init

src/gfxstream/aemu/Stream.cpp

@@ -0,0 +1,219 @@
+/*
+ * Copyright 2019 Google
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "Stream.h"
+
+#include <assert.h>
+#include <string.h>
+
+namespace gfxstream {
+namespace aemu {
+
+void Stream::putByte(uint8_t value) { write(&value, 1U); }
+
+uint8_t Stream::getByte() {
+    uint8_t value[1] = {0};
+    read(value, 1U);
+    return value[0];
+}
+
+void Stream::putBe16(uint16_t value) {
+    uint8_t b[2] = {(uint8_t)(value >> 8), (uint8_t)value};
+    write(b, 2U);
+}
+
+uint16_t Stream::getBe16() {
+    uint8_t b[2] = {0, 0};
+    read(b, 2U);
+    return ((uint16_t)b[0] << 8) | (uint16_t)b[1];
+}
+
+void Stream::putBe32(uint32_t value) {
+    uint8_t b[4] = {(uint8_t)(value >> 24), (uint8_t)(value >> 16), (uint8_t)(value >> 8),
+                    (uint8_t)value};
+    write(b, 4U);
+}
+
+uint32_t Stream::getBe32() {
+    uint8_t b[4] = {0, 0, 0, 0};
+    read(b, 4U);
+    return ((uint32_t)b[0] << 24) | ((uint32_t)b[1] << 16) | ((uint32_t)b[2] << 8) | (uint32_t)b[3];
+}
+
+void Stream::putBe64(uint64_t value) {
+    uint8_t b[8] = {(uint8_t)(value >> 56), (uint8_t)(value >> 48), (uint8_t)(value >> 40),
+                    (uint8_t)(value >> 32), (uint8_t)(value >> 24), (uint8_t)(value >> 16),
+                    (uint8_t)(value >> 8),  (uint8_t)value};
+    write(b, 8U);
+}
+
+uint64_t Stream::getBe64() {
+    uint8_t b[8] = {0, 0, 0, 0, 0, 0, 0, 0};
+    read(b, 8U);
+    return ((uint64_t)b[0] << 56) | ((uint64_t)b[1] << 48) | ((uint64_t)b[2] << 40) |
+           ((uint64_t)b[3] << 32) | ((uint64_t)b[4] << 24) | ((uint64_t)b[5] << 16) |
+           ((uint64_t)b[6] << 8) | (uint64_t)b[7];
+}
+
+void Stream::putFloat(float v) {
+    union {
+        float f;
+        uint8_t bytes[sizeof(float)];
+    } u;
+    u.f
+    = v;
+    this->write(u.bytes, sizeof(u.bytes));
+}
+
+float Stream::getFloat() {
+    union {
+        float f;
+        uint8_t bytes[sizeof(float)];
+    } u;
+    this->read(u.bytes, sizeof(u.bytes));
+    return u.f;
+}
+
+void Stream::putString(const char* str) { putString(str, strlen(str)); }
+
+void Stream::putString(const char* str, size_t len) {
+    this->putBe32(len);
+    this->write(str, len);
+}
+
+void Stream::putString(const std::string& str) { putString(str.c_str()); }
+
+std::string Stream::getString() {
+    std::string result;
+    size_t len = this->getBe32();
+    if (len > 0) {
+        result.resize(len);
+        if (this->read(&result[0], len) != static_cast<ssize_t>(len)) {
+            result.clear();
+        }
+    }
+#ifdef _WIN32
+    else {
+        // std::string in GCC's STL still uses copy on write implementation
+        // with a single shared buffer for an empty string. Its dtor has
+        // a check for that shared buffer, and it deallocates memory only if
+        // the current string's instance address != shared empty string address
+        // Unfortunately, in Windows DLLs each DLL has its own copy of this
+        // empty string (that's just the way Windows DLLs work), so if this
+        // code creates an empty string and passes it over into another module,
+        // that module's std::string::~string() will compare address with its
+        // empty string object, find that they are different and will try to
+        // free() a static object.
+        // To mitigate it we make sure the string allocates something, so it
+        // isn't empty internally and dtor is OK to delete the storage.
+        result.reserve(1);
+    }
+#endif
+    return result;
+}
+
+void Stream::putPackedNum(uint64_t num) {
+    do {
+        auto byte = uint8_t(num & 0x7f);
+        num >>= 7;
+        if (num) {
+            byte |= 0x80;
+        }
+        putByte(byte);
+    } while (num != 0);
+}
+
+uint64_t Stream::getPackedNum() {
+    uint64_t res = 0;
+    uint8_t byte;
+    int i = 0;
+    do {
+        byte = getByte();
+        res |= uint64_t(byte & 0x7f) << (i++ * 7);
+    } while (byte & 0x80 && i < 10);
+    return res;
+}
+
+void Stream::putPackedSignedNum(int64_t num) {
+    if (num >= 0) {
+        assert((uint64_t(num) & (1ULL << 63)) == 0);
+        putPackedNum(uint64_t(num) << 1);
+    } else {
+        assert((uint64_t(-num) & (1ULL << 63)) == 0);
+        putPackedNum((uint64_t(-num) << 1) | 1);
+    }
+}
+
+int64_t Stream::getPackedSignedNum() {
+    auto num = getPackedNum();
+    auto sign = num & 1;
+    return sign ? -int64_t(num >> 1) : (num >> 1);
+}
+
+// Static big-endian conversions
+
+// the |v| pointer is unlikely to be aligned---use memcpy throughout
+
+void Stream::toByte(uint8_t*) {}  // no conversion
+
+void Stream::toBe16(uint8_t* v) {
+    uint16_t value;
+    memcpy(&value, v, sizeof(uint16_t));
+    uint8_t b[2] = {(uint8_t)(value >> 8), (uint8_t)value};
+    memcpy(v, b, sizeof(uint16_t));
+}
+
+void Stream::toBe32(uint8_t* v) {
+    uint32_t value;
+    memcpy(&value, v, sizeof(uint32_t));
+    uint8_t b[4] = {(uint8_t)(value >> 24), (uint8_t)(value >> 16), (uint8_t)(value >> 8),
+                    (uint8_t)value};
+    memcpy(v, b, sizeof(uint32_t));
+}
+
+void Stream::toBe64(uint8_t* v) {
+    uint64_t value;
+    memcpy(&value, v, sizeof(uint64_t));
+    uint8_t b[8] = {(uint8_t)(value >> 56), (uint8_t)(value >> 48), (uint8_t)(value >> 40),
+                    (uint8_t)(value >> 32), (uint8_t)(value >> 24), (uint8_t)(value >> 16),
+                    (uint8_t)(value >> 8),  (uint8_t)value};
+    memcpy(v, b, sizeof(uint64_t));
+}
+
+void Stream::fromByte(uint8_t*) {}  // no conversion
+
+void Stream::fromBe16(uint8_t* v) {
+    uint8_t b[2];
+    memcpy(b, v, sizeof(uint16_t));
+    uint16_t value = ((uint16_t)b[0] << 8) | (uint16_t)b[1];
+    memcpy(v, &value, sizeof(uint16_t));
+}
+
+void Stream::fromBe32(uint8_t* v) {
+    uint8_t b[4];
+    memcpy(b, v, sizeof(uint32_t));
+    uint32_t value =
+        ((uint32_t)b[0] << 24) | ((uint32_t)b[1] << 16) | ((uint32_t)b[2] << 8) | (uint32_t)b[3];
+    memcpy(v, &value, sizeof(uint32_t));
+}
+
+void Stream::fromBe64(uint8_t* v) {
+    uint8_t b[8];
+    memcpy(b, v, sizeof(uint64_t));
+    uint64_t value = ((uint64_t)b[0] << 56) | ((uint64_t)b[1] << 48) | ((uint64_t)b[2] << 40) |
+                     ((uint64_t)b[3] << 32) | ((uint64_t)b[4] << 24) | ((uint64_t)b[5] << 16) |
+                     ((uint64_t)b[6] << 8) | (uint64_t)b[7];
+    memcpy(v, &value, sizeof(uint64_t));
+}
+
+void saveStringArray(Stream* stream, const char* const* strings, uint32_t count) {
+    stream->putBe32(count);
+    for (uint32_t i = 0; i < count; ++i) {
+        stream->putString(strings[i]);
+    }
+}
+
+}  // namespace aemu
+}  // namespace gfxstream

src/gfxstream/aemu/include/AlignedBuf.h

@@ -0,0 +1,150 @@
+/*
+ * Copyright 2018 Google
+ * SPDX-License-Identifier: MIT
+ */
+
+#pragma once
+
+#include <stdio.h>
+
+#include <algorithm>
+#include <atomic>
+#include <cinttypes>
+#include <cstdlib>
+#include <cstring>
+#include <type_traits>
+#include <vector>
+
+#ifdef _WIN32
+#include <malloc.h>
+#endif
+
+namespace gfxstream {
+
+/**
+ * Do not abuse this by using any complicated T. Use it for POD or primitives
+ */
+template <class T, size_t align>
+class AlignedBuf {
+    constexpr static bool triviallyCopyable() {
+#if (defined(__GNUC__) && !defined(__clang__) && __GNUC__ <= 4) || defined(__OLD_STD_VERSION__)
+        // Older g++ doesn't support std::is_trivially_copyable.
+        constexpr bool triviallyCopyable = std::has_trivial_copy_constructor<T>::value;
+#else
+        constexpr bool triviallyCopyable = std::is_trivially_copyable<T>::value;
+#endif
+        return triviallyCopyable;
+    }
+    static_assert(triviallyCopyable() && std::is_standard_layout<T>::value &&
+                  std::is_trivially_default_constructible<T>::value);
+
+   public:
+    explicit AlignedBuf(size_t size) {
+        static_assert(align && ((align & (align - 1)) == 0),
+                      "AlignedBuf only supports power-of-2 aligments.");
+        resizeImpl(size);
+    }
+
+    AlignedBuf(const AlignedBuf& other) : AlignedBuf(other.mSize) {
+        if (other.mBuffer) {  // could have got moved out
+            std::copy(other.mBuffer, other.mBuffer + other.mSize, mBuffer);
+        }
+    }
+
+    AlignedBuf& operator=(const AlignedBuf& other) {
+        if (this != &other) {
+            AlignedBuf tmp(other);
+            *this = std::move(tmp);
+        }
+        return *this;
+    }
+
+    AlignedBuf(AlignedBuf&& other) { *this = std::move(other); }
+
+    AlignedBuf& operator=(AlignedBuf&& other) {
+        mBuffer = other.mBuffer;
+        mSize = other.mSize;
+
+        other.mBuffer = nullptr;
+        other.mSize = 0;
+
+        return *this;
+    }
+
+    ~AlignedBuf() {
+        if (mBuffer) freeImpl(mBuffer);
+    }  // account for getting moved out
+
+    void resize(size_t newSize) { resizeImpl(newSize); }
+
+    size_t size() const { return mSize; }
+
+    T* data() { return mBuffer; }
+
+    T& operator[](size_t index) { return mBuffer[index]; }
+
+    const T& operator[](size_t index) const { return mBuffer[index]; }
+
+    bool operator==(const AlignedBuf& other) const {
+        return 0 == std::memcmp(mBuffer, other.mBuffer, sizeof(T) * std::min(mSize, other.mSize));
+    }
+
+   private:
+    T* getNewBuffer(size_t newSize) {
+        if (newSize == 0) {
+            return nullptr;
+        }
+        size_t pad = std::max(align, sizeof(T));
+        size_t newSizeBytes = ((align - 1 + newSize * sizeof(T) + pad) / align) * align;
+        return static_cast<T*>(reallocImpl(nullptr, newSizeBytes));
+    }
+
+    void resizeImpl(size_t newSize) {
+        T* new_buffer = getNewBuffer(newSize);
+        if (new_buffer && mBuffer) {
+            size_t keepSize = std::min(newSize, mSize);
+            std::copy(mBuffer, mBuffer + keepSize, new_buffer);
+        }
+        if (mBuffer) {
+            freeImpl(mBuffer);
+        }
+        mBuffer = new_buffer;
+        mSize = (new_buffer ? newSize : 0);
+    }
+
+    void* reallocImpl(void* oldPtr, size_t sizeBytes) {
+        if (oldPtr) {
+            freeImpl(oldPtr);
+        }
+        // Platform aligned malloc might not behave right
+        // if we give it an alignment value smaller than sizeof(void*).
+        size_t actualAlign = std::max(align, sizeof(void*));
+#ifdef _WIN32
+        return _aligned_malloc(sizeBytes, actualAlign);
+#else
+        void* res;
+        if (posix_memalign(&res, actualAlign, sizeBytes)) {
+            fprintf(stderr, "%s: failed to alloc aligned memory\n", __func__);
+            abort();
+        }
+        return res;
+#endif
+    }
+
+    void freeImpl(void* ptr) {
+#ifdef _WIN32
+        _aligned_free(ptr);
+#else
+        free(ptr);
+#endif
+    }
+
+    T* mBuffer = nullptr;
+    size_t mSize = 0;
+};
+
+// Convenience function for aligned malloc across platforms
+void* aligned_buf_alloc(size_t align, size_t size);
+void aligned_buf_free(void* buf);
+
+}  // namespace gfxstream

src/gfxstream/aemu/include/Allocator.h

@@ -0,0 +1,59 @@
+/*
+ * Copyright 2021 Google
+ * SPDX-License-Identifier: MIT
+ */
+#pragma once
+
+#include <inttypes.h>
+#include <stddef.h>
+#include <string.h>
+
+namespace gfxstream {
+namespace aemu {
+
+// A generic memory allocator interface which could be used to allocate
+// a certain size of memory region, or memory region for arrays / strings.
+// How the memory are recycled / freed is up to derived classes.
+class Allocator {
+   public:
+    Allocator() = default;
+    virtual ~Allocator() = default;
+
+    virtual void* alloc(size_t wantedSize) = 0;
+
+    // Convenience function to allocate an array
+    // of objects of type T.
+    template <class T>
+    T* allocArray(size_t count) {
+        size_t bytes = sizeof(T) * count;
+        void* res = alloc(bytes);
+        return (T*)res;
+    }
+
+    char* strDup(const char* toCopy) {
+        size_t bytes = strlen(toCopy) + 1;
+        void* res = alloc(bytes);
+        memset(res, 0x0, bytes);
+        memcpy(res, toCopy, bytes);
+        return (char*)res;
+    }
+
+    char** strDupArray(const char* const* arrayToCopy, size_t count) {
+        char** res = allocArray<char*>(count);
+
+        for (size_t i = 0; i < count; i++) {
+            res[i] = strDup(arrayToCopy[i]);
+        }
+
+        return res;
+    }
+
+    void* dupArray(const void* buf, size_t bytes) {
+        void* res = alloc(bytes);
+        memcpy(res, buf, bytes);
+        return res;
+    }
+};
+
+}  // namespace aemu
+}  // namespace gfxstream

src/gfxstream/aemu/include/BumpPool.h

@@ -0,0 +1,69 @@
+/*
+ * Copyright 2019 Google
+ * SPDX-License-Identifier: MIT
+ */
+#pragma once
+
+#include <inttypes.h>
+
+#include <unordered_set>
+#include <vector>
+
+#include "AlignedBuf.h"
+#include "Allocator.h"
+
+namespace gfxstream {
+namespace aemu {
+
+// Class to make it easier to set up memory regions where it is fast
+// to allocate buffers AND we don't care about freeing individual pieces,
+// BUT it's necessary to preserve previous pointer values in between the first
+// alloc() after a freeAll(), and the freeAll() itself, allowing some sloppy use of
+// malloc in the first pass while we find out how much data was needed.
+class BumpPool : public Allocator {
+   public:
+    BumpPool(size_t startingBytes = 4096) : mStorage(startingBytes / sizeof(uint64_t)) {}
+    // All memory allocated by this pool
+    // is automatically deleted when the pool
+    // is deconstructed.
+    ~BumpPool() { freeAll(); }
+
+    void* alloc(size_t wantedSize) override {
+        size_t wantedSizeRoundedUp =
+            sizeof(uint64_t) * ((wantedSize + sizeof(uint64_t) - 1) / (sizeof(uint64_t)));
+
+        mTotalWantedThisGeneration += wantedSizeRoundedUp;
+        if (mAllocPos + wantedSizeRoundedUp > mStorage.size() * sizeof(uint64_t)) {
+            mNeedRealloc = true;
+            void* fallbackPtr = malloc(wantedSizeRoundedUp);
+            mFallbackPtrs.insert(fallbackPtr);
+            return fallbackPtr;
+        }
+        void* allocPtr = (void*)(((unsigned char*)mStorage.data()) + mAllocPos);
+        mAllocPos += wantedSizeRoundedUp;
+        return allocPtr;
+    }
+
+    void freeAll() {
+        mAllocPos = 0;
+        if (mNeedRealloc) {
+            mStorage.resize((mTotalWantedThisGeneration * 2) / sizeof(uint64_t));
+            mNeedRealloc = false;
+            for (auto ptr : mFallbackPtrs) {
+                free(ptr);
+            }
+            mFallbackPtrs.clear();
+        }
+        mTotalWantedThisGeneration = 0;
+    }
+
+   private:
+    AlignedBuf<uint64_t, 8> mStorage;
+    std::unordered_set<void*> mFallbackPtrs;
+    size_t mAllocPos = 0;
+    size_t mTotalWantedThisGeneration = 0;
+    bool mNeedRealloc = false;
+};
+
+}  // namespace aemu
+}  // namespace gfxstream

src/gfxstream/aemu/include/Stream.h

@@ -0,0 +1,113 @@
+/*
+ * Copyright 2019 Google
+ * SPDX-License-Identifier: MIT
+ */
+
+#pragma once
+
+#include <inttypes.h>
+#include <sys/types.h>
+
+#include <string>
+
+namespace gfxstream {
+namespace aemu {
+
+// Abstract interface to byte streams of all kind.
+// This is mainly used to implement disk serialization.
+class Stream {
+   public:
+    // Default constructor.
+    Stream() = default;
+
+    // Destructor.
+    virtual ~Stream() = default;
+
+    // Read up to |size| bytes and copy them to |buffer|. Return the number
+    // of bytes that were actually transferred, or -errno value on error.
+    virtual ssize_t read(void* buffer, size_t size) = 0;
+
+    // Write up to |size| bytes from |buffer| into the stream. Return the
+    // number of bytes that were actually transferred, or -errno value on
+    // error.
+    virtual ssize_t write(const void* buffer, size_t size) = 0;
+
+    virtual void* getProtobuf() { return nullptr; }
+
+    // Write a single byte |value| into the stream. Ignore errors.
+    void putByte(uint8_t value);
+
+    // Write a 16-bit |value| as big-endian into the stream. Ignore errors.
+    void putBe16(uint16_t value);
+
+    // Write a 32-bit |value| as big-endian into the stream. Ignore errors.
+    void putBe32(uint32_t value);
+
+    // Write a 64-bit |value| as big-endian into the stream. Ignore errors.
+    void putBe64(uint64_t value);
+
+    // Read a single byte from the stream. Return 0 on error.
+    uint8_t getByte();
+
+    // Read a single big-endian 16-bit value from the stream.
+    // Return 0 on error.
+    uint16_t getBe16();
+
+    // Read a single big-endian 32-bit value from the stream.
+    // Return 0 on error.
+    uint32_t getBe32();
+
+    // Read a single big-endian 64-bit value from the stream.
+    // Return 0 on error.
+    uint64_t getBe64();
+
+    // Write a 32-bit float |value| to the stream.
+    void putFloat(float value);
+
+    // Read a single 32-bit float value from the stream.
+    float getFloat();
+
+    // Write a 0-terminated C string |str| into the stream. Ignore error.
+    void putString(const char* str);
+    void putString(const std::string& str);
+
+    // Write a string |str| of |strlen| bytes into the stream.
+    // Ignore errors.
+    void putString(const char* str, size_t strlen);
+
+    // Read a string from the stream. Return a new string instance,
+    // which will be empty on error. Note that this can only be used
+    // to read strings that were written with putString().
+    std::string getString();
+
+    // Put/gen an integer number into the stream, making it use as little space
+    // there as possible.
+    // It uses a simple byte-by-byte encoding scheme, putting 7 bits of the
+    // number with the 8th bit set when there's more data to read, until the
+    // whole number is read.
+    // The compression is efficient if the number range is small, but it starts
+    // wasting space when values approach 14 bits for int16 (16K), 28 bits for
+    // int32 (268M) or 56 bits for int64 (still a lot).
+    void putPackedNum(uint64_t num);
+    uint64_t getPackedNum();
+
+    // Same thing, but encode negative numbers efficiently as well (single sign
+    // bit + packed unsigned representation)
+    void putPackedSignedNum(int64_t num);
+    int64_t getPackedSignedNum();
+
+    // Static big-endian conversions
+    static void toByte(uint8_t*);
+    static void toBe16(uint8_t*);
+    static void toBe32(uint8_t*);
+    static void toBe64(uint8_t*);
+    static void fromByte(uint8_t*);
+    static void fromBe16(uint8_t*);
+    static void fromBe32(uint8_t*);
+    static void fromBe64(uint8_t*);
+};
+
+void saveStringArray(Stream* stream, const char* const* strings, uint32_t count);
+
+}  // namespace aemu
+}  // namespace gfxstream

src/gfxstream/aemu/include/ring_buffer.h

@@ -0,0 +1,179 @@
+/*
+ * Copyright 2018 Google
+ * SPDX-License-Identifier: MIT
+ */
+#pragma once
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdbool.h>
+#include <stdint.h>
+
+#define RING_BUFFER_SHIFT 11
+#define RING_BUFFER_SIZE (1 << RING_BUFFER_SHIFT)
+#define NUM_CONFIG_FIELDS 32
+
+// Single producer/consumer ring buffer struct that can be shared
+// between host and guest as-is.
+struct ring_buffer {
+    uint32_t host_version;
+    uint32_t guest_version;
+    uint32_t write_pos;    // Atomically updated for the consumer
+    uint32_t unused0[13];  // Separate cache line
+    uint32_t read_pos;     // Atomically updated for the producer
+    uint32_t read_live_count;
+    uint32_t read_yield_count;
+    uint32_t read_sleep_us_count;
+    uint32_t unused1[12];  // Separate cache line
+    uint8_t buf[RING_BUFFER_SIZE];
+    uint32_t state;  // An atomically updated variable from both
+                     // producer and consumer for other forms of
+                     // coordination.
+    uint32_t config[NUM_CONFIG_FIELDS];
+};
+
+void ring_buffer_init(struct ring_buffer* r);
+
+// Writes or reads step_size at a time. Sets errno=EAGAIN if full or empty.
+// Returns the number of step_size steps read.
+long ring_buffer_write(struct ring_buffer* r, const void* data, uint32_t step_size, uint32_t steps);
+long ring_buffer_read(struct ring_buffer* r, void* data, uint32_t step_size, uint32_t steps);
+// Like ring_buffer_write / ring_buffer_read, but merely advances the counters
+// without reading or writing anything. Returns the number of step_size steps
+// advanced.
+long ring_buffer_advance_write(struct ring_buffer* r, uint32_t step_size, uint32_t steps);
+long ring_buffer_advance_read(struct ring_buffer* r, uint32_t step_size, uint32_t steps);
+
+// If we want to work with dynamically allocated buffers, a separate struct is
+// needed; the host and guest are in different address spaces and thus have
+// different views of the same memory, with the host and guest having different
+// copies of this struct.
+struct ring_buffer_view {
+    uint8_t* buf;
+    uint32_t size;
+    uint32_t mask;
+};
+
+// Convenience struct that holds a pointer to a ring along with a view.  It's a
+// common pattern for the ring and the buffer of the view to be shared between
+// two entities (in this case, usually guest and host).
+struct ring_buffer_with_view {
+    struct ring_buffer* ring;
+    struct ring_buffer_view view;
+};
+
+// Calculates the highest power of 2 so that
+// (1 << shift) <= size.
+uint32_t ring_buffer_calc_shift(uint32_t size);
+
+// Initializes ring buffer with view using |buf|. If |size| is not a power of
+// two, then the buffer will assume a size equal to the greater power of two
+// less than |size|.
+void ring_buffer_view_init(struct ring_buffer* r, struct ring_buffer_view* v, uint8_t* buf,
+                           uint32_t size);
+
+void ring_buffer_init_view_only(struct ring_buffer_view* v, uint8_t* buf, uint32_t size);
+
+// Read/write functions with the view.
+long ring_buffer_view_write(struct ring_buffer* r, struct ring_buffer_view* v, const void* data,
+                            uint32_t step_size, uint32_t steps);
+long ring_buffer_view_read(struct ring_buffer* r, struct ring_buffer_view* v, void* data,
+                           uint32_t step_size, uint32_t steps);
+
+// Usage of ring_buffer as a waitable object.
+// These functions will back off if spinning too long.
+//
+// if |v| is null, it is assumed that the statically allocated ring buffer is
+// used.
+//
+// Returns true if ring buffer became available, false if timed out.
+bool ring_buffer_wait_write(const struct ring_buffer* r, const struct ring_buffer_view* v,
+                            uint32_t bytes, uint64_t timeout_us);
+bool ring_buffer_wait_read(const struct ring_buffer* r, const struct ring_buffer_view* v,
+                           uint32_t bytes, uint64_t timeout_us);
+
+// read/write fully, blocking if there is nothing to read/write.
+void ring_buffer_write_fully(struct ring_buffer* r, struct ring_buffer_view* v, const void* data,
+                             uint32_t bytes);
+void ring_buffer_read_fully(struct ring_buffer* r, struct ring_buffer_view* v, void* data,
+                            uint32_t bytes);
+
+// Like read/write fully, but with an abort value. The value is read from
+// |abortPtr| each time. If |abortPtr| is null, then behaves the same
+// as ring_buffer_(read|write)_fully.
+// Returns the actual number of bytes sent or received.
+uint32_t ring_buffer_write_fully_with_abort(struct ring_buffer* r, struct ring_buffer_view* v,
+                                            const void* data, uint32_t bytes, uint32_t abort_value,
+                                            const volatile uint32_t* abort_ptr);
+uint32_t ring_buffer_read_fully_with_abort(struct ring_buffer* r, struct ring_buffer_view* v,
+                                           void* data, uint32_t bytes, uint32_t abort_value,
+                                           const volatile uint32_t* abort_ptr);
+
+uint32_t ring_buffer_view_get_ring_pos(const struct ring_buffer_view* v, uint32_t index);
+
+bool ring_buffer_can_write(const struct ring_buffer* r, uint32_t bytes);
+bool ring_buffer_can_read(const struct ring_buffer* r, uint32_t bytes);
+bool ring_buffer_view_can_write(const struct ring_buffer* r, const struct ring_buffer_view* v,
+                                uint32_t bytes);
+bool ring_buffer_view_can_read(const struct ring_buffer* r, const struct ring_buffer_view* v,
+                               uint32_t bytes);
+uint32_t ring_buffer_available_read(const struct ring_buffer* r, const struct ring_buffer_view* v);
+uint32_t ring_buffer_available_write(const struct ring_buffer* r, const struct ring_buffer_view* v);
+// Copies out contents from the consumer side of
+// ring buffer/view |r,v|.
+// If there is less available read than |wanted_bytes|,
+// returns -1.
+// On success, returns 0.
+int ring_buffer_copy_contents(const struct ring_buffer* r, const struct ring_buffer_view* v,
+                              uint32_t wanted_bytes, uint8_t* res);
+
+// Lockless synchronization where the consumer is allowed to hang up and go to
+// sleep. This can be considered a sort of asymmetric lock for two threads,
+// where the consumer can be more sleepy. It captures the pattern we usually use
+// for emulator devices; the guest asks the host for something, and some host
+// thread services the request and goes back to sleep.
+enum ring_buffer_sync_state {
+    RING_BUFFER_SYNC_PRODUCER_IDLE = 0,
+    RING_BUFFER_SYNC_PRODUCER_ACTIVE = 1,
+    RING_BUFFER_SYNC_CONSUMER_HANGING_UP = 2,
+    RING_BUFFER_SYNC_CONSUMER_HUNG_UP = 3,
+};
+
+// Sync state is RING_BUFFER_SYNC_PRODUCER_IDLE.
+void ring_buffer_sync_init(struct ring_buffer* r);
+
+// Tries to acquire the channel for sending.
+// Returns false if the consumer was in the middle of hanging up,
+// true if the producer successfully acquired the channel
+// (put it in the RING_BUFFER_SYNC_PRODUCER_ACTIVE state).
+bool ring_buffer_producer_acquire(struct ring_buffer* r);
+// Same as above, but acquires from RING_BUFFER_SYNC_CONSUMER_HUNG_UP.
+bool ring_buffer_producer_acquire_from_hangup(struct ring_buffer* r);
+// Waits until the consumer hangs up.
+void ring_buffer_producer_wait_hangup(struct ring_buffer* r);
+// Sets the state back to RING_BUFFER_SYNC_PRODUCER_IDLE.
+void ring_buffer_producer_idle(struct ring_buffer* r);
+
+// There is no symmetric consumer acquire because the consumer can consume with
+// the ring buffer being in any state (albeit with long waiting if the producer
+// does not send anything)
+
+// Tries to acquire the channel on the consumer side for
+// hanging up. Returns false if the producer is in the middle of sending,
+// true if the consumer successfully hung up the channel
+// (put it in the RING_BUFFER_SYNC_CONSUMER_HUNG_UP state).
+bool ring_buffer_consumer_hangup(struct ring_buffer* r);
+// Waits until the producer has set the state to
+// RING_BUFFER_SYNC_PRODUCER_IDLE.
+void ring_buffer_consumer_wait_producer_idle(struct ring_buffer* r);
+// Sets the state to hung up.
+void ring_buffer_consumer_hung_up(struct ring_buffer* r);
+
+// Convenient function to reschedule thread
+void ring_buffer_yield();
+
+#ifdef __cplusplus
+}
+#endif

src/gfxstream/aemu/meson.build

@@ -0,0 +1,8 @@
+# Copyright 2024 Google LLC
+# SPDX-License-Identifier: MIT
+
+inc_aemu = include_directories('include')
+
+files_libaemu = files('Stream.cpp', 'ring_buffer.cpp')
+
+libaemu = static_library('aemu', files_libaemu, include_directories: [inc_aemu])

src/gfxstream/aemu/ring_buffer.cpp

@@ -0,0 +1,545 @@
+/*
+ * Copyright 2018 Google
+ * SPDX-License-Identifier: MIT
+ */
+#include "ring_buffer.h"
+
+#include <errno.h>
+#include <string.h>
+#ifdef _MSC_VER
+#include "aemu/base/msvc.h"
+#else
+#include <sys/time.h>
+#endif
+
+#ifdef __x86_64__
+#include <emmintrin.h>
+#endif
+
+#ifdef _WIN32
+#include <windows.h>
+#else
+#include <sched.h>
+#include <unistd.h>
+#endif
+
+#define RING_BUFFER_MASK (RING_BUFFER_SIZE - 1)
+
+#define RING_BUFFER_VERSION 1
+
+void ring_buffer_init(struct ring_buffer* r) {
+    r->host_version = 1;
+    r->write_pos = 0;
+    r->read_pos = 0;
+
+    r->read_live_count = 0;
+    r->read_yield_count = 0;
+    r->read_sleep_us_count = 0;
+
+    r->state = 0;
+}
+
+static uint32_t get_ring_pos(uint32_t index) { return index & RING_BUFFER_MASK; }
+
+bool ring_buffer_can_write(const struct ring_buffer* r, uint32_t bytes) {
+    uint32_t read_view;
+    __atomic_load(&r->read_pos, &read_view, __ATOMIC_SEQ_CST);
+    return get_ring_pos(read_view - r->write_pos - 1) >= bytes;
+}
+
+bool ring_buffer_can_read(const struct ring_buffer* r, uint32_t bytes) {
+    uint32_t write_view;
+    __atomic_load(&r->write_pos, &write_view, __ATOMIC_SEQ_CST);
+    return get_ring_pos(write_view - r->read_pos) >= bytes;
+}
+
+long ring_buffer_write(struct ring_buffer* r, const void* data, uint32_t step_size,
+                       uint32_t steps) {
+    const uint8_t* data_bytes = (const uint8_t*)data;
+    uint32_t i;
+
+    for (i = 0; i < steps; ++i) {
+        if (!ring_buffer_can_write(r, step_size)) {
+            errno = -EAGAIN;
+            return (long)i;
+        }
+
+        // Needs to be split up into 2 writes for the edge case.
+        uint32_t available_at_end = RING_BUFFER_SIZE - get_ring_pos(r->write_pos);
+
+        if (step_size > available_at_end) {
+            uint32_t remaining = step_size - available_at_end;
+            memcpy(&r->buf[get_ring_pos(r->write_pos)], data_bytes + i * step_size,
+                   available_at_end);
+            memcpy(&r->buf[get_ring_pos(r->write_pos + available_at_end)],
+                   data_bytes + i * step_size + available_at_end, remaining);
+        } else {
+            memcpy(&r->buf[get_ring_pos(r->write_pos)], data_bytes + i * step_size, step_size);
+        }
+
+        __atomic_add_fetch(&r->write_pos, step_size, __ATOMIC_SEQ_CST);
+    }
+
+    errno = 0;
+    return (long)steps;
+}
+
+long ring_buffer_read(struct ring_buffer* r, void* data, uint32_t step_size, uint32_t steps) {
+    uint8_t* data_bytes = (uint8_t*)data;
+    uint32_t i;
+
+    for (i = 0; i < steps; ++i) {
+        if (!ring_buffer_can_read(r, step_size)) {
+            errno = -EAGAIN;
+            return (long)i;
+        }
+
+        // Needs to be split up into 2 reads for the edge case.
+        uint32_t available_at_end = RING_BUFFER_SIZE - get_ring_pos(r->read_pos);
+
+        if (step_size > available_at_end) {
+            uint32_t remaining = step_size - available_at_end;
+            memcpy(data_bytes + i * step_size, &r->buf[get_ring_pos(r->read_pos)],
+                   available_at_end);
+            memcpy(data_bytes + i * step_size + available_at_end,
+                   &r->buf[get_ring_pos(r->read_pos + available_at_end)], remaining);
+        } else {
+            memcpy(data_bytes + i * step_size, &r->buf[get_ring_pos(r->read_pos)], step_size);
+        }
+
+        __atomic_add_fetch(&r->read_pos, step_size, __ATOMIC_SEQ_CST);
+    }
+
+    errno = 0;
+    return (long)steps;
+}
+
+long ring_buffer_advance_write(struct ring_buffer* r, uint32_t step_size, uint32_t steps) {
+    uint32_t i;
+
+    for (i = 0; i < steps; ++i) {
+        if (!ring_buffer_can_write(r, step_size)) {
+            errno = -EAGAIN;
+            return (long)i;
+        }
+
+        __atomic_add_fetch(&r->write_pos, step_size, __ATOMIC_SEQ_CST);
+    }
+
+    errno = 0;
+    return (long)steps;
+}
+
+long ring_buffer_advance_read(struct ring_buffer* r, uint32_t step_size, uint32_t steps) {
+    uint32_t i;
+
+    for (i = 0; i < steps; ++i) {
+        if (!ring_buffer_can_read(r, step_size)) {
+            errno = -EAGAIN;
+            return (long)i;
+        }
+
+        __atomic_add_fetch(&r->read_pos, step_size, __ATOMIC_SEQ_CST);
+    }
+
+    errno = 0;
+    return (long)steps;
+}
+
+uint32_t ring_buffer_calc_shift(uint32_t size) {
+    uint32_t shift = 0;
+    while ((1u << shift) < size) {
+        ++shift;
+    }
+
+    // if size is not a power of 2,
+    if ((1u << shift) > size) {
+        --shift;
+    }
+    return shift;
+}
+
+void ring_buffer_view_init(struct ring_buffer* r, struct ring_buffer_view* v, uint8_t* buf,
+                           uint32_t size) {
+    uint32_t shift = ring_buffer_calc_shift(size);
+
+    ring_buffer_init(r);
+
+    v->buf = buf;
+    v->size = (1u << shift);
+    v->mask = (1u << shift) - 1;
+}
+
+void ring_buffer_init_view_only(struct ring_buffer_view* v, uint8_t* buf, uint32_t size) {
+    uint32_t shift = ring_buffer_calc_shift(size);
+
+    v->buf = buf;
+    v->size = (1u << shift);
+    v->mask = (1u << shift) - 1;
+}
+
+uint32_t ring_buffer_view_get_ring_pos(const struct ring_buffer_view* v, uint32_t index) {
+    return index & v->mask;
+}
+
+bool ring_buffer_view_can_write(const struct ring_buffer* r, const struct ring_buffer_view* v,
+                                uint32_t bytes) {
+    uint32_t read_view;
+    __atomic_load(&r->read_pos, &read_view, __ATOMIC_SEQ_CST);
+    return ring_buffer_view_get_ring_pos(v, read_view - r->write_pos - 1) >= bytes;
+}
+
+bool ring_buffer_view_can_read(const struct ring_buffer* r, const struct ring_buffer_view* v,
+                               uint32_t bytes) {
+    uint32_t write_view;
+    __atomic_load(&r->write_pos, &write_view, __ATOMIC_SEQ_CST);
+    return ring_buffer_view_get_ring_pos(v, write_view - r->read_pos) >= bytes;
+}
+
+uint32_t ring_buffer_available_read(const struct ring_buffer* r, const struct ring_buffer_view* v) {
+    uint32_t write_view;
+    __atomic_load(&r->write_pos, &write_view, __ATOMIC_SEQ_CST);
+    if (v) {
+        return ring_buffer_view_get_ring_pos(v, write_view - r->read_pos);
+    } else {
+        return get_ring_pos(write_view - r->read_pos);
+    }
+}
+
+uint32_t ring_buffer_available_write(const struct ring_buffer* r,
+                                     const struct ring_buffer_view* v) {
+    uint32_t read_view;
+    __atomic_load(&r->read_pos, &read_view, __ATOMIC_SEQ_CST);
+    if (v) {
+        return ring_buffer_view_get_ring_pos(v, read_view - r->write_pos - 1);
+    } else {
+        return get_ring_pos(read_view - r->write_pos - 1);
+    }
+}
+
+int ring_buffer_copy_contents(const struct ring_buffer* r, const struct ring_buffer_view* v,
+                              uint32_t wanted_bytes, uint8_t* res) {
+    uint32_t total_available = ring_buffer_available_read(r, v);
+    uint32_t available_at_end = 0;
+
+    if (v) {
+        available_at_end = v->size - ring_buffer_view_get_ring_pos(v, r->read_pos);
+    } else {
+        available_at_end = RING_BUFFER_SIZE - get_ring_pos(r->write_pos);
+    }
+
+    if (total_available < wanted_bytes) {
+        return -1;
+    }
+
+    if (v) {
+        if (wanted_bytes > available_at_end) {
+            uint32_t remaining = wanted_bytes - available_at_end;
+            memcpy(res, &v->buf[ring_buffer_view_get_ring_pos(v, r->read_pos)], available_at_end);
+            memcpy(res + available_at_end,
+                   &v->buf[ring_buffer_view_get_ring_pos(v, r->read_pos + available_at_end)],
+                   remaining);
+        } else {
+            memcpy(res, &v->buf[ring_buffer_view_get_ring_pos(v, r->read_pos)], wanted_bytes);
+        }
+    } else {
+        if (wanted_bytes > available_at_end) {
+            uint32_t remaining = wanted_bytes - available_at_end;
+            memcpy(res, &r->buf[get_ring_pos(r->read_pos)], available_at_end);
+            memcpy(res + available_at_end, &r->buf[get_ring_pos(r->read_pos + available_at_end)],
+                   remaining);
+        } else {
+            memcpy(res, &r->buf[get_ring_pos(r->read_pos)], wanted_bytes);
+        }
+    }
+    return 0;
+}
+
+long ring_buffer_view_write(struct ring_buffer* r, struct ring_buffer_view* v, const void* data,
+                            uint32_t step_size, uint32_t steps) {
+    uint8_t* data_bytes = (uint8_t*)data;
+    uint32_t i;
+
+    for (i = 0; i < steps; ++i) {
+        if (!ring_buffer_view_can_write(r, v, step_size)) {
+            errno = -EAGAIN;
+            return (long)i;
+        }
+
+        // Needs to be split up into 2 writes for the edge case.
+        uint32_t available_at_end = v->size - ring_buffer_view_get_ring_pos(v, r->write_pos);
+
+        if (step_size > available_at_end) {
+            uint32_t remaining = step_size - available_at_end;
+            memcpy(&v->buf[ring_buffer_view_get_ring_pos(v, r->write_pos)],
+                   data_bytes + i * step_size, available_at_end);
+            memcpy(&v->buf[ring_buffer_view_get_ring_pos(v, r->write_pos + available_at_end)],
+                   data_bytes + i * step_size + available_at_end, remaining);
+        } else {
+            memcpy(&v->buf[ring_buffer_view_get_ring_pos(v, r->write_pos)],
+                   data_bytes + i * step_size, step_size);
+        }
+
+        __atomic_add_fetch(&r->write_pos, step_size, __ATOMIC_SEQ_CST);
+    }
+
+    errno = 0;
+    return (long)steps;
+}
+
+long ring_buffer_view_read(struct ring_buffer* r, struct ring_buffer_view* v, void* data,
+                           uint32_t step_size, uint32_t steps) {
+    uint8_t* data_bytes = (uint8_t*)data;
+    uint32_t i;
+
+    for (i = 0; i < steps; ++i) {
+        if (!ring_buffer_view_can_read(r, v, step_size)) {
+            errno = -EAGAIN;
+            return (long)i;
+        }
+
+        // Needs to be split up into 2 reads for the edge case.
+        uint32_t available_at_end = v->size - ring_buffer_view_get_ring_pos(v, r->read_pos);
+
+        if (step_size > available_at_end) {
+            uint32_t remaining = step_size - available_at_end;
+            memcpy(data_bytes + i * step_size,
+                   &v->buf[ring_buffer_view_get_ring_pos(v, r->read_pos)], available_at_end);
+            memcpy(data_bytes + i * step_size + available_at_end,
+                   &v->buf[ring_buffer_view_get_ring_pos(v, r->read_pos + available_at_end)],
+                   remaining);
+        } else {
+            memcpy(data_bytes + i * step_size,
+                   &v->buf[ring_buffer_view_get_ring_pos(v, r->read_pos)], step_size);
+        }
+        __atomic_add_fetch(&r->read_pos, step_size, __ATOMIC_SEQ_CST);
+    }
+
+    errno = 0;
+    return (long)steps;
+}
+
+void ring_buffer_yield() {
+#ifdef _WIN32
+    _mm_pause();
+#else
+    sched_yield();
+#endif
+}
+
+static void ring_buffer_sleep() {
+#ifdef _WIN32
+    Sleep(2);
+#else
+    usleep(2000);
+#endif
+}
+
+static uint64_t ring_buffer_curr_us() {
+    uint64_t res;
+    struct timeval tv;
+    gettimeofday(&tv, NULL);
+    res = tv.tv_sec * 1000000ULL + tv.tv_usec;
+    return res;
+}
+
+static const uint32_t yield_backoff_us = 1000;
+static const uint32_t sleep_backoff_us = 2000;
+
+bool ring_buffer_wait_write(const struct ring_buffer* r, const struct ring_buffer_view* v,
+                            uint32_t bytes, uint64_t timeout_us) {
+    uint64_t start_us = ring_buffer_curr_us();
+    uint64_t curr_wait_us;
+
+    bool can_write = v ? ring_buffer_view_can_write(r, v, bytes) : ring_buffer_can_write(r, bytes);
+
+    while (!can_write) {
+#ifdef __x86_64
+        _mm_pause();
+#endif
+        curr_wait_us = ring_buffer_curr_us() - start_us;
+
+        if (curr_wait_us > yield_backoff_us) {
+            ring_buffer_yield();
+        }
+
+        if (curr_wait_us > sleep_backoff_us) {
+            ring_buffer_sleep();
+        }
+
+        if (curr_wait_us > timeout_us) {
+            return false;
+        }
+
+        can_write = v ? ring_buffer_view_can_write(r, v, bytes) : ring_buffer_can_write(r, bytes);
+    }
+
+    return true;
+}
+
+bool ring_buffer_wait_read(const struct ring_buffer* r, const struct ring_buffer_view* v,
+                           uint32_t bytes, uint64_t timeout_us) {
+    uint64_t start_us = ring_buffer_curr_us();
+    uint64_t curr_wait_us;
+
+    bool can_read = v ? ring_buffer_view_can_read(r, v, bytes) : ring_buffer_can_read(r, bytes);
+
+    while (!can_read) {
+        // TODO(bohu): find aarch64 equivalent
+#ifdef __x86_64
+        _mm_pause();
+#endif
+        curr_wait_us = ring_buffer_curr_us() - start_us;
+
+        if (curr_wait_us > yield_backoff_us) {
+            ring_buffer_yield();
+            ((struct ring_buffer*)r)->read_yield_count++;
+        }
+
+        if (curr_wait_us > sleep_backoff_us) {
+            ring_buffer_sleep();
+            ((struct ring_buffer*)r)->read_sleep_us_count += 2000;
+        }
+
+        if (curr_wait_us > timeout_us) {
+            return false;
+        }
+
+        can_read = v ? ring_buffer_view_can_read(r, v, bytes) : ring_buffer_can_read(r, bytes);
+    }
+
+    ((struct ring_buffer*)r)->read_live_count++;
+    return true;
+}
+
+static uint32_t get_step_size(struct ring_buffer* r, struct ring_buffer_view* v, uint32_t bytes) {
+    uint32_t available = v ? (v->size >> 1) : (RING_BUFFER_SIZE >> 1);
+    uint32_t res = available < bytes ? available : bytes;
+
+    return res;
+}
+
+void ring_buffer_write_fully(struct ring_buffer* r, struct ring_buffer_view* v, const void* data,
+                             uint32_t bytes) {
+    ring_buffer_write_fully_with_abort(r, v, data, bytes, 0, 0);
+}
+
+void ring_buffer_read_fully(struct ring_buffer* r, struct ring_buffer_view* v, void* data,
+                            uint32_t bytes) {
+    ring_buffer_read_fully_with_abort(r, v, data, bytes, 0, 0);
+}
+
+uint32_t ring_buffer_write_fully_with_abort(struct ring_buffer* r, struct ring_buffer_view* v,
+                                            const void* data, uint32_t bytes, uint32_t abort_value,
+                                            const volatile uint32_t* abort_ptr) {
+    uint32_t candidate_step = get_step_size(r, v, bytes);
+    uint32_t processed = 0;
+
+    uint8_t* dst = (uint8_t*)data;
+
+    while (processed < bytes) {
+        if (bytes - processed < candidate_step) {
+            candidate_step = bytes - processed;
+        }
+
+        long processed_here = 0;
+        ring_buffer_wait_write(r, v, candidate_step, (uint64_t)(-1));
+
+        if (v) {
+            processed_here = ring_buffer_view_write(r, v, dst + processed, candidate_step, 1);
+        } else {
+            processed_here = ring_buffer_write(r, dst + processed, candidate_step, 1);
+        }
+
+        processed += processed_here ? candidate_step : 0;
+
+        if (abort_ptr && (abort_value == *abort_ptr)) {
+            return processed;
+        }
+    }
+
+    return processed;
+}
+
+uint32_t ring_buffer_read_fully_with_abort(struct ring_buffer* r, struct ring_buffer_view* v,
+                                           void* data, uint32_t bytes, uint32_t abort_value,
+                                           const volatile uint32_t* abort_ptr) {
+    uint32_t candidate_step = get_step_size(r, v, bytes);
+    uint32_t processed = 0;
+
+    uint8_t* dst = (uint8_t*)data;
+
+    while (processed < bytes) {
+#ifdef __x86_64
+        _mm_pause();
+#endif
+        if (bytes - processed < candidate_step) {
+            candidate_step = bytes - processed;
+        }
+
+        long processed_here = 0;
+        ring_buffer_wait_read(r, v, candidate_step, (uint64_t)(-1));
+
+        if (v) {
+            processed_here = ring_buffer_view_read(r, v, dst + processed, candidate_step, 1);
+        } else {
+            processed_here = ring_buffer_read(r, dst + processed, candidate_step, 1);
+        }
+
+        processed += processed_here ? candidate_step : 0;
+
+        if (abort_ptr && (abort_value == *abort_ptr)) {
+            return processed;
+        }
+    }
+
+    return processed;
+}
+
+void ring_buffer_sync_init(struct ring_buffer* r) {
+    __atomic_store_n(&r->state, RING_BUFFER_SYNC_PRODUCER_IDLE, __ATOMIC_SEQ_CST);
+}
+
+bool ring_buffer_producer_acquire(struct ring_buffer* r) {
+    uint32_t expected_idle = RING_BUFFER_SYNC_PRODUCER_IDLE;
+    bool success =
+        __atomic_compare_exchange_n(&r->state, &expected_idle, RING_BUFFER_SYNC_PRODUCER_ACTIVE,
+                                    false /* strong */, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
+    return success;
+}
+
+bool ring_buffer_producer_acquire_from_hangup(struct ring_buffer* r) {
+    uint32_t expected_hangup = RING_BUFFER_SYNC_CONSUMER_HUNG_UP;
+    bool success =
+        __atomic_compare_exchange_n(&r->state, &expected_hangup, RING_BUFFER_SYNC_PRODUCER_ACTIVE,
+                                    false /* strong */, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
+    return success;
+}
+
+void ring_buffer_producer_wait_hangup(struct ring_buffer* r) {
+    while (__atomic_load_n(&r->state, __ATOMIC_SEQ_CST) != RING_BUFFER_SYNC_CONSUMER_HUNG_UP) {
+        ring_buffer_yield();
+    }
+}
+
+void ring_buffer_producer_idle(struct ring_buffer* r) {
+    __atomic_store_n(&r->state, RING_BUFFER_SYNC_PRODUCER_IDLE, __ATOMIC_SEQ_CST);
+}
+
+bool ring_buffer_consumer_hangup(struct ring_buffer* r) {
+    uint32_t expected_idle = RING_BUFFER_SYNC_PRODUCER_IDLE;
+    bool success =
+        __atomic_compare_exchange_n(&r->state, &expected_idle, RING_BUFFER_SYNC_CONSUMER_HANGING_UP,
+                                    false /* strong */, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
+    return success;
+}
+
+void ring_buffer_consumer_wait_producer_idle(struct ring_buffer* r) {
+    while (__atomic_load_n(&r->state, __ATOMIC_SEQ_CST) != RING_BUFFER_SYNC_PRODUCER_IDLE) {
+        ring_buffer_yield();
+    }
+}
+
+void ring_buffer_consumer_hung_up(struct ring_buffer* r) {
+    __atomic_store_n(&r->state, RING_BUFFER_SYNC_CONSUMER_HUNG_UP, __ATOMIC_SEQ_CST);
+}