init

src/freedreno/drm/freedreno_bo.c

@@ -0,0 +1,847 @@
+/*
+ * Copyright © 2012-2018 Rob Clark <[email protected]>
+ * SPDX-License-Identifier: MIT
+ *
+ * Authors:
+ *    Rob Clark <[email protected]>
+ */
+
+#include "util/os_mman.h"
+
+#include "freedreno_drmif.h"
+#include "freedreno_drm_perfetto.h"
+#include "freedreno_priv.h"
+
+simple_mtx_t table_lock = SIMPLE_MTX_INITIALIZER;
+simple_mtx_t fence_lock = SIMPLE_MTX_INITIALIZER;
+
+/* set buffer name, and add to table, call w/ table_lock held: */
+static void
+set_name(struct fd_bo *bo, uint32_t name)
+{
+   bo->name = name;
+   /* add ourself into the handle table: */
+   _mesa_hash_table_insert(bo->dev->name_table, &bo->name, bo);
+}
+
+static struct fd_bo zombie;
+
+/* lookup a buffer, call w/ table_lock held: */
+static struct fd_bo *
+lookup_bo(struct hash_table *tbl, uint32_t key)
+{
+   struct fd_bo *bo = NULL;
+   struct hash_entry *entry;
+
+   simple_mtx_assert_locked(&table_lock);
+
+   entry = _mesa_hash_table_search(tbl, &key);
+   if (entry) {
+      bo = entry->data;
+
+      /* We could be racing with final unref in another thread, and won
+       * the table_lock preventing the other thread from being able to
+       * remove an object it is about to free.  Fortunately since table
+       * lookup and removal are protected by the same lock (and table
+       * removal happens before obj free) we can easily detect this by
+       * checking for refcnt==0 (ie. 1 after p_atomic_inc_return).
+       */
+      if (p_atomic_inc_return(&bo->refcnt) == 1) {
+         /* Restore the zombified reference count, so if another thread
+          * that ends up calling lookup_bo() gets the table_lock before
+          * the thread deleting the bo does, it doesn't mistakenly see
+          * that the BO is live.
+          *
+          * We are holding the table_lock here so we can't be racing
+          * with another caller of lookup_bo()
+          */
+         p_atomic_dec(&bo->refcnt);
+         return &zombie;
+      }
+
+      if (!list_is_empty(&bo->node)) {
+         mesa_logw("bo was in cache, size=%u, alloc_flags=0x%x\n",
+                   bo->size, bo->alloc_flags);
+      }
+
+      /* don't break the bucket if this bo was found in one */
+      list_delinit(&bo->node);
+   }
+   return bo;
+}
+
+void
+fd_bo_init_common(struct fd_bo *bo, struct fd_device *dev)
+{
+   /* Backend should have initialized these: */
+   assert(bo->size);
+   assert(bo->handle);
+   assert(bo->funcs);
+
+   bo->dev = dev;
+   bo->iova = bo->funcs->iova(bo);
+   bo->reloc_flags = FD_RELOC_FLAGS_INIT;
+
+   p_atomic_set(&bo->refcnt, 1);
+   list_inithead(&bo->node);
+
+   bo->max_fences = 1;
+   bo->fences = &bo->_inline_fence;
+
+   if (!bo->map)
+      VG_BO_ALLOC(bo);
+}
+
+/* allocate a new buffer object, call w/ table_lock held */
+static struct fd_bo *
+import_bo_from_handle(struct fd_device *dev, uint32_t size, uint32_t handle)
+{
+   struct fd_bo *bo;
+
+   simple_mtx_assert_locked(&table_lock);
+
+   bo = dev->funcs->bo_from_handle(dev, size, handle);
+   if (!bo) {
+      struct drm_gem_close req = {
+         .handle = handle,
+      };
+      drmIoctl(dev->fd, DRM_IOCTL_GEM_CLOSE, &req);
+      return NULL;
+   }
+
+   bo->alloc_flags |= FD_BO_SHARED;
+
+   /* add ourself into the handle table: */
+   _mesa_hash_table_insert(dev->handle_table, &bo->handle, bo);
+
+   return bo;
+}
+
+static struct fd_bo *
+bo_new(struct fd_device *dev, uint32_t size, uint32_t flags,
+       struct fd_bo_cache *cache)
+{
+   struct fd_bo *bo = NULL;
+
+   if (size < FD_BO_HEAP_BLOCK_SIZE) {
+      uint32_t alloc_flags = flags & ~_FD_BO_HINTS;
+      if ((alloc_flags == 0) && dev->default_heap)
+         bo = fd_bo_heap_alloc(dev->default_heap, size, flags);
+      else if ((flags == RING_FLAGS) && dev->ring_heap)
+         bo = fd_bo_heap_alloc(dev->ring_heap, size, flags);
+      if (bo)
+         return bo;
+   }
+
+   /* demote cached-coherent to WC if not supported: */
+   if ((flags & FD_BO_CACHED_COHERENT) && !dev->has_cached_coherent)
+      flags &= ~FD_BO_CACHED_COHERENT;
+
+   bo = fd_bo_cache_alloc(cache, &size, flags);
+   if (bo)
+      return bo;
+
+   bo = dev->funcs->bo_new(dev, size, flags);
+   if (!bo)
+      return NULL;
+
+   simple_mtx_lock(&table_lock);
+   /* add ourself into the handle table: */
+   _mesa_hash_table_insert(dev->handle_table, &bo->handle, bo);
+   simple_mtx_unlock(&table_lock);
+
+   bo->alloc_flags = flags;
+
+   fd_alloc_log(bo, FD_ALLOC_NONE, FD_ALLOC_ACTIVE);
+
+   return bo;
+}
+
+struct fd_bo *
+_fd_bo_new(struct fd_device *dev, uint32_t size, uint32_t flags)
+{
+   struct fd_bo *bo = bo_new(dev, size, flags, &dev->bo_cache);
+   if (bo)
+      bo->bo_reuse = BO_CACHE;
+   return bo;
+}
+
+void
+_fd_bo_set_name(struct fd_bo *bo, const char *fmt, va_list ap)
+{
+   bo->funcs->set_name(bo, fmt, ap);
+}
+
+/* internal function to allocate bo's that use the ringbuffer cache
+ * instead of the normal bo_cache.  The purpose is, because cmdstream
+ * bo's get vmap'd on the kernel side, and that is expensive, we want
+ * to re-use cmdstream bo's for cmdstream and not unrelated purposes.
+ */
+struct fd_bo *
+fd_bo_new_ring(struct fd_device *dev, uint32_t size)
+{
+   struct fd_bo *bo = bo_new(dev, size, RING_FLAGS, &dev->ring_cache);
+   if (bo) {
+      bo->bo_reuse = RING_CACHE;
+      bo->reloc_flags |= FD_RELOC_DUMP;
+      fd_bo_set_name(bo, "cmdstream");
+   }
+   return bo;
+}
+
+struct fd_bo *
+fd_bo_from_handle(struct fd_device *dev, uint32_t handle, uint32_t size)
+{
+   struct fd_bo *bo = NULL;
+
+   simple_mtx_lock(&table_lock);
+
+   bo = lookup_bo(dev->handle_table, handle);
+   if (bo)
+      goto out_unlock;
+
+   bo = import_bo_from_handle(dev, size, handle);
+
+   VG_BO_ALLOC(bo);
+
+out_unlock:
+   simple_mtx_unlock(&table_lock);
+
+   /* We've raced with the handle being closed, so the handle is no longer
+    * valid.  Friends don't let friends share handles.
+    */
+   if (bo == &zombie)
+      return NULL;
+
+   return bo;
+}
+
+uint32_t
+fd_handle_from_dmabuf_drm(struct fd_device *dev, int fd)
+{
+   uint32_t handle;
+   int ret = drmPrimeFDToHandle(dev->fd, fd, &handle);
+   if (ret)
+      return 0;
+   return handle;
+}
+
+struct fd_bo *
+fd_bo_from_dmabuf_drm(struct fd_device *dev, int fd)
+{
+   int size;
+   uint32_t handle;
+   struct fd_bo *bo;
+
+restart:
+   simple_mtx_lock(&table_lock);
+   handle = dev->funcs->handle_from_dmabuf(dev, fd);
+   if (!handle) {
+      simple_mtx_unlock(&table_lock);
+      return NULL;
+   }
+
+   bo = lookup_bo(dev->handle_table, handle);
+   if (bo)
+      goto out_unlock;
+
+   /* lseek() to get bo size */
+   size = lseek(fd, 0, SEEK_END);
+   lseek(fd, 0, SEEK_CUR);
+
+   bo = import_bo_from_handle(dev, size, handle);
+
+   VG_BO_ALLOC(bo);
+
+out_unlock:
+   simple_mtx_unlock(&table_lock);
+
+   if (bo == &zombie)
+      goto restart;
+
+   return bo;
+}
+
+struct fd_bo *
+fd_bo_from_dmabuf(struct fd_device *dev, int fd)
+{
+   return dev->funcs->bo_from_dmabuf(dev, fd);
+}
+
+struct fd_bo *
+fd_bo_from_name(struct fd_device *dev, uint32_t name)
+{
+   struct drm_gem_open req = {
+      .name = name,
+   };
+   struct fd_bo *bo;
+
+   simple_mtx_lock(&table_lock);
+
+   /* check name table first, to see if bo is already open: */
+   bo = lookup_bo(dev->name_table, name);
+   if (bo)
+      goto out_unlock;
+
+restart:
+   if (drmIoctl(dev->fd, DRM_IOCTL_GEM_OPEN, &req)) {
+      ERROR_MSG("gem-open failed: %s", strerror(errno));
+      goto out_unlock;
+   }
+
+   bo = lookup_bo(dev->handle_table, req.handle);
+   if (bo)
+      goto out_unlock;
+
+   bo = import_bo_from_handle(dev, req.size, req.handle);
+   if (bo) {
+      set_name(bo, name);
+      VG_BO_ALLOC(bo);
+   }
+
+out_unlock:
+   simple_mtx_unlock(&table_lock);
+
+   if (bo == &zombie)
+      goto restart;
+
+   return bo;
+}
+
+void
+fd_bo_mark_for_dump(struct fd_bo *bo)
+{
+   bo->reloc_flags |= FD_RELOC_DUMP;
+}
+
+struct fd_bo *
+fd_bo_ref(struct fd_bo *bo)
+{
+   ref(&bo->refcnt);
+   return bo;
+}
+
+static void
+bo_finalize(struct fd_bo *bo)
+{
+   if (bo->funcs->finalize)
+      bo->funcs->finalize(bo);
+}
+
+static void
+dev_flush(struct fd_device *dev)
+{
+   if (dev->funcs->flush)
+      dev->funcs->flush(dev);
+}
+
+static void
+bo_del(struct fd_bo *bo)
+{
+   bo->funcs->destroy(bo);
+}
+
+static bool
+try_recycle(struct fd_bo *bo)
+{
+   struct fd_device *dev = bo->dev;
+
+   /* No point in BO cache for suballocated buffers: */
+   if (suballoc_bo(bo))
+      return false;
+
+   if (bo->bo_reuse == BO_CACHE)
+      return fd_bo_cache_free(&dev->bo_cache, bo) == 0;
+
+   if (bo->bo_reuse == RING_CACHE)
+      return fd_bo_cache_free(&dev->ring_cache, bo) == 0;
+
+   return false;
+}
+
+void
+fd_bo_del(struct fd_bo *bo)
+{
+   if (!unref(&bo->refcnt))
+      return;
+
+   if (try_recycle(bo))
+      return;
+
+   struct fd_device *dev = bo->dev;
+
+   bo_finalize(bo);
+   dev_flush(dev);
+   fd_alloc_log(bo, FD_ALLOC_ACTIVE, FD_ALLOC_NONE);
+   bo_del(bo);
+}
+
+void
+fd_bo_del_array(struct fd_bo **bos, int count)
+{
+   if (!count)
+      return;
+
+   struct fd_device *dev = bos[0]->dev;
+
+   /*
+    * First pass, remove objects from the table that either (a) still have
+    * a live reference, or (b) no longer have a reference but are released
+    * to the BO cache:
+    */
+
+   for (int i = 0; i < count; i++) {
+      if (!unref(&bos[i]->refcnt) || try_recycle(bos[i])) {
+         bos[i--] = bos[--count];
+      } else {
+         /* We are going to delete this one, so finalize it first: */
+         bo_finalize(bos[i]);
+      }
+   }
+
+   dev_flush(dev);
+
+   /*
+    * Second pass, delete all of the objects remaining after first pass.
+    */
+
+   for (int i = 0; i < count; i++) {
+      fd_alloc_log(bos[i], FD_ALLOC_ACTIVE, FD_ALLOC_NONE);
+      bo_del(bos[i]);
+   }
+}
+
+/**
+ * Special interface for fd_bo_cache to batch delete a list of handles.
+ * Similar to fd_bo_del_array() but bypasses the BO cache (since it is
+ * called from the BO cache to expire a list of BOs).
+ */
+void
+fd_bo_del_list_nocache(struct list_head *list)
+{
+   if (list_is_empty(list))
+      return;
+
+   struct fd_device *dev = first_bo(list)->dev;
+
+   foreach_bo (bo, list) {
+      bo_finalize(bo);
+   }
+
+   dev_flush(dev);
+
+   foreach_bo_safe (bo, list) {
+      assert(bo->refcnt == 0);
+      bo_del(bo);
+   }
+}
+
+void
+fd_bo_fini_fences(struct fd_bo *bo)
+{
+   for (int i = 0; i < bo->nr_fences; i++)
+      fd_fence_del(bo->fences[i]);
+
+   if (bo->fences != &bo->_inline_fence)
+      free(bo->fences);
+}
+
+void
+fd_bo_close_handle_drm(struct fd_bo *bo)
+{
+   struct drm_gem_close req = {
+      .handle = bo->handle,
+   };
+   drmIoctl(bo->dev->fd, DRM_IOCTL_GEM_CLOSE, &req);
+}
+
+/**
+ * Helper called by backends bo->funcs->destroy()
+ *
+ * Called under table_lock, bo_del_flush() *must* be called before
+ * table_lock is released (but bo->funcs->destroy() can be called
+ * multiple times before bo_del_flush(), as long as table_lock is
+ * held the entire time)
+ */
+void
+fd_bo_fini_common(struct fd_bo *bo)
+{
+   struct fd_device *dev = bo->dev;
+   uint32_t handle = bo->handle;
+
+   VG_BO_FREE(bo);
+
+   fd_bo_fini_fences(bo);
+
+   if (bo->map)
+      os_munmap(bo->map, bo->size);
+
+   if (handle) {
+      simple_mtx_lock(&table_lock);
+      dev->funcs->bo_close_handle(bo);
+      _mesa_hash_table_remove_key(dev->handle_table, &handle);
+      if (bo->name)
+         _mesa_hash_table_remove_key(dev->name_table, &bo->name);
+      simple_mtx_unlock(&table_lock);
+   }
+
+   free(bo);
+}
+
+static void
+bo_flush(struct fd_bo *bo)
+{
+   MESA_TRACE_FUNC();
+
+   simple_mtx_lock(&fence_lock);
+   unsigned nr = bo->nr_fences;
+   struct fd_fence *fences[nr];
+   for (unsigned i = 0; i < nr; i++)
+      fences[i] = fd_fence_ref_locked(bo->fences[i]);
+   simple_mtx_unlock(&fence_lock);
+
+   for (unsigned i = 0; i < nr; i++) {
+      fd_fence_flush(fences[i]);
+      fd_fence_del(fences[i]);
+   }
+}
+
+int
+fd_bo_get_name(struct fd_bo *bo, uint32_t *name)
+{
+   if (suballoc_bo(bo))
+      return -1;
+
+   if (!bo->name) {
+      struct drm_gem_flink req = {
+         .handle = bo->handle,
+      };
+      int ret;
+
+      ret = drmIoctl(bo->dev->fd, DRM_IOCTL_GEM_FLINK, &req);
+      if (ret) {
+         return ret;
+      }
+
+      simple_mtx_lock(&table_lock);
+      set_name(bo, req.name);
+      simple_mtx_unlock(&table_lock);
+      bo->bo_reuse = NO_CACHE;
+      bo->alloc_flags |= FD_BO_SHARED;
+      bo_flush(bo);
+   }
+
+   *name = bo->name;
+
+   return 0;
+}
+
+uint32_t
+fd_bo_handle(struct fd_bo *bo)
+{
+   if (suballoc_bo(bo))
+      return 0;
+   bo->bo_reuse = NO_CACHE;
+   bo->alloc_flags |= FD_BO_SHARED;
+   bo_flush(bo);
+   return bo->handle;
+}
+
+int
+fd_bo_dmabuf_drm(struct fd_bo *bo)
+{
+   int ret, prime_fd;
+
+   ret = drmPrimeHandleToFD(bo->dev->fd, bo->handle, DRM_CLOEXEC | DRM_RDWR,
+                            &prime_fd);
+   if (ret < 0)
+      return ret;
+
+   return prime_fd;
+}
+
+int
+fd_bo_dmabuf(struct fd_bo *bo)
+{
+   int ret;
+
+   if (suballoc_bo(bo))
+      return -1;
+
+   ret = bo->funcs->dmabuf(bo);
+   if (ret < 0) {
+      ERROR_MSG("failed to get dmabuf fd: %d", ret);
+      return ret;
+   }
+
+   bo->bo_reuse = NO_CACHE;
+   bo->alloc_flags |= FD_BO_SHARED;
+   bo_flush(bo);
+
+   return ret;
+}
+
+uint32_t
+fd_bo_size(struct fd_bo *bo)
+{
+   return bo->size;
+}
+
+bool
+fd_bo_is_cached(struct fd_bo *bo)
+{
+   return !!(bo->alloc_flags & FD_BO_CACHED_COHERENT);
+}
+
+void
+fd_bo_set_metadata(struct fd_bo *bo, void *metadata, uint32_t metadata_size)
+{
+   if (!bo->funcs->set_metadata)
+      return;
+   bo->funcs->set_metadata(bo, metadata, metadata_size);
+}
+
+int
+fd_bo_get_metadata(struct fd_bo *bo, void *metadata, uint32_t metadata_size)
+{
+   if (!bo->funcs->get_metadata)
+      return -ENOSYS;
+   return bo->funcs->get_metadata(bo, metadata, metadata_size);
+}
+
+void *
+fd_bo_map_os_mmap(struct fd_bo *bo)
+{
+   uint64_t offset;
+   int ret;
+   ret = bo->funcs->offset(bo, &offset);
+   if (ret) {
+      return NULL;
+   }
+   return os_mmap(0, bo->size, PROT_READ | PROT_WRITE, MAP_SHARED,
+                  bo->dev->fd, offset);
+}
+
+/* For internal use only, does not check FD_BO_NOMAP: */
+void *
+__fd_bo_map(struct fd_bo *bo)
+{
+   if (!bo->map) {
+      bo->map = bo->funcs->map(bo);
+      if (bo->map == MAP_FAILED) {
+         ERROR_MSG("mmap failed: %s", strerror(errno));
+         bo->map = NULL;
+      }
+   }
+
+   return bo->map;
+}
+
+void *
+fd_bo_map(struct fd_bo *bo)
+{
+   /* don't allow mmap'ing something allocated with FD_BO_NOMAP
+    * for sanity
+    */
+   if (bo->alloc_flags & FD_BO_NOMAP)
+      return NULL;
+
+   return __fd_bo_map(bo);
+}
+
+static void *
+fd_bo_map_for_upload(struct fd_bo *bo)
+{
+   void *addr = __fd_bo_map(bo);
+   if (bo->alloc_flags & FD_BO_NOMAP)
+      VG_BO_MAPPED(bo);
+
+   return addr;
+}
+
+void
+fd_bo_upload(struct fd_bo *bo, void *src, unsigned off, unsigned len)
+{
+   if (bo->funcs->upload) {
+      bo->funcs->upload(bo, src, off, len);
+      return;
+   }
+
+   memcpy((uint8_t *)fd_bo_map_for_upload(bo) + off, src, len);
+}
+
+bool
+fd_bo_prefer_upload(struct fd_bo *bo, unsigned len)
+{
+   if (bo->funcs->prefer_upload)
+      return bo->funcs->prefer_upload(bo, len);
+
+   return false;
+}
+
+/* a bit odd to take the pipe as an arg, but it's a, umm, quirk of kgsl.. */
+int
+fd_bo_cpu_prep(struct fd_bo *bo, struct fd_pipe *pipe, uint32_t op)
+{
+   enum fd_bo_state state = fd_bo_state(bo);
+
+   if (state == FD_BO_STATE_IDLE)
+      return 0;
+
+   MESA_TRACE_FUNC();
+
+   if (op & (FD_BO_PREP_NOSYNC | FD_BO_PREP_FLUSH)) {
+      if (op & FD_BO_PREP_FLUSH)
+         bo_flush(bo);
+
+      /* If we have *only* been asked to flush, then we aren't really
+       * interested about whether shared buffers are busy, so avoid
+       * the kernel ioctl.
+       */
+      if ((state == FD_BO_STATE_BUSY) ||
+          (op == FD_BO_PREP_FLUSH))
+         return -EBUSY;
+   }
+
+   /* In case the bo is referenced by a deferred submit, flush up to the
+    * required fence now:
+    */
+   bo_flush(bo);
+
+   /* FD_BO_PREP_FLUSH is purely a frontend flag, and is not seen/handled
+    * by backend or kernel:
+    */
+   op &= ~FD_BO_PREP_FLUSH;
+
+   if (!op)
+      return 0;
+
+   /* Wait on fences.. first grab a reference under the fence lock, and then
+    * wait and drop ref.
+    */
+   simple_mtx_lock(&fence_lock);
+   unsigned nr = bo->nr_fences;
+   struct fd_fence *fences[nr];
+   for (unsigned i = 0; i < nr; i++)
+      fences[i] = fd_fence_ref_locked(bo->fences[i]);
+   simple_mtx_unlock(&fence_lock);
+
+   for (unsigned i = 0; i < nr; i++) {
+      fd_fence_wait(fences[i]);
+      fd_fence_del(fences[i]);
+   }
+
+   /* expire completed fences */
+   fd_bo_state(bo);
+
+   /* None shared buffers will not have any external usage (ie. fences
+    * that we are not aware of) so nothing more to do.
+    */
+   if (!(bo->alloc_flags & FD_BO_SHARED))
+      return 0;
+
+   /* If buffer is shared, but we are using explicit sync, no need to
+    * fallback to implicit sync:
+    */
+   if (pipe && pipe->no_implicit_sync)
+      return 0;
+
+   return bo->funcs->cpu_prep(bo, pipe, op);
+}
+
+/**
+ * Cleanup fences, dropping pipe references.  If 'expired' is true, only
+ * cleanup expired fences.
+ *
+ * Normally we expect at most a single fence, the exception being bo's
+ * shared between contexts
+ */
+static void
+cleanup_fences(struct fd_bo *bo)
+{
+   simple_mtx_assert_locked(&fence_lock);
+
+   for (int i = 0; i < bo->nr_fences; i++) {
+      struct fd_fence *f = bo->fences[i];
+
+      if (fd_fence_before(f->pipe->control->fence, f->ufence))
+         continue;
+
+      bo->nr_fences--;
+
+      if (bo->nr_fences > 0) {
+         /* Shuffle up the last entry to replace the current slot: */
+         bo->fences[i] = bo->fences[bo->nr_fences];
+         i--;
+      }
+
+      fd_fence_del_locked(f);
+   }
+}
+
+void
+fd_bo_add_fence(struct fd_bo *bo, struct fd_fence *fence)
+{
+   simple_mtx_assert_locked(&fence_lock);
+
+   if (bo->alloc_flags & _FD_BO_NOSYNC)
+      return;
+
+   /* The common case is bo re-used on the same pipe it had previously
+    * been used on, so just replace the previous fence.
+    */
+   for (int i = 0; i < bo->nr_fences; i++) {
+      struct fd_fence *f = bo->fences[i];
+      if (f == fence)
+         return;
+      if (f->pipe == fence->pipe) {
+         assert(fd_fence_before(f->ufence, fence->ufence));
+         fd_fence_del_locked(f);
+         bo->fences[i] = fd_fence_ref_locked(fence);
+         return;
+      }
+   }
+
+   cleanup_fences(bo);
+
+   /* The first time we grow past a single fence, we need some special
+    * handling, as we've been using the embedded _inline_fence to avoid
+    * a separate allocation:
+    */
+   if (unlikely((bo->nr_fences == 1) &&
+                (bo->fences == &bo->_inline_fence))) {
+      bo->nr_fences = bo->max_fences = 0;
+      bo->fences = NULL;
+      APPEND(bo, fences, bo->_inline_fence);
+   }
+
+   APPEND(bo, fences, fd_fence_ref_locked(fence));
+}
+
+enum fd_bo_state
+fd_bo_state(struct fd_bo *bo)
+{
+   /* NOTE: check the nosync case before touching fence_lock in case we end
+    * up here recursively from dropping pipe reference in cleanup_fences().
+    * The pipe's control buffer is specifically nosync to avoid recursive
+    * lock problems here.
+    */
+   if (bo->alloc_flags & (FD_BO_SHARED | _FD_BO_NOSYNC))
+      return FD_BO_STATE_UNKNOWN;
+
+   /* Speculatively check, if we already know we're idle, no need to acquire
+    * lock and do the cleanup_fences() dance:
+    */
+   if (!bo->nr_fences)
+      return FD_BO_STATE_IDLE;
+
+   simple_mtx_lock(&fence_lock);
+   cleanup_fences(bo);
+   simple_mtx_unlock(&fence_lock);
+
+   if (!bo->nr_fences)
+      return FD_BO_STATE_IDLE;
+
+   return FD_BO_STATE_BUSY;
+}
+

src/freedreno/drm/freedreno_bo_cache.c

@@ -0,0 +1,309 @@
+/*
+ * Copyright © 2012-2018 Rob Clark <[email protected]>
+ * SPDX-License-Identifier: MIT
+ *
+ * Authors:
+ *    Rob Clark <[email protected]>
+ */
+
+#include "freedreno_drmif.h"
+#include "freedreno_drm_perfetto.h"
+#include "freedreno_priv.h"
+
+#define FD_BO_CACHE_STATS 0
+
+#define BO_CACHE_LOG(cache, fmt, ...) do {                      \
+      if (FD_BO_CACHE_STATS) {                                  \
+         mesa_logi("%s: "fmt, (cache)->name, ##__VA_ARGS__);    \
+      }                                                         \
+   } while (0)
+
+#define BO_CACHE_INC(bucket, field) do {                        \
+      if (FD_BO_CACHE_STATS) {                                  \
+         (bucket)->field++;                                     \
+      }                                                         \
+   } while (0)
+
+#define BO_CACHE_DEC(bucket, field) do {                        \
+      if (FD_BO_CACHE_STATS) {                                  \
+         (bucket)->field++;                                     \
+      }                                                         \
+   } while (0)
+
+static void
+bo_remove_from_bucket(struct fd_bo_bucket *bucket, struct fd_bo *bo)
+{
+   list_delinit(&bo->node);
+   BO_CACHE_DEC(bucket, count);
+}
+
+static void
+dump_cache_stats(struct fd_bo_cache *cache)
+{
+   if (!FD_BO_CACHE_STATS)
+      return;
+
+   static int cnt;
+
+   if ((++cnt % 32))
+      return;
+
+   int size = 0;
+   int count = 0;
+   int hits = 0;
+   int misses = 0;
+   int expired = 0;
+
+   for (int i = 0; i < cache->num_buckets; i++) {
+      char *state = "";
+
+      struct fd_bo_bucket *bucket = &cache->cache_bucket[i];
+
+      if (bucket->count > 0) {
+         struct fd_bo *bo = first_bo(&bucket->list);
+         if (fd_bo_state(bo) == FD_BO_STATE_IDLE)
+            state = " (idle)";
+      }
+
+      BO_CACHE_LOG(cache, "bucket[%u]: count=%d\thits=%d\tmisses=%d\texpired=%d%s",
+                   bucket->size, bucket->count, bucket->hits,
+                   bucket->misses, bucket->expired, state);
+
+      size += bucket->size * bucket->count;
+      count += bucket->count;
+      hits += bucket->hits;
+      misses += bucket->misses;
+      expired += bucket->expired;
+   }
+
+   BO_CACHE_LOG(cache, "size=%d\tcount=%d\thits=%d\tmisses=%d\texpired=%d",
+                size, count, hits, misses, expired);
+}
+
+static void
+add_bucket(struct fd_bo_cache *cache, int size)
+{
+   unsigned int i = cache->num_buckets;
+   struct fd_bo_bucket *bucket = &cache->cache_bucket[i];
+
+   assert(i < ARRAY_SIZE(cache->cache_bucket));
+
+   list_inithead(&bucket->list);
+   bucket->size = size;
+   bucket->count = 0;
+   bucket->hits = 0;
+   bucket->misses = 0;
+   bucket->expired = 0;
+   cache->num_buckets++;
+}
+
+/**
+ * @coarse: if true, only power-of-two bucket sizes, otherwise
+ *    fill in for a bit smoother size curve..
+ */
+void
+fd_bo_cache_init(struct fd_bo_cache *cache, int coarse, const char *name)
+{
+   unsigned long size, cache_max_size = 64 * 1024 * 1024;
+
+   cache->name = name;
+   simple_mtx_init(&cache->lock, mtx_plain);
+
+   /* OK, so power of two buckets was too wasteful of memory.
+    * Give 3 other sizes between each power of two, to hopefully
+    * cover things accurately enough.  (The alternative is
+    * probably to just go for exact matching of sizes, and assume
+    * that for things like composited window resize the tiled
+    * width/height alignment and rounding of sizes to pages will
+    * get us useful cache hit rates anyway)
+    */
+   add_bucket(cache, os_page_size);
+   add_bucket(cache, os_page_size * 2);
+   if (!coarse)
+      add_bucket(cache, os_page_size * 3);
+
+   /* Initialize the linked lists for BO reuse cache. */
+   for (size = 4 * os_page_size; size <= cache_max_size; size *= 2) {
+      add_bucket(cache, size);
+      if (!coarse) {
+         add_bucket(cache, size + size * 1 / 4);
+         add_bucket(cache, size + size * 2 / 4);
+         add_bucket(cache, size + size * 3 / 4);
+      }
+   }
+}
+
+/* Frees older cached buffers.  Called under table_lock */
+void
+fd_bo_cache_cleanup(struct fd_bo_cache *cache, time_t time)
+{
+   int i, cnt = 0;
+
+   if (cache->time == time)
+      return;
+
+   struct list_head freelist;
+
+   list_inithead(&freelist);
+
+   simple_mtx_lock(&cache->lock);
+   for (i = 0; i < cache->num_buckets; i++) {
+      struct fd_bo_bucket *bucket = &cache->cache_bucket[i];
+      struct fd_bo *bo;
+
+      while (!list_is_empty(&bucket->list)) {
+         bo = first_bo(&bucket->list);
+
+         /* keep things in cache for at least 1 second: */
+         if (time && ((time - bo->free_time) <= 1))
+            break;
+
+         if (cnt == 0) {
+            BO_CACHE_LOG(cache, "cache cleanup");
+            dump_cache_stats(cache);
+         }
+
+         VG_BO_OBTAIN(bo);
+         bo_remove_from_bucket(bucket, bo);
+         BO_CACHE_INC(bucket, expired);
+         list_addtail(&bo->node, &freelist);
+         fd_alloc_log(bo, FD_ALLOC_CACHE, FD_ALLOC_NONE);
+
+         cnt++;
+      }
+   }
+   simple_mtx_unlock(&cache->lock);
+
+   fd_bo_del_list_nocache(&freelist);
+
+   if (cnt > 0) {
+      BO_CACHE_LOG(cache, "cache cleaned %u BOs", cnt);
+      dump_cache_stats(cache);
+   }
+
+   cache->time = time;
+}
+
+static struct fd_bo_bucket *
+get_bucket(struct fd_bo_cache *cache, uint32_t size)
+{
+   int i;
+
+   /* hmm, this is what intel does, but I suppose we could calculate our
+    * way to the correct bucket size rather than looping..
+    */
+   for (i = 0; i < cache->num_buckets; i++) {
+      struct fd_bo_bucket *bucket = &cache->cache_bucket[i];
+      if (bucket->size >= size) {
+         return bucket;
+      }
+   }
+
+   return NULL;
+}
+
+static struct fd_bo *
+find_in_bucket(struct fd_bo_bucket *bucket, uint32_t flags)
+{
+   struct fd_bo *bo = NULL;
+
+   /* TODO .. if we had an ALLOC_FOR_RENDER flag like intel, we could
+    * skip the busy check.. if it is only going to be a render target
+    * then we probably don't need to stall..
+    *
+    * NOTE that intel takes ALLOC_FOR_RENDER bo's from the list tail
+    * (MRU, since likely to be in GPU cache), rather than head (LRU)..
+    */
+   foreach_bo (entry, &bucket->list) {
+      if (fd_bo_state(entry) != FD_BO_STATE_IDLE) {
+         break;
+      }
+      if (entry->alloc_flags == flags) {
+         bo = entry;
+         bo_remove_from_bucket(bucket, bo);
+         break;
+      }
+   }
+
+   return bo;
+}
+
+/* NOTE: size is potentially rounded up to bucket size: */
+struct fd_bo *
+fd_bo_cache_alloc(struct fd_bo_cache *cache, uint32_t *size, uint32_t flags)
+{
+   struct fd_bo *bo = NULL;
+   struct fd_bo_bucket *bucket;
+
+   *size = align(*size, os_page_size);
+   bucket = get_bucket(cache, *size);
+
+   struct list_head freelist;
+
+   list_inithead(&freelist);
+
+   /* see if we can be green and recycle: */
+retry:
+   if (bucket) {
+      *size = bucket->size;
+      simple_mtx_lock(&cache->lock);
+      bo = find_in_bucket(bucket, flags);
+      simple_mtx_unlock(&cache->lock);
+      if (bo) {
+         VG_BO_OBTAIN(bo);
+         if (bo->funcs->madvise(bo, true) <= 0) {
+            /* we've lost the backing pages, delete and try again: */
+            list_addtail(&bo->node, &freelist);
+            fd_alloc_log(bo, FD_ALLOC_CACHE, FD_ALLOC_NONE);
+            goto retry;
+         }
+         p_atomic_set(&bo->refcnt, 1);
+         bo->reloc_flags = FD_RELOC_FLAGS_INIT;
+         BO_CACHE_INC(bucket, hits);
+         fd_alloc_log(bo, FD_ALLOC_CACHE, FD_ALLOC_ACTIVE);
+         return bo;
+      }
+      BO_CACHE_INC(bucket, misses);
+   }
+
+   fd_bo_del_list_nocache(&freelist);
+
+   BO_CACHE_LOG(cache, "miss on size=%u, flags=0x%x, bucket=%u", *size, flags,
+                bucket ? bucket->size : 0);
+   dump_cache_stats(cache);
+
+   return NULL;
+}
+
+int
+fd_bo_cache_free(struct fd_bo_cache *cache, struct fd_bo *bo)
+{
+   if (bo->alloc_flags & (FD_BO_SHARED | _FD_BO_NOSYNC))
+      return -1;
+
+   struct fd_bo_bucket *bucket = get_bucket(cache, bo->size);
+
+   /* see if we can be green and recycle: */
+   if (bucket) {
+      struct timespec time;
+
+      bo->funcs->madvise(bo, false);
+
+      clock_gettime(CLOCK_MONOTONIC, &time);
+
+      bo->free_time = time.tv_sec;
+      VG_BO_RELEASE(bo);
+
+      simple_mtx_lock(&cache->lock);
+      list_addtail(&bo->node, &bucket->list);
+      BO_CACHE_INC(bucket, count);
+      simple_mtx_unlock(&cache->lock);
+
+      fd_alloc_log(bo, FD_ALLOC_ACTIVE, FD_ALLOC_CACHE);
+      fd_bo_cache_cleanup(cache, time.tv_sec);
+
+      return 0;
+   }
+
+   return -1;
+}

src/freedreno/drm/freedreno_bo_heap.c

@@ -0,0 +1,260 @@
+/*
+ * Copyright © 2022 Google, Inc.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "freedreno_drmif.h"
+#include "freedreno_drm_perfetto.h"
+#include "freedreno_priv.h"
+
+struct sa_bo {
+   struct fd_bo base;
+   struct fd_bo_heap *heap;
+   unsigned offset;
+};
+FD_DEFINE_CAST(fd_bo, sa_bo);
+
+#define HEAP_DEBUG 0
+
+static void heap_clean(struct fd_bo_heap *heap, bool idle);
+static void heap_dump(struct fd_bo_heap *heap);
+
+struct fd_bo_heap *
+fd_bo_heap_new(struct fd_device *dev, uint32_t flags)
+{
+   struct fd_bo_heap *heap;
+
+   /* We cannot suballocate shared buffers! Implicit sync is not supported! */
+   assert(!(flags & FD_BO_SHARED));
+
+   /* No internal buffers either, we need userspace fencing: */
+   assert(!(flags & _FD_BO_NOSYNC));
+
+   heap = calloc(1, sizeof(*heap));
+
+   heap->dev = dev;
+   heap->flags = flags;
+   simple_mtx_init(&heap->lock, mtx_plain);
+   list_inithead(&heap->freelist);
+
+   /* Note that util_vma_heap_init doesn't like offset==0, so we shift the
+    * entire range by one block size (see block_idx()):
+    */
+   util_vma_heap_init(&heap->heap, FD_BO_HEAP_BLOCK_SIZE,
+                      FD_BO_HEAP_BLOCK_SIZE * ARRAY_SIZE(heap->blocks));
+   heap->heap.alloc_high = false;
+   heap->heap.nospan_shift = ffs(FD_BO_HEAP_BLOCK_SIZE) - 1;
+
+   heap_dump(heap);
+
+   return heap;
+}
+
+void fd_bo_heap_destroy(struct fd_bo_heap *heap)
+{
+   /* drain the freelist: */
+   heap_clean(heap, false);
+
+   util_vma_heap_finish(&heap->heap);
+   for (unsigned i = 0; i < ARRAY_SIZE(heap->blocks); i++)
+      if (heap->blocks[i])
+         fd_bo_del(heap->blocks[i]);
+   free(heap);
+}
+
+static bool
+sa_idle(struct fd_bo *bo)
+{
+   enum fd_bo_state state = fd_bo_state(bo);
+   assert(state != FD_BO_STATE_UNKNOWN);
+   return state == FD_BO_STATE_IDLE;
+}
+
+/**
+ * The backing block is determined by the offset within the heap, since all
+ * the blocks are equal size
+ */
+static unsigned
+block_idx(struct sa_bo *s)
+{
+   /* The vma allocator doesn't like offset=0 so the range is shifted up
+    * by one block size:
+    */
+   return (s->offset / FD_BO_HEAP_BLOCK_SIZE) - 1;
+}
+
+static unsigned
+block_offset(struct sa_bo *s)
+{
+   return s->offset % FD_BO_HEAP_BLOCK_SIZE;
+}
+
+static void
+heap_dump(struct fd_bo_heap *heap)
+{
+   if (!HEAP_DEBUG)
+      return;
+   fprintf(stderr, "HEAP[%x]: freelist: %u\n", heap->flags, list_length(&heap->freelist));
+   util_vma_heap_print(&heap->heap, stderr, "",
+                       FD_BO_HEAP_BLOCK_SIZE * ARRAY_SIZE(heap->blocks));
+}
+
+static void
+sa_release(struct fd_bo *bo)
+{
+   struct sa_bo *s = to_sa_bo(bo);
+
+   simple_mtx_assert_locked(&s->heap->lock);
+
+   /*
+    * We don't track heap allocs in valgrind
+    * VG_BO_FREE(bo);
+    */
+
+   fd_bo_fini_fences(bo);
+
+   if (HEAP_DEBUG)
+      mesa_logi("release: %08x-%x idx=%d", s->offset, bo->size, block_idx(s));
+
+   util_vma_heap_free(&s->heap->heap, s->offset, bo->size);
+
+   /* The BO has already been moved ACTIVE->NONE, now move it back to heap: */
+   fd_alloc_log(bo, FD_ALLOC_NONE, FD_ALLOC_HEAP);
+
+   /* Drop our reference to the backing block object: */
+   fd_bo_del(s->heap->blocks[block_idx(s)]);
+
+   list_del(&bo->node);
+
+   if ((++s->heap->cnt % 256) == 0)
+      heap_dump(s->heap);
+
+   free(bo);
+}
+
+static int
+sa_madvise(struct fd_bo *bo, int willneed)
+{
+   return willneed;
+}
+
+static uint64_t
+sa_iova(struct fd_bo *bo)
+{
+   struct sa_bo *s = to_sa_bo(bo);
+
+   return s->heap->blocks[block_idx(s)]->iova + block_offset(s);
+}
+
+static void
+sa_set_name(struct fd_bo *bo, const char *fmt, va_list ap)
+{
+   /* No-op, kernel has a single name for the entire buffer we suballoc from */
+}
+
+static void
+sa_destroy(struct fd_bo *bo)
+{
+   struct fd_bo_heap *heap = to_sa_bo(bo)->heap;
+
+   simple_mtx_lock(&heap->lock);
+   list_addtail(&bo->node, &heap->freelist);
+   simple_mtx_unlock(&heap->lock);
+}
+
+static struct fd_bo_funcs heap_bo_funcs = {
+      .madvise = sa_madvise,
+      .iova = sa_iova,
+      .map = fd_bo_map_os_mmap,
+      .set_name = sa_set_name,
+      .destroy = sa_destroy,
+};
+
+/**
+ * Get the backing heap block of a suballocated bo
+ */
+struct fd_bo *
+fd_bo_heap_block(struct fd_bo *bo)
+{
+   assert(suballoc_bo(bo));
+
+   struct sa_bo *s = to_sa_bo(bo);
+   return s->heap->blocks[block_idx(s)];
+}
+
+static void
+heap_clean(struct fd_bo_heap *heap, bool idle)
+{
+   simple_mtx_lock(&heap->lock);
+   foreach_bo_safe (bo, &heap->freelist) {
+      /* It might be nice if we could keep freelist sorted by fence # */
+      if (idle && !sa_idle(bo))
+         break;
+      sa_release(bo);
+   }
+   simple_mtx_unlock(&heap->lock);
+}
+
+struct fd_bo *
+fd_bo_heap_alloc(struct fd_bo_heap *heap, uint32_t size, uint32_t flags)
+{
+   heap_clean(heap, true);
+
+   /* util_vma does not like zero byte allocations, which we get, for
+    * ex, with the initial query buffer allocation on pre-a5xx:
+    */
+   size = MAX2(size, SUBALLOC_ALIGNMENT);
+
+   size = align(size, SUBALLOC_ALIGNMENT);
+
+   simple_mtx_lock(&heap->lock);
+   /* Allocate larger buffers from the bottom, and smaller buffers from top
+    * to help limit fragmentation:
+    *
+    * (The 8k threshold is just a random guess, but seems to work ok)
+    */
+   heap->heap.alloc_high = (size <= 8 * 1024);
+   uint64_t offset = util_vma_heap_alloc(&heap->heap, size, SUBALLOC_ALIGNMENT);
+   if (!offset) {
+      simple_mtx_unlock(&heap->lock);
+      return NULL;
+   }
+
+   struct sa_bo *s = calloc(1, sizeof(*s));
+
+   s->heap = heap;
+   s->offset = offset;
+
+   assert((s->offset / FD_BO_HEAP_BLOCK_SIZE) == (s->offset + size - 1) / FD_BO_HEAP_BLOCK_SIZE);
+   unsigned idx = block_idx(s);
+   if (HEAP_DEBUG)
+      mesa_logi("alloc: %08x-%x idx=%d", s->offset, size, idx);
+   if (!heap->blocks[idx]) {
+      heap->blocks[idx] = fd_bo_new(
+            heap->dev, FD_BO_HEAP_BLOCK_SIZE, heap->flags | _FD_BO_HINT_HEAP,
+            "heap-%x-block-%u", heap->flags, idx);
+      if (heap->flags == RING_FLAGS)
+         fd_bo_mark_for_dump(heap->blocks[idx]);
+   }
+   /* Take a reference to the backing obj: */
+   fd_bo_ref(heap->blocks[idx]);
+   simple_mtx_unlock(&heap->lock);
+
+   struct fd_bo *bo = &s->base;
+
+   bo->size = size;
+   bo->funcs = &heap_bo_funcs;
+   bo->handle = 1; /* dummy handle to make fd_bo_init_common() happy */
+   bo->alloc_flags = flags;
+
+   /* Pre-initialize mmap ptr, to avoid trying to os_mmap() */
+   bo->map = ((uint8_t *)fd_bo_map(heap->blocks[idx])) + block_offset(s);
+
+   fd_bo_init_common(bo, heap->dev);
+
+   bo->handle = FD_BO_SUBALLOC_HANDLE;
+
+   fd_alloc_log(bo, FD_ALLOC_HEAP, FD_ALLOC_ACTIVE);
+
+   return bo;
+}

src/freedreno/drm/freedreno_device.c

@@ -0,0 +1,252 @@
+/*
+ * Copyright © 2012-2018 Rob Clark <[email protected]>
+ * SPDX-License-Identifier: MIT
+ *
+ * Authors:
+ *    Rob Clark <[email protected]>
+ */
+
+#include <unistd.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+
+#include "util/os_file.h"
+#include "util/u_process.h"
+
+#include "freedreno_rd_output.h"
+
+#include "freedreno_drmif.h"
+#include "freedreno_drm_perfetto.h"
+#include "freedreno_priv.h"
+
+struct fd_device *msm_device_new(int fd, drmVersionPtr version);
+#ifdef HAVE_FREEDRENO_VIRTIO
+struct fd_device *virtio_device_new(int fd, drmVersionPtr version);
+#endif
+
+uint64_t os_page_size = 4096;
+
+struct fd_device *
+fd_device_new(int fd)
+{
+   struct fd_device *dev = NULL;
+   drmVersionPtr version;
+   bool use_heap = false;
+
+   os_get_page_size(&os_page_size);
+
+   /* figure out if we are kgsl or msm drm driver: */
+   version = drmGetVersion(fd);
+   if (!version) {
+      ERROR_MSG("cannot get version: %s", strerror(errno));
+      return NULL;
+   }
+
+#ifdef HAVE_FREEDRENO_VIRTIO
+   if (debug_get_bool_option("FD_FORCE_VTEST", false)) {
+      DEBUG_MSG("virtio_gpu vtest device");
+      dev = virtio_device_new(-1, version);
+   } else
+#endif
+   if (!strcmp(version->name, "msm")) {
+      DEBUG_MSG("msm DRM device");
+      if (version->version_major != 1) {
+         ERROR_MSG("unsupported version: %u.%u.%u", version->version_major,
+                   version->version_minor, version->version_patchlevel);
+         goto out;
+      }
+
+      dev = msm_device_new(fd, version);
+#ifdef HAVE_FREEDRENO_VIRTIO
+   } else if (!strcmp(version->name, "virtio_gpu")) {
+      DEBUG_MSG("virtio_gpu DRM device");
+      dev = virtio_device_new(fd, version);
+      /* Only devices that support a hypervisor are a6xx+, so avoid the
+       * extra guest<->host round trips associated with pipe creation:
+       */
+      use_heap = true;
+#endif
+#if HAVE_FREEDRENO_KGSL
+   } else if (!strcmp(version->name, "kgsl")) {
+      DEBUG_MSG("kgsl DRM device");
+      dev = kgsl_device_new(fd);
+#endif
+   }
+
+   if (!dev) {
+      INFO_MSG("unsupported device: %s", version->name);
+      goto out;
+   }
+
+out:
+   drmFreeVersion(version);
+
+   if (!dev)
+      return NULL;
+
+   fd_drm_perfetto_init();
+
+   fd_rd_dump_env_init();
+   fd_rd_output_init(&dev->rd, util_get_process_name());
+
+   p_atomic_set(&dev->refcnt, 1);
+   dev->fd = fd;
+   dev->handle_table =
+      _mesa_hash_table_create(NULL, _mesa_hash_u32, _mesa_key_u32_equal);
+   dev->name_table =
+      _mesa_hash_table_create(NULL, _mesa_hash_u32, _mesa_key_u32_equal);
+   fd_bo_cache_init(&dev->bo_cache, false, "bo");
+   fd_bo_cache_init(&dev->ring_cache, true, "ring");
+
+   list_inithead(&dev->deferred_submits);
+   simple_mtx_init(&dev->submit_lock, mtx_plain);
+   simple_mtx_init(&dev->suballoc_lock, mtx_plain);
+
+   if (!use_heap) {
+      struct fd_pipe *pipe = fd_pipe_new(dev, FD_PIPE_3D);
+
+      if (!pipe)
+         goto fail;
+
+      /* Userspace fences don't appear to be reliable enough (missing some
+       * cache flushes?) on older gens, so limit sub-alloc heaps to a6xx+
+       * for now:
+       */
+      use_heap = fd_dev_gen(&pipe->dev_id) >= 6;
+
+      fd_pipe_del(pipe);
+   }
+
+   if (use_heap) {
+      dev->ring_heap = fd_bo_heap_new(dev, RING_FLAGS);
+      dev->default_heap = fd_bo_heap_new(dev, 0);
+   }
+
+   return dev;
+
+fail:
+   fd_device_del(dev);
+   return NULL;
+}
+
+/* like fd_device_new() but creates it's own private dup() of the fd
+ * which is close()d when the device is finalized.
+ */
+struct fd_device *
+fd_device_new_dup(int fd)
+{
+   int dup_fd = os_dupfd_cloexec(fd);
+   struct fd_device *dev = fd_device_new(dup_fd);
+   if (dev)
+      dev->closefd = 1;
+   else
+      close(dup_fd);
+   return dev;
+}
+
+/* Convenience helper to open the drm device and return new fd_device:
+ */
+struct fd_device *
+fd_device_open(void)
+{
+   int fd;
+
+   fd = drmOpenWithType("msm", NULL, DRM_NODE_RENDER);
+#ifdef HAVE_FREEDRENO_VIRTIO
+   if (fd < 0)
+      fd = drmOpenWithType("virtio_gpu", NULL, DRM_NODE_RENDER);
+#endif
+   if (fd < 0)
+      return NULL;
+
+   return fd_device_new(fd);
+}
+
+struct fd_device *
+fd_device_ref(struct fd_device *dev)
+{
+   ref(&dev->refcnt);
+   return dev;
+}
+
+void
+fd_device_purge(struct fd_device *dev)
+{
+   fd_bo_cache_cleanup(&dev->bo_cache, 0);
+   fd_bo_cache_cleanup(&dev->ring_cache, 0);
+}
+
+void
+fd_device_del(struct fd_device *dev)
+{
+   if (!unref(&dev->refcnt))
+      return;
+
+   fd_rd_output_fini(&dev->rd);
+
+   assert(list_is_empty(&dev->deferred_submits));
+   assert(!dev->deferred_submits_fence);
+
+   if (dev->suballoc_bo)
+      fd_bo_del(dev->suballoc_bo);
+
+   if (dev->ring_heap)
+      fd_bo_heap_destroy(dev->ring_heap);
+
+   if (dev->default_heap)
+      fd_bo_heap_destroy(dev->default_heap);
+
+   fd_bo_cache_cleanup(&dev->bo_cache, 0);
+   fd_bo_cache_cleanup(&dev->ring_cache, 0);
+
+   /* Needs to be after bo cache cleanup in case backend has a
+    * util_vma_heap that it destroys:
+    */
+   dev->funcs->destroy(dev);
+
+   _mesa_hash_table_destroy(dev->handle_table, NULL);
+   _mesa_hash_table_destroy(dev->name_table, NULL);
+
+   if (fd_device_threaded_submit(dev))
+      util_queue_destroy(&dev->submit_queue);
+
+   if (dev->closefd)
+      close(dev->fd);
+
+   free(dev);
+}
+
+int
+fd_device_fd(struct fd_device *dev)
+{
+   return dev->fd;
+}
+
+enum fd_version
+fd_device_version(struct fd_device *dev)
+{
+   return dev->version;
+}
+
+void
+fd_device_disable_explicit_sync_heuristic(struct fd_device *dev)
+{
+   dev->disable_explicit_sync_heuristic = true;
+}
+
+DEBUG_GET_ONCE_BOOL_OPTION(libgl, "LIBGL_DEBUG", false)
+
+bool
+fd_dbg(void)
+{
+   return debug_get_option_libgl();
+}
+
+bool
+fd_has_syncobj(struct fd_device *dev)
+{
+   uint64_t value;
+   if (drmGetCap(dev->fd, DRM_CAP_SYNCOBJ, &value))
+      return false;
+   return value && dev->version >= FD_VERSION_FENCE_FD;
+}

src/freedreno/drm/freedreno_drm_perfetto.cc

@@ -0,0 +1,177 @@
+/*
+ * Copyright © 2024 Google, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <perfetto.h>
+
+#include "freedreno_drm_perfetto.h"
+#include "freedreno_drmif.h"
+
+#include "util/log.h"
+#include "util/perf/u_perfetto.h"
+#include "util/simple_mtx.h"
+
+class FdMemoryDataSource : public perfetto::DataSource<FdMemoryDataSource> {
+ public:
+   void OnSetup(const SetupArgs &) override
+   {
+   }
+
+   void OnStart(const StartArgs &) override
+   {
+      PERFETTO_LOG("Memory tracing started");
+   }
+
+   void OnStop(const StopArgs &) override
+   {
+      PERFETTO_LOG("Memory tracing stopped");
+   }
+};
+
+PERFETTO_DECLARE_DATA_SOURCE_STATIC_MEMBERS(FdMemoryDataSource);
+PERFETTO_DEFINE_DATA_SOURCE_STATIC_MEMBERS(FdMemoryDataSource);
+
+bool fd_drm_perfetto_active = false;
+
+extern "C" void
+fd_drm_perfetto_init(void)
+{
+   if (!fd_drm_perfetto_active && util_perfetto_is_tracing_enabled()) {
+      perfetto::DataSourceDescriptor dsd;
+      dsd.set_name("gpu.memory.msm");
+      FdMemoryDataSource::Register(dsd);
+      fd_drm_perfetto_active = true;
+   }
+}
+
+extern "C" void
+fd_alloc_log(struct fd_bo *bo, enum fd_alloc_category from, enum fd_alloc_category to)
+{
+   if (!fd_drm_perfetto_active)
+      return;
+
+   /* Special case for BOs that back heap chunks, they don't immediately
+    * transition to active, despite what the caller thinks:
+    */
+   if (bo->alloc_flags & _FD_BO_HINT_HEAP) {
+      if (to == FD_ALLOC_ACTIVE) {
+         to = FD_ALLOC_HEAP;
+      } else if (from == FD_ALLOC_ACTIVE) {
+         from = FD_ALLOC_HEAP;
+      }
+   }
+
+#define MEMORY_DEBUGGING 0
+   if (MEMORY_DEBUGGING) {
+      static simple_mtx_t lock = SIMPLE_MTX_INITIALIZER;
+
+      assert(bo->size);
+
+      simple_mtx_lock(&lock);
+
+      static uint32_t sizes[4];
+      static uint32_t size_buffer, size_image, size_command, size_internal, *size_cat;
+
+      if (from != FD_ALLOC_NONE) {
+         assert(sizes[from] >= bo->size);
+         sizes[from] -= bo->size;
+      }
+
+      if (to != FD_ALLOC_NONE) {
+         sizes[to] += bo->size;
+      }
+
+      if (bo->alloc_flags & FD_BO_HINT_BUFFER) {
+         size_cat = &size_buffer;
+      } else if (bo->alloc_flags & FD_BO_HINT_IMAGE) {
+         size_cat = &size_image;
+      } else if (bo->alloc_flags & FD_BO_HINT_COMMAND) {
+         size_cat = &size_command;
+      } else {
+         size_cat = &size_internal;
+      }
+      if (to == FD_ALLOC_ACTIVE) {
+         *size_cat += bo->size;
+      } else if (from == FD_ALLOC_ACTIVE) {
+         assert(*size_cat >= bo->size);
+         *size_cat -= bo->size;
+      }
+
+      static time_t last_time;
+      struct timespec time;
+
+      clock_gettime(CLOCK_MONOTONIC, &time);
+
+      if (last_time != time.tv_sec) {
+         mesa_logi("active=%'u, heap=%'u, cache=%'u, buffer=%'u, image=%'u, command=%'u, internal=%'u",
+                   sizes[FD_ALLOC_ACTIVE], sizes[FD_ALLOC_HEAP], sizes[FD_ALLOC_CACHE],
+                   size_buffer, size_image, size_command, size_internal);
+         last_time = time.tv_sec;
+      }
+
+      simple_mtx_unlock(&lock);
+   }
+
+   if ((to != FD_ALLOC_ACTIVE) && (from != FD_ALLOC_ACTIVE))
+      return;
+
+   FdMemoryDataSource::Trace([=](FdMemoryDataSource::TraceContext tctx) {
+      auto packet = tctx.NewTracePacket();
+
+      packet->set_timestamp(perfetto::base::GetBootTimeNs().count());
+
+      auto event = packet->set_vulkan_memory_event();
+
+      event->set_timestamp(perfetto::base::GetBootTimeNs().count());
+      event->set_memory_size(bo->size);
+      event->set_memory_address(bo->iova);
+      event->set_allocation_scope(perfetto::protos::pbzero::perfetto_pbzero_enum_VulkanMemoryEvent::SCOPE_COMMAND);
+      event->set_pid(getpid());
+
+      if (bo->alloc_flags & FD_BO_HINT_BUFFER) {
+         event->set_source(perfetto::protos::pbzero::perfetto_pbzero_enum_VulkanMemoryEvent::SOURCE_BUFFER);
+         event->set_memory_type(1);
+      } else if (bo->alloc_flags & FD_BO_HINT_IMAGE) {
+         event->set_source(perfetto::protos::pbzero::perfetto_pbzero_enum_VulkanMemoryEvent::SOURCE_IMAGE);
+         event->set_memory_type(2);
+      } else {
+         event->set_source(perfetto::protos::pbzero::perfetto_pbzero_enum_VulkanMemoryEvent::SOURCE_DRIVER);
+         event->set_memory_type(3);
+      }
+
+      if (bo->alloc_flags & (FD_BO_HINT_BUFFER | FD_BO_HINT_IMAGE)) {
+         /* For IMAGE/BUFFER, the trace processor is looking for BIND/DESTROY_BOUND: */
+         if (to == FD_ALLOC_ACTIVE) {
+            event->set_operation(perfetto::protos::pbzero::perfetto_pbzero_enum_VulkanMemoryEvent::OP_BIND);
+         } else {
+            event->set_operation(perfetto::protos::pbzero::perfetto_pbzero_enum_VulkanMemoryEvent::OP_DESTROY_BOUND);
+         }
+      } else {
+         /* For SOURCE_DRIVER, the relevant ops are CREATE/DESTROY */
+         if (to == FD_ALLOC_ACTIVE) {
+            event->set_operation(perfetto::protos::pbzero::perfetto_pbzero_enum_VulkanMemoryEvent::OP_CREATE);
+         } else {
+            event->set_operation(perfetto::protos::pbzero::perfetto_pbzero_enum_VulkanMemoryEvent::OP_DESTROY);
+         }
+      }
+   });
+}

src/freedreno/drm/freedreno_drm_perfetto.h

@@ -0,0 +1,46 @@
+/*
+ * Copyright © 2024 Google, Inc.
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef FREEDRENO_DRM_PERFETTO_H_
+#define FREEDRENO_DRM_PERFETTO_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Memory events are logged as transition between allocation categories.
+ * Ie. a new allocation from kernel would be NONE -> ACTIVE, while a
+ * freed buffer going to the BO cache would be ACTIVE -> CACHE, and then
+ * if it is eventually freed from the cache, CACHE -> NONE.
+ */
+enum fd_alloc_category {
+   FD_ALLOC_NONE,       /* freed / not allocated */
+   FD_ALLOC_HEAP,       /* unused bo heap memory */
+   FD_ALLOC_CACHE,      /* unused bo cache memory */
+   FD_ALLOC_ACTIVE,     /* actively used */
+};
+
+struct fd_bo;
+
+#ifdef HAVE_PERFETTO
+void fd_drm_perfetto_init(void);
+void fd_alloc_log(struct fd_bo *bo, enum fd_alloc_category from, enum fd_alloc_category to);
+#else
+static inline void
+fd_drm_perfetto_init(void)
+{
+}
+static inline void
+fd_alloc_log(struct fd_bo *bo, enum fd_alloc_category from, enum fd_alloc_category to)
+{
+}
+#endif
+
+#ifdef __cplusplus
+} /* end of extern "C" */
+#endif
+
+#endif /* FREEDRENO_DRM_PERFETTO_H_ */

src/freedreno/drm/freedreno_drmif.h

@@ -0,0 +1,317 @@
+/*
+ * Copyright © 2012-2018 Rob Clark <[email protected]>
+ * SPDX-License-Identifier: MIT
+ *
+ * Authors:
+ *    Rob Clark <[email protected]>
+ */
+
+#ifndef FREEDRENO_DRMIF_H_
+#define FREEDRENO_DRMIF_H_
+
+#include <stdint.h>
+
+#include "util/bitset.h"
+#include "util/list.h"
+#include "util/u_debug.h"
+#include "util/u_queue.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct fd_bo;
+struct fd_pipe;
+struct fd_device;
+
+enum fd_pipe_id {
+   FD_PIPE_3D = 1,
+   FD_PIPE_2D = 2,
+   /* some devices have two 2d blocks.. not really sure how to
+    * use that yet, so just ignoring the 2nd 2d pipe for now
+    */
+   FD_PIPE_MAX
+};
+
+enum fd_param_id {
+   FD_DEVICE_ID,
+   FD_GMEM_SIZE,
+   FD_GMEM_BASE,     /* 64b */
+   FD_GPU_ID,
+   FD_CHIP_ID,       /* 64b */
+   FD_MAX_FREQ,
+   FD_TIMESTAMP,
+   FD_NR_PRIORITIES,      /* # of rings == # of distinct priority levels */
+   FD_CTX_FAULTS,    /* # of per context faults */
+   FD_GLOBAL_FAULTS, /* # of global (all context) faults */
+   FD_SUSPEND_COUNT, /* # of times the GPU has suspended, and potentially lost state */
+   FD_SYSPROF,       /* Settable (for CAP_SYS_ADMIN) param for system profiling */
+   FD_VA_SIZE,       /* GPU virtual address size */
+   FD_UCHE_TRAP_BASE,
+};
+
+/**
+ * Helper for fence/seqno comparisions which deals properly with rollover.
+ * Returns true if fence 'a' is before fence 'b'
+ */
+static inline bool
+fd_fence_before(uint32_t a, uint32_t b)
+{
+   return (int32_t)(a - b) < 0;
+}
+
+static inline bool
+fd_fence_after(uint32_t a, uint32_t b)
+{
+   return (int32_t)(a - b) > 0;
+}
+
+/**
+ * Encapsulates submit out-fence(s), which consist of a 'timestamp' (per-
+ * pipe (submitqueue) sequence number) and optionally, if requested, an
+ * out-fence-fd
+ *
+ * Per submit, there are actually two fences:
+ *  1) The userspace maintained fence, which is used to optimistically
+ *     avoid kernel ioctls to query if specific rendering is completed
+ *  2) The kernel maintained fence, which we cannot directly do anything
+ *     with, other than pass it back to the kernel
+ *
+ * The userspace fence is mostly internal to the drm layer, but we want
+ * the gallium layer to be able to pass it back to us for things like
+ * fd_pipe_wait().  So this struct encapsulates the two.
+ */
+struct fd_fence {
+   /**
+    * Note refcnt is *not* atomic, but protected by fence_lock, since the
+    * fence_lock is held in fd_bo_add_fence(), which is the hotpath.
+    */
+   int32_t refcnt;
+
+   struct fd_pipe *pipe;
+
+   /**
+    * The ready fence is signaled once the submit is actually flushed down
+    * to the kernel, and fence/fence_fd are populated.  You must wait for
+    * this fence to be signaled before reading fence/fence_fd.
+    */
+   struct util_queue_fence ready;
+
+   uint32_t kfence;     /* kernel fence */
+   uint32_t ufence;     /* userspace fence */
+
+   /**
+    * Optional dma_fence fd, returned by submit if use_fence_fd is true
+    */
+   int fence_fd;
+   bool use_fence_fd;
+};
+
+struct fd_fence *fd_fence_new(struct fd_pipe *pipe, bool use_fence_fd);
+struct fd_fence *fd_fence_ref(struct fd_fence *f);
+struct fd_fence *fd_fence_ref_locked(struct fd_fence *f);
+void fd_fence_del(struct fd_fence *f);
+void fd_fence_del_locked(struct fd_fence *f);
+void fd_fence_flush(struct fd_fence *f);
+int fd_fence_wait(struct fd_fence *f);
+
+/*
+ * bo flags:
+ */
+
+#define FD_BO_CACHED_COHERENT     BITSET_BIT(0) /* Default caching is WRITECOMBINE */
+#define FD_BO_GPUREADONLY         BITSET_BIT(1)
+#define FD_BO_NOMAP               BITSET_BIT(2) /* Hint that the bo will not be mmap'd */
+
+/* Hint that the bo will be exported/shared: */
+#define FD_BO_SHARED              BITSET_BIT(4)
+#define FD_BO_SCANOUT             BITSET_BIT(5)
+
+/* internal bo flags: */
+#define _FD_BO_NOSYNC             BITSET_BIT(7) /* Avoid userspace fencing on control buffers */
+
+/* Additional flags hinting usage, only used for tracing.  Buffers without
+ * one of these flags set will be presumed to be driver internal.
+ */
+#define FD_BO_HINT_BUFFER         BITSET_BIT(8)
+#define FD_BO_HINT_IMAGE          BITSET_BIT(9)
+#define FD_BO_HINT_COMMAND        BITSET_BIT(10)
+#define _FD_BO_HINT_HEAP          BITSET_BIT(11)
+#define _FD_BO_HINTS              ( \
+   FD_BO_HINT_BUFFER | \
+   FD_BO_HINT_IMAGE | \
+   FD_BO_HINT_COMMAND | \
+   _FD_BO_HINT_HEAP | \
+   0)
+
+/*
+ * bo access flags: (keep aligned to MSM_PREP_x)
+ */
+#define FD_BO_PREP_READ   BITSET_BIT(0)
+#define FD_BO_PREP_WRITE  BITSET_BIT(1)
+#define FD_BO_PREP_NOSYNC BITSET_BIT(2)
+#define FD_BO_PREP_FLUSH  BITSET_BIT(3)
+
+
+/* device functions:
+ */
+
+struct fd_device *fd_device_new(int fd);
+struct fd_device *fd_device_new_dup(int fd);
+struct fd_device *fd_device_open(void);
+struct fd_device *fd_device_ref(struct fd_device *dev);
+void fd_device_purge(struct fd_device *dev);
+void fd_device_del(struct fd_device *dev);
+int fd_device_fd(struct fd_device *dev);
+
+enum fd_version {
+   FD_VERSION_MADVISE = 1,             /* kernel supports madvise */
+   FD_VERSION_UNLIMITED_CMDS = 1,      /* submits w/ >4 cmd buffers (growable ringbuffer) */
+   FD_VERSION_FENCE_FD = 2,            /* submit command supports in/out fences */
+   FD_VERSION_GMEM_BASE = 3,           /* supports querying GMEM base address */
+   FD_VERSION_SUBMIT_QUEUES = 3,       /* submit queues and multiple priority levels */
+   FD_VERSION_BO_IOVA = 3,             /* supports fd_bo_get/put_iova() */
+   FD_VERSION_SOFTPIN = 4,             /* adds softpin, bo name, and dump flag */
+   FD_VERSION_ROBUSTNESS = 5,          /* adds FD_NR_FAULTS and FD_PP_PGTABLE */
+   FD_VERSION_MEMORY_FD = 2,           /* supports shared memory objects */
+   FD_VERSION_SUSPENDS = 7,            /* Adds MSM_PARAM_SUSPENDS to detect device suspend */
+   FD_VERSION_CACHED_COHERENT = 8,     /* Adds cached-coherent support (a6xx+) */
+   FD_VERSION_VA_SIZE = 9,
+};
+enum fd_version fd_device_version(struct fd_device *dev);
+void fd_device_disable_explicit_sync_heuristic(struct fd_device *dev);
+
+bool fd_has_syncobj(struct fd_device *dev);
+
+/* pipe functions:
+ */
+
+struct fd_pipe *fd_pipe_new(struct fd_device *dev, enum fd_pipe_id id);
+struct fd_pipe *fd_pipe_new2(struct fd_device *dev, enum fd_pipe_id id,
+                             uint32_t prio);
+struct fd_pipe *fd_pipe_ref(struct fd_pipe *pipe);
+struct fd_pipe *fd_pipe_ref_locked(struct fd_pipe *pipe);
+void fd_pipe_del(struct fd_pipe *pipe);
+void fd_pipe_purge(struct fd_pipe *pipe);
+const struct fd_dev_id * fd_pipe_dev_id(struct fd_pipe *pipe);
+int fd_pipe_get_param(struct fd_pipe *pipe, enum fd_param_id param,
+                      uint64_t *value);
+int fd_pipe_set_param(struct fd_pipe *pipe, enum fd_param_id param,
+                      uint64_t value);
+int fd_pipe_wait(struct fd_pipe *pipe, const struct fd_fence *fence);
+/* timeout in nanosec */
+int fd_pipe_wait_timeout(struct fd_pipe *pipe, const struct fd_fence *fence,
+                         uint64_t timeout);
+
+/* buffer-object functions:
+ */
+
+struct fd_bo {
+   struct fd_device *dev;
+   uint32_t size;
+   uint32_t handle;
+   uint32_t name;
+   int32_t refcnt;
+   uint32_t reloc_flags; /* flags like FD_RELOC_DUMP to use for relocs to this BO */
+   uint32_t alloc_flags; /* flags that control allocation/mapping, ie. FD_BO_x */
+   uint64_t iova;
+   void *map;
+   const struct fd_bo_funcs *funcs;
+
+   enum {
+      NO_CACHE = 0,
+      BO_CACHE = 1,
+      RING_CACHE = 2,
+   } bo_reuse : 2;
+
+   /* Most recent index in submit's bo table, used to optimize the common
+    * case where a bo is used many times in the same submit.
+    */
+   uint32_t idx;
+
+   struct list_head node; /* bucket-list entry */
+   time_t free_time;      /* time when added to bucket-list */
+
+   unsigned short nr_fences, max_fences;
+   struct fd_fence **fences;
+
+   /* In the common case, there is no more than one fence attached.
+    * This provides storage for the fences table until it grows to
+    * be larger than a single element.
+    */
+   struct fd_fence *_inline_fence;
+};
+
+struct fd_bo *_fd_bo_new(struct fd_device *dev, uint32_t size, uint32_t flags);
+void _fd_bo_set_name(struct fd_bo *bo, const char *fmt, va_list ap);
+
+static inline void fd_bo_set_name(struct fd_bo *bo, const char *fmt, ...)
+   _util_printf_format(2, 3);
+
+static inline void
+fd_bo_set_name(struct fd_bo *bo, const char *fmt, ...)
+{
+#ifndef NDEBUG
+   va_list ap;
+   va_start(ap, fmt);
+   _fd_bo_set_name(bo, fmt, ap);
+   va_end(ap);
+#endif
+}
+
+static inline struct fd_bo *fd_bo_new(struct fd_device *dev, uint32_t size,
+                                      uint32_t flags, const char *fmt, ...)
+   _util_printf_format(4, 5);
+
+static inline struct fd_bo *
+fd_bo_new(struct fd_device *dev, uint32_t size, uint32_t flags, const char *fmt,
+          ...)
+{
+   struct fd_bo *bo = _fd_bo_new(dev, size, flags);
+#ifndef NDEBUG
+   if (fmt) {
+      va_list ap;
+      va_start(ap, fmt);
+      _fd_bo_set_name(bo, fmt, ap);
+      va_end(ap);
+   }
+#endif
+   return bo;
+}
+
+struct fd_bo *fd_bo_from_handle(struct fd_device *dev, uint32_t handle,
+                                uint32_t size);
+struct fd_bo *fd_bo_from_name(struct fd_device *dev, uint32_t name);
+struct fd_bo *fd_bo_from_dmabuf(struct fd_device *dev, int fd);
+void fd_bo_mark_for_dump(struct fd_bo *bo);
+
+static inline uint64_t
+fd_bo_get_iova(struct fd_bo *bo)
+{
+   /* ancient kernels did not support this */
+   assert(bo->iova != 0);
+   return bo->iova;
+}
+
+struct fd_bo *fd_bo_ref(struct fd_bo *bo);
+void fd_bo_del(struct fd_bo *bo);
+void fd_bo_del_array(struct fd_bo **bos, int count);
+void fd_bo_del_list_nocache(struct list_head *list);
+int fd_bo_get_name(struct fd_bo *bo, uint32_t *name);
+uint32_t fd_bo_handle(struct fd_bo *bo);
+int fd_bo_dmabuf_drm(struct fd_bo *bo);
+int fd_bo_dmabuf(struct fd_bo *bo);
+uint32_t fd_bo_size(struct fd_bo *bo);
+void *fd_bo_map(struct fd_bo *bo);
+void fd_bo_upload(struct fd_bo *bo, void *src, unsigned off, unsigned len);
+bool fd_bo_prefer_upload(struct fd_bo *bo, unsigned len);
+int fd_bo_cpu_prep(struct fd_bo *bo, struct fd_pipe *pipe, uint32_t op);
+bool fd_bo_is_cached(struct fd_bo *bo);
+void fd_bo_set_metadata(struct fd_bo *bo, void *metadata, uint32_t metadata_size);
+int fd_bo_get_metadata(struct fd_bo *bo, void *metadata, uint32_t metadata_size);
+
+#ifdef __cplusplus
+} /* end of extern "C" */
+#endif
+
+#endif /* FREEDRENO_DRMIF_H_ */

src/freedreno/drm/freedreno_pipe.c

@@ -0,0 +1,294 @@
+/*
+ * Copyright © 2012-2018 Rob Clark <[email protected]>
+ * SPDX-License-Identifier: MIT
+ *
+ * Authors:
+ *    Rob Clark <[email protected]>
+ */
+
+#include "freedreno_drmif.h"
+#include "freedreno_priv.h"
+
+/**
+ * priority of zero is highest priority, and higher numeric values are
+ * lower priorities
+ */
+struct fd_pipe *
+fd_pipe_new2(struct fd_device *dev, enum fd_pipe_id id, uint32_t prio)
+{
+   struct fd_pipe *pipe;
+   uint64_t val;
+
+   if (id > FD_PIPE_MAX) {
+      ERROR_MSG("invalid pipe id: %d", id);
+      return NULL;
+   }
+
+   if ((prio != 1) && (fd_device_version(dev) < FD_VERSION_SUBMIT_QUEUES)) {
+      ERROR_MSG("invalid priority!");
+      return NULL;
+   }
+
+   pipe = dev->funcs->pipe_new(dev, id, prio);
+   if (!pipe) {
+      ERROR_MSG("allocation failed");
+      return NULL;
+   }
+
+   pipe->dev = dev;
+   pipe->id = id;
+   p_atomic_set(&pipe->refcnt, 1);
+
+   fd_pipe_get_param(pipe, FD_GPU_ID, &val);
+   pipe->dev_id.gpu_id = val;
+
+   fd_pipe_get_param(pipe, FD_CHIP_ID, &val);
+   pipe->dev_id.chip_id = val;
+
+   if (!fd_dev_is_supported(&pipe->dev_id)) {
+      ERROR_MSG("unsupported GPU id 0x%" PRIx32 " / chip id 0x%" PRIx64,
+                pipe->dev_id.gpu_id,
+                pipe->dev_id.chip_id);
+      return NULL;
+   }
+
+   pipe->gen = fd_dev_gen(&pipe->dev_id);
+   pipe->is_64bit = fd_dev_64b(&pipe->dev_id);
+
+   /* Use the _NOSYNC flags because we don't want the control_mem bo to hold
+    * a reference to the ourself.  This also means that we won't be able
+    * to determine if the buffer is idle which is needed by bo-cache.  But
+    * pipe creation/destroy is not a high frequency event.
+    */
+   pipe->control_mem = fd_bo_new(dev, sizeof(*pipe->control),
+                                 FD_BO_CACHED_COHERENT | _FD_BO_NOSYNC,
+                                 "pipe-control");
+   pipe->control = fd_bo_map(pipe->control_mem);
+
+   /* We could be getting a bo from the bo-cache, make sure the fence value
+    * is not garbage:
+    */
+   pipe->control->fence = 0;
+   pipe->control_mem->bo_reuse = NO_CACHE;
+
+   return pipe;
+}
+
+struct fd_pipe *
+fd_pipe_new(struct fd_device *dev, enum fd_pipe_id id)
+{
+   return fd_pipe_new2(dev, id, 1);
+}
+
+struct fd_pipe *
+fd_pipe_ref(struct fd_pipe *pipe)
+{
+   simple_mtx_lock(&fence_lock);
+   fd_pipe_ref_locked(pipe);
+   simple_mtx_unlock(&fence_lock);
+   return pipe;
+}
+
+struct fd_pipe *
+fd_pipe_ref_locked(struct fd_pipe *pipe)
+{
+   simple_mtx_assert_locked(&fence_lock);
+   pipe->refcnt++;
+   return pipe;
+}
+
+void
+fd_pipe_del(struct fd_pipe *pipe)
+{
+   simple_mtx_lock(&fence_lock);
+   fd_pipe_del_locked(pipe);
+   simple_mtx_unlock(&fence_lock);
+}
+
+void
+fd_pipe_del_locked(struct fd_pipe *pipe)
+{
+   simple_mtx_assert_locked(&fence_lock);
+   if (--pipe->refcnt)
+      return;
+
+   fd_bo_del(pipe->control_mem);
+   pipe->funcs->destroy(pipe);
+}
+
+/**
+ * Flush any unflushed deferred submits.  This is called at context-
+ * destroy to make sure we don't leak unflushed submits.
+ */
+void
+fd_pipe_purge(struct fd_pipe *pipe)
+{
+   struct fd_device *dev = pipe->dev;
+   struct fd_fence *unflushed_fence = NULL;
+
+   simple_mtx_lock(&dev->submit_lock);
+
+   /* We only queue up deferred submits for a single pipe at a time, so
+    * if there is a deferred_submits_fence on the same pipe as us, we
+    * know we have deferred_submits queued, which need to be flushed:
+    */
+   if (dev->deferred_submits_fence && dev->deferred_submits_fence->pipe == pipe) {
+      unflushed_fence = fd_fence_ref(dev->deferred_submits_fence);
+   }
+
+   simple_mtx_unlock(&dev->submit_lock);
+
+   if (unflushed_fence) {
+      fd_fence_flush(unflushed_fence);
+      fd_fence_del(unflushed_fence);
+   }
+
+   if (pipe->funcs->finish)
+      pipe->funcs->finish(pipe);
+}
+
+int
+fd_pipe_get_param(struct fd_pipe *pipe, enum fd_param_id param, uint64_t *value)
+{
+   return pipe->funcs->get_param(pipe, param, value);
+}
+
+int
+fd_pipe_set_param(struct fd_pipe *pipe, enum fd_param_id param, uint64_t value)
+{
+   return pipe->funcs->set_param(pipe, param, value);
+}
+
+const struct fd_dev_id *
+fd_pipe_dev_id(struct fd_pipe *pipe)
+{
+   return &pipe->dev_id;
+}
+
+int
+fd_pipe_wait(struct fd_pipe *pipe, const struct fd_fence *fence)
+{
+   return fd_pipe_wait_timeout(pipe, fence, ~0);
+}
+
+int
+fd_pipe_wait_timeout(struct fd_pipe *pipe, const struct fd_fence *fence,
+                     uint64_t timeout)
+{
+   if (!fd_fence_after(fence->ufence, pipe->control->fence))
+      return 0;
+
+   if (!timeout)
+      return -ETIMEDOUT;
+
+   fd_pipe_flush(pipe, fence->ufence);
+
+   return pipe->funcs->wait(pipe, fence, timeout);
+}
+
+uint32_t
+fd_pipe_emit_fence(struct fd_pipe *pipe, struct fd_ringbuffer *ring)
+{
+   uint32_t fence = ++pipe->last_fence;
+   unsigned gen = pipe->gen;
+
+   if (gen >= A7XX) {
+      OUT_PKT7(ring, CP_EVENT_WRITE7, 4);
+      OUT_RING(ring, CP_EVENT_WRITE_0_EVENT(CACHE_FLUSH_TS) |
+               CP_EVENT_WRITE7_0_WRITE_SRC(EV_WRITE_USER_32B) |
+               CP_EVENT_WRITE7_0_WRITE_DST(EV_DST_RAM) |
+               CP_EVENT_WRITE7_0_WRITE_ENABLED);
+      OUT_RELOC(ring, control_ptr(pipe, fence));   /* ADDR_LO/HI */
+      OUT_RING(ring, fence);
+   } else if (gen >= A5XX) {
+      OUT_PKT7(ring, CP_EVENT_WRITE, 4);
+      OUT_RING(ring, CP_EVENT_WRITE_0_EVENT(CACHE_FLUSH_TS));
+      OUT_RELOC(ring, control_ptr(pipe, fence));   /* ADDR_LO/HI */
+      OUT_RING(ring, fence);
+   } else {
+      OUT_PKT3(ring, CP_EVENT_WRITE, 3);
+      OUT_RING(ring, CP_EVENT_WRITE_0_EVENT(CACHE_FLUSH_TS));
+      OUT_RELOC(ring, control_ptr(pipe, fence));   /* ADDR */
+      OUT_RING(ring, fence);
+   }
+
+   return fence;
+}
+
+struct fd_fence *
+fd_fence_new(struct fd_pipe *pipe, bool use_fence_fd)
+{
+   struct fd_fence *f = calloc(1, sizeof(*f));
+
+   f->refcnt = 1;
+   f->pipe = fd_pipe_ref(pipe);
+   util_queue_fence_init(&f->ready);
+   f->use_fence_fd = use_fence_fd;
+   f->fence_fd = -1;
+
+   return f;
+}
+
+struct fd_fence *
+fd_fence_ref(struct fd_fence *f)
+{
+   simple_mtx_lock(&fence_lock);
+   fd_fence_ref_locked(f);
+   simple_mtx_unlock(&fence_lock);
+
+   return f;
+}
+
+struct fd_fence *
+fd_fence_ref_locked(struct fd_fence *f)
+{
+   simple_mtx_assert_locked(&fence_lock);
+   f->refcnt++;
+   return f;
+}
+
+void
+fd_fence_del(struct fd_fence *f)
+{
+   simple_mtx_lock(&fence_lock);
+   fd_fence_del_locked(f);
+   simple_mtx_unlock(&fence_lock);
+}
+
+void
+fd_fence_del_locked(struct fd_fence *f)
+{
+   simple_mtx_assert_locked(&fence_lock);
+
+   if (--f->refcnt)
+      return;
+
+   fd_pipe_del_locked(f->pipe);
+
+   if (f->use_fence_fd && (f->fence_fd != -1))
+      close(f->fence_fd);
+
+   free(f);
+}
+
+/**
+ * Wait until corresponding submit is flushed to kernel
+ */
+void
+fd_fence_flush(struct fd_fence *f)
+{
+   MESA_TRACE_FUNC();
+   /*
+    * TODO we could simplify this to remove the flush_sync part of
+    * fd_pipe_sp_flush() and just rely on the util_queue_fence_wait()
+    */
+   fd_pipe_flush(f->pipe, f->ufence);
+   util_queue_fence_wait(&f->ready);
+}
+
+int
+fd_fence_wait(struct fd_fence *f)
+{
+   MESA_TRACE_FUNC();
+   return fd_pipe_wait(f->pipe, f);
+}

src/freedreno/drm/freedreno_priv.h

@@ -0,0 +1,586 @@
+/*
+ * Copyright © 2012-2018 Rob Clark <[email protected]>
+ * SPDX-License-Identifier: MIT
+ *
+ * Authors:
+ *    Rob Clark <[email protected]>
+ */
+
+#ifndef FREEDRENO_PRIV_H_
+#define FREEDRENO_PRIV_H_
+
+#include <errno.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+
+#include <xf86drm.h>
+
+#include "util/hash_table.h"
+#include "util/list.h"
+#include "util/log.h"
+#include "util/perf/cpu_trace.h"
+#include "util/simple_mtx.h"
+#include "util/slab.h"
+#include "util/u_atomic.h"
+#include "util/u_debug.h"
+#include "util/u_math.h"
+#include "util/vma.h"
+
+#include "freedreno_common.h"
+#include "freedreno_dev_info.h"
+#include "freedreno_drmif.h"
+#include "freedreno_rd_output.h"
+#include "freedreno_ringbuffer.h"
+
+extern simple_mtx_t table_lock;
+extern simple_mtx_t fence_lock;
+extern uint64_t os_page_size;
+
+#define SUBALLOC_SIZE (32 * 1024)
+/* Maximum known alignment requirement is a6xx's TEX_CONST at 16 dwords */
+#define SUBALLOC_ALIGNMENT 64
+#define RING_FLAGS (FD_BO_GPUREADONLY | FD_BO_CACHED_COHERENT | FD_BO_HINT_COMMAND)
+
+/*
+ * Stupid/simple growable array implementation:
+ */
+
+#define MAX_ARRAY_SIZE ((unsigned short)~0)
+
+static inline void
+grow(void **ptr, uint16_t nr, uint16_t *max, uint16_t sz)
+{
+   assert((nr + 1) < MAX_ARRAY_SIZE);
+   if ((nr + 1) > *max) {
+      if (*max > MAX_ARRAY_SIZE/2)
+         *max = MAX_ARRAY_SIZE;
+      else if ((*max * 2) < (nr + 1))
+         *max = nr + 5;
+      else
+         *max = *max * 2;
+      *ptr = realloc(*ptr, *max * sz);
+   }
+}
+
+#define DECLARE_ARRAY(type, name)                                              \
+   unsigned short nr_##name, max_##name;                                       \
+   type *name;
+
+#define APPEND(x, name, ...)                                                   \
+   ({                                                                          \
+      grow((void **)&(x)->name, (x)->nr_##name, &(x)->max_##name,              \
+           sizeof((x)->name[0]));                                              \
+      (x)->name[(x)->nr_##name] = __VA_ARGS__;                                 \
+      (x)->nr_##name++;                                                        \
+   })
+
+#define READ_ONCE(x) (*(volatile __typeof__(x) *)&(x))
+
+
+struct fd_device_funcs {
+   /* Create a new buffer object:
+    */
+   struct fd_bo *(*bo_new)(struct fd_device *dev, uint32_t size, uint32_t flags);
+
+   /* Create a new buffer object from existing handle (ie. dma-buf or
+    * flink import):
+    */
+   struct fd_bo *(*bo_from_handle)(struct fd_device *dev, uint32_t size,
+                                   uint32_t handle);
+   uint32_t (*handle_from_dmabuf)(struct fd_device *dev, int fd);
+   struct fd_bo *(*bo_from_dmabuf)(struct fd_device *dev, int fd);
+   void (*bo_close_handle)(struct fd_bo *bo);
+
+   struct fd_pipe *(*pipe_new)(struct fd_device *dev, enum fd_pipe_id id,
+                               unsigned prio);
+   int (*flush)(struct fd_device *dev);
+   void (*destroy)(struct fd_device *dev);
+};
+
+struct fd_bo_bucket {
+   uint32_t size;
+   int count, hits, misses, expired;
+   struct list_head list;
+};
+
+struct fd_bo_cache {
+   const char *name;
+   simple_mtx_t lock;
+   struct fd_bo_bucket cache_bucket[14 * 4];
+   int num_buckets;
+   time_t time;
+};
+
+/* Probably good for the block size to be a multiple of an available
+ * large-page size.  For overlap of what both the MMU (with 4kb granule)
+ * and SMMU support, 2MB is that overlap.  (Well, 4kb is as well, but
+ * too small to be practical ;-))
+ */
+#define FD_BO_HEAP_BLOCK_SIZE (4 * 1024 * 1024)
+
+/* Zero is an invalid handle, use it to indicate buffers that have been sub-
+ * allocated from a larger backing heap block buffer.
+ */
+#define FD_BO_SUBALLOC_HANDLE 0
+
+static inline bool
+suballoc_bo(struct fd_bo *bo)
+{
+   return bo->handle == FD_BO_SUBALLOC_HANDLE;
+}
+
+/**
+ * A heap is a virtual range of memory that is backed by N physical buffers,
+ * from which buffers can be suballocated.  This requires kernel support for
+ * userspace allocated iova.
+ */
+struct fd_bo_heap {
+   struct fd_device *dev;
+
+   int cnt;
+
+   /**
+    * Buffer allocation flags for buffers allocated from this heap.
+    */
+   uint32_t flags;
+
+   simple_mtx_t lock;
+
+   /**
+    * Ranges of the backing buffer are allocated at a granularity of
+    * SUBALLOC_ALIGNMENT
+    */
+   struct util_vma_heap heap;
+
+   /**
+    * List of recently freed suballocated BOs from this allocator until they
+    * become idle.  Backend should periodically call fd_bo_suballoc_clean()
+    * to check for newly idle entries on the freelist, so that the memory can
+    * be returned to the free heap.
+    */
+   struct list_head freelist;
+
+   /**
+    * The backing buffers.  Maximum total heap size is:
+    *   FD_BO_HEAP_BLOCK_SIZE * ARRAY_SIZE(heap->blocks)
+    */
+   struct fd_bo *blocks[256];
+};
+
+struct fd_bo_heap *fd_bo_heap_new(struct fd_device *dev, uint32_t flags);
+void fd_bo_heap_destroy(struct fd_bo_heap *heap);
+
+struct fd_bo *fd_bo_heap_block(struct fd_bo *bo);
+struct fd_bo *fd_bo_heap_alloc(struct fd_bo_heap *heap, uint32_t size, uint32_t flags);
+
+static inline uint32_t
+submit_offset(struct fd_bo *bo, uint32_t offset)
+{
+   if (suballoc_bo(bo)) {
+      offset += bo->iova - fd_bo_heap_block(bo)->iova;
+   }
+   return offset;
+}
+
+struct fd_device {
+   int fd;
+   enum fd_version version;
+   int32_t refcnt;
+
+   /* tables to keep track of bo's, to avoid "evil-twin" fd_bo objects:
+    *
+    *   handle_table: maps handle to fd_bo
+    *   name_table: maps flink name to fd_bo
+    *
+    * We end up needing two tables, because DRM_IOCTL_GEM_OPEN always
+    * returns a new handle.  So we need to figure out if the bo is already
+    * open in the process first, before calling gem-open.
+    */
+   struct hash_table *handle_table, *name_table;
+
+   const struct fd_device_funcs *funcs;
+
+   struct fd_bo_cache bo_cache;
+   struct fd_bo_cache ring_cache;
+
+   /**
+    * Heap for mappable + cached-coherent + gpu-readonly (ie. cmdstream)
+    */
+   struct fd_bo_heap *ring_heap;
+
+   /**
+    * Heap for mappable (ie. majority of small buffer allocations, etc)
+    */
+   struct fd_bo_heap *default_heap;
+
+   bool has_cached_coherent;
+
+   bool closefd; /* call close(fd) upon destruction */
+
+   bool disable_explicit_sync_heuristic;
+
+   /* just for valgrind: */
+   int bo_size;
+
+   /**
+    * List of deferred submits, protected by submit_lock.  The deferred
+    * submits are tracked globally per-device, even if they execute in
+    * different order on the kernel side (ie. due to different priority
+    * submitqueues, etc) to preserve the order that they are passed off
+    * to the kernel.  Once the kernel has them, it is the fences' job
+    * to preserve correct order of execution.
+    */
+   struct list_head deferred_submits;
+   struct fd_fence *deferred_submits_fence;
+   unsigned deferred_cmds;
+   simple_mtx_t submit_lock;
+
+   /**
+    * BO for suballocating long-lived state objects.
+    *
+    * Note: one would be tempted to put this in fd_pipe to avoid locking.
+    * But that is a bad idea for a couple of reasons:
+    *
+    *  1) With TC, stateobj allocation can happen in either frontend thread
+    *     (ie. most CSOs), and also driver thread (a6xx cached tex state)
+    *  2) It is best for fd_pipe to not hold a reference to a BO that can
+    *     be free'd to bo cache, as that can cause unexpected re-entrancy
+    *     (fd_bo_cache_alloc() -> find_in_bucket() -> fd_bo_state() ->
+    *     cleanup_fences() -> drop pipe ref which free's bo's).
+    */
+   struct fd_bo *suballoc_bo;
+   uint32_t suballoc_offset;
+   simple_mtx_t suballoc_lock;
+
+   struct util_queue submit_queue;
+
+   struct fd_rd_output rd;
+};
+
+static inline bool
+fd_device_threaded_submit(struct fd_device *dev)
+{
+   return util_queue_is_initialized(&dev->submit_queue);
+}
+
+#define foreach_submit(name, list) \
+   list_for_each_entry(struct fd_submit, name, list, node)
+#define foreach_submit_safe(name, list) \
+   list_for_each_entry_safe(struct fd_submit, name, list, node)
+#define last_submit(list) \
+   list_last_entry(list, struct fd_submit, node)
+
+#define foreach_bo(name, list) \
+   list_for_each_entry(struct fd_bo, name, list, node)
+#define foreach_bo_safe(name, list) \
+   list_for_each_entry_safe(struct fd_bo, name, list, node)
+#define first_bo(list) \
+   list_first_entry(list, struct fd_bo, node)
+
+
+void fd_bo_cache_init(struct fd_bo_cache *cache, int coarse, const char *name);
+void fd_bo_cache_cleanup(struct fd_bo_cache *cache, time_t time);
+struct fd_bo *fd_bo_cache_alloc(struct fd_bo_cache *cache, uint32_t *size,
+                                uint32_t flags);
+int fd_bo_cache_free(struct fd_bo_cache *cache, struct fd_bo *bo);
+
+/* for where @fence_lock is already held: */
+void fd_pipe_del_locked(struct fd_pipe *pipe);
+
+struct fd_pipe_funcs {
+   struct fd_ringbuffer *(*ringbuffer_new_object)(struct fd_pipe *pipe,
+                                                  uint32_t size);
+   struct fd_submit *(*submit_new)(struct fd_pipe *pipe);
+
+   /**
+    * Flush any deferred submits (if deferred submits are supported by
+    * the pipe implementation)
+    */
+   void (*flush)(struct fd_pipe *pipe, uint32_t fence);
+   void (*finish)(struct fd_pipe *pipe);
+
+   int (*get_param)(struct fd_pipe *pipe, enum fd_param_id param,
+                    uint64_t *value);
+   int (*set_param)(struct fd_pipe *pipe, enum fd_param_id param,
+                    uint64_t value);
+   int (*wait)(struct fd_pipe *pipe, const struct fd_fence *fence,
+               uint64_t timeout);
+   void (*destroy)(struct fd_pipe *pipe);
+};
+
+struct fd_pipe_control {
+   uint32_t fence;
+};
+#define control_ptr(pipe, member) \
+   (pipe)->control_mem, offsetof(struct fd_pipe_control, member), 0, 0
+
+struct fd_pipe {
+   struct fd_device *dev;
+   enum fd_pipe_id id;
+   struct fd_dev_id dev_id;
+   uint8_t gen;
+
+   /**
+    * Note refcnt is *not* atomic, but protected by fence_lock, since the
+    * fence_lock is held in fd_bo_add_fence(), which is the hotpath.
+    */
+   int32_t refcnt;
+
+   /**
+    * Previous fence seqno allocated for this pipe.  The fd_pipe represents
+    * a single timeline, fences allocated by this pipe can be compared to
+    * each other, but fences from different pipes are not comparable (as
+    * there could be preemption of multiple priority level submitqueues at
+    * play)
+    */
+   uint32_t last_fence;
+
+   /**
+    * The last fence seqno that was flushed to kernel (doesn't mean that it
+    * is complete, just that the kernel knows about it)
+    */
+   uint32_t last_submit_fence;
+
+   uint32_t last_enqueue_fence;   /* just for debugging */
+
+   /**
+    * Counter for assigning each submit a unique seqno.
+    */
+   seqno_t submit_seqno;
+
+   /**
+    * If we *ever* see an in-fence-fd, assume that userspace is
+    * not relying on implicit fences.
+    */
+   bool no_implicit_sync;
+
+   bool is_64bit;
+
+   struct fd_bo *control_mem;
+   volatile struct fd_pipe_control *control;
+
+   struct slab_parent_pool ring_pool;
+
+   const struct fd_pipe_funcs *funcs;
+};
+
+uint32_t fd_pipe_emit_fence(struct fd_pipe *pipe, struct fd_ringbuffer *ring);
+
+static inline void
+fd_pipe_flush(struct fd_pipe *pipe, uint32_t fence)
+{
+   if (!pipe->funcs->flush)
+      return;
+   pipe->funcs->flush(pipe, fence);
+}
+
+struct fd_submit_funcs {
+   struct fd_ringbuffer *(*new_ringbuffer)(struct fd_submit *submit,
+                                           uint32_t size,
+                                           enum fd_ringbuffer_flags flags);
+   struct fd_fence *(*flush)(struct fd_submit *submit, int in_fence_fd,
+                             bool use_fence_fd);
+   void (*destroy)(struct fd_submit *submit);
+};
+
+struct fd_submit {
+   int32_t refcnt;
+   struct fd_pipe *pipe;
+   struct fd_device *dev;
+   const struct fd_submit_funcs *funcs;
+
+   struct fd_ringbuffer *primary;
+   uint32_t fence;
+   struct list_head node;  /* node in fd_pipe::deferred_submits */
+};
+
+static inline unsigned
+fd_dev_count_deferred_cmds(struct fd_device *dev)
+{
+   unsigned nr = 0;
+
+   simple_mtx_assert_locked(&dev->submit_lock);
+
+   list_for_each_entry (struct fd_submit, submit, &dev->deferred_submits, node) {
+      nr += fd_ringbuffer_cmd_count(submit->primary);
+   }
+
+   return nr;
+}
+
+struct fd_bo_funcs {
+   int (*offset)(struct fd_bo *bo, uint64_t *offset);
+   void *(*map)(struct fd_bo *bo);
+   int (*cpu_prep)(struct fd_bo *bo, struct fd_pipe *pipe, uint32_t op);
+   int (*madvise)(struct fd_bo *bo, int willneed);
+   uint64_t (*iova)(struct fd_bo *bo);
+   void (*set_name)(struct fd_bo *bo, const char *fmt, va_list ap);
+   int (*dmabuf)(struct fd_bo *bo);
+
+   /**
+    * Optional hook that is called before ->destroy().  In the case of
+    * batch deletes (such as BO cache cleanup or cleaning up a submit)
+    * the ->finalize() hook will be called for all of the BOs being
+    * destroyed followed by dev->flush() and then bo->destroy().  This
+    * allows the backend to batch up processing.  (Ie. this is for
+    * virtio backend to batch ccmds to the host)
+    *
+    * In all cases, dev->flush() will happen after bo->finalize() and
+    * bo->destroy().
+    */
+   void (*finalize)(struct fd_bo *bo);
+   void (*destroy)(struct fd_bo *bo);
+
+   /**
+    * Optional, copy data into bo, falls back to mmap+memcpy.  If not
+    * implemented, it must be possible to mmap all buffers
+    */
+   void (*upload)(struct fd_bo *bo, void *src, unsigned off, unsigned len);
+
+   /**
+    * Optional, if upload is supported, should upload be preferred?
+    */
+   bool (*prefer_upload)(struct fd_bo *bo, unsigned len);
+
+   void (*set_metadata)(struct fd_bo *bo, void *metadata, uint32_t metadata_size);
+   int (*get_metadata)(struct fd_bo *bo, void *metadata, uint32_t metadata_size);
+};
+
+void fd_bo_add_fence(struct fd_bo *bo, struct fd_fence *fence);
+void *fd_bo_map_os_mmap(struct fd_bo *bo);
+void *__fd_bo_map(struct fd_bo *bo);
+
+enum fd_bo_state {
+   FD_BO_STATE_IDLE,
+   FD_BO_STATE_BUSY,
+   FD_BO_STATE_UNKNOWN,
+};
+enum fd_bo_state fd_bo_state(struct fd_bo *bo);
+
+void fd_bo_init_common(struct fd_bo *bo, struct fd_device *dev);
+void fd_bo_fini_fences(struct fd_bo *bo);
+void fd_bo_fini_common(struct fd_bo *bo);
+
+struct fd_bo *fd_bo_new_ring(struct fd_device *dev, uint32_t size);
+
+uint32_t fd_handle_from_dmabuf_drm(struct fd_device *dev, int fd);
+struct fd_bo *fd_bo_from_dmabuf_drm(struct fd_device *dev, int fd);
+void fd_bo_close_handle_drm(struct fd_bo *bo);
+
+#define enable_debug 0 /* TODO make dynamic */
+
+bool fd_dbg(void);
+
+#define INFO_MSG(fmt, ...)                                                     \
+   do {                                                                        \
+      if (fd_dbg())                                                            \
+         mesa_logi("%s:%d: " fmt, __func__, __LINE__, ##__VA_ARGS__);          \
+   } while (0)
+#define DEBUG_MSG(fmt, ...)                                                    \
+   do                                                                          \
+      if (enable_debug) {                                                      \
+         mesa_logd("%s:%d: " fmt, __func__, __LINE__, ##__VA_ARGS__);          \
+      }                                                                        \
+   while (0)
+#define WARN_MSG(fmt, ...)                                                     \
+   do {                                                                        \
+      mesa_logw("%s:%d: " fmt, __func__, __LINE__, ##__VA_ARGS__);             \
+   } while (0)
+#define ERROR_MSG(fmt, ...)                                                    \
+   do {                                                                        \
+      mesa_loge("%s:%d: " fmt, __func__, __LINE__, ##__VA_ARGS__);             \
+   } while (0)
+
+#define U642VOID(x) ((void *)(unsigned long)(x))
+#define VOID2U64(x) ((uint64_t)(unsigned long)(x))
+
+#ifdef HAVE_VALGRIND
+#include <memcheck.h>
+
+/*
+ * For tracking the backing memory (if valgrind enabled, we force a mmap
+ * for the purposes of tracking)
+ */
+static inline void
+VG_BO_ALLOC(struct fd_bo *bo)
+{
+   if (bo && RUNNING_ON_VALGRIND) {
+      VALGRIND_MALLOCLIKE_BLOCK(fd_bo_map(bo), bo->size, 0, 1);
+   }
+}
+
+static inline void
+VG_BO_FREE(struct fd_bo *bo)
+{
+   VALGRIND_FREELIKE_BLOCK(bo->map, 0);
+}
+
+/*
+ * For tracking bo structs that are in the buffer-cache, so that valgrind
+ * doesn't attribute ownership to the first one to allocate the recycled
+ * bo.
+ *
+ * Note that the list_head in fd_bo is used to track the buffers in cache
+ * so disable error reporting on the range while they are in cache so
+ * valgrind doesn't squawk about list traversal.
+ *
+ */
+static inline void
+VG_BO_RELEASE(struct fd_bo *bo)
+{
+   if (RUNNING_ON_VALGRIND) {
+      VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(bo, bo->dev->bo_size);
+      VALGRIND_MAKE_MEM_NOACCESS(bo, bo->dev->bo_size);
+      VALGRIND_FREELIKE_BLOCK(bo->map, 0);
+   }
+}
+static inline void
+VG_BO_OBTAIN(struct fd_bo *bo)
+{
+   if (RUNNING_ON_VALGRIND) {
+      VALGRIND_MAKE_MEM_DEFINED(bo, bo->dev->bo_size);
+      VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(bo, bo->dev->bo_size);
+      VALGRIND_MALLOCLIKE_BLOCK(bo->map, bo->size, 0, 1);
+   }
+}
+/* special case for fd_bo_upload */
+static inline void
+VG_BO_MAPPED(struct fd_bo *bo)
+{
+   VALGRIND_MALLOCLIKE_BLOCK(bo->map, bo->size, 0, 1);
+}
+#else
+static inline void
+VG_BO_ALLOC(struct fd_bo *bo)
+{
+}
+static inline void
+VG_BO_FREE(struct fd_bo *bo)
+{
+}
+static inline void
+VG_BO_RELEASE(struct fd_bo *bo)
+{
+}
+static inline void
+VG_BO_OBTAIN(struct fd_bo *bo)
+{
+}
+static inline void
+VG_BO_MAPPED(struct fd_bo *bo)
+{
+}
+#endif
+
+#define FD_DEFINE_CAST(parent, child)                                          \
+   static inline struct child *to_##child(struct parent *x)                    \
+   {                                                                           \
+      return (struct child *)x;                                                \
+   }
+
+#endif /* FREEDRENO_PRIV_H_ */

src/freedreno/drm/freedreno_ringbuffer.c

@@ -0,0 +1,89 @@
+/*
+ * Copyright © 2012-2018 Rob Clark <[email protected]>
+ * SPDX-License-Identifier: MIT
+ *
+ * Authors:
+ *    Rob Clark <[email protected]>
+ */
+
+#include <assert.h>
+
+#include "freedreno_drmif.h"
+#include "freedreno_priv.h"
+#include "freedreno_ringbuffer.h"
+
+struct fd_submit *
+fd_submit_new(struct fd_pipe *pipe)
+{
+   struct fd_submit *submit = pipe->funcs->submit_new(pipe);
+   submit->refcnt = 1;
+   submit->pipe = fd_pipe_ref(pipe);
+   submit->dev = fd_device_ref(pipe->dev);
+   return submit;
+}
+
+void
+fd_submit_del(struct fd_submit *submit)
+{
+   if (!unref(&submit->refcnt))
+      return;
+
+   if (submit->primary)
+      fd_ringbuffer_del(submit->primary);
+
+   struct fd_pipe *pipe = submit->pipe;
+   struct fd_device *dev = submit->dev;
+
+   submit->funcs->destroy(submit);
+
+   fd_pipe_del(pipe);
+   fd_device_del(dev);
+}
+
+struct fd_submit *
+fd_submit_ref(struct fd_submit *submit)
+{
+   ref(&submit->refcnt);
+   return submit;
+}
+
+struct fd_fence *
+fd_submit_flush(struct fd_submit *submit, int in_fence_fd, bool use_fence_fd)
+{
+   submit->fence = fd_pipe_emit_fence(submit->pipe, submit->primary);
+   return submit->funcs->flush(submit, in_fence_fd, use_fence_fd);
+}
+
+struct fd_ringbuffer *
+fd_submit_new_ringbuffer(struct fd_submit *submit, uint32_t size,
+                         enum fd_ringbuffer_flags flags)
+{
+   assert(!(flags & _FD_RINGBUFFER_OBJECT));
+   if (flags & FD_RINGBUFFER_STREAMING) {
+      assert(!(flags & FD_RINGBUFFER_GROWABLE));
+      assert(!(flags & FD_RINGBUFFER_PRIMARY));
+   }
+   struct fd_ringbuffer *ring =
+         submit->funcs->new_ringbuffer(submit, size, flags);
+
+   if (flags & FD_RINGBUFFER_PRIMARY) {
+      assert(!submit->primary);
+      submit->primary = fd_ringbuffer_ref(ring);
+   }
+
+   return ring;
+}
+
+struct fd_ringbuffer *
+fd_ringbuffer_new_object(struct fd_pipe *pipe, uint32_t size)
+{
+   return pipe->funcs->ringbuffer_new_object(pipe, size);
+}
+
+struct fd_ringbuffer *
+fd_ringbuffer_new_shareable_object(struct fd_pipe *pipe, uint32_t size)
+{
+   struct fd_ringbuffer *ring = fd_ringbuffer_new_object(pipe, size);
+   ring->flags |= FD_RINGBUFFER_SHAREABLE;
+   return ring;
+}

src/freedreno/drm/freedreno_ringbuffer.h

@@ -0,0 +1,420 @@
+/*
+ * Copyright © 2012-2018 Rob Clark <[email protected]>
+ * SPDX-License-Identifier: MIT
+ *
+ * Authors:
+ *    Rob Clark <[email protected]>
+ */
+
+#ifndef FREEDRENO_RINGBUFFER_H_
+#define FREEDRENO_RINGBUFFER_H_
+
+#include <stdio.h>
+#include "util/u_atomic.h"
+#include "util/u_debug.h"
+
+#include "common/fd_hw_common.h"
+#include "freedreno_drmif.h"
+#include "freedreno_pm4.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct fd_submit;
+struct fd_ringbuffer;
+
+enum fd_ringbuffer_flags {
+
+   /* Primary ringbuffer for a submit, ie. an IB1 level rb
+    * which kernel must setup RB->IB1 CP_INDIRECT_BRANCH
+    * packets.
+    */
+   FD_RINGBUFFER_PRIMARY = 0x1,
+
+   /* Hint that the stateobj will be used for streaming state
+    * that is used once or a few times and then discarded.
+    *
+    * For sub-allocation, non streaming stateobj's should be
+    * sub-allocated from a page size buffer, so one long lived
+    * state obj doesn't prevent other pages from being freed.
+    * (Ie. it would be no worse than allocating a page sized
+    * bo for each small non-streaming stateobj).
+    *
+    * But streaming stateobj's could be sub-allocated from a
+    * larger buffer to reduce the alloc/del overhead.
+    */
+   FD_RINGBUFFER_STREAMING = 0x2,
+
+   /* Indicates that "growable" cmdstream can be used,
+    * consisting of multiple physical cmdstream buffers
+    */
+   FD_RINGBUFFER_GROWABLE = 0x4,
+
+   /* Indicates an _FD_RINGBUFFER_OBJECT that can be shared
+    * across threads
+    */
+   FD_RINGBUFFER_SHAREABLE = 0x8,
+
+   /* Internal use only: */
+   _FD_RINGBUFFER_OBJECT = 0x10,
+};
+
+/* A submit object manages/tracks all the state buildup for a "submit"
+ * ioctl to the kernel.  Additionally, with the exception of long-lived
+ * non-STREAMING stateobj rb's, rb's are allocated from the submit.
+ */
+struct fd_submit *fd_submit_new(struct fd_pipe *pipe);
+
+/* NOTE: all ringbuffer's create from the submit should be unref'd
+ * before destroying the submit.
+ */
+void fd_submit_del(struct fd_submit *submit);
+
+struct fd_submit * fd_submit_ref(struct fd_submit *submit);
+
+/* Allocate a new rb from the submit. */
+struct fd_ringbuffer *fd_submit_new_ringbuffer(struct fd_submit *submit,
+                                               uint32_t size,
+                                               enum fd_ringbuffer_flags flags);
+
+/* in_fence_fd: -1 for no in-fence, else fence fd
+ * if use_fence_fd is true the output fence will be dma_fence fd backed
+ */
+struct fd_fence *fd_submit_flush(struct fd_submit *submit, int in_fence_fd,
+                                 bool use_fence_fd);
+
+struct fd_ringbuffer;
+struct fd_reloc;
+
+struct fd_ringbuffer_funcs {
+   void (*grow)(struct fd_ringbuffer *ring, uint32_t size);
+
+   /**
+    * Alternative to emit_reloc for the softpin case, where we only need
+    * to track that the bo is used (and not track all the extra info that
+    * the kernel would need to do a legacy reloc.
+    */
+   void (*attach_bo)(struct fd_ringbuffer *ring, struct fd_bo *bo);
+   uint32_t (*attach_ring)(struct fd_ringbuffer *ring, struct fd_ringbuffer *target,
+                           uint32_t cmd_idx, uint64_t *iova);
+   void (*assert_attached)(struct fd_ringbuffer *ring, struct fd_bo *bo);
+
+   void (*emit_reloc)(struct fd_ringbuffer *ring, const struct fd_reloc *reloc);
+   uint32_t (*emit_reloc_ring)(struct fd_ringbuffer *ring,
+                               struct fd_ringbuffer *target, uint32_t cmd_idx);
+   uint32_t (*cmd_count)(struct fd_ringbuffer *ring);
+   bool (*check_size)(struct fd_ringbuffer *ring);
+   void (*destroy)(struct fd_ringbuffer *ring);
+};
+
+/* the ringbuffer object is not opaque so that OUT_RING() type stuff
+ * can be inlined.  Note that users should not make assumptions about
+ * the size of this struct.
+ */
+struct fd_ringbuffer {
+   uint32_t *cur, *end, *start;
+   const struct fd_ringbuffer_funcs *funcs;
+
+   // size or end coudl probably go away
+   int size;
+   int32_t refcnt;
+   enum fd_ringbuffer_flags flags;
+};
+
+/* Allocate a new long-lived state object, not associated with
+ * a submit:
+ */
+struct fd_ringbuffer *fd_ringbuffer_new_object(struct fd_pipe *pipe,
+                                               uint32_t size);
+struct fd_ringbuffer *
+fd_ringbuffer_new_shareable_object(struct fd_pipe *pipe, uint32_t size);
+
+/*
+ * Helpers for ref/unref with some extra debugging.. unref() returns true if
+ * the object is still live
+ */
+
+static inline void
+ref(int32_t *ref)
+{
+   ASSERTED int32_t count = p_atomic_inc_return(ref);
+   /* We should never see a refcnt transition 0->1, this is a sign of a
+    * zombie coming back from the dead!
+    */
+   assert(count != 1);
+}
+
+static inline bool
+unref(int32_t *ref)
+{
+   int32_t count = p_atomic_dec_return(ref);
+   assert(count != -1);
+   return count == 0;
+}
+
+static inline void
+fd_ringbuffer_del(struct fd_ringbuffer *ring)
+{
+   if (--ring->refcnt > 0)
+      return;
+
+   ring->funcs->destroy(ring);
+}
+
+static inline struct fd_ringbuffer *
+fd_ringbuffer_ref(struct fd_ringbuffer *ring)
+{
+   ring->refcnt++;
+   return ring;
+}
+
+static inline void
+fd_ringbuffer_grow(struct fd_ringbuffer *ring, uint32_t ndwords)
+{
+   assert(ring->funcs->grow); /* unsupported on kgsl */
+
+   ring->funcs->grow(ring, ring->size);
+}
+
+static inline bool
+fd_ringbuffer_check_size(struct fd_ringbuffer *ring)
+{
+   return ring->funcs->check_size(ring);
+}
+
+static inline void
+fd_ringbuffer_emit(struct fd_ringbuffer *ring, uint32_t data)
+{
+   (*ring->cur++) = data;
+}
+
+struct fd_reloc {
+   struct fd_bo *bo;
+   uint64_t iova;
+   uint64_t orval;
+#define FD_RELOC_READ  0x0001
+#define FD_RELOC_WRITE 0x0002
+#define FD_RELOC_DUMP  0x0004
+   uint32_t offset;
+   int32_t shift;
+};
+
+/* We always mark BOs for write, instead of tracking it across reloc
+ * sources in userspace.  On the kernel side, this means we track a single
+ * excl fence in the BO instead of a set of read fences, which is cheaper.
+ * The downside is that a dmabuf-shared device won't be able to read in
+ * parallel with a read-only access by freedreno, but most other drivers
+ * have decided that that usecase isn't important enough to do this
+ * tracking, as well.
+ */
+#define FD_RELOC_FLAGS_INIT (FD_RELOC_READ | FD_RELOC_WRITE)
+
+/* NOTE: relocs are 2 dwords on a5xx+ */
+
+static inline void
+fd_ringbuffer_attach_bo(struct fd_ringbuffer *ring, struct fd_bo *bo)
+{
+   ring->funcs->attach_bo(ring, bo);
+}
+
+static inline
+uint32_t fd_ringbuffer_attach_ring(struct fd_ringbuffer *ring,
+                                   struct fd_ringbuffer *target,
+                                   uint32_t cmd_idx, uint64_t *iova)
+{
+   return ring->funcs->attach_ring(ring, target, cmd_idx, iova);
+}
+
+static inline void
+fd_ringbuffer_assert_attached(struct fd_ringbuffer *ring, struct fd_bo *bo)
+{
+#ifndef NDEBUG
+   ring->funcs->assert_attached(ring, bo);
+#endif
+}
+
+static inline void
+fd_ringbuffer_reloc(struct fd_ringbuffer *ring, const struct fd_reloc *reloc)
+{
+   ring->funcs->emit_reloc(ring, reloc);
+}
+
+static inline uint32_t
+fd_ringbuffer_cmd_count(struct fd_ringbuffer *ring)
+{
+   if (!ring->funcs->cmd_count)
+      return 1;
+   return ring->funcs->cmd_count(ring);
+}
+
+static inline uint32_t
+fd_ringbuffer_emit_reloc_ring_full(struct fd_ringbuffer *ring,
+                                   struct fd_ringbuffer *target,
+                                   uint32_t cmd_idx)
+{
+   return ring->funcs->emit_reloc_ring(ring, target, cmd_idx);
+}
+
+static inline uint32_t
+offset_bytes(void *end, void *start)
+{
+   return ((char *)end) - ((char *)start);
+}
+
+static inline uint32_t
+fd_ringbuffer_size(struct fd_ringbuffer *ring)
+{
+   /* only really needed for stateobj ringbuffers, and won't really
+    * do what you expect for growable rb's.. so lets just restrict
+    * this to stateobj's for now:
+    */
+   assert(!(ring->flags & FD_RINGBUFFER_GROWABLE));
+   return offset_bytes(ring->cur, ring->start);
+}
+
+static inline bool
+fd_ringbuffer_empty(struct fd_ringbuffer *ring)
+{
+   return (fd_ringbuffer_cmd_count(ring) == 1) &&
+          (offset_bytes(ring->cur, ring->start) == 0);
+}
+
+#define LOG_DWORDS 0
+
+static inline void
+OUT_RING(struct fd_ringbuffer *ring, uint32_t data)
+{
+   if (LOG_DWORDS) {
+      fprintf(stderr, "ring[%p]: OUT_RING   %04x:  %08x", ring,
+              (uint32_t)(ring->cur - ring->start), data);
+   }
+   fd_ringbuffer_emit(ring, data);
+}
+
+static inline uint64_t
+__reloc_iova(struct fd_bo *bo, uint32_t offset, uint64_t orval, int32_t shift)
+{
+   uint64_t iova = fd_bo_get_iova(bo) + offset;
+
+   if (shift < 0)
+      iova >>= -shift;
+   else
+      iova <<= shift;
+
+   iova |= orval;
+
+   return iova;
+}
+
+static inline void
+OUT_RB(struct fd_ringbuffer *ring, struct fd_ringbuffer *target)
+{
+   fd_ringbuffer_emit_reloc_ring_full(ring, target, 0);
+}
+
+static inline void
+BEGIN_RING(struct fd_ringbuffer *ring, uint32_t ndwords)
+{
+   if (unlikely(ring->cur + ndwords > ring->end))
+      fd_ringbuffer_grow(ring, ndwords);
+}
+
+static inline void
+OUT_PKT0(struct fd_ringbuffer *ring, uint16_t regindx, uint16_t cnt)
+{
+   BEGIN_RING(ring, cnt + 1);
+   OUT_RING(ring, pm4_pkt0_hdr(regindx, cnt));
+}
+
+static inline void
+OUT_PKT2(struct fd_ringbuffer *ring)
+{
+   BEGIN_RING(ring, 1);
+   OUT_RING(ring, CP_TYPE2_PKT);
+}
+
+static inline void
+OUT_PKT3(struct fd_ringbuffer *ring, uint8_t opcode, uint16_t cnt)
+{
+   BEGIN_RING(ring, cnt + 1);
+   OUT_RING(ring, CP_TYPE3_PKT | ((cnt - 1) << 16) | ((opcode & 0xFF) << 8));
+}
+
+/*
+ * Starting with a5xx, pkt4/pkt7 are used instead of pkt0/pkt3
+ */
+
+static inline void
+OUT_PKT4(struct fd_ringbuffer *ring, uint16_t regindx, uint16_t cnt)
+{
+   BEGIN_RING(ring, cnt + 1);
+   OUT_RING(ring, pm4_pkt4_hdr((uint16_t)regindx, (uint16_t)cnt));
+}
+
+static inline void
+OUT_PKT7(struct fd_ringbuffer *ring, uint32_t opcode, uint32_t cnt)
+{
+   BEGIN_RING(ring, cnt + 1);
+   OUT_RING(ring, pm4_pkt7_hdr((uint8_t)opcode, (uint16_t)cnt));
+}
+
+static inline void
+OUT_WFI(struct fd_ringbuffer *ring)
+{
+   OUT_PKT3(ring, CP_WAIT_FOR_IDLE, 1);
+   OUT_RING(ring, 0x00000000);
+}
+
+static inline void
+OUT_WFI5(struct fd_ringbuffer *ring)
+{
+   OUT_PKT7(ring, CP_WAIT_FOR_IDLE, 0);
+}
+
+#ifdef __cplusplus
+} /* end of extern "C" */
+#endif
+
+/*
+ * NOTE: OUT_RELOC() is 2 dwords (64b) on a5xx+
+ */
+static inline void
+OUT_RELOC(struct fd_ringbuffer *ring, struct fd_bo *bo, uint32_t offset,
+          uint64_t orval, int32_t shift)
+{
+   if (LOG_DWORDS) {
+      fprintf(stderr, "ring[%p]: OUT_RELOC   %04x:  %p+%u << %d", ring,
+              (uint32_t)(ring->cur - ring->start), bo, offset, shift);
+   }
+   assert(offset < fd_bo_size(bo));
+
+   uint64_t iova = __reloc_iova(bo, offset, orval, shift);
+
+#if FD_BO_NO_HARDPIN
+   uint64_t *cur = (uint64_t *)ring->cur;
+   *cur = iova;
+   ring->cur += 2;
+   fd_ringbuffer_assert_attached(ring, bo);
+#else
+   struct fd_reloc reloc = {
+         .bo = bo,
+         .iova = iova,
+         .orval = orval,
+         .offset = offset,
+         .shift = shift,
+   };
+
+   fd_ringbuffer_reloc(ring, &reloc);
+#endif
+}
+
+#ifdef __cplusplus
+static inline void
+OUT_RELOC(struct fd_ringbuffer *ring, struct fd_bo *bo, uint32_t offset)
+{
+   OUT_RELOC(ring, bo, offset, 0, 0);
+}
+#endif
+
+#endif /* FREEDRENO_RINGBUFFER_H_ */

src/freedreno/drm/freedreno_ringbuffer_sp.c

@@ -0,0 +1,871 @@
+/*
+ * Copyright © 2018 Rob Clark <[email protected]>
+ * SPDX-License-Identifier: MIT
+ *
+ * Authors:
+ *    Rob Clark <[email protected]>
+ */
+
+#include <assert.h>
+#include <inttypes.h>
+#include <pthread.h>
+
+#include "util/hash_table.h"
+#include "util/libsync.h"
+#include "util/os_file.h"
+#include "util/slab.h"
+
+#include "freedreno_ringbuffer_sp.h"
+
+/* A "softpin" implementation of submit/ringbuffer, which lowers CPU overhead
+ * by avoiding the additional tracking necessary to build cmds/relocs tables
+ * (but still builds a bos table)
+ */
+
+/* In the pipe->flush() path, we don't have a util_queue_fence we can wait on,
+ * instead use a condition-variable.  Note that pipe->flush() is not expected
+ * to be a common/hot path.
+ */
+static pthread_cond_t  flush_cnd = PTHREAD_COND_INITIALIZER;
+static pthread_mutex_t flush_mtx = PTHREAD_MUTEX_INITIALIZER;
+
+static void finalize_current_cmd(struct fd_ringbuffer *ring);
+static struct fd_ringbuffer *
+fd_ringbuffer_sp_init(struct fd_ringbuffer_sp *fd_ring, uint32_t size,
+                      enum fd_ringbuffer_flags flags);
+
+
+static inline bool
+check_append_suballoc_bo(struct fd_submit_sp *submit, struct fd_bo *bo, bool check)
+{
+   uint32_t idx = READ_ONCE(bo->idx);
+
+   if (unlikely((idx >= submit->nr_suballoc_bos) ||
+       (submit->suballoc_bos[idx] != bo))) {
+      uint32_t hash = _mesa_hash_pointer(bo);
+      struct hash_entry *entry;
+
+      entry = _mesa_hash_table_search_pre_hashed(
+            submit->suballoc_bo_table, hash, bo);
+      if (entry) {
+         /* found */
+         idx = (uint32_t)(uintptr_t)entry->data;
+      } else if (unlikely(check)) {
+         return false;
+      } else {
+         idx = APPEND(submit, suballoc_bos, fd_bo_ref(bo));
+
+         _mesa_hash_table_insert_pre_hashed(
+               submit->suballoc_bo_table, hash, bo, (void *)(uintptr_t)idx);
+      }
+      bo->idx = idx;
+   }
+
+   return true;
+}
+
+static inline uint32_t
+check_append_bo(struct fd_submit_sp *submit, struct fd_bo *bo, bool check)
+{
+   if (suballoc_bo(bo)) {
+      if (check) {
+         if (!check_append_suballoc_bo(submit, bo, true)) {
+            return ~0;
+         }
+         bo = fd_bo_heap_block(bo);
+      } else {
+         check_append_suballoc_bo(submit, bo, false);
+         bo = fd_bo_heap_block(bo);
+      }
+   }
+
+   /* NOTE: it is legal to use the same bo on different threads for
+    * different submits.  But it is not legal to use the same submit
+    * from different threads.
+    */
+   uint32_t idx = READ_ONCE(bo->idx);
+
+   if (unlikely((idx >= submit->nr_bos) || (submit->bos[idx] != bo))) {
+      uint32_t hash = _mesa_hash_pointer(bo);
+      struct hash_entry *entry;
+
+      entry = _mesa_hash_table_search_pre_hashed(submit->bo_table, hash, bo);
+      if (entry) {
+         /* found */
+         idx = (uint32_t)(uintptr_t)entry->data;
+      } else if (unlikely(check)) {
+         return ~0;
+      } else {
+         idx = APPEND(submit, bos, fd_bo_ref(bo));
+
+         _mesa_hash_table_insert_pre_hashed(submit->bo_table, hash, bo,
+                                            (void *)(uintptr_t)idx);
+      }
+      bo->idx = idx;
+   }
+
+   return idx;
+}
+
+/* add (if needed) bo to submit and return index: */
+uint32_t
+fd_submit_append_bo(struct fd_submit_sp *submit, struct fd_bo *bo)
+{
+   return check_append_bo(submit, bo, false);
+}
+
+static void
+fd_submit_suballoc_ring_bo(struct fd_submit *submit,
+                           struct fd_ringbuffer_sp *fd_ring, uint32_t size)
+{
+   struct fd_submit_sp *fd_submit = to_fd_submit_sp(submit);
+   unsigned suballoc_offset = 0;
+   struct fd_bo *suballoc_bo = NULL;
+
+   if (fd_submit->suballoc_ring) {
+      struct fd_ringbuffer_sp *suballoc_ring =
+         to_fd_ringbuffer_sp(fd_submit->suballoc_ring);
+
+      suballoc_bo = suballoc_ring->ring_bo;
+      suballoc_offset =
+         fd_ringbuffer_size(fd_submit->suballoc_ring) + suballoc_ring->offset;
+
+      suballoc_offset = align(suballoc_offset, SUBALLOC_ALIGNMENT);
+
+      if ((size + suballoc_offset) > suballoc_bo->size) {
+         suballoc_bo = NULL;
+      }
+   }
+
+   if (!suballoc_bo) {
+      // TODO possibly larger size for streaming bo?
+      fd_ring->ring_bo = fd_bo_new_ring(submit->pipe->dev, SUBALLOC_SIZE);
+      fd_ring->offset = 0;
+   } else {
+      fd_ring->ring_bo = fd_bo_ref(suballoc_bo);
+      fd_ring->offset = suballoc_offset;
+   }
+
+   struct fd_ringbuffer *old_suballoc_ring = fd_submit->suballoc_ring;
+
+   fd_submit->suballoc_ring = fd_ringbuffer_ref(&fd_ring->base);
+
+   if (old_suballoc_ring)
+      fd_ringbuffer_del(old_suballoc_ring);
+}
+
+static struct fd_ringbuffer *
+fd_submit_sp_new_ringbuffer(struct fd_submit *submit, uint32_t size,
+                            enum fd_ringbuffer_flags flags)
+{
+   struct fd_submit_sp *fd_submit = to_fd_submit_sp(submit);
+   struct fd_ringbuffer_sp *fd_ring;
+
+   fd_ring = slab_alloc(&fd_submit->ring_pool);
+
+   fd_ring->u.submit = submit;
+
+   /* NOTE: needs to be before _suballoc_ring_bo() since it could
+    * increment the refcnt of the current ring
+    */
+   fd_ring->base.refcnt = 1;
+
+   if (flags & FD_RINGBUFFER_STREAMING) {
+      fd_submit_suballoc_ring_bo(submit, fd_ring, size);
+   } else {
+      if (flags & FD_RINGBUFFER_GROWABLE)
+         size = SUBALLOC_SIZE;
+
+      fd_ring->offset = 0;
+      fd_ring->ring_bo = fd_bo_new_ring(submit->pipe->dev, size);
+   }
+
+   if (!fd_ringbuffer_sp_init(fd_ring, size, flags))
+      return NULL;
+
+   return &fd_ring->base;
+}
+
+/**
+ * Prepare submit for flush, always done synchronously.
+ *
+ * 1) Finalize primary ringbuffer, at this point no more cmdstream may
+ *    be written into it, since from the PoV of the upper level driver
+ *    the submit is flushed, even if deferred
+ * 2) Add cmdstream bos to bos table
+ * 3) Update bo fences
+ */
+static bool
+fd_submit_sp_flush_prep(struct fd_submit *submit, int in_fence_fd,
+                        struct fd_fence *out_fence)
+{
+   struct fd_submit_sp *fd_submit = to_fd_submit_sp(submit);
+   bool has_shared = false;
+
+   finalize_current_cmd(submit->primary);
+
+   struct fd_ringbuffer_sp *primary =
+      to_fd_ringbuffer_sp(submit->primary);
+
+   for (unsigned i = 0; i < primary->u.nr_cmds; i++)
+      fd_submit_append_bo(fd_submit, primary->u.cmds[i].ring_bo);
+
+   out_fence->ufence = submit->fence;
+
+   simple_mtx_lock(&fence_lock);
+   for (unsigned i = 0; i < fd_submit->nr_bos; i++) {
+      fd_bo_add_fence(fd_submit->bos[i], out_fence);
+      has_shared |= fd_submit->bos[i]->alloc_flags & FD_BO_SHARED;
+   }
+   for (unsigned i = 0; i < fd_submit->nr_suballoc_bos; i++) {
+      fd_bo_add_fence(fd_submit->suballoc_bos[i], out_fence);
+   }
+   simple_mtx_unlock(&fence_lock);
+
+   fd_submit->out_fence   = fd_fence_ref(out_fence);
+   fd_submit->in_fence_fd = (in_fence_fd == -1) ?
+         -1 : os_dupfd_cloexec(in_fence_fd);
+
+   return has_shared;
+}
+
+static void
+fd_submit_sp_flush_execute(void *job, void *gdata, int thread_index)
+{
+   struct fd_submit *submit = job;
+   struct fd_submit_sp *fd_submit = to_fd_submit_sp(submit);
+   struct fd_pipe *pipe = submit->pipe;
+
+   fd_submit->flush_submit_list(&fd_submit->submit_list);
+
+   pthread_mutex_lock(&flush_mtx);
+   assert(fd_fence_before(pipe->last_submit_fence, fd_submit->base.fence));
+   pipe->last_submit_fence = fd_submit->base.fence;
+   pthread_cond_broadcast(&flush_cnd);
+   pthread_mutex_unlock(&flush_mtx);
+
+   DEBUG_MSG("finish: %u", submit->fence);
+}
+
+static void
+fd_submit_sp_flush_cleanup(void *job, void *gdata, int thread_index)
+{
+   struct fd_submit *submit = job;
+   fd_submit_del(submit);
+}
+
+static void
+flush_deferred_submits(struct fd_device *dev)
+{
+   MESA_TRACE_FUNC();
+
+   simple_mtx_assert_locked(&dev->submit_lock);
+
+   if (list_is_empty(&dev->deferred_submits))
+      return;
+
+   struct fd_submit *submit = last_submit(&dev->deferred_submits);
+   struct fd_submit_sp *fd_submit = to_fd_submit_sp(submit);
+   list_replace(&dev->deferred_submits, &fd_submit->submit_list);
+   list_inithead(&dev->deferred_submits);
+   dev->deferred_cmds = 0;
+
+   /* If we have multiple submits with in-fence-fd's then merge them: */
+   foreach_submit (submit, &fd_submit->submit_list) {
+      struct fd_submit_sp *fd_deferred_submit = to_fd_submit_sp(submit);
+
+      if (fd_deferred_submit == fd_submit)
+         break;
+
+      if (fd_deferred_submit->in_fence_fd != -1) {
+         sync_accumulate("freedreno",
+                         &fd_submit->in_fence_fd,
+                         fd_deferred_submit->in_fence_fd);
+         close(fd_deferred_submit->in_fence_fd);
+         fd_deferred_submit->in_fence_fd = -1;
+      }
+   }
+
+   fd_fence_del(dev->deferred_submits_fence);
+   dev->deferred_submits_fence = NULL;
+
+   struct util_queue_fence *fence = &fd_submit->out_fence->ready;
+
+   DEBUG_MSG("enqueue: %u", submit->fence);
+
+   if (fd_device_threaded_submit(submit->pipe->dev)) {
+      util_queue_add_job(&submit->pipe->dev->submit_queue,
+                         submit, fence,
+                         fd_submit_sp_flush_execute,
+                         fd_submit_sp_flush_cleanup,
+                         0);
+   } else {
+      fd_submit_sp_flush_execute(submit, NULL, 0);
+      fd_submit_sp_flush_cleanup(submit, NULL, 0);
+   }
+}
+
+static bool
+should_defer(struct fd_submit *submit)
+{
+   struct fd_submit_sp *fd_submit = to_fd_submit_sp(submit);
+
+   /* if too many bo's, it may not be worth the CPU cost of submit merging: */
+   if (fd_submit->nr_bos > 30)
+      return false;
+
+   /* On the kernel side, with 32K ringbuffer, we have an upper limit of 2k
+    * cmds before we exceed the size of the ringbuffer, which results in
+    * deadlock writing into the RB (ie. kernel doesn't finish writing into
+    * the RB so it doesn't kick the GPU to start consuming from the RB)
+    */
+   if (submit->pipe->dev->deferred_cmds > 128)
+      return false;
+
+   return true;
+}
+
+static struct fd_fence *
+fd_submit_sp_flush(struct fd_submit *submit, int in_fence_fd, bool use_fence_fd)
+{
+   struct fd_device *dev = submit->pipe->dev;
+   struct fd_pipe *pipe = submit->pipe;
+
+   MESA_TRACE_FUNC();
+
+   /* Acquire lock before flush_prep() because it is possible to race between
+    * this and pipe->flush():
+    */
+   simple_mtx_lock(&dev->submit_lock);
+
+   /* If there are deferred submits from another fd_pipe, flush them now,
+    * since we can't merge submits from different submitqueue's (ie. they
+    * could have different priority, etc)
+    */
+   if (!list_is_empty(&dev->deferred_submits) &&
+       (last_submit(&dev->deferred_submits)->pipe != submit->pipe)) {
+      flush_deferred_submits(dev);
+   }
+
+   list_addtail(&fd_submit_ref(submit)->node, &dev->deferred_submits);
+
+   if (!dev->deferred_submits_fence)
+      dev->deferred_submits_fence = fd_fence_new(submit->pipe, use_fence_fd);
+
+   struct fd_fence *out_fence = fd_fence_ref(dev->deferred_submits_fence);
+
+   /* upgrade the out_fence for the deferred submits, if needed: */
+   if (use_fence_fd)
+      out_fence->use_fence_fd = true;
+
+   bool has_shared = fd_submit_sp_flush_prep(submit, in_fence_fd, out_fence);
+
+   if (((in_fence_fd != -1) || out_fence->use_fence_fd) && !dev->disable_explicit_sync_heuristic)
+      pipe->no_implicit_sync = true;
+
+   /* The rule about skipping submit merging with shared buffers is only
+    * needed for implicit-sync.
+    */
+   if (pipe->no_implicit_sync)
+      has_shared = false;
+
+   assert(fd_fence_before(pipe->last_enqueue_fence, submit->fence));
+   pipe->last_enqueue_fence = submit->fence;
+
+   /* If we don't need an out-fence, we can defer the submit.
+    *
+    * TODO we could defer submits with in-fence as well.. if we took our own
+    * reference to the fd, and merged all the in-fence-fd's when we flush the
+    * deferred submits
+    */
+   if (!use_fence_fd && !has_shared && should_defer(submit)) {
+      DEBUG_MSG("defer: %u", submit->fence);
+      dev->deferred_cmds += fd_ringbuffer_cmd_count(submit->primary);
+      assert(dev->deferred_cmds == fd_dev_count_deferred_cmds(dev));
+      simple_mtx_unlock(&dev->submit_lock);
+
+      return out_fence;
+   }
+
+   flush_deferred_submits(dev);
+
+   simple_mtx_unlock(&dev->submit_lock);
+
+   return out_fence;
+}
+
+void
+fd_pipe_sp_flush(struct fd_pipe *pipe, uint32_t fence)
+{
+   struct fd_device *dev = pipe->dev;
+
+   if (!fd_fence_before(pipe->last_submit_fence, fence))
+      return;
+
+   MESA_TRACE_FUNC();
+
+   simple_mtx_lock(&dev->submit_lock);
+
+   assert(!fd_fence_after(fence, pipe->last_enqueue_fence));
+
+   flush_deferred_submits(dev);
+
+   simple_mtx_unlock(&dev->submit_lock);
+
+   if (!fd_device_threaded_submit(pipe->dev))
+      return;
+
+   /* Once we are sure that we've enqueued at least up to the requested
+    * submit, we need to be sure that submitq has caught up and flushed
+    * them to the kernel
+    */
+   pthread_mutex_lock(&flush_mtx);
+   while (fd_fence_before(pipe->last_submit_fence, fence)) {
+      pthread_cond_wait(&flush_cnd, &flush_mtx);
+   }
+   pthread_mutex_unlock(&flush_mtx);
+}
+
+static void
+fd_submit_sp_destroy(struct fd_submit *submit)
+{
+   struct fd_submit_sp *fd_submit = to_fd_submit_sp(submit);
+
+   if (fd_submit->suballoc_ring)
+      fd_ringbuffer_del(fd_submit->suballoc_ring);
+
+   _mesa_hash_table_destroy(fd_submit->bo_table, NULL);
+   _mesa_hash_table_destroy(fd_submit->suballoc_bo_table, NULL);
+
+   // TODO it would be nice to have a way to assert() if all
+   // rb's haven't been free'd back to the slab, because that is
+   // an indication that we are leaking bo's
+   slab_destroy_child(&fd_submit->ring_pool);
+
+   fd_bo_del_array(fd_submit->bos, fd_submit->nr_bos);
+   free(fd_submit->bos);
+
+   fd_bo_del_array(fd_submit->suballoc_bos, fd_submit->nr_suballoc_bos);
+   free(fd_submit->suballoc_bos);
+
+   if (fd_submit->out_fence)
+      fd_fence_del(fd_submit->out_fence);
+
+   free(fd_submit);
+}
+
+static const struct fd_submit_funcs submit_funcs = {
+   .new_ringbuffer = fd_submit_sp_new_ringbuffer,
+   .flush = fd_submit_sp_flush,
+   .destroy = fd_submit_sp_destroy,
+};
+
+struct fd_submit *
+fd_submit_sp_new(struct fd_pipe *pipe, flush_submit_list_fn flush_submit_list)
+{
+   struct fd_submit_sp *fd_submit = calloc(1, sizeof(*fd_submit));
+   struct fd_submit *submit;
+
+   fd_submit->bo_table = _mesa_pointer_hash_table_create(NULL);
+   fd_submit->suballoc_bo_table = _mesa_pointer_hash_table_create(NULL);
+
+   slab_create_child(&fd_submit->ring_pool, &pipe->ring_pool);
+
+   fd_submit->flush_submit_list = flush_submit_list;
+   fd_submit->seqno = seqno_next(&pipe->submit_seqno);
+
+   submit = &fd_submit->base;
+   submit->funcs = &submit_funcs;
+
+   return submit;
+}
+
+void
+fd_pipe_sp_ringpool_init(struct fd_pipe *pipe)
+{
+   // TODO tune size:
+   slab_create_parent(&pipe->ring_pool, sizeof(struct fd_ringbuffer_sp), 16);
+}
+
+void
+fd_pipe_sp_ringpool_fini(struct fd_pipe *pipe)
+{
+   if (pipe->ring_pool.num_elements)
+      slab_destroy_parent(&pipe->ring_pool);
+}
+
+static void
+finalize_current_cmd(struct fd_ringbuffer *ring)
+{
+   assert(!(ring->flags & _FD_RINGBUFFER_OBJECT));
+
+   struct fd_ringbuffer_sp *fd_ring = to_fd_ringbuffer_sp(ring);
+   APPEND(&fd_ring->u, cmds,
+          (struct fd_cmd_sp){
+             .ring_bo = fd_bo_ref(fd_ring->ring_bo),
+             .size = offset_bytes(ring->cur, ring->start),
+          });
+}
+
+static void
+fd_ringbuffer_sp_grow(struct fd_ringbuffer *ring, uint32_t size)
+{
+   struct fd_ringbuffer_sp *fd_ring = to_fd_ringbuffer_sp(ring);
+   struct fd_pipe *pipe = fd_ring->u.submit->pipe;
+
+   assert(ring->flags & FD_RINGBUFFER_GROWABLE);
+
+   finalize_current_cmd(ring);
+
+   fd_bo_del(fd_ring->ring_bo);
+   fd_ring->ring_bo = fd_bo_new_ring(pipe->dev, size);
+
+   ring->start = fd_bo_map(fd_ring->ring_bo);
+   ring->end = &(ring->start[size / 4]);
+   ring->cur = ring->start;
+   ring->size = size;
+}
+
+static inline bool
+fd_ringbuffer_references_bo(struct fd_ringbuffer *ring, struct fd_bo *bo)
+{
+   struct fd_ringbuffer_sp *fd_ring = to_fd_ringbuffer_sp(ring);
+
+   for (int i = 0; i < fd_ring->u.nr_reloc_bos; i++) {
+      if (fd_ring->u.reloc_bos[i] == bo)
+         return true;
+   }
+   return false;
+}
+
+static void
+fd_ringbuffer_sp_attach_bo_nonobj(struct fd_ringbuffer *ring, struct fd_bo *bo)
+{
+   assert(!(ring->flags & _FD_RINGBUFFER_OBJECT));
+
+   struct fd_ringbuffer_sp *fd_ring = to_fd_ringbuffer_sp(ring);
+   struct fd_submit_sp *fd_submit = to_fd_submit_sp(fd_ring->u.submit);
+
+   fd_submit_append_bo(fd_submit, bo);
+}
+
+static void
+fd_ringbuffer_sp_assert_attached_nonobj(struct fd_ringbuffer *ring, struct fd_bo *bo)
+{
+#ifndef NDEBUG
+   struct fd_ringbuffer_sp *fd_ring = to_fd_ringbuffer_sp(ring);
+   struct fd_submit_sp *fd_submit = to_fd_submit_sp(fd_ring->u.submit);
+   assert(check_append_bo(fd_submit, bo, true) != ~0);
+#endif
+}
+
+static struct fd_bo *
+find_target_bo(struct fd_ringbuffer *target, uint32_t cmd_idx, uint32_t *size)
+{
+   struct fd_ringbuffer_sp *fd_target = to_fd_ringbuffer_sp(target);
+
+   if ((target->flags & FD_RINGBUFFER_GROWABLE) &&
+       (cmd_idx < fd_target->u.nr_cmds)) {
+      *size = fd_target->u.cmds[cmd_idx].size;
+      return fd_target->u.cmds[cmd_idx].ring_bo;
+   } else {
+      *size = offset_bytes(target->cur, target->start);
+      return fd_target->ring_bo;
+   }
+}
+
+static uint32_t
+fd_ringbuffer_sp_attach_ring_nonobj(struct fd_ringbuffer *ring,
+                                    struct fd_ringbuffer *target,
+                                    uint32_t cmd_idx,
+                                    uint64_t *iova)
+{
+   struct fd_ringbuffer_sp *fd_target = to_fd_ringbuffer_sp(target);
+   struct fd_ringbuffer_sp *fd_ring = to_fd_ringbuffer_sp(ring);
+   struct fd_submit_sp *fd_submit = to_fd_submit_sp(fd_ring->u.submit);
+   struct fd_bo *bo;
+   uint32_t size;
+
+   bo = find_target_bo(target, cmd_idx, &size);
+
+   *iova = bo->iova + fd_target->offset;
+
+   fd_ringbuffer_sp_attach_bo_nonobj(ring, bo);
+
+   if (!(target->flags & _FD_RINGBUFFER_OBJECT))
+      return size;
+
+   if ((fd_submit->seqno != fd_target->u.last_submit_seqno) ||
+       (target->flags & FD_RINGBUFFER_SHAREABLE)) {
+      for (unsigned i = 0; i < fd_target->u.nr_reloc_bos; i++) {
+         fd_submit_append_bo(fd_submit, fd_target->u.reloc_bos[i]);
+      }
+      fd_target->u.last_submit_seqno = fd_submit->seqno;
+   }
+
+#ifndef NDEBUG
+   /* Dealing with assert'd BOs is deferred until the submit is known,
+    * since the batch resource tracking attaches BOs directly to
+    * the submit instead of the long lived stateobj
+    */
+   for (unsigned i = 0; i < fd_target->u.nr_assert_bos; i++) {
+      fd_ringbuffer_sp_assert_attached_nonobj(ring, fd_target->u.assert_bos[i]);
+   }
+#endif
+
+   return size;
+}
+
+static void
+fd_ringbuffer_sp_attach_bo_obj(struct fd_ringbuffer *ring, struct fd_bo *bo)
+{
+   assert(ring->flags & _FD_RINGBUFFER_OBJECT);
+
+   struct fd_ringbuffer_sp *fd_ring = to_fd_ringbuffer_sp(ring);
+
+   /* Avoid emitting duplicate BO references into the list.  Ringbuffer
+    * objects are long-lived, so this saves ongoing work at draw time in
+    * exchange for a bit at context setup/first draw.  And the number of
+    * relocs per ringbuffer object is fairly small, so the O(n^2) doesn't
+    * hurt much.
+    */
+   if (!fd_ringbuffer_references_bo(ring, bo)) {
+      APPEND(&fd_ring->u, reloc_bos, fd_bo_ref(bo));
+   }
+}
+
+static void
+fd_ringbuffer_sp_assert_attached_obj(struct fd_ringbuffer *ring, struct fd_bo *bo)
+{
+#ifndef NDEBUG
+   /* If the stateobj already references the bo, nothing more to do: */
+   if (fd_ringbuffer_references_bo(ring, bo))
+      return;
+
+   /* If not, we need to defer the assert.. because the batch resource
+    * tracking may have attached the bo to the submit that the stateobj
+    * will eventually be referenced by:
+    */
+   struct fd_ringbuffer_sp *fd_ring = to_fd_ringbuffer_sp(ring);
+   for (int i = 0; i < fd_ring->u.nr_assert_bos; i++)
+      if (fd_ring->u.assert_bos[i] == bo)
+         return;
+
+   APPEND(&fd_ring->u, assert_bos, fd_bo_ref(bo));
+#endif
+}
+
+static uint32_t
+fd_ringbuffer_sp_attach_ring_obj(struct fd_ringbuffer *ring,
+                                 struct fd_ringbuffer *target,
+                                 uint32_t cmd_idx,
+                                 uint64_t *iova)
+{
+   struct fd_ringbuffer_sp *fd_target = to_fd_ringbuffer_sp(target);
+   struct fd_ringbuffer_sp *fd_ring = to_fd_ringbuffer_sp(ring);
+   struct fd_bo *bo;
+   uint32_t size;
+
+   bo = find_target_bo(target, cmd_idx, &size);
+
+   *iova = bo->iova + fd_target->offset;
+
+   fd_ringbuffer_sp_attach_bo_obj(ring, bo);
+
+   if (!(target->flags & _FD_RINGBUFFER_OBJECT))
+      return size;
+
+   for (unsigned i = 0; i < fd_target->u.nr_reloc_bos; i++) {
+      struct fd_bo *target_bo = fd_target->u.reloc_bos[i];
+      if (!fd_ringbuffer_references_bo(ring, target_bo))
+         APPEND(&fd_ring->u, reloc_bos, fd_bo_ref(target_bo));
+   }
+
+   return size;
+}
+
+#define PTRSZ 64
+#include "freedreno_ringbuffer_sp_reloc.h"
+#undef PTRSZ
+#define PTRSZ 32
+#include "freedreno_ringbuffer_sp_reloc.h"
+#undef PTRSZ
+
+static uint32_t
+fd_ringbuffer_sp_cmd_count(struct fd_ringbuffer *ring)
+{
+   if (ring->flags & FD_RINGBUFFER_GROWABLE)
+      return to_fd_ringbuffer_sp(ring)->u.nr_cmds + 1;
+   return 1;
+}
+
+static bool
+fd_ringbuffer_sp_check_size(struct fd_ringbuffer *ring)
+{
+   assert(!(ring->flags & _FD_RINGBUFFER_OBJECT));
+   struct fd_ringbuffer_sp *fd_ring = to_fd_ringbuffer_sp(ring);
+   struct fd_submit *submit = fd_ring->u.submit;
+
+   if (to_fd_submit_sp(submit)->nr_bos > MAX_ARRAY_SIZE/2) {
+      return false;
+   }
+
+   if (to_fd_submit_sp(submit)->nr_suballoc_bos > MAX_ARRAY_SIZE/2) {
+      return false;
+   }
+
+   return true;
+}
+
+static void
+fd_ringbuffer_sp_destroy(struct fd_ringbuffer *ring)
+{
+   struct fd_ringbuffer_sp *fd_ring = to_fd_ringbuffer_sp(ring);
+
+   fd_bo_del(fd_ring->ring_bo);
+
+   if (ring->flags & _FD_RINGBUFFER_OBJECT) {
+      fd_bo_del_array(fd_ring->u.reloc_bos, fd_ring->u.nr_reloc_bos);
+      free(fd_ring->u.reloc_bos);
+#ifndef NDEBUG
+      fd_bo_del_array(fd_ring->u.assert_bos, fd_ring->u.nr_assert_bos);
+      free(fd_ring->u.assert_bos);
+#endif
+      free(fd_ring);
+   } else {
+      struct fd_submit *submit = fd_ring->u.submit;
+
+      // TODO re-arrange the data structures so we can use fd_bo_del_array()
+      for (unsigned i = 0; i < fd_ring->u.nr_cmds; i++) {
+         fd_bo_del(fd_ring->u.cmds[i].ring_bo);
+      }
+      free(fd_ring->u.cmds);
+
+      slab_free(&to_fd_submit_sp(submit)->ring_pool, fd_ring);
+   }
+}
+
+static const struct fd_ringbuffer_funcs ring_funcs_nonobj_32 = {
+   .grow = fd_ringbuffer_sp_grow,
+   .attach_bo = fd_ringbuffer_sp_attach_bo_nonobj,
+   .attach_ring = fd_ringbuffer_sp_attach_ring_nonobj,
+   .assert_attached = fd_ringbuffer_sp_assert_attached_nonobj,
+   .emit_reloc = fd_ringbuffer_sp_emit_reloc_nonobj_32,
+   .emit_reloc_ring = fd_ringbuffer_sp_emit_reloc_ring_nonobj_32,
+   .cmd_count = fd_ringbuffer_sp_cmd_count,
+   .check_size = fd_ringbuffer_sp_check_size,
+   .destroy = fd_ringbuffer_sp_destroy,
+};
+
+static const struct fd_ringbuffer_funcs ring_funcs_obj_32 = {
+   .grow = fd_ringbuffer_sp_grow,
+   .attach_bo = fd_ringbuffer_sp_attach_bo_obj,
+   .attach_ring = fd_ringbuffer_sp_attach_ring_obj,
+   .assert_attached = fd_ringbuffer_sp_assert_attached_obj,
+   .emit_reloc = fd_ringbuffer_sp_emit_reloc_obj_32,
+   .emit_reloc_ring = fd_ringbuffer_sp_emit_reloc_ring_obj_32,
+   .cmd_count = fd_ringbuffer_sp_cmd_count,
+   .destroy = fd_ringbuffer_sp_destroy,
+};
+
+static const struct fd_ringbuffer_funcs ring_funcs_nonobj_64 = {
+   .grow = fd_ringbuffer_sp_grow,
+   .attach_bo = fd_ringbuffer_sp_attach_bo_nonobj,
+   .attach_ring = fd_ringbuffer_sp_attach_ring_nonobj,
+   .assert_attached = fd_ringbuffer_sp_assert_attached_nonobj,
+   .emit_reloc = fd_ringbuffer_sp_emit_reloc_nonobj_64,
+   .emit_reloc_ring = fd_ringbuffer_sp_emit_reloc_ring_nonobj_64,
+   .cmd_count = fd_ringbuffer_sp_cmd_count,
+   .check_size = fd_ringbuffer_sp_check_size,
+   .destroy = fd_ringbuffer_sp_destroy,
+};
+
+static const struct fd_ringbuffer_funcs ring_funcs_obj_64 = {
+   .grow = fd_ringbuffer_sp_grow,
+   .attach_bo = fd_ringbuffer_sp_attach_bo_obj,
+   .attach_ring = fd_ringbuffer_sp_attach_ring_obj,
+   .assert_attached = fd_ringbuffer_sp_assert_attached_obj,
+   .emit_reloc = fd_ringbuffer_sp_emit_reloc_obj_64,
+   .emit_reloc_ring = fd_ringbuffer_sp_emit_reloc_ring_obj_64,
+   .cmd_count = fd_ringbuffer_sp_cmd_count,
+   .destroy = fd_ringbuffer_sp_destroy,
+};
+
+static inline struct fd_ringbuffer *
+fd_ringbuffer_sp_init(struct fd_ringbuffer_sp *fd_ring, uint32_t size,
+                      enum fd_ringbuffer_flags flags)
+{
+   struct fd_ringbuffer *ring = &fd_ring->base;
+
+   assert(fd_ring->ring_bo);
+
+   uint8_t *base = fd_bo_map(fd_ring->ring_bo);
+   ring->start = (void *)(base + fd_ring->offset);
+   ring->end = &(ring->start[size / 4]);
+   ring->cur = ring->start;
+
+   ring->size = size;
+   ring->flags = flags;
+
+   if (flags & _FD_RINGBUFFER_OBJECT) {
+      if (fd_ring->u.pipe->is_64bit) {
+         ring->funcs = &ring_funcs_obj_64;
+      } else {
+         ring->funcs = &ring_funcs_obj_32;
+      }
+   } else {
+      if (fd_ring->u.submit->pipe->is_64bit) {
+         ring->funcs = &ring_funcs_nonobj_64;
+      } else {
+         ring->funcs = &ring_funcs_nonobj_32;
+      }
+   }
+
+   // TODO initializing these could probably be conditional on flags
+   // since unneed for FD_RINGBUFFER_STAGING case..
+   fd_ring->u.cmds = NULL;
+   fd_ring->u.nr_cmds = fd_ring->u.max_cmds = 0;
+
+   fd_ring->u.reloc_bos = NULL;
+   fd_ring->u.nr_reloc_bos = fd_ring->u.max_reloc_bos = 0;
+#ifndef NDEBUG
+   fd_ring->u.assert_bos = NULL;
+   fd_ring->u.nr_assert_bos = fd_ring->u.max_assert_bos = 0;
+#endif
+
+   return ring;
+}
+
+struct fd_ringbuffer *
+fd_ringbuffer_sp_new_object(struct fd_pipe *pipe, uint32_t size)
+{
+   struct fd_device *dev = pipe->dev;
+   struct fd_ringbuffer_sp *fd_ring = malloc(sizeof(*fd_ring));
+
+   /* Lock access to the fd_pipe->suballoc_* since ringbuffer object allocation
+    * can happen both on the frontend (most CSOs) and the driver thread (a6xx
+    * cached tex state, for example)
+    */
+   simple_mtx_lock(&dev->suballoc_lock);
+
+   fd_ring->offset = align(dev->suballoc_offset, SUBALLOC_ALIGNMENT);
+   if (!dev->suballoc_bo ||
+       fd_ring->offset + size > fd_bo_size(dev->suballoc_bo)) {
+      if (dev->suballoc_bo)
+         fd_bo_del(dev->suballoc_bo);
+      dev->suballoc_bo =
+         fd_bo_new_ring(dev, MAX2(SUBALLOC_SIZE, align(size, os_page_size)));
+      fd_ring->offset = 0;
+   }
+
+   fd_ring->u.pipe = pipe;
+   fd_ring->ring_bo = fd_bo_ref(dev->suballoc_bo);
+   fd_ring->base.refcnt = 1;
+   fd_ring->u.last_submit_seqno = 0;
+
+   dev->suballoc_offset = fd_ring->offset + size;
+
+   simple_mtx_unlock(&dev->suballoc_lock);
+
+   return fd_ringbuffer_sp_init(fd_ring, size, _FD_RINGBUFFER_OBJECT);
+}

src/freedreno/drm/freedreno_ringbuffer_sp.h

@@ -0,0 +1,130 @@
+/*
+ * Copyright © 2018 Rob Clark <[email protected]>
+ * SPDX-License-Identifier: MIT
+ *
+ * Authors:
+ *    Rob Clark <[email protected]>
+ */
+
+#ifndef FREEDRENO_RINGBUFFER_SP_H_
+#define FREEDRENO_RINGBUFFER_SP_H_
+
+#include <assert.h>
+#include <inttypes.h>
+#include <pthread.h>
+
+#include "util/hash_table.h"
+#include "util/os_file.h"
+#include "util/slab.h"
+
+#include "freedreno_priv.h"
+#include "freedreno_ringbuffer.h"
+
+/* A "softpin" implementation of submit/ringbuffer, which lowers CPU overhead
+ * by avoiding the additional tracking necessary to build cmds/relocs tables
+ * (but still builds a bos table)
+ */
+
+typedef int (*flush_submit_list_fn)(struct list_head *submit_list);
+
+struct fd_submit_sp {
+   struct fd_submit base;
+
+   DECLARE_ARRAY(struct fd_bo *, bos);
+
+   /* Keep a separate table of sub-alloc BOs.. the backing objects are
+    * tracked in the main bos table (because this is what the kernel
+    * sees), but we need to attach userspace fences to the sub-alloc'd
+    * BOs so the driver knows when they are idle
+    */
+   DECLARE_ARRAY(struct fd_bo *, suballoc_bos);
+
+   /* maps fd_bo to idx in bos table: */
+   struct hash_table *bo_table;
+
+   /* maps fd_bo to idx in suballoc_bos table: */
+   struct hash_table *suballoc_bo_table;
+
+   struct slab_child_pool ring_pool;
+
+   /* Allow for sub-allocation of stateobj ring buffers (ie. sharing
+    * the same underlying bo)..
+    *
+    * We also rely on previous stateobj having been fully constructed
+    * so we can reclaim extra space at it's end.
+    */
+   struct fd_ringbuffer *suballoc_ring;
+
+   /* Flush args, potentially attached to the last submit in the list
+    * of submits to merge:
+    */
+   int in_fence_fd;
+   struct fd_fence *out_fence;
+
+   /* State for enqueued submits:
+    */
+   struct list_head submit_list;   /* includes this submit as last element */
+
+   /* Used by retire_queue, if used by backend: */
+   struct util_queue_fence retire_fence;
+
+   flush_submit_list_fn flush_submit_list;
+
+   uint32_t seqno;
+};
+FD_DEFINE_CAST(fd_submit, fd_submit_sp);
+
+/* for FD_RINGBUFFER_GROWABLE rb's, tracks the 'finalized' cmdstream buffers
+ * and sizes.  Ie. a finalized buffer can have no more commands appended to
+ * it.
+ */
+struct fd_cmd_sp {
+   struct fd_bo *ring_bo;
+   unsigned size;
+};
+
+struct fd_ringbuffer_sp {
+   struct fd_ringbuffer base;
+
+   /* for FD_RINGBUFFER_STREAMING rb's which are sub-allocated */
+   unsigned offset;
+
+   union {
+      /* for _FD_RINGBUFFER_OBJECT case, the array of BOs referenced from
+       * this one
+       */
+      struct {
+         struct fd_pipe *pipe;
+         DECLARE_ARRAY(struct fd_bo *, reloc_bos);
+#ifndef NDEBUG
+         /* BOs to assert are attached to submit: */
+         DECLARE_ARRAY(struct fd_bo *, assert_bos);
+#endif
+
+         /**
+          * The seqno of the last submit we were emitted to.  For stateobjs
+          * it is common to be re-emitted multiple times to the same submit,
+          * we can use this to detect the case.
+          */
+         uint32_t last_submit_seqno;
+      };
+      /* for other cases: */
+      struct {
+         struct fd_submit *submit;
+         DECLARE_ARRAY(struct fd_cmd_sp, cmds);
+      };
+   } u;
+
+   struct fd_bo *ring_bo;
+};
+FD_DEFINE_CAST(fd_ringbuffer, fd_ringbuffer_sp);
+
+void fd_pipe_sp_flush(struct fd_pipe *pipe, uint32_t fence);
+uint32_t fd_submit_append_bo(struct fd_submit_sp *submit, struct fd_bo *bo);
+struct fd_submit *fd_submit_sp_new(struct fd_pipe *pipe,
+                                   flush_submit_list_fn flush_submit_list);
+void fd_pipe_sp_ringpool_init(struct fd_pipe *pipe);
+void fd_pipe_sp_ringpool_fini(struct fd_pipe *pipe);
+struct fd_ringbuffer *fd_ringbuffer_sp_new_object(struct fd_pipe *pipe, uint32_t size);
+
+#endif /* FREEDRENO_RINGBUFFER_SP_H_ */

src/freedreno/drm/freedreno_ringbuffer_sp_reloc.h

@@ -0,0 +1,63 @@
+/*
+ * Copyright © 2021 Google, Inc.
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifdef X
+#undef X
+#endif
+
+#if PTRSZ == 32
+#define X(n) n##_32
+#else
+#define X(n) n##_64
+#endif
+
+static void X(emit_reloc_common)(struct fd_ringbuffer *ring, uint64_t iova)
+{
+#if PTRSZ == 64
+   uint64_t *p64 = (uint64_t *)ring->cur;
+   *p64 = iova;
+   ring->cur += 2;
+#else
+   (*ring->cur++) = (uint32_t)iova;
+#endif
+}
+
+static void X(fd_ringbuffer_sp_emit_reloc_nonobj)(struct fd_ringbuffer *ring,
+                                                  const struct fd_reloc *reloc)
+{
+   X(emit_reloc_common)(ring, reloc->iova);
+   fd_ringbuffer_sp_attach_bo_nonobj(ring, reloc->bo);
+}
+
+static void X(fd_ringbuffer_sp_emit_reloc_obj)(struct fd_ringbuffer *ring,
+                                               const struct fd_reloc *reloc)
+{
+   X(emit_reloc_common)(ring, reloc->iova);
+   fd_ringbuffer_sp_attach_bo_obj(ring, reloc->bo);
+}
+
+static uint32_t X(fd_ringbuffer_sp_emit_reloc_ring_nonobj)(
+   struct fd_ringbuffer *ring, struct fd_ringbuffer *target, uint32_t cmd_idx)
+{
+   uint64_t iova;
+   uint32_t size;
+
+   size = fd_ringbuffer_sp_attach_ring_nonobj(ring, target, cmd_idx, &iova);
+   X(emit_reloc_common)(ring, iova);
+
+   return size;
+}
+
+static uint32_t X(fd_ringbuffer_sp_emit_reloc_ring_obj)(
+   struct fd_ringbuffer *ring, struct fd_ringbuffer *target, uint32_t cmd_idx)
+{
+   uint64_t iova;
+   uint32_t size;
+
+   size = fd_ringbuffer_sp_attach_ring_obj(ring, target, cmd_idx, &iova);
+   X(emit_reloc_common)(ring, iova);
+
+   return size;
+}

src/freedreno/drm/meson.build

@@ -0,0 +1,80 @@
+# Copyright © 2018 Rob Clark
+# SPDX-License-Identifier: MIT
+
+libfreedreno_drm_files = files(
+  'freedreno_bo.c',
+  'freedreno_bo_heap.c',
+  'freedreno_bo_cache.c',
+  'freedreno_device.c',
+  'freedreno_drmif.h',
+  'freedreno_pipe.c',
+  'freedreno_priv.h',
+  'freedreno_ringbuffer.c',
+  'freedreno_ringbuffer.h',
+  'freedreno_ringbuffer_sp.c',
+  'freedreno_ringbuffer_sp.h',
+  'freedreno_ringbuffer_sp_reloc.h',
+)
+libfreedreno_drm_flags = []
+libfreedreno_drm_includes = [
+  inc_freedreno,
+  inc_include,
+  inc_src,
+]
+libfreedreno_drm_deps = [
+  dep_libdrm,
+  dep_valgrind,
+  idep_mesautil,
+]
+libfreedreno_drm_link_with = []
+
+if with_perfetto
+  libfreedreno_drm_deps += dep_perfetto
+  libfreedreno_drm_files += 'freedreno_drm_perfetto.cc'
+endif
+
+# The header file ends up part of the build (but just a stub)
+# in either case:
+libfreedreno_drm_files += 'freedreno_drm_perfetto.h'
+
+libfreedreno_drm_msm_files = files(
+  'msm/msm_bo.c',
+  'msm/msm_device.c',
+  'msm/msm_pipe.c',
+  'msm/msm_priv.h',
+  'msm/msm_ringbuffer.c',
+  'msm/msm_ringbuffer_sp.c',
+)
+libfreedreno_drm_files += libfreedreno_drm_msm_files
+
+libfreedreno_drm_virtio_files = files(
+  'virtio/virtio_bo.c',
+  'virtio/virtio_device.c',
+  'virtio/virtio_pipe.c',
+  'virtio/virtio_priv.h',
+  'virtio/virtio_ringbuffer.c',
+)
+if freedreno_kmds.contains('virtio')
+  libfreedreno_drm_files += libfreedreno_drm_virtio_files
+  libfreedreno_drm_flags += '-DHAVE_FREEDRENO_VIRTIO'
+  libfreedreno_drm_includes += [
+    inc_virtio_gpu,
+    inc_virtio_vdrm,
+  ]
+  libfreedreno_drm_link_with += libvdrm
+endif
+
+libfreedreno_drm = static_library(
+  'freedreno_drm',
+  [
+    libfreedreno_drm_files,
+    freedreno_xml_header_files,
+  ],
+  include_directories : libfreedreno_drm_includes,
+  c_args : [no_override_init_args, libfreedreno_drm_flags],
+  gnu_symbol_visibility : 'hidden',
+  dependencies : libfreedreno_drm_deps,
+  link_with : libfreedreno_drm_link_with,
+  build_by_default : false,
+)
+

src/freedreno/drm/msm/msm_bo.c

@@ -0,0 +1,232 @@
+/*
+ * Copyright © 2012-2018 Rob Clark <[email protected]>
+ * SPDX-License-Identifier: MIT
+ *
+ * Authors:
+ *    Rob Clark <[email protected]>
+ */
+
+#include "msm_priv.h"
+
+static int
+bo_allocate(struct msm_bo *msm_bo)
+{
+   struct fd_bo *bo = &msm_bo->base;
+   if (!msm_bo->offset) {
+      struct drm_msm_gem_info req = {
+         .handle = bo->handle,
+         .info = MSM_INFO_GET_OFFSET,
+      };
+      int ret;
+
+      /* if the buffer is already backed by pages then this
+       * doesn't actually do anything (other than giving us
+       * the offset)
+       */
+      ret =
+         drmCommandWriteRead(bo->dev->fd, DRM_MSM_GEM_INFO, &req, sizeof(req));
+      if (ret) {
+         ERROR_MSG("alloc failed: %s", strerror(errno));
+         return ret;
+      }
+
+      msm_bo->offset = req.value;
+   }
+
+   return 0;
+}
+
+static int
+msm_bo_offset(struct fd_bo *bo, uint64_t *offset)
+{
+   struct msm_bo *msm_bo = to_msm_bo(bo);
+   int ret = bo_allocate(msm_bo);
+   if (ret)
+      return ret;
+   *offset = msm_bo->offset;
+   return 0;
+}
+
+static int
+msm_bo_cpu_prep(struct fd_bo *bo, struct fd_pipe *pipe, uint32_t op)
+{
+   struct drm_msm_gem_cpu_prep req = {
+      .handle = bo->handle,
+      .op = op,
+   };
+
+   get_abs_timeout(&req.timeout, OS_TIMEOUT_INFINITE);
+
+   return drmCommandWrite(bo->dev->fd, DRM_MSM_GEM_CPU_PREP, &req, sizeof(req));
+}
+
+static int
+msm_bo_madvise(struct fd_bo *bo, int willneed)
+{
+   struct drm_msm_gem_madvise req = {
+      .handle = bo->handle,
+      .madv = willneed ? MSM_MADV_WILLNEED : MSM_MADV_DONTNEED,
+   };
+   int ret;
+
+   /* older kernels do not support this: */
+   if (bo->dev->version < FD_VERSION_MADVISE)
+      return willneed;
+
+   ret =
+      drmCommandWriteRead(bo->dev->fd, DRM_MSM_GEM_MADVISE, &req, sizeof(req));
+   if (ret)
+      return ret;
+
+   return req.retained;
+}
+
+static uint64_t
+msm_bo_iova(struct fd_bo *bo)
+{
+   struct drm_msm_gem_info req = {
+      .handle = bo->handle,
+      .info = MSM_INFO_GET_IOVA,
+   };
+   int ret;
+
+   ret = drmCommandWriteRead(bo->dev->fd, DRM_MSM_GEM_INFO, &req, sizeof(req));
+   if (ret)
+      return 0;
+
+   return req.value;
+}
+
+static void
+msm_bo_set_name(struct fd_bo *bo, const char *fmt, va_list ap)
+{
+   struct drm_msm_gem_info req = {
+      .handle = bo->handle,
+      .info = MSM_INFO_SET_NAME,
+   };
+   char buf[32];
+   int sz;
+
+   if (bo->dev->version < FD_VERSION_SOFTPIN)
+      return;
+
+   sz = vsnprintf(buf, sizeof(buf), fmt, ap);
+
+   req.value = VOID2U64(buf);
+   req.len = MIN2(sz, sizeof(buf));
+
+   drmCommandWrite(bo->dev->fd, DRM_MSM_GEM_INFO, &req, sizeof(req));
+}
+
+static void
+msm_bo_set_metadata(struct fd_bo *bo, void *metadata, uint32_t metadata_size)
+{
+   struct drm_msm_gem_info req = {
+      .handle = bo->handle,
+      .info = MSM_INFO_SET_METADATA,
+      .value = (uintptr_t)(void *)metadata,
+      .len = metadata_size,
+   };
+
+   int ret = drmCommandWrite(bo->dev->fd, DRM_MSM_GEM_INFO, &req, sizeof(req));
+   if (ret) {
+      mesa_logw_once("Failed to set BO metadata with DRM_MSM_GEM_INFO: %d",
+                     ret);
+   }
+}
+
+static int
+msm_bo_get_metadata(struct fd_bo *bo, void *metadata, uint32_t metadata_size)
+{
+   struct drm_msm_gem_info req = {
+      .handle = bo->handle,
+      .info = MSM_INFO_GET_METADATA,
+      .value = (uintptr_t)(void *)metadata,
+      .len = metadata_size,
+   };
+
+   int ret = drmCommandWrite(bo->dev->fd, DRM_MSM_GEM_INFO, &req, sizeof(req));
+   if (ret) {
+      mesa_logw_once("Failed to get BO metadata with DRM_MSM_GEM_INFO: %d",
+                     ret);
+   }
+
+   return ret;
+}
+
+static const struct fd_bo_funcs funcs = {
+   .offset = msm_bo_offset,
+   .map = fd_bo_map_os_mmap,
+   .cpu_prep = msm_bo_cpu_prep,
+   .madvise = msm_bo_madvise,
+   .iova = msm_bo_iova,
+   .set_name = msm_bo_set_name,
+   .set_metadata = msm_bo_set_metadata,
+   .get_metadata = msm_bo_get_metadata,
+   .dmabuf = fd_bo_dmabuf_drm,
+   .destroy = fd_bo_fini_common,
+};
+
+/* allocate a buffer handle: */
+static int
+new_handle(struct fd_device *dev, uint32_t size, uint32_t flags, uint32_t *handle)
+{
+   struct drm_msm_gem_new req = {
+      .size = size,
+   };
+   int ret;
+
+   if (flags & FD_BO_SCANOUT)
+      req.flags |= MSM_BO_SCANOUT;
+
+   if (flags & FD_BO_GPUREADONLY)
+      req.flags |= MSM_BO_GPU_READONLY;
+
+   if (flags & FD_BO_CACHED_COHERENT)
+      req.flags |= MSM_BO_CACHED_COHERENT;
+   else
+      req.flags |= MSM_BO_WC;
+
+   ret = drmCommandWriteRead(dev->fd, DRM_MSM_GEM_NEW, &req, sizeof(req));
+   if (ret)
+      return ret;
+
+   *handle = req.handle;
+
+   return 0;
+}
+
+/* allocate a new buffer object */
+struct fd_bo *
+msm_bo_new(struct fd_device *dev, uint32_t size, uint32_t flags)
+{
+   uint32_t handle;
+   int ret;
+
+   ret = new_handle(dev, size, flags, &handle);
+   if (ret)
+      return NULL;
+
+   return msm_bo_from_handle(dev, size, handle);
+}
+
+/* allocate a new buffer object from existing handle (import) */
+struct fd_bo *
+msm_bo_from_handle(struct fd_device *dev, uint32_t size, uint32_t handle)
+{
+   struct msm_bo *msm_bo;
+   struct fd_bo *bo;
+
+   msm_bo = calloc(1, sizeof(*msm_bo));
+   if (!msm_bo)
+      return NULL;
+
+   bo = &msm_bo->base;
+   bo->size = size;
+   bo->handle = handle;
+   bo->funcs = &funcs;
+
+   fd_bo_init_common(bo, dev);
+
+   return bo;
+}

src/freedreno/drm/msm/msm_device.c

@@ -0,0 +1,72 @@
+/*
+ * Copyright © 2012-2018 Rob Clark <[email protected]>
+ * SPDX-License-Identifier: MIT
+ *
+ * Authors:
+ *    Rob Clark <[email protected]>
+ */
+
+#include <unistd.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+
+#include "msm_priv.h"
+
+static void
+msm_device_destroy(struct fd_device *dev)
+{
+}
+
+static const struct fd_device_funcs funcs = {
+   .bo_new = msm_bo_new,
+   .bo_from_handle = msm_bo_from_handle,
+   .handle_from_dmabuf = fd_handle_from_dmabuf_drm,
+   .bo_from_dmabuf = fd_bo_from_dmabuf_drm,
+   .bo_close_handle = fd_bo_close_handle_drm,
+   .pipe_new = msm_pipe_new,
+   .destroy = msm_device_destroy,
+};
+
+struct fd_device *
+msm_device_new(int fd, drmVersionPtr version)
+{
+   struct msm_device *msm_dev;
+   struct fd_device *dev;
+
+   STATIC_ASSERT(FD_BO_PREP_READ == MSM_PREP_READ);
+   STATIC_ASSERT(FD_BO_PREP_WRITE == MSM_PREP_WRITE);
+   STATIC_ASSERT(FD_BO_PREP_NOSYNC == MSM_PREP_NOSYNC);
+
+   msm_dev = calloc(1, sizeof(*msm_dev));
+   if (!msm_dev)
+      return NULL;
+
+   dev = &msm_dev->base;
+   dev->funcs = &funcs;
+   dev->version = version->version_minor;
+
+   if (version->version_minor >= FD_VERSION_CACHED_COHERENT) {
+      struct drm_msm_gem_new new_req = {
+         .size = os_page_size,
+         .flags = MSM_BO_CACHED_COHERENT,
+      };
+
+      /* The kernel is new enough to support MSM_BO_CACHED_COHERENT,
+       * but that is not a guarantee that the device we are running
+       * on supports it.  So do a test allocation to find out.
+       */
+      if (!drmCommandWriteRead(fd, DRM_MSM_GEM_NEW,
+                               &new_req, sizeof(new_req))) {
+         struct drm_gem_close close_req = {
+            .handle = new_req.handle,
+         };
+         drmIoctl(fd, DRM_IOCTL_GEM_CLOSE, &close_req);
+
+         dev->has_cached_coherent = true;
+      }
+   }
+
+   dev->bo_size = sizeof(struct msm_bo);
+
+   return dev;
+}

src/freedreno/drm/msm/msm_pipe.c

@@ -0,0 +1,295 @@
+/*
+ * Copyright © 2012-2018 Rob Clark <[email protected]>
+ * SPDX-License-Identifier: MIT
+ *
+ * Authors:
+ *    Rob Clark <[email protected]>
+ */
+
+#include "util/slab.h"
+
+#include "freedreno_ringbuffer_sp.h"
+#include "msm_priv.h"
+
+static int
+query_param(struct fd_pipe *pipe, uint32_t param, uint64_t *value)
+{
+   struct msm_pipe *msm_pipe = to_msm_pipe(pipe);
+   struct drm_msm_param req = {
+      .pipe = msm_pipe->pipe,
+      .param = param,
+   };
+   int ret;
+
+   ret =
+      drmCommandWriteRead(pipe->dev->fd, DRM_MSM_GET_PARAM, &req, sizeof(req));
+   if (ret)
+      return ret;
+
+   *value = req.value;
+
+   return 0;
+}
+
+static int
+query_queue_param(struct fd_pipe *pipe, uint32_t param, uint64_t *value)
+{
+   struct msm_pipe *msm_pipe = to_msm_pipe(pipe);
+   struct drm_msm_submitqueue_query req = {
+      .data = VOID2U64(value),
+      .id = msm_pipe->queue_id,
+      .param = param,
+      .len = sizeof(*value),
+   };
+   int ret;
+
+   ret = drmCommandWriteRead(pipe->dev->fd, DRM_MSM_SUBMITQUEUE_QUERY, &req,
+                             sizeof(req));
+   if (ret)
+      return ret;
+
+   return 0;
+}
+
+static int
+msm_pipe_get_param(struct fd_pipe *pipe, enum fd_param_id param,
+                   uint64_t *value)
+{
+   struct msm_pipe *msm_pipe = to_msm_pipe(pipe);
+   switch (param) {
+   case FD_DEVICE_ID: // XXX probably get rid of this..
+   case FD_GPU_ID:
+      *value = msm_pipe->gpu_id;
+      return 0;
+   case FD_GMEM_SIZE:
+      *value = msm_pipe->gmem;
+      return 0;
+   case FD_GMEM_BASE:
+      *value = msm_pipe->gmem_base;
+      return 0;
+   case FD_CHIP_ID:
+      *value = msm_pipe->chip_id;
+      return 0;
+   case FD_MAX_FREQ:
+      return query_param(pipe, MSM_PARAM_MAX_FREQ, value);
+   case FD_TIMESTAMP:
+      return query_param(pipe, MSM_PARAM_TIMESTAMP, value);
+   case FD_NR_PRIORITIES:
+      return query_param(pipe, MSM_PARAM_PRIORITIES, value);
+   case FD_CTX_FAULTS:
+      return query_queue_param(pipe, MSM_SUBMITQUEUE_PARAM_FAULTS, value);
+   case FD_GLOBAL_FAULTS:
+      return query_param(pipe, MSM_PARAM_FAULTS, value);
+   case FD_SUSPEND_COUNT:
+      return query_param(pipe, MSM_PARAM_SUSPENDS, value);
+   case FD_VA_SIZE:
+      return query_param(pipe, MSM_PARAM_VA_SIZE, value);
+   case FD_UCHE_TRAP_BASE:
+      return query_param(pipe, MSM_PARAM_UCHE_TRAP_BASE, value);
+   default:
+      ERROR_MSG("invalid param id: %d", param);
+      return -1;
+   }
+}
+
+static int
+set_param(struct fd_pipe *pipe, uint32_t param, uint64_t value)
+{
+   struct msm_pipe *msm_pipe = to_msm_pipe(pipe);
+   struct drm_msm_param req = {
+      .pipe  = msm_pipe->pipe,
+      .param = param,
+      .value = value,
+   };
+
+   return drmCommandWriteRead(pipe->dev->fd, DRM_MSM_SET_PARAM,
+                              &req, sizeof(req));
+}
+
+static int
+msm_pipe_set_param(struct fd_pipe *pipe, enum fd_param_id param, uint64_t value)
+{
+   switch (param) {
+   case FD_SYSPROF:
+      return set_param(pipe, MSM_PARAM_SYSPROF, value);
+   default:
+      ERROR_MSG("invalid param id: %d", param);
+      return -1;
+   }
+}
+
+static int
+msm_pipe_wait(struct fd_pipe *pipe, const struct fd_fence *fence, uint64_t timeout)
+{
+   struct fd_device *dev = pipe->dev;
+   struct drm_msm_wait_fence req = {
+      .fence = fence->kfence,
+      .queueid = to_msm_pipe(pipe)->queue_id,
+   };
+   int ret;
+
+   get_abs_timeout(&req.timeout, timeout);
+
+   ret = drmCommandWrite(dev->fd, DRM_MSM_WAIT_FENCE, &req, sizeof(req));
+   if (ret && (ret != -ETIMEDOUT)) {
+      ERROR_MSG("wait-fence failed! %d (%s)", ret, strerror(errno));
+   }
+
+   return ret;
+}
+
+static int
+__open_submitqueue(struct fd_pipe *pipe, uint32_t prio, uint32_t flags)
+{
+   struct drm_msm_submitqueue req = {
+      .flags = flags,
+      .prio = prio,
+   };
+   uint64_t nr_prio = 1;
+   int ret;
+
+   if (fd_device_version(pipe->dev) < FD_VERSION_SUBMIT_QUEUES) {
+      to_msm_pipe(pipe)->queue_id = 0;
+      return 0;
+   }
+
+   msm_pipe_get_param(pipe, FD_NR_PRIORITIES, &nr_prio);
+
+   req.prio = MIN2(req.prio, MAX2(nr_prio, 1) - 1);
+
+   ret = drmCommandWriteRead(pipe->dev->fd, DRM_MSM_SUBMITQUEUE_NEW, &req,
+                             sizeof(req));
+   if (ret)
+      return ret;
+
+   to_msm_pipe(pipe)->queue_id = req.id;
+   return 0;
+}
+
+static int
+open_submitqueue(struct fd_pipe *pipe, uint32_t prio)
+{
+   const struct fd_dev_info *info = fd_dev_info_raw(&pipe->dev_id);
+   int ret = -1;
+
+   if (info && info->chip >= A7XX)
+      ret = __open_submitqueue(pipe, prio, MSM_SUBMITQUEUE_ALLOW_PREEMPT);
+
+   /* If kernel doesn't support preemption, try again without: */
+   if (ret)
+      ret = __open_submitqueue(pipe, prio, 0);
+
+   if (ret) {
+      ERROR_MSG("could not create submitqueue! %d (%s)", ret, strerror(errno));
+      return ret;
+   }
+
+   return 0;
+}
+
+static void
+close_submitqueue(struct fd_pipe *pipe, uint32_t queue_id)
+{
+   if (fd_device_version(pipe->dev) < FD_VERSION_SUBMIT_QUEUES)
+      return;
+
+   drmCommandWrite(pipe->dev->fd, DRM_MSM_SUBMITQUEUE_CLOSE, &queue_id,
+                   sizeof(queue_id));
+}
+
+static void
+msm_pipe_destroy(struct fd_pipe *pipe)
+{
+   struct msm_pipe *msm_pipe = to_msm_pipe(pipe);
+
+   close_submitqueue(pipe, msm_pipe->queue_id);
+   fd_pipe_sp_ringpool_fini(pipe);
+   free(msm_pipe);
+}
+
+static const struct fd_pipe_funcs sp_funcs = {
+   .ringbuffer_new_object = fd_ringbuffer_sp_new_object,
+   .submit_new = msm_submit_sp_new,
+   .flush = fd_pipe_sp_flush,
+   .get_param = msm_pipe_get_param,
+   .set_param = msm_pipe_set_param,
+   .wait = msm_pipe_wait,
+   .destroy = msm_pipe_destroy,
+};
+
+static const struct fd_pipe_funcs legacy_funcs = {
+   .ringbuffer_new_object = msm_ringbuffer_new_object,
+   .submit_new = msm_submit_new,
+   .get_param = msm_pipe_get_param,
+   .set_param = msm_pipe_set_param,
+   .wait = msm_pipe_wait,
+   .destroy = msm_pipe_destroy,
+};
+
+static uint64_t
+get_param(struct fd_pipe *pipe, uint32_t param)
+{
+   uint64_t value;
+   int ret = query_param(pipe, param, &value);
+   if (ret) {
+      ERROR_MSG("get-param failed! %d (%s)", ret, strerror(errno));
+      return 0;
+   }
+   return value;
+}
+
+struct fd_pipe *
+msm_pipe_new(struct fd_device *dev, enum fd_pipe_id id, uint32_t prio)
+{
+   static const uint32_t pipe_id[] = {
+      [FD_PIPE_3D] = MSM_PIPE_3D0,
+      [FD_PIPE_2D] = MSM_PIPE_2D0,
+   };
+   struct msm_pipe *msm_pipe = NULL;
+   struct fd_pipe *pipe = NULL;
+
+   msm_pipe = calloc(1, sizeof(*msm_pipe));
+   if (!msm_pipe) {
+      ERROR_MSG("allocation failed");
+      goto fail;
+   }
+
+   pipe = &msm_pipe->base;
+
+   if (fd_device_version(dev) >= FD_VERSION_SOFTPIN) {
+      pipe->funcs = &sp_funcs;
+   } else {
+      pipe->funcs = &legacy_funcs;
+   }
+
+   /* initialize before get_param(): */
+   pipe->dev = dev;
+   msm_pipe->pipe = pipe_id[id];
+
+   /* these params should be supported since the first version of drm/msm: */
+   msm_pipe->gpu_id = get_param(pipe, MSM_PARAM_GPU_ID);
+   msm_pipe->gmem = get_param(pipe, MSM_PARAM_GMEM_SIZE);
+   msm_pipe->chip_id = get_param(pipe, MSM_PARAM_CHIP_ID);
+
+   if (fd_device_version(pipe->dev) >= FD_VERSION_GMEM_BASE)
+      msm_pipe->gmem_base = get_param(pipe, MSM_PARAM_GMEM_BASE);
+
+   if (!(msm_pipe->gpu_id || msm_pipe->chip_id))
+      goto fail;
+
+   INFO_MSG("Pipe Info:");
+   INFO_MSG(" GPU-id:          %d", msm_pipe->gpu_id);
+   INFO_MSG(" Chip-id:         0x%016"PRIx64, msm_pipe->chip_id);
+   INFO_MSG(" GMEM size:       0x%08x", msm_pipe->gmem);
+
+   if (open_submitqueue(pipe, prio))
+      goto fail;
+
+   fd_pipe_sp_ringpool_init(pipe);
+
+   return pipe;
+fail:
+   if (pipe)
+      fd_pipe_del(pipe);
+   return NULL;
+}

src/freedreno/drm/msm/msm_priv.h

@@ -0,0 +1,188 @@
+/*
+ * Copyright © 2012-2018 Rob Clark <[email protected]>
+ * SPDX-License-Identifier: MIT
+ *
+ * Authors:
+ *    Rob Clark <[email protected]>
+ */
+
+#ifndef MSM_PRIV_H_
+#define MSM_PRIV_H_
+
+#include "freedreno_drmif.h"
+#include "freedreno_priv.h"
+#include "freedreno_rd_output.h"
+
+#include "util/timespec.h"
+#include "util/u_process.h"
+
+#ifndef __user
+#define __user
+#endif
+
+#include "drm-uapi/msm_drm.h"
+
+struct msm_device {
+   struct fd_device base;
+};
+FD_DEFINE_CAST(fd_device, msm_device);
+
+struct fd_device *msm_device_new(int fd, drmVersionPtr version);
+
+struct msm_pipe {
+   struct fd_pipe base;
+   uint32_t pipe;
+   uint32_t gpu_id;
+   uint64_t chip_id;
+   uint64_t gmem_base;
+   uint32_t gmem;
+   uint32_t queue_id;
+};
+FD_DEFINE_CAST(fd_pipe, msm_pipe);
+
+struct fd_pipe *msm_pipe_new(struct fd_device *dev, enum fd_pipe_id id,
+                             uint32_t prio);
+
+struct fd_ringbuffer *msm_ringbuffer_new_object(struct fd_pipe *pipe,
+                                                uint32_t size);
+
+struct fd_submit *msm_submit_new(struct fd_pipe *pipe);
+struct fd_submit *msm_submit_sp_new(struct fd_pipe *pipe);
+
+struct msm_bo {
+   struct fd_bo base;
+   uint64_t offset;
+};
+FD_DEFINE_CAST(fd_bo, msm_bo);
+
+struct fd_bo *msm_bo_new(struct fd_device *dev, uint32_t size, uint32_t flags);
+struct fd_bo *msm_bo_from_handle(struct fd_device *dev, uint32_t size,
+                                 uint32_t handle);
+
+static inline void
+msm_dump_submit(struct drm_msm_gem_submit *req)
+{
+   for (unsigned i = 0; i < req->nr_bos; i++) {
+      struct drm_msm_gem_submit_bo *bos = U642VOID(req->bos);
+      struct drm_msm_gem_submit_bo *bo = &bos[i];
+      ERROR_MSG("  bos[%d]: handle=%u, flags=%x", i, bo->handle, bo->flags);
+   }
+   for (unsigned i = 0; i < req->nr_cmds; i++) {
+      struct drm_msm_gem_submit_cmd *cmds = U642VOID(req->cmds);
+      struct drm_msm_gem_submit_cmd *cmd = &cmds[i];
+      struct drm_msm_gem_submit_reloc *relocs = U642VOID(cmd->relocs);
+      ERROR_MSG("  cmd[%d]: type=%u, submit_idx=%u, submit_offset=%u, size=%u",
+                i, cmd->type, cmd->submit_idx, cmd->submit_offset, cmd->size);
+      for (unsigned j = 0; j < cmd->nr_relocs; j++) {
+         struct drm_msm_gem_submit_reloc *r = &relocs[j];
+         ERROR_MSG(
+            "    reloc[%d]: submit_offset=%u, or=%08x, shift=%d, reloc_idx=%u"
+            ", reloc_offset=%" PRIu64,
+            j, r->submit_offset, r->or, r->shift, r->reloc_idx,
+            (uint64_t)r->reloc_offset);
+      }
+   }
+}
+
+static inline bool
+__should_dump(struct fd_bo *bo)
+{
+   return (bo->reloc_flags & FD_RELOC_DUMP) || FD_RD_DUMP(FULL);
+}
+
+static inline void
+__snapshot_buf(struct fd_rd_output *rd, struct fd_bo *bo, uint64_t iova,
+               uint32_t size, bool full)
+{
+   uint64_t offset = 0;
+
+   if (iova) {
+      offset = iova - fd_bo_get_iova(bo);
+   } else {
+      iova = fd_bo_get_iova(bo);
+      size = bo->size;
+   }
+
+   fd_rd_output_write_section(rd, RD_GPUADDR, (uint32_t[]){
+      iova, size, iova >> 32
+   }, 12);
+
+   if (!full)
+      return;
+
+   const char *buf = __fd_bo_map(bo);
+   buf += offset;
+   fd_rd_output_write_section(rd, RD_BUFFER_CONTENTS, buf, size);
+}
+
+static inline void
+msm_dump_rd(struct fd_pipe *pipe, struct drm_msm_gem_submit *req)
+{
+   struct fd_rd_output *rd = &pipe->dev->rd;
+
+   if (!fd_rd_dump_env.flags || !req->nr_cmds ||
+       !fd_rd_output_begin(rd, UINT_MAX, req->fence))
+      return;
+
+   if (FD_RD_DUMP(FULL)) {
+      fd_pipe_wait(pipe, &(struct fd_fence) {
+         /* this is cheating a bit, but msm_pipe_wait only needs kfence */
+         .kfence = req->fence,
+      });
+   }
+
+   const char *procname = util_get_process_name();
+   fd_rd_output_write_section(rd, RD_CHIP_ID, &to_msm_pipe(pipe)->chip_id, 8);
+   fd_rd_output_write_section(rd, RD_CMD, procname, strlen(procname));
+
+   struct drm_msm_gem_submit_bo *bos = U642VOID(req->bos);
+   struct drm_msm_gem_submit_cmd *cmds = U642VOID(req->cmds);
+
+   for (unsigned i = 0; i < req->nr_bos; i++) {
+      /* This size param to fd_bo_from_handle() only matters if the bo isn't already in
+       * the handle table.  Which it should be.
+       */
+      struct fd_bo *bo = fd_bo_from_handle(pipe->dev, bos[i].handle, 0);
+
+      __snapshot_buf(rd, bo, 0, 0, __should_dump(bo));
+
+      fd_bo_del(bo);
+   }
+
+   for (unsigned i = 0; i < req->nr_cmds; i++) {
+      struct drm_msm_gem_submit_cmd *cmd = &cmds[i];
+      struct fd_bo *bo = fd_bo_from_handle(pipe->dev, bos[cmd->submit_idx].handle, 0);
+      uint64_t iova = fd_bo_get_iova(bo) + cmd->submit_offset;
+
+      /* snapshot cmdstream bo's (if we haven't already): */
+      if (!__should_dump(bo))
+         __snapshot_buf(rd, bo, iova, cmd->size, true);
+
+      fd_rd_output_write_section(rd, RD_CMDSTREAM_ADDR, (uint32_t[]){
+         iova, cmd->size >> 2, iova >> 32
+      }, 12);
+
+      fd_bo_del(bo);
+   }
+
+   fd_rd_output_end(rd);
+}
+
+static inline void
+get_abs_timeout(struct drm_msm_timespec *tv, uint64_t ns)
+{
+   struct timespec t;
+
+   if (ns == OS_TIMEOUT_INFINITE)
+      ns = 3600ULL * NSEC_PER_SEC; /* 1 hour timeout is almost infinite */
+
+   clock_gettime(CLOCK_MONOTONIC, &t);
+   tv->tv_sec = t.tv_sec + ns / NSEC_PER_SEC;
+   tv->tv_nsec = t.tv_nsec + ns % NSEC_PER_SEC;
+   if (tv->tv_nsec >= NSEC_PER_SEC) { /* handle nsec overflow */
+      tv->tv_nsec -= NSEC_PER_SEC;
+      tv->tv_sec++;
+   }
+}
+
+#endif /* MSM_PRIV_H_ */

src/freedreno/drm/msm/msm_ringbuffer.c

@@ -0,0 +1,686 @@
+/*
+ * Copyright © 2012-2018 Rob Clark <[email protected]>
+ * SPDX-License-Identifier: MIT
+ *
+ * Authors:
+ *    Rob Clark <[email protected]>
+ */
+
+#include <assert.h>
+#include <inttypes.h>
+
+#include "util/hash_table.h"
+#include "util/set.h"
+#include "util/slab.h"
+
+#include "drm/freedreno_ringbuffer.h"
+#include "msm_priv.h"
+
+/* The legacy implementation of submit/ringbuffer, which still does the
+ * traditional reloc and cmd tracking
+ */
+
+#define INIT_SIZE 0x1000
+
+struct msm_submit {
+   struct fd_submit base;
+
+   DECLARE_ARRAY(struct drm_msm_gem_submit_bo, submit_bos);
+   DECLARE_ARRAY(struct fd_bo *, bos);
+
+   /* maps fd_bo to idx in bos table: */
+   struct hash_table *bo_table;
+
+   struct slab_mempool ring_pool;
+
+   /* hash-set of associated rings: */
+   struct set *ring_set;
+
+   /* Allow for sub-allocation of stateobj ring buffers (ie. sharing
+    * the same underlying bo)..
+    *
+    * We also rely on previous stateobj having been fully constructed
+    * so we can reclaim extra space at it's end.
+    */
+   struct fd_ringbuffer *suballoc_ring;
+};
+FD_DEFINE_CAST(fd_submit, msm_submit);
+
+/* for FD_RINGBUFFER_GROWABLE rb's, tracks the 'finalized' cmdstream buffers
+ * and sizes.  Ie. a finalized buffer can have no more commands appended to
+ * it.
+ */
+struct msm_cmd {
+   struct fd_bo *ring_bo;
+   unsigned size;
+   DECLARE_ARRAY(struct drm_msm_gem_submit_reloc, relocs);
+};
+
+static struct msm_cmd *
+cmd_new(struct fd_bo *ring_bo)
+{
+   struct msm_cmd *cmd = malloc(sizeof(*cmd));
+   cmd->ring_bo = fd_bo_ref(ring_bo);
+   cmd->size = 0;
+   cmd->nr_relocs = cmd->max_relocs = 0;
+   cmd->relocs = NULL;
+   return cmd;
+}
+
+static void
+cmd_free(struct msm_cmd *cmd)
+{
+   fd_bo_del(cmd->ring_bo);
+   free(cmd->relocs);
+   free(cmd);
+}
+
+struct msm_ringbuffer {
+   struct fd_ringbuffer base;
+
+   /* for FD_RINGBUFFER_STREAMING rb's which are sub-allocated */
+   unsigned offset;
+
+   union {
+      /* for _FD_RINGBUFFER_OBJECT case: */
+      struct {
+         struct fd_pipe *pipe;
+         DECLARE_ARRAY(struct fd_bo *, reloc_bos);
+         struct set *ring_set;
+      };
+      /* for other cases: */
+      struct {
+         struct fd_submit *submit;
+         DECLARE_ARRAY(struct msm_cmd *, cmds);
+      };
+   } u;
+
+   struct msm_cmd *cmd; /* current cmd */
+   struct fd_bo *ring_bo;
+};
+FD_DEFINE_CAST(fd_ringbuffer, msm_ringbuffer);
+
+static void finalize_current_cmd(struct fd_ringbuffer *ring);
+static struct fd_ringbuffer *
+msm_ringbuffer_init(struct msm_ringbuffer *msm_ring, uint32_t size,
+                    enum fd_ringbuffer_flags flags);
+
+/* add (if needed) bo to submit and return index: */
+static uint32_t
+append_bo(struct msm_submit *submit, struct fd_bo *bo)
+{
+   uint32_t idx;
+
+   /* NOTE: it is legal to use the same bo on different threads for
+    * different submits.  But it is not legal to use the same submit
+    * from given threads.
+    */
+   idx = READ_ONCE(bo->idx);
+
+   if (unlikely((idx >= submit->nr_submit_bos) ||
+                (submit->submit_bos[idx].handle != bo->handle))) {
+      uint32_t hash = _mesa_hash_pointer(bo);
+      struct hash_entry *entry;
+
+      entry = _mesa_hash_table_search_pre_hashed(submit->bo_table, hash, bo);
+      if (entry) {
+         /* found */
+         idx = (uint32_t)(uintptr_t)entry->data;
+      } else {
+         idx = APPEND(
+            submit, submit_bos,
+            (struct drm_msm_gem_submit_bo){
+               .flags = bo->reloc_flags & (MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE),
+               .handle = bo->handle,
+               .presumed = 0,
+            });
+         APPEND(submit, bos, fd_bo_ref(bo));
+
+         _mesa_hash_table_insert_pre_hashed(submit->bo_table, hash, bo,
+                                            (void *)(uintptr_t)idx);
+      }
+      bo->idx = idx;
+   }
+
+   return idx;
+}
+
+static void
+append_ring(struct set *set, struct fd_ringbuffer *ring)
+{
+   uint32_t hash = _mesa_hash_pointer(ring);
+
+   if (!_mesa_set_search_pre_hashed(set, hash, ring)) {
+      fd_ringbuffer_ref(ring);
+      _mesa_set_add_pre_hashed(set, hash, ring);
+   }
+}
+
+static void
+msm_submit_suballoc_ring_bo(struct fd_submit *submit,
+                            struct msm_ringbuffer *msm_ring, uint32_t size)
+{
+   struct msm_submit *msm_submit = to_msm_submit(submit);
+   unsigned suballoc_offset = 0;
+   struct fd_bo *suballoc_bo = NULL;
+
+   if (msm_submit->suballoc_ring) {
+      struct msm_ringbuffer *suballoc_ring =
+         to_msm_ringbuffer(msm_submit->suballoc_ring);
+
+      suballoc_bo = suballoc_ring->ring_bo;
+      suballoc_offset =
+         fd_ringbuffer_size(msm_submit->suballoc_ring) + suballoc_ring->offset;
+
+      suballoc_offset = align(suballoc_offset, 0x10);
+
+      if ((size + suballoc_offset) > suballoc_bo->size) {
+         suballoc_bo = NULL;
+      }
+   }
+
+   if (!suballoc_bo) {
+      // TODO possibly larger size for streaming bo?
+      msm_ring->ring_bo = fd_bo_new_ring(submit->pipe->dev, 0x8000);
+      msm_ring->offset = 0;
+   } else {
+      msm_ring->ring_bo = fd_bo_ref(suballoc_bo);
+      msm_ring->offset = suballoc_offset;
+   }
+
+   struct fd_ringbuffer *old_suballoc_ring = msm_submit->suballoc_ring;
+
+   msm_submit->suballoc_ring = fd_ringbuffer_ref(&msm_ring->base);
+
+   if (old_suballoc_ring)
+      fd_ringbuffer_del(old_suballoc_ring);
+}
+
+static struct fd_ringbuffer *
+msm_submit_new_ringbuffer(struct fd_submit *submit, uint32_t size,
+                          enum fd_ringbuffer_flags flags)
+{
+   struct msm_submit *msm_submit = to_msm_submit(submit);
+   struct msm_ringbuffer *msm_ring;
+
+   msm_ring = slab_alloc_st(&msm_submit->ring_pool);
+
+   msm_ring->u.submit = submit;
+
+   /* NOTE: needs to be before _suballoc_ring_bo() since it could
+    * increment the refcnt of the current ring
+    */
+   msm_ring->base.refcnt = 1;
+
+   if (flags & FD_RINGBUFFER_STREAMING) {
+      msm_submit_suballoc_ring_bo(submit, msm_ring, size);
+   } else {
+      if (flags & FD_RINGBUFFER_GROWABLE)
+         size = INIT_SIZE;
+
+      msm_ring->offset = 0;
+      msm_ring->ring_bo = fd_bo_new_ring(submit->pipe->dev, size);
+   }
+
+   if (!msm_ringbuffer_init(msm_ring, size, flags))
+      return NULL;
+
+   return &msm_ring->base;
+}
+
+static struct drm_msm_gem_submit_reloc *
+handle_stateobj_relocs(struct msm_submit *submit, struct msm_ringbuffer *ring)
+{
+   struct msm_cmd *cmd = ring->cmd;
+   struct drm_msm_gem_submit_reloc *relocs;
+
+   relocs = malloc(cmd->nr_relocs * sizeof(*relocs));
+
+   for (unsigned i = 0; i < cmd->nr_relocs; i++) {
+      unsigned idx = cmd->relocs[i].reloc_idx;
+      struct fd_bo *bo = ring->u.reloc_bos[idx];
+
+      relocs[i] = cmd->relocs[i];
+      relocs[i].reloc_idx = append_bo(submit, bo);
+   }
+
+   return relocs;
+}
+
+static struct fd_fence *
+msm_submit_flush(struct fd_submit *submit, int in_fence_fd, bool use_fence_fd)
+{
+   struct msm_submit *msm_submit = to_msm_submit(submit);
+   struct msm_pipe *msm_pipe = to_msm_pipe(submit->pipe);
+   struct drm_msm_gem_submit req = {
+      .flags = msm_pipe->pipe,
+      .queueid = msm_pipe->queue_id,
+   };
+   int ret;
+
+   finalize_current_cmd(submit->primary);
+   append_ring(msm_submit->ring_set, submit->primary);
+
+   unsigned nr_cmds = 0;
+   unsigned nr_objs = 0;
+
+   set_foreach (msm_submit->ring_set, entry) {
+      struct fd_ringbuffer *ring = (void *)entry->key;
+      if (ring->flags & _FD_RINGBUFFER_OBJECT) {
+         nr_cmds += 1;
+         nr_objs += 1;
+      } else {
+         if (ring != submit->primary)
+            finalize_current_cmd(ring);
+         nr_cmds += to_msm_ringbuffer(ring)->u.nr_cmds;
+      }
+   }
+
+   void *obj_relocs[nr_objs];
+   struct drm_msm_gem_submit_cmd cmds[nr_cmds];
+   unsigned i = 0, o = 0;
+
+   set_foreach (msm_submit->ring_set, entry) {
+      struct fd_ringbuffer *ring = (void *)entry->key;
+      struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
+
+      assert(i < nr_cmds);
+
+      // TODO handle relocs:
+      if (ring->flags & _FD_RINGBUFFER_OBJECT) {
+
+         assert(o < nr_objs);
+
+         void *relocs = handle_stateobj_relocs(msm_submit, msm_ring);
+         obj_relocs[o++] = relocs;
+
+         cmds[i].type = MSM_SUBMIT_CMD_IB_TARGET_BUF;
+         cmds[i].submit_idx = append_bo(msm_submit, msm_ring->ring_bo);
+         cmds[i].submit_offset = submit_offset(msm_ring->ring_bo, msm_ring->offset);
+         cmds[i].size = offset_bytes(ring->cur, ring->start);
+         cmds[i].pad = 0;
+         cmds[i].nr_relocs = msm_ring->cmd->nr_relocs;
+         cmds[i].relocs = VOID2U64(relocs);
+
+         i++;
+      } else {
+         for (unsigned j = 0; j < msm_ring->u.nr_cmds; j++) {
+            if (ring->flags & FD_RINGBUFFER_PRIMARY) {
+               cmds[i].type = MSM_SUBMIT_CMD_BUF;
+            } else {
+               cmds[i].type = MSM_SUBMIT_CMD_IB_TARGET_BUF;
+            }
+            struct fd_bo *ring_bo = msm_ring->u.cmds[j]->ring_bo;
+            cmds[i].submit_idx = append_bo(msm_submit, ring_bo);
+            cmds[i].submit_offset = submit_offset(ring_bo, msm_ring->offset);
+            cmds[i].size = msm_ring->u.cmds[j]->size;
+            cmds[i].pad = 0;
+            cmds[i].nr_relocs = msm_ring->u.cmds[j]->nr_relocs;
+            cmds[i].relocs = VOID2U64(msm_ring->u.cmds[j]->relocs);
+
+            i++;
+         }
+      }
+   }
+
+   struct fd_fence *out_fence = fd_fence_new(submit->pipe, use_fence_fd);
+
+   simple_mtx_lock(&fence_lock);
+   for (unsigned j = 0; j < msm_submit->nr_bos; j++) {
+      fd_bo_add_fence(msm_submit->bos[j], out_fence);
+   }
+   simple_mtx_unlock(&fence_lock);
+
+   if (in_fence_fd != -1) {
+      req.flags |= MSM_SUBMIT_FENCE_FD_IN | MSM_SUBMIT_NO_IMPLICIT;
+      req.fence_fd = in_fence_fd;
+   }
+
+   if (out_fence->use_fence_fd) {
+      req.flags |= MSM_SUBMIT_FENCE_FD_OUT;
+   }
+
+   /* needs to be after get_cmd() as that could create bos/cmds table: */
+   req.bos = VOID2U64(msm_submit->submit_bos),
+   req.nr_bos = msm_submit->nr_submit_bos;
+   req.cmds = VOID2U64(cmds), req.nr_cmds = nr_cmds;
+
+   DEBUG_MSG("nr_cmds=%u, nr_bos=%u", req.nr_cmds, req.nr_bos);
+
+   ret = drmCommandWriteRead(submit->pipe->dev->fd, DRM_MSM_GEM_SUBMIT, &req,
+                             sizeof(req));
+   if (ret) {
+      ERROR_MSG("submit failed: %d (%s)", ret, strerror(errno));
+      fd_fence_del(out_fence);
+      out_fence = NULL;
+      msm_dump_submit(&req);
+   } else if (!ret && out_fence) {
+      out_fence->kfence = req.fence;
+      out_fence->ufence = submit->fence;
+      out_fence->fence_fd = req.fence_fd;
+   }
+
+   for (unsigned o = 0; o < nr_objs; o++)
+      free(obj_relocs[o]);
+
+   return out_fence;
+}
+
+static void
+unref_rings(struct set_entry *entry)
+{
+   struct fd_ringbuffer *ring = (void *)entry->key;
+   fd_ringbuffer_del(ring);
+}
+
+static void
+msm_submit_destroy(struct fd_submit *submit)
+{
+   struct msm_submit *msm_submit = to_msm_submit(submit);
+
+   if (msm_submit->suballoc_ring)
+      fd_ringbuffer_del(msm_submit->suballoc_ring);
+
+   _mesa_hash_table_destroy(msm_submit->bo_table, NULL);
+   _mesa_set_destroy(msm_submit->ring_set, unref_rings);
+
+   // TODO it would be nice to have a way to assert() if all
+   // rb's haven't been free'd back to the slab, because that is
+   // an indication that we are leaking bo's
+   slab_destroy(&msm_submit->ring_pool);
+
+   for (unsigned i = 0; i < msm_submit->nr_bos; i++)
+      fd_bo_del(msm_submit->bos[i]);
+
+   free(msm_submit->submit_bos);
+   free(msm_submit->bos);
+   free(msm_submit);
+}
+
+static const struct fd_submit_funcs submit_funcs = {
+   .new_ringbuffer = msm_submit_new_ringbuffer,
+   .flush = msm_submit_flush,
+   .destroy = msm_submit_destroy,
+};
+
+struct fd_submit *
+msm_submit_new(struct fd_pipe *pipe)
+{
+   struct msm_submit *msm_submit = calloc(1, sizeof(*msm_submit));
+   struct fd_submit *submit;
+
+   msm_submit->bo_table = _mesa_hash_table_create(NULL, _mesa_hash_pointer,
+                                                  _mesa_key_pointer_equal);
+   msm_submit->ring_set =
+      _mesa_set_create(NULL, _mesa_hash_pointer, _mesa_key_pointer_equal);
+   // TODO tune size:
+   slab_create(&msm_submit->ring_pool, sizeof(struct msm_ringbuffer), 16);
+
+   submit = &msm_submit->base;
+   submit->funcs = &submit_funcs;
+
+   return submit;
+}
+
+static void
+finalize_current_cmd(struct fd_ringbuffer *ring)
+{
+   struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
+
+   assert(!(ring->flags & _FD_RINGBUFFER_OBJECT));
+
+   if (!msm_ring->cmd)
+      return;
+
+   assert(msm_ring->cmd->ring_bo == msm_ring->ring_bo);
+
+   msm_ring->cmd->size = offset_bytes(ring->cur, ring->start);
+   APPEND(&msm_ring->u, cmds, msm_ring->cmd);
+   msm_ring->cmd = NULL;
+}
+
+static void
+msm_ringbuffer_grow(struct fd_ringbuffer *ring, uint32_t size)
+{
+   struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
+   struct fd_pipe *pipe = msm_ring->u.submit->pipe;
+
+   assert(ring->flags & FD_RINGBUFFER_GROWABLE);
+
+   finalize_current_cmd(ring);
+
+   fd_bo_del(msm_ring->ring_bo);
+   msm_ring->ring_bo = fd_bo_new_ring(pipe->dev, size);
+   msm_ring->cmd = cmd_new(msm_ring->ring_bo);
+
+   ring->start = fd_bo_map(msm_ring->ring_bo);
+   ring->end = &(ring->start[size / 4]);
+   ring->cur = ring->start;
+   ring->size = size;
+}
+
+static void
+msm_ringbuffer_emit_reloc(struct fd_ringbuffer *ring,
+                          const struct fd_reloc *reloc)
+{
+   struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
+   struct fd_pipe *pipe;
+   unsigned reloc_idx;
+
+   if (ring->flags & _FD_RINGBUFFER_OBJECT) {
+      unsigned idx = APPEND(&msm_ring->u, reloc_bos, fd_bo_ref(reloc->bo));
+
+      /* this gets fixed up at submit->flush() time, since this state-
+       * object rb can be used with many different submits
+       */
+      reloc_idx = idx;
+
+      pipe = msm_ring->u.pipe;
+   } else {
+      struct msm_submit *msm_submit = to_msm_submit(msm_ring->u.submit);
+
+      reloc_idx = append_bo(msm_submit, reloc->bo);
+
+      pipe = msm_ring->u.submit->pipe;
+   }
+
+   APPEND(msm_ring->cmd, relocs,
+          (struct drm_msm_gem_submit_reloc){
+             .reloc_idx = reloc_idx,
+             .reloc_offset = reloc->offset,
+             .or = reloc->orval,
+             .shift = reloc->shift,
+             .submit_offset =
+                offset_bytes(ring->cur, ring->start) + msm_ring->offset,
+          });
+
+   ring->cur++;
+
+   if (pipe->is_64bit) {
+      APPEND(msm_ring->cmd, relocs,
+             (struct drm_msm_gem_submit_reloc){
+                .reloc_idx = reloc_idx,
+                .reloc_offset = reloc->offset,
+                .or = reloc->orval >> 32,
+                .shift = reloc->shift - 32,
+                .submit_offset =
+                   offset_bytes(ring->cur, ring->start) + msm_ring->offset,
+             });
+
+      ring->cur++;
+   }
+}
+
+static void
+append_stateobj_rings(struct msm_submit *submit, struct fd_ringbuffer *target)
+{
+   struct msm_ringbuffer *msm_target = to_msm_ringbuffer(target);
+
+   assert(target->flags & _FD_RINGBUFFER_OBJECT);
+
+   set_foreach (msm_target->u.ring_set, entry) {
+      struct fd_ringbuffer *ring = (void *)entry->key;
+
+      append_ring(submit->ring_set, ring);
+
+      if (ring->flags & _FD_RINGBUFFER_OBJECT) {
+         append_stateobj_rings(submit, ring);
+      }
+   }
+}
+
+static uint32_t
+msm_ringbuffer_emit_reloc_ring(struct fd_ringbuffer *ring,
+                               struct fd_ringbuffer *target, uint32_t cmd_idx)
+{
+   struct msm_ringbuffer *msm_target = to_msm_ringbuffer(target);
+   struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
+   struct fd_bo *bo;
+   uint32_t size;
+
+   if ((target->flags & FD_RINGBUFFER_GROWABLE) &&
+       (cmd_idx < msm_target->u.nr_cmds)) {
+      bo = msm_target->u.cmds[cmd_idx]->ring_bo;
+      size = msm_target->u.cmds[cmd_idx]->size;
+   } else {
+      bo = msm_target->ring_bo;
+      size = offset_bytes(target->cur, target->start);
+   }
+
+   msm_ringbuffer_emit_reloc(ring, &(struct fd_reloc){
+                                      .bo = bo,
+                                      .iova = bo->iova + msm_target->offset,
+                                      .offset = msm_target->offset,
+                                   });
+
+   if (!size)
+      return 0;
+
+   if ((target->flags & _FD_RINGBUFFER_OBJECT) &&
+       !(ring->flags & _FD_RINGBUFFER_OBJECT)) {
+      struct msm_submit *msm_submit = to_msm_submit(msm_ring->u.submit);
+
+      append_stateobj_rings(msm_submit, target);
+   }
+
+   if (ring->flags & _FD_RINGBUFFER_OBJECT) {
+      append_ring(msm_ring->u.ring_set, target);
+   } else {
+      struct msm_submit *msm_submit = to_msm_submit(msm_ring->u.submit);
+      append_ring(msm_submit->ring_set, target);
+   }
+
+   return size;
+}
+
+static uint32_t
+msm_ringbuffer_cmd_count(struct fd_ringbuffer *ring)
+{
+   if (ring->flags & FD_RINGBUFFER_GROWABLE)
+      return to_msm_ringbuffer(ring)->u.nr_cmds + 1;
+   return 1;
+}
+
+static bool
+msm_ringbuffer_check_size(struct fd_ringbuffer *ring)
+{
+   assert(!(ring->flags & _FD_RINGBUFFER_OBJECT));
+   struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
+   struct fd_submit *submit = msm_ring->u.submit;
+   struct fd_pipe *pipe = submit->pipe;
+
+   if ((fd_device_version(pipe->dev) < FD_VERSION_UNLIMITED_CMDS) &&
+       ((ring->cur - ring->start) > (ring->size / 4 - 0x1000))) {
+      return false;
+   }
+
+   if (to_msm_submit(submit)->nr_bos > MAX_ARRAY_SIZE/2) {
+      return false;
+   }
+
+   return true;
+}
+
+static void
+msm_ringbuffer_destroy(struct fd_ringbuffer *ring)
+{
+   struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
+
+   fd_bo_del(msm_ring->ring_bo);
+   if (msm_ring->cmd)
+      cmd_free(msm_ring->cmd);
+
+   if (ring->flags & _FD_RINGBUFFER_OBJECT) {
+      for (unsigned i = 0; i < msm_ring->u.nr_reloc_bos; i++) {
+         fd_bo_del(msm_ring->u.reloc_bos[i]);
+      }
+
+      _mesa_set_destroy(msm_ring->u.ring_set, unref_rings);
+
+      free(msm_ring->u.reloc_bos);
+      free(msm_ring);
+   } else {
+      struct fd_submit *submit = msm_ring->u.submit;
+
+      for (unsigned i = 0; i < msm_ring->u.nr_cmds; i++) {
+         cmd_free(msm_ring->u.cmds[i]);
+      }
+
+      free(msm_ring->u.cmds);
+      slab_free_st(&to_msm_submit(submit)->ring_pool, msm_ring);
+   }
+}
+
+static const struct fd_ringbuffer_funcs ring_funcs = {
+   .grow = msm_ringbuffer_grow,
+   .emit_reloc = msm_ringbuffer_emit_reloc,
+   .emit_reloc_ring = msm_ringbuffer_emit_reloc_ring,
+   .cmd_count = msm_ringbuffer_cmd_count,
+   .check_size = msm_ringbuffer_check_size,
+   .destroy = msm_ringbuffer_destroy,
+};
+
+static inline struct fd_ringbuffer *
+msm_ringbuffer_init(struct msm_ringbuffer *msm_ring, uint32_t size,
+                    enum fd_ringbuffer_flags flags)
+{
+   struct fd_ringbuffer *ring = &msm_ring->base;
+
+   assert(msm_ring->ring_bo);
+
+   uint8_t *base = fd_bo_map(msm_ring->ring_bo);
+   ring->start = (void *)(base + msm_ring->offset);
+   ring->end = &(ring->start[size / 4]);
+   ring->cur = ring->start;
+
+   ring->size = size;
+   ring->flags = flags;
+
+   ring->funcs = &ring_funcs;
+
+   msm_ring->u.cmds = NULL;
+   msm_ring->u.nr_cmds = msm_ring->u.max_cmds = 0;
+
+   msm_ring->cmd = cmd_new(msm_ring->ring_bo);
+
+   return ring;
+}
+
+struct fd_ringbuffer *
+msm_ringbuffer_new_object(struct fd_pipe *pipe, uint32_t size)
+{
+   struct msm_ringbuffer *msm_ring = malloc(sizeof(*msm_ring));
+
+   msm_ring->u.pipe = pipe;
+   msm_ring->offset = 0;
+   msm_ring->ring_bo = fd_bo_new_ring(pipe->dev, size);
+   msm_ring->base.refcnt = 1;
+
+   msm_ring->u.reloc_bos = NULL;
+   msm_ring->u.nr_reloc_bos = msm_ring->u.max_reloc_bos = 0;
+
+   msm_ring->u.ring_set =
+      _mesa_set_create(NULL, _mesa_hash_pointer, _mesa_key_pointer_equal);
+
+   return msm_ringbuffer_init(msm_ring, size, _FD_RINGBUFFER_OBJECT);
+}

src/freedreno/drm/msm/msm_ringbuffer_sp.c

@@ -0,0 +1,162 @@
+/*
+ * Copyright © 2018 Rob Clark <[email protected]>
+ * SPDX-License-Identifier: MIT
+ *
+ * Authors:
+ *    Rob Clark <[email protected]>
+ */
+
+#include <assert.h>
+#include <inttypes.h>
+#include <pthread.h>
+
+#include "util/os_file.h"
+
+#include "drm/freedreno_ringbuffer_sp.h"
+#include "freedreno_rd_output.h"
+#include "msm_priv.h"
+
+static int
+flush_submit_list(struct list_head *submit_list)
+{
+   struct fd_submit_sp *fd_submit = to_fd_submit_sp(last_submit(submit_list));
+   struct fd_pipe *pipe = fd_submit->base.pipe;
+   struct msm_pipe *msm_pipe = to_msm_pipe(pipe);
+   struct drm_msm_gem_submit req = {
+      .flags = msm_pipe->pipe,
+      .queueid = msm_pipe->queue_id,
+   };
+   int ret;
+
+   unsigned nr_cmds = 0;
+
+   MESA_TRACE_FUNC();
+
+   /* Determine the number of extra cmds's from deferred submits that
+    * we will be merging in:
+    */
+   foreach_submit (submit, submit_list) {
+      assert(submit->pipe == &msm_pipe->base);
+      nr_cmds += to_fd_ringbuffer_sp(submit->primary)->u.nr_cmds;
+   }
+
+   struct drm_msm_gem_submit_cmd cmds[nr_cmds];
+
+   unsigned cmd_idx = 0;
+
+   /* Build up the table of cmds, and for all but the last submit in the
+    * list, merge their bo tables into the last submit.
+    */
+   foreach_submit_safe (submit, submit_list) {
+      struct fd_ringbuffer_sp *deferred_primary =
+         to_fd_ringbuffer_sp(submit->primary);
+
+      for (unsigned i = 0; i < deferred_primary->u.nr_cmds; i++) {
+         struct fd_bo *ring_bo = deferred_primary->u.cmds[i].ring_bo;
+         cmds[cmd_idx].type = MSM_SUBMIT_CMD_BUF;
+         cmds[cmd_idx].submit_idx = fd_submit_append_bo(fd_submit, ring_bo);
+         cmds[cmd_idx].submit_offset = submit_offset(ring_bo, deferred_primary->offset);
+         cmds[cmd_idx].size = deferred_primary->u.cmds[i].size;
+         cmds[cmd_idx].pad = 0;
+         cmds[cmd_idx].nr_relocs = 0;
+
+         cmd_idx++;
+      }
+
+      /* We are merging all the submits in the list into the last submit,
+       * so the remainder of the loop body doesn't apply to the last submit
+       */
+      if (submit == last_submit(submit_list)) {
+         DEBUG_MSG("merged %u submits", cmd_idx);
+         break;
+      }
+
+      struct fd_submit_sp *fd_deferred_submit = to_fd_submit_sp(submit);
+      for (unsigned i = 0; i < fd_deferred_submit->nr_bos; i++) {
+         /* Note: if bo is used in both the current submit and the deferred
+          * submit being merged, we expect to hit the fast-path as we add it
+          * to the current submit:
+          */
+         fd_submit_append_bo(fd_submit, fd_deferred_submit->bos[i]);
+      }
+
+      /* Now that the cmds/bos have been transfered over to the current submit,
+       * we can remove the deferred submit from the list and drop it's reference
+       */
+      list_del(&submit->node);
+      fd_submit_del(submit);
+   }
+
+   if (fd_submit->in_fence_fd != -1) {
+      req.flags |= MSM_SUBMIT_FENCE_FD_IN;
+      req.fence_fd = fd_submit->in_fence_fd;
+   }
+
+   if (pipe->no_implicit_sync) {
+      req.flags |= MSM_SUBMIT_NO_IMPLICIT;
+   }
+
+   if (fd_submit->out_fence->use_fence_fd) {
+      req.flags |= MSM_SUBMIT_FENCE_FD_OUT;
+   }
+
+   /* Needs to be after get_cmd() as that could create bos/cmds table:
+    *
+    * NOTE allocate on-stack in the common case, but with an upper-
+    * bound to limit on-stack allocation to 4k:
+    */
+   const unsigned bo_limit = 4096 / sizeof(struct drm_msm_gem_submit_bo);
+   bool bos_on_stack = fd_submit->nr_bos < bo_limit;
+   struct drm_msm_gem_submit_bo
+      _submit_bos[bos_on_stack ? fd_submit->nr_bos : 0];
+   struct drm_msm_gem_submit_bo *submit_bos;
+   if (bos_on_stack) {
+      submit_bos = _submit_bos;
+   } else {
+      submit_bos = malloc(fd_submit->nr_bos * sizeof(submit_bos[0]));
+   }
+
+   for (unsigned i = 0; i < fd_submit->nr_bos; i++) {
+      submit_bos[i].flags = fd_submit->bos[i]->reloc_flags;
+      submit_bos[i].handle = fd_submit->bos[i]->handle;
+      submit_bos[i].presumed = 0;
+   }
+
+   req.bos = VOID2U64(submit_bos);
+   req.nr_bos = fd_submit->nr_bos;
+   req.cmds = VOID2U64(cmds);
+   req.nr_cmds = nr_cmds;
+
+   DEBUG_MSG("nr_cmds=%u, nr_bos=%u", req.nr_cmds, req.nr_bos);
+
+   ret = drmCommandWriteRead(pipe->dev->fd, DRM_MSM_GEM_SUBMIT, &req,
+                             sizeof(req));
+   if (ret) {
+      ERROR_MSG("submit failed: %d (%s)", ret, strerror(errno));
+      msm_dump_submit(&req);
+   } else if (!ret) {
+      fd_submit->out_fence->kfence = req.fence;
+      fd_submit->out_fence->fence_fd = req.fence_fd;
+   }
+
+   msm_dump_rd(pipe, &req);
+
+   if (!bos_on_stack)
+      free(submit_bos);
+
+   if (fd_submit->in_fence_fd != -1)
+      close(fd_submit->in_fence_fd);
+
+   return ret;
+}
+
+struct fd_submit *
+msm_submit_sp_new(struct fd_pipe *pipe)
+{
+   /* We don't do any translation from internal FD_RELOC flags to MSM flags. */
+   STATIC_ASSERT(FD_RELOC_READ == MSM_SUBMIT_BO_READ);
+   STATIC_ASSERT(FD_RELOC_WRITE == MSM_SUBMIT_BO_WRITE);
+   STATIC_ASSERT(FD_RELOC_DUMP == MSM_SUBMIT_BO_DUMP);
+
+   return fd_submit_sp_new(pipe, flush_submit_list);
+}

src/freedreno/drm/virtio/virtio_bo.c

@@ -0,0 +1,356 @@
+/*
+ * Copyright © 2022 Google, Inc.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "util/libsync.h"
+
+#include "virtio_priv.h"
+
+static void *
+virtio_bo_mmap(struct fd_bo *bo)
+{
+   struct vdrm_device *vdrm = to_virtio_device(bo->dev)->vdrm;
+   struct virtio_bo *virtio_bo = to_virtio_bo(bo);
+
+   /* If we have uploaded, we need to wait for host to handle that
+    * before we can allow guest-side CPU access:
+    */
+   if (virtio_bo->has_upload_seqno) {
+
+      virtio_bo->has_upload_seqno = false;
+
+      vdrm_flush(vdrm);
+      vdrm_host_sync(vdrm, &(struct vdrm_ccmd_req) {
+         .seqno = virtio_bo->upload_seqno,
+      });
+   }
+
+   return vdrm_bo_map(vdrm, bo->handle, bo->size, NULL);
+}
+
+static int
+virtio_bo_cpu_prep(struct fd_bo *bo, struct fd_pipe *pipe, uint32_t op)
+{
+   MESA_TRACE_FUNC();
+   struct vdrm_device *vdrm = to_virtio_device(bo->dev)->vdrm;
+   int ret;
+
+   /*
+    * Wait first in the guest, to avoid a blocking call in host.
+    * If implicit sync it used, we still need to *also* wait in
+    * host, if it is a shared buffer, because the guest doesn't
+    * know about usage of the bo in the host (or other guests).
+    */
+
+   ret = vdrm_bo_wait(vdrm, bo->handle);
+   if (ret)
+      goto out;
+
+   /*
+    * The buffer could be shared with other things on the host side
+    * so have to poll the host.  But we only get here with the shared
+    * buffers plus implicit sync.  Hopefully that is rare enough.
+    */
+
+   struct msm_ccmd_gem_cpu_prep_req req = {
+         .hdr = MSM_CCMD(GEM_CPU_PREP, sizeof(req)),
+         .res_id = to_virtio_bo(bo)->res_id,
+         .op = op,
+   };
+   struct msm_ccmd_gem_cpu_prep_rsp *rsp;
+
+   /* We can't do a blocking wait in the host, so we have to poll: */
+   do {
+      rsp = vdrm_alloc_rsp(vdrm, &req.hdr, sizeof(*rsp));
+
+      ret = vdrm_send_req(vdrm, &req.hdr, true);
+      if (ret)
+         goto out;
+
+      ret = rsp->ret;
+   } while (ret == -EBUSY);
+
+out:
+   return ret;
+}
+
+static int
+virtio_bo_madvise(struct fd_bo *bo, int willneed)
+{
+   /* TODO:
+    * Currently unsupported, synchronous WILLNEED calls would introduce too
+    * much latency.. ideally we'd keep state in the guest and only flush
+    * down to host when host is under memory pressure.  (Perhaps virtio-balloon
+    * could signal this?)
+    */
+   return willneed;
+}
+
+static uint64_t
+virtio_bo_iova(struct fd_bo *bo)
+{
+   /* The shmem bo is allowed to have no iova, as it is only used for
+    * guest<->host communications:
+    */
+   assert(bo->iova || (to_virtio_bo(bo)->blob_id == 0));
+   return bo->iova;
+}
+
+static void
+virtio_bo_set_name(struct fd_bo *bo, const char *fmt, va_list ap)
+{
+   char name[32];
+   int sz;
+
+   /* Note, we cannot set name on the host for the shmem bo, as
+    * that isn't a real gem obj on the host side.. not having
+    * an iova is a convenient way to detect this case:
+    */
+   if (!bo->iova)
+      return;
+
+   sz = vsnprintf(name, sizeof(name), fmt, ap);
+   sz = MIN2(sz, sizeof(name));
+
+   unsigned req_len = sizeof(struct msm_ccmd_gem_set_name_req) + align(sz, 4);
+
+   uint8_t buf[req_len];
+   struct msm_ccmd_gem_set_name_req *req = (void *)buf;
+
+   req->hdr = MSM_CCMD(GEM_SET_NAME, req_len);
+   req->res_id = to_virtio_bo(bo)->res_id;
+   req->len = sz;
+
+   memcpy(req->payload, name, sz);
+
+   vdrm_send_req(to_virtio_device(bo->dev)->vdrm, &req->hdr, false);
+}
+
+static int
+virtio_bo_dmabuf(struct fd_bo *bo)
+{
+   struct virtio_device *virtio_dev = to_virtio_device(bo->dev);
+
+   return vdrm_bo_export_dmabuf(virtio_dev->vdrm, bo->handle);
+}
+
+static void
+bo_upload(struct fd_bo *bo, unsigned off, void *src, unsigned len)
+{
+   MESA_TRACE_FUNC();
+   unsigned req_len = sizeof(struct msm_ccmd_gem_upload_req) + align(len, 4);
+   struct virtio_bo *virtio_bo = to_virtio_bo(bo);
+
+   uint8_t buf[req_len];
+   struct msm_ccmd_gem_upload_req *req = (void *)buf;
+
+   req->hdr = MSM_CCMD(GEM_UPLOAD, req_len);
+   req->res_id = virtio_bo->res_id;
+   req->pad = 0;
+   req->off = off;
+   req->len = len;
+
+   memcpy(req->payload, src, len);
+
+   vdrm_send_req(to_virtio_device(bo->dev)->vdrm, &req->hdr, false);
+
+   virtio_bo->upload_seqno = req->hdr.seqno;
+   virtio_bo->has_upload_seqno = true;
+}
+
+static void
+virtio_bo_upload(struct fd_bo *bo, void *src, unsigned off, unsigned len)
+{
+   while (len > 0) {
+      unsigned sz = MIN2(len, 0x1000);
+      bo_upload(bo, off, src, sz);
+      off += sz;
+      src += sz;
+      len -= sz;
+   }
+}
+
+/**
+ * For recently allocated buffers, an immediate mmap would stall waiting
+ * for the host to handle the allocation and map to the guest, which
+ * could take a few ms.  So for small transfers to recently allocated
+ * buffers, we'd prefer to use the upload path instead.
+ */
+static bool
+virtio_bo_prefer_upload(struct fd_bo *bo, unsigned len)
+{
+   struct virtio_bo *virtio_bo = to_virtio_bo(bo);
+
+   /* If we've already taken the hit of mmap'ing the buffer, then no reason
+    * to take the upload path:
+    */
+   if (bo->map)
+      return false;
+
+   if (len > 0x4000)
+      return false;
+
+   int64_t age_ns = os_time_get_nano() - virtio_bo->alloc_time_ns;
+   if (age_ns > 5000000)
+      return false;
+
+   return true;
+}
+
+static void
+set_iova(struct fd_bo *bo, uint64_t iova)
+{
+   struct msm_ccmd_gem_set_iova_req req = {
+         .hdr = MSM_CCMD(GEM_SET_IOVA, sizeof(req)),
+         .res_id = to_virtio_bo(bo)->res_id,
+         .iova = iova,
+   };
+
+   vdrm_send_req(to_virtio_device(bo->dev)->vdrm, &req.hdr, false);
+}
+
+static void
+virtio_bo_finalize(struct fd_bo *bo)
+{
+   /* Release iova by setting to zero: */
+   if (bo->iova) {
+      set_iova(bo, 0);
+
+      virtio_dev_free_iova(bo->dev, bo->iova, bo->size);
+   }
+}
+
+static const struct fd_bo_funcs funcs = {
+   .map = virtio_bo_mmap,
+   .cpu_prep = virtio_bo_cpu_prep,
+   .madvise = virtio_bo_madvise,
+   .iova = virtio_bo_iova,
+   .set_name = virtio_bo_set_name,
+   .dmabuf = virtio_bo_dmabuf,
+   .upload = virtio_bo_upload,
+   .prefer_upload = virtio_bo_prefer_upload,
+   .finalize = virtio_bo_finalize,
+   .destroy = fd_bo_fini_common,
+};
+
+static struct fd_bo *
+bo_from_handle(struct fd_device *dev, uint32_t size, uint32_t handle)
+{
+   struct virtio_device *virtio_dev = to_virtio_device(dev);
+   struct virtio_bo *virtio_bo;
+   struct fd_bo *bo;
+
+   virtio_bo = calloc(1, sizeof(*virtio_bo));
+   if (!virtio_bo)
+      return NULL;
+
+   virtio_bo->alloc_time_ns = os_time_get_nano();
+
+   bo = &virtio_bo->base;
+
+   /* Note we need to set these because allocation_wait_execute() could
+    * run before bo_init_commont():
+    */
+   bo->dev = dev;
+   p_atomic_set(&bo->refcnt, 1);
+
+   bo->size = size;
+   bo->funcs = &funcs;
+   bo->handle = handle;
+
+   /* Don't assume we can mmap an imported bo: */
+   bo->alloc_flags = FD_BO_NOMAP;
+
+   virtio_bo->res_id = vdrm_handle_to_res_id(virtio_dev->vdrm, handle);
+
+   fd_bo_init_common(bo, dev);
+
+   return bo;
+}
+
+/* allocate a new buffer object from existing handle */
+struct fd_bo *
+virtio_bo_from_handle(struct fd_device *dev, uint32_t size, uint32_t handle)
+{
+   struct fd_bo *bo = bo_from_handle(dev, size, handle);
+
+   if (!bo)
+      return NULL;
+
+   bo->iova = virtio_dev_alloc_iova(dev, size);
+   if (!bo->iova)
+      goto fail;
+
+   set_iova(bo, bo->iova);
+
+   return bo;
+
+fail:
+   virtio_bo_finalize(bo);
+   fd_bo_fini_common(bo);
+   return NULL;
+}
+
+/* allocate a buffer object: */
+struct fd_bo *
+virtio_bo_new(struct fd_device *dev, uint32_t size, uint32_t flags)
+{
+   struct virtio_device *virtio_dev = to_virtio_device(dev);
+   struct msm_ccmd_gem_new_req req = {
+         .hdr = MSM_CCMD(GEM_NEW, sizeof(req)),
+         .size = size,
+   };
+
+   if (flags & FD_BO_SCANOUT)
+      req.flags |= MSM_BO_SCANOUT;
+
+   if (flags & FD_BO_GPUREADONLY)
+      req.flags |= MSM_BO_GPU_READONLY;
+
+   if (flags & FD_BO_CACHED_COHERENT) {
+      req.flags |= MSM_BO_CACHED_COHERENT;
+   } else {
+      req.flags |= MSM_BO_WC;
+   }
+
+   uint32_t blob_flags = 0;
+   if (flags & (FD_BO_SHARED | FD_BO_SCANOUT)) {
+      blob_flags = VIRTGPU_BLOB_FLAG_USE_SHAREABLE;
+      if (virtio_dev->vdrm->supports_cross_device)
+         blob_flags |= VIRTGPU_BLOB_FLAG_USE_CROSS_DEVICE;
+   }
+
+   if (!(flags & FD_BO_NOMAP)) {
+      blob_flags |= VIRTGPU_BLOB_FLAG_USE_MAPPABLE;
+   }
+
+   uint32_t blob_id = p_atomic_inc_return(&virtio_dev->next_blob_id);
+
+   /* tunneled cmds are processed separately on host side,
+    * before the renderer->get_blob() callback.. the blob_id
+    * is used to link the created bo to the get_blob() call
+    */
+   req.blob_id = blob_id;
+   req.iova = virtio_dev_alloc_iova(dev, size);
+   if (!req.iova)
+      goto fail;
+
+   uint32_t handle =
+      vdrm_bo_create(virtio_dev->vdrm, size, blob_flags, blob_id, &req.hdr);
+   if (!handle)
+      goto fail;
+
+   struct fd_bo *bo = bo_from_handle(dev, size, handle);
+   struct virtio_bo *virtio_bo = to_virtio_bo(bo);
+
+   virtio_bo->blob_id = blob_id;
+   bo->iova = req.iova;
+
+   return bo;
+
+fail:
+   if (req.iova)
+      virtio_dev_free_iova(dev, req.iova, size);
+   return NULL;
+}

src/freedreno/drm/virtio/virtio_device.c

@@ -0,0 +1,207 @@
+/*
+ * Copyright © 2022 Google, Inc.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include <unistd.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+
+#include "util/libsync.h"
+#include "util/u_process.h"
+
+#include "virtio_priv.h"
+
+
+static int virtio_execbuf_flush(struct fd_device *dev);
+
+static void
+virtio_device_destroy(struct fd_device *dev)
+{
+   struct virtio_device *virtio_dev = to_virtio_device(dev);
+
+   util_vma_heap_finish(&virtio_dev->address_space);
+}
+
+static uint32_t
+virtio_handle_from_dmabuf(struct fd_device *dev, int fd)
+{
+   struct virtio_device *virtio_dev = to_virtio_device(dev);
+
+   return vdrm_dmabuf_to_handle(virtio_dev->vdrm, fd);
+}
+
+static void
+virtio_close_handle(struct fd_bo *bo)
+{
+   struct virtio_device *virtio_dev = to_virtio_device(bo->dev);
+
+   vdrm_bo_close(virtio_dev->vdrm, bo->handle);
+}
+
+static const struct fd_device_funcs funcs = {
+   .bo_new = virtio_bo_new,
+   .bo_from_handle = virtio_bo_from_handle,
+   .handle_from_dmabuf = virtio_handle_from_dmabuf,
+   .bo_from_dmabuf = fd_bo_from_dmabuf_drm,
+   .bo_close_handle = virtio_close_handle,
+   .pipe_new = virtio_pipe_new,
+   .flush = virtio_execbuf_flush,
+   .destroy = virtio_device_destroy,
+};
+
+static void
+set_debuginfo(struct fd_device *dev)
+{
+   const char *comm = util_get_process_name();
+   static char cmdline[0x1000];
+
+   if (!comm || !util_get_command_line(cmdline, sizeof(cmdline)))
+      return;
+
+   unsigned comm_len = strlen(comm) + 1;
+   unsigned cmdline_len = strlen(cmdline) + 1;
+
+   struct msm_ccmd_set_debuginfo_req *req;
+
+   unsigned req_len = align(sizeof(*req) + comm_len + cmdline_len, 4);
+
+   req = malloc(req_len);
+
+   req->hdr         = MSM_CCMD(SET_DEBUGINFO, req_len);
+   req->comm_len    = comm_len;
+   req->cmdline_len = cmdline_len;
+
+   memcpy(&req->payload[0], comm, comm_len);
+   memcpy(&req->payload[comm_len], cmdline, cmdline_len);
+
+   vdrm_send_req(to_virtio_device(dev)->vdrm, &req->hdr, false);
+
+   free(req);
+}
+
+struct fd_device *
+virtio_device_new(int fd, drmVersionPtr version)
+{
+   struct virgl_renderer_capset_drm caps;
+   struct virtio_device *virtio_dev;
+   struct vdrm_device *vdrm;
+   struct fd_device *dev;
+
+   STATIC_ASSERT(FD_BO_PREP_READ == MSM_PREP_READ);
+   STATIC_ASSERT(FD_BO_PREP_WRITE == MSM_PREP_WRITE);
+   STATIC_ASSERT(FD_BO_PREP_NOSYNC == MSM_PREP_NOSYNC);
+
+   /* Debug option to force fallback to virgl: */
+   if (debug_get_bool_option("FD_NO_VIRTIO", false))
+      return NULL;
+
+   vdrm = vdrm_device_connect(fd, VIRTGPU_DRM_CONTEXT_MSM);
+   if (!vdrm) {
+      INFO_MSG("could not connect vdrm");
+      return NULL;
+   }
+
+   caps = vdrm->caps;
+
+   INFO_MSG("wire_format_version: %u", caps.wire_format_version);
+   INFO_MSG("version_major:       %u", caps.version_major);
+   INFO_MSG("version_minor:       %u", caps.version_minor);
+   INFO_MSG("version_patchlevel:  %u", caps.version_patchlevel);
+   INFO_MSG("has_cached_coherent: %u", caps.u.msm.has_cached_coherent);
+   INFO_MSG("va_start:            0x%0"PRIx64, caps.u.msm.va_start);
+   INFO_MSG("va_size:             0x%0"PRIx64, caps.u.msm.va_size);
+   INFO_MSG("gpu_id:              %u", caps.u.msm.gpu_id);
+   INFO_MSG("gmem_size:           %u", caps.u.msm.gmem_size);
+   INFO_MSG("gmem_base:           0x%0" PRIx64, caps.u.msm.gmem_base);
+   INFO_MSG("chip_id:             0x%0" PRIx64, caps.u.msm.chip_id);
+   INFO_MSG("max_freq:            %u", caps.u.msm.max_freq);
+
+   if (caps.wire_format_version != 2) {
+      ERROR_MSG("Unsupported protocol version: %u", caps.wire_format_version);
+      goto error;
+   }
+
+   if ((caps.version_major != 1) || (caps.version_minor < FD_VERSION_SOFTPIN)) {
+      ERROR_MSG("unsupported version: %u.%u.%u", caps.version_major,
+                caps.version_minor, caps.version_patchlevel);
+      goto error;
+   }
+
+   if (!caps.u.msm.va_size) {
+      ERROR_MSG("No address space");
+      goto error;
+   }
+
+   virtio_dev = calloc(1, sizeof(*virtio_dev));
+   if (!virtio_dev)
+      goto error;
+
+   dev = &virtio_dev->base;
+   dev->funcs = &funcs;
+   dev->fd = fd;
+   dev->version = caps.version_minor;
+   dev->has_cached_coherent = caps.u.msm.has_cached_coherent;
+
+   p_atomic_set(&virtio_dev->next_blob_id, 1);
+   virtio_dev->shmem = to_msm_shmem(vdrm->shmem);
+   virtio_dev->vdrm = vdrm;
+
+   util_queue_init(&dev->submit_queue, "sq", 8, 1, 0, NULL);
+
+   dev->bo_size = sizeof(struct virtio_bo);
+
+   set_debuginfo(dev);
+
+   util_vma_heap_init(&virtio_dev->address_space,
+                      caps.u.msm.va_start,
+                      caps.u.msm.va_size);
+   simple_mtx_init(&virtio_dev->address_space_lock, mtx_plain);
+
+   return dev;
+
+error:
+   vdrm_device_close(vdrm);
+   return NULL;
+}
+
+static int
+virtio_execbuf_flush(struct fd_device *dev)
+{
+   return vdrm_flush(to_virtio_device(dev)->vdrm);
+}
+
+/**
+ * Helper for simple pass-thru ioctls
+ */
+int
+virtio_simple_ioctl(struct fd_device *dev, unsigned cmd, void *_req)
+{
+   MESA_TRACE_FUNC();
+   struct vdrm_device *vdrm = to_virtio_device(dev)->vdrm;
+   unsigned req_len = sizeof(struct msm_ccmd_ioctl_simple_req);
+   unsigned rsp_len = sizeof(struct msm_ccmd_ioctl_simple_rsp);
+
+   req_len += _IOC_SIZE(cmd);
+   if (cmd & IOC_OUT)
+      rsp_len += _IOC_SIZE(cmd);
+
+   uint8_t buf[req_len];
+   struct msm_ccmd_ioctl_simple_req *req = (void *)buf;
+   struct msm_ccmd_ioctl_simple_rsp *rsp;
+
+   req->hdr = MSM_CCMD(IOCTL_SIMPLE, req_len);
+   req->cmd = cmd;
+   memcpy(req->payload, _req, _IOC_SIZE(cmd));
+
+   rsp = vdrm_alloc_rsp(vdrm, &req->hdr, rsp_len);
+
+   int ret = vdrm_send_req(vdrm, &req->hdr, true);
+
+   if (cmd & IOC_OUT)
+      memcpy(_req, rsp->payload, _IOC_SIZE(cmd));
+
+   ret = rsp->ret;
+
+   return ret;
+}

src/freedreno/drm/virtio/virtio_pipe.c

@@ -0,0 +1,285 @@
+/*
+ * Copyright © 2022 Google, Inc.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "util/libsync.h"
+#include "util/slab.h"
+
+#include "freedreno_ringbuffer_sp.h"
+#include "virtio_priv.h"
+
+static int
+query_param(struct fd_pipe *pipe, uint32_t param, uint64_t *value)
+{
+   struct virtio_pipe *virtio_pipe = to_virtio_pipe(pipe);
+   struct drm_msm_param req = {
+      .pipe = virtio_pipe->pipe,
+      .param = param,
+   };
+   int ret;
+
+   ret = virtio_simple_ioctl(pipe->dev, DRM_IOCTL_MSM_GET_PARAM, &req);
+   if (ret)
+      return ret;
+
+   *value = req.value;
+
+   return 0;
+}
+
+static int
+query_faults(struct fd_pipe *pipe, uint64_t *value)
+{
+   struct virtio_device *virtio_dev = to_virtio_device(pipe->dev);
+   uint32_t async_error = 0;
+   uint64_t global_faults;
+
+   if (vdrm_shmem_has_field(virtio_dev->shmem, async_error))
+      async_error = virtio_dev->shmem->async_error;
+
+   if (vdrm_shmem_has_field(virtio_dev->shmem, global_faults)) {
+      global_faults = virtio_dev->shmem->global_faults;
+   } else {
+      int ret = query_param(pipe, MSM_PARAM_FAULTS, &global_faults);
+      if (ret)
+         return ret;
+   }
+
+   *value = global_faults + async_error;
+
+   return 0;
+}
+
+static int
+virtio_pipe_get_param(struct fd_pipe *pipe, enum fd_param_id param,
+                   uint64_t *value)
+{
+   struct virtio_pipe *virtio_pipe = to_virtio_pipe(pipe);
+   struct virtio_device *virtio_dev = to_virtio_device(pipe->dev);
+
+   switch (param) {
+   case FD_DEVICE_ID: // XXX probably get rid of this..
+   case FD_GPU_ID:
+      *value = virtio_pipe->gpu_id;
+      return 0;
+   case FD_GMEM_SIZE:
+      *value = virtio_pipe->gmem;
+      return 0;
+   case FD_GMEM_BASE:
+      *value = virtio_pipe->gmem_base;
+      return 0;
+   case FD_CHIP_ID:
+      *value = virtio_pipe->chip_id;
+      return 0;
+   case FD_MAX_FREQ:
+      *value = virtio_dev->vdrm->caps.u.msm.max_freq;
+      return 0;
+   case FD_TIMESTAMP:
+      return query_param(pipe, MSM_PARAM_TIMESTAMP, value);
+   case FD_NR_PRIORITIES:
+      *value = virtio_dev->vdrm->caps.u.msm.priorities;
+      return 0;
+   case FD_CTX_FAULTS:
+   case FD_GLOBAL_FAULTS:
+      return query_faults(pipe, value);
+   case FD_SUSPEND_COUNT:
+      return query_param(pipe, MSM_PARAM_SUSPENDS, value);
+   case FD_VA_SIZE:
+      *value = virtio_dev->vdrm->caps.u.msm.va_size;
+      return 0;
+   case FD_UCHE_TRAP_BASE:
+      return query_param(pipe, MSM_PARAM_UCHE_TRAP_BASE, value);
+   default:
+      ERROR_MSG("invalid param id: %d", param);
+      return -1;
+   }
+}
+
+static void
+virtio_pipe_finish(struct fd_pipe *pipe)
+{
+   struct virtio_pipe *virtio_pipe = to_virtio_pipe(pipe);
+   if (util_queue_is_initialized(&virtio_pipe->retire_queue))
+      util_queue_finish(&virtio_pipe->retire_queue);
+}
+
+static int
+virtio_pipe_wait(struct fd_pipe *pipe, const struct fd_fence *fence, uint64_t timeout)
+{
+   MESA_TRACE_FUNC();
+   struct vdrm_device *vdrm = to_virtio_device(pipe->dev)->vdrm;
+   struct msm_ccmd_wait_fence_req req = {
+         .hdr = MSM_CCMD(WAIT_FENCE, sizeof(req)),
+         .queue_id = to_virtio_pipe(pipe)->queue_id,
+         .fence = fence->kfence,
+   };
+   struct msm_ccmd_submitqueue_query_rsp *rsp;
+   int64_t end_time = os_time_get_nano() + timeout;
+   int ret;
+
+   /* Do a non-blocking wait to trigger host-side wait-boost,
+    * if the host kernel is new enough
+    */
+   rsp = vdrm_alloc_rsp(vdrm, &req.hdr, sizeof(*rsp));
+   ret = vdrm_send_req(vdrm, &req.hdr, false);
+   if (ret)
+      goto out;
+
+   vdrm_flush(vdrm);
+
+   if (fence->use_fence_fd)
+      return sync_wait(fence->fence_fd, timeout / 1000000);
+
+   do {
+      rsp = vdrm_alloc_rsp(vdrm, &req.hdr, sizeof(*rsp));
+
+      ret = vdrm_send_req(vdrm, &req.hdr, true);
+      if (ret)
+         goto out;
+
+      ret = rsp->ret;
+
+      if ((timeout != OS_TIMEOUT_INFINITE) &&
+          (os_time_get_nano() >= end_time))
+         break;
+   } while (ret == -ETIMEDOUT);
+
+out:
+   return ret;
+}
+
+static int
+__open_submitqueue(struct fd_pipe *pipe, uint32_t prio, uint32_t flags)
+{
+   struct virtio_pipe *virtio_pipe = to_virtio_pipe(pipe);
+
+   struct drm_msm_submitqueue req = {
+      .flags = flags,
+      .prio = prio,
+   };
+   uint64_t nr_prio = 1;
+   int ret;
+
+   virtio_pipe_get_param(pipe, FD_NR_PRIORITIES, &nr_prio);
+
+   req.prio = MIN2(req.prio, MAX2(nr_prio, 1) - 1);
+
+   ret = virtio_simple_ioctl(pipe->dev, DRM_IOCTL_MSM_SUBMITQUEUE_NEW, &req);
+   if (ret) {
+      ERROR_MSG("could not create submitqueue! %d (%s)", ret, strerror(errno));
+      return ret;
+   }
+
+   virtio_pipe->queue_id = req.id;
+   virtio_pipe->ring_idx = req.prio + 1;
+
+   return 0;
+}
+
+static int
+open_submitqueue(struct fd_pipe *pipe, uint32_t prio)
+{
+   struct virtio_device *virtio_dev = to_virtio_device(pipe->dev);
+   const struct fd_dev_info *info = fd_dev_info_raw(&pipe->dev_id);
+   int ret = -1;
+
+   if (info && info->chip >= A7XX &&
+       (virtio_dev->vdrm->caps.u.msm.has_preemption != VIRTGPU_CAP_BOOL_FALSE))
+      ret = __open_submitqueue(pipe, prio, MSM_SUBMITQUEUE_ALLOW_PREEMPT);
+
+   /* If kernel doesn't support preemption, try again without: */
+   if (ret)
+      ret = __open_submitqueue(pipe, prio, 0);
+
+   if (ret) {
+      ERROR_MSG("could not create submitqueue! %d (%s)", ret, strerror(errno));
+      return ret;
+   }
+
+   return 0;
+}
+
+static void
+close_submitqueue(struct fd_pipe *pipe, uint32_t queue_id)
+{
+   virtio_simple_ioctl(pipe->dev, DRM_IOCTL_MSM_SUBMITQUEUE_CLOSE, &queue_id);
+}
+
+static void
+virtio_pipe_destroy(struct fd_pipe *pipe)
+{
+   struct virtio_pipe *virtio_pipe = to_virtio_pipe(pipe);
+
+   if (util_queue_is_initialized(&virtio_pipe->retire_queue))
+      util_queue_destroy(&virtio_pipe->retire_queue);
+
+   close_submitqueue(pipe, virtio_pipe->queue_id);
+   fd_pipe_sp_ringpool_fini(pipe);
+   free(virtio_pipe);
+}
+
+static const struct fd_pipe_funcs funcs = {
+   .ringbuffer_new_object = fd_ringbuffer_sp_new_object,
+   .submit_new = virtio_submit_new,
+   .flush = fd_pipe_sp_flush,
+   .finish = virtio_pipe_finish,
+   .get_param = virtio_pipe_get_param,
+   .wait = virtio_pipe_wait,
+   .destroy = virtio_pipe_destroy,
+};
+
+struct fd_pipe *
+virtio_pipe_new(struct fd_device *dev, enum fd_pipe_id id, uint32_t prio)
+{
+   static const uint32_t pipe_id[] = {
+      [FD_PIPE_3D] = MSM_PIPE_3D0,
+      [FD_PIPE_2D] = MSM_PIPE_2D0,
+   };
+   struct virtio_device *virtio_dev = to_virtio_device(dev);
+   struct vdrm_device *vdrm = virtio_dev->vdrm;
+   struct virtio_pipe *virtio_pipe = NULL;
+   struct fd_pipe *pipe = NULL;
+
+   virtio_pipe = calloc(1, sizeof(*virtio_pipe));
+   if (!virtio_pipe) {
+      ERROR_MSG("allocation failed");
+      goto fail;
+   }
+
+   pipe = &virtio_pipe->base;
+
+   pipe->funcs = &funcs;
+
+   /* initialize before get_param(): */
+   pipe->dev = dev;
+   virtio_pipe->pipe = pipe_id[id];
+
+   virtio_pipe->gpu_id = vdrm->caps.u.msm.gpu_id;
+   virtio_pipe->gmem = vdrm->caps.u.msm.gmem_size;
+   virtio_pipe->gmem_base = vdrm->caps.u.msm.gmem_base;
+   virtio_pipe->chip_id = vdrm->caps.u.msm.chip_id;
+
+
+   if (!(virtio_pipe->gpu_id || virtio_pipe->chip_id))
+      goto fail;
+
+   util_queue_init(&virtio_pipe->retire_queue, "rq", 8, 1,
+                   UTIL_QUEUE_INIT_RESIZE_IF_FULL, NULL);
+
+   INFO_MSG("Pipe Info:");
+   INFO_MSG(" GPU-id:          %d", virtio_pipe->gpu_id);
+   INFO_MSG(" Chip-id:         0x%016"PRIx64, virtio_pipe->chip_id);
+   INFO_MSG(" GMEM size:       0x%08x", virtio_pipe->gmem);
+
+   if (open_submitqueue(pipe, prio))
+      goto fail;
+
+   fd_pipe_sp_ringpool_init(pipe);
+
+   return pipe;
+fail:
+   if (pipe)
+      fd_pipe_del(pipe);
+   return NULL;
+}

src/freedreno/drm/virtio/virtio_priv.h

@@ -0,0 +1,145 @@
+/*
+ * Copyright © 2022 Google, Inc.
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef VIRTIO_PRIV_H_
+#define VIRTIO_PRIV_H_
+
+#include <poll.h>
+
+#include "freedreno_priv.h"
+
+#include "util/perf/cpu_trace.h"
+#include "util/u_atomic.h"
+#include "util/slab.h"
+#include "util/timespec.h"
+#include "util/vma.h"
+
+#include "drm-uapi/virtgpu_drm.h"
+/* We also use some types/defines from the host drm/msm uabi: */
+#include "drm-uapi/msm_drm.h"
+
+#include "vdrm.h"
+#include "msm_proto.h"
+
+struct virtio_device {
+   struct fd_device base;
+
+   struct vdrm_device *vdrm;
+
+   uint32_t next_blob_id;
+   struct msm_shmem *shmem;
+
+   /*
+    * Notes on address space allocation:
+    *
+    * In both the import (GEM_INFO) and new (GEM_NEW) path we allocate
+    * the iova.  Since the iova (vma on kernel side) is local to the
+    * address space, and that is 1:1 with drm fd (which is 1:1 with
+    * virtio_device and therefore address_space) which is not shared
+    * with anything outside of the driver, and because of the handle
+    * de-duplication, we can safely assume that an iova has not yet
+    * been set on imported buffers.
+    *
+    * The other complication with userspace allocated iova is that
+    * the kernel holds on to a reference to the bo (and the GPU is
+    * still using it's iova) until the submit retires.  So a per-pipe
+    * retire_queue is used to hold an extra reference to the submit
+    * (and indirectly all the bo's referenced) until the out-fence is
+    * signaled.
+    */
+   struct util_vma_heap address_space;
+   simple_mtx_t address_space_lock;
+};
+FD_DEFINE_CAST(fd_device, virtio_device);
+
+struct fd_device *virtio_device_new(int fd, drmVersionPtr version);
+
+static inline void
+virtio_dev_free_iova(struct fd_device *dev, uint64_t iova, uint32_t size)
+{
+   struct virtio_device *virtio_dev = to_virtio_device(dev);
+
+   simple_mtx_lock(&virtio_dev->address_space_lock);
+   util_vma_heap_free(&virtio_dev->address_space, iova, size);
+   simple_mtx_unlock(&virtio_dev->address_space_lock);
+}
+
+static inline uint64_t
+virtio_dev_alloc_iova(struct fd_device *dev, uint32_t size)
+{
+   struct virtio_device *virtio_dev = to_virtio_device(dev);
+   uint64_t iova;
+
+   simple_mtx_lock(&virtio_dev->address_space_lock);
+   iova = util_vma_heap_alloc(&virtio_dev->address_space, size, os_page_size);
+   simple_mtx_unlock(&virtio_dev->address_space_lock);
+
+   return iova;
+}
+
+struct virtio_pipe {
+   struct fd_pipe base;
+   uint32_t pipe;
+   uint32_t gpu_id;
+   uint64_t chip_id;
+   uint64_t gmem_base;
+   uint32_t gmem;
+   uint32_t queue_id;
+   uint32_t ring_idx;
+   struct slab_parent_pool ring_pool;
+
+   /**
+    * We know that the kernel allocated fence seqno's sequentially per-
+    * submitqueue in a range 1..INT_MAX, which is incremented *after* any
+    * point where the submit ioctl could be restarted.  So we just *guess*
+    * what the next seqno fence will be to avoid having to synchronize the
+    * submit with the host.
+    *
+    * TODO maybe we need version_minor bump so we can make the 1..INT_MAX
+    * assumption.. it is only really true after:
+    *
+    *   ca3ffcbeb0c8 ("drm/msm/gpu: Don't allow zero fence_id")
+    */
+   int32_t next_submit_fence;
+
+   /**
+    * When userspace allocates iova, we need to defer deleting bo's (and
+    * therefore releasing their address) until submits referencing them
+    * have completed.  This is accomplished by enqueueing a job, holding
+    * a reference to the submit, that waits on the submit's out-fence
+    * before dropping the reference to the submit.  The submit holds a
+    * reference to the associated ring buffers, which in turn hold a ref
+    * to the associated bo's.
+    */
+   struct util_queue retire_queue;
+};
+FD_DEFINE_CAST(fd_pipe, virtio_pipe);
+
+struct fd_pipe *virtio_pipe_new(struct fd_device *dev, enum fd_pipe_id id,
+                                uint32_t prio);
+
+struct fd_submit *virtio_submit_new(struct fd_pipe *pipe);
+
+struct virtio_bo {
+   struct fd_bo base;
+   uint64_t alloc_time_ns;
+   uint64_t offset;
+   uint32_t res_id;
+   uint32_t blob_id;
+   uint32_t upload_seqno;
+   bool has_upload_seqno;
+};
+FD_DEFINE_CAST(fd_bo, virtio_bo);
+
+struct fd_bo *virtio_bo_new(struct fd_device *dev, uint32_t size, uint32_t flags);
+struct fd_bo *virtio_bo_from_handle(struct fd_device *dev, uint32_t size,
+                                    uint32_t handle);
+
+/*
+ * Internal helpers:
+ */
+int virtio_simple_ioctl(struct fd_device *dev, unsigned cmd, void *req);
+
+#endif /* VIRTIO_PRIV_H_ */

src/freedreno/drm/virtio/virtio_ringbuffer.c

@@ -0,0 +1,220 @@
+/*
+ * Copyright © 2022 Google, Inc.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include <assert.h>
+#include <inttypes.h>
+#include <pthread.h>
+
+#include "util/libsync.h"
+#include "util/os_file.h"
+
+#include "drm/freedreno_ringbuffer_sp.h"
+#include "virtio_priv.h"
+
+static void
+retire_execute(void *job, void *gdata, int thread_index)
+{
+   struct fd_submit_sp *fd_submit = job;
+
+   MESA_TRACE_FUNC();
+
+   fd_fence_wait(fd_submit->out_fence);
+}
+
+static void
+retire_cleanup(void *job, void *gdata, int thread_index)
+{
+   struct fd_submit_sp *fd_submit = job;
+   fd_submit_del(&fd_submit->base);
+}
+
+static int
+flush_submit_list(struct list_head *submit_list)
+{
+   struct fd_submit_sp *fd_submit = to_fd_submit_sp(last_submit(submit_list));
+   struct virtio_pipe *virtio_pipe = to_virtio_pipe(fd_submit->base.pipe);
+   struct fd_pipe *pipe = &virtio_pipe->base;
+   struct fd_device *dev = pipe->dev;
+
+   unsigned nr_cmds = 0;
+
+   MESA_TRACE_FUNC();
+
+   /* Determine the number of extra cmds's from deferred submits that
+    * we will be merging in:
+    */
+   foreach_submit (submit, submit_list) {
+      assert(submit->pipe == &virtio_pipe->base);
+      nr_cmds += to_fd_ringbuffer_sp(submit->primary)->u.nr_cmds;
+   }
+
+   /* TODO we can get rid of the extra copy into the req by just
+    * assuming the max amount that nr->bos will grow is by the
+    * nr_cmds, and just over-allocate a bit.
+    */
+
+   struct drm_msm_gem_submit_cmd cmds[nr_cmds];
+
+   unsigned cmd_idx = 0;
+
+   /* Build up the table of cmds, and for all but the last submit in the
+    * list, merge their bo tables into the last submit.
+    */
+   foreach_submit_safe (submit, submit_list) {
+      struct fd_ringbuffer_sp *deferred_primary =
+         to_fd_ringbuffer_sp(submit->primary);
+
+      for (unsigned i = 0; i < deferred_primary->u.nr_cmds; i++) {
+         struct fd_bo *ring_bo = deferred_primary->u.cmds[i].ring_bo;
+         cmds[cmd_idx].type = MSM_SUBMIT_CMD_BUF;
+         cmds[cmd_idx].submit_idx = fd_submit_append_bo(fd_submit, ring_bo);
+         cmds[cmd_idx].submit_offset = submit_offset(ring_bo, deferred_primary->offset);
+         cmds[cmd_idx].size = deferred_primary->u.cmds[i].size;
+         cmds[cmd_idx].pad = 0;
+         cmds[cmd_idx].nr_relocs = 0;
+
+         cmd_idx++;
+      }
+
+      /* We are merging all the submits in the list into the last submit,
+       * so the remainder of the loop body doesn't apply to the last submit
+       */
+      if (submit == last_submit(submit_list)) {
+         DEBUG_MSG("merged %u submits", cmd_idx);
+         break;
+      }
+
+      struct fd_submit_sp *fd_deferred_submit = to_fd_submit_sp(submit);
+      for (unsigned i = 0; i < fd_deferred_submit->nr_bos; i++) {
+         /* Note: if bo is used in both the current submit and the deferred
+          * submit being merged, we expect to hit the fast-path as we add it
+          * to the current submit:
+          */
+         fd_submit_append_bo(fd_submit, fd_deferred_submit->bos[i]);
+      }
+
+      /* Now that the cmds/bos have been transfered over to the current submit,
+       * we can remove the deferred submit from the list and drop it's reference
+       */
+      list_del(&submit->node);
+      fd_submit_del(submit);
+   }
+
+   /* Needs to be after get_cmd() as that could create bos/cmds table:
+    *
+    * NOTE allocate on-stack in the common case, but with an upper-
+    * bound to limit on-stack allocation to 4k:
+    */
+   const unsigned bo_limit = 4096 / sizeof(struct drm_msm_gem_submit_bo);
+   bool bos_on_stack = fd_submit->nr_bos < bo_limit;
+   struct drm_msm_gem_submit_bo
+      _submit_bos[bos_on_stack ? fd_submit->nr_bos : 0];
+   struct drm_msm_gem_submit_bo *submit_bos;
+   uint32_t _guest_handles[bos_on_stack ? fd_submit->nr_bos : 0];
+   uint32_t *guest_handles;
+   if (bos_on_stack) {
+      submit_bos = _submit_bos;
+      guest_handles = _guest_handles;
+   } else {
+      submit_bos = malloc(fd_submit->nr_bos * sizeof(submit_bos[0]));
+      guest_handles = malloc(fd_submit->nr_bos * sizeof(guest_handles[0]));
+   }
+
+   uint32_t nr_guest_handles = 0;
+   for (unsigned i = 0; i < fd_submit->nr_bos; i++) {
+      struct virtio_bo *virtio_bo = to_virtio_bo(fd_submit->bos[i]);
+
+      if (virtio_bo->base.alloc_flags & FD_BO_SHARED)
+         guest_handles[nr_guest_handles++] = virtio_bo->base.handle;
+
+      submit_bos[i].flags = fd_submit->bos[i]->reloc_flags;
+      submit_bos[i].handle = virtio_bo->res_id;
+      submit_bos[i].presumed = 0;
+   }
+
+   if (virtio_pipe->next_submit_fence <= 0)
+      virtio_pipe->next_submit_fence = 1;
+
+   uint32_t kfence = virtio_pipe->next_submit_fence++;
+
+   /* TODO avoid extra memcpy, and populate bo's and cmds directly
+    * into the req msg
+    */
+   unsigned bos_len = fd_submit->nr_bos * sizeof(struct drm_msm_gem_submit_bo);
+   unsigned cmd_len = nr_cmds * sizeof(struct drm_msm_gem_submit_cmd);
+   unsigned req_len = sizeof(struct msm_ccmd_gem_submit_req) + bos_len + cmd_len;
+   struct msm_ccmd_gem_submit_req *req = malloc(req_len);
+
+   req->hdr      = MSM_CCMD(GEM_SUBMIT, req_len);
+   req->flags    = virtio_pipe->pipe;
+   req->queue_id = virtio_pipe->queue_id;
+   req->nr_bos   = fd_submit->nr_bos;
+   req->nr_cmds  = nr_cmds;
+   req->fence    = kfence;
+
+   memcpy(req->payload, submit_bos, bos_len);
+   memcpy(req->payload + bos_len, cmds, cmd_len);
+
+   struct fd_fence *out_fence = fd_submit->out_fence;
+
+   out_fence->kfence = kfence;
+
+   /* Even if gallium driver hasn't requested a fence-fd, request one.
+    * This way, if we have to block waiting for the fence, we can do
+    * it in the guest, rather than in the single-threaded host.
+    */
+   out_fence->use_fence_fd = true;
+
+   if (pipe->no_implicit_sync) {
+      req->flags |= MSM_SUBMIT_NO_IMPLICIT;
+      nr_guest_handles = 0;
+   }
+
+   struct vdrm_execbuf_params p = {
+      .req = &req->hdr,
+      .handles = guest_handles,
+      .num_handles = nr_guest_handles,
+      .has_in_fence_fd = !!(fd_submit->in_fence_fd != -1),
+      .needs_out_fence_fd = true,
+      .fence_fd = fd_submit->in_fence_fd,
+      .ring_idx = virtio_pipe->ring_idx,
+   };
+   vdrm_execbuf(to_virtio_device(dev)->vdrm, &p);
+
+   out_fence->fence_fd = p.fence_fd;
+
+   free(req);
+
+   if (!bos_on_stack) {
+      free(submit_bos);
+      free(guest_handles);
+   }
+
+   if (fd_submit->in_fence_fd != -1)
+      close(fd_submit->in_fence_fd);
+
+   fd_submit_ref(&fd_submit->base);
+
+   util_queue_fence_init(&fd_submit->retire_fence);
+
+   util_queue_add_job(&virtio_pipe->retire_queue,
+                      fd_submit, &fd_submit->retire_fence,
+                      retire_execute,
+                      retire_cleanup,
+                      0);
+
+   return 0;
+}
+
+struct fd_submit *
+virtio_submit_new(struct fd_pipe *pipe)
+{
+   /* We don't do any translation from internal FD_RELOC flags to MSM flags. */
+   STATIC_ASSERT(FD_RELOC_READ == MSM_SUBMIT_BO_READ);
+   STATIC_ASSERT(FD_RELOC_WRITE == MSM_SUBMIT_BO_WRITE);
+   STATIC_ASSERT(FD_RELOC_DUMP == MSM_SUBMIT_BO_DUMP);
+
+   return fd_submit_sp_new(pipe, flush_submit_list);
+}