init

src/panfrost/lib/.gitignore

@@ -0,0 +1 @@
+panloader

src/panfrost/lib/kmod/meson.build

@@ -0,0 +1,23 @@
+# Copyright © 2023 Collabora
+# SPDX-License-Identifier: MIT
+
+libpankmod_lib_files = files(
+  'pan_kmod.c',
+  'panfrost_kmod.c',
+  'panthor_kmod.c',
+)
+
+libpankmod_lib = static_library(
+  'pankmod_lib',
+  [libpankmod_lib_files],
+  include_directories : [inc_include, inc_src, inc_panfrost],
+  c_args : [no_override_init_args],
+  gnu_symbol_visibility : 'hidden',
+  dependencies: [dep_libdrm, idep_mesautil, idep_pan_packers, libpanfrost_model_dep],
+  build_by_default : false,
+)
+
+libpankmod_dep = declare_dependency(
+  include_directories: [inc_include, inc_src],
+  dependencies: [dep_libdrm],
+)

src/panfrost/lib/kmod/pan_kmod.c

@@ -0,0 +1,301 @@
+/*
+ * Copyright © 2023 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include <string.h>
+#include <xf86drm.h>
+
+#include "util/cache_ops.h"
+#include "util/u_memory.h"
+#include "util/macros.h"
+#include "pan_kmod.h"
+#include "pan_kmod_backend.h"
+
+extern const struct pan_kmod_ops panfrost_kmod_ops;
+extern const struct pan_kmod_ops panthor_kmod_ops;
+
+static const struct {
+   const char *name;
+   const struct pan_kmod_ops *ops;
+} drivers[] = {
+   {
+      "panfrost",
+      &panfrost_kmod_ops,
+   },
+   {
+      "panthor",
+      &panthor_kmod_ops,
+   },
+};
+
+static void *
+default_zalloc(const struct pan_kmod_allocator *allocator, size_t size,
+               UNUSED bool transient)
+{
+   return os_calloc(1, size);
+}
+
+static void
+default_free(const struct pan_kmod_allocator *allocator, void *data)
+{
+   os_free(data);
+}
+
+static const struct pan_kmod_allocator default_allocator = {
+   .zalloc = default_zalloc,
+   .free = default_free,
+};
+
+struct pan_kmod_dev *
+pan_kmod_dev_create(int fd, uint32_t flags,
+                    const struct pan_kmod_allocator *allocator)
+{
+   drmVersionPtr version = drmGetVersion(fd);
+   struct pan_kmod_dev *dev = NULL;
+
+   if (!version)
+      return NULL;
+
+   if (!allocator)
+      allocator = &default_allocator;
+
+   const char *drv_name = version->name;
+   const struct pan_kmod_driver drv_info = {
+      .version = {
+         .major = version->version_major,
+         .minor = version->version_minor,
+      },
+   };
+
+   for (unsigned i = 0; i < ARRAY_SIZE(drivers); i++) {
+      if (!strcmp(drivers[i].name, drv_name)) {
+         const struct pan_kmod_ops *ops = drivers[i].ops;
+
+         dev = ops->dev_create(fd, flags, &drv_info, allocator);
+         break;
+      }
+   }
+
+   drmFreeVersion(version);
+   return dev;
+}
+
+void
+pan_kmod_dev_destroy(struct pan_kmod_dev *dev)
+{
+   dev->ops->dev_destroy(dev);
+}
+
+struct pan_kmod_bo *
+pan_kmod_bo_alloc(struct pan_kmod_dev *dev, struct pan_kmod_vm *exclusive_vm,
+                  uint64_t size, uint32_t flags)
+{
+   struct pan_kmod_bo *bo;
+
+   bo = dev->ops->bo_alloc(dev, exclusive_vm, size, flags);
+   if (!bo)
+      return NULL;
+
+   /* We intentionally don't take the lock when filling the sparse array,
+    * because we just created the BO, and haven't exported it yet, so
+    * there's no risk of imports racing with our BO insertion.
+    */
+   struct pan_kmod_bo **slot =
+      util_sparse_array_get(&dev->handle_to_bo.array, bo->handle);
+
+   if (!slot) {
+      mesa_loge("failed to allocate slot in the handle_to_bo array");
+      bo->dev->ops->bo_free(bo);
+      return NULL;
+   }
+
+   assert(*slot == NULL);
+   *slot = bo;
+   return bo;
+}
+
+void
+pan_kmod_bo_put(struct pan_kmod_bo *bo)
+{
+   if (!bo)
+      return;
+
+   int32_t refcnt = p_atomic_dec_return(&bo->refcnt);
+
+   assert(refcnt >= 0);
+
+   if (refcnt)
+      return;
+
+   struct pan_kmod_dev *dev = bo->dev;
+
+   simple_mtx_lock(&dev->handle_to_bo.lock);
+
+   /* If some import took a ref on this BO while we were trying to acquire the
+    * lock, skip the destruction.
+    */
+   if (!p_atomic_read(&bo->refcnt)) {
+      struct pan_kmod_bo **slot = (struct pan_kmod_bo **)util_sparse_array_get(
+         &dev->handle_to_bo.array, bo->handle);
+
+      assert(slot);
+      *slot = NULL;
+      bo->dev->ops->bo_free(bo);
+   }
+
+   simple_mtx_unlock(&dev->handle_to_bo.lock);
+}
+
+struct pan_kmod_bo *
+pan_kmod_bo_import(struct pan_kmod_dev *dev, int fd)
+{
+   struct pan_kmod_bo *bo = NULL;
+   struct pan_kmod_bo **slot;
+
+   simple_mtx_lock(&dev->handle_to_bo.lock);
+
+   uint32_t handle;
+   int ret = drmPrimeFDToHandle(dev->fd, fd, &handle);
+   if (ret)
+      goto err_unlock;
+
+   slot = util_sparse_array_get(&dev->handle_to_bo.array, handle);
+   if (!slot)
+      goto err_close_handle;
+
+   if (*slot) {
+      bo = *slot;
+
+      p_atomic_inc(&bo->refcnt);
+   } else {
+      size_t size = lseek(fd, 0, SEEK_END);
+      if (size == 0 || size == (size_t)-1) {
+         mesa_loge("invalid dmabuf size");
+         goto err_close_handle;
+      }
+
+      bo = dev->ops->bo_import(dev, handle, size);
+      if (!bo)
+         goto err_close_handle;
+
+      *slot = bo;
+   }
+
+   assert(p_atomic_read(&bo->refcnt) > 0);
+
+   simple_mtx_unlock(&dev->handle_to_bo.lock);
+
+   return bo;
+
+err_close_handle:
+   drmCloseBufferHandle(dev->fd, handle);
+
+err_unlock:
+   simple_mtx_unlock(&dev->handle_to_bo.lock);
+
+   return NULL;
+}
+
+void
+pan_kmod_flush_bo_map_syncs_locked(struct pan_kmod_dev *dev)
+{
+   ASSERTED int ret = dev->ops->flush_bo_map_syncs(dev);
+   assert(!ret);
+
+   util_dynarray_foreach(&dev->pending_bo_syncs.array,
+                         struct pan_kmod_deferred_bo_sync, sync)
+      sync->bo->has_pending_deferred_syncs = false;
+
+   util_dynarray_clear(&dev->pending_bo_syncs.array);
+}
+
+void
+pan_kmod_flush_bo_map_syncs(struct pan_kmod_dev *dev)
+{
+   if (dev->props.is_io_coherent)
+      return;
+
+   /* Barrier to make sure all flush/invalidate requests are effective. */
+   if (p_atomic_xchg(&dev->pending_bo_syncs.user_cache_ops_pending, false))
+      util_post_flush_inval_fence();
+
+   /* This can be racy, but that's fine, because we expect a future call to
+    * pan_kmod_flush_bo_map_syncs() if new ops are being added while we check
+    * this value.
+    */
+   if (!util_dynarray_num_elements(&dev->pending_bo_syncs.array,
+                                   struct pan_kmod_deferred_bo_sync))
+      return;
+
+   simple_mtx_lock(&dev->pending_bo_syncs.lock);
+   pan_kmod_flush_bo_map_syncs_locked(dev);
+   simple_mtx_unlock(&dev->pending_bo_syncs.lock);
+}
+
+/* Arbitrary limit for now. Pick something bigger or make it configurable if it
+ * becomes problematic.
+ */
+#define MAX_PENDING_SYNC_OPS 4096
+
+void
+pan_kmod_queue_bo_map_sync(struct pan_kmod_bo *bo, uint64_t bo_offset,
+                           void *cpu_ptr, uint64_t range,
+                           enum pan_kmod_bo_sync_type type)
+{
+   struct pan_kmod_dev *dev = bo->dev;
+
+   /* Nothing to do if the buffer is IO coherent or if the BO is not mapped
+    * cacheable.
+    */
+   if (!(bo->flags & PAN_KMOD_BO_FLAG_WB_MMAP) ||
+       (bo->flags & PAN_KMOD_BO_FLAG_IO_COHERENT))
+      return;
+
+   /* If we have userspace cache flushing ops, use them instead of trapping
+    * through to the kernel.
+    */
+   if (pan_kmod_can_sync_bo_map_from_userland(dev)) {
+      /* Pre-flush needs to be executed before each flush/inval operation, but
+       * we can batch the post flush/inval fence. util_pre_flush_fence() being
+       * a NOP on aarch64, it's effectively free there, but we keep it here for
+       * clarity (not sure we care about Mali on x86 to be honest :D).
+       */
+      util_pre_flush_fence();
+
+      if (type == PAN_KMOD_BO_SYNC_CPU_CACHE_FLUSH)
+         util_flush_range_no_fence(cpu_ptr, range);
+      else
+         util_flush_inval_range_no_fence(cpu_ptr, range);
+
+      /* The util_pre_flush_inval_fence() is inserted by
+       * pan_kmod_flush_bo_map_syncs() to avoid unnecessary serialization when
+       * flush/invalidate operations are batched.
+       */
+      p_atomic_set(&dev->pending_bo_syncs.user_cache_ops_pending, true);
+      return;
+   }
+
+   simple_mtx_lock(&dev->pending_bo_syncs.lock);
+
+   /* If we reach the limit, flush the pending ops before queuing new ones. */
+   if (util_dynarray_num_elements(&dev->pending_bo_syncs.array,
+                                  struct pan_kmod_deferred_bo_sync) >=
+       MAX_PENDING_SYNC_OPS)
+      pan_kmod_flush_bo_map_syncs_locked(dev);
+
+   uint64_t start = bo_offset & ~((uint64_t)util_cache_granularity() - 1);
+   uint64_t end = ALIGN_POT(bo_offset + range, util_cache_granularity());
+
+   struct pan_kmod_deferred_bo_sync new_sync = {
+      .bo = bo,
+      .start = start,
+      .size = end - start,
+      .type = type,
+   };
+
+   bo->has_pending_deferred_syncs = true;
+   util_dynarray_append(&dev->pending_bo_syncs.array, new_sync);
+
+   simple_mtx_unlock(&dev->pending_bo_syncs.lock);
+}

src/panfrost/lib/kmod/pan_kmod.h

@@ -0,0 +1,788 @@
+/*
+ * Copyright © 2023 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+/*
+ * This file exposes some core KMD functionalities in a driver-agnostic way.
+ * The drivers are still assumed to be regular DRM drivers, such that some
+ * operations can be handled generically.
+ *
+ * Any operation that's too specific to be abstracted can either have a backend
+ * specific helper exposed through pan_kmod_<backend>.h, or no helper at all
+ * (in the latter case, users are expected to call the ioctl directly).
+ *
+ * If some operations are not natively supported by a KMD, the kmod backend
+ * should fail or emulate the functionality (if deemed necessary).
+ */
+
+#pragma once
+
+#include <fcntl.h>
+#include <inttypes.h>
+#include <unistd.h>
+#include <xf86drm.h>
+
+#include "drm-uapi/drm.h"
+
+#include "util/cache_ops.h"
+#include "util/log.h"
+#include "util/macros.h"
+#include "util/os_file.h"
+#include "util/os_mman.h"
+#include "util/ralloc.h"
+#include "util/simple_mtx.h"
+#include "util/sparse_array.h"
+#include "util/u_atomic.h"
+#include "util/u_dynarray.h"
+
+#include "kmod/panthor_kmod.h"
+#include "pan_trace.h"
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+struct pan_kmod_dev;
+
+/* GPU VM creation flags. */
+enum pan_kmod_vm_flags {
+   /* Set if you want the VM to automatically assign virtual addresses when
+    * pan_kmod_vm_map(). If this flag is set, all pan_kmod_vm_map() calls
+    * must have va=PAN_KMOD_VM_MAP_AUTO_VA.
+    */
+   PAN_KMOD_VM_FLAG_AUTO_VA = BITFIELD_BIT(0),
+
+   /* Let the backend know whether it should track the VM activity or not.
+    * Needed if PAN_KMOD_VM_OP_MODE_DEFER_TO_NEXT_IDLE_POINT is used.
+    */
+   PAN_KMOD_VM_FLAG_TRACK_ACTIVITY = BITFIELD_BIT(1),
+};
+
+#define PAN_PGSIZE_4K (uint64_t)0x1000
+#define PAN_PGSIZE_2M (uint64_t)0x200000
+
+/* Object representing a GPU VM. */
+struct pan_kmod_vm {
+   /* Page sizes supported by this VM. */
+   uint64_t pgsize_bitmap;
+
+   /* Combination of pan_kmod_vm_flags flags. */
+   uint32_t flags;
+
+   /* The VM handle returned by the KMD. If the KMD supports only one VM per
+    * context, this should be zero.
+    */
+   uint32_t handle;
+
+   /* Device this VM was created from. */
+   struct pan_kmod_dev *dev;
+};
+
+/* Buffer object flags. */
+enum pan_kmod_bo_flags {
+   /* Allow GPU execution on this buffer. */
+   PAN_KMOD_BO_FLAG_EXECUTABLE = BITFIELD_BIT(0),
+
+   /* Allocate memory when a GPU fault occurs instead of allocating
+    * up-front.
+    */
+   PAN_KMOD_BO_FLAG_ALLOC_ON_FAULT = BITFIELD_BIT(1),
+
+   /* If set, the buffer object will never be CPU-mapped in userspace. */
+   PAN_KMOD_BO_FLAG_NO_MMAP = BITFIELD_BIT(2),
+
+   /* Set when the buffer object has been exported. Users don't directly
+    * control this flag, it's set when pan_kmod_bo_export() is called.
+    */
+   PAN_KMOD_BO_FLAG_EXPORTED = BITFIELD_BIT(3),
+
+   /* Set when the buffer object has been impported. Users don't directly
+    * control this flag, it's set when pan_kmod_bo_import() is called.
+    */
+   PAN_KMOD_BO_FLAG_IMPORTED = BITFIELD_BIT(4),
+
+   /* If set, the buffer in mapped GPU-uncached when pan_kmod_vm_map()
+    * is called.
+    */
+   PAN_KMOD_BO_FLAG_GPU_UNCACHED = BITFIELD_BIT(5),
+
+   /* CPU map the buffer object in userspace by forcing the "Write-Back
+    * Cacheable" cacheability attribute. The mapping otherwise uses the
+    * "Non-Cacheable" attribute if the ACE-Lite coherency protocol isn't
+    * supported by the GPU.
+    */
+   PAN_KMOD_BO_FLAG_WB_MMAP = BITFIELD_BIT(6),
+
+   /* Set by default when the device is IO coherent. We might want to
+    * make it optional at some point and pass a NON_COHERENT flag to
+    * the KMD to force non-coherent mappings on IO coherent setup.
+    */
+   PAN_KMOD_BO_FLAG_IO_COHERENT = BITFIELD_BIT(7),
+};
+
+/* Allowed group priority flags. */
+enum pan_kmod_group_allow_priority_flags {
+   /* Allow low priority group. */
+   PAN_KMOD_GROUP_ALLOW_PRIORITY_LOW = BITFIELD_BIT(0),
+
+   /* Allow medium priority group. */
+   PAN_KMOD_GROUP_ALLOW_PRIORITY_MEDIUM = BITFIELD_BIT(1),
+
+   /* Allow high priority group. */
+   PAN_KMOD_GROUP_ALLOW_PRIORITY_HIGH = BITFIELD_BIT(2),
+
+   /* Allow realtime priority group. */
+   PAN_KMOD_GROUP_ALLOW_PRIORITY_REALTIME = BITFIELD_BIT(3),
+};
+
+/* Buffer object. */
+struct pan_kmod_bo {
+   /* Atomic reference count. The only reason we need to refcnt BOs at this
+    * level is because of how DRM prime import works: the import logic
+    * returns the handle of an existing object if the object was previously
+    * imported or was created by the driver.
+    * In order to prevent call GEM_CLOSE on an object that's still supposed
+    * to be active, we need count the number of users left.
+    */
+   int32_t refcnt;
+
+   /* Size of the buffer object. */
+   uint64_t size;
+
+   /* Handle attached to the buffer object. */
+   uint32_t handle;
+
+   /* Combination of pan_kmod_bo_flags flags. */
+   uint32_t flags;
+
+   /* True if some deferred syncs targetting this BO are pending. */
+   bool has_pending_deferred_syncs;
+
+   /* If non-NULL, the buffer object can only by mapped on this VM. Typical
+    * the case for all internal/non-shareable buffers. The backend can
+    * optimize things based on this information. Calling pan_kmod_bo_export()
+    * on such buffer objects is forbidden.
+    */
+   struct pan_kmod_vm *exclusive_vm;
+
+   /* The device this buffer object was created from. */
+   struct pan_kmod_dev *dev;
+
+   /* User private data. Use pan_kmod_bo_{set,get}_user_priv() to access it. */
+   void *user_priv;
+};
+
+/* List of GPU properties needed by the UMD. */
+struct pan_kmod_dev_props {
+   /* GPU ID. */
+   uint64_t gpu_id;
+
+   /* GPU variant. */
+   uint32_t gpu_variant;
+
+   /* Bitmask encoding the number of shader cores exposed by the GPU. */
+   uint64_t shader_present;
+
+   /* Tiler features bits. */
+   uint32_t tiler_features;
+
+   /* Memory related feature bits. */
+   uint32_t mem_features;
+
+   /* MMU feature bits. */
+   uint32_t mmu_features;
+#define MMU_FEATURES_VA_BITS(mmu_features) (mmu_features & 0xff)
+
+   /* Texture feature bits. */
+   uint32_t texture_features[4];
+
+   /* Maximum number of threads per core. */
+   uint32_t max_threads_per_core;
+
+   /* Maximum number of compute tasks per core. */
+   uint8_t max_tasks_per_core;
+
+   /* Maximum number of threads per workgroup. */
+   uint32_t max_threads_per_wg;
+
+   /* Number of registers per core. Can be used to determine the maximum
+    * number of threads that can be allocated for a specific shader based on
+    * the number of registers assigned to this shader.
+    */
+   uint32_t num_registers_per_core;
+
+   /* Maximum number of thread-local storage instance per core.
+    * If the GPU doesn't have a THREAD_TLS_ALLOC register or the register
+    * value is zero, the backend should assign the value of max_threads_per_core
+    * here.
+    */
+   uint32_t max_tls_instance_per_core;
+
+   /* AFBC feature bits. */
+   uint32_t afbc_features;
+
+   /* Support cycle count and timestamp propagation as job requirement */
+   bool gpu_can_query_timestamp;
+
+   /* Cycle counter and timestamp device coherent propogation is enabled */
+   bool timestamp_device_coherent;
+
+   /* GPU Timestamp frequency */
+   uint64_t timestamp_frequency;
+
+   /* A mask of flags containing the allowed group priorities. */
+   enum pan_kmod_group_allow_priority_flags allowed_group_priorities_mask;
+
+   /* Mask of BO flags supported by the KMD. */
+   uint32_t supported_bo_flags;
+
+   /* GPU is IO coherent, meaning BOs can be created with WB_MMAP without
+    * requiring explicit CPU cache maintenance.
+    */
+   bool is_io_coherent;
+};
+
+/* Memory allocator for kmod internal allocations. */
+struct pan_kmod_allocator {
+   /* Allocate and set to zero. */
+   void *(*zalloc)(const struct pan_kmod_allocator *allocator, size_t size,
+                   bool transient);
+
+   /* Free. */
+   void (*free)(const struct pan_kmod_allocator *allocator, void *data);
+
+   /* Private data allocator data. Can be NULL if unused. */
+   void *priv;
+};
+
+/* Synchronization type. */
+enum pan_kmod_sync_type {
+   PAN_KMOD_SYNC_TYPE_WAIT = 0,
+   PAN_KMOD_SYNC_TYPE_SIGNAL,
+};
+
+/* Synchronization operation. */
+struct pan_kmod_sync_op {
+   /* Type of operation. */
+   enum pan_kmod_sync_type type;
+
+   /* Syncobj handle. */
+   uint32_t handle;
+
+   /* Syncobj point. Zero for binary syncobjs. */
+   uint64_t point;
+};
+
+/* Special value passed to pan_kmod_vm_map() to signify the VM it should
+ * automatically allocate a VA. Only valid if the VM was created with
+ * PAN_KMOD_VM_FLAG_AUTO_VA.
+ */
+#define PAN_KMOD_VM_MAP_AUTO_VA ~0ull
+
+/* Special value return when the vm_map() operation failed. */
+#define PAN_KMOD_VM_MAP_FAILED ~0ull
+
+/* VM operations can be executed in different modes. */
+enum pan_kmod_vm_op_mode {
+   /* The map/unmap operation is executed immediately, which might cause
+    * GPU faults if the GPU was still accessing buffers when we unmap or
+    * remap.
+    */
+   PAN_KMOD_VM_OP_MODE_IMMEDIATE,
+
+   /* The map/unmap operation is executed asynchronously, and the user
+    * provides explicit wait/signal sync operations.
+    */
+   PAN_KMOD_VM_OP_MODE_ASYNC,
+
+   /* The map/unmap operation is executed when the next GPU/VM idle-point
+    * is reached. This guarantees fault-free unmap/remap operations when the
+    * kmod user doesn't want to deal with synchronizations explicitly.
+    */
+   PAN_KMOD_VM_OP_MODE_DEFER_TO_NEXT_IDLE_POINT,
+};
+
+/* VM operation type. */
+enum pan_kmod_vm_op_type {
+   /* Map a buffer object. */
+   PAN_KMOD_VM_OP_TYPE_MAP,
+
+   /* Unmap a VA range. */
+   PAN_KMOD_VM_OP_TYPE_UNMAP,
+
+   /* Do nothing. Used as a way to execute sync operations on a VM queue,
+    * without touching the VM.
+    */
+   PAN_KMOD_VM_OP_TYPE_SYNC_ONLY,
+};
+
+/* VM operation data. */
+struct pan_kmod_vm_op {
+   /* The type of operation being requested. */
+   enum pan_kmod_vm_op_type type;
+
+   /* VA range. */
+   struct {
+      /* Start of the VA range.
+       * Must be PAN_KMOD_VM_MAP_AUTO_VA if PAN_KMOD_VM_FLAG_AUTO_VA was set
+       * at VM creation time. In that case, the allocated VA is returned
+       * in this field.
+       */
+      uint64_t start;
+
+      /* Size of the VA range */
+      uint64_t size;
+   } va;
+
+   union {
+      /* Arguments specific to map operations. */
+      struct {
+         /* Buffer object to map. */
+         struct pan_kmod_bo *bo;
+
+         /* Offset in the buffer object. */
+         off_t bo_offset;
+      } map;
+   };
+
+   /* Synchronization operations attached to the VM operation. */
+   struct {
+      /* Number of synchronization operations. Must be zero if mode is
+       * PAN_KMOD_VM_OP_MODE_IMMEDIATE or PAN_KMOD_VM_OP_MODE_WAIT_IDLE.
+       */
+      uint32_t count;
+
+      /* Array of synchronization operation descriptors. NULL if count is zero. */
+      const struct pan_kmod_sync_op *array;
+   } syncs;
+};
+
+/* VM state. */
+enum pan_kmod_vm_state {
+   PAN_KMOD_VM_USABLE,
+   PAN_KMOD_VM_FAULTY,
+};
+
+/* Device flags. */
+enum pan_kmod_dev_flags {
+   /* Set when the fd passed to pan_kmod_create() is expected to be
+    * owned by the device, iff the device creation succeeded.
+    */
+   PAN_KMOD_DEV_FLAG_OWNS_FD = (1 << 0),
+
+   /* Force BO syncs through the kernel. */
+   PAN_KMOD_DEV_FLAG_MMAP_SYNC_THROUGH_KERNEL = (1 << 1),
+};
+
+/* Encode a virtual address range. */
+struct pan_kmod_va_range {
+   /* Start of the VA range. */
+   uint64_t start;
+
+   /* Size of the VA range. */
+   uint64_t size;
+};
+
+/* KMD information. */
+struct pan_kmod_driver {
+   /* KMD version. */
+   struct {
+      uint32_t major;
+      uint32_t minor;
+   } version;
+};
+
+/* KMD backend vtable.
+ *
+ * All methods described there are mandatory, unless explicitly flagged as
+ * optional.
+ */
+struct pan_kmod_ops {
+   /* Create a pan_kmod_dev object.
+    * Return NULL if the creation fails for any reason.
+    */
+   struct pan_kmod_dev *(*dev_create)(
+      int fd, uint32_t flags, const struct pan_kmod_driver *drv_info,
+      const struct pan_kmod_allocator *allocator);
+
+   /* Destroy a pan_kmod_dev object. */
+   void (*dev_destroy)(struct pan_kmod_dev *dev);
+
+   /* Query the maxium user VA range.
+    * Users are free to use a subset of this range if they need less VA space.
+    * This method is optional, when not specified, kmod assumes the whole VA
+    * space (extracted from MMU_FEATURES.VA_BITS) is usable.
+    */
+   struct pan_kmod_va_range (*dev_query_user_va_range)(
+      const struct pan_kmod_dev *dev);
+
+   /* Allocate a buffer object.
+    * Return NULL if the creation fails for any reason.
+    */
+   struct pan_kmod_bo *(*bo_alloc)(struct pan_kmod_dev *dev,
+                                   struct pan_kmod_vm *exclusive_vm,
+                                   uint64_t size, uint32_t flags);
+
+   /* Free buffer object. */
+   void (*bo_free)(struct pan_kmod_bo *bo);
+
+   /* Import a buffer object.
+    * Return NULL if the import fails for any reason.
+    */
+   struct pan_kmod_bo *(*bo_import)(struct pan_kmod_dev *dev, uint32_t handle,
+                                    uint64_t size);
+
+   /* Post export operations.
+    * Return 0 on success, -1 otherwise.
+    * This method is optional.
+    */
+   int (*bo_export)(struct pan_kmod_bo *bo, int dmabuf_fd);
+
+   /* Get the file offset to use to mmap() a buffer object. */
+   off_t (*bo_get_mmap_offset)(struct pan_kmod_bo *bo);
+
+   /* Flush the pending BO map syncs. */
+   int (*flush_bo_map_syncs)(struct pan_kmod_dev *dev);
+
+   /* Wait for a buffer object to be ready for read or read/write accesses. */
+   bool (*bo_wait)(struct pan_kmod_bo *bo, int64_t timeout_ns,
+                   bool for_read_only_access);
+
+   /* Make a buffer object evictable. This method is optional. */
+   void (*bo_make_evictable)(struct pan_kmod_bo *bo);
+
+   /* Make the buffer object unevictable. This method is optional. */
+   bool (*bo_make_unevictable)(struct pan_kmod_bo *bo);
+
+   /* Create a VM object. */
+   struct pan_kmod_vm *(*vm_create)(struct pan_kmod_dev *dev, uint32_t flags,
+                                    uint64_t va_start, uint64_t va_range);
+
+   /* Destroy a VM object. */
+   void (*vm_destroy)(struct pan_kmod_vm *vm);
+
+   /* Execute VM operations.
+    * Return 0 if the submission suceeds, -1 otherwise.
+    * For PAN_KMOD_VM_OP_MODE_IMMEDIATE submissions, the return value also
+    * reflects the successfulness of the VM operation, for other modes,
+    * if any of the VM operation fails, the VM might be flagged as unusable
+    * and users should create a new VM to recover.
+    */
+   int (*vm_bind)(struct pan_kmod_vm *vm, enum pan_kmod_vm_op_mode mode,
+                  struct pan_kmod_vm_op *ops, uint32_t op_count);
+
+   /* Query the VM state.
+    * This method is optional. When missing the VM is assumed to always be
+    * usable.
+    */
+   enum pan_kmod_vm_state (*vm_query_state)(struct pan_kmod_vm *vm);
+
+   /* Query the current GPU timestamp */
+   uint64_t (*query_timestamp)(const struct pan_kmod_dev *dev);
+
+   /* Label the BO */
+   void (*bo_set_label)(struct pan_kmod_dev *dev, struct pan_kmod_bo *bo, const char *label);
+};
+
+static inline bool
+pan_kmod_driver_version_at_least(const struct pan_kmod_driver *driver,
+                                 uint32_t major, uint32_t minor)
+{
+   if (driver->version.major < major)
+      return false;
+
+   if (driver->version.major > major)
+      return true;
+
+   return driver->version.minor >= minor;
+}
+
+enum pan_kmod_bo_sync_type {
+   PAN_KMOD_BO_SYNC_CPU_CACHE_FLUSH,
+   PAN_KMOD_BO_SYNC_CPU_CACHE_FLUSH_AND_INVALIDATE,
+};
+
+/* Used to queue BO sync operations. */
+struct pan_kmod_deferred_bo_sync {
+   struct pan_kmod_bo *bo;
+   uint64_t start;
+   uint64_t size;
+   enum pan_kmod_bo_sync_type type;
+};
+
+/* Device object. */
+struct pan_kmod_dev {
+   /* FD attached to the device. */
+   int fd;
+
+   /* Device flags. */
+   uint32_t flags;
+
+   /* KMD backing this device. */
+   struct pan_kmod_driver driver;
+
+   /* KMD-agnostic device properties. */
+   struct pan_kmod_dev_props props;
+
+   /* kmod backend ops assigned at device creation. */
+   const struct pan_kmod_ops *ops;
+
+   /* DRM prime import returns the handle of a pre-existing GEM if we are
+    * importing an object that was created by us or previously imported.
+    * We need to make sure we return the same pan_kmod_bo in that case,
+    * otherwise freeing one pan_kmod_bo will make all other BOs sharing
+    * the same handle invalid.
+    */
+   struct {
+      struct util_sparse_array array;
+      simple_mtx_t lock;
+   } handle_to_bo;
+
+   /* Pending BO syncs. */
+   struct {
+      bool user_cache_ops_pending;
+      struct util_dynarray array;
+      simple_mtx_t lock;
+   } pending_bo_syncs;
+
+   /* Allocator attached to the device. */
+   const struct pan_kmod_allocator *allocator;
+
+   /* User private data. Use pan_kmod_dev_{set,get}_user_priv() to access it. */
+   void *user_priv;
+};
+
+#define pan_kmod_ioctl(fd, op, arg)                                          \
+   ({                                                                        \
+      PAN_TRACE_SCOPE(PAN_TRACE_LIB_KMOD, "pan_kmod_ioctl op=" #op);         \
+      drmIoctl(fd, op, arg);                                                 \
+   })
+
+struct pan_kmod_dev *
+pan_kmod_dev_create(int fd, uint32_t flags,
+                    const struct pan_kmod_allocator *allocator);
+
+void pan_kmod_dev_destroy(struct pan_kmod_dev *dev);
+
+static inline struct pan_kmod_va_range
+pan_kmod_dev_query_user_va_range(const struct pan_kmod_dev *dev)
+{
+   if (dev->ops->dev_query_user_va_range)
+      return dev->ops->dev_query_user_va_range(dev);
+
+   return (struct pan_kmod_va_range){
+      .start = 0,
+      .size = 1ull << MMU_FEATURES_VA_BITS(dev->props.mmu_features),
+   };
+}
+
+static inline void
+pan_kmod_dev_set_user_priv(struct pan_kmod_dev *dev, void *data)
+{
+   dev->user_priv = data;
+}
+
+static inline void *
+pan_kmod_dev_get_user_priv(struct pan_kmod_dev *dev)
+{
+   return dev->user_priv;
+}
+
+struct pan_kmod_bo *pan_kmod_bo_alloc(struct pan_kmod_dev *dev,
+                                      struct pan_kmod_vm *exclusive_vm,
+                                      uint64_t size, uint32_t flags);
+
+static inline struct pan_kmod_bo *
+pan_kmod_bo_get(struct pan_kmod_bo *bo)
+{
+   if (!bo)
+      return NULL;
+
+   ASSERTED int32_t refcnt = p_atomic_inc_return(&bo->refcnt);
+
+   /* If refcnt was zero before our increment, we're in trouble. */
+   assert(refcnt > 1);
+
+   return bo;
+}
+
+void pan_kmod_bo_put(struct pan_kmod_bo *bo);
+
+static inline void *
+pan_kmod_bo_cmdxchg_user_priv(struct pan_kmod_bo *bo, void *old_data,
+                              void *new_data)
+{
+   return (void *)p_atomic_cmpxchg((uintptr_t *)&bo->user_priv,
+                                   (uintptr_t)old_data, (uintptr_t)new_data);
+}
+
+static inline void
+pan_kmod_bo_set_user_priv(struct pan_kmod_bo *bo, void *data)
+{
+   bo->user_priv = data;
+}
+
+static inline void *
+pan_kmod_bo_get_user_priv(const struct pan_kmod_bo *bo)
+{
+   return bo->user_priv;
+}
+
+struct pan_kmod_bo *pan_kmod_bo_import(struct pan_kmod_dev *dev, int fd);
+
+static inline int
+pan_kmod_bo_export(struct pan_kmod_bo *bo)
+{
+   PAN_TRACE_FUNC(PAN_TRACE_LIB_KMOD);
+
+   int fd;
+
+   if (drmPrimeHandleToFD(bo->dev->fd, bo->handle, DRM_CLOEXEC | DRM_RDWR,
+                          &fd)) {
+      mesa_loge("drmPrimeHandleToFD() failed (err=%d)", errno);
+      return -1;
+   }
+
+   if (bo->dev->ops->bo_export && bo->dev->ops->bo_export(bo, fd)) {
+      close(fd);
+      return -1;
+   }
+
+   bo->flags |= PAN_KMOD_BO_FLAG_EXPORTED;
+   return fd;
+}
+
+static inline bool
+pan_kmod_bo_wait(struct pan_kmod_bo *bo, int64_t timeout_ns,
+                 bool for_read_only_access)
+{
+   PAN_TRACE_FUNC(PAN_TRACE_LIB_KMOD);
+
+   return bo->dev->ops->bo_wait(bo, timeout_ns, for_read_only_access);
+}
+
+static inline void
+pan_kmod_bo_make_evictable(struct pan_kmod_bo *bo)
+{
+   if (bo->dev->ops->bo_make_evictable)
+      bo->dev->ops->bo_make_evictable(bo);
+}
+
+static inline bool
+pan_kmod_bo_make_unevictable(struct pan_kmod_bo *bo)
+{
+   if (bo->dev->ops->bo_make_unevictable)
+      return bo->dev->ops->bo_make_unevictable(bo);
+
+   return true;
+}
+
+static inline void *
+pan_kmod_bo_mmap(struct pan_kmod_bo *bo, int prot, int flags, void *host_addr)
+{
+   PAN_TRACE_FUNC(PAN_TRACE_LIB_KMOD);
+
+   off_t mmap_offset;
+
+   /* Don't bother trying an mmap() if it's not allowed. */
+   if (bo->flags & PAN_KMOD_BO_FLAG_NO_MMAP)
+      return MAP_FAILED;
+
+   mmap_offset = bo->dev->ops->bo_get_mmap_offset(bo);
+   if (mmap_offset < 0)
+      return MAP_FAILED;
+
+   host_addr =
+      os_mmap(host_addr, bo->size, prot, flags, bo->dev->fd, mmap_offset);
+   if (host_addr == MAP_FAILED)
+      mesa_loge("mmap(..., size=%" PRIu64 ", prot=%d, flags=0x%x) failed: %s",
+                bo->size, prot, flags, strerror(errno));
+
+   return host_addr;
+}
+
+static inline bool
+pan_kmod_can_sync_bo_map_from_userland(struct pan_kmod_dev *dev)
+{
+   return util_has_cache_ops() &&
+          !(dev->flags & PAN_KMOD_DEV_FLAG_MMAP_SYNC_THROUGH_KERNEL);
+}
+
+void pan_kmod_queue_bo_map_sync(struct pan_kmod_bo *bo, uint64_t bo_offset,
+                                void *cpu_ptr, uint64_t range,
+                                enum pan_kmod_bo_sync_type type);
+
+void pan_kmod_flush_bo_map_syncs(struct pan_kmod_dev *dev);
+
+static inline void
+pan_kmod_set_bo_label(struct pan_kmod_dev *dev, struct pan_kmod_bo *bo, const char *label)
+{
+   PAN_TRACE_FUNC(PAN_TRACE_LIB_KMOD);
+
+   if (dev->ops->bo_set_label)
+      dev->ops->bo_set_label(dev, bo, label);
+}
+
+static inline uint64_t
+pan_kmod_bo_size(struct pan_kmod_bo *bo)
+{
+   return bo->size;
+}
+
+static inline uint32_t
+pan_kmod_bo_handle(struct pan_kmod_bo *bo)
+{
+   return bo->handle;
+}
+
+static inline struct pan_kmod_vm *
+pan_kmod_vm_create(struct pan_kmod_dev *dev, uint32_t flags, uint64_t va_start,
+                   uint64_t va_range)
+{
+   PAN_TRACE_FUNC(PAN_TRACE_LIB_KMOD);
+
+   return dev->ops->vm_create(dev, flags, va_start, va_range);
+}
+
+static inline void
+pan_kmod_vm_destroy(struct pan_kmod_vm *vm)
+{
+   PAN_TRACE_FUNC(PAN_TRACE_LIB_KMOD);
+
+   vm->dev->ops->vm_destroy(vm);
+}
+
+static inline int
+pan_kmod_vm_bind(struct pan_kmod_vm *vm, enum pan_kmod_vm_op_mode mode,
+                 struct pan_kmod_vm_op *ops, uint32_t op_count)
+{
+   PAN_TRACE_FUNC(PAN_TRACE_LIB_KMOD);
+
+   return vm->dev->ops->vm_bind(vm, mode, ops, op_count);
+}
+
+static inline enum pan_kmod_vm_state
+pan_kmod_vm_query_state(struct pan_kmod_vm *vm)
+{
+   if (vm->dev->ops->vm_query_state)
+      return vm->dev->ops->vm_query_state(vm);
+
+   return PAN_KMOD_VM_USABLE;
+}
+
+static inline uint32_t
+pan_kmod_vm_handle(struct pan_kmod_vm *vm)
+{
+   return vm->handle;
+}
+
+static inline uint64_t
+pan_kmod_query_timestamp(const struct pan_kmod_dev *dev)
+{
+   return dev->ops->query_timestamp(dev);
+}
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif

src/panfrost/lib/kmod/pan_kmod_backend.h

@@ -0,0 +1,142 @@
+/*
+ * Copyright © 2023 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#pragma once
+
+#include "util/log.h"
+
+#include "pan_kmod.h"
+
+static inline void
+pan_kmod_dev_init(struct pan_kmod_dev *dev, int fd, uint32_t flags,
+                  const struct pan_kmod_driver *drv_info,
+                  const struct pan_kmod_ops *ops,
+                  const struct pan_kmod_allocator *allocator)
+{
+   simple_mtx_init(&dev->handle_to_bo.lock, mtx_plain);
+   util_sparse_array_init(&dev->handle_to_bo.array,
+                          sizeof(struct pan_kmod_bo *), 512);
+   simple_mtx_init(&dev->pending_bo_syncs.lock, mtx_plain);
+   util_dynarray_init(&dev->pending_bo_syncs.array, NULL);
+   dev->driver = *drv_info;
+   dev->fd = fd;
+   dev->flags = flags;
+   dev->ops = ops;
+   dev->allocator = allocator;
+}
+
+static inline void
+pan_kmod_dev_cleanup(struct pan_kmod_dev *dev)
+{
+   if (dev->flags & PAN_KMOD_DEV_FLAG_OWNS_FD)
+      close(dev->fd);
+
+   util_dynarray_fini(&dev->pending_bo_syncs.array);
+   util_sparse_array_finish(&dev->handle_to_bo.array);
+   simple_mtx_destroy(&dev->handle_to_bo.lock);
+   simple_mtx_destroy(&dev->pending_bo_syncs.lock);
+}
+
+static inline void *
+pan_kmod_alloc(const struct pan_kmod_allocator *allocator, size_t size)
+{
+   return allocator->zalloc(allocator, size, false);
+}
+
+static inline void *
+pan_kmod_alloc_transient(const struct pan_kmod_allocator *allocator,
+                         size_t size)
+{
+   return allocator->zalloc(allocator, size, true);
+}
+
+static inline void
+pan_kmod_free(const struct pan_kmod_allocator *allocator, void *data)
+{
+   return allocator->free(allocator, data);
+}
+
+static inline void *
+pan_kmod_dev_alloc(struct pan_kmod_dev *dev, size_t size)
+{
+   return pan_kmod_alloc(dev->allocator, size);
+}
+
+static inline void *
+pan_kmod_dev_alloc_transient(struct pan_kmod_dev *dev, size_t size)
+{
+   return pan_kmod_alloc_transient(dev->allocator, size);
+}
+
+static inline void
+pan_kmod_dev_free(const struct pan_kmod_dev *dev, void *data)
+{
+   return pan_kmod_free(dev->allocator, data);
+}
+
+static inline void
+pan_kmod_bo_init(struct pan_kmod_bo *bo, struct pan_kmod_dev *dev,
+                 struct pan_kmod_vm *exclusive_vm, uint64_t size, uint32_t flags,
+                 uint32_t handle)
+{
+   /* Set by default when the device is IO coherent. We might want to
+    * make it optional at some point and pass a NON_COHERENT flag to
+    * the KMD to force non-coherent mappings on IO coherent setup.
+    */
+   if (dev->props.is_io_coherent)
+      flags |= PAN_KMOD_BO_FLAG_IO_COHERENT;
+
+   bo->dev = dev;
+   bo->exclusive_vm = exclusive_vm;
+   bo->size = size;
+   bo->flags = flags;
+   bo->handle = handle;
+   p_atomic_set(&bo->refcnt, 1);
+}
+
+void pan_kmod_flush_bo_map_syncs_locked(struct pan_kmod_dev *dev);
+
+static inline void
+pan_kmod_bo_cleanup(struct pan_kmod_bo *bo)
+{
+   if (bo->has_pending_deferred_syncs) {
+      struct pan_kmod_dev *dev = bo->dev;
+
+      simple_mtx_lock(&dev->pending_bo_syncs.lock);
+      pan_kmod_flush_bo_map_syncs_locked(dev);
+      simple_mtx_unlock(&dev->pending_bo_syncs.lock);
+   }
+}
+
+static inline void
+pan_kmod_vm_init(struct pan_kmod_vm *vm, struct pan_kmod_dev *dev,
+                 uint32_t handle, uint32_t flags, uint64_t pgsize_bitmap)
+{
+   vm->dev = dev;
+   vm->handle = handle;
+   vm->flags = flags;
+   vm->pgsize_bitmap = pgsize_bitmap;
+}
+
+static inline int
+pan_kmod_vm_op_check(struct pan_kmod_vm *vm, enum pan_kmod_vm_op_mode mode,
+                     struct pan_kmod_vm_op *op)
+{
+   /* We should only have sync operations on an async VM bind request. */
+   if (mode != PAN_KMOD_VM_OP_MODE_ASYNC && op->syncs.count) {
+      mesa_loge("only PAN_KMOD_VM_OP_MODE_ASYNC can be passed sync operations");
+      return -1;
+   }
+
+   /* Make sure the PAN_KMOD_VM_FLAG_AUTO_VA and VA passed to the op match. */
+   if (op->type == PAN_KMOD_VM_OP_TYPE_MAP &&
+       !!(vm->flags & PAN_KMOD_VM_FLAG_AUTO_VA) !=
+          (op->va.start == PAN_KMOD_VM_MAP_AUTO_VA)) {
+      mesa_loge("op->va.start and vm->flags don't match");
+      return -1;
+   }
+
+   return 0;
+}

src/panfrost/lib/kmod/panfrost_kmod.c

@@ -0,0 +1,627 @@
+/*
+ * Copyright © 2023 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include <errno.h>
+#include <fcntl.h>
+#include <string.h>
+#include <xf86drm.h>
+
+#include "util/hash_table.h"
+#include "util/macros.h"
+#include "util/simple_mtx.h"
+#include "util/stack_array.h"
+
+#include "drm-uapi/panfrost_drm.h"
+
+#include "pan_kmod_backend.h"
+
+#include "pan_props.h"
+
+/* Maximum kmod BO label length, including NUL-terminator */
+#define PANFROST_BO_LABEL_MAXLEN 4096
+
+const struct pan_kmod_ops panfrost_kmod_ops;
+
+struct panfrost_kmod_vm {
+   struct pan_kmod_vm base;
+};
+
+struct panfrost_kmod_dev {
+   struct pan_kmod_dev base;
+   struct panfrost_kmod_vm *vm;
+};
+
+struct panfrost_kmod_bo {
+   struct pan_kmod_bo base;
+
+   /* This is actually the VA assigned to the BO at creation/import time.
+    * We don't control it, it's automatically assigned by the kernel driver.
+    */
+   uint64_t offset;
+};
+
+/* Abstraction over the raw drm_panfrost_get_param ioctl for fetching
+ * information about devices.
+ */
+static __u64
+panfrost_query_raw(int fd, enum drm_panfrost_param param, bool required,
+                   unsigned default_value)
+{
+   struct drm_panfrost_get_param get_param = {};
+   ASSERTED int ret;
+
+   get_param.param = param;
+   ret = pan_kmod_ioctl(fd, DRM_IOCTL_PANFROST_GET_PARAM, &get_param);
+
+   if (ret) {
+      assert(!required);
+      return default_value;
+   }
+
+   return get_param.value;
+}
+
+static void
+panfrost_dev_query_thread_props(struct panfrost_kmod_dev *panfrost_dev)
+{
+   struct pan_kmod_dev_props *props = &panfrost_dev->base.props;
+   const struct pan_kmod_dev *dev = &panfrost_dev->base;
+   int fd = dev->fd;
+
+   props->max_threads_per_core =
+      panfrost_query_raw(fd, DRM_PANFROST_PARAM_MAX_THREADS, true, 0);
+   if (!props->max_threads_per_core) {
+      switch (pan_arch(props->gpu_id)) {
+      case 4:
+      case 5:
+         props->max_threads_per_core = 256;
+         break;
+
+      case 6:
+         /* Bifrost, first generation */
+         props->max_threads_per_core = 384;
+         break;
+
+      case 7:
+         /* Bifrost, second generation (G31 is 512 but it doesn't matter) */
+         props->max_threads_per_core = 768;
+         break;
+
+      case 9:
+         /* Valhall, first generation. */
+         props->max_threads_per_core = 512;
+         break;
+
+      default:
+         assert(!"Unsupported arch");
+      }
+   }
+
+   props->max_threads_per_wg = panfrost_query_raw(
+      fd, DRM_PANFROST_PARAM_THREAD_MAX_WORKGROUP_SZ, true, 0);
+   if (!props->max_threads_per_wg)
+      props->max_threads_per_wg = props->max_threads_per_core;
+
+   uint32_t thread_features =
+      panfrost_query_raw(fd, DRM_PANFROST_PARAM_THREAD_FEATURES, true, 0);
+   props->max_tasks_per_core = MAX2(thread_features >> 24, 1);
+   props->num_registers_per_core = thread_features & 0xffff;
+   if (!props->num_registers_per_core) {
+      switch (pan_arch(props->gpu_id)) {
+      case 4:
+      case 5:
+         /* Assume we can always schedule max_threads_per_core when using 4
+          * registers per-shader or less.
+          */
+         props->num_registers_per_core = props->max_threads_per_core * 4;
+         break;
+
+      case 6:
+         /* Assume we can always schedule max_threads_per_core for shader
+          * using the full per-shader register file (64 regs).
+          */
+         props->num_registers_per_core = props->max_threads_per_core * 64;
+         break;
+
+      case 7:
+      case 9:
+         /* Assume we can always schedule max_threads_per_core for shaders
+          * using half the per-shader register file (32 regs).
+          */
+         props->num_registers_per_core = props->max_threads_per_core * 32;
+         break;
+
+      default:
+         assert(!"Unsupported arch");
+      }
+   }
+
+   props->max_tls_instance_per_core =
+      panfrost_query_raw(fd, DRM_PANFROST_PARAM_THREAD_TLS_ALLOC, true, 0);
+   if (!props->max_tls_instance_per_core)
+      props->max_tls_instance_per_core = props->max_threads_per_core;
+}
+
+static void
+panfrost_dev_query_props(struct panfrost_kmod_dev *panfrost_dev)
+{
+   struct pan_kmod_dev_props *props = &panfrost_dev->base.props;
+   const struct pan_kmod_dev *dev = &panfrost_dev->base;
+   int fd = dev->fd;
+
+   memset(props, 0, sizeof(*props));
+   props->gpu_id =
+      (panfrost_query_raw(fd, DRM_PANFROST_PARAM_GPU_PROD_ID, true, 0) << 16) |
+      panfrost_query_raw(fd, DRM_PANFROST_PARAM_GPU_REVISION, true, 0);
+   props->shader_present =
+      panfrost_query_raw(fd, DRM_PANFROST_PARAM_SHADER_PRESENT, true, 0);
+   props->tiler_features =
+      panfrost_query_raw(fd, DRM_PANFROST_PARAM_TILER_FEATURES, true, 0);
+   props->mem_features =
+      panfrost_query_raw(fd, DRM_PANFROST_PARAM_MEM_FEATURES, true, 0);
+   props->mmu_features =
+      panfrost_query_raw(fd, DRM_PANFROST_PARAM_MMU_FEATURES, true, 0);
+
+   for (unsigned i = 0; i < ARRAY_SIZE(props->texture_features); i++) {
+      props->texture_features[i] = panfrost_query_raw(
+         fd, DRM_PANFROST_PARAM_TEXTURE_FEATURES0 + i, true, 0);
+   }
+
+   props->afbc_features =
+      panfrost_query_raw(fd, DRM_PANFROST_PARAM_AFBC_FEATURES, true, 0);
+
+   panfrost_dev_query_thread_props(panfrost_dev);
+
+   if (pan_kmod_driver_version_at_least(&dev->driver, 1, 3)) {
+      props->gpu_can_query_timestamp = true;
+      props->timestamp_frequency = panfrost_query_raw(
+         fd, DRM_PANFROST_PARAM_SYSTEM_TIMESTAMP_FREQUENCY, true, 0);
+   }
+
+   /* Device coherent timestamps are always enabled on panfrost */
+   props->timestamp_device_coherent = true;
+
+   /* Support for priorities was added in panfrost 1.5, assumes default
+    * priority as medium if the param doesn't exist. */
+   uint64_t prios =
+      panfrost_query_raw(fd, DRM_PANFROST_PARAM_ALLOWED_JM_CTX_PRIORITIES,
+                         false, BITFIELD_BIT(PANFROST_JM_CTX_PRIORITY_MEDIUM));
+
+   if (prios & BITFIELD_BIT(PANFROST_JM_CTX_PRIORITY_LOW))
+      props->allowed_group_priorities_mask |= PAN_KMOD_GROUP_ALLOW_PRIORITY_LOW;
+   if (prios & BITFIELD_BIT(PANFROST_JM_CTX_PRIORITY_MEDIUM))
+      props->allowed_group_priorities_mask |=
+         PAN_KMOD_GROUP_ALLOW_PRIORITY_MEDIUM;
+   if (prios & BITFIELD_BIT(PANFROST_JM_CTX_PRIORITY_HIGH))
+      props->allowed_group_priorities_mask |=
+         PAN_KMOD_GROUP_ALLOW_PRIORITY_HIGH;
+
+   props->supported_bo_flags = PAN_KMOD_BO_FLAG_EXECUTABLE |
+                               PAN_KMOD_BO_FLAG_ALLOC_ON_FAULT |
+                               PAN_KMOD_BO_FLAG_NO_MMAP;
+
+   if (pan_kmod_driver_version_at_least(&dev->driver, 1, 6)) {
+      uint32_t selected_coherency =
+         panfrost_query_raw(fd, DRM_PANFROST_PARAM_SELECTED_COHERENCY, true,
+                            DRM_PANFROST_GPU_COHERENCY_NONE);
+
+      props->supported_bo_flags |= PAN_KMOD_BO_FLAG_WB_MMAP;
+      props->is_io_coherent =
+         selected_coherency != DRM_PANFROST_GPU_COHERENCY_NONE;
+   }
+}
+
+static struct pan_kmod_dev *
+panfrost_kmod_dev_create(int fd, uint32_t flags,
+                         const struct pan_kmod_driver *drv_info,
+                         const struct pan_kmod_allocator *allocator)
+{
+   if (!pan_kmod_driver_version_at_least(drv_info, 1, 1)) {
+      mesa_loge("kernel driver is too old (requires at least 1.1, found %d.%d)",
+                drv_info->version.major, drv_info->version.minor);
+      return NULL;
+   }
+
+   struct panfrost_kmod_dev *panfrost_dev =
+      pan_kmod_alloc(allocator, sizeof(*panfrost_dev));
+   if (!panfrost_dev) {
+      mesa_loge("failed to allocate a panfrost_kmod_dev object");
+      return NULL;
+   }
+
+   pan_kmod_dev_init(&panfrost_dev->base, fd, flags, drv_info,
+                     &panfrost_kmod_ops, allocator);
+   panfrost_dev_query_props(panfrost_dev);
+
+   return &panfrost_dev->base;
+}
+
+static void
+panfrost_kmod_dev_destroy(struct pan_kmod_dev *dev)
+{
+   struct panfrost_kmod_dev *panfrost_dev =
+      container_of(dev, struct panfrost_kmod_dev, base);
+
+   pan_kmod_dev_cleanup(dev);
+   pan_kmod_free(dev->allocator, panfrost_dev);
+}
+
+static uint32_t
+to_panfrost_bo_flags(struct pan_kmod_dev *dev, uint32_t flags)
+{
+   uint32_t panfrost_flags = 0;
+
+   if (pan_kmod_driver_version_at_least(&dev->driver, 1, 1)) {
+      /* The alloc-on-fault feature is only used for the tiler HEAP object,
+       * hence the name of the flag on panfrost.
+       */
+      if (flags & PAN_KMOD_BO_FLAG_ALLOC_ON_FAULT)
+         panfrost_flags |= PANFROST_BO_HEAP;
+
+      if (!(flags & PAN_KMOD_BO_FLAG_EXECUTABLE))
+         panfrost_flags |= PANFROST_BO_NOEXEC;
+   }
+
+   if (flags & PAN_KMOD_BO_FLAG_WB_MMAP) {
+      assert(!(flags & PAN_KMOD_BO_FLAG_NO_MMAP));
+      panfrost_flags |= PANFROST_BO_WB_MMAP;
+   }
+
+   return panfrost_flags;
+}
+
+static struct pan_kmod_bo *
+panfrost_kmod_bo_alloc(struct pan_kmod_dev *dev,
+                       struct pan_kmod_vm *exclusive_vm, uint64_t size,
+                       uint32_t flags)
+{
+   /* The ioctl uses u32 for size. */
+   if ((uint64_t)(uint32_t)size != size)
+      return NULL;
+
+   /* We can't map GPU uncached. */
+   if (flags & PAN_KMOD_BO_FLAG_GPU_UNCACHED)
+      return NULL;
+
+   struct panfrost_kmod_bo *bo = pan_kmod_dev_alloc(dev, sizeof(*bo));
+   if (!bo)
+      return NULL;
+
+   struct drm_panfrost_create_bo req = {
+      .size = size,
+      .flags = to_panfrost_bo_flags(dev, flags),
+   };
+
+   int ret = pan_kmod_ioctl(dev->fd, DRM_IOCTL_PANFROST_CREATE_BO, &req);
+   if (ret) {
+      mesa_loge("DRM_IOCTL_PANFROST_CREATE_BO failed (err=%d)", errno);
+      goto err_free_bo;
+   }
+
+   pan_kmod_bo_init(&bo->base, dev, exclusive_vm, req.size, flags, req.handle);
+   bo->offset = req.offset;
+   return &bo->base;
+
+err_free_bo:
+   pan_kmod_dev_free(dev, bo);
+   return NULL;
+}
+
+static void
+panfrost_kmod_bo_free(struct pan_kmod_bo *bo)
+{
+   pan_kmod_bo_cleanup(bo);
+   drmCloseBufferHandle(bo->dev->fd, bo->handle);
+   pan_kmod_dev_free(bo->dev, bo);
+}
+
+static struct pan_kmod_bo *
+panfrost_kmod_bo_import(struct pan_kmod_dev *dev, uint32_t handle,
+                        uint64_t size)
+{
+   struct panfrost_kmod_bo *panfrost_bo =
+      pan_kmod_dev_alloc(dev, sizeof(*panfrost_bo));
+   if (!panfrost_bo) {
+      mesa_loge("failed to allocate a panfrost_kmod_bo object");
+      return NULL;
+   }
+
+   struct drm_panfrost_get_bo_offset get_bo_offset = {.handle = handle, 0};
+   int ret =
+      pan_kmod_ioctl(dev->fd, DRM_IOCTL_PANFROST_GET_BO_OFFSET,
+                     &get_bo_offset);
+   if (ret) {
+      mesa_loge("DRM_IOCTL_PANFROST_GET_BO_OFFSET failed (err=%d)", errno);
+      goto err_free_bo;
+   }
+
+   panfrost_bo->offset = get_bo_offset.offset;
+
+   uint32_t flags = PAN_KMOD_BO_FLAG_IMPORTED;
+   if (pan_kmod_driver_version_at_least(&dev->driver, 1, 6)) {
+      struct drm_panfrost_query_bo_info args = {
+         .handle = handle,
+      };
+
+      ret = drmIoctl(dev->fd, DRM_IOCTL_PANFROST_QUERY_BO_INFO, &args);
+      if (ret) {
+         mesa_loge("PANFROST_BO_QUERY_INFO failed (err=%d)", errno);
+         goto err_free_bo;
+      }
+
+      /* FIXME: If the BO comes from a different subsystem
+       * (args.extra_flags & DRM_PANTHOR_BO_IS_IMPORTED), we should normally
+       * add extra DMA_BUF_IOCTL_SYNC calls around CPU accesses to ensure the
+       * CPU mapping consistency, but this is something we never worried about
+       * (we've always assumed exporters were exposing uncached mappings with
+       * NOP {begin,end}_cpu_access() implementations), and it worked fine until
+       * now.
+       * The long term plan is to hook up DMA_BUF_IOCTL_SYNC, but this requires
+       * more work.
+       */
+   }
+
+   pan_kmod_bo_init(&panfrost_bo->base, dev, NULL, size, flags, handle);
+   return &panfrost_bo->base;
+
+err_free_bo:
+   pan_kmod_dev_free(dev, panfrost_bo);
+   return NULL;
+}
+
+static off_t
+panfrost_kmod_bo_get_mmap_offset(struct pan_kmod_bo *bo)
+{
+   struct drm_panfrost_mmap_bo mmap_bo = {.handle = bo->handle};
+   int ret = pan_kmod_ioctl(bo->dev->fd, DRM_IOCTL_PANFROST_MMAP_BO,
+                            &mmap_bo);
+   if (ret) {
+      fprintf(stderr, "DRM_IOCTL_PANFROST_MMAP_BO failed: %m\n");
+      assert(0);
+   }
+
+   return mmap_bo.offset;
+}
+
+static bool
+panfrost_kmod_bo_wait(struct pan_kmod_bo *bo, int64_t timeout_ns,
+                      bool for_read_only_access)
+{
+   struct drm_panfrost_wait_bo req = {
+      .handle = bo->handle,
+      .timeout_ns = timeout_ns,
+   };
+
+   /* The ioctl returns >= 0 value when the BO we are waiting for is ready
+    * -1 otherwise.
+    */
+   if (pan_kmod_ioctl(bo->dev->fd, DRM_IOCTL_PANFROST_WAIT_BO, &req) != -1)
+      return true;
+
+   assert(errno == ETIMEDOUT || errno == EBUSY);
+   return false;
+}
+
+static int
+panfrost_kmod_flush_bo_map_syncs(struct pan_kmod_dev *dev)
+{
+   STACK_ARRAY(struct drm_panfrost_bo_sync_op, panfrost_ops,
+               util_dynarray_num_elements(&dev->pending_bo_syncs.array,
+                                          struct pan_kmod_deferred_bo_sync));
+
+   uint32_t panfrost_count = 0;
+   util_dynarray_foreach(&dev->pending_bo_syncs.array,
+                         struct pan_kmod_deferred_bo_sync, sync) {
+      panfrost_ops[panfrost_count++] = (struct drm_panfrost_bo_sync_op){
+         .handle = sync->bo->handle,
+         .type = sync->type == PAN_KMOD_BO_SYNC_CPU_CACHE_FLUSH
+                    ? PANFROST_BO_SYNC_CPU_CACHE_FLUSH
+                    : PANFROST_BO_SYNC_CPU_CACHE_FLUSH_AND_INVALIDATE,
+         .offset = sync->start,
+         .size = sync->size,
+      };
+   }
+
+   struct drm_panfrost_sync_bo req = {
+      .ops = (uintptr_t)panfrost_ops,
+      .op_count = panfrost_count,
+   };
+   int ret = pan_kmod_ioctl(dev->fd, DRM_IOCTL_PANFROST_SYNC_BO, &req);
+   if (ret)
+      mesa_loge("DRM_IOCTL_PANFROST_BO_SYNC failed (err=%d)", errno);
+
+   STACK_ARRAY_FINISH(panfrost_ops);
+
+   return ret;
+}
+
+static void
+panfrost_kmod_bo_make_evictable(struct pan_kmod_bo *bo)
+{
+   struct drm_panfrost_madvise req = {
+      .handle = bo->handle,
+      .madv = PANFROST_MADV_DONTNEED,
+   };
+
+   pan_kmod_ioctl(bo->dev->fd, DRM_IOCTL_PANFROST_MADVISE, &req);
+}
+
+static bool
+panfrost_kmod_bo_make_unevictable(struct pan_kmod_bo *bo)
+{
+   struct drm_panfrost_madvise req = {
+      .handle = bo->handle,
+      .madv = PANFROST_MADV_WILLNEED,
+   };
+
+   if (pan_kmod_ioctl(bo->dev->fd, DRM_IOCTL_PANFROST_MADVISE, &req) == 0 &&
+       req.retained == 0)
+      return false;
+
+   return true;
+}
+
+/* The VA range is restricted by the kernel driver. Lower 32MB are reserved, and
+ * the address space is limited to 32-bit.
+ */
+#define PANFROST_KMOD_VA_START 0x2000000ull
+#define PANFROST_KMOD_VA_END   (1ull << 32)
+
+static struct pan_kmod_va_range
+panfrost_kmod_dev_query_user_va_range(const struct pan_kmod_dev *dev)
+{
+   return (struct pan_kmod_va_range){
+      .start = PANFROST_KMOD_VA_START,
+      .size = PANFROST_KMOD_VA_END - PANFROST_KMOD_VA_START,
+   };
+}
+
+static struct pan_kmod_vm *
+panfrost_kmod_vm_create(struct pan_kmod_dev *dev, uint32_t flags,
+                        uint64_t va_start, uint64_t va_range)
+{
+   struct panfrost_kmod_dev *panfrost_dev =
+      container_of(dev, struct panfrost_kmod_dev, base);
+
+   /* Only one VM per device. */
+   if (panfrost_dev->vm) {
+      mesa_loge("panfrost_kmod only supports one VM per device");
+      return NULL;
+   }
+
+   /* Panfrost kernel driver doesn't support userspace VA management. */
+   if (!(flags & PAN_KMOD_VM_FLAG_AUTO_VA)) {
+      mesa_loge("panfrost_kmod only supports PAN_KMOD_VM_FLAG_AUTO_VA");
+      assert(0);
+      return NULL;
+   }
+
+   struct panfrost_kmod_vm *vm = pan_kmod_dev_alloc(dev, sizeof(*vm));
+   if (!vm) {
+      mesa_loge("failed to allocate a panfrost_kmod_vm object");
+      return NULL;
+   }
+
+   pan_kmod_vm_init(&vm->base, dev, 0, flags, PAN_PGSIZE_4K);
+   panfrost_dev->vm = vm;
+   return &vm->base;
+}
+
+static void
+panfrost_kmod_vm_destroy(struct pan_kmod_vm *vm)
+{
+   struct panfrost_kmod_dev *panfrost_dev =
+      container_of(vm->dev, struct panfrost_kmod_dev, base);
+
+   panfrost_dev->vm = NULL;
+   pan_kmod_dev_free(vm->dev, vm);
+}
+
+static int
+panfrost_kmod_vm_bind(struct pan_kmod_vm *vm, enum pan_kmod_vm_op_mode mode,
+                      struct pan_kmod_vm_op *ops, uint32_t op_count)
+{
+   UNUSED struct panfrost_kmod_vm *panfrost_vm =
+      container_of(vm, struct panfrost_kmod_vm, base);
+
+   /* We only support IMMEDIATE and WAIT_IDLE mode. Actually we always do
+    * WAIT_IDLE in practice, but it shouldn't matter.
+    */
+   if (mode != PAN_KMOD_VM_OP_MODE_IMMEDIATE &&
+       mode != PAN_KMOD_VM_OP_MODE_DEFER_TO_NEXT_IDLE_POINT) {
+      mesa_loge("panfrost_kmod doesn't support mode=%d", mode);
+      assert(0);
+      return -1;
+   }
+
+   for (uint32_t i = 0; i < op_count; i++) {
+
+      if (ops[i].type == PAN_KMOD_VM_OP_TYPE_MAP) {
+         struct panfrost_kmod_bo *panfrost_bo =
+            container_of(ops[i].map.bo, struct panfrost_kmod_bo, base);
+
+         /* Panfrost kernel driver doesn't support userspace VA management. */
+         if (ops[i].va.start != PAN_KMOD_VM_MAP_AUTO_VA) {
+            mesa_loge("panfrost_kmod can only do auto-VA allocation");
+            assert(0);
+            return -1;
+         }
+
+         /* Panfrost kernel driver only support full BO mapping. */
+         if (ops[i].map.bo_offset != 0 ||
+             ops[i].va.size != ops[i].map.bo->size) {
+            mesa_loge("panfrost_kmod doesn't support partial BO mapping");
+            assert(0);
+            return -1;
+         }
+
+         ops[i].va.start = panfrost_bo->offset;
+      } else if (ops[i].type == PAN_KMOD_VM_OP_TYPE_UNMAP) {
+         /* Do nothing, unmapping is done at BO destruction time. */
+      } else {
+         /* We reject PAN_KMOD_VM_OP_TYPE_SYNC_ONLY as this implies
+          * supporting PAN_KMOD_VM_OP_MODE_ASYNC, which we don't support.
+          */
+         mesa_loge("panfrost_kmod doesn't support op=%d", ops[i].type);
+         assert(0);
+         return -1;
+      }
+   }
+
+   return 0;
+}
+
+static uint64_t
+panfrost_kmod_query_timestamp(const struct pan_kmod_dev *dev)
+{
+   return panfrost_query_raw(dev->fd, DRM_PANFROST_PARAM_SYSTEM_TIMESTAMP,
+                             false, 0);
+}
+
+static void
+panfrost_kmod_bo_label(struct pan_kmod_dev *dev, struct pan_kmod_bo *bo, const char *label)
+{
+   char truncated_label[PANFROST_BO_LABEL_MAXLEN];
+
+   if (!pan_kmod_driver_version_at_least(&dev->driver, 1, 4))
+      return;
+
+   if (strnlen(label, PANFROST_BO_LABEL_MAXLEN) == PANFROST_BO_LABEL_MAXLEN) {
+      strncpy(truncated_label, label, PANFROST_BO_LABEL_MAXLEN - 1);
+      truncated_label[PANFROST_BO_LABEL_MAXLEN - 1] = '\0';
+      label = truncated_label;
+   }
+
+   struct drm_panfrost_set_label_bo set_label =
+      (struct drm_panfrost_set_label_bo) {
+      .handle = bo->handle,
+      .label = (uint64_t)(uintptr_t)label,
+   };
+
+   int ret =
+      pan_kmod_ioctl(dev->fd, DRM_IOCTL_PANFROST_SET_LABEL_BO,
+                     &set_label);
+   if (ret)
+      mesa_loge("DRM_IOCTL_PANFROST_SET_LABEL_BO failed (err=%d)", errno);
+}
+
+const struct pan_kmod_ops panfrost_kmod_ops = {
+   .dev_create = panfrost_kmod_dev_create,
+   .dev_destroy = panfrost_kmod_dev_destroy,
+   .dev_query_user_va_range = panfrost_kmod_dev_query_user_va_range,
+   .bo_alloc = panfrost_kmod_bo_alloc,
+   .bo_free = panfrost_kmod_bo_free,
+   .bo_import = panfrost_kmod_bo_import,
+   .bo_get_mmap_offset = panfrost_kmod_bo_get_mmap_offset,
+   .bo_wait = panfrost_kmod_bo_wait,
+   .flush_bo_map_syncs = panfrost_kmod_flush_bo_map_syncs,
+   .bo_make_evictable = panfrost_kmod_bo_make_evictable,
+   .bo_make_unevictable = panfrost_kmod_bo_make_unevictable,
+   .vm_create = panfrost_kmod_vm_create,
+   .vm_destroy = panfrost_kmod_vm_destroy,
+   .vm_bind = panfrost_kmod_vm_bind,
+   .query_timestamp = panfrost_kmod_query_timestamp,
+   .bo_set_label = panfrost_kmod_bo_label,
+};

src/panfrost/lib/kmod/panthor_kmod.h

@@ -0,0 +1,37 @@
+/*
+ * Copyright © 2023 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#pragma once
+
+#include <stdint.h>
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+struct drm_panthor_csif_info;
+
+struct pan_kmod_bo;
+struct pan_kmod_dev;
+struct pan_kmod_vm;
+
+int panthor_kmod_bo_attach_sync_point(struct pan_kmod_bo *bo,
+                                      uint32_t sync_handle,
+                                      uint64_t sync_point, bool written);
+int panthor_kmod_bo_get_sync_point(struct pan_kmod_bo *bo,
+                                   uint32_t *sync_handle, uint64_t *sync_point,
+                                   bool read_only);
+uint32_t panthor_kmod_vm_sync_handle(struct pan_kmod_vm *vm);
+uint64_t panthor_kmod_vm_sync_lock(struct pan_kmod_vm *vm);
+void panthor_kmod_vm_sync_unlock(struct pan_kmod_vm *vm,
+                                 uint64_t new_sync_point);
+uint32_t panthor_kmod_get_flush_id(const struct pan_kmod_dev *dev);
+
+const struct drm_panthor_csif_info *
+panthor_kmod_get_csif_props(const struct pan_kmod_dev *dev);
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif

src/panfrost/lib/meson.build

@@ -0,0 +1,133 @@
+# Copyright © 2018 Rob Clark
+# Copyright © 2019 Collabora
+# SPDX-License-Identifier: MIT
+
+subdir('kmod')
+
+pixel_format_versions = ['5', '6', '7', '9', '10', '12', '13', '14']
+libpanfrost_pixel_format = []
+
+deps_for_libpanfrost = [dep_libdrm, idep_pan_packers, idep_mesautil, libpanfrost_model_dep]
+
+foreach ver : pixel_format_versions
+  libpanfrost_pixel_format += static_library(
+    'pan-format-v' + ver,
+    ['pan_format.c', pan_packers],
+    include_directories : [inc_src, inc_include, inc_panfrost],
+    c_args : ['-DPAN_ARCH=' + ver],
+    gnu_symbol_visibility : 'hidden',
+    dependencies : [deps_for_libpanfrost, dep_valgrind],
+)
+endforeach
+
+libpanfrost_per_arch = []
+
+foreach ver : ['4', '5', '6', '7', '9', '10', '12', '13', '14']
+  libpanfrost_per_arch += static_library(
+    'pan-arch-v' + ver,
+    [
+      'pan_blend.c',
+      'pan_buffer.c',
+      'pan_desc.c',
+      'pan_fb.c',
+      'pan_fb_nir.c',
+      'pan_mod.c',
+      'pan_texture.c',
+    ],
+    include_directories : [inc_include, inc_src, inc_panfrost],
+    c_args : ['-DPAN_ARCH=' + ver],
+    gnu_symbol_visibility : 'hidden',
+    dependencies : [deps_for_libpanfrost, idep_nir],
+  )
+endforeach
+
+libpanfrost_lib_files = files(
+  'pan_encoder.h',
+
+  'pan_blend.c',
+  'pan_clear.c',
+  'pan_earlyzs.c',
+  'pan_samples.c',
+  'pan_tiler.c',
+  'pan_layout.c',
+  'pan_scratch.c',
+  'pan_trace.c',
+  'pan_props.c',
+  'pan_util.c',
+  'pan_afbc.c',
+)
+
+libpanfrost_lib = static_library(
+  'panfrost_lib',
+  [libpanfrost_lib_files, pan_packers],
+  include_directories : [inc_include, inc_src, inc_panfrost],
+  c_args : [no_override_init_args, soong_compat_c_args],
+  gnu_symbol_visibility : 'hidden',
+  dependencies: [dep_libdrm, idep_nir, idep_mesautil, libpanfrost_model_dep],
+  build_by_default : false,
+  link_with: [libpanfrost_pixel_format, libpanfrost_per_arch, libpankmod_lib],
+)
+
+libpanfrost_dep = declare_dependency(
+  link_with: [libpanfrost_lib, libpanfrost_decode, libpanfrost_compiler, libpanfrost_pixel_format, libpanfrost_per_arch],
+  include_directories: [inc_include, inc_src, inc_panfrost],
+  dependencies: [deps_for_libpanfrost, libpankmod_dep, idep_nir],
+)
+
+if with_tests
+  test(
+    'panfrost_attribute',
+    executable(
+      'panfrost_attribute_test',
+      files('tests/test-attribute.c'),
+      c_args : [c_msvc_compat_args, no_override_init_args],
+      gnu_symbol_visibility : 'hidden',
+      include_directories : [inc_include, inc_src],
+      dependencies: [libpanfrost_dep],
+    ),
+    suite : ['panfrost'],
+  )
+
+  test(
+    'panfrost_blend',
+    executable(
+      'panfrost_blend_test',
+      files('tests/test-blend.c'),
+      c_args : [c_msvc_compat_args, no_override_init_args],
+      gnu_symbol_visibility : 'hidden',
+      include_directories : [inc_include, inc_src],
+      dependencies: [libpanfrost_dep],
+    ),
+    suite : ['panfrost'],
+  )
+
+  test(
+    'panfrost_clear',
+    executable(
+      'panfrost_clear_test',
+      files('tests/test-clear.c'),
+      c_args : [c_msvc_compat_args, no_override_init_args],
+      gnu_symbol_visibility : 'hidden',
+      include_directories : [inc_include, inc_src],
+      dependencies: [libpanfrost_dep],
+    ),
+    suite : ['panfrost'],
+  )
+
+  test(
+    'panfrost_tests',
+    executable(
+      'panfrost_tests',
+      files(
+        'tests/test-earlyzs.cpp',
+        'tests/test-layout.cpp',
+      ),
+      cpp_args : cpp_msvc_compat_args,
+      gnu_symbol_visibility : 'hidden',
+      include_directories : [inc_include, inc_src, inc_panfrost],
+      dependencies: [idep_gtest, libpanfrost_dep],
+    ),
+    suite : ['panfrost'],
+    protocol : 'gtest',
+  )
+endif

src/panfrost/lib/pan_afbc.c

@@ -0,0 +1,213 @@
+/*
+ * Copyright © 2023-2025 Amazon.com, Inc. or its affiliates.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "pan_afbc.h"
+#include "pan_trace.h"
+#include "util/detect_arch.h"
+#include "util/u_cpu_detect.h"
+
+#if DETECT_ARCH_AARCH64 || (DETECT_ARCH_ARM && !defined(__SOFTFP__))
+/* armhf builds default to VFP, not NEON, and refuses to compile NEON
+ * intrinsics unless you tell it "no really". */
+#if DETECT_ARCH_ARM
+#pragma GCC target ("fpu=neon")
+#endif
+#include <arm_neon.h>
+#endif
+
+#if DETECT_ARCH_AARCH64
+
+/* Arm A64 NEON intrinsics version. */
+uint32_t
+pan_afbc_payload_layout_packed(unsigned arch,
+                               const struct pan_afbc_headerblock *headers,
+                               struct pan_afbc_payload_extent *layout,
+                               uint32_t nr_blocks, enum pipe_format format,
+                               uint64_t modifier)
+{
+   PAN_TRACE_FUNC(PAN_TRACE_LIB_AFBC);
+
+   uint32_t uncompressed_size =
+      pan_afbc_payload_uncompressed_size(format, modifier);
+   uint32_t body_size = 0;
+
+   alignas(16) static const uint8_t idx0[16] =
+      { 4, 5, 6, ~0, 7, 8, 9, ~0, 10, 11, 12, ~0, 13, 14, 15, ~0 };
+   alignas(16) static const uint8_t idx1[16] =
+      { 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60 };
+   alignas(16) static const uint8_t mask[16] =
+      { 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63 };
+   alignas(16) static const uint8_t ones[16] =
+      { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 };
+
+   uint8x16_t vidx0 = vld1q_u8(idx0);
+   uint8x16_t vidx1 = vld1q_u8(idx1);
+   uint8x16_t vmask = vld1q_u8(mask);
+   uint8x16_t vones = vld1q_u8(ones);
+
+   for (uint32_t i = 0; i < nr_blocks; ++i) {
+      uint32_t payload_size = 0;
+
+      /* Skip sum if the 1st subblock is 0 (solid color encoding). */
+      if (arch < 7 || headers[i].payload.subblock_sizes[0] & 0x3f) {
+         uint8x16_t vhdr = vld1q_u8((uint8_t *)&headers[i]);
+
+         /* Dispatch 6-bit packed 16 subblock sizes into 8-bit vector. */
+         uint8x16_t v0 = vqtbl1q_u8(vhdr, vidx0);
+         uint8x16_t v1 = vreinterpretq_u8_u32(
+            vshrq_n_u32(vreinterpretq_u32_u8(v0), 6));
+         uint8x16_t v2 = vreinterpretq_u8_u32(
+            vshrq_n_u32(vreinterpretq_u32_u8(v0), 12));
+         uint8x16_t v3 = vreinterpretq_u8_u32(
+            vshrq_n_u32(vreinterpretq_u32_u8(v0), 18));
+         uint8x16x4_t vtbl = {{ v0, v1, v2, v3 }};
+         v0 = vqtbl4q_u8(vtbl, vidx1);
+         v0 = vandq_u8(v0, vmask);
+
+         /* Sum across vector. */
+         payload_size = vaddlvq_u8(v0);
+
+         /* Number of subblocks of size 1. */
+         v0 = vceqq_u8(v0, vones);
+         v0 = vandq_u8(v0, vones);
+         uint32_t nr_ones = vaddvq_u8(v0);
+
+         /* Payload size already stores subblocks of size 1. Fix-up the sum
+          * using the number of such subblocks. */
+         payload_size += nr_ones * (uncompressed_size - 1);
+
+         payload_size = ALIGN_POT(payload_size, 16);
+      }
+
+      layout[i].size = payload_size;
+      layout[i].offset = body_size;
+      body_size += payload_size;
+   }
+
+   return body_size;
+}
+
+#else
+
+/* Arm A32 NEON intrinsics and generic version. */
+uint32_t
+pan_afbc_payload_layout_packed(unsigned arch,
+                               const struct pan_afbc_headerblock *headers,
+                               struct pan_afbc_payload_extent *layout,
+                               uint32_t nr_blocks, enum pipe_format format,
+                               uint64_t modifier)
+{
+   PAN_TRACE_FUNC(PAN_TRACE_LIB_AFBC);
+
+   uint32_t uncompressed_size =
+      pan_afbc_payload_uncompressed_size(format, modifier);
+   uint32_t body_size = 0;
+
+#if DETECT_ARCH_ARM && !defined(__SOFTFP__)
+
+   if (unlikely(!util_get_cpu_caps()->has_neon))
+      goto no_neon;
+
+   /* Arm A32 NEON intrinsics version. */
+
+   alignas(16) static const uint8_t idx0[2][8] =
+      { { 4, 5, 6, ~0, 7, 8, 9, ~0 }, { 2, 3, 4, ~0, 5, 6, 7, ~0 } };
+   alignas(8) static const uint8_t idx1[8] =
+      { 0, 4, 8, 12, 16, 20, 24, 28 };
+   alignas(16) static const uint8_t mask[16] =
+      { 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63 };
+   alignas(16) static const uint8_t ones[16] =
+      { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 };
+
+   uint8x8_t vidx00 = vld1_u8(idx0[0]);
+   uint8x8_t vidx01 = vld1_u8(idx0[1]);
+   uint8x8_t vidx1 = vld1_u8(idx1);
+   uint8x16_t vmask = vld1q_u8(mask);
+   uint8x16_t vones = vld1q_u8(ones);
+
+   for (uint32_t i = 0; i < nr_blocks; ++i) {
+      uint32_t payload_size = 0;
+
+      /* Skip sum if the 1st subblock is 0 (solid color encoding). */
+      if (arch < 7 || headers[i].payload.subblock_sizes[0] & 0x3f) {
+         /* vld1_u8_x2() isn't widely available yet. */
+         uint8x8_t vhdr0 = vld1_u8(&headers[i].u8[0]);
+         uint8x8_t vhdr1 = vld1_u8(&headers[i].u8[8]);
+         uint8x8x2_t vhdr = {{ vhdr0, vhdr1 }};
+
+         /* Dispatch 6-bit packed 16 payload sizes into 8-bit vector. Note
+          * that the NEON TBL instr in A32 only supports doubleword operands
+          * while VSHR also supports quadword. Not sure how to mix doubleword
+          * and quadword intrinsics and get compilers to correctly alias D and
+          * Q registers though (128-bit register Q0 is an alias for the two
+          * consecutive 64-bit registers D0 and D1), so stick with doubleword
+          * intrinsics here. */
+         uint8x8_t v00 = vtbl2_u8(vhdr, vidx00);
+         uint8x8_t v01 = vtbl1_u8(vhdr1, vidx01);
+         uint8x8_t v10 = vreinterpret_u8_u32(
+            vshr_n_u32(vreinterpret_u32_u8(v00), 6));
+         uint8x8_t v11 = vreinterpret_u8_u32(
+            vshr_n_u32(vreinterpret_u32_u8(v01), 6));
+         uint8x8_t v20 = vreinterpret_u8_u32(
+            vshr_n_u32(vreinterpret_u32_u8(v00), 12));
+         uint8x8_t v21 = vreinterpret_u8_u32(
+            vshr_n_u32(vreinterpret_u32_u8(v01), 12));
+         uint8x8_t v30 = vreinterpret_u8_u32(
+            vshr_n_u32(vreinterpret_u32_u8(v00), 18));
+         uint8x8_t v31 = vreinterpret_u8_u32(
+            vshr_n_u32(vreinterpret_u32_u8(v01), 18));
+         uint8x8x4_t vtbl0 = {{ v00, v01, v10, v11 }};
+         uint8x8x4_t vtbl1 = {{ v20, v21, v30, v31 }};
+         v00 = vtbl4_u8(vtbl0, vidx1);
+         v01 = vtbl4_u8(vtbl1, vidx1);
+         uint8x16_t v0 = vandq_u8(vcombine_u8(v00, v01), vmask);
+
+         /* Sum across vector. */
+         uint64x2_t v1 = vpaddlq_u32(vpaddlq_u16(vpaddlq_u8(v0)));
+         payload_size = vget_lane_u64(vadd_u64(vget_low_u64(v1),
+                                               vget_high_u64(v1)), 0);
+
+         /* Number of subblocks of size 1. */
+         v0 = vceqq_u8(v0, vones);
+         v0 = vandq_u8(v0, vones);
+         v1 = vpaddlq_u32(vpaddlq_u16(vpaddlq_u8(v0)));
+         uint32_t nr_ones = vget_lane_u64(vadd_u64(vget_low_u64(v1),
+                                                   vget_high_u64(v1)), 0);
+
+         /* Payload size already stores subblocks of size 1. Fix-up the sum
+          * using the number of such subblocks. */
+         payload_size += nr_ones * (uncompressed_size - 1);
+
+         payload_size = ALIGN_POT(payload_size, 16);
+      }
+
+      layout[i].size = payload_size;
+      layout[i].offset = body_size;
+      body_size += payload_size;
+   }
+
+   return body_size;
+
+ no_neon:
+#endif
+
+   /* Generic version. */
+
+   /* XXX: It might be faster to copy the header from non-cacheable memory
+    * into a cacheline sized chunk in cacheable memory in order to avoid too
+    * many uncached transactions. Not sure though, so it needs testing. */
+
+   for (uint32_t i = 0; i < nr_blocks; i++) {
+      uint32_t payload_size =
+         pan_afbc_payload_size(arch, headers[i], uncompressed_size);
+      layout[i].size = payload_size;
+      layout[i].offset = body_size;
+      body_size += payload_size;
+   }
+
+   return body_size;
+}
+
+#endif

src/panfrost/lib/pan_afbc.h

@@ -0,0 +1,901 @@
+/*
+ * Copyright (C) 2008 VMware, Inc.
+ * Copyright (C) 2014 Broadcom
+ * Copyright (C) 2018-2019 Alyssa Rosenzweig
+ * Copyright (C) 2019-2020 Collabora, Ltd.
+ * Copyright (C) 2026 Arm Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef __PAN_AFBC_H
+#define __PAN_AFBC_H
+
+#include "pan_format.h"
+#include "pan_layout.h"
+
+#include "drm-uapi/drm_fourcc.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Arm FrameBuffer Compression (AFBC) is a lossless compression scheme natively
+ * implemented in Mali GPUs (as well as many display controllers paired with
+ * Mali GPUs, etc). Where possible, Panfrost prefers to use AFBC for both
+ * rendering and texturing. In most cases, this is a performance-win due to a
+ * dramatic reduction in memory bandwidth and cache locality compared to a
+ * linear resources.
+ *
+ * AFBC divides the framebuffer into 16x16 tiles (other sizes possible, TODO:
+ * do we need to support this?). So, the width and height each must be aligned
+ * up to 16 pixels. This is inherently good for performance; note that for a 4
+ * byte-per-pixel format like RGBA8888, that means that rows are 16*4=64 byte
+ * aligned, which is the cache-line size.
+ *
+ * For each AFBC-compressed resource, there is a single contiguous
+ * (CPU/GPU-shared) buffer. This buffer itself is divided into two parts:
+ * header and body, placed immediately after each other.
+ *
+ * The AFBC header contains 16 bytes of metadata per tile.
+ *
+ * The AFBC body is the same size as the original linear resource (padded to
+ * the nearest tile). Although the body comes immediately after the header, it
+ * must also be cache-line aligned, so there can sometimes be a bit of padding
+ * between the header and body.
+ *
+ * As an example, a 64x64 RGBA framebuffer contains 64/16 = 4 tiles horizontally
+ * and 4 tiles vertically. There are 4*4=16 tiles in total, each containing 16
+ * bytes of metadata, so there is a 16*16=256 byte header. 64x64 is already
+ * tile aligned, so the body is 64*64 * 4 bytes per pixel = 16384 bytes of
+ * body.
+ *
+ * From userspace, Panfrost needs to be able to calculate these sizes. It
+ * explicitly does not and can not know the format of the data contained within
+ * this header and body. The GPU has native support for AFBC encode/decode. For
+ * an internal FBO or a framebuffer used for scanout with an AFBC-compatible
+ * winsys/display-controller, the buffer is maintained AFBC throughout flight,
+ * and the driver never needs to know the internal data. For edge cases where
+ * the driver really does need to read/write from the AFBC resource, we
+ * generate a linear staging buffer and use the GPU to blit AFBC<--->linear.
+ */
+
+#define AFBC_HEADER_BYTES_PER_TILE 16
+
+/* AFBC format mode. The ordering is intended to match the Valhall hardware enum
+ * ("AFBC Compression Mode"), but this enum is required in software on older
+ * hardware for correct handling of texture views. Defining the enum lets us
+ * unify these code paths.
+ */
+enum pan_afbc_mode {
+   PAN_AFBC_MODE_R8,
+   PAN_AFBC_MODE_R8G8,
+   PAN_AFBC_MODE_R5G6B5,
+   PAN_AFBC_MODE_R4G4B4A4,
+   PAN_AFBC_MODE_R5G5B5A1,
+   PAN_AFBC_MODE_R8G8B8,
+   PAN_AFBC_MODE_R8G8B8A8,
+   PAN_AFBC_MODE_R10G10B10A2,
+   PAN_AFBC_MODE_R11G11B10,
+
+   /* 16 bit modes, available on v10+ */
+   PAN_AFBC_MODE_R16,
+   PAN_AFBC_MODE_R16G16,
+   PAN_AFBC_MODE_R16G16B16,
+   PAN_AFBC_MODE_R16G16B16A16,
+
+   /* YUV special modes */
+   PAN_AFBC_MODE_YUV420_6C8,
+   PAN_AFBC_MODE_YUV420_2C8,
+   PAN_AFBC_MODE_YUV420_1C8,
+   PAN_AFBC_MODE_YUV420_6C10,
+   PAN_AFBC_MODE_YUV420_2C10,
+   PAN_AFBC_MODE_YUV420_1C10,
+
+   PAN_AFBC_MODE_YUV422_4C8,
+   PAN_AFBC_MODE_YUV422_2C8,
+   PAN_AFBC_MODE_YUV422_1C8,
+   PAN_AFBC_MODE_YUV422_4C10,
+   PAN_AFBC_MODE_YUV422_2C10,
+   PAN_AFBC_MODE_YUV422_1C10,
+
+   /* Sentintel signalling a format that cannot be compressed */
+   PAN_AFBC_MODE_INVALID
+};
+
+/*
+ * An AFBC header block provides access to an associated superblock payload of
+ * 4x4 subblocks or to an embedded solid color.
+ */
+struct pan_afbc_headerblock {
+   union {
+      /* Superblock payload. */
+      struct {
+         /* Offset in bytes from the start of the AFBC buffer (1st header
+          * block) to the start of the superblock payload data. */
+         uint32_t offset;
+
+         /* Sizes in bytes of the 4x4 6-bit subblocks. */
+         uint8_t subblock_sizes[12];
+      } payload;
+
+      /* Solid color. */
+      struct {
+         uint64_t reserved;
+
+         /* RGBA 8-8-8-8 color format. */
+         /* XXX: Add other formats. */
+         struct {
+            uint8_t r, g, b, a;
+            uint32_t reserved;
+         } rgba8888;
+      } color;
+
+      /* Random access. */
+      uint8_t u8[16];
+      uint16_t u16[8];
+      uint32_t u32[4];
+      uint64_t u64[2];
+   };
+};
+
+/*
+ * An AFBC payload extent describes the extent of the payload data (compressed
+ * superblock data) associated to a pan_afbc_headerblock.
+ */
+struct pan_afbc_payload_extent {
+   uint32_t size;
+   uint32_t offset;
+};
+
+/*
+ * Given an AFBC modifier, return the superblock size.
+ *
+ * We do not yet have any use cases for multiplanar YCBCr formats with different
+ * superblock sizes on the luma and chroma planes. These formats are unsupported
+ * for now.
+ */
+static inline struct pan_image_block_size
+pan_afbc_superblock_size(uint64_t modifier)
+{
+   unsigned index = (modifier & AFBC_FORMAT_MOD_BLOCK_SIZE_MASK);
+
+   assert(drm_is_afbc(modifier));
+
+   switch (index) {
+   case AFBC_FORMAT_MOD_BLOCK_SIZE_16x16:
+      return (struct pan_image_block_size){16, 16};
+   case AFBC_FORMAT_MOD_BLOCK_SIZE_32x8:
+      return (struct pan_image_block_size){32, 8};
+   case AFBC_FORMAT_MOD_BLOCK_SIZE_64x4:
+      return (struct pan_image_block_size){64, 4};
+   default:
+      assert(!"Unsupported AFBC block size");
+      return (struct pan_image_block_size){0, 0};
+   }
+}
+
+/* Same as pan_afbc_superblock_size_el() but counted in block elements
+ * instead of pixels. For anything non-YUV this is the same. */
+static inline struct pan_image_block_size
+pan_afbc_superblock_size_el(enum pipe_format format, uint64_t modifier)
+{
+   struct pan_image_block_size sb_size_px = pan_afbc_superblock_size(modifier);
+
+   assert(sb_size_px.width % util_format_get_blockwidth(format) == 0);
+   assert(sb_size_px.height % util_format_get_blockheight(format) == 0);
+
+   return (struct pan_image_block_size){
+      .width = sb_size_px.width / util_format_get_blockwidth(format),
+      .height = sb_size_px.height / util_format_get_blockheight(format),
+   };
+}
+
+/*
+ * Given an AFBC modifier, return the render size.
+ */
+static inline struct pan_image_block_size
+pan_afbc_renderblock_size(uint64_t modifier)
+{
+   struct pan_image_block_size blk_size = pan_afbc_superblock_size(modifier);
+
+   /* The GPU needs to render 16x16 tiles. For wide tiles, that means we
+    * have to extend the render region to have a height of 16 pixels.
+    */
+   blk_size.height = ALIGN_POT(blk_size.height, 16);
+   return blk_size;
+}
+
+
+/* Same as pan_afbc_renderblock_size() but counted in block elements
+ * instead of pixels. For anything non-YUV this is the same. */
+static inline struct pan_image_block_size
+pan_afbc_renderblock_size_el(enum pipe_format format, uint64_t modifier)
+{
+   struct pan_image_block_size rb_size_px = pan_afbc_renderblock_size(modifier);
+
+   assert(rb_size_px.width % util_format_get_blockwidth(format) == 0);
+   assert(rb_size_px.height % util_format_get_blockheight(format) == 0);
+
+   return (struct pan_image_block_size){
+      .width = rb_size_px.width / util_format_get_blockwidth(format),
+      .height = rb_size_px.height / util_format_get_blockheight(format),
+   };
+}
+
+/*
+ * Given an AFBC modifier, return the width of the superblock.
+ */
+static inline unsigned
+pan_afbc_superblock_width(uint64_t modifier)
+{
+   return pan_afbc_superblock_size(modifier).width;
+}
+
+/*
+ * Given an AFBC modifier, return the height of the superblock.
+ */
+static inline unsigned
+pan_afbc_superblock_height(uint64_t modifier)
+{
+   return pan_afbc_superblock_size(modifier).height;
+}
+
+/*
+ * Given an AFBC modifier, return if "wide blocks" are used. Wide blocks are
+ * defined as superblocks wider than 16 pixels, the minimum (and default) super
+ * block width.
+ */
+static inline bool
+pan_afbc_is_wide(uint64_t modifier)
+{
+   return pan_afbc_superblock_width(modifier) > 16;
+}
+
+/*
+ * Given an AFBC modifier, return the subblock size (subdivision of a
+ * superblock). This is always 4x4 for now as we only support one AFBC
+ * superblock layout.
+ */
+static inline struct pan_image_block_size
+pan_afbc_subblock_size(uint64_t modifier)
+{
+   return (struct pan_image_block_size){4, 4};
+}
+
+/*
+ * Given an AFBC header block, return the size of the subblock at the given
+ * index in the range [0, 15].
+ */
+static inline unsigned
+pan_afbc_header_subblock_size(struct pan_afbc_headerblock header,
+                              uint32_t index)
+{
+   uint64_t mask = BITFIELD_MASK(6);
+
+   switch (index) {
+   case  0: return  (header.u64[0] >> 32)  & mask; break;
+   case  1: return  (header.u64[0] >> 38)  & mask; break;
+   case  2: return  (header.u64[0] >> 44)  & mask; break;
+   case  3: return  (header.u64[0] >> 50)  & mask; break;
+   case  4: return  (header.u64[0] >> 56)  & mask; break;
+   case  5: return ((header.u64[0] >> 62) |
+                    (header.u64[1] <<  2)) & mask; break;
+   case  6: return  (header.u64[1] >>  4)  & mask; break;
+   case  7: return  (header.u64[1] >> 10)  & mask; break;
+   case  8: return  (header.u64[1] >> 16)  & mask; break;
+   case  9: return  (header.u64[1] >> 22)  & mask; break;
+   case 10: return  (header.u64[1] >> 28)  & mask; break;
+   case 11: return  (header.u64[1] >> 34)  & mask; break;
+   case 12: return  (header.u64[1] >> 40)  & mask; break;
+   case 13: return  (header.u64[1] >> 46)  & mask; break;
+   case 14: return  (header.u64[1] >> 52)  & mask; break;
+   case 15: return  (header.u64[1] >> 58)  & mask; break;
+   default: UNREACHABLE("invalid index"); return 0;
+   }
+}
+
+/*
+ * Given an AFBC header block, return the size in bytes of the associated
+ * superblock payload data (for the superblock layouts 0, 3, 4 and 7).
+ */
+static inline uint32_t
+pan_afbc_payload_size(unsigned arch,
+                      struct pan_afbc_headerblock header,
+                      uint32_t uncompressed_size)
+{
+   /* Skip sum if the 1st subblock is 0 (solid color encoding). */
+   if (arch >= 7 && pan_afbc_header_subblock_size(header, 0) == 0)
+      return 0;
+
+   uint64_t size = 0;
+
+   for (unsigned i = 0; i < 16; i++) {
+      unsigned sub_size = pan_afbc_header_subblock_size(header, i);
+      size += sub_size != 1 ? sub_size : uncompressed_size;
+   }
+
+   return ALIGN_POT(size, 16);
+}
+
+/*
+ * Given a format and a modifier, return the size in bytes of an uncompressed
+ * superblock payload.
+ */
+static inline uint32_t
+pan_afbc_payload_uncompressed_size(enum pipe_format format, uint64_t modifier)
+{
+   struct pan_image_block_size size_px = pan_afbc_subblock_size(modifier);
+   uint32_t size_B = util_format_get_blocksizebits(format) / 8;
+   size_B *= size_px.width * size_px.height;
+
+   assert(size_B == ALIGN_POT(size_B, 16));
+
+   return size_B;
+}
+
+/*
+ * Calculate the size of each AFBC superblock payload data from the given
+ * header blocks, generate a packed AFBC payload layout and return the body
+ * size.
+ */
+uint32_t
+pan_afbc_payload_layout_packed(unsigned arch,
+                               const struct pan_afbc_headerblock *headers,
+                               struct pan_afbc_payload_extent *layout,
+                               uint32_t nr_blocks, enum pipe_format format,
+                               uint64_t modifier);
+
+static inline uint32_t
+pan_afbc_header_row_stride_align(unsigned arch, enum pipe_format format,
+                                 uint64_t modifier)
+{
+   if (arch <= 7 || !(modifier & AFBC_FORMAT_MOD_TILED))
+      return 16;
+
+   if (util_format_get_blocksizebits(format) <= 32)
+      return 1024;
+   else
+      return 256;
+}
+
+static inline uint32_t
+pan_afbc_header_align(unsigned arch, uint64_t modifier)
+{
+   if (modifier & AFBC_FORMAT_MOD_TILED)
+      return 4096;
+   else if (arch >= 6)
+      return 128;
+   else
+      return 64;
+}
+
+/*
+ * Determine the required alignment for the body offset of an AFBC image. For
+ * now, this depends only on whether tiling is in use. These minimum alignments
+ * are required on all current GPUs.
+ */
+static inline uint32_t
+pan_afbc_body_align(unsigned arch, uint64_t modifier)
+{
+   /* Body and header alignments are actually the same. */
+   return pan_afbc_header_align(arch, modifier);
+}
+
+/* Get the body offset for a given AFBC header size. */
+static inline uint32_t
+pan_afbc_body_offset(unsigned arch, uint64_t modifier, uint32_t header_size)
+{
+   return ALIGN_POT(header_size, pan_afbc_body_align(arch, modifier));
+}
+
+/*
+ * Determine the tile size used by AFBC. This tiles superblocks themselves.
+ * Current GPUs support either 8x8 tiling or no tiling (1x1) for most
+ * formats; but note that formats with more than 32bpp use 4x4 tiling
+ */
+static inline unsigned
+pan_afbc_tile_size(enum pipe_format format, uint64_t modifier)
+{
+   if ( !(modifier & AFBC_FORMAT_MOD_TILED) ) {
+      return 1;
+   }
+   return util_format_get_blocksizebits(format) <= 32 ? 8 : 4;
+}
+
+/*
+ * Determine the number of bytes between header rows for an AFBC image. For an
+ * image with linear headers, this is simply the number of header blocks
+ * (=superblocks) per row times the numbers of bytes per header block. For an
+ * image with tiled headers, this is multipled by the number of rows of
+ * header blocks are in a tile together.
+ */
+static inline uint32_t
+pan_afbc_row_stride(enum pipe_format format, uint64_t modifier, uint32_t width)
+{
+   unsigned block_width = pan_afbc_superblock_width(modifier);
+
+   return (width / block_width) * pan_afbc_tile_size(format, modifier) *
+          AFBC_HEADER_BYTES_PER_TILE;
+}
+
+/*
+ * Determine the number of header blocks between header rows. This is equal to
+ * the number of bytes between header rows divided by the bytes per blocks of a
+ * header tile. This is also divided by the tile size to give a "line stride" in
+ * blocks, rather than a real row stride. This is required by Bifrost.
+ */
+static inline uint32_t
+pan_afbc_stride_blocks(enum pipe_format format, uint64_t modifier, uint32_t row_stride_bytes)
+{
+   return row_stride_bytes /
+      (AFBC_HEADER_BYTES_PER_TILE * pan_afbc_tile_size(format, modifier));
+}
+
+/* Returns a height in superblocks taking into account the tile alignment
+ * requirement coming from the modifier.
+ */
+static inline uint32_t
+pan_afbc_height_blocks(enum pipe_format format, uint64_t modifier, uint32_t height_px)
+{
+   return ALIGN_POT(
+      DIV_ROUND_UP(height_px, pan_afbc_superblock_height(modifier)),
+      pan_afbc_tile_size(format, modifier));
+}
+
+static inline enum pipe_format
+pan_afbc_unswizzled_format(unsigned arch, enum pipe_format format)
+{
+   switch (format) {
+   case PIPE_FORMAT_A8_UNORM:
+   case PIPE_FORMAT_L8_UNORM:
+   case PIPE_FORMAT_I8_UNORM:
+      return PIPE_FORMAT_R8_UNORM;
+
+   case PIPE_FORMAT_L8A8_UNORM:
+      return PIPE_FORMAT_R8G8_UNORM;
+
+   case PIPE_FORMAT_B8G8R8_UNORM:
+      return PIPE_FORMAT_R8G8B8_UNORM;
+
+   case PIPE_FORMAT_R8G8B8X8_UNORM:
+   case PIPE_FORMAT_B8G8R8A8_UNORM:
+   case PIPE_FORMAT_B8G8R8X8_UNORM:
+      return PIPE_FORMAT_R8G8B8A8_UNORM;
+   case PIPE_FORMAT_A8R8G8B8_UNORM:
+   case PIPE_FORMAT_X8R8G8B8_UNORM:
+   case PIPE_FORMAT_X8B8G8R8_UNORM:
+   case PIPE_FORMAT_A8B8G8R8_UNORM:
+      /* v7 can only support AFBC for RGB and BGR */
+      return arch == 7 ? format : PIPE_FORMAT_R8G8B8A8_UNORM;
+
+   case PIPE_FORMAT_B5G6R5_UNORM:
+      return PIPE_FORMAT_R5G6B5_UNORM;
+
+   case PIPE_FORMAT_B5G5R5A1_UNORM:
+      return PIPE_FORMAT_R5G5B5A1_UNORM;
+
+   case PIPE_FORMAT_R10G10B10X2_UNORM:
+   case PIPE_FORMAT_B10G10R10A2_UNORM:
+   case PIPE_FORMAT_B10G10R10X2_UNORM:
+      return PIPE_FORMAT_R10G10B10A2_UNORM;
+
+   case PIPE_FORMAT_B4G4R4A4_UNORM:
+      return PIPE_FORMAT_R4G4B4A4_UNORM;
+   case PIPE_FORMAT_A4B4G4R4_UNORM:
+      /* v7 can only support AFBC for RGB and BGR */
+      return arch == 7 ? format : PIPE_FORMAT_R4G4B4A4_UNORM;
+
+   default:
+      return format;
+   }
+}
+
+/* AFBC supports compressing a few canonical formats. Additional formats are
+ * available by using a canonical internal format. Given a PIPE format, find
+ * the canonical AFBC internal format if it exists, or NONE if the format
+ * cannot be compressed. */
+
+static inline enum pan_afbc_mode
+pan_afbc_format(unsigned arch, enum pipe_format format, unsigned plane_idx)
+{
+   assert(plane_idx < util_format_get_num_planes(format));
+
+   switch (format) {
+   case PIPE_FORMAT_R8_G8B8_420_UNORM:
+   case PIPE_FORMAT_R8_B8G8_420_UNORM:
+      return plane_idx == 0 ? PAN_AFBC_MODE_YUV420_1C8
+                            : PAN_AFBC_MODE_YUV420_2C8;
+   case PIPE_FORMAT_R8_G8_B8_420_UNORM:
+   case PIPE_FORMAT_R8_B8_G8_420_UNORM:
+      return PAN_AFBC_MODE_YUV420_1C8;
+   case PIPE_FORMAT_R8_G8B8_422_UNORM:
+      return plane_idx == 0 ? PAN_AFBC_MODE_YUV422_1C8
+                            : PAN_AFBC_MODE_YUV422_2C8;
+   case PIPE_FORMAT_R10_G10B10_420_UNORM:
+      return plane_idx == 0 ? PAN_AFBC_MODE_YUV420_1C10
+                            : PAN_AFBC_MODE_YUV420_2C10;
+   case PIPE_FORMAT_R10_G10B10_422_UNORM:
+      return plane_idx == 0 ? PAN_AFBC_MODE_YUV422_1C10
+                            : PAN_AFBC_MODE_YUV422_2C10;
+   case PIPE_FORMAT_R8G8B8_420_UNORM_PACKED:
+      return PAN_AFBC_MODE_YUV420_6C8;
+   case PIPE_FORMAT_R10G10B10_420_UNORM_PACKED:
+      return PAN_AFBC_MODE_YUV420_6C10;
+   case PIPE_FORMAT_R8G8_R8B8_UNORM:
+   case PIPE_FORMAT_R8B8_R8G8_UNORM:
+      return PAN_AFBC_MODE_YUV422_4C8;
+   case PIPE_FORMAT_X6R10X6G10_X6R10X6B10_422_UNORM:
+      return PAN_AFBC_MODE_YUV422_4C10;
+   default:
+      break;
+   }
+
+   /* sRGB does not change the pixel format itself, only the
+    * interpretation. The interpretation is handled by conversion hardware
+    * independent to the compression hardware, so we can compress sRGB
+    * formats by using the corresponding linear format.
+    */
+   format = util_format_linear(format);
+
+   /* AFBC is type agnostic on v9+, so the following logic is done on UNORM
+    * for simplicity, so make sure we convert to _UNORM first.
+    */
+   if (arch >= 9)
+      format = util_format_any_to_unorm(format);
+
+   /* Luminance-alpha not supported for AFBC on v7+ */
+   /* 16 bit modes only available for AFBC on v10 and later */
+   switch (format) {
+   case PIPE_FORMAT_A8_UNORM:
+   case PIPE_FORMAT_L8_UNORM:
+   case PIPE_FORMAT_I8_UNORM:
+   case PIPE_FORMAT_L8A8_UNORM:
+      if (arch >= 7)
+         return PAN_AFBC_MODE_INVALID;
+      else
+         break;
+   case PIPE_FORMAT_R16_UNORM:
+   case PIPE_FORMAT_R16G16_UNORM:
+   case PIPE_FORMAT_R16G16B16_UNORM:
+   case PIPE_FORMAT_R16G16B16A16_UNORM:
+      if (arch < 10)
+         return PAN_AFBC_MODE_INVALID;
+      else
+         break;
+   case PIPE_FORMAT_S8_UINT:
+   case PIPE_FORMAT_R11G11B10_FLOAT:
+      if (arch < 9)
+         return PAN_AFBC_MODE_INVALID;
+      else
+         break;
+   default:
+      break;
+   }
+
+   /* We handle swizzling orthogonally to AFBC */
+   format = pan_afbc_unswizzled_format(arch, format);
+
+   /* clang-format off */
+   switch (format) {
+   case PIPE_FORMAT_R8_UNORM:          return PAN_AFBC_MODE_R8;
+   case PIPE_FORMAT_R8G8_UNORM:        return PAN_AFBC_MODE_R8G8;
+   case PIPE_FORMAT_R8G8B8_UNORM:      return PAN_AFBC_MODE_R8G8B8;
+   case PIPE_FORMAT_R8G8B8A8_UNORM:    return PAN_AFBC_MODE_R8G8B8A8;
+   case PIPE_FORMAT_R5G6B5_UNORM:      return PAN_AFBC_MODE_R5G6B5;
+   case PIPE_FORMAT_R5G5B5A1_UNORM:    return PAN_AFBC_MODE_R5G5B5A1;
+   case PIPE_FORMAT_R10G10B10A2_UNORM: return PAN_AFBC_MODE_R10G10B10A2;
+   case PIPE_FORMAT_R4G4B4A4_UNORM:    return PAN_AFBC_MODE_R4G4B4A4;
+   case PIPE_FORMAT_Z16_UNORM:         return PAN_AFBC_MODE_R8G8;
+
+   case PIPE_FORMAT_Z32_FLOAT:         return PAN_AFBC_MODE_R8G8B8A8;
+   case PIPE_FORMAT_Z24_UNORM_S8_UINT: return PAN_AFBC_MODE_R8G8B8A8;
+   case PIPE_FORMAT_Z24X8_UNORM:       return PAN_AFBC_MODE_R8G8B8A8;
+   case PIPE_FORMAT_X24S8_UINT:        return PAN_AFBC_MODE_R8G8B8A8;
+   case PIPE_FORMAT_Z24_UNORM_PACKED:  return PAN_AFBC_MODE_R8G8B8;
+
+   case PIPE_FORMAT_S8_UINT:           return PAN_AFBC_MODE_R8;
+   case PIPE_FORMAT_R11G11B10_FLOAT:   return PAN_AFBC_MODE_R11G11B10;
+
+   case PIPE_FORMAT_R16_UNORM:         return PAN_AFBC_MODE_R16;
+   case PIPE_FORMAT_R16G16_UNORM:      return PAN_AFBC_MODE_R16G16;
+   case PIPE_FORMAT_R16G16B16_UNORM:   return PAN_AFBC_MODE_R16G16B16;
+   case PIPE_FORMAT_R16G16B16A16_UNORM:
+                                       return PAN_AFBC_MODE_R16G16B16A16;
+
+   default:                            return PAN_AFBC_MODE_INVALID;
+   }
+   /* clang-format on */
+}
+
+/* A format may be compressed as AFBC if it has an AFBC internal format */
+
+static inline bool
+pan_afbc_supports_format(unsigned arch, enum pipe_format format)
+{
+   unsigned plane_count = util_format_get_num_planes(format);
+
+   for (unsigned i = 0; i < plane_count; i++) {
+      if (pan_afbc_format(arch, format, i) == PAN_AFBC_MODE_INVALID)
+         return false;
+   }
+
+   return true;
+}
+
+/* The lossless colour transform (AFBC_FORMAT_MOD_YTR) requires RGB. */
+
+static inline bool
+pan_afbc_can_ytr(enum pipe_format format)
+{
+   const struct util_format_description *desc = util_format_description(format);
+
+   /* YTR is only defined for RGB(A) */
+   if (desc->nr_channels != 3 && desc->nr_channels != 4)
+      return false;
+
+   /* The fourth channel if it exists doesn't matter */
+   return desc->colorspace == UTIL_FORMAT_COLORSPACE_RGB;
+}
+
+static inline bool
+pan_afbc_check_params(enum pan_afbc_mode mode, uint64_t modifier)
+{
+   unsigned block_width = pan_afbc_superblock_width(modifier);
+   bool is_split = modifier & AFBC_FORMAT_MOD_SPLIT;
+
+   /* YTR is not useful on RGB formats, nor with one- or two-channel
+    * formats */
+   bool is_ytr = modifier & AFBC_FORMAT_MOD_YTR;
+   switch (mode) {
+   case PAN_AFBC_MODE_YUV420_6C8:
+   case PAN_AFBC_MODE_YUV420_2C8:
+   case PAN_AFBC_MODE_YUV420_1C8:
+   case PAN_AFBC_MODE_YUV420_6C10:
+   case PAN_AFBC_MODE_YUV420_2C10:
+   case PAN_AFBC_MODE_YUV420_1C10:
+   case PAN_AFBC_MODE_YUV422_4C8:
+   case PAN_AFBC_MODE_YUV422_2C8:
+   case PAN_AFBC_MODE_YUV422_1C8:
+   case PAN_AFBC_MODE_YUV422_4C10:
+   case PAN_AFBC_MODE_YUV422_2C10:
+   case PAN_AFBC_MODE_YUV422_1C10:
+   case PAN_AFBC_MODE_R8:
+   case PAN_AFBC_MODE_R8G8:
+   case PAN_AFBC_MODE_R16:
+   case PAN_AFBC_MODE_R16G16:
+      if (is_ytr)
+         return false;
+      break;
+   default:
+      break;
+   }
+
+   /* 16x16 superblocks are universally compatible */
+   if (block_width == 16)
+      return true;
+
+   /* 64x4 blocks are only usable for multi-planar YUV formats,
+    * and not in block-split mode. */
+   if (block_width == 64) {
+      switch (mode) {
+      case PAN_AFBC_MODE_YUV420_2C8:
+      case PAN_AFBC_MODE_YUV420_1C8:
+      case PAN_AFBC_MODE_YUV420_2C10:
+      case PAN_AFBC_MODE_YUV420_1C10:
+      case PAN_AFBC_MODE_YUV422_2C8:
+      case PAN_AFBC_MODE_YUV422_1C8:
+      case PAN_AFBC_MODE_YUV422_2C10:
+      case PAN_AFBC_MODE_YUV422_1C10:
+         return !is_split;
+      default:
+         return false;
+      }
+   }
+
+   assert(block_width == 32);
+
+   /* 32x8 superblocks have different requirements based on the format:
+    * block-split mode is required for effective bpp >16, and prohibited
+    * for effective bpp <= 16. */
+   switch (mode) {
+   case PAN_AFBC_MODE_R8G8B8:
+   case PAN_AFBC_MODE_R8G8B8A8:
+   case PAN_AFBC_MODE_R10G10B10A2:
+   case PAN_AFBC_MODE_R11G11B10:
+   case PAN_AFBC_MODE_R16G16:
+   case PAN_AFBC_MODE_R16G16B16:
+   case PAN_AFBC_MODE_R16G16B16A16:
+      return is_split;
+   case PAN_AFBC_MODE_R8:
+   case PAN_AFBC_MODE_R8G8:
+   case PAN_AFBC_MODE_R5G6B5:
+   case PAN_AFBC_MODE_R4G4B4A4:
+   case PAN_AFBC_MODE_R5G5B5A1:
+   case PAN_AFBC_MODE_R16:
+   case PAN_AFBC_MODE_YUV420_6C8:
+   case PAN_AFBC_MODE_YUV420_6C10:
+   case PAN_AFBC_MODE_YUV422_4C8:
+   case PAN_AFBC_MODE_YUV422_4C10:
+      return !is_split;
+   /* The following split luma/chroma layouts require
+    * AFBC_FORMAT_MOD_BLOCK_SIZE_32x8_64x4 to be usable, since the luma
+    * planes require non-split 32x8, and the chroma planes require 64x4.
+    */
+   case PAN_AFBC_MODE_YUV420_1C8:
+   case PAN_AFBC_MODE_YUV420_1C10:
+   case PAN_AFBC_MODE_YUV422_1C8:
+   case PAN_AFBC_MODE_YUV422_1C10:
+      return !is_split;
+   case PAN_AFBC_MODE_YUV420_2C8:
+   case PAN_AFBC_MODE_YUV420_2C10:
+   case PAN_AFBC_MODE_YUV422_2C8:
+   case PAN_AFBC_MODE_YUV422_2C10:
+      return false;
+   default:
+      assert(!"unknown mode in pan_afbc_check_params()");
+      return false;
+   }
+}
+
+/* Only support packing for RGB formats for now. */
+
+static inline bool
+pan_afbc_can_pack(enum pipe_format format)
+{
+   const struct util_format_description *desc = util_format_description(format);
+
+   return desc->colorspace == UTIL_FORMAT_COLORSPACE_RGB;
+}
+
+/*
+ * Check if a gen supports AFBC with tiled headers (and hence also solid
+ * colour blocks).
+ */
+static inline bool
+pan_afbc_can_tile(unsigned arch)
+{
+   return arch >= 7;
+}
+
+/*
+ * Check if a gen supports AFBC with split-block mode.
+ */
+static inline bool
+pan_afbc_can_split(unsigned arch)
+{
+   return arch >= 6;
+}
+
+#if PAN_ARCH >= 9
+static inline enum mali_afbc_compression_mode
+pan_afbc_compression_mode(enum pan_afbc_mode mode)
+{
+   /* Map canonical formats to the hardware enum. This only
+    * needs to handle the subset of formats returned by
+    * pan_afbc_format.
+    */
+   switch (mode) {
+   case PAN_AFBC_MODE_R8:
+      return MALI_AFBC_COMPRESSION_MODE_R8;
+   case PAN_AFBC_MODE_R8G8:
+      return MALI_AFBC_COMPRESSION_MODE_R8G8;
+   case PAN_AFBC_MODE_R5G6B5:
+      return MALI_AFBC_COMPRESSION_MODE_R5G6B5;
+   case PAN_AFBC_MODE_R4G4B4A4:
+      return MALI_AFBC_COMPRESSION_MODE_R4G4B4A4;
+   case PAN_AFBC_MODE_R5G5B5A1:
+      return MALI_AFBC_COMPRESSION_MODE_R5G5B5A1;
+   case PAN_AFBC_MODE_R8G8B8:
+      return MALI_AFBC_COMPRESSION_MODE_R8G8B8;
+   case PAN_AFBC_MODE_R8G8B8A8:
+      return MALI_AFBC_COMPRESSION_MODE_R8G8B8A8;
+   case PAN_AFBC_MODE_R10G10B10A2:
+      return MALI_AFBC_COMPRESSION_MODE_R10G10B10A2;
+   case PAN_AFBC_MODE_R11G11B10:
+      return MALI_AFBC_COMPRESSION_MODE_R11G11B10;
+#if PAN_ARCH >= 10
+   case PAN_AFBC_MODE_R16:
+      return MALI_AFBC_COMPRESSION_MODE_R16;
+   case PAN_AFBC_MODE_R16G16:
+      return MALI_AFBC_COMPRESSION_MODE_R16G16;
+   case PAN_AFBC_MODE_R16G16B16:
+      return MALI_AFBC_COMPRESSION_MODE_R16G16B16;
+   case PAN_AFBC_MODE_R16G16B16A16:
+      return MALI_AFBC_COMPRESSION_MODE_R16G16B16A16;
+#endif
+#if PAN_ARCH >= 14
+   case PAN_AFBC_MODE_YUV420_6C8:
+      return MALI_AFBC_COMPRESSION_MODE_Y8U8V8_420;
+   case PAN_AFBC_MODE_YUV420_2C8:
+      return MALI_AFBC_COMPRESSION_MODE_R8G8;
+   case PAN_AFBC_MODE_YUV420_1C8:
+      return MALI_AFBC_COMPRESSION_MODE_R8;
+   case PAN_AFBC_MODE_YUV420_6C10:
+      return MALI_AFBC_COMPRESSION_MODE_Y10U10V10_420;
+   case PAN_AFBC_MODE_YUV420_2C10:
+      return MALI_AFBC_COMPRESSION_MODE_R10G10;
+   case PAN_AFBC_MODE_YUV420_1C10:
+      return MALI_AFBC_COMPRESSION_MODE_R10;
+   case PAN_AFBC_MODE_YUV422_4C8:
+      return MALI_AFBC_COMPRESSION_MODE_Y8U8Y8V8_422;
+   case PAN_AFBC_MODE_YUV422_2C8:
+      return MALI_AFBC_COMPRESSION_MODE_R8G8;
+   case PAN_AFBC_MODE_YUV422_1C8:
+      return MALI_AFBC_COMPRESSION_MODE_R8;
+   case PAN_AFBC_MODE_YUV422_4C10:
+      return MALI_AFBC_COMPRESSION_MODE_Y10U10Y10V10_422;
+   case PAN_AFBC_MODE_YUV422_2C10:
+      return MALI_AFBC_COMPRESSION_MODE_R10G10;
+   case PAN_AFBC_MODE_YUV422_1C10:
+      return MALI_AFBC_COMPRESSION_MODE_R10;
+#else
+   case PAN_AFBC_MODE_YUV420_6C8:
+      return MALI_AFBC_COMPRESSION_MODE_YUV420_6C8;
+   case PAN_AFBC_MODE_YUV420_2C8:
+      return MALI_AFBC_COMPRESSION_MODE_YUV420_2C8;
+   case PAN_AFBC_MODE_YUV420_1C8:
+      return MALI_AFBC_COMPRESSION_MODE_YUV420_1C8;
+   case PAN_AFBC_MODE_YUV420_6C10:
+      return MALI_AFBC_COMPRESSION_MODE_YUV420_6C10;
+   case PAN_AFBC_MODE_YUV420_2C10:
+      return MALI_AFBC_COMPRESSION_MODE_YUV420_2C10;
+   case PAN_AFBC_MODE_YUV420_1C10:
+      return MALI_AFBC_COMPRESSION_MODE_YUV420_1C10;
+   case PAN_AFBC_MODE_YUV422_4C8:
+      return MALI_AFBC_COMPRESSION_MODE_YUV422_4C8;
+   case PAN_AFBC_MODE_YUV422_2C8:
+      return MALI_AFBC_COMPRESSION_MODE_YUV422_2C8;
+   case PAN_AFBC_MODE_YUV422_1C8:
+      return MALI_AFBC_COMPRESSION_MODE_YUV422_1C8;
+   case PAN_AFBC_MODE_YUV422_4C10:
+      return MALI_AFBC_COMPRESSION_MODE_YUV422_4C10;
+   case PAN_AFBC_MODE_YUV422_2C10:
+      return MALI_AFBC_COMPRESSION_MODE_YUV422_2C10;
+   case PAN_AFBC_MODE_YUV422_1C10:
+      return MALI_AFBC_COMPRESSION_MODE_YUV422_1C10;
+#endif /* PAN_ARCH >= 14 */
+#if PAN_ARCH == 9
+   case PAN_AFBC_MODE_R16:
+   case PAN_AFBC_MODE_R16G16:
+   case PAN_AFBC_MODE_R16G16B16:
+   case PAN_AFBC_MODE_R16G16B16A16:
+#endif
+   case PAN_AFBC_MODE_INVALID:
+      UNREACHABLE("Invalid AFBC format");
+   }
+
+   UNREACHABLE("all AFBC formats handled");
+}
+
+static inline enum mali_afbc_compression_mode
+pan_afbc_decompression_mode(enum pipe_format view_format,
+                            enum pan_afbc_mode img_mode)
+{
+   /* There are special cases for texture views of stencil images. */
+   if (img_mode == PAN_AFBC_MODE_R8G8B8A8 &&
+       view_format == PIPE_FORMAT_X24S8_UINT)
+      return MALI_AFBC_COMPRESSION_MODE_X24S8;
+   else if (view_format == PIPE_FORMAT_S8_UINT)
+      return MALI_AFBC_COMPRESSION_MODE_S8;
+
+   return pan_afbc_compression_mode(img_mode);
+}
+#endif
+
+static inline bool
+format_requires_afbc(enum pipe_format format) {
+   switch (format) {
+   case PIPE_FORMAT_R8G8B8_420_UNORM_PACKED:
+   case PIPE_FORMAT_R10G10B10_420_UNORM_PACKED:
+   case PIPE_FORMAT_X6R10X6G10_X6R10X6B10_422_UNORM:
+      return true;
+   default:
+      return false;
+   }
+}
+
+#ifdef __cplusplus
+} /* extern C */
+#endif
+
+#endif

src/panfrost/lib/pan_afrc.h

@@ -0,0 +1,448 @@
+/*
+ * Copyright (C) 2023 Collabora, Ltd.
+ * Copyright (C) 2026 Arm Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef __PAN_AFRC_H
+#define __PAN_AFRC_H
+
+#include "pan_format.h"
+#include "pan_layout.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Arm Fixed-Rate Compression (AFRC) is a lossy compression scheme natively
+ * implemented in Mali GPUs. AFRC images can only be rendered or textured
+ * from. It is currently not possible to do image reads or writes to such
+ * resources.
+ *
+ * AFRC divides the image into an array of fixed-size coding units which are
+ * grouped into paging tiles. The size of the coding units (clump size)
+ * depends on the image format and the pixel layout (whether it is optimized
+ * for 2D locality and rotation, or for scan line order access). The last
+ * parameter is the size of the compressed block that can be either 16, 24,
+ * or 32 bytes.
+ *
+ * The compression rate can be calculated by dividing the compressed block
+ * size by the uncompressed block size (clump size multiplied by the component
+ * size and the number of components).
+ */
+
+#define AFRC_CLUMPS_PER_TILE 64
+
+enum pan_afrc_rate {
+   PAN_AFRC_RATE_NONE,
+   PAN_AFRC_RATE_1BPC,
+   PAN_AFRC_RATE_2BPC,
+   PAN_AFRC_RATE_3BPC,
+   PAN_AFRC_RATE_4BPC,
+   PAN_AFRC_RATE_5BPC,
+   PAN_AFRC_RATE_6BPC,
+   PAN_AFRC_RATE_7BPC,
+   PAN_AFRC_RATE_8BPC,
+   PAN_AFRC_RATE_9BPC,
+   PAN_AFRC_RATE_10BPC,
+   PAN_AFRC_RATE_11BPC,
+   PAN_AFRC_RATE_12BPC,
+   PAN_AFRC_RATE_DEFAULT = 0xF
+};
+
+enum pan_afrc_interchange_format {
+   PAN_AFRC_ICHANGE_FORMAT_RAW,
+   PAN_AFRC_ICHANGE_FORMAT_YUV444,
+   PAN_AFRC_ICHANGE_FORMAT_YUV422,
+   PAN_AFRC_ICHANGE_FORMAT_YUV420,
+};
+
+struct pan_afrc_format_info {
+   unsigned bpc         : 4;
+   unsigned num_comps   : 3;
+   unsigned ichange_fmt : 2;
+   unsigned num_planes  : 2;
+};
+
+/*
+ * Given an AFRC modifier, return whether the layout is optimized for scan
+ * order (vs rotation order).
+ */
+static inline bool
+pan_afrc_is_scan(uint64_t modifier)
+{
+   return modifier & AFRC_FORMAT_MOD_LAYOUT_SCAN;
+}
+
+static inline struct pan_afrc_format_info
+pan_afrc_get_format_info(enum pipe_format format)
+{
+   const struct util_format_description *desc = util_format_description(format);
+   struct pan_afrc_format_info info = {0};
+
+   /* No AFRC(compressed) */
+   if (util_format_is_compressed(format))
+      return info;
+
+   /* No AFRC(ZS). */
+   if (desc->colorspace == UTIL_FORMAT_COLORSPACE_ZS)
+      return info;
+
+   /* No AFRC(YUV) yet. */
+   if (pan_format_is_yuv(format))
+      return info;
+
+   unsigned bpc = 0;
+   for (unsigned c = 0; c < desc->nr_channels; c++) {
+      if (bpc && bpc != desc->channel[c].size)
+         return info;
+
+      bpc = desc->channel[0].size;
+   }
+
+   info.bpc = bpc;
+
+   assert(desc->colorspace == UTIL_FORMAT_COLORSPACE_RGB ||
+          desc->colorspace == UTIL_FORMAT_COLORSPACE_SRGB);
+   info.ichange_fmt = PAN_AFRC_ICHANGE_FORMAT_RAW;
+   info.num_planes = util_format_get_num_planes(format);
+   info.num_comps = util_format_get_nr_components(format);
+   return info;
+}
+
+struct pan_afrc_block_size {
+   unsigned size;          /* Block size in bytes */
+   unsigned alignment;     /* Buffer alignment */
+   uint64_t modifier_flag; /* Part of the modifier for CU size */
+};
+
+#define BLOCK_SIZE(block_size, buffer_alignment)                               \
+   {                                                                           \
+      .size = block_size,                                                      \
+      .alignment = buffer_alignment,                                           \
+      .modifier_flag = AFRC_FORMAT_MOD_CU_SIZE_##block_size,                   \
+   }
+
+#define AFRC_BLOCK_SIZES(__name)                                               \
+   static const struct pan_afrc_block_size __name[] = {                        \
+      BLOCK_SIZE(16, 1024),                                                    \
+      BLOCK_SIZE(24, 512),                                                     \
+      BLOCK_SIZE(32, 2048),                                                    \
+   }
+
+static inline struct pan_image_block_size
+pan_afrc_clump_size(enum pipe_format format, bool scan)
+{
+   struct pan_afrc_format_info finfo = pan_afrc_get_format_info(format);
+
+   switch (finfo.num_comps) {
+   case 1:
+      return scan ? (struct pan_image_block_size){16, 4}
+                  : (struct pan_image_block_size){8, 8};
+   case 2:
+      return (struct pan_image_block_size){8, 4};
+   case 3:
+   case 4:
+      return (struct pan_image_block_size){4, 4};
+   default:
+      assert(0);
+      return (struct pan_image_block_size){0, 0};
+   }
+}
+
+/* Total number of components in a AFRC coding unit */
+static inline unsigned
+pan_afrc_clump_get_nr_components(enum pipe_format format, bool scan)
+{
+   const struct util_format_description *desc = util_format_description(format);
+   struct pan_image_block_size clump_sz = pan_afrc_clump_size(format, scan);
+   return clump_sz.width * clump_sz.height * desc->nr_channels;
+}
+
+static inline bool
+pan_afrc_supports_format(enum pipe_format format)
+{
+   struct pan_afrc_format_info finfo = pan_afrc_get_format_info(format);
+
+   return finfo.num_comps != 0;
+}
+
+static inline unsigned
+pan_afrc_query_rates(enum pipe_format format, unsigned max, uint32_t *rates)
+{
+   if (!pan_afrc_supports_format(format))
+      return 0;
+
+   unsigned clump_comps = pan_afrc_clump_get_nr_components(format, false);
+   unsigned nr_rates = 0;
+
+   /**
+    * From EGL_EXT_surface_compression:
+    *
+    * "For pixel formats with different number of bits per component, the
+    * specified fixed-rate compression rate applies to the component with
+    * the highest number of bits."
+    *
+    * We only support formats where all components have the same size for now.
+    * Let's just use the first component size for calculation.
+    */
+   unsigned uncompressed_rate =
+      util_format_get_component_bits(format, UTIL_FORMAT_COLORSPACE_RGB, 0);
+
+   AFRC_BLOCK_SIZES(block_sizes);
+
+   for (unsigned i = 0; i < ARRAY_SIZE(block_sizes); ++i) {
+      unsigned clump_sz = block_sizes[i].size * 8;
+      unsigned rate = clump_sz / clump_comps;
+
+      if (rate >= uncompressed_rate)
+         continue;
+
+      if (nr_rates < max)
+         rates[nr_rates] = rate;
+      nr_rates++;
+
+      if (max > 0 && nr_rates == max)
+         break;
+   }
+
+   return nr_rates;
+}
+
+static inline unsigned
+pan_afrc_get_modifiers(enum pipe_format format, uint32_t rate, unsigned max,
+                       uint64_t *modifiers)
+{
+   if (!pan_afrc_supports_format(format))
+      return 0;
+
+   /* For now, the number of components in a clump is always the same no
+    * matter the layout for all supported formats */
+   unsigned clump_comps = pan_afrc_clump_get_nr_components(format, false);
+   unsigned count = 0;
+
+   /* FIXME Choose a more sensitive default compression rate? */
+   if (rate == PAN_AFRC_RATE_DEFAULT) {
+      if (max > 0)
+         modifiers[0] = DRM_FORMAT_MOD_ARM_AFRC(AFRC_FORMAT_MOD_CU_SIZE_24);
+
+      if (max > 1)
+         modifiers[1] = DRM_FORMAT_MOD_ARM_AFRC(AFRC_FORMAT_MOD_CU_SIZE_24 |
+                                                AFRC_FORMAT_MOD_LAYOUT_SCAN);
+
+      return 2;
+   }
+
+   AFRC_BLOCK_SIZES(block_sizes);
+   for (unsigned i = 0; i < ARRAY_SIZE(block_sizes); ++i) {
+      unsigned clump_sz = block_sizes[i].size * 8;
+      if (rate == clump_sz / clump_comps) {
+         for (unsigned scan = 0; scan < 2; ++scan) {
+            if (count < max) {
+               modifiers[count] = DRM_FORMAT_MOD_ARM_AFRC(
+                  block_sizes[i].modifier_flag |
+                  (scan ? AFRC_FORMAT_MOD_LAYOUT_SCAN : 0));
+            }
+            count++;
+         }
+      }
+   }
+
+   return count;
+}
+
+static inline unsigned
+pan_afrc_block_size_from_modifier(uint64_t modifier)
+{
+   switch (modifier & AFRC_FORMAT_MOD_CU_SIZE_MASK) {
+   case AFRC_FORMAT_MOD_CU_SIZE_16:
+      return 16;
+   case AFRC_FORMAT_MOD_CU_SIZE_24:
+      return 24;
+   case AFRC_FORMAT_MOD_CU_SIZE_32:
+      return 32;
+   default:
+      UNREACHABLE("invalid coding unit size flag in modifier");
+   };
+}
+
+static inline uint32_t
+pan_afrc_get_rate(enum pipe_format format, uint64_t modifier)
+{
+   if (!drm_is_afrc(modifier) || !pan_afrc_supports_format(format))
+      return PAN_AFRC_RATE_NONE;
+
+   bool scan = pan_afrc_is_scan(modifier);
+   unsigned block_comps = pan_afrc_clump_get_nr_components(format, scan);
+   uint32_t block_sz = pan_afrc_block_size_from_modifier(modifier) * 8;
+
+   return block_sz / block_comps;
+}
+
+static inline struct pan_image_block_size
+pan_afrc_layout_size(uint64_t modifier)
+{
+   if (pan_afrc_is_scan(modifier))
+      return (struct pan_image_block_size){16, 4};
+   else
+      return (struct pan_image_block_size){8, 8};
+}
+
+static inline struct pan_image_block_size
+pan_afrc_tile_size(enum pipe_format format, uint64_t modifier)
+{
+   bool scan = pan_afrc_is_scan(modifier);
+   struct pan_image_block_size clump_sz = pan_afrc_clump_size(format, scan);
+   struct pan_image_block_size layout_sz = pan_afrc_layout_size(modifier);
+
+   return (struct pan_image_block_size){clump_sz.width * layout_sz.width,
+                                        clump_sz.height * layout_sz.height};
+}
+
+static inline unsigned
+pan_afrc_buffer_alignment_from_modifier(uint64_t modifier)
+{
+   switch (modifier & AFRC_FORMAT_MOD_CU_SIZE_MASK) {
+   case AFRC_FORMAT_MOD_CU_SIZE_16:
+      return 1024;
+   case AFRC_FORMAT_MOD_CU_SIZE_24:
+      return 512;
+   case AFRC_FORMAT_MOD_CU_SIZE_32:
+      return 2048;
+   default:
+      UNREACHABLE("invalid coding unit size flag in modifier");
+   };
+}
+
+/*
+ * Determine the number of bytes between rows of paging tiles in an AFRC image
+ */
+static inline uint32_t
+pan_afrc_row_stride(enum pipe_format format, uint64_t modifier, uint32_t width)
+{
+   struct pan_image_block_size tile_size = pan_afrc_tile_size(format, modifier);
+   unsigned block_size = pan_afrc_block_size_from_modifier(modifier);
+
+   return (width / tile_size.width) * block_size * AFRC_CLUMPS_PER_TILE;
+}
+
+#if PAN_ARCH >= 10
+static inline enum mali_afrc_format
+pan_afrc_format(struct pan_afrc_format_info info, uint64_t modifier,
+                unsigned plane)
+{
+   bool scan = pan_afrc_is_scan(modifier);
+
+   assert(info.bpc == 8 || info.bpc == 10);
+   assert(info.num_comps > 0 && info.num_comps <= 4);
+
+   switch (info.ichange_fmt) {
+   case PAN_AFRC_ICHANGE_FORMAT_RAW:
+      assert(plane == 0);
+
+      if (info.bpc == 8)
+         return (scan ? MALI_AFRC_FORMAT_R8_SCAN : MALI_AFRC_FORMAT_R8_ROT) +
+                (info.num_comps - 1);
+
+      assert(info.num_comps == 4);
+      return (scan ? MALI_AFRC_FORMAT_R10G10B10A10_SCAN
+                   : MALI_AFRC_FORMAT_R10G10B10A10_ROT);
+
+#if PAN_ARCH >= 14
+   case PAN_AFRC_ICHANGE_FORMAT_YUV444:
+   case PAN_AFRC_ICHANGE_FORMAT_YUV422:
+   case PAN_AFRC_ICHANGE_FORMAT_YUV420:
+      if (info.bpc == 8) {
+         if (plane == 0 || info.num_planes == 3)
+            return (scan ? MALI_AFRC_FORMAT_R8_SCAN : MALI_AFRC_FORMAT_R8_ROT);
+
+         return (scan ? MALI_AFRC_FORMAT_R8G8_SCAN : MALI_AFRC_FORMAT_R8G8_ROT);
+      }
+
+      if (plane == 0 || info.num_planes == 3)
+         return (scan ? MALI_AFRC_FORMAT_R10_SCAN : MALI_AFRC_FORMAT_R10_ROT);
+
+      assert(info.ichange_fmt == PAN_AFRC_ICHANGE_FORMAT_YUV422 ||
+             info.ichange_fmt == PAN_AFRC_ICHANGE_FORMAT_YUV420);
+      return (scan ? MALI_AFRC_FORMAT_R10G10_SCAN
+                   : MALI_AFRC_FORMAT_R10G10_ROT);
+#else
+   case PAN_AFRC_ICHANGE_FORMAT_YUV444:
+      if (info.bpc == 8) {
+         if (plane == 0 || info.num_planes == 3)
+            return (scan ? MALI_AFRC_FORMAT_R8_444_SCAN
+                         : MALI_AFRC_FORMAT_R8_444_ROT);
+
+         return (scan ? MALI_AFRC_FORMAT_R8G8_444_SCAN
+                      : MALI_AFRC_FORMAT_R8G8_444_ROT);
+      }
+
+      assert(info.num_planes == 3);
+      return (scan ? MALI_AFRC_FORMAT_R10_444_SCAN
+                   : MALI_AFRC_FORMAT_R10_444_ROT);
+
+   case PAN_AFRC_ICHANGE_FORMAT_YUV422:
+      if (info.bpc == 8) {
+         if (plane == 0 || info.num_planes == 3)
+            return (scan ? MALI_AFRC_FORMAT_R8_422_SCAN
+                         : MALI_AFRC_FORMAT_R8_422_ROT);
+
+         return (scan ? MALI_AFRC_FORMAT_R8G8_422_SCAN
+                      : MALI_AFRC_FORMAT_R8G8_422_ROT);
+      }
+
+      if (plane == 0 || info.num_planes == 3)
+         return (scan ? MALI_AFRC_FORMAT_R10_422_SCAN
+                      : MALI_AFRC_FORMAT_R10_422_ROT);
+
+      return (scan ? MALI_AFRC_FORMAT_R10G10_422_SCAN
+                   : MALI_AFRC_FORMAT_R10G10_422_ROT);
+
+   case PAN_AFRC_ICHANGE_FORMAT_YUV420:
+      if (info.bpc == 8) {
+         if (plane == 0 || info.num_planes == 3)
+            return (scan ? MALI_AFRC_FORMAT_R8_420_SCAN
+                         : MALI_AFRC_FORMAT_R8_420_ROT);
+
+         return (scan ? MALI_AFRC_FORMAT_R8G8_420_SCAN
+                      : MALI_AFRC_FORMAT_R8G8_420_ROT);
+      }
+
+      if (plane == 0 || info.num_planes == 3)
+         return (scan ? MALI_AFRC_FORMAT_R10_420_SCAN
+                      : MALI_AFRC_FORMAT_R10_420_ROT);
+
+      return (scan ? MALI_AFRC_FORMAT_R10G10_420_SCAN
+                   : MALI_AFRC_FORMAT_R10G10_420_ROT);
+#endif /* PAN_ARCH >= 14 */
+
+   default:
+      return MALI_AFRC_FORMAT_INVALID;
+   }
+}
+
+static inline enum mali_afrc_block_size
+pan_afrc_block_size(uint64_t modifier, unsigned index)
+{
+   /* Clump size flag for planes 1 and 2 is shifted by 4 bits */
+   unsigned shift = index == 0 ? 0 : 4;
+   uint64_t flag = (modifier >> shift) & AFRC_FORMAT_MOD_CU_SIZE_MASK;
+
+   switch (flag) {
+   case AFRC_FORMAT_MOD_CU_SIZE_16:
+      return MALI_AFRC_BLOCK_SIZE_16;
+   case AFRC_FORMAT_MOD_CU_SIZE_24:
+      return MALI_AFRC_BLOCK_SIZE_24;
+   case AFRC_FORMAT_MOD_CU_SIZE_32:
+      return MALI_AFRC_BLOCK_SIZE_32;
+   default:
+      UNREACHABLE("invalid code unit size");
+   }
+}
+#endif
+
+#ifdef __cplusplus
+} /* extern C */
+#endif
+
+#endif

src/panfrost/lib/pan_blend.h

@@ -0,0 +1,137 @@
+/*
+ * Copyright (C) 2018 Alyssa Rosenzweig
+ * Copyright (C) 2019-2021 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef __PAN_BLEND_H__
+#define __PAN_BLEND_H__
+
+#include "genxml/gen_macros.h"
+
+#include "compiler/nir/nir.h"
+#include "util/blend.h"
+#include "util/format/u_format.h"
+
+struct MALI_BLEND_EQUATION;
+
+struct pan_blend_equation {
+   unsigned blend_enable                  : 1;
+   unsigned is_float                      : 1;
+   enum pipe_blend_func rgb_func          : 3;
+   enum pipe_blendfactor rgb_src_factor   : 5;
+   enum pipe_blendfactor rgb_dst_factor   : 5;
+   enum pipe_blend_func alpha_func        : 3;
+   enum pipe_blendfactor alpha_src_factor : 5;
+   enum pipe_blendfactor alpha_dst_factor : 5;
+   unsigned color_mask                    : 4;
+};
+
+struct pan_blend_rt_state {
+   /* RT format */
+   enum pipe_format format;
+
+   /* Number of samples */
+   unsigned nr_samples;
+
+   struct pan_blend_equation equation;
+};
+
+struct pan_blend_state {
+   bool alpha_to_one;
+   bool logicop_enable;
+   enum pipe_logicop logicop_func;
+   float constants[4];
+   unsigned rt_count;
+   struct pan_blend_rt_state rts[8];
+};
+
+struct pan_blend_shader_key {
+   enum pipe_format format;
+   nir_alu_type src0_type, src1_type;
+   uint32_t rt             : 3;
+   uint32_t logicop_enable : 1;
+   uint32_t logicop_func   : 4;
+   uint32_t nr_samples     : 5;
+   uint32_t alpha_to_one   : 1;
+   uint32_t padding        : 17;
+   struct pan_blend_equation equation;
+};
+
+bool pan_blend_reads_dest(const struct pan_blend_equation eq);
+
+bool pan_blend_can_fixed_function(unsigned arch,
+                                  const struct pan_blend_equation equation,
+                                  bool supports_2src);
+
+bool pan_blend_is_opaque(const struct pan_blend_equation eq);
+
+bool pan_blend_alpha_zero_nop(const struct pan_blend_equation eq);
+
+bool pan_blend_alpha_one_store(const struct pan_blend_equation eq);
+
+unsigned pan_blend_constant_mask(const struct pan_blend_equation eq);
+
+void pan_blend_optimize_equation(struct pan_blend_equation *eq,
+                                 enum pipe_format format,
+                                 const float *constants);
+
+/* Fixed-function blending only supports a single constant, so if multiple bits
+ * are set in constant_mask, the constants must match. Therefore we may pick
+ * just the first constant. */
+
+static inline float
+pan_blend_get_constant(unsigned mask, const float *constants)
+{
+   return mask ? constants[ffs(mask) - 1] : 0.0;
+}
+
+/* v6 doesn't support blend constants in FF blend equations whatsoever, and v7
+ * only uses the constant from RT 0 (TODO: what if it's the same constant? or a
+ * constant is shared?) */
+
+static inline bool
+pan_blend_supports_constant(unsigned arch, unsigned rt)
+{
+   return !((arch == 6) || (arch == 7 && rt > 0));
+}
+
+/* The SOURCE_2 value is new in Bifrost */
+
+static inline bool
+pan_blend_supports_2src(unsigned arch)
+{
+   return (arch >= 6);
+}
+
+bool pan_blend_is_homogenous_constant(unsigned mask, const float *constants);
+
+uint16_t pan_pack_blend_constant(enum pipe_format format, float cons);
+
+void pan_blend_to_fixed_function_equation(const struct pan_blend_equation eq,
+                                          struct MALI_BLEND_EQUATION *equation);
+
+enum mali_register_file_format pan_blend_type_from_nir(nir_alu_type nir_type);
+uint32_t pan_pack_blend(const struct pan_blend_equation equation);
+
+#ifdef PAN_ARCH
+
+enum pipe_format GENX(pan_blend_shader_fmt)(enum pipe_format format);
+
+nir_shader *GENX(pan_blend_create_shader)(const struct pan_blend_state *state,
+                                          nir_alu_type src0_type,
+                                          nir_alu_type src1_type, unsigned rt);
+
+#if PAN_ARCH >= 6
+uint64_t GENX(pan_blend_get_internal_desc)(enum pipe_format fmt, unsigned rt,
+                                           unsigned force_size, bool dithered);
+
+#if PAN_ARCH < 9
+enum mali_register_file_format
+   GENX(pan_fixup_blend_type)(nir_alu_type T_size, enum pipe_format format);
+#endif
+#endif
+
+#endif
+
+#endif

src/panfrost/lib/pan_buffer.c

@@ -0,0 +1,120 @@
+/*
+ * Copyright (C) 2026 Arm Ltd.
+ *
+ * Derived from pan_texture.c which is:
+ * Copyright (C) 2008 VMware, Inc.
+ * Copyright (C) 2014 Broadcom
+ * Copyright (C) 2018-2019 Alyssa Rosenzweig
+ * Copyright (C) 2019-2020 Collabora, Ltd.
+ * Copyright (C) 2024 Arm Ltd.
+ *
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "pan_buffer.h"
+#include "pan_desc.h"
+#include "pan_format.h"
+#include "pan_texture.h"
+#include "pan_util.h"
+
+#if PAN_ARCH >= 9
+
+void
+GENX(pan_buffer_texture_emit)(const struct pan_buffer_view *bview,
+                              struct mali_buffer_packed *out)
+{
+   unsigned stride = util_format_get_blocksize(bview->format);
+   struct MALI_INTERNAL_CONVERSION conv = {
+      .memory_format = GENX(pan_format_from_pipe_format)(bview->format)->hw,
+      .raw = false,
+   };
+
+   uint64_t buffer_size = bview->width_el * stride;
+
+   pan_pack(out, BUFFER, cfg) {
+      cfg.type = MALI_DESCRIPTOR_TYPE_BUFFER;
+      cfg.buffer_type = MALI_BUFFER_TYPE_STRUCTURE;
+
+#if PAN_ARCH >= 11
+      cfg.size = buffer_size & BITFIELD_MASK(32);
+      cfg.size_hi = buffer_size >> 32;
+#else
+      cfg.size = buffer_size;
+#endif
+
+      cfg.address = bview->base;
+      cfg.stride = stride;
+      cfg.conversion = conv;
+   }
+}
+
+#elif PAN_ARCH >= 6
+
+void
+GENX(pan_buffer_texture_emit)(const struct pan_buffer_view *bview,
+                              struct mali_attribute_buffer_packed *out_buf,
+                              struct mali_attribute_packed *out_attrib)
+{
+   unsigned stride = util_format_get_blocksize(bview->format);
+   uint32_t hw_fmt = GENX(pan_format_from_pipe_format)(bview->format)->hw;
+
+   /* The base address has shr(6) applied so we need to use the offset of the
+    * attribute to get the last 6 bits.
+    */
+   uint64_t base = bview->base & ~0x3f;
+   uint32_t offset = bview->base & 0x3f;
+
+   pan_pack(out_buf, ATTRIBUTE_BUFFER, cfg) {
+      cfg.type = MALI_ATTRIBUTE_TYPE_1D;
+      cfg.pointer = base;
+      cfg.stride = stride;
+      /* The ATTRIBUTE_BUFFER.size specifies the size of the buffer starting
+       * at ATTRIBUTE_BUFFER.pointer and and ATTRIBUTE.offset specifies an
+       * offset into that window. If we're going to use it to offset the base
+       * of the buffer, we need to increase the size as well.
+       */
+      cfg.size = bview->width_el * stride + offset;
+   }
+
+   pan_pack(out_attrib, ATTRIBUTE, cfg) {
+      cfg.format = hw_fmt;
+      cfg.offset = offset;
+      cfg.offset_enable = offset != 0;
+   }
+}
+
+#else
+
+void
+GENX(pan_buffer_texture_emit)(const struct pan_buffer_view *bview,
+                              struct mali_texture_packed *out,
+                              const struct pan_ptr *payload)
+{
+   uint32_t mali_format = GENX(pan_format_from_pipe_format)(bview->format)->hw;
+   static const unsigned char rgba_swizzle[4] = {
+      PIPE_SWIZZLE_X,
+      PIPE_SWIZZLE_Y,
+      PIPE_SWIZZLE_Z,
+      PIPE_SWIZZLE_W,
+   };
+
+   pan_cast_and_pack(payload->cpu, SURFACE_WITH_STRIDE, cfg) {
+      cfg.pointer = bview->base;
+      cfg.row_stride = 0;
+      cfg.surface_stride = 0;
+   }
+
+   pan_pack(out, TEXTURE, cfg) {
+      cfg.dimension = MALI_TEXTURE_DIMENSION_1D;
+      cfg.format = mali_format;
+      cfg.width = bview->width_el;
+      cfg.height = 1;
+      cfg.sample_count = 1;
+      cfg.swizzle = pan_translate_swizzle_4(rgba_swizzle);
+      cfg.texel_ordering = MALI_TEXTURE_LAYOUT_LINEAR;
+      cfg.levels = 1;
+      cfg.array_size = 1;
+   }
+}
+
+#endif

src/panfrost/lib/pan_buffer.h

@@ -0,0 +1,43 @@
+/*
+ * Copyright (C) 2026 Arm Ltd.
+ *
+ * Derived from pan_texture.h which is:
+ * Copyright (C) 2008 VMware, Inc.
+ * Copyright (C) 2014 Broadcom
+ * Copyright (C) 2018-2019 Alyssa Rosenzweig
+ * Copyright (C) 2019-2020 Collabora, Ltd.
+ * Copyright (C) 2025 Arm Ltd.
+ *
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef __PAN_BUFFER_H
+#define __PAN_BUFFER_H
+
+#ifndef PAN_ARCH
+#error "PAN_ARCH must be defined"
+#endif
+
+#include "genxml/gen_macros.h"
+
+struct pan_buffer_view;
+struct mali_buffer_packed;
+struct mali_attribute_buffer_packed;
+struct mali_attribute_packed;
+struct mali_texture_packed;
+struct pan_ptr;
+
+#if PAN_ARCH >= 9
+void GENX(pan_buffer_texture_emit)(const struct pan_buffer_view *bview,
+                                   struct mali_buffer_packed *out);
+#elif PAN_ARCH >= 6
+void GENX(pan_buffer_texture_emit)(const struct pan_buffer_view *bview,
+                                   struct mali_attribute_buffer_packed *out_buf,
+                                   struct mali_attribute_packed *out_attrib);
+#else
+void GENX(pan_buffer_texture_emit)(const struct pan_buffer_view *bview,
+                                   struct mali_texture_packed *out,
+                                   const struct pan_ptr *payload);
+#endif
+
+#endif

src/panfrost/lib/pan_clear.c

@@ -0,0 +1,153 @@
+/*
+ * Copyright (C) 2019-2021 Collabora, Ltd.
+ * Copyright (C) 2019 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "genxml/gen_macros.h"
+
+#include <string.h>
+#include "util/format_srgb.h"
+#include "util/rounding.h"
+#include "util/u_pack_color.h"
+#include "pan_format.h"
+#include "pan_util.h"
+
+/* Clear colours are packed as the internal format of the tilebuffer, looked up
+ * in the blendable formats table given the render target format.
+ *
+ * Raw formats may emulate arbitrary formats with blend shaders. For these, we
+ * defer to util_pack_colour to pack in the API format.
+ *
+ * Blendable formats, on the other hand, include extra "fractional" bits in the
+ * tilebuffer for dithering. These have a packed fixed-point representation:
+ * for a channel with m integer bits and n fractional bits, multiply by ((2^m)
+ * - 1) * 2^n and round to the nearest even.
+ */
+
+/* Replicate a 32-bit value to fill 128-bit */
+
+static void
+pan_pack_color_32(uint32_t *packed, uint32_t v)
+{
+   for (unsigned i = 0; i < 4; ++i)
+      packed[i] = v;
+}
+
+/* For m integer bits and n fractional bits, calculate the conversion factor,
+ * multiply the source value, and convert to integer rounding to even. When
+ * dithering, the fractional bits are used. When not dithered, only the integer
+ * bits are used and the fractional bits must remain zero. */
+
+static inline uint32_t
+float_to_fixed(float f, unsigned bits_int, unsigned bits_frac, bool dither)
+{
+   uint32_t m = (1 << bits_int) - 1;
+
+   if (dither) {
+      float factor = m << bits_frac;
+      return _mesa_roundevenf(f * factor);
+   } else {
+      uint32_t v = _mesa_roundevenf(f * (float)m);
+      return v << bits_frac;
+   }
+}
+
+struct mali_tib_layout {
+   unsigned int_r, frac_r;
+   unsigned int_g, frac_g;
+   unsigned int_b, frac_b;
+   unsigned int_a, frac_a;
+};
+
+/* clang-format off */
+static const struct mali_tib_layout tib_layouts[] = {
+   [MALI_COLOR_BUFFER_INTERNAL_FORMAT_R8G8B8A8]    = {  8, 0,  8, 0,  8, 0, 8, 0 },
+   [MALI_COLOR_BUFFER_INTERNAL_FORMAT_R10G10B10A2] = { 10, 0, 10, 0, 10, 0, 2, 0 },
+   [MALI_COLOR_BUFFER_INTERNAL_FORMAT_R8G8B8A2]    = {  8, 2,  8, 2,  8, 2, 2, 0 },
+   [MALI_COLOR_BUFFER_INTERNAL_FORMAT_R4G4B4A4]    = {  4, 4,  4, 4,  4, 4, 4, 4 },
+   [MALI_COLOR_BUFFER_INTERNAL_FORMAT_R5G6B5A0]    = {  5, 5,  6, 4,  5, 5, 0, 2 },
+   [MALI_COLOR_BUFFER_INTERNAL_FORMAT_R5G5B5A1]    = {  5, 5,  5, 5,  5, 5, 1, 1 },
+};
+/* clang-format on */
+
+/* Raw values are stored as-is but replicated for multisampling */
+
+static void
+pan_pack_raw(uint32_t *packed, const union pipe_color_union *color,
+             enum pipe_format format)
+{
+   union util_color out = {0};
+   unsigned size = util_format_get_blocksize(format);
+   assert(size <= 16);
+
+   util_pack_color(color->f, format, &out);
+
+   if (size == 1) {
+      unsigned b = out.ui[0] & 0xff;
+      unsigned w = b | (b << 8);
+      pan_pack_color_32(packed, w | (w << 16));
+   } else if (size == 2) {
+      unsigned w = out.ui[0] & 0xffff;
+      pan_pack_color_32(packed, w | (w << 16));
+   } else if (size <= 4) {
+      pan_pack_color_32(packed, out.ui[0]);
+   } else if (size <= 8) {
+      memcpy(packed + 0, out.ui, 8);
+      memcpy(packed + 2, out.ui, 8);
+   } else {
+      memcpy(packed, out.ui, 16);
+   }
+}
+
+void
+pan_pack_color(const struct pan_blendable_format *blendable_formats,
+               uint32_t *packed, const union pipe_color_union *color,
+               enum pipe_format format, bool dithered)
+{
+   enum mali_color_buffer_internal_format internal =
+      blendable_formats[format].internal;
+
+   if (util_format_is_float(format) ||
+       internal == MALI_COLOR_BUFFER_INTERNAL_FORMAT_RAW_VALUE) {
+      pan_pack_raw(packed, color, format);
+      return;
+   }
+
+   /* Saturate to [0, 1] by definition of UNORM. Prevents overflow. */
+   float r = SATURATE(color->f[0]);
+   float g = SATURATE(color->f[1]);
+   float b = SATURATE(color->f[2]);
+   float a = SATURATE(color->f[3]);
+
+   /* Fill in alpha = 1.0 by default */
+   if (!util_format_has_alpha(format))
+      a = 1.0;
+
+   /* Convert colourspace while we still have floats */
+   if (util_format_is_srgb(format)) {
+      r = util_format_linear_to_srgb_float(r);
+      g = util_format_linear_to_srgb_float(g);
+      b = util_format_linear_to_srgb_float(b);
+   }
+
+   /* Look up the layout of the tilebuffer */
+   assert(internal < ARRAY_SIZE(tib_layouts));
+   struct mali_tib_layout l = tib_layouts[internal];
+
+   unsigned count_r = l.int_r + l.frac_r;
+   unsigned count_g = l.int_g + l.frac_g + count_r;
+   unsigned count_b = l.int_b + l.frac_b + count_g;
+   ASSERTED unsigned count_a = l.int_a + l.frac_a + count_b;
+
+   /* Must fill the word */
+   assert(count_a == 32);
+
+   /* Convert the transformed float colour to the given layout */
+   uint32_t ur = float_to_fixed(r, l.int_r, l.frac_r, dithered) << 0;
+   uint32_t ug = float_to_fixed(g, l.int_g, l.frac_g, dithered) << count_r;
+   uint32_t ub = float_to_fixed(b, l.int_b, l.frac_b, dithered) << count_g;
+   uint32_t ua = float_to_fixed(a, l.int_a, l.frac_a, dithered) << count_b;
+
+   pan_pack_color_32(packed, ur | ug | ub | ua);
+}

src/panfrost/lib/pan_desc.h

@@ -0,0 +1,417 @@
+/*
+ * Copyright (C) 2021 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef __PAN_DESC_H
+#define __PAN_DESC_H
+
+#include "genxml/gen_macros.h"
+
+#include "kmod/pan_kmod.h"
+#include "pan_image.h"
+#include "pan_pool.h"
+
+struct pan_buffer_view {
+   enum pipe_format format;
+   struct {
+      unsigned narrow;
+      unsigned hdr;
+   } astc;
+   unsigned width_el;
+   uint64_t base;
+};
+
+struct pan_compute_dim {
+   uint32_t x, y, z;
+};
+
+struct pan_fb_color_attachment {
+   const struct pan_image_view *view;
+   bool *crc_valid;
+   bool clear;
+   bool preload;
+   bool discard;
+   uint32_t clear_value[4];
+};
+
+struct pan_fb_zs_attachment {
+   struct {
+      const struct pan_image_view *zs, *s;
+   } view;
+
+   struct {
+      bool z, s;
+   } clear;
+
+   struct {
+      bool z, s;
+   } discard;
+
+   struct {
+      bool z, s;
+   } preload;
+
+   struct {
+      float depth;
+      uint8_t stencil;
+   } clear_value;
+};
+
+struct pan_tiler_context {
+   union {
+      struct {
+         uint64_t desc;
+         /* A tiler descriptor can only handle a limited amount of layers.
+          * If the number of layers is bigger than this, several tiler
+          * descriptors will be issued, each with a different layer_offset.
+          *
+          * Note: on v14+, only up to one tiler descriptor is supported and
+          * layer_offset is deprecated.
+          */
+         uint8_t layer_offset;
+      } valhall;
+      struct {
+         uint64_t desc;
+      } bifrost;
+      struct {
+         /* Sum of vertex counts (for non-indexed draws), index counts, or ~0 if
+          * any indirect draws are used. Helps tune hierarchy masks.
+          */
+         uint32_t vertex_count;
+         bool disable;
+         bool no_hierarchical_tiling;
+         uint64_t polygon_list;
+         struct {
+            uint64_t start;
+            unsigned size;
+         } heap;
+      } midgard;
+   };
+};
+
+struct pan_tls_info {
+   struct {
+      uint64_t ptr;
+      unsigned size;
+   } tls;
+
+   struct {
+      unsigned instances;
+      uint64_t ptr;
+      unsigned size;
+   } wls;
+};
+
+struct pan_fb_bifrost_info {
+   struct {
+      struct pan_ptr dcds;
+      unsigned modes[3];
+   } pre_post;
+};
+
+struct pan_bbox {
+   unsigned minx, miny, maxx, maxy;
+};
+
+struct pan_fb_info {
+   unsigned width, height;
+   /* Draw-extent controlled by viewports/scissors.
+    * Max values are exclusive */
+   struct pan_bbox draw_extent;
+   /* frame_bounding_box controls the bounding box in the framebuffer
+    * descriptor for the entire pass. This is being controlled by the
+    * renderArea of a renderpass in Vulkan. On GL, this covers the
+    * entire frame. Max values are exclusive. */
+   struct pan_bbox frame_bounding_box;
+   unsigned nr_samples;
+   unsigned force_samples; /* samples used for rasterization */
+   unsigned rt_count;
+   struct pan_fb_color_attachment rts[8];
+   struct pan_fb_zs_attachment zs;
+
+   struct {
+      unsigned stride;
+      uint64_t base;
+   } tile_map;
+
+   union {
+      struct pan_fb_bifrost_info bifrost;
+   };
+
+   /* Optimal tile buffer size. */
+   unsigned tile_buf_budget;
+   unsigned z_tile_buf_budget;
+   unsigned tile_size;
+   unsigned cbuf_allocation;
+
+   /* Sample position array. */
+   uint64_t sample_positions;
+
+   /* Only used on Valhall */
+   bool sprite_coord_origin;
+   bool first_provoking_vertex;
+   bool allow_hsr_prepass;
+
+   /* indicates whether pixel local storage is enabled */
+   bool pls_enabled;
+};
+
+struct pan_clean_tile {
+   /* clean_tile_write_enable mask on the 8 color attachments. */
+   uint8_t write_rt_mask;
+
+   /* clean_tile_write_enable flag on the depth/stencil attachment. */
+   uint8_t write_zs : 1;
+};
+
+static inline bool
+pan_clean_tile_write_rt_enabled(struct pan_clean_tile clean_tile,
+                                unsigned index)
+{
+   return (clean_tile.write_rt_mask >> index) & 1;
+}
+
+static inline bool
+pan_clean_tile_write_zs_enabled(struct pan_clean_tile clean_tile)
+{
+   return clean_tile.write_zs;
+}
+
+static inline bool
+pan_clean_tile_write_any_set(struct pan_clean_tile clean_tile)
+{
+   return clean_tile.write_rt_mask || clean_tile.write_zs;
+}
+
+static unsigned
+pan_zsbuf_bytes_per_pixel(const struct pan_fb_info *fb)
+{
+   unsigned samples = fb->nr_samples;
+
+   const struct pan_image_view *zs_view = fb->zs.view.zs;
+   if (zs_view)
+      samples = zs_view->nr_samples;
+
+   const struct pan_image_view *s_view = fb->zs.view.s;
+   if (s_view)
+      samples = MAX2(samples, s_view->nr_samples);
+
+   /* Depth is always stored in a 32-bit float. Stencil requires depth to
+    * be allocated, but doesn't have it's own budget; it's tied to the
+    * depth buffer.
+    */
+   return sizeof(float) * samples;
+}
+
+static inline unsigned
+pan_wls_instances(const struct pan_compute_dim *dim)
+{
+   return util_next_power_of_two(dim->x) * util_next_power_of_two(dim->y) *
+          util_next_power_of_two(dim->z);
+}
+
+static inline unsigned
+pan_wls_adjust_size(unsigned wls_size)
+{
+   return util_next_power_of_two(MAX2(wls_size, 128));
+}
+
+static inline unsigned
+pan_calc_workgroups_per_task(const struct pan_compute_dim *shader_local_size,
+                             const struct pan_kmod_dev_props *props)
+{
+   /* Each shader core can run N tasks and a total of M threads at any single
+    * time, thus each task should ideally have no more than M/N threads. */
+   unsigned max_threads_per_task =
+      props->max_threads_per_core / props->max_tasks_per_core;
+
+   /* To achieve the best utilization, we should aim for as many workgroups
+    * per tasks as we can fit without exceeding the above thread limit */
+   unsigned threads_per_wg =
+      shader_local_size->x * shader_local_size->y * shader_local_size->z;
+   assert(threads_per_wg > 0 && threads_per_wg <= props->max_threads_per_wg);
+   unsigned wg_per_task = DIV_ROUND_UP(max_threads_per_task, threads_per_wg);
+   assert(wg_per_task > 0 && wg_per_task <= max_threads_per_task);
+
+   return wg_per_task;
+}
+
+static inline unsigned
+pan_calc_wls_instances(const struct pan_compute_dim *shader_local_size,
+                       const struct pan_kmod_dev_props *props,
+                       const struct pan_compute_dim *dim)
+{
+   /* NOTE: If the instance count is lower than the number of workgroups
+    * being dispatched, the HW will hold back workgroups until instances
+    * can be reused. */
+   unsigned instances;
+   unsigned wg_per_task =
+      pan_calc_workgroups_per_task(shader_local_size, props);
+   unsigned max_instances_per_core =
+      util_next_power_of_two(wg_per_task * props->max_tasks_per_core);
+
+   /* Not passing workgroup dimensions implies indirect compute. */
+   if (!dim) {
+      /* Assume we utilize all shader cores to the max */
+      instances = max_instances_per_core;
+   } else {
+      /* NOTE: There is no benefit from allocating more instances than what
+       * can concurrently be used by the HW */
+      instances = MIN2(pan_wls_instances(dim), max_instances_per_core);
+   }
+   return instances;
+}
+
+static inline unsigned
+pan_calc_total_wls_size(unsigned wls_size, unsigned wls_instances,
+                        unsigned max_core_id_plus_one)
+{
+   unsigned size = pan_wls_adjust_size(wls_size);
+
+   return size * wls_instances * max_core_id_plus_one;
+}
+
+#ifdef PAN_ARCH
+
+static inline enum mali_sample_pattern
+pan_sample_pattern(unsigned samples)
+{
+   switch (samples) {
+   case 1:
+      return MALI_SAMPLE_PATTERN_SINGLE_SAMPLED;
+#if PAN_ARCH >= 12
+   case 2:
+      return MALI_SAMPLE_PATTERN_ROTATED_2X_GRID;
+#endif
+   case 4:
+      return MALI_SAMPLE_PATTERN_ROTATED_4X_GRID;
+   case 8:
+      return MALI_SAMPLE_PATTERN_D3D_8X_GRID;
+#if PAN_ARCH >= 5
+   case 16:
+      return MALI_SAMPLE_PATTERN_D3D_16X_GRID;
+#endif
+   default:
+      UNREACHABLE("Unsupported sample count");
+   }
+}
+
+static inline struct pan_image_block_size
+pan_effective_tile_block_size(unsigned tile_size)
+{
+   /* Tile is either a square or a rect whose width is twice the height. */
+   unsigned shift_h = util_logbase2(tile_size);
+   unsigned shift_w = shift_h + 1;
+   unsigned h = 1 << (shift_h >> 1);
+   unsigned w = 1 << (shift_w >> 1);
+
+   return (struct pan_image_block_size){w, h};
+}
+
+#if PAN_ARCH >= 6
+/* All GPUs starting from Bifrost are affected by issue TSIX-2033:
+ *
+ *      Forcing clean_tile_writes breaks INTERSECT readbacks
+ *
+ * To workaround, use the pre-frame shader mode ALWAYS instead of INTERSECT if
+ * clean_tile_write_enable is set on either one of the color, depth or stencil
+ * buffers. Since INTERSECT is a hint that the hardware may ignore, this
+ * cannot affect correctness, only performance. */
+
+static enum mali_pre_post_frame_shader_mode
+pan_fix_frame_shader_mode(enum mali_pre_post_frame_shader_mode mode,
+                          bool force_clean_tile)
+{
+   if (force_clean_tile && mode == MALI_PRE_POST_FRAME_SHADER_MODE_INTERSECT)
+      return MALI_PRE_POST_FRAME_SHADER_MODE_ALWAYS;
+   else
+      return mode;
+}
+#endif
+
+void GENX(pan_select_tile_size)(struct pan_fb_info *fb);
+
+bool GENX(pan_force_clean_write_on)(const struct pan_image *image,
+                                    unsigned fb_tile_size_px);
+
+struct pan_clean_tile
+   GENX(pan_get_clean_tile_info)(const struct pan_fb_info *fb);
+
+void GENX(pan_emit_tls)(const struct pan_tls_info *info,
+                        struct mali_local_storage_packed *out);
+
+int GENX(pan_select_crc_rt)(const struct pan_fb_info *fb, unsigned tile_size);
+
+struct pan_attachment_info {
+   const struct pan_image_view *iview;
+   unsigned layer_or_z_slice;
+   unsigned fb_tile_size_px;
+};
+
+#if PAN_ARCH >= 5
+void GENX(pan_emit_default_color_attachment)(enum pipe_format format,
+                                             void *payload);
+void GENX(pan_emit_linear_color_attachment)(const struct pan_attachment_info *att,
+                                            void *payload);
+void GENX(pan_emit_linear_s_attachment)(const struct pan_attachment_info *att,
+                                        void *payload);
+void GENX(pan_emit_linear_zs_attachment)(const struct pan_attachment_info *att,
+                                         void *payload);
+void GENX(pan_emit_u_tiled_color_attachment)(const struct pan_attachment_info *att,
+                                             void *payload);
+void GENX(pan_emit_u_tiled_s_attachment)(const struct pan_attachment_info *att,
+                                         void *payload);
+void GENX(pan_emit_u_tiled_zs_attachment)(const struct pan_attachment_info *att,
+                                          void *payload);
+void GENX(pan_emit_afbc_color_attachment)(const struct pan_attachment_info *att,
+                                          void *payload);
+void GENX(pan_emit_afbc_zs_attachment)(const struct pan_attachment_info *att,
+                                       void *payload);
+void GENX(pan_emit_afbc_s_attachment)(const struct pan_attachment_info *att,
+                                      void *payload);
+#endif
+
+#if PAN_ARCH >= 10
+void
+GENX(pan_emit_interleaved_64k_color_attachment)(const struct pan_attachment_info *att,
+                                                void *payload);
+void GENX(pan_emit_interleaved_64k_zs_attachment)(const struct pan_attachment_info *att,
+                                                  void *payload);
+void GENX(pan_emit_interleaved_64k_s_attachment)(const struct pan_attachment_info *att,
+                                                 void *payload);
+
+void GENX(pan_emit_afrc_color_attachment)(const struct pan_attachment_info *att,
+                                          void *payload);
+#endif
+
+struct pan_fbd_descs {
+#if PAN_ARCH <= 13
+   struct mali_framebuffer_packed *fbd;
+#endif
+#if PAN_ARCH >= 5
+   struct mali_zs_crc_extension_packed *zs_crc;
+   struct mali_render_target_packed *rts;
+#endif
+};
+
+unsigned GENX(pan_emit_fbd)(const struct pan_fb_info *fb, unsigned layer_idx,
+                            const struct pan_tls_info *tls,
+                            const struct pan_tiler_context *tiler_ctx,
+                            const struct pan_fbd_descs *out);
+
+#if PAN_ARCH >= 6
+unsigned GENX(pan_select_tiler_hierarchy_mask)(uint32_t width, uint32_t height,
+                                               uint32_t max_levels,
+                                               uint32_t tile_size,
+                                               uint32_t mem_budget);
+#endif
+
+#if PAN_ARCH <= 9
+void GENX(pan_emit_fragment_job_payload)(const struct pan_fb_info *fb,
+                                         uint64_t fbd, void *out);
+#endif
+
+#endif /* ifdef PAN_ARCH */
+
+#endif

src/panfrost/lib/pan_earlyzs.c

@@ -0,0 +1,148 @@
+/*
+ * Copyright (C) 2022 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "genxml/gen_macros.h"
+
+#include "pan_earlyzs.h"
+#include "panfrost/compiler/pan_compiler.h"
+
+
+/*
+ * Return an "early" mode. If it is known that the depth/stencil tests always
+ * pass (so the shader is always executed), weak early is usually faster than
+ * force early.
+ */
+static enum mali_pixel_kill
+best_early_mode(bool zs_always_passes, bool force_early)
+{
+   if (zs_always_passes && !force_early)
+      return MALI_PIXEL_KILL_WEAK_EARLY;
+   else
+      return MALI_PIXEL_KILL_FORCE_EARLY;
+}
+
+/*
+ * Analyze a fragment shader and provided API state to determine the early-ZS
+ * configuration. The order of arguments must match the order of states in the
+ * lookup table, synchronized with pan_earlyzs_get.
+ */
+static struct pan_earlyzs_state
+analyze(const struct pan_shader_info *s, bool writes_zs_or_oq,
+        bool alpha_to_coverage, bool zs_always_passes,
+        enum pan_earlyzs_zs_tilebuf_read zs_read,
+        bool should_force_early_update)
+{
+   /* If the shader writes depth or stencil, all depth/stencil tests must
+    * be deferred until the value is known after the ZS_EMIT instruction,
+    * if present. ZS_EMIT must precede ATEST, so the value is known when
+    * ATEST executes, justifying the late test/update.
+    * Also, if alpha_to_coverage is set that also forces a late update.
+    * NOTE: it's not at all clear why alpha_to_coverage always requires
+    * a late update; the late update should only really be required if
+    * we're writing z or stencil, or testing for occlusion queries.
+    * The docs are somewhat contradictory on this point.
+    * But empirically we observe this requirement on Valhall, and doing
+    * the update later is never wrong (just potentially a bit slower).
+    */
+   bool shader_writes_zs = (s->fs.writes_depth || s->fs.writes_stencil);
+   bool late_update = shader_writes_zs || alpha_to_coverage;
+   bool late_kill = shader_writes_zs;
+   bool force_early_update =
+      s->fs.early_fragment_tests || should_force_early_update;
+   bool force_early_kill = s->fs.early_fragment_tests;
+
+   /* Late coverage updates are required if the coverage mask depends on
+    * the results of the shader. Discards are implemented as coverage mask
+    * updates and must be considered. Strictly, depth/stencil writes may
+    * also update the coverage mask, but these already force late updates.
+    */
+   bool late_coverage =
+      s->fs.writes_coverage || s->fs.can_discard || alpha_to_coverage;
+
+   /* Late coverage mask updates may affect the value written to the
+    * depth/stencil buffer (if a pixel is discarded entirely). However,
+    * they do not affect depth/stencil testing. So they may only matter if
+    * depth or stencil is written.
+    *
+    * That dependency does mean late coverage mask updates require late
+    * depth/stencil updates.
+    *
+    * Similarly, occlusion queries count samples that pass the
+    * depth/stencil tests, so occlusion queries with late coverage also
+    * require a late update.
+    */
+   late_update |= (late_coverage && writes_zs_or_oq);
+
+   /* Side effects require late depth/stencil tests to ensure the shader
+    * isn't killed before the side effects execute.
+    */
+   late_kill |= s->writes_global;
+
+   /* Shader reads require late depth/stencil tests to ensure the shader
+    * isn't killed before the side effects execute, unless the HW supports
+    * read-only ZS optimization, in which case it can be lowered to
+    * force-early. */
+   bool optimize_shader_read_only_zs = false;
+   if (zs_read != PAN_EARLYZS_ZS_TILEBUF_NOT_READ) {
+      if (!late_update && zs_read == PAN_EARLYZS_ZS_TILEBUF_READ_OPT) {
+         optimize_shader_read_only_zs = true;
+         force_early_update |= true;
+      } else {
+         late_update |= true;
+      }
+
+      if (!late_kill && zs_read == PAN_EARLYZS_ZS_TILEBUF_READ_OPT) {
+         optimize_shader_read_only_zs = true;
+         force_early_kill |= true;
+      }
+   }
+
+   /* Finally, the shader may override and force early fragment tests */
+   late_update &= !s->fs.early_fragment_tests;
+   late_kill &= !s->fs.early_fragment_tests;
+
+   /* Collect results */
+   return (struct pan_earlyzs_state){
+      .update = late_update
+                   ? MALI_PIXEL_KILL_FORCE_LATE
+                   : best_early_mode(zs_always_passes, force_early_update),
+      .kill = late_kill ? MALI_PIXEL_KILL_FORCE_LATE
+                        : best_early_mode(zs_always_passes, force_early_kill),
+      .shader_readonly_zs = optimize_shader_read_only_zs,
+   };
+}
+
+/*
+ * Analyze a fragment shader to determine all possible early-ZS configurations.
+ * Returns a lookup table of configurations indexed by the API state.
+ */
+struct pan_earlyzs_lut
+pan_earlyzs_analyze(const struct pan_shader_info *s, unsigned arch)
+{
+   /* On v11+, update operation cannot be weak early */
+   bool should_force_early_update = arch >= 11;
+
+   struct pan_earlyzs_lut lut;
+
+   for (unsigned v0 = 0; v0 < 2; ++v0) {
+      for (unsigned v1 = 0; v1 < 2; ++v1) {
+         for (unsigned v2 = 0; v2 < 2; ++v2) {
+            for (unsigned v3 = 0; v3 < PAN_EARLYZS_ZS_TILEBUF_MODE_COUNT;
+                 ++v3) {
+               enum pan_earlyzs_zs_tilebuf_read zs_read = v3;
+
+               /* Shader read-only ZS optimization only exists on v10. */
+               if (arch != 10 && v3 == PAN_EARLYZS_ZS_TILEBUF_READ_OPT)
+                  zs_read = PAN_EARLYZS_ZS_TILEBUF_READ_NO_OPT;
+
+               lut.states[v0][v1][v2][v3] =
+                  analyze(s, v0, v1, v2, zs_read, should_force_early_update);
+            }
+         }
+      }
+   }
+
+   return lut;
+}

src/panfrost/lib/pan_earlyzs.h

@@ -0,0 +1,79 @@
+/*
+ * Copyright (C) 2022 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef __PAN_EARLYZS_H__
+#define __PAN_EARLYZS_H__
+
+#include <stdbool.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Early-ZS pair. */
+struct pan_earlyzs_state {
+   /* Z/S test and update */
+   unsigned update : 2;
+
+   /* Pixel kill */
+   unsigned kill : 2;
+
+   /* True if the shader read-only ZS optimization should be enabled */
+   bool shader_readonly_zs : 1;
+
+   /* So it fits in a byte */
+   unsigned padding : 3;
+};
+
+/* ZS tilebuf read access */
+enum pan_earlyzs_zs_tilebuf_read {
+   /* The ZS tile buffer is not read */
+   PAN_EARLYZS_ZS_TILEBUF_NOT_READ = 0,
+
+   /* The ZS tile buffer is read but the read-only optimization must be
+    * disabled
+    */
+   PAN_EARLYZS_ZS_TILEBUF_READ_NO_OPT,
+
+   /* The ZS tile buffer is read and the read-only optimization can be
+    * enabled
+    */
+   PAN_EARLYZS_ZS_TILEBUF_READ_OPT,
+
+   /* Number of ZS read modes */
+   PAN_EARLYZS_ZS_TILEBUF_MODE_COUNT,
+};
+
+/* Internal lookup table. Users should treat as an opaque structure and only
+ * access through pan_earlyzs_get and pan_earlyzs_analyze. See pan_earlyzs_get
+ * for definition of the arrays.
+ */
+struct pan_earlyzs_lut {
+   struct pan_earlyzs_state states[2][2][2][PAN_EARLYZS_ZS_TILEBUF_MODE_COUNT];
+};
+
+/*
+ * Look up early-ZS state. This is in the draw hot path on Valhall, so this is
+ * defined inline in the header.
+ */
+static inline struct pan_earlyzs_state
+pan_earlyzs_get(struct pan_earlyzs_lut lut, bool writes_zs_or_oq,
+                bool alpha_to_coverage, bool zs_always_passes,
+                enum pan_earlyzs_zs_tilebuf_read zs_read)
+{
+   return lut.states[writes_zs_or_oq][alpha_to_coverage][zs_always_passes]
+                    [zs_read];
+}
+
+struct pan_shader_info;
+
+struct pan_earlyzs_lut pan_earlyzs_analyze(const struct pan_shader_info *s,
+                                           unsigned arch);
+
+#ifdef __cplusplus
+} /* extern C */
+#endif
+
+#endif

src/panfrost/lib/pan_encoder.h

@@ -0,0 +1,283 @@
+/*
+ * Copyright (C) 2019 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef __PAN_ENCODER_H
+#define __PAN_ENCODER_H
+
+#ifndef __OPENCL_VERSION__
+#include "util/macros.h"
+
+#include <stdbool.h>
+#include "util/format/u_format.h"
+#include "pan_pool.h"
+#else
+#include "compiler/libcl/libcl.h"
+#endif
+
+#include "genxml/gen_macros.h"
+
+/* Tiler structure size computation */
+
+unsigned pan_tiler_header_size(unsigned width, unsigned height, unsigned mask,
+                               bool hierarchy);
+
+unsigned pan_tiler_full_size(unsigned width, unsigned height, unsigned mask,
+                             bool hierarchy);
+
+unsigned pan_choose_hierarchy_mask(unsigned width, unsigned height,
+                                   unsigned vertex_count, bool hierarchy);
+
+#if defined(PAN_ARCH) && PAN_ARCH <= 5
+static inline unsigned
+pan_tiler_get_polygon_list_size(unsigned fb_width, unsigned fb_height,
+                                unsigned vertex_count, bool hierarchy)
+{
+   if (!vertex_count)
+      return MALI_MIDGARD_TILER_MINIMUM_HEADER_SIZE + 4;
+
+   unsigned hierarchy_mask =
+      pan_choose_hierarchy_mask(fb_width, fb_height, vertex_count, hierarchy);
+
+   return pan_tiler_full_size(fb_width, fb_height, hierarchy_mask, hierarchy) +
+          pan_tiler_header_size(fb_width, fb_height, hierarchy_mask, hierarchy);
+}
+#endif
+
+/* Stack sizes */
+
+unsigned pan_get_stack_shift(unsigned stack_size);
+
+unsigned pan_get_total_stack_size(unsigned thread_size,
+                                  unsigned threads_per_core,
+                                  unsigned core_id_range);
+
+/* Attributes / instancing */
+
+static inline unsigned
+pan_padded_vertex_count(unsigned vertex_count)
+{
+   if (vertex_count < 10)
+      return vertex_count;
+
+   if (vertex_count < 20)
+      return (vertex_count + 1) & ~1;
+
+   /* First, we have to find the highest set one */
+   unsigned highest = 32 - __builtin_clz(vertex_count);
+
+   /* Using that, we mask out the highest 4-bits */
+   unsigned n = highest - 4;
+   unsigned nibble = (vertex_count >> n) & 0xF;
+
+   /* Great, we have the nibble. Now we can just try possibilities. Note
+    * that we don't care about the bottom most bit in most cases, and we
+    * know the top bit must be 1 */
+
+   unsigned middle_two = (nibble >> 1) & 0x3;
+
+   switch (middle_two) {
+   case 0b00:
+      if (!(nibble & 1))
+         return (1 << n) * 9;
+      else
+         return (1 << (n + 1)) * 5;
+   case 0b01:
+      return (1 << (n + 2)) * 3;
+   case 0b10:
+      return (1 << (n + 1)) * 7;
+   case 0b11:
+      return (1 << (n + 4));
+   default:
+      return 0; /* unreachable */
+   }
+}
+
+static inline unsigned
+pan_compute_npot_divisor(unsigned hw_divisor, unsigned *divisor_r,
+                         unsigned *divisor_e)
+{
+   unsigned r = util_logbase2(hw_divisor);
+
+   uint64_t shift_hi = 32 + r;
+   uint64_t t = (uint64_t)1 << shift_hi;
+   uint64_t f0 = t + hw_divisor / 2;
+   uint64_t fi = f0 / hw_divisor;
+   uint64_t ff = f0 - fi * hw_divisor;
+
+   uint64_t d = fi - (1ul << 31);
+   *divisor_r = r;
+   *divisor_e = ff > hw_divisor / 2 ? 1 : 0;
+
+   return d;
+}
+
+#ifdef PAN_ARCH
+/* Records for gl_VertexID and gl_InstanceID use special encodings on Midgard */
+
+#if PAN_ARCH <= 5
+static inline void
+pan_vertex_id(unsigned padded_count,
+              struct mali_attribute_vertex_id_packed *attr, bool instanced)
+{
+   pan_pack(attr, ATTRIBUTE_VERTEX_ID, cfg) {
+      if (instanced) {
+         cfg.divisor_r = __builtin_ctz(padded_count);
+         cfg.divisor_p = padded_count >> (cfg.divisor_r + 1);
+      } else {
+         /* Large values so the modulo is a no-op */
+         cfg.divisor_r = 0x1F;
+         cfg.divisor_p = 0x4;
+      }
+   }
+}
+
+static inline void
+pan_instance_id(unsigned padded_count,
+                struct mali_attribute_instance_id_packed *attr, bool instanced)
+{
+   pan_pack(attr, ATTRIBUTE_INSTANCE_ID, cfg) {
+      if (!instanced || padded_count <= 1) {
+         /* Divide by large number to force to 0 */
+         cfg.divisor_p = ((1u << 31) - 1);
+         cfg.divisor_r = 0x1F;
+         cfg.divisor_e = 0x1;
+      } else if (util_is_power_of_two_or_zero(padded_count)) {
+         /* Can't underflow since padded_count >= 2 */
+         cfg.divisor_r = __builtin_ctz(padded_count) - 1;
+      } else {
+         cfg.divisor_p = pan_compute_npot_divisor(padded_count, &cfg.divisor_r,
+                                                  &cfg.divisor_e);
+      }
+   }
+}
+#endif /* PAN_ARCH <= 5 */
+
+/* Sampler comparison functions are flipped in OpenGL from the hardware, so we
+ * need to be able to flip accordingly */
+
+static inline enum mali_func
+pan_flip_compare_func(enum mali_func f)
+{
+   switch (f) {
+   case MALI_FUNC_LESS:
+      return MALI_FUNC_GREATER;
+   case MALI_FUNC_GREATER:
+      return MALI_FUNC_LESS;
+   case MALI_FUNC_LEQUAL:
+      return MALI_FUNC_GEQUAL;
+   case MALI_FUNC_GEQUAL:
+      return MALI_FUNC_LEQUAL;
+   default:
+      return f;
+   }
+}
+
+#if PAN_ARCH < 9
+/* Compute shaders are invoked with a gl_NumWorkGroups X/Y/Z triplet. Vertex
+ * shaders are invoked as (1, vertex_count, instance_count). Compute shaders
+ * also have a gl_WorkGroupSize X/Y/Z triplet. These 6 values are packed
+ * together in a dynamic bitfield, packed by this routine. */
+
+static inline void
+pan_pack_work_groups_compute(struct mali_invocation_packed *out, unsigned num_x,
+                             unsigned num_y, unsigned num_z, unsigned size_x,
+                             unsigned size_y, unsigned size_z,
+                             bool quirk_graphics, bool indirect_dispatch)
+{
+   /* The values needing packing, in order, and the corresponding shifts.
+    * Indices into shift are off-by-one to make the logic easier */
+
+   unsigned values[6] = {size_x, size_y, size_z, num_x, num_y, num_z};
+   unsigned shifts[7] = {0};
+   uint32_t packed = 0;
+
+   for (unsigned i = 0; i < 6; ++i) {
+      /* Must be positive, otherwise we underflow */
+      assert(values[i] >= 1);
+
+      /* OR it in, shifting as required */
+      packed |= ((values[i] - 1) << shifts[i]);
+
+      /* How many bits did we use? */
+      unsigned bit_count = util_logbase2_ceil(values[i]);
+
+      /* Set the next shift accordingly */
+      shifts[i + 1] = shifts[i] + bit_count;
+   }
+
+   pan_pack(out, INVOCATION, cfg) {
+      cfg.invocations = packed;
+      cfg.size_y_shift = shifts[1];
+      cfg.size_z_shift = shifts[2];
+      cfg.workgroups_x_shift = shifts[3];
+
+      if (!indirect_dispatch) {
+         /* Leave zero for the dispatch shader */
+         cfg.workgroups_y_shift = shifts[4];
+         cfg.workgroups_z_shift = shifts[5];
+      }
+
+      /* Quirk: for non-instanced graphics, the blob sets
+       * workgroups_z_shift = 32. This doesn't appear to matter to
+       * the hardware, but it's good to be bit-identical. */
+
+      if (quirk_graphics && (num_z <= 1))
+         cfg.workgroups_z_shift = 32;
+
+      /* For graphics, set to the minimum efficient value. For
+       * compute, must equal the workgroup X shift for barriers to
+       * function correctly */
+
+      cfg.thread_group_split =
+         quirk_graphics ? MALI_SPLIT_MIN_EFFICIENT : cfg.workgroups_x_shift;
+   }
+}
+#endif
+
+#ifndef __OPENCL_VERSION__
+#if PAN_ARCH >= 5
+/* Format conversion */
+static inline enum mali_z_internal_format
+pan_get_z_internal_format(enum pipe_format fmt)
+{
+   switch (fmt) {
+   case PIPE_FORMAT_Z16_UNORM:
+   case PIPE_FORMAT_Z16_UNORM_S8_UINT:
+      return MALI_Z_INTERNAL_FORMAT_D16;
+   case PIPE_FORMAT_Z24_UNORM_S8_UINT:
+   case PIPE_FORMAT_Z24X8_UNORM:
+   case PIPE_FORMAT_Z24_UNORM_PACKED:
+      return MALI_Z_INTERNAL_FORMAT_D24;
+   case PIPE_FORMAT_Z32_FLOAT:
+   case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT:
+      return MALI_Z_INTERNAL_FORMAT_D32;
+   default:
+      UNREACHABLE("Unsupported depth/stencil format.");
+   }
+}
+#endif
+#endif
+
+#endif /* PAN_ARCH */
+
+#ifndef __OPENCL_VERSION__
+#if PAN_ARCH >= 9
+static inline void
+pan_make_resource_table(struct pan_ptr base, unsigned index, uint64_t address,
+                        unsigned resource_count)
+{
+   if (resource_count == 0)
+      return;
+
+   struct mali_resource_packed *res = base.cpu;
+   pan_pack(&res[index], RESOURCE, cfg) {
+      cfg.address = address;
+      cfg.size = resource_count * pan_size(BUFFER);
+   }
+}
+#endif
+#endif
+
+#endif

src/panfrost/lib/pan_fb.c

@@ -0,0 +1,800 @@
+/*
+ * Copyright (C) 2026 Collabora, Ltd.
+ * Copyright (C) 2026 Arm Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+#include "pan_fb.h"
+
+#include "pan_afbc.h"
+#include "pan_afrc.h"
+#include "pan_desc.h"
+#include "pan_encoder.h"
+#include "pan_format.h"
+#include "pan_image.h"
+#include "pan_props.h"
+#include "pan_util.h"
+
+static unsigned
+pan_bytes_per_pixel_tib(enum pipe_format format)
+{
+   const struct pan_blendable_format *bf =
+      GENX(pan_blendable_format_from_pipe_format)(format);
+   return pan_format_tib_size(format, bf->internal);
+}
+
+void
+GENX(pan_select_fb_tile_size)(struct pan_fb_layout *fb)
+{
+   uint32_t rt_B_per_sa = 0;
+   for (unsigned rt = 0; rt < fb->rt_count; rt++) {
+      enum pipe_format format = fb->rt_formats[rt];
+      if (format == PIPE_FORMAT_NONE)
+         continue;
+
+      const struct pan_blendable_format *bf =
+         GENX(pan_blendable_format_from_pipe_format)(format);
+      rt_B_per_sa += pan_format_tib_size(format, bf->internal);
+   }
+
+   /* The PLS area overlaps with the color targets */
+   rt_B_per_sa = MAX2(rt_B_per_sa, fb->pls_size_B);
+
+   const uint32_t rt_B_per_px = rt_B_per_sa * fb->sample_count;
+
+   /* We always have depth and it's is always stored in a 32-bit float.
+    * Stencil requires depth to be allocated, but doesn't have it's own
+    * budget; it's tied to the depth buffer.
+    */
+   const uint32_t z_B_per_px = sizeof(float) * fb->sample_count;
+
+   fb->tile_size_px =
+      MIN2(fb->tile_rt_budget_B >> util_logbase2_ceil(rt_B_per_px),
+           fb->tile_z_budget_B >> util_logbase2_ceil(z_B_per_px));
+
+   /* Check if we're using too much tile-memory; if we are, try disabling
+    * pipelining. This works because we're starting with an optimistic half
+    * of the tile-budget, so we actually have another half that can be used.
+    *
+    * On v6 GPUs, doing this is not allowed; they *have* to pipeline.
+    */
+    if (PAN_ARCH != 6 && fb->tile_size_px < 4 * 4)
+       fb->tile_size_px *= 2;
+
+   /* Clamp tile size to hardware limits */
+   fb->tile_size_px =
+      MIN2(fb->tile_size_px, pan_max_effective_tile_size(PAN_ARCH));
+   assert(util_is_power_of_two_nonzero(fb->tile_size_px));
+   assert(fb->tile_size_px >= 4 * 4);
+
+   /* Colour buffer allocations must be 1K aligned. */
+   fb->tile_rt_alloc_B = ALIGN_POT(rt_B_per_px * fb->tile_size_px, 1024);
+
+#if PAN_ARCH == 6
+   assert(fb->tile_rt_alloc_B <= fb->tile_rt_budget_B && "tile too big");
+#else
+   assert(fb->tile_rt_alloc_B <= fb->tile_rt_budget_B * 2 && "tile too big");
+#endif
+}
+
+static void
+align_fb_tiling_area_for_image_plane(struct pan_fb_layout *fb,
+                                     struct pan_image_plane_ref pref)
+{
+   if (!pref.image)
+      return;
+
+   struct pan_image_block_size block_size;
+   if (drm_is_afbc(pref.image->props.modifier)) {
+      /* For AFBC render targets, the hardware always writes full superblocks.
+       * In order to ensure we don't write garbage, we need to expand the
+       * render area accordingly and load the border pixels.
+       */
+      block_size = pan_afbc_renderblock_size(pref.image->props.modifier);
+      assert(block_size.width >= 16 && block_size.height >= 16);
+   } else if (drm_is_afrc(pref.image->props.modifier)) {
+      /* For AFRC render targets, the hardware always writes full clumps.  In
+       * order to ensure we don't write garbage, we need to expand the render
+       * area accordingly and load the border pixels.
+       */
+      bool scan = pan_afrc_is_scan(pref.image->props.modifier);
+      block_size = pan_afrc_clump_size(pref.image->props.format, scan);
+   } else {
+      /* No alignment requirements */
+      return;
+   }
+
+   fb->tiling_area_px = pan_fb_bbox_align(fb->tiling_area_px,
+                                          block_size.width,
+                                          block_size.height);
+}
+
+void
+GENX(pan_align_fb_tiling_area)(struct pan_fb_layout *fb,
+                               const struct pan_fb_store *store)
+{
+   if (store == NULL)
+      return;
+
+   for (unsigned rt = 0; rt < fb->rt_count; rt++) {
+      if (store->rts[rt].store) {
+         align_fb_tiling_area_for_image_plane(fb,
+            pan_image_view_get_color_plane(store->rts[rt].iview));
+      }
+   }
+
+   if (store->zs.store) {
+      align_fb_tiling_area_for_image_plane(fb,
+         pan_image_view_get_zs_plane(store->zs.iview));
+   }
+
+   if (store->s.store) {
+      align_fb_tiling_area_for_image_plane(fb,
+         pan_image_view_get_s_plane(store->s.iview));
+   }
+}
+
+static void
+fold_sample_0_image_load(struct pan_fb_resolve_op_msaa *rm,
+                         const struct pan_image_view *iview,
+                         uint8_t fb_sample_count)
+{
+   if (rm->resolve != PAN_FB_RESOLVE_IMAGE)
+      return;
+
+   if (rm->msaa != PAN_FB_MSAA_COPY_ALL)
+      return;
+
+   assert(pan_image_view_get_nr_samples(iview) == fb_sample_count);
+
+   /* If we're loading all samples of the image but only storing SAMPLE_0,
+    * then we don't actually need all samples in the image.  Drop it to a
+    * SAMPLE_0 load to save bandwidth.
+    */
+   rm->msaa = PAN_FB_MSAA_COPY_SAMPLE_0;
+}
+
+static bool
+resolve_msaa_can_fold(struct pan_fb_resolve_op_msaa rm,
+                      enum pan_fb_resolve_op self_op,
+                      enum pipe_format format)
+{
+   if (rm.resolve == PAN_FB_RESOLVE_IMAGE) {
+      /* We already folded away COPY_ALL and all the other MSAA modes produce
+       * a single per-pixel result across all the samples, i.e. COPY_IDENTICAL.
+       * We can fold that with any resolve mode.
+       */
+      assert(rm.msaa != PAN_FB_MSAA_COPY_ALL);
+      return true;
+   }
+
+   if (rm.resolve != self_op)
+      return false;
+
+   /* PAN_FB_MSAA_COPY_IDENTICAL is always foldable */
+   if (rm.msaa == PAN_FB_MSAA_COPY_IDENTICAL)
+      return true;
+
+   if (self_op == PAN_FB_RESOLVE_Z || self_op == PAN_FB_RESOLVE_S) {
+      /* Z/S only supports SAMPLE_0 */
+      return rm.msaa == PAN_FB_MSAA_COPY_SAMPLE_0;
+   } else {
+      if (rm.msaa == PAN_FB_MSAA_COPY_AVERAGE &&
+          !GENX(pan_format_supports_msaa_average)(format))
+         return false;
+
+      /* The color hardware supports SAMPLE_0 and AVERAGE */
+      return rm.msaa == PAN_FB_MSAA_COPY_SAMPLE_0 ||
+             rm.msaa == PAN_FB_MSAA_COPY_AVERAGE;
+   }
+}
+
+static void
+fold_resolve_into_store_target(enum pipe_format format,
+                               uint8_t fb_sample_count,
+                               enum pan_fb_resolve_op self_op,
+                               struct pan_fb_resolve_target *resolve,
+                               struct pan_fb_store_target *store)
+{
+   if (format == PIPE_FORMAT_NONE)
+      return;
+
+   /* We need SAMPLE_0 if we're going to fold in the resolve */
+   if (store->msaa != PAN_FB_MSAA_COPY_SAMPLE_0)
+      return;
+
+   fold_sample_0_image_load(&resolve->in_bounds, resolve->iview,
+                            fb_sample_count);
+   fold_sample_0_image_load(&resolve->in_bounds, resolve->iview,
+                            fb_sample_count);
+
+   if (!resolve_msaa_can_fold(resolve->in_bounds, self_op, format) ||
+       !resolve_msaa_can_fold(resolve->border, self_op, format))
+      return;
+
+   enum pan_fb_msaa_copy_op msaa;
+   if (resolve->in_bounds.resolve == PAN_FB_RESOLVE_IMAGE) {
+      /* If they're both IMAGE, there's nothing to fold */
+      if (resolve->border.resolve == PAN_FB_RESOLVE_IMAGE)
+         return;
+      msaa = resolve->border.msaa;
+   } else if (resolve->border.resolve == PAN_FB_RESOLVE_IMAGE) {
+      msaa = resolve->in_bounds.msaa;
+   } else {
+      /* If neither is an image resolve, the MSAA copy op has to match or one
+       * of them has to be PAN_FB_MSAA_COPY_IDENTICAL.
+       */
+      if (resolve->in_bounds.msaa != PAN_FB_MSAA_COPY_IDENTICAL) {
+         if (resolve->border.msaa != PAN_FB_MSAA_COPY_IDENTICAL &&
+             resolve->border.msaa != resolve->in_bounds.msaa)
+            return;
+         msaa = resolve->in_bounds.msaa;
+      } else {
+         msaa = resolve->border.msaa;
+      }
+   }
+
+   /* PAN_FB_MSAA_COPY_IDENTICAL isn't real. Use SAMPLE_0 */
+   if (msaa == PAN_FB_MSAA_COPY_IDENTICAL)
+      msaa = PAN_FB_MSAA_COPY_SAMPLE_0;
+
+   /* At this point, we know we can fold the resolve */
+   store->msaa = msaa;
+
+   if (resolve->in_bounds.resolve != PAN_FB_RESOLVE_IMAGE) {
+      resolve->in_bounds.resolve = PAN_FB_RESOLVE_NONE;
+      resolve->in_bounds.msaa = PAN_FB_MSAA_COPY_ALL;
+   }
+
+   if (resolve->border.resolve != PAN_FB_RESOLVE_IMAGE) {
+      resolve->border.resolve = PAN_FB_RESOLVE_NONE;
+      resolve->border.msaa = PAN_FB_MSAA_COPY_ALL;
+   }
+}
+
+void
+GENX(pan_fb_fold_resolve_into_store)(const struct pan_fb_layout *fb,
+                                     struct pan_fb_resolve *resolve,
+                                     struct pan_fb_store *store)
+{
+   if (fb->sample_count == 1)
+      return;
+
+   fold_resolve_into_store_target(fb->z_format, fb->sample_count,
+                                  PAN_FB_RESOLVE_Z, &resolve->z, &store->zs);
+   fold_resolve_into_store_target(fb->s_format, fb->sample_count,
+                                  PAN_FB_RESOLVE_S, &resolve->s, &store->s);
+   for (unsigned rt = 0; rt < fb->rt_count; rt++) {
+      fold_resolve_into_store_target(fb->rt_formats[rt], fb->sample_count,
+                                     PAN_FB_RESOLVE_RT(rt),
+                                     &resolve->rts[rt], &store->rts[rt]);
+   }
+}
+
+void
+GENX(pan_fill_fb_info)(const struct pan_fb_desc_info *info,
+                       struct pan_fb_info *fbinfo)
+{
+   struct pan_bbox bbox = { 0, 0, 0, 0 };
+   if (info->fb->width_px > 0 && info->fb->width_px > 0) {
+      const struct pan_fb_bbox fb_area_px =
+         pan_fb_bbox_from_xywh(0, 0, info->fb->width_px, info->fb->height_px);
+
+      assert(pan_fb_bbox_is_valid(info->fb->tiling_area_px));
+      const struct pan_fb_bbox bbox_px =
+         pan_fb_bbox_clamp(info->fb->tiling_area_px, fb_area_px);
+
+      bbox = (struct pan_bbox) {
+         .minx = bbox_px.min_x,
+         .miny = bbox_px.min_y,
+         .maxx = bbox_px.max_x,
+         .maxy = bbox_px.max_y,
+      };
+   }
+
+   *fbinfo = (struct pan_fb_info) {
+      .width = info->fb->width_px,
+      .height = info->fb->height_px,
+      .draw_extent = bbox,
+      .frame_bounding_box = bbox,
+      .nr_samples = info->fb->sample_count,
+      .rt_count = info->fb->rt_count,
+      .pls_enabled = info->fb->pls_size_B > 0,
+      .bifrost.pre_post = {
+         .dcds.gpu = info->frame_shaders.dcd_pointer,
+         .modes = {
+            info->frame_shaders.modes[0],
+            info->frame_shaders.modes[1],
+            info->frame_shaders.modes[2],
+         },
+      },
+
+      .tile_buf_budget = info->fb->tile_rt_budget_B,
+      .z_tile_buf_budget = info->fb->tile_z_budget_B,
+      .tile_size = info->fb->tile_size_px,
+      .cbuf_allocation = info->fb->tile_rt_alloc_B,
+
+      .sample_positions = info->sample_pos_array_pointer,
+      .sprite_coord_origin = info->sprite_coord_origin_max_y,
+      .first_provoking_vertex = info->provoking_vertex_first,
+      .allow_hsr_prepass = info->provoking_vertex_first,
+   };
+
+   /* There are cases where we only want to fill out a partial fb_info */
+   if (info->load == NULL && info->store == NULL)
+      return;
+
+   for (unsigned rt = 0; rt < info->fb->rt_count; rt++) {
+      fbinfo->rts[rt] = (struct pan_fb_color_attachment) {
+         .view = info->store->rts[rt].iview,
+         .clear = info->load->rts[rt].border_load == PAN_FB_LOAD_CLEAR ||
+                  info->load->rts[rt].in_bounds_load == PAN_FB_LOAD_CLEAR,
+         .preload = info->load->rts[rt].in_bounds_load == PAN_FB_LOAD_IMAGE,
+         .discard = !info->store->rts[rt].store,
+      };
+
+      if (fbinfo->rts[rt].clear) {
+         pan_pack_color(GENX(pan_blendable_formats),
+                        fbinfo->rts[rt].clear_value,
+                        &info->load->rts[rt].clear.color,
+                        info->fb->rt_formats[rt],
+                        false /* dithered */);
+      }
+   }
+
+   if (info->fb->z_format != PIPE_FORMAT_NONE) {
+      fbinfo->zs.view.zs = info->store->zs.iview;
+      fbinfo->zs.clear.z = info->load->z.border_load == PAN_FB_LOAD_CLEAR ||
+                           info->load->z.in_bounds_load == PAN_FB_LOAD_CLEAR;
+      fbinfo->zs.preload.z = info->load->z.in_bounds_load == PAN_FB_LOAD_IMAGE;
+      fbinfo->zs.discard.z = !info->store->zs.store;
+      if (fbinfo->zs.clear.z)
+         fbinfo->zs.clear_value.depth = info->load->z.clear.depth;
+   }
+
+   if (info->fb->s_format != PIPE_FORMAT_NONE) {
+      fbinfo->zs.view.s = info->store->s.iview;
+      fbinfo->zs.clear.s = info->load->s.border_load == PAN_FB_LOAD_CLEAR ||
+                           info->load->s.in_bounds_load == PAN_FB_LOAD_CLEAR;
+      fbinfo->zs.preload.s = info->load->s.in_bounds_load == PAN_FB_LOAD_IMAGE;
+      fbinfo->zs.discard.s = !info->store->s.store;
+      if (fbinfo->zs.clear.s)
+         fbinfo->zs.clear_value.stencil = info->load->s.clear.stencil;
+   }
+}
+
+#if PAN_ARCH >= 5
+static enum mali_msaa
+translate_msaa_copy_op(const struct pan_fb_layout *fb,
+                       const struct pan_image_view *iview,
+                       enum pan_fb_msaa_copy_op msaa)
+{
+   switch (msaa) {
+   case PAN_FB_MSAA_COPY_ALL:
+      assert(pan_image_view_get_nr_samples(iview) == fb->sample_count);
+      if (fb->sample_count > 1)
+         return MALI_MSAA_LAYERED;
+      else
+         return MALI_MSAA_SINGLE;
+
+   case PAN_FB_MSAA_COPY_SINGLE:
+   case PAN_FB_MSAA_COPY_SAMPLE_0:
+      return MALI_MSAA_SINGLE;
+
+   case PAN_FB_MSAA_COPY_AVERAGE:
+      /* It must be a blendable format or the hardware can't average */
+      assert(GENX(pan_blendable_format_from_pipe_format)(iview->format)->internal);
+      if (fb->sample_count > 1)
+         return MALI_MSAA_AVERAGE;
+      else
+         return MALI_MSAA_SINGLE;
+
+   default:
+      UNREACHABLE("Unsupported MSAA copy op");
+   }
+}
+
+static bool
+pan_fb_load_target_always(const struct pan_fb_load_target *target)
+{
+   if (target->in_bounds_load == PAN_FB_LOAD_NONE &&
+       target->border_load == PAN_FB_LOAD_NONE)
+      return false;
+
+   return target->always;
+}
+
+static bool
+pan_fb_store_target_always(const struct pan_fb_store_target *target)
+{
+   return target->store && target->always;
+}
+
+struct pan_fb_clean_tile
+GENX(pan_fb_get_clean_tile)(const struct pan_fb_desc_info *info)
+{
+   const struct pan_fb_layout *fb = info->fb;
+   const struct pan_fb_load *load = info->load;
+   const struct pan_fb_store *store = info->store;
+
+   struct pan_fb_clean_tile ct = { };
+
+   for (unsigned rt = 0; rt < fb->rt_count; rt++) {
+      if (fb->rt_formats[rt] == PIPE_FORMAT_NONE)
+         continue;
+
+      if ((load && pan_fb_load_target_always(&load->rts[rt])) ||
+          (store && pan_fb_store_target_always(&store->rts[rt])))
+         ct.rts |= BITFIELD_BIT(rt);
+
+      if (store && store->rts[rt].store) {
+         const struct pan_image *img =
+            pan_image_view_get_color_plane(store->rts[rt].iview).image;
+         if (GENX(pan_force_clean_write_on)(img, fb->tile_size_px))
+            ct.rts |= BITFIELD_BIT(rt);
+      }
+   }
+
+   const bool z_always_load = load && pan_fb_load_target_always(&load->z);
+   const bool s_always_load = load && pan_fb_load_target_always(&load->s);
+   const bool zs_always_store = store && pan_fb_store_target_always(&store->zs);
+   const bool s_always_store = store && pan_fb_store_target_always(&store->s);
+
+   if (fb->z_format != PIPE_FORMAT_NONE) {
+      ct.zs = z_always_load || zs_always_store;
+
+      if (store && store->zs.store) {
+         const struct pan_image *img =
+            pan_image_view_get_zs_plane(store->zs.iview).image;
+         assert(util_format_get_depth_bits(img->props.format) ==
+                util_format_get_depth_bits(fb->z_format));
+         assert(util_format_get_depth_bits(store->zs.iview->format) ==
+                util_format_get_depth_bits(fb->z_format));
+         const struct util_format_description *zs_fmt_desc =
+            util_format_description(img->props.format);
+
+         /* If ZS writes stencil, we have to also include stencil loads */
+         if (util_format_has_stencil(zs_fmt_desc) && s_always_load)
+            ct.zs = true;
+
+         if (GENX(pan_force_clean_write_on)(img, fb->tile_size_px))
+            ct.zs = true;
+      }
+   }
+
+   if (fb->s_format != PIPE_FORMAT_NONE) {
+      ct.s = s_always_load || s_always_store;
+
+      if (store && store->s.store) {
+         const struct pan_image *img =
+            pan_image_view_get_s_plane(store->s.iview).image;
+         if (GENX(pan_force_clean_write_on)(img, fb->tile_size_px))
+            ct.s = true;
+      }
+   }
+
+   return ct;
+}
+
+static void
+emit_zs_crc_desc(const struct pan_fb_desc_info *info,
+                 const struct pan_fb_clean_tile ct,
+                 struct mali_zs_crc_extension_packed *zs_crc)
+{
+   const struct pan_fb_layout *fb = info->fb;
+   const struct pan_fb_store *store = info->store;
+
+   pan_pack(zs_crc, ZS_CRC_EXTENSION, cfg) {
+      if (store && store->zs.store) {
+         cfg.zs.msaa = translate_msaa_copy_op(fb, store->zs.iview,
+                                              store->zs.msaa);
+
+#if PAN_ARCH >= 6
+         cfg.zs.clean_tile_write_enable = ct.zs;
+#else
+         cfg.zs.clean_pixel_write_enable = ct.zs;
+#endif
+
+      }
+
+      if (store && store->s.store) {
+         cfg.s.msaa = translate_msaa_copy_op(fb, store->s.iview,
+                                             store->s.msaa);
+
+#if PAN_ARCH >= 6
+         cfg.s.clean_tile_write_enable = ct.s;
+#else
+         cfg.s.clean_pixel_write_enable = ct.s;
+#endif
+      }
+
+      /* TODO CRC */
+   }
+
+   if (store && store->zs.store) {
+      const struct pan_image_view *iview = store->zs.iview;
+      const struct pan_mod_handler *mod_handler =
+         pan_image_view_get_zs_plane(iview).image->mod_handler;
+
+      assert(info->layer < pan_image_view_get_layer_count(iview));
+      const struct pan_attachment_info att = {
+         .iview = iview,
+         .layer_or_z_slice = iview->first_layer + info->layer,
+         .fb_tile_size_px = fb->tile_size_px,
+      };
+
+      struct mali_zs_crc_extension_packed zs_part;
+      mod_handler->emit_zs_attachment(&att, &zs_part);
+      pan_merge(zs_crc, &zs_part, ZS_CRC_EXTENSION);
+   }
+
+   if (store && store->s.store) {
+      const struct pan_image_view *iview = store->s.iview;
+      const struct pan_mod_handler *mod_handler =
+         pan_image_view_get_s_plane(iview).image->mod_handler;
+
+      assert(info->layer < pan_image_view_get_layer_count(iview));
+      const struct pan_attachment_info att = {
+         .iview = iview,
+         .layer_or_z_slice = iview->first_layer + info->layer,
+         .fb_tile_size_px = fb->tile_size_px,
+      };
+
+      struct mali_zs_crc_extension_packed s_part;
+      mod_handler->emit_s_attachment(&att, &s_part);
+      pan_merge(zs_crc, &s_part, ZS_CRC_EXTENSION);
+   }
+
+   /* TODO: CRC */
+}
+
+static void
+emit_rgb_rt_desc(const struct pan_fb_desc_info *info,
+                 const struct pan_fb_clean_tile ct,
+                 unsigned rt, uint32_t tile_rt_offset_B,
+                 struct mali_rgb_render_target_packed *rgb_rt)
+{
+   const struct pan_fb_layout *fb = info->fb;
+   const struct pan_fb_load *load = info->load;
+   const struct pan_fb_store *store = info->store;
+
+   if (fb->rt_formats[rt] == PIPE_FORMAT_NONE) {
+      pan_pack(rgb_rt, RGB_RENDER_TARGET, cfg) {
+         /* Place non-existent targets at the end.  Hardware bounds
+          * checking should just discard writes and zero reads.
+          */
+         cfg.internal_buffer_offset = fb->tile_rt_alloc_B;
+         cfg.internal_format = MALI_COLOR_BUFFER_INTERNAL_FORMAT_R8G8B8A8;
+         cfg.write_enable = false;
+#if PAN_ARCH >= 7
+         cfg.writeback_block_format = MALI_BLOCK_FORMAT_TILED_U_INTERLEAVED;
+#endif
+      }
+      return;
+   }
+
+   pan_pack(rgb_rt, RGB_RENDER_TARGET, cfg) {
+      cfg.internal_buffer_offset = tile_rt_offset_B;
+      cfg.dithering_enable = true;
+
+      if (store && store->rts[rt].store) {
+         cfg.writeback_msaa =
+            translate_msaa_copy_op(fb, store->rts[rt].iview,
+                                   store->rts[rt].msaa);
+      }
+
+#if PAN_ARCH >= 6
+      cfg.clean_tile_write_enable = !!(ct.rts & BITFIELD_BIT(rt));
+#else
+      cfg.clean_pixel_write_enable = !!(ct.rts & BITFIELD_BIT(rt));
+#endif
+
+      if (load && pan_target_has_clear(&load->rts[rt])) {
+         uint32_t packed[4] = {};
+         pan_pack_color(GENX(pan_blendable_formats), packed,
+                        &load->rts[rt].clear.color, fb->rt_formats[rt],
+                        false /* dithered */);
+
+         cfg.clear = (struct MALI_RT_CLEAR) {
+            .color_0 = packed[0],
+            .color_1 = packed[1],
+            .color_2 = packed[2],
+            .color_3 = packed[3],
+         };
+      }
+   }
+
+   struct mali_render_target_packed desc;
+   if (store && store->rts[rt].store) {
+      const struct pan_image_view *iview = store->rts[rt].iview;
+      const struct pan_mod_handler *mod_handler =
+         pan_image_view_get_color_plane(iview).image->mod_handler;
+
+      assert(info->layer < pan_image_view_get_layer_count(iview));
+      const struct pan_attachment_info att = {
+         .iview = iview,
+         .layer_or_z_slice = iview->first_layer + info->layer,
+         .fb_tile_size_px = fb->tile_size_px,
+      };
+
+      mod_handler->emit_color_attachment(&att, &desc);
+   } else {
+      GENX(pan_emit_default_color_attachment)(fb->rt_formats[rt], &desc);
+   }
+   pan_merge(rgb_rt, &desc, RGB_RENDER_TARGET);
+}
+
+static void
+emit_rts(const struct pan_fb_desc_info *info,
+         struct mali_rgb_render_target_packed *rts)
+{
+   const struct pan_fb_layout *fb = info->fb;
+   const struct pan_fb_clean_tile ct = GENX(pan_fb_get_clean_tile)(info);
+
+   uint32_t tile_rt_offset_B = 0;
+   for (unsigned rt = 0; rt < fb->rt_count; rt++) {
+      emit_rgb_rt_desc(info, ct, rt, tile_rt_offset_B, rts);
+      rts++;
+
+      if (fb->rt_formats[rt] != PIPE_FORMAT_NONE) {
+         tile_rt_offset_B += pan_bytes_per_pixel_tib(fb->rt_formats[rt]) *
+                             fb->tile_size_px * fb->sample_count;
+      }
+   }
+   assert(tile_rt_offset_B <= fb->tile_rt_alloc_B);
+}
+
+#if PAN_ARCH >= 14
+uint32_t
+GENX(pan_emit_fb_desc)(const struct pan_fb_desc_info *info,
+                       const struct pan_fb_descs *out)
+{
+   if (pan_fb_has_zs(info->fb)) {
+      emit_zs_crc_desc(info, GENX(pan_fb_get_clean_tile)(info), out->zs_crc);
+   }
+
+   emit_rts(info, out->rts);
+
+   return 0;
+}
+#else /* PAN_ARCH < 14 */
+uint32_t
+GENX(pan_emit_fb_desc)(const struct pan_fb_desc_info *info,
+                       const struct pan_fb_descs *out)
+{
+   const struct pan_fb_layout *fb = info->fb;
+   const struct pan_fb_load *load = info->load;
+   const struct pan_fb_store *store = info->store;
+   const struct pan_fb_clean_tile ct = GENX(pan_fb_get_clean_tile)(info);
+
+   const bool has_zs_crc_ext = pan_fb_has_zs(fb);
+
+   struct mali_framebuffer_packed fbd = {};
+
+#if PAN_ARCH <= 5
+   GENX(pan_emit_tls)(info->tls,
+      pan_section_ptr(&fbd, FRAMEBUFFER, LOCAL_STORAGE));
+#endif
+
+   pan_section_pack(&fbd, FRAMEBUFFER, PARAMETERS, cfg) {
+#if PAN_ARCH >= 6
+      cfg.pre_frame_0 = pan_fix_frame_shader_mode(info->frame_shaders.modes[0],
+                                                  ct.rts || ct.zs || ct.s);
+      cfg.pre_frame_1 = pan_fix_frame_shader_mode(info->frame_shaders.modes[1],
+                                                  ct.rts || ct.zs || ct.s);
+      cfg.post_frame = info->frame_shaders.modes[2];
+      cfg.frame_shader_dcds = info->frame_shaders.dcd_pointer;
+
+      cfg.sample_locations = info->sample_pos_array_pointer;
+#endif
+
+#if PAN_ARCH >= 13
+      /* Enabling prepass without pipelineing is generally not good for
+       * performance, so disable HSR in that case.
+       */
+      cfg.hsr_prepass_enable = info->allow_hsr_prepass &&
+                               pan_fb_can_pipeline_zs(fb);
+      cfg.hsr_prepass_interleaving_enable = pan_fb_can_pipeline_zs(fb);
+      cfg.hsr_prepass_filter_enable = true;
+      cfg.hsr_hierarchical_optimizations_enable = true;
+#endif
+
+#if PAN_ARCH >= 9
+      /* internal_layer_index is used to select the right primitive list in
+       * the tiler context, and frame_arg is the value that's passed to the
+       * fragment shader through r62-r63, which we use to pass gl_Layer. Since
+       * the layer_idx only takes 8-bits, we might use the extra 56-bits we
+       * have in frame_argument to pass other information to the fragment
+       * shader at some point.
+       */
+      assert(info->layer >= info->tiler_ctx->valhall.layer_offset);
+      cfg.internal_layer_index =
+         info->layer - info->tiler_ctx->valhall.layer_offset;
+      cfg.frame_argument = info->layer;
+#endif
+
+      cfg.width = fb->width_px;
+      cfg.height = fb->height_px;
+
+      const struct pan_fb_bbox fb_area_px =
+         pan_fb_bbox_from_xywh(0, 0, info->fb->width_px, info->fb->height_px);
+      assert(pan_fb_bbox_is_valid(info->fb->tiling_area_px));
+      const struct pan_fb_bbox bbox_px =
+         pan_fb_bbox_clamp(info->fb->tiling_area_px, fb_area_px);
+      cfg.bound_min_x = bbox_px.min_x;
+      cfg.bound_min_y = bbox_px.min_y;
+      cfg.bound_max_x = bbox_px.max_x;
+      cfg.bound_max_y = bbox_px.max_y;
+
+      cfg.sample_count = fb->sample_count;
+      cfg.sample_pattern = pan_sample_pattern(fb->sample_count);
+
+      /* Ensure we cover the samples on the edge for 16x MSAA */
+      cfg.tie_break_rule = fb->sample_count == 16 ?
+         MALI_TIE_BREAK_RULE_MINUS_180_OUT_0_IN :
+         MALI_TIE_BREAK_RULE_MINUS_180_IN_0_OUT;
+
+      cfg.effective_tile_size = fb->tile_size_px;
+
+#if PAN_ARCH >= 9
+      cfg.point_sprite_coord_origin_max_y = info->sprite_coord_origin_max_y;
+      cfg.first_provoking_vertex = info->provoking_vertex_first;
+#endif
+
+      assert(fb->rt_count > 0);
+      cfg.render_target_count = fb->rt_count;
+      cfg.color_buffer_allocation = fb->tile_rt_alloc_B;
+
+      if (fb->s_format != PIPE_FORMAT_NONE) {
+         cfg.s_clear =
+            load && pan_target_has_clear(&load->s) ? load->s.clear.stencil : 0;
+         cfg.s_write_enable = store && store->s.store;
+      }
+
+      if (fb->z_format != PIPE_FORMAT_NONE) {
+         cfg.z_internal_format = pan_get_z_internal_format(fb->z_format);
+         cfg.z_clear =
+            load && pan_target_has_clear(&load->z) ? load->z.clear.depth : 0;
+         cfg.z_write_enable = store && store->zs.store;
+      } else {
+         /* Default to 24 bit depth if there's no surface. */
+         cfg.z_internal_format = MALI_Z_INTERNAL_FORMAT_D24;
+
+         /* If we want a stencil store and we don't have depth, that should
+          * happen as a stencil store, not combined depth/stencil.
+          */
+         assert(!store || !store->zs.store);
+      }
+
+      cfg.has_zs_crc_extension = has_zs_crc_ext;
+
+#if PAN_ARCH >= 6
+      cfg.tiler = PAN_ARCH >= 9 ? info->tiler_ctx->valhall.desc
+                                : info->tiler_ctx->bifrost.desc;
+#endif
+   }
+
+#if PAN_ARCH <= 5
+   /* TODO: Midgard tiler */
+   assert(info->tiler_ctx == NULL);
+
+   /* All weights set to 0, nothing to do here */
+   pan_section_pack(&fbd, FRAMEBUFFER, TILER_WEIGHTS, w);
+#endif
+
+   memcpy(out->fbd, &fbd, sizeof(fbd));
+
+   if (has_zs_crc_ext) {
+      emit_zs_crc_desc(info, ct, out->zs_crc);
+   }
+
+   emit_rts(info, out->rts);
+
+   struct mali_framebuffer_pointer_packed tag;
+   pan_pack(&tag, FRAMEBUFFER_POINTER, cfg) {
+      cfg.zs_crc_extension_present = has_zs_crc_ext;
+      cfg.render_target_count = fb->rt_count;
+   }
+   return tag.opaque[0];
+}
+#endif /* PAN_ARCH >= 14 */
+#endif /* PAN_ARCH >= 5 */

src/panfrost/lib/pan_fb.h

@@ -0,0 +1,665 @@
+/*
+ * Copyright (C) 2026 Collabora, Ltd.
+ * Copyright (C) 2026 Arm Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef __PAN_FB_H
+#define __PAN_FB_H
+
+#include "genxml/gen_macros.h"
+#include "util/format/u_formats.h"
+#include "compiler/shader_enums.h"
+
+struct nir_shader;
+struct nir_shader_compiler_options;
+struct pan_fb_info;
+struct pan_image_view;
+
+#define PAN_MAX_RTS 8
+
+/** Bounding box used by the framebuffer abstraction */
+struct pan_fb_bbox {
+   /** Minimum x/y value */
+   uint16_t min_x, min_y;
+
+   /** Maximum x/y value
+    *
+    * Like the hardware, the maximums here are inclusive.  A pixel is
+    * in-bounds if min_x <= x <= max_x and min_y <= y <= max_y.
+    */
+   uint16_t max_x, max_y;
+};
+
+static inline bool
+pan_fb_bbox_is_valid(struct pan_fb_bbox b)
+{
+   return b.min_x <= b.max_x && b.min_y <= b.max_y;
+}
+
+static inline bool
+pan_fb_bbox_contains_bbox(struct pan_fb_bbox a, struct pan_fb_bbox b)
+{
+   return a.min_x <= b.min_x && b.max_x <= a.max_x &&
+          a.min_y <= b.min_y && b.max_y <= a.max_y;
+}
+
+static inline bool
+pan_fb_bbox_equal(struct pan_fb_bbox a, struct pan_fb_bbox b)
+{
+   return a.min_x == b.min_x && b.max_x == a.max_x &&
+          a.min_y == b.min_y && b.max_y == a.max_y;
+}
+
+static inline struct pan_fb_bbox
+pan_fb_bbox_clamp(struct pan_fb_bbox a, struct pan_fb_bbox b)
+{
+   assert(pan_fb_bbox_is_valid(a) && pan_fb_bbox_is_valid(a));
+   return (struct pan_fb_bbox) {
+      .min_x = CLAMP(a.min_x, b.min_x, b.max_x),
+      .min_y = CLAMP(a.min_y, b.min_y, b.max_y),
+      .max_x = CLAMP(a.max_x, b.min_x, b.max_x),
+      .max_y = CLAMP(a.max_y, b.min_y, b.max_y),
+   };
+}
+
+static inline struct pan_fb_bbox
+pan_fb_bbox_align(struct pan_fb_bbox b, uint32_t align_x, uint32_t align_y)
+{
+   assert(util_is_power_of_two_nonzero(align_x));
+   assert(util_is_power_of_two_nonzero(align_y));
+   assert(align_x <= UINT16_MAX && align_y <= UINT16_MAX);
+
+   b.min_x = ROUND_DOWN_TO(b.min_x, align_x);
+   b.min_y = ROUND_DOWN_TO(b.min_y, align_y);
+   b.max_x = align((uint32_t)b.max_x + 1, align_x) - 1;
+   b.max_y = align((uint32_t)b.max_y + 1, align_y) - 1;
+
+   return b;
+}
+
+static inline struct pan_fb_bbox
+pan_fb_bbox_from_xywh(uint32_t x, uint32_t y, uint32_t w, uint32_t h)
+{
+   assert(x + w > x && x + w - 1 <= UINT16_MAX);
+   assert(y + h > y && y + h - 1 <= UINT16_MAX);
+   return (struct pan_fb_bbox) {
+      .min_x = x,
+      .min_y = y,
+      .max_x = x + w - 1,
+      .max_y = y + h - 1,
+   };
+}
+
+/** Framebuffer layout
+ *
+ * This describes the layout of framebuffer in tile memory as well as various
+ * properties that are properties of the framebuffer itself, nit any of the
+ * image views attached to it.
+ */
+struct pan_fb_layout {
+   /** Dimensions of the framebuffer itself, in pixels */
+   uint32_t width_px, height_px;
+
+   /** The render area
+    *
+    * This is the API render area (as defined by Vulkan) and may be smaller
+    * than the actual bounding box of the framebuffer.  The in_bounds_load
+    * and border_load members of pan_fb_load_target are relative to the
+    * render area.
+    */
+   struct pan_fb_bbox render_area_px;
+
+   /** The tiling area
+    *
+    * This is based on the render area but may be larger to accomodate tiling
+    * restrictions.
+    */
+   struct pan_fb_bbox tiling_area_px;
+
+   /** Sample count for this framebuffer */
+   uint8_t sample_count;
+
+   /** Color render target formats
+    *
+    * If a render target is unused, set it to PIPE_FORMAT_NONE.
+    */
+   uint8_t rt_count;
+   enum pipe_format rt_formats[PAN_MAX_RTS];
+
+   /** Depth format for this framebuffer, or PIPE_FORMAT_NONE */
+   enum pipe_format z_format;
+
+   /** Stencil format for this framebuffer, or PIPE_FORMAT_NONE */
+   enum pipe_format s_format;
+
+   /** Amount of space reserved for PLS, in bytes.
+    *
+    * This may overlap the color render targets in tile memory.
+    */
+   uint16_t pls_size_B;
+
+   uint32_t tile_rt_budget_B;
+   uint32_t tile_z_budget_B;
+
+   /** Optimal tile buffer size, in pixels
+    *
+    * This must be a power of two.
+    */
+   unsigned tile_size_px;
+
+   /** Amount of tile memory allocated for color render targets */
+   unsigned tile_rt_alloc_B;
+};
+
+static inline bool
+pan_fb_has_zs(const struct pan_fb_layout *fb)
+{
+   /* An intentional choice is made here to base has_zs on the formats in
+    * the pan_fb_layout and not the stores being done in the end.  PanVK in
+    * particular relies on the size of the final framebuffer descriptor being
+    * the same for all the different variants with incremental rendering.
+    */
+   return fb->z_format != PIPE_FORMAT_NONE ||
+          fb->s_format != PIPE_FORMAT_NONE;
+}
+
+static inline bool
+pan_fb_is_fully_covered(const struct pan_fb_layout *fb)
+{
+   const struct pan_fb_bbox fb_area_px =
+      pan_fb_bbox_from_xywh(0, 0, fb->width_px, fb->height_px);
+   return pan_fb_bbox_equal(fb->render_area_px, fb_area_px);
+}
+
+static inline bool
+pan_fb_has_partial_tiles(const struct pan_fb_layout *fb)
+{
+   assert(pan_fb_bbox_contains_bbox(fb->tiling_area_px, fb->render_area_px));
+   return !pan_fb_bbox_equal(fb->tiling_area_px, fb->render_area_px);
+}
+
+#ifdef PAN_ARCH
+void GENX(pan_select_fb_tile_size)(struct pan_fb_layout *fb);
+#endif
+
+enum ENUM_PACKED pan_fb_load_op {
+   PAN_FB_LOAD_NONE = 0,
+   PAN_FB_LOAD_CLEAR,
+   PAN_FB_LOAD_IMAGE,
+   PAN_FB_LOAD_OP_COUNT,
+};
+
+/** Describes how MSAA is handled when copying
+ *
+ * The source or destination may or may not be multisampled independently and
+ * this controls how that is handled in the copy.
+ */
+enum ENUM_PACKED pan_fb_msaa_copy_op {
+   /** Copies all samples of the source to the destination
+    *
+    * This requires that the source and destination sample counts be the same.
+    * If the source and destination are both single-sampled, this is equivalent
+    * as PAN_FB_MSAA_COPY_SINGLE or PAN_FB_MSAA_COPY_SAMPLE_0.
+    */
+   PAN_FB_MSAA_COPY_ALL = 0,
+
+   /** Copies the single sample from the source to the destination
+    *
+    * This requires that the source be single-sampled.  Semantically, this is
+    * the same as PAN_FB_MSAA_COPY_SAMPLE_0, but it's useful to make this
+    * distinction in shader keys so single and multi-sampled sources can be
+    * differentiated.
+    */
+   PAN_FB_MSAA_COPY_SINGLE,
+
+   /** Copies assuming all source samples are identical
+    *
+    * The actual copy can use any resolve mode because we know a priori that
+    * all samples are identical.  All resolve modes yield the same result.
+    */
+   PAN_FB_MSAA_COPY_IDENTICAL,
+
+   /** Copies sample 0 from the source to all samples in the destination */
+   PAN_FB_MSAA_COPY_SAMPLE_0,
+
+   /** Copies the average of the samples in the source to the destination */
+   PAN_FB_MSAA_COPY_AVERAGE,
+
+   /** Copies the minimum of the samples in the source to the destination */
+   PAN_FB_MSAA_COPY_MIN,
+
+   /** Copies the maximum of the samples in the source to the destination */
+   PAN_FB_MSAA_COPY_MAX,
+
+   PAN_FB_MSAA_COPY_OP_COUNT,
+};
+
+struct pan_fb_load_target {
+   /** Load op for pixels inside the bounding box */
+   enum pan_fb_load_op in_bounds_load;
+
+   /** Load op for border (out of bounds) pixels
+    *
+    * This should almost always be LOAD_OP_LOAD, unless you have a very good
+    * reason to make a different choice.
+    */
+   enum pan_fb_load_op border_load;
+
+   /** Always load, even if no primitives intersect the tile
+    *
+    * This should usually be set for clears.
+    */
+   bool always;
+
+   /** MSAA mode used when load op is IMAGE */
+   enum pan_fb_msaa_copy_op msaa;
+
+   /* Image view to load from, or NULL if neither load op is IMAGE */
+   const struct pan_image_view *iview;
+
+   /** Clear value, if either load op is CLEAR */
+   union {
+      union pipe_color_union color;
+      float depth;
+      uint8_t stencil;
+   } clear;
+};
+
+static inline struct pan_fb_load_target
+pan_fb_load_iview(const struct pan_image_view *iview)
+{
+   return (struct pan_fb_load_target) {
+      .in_bounds_load = PAN_FB_LOAD_IMAGE,
+      .border_load = PAN_FB_LOAD_IMAGE,
+      .msaa = PAN_FB_MSAA_COPY_ALL,
+      .iview = iview,
+   };
+}
+
+struct pan_fb_load {
+   /** Color render target loads */
+   struct pan_fb_load_target rts[PAN_MAX_RTS];
+
+   /** Depth target load */
+   struct pan_fb_load_target z;
+
+   /** Stencil target load */
+   struct pan_fb_load_target s;
+};
+
+static inline bool
+pan_fb_has_image_load(const struct pan_fb_load *load, bool include_border)
+{
+   if (load->z.in_bounds_load == PAN_FB_LOAD_IMAGE ||
+       load->s.in_bounds_load == PAN_FB_LOAD_IMAGE)
+      return true;
+
+   if (include_border &&
+       (load->z.border_load == PAN_FB_LOAD_IMAGE ||
+        load->s.border_load == PAN_FB_LOAD_IMAGE))
+      return true;
+
+   for (unsigned rt = 0; rt < PAN_MAX_RTS; rt++) {
+      if (load->rts[rt].in_bounds_load == PAN_FB_LOAD_IMAGE)
+         return true;
+      if (include_border && load->rts[rt].border_load == PAN_FB_LOAD_IMAGE)
+         return true;
+   }
+
+   return false;
+}
+
+/** Describes a resolve operation
+ *
+ * This describes the source data for the resolve operation.  The way in which
+ * MSAA is handled in the source data is defined by the pan_fb_msaa_copy_op.
+ */
+enum ENUM_PACKED pan_fb_resolve_op {
+   /** Do nothing
+    *
+    * In this mode, the framebuffer is left unaffected by the resolve.  If
+    * necessary (sometimes required for Z/S), the old value will be read and
+    * written back to ensure the final value is the same.
+    */
+   PAN_FB_RESOLVE_NONE = 0,
+
+   /** Load from the specified image view */
+   PAN_FB_RESOLVE_IMAGE,
+
+   /** Load from color render target 0 */
+   PAN_FB_RESOLVE_RT_0,
+   PAN_FB_RESOLVE_RT_1,
+   PAN_FB_RESOLVE_RT_2,
+   PAN_FB_RESOLVE_RT_3,
+   PAN_FB_RESOLVE_RT_4,
+   PAN_FB_RESOLVE_RT_5,
+   PAN_FB_RESOLVE_RT_6,
+   PAN_FB_RESOLVE_RT_7,
+
+   /** Load from the depth target
+    *
+    * From the perspective of resolve ops, Z/S are always separate
+    */
+   PAN_FB_RESOLVE_Z,
+
+   /** Load from the stencil target
+    *
+    * From the perspective of resolve ops, Z/S are always separate
+    */
+   PAN_FB_RESOLVE_S,
+   PAN_FB_RESOLVE_OP_COUNT,
+};
+
+static_assert(PAN_FB_RESOLVE_Z == PAN_FB_RESOLVE_RT_0 + PAN_MAX_RTS,
+              "There are PAN_MAX_RTS many RTs");
+
+#define PAN_FB_RESOLVE_RT(rt) \
+   (assert(0 <= (rt) && (rt) < PAN_MAX_RTS), (PAN_FB_RESOLVE_RT_0 + (rt)))
+
+/** Describes a resovle operation on a given target
+ *
+ * For each side of the render area (in-bounds or border), this defines both
+ * a source to copy from (the resolve op) and a MSAA copy op to apply as part
+ * of the copy.  In the common case, a resolve op will read from itself but
+ * that is not strictly a requirement.  Any resolve op can read from any
+ * render target.
+ */
+struct pan_fb_resolve_target {
+   struct pan_fb_resolve_op_msaa {
+      enum pan_fb_resolve_op resolve;
+      enum pan_fb_msaa_copy_op msaa;
+   } in_bounds, border;
+
+   /** For PAN_FB_RESOLVE_IMAGE the image view to load from */
+   const struct pan_image_view *iview;
+};
+
+/** Describes a resolve operation
+ *
+ * A resolve operation is implemented as a post-frame shader (see
+ * pan_fb_resolve_shader_key_fill) and does a copy from render targets to
+ * render targets, applying MSAA copy ops along the way.
+ */
+struct pan_fb_resolve {
+   struct pan_fb_resolve_target rts[PAN_MAX_RTS];
+   struct pan_fb_resolve_target z, s;
+};
+
+struct pan_fb_store_target {
+   /** Whether or not to do a store */
+   bool store;
+
+   /** Always store, even if no primitives intersect the tile
+    *
+    * clean_tile_write_enable will be set if either store.always is set or
+    * load.always is set for the corresponding load[s].
+    */
+   bool always;
+
+   /** MSAA mode to use when copying to the destination image
+    *
+    * This can only be set to AVERAGE for blendable color formats.
+    */
+   enum pan_fb_msaa_copy_op msaa;
+
+   /** Image view to store to */
+   const struct pan_image_view *iview;
+};
+
+static inline struct pan_fb_store_target
+pan_fb_store_iview(const struct pan_image_view *iview)
+{
+   return (struct pan_fb_store_target) {
+      .store = true,
+      .msaa = PAN_FB_MSAA_COPY_ALL,
+      .iview = iview,
+   };
+}
+
+static inline struct pan_fb_store_target
+pan_fb_always_store_iview_s0(const struct pan_image_view *iview)
+{
+   return (struct pan_fb_store_target) {
+      .store = true,
+      .always = true,
+      .msaa = PAN_FB_MSAA_COPY_SAMPLE_0,
+      .iview = iview,
+   };
+}
+
+struct pan_fb_store {
+   /* Color render target stores */
+   struct pan_fb_store_target rts[PAN_MAX_RTS];
+
+   /* Depth/stencil target store
+    *
+    * If the attached image view has a combined depth/stencil format, both
+    * depth and stencil will be written.
+    */
+   struct pan_fb_store_target zs;
+
+   /* Stencil target store */
+   struct pan_fb_store_target s;
+};
+
+#ifdef PAN_ARCH
+void GENX(pan_align_fb_tiling_area)(struct pan_fb_layout *fb,
+                                    const struct pan_fb_store *store);
+
+void GENX(pan_fb_fold_resolve_into_store)(const struct pan_fb_layout *fb,
+                                          struct pan_fb_resolve *resolve,
+                                          struct pan_fb_store *store);
+#endif
+
+struct pan_fb_frame_shaders {
+   uint64_t dcd_pointer;
+   uint8_t modes[3];
+};
+
+struct pan_fb_desc_info {
+   const struct pan_fb_layout *fb;
+   const struct pan_fb_load *load;
+   const struct pan_fb_store *store;
+
+   struct pan_fb_frame_shaders frame_shaders;
+
+   uint64_t sample_pos_array_pointer;
+
+   /* Only used on Valhal */
+   bool sprite_coord_origin_max_y;
+   bool provoking_vertex_first;
+   bool allow_hsr_prepass;
+
+   uint16_t layer;
+
+   const struct pan_tls_info *tls;
+   const struct pan_tiler_context *tiler_ctx;
+};
+
+#ifdef PAN_ARCH
+void GENX(pan_fill_fb_info)(const struct pan_fb_desc_info *info,
+                            struct pan_fb_info *fbinfo);
+
+struct pan_fb_descs {
+#if PAN_ARCH <= 13
+   struct mali_framebuffer_packed *fbd;
+#endif
+   struct mali_zs_crc_extension_packed *zs_crc;
+   struct mali_rgb_render_target_packed *rts;
+};
+
+uint32_t GENX(pan_emit_fb_desc)(const struct pan_fb_desc_info *info,
+                                const struct pan_fb_descs *out);
+#endif
+
+enum ENUM_PACKED pan_fb_shader_op {
+   /** Preserves the current fragment contents
+    *
+    * The MSAA mode is ignored and, if a load does need to occur to preserve
+    * the current fragment contents, PAN_FB_MSAA_COPY_ALL is implied.
+    *
+    * This is only allowed in resolve shaders so this can always be replaced
+    * with a discard.
+    */
+   PAN_FB_SHADER_PRESERVE = 0,
+
+   /** Do whatever is the fastest (lowest bandwidth)
+    *
+    * This could be a CLEAR or PRESERVE or just write zeros.  It will never
+    * write actual random data but the exact behavior may depend on other bits
+    * in the shader key.
+    */
+   PAN_FB_SHADER_DONT_CARE,
+
+   /** Loads the clear color
+    *
+    * This is only allowed in preload shaders and the clear will also be done
+    * in hardware so it can always be replaced with a discard.
+    */
+   PAN_FB_SHADER_LOAD_CLEAR,
+
+   /** Loads from the image bound at the corresponding gl_frag_result */
+   PAN_FB_SHADER_LOAD_IMAGE,
+
+   PAN_FB_SHADER_COPY_RT_0,
+   PAN_FB_SHADER_COPY_RT_1,
+   PAN_FB_SHADER_COPY_RT_2,
+   PAN_FB_SHADER_COPY_RT_3,
+   PAN_FB_SHADER_COPY_RT_4,
+   PAN_FB_SHADER_COPY_RT_5,
+   PAN_FB_SHADER_COPY_RT_6,
+   PAN_FB_SHADER_COPY_RT_7,
+   PAN_FB_SHADER_COPY_Z,
+   PAN_FB_SHADER_COPY_S,
+
+   PAN_FB_SHADER_OP_COUNT,
+};
+
+static_assert(PAN_FB_SHADER_COPY_Z == PAN_FB_SHADER_COPY_RT_0 + PAN_MAX_RTS,
+              "We should have one PAN_FB_SHADER_COPY_RT_N per RT");
+
+#define PAN_FB_SHADER_COPY_RT(rt) \
+   (assert(0 <= (rt) && (rt) < PAN_MAX_RTS), (PAN_FB_SHADER_COPY_RT_0 + (rt)))
+
+static inline bool
+pan_fb_shader_op_can_discard(enum pan_fb_shader_op op)
+{
+   return op == PAN_FB_SHADER_PRESERVE ||
+          op == PAN_FB_SHADER_DONT_CARE ||
+          op == PAN_FB_SHADER_LOAD_CLEAR;
+}
+
+enum ENUM_PACKED pan_fb_shader_data_type {
+   PAN_FB_SHADER_DATA_TYPE_F32 = 0,
+   PAN_FB_SHADER_DATA_TYPE_I32,
+   PAN_FB_SHADER_DATA_TYPE_U32,
+   PAN_FB_SHADER_DATA_TYPE_COUNT,
+};
+
+static_assert(PAN_FB_SHADER_OP_COUNT <= (1 << 4),
+              "pan_fb_shader_op fits in 4 bits");
+static_assert(PAN_FB_MSAA_COPY_OP_COUNT <= (1 << 3),
+              "pan_fb_msaa_copy_op fits in 3 bits");
+static_assert(PAN_FB_SHADER_DATA_TYPE_COUNT <= (1 << 2),
+              "pan_fb_shader_data_type fits in 2 bits");
+
+/* Asserts for glsl_sampler_dim glsl_base_type have to be runtime because
+ * there is no MAX value we can use.
+ */
+
+PRAGMA_DIAGNOSTIC_PUSH
+PRAGMA_DIAGNOSTIC_ERROR(-Wpadded)
+struct pan_fb_shader_key_target {
+   uint32_t in_bounds_op : 4;
+   uint32_t border_op : 4;
+   uint32_t in_bounds_msaa : 3;
+   uint32_t border_msaa : 3;
+   uint32_t sample0_only : 1;
+   uint32_t image_dim : 2;
+   uint32_t image_is_array : 1;
+   uint32_t image_samples_log2 : 3;
+   uint32_t data_type : 2;
+   uint32_t pad : 9;
+};
+PRAGMA_DIAGNOSTIC_POP
+static_assert(sizeof(struct pan_fb_shader_key_target) == 4,
+              "This struct has no holes");
+
+/** Whether or not the given target is written by the FB shader
+ *
+ * GENX(pan_get_fb_shader)() guarantees that if both in_bounds_op and
+ * border_op can be discarded, that it does nothing with that target.  This
+ * way the full outputs_written can be known from just the shader key.
+ */
+static inline bool
+pan_fb_shader_key_target_written(const struct pan_fb_shader_key_target *target)
+{
+   return !pan_fb_shader_op_can_discard(target->in_bounds_op) ||
+          !pan_fb_shader_op_can_discard(target->border_op);
+}
+
+PRAGMA_DIAGNOSTIC_PUSH
+PRAGMA_DIAGNOSTIC_ERROR(-Wpadded)
+struct pan_fb_shader_key {
+   struct pan_fb_shader_key_target rts[PAN_MAX_RTS];
+   struct pan_fb_shader_key_target z, s;
+   uint16_t z_format;
+   uint8_t fb_sample_count;
+   uint8_t _pad;
+};
+PRAGMA_DIAGNOSTIC_POP
+static_assert(sizeof(struct pan_fb_shader_key) == 4 * (PAN_MAX_RTS + 3),
+              "This struct has no holes");
+
+#ifdef PAN_ARCH
+bool GENX(pan_fb_load_shader_key_fill)(struct pan_fb_shader_key *key,
+                                       const struct pan_fb_layout *fb,
+                                       const struct pan_fb_load *load,
+                                       bool zs_prepass);
+
+#if PAN_ARCH >= 5
+struct pan_fb_clean_tile {
+   uint8_t rts;
+   bool zs, s;
+};
+
+struct pan_fb_clean_tile
+   GENX(pan_fb_get_clean_tile)(const struct pan_fb_desc_info *info);
+
+static inline bool
+pan_target_has_clear(const struct pan_fb_load_target *target)
+{
+   return target->in_bounds_load == PAN_FB_LOAD_CLEAR ||
+          target->border_load == PAN_FB_LOAD_CLEAR;
+}
+#endif /* PAN_ARCH >= 5 */
+
+#if PAN_ARCH >= 6
+bool GENX(pan_fb_resolve_shader_key_fill)(struct pan_fb_shader_key *key,
+                                          const struct pan_fb_layout *fb,
+                                          const struct pan_fb_resolve *resolve);
+#endif /* PAN_ARCH >= 6 */
+
+struct nir_shader *
+GENX(pan_get_fb_shader)(const struct pan_fb_shader_key *key,
+                        const struct nir_shader_compiler_options *nir_options);
+
+#if PAN_ARCH >= 13
+/**
+ * Returns true if there's enough space in the tile buffer for at least two
+ * Z/S tiles.
+ */
+static inline bool
+pan_fb_can_pipeline_zs(const struct pan_fb_layout *fb)
+{
+   const uint32_t z_B_per_px = sizeof(float) * fb->sample_count;
+   const uint32_t z_B_per_tile = z_B_per_px * fb->tile_size_px;
+
+   /* The budget is already half the available Z space */
+   return z_B_per_tile < fb->tile_z_budget_B;
+}
+#endif
+#endif /* PAN_ARCH */
+
+#endif /* __PAN_FB_H */

src/panfrost/lib/pan_fb_nir.c

@@ -0,0 +1,859 @@
+/*
+ * Copyright (C) 2026 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "pan_blend.h"
+#include "pan_desc.h"
+#include "pan_fb.h"
+
+#include "compiler/nir/nir.h"
+#include "compiler/nir/nir_builder.h"
+#include "compiler/pan_nir.h"
+
+static enum glsl_sampler_dim
+mali_to_glsl_dim(enum mali_texture_dimension dim)
+{
+   switch (dim) {
+   case MALI_TEXTURE_DIMENSION_1D: return GLSL_SAMPLER_DIM_1D;
+   case MALI_TEXTURE_DIMENSION_CUBE:
+   case MALI_TEXTURE_DIMENSION_2D: return GLSL_SAMPLER_DIM_2D;
+   case MALI_TEXTURE_DIMENSION_3D: return GLSL_SAMPLER_DIM_3D;
+   default:
+      UNREACHABLE("Unknown mali_texture_dimension");
+   }
+}
+
+static enum pan_fb_shader_data_type
+data_type_for_format(enum pipe_format format)
+{
+   if (util_format_is_pure_uint(format))
+      return PAN_FB_SHADER_DATA_TYPE_U32;
+   else if (util_format_is_pure_sint(format))
+      return PAN_FB_SHADER_DATA_TYPE_I32;
+   else
+      return PAN_FB_SHADER_DATA_TYPE_F32;
+}
+
+static nir_alu_type
+nir_alu_type_for_data_type(enum pan_fb_shader_data_type data_type)
+{
+   switch (data_type) {
+   case PAN_FB_SHADER_DATA_TYPE_F32:   return nir_type_float32;
+   case PAN_FB_SHADER_DATA_TYPE_I32:   return nir_type_int32;
+   case PAN_FB_SHADER_DATA_TYPE_U32:   return nir_type_uint32;
+   default:
+      UNREACHABLE("Invalid pan_fb_shader_data_type");
+   }
+}
+
+static enum pan_fb_shader_op
+get_shader_op_for_load(enum pan_fb_load_op op)
+{
+   switch (op) {
+   case PAN_FB_LOAD_NONE:     return PAN_FB_SHADER_DONT_CARE;
+   case PAN_FB_LOAD_CLEAR:    return PAN_FB_SHADER_LOAD_CLEAR;
+   case PAN_FB_LOAD_IMAGE:    return PAN_FB_SHADER_LOAD_IMAGE;
+   default:
+      UNREACHABLE("Invalid load op");
+   }
+}
+
+static inline enum pan_fb_shader_op
+get_shader_op_for_resolve(enum pan_fb_resolve_op op)
+{
+   switch (op) {
+   case PAN_FB_RESOLVE_NONE:  return PAN_FB_SHADER_PRESERVE;
+   case PAN_FB_RESOLVE_IMAGE: return PAN_FB_SHADER_LOAD_IMAGE;
+   case PAN_FB_RESOLVE_RT_0:  return PAN_FB_SHADER_COPY_RT_0;
+   case PAN_FB_RESOLVE_RT_1:  return PAN_FB_SHADER_COPY_RT_1;
+   case PAN_FB_RESOLVE_RT_2:  return PAN_FB_SHADER_COPY_RT_2;
+   case PAN_FB_RESOLVE_RT_3:  return PAN_FB_SHADER_COPY_RT_3;
+   case PAN_FB_RESOLVE_RT_4:  return PAN_FB_SHADER_COPY_RT_4;
+   case PAN_FB_RESOLVE_RT_5:  return PAN_FB_SHADER_COPY_RT_5;
+   case PAN_FB_RESOLVE_RT_6:  return PAN_FB_SHADER_COPY_RT_6;
+   case PAN_FB_RESOLVE_RT_7:  return PAN_FB_SHADER_COPY_RT_7;
+   case PAN_FB_RESOLVE_Z:     return PAN_FB_SHADER_COPY_Z;
+   case PAN_FB_RESOLVE_S:     return PAN_FB_SHADER_COPY_S;
+   default:
+      UNREACHABLE("Invalid resolve op");
+   }
+}
+
+static enum pan_fb_msaa_copy_op
+reduce_msaa_op(enum pan_fb_msaa_copy_op msaa, enum pan_fb_shader_op op,
+               uint8_t fb_sample_count, uint8_t image_sample_count)
+{
+   if (pan_fb_shader_op_can_discard(op))
+      return PAN_FB_MSAA_COPY_ALL;
+
+   const uint8_t src_sample_count = op == PAN_FB_SHADER_LOAD_IMAGE
+                                    ? image_sample_count : fb_sample_count;
+   if (msaa == PAN_FB_MSAA_COPY_ALL)
+      assert(src_sample_count == fb_sample_count);
+
+   if (src_sample_count <= 1)
+      return PAN_FB_MSAA_COPY_SINGLE;
+
+   return msaa;
+}
+
+static bool
+op_needs_sample_count(enum pan_fb_shader_op op,
+                      enum pan_fb_msaa_copy_op msaa)
+{
+   return msaa == PAN_FB_MSAA_COPY_AVERAGE ||
+          msaa == PAN_FB_MSAA_COPY_MIN ||
+          msaa == PAN_FB_MSAA_COPY_MAX;
+}
+
+static const struct pan_fb_shader_key_target key_target_dont_care = {
+   .in_bounds_op = PAN_FB_SHADER_DONT_CARE,
+   .border_op = PAN_FB_SHADER_DONT_CARE,
+};
+
+static const struct pan_fb_shader_key_target key_target_preserve = {
+   .in_bounds_op = PAN_FB_SHADER_PRESERVE,
+   .border_op = PAN_FB_SHADER_PRESERVE,
+};
+
+static struct pan_fb_shader_key_target
+get_key_target(enum pipe_format format,
+               uint8_t fb_sample_count,
+               bool has_border,
+               enum pan_fb_shader_op in_bounds_op,
+               enum pan_fb_shader_op border_op,
+               enum pan_fb_msaa_copy_op in_bounds_msaa,
+               enum pan_fb_msaa_copy_op border_msaa,
+               bool sample0_only,
+               const struct pan_image_view *iview)
+{
+   if (format == PIPE_FORMAT_NONE)
+      return key_target_dont_care;
+
+   /* If we have a full framebuffer, there is no border.  Set the boarder load
+    * equal to the in-bounds load so things in the shader fold nicely.
+    */
+   if (!has_border) {
+      border_op = in_bounds_op;
+      border_msaa = in_bounds_msaa;
+   }
+
+   if (in_bounds_op == PAN_FB_SHADER_DONT_CARE &&
+       border_op == PAN_FB_SHADER_DONT_CARE)
+      return key_target_dont_care;
+
+   /* If both load ops can be implemented as a discard, set the target to
+    * always PRESERVE and we just won't emit any code to do a load.
+    *
+    * Importantly, this is true for all load ops except PAN_FB_LOAD_OP_IMAGE.
+    */
+   if (pan_fb_shader_op_can_discard(in_bounds_op) &&
+       pan_fb_shader_op_can_discard(border_op))
+      return key_target_preserve;
+
+   /* We don't have a bound on this enum so we need a live assert */
+   enum mali_texture_dimension dim = MALI_TEXTURE_DIMENSION_2D;
+   bool is_array = false;
+   uint8_t image_sample_count = 0;
+   if (in_bounds_op == PAN_FB_SHADER_LOAD_IMAGE ||
+       border_op == PAN_FB_SHADER_LOAD_IMAGE) {
+      if (util_format_has_depth(util_format_description(format)))
+         assert(util_format_get_depth_bits(format) ==
+                util_format_get_depth_bits(iview->format));
+      else if (util_format_has_stencil(util_format_description(format)))
+         assert(util_format_has_stencil(util_format_description(iview->format)));
+      else
+         assert(format == iview->format);
+
+      dim = iview->dim;
+      is_array = iview->first_layer != iview->last_layer;
+      image_sample_count = pan_image_view_get_nr_samples(iview);
+   }
+
+   in_bounds_msaa = reduce_msaa_op(in_bounds_msaa, in_bounds_op,
+                                   fb_sample_count, image_sample_count);
+   border_msaa = reduce_msaa_op(border_msaa, border_op,
+                                fb_sample_count, image_sample_count);
+
+   const bool needs_sample_count =
+      op_needs_sample_count(in_bounds_op, in_bounds_msaa) ||
+      op_needs_sample_count(border_op, border_msaa);
+
+   assert(((unsigned)dim) < (1 << 2));
+
+   return (struct pan_fb_shader_key_target) {
+      .in_bounds_op = in_bounds_op,
+      .border_op = border_op,
+      .in_bounds_msaa = in_bounds_msaa,
+      .border_msaa = border_msaa,
+      .sample0_only = sample0_only,
+      .image_dim = dim,
+      .image_is_array = is_array,
+      .image_samples_log2 = needs_sample_count
+                            ? util_logbase2(image_sample_count) : 0,
+      .data_type = data_type_for_format(format),
+   };
+}
+
+static struct pan_fb_shader_key_target
+get_load_key_target(enum pipe_format format,
+                    uint8_t fb_sample_count,
+                    bool has_border,
+                    const struct pan_fb_load_target *load)
+{
+   return get_key_target(format, fb_sample_count, has_border,
+                         get_shader_op_for_load(load->in_bounds_load),
+                         get_shader_op_for_load(load->border_load),
+                         load->msaa, load->msaa, false, load->iview);
+}
+
+bool
+GENX(pan_fb_load_shader_key_fill)(struct pan_fb_shader_key *key,
+                                  const struct pan_fb_layout *fb,
+                                  const struct pan_fb_load *load,
+                                  bool zs_prepass)
+{
+   *key = (struct pan_fb_shader_key) { };
+
+   const bool has_border = pan_fb_has_partial_tiles(fb);
+
+   /* Z/S can only be written from the prepass and color can only be written
+    * from the non-prepass.  Setting everything to zero will cause the shader
+    * to just not write anything in that case.
+    */
+   if (zs_prepass) {
+      key->z = get_load_key_target(fb->z_format, fb->sample_count,
+                                   has_border, &load->z);
+      key->s = get_load_key_target(fb->s_format, fb->sample_count,
+                                   has_border, &load->s);
+      return pan_fb_shader_key_target_written(&key->z) ||
+             pan_fb_shader_key_target_written(&key->s);
+   } else {
+      bool needs_shader = false;
+      for (unsigned rt = 0; rt < fb->rt_count; rt++) {
+         key->rts[rt] = get_load_key_target(fb->rt_formats[rt],
+                                            fb->sample_count,
+                                            has_border, &load->rts[rt]);
+         if (pan_fb_shader_key_target_written(&key->rts[rt]))
+            needs_shader = true;
+      }
+      return needs_shader;
+   }
+}
+
+#if PAN_ARCH >= 6
+static struct pan_fb_shader_key_target
+get_resolve_key_target(enum pipe_format format,
+                       uint8_t fb_sample_count,
+                       bool has_border,
+                       const struct pan_fb_resolve_target *load)
+{
+   const enum pan_fb_shader_op in_bounds_op =
+      get_shader_op_for_resolve(load->in_bounds.resolve);
+   const enum pan_fb_shader_op border_op =
+      get_shader_op_for_resolve(load->border.resolve);
+   return get_key_target(format, fb_sample_count, has_border,
+                         in_bounds_op, border_op,
+                         load->in_bounds.msaa, load->border.msaa,
+                         true, load->iview);
+}
+
+bool
+GENX(pan_fb_resolve_shader_key_fill)(struct pan_fb_shader_key *key,
+                                     const struct pan_fb_layout *fb,
+                                     const struct pan_fb_resolve *resolve)
+{
+   const bool has_border = pan_fb_has_partial_tiles(fb);
+
+   *key = (struct pan_fb_shader_key) {
+      .z = get_resolve_key_target(fb->z_format, fb->sample_count,
+                                  has_border, &resolve->z),
+      .s = get_resolve_key_target(fb->s_format, fb->sample_count,
+                                  has_border, &resolve->s),
+      .z_format = fb->z_format,
+      .fb_sample_count = fb->sample_count,
+   };
+   bool needs_shader = pan_fb_shader_key_target_written(&key->z) ||
+                       pan_fb_shader_key_target_written(&key->s);
+
+   for (unsigned rt = 0; rt < fb->rt_count; rt++) {
+      key->rts[rt] = get_resolve_key_target(fb->rt_formats[rt],
+                                            fb->sample_count,
+                                            has_border,
+                                            &resolve->rts[rt]);
+      if (pan_fb_shader_key_target_written(&key->rts[rt]))
+         needs_shader = true;
+   }
+
+   return needs_shader;
+}
+#endif
+
+static nir_def *
+combine_samples_no_div(nir_builder *b, nir_def **samples, uint8_t sample_count,
+                       const nir_alu_type nir_type,
+                       enum pan_fb_msaa_copy_op msaa)
+{
+   assert(util_is_power_of_two_nonzero(sample_count));
+   if (sample_count == 1)
+      return samples[0];
+
+   nir_def *lo = combine_samples_no_div(b, samples,
+                                        sample_count / 2, nir_type, msaa);
+   nir_def *hi = combine_samples_no_div(b, samples + (sample_count / 2),
+                                        sample_count / 2, nir_type, msaa);
+
+   /* We assume that first half always comes before the second so setting the
+    * cursor after the second half combine instruction will give us the least
+    * common ancestor.
+    */
+   b->cursor = nir_after_instr(nir_def_instr(hi));
+
+   switch (msaa) {
+   case PAN_FB_MSAA_COPY_AVERAGE:
+      assert(nir_alu_type_get_base_type(nir_type) == nir_type_float);
+      return nir_fadd(b, lo, hi);
+
+   case PAN_FB_MSAA_COPY_MIN:
+      switch (nir_alu_type_get_base_type(nir_type)) {
+      case nir_type_float:
+         return nir_fmin(b, lo, hi);
+      case nir_type_uint:
+         return nir_umin(b, lo, hi);
+      case nir_type_int:
+         return nir_imin(b, lo, hi);
+      default:
+         UNREACHABLE("Unsupported NIR type");
+      }
+
+   case PAN_FB_MSAA_COPY_MAX:
+      switch (nir_alu_type_get_base_type(nir_type)) {
+      case nir_type_float:
+         return nir_fmax(b, lo, hi);
+      case nir_type_uint:
+         return nir_umax(b, lo, hi);
+      case nir_type_int:
+         return nir_imax(b, lo, hi);
+      default:
+         UNREACHABLE("Unsupported NIR type");
+      }
+
+   default:
+      UNREACHABLE("Invalid MSAA op");
+   }
+}
+
+static nir_def *
+combine_samples(nir_builder *b, nir_def **samples, uint8_t sample_count,
+                const nir_alu_type nir_type, enum pan_fb_msaa_copy_op msaa)
+{
+   if (msaa == PAN_FB_MSAA_COPY_SAMPLE_0)
+      return samples[0];
+
+   nir_def *val = combine_samples_no_div(b, samples, sample_count,
+                                         nir_type, msaa);
+   if (msaa == PAN_FB_MSAA_COPY_AVERAGE)
+      val = nir_fdiv_imm(b, val, sample_count);
+
+   return val;
+}
+
+static nir_def *
+build_sample_id(nir_builder *b)
+{
+   b->shader->info.fs.uses_sample_shading = true;
+   return nir_load_sample_id(b);
+}
+
+static nir_def *
+build_image_load(nir_builder *b, const nir_alu_type nir_type,
+                 nir_def *pos, gl_frag_result location,
+                 enum pan_fb_msaa_copy_op msaa, uint8_t sample_count,
+                 enum mali_texture_dimension dim, bool is_array)
+{
+   assert(pos->num_components == 3);
+   switch (dim) {
+   case MALI_TEXTURE_DIMENSION_1D:
+      if (is_array)
+         pos = nir_channels(b, pos, 0b101);
+      else
+         pos = nir_channel(b, pos, 0);
+      break;
+
+   case MALI_TEXTURE_DIMENSION_CUBE:
+      assert(is_array);
+      break;
+
+   case MALI_TEXTURE_DIMENSION_2D:
+      if (!is_array)
+         pos = nir_channels(b, pos, 0b011);
+      break;
+
+   case MALI_TEXTURE_DIMENSION_3D:
+      break;
+
+   default:
+      UNREACHABLE("Unsupported dim");
+   }
+
+   nir_def *val;
+   switch (msaa) {
+   case PAN_FB_MSAA_COPY_SINGLE:
+      val = nir_txf(b, pos,
+                    .texture_index = location,
+                    .dim = mali_to_glsl_dim(dim),
+                    .dest_type = nir_type,
+                    .is_array = is_array);
+      break;
+
+   case PAN_FB_MSAA_COPY_ALL:
+   case PAN_FB_MSAA_COPY_IDENTICAL:
+   case PAN_FB_MSAA_COPY_SAMPLE_0: {
+      assert(dim == MALI_TEXTURE_DIMENSION_2D);
+
+      nir_def *sample_id = msaa == PAN_FB_MSAA_COPY_ALL
+                           ? build_sample_id(b) : nir_imm_int(b, 0);
+
+      val = nir_txf_ms(b, pos, sample_id,
+                       .texture_index = location,
+                       .dim = GLSL_SAMPLER_DIM_MS,
+                       .dest_type = nir_type,
+                       .is_array = is_array);
+      break;
+   }
+
+   case PAN_FB_MSAA_COPY_AVERAGE:
+   case PAN_FB_MSAA_COPY_MIN:
+   case PAN_FB_MSAA_COPY_MAX: {
+      assert(dim == MALI_TEXTURE_DIMENSION_2D);
+      assert(sample_count > 0);
+
+      nir_def *samples[16];
+      assert(sample_count <= ARRAY_SIZE(samples));
+      for (uint32_t s = 0; s < sample_count; s++) {
+         samples[s] = nir_txf_ms(b, pos, nir_imm_int(b, s),
+                                 .texture_index = location,
+                                 .dim = GLSL_SAMPLER_DIM_MS,
+                                 .dest_type = nir_type,
+                                 .is_array = is_array);
+      }
+      val = combine_samples(b, samples, sample_count, nir_type, msaa);
+      break;
+   }
+
+   default:
+      UNREACHABLE("Invalid copy op");
+   }
+
+   if (location == FRAG_RESULT_DEPTH || location == FRAG_RESULT_STENCIL)
+      val = nir_channel(b, val, 0);
+
+   return val;
+}
+
+#if PAN_ARCH >= 6
+static uint32_t
+get_fb_conversion(nir_alu_type nir_type, enum pipe_format format)
+{
+#if PAN_ARCH >= 9
+   return 0;
+#else
+   struct mali_internal_conversion_packed conv;
+   pan_pack(&conv, INTERNAL_CONVERSION, cfg) {
+      cfg.register_format = pan_blend_type_from_nir(nir_type);
+      cfg.memory_format = GENX(pan_dithered_format_from_pipe_format)(
+         format, false);
+   }
+   return conv.opaque[0];
+#endif
+}
+
+static nir_def *
+build_single_fb_load(nir_builder *b, const nir_alu_type nir_type,
+                     gl_frag_result location, nir_def *sample,
+                     enum pipe_format z_format)
+{
+   const bool is_zs = location == FRAG_RESULT_DEPTH ||
+                      location == FRAG_RESULT_STENCIL;
+   unsigned num_components = is_zs ? 1 : 4;
+
+   nir_def *conversion;
+   if (location == FRAG_RESULT_DEPTH) {
+      conversion = nir_imm_int(b, get_fb_conversion(nir_type, z_format));
+   } else if (location == FRAG_RESULT_STENCIL) {
+      conversion = nir_imm_int(b,
+         get_fb_conversion(nir_type, PIPE_FORMAT_S8_UINT));
+   } else {
+      assert(location >= FRAG_RESULT_DATA0);
+      uint8_t rt = location - FRAG_RESULT_DATA0;
+      assert(rt < PAN_MAX_RTS);
+
+      conversion = nir_unpack_64_2x32_split_y(b,
+         nir_load_blend_descriptor_pan(b, .base = rt));
+   }
+
+   nir_def *val = nir_load_tile_pan(b,
+      num_components, 32,
+      pan_nir_tile_location_sample(b, location, sample),
+      pan_nir_tile_default_coverage(b),
+      conversion,
+      .dest_type = nir_type,
+      .io_semantics.location = location,
+      .io_semantics.num_slots = 1);
+
+   /* If we loaded the stencil value, the upper 24 bits might contain garbage,
+    * so we need to mask that off before we do any math on the result.
+    */
+   if (location == FRAG_RESULT_STENCIL)
+      val = nir_iand_imm(b, val, 0xff);
+
+   return val;
+}
+
+static nir_def *
+build_fb_load(nir_builder *b, const nir_alu_type nir_type,
+              gl_frag_result location, enum pipe_format z_format,
+              enum pan_fb_msaa_copy_op msaa, uint8_t fb_sample_count)
+{
+   switch (msaa) {
+   case PAN_FB_MSAA_COPY_SINGLE:
+   case PAN_FB_MSAA_COPY_ALL:
+   case PAN_FB_MSAA_COPY_IDENTICAL:
+   case PAN_FB_MSAA_COPY_SAMPLE_0: {
+      nir_def *sample_id = msaa == PAN_FB_MSAA_COPY_ALL
+                           ? build_sample_id(b) : nir_imm_int(b, 0);
+
+      return build_single_fb_load(b, nir_type, location, sample_id, z_format);
+   }
+
+   case PAN_FB_MSAA_COPY_AVERAGE:
+   case PAN_FB_MSAA_COPY_MIN:
+   case PAN_FB_MSAA_COPY_MAX: {
+      nir_def *samples[16];
+      assert(fb_sample_count > 0 && fb_sample_count <= ARRAY_SIZE(samples));
+      for (uint32_t s = 0; s < fb_sample_count; s++) {
+         samples[s] = build_single_fb_load(b, nir_type, location,
+                                           nir_imm_int(b, s), z_format);
+      }
+      return combine_samples(b, samples, fb_sample_count, nir_type, msaa);
+   }
+
+   default:
+      UNREACHABLE("Invalid copy op");
+   }
+}
+#endif
+
+static nir_def *
+build_load(nir_builder *b, nir_def *pos,
+           enum pan_fb_shader_op op, enum pan_fb_msaa_copy_op msaa,
+           gl_frag_result location,
+           const struct pan_fb_shader_key_target *target,
+           const struct pan_fb_shader_key *key)
+{
+   const nir_alu_type nir_type = nir_alu_type_for_data_type(target->data_type);
+   const bool is_zs = location == FRAG_RESULT_DEPTH ||
+                      location == FRAG_RESULT_STENCIL;
+
+   switch (op) {
+#if PAN_ARCH >= 6
+   case PAN_FB_SHADER_PRESERVE:
+      return build_fb_load(b, nir_type, location, key->z_format,
+                           PAN_FB_MSAA_COPY_ALL, key->fb_sample_count);
+#endif
+
+   case PAN_FB_SHADER_DONT_CARE:
+      return nir_imm_zero(b, is_zs ? 1 : 4, 32);
+
+   case PAN_FB_SHADER_LOAD_CLEAR:
+      return nir_load_clear_value_pan(b, is_zs ? 1 : 4, 32,
+                                      .io_semantics.location = location,
+                                      .dest_type = nir_type);
+
+   case PAN_FB_SHADER_LOAD_IMAGE:
+      return build_image_load(b, nir_type, pos, location, msaa,
+                              1 << target->image_samples_log2,
+                              target->image_dim, target->image_is_array);
+
+#if PAN_ARCH >= 6
+   default: { /* PAN_FB_SHADER_COPY_RT_N */
+      uint8_t rt = op - PAN_FB_SHADER_COPY_RT_0;
+      assert(rt < PAN_MAX_RTS);
+      return build_fb_load(b, nir_type, FRAG_RESULT_DATA0 + rt,
+                           PIPE_FORMAT_NONE, msaa, key->fb_sample_count);
+   }
+
+   case PAN_FB_SHADER_COPY_Z:
+      return build_fb_load(b, nir_type, FRAG_RESULT_DEPTH,
+                           key->z_format, msaa, key->fb_sample_count);
+
+   case PAN_FB_SHADER_COPY_S:
+      return build_fb_load(b, nir_type, FRAG_RESULT_STENCIL,
+                           PIPE_FORMAT_S8_UINT, msaa, key->fb_sample_count);
+#else
+   default:
+      UNREACHABLE("Unsupported shader op");
+#endif
+   }
+}
+
+static void
+build_op(nir_builder *b, nir_def *pos, nir_def *in_bounds,
+         gl_frag_result location,
+         const struct pan_fb_shader_key_target *target,
+         const struct pan_fb_shader_key *key)
+{
+   nir_if *nif_s0 = NULL;
+   if (target->sample0_only)
+      nif_s0 = nir_push_if(b, nir_ieq_imm(b, build_sample_id(b), 0));
+
+   nir_def *val;
+   if (target->in_bounds_op == target->border_op &&
+       target->in_bounds_msaa == target->border_msaa) {
+      val = build_load(b, pos, target->in_bounds_op, target->in_bounds_msaa,
+                       location, target, key);
+   } else {
+      nir_def *in_bounds_val, *border_val;
+      nir_if *nif = nir_push_if(b, in_bounds);
+      {
+         in_bounds_val = build_load(b, pos, target->in_bounds_op,
+                                    target->in_bounds_msaa,
+                                    location, target, key);
+      }
+      nir_push_else(b, nif);
+      {
+         border_val = build_load(b, pos, target->border_op,
+                                 target->border_msaa,
+                                 location, target, key);
+      }
+      nir_pop_if(b, nif);
+      val = nir_if_phi(b, in_bounds_val, border_val);
+   }
+
+   nir_store_output(b, val, nir_imm_int(b, 0),
+                    .base = location, .range = 1,
+                    .write_mask = nir_component_mask(val->num_components),
+                    .src_type = nir_alu_type_for_data_type(target->data_type),
+                    .io_semantics.location = location,
+                    .io_semantics.num_slots = 1);
+
+   if (nif_s0)
+      nir_pop_if(b, nif_s0);
+}
+
+struct pan_fb_shader_info {
+   bool discard_in_bounds;
+   bool discard_border;
+   bool sample0_only;
+};
+
+static void
+gather_target_info(struct pan_fb_shader_info *info,
+                   const struct pan_fb_shader_key_target *target)
+{
+   /* Ignore any targets we don't write */
+   if (!pan_fb_shader_key_target_written(target))
+      return;
+
+   if (!pan_fb_shader_op_can_discard(target->in_bounds_op))
+      info->discard_in_bounds = false;
+
+   if (!pan_fb_shader_op_can_discard(target->border_op))
+      info->discard_border = false;
+
+   if (!target->sample0_only)
+      info->sample0_only = false;
+}
+
+static struct pan_fb_shader_info
+gather_shader_info(const struct pan_fb_shader_key *key)
+{
+   struct pan_fb_shader_info info = {
+      .discard_in_bounds = true,
+      .discard_border = true,
+      .sample0_only = true,
+   };
+
+   for (unsigned rt = 0; rt < PAN_MAX_RTS; rt++)
+      gather_target_info(&info, &key->rts[rt]);
+
+   gather_target_info(&info, &key->z);
+   gather_target_info(&info, &key->s);
+
+   return info;
+}
+
+static bool
+opt_sample0_only_intr(nir_builder *b, nir_intrinsic_instr *intr, void *data)
+{
+   switch (intr->intrinsic) {
+   case nir_intrinsic_load_sample_id:
+      b->cursor = nir_before_instr(&intr->instr);
+      nir_def_replace(&intr->def, nir_imm_int(b, 0));
+      return true;
+
+   case nir_intrinsic_load_cumulative_coverage_pan:
+      b->cursor = nir_before_instr(&intr->instr);
+      nir_def_replace(&intr->def, nir_imm_int(b, 1));
+      return true;
+
+   default:
+      return false;
+   }
+}
+
+nir_shader *
+GENX(pan_get_fb_shader)(const struct pan_fb_shader_key *key,
+                        const struct nir_shader_compiler_options *nir_options)
+{
+   nir_builder builder = nir_builder_init_simple_shader(
+      MESA_SHADER_FRAGMENT, nir_options, "pan-fb-load");
+   nir_builder *b = &builder;
+
+   const struct pan_fb_shader_info info = gather_shader_info(key);
+
+   /* We shouldn't even be building a shader at all in this case but, on the
+    * off chance that we do, build one that just discards.
+    */
+   if (info.discard_in_bounds && info.discard_border) {
+      nir_terminate(b);
+      return builder.shader;
+   }
+
+   nir_def *pos_xy = nir_load_pixel_coord(b);
+   nir_def *pos_x = nir_u2u32(b, nir_channel(b, pos_xy, 0));
+   nir_def *pos_y = nir_u2u32(b, nir_channel(b, pos_xy, 1));
+   nir_def *pos = nir_vec3(b, pos_x, pos_y, nir_load_layer_id(b));
+
+   nir_def *ra = nir_load_fb_render_area_pan(b);
+   nir_def *ra_min_xy = nir_channels(b, ra, 0b0011);
+   nir_def *ra_max_xy = nir_channels(b, ra, 0b1100);
+
+   nir_def *in_bounds =
+      nir_ball(b, nir_iand(b, nir_uge(b, pos_xy, ra_min_xy),
+                              nir_uge(b, ra_max_xy, pos_xy)));
+
+   if (PAN_ARCH >= 6 && !pan_fb_shader_key_target_written(&key->z) &&
+                        !pan_fb_shader_key_target_written(&key->s)) {
+      /* We assume that the framebuffer will clear if either load op is set to
+       * clear.  For color-only on bifrost, we can do a bit better by emitting
+       * BLEND instructions directly only writing the tile buffer if we're
+       * actually loading the image.
+       */
+      for (unsigned rt = 0; rt < PAN_MAX_RTS; rt++) {
+         const struct pan_fb_shader_key_target *target = &key->rts[rt];
+         const gl_frag_result location = FRAG_RESULT_DATA0 + rt;
+
+         if (!pan_fb_shader_key_target_written(target))
+            continue;
+
+         nir_def *coverage = nir_load_cumulative_coverage_pan(b);
+         nir_def *z1 = nir_imm_int(b, 0);
+         nir_def *u4 = nir_undef(b, 4, 32);
+
+         nir_if *nif_s0 = NULL;
+         if (target->sample0_only)
+            nif_s0 = nir_push_if(b, nir_ieq_imm(b, build_sample_id(b), 0));
+
+         nir_def *color;
+         if (target->in_bounds_op == target->border_op &&
+             target->in_bounds_msaa == target->border_msaa) {
+            color = build_load(b, pos, target->in_bounds_op,
+                               target->in_bounds_msaa,
+                               location, target, key);
+         } else {
+            nir_def *ib_color, *bd_color, *ib_cov, *bd_cov;
+            nir_if *nif = nir_push_if(b, in_bounds);
+            {
+               if (pan_fb_shader_op_can_discard(target->in_bounds_op)) {
+                  ib_color = u4;
+                  ib_cov = z1;
+               } else {
+                  ib_color = build_load(b, pos, target->in_bounds_op,
+                                        target->in_bounds_msaa,
+                                        location, target, key);
+                  ib_cov = coverage;
+               }
+            }
+            nir_push_else(b, nif);
+            {
+               if (pan_fb_shader_op_can_discard(target->border_op)) {
+                  bd_color = u4;
+                  bd_cov = z1;
+               } else {
+                  bd_color = build_load(b, pos, target->border_op,
+                                        target->border_msaa,
+                                        location, target, key);
+                  bd_cov = coverage;
+               }
+            }
+            nir_pop_if(b, nif);
+            color = nir_if_phi(b, ib_color, bd_color);
+            coverage = nir_if_phi(b, ib_cov, bd_cov);
+         }
+
+         if (nif_s0) {
+            nir_pop_if(b, nif_s0);
+            color = nir_if_phi(b, color, u4);
+            coverage = nir_if_phi(b, coverage, z1);
+         }
+
+         const nir_alu_type nir_type =
+            nir_alu_type_for_data_type(target->data_type);
+         nir_def *blend = nir_load_blend_descriptor_pan(b, .base = rt);
+
+         nir_blend_pan(b, coverage, blend, color,
+                       .src_type = nir_type,
+                       .io_semantics.location = location,
+                       .io_semantics.num_slots = 1);
+      }
+   } else {
+      if (info.discard_in_bounds) {
+         nir_terminate_if(b, in_bounds);
+         in_bounds = nir_imm_false(b);
+      } else if (info.discard_border) {
+         nir_terminate_if(b, nir_inot(b, in_bounds));
+         in_bounds = nir_imm_true(b);
+      }
+
+      if (info.sample0_only) {
+         /* The little optimization we do at the end will take care of most
+          * of what we need for sample0-only but we also need to write
+          * SAMPLE_MASK so pan_nir_lower_fs_outputs will do the right thing.
+          */
+         nir_store_output(b, nir_imm_int(b, 1), nir_imm_int(b, 0),
+                          .base = FRAG_RESULT_SAMPLE_MASK, .range = 1,
+                          .write_mask = 1,
+                          .src_type = nir_type_uint32,
+                          .io_semantics.location = FRAG_RESULT_SAMPLE_MASK,
+                          .io_semantics.num_slots = 1);
+      }
+
+      if (pan_fb_shader_key_target_written(&key->z)) {
+         assert(key->z.sample0_only == info.sample0_only);
+         build_op(b, pos, in_bounds, FRAG_RESULT_DEPTH, &key->z, key);
+      }
+
+      if (pan_fb_shader_key_target_written(&key->s)) {
+         assert(key->s.sample0_only == info.sample0_only);
+         build_op(b, pos, in_bounds, FRAG_RESULT_STENCIL, &key->s, key);
+      }
+
+      for (unsigned rt = 0; rt < PAN_MAX_RTS; rt++) {
+         if (pan_fb_shader_key_target_written(&key->rts[rt])) {
+            assert(key->rts[rt].sample0_only == info.sample0_only);
+            build_op(b, pos, in_bounds, FRAG_RESULT_DATA0 + rt,
+                     &key->rts[rt], key);
+         }
+      }
+   }
+
+   if (info.sample0_only) {
+      NIR_PASS(_, b->shader, nir_shader_intrinsics_pass,
+               opt_sample0_only_intr, nir_metadata_control_flow, NULL);
+      b->shader->info.fs.uses_sample_shading = false;
+   }
+
+   return builder.shader;
+}

src/panfrost/lib/pan_format.c

@@ -0,0 +1,691 @@
+/*
+ * Copyright (C) 2019 Collabora, Ltd.
+ * Copyright (C) 2026 Arm Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "pan_format.h"
+#include "genxml/gen_macros.h"
+#include "util/format/u_format.h"
+
+/* Convenience */
+
+#if PAN_ARCH == 6
+#define MALI_RGBA_SWIZZLE         PAN_V6_SWIZZLE(R, G, B, A)
+#define MALI_RGB1_SWIZZLE         PAN_V6_SWIZZLE(R, G, B, A)
+#else
+#define MALI_RGBA_SWIZZLE         MALI_RGB_COMPONENT_ORDER_RGBA
+#define MALI_RGB1_SWIZZLE         MALI_RGB_COMPONENT_ORDER_RGB1
+#endif
+
+#define MALI_BLEND_AU_R8G8B8A8    (MALI_RGBA8_TB << 12)    | MALI_RGBA_SWIZZLE
+#define MALI_BLEND_PU_R8G8B8A8    (MALI_RGBA8_TB << 12)    | MALI_RGBA_SWIZZLE
+#define MALI_BLEND_AU_R10G10B10A2 (MALI_RGB10_A2_TB << 12) | MALI_RGBA_SWIZZLE
+#define MALI_BLEND_PU_R10G10B10A2 (MALI_RGB10_A2_TB << 12) | MALI_RGBA_SWIZZLE
+#define MALI_BLEND_AU_R8G8B8A2    (MALI_RGB8_A2_AU << 12)  | MALI_RGBA_SWIZZLE
+#define MALI_BLEND_PU_R8G8B8A2    (MALI_RGB8_A2_PU << 12)  | MALI_RGBA_SWIZZLE
+#define MALI_BLEND_AU_R4G4B4A4    (MALI_RGBA4_AU << 12)    | MALI_RGBA_SWIZZLE
+#define MALI_BLEND_PU_R4G4B4A4    (MALI_RGBA4_PU << 12)    | MALI_RGBA_SWIZZLE
+#define MALI_BLEND_AU_R5G6B5A0    (MALI_R5G6B5_AU << 12)   | MALI_RGB1_SWIZZLE
+#define MALI_BLEND_PU_R5G6B5A0    (MALI_R5G6B5_PU << 12)   | MALI_RGB1_SWIZZLE
+#define MALI_BLEND_AU_R5G5B5A1    (MALI_RGB5_A1_AU << 12)  | MALI_RGBA_SWIZZLE
+#define MALI_BLEND_PU_R5G5B5A1    (MALI_RGB5_A1_PU << 12)  | MALI_RGBA_SWIZZLE
+
+#define MALI_BLEND_R16F        (MALI_R16F << 12)           | MALI_RGBA_SWIZZLE
+#define MALI_BLEND_RG16F       (MALI_RG16F << 12)          | MALI_RGBA_SWIZZLE
+#define MALI_BLEND_RGBA16F     (MALI_RGBA16F << 12)        | MALI_RGBA_SWIZZLE
+#define MALI_BLEND_RG11F_B10F  (MALI_R11F_G11F_B10F << 12) | MALI_RGBA_SWIZZLE
+
+#if PAN_ARCH <= 5
+#define BFMT2(pipe, internal, writeback, srgb)                                 \
+   [PIPE_FORMAT_##pipe] = {                                                    \
+      MALI_COLOR_BUFFER_INTERNAL_FORMAT_##internal,                            \
+      MALI_COLOR_FORMAT_##writeback,                                           \
+      { 0, 0 },                                                                \
+   }
+#else
+#define BFMT2(pipe, internal, writeback, srgb)                                 \
+   [PIPE_FORMAT_##pipe] = {                                                    \
+      MALI_COLOR_BUFFER_INTERNAL_FORMAT_##internal,                            \
+      MALI_COLOR_FORMAT_##writeback,                                           \
+      {                                                                        \
+         MALI_BLEND_PU_##internal | (srgb ? (1 << 20) : 0),                    \
+         MALI_BLEND_AU_##internal | (srgb ? (1 << 20) : 0),                    \
+      },                                                                       \
+   }
+#endif
+
+#define BFMT(pipe, internal_and_writeback)                                     \
+   BFMT2(pipe, internal_and_writeback, internal_and_writeback, 0)
+
+#define BFMT_SRGB(pipe, writeback)                                             \
+   BFMT2(pipe##_UNORM, R8G8B8A8, writeback, 0),                                \
+      BFMT2(pipe##_SRGB, R8G8B8A8, writeback, 1)
+
+#if PAN_ARCH >= 9
+#define BFMT_FLOAT(pipe, internal_and_writeback)                               \
+   [PIPE_FORMAT_##pipe] = {                                                    \
+      MALI_COLOR_BUFFER_INTERNAL_FORMAT_##internal_and_writeback,              \
+      MALI_FLOAT_COLOR_FORMAT_##internal_and_writeback,                        \
+      {                                                                        \
+         MALI_BLEND_##internal_and_writeback,                                  \
+         MALI_BLEND_##internal_and_writeback,                                  \
+      },                                                                       \
+   }
+#endif
+
+const struct pan_blendable_format
+   GENX(pan_blendable_formats)[PIPE_FORMAT_COUNT] = {
+      BFMT_SRGB(L8, R8),
+      BFMT_SRGB(L8A8, R8G8),
+      BFMT_SRGB(R8, R8),
+      BFMT_SRGB(R8G8, R8G8),
+      BFMT_SRGB(R8G8B8, R8G8B8),
+      BFMT_SRGB(B8G8R8, R8G8B8),
+
+      BFMT_SRGB(B8G8R8A8, R8G8B8A8),
+      BFMT_SRGB(B8G8R8X8, R8G8B8A8),
+      BFMT_SRGB(A8R8G8B8, R8G8B8A8),
+      BFMT_SRGB(X8R8G8B8, R8G8B8A8),
+      BFMT_SRGB(A8B8G8R8, R8G8B8A8),
+      BFMT_SRGB(X8B8G8R8, R8G8B8A8),
+      BFMT_SRGB(R8G8B8X8, R8G8B8A8),
+      BFMT_SRGB(R8G8B8A8, R8G8B8A8),
+
+      BFMT2(A8_UNORM, R8G8B8A8, R8, 0),
+#if PAN_ARCH < 6
+      BFMT2(I8_UNORM, R8G8B8A8, R8, 0),
+#endif
+      BFMT2(R5G6B5_UNORM, R5G6B5A0, R5G6B5, 0),
+      BFMT2(B5G6R5_UNORM, R5G6B5A0, R5G6B5, 0),
+
+      BFMT(R4G4B4A4_UNORM, R4G4B4A4),
+      BFMT(B4G4R4A4_UNORM, R4G4B4A4),
+      BFMT(A4R4G4B4_UNORM, R4G4B4A4),
+      BFMT(A4B4G4R4_UNORM, R4G4B4A4),
+
+      BFMT(R10G10B10A2_UNORM, R10G10B10A2),
+      BFMT(B10G10R10A2_UNORM, R10G10B10A2),
+      BFMT(R10G10B10X2_UNORM, R10G10B10A2),
+      BFMT(B10G10R10X2_UNORM, R10G10B10A2),
+
+      BFMT(B5G5R5A1_UNORM, R5G5B5A1),
+      BFMT(R5G5B5A1_UNORM, R5G5B5A1),
+      BFMT(B5G5R5X1_UNORM, R5G5B5A1),
+
+#if PAN_ARCH >= 9
+      BFMT_FLOAT(R16_FLOAT, R16F),
+      BFMT_FLOAT(R16G16_FLOAT, RG16F),
+      BFMT_FLOAT(R16G16B16A16_FLOAT, RGBA16F),
+      BFMT_FLOAT(R11G11B10_FLOAT, RG11F_B10F),
+#endif
+};
+
+/* Convenience */
+
+#define _V PAN_BIND_VERTEX_BUFFER
+#define _T PAN_BIND_SAMPLER_VIEW
+#define _R PAN_BIND_RENDER_TARGET
+#define _Z PAN_BIND_DEPTH_STENCIL
+#define _I PAN_BIND_STORAGE_IMAGE
+#define _B PAN_BIND_TEXEL_BUFFER
+
+#define FLAGS_V_____ (_V)
+#define FLAGS__T____ (_T)
+#define FLAGS_VTR___ (_V | _T | _R)
+#define FLAGS_VTR_IB (_V | _T | _R | _I | _B)
+#define FLAGS_VT____ (_V | _T)
+#define FLAGS_VT__IB (_V | _T | _I | _B)
+#define FLAGS__T_Z__ (_T | _Z)
+
+#define FMT(pipe, mali, swizzle, srgb, flags)                                  \
+   [PIPE_FORMAT_##pipe] = {                                                    \
+      .hw = MALI_PACK_FMT(mali, swizzle, srgb),                                \
+      .bind = FLAGS_##flags,                                                   \
+   }
+
+#if PAN_ARCH >= 7
+#define YUV_NO_SWAP (0)
+#define YUV_SWAP    (1)
+
+#if PAN_ARCH < 14
+#define MALI_YUV_CR_SITING_CENTER_422 (MALI_YUV_CR_SITING_CENTER_Y)
+#else
+#define MALI_YUV_CR_SITING_CENTER_422 (MALI_YUV_CR_SITING_CENTER_X)
+#endif
+
+#define FMT_YUV(pipe, mali, swizzle, swap, siting, flags)                      \
+   [PIPE_FORMAT_##pipe] = {                                                    \
+      .hw = (MALI_YUV_SWIZZLE_##swizzle) | ((YUV_##swap) << 3) |               \
+            ((MALI_YUV_CR_SITING_##siting) << 9) | ((MALI_##mali) << 12),      \
+      .bind = FLAGS_##flags,                                                   \
+   }
+#endif
+
+#if PAN_ARCH < 9
+#define FMTC(pipe, texfeat, interchange, swizzle, srgb)                        \
+   [PIPE_FORMAT_##pipe] = {                                                    \
+      .hw = MALI_PACK_FMT(texfeat, swizzle, srgb),                             \
+      .bind = (PAN_BIND_SAMPLER_VIEW),                                         \
+      .texfeat_bit = MALI_##texfeat,                                           \
+   }
+#else
+   /* Map to interchange format, as compression is specified in the plane
+    * descriptor on Valhall.
+    */
+#define FMTC(pipe, texfeat, interchange, swizzle, srgb)                        \
+   [PIPE_FORMAT_##pipe] = {                                                    \
+      .hw = MALI_PACK_FMT(interchange, swizzle, srgb),                         \
+      .bind = (PAN_BIND_SAMPLER_VIEW),                                         \
+      .texfeat_bit = MALI_##texfeat,                                           \
+   }
+#endif
+
+/* clang-format off */
+const struct pan_format GENX(pan_pipe_format)[PIPE_FORMAT_COUNT] = {
+   FMT(NONE,                    CONSTANT,        0000, L, VTR_IB),
+
+#if PAN_ARCH >= 14
+   /* Multiplane formats */
+   FMT_YUV(R8G8_R8B8_UNORM, Y8U8Y8V8_422, UVYA, NO_SWAP, CENTER_422, _T____),
+   FMT_YUV(G8R8_B8R8_UNORM, U8Y8V8Y8_422, UYVA, SWAP,    CENTER_422, _T____),
+   FMT_YUV(R8B8_R8G8_UNORM, Y8U8Y8V8_422, VYUA, NO_SWAP, CENTER_422, _T____),
+   FMT_YUV(B8R8_G8R8_UNORM, U8Y8V8Y8_422, VUYA, SWAP,    CENTER_422, _T____),
+   FMT_YUV(R8_G8B8_420_UNORM, Y8U8V8_420, YUVA, NO_SWAP, CENTER, _T____),
+   FMT_YUV(R8_B8G8_420_UNORM, Y8U8V8_420, YVUA, NO_SWAP, CENTER, _T____),
+   FMT_YUV(R8_G8_B8_420_UNORM, Y8U8V8_420, YUVA, NO_SWAP, CENTER, _T____),
+   FMT_YUV(R8_B8_G8_420_UNORM, Y8U8V8_420, YVUA, NO_SWAP, CENTER, _T____),
+
+   FMT_YUV(R8_G8B8_422_UNORM, Y8U8Y8V8_422, YUVA, NO_SWAP, CENTER_422, _T____),
+   FMT_YUV(R8_B8G8_422_UNORM, U8Y8V8Y8_422, YVUA, NO_SWAP, CENTER_422, _T____),
+
+   FMT_YUV(R10_G10B10_420_UNORM, YUYAAYVYAA_420, YUVA, NO_SWAP, CENTER, _T____),
+   FMT_YUV(R10_G10B10_422_UNORM, Y10X6U10X6Y10X6V10X6_422, YUVA, NO_SWAP, CENTER_422, _T____),
+   /* special internal formats */
+   FMT_YUV(R8G8B8_420_UNORM_PACKED, Y8U8V8_420, YUVA, NO_SWAP, CENTER, _T____),
+   FMT_YUV(R10G10B10_420_UNORM_PACKED, Y10U10V10_420, YUVA, NO_SWAP, CENTER, _T____),
+   FMT_YUV(X6R10X6G10_X6R10X6B10_422_UNORM, Y10X6U10X6Y10X6V10X6_422, UVYA, NO_SWAP, CENTER_422, _T____),
+#elif PAN_ARCH >= 7
+   /* Multiplane formats */
+   FMT_YUV(R8G8_R8B8_UNORM, YUYV8, UVYA, NO_SWAP, CENTER_422, _T____),
+   FMT_YUV(G8R8_B8R8_UNORM, VYUY8, UYVA, SWAP,    CENTER_422, _T____),
+   FMT_YUV(R8B8_R8G8_UNORM, YUYV8, VYUA, NO_SWAP, CENTER_422, _T____),
+   FMT_YUV(B8R8_G8R8_UNORM, VYUY8, VUYA, SWAP,    CENTER_422, _T____),
+   FMT_YUV(R8_G8B8_420_UNORM, Y8_UV8_420, YUVA, NO_SWAP, CENTER, _T____),
+   FMT_YUV(R8_B8G8_420_UNORM, Y8_UV8_420, YVUA, NO_SWAP, CENTER, _T____),
+   FMT_YUV(R8_G8_B8_420_UNORM, Y8_U8_V8_420, YUVA, NO_SWAP, CENTER, _T____),
+   FMT_YUV(R8_B8_G8_420_UNORM, Y8_U8_V8_420, YVUA, NO_SWAP, CENTER, _T____),
+
+   FMT_YUV(R8_G8B8_422_UNORM, Y8_UV8_422, YUVA, NO_SWAP, CENTER_422, _T____),
+   FMT_YUV(R8_B8G8_422_UNORM, Y8_UV8_422, YVUA, NO_SWAP, CENTER_422, _T____),
+
+   FMT_YUV(R10_G10B10_420_UNORM, Y10_UV10_420, YUVA, NO_SWAP, CENTER, _T____),
+   FMT_YUV(R10_G10B10_422_UNORM, Y10_UV10_422, YUVA, NO_SWAP, CENTER_422, _T____),
+   /* special internal formats */
+   FMT_YUV(R8G8B8_420_UNORM_PACKED, Y8_UV8_420, YUVA, NO_SWAP, CENTER, _T____),
+   FMT_YUV(R10G10B10_420_UNORM_PACKED, Y10_UV10_420, YUVA, NO_SWAP, CENTER, _T____),
+   FMT_YUV(X6R10X6G10_X6R10X6B10_422_UNORM, YUYV10, UVYA, NO_SWAP, CENTER_422, _T____),
+#endif
+
+   FMTC(ETC1_RGB8,               ETC2_RGB8,       RGBA8_UNORM, RGB1, L),
+   FMTC(ETC2_RGB8,               ETC2_RGB8,       RGBA8_UNORM, RGB1, L),
+   FMTC(ETC2_SRGB8,              ETC2_RGB8,       RGBA8_UNORM, RGB1, S),
+   FMTC(ETC2_R11_UNORM,          ETC2_R11_UNORM,  R16_UNORM,   R001, L),
+   FMTC(ETC2_RGBA8,              ETC2_RGBA8,      RGBA8_UNORM, RGBA, L),
+   FMTC(ETC2_SRGBA8,             ETC2_RGBA8,      RGBA8_UNORM, RGBA, S),
+   FMTC(ETC2_RG11_UNORM,         ETC2_RG11_UNORM, RG16_UNORM,  RG01, L),
+   FMTC(ETC2_R11_SNORM,          ETC2_R11_SNORM,  R16_SNORM,   R001, L),
+   FMTC(ETC2_RG11_SNORM,         ETC2_RG11_SNORM, RG16_SNORM,  RG01, L),
+   FMTC(ETC2_RGB8A1,             ETC2_RGB8A1,     RGBA8_UNORM, RGBA, L),
+   FMTC(ETC2_SRGB8A1,            ETC2_RGB8A1,     RGBA8_UNORM, RGBA, S),
+   FMTC(DXT1_RGB,                BC1_UNORM,       RGBA8_UNORM, RGB1, L),
+   FMTC(DXT1_RGBA,               BC1_UNORM,       RGBA8_UNORM, RGBA, L),
+   FMTC(DXT1_SRGB,               BC1_UNORM,       RGBA8_UNORM, RGB1, S),
+   FMTC(DXT1_SRGBA,              BC1_UNORM,       RGBA8_UNORM, RGBA, S),
+   FMTC(DXT3_RGBA,               BC2_UNORM,       RGBA8_UNORM, RGBA, L),
+   FMTC(DXT3_SRGBA,              BC2_UNORM,       RGBA8_UNORM, RGBA, S),
+   FMTC(DXT5_RGBA,               BC3_UNORM,       RGBA8_UNORM, RGBA, L),
+   FMTC(DXT5_SRGBA,              BC3_UNORM,       RGBA8_UNORM, RGBA, S),
+   FMTC(RGTC1_UNORM,             BC4_UNORM,       R16_UNORM,   R001, L),
+   FMTC(RGTC1_SNORM,             BC4_SNORM,       R16_SNORM,   R001, L),
+   FMTC(RGTC2_UNORM,             BC5_UNORM,       RG16_UNORM,  RG01, L),
+   FMTC(RGTC2_SNORM,             BC5_SNORM,       RG16_SNORM,  RG01, L),
+   FMTC(BPTC_RGB_FLOAT,          BC6H_SF16,       RGBA16F,     RGB1, L),
+   FMTC(BPTC_RGB_UFLOAT,         BC6H_UF16,       RGBA16F,     RGB1, L),
+   FMTC(BPTC_RGBA_UNORM,         BC7_UNORM,       RGBA8_UNORM, RGBA, L),
+   FMTC(BPTC_SRGBA,              BC7_UNORM,       RGBA8_UNORM, RGBA, S),
+
+   /* If ASTC decode mode is set to RGBA8, the hardware format
+    * will be overriden to RGBA8_UNORM later on.
+    */
+   FMTC(ASTC_4x4,                ASTC_2D_LDR,     RGBA16F,     RGBA, L),
+   FMTC(ASTC_5x4,                ASTC_2D_LDR,     RGBA16F,     RGBA, L),
+   FMTC(ASTC_5x5,                ASTC_2D_LDR,     RGBA16F,     RGBA, L),
+   FMTC(ASTC_6x5,                ASTC_2D_LDR,     RGBA16F,     RGBA, L),
+   FMTC(ASTC_6x6,                ASTC_2D_LDR,     RGBA16F,     RGBA, L),
+   FMTC(ASTC_8x5,                ASTC_2D_LDR,     RGBA16F,     RGBA, L),
+   FMTC(ASTC_8x6,                ASTC_2D_LDR,     RGBA16F,     RGBA, L),
+   FMTC(ASTC_8x8,                ASTC_2D_LDR,     RGBA16F,     RGBA, L),
+   FMTC(ASTC_10x5,               ASTC_2D_LDR,     RGBA16F,     RGBA, L),
+   FMTC(ASTC_10x6,               ASTC_2D_LDR,     RGBA16F,     RGBA, L),
+   FMTC(ASTC_10x8,               ASTC_2D_LDR,     RGBA16F,     RGBA, L),
+   FMTC(ASTC_10x10,              ASTC_2D_LDR,     RGBA16F,     RGBA, L),
+   FMTC(ASTC_12x10,              ASTC_2D_LDR,     RGBA16F,     RGBA, L),
+   FMTC(ASTC_12x12,              ASTC_2D_LDR,     RGBA16F,     RGBA, L),
+   FMTC(ASTC_3x3x3,              ASTC_3D_LDR,     RGBA16F,     RGBA, L),
+   FMTC(ASTC_4x3x3,              ASTC_3D_LDR,     RGBA16F,     RGBA, L),
+   FMTC(ASTC_4x4x3,              ASTC_3D_LDR,     RGBA16F,     RGBA, L),
+   FMTC(ASTC_4x4x4,              ASTC_3D_LDR,     RGBA16F,     RGBA, L),
+   FMTC(ASTC_5x4x4,              ASTC_3D_LDR,     RGBA16F,     RGBA, L),
+   FMTC(ASTC_5x5x4,              ASTC_3D_LDR,     RGBA16F,     RGBA, L),
+   FMTC(ASTC_5x5x5,              ASTC_3D_LDR,     RGBA16F,     RGBA, L),
+   FMTC(ASTC_6x5x5,              ASTC_3D_LDR,     RGBA16F,     RGBA, L),
+   FMTC(ASTC_6x6x5,              ASTC_3D_LDR,     RGBA16F,     RGBA, L),
+   FMTC(ASTC_6x6x6,              ASTC_3D_LDR,     RGBA16F,     RGBA, L),
+
+   /* By definition, sRGB formats are narrow */
+   FMTC(ASTC_4x4_SRGB,           ASTC_2D_LDR,     RGBA8_UNORM, RGBA, S),
+   FMTC(ASTC_5x4_SRGB,           ASTC_2D_LDR,     RGBA8_UNORM, RGBA, S),
+   FMTC(ASTC_5x5_SRGB,           ASTC_2D_LDR,     RGBA8_UNORM, RGBA, S),
+   FMTC(ASTC_6x5_SRGB,           ASTC_2D_LDR,     RGBA8_UNORM, RGBA, S),
+   FMTC(ASTC_6x6_SRGB,           ASTC_2D_LDR,     RGBA8_UNORM, RGBA, S),
+   FMTC(ASTC_8x5_SRGB,           ASTC_2D_LDR,     RGBA8_UNORM, RGBA, S),
+   FMTC(ASTC_8x6_SRGB,           ASTC_2D_LDR,     RGBA8_UNORM, RGBA, S),
+   FMTC(ASTC_8x8_SRGB,           ASTC_2D_LDR,     RGBA8_UNORM, RGBA, S),
+   FMTC(ASTC_10x5_SRGB,          ASTC_2D_LDR,     RGBA8_UNORM, RGBA, S),
+   FMTC(ASTC_10x6_SRGB,          ASTC_2D_LDR,     RGBA8_UNORM, RGBA, S),
+   FMTC(ASTC_10x8_SRGB,          ASTC_2D_LDR,     RGBA8_UNORM, RGBA, S),
+   FMTC(ASTC_10x10_SRGB,         ASTC_2D_LDR,     RGBA8_UNORM, RGBA, S),
+   FMTC(ASTC_12x10_SRGB,         ASTC_2D_LDR,     RGBA8_UNORM, RGBA, S),
+   FMTC(ASTC_12x12_SRGB,         ASTC_2D_LDR,     RGBA8_UNORM, RGBA, S),
+   FMTC(ASTC_3x3x3_SRGB,         ASTC_3D_LDR,     RGBA8_UNORM, RGBA, S),
+   FMTC(ASTC_4x3x3_SRGB,         ASTC_3D_LDR,     RGBA8_UNORM, RGBA, S),
+   FMTC(ASTC_4x4x3_SRGB,         ASTC_3D_LDR,     RGBA8_UNORM, RGBA, S),
+   FMTC(ASTC_4x4x4_SRGB,         ASTC_3D_LDR,     RGBA8_UNORM, RGBA, S),
+   FMTC(ASTC_5x4x4_SRGB,         ASTC_3D_LDR,     RGBA8_UNORM, RGBA, S),
+   FMTC(ASTC_5x5x4_SRGB,         ASTC_3D_LDR,     RGBA8_UNORM, RGBA, S),
+   FMTC(ASTC_5x5x5_SRGB,         ASTC_3D_LDR,     RGBA8_UNORM, RGBA, S),
+   FMTC(ASTC_6x5x5_SRGB,         ASTC_3D_LDR,     RGBA8_UNORM, RGBA, S),
+   FMTC(ASTC_6x6x5_SRGB,         ASTC_3D_LDR,     RGBA8_UNORM, RGBA, S),
+   FMTC(ASTC_6x6x6_SRGB,         ASTC_3D_LDR,     RGBA8_UNORM, RGBA, S),
+
+   FMTC(ASTC_4x4_FLOAT,          ASTC_2D_HDR,     RGBA16F,     RGBA, L),
+   FMTC(ASTC_5x4_FLOAT,          ASTC_2D_HDR,     RGBA16F,     RGBA, L),
+   FMTC(ASTC_5x5_FLOAT,          ASTC_2D_HDR,     RGBA16F,     RGBA, L),
+   FMTC(ASTC_6x5_FLOAT,          ASTC_2D_HDR,     RGBA16F,     RGBA, L),
+   FMTC(ASTC_6x6_FLOAT,          ASTC_2D_HDR,     RGBA16F,     RGBA, L),
+   FMTC(ASTC_8x5_FLOAT,          ASTC_2D_HDR,     RGBA16F,     RGBA, L),
+   FMTC(ASTC_8x6_FLOAT,          ASTC_2D_HDR,     RGBA16F,     RGBA, L),
+   FMTC(ASTC_8x8_FLOAT,          ASTC_2D_HDR,     RGBA16F,     RGBA, L),
+   FMTC(ASTC_10x5_FLOAT,         ASTC_2D_HDR,     RGBA16F,     RGBA, L),
+   FMTC(ASTC_10x6_FLOAT,         ASTC_2D_HDR,     RGBA16F,     RGBA, L),
+   FMTC(ASTC_10x8_FLOAT,         ASTC_2D_HDR,     RGBA16F,     RGBA, L),
+   FMTC(ASTC_10x10_FLOAT,        ASTC_2D_HDR,     RGBA16F,     RGBA, L),
+   FMTC(ASTC_12x10_FLOAT,        ASTC_2D_HDR,     RGBA16F,     RGBA, L),
+   FMTC(ASTC_12x12_FLOAT,        ASTC_2D_HDR,     RGBA16F,     RGBA, L),
+
+   FMT(R5G6B5_UNORM,            RGB565,          RGB1, L, VTR_IB),
+   FMT(B5G6R5_UNORM,            RGB565,          BGR1, L, VTR_IB),
+   FMT(R5G5B5X1_UNORM,          RGB5_A1_UNORM,   RGB1, L, VT__IB),
+   FMT(B5G5R5X1_UNORM,          RGB5_A1_UNORM,   BGR1, L, VT__IB),
+   FMT(R5G5B5A1_UNORM,          RGB5_A1_UNORM,   RGBA, L, VTR_IB),
+   FMT(B5G5R5A1_UNORM,          RGB5_A1_UNORM,   BGRA, L, VTR_IB),
+   FMT(R10G10B10X2_UNORM,       RGB10_A2_UNORM,  RGB1, L, VTR_IB),
+   FMT(B10G10R10X2_UNORM,       RGB10_A2_UNORM,  BGR1, L, VTR_IB),
+   FMT(R10G10B10A2_UNORM,       RGB10_A2_UNORM,  RGBA, L, VTR_IB),
+   FMT(B10G10R10A2_UNORM,       RGB10_A2_UNORM,  BGRA, L, VTR_IB),
+#if PAN_ARCH <= 5
+   FMT(R10G10B10X2_SNORM,       RGB10_A2_SNORM,  RGB1, L, VT__IB),
+   FMT(R10G10B10A2_SNORM,       RGB10_A2_SNORM,  RGBA, L, VT__IB),
+   FMT(B10G10R10A2_SNORM,       RGB10_A2_SNORM,  BGRA, L, VT__IB),
+   FMT(R3G3B2_UNORM,            RGB332_UNORM,    RGB1, L, _T____),
+#else
+   FMT(R10G10B10X2_SNORM,       RGB10_A2_SNORM,  RGB1, L, V_____),
+   FMT(R10G10B10A2_SNORM,       RGB10_A2_SNORM,  RGBA, L, V_____),
+   FMT(B10G10R10A2_SNORM,       RGB10_A2_SNORM,  BGRA, L, V_____),
+#endif
+   FMT(R10G10B10A2_UINT,        RGB10_A2UI,      RGBA, L, VTR_IB),
+   FMT(B10G10R10A2_UINT,        RGB10_A2UI,      BGRA, L, VTR_IB),
+   FMT(R10G10B10A2_USCALED,     RGB10_A2UI,      RGBA, L, V_____),
+   FMT(B10G10R10A2_USCALED,     RGB10_A2UI,      BGRA, L, V_____),
+   FMT(R10G10B10A2_SINT,        RGB10_A2I,       RGBA, L, VTR___),
+   FMT(B10G10R10A2_SINT,        RGB10_A2I,       BGRA, L, VTR___),
+   FMT(R10G10B10A2_SSCALED,     RGB10_A2I,       RGBA, L, V_____),
+   FMT(B10G10R10A2_SSCALED,     RGB10_A2I,       BGRA, L, V_____),
+   FMT(R8_SSCALED,              R8I,             R001, L, V_____),
+   FMT(R8G8_SSCALED,            RG8I,            RG01, L, V_____),
+   FMT(R8G8B8_SSCALED,          RGB8I,           RGB1, L, V_____),
+   FMT(B8G8R8_SSCALED,          RGB8I,           BGR1, L, V_____),
+   FMT(R8G8B8A8_SSCALED,        RGBA8I,          RGBA, L, V_____),
+   FMT(B8G8R8A8_SSCALED,        RGBA8I,          BGRA, L, V_____),
+   FMT(A8B8G8R8_SSCALED,        RGBA8I,          ABGR, L, V_____),
+   FMT(R8_USCALED,              R8UI,            R001, L, V_____),
+   FMT(R8G8_USCALED,            RG8UI,           RG01, L, V_____),
+   FMT(R8G8B8_USCALED,          RGB8UI,          RGB1, L, V_____),
+   FMT(B8G8R8_USCALED,          RGB8UI,          BGR1, L, V_____),
+   FMT(R8G8B8A8_USCALED,        RGBA8UI,         RGBA, L, V_____),
+   FMT(B8G8R8A8_USCALED,        RGBA8UI,         BGRA, L, V_____),
+   FMT(A8B8G8R8_USCALED,        RGBA8UI,         ABGR, L, V_____),
+   FMT(R16_USCALED,             R16UI,           R001, L, V_____),
+   FMT(R16G16_USCALED,          RG16UI,          RG01, L, V_____),
+   FMT(R16G16B16A16_USCALED,    RGBA16UI,        RGBA, L, V_____),
+   FMT(R16_SSCALED,             R16I,            R001, L, V_____),
+   FMT(R16G16_SSCALED,          RG16I,           RG01, L, V_____),
+   FMT(R16G16B16A16_SSCALED,    RGBA16I,         RGBA, L, V_____),
+   FMT(R32_USCALED,             R32UI,           R001, L, V_____),
+   FMT(R32G32_USCALED,          RG32UI,          RG01, L, V_____),
+   FMT(R32G32B32_USCALED,       RGB32UI,         RGB1, L, V_____),
+   FMT(R32G32B32A32_USCALED,    RGBA32UI,        RGBA, L, V_____),
+   FMT(R32_SSCALED,             R32I,            R001, L, V_____),
+   FMT(R32G32_SSCALED,          RG32I,           RG01, L, V_____),
+   FMT(R32G32B32_SSCALED,       RGB32I,          RGB1, L, V_____),
+   FMT(R32G32B32A32_SSCALED,    RGBA32I,         RGBA, L, V_____),
+   FMT(R32_FIXED,               R32_FIXED,       R001, L, V_____),
+   FMT(R32G32_FIXED,            RG32_FIXED,      RG01, L, V_____),
+   FMT(R32G32B32_FIXED,         RGB32_FIXED,     RGB1, L, V_____),
+   FMT(R32G32B32A32_FIXED,      RGBA32_FIXED,    RGBA, L, V_____),
+   FMT(R11G11B10_FLOAT,         R11F_G11F_B10F,  RGB1, L, VTR_IB),
+#if PAN_ARCH < 7
+   FMT(R9G9B9E5_FLOAT,          R9F_G9F_B9F_E5F, RGB1, L, _T____),
+#else
+   FMT(R9G9B9E5_FLOAT,          R9F_G9F_B9F_E5F, RGB1, L, VTR_IB),
+#endif
+#if PAN_ARCH >= 6
+   /* SNORM is renderable on Bifrost (with blend shaders) */
+   FMT(R8_SNORM,                R8_SNORM,        R001, L, VTR_IB),
+   FMT(R16_SNORM,               R16_SNORM,       R001, L, VTR_IB),
+   FMT(R8G8_SNORM,              RG8_SNORM,       RG01, L, VTR_IB),
+   FMT(R16G16_SNORM,            RG16_SNORM,      RG01, L, VTR_IB),
+   FMT(R8G8B8_SNORM,            RGB8_SNORM,      RGB1, L, VTR_IB),
+   FMT(R8G8B8A8_SNORM,          RGBA8_SNORM,     RGBA, L, VTR_IB),
+   FMT(B8G8R8A8_SNORM,          RGBA8_SNORM,     BGRA, L, VTR_IB),
+   FMT(R16G16B16A16_SNORM,      RGBA16_SNORM,    RGBA, L, VTR_IB),
+#else
+   /* So far we haven't needed SNORM rendering on Midgard */
+   FMT(R8_SNORM,                R8_SNORM,        R001, L, VT__IB),
+   FMT(R16_SNORM,               R16_SNORM,       R001, L, VT__IB),
+   FMT(R8G8_SNORM,              RG8_SNORM,       RG01, L, VT__IB),
+   FMT(R16G16_SNORM,            RG16_SNORM,      RG01, L, VT__IB),
+   FMT(R8G8B8_SNORM,            RGB8_SNORM,      RGB1, L, VT__IB),
+   FMT(R8G8B8A8_SNORM,          RGBA8_SNORM,     RGBA, L, VT__IB),
+   FMT(B8G8R8A8_SNORM,          RGBA8_SNORM,     BGRA, L, VT__IB),
+   FMT(R16G16B16A16_SNORM,      RGBA16_SNORM,    RGBA, L, VT__IB),
+#endif
+   FMT(I8_SINT,                 R8I,             RRRR, L, VTR_IB),
+   FMT(L8_SINT,                 R8I,             RRR1, L, VTR_IB),
+   FMT(I8_UINT,                 R8UI,            RRRR, L, VTR_IB),
+   FMT(L8_UINT,                 R8UI,            RRR1, L, VTR_IB),
+   FMT(I16_SINT,                R16I,            RRRR, L, VTR_IB),
+   FMT(L16_SINT,                R16I,            RRR1, L, VTR_IB),
+   FMT(I16_UINT,                R16UI,           RRRR, L, VTR_IB),
+   FMT(L16_UINT,                R16UI,           RRR1, L, VTR_IB),
+   FMT(I32_SINT,                R32I,            RRRR, L, VTR_IB),
+   FMT(L32_SINT,                R32I,            RRR1, L, VTR_IB),
+   FMT(I32_UINT,                R32UI,           RRRR, L, VTR_IB),
+   FMT(L32_UINT,                R32UI,           RRR1, L, VTR_IB),
+   FMT(B8G8R8_UINT,             RGB8UI,          BGR1, L, V_____),
+   FMT(B8G8R8_SINT,             RGB8I,           BGR1, L, V_____),
+   FMT(B8G8R8A8_UINT,           RGBA8UI,         BGRA, L, VTR_IB),
+   FMT(B8G8R8A8_SINT,           RGBA8I,          BGRA, L, VTR_IB),
+   FMT(A8R8G8B8_UINT,           RGBA8UI,         GBAR, L, VTR_IB),
+   FMT(A8B8G8R8_UINT,           RGBA8UI,         ABGR, L, VTR_IB),
+   FMT(R8_UINT,                 R8UI,            R001, L, VTR_IB),
+   FMT(R16_UINT,                R16UI,           R001, L, VTR_IB),
+   FMT(R32_UINT,                R32UI,           R001, L, VTR_IB),
+   FMT(R8G8_UINT,               RG8UI,           RG01, L, VTR_IB),
+   FMT(R16G16_UINT,             RG16UI,          RG01, L, VTR_IB),
+   FMT(R32G32_UINT,             RG32UI,          RG01, L, VTR_IB),
+   FMT(R8G8B8_UINT,             RGB8UI,          RGB1, L, V_____),
+   FMT(R32G32B32_UINT,          RGB32UI,         RGB1, L, VTR_IB),
+   FMT(R8G8B8A8_UINT,           RGBA8UI,         RGBA, L, VTR_IB),
+   FMT(R16G16B16A16_UINT,       RGBA16UI,        RGBA, L, VTR_IB),
+   FMT(R32G32B32A32_UINT,       RGBA32UI,        RGBA, L, VTR_IB),
+   FMT(R32_FLOAT,               R32F,            R001, L, VTR_IB),
+   FMT(R32G32_FLOAT,            RG32F,           RG01, L, VTR_IB),
+   FMT(R32G32B32_FLOAT,         RGB32F,          RGB1, L, VTR_IB),
+   FMT(R32G32B32A32_FLOAT,      RGBA32F,         RGBA, L, VTR_IB),
+   FMT(R8_UNORM,                R8_UNORM,        R001, L, VTR_IB),
+   FMT(R16_UNORM,               R16_UNORM,       R001, L, VTR_IB),
+   FMT(R8G8_UNORM,              RG8_UNORM,       RG01, L, VTR_IB),
+   FMT(R16G16_UNORM,            RG16_UNORM,      RG01, L, VTR_IB),
+   FMT(R8G8B8_UNORM,            RGB8_UNORM,      RGB1, L, VTR_IB),
+   FMT(B8G8R8_UNORM,            RGB8_UNORM,      BGR1, L, VTR_IB),
+
+   /* 32-bit NORM is not texturable in v7 onwards. It's renderable
+    * everywhere, but rendering without texturing is not useful.
+    */
+#if PAN_ARCH <= 6
+   FMT(R32_UNORM,               R32_UNORM,       R001, L, VTR_IB),
+   FMT(R32G32_UNORM,            RG32_UNORM,      RG01, L, VTR_IB),
+   FMT(R32G32B32_UNORM,         RGB32_UNORM,     RGB1, L, VT__IB),
+   FMT(R32G32B32A32_UNORM,      RGBA32_UNORM,    RGBA, L, VTR_IB),
+   FMT(R32_SNORM,               R32_SNORM,       R001, L, VT__IB),
+   FMT(R32G32_SNORM,            RG32_SNORM,      RG01, L, VT__IB),
+   FMT(R32G32B32_SNORM,         RGB32_SNORM,     RGB1, L, VT__IB),
+   FMT(R32G32B32A32_SNORM,      RGBA32_SNORM,    RGBA, L, VT__IB),
+#else
+   FMT(R32_UNORM,               R32_UNORM,       R001, L, V_____),
+   FMT(R32G32_UNORM,            RG32_UNORM,      RG01, L, V_____),
+   FMT(R32G32B32_UNORM,         RGB32_UNORM,     RGB1, L, V_____),
+   FMT(R32G32B32A32_UNORM,      RGBA32_UNORM,    RGBA, L, V_____),
+   FMT(R32_SNORM,               R32_SNORM,       R001, L, V_____),
+   FMT(R32G32_SNORM,            RG32_SNORM,      RG01, L, V_____),
+   FMT(R32G32B32_SNORM,         RGB32_SNORM,     RGB1, L, V_____),
+   FMT(R32G32B32A32_SNORM,      RGBA32_SNORM,    RGBA, L, V_____),
+#endif
+
+   /* Don't allow render/texture for 48-bit  */
+   FMT(R16G16B16_UNORM,         RGB16_UNORM,     RGB1, L, V_____),
+   FMT(R16G16B16_SINT,          RGB16I,          RGB1, L, V_____),
+   FMT(R16G16B16_FLOAT,         RGB16F,          RGB1, L, V_____),
+   FMT(R16G16B16_USCALED,       RGB16UI,         RGB1, L, V_____),
+   FMT(R16G16B16_SSCALED,       RGB16I,          RGB1, L, V_____),
+   FMT(R16G16B16_SNORM,         RGB16_SNORM,     RGB1, L, V_____),
+   FMT(R16G16B16_UINT,          RGB16UI,         RGB1, L, V_____),
+   FMT(R4G4B4A4_UNORM,          RGBA4_UNORM,     RGBA, L, VTR___),
+   FMT(B4G4R4A4_UNORM,          RGBA4_UNORM,     BGRA, L, VTR___),
+   FMT(A4R4G4B4_UNORM,          RGBA4_UNORM,     ARGB, L, VTR___),
+   FMT(A4B4G4R4_UNORM,          RGBA4_UNORM,     ABGR, L, VTR___),
+   FMT(R16G16B16A16_UNORM,      RGBA16_UNORM,    RGBA, L, VTR_IB),
+#if PAN_ARCH >= 11
+   FMT(X6R10X6G10X6B10X6A10_UNORM, R10X6G10X6B10X6A10X6_UNORM, RGBA, L, _T____),
+#endif
+   FMT(B8G8R8A8_UNORM,          RGBA8_UNORM,     BGRA, L, VTR_IB),
+   FMT(B8G8R8X8_UNORM,          RGBA8_UNORM,     BGR1, L, VTR_IB),
+   FMT(A8R8G8B8_UNORM,          RGBA8_UNORM,     GBAR, L, VTR_IB),
+   FMT(X8R8G8B8_UNORM,          RGBA8_UNORM,     GBA1, L, VTR_IB),
+   FMT(A8B8G8R8_UNORM,          RGBA8_UNORM,     ABGR, L, VTR_IB),
+   FMT(X8B8G8R8_UNORM,          RGBA8_UNORM,     ABG1, L, VTR_IB),
+   FMT(R8G8B8X8_UNORM,          RGBA8_UNORM,     RGB1, L, VTR_IB),
+   FMT(R8G8B8A8_UNORM,          RGBA8_UNORM,     RGBA, L, VTR_IB),
+   FMT(R8G8B8X8_SNORM,          RGBA8_SNORM,     RGB1, L, VT__IB),
+   FMT(R8G8B8X8_SRGB,           RGBA8_UNORM,     RGB1, S, VTR_IB),
+   FMT(R8G8B8X8_UINT,           RGBA8UI,         RGB1, L, VTR_IB),
+   FMT(R8G8B8X8_SINT,           RGBA8I,          RGB1, L, VTR_IB),
+   FMT(L8_UNORM,                R8_UNORM,        RRR1, L, VTR_IB),
+   FMT(I8_UNORM,                R8_UNORM,        RRRR, L, VTR_IB),
+   FMT(L16_UNORM,               R16_UNORM,       RRR1, L, VT__IB),
+   FMT(I16_UNORM,               R16_UNORM,       RRRR, L, VT__IB),
+   FMT(L8_SNORM,                R8_SNORM,        RRR1, L, VT__IB),
+   FMT(I8_SNORM,                R8_SNORM,        RRRR, L, VT__IB),
+   FMT(L16_SNORM,               R16_SNORM,       RRR1, L, VT__IB),
+   FMT(I16_SNORM,               R16_SNORM,       RRRR, L, VT__IB),
+   FMT(L16_FLOAT,               R16F,            RRR1, L, VTR_IB),
+   FMT(I16_FLOAT,               R16F,            RRRR, L, VTR_IB),
+   FMT(L8_SRGB,                 R8_UNORM,        RRR1, S, VTR_IB),
+   FMT(R8_SRGB,                 R8_UNORM,        R001, S, VTR_IB),
+   FMT(R8G8_SRGB,               RG8_UNORM,       RG01, S, VTR_IB),
+   FMT(R8G8B8_SRGB,             RGB8_UNORM,      RGB1, S, VTR_IB),
+   FMT(B8G8R8_SRGB,             RGB8_UNORM,      BGR1, S, VTR_IB),
+   FMT(R8G8B8A8_SRGB,           RGBA8_UNORM,     RGBA, S, VTR_IB),
+   FMT(A8B8G8R8_SRGB,           RGBA8_UNORM,     ABGR, S, VTR_IB),
+   FMT(X8B8G8R8_SRGB,           RGBA8_UNORM,     ABG1, S, VTR_IB),
+   FMT(B8G8R8A8_SRGB,           RGBA8_UNORM,     BGRA, S, VTR_IB),
+   FMT(B8G8R8X8_SRGB,           RGBA8_UNORM,     BGR1, S, VTR_IB),
+   FMT(A8R8G8B8_SRGB,           RGBA8_UNORM,     GBAR, S, VTR_IB),
+   FMT(X8R8G8B8_SRGB,           RGBA8_UNORM,     GBA1, S, VTR_IB),
+   FMT(R8_SINT,                 R8I,             R001, L, VTR_IB),
+   FMT(R16_SINT,                R16I,            R001, L, VTR_IB),
+   FMT(R32_SINT,                R32I,            R001, L, VTR_IB),
+   FMT(R16_FLOAT,               R16F,            R001, L, VTR_IB),
+   FMT(R8G8_SINT,               RG8I,            RG01, L, VTR_IB),
+   FMT(R16G16_SINT,             RG16I,           RG01, L, VTR_IB),
+   FMT(R32G32_SINT,             RG32I,           RG01, L, VTR_IB),
+   FMT(R16G16_FLOAT,            RG16F,           RG01, L, VTR_IB),
+   FMT(R8G8B8_SINT,             RGB8I,           RGB1, L, V_____),
+   FMT(R32G32B32_SINT,          RGB32I,          RGB1, L, VTR_IB),
+   FMT(R8G8B8A8_SINT,           RGBA8I,          RGBA, L, VTR_IB),
+   FMT(R16G16B16A16_SINT,       RGBA16I,         RGBA, L, VTR_IB),
+   FMT(R32G32B32A32_SINT,       RGBA32I,         RGBA, L, VTR_IB),
+   FMT(R16G16B16A16_FLOAT,      RGBA16F,         RGBA, L, VTR_IB),
+   FMT(R16G16B16X16_UNORM,      RGBA16_UNORM,    RGB1, L, VTR_IB),
+   FMT(R16G16B16X16_SNORM,      RGBA16_SNORM,    RGB1, L, VT__IB),
+   FMT(R16G16B16X16_FLOAT,      RGBA16F,         RGB1, L, VTR_IB),
+   FMT(R16G16B16X16_UINT,       RGBA16UI,        RGB1, L, VTR_IB),
+   FMT(R16G16B16X16_SINT,       RGBA16I,         RGB1, L, VTR_IB),
+   FMT(R32G32B32X32_FLOAT,      RGBA32F,         RGB1, L, VTR_IB),
+   FMT(R32G32B32X32_UINT,       RGBA32UI,        RGB1, L, VTR_IB),
+   FMT(R32G32B32X32_SINT,       RGBA32I,         RGB1, L, VTR_IB),
+
+#if PAN_ARCH <= 6
+   FMT(Z16_UNORM,               R16_UNORM,       RRRR, L, _T_Z__),
+   FMT(Z24_UNORM_S8_UINT,       Z24X8_UNORM,     RRRR, L, _T_Z__),
+   FMT(Z24X8_UNORM,             Z24X8_UNORM,     RRRR, L, _T_Z__),
+   FMT(Z32_FLOAT,               R32F,            RRRR, L, _T_Z__),
+   FMT(Z32_FLOAT_S8X24_UINT,    RG32F,           RRRR, L, _T_Z__),
+   FMT(X32_S8X24_UINT,          X32_S8X24,       GGGG, L, _T_Z__),
+   FMT(X24S8_UINT,              RGBA8UI,         AAAA, L, _T_Z__),
+   FMT(S8_UINT,                 R8UI,            RRRR, L, _T_Z__),
+
+   FMT(A8_UNORM,                R8_UNORM,        000R, L, VTR___),
+   FMT(L8A8_UNORM,              RG8_UNORM,       RRRG, L, VTR_IB),
+   FMT(L8A8_SRGB,               RG8_UNORM,       RRRG, S, VTR_IB),
+
+   /* These formats were removed in v7 */
+   FMT(A8_SNORM,                R8_SNORM,        000R, L, VT__IB),
+   FMT(A8_SINT,                 R8I,             000R, L, VTR_IB),
+   FMT(A8_UINT,                 R8UI,            000R, L, VTR_IB),
+   FMT(A16_SINT,                R16I,            000R, L, VTR_IB),
+   FMT(A16_UINT,                R16UI,           000R, L, VTR_IB),
+   FMT(A32_SINT,                R32I,            000R, L, VTR_IB),
+   FMT(A32_UINT,                R32UI,           000R, L, VTR_IB),
+   FMT(A16_UNORM,               R16_UNORM,       000R, L, VT__IB),
+   FMT(A16_SNORM,               R16_SNORM,       000R, L, VT__IB),
+   FMT(A16_FLOAT,               R16F,            000R, L, VTR_IB),
+
+#else
+   FMT(Z16_UNORM,               Z16_UNORM,       RGBA, L, _T_Z__),
+   FMT(Z24_UNORM_S8_UINT,       Z24X8_UNORM,     RGBA, L, _T_Z__),
+   FMT(Z24X8_UNORM,             Z24X8_UNORM,     RGBA, L, _T_Z__),
+   FMT(Z32_FLOAT,               Z32,             RGBA, L, _T_Z__),
+
+#if PAN_ARCH >= 9
+   /* Specify interchange formats, the actual format for depth/stencil is
+    * determined by the plane descriptor on Valhall.
+    *
+    * On Valhall, S8 logically acts like "X8S8", so "S8 RGBA" is logically
+    * "0s00" and "S8 GRBA" is logically "s000". For Bifrost compatibility
+    * we want stencil in the red channel, so we use the GRBA swizzles.
+    */
+   FMT(Z32_FLOAT_S8X24_UINT,    R32F,            GRBA, L, _T_Z__),
+   FMT(Z24_UNORM_PACKED,        Z24X8_UNORM,     RGBA, L, _T_Z__),
+   FMT(X32_S8X24_UINT,          S8,              GRBA, L, _T_Z__),
+   FMT(X24S8_UINT,              S8,              GRBA, L, _T_Z__),
+   FMT(S8_UINT,                 S8,              GRBA, L, _T_Z__),
+
+   /* similarly, the interchange format is RGBA8, but we only
+      actually store 1 component in memory here */
+   FMT(A8_UNORM,                RGBA8_UNORM,     000A, L, VTR___),
+#else
+   /* Specify real formats on Bifrost */
+   FMT(Z32_FLOAT_S8X24_UINT,    Z32_X32,         RGBA, L, _T_Z__),
+   FMT(X32_S8X24_UINT,          X32_S8X24,       GRBA, L, _T_Z__),
+   FMT(X24S8_UINT,              X24S8,           GRBA, L, _T_Z__),
+   FMT(S8_UINT,                 S8,              GRBA, L, _T_Z__),
+
+   /* Obsolete formats removed in Valhall */
+   FMT(A8_UNORM,                A8_UNORM,        000A, L, VTR___),
+   FMT(L8A8_UNORM,              R8A8_UNORM,      RRRA, L, VTR_IB),
+   FMT(L8A8_SRGB,               R8A8_UNORM,      RRRA, S, VTR_IB),
+#endif
+
+#endif
+};
+/* clang-format on */
+
+/* This is only needed on Midgard. It's the same on both v4 and v5, so only
+ * compile once to avoid the GenXML dependency for calls.
+ */
+#if PAN_ARCH == 5
+uint8_t
+pan_raw_format_mask_midgard(enum pipe_format *formats)
+{
+   uint8_t out = 0;
+
+   for (unsigned i = 0; i < 8; i++) {
+      enum pipe_format fmt = formats[i];
+      unsigned wb_fmt = pan_blendable_formats_v6[fmt].writeback;
+
+      if (wb_fmt < MALI_COLOR_FORMAT_R8)
+         out |= BITFIELD_BIT(i);
+   }
+
+   return out;
+}
+#endif
+
+#if PAN_ARCH == 7 || PAN_ARCH >= 10
+/*
+ * Decompose a component ordering swizzle into a component ordering (applied
+ * first) and a swizzle (applied second). The output ordering "pre" is allowed
+ * with compression and the swizzle "post" is a bijection.
+ *
+ * These properties allow any component ordering to be used with compression, by
+ * using the output "pre" ordering, composing the API swizzle with the "post"
+ * ordering, and applying the inverse of the "post" ordering to the border
+ * colour to undo what we compose into the API swizzle.
+ *
+ * Note that "post" is a swizzle, not a component ordering, which means it is
+ * inverted from the ordering. E.g. ARGB ordering uses a GBAR (YZWX) swizzle.
+ */
+struct pan_decomposed_swizzle
+GENX(pan_decompose_swizzle)(enum mali_rgb_component_order order)
+{
+#define CASE(case_, pre_, R_, G_, B_, A_)                                      \
+   case MALI_RGB_COMPONENT_ORDER_##case_:                                      \
+      return (struct pan_decomposed_swizzle){                                  \
+         MALI_RGB_COMPONENT_ORDER_##pre_,                                      \
+         {                                                                     \
+            PIPE_SWIZZLE_##R_,                                                 \
+            PIPE_SWIZZLE_##G_,                                                 \
+            PIPE_SWIZZLE_##B_,                                                 \
+            PIPE_SWIZZLE_##A_,                                                 \
+         },                                                                    \
+      };
+
+   switch (order) {
+      CASE(RGBA, RGBA, X, Y, Z, W);
+      CASE(GRBA, RGBA, Y, X, Z, W);
+      CASE(BGRA, RGBA, Z, Y, X, W);
+      CASE(ARGB, RGBA, Y, Z, W, X);
+      CASE(AGRB, RGBA, Z, Y, W, X);
+      CASE(ABGR, RGBA, W, Z, Y, X);
+      CASE(RGB1, RGB1, X, Y, Z, W);
+      CASE(GRB1, RGB1, Y, X, Z, W);
+      CASE(BGR1, RGB1, Z, Y, X, W);
+      CASE(1RGB, RGB1, Y, Z, W, X);
+      CASE(1GRB, RGB1, Z, Y, W, X);
+      CASE(1BGR, RGB1, W, Z, Y, X);
+      CASE(RRRR, RRRR, X, Y, Z, W);
+      CASE(RRR1, RRR1, X, Y, Z, W);
+      CASE(RRRA, RRRA, X, Y, Z, W);
+      CASE(000A, 000A, X, Y, Z, W);
+      CASE(0001, 0001, X, Y, Z, W);
+      CASE(0000, 0000, X, Y, Z, W);
+   default:
+      UNREACHABLE("Invalid case for texturing");
+   }
+
+#undef CASE
+}
+#endif

src/panfrost/lib/pan_format.h

@@ -0,0 +1,370 @@
+/*
+ * Copyright (C) 2008 VMware, Inc.
+ * Copyright (C) 2014 Broadcom
+ * Copyright (C) 2018-2019 Alyssa Rosenzweig
+ * Copyright (C) 2019-2020 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef __PAN_FORMAT_H
+#define __PAN_FORMAT_H
+
+#include "genxml/gen_macros.h"
+
+#include "util/format/u_format.h"
+
+#include "drm-uapi/drm_fourcc.h"
+
+/*
+ * List of supported modifiers, in descending order of preference. AFBC is
+ * faster than u-interleaved tiling which is faster than linear. Within AFBC,
+ * enabling the YUV-like transform is typically a win where possible.
+ * AFRC is only used if explicitly asked for (only for RGB formats).
+ * Similarly MTK 16L32 is only used if explicitly asked for.
+ */
+#define PAN_SUPPORTED_MODIFIERS(__name)                                        \
+   static const uint64_t __name[] = {                                          \
+      DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_32x8 |                \
+                              AFBC_FORMAT_MOD_SPARSE),                         \
+      DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_32x8 |                \
+                              AFBC_FORMAT_MOD_SPARSE | AFBC_FORMAT_MOD_SPLIT), \
+      DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_32x8 |                \
+                              AFBC_FORMAT_MOD_SPARSE | AFBC_FORMAT_MOD_SPLIT | \
+                              AFBC_FORMAT_MOD_YTR),                            \
+                                                                               \
+      DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_16x16 |               \
+                              AFBC_FORMAT_MOD_TILED | AFBC_FORMAT_MOD_SC |     \
+                              AFBC_FORMAT_MOD_SPARSE | AFBC_FORMAT_MOD_YTR),   \
+                                                                               \
+      DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_16x16 |               \
+                              AFBC_FORMAT_MOD_TILED | AFBC_FORMAT_MOD_SC |     \
+                              AFBC_FORMAT_MOD_SPARSE),                         \
+                                                                               \
+      DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_16x16 |               \
+                              AFBC_FORMAT_MOD_SPARSE | AFBC_FORMAT_MOD_YTR),   \
+                                                                               \
+      DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_16x16 |               \
+                              AFBC_FORMAT_MOD_SPARSE),                         \
+                                                                               \
+      DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_16x16 |               \
+                              AFBC_FORMAT_MOD_SPARSE | AFBC_FORMAT_MOD_SPLIT), \
+                                                                               \
+      DRM_FORMAT_MOD_ARM_INTERLEAVED_64K,                                      \
+      DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED,                            \
+      DRM_FORMAT_MOD_LINEAR,                                                   \
+                                                                               \
+      DRM_FORMAT_MOD_ARM_AFRC(                                                 \
+         AFRC_FORMAT_MOD_CU_SIZE_P0(AFRC_FORMAT_MOD_CU_SIZE_16)),              \
+      DRM_FORMAT_MOD_ARM_AFRC(                                                 \
+         AFRC_FORMAT_MOD_CU_SIZE_P0(AFRC_FORMAT_MOD_CU_SIZE_24)),              \
+      DRM_FORMAT_MOD_ARM_AFRC(                                                 \
+         AFRC_FORMAT_MOD_CU_SIZE_P0(AFRC_FORMAT_MOD_CU_SIZE_32)),              \
+      DRM_FORMAT_MOD_ARM_AFRC(                                                 \
+         AFRC_FORMAT_MOD_CU_SIZE_P0(AFRC_FORMAT_MOD_CU_SIZE_16) |              \
+         AFRC_FORMAT_MOD_LAYOUT_SCAN),                                         \
+      DRM_FORMAT_MOD_ARM_AFRC(                                                 \
+         AFRC_FORMAT_MOD_CU_SIZE_P0(AFRC_FORMAT_MOD_CU_SIZE_24) |              \
+         AFRC_FORMAT_MOD_LAYOUT_SCAN),                                         \
+      DRM_FORMAT_MOD_ARM_AFRC(                                                 \
+         AFRC_FORMAT_MOD_CU_SIZE_P0(AFRC_FORMAT_MOD_CU_SIZE_32) |              \
+         AFRC_FORMAT_MOD_LAYOUT_SCAN),                                         \
+   }
+
+/* DRM modifier helper */
+
+#define drm_is_afbc(mod)                                                       \
+   ((mod >> 52) ==                                                             \
+    (DRM_FORMAT_MOD_ARM_TYPE_AFBC | (DRM_FORMAT_MOD_VENDOR_ARM << 4)))
+
+#define drm_is_afrc(mod)                                                       \
+   ((mod >> 52) ==                                                             \
+    (DRM_FORMAT_MOD_ARM_TYPE_AFRC | (DRM_FORMAT_MOD_VENDOR_ARM << 4)))
+
+static inline bool
+pan_u_tiled_or_linear_supports_format(enum pipe_format format)
+{
+   switch (format) {
+   case PIPE_FORMAT_R8G8B8_420_UNORM_PACKED:
+   case PIPE_FORMAT_R10G10B10_420_UNORM_PACKED:
+      return false;
+
+   default:
+      return true;
+   }
+}
+
+/* Formats */
+
+static inline unsigned
+pan_format_get_plane_blocksize(enum pipe_format format, unsigned plane_idx)
+{
+   switch (format) {
+   case PIPE_FORMAT_R8_G8B8_420_UNORM:
+   case PIPE_FORMAT_R8_B8G8_420_UNORM:
+   case PIPE_FORMAT_R8_G8B8_422_UNORM:
+   case PIPE_FORMAT_R8_B8G8_422_UNORM:
+      return plane_idx ? 2 : 1;
+   case PIPE_FORMAT_R10_G10B10_420_UNORM:
+   case PIPE_FORMAT_R10_G10B10_422_UNORM:
+      return plane_idx ? 10 : 5;
+   case PIPE_FORMAT_R8_G8_B8_420_UNORM:
+   case PIPE_FORMAT_R8_B8_G8_420_UNORM:
+      return 1;
+   default:
+      assert(util_format_get_num_planes(format) == 1);
+      return util_format_get_blocksize(format);
+   }
+}
+
+static inline unsigned
+pan_format_tib_size(enum pipe_format format, bool internal)
+{
+   if (internal && !util_format_is_float(format)) {
+      /* Blendable UNORM and sRGB formats are always 32-bits in the tile
+       * buffer, extra bits are used as padding or to dither */
+      return 4;
+   } else {
+      /* Non-blendable and float formats are raw, rounded up to the nearest
+       * power-of-two size */
+      unsigned bytes = util_format_get_blocksize(format);
+      return util_next_power_of_two(bytes);
+   }
+}
+
+typedef uint32_t mali_pixel_format;
+
+/* pan bind flags */
+#define PAN_BIND_DEPTH_STENCIL BITFIELD_BIT(0)
+#define PAN_BIND_RENDER_TARGET BITFIELD_BIT(1)
+#define PAN_BIND_SAMPLER_VIEW  BITFIELD_BIT(2)
+#define PAN_BIND_VERTEX_BUFFER BITFIELD_BIT(3)
+#define PAN_BIND_STORAGE_IMAGE BITFIELD_BIT(4)
+#define PAN_BIND_TEXEL_BUFFER  BITFIELD_BIT(5)
+
+struct pan_format {
+   uint32_t hw          : 21;
+   uint32_t texfeat_bit : 5;
+   uint32_t bind        : 6;
+};
+
+struct pan_blendable_format {
+   /* enum mali_color_buffer_internal_format */ uint16_t internal;
+   /* enum mali_mfbd_color_format */ uint16_t writeback;
+
+   /* Indexed by the dithered? flag. So _PU first, then _AU */
+   mali_pixel_format bifrost[2];
+};
+
+#define pan_blendable_formats_v4 pan_blendable_formats_v5
+extern const struct pan_blendable_format
+   pan_blendable_formats_v5[PIPE_FORMAT_COUNT];
+extern const struct pan_blendable_format
+   pan_blendable_formats_v6[PIPE_FORMAT_COUNT];
+extern const struct pan_blendable_format
+   pan_blendable_formats_v7[PIPE_FORMAT_COUNT];
+extern const struct pan_blendable_format
+   pan_blendable_formats_v9[PIPE_FORMAT_COUNT];
+extern const struct pan_blendable_format
+   pan_blendable_formats_v10[PIPE_FORMAT_COUNT];
+extern const struct pan_blendable_format
+   pan_blendable_formats_v12[PIPE_FORMAT_COUNT];
+extern const struct pan_blendable_format
+   pan_blendable_formats_v13[PIPE_FORMAT_COUNT];
+extern const struct pan_blendable_format
+   pan_blendable_formats_v14[PIPE_FORMAT_COUNT];
+
+uint8_t pan_raw_format_mask_midgard(enum pipe_format *formats);
+
+static inline const struct pan_blendable_format *
+pan_blendable_format_table(unsigned arch)
+{
+   switch (arch) {
+#define FMT_TABLE(x) case x: return pan_blendable_formats_v ## x
+   FMT_TABLE(4);
+   FMT_TABLE(5);
+   FMT_TABLE(6);
+   FMT_TABLE(7);
+   FMT_TABLE(9);
+   FMT_TABLE(10);
+   FMT_TABLE(12);
+   FMT_TABLE(13);
+   FMT_TABLE(14);
+#undef FMT_TABLE
+   default:
+      assert(!"Unsupported architecture");
+      return NULL;
+   }
+}
+
+#define pan_pipe_format_v4 pan_pipe_format_v5
+extern const struct pan_format pan_pipe_format_v5[PIPE_FORMAT_COUNT];
+extern const struct pan_format pan_pipe_format_v6[PIPE_FORMAT_COUNT];
+extern const struct pan_format pan_pipe_format_v7[PIPE_FORMAT_COUNT];
+extern const struct pan_format pan_pipe_format_v9[PIPE_FORMAT_COUNT];
+extern const struct pan_format pan_pipe_format_v10[PIPE_FORMAT_COUNT];
+extern const struct pan_format pan_pipe_format_v12[PIPE_FORMAT_COUNT];
+extern const struct pan_format pan_pipe_format_v13[PIPE_FORMAT_COUNT];
+extern const struct pan_format pan_pipe_format_v14[PIPE_FORMAT_COUNT];
+
+static inline const struct pan_format *
+pan_format_table(unsigned arch)
+{
+   switch (arch) {
+#define FMT_TABLE(x) case x: return pan_pipe_format_v ## x
+   FMT_TABLE(4);
+   FMT_TABLE(5);
+   FMT_TABLE(6);
+   FMT_TABLE(7);
+   FMT_TABLE(9);
+   FMT_TABLE(10);
+   FMT_TABLE(12);
+   FMT_TABLE(13);
+   FMT_TABLE(14);
+#undef FMT_TABLE
+   default:
+      assert(!"Unsupported architecture");
+      return NULL;
+   }
+}
+
+/* Helpers to construct swizzles */
+
+#define PAN_V6_SWIZZLE(R, G, B, A)                                             \
+   (((MALI_CHANNEL_##R) << 0) | ((MALI_CHANNEL_##G) << 3) |                    \
+    ((MALI_CHANNEL_##B) << 6) | ((MALI_CHANNEL_##A) << 9))
+
+static inline unsigned
+pan_get_default_swizzle(unsigned components)
+{
+   switch (components) {
+   case 1:
+      return PAN_V6_SWIZZLE(R, 0, 0, 1);
+   case 2:
+      return PAN_V6_SWIZZLE(R, G, 0, 1);
+   case 3:
+      return PAN_V6_SWIZZLE(R, G, B, 1);
+   case 4:
+      return PAN_V6_SWIZZLE(R, G, B, A);
+   default:
+      UNREACHABLE("Invalid number of components");
+   }
+}
+
+#if PAN_ARCH == 7 || PAN_ARCH >= 10
+struct pan_decomposed_swizzle {
+   /* Component ordering to apply first */
+   enum mali_rgb_component_order pre;
+
+   /* Bijective swizzle applied after */
+   unsigned char post[4];
+};
+
+struct pan_decomposed_swizzle
+   GENX(pan_decompose_swizzle)(enum mali_rgb_component_order order);
+#endif
+
+#define MALI_SRGB_L (0)
+#define MALI_SRGB_S (1)
+
+#if PAN_ARCH <= 6
+
+#define MALI_V6_0000 PAN_V6_SWIZZLE(0, 0, 0, 0)
+#define MALI_V6_000R PAN_V6_SWIZZLE(0, 0, 0, R)
+#define MALI_V6_0R00 PAN_V6_SWIZZLE(0, R, 0, 0)
+#define MALI_V6_0A00 PAN_V6_SWIZZLE(0, A, 0, 0)
+#define MALI_V6_AAAA PAN_V6_SWIZZLE(A, A, A, A)
+#define MALI_V6_A001 PAN_V6_SWIZZLE(A, 0, 0, 1)
+#define MALI_V6_ABG1 PAN_V6_SWIZZLE(A, B, G, 1)
+#define MALI_V6_ABGR PAN_V6_SWIZZLE(A, B, G, R)
+#define MALI_V6_ARGB PAN_V6_SWIZZLE(A, R, G, B)
+#define MALI_V6_BGR1 PAN_V6_SWIZZLE(B, G, R, 1)
+#define MALI_V6_BGRA PAN_V6_SWIZZLE(B, G, R, A)
+#define MALI_V6_GBA1 PAN_V6_SWIZZLE(G, B, A, 1)
+#define MALI_V6_GBAR PAN_V6_SWIZZLE(G, B, A, R)
+#define MALI_V6_R000 PAN_V6_SWIZZLE(R, 0, 0, 0)
+#define MALI_V6_R001 PAN_V6_SWIZZLE(R, 0, 0, 1)
+#define MALI_V6_RG01 PAN_V6_SWIZZLE(R, G, 0, 1)
+#define MALI_V6_RGB1 PAN_V6_SWIZZLE(R, G, B, 1)
+#define MALI_V6_RGBA PAN_V6_SWIZZLE(R, G, B, A)
+#define MALI_V6_RRR1 PAN_V6_SWIZZLE(R, R, R, 1)
+#define MALI_V6_RRRG PAN_V6_SWIZZLE(R, R, R, G)
+#define MALI_V6_RRRR PAN_V6_SWIZZLE(R, R, R, R)
+#define MALI_V6_GGGG PAN_V6_SWIZZLE(G, G, G, G)
+
+#define MALI_PACK_FMT(mali, swizzle, srgb)                                     \
+   (MALI_V6_##swizzle) | ((MALI_##mali) << 12) | (((MALI_SRGB_##srgb)) << 20)
+
+#else
+
+#define MALI_RGB_COMPONENT_ORDER_R001 MALI_RGB_COMPONENT_ORDER_RGB1
+#define MALI_RGB_COMPONENT_ORDER_RG01 MALI_RGB_COMPONENT_ORDER_RGB1
+#define MALI_RGB_COMPONENT_ORDER_GBAR MALI_RGB_COMPONENT_ORDER_ARGB
+#define MALI_RGB_COMPONENT_ORDER_GBA1 MALI_RGB_COMPONENT_ORDER_1RGB
+#define MALI_RGB_COMPONENT_ORDER_ABG1 MALI_RGB_COMPONENT_ORDER_1BGR
+
+#define MALI_PACK_FMT(mali, swizzle, srgb)                                     \
+   (MALI_RGB_COMPONENT_ORDER_##swizzle) | ((MALI_##mali) << 12) |              \
+      (((MALI_SRGB_##srgb)) << 20)
+
+#endif
+
+#define MALI_EXTRACT_INDEX(pixfmt) (((pixfmt) >> 12) & 0xFF)
+
+static inline bool
+pan_format_is_yuv(enum pipe_format f)
+{
+   enum util_format_layout layout = util_format_description(f)->layout;
+
+   /* Mesa's subsampled RGB formats are considered YUV formats on Mali */
+   return layout == UTIL_FORMAT_LAYOUT_SUBSAMPLED ||
+          layout == UTIL_FORMAT_LAYOUT_PLANAR2 ||
+          layout == UTIL_FORMAT_LAYOUT_PLANAR3;
+}
+
+#ifdef PAN_ARCH
+static inline const struct pan_format *
+GENX(pan_format_from_pipe_format)(enum pipe_format f)
+{
+   return &GENX(pan_pipe_format)[f];
+}
+
+static inline const struct pan_blendable_format *
+GENX(pan_blendable_format_from_pipe_format)(enum pipe_format f)
+{
+   return &GENX(pan_blendable_formats)[f];
+}
+
+#if PAN_ARCH >= 6
+static inline unsigned
+GENX(pan_dithered_format_from_pipe_format)(enum pipe_format f, bool dithered)
+{
+   mali_pixel_format pixfmt = GENX(pan_blendable_formats)[f].bifrost[dithered];
+
+   /* Formats requiring blend shaders are stored raw in the tilebuffer and will
+    * have 0 as their pixel format. Assumes dithering is set, I don't know of a
+    * case when it makes sense to turn off dithering. */
+   return pixfmt ?: GENX(pan_format_from_pipe_format)(f)->hw;
+}
+#endif
+
+static inline bool
+GENX(pan_format_supports_hw_blend)(enum pipe_format format)
+{
+   return GENX(pan_blendable_format_from_pipe_format)(format)->internal;
+}
+
+static inline bool
+GENX(pan_format_supports_msaa_average)(enum pipe_format format)
+{
+   if (!GENX(pan_format_supports_hw_blend)(format))
+      return false;
+
+   /* F16 formats are blendable on v10 but don't support averaging until v11 */
+   if (util_format_is_float16(format) && PAN_ARCH < 11)
+      return false;
+
+   return true;
+}
+#endif
+
+#endif

src/panfrost/lib/pan_image.h

@@ -0,0 +1,354 @@
+/*
+ * Copyright (C) 2008 VMware, Inc.
+ * Copyright (C) 2014 Broadcom
+ * Copyright (C) 2018-2019 Alyssa Rosenzweig
+ * Copyright (C) 2019-2020 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef __PAN_IMAGE_H
+#define __PAN_IMAGE_H
+
+#include "genxml/gen_macros.h"
+
+#include <stdbool.h>
+#include "util/format/u_format.h"
+#include "pan_format.h"
+#include "pan_layout.h"
+#include "pan_mod.h"
+#include "pan_props.h"
+
+#include "kmod/pan_kmod.h"
+
+#include "util/log.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct pan_mod_handler;
+
+struct pan_image_plane {
+   struct pan_image_layout layout;
+   uint64_t base;
+};
+
+struct pan_image {
+   struct pan_image_props props;
+   const struct pan_mod_handler *mod_handler;
+   struct pan_image_plane *planes[MAX_IMAGE_PLANES];
+};
+
+struct pan_image_plane_ref {
+   struct pan_image *image;
+   uint32_t plane_idx;
+};
+
+struct pan_image_view {
+   /* Format, dimension and sample count of the view might differ from
+    * those of the image (2D view of a 3D image surface for instance).
+    */
+   enum pipe_format format;
+   enum mali_texture_dimension dim;
+   unsigned first_level, last_level;
+   unsigned first_layer, last_layer;
+   float min_lod;
+   unsigned char swizzle[4];
+
+   /* planes 1 and 2 are NULL for single plane formats */
+   struct pan_image_plane_ref planes[MAX_IMAGE_PLANES];
+
+   /* If EXT_multisampled_render_to_texture is used, this may be
+    * greater than image->layout.nr_samples. */
+   unsigned nr_samples;
+
+   struct {
+      unsigned narrow;
+      unsigned hdr;
+   } astc;
+};
+
+static inline struct pan_image_plane_ref
+pan_image_view_get_plane(const struct pan_image_view *iview, uint32_t idx)
+{
+   if (idx >= ARRAY_SIZE(iview->planes))
+      return (struct pan_image_plane_ref){0};
+
+   return iview->planes[idx];
+}
+
+static inline unsigned
+pan_image_view_get_plane_mask(const struct pan_image_view *iview)
+{
+   unsigned mask = 0;
+
+   for (unsigned i = 0; i < ARRAY_SIZE(iview->planes); i++) {
+      if (iview->planes[i].image)
+         mask |= BITFIELD_BIT(i);
+   }
+
+   return mask;
+}
+
+static inline unsigned
+pan_image_view_get_first_plane_idx(const struct pan_image_view *iview)
+{
+   unsigned mask = pan_image_view_get_plane_mask(iview);
+
+   assert(mask);
+   return ffs(mask) - 1;
+}
+
+static inline struct pan_image_plane_ref
+pan_image_view_get_first_plane(const struct pan_image_view *iview)
+{
+   unsigned first_plane_idx = pan_image_view_get_first_plane_idx(iview);
+   return pan_image_view_get_plane(iview, first_plane_idx);
+}
+
+static inline uint32_t
+pan_image_view_get_nr_samples(const struct pan_image_view *iview)
+{
+   const struct pan_image_plane_ref pref = pan_image_view_get_first_plane(iview);
+
+   if (!pref.image)
+      return 0;
+
+   return pref.image->props.nr_samples;
+}
+
+static inline uint32_t
+pan_image_view_get_layer_count(const struct pan_image_view *iview)
+{
+   const struct pan_image_plane_ref pref = pan_image_view_get_first_plane(iview);
+
+   if (!pref.image)
+      return 0;
+
+   return iview->dim == MALI_TEXTURE_DIMENSION_3D
+                        ? pref.image->props.extent_px.depth
+                        : iview->last_layer - iview->first_layer + 1;
+}
+
+static inline struct pan_image_plane_ref
+pan_image_view_get_color_plane(const struct pan_image_view *iview)
+{
+   /* We only support rendering to plane 0 */
+   assert(pan_image_view_get_plane(iview, 1).image == NULL);
+   return pan_image_view_get_plane(iview, 0);
+}
+
+static inline bool
+pan_image_view_has_crc(const struct pan_image_view *iview)
+{
+   const struct pan_image_plane_ref p = pan_image_view_get_color_plane(iview);
+
+   if (!p.image)
+      return false;
+
+   return p.image->props.crc;
+}
+
+static inline struct pan_image_plane_ref
+pan_image_view_get_s_plane(const struct pan_image_view *iview)
+{
+   ASSERTED const struct util_format_description *fdesc =
+      util_format_description(iview->format);
+   assert(util_format_has_stencil(fdesc));
+
+   /* In case of multiplanar depth/stencil, the stencil is always on
+    * plane 1. Combined depth/stencil only has one plane, so depth
+    * will be on plane 0 in either case.
+    */
+   const struct pan_image_plane_ref pref =
+      iview->planes[1].image ? iview->planes[1] : iview->planes[0];
+
+   assert(pref.image);
+   fdesc = util_format_description(pref.image->props.format);
+   assert(util_format_has_stencil(fdesc));
+   return pref;
+}
+
+static inline struct pan_image_plane_ref
+pan_image_view_get_zs_plane(const struct pan_image_view *iview)
+{
+   assert(util_format_is_depth_or_stencil(iview->format));
+
+   /* Depth or combined depth-stencil is always on plane 0. */
+   return pan_image_view_get_plane(iview, 0);
+}
+
+static inline void
+pan_image_view_check(const struct pan_image_view *iview)
+{
+#ifndef NDEBUG
+   unsigned nplanes = util_format_get_num_planes(iview->format);
+   struct pan_image_plane_ref pref;
+
+   for (unsigned i = 0; i < nplanes; i++) {
+      if (util_format_is_depth_or_stencil(iview->format)) {
+         const struct util_format_description *fdesc =
+            util_format_description(iview->format);
+
+         if (util_format_has_stencil(fdesc))
+            pref = pan_image_view_get_s_plane(iview);
+         else
+            pref = pan_image_view_get_zs_plane(iview);
+      } else {
+         pref = iview->planes[i];
+      }
+
+      /* Make sure we have an image and the plane we point to exists. */
+      assert(pref.image);
+      assert(pref.plane_idx <
+             util_format_get_num_planes(pref.image->props.format));
+
+      enum pipe_format view_format =
+         util_format_get_plane_format(iview->format, i);
+      enum pipe_format img_format =
+         util_format_get_plane_format(pref.image->props.format, pref.plane_idx);
+
+      /* View-based pixel re-interpretation only allowed if the formats
+       * blocksize match. Exception is for Z24X8 with AFBC enabled since then
+       * we can lower it to Z24 packed format internally. */
+      assert((util_format_get_blocksize(view_format) ==
+              util_format_get_blocksize(img_format)) ||
+             (view_format == PIPE_FORMAT_Z24X8_UNORM &&
+              img_format == PIPE_FORMAT_Z24_UNORM_PACKED &&
+              drm_is_afbc(pref.image->props.modifier)));
+   }
+#endif
+}
+
+static inline uint64_t
+pan_image_mip_level_size(const struct pan_image *image, unsigned plane_idx,
+                         unsigned mip_level)
+{
+   assert(plane_idx < ARRAY_SIZE(image->planes) &&
+          plane_idx < util_format_get_num_planes(image->props.format));
+   assert(mip_level < image->props.nr_slices);
+   assert(image->planes[plane_idx]);
+
+   uint64_t size = image->planes[plane_idx]->layout.slices[mip_level].size_B;
+
+   /* If this is an array, we need to cover the whole array. */
+   if (image->props.array_size > 1) {
+      size += image->planes[plane_idx]->layout.array_stride_B *
+              (image->props.array_size - 1);
+   }
+
+   return size;
+}
+
+static inline uint32_t
+pan_image_get_wsi_row_pitch(const struct pan_image *image, unsigned plane_idx,
+                            unsigned mip_level)
+{
+   assert(image->mod_handler);
+   assert(image->mod_handler->get_wsi_row_pitch);
+   assert(plane_idx < ARRAY_SIZE(image->planes) &&
+          plane_idx < util_format_get_num_planes(image->props.format));
+   assert(image->planes[plane_idx]);
+
+   return image->mod_handler->get_wsi_row_pitch(image, plane_idx, mip_level);
+}
+
+static inline uint64_t
+pan_image_get_wsi_offset(const struct pan_image *image, unsigned plane_idx,
+                         unsigned mip_level)
+{
+   assert(plane_idx < ARRAY_SIZE(image->planes) &&
+          plane_idx < util_format_get_num_planes(image->props.format));
+   assert(mip_level < image->props.nr_slices);
+   assert(image->planes[plane_idx]);
+
+   return image->planes[plane_idx]->layout.slices[mip_level].offset_B;
+}
+
+bool pan_image_layout_init(
+   unsigned arch, struct pan_image *image, unsigned plane_idx,
+   const struct pan_image_layout_constraints *explicit_layout_constraints);
+
+struct pan_image_usage {
+   /* PAN_BIND_xxx flags. */
+   uint32_t bind;
+
+   /* Image needs to be mappable at standard sparse block granularity. */
+   bool standard_sparse_mapping_granularity;
+
+   /* Image filled directly from the CPU. */
+   bool host_copy;
+
+   /* Image frequently updated with host data. */
+   bool frequent_host_updates;
+
+   /* Legacy scanout image. */
+   bool legacy_scanout;
+
+   /* Image created by WSI. */
+   bool wsi;
+};
+
+static inline enum pan_mod_support
+pan_image_test_props(const struct pan_kmod_dev_props *dprops,
+                     const struct pan_image_props *iprops,
+                     const struct pan_image_usage *iusage)
+{
+   const unsigned arch = pan_arch(dprops->gpu_id);
+   struct pan_image image = {
+      .props = *iprops,
+      .mod_handler = pan_mod_get_handler(arch, iprops->modifier),
+   };
+
+   if (!image.mod_handler)
+      return PAN_MOD_NOT_SUPPORTED;
+
+   enum pan_mod_support ret =
+      image.mod_handler->test_props(dprops, &image.props, iusage);
+   if (ret == PAN_MOD_NOT_SUPPORTED)
+      return ret;
+
+   /* Now make sure the layout can be properly initialized on all planes. */
+   uint32_t plane_count = util_format_get_num_planes(image.props.format);
+   for (uint32_t p = 0; p < plane_count; p++) {
+      struct pan_image_plane plane;
+
+      memset(&plane, 0, sizeof(plane));
+      image.planes[p] = &plane;
+
+      if (!pan_image_layout_init(arch, &image, p, NULL))
+         return PAN_MOD_NOT_SUPPORTED;
+   }
+
+   return ret;
+}
+
+static inline bool
+pan_image_test_modifier_with_format(const struct pan_kmod_dev_props *dprops,
+                                    uint64_t modifier, enum pipe_format format)
+{
+   /* To check if a <modifier,format> pair is supported, we define the smallest
+    * possible 2D image (or 3D image if this is a 3D compressed format). */
+   const struct pan_image_props iprops = {
+      .modifier = modifier,
+      .format = format,
+      .extent_px = {
+            .width = util_format_get_blockwidth(format),
+            .height = util_format_get_blockheight(format),
+            .depth = util_format_get_blockdepth(format),
+      },
+      .nr_samples = 1,
+      .dim = util_format_get_blockdepth(format) > 1 ? MALI_TEXTURE_DIMENSION_3D
+                                                    : MALI_TEXTURE_DIMENSION_2D,
+      .nr_slices = 1,
+      .array_size = 1,
+   };
+
+   return pan_image_test_props(dprops, &iprops, NULL) != PAN_MOD_NOT_SUPPORTED;
+}
+
+#ifdef __cplusplus
+} /* extern C */
+#endif
+
+#endif

src/panfrost/lib/pan_jc.h

@@ -0,0 +1,248 @@
+/*
+ * Copyright (C) 2019-2020 Collabora Ltd.
+ * Copyright (C) 2019 Alyssa Rosenzweig
+ * Copyright (C) 2014-2017 Broadcom
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef __PAN_JC_H__
+#define __PAN_JC_H__
+
+#include "genxml/gen_macros.h"
+#include "pan_pool.h"
+
+/* Job chain */
+struct pan_jc {
+   /* The first job in the chain */
+   uint64_t first_job;
+
+   /* The number of jobs in the chain, essentially */
+   unsigned job_index;
+
+   /* A CPU-side pointer to the previous job for next_job linking */
+   struct mali_job_header_packed *prev_job;
+
+   /* A CPU-side pointer to the first tiler job for dep updates when
+    * injecting a reload tiler job.
+    */
+   struct mali_job_header_packed *first_tiler;
+   uint32_t first_tiler_dep1;
+
+   /* The dependency for tiler jobs (i.e. the index of the last emitted
+    * tiler job, or zero if none have been emitted) */
+   unsigned tiler_dep;
+
+   /* The job index of the WRITE_VALUE job (before it has been created) */
+   unsigned write_value_index;
+};
+
+#if defined(PAN_ARCH) && PAN_ARCH <= 9
+/*
+ * There are various types of Mali jobs:
+ *
+ *  - WRITE_VALUE: generic write primitive, used to zero tiler field
+ *  - VERTEX: runs a vertex shader
+ *  - TILER: runs tiling and sets up a fragment shader
+ *  - FRAGMENT: runs fragment shaders and writes out
+ *  - COMPUTE: runs a compute shader
+ *  - FUSED: vertex+tiler fused together, implicit intradependency (Bifrost)
+ *  - GEOMETRY: runs a geometry shader (unimplemented)
+ *  - CACHE_FLUSH: unseen in the wild, theoretically cache flush
+ *
+ * In between a full batch and a single Mali job is the "job chain", a series
+ * of Mali jobs together forming a linked list. Within the job chain, each Mali
+ * job can set (up to) two dependencies on other earlier jobs in the chain.
+ * This dependency graph forms a scoreboard. The general idea of a scoreboard
+ * applies: when there is a data dependency of job B on job A, job B sets one
+ * of its dependency indices to job A, ensuring that job B won't start until
+ * job A finishes.
+ *
+ * More specifically, here are a set of rules:
+ *
+ * - A write value job must appear if and only if there is at least one tiler
+ *   job, and tiler jobs must depend on it.
+ *
+ * - Vertex jobs and tiler jobs are independent.
+ *
+ * - A tiler job must have a dependency on its data source. If it's getting
+ *   data from a vertex job, it depends on the vertex job. If it's getting data
+ *   from software, this is null.
+ *
+ * - Tiler jobs must depend on the write value job (chained or otherwise).
+ *
+ * - Tiler jobs must be strictly ordered. So each tiler job must depend on the
+ *   previous job in the chain.
+ *
+ * - Jobs linking via next_job has no bearing on order of execution, rather it
+ *   just establishes the linked list of jobs, EXCEPT:
+ *
+ * - A job's dependencies must appear earlier in the linked list (job chain).
+ *
+ * Justification for each rule:
+ *
+ * - Write value jobs are used to write a zero into a magic tiling field, which
+ *   enables tiling to work. If tiling occurs, they are needed; if it does not,
+ *   we cannot emit them since then tiling partially occurs and it's bad.
+ *
+ * - The hardware has no notion of a "vertex/tiler job" (at least not our
+ *   hardware -- other revs have fused jobs, but --- crap, this just got even
+ *   more complicated). They are independent units that take in data, process
+ *   it, and spit out data.
+ *
+ * - Any job must depend on its data source, in fact, or risk a
+ *   read-before-write hazard. Tiler jobs get their data from vertex jobs, ergo
+ *   tiler jobs depend on the corresponding vertex job (if it's there).
+ *
+ * - The tiler is not thread-safe; this dependency prevents race conditions
+ *   between two different jobs trying to write to the tiler outputs at the
+ *   same time.
+ *
+ * - Internally, jobs are scoreboarded; the next job fields just form a linked
+ *   list to allow the jobs to be read in; the execution order is from
+ *   resolving the dependency fields instead.
+ *
+ * - The hardware cannot set a dependency on a job it doesn't know about yet,
+ *   and dependencies are processed in-order of the next job fields.
+ *
+ */
+
+/**
+ * Does this job use the tiler? Beyond tiler jobs, index-driven vertex shading
+ * jobs also do.
+ */
+static inline bool
+job_uses_tiling(enum mali_job_type type)
+{
+#if PAN_ARCH >= 9
+   if (type == MALI_JOB_TYPE_MALLOC_VERTEX)
+      return true;
+#endif
+
+#if PAN_ARCH >= 6
+   if (type == MALI_JOB_TYPE_INDEXED_VERTEX)
+      return true;
+#endif
+
+   return (type == MALI_JOB_TYPE_TILER);
+}
+
+/* Generates, uploads, and queues a a new job. All fields are written in order
+ * except for next_job accounting (TODO: Should we be clever and defer the
+ * upload of the header here until next job to keep the access pattern totally
+ * linear? Or is that just a micro op at this point?). Returns the generated
+ * index for dep management.
+ *
+ * Inject is used to inject a job at the front, for wallpapering. If you are
+ * not wallpapering and set this, dragons will eat you. */
+
+static inline unsigned
+pan_jc_add_job(struct pan_jc *jc, enum mali_job_type type, bool barrier,
+               bool suppress_prefetch, unsigned local_dep, unsigned global_dep,
+               const struct pan_ptr *job, bool inject)
+{
+   if (job_uses_tiling(type)) {
+      /* Tiler jobs must be chained, and on Midgard, the first tiler
+       * job must depend on the write value job, whose index we
+       * reserve now */
+
+      if (PAN_ARCH <= 5 && !jc->write_value_index)
+         jc->write_value_index = ++jc->job_index;
+
+      if (jc->tiler_dep && !inject)
+         global_dep = jc->tiler_dep;
+      else if (PAN_ARCH <= 5)
+         global_dep = jc->write_value_index;
+   }
+
+   /* Assign the index */
+   unsigned index = ++jc->job_index;
+
+   pan_cast_and_pack(job->cpu, JOB_HEADER, header) {
+      header.type = type;
+      header.barrier = barrier;
+      header.suppress_prefetch = suppress_prefetch;
+      header.index = index;
+      header.dependency_1 = local_dep;
+      header.dependency_2 = global_dep;
+
+      if (inject)
+         header.next = jc->first_job;
+   }
+
+   if (inject) {
+      assert(type == MALI_JOB_TYPE_TILER && "only for blit shaders");
+
+      if (jc->first_tiler) {
+         /* Manual update of the dep2 field. This is bad,
+          * don't copy this pattern.
+          */
+         jc->first_tiler->opaque[5] =
+            jc->first_tiler_dep1 | (index << 16);
+      }
+
+      jc->first_tiler = (void *)job->cpu;
+      jc->first_tiler_dep1 = local_dep;
+      jc->first_job = job->gpu;
+      return index;
+   }
+
+   /* Form a chain */
+   if (job_uses_tiling(type)) {
+      if (!jc->first_tiler) {
+         jc->first_tiler = (void *)job->cpu;
+         jc->first_tiler_dep1 = local_dep;
+      }
+      jc->tiler_dep = index;
+   }
+
+   if (jc->prev_job) {
+      /* Manual update of the next pointer. This is bad, don't copy
+       * this pattern.
+       * TODO: Find a way to defer last job header emission until we
+       * have a new job to queue or the batch is ready for execution.
+       */
+      jc->prev_job->opaque[6] = job->gpu;
+      jc->prev_job->opaque[7] = job->gpu >> 32;
+   } else {
+      jc->first_job = job->gpu;
+   }
+
+   jc->prev_job = (struct mali_job_header_packed *)job->cpu;
+   return index;
+}
+
+/* Generates a write value job, used to initialize the tiler structures. Note
+ * this is called right before frame submission. */
+
+static inline struct pan_ptr
+pan_jc_initialize_tiler(struct pan_pool *pool, struct pan_jc *jc,
+                        uint64_t polygon_list)
+{
+   struct pan_ptr transfer = {0};
+
+   /* Check if we even need tiling */
+   if (PAN_ARCH >= 6 || !jc->first_tiler)
+      return transfer;
+
+   /* Okay, we do. Let's generate it. We'll need the job's polygon list
+    * regardless of size. */
+
+   transfer = pan_pool_alloc_desc(pool, WRITE_VALUE_JOB);
+
+   pan_section_pack(transfer.cpu, WRITE_VALUE_JOB, HEADER, header) {
+      header.type = MALI_JOB_TYPE_WRITE_VALUE;
+      header.index = jc->write_value_index;
+      header.next = jc->first_job;
+   }
+
+   pan_section_pack(transfer.cpu, WRITE_VALUE_JOB, PAYLOAD, payload) {
+      payload.address = polygon_list;
+      payload.type = MALI_WRITE_VALUE_TYPE_ZERO;
+   }
+
+   jc->first_job = transfer.gpu;
+   return transfer;
+}
+#endif /* PAN_ARCH */
+
+#endif

src/panfrost/lib/pan_layout.c

@@ -0,0 +1,161 @@
+/*
+ * Copyright (C) 2019-2022 Collabora, Ltd.
+ * Copyright (C) 2018-2019 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "pan_layout.h"
+#include "util/log.h"
+#include "util/macros.h"
+#include "util/u_math.h"
+#include "pan_afbc.h"
+#include "pan_afrc.h"
+#include "pan_image.h"
+#include "pan_mod.h"
+#include "pan_props.h"
+
+/*
+ * Computes sizes for checksumming, which is 8 bytes per 16x16 tile.
+ * Checksumming is believed to be a CRC variant (CRC64 based on the size?).
+ * This feature is also known as "transaction elimination".
+ * CRC values are prefetched by 32x32 (64x64 on v12+) regions so size needs to
+ * be aligned.
+ */
+
+#define CHECKSUM_TILE_WIDTH     16
+#define CHECKSUM_TILE_HEIGHT    16
+#define CHECKSUM_BYTES_PER_TILE 8
+
+static void
+init_slice_crc_info(unsigned arch, struct pan_image_slice_layout *slice,
+                    unsigned width_px, unsigned height_px, uint64_t offset_B)
+{
+   unsigned checksum_region_size_px = pan_meta_tile_size(arch);
+   unsigned checksum_x_tile_per_region =
+      (checksum_region_size_px / CHECKSUM_TILE_WIDTH);
+   unsigned checksum_y_tile_per_region =
+      (checksum_region_size_px / CHECKSUM_TILE_HEIGHT);
+
+   unsigned tile_count_x = checksum_x_tile_per_region *
+                           DIV_ROUND_UP(width_px, checksum_region_size_px);
+   unsigned tile_count_y = checksum_y_tile_per_region *
+                           DIV_ROUND_UP(height_px, checksum_region_size_px);
+
+   slice->crc.offset_B = offset_B;
+   slice->crc.stride_B = tile_count_x * CHECKSUM_BYTES_PER_TILE;
+   slice->crc.size_B = slice->crc.stride_B * tile_count_y;
+}
+
+static struct pan_image_extent
+get_mip_level_extent(const struct pan_image_props *props, unsigned plane_idx,
+                     unsigned mip_level)
+{
+   return (struct pan_image_extent){
+      .width = u_minify(util_format_get_plane_width(props->format, plane_idx,
+                                                    props->extent_px.width),
+                        mip_level),
+      .height = u_minify(util_format_get_plane_height(props->format, plane_idx,
+                                                      props->extent_px.height),
+                         mip_level),
+      .depth = u_minify(props->extent_px.depth, mip_level),
+   };
+}
+
+bool
+pan_image_layout_init(
+   unsigned arch, struct pan_image *image, unsigned plane_idx,
+   const struct pan_image_layout_constraints *explicit_layout_constraints)
+{
+   assert(image->mod_handler);
+   assert(plane_idx < ARRAY_SIZE(image->planes) &&
+          plane_idx < util_format_get_num_planes(image->props.format));
+   assert(image->planes[plane_idx]);
+
+   const struct pan_mod_handler *mod_handler = image->mod_handler;
+   const struct pan_image_props *props = &image->props;
+   struct pan_image_layout *layout = &image->planes[plane_idx]->layout;
+
+   /* Use explicit layout only when wsi_row_pitch_B is non-zero */
+   struct pan_image_layout_constraints layout_constraints = {0};
+   if (explicit_layout_constraints)
+      layout_constraints = *explicit_layout_constraints;
+
+   const bool use_explicit_layout = layout_constraints.wsi_row_pitch_B != 0;
+
+   /* Explicit stride only work with non-mipmap, non-array, single-sample
+    * 2D image without CRC.
+    */
+   if (use_explicit_layout &&
+       (props->extent_px.depth > 1 || props->nr_samples > 1 ||
+        props->array_size > 1 || props->dim != MALI_TEXTURE_DIMENSION_2D ||
+        props->nr_slices > 1 || props->crc))
+      return false;
+
+   if (plane_idx >= util_format_get_num_planes(props->format))
+      return false;
+
+   /* Init plane layout data. */
+   if (mod_handler->init_plane_layout &&
+       !mod_handler->init_plane_layout(props, plane_idx, layout))
+      return false;
+
+   /* MSAA is implemented as a 3D texture with z corresponding to the
+    * sample #, horrifyingly enough */
+
+   assert(props->extent_px.depth == 1 || props->nr_samples == 1);
+
+   /* Make sure the extent/sample_count is not zero. */
+   assert(props->extent_px.width && props->extent_px.height &&
+          props->extent_px.depth && props->nr_samples);
+
+   struct pan_image_extent mip_extent_px = {
+      .width = util_format_get_plane_width(props->format, plane_idx,
+                                           props->extent_px.width),
+      .height = util_format_get_plane_height(props->format, plane_idx,
+                                             props->extent_px.height),
+      .depth = props->extent_px.depth,
+   };
+
+   for (unsigned l = 0; l < props->nr_slices; ++l) {
+      struct pan_image_slice_layout *slayout = &layout->slices[l];
+
+      if (!mod_handler->init_slice_layout(props, plane_idx, mip_extent_px,
+                                          &layout_constraints, slayout))
+         return false;
+
+      layout_constraints.offset_B += slayout->size_B;
+
+      /* Add a checksum region if necessary */
+      if (props->crc) {
+         init_slice_crc_info(arch, slayout, mip_extent_px.width,
+                             mip_extent_px.height, layout_constraints.offset_B);
+         layout_constraints.offset_B += slayout->crc.size_B;
+         slayout->size_B += slayout->crc.size_B;
+      }
+
+      mip_extent_px.width = u_minify(mip_extent_px.width, 1);
+      mip_extent_px.height = u_minify(mip_extent_px.height, 1);
+      mip_extent_px.depth = u_minify(mip_extent_px.depth, 1);
+   }
+
+   /* Arrays and cubemaps have the entire miptree duplicated */
+   layout->array_stride_B =
+      ALIGN_POT(layout_constraints.offset_B - layout->slices[0].offset_B, 64);
+
+   if (use_explicit_layout) {
+      layout->data_size_B =
+         layout_constraints.offset_B - explicit_layout_constraints->offset_B;
+   } else {
+      /* Native images start from offset 0, and the planar plane offset has
+       * been at least 4K page aligned below. So the base level slice offset
+       * should always be the same with the plane offset.
+       */
+      assert(!explicit_layout_constraints ||
+             explicit_layout_constraints->offset_B ==
+                layout->slices[0].offset_B);
+      layout->data_size_B = ALIGN_POT(
+         (uint64_t)layout->array_stride_B * (uint64_t)props->array_size, 4096);
+   }
+
+   return true;
+}

src/panfrost/lib/pan_layout.h

@@ -0,0 +1,267 @@
+/*
+ * Copyright (C) 2008 VMware, Inc.
+ * Copyright (C) 2014 Broadcom
+ * Copyright (C) 2018-2019 Alyssa Rosenzweig
+ * Copyright (C) 2019-2020 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef __PAN_LAYOUT_H
+#define __PAN_LAYOUT_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdbool.h>
+
+#include "genxml/gen_macros.h"
+
+#include "util/format/u_format.h"
+
+#define MAX_MIP_LEVELS   17
+#define MAX_IMAGE_PLANES 3
+
+struct pan_mod_handler;
+
+struct pan_afbc_image_slice_layout {
+   struct {
+      /* Number of bytes between two rows of AFBC headers. */
+      uint32_t row_stride_B;
+
+      /* For 3D textures, this is the size in bytes of AFBC headers covering
+       * a single Z slice. For 2D this is the total header size. This size is
+       * the utile header size, it doesn't count the padding needed to meet the
+       * body alignment constraints. Pass this to pan_afbc_body_offset() to get
+       * the body offset.
+       */
+      uint32_t surface_size_B;
+   } header;
+
+   /* For 3D textures, this is the stride in bytes between AFBC headers of two
+    * consecutive Z slices. For 2D, this is the total size of the 2D level.
+    */
+   uint64_t surface_stride_B;
+};
+
+struct pan_tiled_or_linear_image_slice_layout {
+   /* Number of bytes between two rows of tiles/lines. */
+   uint32_t row_stride_B;
+
+   /* For 3D textures, this is the stride in bytes between two
+    * consecutive Z slices. For 2DMS textures, this is the stride in bytes
+    * between two sample planes.
+    */
+   uint64_t surface_stride_B;
+};
+
+struct pan_image_slice_layout {
+   /* Offset in bytes relative to the base bo bound.
+    *
+    * Unlike gallium, vulkan has to report explicit image subres layout which
+    * disallows to hide the planar plane offset into the bo mapping. So we let
+    * the slice offsets to include the plane offset of the native multi-planar
+    * images to be consistent with the imported ones via explicit layout info.
+    * Doing so allows us to use a single code path to correctly:
+    * - report image subres layout and memory requirement
+    * - bind image memory
+    */
+   uint64_t offset_B;
+
+   /* Size of the MIP level in bytes. */
+   uint64_t size_B;
+
+   /* Some properties have a different meaning depending on the modifier.
+    * Those are placed in different structs under a union. */
+   union {
+      /* Used only for AFBC images. */
+      struct pan_afbc_image_slice_layout afbc;
+      /* Used for linear, u-tiled and AFRC images. */
+      struct pan_tiled_or_linear_image_slice_layout tiled_or_linear;
+   };
+
+   /* If checksumming is enabled following the slice, what
+    * is its offset/stride? */
+   struct {
+      uint64_t offset_B;
+      uint32_t stride_B;
+      uint32_t size_B;
+   } crc;
+};
+
+struct pan_image_extent {
+   unsigned width;
+   unsigned height;
+   unsigned depth;
+};
+
+struct pan_image_props {
+   uint64_t modifier;
+   enum pipe_format format;
+   struct pan_image_extent extent_px;
+   unsigned nr_samples;
+   enum mali_texture_dimension dim;
+   unsigned nr_slices;
+   unsigned array_size;
+   bool crc;
+};
+
+struct pan_image_layout {
+   struct pan_image_slice_layout slices[MAX_MIP_LEVELS];
+
+   union {
+      struct {
+         int mode;
+      } afbc;
+   };
+
+   /* Image plane data size in bytes */
+   uint64_t data_size_B;
+   uint64_t array_stride_B;
+};
+
+struct pan_image_layout_constraints {
+   /*
+    * Plane offset in bytes
+    * - For native images, it's the planar plane offset.
+    * - For imported images, it's the user specified explicit offset.
+    *
+    * To be noted, this offset might be adjusted to choose an optimal alignment,
+    * unless the layout constraints are explicit (wsi_row_patch_B != 0).
+    */
+   uint64_t offset_B;
+
+   /* Row pitch in bytes. Non-zero if layout is explicit. */
+   uint32_t wsi_row_pitch_B;
+
+   /* When true, AFBC/AFRC imports are stricter than they were when those
+    * modifiers where introduced. */
+   bool strict;
+};
+
+/*
+ * Represents the block size of a single plane. For AFBC, this represents the
+ * superblock size. For u-interleaving, this represents the tile size.
+ */
+struct pan_image_block_size {
+   /** Width of block */
+   unsigned width;
+
+   /** Height of blocks */
+   unsigned height;
+};
+
+/*
+ * Determine the required alignment for the slice offset of an image. For
+ * now, this is always aligned on 64-byte boundaries. */
+static inline uint32_t
+pan_image_slice_align(uint64_t modifier)
+{
+   return 64;
+}
+
+static inline uint32_t
+pan_linear_or_tiled_row_align_req(unsigned arch, enum pipe_format format,
+                                  unsigned plane_idx)
+{
+   if (arch < 7) {
+      unsigned nplanes = util_format_get_num_planes(format);
+
+      /* If this is a planar format, align on the plane blocksize. */
+      if (nplanes > 1) {
+         enum pipe_format plane_format =
+            util_format_get_plane_format(format, plane_idx);
+
+         return util_next_power_of_two(util_format_get_blocksize(plane_format));
+      }
+
+      /* Align on blocksize if the format is compressed. */
+      if (util_format_is_compressed(format))
+         return util_next_power_of_two(util_format_get_blocksize(format));
+
+      const struct util_format_description *fdesc =
+         util_format_description(format);
+      unsigned comp_sz_bits = 0;
+      for (unsigned i = 0; i < ARRAY_SIZE(fdesc->channel); i++) {
+         if (!fdesc->channel[0].size)
+            continue;
+
+         /* Align on a pixel if any component is not 8-bit aligned or not a
+          * power of two. */
+         if (fdesc->channel[0].size % 8 != 0 ||
+             !util_is_power_of_two_nonzero(fdesc->channel[0].size))
+            return util_next_power_of_two(util_format_get_blocksize(format));
+
+         /* Align on a pixel if not all components have the same size. */
+         if (comp_sz_bits != 0 && comp_sz_bits != fdesc->channel[0].size)
+            return util_next_power_of_two(util_format_get_blocksize(format));
+
+         comp_sz_bits = fdesc->channel[0].size;
+      }
+
+      /* If all components are the same size, 8-bit aligned and a power of two,
+       * align on a component. */
+      return comp_sz_bits / 8;
+   }
+
+   switch (format) {
+   /* For v7+, NV12/NV21/I420 have a looser alignment requirement of 16 bytes */
+   case PIPE_FORMAT_R8G8B8_420_UNORM_PACKED:
+   case PIPE_FORMAT_R8_G8B8_420_UNORM:
+   case PIPE_FORMAT_G8_B8R8_420_UNORM:
+   case PIPE_FORMAT_R8_G8_B8_420_UNORM:
+   case PIPE_FORMAT_R8_B8_G8_420_UNORM:
+   case PIPE_FORMAT_R8_G8B8_422_UNORM:
+   case PIPE_FORMAT_R8_B8G8_422_UNORM:
+      return 16;
+   /* the 10 bit formats have even looser alignment */
+   case PIPE_FORMAT_R10G10B10_420_UNORM_PACKED:
+   case PIPE_FORMAT_R10_G10B10_420_UNORM:
+   case PIPE_FORMAT_R10_G10B10_422_UNORM:
+      return 1;
+   default:
+      return 64;
+   }
+}
+
+/*
+ * Given a format, determine the tile size used for u-interleaving. For formats
+ * that are already block compressed, this is 4x4. For all other formats, this
+ * is 16x16, hence the modifier name.
+ */
+static inline struct pan_image_block_size
+pan_u_interleaved_tile_size_el(enum pipe_format format)
+{
+   if (util_format_is_compressed(format)) {
+      return (struct pan_image_block_size){4, 4};
+   } else {
+      assert(16 % util_format_get_blockwidth(format) == 0);
+      assert(16 % util_format_get_blockheight(format) == 0);
+      return (struct pan_image_block_size){
+         .width = 16 / util_format_get_blockwidth(format),
+         .height = 16 / util_format_get_blockheight(format),
+      };
+   }
+}
+
+/* Given a format, determine the tile size used for interleaved 64k. */
+static inline struct pan_image_block_size
+pan_interleaved_64k_tile_size_el(enum pipe_format format)
+{
+   switch (util_format_get_blocksize(format)) {
+   case 1: return (struct pan_image_block_size){256, 256};
+   case 2: return (struct pan_image_block_size){256, 128};
+   case 4: return (struct pan_image_block_size){128, 128};
+   case 8: return (struct pan_image_block_size){128, 64};
+   case 16: return (struct pan_image_block_size){64, 64};
+   default:
+      UNREACHABLE("unsupported format");
+      return (struct pan_image_block_size){0, 0};
+   }
+}
+
+#ifdef __cplusplus
+} /* extern C */
+#endif
+
+#endif

src/panfrost/lib/pan_mod.c

@@ -0,0 +1,826 @@
+/*
+ * Copyright (C) 2019-2025 Collabora, Ltd.
+ * Copyright (C) 2018-2019 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "pan_mod.h"
+#include "pan_afbc.h"
+#include "pan_afrc.h"
+#include "pan_desc.h"
+#include "pan_format.h"
+#include "pan_image.h"
+#include "pan_layout.h"
+#include "pan_props.h"
+#include "pan_texture.h"
+
+#include "util/format/u_format.h"
+
+static bool
+pan_mod_afbc_match(uint64_t mod)
+{
+   return drm_is_afbc(mod);
+}
+
+static uint32_t
+pan_mod_afbc_get_wsi_row_pitch(const struct pan_image *image,
+                               unsigned plane_idx, unsigned mip_level)
+{
+   const struct pan_image_props *props = &image->props;
+   const struct pan_image_layout *layout = &image->planes[plane_idx]->layout;
+   const unsigned header_row_stride_B =
+      layout->slices[mip_level].afbc.header.row_stride_B;
+   const struct pan_image_block_size tile_extent_el =
+      pan_afbc_superblock_size_el(props->format, props->modifier);
+   const unsigned tile_payload_size_B =
+      tile_extent_el.width * tile_extent_el.height *
+      pan_format_get_plane_blocksize(props->format, plane_idx);
+   const unsigned tile_row_payload_size_B =
+      pan_afbc_stride_blocks(props->format,
+                             props->modifier, header_row_stride_B) *
+      tile_payload_size_B;
+
+   return tile_row_payload_size_B / pan_afbc_superblock_height(props->modifier);
+}
+
+static bool
+pan_mod_afbc_init_plane_layout(const struct pan_image_props *props,
+                               unsigned plane_idx,
+                               struct pan_image_layout *plane_layout)
+{
+   enum pan_afbc_mode mode =
+      pan_afbc_format(PAN_ARCH, props->format, plane_idx);
+   if (mode == PAN_AFBC_MODE_INVALID)
+      return false;
+
+   plane_layout->afbc.mode = mode;
+   return true;
+}
+
+static bool
+pan_mod_afbc_init_slice_layout(
+   const struct pan_image_props *props, unsigned plane_idx,
+   struct pan_image_extent mip_extent_px,
+   const struct pan_image_layout_constraints *layout_constraints,
+   struct pan_image_slice_layout *slayout)
+{
+   /* Use explicit layout only when wsi_row_pitch_B is non-zero */
+   const bool use_explicit_layout =
+      layout_constraints && layout_constraints->wsi_row_pitch_B;
+   struct pan_image_block_size afbc_tile_extent_px =
+      pan_afbc_superblock_size(props->modifier);
+   unsigned offset_align_mask =
+      pan_afbc_header_align(PAN_ARCH, props->modifier) - 1;
+   unsigned row_align_mask = pan_afbc_header_row_stride_align(
+                                PAN_ARCH, props->format, props->modifier) -
+                             1;
+   struct pan_image_block_size afbc_tile_extent_el =
+      pan_afbc_superblock_size_el(props->format, props->modifier);
+   unsigned afbc_tile_payload_size_B =
+      afbc_tile_extent_el.width * afbc_tile_extent_el.height *
+      pan_format_get_plane_blocksize(props->format, plane_idx);
+
+   struct pan_image_block_size align_px =
+      pan_afbc_renderblock_size(props->modifier);
+
+   /* If superblock tiling is used, align on a superblock tile. */
+   if (props->modifier & AFBC_FORMAT_MOD_TILED) {
+      align_px.width =
+         ALIGN_POT(align_px.width, afbc_tile_extent_px.width *
+                   pan_afbc_tile_size(props->format, props->modifier));
+      align_px.height =
+         ALIGN_POT(align_px.height, afbc_tile_extent_px.height *
+                   pan_afbc_tile_size(props->format, props->modifier));
+   }
+
+   struct pan_image_extent aligned_extent_px = {
+      .width = ALIGN_POT(mip_extent_px.width, align_px.width),
+      .height = ALIGN_POT(mip_extent_px.height, align_px.height),
+      .depth = mip_extent_px.depth,
+   };
+
+   if (use_explicit_layout) {
+      unsigned afbc_tile_payload_row_stride_B =
+         layout_constraints->wsi_row_pitch_B *
+         pan_afbc_superblock_height(props->modifier);
+
+      /* For quite some time, we've been accepting WSI row pitch that
+       * didn't match exactly the image size and have been assuming tightly
+       * packed tile rows instead of using the explicit stride in that case.
+       * This is something we can't change without risking breaking existing
+       * users, so we enforce this explicit tile alignment only if we were
+       * asked to. */
+      if (layout_constraints->strict &&
+          (afbc_tile_payload_row_stride_B % afbc_tile_payload_size_B)) {
+         mesa_loge("WSI pitch is not aligned on an AFBC tile");
+         return false;
+      }
+
+      unsigned width_from_wsi_row_stride =
+         (afbc_tile_payload_row_stride_B / afbc_tile_payload_size_B) *
+         pan_afbc_superblock_width(props->modifier);
+
+      if (width_from_wsi_row_stride < mip_extent_px.width) {
+         mesa_loge("WSI pitch too small");
+         return false;
+      }
+
+      slayout->afbc.header.row_stride_B =
+         pan_afbc_row_stride(props->format, props->modifier, width_from_wsi_row_stride);
+      if (slayout->afbc.header.row_stride_B & row_align_mask) {
+         mesa_loge("WSI pitch not properly aligned");
+         return false;
+      }
+
+      slayout->offset_B = layout_constraints->offset_B;
+      if (slayout->offset_B & offset_align_mask) {
+         mesa_loge("WSI offset not properly aligned");
+         return false;
+      }
+
+      /* If this is not a strict import, ignore the WSI row pitch and use
+       * the resource width to get the size. */
+      if (!layout_constraints->strict) {
+         slayout->afbc.header.row_stride_B = ALIGN_POT(
+            pan_afbc_row_stride(props->format, props->modifier, aligned_extent_px.width),
+            row_align_mask + 1);
+      }
+   } else {
+      slayout->offset_B =
+         ALIGN_POT(layout_constraints ? layout_constraints->offset_B : 0,
+                   offset_align_mask + 1);
+      slayout->afbc.header.row_stride_B = ALIGN_POT(
+         pan_afbc_row_stride(props->format, props->modifier, aligned_extent_px.width),
+         row_align_mask + 1);
+   }
+
+   const unsigned row_stride_sb = pan_afbc_stride_blocks(props->format,
+                                                         props->modifier,
+                                                         slayout->afbc.header.row_stride_B);
+   const unsigned surface_stride_sb =
+      row_stride_sb * (aligned_extent_px.height / afbc_tile_extent_px.height);
+
+   uint64_t hdr_surf_size_B =
+      (uint64_t)surface_stride_sb * AFBC_HEADER_BYTES_PER_TILE;
+   uint64_t body_offset_B =
+      pan_afbc_body_offset(PAN_ARCH, props->modifier, hdr_surf_size_B);
+   uint64_t surf_stride_B =
+      body_offset_B + ((uint64_t)surface_stride_sb * afbc_tile_payload_size_B);
+
+   slayout->afbc.header.surface_size_B = hdr_surf_size_B;
+   slayout->afbc.surface_stride_B = surf_stride_B;
+   slayout->size_B = surf_stride_B * mip_extent_px.depth;
+
+   if (hdr_surf_size_B > UINT32_MAX)
+      return false;
+
+   return true;
+}
+
+static enum pan_mod_support
+pan_mod_afbc_test_props(const struct pan_kmod_dev_props *dprops,
+                        const struct pan_image_props *iprops,
+                        const struct pan_image_usage *iusage)
+{
+   /* AFBC not supported. */
+   if (!pan_query_afbc(dprops))
+      return PAN_MOD_NOT_SUPPORTED;
+
+   /* Not all architectures support tiled mode */
+   if ((iprops->modifier & AFBC_FORMAT_MOD_TILED) && !pan_afbc_can_tile(PAN_ARCH))
+      return PAN_MOD_NOT_SUPPORTED;
+
+   /* ... or split mode */
+   if ((iprops->modifier & AFBC_FORMAT_MOD_SPLIT) && !pan_afbc_can_split(PAN_ARCH))
+      return PAN_MOD_NOT_SUPPORTED;
+
+   /* Check compatibility between formats and parameters */
+   unsigned plane_count = util_format_get_num_planes(iprops->format);
+   for (unsigned p = 0; p < plane_count; p++) {
+      enum pan_afbc_mode mode = pan_afbc_format(PAN_ARCH, iprops->format, p);
+      if (mode == PAN_AFBC_MODE_INVALID)
+         return PAN_MOD_NOT_SUPPORTED;
+      if (!pan_afbc_check_params(mode, iprops->modifier))
+         return PAN_MOD_NOT_SUPPORTED;
+   }
+
+   /* AFBC can't do multisampling. */
+   if (iprops->nr_samples > 1)
+      return PAN_MOD_NOT_SUPPORTED;
+
+   /* AFBC(2D) or AFBC(3D) on v7+ only. */
+   if ((iprops->dim == MALI_TEXTURE_DIMENSION_3D && PAN_ARCH < 7) ||
+       iprops->dim != MALI_TEXTURE_DIMENSION_2D)
+      return PAN_MOD_NOT_SUPPORTED;
+
+   /* No image store. */
+   if (iusage && iusage->bind & PAN_BIND_STORAGE_IMAGE)
+      return PAN_MOD_NOT_SUPPORTED;
+
+   /* ZS buffer descriptors can't pass split/wide/YTR modifiers. */
+   if (iusage && (iusage->bind & PAN_BIND_DEPTH_STENCIL) &&
+       (pan_afbc_superblock_width(iprops->modifier) != 16 ||
+        (iprops->modifier & (AFBC_FORMAT_MOD_SPLIT | AFBC_FORMAT_MOD_YTR))))
+      return PAN_MOD_NOT_SUPPORTED;
+
+   /* We don't implement mapping individual tiles with AFBC. */
+   if (iusage && iusage->standard_sparse_mapping_granularity)
+      return PAN_MOD_NOT_SUPPORTED;
+
+   struct pan_image_block_size superblock_extent_px = pan_afbc_superblock_size(iprops->modifier);
+
+   if (iprops->modifier & AFBC_FORMAT_MOD_TILED) {
+      struct pan_image_block_size tile_extent_px = {
+         superblock_extent_px.width * pan_afbc_tile_size(iprops->format, iprops->modifier),
+         superblock_extent_px.height * pan_afbc_tile_size(iprops->format, iprops->modifier),
+      };
+
+      /* Tiled mode has some overhead we don't want to pay if the image size is
+       * too thin. */
+      if (iprops->extent_px.width < tile_extent_px.width / 2 ||
+          iprops->extent_px.height < tile_extent_px.height / 2)
+         return PAN_MOD_NOT_OPTIMAL;
+   } else {
+      /* For images thinner than a superblock, fall through to U-interleaved. */
+      if (iprops->extent_px.width < superblock_extent_px.width ||
+          iprops->extent_px.height < superblock_extent_px.height)
+         return PAN_MOD_NOT_OPTIMAL;
+   }
+
+   /* Only use 32x8 superblock for WSI images. */
+   if (iusage && !iusage->wsi &&
+       pan_afbc_superblock_width(iprops->modifier) != 16)
+      return PAN_MOD_NOT_OPTIMAL;
+
+   /* Prefer YTR when available. */
+   if (pan_afbc_can_ytr(iprops->format) &&
+       !(iprops->modifier & AFBC_FORMAT_MOD_YTR))
+      return PAN_MOD_NOT_OPTIMAL;
+
+   /* Packing/unpacking AFBC payload requires a COMPUTE job which we'd rather
+    * avoid.
+    */
+   if (iusage &&
+       (iusage->bind & (PAN_BIND_DEPTH_STENCIL | PAN_BIND_RENDER_TARGET)) &&
+       !(iprops->modifier & AFBC_FORMAT_MOD_SPARSE))
+      return PAN_MOD_NOT_OPTIMAL;
+
+   return PAN_MOD_OPTIMAL;
+}
+
+#define pan_mod_afbc_emit_tex_payload_entry                                    \
+   GENX(pan_tex_emit_afbc_payload_entry)
+#define pan_mod_afbc_emit_color_attachment GENX(pan_emit_afbc_color_attachment)
+#define pan_mod_afbc_emit_zs_attachment    GENX(pan_emit_afbc_zs_attachment)
+#define pan_mod_afbc_emit_s_attachment     GENX(pan_emit_afbc_s_attachment)
+
+#if PAN_ARCH >= 10
+static bool
+pan_mod_afrc_match(uint64_t mod)
+{
+   return drm_is_afrc(mod);
+}
+
+static enum pan_mod_support
+pan_mod_afrc_test_props(const struct pan_kmod_dev_props *dprops,
+                        const struct pan_image_props *iprops,
+                        const struct pan_image_usage *iusage)
+{
+   /* AFRC not supported. */
+   if (!pan_query_afrc(dprops))
+      return PAN_MOD_NOT_SUPPORTED;
+
+   /* Format not AFRC-able. */
+   if (!pan_afrc_supports_format(iprops->format))
+      return PAN_MOD_NOT_SUPPORTED;
+
+   /* AFRC does not support layered multisampling. */
+   if (iprops->nr_samples > 1)
+      return PAN_MOD_NOT_SUPPORTED;
+
+   /* No image store. */
+   if (iusage && iusage->bind & PAN_BIND_STORAGE_IMAGE)
+      return PAN_MOD_NOT_SUPPORTED;
+
+   /* We don't implement mapping individual tiles with AFRC. */
+   if (iusage && iusage->standard_sparse_mapping_granularity)
+      return PAN_MOD_NOT_SUPPORTED;
+
+   /* We can't write to an AFRC resource directly. */
+   if (iusage && iusage->host_copy)
+      return PAN_MOD_NOT_SUPPORTED;
+
+   /* Host updates require an extra blit which we would rather avoid. */
+   if (iusage && iusage->frequent_host_updates)
+      return PAN_MOD_NOT_OPTIMAL;
+
+   /* There's nothing preventing 1D AFRC, but it's pointless. */
+   if (iprops->dim == MALI_TEXTURE_DIMENSION_1D)
+      return PAN_MOD_NOT_OPTIMAL;
+
+   return PAN_MOD_OPTIMAL;
+}
+
+static uint32_t
+pan_mod_afrc_get_wsi_row_pitch(const struct pan_image *image,
+                               unsigned plane_idx, unsigned mip_level)
+{
+   const struct pan_image_props *props = &image->props;
+   const struct pan_image_layout *layout = &image->planes[plane_idx]->layout;
+   const struct pan_image_block_size tile_extent_px =
+      pan_afrc_tile_size(props->format, props->modifier);
+
+   return layout->slices[mip_level].tiled_or_linear.row_stride_B /
+          tile_extent_px.height;
+}
+
+#define pan_mod_afrc_init_plane_layout NULL
+
+static bool
+pan_mod_afrc_init_slice_layout(
+   const struct pan_image_props *props, unsigned plane_idx,
+   struct pan_image_extent mip_extent_px,
+   const struct pan_image_layout_constraints *layout_constraints,
+   struct pan_image_slice_layout *slayout)
+{
+   /* Use explicit layout only when wsi_row_pitch_B is non-zero */
+   const bool use_explicit_layout =
+      layout_constraints && layout_constraints->wsi_row_pitch_B;
+   const unsigned align_mask =
+      pan_afrc_buffer_alignment_from_modifier(props->modifier) - 1;
+   struct pan_image_block_size tile_extent_px =
+      pan_afrc_tile_size(props->format, props->modifier);
+   struct pan_image_extent aligned_extent_px = {
+      .width = ALIGN_POT(mip_extent_px.width, tile_extent_px.width),
+      .height = ALIGN_POT(mip_extent_px.height, tile_extent_px.height),
+      .depth = mip_extent_px.depth,
+   };
+
+   if (use_explicit_layout) {
+      slayout->tiled_or_linear.row_stride_B =
+         layout_constraints->wsi_row_pitch_B * tile_extent_px.height;
+      if (slayout->tiled_or_linear.row_stride_B & align_mask) {
+         mesa_loge("WSI pitch not properly aligned");
+         return false;
+      }
+
+      slayout->offset_B = layout_constraints->offset_B;
+      if (slayout->offset_B & align_mask) {
+         mesa_loge("WSI offset not properly aligned");
+         return false;
+      }
+
+      unsigned afrc_blk_size_B =
+         pan_afrc_block_size_from_modifier(props->modifier) *
+         AFRC_CLUMPS_PER_TILE;
+      unsigned width_from_wsi_row_stride =
+         (slayout->tiled_or_linear.row_stride_B / afrc_blk_size_B) *
+         tile_extent_px.width;
+
+      if (width_from_wsi_row_stride < mip_extent_px.width) {
+         mesa_loge("WSI pitch too small");
+         return false;
+      }
+
+      /* If this is not a strict import, ignore the WSI row pitch and use
+       * the resource width to get the size. */
+      if (!layout_constraints->strict) {
+         slayout->tiled_or_linear.row_stride_B = pan_afrc_row_stride(
+            props->format, props->modifier, mip_extent_px.width);
+      }
+   } else {
+      /* Align levels to cache-line as a performance improvement for
+       * linear/tiled and as a requirement for AFBC */
+      slayout->offset_B = ALIGN_POT(
+         layout_constraints ? layout_constraints->offset_B : 0, align_mask + 1);
+      slayout->tiled_or_linear.row_stride_B =
+         ALIGN_POT(pan_afrc_row_stride(props->format, props->modifier,
+                                       mip_extent_px.width),
+                   align_mask + 1);
+   }
+
+   uint64_t surf_stride_B =
+      (uint64_t)slayout->tiled_or_linear.row_stride_B *
+      (aligned_extent_px.height / tile_extent_px.height);
+
+   slayout->tiled_or_linear.surface_stride_B = surf_stride_B;
+   slayout->size_B =
+      surf_stride_B * aligned_extent_px.depth * props->nr_samples;
+
+   return true;
+}
+
+#define pan_mod_afrc_emit_tex_payload_entry                                    \
+   GENX(pan_tex_emit_afrc_payload_entry)
+#define pan_mod_afrc_emit_color_attachment GENX(pan_emit_afrc_color_attachment)
+#define pan_mod_afrc_emit_zs_attachment    NULL
+#define pan_mod_afrc_emit_s_attachment     NULL
+#endif
+
+static bool
+pan_mod_u_tiled_match(uint64_t mod)
+{
+   return mod == DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED;
+}
+
+static enum pan_mod_support
+pan_mod_u_tiled_test_props(const struct pan_kmod_dev_props *dprops,
+                           const struct pan_image_props *iprops,
+                           const struct pan_image_usage *iusage)
+{
+   assert(GENX(pan_format_from_pipe_format)(iprops->format)->hw);
+
+   /* YUV not supported. */
+   if (pan_format_is_yuv(iprops->format))
+      return PAN_MOD_NOT_SUPPORTED;
+
+   /* We don't implement mapping individual tiles in this layout. */
+   if (iusage && iusage->standard_sparse_mapping_granularity)
+      return PAN_MOD_NOT_SUPPORTED;
+
+   /* The purpose of tiling is improving locality in both X- and
+    * Y-directions. If there is only a single pixel in either direction,
+    * tiling does not make sense; using a linear layout instead is optimal
+    * for both memory usage and performance.
+    */
+   if (MIN2(iprops->extent_px.width, iprops->extent_px.height) < 2)
+      return PAN_MOD_NOT_OPTIMAL;
+
+   return PAN_MOD_OPTIMAL;
+}
+
+static uint32_t
+pan_mod_u_tiled_get_wsi_row_pitch(const struct pan_image *image,
+                                  unsigned plane_idx, unsigned mip_level)
+{
+   const struct pan_image_props *props = &image->props;
+   const struct pan_image_layout *layout = &image->planes[plane_idx]->layout;
+
+   return layout->slices[mip_level].tiled_or_linear.row_stride_B /
+          pan_u_interleaved_tile_size_el(props->format).height;
+}
+
+#define pan_mod_u_tiled_init_plane_layout NULL
+
+static bool
+pan_mod_u_tiled_init_slice_layout(
+   const struct pan_image_props *props, unsigned plane_idx,
+   struct pan_image_extent mip_extent_px,
+   const struct pan_image_layout_constraints *layout_constraints,
+   struct pan_image_slice_layout *slayout)
+{
+   /* Use explicit layout only when wsi_row_pitch_B is non-zero */
+   const bool use_explicit_layout =
+      layout_constraints && layout_constraints->wsi_row_pitch_B;
+   unsigned align_mask =
+      pan_linear_or_tiled_row_align_req(PAN_ARCH, props->format, plane_idx) - 1;
+   struct pan_image_block_size tile_extent_el =
+      pan_u_interleaved_tile_size_el(props->format);
+   struct pan_image_extent mip_extent_el;
+   unsigned tile_size_B;
+
+   if (util_format_is_compressed(props->format)) {
+      assert(util_format_get_num_planes(props->format) == 1);
+      mip_extent_el.width = DIV_ROUND_UP(
+         mip_extent_px.width, util_format_get_blockwidth(props->format));
+      mip_extent_el.height = DIV_ROUND_UP(
+         mip_extent_px.height, util_format_get_blockheight(props->format));
+      mip_extent_el.depth = DIV_ROUND_UP(
+         mip_extent_px.depth, util_format_get_blockdepth(props->format));
+      tile_size_B = tile_extent_el.width * tile_extent_el.height *
+                    pan_format_get_plane_blocksize(props->format, plane_idx);
+   } else {
+      /* Block-based YUV needs special care, because the U-tile extent
+       * is in pixels, not blocks in that case. */
+      assert(tile_extent_el.width % util_format_get_blockwidth(props->format) ==
+             0);
+      assert(tile_extent_el.height %
+                util_format_get_blockheight(props->format) ==
+             0);
+      mip_extent_el = mip_extent_px;
+      tile_size_B =
+         (tile_extent_el.width / util_format_get_blockwidth(props->format)) *
+         (tile_extent_el.height / util_format_get_blockheight(props->format)) *
+         pan_format_get_plane_blocksize(props->format, plane_idx);
+   }
+
+   if (use_explicit_layout) {
+      slayout->tiled_or_linear.row_stride_B =
+         layout_constraints->wsi_row_pitch_B * tile_extent_el.height;
+      if (slayout->tiled_or_linear.row_stride_B & align_mask) {
+         mesa_loge("WSI pitch not properly aligned");
+         return false;
+      }
+
+      const unsigned width_from_wsi_row_stride =
+         (slayout->tiled_or_linear.row_stride_B / tile_size_B) *
+         tile_extent_el.width;
+
+      if (width_from_wsi_row_stride < mip_extent_el.width) {
+         mesa_loge("WSI pitch too small");
+         return false;
+      }
+
+      slayout->offset_B = layout_constraints->offset_B;
+      if (slayout->offset_B & align_mask) {
+         mesa_loge("WSI offset not properly aligned");
+         return false;
+      }
+   } else {
+      /* When we can decide of the layout, we want things aligned on at least a
+       * cacheline for performance reasons. */
+      align_mask = MAX2(align_mask, 63);
+      slayout->offset_B = ALIGN_POT(
+         layout_constraints ? layout_constraints->offset_B : 0,
+         MAX2(align_mask + 1, pan_image_slice_align(props->modifier)));
+      slayout->tiled_or_linear.row_stride_B = ALIGN_POT(
+         tile_size_B * DIV_ROUND_UP(mip_extent_el.width, tile_extent_el.width),
+         align_mask + 1);
+   }
+
+   uint64_t surf_stride_B =
+      (uint64_t)slayout->tiled_or_linear.row_stride_B *
+      DIV_ROUND_UP(mip_extent_el.height, tile_extent_el.height);
+   surf_stride_B = ALIGN_POT(surf_stride_B, (uint64_t)align_mask + 1);
+
+   slayout->tiled_or_linear.surface_stride_B = surf_stride_B;
+   slayout->size_B = surf_stride_B * mip_extent_el.depth * props->nr_samples;
+
+   return true;
+}
+
+#define pan_mod_u_tiled_emit_tex_payload_entry                                 \
+   GENX(pan_tex_emit_u_tiled_payload_entry)
+#define pan_mod_u_tiled_emit_color_attachment                                  \
+   GENX(pan_emit_u_tiled_color_attachment)
+#define pan_mod_u_tiled_emit_zs_attachment GENX(pan_emit_u_tiled_zs_attachment)
+#define pan_mod_u_tiled_emit_s_attachment  GENX(pan_emit_u_tiled_s_attachment)
+
+#if PAN_ARCH >= 10
+#define pan_mod_interleaved_64k_init_plane_layout NULL
+
+static bool
+pan_mod_interleaved_64k_init_slice_layout(
+   const struct pan_image_props *props, unsigned plane_idx,
+   struct pan_image_extent mip_extent_px,
+   const struct pan_image_layout_constraints *layout_constraints,
+   struct pan_image_slice_layout *slayout)
+{
+   assert(!(layout_constraints && layout_constraints->wsi_row_pitch_B));
+
+   struct pan_image_block_size tile_extent_el =
+      pan_interleaved_64k_tile_size_el(props->format);
+   struct pan_image_block_size tile_extent_px = {
+      tile_extent_el.width * util_format_get_blockwidth(props->format),
+      tile_extent_el.height * util_format_get_blockheight(props->format),
+   };
+   uint64_t tile_size_B = 65536;
+
+   struct pan_image_extent mip_extent_tiles = {
+      DIV_ROUND_UP(mip_extent_px.width, tile_extent_px.width),
+      DIV_ROUND_UP(mip_extent_px.height, tile_extent_px.height),
+      mip_extent_px.depth,
+   };
+
+   uint64_t row_stride_B = mip_extent_tiles.width * tile_size_B;
+   uint64_t surf_stride_B = mip_extent_tiles.height * row_stride_B;
+
+   slayout->offset_B = layout_constraints ? layout_constraints->offset_B : 0;
+   slayout->size_B = mip_extent_tiles.depth * surf_stride_B;
+   slayout->tiled_or_linear.row_stride_B = row_stride_B;
+   slayout->tiled_or_linear.surface_stride_B = surf_stride_B;
+
+   return true;
+}
+
+static uint32_t
+pan_mod_interleaved_64k_get_wsi_row_pitch(const struct pan_image *image,
+                                          unsigned plane_idx, unsigned mip_level)
+{
+   UNREACHABLE("interleaved 64k cannot be used for wsi");
+}
+
+static bool
+pan_mod_interleaved_64k_match(uint64_t mod)
+{
+   return mod == DRM_FORMAT_MOD_ARM_INTERLEAVED_64K;
+}
+
+static enum pan_mod_support
+pan_mod_interleaved_64k_test_props(const struct pan_kmod_dev_props *dprops,
+                                   const struct pan_image_props *iprops,
+                                   const struct pan_image_usage *iusage)
+{
+   assert(GENX(pan_format_from_pipe_format)(iprops->format)->hw);
+
+   /* YUV not supported. */
+   if (pan_format_is_yuv(iprops->format))
+      return PAN_MOD_NOT_SUPPORTED;
+
+   /* Non-po2 byte texel blocks not supported. */
+   if (!util_is_power_of_two_nonzero(util_format_get_blocksize(iprops->format)))
+      return PAN_MOD_NOT_SUPPORTED;
+
+   /* We don't implement multisampling with this layout. */
+   if (iprops->nr_samples > 1)
+      return PAN_MOD_NOT_SUPPORTED;
+
+   /* We don't implement tiling/detiling of this layout on host. */
+   if (iusage && iusage->host_copy)
+      return PAN_MOD_NOT_SUPPORTED;
+
+   /* We don't respect wsi_row_pitch_B so this layout is not usable for WSI. */
+   if (iusage && iusage->wsi)
+      return PAN_MOD_NOT_SUPPORTED;
+
+   struct pan_image_block_size tile_extent_el =
+      pan_interleaved_64k_tile_size_el(iprops->format);
+   struct pan_image_block_size tile_extent_px = {
+      tile_extent_el.width * util_format_get_blockwidth(iprops->format),
+      tile_extent_el.height * util_format_get_blockheight(iprops->format),
+   };
+
+   /* If our image is too thin it might make more sense to use U-interleaved or
+    * even linear */
+   if (iprops->extent_px.width < tile_extent_px.width / 2 ||
+       iprops->extent_px.height < tile_extent_px.height / 2)
+      return PAN_MOD_NOT_OPTIMAL;
+
+   return PAN_MOD_OPTIMAL;
+}
+
+#define pan_mod_interleaved_64k_emit_tex_payload_entry                                 \
+   GENX(pan_tex_emit_interleaved_64k_payload_entry)
+#define pan_mod_interleaved_64k_emit_color_attachment                                  \
+   GENX(pan_emit_interleaved_64k_color_attachment)
+#define pan_mod_interleaved_64k_emit_zs_attachment GENX(pan_emit_interleaved_64k_zs_attachment)
+#define pan_mod_interleaved_64k_emit_s_attachment  GENX(pan_emit_interleaved_64k_s_attachment)
+#endif
+
+static bool
+pan_mod_linear_match(uint64_t mod)
+{
+   return mod == DRM_FORMAT_MOD_LINEAR;
+}
+
+static enum pan_mod_support
+pan_mod_linear_test_props(const struct pan_kmod_dev_props *dprops,
+                          const struct pan_image_props *iprops,
+                          const struct pan_image_usage *iusage)
+{
+   assert(GENX(pan_format_from_pipe_format)(iprops->format)->hw);
+
+   switch (iprops->format) {
+   case PIPE_FORMAT_R8G8B8_420_UNORM_PACKED:
+   case PIPE_FORMAT_R10G10B10_420_UNORM_PACKED:
+   case PIPE_FORMAT_X6R10X6G10_X6R10X6B10_422_UNORM:
+      /* These formats have no linear representation. */
+      return PAN_MOD_NOT_SUPPORTED;
+   default:
+      break;
+   }
+
+   /* We can't implement mapping of tiles at standard sparse granularity using
+    * this layout. */
+   if (iusage && iusage->standard_sparse_mapping_granularity)
+      return PAN_MOD_NOT_SUPPORTED;
+
+   /* We assume that all "better" mods have been tested before linear, and
+    * declare it as optimal so it's always picked when tested, unless it's
+    * not supported.
+    */
+   return PAN_MOD_OPTIMAL;
+}
+
+static uint32_t
+pan_mod_linear_get_wsi_row_pitch(const struct pan_image *image,
+                                 unsigned plane_idx, unsigned mip_level)
+{
+   const struct pan_image_layout *layout = &image->planes[plane_idx]->layout;
+
+   return layout->slices[mip_level].tiled_or_linear.row_stride_B;
+}
+
+#define pan_mod_linear_init_plane_layout NULL
+
+static bool
+pan_mod_linear_init_slice_layout(
+   const struct pan_image_props *props, unsigned plane_idx,
+   struct pan_image_extent mip_extent_px,
+   const struct pan_image_layout_constraints *layout_constraints,
+   struct pan_image_slice_layout *slayout)
+{
+   /* Use explicit layout only when wsi_row_pitch_B is non-zero */
+   const bool use_explicit_layout =
+      layout_constraints && layout_constraints->wsi_row_pitch_B;
+   unsigned align_mask =
+      pan_linear_or_tiled_row_align_req(PAN_ARCH, props->format, plane_idx) - 1;
+   const unsigned fmt_blksize_B =
+      pan_format_get_plane_blocksize(props->format, plane_idx);
+   struct pan_image_extent mip_extent_el;
+
+   if (util_format_is_compressed(props->format)) {
+      assert(util_format_get_num_planes(props->format) == 1);
+      mip_extent_el.width = DIV_ROUND_UP(
+         mip_extent_px.width, util_format_get_blockwidth(props->format));
+      mip_extent_el.height = DIV_ROUND_UP(
+         mip_extent_px.height, util_format_get_blockheight(props->format));
+      mip_extent_el.depth = DIV_ROUND_UP(
+         mip_extent_px.depth, util_format_get_blockdepth(props->format));
+   } else {
+      mip_extent_el = mip_extent_px;
+   }
+
+   if (use_explicit_layout) {
+      unsigned width_from_wsi_row_stride =
+         layout_constraints->wsi_row_pitch_B / fmt_blksize_B;
+
+      if (!util_format_is_compressed(props->format))
+         width_from_wsi_row_stride *= util_format_get_blockwidth(props->format);
+
+      if (width_from_wsi_row_stride < mip_extent_el.width) {
+         mesa_loge("WSI pitch too small");
+         return false;
+      }
+
+      slayout->tiled_or_linear.row_stride_B =
+         layout_constraints->wsi_row_pitch_B;
+      if (slayout->tiled_or_linear.row_stride_B & align_mask) {
+         mesa_loge("WSI pitch not properly aligned");
+         return false;
+      }
+
+      slayout->offset_B = layout_constraints->offset_B;
+      if (slayout->offset_B & align_mask) {
+         mesa_loge("WSI offset not properly aligned");
+         return false;
+      }
+   } else {
+      /* When we can decide of the layout, we want things aligned on at least a
+       * cacheline for performance reasons. */
+      align_mask = MAX2(align_mask, 63);
+      slayout->offset_B = ALIGN_POT(
+         layout_constraints ? layout_constraints->offset_B : 0,
+         MAX2(align_mask + 1, pan_image_slice_align(props->modifier)));
+      slayout->tiled_or_linear.row_stride_B =
+         ALIGN_POT(mip_extent_el.width * fmt_blksize_B, align_mask + 1);
+   }
+
+   uint64_t surf_stride_B =
+      (uint64_t)slayout->tiled_or_linear.row_stride_B * mip_extent_el.height;
+   surf_stride_B = ALIGN_POT(surf_stride_B, (uint64_t)align_mask + 1);
+
+   slayout->tiled_or_linear.surface_stride_B = surf_stride_B;
+   slayout->size_B = surf_stride_B * mip_extent_el.depth * props->nr_samples;
+   return true;
+}
+
+#define pan_mod_linear_emit_tex_payload_entry                                  \
+   GENX(pan_tex_emit_linear_payload_entry)
+#define pan_mod_linear_emit_color_attachment                                   \
+   GENX(pan_emit_linear_color_attachment)
+#define pan_mod_linear_emit_zs_attachment GENX(pan_emit_linear_zs_attachment)
+#define pan_mod_linear_emit_s_attachment  GENX(pan_emit_linear_s_attachment)
+
+#if PAN_ARCH >= 5
+#define EMIT_ATT(__name)                                                       \
+   .emit_color_attachment = pan_mod_##__name##_emit_color_attachment,          \
+   .emit_zs_attachment = pan_mod_##__name##_emit_zs_attachment,                \
+   .emit_s_attachment = pan_mod_##__name##_emit_s_attachment
+#else
+#define EMIT_ATT(__name)                                                       \
+   .emit_color_attachment = NULL, .emit_zs_attachment = NULL,                  \
+   .emit_s_attachment = NULL
+#endif
+
+#define PAN_MOD_DEF(__name)                                                    \
+   {                                                                           \
+      .match = pan_mod_##__name##_match,                                       \
+      .test_props = pan_mod_##__name##_test_props,                             \
+      .get_wsi_row_pitch = pan_mod_##__name##_get_wsi_row_pitch,               \
+      .init_plane_layout = pan_mod_##__name##_init_plane_layout,               \
+      .init_slice_layout = pan_mod_##__name##_init_slice_layout,               \
+      .emit_tex_payload_entry = pan_mod_##__name##_emit_tex_payload_entry,     \
+      EMIT_ATT(__name),                                                        \
+   }
+
+static const struct pan_mod_handler pan_mod_handlers[] = {
+   PAN_MOD_DEF(afbc),
+   PAN_MOD_DEF(u_tiled),
+   PAN_MOD_DEF(linear),
+#if PAN_ARCH >= 10
+   PAN_MOD_DEF(interleaved_64k),
+   PAN_MOD_DEF(afrc),
+#endif
+};
+
+const struct pan_mod_handler *
+GENX(pan_mod_get_handler)(uint64_t modifier)
+{
+   for (unsigned i = 0; i < ARRAY_SIZE(pan_mod_handlers); i++) {
+      if (pan_mod_handlers[i].match(modifier))
+         return &pan_mod_handlers[i];
+   }
+
+   return NULL;
+}

src/panfrost/lib/pan_mod.h

@@ -0,0 +1,121 @@
+/*
+ * Copyright (C) 2025 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef __PAN_MOD_H
+#define __PAN_MOD_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "util/format/u_format.h"
+
+#include "pan_layout.h"
+
+struct pan_attachment_info;
+struct pan_image;
+struct pan_image_view;
+struct pan_image_usage;
+struct pan_kmod_dev_props;
+struct pan_mod_handler;
+
+enum pan_mod_support {
+   PAN_MOD_NOT_SUPPORTED = 0,
+   PAN_MOD_NOT_OPTIMAL,
+   PAN_MOD_OPTIMAL,
+};
+
+struct pan_mod_handler {
+   bool (*match)(uint64_t mod);
+
+   /* Used to check if a set of image properties is valid. Passing a NULL iusage
+    * is valid and means "optimal set of usage for this mod". This implies
+    * that some non-supported cases can't be detected or can be reported as
+    * optimal when specific usage flags would report it non-optimal.
+    */
+   enum pan_mod_support (*test_props)(const struct pan_kmod_dev_props *dprops,
+                                      const struct pan_image_props *iprops,
+                                      const struct pan_image_usage *iusage);
+
+   /* Optional method used to initialize modifier-specific per-plane layout
+    * data. This is called before init_slice_layout().
+    */
+   bool (*init_plane_layout)(const struct pan_image_props *props,
+                             unsigned plane_idx,
+                             struct pan_image_layout *plane_layout);
+
+   bool (*init_slice_layout)(
+      const struct pan_image_props *props, unsigned plane_idx,
+      struct pan_image_extent mip_extent_px,
+      const struct pan_image_layout_constraints *layout_constraints,
+      struct pan_image_slice_layout *slice);
+   uint32_t (*get_wsi_row_pitch)(const struct pan_image *image,
+                                 unsigned plane_idx, unsigned mip_level);
+
+   void (*emit_tex_payload_entry)(const struct pan_image_view *iview,
+                                  unsigned mip_level, unsigned layer_or_z_slice,
+                                  unsigned sample, void **payload);
+
+   void (*emit_color_attachment)(const struct pan_attachment_info *att,
+                                 void *payload);
+   void (*emit_zs_attachment)(const struct pan_attachment_info *att,
+                              void *payload);
+   void (*emit_s_attachment)(const struct pan_attachment_info *att,
+                             void *payload);
+};
+
+#ifdef PAN_ARCH
+const struct pan_mod_handler *GENX(pan_mod_get_handler)(uint64_t modifier);
+
+static inline const struct pan_mod_handler *
+pan_mod_get_handler(unsigned arch, uint64_t modifier)
+{
+   assert(arch == PAN_ARCH);
+   return GENX(pan_mod_get_handler)(modifier);
+}
+#else
+const struct pan_mod_handler *pan_mod_get_handler_v4(uint64_t modifier);
+const struct pan_mod_handler *pan_mod_get_handler_v5(uint64_t modifier);
+const struct pan_mod_handler *pan_mod_get_handler_v6(uint64_t modifier);
+const struct pan_mod_handler *pan_mod_get_handler_v7(uint64_t modifier);
+const struct pan_mod_handler *pan_mod_get_handler_v9(uint64_t modifier);
+const struct pan_mod_handler *pan_mod_get_handler_v10(uint64_t modifier);
+const struct pan_mod_handler *pan_mod_get_handler_v12(uint64_t modifier);
+const struct pan_mod_handler *pan_mod_get_handler_v13(uint64_t modifier);
+const struct pan_mod_handler *pan_mod_get_handler_v14(uint64_t modifier);
+
+static inline const struct pan_mod_handler *
+pan_mod_get_handler(unsigned arch, uint64_t modifier)
+{
+   switch (arch) {
+   case 4:
+      return pan_mod_get_handler_v4(modifier);
+   case 5:
+      return pan_mod_get_handler_v5(modifier);
+   case 6:
+      return pan_mod_get_handler_v6(modifier);
+   case 7:
+      return pan_mod_get_handler_v7(modifier);
+   case 9:
+      return pan_mod_get_handler_v9(modifier);
+   case 10:
+      return pan_mod_get_handler_v10(modifier);
+   case 12:
+      return pan_mod_get_handler_v12(modifier);
+   case 13:
+      return pan_mod_get_handler_v13(modifier);
+   case 14:
+      return pan_mod_get_handler_v14(modifier);
+   default:
+      UNREACHABLE("Unsupported arch");
+   }
+}
+#endif
+
+#ifdef __cplusplus
+} /* extern C */
+#endif
+
+#endif

src/panfrost/lib/pan_pool.h

@@ -0,0 +1,118 @@
+/*
+ * © Copyright 2017-2018 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef __PAN_POOL_H__
+#define __PAN_POOL_H__
+
+#include <stddef.h>
+#include <genxml/gen_macros.h>
+
+#include "util/u_dynarray.h"
+
+struct pan_ptr {
+   /* CPU address */
+   void *cpu;
+
+   /* GPU address */
+   uint64_t gpu;
+};
+
+static inline struct pan_ptr
+pan_ptr_offset(struct pan_ptr ptr, int64_t offset)
+{
+   ptr.cpu = ((char *)ptr.cpu) + offset;
+   ptr.gpu += offset;
+   return ptr;
+}
+
+/* Represents grow-only memory. */
+
+struct pan_pool {
+   /* Minimum size for allocated BOs. */
+   size_t slab_size;
+};
+
+static inline void
+pan_pool_init(struct pan_pool *pool, size_t slab_size)
+{
+   pool->slab_size = slab_size;
+}
+
+/* Represents a fat pointer for GPU-mapped memory, returned from the transient
+ * allocator and not used for much else */
+
+struct pan_ptr pan_pool_alloc_aligned(struct pan_pool *pool, size_t sz,
+                                      unsigned alignment);
+
+#define PAN_POOL_ALLOCATOR(pool_subclass, alloc_func)                          \
+   struct pan_ptr pan_pool_alloc_aligned(struct pan_pool *p, size_t sz,        \
+                                         unsigned alignment)                   \
+   {                                                                           \
+      pool_subclass *pool = container_of(p, pool_subclass, base);              \
+      return alloc_func(pool, sz, alignment);                                  \
+   }
+
+static inline uint64_t
+pan_pool_upload_aligned(struct pan_pool *pool, const void *data, size_t sz,
+                        unsigned alignment)
+{
+   struct pan_ptr transfer = pan_pool_alloc_aligned(pool, sz, alignment);
+
+   if (transfer.cpu)
+      memcpy(transfer.cpu, data, sz);
+
+   return transfer.gpu;
+}
+
+static inline uint64_t
+pan_pool_upload(struct pan_pool *pool, const void *data, size_t sz)
+{
+   return pan_pool_upload_aligned(pool, data, sz, sz);
+}
+
+struct pan_desc_alloc_info {
+   unsigned size;
+   unsigned align;
+   unsigned nelems;
+};
+
+#define PAN_DESC_ARRAY(count, name)                                            \
+   {                                                                           \
+      .size = pan_size(name), .align = pan_alignment(name), .nelems = count,   \
+   }
+
+#define PAN_DESC(name) PAN_DESC_ARRAY(1, name)
+
+#define PAN_DESC_AGGREGATE(...)                                                \
+   (struct pan_desc_alloc_info[])                                              \
+   {                                                                           \
+      __VA_ARGS__, {0},                                                        \
+   }
+
+static inline struct pan_ptr
+pan_pool_alloc_descs(struct pan_pool *pool,
+                     const struct pan_desc_alloc_info *descs)
+{
+   unsigned size = 0;
+   unsigned align = descs[0].align;
+
+   for (unsigned i = 0; descs[i].size; i++) {
+      assert(!(size & (descs[i].align - 1)));
+      size += descs[i].size * descs[i].nelems;
+   }
+
+   return pan_pool_alloc_aligned(pool, size, align);
+}
+
+#define pan_pool_alloc_desc(pool, name)                                        \
+   pan_pool_alloc_descs(pool, PAN_DESC_AGGREGATE(PAN_DESC(name)))
+
+#define pan_pool_alloc_desc_array(pool, count, name)                           \
+   pan_pool_alloc_descs(pool, PAN_DESC_AGGREGATE(PAN_DESC_ARRAY(count, name)))
+
+#define pan_pool_alloc_desc_aggregate(pool, ...)                               \
+   pan_pool_alloc_descs(pool, PAN_DESC_AGGREGATE(__VA_ARGS__))
+
+#endif

src/panfrost/lib/pan_props.c

@@ -0,0 +1,162 @@
+/*
+ * Copyright (C) 2019 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "util/macros.h"
+
+#include "kmod/pan_kmod.h"
+#include "pan_props.h"
+
+#include <genxml/gen_macros.h>
+
+unsigned
+pan_query_l2_slices(const struct pan_kmod_dev_props *props)
+{
+   /* L2_SLICES is MEM_FEATURES[11:8] minus(1) */
+   return ((props->mem_features >> 8) & 0xF) + 1;
+}
+
+struct pan_tiler_features
+pan_query_tiler_features(const struct pan_kmod_dev_props *props)
+{
+   /* Default value (2^9 bytes and 8 levels) to match old behaviour */
+   uint32_t raw = props->tiler_features;
+
+   /* Bin size is log2 in the first byte, max levels in the second byte */
+   return (struct pan_tiler_features){
+      .bin_size = (1 << (raw & BITFIELD_MASK(5))),
+      .max_levels = (raw >> 8) & BITFIELD_MASK(4),
+   };
+}
+
+unsigned
+pan_query_core_count(const struct pan_kmod_dev_props *props,
+                     unsigned *core_id_range)
+{
+   /* On older kernels, worst-case to 16 cores */
+
+   unsigned mask = props->shader_present;
+
+   /* Some cores might be absent. In some cases, we care
+    * about the range of core IDs (that is, the greatest core ID + 1). If
+    * the core mask is contiguous, this equals the core count.
+    */
+   *core_id_range = util_last_bit(mask);
+
+   /* The actual core count skips overs the gaps */
+   return util_bitcount(mask);
+}
+
+unsigned
+pan_query_thread_tls_alloc(const struct pan_kmod_dev_props *props)
+{
+   return props->max_tls_instance_per_core ?: props->max_threads_per_core;
+}
+
+unsigned
+pan_compute_max_thread_count(const struct pan_kmod_dev_props *props,
+                             unsigned work_reg_count)
+{
+   unsigned aligned_reg_count;
+
+   /* 4, 8 or 16 registers per shader on Midgard
+    * 32 or 64 registers per shader on Bifrost
+    */
+   if (pan_arch(props->gpu_id) <= 5) {
+      aligned_reg_count = util_next_power_of_two(MAX2(work_reg_count, 4));
+      assert(aligned_reg_count <= 16);
+   } else {
+      aligned_reg_count = work_reg_count <= 32 ? 32 : 64;
+   }
+
+   return MIN3(props->max_threads_per_wg, props->max_threads_per_core,
+               props->num_registers_per_core / aligned_reg_count);
+}
+
+uint32_t
+pan_query_compressed_formats(const struct pan_kmod_dev_props *props)
+{
+   return props->texture_features[0];
+}
+
+/* Check for AFBC hardware support. AFBC is introduced in v5. Implementations
+ * may omit it, signaled as a nonzero value in the AFBC_FEATURES property. */
+
+bool
+pan_query_afbc(const struct pan_kmod_dev_props *props)
+{
+   unsigned reg = props->afbc_features;
+
+   return (pan_arch(props->gpu_id) >= 5) && (reg == 0);
+}
+
+/* Check for AFRC hardware support. AFRC is introduced in v10. Implementations
+ * may omit it, signaled in bit 25 of TEXTURE_FEATURES_0 property. */
+
+bool
+pan_query_afrc(const struct pan_kmod_dev_props *props)
+{
+   return (pan_arch(props->gpu_id) >= 10) &&
+          (props->texture_features[0] & (1 << 25));
+}
+
+/*
+ * To pipeline multiple tiles, a given tile may use at most half of the tile
+ * buffer. This function returns the optimal size (assuming pipelining).
+ *
+ * For Mali-G510 and Mali-G310, we will need extra logic to query the tilebuffer
+ * size for the particular variant. The CORE_FEATURES register might help.
+ */
+unsigned
+pan_query_tib_size(const struct pan_model *model)
+{
+   /* Preconditions ensure the returned value is a multiple of 1 KiB, the
+    * granularity of the colour buffer allocation field.
+    */
+   assert(model->tilebuffer.color_size >= 2048);
+   assert(util_is_power_of_two_nonzero(model->tilebuffer.color_size));
+
+   return model->tilebuffer.color_size;
+}
+
+unsigned
+pan_query_z_tib_size(const struct pan_model *model)
+{
+   /* Preconditions ensure the returned value is a multiple of 1 KiB, the
+    * granularity of the colour buffer allocation field.
+    */
+   assert(model->tilebuffer.z_size >= 1024);
+   assert(util_is_power_of_two_nonzero(model->tilebuffer.z_size));
+
+   return model->tilebuffer.z_size;
+}
+
+uint64_t
+pan_clamp_to_usable_va_range(const struct pan_kmod_dev *dev, uint64_t va)
+{
+   struct pan_kmod_va_range user_va_range =
+      pan_kmod_dev_query_user_va_range(dev);
+
+   if (va < user_va_range.start)
+      return user_va_range.start;
+   else if (va > user_va_range.start + user_va_range.size)
+      return user_va_range.start + user_va_range.size;
+
+   return va;
+}
+
+uint64_t
+pan_choose_gpu_va_alignment(const struct pan_kmod_vm *vm, uint64_t size)
+{
+   assert(vm->pgsize_bitmap != 0);
+
+   uint64_t align = 0;
+   u_foreach_bit64(pgsize_bit, vm->pgsize_bitmap) {
+      uint64_t pgsize = (uint64_t)1 << pgsize_bit;
+      if (align > 0 && pgsize > size)
+         break;
+      align = pgsize;
+   }
+   return align;
+}

src/panfrost/lib/pan_props.h

@@ -0,0 +1,132 @@
+/*
+ * Copyright (C) 2019 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef PAN_PROPS_H
+#define PAN_PROPS_H
+
+#include <stdbool.h>
+#include <stdint.h>
+
+#include "util/macros.h"
+
+#include "pan_model.h"
+
+struct pan_kmod_dev;
+struct pan_kmod_dev_props;
+struct pan_kmod_vm;
+
+#define ARM_VENDOR_ID 0x13B5
+
+unsigned pan_query_l2_slices(const struct pan_kmod_dev_props *props);
+
+struct pan_tiler_features
+pan_query_tiler_features(const struct pan_kmod_dev_props *props);
+
+unsigned pan_query_thread_tls_alloc(const struct pan_kmod_dev_props *props);
+
+uint32_t pan_query_compressed_formats(const struct pan_kmod_dev_props *props);
+
+unsigned pan_query_core_count(const struct pan_kmod_dev_props *props,
+                              unsigned *core_id_range);
+
+bool pan_query_afbc(const struct pan_kmod_dev_props *props);
+
+bool pan_query_afrc(const struct pan_kmod_dev_props *props);
+
+unsigned pan_query_tib_size(const struct pan_model *model);
+
+unsigned pan_query_z_tib_size(const struct pan_model *model);
+
+static inline unsigned
+pan_query_optimal_tib_size(unsigned arch, const struct pan_model *model)
+{
+   unsigned tib_size = pan_query_tib_size(model);
+
+   /* On V5, as well as V7 and later, we can disable pipelining to gain some
+    * extra tib memory.
+    */
+   if (arch > 4 && arch != 6)
+      return tib_size / 2;
+
+   return tib_size;
+}
+
+static inline unsigned
+pan_query_optimal_z_tib_size(unsigned arch, const struct pan_model *model)
+{
+   unsigned tib_size = pan_query_z_tib_size(model);
+
+   /* On V5, as well as V7 and later, we can disable pipelining to gain some
+    * extra tib memory.
+    */
+   if (arch > 4 && arch != 6)
+      return tib_size / 2;
+
+   return tib_size;
+}
+
+uint64_t pan_clamp_to_usable_va_range(const struct pan_kmod_dev *dev,
+                                      uint64_t va);
+
+uint64_t pan_choose_gpu_va_alignment(const struct pan_kmod_vm *vm,
+                                     uint64_t size);
+
+unsigned pan_compute_max_thread_count(const struct pan_kmod_dev_props *props,
+                                      unsigned work_reg_count);
+
+static inline unsigned
+pan_max_effective_tile_size(unsigned arch)
+{
+   if (arch >= 12)
+      return 64 * 64;
+
+   if (arch >= 10)
+      return 32 * 32;
+
+   return 16 * 16;
+}
+
+static inline unsigned
+pan_meta_tile_size(unsigned arch)
+{
+   if (arch >= 12)
+      return 64;
+
+   return 32;
+}
+
+static inline uint32_t
+pan_get_max_cbufs(unsigned arch, unsigned max_tib_size)
+{
+   if (arch < 5)
+      return 1;
+
+   const unsigned min_msaa = 4;            /* Vulkan *requires* at least 4x MSAA support */
+   const unsigned max_cbuf_format = 4 * 4; /* R32G32B32A32 */
+   const unsigned min_tile_size = 4 * 4;
+
+   unsigned max_cbufs =
+      max_tib_size / (min_msaa * max_cbuf_format * min_tile_size);
+
+   return MIN2(max_cbufs, 8);
+}
+
+static inline unsigned
+pan_get_max_msaa(unsigned arch, unsigned max_tib_size, unsigned max_cbuf_atts,
+                 unsigned format_size)
+{
+   if (arch < 5)
+      return 8;
+
+   assert(max_cbuf_atts > 0);
+   assert(format_size > 0);
+
+   const unsigned min_tile_size = 4 * 4;
+   unsigned max_msaa = max_tib_size / (max_cbuf_atts * format_size *
+                                       min_tile_size);
+   return MIN2(max_msaa, 16);
+}
+
+#endif

src/panfrost/lib/pan_samples.c

@@ -0,0 +1,139 @@
+/*
+ * Copyright (C) 2019 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include <stdint.h>
+
+#include "pan_samples.h"
+
+/* Sample positions are specified partially in hardware, partially in software
+ * on Mali. On Midgard, sample positions are completely fixed but need to be
+ * software accessible to implement gl_SamplePosition. On Bifrost, sample
+ * positions are part fixed, part programmable, and again need to be accessible
+ * for gl_SamplePosition. The mali_sample_position data structure is
+ * hardware-defined on Bifrost as a packed 8:8 fixed point format. The
+ * mali_sample_positions array is part-hardware, part-software-defined: it must
+ * give the sample position when indexed by the sample index, as well as
+ * reading back the origin when indexed by 32.
+ *
+ * The upshot is all of this is known at compile-time, so we can just hardcode
+ * the LUT, upload it to GPU memory on device initialization, and forget about
+ * it.
+ */
+
+struct mali_sample_position {
+   uint16_t x, y;
+} __attribute__((packed));
+
+struct mali_sample_positions {
+   struct mali_sample_position positions[32];
+   struct mali_sample_position origin;
+   struct mali_sample_position padding[64 - (32 + 1)];
+} __attribute__((packed));
+
+/* SAMPLE16 constructs a single sample in terms of 1/16's of the grid, centered
+ * at the origin. SAMPLE4/8 swap the units for legibility. */
+
+#define SAMPLE16(x, y)                                                         \
+   {                                                                           \
+      (((x) + 8) * (256 / 16)), (((y) + 8) * (256 / 16))                       \
+   }
+
+#define SAMPLE8(x, y) SAMPLE16((x)*2, (y)*2)
+#define SAMPLE4(x, y) SAMPLE16((x)*4, (y)*4)
+
+/* clang-format off */
+const struct mali_sample_positions sample_position_lut[] = {
+   [MALI_SAMPLE_PATTERN_SINGLE_SAMPLED] = {
+      .positions = {
+         SAMPLE4(0, 0)
+      },
+      .origin = SAMPLE4(0, 0)
+   },
+
+   [MALI_SAMPLE_PATTERN_ORDERED_4X_GRID] = {
+      .positions = {
+         SAMPLE4(-1, -1),
+         SAMPLE4( 1, -1),
+         SAMPLE4(-1,  1),
+         SAMPLE4( 1,  1),
+      },
+      .origin = SAMPLE4(0, 0)
+   },
+
+   [MALI_SAMPLE_PATTERN_ROTATED_4X_GRID] = {
+      .positions = {
+         SAMPLE8(-1, -3),
+         SAMPLE8( 3, -1),
+         SAMPLE8(-3,  1),
+         SAMPLE8( 1,  3),
+      },
+      .origin = SAMPLE8(0, 0)
+   },
+
+   [MALI_SAMPLE_PATTERN_D3D_8X_GRID] = {
+      .positions = {
+         SAMPLE16( 1, -3),
+         SAMPLE16(-1,  3),
+         SAMPLE16( 5,  1),
+         SAMPLE16(-3, -5),
+         SAMPLE16(-5,  5),
+         SAMPLE16(-7, -1),
+         SAMPLE16( 3,  7),
+         SAMPLE16( 7, -7),
+      },
+      .origin = SAMPLE16(0, 0)
+   },
+
+   [MALI_SAMPLE_PATTERN_D3D_16X_GRID] = {
+      .positions = {
+         SAMPLE16( 1,  1),
+         SAMPLE16(-1, -3),
+         SAMPLE16(-3,  2),
+         SAMPLE16( 4, -1),
+         SAMPLE16(-5, -2),
+         SAMPLE16( 2,  5),
+         SAMPLE16( 5,  3),
+         SAMPLE16( 3, -5),
+         SAMPLE16(-2,  6),
+         SAMPLE16( 0, -7),
+         SAMPLE16(-4, -6),
+         SAMPLE16(-6,  4),
+         SAMPLE16(-8,  0),
+         SAMPLE16( 7, -4),
+         SAMPLE16( 6,  7),
+         SAMPLE16(-7, -8),
+      },
+      .origin = SAMPLE16(0, 0)
+   },
+
+   [MALI_SAMPLE_PATTERN_ROTATED_2X_GRID] = {
+      .positions = {
+         SAMPLE4( 1,  1),
+         SAMPLE4(-1, -1),
+      },
+      .origin = SAMPLE4(0, 0)
+   }
+};
+/* clang-format on */
+
+unsigned
+pan_sample_positions_buffer_size(void)
+{
+   return sizeof(sample_position_lut);
+}
+
+unsigned
+pan_sample_positions_offset(enum mali_sample_pattern pattern)
+{
+   assert(pattern < ARRAY_SIZE(sample_position_lut));
+   unsigned offset = (pattern * sizeof(sample_position_lut[0]));
+   return offset;
+}
+
+void
+pan_upload_sample_positions(void *buffer)
+{
+   memcpy(buffer, sample_position_lut, sizeof(sample_position_lut));
+}

src/panfrost/lib/pan_samples.h

@@ -0,0 +1,17 @@
+/*
+ * Copyright (C) 2019 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef PAN_SAMPLES_H
+#define PAN_SAMPLES_H
+
+#include <genxml/gen_macros.h>
+
+unsigned pan_sample_positions_buffer_size(void);
+
+void pan_upload_sample_positions(void *buffer);
+
+unsigned pan_sample_positions_offset(enum mali_sample_pattern pattern);
+
+#endif

src/panfrost/lib/pan_scratch.c

@@ -0,0 +1,65 @@
+/*
+ * Copyright (C) 2019 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "util/macros.h"
+#include "util/u_math.h"
+#include "pan_encoder.h"
+
+/* Midgard has a small register file, so shaders with high register pressure
+ * need to spill from the register file onto the stack. In addition to
+ * spilling, it is desireable to allocate temporary arrays on the stack (for
+ * instance because the register file does not support indirect access but the
+ * stack does).
+ *
+ * The stack is located in "Thread Local Storage", sometimes abbreviated TLS in
+ * the kernel source code. Thread local storage is allocated per-thread,
+ * per-core, so threads executing concurrently do not interfere with each
+ * other's stacks. On modern kernels, we may query
+ * DRM_PANFROST_PARAM_THREAD_TLS_ALLOC for the number of threads per core we
+ * must allocate for, and DRM_PANFROST_PARAM_SHADER_PRESENT for a bitmask of
+ * shader cores (so take a popcount of that mask for the number of shader
+ * cores). On older kernels that do not support querying these values,
+ * following kbase, we may use the worst-case value of 256 threads for
+ * THREAD_TLS_ALLOC, and the worst-case value of 16 cores for Midgard per the
+ * "shader core count" column of the implementations table in
+ * https://en.wikipedia.org/wiki/Mali_%28GPU% [citation needed]
+ *
+ * Within a particular thread, there is stack allocated. If it is present, its
+ * size is a power-of-two, and it is at least 16 bytes. Stack is allocated
+ * with the shared memory descriptor used for all shaders within a frame (note
+ * that they don't execute concurrently so it's fine). So, consider the maximum
+ * stack size used by any shader within a job, and then compute (where npot
+ * denotes the next power of two):
+ *
+ *      bytes/thread = npot(max(size, 16))
+ *      allocated = (# of bytes/thread) * (# of threads/core) * (# of cores)
+ *
+ * The size of Thread Local Storage is signaled to the GPU in the tls_size
+ * field, which has a log2 modifier and is in units of 16 bytes.
+ */
+
+/* Computes log_stack_size = log2(ceil(s / 16)) */
+
+unsigned
+pan_get_stack_shift(unsigned stack_size)
+{
+   if (stack_size)
+      return util_logbase2_ceil(DIV_ROUND_UP(stack_size, 16));
+   else
+      return 0;
+}
+
+/* Computes the aligned stack size given the shift and thread count. */
+
+unsigned
+pan_get_total_stack_size(unsigned thread_size, unsigned threads_per_core,
+                         unsigned core_id_range)
+{
+   unsigned size_per_thread =
+      (thread_size == 0) ? 0
+                         : util_next_power_of_two(ALIGN_POT(thread_size, 16));
+
+   return size_per_thread * threads_per_core * core_id_range;
+}

src/panfrost/lib/pan_shader.h

@@ -0,0 +1,217 @@
+/*
+ * Copyright (C) 2021 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef __PAN_SHADER_H__
+#define __PAN_SHADER_H__
+
+#include "compiler/nir/nir.h"
+#include "genxml/gen_macros.h"
+#include "panfrost/lib/pan_props.h"
+#include "panfrost/compiler/pan_compiler.h"
+
+#ifdef PAN_ARCH
+
+#if PAN_ARCH >= 9
+static inline enum mali_shader_stage
+pan_shader_stage(const struct pan_shader_info *info)
+{
+   switch (info->stage) {
+   case MESA_SHADER_VERTEX:
+      return MALI_SHADER_STAGE_VERTEX;
+   case MESA_SHADER_FRAGMENT:
+      return MALI_SHADER_STAGE_FRAGMENT;
+   default:
+      return MALI_SHADER_STAGE_COMPUTE;
+   }
+}
+#endif
+
+#if PAN_ARCH >= 7
+static inline enum mali_shader_register_allocation
+pan_register_allocation(unsigned work_reg_count)
+{
+   return (work_reg_count <= 32)
+             ? MALI_SHADER_REGISTER_ALLOCATION_32_PER_THREAD
+             : MALI_SHADER_REGISTER_ALLOCATION_64_PER_THREAD;
+}
+#endif
+
+static inline enum mali_depth_source
+pan_depth_source(const struct pan_shader_info *info)
+{
+   return info->fs.writes_depth ? MALI_DEPTH_SOURCE_SHADER
+                                : MALI_DEPTH_SOURCE_FIXED_FUNCTION;
+}
+
+#if PAN_ARCH < 9
+#if PAN_ARCH <= 5
+static inline void
+pan_shader_prepare_midgard_rsd(const struct pan_shader_info *info,
+                               struct MALI_RENDERER_STATE *rsd)
+{
+   assert((info->push.count & 3) == 0);
+
+   rsd->properties.uniform_count = info->push.count / 4;
+   rsd->properties.shader_has_side_effects = info->writes_global;
+   rsd->properties.fp_mode = MALI_FP_MODE_GL_INF_NAN_ALLOWED;
+
+   /* For fragment shaders, work register count, early-z, reads at draw-time */
+
+   if (info->stage != MESA_SHADER_FRAGMENT) {
+      rsd->properties.work_register_count = info->work_reg_count;
+   } else {
+      rsd->properties.shader_reads_tilebuffer = info->fs.outputs_read;
+
+      /* However, forcing early-z in the shader overrides draw-time */
+      rsd->properties.force_early_z = info->fs.early_fragment_tests;
+   }
+}
+
+#else
+
+#define pan_preloads(reg) (preload & BITFIELD64_BIT(reg))
+
+static void
+pan_make_preload(mesa_shader_stage stage, uint64_t preload,
+                 struct MALI_PRELOAD *out)
+{
+   switch (stage) {
+   case MESA_SHADER_VERTEX:
+      out->vertex.position_result_address_lo = pan_preloads(58);
+      out->vertex.position_result_address_hi = pan_preloads(59);
+      out->vertex.vertex_id = pan_preloads(61);
+      out->vertex.instance_id = pan_preloads(62);
+      break;
+
+   case MESA_SHADER_FRAGMENT:
+      out->fragment.primitive_id = pan_preloads(57);
+      out->fragment.primitive_flags = pan_preloads(58);
+      out->fragment.fragment_position = pan_preloads(59);
+      out->fragment.sample_mask_id = pan_preloads(61);
+      out->fragment.coverage = true;
+      break;
+
+   default:
+      out->compute.local_invocation_xy = pan_preloads(55);
+      out->compute.local_invocation_z = pan_preloads(56);
+      out->compute.work_group_x = pan_preloads(57);
+      out->compute.work_group_y = pan_preloads(58);
+      out->compute.work_group_z = pan_preloads(59);
+      out->compute.global_invocation_x = pan_preloads(60);
+      out->compute.global_invocation_y = pan_preloads(61);
+      out->compute.global_invocation_z = pan_preloads(62);
+      break;
+   }
+}
+
+#if PAN_ARCH == 7
+static inline void
+pan_pack_message_preload(struct MALI_MESSAGE_PRELOAD *cfg,
+                         const struct bifrost_message_preload *msg)
+{
+   enum mali_message_preload_register_format regfmt =
+      msg->fp16 ? MALI_MESSAGE_PRELOAD_REGISTER_FORMAT_F16
+                : MALI_MESSAGE_PRELOAD_REGISTER_FORMAT_F32;
+
+   if (msg->enabled && msg->texture) {
+      cfg->type = MALI_MESSAGE_TYPE_VAR_TEX;
+      cfg->var_tex.varying_index = msg->varying_index;
+      cfg->var_tex.texture_index = msg->texture_index;
+      cfg->var_tex.register_format = regfmt;
+      cfg->var_tex.skip = msg->skip;
+      cfg->var_tex.zero_lod = msg->zero_lod;
+   } else if (msg->enabled) {
+      cfg->type = MALI_MESSAGE_TYPE_LD_VAR;
+      cfg->ld_var.varying_index = msg->varying_index;
+      cfg->ld_var.register_format = regfmt;
+      cfg->ld_var.num_components = msg->num_components;
+   } else {
+      cfg->type = MALI_MESSAGE_TYPE_DISABLED;
+   }
+}
+#endif
+
+static inline void
+pan_shader_prepare_bifrost_rsd(const struct pan_shader_info *info,
+                               struct MALI_RENDERER_STATE *rsd)
+{
+   unsigned fau_count = DIV_ROUND_UP(info->push.count, 2);
+   rsd->preload.uniform_count = fau_count;
+
+#if PAN_ARCH >= 7
+   rsd->properties.shader_register_allocation =
+      pan_register_allocation(info->work_reg_count);
+#endif
+
+   pan_make_preload(info->stage, info->preload, &rsd->preload);
+
+   if (info->stage == MESA_SHADER_FRAGMENT) {
+      rsd->properties.shader_modifies_coverage =
+         info->fs.writes_coverage || info->fs.can_discard;
+
+      rsd->properties.allow_forward_pixel_to_be_killed = !info->writes_global;
+
+#if PAN_ARCH >= 7
+      rsd->properties.shader_wait_dependency_6 = info->bifrost.wait_6;
+      rsd->properties.shader_wait_dependency_7 = info->bifrost.wait_7;
+
+      pan_pack_message_preload(&rsd->message_preload_1,
+                               &info->bifrost.messages[0]);
+      pan_pack_message_preload(&rsd->message_preload_2,
+                               &info->bifrost.messages[1]);
+#endif
+   } else if (info->stage == MESA_SHADER_VERTEX && info->vs.secondary_enable) {
+      rsd->secondary_preload.uniform_count = fau_count;
+
+      pan_make_preload(info->stage, info->vs.secondary_preload,
+                       &rsd->secondary_preload);
+
+      rsd->secondary_shader = rsd->shader.shader + info->vs.secondary_offset;
+
+#if PAN_ARCH >= 7
+      rsd->properties.secondary_shader_register_allocation =
+         pan_register_allocation(info->vs.secondary_work_reg_count);
+#endif
+   }
+}
+
+#endif
+
+static inline void
+pan_shader_prepare_rsd(const struct pan_shader_info *shader_info,
+                       uint64_t shader_ptr, struct MALI_RENDERER_STATE *rsd)
+{
+#if PAN_ARCH <= 5
+   shader_ptr |= shader_info->midgard.first_tag;
+#endif
+
+   rsd->shader.shader = shader_ptr;
+   rsd->shader.attribute_count = shader_info->attribute_count;
+   rsd->shader.varying_count = shader_info->varyings.formats.count;
+   rsd->shader.texture_count = shader_info->texture_count;
+   rsd->shader.sampler_count = shader_info->sampler_count;
+   rsd->properties.shader_contains_barrier = shader_info->contains_barrier;
+   rsd->properties.uniform_buffer_count = shader_info->ubo_count;
+
+   if (shader_info->stage == MESA_SHADER_FRAGMENT) {
+      rsd->properties.stencil_from_shader = shader_info->fs.writes_stencil;
+      rsd->properties.depth_source = pan_depth_source(shader_info);
+
+      /* This also needs to be set if the API forces per-sample
+       * shading, but that'll just got ORed in */
+      rsd->multisample_misc.evaluate_per_sample =
+         shader_info->fs.sample_shading;
+   }
+
+#if PAN_ARCH >= 6
+   pan_shader_prepare_bifrost_rsd(shader_info, rsd);
+#else
+   pan_shader_prepare_midgard_rsd(shader_info, rsd);
+#endif
+}
+#endif /* PAN_ARCH */
+#endif
+
+#endif

src/panfrost/lib/pan_texture.h

@@ -0,0 +1,79 @@
+/*
+ * Copyright (C) 2008 VMware, Inc.
+ * Copyright (C) 2014 Broadcom
+ * Copyright (C) 2018-2019 Alyssa Rosenzweig
+ * Copyright (C) 2019-2020 Collabora, Ltd.
+ * Copyright (C) 2025 Arm Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef __PAN_TEXTURE_H
+#define __PAN_TEXTURE_H
+
+#ifndef PAN_ARCH
+#error "PAN_ARCH must be defined"
+#endif
+
+#include "pan_image.h"
+
+struct pan_ptr;
+struct mali_texture_packed;
+
+#if PAN_ARCH >= 7
+void GENX(pan_texture_swizzle_replicate_x)(struct pan_image_view *iview);
+#endif
+
+#if PAN_ARCH == 7
+void GENX(pan_texture_afbc_reswizzle)(struct pan_image_view *iview);
+#endif
+
+unsigned
+GENX(pan_texture_estimate_payload_size)(const struct pan_image_view *iview);
+
+void GENX(pan_sampled_texture_emit)(const struct pan_image_view *iview,
+                                    struct mali_texture_packed *out,
+                                    const struct pan_ptr *payload);
+
+void GENX(pan_tex_emit_linear_payload_entry)(const struct pan_image_view *iview,
+                                             unsigned mip_level,
+                                             unsigned layer_or_z_slice,
+                                             unsigned sample, void **payload);
+
+void GENX(pan_tex_emit_u_tiled_payload_entry)(
+   const struct pan_image_view *iview, unsigned mip_level,
+   unsigned layer_or_z_slice, unsigned sample, void **payload);
+
+void
+GENX(pan_tex_emit_afbc_payload_entry)(const struct pan_image_view *iview,
+                                      unsigned mip_level, unsigned layer_or_z_slice,
+                                      unsigned sample, void **payload);
+
+#if PAN_ARCH >= 9
+void GENX(pan_storage_texture_emit)(const struct pan_image_view *iview,
+                                    struct mali_texture_packed *out,
+                                    const struct pan_ptr *payload);
+#endif
+
+void GENX(pan_tex_emit_linear_payload_entry)(const struct pan_image_view *iview,
+                                             unsigned mip_level,
+                                             unsigned layer_or_z_slice,
+                                             unsigned sample, void **payload);
+void GENX(pan_tex_emit_u_tiled_payload_entry)(
+   const struct pan_image_view *iview, unsigned mip_level,
+   unsigned layer_or_z_slice, unsigned sample, void **payload);
+void GENX(pan_tex_emit_afbc_payload_entry)(const struct pan_image_view *iview,
+                                           unsigned mip_level,
+                                           unsigned layer_or_z_slice,
+                                           unsigned sample, void **payload);
+
+#if PAN_ARCH >= 10
+void GENX(pan_tex_emit_interleaved_64k_payload_entry)(
+      const struct pan_image_view *iview, unsigned mip_level,
+      unsigned layer_or_z_slice, unsigned sample, void **payload);
+
+void GENX(pan_tex_emit_afrc_payload_entry)(
+      const struct pan_image_view *iview, unsigned mip_level,
+      unsigned layer_or_z_slice, unsigned sample, void **payload);
+#endif
+
+#endif

src/panfrost/lib/pan_tiler.c

@@ -0,0 +1,352 @@
+/*
+ * Copyright (C) 2019 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "util/macros.h"
+#include "util/u_math.h"
+#include "pan_encoder.h"
+
+/* Mali GPUs are tiled-mode renderers, rather than immediate-mode.
+ * Conceptually, the screen is divided into 16x16 tiles. Vertex shaders run.
+ * Then, a fixed-function hardware block (the tiler) consumes the gl_Position
+ * results. For each triangle specified, it marks each containing tile as
+ * containing that triangle. This set of "triangles per tile" form the "polygon
+ * list". Finally, the rasterization unit consumes the polygon list to invoke
+ * the fragment shader.
+ *
+ * In practice, it's a bit more complicated than this. On Midgard chips with an
+ * "advanced tiling unit" (all except T720/T820/T830), 16x16 is the logical
+ * tile size, but Midgard features "hierarchical tiling", where power-of-two
+ * multiples of the base tile size can be used: hierarchy level 0 (16x16),
+ * level 1 (32x32), level 2 (64x64), per public information about Midgard's
+ * tiling. In fact, tiling goes up to 4096x4096 (!), although in practice
+ * 128x128 is the largest usually used (though higher modes are enabled).  The
+ * idea behind hierarchical tiling is to use low tiling levels for small
+ * triangles and high levels for large triangles, to minimize memory bandwidth
+ * and repeated fragment shader invocations (the former issue inherent to
+ * immediate-mode rendering and the latter common in traditional tilers).
+ *
+ * The tiler itself works by reading varyings in and writing a polygon list
+ * out. Unfortunately (for us), both of these buffers are managed in main
+ * memory; although they ideally will be cached, it is the drivers'
+ * responsibility to allocate these buffers. Varying buffer allocation is
+ * handled elsewhere, as it is not tiler specific; the real issue is allocating
+ * the polygon list.
+ *
+ * This is hard, because from the driver's perspective, we have no information
+ * about what geometry will actually look like on screen; that information is
+ * only gained from running the vertex shader. (Theoretically, we could run the
+ * vertex shaders in software as a prepass, or in hardware with transform
+ * feedback as a prepass, but either idea is ludicrous on so many levels).
+ *
+ * Instead, Mali uses a bit of a hybrid approach, splitting the polygon list
+ * into three distinct pieces. First, the driver statically determines which
+ * tile hierarchy levels to use (more on that later). At this point, we know the
+ * framebuffer dimensions and all the possible tilings of the framebuffer, so
+ * we know exactly how many tiles exist across all hierarchy levels. The first
+ * piece of the polygon list is the header, which is exactly 8 bytes per tile,
+ * plus padding and a small 64-byte prologue. (If that doesn't remind you of
+ * AFBC, it should. See pan_afbc.c for some fun parallels). The next part is
+ * the polygon list body, which seems to contain 512 bytes per tile, again
+ * across every level of the hierarchy. These two parts form the polygon list
+ * buffer. This buffer has a statically determinable size, approximately equal
+ * to the # of tiles across all hierarchy levels * (8 bytes + 512 bytes), plus
+ * alignment / minimum restrictions / etc.
+ *
+ * The third piece is the easy one (for us): the tiler heap. In essence, the
+ * tiler heap is a gigantic slab that's as big as could possibly be necessary
+ * in the worst case imaginable. Just... a gigantic allocation that we give a
+ * start and end pointer to. What's the catch? The tiler heap is lazily
+ * allocated; that is, a huge amount of memory is _reserved_, but only a tiny
+ * bit is actually allocated upfront. The GPU just keeps using the
+ * unallocated-but-reserved portions as it goes along, generating page faults
+ * if it goes beyond the allocation, and then the kernel is instructed to
+ * expand the allocation on page fault (known in the vendor kernel as growable
+ * memory). This is quite a bit of bookkeeping of its own, but that task is
+ * pushed to kernel space and we can mostly ignore it here, just remembering to
+ * set the GROWABLE flag so the kernel actually uses this path rather than
+ * allocating a gigantic amount up front and burning a hole in RAM.
+ *
+ * As far as determining which hierarchy levels to use, the simple answer is
+ * that right now, we don't. In the tiler configuration fields (consistent from
+ * the earliest Midgard's SFBD through the latest Bifrost traces we have),
+ * there is a hierarchy_mask field, controlling which levels (tile sizes) are
+ * enabled. Ideally, the hierarchical tiling dream -- mapping big polygons to
+ * big tiles and small polygons to small tiles -- would be realized here as
+ * well. As long as there are polygons at all needing tiling, we always have to
+ * have big tiles available, in case there are big polygons. But we don't
+ * necessarily need small tiles available. Ideally, when there are small
+ * polygons, small tiles are enabled (to avoid waste from putting small
+ * triangles in the big tiles); when there are not, small tiles are disabled to
+ * avoid enabling more levels than necessary, which potentially costs in memory
+ * bandwidth / power / tiler performance.
+ *
+ * Of course, the driver has to figure this out statically. When tile
+ * hiearchies are actually established, this occurs by the tiler in
+ * fixed-function hardware, after the vertex shaders have run and there is
+ * sufficient information to figure out the size of triangles. The driver has
+ * no such luxury, again barring insane hacks like additionally running the
+ * vertex shaders in software or in hardware via transform feedback. Thus, for
+ * the driver, we need a heuristic approach.
+ *
+ * There are lots of heuristics to guess triangle size statically you could
+ * imagine, but one approach shines as particularly simple-stupid: assume all
+ * on-screen triangles are equal size and spread equidistantly throughout the
+ * screen. Let's be clear, this is NOT A VALID ASSUMPTION. But if we roll with
+ * it, then we see:
+ *
+ *      Triangle Area   = (Screen Area / # of triangles)
+ *                      = (Width * Height) / (# of triangles)
+ *
+ * Or if you prefer, we can also make a third CRAZY assumption that we only draw
+ * right triangles with edges parallel/perpendicular to the sides of the screen
+ * with no overdraw, forming a triangle grid across the screen:
+ *
+ * |--w--|
+ *  _____   |
+ * | /| /|  |
+ * |/_|/_|  h
+ * | /| /|  |
+ * |/_|/_|  |
+ *
+ * Then you can use some middle school geometry and algebra to work out the
+ * triangle dimensions. I started working on this, but realised I didn't need
+ * to to make my point, but couldn't bare to erase that ASCII art. Anyway.
+ *
+ * POINT IS, by considering the ratio of screen area and triangle count, we can
+ * estimate the triangle size. For a small size, use small bins; for a large
+ * size, use large bins. Intuitively, this metric makes sense: when there are
+ * few triangles on a large screen, you're probably compositing a UI and
+ * therefore the triangles are large; when there are a lot of triangles on a
+ * small screen, you're probably rendering a 3D mesh and therefore the
+ * triangles are tiny. (Or better said -- there will be tiny triangles, even if
+ * there are also large triangles. There have to be unless you expect crazy
+ * overdraw. Generally, it's better to allow more small bin sizes than
+ * necessary than not allow enough.)
+ *
+ * From this heuristic (or whatever), we determine the minimum allowable tile
+ * size, and we use that to decide the hierarchy masking, selecting from the
+ * minimum "ideal" tile size to the maximum tile size (2048x2048 in practice).
+ *
+ * Once we have that mask and the framebuffer dimensions, we can compute the
+ * size of the statically-sized polygon list structures, allocate them, and go!
+ *
+ * -----
+ *
+ * On T720, T820, and T830, there is no support for hierarchical tiling.
+ * Instead, the hardware allows the driver to select the tile size dynamically
+ * on a per-framebuffer basis, including allowing rectangular/non-square tiles.
+ * Rules for tile size selection are as follows:
+ *
+ *  - Dimensions must be powers-of-two.
+ *  - The smallest tile is 16x16.
+ *  - The tile width/height is at most the framebuffer w/h (clamp up to 16 pix)
+ *  - There must be no more than 64 tiles in either dimension.
+ *
+ * Within these constraints, the driver is free to pick a tile size according
+ * to some heuristic, similar to units with an advanced tiling unit.
+ *
+ * To pick a size without any heuristics, we may satisfy the constraints by
+ * defaulting to 16x16 (a power-of-two). This fits the minimum. For the size
+ * constraint, consider:
+ *
+ *      # of tiles < 64
+ *      ceil (fb / tile) < 64
+ *      (fb / tile) <= (64 - 1)
+ *      tile <= fb / (64 - 1) <= next_power_of_two(fb / (64 - 1))
+ *
+ * Hence we clamp up to align_pot(fb / (64 - 1)).
+
+ * Extending to use a selection heuristic left for future work.
+ *
+ * Once the tile size (w, h) is chosen, we compute the hierarchy "mask":
+ *
+ *      hierarchy_mask = (log2(h / 16) << 6) | log2(w / 16)
+ *
+ * Of course with no hierarchical tiling, this is not a mask; it's just a field
+ * specifying the tile size. But I digress.
+ *
+ * We also compute the polgon list sizes (with framebuffer size W, H) as:
+ *
+ *      full_size = 0x200 + 0x200 * ceil(W / w) * ceil(H / h)
+ *      offset = 8 * ceil(W / w) * ceil(H / h)
+ *
+ * It further appears necessary to round down offset to the nearest 0x200.
+ * Possibly we would also round down full_size to the nearest 0x200 but
+ * full_size/0x200 = (1 + ceil(W / w) * ceil(H / h)) is an integer so there's
+ * nothing to do.
+ */
+
+/* Hierarchical tiling spans from 16x16 to 4096x4096 tiles */
+
+#define MIN_TILE_SIZE 16
+#define MAX_TILE_SIZE 4096
+
+/* Constants as shifts for easier power-of-two iteration */
+
+#define MIN_TILE_SHIFT util_logbase2(MIN_TILE_SIZE)
+#define MAX_TILE_SHIFT util_logbase2(MAX_TILE_SIZE)
+
+/* The hierarchy has a 64-byte prologue */
+#define PROLOGUE_SIZE 0x40
+
+/* For each tile (across all hierarchy levels), there is 8 bytes of header */
+#define HEADER_BYTES_PER_TILE 0x8
+
+/* Likewise, each tile per level has 512 bytes of body */
+#define FULL_BYTES_PER_TILE 0x200
+
+static unsigned
+pan_hierarchy_size(unsigned width, unsigned height, unsigned mask,
+                   unsigned bytes_per_tile)
+{
+   unsigned size = PROLOGUE_SIZE;
+
+   /* Iterate hierarchy levels */
+   u_foreach_bit(level, mask) {
+      assert(level <= (MAX_TILE_SHIFT - MIN_TILE_SHIFT) &&
+             "invalid hierarchy mask");
+
+      /* Levels are power-of-two sizes */
+      unsigned tile_size = MIN_TILE_SIZE << level;
+
+      size += DIV_ROUND_UP(width, tile_size) * DIV_ROUND_UP(height, tile_size) *
+              bytes_per_tile;
+   }
+
+   /* This size will be used as an offset, so ensure it's aligned */
+   return ALIGN_POT(size, 0x200);
+}
+
+/* Implement the formula:
+ *
+ *      0x200 + bytes_per_tile * ceil(W / w) * ceil(H / h)
+ *
+ * rounding down the answer to the nearest 0x200. This is used to compute both
+ * header and body sizes for GPUs without hierarchical tiling. Essentially,
+ * computing a single hierarchy level, since there isn't any hierarchy!
+ */
+
+static unsigned
+pan_flat_size(unsigned width, unsigned height, unsigned dim,
+              unsigned bytes_per_tile)
+{
+   /* First, extract the tile dimensions */
+   unsigned tw = (1 << (dim & 0b111)) * 8;
+   unsigned th = (1 << ((dim & (0b111 << 6)) >> 6)) * 8;
+
+   /* Calculate the raw size */
+   unsigned raw =
+      DIV_ROUND_UP(width, tw) * DIV_ROUND_UP(height, th) * bytes_per_tile;
+
+   /* Round down and add offset */
+   return 0x200 + ((raw / 0x200) * 0x200);
+}
+
+/* Given a hierarchy mask and a framebuffer size, compute the header size */
+
+unsigned
+pan_tiler_header_size(unsigned width, unsigned height, unsigned mask,
+                      bool hierarchy)
+{
+   if (hierarchy)
+      return pan_hierarchy_size(width, height, mask, HEADER_BYTES_PER_TILE);
+   else
+      return pan_flat_size(width, height, mask, HEADER_BYTES_PER_TILE);
+}
+
+/* The combined header/body is sized similarly (but it is significantly
+ * larger), except that it can be empty when the tiler disabled, rather than
+ * getting clamped to a minimum size.
+ */
+
+unsigned
+pan_tiler_full_size(unsigned width, unsigned height, unsigned mask,
+                    bool hierarchy)
+{
+   if (hierarchy)
+      return pan_hierarchy_size(width, height, mask, FULL_BYTES_PER_TILE);
+   else
+      return pan_flat_size(width, height, mask, FULL_BYTES_PER_TILE);
+}
+
+/* On GPUs without hierarchical tiling, we choose a tile size directly and
+ * stuff it into the field otherwise known as hierarchy mask (not a mask). */
+
+static unsigned
+pan_choose_tile_size(unsigned width, unsigned height, unsigned vertex_count)
+{
+   /* Figure out the ideal tile size. Eventually a heuristic should be
+    * used for this */
+
+   unsigned best_w = 16;
+   unsigned best_h = 16;
+
+   /* Clamp so there are less than 64 tiles in each direction */
+
+   best_w = MAX2(best_w, util_next_power_of_two(width / 63));
+   best_h = MAX2(best_h, util_next_power_of_two(height / 63));
+
+   /* We have our ideal tile size, so encode */
+
+   unsigned exp_w = util_logbase2(best_w / 16);
+   unsigned exp_h = util_logbase2(best_h / 16);
+
+   return exp_w | (exp_h << 6);
+}
+
+unsigned
+pan_choose_hierarchy_mask(unsigned width, unsigned height,
+                          unsigned vertex_count, bool hierarchy)
+{
+   /* If there is no geometry, we don't bother enabling anything */
+
+   if (!vertex_count)
+      return 0x00;
+
+   if (!hierarchy)
+      return pan_choose_tile_size(width, height, vertex_count);
+
+   /* Heuristic: choose the largest minimum bin size such that there are an
+    * average of k vertices per bin at the lowest level. This is modeled as:
+    *
+    *    k = vertex_count / ((fb width / bin width) * (fb height / bin height))
+    *
+    * Bins are square, so solving for bin size = bin width = bin height:
+    *
+    *    bin size = sqrt(((k) (fb width) (fb height) / vertex count))
+    *
+    * k = 4 represents each bin as a QUAD. If the screen is completely tiled
+    * into nonoverlapping uniform power-of-two squares, then this heuristic sets
+    * the bin size to the quad size, which seems like an ok choice.
+    */
+   unsigned k = 4;
+   unsigned log2_min_bin_size =
+      util_logbase2_ceil((k * width * height) / vertex_count) / 2;
+
+   /* Do not use bins larger than the framebuffer. They will be empty. */
+   unsigned log2_max_bin_size = util_logbase2_ceil(MAX2(width, height));
+
+   /* For small framebuffers, use one big tile */
+   log2_min_bin_size = MIN2(log2_min_bin_size, log2_max_bin_size);
+
+   /* Clamp to valid bin sizes */
+   log2_min_bin_size = CLAMP(log2_min_bin_size, MIN_TILE_SHIFT, MAX_TILE_SHIFT);
+   log2_max_bin_size = CLAMP(log2_max_bin_size, MIN_TILE_SHIFT, MAX_TILE_SHIFT);
+
+   /* Bin indices are numbered from 0 started with MIN_TILE_SIZE */
+   unsigned min_bin_index = log2_min_bin_size - MIN_TILE_SHIFT;
+   unsigned max_bin_index = log2_max_bin_size - MIN_TILE_SHIFT;
+
+   /* Enable up to 8 bins starting from the heuristic selected minimum. 8
+    * is the implementation specific maximum in supported Midgard devices.
+    */
+   unsigned mask =
+      (BITFIELD_MASK(8) << min_bin_index) & BITFIELD_MASK(max_bin_index + 1);
+
+   assert(mask != 0 && "too few levels");
+   assert(util_bitcount(mask) <= 8 && "too many levels");
+
+   return mask;
+}

src/panfrost/lib/pan_trace.c

@@ -0,0 +1,102 @@
+/*
+ * Copyright © 2026 Amazon.com, Inc. or its affiliates.
+ *
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "pan_trace.h"
+
+#include "util/os_misc.h"
+
+#define PAN_TRACE_ENV_VAR "PAN_CPU_TRACE"
+
+#define CATEGORY(str, flag) { str, ARRAY_SIZE(str) - 1, (uint64_t) (flag) }
+
+/* To be kept in sync with the pan_trace_category enum in pan_trace.h. */
+/* clang-format off */
+static struct {
+   const char *str;
+   size_t len;
+   uint64_t flag;
+} categories_table[] = {
+   /* Library categories. */
+   CATEGORY("lib.afbc",      PAN_TRACE_LIB_AFBC),
+   CATEGORY("lib.desc",      PAN_TRACE_LIB_DESC),
+   CATEGORY("lib.kmod",      PAN_TRACE_LIB_KMOD),
+
+   CATEGORY("lib",           PAN_TRACE_LIB_AFBC |
+                             PAN_TRACE_LIB_DESC |
+                             PAN_TRACE_LIB_KMOD),
+
+   /* Gallium categories. */
+   CATEGORY("gl.blit",       PAN_TRACE_GL_BLIT),
+   CATEGORY("gl.bo",         PAN_TRACE_GL_BO),
+   CATEGORY("gl.cmdstream",  PAN_TRACE_GL_CMDSTREAM),
+   CATEGORY("gl.context",    PAN_TRACE_GL_CONTEXT),
+   CATEGORY("gl.csf",        PAN_TRACE_GL_CSF),
+   CATEGORY("gl.disk_cache", PAN_TRACE_GL_DISK_CACHE),
+   CATEGORY("gl.fb_preload", PAN_TRACE_GL_FB_PRELOAD),
+   CATEGORY("gl.jm",         PAN_TRACE_GL_JM),
+   CATEGORY("gl.job",        PAN_TRACE_GL_JOB),
+   CATEGORY("gl.mempool",    PAN_TRACE_GL_MEMPOOL),
+   CATEGORY("gl.resource",   PAN_TRACE_GL_RESOURCE),
+   CATEGORY("gl.shader",     PAN_TRACE_GL_SHADER),
+
+   CATEGORY("gl",            PAN_TRACE_GL_BLIT |
+                             PAN_TRACE_GL_BO |
+                             PAN_TRACE_GL_CMDSTREAM |
+                             PAN_TRACE_GL_CONTEXT |
+                             PAN_TRACE_GL_CSF |
+                             PAN_TRACE_GL_DISK_CACHE |
+                             PAN_TRACE_GL_FB_PRELOAD |
+                             PAN_TRACE_GL_JM |
+                             PAN_TRACE_GL_JOB |
+                             PAN_TRACE_GL_MEMPOOL |
+                             PAN_TRACE_GL_RESOURCE |
+                             PAN_TRACE_GL_SHADER),
+
+   /* Vulkan categories. */
+   CATEGORY("vk.csf",        PAN_TRACE_VK_CSF),
+
+   CATEGORY("vk",            PAN_TRACE_VK_CSF),
+};
+/* clang-format on */
+
+uint64_t pan_trace_categories = 0;
+
+static bool
+is_separator(char c)
+{
+   return c == ',' || c == ';' || c == ' ';
+}
+
+void
+pan_trace_init(void)
+{
+   const char *list = os_get_option(PAN_TRACE_ENV_VAR);
+   const char *str = NULL;
+   uint64_t categories = 0;
+   char prev_char = ',';
+
+   if (!list)
+      return;
+
+   /* Parse list and flag enabled categories. */
+   for (int i = 0; prev_char; prev_char = list[i++]) {
+      if (!is_separator(list[i]) && list[i]) {
+         if (is_separator(prev_char))
+            str = &list[i];
+      } else if (!is_separator(prev_char)) {
+         for (int j = 0; j < ARRAY_SIZE(categories_table); j++) {
+            size_t len = &list[i] - str;
+            if (categories_table[j].len == len &&
+                !strncasecmp(categories_table[j].str, str, len)) {
+               categories |= categories_table[j].flag;
+               break;
+            }
+         }
+      }
+   }
+
+   pan_trace_categories = categories;
+}

src/panfrost/lib/pan_trace.h

@@ -0,0 +1,93 @@
+/*
+ * Copyright © 2026 Amazon.com, Inc. or its affiliates.
+ *
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef __PAN_TRACE_H
+#define __PAN_TRACE_H
+
+#include "util/perf/cpu_trace.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Panfrost wrappers for Mesa CPU scope traces.
+ *
+ * CPU tracing (not to be confused with GPU command stream tracing with
+ * PAN_MESA_DEBUG=trace) is often enabled in release builds. Since each trace
+ * pushed down the underlying tracing backend (Perfetto, Gpuvis, sysprof, etc)
+ * has a slight cost, Panfrost extends the Mesa CPU scope traces by
+ * associating each trace to a category that must be enabled at run-time. This
+ * allows to keep adding useful trace points into Panfrost without worrying
+ * about the implied inherent latency.
+ *
+ * There are 3 categories ("lib" for the shared utilities, "gl" for Gallium,
+ * "vk" for PanVK) divided into subcategories so that subsystems can easily be
+ * traced. A list of categories can be passed to Panfrost through the
+ * PAN_CPU_TRACE environment variable like that:
+ *
+ *   $ PAN_CPU_TRACE=gl,lib program
+ *
+ * Passing a category enables traces for all its subcategories. Subcategories
+ * can be passed instead for finer grained traces like that:
+ *
+ *   $ PAN_CPU_TRACE=gl.blit,gl.bo,gl.job program
+ */
+
+/* Panfrost trace categories.
+ *
+ * To be kept in sync with the categories_table array in pan_trace.c.
+ */
+/* clang-format off */
+enum pan_trace_category {
+   /* Library categories. */
+   PAN_TRACE_LIB_AFBC      = BITFIELD_BIT(1),  /* "lib.afbc" */
+   PAN_TRACE_LIB_DESC      = BITFIELD_BIT(2),  /* "lib.desc" */
+   PAN_TRACE_LIB_KMOD      = BITFIELD_BIT(3),  /* "lib.kmod" */
+
+   /* Gallium categories. */
+   PAN_TRACE_GL_BLIT       = BITFIELD_BIT(4),  /* "gl.blit" */
+   PAN_TRACE_GL_BO         = BITFIELD_BIT(5),  /* "gl.bo" */
+   PAN_TRACE_GL_CMDSTREAM  = BITFIELD_BIT(6),  /* "gl.cmdstream" */
+   PAN_TRACE_GL_CONTEXT    = BITFIELD_BIT(7),  /* "gl.context" */
+   PAN_TRACE_GL_CSF        = BITFIELD_BIT(8),  /* "gl.csf" */
+   PAN_TRACE_GL_DISK_CACHE = BITFIELD_BIT(9),  /* "gl.disk_cache" */
+   PAN_TRACE_GL_FB_PRELOAD = BITFIELD_BIT(10), /* "gl.fb_preload" */
+   PAN_TRACE_GL_JM         = BITFIELD_BIT(11), /* "gl.jm" */
+   PAN_TRACE_GL_JOB        = BITFIELD_BIT(12), /* "gl.job" */
+   PAN_TRACE_GL_MEMPOOL    = BITFIELD_BIT(13), /* "gl.mempool" */
+   PAN_TRACE_GL_RESOURCE   = BITFIELD_BIT(14), /* "gl.resource" */
+   PAN_TRACE_GL_SHADER     = BITFIELD_BIT(15), /* "gl.shader" */
+
+   /* Vulkan categories. */
+   PAN_TRACE_VK_CSF        = BITFIELD_BIT(16), /* "vk.csf" */
+};
+/* clang-format on */
+
+extern uint64_t pan_trace_categories;
+
+/* Add a Mesa CPU scope trace for a given Panfrost category using printf like
+ * formatting.
+ */
+#define PAN_TRACE_SCOPE(category, format, ...)                               \
+   MESA_TRACE_SCOPE_IF(category & pan_trace_categories, format, ##__VA_ARGS__)
+
+/* Add a Mesa CPU scope trace for a given Panfrost category using current
+ * function name.
+ */
+#define PAN_TRACE_FUNC(category)                                             \
+   MESA_TRACE_FUNC_IF(category & pan_trace_categories)
+
+/* Parse the PAN_CPU_TRACE environment variable and initialize CPU tracing.
+ * The PAN_CPU_TRACE environment variable stores a list of categories (see
+ * enum pan_trace_category) separated by a comma, a semicolon or a space.
+ */
+void pan_trace_init(void);
+
+#ifdef __cplusplus
+} /* extern C */
+#endif
+
+#endif /* __PAN_TRACE_H */

src/panfrost/lib/pan_util.c

@@ -0,0 +1,53 @@
+/*
+ * Copyright (C) 2019 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "pan_util.h"
+#include <stdio.h>
+
+/* Translate a PIPE swizzle quad to a 12-bit Mali swizzle code. PIPE
+ * swizzles line up with Mali swizzles for the XYZW01, but PIPE swizzles have
+ * an additional "NONE" field that we have to mask out to zero. Additionally,
+ * PIPE swizzles are sparse but Mali swizzles are packed */
+
+unsigned
+pan_translate_swizzle_4(const unsigned char swizzle[4])
+{
+   unsigned out = 0;
+
+   for (unsigned i = 0; i < 4; ++i) {
+      assert(swizzle[i] <= PIPE_SWIZZLE_1);
+      out |= (swizzle[i] << (3 * i));
+   }
+
+   return out;
+}
+
+void
+pan_invert_swizzle(const unsigned char *in, unsigned char *out)
+{
+   /* First, default to all zeroes, both to prevent uninitialized junk
+      and to provide a known baseline so we can tell when components
+      have been modified
+    */
+
+   for (unsigned c = 0; c < 4; ++c)
+      out[c] = PIPE_SWIZZLE_0;
+
+   /* Now "do" what the swizzle says */
+
+   for (unsigned c = 0; c < 4; ++c) {
+      unsigned char i = in[c];
+
+      /* Who cares? */
+      assert(PIPE_SWIZZLE_X == 0);
+      if (i > PIPE_SWIZZLE_W)
+         continue;
+
+      /* Invert (only if we haven't already applied) */
+      unsigned idx = i - PIPE_SWIZZLE_X;
+      if (out[idx] == PIPE_SWIZZLE_0)
+         out[idx] = PIPE_SWIZZLE_X + c;
+   }
+}

src/panfrost/lib/pan_util.h

@@ -0,0 +1,60 @@
+/*
+ * Copyright 2019 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef PAN_UTIL_H
+#define PAN_UTIL_H
+
+#include <stdbool.h>
+#include <stdint.h>
+#include "util/format/u_format.h"
+
+#define PAN_DBG_PERF  0x0001
+#define PAN_DBG_TRACE 0x0002
+/* 0x4 unused */
+#define PAN_DBG_DIRTY 0x0008
+#define PAN_DBG_SYNC  0x0010
+/* 0x20 unused */
+#define PAN_DBG_NOFP16  0x0040
+#define PAN_DBG_NO_CRC  0x0080
+#define PAN_DBG_GL3     0x0100
+#define PAN_DBG_NO_AFBC 0x0200
+/* 0x400 unused */
+#define PAN_DBG_STRICT_IMPORT 0x0800
+#define PAN_DBG_LINEAR   0x1000
+#define PAN_DBG_NO_CACHE 0x2000
+#define PAN_DBG_DUMP     0x4000
+
+#ifndef NDEBUG
+#define PAN_DBG_OVERFLOW 0x8000
+#endif
+
+#define PAN_DBG_YUV        0x20000
+#define PAN_DBG_FORCE_PACK 0x40000
+#define PAN_DBG_CS         0x80000
+
+struct pan_blendable_format;
+
+unsigned pan_translate_swizzle_4(const unsigned char swizzle[4]);
+
+void pan_invert_swizzle(const unsigned char *in, unsigned char *out);
+
+void pan_pack_color(const struct pan_blendable_format *blendable_formats,
+                    uint32_t *packed, const union pipe_color_union *color,
+                    enum pipe_format format, bool dithered);
+
+/* Get the last blend shader, for an erratum workaround on v5 */
+
+static inline uint64_t
+pan_last_nonnull(uint64_t *ptrs, unsigned count)
+{
+   for (signed i = ((signed)count - 1); i >= 0; --i) {
+      if (ptrs[i])
+         return ptrs[i];
+   }
+
+   return 0;
+}
+
+#endif /* PAN_UTIL_H */

src/panfrost/lib/tests/test-attribute.c

@@ -0,0 +1,67 @@
+/*
+ * Copyright 2025 Collabora Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+#include "pan_encoder.h"
+
+struct test {
+   const char *label;
+   uint32_t offset_id;
+   uint32_t divisor;
+};
+
+#define TEST_DEF(label, offset_id, divisor) {label, offset_id, divisor}
+
+static const struct test attribute_tests[] = {
+   TEST_DEF("Sanity check even", 2, 2),
+   TEST_DEF("Sanity check odd", 9, 3),
+   TEST_DEF("Sanity check random #1", 42, 3),
+   TEST_DEF("Sanity check random #2", 999, 3),
+   TEST_DEF("Sanity check max #1", UINT32_MAX, UINT32_MAX),
+   TEST_DEF("Sanity check max #2", UINT32_MAX, INT32_MAX),
+   TEST_DEF("Regression test case #1", 404896682, 444453),
+   TEST_DEF("Regression test case #2", 449209605, 980807),
+   TEST_DEF("Regression test case #3", 412962763, 103240691),
+   TEST_DEF("Regression test case #4", 1269169964, 507465),
+   TEST_DEF("Regression test case #5", 1771745081, 6867229),
+};
+
+static uint32_t
+compute_effective_id(uint64_t offset_id, uint64_t r, uint64_t e, uint64_t d)
+{
+   return (offset_id + e) * (d + (1u << 31)) / ((uint64_t)1 << (32 + r));
+}
+
+#define ASSERT_EQ(x, y)                                                        \
+   do {                                                                        \
+      if (x == y) {                                                            \
+         nr_pass++;                                                            \
+      } else {                                                                 \
+         nr_fail++;                                                            \
+         fprintf(stderr, "%s: Assertion failed %s (%x) != %s (%x)\n", T.label, \
+                 #x, x, #y, y);                                                \
+      }                                                                        \
+   } while (0)
+
+int
+main(int argc, const char **argv)
+{
+   unsigned nr_pass = 0, nr_fail = 0;
+
+   for (unsigned i = 0; i < ARRAY_SIZE(attribute_tests); ++i) {
+      struct test T = attribute_tests[i];
+
+      uint32_t e;
+      uint32_t r;
+      uint32_t d = pan_compute_npot_divisor(T.divisor, &r, &e);
+
+      uint32_t expected_effective_id = T.offset_id / T.divisor;
+      uint32_t computed_effective_id =
+         compute_effective_id(T.offset_id, r, e, d);
+
+      ASSERT_EQ(expected_effective_id, computed_effective_id);
+   }
+
+   printf("Passed %u/%u\n", nr_pass, nr_pass + nr_fail);
+   return nr_fail ? 1 : 0;
+}

src/panfrost/lib/tests/test-blend.c

@@ -0,0 +1,308 @@
+/*
+ * Copyright (C) 2021 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "pan_blend.h"
+
+/* A test consists of a given blend mode and its translated form */
+struct test {
+   const char *label;
+   struct pan_blend_equation eq;
+   unsigned constant_mask;
+   bool reads_dest;
+   bool opaque;
+   bool fixed_function;
+   bool alpha_zero_nop;
+   bool alpha_one_store;
+   uint32_t hardware;
+};
+
+/* clang-format off */
+#define RGBA(key, value) \
+   .rgb_ ## key = value, \
+   .alpha_ ## key = value
+
+static const struct test blend_tests[] = {
+   {
+      "Replace",
+      {
+         .blend_enable = false,
+         .color_mask = 0xF,
+      },
+      .constant_mask = 0x0,
+      .reads_dest = false,
+      .opaque = true,
+      .fixed_function = true,
+      .alpha_zero_nop = false,
+      .alpha_one_store = false,
+      .hardware = 0xF0122122
+   },
+   {
+      "Alpha",
+      {
+         .blend_enable = true,
+         .color_mask = 0xF,
+
+         RGBA(func, PIPE_BLEND_ADD),
+         RGBA(src_factor, PIPE_BLENDFACTOR_SRC_ALPHA),
+         RGBA(dst_factor, PIPE_BLENDFACTOR_INV_SRC_ALPHA),
+      },
+      .constant_mask = 0x0,
+      .reads_dest = true,
+      .opaque = false,
+      .fixed_function = true,
+      .alpha_zero_nop = true,
+      .alpha_one_store = true,
+      .hardware = 0xF0503503
+   },
+   {
+      "Additive",
+      {
+         .blend_enable = true,
+         .color_mask = 0xF,
+
+         RGBA(func, PIPE_BLEND_ADD),
+         RGBA(src_factor, PIPE_BLENDFACTOR_ONE),
+         RGBA(dst_factor, PIPE_BLENDFACTOR_ONE),
+      },
+      .constant_mask = 0x0,
+      .reads_dest = true,
+      .opaque = false,
+      .fixed_function = true,
+      .alpha_zero_nop = false,
+      .alpha_one_store = false,
+      .hardware = 0xF0932932 /* equivalently 0xF0923923 */
+   },
+   {
+      "Additive-Alpha",
+      {
+         .blend_enable = true,
+         .color_mask = 0xF,
+
+         RGBA(func, PIPE_BLEND_ADD),
+         RGBA(src_factor, PIPE_BLENDFACTOR_SRC_ALPHA),
+         RGBA(dst_factor, PIPE_BLENDFACTOR_ONE),
+      },
+      .constant_mask = 0x0,
+      .reads_dest = true,
+      .opaque = false,
+      .fixed_function = true,
+      .alpha_zero_nop = true,
+      .alpha_one_store = false,
+      .hardware = 0xF0523523
+   },
+   {
+      "Subtractive",
+      {
+         .blend_enable = true,
+         .color_mask = 0xF,
+
+         RGBA(func, PIPE_BLEND_SUBTRACT),
+         RGBA(src_factor, PIPE_BLENDFACTOR_ONE),
+         RGBA(dst_factor, PIPE_BLENDFACTOR_ONE),
+      },
+      .constant_mask = 0x0,
+      .reads_dest = true,
+      .opaque = false,
+      .fixed_function = true,
+      .alpha_zero_nop = false,
+      .alpha_one_store = false,
+      .hardware = 0xF09B29B2 /* equivalently 0xF09A39A3 */
+   },
+   {
+      "Subtractive-Alpha",
+      {
+         .blend_enable = true,
+         .color_mask = 0xF,
+
+         RGBA(func, PIPE_BLEND_SUBTRACT),
+         RGBA(src_factor, PIPE_BLENDFACTOR_SRC_ALPHA),
+         RGBA(dst_factor, PIPE_BLENDFACTOR_ONE),
+      },
+      .constant_mask = 0x0,
+      .reads_dest = true,
+      .opaque = false,
+      .fixed_function = true,
+      .alpha_zero_nop = false,
+      .alpha_one_store = false,
+      .hardware = 0xF052B52b /* equivalently 0xF05A35A3 */
+   },
+   {
+      "Modulate",
+      {
+         .blend_enable = true,
+         .color_mask = 0xF,
+
+         RGBA(func, PIPE_BLEND_ADD),
+         RGBA(src_factor, PIPE_BLENDFACTOR_ZERO),
+         RGBA(dst_factor, PIPE_BLENDFACTOR_SRC_COLOR),
+      },
+      .constant_mask = 0x0,
+      .reads_dest = true,
+      .opaque = false,
+      .fixed_function = true,
+      .alpha_zero_nop = false,
+      .alpha_one_store = false,
+      .hardware = 0xF0231231 /* equivalently 0xF0321321 */
+   },
+   {
+      "Replace masked",
+      {
+         .blend_enable = false,
+         .color_mask = 0x3,
+      },
+      .constant_mask = 0x0,
+      .reads_dest = true,
+      .opaque = false,
+      .fixed_function = true,
+      .alpha_zero_nop = false,
+      .alpha_one_store = false,
+      .hardware = 0x30122122
+   },
+   {
+      "Modulate masked",
+      {
+         .blend_enable = true,
+         .color_mask = 0xA,
+
+         RGBA(func, PIPE_BLEND_ADD),
+         RGBA(src_factor, PIPE_BLENDFACTOR_ZERO),
+         RGBA(dst_factor, PIPE_BLENDFACTOR_SRC_COLOR),
+      },
+      .constant_mask = 0x0,
+      .reads_dest = true,
+      .opaque = false,
+      .fixed_function = true,
+      .alpha_zero_nop = false,
+      .alpha_one_store = false,
+      .hardware = 0xA0231231 /* equivalently 0xA0321321 */
+   },
+   {
+      "src*dst + dst*src",
+      {
+         .blend_enable = true,
+         .color_mask = 0xF,
+
+         RGBA(func, PIPE_BLEND_ADD),
+         RGBA(src_factor, PIPE_BLENDFACTOR_DST_COLOR),
+         RGBA(dst_factor, PIPE_BLENDFACTOR_SRC_COLOR),
+      },
+      .constant_mask = 0x0,
+      .reads_dest = true,
+      .opaque = false,
+      .fixed_function = true,
+      .alpha_zero_nop = false,
+      .alpha_one_store = false,
+      .hardware = 0xF0431431 /* 0 + dest * (2*src) */
+   },
+   {
+      "Mixed src*dst + dst*src masked I",
+      {
+         .blend_enable = true,
+         .color_mask = 0xC,
+
+         .rgb_func = PIPE_BLEND_ADD,
+         .rgb_src_factor = PIPE_BLENDFACTOR_ONE,
+         .rgb_dst_factor= PIPE_BLENDFACTOR_ZERO,
+
+         .alpha_func = PIPE_BLEND_ADD,
+         .alpha_src_factor = PIPE_BLENDFACTOR_DST_COLOR,
+         .alpha_dst_factor= PIPE_BLENDFACTOR_SRC_COLOR,
+      },
+      .constant_mask = 0x0,
+      .reads_dest = true,
+      .opaque = false,
+      .fixed_function = true,
+      .alpha_zero_nop = false,
+      .alpha_one_store = false,
+      .hardware = 0xC0431132 /* 0 + dest * (2*src); equivalent 0xC0431122 */
+   },
+   {
+      "Mixed src*dst + dst*src masked II",
+      {
+         .blend_enable = true,
+         .color_mask = 0xC,
+
+         .rgb_func = PIPE_BLEND_ADD,
+         .rgb_src_factor = PIPE_BLENDFACTOR_ONE,
+         .rgb_dst_factor = PIPE_BLENDFACTOR_ZERO,
+
+         .alpha_func = PIPE_BLEND_ADD,
+         .alpha_src_factor = PIPE_BLENDFACTOR_DST_ALPHA,
+         .alpha_dst_factor= PIPE_BLENDFACTOR_SRC_COLOR,
+      },
+      .constant_mask = 0x0,
+      .reads_dest = true,
+      .opaque = false,
+      .fixed_function = true,
+      .alpha_zero_nop = false,
+      .alpha_one_store = false,
+      .hardware = 0xC0431132 /* 0 + dest * (2*src); equivalent 0xC0431122 */
+   },
+   {
+      "Mixed src*dst + dst*src masked III",
+      {
+         .blend_enable = true,
+         .color_mask = 0xC,
+
+         .rgb_func = PIPE_BLEND_ADD,
+         .rgb_src_factor = PIPE_BLENDFACTOR_ONE,
+         .rgb_dst_factor = PIPE_BLENDFACTOR_ZERO,
+
+         .alpha_func = PIPE_BLEND_ADD,
+         .alpha_src_factor = PIPE_BLENDFACTOR_DST_ALPHA,
+         .alpha_dst_factor= PIPE_BLENDFACTOR_SRC_ALPHA,
+      },
+      .constant_mask = 0x0,
+      .reads_dest = true,
+      .opaque = false,
+      .fixed_function = true,
+      .alpha_zero_nop = false,
+      .alpha_one_store = false,
+      .hardware = 0xC0431132 /* 0 + dest * (2*src); equivalent 0xC0431122 */
+   }
+};
+/* clang-format on */
+
+#define ASSERT_EQ(x, y)                                                        \
+   do {                                                                        \
+      if (x == y) {                                                            \
+         nr_pass++;                                                            \
+      } else {                                                                 \
+         nr_fail++;                                                            \
+         fprintf(stderr, "%s: Assertion failed %s (%x) != %s (%x)\n", T.label, \
+                 #x, x, #y, y);                                                \
+      }                                                                        \
+   } while (0)
+
+int
+main(int argc, const char **argv)
+{
+   unsigned nr_pass = 0, nr_fail = 0;
+
+   /* The architecture we test for is arbitrary and used only for checking
+    * whether we can use fixed function. If we really wanted to be paranoid
+    * we could add a loop checking all architectures, but in practice there's
+    * not much difference and we're only checking for internal consistency
+    * anyway
+    */
+   unsigned arch = 10;
+
+   for (unsigned i = 0; i < ARRAY_SIZE(blend_tests); ++i) {
+      struct test T = blend_tests[i];
+      ASSERT_EQ(T.constant_mask, pan_blend_constant_mask(T.eq));
+      ASSERT_EQ(T.reads_dest, pan_blend_reads_dest(T.eq));
+      ASSERT_EQ(T.opaque, pan_blend_is_opaque(T.eq));
+      ASSERT_EQ(T.fixed_function, pan_blend_can_fixed_function(arch, T.eq, true));
+      ASSERT_EQ(T.alpha_zero_nop, pan_blend_alpha_zero_nop(T.eq));
+      ASSERT_EQ(T.alpha_one_store, pan_blend_alpha_one_store(T.eq));
+
+      if (pan_blend_can_fixed_function(arch, T.eq, true)) {
+         ASSERT_EQ(T.hardware, pan_pack_blend(T.eq));
+      }
+   }
+
+   printf("Passed %u/%u\n", nr_pass, nr_pass + nr_fail);
+   return nr_fail ? 1 : 0;
+}

src/panfrost/lib/tests/test-clear.c

@@ -0,0 +1,169 @@
+/*
+ * Copyright (C) 2021 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "pan_image.h"
+#include "pan_util.h"
+
+/* A test consists of a render target format, clear colour, dither state, and
+ * translated form. Dither state matters when the tilebuffer format is more
+ * precise than the final format. */
+struct test {
+   enum pipe_format format;
+   bool dithered;
+   union pipe_color_union colour;
+   uint32_t packed[4];
+};
+
+#define RRRR(r)                                                                \
+   {                                                                           \
+      r, r, r, r                                                               \
+   }
+#define RGRG(r, g)                                                             \
+   {                                                                           \
+      r, g, r, g                                                               \
+   }
+#define F(r, g, b, a)                                                          \
+   {                                                                           \
+      .f = { r, g, b, a }                                                      \
+   }
+#define UI(r, g, b, a)                                                         \
+   {                                                                           \
+      .ui = { r, g, b, a }                                                     \
+   }
+#define D (true)
+#define _ (false)
+
+/* clang-format off */
+static const struct test clear_tests[] = {
+   /* Basic tests */
+   { PIPE_FORMAT_R8G8B8A8_UNORM,    D, F(0.0,   0.0, 0.0, 0.0),   RRRR(0x00000000) },
+   { PIPE_FORMAT_R8G8B8A8_UNORM,    _, F(0.0,   0.0, 0.0, 0.0),   RRRR(0x00000000) },
+   { PIPE_FORMAT_R8G8B8A8_UNORM,    D, F(1.0,   0.0, 0.0, 1.0),   RRRR(0xFF0000FF) },
+   { PIPE_FORMAT_R8G8B8A8_UNORM,    _, F(1.0,   0.0, 0.0, 1.0),   RRRR(0xFF0000FF) },
+   { PIPE_FORMAT_B8G8R8A8_UNORM,    D, F(1.0,   0.0, 0.0, 1.0),   RRRR(0xFF0000FF) },
+   { PIPE_FORMAT_B8G8R8A8_UNORM,    _, F(1.0,   0.0, 0.0, 1.0),   RRRR(0xFF0000FF) },
+   { PIPE_FORMAT_R8G8B8A8_UNORM,    D, F(0.664, 0.0, 0.0, 0.0),   RRRR(0x000000A9) },
+   { PIPE_FORMAT_R8G8B8A8_UNORM,    _, F(0.664, 0.0, 0.0, 0.0),   RRRR(0x000000A9) },
+   { PIPE_FORMAT_R4G4B4A4_UNORM,    D, F(0.664, 0.0, 0.0, 0.0),   RRRR(0x0000009F) },
+   { PIPE_FORMAT_R4G4B4A4_UNORM,    _, F(0.664, 0.0, 0.0, 0.0),   RRRR(0x000000A0) },
+
+   /* Test rounding to nearest even. The values are cherrypicked to multiply
+    * out to a fractional part of 0.5. The first test should round down and
+    * second test should round up. */
+
+   { PIPE_FORMAT_R4G4B4A4_UNORM,    D, F(0.41875, 0.0, 0.0, 1.0), RRRR(0xF0000064) },
+   { PIPE_FORMAT_R4G4B4A4_UNORM,    D, F(0.40625, 0.0, 0.0, 1.0), RRRR(0xF0000062) },
+
+   /* Testing rounding direction when dithering is disabled. The packed value
+    * is what gets rounded. This behaves as round-to-even with the cutoff point
+    * at 2^-m + 2^-n for an m:n representation. */
+
+   { PIPE_FORMAT_R4G4B4A4_UNORM,    _, F(0.03125, 0.0, 0.0, 1.0),  RRRR(0xF0000000) },
+   { PIPE_FORMAT_R4G4B4A4_UNORM,    _, F(0.03125, 0.0, 0.0, 1.0),  RRRR(0xF0000000) },
+   { PIPE_FORMAT_R4G4B4A4_UNORM,    _, F(0.03333, 0.0, 0.0, 1.0),  RRRR(0xF0000000) },
+   { PIPE_FORMAT_R4G4B4A4_UNORM,    _, F(0.03334, 0.0, 0.0, 1.0),  RRRR(0xF0000010) },
+   { PIPE_FORMAT_R4G4B4A4_UNORM,    _, F(0.09375, 0.0, 0.0, 1.0),  RRRR(0xF0000010) },
+
+   /* Check all the special formats with different edge cases */
+
+   { PIPE_FORMAT_R4G4B4A4_UNORM,    D, F(0.127, 2.4, -1.0, 0.5), RRRR(0x7800F01E) },
+   { PIPE_FORMAT_R5G5B5A1_UNORM,    D, F(0.127, 2.4, -1.0, 0.5), RRRR(0x400F807E) },
+   { PIPE_FORMAT_R5G6B5_UNORM,      D, F(0.127, 2.4, -1.0, 0.5), RRRR(0x000FC07E) },
+   { PIPE_FORMAT_R10G10B10A2_UNORM, D, F(0.127, 2.4, -1.0, 0.5), RRRR(0x800FFC82) },
+   { PIPE_FORMAT_R8G8B8A8_SRGB,     D, F(0.127, 2.4, -1.0, 0.5), RRRR(0x8000FF64) },
+
+   { PIPE_FORMAT_R4G4B4A4_UNORM,    D, F(0.718, 0.18, 1.0, 2.0), RRRR(0xF0F02BAC) },
+   { PIPE_FORMAT_R5G6B5_UNORM,      D, F(0.718, 0.18, 1.0, 2.0), RRRR(0x3E02D6C8) },
+   { PIPE_FORMAT_R5G5B5A1_UNORM,    D, F(0.718, 0.18, 1.0, 2.0), RRRR(0xBE02CEC8) },
+   { PIPE_FORMAT_R10G10B10A2_UNORM, D, F(0.718, 0.18, 1.0, 2.0), RRRR(0xFFF2E2DF) },
+   { PIPE_FORMAT_R8G8B8A8_SRGB,     D, F(0.718, 0.18, 1.0, 2.0), RRRR(0xFFFF76DC) },
+
+   /* Check that values are padded when dithering is disabled */
+
+   { PIPE_FORMAT_R4G4B4A4_UNORM,    _, F(0.718, 0.18, 1.0, 2.0), RRRR(0xF0F030B0) },
+   { PIPE_FORMAT_R5G6B5_UNORM,      _, F(0.718, 0.18, 1.0, 2.0), RRRR(0x3E02C2C0) },
+   { PIPE_FORMAT_R5G5B5A1_UNORM,    _, F(0.718, 0.18, 1.0, 2.0), RRRR(0xBE0302C0) },
+   { PIPE_FORMAT_R10G10B10A2_UNORM, _, F(0.718, 0.18, 1.0, 2.0), RRRR(0xFFF2E2DF) },
+   { PIPE_FORMAT_R8G8B8A8_SRGB,     _, F(0.718, 0.18, 1.0, 2.0), RRRR(0xFFFF76DC) },
+
+   /* Check that blendable tilebuffer values are invariant under swizzling */
+
+   { PIPE_FORMAT_B4G4R4A4_UNORM,    D, F(0.127, 2.4, -1.0, 0.5), RRRR(0x7800F01E) },
+   { PIPE_FORMAT_B5G5R5A1_UNORM,    D, F(0.127, 2.4, -1.0, 0.5), RRRR(0x400F807E) },
+   { PIPE_FORMAT_B5G6R5_UNORM,      D, F(0.127, 2.4, -1.0, 0.5), RRRR(0x000FC07E) },
+   { PIPE_FORMAT_B10G10R10A2_UNORM, D, F(0.127, 2.4, -1.0, 0.5), RRRR(0x800FFC82) },
+   { PIPE_FORMAT_B8G8R8A8_SRGB,     D, F(0.127, 2.4, -1.0, 0.5), RRRR(0x8000FF64) },
+
+   { PIPE_FORMAT_B4G4R4A4_UNORM,    D, F(0.718, 0.18, 1.0, 2.0), RRRR(0xF0F02BAC) },
+   { PIPE_FORMAT_B5G6R5_UNORM,      D, F(0.718, 0.18, 1.0, 2.0), RRRR(0x3E02D6C8) },
+   { PIPE_FORMAT_B5G5R5A1_UNORM,    D, F(0.718, 0.18, 1.0, 2.0), RRRR(0xBE02CEC8) },
+   { PIPE_FORMAT_B10G10R10A2_UNORM, D, F(0.718, 0.18, 1.0, 2.0), RRRR(0xFFF2E2DF) },
+   { PIPE_FORMAT_B8G8R8A8_SRGB,     D, F(0.718, 0.18, 1.0, 2.0), RRRR(0xFFFF76DC) },
+
+   { PIPE_FORMAT_B4G4R4A4_UNORM,    _, F(0.718, 0.18, 1.0, 2.0), RRRR(0xF0F030B0) },
+   { PIPE_FORMAT_B5G6R5_UNORM,      _, F(0.718, 0.18, 1.0, 2.0), RRRR(0x3E02C2C0) },
+   { PIPE_FORMAT_B5G5R5A1_UNORM,    _, F(0.718, 0.18, 1.0, 2.0), RRRR(0xBE0302C0) },
+   { PIPE_FORMAT_B10G10R10A2_UNORM, _, F(0.718, 0.18, 1.0, 2.0), RRRR(0xFFF2E2DF) },
+   { PIPE_FORMAT_B8G8R8A8_SRGB,     _, F(0.718, 0.18, 1.0, 2.0), RRRR(0xFFFF76DC) },
+
+   /* Check raw formats, which are not invariant under swizzling. Raw formats
+    * cannot be dithered. */
+
+   { PIPE_FORMAT_R8G8B8A8_UINT,   D, UI(0xCA, 0xFE, 0xBA, 0xBE), RRRR(0xBEBAFECA) },
+   { PIPE_FORMAT_R8G8B8A8_UINT,   _, UI(0xCA, 0xFE, 0xBA, 0xBE), RRRR(0xBEBAFECA) },
+
+   { PIPE_FORMAT_B8G8R8A8_UINT,   D, UI(0xCA, 0xFE, 0xBA, 0xBE), RRRR(0xBECAFEBA) },
+   { PIPE_FORMAT_B8G8R8A8_UINT,   _, UI(0xCA, 0xFE, 0xBA, 0xBE), RRRR(0xBECAFEBA) },
+
+   /* Check that larger raw formats are replicated correctly */
+
+   { PIPE_FORMAT_R16G16B16A16_UINT, D, UI(0xCAFE, 0xBABE, 0xABAD, 0x1DEA),
+                                       RGRG(0xBABECAFE, 0x1DEAABAD) },
+
+   { PIPE_FORMAT_R16G16B16A16_UINT, _, UI(0xCAFE, 0xBABE, 0xABAD, 0x1DEA),
+                                       RGRG(0xBABECAFE, 0x1DEAABAD) },
+
+   { PIPE_FORMAT_R32G32B32A32_UINT, D,
+      UI(0xCAFEBABE, 0xABAD1DEA, 0xDEADBEEF, 0xABCDEF01),
+      { 0xCAFEBABE, 0xABAD1DEA, 0xDEADBEEF, 0xABCDEF01 } },
+
+   { PIPE_FORMAT_R32G32B32A32_UINT, _,
+      UI(0xCAFEBABE, 0xABAD1DEA, 0xDEADBEEF, 0xABCDEF01),
+      { 0xCAFEBABE, 0xABAD1DEA, 0xDEADBEEF, 0xABCDEF01 } },
+};
+/* clang-format on */
+
+#define ASSERT_EQ(x, y)                                                                      \
+   do {                                                                                      \
+      if ((x[0] == y[0]) && (x[1] == y[1]) && (x[2] == y[2]) &&                              \
+          (x[3] == y[3])) {                                                                  \
+         nr_pass++;                                                                          \
+      } else {                                                                               \
+         nr_fail++;                                                                          \
+         fprintf(                                                                            \
+            stderr,                                                                          \
+            "%s%s: Assertion failed %s (%08X %08X %08X %08X) != %s (%08X %08X %08X %08X)\n", \
+            util_format_short_name(T.format), T.dithered ? " dithered" : "",                 \
+            #x, x[0], x[1], x[2], x[3], #y, y[0], y[1], y[2], y[3]);                         \
+      }                                                                                      \
+   } while (0)
+
+int
+main(int argc, const char **argv)
+{
+   unsigned nr_pass = 0, nr_fail = 0;
+
+   for (unsigned i = 0; i < ARRAY_SIZE(clear_tests); ++i) {
+      struct test T = clear_tests[i];
+      uint32_t packed[4];
+      pan_pack_color(pan_blendable_formats_v7, &packed[0], &T.colour, T.format,
+                     T.dithered);
+
+      ASSERT_EQ(T.packed, packed);
+   }
+
+   printf("Passed %u/%u\n", nr_pass, nr_pass + nr_fail);
+   return nr_fail ? 1 : 0;
+}

src/panfrost/lib/tests/test-earlyzs.cpp

@@ -0,0 +1,167 @@
+/*
+ * Copyright (C) 2022 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "compiler/pan_compiler.h"
+#include "pan_earlyzs.h"
+#include "genxml/gen_macros.h"
+
+#include <gtest/gtest.h>
+
+/*
+ * Test the early-ZS helpers used on Bifrost and Valhall. Early-ZS state depends
+ * on both shader state and draw-time API state. As such, there are two helpers
+ * -- analzye and get -- that separate the link-time analysis of a fragment
+ * shader from the draw-time classification. The internal data structure is not
+ * under test, only the external API. So we test only the composition.
+ */
+
+#define ZS_WRITEMASK             BITFIELD_BIT(0)
+#define ALPHA2COV                BITFIELD_BIT(1)
+#define ZS_ALWAYS_PASSES         BITFIELD_BIT(2)
+#define DISCARD                  BITFIELD_BIT(3)
+#define WRITES_Z                 BITFIELD_BIT(4)
+#define WRITES_S                 BITFIELD_BIT(5)
+#define WRITES_COV               BITFIELD_BIT(6)
+#define SIDEFX                   BITFIELD_BIT(7)
+#define API_EARLY                BITFIELD_BIT(8)
+#define SHADER_READS_ZS          BITFIELD_BIT(9)
+#define ARCH_HAS_READONLY_ZS_OPT BITFIELD_BIT(10)
+#define ARCH_HAS_STATE_TRACK_OPT BITFIELD_BIT(11)
+
+static void
+test(enum mali_pixel_kill expected_update, enum mali_pixel_kill expected_kill,
+     bool expected_shader_readonly_zs, uint32_t flags)
+{
+   enum pan_earlyzs_zs_tilebuf_read zs_read = PAN_EARLYZS_ZS_TILEBUF_NOT_READ;
+   struct pan_shader_info info = {};
+   info.fs.can_discard = !!(flags & DISCARD);
+   info.fs.writes_depth = !!(flags & WRITES_Z);
+   info.fs.writes_stencil = !!(flags & WRITES_S);
+   info.fs.writes_coverage = !!(flags & WRITES_COV);
+   info.fs.early_fragment_tests = !!(flags & API_EARLY);
+   info.writes_global = !!(flags & SIDEFX);
+
+   unsigned arch = 9;
+
+   if (flags & ARCH_HAS_STATE_TRACK_OPT)
+      arch = 11;
+   else if (flags & ARCH_HAS_READONLY_ZS_OPT)
+      arch = 10;
+
+   if (flags & SHADER_READS_ZS) {
+      if (flags & (WRITES_Z | WRITES_S))
+         zs_read = PAN_EARLYZS_ZS_TILEBUF_READ_NO_OPT;
+      else
+         zs_read = PAN_EARLYZS_ZS_TILEBUF_READ_OPT;
+   }
+
+   struct pan_earlyzs_state result = pan_earlyzs_get(
+      pan_earlyzs_analyze(&info, arch), !!(flags & ZS_WRITEMASK),
+      !!(flags & ALPHA2COV), !!(flags & ZS_ALWAYS_PASSES), zs_read);
+
+   ASSERT_EQ(result.update, expected_update);
+   ASSERT_EQ(result.kill, expected_kill);
+   ASSERT_EQ(result.shader_readonly_zs, expected_shader_readonly_zs);
+}
+
+#define CASE(expected_update, expected_kill, flags)                            \
+   test(MALI_PIXEL_KILL_##expected_update, MALI_PIXEL_KILL_##expected_kill, false,     \
+        flags)
+
+#define CASE_RO_ZS(expected_update, expected_kill, expected_ro_zs, flags)      \
+   test(MALI_PIXEL_KILL_##expected_update, MALI_PIXEL_KILL_##expected_kill,            \
+        expected_ro_zs, flags)
+
+TEST(EarlyZS, APIForceEarly)
+{
+   CASE(FORCE_EARLY, FORCE_EARLY, API_EARLY);
+   CASE(FORCE_EARLY, FORCE_EARLY, API_EARLY | WRITES_Z | WRITES_S);
+   CASE(FORCE_EARLY, FORCE_EARLY, API_EARLY | ALPHA2COV | DISCARD);
+}
+
+TEST(EarlyZS, ShaderCalculatesZS)
+{
+   CASE(FORCE_LATE, FORCE_LATE, WRITES_Z);
+   CASE(FORCE_LATE, FORCE_LATE, WRITES_S);
+   CASE(FORCE_LATE, FORCE_LATE, WRITES_Z | WRITES_S);
+   CASE(FORCE_LATE, FORCE_LATE, WRITES_Z | WRITES_S | SIDEFX);
+   CASE(FORCE_LATE, FORCE_LATE, WRITES_Z | WRITES_S | ZS_ALWAYS_PASSES);
+   CASE(FORCE_LATE, FORCE_LATE, WRITES_Z | ZS_ALWAYS_PASSES | ALPHA2COV);
+}
+
+TEST(EarlyZS, ModifiesCoverageWritesZSNoSideFX)
+{
+   CASE(FORCE_LATE, FORCE_EARLY, ZS_WRITEMASK | WRITES_COV);
+   CASE(FORCE_LATE, FORCE_EARLY, ZS_WRITEMASK | DISCARD);
+   CASE(FORCE_LATE, FORCE_EARLY, ZS_WRITEMASK | ALPHA2COV);
+   CASE(FORCE_LATE, FORCE_EARLY,
+        ZS_WRITEMASK | WRITES_COV | DISCARD | ALPHA2COV);
+}
+
+TEST(EarlyZS, ModifiesCoverageWritesZSNoSideFXAlt)
+{
+   CASE(FORCE_LATE, WEAK_EARLY, ZS_ALWAYS_PASSES | ZS_WRITEMASK | WRITES_COV);
+   CASE(FORCE_LATE, WEAK_EARLY, ZS_ALWAYS_PASSES | ZS_WRITEMASK | DISCARD);
+   CASE(FORCE_LATE, WEAK_EARLY, ZS_ALWAYS_PASSES | ZS_WRITEMASK | ALPHA2COV);
+   CASE(FORCE_LATE, WEAK_EARLY,
+        ZS_ALWAYS_PASSES | ZS_WRITEMASK | WRITES_COV | DISCARD | ALPHA2COV);
+}
+
+TEST(EarlyZS, ModifiesCoverageWritesZSSideFX)
+{
+   CASE(FORCE_LATE, FORCE_LATE, ZS_WRITEMASK | SIDEFX | WRITES_COV);
+   CASE(FORCE_LATE, FORCE_LATE, ZS_WRITEMASK | SIDEFX | DISCARD);
+   CASE(FORCE_LATE, FORCE_LATE, ZS_WRITEMASK | SIDEFX | ALPHA2COV);
+   CASE(FORCE_LATE, FORCE_LATE,
+        ZS_WRITEMASK | SIDEFX | WRITES_COV | DISCARD | ALPHA2COV);
+}
+
+TEST(EarlyZS, SideFXNoShaderZS)
+{
+   CASE(FORCE_EARLY, FORCE_LATE, SIDEFX);
+   CASE(FORCE_EARLY, FORCE_LATE, SIDEFX | DISCARD);
+   CASE(FORCE_EARLY, FORCE_LATE, SIDEFX | WRITES_COV);
+   CASE(FORCE_LATE, FORCE_LATE, SIDEFX | ALPHA2COV);
+}
+
+TEST(EarlyZS, SideFXNoShaderZSAlt)
+{
+   CASE(WEAK_EARLY, FORCE_LATE, ZS_ALWAYS_PASSES | SIDEFX);
+   CASE(WEAK_EARLY, FORCE_LATE, ZS_ALWAYS_PASSES | SIDEFX | DISCARD);
+   CASE(WEAK_EARLY, FORCE_LATE, ZS_ALWAYS_PASSES | SIDEFX | WRITES_COV);
+   CASE(FORCE_LATE, FORCE_LATE, ZS_ALWAYS_PASSES | SIDEFX | ALPHA2COV);
+}
+
+TEST(EarlyZS, NoSideFXNoShaderZS)
+{
+   CASE(FORCE_EARLY, FORCE_EARLY, 0);
+   CASE(FORCE_LATE, FORCE_EARLY, ALPHA2COV | DISCARD | WRITES_COV);
+   CASE(FORCE_EARLY, FORCE_EARLY, ZS_WRITEMASK);
+}
+
+TEST(EarlyZS, ShaderReadZS)
+{
+   CASE_RO_ZS(FORCE_LATE, FORCE_LATE, false, SIDEFX | SHADER_READS_ZS);
+   CASE_RO_ZS(FORCE_EARLY, FORCE_LATE, true,
+              SIDEFX | SHADER_READS_ZS | ARCH_HAS_READONLY_ZS_OPT);
+   CASE_RO_ZS(FORCE_LATE, FORCE_LATE, false,
+              SIDEFX | SHADER_READS_ZS | WRITES_Z | ARCH_HAS_READONLY_ZS_OPT);
+   CASE_RO_ZS(FORCE_EARLY, FORCE_EARLY, true,
+              SHADER_READS_ZS | ARCH_HAS_READONLY_ZS_OPT);
+   CASE_RO_ZS(FORCE_LATE, FORCE_LATE, false,
+              SHADER_READS_ZS | WRITES_Z | ARCH_HAS_READONLY_ZS_OPT);
+   CASE_RO_ZS(FORCE_LATE, WEAK_EARLY, false,
+              SHADER_READS_ZS | ZS_ALWAYS_PASSES);
+   CASE_RO_ZS(FORCE_LATE, FORCE_EARLY, false, SHADER_READS_ZS);
+}
+
+TEST(EarlyZS, NoSideFXNoShaderZSAlt)
+{
+   CASE(WEAK_EARLY, WEAK_EARLY, ZS_ALWAYS_PASSES);
+   CASE(FORCE_EARLY, WEAK_EARLY, ZS_ALWAYS_PASSES | ARCH_HAS_STATE_TRACK_OPT);
+   CASE(FORCE_LATE, WEAK_EARLY,
+        ZS_ALWAYS_PASSES | ALPHA2COV | DISCARD | WRITES_COV);
+   CASE(WEAK_EARLY, WEAK_EARLY, ZS_ALWAYS_PASSES | ZS_WRITEMASK);
+}

src/panfrost/lib/wrap.h

@@ -0,0 +1,57 @@
+/*
+ * Copyright (C) 2017-2019 Lyude Paul
+ * Copyright (C) 2017-2019 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef __PAN_DECODE_PUBLIC_H__
+#define __PAN_DECODE_PUBLIC_H__
+
+#include <inttypes.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+/* Public entrypoints for the tracing infrastructure. This API should be kept
+ * more or less stable. Don't feel bad if you have to change it; just feel
+ * slightly guilty about creating more work for me later. -Alyssa <3
+ *
+ * I'm joking. Mostly. panwrap (out-of-tree) includes this, so update that if
+ * you need to change something here. panwrap is open-source but cannot be
+ * included in-tree.
+ */
+
+// TODO: update panwrap
+
+struct pandecode_context;
+
+struct pandecode_context *pandecode_create_context(bool to_stderr);
+
+void pandecode_next_frame(struct pandecode_context *ctx);
+
+void pandecode_destroy_context(struct pandecode_context *ctx);
+
+void pandecode_inject_mmap(struct pandecode_context *ctx, uint64_t gpu_va,
+                           void *cpu, unsigned sz, const char *name);
+
+void pandecode_inject_free(struct pandecode_context *ctx, uint64_t gpu_va,
+                           unsigned sz);
+
+void pandecode_jc(struct pandecode_context *ctx, uint64_t jc_gpu_va,
+                  uint64_t gpu_id);
+
+void pandecode_interpret_cs(struct pandecode_context *ctx,
+                            uint64_t queue_gpu_va, uint32_t size,
+                            uint64_t gpu_id, uint32_t *regs);
+
+void pandecode_cs_binary(struct pandecode_context *ctx, uint64_t binary_gpu_va,
+                         uint32_t size, uint64_t gpu_id);
+
+void pandecode_cs_trace(struct pandecode_context *ctx, uint64_t trace_gpu_va,
+                        uint32_t size, uint64_t gpu_id);
+
+void pandecode_abort_on_fault(struct pandecode_context *ctx, uint64_t jc_gpu_va,
+                              uint64_t gpu_id);
+
+#endif /* __MMAP_TRACE_H__ */