init

src/freedreno/computerator/README.rst

@@ -0,0 +1,55 @@
+Overview
+========
+
+Computerator is a tool to launch compute shaders, written in assembly.
+The main purpose is to have an easy way to experiment with instructions
+without dealing with the entire compiler stack (which makes controlling
+the order of instructions, the registers chosen, etc, difficult).  The
+choice of compute shaders is simply because there is far less state
+setup required.
+
+Headers
+-------
+
+The shader assembly can be prefixed with headers to control state setup:
+
+* ``@localsize X, Y, Z`` - configures local workgroup size
+* ``@buf SZ (cN.c)`` - configures an SSBO of the specified size (in dwords).
+  The order of the ``@buf`` headers determines the index, ie the first
+  ``@buf`` header is ``g[0]``, the second ``g[1]``, and so on.
+  The iova of the buffer is written as a vec2 to ``cN.c``
+* ``@const(cN.c)`` configures a const vec4 starting at specified
+  const register, ie ``@const(c1.x) 1.0, 2.0, 3.0, 4.0`` will populate
+  ``c1.xyzw`` with ``vec4(1.0, 2.0, 3.0, 4.0)``
+* ``@invocationid(rN.c)`` will populate a vec3 starting at the specified
+  register with the local invocation-id
+* ``@wgid(rN.c)`` will populate a vec3 starting at the specified register
+  with the workgroup-id (must be a high-reg, ie. ``r48.x`` and above)
+* ``@numwg(cN.c)`` will populate a vec3 starting at the specified const
+  register
+
+Example
+-------
+
+```
+@localsize 32, 1, 1
+@buf 32  ; g[0]
+@const(c0.x)  0.0, 0.0, 0.0, 0.0
+@const(c1.x)  1.0, 2.0, 3.0, 4.0
+@wgid(r48.x)        ; r48.xyz
+@invocationid(r0.x) ; r0.xyz
+@numwg(c2.x)        ; c2.xyz
+mov.u32u32 r0.y, r0.x
+(rpt5)nop
+stib.untyped.1d.u32.1 g[0] + r0.y, r0.x
+end
+nop
+```
+
+Usage
+-----
+
+```
+cat myshader.asm | ./computerator --disasm --groups=4,4,4
+```
+

src/freedreno/computerator/a4xx.cc

@@ -0,0 +1,352 @@
+/*
+ * Copyright © 2021 Ilia Mirkin
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "ir3/ir3_compiler.h"
+
+#include "util/u_math.h"
+#include "util/u_queue.h"
+
+#include "common/fd4_hw.h"
+#include "ir3_asm.h"
+#include "main.h"
+
+struct a4xx_backend {
+   struct backend base;
+
+   struct ir3_compiler *compiler;
+   struct fd_device *dev;
+};
+define_cast(backend, a4xx_backend);
+
+/*
+ * Backend implementation:
+ */
+
+static struct kernel *
+a4xx_assemble(struct backend *b, FILE *in)
+{
+   struct a4xx_backend *a4xx_backend = to_a4xx_backend(b);
+   struct ir3_kernel *ir3_kernel = ir3_asm_assemble(a4xx_backend->compiler, in);
+   ir3_kernel->backend = b;
+   return &ir3_kernel->base;
+}
+
+static void
+a4xx_disassemble(struct kernel *kernel, struct ir3_disasm_options *options)
+{
+   ir3_asm_disassemble(to_ir3_kernel(kernel), options);
+}
+
+static void
+cs_program_emit(struct fd_ringbuffer *ring, struct kernel *kernel)
+{
+   struct ir3_kernel *ir3_kernel = to_ir3_kernel(kernel);
+   struct ir3_shader_variant *v = ir3_kernel->v;
+   const struct ir3_info *i = &v->info;
+   enum a3xx_threadsize thrsz = i->double_threadsize ? FOUR_QUADS : TWO_QUADS;
+
+   OUT_PKT0(ring, REG_A4XX_UCHE_INVALIDATE0, 2);
+   OUT_RING(ring, 0x00000000);
+   OUT_RING(ring, 0x00000012);
+
+   OUT_WFI(ring);
+
+   OUT_PKT0(ring, REG_A4XX_SP_MODE_CONTROL, 1);
+   OUT_RING(ring, 0x0000001e);
+
+   OUT_PKT0(ring, REG_A4XX_TPL1_TP_MODE_CONTROL, 1);
+   OUT_RING(ring, 0x00000038);
+
+   OUT_PKT0(ring, REG_A4XX_TPL1_TP_FS_TEX_COUNT, 1);
+   OUT_RING(ring, 0x00000000);
+
+   OUT_WFI(ring);
+
+   OUT_PKT0(ring, REG_A4XX_HLSQ_MODE_CONTROL, 1);
+   OUT_RING(ring, 0x00000003);
+
+   OUT_PKT0(ring, REG_A4XX_HLSQ_CONTROL_0_REG, 1);
+   OUT_RING(ring, 0x080005f0);
+
+   OUT_PKT0(ring, REG_A4XX_HLSQ_UPDATE_CONTROL, 1);
+   OUT_RING(ring, 0x00000038);
+
+   OUT_PKT0(ring, REG_A4XX_SP_SP_CTRL_REG, 1);
+   OUT_RING(ring, 0x00860010);
+   // OUT_RING(ring, 0x00920000);
+
+   OUT_PKT0(ring, REG_A4XX_SP_INSTR_CACHE_CTRL, 1);
+   OUT_RING(ring, 0x000004ff);
+   // OUT_RING(ring, 0x00000260);
+
+   OUT_PKT0(ring, REG_A4XX_SP_FS_CTRL_REG1, 1);
+   OUT_RING(ring, 0x80000000);
+
+   OUT_PKT0(ring, REG_A4XX_SP_CS_CTRL_REG0, 1);
+   OUT_RING(ring,
+            A4XX_SP_CS_CTRL_REG0_THREADSIZE(thrsz) |
+            A4XX_SP_CS_CTRL_REG0_SUPERTHREADMODE |
+            A4XX_SP_CS_CTRL_REG0_HALFREGFOOTPRINT(i->max_half_reg + 1) |
+            A4XX_SP_CS_CTRL_REG0_FULLREGFOOTPRINT(i->max_reg + 1));
+
+   OUT_PKT0(ring, REG_A4XX_HLSQ_CS_CONTROL_REG, 1);
+   OUT_RING(ring, A4XX_HLSQ_CS_CONTROL_REG_CONSTOBJECTOFFSET(0) |
+                     A4XX_HLSQ_CS_CONTROL_REG_SHADEROBJOFFSET(0) |
+                     A4XX_HLSQ_CS_CONTROL_REG_ENABLED |
+                     A4XX_HLSQ_CS_CONTROL_REG_INSTRLENGTH(1) |
+                     COND(v->has_ssbo, A4XX_HLSQ_CS_CONTROL_REG_SSBO_ENABLE) |
+                     A4XX_HLSQ_CS_CONTROL_REG_CONSTLENGTH(v->constlen / 4));
+
+   OUT_PKT0(ring, REG_A4XX_SP_CS_OBJ_START, 1);
+   OUT_RELOC(ring, v->bo, 0, 0, 0); /* SP_CS_OBJ_START */
+
+   OUT_PKT0(ring, REG_A4XX_SP_CS_LENGTH_REG, 1);
+   OUT_RING(ring, v->instrlen);
+
+   uint32_t local_invocation_id, work_group_id, num_wg_id;
+   local_invocation_id =
+      ir3_find_sysval_regid(v, SYSTEM_VALUE_LOCAL_INVOCATION_ID);
+   work_group_id = ir3_kernel->info.wgid;
+   num_wg_id = ir3_kernel->info.numwg;
+
+   OUT_PKT0(ring, REG_A4XX_HLSQ_CL_CONTROL_0, 2);
+   OUT_RING(ring, A4XX_HLSQ_CL_CONTROL_0_WGIDCONSTID(work_group_id) |
+                     A4XX_HLSQ_CL_CONTROL_0_KERNELDIMCONSTID(regid(63, 0)) |
+                     A4XX_HLSQ_CL_CONTROL_0_LOCALIDREGID(local_invocation_id));
+   OUT_RING(ring, A4XX_HLSQ_CL_CONTROL_1_UNK0CONSTID(regid(63, 0)) |
+                     A4XX_HLSQ_CL_CONTROL_1_WORKGROUPSIZECONSTID(regid(63, 0)));
+
+   OUT_PKT0(ring, REG_A4XX_HLSQ_CL_KERNEL_CONST, 1);
+   OUT_RING(ring, A4XX_HLSQ_CL_KERNEL_CONST_UNK0CONSTID(regid(63, 0)) |
+                     A4XX_HLSQ_CL_KERNEL_CONST_NUMWGCONSTID(num_wg_id));
+
+   OUT_PKT0(ring, REG_A4XX_HLSQ_CL_WG_OFFSET, 1);
+   OUT_RING(ring, A4XX_HLSQ_CL_WG_OFFSET_UNK0CONSTID(regid(63, 0)));
+
+   OUT_PKT3(ring, CP_LOAD_STATE4, 2);
+   OUT_RING(ring, CP_LOAD_STATE4_0_DST_OFF(0) |
+                     CP_LOAD_STATE4_0_STATE_SRC(SS4_INDIRECT) |
+                     CP_LOAD_STATE4_0_STATE_BLOCK(SB4_CS_SHADER) |
+                     CP_LOAD_STATE4_0_NUM_UNIT(v->instrlen));
+   OUT_RELOC(ring, v->bo, 0, CP_LOAD_STATE4_1_STATE_TYPE(ST4_SHADER), 0);
+}
+
+static void
+emit_const(struct fd_ringbuffer *ring, struct kernel *kernel, uint32_t constid, uint32_t sizedwords,
+           const uint32_t *dwords)
+{
+   uint32_t align_sz;
+
+   assert((constid % 4) == 0);
+
+   /* Overwrite appropriate entries with buffer addresses */
+   struct fd_bo **replacements =
+      (struct fd_bo **)calloc(sizedwords, sizeof(struct fd_bo *));
+   for (int i = 0; i < MAX_BUFS; i++) {
+      if (kernel->buf_addr_regs[i] != INVALID_REG) {
+         int idx = kernel->buf_addr_regs[i];
+         assert(idx < sizedwords);
+
+         replacements[idx] = kernel->bufs[i];
+      }
+   }
+
+   align_sz = align(sizedwords, 4);
+
+   OUT_PKT3(ring, CP_LOAD_STATE4, 2 + align_sz);
+   OUT_RING(ring, CP_LOAD_STATE4_0_DST_OFF(constid / 4) |
+                     CP_LOAD_STATE4_0_STATE_SRC(SS4_DIRECT) |
+                     CP_LOAD_STATE4_0_STATE_BLOCK(SB4_CS_SHADER) |
+                     CP_LOAD_STATE4_0_NUM_UNIT(DIV_ROUND_UP(sizedwords, 4)));
+   OUT_RING(ring, CP_LOAD_STATE4_1_EXT_SRC_ADDR(0) |
+                     CP_LOAD_STATE4_1_STATE_TYPE(ST4_CONSTANTS));
+   for (unsigned i = 0; i < sizedwords; i++) {
+      if (replacements[i])
+         OUT_RELOC(ring, replacements[i], 0, 0, 0);
+      else
+         OUT_RING(ring, dwords[i]);
+   }
+
+   /* Zero-pad to multiple of 4 dwords */
+   for (uint32_t i = sizedwords; i < align_sz; i++) {
+      OUT_RING(ring, 0);
+   }
+
+   free(replacements);
+}
+
+static void
+cs_const_emit(struct fd_ringbuffer *ring, struct kernel *kernel,
+              uint32_t grid[3])
+{
+   struct ir3_kernel *ir3_kernel = to_ir3_kernel(kernel);
+   struct ir3_shader_variant *v = ir3_kernel->v;
+
+   const struct ir3_const_state *const_state = ir3_const_state(v);
+   uint32_t base = const_state->allocs.max_const_offset_vec4;
+   const struct ir3_imm_const_state *imm_state = &v->imm_state;
+   int size = DIV_ROUND_UP(imm_state->count, 4);
+
+   /* truncate size to avoid writing constants that shader
+    * does not use:
+    */
+   size = MIN2(size + base, v->constlen) - base;
+
+   /* convert out of vec4: */
+   base *= 4;
+   size *= 4;
+
+   if (size > 0) {
+      emit_const(ring, kernel, base, size, imm_state->values);
+   }
+}
+
+static void
+cs_ibo_emit(struct fd_ringbuffer *ring, struct fd_submit *submit,
+            struct kernel *kernel)
+{
+   OUT_PKT3(ring, CP_LOAD_STATE4, 2 + (4 * kernel->num_bufs));
+   OUT_RING(ring, CP_LOAD_STATE4_0_DST_OFF(0) |
+         CP_LOAD_STATE4_0_STATE_SRC(SS4_DIRECT) |
+         CP_LOAD_STATE4_0_STATE_BLOCK(SB4_CS_SSBO) |
+         CP_LOAD_STATE4_0_NUM_UNIT(kernel->num_bufs));
+   OUT_RING(ring, CP_LOAD_STATE4_1_STATE_TYPE(ST4_SHADER) |
+         CP_LOAD_STATE4_1_EXT_SRC_ADDR(0));
+   for (unsigned i = 0; i < kernel->num_bufs; i++) {
+      OUT_RELOC(ring, kernel->bufs[i], 0, 0, 0);
+#if 1
+      OUT_RING(ring, 0);
+      OUT_RING(ring, 0);
+      OUT_RING(ring, 0);
+#else
+      OUT_RING(ring, kernel->buf_sizes[i]);
+      OUT_RING(ring, kernel->buf_sizes[i]);
+      OUT_RING(ring, 0x00000004);
+#endif
+   }
+
+   OUT_PKT3(ring, CP_LOAD_STATE4, 2 + (2 * kernel->num_bufs));
+   OUT_RING(ring, CP_LOAD_STATE4_0_DST_OFF(0) |
+         CP_LOAD_STATE4_0_STATE_SRC(SS4_DIRECT) |
+         CP_LOAD_STATE4_0_STATE_BLOCK(SB4_CS_SSBO) |
+         CP_LOAD_STATE4_0_NUM_UNIT(kernel->num_bufs));
+   OUT_RING(ring, CP_LOAD_STATE4_1_STATE_TYPE(ST4_CONSTANTS) |
+         CP_LOAD_STATE4_1_EXT_SRC_ADDR(0));
+   for (unsigned i = 0; i < kernel->num_bufs; i++) {
+      unsigned sz = kernel->buf_sizes[i];
+
+      /* width is in dwords, overflows into height: */
+      sz /= 4;
+
+#if 1
+      OUT_RING(ring, A4XX_SSBO_1_0_WIDTH(sz));
+      OUT_RING(ring, A4XX_SSBO_1_1_HEIGHT(sz >> 16));
+#else
+      OUT_RING(ring, A4XX_SSBO_1_0_WIDTH(sz) |
+            A4XX_SSBO_1_0_FMT(RB4_R32_UINT) |
+            A4XX_SSBO_1_0_CPP(4));
+      OUT_RING(ring, A4XX_SSBO_1_1_HEIGHT(DIV_ROUND_UP(sz, 1 << 16)) |
+            A4XX_SSBO_1_1_DEPTH(1));
+#endif
+   }
+}
+
+static void
+a4xx_emit_grid(struct kernel *kernel, uint32_t grid[3],
+               struct fd_submit *submit)
+{
+   struct fd_ringbuffer *ring = fd_submit_new_ringbuffer(
+      submit, 0,
+      (enum fd_ringbuffer_flags)(FD_RINGBUFFER_PRIMARY | FD_RINGBUFFER_GROWABLE));
+
+   cs_program_emit(ring, kernel);
+   cs_const_emit(ring, kernel, grid);
+   cs_ibo_emit(ring, submit, kernel);
+
+   const unsigned *local_size = kernel->local_size;
+   const unsigned *num_groups = grid;
+
+   unsigned work_dim = 0;
+   for (int i = 0; i < 3; i++) {
+      if (!grid[i])
+         break;
+      work_dim++;
+   }
+
+   OUT_PKT0(ring, REG_A4XX_HLSQ_CL_NDRANGE_0, 7);
+   OUT_RING(ring, A4XX_HLSQ_CL_NDRANGE_0_KERNELDIM(work_dim) |
+                     A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEX(local_size[0] - 1) |
+                     A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEY(local_size[1] - 1) |
+                     A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEZ(local_size[2] - 1));
+   OUT_RING(ring,
+            A4XX_HLSQ_CL_NDRANGE_1_SIZE_X(local_size[0] * num_groups[0]));
+   OUT_RING(ring, 0); /* HLSQ_CL_NDRANGE_2_GLOBALOFF_X */
+   OUT_RING(ring,
+            A4XX_HLSQ_CL_NDRANGE_3_SIZE_Y(local_size[1] * num_groups[1]));
+   OUT_RING(ring, 0); /* HLSQ_CL_NDRANGE_4_GLOBALOFF_Y */
+   OUT_RING(ring,
+            A4XX_HLSQ_CL_NDRANGE_5_SIZE_Z(local_size[2] * num_groups[2]));
+   OUT_RING(ring, 0); /* HLSQ_CL_NDRANGE_6_GLOBALOFF_Z */
+
+#if 1
+   OUT_PKT3(ring, CP_EXEC_CS, 4);
+   OUT_RING(ring, 0x00000000);
+   OUT_RING(ring, CP_EXEC_CS_1_NGROUPS_X(grid[0]));
+   OUT_RING(ring, CP_EXEC_CS_2_NGROUPS_Y(grid[1]));
+   OUT_RING(ring, CP_EXEC_CS_3_NGROUPS_Z(grid[2]));
+#else
+   OUT_PKT0(ring, REG_A4XX_HLSQ_CL_KERNEL_GROUP_X, 3);
+   OUT_RING(ring, grid[0]); /* HLSQ_CL_KERNEL_GROUP_X */
+   OUT_RING(ring, grid[1]); /* HLSQ_CL_KERNEL_GROUP_Y */
+   OUT_RING(ring, grid[2]); /* HLSQ_CL_KERNEL_GROUP_Z */
+
+   OUT_PKT3(ring, CP_RUN_OPENCL, 1);
+   OUT_RING(ring, 0);
+#endif
+
+   OUT_WFI(ring);
+
+   /* TODO: cache_flush */
+}
+
+struct backend *
+a4xx_init(struct fd_device *dev, const struct fd_dev_id *dev_id)
+{
+   struct a4xx_backend *a4xx_backend =
+      (struct a4xx_backend *)calloc(1, sizeof(*a4xx_backend));
+
+   a4xx_backend->base = (struct backend){
+      .assemble = a4xx_assemble,
+      .disassemble = a4xx_disassemble,
+      .emit_grid = a4xx_emit_grid,
+   };
+
+   struct ir3_compiler_options compiler_options = {
+      .disable_cache = true,
+   };
+   a4xx_backend->compiler =
+      ir3_compiler_create(dev, dev_id, fd_dev_info_raw(dev_id), &compiler_options);
+   a4xx_backend->dev = dev;
+
+   return &a4xx_backend->base;
+}

src/freedreno/computerator/a6xx.cc

@@ -0,0 +1,720 @@
+/*
+ * Copyright © 2020 Google, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "ir3/ir3_compiler.h"
+
+#include "util/u_math.h"
+
+#include "fd6_hw.h"
+
+#include "common/freedreno_dev_info.h"
+#include "fdl/freedreno_layout.h"
+
+#include "ir3_asm.h"
+#include "main.h"
+
+#define FD_BO_NO_HARDPIN 1
+#include "common/fd6_pack.h"
+
+struct a6xx_backend {
+   struct backend base;
+
+   struct ir3_compiler *compiler;
+   struct fd_device *dev;
+
+   const struct fd_dev_info *info;
+
+   unsigned seqno;
+   struct fd_bo *control_mem;
+
+   struct fd_bo *query_mem;
+   const struct perfcntr *perfcntrs;
+   unsigned num_perfcntrs;
+};
+define_cast(backend, a6xx_backend);
+
+/*
+ * Data structures shared with GPU:
+ */
+
+/* This struct defines the layout of the fd6_context::control buffer: */
+struct fd6_control {
+   uint32_t seqno; /* seqno for async CP_EVENT_WRITE, etc */
+   uint32_t _pad0;
+   volatile uint32_t vsc_overflow;
+   uint32_t _pad1;
+   /* flag set from cmdstream when VSC overflow detected: */
+   uint32_t vsc_scratch;
+   uint32_t _pad2;
+   uint32_t _pad3;
+   uint32_t _pad4;
+
+   /* scratch space for VPC_SO[i].FLUSH_BASE_LO/HI, start on 32 byte boundary. */
+   struct {
+      uint32_t offset;
+      uint32_t pad[7];
+   } flush_base[4];
+};
+
+#define control_ptr(a6xx_backend, member)                                      \
+   (a6xx_backend)->control_mem, offsetof(struct fd6_control, member)
+
+struct PACKED fd6_query_sample {
+   uint64_t start;
+   uint64_t result;
+   uint64_t stop;
+};
+
+/* offset of a single field of an array of fd6_query_sample: */
+#define query_sample_idx(a6xx_backend, idx, field)                             \
+   (a6xx_backend)->query_mem,                                                  \
+      (idx * sizeof(struct fd6_query_sample)) +                                \
+         offsetof(struct fd6_query_sample, field)
+
+/*
+ * Backend implementation:
+ */
+
+static struct kernel *
+a6xx_assemble(struct backend *b, FILE *in)
+{
+   struct a6xx_backend *a6xx_backend = to_a6xx_backend(b);
+   struct ir3_kernel *ir3_kernel = ir3_asm_assemble(a6xx_backend->compiler, in);
+   ir3_kernel->backend = b;
+   return &ir3_kernel->base;
+}
+
+static void
+a6xx_disassemble(struct kernel *kernel, struct ir3_disasm_options *options)
+{
+   ir3_asm_disassemble(to_ir3_kernel(kernel), options);
+}
+
+template<chip CHIP>
+static void
+cs_restore_emit(fd_cs &cs, struct a6xx_backend *a6xx_backend)
+{
+   fd_ncrb<CHIP> ncrb(cs, 2 + ARRAY_SIZE(a6xx_backend->info->magic_raw));
+
+   ncrb.add(A6XX_SP_PERFCTR_SHADER_MASK(.cs = true));
+
+   /* KMD programs this (and blocks UMD access) on gen8+: */
+   if (CHIP < A8XX)
+      ncrb.add(A6XX_SP_NC_MODE_CNTL_2());
+
+   for (size_t i = 0; i < ARRAY_SIZE(a6xx_backend->info->magic_raw); i++) {
+      auto magic_reg = a6xx_backend->info->magic_raw[i];
+      if (!magic_reg.reg)
+         break;
+
+      ncrb.add({magic_reg.reg, magic_reg.value});
+   }
+}
+
+template<chip CHIP>
+static void
+cs_program_emit_regs(fd_cs &cs, struct kernel *kernel)
+{
+   struct ir3_kernel *ir3_kernel = to_ir3_kernel(kernel);
+   struct a6xx_backend *a6xx_backend = to_a6xx_backend(ir3_kernel->backend);
+   struct ir3_shader_variant *v = ir3_kernel->v;
+   const unsigned *local_size = kernel->local_size;
+   const struct ir3_info *i = &v->info;
+   enum a6xx_threadsize thrsz = i->double_threadsize ? THREAD128 : THREAD64;
+   fd_crb crb(cs, 25);
+
+   crb.add(A6XX_SP_MODE_CNTL(
+      .constant_demotion_enable = true,
+      .isammode = ISAMMODE_GL,
+      .shared_consts_enable = false,
+   ));
+
+   crb.add(SP_UPDATE_CNTL(CHIP,
+      .vs_state = true,
+      .hs_state = true,
+      .ds_state = true,
+      .gs_state = true,
+      .fs_state = true,
+      .cs_state = true,
+      .gfx_uav = true,
+   ));
+
+   unsigned constlen = align(v->constlen, 4);
+   crb.add(SP_CS_CONST_CONFIG(CHIP, .constlen = constlen, .enabled = true, ));
+
+   crb.add(A6XX_SP_CS_CONFIG(
+      .enabled = true,
+      .ntex = v->num_samp,
+      .nsamp = v->num_samp,
+      .nuav = kernel->num_bufs,
+   ));
+   crb.add(A6XX_SP_CS_INSTR_SIZE(v->instrlen));
+
+   crb.add(A6XX_SP_CS_CNTL_0(
+      .halfregfootprint = i->max_half_reg + 1,
+      .fullregfootprint = i->max_reg + 1,
+      .branchstack = ir3_shader_branchstack_hw(v),
+      .threadsize = thrsz,
+      .earlypreamble = v->early_preamble,
+      .mergedregs = v->mergedregs,
+   ));
+
+   if (CHIP >= A7XX) {
+      crb.add(SP_PS_WAVE_CNTL(CHIP, .threadsize = THREAD64));
+
+      crb.add(SP_REG_PROG_ID_0(CHIP, .dword = 0xfcfcfcfc));
+      crb.add(SP_REG_PROG_ID_1(CHIP, .dword = 0xfcfcfcfc));
+      crb.add(SP_REG_PROG_ID_2(CHIP, .dword = 0xfcfcfcfc));
+      crb.add(SP_REG_PROG_ID_3(CHIP, .dword = 0x0000fc00));
+   }
+
+   uint32_t shared_size = MAX2(((int)v->shared_size - 1) / 1024, 1);
+   enum a6xx_const_ram_mode mode =
+      v->constlen > 256 ? CONSTLEN_512 :
+      (v->constlen > 192 ? CONSTLEN_256 :
+      (v->constlen > 128 ? CONSTLEN_192 : CONSTLEN_128));
+   crb.add(A6XX_SP_CS_CNTL_1(.shared_size = shared_size, .constantrammode = mode));
+
+   if (CHIP == A6XX && a6xx_backend->info->props.has_lpac) {
+      crb.add(HLSQ_CS_CTRL_REG1(CHIP, .shared_size = 1, .constantrammode = mode));
+   }
+
+   uint32_t local_invocation_id, work_group_id;
+   local_invocation_id =
+      ir3_find_sysval_regid(v, SYSTEM_VALUE_LOCAL_INVOCATION_ID);
+   work_group_id = ir3_find_sysval_regid(v, SYSTEM_VALUE_WORKGROUP_ID);
+
+   if (CHIP == A6XX) {
+      crb.add(SP_CS_CONST_CONFIG_0(CHIP,
+         .wgidconstid = work_group_id,
+         .wgsizeconstid = INVALID_REG,
+         .wgoffsetconstid = INVALID_REG,
+         .localidregid = local_invocation_id,
+      ));
+      crb.add(SP_CS_WGE_CNTL(CHIP,
+         .linearlocalidregid = INVALID_REG,
+         .threadsize = thrsz,
+      ));
+   } else {
+      unsigned tile_height = (local_size[1] % 8 == 0)   ? 3
+                             : (local_size[1] % 4 == 0) ? 5
+                             : (local_size[1] % 2 == 0) ? 9
+                                                        : 17;
+
+      crb.add(SP_CS_WGE_CNTL(CHIP,
+         .linearlocalidregid = INVALID_REG,
+         .threadsize = thrsz,
+         .workgrouprastorderzfirsten = true,
+         .wgtilewidth = 4,
+         .wgtileheight = tile_height,
+      ));
+   }
+
+   if (CHIP >= A7XX || a6xx_backend->info->props.has_lpac) {
+      crb.add(A6XX_SP_CS_WIE_CNTL_0(
+         .wgidconstid = work_group_id,
+         .wgsizeconstid = INVALID_REG,
+         .wgoffsetconstid = INVALID_REG,
+         .localidregid = local_invocation_id,
+      ));
+
+      if (CHIP >= A7XX) {
+         /* TODO allow the shader to control the tiling */
+         crb.add(SP_CS_WIE_CNTL_1(CHIP,
+            .linearlocalidregid = INVALID_REG,
+            .threadsize = thrsz,
+            .workitemrastorder = WORKITEMRASTORDER_LINEAR,
+         ));
+      } else {
+         crb.add(SP_CS_WIE_CNTL_1(CHIP,
+            .linearlocalidregid = INVALID_REG,
+            .threadsize = thrsz,
+         ));
+      }
+   }
+
+   crb.attach_bo(v->bo);
+
+   crb.add(A6XX_SP_CS_BASE(v->bo));
+   crb.add(A6XX_SP_CS_INSTR_SIZE(v->instrlen));
+
+   if (v->pvtmem_size > 0) {
+      uint32_t per_fiber_size = v->pvtmem_size;
+      uint32_t per_sp_size =
+         align(per_fiber_size * a6xx_backend->info->fibers_per_sp, 1 << 12);
+      uint32_t total_size = per_sp_size * a6xx_backend->info->num_sp_cores;
+
+      struct fd_bo *pvtmem = fd_bo_new(a6xx_backend->dev, total_size, 0, "pvtmem");
+      crb.add(A6XX_SP_CS_PVT_MEM_PARAM(.memsizeperitem = per_fiber_size));
+      crb.add(A6XX_SP_CS_PVT_MEM_BASE(pvtmem));
+      crb.add(A6XX_SP_CS_PVT_MEM_SIZE(
+         .totalpvtmemsize = per_sp_size,
+         .perwavememlayout = v->pvtmem_per_wave,
+      ));
+
+      crb.add(A6XX_SP_CS_PVT_MEM_STACK_OFFSET(.offset = per_sp_size));
+   }
+}
+
+template<chip CHIP>
+static void
+cs_program_emit(fd_cs &cs, struct kernel *kernel)
+{
+   struct ir3_kernel *ir3_kernel = to_ir3_kernel(kernel);
+   struct a6xx_backend *a6xx_backend = to_a6xx_backend(ir3_kernel->backend);
+   struct ir3_shader_variant *v = ir3_kernel->v;
+
+   cs_program_emit_regs<CHIP>(cs, kernel);
+
+   uint32_t shader_preload_size =
+      MIN2(v->instrlen, a6xx_backend->info->props.instr_cache_size);
+
+   fd_pkt7(cs, CP_LOAD_STATE6_FRAG, 3)
+      .add(CP_LOAD_STATE6_0(
+         .state_type = ST6_SHADER,
+         .state_src = SS6_INDIRECT,
+         .state_block = SB6_CS_SHADER,
+         .num_unit = shader_preload_size,
+      ))
+      .add(CP_LOAD_STATE6_EXT_SRC_ADDR(v->bo));
+}
+
+template<chip CHIP>
+static void
+emit_const(fd_cs &cs, uint32_t regid, uint32_t sizedwords, const uint32_t *dwords)
+{
+   uint32_t zero[4] = {};
+   uint32_t align_sz;
+
+   assert((regid % 4) == 0);
+
+   align_sz = align(sizedwords, 4);
+
+   fd_pkt7(cs, CP_LOAD_STATE6_FRAG, 3 + align_sz)
+      .add(CP_LOAD_STATE6_0(
+         .dst_off = regid / 4,
+         .state_type = ST6_CONSTANTS,
+         .state_src = SS6_DIRECT,
+         .state_block = SB6_CS_SHADER,
+         .num_unit = DIV_ROUND_UP(sizedwords, 4)
+      ))
+      .add(CP_LOAD_STATE6_EXT_SRC_ADDR())
+      .add(dwords, sizedwords)
+      /* Zero-pad to multiple of 4 dwords */
+      .add(zero, align_sz - sizedwords);
+}
+
+template<chip CHIP>
+static void
+cs_const_emit(fd_cs &cs, struct kernel *kernel, uint32_t grid[3])
+{
+   struct ir3_kernel *ir3_kernel = to_ir3_kernel(kernel);
+   struct ir3_shader_variant *v = ir3_kernel->v;
+
+   const struct ir3_const_state *const_state = ir3_const_state(v);
+   uint32_t base = const_state->allocs.max_const_offset_vec4;
+   const struct ir3_imm_const_state *imm_state = &v->imm_state;
+   int size = DIV_ROUND_UP(imm_state->count, 4);
+
+   if (ir3_kernel->info.numwg != INVALID_REG) {
+      assert((ir3_kernel->info.numwg & 0x3) == 0);
+      int idx = ir3_kernel->info.numwg >> 2;
+      imm_state->values[idx * 4 + 0] = grid[0];
+      imm_state->values[idx * 4 + 1] = grid[1];
+      imm_state->values[idx * 4 + 2] = grid[2];
+   }
+
+   for (int i = 0; i < MAX_BUFS; i++) {
+      if (kernel->buf_addr_regs[i] != INVALID_REG) {
+         assert((kernel->buf_addr_regs[i] & 0x3) == 0);
+         int idx = kernel->buf_addr_regs[i] >> 2;
+
+         uint64_t iova = fd_bo_get_iova(kernel->bufs[i]);
+
+         imm_state->values[idx * 4 + 1] = iova >> 32;
+         imm_state->values[idx * 4 + 0] = (iova << 32) >> 32;
+      }
+   }
+
+   /* truncate size to avoid writing constants that shader
+    * does not use:
+    */
+   size = MIN2(size + base, v->constlen) - base;
+
+   /* convert out of vec4: */
+   base *= 4;
+   size *= 4;
+
+   if (size > 0) {
+      emit_const<CHIP>(cs, base, size, imm_state->values);
+   }
+}
+
+static unsigned
+kernel_num_bufs(struct kernel *kernel, enum kernel_buf_type buf_type)
+{
+   unsigned num_bufs = 0;
+
+   for (unsigned i = 0; i < kernel->num_bufs; i++) {
+      if (kernel->buf_types[i] == buf_type) {
+         num_bufs++;
+      }
+   }
+
+   return num_bufs;
+}
+
+template<chip CHIP>
+static void
+cs_uav_emit(fd_cs &cs, struct fd_device *dev, struct kernel *kernel)
+{
+   unsigned num_bufs = kernel_num_bufs(kernel, KERNEL_BUF_UAV);
+
+   if (num_bufs == 0) {
+      return;
+   }
+
+   struct fd_bo *state = fd_bo_new(dev, kernel->num_bufs * 16 * 4,
+                                   FD_BO_GPUREADONLY | FD_BO_HINT_COMMAND,
+                                   "tex_desc");
+   fd_bo_mark_for_dump(state);
+
+   cs.attach_bo(state);
+
+   uint32_t *buf = (uint32_t *)fd_bo_map(state);
+
+   for (unsigned i = 0; i < kernel->num_bufs; i++) {
+      if (kernel->buf_types[i] != KERNEL_BUF_UAV) {
+         continue;
+      }
+
+      cs.attach_bo(kernel->bufs[i]);
+
+      uint64_t iova = fd_bo_get_iova(kernel->bufs[i]);
+      uint32_t descriptor[16];
+
+      static const uint8_t swiz_identity[4] = {
+         PIPE_SWIZZLE_X, PIPE_SWIZZLE_Y,
+         PIPE_SWIZZLE_Z, PIPE_SWIZZLE_W,
+      };
+
+      fdl6_buffer_view_init<CHIP>(descriptor, PIPE_FORMAT_R32_UINT, swiz_identity,
+                                  iova, kernel->buf_sizes[i] * 4);
+
+      memcpy(buf, descriptor, 16 * 4);
+      buf += 16;
+   }
+
+   fd_pkt7(cs, CP_LOAD_STATE6_FRAG, 3)
+      .add(CP_LOAD_STATE6_0(
+         .state_type = ST6_UAV,
+         .state_src = SS6_INDIRECT,
+         .state_block = SB6_CS_SHADER,
+         .num_unit = num_bufs,
+      ))
+      .add(CP_LOAD_STATE6_EXT_SRC_ADDR(state));
+
+   fd_crb crb(cs, 3);
+
+   crb.add(SP_CS_UAV_BASE(CHIP, state));
+   crb.add(A6XX_SP_CS_USIZE(num_bufs));
+
+   fd_bo_del(state);
+}
+
+static void
+cs_ubo_emit(fd_cs &cs, struct kernel *kernel)
+{
+   unsigned num_bufs = kernel_num_bufs(kernel, KERNEL_BUF_UBO);
+
+   if (num_bufs == 0) {
+      return;
+   }
+
+   for (unsigned i = 0, offset = 0; i < kernel->num_bufs; i++) {
+      if (kernel->buf_types[i] != KERNEL_BUF_UBO) {
+         continue;
+      }
+
+      cs.attach_bo(kernel->bufs[i]);
+
+      unsigned size_vec4s = DIV_ROUND_UP(kernel->buf_sizes[i], 4);
+
+      fd_pkt7(cs, CP_LOAD_STATE6_FRAG, 5)
+         .add(CP_LOAD_STATE6_0(
+            .dst_off = offset,
+            .state_type = ST6_UBO,
+            .state_src = SS6_DIRECT,
+            .state_block = SB6_CS_SHADER,
+            .num_unit = 1,
+         ))
+         .add(CP_LOAD_STATE6_EXT_SRC_ADDR())
+         .add(A6XX_UBO_DESC(0, kernel->bufs[i], 0, size_vec4s));
+
+      offset++;
+   }
+}
+
+template<chip CHIP>
+static inline unsigned
+event_write(fd_cs &cs, struct kernel *kernel, enum vgt_event_type evt, bool timestamp)
+{
+   unsigned seqno = 0;
+   unsigned len = timestamp ? 4 : 1;
+
+   fd_pkt7 pkt(cs, CP_EVENT_WRITE, len);
+
+   if (CHIP == A6XX) {
+      pkt.add(CP_EVENT_WRITE_0_EVENT(evt));
+   } else {
+      pkt.add(CP_EVENT_WRITE7_0_EVENT(evt) |
+              CP_EVENT_WRITE7_0_WRITE_SRC(EV_WRITE_USER_32B) |
+              COND(timestamp, CP_EVENT_WRITE7_0_WRITE_ENABLED));
+   }
+
+   if (timestamp) {
+      struct ir3_kernel *ir3_kernel = to_ir3_kernel(kernel);
+      struct a6xx_backend *a6xx_backend = to_a6xx_backend(ir3_kernel->backend);
+      seqno = ++a6xx_backend->seqno;
+      pkt.add(CP_EVENT_WRITE_ADDR(control_ptr(a6xx_backend, seqno)));
+      pkt.add(seqno);
+   }
+
+   return seqno;
+}
+
+template<chip CHIP>
+static inline void
+cache_flush(fd_cs &cs, struct kernel *kernel)
+{
+   struct ir3_kernel *ir3_kernel = to_ir3_kernel(kernel);
+   struct a6xx_backend *a6xx_backend = to_a6xx_backend(ir3_kernel->backend);
+   unsigned seqno;
+
+   seqno = event_write<CHIP>(cs, kernel, RB_DONE_TS, true);
+
+   fd_pkt7(cs, CP_WAIT_REG_MEM, 6)
+      .add(CP_WAIT_REG_MEM_0(.function = WRITE_EQ, .poll = POLL_MEMORY))
+      .add(CP_WAIT_REG_MEM_POLL_ADDR(control_ptr(a6xx_backend, seqno)))
+      .add(CP_WAIT_REG_MEM_3(.ref = seqno))
+      .add(CP_WAIT_REG_MEM_4(.mask = ~0))
+      .add(CP_WAIT_REG_MEM_5(.delay_loop_cycles = 16));
+
+   if (CHIP == A6XX) {
+      seqno = event_write<CHIP>(cs, kernel, CACHE_FLUSH_TS, true);
+
+      fd_pkt7(cs, CP_WAIT_MEM_GTE, 4)
+         .add(CP_WAIT_MEM_GTE_0())
+         .add(CP_WAIT_MEM_GTE_POLL_ADDR(control_ptr(a6xx_backend, seqno)))
+         .add(CP_WAIT_MEM_GTE_3(.ref = seqno));
+   } else {
+      event_write<CHIP>(cs, kernel, CACHE_FLUSH7, false);
+   }
+}
+
+template<chip CHIP>
+static void
+a6xx_emit_grid(struct kernel *kernel, uint32_t grid[3],
+               struct fd_submit *submit)
+{
+   struct ir3_kernel *ir3_kernel = to_ir3_kernel(kernel);
+   struct a6xx_backend *a6xx_backend = to_a6xx_backend(ir3_kernel->backend);
+   fd_cs cs(fd_submit_new_ringbuffer(submit, 0,
+      (enum fd_ringbuffer_flags)(FD_RINGBUFFER_PRIMARY | FD_RINGBUFFER_GROWABLE)));
+
+   cs.attach_bo(a6xx_backend->control_mem);
+
+   cs_restore_emit<CHIP>(cs, a6xx_backend);
+   cs_program_emit<CHIP>(cs, kernel);
+   cs_const_emit<CHIP>(cs, kernel, grid);
+   cs_uav_emit<CHIP>(cs, a6xx_backend->dev, kernel);
+   cs_ubo_emit(cs, kernel);
+
+   if (CHIP >= A8XX) {
+      fd_pkt7(cs, CP_SET_MARKER, 1)
+         .add(A8XX_CP_SET_MARKER_0(.mode = RM6_COMPUTE));
+   } else{
+      fd_pkt7(cs, CP_SET_MARKER, 1)
+         .add(A6XX_CP_SET_MARKER_0(.mode = RM6_COMPUTE));
+   }
+
+   const unsigned *local_size = kernel->local_size;
+   const unsigned *num_groups = grid;
+
+   unsigned work_dim = 0;
+   for (int i = 0; i < 3; i++) {
+      if (!grid[i])
+         break;
+      work_dim++;
+   }
+
+   with_crb (cs, 11) {
+      crb.add(SP_CS_NDRANGE_0(CHIP,
+         .kerneldim = work_dim,
+         .localsizex = local_size[0] - 1,
+         .localsizey = local_size[1] - 1,
+         .localsizez = local_size[2] - 1,
+      ));
+
+      if (CHIP >= A7XX) {
+         crb.add(SP_CS_NDRANGE_7(CHIP,
+            .localsizex = local_size[0] - 1,
+            .localsizey = local_size[1] - 1,
+            .localsizez = local_size[2] - 1,
+         ));
+      }
+
+      crb.add(SP_CS_NDRANGE_1(CHIP,
+         .globalsize_x = local_size[0] * num_groups[0],
+      ));
+      crb.add(SP_CS_NDRANGE_2(CHIP, 0));
+      crb.add(SP_CS_NDRANGE_3(CHIP,
+         .globalsize_y = local_size[1] * num_groups[1],
+      ));
+      crb.add(SP_CS_NDRANGE_4(CHIP, 0));
+      crb.add(SP_CS_NDRANGE_5(CHIP,
+         .globalsize_z = local_size[2] * num_groups[2],
+      ));
+      crb.add(SP_CS_NDRANGE_6(CHIP, 0));
+
+      crb.add(SP_CS_KERNEL_GROUP_X(CHIP, 1));
+      crb.add(SP_CS_KERNEL_GROUP_Y(CHIP, 1));
+      crb.add(SP_CS_KERNEL_GROUP_Z(CHIP, 1));
+   }
+
+   if (a6xx_backend->num_perfcntrs > 0) {
+      a6xx_backend->query_mem = fd_bo_new(
+         a6xx_backend->dev,
+         a6xx_backend->num_perfcntrs * sizeof(struct fd6_query_sample), 0, "query");
+
+      /* configure the performance counters to count the requested
+       * countables:
+       */
+      for (unsigned i = 0; i < a6xx_backend->num_perfcntrs; i++) {
+         const struct perfcntr *counter = &a6xx_backend->perfcntrs[i];
+
+         fd_pkt4(cs, 1).add({
+            .reg = counter->select_reg,
+            .value = counter->selector,
+         });
+      }
+
+      fd_pkt7(cs, CP_WAIT_FOR_IDLE, 0);
+
+      /* and snapshot the start values: */
+      for (unsigned i = 0; i < a6xx_backend->num_perfcntrs; i++) {
+         const struct perfcntr *counter = &a6xx_backend->perfcntrs[i];
+
+         fd_pkt7(cs, CP_REG_TO_MEM, 3)
+            .add(CP_REG_TO_MEM_0(.reg = counter->counter_reg_lo, ._64b = true))
+            .add(A5XX_CP_REG_TO_MEM_DEST(query_sample_idx(a6xx_backend, i, start)));
+      }
+   }
+
+   fd_pkt7(cs, CP_EXEC_CS, 4)
+      .add(CP_EXEC_CS_0())
+      .add(CP_EXEC_CS_1(.ngroups_x = grid[0]))
+      .add(CP_EXEC_CS_2(.ngroups_y = grid[1]))
+      .add(CP_EXEC_CS_3(.ngroups_z = grid[2]));
+
+   fd_pkt7(cs, CP_WAIT_FOR_IDLE, 0);
+
+   if (a6xx_backend->num_perfcntrs > 0) {
+      /* snapshot the end values: */
+      for (unsigned i = 0; i < a6xx_backend->num_perfcntrs; i++) {
+         const struct perfcntr *counter = &a6xx_backend->perfcntrs[i];
+
+         fd_pkt7(cs, CP_REG_TO_MEM, 3)
+            .add(CP_REG_TO_MEM_0(.reg = counter->counter_reg_lo, ._64b = true))
+            .add(A5XX_CP_REG_TO_MEM_DEST(query_sample_idx(a6xx_backend, i, stop)));
+      }
+
+      /* and compute the result: */
+      for (unsigned i = 0; i < a6xx_backend->num_perfcntrs; i++) {
+         /* result += stop - start: */
+         fd_pkt7(cs, CP_MEM_TO_MEM, 9)
+            .add(CP_MEM_TO_MEM_0(.neg_c = true, ._double = true))
+            .add(CP_MEM_TO_MEM_DST(query_sample_idx(a6xx_backend, i, result)))
+            .add(CP_MEM_TO_MEM_SRC_A(query_sample_idx(a6xx_backend, i, result)))
+            .add(CP_MEM_TO_MEM_SRC_B(query_sample_idx(a6xx_backend, i, stop)))
+            .add(CP_MEM_TO_MEM_SRC_C(query_sample_idx(a6xx_backend, i, start)));
+      }
+   }
+
+   cache_flush<CHIP>(cs, kernel);
+}
+
+static void
+a6xx_set_perfcntrs(struct backend *b, const struct perfcntr *perfcntrs,
+                   unsigned num_perfcntrs)
+{
+   struct a6xx_backend *a6xx_backend = to_a6xx_backend(b);
+
+   a6xx_backend->perfcntrs = perfcntrs;
+   a6xx_backend->num_perfcntrs = num_perfcntrs;
+}
+
+static void
+a6xx_read_perfcntrs(struct backend *b, uint64_t *results)
+{
+   struct a6xx_backend *a6xx_backend = to_a6xx_backend(b);
+
+   fd_bo_cpu_prep(a6xx_backend->query_mem, NULL, FD_BO_PREP_READ);
+   struct fd6_query_sample *samples =
+      (struct fd6_query_sample *)fd_bo_map(a6xx_backend->query_mem);
+
+   for (unsigned i = 0; i < a6xx_backend->num_perfcntrs; i++) {
+      results[i] = samples[i].result;
+   }
+}
+
+template<chip CHIP>
+struct backend *
+a6xx_init(struct fd_device *dev, const struct fd_dev_id *dev_id)
+{
+   struct a6xx_backend *a6xx_backend =
+      (struct a6xx_backend *)calloc(1, sizeof(*a6xx_backend));
+
+   a6xx_backend->base = (struct backend){
+      .assemble = a6xx_assemble,
+      .disassemble = a6xx_disassemble,
+      .emit_grid = a6xx_emit_grid<CHIP>,
+      .set_perfcntrs = a6xx_set_perfcntrs,
+      .read_perfcntrs = a6xx_read_perfcntrs,
+   };
+
+   struct ir3_compiler_options compiler_options = {
+      .disable_cache = true,
+   };
+   a6xx_backend->compiler =
+      ir3_compiler_create(dev, dev_id, fd_dev_info_raw(dev_id), &compiler_options);
+   a6xx_backend->dev = dev;
+
+   a6xx_backend->info = fd_dev_info_raw(dev_id);
+
+   a6xx_backend->control_mem =
+      fd_bo_new(dev, 0x1000, 0, "control");
+
+   return &a6xx_backend->base;
+}
+FD_GENX(a6xx_init);

src/freedreno/computerator/examples/branch.asm

@@ -0,0 +1,15 @@
+@localsize 16, 1, 1
+@buf 16  ; g[0]
+@invocationid(r0.x) ; r0.xyz
+@branchstack 1
+cmps.u.gt p0.x, r0.x, 0
+mov.u32u32 r1.x, 0x87654321
+(rpt5)nop
+br !p0.x, #endif
+mov.u32u32 r1.x, 0x12345678
+endif:
+(jp)(rpt5)nop
+stib.b.untyped.1d.u32.1.imm r1.x, r0.x, 0
+end
+nop
+

src/freedreno/computerator/examples/buf-init.asm

@@ -0,0 +1,9 @@
+@localsize 8, 1, 1
+@buf 8 1, 2, 3, 4, 5, 6, 7, 8
+@invocationid(r0.x)
+
+ldib.b.untyped.1d.u32.4.imm r1.x, r0.x, 0
+(sy)add.u r1.x, r1.x, 5
+(rpt5)nop
+stib.b.untyped.1d.u32.4.imm r1.x, r0.x, 0
+end

src/freedreno/computerator/examples/early_preamble.asm

@@ -0,0 +1,25 @@
+@localsize 1, 1, 1
+@buf 4  ; g[0]
+@invocationid(r0.x) ; r0.xyz
+@const(c0.x)  0.0, 0.0, 0.0, 0.0
+@earlypreamble
+
+shps #l_preamble_end
+getone #l_preamble_end
+
+mov.u32u32 r48.x, 1
+mov.u32u32 r48.y, 2
+mov.u32u32 r48.z, 3
+mov.u32u32 r48.w, 4
+(rpt5)nop
+stc.u32 c[0], r48.x, 4
+
+(sy)(ss)shpe
+
+l_preamble_end:
+(jp)nop
+
+(rpt3)mov.u32u32 r1.x, (r)c0.x
+(rpt5)nop
+stib.b.untyped.1d.u32.4.imm r1.x, r0.x, 0
+end

src/freedreno/computerator/examples/invocationid.asm

@@ -0,0 +1,13 @@
+@localsize 32, 1, 1
+@buf 32  ; g[0]
+@const(c0.x)  0.0, 0.0, 0.0, 0.0
+@const(c1.x)  1.0, 2.0, 3.0, 4.0
+@wgid(r48.x)        ; r48.xyz
+@invocationid(r0.x) ; r0.xyz
+@numwg(c2.x)        ; c2.xyz
+mov.u32u32 r0.y, r0.x
+(rpt5)nop
+stib.b.untyped.1d.u32.1.imm r0.x, r0.y, 0
+end
+nop
+

src/freedreno/computerator/examples/localmem.asm

@@ -0,0 +1,18 @@
+@localsize 8, 1, 1
+@buf 8
+@invocationid(r0.x)
+@localmem 4
+@branchstack 1
+
+mov.u32u32 r1.x, 0
+
+getone #l_all
+mov.u32u32 r1.y, 5
+(rpt2)nop
+stlw.u32 l[r1.x], r1.y, 1
+
+l_all:
+(jp)ldlw.u32 r1.z, l[r1.x], 1
+(ss)nop
+stib.b.untyped.1d.u32.4.imm r1.z, r0.x, 0
+end

src/freedreno/computerator/examples/pvtmem.asm

@@ -0,0 +1,14 @@
+@localsize 1, 1, 1
+@buf 4  ; g[0]
+@pvtmem 4
+mov.u32u32 r1.x, 0x12345678
+mov.u32u32 r0.x, 0
+(rpt5)nop
+stp.u32 p[r0.x + 0], r1.x, 1
+ldp.u32 r0.x, p[r0.x + 0], 1
+mov.u32u32 r0.y, 0x00000000
+(sy)(rpt5)nop
+stib.b.untyped.1d.u32.1.imm r0.x, r0.y, 0
+end
+nop
+

src/freedreno/computerator/examples/simple.asm

@@ -0,0 +1,28 @@
+@localsize 1, 1, 1
+@buf 4  ; g[0]
+@const(c0.x)  0.0, 0.0, 0.0, 0.0
+@const(c1.x)  1.0, 2.0, 3.0, 4.0
+@wgid(r48.x)        ; r48.xyz
+@invocationid(r0.x) ; r0.xyz
+@numwg(c2.x)        ; c2.xyz
+mov.f32f32 r2.x, c0.y
+mov.u32u32 r0.x, 0x12345678
+mov.u32u32 r0.y, 0x12345678
+mov.u32u32 r0.z, 0x12345678
+add.u r2.x, c0.x, r2.x
+mov.u32u32 r0.w, 0x12345678
+mov.u32u32 r1.x, 0x12345678
+mov.u32u32 r1.y, 0x12345678
+cov.u32s16 hr4.x, r2.x
+mov.u32u32 r1.z, 0x12345678
+mov.u32u32 r1.w, 0x12345678
+nop
+mova a0.x, hr4.x
+(rpt5)nop
+(ul)mov.u32u32 r0.x, r<a0.x>
+mov.u32u32 r0.y, 0x00000000
+(rpt5)nop
+stib.b.untyped.1d.u32.1.imm r0.x, r0.y, 0
+end
+nop
+

src/freedreno/computerator/examples/stg_ldg_offset.asm

@@ -0,0 +1,17 @@
+@localsize 16, 1, 1
+@buf 128 (c2.x)  ; c2.xy
+@invocationid(r0.x) ; r0.xyz
+mov.u32u32 r0.y, r0.x
+mov.u32u32 r1.x, c2.x
+mov.u32u32 r1.y, c2.y
+mov.u32u32 r2.x, 0xff
+(rpt5)nop
+stg.a.u32 g[r1.x+r0.y<<4+2<<2], r2.x, 1
+nop(sy)
+ldg.a.u32 r4.x, g[r1.x+r0.y<<4+2<<2], 1
+nop(sy)
+add.u r4.x, r4.x, 1
+(rpt3)nop
+stg.a.u32 g[r1.x+r0.y<<4+1<<2], r4.x, 1
+end
+nop

src/freedreno/computerator/examples/test-flut.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+set -e
+
+gen_shader() {
+	imm=$1
+	if [ "$2" = "half" ]; then
+		h="h"
+		mov="cov.u16u32"
+	else
+		h=""
+		mov="mov.u32u32"
+	fi
+	cat <<EOF
+@localsize 1, 1, 1
+@buf 4  ; g[0]
+mov.u32u32 r0.x, 0
+(rpt5)nop
+(rpt5)nop
+add.f ${h}r2.x, ${h}r0.x, $imm
+(rpt5)nop
+$mov r1.x, ${h}r2.x
+(rpt5)nop
+; and store results:
+stib.b.untyped.1d.u32.1.imm r1.x, r0.x, 0
+(sy)nop
+end
+EOF
+}
+
+
+run() {
+	echo "TEST: $*"
+	gen_shader $* | ./computerator -g 1,1,1 -d
+}
+
+for n in `seq 0 16`; do
+	run "$n" "full"
+	run "h($n)" "half"
+done

src/freedreno/computerator/examples/test-opcodes.sh

@@ -0,0 +1,297 @@
+#!/bin/bash
+#
+# Test various instructions to check whether half<->full widening/narrowing
+# works.  The basic premise is to perform the same instruction with and
+# without the widening/narrowing folded in and check if the results match.
+#
+# Note this doesn't currently diferentiate between signed/unsigned/bool,
+# and just assumes int is signed (since unsigned is basically(ish) like
+# signed but without sign extension)
+#
+# TODO probably good pick numeric src values that are better at triggering
+# edge cases, while still not loosing precision in a full->half->full
+# seqeuence.. but some instructions like absneg don't even appear to be
+# subtlely wrong when you try to fold in a precision conversion.
+#
+# add '-v' arg to see the result values
+
+set -e
+
+#
+# Templates for float->float instructions:
+#
+f2f_instrs=(
+	'add.f $dst, $src1, $src2'
+	'min.f $dst, $src1, $src2'
+	'min.f $dst, $src2, $src1'
+	'max.f $dst, $src1, $src2'
+	'max.f $dst, $src2, $src1'
+	'mul.f $dst, $src1, $src2'
+	'sign.f $dst, $src1'
+	'absneg.f $dst, \(neg\)$src1'
+	'absneg.f $dst, \(abs\)$src1'
+	'floor.f $dst, $src1'
+	'ceil.f $dst, $src1'
+	'rndne.f $dst, $src1'
+	'rndaz.f $dst, $src1'
+	'trunc.f $dst, $src1'
+)
+
+#
+# Templates for float->int instructions:
+#
+f2i_instrs=(
+	'cmps.f.gt $dst, $src1, $src2'
+	'cmps.f.lt $dst, $src1, $src2'
+	'cmpv.f.gt $dst, $src1, $src2'
+	'cmpv.f.lt $dst, $src1, $src2'
+)
+
+#
+# Templates for int->int instructions:
+#
+i2i_instrs=(
+	'add.u $dst, $src1, $src2'
+	'add.s $dst, $src1, $src2'
+	'sub.u $dst, $src1, $src2'
+	'sub.s $dst, $src1, $src2'
+	'cmps.f.gt $dst, $src1, $src2'
+	'cmps.f.lt $dst, $src1, $src2'
+	'min.u $dst, $src1, $src2'
+	'min.u $dst, $src2, $src1'
+	'min.s $dst, $src1, $src2'
+	'min.s $dst, $src2, $src1'
+	'max.u $dst, $src1, $src2'
+	'max.u $dst, $src2, $src1'
+	'max.s $dst, $src1, $src2'
+	'max.s $dst, $src2, $src1'
+	'absneg.s $dst, \(neg\)$src1'
+	'absneg.s $dst, \(abs\)$src1'
+	'and.b $dst, $src2, $src3'
+	'or.b $dst, $src1, $src2'
+	'not.b $dst, $src1'
+	'xor.b $dst, $src1, $src2'
+	'cmpv.u.gt $dst, $src1, $src2'
+	'cmpv.u.lt $dst, $src1, $src2'
+	'cmpv.s.gt $dst, $src1, $src2'
+	'cmpv.s.lt $dst, $src1, $src2'
+	'mul.u24 $dst, $src1, $src2'
+	'mul.s24 $dst, $src1, $src2'
+	'mull.u $dst, $src1, $src2'
+	'bfrev.b $dst, $src1'
+	'clz.s $dst, $src2'
+	'clz.b $dst, $src2'
+	'shl.b $dst, $src1, $src2'
+	'shr.b $dst, $src3, $src1'
+	'ashr.b $dst, $src3, $src1'
+	'mgen.b $dst, $src1, $src2'
+	'getbit.b $dst, $src3, $src2'
+	'setrm $dst, $src1'
+	'cbits.b $dst, $src3'
+	'shb $dst, $src1, $src2'
+	'msad $dst, $src1, $src2'
+)
+
+#
+# Helper to expand instruction template:
+#
+expand() {
+	instr=$1
+	dst=$2
+	src1=$3
+	src2=$4
+	src3=$5
+	eval echo $instr
+}
+
+expand_test() {
+	instr=$1
+
+	echo '; control, half->half:'
+	expand $instr "hr1.x" "hr0.x" "hr0.y" "hr0.z"
+	echo '; test, full->half:'
+	expand $instr "hr1.y" "r1.x" "r1.y" "r1.z"
+
+	echo '; control, full->full:'
+	expand $instr "r2.x" "r1.x" "r1.y" "r1.z"
+	echo '; test, half->full:'
+	expand $instr "r2.y" "hr0.x" "hr0.y" "hr0.z"
+
+	echo "(rpt5)nop"
+}
+
+#
+# Helpers to construct test program assembly:
+#
+header_asm() {
+	cat <<EOF
+@localsize 1, 1, 1
+@buf 4  ; g[0]
+EOF
+}
+
+footer_asm() {
+	cat <<EOF
+; dest offsets:
+mov.u32u32 r3.x, 0
+mov.u32u32 r3.y, 1
+mov.u32u32 r3.z, 2
+mov.u32u32 r3.w, 3
+(rpt5)nop
+
+; and store results:
+stib.untyped.1d.u32.1 r2.x, r3.x, 0   ; control: full->full
+stib.untyped.1d.u32.1 r2.y, r3.y, 0   ; test:    half->full
+stib.untyped.1d.u32.1 r2.z, r3.z, 0   ; control: half->half
+stib.untyped.1d.u32.1 r2.w, r3.w, 0   ; test:    full->half
+(sy)nop
+end
+EOF
+}
+
+setup_asm_float() {
+	cat <<EOF
+; hr0->hr1 (r0) avail for half, hr0 for src, hr1 for dst
+; r1->r2 avail for full, r1 for src, r2 for dst
+cov.f32f16 hr0.x, (1.0)
+cov.f32f16 hr0.y, (2.0)
+cov.f32f16 hr0.z, (3.0)
+mov.f32f32 r1.x,  (1.0)
+mov.f32f32 r1.y,  (2.0)
+mov.f32f32 r1.z,  (3.0)
+(rpt5)nop
+EOF
+}
+
+setup_asm_int() {
+	cat <<EOF
+; hr0->hr1 (r0) avail for half, hr0 for src, hr1 for dst
+; r1->r2 avail for full, r1 for src, r2 for dst
+cov.s32s16 hr0.x,  1
+cov.s32s16 hr0.y, -2
+cov.s32s16 hr0.z,  3
+mov.s32s32 r1.x,   1
+mov.s32s32 r1.y,  -2
+mov.s32s32 r1.z,   3
+(rpt5)nop
+EOF
+}
+
+#
+# Generate assembly code to test float->float opcode
+#
+f2f_asm() {
+	instr=$1
+
+	header_asm
+	setup_asm_float
+	expand_test $instr
+
+	cat <<EOF
+; convert half results back to full:
+cov.f16f32 r2.z, hr1.x
+cov.f16f32 r2.w, hr1.y
+EOF
+
+	footer_asm
+}
+
+#
+# Generate assembly code to test float->int opcode
+#
+f2i_asm() {
+	instr=$1
+
+	header_asm
+	setup_asm_float
+	expand_test $instr
+
+	cat <<EOF
+; convert half results back to full:
+cov.s16s32 r2.z, hr1.x
+cov.s16s32 r2.w, hr1.y
+EOF
+
+	footer_asm
+}
+
+#
+# Generate assembly code to test int->int opcode
+#
+i2i_asm() {
+	instr=$1
+
+	header_asm
+	setup_asm_int
+	expand_test $instr
+
+	cat <<EOF
+; convert half results back to full:
+cov.s16s32 r2.z, hr1.x
+cov.s16s32 r2.w, hr1.y
+EOF
+
+	footer_asm
+}
+
+
+#
+# Helper to parse computerator output and print results:
+#
+check_results() {
+	str=`cat - | grep "	" | head -1 | xargs`
+
+	if [ "$verbose" = "true" ]; then
+		echo $str
+	fi
+
+	# Split components of result buffer:
+	cf=$(echo $str | cut -f1 -d' ')
+	tf=$(echo $str | cut -f2 -d' ')
+	ch=$(echo $str | cut -f3 -d' ')
+	th=$(echo $str | cut -f4 -d' ')
+
+	# Sanity test, make sure the control results match:
+	if [ $cf != $ch ]; then
+		echo "    FAIL: control results do not match!  Half vs full op is not equivalent!"
+		echo "    full=$cf half=$ch"
+	fi
+
+	# Compare test (with conversion folded) to control:
+	if [ $cf != $tf ]; then
+		echo "    FAIL: half -> full widening result does not match control!"
+		echo "    control=$cf result=$tf"
+	fi
+	if [ $ch != $th ]; then
+		echo "    FAIL: full -> half narrowing result does not match control!"
+		echo "    control=$ch result=$th"
+	fi
+
+	# HACK without a delay different invocations
+	# of computerator seem to somehow clobber each
+	# other.. which isn't great..
+	sleep 0.1
+}
+
+#
+# Run the tests!
+#
+
+if [ "$1" = "-v" ]; then
+	verbose="true"
+fi
+
+IFS=""
+for instr in ${f2f_instrs[@]}; do
+	echo "TEST: $instr"
+	f2f_asm $instr | ./computerator -g 1,1,1 | check_results
+done
+for instr in ${f2i_instrs[@]}; do
+	echo "TEST: $instr"
+	f2i_asm $instr | ./computerator -g 1,1,1 | check_results
+done
+for instr in ${i2i_instrs[@]}; do
+	echo "TEST: $instr"
+	i2i_asm $instr | ./computerator -g 1,1,1 | check_results
+done
+

src/freedreno/computerator/examples/test-regfile.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+# This tests for the size of the register file. We do this by launching a
+# lot of workgroups with only one invocation, which causes the GPU to be
+# saturated with in-flight waves. Each thread records its wave id using "getwid"
+# (only available in a6xx+!) and stores it in the buffer. We then vary the
+# register footprint by introducing uses of higher and higher registers. This
+# lets us determine:
+# 1. The total number of waves available (always 16 for known models)
+# 2. The wave granularity (how many waves are always launched together, always 2
+# for known models).
+# 3. The total size of the register file that is divvied up between the waves.
+
+set -e
+
+gen_shader() {
+	n=$1;
+	cat <<EOF
+@localsize 1, 1, 1
+@buf 128  ; g[0]
+@wgid(r48.x)
+getwid.u32 r1.x
+mov.u32u32 r0.x, r48.x
+
+; busy loop to make sure it actually uses all possible waves
+mov.u32u32 r0.y, 16
+(rpt2)nop
+loop:
+cmps.u.gt p0.x, r0.y, 0
+sub.u r0.y, r0.y, 1
+(rpt5)nop
+br p0.x, #loop
+add.f r1.y, r1.x, r$n.w
+
+(ss)(sy)(rpt5)nop
+stib.b.untyped.1d.u32.1.imm r1.x, r0.x, 0
+end
+nop
+EOF
+}
+
+# generate reference:
+gen_shader 1 | ./computerator -g 128,1,1 | tee reference.log
+
+for n in `seq 2 32`; do
+	echo "Trying max reg: r$n"
+	gen_shader $n | ./computerator -g 128,1,1 | tee new.log
+	diff reference.log new.log
+	if [ "$?" != "0" ]; then
+		echo "Changes at r$n"
+		break
+	fi
+done

src/freedreno/computerator/examples/ubo.asm

@@ -0,0 +1,29 @@
+@localsize 1, 1, 1
+@ubo 4 1, 2, 3, 4 ; UBO 0
+@ubo 4 5, 6, 7, 8 ; UBO 1
+@buf 4
+@invocationid(r0.x)
+@branchstack 1
+@earlypreamble
+
+shps #main
+getone #main
+
+mova1 a1.x, (r)0
+(ss)nop
+
+; copy UBO 0 to c0
+ldc.4.k.imm c[a1.x], 0, 0
+
+(sy)(ss)shpe
+
+main:
+
+; load UBO 1 to r1
+(jp)ldc.offset0.4.imm r1.x, 0, 1
+
+(sy)(rpt3)add.u r1.x, (r)r1.x, (r)c0.x
+(rpt2)nop
+
+stib.b.untyped.1d.u32.4.imm r1.x, r0.x, 0
+end

src/freedreno/computerator/ir3_asm.cc

@@ -0,0 +1,70 @@
+/*
+ * Copyright © 2020 Google, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "ir3/ir3_assembler.h"
+#include "ir3/ir3_compiler.h"
+
+#include "ir3_asm.h"
+
+struct ir3_kernel *
+ir3_asm_assemble(struct ir3_compiler *c, FILE *in)
+{
+   struct ir3_kernel *kernel = (struct ir3_kernel *)calloc(1, sizeof(*kernel));
+   struct ir3_shader *shader = ir3_parse_asm(c, &kernel->info, in);
+   if (!shader)
+      errx(-1, "assembler failed");
+   struct ir3_shader_variant *v = shader->variants;
+
+   kernel->v = v;
+   kernel->bin = v->bin;
+
+   kernel->base.local_size[0] = v->local_size[0];
+   kernel->base.local_size[1] = v->local_size[1];
+   kernel->base.local_size[2] = v->local_size[2];
+   kernel->base.num_bufs = kernel->info.num_bufs;
+   memcpy(kernel->base.buf_types, kernel->info.buf_types,
+          sizeof(kernel->base.buf_sizes));
+   memcpy(kernel->base.buf_sizes, kernel->info.buf_sizes,
+          sizeof(kernel->base.buf_sizes));
+   memcpy(kernel->base.buf_addr_regs, kernel->info.buf_addr_regs,
+          sizeof(kernel->base.buf_addr_regs));
+   memcpy(kernel->base.buf_init_data, kernel->info.buf_init_data,
+          sizeof(kernel->base.buf_init_data));
+
+   unsigned sz = v->info.size;
+
+   v->bo = fd_bo_new(c->dev, sz, FD_BO_NOMAP, "%s", ir3_shader_stage(v));
+
+   fd_bo_upload(v->bo, kernel->bin, 0, sz);
+
+   /* Always include shaders in kernel crash dumps. */
+   fd_bo_mark_for_dump(v->bo);
+
+   return kernel;
+}
+
+void
+ir3_asm_disassemble(struct ir3_kernel *k, struct ir3_disasm_options *options)
+{
+   ir3_shader_disasm_options(k->v, (uint32_t *)k->bin, options);
+}

src/freedreno/computerator/ir3_asm.h

@@ -0,0 +1,27 @@
+/*
+ * Copyright © 2020 Google, Inc.
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef __IR3_ASM_H__
+#define __IR3_ASM_H__
+
+#include "main.h"
+
+#include "ir3/ir3_parser.h"
+#include "ir3/ir3_shader.h"
+
+struct ir3_kernel {
+   struct kernel base;
+   struct ir3_kernel_info info;
+   struct backend *backend;
+   struct ir3_shader_variant *v;
+   void *bin;
+};
+define_cast(kernel, ir3_kernel);
+
+struct ir3_kernel *ir3_asm_assemble(struct ir3_compiler *c, FILE *in);
+void ir3_asm_disassemble(struct ir3_kernel *k,
+                         struct ir3_disasm_options *options);
+
+#endif /* __IR3_ASM_H__ */

src/freedreno/computerator/main.cc

@@ -0,0 +1,340 @@
+/*
+ * Copyright © 2020 Google, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <getopt.h>
+#include <inttypes.h>
+#include <locale.h>
+#include <stdlib.h>
+#include <xf86drm.h>
+
+#include "util/u_math.h"
+#include "util/stack_array.h"
+
+#include "ir3/ir3_shader.h"
+#include "perfcntrs/freedreno_perfcntr.h"
+
+#include "main.h"
+
+static void
+dump_float(void *buf, int sz)
+{
+   uint8_t *ptr = (uint8_t *)buf;
+   uint8_t *end = ptr + sz - 3;
+   int i = 0;
+
+   while (ptr < end) {
+      uint32_t d = 0;
+
+      printf((i % 8) ? " " : "\t");
+
+      d |= *(ptr++) << 0;
+      d |= *(ptr++) << 8;
+      d |= *(ptr++) << 16;
+      d |= *(ptr++) << 24;
+
+      printf("%8f", uif(d));
+
+      if ((i % 8) == 7) {
+         printf("\n");
+      }
+
+      i++;
+   }
+
+   if (i % 8) {
+      printf("\n");
+   }
+}
+
+static void
+dump_hex(void *buf, int sz)
+{
+   uint8_t *ptr = (uint8_t *)buf;
+   uint8_t *end = ptr + sz;
+   int i = 0;
+
+   while (ptr < end) {
+      uint32_t d = 0;
+
+      printf((i % 8) ? " " : "\t");
+
+      d |= *(ptr++) << 0;
+      d |= *(ptr++) << 8;
+      d |= *(ptr++) << 16;
+      d |= *(ptr++) << 24;
+
+      printf("%08x", d);
+
+      if ((i % 8) == 7) {
+         printf("\n");
+      }
+
+      i++;
+   }
+
+   if (i % 8) {
+      printf("\n");
+   }
+}
+
+static const char *shortopts = "drf:g:hp:";
+
+static const struct option longopts[] = {
+   {"disasm", no_argument, 0, 'd'},         {"file", required_argument, 0, 'f'},
+   {"groups", required_argument, 0, 'g'},   {"help", no_argument, 0, 'h'},
+   {"perfcntr", required_argument, 0, 'p'},
+   {"disasm-print-raw", no_argument, 0, 'r'},
+   {0, 0, 0, 0}};
+
+static void
+usage(const char *name)
+{
+   printf(
+      "Usage: %s [-dfgh]\n"
+      "\n"
+      "options:\n"
+      "    -d, --disasm             print disassembled shader\n"
+      "    -f, --file=FILE          read shader from file (instead of stdin)\n"
+      "    -g, --groups=X,Y,Z       use specified group size\n"
+      "    -h, --help               show this message\n"
+      "    -p, --perfcntr=LIST      sample specified performance counters "
+      "(comma\n"
+      "                             separated list)\n",
+      name);
+}
+
+/* performance counter description: */
+static unsigned num_groups;
+static const struct fd_perfcntr_group *groups;
+
+/* Track enabled counters per group: */
+static unsigned *enabled_counters;
+
+static void
+setup_counter(const char *name, struct perfcntr *c)
+{
+   for (int i = 0; i < num_groups; i++) {
+      const struct fd_perfcntr_group *group = &groups[i];
+      const struct fd_perfcntr_countable *countable =
+         fd_perfcntrs_countable(group, name);
+
+      if (countable) {
+         /*
+          * Allocate a counter to use to monitor the requested countable:
+          */
+         if (enabled_counters[i] >= group->num_counters) {
+            errx(-1, "Too many counters selected in group: %s", group->name);
+         }
+
+         unsigned idx = enabled_counters[i]++;
+         const struct fd_perfcntr_counter *counter = &group->counters[idx];
+
+         /*
+          * And initialize the perfcntr struct, pulling together the info
+          * about selected counter and countable, to simplify life for the
+          * backend:
+          */
+         c->name = name;
+         c->select_reg = counter->select_reg;
+         c->counter_reg_lo = counter->counter_reg_lo;
+         c->counter_reg_hi = counter->counter_reg_hi;
+         c->selector = countable->selector;
+
+         return;
+      }
+   }
+
+   errx(-1, "could not find countable: %s", name);
+}
+
+static struct perfcntr *
+parse_perfcntrs(const struct fd_dev_id *dev_id, const char *perfcntrstr,
+                unsigned *num_perfcntrs)
+{
+   struct perfcntr *counters = NULL;
+   char *cnames, *s;
+   unsigned cnt = 0;
+
+   groups = fd_perfcntrs(dev_id, &num_groups);
+   enabled_counters = (uint32_t *) calloc(num_groups, sizeof(enabled_counters[0]));
+
+   cnames = strdup(perfcntrstr);
+   while ((s = strstr(cnames, ","))) {
+      char *name = cnames;
+      s[0] = '\0';
+      cnames = &s[1];
+
+      counters =
+         (struct perfcntr *)realloc(counters, ++cnt * sizeof(counters[0]));
+      setup_counter(name, &counters[cnt - 1]);
+   }
+
+   char *name = cnames;
+   counters = (struct perfcntr *)realloc(counters, ++cnt * sizeof(counters[0]));
+   setup_counter(name, &counters[cnt - 1]);
+
+   *num_perfcntrs = cnt;
+
+   return counters;
+}
+
+int
+main(int argc, char **argv)
+{
+   FILE *in = stdin;
+   const char *perfcntrstr = NULL;
+   struct perfcntr *perfcntrs = NULL;
+   unsigned num_perfcntrs = 0;
+   bool disasm = false;
+   bool disasm_print_raw = false;
+   uint32_t grid[3] = {0};
+   int opt, ret;
+
+   setlocale(LC_NUMERIC, "en_US.UTF-8");
+
+   while ((opt = getopt_long_only(argc, argv, shortopts, longopts, NULL)) !=
+          -1) {
+      switch (opt) {
+      case 'd':
+         disasm = true;
+         break;
+      case 'r':
+         disasm_print_raw = true;
+         break;
+      case 'f':
+         in = fopen(optarg, "r");
+         if (!in)
+            err(1, "could not open '%s'", optarg);
+         break;
+      case 'g':
+         ret = sscanf(optarg, "%u,%u,%u", &grid[0], &grid[1], &grid[2]);
+         if (ret != 3) {
+            usage(argv[0]);
+            return -1;
+         }
+         break;
+      case 'h':
+         usage(argv[0]);
+         return -1;
+      case 'p':
+         perfcntrstr = optarg;
+         break;
+      default:
+         printf("unrecognized arg: %c\n", opt);
+         usage(argv[0]);
+         return -1;
+      }
+   }
+
+   struct fd_device *dev = fd_device_open();
+   struct fd_pipe *pipe = fd_pipe_new(dev, FD_PIPE_3D);
+
+   const struct fd_dev_id *dev_id = fd_pipe_dev_id(pipe);
+
+   printf("got gpu: %s\n", fd_dev_name(dev_id));
+
+   struct backend *backend;
+   switch (fd_dev_gen(dev_id)) {
+   case 4:
+      backend = a4xx_init(dev, dev_id);
+      break;
+   case 6:
+      backend = a6xx_init<A6XX>(dev, dev_id);
+      break;
+   case 7:
+      backend = a6xx_init<A7XX>(dev, dev_id);
+      break;
+   case 8:
+      backend = a6xx_init<A8XX>(dev, dev_id);
+      break;
+   default:
+      err(1, "unsupported gpu generation: a%uxx", fd_dev_gen(dev_id));
+   }
+
+   struct kernel *kernel = backend->assemble(backend, in);
+   printf("localsize: %dx%dx%d\n", kernel->local_size[0], kernel->local_size[1],
+          kernel->local_size[2]);
+   for (int i = 0; i < kernel->num_bufs; i++) {
+      printf("buf[%d]: size=%u, type=%s\n", i, kernel->buf_sizes[i],
+             kernel->buf_types[i] == KERNEL_BUF_UBO ? "UBO" : "SSBO");
+      kernel->bufs[i] = fd_bo_new(dev, kernel->buf_sizes[i] * 4, 0, "buf[%d]", i);
+
+      if (kernel->buf_init_data[i]) {
+         fd_bo_cpu_prep(kernel->bufs[i], pipe, FD_BO_PREP_WRITE);
+         void *map = fd_bo_map(kernel->bufs[i]);
+         memcpy(map, kernel->buf_init_data[i], kernel->buf_sizes[i] * 4);
+         free(kernel->buf_init_data[i]);
+      }
+   }
+
+   if (disasm) {
+      struct ir3_disasm_options options = {
+         .out = stdout,
+         .print_raw = disasm_print_raw,
+      };
+      backend->disassemble(kernel, &options);
+   }
+
+   if (grid[0] == 0)
+      return 0;
+
+   struct fd_submit *submit = fd_submit_new(pipe);
+
+   if (perfcntrstr) {
+      if (!backend->set_perfcntrs) {
+         err(1, "performance counters not supported");
+      }
+      perfcntrs = parse_perfcntrs(dev_id, perfcntrstr, &num_perfcntrs);
+      backend->set_perfcntrs(backend, perfcntrs, num_perfcntrs);
+   }
+
+   backend->emit_grid(kernel, grid, submit);
+
+   struct fd_fence *fence = fd_submit_flush(submit, -1, false);
+
+   fd_fence_flush(fence);
+   fd_fence_del(fence);
+
+   for (int i = 0; i < kernel->num_bufs; i++) {
+      fd_bo_cpu_prep(kernel->bufs[i], pipe, FD_BO_PREP_READ);
+      void *map = fd_bo_map(kernel->bufs[i]);
+
+      printf("buf[%d]:\n", i);
+      dump_hex(map, kernel->buf_sizes[i] * 4);
+      dump_float(map, kernel->buf_sizes[i] * 4);
+   }
+
+   if (perfcntrstr) {
+      STACK_ARRAY(uint64_t, results, num_perfcntrs);
+
+      backend->read_perfcntrs(backend, results);
+
+      for (unsigned i = 0; i < num_perfcntrs; i++) {
+         printf("%s:\t%'" PRIu64 "\n", perfcntrs[i].name, results[i]);
+      }
+
+      STACK_ARRAY_FINISH(results);
+   }
+
+   return 0;
+}

src/freedreno/computerator/main.h

@@ -0,0 +1,75 @@
+/*
+ * Copyright © 2020 Google, Inc.
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef __MAIN_H__
+#define __MAIN_H__
+
+#include <err.h>
+#include <stdint.h>
+#include <stdio.h>
+
+#include "drm/freedreno_drmif.h"
+#include "drm/freedreno_ringbuffer.h"
+
+#include "common/fd_hw_common.h"
+
+#include "ir3/ir3_assembler.h"
+
+#define MAX_BUFS 4
+
+struct kernel {
+   /* filled in by backend when shader is assembled: */
+   uint32_t local_size[3];
+   uint32_t num_bufs;
+   enum kernel_buf_type buf_types[MAX_BUFS];
+   uint32_t buf_sizes[MAX_BUFS]; /* size in dwords */
+   uint32_t buf_addr_regs[MAX_BUFS];
+   uint32_t *buf_init_data[MAX_BUFS];
+
+   /* filled in by frontend before launching grid: */
+   struct fd_bo *bufs[MAX_BUFS];
+};
+
+struct perfcntr {
+   const char *name;
+
+   /* for backend to configure/read the counter, describes
+    * the selected counter:
+    */
+   unsigned select_reg;
+   unsigned counter_reg_lo;
+   unsigned counter_reg_hi;
+   /* and selected countable:
+    */
+   unsigned selector;
+};
+
+/* per-generation entry-points: */
+struct backend {
+   struct kernel *(*assemble)(struct backend *b, FILE *in);
+   void (*disassemble)(struct kernel *kernel, struct ir3_disasm_options *);
+   void (*emit_grid)(struct kernel *kernel, uint32_t grid[3],
+                     struct fd_submit *submit);
+
+   /* performance-counter API: */
+   void (*set_perfcntrs)(struct backend *b, const struct perfcntr *perfcntrs,
+                         unsigned num_perfcntrs);
+   void (*read_perfcntrs)(struct backend *b, uint64_t *results);
+};
+
+#define define_cast(_from, _to)                                                \
+   static inline struct _to *to_##_to(struct _from *f)                         \
+   {                                                                           \
+      return (struct _to *)f;                                                  \
+   }
+
+struct backend *a4xx_init(struct fd_device *dev, const struct fd_dev_id *dev_id);
+template<chip CHIP>
+struct backend *a6xx_init(struct fd_device *dev, const struct fd_dev_id *dev_id);
+
+/* for conditionally setting boolean flag(s): */
+#define COND(bool, val) ((bool) ? (val) : 0)
+
+#endif /* __MAIN_H__ */

src/freedreno/computerator/meson.build

@@ -0,0 +1,44 @@
+# Copyright © 2020 Google, Inc
+# SPDX-License-Identifier: MIT
+
+computerator_files = [
+  'a4xx.cc',
+  'a6xx.cc',
+  'ir3_asm.cc',
+  'main.cc',
+  freedreno_xml_header_files,
+  ir3_parser[1],
+]
+
+computerator_cpp_args = cpp.get_supported_arguments([
+  '-Wno-array-bounds',
+  '-Wno-c++11-narrowing',
+  '-Wno-narrowing',
+  '-Wno-sign-compare',
+])
+
+computerator = executable(
+  'computerator',
+  computerator_files,
+  include_directories : [
+    inc_freedreno,
+    inc_include,
+    inc_src,
+  ],
+  link_with : [
+    libfreedreno_drm,
+    libfreedreno_ir3,
+    libfreedreno_layout,
+    libfreedreno_perfcntrs,
+    libfreedreno_common,
+  ],
+  dependencies : [
+    dep_libdrm,
+    idep_mesautil,
+    # We don't actually use nir, but ir3 wants some nir headers:
+    idep_nir,
+  ],
+  cpp_args : [computerator_cpp_args],
+  build_by_default : with_tools.contains('freedreno'),
+  install : with_tools.contains('freedreno'),
+)