init

src/freedreno/computerator/a6xx.cc

@@ -0,0 +1,720 @@
+/*
+ * Copyright © 2020 Google, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "ir3/ir3_compiler.h"
+
+#include "util/u_math.h"
+
+#include "fd6_hw.h"
+
+#include "common/freedreno_dev_info.h"
+#include "fdl/freedreno_layout.h"
+
+#include "ir3_asm.h"
+#include "main.h"
+
+#define FD_BO_NO_HARDPIN 1
+#include "common/fd6_pack.h"
+
+struct a6xx_backend {
+   struct backend base;
+
+   struct ir3_compiler *compiler;
+   struct fd_device *dev;
+
+   const struct fd_dev_info *info;
+
+   unsigned seqno;
+   struct fd_bo *control_mem;
+
+   struct fd_bo *query_mem;
+   const struct perfcntr *perfcntrs;
+   unsigned num_perfcntrs;
+};
+define_cast(backend, a6xx_backend);
+
+/*
+ * Data structures shared with GPU:
+ */
+
+/* This struct defines the layout of the fd6_context::control buffer: */
+struct fd6_control {
+   uint32_t seqno; /* seqno for async CP_EVENT_WRITE, etc */
+   uint32_t _pad0;
+   volatile uint32_t vsc_overflow;
+   uint32_t _pad1;
+   /* flag set from cmdstream when VSC overflow detected: */
+   uint32_t vsc_scratch;
+   uint32_t _pad2;
+   uint32_t _pad3;
+   uint32_t _pad4;
+
+   /* scratch space for VPC_SO[i].FLUSH_BASE_LO/HI, start on 32 byte boundary. */
+   struct {
+      uint32_t offset;
+      uint32_t pad[7];
+   } flush_base[4];
+};
+
+#define control_ptr(a6xx_backend, member)                                      \
+   (a6xx_backend)->control_mem, offsetof(struct fd6_control, member)
+
+struct PACKED fd6_query_sample {
+   uint64_t start;
+   uint64_t result;
+   uint64_t stop;
+};
+
+/* offset of a single field of an array of fd6_query_sample: */
+#define query_sample_idx(a6xx_backend, idx, field)                             \
+   (a6xx_backend)->query_mem,                                                  \
+      (idx * sizeof(struct fd6_query_sample)) +                                \
+         offsetof(struct fd6_query_sample, field)
+
+/*
+ * Backend implementation:
+ */
+
+static struct kernel *
+a6xx_assemble(struct backend *b, FILE *in)
+{
+   struct a6xx_backend *a6xx_backend = to_a6xx_backend(b);
+   struct ir3_kernel *ir3_kernel = ir3_asm_assemble(a6xx_backend->compiler, in);
+   ir3_kernel->backend = b;
+   return &ir3_kernel->base;
+}
+
+static void
+a6xx_disassemble(struct kernel *kernel, struct ir3_disasm_options *options)
+{
+   ir3_asm_disassemble(to_ir3_kernel(kernel), options);
+}
+
+template<chip CHIP>
+static void
+cs_restore_emit(fd_cs &cs, struct a6xx_backend *a6xx_backend)
+{
+   fd_ncrb<CHIP> ncrb(cs, 2 + ARRAY_SIZE(a6xx_backend->info->magic_raw));
+
+   ncrb.add(A6XX_SP_PERFCTR_SHADER_MASK(.cs = true));
+
+   /* KMD programs this (and blocks UMD access) on gen8+: */
+   if (CHIP < A8XX)
+      ncrb.add(A6XX_SP_NC_MODE_CNTL_2());
+
+   for (size_t i = 0; i < ARRAY_SIZE(a6xx_backend->info->magic_raw); i++) {
+      auto magic_reg = a6xx_backend->info->magic_raw[i];
+      if (!magic_reg.reg)
+         break;
+
+      ncrb.add({magic_reg.reg, magic_reg.value});
+   }
+}
+
+template<chip CHIP>
+static void
+cs_program_emit_regs(fd_cs &cs, struct kernel *kernel)
+{
+   struct ir3_kernel *ir3_kernel = to_ir3_kernel(kernel);
+   struct a6xx_backend *a6xx_backend = to_a6xx_backend(ir3_kernel->backend);
+   struct ir3_shader_variant *v = ir3_kernel->v;
+   const unsigned *local_size = kernel->local_size;
+   const struct ir3_info *i = &v->info;
+   enum a6xx_threadsize thrsz = i->double_threadsize ? THREAD128 : THREAD64;
+   fd_crb crb(cs, 25);
+
+   crb.add(A6XX_SP_MODE_CNTL(
+      .constant_demotion_enable = true,
+      .isammode = ISAMMODE_GL,
+      .shared_consts_enable = false,
+   ));
+
+   crb.add(SP_UPDATE_CNTL(CHIP,
+      .vs_state = true,
+      .hs_state = true,
+      .ds_state = true,
+      .gs_state = true,
+      .fs_state = true,
+      .cs_state = true,
+      .gfx_uav = true,
+   ));
+
+   unsigned constlen = align(v->constlen, 4);
+   crb.add(SP_CS_CONST_CONFIG(CHIP, .constlen = constlen, .enabled = true, ));
+
+   crb.add(A6XX_SP_CS_CONFIG(
+      .enabled = true,
+      .ntex = v->num_samp,
+      .nsamp = v->num_samp,
+      .nuav = kernel->num_bufs,
+   ));
+   crb.add(A6XX_SP_CS_INSTR_SIZE(v->instrlen));
+
+   crb.add(A6XX_SP_CS_CNTL_0(
+      .halfregfootprint = i->max_half_reg + 1,
+      .fullregfootprint = i->max_reg + 1,
+      .branchstack = ir3_shader_branchstack_hw(v),
+      .threadsize = thrsz,
+      .earlypreamble = v->early_preamble,
+      .mergedregs = v->mergedregs,
+   ));
+
+   if (CHIP >= A7XX) {
+      crb.add(SP_PS_WAVE_CNTL(CHIP, .threadsize = THREAD64));
+
+      crb.add(SP_REG_PROG_ID_0(CHIP, .dword = 0xfcfcfcfc));
+      crb.add(SP_REG_PROG_ID_1(CHIP, .dword = 0xfcfcfcfc));
+      crb.add(SP_REG_PROG_ID_2(CHIP, .dword = 0xfcfcfcfc));
+      crb.add(SP_REG_PROG_ID_3(CHIP, .dword = 0x0000fc00));
+   }
+
+   uint32_t shared_size = MAX2(((int)v->shared_size - 1) / 1024, 1);
+   enum a6xx_const_ram_mode mode =
+      v->constlen > 256 ? CONSTLEN_512 :
+      (v->constlen > 192 ? CONSTLEN_256 :
+      (v->constlen > 128 ? CONSTLEN_192 : CONSTLEN_128));
+   crb.add(A6XX_SP_CS_CNTL_1(.shared_size = shared_size, .constantrammode = mode));
+
+   if (CHIP == A6XX && a6xx_backend->info->props.has_lpac) {
+      crb.add(HLSQ_CS_CTRL_REG1(CHIP, .shared_size = 1, .constantrammode = mode));
+   }
+
+   uint32_t local_invocation_id, work_group_id;
+   local_invocation_id =
+      ir3_find_sysval_regid(v, SYSTEM_VALUE_LOCAL_INVOCATION_ID);
+   work_group_id = ir3_find_sysval_regid(v, SYSTEM_VALUE_WORKGROUP_ID);
+
+   if (CHIP == A6XX) {
+      crb.add(SP_CS_CONST_CONFIG_0(CHIP,
+         .wgidconstid = work_group_id,
+         .wgsizeconstid = INVALID_REG,
+         .wgoffsetconstid = INVALID_REG,
+         .localidregid = local_invocation_id,
+      ));
+      crb.add(SP_CS_WGE_CNTL(CHIP,
+         .linearlocalidregid = INVALID_REG,
+         .threadsize = thrsz,
+      ));
+   } else {
+      unsigned tile_height = (local_size[1] % 8 == 0)   ? 3
+                             : (local_size[1] % 4 == 0) ? 5
+                             : (local_size[1] % 2 == 0) ? 9
+                                                        : 17;
+
+      crb.add(SP_CS_WGE_CNTL(CHIP,
+         .linearlocalidregid = INVALID_REG,
+         .threadsize = thrsz,
+         .workgrouprastorderzfirsten = true,
+         .wgtilewidth = 4,
+         .wgtileheight = tile_height,
+      ));
+   }
+
+   if (CHIP >= A7XX || a6xx_backend->info->props.has_lpac) {
+      crb.add(A6XX_SP_CS_WIE_CNTL_0(
+         .wgidconstid = work_group_id,
+         .wgsizeconstid = INVALID_REG,
+         .wgoffsetconstid = INVALID_REG,
+         .localidregid = local_invocation_id,
+      ));
+
+      if (CHIP >= A7XX) {
+         /* TODO allow the shader to control the tiling */
+         crb.add(SP_CS_WIE_CNTL_1(CHIP,
+            .linearlocalidregid = INVALID_REG,
+            .threadsize = thrsz,
+            .workitemrastorder = WORKITEMRASTORDER_LINEAR,
+         ));
+      } else {
+         crb.add(SP_CS_WIE_CNTL_1(CHIP,
+            .linearlocalidregid = INVALID_REG,
+            .threadsize = thrsz,
+         ));
+      }
+   }
+
+   crb.attach_bo(v->bo);
+
+   crb.add(A6XX_SP_CS_BASE(v->bo));
+   crb.add(A6XX_SP_CS_INSTR_SIZE(v->instrlen));
+
+   if (v->pvtmem_size > 0) {
+      uint32_t per_fiber_size = v->pvtmem_size;
+      uint32_t per_sp_size =
+         align(per_fiber_size * a6xx_backend->info->fibers_per_sp, 1 << 12);
+      uint32_t total_size = per_sp_size * a6xx_backend->info->num_sp_cores;
+
+      struct fd_bo *pvtmem = fd_bo_new(a6xx_backend->dev, total_size, 0, "pvtmem");
+      crb.add(A6XX_SP_CS_PVT_MEM_PARAM(.memsizeperitem = per_fiber_size));
+      crb.add(A6XX_SP_CS_PVT_MEM_BASE(pvtmem));
+      crb.add(A6XX_SP_CS_PVT_MEM_SIZE(
+         .totalpvtmemsize = per_sp_size,
+         .perwavememlayout = v->pvtmem_per_wave,
+      ));
+
+      crb.add(A6XX_SP_CS_PVT_MEM_STACK_OFFSET(.offset = per_sp_size));
+   }
+}
+
+template<chip CHIP>
+static void
+cs_program_emit(fd_cs &cs, struct kernel *kernel)
+{
+   struct ir3_kernel *ir3_kernel = to_ir3_kernel(kernel);
+   struct a6xx_backend *a6xx_backend = to_a6xx_backend(ir3_kernel->backend);
+   struct ir3_shader_variant *v = ir3_kernel->v;
+
+   cs_program_emit_regs<CHIP>(cs, kernel);
+
+   uint32_t shader_preload_size =
+      MIN2(v->instrlen, a6xx_backend->info->props.instr_cache_size);
+
+   fd_pkt7(cs, CP_LOAD_STATE6_FRAG, 3)
+      .add(CP_LOAD_STATE6_0(
+         .state_type = ST6_SHADER,
+         .state_src = SS6_INDIRECT,
+         .state_block = SB6_CS_SHADER,
+         .num_unit = shader_preload_size,
+      ))
+      .add(CP_LOAD_STATE6_EXT_SRC_ADDR(v->bo));
+}
+
+template<chip CHIP>
+static void
+emit_const(fd_cs &cs, uint32_t regid, uint32_t sizedwords, const uint32_t *dwords)
+{
+   uint32_t zero[4] = {};
+   uint32_t align_sz;
+
+   assert((regid % 4) == 0);
+
+   align_sz = align(sizedwords, 4);
+
+   fd_pkt7(cs, CP_LOAD_STATE6_FRAG, 3 + align_sz)
+      .add(CP_LOAD_STATE6_0(
+         .dst_off = regid / 4,
+         .state_type = ST6_CONSTANTS,
+         .state_src = SS6_DIRECT,
+         .state_block = SB6_CS_SHADER,
+         .num_unit = DIV_ROUND_UP(sizedwords, 4)
+      ))
+      .add(CP_LOAD_STATE6_EXT_SRC_ADDR())
+      .add(dwords, sizedwords)
+      /* Zero-pad to multiple of 4 dwords */
+      .add(zero, align_sz - sizedwords);
+}
+
+template<chip CHIP>
+static void
+cs_const_emit(fd_cs &cs, struct kernel *kernel, uint32_t grid[3])
+{
+   struct ir3_kernel *ir3_kernel = to_ir3_kernel(kernel);
+   struct ir3_shader_variant *v = ir3_kernel->v;
+
+   const struct ir3_const_state *const_state = ir3_const_state(v);
+   uint32_t base = const_state->allocs.max_const_offset_vec4;
+   const struct ir3_imm_const_state *imm_state = &v->imm_state;
+   int size = DIV_ROUND_UP(imm_state->count, 4);
+
+   if (ir3_kernel->info.numwg != INVALID_REG) {
+      assert((ir3_kernel->info.numwg & 0x3) == 0);
+      int idx = ir3_kernel->info.numwg >> 2;
+      imm_state->values[idx * 4 + 0] = grid[0];
+      imm_state->values[idx * 4 + 1] = grid[1];
+      imm_state->values[idx * 4 + 2] = grid[2];
+   }
+
+   for (int i = 0; i < MAX_BUFS; i++) {
+      if (kernel->buf_addr_regs[i] != INVALID_REG) {
+         assert((kernel->buf_addr_regs[i] & 0x3) == 0);
+         int idx = kernel->buf_addr_regs[i] >> 2;
+
+         uint64_t iova = fd_bo_get_iova(kernel->bufs[i]);
+
+         imm_state->values[idx * 4 + 1] = iova >> 32;
+         imm_state->values[idx * 4 + 0] = (iova << 32) >> 32;
+      }
+   }
+
+   /* truncate size to avoid writing constants that shader
+    * does not use:
+    */
+   size = MIN2(size + base, v->constlen) - base;
+
+   /* convert out of vec4: */
+   base *= 4;
+   size *= 4;
+
+   if (size > 0) {
+      emit_const<CHIP>(cs, base, size, imm_state->values);
+   }
+}
+
+static unsigned
+kernel_num_bufs(struct kernel *kernel, enum kernel_buf_type buf_type)
+{
+   unsigned num_bufs = 0;
+
+   for (unsigned i = 0; i < kernel->num_bufs; i++) {
+      if (kernel->buf_types[i] == buf_type) {
+         num_bufs++;
+      }
+   }
+
+   return num_bufs;
+}
+
+template<chip CHIP>
+static void
+cs_uav_emit(fd_cs &cs, struct fd_device *dev, struct kernel *kernel)
+{
+   unsigned num_bufs = kernel_num_bufs(kernel, KERNEL_BUF_UAV);
+
+   if (num_bufs == 0) {
+      return;
+   }
+
+   struct fd_bo *state = fd_bo_new(dev, kernel->num_bufs * 16 * 4,
+                                   FD_BO_GPUREADONLY | FD_BO_HINT_COMMAND,
+                                   "tex_desc");
+   fd_bo_mark_for_dump(state);
+
+   cs.attach_bo(state);
+
+   uint32_t *buf = (uint32_t *)fd_bo_map(state);
+
+   for (unsigned i = 0; i < kernel->num_bufs; i++) {
+      if (kernel->buf_types[i] != KERNEL_BUF_UAV) {
+         continue;
+      }
+
+      cs.attach_bo(kernel->bufs[i]);
+
+      uint64_t iova = fd_bo_get_iova(kernel->bufs[i]);
+      uint32_t descriptor[16];
+
+      static const uint8_t swiz_identity[4] = {
+         PIPE_SWIZZLE_X, PIPE_SWIZZLE_Y,
+         PIPE_SWIZZLE_Z, PIPE_SWIZZLE_W,
+      };
+
+      fdl6_buffer_view_init<CHIP>(descriptor, PIPE_FORMAT_R32_UINT, swiz_identity,
+                                  iova, kernel->buf_sizes[i] * 4);
+
+      memcpy(buf, descriptor, 16 * 4);
+      buf += 16;
+   }
+
+   fd_pkt7(cs, CP_LOAD_STATE6_FRAG, 3)
+      .add(CP_LOAD_STATE6_0(
+         .state_type = ST6_UAV,
+         .state_src = SS6_INDIRECT,
+         .state_block = SB6_CS_SHADER,
+         .num_unit = num_bufs,
+      ))
+      .add(CP_LOAD_STATE6_EXT_SRC_ADDR(state));
+
+   fd_crb crb(cs, 3);
+
+   crb.add(SP_CS_UAV_BASE(CHIP, state));
+   crb.add(A6XX_SP_CS_USIZE(num_bufs));
+
+   fd_bo_del(state);
+}
+
+static void
+cs_ubo_emit(fd_cs &cs, struct kernel *kernel)
+{
+   unsigned num_bufs = kernel_num_bufs(kernel, KERNEL_BUF_UBO);
+
+   if (num_bufs == 0) {
+      return;
+   }
+
+   for (unsigned i = 0, offset = 0; i < kernel->num_bufs; i++) {
+      if (kernel->buf_types[i] != KERNEL_BUF_UBO) {
+         continue;
+      }
+
+      cs.attach_bo(kernel->bufs[i]);
+
+      unsigned size_vec4s = DIV_ROUND_UP(kernel->buf_sizes[i], 4);
+
+      fd_pkt7(cs, CP_LOAD_STATE6_FRAG, 5)
+         .add(CP_LOAD_STATE6_0(
+            .dst_off = offset,
+            .state_type = ST6_UBO,
+            .state_src = SS6_DIRECT,
+            .state_block = SB6_CS_SHADER,
+            .num_unit = 1,
+         ))
+         .add(CP_LOAD_STATE6_EXT_SRC_ADDR())
+         .add(A6XX_UBO_DESC(0, kernel->bufs[i], 0, size_vec4s));
+
+      offset++;
+   }
+}
+
+template<chip CHIP>
+static inline unsigned
+event_write(fd_cs &cs, struct kernel *kernel, enum vgt_event_type evt, bool timestamp)
+{
+   unsigned seqno = 0;
+   unsigned len = timestamp ? 4 : 1;
+
+   fd_pkt7 pkt(cs, CP_EVENT_WRITE, len);
+
+   if (CHIP == A6XX) {
+      pkt.add(CP_EVENT_WRITE_0_EVENT(evt));
+   } else {
+      pkt.add(CP_EVENT_WRITE7_0_EVENT(evt) |
+              CP_EVENT_WRITE7_0_WRITE_SRC(EV_WRITE_USER_32B) |
+              COND(timestamp, CP_EVENT_WRITE7_0_WRITE_ENABLED));
+   }
+
+   if (timestamp) {
+      struct ir3_kernel *ir3_kernel = to_ir3_kernel(kernel);
+      struct a6xx_backend *a6xx_backend = to_a6xx_backend(ir3_kernel->backend);
+      seqno = ++a6xx_backend->seqno;
+      pkt.add(CP_EVENT_WRITE_ADDR(control_ptr(a6xx_backend, seqno)));
+      pkt.add(seqno);
+   }
+
+   return seqno;
+}
+
+template<chip CHIP>
+static inline void
+cache_flush(fd_cs &cs, struct kernel *kernel)
+{
+   struct ir3_kernel *ir3_kernel = to_ir3_kernel(kernel);
+   struct a6xx_backend *a6xx_backend = to_a6xx_backend(ir3_kernel->backend);
+   unsigned seqno;
+
+   seqno = event_write<CHIP>(cs, kernel, RB_DONE_TS, true);
+
+   fd_pkt7(cs, CP_WAIT_REG_MEM, 6)
+      .add(CP_WAIT_REG_MEM_0(.function = WRITE_EQ, .poll = POLL_MEMORY))
+      .add(CP_WAIT_REG_MEM_POLL_ADDR(control_ptr(a6xx_backend, seqno)))
+      .add(CP_WAIT_REG_MEM_3(.ref = seqno))
+      .add(CP_WAIT_REG_MEM_4(.mask = ~0))
+      .add(CP_WAIT_REG_MEM_5(.delay_loop_cycles = 16));
+
+   if (CHIP == A6XX) {
+      seqno = event_write<CHIP>(cs, kernel, CACHE_FLUSH_TS, true);
+
+      fd_pkt7(cs, CP_WAIT_MEM_GTE, 4)
+         .add(CP_WAIT_MEM_GTE_0())
+         .add(CP_WAIT_MEM_GTE_POLL_ADDR(control_ptr(a6xx_backend, seqno)))
+         .add(CP_WAIT_MEM_GTE_3(.ref = seqno));
+   } else {
+      event_write<CHIP>(cs, kernel, CACHE_FLUSH7, false);
+   }
+}
+
+template<chip CHIP>
+static void
+a6xx_emit_grid(struct kernel *kernel, uint32_t grid[3],
+               struct fd_submit *submit)
+{
+   struct ir3_kernel *ir3_kernel = to_ir3_kernel(kernel);
+   struct a6xx_backend *a6xx_backend = to_a6xx_backend(ir3_kernel->backend);
+   fd_cs cs(fd_submit_new_ringbuffer(submit, 0,
+      (enum fd_ringbuffer_flags)(FD_RINGBUFFER_PRIMARY | FD_RINGBUFFER_GROWABLE)));
+
+   cs.attach_bo(a6xx_backend->control_mem);
+
+   cs_restore_emit<CHIP>(cs, a6xx_backend);
+   cs_program_emit<CHIP>(cs, kernel);
+   cs_const_emit<CHIP>(cs, kernel, grid);
+   cs_uav_emit<CHIP>(cs, a6xx_backend->dev, kernel);
+   cs_ubo_emit(cs, kernel);
+
+   if (CHIP >= A8XX) {
+      fd_pkt7(cs, CP_SET_MARKER, 1)
+         .add(A8XX_CP_SET_MARKER_0(.mode = RM6_COMPUTE));
+   } else{
+      fd_pkt7(cs, CP_SET_MARKER, 1)
+         .add(A6XX_CP_SET_MARKER_0(.mode = RM6_COMPUTE));
+   }
+
+   const unsigned *local_size = kernel->local_size;
+   const unsigned *num_groups = grid;
+
+   unsigned work_dim = 0;
+   for (int i = 0; i < 3; i++) {
+      if (!grid[i])
+         break;
+      work_dim++;
+   }
+
+   with_crb (cs, 11) {
+      crb.add(SP_CS_NDRANGE_0(CHIP,
+         .kerneldim = work_dim,
+         .localsizex = local_size[0] - 1,
+         .localsizey = local_size[1] - 1,
+         .localsizez = local_size[2] - 1,
+      ));
+
+      if (CHIP >= A7XX) {
+         crb.add(SP_CS_NDRANGE_7(CHIP,
+            .localsizex = local_size[0] - 1,
+            .localsizey = local_size[1] - 1,
+            .localsizez = local_size[2] - 1,
+         ));
+      }
+
+      crb.add(SP_CS_NDRANGE_1(CHIP,
+         .globalsize_x = local_size[0] * num_groups[0],
+      ));
+      crb.add(SP_CS_NDRANGE_2(CHIP, 0));
+      crb.add(SP_CS_NDRANGE_3(CHIP,
+         .globalsize_y = local_size[1] * num_groups[1],
+      ));
+      crb.add(SP_CS_NDRANGE_4(CHIP, 0));
+      crb.add(SP_CS_NDRANGE_5(CHIP,
+         .globalsize_z = local_size[2] * num_groups[2],
+      ));
+      crb.add(SP_CS_NDRANGE_6(CHIP, 0));
+
+      crb.add(SP_CS_KERNEL_GROUP_X(CHIP, 1));
+      crb.add(SP_CS_KERNEL_GROUP_Y(CHIP, 1));
+      crb.add(SP_CS_KERNEL_GROUP_Z(CHIP, 1));
+   }
+
+   if (a6xx_backend->num_perfcntrs > 0) {
+      a6xx_backend->query_mem = fd_bo_new(
+         a6xx_backend->dev,
+         a6xx_backend->num_perfcntrs * sizeof(struct fd6_query_sample), 0, "query");
+
+      /* configure the performance counters to count the requested
+       * countables:
+       */
+      for (unsigned i = 0; i < a6xx_backend->num_perfcntrs; i++) {
+         const struct perfcntr *counter = &a6xx_backend->perfcntrs[i];
+
+         fd_pkt4(cs, 1).add({
+            .reg = counter->select_reg,
+            .value = counter->selector,
+         });
+      }
+
+      fd_pkt7(cs, CP_WAIT_FOR_IDLE, 0);
+
+      /* and snapshot the start values: */
+      for (unsigned i = 0; i < a6xx_backend->num_perfcntrs; i++) {
+         const struct perfcntr *counter = &a6xx_backend->perfcntrs[i];
+
+         fd_pkt7(cs, CP_REG_TO_MEM, 3)
+            .add(CP_REG_TO_MEM_0(.reg = counter->counter_reg_lo, ._64b = true))
+            .add(A5XX_CP_REG_TO_MEM_DEST(query_sample_idx(a6xx_backend, i, start)));
+      }
+   }
+
+   fd_pkt7(cs, CP_EXEC_CS, 4)
+      .add(CP_EXEC_CS_0())
+      .add(CP_EXEC_CS_1(.ngroups_x = grid[0]))
+      .add(CP_EXEC_CS_2(.ngroups_y = grid[1]))
+      .add(CP_EXEC_CS_3(.ngroups_z = grid[2]));
+
+   fd_pkt7(cs, CP_WAIT_FOR_IDLE, 0);
+
+   if (a6xx_backend->num_perfcntrs > 0) {
+      /* snapshot the end values: */
+      for (unsigned i = 0; i < a6xx_backend->num_perfcntrs; i++) {
+         const struct perfcntr *counter = &a6xx_backend->perfcntrs[i];
+
+         fd_pkt7(cs, CP_REG_TO_MEM, 3)
+            .add(CP_REG_TO_MEM_0(.reg = counter->counter_reg_lo, ._64b = true))
+            .add(A5XX_CP_REG_TO_MEM_DEST(query_sample_idx(a6xx_backend, i, stop)));
+      }
+
+      /* and compute the result: */
+      for (unsigned i = 0; i < a6xx_backend->num_perfcntrs; i++) {
+         /* result += stop - start: */
+         fd_pkt7(cs, CP_MEM_TO_MEM, 9)
+            .add(CP_MEM_TO_MEM_0(.neg_c = true, ._double = true))
+            .add(CP_MEM_TO_MEM_DST(query_sample_idx(a6xx_backend, i, result)))
+            .add(CP_MEM_TO_MEM_SRC_A(query_sample_idx(a6xx_backend, i, result)))
+            .add(CP_MEM_TO_MEM_SRC_B(query_sample_idx(a6xx_backend, i, stop)))
+            .add(CP_MEM_TO_MEM_SRC_C(query_sample_idx(a6xx_backend, i, start)));
+      }
+   }
+
+   cache_flush<CHIP>(cs, kernel);
+}
+
+static void
+a6xx_set_perfcntrs(struct backend *b, const struct perfcntr *perfcntrs,
+                   unsigned num_perfcntrs)
+{
+   struct a6xx_backend *a6xx_backend = to_a6xx_backend(b);
+
+   a6xx_backend->perfcntrs = perfcntrs;
+   a6xx_backend->num_perfcntrs = num_perfcntrs;
+}
+
+static void
+a6xx_read_perfcntrs(struct backend *b, uint64_t *results)
+{
+   struct a6xx_backend *a6xx_backend = to_a6xx_backend(b);
+
+   fd_bo_cpu_prep(a6xx_backend->query_mem, NULL, FD_BO_PREP_READ);
+   struct fd6_query_sample *samples =
+      (struct fd6_query_sample *)fd_bo_map(a6xx_backend->query_mem);
+
+   for (unsigned i = 0; i < a6xx_backend->num_perfcntrs; i++) {
+      results[i] = samples[i].result;
+   }
+}
+
+template<chip CHIP>
+struct backend *
+a6xx_init(struct fd_device *dev, const struct fd_dev_id *dev_id)
+{
+   struct a6xx_backend *a6xx_backend =
+      (struct a6xx_backend *)calloc(1, sizeof(*a6xx_backend));
+
+   a6xx_backend->base = (struct backend){
+      .assemble = a6xx_assemble,
+      .disassemble = a6xx_disassemble,
+      .emit_grid = a6xx_emit_grid<CHIP>,
+      .set_perfcntrs = a6xx_set_perfcntrs,
+      .read_perfcntrs = a6xx_read_perfcntrs,
+   };
+
+   struct ir3_compiler_options compiler_options = {
+      .disable_cache = true,
+   };
+   a6xx_backend->compiler =
+      ir3_compiler_create(dev, dev_id, fd_dev_info_raw(dev_id), &compiler_options);
+   a6xx_backend->dev = dev;
+
+   a6xx_backend->info = fd_dev_info_raw(dev_id);
+
+   a6xx_backend->control_mem =
+      fd_bo_new(dev, 0x1000, 0, "control");
+
+   return &a6xx_backend->base;
+}
+FD_GENX(a6xx_init);