init

src/asahi/.clang-format

@@ -0,0 +1,8 @@
+
+BasedOnStyle: InheritParentConfig
+DisableFormat: false
+
+AlignConsecutiveBitFields: Consecutive
+ColumnLimit: 80
+BreakStringLiterals: false
+SpaceBeforeParens: ControlStatementsExceptControlMacros

src/asahi/ci/asahi-agx2-fails.txt

@@ -0,0 +1,19 @@
+# In current VKCTS 1.4.4.0
+dEQP-VK.reconvergence.terminate_invocation.bit_count,Fail
+
+dEQP-VK.sparse_resources.buffer.misc.null_address_read,Fail
+dEQP-VK.sparse_resources.buffer.misc.null_address_read_descriptors,Fail
+dEQP-VK.sparse_resources.buffer.misc.null_address_read_local_inv_idx,Fail
+dEQP-VK.sparse_resources.buffer.misc.null_address_read_local_inv_idx_descriptors,Fail
+dEQP-VK.sparse_resources.buffer.ssbo.read_write.sparse_residency_non_resident_strict,Fail
+dEQP-VK.sparse_resources.buffer.ssbo.sparse_residency.non_resident_buffer_strict_copy_dest_alloc_none_1024,Fail
+dEQP-VK.sparse_resources.buffer.ssbo.sparse_residency.non_resident_buffer_strict_copy_dest_alloc_none_16777216,Fail
+dEQP-VK.sparse_resources.buffer.ssbo.sparse_residency.non_resident_buffer_strict_copy_dest_alloc_none_65536,Fail
+dEQP-VK.sparse_resources.buffer.ssbo.sparse_residency.non_resident_buffer_strict_copy_dest_alloc_partial_16777216,Fail
+dEQP-VK.sparse_resources.buffer.ssbo.sparse_residency.non_resident_buffer_strict_copy_dest_alloc_partial_65536,Fail
+
+# VKCTS 1.4.4.2 -> 1.4.5.3 uprev
+dEQP-VK.compute.shader_object_binary.basic.copy_ssbo_bounds,Fail
+dEQP-VK.compute.shader_object_spirv.basic.copy_ssbo_bounds,Fail
+dEQP-VK.shader_object.tessellation.hlsl.point_mode,Fail
+dEQP-VK.wsi.xcb.present_timing.basic.timing_queue,Fail

src/asahi/ci/asahi-agx2-skips.txt

@@ -0,0 +1,3 @@
+# https://gitlab.freedesktop.org/mesa/mesa/-/issues/15116
+dEQP-VK.wsi.xlib.maintenance1.present_modes.*
+dEQP-VK.wsi.xlib.maintenance1.present_fence.fifo.*

src/asahi/ci/asahi-skips.txt

@@ -0,0 +1,3 @@
+# CTS for subgroups with softfp64 is extremely slow and causes timeouts.
+KHR-Single-GL46.subgroups.*double.*
+KHR-Single-GL46.subgroups.*dvec[234]*

src/asahi/ci/deqp-asahi-agx2-gl.toml

@@ -0,0 +1,57 @@
+# full OpenGL ES test set
+[[deqp]]
+deqp = "/deqp-gles/external/openglcts/modules/glcts"
+caselists = [
+    "/deqp-gles/mustpass/gles2-main.txt",
+    "/deqp-gles/mustpass/gles3-main.txt",
+    "/deqp-gles/mustpass/gles31-main.txt",
+    "/deqp-gles/mustpass/gles2-khr-main.txt",
+    "/deqp-gles/mustpass/gles3-khr-main.txt",
+    "/deqp-gles/mustpass/gles31-khr-main.txt",
+]
+deqp_args = [
+    "--deqp-surface-width=256", "--deqp-surface-height=256",
+    "--deqp-surface-type=pbuffer", "--deqp-visibility=hidden",
+    "--deqp-gl-config-name=rgba8888d24s8ms0",
+]
+renderer_check = "Apple M[12]"
+tests_per_group = 1000
+version_check = "GL ES 3.2.*git"
+
+# Note that KHR-GL* test sets include all tests from the previous
+# version, so we only need to run one test list (unlike dEQP-GLES,
+# where the test sets are separate).
+[[deqp]]
+deqp = "/deqp-gl/external/openglcts/modules/glcts"
+caselists = [
+    "/deqp-gl/mustpass/gl46-main.txt",
+]
+deqp_args = [
+    "--deqp-surface-width=256", "--deqp-surface-height=256",
+    "--deqp-surface-type=pbuffer", "--deqp-visibility=hidden",
+    "--deqp-gl-config-name=rgba8888d24s8ms0",
+]
+renderer_check = "Apple M[12]"
+tests_per_group = 1000
+
+# wayland
+[[deqp]]
+deqp = "/deqp-gles/modules/egl/deqp-egl-wayland"
+caselists = ["/deqp-gles/mustpass/egl-main.txt"]
+deqp_args = [
+    "--deqp-surface-width=256", "--deqp-surface-height=256",
+    "--deqp-surface-type=pbuffer", "--deqp-visibility=hidden",
+    "--deqp-gl-config-name=rgba8888d24s8ms0",
+]
+prefix = "wayland-"
+
+# x11
+[[deqp]]
+deqp = "/deqp-gles/modules/egl/deqp-egl-x11"
+caselists = ["/deqp-gles/mustpass/egl-main.txt"]
+deqp_args = [
+    "--deqp-surface-width=256", "--deqp-surface-height=256",
+    "--deqp-surface-type=pbuffer", "--deqp-visibility=hidden",
+    "--deqp-gl-config-name=rgba8888d24s8ms0",
+]
+prefix = "x11-"

src/asahi/ci/deqp-asahi-agx2-vk.toml

@@ -0,0 +1,5 @@
+[[deqp]]
+deqp = "/deqp-vk/external/vulkancts/modules/vulkan/deqp-vk"
+caselists = ["/deqp-vk/mustpass/vk-main.txt"]
+tests_per_group = 10000
+renderer_check = "Apple M[12]"

src/asahi/clc/asahi_clc.c

@@ -0,0 +1,391 @@
+/*
+ * Copyright 2023 Alyssa Rosenzweig
+ * Copyright 2020 Intel Corporation
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "asahi_clc.h"
+#include "asahi/compiler/agx_compile.h"
+#include "asahi/compiler/agx_nir.h"
+#include "compiler/glsl_types.h"
+#include "compiler/spirv/nir_spirv.h"
+#include "nir.h"
+#include "nir_builder.h"
+#include "nir_builder_opcodes.h"
+#include "nir_intrinsics.h"
+#include "nir_precompiled.h"
+#include "shader_enums.h"
+
+#include <fcntl.h>
+#include <inttypes.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include "util/macros.h"
+#include <sys/mman.h>
+
+const char *targets[] = {"g13g", "g13x"};
+
+#define foreach_target(target)                                                 \
+   for (const char **target = &targets[0];                                     \
+        target < &targets[ARRAY_SIZE(targets)]; ++target)
+
+static const struct spirv_to_nir_options spirv_options = {
+   .environment = NIR_SPIRV_OPENCL,
+   .shared_addr_format = nir_address_format_62bit_generic,
+   .global_addr_format = nir_address_format_62bit_generic,
+   .temp_addr_format = nir_address_format_62bit_generic,
+   .constant_addr_format = nir_address_format_64bit_global,
+   .create_library = true,
+   .printf = true,
+};
+
+/* Standard optimization loop */
+static void
+optimize(nir_shader *nir)
+{
+   bool progress;
+   do {
+      progress = false;
+
+      NIR_PASS(progress, nir, nir_split_var_copies);
+      NIR_PASS(progress, nir, nir_split_struct_vars, nir_var_function_temp);
+      NIR_PASS(progress, nir, nir_lower_var_copies);
+      NIR_PASS(progress, nir, nir_lower_vars_to_ssa);
+
+      NIR_PASS(progress, nir, nir_opt_copy_prop);
+      NIR_PASS(progress, nir, nir_opt_remove_phis);
+      NIR_PASS(progress, nir, nir_lower_all_phis_to_scalar);
+      NIR_PASS(progress, nir, nir_opt_dce);
+      NIR_PASS(progress, nir, nir_opt_dead_cf);
+      NIR_PASS(progress, nir, nir_opt_cse);
+
+      nir_opt_peephole_select_options peephole_select_options = {
+         .limit = 64,
+         .expensive_alu_ok = true,
+      };
+      NIR_PASS(progress, nir, nir_opt_peephole_select,
+               &peephole_select_options);
+      NIR_PASS(progress, nir, nir_opt_phi_precision);
+      NIR_PASS(progress, nir, nir_opt_algebraic);
+      NIR_PASS(progress, nir, nir_opt_constant_folding);
+
+      NIR_PASS(progress, nir, nir_opt_deref);
+      NIR_PASS(progress, nir, nir_opt_copy_prop_vars);
+      NIR_PASS(progress, nir, nir_opt_undef);
+      NIR_PASS(progress, nir, nir_lower_undef_to_zero);
+
+      NIR_PASS(progress, nir, nir_opt_shrink_vectors, true);
+      NIR_PASS(progress, nir, nir_opt_loop_unroll);
+
+   } while (progress);
+}
+
+static nir_shader *
+compile(void *memctx, const uint32_t *spirv, size_t spirv_size)
+{
+   const nir_shader_compiler_options *nir_options = &agx_nir_options;
+
+   assert(spirv_size % 4 == 0);
+   nir_shader *nir =
+      spirv_to_nir(spirv, spirv_size / 4, NULL, MESA_SHADER_KERNEL, "library",
+                   &spirv_options, nir_options);
+   nir_validate_shader(nir, "after spirv_to_nir");
+   ralloc_steal(memctx, nir);
+
+   nir_fixup_is_exported(nir);
+
+   NIR_PASS(_, nir, nir_lower_system_values);
+   NIR_PASS(_, nir, nir_lower_calls_to_builtins);
+
+   nir_lower_compute_system_values_options cs = {.global_id_is_32bit = true};
+   NIR_PASS(_, nir, nir_lower_compute_system_values, &cs);
+
+   NIR_PASS(_, nir, nir_lower_printf,
+            &(const struct nir_lower_printf_options){
+               .hash_format_strings = true,
+            });
+
+   /* We have to lower away local constant initializers right before we
+    * inline functions.  That way they get properly initialized at the top
+    * of the function and not at the top of its caller.
+    */
+   NIR_PASS(_, nir, nir_lower_variable_initializers, nir_var_function_temp);
+   NIR_PASS(_, nir, nir_lower_returns);
+   NIR_PASS(_, nir, nir_inline_functions);
+   nir_remove_non_exported(nir);
+   NIR_PASS(_, nir, nir_opt_copy_prop);
+   NIR_PASS(_, nir, nir_opt_deref);
+
+   /* We can't deal with constant data, get rid of it */
+   nir_lower_constant_to_temp(nir);
+
+   /* We can go ahead and lower the rest of the constant initializers.  We do
+    * this here so that nir_remove_dead_variables and split_per_member_structs
+    * below see the corresponding stores.
+    */
+   NIR_PASS(_, nir, nir_lower_variable_initializers, ~0);
+
+   /* LLVM loves take advantage of the fact that vec3s in OpenCL are 16B
+    * aligned and so it can just read/write them as vec4s.  This results in a
+    * LOT of vec4->vec3 casts on loads and stores.  One solution to this
+    * problem is to get rid of all vec3 variables.
+    */
+   NIR_PASS(_, nir, nir_lower_vec3_to_vec4,
+            nir_var_shader_temp | nir_var_function_temp | nir_var_mem_shared |
+               nir_var_mem_global | nir_var_mem_constant);
+
+   /* We assign explicit types early so that the optimizer can take advantage
+    * of that information and hopefully get rid of some of our memcpys.
+    */
+   NIR_PASS(_, nir, nir_lower_vars_to_explicit_types,
+            nir_var_uniform | nir_var_shader_temp | nir_var_function_temp |
+               nir_var_mem_shared | nir_var_mem_global,
+            glsl_get_cl_type_size_align);
+
+   optimize(nir);
+
+   NIR_PASS(_, nir, nir_remove_dead_variables, nir_var_all, NULL);
+
+   /* Lower again, this time after dead-variables to get more compact variable
+    * layouts.
+    */
+   NIR_PASS(_, nir, nir_lower_vars_to_explicit_types,
+            nir_var_shader_temp | nir_var_function_temp | nir_var_mem_shared |
+               nir_var_mem_global | nir_var_mem_constant,
+            glsl_get_cl_type_size_align);
+   assert(nir->constant_data_size == 0);
+
+   NIR_PASS(_, nir, nir_lower_memcpy);
+
+   NIR_PASS(_, nir, nir_lower_explicit_io, nir_var_mem_constant,
+            nir_address_format_64bit_global);
+
+   NIR_PASS(_, nir, nir_lower_explicit_io, nir_var_uniform,
+            nir_address_format_32bit_offset_as_64bit);
+
+   /* Note: we cannot lower explicit I/O here, because we need derefs in tact
+    * for function calls into the library to work.
+    */
+
+   NIR_PASS(_, nir, nir_lower_convert_alu_types, NULL);
+   NIR_PASS(_, nir, nir_opt_if, 0);
+   NIR_PASS(_, nir, nir_opt_idiv_const, 16);
+
+   NIR_PASS(_, nir, agx_nir_lower_texture_early, false /* support_lod_bias */);
+   NIR_PASS(_, nir, agx_nir_lower_texture);
+   NIR_PASS(_, nir, agx_nir_lower_multisampled_image_store);
+
+   optimize(nir);
+
+   return nir;
+}
+
+static void
+print_shader(FILE *fp, const char *name, const char *suffix, uint32_t variant,
+             struct agx_shader_part *p)
+{
+   struct agx_precompiled_kernel_info info = agx_compact_kernel_info(&p->info);
+   size_t sz_B = sizeof(info) + p->info.binary_size;
+   size_t sz_el = DIV_ROUND_UP(sz_B, 4);
+   uint32_t *mem = calloc(sz_el, 4);
+
+   memcpy(mem, &info, sizeof(info));
+   memcpy((uint8_t *)mem + sizeof(info), p->binary, p->info.binary_size);
+
+   nir_precomp_print_blob(fp, name, suffix, variant, mem, sz_B, true);
+   free(mem);
+}
+
+static bool
+gather_atomic_info(nir_builder *b, nir_intrinsic_instr *intr, void *data)
+{
+   bool *any = data;
+
+   switch (intr->intrinsic) {
+   case nir_intrinsic_global_atomic:
+   case nir_intrinsic_global_atomic_agx:
+   case nir_intrinsic_deref_atomic:
+   case nir_intrinsic_global_atomic_swap:
+   case nir_intrinsic_global_atomic_swap_agx:
+   case nir_intrinsic_deref_atomic_swap:
+      *any = true;
+      return false;
+   default:
+      return false;
+   }
+}
+
+/* G13X variants are only compiled when atomics are used */
+static const char *
+remap_variant(nir_function *func, unsigned variant, const char *target)
+{
+   /* pass_flags is only 32-bit */
+   assert(variant < 32 && "maximum # of variants");
+
+   bool has_atomic = func->pass_flags & BITFIELD_BIT(variant);
+
+   if (!has_atomic && !strcmp(target, "g13x"))
+      return "g13g";
+   else
+      return target;
+}
+
+static nir_def *
+load_kernel_input(nir_builder *b, unsigned num_components, unsigned bit_size,
+                  unsigned offset_B)
+{
+   assert((offset_B & 1) == 0 && "half-aligned");
+   return nir_load_preamble(b, num_components, bit_size, .base = offset_B / 2);
+}
+
+int
+main(int argc, char **argv)
+{
+   if (argc != 4) {
+      fprintf(stderr, "Usage: %s [input spir-v] [output header] [output C]\n",
+              argv[0]);
+      return 1;
+   }
+
+   const char *infile = argv[1];
+   const char *outh_file = argv[2];
+   const char *outc_file = argv[3];
+
+   void *mem_ctx = ralloc_context(NULL);
+
+   int fd = open(infile, O_RDONLY);
+   if (fd < 0) {
+      fprintf(stderr, "Failed to open %s\n", infile);
+      ralloc_free(mem_ctx);
+      return 1;
+   }
+
+   off_t spirv_len = lseek(fd, 0, SEEK_END);
+   const void *spirv_map = mmap(NULL, spirv_len, PROT_READ, MAP_PRIVATE, fd, 0);
+   close(fd);
+   if (spirv_map == MAP_FAILED) {
+      fprintf(stderr, "Failed to mmap the file: errno=%d, %s\n", errno,
+              strerror(errno));
+      ralloc_free(mem_ctx);
+      return 1;
+   }
+
+   FILE *fp_h = fopen(outh_file, "w");
+   FILE *fp_c = fopen(outc_file, "w");
+   glsl_type_singleton_init_or_ref();
+
+   nir_precomp_print_header(fp_c, fp_h, "The Asahi Linux Contributors",
+                            "libagx_shaders.h");
+
+   nir_shader *nir = compile(mem_ctx, spirv_map, spirv_len);
+
+   /* load_preamble works at 16-bit granularity */
+   struct nir_precomp_opts opt = {.arg_align_B = 2};
+
+   nir_foreach_entrypoint(libfunc, nir) {
+      libfunc->pass_flags = 0;
+      struct nir_precomp_layout layout =
+         nir_precomp_derive_layout(&opt, libfunc);
+      unsigned nr_vars = nir_precomp_nr_variants(libfunc);
+
+      nir_precomp_print_layout_struct(fp_h, &opt, libfunc);
+
+      for (unsigned v = 0; v < nr_vars; ++v) {
+         nir_shader *s = nir_precompiled_build_variant(
+            libfunc, MESA_SHADER_COMPUTE, v, &agx_nir_options, &opt,
+            load_kernel_input);
+
+         nir_link_shader_functions(s, nir);
+         NIR_PASS(_, s, nir_inline_functions);
+         nir_remove_non_entrypoints(s);
+         NIR_PASS(_, s, nir_opt_deref);
+         NIR_PASS(_, s, nir_lower_vars_to_ssa);
+         NIR_PASS(_, s, nir_remove_dead_derefs);
+         NIR_PASS(_, s, nir_remove_dead_variables,
+                  nir_var_function_temp | nir_var_shader_temp, NULL);
+         NIR_PASS(_, s, nir_lower_vars_to_explicit_types,
+                  nir_var_shader_temp | nir_var_function_temp,
+                  glsl_get_cl_type_size_align);
+
+         NIR_PASS(_, s, nir_lower_vars_to_explicit_types, nir_var_mem_shared,
+                  glsl_get_cl_type_size_align);
+
+         NIR_PASS(_, s, nir_lower_explicit_io, nir_var_mem_shared,
+                  nir_address_format_62bit_generic);
+
+         agx_preprocess_nir(s);
+
+         NIR_PASS(_, s, nir_opt_deref);
+         NIR_PASS(_, s, nir_lower_vars_to_ssa);
+         NIR_PASS(_, s, nir_lower_explicit_io,
+                  nir_var_shader_temp | nir_var_function_temp |
+                     nir_var_mem_shared | nir_var_mem_global,
+                  nir_address_format_62bit_generic);
+
+         bool has_atomic = false;
+         nir_shader_intrinsics_pass(s, gather_atomic_info, nir_metadata_all,
+                                    &has_atomic);
+         if (has_atomic) {
+            libfunc->pass_flags |= BITFIELD_BIT(v);
+         }
+
+         foreach_target(target)
+         {
+            /* Skip unused variants */
+            if (strcmp(*target, remap_variant(libfunc, v, *target)))
+               continue;
+
+            struct agx_shader_part compiled;
+            bool is_helper = !strcmp(libfunc->name, "libagx_helper");
+            struct agx_shader_key key = {
+               .promote_constants = !is_helper,
+
+               /* Most of the internal programs don't use textures at all, but
+                * the few that do are exclusively bindless, so we want to
+                * promote their access.
+                */
+               .promote_textures = true,
+
+               .reserved_preamble = layout.size_B / 2,
+               .is_helper = is_helper,
+            };
+
+            if (has_atomic) {
+               key.dev.needs_g13x_coherency =
+                  u_tristate_make(!strcmp(*target, "g13x"));
+            }
+
+            nir_shader *clone = nir_shader_clone(NULL, s);
+            agx_compile_shader_nir(clone, &key, &compiled);
+            print_shader(fp_c, libfunc->name, *target, v, &compiled);
+            free(compiled.binary);
+            ralloc_free(clone);
+
+            assert(compiled.info.scratch_size == 0 &&
+                   "internal shaders do not spill");
+
+            assert(compiled.info.preamble_scratch_size == 0 &&
+                   "internal shader preambles do not spill");
+         }
+
+         ralloc_free(s);
+      }
+   }
+
+   nir_precomp_print_program_enum(fp_h, nir, "libagx");
+   nir_precomp_print_dispatch_macros(fp_h, &opt, nir);
+
+   /* For each target, generate a table mapping programs to binaries */
+   foreach_target(target)
+   {
+      nir_precomp_print_extern_binary_map(fp_h, "libagx", *target);
+      nir_precomp_print_binary_map(fp_c, nir, "libagx", *target, remap_variant);
+   }
+
+   glsl_type_singleton_decref();
+   fclose(fp_c);
+   fclose(fp_h);
+   ralloc_free(mem_ctx);
+   return 0;
+}

src/asahi/clc/asahi_clc.h

@@ -0,0 +1,12 @@
+/*
+ * Copyright 2024 Valve Corporation
+ * SPDX-License-Identifier: MIT
+ */
+
+#pragma once
+
+/* The libagx printf/abort buffer addresses is fixed at compile-time for
+ * simplicity.
+ */
+#define LIBAGX_PRINTF_BUFFER_ADDRESS (1ull << 36)
+#define LIBAGX_PRINTF_BUFFER_SIZE    (16384)

src/asahi/clc/meson.build

@@ -0,0 +1,21 @@
+# Copyright 2017 Intel Corporation
+# SPDX-License-Identifier: MIT
+
+if get_option('precomp-compiler') == 'system'
+  prog_asahi_clc = find_program('asahi_clc', native : true)
+else
+  prog_asahi_clc = executable(
+    'asahi_clc',
+    ['asahi_clc.c'],
+    link_with : [libasahi_compiler],
+    include_directories : [inc_include, inc_src],
+    c_args : [pre_args, no_override_init_args],
+    link_args : [ld_args_build_id],
+    dependencies : [idep_vtn, idep_nir, idep_mesautil],
+    # If we can run host binaries directly, just build asahi_clc for the host.
+    # Most commonly this happens when doing a cross compile from an x86_64 build
+    # machine to an x86 host
+    native : not meson.can_run_host_binaries(),
+    install : get_option('install-precomp-compiler'),
+  )
+endif

src/asahi/compiler/README.md

@@ -0,0 +1,89 @@
+# Special registers
+
+`r0l` is the hardware nesting counter.
+
+`r1` is the hardware link register.
+
+`r5` and `r6` are preloaded in vertex shaders to the vertex ID and instance ID.
+
+# ABI
+
+The following section describes the ABI used by non-monolithic programs.
+
+## Vertex
+
+Registers have the following layout at the beginning of the vertex shader
+(written by the vertex prolog):
+
+* `r0-r3` and `r7` undefined. This avoids preloading into the nesting counter or
+  having unaligned values. The prolog is free to use these registers as
+  temporaries.
+* `r4` is the zero-based vertex ID if the vertex shader is running as a hardware
+  compute shader, useful to avoid a redundant special register read in the main
+  shader. Undefined in hardware vertex shaders.
+* `r5-r6` retain their usual meanings, even if the vertex shader is running as a
+  hardware compute shader. This allows software index fetch code to run in the
+  prolog without contaminating the main shader key.
+* `r8` onwards contains 128-bit uniform vectors for each attribute.
+  Accommodates 30 attributes without spilling, exceeding the 16 attribute API
+  minimum. For 32 attributes, we will need to use function calls or the stack.
+
+One useful property is that the GPR usage of the combined program is equal to
+the GPR usage of the main shader. The prolog cannot write higher registers than
+read by the main shader.
+
+Vertex prologs do not have any uniform registers allocated for preamble
+optimization or constant promotion, as this adds complexity without any
+legitimate use case.
+
+For a vertex shader reading $n$ attributes, the following layout is used:
+
+* The first $n$ 64-bit uniforms are the base addresses of each attribute.
+* The next $n$ 32-bit uniforms are the associated clamps (sizes). Presently
+  robustness is always used.
+* The next 2x32-bit uniform is the base vertex and base instance. This must
+  always be reserved because it is unknown at vertex shader compile-time whether
+  any attribute will use instancing. Reserving also the base vertex allows us to
+  push both conveniently with a single USC Uniform word.
+* The next 16-bit is the draw ID.
+* For a hardware compute shader, the next 48-bit is padding.
+* For a hardware compute shader, the next 64-bit uniform is a pointer to the
+  input assembly buffer.
+
+In total, the first $6n + 5$ 16-bit uniform slots are reserved for a hardware
+vertex shader, or $6n + 12$ for a hardware compute shader.
+
+## Fragment
+
+When sample shading is enabled in a non-monolithic fragment shader, the fragment
+shader has the following register inputs:
+
+* `r0l = 0`. This is the hardware nesting counter.
+* `r1l` is the mask of samples currently being shaded. This usually equals to
+  `1 << sample ID`, for "true" per-sample shading.
+
+When sample shading is disabled, no register inputs are defined. The fragment
+prolog (if present) may clobber whatever registers it pleases.
+
+Registers have the following layout at the end of the fragment shader (read by
+the fragment epilog):
+
+* `r0l = 0` if sample shading is enabled. This is implicitly true.
+* `r1l` preserved if sample shading is enabled.
+* `r2` and `r3l` contain the emitted depth/stencil respectively, if
+  depth and/or stencil are written by the fragment shader. Depth/stencil writes
+  must be deferred to the epilog for correctness when the epilog can discard
+  (i.e. when alpha-to-coverage is enabled).
+* `r3h` contains the logically emitted sample mask, if the fragment shader uses
+  forced early tests. This predicates the epilog's stores.
+* The vec4 of 32-bit registers beginning at `r(4 * (i + 1))` contains the colour
+  output for render target `i`. When dual source blending is enabled, there is
+  only a single render target and the dual source colour is treated as the
+  second render target (registers r8-r11).
+
+Uniform registers have the following layout:
+
+* u0_u1: 64-bit render target texture heap
+* u2...u5: Blend constant
+* u6_u7: Root descriptor, so we can fetch the 64-bit fragment invocation counter
+  address and (OpenGL only) the 64-bit polygon stipple address

src/asahi/compiler/agx_builder.h.py

@@ -0,0 +1,177 @@
+template = """/*
+ * Copyright 2021 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef _AGX_BUILDER_
+#define _AGX_BUILDER_
+
+#include "agx_compiler.h"
+#include "util/lut.h"
+
+static inline agx_instr *
+agx_alloc_instr(agx_builder *b, enum agx_opcode op, uint8_t nr_dests, uint8_t nr_srcs)
+{
+   size_t size = sizeof(agx_instr);
+   size += sizeof(agx_index) * nr_dests;
+   size += sizeof(agx_index) * nr_srcs;
+
+   agx_instr *I = (agx_instr *) rzalloc_size(b->shader, size);
+   I->dest = (agx_index *) (I + 1);
+   I->src = I->dest + nr_dests;
+
+   I->op = op;
+   I->nr_dests = nr_dests;
+   I->nr_srcs = nr_srcs;
+   return I;
+}
+
+% for opcode in opcodes:
+<%
+   op = opcodes[opcode]
+   dests = op.dests
+   srcs = op.srcs
+   imms = [x for x in op.imms if (x.name != 'scoreboard' or opcode == 'wait')]
+   suffix = "_to" if dests > 0 else ""
+   nr_dests = "nr_dests" if op.variable_dests else str(dests)
+   nr_srcs = "nr_srcs" if op.variable_srcs else str(srcs)
+%>
+
+static inline agx_instr *
+agx_${opcode}${suffix}(agx_builder *b
+
+% if op.variable_dests:
+   , unsigned nr_dests
+% endif
+
+% for dest in range(dests):
+   , agx_index dst${dest}
+% endfor
+
+% if op.variable_srcs:
+   , unsigned nr_srcs
+% endif
+
+% for src in range(srcs):
+   , agx_index src${src}
+% endfor
+
+% for imm in imms:
+   , ${imm.ctype} ${imm.name}
+% endfor
+
+) {
+   agx_instr *I = agx_alloc_instr(b, AGX_OPCODE_${opcode.upper()}, ${nr_dests}, ${nr_srcs});
+
+% for dest in range(dests):
+   I->dest[${dest}] = dst${dest};
+% endfor
+
+% for src in range(srcs):
+   I->src[${src}] = src${src};
+% endfor
+
+% for imm in imms:
+   I->${imm.name} = ${imm.name};
+% endfor
+
+   agx_builder_insert(&b->cursor, I);
+   return I;
+}
+
+% if dests == 1 and not op.variable_srcs and not op.variable_dests:
+static inline agx_index
+agx_${opcode}(agx_builder *b
+
+% if srcs == 0:
+   , unsigned size
+% endif
+
+% for src in range(srcs):
+   , agx_index src${src}
+% endfor
+
+% for imm in imms:
+   , ${imm.ctype} ${imm.name}
+% endfor
+
+) {
+<%
+   args = ["tmp"]
+   args += ["src" + str(i) for i in range(srcs)]
+   args += [imm.name for imm in imms]
+%>
+% if srcs == 0:
+   agx_index tmp = agx_temp(b->shader, agx_size_for_bits(size));
+% else:
+   agx_index tmp = agx_temp(b->shader, src0.size);
+% endif
+   agx_${opcode}_to(b, ${", ".join(args)});
+   return tmp;
+}
+% endif
+
+% endfor
+
+/* Convenience methods */
+#define BINOP_BITOP(name, expr)                                                      \
+   static inline agx_instr *                                                         \
+   agx_## name ##_to(agx_builder *b, agx_index dst0, agx_index src0, agx_index src1) \
+   {                                                                                 \
+      return agx_bitop_to(b, dst0, src0, src1, UTIL_LUT2(expr));                     \
+   }                                                                                 \
+                                                                                     \
+   static inline agx_index                                                           \
+   agx_## name (agx_builder *b, agx_index src0, agx_index src1)                      \
+   {                                                                                 \
+      agx_index tmp = agx_temp(b->shader, src0.size);                                \
+      agx_##name##_to(b, tmp, src0, src1);                                           \
+      return tmp;                                                                    \
+   }
+
+BINOP_BITOP(nor, ~(a | b))
+BINOP_BITOP(andn2, a & ~b)
+BINOP_BITOP(andn1, ~a & b)
+BINOP_BITOP(xor, a ^ b)
+BINOP_BITOP(nand, ~(a & b))
+BINOP_BITOP(and, a & b)
+BINOP_BITOP(xnor, ~a ^ b)
+BINOP_BITOP(orn2, a | ~b)
+BINOP_BITOP(orn1, ~a | b)
+BINOP_BITOP(or, a | b)
+
+#undef BINOP_BITOP
+
+static inline agx_instr *
+agx_fmov_to(agx_builder *b, agx_index dst0, agx_index src0)
+{
+   return agx_fadd_to(b, dst0, src0, agx_negzero());
+}
+
+static inline agx_instr *
+agx_push_exec(agx_builder *b, unsigned n)
+{
+   return agx_if_fcmp(b, agx_zero(), agx_zero(), n, AGX_FCOND_EQ, false, NULL);
+}
+
+static inline agx_instr *
+agx_ushr_to(agx_builder *b, agx_index dst, agx_index s0, agx_index s1)
+{
+    return agx_bfeil_to(b, dst, agx_zero(), s0, s1, 0);
+}
+
+static inline agx_index
+agx_ushr(agx_builder *b, agx_index s0, agx_index s1)
+{
+    agx_index tmp = agx_temp(b->shader, s0.size);
+    agx_ushr_to(b, tmp, s0, s1);
+    return tmp;
+}
+
+#endif
+"""
+
+from mako.template import Template
+from agx_opcodes import opcodes
+
+print(Template(template).render(opcodes=opcodes))

src/asahi/compiler/agx_compile.h

@@ -0,0 +1,409 @@
+/*
+ * Copyright 2018-2021 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#pragma once
+
+#include "compiler/nir/nir.h"
+#include "util/shader_stats.h"
+#include "util/u_dynarray.h"
+#include "util/u_tristate.h"
+#include "shader_enums.h"
+
+struct agx_cf_binding {
+   /* Base coefficient register */
+   uint8_t cf_base;
+
+   /* Slot being bound */
+   gl_varying_slot slot : 8;
+
+   /* First component bound.
+    *
+    * Must be 2 (Z) or 3 (W) if slot == VARYING_SLOT_POS.
+    */
+   unsigned offset : 2;
+
+   /* Number of components bound */
+   unsigned count : 3;
+
+   /* Is smooth shading enabled? If false, flat shading is used */
+   bool smooth : 1;
+
+   /* Perspective correct interpolation */
+   bool perspective : 1;
+
+   uint8_t pad;
+};
+
+/* Conservative bound, * 4 due to offsets (TODO: maybe worth eliminating
+ * coefficient register aliasing?)
+ */
+#define AGX_MAX_CF_BINDINGS (VARYING_SLOT_MAX * 4)
+
+struct agx_varyings_fs {
+   /* Number of coefficient registers used */
+   unsigned nr_cf;
+
+   /* Number of coefficient register bindings */
+   unsigned nr_bindings;
+
+   /* Whether gl_FragCoord.z is read */
+   bool reads_z;
+
+   /* Coefficient register bindings */
+   struct agx_cf_binding bindings[AGX_MAX_CF_BINDINGS];
+};
+
+union agx_varyings {
+   struct agx_varyings_fs fs;
+};
+
+struct agx_interp_info {
+   /* Bit masks indexed by I/O location of flat and linear varyings */
+   uint64_t flat;
+   uint64_t linear;
+};
+static_assert(sizeof(struct agx_interp_info) == 16, "packed");
+
+struct agx_rodata {
+   /* Offset in the binary */
+   uint32_t offset;
+
+   /* Base uniform to map constants */
+   uint16_t base_uniform;
+
+   /* Number of 16-bit constants to map contiguously there */
+   uint16_t size_16;
+};
+
+struct agx_shader_info {
+   mesa_shader_stage stage;
+   uint32_t binary_size;
+
+   union agx_varyings varyings;
+
+   /* Number of uniforms */
+   uint16_t push_count;
+
+   /* Local memory allocation in bytes */
+   uint16_t local_size;
+
+   /* Local imageblock allocation in bytes per thread */
+   uint16_t imageblock_stride;
+
+   /* Scratch memory allocation in bytes for main/preamble respectively */
+   unsigned scratch_size, preamble_scratch_size;
+
+   /* Size in bytes of the main sahder */
+   unsigned main_size;
+
+   /* Does the shader have a preamble? If so, it is at offset preamble_offset.
+    * The main shader is at offset main_offset. The preamble is executed first.
+    */
+   bool has_preamble;
+   unsigned preamble_offset, main_offset;
+
+   /* Does the shader read the tilebuffer? */
+   bool reads_tib;
+
+   /* Does the shader require early fragment tests? */
+   bool early_fragment_tests;
+
+   /* Does the shader potentially draw to a nonzero viewport? */
+   bool nonzero_viewport;
+
+   /* Does the shader write layer and/or viewport index? Written together */
+   bool writes_layer_viewport;
+
+   /* Does the shader control the sample mask? */
+   bool writes_sample_mask;
+
+   /* Depth layout, never equal to NONE */
+   enum gl_frag_depth_layout depth_layout;
+
+   /* Based only the compiled shader, should tag writes be disabled? This is set
+    * based on what is outputted. Note if rasterizer discard is used, that needs
+    * to disable tag writes regardless of this flag.
+    */
+   bool tag_write_disable;
+
+   /* Shader is incompatible with triangle merging */
+   bool disable_tri_merging;
+
+   /* Reads draw ID system value */
+   bool uses_draw_id;
+
+   /* Reads base vertex/instance */
+   bool uses_base_param;
+
+   /* Uses txf and hence needs a txf sampler mapped */
+   bool uses_txf;
+
+   /* Potentially uses the sampler heap (conservative) */
+   bool uses_sampler_heap;
+
+   /* Number of texture/sampler state registers pushed by the preamble. */
+   uint8_t texture_state_count, sampler_state_count;
+
+   /* Number of 16-bit registers used by the main shader and preamble
+    * respectively.
+    */
+   uint16_t nr_gprs, nr_preamble_gprs;
+
+   /* Output mask set during driver lowering */
+   uint64_t outputs;
+
+   /* Workgroup size */
+   uint16_t workgroup_size[3];
+
+   /* There may be constants in the binary. The driver must map these to uniform
+    * registers as specified hre.
+    */
+   struct agx_rodata rodata;
+
+   struct agx2_stats stats;
+};
+
+struct agx_precompiled_kernel_info {
+   uint32_t preamble_offset, main_offset;
+   uint32_t main_size, binary_size;
+   struct agx_rodata rodata;
+   uint16_t nr_gprs, nr_preamble_gprs;
+   uint16_t push_count;
+   uint16_t workgroup_size[3];
+   uint16_t local_size;
+   uint16_t imageblock_stride;
+   bool uses_txf;
+};
+
+static inline struct agx_precompiled_kernel_info
+agx_compact_kernel_info(struct agx_shader_info *info)
+{
+   assert(info->has_preamble == (info->nr_preamble_gprs > 0));
+   assert(info->texture_state_count <= 8 && "static maximum, no need to plumb");
+
+   return (struct agx_precompiled_kernel_info){
+      .preamble_offset = info->preamble_offset,
+      .main_offset = info->main_offset,
+      .main_size = info->main_size,
+      .binary_size = info->binary_size,
+      .rodata = info->rodata,
+      .nr_gprs = info->nr_gprs,
+      .nr_preamble_gprs = info->nr_preamble_gprs,
+      .push_count = info->push_count,
+      .workgroup_size = {info->workgroup_size[0], info->workgroup_size[1],
+                         info->workgroup_size[2]},
+      .local_size = info->local_size,
+      .imageblock_stride = info->imageblock_stride,
+      .uses_txf = info->uses_txf,
+   };
+}
+
+struct agx_shader_part {
+   struct agx_shader_info info;
+   void *binary;
+};
+
+static inline bool
+agx_is_shader_empty(struct agx_shader_part *s)
+{
+   /* Last instruction is a stop, so if there's one instruction, there is
+    * nothing but a stop. The shader is thus empty.
+    */
+   return (s->info.stats.instrs == 1);
+}
+
+#define AGX_MAX_RTS (8)
+
+enum agx_format {
+   AGX_FORMAT_I8 = 0,
+   AGX_FORMAT_I16 = 1,
+   AGX_FORMAT_I32 = 2,
+   AGX_FORMAT_F16 = 3,
+   AGX_FORMAT_U8NORM = 4,
+   AGX_FORMAT_S8NORM = 5,
+   AGX_FORMAT_U16NORM = 6,
+   AGX_FORMAT_S16NORM = 7,
+   AGX_FORMAT_RGB10A2 = 8,
+   AGX_FORMAT_SRGBA8 = 10,
+   AGX_FORMAT_RG11B10F = 12,
+   AGX_FORMAT_RGB9E5 = 13,
+
+   /* Keep last */
+   AGX_NUM_FORMATS,
+};
+
+struct agx_fs_shader_key {
+   /* Normally, access to the tilebuffer must be guarded by appropriate fencing
+    * instructions to ensure correct results in the presence of out-of-order
+    * hardware optimizations. However, specially dispatched clear shaders are
+    * not subject to these conditions and can omit the wait instructions.
+    *
+    * Must (only) be set for special clear shaders.
+    *
+    * Must not be used with sample mask writes (including discards) or
+    * tilebuffer loads (including blending).
+    */
+   bool ignore_tib_dependencies;
+
+   /* When dynamic sample shading is used, the fragment shader is wrapped in a
+    * loop external to the API shader. This bit indicates that we are compiling
+    * inside the sample loop, meaning the execution nesting counter is already
+    * zero and must be preserved.
+    */
+   bool inside_sample_loop;
+
+   /* Base coefficient register. 0 for API shaders but nonzero for FS prolog */
+   uint8_t cf_base;
+};
+
+struct agx_device_key {
+   /* Does the target GPU need explicit cluster coherency for atomics?
+    * Only used on G13X.
+    */
+   enum u_tristate needs_g13x_coherency;
+
+   /* Is soft fault enabled? This is technically system-wide policy set by the
+    * kernel, but that's functionally a hardware feature.
+    */
+   bool soft_fault;
+};
+
+struct agx_shader_key {
+   /* Device info */
+   struct agx_device_key dev;
+
+   /* Number of reserved preamble slots at the start */
+   unsigned reserved_preamble;
+
+   /* Whether scratch memory is available in the given shader stage */
+   bool has_scratch;
+
+   /* Whether we're compiling the helper program used for scratch allocation.
+    * This has special register allocation requirements.
+    */
+   bool is_helper;
+
+   /* Whether the driver supports uploading constants for this shader. If
+    * false, constants will not be promoted to uniforms.
+    */
+   bool promote_constants;
+
+   /* Similarly whether the driver supports promoting bindless
+    * textures/samplers.  Currently this works only if non-bindless
+    * textures/samplers are not used, but none of our drivers mix bindless /
+    * non-bindless usage.
+    */
+   bool promote_textures;
+
+   /* Set if this is a non-monolithic shader that must be linked with additional
+    * shader parts before the program can be used. This suppresses omission of
+    * `stop` instructions, which the linker must insert instead.
+    */
+   bool no_stop;
+
+   /* Set if this is a secondary shader part (prolog or epilog). This prevents
+    * the compiler from allocating uniform registers. For example, this turns
+    * off preambles.
+    */
+   bool secondary;
+
+   union {
+      struct agx_fs_shader_key fs;
+   };
+};
+
+uint64_t agx_gather_texcoords(nir_shader *nir);
+
+void agx_preprocess_nir(nir_shader *nir);
+bool agx_nir_lower_discard_zs_emit(nir_shader *s);
+bool agx_nir_lower_sample_mask(nir_shader *s);
+bool agx_nir_lower_interpolation(nir_shader *s);
+
+bool agx_nir_lower_cull_distance_vs(struct nir_shader *s);
+bool agx_nir_lower_cull_distance_fs(struct nir_shader *s,
+                                    unsigned nr_distances);
+bool agx_mem_vectorize_cb(unsigned align_mul, unsigned align_offset,
+                          unsigned bit_size, unsigned num_components,
+                          int64_t hole_size, nir_intrinsic_instr *low,
+                          nir_intrinsic_instr *high, void *data);
+
+void agx_compile_shader_nir(nir_shader *nir, struct agx_shader_key *key,
+                            struct agx_shader_part *out);
+
+struct agx_occupancy {
+   unsigned max_registers;
+   unsigned max_threads;
+};
+
+struct agx_occupancy agx_occupancy_for_register_count(unsigned halfregs);
+unsigned agx_max_registers_for_occupancy(unsigned occupancy);
+
+static inline unsigned
+agx_round_registers(unsigned halfregs)
+{
+   return agx_occupancy_for_register_count(halfregs).max_registers;
+}
+
+static const nir_shader_compiler_options agx_nir_options = {
+   .lower_fdiv = true,
+   .float_mul_add16 = nir_float_muladd_support_has_ffma | nir_float_muladd_support_fuse,
+   .float_mul_add32 = nir_float_muladd_support_has_ffma | nir_float_muladd_support_fuse,
+   .lower_flrp16 = true,
+   .lower_flrp32 = true,
+   .lower_fpow = true,
+   .lower_fmod = true,
+   .lower_bitfield_extract8 = true,
+   .lower_bitfield_extract16 = true,
+   .lower_bitfield_insert = true,
+   .lower_ifind_msb = true,
+   .lower_find_lsb = true,
+   .lower_uadd_carry = true,
+   .lower_usub_borrow = true,
+   .lower_fisnormal = true,
+   .lower_scmp = true,
+   .lower_isign = true,
+   .lower_fsign = true,
+   .lower_iabs = true,
+   .lower_fminmax_signed_zero = true,
+   .lower_fdph = true,
+   .lower_ffract = true,
+   .lower_pack_half_2x16 = true,
+   .lower_pack_unorm_2x16 = true,
+   .lower_pack_snorm_2x16 = true,
+   .lower_pack_unorm_4x8 = true,
+   .lower_pack_snorm_4x8 = true,
+   .lower_pack_64_2x32 = true,
+   .lower_unpack_half_2x16 = true,
+   .lower_unpack_unorm_2x16 = true,
+   .lower_unpack_snorm_2x16 = true,
+   .lower_unpack_unorm_4x8 = true,
+   .lower_unpack_snorm_4x8 = true,
+   .lower_extract_byte = true,
+   .lower_insert_byte = true,
+   .lower_insert_word = true,
+   .has_cs_global_id = true,
+   .lower_device_index_to_zero = true,
+   .lower_hadd = true,
+   .has_amul = true,
+   .has_isub = true,
+   .has_load_global_bounded = true,
+   .support_16bit_alu = true,
+   .max_unroll_iterations = 32,
+   .lower_uniforms_to_ubo = true,
+   .late_lower_int64 = true,
+   .lower_int64_options =
+      (nir_lower_int64_options) ~(nir_lower_iadd64 | nir_lower_imul_2x32_64),
+   .lower_doubles_options = (nir_lower_doubles_options)(~0),
+   .support_indirect_inputs = BITFIELD_BIT(MESA_SHADER_TESS_CTRL) |
+                              BITFIELD_BIT(MESA_SHADER_TESS_EVAL) |
+                              BITFIELD_BIT(MESA_SHADER_FRAGMENT),
+   .support_indirect_outputs = (uint8_t)BITFIELD_MASK(MESA_SHADER_STAGES),
+   .max_samples = 4,
+   .lower_fquantize2f16 = true,
+   .compact_arrays = true,
+   .discard_is_demote = true,
+   .scalarize_ddx = true,
+   .io_options = nir_io_always_interpolate_convergent_fs_inputs,
+};

src/asahi/compiler/agx_dce.c

@@ -0,0 +1,58 @@
+/*
+ * Copyright 2021 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "agx_compiler.h"
+
+/**
+ * SSA-based scalar dead code elimination.
+ * This pass assumes that no loop header phis are dead.
+ */
+void
+agx_dce(agx_context *ctx, bool partial)
+{
+   BITSET_WORD *seen = BITSET_CALLOC(ctx->alloc);
+
+   agx_foreach_block(ctx, block) {
+      if (block->loop_header) {
+         agx_foreach_phi_in_block(block, I) {
+            agx_foreach_ssa_src(I, s) {
+               BITSET_SET(seen, I->src[s].value);
+            }
+         }
+      }
+   }
+
+   agx_foreach_block_rev(ctx, block) {
+      agx_foreach_instr_in_block_safe_rev(block, I) {
+         if (block->loop_header && I->op == AGX_OPCODE_PHI)
+            break;
+
+         bool needed = false;
+
+         agx_foreach_ssa_dest(I, d) {
+            /* Eliminate destinations that are never read, as RA needs to
+             * handle them specially. Visible only for instructions that write
+             * multiple destinations (splits) or that write a destination but
+             * cannot be DCE'd (atomics).
+             */
+            if (BITSET_TEST(seen, I->dest[d].value)) {
+               needed = true;
+            } else if (partial) {
+               I->dest[d] = agx_null();
+            }
+         }
+
+         if (!needed && agx_opcodes_info[I->op].can_eliminate) {
+            agx_remove_instruction(I);
+         } else {
+            agx_foreach_ssa_src(I, s) {
+               BITSET_SET(seen, I->src[s].value);
+            }
+         }
+      }
+   }
+
+   free(seen);
+}

src/asahi/compiler/agx_debug.h

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2021 Alyssa Rosenzweig
+ * Copyright 2020 Collabora Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#pragma once
+
+#include "util/macros.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* clang-format off */
+enum agx_compiler_dbg {
+   /* bit 0 unused */
+   AGX_DBG_SHADERS     = BITFIELD_BIT(1),
+   AGX_DBG_SHADERDB    = BITFIELD_BIT(2),
+   AGX_DBG_VERBOSE     = BITFIELD_BIT(3),
+   AGX_DBG_INTERNAL    = BITFIELD_BIT(4),
+   AGX_DBG_NOVALIDATE  = BITFIELD_BIT(5),
+   AGX_DBG_NOOPT       = BITFIELD_BIT(6),
+   AGX_DBG_WAIT        = BITFIELD_BIT(7),
+   AGX_DBG_NOPREAMBLE  = BITFIELD_BIT(8),
+   AGX_DBG_DEMAND      = BITFIELD_BIT(9),
+   AGX_DBG_NOSCHED     = BITFIELD_BIT(10),
+   AGX_DBG_SPILL       = BITFIELD_BIT(11),
+   AGX_DBG_NOPROMOTE   = BITFIELD_BIT(12),
+};
+/* clang-format on */
+
+uint64_t agx_get_compiler_debug(void);
+
+#ifdef __cplusplus
+} /* extern C */
+#endif

src/asahi/compiler/agx_insert_waits.c

@@ -0,0 +1,162 @@
+/*
+ * Copyright 2022 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "agx_builder.h"
+#include "agx_compiler.h"
+#include "agx_debug.h"
+
+#define AGX_MAX_PENDING (8)
+
+/*
+ * Returns whether an instruction is asynchronous and needs a scoreboard slot
+ */
+static bool
+instr_is_async(agx_instr *I)
+{
+   return agx_opcodes_info[I->op].immediates & AGX_IMMEDIATE_SCOREBOARD;
+}
+
+struct slot {
+   /* Set of registers this slot is currently writing */
+   BITSET_DECLARE(writes, AGX_NUM_REGS);
+
+   /* Number of pending messages on this slot. Must not exceed
+    * AGX_MAX_PENDING for correct results.
+    */
+   uint8_t nr_pending;
+};
+
+/*
+ * Insert waits within a block to stall after every async instruction. Useful
+ * for debugging.
+ */
+static void
+agx_insert_waits_trivial(agx_context *ctx, agx_block *block)
+{
+   agx_foreach_instr_in_block_safe(block, I) {
+      if (instr_is_async(I)) {
+         agx_builder b = agx_init_builder(ctx, agx_after_instr(I));
+         agx_wait(&b, I->scoreboard);
+      }
+   }
+}
+
+/*
+ * Insert waits within a block, assuming scoreboard slots have already been
+ * assigned. This waits for everything at the end of the block, rather than
+ * doing something more intelligent/global. This should be optimized.
+ *
+ * XXX: Do any instructions read their sources asynchronously?
+ */
+static void
+agx_insert_waits_local(agx_context *ctx, agx_block *block)
+{
+   struct slot slots[2] = {0};
+
+   agx_foreach_instr_in_block_safe(block, I) {
+      uint8_t wait_mask = 0;
+
+      /* Check for read-after-write */
+      agx_foreach_src(I, s) {
+         if (I->src[s].type != AGX_INDEX_REGISTER)
+            continue;
+
+         unsigned nr_read = agx_index_size_16(I->src[s]);
+         for (unsigned slot = 0; slot < ARRAY_SIZE(slots); ++slot) {
+            if (BITSET_TEST_COUNT(slots[slot].writes, I->src[s].value, nr_read))
+               wait_mask |= BITSET_BIT(slot);
+         }
+      }
+
+      /* Check for write-after-write */
+      agx_foreach_dest(I, d) {
+         if (I->dest[d].type != AGX_INDEX_REGISTER)
+            continue;
+
+         unsigned nr_writes = agx_index_size_16(I->dest[d]);
+         for (unsigned slot = 0; slot < ARRAY_SIZE(slots); ++slot) {
+            if (BITSET_TEST_COUNT(slots[slot].writes, I->dest[d].value,
+                                  nr_writes))
+               wait_mask |= BITSET_BIT(slot);
+         }
+      }
+
+      /* Check for barriers */
+      if (I->op == AGX_OPCODE_THREADGROUP_BARRIER ||
+          I->op == AGX_OPCODE_MEMORY_BARRIER) {
+
+         for (unsigned slot = 0; slot < ARRAY_SIZE(slots); ++slot) {
+            if (slots[slot].nr_pending)
+               wait_mask |= BITSET_BIT(slot);
+         }
+      }
+
+      /* Try to assign a free slot */
+      if (instr_is_async(I)) {
+         for (unsigned slot = 0; slot < ARRAY_SIZE(slots); ++slot) {
+            if (slots[slot].nr_pending == 0) {
+               I->scoreboard = slot;
+               break;
+            }
+         }
+      }
+
+      /* Check for slot overflow */
+      if (instr_is_async(I) &&
+          slots[I->scoreboard].nr_pending >= AGX_MAX_PENDING)
+         wait_mask |= BITSET_BIT(I->scoreboard);
+
+      /* Insert the appropriate waits, clearing the slots */
+      u_foreach_bit(slot, wait_mask) {
+         agx_builder b = agx_init_builder(ctx, agx_before_instr(I));
+         agx_wait(&b, slot);
+
+         BITSET_ZERO(slots[slot].writes);
+         slots[slot].nr_pending = 0;
+      }
+
+      /* Record access */
+      if (instr_is_async(I)) {
+         agx_foreach_dest(I, d) {
+            if (agx_is_null(I->dest[d]))
+               continue;
+
+            assert(I->dest[d].type == AGX_INDEX_REGISTER);
+            BITSET_SET_COUNT(slots[I->scoreboard].writes, I->dest[d].value,
+                             agx_index_size_16(I->dest[d]));
+         }
+
+         slots[I->scoreboard].nr_pending++;
+      }
+   }
+
+   /* If there are outstanding messages, wait for them. We don't do this for the
+    * exit block, though, since nothing else will execute in the shader so
+    * waiting is pointless.
+    */
+   if (block != agx_exit_block(ctx)) {
+      agx_builder b = agx_init_builder(ctx, agx_after_block_logical(block));
+
+      for (unsigned slot = 0; slot < ARRAY_SIZE(slots); ++slot) {
+         if (slots[slot].nr_pending)
+            agx_wait(&b, slot);
+      }
+   }
+}
+
+/*
+ * Assign scoreboard slots to asynchronous instructions and insert waits for the
+ * appropriate hazard tracking.
+ */
+void
+agx_insert_waits(agx_context *ctx)
+{
+   agx_foreach_block(ctx, block) {
+      if (agx_compiler_debug & AGX_DBG_WAIT)
+         agx_insert_waits_trivial(ctx, block);
+      else
+         agx_insert_waits_local(ctx, block);
+   }
+}

src/asahi/compiler/agx_ir.c

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2022 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "agx_compiler.h"
+#include "agx_opcodes.h"
+
+bool
+agx_allows_16bit_immediate(agx_instr *I)
+{
+   return (I->op == AGX_OPCODE_DEVICE_LOAD) ||
+          (I->op == AGX_OPCODE_DEVICE_STORE) ||
+          (I->op == AGX_OPCODE_STACK_LOAD) ||
+          (I->op == AGX_OPCODE_STACK_STORE) ||
+          (I->op == AGX_OPCODE_UNIFORM_STORE) || (I->op == AGX_OPCODE_ATOMIC) ||
+          (I->op == AGX_OPCODE_PHI) || (I->op == AGX_OPCODE_TEX_STATE_STORE) ||
+          (I->op == AGX_OPCODE_SAMPLER_STATE_STORE);
+}
+
+unsigned
+agx_negate_src_index(agx_instr *I)
+{
+   switch (I->op) {
+   case AGX_OPCODE_IMAD:
+      return 2;
+   case AGX_OPCODE_IADD:
+      return 1;
+   default:
+      UNREACHABLE("not allowed");
+   }
+}

src/asahi/compiler/agx_liveness.c

@@ -0,0 +1,99 @@
+/*
+ * Copyright 2021 Alyssa Rosenzweig
+ * Copyright 2019-2020 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "util/sparse_bitset.h"
+#include "agx_compiler.h"
+
+/* Liveness analysis is a backwards-may dataflow analysis pass. Within a block,
+ * we compute live_out from live_in. The intrablock pass is linear-time. It
+ * returns whether progress was made. */
+
+/* live_in[s] = GEN[s] + (live_out[s] - KILL[s]) */
+
+void
+agx_liveness_ins_update(struct u_sparse_bitset *live, agx_instr *I)
+{
+   agx_foreach_ssa_dest(I, d)
+      u_sparse_bitset_clear(live, I->dest[d].value);
+
+   agx_foreach_ssa_src(I, s) {
+      /* If the source is not live after this instruction, but becomes live
+       * at this instruction, this is the use that kills the source
+       */
+      I->src[s].kill = !u_sparse_bitset_test(live, I->src[s].value);
+      u_sparse_bitset_set(live, I->src[s].value);
+   }
+}
+
+/* Globally, liveness analysis uses a fixed-point algorithm based on a
+ * worklist. We initialize a work list with the exit block. We iterate the work
+ * list to compute live_in from live_out for each block on the work list,
+ * adding the predecessors of the block to the work list if we made progress.
+ */
+
+void
+agx_compute_liveness(agx_context *ctx)
+{
+   u_worklist worklist;
+   u_worklist_init(&worklist, ctx->num_blocks, NULL);
+
+   /* Free any previous liveness, and allocate */
+   agx_foreach_block(ctx, block) {
+      u_sparse_bitset_free(&block->live_in);
+      u_sparse_bitset_free(&block->live_out);
+
+      u_sparse_bitset_init(&block->live_in, ctx->alloc, block);
+      u_sparse_bitset_init(&block->live_out, ctx->alloc, block);
+
+      agx_worklist_push_head(&worklist, block);
+   }
+
+   /* Iterate the work list */
+   while (!u_worklist_is_empty(&worklist)) {
+      /* Pop in reverse order since liveness is a backwards pass */
+      agx_block *blk = agx_worklist_pop_head(&worklist);
+
+      /* Update its liveness information */
+      u_sparse_bitset_dup(&blk->live_in, &blk->live_out);
+
+      agx_foreach_instr_in_block_rev(blk, I) {
+         if (I->op != AGX_OPCODE_PHI)
+            agx_liveness_ins_update(&blk->live_in, I);
+      }
+
+      /* Propagate the live in of the successor (blk) to the live out of
+       * predecessors.
+       *
+       * Phi nodes are logically on the control flow edge and act in parallel.
+       * To handle when propagating, we kill writes from phis and make live the
+       * corresponding sources.
+       */
+      agx_foreach_predecessor(blk, pred) {
+         struct u_sparse_bitset live;
+         u_sparse_bitset_dup(&live, &blk->live_in);
+
+         /* Kill write */
+         agx_foreach_phi_in_block(blk, phi) {
+            assert(phi->dest[0].type == AGX_INDEX_NORMAL);
+            u_sparse_bitset_clear(&live, phi->dest[0].value);
+         }
+
+         /* Make live the corresponding source */
+         agx_foreach_phi_in_block(blk, phi) {
+            agx_index operand = phi->src[agx_predecessor_index(blk, *pred)];
+            if (operand.type == AGX_INDEX_NORMAL) {
+               u_sparse_bitset_set(&live, operand.value);
+               phi->src[agx_predecessor_index(blk, *pred)].kill = false;
+            }
+         }
+
+         if (u_sparse_bitset_merge(&(*pred)->live_out, &live))
+            agx_worklist_push_tail(&worklist, *pred);
+      }
+   }
+
+   u_worklist_fini(&worklist);
+}

src/asahi/compiler/agx_lower_64bit.c

@@ -0,0 +1,63 @@
+/*
+ * Copyright 2022 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "agx_builder.h"
+#include "agx_compiler.h"
+
+/*
+ * Lower 64-bit moves to 32-bit moves. Although there are not 64-bit moves in
+ * the ISA, it is convenient to pretend there are for instruction selection.
+ * They are lowered trivially after register allocation.
+ *
+ * General 64-bit lowering happens in nir_lower_int64.
+ */
+static bool
+lower(agx_builder *b, agx_instr *I)
+{
+   if (I->op != AGX_OPCODE_MOV && I->op != AGX_OPCODE_MOV_IMM)
+      return false;
+
+   if (I->dest[0].size != AGX_SIZE_64)
+      return false;
+
+   agx_index dest = I->dest[0];
+   dest.size = AGX_SIZE_32;
+
+   if (I->op == AGX_OPCODE_MOV) {
+      assert(I->src[0].type == AGX_INDEX_REGISTER ||
+             I->src[0].type == AGX_INDEX_UNIFORM);
+      assert(I->src[0].size == AGX_SIZE_64);
+      agx_index src = I->src[0];
+      src.size = AGX_SIZE_32;
+
+      /* Low 32-bit */
+      agx_mov_to(b, dest, src);
+
+      /* High 32-bits */
+      dest.value += 2;
+      src.value += 2;
+      agx_mov_to(b, dest, src);
+   } else {
+      /* Low 32-bit */
+      agx_mov_imm_to(b, dest, I->imm & BITFIELD_MASK(32));
+
+      /* High 32-bits */
+      dest.value += 2;
+      agx_mov_imm_to(b, dest, I->imm >> 32);
+   }
+
+   return true;
+}
+
+void
+agx_lower_64bit_postra(agx_context *ctx)
+{
+   agx_foreach_instr_global_safe(ctx, I) {
+      agx_builder b = agx_init_builder(ctx, agx_before_instr(I));
+
+      if (lower(&b, I))
+         agx_remove_instruction(I);
+   }
+}

src/asahi/compiler/agx_lower_divergent_shuffle.c

@@ -0,0 +1,71 @@
+/*
+ * Copyright 2024 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "agx_builder.h"
+#include "agx_compiler.h"
+#include "agx_opcodes.h"
+
+static bool
+is_shuffle(enum agx_opcode op)
+{
+   switch (op) {
+   case AGX_OPCODE_SHUFFLE:
+   case AGX_OPCODE_SHUFFLE_UP:
+   case AGX_OPCODE_SHUFFLE_DOWN:
+   case AGX_OPCODE_SHUFFLE_XOR:
+   case AGX_OPCODE_QUAD_SHUFFLE:
+   case AGX_OPCODE_QUAD_SHUFFLE_UP:
+   case AGX_OPCODE_QUAD_SHUFFLE_DOWN:
+   case AGX_OPCODE_QUAD_SHUFFLE_XOR:
+      return true;
+   default:
+      return false;
+   }
+}
+
+/*
+ * AGX shuffle instructions read indices to shuffle with from the entire quad
+ * and accumulate them. That means that an inactive thread anywhere in the quad
+ * can make the whole shuffle undefined! To workaround, we reserve a scratch
+ * register (r0h) which we keep zero throughout the program... except for when
+ * actually shuffling, when we copy the shuffle index into r0h for the
+ * operation. This ensures that inactive threads read 0 for their index and
+ * hence do not contribute to the accumulated index.
+ */
+void
+agx_lower_divergent_shuffle(agx_context *ctx)
+{
+   agx_builder b = agx_init_builder(ctx, agx_before_function(ctx));
+   agx_index scratch = agx_register(1, AGX_SIZE_16);
+
+   assert(ctx->any_quad_divergent_shuffle);
+   agx_mov_imm_to(&b, scratch, 0);
+
+   agx_foreach_block(ctx, block) {
+      bool needs_zero = false;
+
+      agx_foreach_instr_in_block_safe(block, I) {
+         if (is_shuffle(I->op) && I->src[1].type == AGX_INDEX_REGISTER) {
+            assert(I->dest[0].value != scratch.value);
+            assert(I->src[0].value != scratch.value);
+            assert(I->src[1].value != scratch.value);
+
+            /* Use scratch register for our input, then zero it at the end of
+             * the block so all inactive threads read zero.
+             */
+            b.cursor = agx_before_instr(I);
+            agx_mov_to(&b, scratch, I->src[1]);
+            needs_zero = true;
+
+            I->src[1] = scratch;
+         }
+      }
+
+      if (needs_zero) {
+         b.cursor = agx_after_block_logical(block);
+         agx_mov_imm_to(&b, scratch, 0);
+      }
+   }
+}

src/asahi/compiler/agx_lower_parallel_copy.c

@@ -0,0 +1,402 @@
+/*
+ * Copyright 2022 Alyssa Rosenzweig
+ * Copyright 2021 Valve Corporation
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "agx_builder.h"
+#include "agx_compiler.h"
+
+UNUSED static void
+print_copy(const struct agx_copy *cp)
+{
+   printf("%sr%u = ", cp->dest_mem ? "m" : "", cp->dest);
+   agx_print_index(cp->src, false, stdout);
+   printf("\n");
+}
+
+UNUSED static void
+print_copies(const struct agx_copy *copies, unsigned nr)
+{
+   printf("[\n");
+
+   for (unsigned i = 0; i < nr; ++i) {
+      printf("  ");
+      print_copy(&copies[i]);
+   }
+
+   printf("]\n");
+}
+
+/*
+ * Emits code for
+ *
+ *    for (int i = 0; i < n; ++i)
+ *       registers[dests[i]] = registers[srcs[i]];
+ *
+ * ...with all copies happening in parallel.
+ *
+ * That is, emit machine instructions equivalent to a parallel copy. This is
+ * used to lower not only parallel copies but also collects and splits, which
+ * also have parallel copy semantics.
+ *
+ * We only handles register-register copies, not general agx_index sources. This
+ * suffices for its internal use for register allocation.
+ */
+
+static void
+do_copy(agx_builder *b, const struct agx_copy *copy)
+{
+   agx_index dst = copy->dest_mem
+                      ? agx_memory_register(copy->dest, copy->src.size)
+                      : agx_register(copy->dest, copy->src.size);
+
+   if (copy->dest_mem && copy->src.memory) {
+      /* Memory-memory copies need to be lowered to memory-register and
+       * register-memory, using a reserved scratch register.
+       */
+      agx_index scratch_reg = agx_register(2, copy->src.size);
+      agx_mov_to(b, scratch_reg, copy->src);
+      agx_mov_to(b, dst, scratch_reg);
+   } else if (copy->src.type == AGX_INDEX_IMMEDIATE) {
+      agx_mov_imm_to(b, dst, copy->src.value);
+   } else {
+      agx_mov_to(b, dst, copy->src);
+   }
+}
+
+static void
+do_swap(agx_builder *b, const struct agx_copy *copy)
+{
+   assert(copy->src.type == AGX_INDEX_REGISTER && "only GPRs are swapped");
+
+   if (copy->dest == copy->src.value)
+      return;
+
+   agx_index x = copy->dest_mem
+                    ? agx_memory_register(copy->dest, copy->src.size)
+                    : agx_register(copy->dest, copy->src.size);
+   agx_index y = copy->src;
+   assert(x.memory == y.memory);
+
+   /* Memory-memory swaps lowered here, GPR swaps lowered later */
+   if (x.memory) {
+      agx_index temp1 = agx_register(4, copy->src.size);
+      agx_index temp2 = agx_register(6, copy->src.size);
+
+      agx_mov_to(b, temp1, x);
+      agx_mov_to(b, temp2, y);
+      agx_mov_to(b, y, temp1);
+      agx_mov_to(b, x, temp2);
+   } else {
+      agx_swap(b, x, y);
+   }
+}
+
+struct copy_ctx {
+   /* Number of copies being processed */
+   unsigned entry_count;
+
+   /* For each physreg, the number of pending copy entries that use it as a
+    * source. Once this drops to zero, then the physreg is unblocked and can
+    * be moved to.
+    */
+   unsigned physreg_use_count[AGX_NUM_MODELED_REGS];
+
+   /* For each physreg, the pending copy_entry that uses it as a dest. */
+   struct agx_copy *physreg_dest[AGX_NUM_MODELED_REGS];
+
+   struct agx_copy entries[AGX_NUM_MODELED_REGS];
+};
+
+static bool
+entry_blocked(struct agx_copy *entry, struct copy_ctx *ctx)
+{
+   for (unsigned i = 0; i < agx_size_align_16(entry->src.size); i++) {
+      if (ctx->physreg_use_count[entry->dest + i] != 0)
+         return true;
+   }
+
+   return false;
+}
+
+static bool
+is_real(struct agx_copy *entry)
+{
+   return entry->src.type == AGX_INDEX_REGISTER &&
+          entry->dest_mem == entry->src.memory;
+}
+
+/* TODO: Generalize to other bit sizes */
+static void
+split_32bit_copy(struct copy_ctx *ctx, struct agx_copy *entry)
+{
+   assert(!entry->done);
+   assert(is_real(entry));
+   assert(agx_size_align_16(entry->src.size) == 2);
+   struct agx_copy *new_entry = &ctx->entries[ctx->entry_count++];
+
+   new_entry->dest = entry->dest + 1;
+   new_entry->dest_mem = entry->dest_mem;
+   new_entry->src = entry->src;
+   new_entry->src.value += 1;
+   new_entry->done = false;
+   entry->src.size = AGX_SIZE_16;
+   new_entry->src.size = AGX_SIZE_16;
+   ctx->physreg_dest[entry->dest + 1] = new_entry;
+}
+
+static void
+agx_emit_parallel_copies_for_class(agx_builder *b, struct agx_copy *copies,
+                                   unsigned num_copies, bool cls)
+{
+   /* First, lower away 64-bit copies to smaller chunks, since we don't have
+    * 64-bit ALU so we always want to split.
+    */
+   struct agx_copy *copies2 = calloc(sizeof(copies[0]), num_copies * 2);
+   unsigned num_copies2 = 0;
+
+   for (unsigned i = 0; i < num_copies; ++i) {
+      struct agx_copy copy = copies[i];
+
+      /* Filter by class */
+      if (copy.dest_mem != cls)
+         continue;
+
+      assert(copy.dest < AGX_NUM_MODELED_REGS);
+
+      if (copy.src.size == AGX_SIZE_64) {
+         copy.src.size = AGX_SIZE_32;
+         copies2[num_copies2++] = copy;
+
+         if (copy.src.type == AGX_INDEX_IMMEDIATE) {
+            static_assert(sizeof(copy.src.value) * 8 == 32, "known size");
+            copy.src.value = 0;
+         } else {
+            assert(copy.src.type == AGX_INDEX_REGISTER ||
+                   copy.src.type == AGX_INDEX_UNIFORM);
+
+            copy.src.value += 2;
+         }
+
+         copy.dest += 2;
+         copies2[num_copies2++] = copy;
+      } else {
+         copies2[num_copies2++] = copy;
+      }
+   }
+
+   copies = copies2;
+   num_copies = num_copies2;
+
+   /* Set up the bookkeeping */
+   struct copy_ctx _ctx = {.entry_count = num_copies};
+   struct copy_ctx *ctx = &_ctx;
+
+   memset(ctx->physreg_dest, 0, sizeof(ctx->physreg_dest));
+   memset(ctx->physreg_use_count, 0, sizeof(ctx->physreg_use_count));
+
+   for (unsigned i = 0; i < ctx->entry_count; i++) {
+      struct agx_copy *entry = &copies[i];
+
+      ctx->entries[i] = *entry;
+
+      for (unsigned j = 0; j < agx_size_align_16(entry->src.size); j++) {
+         if (is_real(entry))
+            ctx->physreg_use_count[entry->src.value + j]++;
+
+         /* Copies should not have overlapping destinations. */
+         assert(!ctx->physreg_dest[entry->dest + j]);
+         ctx->physreg_dest[entry->dest + j] = &ctx->entries[i];
+      }
+   }
+
+   /* Try to vectorize aligned 16-bit copies to use 32-bit operations instead */
+   for (unsigned i = 0; i < ctx->entry_count; i++) {
+      struct agx_copy *entry = &ctx->entries[i];
+      if (entry->src.size != AGX_SIZE_16)
+         continue;
+
+      if ((entry->dest & 1) || (entry->src.value & 1))
+         continue;
+
+      if (entry->src.type != AGX_INDEX_UNIFORM &&
+          entry->src.type != AGX_INDEX_REGISTER)
+         continue;
+
+      unsigned next_dest = entry->dest + 1;
+      assert(next_dest < ARRAY_SIZE(ctx->physreg_dest) && "aligned reg");
+
+      struct agx_copy *next_copy = ctx->physreg_dest[next_dest];
+      if (!next_copy)
+         continue;
+
+      assert(next_copy->dest == next_dest && "data structure invariant");
+      assert(next_copy->src.size == AGX_SIZE_16 && "unaligned copy");
+
+      if (next_copy->src.type != entry->src.type)
+         continue;
+
+      if (next_copy->src.value != (entry->src.value + 1))
+         continue;
+
+      /* Vectorize the copies */
+      ctx->physreg_dest[next_dest] = entry;
+      entry->src.size = AGX_SIZE_32;
+      next_copy->done = true;
+   }
+
+   bool progress = true;
+   while (progress) {
+      progress = false;
+
+      /* Step 1: resolve paths in the transfer graph. This means finding
+       * copies whose destination aren't blocked by something else and then
+       * emitting them, continuing this process until every copy is blocked
+       * and there are only cycles left.
+       *
+       * TODO: We should note that src is also available in dest to unblock
+       * cycles that src is involved in.
+       */
+
+      for (unsigned i = 0; i < ctx->entry_count; i++) {
+         struct agx_copy *entry = &ctx->entries[i];
+         if (!entry->done && !entry_blocked(entry, ctx)) {
+            entry->done = true;
+            progress = true;
+            do_copy(b, entry);
+            for (unsigned j = 0; j < agx_size_align_16(entry->src.size); j++) {
+               if (is_real(entry))
+                  ctx->physreg_use_count[entry->src.value + j]--;
+               ctx->physreg_dest[entry->dest + j] = NULL;
+            }
+         }
+      }
+
+      if (progress)
+         continue;
+
+      /* Step 2: Find partially blocked copies and split them. In the
+       * mergedregs case, we can 32-bit copies which are only blocked on one
+       * 16-bit half, and splitting them helps get things moving.
+       *
+       * We can skip splitting copies if the source isn't a register,
+       * however, because it does not unblock anything and therefore doesn't
+       * contribute to making forward progress with step 1. These copies
+       * should still be resolved eventually in step 1 because they can't be
+       * part of a cycle.
+       */
+      for (unsigned i = 0; i < ctx->entry_count; i++) {
+         struct agx_copy *entry = &ctx->entries[i];
+         if (entry->done || (agx_size_align_16(entry->src.size) != 2))
+            continue;
+
+         if (((ctx->physreg_use_count[entry->dest] == 0 ||
+               ctx->physreg_use_count[entry->dest + 1] == 0)) &&
+             is_real(entry)) {
+            split_32bit_copy(ctx, entry);
+            progress = true;
+         }
+      }
+   }
+
+   /* Step 3: resolve cycles through swapping.
+    *
+    * At this point, the transfer graph should consist of only cycles.
+    * The reason is that, given any physreg n_1 that's the source of a
+    * remaining entry, it has a destination n_2, which (because every
+    * copy is blocked) is the source of some other copy whose destination
+    * is n_3, and so we can follow the chain until we get a cycle. If we
+    * reached some other node than n_1:
+    *
+    *  n_1 -> n_2 -> ... -> n_i
+    *          ^             |
+    *          |-------------|
+    *
+    *  then n_2 would be the destination of 2 copies, which is illegal
+    *  (checked above in an assert). So n_1 must be part of a cycle:
+    *
+    *  n_1 -> n_2 -> ... -> n_i
+    *  ^                     |
+    *  |---------------------|
+    *
+    *  and this must be only cycle n_1 is involved in, because any other
+    *  path starting from n_1 would also have to end in n_1, resulting in
+    *  a node somewhere along the way being the destination of 2 copies
+    *  when the 2 paths merge.
+    *
+    *  The way we resolve the cycle is through picking a copy (n_1, n_2)
+    *  and swapping n_1 and n_2. This moves n_1 to n_2, so n_2 is taken
+    *  out of the cycle:
+    *
+    *  n_1 -> ... -> n_i
+    *  ^              |
+    *  |--------------|
+    *
+    *  and we can keep repeating this until the cycle is empty.
+    */
+
+   for (unsigned i = 0; i < ctx->entry_count; i++) {
+      struct agx_copy *entry = &ctx->entries[i];
+      if (entry->done)
+         continue;
+
+      assert(is_real(entry));
+
+      /* catch trivial copies */
+      if (entry->dest == entry->src.value) {
+         entry->done = true;
+         continue;
+      }
+
+      do_swap(b, entry);
+
+      /* Split any blocking copies whose sources are only partially
+       * contained within our destination.
+       */
+      if (agx_size_align_16(entry->src.size) == 1) {
+         for (unsigned j = 0; j < ctx->entry_count; j++) {
+            struct agx_copy *blocking = &ctx->entries[j];
+
+            if (blocking->done)
+               continue;
+
+            if (blocking->src.value <= entry->dest &&
+                blocking->src.value + 1 >= entry->dest &&
+                agx_size_align_16(blocking->src.size) == 2) {
+               split_32bit_copy(ctx, blocking);
+            }
+         }
+      }
+
+      /* Update sources of blocking copies.
+       *
+       * Note: at this point, every blocking copy's source should be
+       * contained within our destination.
+       */
+      for (unsigned j = 0; j < ctx->entry_count; j++) {
+         struct agx_copy *blocking = &ctx->entries[j];
+         if (blocking->src.value >= entry->dest &&
+             blocking->src.value <
+                entry->dest + agx_size_align_16(entry->src.size)) {
+            blocking->src.value =
+               entry->src.value + (blocking->src.value - entry->dest);
+         }
+      }
+
+      entry->done = true;
+   }
+
+   free(copies2);
+}
+
+void
+agx_emit_parallel_copies(agx_builder *b, struct agx_copy *copies,
+                         unsigned num_copies)
+{
+   /* Emit copies fo reach register class separately because we don't have
+    * register class awareness in the parallel copy lowering data structure.
+    */
+   agx_emit_parallel_copies_for_class(b, copies, num_copies, false);
+   agx_emit_parallel_copies_for_class(b, copies, num_copies, true);
+}

src/asahi/compiler/agx_lower_pseudo.c

@@ -0,0 +1,142 @@
+/*
+ * Copyright 2022 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "util/lut.h"
+#include "agx_builder.h"
+#include "agx_compiler.h"
+#include "agx_opcodes.h"
+
+/* Lower pseudo instructions created during optimization. */
+
+static agx_instr *
+while_for_break_if(agx_builder *b, agx_instr *I)
+{
+   if (I->op == AGX_OPCODE_BREAK_IF_FCMP) {
+      return agx_while_fcmp(b, I->src[0], I->src[1], I->nest, I->fcond,
+                            !I->invert_cond, NULL);
+   } else {
+      return agx_while_icmp(b, I->src[0], I->src[1], I->nest, I->icond,
+                            !I->invert_cond, NULL);
+   }
+}
+
+static agx_instr *
+cmpsel_for_break_if(agx_builder *b, agx_instr *I)
+{
+   agx_index r0l = agx_register(0, AGX_SIZE_16);
+
+   /* If the condition is true, set r0l to nest to break */
+   agx_index t = agx_immediate(I->nest);
+   agx_index f = r0l;
+
+   if (I->invert_cond) {
+      SWAP(t, f);
+   }
+
+   if (I->op == AGX_OPCODE_BREAK_IF_FCMP)
+      agx_fcmpsel_to(b, r0l, I->src[0], I->src[1], t, f, I->fcond);
+   else
+      agx_icmpsel_to(b, r0l, I->src[0], I->src[1], t, f, I->icond);
+
+   return agx_push_exec(b, 0);
+}
+
+static void
+swap(agx_builder *b, agx_index x, agx_index y)
+{
+   assert(!x.memory && "already lowered");
+   assert(!y.memory && "already lowered");
+
+   /* We can swap lo/hi halves of a 32-bit register with a 32-bit extr */
+   if (x.size == AGX_SIZE_16 && (x.value >> 1) == (y.value >> 1)) {
+
+      assert(((x.value & 1) == (1 - (y.value & 1))) &&
+             "no trivial swaps, and only 2 halves of a register");
+
+      /* r0 = extr r0, r0, #16
+       *    = (((r0 << 32) | r0) >> 16) & 0xFFFFFFFF
+       *    = (((r0 << 32) >> 16) & 0xFFFFFFFF) | (r0 >> 16)
+       *    = (r0l << 16) | r0h
+       */
+      agx_index reg32 = agx_register(x.value & ~1, AGX_SIZE_32);
+      agx_extr_to(b, reg32, reg32, reg32, agx_immediate(16), 0);
+   } else {
+      /* Otherwise, we're swapping GPRs and fallback on a XOR swap. */
+      agx_xor_to(b, x, x, y);
+      agx_xor_to(b, y, x, y);
+      agx_xor_to(b, x, x, y);
+   }
+}
+
+static agx_instr *
+lower(agx_builder *b, agx_instr *I)
+{
+   switch (I->op) {
+
+   /* Various instructions are implemented as bitwise truth tables */
+   case AGX_OPCODE_MOV:
+      return agx_bitop_to(b, I->dest[0], I->src[0], agx_zero(), UTIL_LUT2(a));
+
+   case AGX_OPCODE_NOT:
+      return agx_bitop_to(b, I->dest[0], I->src[0], agx_zero(), UTIL_LUT2(~a));
+
+   /* We can sign-extend with an add */
+   case AGX_OPCODE_SIGNEXT:
+      return agx_iadd_to(b, I->dest[0], I->src[0], agx_zero(), 0);
+
+   /* Unfused comparisons are fused with a 0/1 select */
+   case AGX_OPCODE_ICMP:
+      return agx_icmpsel_to(b, I->dest[0], I->src[0], I->src[1],
+                            agx_immediate(I->invert_cond ? 0 : 1),
+                            agx_immediate(I->invert_cond ? 1 : 0), I->icond);
+
+   case AGX_OPCODE_FCMP:
+      return agx_fcmpsel_to(b, I->dest[0], I->src[0], I->src[1],
+                            agx_immediate(I->invert_cond ? 0 : 1),
+                            agx_immediate(I->invert_cond ? 1 : 0), I->fcond);
+
+   case AGX_OPCODE_BALLOT:
+      return agx_icmp_ballot_to(b, I->dest[0], I->src[0], agx_zero(),
+                                AGX_ICOND_UEQ, true /* invert */);
+
+   case AGX_OPCODE_QUAD_BALLOT:
+      return agx_icmp_quad_ballot_to(b, I->dest[0], I->src[0], agx_zero(),
+                                     AGX_ICOND_UEQ, true /* invert */);
+
+   /* Writes to the nesting counter lowered to the real register */
+   case AGX_OPCODE_BEGIN_CF:
+      return agx_mov_imm_to(b, agx_register(0, AGX_SIZE_16), 0);
+
+   case AGX_OPCODE_BREAK:
+      agx_mov_imm_to(b, agx_register(0, AGX_SIZE_16), I->nest);
+      return agx_pop_exec(b, 0);
+
+   case AGX_OPCODE_BREAK_IF_ICMP:
+   case AGX_OPCODE_BREAK_IF_FCMP: {
+      if (I->nest == 1)
+         return while_for_break_if(b, I);
+      else
+         return cmpsel_for_break_if(b, I);
+   }
+
+   case AGX_OPCODE_SWAP:
+      swap(b, I->src[0], I->src[1]);
+      return (void *)true;
+
+   default:
+      return NULL;
+   }
+}
+
+void
+agx_lower_pseudo(agx_context *ctx)
+{
+   agx_foreach_instr_global_safe(ctx, I) {
+      agx_builder b = agx_init_builder(ctx, agx_before_instr(I));
+
+      if (lower(&b, I))
+         agx_remove_instruction(I);
+   }
+}

src/asahi/compiler/agx_lower_spill.c

@@ -0,0 +1,82 @@
+/*
+ * Copyright 2023 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "util/macros.h"
+#include "agx_builder.h"
+#include "agx_compile.h"
+#include "agx_compiler.h"
+
+/* Lower moves involving memory registers (created when spilling) to concrete
+ * spills and fills.
+ */
+
+static void
+spill_fill(agx_builder *b, agx_instr *I, enum agx_size size, unsigned channels,
+           unsigned component_offset_el)
+{
+   unsigned size_B = agx_size_align_16(size) * 2;
+   enum agx_format format = size_B == 2 ? AGX_FORMAT_I16 : AGX_FORMAT_I32;
+   unsigned format_size_B = size_B == 2 ? 2 : 4;
+
+   unsigned offset_B = component_offset_el * size_B;
+   unsigned effective_chans = size == AGX_SIZE_64 ? (channels * 2) : channels;
+   unsigned mask = BITFIELD_MASK(effective_chans);
+
+   assert(effective_chans <= 4);
+
+   /* Pick off the memory and register parts of the move */
+   agx_index mem = I->dest[0].memory ? I->dest[0] : I->src[0];
+   agx_index reg = I->dest[0].memory ? I->src[0] : I->dest[0];
+
+   assert(mem.type == AGX_INDEX_REGISTER && mem.memory);
+   assert(reg.type == AGX_INDEX_REGISTER && !reg.memory);
+
+   /* Slice the register according to the part of the spill we're handling */
+   if (component_offset_el > 0 || channels != agx_channels(reg)) {
+      reg.value += component_offset_el * agx_size_align_16(reg.size);
+      reg.channels_m1 = channels - 1;
+   }
+
+   /* Calculate stack offset in bytes. IR registers are 2-bytes each. */
+   unsigned stack_offs_B = b->shader->spill_base_B + (mem.value * 2) + offset_B;
+   unsigned stack_offs_end_B = stack_offs_B + (effective_chans * format_size_B);
+
+   assert(stack_offs_end_B <= b->shader->scratch_size_B &&
+          "RA allocates enough scratch");
+
+   /* Emit the spill/fill */
+   if (I->dest[0].memory) {
+      agx_stack_store(b, reg, agx_immediate(stack_offs_B), format, mask);
+   } else {
+      agx_stack_load_to(b, reg, agx_immediate(stack_offs_B), format, mask);
+   }
+}
+
+void
+agx_lower_spill(agx_context *ctx)
+{
+   agx_foreach_instr_global_safe(ctx, I) {
+      if (I->op != AGX_OPCODE_MOV || (!I->dest[0].memory && !I->src[0].memory))
+         continue;
+
+      enum agx_size size = I->dest[0].size;
+      unsigned channels = agx_channels(I->dest[0]);
+
+      assert(size == I->src[0].size);
+      assert(channels == agx_channels(I->src[0]));
+
+      /* Texture gradient sources can be vec6, and if such a vector is spilled,
+       * we need to be able to spill/fill a vec6. Since stack_store/stack_load
+       * only work up to vec4, we break up into (at most) vec4 components.
+       */
+      agx_builder b = agx_init_builder(ctx, agx_before_instr(I));
+
+      for (unsigned c = 0; c < channels; c += 4) {
+         spill_fill(&b, I, size, MIN2(channels - c, 4), c);
+      }
+
+      agx_remove_instruction(I);
+   }
+}

src/asahi/compiler/agx_lower_uniform_sources.c

@@ -0,0 +1,94 @@
+/*
+ * Copyright 2023 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+#include "agx_builder.h"
+#include "agx_compiler.h"
+
+/*
+ * Not all instructions can take uniforms. Memory instructions can take
+ * uniforms, but only for their base (first) source and only in the
+ * low-half of the uniform file.
+ *
+ * This pass lowers invalid uniforms.
+ */
+bool
+agx_instr_accepts_uniform(enum agx_opcode op, unsigned src_index,
+                          unsigned value, enum agx_size size)
+{
+   /* Some instructions only seem able to access uniforms in the low half */
+   bool high = value >= 256;
+
+   /* ALU cannot access 64-bit uniforms */
+   bool is_64 = size == AGX_SIZE_64;
+
+   switch (op) {
+   case AGX_OPCODE_IMAGE_LOAD:
+   case AGX_OPCODE_TEXTURE_LOAD:
+   case AGX_OPCODE_TEXTURE_SAMPLE:
+      /* Unknown if this works, but the driver will never hit this. */
+      assert(!(src_index == 2 && high) && "texture heap always low");
+      return !high && (src_index == 1 || src_index == 2);
+
+   case AGX_OPCODE_TEX_STATE_STORE:
+   case AGX_OPCODE_SAMPLER_STATE_STORE:
+      assert(!high && "texture heap always low");
+      return !high && src_index == 0;
+
+   case AGX_OPCODE_DEVICE_LOAD:
+      return src_index == 0 && !high;
+   case AGX_OPCODE_DEVICE_STORE:
+   case AGX_OPCODE_ATOMIC:
+      return src_index == 1 && !high;
+   case AGX_OPCODE_LOCAL_LOAD:
+      return src_index == 0;
+   case AGX_OPCODE_LOCAL_STORE:
+      return src_index == 1;
+   case AGX_OPCODE_IMAGE_WRITE:
+      return src_index == 3;
+   case AGX_OPCODE_BLOCK_IMAGE_STORE:
+   case AGX_OPCODE_ZS_EMIT:
+   case AGX_OPCODE_ST_TILE:
+   case AGX_OPCODE_LD_TILE:
+   case AGX_OPCODE_UNIFORM_STORE:
+   case AGX_OPCODE_ST_VARY:
+   case AGX_OPCODE_LOCAL_ATOMIC:
+   case AGX_OPCODE_SAMPLE_MASK:
+   case AGX_OPCODE_ITER:
+   case AGX_OPCODE_ITERPROJ:
+   case AGX_OPCODE_STACK_LOAD:
+   case AGX_OPCODE_STACK_STORE:
+   case AGX_OPCODE_BALLOT:
+   case AGX_OPCODE_FCMP_BALLOT:
+   case AGX_OPCODE_ICMP_BALLOT:
+   case AGX_OPCODE_QUAD_BALLOT:
+   case AGX_OPCODE_FCMP_QUAD_BALLOT:
+   case AGX_OPCODE_ICMP_QUAD_BALLOT:
+      return false;
+   case AGX_OPCODE_EXPORT:
+   case AGX_OPCODE_PHI:
+      /* We would fail validation otherwise */
+      return true;
+   default:
+      return !is_64;
+   }
+}
+
+void
+agx_lower_uniform_sources(agx_context *ctx)
+{
+   agx_foreach_instr_global_safe(ctx, I) {
+      agx_builder b = agx_init_builder(ctx, agx_before_instr(I));
+
+      agx_foreach_src(I, s) {
+         if (I->src[s].type == AGX_INDEX_UNIFORM &&
+             !agx_instr_accepts_uniform(I->op, s, I->src[s].value,
+                                        I->src[s].size)) {
+
+            agx_index idx = I->src[s];
+            idx.abs = idx.neg = false;
+            agx_replace_src(I, s, agx_mov(&b, idx));
+         }
+      }
+   }
+}

src/asahi/compiler/agx_nir.h

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2023 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#pragma once
+
+#include <stdbool.h>
+
+struct nir_shader;
+struct nir_instr;
+
+bool agx_nir_lower_address(struct nir_shader *shader);
+bool agx_nir_lower_algebraic_late(struct nir_shader *shader);
+bool agx_nir_cleanup_amul(struct nir_shader *shader);
+bool agx_nir_fuse_lea(struct nir_shader *shader);
+bool agx_nir_lower_lea(struct nir_shader *shader);
+bool agx_nir_fuse_selects(struct nir_shader *shader);
+bool agx_nir_fuse_algebraic_late(struct nir_shader *shader);
+bool agx_nir_fence_images(struct nir_shader *shader);
+bool agx_nir_lower_layer(struct nir_shader *s);
+bool agx_nir_lower_clip_distance(struct nir_shader *s);
+bool agx_nir_lower_subgroups(struct nir_shader *s);
+bool agx_nir_lower_fminmax(struct nir_shader *s);
+
+bool agx_nir_lower_texture_early(struct nir_shader *s, bool support_lod_bias);
+bool agx_nir_lower_texture(struct nir_shader *s);
+bool agx_nir_lower_multisampled_image_store(struct nir_shader *s);
+bool agx_nir_needs_texture_crawl(struct nir_instr *instr);

src/asahi/compiler/agx_nir_algebraic.py

@@ -0,0 +1,359 @@
+# Copyright 2022 Alyssa Rosenzweig
+# Copyright 2021 Collabora, Ltd.
+# Copyright 2016 Intel Corporation
+# SPDX-License-Identifier: MIT
+
+import argparse
+import sys
+import math
+
+a = 'a'
+b = 'b'
+c = 'c'
+d = 'd'
+e = 'e'
+
+lower_sm5_shift = []
+
+# Our shifts differ from SM5 for the upper bits. Mask to match the NIR
+# behaviour. Because this happens as a late lowering, NIR won't optimize the
+# masking back out (that happens in the main nir_opt_algebraic).
+for s in [8, 16, 32, 64]:
+    for shift in ["ishl", "ishr", "ushr"]:
+        lower_sm5_shift += [((shift, f'a@{s}', b),
+                             (shift, a, ('iand', b, s - 1)))]
+
+lower_pack = [
+    (('pack_half_2x16_split', a, b),
+     ('pack_32_2x16_split', ('f2f16', a), ('f2f16', b))),
+
+    # We don't have 8-bit ALU, so we need to lower this. But if we lower it like
+    # this, we can at least coalesce the pack_32_2x16_split and only pay the
+    # cost of the iors and ishl. (u2u16 of 8-bit is assumed free.)
+    (('pack_32_4x8_split', a, b, c, d),
+     ('pack_32_2x16_split', ('ior', ('u2u16', a), ('ishl', ('u2u16', b), 8)),
+                            ('ior', ('u2u16', c), ('ishl', ('u2u16', d), 8)))),
+
+    (('extract_u16', 'a@32', 0), ('u2u32', ('unpack_32_2x16_split_x', a))),
+    (('extract_u16', 'a@32', 1), ('u2u32', ('unpack_32_2x16_split_y', a))),
+    (('extract_i16', 'a@32', 0), ('i2i32', ('unpack_32_2x16_split_x', a))),
+    (('extract_i16', 'a@32', 1), ('i2i32', ('unpack_32_2x16_split_y', a))),
+
+    # For optimizing extract->convert sequences for unpack/pack norm
+    (('u2f32', ('u2u32', a)), ('u2f32', a)),
+    (('i2f32', ('i2i32', a)), ('i2f32', a)),
+
+    # Chew through some 8-bit before the backend has to deal with it
+    (('f2u8', a), ('u2u8', ('f2u16', a))),
+    (('f2i8', a), ('i2i8', ('f2i16', a))),
+
+    # Duplicated from nir_opt_algebraic since this pattern is generated by our
+    # bounds checking optimization which needs to run relatively late.
+    (('unpack_64_2x32_split_x', ('pack_64_2x32_split', a, b)), a),
+    (('unpack_64_2x32_split_y', ('pack_64_2x32_split', a, b)), b),
+
+    # Based on the VIR lowering
+    (('f2f16_rtz', 'a@32'),
+     ('bcsel', ('flt', ('fabs', a), ('fabs', ('f2f32', ('f2f16_rtne', a)))),
+      ('isub', ('f2f16_rtne', a), 1), ('f2f16_rtne', a))),
+
+    # These are based on the lowerings from nir_opt_algebraic, but conditioned
+    # on the number of bits not being constant. If the bit count is constant
+    # (the happy path) we can use our native instruction instead.
+    (('ibitfield_extract', 'value', 'offset', 'bits(is_not_const)'),
+     ('bcsel', ('ieq', 0, 'bits'),
+      0,
+      ('ishr',
+       ('ishl', 'value', ('isub', ('isub', 32, 'bits'), 'offset')),
+       ('isub', 32, 'bits')))),
+
+    (('ubitfield_extract', 'value', 'offset', 'bits(is_not_const)'),
+     ('iand',
+      ('ushr', 'value', 'offset'),
+      ('bcsel', ('ieq', 'bits', 32),
+       0xffffffff,
+       ('isub', ('ishl', 1, 'bits'), 1)))),
+
+    # Codegen depends on this trivial case being optimized out.
+    (('ubitfield_extract', 'value', 'offset', 0), 0),
+    (('ibitfield_extract', 'value', 'offset', 0), 0),
+
+    # At this point, bitfield extracts are constant. We can only do constant
+    # unsigned bitfield extract, so lower signed to unsigned + sign extend.
+    (('ibitfield_extract', a, b, '#bits'),
+     ('ishr', ('ishl', ('ubitfield_extract', a, b, 'bits'), ('isub', 32, 'bits')),
+      ('isub', 32, 'bits'))),
+]
+
+lower_selects = []
+for T, sizes, one in [('f', [16, 32], 1.0),
+                      ('i', [8, 16, 32], 1),
+                      ('b', [16, 32], -1)]:
+    for size in sizes:
+        lower_selects.extend([
+            ((f'b2{T}{size}', ('inot', 'a@1')), ('bcsel', a, 0, one)),
+            ((f'b2{T}{size}', 'a@1'), ('bcsel', a, one, 0)),
+        ])
+
+# Rewriting bcsel(a || b, ...) in terms of bcsel(a, ...) and bcsel(b, ...) lets
+# our rules to fuse compare-and-select do a better job, assuming that a and b
+# are comparisons themselves.
+#
+# This needs to be a separate pass that runs after lower_selects, in order to
+# pick up patterns like b2f32(iand(...))
+opt_selects = [
+        (('bcsel', ('ior(is_used_once)', ('inot', a), b), c, d),
+         ('bcsel', a, ('bcsel', b, c, d), c)),
+
+        (('bcsel', ('ior(is_used_once)', a, b), c, d),
+         ('bcsel', a, c, ('bcsel', b, c, d))),
+
+        (('bcsel', ('iand(is_used_once)', ('inot', a), b), c, d),
+         ('bcsel', a, d, ('bcsel', b, c, d))),
+
+        (('bcsel', ('iand(is_used_once)', a, b), c, d),
+         ('bcsel', a, ('bcsel', b, c, d), d)),
+]
+
+# When the ior/iand is used multiple times, we can instead fuse the other way.
+opt_selects.extend([
+        (('iand', ('inot', 'a@1'), b), ('bcsel', a, False, b)),
+        (('iand', 'a@1', b), ('bcsel', a, b, False)),
+
+        (('ior', ('inot', 'a@1'), b), ('bcsel', a, b, True)),
+        (('ior', 'a@1', b), ('bcsel', a, True, b)),
+])
+
+# Rewriting fneg to ixor costs a uniform but lets us CSE the bcsel if this is
+# applied to a whole vector, hence the has_multiple_uses heuristic.
+opt_selects.extend([
+        (('bcsel', 'a(has_multiple_uses)', ('fneg(is_used_once)', 'b@32'), b),
+         ('ixor', b, ('bcsel', a, 1 << 31, 0))),
+
+        (('bcsel', 'a(has_multiple_uses)', b, ('fneg(is_used_once)', 'b@32')),
+         ('ixor', b, ('bcsel', a, 0, 1 << 31))),
+])
+
+fuse_extr = []
+for start in range(32):
+    fuse_extr.extend([
+        (('ior', ('ushr', 'a@32', start), ('ishl', 'b@32', 32 - start)),
+         ('extr_agx', a, b, start, 0)),
+    ])
+
+fuse_ubfe = []
+for bits in range(1, 32):
+    fuse_ubfe.extend([
+        (('iand', ('ushr', 'a@32', b), (1 << bits) - 1),
+         ('ubitfield_extract', a, b, bits))
+    ])
+
+# (x * y) + s = (x * y) + (s << 0)
+def imad(x, y, z):
+    return ('imadshl_agx', x, y, z, 0)
+
+# (x * y) - s = (x * y) - (s << 0)
+def imsub(x, y, z):
+    return ('imsubshl_agx', x, y, z, 0)
+
+# x + (y << s) = (x * 1) + (y << s)
+def iaddshl(x, y, s):
+    return ('imadshl_agx', x, 1, y, s)
+
+# x - (y << s) = (x * 1) - (y << s)
+def isubshl(x, y, s):
+    return ('imsubshl_agx', x, 1, y, s)
+
+fuse_imad = [
+    # Reassociate imul+iadd chain in order to fuse imads. This pattern comes up
+    # in compute shader lowering.
+    (('iadd', ('iadd(is_used_once)', ('imul(is_used_once)', a, b),
+              ('imul(is_used_once)', c, d)), e),
+     imad(a, b, imad(c, d, e))),
+
+    # Fuse regular imad
+    (('iadd', ('imul(is_used_once)', a, b), c), imad(a, b, c)),
+    (('isub', ('imul(is_used_once)', a, b), c), imsub(a, b, c)),
+]
+
+for s in range(1, 5):
+    fuse_imad += [
+        # Definitions
+        (('iadd', a, ('ishl(is_used_once)', b, s)), iaddshl(a, b, s)),
+        (('isub', a, ('ishl(is_used_once)', b, s)), isubshl(a, b, s)),
+
+        # ineg(x) is 0 - x
+        (('ineg', ('ishl(is_used_once)', b, s)), isubshl(0, b, s)),
+
+        # Definitions
+        (imad(a, b, ('ishl(is_used_once)', c, s)), ('imadshl_agx', a, b, c, s)),
+        (imsub(a, b, ('ishl(is_used_once)', c, s)), ('imsubshl_agx', a, b, c, s)),
+
+        # The above but after the below shift lowering
+        (imad(a, b, ('imadshl_agx(is_used_once)', 0, 1, c, s)), ('imadshl_agx', a, b, c, s)),
+        (imsub(a, b, ('imadshl_agx(is_used_once)', 0, 1, c, s)), ('imsubshl_agx', a, b, c, s)),
+
+        # a + (a << s) = a + a * (1 << s) = a * (1 + (1 << s))
+        (('imul', a, 1 + (1 << s)), iaddshl(a, a, s)),
+
+        # a - (a << s) = a - a * (1 << s) = a * (1 - (1 << s))
+        (('imul', a, 1 - (1 << s)), isubshl(a, a, s)),
+
+        # a - (a << s) = a * (1 - (1 << s)) = -(a * (1 << s) - 1)
+        (('ineg', ('imul(is_used_once)', a, (1 << s) - 1)), isubshl(a, a, s)),
+
+        # iadd is SCIB, general shfit is IC (slower)
+        (('ishl', a, s), iaddshl(0, a, s)),
+    ]
+
+# If the above rules failed, we have a large constant shift on the IC unit.
+# Might as well fuse an add to form an imad, if we're on the IC anyway.
+fuse_imad += [
+    (('iadd', a, ('ishl(is_used_once)', b, '#c')), imad(b, ('ishl', 1, c), a)),
+]
+
+# Discard lowering generates this pattern, clean it up
+ixor_bcsel = [
+   (('ixor', ('bcsel', a, '#b', '#c'), '#d'),
+    ('bcsel', a, ('ixor', b, d), ('ixor', c, d))),
+]
+
+# The main NIR optimizer works on imul, not iadd. We need just enough patterns
+# for amul to let us fuse lea.
+cleanup_amul = [
+   # Neither operation overflows so we can keep the amul.
+   (('amul', ('amul', a, '#b'), '#c'), ('amul', a, ('imul', b, c))),
+
+   # Result of u2u64 has zero in upper half, so the shift doesn't overflow, so
+   # neither multiplication overflows.
+   (('amul', ('ishl', ('u2u64', 'a@32'), '#b(is_ult_32)'), '#c'),
+    ('amul', ('u2u64', a), ('ishl', c, b))),
+
+   # We depending on simplifying this.
+   (('iadd', a, 0), a),
+]
+
+fuse_lea = []
+
+# Handle 64-bit address arithmetic (OpenCL)
+for s in range(1, 5):
+    pot = 1 << s
+
+    fuse_lea += [
+        # A + (#b + c) 2^s = (A + c 2^s) + #b 2^s
+        (('iadd', 'a@64', ('amul', pot, ('iadd', '#b(is_upper_half_zero)', ('u2u64', 'c@32')))),
+         ('ulea_agx', ('ulea_agx', a, c, s), ('u2u32', b), s)),
+
+        # A + (B + c) 2^s = (A + B 2^s) + c 2^s
+        (('iadd', 'a@64', ('amul', ('iadd', 'b@64', ('i2i64', 'c@32')), pot)),
+         ('ilea_agx', ('iadd', a, ('ishl', b, s)), c, s)),
+
+        # A + 2^s (B + (C + d)) = (A + (B + C)2^s) + d 2^s
+        (('iadd', 'a@64', ('amul', ('iadd', 'b@64',
+                                   ('iadd', 'c@64', ('u2u64', 'd@32'))), pot)),
+         ('ulea_agx', ('iadd', a, ('ishl', ('iadd', b, c), s)), d, s)),
+    ]
+
+    for sgn in ["u", "i"]:
+        upconv = f'{sgn}2{sgn}64'
+        lea = f'{sgn}lea_agx'
+
+        fuse_lea += [
+            # Basic pattern match
+            (('iadd', 'a@64', ('amul', (upconv, 'b@32'), pot)), (lea, a, b, s)),
+            (('iadd', 'a@64', ('ishl', (upconv, 'b@32'), s)), (lea, a, b, s)),
+        ]
+
+# Handle relaxed 32-bit address arithmetic (OpenGL, Vulkan)
+for s_ in range(1, 5):
+    # Iterate backwards
+    s = 5 - s_
+
+    v = 1 << s
+    is_mult = f'(is_unsigned_multiple_of_{v})'
+
+    fuse_lea += [
+        # A + b * s = A + B * s with relaxed multiply
+        (('iadd', 'a@64', ('u2u64', ('amul', 'b@32', v))),
+         ('ulea_agx', a, b, s)),
+
+        # A + (b * c 2^s) = A + (b * c) 2^s with relaxed multiply
+        (('iadd', 'a@64', ('u2u64', ('amul', 'b@32', f'#c{is_mult}'))),
+         ('ulea_agx', a, ('imul', b, ('ushr', c, s)), s)),
+
+        # A + (b 2^s + c d 2^s) = A + (b + cd) 2^s with relaxation.
+        #
+        # amul is bounded by the buffer size by definition, and both the GL & VK
+        # limit UBOs and SSBOs to INT32_MAX bytes. Therefore, amul has no signed
+        # wrap.
+        #
+        # Further, because we are zero-extending the 32-bit result, the 32-bit
+        # sum must be nonnegative -- if it were negative, it would represent an
+        # offset above INT32_MAX which would be invalid given the amul and
+        # max buffer size. Thus with signed math
+        #
+        #   0 <= b 2^s + cd 2^s < INT32_MAX
+        #
+        # ..and hence
+        #
+        #   0 <= b + cd < INT32_MAX
+        #
+        # Those bounds together with distributivity mean that
+        #
+        #   (b 2^s + cd 2^s) mod 2^32 = 2^s ((b + cd) mod 2^32)
+        #
+        # ...which is exactly what we need to factor out the shift.
+        (('iadd', 'a@64', ('u2u64', ('iadd', f'#b{is_mult}',
+                                     ('amul', 'c@32', f'#d{is_mult}')))),
+         ('ulea_agx', a, ('iadd', ('ishr', b, s),
+                                  ('amul', 'c@32', ('ishr', d, s))), s)),
+    ]
+
+# 8-bit rules are lowest precedence since we really like to fuse shifts
+fuse_lea += [
+    (('iadd', a, ('u2u64', 'b@32')), ('ulea_agx', a, b, 0)),
+    (('iadd', a, ('i2i64', 'b@32')), ('ilea_agx', a, b, 0)),
+
+    (('iadd', a, ('iadd', ('u2u64', 'b@32'), c)),
+     ('ulea_agx', ('iadd', a, c), b, 0)),
+    (('iadd', a, ('iadd', ('i2i64', 'b@32'), c)),
+     ('ilea_agx', ('iadd', a, c), b, 0)),
+]
+
+# After lowering address arithmetic, the various address arithmetic opcodes are
+# no longer useful. Lower them to regular arithmetic to let nir_opt_algebraic
+# take over.
+lower_lea = [
+    (('amul', a, b), ('imul', a, b)),
+    (('ulea_agx', a, b, c), ('iadd', a, ('ishl', ('u2u64', b), c))),
+    (('ilea_agx', a, b, c), ('iadd', a, ('ishl', ('i2i64', b), c))),
+]
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-p', '--import-path', required=True)
+    args = parser.parse_args()
+    sys.path.insert(0, args.import_path)
+    run()
+
+def run():
+    import nir_algebraic  # pylint: disable=import-error
+
+    print('#include "agx_nir.h"')
+
+    print(nir_algebraic.AlgebraicPass("agx_nir_cleanup_amul", cleanup_amul).render())
+    print(nir_algebraic.AlgebraicPass("agx_nir_fuse_lea", fuse_lea).render())
+    print(nir_algebraic.AlgebraicPass("agx_nir_lower_lea", lower_lea).render())
+
+    print(nir_algebraic.AlgebraicPass("agx_nir_lower_algebraic_late",
+                                      lower_sm5_shift + lower_pack +
+                                      lower_selects).render())
+    print(nir_algebraic.AlgebraicPass("agx_nir_fuse_selects",
+                                      opt_selects).render())
+    print(nir_algebraic.AlgebraicPass("agx_nir_fuse_algebraic_late",
+                                      fuse_extr + fuse_ubfe +
+                                      fuse_imad + ixor_bcsel).render())
+
+
+if __name__ == '__main__':
+    main()

src/asahi/compiler/agx_nir_lower_address.c

@@ -0,0 +1,179 @@
+/*
+ * Copyright 2022 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#include <stdint.h>
+#include "compiler/nir/nir_builder.h"
+#include "agx_nir.h"
+#include "nir.h"
+#include "nir_intrinsics.h"
+#include "nir_opcodes.h"
+
+struct match {
+   nir_scalar base, offset;
+   bool sign_extend;
+   uint8_t shift;
+};
+
+static enum pipe_format
+format_for_bitsize(unsigned bitsize)
+{
+   switch (bitsize) {
+   case 8:
+      return PIPE_FORMAT_R8_UINT;
+   case 16:
+      return PIPE_FORMAT_R16_UINT;
+   case 32:
+      return PIPE_FORMAT_R32_UINT;
+   default:
+      UNREACHABLE("should have been lowered");
+   }
+}
+
+static bool
+pass(struct nir_builder *b, nir_intrinsic_instr *intr, void *data)
+{
+   if (intr->intrinsic != nir_intrinsic_load_global &&
+       intr->intrinsic != nir_intrinsic_load_global_constant &&
+       intr->intrinsic != nir_intrinsic_global_atomic &&
+       intr->intrinsic != nir_intrinsic_global_atomic_swap &&
+       intr->intrinsic != nir_intrinsic_store_global)
+      return false;
+
+   b->cursor = nir_before_instr(&intr->instr);
+
+   unsigned bitsize = intr->intrinsic == nir_intrinsic_store_global
+                         ? nir_src_bit_size(intr->src[0])
+                         : intr->def.bit_size;
+   enum pipe_format format = format_for_bitsize(bitsize);
+   unsigned format_shift = util_logbase2(util_format_get_blocksize(format));
+
+   nir_src *orig_offset = nir_get_io_offset_src(intr);
+   nir_scalar base = nir_scalar_resolved(orig_offset->ssa, 0);
+   struct match match = {.base = base};
+   bool shift_must_match =
+      (intr->intrinsic == nir_intrinsic_global_atomic) ||
+      (intr->intrinsic == nir_intrinsic_global_atomic_swap);
+   unsigned max_shift = format_shift + (shift_must_match ? 0 : 2);
+
+   if (nir_scalar_is_alu(base)) {
+      nir_op op = nir_scalar_alu_op(base);
+      if (op == nir_op_ulea_agx || op == nir_op_ilea_agx) {
+         unsigned shift = nir_scalar_as_uint(nir_scalar_chase_alu_src(base, 2));
+         if (shift >= format_shift && shift <= max_shift) {
+            match = (struct match){
+               .base = nir_scalar_chase_alu_src(base, 0),
+               .offset = nir_scalar_chase_alu_src(base, 1),
+               .shift = shift - format_shift,
+               .sign_extend = (op == nir_op_ilea_agx),
+            };
+         }
+      } else if (op == nir_op_iadd) {
+         for (unsigned i = 0; i < 2; ++i) {
+            nir_scalar const_scalar = nir_scalar_chase_alu_src(base, i);
+            if (!nir_scalar_is_const(const_scalar))
+               continue;
+
+            /* Put scalar into form (k*2^n), clamping n at the maximum hardware
+             * shift.
+             */
+            int64_t raw_scalar = nir_scalar_as_uint(const_scalar);
+            assert(raw_scalar != 0 && "must have been optimized out");
+
+            uint32_t shift = MIN2(__builtin_ctz(raw_scalar), max_shift);
+            int64_t k = raw_scalar >> shift;
+
+            /* See if the reduced scalar is from a sign extension. We must have
+             * at least the format shift to avoid underflowing.
+             */
+            if (k > INT32_MAX || k < INT32_MIN || shift < format_shift)
+               break;
+
+            /* Match the constant */
+            match = (struct match){
+               .base = nir_scalar_chase_alu_src(base, 1 - i),
+               .offset = nir_get_scalar(nir_imm_int(b, k), 0),
+               .shift = shift - format_shift,
+               .sign_extend = true,
+            };
+
+            break;
+         }
+      }
+   }
+
+   nir_def *offset = match.offset.def != NULL
+                        ? nir_channel(b, match.offset.def, match.offset.comp)
+                        : nir_imm_int(b, 0);
+
+   nir_def *new_base = nir_channel(b, match.base.def, match.base.comp);
+
+   nir_def *repl = NULL;
+   bool has_dest = (intr->intrinsic != nir_intrinsic_store_global);
+   unsigned num_components = has_dest ? intr->def.num_components : 0;
+   unsigned bit_size = has_dest ? intr->def.bit_size : 0;
+
+   if (intr->intrinsic == nir_intrinsic_load_global) {
+      repl =
+         nir_load_agx(b, num_components, bit_size, new_base, offset,
+                      .access = nir_intrinsic_access(intr), .base = match.shift,
+                      .format = format, .sign_extend = match.sign_extend);
+
+   } else if (intr->intrinsic == nir_intrinsic_load_global_constant) {
+      repl = nir_load_constant_agx(b, num_components, bit_size, new_base,
+                                   offset, .access = nir_intrinsic_access(intr),
+                                   .base = match.shift, .format = format,
+                                   .sign_extend = match.sign_extend);
+   } else if (intr->intrinsic == nir_intrinsic_global_atomic) {
+      repl =
+         nir_global_atomic_agx(b, bit_size, new_base, offset, intr->src[1].ssa,
+                               .atomic_op = nir_intrinsic_atomic_op(intr),
+                               .sign_extend = match.sign_extend);
+   } else if (intr->intrinsic == nir_intrinsic_global_atomic_swap) {
+      repl = nir_global_atomic_swap_agx(
+         b, bit_size, new_base, offset, intr->src[1].ssa, intr->src[2].ssa,
+         .atomic_op = nir_intrinsic_atomic_op(intr),
+         .sign_extend = match.sign_extend);
+   } else {
+      nir_store_agx(b, intr->src[0].ssa, new_base, offset,
+                    .access = nir_intrinsic_access(intr), .base = match.shift,
+                    .format = format, .sign_extend = match.sign_extend);
+   }
+
+   if (repl)
+      nir_def_rewrite_uses(&intr->def, repl);
+
+   nir_instr_remove(&intr->instr);
+   return true;
+}
+
+bool
+agx_nir_lower_address(nir_shader *nir)
+{
+   bool progress = false;
+
+   /* First, clean up as much as possible. This will make fusing more effective.
+    */
+   do {
+      progress = false;
+      NIR_PASS(progress, nir, agx_nir_cleanup_amul);
+      NIR_PASS(progress, nir, nir_opt_constant_folding);
+      NIR_PASS(progress, nir, nir_opt_dce);
+   } while (progress);
+
+   /* Then, fuse as many lea as possible */
+   NIR_PASS(progress, nir, agx_nir_fuse_lea);
+
+   /* Next, lower load/store using the lea's */
+   NIR_PASS(progress, nir, nir_shader_intrinsics_pass, pass,
+            nir_metadata_control_flow, NULL);
+
+   /* Finally, lower any leftover lea instructions back to ALU to let
+    * nir_opt_algebraic simplify them from here.
+    */
+   NIR_PASS(progress, nir, agx_nir_lower_lea);
+   NIR_PASS(progress, nir, nir_opt_dce);
+
+   return progress;
+}

src/asahi/compiler/agx_nir_lower_cull_distance.c

@@ -0,0 +1,127 @@
+/*
+ * Copyright 2023 Valve Corporation
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "compiler/nir/nir.h"
+#include "compiler/nir/nir_builder.h"
+#include "util/macros.h"
+#include "agx_compile.h"
+#include "agx_nir.h"
+#include "glsl_types.h"
+#include "nir_builder_opcodes.h"
+#include "shader_enums.h"
+
+/*
+ * Lower cull distance to discard. From the spec:
+ *
+ *    If the cull distance for any enabled cull half-space is negative for all
+ *    of the vertices of the primitive under consideration, the primitive is
+ *    discarded.
+ *
+ * We don't have a direct way to read the cull distance at non-provoking
+ * vertices in the fragment shader. Instead, we interpolate the quantity:
+ *
+ *    cull distance >= 0.0 ? 1.0 : 0.0
+ *
+ * Then, the discard condition is equivalent to:
+ *
+ *    "quantity is zero for all vertices of the primitive"
+ *
+ * which by linearity is equivalent to:
+ *
+ *    quantity is zero somewhere in the primitive and quantity has zero
+ *    first-order screen space derivatives.
+ *
+ * which we can determine with ease in the fragment shader.
+ */
+
+static bool
+lower_write(nir_builder *b, nir_intrinsic_instr *intr, UNUSED void *data)
+{
+   if (intr->intrinsic != nir_intrinsic_store_output)
+      return false;
+
+   nir_io_semantics sem = nir_intrinsic_io_semantics(intr);
+   if (sem.location != VARYING_SLOT_CLIP_DIST0 &&
+       sem.location != VARYING_SLOT_CLIP_DIST1)
+      return false;
+
+   signed loc = sem.location + nir_src_as_uint(intr->src[1]);
+   unsigned total_component =
+      (loc - VARYING_SLOT_CLIP_DIST0) * 4 + nir_intrinsic_component(intr);
+
+   unsigned base = b->shader->info.clip_distance_array_size;
+   if (total_component < base)
+      return false;
+
+   unsigned component = total_component - base;
+   if (component >= b->shader->info.cull_distance_array_size)
+      return false;
+
+   assert(nir_src_num_components(intr->src[0]) == 1 && "must be scalarized");
+
+   b->cursor = nir_before_instr(&intr->instr);
+   nir_def *offs = nir_imm_int(b, component / 4);
+   nir_def *v = nir_b2f32(b, nir_fge_imm(b, intr->src[0].ssa, 0.0));
+
+   nir_store_output(b, v, offs, .component = component % 4,
+                    .src_type = nir_type_float32,
+                    .io_semantics.location = VARYING_SLOT_CULL_PRIMITIVE,
+                    .io_semantics.num_slots = 2);
+   return true;
+}
+
+bool
+agx_nir_lower_cull_distance_vs(nir_shader *s)
+{
+   assert(s->info.stage == MESA_SHADER_VERTEX ||
+          s->info.stage == MESA_SHADER_TESS_EVAL);
+
+   nir_shader_intrinsics_pass(s, lower_write, nir_metadata_control_flow, NULL);
+
+   s->info.outputs_written |=
+      BITFIELD64_RANGE(VARYING_SLOT_CULL_PRIMITIVE,
+                       DIV_ROUND_UP(s->info.cull_distance_array_size, 4));
+   return true;
+}
+
+bool
+agx_nir_lower_cull_distance_fs(nir_shader *s, unsigned nr_distances)
+{
+   assert(s->info.stage == MESA_SHADER_FRAGMENT);
+   assert(nr_distances > 0);
+
+   nir_builder b_ =
+      nir_builder_at(nir_before_impl(nir_shader_get_entrypoint(s)));
+   nir_builder *b = &b_;
+
+   /* Test each half-space */
+   nir_def *culled = nir_imm_false(b);
+
+   for (unsigned i = 0; i < nr_distances; ++i) {
+      /* Load the coefficient vector for this half-space. Imaginapple
+       * partial derivatives and the value somewhere.
+       */
+      nir_def *cf = nir_load_coefficients_agx(
+         b, nir_imm_int(b, 0), .component = i & 3,
+         .io_semantics.location = VARYING_SLOT_CULL_PRIMITIVE + (i / 4),
+         .io_semantics.num_slots = nr_distances / 4,
+         .interp_mode = INTERP_MODE_NOPERSPECTIVE);
+
+      /* If the coefficients are identically zero, then the quantity is
+       * zero across the primtive <==> cull distance is negative across the
+       * primitive <==> the primitive is culled.
+       */
+      culled = nir_ior(b, culled, nir_ball(b, nir_feq_imm(b, cf, 0)));
+   }
+
+   /* Emulate primitive culling by discarding fragments */
+   nir_demote_if(b, culled);
+
+   s->info.inputs_read |= BITFIELD64_RANGE(VARYING_SLOT_CULL_PRIMITIVE,
+                                           DIV_ROUND_UP(nr_distances, 4));
+
+   s->info.fs.uses_discard = true;
+   return nir_progress(true, b->impl, nir_metadata_control_flow);
+}

src/asahi/compiler/agx_nir_lower_discard_zs_emit.c

@@ -0,0 +1,148 @@
+/*
+ * Copyright 2022 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "compiler/nir/nir.h"
+#include "compiler/nir/nir_builder.h"
+#include "agx_compiler.h"
+#include "nir_builder_opcodes.h"
+
+#define ALL_SAMPLES 0xFF
+#define BASE_Z      1
+#define BASE_S      2
+
+static bool
+lower_zs_emit(nir_block *block, bool force_early_z)
+{
+   nir_intrinsic_instr *zs_emit = NULL;
+   bool progress = false;
+
+   nir_foreach_instr_reverse_safe(instr, block) {
+      if (instr->type != nir_instr_type_intrinsic)
+         continue;
+
+      nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
+      if (intr->intrinsic != nir_intrinsic_store_output)
+         continue;
+
+      nir_io_semantics sem = nir_intrinsic_io_semantics(intr);
+      if (sem.location != FRAG_RESULT_DEPTH &&
+          sem.location != FRAG_RESULT_STENCIL)
+         continue;
+
+      /* If early-Z is forced, z/s writes are a no-op (and will cause problems
+       * later in the compile). Piglit early-z tests this. Just remove the
+       * offending writes.
+       */
+      if (force_early_z) {
+         nir_instr_remove(instr);
+         progress = true;
+         continue;
+      }
+
+      nir_builder b = nir_builder_at(nir_before_instr(instr));
+
+      nir_def *value = intr->src[0].ssa;
+      bool z = (sem.location == FRAG_RESULT_DEPTH);
+
+      unsigned src_idx = z ? 1 : 2;
+      unsigned base = z ? BASE_Z : BASE_S;
+
+      /* In the hw, depth is 32-bit but stencil is 16-bit. Instruction
+       * selection checks this, so emit the conversion now.
+       */
+      if (z)
+         value = nir_f2f32(&b, value);
+      else
+         value = nir_u2u16(&b, value);
+
+      if (zs_emit == NULL) {
+         /* Multisampling will get lowered later if needed, default to
+          * broadcast
+          */
+         nir_def *sample_mask = nir_imm_intN_t(&b, ALL_SAMPLES, 16);
+         zs_emit =
+            nir_store_zs_agx(&b, sample_mask, nir_undef(&b, 1, 32) /* depth */,
+                             nir_undef(&b, 1, 16) /* stencil */);
+      }
+
+      assert((nir_intrinsic_base(zs_emit) & base) == 0 &&
+             "each of depth/stencil may only be written once");
+
+      nir_src_rewrite(&zs_emit->src[src_idx], value);
+      nir_intrinsic_set_base(zs_emit, nir_intrinsic_base(zs_emit) | base);
+
+      nir_instr_remove(instr);
+      progress = true;
+   }
+
+   return progress;
+}
+
+static bool
+lower_discard(nir_builder *b, nir_intrinsic_instr *intr, UNUSED void *data)
+{
+   if (intr->intrinsic != nir_intrinsic_demote &&
+       intr->intrinsic != nir_intrinsic_demote_if)
+      return false;
+
+   b->cursor = nir_before_instr(&intr->instr);
+
+   nir_def *all_samples = nir_imm_intN_t(b, ALL_SAMPLES, 16);
+   nir_def *no_samples = nir_imm_intN_t(b, 0, 16);
+   nir_def *killed_samples = all_samples;
+
+   if (intr->intrinsic == nir_intrinsic_demote_if)
+      killed_samples = nir_bcsel(b, intr->src[0].ssa, all_samples, no_samples);
+
+   /* This will get lowered later as needed */
+   nir_demote_samples(b, killed_samples);
+   nir_instr_remove(&intr->instr);
+   return true;
+}
+
+static bool
+agx_nir_lower_discard(nir_shader *s)
+{
+   if (!s->info.fs.uses_discard)
+      return false;
+
+   return nir_shader_intrinsics_pass(s, lower_discard,
+                                     nir_metadata_control_flow, NULL);
+}
+
+static bool
+agx_nir_lower_zs_emit(nir_shader *s)
+{
+   /* If depth/stencil isn't written, there's nothing to lower */
+   if (!(s->info.outputs_written & (BITFIELD64_BIT(FRAG_RESULT_STENCIL) |
+                                    BITFIELD64_BIT(FRAG_RESULT_DEPTH))))
+      return false;
+
+   bool any_progress = false;
+
+   nir_foreach_function_impl(impl, s) {
+      bool progress = false;
+
+      nir_foreach_block(block, impl) {
+         progress |= lower_zs_emit(block, s->info.fs.early_fragment_tests);
+      }
+
+      any_progress |= nir_progress(progress, impl, nir_metadata_control_flow);
+   }
+
+   return any_progress;
+}
+
+bool
+agx_nir_lower_discard_zs_emit(nir_shader *s)
+{
+   bool progress = false;
+
+   /* Lower depth/stencil writes before discard so the interaction works */
+   progress |= agx_nir_lower_zs_emit(s);
+   progress |= agx_nir_lower_discard(s);
+
+   return progress;
+}

src/asahi/compiler/agx_nir_lower_fminmax.c

@@ -0,0 +1,78 @@
+/*
+ * Copyright 2024 Valve Corporation
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "compiler/nir/nir.h"
+#include "compiler/nir/nir_builder.h"
+#include "agx_nir.h"
+
+/*
+ * AGX generally flushes FP32 denorms. However, the min/max instructions do not
+ * as they are implemented with cmpsel. We need to flush the results of fp32
+ * min/max for correctness. Doing so naively will generate redundant flushes, so
+ * this pass tries to be clever and elide flushes when possible.
+ *
+ * This pass is still pretty simple, it doesn't see through phis or bcsels yet.
+ */
+static bool
+could_be_denorm(nir_scalar s)
+{
+   /* Constants can be denorms only if they are denorms. */
+   if (nir_scalar_is_const(s)) {
+      return fpclassify(nir_scalar_as_float(s)) == FP_SUBNORMAL;
+   }
+
+   /* Floating-point instructions flush denormals, so ALU results can only be
+    * denormal if they are not from a float instruction. Crucially fmin/fmax
+    * flushes in NIR, so this pass handles chains of fmin/fmax properly.
+    */
+   if (nir_scalar_is_alu(s)) {
+      nir_op op = nir_scalar_alu_op(s);
+      nir_alu_type T = nir_op_infos[op].output_type;
+
+      return nir_alu_type_get_base_type(T) != nir_type_float &&
+             op != nir_op_fmin_agx && op != nir_op_fmax_agx;
+   }
+
+   /* Otherwise, assume it could be denormal (say, loading from a buffer). */
+   return true;
+}
+
+static bool
+lower(nir_builder *b, nir_alu_instr *alu, void *data)
+{
+   if ((alu->op != nir_op_fmin && alu->op != nir_op_fmax) ||
+       (alu->def.bit_size != 32))
+      return false;
+
+   /* Lower the op, we'll fix up the denorms right after. */
+   if (alu->op == nir_op_fmax)
+      alu->op = nir_op_fmax_agx;
+   else
+      alu->op = nir_op_fmin_agx;
+
+   /* We need to canonicalize the result if the output could be a denorm. That
+    * occurs only when one of the sources could be a denorm. Check each source.
+    * Swizzles don't affect denormalness so we can grab the def directly.
+    */
+   nir_scalar scalar = nir_get_scalar(&alu->def, 0);
+   nir_scalar src0 = nir_scalar_chase_alu_src(scalar, 0);
+   nir_scalar src1 = nir_scalar_chase_alu_src(scalar, 1);
+
+   if (could_be_denorm(src0) || could_be_denorm(src1)) {
+      b->cursor = nir_after_instr(&alu->instr);
+      b->fp_math_ctrl = nir_fp_no_fast_math;
+
+      nir_def *canonicalized = nir_fadd_imm(b, &alu->def, -0.0);
+      nir_def_rewrite_uses_after(&alu->def, canonicalized);
+   }
+
+   return true;
+}
+
+bool
+agx_nir_lower_fminmax(nir_shader *s)
+{
+   return nir_shader_alu_pass(s, lower, nir_metadata_control_flow, NULL);
+}

src/asahi/compiler/agx_nir_lower_frag_sidefx.c

@@ -0,0 +1,103 @@
+/*
+ * Copyright 2022 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "compiler/nir/nir.h"
+#include "compiler/nir/nir_builder.h"
+#include "agx_compiler.h"
+
+/* Fragment shaders with side effects require special handling to ensure the
+ * side effects execute as intended. By default, they require late depth
+ * testing, to ensure the side effects happen even for killed pixels. To handle,
+ * the driver inserts a dummy `gl_FragDepth = gl_Position.z` in shaders that
+ * don't otherwise write their depth, forcing a late depth test.
+ *
+ * For side effects with force early testing forced, the sample mask is written
+ * at the *beginning* of the shader.
+ */
+
+#define ALL_SAMPLES (0xFF)
+
+static void
+insert_z_write(nir_builder *b)
+{
+   /* When forcing depth test to NEVER, force Z to NaN.  This works around the
+    * hardware discarding depth=NEVER.
+    *
+    * TODO: Check if Apple has a better way of handling this.
+    */
+   nir_def *bias = nir_bcsel(b, nir_ine_imm(b, nir_load_depth_never_agx(b), 0),
+                             nir_imm_float(b, NAN), nir_imm_float(b, -0.0));
+
+   nir_def *z = nir_fadd(b, bias, nir_load_frag_coord_z(b));
+
+   nir_store_output(b, z, nir_imm_int(b, 0),
+                    .io_semantics.location = FRAG_RESULT_DEPTH,
+                    .src_type = nir_type_float32);
+
+   b->shader->info.outputs_written |= BITFIELD64_BIT(FRAG_RESULT_DEPTH);
+}
+
+static bool
+pass(struct nir_builder *b, nir_intrinsic_instr *intr, void *data)
+{
+   if (intr->intrinsic != nir_intrinsic_store_output)
+      return false;
+
+   /* Only lower once */
+   bool *done = data;
+   if (*done)
+      return false;
+   *done = true;
+
+   b->cursor = nir_before_instr(&intr->instr);
+   insert_z_write(b);
+   return true;
+}
+
+bool
+agx_nir_lower_frag_sidefx(nir_shader *s)
+{
+   assert(s->info.stage == MESA_SHADER_FRAGMENT);
+
+   /* If there are no side effects, there's nothing to lower */
+   if (!s->info.writes_memory)
+      return false;
+
+   /* Lower writes from helper invocations with the common pass. The hardware
+    * will predicate simple writes for us, but that fails with sample shading so
+    * we do that in software too.
+    */
+   NIR_PASS(_, s, nir_lower_helper_writes, s->info.fs.uses_sample_shading);
+
+   bool writes_zs =
+      s->info.outputs_written &
+      (BITFIELD64_BIT(FRAG_RESULT_STENCIL) | BITFIELD64_BIT(FRAG_RESULT_DEPTH));
+
+   /* If the shader wants early fragment tests, the sample mask lowering pass
+    * will trigger an early test at the beginning of the shader. This lets us
+    * use a Passthrough punch type, instead of Opaque which may result in the
+    * shader getting skipped incorrectly and then the side effects not kicking
+    * in. But this happens there to avoid it happening twice with a discard.
+    */
+   if (s->info.fs.early_fragment_tests)
+      return false;
+
+   /* If depth/stencil feedback is already used, we're done */
+   if (writes_zs)
+      return false;
+
+   bool done = false;
+   nir_shader_intrinsics_pass(s, pass, nir_metadata_control_flow, &done);
+
+   /* If there's no render targets written, just put the write at the end */
+   if (!done) {
+      nir_function_impl *impl = nir_shader_get_entrypoint(s);
+      nir_builder b = nir_builder_at(nir_after_impl(impl));
+
+      insert_z_write(&b);
+   }
+
+   return true;
+}

src/asahi/compiler/agx_nir_lower_interpolation.c

@@ -0,0 +1,169 @@
+/*
+ * Copyright 2023 Valve Corporation
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "compiler/shader_enums.h"
+#include "agx_compile.h"
+#include "nir.h"
+#include "nir_builder.h"
+#include "nir_builder_opcodes.h"
+#include "nir_intrinsics.h"
+#include "nir_intrinsics_indices.h"
+
+/*
+ * In AGX, the values of fragment shader inputs are represented as coefficient
+ * vectors <A, B, C>, which are dotted with <x, y, 1> to perform interpolation.
+ * x and y are relative to the tile. In other words, A and B are the
+ * screen-space partial derivatives of the input, and C is the value at the
+ * corner of the tile.
+ *
+ * For some interpolation modes, the dot product happens in the iterator
+ * hardware. Other modes are implemented in this file, by lowering to math on
+ * the coefficient vectors.
+ */
+
+/* XXX: It's not clear what this is for, but seems necessary */
+static nir_def *
+cf_valid(nir_builder *b, nir_def *cf)
+{
+   nir_def *bit = nir_ieq_imm(b, nir_iand_imm(b, nir_channel(b, cf, 0), 1), 0);
+
+   /* XXX: Apple's compiler actually checks that the significand is nonzero and
+    * the exponent is 0 or 1. This is probably a typo -- it doesn't make any
+    * logical sense.  Presumably they just meant to check for denorms, so let's
+    * do that. Either way the tests pass.
+    */
+   nir_def *cf01 = nir_trim_vector(b, cf, 2);
+   return nir_ior(b, bit, nir_fisnormal(b, cf01));
+}
+
+static nir_def *
+interpolate_at_offset(nir_builder *b, nir_def *cf, nir_def *offset,
+                      bool perspective)
+{
+   /* Get the coordinate of the pixel within the tile */
+   nir_def *pixel_coords = nir_load_pixel_coord(b);
+   nir_def *tile_offs = nir_umod_imm(b, pixel_coords, 32);
+
+   /* Convert to float, getting the center of the pixel */
+   nir_def *center = nir_fadd_imm(b, nir_u2f32(b, tile_offs), 0.5);
+
+   /* Calculate the location to interpolate. offset is defined relative to the
+    * center of the pixel and is a float.
+    */
+   nir_def *pos = nir_fadd(b, center, nir_f2f32(b, offset));
+
+   /* Interpolate with the given coefficients */
+   nir_def *interp = nir_ffma_weak(b, nir_channel(b, pos, 1), nir_channel(b, cf, 1),
+                                   nir_channel(b, cf, 2));
+
+   interp = nir_ffma_weak(b, nir_channel(b, pos, 0), nir_channel(b, cf, 0), interp);
+
+   /* Divide by RHW. This load will be lowered recursively. */
+   if (perspective) {
+      nir_def *bary = nir_load_barycentric_at_offset(
+         b, 32, offset, .interp_mode = INTERP_MODE_NOPERSPECTIVE);
+
+      nir_def *rhw = nir_load_interpolated_input(
+         b, 1, 32, bary, nir_imm_int(b, 0), .component = 3,
+         .io_semantics = {
+            .location = VARYING_SLOT_POS,
+            .num_slots = 1,
+         });
+
+      interp = nir_fdiv(b, interp, rhw);
+   }
+
+   /* Replace invalid interpolations with the constant channel */
+   return nir_bcsel(b, cf_valid(b, cf), interp, nir_channel(b, cf, 2));
+}
+
+static nir_def *
+interpolate_flat(nir_builder *b, nir_def *coefficients)
+{
+   /* Same value anywhere, so just take the constant (affine) component. For
+    * triangle fans with the first provoking vertex, the CF layout is slightly
+    * different. I am unsure why, but Apple does the same and the bcsel is
+    * required for corrctness.
+    */
+   return nir_bcsel(b, nir_load_is_first_fan_agx(b),
+                    nir_channel(b, coefficients, 1),
+                    nir_channel(b, coefficients, 2));
+}
+
+static enum glsl_interp_mode
+interp_mode_for_load(nir_intrinsic_instr *load)
+{
+   if (load->intrinsic == nir_intrinsic_load_input)
+      return INTERP_MODE_FLAT;
+   else
+      return nir_intrinsic_interp_mode(nir_src_as_intrinsic(load->src[0]));
+}
+
+static bool
+needs_lower(const nir_instr *instr, UNUSED const void *_)
+{
+   if (instr->type != nir_instr_type_intrinsic)
+      return false;
+
+   const nir_intrinsic_instr *load = nir_instr_as_intrinsic(instr);
+
+   /* at_offset barycentrics need to be lowered */
+   if (load->intrinsic == nir_intrinsic_load_interpolated_input) {
+      return (nir_src_as_intrinsic(load->src[0])->intrinsic ==
+              nir_intrinsic_load_barycentric_at_offset);
+   }
+
+   /* Flat shading always lowered */
+   return (load->intrinsic == nir_intrinsic_load_input);
+}
+
+static nir_def *
+interpolate_channel(nir_builder *b, nir_intrinsic_instr *load, unsigned channel)
+{
+   nir_def *coefficients = nir_load_coefficients_agx(
+      b, nir_get_io_offset_src(load)->ssa,
+      .component = nir_intrinsic_component(load) + channel,
+      .interp_mode = interp_mode_for_load(load),
+      .io_semantics = nir_intrinsic_io_semantics(load));
+
+   if (load->intrinsic == nir_intrinsic_load_input) {
+      assert(load->def.bit_size == 32);
+
+      if (nir_intrinsic_io_semantics(load).location == VARYING_SLOT_LAYER)
+         return nir_load_layer_id(b);
+      else
+         return interpolate_flat(b, coefficients);
+   } else {
+      nir_intrinsic_instr *bary = nir_src_as_intrinsic(load->src[0]);
+
+      nir_def *interp = interpolate_at_offset(
+         b, coefficients, bary->src[0].ssa,
+         nir_intrinsic_interp_mode(bary) != INTERP_MODE_NOPERSPECTIVE);
+
+      return nir_f2fN(b, interp, load->def.bit_size);
+   }
+}
+
+static nir_def *
+lower(nir_builder *b, nir_instr *instr, void *data)
+{
+   nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
+
+   /* Each component is loaded separated */
+   nir_def *values[NIR_MAX_VEC_COMPONENTS] = {NULL};
+   for (unsigned i = 0; i < intr->def.num_components; ++i) {
+      values[i] = interpolate_channel(b, intr, i);
+   }
+
+   return nir_vec(b, values, intr->def.num_components);
+}
+
+bool
+agx_nir_lower_interpolation(nir_shader *s)
+{
+   assert(s->info.stage == MESA_SHADER_FRAGMENT);
+
+   return nir_shader_lower_instructions(s, needs_lower, lower, NULL);
+}

src/asahi/compiler/agx_nir_lower_sample_mask.c

@@ -0,0 +1,226 @@
+/*
+ * Copyright 2023 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "compiler/list.h"
+#include "compiler/nir/nir_builder.h"
+#include "agx_compiler.h"
+#include "nir.h"
+#include "nir_builder_opcodes.h"
+#include "nir_intrinsics.h"
+
+/*
+ * sample_mask takes two bitmasks as arguments, TARGET and LIVE. Each bit refers
+ * to an indexed sample. Roughly, the instruction does:
+ *
+ *    foreach sample in TARGET {
+ *       if sample in LIVE {
+ *          run depth/stencil/occlusion test/update
+ *       } else {
+ *          kill sample
+ *       }
+ *    }
+ *
+ * As a special case, TARGET may be set to all-1s (~0) to refer to all samples
+ * regardless of the framebuffer sample count.
+ *
+ * For example, to discard an entire pixel unconditionally, we could run:
+ *
+ *    sample_mask ~0, 0
+ *
+ * sample_mask must follow these rules:
+ *
+ * 1. All sample_mask instructions affecting a sample must execute before a
+ *    local_store_pixel instruction targeting that sample. This ensures that
+ *    nothing is written for discarded samples (whether discarded in shader or
+ *    due to a failed depth/stencil test).
+ *
+ * 2. If sample_mask is used anywhere in a shader, then on every execution path,
+ *    every sample must be killed or else run depth/stencil tests exactly ONCE.
+ *
+ * 3. If a sample is killed, future sample_mask instructions have
+ *    no effect on that sample. The following code sequence correctly implements
+ *    a conditional discard (if there are no other sample_mask instructions in
+ *    the shader):
+ *
+ *       sample_mask discarded, 0
+ *       sample_mask ~0, ~0
+ *
+ *    but this sequence is incorrect:
+ *
+ *       sample_mask ~0, ~discarded
+ *       sample_mask ~0, ~0         <-- incorrect: depth/stencil tests run twice
+ *
+ * 4. Conversely, if a sample is tested, future sample_mask instructions may not
+ *    discard that sample. The following code is invalid:
+ *
+ *      sample_mask ~0, ~0
+ *      sample_mask discarded, 0
+ *      st_tile
+ *
+ *    To implement the semantic of "force early tests with discard", manual
+ *    colour masking must be used. It's a weird case, but CTS does it.
+ *
+ * 5. zs_emit may be used in the shader exactly once to trigger tests.
+ * sample_mask with 0 may be used to discard early.
+ *
+ * This pass lowers demote_samples to sample_mask instructions satisfying these
+ * rules. Other passes should not generate sample_mask instructions, as there
+ * are too many footguns.
+ */
+
+#define ALL_SAMPLES (0xFF)
+#define BASE_Z      1
+#define BASE_S      2
+
+static bool
+lower_discard_to_sample_mask_0(nir_builder *b, nir_intrinsic_instr *intr,
+                               UNUSED void *data)
+{
+   if (intr->intrinsic != nir_intrinsic_demote_samples)
+      return false;
+
+   b->cursor = nir_before_instr(&intr->instr);
+   nir_sample_mask_agx(b, nir_u2u16(b, intr->src[0].ssa),
+                       nir_imm_intN_t(b, 0, 16));
+   nir_instr_remove(&intr->instr);
+   return true;
+}
+
+static nir_intrinsic_instr *
+last_discard_in_block(nir_block *block)
+{
+   nir_foreach_instr_reverse(instr, block) {
+      if (instr->type != nir_instr_type_intrinsic)
+         continue;
+
+      nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
+      if (intr->intrinsic == nir_intrinsic_demote_samples)
+         return intr;
+   }
+
+   return NULL;
+}
+
+static bool
+cf_node_contains_discard(nir_cf_node *node)
+{
+   nir_foreach_block_in_cf_node(block, node) {
+      if (last_discard_in_block(block))
+         return true;
+   }
+
+   return false;
+}
+
+/*
+ * We want to run depth/stencil tests as early as possible, but we have to
+ * wait until after the last discard. We find the last discard and
+ * execute depth/stencil tests in the first unconditional block after (if
+ * in conditional control flow), or fuse depth/stencil tests into the
+ * sample instruction (if in unconditional control flow).
+ *
+ * To do so, we walk the root control flow list backwards, looking for the
+ * earliest unconditionally executed instruction after all discard.
+ */
+static void
+run_tests_after_last_discard(nir_builder *b)
+{
+   foreach_list_typed_reverse(nir_cf_node, node, node, &b->impl->body) {
+      if (node->type == nir_cf_node_block) {
+         /* Unconditionally executed block */
+         nir_block *block = nir_cf_node_as_block(node);
+         nir_intrinsic_instr *intr = last_discard_in_block(block);
+
+         if (intr) {
+            /* Last discard is executed unconditionally, so fuse tests:
+             *
+             *    sample_mask (testing | killed), ~killed
+             *
+             * When testing, this is `sample_mask ~0, ~killed` which kills the
+             * kill set and triggers tests on the rest.
+             *
+             * When not testing, this is `sample_mask killed, ~killed` which is
+             * equivalent to `sample_mask killed, 0`, killing without testing.
+             */
+            b->cursor = nir_before_instr(&intr->instr);
+
+            nir_def *all_samples = nir_imm_intN_t(b, ALL_SAMPLES, 16);
+            nir_def *killed = nir_u2u16(b, intr->src[0].ssa);
+            nir_def *live = nir_ixor(b, killed, all_samples);
+
+            nir_def *testing = nir_load_shader_part_tests_zs_agx(b);
+            nir_def *affected = nir_ior(b, testing, killed);
+
+            nir_sample_mask_agx(b, affected, live);
+            nir_instr_remove(&intr->instr);
+            return;
+         } else {
+            /* Set cursor for insertion due to a preceding conditionally
+             * executed discard.
+             */
+            b->cursor = nir_before_block_after_phis(block);
+         }
+      } else if (cf_node_contains_discard(node)) {
+         /* Conditionally executed block contains the last discard. Test
+          * depth/stencil for remaining samples in unconditional code after.
+          *
+          * If we're not testing, this turns into sample_mask(0, ~0) which is a
+          * no-op.
+          */
+         nir_sample_mask_agx(b, nir_load_shader_part_tests_zs_agx(b),
+                             nir_imm_intN_t(b, ALL_SAMPLES, 16));
+         return;
+      }
+   }
+}
+
+static void
+run_tests_at_start(nir_shader *shader)
+{
+   nir_function_impl *impl = nir_shader_get_entrypoint(shader);
+   nir_builder b = nir_builder_at(nir_before_impl(impl));
+
+   nir_sample_mask_agx(&b, nir_imm_intN_t(&b, ALL_SAMPLES, 16),
+                       nir_imm_intN_t(&b, ALL_SAMPLES, 16));
+}
+
+bool
+agx_nir_lower_sample_mask(nir_shader *shader)
+{
+   nir_function_impl *impl = nir_shader_get_entrypoint(shader);
+
+   bool writes_zs =
+      shader->info.outputs_written &
+      (BITFIELD64_BIT(FRAG_RESULT_STENCIL) | BITFIELD64_BIT(FRAG_RESULT_DEPTH));
+
+   if (shader->info.fs.early_fragment_tests) {
+      /* run tests early, if we need testing */
+      if (shader->info.fs.uses_discard || writes_zs ||
+          shader->info.writes_memory) {
+
+         run_tests_at_start(shader);
+      }
+   } else if (shader->info.fs.uses_discard) {
+      /* If we have zs_emit, the tests will be triggered by zs_emit, otherwise
+       * we need to trigger tests explicitly. Allow sample_mask with zs_emit.
+       */
+      if (!writes_zs) {
+         nir_builder b = nir_builder_create(impl);
+
+         /* run tests late */
+         run_tests_after_last_discard(&b);
+      }
+   } else {
+      /* regular shaders that don't use discard have nothing to lower */
+      return nir_no_progress(impl);
+   }
+
+   nir_progress(true, impl, nir_metadata_control_flow);
+
+   nir_shader_intrinsics_pass(shader, lower_discard_to_sample_mask_0,
+                              nir_metadata_control_flow, NULL);
+
+   return true;
+}

src/asahi/compiler/agx_nir_lower_shared_bitsize.c

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2022 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "compiler/nir/nir_builder.h"
+#include "agx_compiler.h"
+
+/* Local memory instructions require 16-bit offsets, so we add conversions. */
+static bool
+pass(struct nir_builder *b, nir_intrinsic_instr *intr, UNUSED void *data)
+{
+   nir_src *offset = nir_get_io_offset_src(intr);
+   if (!nir_is_shared_access(intr) || nir_src_bit_size(*offset) == 16)
+      return false;
+
+   b->cursor = nir_before_instr(&intr->instr);
+   nir_src_rewrite(offset, nir_u2u16(b, offset->ssa));
+   return true;
+}
+
+bool
+agx_nir_lower_shared_bitsize(nir_shader *shader)
+{
+   return nir_shader_intrinsics_pass(shader, pass, nir_metadata_control_flow,
+                                     NULL);
+}

src/asahi/compiler/agx_nir_lower_subgroups.c

@@ -0,0 +1,237 @@
+/*
+ * Copyright 2023 Valve Corporation
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "compiler/nir/nir.h"
+#include "compiler/nir/nir_builder.h"
+#include "util/list.h"
+#include "agx_nir.h"
+#include "nir_builder_opcodes.h"
+#include "nir_intrinsics.h"
+#include "nir_intrinsics_indices.h"
+#include "nir_opcodes.h"
+
+/* XXX: cribbed from nak, move to common */
+static nir_def *
+nir_udiv_round_up(nir_builder *b, nir_def *n, nir_def *d)
+{
+   return nir_udiv(b, nir_iadd(b, n, nir_iadd_imm(b, d, -1)), d);
+}
+
+static bool
+lower(nir_builder *b, nir_intrinsic_instr *intr, void *data)
+{
+   b->cursor = nir_before_instr(&intr->instr);
+
+   switch (intr->intrinsic) {
+   case nir_intrinsic_quad_vote_any: {
+      nir_def *ballot = nir_quad_ballot_agx(b, 16, intr->src[0].ssa);
+      nir_def_rewrite_uses(&intr->def, nir_ine_imm(b, ballot, 0));
+      return true;
+   }
+
+   case nir_intrinsic_quad_vote_all: {
+      nir_def *ballot =
+         nir_quad_ballot_agx(b, 16, nir_inot(b, intr->src[0].ssa));
+      nir_def_rewrite_uses(&intr->def, nir_ieq_imm(b, ballot, 0));
+      return true;
+   }
+
+   case nir_intrinsic_elect: {
+      nir_def *active_id = nir_load_active_subgroup_invocation_agx(b, 16);
+      nir_def_rewrite_uses(&intr->def, nir_ieq_imm(b, active_id, 0));
+      return true;
+   }
+
+   case nir_intrinsic_first_invocation: {
+      nir_def *active_id = nir_load_active_subgroup_invocation_agx(b, 16);
+      nir_def *is_first = nir_ieq_imm(b, active_id, 0);
+      nir_def *first_bit = nir_ballot(b, 1, 32, is_first);
+      nir_def_rewrite_uses(&intr->def, nir_ufind_msb(b, first_bit));
+      return true;
+   }
+
+   case nir_intrinsic_last_invocation: {
+      nir_def *active_mask = nir_ballot(b, 1, 32, nir_imm_true(b));
+      nir_def_rewrite_uses(&intr->def, nir_ufind_msb(b, active_mask));
+      return true;
+   }
+
+   case nir_intrinsic_vote_ieq:
+   case nir_intrinsic_vote_feq: {
+      /* The common lowering does:
+       *
+       *    vote_all(x == read_first(x))
+       *
+       * This is not optimal for AGX, since we have ufind_msb but not ctz, so
+       * it's cheaper to read the last invocation than the first. So we do:
+       *
+       *    vote_all(x == read_last(x))
+       *
+       * implemented with lowered instructions as
+       *
+       *    ballot(x != broadcast(x, ffs(ballot(true)))) == 0
+       */
+      nir_def *active_mask = nir_ballot(b, 1, 32, nir_imm_true(b));
+      nir_def *active_bit = nir_ufind_msb(b, active_mask);
+      nir_def *other = nir_read_invocation(b, intr->src[0].ssa, active_bit);
+      nir_def *is_ne;
+
+      if (intr->intrinsic == nir_intrinsic_vote_feq) {
+         is_ne = nir_fneu(b, other, intr->src[0].ssa);
+      } else {
+         is_ne = nir_ine(b, other, intr->src[0].ssa);
+      }
+
+      nir_def *ballot = nir_ballot(b, 1, 32, is_ne);
+      nir_def_rewrite_uses(&intr->def, nir_ieq_imm(b, ballot, 0));
+      return true;
+   }
+
+   case nir_intrinsic_load_num_subgroups: {
+      nir_def *workgroup_size = nir_load_workgroup_size(b);
+      workgroup_size = nir_imul(b,
+                                nir_imul(b, nir_channel(b, workgroup_size, 0),
+                                         nir_channel(b, workgroup_size, 1)),
+                                nir_channel(b, workgroup_size, 2));
+      nir_def *subgroup_size = nir_imm_int(b, 32);
+      nir_def *num_subgroups =
+         nir_udiv_round_up(b, workgroup_size, subgroup_size);
+      nir_def_rewrite_uses(&intr->def, num_subgroups);
+      return true;
+   }
+
+   case nir_intrinsic_shuffle: {
+      nir_def *data = intr->src[0].ssa;
+      nir_def *target = intr->src[1].ssa;
+
+      /* The hardware shuffle instruction chooses a single index within the
+       * target quad to shuffle each source quad with. Consequently, the low
+       * 2-bits of shuffle indices should not be quad divergent.  To implement
+       * arbitrary shuffle, pull each low 2-bits index in the quad separately.
+       */
+      nir_def *quad_start = nir_iand_imm(b, target, 0x1c);
+      nir_def *result = NULL;
+
+      for (unsigned i = 0; i < 4; ++i) {
+         nir_def *target_i = nir_iadd_imm(b, quad_start, i);
+         nir_def *shuf = nir_read_invocation(b, data, target_i);
+
+         if (result)
+            result = nir_bcsel(b, nir_ieq(b, target, target_i), shuf, result);
+         else
+            result = shuf;
+      }
+
+      nir_def_rewrite_uses(&intr->def, result);
+      return true;
+   }
+
+   case nir_intrinsic_inclusive_scan: {
+      /* If we got here, we support the corresponding exclusive scan in
+       * hardware, so just handle the last element.
+       */
+      nir_op red_op = nir_intrinsic_reduction_op(intr);
+      nir_def *data = intr->src[0].ssa;
+
+      b->cursor = nir_after_instr(&intr->instr);
+      intr->intrinsic = nir_intrinsic_exclusive_scan;
+      nir_def *accum = nir_build_alu2(b, red_op, data, &intr->def);
+      nir_def_rewrite_uses_after(&intr->def, accum);
+      return true;
+   }
+
+   case nir_intrinsic_ballot: {
+      /* Optimize popcount(ballot(true)) to load_active_subgroup_count_agx() */
+      if (!nir_src_is_const(intr->src[0]) || !nir_src_as_bool(intr->src[0]) ||
+          !list_is_singular(&intr->def.uses))
+         return false;
+
+      nir_src *use = list_first_entry(&intr->def.uses, nir_src, use_link);
+      nir_instr *parent = nir_src_use_instr(use);
+      if (parent->type != nir_instr_type_alu)
+         return false;
+
+      nir_alu_instr *alu = nir_instr_as_alu(parent);
+      if (alu->op != nir_op_bit_count)
+         return false;
+
+      nir_def_rewrite_uses(&alu->def,
+                           nir_load_active_subgroup_count_agx(b, 32));
+      return true;
+   }
+
+   default:
+      return false;
+   }
+}
+
+static bool
+lower_subgroup_filter(const nir_intrinsic_instr *intr, UNUSED const void *data)
+{
+   /* Use default behaviour for everything but scans */
+   if (intr->intrinsic != nir_intrinsic_exclusive_scan &&
+       intr->intrinsic != nir_intrinsic_inclusive_scan &&
+       intr->intrinsic != nir_intrinsic_reduce)
+      return true;
+
+   if (intr->def.num_components > 1 || intr->def.bit_size == 1)
+      return true;
+
+   /* Hardware supports quad ops but no other support clustered reductions. */
+   if (nir_intrinsic_has_cluster_size(intr)) {
+      unsigned cluster = nir_intrinsic_cluster_size(intr);
+      if (cluster && cluster != 4 && cluster < 32)
+         return true;
+   }
+
+   switch (nir_intrinsic_reduction_op(intr)) {
+   case nir_op_imul:
+   case nir_op_fmin:
+   case nir_op_fmax:
+      /* no hardware scan for this opcode, always lower it */
+      return true;
+
+   case nir_op_iadd:
+   case nir_op_iand:
+   case nir_op_ixor:
+   case nir_op_ior:
+      /* these have dedicated 64-bit lowering paths that use the 32-bit hardware
+       * instructions so are likely better than the full lowering.
+       */
+      return false;
+
+   default:
+      /* otherwise, lower 64-bit, since the hw ops are at most 32-bit. */
+      return intr->def.bit_size == 64;
+   }
+}
+
+bool
+agx_nir_lower_subgroups(nir_shader *s)
+{
+   /* First, do as much common lowering as we can */
+   nir_lower_subgroups_options opts = {
+      .filter = lower_subgroup_filter,
+      .lower_read_first_invocation = true,
+      .lower_inverse_ballot = true,
+      .lower_to_scalar = true,
+      .lower_relative_shuffle = true,
+      .lower_rotate_to_shuffle = true,
+      .lower_subgroup_masks = true,
+      .lower_reduce = true,
+      .lower_vote = true,
+      .ballot_components = 1,
+      .ballot_bit_size = 32,
+      .subgroup_size = 32,
+   };
+
+   bool progress = nir_lower_subgroups(s, &opts);
+
+   /* Then do AGX-only lowerings on top */
+   progress |=
+      nir_shader_intrinsics_pass(s, lower, nir_metadata_control_flow, NULL);
+
+   return progress;
+}

src/asahi/compiler/agx_nir_lower_texture.c

@@ -0,0 +1,848 @@
+/*
+ * Copyright 2023 Valve Corporation
+ * Copyright 2021 Alyssa Rosenzweig
+ * Copyright 2020 Collabora Ltd.
+ * Copyright 2016 Broadcom
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "compiler/nir/nir.h"
+#include "compiler/nir/nir_builder.h"
+#include "agx_nir.h"
+#include "agx_nir_texture.h"
+#include "glsl_types.h"
+#include "libagx.h"
+#include "nir_builder_opcodes.h"
+#include "nir_builtin_builder.h"
+#include "nir_intrinsics.h"
+#include "nir_intrinsics_indices.h"
+#include "shader_enums.h"
+
+/* Residency flags are inverted from NIR */
+#define AGX_RESIDENT (0)
+
+static bool
+fence_image(struct nir_builder *b, nir_intrinsic_instr *intr, void *data)
+{
+   b->cursor = nir_after_instr(&intr->instr);
+
+   /* If the image is write-only, there is no fencing needed */
+   if (nir_intrinsic_has_access(intr) &&
+       (nir_intrinsic_access(intr) & ACCESS_NON_READABLE)) {
+      return false;
+   }
+
+   switch (intr->intrinsic) {
+   case nir_intrinsic_image_store:
+   case nir_intrinsic_bindless_image_store:
+      nir_fence_pbe_to_tex_agx(b);
+      return true;
+
+   case nir_intrinsic_image_atomic:
+   case nir_intrinsic_bindless_image_atomic:
+   case nir_intrinsic_image_atomic_swap:
+   case nir_intrinsic_bindless_image_atomic_swap:
+      nir_fence_mem_to_tex_agx(b);
+      return true;
+
+   default:
+      return false;
+   }
+}
+
+static nir_def *
+texture_descriptor_ptr(nir_builder *b, nir_tex_instr *tex)
+{
+   int handle_idx = nir_tex_instr_src_index(tex, nir_tex_src_texture_handle);
+   assert(handle_idx >= 0 && "must be bindless");
+   return nir_load_from_texture_handle_agx(b, tex->src[handle_idx].src.ssa);
+}
+
+static bool
+has_nonzero_lod(nir_tex_instr *tex)
+{
+   int idx = nir_tex_instr_src_index(tex, nir_tex_src_lod);
+   if (idx < 0)
+      return false;
+
+   nir_src src = tex->src[idx].src;
+   return !(nir_src_is_const(src) && nir_src_as_uint(src) == 0);
+}
+
+static bool
+lower_tex_crawl(nir_builder *b, nir_instr *instr, UNUSED void *data)
+{
+   if (instr->type != nir_instr_type_tex)
+      return false;
+
+   nir_tex_instr *tex = nir_instr_as_tex(instr);
+   b->cursor = nir_before_instr(instr);
+
+   if (tex->op != nir_texop_txs && tex->op != nir_texop_texture_samples &&
+       tex->op != nir_texop_query_levels)
+      return false;
+
+   nir_def *ptr = texture_descriptor_ptr(b, tex);
+   unsigned nr_comps = tex->def.num_components;
+   assert(nr_comps <= 3);
+
+   int lod_idx = nir_tex_instr_src_index(tex, nir_tex_src_lod);
+   nir_def *lod = lod_idx >= 0 ? nir_u2u16(b, tex->src[lod_idx].src.ssa)
+                               : nir_imm_intN_t(b, 0, 16);
+
+   nir_def *res;
+   if (tex->op == nir_texop_txs) {
+      res =
+         libagx_txs(b, ptr, lod, nir_imm_int(b, nr_comps),
+                    nir_imm_bool(b, tex->sampler_dim == GLSL_SAMPLER_DIM_BUF),
+                    nir_imm_bool(b, tex->sampler_dim == GLSL_SAMPLER_DIM_1D),
+                    nir_imm_bool(b, tex->sampler_dim == GLSL_SAMPLER_DIM_2D),
+                    nir_imm_bool(b, tex->sampler_dim == GLSL_SAMPLER_DIM_CUBE),
+                    nir_imm_bool(b, tex->is_array));
+   } else if (tex->op == nir_texop_query_levels) {
+      res = libagx_texture_levels(b, ptr);
+   } else {
+      res = libagx_texture_samples(b, ptr);
+   }
+
+   nir_def_rewrite_uses(&tex->def, nir_trim_vector(b, res, nr_comps));
+   nir_instr_remove(instr);
+   return true;
+}
+
+/*
+ * Given a 1D buffer texture coordinate, calculate the 2D coordinate vector that
+ * will be used to access the linear 2D texture bound to the buffer.
+ */
+static nir_def *
+coords_for_buffer_texture(nir_builder *b, nir_def *coord)
+{
+   return nir_vec2(b, nir_umod_imm(b, coord, AGX_TEXTURE_BUFFER_WIDTH),
+                   nir_udiv_imm(b, coord, AGX_TEXTURE_BUFFER_WIDTH));
+}
+
+/*
+ * Buffer textures are lowered to 2D (1024xN) textures in the driver to access
+ * more storage. When lowering, we need to fix up the coordinate accordingly.
+ *
+ * Furthermore, RGB32 formats are emulated by lowering to global memory access,
+ * so to read a buffer texture we generate code that looks like:
+ *
+ *    if (descriptor->format == RGB32)
+ *       return ((uint32_t *) descriptor->address)[x];
+ *    else
+ *       return txf(texture_as_2d, vec2(x % 1024, x / 1024));
+ */
+static bool
+lower_buffer_texture(nir_builder *b, nir_tex_instr *tex)
+{
+   nir_def *coord = nir_steal_tex_src(tex, nir_tex_src_coord);
+   nir_def *size = nir_get_texture_size(b, tex);
+   nir_def *oob = nir_uge(b, coord, size);
+
+   /* Apply the buffer offset after calculating oob but before remapping */
+   nir_def *desc = texture_descriptor_ptr(b, tex);
+   coord = libagx_buffer_texture_offset(b, desc, coord);
+
+   /* Map out-of-bounds indices to out-of-bounds coordinates for robustness2
+    * semantics from the hardware.
+    */
+   coord = nir_bcsel(b, oob, nir_imm_int(b, -1), coord);
+
+   bool is_float = nir_alu_type_get_base_type(tex->dest_type) == nir_type_float;
+
+   /* Lower RGB32 reads if the format requires. If we are out-of-bounds, we use
+    * the hardware path so we get a zero texel.
+    */
+   nir_if *nif = nir_push_if(
+      b, nir_iand(b, libagx_texture_is_rgb32(b, desc), nir_inot(b, oob)));
+
+   nir_def *rgb32 = nir_trim_vector(
+      b, libagx_texture_load_rgb32(b, desc, coord, nir_imm_bool(b, is_float)),
+      nir_tex_instr_result_size(tex));
+
+   /* Raw loads do not return residency information, but residency queries are
+    * supported on buffer textures. Fortunately, we do not need to support
+    * sparse RGB32 buffers, so we simply claim all RGB32 loads were resident.
+    * Nothing should hit this in practice, but if we don't do *something* here
+    * we'll get vector size mismatches which blow up in vkd3d-proton.
+    */
+   if (tex->is_sparse) {
+      rgb32 = nir_pad_vector_imm_int(b, rgb32, AGX_RESIDENT,
+                                     rgb32->num_components + 1);
+   }
+
+   nir_push_else(b, nif);
+
+   /* Otherwise, lower the texture instruction to read from 2D */
+   assert(coord->num_components == 1 && "buffer textures are 1D");
+   tex->sampler_dim = GLSL_SAMPLER_DIM_2D;
+
+   nir_def *coord2d = coords_for_buffer_texture(b, coord);
+   nir_instr_remove(&tex->instr);
+   nir_builder_instr_insert(b, &tex->instr);
+   nir_tex_instr_add_src(tex, nir_tex_src_backend1, coord2d);
+   nir_steal_tex_src(tex, nir_tex_src_sampler_handle);
+   nir_steal_tex_src(tex, nir_tex_src_sampler_offset);
+   nir_block *else_block = nir_cursor_current_block(b->cursor);
+   nir_pop_if(b, nif);
+
+   /* Put it together with a phi */
+   nir_def *phi = nir_if_phi(b, rgb32, &tex->def);
+   nir_def_rewrite_uses(&tex->def, phi);
+   nir_phi_instr *phi_instr = nir_def_as_phi(phi);
+   nir_phi_src *else_src = nir_phi_get_src_from_block(phi_instr, else_block);
+   nir_src_rewrite(&else_src->src, &tex->def);
+   return true;
+}
+
+/*
+ * NIR indexes into array textures with unclamped floats (integer for txf). AGX
+ * requires the index to be a clamped integer. Lower tex_src_coord into
+ * tex_src_backend1 for array textures by type-converting and clamping.
+ */
+static bool
+lower_regular_texture(nir_builder *b, nir_instr *instr, UNUSED void *data)
+{
+   if (instr->type != nir_instr_type_tex)
+      return false;
+
+   nir_tex_instr *tex = nir_instr_as_tex(instr);
+   b->cursor = nir_before_instr(instr);
+
+   if (nir_tex_instr_is_query(tex) && tex->op != nir_texop_lod)
+      return false;
+
+   if (tex->sampler_dim == GLSL_SAMPLER_DIM_BUF)
+      return lower_buffer_texture(b, tex);
+
+   /* Don't lower twice */
+   if (nir_tex_instr_src_index(tex, nir_tex_src_backend1) >= 0)
+      return false;
+
+   /* Get the coordinates */
+   nir_def *coord = nir_steal_tex_src(tex, nir_tex_src_coord);
+   nir_def *ms_idx = nir_steal_tex_src(tex, nir_tex_src_ms_index);
+
+   /* Apply txf workaround, see libagx_lower_txf_robustness */
+   bool is_txf = ((tex->op == nir_texop_txf) || (tex->op == nir_texop_txf_ms));
+
+   if (is_txf &&
+       (has_nonzero_lod(tex) || tex->is_array ||
+        nir_tex_instr_src_index(tex, nir_tex_src_min_lod) >= 0) &&
+       !(tex->backend_flags & AGX_TEXTURE_FLAG_NO_CLAMP)) {
+
+      int lod_idx = nir_tex_instr_src_index(tex, nir_tex_src_lod);
+      nir_def *lod =
+         lod_idx >= 0 ? tex->src[lod_idx].src.ssa : nir_undef(b, 1, 16);
+
+      nir_def *min_lod = nir_steal_tex_src(tex, nir_tex_src_min_lod);
+
+      unsigned lidx = coord->num_components - 1;
+      nir_def *layer = nir_channel(b, coord, lidx);
+
+      nir_def *replaced = libagx_lower_txf_robustness(
+         b, texture_descriptor_ptr(b, tex),
+         nir_imm_bool(b, has_nonzero_lod(tex)), lod,
+         nir_imm_bool(b, min_lod != NULL), min_lod ?: nir_undef(b, 1, 16),
+         nir_imm_bool(b, tex->is_array), layer, nir_channel(b, coord, 0));
+
+      coord = nir_vector_insert_imm(b, coord, replaced, 0);
+   }
+
+   /* The layer is always the last component of the NIR coordinate, split it off
+    * because we'll need to swizzle.
+    */
+   nir_def *layer = NULL;
+
+   if (tex->is_array && tex->op != nir_texop_lod) {
+      unsigned lidx = coord->num_components - 1;
+      nir_def *unclamped_layer = nir_channel(b, coord, lidx);
+      coord = nir_trim_vector(b, coord, lidx);
+
+      /* Round layer to nearest even */
+      if (!is_txf) {
+         unclamped_layer = nir_fround_even(b, unclamped_layer);
+
+         /* Explicitly round negative to avoid undefined behaviour when constant
+          * folding. This is load bearing on x86 builds.
+          */
+         unclamped_layer =
+            nir_f2u32(b, nir_fmax(b, unclamped_layer, nir_imm_float(b, 0.0f)));
+      }
+
+      /* For a cube array, the layer is zero-indexed component 3 of the
+       * coordinate but the number of layers is component 2 of the txs result.
+       */
+      if (tex->sampler_dim == GLSL_SAMPLER_DIM_CUBE) {
+         assert(lidx == 3 && "4 components");
+         lidx = 2;
+      }
+
+      /* Clamp to max layer = (# of layers - 1) for out-of-bounds handling.
+       * Layer must be 16-bits for the hardware, drop top bits after clamping.
+       *
+       * For txf, we drop out-of-bounds components rather than clamp, see the
+       * above txf robustness workaround.
+       */
+      if (!(tex->backend_flags & AGX_TEXTURE_FLAG_NO_CLAMP) && !is_txf) {
+         nir_def *txs = nir_get_texture_size(b, tex);
+         nir_def *nr_layers = nir_channel(b, txs, lidx);
+         nir_def *max_layer = nir_iadd_imm(b, nr_layers, -1);
+         layer = nir_umin(b, unclamped_layer, max_layer);
+      } else {
+         layer = unclamped_layer;
+      }
+
+      layer = nir_u2u16(b, layer);
+   }
+
+   /* Combine layer and multisample index into 32-bit so we don't need a vec5 or
+    * vec6 16-bit coordinate tuple, which would be inconvenient in NIR for
+    * little benefit (a minor optimization, I guess).
+    */
+   nir_def *sample_array = (ms_idx && layer)
+                              ? nir_pack_32_2x16_split(b, ms_idx, layer)
+                           : ms_idx ? nir_u2u32(b, ms_idx)
+                           : layer  ? nir_u2u32(b, layer)
+                                    : NULL;
+
+   /* Combine into the final 32-bit tuple */
+   if (sample_array != NULL) {
+      unsigned end = coord->num_components;
+      coord = nir_pad_vector(b, coord, end + 1);
+      coord = nir_vector_insert_imm(b, coord, sample_array, end);
+   }
+
+   nir_tex_instr_add_src(tex, nir_tex_src_backend1, coord);
+
+   /* Furthermore, if there is an offset vector, it must be packed */
+   nir_def *offset = nir_steal_tex_src(tex, nir_tex_src_offset);
+
+   if (offset != NULL) {
+      nir_def *packed = NULL;
+
+      for (unsigned c = 0; c < offset->num_components; ++c) {
+         nir_def *nibble = nir_iand_imm(b, nir_channel(b, offset, c), 0xF);
+         nir_def *shifted = nir_ishl_imm(b, nibble, 4 * c);
+
+         /* We pack with iadd instead of ior to let us fuse in the shift with an
+          * iadd-lsl instruction.
+          */
+         if (packed != NULL)
+            packed = nir_iadd(b, packed, shifted);
+         else
+            packed = shifted;
+      }
+
+      nir_tex_instr_add_src(tex, nir_tex_src_backend2, packed);
+   }
+
+   if (nir_tex_instr_src_index(tex, nir_tex_src_bias) >= 0 &&
+       nir_tex_instr_src_index(tex, nir_tex_src_min_lod) >= 0) {
+
+      nir_def *bias = nir_steal_tex_src(tex, nir_tex_src_bias);
+      nir_def *min_lod = nir_steal_tex_src(tex, nir_tex_src_min_lod);
+      nir_def *packed = nir_pack_32_2x16_split(b, bias, min_lod);
+      nir_tex_instr_add_src(tex, nir_tex_src_lod_bias_min_agx, packed);
+   }
+
+   /* We reserve bound sampler #0, so we offset bound samplers by 1 and
+    * otherwise map bound samplers as-is.
+    */
+   nir_def *sampler = nir_steal_tex_src(tex, nir_tex_src_sampler_offset);
+   if (!sampler)
+      sampler = nir_imm_intN_t(b, tex->sampler_index, 16);
+
+   if (!is_txf &&
+       nir_tex_instr_src_index(tex, nir_tex_src_sampler_handle) < 0) {
+
+      nir_tex_instr_add_src(tex, nir_tex_src_sampler_handle,
+                            nir_iadd_imm(b, nir_u2u16(b, sampler), 1));
+   }
+
+   return true;
+}
+
+static bool
+legalize_image_lod(nir_builder *b, nir_intrinsic_instr *intr, UNUSED void *data)
+{
+   nir_src *src;
+
+#define CASE(op, idx)                                                          \
+   case nir_intrinsic_##op:                                                    \
+   case nir_intrinsic_bindless_##op:                                           \
+      src = &intr->src[idx];                                                   \
+      break;
+
+   switch (intr->intrinsic) {
+      CASE(image_load, 3)
+      CASE(image_sparse_load, 3)
+      CASE(image_store, 4)
+      CASE(image_size, 1)
+   default:
+      return false;
+   }
+
+#undef CASE
+
+   if (src->ssa->bit_size == 16)
+      return false;
+
+   b->cursor = nir_before_instr(&intr->instr);
+   nir_src_rewrite(src, nir_i2i16(b, src->ssa));
+   return true;
+}
+
+static nir_def *
+txs_for_image(nir_builder *b, nir_intrinsic_instr *intr,
+              unsigned num_components, unsigned bit_size, bool query_samples)
+{
+   enum glsl_sampler_dim dim = nir_intrinsic_image_dim(intr);
+   nir_def *lod = query_samples ? NULL : intr->src[1].ssa;
+   nir_texop op = query_samples ? nir_texop_texture_samples : nir_texop_txs;
+
+   nir_def *res =
+      nir_build_tex(b, op, .texture_handle = intr->src[0].ssa, .lod = lod,
+                    .dim = dim, .is_array = nir_intrinsic_image_array(intr),
+                    .can_speculate = nir_instr_can_speculate(&intr->instr));
+
+   /* Cube images are implemented as 2D arrays, so we need to divide here. */
+   if (dim == GLSL_SAMPLER_DIM_CUBE && res->num_components > 2 &&
+       !query_samples) {
+      nir_def *divided = nir_udiv_imm(b, nir_channel(b, res, 2), 6);
+      res = nir_vector_insert_imm(b, res, divided, 2);
+   }
+
+   return res;
+}
+
+static nir_def *
+image_texel_address(nir_builder *b, nir_intrinsic_instr *intr,
+                    bool return_index)
+{
+   /* First, calculate the address of the PBE descriptor */
+   nir_def *desc_address =
+      nir_load_from_texture_handle_agx(b, intr->src[0].ssa);
+
+   nir_def *coord = intr->src[1].ssa;
+
+   /* For atomics, we always infer the format. We only go down this path with
+    * formatless intrinsics when lowering multisampled image stores, but that
+    * uses the return_index path that ignores the block size.
+    */
+   enum pipe_format format = nir_intrinsic_format(intr);
+   assert(return_index || format != PIPE_FORMAT_NONE);
+
+   nir_def *blocksize_B = nir_imm_int(b, util_format_get_blocksize(format));
+
+   enum glsl_sampler_dim dim = nir_intrinsic_image_dim(intr);
+   bool layered = nir_intrinsic_image_array(intr) ||
+                  (dim == GLSL_SAMPLER_DIM_CUBE) ||
+                  (dim == GLSL_SAMPLER_DIM_3D);
+
+   if (dim == GLSL_SAMPLER_DIM_BUF && return_index) {
+      return nir_channel(b, coord, 0);
+   } else if (dim == GLSL_SAMPLER_DIM_BUF) {
+      return libagx_buffer_texel_address(b, desc_address, coord, blocksize_B);
+   } else {
+      return libagx_image_texel_address(
+         b, desc_address, coord, nir_u2u32(b, intr->src[2].ssa), blocksize_B,
+         nir_imm_bool(b, dim == GLSL_SAMPLER_DIM_1D),
+         nir_imm_bool(b, dim == GLSL_SAMPLER_DIM_MS), nir_imm_bool(b, layered),
+         nir_imm_bool(b, return_index));
+   }
+}
+
+static void
+lower_buffer_image(nir_builder *b, nir_intrinsic_instr *intr)
+{
+   nir_def *coord_vector = intr->src[1].ssa;
+   nir_def *coord = nir_channel(b, coord_vector, 0);
+
+   /* If we're not bindless, assume we don't need an offset (GL driver) */
+   if (intr->intrinsic == nir_intrinsic_bindless_image_load ||
+       intr->intrinsic == nir_intrinsic_bindless_image_sparse_load) {
+
+      nir_def *desc = nir_load_from_texture_handle_agx(b, intr->src[0].ssa);
+      coord = libagx_buffer_texture_offset(b, desc, coord);
+   } else if (intr->intrinsic == nir_intrinsic_bindless_image_store) {
+      nir_def *desc = nir_load_from_texture_handle_agx(b, intr->src[0].ssa);
+      coord = libagx_buffer_image_offset(b, desc, coord);
+   }
+
+   /* Lower the buffer load/store to a 2D image load/store, matching the 2D
+    * texture/PBE descriptor the driver supplies for buffer images.
+    */
+   nir_def *coord2d = coords_for_buffer_texture(b, coord);
+   nir_src_rewrite(&intr->src[1], nir_pad_vector(b, coord2d, 4));
+   nir_intrinsic_set_image_dim(intr, GLSL_SAMPLER_DIM_2D);
+}
+
+static void
+lower_1d_image(nir_builder *b, nir_intrinsic_instr *intr)
+{
+   nir_def *coord = intr->src[1].ssa;
+   bool is_array = nir_intrinsic_image_array(intr);
+   nir_def *zero = nir_imm_intN_t(b, 0, coord->bit_size);
+
+   if (is_array) {
+      assert(coord->num_components >= 2);
+      coord =
+         nir_vec3(b, nir_channel(b, coord, 0), zero, nir_channel(b, coord, 1));
+   } else {
+      assert(coord->num_components >= 1);
+      coord = nir_vec2(b, coord, zero);
+   }
+
+   nir_src_rewrite(&intr->src[1], nir_pad_vector(b, coord, 4));
+   nir_intrinsic_set_image_dim(intr, GLSL_SAMPLER_DIM_2D);
+}
+
+/*
+ * Just like for txf, we need special handling around layers (and LODs, but we
+ * don't support mipmapped images yet) for robust image_loads. See
+ * libagx_lower_txf_robustness for more info.
+ */
+static bool
+lower_image_load_robustness(nir_builder *b, nir_intrinsic_instr *intr)
+{
+   if (nir_intrinsic_access(intr) & ACCESS_IN_BOUNDS)
+      return false;
+
+   /* We only need to worry about array-like loads */
+   enum glsl_sampler_dim dim = nir_intrinsic_image_dim(intr);
+   if (!nir_intrinsic_image_array(intr) && dim != GLSL_SAMPLER_DIM_CUBE)
+      return false;
+
+   /* Determine the coordinate component of the layer. Cubes and cube arrays
+    * keep their array in their last non-array coordinate component, other
+    * arrays are immediately after.
+    */
+   unsigned lidx = glsl_get_sampler_dim_coordinate_components(dim);
+   if (dim == GLSL_SAMPLER_DIM_CUBE)
+      lidx--;
+
+   nir_def *coord = intr->src[1].ssa;
+   nir_def *lod = nir_undef(b, 1, 16);
+   nir_def *layer = nir_channel(b, coord, lidx);
+
+   /* image_load is effectively the same as txf, reuse the txf lower */
+   nir_def *replaced = libagx_lower_txf_robustness(
+      b, nir_load_from_texture_handle_agx(b, intr->src[0].ssa),
+      nir_imm_bool(b, false /* lower LOD */), lod,
+      nir_imm_bool(b, false /* lower min LOD */), lod,
+      nir_imm_bool(b, true /* lower layer */), layer, nir_channel(b, coord, 0));
+
+   nir_src_rewrite(&intr->src[1], nir_vector_insert_imm(b, coord, replaced, 0));
+   return true;
+}
+
+static bool
+lower_images(nir_builder *b, nir_intrinsic_instr *intr, UNUSED void *data)
+{
+   b->cursor = nir_before_instr(&intr->instr);
+
+   switch (intr->intrinsic) {
+   case nir_intrinsic_image_load:
+   case nir_intrinsic_image_store:
+   case nir_intrinsic_bindless_image_load:
+   case nir_intrinsic_bindless_image_sparse_load:
+   case nir_intrinsic_bindless_image_store: {
+      /* Legalize MSAA index */
+      nir_src_rewrite(&intr->src[2], nir_u2u16(b, intr->src[2].ssa));
+
+      if (intr->intrinsic == nir_intrinsic_image_load ||
+          intr->intrinsic == nir_intrinsic_bindless_image_load ||
+          intr->intrinsic == nir_intrinsic_bindless_image_sparse_load) {
+         lower_image_load_robustness(b, intr);
+      }
+
+      switch (nir_intrinsic_image_dim(intr)) {
+      case GLSL_SAMPLER_DIM_1D:
+         lower_1d_image(b, intr);
+         return true;
+
+      case GLSL_SAMPLER_DIM_BUF:
+         lower_buffer_image(b, intr);
+         return true;
+
+      default:
+         return true;
+      }
+   }
+
+   case nir_intrinsic_bindless_image_size:
+   case nir_intrinsic_bindless_image_samples:
+      nir_def_rewrite_uses(
+         &intr->def,
+         txs_for_image(
+            b, intr, intr->def.num_components, intr->def.bit_size,
+            intr->intrinsic == nir_intrinsic_bindless_image_samples));
+      return true;
+
+   case nir_intrinsic_bindless_image_texel_address:
+      nir_def_rewrite_uses(&intr->def, image_texel_address(b, intr, false));
+      return true;
+
+   case nir_intrinsic_is_sparse_texels_resident:
+      /* Residency information is in bit 0, so we need to mask. Unclear what's
+       * in the upper bits. For now, let's match the blob.
+       */
+      nir_def_replace(
+         &intr->def,
+         nir_ieq_imm(b, nir_iand_imm(b, intr->src[0].ssa, 1), AGX_RESIDENT));
+      return true;
+
+   case nir_intrinsic_sparse_residency_code_and:
+      /* ior because residency codes are inverted from NIR */
+      nir_def_replace(&intr->def,
+                      nir_ior(b, intr->src[0].ssa, intr->src[1].ssa));
+      return true;
+
+   case nir_intrinsic_image_size:
+   case nir_intrinsic_image_texel_address:
+      UNREACHABLE("should've been lowered");
+
+   default:
+      return false;
+   }
+}
+
+/*
+ * Map out-of-bounds storage texel buffer accesses and multisampled image stores
+ * to -1 indices, which will become an out-of-bounds hardware access. This gives
+ * cheap robustness2.
+ */
+static bool
+lower_robustness(nir_builder *b, nir_intrinsic_instr *intr, UNUSED void *data)
+{
+   b->cursor = nir_before_instr(&intr->instr);
+
+   switch (intr->intrinsic) {
+   case nir_intrinsic_image_deref_load:
+   case nir_intrinsic_image_deref_sparse_load:
+   case nir_intrinsic_image_deref_store:
+      break;
+   default:
+      return false;
+   }
+
+   enum glsl_sampler_dim dim = nir_intrinsic_image_dim(intr);
+   bool array = nir_intrinsic_image_array(intr);
+   unsigned size_components = nir_image_intrinsic_coord_components(intr);
+
+   nir_def *deref = intr->src[0].ssa;
+   nir_def *coord = intr->src[1].ssa;
+
+   if (dim != GLSL_SAMPLER_DIM_BUF &&
+       !(dim == GLSL_SAMPLER_DIM_MS &&
+         intr->intrinsic == nir_intrinsic_image_deref_store))
+      return false;
+
+   /* Bounds check the coordinate */
+   nir_def *size =
+      nir_image_deref_size(b, size_components, 32, deref, nir_imm_int(b, 0),
+                           .image_dim = dim, .image_array = array);
+   nir_def *oob = nir_bany(b, nir_uge(b, coord, size));
+
+   /* Bounds check the sample */
+   if (dim == GLSL_SAMPLER_DIM_MS) {
+      nir_def *samples = nir_image_deref_samples(b, 32, deref, .image_dim = dim,
+                                                 .image_array = array);
+
+      oob = nir_ior(b, oob, nir_uge(b, intr->src[2].ssa, samples));
+   }
+
+   /* Replace the last coordinate component with a large coordinate for
+    * out-of-bounds. We pick 0xFFF0 as it fits in 16-bit, and it is not signed
+    * as 32-bit so we won't get in-bounds coordinates for arrays due to two's
+    * complement wraparound. Additionally it still meets this requirement after
+    * adding 0xF, the maximum tail offset.
+    *
+    * This ensures the resulting hardware coordinate is definitely
+    * out-of-bounds, giving hardware-level robustness2 behaviour.
+    */
+   unsigned c = size_components - 1;
+   nir_def *r =
+      nir_bcsel(b, oob, nir_imm_int(b, 0xFFF0), nir_channel(b, coord, c));
+
+   nir_src_rewrite(&intr->src[1], nir_vector_insert_imm(b, coord, r, c));
+   return true;
+}
+
+/*
+ * Early texture lowering passes, called by the driver before lowering
+ * descriptor bindings. That means these passes operate on texture derefs. The
+ * purpose is to make descriptor crawls explicit in the NIR, so that the driver
+ * can accurately lower descriptors after this pass but before calling
+ * the full agx_nir_lower_texture.
+ */
+bool
+agx_nir_lower_texture_early(nir_shader *s, bool support_lod_bias)
+{
+   bool progress = false;
+
+   NIR_PASS(progress, s, nir_shader_intrinsics_pass, lower_robustness,
+            nir_metadata_control_flow, NULL);
+
+   nir_lower_tex_options lower_tex_options = {
+      .lower_txp = ~0,
+      .lower_invalid_implicit_lod = true,
+      .lower_tg4_offsets = true,
+      .lower_index_to_offset = true,
+      .lower_sampler_lod_bias = support_lod_bias,
+
+      /* Unclear if/how mipmapped 1D textures work in the hardware. */
+      .lower_1d = true,
+
+      /* XXX: Metal seems to handle just like 3D txd, so why doesn't it work?
+       * TODO: Stop using this lowering
+       */
+      .lower_txd_cube_map = true,
+   };
+
+   NIR_PASS(progress, s, nir_lower_tex, &lower_tex_options);
+   return progress;
+}
+
+bool
+agx_nir_lower_texture(nir_shader *s)
+{
+   bool progress = false;
+
+   nir_tex_src_type_constraints tex_constraints = {
+      [nir_tex_src_lod] = {true, 16},
+      [nir_tex_src_bias] = {true, 16},
+      [nir_tex_src_ms_index] = {true, 16},
+      [nir_tex_src_min_lod] = {true, 16},
+      [nir_tex_src_texture_offset] = {true, 16},
+      [nir_tex_src_sampler_offset] = {true, 16},
+   };
+
+   /* Insert fences before lowering image atomics, since image atomics need
+    * different fencing than other image operations.
+    */
+   NIR_PASS(progress, s, nir_shader_intrinsics_pass, fence_image,
+            nir_metadata_control_flow, NULL);
+
+   NIR_PASS(progress, s, nir_lower_image_atomics_to_global, NULL, NULL);
+
+   NIR_PASS(progress, s, nir_shader_intrinsics_pass, legalize_image_lod,
+            nir_metadata_control_flow, NULL);
+   NIR_PASS(progress, s, nir_shader_intrinsics_pass, lower_images,
+            nir_metadata_control_flow, NULL);
+   NIR_PASS(progress, s, nir_legalize_16bit_sampler_srcs, tex_constraints);
+
+   /* Fold constants after nir_legalize_16bit_sampler_srcs so we can detect 0 in
+    * lower_regular_texture. This is required for correctness.
+    */
+   NIR_PASS(progress, s, nir_opt_constant_folding);
+
+   /* Lower texture sources after legalizing types (as the lowering depends on
+    * 16-bit multisample indices) but before lowering queries (as the lowering
+    * generates txs for array textures).
+    */
+   NIR_PASS(progress, s, nir_shader_instructions_pass, lower_regular_texture,
+            nir_metadata_none, NULL);
+   NIR_PASS(progress, s, nir_shader_instructions_pass, lower_tex_crawl,
+            nir_metadata_control_flow, NULL);
+
+   return progress;
+}
+
+static bool
+lower_multisampled_store(nir_builder *b, nir_intrinsic_instr *intr,
+                         UNUSED void *data)
+{
+   b->cursor = nir_before_instr(&intr->instr);
+
+   if (intr->intrinsic != nir_intrinsic_bindless_image_store)
+      return false;
+
+   if (nir_intrinsic_image_dim(intr) != GLSL_SAMPLER_DIM_MS)
+      return false;
+
+   nir_def *index_px = nir_u2u32(b, image_texel_address(b, intr, true));
+   nir_def *coord2d = coords_for_buffer_texture(b, index_px);
+
+   nir_src_rewrite(&intr->src[1], nir_pad_vector(b, coord2d, 4));
+   nir_src_rewrite(&intr->src[2], nir_imm_int(b, 0));
+   nir_intrinsic_set_image_dim(intr, GLSL_SAMPLER_DIM_2D);
+   nir_intrinsic_set_image_array(intr, false);
+   return true;
+}
+
+bool
+agx_nir_lower_multisampled_image_store(nir_shader *s)
+{
+   return nir_shader_intrinsics_pass(s, lower_multisampled_store,
+                                     nir_metadata_control_flow, NULL);
+}
+
+/*
+ * Given a non-bindless instruction, return whether agx_nir_lower_texture will
+ * lower it to something involving a descriptor crawl. This requires the driver
+ * to lower the instruction to bindless before calling agx_nir_lower_texture.
+ * The implementation just enumerates the cases handled in this file.
+ */
+bool
+agx_nir_needs_texture_crawl(nir_instr *instr)
+{
+   if (instr->type == nir_instr_type_intrinsic) {
+      nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
+
+      switch (intr->intrinsic) {
+      /* Queries, atomics always become a crawl */
+      case nir_intrinsic_image_size:
+      case nir_intrinsic_image_deref_size:
+      case nir_intrinsic_image_samples:
+      case nir_intrinsic_image_deref_samples:
+      case nir_intrinsic_image_atomic:
+      case nir_intrinsic_image_deref_atomic:
+      case nir_intrinsic_image_atomic_swap:
+      case nir_intrinsic_image_deref_atomic_swap:
+         return true;
+
+      /* Multisampled stores need a crawl, others do not */
+      case nir_intrinsic_image_store:
+      case nir_intrinsic_image_deref_store:
+         return nir_intrinsic_image_dim(intr) == GLSL_SAMPLER_DIM_MS;
+
+      /* Array loads need a crawl, other load do not */
+      case nir_intrinsic_image_load:
+         return nir_intrinsic_image_array(intr) ||
+                nir_intrinsic_image_dim(intr) == GLSL_SAMPLER_DIM_CUBE;
+
+      default:
+         return false;
+      }
+   } else if (instr->type == nir_instr_type_tex) {
+      nir_tex_instr *tex = nir_instr_as_tex(instr);
+
+      /* Array textures get clamped to their size via txs */
+      if (tex->is_array && !(tex->backend_flags & AGX_TEXTURE_FLAG_NO_CLAMP))
+         return true;
+
+      switch (tex->op) {
+      /* Queries always become a crawl */
+      case nir_texop_txs:
+      case nir_texop_texture_samples:
+      case nir_texop_query_levels:
+         return true;
+
+      /* Buffer textures need their format read and txf needs its LOD/layer
+       * clamped.  Buffer textures are only read through txf.
+       */
+      case nir_texop_txf:
+      case nir_texop_txf_ms:
+         return has_nonzero_lod(tex) || tex->is_array ||
+                tex->sampler_dim == GLSL_SAMPLER_DIM_BUF;
+
+      default:
+         return false;
+      }
+   }
+
+   return false;
+}

src/asahi/compiler/agx_nir_opt_preamble.c

@@ -0,0 +1,447 @@
+/*
+ * Copyright 2022 Alyssa Rosenzweig
+ * Copyright 2021 Valve Corporation
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "compiler/nir/nir_builder.h"
+#include "util/macros.h"
+#include "agx_compiler.h"
+#include "nir.h"
+#include "nir_builder_opcodes.h"
+#include "nir_intrinsics.h"
+#include "nir_intrinsics_indices.h"
+#include "nir_opcodes.h"
+
+static nir_preamble_class
+preamble_class(nir_def *def)
+{
+   nir_intrinsic_instr *intr = nir_def_as_intrinsic_or_null(def);
+   if (!intr || (nir_intrinsic_has_desc_set(intr) &&
+                 nir_intrinsic_desc_set(intr) >= 32 /* encoding restriction */))
+      return nir_preamble_class_general;
+
+   if (intr->intrinsic == nir_intrinsic_bindless_image_agx)
+      return nir_preamble_class_image;
+   else if (intr->intrinsic == nir_intrinsic_bindless_sampler_agx)
+      return nir_preamble_class_sampler;
+   else
+      return nir_preamble_class_general;
+}
+
+static void
+def_size(nir_def *def, unsigned *size, unsigned *align,
+         nir_preamble_class *class)
+{
+   unsigned bit_size = MAX2(def->bit_size, 16);
+
+   *size = (bit_size * def->num_components) / 16;
+   *align = bit_size / 16;
+   *class = preamble_class(def);
+}
+
+static bool
+all_uses_float(nir_def *def)
+{
+   nir_foreach_use_including_if(use, def) {
+      if (nir_src_is_if(use))
+         return false;
+
+      nir_instr *use_instr = nir_src_use_instr(use);
+      if (use_instr->type != nir_instr_type_alu)
+         return false;
+
+      nir_alu_instr *use_alu = nir_instr_as_alu(use_instr);
+      unsigned src_index = ~0;
+      for (unsigned i = 0; i < nir_op_infos[use_alu->op].num_inputs; i++) {
+         if (&use_alu->src[i].src == use) {
+            src_index = i;
+            break;
+         }
+      }
+
+      assert(src_index != ~0);
+      nir_alu_type src_type = nir_alu_type_get_base_type(
+         nir_op_infos[use_alu->op].input_types[src_index]);
+
+      if (src_type != nir_type_float)
+         return false;
+
+      /* No float modifiers on G13 */
+      if (use_alu->op == nir_op_fmax || use_alu->op == nir_op_fmin)
+         return false;
+   }
+
+   return true;
+}
+
+static float
+alu_cost(nir_alu_instr *alu)
+{
+   /* TODO: Model 64-bit better */
+   if (alu->def.bit_size == 64)
+      return 10.0f;
+
+   switch (alu->op) {
+   case nir_op_fsat:
+   case nir_op_f2fmp:
+   case nir_op_f2f16:
+   case nir_op_f2f16_rtne:
+   case nir_op_fadd:
+   case nir_op_fmul:
+   case nir_op_ffma:
+   case nir_op_iadd:
+   case nir_op_inot:
+   case nir_op_iand:
+   case nir_op_ior:
+   case nir_op_ixor:
+   case nir_op_feq:
+   case nir_op_flt:
+   case nir_op_fge:
+   case nir_op_fneu:
+   case nir_op_ieq:
+   case nir_op_ine:
+   case nir_op_ilt:
+   case nir_op_ige:
+   case nir_op_ult:
+   case nir_op_uge:
+   case nir_op_fmin:
+   case nir_op_fmax:
+   case nir_op_imin:
+   case nir_op_imax:
+   case nir_op_umin:
+   case nir_op_umax:
+   case nir_op_isub:
+   case nir_op_ineg:
+   case nir_op_bcsel:
+   case nir_op_b2b1:
+   case nir_op_b2b8:
+   case nir_op_b2b16:
+   case nir_op_b2b32:
+   case nir_op_b2i8:
+   case nir_op_b2i16:
+   case nir_op_b2i32:
+   case nir_op_b2f16:
+   case nir_op_b2f32:
+   case nir_op_i2i32:
+   case nir_op_i2i16:
+   case nir_op_u2u32:
+   case nir_op_u2u16:
+   case nir_op_u2u8:
+   case nir_op_i2i8:
+   case nir_op_iadd_sat:
+   case nir_op_isub_sat:
+   case nir_op_uadd_sat:
+   case nir_op_usub_sat:
+   case nir_op_iabs:
+      /* SCIB */
+      return 1.0;
+
+   case nir_op_ffloor:
+   case nir_op_fceil:
+   case nir_op_ftrunc:
+   case nir_op_fround_even:
+   case nir_op_bit_count:
+   case nir_op_bitfield_reverse:
+   case nir_op_ufind_msb:
+   case nir_op_imul:
+   case nir_op_imadshl_agx:
+   case nir_op_imsubshl_agx:
+   case nir_op_ishl:
+   case nir_op_ishr:
+   case nir_op_ushr:
+   case nir_op_flog2:
+   case nir_op_fexp2:
+   case nir_op_extr_agx:
+   case nir_op_ubitfield_extract:
+   case nir_op_f2i8:
+   case nir_op_f2i16:
+   case nir_op_f2i32:
+   case nir_op_f2u8:
+   case nir_op_f2u16:
+   case nir_op_f2u32:
+   case nir_op_i2fmp:
+   case nir_op_i2f16:
+   case nir_op_i2f32:
+   case nir_op_u2fmp:
+   case nir_op_u2f16:
+   case nir_op_u2f32:
+   case nir_op_interleave:
+      /* IC */
+      return 4.0;
+
+   case nir_op_frcp:
+      /* IC */
+      return 6.0;
+
+   case nir_op_frsq:
+      /* IC */
+      return 8.0;
+
+   case nir_op_fsqrt:
+      /* IC + F32 */
+      return 8.5;
+
+   case nir_op_imul_high:
+   case nir_op_umul_high:
+   case nir_op_imul_2x32_64:
+   case nir_op_umul_2x32_64:
+      /* IC */
+      return 8.0;
+
+   case nir_op_fsin_agx:
+      /* 2 IC + 1 F32 in parallel */
+      return 8.5;
+
+   case nir_op_fneg:
+   case nir_op_fabs:
+   case nir_op_f2f32:
+      /* Float source modifiers will be propagated */
+      return all_uses_float(&alu->def) ? 0.0 : 1.0;
+
+   case nir_op_mov:
+   case nir_op_vec2:
+   case nir_op_vec3:
+   case nir_op_vec4:
+   case nir_op_pack_32_2x16_split:
+   case nir_op_pack_64_2x32_split:
+   case nir_op_unpack_64_2x32_split_x:
+   case nir_op_unpack_64_2x32_split_y:
+   case nir_op_unpack_32_2x16_split_x:
+   case nir_op_unpack_32_2x16_split_y:
+   case nir_op_extract_i8:
+   case nir_op_extract_u8:
+   case nir_op_extract_i16:
+   case nir_op_extract_u16:
+      /* We optimistically assume that moves get coalesced */
+      return 0.0;
+
+   default:
+      /* Shrug */
+      return 2.0;
+   }
+}
+
+static float
+instr_cost(nir_instr *instr, const void *data)
+{
+   switch (instr->type) {
+   case nir_instr_type_intrinsic:
+      switch (nir_instr_as_intrinsic(instr)->intrinsic) {
+      case nir_intrinsic_load_global:
+      case nir_intrinsic_load_agx:
+      case nir_intrinsic_load_global_constant:
+      case nir_intrinsic_load_constant_agx:
+      case nir_intrinsic_load_ubo:
+         return 10.0;
+      case nir_intrinsic_ddx:
+      case nir_intrinsic_ddx_fine:
+      case nir_intrinsic_ddx_coarse:
+      case nir_intrinsic_ddy:
+      case nir_intrinsic_ddy_fine:
+      case nir_intrinsic_ddy_coarse:
+         return 1.0;
+      case nir_intrinsic_bindless_image_agx:
+      case nir_intrinsic_bindless_sampler_agx:
+         /* It's worth promoting even with a constant source, but it doesn't
+          * turn into instructions so should be less than any other normal
+          * instruction... But just enough to get over the image rewrite_cost.
+          */
+         return 2.5;
+      default:
+         /* Assume it's a sysval or something */
+         return 0.0;
+      }
+
+   case nir_instr_type_tex:
+      /* Texturing involes lots of memory bandwidth */
+      return 20.0;
+
+   case nir_instr_type_alu:
+      return alu_cost(nir_instr_as_alu(instr));
+
+   default:
+      return 1.0;
+   }
+}
+
+static float
+rewrite_cost(nir_def *def, const void *data)
+{
+   bool mov_needed = false, vectorizable = true;
+   nir_foreach_use(use, def) {
+      nir_instr *parent_instr = nir_src_use_instr(use);
+      if (parent_instr->type == nir_instr_type_tex) {
+         /* TODO: Maybe check the source index, but biases can be uniform */
+         break;
+      } else if (parent_instr->type == nir_instr_type_phi) {
+         /* Assume we'd eat a move anyway */
+      } else if (parent_instr->type != nir_instr_type_alu) {
+         mov_needed = true;
+         vectorizable = false;
+         break;
+      } else {
+         nir_alu_instr *alu = nir_instr_as_alu(parent_instr);
+         if (alu->op == nir_op_vec2 || alu->op == nir_op_vec3 ||
+             alu->op == nir_op_vec4) {
+            mov_needed = true;
+            break;
+         } else if (alu->op == nir_op_mov) {
+            mov_needed = true;
+            vectorizable = false;
+         } else {
+            /* Assume for non-moves that the const is folded into the src */
+         }
+      }
+   }
+
+   return mov_needed ? ((float)(def->num_components * def->bit_size) /
+                        (vectorizable ? 32.0 : 16.0))
+                     : 0;
+}
+
+static bool
+avoid_instr(const nir_instr *instr, const void *data)
+{
+   return false;
+}
+
+static const nir_opt_preamble_options preamble_options = {
+   .drawid_uniform = true,
+   .subgroup_size_uniform = true,
+   /* not supported in hardware */
+   .load_workgroup_size_allowed = false,
+   .def_size = def_size,
+   .instr_cost_cb = instr_cost,
+   .rewrite_cost_cb = rewrite_cost,
+   .avoid_instr_cb = avoid_instr,
+
+   /* hardware size is 512, but it's polite to leave some wiggle room to push
+    * hot constants so we don't end up rematerializing all over the place.
+    * 480 seems to be a sweetspot, based on a few minutes of shader-db.
+    */
+   .preamble_storage_size[nir_preamble_class_general] = 480,
+
+   /* We have at least 32 texture state registers. TODO: check for more? */
+   .preamble_storage_size[nir_preamble_class_image] = 32,
+
+   /* We have at least 16 sampler state registers. TODO: check for more? */
+   .preamble_storage_size[nir_preamble_class_sampler] = 16,
+};
+
+/*
+ * Bindless image handles can't be stored to uniforms, so we move them back to
+ * the main shader. Effectively un-optimizing the preamble.
+ */
+static bool
+lower_store_preamble(nir_builder *b, nir_intrinsic_instr *intr, void *data)
+{
+   int16_t *heaps = data;
+   if (intr->intrinsic != nir_intrinsic_store_preamble ||
+       nir_intrinsic_preamble_class(intr) == nir_preamble_class_image)
+      return false;
+
+   nir_intrinsic_instr *handle = nir_src_as_intrinsic(intr->src[0]);
+   if (!handle || handle->intrinsic != nir_intrinsic_bindless_image_agx)
+      return false;
+
+   heaps[nir_intrinsic_base(intr)] = nir_intrinsic_desc_set(handle);
+   nir_src_rewrite(&intr->src[0], handle->src[0].ssa);
+   return true;
+}
+
+static bool
+lower_preamble(nir_builder *b, nir_intrinsic_instr *intr, void *data)
+{
+   if (intr->intrinsic == nir_intrinsic_bindless_sampler_agx) {
+      /* Rematerialize bindless_sampler_agx before store_preamble with only the
+       * byte offset (first source), not the sampler index.
+       */
+      nir_foreach_use_safe(use, &intr->def) {
+         nir_instr *parent = nir_src_use_instr(use);
+         if (parent->type != nir_instr_type_intrinsic)
+            continue;
+         nir_intrinsic_instr *pintr = nir_instr_as_intrinsic(parent);
+         if (pintr->intrinsic != nir_intrinsic_store_preamble ||
+             nir_intrinsic_preamble_class(pintr) != nir_preamble_class_sampler)
+            continue;
+
+         b->cursor = nir_before_src(use);
+         nir_def *repl =
+            nir_bindless_sampler_agx(b, intr->src[0].ssa, nir_undef(b, 1, 16),
+                                     .desc_set = nir_intrinsic_desc_set(intr));
+         nir_src_rewrite(use, repl);
+      }
+
+      /* Replace other uses with just the sampler index. */
+      nir_def_replace(&intr->def, intr->src[1].ssa);
+      return true;
+   }
+
+   if (intr->intrinsic != nir_intrinsic_load_preamble)
+      return false;
+
+   int16_t *heaps = data;
+   b->cursor = nir_after_instr(&intr->instr);
+
+   unsigned base = nir_intrinsic_base(intr);
+   nir_def *new_ = NULL;
+   bool ts = nir_intrinsic_preamble_class(intr) == nir_preamble_class_image;
+   bool ss = nir_intrinsic_preamble_class(intr) == nir_preamble_class_sampler;
+   if (!ts && heaps[base] >= 0) {
+      new_ = nir_bindless_image_agx(b, &intr->def, .desc_set = heaps[base]);
+   }
+
+   nir_foreach_use_safe(use, &intr->def) {
+      nir_instr *parent = nir_src_use_instr(use);
+
+      if (parent->type == nir_instr_type_intrinsic) {
+         nir_intrinsic_instr *pintr = nir_instr_as_intrinsic(parent);
+
+         if (ts) {
+            nir_rewrite_image_intrinsic(pintr, nir_imm_intN_t(b, base / 2, 16),
+                                        nir_image_intrinsic_type_default);
+         } else if (new_ != NULL &&
+                    pintr->intrinsic != nir_intrinsic_bindless_image_agx) {
+            nir_src_rewrite(use, new_);
+         }
+      } else if (parent->type == nir_instr_type_tex) {
+         nir_tex_instr *tex = nir_instr_as_tex(parent);
+         nir_tex_src *src = (nir_tex_src *)use;
+
+         if (src->src_type == nir_tex_src_sampler_handle && ss) {
+            nir_steal_tex_src(tex, nir_tex_src_sampler_handle);
+            tex->sampler_index = base;
+         } else if (src->src_type == nir_tex_src_texture_handle && ts) {
+            nir_steal_tex_src(tex, nir_tex_src_texture_handle);
+            tex->texture_index = base / 2;
+         } else if (src->src_type == nir_tex_src_texture_handle) {
+            assert(new_ != NULL);
+            nir_src_rewrite(use, new_);
+         }
+      }
+   }
+
+   return true;
+}
+
+bool
+agx_nir_opt_preamble(nir_shader *nir, unsigned *sizes)
+{
+   bool progress = false;
+   NIR_PASS(progress, nir, nir_opt_preamble, &preamble_options, sizes);
+
+   int16_t heap[512];
+   memset(heap, ~0, sizeof(heap));
+
+   if (progress) {
+      nir_function_intrinsics_pass(nir_shader_get_preamble(nir),
+                                   lower_store_preamble,
+                                   nir_metadata_control_flow, heap);
+   }
+
+   NIR_PASS(progress, nir, nir_shader_intrinsics_pass, lower_preamble,
+            nir_metadata_control_flow, heap);
+
+   return progress;
+}

src/asahi/compiler/agx_nir_texture.h

@@ -0,0 +1,18 @@
+/*
+ * Copyright 2023 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#pragma once
+
+/* Texture backend flags */
+#define AGX_TEXTURE_FLAG_NO_CLAMP (1 << 0)
+
+/* Indicates that the sampler should be overriden to clamp to 0 instead of 1 */
+#define AGX_TEXTURE_FLAG_CLAMP_TO_0 (1 << 1)
+
+/* Texel buffers lowered to (at most) 16384x16384 2D textures */
+#define AGX_TEXTURE_BUFFER_WIDTH      16384
+#define AGX_TEXTURE_BUFFER_MAX_HEIGHT 16384
+#define AGX_TEXTURE_BUFFER_MAX_SIZE                                            \
+   (AGX_TEXTURE_BUFFER_WIDTH * AGX_TEXTURE_BUFFER_MAX_HEIGHT)

src/asahi/compiler/agx_opcodes.c.py

@@ -0,0 +1,39 @@
+template = """/*
+ * Copyright 2021 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "agx_opcodes.h"
+
+<%
+def make_encoding(encoding):
+   if encoding is None:
+      return "{ 0 }"
+
+   return "{{ {}, {}, {} }}".format(hex(encoding.exact), encoding.length_short, int(encoding.extensible))
+%>
+
+const struct agx_opcode_info agx_opcodes_info[AGX_NUM_OPCODES] = {
+% for opcode in opcodes:
+<%
+   op = opcodes[opcode]
+   imms = ["AGX_IMMEDIATE_" + imm.name.upper() for imm in op.imms]
+   if len(imms) == 0:
+      imms = ["0"]
+%>
+   [AGX_OPCODE_${opcode.upper()}] = {
+      "${opcode}", ${op.srcs}, ${op.dests}, ${" | ".join(imms)},
+      ${make_encoding(op.encoding)},
+      AGX_SCHEDULE_CLASS_${op.schedule_class.upper()},
+      ${int(op.is_float)},
+      ${int(op.can_eliminate)},
+      ${int(op.can_reorder)},
+   },
+% endfor
+};
+"""
+
+from mako.template import Template
+from agx_opcodes import opcodes
+
+print(Template(template).render(opcodes=opcodes))

src/asahi/compiler/agx_opcodes.h.py

@@ -0,0 +1,90 @@
+template = """/*
+ * Copyright 2021 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef _AGX_OPCODES_
+#define _AGX_OPCODES_
+
+#include <stdbool.h>
+#include <stdint.h>
+#include "util/macros.h"
+
+enum agx_schedule_class {
+   AGX_SCHEDULE_CLASS_INVALID,
+   AGX_SCHEDULE_CLASS_NONE,
+   AGX_SCHEDULE_CLASS_LOAD,
+   AGX_SCHEDULE_CLASS_STORE,
+   AGX_SCHEDULE_CLASS_ATOMIC,
+   AGX_SCHEDULE_CLASS_COVERAGE,
+   AGX_SCHEDULE_CLASS_PRELOAD,
+   AGX_SCHEDULE_CLASS_BARRIER,
+};
+
+/* Listing of opcodes */
+
+enum agx_opcode {
+% for op in opcodes:
+   AGX_OPCODE_${op.upper()},
+% endfor
+   AGX_NUM_OPCODES
+};
+
+% for name in enums:
+enum agx_${name} {
+% for k, v in enums[name].items():
+   AGX_${name.upper()}_${v.replace('.', '_').upper()} = ${k},
+% endfor
+};
+
+static inline const char *
+agx_${name}_as_str(enum agx_${name} x)
+{
+    switch (x) {
+% for k, v in enums[name].items():
+    case AGX_${name.upper()}_${v.replace('.', '_').upper()}: return "${v}";
+% endfor
+    default: UNREACHABLE("Nonexhaustive enum");
+    }
+}
+
+% endfor
+
+/* Runtime accessible info on each defined opcode */
+
+<% assert(len(immediates) < 64); %>
+
+enum agx_immediate {
+% for i, imm in enumerate(immediates):
+   AGX_IMMEDIATE_${imm.upper()} = (1ull << ${i}),
+% endfor
+};
+
+struct agx_encoding {
+   uint64_t exact;
+   unsigned length_short : 4;
+   bool extensible : 1;
+};
+
+struct agx_opcode_info {
+   const char *name;
+   unsigned nr_srcs;
+   unsigned nr_dests;
+   uint64_t immediates;
+   struct agx_encoding encoding;
+   enum agx_schedule_class schedule_class;
+   bool is_float : 1;
+   bool can_eliminate : 1;
+   bool can_reorder : 1;
+};
+
+extern const struct agx_opcode_info agx_opcodes_info[AGX_NUM_OPCODES];
+
+#endif
+"""
+
+from mako.template import Template
+from agx_opcodes import opcodes, immediates, enums
+
+print(Template(template).render(opcodes=opcodes, immediates=immediates,
+         enums=enums))

src/asahi/compiler/agx_opcodes.py

@@ -0,0 +1,486 @@
+"""
+Copyright 2021 Alyssa Rosenzweig
+# SPDX-License-Identifier: MIT
+"""
+
+opcodes = {}
+immediates = {}
+enums = {}
+
+VARIABLE = ~0
+
+class Opcode(object):
+   def __init__(self, name, dests, srcs, imms, is_float, can_eliminate,
+                can_reorder, schedule_class, encoding):
+      self.name = name
+      self.dests = dests if dests != VARIABLE else 0
+      self.srcs = srcs if srcs != VARIABLE else 0
+      self.variable_srcs = (srcs == VARIABLE)
+      self.variable_dests = (dests == VARIABLE)
+      self.imms = imms
+      self.is_float = is_float
+      self.can_eliminate = can_eliminate
+      self.can_reorder = can_reorder
+      self.schedule_class = schedule_class
+      self.encoding = encoding
+
+class Immediate(object):
+   def __init__(self, name, ctype):
+      self.name = name
+      self.ctype = ctype
+
+class Encoding(object):
+   def __init__(self, description):
+      (exact, length_short, length_long) = description
+
+      # Convenience
+      if length_long is None:
+         length_long = length_short
+
+      self.exact = exact
+      self.length_short = length_short
+      self.extensible = length_short != length_long
+
+      if self.extensible:
+         assert(length_long == length_short + (4 if length_short > 8 else 2))
+
+def op(name, encoding, dests = 1, srcs = 0, imms = [], is_float = False,
+        can_eliminate = True, can_reorder = True, schedule_class = "none"):
+   encoding = Encoding(encoding) if encoding is not None else None
+
+   opcodes[name] = Opcode(name, dests, srcs, imms, is_float, can_eliminate,
+                          can_reorder, schedule_class,  encoding)
+
+def immediate(name, ctype = "uint32_t"):
+   imm = Immediate(name, ctype)
+   immediates[name] = imm
+   return imm
+
+def enum(name, value_dict):
+   enums[name] = value_dict
+   return immediate(name, "enum agx_" + name)
+
+L = (1 << 15)
+_ = None
+
+FORMAT = immediate("format", "enum agx_format")
+IMM = immediate("imm", "uint64_t")
+WRITEOUT = immediate("writeout")
+INDEX = immediate("index")
+COMPONENT = immediate("component")
+CHANNELS = immediate("channels")
+TRUTH_TABLE = immediate("truth_table")
+ROUND = immediate("round", "enum agx_round")
+SHIFT = immediate("shift")
+MASK = immediate("mask")
+BFI_MASK = immediate("bfi_mask")
+LOD_MODE = immediate("lod_mode", "enum agx_lod_mode")
+PIXEL_OFFSET = immediate("pixel_offset")
+STACK_SIZE = immediate("stack_size", 'int16_t')
+EXPLICIT_COORDS = immediate("explicit_coords", "bool")
+
+DIM = enum("dim", {
+    0: '1d',
+    1: '1d_array',
+    2: '2d',
+    3: '2d_array',
+    4: '2d_ms',
+    5: '3d',
+    6: 'cube',
+    7: 'cube_array',
+    8: '2d_ms_array',
+})
+
+GATHER = enum("gather", {
+	0b000: "none",
+	0b001: "r",
+	0b011: "g",
+	0b101: "b",
+	0b111: "a",
+})
+
+OFFSET = immediate("offset", "bool")
+SHADOW = immediate("shadow", "bool")
+SPARSE = immediate("sparse", "bool")
+QUERY_LOD = immediate("query_lod", "bool")
+COHERENT = immediate("coherent", "bool")
+SCOREBOARD = immediate("scoreboard")
+ICOND = immediate("icond", "enum agx_icond")
+FCOND = immediate("fcond", "enum agx_fcond")
+NEST = immediate("nest")
+INVERT_COND = immediate("invert_cond")
+NEST = immediate("nest")
+TARGET = immediate("target", "agx_block *")
+ZS = immediate("zs")
+SR = enum("sr", {
+   0:  'threadgroup_position_in_grid.x',
+   1:  'threadgroup_position_in_grid.y',
+   2:  'threadgroup_position_in_grid.z',
+   4:  'threads_per_threadgroup.x',
+   5:  'threads_per_threadgroup.y',
+   6:  'threads_per_threadgroup.z',
+   8:  'dispatch_threads_per_threadgroup.x',
+   9:  'dispatch_threads_per_threadgroup.y',
+   10: 'dispatch_threads_per_threadgroup.z',
+   14: 'samples_log2',
+   20: 'core_id',
+   21: 'vm_slot',
+   48: 'thread_position_in_threadgroup.x',
+   49: 'thread_position_in_threadgroup.y',
+   50: 'thread_position_in_threadgroup.z',
+   51: 'thread_index_in_threadgroup',
+   52: 'thread_index_in_subgroup',
+   53: 'subgroup_index_in_threadgroup',
+   56: 'active_thread_index_in_quad',
+   57: 'total_active_threads_in_quad',
+   58: 'active_thread_index_in_subgroup',
+   59: 'total_active_threads_in_subgroup',
+   60: 'coverage_mask',
+   62: 'backfacing',
+   63: 'is_active_thread',
+   80: 'thread_position_in_grid.x',
+   81: 'thread_position_in_grid.y',
+   82: 'thread_position_in_grid.z',
+   124: 'input_sample_mask',
+   144: 'helper_op',
+   146: 'helper_arg_l',
+   147: 'helper_arg_h',
+})
+
+ATOMIC_OPC = enum("atomic_opc", {
+	0: 'add',
+	1: 'sub',
+	2: 'xchg',
+	3: 'cmpxchg',
+	4: 'umin',
+	5: 'imin',
+	6: 'umax',
+	7: 'imax',
+	8: 'and',
+	9: 'or',
+	10: 'xor',
+})
+
+INTERPOLATION = enum("interpolation", {
+    0: 'center',
+    1: 'sample',
+    2: 'centroid',
+    # We translate sample -> sample_register at pack time for simplicity
+    3: 'sample_register',
+})
+
+SIMD_OP = enum("simd_op", {
+    0b00000: 'and',
+    0b00001: 'fadd',
+    0b00010: 'or',
+    0b00011: 'fmul',
+    0b00100: 'xor',
+    0b00101: 'fmin',
+    0b00111: 'fmax',
+    0b10000: 'iadd',
+    0b10100: 'smin',
+    0b10110: 'smax',
+    0b11100: 'umin',
+    0b11110: 'umax',
+})
+
+FUNOP = lambda x: (x << 28)
+FUNOP_MASK = FUNOP((1 << 14) - 1)
+
+def funop(name, opcode, schedule_class = "none"):
+   op(name, (0x0A | (opcode << 28), 4, 6),
+      srcs = 1, is_float = True, schedule_class = schedule_class)
+
+def iunop(name, opcode):
+    assert(opcode < 4)
+    op(name, (0x3E | (opcode << 26), 6, _), srcs = 1)
+
+# Listing of opcodes
+funop("floor",     0b000000)
+funop("srsqrt",    0b000001)
+funop("dfdx",      0b000100, schedule_class = "coverage")
+funop("dfdy",      0b000110, schedule_class = "coverage")
+funop("rcp",       0b001000)
+funop("rsqrt",     0b001001)
+funop("sin_pt_1",  0b001010)
+funop("log2",      0b001100)
+funop("exp2",      0b001101)
+funop("sin_pt_2",  0b001110)
+funop("ceil",      0b010000)
+funop("trunc",     0b100000)
+funop("roundeven", 0b110000)
+
+iunop("bitrev",    0b01)
+iunop("popcount",  0b10)
+iunop("ffs",       0b11)
+
+op("fadd", encoding = (0x2A, 4, 6), srcs = 2, is_float = True)
+op("ffma", encoding = (0x3A, 6, 8), srcs = 3, is_float = True)
+op("fmul", encoding = (0x1A, 4, 6), srcs = 2, is_float = True)
+op("hadd", encoding = (0x26, 4, 6), srcs = 2, is_float = True)
+op("hfma", encoding = (0x36, 6, 8), srcs = 3, is_float = True)
+op("hmul", encoding = (0x16, 4, 6), srcs = 2, is_float = True)
+op("mov_imm", encoding = (0x62, 6, 8), imms = [IMM])
+op("iadd", encoding = (0x0E, 8, _), srcs = 2, imms = [SHIFT])
+op("imad", encoding = (0x1E, 8, _), srcs = 3, imms = [SHIFT])
+op("bfi", encoding = (0x2E, 8, _), srcs = 3, imms = [BFI_MASK])
+op("bfeil", encoding = (0x2E | L, 8, _), srcs = 3, imms = [BFI_MASK])
+op("extr", encoding = (0x2E | (0x1 << 26), 8, _), srcs = 3, imms = [BFI_MASK])
+op("asr", encoding = (0x2E | L | (0x1 << 26), 8, _), srcs = 2)
+op("icmpsel", encoding = (0x12, 8, 10), srcs = 4, imms = [ICOND])
+op("fcmpsel", encoding = (0x02, 8, 10), srcs = 4, imms = [FCOND])
+
+def subgroup_op(name, opc):
+    exact      = 0b01101111 | L | (opc << 29)
+    op(name, encoding = (exact, 6, _), srcs = 1, imms = [SIMD_OP])
+
+subgroup_op("quad_reduce", 0x0)
+subgroup_op("simd_reduce", 0x1)
+subgroup_op("quad_prefix", 0x2)
+subgroup_op("simd_prefix", 0x3)
+
+for window, w_bit in [('quad_', 0), ('', 1)]:
+    for s, shuffle in enumerate(['', '_xor', '_up', '_down']):
+        op(f"{window}shuffle{shuffle}",
+            encoding = (0b01101111 | (w_bit << 26) | (s << 38), 6, _),
+            srcs = 2)
+
+    # Pseudo-instruction ballotting a boolean
+    op(f"{window}ballot", _, srcs = 1)
+
+    for T, T_bit, cond in [('f', 0, FCOND), ('i', 1, ICOND)]:
+        op(f"{T}cmp_{window}ballot",
+           encoding = (0b0100010 | (T_bit << 4) | (w_bit << 48), 8, _),
+           srcs = 2, imms = [cond, INVERT_COND])
+
+# Pseudo-instructions for compares returning 1/0
+op("icmp", _, srcs = 2, imms = [ICOND, INVERT_COND])
+op("fcmp", _, srcs = 2, imms = [FCOND, INVERT_COND])
+
+# sources are coordinates, LOD, texture bindless base (zero for texture state
+# registers), texture, sampler, shadow/offset
+op("texture_sample",
+      encoding = (0x31, 8, 10), # XXX WRONG SIZE
+      srcs = 6, imms = [DIM, LOD_MODE, MASK, SCOREBOARD, OFFSET, SHADOW,
+                        QUERY_LOD, GATHER, SPARSE])
+for memory, can_reorder in [("texture", True), ("image", False)]:
+    coherency = [COHERENT] if not can_reorder else []
+    op(f"{memory}_load", encoding = (0x71, 8, 10), # XXX WRONG SIZE
+       srcs = 6, imms = [DIM, LOD_MODE, MASK, SCOREBOARD, OFFSET, SPARSE] + coherency,
+       can_reorder = can_reorder,
+       schedule_class = "none" if can_reorder else "load")
+
+# sources are base, index
+op("device_load",
+      encoding = (0x05, 6, 8),
+      srcs = 2, imms = [FORMAT, MASK, SHIFT, SCOREBOARD, COHERENT], can_reorder = False,
+      schedule_class = "load")
+
+# sources are base (relative to workgroup memory), index
+op("local_load",
+      encoding = (0b1101001, 6, 8),
+      srcs = 2, imms = [FORMAT, MASK], can_reorder = False,
+      schedule_class = "load")
+
+# sources are value, base, index
+# TODO: Consider permitting the short form
+op("device_store",
+      encoding = (0x45 | (1 << 47), 8, _),
+      dests = 0, srcs = 3, imms = [FORMAT, MASK, SHIFT, SCOREBOARD, COHERENT], can_eliminate = False,
+      schedule_class = "store")
+
+# sources are value, base, index
+op("local_store",
+      encoding = (0b0101001, 6, 8),
+      dests = 0, srcs = 3, imms = [FORMAT, MASK],
+      can_eliminate=False, schedule_class = "store")
+
+# sources are value, index
+# TODO: Consider permitting the short form
+op("uniform_store",
+      encoding = ((0b111 << 27) | 0b1000101 | (1 << 47), 8, _),
+      dests = 0, srcs = 2, imms = [MASK], can_eliminate = False)
+
+# sources are base, offset
+for kind, bit in [("tex", 1 << 6), ("sampler", 0)]:
+    op(f"{kind}_state_store", encoding = (0b10101101 | bit | (1 << 20), 8, _),
+          dests = 0, srcs = 2, imms = [IMM, SCOREBOARD], can_eliminate = False,
+          schedule_class = "store")
+
+# sources are value, base, index
+op("atomic",
+      encoding = (0x15 | (1 << 26) | (1 << 31) | (5 << 44), 8, _),
+      dests = 1, srcs = 3, imms = [ATOMIC_OPC, SCOREBOARD],
+      can_eliminate = False, schedule_class = "atomic")
+
+# XXX: stop hardcoding the long form
+op("local_atomic",
+      encoding = (0x19 | (1 << 15) | (1 << 36) | (1 << 47), 10, _),
+      dests = 1, srcs = 3, imms = [ATOMIC_OPC], schedule_class = "atomic",
+      can_eliminate = False)
+
+op("wait", (0x38, 2, _), dests = 0,
+      can_eliminate = False, imms = [SCOREBOARD], schedule_class = "invalid")
+
+for (suffix, schedule_class) in [("", "none"), ("_coverage", "coverage"), ("_barrier", "barrier")]:
+    op(f"get_sr{suffix}", (0x72, 4, _), dests = 1, imms = [SR],
+       schedule_class = schedule_class, can_reorder = schedule_class == "none")
+
+op("sample_mask", (0x7fc1, 6, _), dests = 0, srcs = 2,
+    can_eliminate = False, schedule_class = "coverage")
+
+# Sources: sample mask, combined depth/stencil
+op("zs_emit", (0x41, 4, _), dests = 0, srcs = 2,
+              can_eliminate = False, imms = [ZS], schedule_class = "coverage")
+
+# Sources: sample mask, explicit coords (if present)
+op("ld_tile", (0x49, 8, _), dests = 1, srcs = 2,
+        imms = [FORMAT, MASK, PIXEL_OFFSET, EXPLICIT_COORDS], can_reorder = False,
+        schedule_class = "coverage")
+
+# Sources: value, sample mask, explicit coords (if present)
+op("st_tile", (0x09, 8, _), dests = 0, srcs = 3,
+      can_eliminate = False, imms = [FORMAT, MASK, PIXEL_OFFSET, EXPLICIT_COORDS],
+      schedule_class = "coverage")
+
+for (name, exact) in [("any", 0xC000), ("none", 0xC020), ("none_after", 0xC020)]:
+   op("jmp_exec_" + name, (exact, 6, _), dests = 0, srcs = 0,
+         can_eliminate = False, schedule_class = "invalid", imms = [TARGET])
+
+# TODO: model implicit r0l destinations
+op("pop_exec", (0x52 | (0x3 << 9), 6, _),
+      dests = 0, srcs = 0, can_eliminate = False, schedule_class = "invalid",
+      imms = [NEST])
+
+for is_float in [False, True]:
+   mod_mask = 0 if is_float else (0x3 << 26) | (0x3 << 38)
+
+   for (cf, cf_op) in [("if", 0), ("else", 1), ("while", 2)]:
+      name = "{}_{}cmp".format(cf, "f" if is_float else "i")
+      exact = 0x42 | (0x0 if is_float else 0x10) | (cf_op << 9)
+      imms = [NEST, FCOND if is_float else ICOND, INVERT_COND, TARGET]
+
+      op(name, (exact, 6, _), dests = 0, srcs = 2, can_eliminate = False,
+            imms = imms, is_float = is_float,
+            schedule_class = "preload" if cf == "else" else "invalid")
+
+op("bitop", (0x7E, 6, _), srcs = 2, imms = [TRUTH_TABLE])
+op("intl", (0x3E, 6, _), srcs = 2, imms = [])
+op("convert", (0x3E | L, 6, _), srcs = 2, imms = [ROUND]) 
+
+# Sources are the coeffient register and the sample index (if applicable)
+op("iter", (0x21, 8, _), srcs = 2, imms = [CHANNELS, INTERPOLATION])
+
+# Sources are the coeffient register for the varying, the coefficient register
+# for W, and the sample index (if applicable)
+op("iterproj", (0x21, 8, _), srcs = 3, imms = [CHANNELS, INTERPOLATION])
+
+op("ldcf", (0xA1, 8, _), srcs = 1, imms = [CHANNELS])
+op("st_vary", None, dests = 0, srcs = 2, can_eliminate = False)
+op("no_varyings", (0x80000051, 4, _), dests = 0, can_eliminate = False)
+op("stop", (0x88, 2, _), dests = 0, can_eliminate = False,
+   schedule_class = "invalid")
+op("trap", (0x08, 2, _), dests = 0, can_eliminate = False,
+   schedule_class = "invalid")
+
+# These are modelled as total barriers since they can guard global memory
+# access too, and even need to be properly ordered with loads.
+op("wait_pix", (0x48, 4, _), dests = 0, imms = [WRITEOUT],
+   can_eliminate = False, schedule_class = "barrier")
+op("signal_pix", (0x58, 4, _), dests = 0, imms = [WRITEOUT],
+   can_eliminate = False, schedule_class = "barrier")
+
+# Sources are the data vector, the coordinate vector, the LOD, the bindless
+# table if present (zero for texture state registers), and texture index.
+op("image_write", (0xF1 | (1 << 23), 6, 8), dests = 0, srcs = 5, imms
+   = [DIM, COHERENT], can_eliminate = False, schedule_class = "store")
+
+# Sources are the image base, image index, the offset within shared memory, and
+# the coordinates (or just the layer if implicit).
+# TODO: Do we need the short encoding?
+op("block_image_store", (0xB1, 10, _), dests = 0, srcs = 4,
+   imms = [FORMAT, DIM, EXPLICIT_COORDS], can_eliminate = False, schedule_class = "store")
+
+# Barriers
+op("threadgroup_barrier", (0x0068, 2, _), dests = 0, srcs = 0,
+   can_eliminate = False, schedule_class = "barrier")
+
+def memory_barrier(name, a, b, c):
+    op(name, (0xF5 | (a << 10) | (b << 8) | (c << 12), 2, _), dests = 0, srcs = 0,
+       can_eliminate = False, schedule_class = "barrier")
+
+memory_barrier("memory_barrier", 1, 2, 9)
+
+# TODO: Not clear what these individually are. Some might be cache flushes?
+memory_barrier("image_barrier_1", 2, 2, 10)
+memory_barrier("image_barrier_2", 3, 2, 10)
+memory_barrier("image_barrier_3", 2, 1, 10)
+memory_barrier("image_barrier_4", 3, 1, 10)
+
+memory_barrier("flush_memory_to_texture", 0, 0, 4)
+
+memory_barrier("memory_barrier_2", 2, 2, 9)
+memory_barrier("memory_barrier_3", 2, 1, 9)
+memory_barrier("unknown_barrier_1", 0, 3, 3)
+memory_barrier("unknown_barrier_2", 0, 3, 0)
+
+# Seen with device-scope memory barriers. Again not clear what's what.
+memory_barrier("device_barrier_1", 3, 1, 9)
+memory_barrier("device_barrier_2", 3, 2, 9)
+
+op("doorbell", (0x60020 | 0x28 << 32, 6, _), dests = 0,
+      can_eliminate = False, can_reorder = False, imms = [IMM])
+
+op("stack_unmap", (0x00075, 8, _), dests = 1, srcs = 0, can_eliminate = False, can_reorder = False, imms = [IMM])
+op("stack_map",   (0x10075, 8, _), dests = 0, srcs = 1, can_eliminate = False, can_reorder = False, imms = [IMM])
+
+op("stack_adjust", encoding = (0x10100b5, 8, _),
+      dests = 0, srcs = 0, can_eliminate = False, can_reorder = False,
+      imms = [STACK_SIZE], schedule_class = "store")
+
+# source is offset
+op("stack_load", encoding = (0x35, 6, 8),
+      srcs = 1, imms = [FORMAT, MASK, SCOREBOARD], can_reorder = False,
+      schedule_class = "load")
+
+# sources are value and offset
+op("stack_store", encoding = (0xb5, 6, 8),
+      dests = 0, srcs = 2, imms = [FORMAT, MASK, SCOREBOARD],
+      can_eliminate=False, schedule_class = "store")
+
+# Convenient aliases.
+op("mov", _, srcs = 1)
+op("not", _, srcs = 1)
+op("signext", _, srcs = 1)
+
+op("collect", _, srcs = VARIABLE)
+op("split", _, srcs = 1, dests = VARIABLE)
+op("phi", _, srcs = VARIABLE, schedule_class = "preload")
+
+# The srcs double as destinations. Only deals in registers. This is generated by
+# parallel copy lowering and lowered soon after. We need this as a dedicated
+# instruction only for RA validation.
+op("swap", _, dests = 0, srcs = 2)
+
+op("unit_test", _, dests = 0, srcs = 1, can_eliminate = False)
+
+# Like mov, but takes a register and can only appear at the start. Guaranteed
+# to be coalesced during RA, rather than lowered to a real move. 
+op("preload", _, srcs = 1, schedule_class = "preload")
+
+# Opposite of preload. Exports a scalar value to a particular register at the
+# end of the shader part. Must only appear after the logical end of the exit
+# block, this avoids special casing the source's liveness. Logically all exports
+# happen in parallel at the end of the shader part.
+op("export", _, dests = 0, srcs = 1, imms = [IMM], can_eliminate = False,
+   schedule_class = "invalid")
+
+# Pseudo-instructions to set the nesting counter. Lowers to r0l writes after RA.
+op("begin_cf", _, dests = 0, can_eliminate = False)
+op("break", _, dests = 0, imms = [NEST, TARGET], can_eliminate = False,
+   schedule_class = "invalid")
+
+for (name, is_float) in [("break_if_icmp", False), ("break_if_fcmp", True)]:
+    op(name, _, dests = 0, srcs = 2,
+       imms = [NEST, INVERT_COND, FCOND if is_float else ICOND, TARGET],
+       can_eliminate = False, schedule_class = "invalid")

src/asahi/compiler/agx_opt_break_if.c

@@ -0,0 +1,81 @@
+/*
+ * Copyright 2023 Valve Corporation
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "util/list.h"
+#include "agx_builder.h"
+#include "agx_compiler.h"
+
+/*
+ * Replaces conditional breaks:
+ *
+ *    if_cmp x, y, n=1 {
+ *       break #value
+ *    }
+ *    pop_exec n=1
+ *
+ * with break_if_*cmp pseudo-instructions for further optimization. This
+ * assumes agx_opt_empty_else has already run.
+ */
+
+static void
+match_block(agx_context *ctx, agx_block *block)
+{
+   agx_instr *if_ = agx_last_instr(block);
+   if (!if_ ||
+       (if_->op != AGX_OPCODE_IF_ICMP && if_->op != AGX_OPCODE_IF_FCMP) ||
+       if_->nest != 1)
+      return;
+
+   /* If's fallthrough to the then */
+   agx_block *then_block = agx_next_block(block);
+   assert(block->successors[0] == then_block && "successors for if");
+
+   /* We're searching for a single block then, so the next block is else */
+   agx_block *else_block = agx_next_block(then_block);
+   if (block->successors[1] != else_block)
+      return;
+
+   if (!list_is_singular(&then_block->instructions) ||
+       !list_is_singular(&else_block->instructions))
+      return;
+
+   agx_instr *break_ = agx_last_instr(then_block);
+   agx_instr *pop = agx_last_instr(else_block);
+
+   if (break_->op != AGX_OPCODE_BREAK || pop->op != AGX_OPCODE_POP_EXEC ||
+       pop->nest != 1)
+      return;
+
+   /* Find the successor of the if */
+   agx_block *after = else_block->successors[0];
+   assert(else_block->successors[1] == NULL && "single successor");
+
+   /* We are dropping one level of nesting (the if) when rewriting */
+   assert(break_->nest >= 1 && "must at least break out of the if");
+   unsigned new_nest = break_->nest - 1;
+
+   /* Rewrite */
+   agx_builder b = agx_init_builder(ctx, agx_before_block(after));
+
+   if (if_->op == AGX_OPCODE_IF_FCMP) {
+      agx_break_if_fcmp(&b, if_->src[0], if_->src[1], new_nest,
+                        if_->invert_cond, if_->fcond, break_->target);
+   } else {
+      agx_break_if_icmp(&b, if_->src[0], if_->src[1], new_nest,
+                        if_->invert_cond, if_->icond, break_->target);
+   }
+
+   agx_remove_instruction(if_);
+   agx_remove_instruction(break_);
+   agx_remove_instruction(pop);
+}
+
+void
+agx_opt_break_if(agx_context *ctx)
+{
+   agx_foreach_block(ctx, block) {
+      match_block(ctx, block);
+   }
+}

src/asahi/compiler/agx_opt_compact_constants.c

@@ -0,0 +1,62 @@
+/*
+ * Copyright 2024 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "util/bitset.h"
+#include "util/half_float.h"
+#include "agx_builder.h"
+#include "agx_compiler.h"
+#include "agx_opcodes.h"
+
+/*
+ * AGX can convert 16-bit sources to 32-bit for free, so it's beneficial to
+ * compact 32-bit constants down to 16-bit when doing so is lossless. This
+ * reduces register pressure (GPR or uniform, depending on whether the constant
+ * is promoted).
+ */
+void
+agx_opt_compact_constants(agx_context *ctx)
+{
+   /* TODO: Handle ints too */
+   BITSET_WORD *src_float = calloc(ctx->alloc, sizeof(BITSET_WORD));
+   BITSET_WORD *src_other = calloc(ctx->alloc, sizeof(BITSET_WORD));
+   BITSET_WORD *replaced = calloc(ctx->alloc, sizeof(BITSET_WORD));
+
+   /* Analyze the types that we read constants as */
+   agx_foreach_instr_global(ctx, I) {
+      agx_foreach_ssa_src(I, s) {
+         if (agx_is_float_src(I, s))
+            BITSET_SET(src_float, I->src[s].value);
+         else
+            BITSET_SET(src_other, I->src[s].value);
+      }
+   }
+
+   agx_foreach_instr_global(ctx, I) {
+      if (I->op == AGX_OPCODE_MOV_IMM && I->dest[0].size == AGX_SIZE_32) {
+         unsigned v = I->dest[0].value;
+
+         if (BITSET_TEST(src_float, v) && !BITSET_TEST(src_other, v)) {
+            /* Try to compact to f16 */
+            uint16_t compact = _mesa_float_to_half(uif(I->imm));
+
+            if (I->imm == fui(_mesa_half_to_float(compact))) {
+               I->dest[0].size = AGX_SIZE_16;
+               I->imm = compact;
+               BITSET_SET(replaced, v);
+            }
+         }
+      } else {
+         agx_foreach_ssa_src(I, s) {
+            if (BITSET_TEST(replaced, I->src[s].value)) {
+               I->src[s].size = AGX_SIZE_16;
+            }
+         }
+      }
+   }
+
+   free(replaced);
+   free(src_float);
+   free(src_other);
+}

src/asahi/compiler/agx_opt_cse.c

@@ -0,0 +1,139 @@
+/*
+ * Copyright 2022 Alyssa Rosenzweig
+ * Copyright 2021 Collabora, Ltd.
+ * Copyright 2014 Valve Corporation
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "util/compiler.h"
+#include "agx_builder.h"
+
+#define XXH_INLINE_ALL
+#include "util/xxhash.h"
+
+/*
+ * This pass handles CSE'ing repeated expressions created in the process of
+ * translating from NIR. Also, currently this is intra-block only, to make it
+ * work over multiple block we'd need to bring forward dominance calculation.
+ */
+
+static inline uint32_t
+HASH(uint32_t hash, unsigned data)
+{
+   return XXH32(&data, sizeof(data), hash);
+}
+
+/* Hash an instruction. XXH32 isn't too speedy, so this is hotspot. */
+static uint32_t
+hash_instr(const void *data)
+{
+   const agx_instr *I = data;
+   uint32_t hash = 0;
+
+   /* Explicitly skip destinations, except for size and type */
+   agx_foreach_dest(I, d) {
+      hash = HASH(hash, ((uint32_t)I->dest[d].type) |
+                           (((uint32_t)I->dest[d].size) << 16));
+   }
+
+   /* Hash the source array as-is */
+   hash = XXH32(I->src, sizeof(agx_index) * I->nr_srcs, hash);
+
+   /* Hash everything else in the instruction starting from the opcode */
+   hash = XXH32(&I->op, sizeof(agx_instr) - offsetof(agx_instr, op), hash);
+
+   return hash;
+}
+
+static bool
+instrs_equal(const void *_i1, const void *_i2)
+{
+   const agx_instr *i1 = _i1, *i2 = _i2;
+
+   if (i1->op != i2->op)
+      return false;
+   if (i1->nr_srcs != i2->nr_srcs)
+      return false;
+   if (i1->nr_dests != i2->nr_dests)
+      return false;
+
+   /* Explicitly skip everything but size and type */
+   agx_foreach_dest(i1, d) {
+      if (i1->dest[d].type != i2->dest[d].type)
+         return false;
+      if (i1->dest[d].size != i2->dest[d].size)
+         return false;
+   }
+
+   agx_foreach_src(i1, s) {
+      agx_index s1 = i1->src[s], s2 = i2->src[s];
+
+      if (memcmp(&s1, &s2, sizeof(s1)) != 0)
+         return false;
+   }
+
+   if (i1->imm != i2->imm)
+      return false;
+   if (i1->invert_cond != i2->invert_cond)
+      return false;
+   if (i1->dim != i2->dim)
+      return false;
+   if (i1->offset != i2->offset)
+      return false;
+   if (i1->shadow != i2->shadow)
+      return false;
+   if (i1->shift != i2->shift)
+      return false;
+   if (i1->saturate != i2->saturate)
+      return false;
+   if (i1->mask != i2->mask)
+      return false;
+
+   return true;
+}
+
+/* Determines what instructions the above routines have to handle */
+static bool
+instr_can_cse(const agx_instr *I)
+{
+   return agx_opcodes_info[I->op].can_eliminate &&
+          agx_opcodes_info[I->op].can_reorder;
+}
+
+void
+agx_opt_cse(agx_context *ctx)
+{
+   struct set *instr_set = _mesa_set_create(NULL, hash_instr, instrs_equal);
+   agx_index *replacement = malloc(sizeof(agx_index) * ctx->alloc);
+
+   agx_foreach_block(ctx, block) {
+      memset(replacement, 0, sizeof(agx_index) * ctx->alloc);
+      _mesa_set_clear(instr_set, NULL);
+
+      agx_foreach_instr_in_block(block, instr) {
+         /* Rewrite as we go so we converge locally in 1 iteration */
+         agx_foreach_ssa_src(instr, s) {
+            agx_index repl = replacement[instr->src[s].value];
+            if (!agx_is_null(repl))
+               agx_replace_src(instr, s, repl);
+         }
+
+         if (!instr_can_cse(instr))
+            continue;
+
+         bool found;
+         struct set_entry *entry =
+            _mesa_set_search_or_add(instr_set, instr, &found);
+         if (found) {
+            const agx_instr *match = entry->key;
+
+            agx_foreach_dest(instr, d) {
+               replacement[instr->dest[d].value] = match->dest[d];
+            }
+         }
+      }
+   }
+
+   free(replacement);
+   _mesa_set_destroy(instr_set, NULL);
+}

src/asahi/compiler/agx_opt_empty_else.c

@@ -0,0 +1,101 @@
+/*
+ * Copyright 2023 Valve Corporation
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "util/list.h"
+#include "agx_builder.h"
+#include "agx_compiler.h"
+#include "agx_opcodes.h"
+
+/*
+ * Detect blocks with the sole contents:
+ *
+ *    else n=1
+ *    pop_exec n=1
+ *
+ * The else instruction is a no-op. To see that, consider the pseudocode for the
+ * sequence of operations "else n=1; pop_exec n=1":
+ *
+ *   # else n=1
+ *   if r0l == 0:
+ *     r0l = 1
+ *   elif r0l == 1:
+ *     if [...]:
+ *       r0l = 0
+ *     else:
+ *       r0l = 1
+ *   exec_mask[thread] = (r0l == 0)
+ *
+ *   # pop_exec n=1
+ *   if r0l > 0:
+ *     r0l -= 1
+ *   exec_mask[thread] = (r0l == 0)
+ *
+ * That logic code simplifies to:
+ *
+ *   if r0l > 0:
+ *     r0l = r0l - 1
+ *   exec_mask[thread] = (r0l == 0)
+ *
+ * which is just "pop_exec n=1".
+ *
+ * Therefore, this pass detects these blocks and deletes the else instruction.
+ * This has the effect of removing empty else blocks. Logically, that creates
+ * critical edges, so this pass can only run late (post-RA).
+ *
+ * The pass itself uses a simple state machine for pattern matching.
+ */
+
+enum block_state {
+   STATE_ELSE = 0,
+   STATE_POP_EXEC,
+   STATE_DONE,
+
+   /* Must be last */
+   STATE_NONE,
+};
+
+static enum block_state
+state_for_instr(const agx_instr *I)
+{
+   switch (I->op) {
+   case AGX_OPCODE_ELSE_ICMP:
+   case AGX_OPCODE_ELSE_FCMP:
+      return (I->nest == 1) ? STATE_ELSE : STATE_NONE;
+
+   case AGX_OPCODE_POP_EXEC:
+      return (I->nest == 1) ? STATE_POP_EXEC : STATE_NONE;
+
+   default:
+      return STATE_NONE;
+   }
+}
+
+static bool
+match_block(agx_block *blk)
+{
+   enum block_state state = STATE_ELSE;
+
+   agx_foreach_instr_in_block(blk, I) {
+      if (state_for_instr(I) == state)
+         state++;
+      else
+         return false;
+   }
+
+   return (state == STATE_DONE);
+}
+
+void
+agx_opt_empty_else(agx_context *ctx)
+{
+   agx_foreach_block(ctx, blk) {
+      if (match_block(blk)) {
+         agx_instr *else_instr = agx_first_instr(blk);
+         assert(state_for_instr(else_instr) == STATE_ELSE && "block matched");
+
+         agx_remove_instruction(else_instr);
+      }
+   }
+}

src/asahi/compiler/agx_opt_jmp_none.c

@@ -0,0 +1,184 @@
+/*
+ * Copyright 2023 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "agx_builder.h"
+#include "agx_compiler.h"
+#include "agx_opcodes.h"
+
+/*
+ * AGX control flow instructions predicate out threads. No forward branches are
+ * inserted during instruction selection, only backwards branches at the end of
+ * loops exist before this pass. This means, prior to this pass, we would always
+ * execute both sides of an if.
+ *
+ * To improve performance, this pass inserts conservative forward branches after
+ * if_*cmp, else_*cmp, and break_if_*cmp instructions, jumping the
+ * subgroup to their logical destination if all threads in the subgroup are
+ * inactive. This has the effect of skipping over the unexecuted half of an if.
+ * That means this pass is critical for control flow performance.
+ */
+
+/* Estimated cost of inserting a jmp_exec_none. This value is tuned to Dolphin
+ * ubershaders. It needs to be retuned in lockstep with changes to the cost
+ * estimation heuristic.
+ */
+#define COST_JMP (19)
+
+static uint32_t
+cost_instr(agx_instr *I)
+{
+   /* TODO: Better heuristic */
+   switch (I->op) {
+   case AGX_OPCODE_DEVICE_LOAD:
+      return 10;
+   case AGX_OPCODE_TEXTURE_LOAD:
+   case AGX_OPCODE_TEXTURE_SAMPLE:
+      return 50;
+   default:
+      return 1;
+   }
+}
+
+/*
+ * Estimate the cost between the instruction and the branch target. This is an
+ * input for our heuristic. The branch target is guaranteed to be a forward
+ * branch.
+ */
+static uint32_t
+cost_between(agx_context *ctx, agx_block *from, agx_instr *from_I,
+             agx_block *target, bool skip_to_end_of_target)
+{
+   uint32_t cost = 0;
+
+   /* Consider the cost in the rest of this block */
+   if (from_I != agx_last_instr(from)) {
+      agx_foreach_instr_in_block_from(from, J, from_I) {
+         /* If we reach the end, we're done */
+         if (from == target && skip_to_end_of_target &&
+             J == agx_last_instr(target))
+            break;
+
+         cost += cost_instr(J);
+      }
+   }
+
+   if (from == target)
+      return cost;
+
+   /* Consider the cost in the subsequent blocks */
+   agx_foreach_block_from(ctx, from, block) {
+      if (block == from)
+         continue;
+
+      if (block == target && !skip_to_end_of_target)
+         break;
+
+      agx_foreach_instr_in_block(block, I) {
+         if (block == target && I == agx_last_instr(target))
+            break;
+
+         cost += cost_instr(I);
+      }
+
+      if (block == target) {
+         assert(skip_to_end_of_target);
+         break;
+      }
+   }
+
+   return cost;
+}
+
+static void
+try_insert_jmp(agx_context *ctx, agx_block *from, agx_instr *from_I,
+               agx_block *target, bool skip_to_end_of_target,
+               unsigned inverse_probability)
+{
+   agx_builder b = agx_init_builder(ctx, agx_after_instr(from_I));
+
+   /* If the control flow instruction was only inserted for its side effects,
+    * there is nowhere to jump. Bail.
+    */
+   if (!target)
+      return;
+
+   /* If we do not insert a jump, we execute the predicated instructions
+    * unconditionally, with an expected cost C.
+    *
+    * If we do insert a jump, then we pay the cost J of the jump, AND if we do
+    * not take the jump, also the cost of the instructions C. The expected cost
+    * if we insert a jump is therefore J + P(not all threads inactive) C.
+    *
+    * Therefore, we should insert a jump if:
+    *
+    *    J + P(not all threads inactive) C < C
+    *
+    * To model the implicit (i-cache, etc) costs of inserting a jump
+    * instruction, we tie break conservatively, comparing with < instead of <=.
+    *
+    * Rearranging terms, we should NOT insert a jump if:
+    *
+    *    C < J / P(all threads inactive).
+    */
+   uint32_t cost_instructions =
+      cost_between(ctx, from, from_I, target, skip_to_end_of_target);
+
+   if (cost_instructions < COST_JMP * inverse_probability)
+      return;
+
+   /* It looks like inserting a jump will be a win. Do so. */
+   if (skip_to_end_of_target)
+      agx_jmp_exec_none_after(&b, target);
+   else
+      agx_jmp_exec_none(&b, target);
+}
+
+void
+agx_opt_jmp_none(agx_context *ctx)
+{
+   agx_foreach_block(ctx, blk) {
+      /* Handle the beginning of blocks */
+      agx_instr *first_ = agx_first_instr(blk);
+      if (first_ && (first_->op == AGX_OPCODE_ELSE_ICMP ||
+                     first_->op == AGX_OPCODE_ELSE_FCMP)) {
+
+         /* The target of the else is the last block of the else, so we skip
+          * to the end of the block (to start execution with the pop_exec).
+          */
+         try_insert_jmp(ctx, blk, first_, first_->target, true, 2);
+      } else if (first_ &&
+                 (first_->op == AGX_OPCODE_BREAK_IF_ICMP ||
+                  first_->op == AGX_OPCODE_BREAK_IF_FCMP) &&
+                 first_->nest == 1) {
+         /* The target of the break is the block immediately after the end of
+          * the loop, so jump to the end of the previous block to get the
+          * appropriate pop_exec.
+          *
+          * Also, note we only do this for nest=1 to ensure we don't insert
+          * jumps inside if-statements inside breaks. We can't insert a
+          * jmp_exec_none inside the if because it would break out of the loop
+          * for threads that are still running the loop but merely predicated
+          * out due to the if-condition. This is similarly why we don't bother
+          * handling unconditional break.
+          *
+          * TODO: This is not optimal, but fixing this would require
+          * considerably more CFG gymnastics.
+          */
+         agx_block *target = agx_prev_block(first_->target);
+         try_insert_jmp(ctx, blk, first_, target, true, 10);
+      }
+
+      /* Handle end of block instructions */
+      agx_foreach_instr_in_block_rev(blk, I) {
+         if (!instr_after_logical_end(I))
+            break;
+
+         if (I->op == AGX_OPCODE_IF_ICMP || I->op == AGX_OPCODE_IF_FCMP) {
+            try_insert_jmp(ctx, blk, I, I->target, false, 2);
+            break;
+         }
+      }
+   }
+}

src/asahi/compiler/agx_opt_promote_constants.c

@@ -0,0 +1,189 @@
+/*
+ * Copyright 2023 Valve Corporation
+ * SPDX-License-Identifier: MIT
+ */
+
+#include <stdlib.h>
+#include "util/bitset.h"
+#include "util/hash_table.h"
+#include "util/ralloc.h"
+#include "agx_compiler.h"
+#include "agx_opcodes.h"
+
+/*
+ * Information about a constant, indexed by its 64-bit value. This describes the
+ * value, not the move that generated it. If there are multiple moves in the
+ * shader with the same immediate value, they resolve to the same constant.
+ */
+struct constant_info {
+   uint64_t value;
+
+   /* Number of uses of the constant that could be promoted */
+   unsigned nr_promotable_uses;
+
+   /* If we push, the uniform used */
+   uint16_t uniform;
+
+   /* Alignment in 16-bit units needed for the constant */
+   uint8_t align_16;
+
+   /* True if the constant was promoted to a uniform */
+   bool promoted;
+};
+
+/*
+ * Choosing constants to promote is similar to the 0-1 knapsack problem. We use
+ * a well-known heuristic: sort by benefit divided by size. We approximate
+ * benefit by use count.
+ */
+static int
+constant_priority(const struct constant_info *const info)
+{
+   int size = info->align_16;
+   assert(size == 1 || size == 2 || size == 4);
+   int inverse_size = (size == 1) ? 4 : (size == 2) ? 2 : 1;
+
+   return info->nr_promotable_uses * inverse_size;
+}
+
+static int
+priority_compare(const void *A_, const void *B_)
+{
+   const struct constant_info *const *A = A_;
+   const struct constant_info *const *B = B_;
+
+   /* This is backwards from qsort's documentation, because we want descending
+    * order and qsort returns ascending.
+    */
+   return constant_priority(*B) - constant_priority(*A);
+}
+
+static void
+record_use(void *memctx, struct hash_table_u64 *constants, uint64_t imm,
+           enum agx_size size)
+{
+   struct constant_info *info = _mesa_hash_table_u64_search(constants, imm);
+
+   if (!info) {
+      info = rzalloc(memctx, struct constant_info);
+      info->value = imm;
+      _mesa_hash_table_u64_insert(constants, imm, info);
+   }
+
+   info->nr_promotable_uses++;
+   info->align_16 = MAX2(info->align_16, agx_size_align_16(size));
+}
+
+static void
+pass(agx_context *ctx, void *memctx)
+{
+   /* Map from SSA indices to struct constant_info */
+   struct hash_table_u64 *constants = _mesa_hash_table_u64_create(memctx);
+
+   /* Map from SSA indices to immediate values */
+   uint64_t *values = rzalloc_array(memctx, uint64_t, ctx->alloc);
+
+   /* Set of SSA indices that map to immediate values */
+   BITSET_WORD *is_immediate = BITSET_RZALLOC(memctx, ctx->alloc);
+
+   /* Gather constant definitions and use */
+   agx_foreach_instr_global(ctx, I) {
+      if (I->op == AGX_OPCODE_MOV_IMM) {
+         assert(I->dest[0].type == AGX_INDEX_NORMAL);
+         BITSET_SET(is_immediate, I->dest[0].value);
+         values[I->dest[0].value] = I->imm;
+      } else {
+         agx_foreach_ssa_src(I, s) {
+            if (BITSET_TEST(is_immediate, I->src[s].value) &&
+                agx_instr_accepts_uniform(I->op, s, ctx->out->push_count,
+                                          I->src[s].size)) {
+
+               record_use(memctx, constants, values[I->src[s].value],
+                          I->src[s].size);
+            }
+         }
+      }
+   }
+
+   /* Early exit if there were no constants */
+   unsigned nr_nodes = _mesa_hash_table_u64_num_entries(constants);
+   if (nr_nodes == 0)
+      return;
+
+   /* Collect nodes that are promotable */
+   struct constant_info **flat =
+      rzalloc_array(memctx, struct constant_info *, nr_nodes);
+
+   unsigned flat_count = 0;
+   hash_table_u64_foreach(constants, entry) {
+      flat[flat_count++] = entry.data;
+   }
+
+   /* Select constants. Even when we can promote everything, sorting keeps hot
+    * constants in lower uniforms, required by some instructions.
+    */
+   qsort(flat, flat_count, sizeof(*flat), priority_compare);
+
+   /* Promote as many constants as we can */
+   for (unsigned i = 0; i < flat_count; ++i) {
+      struct constant_info *info = flat[i];
+      assert(info->nr_promotable_uses > 0);
+
+      /* Try to assign a uniform */
+      unsigned uniform = ALIGN_POT(ctx->out->push_count, info->align_16);
+      unsigned new_count = uniform + info->align_16;
+      if (new_count > AGX_NUM_UNIFORMS)
+         break;
+
+      /* Set the base for the first constant. This avoids uploading an extra
+       * zero if we needed to pad, compared to doing this before the loop. Same
+       * shader code, but slightly more efficient USC record.
+       */
+      if (i == 0) {
+         ctx->out->rodata.base_uniform = uniform;
+      }
+
+      info->uniform = uniform;
+      info->promoted = true;
+      ctx->out->push_count = new_count;
+
+      unsigned size_B = info->align_16 * 2;
+      unsigned uniform_offs = uniform - ctx->out->rodata.base_uniform;
+      assert(uniform_offs < ARRAY_SIZE(ctx->rodata));
+      memcpy(&ctx->rodata[uniform_offs], &info->value, size_B);
+
+      ctx->out->rodata.size_16 = new_count - ctx->out->rodata.base_uniform;
+   }
+
+   /* Promote in the IR */
+   agx_foreach_instr_global(ctx, I) {
+      agx_foreach_ssa_src(I, s) {
+         if (!BITSET_TEST(is_immediate, I->src[s].value))
+            continue;
+
+         struct constant_info *info =
+            _mesa_hash_table_u64_search(constants, values[I->src[s].value]);
+
+         if (info && info->promoted &&
+             agx_instr_accepts_uniform(I->op, s, info->uniform,
+                                       I->src[s].size)) {
+
+            agx_replace_src(I, s, agx_uniform(info->uniform, I->src[s].size));
+         }
+      }
+   }
+}
+
+void
+agx_opt_promote_constants(agx_context *ctx)
+{
+   /* We do not promote constants in preambles since it's pointless and wastes
+    * uniform slots.
+    */
+   if (ctx->is_preamble)
+      return;
+
+   void *memctx = ralloc_context(NULL);
+   pass(ctx, memctx);
+   ralloc_free(memctx);
+}

src/asahi/compiler/agx_opt_register_cache.c

@@ -0,0 +1,154 @@
+/*
+ * Copyright 2025 Valve Corporation
+ * Copyright 2019-2022 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "util/bitset.h"
+#include "agx_compiler.h"
+#include "shader_enums.h"
+
+static void
+postra_liveness_ins(BITSET_WORD *live, agx_instr *I)
+{
+   agx_foreach_reg_dest(I, d) {
+      unsigned reg = I->dest[d].value;
+      BITSET_CLEAR_COUNT(live, reg, agx_index_size_16(I->dest[d]));
+   }
+
+   agx_foreach_reg_src(I, s) {
+      unsigned reg = I->src[s].value;
+      BITSET_SET_COUNT(live, reg, agx_index_size_16(I->src[s]));
+   }
+}
+
+/*
+ * Globally, liveness analysis uses a fixed-point algorithm based on a
+ * worklist. We initialize a work list with the exit block. We iterate the work
+ * list to compute live_in from live_out for each block on the work list,
+ * adding the predecessors of the block to the work list if we made progress.
+ */
+static void
+postra_liveness(agx_context *ctx)
+{
+   u_worklist worklist;
+   agx_worklist_init(ctx, &worklist);
+
+   agx_foreach_block(ctx, block) {
+      BITSET_ZERO(block->reg_live_in);
+      BITSET_ZERO(block->reg_live_out);
+
+      agx_worklist_push_tail(&worklist, block);
+   }
+
+   while (!u_worklist_is_empty(&worklist)) {
+      /* Pop off in reverse order since liveness is backwards */
+      agx_block *blk = agx_worklist_pop_tail(&worklist);
+
+      /* Calculate liveness locally */
+      agx_foreach_successor(blk, succ) {
+         BITSET_OR(blk->reg_live_out, blk->reg_live_out, succ->reg_live_in);
+      }
+
+      BITSET_DECLARE(live, AGX_NUM_REGS);
+      memcpy(live, blk->reg_live_out, sizeof(live));
+
+      agx_foreach_instr_in_block_rev(blk, ins) {
+         postra_liveness_ins(live, ins);
+      }
+
+      if (BITSET_EQUAL(blk->reg_live_in, live))
+         continue;
+
+      /* We made progress, so we need to reprocess the predecessors */
+      memcpy(blk->reg_live_in, live, sizeof(live));
+      agx_foreach_predecessor(blk, pred) {
+         agx_worklist_push_head(&worklist, *pred);
+      }
+   }
+
+   u_worklist_fini(&worklist);
+}
+
+static bool
+writes_reg(const agx_instr *I, unsigned reg)
+{
+   agx_foreach_reg_dest(I, d) {
+      unsigned count = agx_index_size_16(I->dest[d]);
+
+      if (reg >= I->dest[d].value && (reg - I->dest[d].value) < count)
+         return true;
+   }
+
+   return false;
+}
+
+/*
+ * Mark last-use sources to allow the hardware to discard from the register
+ * cache. Last use information follows immediately from (post-RA) liveness
+ * analysis: a register is dead immediately after its last use.
+ *
+ * Mark cache hints on sources/destinations to encourage the hardware to make
+ * better use of the register cache. This is a simple local analysis.
+ */
+void
+agx_opt_register_cache(agx_context *ctx)
+{
+   /* Analyze the shader globally */
+   postra_liveness(ctx);
+
+   agx_foreach_block(ctx, block) {
+      /* Live-set at each point in the program */
+      BITSET_DECLARE(live, AGX_NUM_REGS);
+      memcpy(live, block->reg_live_out, sizeof(live));
+
+      /* Set of registers read "soon" by an ALU instruction. These are
+       * candidates for the .cache bit.
+       */
+      BITSET_DECLARE(alu_reads, AGX_NUM_REGS) = {0};
+
+      agx_foreach_instr_in_block_rev(block, I) {
+         agx_foreach_reg_dest(I, d) {
+            uint64_t reg = I->dest[d].value;
+            unsigned nr = agx_index_size_16(I->dest[d]);
+
+            I->dest[d].cache = BITSET_TEST(alu_reads, I->dest[d].value);
+            BITSET_CLEAR_COUNT(alu_reads, reg, nr);
+         }
+
+         agx_foreach_reg_src(I, s) {
+            I->src[s].cache = BITSET_TEST(alu_reads, I->src[s].value);
+         }
+
+         agx_foreach_reg_src(I, s) {
+            uint64_t reg = I->src[s].value;
+            unsigned nr = agx_index_size_16(I->src[s]);
+
+            /* If the register dead after this instruction, it's the last use.
+             * That includes if the register is overwritten this cycle, but that
+             * won't show up in the liveness analysis.
+             */
+            bool lu = !BITSET_TEST_COUNT(live, reg, nr) || writes_reg(I, reg);
+
+            /* Handling divergent blocks would require physical CFG awareness.
+             * Just bail for now, skipping this pass won't affect correctness.
+             */
+            I->src[s].discard = lu && !block->divergent;
+
+            /* Mark any source read by an ALU instruction in the same block as
+             * wanting a .cache hint. This is better than just marking
+             * everything, although it overly hints for very long blocks and
+             * underhints for registers used across block boundaries. It's
+             * probably good enough, though, and it's not clear how to do much
+             * better given our limited understanding of the hardware.
+             */
+            if (agx_is_alu(I))
+               BITSET_SET_COUNT(alu_reads, reg, nr);
+
+            assert(!(I->src[s].discard && I->src[s].cache));
+         }
+
+         postra_liveness_ins(live, I);
+      }
+   }
+}

src/asahi/compiler/agx_optimizer.c

@@ -0,0 +1,655 @@
+/*
+ * Copyright 2021 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "util/lut.h"
+#include "util/macros.h"
+#include "agx_builder.h"
+#include "agx_compiler.h"
+#include "agx_opcodes.h"
+
+/* AGX peephole optimizer responsible for instruction combining. It operates in
+ * a forward direction and a backward direction, in each case traversing in
+ * source order. SSA means the forward pass satisfies the invariant:
+ *
+ *    Every def is visited before any of its uses.
+ *
+ * Dually, the backend pass satisfies the invariant:
+ *
+ *    Every use of a def is visited before the def.
+ *
+ * This means the forward pass can propagate modifiers forward, whereas the
+ * backwards pass propagates modifiers backward. Consider an example:
+ *
+ *    1 = fabs 0
+ *    2 = fround 1
+ *    3 = fsat 1
+ *
+ * The forwards pass would propagate the fabs to the fround (since we can
+ * lookup the fabs from the fround source and do the replacement). By contrast
+ * the backwards pass would propagate the fsat back to the fround (since when
+ * we see the fround we know it has only a single user, fsat).  Propagatable
+ * instruction have natural directions (like pushforwards and pullbacks).
+ *
+ * We are careful to update the tracked state whenever we modify an instruction
+ * to ensure the passes are linear-time and converge in a single iteration.
+ *
+ * Size conversions are worth special discussion. Consider the snippet:
+ *
+ *    2 = fadd 0, 1
+ *    3 = f2f16 2
+ *    4 = fround 3
+ *
+ * A priori, we can move the f2f16 in either direction. But it's not equal --
+ * if we move it up to the fadd, we get FP16 for two instructions, whereas if
+ * we push it into the fround, we effectively get FP32 for two instructions. So
+ * f2f16 is backwards. Likewise, consider
+ *
+ *    2 = fadd 0, 1
+ *    3 = f2f32 1
+ *    4 = fround 3
+ *
+ * This time if we move f2f32 up to the fadd, we get FP32 for two, but if we
+ * move it down to the fround, we get FP16 to too. So f2f32 is backwards.
+ */
+
+static bool
+agx_is_fmov(agx_instr *def)
+{
+   return (def->op == AGX_OPCODE_FADD || def->op == AGX_OPCODE_HADD) &&
+          agx_is_equiv(def->src[1], agx_negzero());
+}
+
+/* Compose floating-point modifiers with floating-point sources */
+
+static agx_index
+agx_compose_float_src(agx_index to, agx_index from)
+{
+   if (to.abs) {
+      from.neg = false;
+      from.abs = true;
+   }
+
+   from.neg ^= to.neg;
+
+   return from;
+}
+
+static void
+agx_optimizer_fmov(agx_instr **defs, agx_instr *ins)
+{
+   agx_foreach_ssa_src(ins, s) {
+      agx_index src = ins->src[s];
+      agx_instr *def = defs[src.value];
+
+      if (def == NULL)
+         continue; /* happens for phis in loops */
+      if (!agx_is_fmov(def))
+         continue;
+      if (def->saturate)
+         continue;
+      if (ins->op == AGX_OPCODE_FCMPSEL && s >= 2)
+         continue;
+
+      /* We can fold f2f32 into 32-bit instructions, but we can't fold f2f16
+       * into 16-bit instructions, since the latter would implicitly promote to
+       * a 32-bit instruction which is not exact.
+       */
+      assert(def->src[0].size == AGX_SIZE_32 ||
+             def->src[0].size == AGX_SIZE_16);
+      assert(src.size == AGX_SIZE_32 || src.size == AGX_SIZE_16);
+
+      if (src.size == AGX_SIZE_16 && def->src[0].size == AGX_SIZE_32)
+         continue;
+
+      ins->src[s] = agx_compose_float_src(src, def->src[0]);
+   }
+}
+
+static bool
+image_write_source_can_be_immediate(agx_instr *I, unsigned s)
+{
+   bool block = I->op == AGX_OPCODE_BLOCK_IMAGE_STORE;
+   assert(I->op == AGX_OPCODE_IMAGE_WRITE || block);
+
+   /* LOD can always be immediate. Actually, it's just zero so far, we don't
+    * support nonzero LOD for images yet.
+    */
+   if (s == 2 && !block)
+      return true;
+
+   /* If the "bindless" source (source 3) is an immediate, it means we don't
+    * have a bindless image, instead we have a texture state index. We're
+    * allowed to have immediate texture state registers (source 4). However,
+    * we're not allowed to have immediate bindless offsets (also source 4).
+    */
+   unsigned base = block ? 0 : 3;
+   bool is_texture_state = (I->src[base].type == AGX_INDEX_IMMEDIATE);
+   if (s == (base + 1) && is_texture_state)
+      return true;
+
+   /* Otherwise, must be from a register */
+   return false;
+}
+
+static void
+agx_optimizer_inline_imm(agx_instr **defs, agx_instr *I)
+{
+   agx_foreach_ssa_src(I, s) {
+      agx_index src = I->src[s];
+      if (src.neg)
+         continue;
+
+      agx_instr *def = defs[src.value];
+      if (!def || def->op != AGX_OPCODE_MOV_IMM)
+         continue;
+
+      uint8_t value = def->imm;
+      uint16_t value_u16 = def->imm;
+
+      bool float_src = agx_is_float_src(I, s);
+
+      if (I->op == AGX_OPCODE_ST_TILE && s == 0)
+         continue;
+      if (I->op == AGX_OPCODE_ZS_EMIT && s != 0)
+         continue;
+      if (I->op == AGX_OPCODE_TEXTURE_SAMPLE && s != 4)
+         continue;
+      if ((I->op == AGX_OPCODE_DEVICE_STORE ||
+           I->op == AGX_OPCODE_LOCAL_STORE || I->op == AGX_OPCODE_ATOMIC ||
+           I->op == AGX_OPCODE_LOCAL_ATOMIC) &&
+          s != 2)
+         continue;
+      if (I->op == AGX_OPCODE_ST_VARY && s != 0)
+         continue;
+      if ((I->op == AGX_OPCODE_LOCAL_LOAD || I->op == AGX_OPCODE_DEVICE_LOAD ||
+           I->op == AGX_OPCODE_STACK_STORE) &&
+          s != 1)
+         continue;
+      if (I->op == AGX_OPCODE_SPLIT)
+         continue;
+
+      if ((I->op == AGX_OPCODE_IMAGE_WRITE ||
+           I->op == AGX_OPCODE_BLOCK_IMAGE_STORE) &&
+          !image_write_source_can_be_immediate(I, s))
+         continue;
+
+      if (float_src) {
+         bool fp16 = (def->dest[0].size == AGX_SIZE_16);
+         assert(fp16 || (def->dest[0].size == AGX_SIZE_32));
+
+         float f = fp16 ? _mesa_half_to_float(def->imm) : uif(def->imm);
+         if (!agx_minifloat_exact(f))
+            continue;
+
+         I->src[s] = agx_immediate_f(f);
+      } else if (value == def->imm) {
+         I->src[s] = agx_immediate(value);
+      } else if (value_u16 == def->imm && agx_allows_16bit_immediate(I)) {
+         I->src[s] = agx_abs(agx_immediate(value_u16));
+      } else if ((I->op == AGX_OPCODE_IADD || I->op == AGX_OPCODE_IMAD) &&
+                 s == agx_negate_src_index(I)) {
+         unsigned bits = agx_size_align_16(def->dest[0].size) * 16;
+         uint64_t mask = BITFIELD64_MASK(bits);
+         uint64_t negated = (-def->imm) & mask;
+         value = negated;
+
+         /* Try to negate the immediate */
+         if (value == negated) {
+            I->src[s] = agx_neg(agx_immediate(value));
+         }
+      }
+   }
+}
+
+/*
+ * Fuse not into and/or/xor. Specifically, acts on not and fuses:
+ *
+ *    not(and(x, y) -> nand(x, y)
+ *    not(or(x, y) -> nor(x, y)
+ *    not(xor(x, y) -> xnor(x, y)
+ */
+static bool
+agx_optimizer_not(agx_instr *I, agx_instr *use)
+{
+   /* Check for bit op and use of not op */
+   if (I->op != AGX_OPCODE_BITOP || use->op != AGX_OPCODE_NOT)
+      return false;
+
+   /* Remap operation to the appropriate one */
+   I->truth_table ^= 0xF;
+   I->dest[0] = use->dest[0];
+
+   return true;
+}
+
+static bool
+agx_optimizer_fmov_rev(agx_instr *I, agx_instr *use)
+{
+   if (!agx_is_fmov(use))
+      return false;
+   if (use->src[0].neg || use->src[0].abs)
+      return false;
+
+   /* We can fold f2f16 into 32-bit instructions, but we can't fold f2f32 into
+    * 16-bit instructions, since the latter would implicitly promote to a 32-bit
+    * instruction which is not exact.
+    */
+   assert(use->dest[0].size == AGX_SIZE_32 || use->dest[0].size == AGX_SIZE_16);
+   assert(I->dest[0].size == AGX_SIZE_32 || I->dest[0].size == AGX_SIZE_16);
+
+   if (I->dest[0].size == AGX_SIZE_16 && use->dest[0].size == AGX_SIZE_32)
+      return false;
+
+   /* saturate(saturate(x)) = saturate(x) */
+   I->saturate |= use->saturate;
+   I->dest[0] = use->dest[0];
+   return true;
+}
+
+static bool
+agx_icond_is_unsigned(enum agx_icond cond)
+{
+   switch (cond) {
+   case AGX_ICOND_UEQ:
+   case AGX_ICOND_ULT:
+   case AGX_ICOND_UGT:
+      return true;
+
+   case AGX_ICOND_SEQ:
+   case AGX_ICOND_SLT:
+   case AGX_ICOND_SGT:
+      return false;
+   }
+
+   UNREACHABLE("invalid condition");
+}
+
+static bool
+agx_supports_zext(agx_instr *I, unsigned s)
+{
+   switch (I->op) {
+   case AGX_OPCODE_IADD:
+   case AGX_OPCODE_IMAD:
+   case AGX_OPCODE_INTL:
+   case AGX_OPCODE_FFS:
+   case AGX_OPCODE_BITREV:
+   case AGX_OPCODE_BFI:
+   case AGX_OPCODE_BFEIL:
+   case AGX_OPCODE_EXTR:
+   case AGX_OPCODE_BITOP:
+   case AGX_OPCODE_WHILE_ICMP:
+   case AGX_OPCODE_IF_ICMP:
+   case AGX_OPCODE_ELSE_ICMP:
+   case AGX_OPCODE_BREAK_IF_ICMP:
+   case AGX_OPCODE_ICMP_BALLOT:
+   case AGX_OPCODE_ICMP_QUAD_BALLOT:
+      return true;
+
+   case AGX_OPCODE_ICMP:
+   case AGX_OPCODE_ICMPSEL:
+      /* Only the comparisons can be extended, not the selection. And we can
+       * only zero-extend with unsigned comparison. Presumably the hardware
+       * sign-extends with signed comparisons but we don't handle that yet.
+       */
+      return (s < 2) && agx_icond_is_unsigned(I->icond);
+
+   default:
+      return false;
+   }
+}
+
+static void
+agx_optimizer_copyprop(agx_context *ctx, agx_instr **defs, agx_instr *I)
+{
+   agx_foreach_ssa_src(I, s) {
+      agx_index src = I->src[s];
+      agx_instr *def = defs[src.value];
+
+      if (def == NULL)
+         continue; /* happens for phis in loops */
+      if (def->op != AGX_OPCODE_MOV)
+         continue;
+
+      /* At the moment, not all instructions support size conversions. Notably
+       * RA pseudo instructions don't handle size conversions. This should be
+       * refined in the future.
+       */
+      if (def->src[0].size != src.size &&
+          !(def->src[0].size < src.size && agx_supports_zext(I, s)))
+         continue;
+
+      /* Optimize split(64-bit uniform) so we can get better copyprop of the
+       * 32-bit uniform parts. This helps reduce moves with 64-bit uniforms.
+       */
+      if (I->op == AGX_OPCODE_SPLIT && def->src[0].type == AGX_INDEX_UNIFORM &&
+          src.size == AGX_SIZE_64 && I->dest[0].size == AGX_SIZE_32) {
+
+         assert(I->nr_dests == 2 && "decomposing a 64-bit scalar");
+         agx_builder b = agx_init_builder(ctx, agx_before_instr(I));
+
+         agx_index lo = def->src[0];
+         lo.size = AGX_SIZE_32;
+
+         agx_index hi = lo;
+         hi.value += 2 /* half of 64-bits = 32-bits = 2 x 16-bits */;
+
+         defs[I->dest[0].value] = agx_mov_to(&b, I->dest[0], lo);
+         defs[I->dest[1].value] = agx_mov_to(&b, I->dest[1], hi);
+
+         agx_remove_instruction(I);
+         continue;
+      }
+
+      /* Immediate inlining happens elsewhere */
+      if (def->src[0].type == AGX_INDEX_IMMEDIATE)
+         continue;
+
+      /* ALU instructions cannot take 64-bit */
+      if (def->src[0].size == AGX_SIZE_64 &&
+          !(I->op == AGX_OPCODE_DEVICE_LOAD && s == 0) &&
+          !(I->op == AGX_OPCODE_DEVICE_STORE && s == 1) &&
+          !(I->op == AGX_OPCODE_ATOMIC && s == 1))
+         continue;
+
+      agx_replace_src(I, s, def->src[0]);
+
+      /* If we are zero-extending into an instruction that distinguishes sign
+       * and zero extend, make sure we pick zero-extend.
+       */
+      if (def->src[0].size < src.size &&
+          (I->op == AGX_OPCODE_IMAD || I->op == AGX_OPCODE_IADD)) {
+
+         assert(agx_supports_zext(I, s));
+         I->src[s].abs = true;
+      }
+   }
+}
+
+/*
+ * Fuse conditions into if. Specifically, acts on if_icmp and fuses:
+ *
+ *    if_icmp(cmp(x, y, *), 0, ne/eq) -> if_cmp(x, y, *)
+ */
+static void
+agx_optimizer_if_cmp(agx_instr **defs, agx_instr *I)
+{
+   /* Check for unfused if */
+   if (!agx_is_equiv(I->src[1], agx_zero()) || I->icond != AGX_ICOND_UEQ ||
+       I->src[0].type != AGX_INDEX_NORMAL)
+      return;
+
+   /* Check for condition */
+   agx_instr *def = defs[I->src[0].value];
+   if (def->op != AGX_OPCODE_ICMP && def->op != AGX_OPCODE_FCMP)
+      return;
+
+   /* Fuse */
+   I->src[0] = def->src[0];
+   I->src[1] = def->src[1];
+   I->invert_cond = def->invert_cond ^ !I->invert_cond;
+
+   if (def->op == AGX_OPCODE_ICMP) {
+      I->op = AGX_OPCODE_IF_ICMP;
+      I->icond = def->icond;
+   } else {
+      I->op = AGX_OPCODE_IF_FCMP;
+      I->fcond = def->fcond;
+   }
+}
+
+/*
+ * Fuse invert into if. Acts on if_icmp and fuses:
+ *
+ *    if_icmp(xor(x, 1), 0, ne) -> if_cmp(x, 0, eq)
+ */
+static void
+agx_optimizer_if_not(agx_instr **defs, agx_instr *I)
+{
+   /* Check for unfused if */
+   if (!agx_is_equiv(I->src[1], agx_zero()) || I->icond != AGX_ICOND_UEQ ||
+       I->src[0].type != AGX_INDEX_NORMAL)
+      return;
+
+   /* Check for invert */
+   agx_instr *def = defs[I->src[0].value];
+   if (def->op != AGX_OPCODE_BITOP ||
+       !agx_is_equiv(def->src[1], agx_immediate(1)) ||
+       def->truth_table != UTIL_LUT2(a ^ b))
+      return;
+
+   /* Fuse */
+   I->src[0] = def->src[0];
+   I->invert_cond = !I->invert_cond;
+}
+
+/*
+ * Fuse conditions into select. Specifically, acts on icmpsel and fuses:
+ *
+ *    icmpsel(cmp(x, y, *), 0, z, w, eq) -> cmpsel(x, y, w, z, *)
+ *
+ * Care must be taken to invert the condition by swapping cmpsel arguments.
+ */
+static void
+agx_optimizer_cmpsel(agx_instr **defs, agx_instr *I)
+{
+   /* Check for unfused select */
+   if (!agx_is_equiv(I->src[1], agx_zero()) || I->icond != AGX_ICOND_UEQ ||
+       I->src[0].type != AGX_INDEX_NORMAL)
+      return;
+
+   /* Check for condition */
+   agx_instr *def = defs[I->src[0].value];
+   if (def->op != AGX_OPCODE_ICMP && def->op != AGX_OPCODE_FCMP)
+      return;
+
+   /* Fuse */
+   I->src[0] = def->src[0];
+   I->src[1] = def->src[1];
+
+   /* In the unfused select, the condition is inverted due to the form:
+    *
+    *    (cond == 0) ? x : y
+    *
+    * So we need to swap the arguments when fusing to become cond ? y : x. If
+    * the condition was supposed to be inverted, we don't swap since it's
+    * already inverted. cmpsel does not have an invert_cond bit to use.
+    */
+   if (!def->invert_cond) {
+      agx_index temp = I->src[2];
+      I->src[2] = I->src[3];
+      I->src[3] = temp;
+   }
+
+   if (def->op == AGX_OPCODE_ICMP) {
+      I->op = AGX_OPCODE_ICMPSEL;
+      I->icond = def->icond;
+   } else {
+      I->op = AGX_OPCODE_FCMPSEL;
+      I->fcond = def->fcond;
+   }
+}
+
+/*
+ * Fuse conditions into ballots:
+ *
+ *    ballot(cmp(x, y)) -> ballot_cmp(x, y)
+ */
+static void
+agx_optimizer_ballot(agx_context *ctx, agx_instr **defs, agx_instr *I)
+{
+   if (I->src[0].type != AGX_INDEX_NORMAL)
+      return;
+
+   agx_instr *def = defs[I->src[0].value];
+   if (!def || (def->op != AGX_OPCODE_ICMP && def->op != AGX_OPCODE_FCMP))
+      return;
+
+   bool quad = I->op == AGX_OPCODE_QUAD_BALLOT;
+   assert(quad || I->op == AGX_OPCODE_BALLOT);
+
+   /* Replace with a fused instruction since the # of sources changes */
+   agx_builder b = agx_init_builder(ctx, agx_before_instr(I));
+
+   agx_instr *fused = agx_icmp_ballot_to(
+      &b, I->dest[0], def->src[0], def->src[1], def->icond, def->invert_cond);
+
+   if (def->op == AGX_OPCODE_ICMP) {
+      fused->op = quad ? AGX_OPCODE_ICMP_QUAD_BALLOT : AGX_OPCODE_ICMP_BALLOT;
+   } else {
+      fused->op = quad ? AGX_OPCODE_FCMP_QUAD_BALLOT : AGX_OPCODE_FCMP_BALLOT;
+   }
+
+   agx_remove_instruction(I);
+}
+
+/*
+ * Fuse not srcs into bitop.
+ */
+static void
+agx_optimizer_bitop(agx_instr **defs, agx_instr *I)
+{
+   agx_foreach_ssa_src(I, s) {
+      agx_index src = I->src[s];
+      agx_instr *def = defs[src.value];
+
+      /* If we find a not, select new operation and fuse */
+      if (def->op == AGX_OPCODE_NOT) {
+         I->truth_table = util_lut2_invert_source(I->truth_table, s);
+         I->src[s] = def->src[0];
+      }
+   }
+}
+
+/*
+ * Fuse sign-extends into addition-like instructions:
+ */
+static void
+agx_optimizer_signext(agx_instr **defs, agx_instr *I)
+{
+   agx_foreach_ssa_src(I, s) {
+      agx_index src = I->src[s];
+      agx_instr *def = defs[src.value];
+
+      if (def == NULL || def->op != AGX_OPCODE_SIGNEXT)
+         continue;
+
+      agx_replace_src(I, s, def->src[0]);
+      assert(!I->src[s].abs && "sign-extended");
+   }
+}
+
+void
+agx_optimizer_forward(agx_context *ctx)
+{
+   agx_instr **defs = calloc(ctx->alloc, sizeof(*defs));
+
+   agx_foreach_instr_global_safe(ctx, I) {
+      struct agx_opcode_info info = agx_opcodes_info[I->op];
+
+      agx_foreach_ssa_dest(I, d) {
+         defs[I->dest[d].value] = I;
+      }
+
+      /* Optimize moves */
+      agx_optimizer_copyprop(ctx, defs, I);
+
+      /* Propagate fmov down */
+      if (info.is_float || I->op == AGX_OPCODE_FCMPSEL ||
+          I->op == AGX_OPCODE_FCMP)
+         agx_optimizer_fmov(defs, I);
+
+      /* Inline immediates if we can. TODO: systematic */
+      if (I->op != AGX_OPCODE_COLLECT && I->op != AGX_OPCODE_IMAGE_LOAD &&
+          I->op != AGX_OPCODE_TEXTURE_LOAD &&
+          I->op != AGX_OPCODE_UNIFORM_STORE && I->op != AGX_OPCODE_EXPORT)
+         agx_optimizer_inline_imm(defs, I);
+
+      if (I->op == AGX_OPCODE_IF_ICMP) {
+         agx_optimizer_if_not(defs, I);
+         agx_optimizer_if_cmp(defs, I);
+      } else if (I->op == AGX_OPCODE_ICMPSEL) {
+         agx_optimizer_cmpsel(defs, I);
+      } else if (I->op == AGX_OPCODE_BALLOT ||
+                 I->op == AGX_OPCODE_QUAD_BALLOT) {
+         agx_optimizer_ballot(ctx, defs, I);
+      } else if (I->op == AGX_OPCODE_BITOP) {
+         agx_optimizer_bitop(defs, I);
+      } else if (I->op == AGX_OPCODE_IADD || I->op == AGX_OPCODE_IMAD) {
+         agx_optimizer_signext(defs, I);
+      }
+   }
+
+   free(defs);
+}
+
+static void
+record_use(agx_instr **uses, BITSET_WORD *multiple, agx_instr *I, unsigned src)
+{
+   unsigned v = I->src[src].value;
+
+   if (uses[v])
+      BITSET_SET(multiple, v);
+   else
+      uses[v] = I;
+}
+
+void
+agx_optimizer_backward(agx_context *ctx)
+{
+   agx_instr **uses = calloc(ctx->alloc, sizeof(*uses));
+   BITSET_WORD *multiple = BITSET_CALLOC(ctx->alloc);
+
+   agx_foreach_block_rev(ctx, block) {
+      /* Phi sources are logically read at the end of predecessor, so process
+       * our source in our successors' phis firsts. This ensures we set
+       * `multiple` correctly with phi sources.
+       */
+      agx_foreach_successor(block, succ) {
+         unsigned s = agx_predecessor_index(succ, block);
+
+         agx_foreach_phi_in_block(succ, phi) {
+            record_use(uses, multiple, phi, s);
+         }
+      }
+
+      agx_foreach_instr_in_block_rev(block, I) {
+         struct agx_opcode_info info = agx_opcodes_info[I->op];
+
+         /* Skip phis, they're handled specially */
+         if (I->op == AGX_OPCODE_PHI) {
+            continue;
+         }
+
+         agx_foreach_ssa_src(I, s) {
+            record_use(uses, multiple, I, s);
+         }
+
+         if (info.nr_dests != 1)
+            continue;
+
+         if (I->dest[0].type != AGX_INDEX_NORMAL)
+            continue;
+
+         agx_instr *use = uses[I->dest[0].value];
+
+         if (!use || BITSET_TEST(multiple, I->dest[0].value))
+            continue;
+
+         if (agx_optimizer_not(I, use)) {
+            agx_remove_instruction(use);
+            continue;
+         }
+
+         /* Destination has a single use, try to propagate */
+         if (info.is_float && agx_optimizer_fmov_rev(I, use)) {
+            agx_remove_instruction(use);
+            continue;
+         }
+      }
+   }
+
+   free(uses);
+   free(multiple);
+}

src/asahi/compiler/agx_performance.c

@@ -0,0 +1,156 @@
+/*
+ * Copyright 2023 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "agx_compile.h"
+#include "agx_compiler.h"
+#include "agx_opcodes.h"
+
+/* Table describing the relationship between registers pressure and thread
+ * count. Each entry describes a maximum number of registers and the associated
+ * best-case thread count.
+ *
+ * Sorted in ascending order of maximum registers for easy lookup.
+ */
+static const struct agx_occupancy occupancies[] = {
+   {104, 1024}, {112, 896}, {128, 832}, {136, 768}, {144, 704},
+   {160, 640},  {184, 576}, {208, 512}, {232, 448}, {256, 384},
+};
+
+struct agx_occupancy
+agx_occupancy_for_register_count(unsigned halfregs)
+{
+   for (unsigned i = 0; i < ARRAY_SIZE(occupancies); ++i) {
+      unsigned max = occupancies[i].max_registers;
+      assert((i == 0 || max > occupancies[i - 1].max_registers) && "ascending");
+
+      if (halfregs <= max)
+         return occupancies[i];
+   }
+
+   UNREACHABLE("Register count must be less than the maximum");
+}
+
+unsigned
+agx_max_registers_for_occupancy(unsigned occupancy)
+{
+   unsigned max_regs = 0;
+
+   for (unsigned i = 0; i < ARRAY_SIZE(occupancies); ++i) {
+      if (occupancy <= occupancies[i].max_threads)
+         max_regs = occupancies[i].max_registers;
+      else
+         break;
+   }
+
+   assert(max_regs > 0 && "Thread count must be less than the maximum");
+   return max_regs;
+}
+
+/* Crude cycle model for G13G */
+enum alu_unit {
+   NONE,
+   SCIB,
+   IC,
+   F32,
+   F16,
+};
+
+struct alu_timing {
+   enum alu_unit unit;
+   unsigned latency;
+   unsigned tp;
+};
+
+/* clang-format off */
+struct alu_timing op_timings[] = {
+   [AGX_OPCODE_FFMA]          = { F32, 2, 1 },
+   [AGX_OPCODE_FADD]          = { F32, 2, 1 },
+   [AGX_OPCODE_FMUL]          = { F32, 2, 1 },
+
+   [AGX_OPCODE_HFMA]          = { F16, 2, 1 },
+   [AGX_OPCODE_HADD]          = { F16, 2, 1 },
+   [AGX_OPCODE_HMUL]          = { F16, 2, 1 },
+
+   [AGX_OPCODE_MOV_IMM]       = { SCIB, 1, 1 },
+   [AGX_OPCODE_BITOP]         = { SCIB, 2, 1 }, /* tp might be 2 for 32-bit / no $? */
+   [AGX_OPCODE_ICMPSEL]       = { SCIB, 2, 1 },
+   [AGX_OPCODE_FCMPSEL]       = { SCIB, 2, 1 },
+   [AGX_OPCODE_IADD]          = { SCIB, 2, 1 },
+
+   [AGX_OPCODE_GET_SR]          = { SCIB, 2, 2 },
+   [AGX_OPCODE_GET_SR_BARRIER]  = { SCIB, 2, 2 },
+   [AGX_OPCODE_GET_SR_COVERAGE] = { SCIB, 2, 2 },
+
+   [AGX_OPCODE_IMAD]          = { IC, 3, 2 },
+   [AGX_OPCODE_BFI]           = { IC, 3, 2 },
+   [AGX_OPCODE_BFEIL]         = { IC, 3, 2 },
+   [AGX_OPCODE_EXTR]          = { IC, 3, 2 },
+   [AGX_OPCODE_ASR]           = { IC, 3, 2 },
+   [AGX_OPCODE_FLOOR]         = { IC, 3, 2 },
+   [AGX_OPCODE_SIN_PT_1]      = { IC, 3, 2 },
+   [AGX_OPCODE_SIN_PT_2]      = { IC, 5, 2 },
+   [AGX_OPCODE_LOG2]          = { IC, 5, 2 },
+   [AGX_OPCODE_EXP2]          = { IC, 5, 2 },
+   [AGX_OPCODE_RCP]           = { IC, 5, 3 },
+   [AGX_OPCODE_RSQRT]         = { IC, 6, 4 },
+   [AGX_OPCODE_SRSQRT]        = { IC, 6, 4 },
+
+   // XXX: check this
+   [AGX_OPCODE_SIMD_PREFIX]      = { SCIB, 18, 18 },
+   [AGX_OPCODE_SIMD_REDUCE]      = { SCIB, 24, 24 },
+   [AGX_OPCODE_QUAD_PREFIX]      = { SCIB, 18, 18 },
+   [AGX_OPCODE_QUAD_REDUCE]      = { SCIB, 24, 24 },
+   [AGX_OPCODE_SHUFFLE]          = { SCIB, 5, 2   },
+   [AGX_OPCODE_QUAD_SHUFFLE]     = { SCIB, 5, 2   },
+
+   [AGX_OPCODE_ICMP_BALLOT]      = { SCIB, 5, 2   },
+   [AGX_OPCODE_FCMP_BALLOT]      = { SCIB, 5, 2   },
+   [AGX_OPCODE_ICMP_QUAD_BALLOT] = { SCIB, 4, 2   },
+   [AGX_OPCODE_FCMP_QUAD_BALLOT] = { SCIB, 4, 2   },
+};
+/* clang-format on */
+
+static struct alu_timing
+alu_timing(const agx_instr *I)
+{
+   return I->op < ARRAY_SIZE(op_timings) ? op_timings[I->op]
+                                         : (struct alu_timing){0};
+}
+
+bool
+agx_is_alu(const agx_instr *I)
+{
+   return alu_timing(I).unit != NONE;
+}
+
+/*
+ * TODO: Model non-ALU instructions, latency, register cache, 64-bit, etc.
+ */
+struct agx_cycle_estimate
+agx_estimate_cycles(agx_context *ctx)
+{
+   struct agx_cycle_estimate est = {0};
+
+   agx_foreach_instr_global(ctx, I) {
+      struct alu_timing alu = alu_timing(I);
+
+      if (alu.unit == IC) {
+         est.ic += alu.tp * 2;
+
+         /* In addition to the IC work, IC ops appear to be dispatched along
+          * with SCIB.
+          */
+         est.f_scib += 1;
+      } else if (alu.unit) {
+         est.f_scib += alu.tp;
+      } else {
+         /* TODO */
+      }
+   }
+
+   /* IC and F/SCIB run in parallel across warps */
+   est.alu = MAX2(est.ic, est.f_scib);
+   return est;
+}

src/asahi/compiler/agx_pressure_schedule.c

@@ -0,0 +1,271 @@
+/*
+ * Copyright 2023 Alyssa Rosenzweig
+ * Copyright 2022 Collabora Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+/* Bottom-up local scheduler to reduce register pressure */
+
+#include "util/dag.h"
+#include "util/sparse_bitset.h"
+#include "agx_compiler.h"
+#include "agx_opcodes.h"
+
+struct sched_ctx {
+   /* Dependency graph */
+   struct dag *dag;
+
+   /* Live set */
+   struct u_sparse_bitset live;
+};
+
+struct sched_node {
+   struct dag_node dag;
+
+   /* Instruction this node represents */
+   agx_instr *instr;
+};
+
+static void
+add_dep(struct sched_node *a, struct sched_node *b)
+{
+   assert(a != b && "no self-dependencies");
+
+   if (a && b)
+      dag_add_edge(&a->dag, &b->dag, 0);
+}
+
+static void
+serialize(struct sched_node *a, struct sched_node **b)
+{
+   add_dep(a, *b);
+   *b = a;
+}
+
+static struct dag *
+create_dag(agx_context *ctx, agx_block *block, void *memctx)
+{
+   struct dag *dag = dag_create(ctx);
+
+   struct sched_node **last_write =
+      calloc(ctx->alloc, sizeof(struct sched_node *));
+   struct sched_node *coverage = NULL;
+   struct sched_node *preload = NULL;
+
+   /* Last memory load, to serialize stores against */
+   struct sched_node *memory_load = NULL;
+
+   /* Last memory store, to serialize loads and stores against */
+   struct sched_node *memory_store = NULL;
+
+   agx_foreach_instr_in_block(block, I) {
+      /* Don't touch control flow */
+      if (instr_after_logical_end(I))
+         break;
+
+      struct sched_node *node = rzalloc(memctx, struct sched_node);
+      node->instr = I;
+      dag_init_node(dag, &node->dag);
+
+      /* Reads depend on writes, no other hazards in SSA */
+      agx_foreach_ssa_src(I, s) {
+         add_dep(node, last_write[I->src[s].value]);
+      }
+
+      agx_foreach_ssa_dest(I, d) {
+         assert(I->dest[d].value < ctx->alloc);
+         last_write[I->dest[d].value] = node;
+      }
+
+      /* Classify the instruction and add dependencies according to the class */
+      enum agx_schedule_class dep = agx_opcodes_info[I->op].schedule_class;
+      assert(dep != AGX_SCHEDULE_CLASS_INVALID && "invalid instruction seen");
+
+      bool barrier = dep == AGX_SCHEDULE_CLASS_BARRIER;
+      bool discards =
+         I->op == AGX_OPCODE_SAMPLE_MASK || I->op == AGX_OPCODE_ZS_EMIT;
+
+      if (dep == AGX_SCHEDULE_CLASS_STORE)
+         add_dep(node, memory_load);
+      else if (dep == AGX_SCHEDULE_CLASS_ATOMIC || barrier)
+         serialize(node, &memory_load);
+
+      if (dep == AGX_SCHEDULE_CLASS_LOAD || dep == AGX_SCHEDULE_CLASS_STORE ||
+          dep == AGX_SCHEDULE_CLASS_ATOMIC || barrier)
+         serialize(node, &memory_store);
+
+      if (dep == AGX_SCHEDULE_CLASS_COVERAGE || barrier)
+         serialize(node, &coverage);
+
+      /* Make sure side effects happen before a discard */
+      if (discards)
+         add_dep(node, memory_store);
+
+      if (dep == AGX_SCHEDULE_CLASS_PRELOAD)
+         serialize(node, &preload);
+      else
+         add_dep(node, preload);
+   }
+
+   free(last_write);
+
+   return dag;
+}
+
+/*
+ * Calculate the change in register pressure from scheduling a given
+ * instruction. Equivalently, calculate the difference in the number of live
+ * registers before and after the instruction, given the live set after the
+ * instruction. This calculation follows immediately from the dataflow
+ * definition of liveness:
+ *
+ *      live_in = (live_out - KILL) + GEN
+ */
+static signed
+calculate_pressure_delta(agx_instr *I, struct u_sparse_bitset *live)
+{
+   signed delta = 0;
+
+   /* Destinations must be unique */
+   agx_foreach_ssa_dest(I, d) {
+      if (u_sparse_bitset_test(live, I->dest[d].value))
+         delta -= agx_index_size_16(I->dest[d]);
+   }
+
+   agx_foreach_ssa_src(I, src) {
+      /* Filter duplicates */
+      bool dupe = false;
+
+      for (unsigned i = 0; i < src; ++i) {
+         if (agx_is_equiv(I->src[i], I->src[src])) {
+            dupe = true;
+            break;
+         }
+      }
+
+      if (!dupe && !u_sparse_bitset_test(live, I->src[src].value))
+         delta += agx_index_size_16(I->src[src]);
+   }
+
+   return delta;
+}
+
+/*
+ * Choose the next instruction, bottom-up. For now we use a simple greedy
+ * heuristic: choose the instruction that has the best effect on liveness, while
+ * hoisting sample_mask.
+ */
+static struct sched_node *
+choose_instr(struct sched_ctx *s)
+{
+   int32_t min_delta = INT32_MAX;
+   struct sched_node *best = NULL;
+
+   list_for_each_entry(struct sched_node, n, &s->dag->heads, dag.link) {
+      /* Heuristic: hoist sample_mask/zs_emit. This allows depth/stencil tests
+       * to run earlier, and potentially to discard the entire quad invocation
+       * earlier, reducing how much redundant fragment shader we run.
+       *
+       * Since we schedule backwards, we make that happen by only choosing
+       * sample_mask when all other instructions have been exhausted.
+       */
+      if (n->instr->op == AGX_OPCODE_SAMPLE_MASK ||
+          n->instr->op == AGX_OPCODE_ZS_EMIT) {
+
+         if (!best) {
+            best = n;
+            assert(min_delta == INT32_MAX);
+         }
+
+         continue;
+      }
+
+      /* Heuristic: sink wait_pix to increase parallelism. Since wait_pix does
+       * not read or write registers, this has no effect on pressure.
+       */
+      if (n->instr->op == AGX_OPCODE_WAIT_PIX)
+         return n;
+
+      int32_t delta = calculate_pressure_delta(n->instr, &s->live);
+
+      if (delta < min_delta) {
+         best = n;
+         min_delta = delta;
+      }
+   }
+
+   return best;
+}
+
+static void
+pressure_schedule_block(agx_context *ctx, agx_block *block, struct sched_ctx *s)
+{
+   /* off by a constant, that's ok */
+   signed pressure = 0;
+   signed orig_max_pressure = 0;
+   unsigned nr_ins = 0;
+
+   u_sparse_bitset_dup(&s->live, &block->live_out);
+
+   agx_foreach_instr_in_block_rev(block, I) {
+      pressure += calculate_pressure_delta(I, &s->live);
+      orig_max_pressure = MAX2(pressure, orig_max_pressure);
+      agx_liveness_ins_update(&s->live, I);
+      nr_ins++;
+   }
+
+   u_sparse_bitset_dup(&s->live, &block->live_out);
+
+   /* off by a constant, that's ok */
+   signed max_pressure = 0;
+   pressure = 0;
+
+   struct sched_node **schedule = calloc(nr_ins, sizeof(struct sched_node *));
+   nr_ins = 0;
+
+   while (!list_is_empty(&s->dag->heads)) {
+      struct sched_node *node = choose_instr(s);
+      pressure += calculate_pressure_delta(node->instr, &s->live);
+      max_pressure = MAX2(pressure, max_pressure);
+      dag_prune_head(s->dag, &node->dag);
+
+      schedule[nr_ins++] = node;
+      agx_liveness_ins_update(&s->live, node->instr);
+   }
+
+   /* Bail if it looks like it's worse */
+   if (max_pressure >= orig_max_pressure) {
+      free(schedule);
+      return;
+   }
+
+   /* Apply the schedule */
+   for (unsigned i = 0; i < nr_ins; ++i) {
+      agx_remove_instruction(schedule[i]->instr);
+      list_add(&schedule[i]->instr->link, &block->instructions);
+   }
+
+   free(schedule);
+}
+
+void
+agx_pressure_schedule(agx_context *ctx)
+{
+   agx_compute_liveness(ctx);
+   void *memctx = ralloc_context(ctx);
+
+   agx_foreach_block(ctx, block) {
+      struct sched_ctx sctx = {.dag = create_dag(ctx, block, memctx)};
+      u_sparse_bitset_init(&sctx.live, ctx->alloc, memctx);
+
+      pressure_schedule_block(ctx, block, &sctx);
+   }
+
+   /* Clean up after liveness analysis */
+   agx_foreach_instr_global(ctx, I) {
+      agx_foreach_ssa_src(I, s)
+         I->src[s].kill = false;
+   }
+
+   ralloc_free(memctx);
+}

src/asahi/compiler/agx_print.c

@@ -0,0 +1,272 @@
+/*
+ * Copyright 2021 Alyssa Rosenzweig
+ * Copyright 2019-2020 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "util/lut.h"
+#include "util/u_math.h"
+#include "agx_builder.h"
+#include "agx_compiler.h"
+
+static void
+agx_print_sized(char prefix, unsigned value, enum agx_size size, FILE *fp)
+{
+   switch (size) {
+   case AGX_SIZE_16:
+      fprintf(fp, "%c%u%c", prefix, value >> 1, (value & 1) ? 'h' : 'l');
+      return;
+   case AGX_SIZE_32:
+      assert((value & 1) == 0);
+      fprintf(fp, "%c%u", prefix, value >> 1);
+      return;
+   case AGX_SIZE_64:
+      assert((value & 1) == 0);
+      fprintf(fp, "%c%u:%c%u", prefix, value >> 1, prefix, (value >> 1) + 1);
+      return;
+   }
+
+   UNREACHABLE("Invalid size");
+}
+
+static void
+agx_print_reg(agx_index index, unsigned value, FILE *fp)
+{
+   agx_print_sized('r', value, index.size, fp);
+
+   if (agx_channels(index) > 1) {
+      unsigned last =
+         value + agx_size_align_16(index.size) * (agx_channels(index) - 1);
+
+      fprintf(fp, "...");
+
+      if (index.memory)
+         fprintf(fp, "m");
+      agx_print_sized('r', last, index.size, fp);
+   }
+}
+
+void
+agx_print_index(agx_index index, bool is_float, FILE *fp)
+{
+   if (index.memory)
+      fprintf(fp, "m");
+
+   switch (index.type) {
+   case AGX_INDEX_NULL:
+      fprintf(fp, "_");
+      return;
+
+   case AGX_INDEX_NORMAL:
+      if (index.kill)
+         fprintf(fp, "*");
+
+      fprintf(fp, "%u", index.value);
+      break;
+
+   case AGX_INDEX_IMMEDIATE:
+      if (is_float) {
+         assert(index.value < 0x100);
+         fprintf(fp, "#%f", agx_minifloat_decode(index.value));
+      } else {
+         fprintf(fp, "#%u", index.value);
+      }
+
+      break;
+
+   case AGX_INDEX_UNDEF:
+      fprintf(fp, "undef");
+      break;
+
+   case AGX_INDEX_UNIFORM:
+      agx_print_sized('u', index.value, index.size, fp);
+      break;
+
+   case AGX_INDEX_REGISTER:
+      if (index.cache)
+         fprintf(fp, "$");
+
+      if (index.discard)
+         fprintf(fp, "^");
+
+      agx_print_reg(index, index.value, fp);
+      break;
+
+   default:
+      UNREACHABLE("Invalid index type");
+   }
+
+   if (index.type == AGX_INDEX_NORMAL) {
+      /* Print length suffixes if not implied */
+      if (index.size == AGX_SIZE_16)
+         fprintf(fp, "h");
+      else if (index.size == AGX_SIZE_64)
+         fprintf(fp, "d");
+
+      /* Print assigned register if we have one */
+      if (index.has_reg) {
+         fprintf(fp, "(");
+         if (index.memory)
+            fprintf(fp, "m");
+
+         agx_print_reg(index, index.reg, fp);
+         fprintf(fp, ")");
+      }
+   }
+
+   if (index.abs)
+      fprintf(fp, ".abs");
+
+   if (index.neg)
+      fprintf(fp, ".neg");
+}
+
+static const char *lut2_to_function[16] = {
+   "zero", "nor",  "andn2", "not2", "andn1", "not1", "xor", "nand",
+   "and",  "xnor", "mov1",  "orn2", "mov2",  "orn1", "or",  "one",
+};
+
+void
+agx_print_instr(const agx_instr *I, FILE *fp)
+{
+   assert(I->op < AGX_NUM_OPCODES);
+   struct agx_opcode_info info = agx_opcodes_info[I->op];
+   bool print_comma = false;
+
+   if (I->op == AGX_OPCODE_BITOP) {
+      info.name = lut2_to_function[I->truth_table];
+      info.immediates &= ~AGX_IMMEDIATE_TRUTH_TABLE;
+   }
+
+   fprintf(fp, "   ");
+
+   agx_foreach_dest(I, d) {
+      if (print_comma)
+         fprintf(fp, ", ");
+      else
+         print_comma = true;
+
+      agx_print_index(I->dest[d], false, fp);
+   }
+
+   if (I->nr_dests) {
+      fprintf(fp, " = ");
+      print_comma = false;
+   }
+
+   fprintf(fp, "%s", info.name);
+
+   if (I->saturate)
+      fprintf(fp, ".sat");
+
+   if (I->last)
+      fprintf(fp, ".last");
+
+   fprintf(fp, " ");
+
+   agx_foreach_src(I, s) {
+      if (print_comma)
+         fprintf(fp, ", ");
+      else
+         print_comma = true;
+
+      agx_print_index(I->src[s],
+                      agx_opcodes_info[I->op].is_float &&
+                         !(s >= 2 && I->op == AGX_OPCODE_FCMPSEL),
+                      fp);
+   }
+
+   if (I->mask) {
+      fprintf(fp, ", ");
+
+      for (unsigned i = 0; i < 4; ++i) {
+         if (I->mask & (1 << i))
+            fprintf(fp, "%c", "xyzw"[i]);
+      }
+   }
+
+   /* TODO: Do better for enums, truth tables, etc */
+   if (info.immediates) {
+      if (print_comma)
+         fprintf(fp, ", ");
+      else
+         print_comma = true;
+
+      fprintf(fp, "#0x%" PRIx64, I->imm);
+      if (I->imm && util_is_probably_float(I->imm)) {
+         fprintf(fp, " /* %f */", uif(I->imm));
+      }
+   }
+
+   if (info.immediates & AGX_IMMEDIATE_DIM) {
+      if (print_comma)
+         fprintf(fp, ", ");
+      else
+         print_comma = true;
+
+      fputs(agx_dim_as_str(I->dim), fp);
+   }
+
+   if (info.immediates & AGX_IMMEDIATE_SCOREBOARD) {
+      if (print_comma)
+         fprintf(fp, ", ");
+      else
+         print_comma = true;
+
+      fprintf(fp, "slot %u", I->scoreboard);
+   }
+
+   if (info.immediates & AGX_IMMEDIATE_NEST) {
+      if (print_comma)
+         fprintf(fp, ", ");
+      else
+         print_comma = true;
+
+      fprintf(fp, "n=%u", I->nest);
+   }
+
+   if ((info.immediates & AGX_IMMEDIATE_INVERT_COND) && I->invert_cond) {
+      if (print_comma)
+         fprintf(fp, ", ");
+      else
+         print_comma = true;
+
+      fprintf(fp, "inv");
+   }
+
+   fprintf(fp, "\n");
+}
+
+void
+agx_print_block(const agx_block *block, FILE *fp)
+{
+   fprintf(fp, "block%u {\n", block->index);
+
+   agx_foreach_instr_in_block(block, ins)
+      agx_print_instr(ins, fp);
+
+   fprintf(fp, "}");
+
+   if (block->successors[0]) {
+      fprintf(fp, " -> ");
+
+      agx_foreach_successor(block, succ)
+         fprintf(fp, "block%u ", succ->index);
+   }
+
+   if (block->predecessors.size) {
+      fprintf(fp, " from");
+
+      agx_foreach_predecessor(block, pred)
+         fprintf(fp, " block%u", (*pred)->index);
+   }
+
+   fprintf(fp, "\n\n");
+}
+
+void
+agx_print_shader(const agx_context *ctx, FILE *fp)
+{
+   agx_foreach_block(ctx, block)
+      agx_print_block(block, fp);
+}

src/asahi/compiler/agx_reindex_ssa.c

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2024 Valve Corporation
+ * SPDX-License-Identifier: MIT
+ */
+
+#include <stdlib.h>
+#include "agx_compiler.h"
+
+/* Reindex SSA to reduce memory usage */
+
+void
+agx_reindex_ssa(agx_context *ctx)
+{
+   unsigned *remap = calloc(ctx->alloc, sizeof(*remap));
+
+   ctx->alloc = 0;
+
+   agx_foreach_instr_global(ctx, I) {
+      agx_foreach_ssa_dest(I, d) {
+         assert(!remap[I->dest[d].value] && "input is SSA");
+         remap[I->dest[d].value] = ctx->alloc++;
+         I->dest[d].value = remap[I->dest[d].value];
+      }
+   }
+
+   agx_foreach_instr_global(ctx, I) {
+      agx_foreach_ssa_src(I, s) {
+         I->src[s].value = remap[I->src[s].value];
+      }
+   }
+
+   free(remap);
+}

src/asahi/compiler/agx_repair_ssa.c

@@ -0,0 +1,331 @@
+/*
+ * Copyright 2023 Alyssa Rosenzweig
+ * Copyright 2023 Valve Corporation
+ * Copyright 2022 Collabora Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+/*
+ * Implementation of "Simple and Efficient
+ * Construction of Static Single Assignment Form", also by Braun et al.
+ * https://link.springer.com/content/pdf/10.1007/978-3-642-37051-9_6.pdf
+ */
+
+#include "util/hash_table.h"
+#include "util/ralloc.h"
+#include "util/u_dynarray.h"
+#include "agx_builder.h"
+#include "agx_compiler.h"
+#include "agx_opcodes.h"
+
+struct repair_block {
+   /* For a loop header, whether phis operands have been added */
+   bool sealed;
+
+   /* Sparse map: variable name -> agx_index.
+    *
+    * Definition of a variable at the end of the block.
+    */
+   struct hash_table_u64 *defs;
+};
+
+struct repair_ctx {
+   agx_context *shader;
+
+   /* Number of variables */
+   unsigned n;
+
+   /* Information on blocks indexed in source order */
+   struct repair_block *blocks;
+};
+
+static inline struct repair_block *
+repair_block(struct repair_ctx *ctx, agx_block *block)
+{
+   return &ctx->blocks[block->index];
+}
+
+static void
+record_write(struct repair_ctx *ctx, agx_block *block, unsigned node,
+             agx_index val)
+{
+   assert(node < ctx->n);
+   struct hash_table_u64 *defs = repair_block(ctx, block)->defs;
+   _mesa_hash_table_u64_insert(defs, node,
+                               ralloc_memdup(defs, &val, sizeof(val)));
+}
+
+static void add_phi_operands(struct repair_ctx *ctx, agx_block *block,
+                             agx_instr *phi, agx_index node);
+
+static agx_index
+resolve_read(struct repair_ctx *ctx, agx_block *block, agx_index node)
+{
+   struct repair_block *rb = repair_block(ctx, block);
+
+   /* Local value numbering */
+   assert(node.type == AGX_INDEX_NORMAL);
+   agx_index *local = _mesa_hash_table_u64_search(rb->defs, node.value);
+
+   if (local) {
+      assert(!agx_is_null(*local));
+      return *local;
+   }
+
+   /* Global value numbering. readValueRecursive in the paper */
+   unsigned nr_preds = agx_num_predecessors(block);
+   agx_index val;
+
+   assert(nr_preds > 0);
+
+   /* Loop headers are not in the "sealedBlock" set in the paper. To
+    * handle, we insert an incomplete phi to be filled in after the rest of
+    * the loop is processed.
+    */
+   if (block->loop_header && !rb->sealed) {
+      val = agx_temp_like(ctx->shader, node);
+      agx_builder b = agx_init_builder(ctx->shader, agx_before_block(block));
+      agx_instr *phi = agx_phi_to(&b, val, nr_preds);
+      phi->shadow = true;
+
+      /* Stash the variable in for an intrusive incompletePhis map */
+      phi->imm = node.value + 1;
+   } else if (nr_preds == 1) {
+      /* No phi needed */
+      agx_block *pred =
+         *util_dynarray_element(&block->predecessors, agx_block *, 0);
+      val = resolve_read(ctx, pred, node);
+   } else {
+      /* Insert phi first to break cycles */
+      val = agx_temp_like(ctx->shader, node);
+      agx_builder b = agx_init_builder(ctx->shader, agx_before_block(block));
+      agx_instr *phi = agx_phi_to(&b, val, nr_preds);
+      phi->shadow = true;
+      record_write(ctx, block, node.value, val);
+      add_phi_operands(ctx, block, phi, node);
+   }
+
+   assert(!agx_is_null(val));
+   record_write(ctx, block, node.value, val);
+   return val;
+}
+
+static void
+add_phi_operands(struct repair_ctx *ctx, agx_block *block, agx_instr *phi,
+                 agx_index node)
+{
+   /* Add phi operands */
+   agx_foreach_predecessor(block, pred) {
+      unsigned s = agx_predecessor_index(block, *pred);
+      phi->src[s] = resolve_read(ctx, *pred, node);
+   }
+}
+
+static void
+seal_block(struct repair_ctx *ctx, agx_block *block)
+{
+   agx_foreach_phi_in_block(block, phi) {
+      /* We use phi->imm as a sideband to pass the variable name. */
+      if (phi->imm) {
+         agx_index var = agx_get_vec_index(phi->imm - 1, phi->dest[0].size,
+                                           agx_channels(phi->dest[0]));
+         var.memory = phi->dest[0].memory;
+         add_phi_operands(ctx, block, phi, var);
+         phi->imm = 0;
+      }
+   }
+
+   repair_block(ctx, block)->sealed = true;
+}
+
+static void
+seal_loop_headers(struct repair_ctx *ctx, struct agx_block *exit)
+{
+   agx_foreach_successor(exit, succ) {
+      /* Only loop headers need to be sealed late */
+      if (!succ->loop_header)
+         continue;
+
+      /* Check if all predecessors have been processed */
+      bool any_unprocessed = false;
+
+      agx_foreach_predecessor(succ, P) {
+         if ((*P)->index > exit->index) {
+            any_unprocessed = true;
+            break;
+         }
+      }
+
+      /* Seal once all predecessors are processed */
+      if (!any_unprocessed)
+         seal_block(ctx, succ);
+   }
+}
+
+static void
+agx_opt_trivial_phi(agx_context *ctx)
+{
+   agx_index *remap = calloc(ctx->alloc, sizeof(*remap));
+   for (;;) {
+      bool progress = false;
+      memset(remap, 0, ctx->alloc * sizeof(*remap));
+
+      agx_foreach_block(ctx, block) {
+         agx_foreach_phi_in_block_safe(block, phi) {
+            agx_index same = agx_null();
+            bool all_same = true;
+
+            agx_foreach_src(phi, s) {
+               /* Only optimize trivial phis with normal sources. It is possible
+                * to optimize something like `phi #0, #0` but...
+                *
+                * 1. It would inadvently propagate constants which may be
+                * invalid. Copyprop knows the rules for this, but we don't here.
+                *
+                * 2. These trivial phis should be optimized at the NIR level.
+                * This pass is just to clean up spilling.
+                *
+                * So skip them for correctness in case NIR misses something
+                * (which can happen depending on pass order).
+                *
+                * ---
+                *
+                * TODO: Handle cycles faster.
+                */
+               if (phi->src[s].type != AGX_INDEX_NORMAL ||
+                   agx_is_null(remap[phi->src[s].value])) {
+                  all_same = false;
+                  break;
+               }
+
+               /* Same value or self-reference */
+               if (agx_is_equiv(phi->src[s], same) ||
+                   agx_is_equiv(phi->src[s], phi->dest[0]))
+                  continue;
+
+               if (!agx_is_null(same)) {
+                  all_same = false;
+                  break;
+               }
+
+               same = phi->src[s];
+            }
+
+            if (all_same) {
+               remap[phi->dest[0].value] = same;
+               agx_remove_instruction(phi);
+               progress = true;
+            }
+         }
+      }
+
+      if (!progress)
+         break;
+
+      agx_foreach_instr_global(ctx, I) {
+         agx_foreach_ssa_src(I, s) {
+            if (!agx_is_null(remap[I->src[s].value])) {
+               agx_replace_src(I, s, remap[I->src[s].value]);
+            }
+         }
+      }
+   }
+
+   free(remap);
+}
+
+void
+agx_repair_ssa(agx_context *ctx)
+{
+   struct repair_block *blocks =
+      rzalloc_array(NULL, struct repair_block, ctx->num_blocks);
+
+   agx_foreach_block(ctx, block) {
+      struct repair_block *rb = &blocks[block->index];
+      rb->defs = _mesa_hash_table_u64_create(blocks);
+   }
+
+   unsigned n = ctx->alloc;
+
+   agx_foreach_block(ctx, block) {
+      struct repair_ctx rctx = {
+         .shader = ctx,
+         .n = n,
+         .blocks = blocks,
+      };
+
+      agx_foreach_instr_in_block(block, I) {
+         /* Repair SSA for the instruction */
+         if (I->op != AGX_OPCODE_PHI) {
+            agx_foreach_ssa_src(I, s) {
+               assert(I->src[s].value < n);
+               agx_replace_src(I, s, resolve_read(&rctx, block, I->src[s]));
+            }
+         }
+
+         agx_foreach_ssa_dest(I, d) {
+            unsigned handle = I->dest[d].value;
+
+            /* Skip phis that we just created when processing loops */
+            if (handle >= n) {
+               assert(I->op == AGX_OPCODE_PHI);
+               continue;
+            }
+
+            I->dest[d] =
+               agx_replace_index(I->dest[d], agx_temp_like(ctx, I->dest[d]));
+
+            record_write(&rctx, block, handle, I->dest[d]);
+         }
+      }
+
+      seal_loop_headers(&rctx, block);
+   }
+
+   agx_foreach_block(ctx, block) {
+      agx_foreach_phi_in_block(block, phi) {
+         /* The kill bit is invalid (and meaningless) for phis. Liveness
+          * analysis does not produce it. However, we're ingesting broken SSA
+          * where we can have random kill bits set on phis. Strip them as part
+          * of the SSA repair.
+          *
+          * The register allocator depends on this for correctness.
+          */
+         phi->dest[0].kill = false;
+
+         agx_foreach_src(phi, s) {
+            phi->src[s].kill = false;
+         }
+
+         /* Skip the phis that we just created */
+         if (phi->shadow) {
+            phi->shadow = false;
+            continue;
+         }
+
+         agx_foreach_ssa_src(phi, s) {
+            /* Phis (uniquely) read their sources in their corresponding
+             * predecessors, so chain through for that.
+             */
+            agx_block *read_block =
+               *util_dynarray_element(&block->predecessors, agx_block *, s);
+
+            assert(phi->src[s].value < n);
+
+            struct repair_ctx rctx = {
+               .shader = ctx,
+               .n = n,
+               .blocks = blocks,
+            };
+
+            agx_replace_src(phi, s,
+                            resolve_read(&rctx, read_block, phi->src[s]));
+         }
+      }
+   }
+
+   ralloc_free(blocks);
+
+   agx_opt_trivial_phi(ctx);
+   agx_reindex_ssa(ctx);
+}

src/asahi/compiler/agx_validate.c

@@ -0,0 +1,567 @@
+/*
+ * Copyright 2022 Alyssa Rosenzweig
+ * Copyright 2021 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "util/compiler.h"
+#include "agx_compiler.h"
+#include "agx_debug.h"
+#include "agx_opcodes.h"
+
+/* Validatation doesn't make sense in release builds */
+#ifndef NDEBUG
+
+#define agx_validate_assert(stmt)                                              \
+   if (!(stmt)) {                                                              \
+      return false;                                                            \
+   }
+
+/*
+ * If a block contains phi nodes, they must come at the start of the block. If a
+ * block contains control flow, it must come at the beginning/end as applicable.
+ * Therefore the form of a valid block is:
+ *
+ *       Control flow instructions (else)
+ *       Phi nodes
+ *       General instructions
+ *       Control flow instructions (except else)
+ *
+ * Validate that this form is satisfied.
+ */
+enum agx_block_state {
+   AGX_BLOCK_STATE_CF_ELSE = 0,
+   AGX_BLOCK_STATE_PHI = 1,
+   AGX_BLOCK_STATE_BODY = 2,
+   AGX_BLOCK_STATE_CF = 3
+};
+
+static bool
+agx_validate_block_form(agx_block *block)
+{
+   enum agx_block_state state = AGX_BLOCK_STATE_CF_ELSE;
+
+   agx_foreach_instr_in_block(block, I) {
+      switch (I->op) {
+      case AGX_OPCODE_PRELOAD:
+      case AGX_OPCODE_ELSE_ICMP:
+      case AGX_OPCODE_ELSE_FCMP:
+         agx_validate_assert(state == AGX_BLOCK_STATE_CF_ELSE);
+         break;
+
+      case AGX_OPCODE_PHI:
+         agx_validate_assert(state == AGX_BLOCK_STATE_CF_ELSE ||
+                             state == AGX_BLOCK_STATE_PHI);
+
+         state = AGX_BLOCK_STATE_PHI;
+         break;
+
+      case AGX_OPCODE_EXPORT:
+         agx_validate_assert(agx_num_successors(block) == 0);
+         state = AGX_BLOCK_STATE_CF;
+         break;
+
+      default:
+         if (instr_after_logical_end(I)) {
+            state = AGX_BLOCK_STATE_CF;
+         } else {
+            agx_validate_assert(state != AGX_BLOCK_STATE_CF);
+            state = AGX_BLOCK_STATE_BODY;
+         }
+         break;
+      }
+   }
+
+   return true;
+}
+
+/*
+ * Only moves and phis use stack. Phis cannot use moves due to their
+ * parallel nature, so we allow phis to take memory, later lowered to moves.
+ */
+static bool
+is_stack_valid(agx_instr *I)
+{
+   return (I->op == AGX_OPCODE_MOV) || (I->op == AGX_OPCODE_PHI);
+}
+
+static bool
+agx_validate_sources(agx_instr *I)
+{
+   agx_foreach_src(I, s) {
+      agx_index src = I->src[s];
+
+      if (src.type == AGX_INDEX_IMMEDIATE) {
+         agx_validate_assert(!src.kill);
+         agx_validate_assert(!src.cache);
+         agx_validate_assert(!src.discard);
+
+         bool ldst = agx_allows_16bit_immediate(I);
+
+         /* Immediates are encoded as 8-bit (16-bit for memory load/store). For
+          * integers, they extend to 16-bit. For floating point, they are 8-bit
+          * minifloats. The 8-bit minifloats are a strict subset of 16-bit
+          * standard floats, so we treat them as such in the IR, with an
+          * implicit f16->f32 for 32-bit floating point operations.
+          */
+         agx_validate_assert(src.size == AGX_SIZE_16);
+         agx_validate_assert(src.value < (1 << (ldst ? 16 : 8)));
+      } else if (I->op == AGX_OPCODE_COLLECT && !agx_is_null(src)) {
+         agx_validate_assert(src.size == I->src[0].size);
+      } else if (I->op == AGX_OPCODE_PHI) {
+         agx_validate_assert(src.size == I->dest[0].size);
+         agx_validate_assert(!agx_is_null(src));
+      }
+
+      agx_validate_assert(!src.memory || is_stack_valid(I));
+   }
+
+   return true;
+}
+
+static bool
+agx_validate_defs(agx_instr *I, BITSET_WORD *defs)
+{
+   agx_foreach_ssa_src(I, s) {
+      /* Skip phis, they're special in loop headers */
+      if (I->op == AGX_OPCODE_PHI)
+         break;
+
+      /* Sources must be defined before their use */
+      if (!BITSET_TEST(defs, I->src[s].value))
+         return false;
+   }
+
+   agx_foreach_ssa_dest(I, d) {
+      /* Static single assignment */
+      if (BITSET_TEST(defs, I->dest[d].value))
+         return false;
+
+      BITSET_SET(defs, I->dest[d].value);
+
+      if (I->dest[d].memory && !is_stack_valid(I))
+         return false;
+   }
+
+   return true;
+}
+
+/** Returns number of registers written by an instruction */
+static unsigned
+agx_write_registers(const agx_instr *I, unsigned d)
+{
+   unsigned size = agx_size_align_16(I->dest[d].size);
+
+   switch (I->op) {
+   case AGX_OPCODE_MOV:
+   case AGX_OPCODE_PHI:
+      /* Tautological */
+      return agx_index_size_16(I->dest[d]);
+
+   case AGX_OPCODE_ITER:
+   case AGX_OPCODE_ITERPROJ:
+      assert(1 <= I->channels && I->channels <= 4);
+      return I->channels * size;
+
+   case AGX_OPCODE_IMAGE_LOAD:
+   case AGX_OPCODE_TEXTURE_LOAD:
+   case AGX_OPCODE_TEXTURE_SAMPLE:
+      /* Even when masked out, these clobber 4 registers.
+       *
+       * TODO: Figure out the sparse interaction.
+       */
+      return (I->sparse ? 8 : 4) * size;
+
+   case AGX_OPCODE_DEVICE_LOAD:
+   case AGX_OPCODE_LOCAL_LOAD:
+   case AGX_OPCODE_STACK_LOAD:
+   case AGX_OPCODE_LD_TILE:
+      /* Can write 16-bit or 32-bit. Anything logically 64-bit is already
+       * expanded to 32-bit in the mask.
+       */
+      return util_bitcount(I->mask) * MIN2(size, 2);
+
+   case AGX_OPCODE_LDCF:
+      return 6;
+   case AGX_OPCODE_COLLECT:
+      return I->nr_srcs * agx_size_align_16(I->src[0].size);
+   default:
+      return size;
+   }
+}
+
+struct dim_info {
+   unsigned comps;
+   bool array;
+};
+
+static struct dim_info
+agx_dim_info(enum agx_dim dim)
+{
+   switch (dim) {
+   case AGX_DIM_1D:
+      return (struct dim_info){1, false};
+   case AGX_DIM_1D_ARRAY:
+      return (struct dim_info){1, true};
+   case AGX_DIM_2D:
+      return (struct dim_info){2, false};
+   case AGX_DIM_2D_ARRAY:
+      return (struct dim_info){2, true};
+   case AGX_DIM_2D_MS:
+      return (struct dim_info){3, false};
+   case AGX_DIM_3D:
+      return (struct dim_info){3, false};
+   case AGX_DIM_CUBE:
+      return (struct dim_info){3, false};
+   case AGX_DIM_CUBE_ARRAY:
+      return (struct dim_info){3, true};
+   case AGX_DIM_2D_MS_ARRAY:
+      return (struct dim_info){2, true};
+   default:
+      UNREACHABLE("invalid dim");
+   }
+}
+
+/*
+ * Return number of registers required for coordinates for a texture/image
+ * instruction. We handle layer + sample index as 32-bit even when only the
+ * lower 16-bits are present. LOD queries do not take a layer.
+ */
+static unsigned
+agx_coordinate_registers(const agx_instr *I, enum agx_size size)
+{
+   struct dim_info dim = agx_dim_info(I->dim);
+   bool has_array = !I->query_lod;
+
+   return agx_size_align_16(size) * (dim.comps + (has_array && dim.array));
+}
+
+static unsigned
+agx_read_registers(const agx_instr *I, unsigned s)
+{
+   unsigned size = agx_size_align_16(I->src[s].size);
+
+   switch (I->op) {
+   case AGX_OPCODE_MOV:
+   case AGX_OPCODE_EXPORT:
+      /* Tautological */
+      return agx_index_size_16(I->src[0]);
+
+   case AGX_OPCODE_PHI:
+      if (I->src[s].type == AGX_INDEX_IMMEDIATE)
+         return size;
+      else
+         return agx_index_size_16(I->dest[0]);
+
+   case AGX_OPCODE_SPLIT:
+      return I->nr_dests * agx_size_align_16(agx_split_width(I));
+
+   case AGX_OPCODE_UNIFORM_STORE:
+      if (s == 0)
+         return util_bitcount(I->mask) * size;
+      else
+         return size;
+
+   case AGX_OPCODE_DEVICE_STORE:
+   case AGX_OPCODE_LOCAL_STORE:
+   case AGX_OPCODE_STACK_STORE:
+   case AGX_OPCODE_ST_TILE:
+      /* See agx_write_registers */
+      if (s == 0)
+         return util_bitcount(I->mask) * MIN2(size, 2);
+      else if (s == 2 && I->explicit_coords)
+         return 2;
+      else
+         return size;
+
+   case AGX_OPCODE_ZS_EMIT:
+      if (s == 1) {
+         /* Depth (bit 0) is fp32, stencil (bit 1) is u16 in the hw but we pad
+          * up to u32 for simplicity
+          */
+         bool z = !!(I->zs & 1);
+         bool s = !!(I->zs & 2);
+         assert(z || s);
+
+         return (z && s) ? 4 : z ? 2 : 1;
+      } else {
+         return 1;
+      }
+
+   case AGX_OPCODE_IMAGE_WRITE:
+      if (s == 0)
+         return 4 * size /* data */;
+      else if (s == 1)
+         return agx_coordinate_registers(I, I->src[1].size);
+      else
+         return size;
+
+   case AGX_OPCODE_IMAGE_LOAD:
+   case AGX_OPCODE_TEXTURE_LOAD:
+   case AGX_OPCODE_TEXTURE_SAMPLE:
+      if (s == 0) {
+         return agx_coordinate_registers(I, I->src[0].size);
+      } else if (s == 1) {
+         /* LOD */
+         if (I->lod_mode == AGX_LOD_MODE_LOD_GRAD ||
+             I->lod_mode == AGX_LOD_MODE_LOD_GRAD_MIN) {
+
+            /* Technically only 16-bit but we model as 32-bit to keep the IR
+             * simple, since the gradient is otherwise 32-bit.
+             */
+            unsigned min = I->lod_mode == AGX_LOD_MODE_LOD_GRAD_MIN ? 2 : 0;
+
+            switch (I->dim) {
+            case AGX_DIM_1D:
+            case AGX_DIM_1D_ARRAY:
+               return (2 * 2 * 1) + min;
+            case AGX_DIM_2D:
+            case AGX_DIM_2D_ARRAY:
+            case AGX_DIM_2D_MS_ARRAY:
+            case AGX_DIM_2D_MS:
+               return (2 * 2 * 2) + min;
+            case AGX_DIM_CUBE:
+            case AGX_DIM_CUBE_ARRAY:
+            case AGX_DIM_3D:
+               return (2 * 2 * 3) + min;
+            }
+
+            UNREACHABLE("Invalid texture dimension");
+         } else if (I->lod_mode == AGX_LOD_MODE_AUTO_LOD_BIAS_MIN) {
+            return 2;
+         } else {
+            return 1;
+         }
+      } else if (s == 5) {
+         /* Compare/offset */
+         return 2 * ((!!I->shadow) + (!!I->offset));
+      } else {
+         return size;
+      }
+
+   case AGX_OPCODE_BLOCK_IMAGE_STORE:
+      if (s == 3 && I->explicit_coords)
+         return agx_coordinate_registers(I, I->src[3].size);
+      else
+         return size;
+
+   case AGX_OPCODE_ATOMIC:
+   case AGX_OPCODE_LOCAL_ATOMIC:
+      if (s == 0 && I->atomic_opc == AGX_ATOMIC_OPC_CMPXCHG)
+         return size * 2;
+      else
+         return size;
+
+   default:
+      return size;
+   }
+}
+
+/* Type check the dimensionality of sources and destinations. */
+static bool
+agx_validate_width(agx_context *ctx)
+{
+   bool succ = true;
+   enum agx_size *sizes = calloc(ctx->alloc, sizeof(*sizes));
+
+   agx_foreach_instr_global(ctx, I) {
+      agx_foreach_dest(I, d) {
+         unsigned exp = agx_write_registers(I, d);
+         unsigned act =
+            agx_channels(I->dest[d]) * agx_size_align_16(I->dest[d].size);
+
+         if (exp != act) {
+            succ = false;
+            fprintf(stderr, "destination %u, expected width %u, got width %u\n",
+                    d, exp, act);
+            agx_print_instr(I, stderr);
+            fprintf(stderr, "\n");
+         }
+
+         if (I->dest[d].type == AGX_INDEX_NORMAL)
+            sizes[I->dest[d].value] = I->dest[d].size;
+      }
+
+      agx_foreach_src(I, s) {
+         if (I->src[s].type == AGX_INDEX_NULL)
+            continue;
+
+         unsigned exp = agx_read_registers(I, s);
+         unsigned act =
+            agx_channels(I->src[s]) * agx_size_align_16(I->src[s].size);
+
+         if (exp != act) {
+            succ = false;
+            fprintf(stderr, "source %u, expected width %u, got width %u\n", s,
+                    exp, act);
+            agx_print_instr(I, stderr);
+            fprintf(stderr, "\n");
+         }
+      }
+   }
+
+   /* Check sources after all defs processed for proper backedge handling */
+   agx_foreach_instr_global(ctx, I) {
+      agx_foreach_ssa_src(I, s) {
+         if (sizes[I->src[s].value] != I->src[s].size) {
+            succ = false;
+            fprintf(stderr, "source %u, expected el size %u, got el size %u\n",
+                    s, agx_size_align_16(sizes[I->src[s].value]),
+                    agx_size_align_16(I->src[s].size));
+            agx_print_instr(I, stderr);
+            fprintf(stderr, "\n");
+         }
+      }
+   }
+
+   free(sizes);
+   return succ;
+}
+
+static bool
+agx_validate_predecessors(agx_block *block)
+{
+   /* Loop headers (only) have predecessors that are later in source form */
+   bool has_later_preds = false;
+
+   agx_foreach_predecessor(block, pred) {
+      if ((*pred)->index >= block->index)
+         has_later_preds = true;
+   }
+
+   if (has_later_preds && !block->loop_header)
+      return false;
+
+   /* Successors and predecessors are found together */
+   agx_foreach_predecessor(block, pred) {
+      bool found = false;
+
+      agx_foreach_successor((*pred), succ) {
+         if (succ == block)
+            found = true;
+      }
+
+      if (!found)
+         return false;
+   }
+
+   return true;
+}
+
+static bool
+agx_validate_sr(const agx_instr *I)
+{
+   bool none = (I->op == AGX_OPCODE_GET_SR);
+   bool coverage = (I->op == AGX_OPCODE_GET_SR_COVERAGE);
+   bool barrier = (I->op == AGX_OPCODE_GET_SR_BARRIER);
+
+   /* Filter get_sr instructions */
+   if (!(none || coverage || barrier))
+      return true;
+
+   switch (I->sr) {
+   case AGX_SR_ACTIVE_THREAD_INDEX_IN_QUAD:
+   case AGX_SR_ACTIVE_THREAD_INDEX_IN_SUBGROUP:
+   case AGX_SR_TOTAL_ACTIVE_THREADS_IN_QUAD:
+   case AGX_SR_TOTAL_ACTIVE_THREADS_IN_SUBGROUP:
+   case AGX_SR_COVERAGE_MASK:
+   case AGX_SR_IS_ACTIVE_THREAD:
+      return coverage;
+
+   case AGX_SR_HELPER_OP:
+   case AGX_SR_HELPER_ARG_L:
+   case AGX_SR_HELPER_ARG_H:
+      return barrier;
+
+   default:
+      return none;
+   }
+}
+
+void
+agx_validate(agx_context *ctx, const char *after)
+{
+   bool fail = false;
+
+   if (agx_compiler_debug & AGX_DBG_NOVALIDATE)
+      return;
+
+   int last_index = -1;
+
+   agx_foreach_block(ctx, block) {
+      if ((int)block->index < last_index) {
+         fprintf(stderr, "Out-of-order block index %d vs %d after %s\n",
+                 block->index, last_index, after);
+         agx_print_block(block, stderr);
+         fail = true;
+      }
+
+      last_index = block->index;
+
+      if (!agx_validate_block_form(block)) {
+         fprintf(stderr, "Invalid block form after %s\n", after);
+         agx_print_block(block, stderr);
+         fail = true;
+      }
+
+      if (!agx_validate_predecessors(block)) {
+         fprintf(stderr, "Invalid loop header flag after %s\n", after);
+         agx_print_block(block, stderr);
+         fail = true;
+      }
+   }
+
+   {
+      BITSET_WORD *defs = BITSET_CALLOC(ctx->alloc);
+
+      agx_foreach_instr_global(ctx, I) {
+         if (!agx_validate_defs(I, defs)) {
+            fprintf(stderr, "Invalid defs after %s\n", after);
+            agx_print_instr(I, stderr);
+            fail = true;
+         }
+      }
+
+      /* agx_validate_defs skips phi sources, so validate them now */
+      agx_foreach_block(ctx, block) {
+         agx_foreach_phi_in_block(block, phi) {
+            agx_foreach_ssa_src(phi, s) {
+               if (!BITSET_TEST(defs, phi->src[s].value)) {
+                  fprintf(stderr, "Undefined phi source %u after %s\n",
+                          phi->src[s].value, after);
+                  agx_print_instr(phi, stderr);
+                  fail = true;
+               }
+            }
+         }
+      }
+
+      free(defs);
+   }
+
+   agx_foreach_instr_global(ctx, I) {
+      if (!agx_validate_sources(I)) {
+         fprintf(stderr, "Invalid sources form after %s\n", after);
+         agx_print_instr(I, stderr);
+         fail = true;
+      }
+
+      if (!agx_validate_sr(I)) {
+         fprintf(stderr, "Invalid SR after %s\n", after);
+         agx_print_instr(I, stdout);
+         fail = true;
+      }
+   }
+
+   if (!agx_validate_width(ctx)) {
+      fprintf(stderr, "Invalid vectors after %s\n", after);
+      fail = true;
+   }
+
+   if (fail) {
+      agx_print_shader(ctx, stderr);
+      exit(1);
+   }
+}
+
+#endif /* NDEBUG */

src/asahi/compiler/agx_validate_ra.c

@@ -0,0 +1,304 @@
+/*
+ * Copyright 2024 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "agx_compiler.h"
+#include "agx_opcodes.h"
+
+/* Validatation doesn't make sense in release builds */
+#ifndef NDEBUG
+
+/* Represents a single 16-bit slice of an SSA var */
+struct var_offset {
+   uint32_t var;
+   uint8_t offset;
+   uint8_t defined;
+   uint8_t pad[2];
+};
+static_assert(sizeof(struct var_offset) == 8);
+
+static struct var_offset
+var_index(agx_index idx, uint32_t offset)
+{
+   assert(idx.type == AGX_INDEX_NORMAL);
+
+   return (struct var_offset){
+      .var = idx.value,
+      .offset = offset,
+      .defined = true,
+   };
+}
+
+static struct var_offset
+var_undef()
+{
+   return (struct var_offset){.defined = false};
+}
+
+static bool
+vars_equal(struct var_offset x, struct var_offset y)
+{
+   return x.defined && y.defined && x.var == y.var && x.offset == y.offset;
+}
+
+/* Represents the contents of a single register file */
+struct regfile {
+   struct var_offset r[RA_CLASSES][AGX_NUM_MODELED_REGS];
+};
+
+static void
+print_regfile(struct regfile *file, FILE *fp)
+{
+   fprintf(fp, "regfile: \n");
+   for (enum ra_class cls = 0; cls < RA_CLASSES; ++cls) {
+      for (unsigned r = 0; r < AGX_NUM_MODELED_REGS; ++r) {
+         struct var_offset v = file->r[cls][r];
+
+         if (v.defined) {
+            fprintf(fp, "   %c%u = %u[%u]\n", cls == RA_MEM ? 'm' : 'h', r,
+                    v.var, v.offset);
+         }
+      }
+   }
+   fprintf(fp, "\n");
+}
+
+#define agx_validate_assert(file, I, s, offs, stmt)                            \
+   if (!(stmt)) {                                                              \
+      fprintf(stderr, "failed to validate RA source %u offs %u: " #stmt "\n",  \
+              s, offs);                                                        \
+      agx_print_instr(I, stderr);                                              \
+      print_regfile(file, stderr);                                             \
+      return false;                                                            \
+   }
+
+static void
+copy_reg(struct regfile *file, agx_index dst, agx_index src)
+{
+   assert(dst.type == AGX_INDEX_REGISTER);
+   assert(src.type == AGX_INDEX_REGISTER);
+
+   enum ra_class dst_cls = ra_class_for_index(dst);
+   enum ra_class src_cls = ra_class_for_index(src);
+
+   for (uint8_t offs = 0; offs < agx_index_size_16(dst); ++offs) {
+      assert(dst.value + offs < ARRAY_SIZE(file->r[dst_cls]));
+      file->r[dst_cls][dst.value + offs] = file->r[src_cls][src.value + offs];
+   }
+}
+
+static void
+swap_regs(struct regfile *file, agx_index a, agx_index b)
+{
+   assert(a.type == AGX_INDEX_REGISTER);
+   assert(b.type == AGX_INDEX_REGISTER);
+
+   enum ra_class a_cls = ra_class_for_index(a);
+   enum ra_class b_cls = ra_class_for_index(b);
+
+   unsigned size = agx_index_size_16(a);
+   assert(size == agx_index_size_16(b));
+
+   for (uint8_t offs = 0; offs < size; ++offs) {
+      assert(a.value + offs < ARRAY_SIZE(file->r[a_cls]));
+      assert(b.value + offs < ARRAY_SIZE(file->r[b_cls]));
+
+      struct var_offset tmp = file->r[a_cls][a.value + offs];
+      file->r[a_cls][a.value + offs] = file->r[b_cls][b.value + offs];
+      file->r[b_cls][b.value + offs] = tmp;
+   }
+}
+
+static void
+record_dest(struct regfile *file, agx_index idx)
+{
+   assert(idx.type == AGX_INDEX_NORMAL && idx.has_reg);
+   enum ra_class cls = ra_class_for_index(idx);
+
+   for (uint8_t offs = 0; offs < agx_index_size_16(idx); ++offs) {
+      assert(idx.reg + offs < ARRAY_SIZE(file->r[cls]));
+      file->r[cls][idx.reg + offs] = var_index(idx, offs);
+   }
+}
+
+static bool
+validate_src(agx_instr *I, unsigned s, struct regfile *file, agx_index idx)
+{
+   assert(idx.type == AGX_INDEX_NORMAL && idx.has_reg);
+   enum ra_class cls = ra_class_for_index(idx);
+
+   for (uint8_t offs = 0; offs < agx_index_size_16(idx); ++offs) {
+      assert(idx.reg + offs < ARRAY_SIZE(file->r[cls]));
+      struct var_offset actual = file->r[cls][idx.reg + offs];
+
+      agx_validate_assert(file, I, s, offs, actual.defined);
+      agx_validate_assert(file, I, s, offs, actual.var == idx.value);
+      agx_validate_assert(file, I, s, offs, actual.offset == offs);
+   }
+
+   return true;
+}
+
+static bool
+validate_block(agx_context *ctx, agx_block *block, struct regfile *blocks)
+{
+   struct regfile *file = &blocks[block->index];
+   bool success = true;
+
+   /* Pathological shaders can end up with loop headers that have only a single
+    * predecessor and act like normal blocks. Validate them as such, since RA
+    * treats them as such implicitly. Affects:
+    *
+    * dEQP-VK.graphicsfuzz.spv-stable-mergesort-dead-code
+    */
+   bool loop_header = block->loop_header && agx_num_predecessors(block) > 1;
+
+   /* Initialize the register file based on predecessors. This only works in
+    * non-loop headers, since loop headers have unprocessed predecessors.
+    * However, loop headers phi-declare everything instead of using implicit
+    * live-in sources, so that's ok.
+    */
+   if (!loop_header) {
+      bool first_pred = true;
+      agx_foreach_predecessor(block, pred) {
+         struct regfile *pred_file = &blocks[(*pred)->index];
+
+         for (enum ra_class cls = 0; cls < RA_CLASSES; ++cls) {
+            for (unsigned r = 0; r < AGX_NUM_MODELED_REGS; ++r) {
+               if (first_pred)
+                  file->r[cls][r] = pred_file->r[cls][r];
+               else if (!vars_equal(file->r[cls][r], pred_file->r[cls][r]))
+                  file->r[cls][r] = var_undef();
+            }
+         }
+
+         first_pred = false;
+      }
+   }
+
+   agx_foreach_instr_in_block(block, I) {
+      /* Phis are special since they happen along the edge */
+      if (I->op != AGX_OPCODE_PHI) {
+         agx_foreach_ssa_src(I, s) {
+            success &= validate_src(I, s, file, I->src[s]);
+         }
+      }
+
+      agx_foreach_ssa_dest(I, d) {
+         record_dest(file, I->dest[d]);
+      }
+
+      /* Lowered live range splits don't have SSA associated, handle
+       * directly at the register level.
+       */
+      if (I->op == AGX_OPCODE_MOV && I->dest[0].type == AGX_INDEX_REGISTER &&
+          I->src[0].type == AGX_INDEX_REGISTER) {
+
+         copy_reg(file, I->dest[0], I->src[0]);
+      } else if (I->op == AGX_OPCODE_SWAP) {
+         swap_regs(file, I->src[0], I->src[1]);
+      } else if (I->op == AGX_OPCODE_PHI &&
+                 I->dest[0].type == AGX_INDEX_REGISTER) {
+         /* Register-only phis which resolve to the same variable in all blocks.
+          * This is generated for edge case live range splits.
+          */
+         assert(!I->dest[0].memory);
+         assert(!loop_header);
+         for (uint8_t offs = 0; offs < agx_index_size_16(I->dest[0]); ++offs) {
+            bool all_same = true;
+            bool first = true;
+            struct var_offset same = var_undef();
+            agx_foreach_predecessor(block, pred) {
+               unsigned idx = agx_predecessor_index(block, *pred);
+               agx_index src = I->src[idx];
+
+               assert(!src.memory);
+               if (src.type != AGX_INDEX_REGISTER) {
+                  all_same = false;
+                  first = false;
+                  continue;
+               }
+
+               struct regfile *pred_file = &blocks[(*pred)->index];
+               struct var_offset var = pred_file->r[RA_GPR][src.value + offs];
+               all_same &= first || vars_equal(var, same);
+               same = var;
+               first = false;
+            }
+
+            if (all_same) {
+               file->r[RA_GPR][I->dest[0].value + offs] = same;
+            }
+         }
+      } else if (I->op == AGX_OPCODE_PHI && I->dest[0].has_reg &&
+                 !loop_header) {
+         /* Phis which resolve to the same variable in all blocks.
+          * This is generated for live range splits.
+          */
+         enum ra_class cls = ra_class_for_index(I->dest[0]);
+         for (uint8_t offs = 0; offs < agx_index_size_16(I->dest[0]); ++offs) {
+            bool all_same = true;
+            bool first = true;
+            struct var_offset same = var_undef();
+            agx_foreach_predecessor(block, pred) {
+               unsigned idx = agx_predecessor_index(block, *pred);
+               agx_index src = I->src[idx];
+
+               assert(ra_class_for_index(src) == cls);
+               if (!src.has_reg) {
+                  all_same = false;
+                  first = false;
+                  continue;
+               }
+
+               struct regfile *pred_file = &blocks[(*pred)->index];
+               struct var_offset var = pred_file->r[cls][src.reg + offs];
+               all_same &= first || vars_equal(var, same);
+               same = var;
+               first = false;
+            }
+
+            if (all_same && I->dest[0].value == I->src[0].value) {
+               file->r[cls][I->dest[0].reg + offs] = same;
+            }
+         }
+      }
+   }
+
+   /* After processing a block, process the block's source in its successors'
+    * phis. These happen on the edge so we have all the information here, even
+    * with backedges.
+    */
+   agx_foreach_successor(block, succ) {
+      unsigned idx = agx_predecessor_index(succ, block);
+
+      agx_foreach_phi_in_block(succ, phi) {
+         if (phi->src[idx].type == AGX_INDEX_NORMAL) {
+            success &= validate_src(phi, idx, file, phi->src[idx]);
+         }
+      }
+   }
+
+   return success;
+}
+
+void
+agx_validate_ra(agx_context *ctx)
+{
+   bool succ = true;
+   struct regfile *blocks = calloc(ctx->num_blocks, sizeof(*blocks));
+
+   agx_foreach_block(ctx, block) {
+      succ &= validate_block(ctx, block, blocks);
+   }
+
+   if (!succ) {
+      agx_print_shader(ctx, stderr);
+      UNREACHABLE("invalid RA");
+   }
+
+   free(blocks);
+}
+
+#endif /* NDEBUG */

src/asahi/compiler/meson.build

@@ -0,0 +1,124 @@
+# Copyright 2018 Rob Clark
+# Copyright 2019 Collabora, Ltd.
+# SPDX-License-Identifier: MIT
+
+libasahi_agx_files = files(
+  'agx_compile.c',
+  'agx_dce.c',
+  'agx_liveness.c',
+  'agx_insert_waits.c',
+  'agx_nir_lower_address.c',
+  'agx_nir_lower_cull_distance.c',
+  'agx_nir_lower_fminmax.c',
+  'agx_nir_lower_frag_sidefx.c',
+  'agx_nir_lower_sample_mask.c',
+  'agx_nir_lower_discard_zs_emit.c',
+  'agx_nir_lower_interpolation.c',
+  'agx_nir_lower_shared_bitsize.c',
+  'agx_nir_lower_subgroups.c',
+  'agx_nir_lower_texture.c',
+  'agx_nir_opt_preamble.c',
+  'agx_lower_64bit.c',
+  'agx_lower_divergent_shuffle.c',
+  'agx_lower_parallel_copy.c',
+  'agx_lower_pseudo.c',
+  'agx_lower_spill.c',
+  'agx_lower_uniform_sources.c',
+  'agx_pack.c',
+  'agx_performance.c',
+  'agx_print.c',
+  'agx_pressure_schedule.c',
+  'agx_ir.c',
+  'agx_opt_cse.c',
+  'agx_opt_break_if.c',
+  'agx_opt_empty_else.c',
+  'agx_opt_jmp_none.c',
+  'agx_opt_compact_constants.c',
+  'agx_opt_promote_constants.c',
+  'agx_opt_register_cache.c',
+  'agx_optimizer.c',
+  'agx_repair_ssa.c',
+  'agx_reindex_ssa.c',
+  'agx_spill.c',
+  'agx_register_allocate.c',
+  'agx_validate.c',
+  'agx_validate_ra.c',
+)
+
+agx_nir_algebraic_c = custom_target(
+  input : 'agx_nir_algebraic.py',
+  output : 'agx_nir_algebraic.c',
+  command : [
+    prog_python, '@INPUT@', '-p', dir_compiler_nir,
+  ],
+  capture : true,
+  depend_files : nir_algebraic_depends,
+)
+
+agx_opcodes_h = custom_target(
+  input : ['agx_opcodes.h.py'],
+  output : 'agx_opcodes.h',
+  command : [prog_python, '@INPUT@'],
+  capture : true,
+  depend_files : files('agx_opcodes.py'),
+)
+
+idep_agx_opcodes_h = declare_dependency(
+  sources : [agx_opcodes_h],
+  include_directories : include_directories('.'),
+)
+
+agx_opcodes_c = custom_target(
+  input : ['agx_opcodes.c.py'],
+  output : 'agx_opcodes.c',
+  command : [prog_python, '@INPUT@'],
+  capture : true,
+  depend_files : files('agx_opcodes.py'),
+)
+
+agx_builder_h = custom_target(
+  input : ['agx_builder.h.py'],
+  output : 'agx_builder.h',
+  command : [prog_python, '@INPUT@'],
+  capture : true,
+  depend_files : files('agx_opcodes.py'),
+)
+
+idep_agx_builder_h = declare_dependency(
+  sources : [agx_builder_h],
+  include_directories : include_directories('.'),
+)
+
+libasahi_compiler = static_library(
+  'asahi_compiler',
+  [libasahi_agx_files, agx_opcodes_c, agx_nir_algebraic_c],
+  dependencies: [idep_nir, idep_agx_opcodes_h, idep_agx_builder_h, idep_mesautil, idep_libagx],
+  link_with : [libagx2_disasm],
+  c_args : [no_override_init_args, '-Wno-c2x-extensions'],
+  gnu_symbol_visibility : 'hidden',
+  build_by_default : false,
+)
+
+if with_tests
+  test(
+    'agx_tests',
+    executable(
+      'agx_tests',
+      files(
+         'test/test-compact-constants.cpp',
+         'test/test-minifloat.cpp',
+         'test/test-optimizer.cpp',
+         'test/test-lower-pseudo.cpp',
+         'test/test-lower-parallel-copy.cpp',
+         'test/test-lower-spill.cpp',
+         'test/test-repair-ssa.cpp',
+      ),
+      c_args : [c_msvc_compat_args, no_override_init_args],
+      gnu_symbol_visibility : 'hidden',
+      dependencies: [idep_gtest, idep_nir, idep_agx_opcodes_h, idep_agx_builder_h, idep_mesautil],
+      link_with : [libasahi_compiler],
+    ),
+    suite : ['asahi'],
+    protocol : 'gtest',
+  )
+endif

src/asahi/compiler/test/agx_test.h

@@ -0,0 +1,130 @@
+/*
+ * Copyright 2020-2021 Collabora Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef __AGX_TEST_H
+#define __AGX_TEST_H
+
+#include <inttypes.h>
+#include <stdio.h>
+#include "agx_builder.h"
+#include "agx_compiler.h"
+
+/* Helper to generate a agx_builder suitable for creating test instructions */
+static inline agx_builder *
+agx_test_builder(void *memctx)
+{
+   agx_context *ctx = rzalloc(memctx, agx_context);
+   list_inithead(&ctx->blocks);
+
+   agx_block *blk = rzalloc(ctx, agx_block);
+   blk->predecessors = UTIL_DYNARRAY_INIT;
+   ctx->num_blocks = 1;
+
+   list_addtail(&blk->link, &ctx->blocks);
+   list_inithead(&blk->instructions);
+
+   agx_builder *b = rzalloc(memctx, agx_builder);
+   b->shader = ctx;
+   b->cursor = agx_after_block(blk);
+
+   return b;
+}
+
+static inline agx_block *
+agx_test_block(agx_context *ctx)
+{
+   agx_block *blk = rzalloc(ctx, agx_block);
+
+   util_dynarray_init(&blk->predecessors, blk);
+   list_addtail(&blk->link, &ctx->blocks);
+   list_inithead(&blk->instructions);
+
+   blk->index = ctx->num_blocks++;
+
+   return blk;
+}
+
+/* Helper to compare for logical equality of instructions. Need to compare the
+ * pointers, then compare raw data.
+ */
+static inline bool
+agx_instr_equal(agx_instr *A, agx_instr *B)
+{
+   unsigned pointers = sizeof(struct list_head) + sizeof(agx_index *) * 2;
+
+   if (A->nr_srcs != B->nr_srcs)
+      return false;
+
+   if (memcmp(A->src, B->src, A->nr_srcs * sizeof(agx_index)))
+      return false;
+
+   if (A->nr_dests != B->nr_dests)
+      return false;
+
+   if (memcmp(A->dest, B->dest, A->nr_dests * sizeof(agx_index)))
+      return false;
+
+   return memcmp((uint8_t *)A + pointers, (uint8_t *)B + pointers,
+                 sizeof(agx_instr) - pointers) == 0;
+}
+
+static inline bool
+agx_block_equal(agx_block *A, agx_block *B)
+{
+   if (list_length(&A->instructions) != list_length(&B->instructions))
+      return false;
+
+   list_pair_for_each_entry(agx_instr, insA, insB, &A->instructions,
+                            &B->instructions, link) {
+      if (!agx_instr_equal(insA, insB))
+         return false;
+   }
+
+   return true;
+}
+
+static inline bool
+agx_shader_equal(agx_context *A, agx_context *B)
+{
+   if (list_length(&A->blocks) != list_length(&B->blocks))
+      return false;
+
+   list_pair_for_each_entry(agx_block, blockA, blockB, &A->blocks, &B->blocks,
+                            link) {
+      if (!agx_block_equal(blockA, blockB))
+         return false;
+   }
+
+   return true;
+}
+
+#define ASSERT_SHADER_EQUAL(A, B)                                              \
+   if (!agx_shader_equal(A, B)) {                                              \
+      ADD_FAILURE();                                                           \
+      fprintf(stderr, "Pass produced unexpected results");                     \
+      fprintf(stderr, "  Actual:\n");                                          \
+      agx_print_shader(A, stderr);                                             \
+      fprintf(stderr, " Expected:\n");                                         \
+      agx_print_shader(B, stderr);                                             \
+      fprintf(stderr, "\n");                                                   \
+   }
+
+#define INSTRUCTION_CASE(instr, expected, pass)                                \
+   do {                                                                        \
+      agx_builder *A = agx_test_builder(mem_ctx);                              \
+      agx_builder *B = agx_test_builder(mem_ctx);                              \
+      {                                                                        \
+         agx_builder *b = A;                                                   \
+         instr;                                                                \
+      }                                                                        \
+      {                                                                        \
+         agx_builder *b = B;                                                   \
+         expected;                                                             \
+      }                                                                        \
+      pass(A->shader);                                                         \
+      ASSERT_SHADER_EQUAL(A->shader, B->shader);                               \
+   } while (0)
+
+#endif

src/asahi/compiler/test/test-compact-constants.cpp

@@ -0,0 +1,57 @@
+/*
+ * Copyright 2021 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "agx_builder.h"
+#include "agx_compiler.h"
+#include "agx_test.h"
+
+#include <gtest/gtest.h>
+
+#define CASE(instr, expected, size)                                            \
+   INSTRUCTION_CASE(                                                           \
+      {                                                                        \
+         UNUSED agx_index out = agx_temp(b->shader, AGX_SIZE_##size);          \
+         instr;                                                                \
+         agx_unit_test(b, out);                                                \
+      },                                                                       \
+      {                                                                        \
+         UNUSED agx_index out = agx_temp(b->shader, AGX_SIZE_##size);          \
+         expected;                                                             \
+         agx_unit_test(b, out);                                                \
+      },                                                                       \
+      agx_opt_compact_constants)
+
+#define NEGCASE(instr, size)    CASE(instr, instr, size)
+#define CASE32(instr, expected) CASE(instr, expected, 32)
+#define NEGCASE32(instr)        NEGCASE(instr, 32)
+
+class CompactConstants : public testing::Test {
+ protected:
+   CompactConstants()
+   {
+      mem_ctx = ralloc_context(NULL);
+
+      wx = agx_register(0, AGX_SIZE_32);
+   }
+
+   ~CompactConstants()
+   {
+      ralloc_free(mem_ctx);
+   }
+
+   void *mem_ctx;
+   agx_index wx;
+};
+
+TEST_F(CompactConstants, FP32)
+{
+   CASE32(agx_fadd_to(b, out, wx, agx_mov_imm(b, 32, fui(32768.0))),
+          agx_fadd_to(b, out, wx, agx_mov_imm(b, 16, 0x7800)));
+}
+
+TEST_F(CompactConstants, InexactFP32)
+{
+   NEGCASE32(agx_fadd_to(b, out, wx, agx_mov_imm(b, 32, fui(32769.0))));
+}

src/asahi/compiler/test/test-lower-parallel-copy.cpp

@@ -0,0 +1,302 @@
+/*
+ * Copyright 2021 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "agx_builder.h"
+#include "agx_compiler.h"
+#include "agx_test.h"
+
+#include <gtest/gtest.h>
+
+#define CASE(copies, expected)                                                 \
+   do {                                                                        \
+      agx_builder *A = agx_test_builder(mem_ctx);                              \
+      agx_builder *B = agx_test_builder(mem_ctx);                              \
+                                                                               \
+      agx_emit_parallel_copies(A, copies, ARRAY_SIZE(copies));                 \
+                                                                               \
+      {                                                                        \
+         agx_builder *b = B;                                                   \
+         expected;                                                             \
+      }                                                                        \
+                                                                               \
+      ASSERT_SHADER_EQUAL(A->shader, B->shader);                               \
+   } while (0)
+
+class LowerParallelCopy : public testing::Test {
+ protected:
+   LowerParallelCopy()
+   {
+      mem_ctx = ralloc_context(NULL);
+   }
+
+   ~LowerParallelCopy()
+   {
+      ralloc_free(mem_ctx);
+   }
+
+   void *mem_ctx;
+};
+
+TEST_F(LowerParallelCopy, UnrelatedCopies)
+{
+   struct agx_copy test_1[] = {
+      {.dest = 0, .src = agx_register(2, AGX_SIZE_32)},
+      {.dest = 4, .src = agx_register(6, AGX_SIZE_32)},
+   };
+
+   CASE(test_1, {
+      agx_mov_to(b, agx_register(0, AGX_SIZE_32), agx_register(2, AGX_SIZE_32));
+      agx_mov_to(b, agx_register(4, AGX_SIZE_32), agx_register(6, AGX_SIZE_32));
+   });
+
+   struct agx_copy test_2[] = {
+      {.dest = 0, .src = agx_register(1, AGX_SIZE_16)},
+      {.dest = 4, .src = agx_register(5, AGX_SIZE_16)},
+   };
+
+   CASE(test_2, {
+      agx_mov_to(b, agx_register(0, AGX_SIZE_16), agx_register(1, AGX_SIZE_16));
+      agx_mov_to(b, agx_register(4, AGX_SIZE_16), agx_register(5, AGX_SIZE_16));
+   });
+}
+
+TEST_F(LowerParallelCopy, RelatedSource)
+{
+   struct agx_copy test_1[] = {
+      {.dest = 0, .src = agx_register(2, AGX_SIZE_32)},
+      {.dest = 4, .src = agx_register(2, AGX_SIZE_32)},
+   };
+
+   CASE(test_1, {
+      agx_mov_to(b, agx_register(0, AGX_SIZE_32), agx_register(2, AGX_SIZE_32));
+      agx_mov_to(b, agx_register(4, AGX_SIZE_32), agx_register(2, AGX_SIZE_32));
+   });
+
+   struct agx_copy test_2[] = {
+      {.dest = 0, .src = agx_register(1, AGX_SIZE_16)},
+      {.dest = 4, .src = agx_register(1, AGX_SIZE_16)},
+   };
+
+   CASE(test_2, {
+      agx_mov_to(b, agx_register(0, AGX_SIZE_16), agx_register(1, AGX_SIZE_16));
+      agx_mov_to(b, agx_register(4, AGX_SIZE_16), agx_register(1, AGX_SIZE_16));
+   });
+}
+
+TEST_F(LowerParallelCopy, DependentCopies)
+{
+   struct agx_copy test_1[] = {
+      {.dest = 0, .src = agx_register(2, AGX_SIZE_32)},
+      {.dest = 4, .src = agx_register(0, AGX_SIZE_32)},
+   };
+
+   CASE(test_1, {
+      agx_mov_to(b, agx_register(4, AGX_SIZE_32), agx_register(0, AGX_SIZE_32));
+      agx_mov_to(b, agx_register(0, AGX_SIZE_32), agx_register(2, AGX_SIZE_32));
+   });
+
+   struct agx_copy test_2[] = {
+      {.dest = 0, .src = agx_register(1, AGX_SIZE_16)},
+      {.dest = 4, .src = agx_register(0, AGX_SIZE_16)},
+   };
+
+   CASE(test_2, {
+      agx_mov_to(b, agx_register(4, AGX_SIZE_16), agx_register(0, AGX_SIZE_16));
+      agx_mov_to(b, agx_register(0, AGX_SIZE_16), agx_register(1, AGX_SIZE_16));
+   });
+}
+
+TEST_F(LowerParallelCopy, ManyDependentCopies)
+{
+   struct agx_copy test_1[] = {
+      {.dest = 0, .src = agx_register(2, AGX_SIZE_32)},
+      {.dest = 4, .src = agx_register(0, AGX_SIZE_32)},
+      {.dest = 8, .src = agx_register(6, AGX_SIZE_32)},
+      {.dest = 6, .src = agx_register(4, AGX_SIZE_32)},
+   };
+
+   CASE(test_1, {
+      agx_mov_to(b, agx_register(8, AGX_SIZE_32), agx_register(6, AGX_SIZE_32));
+      agx_mov_to(b, agx_register(6, AGX_SIZE_32), agx_register(4, AGX_SIZE_32));
+      agx_mov_to(b, agx_register(4, AGX_SIZE_32), agx_register(0, AGX_SIZE_32));
+      agx_mov_to(b, agx_register(0, AGX_SIZE_32), agx_register(2, AGX_SIZE_32));
+   });
+
+   struct agx_copy test_2[] = {
+      {.dest = 0, .src = agx_register(1, AGX_SIZE_16)},
+      {.dest = 2, .src = agx_register(0, AGX_SIZE_16)},
+      {.dest = 4, .src = agx_register(3, AGX_SIZE_16)},
+      {.dest = 3, .src = agx_register(2, AGX_SIZE_16)},
+   };
+
+   CASE(test_2, {
+      agx_mov_to(b, agx_register(4, AGX_SIZE_16), agx_register(3, AGX_SIZE_16));
+      agx_mov_to(b, agx_register(3, AGX_SIZE_16), agx_register(2, AGX_SIZE_16));
+      agx_mov_to(b, agx_register(2, AGX_SIZE_16), agx_register(0, AGX_SIZE_16));
+      agx_mov_to(b, agx_register(0, AGX_SIZE_16), agx_register(1, AGX_SIZE_16));
+   });
+}
+
+TEST_F(LowerParallelCopy, Swap)
+{
+   struct agx_copy test_1[] = {
+      {.dest = 0, .src = agx_register(2, AGX_SIZE_32)},
+      {.dest = 2, .src = agx_register(0, AGX_SIZE_32)},
+   };
+
+   CASE(test_1, {
+      agx_swap(b, agx_register(0, AGX_SIZE_32), agx_register(2, AGX_SIZE_32));
+   });
+
+   struct agx_copy test_2[] = {
+      {.dest = 0, .src = agx_register(1, AGX_SIZE_16)},
+      {.dest = 1, .src = agx_register(0, AGX_SIZE_16)},
+   };
+
+   CASE(test_2, {
+      agx_swap(b, agx_register(0, AGX_SIZE_16), agx_register(1, AGX_SIZE_16));
+   });
+}
+
+TEST_F(LowerParallelCopy, Cycle3)
+{
+   struct agx_copy test[] = {
+      {.dest = 0, .src = agx_register(1, AGX_SIZE_16)},
+      {.dest = 1, .src = agx_register(2, AGX_SIZE_16)},
+      {.dest = 2, .src = agx_register(0, AGX_SIZE_16)},
+   };
+
+   CASE(test, {
+      agx_swap(b, agx_register(0, AGX_SIZE_16), agx_register(1, AGX_SIZE_16));
+      agx_swap(b, agx_register(1, AGX_SIZE_16), agx_register(2, AGX_SIZE_16));
+   });
+}
+
+TEST_F(LowerParallelCopy, Immediate64)
+{
+   agx_index imm = agx_immediate(10);
+   imm.size = AGX_SIZE_64;
+
+   struct agx_copy test_1[] = {
+      {.dest = 4, .src = imm},
+   };
+
+   CASE(test_1, {
+      agx_mov_imm_to(b, agx_register(4, AGX_SIZE_32), 10);
+      agx_mov_imm_to(b, agx_register(6, AGX_SIZE_32), 0);
+   });
+}
+
+/* Test case from Hack et al */
+TEST_F(LowerParallelCopy, TwoSwaps)
+{
+   struct agx_copy test[] = {
+      {.dest = 4, .src = agx_register(2, AGX_SIZE_32)},
+      {.dest = 6, .src = agx_register(4, AGX_SIZE_32)},
+      {.dest = 2, .src = agx_register(6, AGX_SIZE_32)},
+      {.dest = 8, .src = agx_register(8, AGX_SIZE_32)},
+   };
+
+   CASE(test, {
+      agx_swap(b, agx_register(4, AGX_SIZE_32), agx_register(2, AGX_SIZE_32));
+      agx_swap(b, agx_register(6, AGX_SIZE_32), agx_register(2, AGX_SIZE_32));
+   });
+}
+
+TEST_F(LowerParallelCopy, VectorizeAlignedHalfRegs)
+{
+   struct agx_copy test[] = {
+      {.dest = 0, .src = agx_register(10, AGX_SIZE_16)},
+      {.dest = 1, .src = agx_register(11, AGX_SIZE_16)},
+      {.dest = 2, .src = agx_uniform(8, AGX_SIZE_16)},
+      {.dest = 3, .src = agx_uniform(9, AGX_SIZE_16)},
+   };
+
+   CASE(test, {
+      agx_mov_to(b, agx_register(0, AGX_SIZE_32),
+                 agx_register(10, AGX_SIZE_32));
+      agx_mov_to(b, agx_register(2, AGX_SIZE_32), agx_uniform(8, AGX_SIZE_32));
+   });
+}
+
+TEST_F(LowerParallelCopy, StackCopies)
+{
+   struct agx_copy test[] = {
+      {.dest = 21, .dest_mem = true, .src = agx_register(20, AGX_SIZE_16)},
+      {.dest = 22, .dest_mem = true, .src = agx_register(22, AGX_SIZE_32)},
+      {.dest = 0, .src = agx_memory_register(10, AGX_SIZE_16)},
+      {.dest = 1, .src = agx_memory_register(11, AGX_SIZE_16)},
+      {.dest = 0, .dest_mem = true, .src = agx_memory_register(12, AGX_SIZE_16)},
+      {.dest = 1, .dest_mem = true, .src = agx_memory_register(13, AGX_SIZE_16)},
+      {.dest = 2,
+       .dest_mem = true,
+       .src = agx_memory_register(804, AGX_SIZE_32)},
+      {.dest = 804,
+       .dest_mem = true,
+       .src = agx_memory_register(2, AGX_SIZE_32)},
+      {.dest = 807,
+       .dest_mem = true,
+       .src = agx_memory_register(808, AGX_SIZE_16)},
+      {.dest = 808,
+       .dest_mem = true,
+       .src = agx_memory_register(807, AGX_SIZE_16)},
+   };
+
+   CASE(test, {
+      /* Vectorized fill */
+      agx_mov_to(b, agx_register(0, AGX_SIZE_32),
+                 agx_memory_register(10, AGX_SIZE_32));
+
+      /* Regular spills */
+      agx_mov_to(b, agx_memory_register(21, AGX_SIZE_16),
+                 agx_register(20, AGX_SIZE_16));
+      agx_mov_to(b, agx_memory_register(22, AGX_SIZE_32),
+                 agx_register(22, AGX_SIZE_32));
+
+      /* Vectorized stack->stack copy */
+      agx_mov_to(b, agx_register(2, AGX_SIZE_32),
+                 agx_memory_register(12, AGX_SIZE_32));
+
+      agx_mov_to(b, agx_memory_register(0, AGX_SIZE_32),
+                 agx_register(2, AGX_SIZE_32));
+
+      /* Stack swap: 32-bit */
+      agx_index temp1 = agx_register(4, AGX_SIZE_32);
+      agx_index temp2 = agx_register(6, AGX_SIZE_32);
+      agx_index spilled_gpr_vec2 = agx_register(0, AGX_SIZE_32);
+      spilled_gpr_vec2.channels_m1++;
+
+      agx_mov_to(b, temp1, agx_memory_register(2, AGX_SIZE_32));
+      agx_mov_to(b, temp2, agx_memory_register(804, AGX_SIZE_32));
+      agx_mov_to(b, agx_memory_register(804, AGX_SIZE_32), temp1);
+      agx_mov_to(b, agx_memory_register(2, AGX_SIZE_32), temp2);
+
+      /* Stack swap: 16-bit */
+      spilled_gpr_vec2.size = AGX_SIZE_16;
+      temp1.size = AGX_SIZE_16;
+      temp2.size = AGX_SIZE_16;
+
+      agx_mov_to(b, temp1, agx_memory_register(807, AGX_SIZE_16));
+      agx_mov_to(b, temp2, agx_memory_register(808, AGX_SIZE_16));
+      agx_mov_to(b, agx_memory_register(808, AGX_SIZE_16), temp1);
+      agx_mov_to(b, agx_memory_register(807, AGX_SIZE_16), temp2);
+   });
+}
+
+#if 0
+TEST_F(LowerParallelCopy, LooksLikeASwap) {
+   struct agx_copy test[] = {
+        { .dest = 0, .src = agx_register(2, AGX_SIZE_32) },
+        { .dest = 2, .src = agx_register(0, AGX_SIZE_32) },
+        { .dest = 4, .src = agx_register(2, AGX_SIZE_32) },
+   };
+
+   CASE(test, {
+         agx_mov_to(b, agx_register(4, AGX_SIZE_32), agx_register(2, AGX_SIZE_32));
+         agx_mov_to(b, agx_register(2, AGX_SIZE_32), agx_register(0, AGX_SIZE_32));
+         agx_mov_to(b, agx_register(0, AGX_SIZE_32), agx_register(4, AGX_SIZE_32));
+   });
+}
+#endif

src/asahi/compiler/test/test-lower-pseudo.cpp

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2021 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "agx_test.h"
+
+#include <gtest/gtest.h>
+
+#define CASE(instr, expected)                                                  \
+   INSTRUCTION_CASE(instr, expected, agx_lower_pseudo)
+#define NEGCASE(instr) CASE(instr, instr)
+
+class LowerPseudo : public testing::Test {
+ protected:
+   LowerPseudo()
+   {
+      mem_ctx = ralloc_context(NULL);
+
+      wx = agx_register(0, AGX_SIZE_32);
+      wy = agx_register(2, AGX_SIZE_32);
+      wz = agx_register(4, AGX_SIZE_32);
+   }
+
+   ~LowerPseudo()
+   {
+      ralloc_free(mem_ctx);
+   }
+
+   void *mem_ctx;
+   agx_index wx, wy, wz;
+};
+
+TEST_F(LowerPseudo, Move)
+{
+   CASE(agx_mov_to(b, wx, wy), agx_bitop_to(b, wx, wy, agx_zero(), 0xA));
+}
+
+TEST_F(LowerPseudo, Not)
+{
+   CASE(agx_not_to(b, wx, wy), agx_bitop_to(b, wx, wy, agx_zero(), 0x5));
+}

src/asahi/compiler/test/test-lower-spill.cpp

@@ -0,0 +1,164 @@
+/*
+ * Copyright 2023 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "agx_builder.h"
+#include "agx_compile.h"
+#include "agx_compiler.h"
+#include "agx_test.h"
+
+#include "util/macros.h"
+#include <gtest/gtest.h>
+
+#define CASE(instr, expected)                                                  \
+   do {                                                                        \
+      agx_builder *A = agx_test_builder(mem_ctx);                              \
+      agx_builder *B = agx_test_builder(mem_ctx);                              \
+      {                                                                        \
+         agx_builder *b = A;                                                   \
+         instr;                                                                \
+      }                                                                        \
+      {                                                                        \
+         agx_builder *b = B;                                                   \
+         expected;                                                             \
+      }                                                                        \
+      A->shader->scratch_size_B = 1024;                                        \
+      agx_lower_spill(A->shader);                                              \
+      ASSERT_SHADER_EQUAL(A->shader, B->shader);                               \
+   } while (0)
+
+class LowerSpill : public testing::Test {
+ protected:
+   LowerSpill()
+   {
+      mem_ctx = ralloc_context(NULL);
+
+      wx = agx_register(0, AGX_SIZE_32);
+      hy = agx_register(2, AGX_SIZE_16);
+
+      mw4 = agx_memory_register(0, AGX_SIZE_32);
+      mh4 = agx_memory_register(0, AGX_SIZE_16);
+      mw4.channels_m1 = 4 - 1;
+      mh4.channels_m1 = 4 - 1;
+
+      wx4 = wx;
+      wx4.channels_m1 = 4 - 1;
+
+      hy4 = hy;
+      hy4.channels_m1 = 4 - 1;
+   }
+
+   ~LowerSpill()
+   {
+      ralloc_free(mem_ctx);
+   }
+
+   void *mem_ctx;
+   agx_index wx, hy, wx4, hy4;
+   agx_index mw4, mh4;
+
+   unsigned scalar = BITFIELD_MASK(1);
+   unsigned vec4 = BITFIELD_MASK(4);
+
+   enum agx_format i16 = AGX_FORMAT_I16;
+   enum agx_format i32 = AGX_FORMAT_I32;
+};
+
+TEST_F(LowerSpill, ScalarSpills)
+{
+   CASE(agx_mov_to(b, agx_memory_register(11, AGX_SIZE_16), hy),
+        agx_stack_store(b, hy, agx_immediate(22), i16, scalar));
+
+   CASE(agx_mov_to(b, agx_memory_register(18, AGX_SIZE_32), wx),
+        agx_stack_store(b, wx, agx_immediate(36), i32, scalar));
+}
+
+TEST_F(LowerSpill, ScalarFills)
+{
+   CASE(agx_mov_to(b, hy, agx_memory_register(11, AGX_SIZE_16)),
+        agx_stack_load_to(b, hy, agx_immediate(22), i16, scalar));
+
+   CASE(agx_mov_to(b, wx, agx_memory_register(18, AGX_SIZE_32)),
+        agx_stack_load_to(b, wx, agx_immediate(36), i32, scalar));
+}
+
+TEST_F(LowerSpill, VectorSpills)
+{
+   CASE(agx_mov_to(b, mh4, hy4),
+        agx_stack_store(b, hy4, agx_immediate(0), i16, vec4));
+
+   CASE(agx_mov_to(b, mw4, wx4),
+        agx_stack_store(b, wx4, agx_immediate(0), i32, vec4));
+}
+
+TEST_F(LowerSpill, VectorFills)
+{
+   CASE(agx_mov_to(b, hy4, mh4),
+        agx_stack_load_to(b, hy4, agx_immediate(0), i16, vec4));
+
+   CASE(agx_mov_to(b, wx4, mw4),
+        agx_stack_load_to(b, wx4, agx_immediate(0), i32, vec4));
+}
+
+TEST_F(LowerSpill, ScalarSpill64)
+{
+   CASE(agx_mov_to(b, agx_memory_register(16, AGX_SIZE_64),
+                   agx_register(8, AGX_SIZE_64)),
+        agx_stack_store(b, agx_register(8, AGX_SIZE_64), agx_immediate(32), i32,
+                        BITFIELD_MASK(2)));
+}
+
+TEST_F(LowerSpill, ScalarFill64)
+{
+   CASE(agx_mov_to(b, agx_register(16, AGX_SIZE_64),
+                   agx_memory_register(8, AGX_SIZE_64)),
+        agx_stack_load_to(b, agx_register(16, AGX_SIZE_64), agx_immediate(16),
+                          i32, BITFIELD_MASK(2)));
+}
+
+TEST_F(LowerSpill, Vec6Spill)
+{
+   CASE(
+      {
+         agx_index mvec6 = agx_memory_register(16, AGX_SIZE_32);
+         agx_index vec6 = agx_register(8, AGX_SIZE_32);
+         vec6.channels_m1 = 6 - 1;
+         mvec6.channels_m1 = 6 - 1;
+
+         agx_mov_to(b, mvec6, vec6);
+      },
+      {
+         agx_index vec4 = agx_register(8, AGX_SIZE_32);
+         agx_index vec2 = agx_register(8 + (4 * 2), AGX_SIZE_32);
+         vec4.channels_m1 = 4 - 1;
+         vec2.channels_m1 = 2 - 1;
+
+         agx_stack_store(b, vec4, agx_immediate(32), i32, BITFIELD_MASK(4));
+         agx_stack_store(b, vec2, agx_immediate(32 + 4 * 4), i32,
+                         BITFIELD_MASK(2));
+      });
+}
+
+TEST_F(LowerSpill, Vec6Fill)
+{
+   CASE(
+      {
+         agx_index mvec6 = agx_memory_register(16, AGX_SIZE_32);
+         agx_index vec6 = agx_register(8, AGX_SIZE_32);
+         vec6.channels_m1 = 6 - 1;
+         mvec6.channels_m1 = 6 - 1;
+
+         agx_mov_to(b, vec6, mvec6);
+      },
+      {
+         agx_index vec4 = agx_register(8, AGX_SIZE_32);
+         agx_index vec2 = agx_register(8 + (4 * 2), AGX_SIZE_32);
+         vec4.channels_m1 = 4 - 1;
+         vec2.channels_m1 = 2 - 1;
+
+         agx_stack_load_to(b, vec4, agx_immediate(32), i32, BITFIELD_MASK(4));
+         agx_stack_load_to(b, vec2, agx_immediate(32 + 4 * 4), i32,
+                           BITFIELD_MASK(2));
+      });
+}

src/asahi/compiler/test/test-minifloat.cpp

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2021-2022 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "agx_test.h"
+
+#include <gtest/gtest.h>
+
+TEST(Minifloat, RepresentativeValues)
+{
+   EXPECT_EQ(agx_minifloat_decode(0), 0.0f);
+   EXPECT_EQ(agx_minifloat_decode(25), 0.390625f);
+   EXPECT_EQ(agx_minifloat_decode(135), -0.109375f);
+   EXPECT_EQ(agx_minifloat_decode(255), -31.0);
+}
+
+TEST(Minifloat, Exactness)
+{
+   EXPECT_TRUE(agx_minifloat_exact(0.0f));
+   EXPECT_TRUE(agx_minifloat_exact(0.390625f));
+   EXPECT_TRUE(agx_minifloat_exact(-0.109375f));
+   EXPECT_TRUE(agx_minifloat_exact(-31.0));
+   EXPECT_FALSE(agx_minifloat_exact(3.141f));
+   EXPECT_FALSE(agx_minifloat_exact(2.718f));
+   EXPECT_FALSE(agx_minifloat_exact(1.618f));
+}
+
+TEST(Minifloat, AllValuesRoundtrip)
+{
+   for (unsigned i = 0; i < 0x100; ++i) {
+      float f = agx_minifloat_decode(i);
+      EXPECT_EQ(agx_minifloat_encode(f), i);
+      EXPECT_TRUE(agx_minifloat_exact(f));
+   }
+}

src/asahi/compiler/test/test-optimizer.cpp

@@ -0,0 +1,490 @@
+/*
+ * Copyright 2021 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "agx_builder.h"
+#include "agx_compiler.h"
+#include "agx_test.h"
+
+#include <gtest/gtest.h>
+
+static void
+agx_optimize_and_dce(agx_context *ctx)
+{
+   agx_optimizer_backward(ctx);
+   agx_optimizer_forward(ctx);
+   agx_dce(ctx, true);
+}
+
+#define CASE(instr, expected, size, returns)                                   \
+   INSTRUCTION_CASE(                                                           \
+      {                                                                        \
+         UNUSED agx_index out = agx_temp(b->shader, AGX_SIZE_##size);          \
+         instr;                                                                \
+         if (returns)                                                          \
+            agx_unit_test(b, out);                                             \
+      },                                                                       \
+      {                                                                        \
+         UNUSED agx_index out = agx_temp(b->shader, AGX_SIZE_##size);          \
+         expected;                                                             \
+         if (returns)                                                          \
+            agx_unit_test(b, out);                                             \
+      },                                                                       \
+      agx_optimize_and_dce)
+
+#define NEGCASE(instr, size) CASE(instr, instr, size, true)
+
+#define CASE16(instr, expected) CASE(instr, expected, 16, true)
+#define CASE32(instr, expected) CASE(instr, expected, 32, true)
+#define CASE64(instr, expected) CASE(instr, expected, 64, true)
+
+#define CASE_NO_RETURN(instr, expected)                                        \
+   CASE(instr, expected, 32 /* irrelevant */, false)
+
+#define NEGCASE16(instr) NEGCASE(instr, 16)
+#define NEGCASE32(instr) NEGCASE(instr, 32)
+
+static inline agx_index
+agx_fmov(agx_builder *b, agx_index s0)
+{
+   agx_index tmp = agx_temp(b->shader, s0.size);
+   agx_fmov_to(b, tmp, s0);
+   return tmp;
+}
+
+class Optimizer : public testing::Test {
+ protected:
+   Optimizer()
+   {
+      mem_ctx = ralloc_context(NULL);
+
+      dx = agx_register(0, AGX_SIZE_64);
+      dz = agx_register(4, AGX_SIZE_64);
+
+      wx = agx_register(0, AGX_SIZE_32);
+      wy = agx_register(2, AGX_SIZE_32);
+      wz = agx_register(4, AGX_SIZE_32);
+
+      hx = agx_register(0, AGX_SIZE_16);
+      hy = agx_register(1, AGX_SIZE_16);
+      hz = agx_register(2, AGX_SIZE_16);
+   }
+
+   ~Optimizer()
+   {
+      ralloc_free(mem_ctx);
+   }
+
+   void *mem_ctx;
+
+   agx_index dx, dz, wx, wy, wz, hx, hy, hz;
+};
+
+TEST_F(Optimizer, FloatCopyprop)
+{
+   CASE32(agx_fadd_to(b, out, agx_abs(agx_fmov(b, wx)), wy),
+          agx_fadd_to(b, out, agx_abs(wx), wy));
+
+   CASE32(agx_fadd_to(b, out, agx_neg(agx_fmov(b, wx)), wy),
+          agx_fadd_to(b, out, agx_neg(wx), wy));
+}
+
+TEST_F(Optimizer, FloatConversion)
+{
+   CASE32(
+      {
+         agx_index cvt = agx_temp(b->shader, AGX_SIZE_32);
+         agx_fmov_to(b, cvt, hx);
+         agx_fadd_to(b, out, cvt, wy);
+      },
+      { agx_fadd_to(b, out, hx, wy); });
+
+   CASE16(
+      {
+         agx_index sum = agx_temp(b->shader, AGX_SIZE_32);
+         agx_fadd_to(b, sum, wx, wy);
+         agx_fmov_to(b, out, sum);
+      },
+      { agx_fadd_to(b, out, wx, wy); });
+}
+
+TEST_F(Optimizer, FusedFABSNEG)
+{
+   CASE32(agx_fadd_to(b, out, agx_fmov(b, agx_abs(wx)), wy),
+          agx_fadd_to(b, out, agx_abs(wx), wy));
+
+   CASE32(agx_fmul_to(b, out, wx, agx_fmov(b, agx_neg(agx_abs(wx)))),
+          agx_fmul_to(b, out, wx, agx_neg(agx_abs(wx))));
+}
+
+TEST_F(Optimizer, FusedFabsAbsorb)
+{
+   CASE32(agx_fadd_to(b, out, agx_abs(agx_fmov(b, agx_abs(wx))), wy),
+          agx_fadd_to(b, out, agx_abs(wx), wy));
+}
+
+TEST_F(Optimizer, FusedFnegCancel)
+{
+   CASE32(agx_fmul_to(b, out, wx, agx_neg(agx_fmov(b, agx_neg(wx)))),
+          agx_fmul_to(b, out, wx, wx));
+
+   CASE32(agx_fmul_to(b, out, wx, agx_neg(agx_fmov(b, agx_neg(agx_abs(wx))))),
+          agx_fmul_to(b, out, wx, agx_abs(wx)));
+}
+
+TEST_F(Optimizer, FusedNot)
+{
+   CASE32(agx_not_to(b, out, agx_and(b, wx, wx)), agx_nand_to(b, out, wx, wx));
+
+   CASE32(agx_not_to(b, out, agx_or(b, wx, wx)), agx_nor_to(b, out, wx, wx));
+
+   CASE32(agx_not_to(b, out, agx_xor(b, wx, wx)), agx_xnor_to(b, out, wx, wx));
+
+   CASE32(agx_xor_to(b, out, agx_not(b, wx), agx_not(b, wx)),
+          agx_xor_to(b, out, wx, wx));
+
+   CASE32(agx_xor_to(b, out, agx_not(b, wx), wx), agx_xnor_to(b, out, wx, wx));
+
+   CASE32(agx_xor_to(b, out, wx, agx_not(b, wx)), agx_xnor_to(b, out, wx, wx));
+
+   CASE32(agx_nand_to(b, out, agx_not(b, wx), agx_not(b, wx)),
+          agx_or_to(b, out, wx, wx));
+
+   CASE32(agx_andn1_to(b, out, agx_not(b, wx), wx), agx_and_to(b, out, wx, wx));
+
+   CASE32(agx_andn1_to(b, out, wx, agx_not(b, wx)), agx_nor_to(b, out, wx, wx));
+
+   CASE32(agx_andn2_to(b, out, agx_not(b, wx), wx), agx_nor_to(b, out, wx, wx));
+
+   CASE32(agx_andn2_to(b, out, wx, agx_not(b, wx)), agx_and_to(b, out, wx, wx));
+
+   CASE32(agx_xor_to(b, out, agx_not(b, wx), agx_uniform(8, AGX_SIZE_32)),
+          agx_xnor_to(b, out, wx, agx_uniform(8, AGX_SIZE_32)));
+
+   CASE32(agx_or_to(b, out, agx_immediate(123), agx_not(b, wx)),
+          agx_orn2_to(b, out, agx_immediate(123), wx));
+
+   CASE32(agx_xor_to(b, out, wx, agx_not(b, wy)), agx_xnor_to(b, out, wx, wy));
+
+   CASE32(agx_xor_to(b, out, wy, agx_not(b, wx)), agx_xnor_to(b, out, wy, wx));
+
+   CASE32(agx_and_to(b, out, agx_not(b, wx), wy), agx_andn1_to(b, out, wx, wy));
+
+   CASE32(agx_or_to(b, out, wx, agx_not(b, wy)), agx_orn2_to(b, out, wx, wy));
+}
+
+TEST_F(Optimizer, FmulFsatF2F16)
+{
+   CASE16(
+      {
+         agx_index tmp = agx_temp(b->shader, AGX_SIZE_32);
+         agx_fmov_to(b, tmp, agx_fmul(b, wx, wy))->saturate = true;
+         agx_fmov_to(b, out, tmp);
+      },
+      { agx_fmul_to(b, out, wx, wy)->saturate = true; });
+}
+
+TEST_F(Optimizer, FsatWithPhi)
+{
+   /*
+    * Construct the loop:
+    *
+    * A:
+    *   ...
+    *
+    * B:
+    *    phi ..., u
+    *    u = wx * phi
+    *    out = fsat u
+    *    --> B
+    *
+    * This example shows that phi sources are read at the end of the
+    * predecessor, not at the start of the successor. If phis are not handled
+    * properly, the fsat would be fused incorrectly.
+    *
+    * This reproduces an issue hit in a Control shader. Astonishingly, it is not
+    * hit anywhere in CTS.
+    */
+   NEGCASE32({
+      agx_block *A = agx_start_block(b->shader);
+      agx_block *B = agx_test_block(b->shader);
+
+      agx_block_add_successor(A, B);
+      agx_block_add_successor(B, B);
+
+      b->cursor = agx_after_block(B);
+      agx_index u = agx_temp(b->shader, AGX_SIZE_32);
+
+      agx_instr *phi = agx_phi_to(b, agx_temp(b->shader, AGX_SIZE_32), 2);
+      phi->src[0] = wx;
+      phi->src[1] = u;
+
+      agx_fmul_to(b, u, wx, phi->dest[0]);
+      agx_fmov_to(b, out, u)->saturate = true;
+   });
+}
+
+TEST_F(Optimizer, Copyprop)
+{
+   CASE32(agx_fmul_to(b, out, wx, agx_mov(b, wy)), agx_fmul_to(b, out, wx, wy));
+   CASE32(agx_fmul_to(b, out, agx_mov(b, wx), agx_mov(b, wy)),
+          agx_fmul_to(b, out, wx, wy));
+}
+
+TEST_F(Optimizer, SourceZeroExtend)
+{
+   CASE32(
+      {
+         agx_index t = agx_temp(b->shader, AGX_SIZE_32);
+         agx_mov_to(b, t, hy);
+         agx_ffs_to(b, out, t);
+      },
+      agx_ffs_to(b, out, hy));
+}
+
+TEST_F(Optimizer, AddSourceZeroExtend)
+{
+   CASE32(
+      {
+         agx_index t = agx_temp(b->shader, AGX_SIZE_32);
+         agx_mov_to(b, t, hy);
+         agx_iadd_to(b, out, wx, t, 1);
+      },
+      agx_iadd_to(b, out, wx, agx_abs(hy), 1));
+}
+
+TEST_F(Optimizer, AddSourceSignExtend)
+{
+   CASE32(
+      {
+         agx_index t = agx_temp(b->shader, AGX_SIZE_32);
+         agx_signext_to(b, t, hy);
+         agx_iadd_to(b, out, wx, t, 1);
+      },
+      agx_iadd_to(b, out, wx, hy, 1));
+}
+
+TEST_F(Optimizer, SubInlineImmediate)
+{
+   CASE16(agx_iadd_to(b, out, hx, agx_mov_imm(b, 16, -2), 0),
+          agx_iadd_to(b, out, hx, agx_neg(agx_immediate(2)), 0));
+
+   CASE32(agx_iadd_to(b, out, wx, agx_mov_imm(b, 32, -1), 0),
+          agx_iadd_to(b, out, wx, agx_neg(agx_immediate(1)), 0));
+
+   CASE64(agx_iadd_to(b, out, dx, agx_mov_imm(b, 64, -17), 0),
+          agx_iadd_to(b, out, dx, agx_neg(agx_immediate(17)), 0));
+
+   CASE16(agx_imad_to(b, out, hx, hy, agx_mov_imm(b, 16, -2), 0),
+          agx_imad_to(b, out, hx, hy, agx_neg(agx_immediate(2)), 0));
+
+   CASE32(agx_imad_to(b, out, wx, wy, agx_mov_imm(b, 32, -1), 0),
+          agx_imad_to(b, out, wx, wy, agx_neg(agx_immediate(1)), 0));
+
+   CASE64(agx_imad_to(b, out, dx, dz, agx_mov_imm(b, 64, -17), 0),
+          agx_imad_to(b, out, dx, dz, agx_neg(agx_immediate(17)), 0));
+}
+
+TEST_F(Optimizer, InlineHazards)
+{
+   NEGCASE32({
+      agx_index zero = agx_mov_imm(b, AGX_SIZE_32, 0);
+      agx_instr *I = agx_collect_to(b, out, 4);
+
+      I->src[0] = zero;
+      I->src[1] = wy;
+      I->src[2] = wz;
+      I->src[3] = wz;
+   });
+}
+
+TEST_F(Optimizer, CopypropRespectsAbsNeg)
+{
+   CASE32(agx_fadd_to(b, out, agx_abs(agx_mov(b, wx)), wy),
+          agx_fadd_to(b, out, agx_abs(wx), wy));
+
+   CASE32(agx_fadd_to(b, out, agx_neg(agx_mov(b, wx)), wy),
+          agx_fadd_to(b, out, agx_neg(wx), wy));
+
+   CASE32(agx_fadd_to(b, out, agx_neg(agx_abs(agx_mov(b, wx))), wy),
+          agx_fadd_to(b, out, agx_neg(agx_abs(wx)), wy));
+}
+
+TEST_F(Optimizer, IntCopyprop)
+{
+   CASE32(agx_xor_to(b, out, agx_mov(b, wx), wy), agx_xor_to(b, out, wx, wy));
+}
+
+TEST_F(Optimizer, CopypropSplitMovedUniform64)
+{
+   CASE32(
+      {
+         /* emit_load_preamble puts in the move, so we do too */
+         agx_index mov = agx_mov(b, agx_uniform(40, AGX_SIZE_64));
+         agx_instr *spl = agx_split(b, 2, mov);
+         spl->dest[0] = agx_temp(b->shader, AGX_SIZE_32);
+         spl->dest[1] = agx_temp(b->shader, AGX_SIZE_32);
+         agx_xor_to(b, out, spl->dest[0], spl->dest[1]);
+      },
+      {
+         agx_xor_to(b, out, agx_uniform(40, AGX_SIZE_32),
+                    agx_uniform(42, AGX_SIZE_32));
+      });
+}
+
+TEST_F(Optimizer, IntCopypropDoesntConvert)
+{
+   NEGCASE32({
+      agx_index cvt = agx_temp(b->shader, AGX_SIZE_32);
+      agx_mov_to(b, cvt, hx);
+      agx_fmul_to(b, out, cvt, wy);
+   });
+}
+
+TEST_F(Optimizer, SkipPreloads)
+{
+   NEGCASE32({
+      agx_index preload = agx_preload(b, agx_register(0, AGX_SIZE_32));
+      agx_xor_to(b, out, preload, wy);
+   });
+}
+
+TEST_F(Optimizer, NoConversionsOn16BitALU)
+{
+   NEGCASE16({
+      agx_index cvt = agx_temp(b->shader, AGX_SIZE_16);
+      agx_fmov_to(b, cvt, wx);
+      agx_fadd_to(b, out, cvt, hy);
+   });
+
+   NEGCASE32(agx_fmov_to(b, out, agx_fadd(b, hx, hy)));
+}
+
+TEST_F(Optimizer, BallotCondition)
+{
+   CASE32(agx_ballot_to(b, out, agx_icmp(b, wx, wy, AGX_ICOND_UEQ, true)),
+          agx_icmp_ballot_to(b, out, wx, wy, AGX_ICOND_UEQ, true));
+
+   CASE32(agx_ballot_to(b, out, agx_fcmp(b, wx, wy, AGX_FCOND_GE, false)),
+          agx_fcmp_ballot_to(b, out, wx, wy, AGX_FCOND_GE, false));
+
+   CASE32(agx_quad_ballot_to(b, out, agx_icmp(b, wx, wy, AGX_ICOND_UEQ, true)),
+          agx_icmp_quad_ballot_to(b, out, wx, wy, AGX_ICOND_UEQ, true));
+
+   CASE32(agx_quad_ballot_to(b, out, agx_fcmp(b, wx, wy, AGX_FCOND_GT, false)),
+          agx_fcmp_quad_ballot_to(b, out, wx, wy, AGX_FCOND_GT, false));
+}
+
+TEST_F(Optimizer, BallotMultipleUses)
+{
+   CASE32(
+      {
+         agx_index cmp = agx_fcmp(b, wx, wy, AGX_FCOND_GT, false);
+         agx_index ballot = agx_quad_ballot(b, cmp);
+         agx_fadd_to(b, out, cmp, ballot);
+      },
+      {
+         agx_index cmp = agx_fcmp(b, wx, wy, AGX_FCOND_GT, false);
+         agx_index ballot =
+            agx_fcmp_quad_ballot(b, wx, wy, AGX_FCOND_GT, false);
+         agx_fadd_to(b, out, cmp, ballot);
+      });
+}
+
+/*
+ * We had a bug where the ballot optimization didn't check the agx_index's type
+ * so would fuse constants with overlapping values. An unrelated common code
+ * change surfaced this in CTS case:
+ *
+ *    dEQP-VK.subgroups.vote.frag_helper.subgroupallequal_bool_fragment
+ *
+ * We passed Vulkan CTS without hitting it though, hence the targeted test.
+ */
+TEST_F(Optimizer, BallotConstant)
+{
+   CASE32(
+      {
+         agx_index cmp = agx_fcmp(b, wx, wy, AGX_FCOND_GT, false);
+         agx_index ballot = agx_quad_ballot(b, agx_immediate(cmp.value));
+         agx_index ballot2 = agx_quad_ballot(b, cmp);
+         agx_fadd_to(b, out, ballot, agx_fadd(b, ballot2, cmp));
+      },
+      {
+         agx_index cmp = agx_fcmp(b, wx, wy, AGX_FCOND_GT, false);
+         agx_index ballot = agx_quad_ballot(b, agx_immediate(cmp.value));
+         agx_index ballot2 =
+            agx_fcmp_quad_ballot(b, wx, wy, AGX_FCOND_GT, false);
+         agx_fadd_to(b, out, ballot, agx_fadd(b, ballot2, cmp));
+      });
+}
+
+TEST_F(Optimizer, IfCondition)
+{
+   CASE_NO_RETURN(agx_if_icmp(b, agx_icmp(b, wx, wy, AGX_ICOND_UEQ, true),
+                              agx_zero(), 1, AGX_ICOND_UEQ, true, NULL),
+                  agx_if_icmp(b, wx, wy, 1, AGX_ICOND_UEQ, true, NULL));
+
+   CASE_NO_RETURN(agx_if_icmp(b, agx_fcmp(b, wx, wy, AGX_FCOND_EQ, true),
+                              agx_zero(), 1, AGX_ICOND_UEQ, true, NULL),
+                  agx_if_fcmp(b, wx, wy, 1, AGX_FCOND_EQ, true, NULL));
+
+   CASE_NO_RETURN(agx_if_icmp(b, agx_fcmp(b, hx, hy, AGX_FCOND_LT, false),
+                              agx_zero(), 1, AGX_ICOND_UEQ, true, NULL),
+                  agx_if_fcmp(b, hx, hy, 1, AGX_FCOND_LT, false, NULL));
+}
+
+TEST_F(Optimizer, SelectCondition)
+{
+   CASE32(agx_icmpsel_to(b, out, agx_icmp(b, wx, wy, AGX_ICOND_UEQ, false),
+                         agx_zero(), wz, wx, AGX_ICOND_UEQ),
+          agx_icmpsel_to(b, out, wx, wy, wx, wz, AGX_ICOND_UEQ));
+
+   CASE32(agx_icmpsel_to(b, out, agx_icmp(b, wx, wy, AGX_ICOND_UEQ, true),
+                         agx_zero(), wz, wx, AGX_ICOND_UEQ),
+          agx_icmpsel_to(b, out, wx, wy, wz, wx, AGX_ICOND_UEQ));
+
+   CASE32(agx_icmpsel_to(b, out, agx_fcmp(b, wx, wy, AGX_FCOND_EQ, false),
+                         agx_zero(), wz, wx, AGX_ICOND_UEQ),
+          agx_fcmpsel_to(b, out, wx, wy, wx, wz, AGX_FCOND_EQ));
+
+   CASE32(agx_icmpsel_to(b, out, agx_fcmp(b, wx, wy, AGX_FCOND_LT, true),
+                         agx_zero(), wz, wx, AGX_ICOND_UEQ),
+          agx_fcmpsel_to(b, out, wx, wy, wz, wx, AGX_FCOND_LT));
+}
+
+TEST_F(Optimizer, IfInverted)
+{
+   CASE_NO_RETURN(
+      agx_if_icmp(b, agx_xor(b, hx, agx_immediate(1)), agx_zero(), 1,
+                  AGX_ICOND_UEQ, true, NULL),
+      agx_if_icmp(b, hx, agx_zero(), 1, AGX_ICOND_UEQ, false, NULL));
+
+   CASE_NO_RETURN(agx_if_icmp(b, agx_xor(b, hx, agx_immediate(1)), agx_zero(),
+                              1, AGX_ICOND_UEQ, false, NULL),
+                  agx_if_icmp(b, hx, agx_zero(), 1, AGX_ICOND_UEQ, true, NULL));
+}
+
+TEST_F(Optimizer, IfInvertedCondition)
+{
+   CASE_NO_RETURN(
+      agx_if_icmp(
+         b,
+         agx_xor(b, agx_icmp(b, wx, wy, AGX_ICOND_UEQ, true), agx_immediate(1)),
+         agx_zero(), 1, AGX_ICOND_UEQ, true, NULL),
+      agx_if_icmp(b, wx, wy, 1, AGX_ICOND_UEQ, false, NULL));
+
+   CASE_NO_RETURN(
+      agx_if_icmp(
+         b,
+         agx_xor(b, agx_fcmp(b, wx, wy, AGX_FCOND_EQ, true), agx_immediate(1)),
+         agx_zero(), 1, AGX_ICOND_UEQ, true, NULL),
+      agx_if_fcmp(b, wx, wy, 1, AGX_FCOND_EQ, false, NULL));
+
+   CASE_NO_RETURN(
+      agx_if_icmp(
+         b,
+         agx_xor(b, agx_fcmp(b, hx, hy, AGX_FCOND_LT, false), agx_immediate(1)),
+         agx_zero(), 1, AGX_ICOND_UEQ, true, NULL),
+      agx_if_fcmp(b, hx, hy, 1, AGX_FCOND_LT, true, NULL));
+}

src/asahi/compiler/test/test-repair-ssa.cpp

@@ -0,0 +1,108 @@
+/*
+ * Copyright 2024 Alyssa Rosenzweig
+ * Copyright 2022 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "agx_builder.h"
+#include "agx_compiler.h"
+#include "agx_test.h"
+
+#include <gtest/gtest.h>
+
+static void
+pass(agx_context *ctx)
+{
+   agx_repair_ssa(ctx);
+   agx_reindex_ssa(ctx);
+}
+
+#define CASE(instr)                                                            \
+   INSTRUCTION_CASE(                                                           \
+      {                                                                        \
+         bool repaired = false;                                                \
+         instr                                                                 \
+      },                                                                       \
+      {                                                                        \
+         bool repaired = true;                                                 \
+         instr                                                                 \
+      },                                                                       \
+      pass)
+
+class RepairSSA : public testing::Test {
+ protected:
+   RepairSSA()
+   {
+      mem_ctx = ralloc_context(NULL);
+   }
+
+   ~RepairSSA()
+   {
+      ralloc_free(mem_ctx);
+   }
+
+   void *mem_ctx;
+};
+
+static agx_index
+agx_phi_2(agx_builder *b, agx_index x, agx_index y)
+{
+   agx_index idx = agx_temp(b->shader, x.size);
+   agx_instr *phi = agx_phi_to(b, idx, 2);
+   phi->src[0] = x;
+   phi->src[1] = y;
+   return idx;
+}
+
+TEST_F(RepairSSA, Local)
+{
+   CASE({
+      agx_index x = agx_mov_imm(b, AGX_SIZE_32, 0xcafe);
+      agx_index y = agx_mov_imm(b, AGX_SIZE_32, 0xefac);
+
+      if (repaired) {
+         agx_unit_test(b, agx_fadd(b, y, x));
+      } else {
+         agx_fadd_to(b, x, y, x);
+         agx_unit_test(b, x);
+      }
+   });
+}
+
+/*      A
+ *     / \
+ *    B   C
+ *     \ /
+ *      D
+ */
+TEST_F(RepairSSA, IfElse)
+{
+   CASE({
+      agx_block *A = agx_start_block(b->shader);
+      agx_block *B = agx_test_block(b->shader);
+      agx_block *C = agx_test_block(b->shader);
+      agx_block *D = agx_test_block(b->shader);
+
+      agx_block_add_successor(A, B);
+      agx_block_add_successor(A, C);
+
+      agx_block_add_successor(B, D);
+      agx_block_add_successor(C, D);
+
+      b->cursor = agx_after_block(B);
+      agx_index x = agx_mov_imm(b, 32, 0xcafe);
+      agx_index y = agx_mov_imm(b, 32, 0xbade);
+
+      b->cursor = agx_after_block(C);
+      agx_index x2 = repaired ? agx_temp(b->shader, AGX_SIZE_32) : x;
+      agx_mov_imm_to(b, x2, 0xefac);
+      agx_index y2 = agx_mov_imm(b, 32, 0xbade);
+
+      b->cursor = agx_after_block(D);
+      if (repaired)
+         x = agx_phi_2(b, x, x2);
+
+      agx_index y3 = agx_phi_2(b, y, y2);
+      agx_unit_test(b, agx_fadd(b, x, y3));
+   });
+}

src/asahi/drm-shim/asahi_noop.c

@@ -0,0 +1,127 @@
+/*
+ * Copyright 2022 Alyssa Rosenzweig
+ * Copyright 2018 Broadcom
+ * SPDX-License-Identifier: MIT
+ */
+
+#include <string.h>
+
+#include "drm-shim/drm_shim.h"
+#include "drm-uapi/asahi_drm.h"
+
+bool drm_shim_driver_prefers_first_render_node = true;
+
+static const struct drm_asahi_params_global params = {
+   .gpu_generation = 13,
+   .gpu_variant = 'G',
+   .gpu_revision = 0,
+   .vm_start = 0x4000,
+   .vm_end = 0x7fffff8000ull,
+   .command_timestamp_frequency_hz = 1000000000,
+};
+
+struct asahi_bo {
+   struct shim_bo base;
+   uint32_t offset;
+};
+
+static struct asahi_bo *
+asahi_bo(struct shim_bo *bo)
+{
+   return (struct asahi_bo *)bo;
+}
+
+struct asahi_device {
+   uint64_t next_offset;
+};
+
+static struct asahi_device asahi = {
+   .next_offset = 0x1000,
+};
+
+static int
+asahi_ioctl_noop(int fd, unsigned long request, void *arg)
+{
+   return 0;
+}
+
+static int
+asahi_ioctl_gem_create(int fd, unsigned long request, void *arg)
+{
+   struct shim_fd *shim_fd = drm_shim_fd_lookup(fd);
+   struct drm_asahi_gem_create *create = arg;
+   struct asahi_bo *bo = calloc(1, sizeof(*bo));
+
+   drm_shim_bo_init(&bo->base, create->size);
+
+   assert(UINT64_MAX - asahi.next_offset > create->size);
+   bo->offset = asahi.next_offset;
+   asahi.next_offset += create->size;
+
+   create->handle = drm_shim_bo_get_handle(shim_fd, &bo->base);
+
+   drm_shim_bo_put(&bo->base);
+
+   return 0;
+}
+
+static int
+asahi_ioctl_gem_mmap_offset(int fd, unsigned long request, void *arg)
+{
+   struct shim_fd *shim_fd = drm_shim_fd_lookup(fd);
+   struct drm_asahi_gem_mmap_offset *map = arg;
+   struct shim_bo *bo = drm_shim_bo_lookup(shim_fd, map->handle);
+
+   map->offset = drm_shim_bo_get_mmap_offset(shim_fd, bo);
+
+   drm_shim_bo_put(bo);
+
+   return 0;
+}
+
+static int
+asahi_ioctl_get_param(int fd, unsigned long request, void *arg)
+{
+   struct drm_asahi_get_params *gp = arg;
+
+   switch (gp->param_group) {
+   case 0:
+      assert(gp->size == sizeof(struct drm_asahi_params_global));
+      memcpy((void *)gp->pointer, &params, gp->size);
+      return 0;
+
+   default:
+      fprintf(stderr, "Unknown DRM_IOCTL_ASAHI_GET_PARAMS %d\n",
+              gp->param_group);
+      return -1;
+   }
+}
+
+static ioctl_fn_t driver_ioctls[] = {
+   [DRM_ASAHI_GET_PARAMS] = asahi_ioctl_get_param,
+   [DRM_ASAHI_VM_CREATE] = asahi_ioctl_noop,
+   [DRM_ASAHI_VM_DESTROY] = asahi_ioctl_noop,
+   [DRM_ASAHI_VM_BIND] = asahi_ioctl_noop,
+   [DRM_ASAHI_GEM_CREATE] = asahi_ioctl_gem_create,
+   [DRM_ASAHI_GEM_MMAP_OFFSET] = asahi_ioctl_gem_mmap_offset,
+   [DRM_ASAHI_QUEUE_CREATE] = asahi_ioctl_noop,
+   [DRM_ASAHI_QUEUE_DESTROY] = asahi_ioctl_noop,
+   [DRM_ASAHI_GEM_BIND_OBJECT] = asahi_ioctl_noop,
+   [DRM_ASAHI_SUBMIT] = asahi_ioctl_noop,
+};
+
+void
+drm_shim_driver_init(void)
+{
+   shim_device.bus_type = DRM_BUS_PLATFORM;
+   shim_device.driver_name = "asahi";
+   shim_device.driver_ioctls = driver_ioctls;
+   shim_device.driver_ioctl_count = ARRAY_SIZE(driver_ioctls);
+
+   drm_shim_override_file("DRIVER=asahi\n"
+                          "OF_FULLNAME=/soc/agx\n"
+                          "OF_COMPATIBLE_0=apple,gpu-g13g\n"
+                          "OF_COMPATIBLE_N=1\n",
+                          "/sys/dev/char/%d:%d/device/uevent", DRM_MAJOR,
+                          render_node_minor);
+}

src/asahi/drm-shim/meson.build

@@ -0,0 +1,11 @@
+# Copyright 2019 Broadcom
+# SPDX-License-Identifier: MIT
+
+libasahi_noop_drm_shim = shared_library(
+  'asahi_noop_drm_shim',
+  'asahi_noop.c',
+  include_directories: [inc_include, inc_src],
+  dependencies: [dep_drm_shim, dep_valgrind],
+  gnu_symbol_visibility : 'hidden',
+  install : true,
+)

src/asahi/genxml/agx_pack_header.h

@@ -0,0 +1,165 @@
+/*
+ * Copyright 2016 Intel Corporation
+ * Copyright 2016 Broadcom
+ * Copyright 2020 Collabora, Ltd.
+ * Copyright 2024 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#pragma once
+
+#include "compiler/libcl/libcl.h"
+#include "util/bitpack_helpers.h"
+
+#ifndef __OPENCL_VERSION__
+#include <inttypes.h>
+#include <stdio.h>
+#include "util/half_float.h"
+#endif
+
+#define __gen_unpack_float(x, y, z) uif(__gen_unpack_uint(x, y, z))
+#define __gen_unpack_half(x, y, z)                                             \
+   _mesa_half_to_float(__gen_unpack_uint(x, y, z))
+
+static inline uint64_t
+__gen_unpack_uint(CONST uint32_t *restrict cl, uint32_t start, uint32_t end)
+{
+   uint64_t val = 0;
+   const int width = end - start + 1;
+   const uint64_t mask =
+      (width == 64) ? ~((uint64_t)0) : ((uint64_t)1 << width) - 1;
+
+   for (unsigned word = start / 32; word < (end / 32) + 1; word++) {
+      val |= ((uint64_t)cl[word]) << ((word - start / 32) * 32);
+   }
+
+   return (val >> (start % 32)) & mask;
+}
+
+/*
+ * LODs are 4:6 fixed point. We must clamp before converting to integers to
+ * avoid undefined behaviour for out-of-bounds inputs like +/- infinity.
+ */
+static inline uint32_t
+__gen_pack_lod(float f, uint32_t start, uint32_t end)
+{
+   uint32_t fixed = CLAMP(f * (1 << 6), 0 /* 0.0 */, 0x380 /* 14.0 */);
+   return util_bitpack_uint(fixed, start, end);
+}
+
+static inline float
+__gen_unpack_lod(CONST uint32_t *restrict cl, uint32_t start, uint32_t end)
+{
+   return ((float)__gen_unpack_uint(cl, start, end)) / (1 << 6);
+}
+
+static inline uint64_t
+__gen_unpack_sint(CONST uint32_t *restrict cl, uint32_t start, uint32_t end)
+{
+   int size = end - start + 1;
+   int64_t val = __gen_unpack_uint(cl, start, end);
+
+   return util_sign_extend(val, size);
+}
+
+static inline uint64_t
+__gen_to_groups(uint32_t value, uint32_t group_size, uint32_t length)
+{
+   /* Zero is not representable, clamp to minimum */
+   if (value == 0)
+      return 1;
+
+   /* Round up to the nearest number of groups */
+   uint32_t groups = DIV_ROUND_UP(value, group_size);
+
+   /* The 0 encoding means "all" */
+   if (groups == ((uint64_t)1) << length)
+      return 0;
+
+   /* Otherwise it's encoded as the identity */
+   assert(groups < (1u << length) && "out of bounds");
+   assert(groups >= 1 && "exhaustive");
+   return groups;
+}
+
+static inline uint64_t
+__gen_from_groups(uint32_t value, uint32_t group_size, uint32_t length)
+{
+   return group_size * (value ? value : (1 << length));
+}
+
+#define agx_pack(dst, T, name)                                                 \
+   for (struct AGX_##T name = {AGX_##T##_header},                              \
+                       *_loop_count = (GLOBAL void *)((uintptr_t)0);           \
+        (uintptr_t)_loop_count < 1; (                                          \
+           {                                                                   \
+              AGX_##T##_pack((GLOBAL uint32_t *)(dst), &name);                 \
+              _loop_count = (GLOBAL void *)(((uintptr_t)_loop_count) + 1);     \
+           }))
+
+#define agx_unpack(fp, src, T, name)                                           \
+   struct AGX_##T name;                                                        \
+   AGX_##T##_unpack(fp, (CONST uint8_t *)(src), &name)
+
+#define agx_print(fp, T, var, indent) AGX_##T##_print(fp, &(var), indent)
+
+static inline void
+agx_merge_helper(uint32_t *dst, const uint32_t *src, size_t bytes)
+{
+   assert((bytes & 3) == 0);
+
+   for (unsigned i = 0; i < (bytes / 4); ++i)
+      dst[i] |= src[i];
+}
+
+#define agx_merge(packed1, packed2, type)                                      \
+   agx_merge_helper((packed1).opaque, (packed2).opaque, AGX_##type##_LENGTH)
+
+#if defined(NDEBUG) || defined(__OPENCL_VERSION__)
+#define agx_genxml_validate_bounds(a, b, c)
+#define agx_genxml_validate_mask(a, b, c, d, e) true
+#define agx_genxml_validate_exact(a, b, c, d)   true
+#else
+static inline void
+agx_genxml_validate_bounds(const char *name, uint64_t value, uint64_t bound)
+{
+   if (unlikely(value >= bound)) {
+      fprintf(stderr, "%s out-of-bounds, got 0x%" PRIx64 ", max %" PRIx64 "\n",
+              name, value, bound);
+
+      UNREACHABLE("Out-of-bounds pack");
+   }
+}
+
+static inline bool
+agx_genxml_validate_mask(FILE *fp, const char *name, const void *cl_,
+                         uint32_t index, uint32_t bad_mask)
+{
+   const uint32_t *cl = (const uint32_t *)cl_;
+   uint32_t bad = cl[index] & bad_mask;
+
+   if (bad && fp != NULL) {
+      fprintf(
+         fp,
+         "XXX: Unknown field of %s unpacked at word %u got %X, bad mask %X\n",
+         name, index, cl[index], bad);
+   }
+
+   return bad == 0;
+}
+
+static bool
+agx_genxml_validate_exact(FILE *fp, const char *name, uint64_t value,
+                          uint64_t exact)
+{
+   if (value != exact && fp != NULL) {
+      fprintf(fp, "XXX: Expected %s to equal %" PRIx64 " but got %" PRIx64 "\n",
+              name, value, exact);
+   }
+
+   return value == exact;
+}
+
+#endif
+
+/* Everything after this is autogenerated from XML. Do not hand edit. */

src/asahi/genxml/gen_pack.py

@@ -0,0 +1,655 @@
+#encoding=utf-8
+
+# Copyright 2016 Intel Corporation
+# Copyright 2016 Broadcom
+# Copyright 2020 Collabora, Ltd.
+# SPDX-License-Identifier: MIT
+
+import xml.parsers.expat
+import sys
+import operator
+import math
+import platform
+from functools import reduce
+
+global_prefix = "agx"
+
+def to_alphanum(name):
+    substitutions = {
+        ' ': '_',
+        '/': '_',
+        '[': '',
+        ']': '',
+        '(': '',
+        ')': '',
+        '-': '_',
+        ':': '',
+        '.': '',
+        ',': '',
+        '=': '',
+        '>': '',
+        '#': '',
+        '&': '',
+        '*': '',
+        '"': '',
+        '+': '',
+        '\'': '',
+        '?': '',
+    }
+
+    for i, j in substitutions.items():
+        name = name.replace(i, j)
+
+    return name
+
+def safe_name(name):
+    name = to_alphanum(name)
+    if not name[0].isalpha():
+        name = '_' + name
+
+    return name
+
+def prefixed_upper_name(prefix, name):
+    if prefix:
+        name = prefix + "_" + name
+    return safe_name(name).upper()
+
+def enum_name(name):
+    return f"{global_prefix}_{safe_name(name)}".lower()
+
+MODIFIERS = ["shr", "minus", "align", "log2", "groups"]
+
+def parse_modifier(modifier):
+    if modifier is None:
+        return None
+
+    for mod in MODIFIERS:
+        if modifier[0:len(mod)] == mod:
+            if mod == "log2":
+                assert(len(mod) == len(modifier))
+                return [mod]
+
+            if modifier[len(mod)] == '(' and modifier[-1] == ')':
+                ret = [mod, int(modifier[(len(mod) + 1):-1])]
+                if ret[0] == 'align':
+                    align = ret[1]
+                    # Make sure the alignment is a power of 2
+                    assert(align > 0 and not(align & (align - 1)));
+
+                return ret
+
+    print("Invalid modifier")
+    assert(False)
+
+class Field(object):
+    def __init__(self, parser, attrs):
+        self.parser = parser
+        if "name" in attrs:
+            self.name = safe_name(attrs["name"]).lower()
+            self.human_name = attrs["name"]
+
+        if ":" in str(attrs["start"]):
+            (word, bit) = attrs["start"].split(":")
+            self.start = (int(word) * 32) + int(bit)
+        else:
+            self.start = int(attrs["start"])
+
+        self.end = self.start + int(attrs["size"]) - 1
+        self.type = attrs["type"]
+
+        if self.type == 'bool' and self.start != self.end:
+            print(f"#error Field {self.name} has bool type but more than one bit of size");
+
+        if "prefix" in attrs:
+            self.prefix = safe_name(attrs["prefix"]).upper()
+        else:
+            self.prefix = None
+
+        self.modifier = parse_modifier(attrs.get("modifier"))
+        self.exact = attrs.get("exact")
+        self.default = None
+
+        if self.exact is not None:
+            self.default = self.exact
+        elif self.modifier is not None:
+            # Set the default value to encode to zero
+            mod = self.modifier
+            if mod[0] == 'log2':
+                self.default = 1
+            elif mod[0] == 'minus':
+                self.default = mod[1]
+            elif mod[0] == 'groups':
+                # The zero encoding means "all"
+                self.default = (1 << int(attrs["size"])) * mod[1]
+            elif mod[0] in ['shr', 'align']:
+                # Zero encodes to zero
+                pass
+            else:
+                assert(0)
+
+        # Map enum values
+        if self.type in self.parser.enums and self.default is not None:
+            self.default = safe_name(f'{global_prefix}_{self.type}_{self.default}').upper()
+
+
+    def emit_template_struct(self, dim):
+        if self.type == 'address':
+            type = 'uint64_t'
+        elif self.type == 'bool':
+            type = 'bool'
+        elif self.type in ['float', 'half', 'lod']:
+            type = 'float'
+        elif self.type in ['uint', 'hex'] and self.end - self.start > 32:
+            type = 'uint64_t'
+        elif self.type == 'int':
+            type = 'int32_t'
+        elif self.type in ['uint', 'hex']:
+            type = 'uint32_t'
+        elif self.type in self.parser.structs:
+            type = 'struct ' + self.parser.gen_prefix(safe_name(self.type.upper()))
+        elif self.type in self.parser.enums:
+            type = 'enum ' + enum_name(self.type)
+        else:
+            print(f"#error unhandled type: {self.type}")
+            type = "uint32_t"
+
+        print("   %-36s %s%s;" % (type, self.name, dim))
+
+        for value in self.values:
+            name = prefixed_upper_name(self.prefix, value.name)
+            print("#define %-40s %d" % (name, value.value))
+
+    def overlaps(self, field):
+        return self != field and max(self.start, field.start) <= min(self.end, field.end)
+
+class Group(object):
+    def __init__(self, parser, parent, start, count, label):
+        self.parser = parser
+        self.parent = parent
+        self.start = start
+        self.count = count
+        self.label = label
+        self.size = 0
+        self.length = 0
+        self.fields = []
+
+    def get_length(self):
+        # Determine number of bytes in this group.
+        calculated = max(field.end // 8 for field in self.fields) + 1 if len(self.fields) > 0 else 0
+        if self.length > 0:
+            assert(self.length >= calculated)
+        else:
+            self.length = calculated
+        return self.length
+
+
+    def emit_template_struct(self, dim):
+        if self.count == 0:
+            print("   /* variable length fields follow */")
+        else:
+            if self.count > 1:
+                dim = "%s[%d]" % (dim, self.count)
+
+            any_fields = False
+            for field in self.fields:
+                if not field.exact:
+                    field.emit_template_struct(dim)
+                    any_fields = True
+
+            if not any_fields:
+                print("   int dummy;")
+
+    class Word:
+        def __init__(self):
+            self.size = 32
+            self.contributors = []
+
+    class FieldRef:
+        def __init__(self, field, path, start, end):
+            self.field = field
+            self.path = path
+            self.start = start
+            self.end = end
+
+    def collect_fields(self, fields, offset, path, all_fields):
+        for field in fields:
+            field_path = f'{path}{field.name}'
+            field_offset = offset + field.start
+
+            if field.type in self.parser.structs:
+                sub_struct = self.parser.structs[field.type]
+                self.collect_fields(sub_struct.fields, field_offset, field_path + '.', all_fields)
+                continue
+
+            start = field_offset
+            end = offset + field.end
+            all_fields.append(self.FieldRef(field, field_path, start, end))
+
+    def collect_words(self, fields, offset, path, words):
+        for field in fields:
+            field_path = f'{path}{field.name}'
+            start = offset + field.start
+
+            if field.type in self.parser.structs:
+                sub_fields = self.parser.structs[field.type].fields
+                self.collect_words(sub_fields, start, field_path + '.', words)
+                continue
+
+            end = offset + field.end
+            contributor = self.FieldRef(field, field_path, start, end)
+            first_word = contributor.start // 32
+            last_word = contributor.end // 32
+            for b in range(first_word, last_word + 1):
+                if not b in words:
+                    words[b] = self.Word()
+                words[b].contributors.append(contributor)
+
+    def emit_pack_function(self):
+        self.get_length()
+
+        words = {}
+        self.collect_words(self.fields, 0, '', words)
+
+        # Validate the modifier is lossless
+        for field in self.fields:
+            if field.modifier is None:
+                continue
+
+            if field.modifier[0] == "shr":
+                shift = field.modifier[1]
+                mask = hex((1 << shift) - 1)
+                print(f"   assert((values->{field.name} & {mask}) == 0);")
+            elif field.modifier[0] == "minus":
+                print(f"   assert(values->{field.name} >= {field.modifier[1]});")
+            elif field.modifier[0] == "log2":
+                print(f"   assert(IS_POT_NONZERO(values->{field.name}));")
+
+        for index in range(math.ceil(self.length / 4)):
+            # Handle MBZ words
+            if not index in words:
+                print("   cl[%2d] = 0;" % index)
+                continue
+
+            word = words[index]
+
+            word_start = index * 32
+
+            v = None
+            prefix = "   cl[%2d] =" % index
+
+            lines = []
+
+            for contributor in word.contributors:
+                field = contributor.field
+                name = field.name
+                start = contributor.start
+                end = contributor.end
+                contrib_word_start = (start // 32) * 32
+                start -= contrib_word_start
+                end -= contrib_word_start
+
+                value = f"values->{contributor.path}"
+                if field.exact:
+                    value = field.default
+
+                # These types all use util_bitpack_uint
+                pack_as_uint = field.type in ["uint", "hex", "address", "bool"]
+                pack_as_uint |= field.type in self.parser.enums
+                start_adjusted = start
+                value_unshifted = None
+
+                if field.modifier is not None:
+                    if field.modifier[0] == "shr":
+                        if pack_as_uint and start >= field.modifier[1]:
+                            # For uint, we fast path.  If we do `(a >> 2) << 2`,
+                            # clang will generate a mask in release builds, even
+                            # though we know we're aligned. So don't generate
+                            # that to avoid the masking.
+                            start_adjusted = start - field.modifier[1]
+                        else:
+                            value = f"{value} >> {field.modifier[1]}"
+                    elif field.modifier[0] == "minus":
+                        value = f"{value} - {field.modifier[1]}"
+                    elif field.modifier[0] == "align":
+                        value = f"ALIGN_POT({value}, {field.modifier[1]})"
+                    elif field.modifier[0] == "log2":
+                        value = f"util_logbase2({value})"
+                    elif field.modifier[0] == "groups":
+                        value = "__gen_to_groups({}, {}, {})".format(value,
+                                field.modifier[1], end - start + 1)
+
+                if pack_as_uint:
+                    bits = (end - start_adjusted + 1)
+                    if bits < 64 and not field.exact:
+                        # Add some nicer error checking
+                        label = f"{self.label}::{name}"
+                        bound = hex(1 << bits)
+                        print(f"   agx_genxml_validate_bounds(\"{label}\", {value}, {bound}ull);")
+
+                    s = f"util_bitpack_uint({value}, {start_adjusted}, {end})"
+                elif field.type == "int":
+                    s = "util_bitpack_sint(%s, %d, %d)" % \
+                        (value, start, end)
+                elif field.type == "float":
+                    assert(start == 0 and end == 31)
+                    s = f"util_bitpack_float({value})"
+                elif field.type == "half":
+                    assert(start == 0 and end == 15)
+                    s = f"_mesa_float_to_half({value})"
+                elif field.type == "lod":
+                    assert(end - start + 1 == 10)
+                    s = "__gen_pack_lod(%s, %d, %d)" % (value, start, end)
+                else:
+                    s = f"#error unhandled field {contributor.path}, type {field.type}"
+
+                if not s == None:
+                    shift = word_start - contrib_word_start
+                    if shift:
+                        s = "%s >> %d" % (s, shift)
+
+                    if contributor == word.contributors[-1]:
+                        lines.append(f"{prefix} {s};")
+                    else:
+                        lines.append(f"{prefix} {s} |")
+                    prefix = "           "
+
+            for ln in lines:
+                print(ln)
+
+            continue
+
+    # Given a field (start, end) contained in word `index`, generate the 32-bit
+    # mask of present bits relative to the word
+    def mask_for_word(self, index, start, end):
+        field_word_start = index * 32
+        start -= field_word_start
+        end -= field_word_start
+        # Cap multiword at one word
+        start = max(start, 0)
+        end = min(end, 32 - 1)
+        count = (end - start + 1)
+        return (((1 << count) - 1) << start)
+
+    def emit_unpack_function(self):
+        # First, verify there is no garbage in unused bits
+        words = {}
+        self.collect_words(self.fields, 0, '', words)
+        validation = []
+
+        for index in range(self.length // 4):
+            base = index * 32
+            word = words.get(index, self.Word())
+            masks = [self.mask_for_word(index, c.start, c.end) for c in word.contributors]
+            mask = reduce(lambda x,y: x | y, masks, 0)
+
+            ALL_ONES = 0xffffffff
+
+            if mask != ALL_ONES:
+                bad_mask = hex(mask ^ ALL_ONES)
+                validation.append(f'agx_genxml_validate_mask(fp, \"{self.label}\", cl, {index}, {bad_mask})')
+
+        fieldrefs = []
+        self.collect_fields(self.fields, 0, '', fieldrefs)
+        for fieldref in fieldrefs:
+            field = fieldref.field
+            convert = None
+
+            args = []
+            args.append('(CONST uint32_t *) cl')
+            args.append(str(fieldref.start))
+            args.append(str(fieldref.end))
+
+            if field.type in set(["uint", "address", "hex"]) | self.parser.enums:
+                convert = "__gen_unpack_uint"
+            elif field.type == "int":
+                convert = "__gen_unpack_sint"
+            elif field.type == "bool":
+                convert = "__gen_unpack_uint"
+            elif field.type == "float":
+                convert = "__gen_unpack_float"
+            elif field.type == "half":
+                convert = "__gen_unpack_half"
+            elif field.type == "lod":
+                convert = "__gen_unpack_lod"
+            else:
+                s = f"/* unhandled field {field.name}, type {field.type} */\n"
+
+            suffix = ""
+            prefix = ""
+            if field.modifier:
+                if field.modifier[0] == "minus":
+                    suffix = f" + {field.modifier[1]}"
+                elif field.modifier[0] == "shr":
+                    suffix = f" << {field.modifier[1]}"
+                if field.modifier[0] == "log2":
+                    prefix = "1 << "
+                elif field.modifier[0] == "groups":
+                    prefix = "__gen_from_groups("
+                    suffix = ", {}, {})".format(field.modifier[1],
+                                                fieldref.end - fieldref.start + 1)
+
+            if field.type in self.parser.enums and not field.exact:
+                prefix = f"(enum {enum_name(field.type)}) {prefix}"
+
+            decoded = f"{prefix}{convert}({', '.join(args)}){suffix}"
+
+            if field.exact:
+                name = self.label
+                validation.append(f'agx_genxml_validate_exact(fp, \"{name}\", {decoded}, {field.default})')
+            else:
+                print(f'   values->{fieldref.path} = {decoded};')
+
+            if field.modifier and field.modifier[0] == "align":
+                assert(not field.exact)
+                mask = hex(field.modifier[1] - 1)
+                print(f'   assert(!(values->{fieldref.path} & {mask}));')
+
+        if len(validation) > 1:
+            print('   bool valid = true;')
+            for v in validation:
+                print(f'   valid &= {v};')
+            print("   return valid;")
+        elif len(validation) == 1:
+            print(f"   return {validation[0]};")
+        else:
+            print("   return true;")
+
+    def emit_print_function(self):
+        for field in self.fields:
+            convert = None
+            name, val = field.human_name, f'values->{field.name}'
+
+            if field.exact:
+                continue
+
+            if field.type in self.parser.structs:
+                pack_name = self.parser.gen_prefix(safe_name(field.type)).upper()
+                print(f'   fprintf(fp, "%*s{field.human_name}:\\n", indent, "");')
+                print(f"   {pack_name}_print(fp, &values->{field.name}, indent + 2);")
+            elif field.type == "address":
+                # TODO resolve to name
+                print(f'   fprintf(fp, "%*s{name}: 0x%" PRIx64 "\\n", indent, "", {val});')
+            elif field.type in self.parser.enums:
+                print(f'   if ({enum_name(field.type)}_as_str({val}))')
+                print(f'     fprintf(fp, "%*s{name}: %s\\n", indent, "", {enum_name(field.type)}_as_str({val}));')
+                print(f'   else')
+                print(f'     fprintf(fp, "%*s{name}: unknown %X (XXX)\\n", indent, "", {val});')
+            elif field.type == "int":
+                print(f'   fprintf(fp, "%*s{name}: %d\\n", indent, "", {val});')
+            elif field.type == "bool":
+                print(f'   fprintf(fp, "%*s{name}: %s\\n", indent, "", {val} ? "true" : "false");')
+            elif field.type in ["float", "lod", "half"]:
+                print(f'   fprintf(fp, "%*s{name}: %f\\n", indent, "", {val});')
+            elif field.type in ["uint", "hex"] and (field.end - field.start) >= 32:
+                print(f'   fprintf(fp, "%*s{name}: 0x%" PRIx64 "\\n", indent, "", {val});')
+            elif field.type == "hex":
+                print(f'   fprintf(fp, "%*s{name}: 0x%" PRIx32 "\\n", indent, "", {val});')
+            else:
+                print(f'   fprintf(fp, "%*s{name}: %u\\n", indent, "", {val});')
+
+class Value(object):
+    def __init__(self, attrs):
+        self.name = attrs["name"]
+        self.value = int(attrs["value"], 0)
+
+class Parser(object):
+    def __init__(self):
+        self.parser = xml.parsers.expat.ParserCreate()
+        self.parser.StartElementHandler = self.start_element
+        self.parser.EndElementHandler = self.end_element
+
+        self.struct = None
+        self.structs = {}
+        # Set of enum names we've seen.
+        self.enums = set()
+
+    def gen_prefix(self, name):
+        return f'{global_prefix.upper()}_{name}'
+
+    def start_element(self, name, attrs):
+        if name == "genxml":
+            print(pack_header)
+        elif name == "struct":
+            name = attrs["name"]
+            object_name = self.gen_prefix(safe_name(name.upper()))
+            self.struct = object_name
+
+            self.group = Group(self, None, 0, 1, name)
+            if "size" in attrs:
+                self.group.length = int(attrs["size"])
+            self.group.align = int(attrs["align"]) if "align" in attrs else None
+            self.structs[attrs["name"]] = self.group
+        elif name == "field" and self.group is not None:
+            self.group.fields.append(Field(self, attrs))
+            self.values = []
+        elif name == "enum":
+            self.values = []
+            self.enum = safe_name(attrs["name"])
+            self.enums.add(attrs["name"])
+            if "prefix" in attrs:
+                self.prefix = attrs["prefix"]
+            else:
+                self.prefix= None
+        elif name == "value":
+            self.values.append(Value(attrs))
+
+    def end_element(self, name):
+        if name == "struct":
+            if self.struct is not None:
+                self.emit_struct()
+                self.struct = None
+
+            self.group = None
+        elif name  == "field" and self.group is not None:
+            self.group.fields[-1].values = self.values
+        elif name  == "enum":
+            self.emit_enum()
+            self.enum = None
+
+    def emit_header(self, name):
+        default_fields = []
+        for field in self.group.fields:
+            if not type(field) is Field or field.exact:
+                continue
+            if field.default is not None:
+                default_fields.append(f"   .{field.name} = {field.default}")
+            elif field.type in self.structs:
+                default_fields.append(f"   .{field.name} = {{ {self.gen_prefix(safe_name(field.type.upper()))}_header }}")
+
+        if default_fields:
+            print('#define %-40s\\' % (name + '_header'))
+            print(",  \\\n".join(default_fields))
+        else:
+            print(f'#define {name}_header 0')
+        print('')
+
+    def emit_template_struct(self, name, group):
+        print("struct %s {" % name)
+        group.emit_template_struct("")
+        print("};\n")
+
+    def emit_pack_function(self, name, group):
+        print("static inline void\n%s_pack(GLOBAL uint32_t * restrict cl,\n%sconst struct %s * restrict values)\n{" %
+              (name, ' ' * (len(name) + 6), name))
+
+        group.emit_pack_function()
+
+        print("}\n")
+
+        print(f"#define {name + '_LENGTH'} {self.group.length}")
+        if self.group.align != None:
+            print(f"#define {name + '_ALIGN'} {self.group.align}")
+
+        # round up to handle 6 half-word USC structures
+        words = (self.group.length + 4 - 1) // 4
+        print(f'struct {name.lower()}_packed {{ uint32_t opaque[{words}];}};')
+
+    def emit_unpack_function(self, name, group):
+        print("static inline bool")
+        print("%s_unpack(FILE *fp, CONST uint8_t * restrict cl,\n%sstruct %s * restrict values)\n{" %
+              (name.upper(), ' ' * (len(name) + 8), name))
+
+        group.emit_unpack_function()
+
+        print("}\n")
+
+    def emit_print_function(self, name, group):
+        print("#ifndef __OPENCL_VERSION__")
+        print("static inline void")
+        print(f"{name.upper()}_print(FILE *fp, const struct {name} * values, unsigned indent)\n{{")
+
+        group.emit_print_function()
+
+        print("}")
+        print("#endif\n")
+
+    def emit_struct(self):
+        name = self.struct
+
+        self.emit_template_struct(self.struct, self.group)
+        self.emit_header(name)
+        self.emit_pack_function(self.struct, self.group)
+        self.emit_unpack_function(self.struct, self.group)
+        self.emit_print_function(self.struct, self.group)
+
+    def enum_prefix(self, name):
+        return 
+
+    def emit_enum(self):
+        e_name = enum_name(self.enum)
+        prefix = e_name if self.enum != 'Format' else global_prefix
+        print(f'enum {e_name} {{')
+
+        for value in self.values:
+            name = f'{prefix}_{value.name}'
+            name = safe_name(name).upper()
+            print(f'   {name} = {value.value},')
+        print('};\n')
+
+        print("#ifndef __OPENCL_VERSION__")
+        print("static inline const char *")
+        print(f"{e_name.lower()}_as_str(enum {e_name} imm)\n{{")
+        print("    switch (imm) {")
+        for value in self.values:
+            name = f'{prefix}_{value.name}'
+            name = safe_name(name).upper()
+            print(f'    case {name}: return "{value.name}";')
+        print('    default: return NULL;')
+        print("    }")
+        print("}")
+        print("#endif\n")
+
+    def parse(self, filename):
+        file = open(filename, "rb")
+        self.parser.ParseFile(file)
+        file.close()
+
+if len(sys.argv) < 3:
+    print("Missing input files file specified")
+    sys.exit(1)
+
+input_file = sys.argv[1]
+pack_header = open(sys.argv[2]).read()
+
+p = Parser()
+p.parse(input_file)

src/asahi/genxml/meson.build

@@ -0,0 +1,17 @@
+# Copyright 2017 Intel Corporation
+# SPDX-License-Identifier: MIT
+
+agx_pack = custom_target(
+  input : ['gen_pack.py', 'cmdbuf.xml', 'agx_pack_header.h'],
+  output : 'agx_pack.h',
+  command : [prog_python, '@INPUT@'],
+  capture : true,
+)
+
+idep_agx_pack = declare_dependency(
+  sources : [agx_pack],
+  dependencies : dep_valgrind,
+  include_directories : include_directories('.'),
+)
+
+

src/asahi/isa/agx_minifloat.h

@@ -0,0 +1,64 @@
+/*
+ * Copyright 2021 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#pragma once
+
+#include <math.h>
+#include <stdbool.h>
+#include <string.h>
+#include "util/macros.h"
+
+/* AGX includes an 8-bit floating-point format for small dyadic immediates,
+ * consisting of 3 bits for the exponent, 4 bits for the mantissa, and 1-bit
+ * for sign, in the usual order. Zero exponent has special handling. */
+
+static inline float
+agx_minifloat_decode(uint8_t imm)
+{
+   float sign = (imm & 0x80) ? -1.0 : 1.0;
+   signed exp = (imm & 0x70) >> 4;
+   unsigned mantissa = (imm & 0xF);
+
+   if (exp)
+      return ldexpf(sign * (float)(mantissa | 0x10), exp - 7);
+   else
+      return ldexpf(sign * ((float)mantissa), -6);
+}
+
+/* Encodes a float. Results are only valid if the float can be represented
+ * exactly, if not the result of this function is UNDEFINED. However, it is
+ * guaranteed that this function will not crash on out-of-spec inputs, so it is
+ * safe to call on any input. signbit() is used to ensure -0.0 is handled
+ * correctly.
+ */
+static inline uint8_t
+agx_minifloat_encode(float f)
+{
+   unsigned sign = signbit(f) ? 0x80 : 0;
+   f = fabsf(f);
+
+   /* frac is in [0.5, 1) and f = frac * 2^exp */
+   int exp = 0;
+   float frac = frexpf(f, &exp);
+
+   if (f >= 0.25) {
+      unsigned mantissa = (frac * 32.0);
+      exp -= 5; /* 2^5 = 32 */
+      exp = CLAMP(exp + 7, 0, 7);
+
+      return sign | (exp << 4) | (mantissa & 0xF);
+   } else {
+      unsigned mantissa = (f * 64.0f);
+
+      return sign | mantissa;
+   }
+}
+
+static inline bool
+agx_minifloat_exact(float f)
+{
+   float f_ = agx_minifloat_decode(agx_minifloat_encode(f));
+   return memcmp(&f, &f_, sizeof(float)) == 0;
+}

src/asahi/isa/disasm-internal.h

@@ -0,0 +1,258 @@
+/*
+ * Copyright 2025 Valve Corporation
+ * SPDX-License-Identifier: MIT
+ */
+#include <assert.h>
+#include <inttypes.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include "asahi/isa/agx_minifloat.h"
+#include "util/bitset.h"
+#include "util/u_math.h"
+#include "disasm.h"
+
+struct disasm_ctx {
+   FILE *fp;
+   bool any_operands;
+   bool error;
+};
+
+static inline uint64_t
+bits(BITSET_WORD *word, unsigned start, unsigned size, unsigned shift)
+{
+   return ((uint64_t)__bitset_extract(word, start, size)) << shift;
+}
+
+typedef void (*disasm_instr)(struct disasm_ctx *ctx, BITSET_WORD *code);
+
+struct disasm_spec {
+   const char *display;
+   disasm_instr disassemble;
+   unsigned length_bit;
+   unsigned length_short;
+   unsigned length_long;
+   BITSET_WORD mask[4];
+   BITSET_WORD exact[4];
+   BITSET_WORD known[4];
+};
+
+static void
+introduce_operand(struct disasm_ctx *ctx)
+{
+   if (ctx->any_operands) {
+      fprintf(ctx->fp, ", ");
+   } else {
+      fprintf(ctx->fp, " ");
+   }
+
+   ctx->any_operands = true;
+}
+
+enum operand_kind {
+   KIND_NONE = 0,
+   KIND_REG,
+   KIND_UNIFORM,
+   KIND_CF,
+   KIND_TS,
+   KIND_SS,
+   KIND_IMM,
+   KIND_FIMM,
+};
+
+struct operand_desc {
+   enum operand_kind kind;
+   int32_t value;
+   unsigned hint, count;
+   bool optional, size16, size32, size64, abs, neg, sx, cache, lu;
+};
+
+static void
+_print_operand(struct disasm_ctx *ctx, struct operand_desc d)
+{
+   if (d.kind == KIND_NONE) {
+      if (!d.optional) {
+         introduce_operand(ctx);
+         fprintf(ctx->fp, "_");
+      }
+
+      return;
+   }
+
+   unsigned size = d.size64 ? 64 : d.size32 ? 32 : 16;
+   bool cache = d.cache || d.hint == 2;
+   bool lu = d.lu || d.hint == 3;
+   unsigned count = MAX2(d.count, 1);
+
+   introduce_operand(ctx);
+
+   if (lu && cache) {
+      fprintf(ctx->fp, "XXX invalid cache+lu set\n");
+      ctx->error = true;
+   }
+
+   if (lu)
+      fprintf(ctx->fp, "^");
+   if (cache)
+      fprintf(ctx->fp, "$");
+
+   const char *prefixes[][2] = {
+      [KIND_REG] = {"r", "dr"}, [KIND_UNIFORM] = {"u", "du"},
+      [KIND_CF] = {"cf"},       [KIND_TS] = {"ts"},
+      [KIND_SS] = {"ss"},
+   };
+
+   if (d.kind == KIND_IMM) {
+      fprintf(ctx->fp, "%d", d.value);
+   } else if (d.kind == KIND_FIMM) {
+      float f = agx_minifloat_decode(d.value);
+
+      /* Match python's float printing */
+      if (f == (int)f)
+         fprintf(ctx->fp, "%g.0", agx_minifloat_decode(d.value));
+      else
+         fprintf(ctx->fp, "%g", agx_minifloat_decode(d.value));
+   } else if (d.kind == KIND_CF || d.kind == KIND_TS || d.kind == KIND_SS) {
+      fprintf(ctx->fp, "%s%u", prefixes[d.kind][0], d.value);
+   } else {
+      for (unsigned i = 0; i < count; ++i) {
+         if (i)
+            fprintf(ctx->fp, "_");
+
+         unsigned reg = d.value + (i * (size / 16));
+         unsigned whole = reg >> 1;
+         unsigned part = reg & 1;
+         const char *prefix = prefixes[d.kind][size == 64];
+
+         if (size == 16) {
+            fprintf(ctx->fp, "%s%u%c", prefix, whole, "lh"[part]);
+         } else {
+            if (part != 0) {
+               fprintf(ctx->fp, "# 32-bit must be expected, but got raw %u\n",
+                       reg);
+               ctx->error = true;
+            }
+
+            fprintf(ctx->fp, "%s%u", prefix, whole);
+         }
+      }
+   }
+
+   if (d.abs)
+      fprintf(ctx->fp, ".abs");
+   if (d.neg)
+      fprintf(ctx->fp, ".neg");
+   if (d.sx)
+      fprintf(ctx->fp, ".sx");
+}
+
+static void
+print_immediate(struct disasm_ctx *ctx, bool is_signed, uint32_t value)
+{
+   introduce_operand(ctx);
+   fprintf(ctx->fp, is_signed ? "%d" : "%u", value);
+}
+
+static void
+_print_enum(struct disasm_ctx *ctx, const char **arr, unsigned n,
+            unsigned value)
+{
+   if (value >= n || arr[value] == NULL) {
+      introduce_operand(ctx);
+      fprintf(ctx->fp, "XXX: Unknown enum value %u", value);
+      ctx->error = true;
+   } else if (arr[value][0] != 0) {
+      introduce_operand(ctx);
+      fprintf(ctx->fp, "%s", arr[value]);
+   }
+}
+
+#define print_enum(ctx, arr, value)                                            \
+   _print_enum(ctx, arr, ARRAY_SIZE(arr), value)
+
+static void
+print_modifier(struct disasm_ctx *ctx, const char *display, unsigned value)
+{
+   if (value) {
+      introduce_operand(ctx);
+      fprintf(ctx->fp, "%s", display);
+   }
+}
+
+static signed
+_disassemble_instr(BITSET_WORD *code, FILE *fp, struct disasm_spec *specs,
+                   unsigned nr_specs, unsigned offset, bool verbose)
+{
+   BITSET_WORD tmp[4];
+   memcpy(tmp, code, sizeof(tmp));
+   struct disasm_spec *spec = NULL;
+   unsigned n = 0;
+   bool match = false;
+   BITSET_WORD masked[4] = {0};
+
+   for (unsigned i = 0; i < nr_specs; ++i) {
+      spec = &specs[i];
+
+      n = spec->length_short;
+      if (BITSET_TEST(tmp, spec->length_bit)) {
+         n = spec->length_long;
+      }
+
+      match = true;
+      for (unsigned i = 0; i < ARRAY_SIZE(tmp); ++i) {
+         masked[i] = tmp[i];
+
+         uint32_t bytes_left = (n - (i * 4));
+         if (bytes_left < 4) {
+            masked[i] &= ((1 << (bytes_left * 8)) - 1);
+         }
+
+         if ((masked[i] & spec->mask[i]) != spec->exact[i]) {
+            match = false;
+            break;
+         }
+      }
+
+      if (match)
+         break;
+   }
+
+   struct disasm_ctx ctx = {.fp = fp};
+
+   if (match) {
+      BITSET_WORD unknown[4];
+      __bitset_andnot(unknown, masked, spec->known, ARRAY_SIZE(masked));
+      int i;
+      BITSET_FOREACH_SET(i, unknown, n * 8) {
+         fprintf(fp, "# XXX: Unknown bit set %u\n", i);
+         ctx.error = true;
+      }
+   } else {
+      n = 2;
+   }
+
+   if (verbose) {
+      fprintf(fp, "%4x: ", offset);
+      for (unsigned i = 0; i < n; ++i) {
+         fprintf(fp, "%02x", __bitset_extract(masked, i * 8, 8));
+      }
+      for (unsigned i = n; i < 11; ++i) {
+         fprintf(fp, "  ");
+      }
+      fprintf(fp, " ");
+   }
+
+   if (!match) {
+      fprintf(fp, "<unknown instr>\n");
+      return -n;
+   }
+
+   fprintf(fp, "%s", spec->display);
+
+   if (spec->disassemble)
+      spec->disassemble(&ctx, masked);
+
+   fprintf(fp, "\n");
+
+   return ctx.error ? -n : n;
+}

src/asahi/isa/disasm.h

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2025 Valve Corporation
+ * SPDX-License-Identifier: MIT
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include "util/bitset.h"
+
+signed agx2_disassemble_instr(BITSET_WORD *code, FILE *fp, unsigned offset,
+                              bool verbose);
+
+static inline bool
+agx2_disassemble(uint8_t *code, size_t max_len, FILE *fp)
+{
+   int i = 0;
+   bool errors = false;
+
+   while (i < max_len) {
+      /* Break on trap */
+      if (code[i + 0] == 0x08 && code[i + 1] == 0x00)
+         break;
+
+      signed ret =
+         agx2_disassemble_instr((BITSET_WORD *)(code + i), fp, i, true);
+      if (ret < 0)
+         fprintf(fp, "XXX error here\n");
+
+      errors |= ret < 0;
+      i += abs(ret);
+   }
+
+   return errors;
+}

src/asahi/isa/gen-disasm.py

@@ -0,0 +1,168 @@
+# Copyright 2016 Intel Corporation
+# Copyright 2016 Broadcom
+# Copyright 2020 Collabora, Ltd.
+# Copyright 2025 Valve Corporation
+# SPDX-License-Identifier: MIT
+
+from mako.template import Template
+from mako import exceptions
+from isa import isa
+
+# XXX: deduplicate
+def to_alphanum(name):
+    substitutions = {
+        ' ': '_',
+        '/': '_',
+        '[': '',
+        ']': '',
+        '(': '',
+        ')': '',
+        '-': '_',
+        ':': '',
+        '.': '',
+        ',': '',
+        '=': '',
+        '>': '',
+        '#': '',
+        '&': '',
+        '*': '',
+        '"': '',
+        '+': '',
+        '\'': '',
+    }
+
+    for i, j in substitutions.items():
+        name = name.replace(i, j)
+
+    return name
+
+def safe_name(name):
+    name = to_alphanum(name)
+    if not name[0].isalpha():
+        name = '_' + name
+
+    return name.lower()
+
+def chunk(x):
+    chunks = [hex((x >> (32 * i)) & ((1 << 32) - 1)) for i in range(4)]
+    while len(chunks) > 1 and chunks[-1] == '0x0':
+        chunks.pop()
+
+    return f'{{{", ".join(chunks)}}}'
+
+def instr_is_trivial(I):
+    return len(I.dests + I.sources + I.immediates + I.modifiers) == 0
+
+def extract_bits(x, bitset="r"):
+    parts = []
+    n = 0
+
+    for r in x.bits.ranges:
+        parts.append(f'bits({bitset}, {r.start}, {r.size}, {n})')
+        n += r.size
+
+    return ' | '.join(parts)
+
+template = """
+#include "disasm-internal.h"
+
+% for en in isa.enums:
+UNUSED static const char *enum_${safe_name(isa.enums[en].kind)}[] = {
+% for n, v in isa.enums[en].values.items():
+    [${n}] = "${v.display or ''}",
+% endfor
+};
+% endfor
+
+% for k, enc in isa.encodings.items():
+static void
+print_${k}(struct disasm_ctx *ctx, BITSET_WORD *orig, uint64_t v, bool source)
+{
+   BITSET_WORD r[] = { v, v >> 32 };
+% for i, case in enumerate(enc.cases):
+   ${"} else " if i > 0 else ""}if ((v & ${hex(case.exact.mask)}ull) == ${hex(case.exact.value)}ull) {
+% for mod in case.modifiers:
+% if mod.name == 'hint':
+       if (${extract_bits(mod)} == 0) {
+          fprintf(ctx->fp, "# missing hint");
+       }
+
+% endif
+% endfor
+       _print_operand(ctx, (struct operand_desc){
+% if case.kind is not None:
+          .kind = KIND_${case.kind.name.upper()},
+% if case.kind.kind == 'signed':
+          .value = util_sign_extend(${extract_bits(case.kind)} << ${case.kind.shift}, ${case.kind.bits.size()}),
+% else:
+          .value = ${extract_bits(case.kind)} << ${case.kind.shift},
+% endif
+% endif
+% for mod in case.modifiers:
+% if mod.name == 'mask':
+          .count = util_bitcount(${extract_bits(mod)}),
+% elif mod.name == 'count':
+          .count = (${extract_bits(mod)}) ?: 4,
+% elif mod.name == 'zx':
+          .sx = ${extract_bits(mod)} ^ 1,
+% else:
+          .${mod.name} = ${extract_bits(mod)},
+% endif
+% endfor
+% for prop, v in case.properties.items():
+          .${prop} = ${v},
+% endfor
+          .optional = source,
+       });
+% endfor
+   } else {
+       fprintf(ctx->fp, "# XXX: Invalid value 0x%"PRIx64" for ${k}", v);
+       ctx->error = true;
+   }
+}
+
+% endfor
+
+% for k, instr in isa.instrs.items():
+% if not instr_is_trivial(instr):
+static void
+print_${instr.name}(struct disasm_ctx *ctx, BITSET_WORD *r)
+{
+% for i, op in enumerate(instr.dests):
+   print_${op.kind}(ctx, r, ${extract_bits(op)}, false);
+% endfor
+% for i, op in enumerate(instr.sources):
+   print_${op.kind}(ctx, r, ${extract_bits(op)}, true);
+% endfor
+% for imm in instr.immediates:
+   print_immediate(ctx, ${'true' if imm.kind == 'signed' else 'false'}, ${extract_bits(imm)});
+% endfor
+% for mod in instr.modifiers:
+% if mod.name != 'length':
+% if mod.kind in isa.enums:
+   print_enum(ctx, enum_${mod.kind}, ${extract_bits(mod)});
+% else:
+   print_modifier(ctx, "${mod.display}", ${extract_bits(mod)});
+% endif
+% endif
+% endfor
+}
+
+% endif
+% endfor
+struct disasm_spec decodings[] = {
+% for k, instr in isa.instrs.items():
+    { "${instr.display if instr_is_trivial(instr) else f'{instr.display:<8}'}", ${'NULL' if instr_is_trivial(instr) else f'print_{instr.name}'}, ${instr.length_bit}, ${instr.length[0]}, ${instr.length[-1]}, ${chunk(instr.exact.mask)}, ${chunk(instr.exact.value)}, ${chunk(instr.mask())} },
+% endfor
+};
+
+signed
+agx2_disassemble_instr(BITSET_WORD *code, FILE *fp, unsigned offset, bool verbose)
+{
+  return _disassemble_instr(code, fp, decodings, ARRAY_SIZE(decodings), offset, verbose);
+}"""
+
+try:
+    print(Template(template).render(isa = isa, safe_name = safe_name, chunk = chunk, instr_is_trivial = instr_is_trivial, extract_bits = extract_bits))
+except:
+    print(exceptions.text_error_template().render())

src/asahi/isa/isa.py

@@ -0,0 +1,321 @@
+# Copyright 2025 Valve Corporation
+# SPDX-License-Identifier: MIT
+
+from dataclasses import dataclass
+from functools import reduce
+import xml.etree.ElementTree as ET
+import sys
+import copy
+import os
+
+def ensure(cond, msg):
+    if not cond:
+        print(msg)
+        sys.exit(1)
+
+class BitRange:
+    def __init__(self, start, end):
+        assert(start <= end)
+        self.start = start
+        self.end = end
+
+        if self.start >= 0:
+            self.size = end - start + 1
+            self.size_mask = (1 << self.size) - 1
+            self.mask = self.size_mask << start
+
+    def parse(spec: str):
+        if ':' in spec:
+            start, end = [int(x) for x in spec.split(':')]
+            return BitRange(start, end)
+        else:
+            return BitRange(int(spec), int(spec))
+
+    def specialize(self, length):
+        return BitRange(self.start % length, self.end % length)
+
+    def __repr__(self):
+        if self.start == self.end:
+            return f'{self.start}'
+        else:
+            return f'{self.start}:{self.end}'
+
+class Bits:
+    def __init__(self, spec = None):
+        if spec is not None:
+            self.ranges = [BitRange.parse(r) for r in spec.split(' ')]
+        else:
+            self.ranges = []
+
+    def size(self) -> int:
+        return sum([x.size for x in self.ranges])
+
+    def mask(self) -> int:
+        masks = [r.mask for r in self.ranges]
+        union = 0
+        for mask in masks:
+            union |= mask
+
+        # No overlap
+        assert(sum(masks) == union)
+        return union
+
+    def extract(self, x):
+        accum = 0
+        n = 0
+
+        for r in self.ranges:
+            accum |= ((x >> r.start) & r.size_mask) << n
+            n += r.size
+
+        return accum
+
+    def specialize(self, length):
+        obj = copy.copy(self)
+        obj.ranges = [r.specialize(length) for r in obj.ranges]
+        return obj
+
+    def __repr__(self):
+        return ' '.join([str(r) for r in self.ranges])
+
+def parse_bitstring(bits):
+    value = 0
+
+    # Reverse to deal with the convention
+    bits = bits[::-1]
+
+    for i, bit in enumerate(bits):
+        assert(bit in ['0', '1'])
+        if bit == '1':
+            value |= (1 << i)
+
+    return value
+
+# Spread the bits of value out according to the 1s in mask.
+# Similar to pdep in x86.
+def deposit_bits(mask, value):
+    accum = 0
+    value_i = 0
+
+    assert(mask < (1 << 128))
+    for m in range(128):
+        if mask & (1 << m):
+            if (value & (1 << value_i)) != 0:
+                accum |= (1 << m)
+
+            value_i += 1
+
+    assert((accum & ~mask) == 0)
+    return accum
+
+def substitute(text, substitutions):
+    if text in substitutions:
+        return substitutions[text]
+    else:
+        return text
+
+class Exact:
+    def __init__(self, el = None, subst = None):
+        bits = 0
+        if el is not None:
+            self.mask = Bits(el.attrib['bit']).mask()
+            if el.tag != 'zero':
+                text = substitute(el.text, subst)
+                bits = parse_bitstring(text)
+                assert(len(text) == self.mask.bit_count())
+        else:
+            self.mask = 0
+
+        self.value = deposit_bits(self.mask, bits)
+
+    def union(self, other):
+        self.value |= other.value
+        self.mask |= other.mask
+
+class EnumValue:
+    def __init__(self, el):
+        self.name = el.text
+        self.label = el.attrib.get('label')
+        self.default = bool(el.attrib.get('default', False))
+        self.display = None if self.default else self.name
+
+class Enum:
+    def __init__(self, el):
+        self.name = el.attrib['name']
+        self.kind = el.attrib['kind']
+        self.values = {}
+
+        for child in el:
+            self.values[int(child.attrib['value'])] = EnumValue(child)
+
+class Immediate:
+    def __init__(self, el, name):
+        self.name = name
+        self.kind = el.attrib.get('kind', 'int')
+        self.bits = Bits(el.attrib['bit'])
+        self.shift = int(el.attrib.get('shift', 0))
+
+class Operand:
+    def __init__(self, el, is_dest):
+        self.kind = el.attrib['kind']
+        self.bits = Bits(el.attrib['bit'])
+        self.is_dest = is_dest
+
+    def specialize(self, length):
+        obj = copy.copy(self)
+        obj.bits = obj.bits.specialize(length)
+        return obj
+
+class EncodingCase:
+    def __init__(self, el):
+        self.exact = Exact()
+        self.modifiers = []
+        self.kind = None
+        self.properties = {}
+
+        for ch in el:
+            if ch.tag in ["exact", "zero"]:
+                self.exact.union(Exact(ch, {}))
+            elif ch.tag == "immediate":
+                assert(self.kind is None)
+                self.kind = Immediate(ch, ch.attrib['name'])
+            elif ch.tag == "modifier":
+                self.modifiers.append(Modifier(ch, ch.attrib['name']))
+            elif ch.tag == "property":
+                self.properties[ch.attrib['name']] = ch.text or '1'
+
+    def union(self, other):
+        self.exact.union(other.exact)
+        self.modifiers = other.modifiers + self.modifiers
+        self.properties.update(other.properties)
+
+        assert(self.kind is None or other.kind is None)
+        self.kind = self.kind or other.kind
+
+class Encoding:
+    def __init__(self, el):
+        base = EncodingCase(el)
+        cases = [EncodingCase(ch) for ch in el if ch.tag == 'case']
+
+        if len(cases) == 0:
+            self.cases = [base]
+        else:
+            self.cases = cases
+            for c in self.cases:
+                c.union(base)
+
+class Modifier:
+    def __init__(self, el, name):
+        self.name = name
+        self.display = el.get('display', name)
+        self.kind = el.get('kind', 'int')
+        self.bits = Bits(el.attrib['bit'])
+
+class Instruction:
+    def __init__(self, xml, name, display, substitutions, defs):
+        length = [int(x) for x in xml.attrib['length'].split('/')]
+        exact = Exact()
+        mods = []
+        dests = []
+        srcs = []
+        imms = []
+
+        for el in xml:
+            if el.tag in ["exact", "zero"]:
+                exact.union(Exact(el, substitutions))
+            elif el.tag == "immediate":
+                imms.append(Immediate(el, el.attrib['name']))
+            elif el.tag == "src":
+                srcs.append(Operand(el, False))
+            elif el.tag == "dest":
+                dests.append(Operand(el, True))
+            elif el.tag == "modifier":
+                mods.append(Modifier(el, el.attrib['name']))
+            elif el.tag in defs:
+                d = defs[el.tag]
+                if isinstance(d, Operand):
+                    spec = d.specialize(length[-1] * 8)
+                    if spec.is_dest:
+                        dests.append(spec)
+                    else:
+                        srcs.append(spec)
+                elif isinstance(d, Modifier):
+                    mods.append(d)
+                elif isinstance(d, Immediate):
+                    imms.append(d)
+                else:
+                    assert(0)
+            elif el.tag == 'ins':
+                assert(xml.tag == 'group')
+            else:
+                print(name)
+                assert(0)
+
+        # Infer length bit if needed
+        if len(length) > 1 and not any([x.name == 'length' for x in mods]):
+            mods.append(defs['length'])
+
+        self.length_spec = None
+        self.length_bit = 0
+        for mod in mods:
+            if mod.name == 'length':
+                self.length_spec = mod
+                self.length_bit = mod.bits.ranges[0].start
+        self.name = name
+        self.display = display or name
+        self.length = length
+        self.exact = exact
+        self.modifiers = mods
+        self.dests = dests
+        self.sources = srcs
+        self.immediates = imms
+
+    def mask(self):
+        mask = self.exact.mask
+        parts = self.modifiers + self.immediates + self.sources + self.dests
+        bits = [x.bits.mask() for x in parts]
+        return self.exact.mask | reduce(lambda x, y: x | y, bits, 0)
+
+class ISA:
+    def __init__(self, el):
+        self.enums = {}
+        self.instrs = {}
+        self.encodings = {}
+        defs = {}
+
+        for child in el:
+            name = child.get('name')
+
+            if child.tag == "define":
+                if 'src' in child.attrib:
+                    defs[child.attrib['src']] = Operand(child, False)
+                elif 'dest' in child.attrib:
+                    defs[child.attrib['dest']] = Operand(child, True)
+                elif 'modifier' in child.attrib:
+                    name = child.attrib['modifier']
+                    defs[child.attrib['modifier']] = Modifier(child, name)
+                elif 'immediate' in child.attrib:
+                    name = child.attrib['immediate']
+                    defs[child.attrib['immediate']] = Immediate(child, name)
+                else:
+                    assert(0)
+            elif child.tag == "encoding":
+                self.encodings[child.attrib['name']] = Encoding(child)
+            elif child.tag == "enum":
+                enum = Enum(child)
+                self.enums[enum.kind] = enum
+            elif child.tag == "ins":
+                ins = Instruction(child, name, child.get('display'), {}, defs)
+                ensure(ins.name not in self.instrs, f'Multiple instructions named {ins.name}')
+                self.instrs[ins.name] = ins
+            elif child.tag == "group":
+                ensure(any([c.tag == "ins" for c in child]), f'No instructions in group {name}')
+
+                for c in child:
+                    if c.tag == "ins":
+                        ins = Instruction(child, c.attrib['name'], c.get('display'), c.attrib, defs)
+                        ensure(ins.name not in self.instrs, f'Multiple instructions named {ins.name}')
+                        self.instrs[ins.name] = ins
+
+xmlfile = os.path.join(os.path.dirname(__file__), 'AGX2.xml')
+isa = ISA(ET.parse(xmlfile).getroot())

src/asahi/isa/meson.build

@@ -0,0 +1,43 @@
+# Copyright 2025 Valve Corporation
+# Copyright 2020-2021 Collabora
+# SPDX-License-Identifier: MIT
+
+agx2_disasm_c = custom_target(
+  input : ['gen-disasm.py', 'AGX2.xml'],
+  output : 'agx2_disasm.c',
+  command : [prog_python, '@INPUT@'],
+  capture : true,
+  depend_files : files('isa.py'),
+)
+
+libagx2_disasm = static_library(
+  'agx2_disasm',
+  agx2_disasm_c,
+  include_directories : [inc_include, inc_src],
+  c_args : [no_override_init_args],
+  gnu_symbol_visibility : 'hidden',
+  build_by_default : false,
+)
+
+if with_tests
+  test(
+    'agx2_disasm',
+    executable(
+      'agx2_disasm_test',
+      files('test/test-disassembler.c'),
+      c_args : [c_msvc_compat_args, no_override_init_args],
+      gnu_symbol_visibility : 'hidden',
+      include_directories : [inc_include, inc_src],
+      link_with : [libagx2_disasm],
+    ),
+    suite : ['asahi'],
+    args : files('test/cases.txt'),
+  )
+
+ test(
+    'agx2_python_disasm',
+    prog_python,
+    args : files('test-disasm.py', 'test/cases.txt'),
+    suite : ['asahi']
+  )
+endif

src/asahi/isa/test-disasm.py

@@ -0,0 +1,46 @@
+# Copyright 2025 Valve Corporation
+# SPDX-License-Identifier: MIT
+
+import sys
+import os
+import textwrap
+import binascii
+from test.disasm import disasm
+
+errors = False
+new_cases = []
+
+for case in open(sys.argv[1]).read().strip().split('\n'):
+    first = case.split(' ')[0]
+    ref = case[len(first):].strip()
+
+    # Extract bytes
+    lst = [int(x, 16) for x in textwrap.wrap(first, 2)]
+
+    # Make sure we don't depend on zeroes in upper bits
+    padded = lst + [0xCA, 0xFE, 0xBA, 0xBE] * 16
+    raw = sum([x << (8 * i) for i, x in enumerate(padded)])
+
+    error = False
+    length, asm = disasm(raw)
+    error |= (length != len(lst))
+
+    actual = f'{str(binascii.hexlify(bytes(lst)).decode()):<20} {asm}\n'
+    new_cases.append(actual)
+    error |= (actual[:-1] != case)
+
+    if error:
+        print(case)
+        print(actual)
+    errors |= error
+
+# If there were errors, optionally update expectations.
+if len(sys.argv) >= 3:
+    assert(sys.argv[2] in ['-u', '--update'])
+    open(sys.argv[1], 'w').write(''.join(new_cases))
+
+if errors:
+    print('Fail.')
+    sys.exit(1)
+else:
+    print('Pass.')

src/asahi/isa/test/cases.txt

@@ -0,0 +1,555 @@
+00c086fdffff         jmp_any  -634
+02000000040001e4     csel     r0l, 0.0, r0l, 0, r0l, fgtn
+0200400000421024     csel     r0l, r0l, 0.0, r1l, r0h, flt
+0200404004404064     csel     r0l, r0l, r2l, r0l, r2l, fltn
+02020010040011e4     csel     r0h, 0.0, r0h, 0, r0h, fgtn
+02020010044300b0     csel     r0h, 0.0, r0h, r1h, 0, fgte
+02040010044200a4     csel     r1l, 0.0, r0h, r1l, r0l, fgte
+0204410444432004     csel     r1l, r0h.abs, r0l.abs, r1h, r1l, feq
+02084654044430a4     csel     r2l, r3l.abs, r2h, r2l, r1h, fgte
+020a3020044250a4     csel     r2h, 1.0, r1l, r1l, r2h, fgte
+020c0010044020a4     csel     r3l, 0.0, r0h, r0l, r1l, fgte
+020e3020040071a4     csel     r3h, 1.0, r1l, 0, r3h, fgte
+021900c0249001b0     csel     r6, 0.0, r6, u8, 0, fgte
+02194ce2244ce064     csel     r6, r6, r7, r6, r7, fltn
+02214c02004ea024     csel     r8, r6, 0.0, r7, r5, flt
+0501000d10c0f200     load     r0_r1_r2_r3, du8, 0, i32, xyzw, a
+0501000d80c01200     load     r0, du64, 0, i32, x, a
+0501480d60c07200     load     r0_r1_r2, du52, 4, i32, xyz, a
+0501a80d69c03200     load     r0_r1, du52, 154, i32, xy, a
+0509480e00c01200     load     r1, du4, r2, i32, x, a
+0511440500c03200     load     r2_r3, dr2, 4, i32, xy, a
+05116e0621c01200     load     r2, dr23, r11, i32, x, a
+0521004500c01200     load     r4, dr0, 0, i32, x, b
+0521040500c01200     load     r4, dr2, 0, i32, x, a
+0521100500c01200     load     r4, dr0, 1, i32, x, a
+0521340510c01200     load     r4, dr10, 3, i32, x, a
+05310c0500c03200     load     r6_r7, dr6, 0, i32, xy, a
+0531200510c01200     load     r6, dr8, 2, i32, x, a
+0531400510c01200     load     r6, dr8, 4, i32, x, a
+0531cc0e60c03200     load     r6_r7, du54, r6, i32, xy, a
+0541000510c03200     load     r8_r9, dr8, 0, i32, xy, a
+0541000e00c07200     load     r8_r9_r10, du0, r0, i32, xyz, a
+05410c0500c03200     load     r8_r9, dr6, 0, i32, xy, a
+05410c0500c0f200     load     r8_r9_r10_r11, dr6, 0, i32, xyzw, a
+05410c4500c03200     load     r8_r9, dr6, 0, i32, xy, b
+05412c0500c01200     load     r8, dr6, 2, i32, x, a
+0541f00f0fc8f20f     load     r8_r9_r10_r11, du0, 4095, lsl 2, i32, xyzw, a
+0551300510c01200     load     r10, dr8, 3, i32, x, a
+05610c0e60c0f200     load     r12_r13_r14_r15, du54, r0, i32, xyzw, a
+0561e44610c01200     load     r12, dr10, r7, i32, x, b
+09010004f0fc0003     st_tile  r0_r1_r2_r3, 0, 255, 0, u8norm, xyzw
+0901000e70fc0003     st_tile  r0_r1_r2, 0, 255, 0, rg11b10f_rtz, xyz
+0901000f70fc0003     st_tile  r0_r1_r2, 0, 255, 0, rgb9e5_rtz, xyz
+09100004f0fc0003     st_tile  r4l_r4h_r5l_r5h, 0, 255, 0, u8norm, xyzw
+09111204f8048000     st_tile  r4_r5_r6_r7, r9l_r9h, 1, 0, explicit_coords, u8norm, xyzw
+0a0040c0             log2     r0l, r0l
+0a0040d0             exp2     r0l, r0l
+0a064380             rcp      r1h, r1h
+0a084580             rcp      r2l, r2h
+0a0a4280             rcp      r2h, r1l
+0a0a4590             rsqrt    r2h, r2h
+0a0d4692             rsqrt    r3, r3
+0a0e4790             rsqrt    r3h, r3h
+0e0100c065002000     iadd     r0, 0, r14.sx, lsl 4
+0e01401600000000     iadd     r0, r0.sx, 1
+0e01402664000000     iadd     r0, r0.sx, r1.sx
+0e01408664000000     iadd     r0, r0.sx, r4.sx
+0e014aa6e4000000     iadd     r0, r5.sx, r5.sx, lsl 1
+0e03408334000000     iadd     dr0, dr0, dr4
+0e05420658020000     iadd     r1, r1.sx, u64.sx
+0e05428658000000     iadd     r1, r1.sx, u4.sx
+0e05428664000000     iadd     r1, r1.sx, r4.sx
+0e09424658000000     iadd     r2, r1.sx, u2.sx
+0e09442658020000     iadd     r2, r2.sx, u65.sx
+0e0944a658000000     iadd     r2, r2.sx, u5.sx
+0e0b44c334000000     iadd     dr2, dr2, dr6
+0e0d408664000000     iadd     r3, r0.sx, r4.sx
+0e0d446658000000     iadd     r3, r2.sx, u3.sx
+0e0d464658020000     iadd     r3, r3.sx, u66.sx
+0e14192500000000     iadd     r5l, u12h.sx, 2
+0e14193500000000     iadd     r5l, u12h.sx, 3
+0e194c1600000000     iadd     r6, r6.sx, 1
+0e194ec664000000     iadd     r6, r7.sx, r6.sx
+0e1b4c4335000000     iadd     dr6, dr6, dr10
+0e1b4cc300000000     iadd     dr6, dr6, 12
+0e1b504335000000     iadd     dr6, dr8, dr10
+0e1b50c334000000     iadd     dr6, dr8, dr6
+0e1d01e064000000     iadd     r7, 1, r7.sx
+0e201a1500000000     iadd     r8l, u13l.sx, 1
+0e23448300000000     iadd     dr8, dr2, 8
+0e23504300000000     iadd     dr8, dr8, 4
+0e23504335000000     iadd     dr8, dr8, dr10
+0e23508300000000     iadd     dr8, dr8, 8
+0e23540335000000     iadd     dr8, dr10, dr8
+0e25502665000000     iadd     r9, r8.sx, r9.sx
+0e2900c064002000     iadd     r10, 0, r6.sx, lsl 4
+0e29540665002000     iadd     r10, r10.sx, r8.sx, lsl 4
+0e2b444335000000     iadd     dr10, dr2, dr10
+0e2b448300000000     iadd     dr10, dr2, 8
+0e2b508301000000     iadd     dr10, dr8, 24
+0e314c1600000000     iadd     r12, r6.sx, 1
+0e31581600000000     iadd     r12, r12.sx, 1
+0e690160a9000000     iadd     r26, 1, $r11, lsl 1
+10c060ffffff         call     -160
+10c08affffff         call     -118
+11000880             st_var   r0, r4l
+11008280             st_var   r0, 2
+11040680             st_var   r1, r3l
+11080880             st_var   r2, r4l
+110c0880             st_var   r3, r4l
+11100a80             st_var   r4, r5l
+11140c80             st_var   r5, r6l
+11141080             st_var   r5, r8l
+11208080             st_var   r8, 0
+11248180             st_var   r9, 1
+112c8380             st_var   r11, 3
+11308480             st_var   r12, 4
+11348580             st_var   r13, 5
+11388680             st_var   r14, 6
+113c8780             st_var   r15, 7
+12004c5200404010     csel     r0l, r6, 5, r0l, 4, ieq
+1201401200401050     csel     r0, r0, 1, r0, 1, ugt
+1202506004506044     csel     r0h, r8l, r3l, r8l, r3l, ugt
+12114802004c0010     csel     r4, r4, 0, r6, 0, ieq
+12194802004c0010     csel     r6, r4, 0, r6, 0, ieq
+12194c12004c1050     csel     r6, r6, 1, r6, 1, ugt
+12194e12004e1050     csel     r6, r7, 1, r7, 1, ugt
+1235548225548045     csel     r13, r10, r12, r10, r12, ugt
+12824262180101300002 csel     r0h, r1, u67, 1, 0, ult
+12824482184100300002 csel     r0h, r2, u68, r0h, 0, ult
+1402                 ret      r1
+1500048d20501400     atomic   _, r10, du18, 0, add, coherent1, a
+1500248c10500600     atomic   _, r3, du10, r1.sx, add, coherent1, a
+1500248c90500600     atomic   _, r3, du74, r1.sx, add, coherent1, a
+1502248c10500600     atomic   _, r3, du10, r1.sx, and, coherent1, a
+1508a48c10d00800     atomic   r1, r4, du10, r5.sx, add, coherent1, a
+150aa48c10d00800     atomic   r1, r4, du10, r5.sx, and, coherent1, a
+1520008d20d00200     atomic   r4, r1, du16, 0, add, coherent1, a
+1520048d20d00200     atomic   r4, r1, du18, 0, add, coherent1, a
+15302c8d20d00c00     atomic   r6, r6, du22, 2, add, coherent1, a
+160081f0             hmul     r0l, $r0h, 0.234375
+168040101100         hmul     r0l, r0l, u8h
+168040700400         hmul     r0l, r0l, r3h
+168241700400         hmul     r0h, r0h, r3h
+168442400400         hmul     r1l, r1l, r2l
+168442422400         hmul     r1l, r1l.abs, r2l.abs
+168442700400         hmul     r1l, r1l, r3h
+168640300400         hmul     r1h, r0l, r1h
+168643500400         hmul     r1h, r1h, r2h
+168842300400         hmul     r2l, r1l, r1h
+168844500400         hmul     r2l, r2l, r2h
+168a44400400         hmul     r2h, r2l, r2l
+168a46400400         hmul     r2h, r3l, r2l
+168a48500400         hmul     r2h, r4l, r2h
+168a4aa00400         hmul     r2h, r5l, r5l
+168c46500400         hmul     r3l, r3l, r2h
+168e42200400         hmul     r3h, r1l, r1l
+168e45500400         hmul     r3h, r2h, r2h
+168e4a500400         hmul     r3h, r5l, r2h
+169044700400         hmul     r4l, r2l, r3h
+169249500400         hmul     r4h, r4h, r2h
+16944a000001         hmul     r5l, r5l, 2.0
+19918000548000000800 latomic  r4, r4, 0, add
+19918002548000000800 latomic  r4, r4, 0, xchg
+19918004548000000800 latomic  r4, r4, 0, umin
+19918006548000000800 latomic  r4, r4, 0, umax
+19918008548000000800 latomic  r4, r4, 0, and
+19918009548000000800 latomic  r4, r4, 0, or
+1991800a548000000800 latomic  r4, r4, 0, xor
+19998003548000000c00 latomic  r6, r6_r7, 0, cmpxchg
+1a0182f2             fmul     r0, $r1, 0.234375
+1a9156222500         fmul     r4, r11, r9
+1aa556222500         fmul     r9, r11, r9
+1aad52222500         fmul     r11, r9, r9
+1e0188a558000000     imul     r0, u4.sx, u5.sx
+1e0542a658440602     imadd    r1, r1.sx, u69.sx, r2.sx
+1e09480659440602     imadd    r2, r4.sx, u72.sx, r2.sx
+1e0d422664460600     imadd    r3, r1.sx, r1.sx, r3.sx
+1e0d444664460600     imadd    r3, r2.sx, r2.sx, r3.sx
+1e0d466664000000     imul     r3, r3.sx, r3.sx
+1e1b428201000001     imul     dr6, r1, 88
+1e23428201000001     imul     dr8, r1, 88
+20000600             jmp_if_skipping_doorbell 6
+20c01c010000         jmp_none 284
+20c060010000         jmp_none 352
+20c09e000000         jmp_none 158
+20c0dc000000         jmp_none 220
+20c0fa000000         jmp_none 250
+2199014000000000     iter     r6, cf1
+219d004000000000     iter     r7, cf0
+2602c1f0             hadd     r0h, ^r0h, 0.234375
+268440000300         hadd     r1l, r0l, 1.0
+269642100400         hadd     r5h, r1l, r0h
+2820                 doorbell 32
+2830                 doorbell 48
+2831                 doorbell 49
+2981800214000000     lstore   r0, 0, i32, x
+2989800234000000     lstore   r2_r3, 0, i32, xy
+298d040214000000     lstore   r3, r2l, 0, i32, x
+29914b02f4000000     lstore   r4_r5_r6_r7, u5h, 0, i32, xyzw
+29914c02f4000000     lstore   r4_r5_r6_r7, u6l, 0, i32, xyzw
+2991800274000000     lstore   r4_r5_r6, 0, i32, xyz
+29918002f4000000     lstore   r4_r5_r6_r7, 0, i32, xyzw
+2a05c2f2             fadd     r1, ^r1, 0.234375
+2a81546a2500         fadd     r0, r10.neg, r11
+2a8441040002         fadd     r1l, r0h.abs, -0.0
+2a8441080002         fadd     r1l, r0h.neg, -0.0
+2a9d4a0a0002         fadd     r7, r5.neg, -0.0
+2e01001000400200     shl      r0, 1, r0
+2e014466241e0000     extr     r0, r2, r3, 30, 0
+2e0158a625180000     extr     r0, r12, r13, 24, 0
+2e05001000420200     shl      r1, 1, r1
+2e0500c024080000     shl      r1, r6, 8
+2e055c06241c0000     extr     r1, r14, r0, 28, 0
+2e11004024070000     shl      r4, r2, 7
+2e194ce6241c0000     extr     r6, r6, r7, 28, 0
+2e81000024420200     shr      r0, r0, r1
+2e81008024020000     shr      r0, r4, 2
+2e850040e41b0000     bfeil    r1, 0, r2, 27, 3
+2e8500e025040400     bfeil    r1, 0, r15, 4, 4
+2e91008064090000     bfeil    r4, 0, r4, 9, 1
+2e99008024020000     shr      r6, r4, 2
+2e9900c024500200     shr      r6, r6, r8
+2e9d00e024500200     shr      r7, r7, r8
+2e9d00e0a40a0c00     bfeil    r7, 0, r7, 10, 14
+2ea100e024180400     bfeil    r8, 0, r7, 24, 4
+2ead0020251c0000     shr      r11, r9, 28
+2ead00c0241c0000     shr      r11, r6, 28
+3111081400622f01     smpf     r4_r5_r6_r7, ts0, ss1, r4, u10l, 2d, xyzw, lod_min, a
+31818000ca6321481b201e00 smpf     r0, du54, r5, r4l, r0, u64l, r15, shadow, 2d_array, x, lod_min, a
+31910400c2620f411b008001 smpf     r4_r5_r6_r7, du54, r1, r0h, r2, 0, 2d, xyzw, auto_lod, gather_g, a
+31910400c2620f411b008002 smpf     r4_r5_r6_r7, du54, r1, r0h, r2, 0, 2d, xyzw, auto_lod, gather_b, a
+31910400c2620f411b008003 smpf     r4_r5_r6_r7, du54, r1, r0h, r2, 0, 2d, xyzw, auto_lod, gather_a, a
+31910406c26293411b200000 smpf     r4_r5, du54, r1, r0h, r2, u67l_u67h, 2d, xy, auto_lod_bias_min, a
+31a10400c2620f411b008000 smpf     r8_r9_r10_r11, du54, r1, r0h, r2, 0, 2d, xyzw, auto_lod, gather_r, a
+31a10400c2620f411b008001 smpf     r8_r9_r10_r11, du54, r1, r0h, r2, 0, 2d, xyzw, auto_lod, gather_g, a
+31a10400c2620f411b008002 smpf     r8_r9_r10_r11, du54, r1, r0h, r2, 0, 2d, xyzw, auto_lod, gather_b, a
+31a10400c2620f411b008003 smpf     r8_r9_r10_r11, du54, r1, r0h, r2, 0, 2d, xyzw, auto_lod, gather_a, a
+31a10400c2620f415b008000 smpf     r8_r9_r10_r11, du54, r1, r0h, r2, 0, sparse, 2d, xyzw, auto_lod, gather_r, a
+31a10400c2620f415b008001 smpf     r8_r9_r10_r11, du54, r1, r0h, r2, 0, sparse, 2d, xyzw, auto_lod, gather_g, a
+31a10400c2620f415b008002 smpf     r8_r9_r10_r11, du54, r1, r0h, r2, 0, sparse, 2d, xyzw, auto_lod, gather_b, a
+31a10402c862df415b000000 smpf     r8_r9_r10_r11, du54, r4, r0h, r2, r1l_r1h, sparse, 2d, xyzw, auto_lod_bias_min, a
+31a10406c26291411b200000 smpf     r8, du54, r1, r0h, r2, u67l_u67h, 2d, x, auto_lod_bias_min, a
+31a10804c665df411b000208 smpf     r8_r9_r10_r11, du54, r3, r0h, r4, r2l_r2h, r1l, 3d, xyzw, auto_lod_bias_min, a
+31a10808c2679f411b200000 smpf     r8_r9_r10_r11, du54, r1, r0h, r4, u68l_u68h, cube_array, xyzw, auto_lod_bias_min, a
+31b10806c2629f411b200000 smpf     r12_r13_r14_r15, du54, r1, r0h, r4, u67l_u67h, 2d, xyzw, auto_lod_bias_min, a
+31c10400c2620f415b008000 smpf     r16_r17_r18_r19, du54, r1, r0h, r2, 0, sparse, 2d, xyzw, auto_lod, gather_r, a
+31c10400c2620f415b008001 smpf     r16_r17_r18_r19, du54, r1, r0h, r2, 0, sparse, 2d, xyzw, auto_lod, gather_g, a
+31c10400c2620f415b008002 smpf     r16_r17_r18_r19, du54, r1, r0h, r2, 0, sparse, 2d, xyzw, auto_lod, gather_b, a
+31c11800c2622f415b200000 smpf     r16_r17_r18_r19, du54, r1, r0h, r12, u64l, sparse, 2d, xyzw, lod_min, a
+3219010000800100     ballot   r6
+3219420000000100     ballot   r6, r1l, 0, ieq
+3229010000800100     ballot   r10
+325962a226100100     ballot   r54, r17, r21, ieq
+3509000501c03000     stack_load r1l_r1h, 16, i16, xy, a
+3522000500c01200     stack_load r4, 0, i32, x, a
+3532400500c01200     stack_load r6, 4, i32, x, a
+3532800500c01200     stack_load r6, 8, i32, x, a
+3532800502c01200     stack_load r6, 40, i32, x, a
+353ac00503c01200     stack_load r7, 60, i32, x, a
+35b9c00500c03000     stack_load r23l_r23h, 12, i16, xy, a
+35c1800500c03000     stack_load r24l_r24h, 8, i16, xy, a
+35c9400500c03000     stack_load r25l_r25h, 4, i16, xy, a
+35d1000500c03000     stack_load r26l_r26h, 0, i16, xy, a
+360040440441         hfma     r0l, r0l.neg, r2l, r0h
+360241300442         hfma     r0h, r0h, r1h, r1l
+360245340442         hfma     r0h, r2h.neg, r1h, r1l
+36084a540443         hfma     r2l, r5l.neg, r2h, r1h
+36084a740444         hfma     r2l, r5l.neg, r3h, r2l
+360a43300445         hfma     r2h, r1h, r1h, r2h
+360a45600447         hfma     r2h, r2h, r3l, r3h
+360a46600445         hfma     r2h, r3l, r3l, r2h
+360a48800445         hfma     r2h, r4l, r4l, r2h
+360a49900445         hfma     r2h, r4h, r4h, r2h
+360c41340430         hfma     r3l, r0h.neg, r1h, 1.0
+360c46700430         hfma     r3l, r3l, r3h, 1.0
+360c46700448         hfma     r3l, r3l, r3h, r4l
+360c48700446         hfma     r3l, r4l, r3h, r3l
+360e40000447         hfma     r3h, r0l, r0l, r3h
+360e41100447         hfma     r3h, r0h, r0h, r3h
+361410110449         hfma     r5l, u8l, r0h, r4h
+36144530044a         hfma     r5l, r2h, r1h, r5l
+361446940448         hfma     r5l, r3l.neg, r4h, r4l
+3682414404420400     hfma     r0h, r0h.neg, r2l, r1l.neg
+3684469004480400     hfma     r1l, r3l, r4h, r4l.neg
+36884a9404460400     hfma     r2l, r5l.neg, r4h, r3l.neg
+3692100104120100     hfma     r4h, u8l, r0l, u9l
+3800                 wait     a
+3801                 wait     b
+3a81130124540200     ffma     r0, u9h, r0, r10
+3a914602198c0100     ffma     r4, r3, u8, u6
+3a954622198e0100     ffma     r5, r3, u9, u7
+3a954c22194a0200     ffma     r5, r6, u9, r5
+3e0944500400         intl     r2, r2l, r2h
+3e29540e0000         ffs      r10, r10
+3e800a042400         u32_to_f r0l, r0, rte
+3e800b042400         s32_to_f r0l, r0, rte
+3e8108000400         f_to_u32 r0, r0l, rtz
+3e8108802500         f_to_u32 r0, r12, rtz
+3e8109000400         f_to_s32 r0, r0l, rtz
+3e8109802500         f_to_s32 r0, r12, rtz
+3e810a042400         u32_to_f r0, r0, rte
+3e810b042400         s32_to_f r0, r0, rte
+3e8508202400         f_to_u32 r1, r1, rtz
+3e8509202400         f_to_s32 r1, r1, rtz
+3e850a242400         u32_to_f r1, r1, rte
+3e850b242400         s32_to_f r1, r1, rte
+3e8908402400         f_to_u32 r2, r2, rtz
+3e8909402400         f_to_s32 r2, r2, rtz
+3e8d0a242400         u32_to_f r3, r1, rte
+3e8d0a442400         u32_to_f r3, r2, rte
+417f0423             zs_emit  255, r2
+420000000000         update_exec
+42096026e600         if       r16.abs, r17.abs.neg, 1, fneq
+420a00000000         else     0.0, 0.0, 1, feq
+421000000000         if       0.0, 0.0, 2, feq
+422844801100         if       r2l, u12l, 1, flt
+4501040d80c01200     store    r0, du66, 0, i32, x, a
+4501400e00c43200     store    r0_r1, du0, r2, lsl 1, i32, xy, a
+4501404d80c03200     store    r0_r1, du64, 4, i32, xy, b
+4509080500c01200     store    r1, dr4, 0, i32, x, a
+4521000d80c0f200     store    r4_r5_r6_r7, du64, 0, i32, xyzw, a
+4531400510c01200     store    r6, dr8, 4, i32, x, a
+45410c0500c01200     store    r8, dr6, 0, i32, x, a
+4541800e00c8f200     store    r8_r9_r10_r11, du0, r4, lsl 2, i32, xyzw, a
+45492c0500c01200     store    r9, dr6, 2, i32, x, a
+4561000510c01200     store    r12, dr8, 0, i32, x, a
+4561004510c01200     store    r12, dr8, 0, i32, x, b
+4561200510c03200     store    r12_r13, dr8, 2, i32, xy, a
+4561e44610c01200     store    r12, dr10, r7, i32, x, b
+4800c200             pixwait  512, 3
+48040000             pixwait  4, 0
+48080000             pixwait  8, 0
+480c0000             pixwait  12, 0
+49000004f8fc0003     ld_tile  r0l_r0h_r1l_r1h, 0, 255, 0, u8norm, xyzw
+520841000000         if       r0h, 0, 1, ieq
+52084c020000         if       r6, 0, 1, ieq
+52084c120000         if       r6, 1, 1, ieq
+52084c420000         if       r6, 4, 1, ieq
+52084cf20000         if       r6, 15, 1, ieq
+520850020000         if       r8, 0, 1, ieq
+520941000000         if       r0h, 0, 1, ine
+520e00000000         pop_exec 1
+52284c822400         if       r6, r4, 1, ult
+5501248c10500600     atomic   _, r3, du10, r1.sx, imin, coherent1, a
+5502248c10500600     atomic   _, r3, du10, r1.sx, or, coherent1, a
+5509a48c10d00800     atomic   r1, r4, du10, r5.sx, imin, coherent1, a
+550aa48c10d00800     atomic   r1, r4, du10, r5.sx, or, coherent1, a
+6180010000000000     iter     r0l_r0h_r1l_r1h, cf1, cf0
+6181018000000100     iter     r0_r1, cf1, cf0, 0, sample
+618101c000000200     iter     r0_r1_r2, cf1, cf0, centroid
+618101c000401000     iter     r0_r1_r2, cf1, cf0, elide, kill
+619005c000000000     iter     r4l_r4h_r5l, cf5, cf0
+6191010006000300     iter     r4_r5_r6_r7, cf1, cf0, r3l, sample
+6191018002000300     iter     r4_r5, cf1, cf0, r1l, sample
+6191018006000300     iter     r4_r5, cf1, cf0, r3l, sample
+619101c002000300     iter     r4_r5_r6, cf1, cf0, r1l, sample
+62000000             ldimm    r0l, 0
+620100000040         ldimm    r0, 1073741824
+62020000             ldimm    r0h, 0
+6206003c             ldimm    r1h, 15360
+6206483e             ldimm    r1h, 15944
+62080034             ldimm    r2l, 13312
+62090000adde         ldimm    r2, 3735879680
+620a0074             ldimm    r2h, 29696
+620c0080             ldimm    r3l, 32768
+620cdf2a             ldimm    r3l, 10975
+620cff7f             ldimm    r3l, 32767
+620dffffff07         ldimm    r3, 134217727
+620e48be             ldimm    r3h, 48712
+62100000             ldimm    r4l, 0
+62103432             ldimm    r4l, 12852
+621036a2             ldimm    r4l, 41526
+621053b5             ldimm    r4l, 46419
+621083af             ldimm    r4l, 44931
+621100000000         ldimm    r4, 0
+6211ffffffff         ldimm    r4, 4294967295
+62120000             ldimm    r4h, 0
+621500000000         ldimm    r5, 0
+6216003c             ldimm    r5h, 15360
+6225ff3f0000         ldimm    r9, 16383
+6225ffffffff         ldimm    r9, 4294967295
+6229f0ffffff         ldimm    r10, 4294967280
+6800                 barrier
+6985800214000000     lload    r1, 0, i32, x
+6989000214000000     lload    r2, r0l, 0, i32, x
+6991800234000000     lload    r4_r5, 0, i32, xy
+6991800274000000     lload    r4_r5_r6, 0, i32, xyz
+69918002f4000000     lload    r4_r5_r6_r7, 0, i32, xyzw
+6aad50020002         fadd     r11, r8, -0.0, sat
+6f0241104000         shuffle  r0h, r0h, 1, quad, xor
+6f0241204000         shuffle  r0h, r0h, 2, quad, xor
+6f0241304000         shuffle  r0h, r0h, 3, quad, xor
+6f104c100400         shuffle  r4l, r6l, r0h, quad
+6f124c100000         shuffle  r4h, r6l, 1, quad
+6f124c100400         shuffle  r4h, r6l, r0h, quad
+6f124c200000         shuffle  r4h, r6l, 2, quad
+6f194c160400         shuffle  r6, r6, r0h
+6f1a4c000000         shuffle  r6h, r6l, 0, quad
+6f1a4c300000         shuffle  r6h, r6l, 3, quad
+6f8444600000         simd     r1l, r2l, prefix, and
+6f8444604000         simd     r1l, r2l, prefix, or
+6f8444608000         simd     r1l, r2l, prefix, xor
+6f8444608080         simd     r1l, r2l, prefix, smin
+6f8846604000         simd     r2l, r3l, prefix, or
+6f8846608000         simd     r2l, r3l, prefix, xor
+6f8a44600080         simd     r2h, r2l, prefix, iadd
+6f8a44604000         simd     r2h, r2l, prefix, or
+6f8a44608000         simd     r2h, r2l, prefix, xor
+6f8a4460c080         simd     r2h, r2l, prefix, smax
+6f8e46604000         simd     r3h, r3l, prefix, or
+6f8e46608000         simd     r3h, r3l, prefix, xor
+6f9a4c600080         simd     r6h, r6l, prefix, iadd
+6f9d4c620080         simd     r7, r6, prefix, iadd
+7101040000620f00     smpi     r0_r1_r2_r3, ts0, ss0, r2, 0, 2d, xyzw, auto_lod, a
+72001004             mov      r0l, thread_in_grid_x
+72021104             mov      r0h, thread_in_grid_y
+72050000             mov      r1, threadgroup_in_grid_x
+72051004             mov      r1, thread_in_grid_x
+72051400             mov      r1, core_index
+72090100             mov      r2, threadgroup_in_grid_y
+72091004             mov      r2, thread_in_grid_x
+72091104             mov      r2, thread_in_grid_y
+72091208             mov      r2, helper_arg_l
+720d0200             mov      r3, threadgroup_in_grid_z
+720d1308             mov      r3, helper_arg_h
+720d3000             mov      r3, thread_in_threadgroup_x
+72191008             mov      r6, helper_op
+721d1208             mov      r7, helper_arg_l
+72213100             mov      r8, thread_in_threadgroup_y
+72283a00             mov      r10l, active_thread_index_in_simdgroup
+7540010500c01000     map      r8, 0, 0
+7540300500c01000     unmap    r8, 0, 3
+7548110500c01000     map      r9, 0, 1
+7550210500c01000     map      r10, 0, 2
+7558310500c01000     map      r11, 0, 3
+7560000500c01000     unmap    r12, 0, 0
+7560100500c01000     unmap    r12, 0, 1
+7560200500c01000     unmap    r12, 0, 2
+7e0140088000         mov      r0, r0l
+7e014022a500         and      r0, r0, r9
+7e01402a2400         andn2    r0, r0, r1
+7e0140728000         and      r0, r0, 7
+7e0140f28100         and      r0, r0, 31
+7e015ef28000         and      r0, r15, 15
+7e0190098000         mov      r0, u8
+7e02540a8000         mov      r0h, r10
+7e0414098000         mov      r1l, u10l
+7e0442608400         and      r1l, r1l, r3l
+7e044528c400         or       r1l, r2h, r1l
+7e0500088000         mov      r1, 0
+7e05400a8000         mov      r1, r0
+7e0542064000         not      r1, r1
+7e0542e29802         and      r1, r1, u71
+7e0542f28100         and      r1, r1, 31
+7e0542f28300         and      r1, r1, 63
+7e0543088000         mov      r1, r1h
+7e0644088000         mov      r1h, r2l
+7e0646088000         mov      r1h, r3l
+7e0940088000         mov      r2, r0l
+7e0942c29802         and      r2, r1, u70
+7e094462a400         and      r2, r2, r3
+7e09ac098004         mov      r2, u54
+7e0a45608400         and      r2h, r2h, r3l
+7e0d41088000         mov      r3, r0h
+7e0dae098004         mov      r3, u55
+7e1049088000         mov      r4l, r4h
+7e11400a8000         mov      r4, r0
+7e1142088000         mov      r4, r1l
+7e11442ae400         or       r4, r2, r1
+7e15400a8000         mov      r5, r0
+7e19400a8000         mov      r6, r0
+7e19480a8000         mov      r6, r4
+7e194c22a500         and      r6, r6, r9
+7e1d400a8000         mov      r7, r0
+7e1d4a0a8000         mov      r7, r5
+7e2150f28100         and      r8, r8, 31
+7e2446088000         mov      r9l, r3l
+7e2650088000         mov      r9h, r8l
+7e29480a8000         mov      r10, r4
+7e2d440a8000         mov      r11, r2
+7e2d480a8000         mov      r11, r4
+7e2d4a0a8000         mov      r11, r5
+7e2d740a8000         mov      r11, r26
+7e31ae0a8000         mov      r12, $r23
+7e35700a8000         mov      r13, r24
+7e5d980a8000         mov      r23, $r12
+7e615a0a8000         mov      r24, r13
+7e65960a8000         mov      r25, $r11
+85044c0e00c01000     load     r0h, du6, r2, i16, x, a
+8800                 stop
+8e090060a9000000     iadd     $r2, 0, $r11, lsl 1
+8e09562227000000     iadd     $r2, r11, r25
+91008880             st_var   r0, 8, last
+9500248c10500600     atomic   _, r3, du10, r1.sx, xchg, coherent1, a
+9502248c10500600     atomic   _, r3, du10, r1.sx, xor, coherent1, a
+9508a48c10d00800     atomic   r1, r4, du10, r5.sx, xchg, coherent1, a
+950aa48c10d00800     atomic   r1, r4, du10, r5.sx, xor, coherent1, a
+a181004000000000     ldcf     r0_r1_r2, cf0
+a191014000000000     ldcf     r4_r5_r6, cf1
+a191024000000000     ldcf     r4_r5_r6, cf2
+a191034000000000     ldcf     r4_r5_r6, cf3
+ad01100800000010     load     ss1, du4, r1, a
+ad01116001000024     load     ss1, du8, 88, b
+b1800080004a00000a00 imgwblk  r0l, ts0, r0, 0, 2d, u8norm
+b1848480024a00000a00 imgwblk  r1l, ts2, r2, 0, explicit_coords, 2d, u8norm
+b1880680004880000a00 imgwblk  r2l, ts0, r3, 0, 2d_ms_array, u8norm
+b50081052000         stack_adjust 8
+b500810523800000     stack_adjust 56
+b50081052f8000ff     stack_adjust -8
+b509000501c03000     stack_store r1l_r1h, 16, i16, xy, a
+b521800500c01000     stack_store r4l, 8, i16, x, a
+b522000500c01200     stack_store r4, 0, i32, x, a
+b522400501c01200     stack_store r4, 20, i32, x, a
+b522404500c01200     stack_store r4, 4, i32, x, b
+b522c00500c01200     stack_store r4, 12, i32, x, a
+b53a800501c01200     stack_store r7, 24, i32, x, a
+b552400501c01200     stack_store r10, 20, i32, x, a
+b55a000501c01200     stack_store r11, 16, i32, x, a
+b5b9c00500c03000     stack_store r23l_r23h, 12, i16, xy, a
+b5c1800500c03000     stack_store r24l_r24h, 8, i16, xy, a
+b5c9400500c03000     stack_store r25l_r25h, 4, i16, xy, a
+b5d1000500c03000     stack_store r26l_r26h, 0, i16, xy, a
+c1000100             sample_mask r0l, r0h
+c1008000             sample_mask r0l, 0
+c1020000             sample_mask r0h, r0l
+c1028000             sample_mask r0h, 0
+c102bf0c             sample_mask r0h, 255
+c17f8003             sample_mask 255, 0
+c500003d0880f000     mov      du64, dr0
+c500003d09803000     mov      u72, r0
+c500403d01803000     mov      u10, r0
+c500403d0980f000     mov      du74, dr0
+c500440e00c43000     store    r0l_r0h, du2, r2, lsl 1, i16, xy, a
+c500a03d08803000     mov      u69, r0
+c508c03d08803000     mov      u70, r1
+c508e03d08803000     mov      u71, r1
+c510403d08803000     mov      u66, r2
+c520803d08803000     mov      u68, r4
+c520a03d02807000     mov      u21l_u21h_u22l, r4l_r4h_r5l
+c528903d02803000     mov      u20h_u21l, r5
+c530603d08803000     mov      u67, r6
+ce05864200000000     iadd     $r1, $r3, 4, sat
+d21600000000         pop_exec 2, cache
+d28884020000         if       $r2, 0, 1, cache, seq
+d500688c10500c00     atomic   _, r6_r7, du12, r3.sx, cmpxchg, coherent1, a
+d501248c10500600     atomic   _, r3, du10, r1.sx, imax, coherent1, a
+d508288c10d01000     atomic   r1, r8_r9, du12, r1.sx, cmpxchg, coherent1, a
+d509a48c10d00800     atomic   r1, r4, du10, r5.sx, imax, coherent1, a
+ed00100001000024     load     ts0, du8, 64, a
+ed01100800000000     load     ts1, du0, r1, a
+ed02100800000008     load     ts2, du2, r1, a
+ed02100800000018     load     ts2, du6, r1, a
+f151a080054a         imgw     r20_r21_r22_r23, ts5, r16, 0, 2d
+f1a18c80c24a1b00     imgw     r8_r9_r10_r11, du54, r1, r6, 0, 2d
+f503                 unknown_barrier_2
+f533                 unknown_barrier_1
+f540                 flush_memory_to_texture
+f596                 memory_barrier
+f599                 memory_barrier_3
+f59a                 memory_barrier_2
+f5a9                 image_barrier_3
+f5aa                 image_barrier_1
+f5ad                 image_barrier_4
+f5ae                 image_barrier_2

src/asahi/isa/test/disasm.py

@@ -0,0 +1,158 @@
+# Copyright 2025 Valve Corporation
+# SPDX-License-Identifier: MIT
+
+from isa import isa
+import math
+import sys
+import binascii
+
+def match_instr(raw):
+    for key in isa.instrs:
+        instr = isa.instrs[key]
+        length = instr.length[instr.length_spec.bits.extract(raw) if instr.length_spec is not None else 0]
+        r = raw & ((1 << (length * 8)) - 1)
+        if (r & instr.exact.mask) == instr.exact.value:
+            return length, instr, r
+
+    return 2, None, raw
+
+def map_value(v, kind, length):
+    if kind in isa.enums:
+        val = isa.enums[kind].values.get(v)
+        return f'unk{v}' if val is None else val.display
+    elif kind == 'signed':
+        return str(-((1 << length) - v) if (v & (1 << (length - 1))) else v)
+    else:
+        return str(v)
+
+def map_mod(v, mod):
+    if mod.kind in isa.enums:
+        return map_value(v, mod.kind, 0)
+    else:
+        assert(mod.bits.size() == 1)
+        return mod.display if v != 0 else None
+
+def decode_float_immediate(n):
+	sign = -1.0 if n & 0x80 else 1.0
+	e = (n & 0x70) >> 4
+	f = n & 0xF
+	if e == 0:
+		return sign * f / 64.0
+	else:
+		return sign * float(0x10 | f) * (2.0 ** (e - 7))
+
+def operand(desc, length):
+    count = desc.get('count', 1)
+    if count == 0:
+        count = 4
+    if 'mask' in desc:
+        count = desc['mask'].bit_count()
+    if 'zx' in desc:
+        desc['sx'] = desc['zx'] ^ 1
+
+    size = 64 if desc.get('size64', 0) != 0 else 32 if desc.get('size32', 0) != 0 else 16
+    base = desc.get('base')
+    kind = desc.get('kind')
+    cache = desc.get('cache', False)
+    discard = desc.get('discard', False)
+
+    hint = desc.get('hint')
+    if hint is not None:
+        if hint == 3:
+            discard = True
+        elif hint == 2:
+            cache = True
+        else:
+            assert(hint == 1)
+
+    assert(not (cache and discard))
+    hint = '^' if discard else '$' if cache else ''
+    suffixes = ''.join(['.' + x for x in ['abs', 'neg', 'sx'] if desc.get(x, 0) != 0])
+
+    if kind == None:
+        return None
+    elif kind in ['cf', 'ts', 'ss']:
+        assert(count == 1)
+        return f'{kind}{base}' + suffixes
+    elif kind == 'fimm':
+        return str(decode_float_immediate(base)) + suffixes
+    elif kind in ['imm', 'signed']:
+        return map_value(base, kind, length) + suffixes
+
+    def m(offs):
+        assert(size == 16 or size == 32 or size == 64)
+        reg = base + (offs * (size // 16))
+        whole = reg >> 1
+        part = reg & 1
+        prefix = kind[0]
+
+        if size == 16:
+            part_str = 'h' if part != 0 else 'l'
+            return f'{prefix}{whole}{part_str}'
+        else:
+            assert(part == 0)
+            return f'{"d" if size == 64 else ""}{prefix}{whole}'
+
+    return hint + '_'.join([m(x) for x in range(count)]) + suffixes
+
+def map_op(v, kind, raw, ins, length):
+    for c in isa.encodings[kind].cases:
+        if (v & c.exact.mask) == c.exact.value:
+            desc = {}
+            if c.kind is not None:
+                length = c.kind.bits.size()
+                desc['kind'] = c.kind.name
+                desc['base'] = c.kind.bits.extract(v) << c.kind.shift
+                if c.kind.kind == 'signed':
+                    desc['kind'] = 'signed'
+
+            for mod in c.modifiers:
+                desc[mod.name] = mod.bits.extract(v)
+
+            for prop in c.properties:
+                desc[prop] = int(c.properties[prop])
+
+            return operand(desc, length)
+
+def disasm(raw_in):
+    length, instr, raw = match_instr(raw_in)
+    if instr is None:
+        return length, "<unknown instr>"
+
+    imms = [map_value(imm.bits.extract(raw), imm.kind, imm.bits.size()) for imm in instr.immediates]
+    mods = [map_mod(mod.bits.extract(raw), mod) for mod in instr.modifiers if mod.name != 'length']
+    dests = [map_op(d.bits.extract(raw), d.kind, raw, instr, d.bits.size()) for d in instr.dests]
+    srcs = [map_op(s.bits.extract(raw), s.kind, raw, instr, s.bits.size()) for s in instr.sources]
+
+    dests = ['_' if x is None else str(x) for x in dests]
+    srcs = [str(x) for x in srcs if x is not None]
+    imms = [str(x) for x in imms if x is not None]
+    mods = [str(x) for x in mods if x is not None]
+    end = ', '.join(dests + srcs + imms + mods)
+    if end != "":
+        end = " " + end
+
+    asm = f'{instr.display:<8}{end}'.strip()
+
+    unexpected = raw & ~instr.mask()
+    unexpected = [i for i in range(length * 8) if unexpected & (1 << i)]
+    if len(unexpected) != 0:
+        print(f"Unexpected set bits: {unexpected}")
+        length = -length
+
+    return length, asm
+
+if __name__ == '__main__':
+    prog = open(sys.argv[1], 'rb').read()
+    print("")
+    i = 0
+    while i < len(prog):
+        lst = prog[i:i + 16]
+        raw = sum([int(x) << (8 * i) for i, x in enumerate(lst)])
+        length, asm = disasm(raw)
+        lst = lst[0:length]
+        print(f'{hex(i)[2:]:>4}: {str(binascii.hexlify(lst).decode()):<20} {asm}')
+        if asm.strip() == 'trap':
+            break
+        i += length
+    print("")

src/asahi/isa/test/test-disassembler.c

@@ -0,0 +1,114 @@
+/*
+ * Copyright (C) 2021 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include <inttypes.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "asahi/isa/disasm.h"
+#include "util/macros.h"
+
+static inline uint8_t
+parse_nibble(const char c)
+{
+   return (c >= 'a') ? 10 + (c - 'a') : (c - '0');
+}
+
+/* Given a little endian 8 byte hexdump, parse out the 64-bit value */
+static uint64_t
+parse_hex(const char *in)
+{
+   uint64_t v = 0;
+
+   for (unsigned i = 0; i < 8; ++i) {
+      uint8_t byte = (parse_nibble(in[0]) << 4) | parse_nibble(in[1]);
+      v |= ((uint64_t)byte) << (8 * i);
+
+      /* Skip the space after the byte */
+      in += 3;
+   }
+
+   return v;
+}
+
+int
+main(int argc, const char **argv)
+{
+   if (argc < 2) {
+      fprintf(stderr, "Expected case list\n");
+      return 1;
+   }
+
+   FILE *fp = fopen(argv[1], "r");
+
+   if (fp == NULL) {
+      fprintf(stderr, "Could not open the case list");
+      return 1;
+   }
+
+   char line[128];
+   unsigned nr_fail = 0, nr_pass = 0;
+
+   while (fgets(line, sizeof(line), fp) != NULL) {
+      char *output = NULL;
+      size_t sz = 0;
+      size_t len = strlen(line);
+
+      /* Skip empty lines */
+      if (len <= 1)
+         continue;
+
+      /* Parse the hex */
+      uint8_t code[16];
+      const char *cursor = line;
+      const char *end = line + len;
+      uint32_t i = 0;
+      while ((cursor + 2) <= end && cursor[0] != ' ') {
+         assert(i < ARRAY_SIZE(code));
+         code[i] = (parse_nibble(cursor[0]) << 4) | parse_nibble(cursor[1]);
+         cursor += 2;
+         ++i;
+      }
+
+      /* Skip spacing */
+      while (cursor < end && (*cursor) == ' ')
+         ++cursor;
+
+      FILE *outputp = open_memstream(&output, &sz);
+      signed ret = agx2_disassemble_instr((void *)code, outputp, 0, false);
+      unsigned instr_len = abs(ret);
+      fclose(outputp);
+
+      /* Rest of the line is the reference assembly */
+      bool fail = strcmp(cursor, output);
+      fail |= (instr_len != i);
+      fail |= (ret < 0);
+
+      if (fail) {
+         /* Extra spaces after Got to align with Expected */
+         printf("Got      %sExpected %s\n", output, cursor);
+
+         if (instr_len != i) {
+            printf("Got length %d, expected length %u\n", instr_len, i);
+         }
+
+         if (ret < 0) {
+            printf("Got an error.\n");
+         }
+
+         nr_fail++;
+      } else {
+         nr_pass++;
+      }
+
+      free(output);
+   }
+
+   printf("Passed %u/%u tests.\n", nr_pass, nr_pass + nr_fail);
+   fclose(fp);
+
+   return nr_fail ? 1 : 0;
+}

src/asahi/layout/README

@@ -0,0 +1,30 @@
+# ail
+
+ail is a small library for working with the image (and buffer) layouts
+encountered with AGX hardware. Its design is inspired by isl. In
+particular, ail strives to use isl unit suffixes and to represent
+quantities in a canonical, API-agnostic fashion.
+
+ail conventions differ slightly from isl. See the isl documentation in
+mesa/docs/isl for the conventions that inspired ail, in particular
+mesa/docs/isl/units.rst.
+
+In ail, we have the following units:
+
+**Bytes (B)**: 8-bits.
+
+**Samples (sa)**: An individual sample. The number of bytes per sample depends
+on the format.
+
+**Pixels (px)**: The unit everything starts as from the API. A pixel contains a
+fixed number of samples, given as the layout's `sample_count_sa`. For twiddled
+layouts, these samples are stored together in emmory.
+
+**Elements (el)**: For a block-compressed format, a single compression block
+containing multiple pixels. Otherwise, equal to a single pixel.
+
+**Tiles (tiles)**: Defined only for tiled/twiddled layouts. A group of elements
+forming the layout-specific tile.
+
+We do not support any multisampled block-compressed layouts, so either pixels
+equals either samples or elements.

src/asahi/layout/formats.c

@@ -0,0 +1,278 @@
+/*
+ * Copyright 2021 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "agx_pack.h"
+#include "layout.h"
+
+#define AIL_ISA_FORMAT__ PIPE_FORMAT_NONE
+
+#define AIL_FMT(pipe, channels_, type_, renderable_)                           \
+   [PIPE_FORMAT_##pipe] = {                                                    \
+      .channels = AGX_CHANNELS_##channels_,                                    \
+      .type = AGX_TEXTURE_TYPE_##type_,                                        \
+      .texturable = true,                                                      \
+      .renderable = (enum pipe_format)AIL_ISA_FORMAT_##renderable_,            \
+   }
+
+/* clang-format off */
+const struct ail_pixel_format_entry ail_pixel_format[PIPE_FORMAT_COUNT] = {
+   AIL_FMT(R5G6B5_UNORM,            R5G6B5,                UNORM, F16),
+   AIL_FMT(B5G6R5_UNORM,            R5G6B5,                UNORM, F16),
+
+   AIL_FMT(R5G5B5A1_UNORM,          R5G5B5A1,              UNORM, F16),
+   AIL_FMT(B5G5R5A1_UNORM,          R5G5B5A1,              UNORM, F16),
+   AIL_FMT(R5G5B5X1_UNORM,          R5G5B5A1,              UNORM, F16),
+   AIL_FMT(B5G5R5X1_UNORM,          R5G5B5A1,              UNORM, F16),
+
+   AIL_FMT(R4G4B4A4_UNORM,          R4G4B4A4,              UNORM, F16),
+   AIL_FMT(B4G4R4A4_UNORM,          R4G4B4A4,              UNORM, F16),
+   AIL_FMT(A4B4G4R4_UNORM,          R4G4B4A4,              UNORM, F16),
+   AIL_FMT(A4R4G4B4_UNORM,          R4G4B4A4,              UNORM, F16),
+
+   AIL_FMT(R8_UNORM,                R8,                    UNORM, U8NORM),
+   AIL_FMT(R8G8_UNORM,              R8G8,                  UNORM, U8NORM),
+   AIL_FMT(R8G8B8A8_UNORM,          R8G8B8A8,              UNORM, U8NORM),
+   AIL_FMT(A8R8G8B8_UNORM,          R8G8B8A8,              UNORM, U8NORM),
+   AIL_FMT(A8B8G8R8_UNORM,          R8G8B8A8,              UNORM, U8NORM),
+   AIL_FMT(B8G8R8A8_UNORM,          R8G8B8A8,              UNORM, U8NORM),
+   AIL_FMT(R8G8B8X8_UNORM,          R8G8B8A8,              UNORM, U8NORM),
+   AIL_FMT(X8R8G8B8_UNORM,          R8G8B8A8,              UNORM, U8NORM),
+   AIL_FMT(X8B8G8R8_UNORM,          R8G8B8A8,              UNORM, U8NORM),
+   AIL_FMT(B8G8R8X8_UNORM,          R8G8B8A8,              UNORM, U8NORM),
+
+   AIL_FMT(R16_UNORM,               R16,                   UNORM, U16NORM),
+   AIL_FMT(R16G16_UNORM,            R16G16,                UNORM, U16NORM),
+   AIL_FMT(R16G16B16A16_UNORM,      R16G16B16A16,          UNORM, U16NORM),
+   AIL_FMT(R16_SNORM,               R16,                   SNORM, S16NORM),
+   AIL_FMT(R16G16_SNORM,            R16G16,                SNORM, S16NORM),
+   AIL_FMT(R16G16B16A16_SNORM,      R16G16B16A16,          SNORM, S16NORM),
+
+   AIL_FMT(R8_SRGB,                 R8,                    UNORM, SRGBA8),
+   AIL_FMT(R8G8_SRGB,               R8G8,                  UNORM, SRGBA8),
+   AIL_FMT(R8G8B8A8_SRGB,           R8G8B8A8,              UNORM, SRGBA8),
+   AIL_FMT(A8R8G8B8_SRGB,           R8G8B8A8,              UNORM, SRGBA8),
+   AIL_FMT(A8B8G8R8_SRGB,           R8G8B8A8,              UNORM, SRGBA8),
+   AIL_FMT(B8G8R8A8_SRGB,           R8G8B8A8,              UNORM, SRGBA8),
+   AIL_FMT(R8G8B8X8_SRGB,           R8G8B8A8,              UNORM, SRGBA8),
+   AIL_FMT(X8R8G8B8_SRGB,           R8G8B8A8,              UNORM, SRGBA8),
+   AIL_FMT(X8B8G8R8_SRGB,           R8G8B8A8,              UNORM, SRGBA8),
+   AIL_FMT(B8G8R8X8_SRGB,           R8G8B8A8,              UNORM, SRGBA8),
+
+   AIL_FMT(R8_SNORM,                R8,                    SNORM, S8NORM),
+   AIL_FMT(R8G8_SNORM,              R8G8,                  SNORM, S8NORM),
+   AIL_FMT(R8G8B8A8_SNORM,          R8G8B8A8,              SNORM, S8NORM),
+   AIL_FMT(A8R8G8B8_SNORM,          R8G8B8A8,              SNORM, S8NORM),
+   AIL_FMT(A8B8G8R8_SNORM,          R8G8B8A8,              SNORM, S8NORM),
+   AIL_FMT(B8G8R8A8_SNORM,          R8G8B8A8,              SNORM, S8NORM),
+   AIL_FMT(R8G8B8X8_SNORM,          R8G8B8A8,              SNORM, S8NORM),
+   AIL_FMT(X8R8G8B8_SNORM,          R8G8B8A8,              SNORM, S8NORM),
+   AIL_FMT(X8B8G8R8_SNORM,          R8G8B8A8,              SNORM, S8NORM),
+   AIL_FMT(B8G8R8X8_SNORM,          R8G8B8A8,              SNORM, S8NORM),
+
+   AIL_FMT(R16_FLOAT,               R16,                   FLOAT, F16),
+   AIL_FMT(R16G16_FLOAT,            R16G16,                FLOAT, F16),
+   AIL_FMT(R16G16B16X16_FLOAT,      R16G16B16A16,          FLOAT, F16),
+   AIL_FMT(R16G16B16A16_FLOAT,      R16G16B16A16,          FLOAT, F16),
+
+   AIL_FMT(R32_FLOAT,               R32,                   FLOAT, I32),
+   AIL_FMT(R32G32_FLOAT,            R32G32,                FLOAT, I32),
+   AIL_FMT(R32G32B32X32_FLOAT,      R32G32B32A32,          FLOAT, I32),
+   AIL_FMT(R32G32B32A32_FLOAT,      R32G32B32A32,          FLOAT, I32),
+
+   AIL_FMT(R8_UINT,                 R8,                    UINT,  I8),
+   AIL_FMT(R8G8_UINT,               R8G8,                  UINT,  I8),
+   AIL_FMT(R8G8B8X8_UINT,           R8G8B8A8,              UINT,  I8),
+   AIL_FMT(R8G8B8A8_UINT,           R8G8B8A8,              UINT,  I8),
+   AIL_FMT(B8G8R8X8_UINT,           R8G8B8A8,              UINT,  I8),
+   AIL_FMT(B8G8R8A8_UINT,           R8G8B8A8,              UINT,  I8),
+
+   AIL_FMT(R16_UINT,                R16,                   UINT,  I16),
+   AIL_FMT(R16G16_UINT,             R16G16,                UINT,  I16),
+   AIL_FMT(R16G16B16X16_UINT,       R16G16B16A16,          UINT,  I16),
+   AIL_FMT(R16G16B16A16_UINT,       R16G16B16A16,          UINT,  I16),
+
+   AIL_FMT(R32_UINT,                R32,                   UINT,  I32),
+   AIL_FMT(R32G32_UINT,             R32G32,                UINT,  I32),
+   AIL_FMT(R32G32B32X32_UINT,       R32G32B32A32,          UINT,  I32),
+   AIL_FMT(R32G32B32A32_UINT,       R32G32B32A32,          UINT,  I32),
+
+   AIL_FMT(R8_SINT,                 R8,                    SINT,  I8),
+   AIL_FMT(R8G8_SINT,               R8G8,                  SINT,  I8),
+   AIL_FMT(R8G8B8X8_SINT,           R8G8B8A8,              SINT,  I8),
+   AIL_FMT(R8G8B8A8_SINT,           R8G8B8A8,              SINT,  I8),
+   AIL_FMT(B8G8R8X8_SINT,           R8G8B8A8,              SINT,  I8),
+   AIL_FMT(B8G8R8A8_SINT,           R8G8B8A8,              SINT,  I8),
+
+   AIL_FMT(R16_SINT,                R16,                   SINT,  I16),
+   AIL_FMT(R16G16_SINT,             R16G16,                SINT,  I16),
+   AIL_FMT(R16G16B16X16_SINT,       R16G16B16A16,          SINT,  I16),
+   AIL_FMT(R16G16B16A16_SINT,       R16G16B16A16,          SINT,  I16),
+
+   AIL_FMT(R32_SINT,                R32,                   SINT,  I32),
+   AIL_FMT(R32G32_SINT,             R32G32,                SINT,  I32),
+   AIL_FMT(R32G32B32X32_SINT,       R32G32B32A32,          SINT,  I32),
+   AIL_FMT(R32G32B32A32_SINT,       R32G32B32A32,          SINT,  I32),
+
+   AIL_FMT(Z16_UNORM,               R16,                   UNORM, _),
+   AIL_FMT(Z32_FLOAT,               R32,                   FLOAT, _),
+   AIL_FMT(Z32_FLOAT_S8X24_UINT,    R32,                   FLOAT, _),
+   AIL_FMT(S8_UINT,                 R8,                    UINT,  _),
+
+   /* The stencil part of Z32F + S8 is just S8 */
+   AIL_FMT(X32_S8X24_UINT,          R8,                    UINT,  _),
+
+   /* These must be lowered by u_transfer_helper to Z32F + S8 */
+   AIL_FMT(Z24X8_UNORM,             R32,                   FLOAT, _),
+   AIL_FMT(Z24_UNORM_S8_UINT,       R32,                   FLOAT, _),
+
+   AIL_FMT(R10G10B10A2_UNORM,       R10G10B10A2,           UNORM, RGB10A2),
+   AIL_FMT(R10G10B10X2_UNORM,       R10G10B10A2,           UNORM, RGB10A2),
+   AIL_FMT(B10G10R10A2_UNORM,       R10G10B10A2,           UNORM, RGB10A2),
+   AIL_FMT(B10G10R10X2_UNORM,       R10G10B10A2,           UNORM, RGB10A2),
+
+   AIL_FMT(R10G10B10A2_UINT,        R10G10B10A2,           UINT,  I16),
+   AIL_FMT(B10G10R10A2_UINT,        R10G10B10A2,           UINT,  I16),
+
+   /* I don't see why this wouldn't be renderable, but it doesn't seem to work
+    * properly and it's not in Metal.
+    */
+   AIL_FMT(R10G10B10A2_SINT,        R10G10B10A2,           SINT,  _),
+   AIL_FMT(B10G10R10A2_SINT,        R10G10B10A2,           SINT,  _),
+
+   AIL_FMT(R11G11B10_FLOAT,         R11G11B10,             FLOAT, RG11B10F),
+   AIL_FMT(R9G9B9E5_FLOAT,          R9G9B9E5,              FLOAT, RGB9E5),
+
+   /* These formats are emulated for texture buffers only */
+   AIL_FMT(R32G32B32_FLOAT,         R32G32B32_EMULATED,    FLOAT, _),
+   AIL_FMT(R32G32B32_UINT,          R32G32B32_EMULATED,    UINT,  _),
+   AIL_FMT(R32G32B32_SINT,          R32G32B32_EMULATED,    SINT,  _),
+
+   /* Likewise, luminance/alpha/intensity formats are supported for texturing,
+    * because they are required for texture buffers in the compat profile and
+    * mesa/st is unable to emulate them for texture buffers. Our Gallium driver
+    * handles the swizzles appropriately, so we just need to plumb through the
+    * enums.
+    *
+    * If mesa/st grows emulation for these formats later, we can drop this.
+    */
+   AIL_FMT(A8_UNORM,                R8,                    UNORM, _),
+   AIL_FMT(A16_UNORM,               R16,                   UNORM, _),
+   AIL_FMT(A8_SINT,                 R8,                    SINT,  _),
+   AIL_FMT(A16_SINT,                R16,                   SINT,  _),
+   AIL_FMT(A32_SINT,                R32,                   SINT,  _),
+   AIL_FMT(A8_UINT,                 R8,                    UINT,  _),
+   AIL_FMT(A16_UINT,                R16,                   UINT,  _),
+   AIL_FMT(A32_UINT,                R32,                   UINT,  _),
+   AIL_FMT(A16_FLOAT,               R16,                   FLOAT, _),
+   AIL_FMT(A32_FLOAT,               R32,                   FLOAT, _),
+
+   AIL_FMT(L8_UNORM,                R8,                    UNORM, _),
+   AIL_FMT(L16_UNORM,               R16,                   UNORM, _),
+   AIL_FMT(L8_SINT,                 R8,                    SINT,  _),
+   AIL_FMT(L16_SINT,                R16,                   SINT,  _),
+   AIL_FMT(L32_SINT,                R32,                   SINT,  _),
+   AIL_FMT(L8_UINT,                 R8,                    UINT,  _),
+   AIL_FMT(L16_UINT,                R16,                   UINT,  _),
+   AIL_FMT(L32_UINT,                R32,                   UINT,  _),
+   AIL_FMT(L16_FLOAT,               R16,                   FLOAT, _),
+   AIL_FMT(L32_FLOAT,               R32,                   FLOAT, _),
+
+   AIL_FMT(L8A8_UNORM,              R8G8,                  UNORM, _),
+   AIL_FMT(L16A16_UNORM,            R16G16,                UNORM, _),
+   AIL_FMT(L8A8_SINT,               R8G8,                  SINT,  _),
+   AIL_FMT(L16A16_SINT,             R16G16,                SINT,  _),
+   AIL_FMT(L32A32_SINT,             R32G32,                SINT,  _),
+   AIL_FMT(L8A8_UINT,               R8G8,                  UINT,  _),
+   AIL_FMT(L16A16_UINT,             R16G16,                UINT,  _),
+   AIL_FMT(L32A32_UINT,             R32G32,                UINT,  _),
+   AIL_FMT(L16A16_FLOAT,            R16G16,                FLOAT, _),
+   AIL_FMT(L32A32_FLOAT,            R32G32,                FLOAT, _),
+
+   AIL_FMT(I8_UNORM,                R8,                    UNORM, _),
+   AIL_FMT(I16_UNORM,               R16,                   UNORM, _),
+   AIL_FMT(I8_SINT,                 R8,                    SINT,  _),
+   AIL_FMT(I16_SINT,                R16,                   SINT,  _),
+   AIL_FMT(I32_SINT,                R32,                   SINT,  _),
+   AIL_FMT(I8_UINT,                 R8,                    UINT,  _),
+   AIL_FMT(I16_UINT,                R16,                   UINT,  _),
+   AIL_FMT(I32_UINT,                R32,                   UINT,  _),
+   AIL_FMT(I16_FLOAT,               R16,                   FLOAT, _),
+   AIL_FMT(I32_FLOAT,               R32,                   FLOAT, _),
+
+   AIL_FMT(ETC1_RGB8,               ETC2_RGB8,             UNORM, _),
+   AIL_FMT(ETC2_RGB8,               ETC2_RGB8,             UNORM, _),
+   AIL_FMT(ETC2_SRGB8,              ETC2_RGB8,             UNORM, _),
+   AIL_FMT(ETC2_RGB8A1,             ETC2_RGB8A1,           UNORM, _),
+   AIL_FMT(ETC2_SRGB8A1,            ETC2_RGB8A1,           UNORM, _),
+   AIL_FMT(ETC2_RGBA8,              ETC2_RGBA8,            UNORM, _),
+   AIL_FMT(ETC2_SRGBA8,             ETC2_RGBA8,            UNORM, _),
+   AIL_FMT(ETC2_R11_UNORM,          EAC_R11,               UNORM, _),
+   AIL_FMT(ETC2_R11_SNORM,          EAC_R11,               SNORM, _),
+   AIL_FMT(ETC2_RG11_UNORM,         EAC_RG11,              UNORM, _),
+   AIL_FMT(ETC2_RG11_SNORM,         EAC_RG11,              SNORM, _),
+
+   AIL_FMT(ASTC_4x4,                ASTC_4X4,              UNORM, _),
+   AIL_FMT(ASTC_5x4,                ASTC_5X4,              UNORM, _),
+   AIL_FMT(ASTC_5x5,                ASTC_5X5,              UNORM, _),
+   AIL_FMT(ASTC_6x5,                ASTC_6X5,              UNORM, _),
+   AIL_FMT(ASTC_6x6,                ASTC_6X6,              UNORM, _),
+   AIL_FMT(ASTC_8x5,                ASTC_8X5,              UNORM, _),
+   AIL_FMT(ASTC_8x6,                ASTC_8X6,              UNORM, _),
+   AIL_FMT(ASTC_8x8,                ASTC_8X8,              UNORM, _),
+   AIL_FMT(ASTC_10x5,               ASTC_10X5,             UNORM, _),
+   AIL_FMT(ASTC_10x6,               ASTC_10X6,             UNORM, _),
+   AIL_FMT(ASTC_10x8,               ASTC_10X8,             UNORM, _),
+   AIL_FMT(ASTC_10x10,              ASTC_10X10,            UNORM, _),
+   AIL_FMT(ASTC_12x10,              ASTC_12X10,            UNORM, _),
+   AIL_FMT(ASTC_12x12,              ASTC_12X12,            UNORM, _),
+
+   AIL_FMT(ASTC_4x4_SRGB,           ASTC_4X4,              UNORM, _),
+   AIL_FMT(ASTC_5x4_SRGB,           ASTC_5X4,              UNORM, _),
+   AIL_FMT(ASTC_5x5_SRGB,           ASTC_5X5,              UNORM, _),
+   AIL_FMT(ASTC_6x5_SRGB,           ASTC_6X5,              UNORM, _),
+   AIL_FMT(ASTC_6x6_SRGB,           ASTC_6X6,              UNORM, _),
+   AIL_FMT(ASTC_8x5_SRGB,           ASTC_8X5,              UNORM, _),
+   AIL_FMT(ASTC_8x6_SRGB,           ASTC_8X6,              UNORM, _),
+   AIL_FMT(ASTC_8x8_SRGB,           ASTC_8X8,              UNORM, _),
+   AIL_FMT(ASTC_10x5_SRGB,          ASTC_10X5,             UNORM, _),
+   AIL_FMT(ASTC_10x6_SRGB,          ASTC_10X6,             UNORM, _),
+   AIL_FMT(ASTC_10x8_SRGB,          ASTC_10X8,             UNORM, _),
+   AIL_FMT(ASTC_10x10_SRGB,         ASTC_10X10,            UNORM, _),
+   AIL_FMT(ASTC_12x10_SRGB,         ASTC_12X10,            UNORM, _),
+   AIL_FMT(ASTC_12x12_SRGB,         ASTC_12X12,            UNORM, _),
+
+   AIL_FMT(ASTC_4x4_FLOAT,          ASTC_4X4,              FLOAT, _),
+   AIL_FMT(ASTC_5x4_FLOAT,          ASTC_5X4,              FLOAT, _),
+   AIL_FMT(ASTC_5x5_FLOAT,          ASTC_5X5,              FLOAT, _),
+   AIL_FMT(ASTC_6x5_FLOAT,          ASTC_6X5,              FLOAT, _),
+   AIL_FMT(ASTC_6x6_FLOAT,          ASTC_6X6,              FLOAT, _),
+   AIL_FMT(ASTC_8x5_FLOAT,          ASTC_8X5,              FLOAT, _),
+   AIL_FMT(ASTC_8x6_FLOAT,          ASTC_8X6,              FLOAT, _),
+   AIL_FMT(ASTC_8x8_FLOAT,          ASTC_8X8,              FLOAT, _),
+   AIL_FMT(ASTC_10x5_FLOAT,         ASTC_10X5,             FLOAT, _),
+   AIL_FMT(ASTC_10x6_FLOAT,         ASTC_10X6,             FLOAT, _),
+   AIL_FMT(ASTC_10x8_FLOAT,         ASTC_10X8,             FLOAT, _),
+   AIL_FMT(ASTC_10x10_FLOAT,        ASTC_10X10,            FLOAT, _),
+   AIL_FMT(ASTC_12x10_FLOAT,        ASTC_12X10,            FLOAT, _),
+   AIL_FMT(ASTC_12x12_FLOAT,        ASTC_12X12,            FLOAT, _),
+
+   AIL_FMT(DXT1_RGB,                BC1,                   UNORM, _),
+   AIL_FMT(DXT1_RGBA,               BC1,                   UNORM, _),
+   AIL_FMT(DXT1_SRGB,               BC1,                   UNORM, _),
+   AIL_FMT(DXT1_SRGBA,              BC1,                   UNORM, _),
+   AIL_FMT(DXT3_RGBA,               BC2,                   UNORM, _),
+   AIL_FMT(DXT3_SRGBA,              BC2,                   UNORM, _),
+   AIL_FMT(DXT5_RGBA,               BC3,                   UNORM, _),
+   AIL_FMT(DXT5_SRGBA,              BC3,                   UNORM, _),
+   AIL_FMT(RGTC1_UNORM,             BC4,                   UNORM, _),
+   AIL_FMT(RGTC1_SNORM,             BC4,                   SNORM, _),
+   AIL_FMT(RGTC2_UNORM,             BC5,                   UNORM, _),
+   AIL_FMT(RGTC2_SNORM,             BC5,                   SNORM, _),
+   AIL_FMT(BPTC_RGB_FLOAT,          BC6H,                  FLOAT, _),
+   AIL_FMT(BPTC_RGB_UFLOAT,         BC6H_UFLOAT,           FLOAT, _),
+   AIL_FMT(BPTC_RGBA_UNORM,         BC7,                   UNORM, _),
+   AIL_FMT(BPTC_SRGBA,              BC7,                   UNORM, _),
+};
+/* clang-format on */

src/asahi/layout/layout.c

@@ -0,0 +1,388 @@
+/*
+ * Copyright 2022 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "layout.h"
+#include "util/format/u_format.h"
+#include "util/u_math.h"
+
+static void
+ail_initialize_linear(struct ail_layout *layout)
+{
+   /* Select the optimal stride if none is forced */
+   if (layout->linear_stride_B == 0) {
+      uint32_t minimum_stride_B =
+         util_format_get_stride(layout->format, layout->width_px);
+
+      layout->linear_stride_B = ALIGN_POT(minimum_stride_B, AIL_CACHELINE);
+   }
+
+   assert((layout->linear_stride_B % 16) == 0 && "Strides must be aligned");
+
+   /* Layer stride must be cache line aligned to pack linear 2D arrays */
+   layout->layer_stride_B = align64(
+      (uint64_t)layout->linear_stride_B * layout->height_px, AIL_CACHELINE);
+
+   layout->size_B =
+      layout->level_offsets_B[0] + (layout->layer_stride_B * layout->depth_px);
+}
+
+/*
+ * Get the maximum tile size possible for a given block size. This satisfy
+ * width * height * blocksize = 16384 = page size, so each tile is one page.
+ */
+struct ail_tile
+ail_get_max_tile_size(unsigned blocksize_B)
+{
+   /* clang-format off */
+   switch (blocksize_B) {
+   case  1: return (struct ail_tile) { 128, 128 };
+   case  2: return (struct ail_tile) { 128,  64 };
+   case  4: return (struct ail_tile) {  64,  64 };
+   case  8: return (struct ail_tile) {  64,  32 };
+   case 16: return (struct ail_tile) {  32,  32 };
+   case 32: return (struct ail_tile) {  32,  16 };
+   case 64: return (struct ail_tile) {  16,  16 };
+   default: UNREACHABLE("Invalid blocksize");
+   }
+   /* clang-format on */
+}
+
+/*
+ * Calculate the number of bytes in a block. This must take both block
+ * dimensions and multisampling into account.
+ */
+static uint32_t
+ail_get_block_size_B(struct ail_layout *layout)
+{
+   ASSERTED const struct util_format_description *desc =
+      util_format_description(layout->format);
+
+   assert(((layout->sample_count_sa == 1) ||
+           (desc->block.width == 1 && desc->block.height == 1)) &&
+          "multisampling and block-compression are mutually-exclusive");
+
+   return util_format_get_blocksize(layout->format) * layout->sample_count_sa;
+}
+
+static void
+ail_initialize_gpu_tiled(struct ail_layout *layout)
+{
+   unsigned offset_B = 0;
+   unsigned blocksize_B = ail_get_block_size_B(layout);
+   unsigned w_el = util_format_get_nblocksx(layout->format, layout->width_px);
+   unsigned h_el = util_format_get_nblocksy(layout->format, layout->height_px);
+   unsigned bw_px = util_format_get_blockwidth(layout->format);
+   unsigned bh_px = util_format_get_blockheight(layout->format);
+   bool compressed = util_format_is_compressed(layout->format);
+
+   /* Calculate the tile size used for the large miptree, and the dimensions of
+    * level 0 given that tile size.
+    */
+   struct ail_tile tilesize_el = ail_get_max_tile_size(blocksize_B);
+   unsigned stx_tiles = DIV_ROUND_UP(w_el, tilesize_el.width_el);
+   unsigned sty_tiles = DIV_ROUND_UP(h_el, tilesize_el.height_el);
+   unsigned sarea_tiles = stx_tiles * sty_tiles;
+
+   /* Calculate which level the small power-of-two miptree begins at. The
+    * power-of-two miptree is used when either the width or the height is
+    * smaller than a single large tile.
+    */
+   unsigned pot_level = 0;
+   unsigned pot_w_px = bw_px * w_el;
+   unsigned pot_h_px = bh_px * h_el;
+   do {
+      unsigned pot_w_el = util_format_get_nblocksx(layout->format, pot_w_px);
+      unsigned pot_h_el = util_format_get_nblocksy(layout->format, pot_h_px);
+      if (pot_w_el < tilesize_el.width_el || pot_h_el < tilesize_el.height_el)
+         break;
+      pot_w_px = u_minify(pot_w_px, 1);
+      pot_h_px = u_minify(pot_h_px, 1);
+      pot_level++;
+   } while (1);
+
+   /* First allocate the large miptree. All tiles in the large miptree are of
+    * size tilesize_el and have their dimensions given by stx/sty/sarea.
+    */
+   for (unsigned l = 0; l < MIN2(pot_level, layout->levels); ++l) {
+      unsigned tiles = (sarea_tiles >> (2 * l));
+
+      bool pad_left = (stx_tiles & BITFIELD_MASK(l));
+      bool pad_bottom = (sty_tiles & BITFIELD_MASK(l));
+      bool pad_corner = pad_left && pad_bottom;
+
+      if (pad_left)
+         tiles += (sty_tiles >> l);
+
+      if (pad_bottom)
+         tiles += (stx_tiles >> l);
+
+      if (pad_corner)
+         tiles += 1;
+
+      unsigned size_el = tiles * tilesize_el.width_el * tilesize_el.height_el;
+      layout->level_offsets_B[l] = offset_B;
+      offset_B = ALIGN_POT(offset_B + (blocksize_B * size_el), AIL_CACHELINE);
+
+      layout->stride_el[l] = util_format_get_nblocksx(
+         layout->format, u_minify(layout->width_px, l));
+
+      /* Compressed textures pad the stride in this case */
+      if (compressed && pad_left)
+         layout->stride_el[l]++;
+
+      layout->tilesize_el[l] = tilesize_el;
+   }
+
+   /* Then begin the POT miptree. Note that we round up to a power-of-two
+    * outside the loop. That ensures correct handling of cases like 33x33
+    * images, where the round-down error of right-shifting could cause incorrect
+    * tile size calculations.
+    */
+   unsigned potw_el, poth_el;
+   if (compressed) {
+      /* Compressed formats round then minify instead of minifying then rounding
+       */
+      potw_el = u_minify(util_next_power_of_two(w_el), pot_level);
+      poth_el = u_minify(util_next_power_of_two(h_el), pot_level);
+   } else {
+      potw_el = util_next_power_of_two(u_minify(w_el, pot_level));
+      poth_el = util_next_power_of_two(u_minify(h_el, pot_level));
+   }
+
+   /* Finally we allocate the POT miptree, starting at level pot_level. Each
+    * level uses the largest power-of-two tile that fits the level.
+    */
+   for (unsigned l = pot_level; l < layout->levels; ++l) {
+      unsigned size_el = potw_el * poth_el;
+      layout->level_offsets_B[l] = offset_B;
+      offset_B = ALIGN_POT(offset_B + (blocksize_B * size_el), AIL_CACHELINE);
+
+      /* The tilesize is based on the true mipmap level size, not the POT
+       * rounded size, except for compressed textures */
+      unsigned tilesize_el;
+      if (compressed)
+         tilesize_el = util_next_power_of_two(MIN2(potw_el, poth_el));
+      else
+         tilesize_el = util_next_power_of_two(u_minify(MIN2(w_el, h_el), l));
+      layout->tilesize_el[l] = (struct ail_tile){tilesize_el, tilesize_el};
+      layout->stride_el[l] = util_format_get_nblocksx(
+         layout->format, u_minify(layout->width_px, l));
+
+      potw_el = u_minify(potw_el, 1);
+      poth_el = u_minify(poth_el, 1);
+   }
+
+   /* Add the end offset so we can easily recover the size of a level */
+   assert(layout->levels < ARRAY_SIZE(layout->level_offsets_B));
+   layout->level_offsets_B[layout->levels] = offset_B;
+
+   /* Determine the start of the miptail. From that level on, we can no longer
+    * precisely bind at page granularity.
+    */
+   layout->mip_tail_first_lod = MIN2(pot_level, layout->levels);
+
+   /* Determine the stride of the miptail. Sparse arrayed images inherently
+    * require page-aligned layers to be able to bind individual layers.
+    */
+   unsigned tail_offset_B = layout->level_offsets_B[layout->mip_tail_first_lod];
+   layout->mip_tail_stride = align(offset_B - tail_offset_B, AIL_PAGESIZE);
+
+   /* Align layer size if we have mipmaps and one miptree is larger than one
+    * page */
+   layout->page_aligned_layers = layout->levels != 1 && offset_B > AIL_PAGESIZE;
+
+   /* Single-layer images are not padded unless they are Z/S */
+   bool zs = util_format_is_depth_or_stencil(layout->format);
+   if (layout->depth_px == 1 && !zs)
+      layout->page_aligned_layers = false;
+
+   /* For writable images, we require page-aligned layers. This appears to be
+    * required for PBE stores, including block stores for colour rendering.
+    * Likewise, we specify the ZLS layer stride in pages, so we need
+    * page-aligned layers for renderable depth/stencil targets.
+    */
+   layout->page_aligned_layers |= layout->writeable_image;
+   layout->page_aligned_layers |= layout->renderable && layout->depth_px > 1;
+
+   if (layout->page_aligned_layers)
+      layout->layer_stride_B = ALIGN_POT(offset_B, AIL_PAGESIZE);
+   else
+      layout->layer_stride_B = offset_B;
+
+   layout->size_B = (uint64_t)layout->layer_stride_B * layout->depth_px;
+}
+
+static void
+ail_initialize_twiddled(struct ail_layout *layout)
+{
+   unsigned offset_B = 0;
+   unsigned blocksize_B = ail_get_block_size_B(layout);
+   unsigned w_el = util_format_get_nblocksx(layout->format, layout->width_px);
+   unsigned h_el = util_format_get_nblocksy(layout->format, layout->height_px);
+   bool compressed = util_format_is_compressed(layout->format);
+
+   for (unsigned l = 0; l < layout->levels; ++l) {
+      unsigned alloc_w_el = u_minify(util_next_power_of_two(w_el), l);
+      unsigned alloc_h_el = u_minify(util_next_power_of_two(h_el), l);
+      unsigned logical_w_el = util_next_power_of_two(u_minify(w_el, l));
+      unsigned logical_h_el = util_next_power_of_two(u_minify(h_el, l));
+
+      /* AGX: Even the exceptions have exceptions */
+      if (compressed) {
+         logical_w_el = alloc_w_el;
+         logical_h_el = alloc_h_el;
+      }
+
+      unsigned size_el = alloc_w_el * alloc_h_el;
+      layout->level_offsets_B[l] = offset_B;
+      offset_B = ALIGN_POT(offset_B + (blocksize_B * size_el), AIL_CACHELINE);
+
+      unsigned minor_el = MIN2(logical_w_el, logical_h_el);
+      layout->stride_el[l] = logical_w_el;
+      layout->tilesize_el[l] = (struct ail_tile){minor_el, minor_el};
+   }
+
+   /* Add the end offset so we can easily recover the size of a level */
+   assert(layout->levels < ARRAY_SIZE(layout->level_offsets_B));
+   layout->level_offsets_B[layout->levels] = offset_B;
+
+   /* Determine the start of the miptail. From that level on, we can no longer
+    * precisely bind at page granularity.
+    */
+   /* XXX TODO */
+   layout->mip_tail_first_lod = 0;
+
+   /* Determine the stride of the miptail. Sparse arrayed images inherently
+    * require page-aligned layers to be able to bind individual layers.
+    */
+   unsigned tail_offset_B = layout->level_offsets_B[layout->mip_tail_first_lod];
+   layout->mip_tail_stride = align(offset_B - tail_offset_B, AIL_PAGESIZE);
+
+   layout->page_aligned_layers = true;
+   layout->layer_stride_B = ALIGN_POT(offset_B, AIL_PAGESIZE);
+   layout->size_B = (uint64_t)layout->layer_stride_B * layout->depth_px;
+}
+
+static void
+ail_initialize_compression(struct ail_layout *layout)
+{
+   assert(!util_format_is_compressed(layout->format) &&
+          "Compressed pixel formats not supported");
+   assert(util_format_get_blockwidth(layout->format) == 1);
+   assert(util_format_get_blockheight(layout->format) == 1);
+
+   unsigned width_sa =
+      ail_effective_width_sa(layout->width_px, layout->sample_count_sa);
+   unsigned height_sa =
+      ail_effective_height_sa(layout->height_px, layout->sample_count_sa);
+
+   assert(width_sa >= 16 && "Small textures are never compressed");
+   assert(height_sa >= 16 && "Small textures are never compressed");
+
+   layout->metadata_offset_B = layout->size_B;
+
+   width_sa = ALIGN_POT(width_sa, 16);
+   height_sa = ALIGN_POT(height_sa, 16);
+
+   unsigned compbuf_B = 0;
+
+   for (unsigned l = 0;
+        l < layout->levels && ail_is_level_allocated_compressed(layout, l);
+        ++l) {
+
+      layout->level_offsets_compressed_B[l] = compbuf_B;
+
+      /* The metadata buffer contains 8 bytes per 16x16 compression tile.
+       * Addressing is fully twiddled, so both width and height are padded to
+       * powers-of-two.
+       */
+      unsigned w_tl = DIV_ROUND_UP(util_next_power_of_two(width_sa), 16);
+      unsigned h_tl = DIV_ROUND_UP(util_next_power_of_two(height_sa), 16);
+      unsigned B_per_tl_2 = 8;
+
+      compbuf_B += ALIGN_POT(w_tl * h_tl * B_per_tl_2, AIL_CACHELINE);
+
+      width_sa = DIV_ROUND_UP(width_sa, 2);
+      height_sa = DIV_ROUND_UP(height_sa, 2);
+   }
+
+   layout->compression_layer_stride_B = compbuf_B;
+   layout->size_B +=
+      (uint64_t)(layout->compression_layer_stride_B * layout->depth_px);
+}
+
+static void
+ail_initialize_sparse_table(struct ail_layout *layout)
+{
+   layout->sparse_folios_per_layer =
+      DIV_ROUND_UP(layout->layer_stride_B, AIL_IMAGE_SIZE_PER_FOLIO_B);
+
+   unsigned folios = layout->sparse_folios_per_layer * layout->depth_px;
+   layout->sparse_table_size_B = folios * AIL_FOLIO_SIZE_B;
+}
+
+void
+ail_make_miptree(struct ail_layout *layout)
+{
+   assert(layout->width_px >= 1 && "Invalid dimensions");
+   assert(layout->height_px >= 1 && "Invalid dimensions");
+   assert(layout->depth_px >= 1 && "Invalid dimensions");
+
+   if (layout->tiling == AIL_TILING_LINEAR) {
+      assert(layout->levels == 1 && "Invalid linear layout");
+      assert(layout->sample_count_sa == 1 &&
+             "Multisampled linear layouts not supported");
+      assert(util_format_get_blockwidth(layout->format) == 1 &&
+             "Strided linear block formats unsupported");
+      assert(util_format_get_blockheight(layout->format) == 1 &&
+             "Strided linear block formats unsupported");
+   } else {
+      assert(layout->linear_stride_B == 0 && "Invalid nonlinear layout");
+      assert(layout->levels >= 1 && "Invalid dimensions");
+      assert(layout->sample_count_sa >= 1 && "Invalid sample count");
+      assert(layout->level_offsets_B[0] == 0 && "Invalid offset");
+   }
+
+   assert(!(layout->writeable_image && layout->compressed) &&
+          "Writeable images must not be compressed");
+
+   /* Hardware strides are based on the maximum number of levels, so always
+    * allocate them all.
+    */
+   if (layout->levels > 1) {
+      unsigned major_axis_px = MAX2(layout->width_px, layout->height_px);
+
+      if (layout->mipmapped_z)
+         major_axis_px = MAX2(major_axis_px, layout->depth_px);
+
+      layout->levels = util_logbase2(major_axis_px) + 1;
+   }
+
+   assert(util_format_get_blockdepth(layout->format) == 1 &&
+          "Deep formats unsupported");
+
+   switch (layout->tiling) {
+   case AIL_TILING_LINEAR:
+      ail_initialize_linear(layout);
+      break;
+   case AIL_TILING_GPU:
+      ail_initialize_gpu_tiled(layout);
+      break;
+   case AIL_TILING_TWIDDLED:
+      ail_initialize_twiddled(layout);
+      break;
+   default:
+      UNREACHABLE("Unsupported tiling");
+   }
+
+   if (layout->compressed) {
+      ail_initialize_compression(layout);
+   }
+
+   ail_initialize_sparse_table(layout);
+
+   layout->size_B = ALIGN_POT(layout->size_B, AIL_CACHELINE);
+   assert(layout->size_B > 0 && "Invalid dimensions");
+}

src/asahi/layout/layout.h

@@ -0,0 +1,622 @@
+/*
+ * Copyright 2022 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#pragma once
+
+#include "drm-uapi/drm_fourcc.h"
+#include "util/format/u_format.h"
+#include "util/macros.h"
+#include "util/u_math.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define AIL_CACHELINE      0x80
+#define AIL_PAGESIZE       0x4000
+#define AIL_MAX_MIP_LEVELS 16
+
+enum ail_tiling {
+   /**
+    * Strided linear (raster order). Only allowed for 1D or 2D, without
+    * mipmapping, multisampling, block-compression, or arrays.
+    */
+   AIL_TILING_LINEAR,
+
+   /**
+    * GPU-tiled. Always allowed.
+    */
+   AIL_TILING_GPU,
+
+   /**
+    * Fully twiddled.
+    */
+   AIL_TILING_TWIDDLED,
+};
+
+/*
+ * Represents the dimensions of a single tile. Used to describe tiled layouts.
+ * Width and height are in units of elements, not pixels, to model compressed
+ * textures corrects.
+ *
+ * Invariant: width_el and height_el are powers of two.
+ */
+struct ail_tile {
+   unsigned width_el, height_el;
+};
+
+/*
+ * An AGX image layout.
+ */
+struct ail_layout {
+   /** Width, height, and depth in pixels at level 0 */
+   uint32_t width_px, height_px, depth_px;
+
+   /** Number of samples per pixel. 1 if multisampling is disabled. */
+   uint8_t sample_count_sa;
+
+   /** Number of miplevels. 1 if no mipmapping is used. */
+   uint8_t levels;
+
+   /** Should this image be mipmapped along the Z-axis in addition to the X- and
+    * Y-axes? This should be set for API-level 3D images, but not 2D arrays or
+    * cubes.
+    */
+   bool mipmapped_z;
+
+   /** Tiling mode used */
+   enum ail_tiling tiling;
+
+   /** Whether compression is used. Requires a non-linear layout. */
+   bool compressed;
+
+   /** Texture format */
+   enum pipe_format format;
+
+   /**
+    * If tiling is LINEAR, the number of bytes between adjacent rows of
+    * elements. Otherwise, this field is zero.
+    */
+   uint32_t linear_stride_B;
+
+   /**
+    * Stride between layers of an array texture, including a cube map. Layer i
+    * begins at offset (i * layer_stride_B) from the beginning of the texture.
+    *
+    * If depth_px = 1, the value of this field is UNDEFINED.
+    */
+   uint64_t layer_stride_B;
+
+   /**
+    * Whether the layer stride is aligned to the page size or not. The hardware
+    * needs this flag to compute the implicit layer stride.
+    */
+   bool page_aligned_layers;
+
+   /**
+    * Offsets of mip levels within a layer.
+    */
+   uint64_t level_offsets_B[AIL_MAX_MIP_LEVELS];
+
+   /**
+    * For the compressed buffer, offsets of mip levels within a layer.
+    */
+   uint64_t level_offsets_compressed_B[AIL_MAX_MIP_LEVELS];
+
+   /**
+    * If tiling is nonlinear, the tile size used for each mip level within a
+    * layer. Calculating tile sizes is the sole responsibility of
+    * ail_initialized_twiddled.
+    */
+   struct ail_tile tilesize_el[AIL_MAX_MIP_LEVELS];
+
+   /**
+    * If tiling is nonlinear, the stride in elements used for each mip level
+    * within a layer. Calculating level strides is the sole responsibility of
+    * ail_initialized_twiddled. This is necessary because compressed pixel
+    * formats may add extra stride padding.
+    */
+   uint32_t stride_el[AIL_MAX_MIP_LEVELS];
+
+   /**
+    * If the image is bound sparsely, the first LOD/level of the miptail.
+    * This corresponds to Vulkan imageMipTailFirstLod.
+    */
+   uint32_t mip_tail_first_lod;
+
+   /**
+    * If the image is bound sparsely, the layer stride of the miptail.
+    * Conceptually, this corresponds to Vulkan imageMipTailStride. However, it
+    * is not actually used for imageMipTailStride due to complications with
+    * standard sparse block sizes.
+    */
+   uint32_t mip_tail_stride;
+
+   /* Offset of the start of the compression metadata buffer */
+   uint32_t metadata_offset_B;
+
+   /* Stride between subsequent layers in the compression metadata buffer */
+   uint64_t compression_layer_stride_B;
+
+   /* Size of entire texture */
+   uint64_t size_B;
+
+   /* Size of the sparse page table required to address this image. This is
+    * calculated for all images (whether sparse is used or not), since the
+    * hardware has few requirements imposed on sparse.
+    *
+    * This size does not contribute to size_B. In a sparse image, size_B is the
+    * logical size of the image (if it were completely resident), whereas
+    * sparse_table_size_B is the physical size of the page table required to
+    * address that logical image.
+    */
+   uint64_t sparse_table_size_B;
+
+   /* Number of sparse folios required to cover a single layer of an image. This
+    * is proportional to sparse_table_size_B but split out for convenience.
+    */
+   uint32_t sparse_folios_per_layer;
+
+   /* Must the layout support writeable images? If false, the layout MUST NOT be
+    * used as a writeable image (either PBE or image atomics).
+    */
+   bool writeable_image;
+
+   /* Must the layout support rendering? If false, the layout MUST NOT be used
+    * for rendering, either PBE or ZLS.
+    */
+   bool renderable;
+};
+
+static inline uint32_t
+ail_get_linear_stride_B(const struct ail_layout *layout, ASSERTED uint8_t level)
+{
+   assert(layout->tiling == AIL_TILING_LINEAR && "Invalid usage");
+   assert(level == 0 && "Strided linear mipmapped textures are unsupported");
+
+   return layout->linear_stride_B;
+}
+
+/*
+ * For WSI purposes, we need to associate a stride with all layouts. In the
+ * hardware, only strided linear images have an associated stride, there is no
+ * natural stride associated with nonlinear images. However, various clients
+ * assert that the stride is valid for the image if it were linear (even if it
+ * is in fact not linear). In those cases, by convention we use the minimum
+ * valid such stride.
+ */
+static inline uint32_t
+ail_get_wsi_stride_B(const struct ail_layout *layout, unsigned level)
+{
+   assert(level == 0 && "Mipmaps cannot be shared as WSI");
+
+   if (layout->tiling == AIL_TILING_LINEAR)
+      return ail_get_linear_stride_B(layout, level);
+   else
+      return util_format_get_stride(layout->format, layout->width_px);
+}
+
+static inline uint32_t
+ail_get_layer_offset_B(const struct ail_layout *layout, unsigned z_px)
+{
+   return z_px * layout->layer_stride_B;
+}
+
+static inline uint32_t
+ail_get_level_offset_B(const struct ail_layout *layout, unsigned level)
+{
+   return layout->level_offsets_B[level];
+}
+
+static inline uint32_t
+ail_get_layer_level_B(const struct ail_layout *layout, unsigned z_px,
+                      unsigned level)
+{
+   return ail_get_layer_offset_B(layout, z_px) +
+          ail_get_level_offset_B(layout, level);
+}
+
+static inline uint32_t
+ail_get_level_size_B(const struct ail_layout *layout, unsigned level)
+{
+   if (layout->tiling == AIL_TILING_LINEAR) {
+      assert(level == 0);
+      return layout->layer_stride_B;
+   } else {
+      assert(level + 1 < ARRAY_SIZE(layout->level_offsets_B));
+      return layout->level_offsets_B[level + 1] -
+             layout->level_offsets_B[level];
+   }
+}
+
+static inline uint32_t
+ail_get_linear_pixel_B(const struct ail_layout *layout, ASSERTED unsigned level,
+                       uint32_t x_px, uint32_t y_px, uint32_t z_px)
+{
+   assert(level == 0 && "Strided linear mipmapped textures are unsupported");
+   assert(util_format_get_blockwidth(layout->format) == 1 &&
+          "Strided linear block formats unsupported");
+   assert(util_format_get_blockheight(layout->format) == 1 &&
+          "Strided linear block formats unsupported");
+   assert(layout->sample_count_sa == 1 &&
+          "Strided linear multisampling unsupported");
+
+   return ail_get_layer_offset_B(layout, z_px) +
+          (y_px * ail_get_linear_stride_B(layout, level)) +
+          (x_px * util_format_get_blocksize(layout->format));
+}
+
+static inline uint32_t
+ail_space_bits(unsigned x)
+{
+   uint32_t accum = 0;
+
+   /* With fully twiddled images, we can have up to 16384x16384 tiles,
+    * justifying stopping the loop at 14.
+    */
+   assert(x < (1 << 14) && "offset must be inside the tile");
+
+   for (unsigned i = 0; i < 14; ++i) {
+      accum |= (x & (1 << i)) << i;
+   }
+
+   return accum;
+}
+
+#define MOD_POT(x, y) (x) & ((y) - 1)
+
+static inline uint32_t
+ail_get_blocksize_B(const struct ail_layout *layout)
+{
+   return util_format_get_blocksize(layout->format);
+}
+
+static inline uint32_t
+ail_get_twiddled_block_B(const struct ail_layout *layout, unsigned level,
+                         uint32_t x_px, uint32_t y_px, uint32_t z_px)
+{
+   assert(layout->tiling == AIL_TILING_GPU);
+   assert(level < layout->levels);
+
+   unsigned x_el = util_format_get_nblocksx(layout->format, x_px);
+   unsigned y_el = util_format_get_nblocksy(layout->format, y_px);
+
+   struct ail_tile tile_size = layout->tilesize_el[level];
+   assert((x_el % tile_size.width_el) == 0 && "must be aligned");
+   assert((y_el % tile_size.height_el) == 0 && "must be aligned");
+
+   unsigned stride_el = layout->stride_el[level];
+   unsigned tile_area_el = tile_size.width_el * tile_size.height_el;
+   unsigned tiles_per_row = DIV_ROUND_UP(stride_el, tile_size.width_el);
+
+   unsigned y_rowtile = y_el / tile_size.height_el;
+   unsigned y_tile = y_rowtile * tiles_per_row;
+
+   unsigned tile_idx = y_tile + (x_el / tile_size.width_el);
+   unsigned tile_offset_el = tile_idx * tile_area_el;
+   unsigned tile_offset_B = tile_offset_el * ail_get_blocksize_B(layout);
+
+   return ail_get_layer_level_B(layout, z_px, level) + tile_offset_B;
+}
+
+static inline unsigned
+ail_effective_width_sa(unsigned width_px, unsigned sample_count_sa)
+{
+   return width_px * (sample_count_sa == 4 ? 2 : 1);
+}
+
+static inline unsigned
+ail_effective_height_sa(unsigned height_px, unsigned sample_count_sa)
+{
+   return height_px * (sample_count_sa >= 2 ? 2 : 1);
+}
+
+static inline unsigned
+ail_metadata_width_tl(struct ail_layout *layout, unsigned level)
+{
+   unsigned px = u_minify(layout->width_px, level);
+   uint32_t sa = ail_effective_width_sa(px, layout->sample_count_sa);
+   return DIV_ROUND_UP(sa, 16);
+}
+
+static inline unsigned
+ail_metadata_height_tl(struct ail_layout *layout, unsigned level)
+{
+   unsigned px = u_minify(layout->height_px, level);
+   uint32_t sa = ail_effective_height_sa(px, layout->sample_count_sa);
+   return DIV_ROUND_UP(sa, 16);
+}
+
+/*
+ * Even when the base mip level is compressed, high levels of the miptree
+ * (smaller than 16 pixels on either axis) are not compressed as it would be
+ * pointless. This queries this case.
+ */
+static inline bool
+ail_is_level_allocated_compressed(const struct ail_layout *layout,
+                                  unsigned level)
+{
+   unsigned width_sa = align(
+      ail_effective_width_sa(layout->width_px, layout->sample_count_sa), 16);
+
+   unsigned height_sa = align(
+      ail_effective_height_sa(layout->height_px, layout->sample_count_sa), 16);
+
+   return layout->compressed &&
+          u_minify(MAX2(width_sa, height_sa), level) >= 16;
+}
+
+static inline bool
+ail_is_level_logically_compressed(const struct ail_layout *layout,
+                                  unsigned level)
+{
+   unsigned width_sa =
+      ail_effective_width_sa(layout->width_px, layout->sample_count_sa);
+
+   unsigned height_sa =
+      ail_effective_height_sa(layout->height_px, layout->sample_count_sa);
+
+   return layout->compressed &&
+          u_minify(MIN2(width_sa, height_sa), level) >= 16;
+}
+
+static inline bool
+ail_is_level_twiddled_uncompressed(const struct ail_layout *layout,
+                                   unsigned level)
+{
+   if (layout->compressed) {
+      return !ail_is_level_logically_compressed(layout, level);
+   } else {
+      return layout->tiling != AIL_TILING_LINEAR;
+   }
+}
+
+struct ail_tile ail_get_max_tile_size(unsigned blocksize_B);
+
+void ail_make_miptree(struct ail_layout *layout);
+
+void ail_detile(void *_tiled, void *_linear,
+                const struct ail_layout *tiled_layout, unsigned level,
+                unsigned linear_pitch_B, unsigned sx_px, unsigned sy_px,
+                unsigned width_px, unsigned height_px);
+
+void ail_tile(void *_tiled, void *_linear,
+              const struct ail_layout *tiled_layout, unsigned level,
+              unsigned linear_pitch_B, unsigned sx_px, unsigned sy_px,
+              unsigned width_px, unsigned height_px);
+
+/* Define aliases for the subset formats that are accessible in the ISA. These
+ * subsets disregard component mapping and number of components. This
+ * constitutes ABI with the compiler.
+ */
+enum ail_isa_format {
+   AIL_ISA_FORMAT_I8 = PIPE_FORMAT_R8_UINT,
+   AIL_ISA_FORMAT_I16 = PIPE_FORMAT_R16_UINT,
+   AIL_ISA_FORMAT_I32 = PIPE_FORMAT_R32_UINT,
+   AIL_ISA_FORMAT_F16 = PIPE_FORMAT_R16_FLOAT,
+   AIL_ISA_FORMAT_U8NORM = PIPE_FORMAT_R8_UNORM,
+   AIL_ISA_FORMAT_S8NORM = PIPE_FORMAT_R8_SNORM,
+   AIL_ISA_FORMAT_U16NORM = PIPE_FORMAT_R16_UNORM,
+   AIL_ISA_FORMAT_S16NORM = PIPE_FORMAT_R16_SNORM,
+   AIL_ISA_FORMAT_RGB10A2 = PIPE_FORMAT_R10G10B10A2_UNORM,
+   AIL_ISA_FORMAT_SRGBA8 = PIPE_FORMAT_R8G8B8A8_SRGB,
+   AIL_ISA_FORMAT_RG11B10F = PIPE_FORMAT_R11G11B10_FLOAT,
+   AIL_ISA_FORMAT_RGB9E5 = PIPE_FORMAT_R9G9B9E5_FLOAT
+};
+
+/*
+ * The architecture load/store instructions support masking, but packed formats
+ * are not compatible with masking. Check if a format is packed.
+ */
+static inline bool
+ail_isa_format_supports_mask(enum ail_isa_format format)
+{
+   switch (format) {
+   case AIL_ISA_FORMAT_RGB10A2:
+   case AIL_ISA_FORMAT_RG11B10F:
+   case AIL_ISA_FORMAT_RGB9E5:
+      return false;
+   default:
+      return true;
+   }
+}
+
+struct ail_pixel_format_entry {
+   uint8_t channels;
+   uint8_t type;
+   bool texturable : 1;
+   enum pipe_format renderable;
+};
+
+extern const struct ail_pixel_format_entry ail_pixel_format[PIPE_FORMAT_COUNT];
+
+static inline bool
+ail_is_valid_pixel_format(enum pipe_format format)
+{
+   return ail_pixel_format[format].texturable;
+}
+
+/* Query whether an image with the specified layout is compressible */
+static inline bool
+ail_can_compress(enum pipe_format format, unsigned w_px, unsigned h_px,
+                 unsigned sample_count_sa)
+{
+   assert(sample_count_sa == 1 || sample_count_sa == 2 || sample_count_sa == 4);
+
+   /* We compress via the PBE, so we can only compress PBE-writeable formats. */
+   if (ail_pixel_format[format].renderable == PIPE_FORMAT_NONE &&
+       !util_format_is_depth_or_stencil(format))
+      return false;
+
+   /* Lossy-compressed texture formats cannot be compressed */
+   assert(!util_format_is_compressed(format) &&
+          "block-compressed formats are not renderable");
+
+   /* Small textures cannot be compressed */
+   return ail_effective_width_sa(w_px, sample_count_sa) >= 16 &&
+          ail_effective_height_sa(h_px, sample_count_sa) >= 16;
+}
+
+/* AGX compression mode for an uncompessed subtile. Frustratingly, this seems to
+ * depend on the format. It is possible that modes are actual 8-bit structures
+ * with multiple fields rather than plain enumerations.
+ */
+#define AIL_COMP_UNCOMPRESSED_1    0x1f
+#define AIL_COMP_UNCOMPRESSED_2    0x3f
+#define AIL_COMP_UNCOMPRESSED_4    0x7f
+#define AIL_COMP_UNCOMPRESSED_8_16 0xff
+
+/* AGX compression mode for a solid colour for the subtile. */
+#define AIL_COMP_SOLID_1    0x60
+#define AIL_COMP_SOLID_2    0x01
+#define AIL_COMP_SOLID_4    0x03
+#define AIL_COMP_SOLID_8_16 0x07
+
+static inline uint8_t
+ail_subtile_uncompressed_mode(enum pipe_format format)
+{
+   /* clang-format off */
+   switch (util_format_get_blocksize(format)) {
+   case  1: return AIL_COMP_UNCOMPRESSED_1;
+   case  2: return AIL_COMP_UNCOMPRESSED_2;
+   case  4: return AIL_COMP_UNCOMPRESSED_4;
+   case  8:
+   case 16: return AIL_COMP_UNCOMPRESSED_8_16;
+   default: UNREACHABLE("invalid block size");
+   }
+   /* clang-format on */
+}
+
+static inline uint8_t
+ail_subtile_solid_mode(enum pipe_format format)
+{
+   /* clang-format off */
+   switch (util_format_get_blocksize(format)) {
+   case  1: return AIL_COMP_SOLID_1;
+   case  2: return AIL_COMP_SOLID_2;
+   case  4: return AIL_COMP_SOLID_4;
+   case  8:
+   case 16: return AIL_COMP_SOLID_8_16;
+   default: UNREACHABLE("invalid block size");
+   }
+   /* clang-format on */
+}
+
+/*
+ * Compression modes are 8-bit per 8x4 subtile, but grouped into 64-bit for all
+ * modes in a 16x16 tile. This helper replicates a subtile mode to a tile mode
+ * using a SWAR idiom.
+ */
+static inline uint64_t
+ail_tile_mode_replicated(uint8_t subtile_mode)
+{
+   return (uint64_t)subtile_mode * 0x0101010101010101ULL;
+}
+
+/*
+ * Composed convenience function.
+ */
+static inline uint64_t
+ail_tile_mode_uncompressed(enum pipe_format format)
+{
+   return ail_tile_mode_replicated(ail_subtile_uncompressed_mode(format));
+}
+
+static inline uint64_t
+ail_tile_mode_solid(enum pipe_format format)
+{
+   return ail_tile_mode_replicated(ail_subtile_solid_mode(format));
+}
+
+/*
+ * For compression, compatible formats must have the same number/size/order of
+ * channels, but may differ in data type. For example, R32_SINT is compatible
+ * with Z32_FLOAT, but not with R16G16_SINT. This is the relation given by the
+ * "channels" part of the decomposed format.
+ *
+ * This has not been exhaustively tested and might be missing some corner cases
+ * around XR formats, but is well-motivated and seems to work.
+ */
+static inline bool
+ail_formats_compatible(enum pipe_format a, enum pipe_format b)
+{
+   return ail_pixel_format[a].channels == ail_pixel_format[b].channels;
+}
+
+static inline bool
+ail_is_view_compatible(struct ail_layout *layout, enum pipe_format view)
+{
+   return !layout->compressed || ail_formats_compatible(layout->format, view);
+}
+
+/*
+ * We generally use ail enums instead of DRM format modifiers. These helpers
+ * bridges the gap.
+ */
+static inline enum ail_tiling
+ail_drm_modifier_to_tiling(uint64_t modifier)
+{
+   switch (modifier) {
+   case DRM_FORMAT_MOD_LINEAR:
+      return AIL_TILING_LINEAR;
+   case DRM_FORMAT_MOD_APPLE_GPU_TILED:
+   case DRM_FORMAT_MOD_APPLE_GPU_TILED_COMPRESSED:
+      return AIL_TILING_GPU;
+   default:
+      UNREACHABLE("Unsupported modifier");
+   }
+}
+
+static inline bool
+ail_is_drm_modifier_compressed(uint64_t modifier)
+{
+   switch (modifier) {
+   case DRM_FORMAT_MOD_LINEAR:
+   case DRM_FORMAT_MOD_APPLE_GPU_TILED:
+      return false;
+   case DRM_FORMAT_MOD_APPLE_GPU_TILED_COMPRESSED:
+      return true;
+   default:
+      UNREACHABLE("Unsupported modifier");
+   }
+}
+
+/*
+ * Constants for sparse page tables. See docs/drivers/asahi.rst.
+ */
+#define AIL_SPARSE_ELSIZE_B        4
+#define AIL_PAGES_PER_FOLIO        256
+#define AIL_IMAGE_SIZE_PER_FOLIO_B (AIL_PAGESIZE * AIL_PAGES_PER_FOLIO)
+#define AIL_FOLIO_SIZE_EL          (AIL_PAGES_PER_FOLIO * 2)
+#define AIL_FOLIO_SIZE_B           (AIL_FOLIO_SIZE_EL * AIL_SPARSE_ELSIZE_B)
+
+static inline unsigned
+ail_bytes_to_pages(unsigned x_B)
+{
+   assert((x_B % AIL_PAGESIZE) == 0);
+   return x_B / AIL_PAGESIZE;
+}
+
+/*
+ * Map a logical page of the image (i.e. page = logical address / page_size) to
+ * a word-index into the sparse table.
+ */
+static inline unsigned
+ail_page_to_sparse_index_el(const struct ail_layout *layout, unsigned z,
+                            unsigned page)
+{
+   unsigned z_base = z * layout->sparse_folios_per_layer;
+   unsigned folio = page / AIL_PAGES_PER_FOLIO;
+   unsigned index_in_folio = page % AIL_PAGES_PER_FOLIO;
+   unsigned idx = ((z_base + folio) * AIL_FOLIO_SIZE_EL) + index_in_folio;
+
+   assert((idx * AIL_SPARSE_ELSIZE_B) < layout->sparse_table_size_B);
+   return idx;
+}
+
+#ifdef __cplusplus
+} /* extern C */
+#endif

src/asahi/layout/meson.build

@@ -0,0 +1,38 @@
+# Copyright 2022 Alyssa Rosenzweig
+# SPDX-License-Identifier: MIT
+
+libasahi_layout_files = files(
+  'formats.c',
+  'layout.c',
+  'tiling.cc',
+)
+
+libasahi_layout = static_library(
+  'asahi_layout',
+  [libasahi_layout_files],
+  include_directories : [inc_include, inc_src],
+  dependencies: [idep_mesautil, idep_agx_pack],
+  c_args : [no_override_init_args],
+  gnu_symbol_visibility : 'hidden',
+  build_by_default : false,
+)
+
+if with_tests
+  test(
+    'asahi_layout_tests',
+    executable(
+      'asahi_layout_tests',
+      files(
+        'tests/test-layout.cpp',
+        'tests/test-generated.cpp',
+        'tests/test-twiddling.cpp',
+      ),
+      c_args : [c_msvc_compat_args, no_override_init_args],
+      gnu_symbol_visibility : 'hidden',
+      dependencies: [idep_gtest, idep_mesautil],
+      link_with : [libasahi_layout],
+    ),
+    suite : ['asahi'],
+    protocol : 'gtest',
+  )
+endif

src/asahi/layout/tests/test-generated.cpp

@@ -0,0 +1,171 @@
+/*
+ * Copyright 2022 Alyssa Rosenzweig
+ * Copyright 2022 Asahi Lina
+ * SPDX-License-Identifier: MIT
+ */
+
+#include <gtest/gtest.h>
+#include "layout.h"
+
+/*
+ * Test texture layouts with test cases extracted from texture structure dumps
+ * produced by Metal.
+ *
+ * The extracted test cases are stored in separate files which do not get
+ * clang-formatted. They do still get parsed as C code, though. They may be
+ * sorted with `sort -t"," -k1,1 -k2,5n | uniq`.
+ */
+struct sizetest {
+   enum pipe_format format;
+   uint32_t width, height, depth;
+   uint8_t levels;
+   uint32_t size;
+};
+
+struct miptest {
+   enum pipe_format format;
+   uint32_t width, height;
+   uint8_t levels;
+   uint32_t offsets[16];
+};
+
+struct msaatest {
+   enum pipe_format format;
+   uint32_t width, height, depth;
+   uint8_t levels;
+   uint8_t samples;
+   bool is_compressed;
+   uint32_t meta_offset;
+   uint32_t size;
+};
+
+static struct sizetest comptests[] = {
+#include "comp-twiddled.txt"
+};
+
+static struct sizetest sizetests[] = {
+#include "uncomp-twiddled.txt"
+};
+
+static struct miptest miptests[] = {
+#include "miptree.txt"
+};
+
+static struct msaatest msaatests[] = {
+#include "msaa.txt"
+};
+
+TEST(Generated, CompTwiddled)
+{
+   for (unsigned i = 0; i < ARRAY_SIZE(comptests); ++i) {
+      struct sizetest test = comptests[i];
+
+      struct ail_layout layout = {
+         .width_px = test.width,
+         .height_px = test.height,
+         .depth_px = test.depth,
+         .sample_count_sa = 1,
+         .levels = test.levels,
+         .tiling = AIL_TILING_GPU,
+         .compressed = true,
+         .format = test.format,
+      };
+
+      ail_make_miptree(&layout);
+
+      EXPECT_EQ(layout.size_B, test.size)
+         << test.width << "x" << test.height << "x" << test.depth << " "
+         << (int)test.levels << "L " << util_format_short_name(test.format)
+         << " compressed texture has wrong allocation size, off by "
+         << ((int)layout.size_B - (int)test.size);
+   }
+}
+
+TEST(Generated, UncompTwiddled)
+{
+   for (unsigned i = 0; i < ARRAY_SIZE(sizetests); ++i) {
+      struct sizetest test = sizetests[i];
+
+      struct ail_layout layout = {
+         .width_px = test.width,
+         .height_px = test.height,
+         .depth_px = test.depth,
+         .sample_count_sa = 1,
+         .levels = test.levels,
+         .tiling = AIL_TILING_GPU,
+         .format = test.format,
+      };
+
+      ail_make_miptree(&layout);
+
+      EXPECT_EQ(layout.size_B, test.size)
+         << test.width << "x" << test.height << "x" << test.depth << " "
+         << (int)test.levels << "L " << util_format_short_name(test.format)
+         << " uncompressed texture has wrong allocation size, off by "
+         << ((int)layout.size_B - (int)test.size);
+   }
+}
+
+TEST(Generated, Miptree2D)
+{
+   for (unsigned i = 0; i < ARRAY_SIZE(miptests); ++i) {
+      struct miptest test = miptests[i];
+
+      struct ail_layout layout = {
+         .width_px = test.width,
+         .height_px = test.height,
+         .depth_px = 1,
+         .sample_count_sa = 1,
+         .levels = test.levels,
+         .tiling = AIL_TILING_GPU,
+         .format = test.format,
+      };
+
+      ail_make_miptree(&layout);
+
+      for (unsigned l = 0; l < test.levels; ++l) {
+         EXPECT_EQ(ail_get_level_offset_B(&layout, l), test.offsets[l])
+            << test.width << "x" << test.height << " "
+            << util_format_short_name(test.format)
+            << " texture has wrong offset at level " << l << ", off by "
+            << test.offsets[l] - ail_get_level_offset_B(&layout, l);
+      }
+   }
+}
+
+TEST(Generated, MSAA)
+{
+   for (unsigned i = 0; i < ARRAY_SIZE(msaatests); ++i) {
+      struct msaatest test = msaatests[i];
+
+      struct ail_layout layout = {
+         .width_px = test.width,
+         .height_px = test.height,
+         .depth_px = test.depth,
+         .sample_count_sa = test.samples,
+         .levels = test.levels,
+         .tiling = AIL_TILING_GPU,
+         .compressed = test.is_compressed,
+         .format = test.format,
+      };
+
+      ail_make_miptree(&layout);
+
+      EXPECT_EQ(layout.size_B, test.size)
+         << test.width << "x" << test.height << "x" << test.depth << " "
+         << (int)test.levels << "L " << (int)test.samples << "S "
+         << util_format_short_name(test.format)
+         << (test.is_compressed ? " " : " un")
+         << "compressed texture has wrong allocation size, off by "
+         << ((int)layout.size_B - (int)test.size);
+
+      if (test.is_compressed) {
+         EXPECT_EQ(layout.metadata_offset_B, test.meta_offset)
+            << test.width << "x" << test.height << "x" << test.depth << " "
+            << (int)test.levels << "L " << (int)test.samples << "S "
+            << util_format_short_name(test.format)
+            << "compressed texture has wrong metadata offset size, off by "
+            << ((int)layout.metadata_offset_B - (int)test.meta_offset);
+      }
+   }
+}

src/asahi/layout/tests/test-layout.cpp

@@ -0,0 +1,145 @@
+/*
+ * Copyright 2022 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#include <gtest/gtest.h>
+#include "layout.h"
+
+TEST(Cubemap, Nonmipmapped)
+{
+   struct ail_layout layout = {
+      .width_px = 512,
+      .height_px = 512,
+      .depth_px = 6,
+      .sample_count_sa = 1,
+      .levels = 1,
+      .tiling = AIL_TILING_GPU,
+      .format = PIPE_FORMAT_R8G8B8A8_UNORM,
+   };
+
+   ail_make_miptree(&layout);
+
+   EXPECT_EQ(layout.layer_stride_B, ALIGN_POT(512 * 512 * 4, 0x4000));
+   EXPECT_EQ(layout.size_B, ALIGN_POT(512 * 512 * 4 * 6, 0x4000));
+}
+
+TEST(Cubemap, RoundsToOnePage)
+{
+   struct ail_layout layout = {
+      .width_px = 63,
+      .height_px = 63,
+      .depth_px = 6,
+      .sample_count_sa = 1,
+      .levels = 6,
+      .tiling = AIL_TILING_GPU,
+      .format = PIPE_FORMAT_R32_FLOAT,
+   };
+
+   ail_make_miptree(&layout);
+
+   EXPECT_EQ(layout.level_offsets_B[0], 0);
+   EXPECT_EQ(layout.level_offsets_B[1], 0x4000);
+   EXPECT_EQ(layout.level_offsets_B[2], 0x5000);
+   EXPECT_EQ(layout.level_offsets_B[3], 0x5400);
+   EXPECT_EQ(layout.level_offsets_B[4], 0x5500);
+   EXPECT_TRUE(layout.page_aligned_layers);
+   EXPECT_EQ(layout.layer_stride_B, 0x8000);
+   EXPECT_EQ(layout.size_B, 0x30000);
+}
+
+TEST(Linear, SmokeTestBuffer)
+{
+   struct ail_layout layout = {
+      .width_px = 81946,
+      .height_px = 1,
+      .depth_px = 1,
+      .sample_count_sa = 1,
+      .levels = 1,
+      .tiling = AIL_TILING_LINEAR,
+      .format = PIPE_FORMAT_R8_UINT,
+   };
+
+   ail_make_miptree(&layout);
+
+   EXPECT_EQ(layout.size_B, ALIGN_POT(81946, AIL_CACHELINE));
+}
+
+TEST(Miptree, AllMipLevels)
+{
+   struct ail_layout layout = {
+      .width_px = 1024,
+      .height_px = 1024,
+      .depth_px = 1,
+      .sample_count_sa = 1,
+      .levels = 11,
+      .tiling = AIL_TILING_GPU,
+      .format = PIPE_FORMAT_R8G8B8A8_UINT,
+   };
+
+   ail_make_miptree(&layout);
+
+   EXPECT_EQ(layout.size_B, 0x555680);
+   EXPECT_EQ(layout.sparse_folios_per_layer, 2);
+   EXPECT_EQ(layout.sparse_table_size_B, 0x1000);
+}
+
+TEST(Miptree, SomeMipLevels)
+{
+   struct ail_layout layout = {
+      .width_px = 1024,
+      .height_px = 1024,
+      .depth_px = 1,
+      .sample_count_sa = 1,
+      .levels = 4,
+      .tiling = AIL_TILING_GPU,
+      .format = PIPE_FORMAT_R8G8B8A8_UINT,
+   };
+
+   ail_make_miptree(&layout);
+
+   EXPECT_EQ(layout.size_B, 0x555680);
+   EXPECT_EQ(layout.sparse_folios_per_layer, 2);
+   EXPECT_EQ(layout.sparse_table_size_B, 0x1000);
+}
+
+TEST(Miptree, SmallPartialMiptree2DArray)
+{
+   struct ail_layout layout = {
+      .width_px = 32,
+      .height_px = 16,
+      .depth_px = 64,
+      .sample_count_sa = 1,
+      .levels = 4,
+      .tiling = AIL_TILING_GPU,
+      .format = PIPE_FORMAT_R32_FLOAT,
+   };
+
+   ail_make_miptree(&layout);
+
+   EXPECT_EQ(layout.layer_stride_B, 0xc00);
+   EXPECT_EQ(layout.size_B, 0x30000);
+   EXPECT_EQ(layout.sparse_folios_per_layer, 1);
+   EXPECT_EQ(layout.sparse_table_size_B, 0x20000);
+}
+
+TEST(Miptree, SmallPartialMiptree3D)
+{
+   struct ail_layout layout = {
+      .width_px = 32,
+      .height_px = 16,
+      .depth_px = 64,
+      .sample_count_sa = 1,
+      .levels = 4,
+      .mipmapped_z = true,
+      .tiling = AIL_TILING_GPU,
+      .format = PIPE_FORMAT_R32_FLOAT,
+   };
+
+   ail_make_miptree(&layout);
+
+   EXPECT_EQ(layout.layer_stride_B, 0xc80);
+   EXPECT_EQ(layout.size_B, 0x32000);
+   EXPECT_EQ(layout.sparse_folios_per_layer, 1);
+   EXPECT_EQ(layout.sparse_table_size_B, 0x20000);
+}

src/asahi/layout/tests/test-twiddling.cpp

@@ -0,0 +1,257 @@
+/*
+ * Copyright 2024 Alyssa Rosenzweig
+ * Copyright 2022 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "layout.h"
+
+#include "util/format/u_format.h"
+#include <gtest/gtest.h>
+
+/*
+ * Reference tiling algorithm, written for clarity rather than performance. See
+ * docs/drivers/asahi.rst for details on the format.
+ */
+
+static unsigned
+z_order(unsigned x, unsigned y)
+{
+   unsigned out = 0;
+
+   for (unsigned i = 0; i < 8; ++i) {
+      unsigned bit = (1 << (2 * i));
+
+      if (x & (1 << i))
+         out |= bit;
+
+      if (y & (1 << i))
+         out |= bit << 1;
+   }
+
+   return out;
+}
+
+/* x/y are in blocks */
+static unsigned
+tiled_offset_el(struct ail_layout *layout, unsigned level, unsigned x_el,
+                unsigned y_el)
+{
+   unsigned x_tl = x_el / layout->tilesize_el[level].width_el;
+   unsigned y_tl = y_el / layout->tilesize_el[level].height_el;
+
+   unsigned offs_x_el = x_el % layout->tilesize_el[level].width_el;
+   unsigned offs_y_el = y_el % layout->tilesize_el[level].height_el;
+
+   unsigned offs_in_tile_el = z_order(offs_x_el, offs_y_el);
+
+   unsigned offs_row_el =
+      y_tl *
+      align(layout->stride_el[level], layout->tilesize_el[level].width_el) *
+      layout->tilesize_el[level].height_el;
+
+   unsigned offs_col_el = x_tl * layout->tilesize_el[level].width_el *
+                          layout->tilesize_el[level].height_el;
+
+   return offs_row_el + offs_col_el + offs_in_tile_el;
+}
+
+static unsigned
+linear_offset_B(unsigned x_el, unsigned y_el, unsigned stride_B,
+                unsigned blocksize_B)
+{
+   return (stride_B * y_el) + (x_el * blocksize_B);
+}
+
+static void
+ref_access_tiled(uint8_t *tiled, uint8_t *linear, struct ail_layout *layout,
+                 unsigned region_x_px, unsigned region_y_px, unsigned w_px,
+                 unsigned h_px, uint32_t linear_stride_B, bool dst_is_tiled)
+{
+   unsigned blocksize_B = util_format_get_blocksize(layout->format);
+
+   unsigned w_el = util_format_get_nblocksx(layout->format, w_px);
+   unsigned h_el = util_format_get_nblocksy(layout->format, h_px);
+
+   unsigned region_x_el = util_format_get_nblocksx(layout->format, region_x_px);
+   unsigned region_y_el = util_format_get_nblocksy(layout->format, region_y_px);
+
+   for (unsigned linear_y_el = 0; linear_y_el < h_el; ++linear_y_el) {
+      for (unsigned linear_x_el = 0; linear_x_el < w_el; ++linear_x_el) {
+         unsigned tiled_x_el = region_x_el + linear_x_el;
+         unsigned tiled_y_el = region_y_el + linear_y_el;
+
+         uint8_t *linear_ptr =
+            linear + linear_offset_B(linear_x_el, linear_y_el, linear_stride_B,
+                                     blocksize_B);
+
+         unsigned twiddled_offs_B =
+            blocksize_B * tiled_offset_el(layout, 0, tiled_x_el, tiled_y_el);
+
+         /* While we're at it, check that ail_get_twiddled_block_B does the
+          * right thing where it is defined (at tile boundaries). We can't
+          * otherwise use it here as it is part of the routine under test.
+          */
+         if ((tiled_x_el % layout->tilesize_el[0].width_el) == 0 &&
+             (tiled_y_el % layout->tilesize_el[0].height_el) == 0) {
+
+            unsigned tiled_x_px =
+               tiled_x_el * util_format_get_blockwidth(layout->format);
+
+            unsigned tiled_y_px =
+               tiled_y_el * util_format_get_blockwidth(layout->format);
+
+            EXPECT_EQ(
+               ail_get_twiddled_block_B(layout, 0, tiled_x_px, tiled_y_px, 0),
+               twiddled_offs_B);
+         }
+
+         uint8_t *tiled_ptr = tiled + twiddled_offs_B;
+
+         if (dst_is_tiled) {
+            memcpy(tiled_ptr, linear_ptr, blocksize_B);
+         } else {
+            memcpy(linear_ptr, tiled_ptr, blocksize_B);
+         }
+      }
+   }
+}
+
+/*
+ * Helper to build test cases for tiled texture access. This test suite compares
+ * the above reference tiling algorithm to the optimized algorithm used in
+ * production.
+ */
+static void
+test(unsigned width, unsigned height, unsigned rx, unsigned ry, unsigned rw,
+     unsigned rh, unsigned linear_stride, enum pipe_format format, bool store)
+{
+   unsigned bpp = util_format_get_blocksize(format);
+   struct ail_layout layout = {
+      .width_px = width,
+      .height_px = height,
+      .depth_px = 1,
+      .sample_count_sa = 1,
+      .levels = 1,
+      .tiling = AIL_TILING_GPU,
+      .format = format,
+   };
+
+   ail_make_miptree(&layout);
+
+   uint8_t *tiled = (uint8_t *)calloc(bpp, layout.size_B);
+   uint8_t *linear = (uint8_t *)calloc(bpp, rh * linear_stride);
+   uint8_t *ref =
+      (uint8_t *)calloc(bpp, store ? layout.size_B : (rh * linear_stride));
+
+   if (store) {
+      for (unsigned i = 0; i < bpp * rh * linear_stride; ++i) {
+         linear[i] = (i & 0xFF);
+      }
+
+      ail_tile(tiled, linear, &layout, 0, linear_stride, rx, ry, rw, rh);
+      ref_access_tiled(ref, linear, &layout, rx, ry, rw, rh, linear_stride,
+                       true);
+
+      EXPECT_EQ(memcmp(ref, tiled, layout.size_B), 0);
+   } else {
+      for (unsigned i = 0; i < layout.size_B; ++i) {
+         tiled[i] = (i & 0xFF);
+      }
+
+      ail_detile(tiled, linear, &layout, 0, linear_stride, rx, ry, rw, rh);
+      ref_access_tiled(tiled, ref, &layout, rx, ry, rw, rh, linear_stride,
+                       false);
+
+      EXPECT_EQ(memcmp(ref, linear, bpp * rh * linear_stride), 0);
+   }
+
+   free(ref);
+   free(tiled);
+   free(linear);
+}
+
+static void
+test_ldst(unsigned width, unsigned height, unsigned rx, unsigned ry,
+          unsigned rw, unsigned rh, unsigned linear_stride,
+          enum pipe_format format)
+{
+   test(width, height, rx, ry, rw, rh, linear_stride, format, true);
+   test(width, height, rx, ry, rw, rh, linear_stride, format, false);
+}
+
+TEST(Twiddling, RegularFormats)
+{
+   test_ldst(233, 173, 0, 0, 233, 173, 233, PIPE_FORMAT_R8_UINT);
+   test_ldst(233, 173, 0, 0, 233, 173, 233 * 2, PIPE_FORMAT_R8G8_UINT);
+   test_ldst(233, 173, 0, 0, 233, 173, 233 * 4, PIPE_FORMAT_R32_UINT);
+   test_ldst(173, 143, 0, 0, 173, 143, 173 * 8, PIPE_FORMAT_R32G32_UINT);
+   test_ldst(133, 143, 0, 0, 133, 143, 133 * 16, PIPE_FORMAT_R32G32B32A32_UINT);
+}
+
+TEST(Twiddling, UnpackedStrides)
+{
+   test_ldst(213, 17, 0, 0, 213, 17, 369 * 1, PIPE_FORMAT_R8_SINT);
+   test_ldst(213, 17, 0, 0, 213, 17, 369 * 2, PIPE_FORMAT_R8G8_SINT);
+   test_ldst(213, 17, 0, 0, 213, 17, 369 * 4, PIPE_FORMAT_R32_SINT);
+   test_ldst(213, 17, 0, 0, 213, 17, 369 * 8, PIPE_FORMAT_R32G32_SINT);
+   test_ldst(213, 17, 0, 0, 213, 17, 369 * 16, PIPE_FORMAT_R32G32B32A32_SINT);
+}
+
+TEST(Twiddling, PartialAccess)
+{
+   test_ldst(283, 171, 3, 1, 131, 7, 369 * 1, PIPE_FORMAT_R8_UNORM);
+   test_ldst(283, 171, 3, 1, 131, 7, 369 * 2, PIPE_FORMAT_R8G8_UNORM);
+   test_ldst(283, 171, 3, 1, 131, 7, 369 * 4, PIPE_FORMAT_R32_UNORM);
+   test_ldst(283, 171, 3, 1, 131, 7, 369 * 8, PIPE_FORMAT_R32G32_UNORM);
+   test_ldst(283, 171, 3, 1, 131, 7, 369 * 16, PIPE_FORMAT_R32G32B32A32_UNORM);
+}
+
+TEST(Twiddling, ETC)
+{
+   /* Block alignment assumed */
+   test_ldst(32, 32, 0, 0, 32, 32, 512, PIPE_FORMAT_ETC1_RGB8);
+   test_ldst(32, 256, 0, 0, 32, 256, 512, PIPE_FORMAT_ETC2_RGB8A1);
+   test_ldst(32, 512, 0, 0, 32, 512, 512, PIPE_FORMAT_ETC2_RG11_SNORM);
+}
+
+TEST(Twiddling, PartialETC)
+{
+   /* Block alignment assumed */
+   test_ldst(32, 32, 4, 8, 16, 12, 512, PIPE_FORMAT_ETC1_RGB8);
+   test_ldst(32, 32, 4, 8, 16, 12, 512, PIPE_FORMAT_ETC2_RGB8A1);
+   test_ldst(32, 32, 4, 8, 16, 12, 512, PIPE_FORMAT_ETC2_RG11_SNORM);
+}
+
+TEST(Twiddling, DXT)
+{
+   /* Block alignment assumed */
+   test_ldst(32, 32, 0, 0, 32, 32, 512, PIPE_FORMAT_DXT1_RGB);
+   test_ldst(32, 32, 0, 0, 32, 32, 512, PIPE_FORMAT_DXT3_RGBA);
+   test_ldst(32, 32, 0, 0, 32, 32, 512, PIPE_FORMAT_DXT5_RGBA);
+}
+
+TEST(Twiddling, PartialDXT)
+{
+   /* Block alignment assumed */
+   test_ldst(32, 32, 4, 8, 16, 12, 512, PIPE_FORMAT_DXT1_RGB);
+   test_ldst(32, 32, 4, 8, 16, 12, 512, PIPE_FORMAT_DXT3_RGBA);
+   test_ldst(32, 32, 4, 8, 16, 12, 512, PIPE_FORMAT_DXT5_RGBA);
+}
+
+TEST(Twiddling, ASTC)
+{
+   /* Block alignment assumed */
+   test_ldst(40, 40, 0, 0, 40, 40, 512, PIPE_FORMAT_ASTC_4x4);
+   test_ldst(50, 40, 0, 0, 50, 40, 512, PIPE_FORMAT_ASTC_5x4);
+   test_ldst(50, 50, 0, 0, 50, 50, 512, PIPE_FORMAT_ASTC_5x5);
+}
+
+TEST(Twiddling, PartialASTC)
+{
+   /* Block alignment assumed */
+   test_ldst(40, 40, 4, 4, 16, 8, 512, PIPE_FORMAT_ASTC_4x4);
+   test_ldst(50, 40, 5, 4, 10, 8, 512, PIPE_FORMAT_ASTC_5x4);
+   test_ldst(50, 50, 5, 5, 10, 10, 512, PIPE_FORMAT_ASTC_5x5);
+}