init

2026-11-01 11:18:03 +03:00 · 2026-06-06 13:25:15 +03:00
1 parent 04d4dc8b09
commit 818cab85dc
@@ -0,0 +1 @@
+max_width = 80
@@ -0,0 +1,4 @@
+[[deqp]]
+deqp = "/deqp-vk/external/vulkancts/modules/vulkan/deqp-vk"
+caselists = ["/deqp-vk/mustpass/vk-main.txt"]
+tests_per_group = 10000
@@ -0,0 +1,74 @@
+.nvk-valve-rules:
+  stage: nouveau
+  rules:
+    - !reference [.test, rules]
+    - !reference [.valve-farm-rules, rules]
+    - !reference [.vulkan-rules, rules]
+    - changes: &nvk_file_list
+        - src/nouveau/*
+        - src/nouveau/codegen**/*
+        - src/nouveau/compiler/**/*
+        - src/nouveau/drm/**/*
+        - src/nouveau/nme/**/*
+        - src/nouveau/nil/**/*
+        - src/nouveau/nvidia-headers/**/*
+        - src/nouveau/vulkan/**/*
+        - src/nouveau/winsys/**/*
+        - src/nouveau/ci/gitlab-ci.yml
+        - src/nouveau/ci/gitlab-ci-inc.yml
+        - src/nouveau/ci/deqp-$DEQP_SUITE.toml
+        - src/nouveau/ci/$GPU_VERSION-fails.txt
+        - src/nouveau/ci/$GPU_VERSION-flakes.txt
+        - src/nouveau/ci/$GPU_VERSION-skips.txt
+        - src/nouveau/ci/$GPU_VERSION-vkd3d-fails.txt
+        - src/nouveau/ci/$GPU_VERSION-vkd3d-flakes.txt
+        - src/nouveau/ci/$GPU_VERSION-vkd3d-skips.txt
+      when: on_success
+
+.nvk-valve-manual-rules:
+  stage: nouveau-nightly
+  extends: .no-auto-retry
+  rules:
+    - !reference [.test, rules]
+    - !reference [.valve-farm-manual-rules, rules]
+    - !reference [.vulkan-manual-rules, rules]
+    - changes:
+        *nvk_file_list
+      when: manual
+
+.nvk-ci-tron-common:
+  variables:
+    CI_TRON_PATTERN__JOB_WARN__REGEX: 'nouveau .+ job timeout|NMI backtrace for cpu \d+'
+    CI_TRON_INITRAMFS__DEPMOD__URL: "https://gitlab.freedesktop.org/gfx-ci/boot2container/-/releases/$B2C_VERSION/downloads/linux-x86_64.depmod.cpio.xz"
+    CI_TRON_INITRAMFS__GPU__URL: "https://gitlab.freedesktop.org/gfx-ci/boot2container/-/releases/$B2C_VERSION/downloads/linux-x86_64.gpu.cpio"
+    CI_TRON_INITRAMFS__GPU__FORMAT__0__ARCHIVE__KEEP__0__PATH: ${GPU_FIRMWARE_KEEP_PATH_REGEX}
+    CI_TRON_KERNEL_CMDLINE__LOAD_MODULES: 'modules_load=nouveau'
+    # GSP is not loaded by default
+    CI_TRON_KERNEL_CMDLINE__NOUVEAU_LOAD_GSP: 'nouveau.config=NvGspRm=1'
+
+.nvk-vkcts:
+  variables:
+    VK_DRIVER: nouveau
+    DEQP_SUITE: nvk
+    MESA_VK_IGNORE_CONFORMANCE_WARNING: 1
+    NVK_I_WANT_A_BROKEN_VULKAN_DRIVER: 1
+
+.nvk-ga106-valve:
+  extends:
+    - .nvk-ci-tron-common
+  variables:
+    GPU_VERSION: nvk-ga106
+    FDO_CI_CONCURRENT: 16
+    RUNNER_FARM_LOCATION: mupuf
+    CI_TRON_DUT_SETUP_TAGS: nvidia:codename:GA106
+    GPU_FIRMWARE_KEEP_PATH_REGEX: '(lib/(modules|firmware/nvidia/ga10\d)/.*)'
+
+.nvk-ad106-valve:
+  extends:
+    - .nvk-ci-tron-common
+  variables:
+    GPU_VERSION: nvk-ad106
+    FDO_CI_CONCURRENT: 32
+    RUNNER_FARM_LOCATION: keywords
+    CI_TRON_DUT_SETUP_TAGS: nvidia:codename:AD106
+    GPU_FIRMWARE_KEEP_PATH_REGEX: '(lib/(modules|firmware/nvidia/(ga|ad)10\d)/.*)'
@@ -0,0 +1,49 @@
+include:
+  - local: 'src/nouveau/ci/gitlab-ci-inc.yml'
+
+nvk-ga106-vkcts-valve:
+  extends:
+    - .ci-tron-x86_64-test-vk-manual
+    - .nvk-vkcts
+    - .nvk-ga106-valve
+    - .nvk-valve-manual-rules
+  timeout: 95m
+  variables:
+    CI_TRON_TIMEOUT__OVERALL__MINUTES: 90  # usual run time: 75min
+
+nvk-ad106-vkcts-valve:
+  extends:
+    - .ci-tron-x86_64-test-vk-manual
+    - .nvk-vkcts
+    - .nvk-ad106-valve
+    - .nvk-valve-manual-rules
+  timeout: 60m
+  parallel: 2
+  variables:
+    CI_TRON_TIMEOUT__OVERALL__MINUTES: 55  # usual run time: 37min
+
+nvk-ga106-vkd3d-valve:
+  extends:
+    - .ci-tron-x86_64-test-vk-manual
+    - .nvk-vkcts
+    - .nvk-ga106-valve
+    - .nvk-valve-manual-rules
+    - .test-vkd3d-proton
+  timeout: 15m
+  variables:
+    CI_TRON_TIMEOUT__OVERALL__MINUTES: 10  # usual run time: 5.5min
+    # Random tests fail/crash when trying to run them in parallel
+    FDO_CI_CONCURRENT: 1
+
+nvk-ad106-vkd3d-valve:
+  extends:
+    - .ci-tron-x86_64-test-vk-manual
+    - .nvk-vkcts
+    - .nvk-ad106-valve
+    - .nvk-valve-manual-rules
+    - .test-vkd3d-proton
+  timeout: 15m
+  variables:
+    CI_TRON_TIMEOUT__OVERALL__MINUTES: 10  # usual run time: 5.5min
+    # Random tests fail/crash when trying to run them in parallel
+    FDO_CI_CONCURRENT: 1
@@ -0,0 +1,29 @@
+dEQP-VK.api.external.semaphore.sync_fd.export_multiple_times_temporary
+dEQP-VK.api.info.vulkan1p2.feature_bits_influence
+dEQP-VK.api.object_management.multithreaded_shared_resources.device
+dEQP-VK.draw.dynamic_rendering.complete_secondary_cmd_buff.shader_layer.tessellation_shader_256
+dEQP-VK.draw.dynamic_rendering.complete_secondary_cmd_buff.shader_layer.vertex_shader_256
+dEQP-VK.draw.dynamic_rendering.partial_secondary_cmd_buff.shader_layer.tessellation_shader_256
+dEQP-VK.draw.dynamic_rendering.partial_secondary_cmd_buff.shader_layer.vertex_shader_256
+dEQP-VK.draw.dynamic_rendering.primary_cmd_buff.shader_layer.tessellation_shader_256
+dEQP-VK.draw.dynamic_rendering.primary_cmd_buff.shader_layer.vertex_shader_256
+dEQP-VK.draw.renderpass.shader_layer.tessellation_shader_256
+dEQP-VK.draw.renderpass.shader_layer.vertex_shader_256
+dEQP-VK.dynamic_rendering.primary_cmd_buff.suballocation.multisample_resolve.*
+dEQP-VK.pipeline.monolithic.blend.dual_source.multi_attachments.a2b10g10r10_unorm_pack32
+dEQP-VK.pipeline.shader_object_unlinked_spirv.blend.dual_source.multi_attachments.r8g8b8a8_snorm
+dEQP-VK.renderpass.suballocation.multisample_resolve.*
+dEQP-VK.renderpass2.suballocation.multisample_resolve.*
+dEQP-VK.synchronization.basic.timeline_semaphore.chain
+dEQP-VK.wsi.display.create_display_mode
+dEQP-VK.wsi.display.create_display_plane_surface
+dEQP-VK.wsi.display.get_display_mode_properties
+dEQP-VK.wsi.display.get_display_mode_properties2
+dEQP-VK.wsi.display.get_display_plane_capabilities
+dEQP-VK.wsi.display.get_display_plane_capabilities2
+dEQP-VK.wsi.display.get_display_plane_properties
+dEQP-VK.wsi.display.get_display_plane_properties2
+dEQP-VK.wsi.display.get_display_plane_supported_displays
+dEQP-VK.wsi.display.get_display_properties
+dEQP-VK.wsi.display.get_display_properties2
+dEQP-VK.wsi.display.surface_counters
@@ -0,0 +1,27 @@
+# These tests are timeout'ing
+dEQP-VK.binding_model.descriptor_buffer.sparse_binding_buffer.multiple.compute_comp_buffers32_sets1
+dEQP-VK.binding_model.descriptor_buffer.sparse_binding_buffer.multiple.graphics_comp_buffers32_sets1
+dEQP-VK.binding_model.descriptor_buffer.sparse_binding_buffer.multiple.graphics_frag_buffers32_sets1
+dEQP-VK.binding_model.descriptor_buffer.sparse_binding_buffer.multiple.graphics_vert_buffers32_sets1
+dEQP-VK.binding_model.descriptor_buffer.sparse_residency_buffer.multiple.compute_comp_buffers32_sets1
+dEQP-VK.binding_model.descriptor_buffer.sparse_residency_buffer.multiple.graphics_comp_buffers32_sets1
+dEQP-VK.binding_model.descriptor_buffer.sparse_residency_buffer.multiple.graphics_frag_buffers32_sets1
+dEQP-VK.binding_model.descriptor_buffer.sparse_residency_buffer.multiple.graphics_vert_buffers32_sets1
+dEQP-VK.binding_model.descriptor_buffer.traditional_buffer.multiple.compute_comp_buffers32_sets1
+dEQP-VK.binding_model.descriptor_buffer.traditional_buffer.multiple.graphics_comp_buffers32_sets1
+dEQP-VK.binding_model.descriptor_buffer.traditional_buffer.multiple.graphics_frag_buffers32_sets1
+dEQP-VK.binding_model.descriptor_buffer.traditional_buffer.multiple.graphics_vert_buffers32_sets1
+dEQP-VK.memory.pipeline_barrier.host_write_transfer_src.1048576
+dEQP-VK.memory.pipeline_barrier.host_write_vertex_buffer.65536_vertex_buffer_stride_4
+dEQP-VK.memory.pipeline_barrier.transfer_src_transfer_dst.1048576
+dEQP-VK.sparse_resources.image_sparse_binding.multiple_bind_sparse_info.2d_array.*
+dEQP-VK.sparse_resources.image_sparse_binding.multiple_bind_sparse_info.cube_array.*
+dEQP-VK.sparse_resources.image_sparse_binding.multiple_sparse_image_opaque_memory_bind_info.3d.*
+dEQP-VK.sparse_resources.image_sparse_binding.multiple_sparse_image_opaque_memory_bind_info.cube_array.*
+dEQP-VK.sparse_resources.image_sparse_binding.multiple_sparse_memory_bind.cube_array.*
+dEQP-VK.sparse_resources.image_sparse_residency.cube_array.*
+
+# https://github.com/KhronosGroup/VK-GL-CTS/issues/546
+dEQP-VK.spirv_assembly.instruction.compute.untyped_pointers.vulkan_memory_model.cooperative_matrix.type_punning.*
+# https://github.com/KhronosGroup/VK-GL-CTS/issues/553
+dEQP-VK.spirv_assembly.instruction.compute.untyped_pointers.vulkan_memory_model.cooperative_matrix.mixed.load.*
@@ -0,0 +1,20 @@
+# nouveau 0000:2d:00.0: gsp: mmu fault queued
+# nouveau 0000:2d:00.0: gsp: rc engn:00000001 chid:24 type:31 scope:1 part:233
+# nouveau 0000:2d:00.0: fifo:c00000:0003:0018:[d3d12[1056]] errored - disabling channel
+# nouveau 0000:2d:00.0: d3d12[1056]: channel 24 killed!
+test_uav_counter_null_behavior_dxbc,Crash
+test_uav_counter_null_behavior_dxil,Crash
+
+test_custom_border_color_limits,Fail
+test_derivative_hoisting_dxil,Fail
+
+test_undefined_structured_raw_alias_dxbc,Fail
+test_undefined_structured_raw_alias_dxil,Fail
+test_undefined_structured_raw_read_typed_dxbc,Fail
+test_undefined_structured_raw_read_typed_dxil,Fail
+
+test_large_buffer_descriptors,Crash
+
+test_line_tessellation_dxbc,Fail
+test_line_tessellation_dxil,Fail
+test_tessellation_read_tesslevel,Fail
@@ -0,0 +1,6 @@
+dEQP-VK.pipeline.fast_linked_library.pipeline_cache.robustness2.storage,Fail
+dEQP-VK.pipeline.pipeline_library.pipeline_cache.robustness2.storage,Fail
+dEQP-VK.pipeline.shader_object_linked_binary.pipeline_cache.robustness2.storage,Fail
+dEQP-VK.pipeline.shader_object_linked_spirv.pipeline_cache.robustness2.storage,Fail
+dEQP-VK.pipeline.shader_object_unlinked_binary.pipeline_cache.robustness2.storage,Fail
+dEQP-VK.pipeline.shader_object_unlinked_spirv.pipeline_cache.robustness2.storage,Fail
@@ -0,0 +1,149 @@
+dEQP-VK.api.copy_and_blit.core.image_to_image.dimensions.src32768x6_dst32768x6.r4g4b4a4_unorm_pack16.r16_sfloat.general_general
+
+dEQP-VK.binding_model.descriptor_buffer.sparse_residency_buffer.max.graphics_tese_sampler15_resource1
+dEQP-VK.binding_model.descriptor_buffer.traditional_buffer.multiple.graphics_frag_buffers3_sets1
+dEQP-VK.binding_model.descriptor_buffer.traditional_buffer.mutable_descriptor.graphics_tesc_type_mask8
+dEQP-VK.binding_model.descriptor_buffer.traditional_buffer.single.graphics_frag_sampled_image
+dEQP-VK.binding_model.descriptorset_random.sets8.runtimesize.noubo.sbolimitlow.sampledimghigh.outimgtexlow.iublimithigh.nouab.frag.ialimitlow.0
+dEQP-VK.binding_model.mutable_descriptor.single.uniform_texel_buffer.update_copy.nonmutable_source.normal_source.pool_same_types.update_after_bind.no_array.tese
+dEQP-VK.binding_model.shader_access.primary_cmd_buf.bind.with_template.sampler_mutable.compute.multiple_contiguous_descriptors.1d_base_slice
+dEQP-VK.binding_model.shader_access.primary_cmd_buf.bind.with_template.sampler_mutable.vertex_fragment.multiple_discontiguous_descriptor_sets.multiple_arbitrary_descriptors.2d
+
+dEQP-VK.dynamic_rendering.primary_cmd_buff.suballocation.multisample_resolve.layers_3.a8b8g8r8_srgb_pack32.samples_8
+dEQP-VK.dynamic_rendering.primary_cmd_buff.suballocation.multisample_resolve.layers_3.a8b8g8r8_uint_pack32.samples_8
+dEQP-VK.dynamic_rendering.primary_cmd_buff.suballocation.multisample_resolve.layers_3.b8g8r8a8_srgb.samples_8_resolve_level_3
+dEQP-VK.dynamic_rendering.primary_cmd_buff.suballocation.multisample_resolve.layers_3.b8g8r8a8_unorm.samples_8_baseLayer1
+dEQP-VK.dynamic_rendering.primary_cmd_buff.suballocation.multisample_resolve.layers_3.r16g16b16a16_sint.samples_8_resolve_level_4
+dEQP-VK.dynamic_rendering.primary_cmd_buff.suballocation.multisample_resolve.layers_3.r8g8b8a8_srgb.samples_8_resolve_level_4
+dEQP-VK.dynamic_rendering.primary_cmd_buff.suballocation.multisample_resolve.layers_3.r8g8b8a8_unorm.samples_4_resolve_level_3
+dEQP-VK.dynamic_rendering.primary_cmd_buff.suballocation.multisample_resolve.layers_3.r8g8b8a8_unorm.samples_8_resolve_level_3
+dEQP-VK.dynamic_rendering.primary_cmd_buff.suballocation.multisample_resolve.layers_6.r32g32b32a32_uint.samples_2_resolve_level_4
+dEQP-VK.dynamic_rendering.primary_cmd_buff.suballocation.multisample_resolve.r32g32b32a32_sint.samples_8_resolve_level_4
+dEQP-VK.dynamic_rendering.primary_cmd_buff.suballocation.multisample_resolve.r8_uint.samples_8_resolve_level_2
+
+dEQP-VK.image.host_image_copy.large_images.image_to_memory_r32_sfloat_4096_4096
+dEQP-VK.image.host_image_copy.large_images.image_to_memory_r8g8b8a8_unorm_4096_4096
+dEQP-VK.image.host_image_copy.optimal.image_to_image_copy.shader_read_only_optimal.stencil_read_only_optimal.51x63x1_1.64.r8g8b8a8_unorm
+dEQP-VK.image.host_image_copy.optimal.image_to_image_copy.transfer_src_optimal.general.32x32x1_2.64.r16_sfloat
+
+dEQP-VK.memory.pipeline_barrier.host_write_transfer_src.1048576
+
+dEQP-VK.pipeline.*.pipeline_cache.robustness2.storage
+dEQP-VK.pipeline.monolithic.creation_cache_control.compute_pipelines.duplicate_batch_pipelines_no_cache
+
+dEQP-VK.rasterization.primitive_size.points.point_size_10000
+
+dEQP-VK.renderpass.suballocation.multisample_resolve.layers_3.a8b8g8r8_uint_pack32.samples_8
+dEQP-VK.renderpass.suballocation.multisample_resolve.layers_3.r16g16_unorm.samples_8_baseLayer1
+dEQP-VK.renderpass.suballocation.multisample_resolve.layers_3.r32g32b32a32_sint.samples_8_baseLayer1
+dEQP-VK.renderpass.suballocation.multisample_resolve.layers_3.r32g32b32a32_sint.samples_8_resolve_level_4
+dEQP-VK.renderpass.suballocation.multisample_resolve.layers_6.r16g16b16a16_sfloat.samples_4_resolve_level_4
+dEQP-VK.renderpass.suballocation.multisample_resolve.layers_6.r32g32b32a32_sfloat.samples_4_resolve_level_4
+dEQP-VK.renderpass2.suballocation.multisample_resolve.layers_3.a8b8g8r8_unorm_pack32.samples_8_resolve_level_3
+dEQP-VK.renderpass2.suballocation.multisample_resolve.layers_3.r16g16b16a16_uint.samples_8_baseLayer1
+dEQP-VK.renderpass2.suballocation.multisample_resolve.layers_3.r32g32b32a32_sfloat.samples_8
+dEQP-VK.renderpass2.suballocation.multisample_resolve.layers_3.r8g8b8a8_snorm.samples_8
+
+dEQP-VK.shader_object.rendering.color_attachment_count_4.extra_attachment_after_2.none.none.random_color_formats.after.none.r8g8b8a8_srgb_d24_unorm_s8_uint
+
+dEQP-VK.sparse_resources.image_sparse_binding.cube_array.rg16ui.256_256_6
+dEQP-VK.sparse_resources.image_sparse_binding.cube_array.rgba32i.256_256_6
+dEQP-VK.sparse_resources.image_sparse_binding.multiple_bind_sparse_info.2d_array.rg16_snorm.1024_128_8
+dEQP-VK.sparse_resources.image_sparse_binding.multiple_bind_sparse_info.3d.rgba32ui.1024_128_8
+dEQP-VK.sparse_resources.image_sparse_binding.multiple_bind_sparse_info.cube_array.rg32ui.256_256_6
+dEQP-VK.sparse_resources.image_sparse_binding.multiple_sparse_image_opaque_memory_bind_info.cube_array.rgba32i.256_256_6
+dEQP-VK.sparse_resources.image_sparse_binding.multiple_sparse_image_opaque_memory_bind_info.cube_array.rgba32ui.256_256_6
+dEQP-VK.sparse_resources.image_sparse_binding.multiple_sparse_memory_bind.cube_array.rgba32ui.256_256_6
+dEQP-VK.sparse_resources.image_sparse_residency.cube.rgba16i.128_128_1
+dEQP-VK.sparse_resources.transfer_queue.cube_array.rgba32ui.256_256_6
+
+dEQP-VK.synchronization.signal_order.shared_timeline_semaphore.write_image_compute_indirect_read_blit_image.image_128x128_r16_uint_opaque_fd
+dEQP-VK.synchronization.signal_order.shared_timeline_semaphore.write_image_fragment_read_image_vertex.image_128x128_r16g16b16a16_uint_opaque_fd
+dEQP-VK.synchronization2.timeline_semaphore.wait_before_signal.write_image_tess_eval_read_image_tess_eval.image_128x128_r16g16b16a16_uint
+
+dEQP-VK.wsi.display.create_display_mode
+dEQP-VK.wsi.display.create_display_plane_surface
+dEQP-VK.wsi.display.get_display_mode_properties
+dEQP-VK.wsi.display.get_display_mode_properties2
+dEQP-VK.wsi.display.get_display_plane_capabilities
+dEQP-VK.wsi.display.get_display_plane_capabilities2
+dEQP-VK.wsi.display.get_display_plane_properties
+dEQP-VK.wsi.display.get_display_plane_properties2
+dEQP-VK.wsi.display.get_display_plane_supported_displays
+dEQP-VK.wsi.display.get_display_properties
+dEQP-VK.wsi.display.get_display_properties2
+dEQP-VK.wsi.display.surface_counters
+
+KHR-GL46.conditional_render_inverted.functional
+KHR-GL46.gpu_shader_fp64.builtin.abs_dvec2
+KHR-GL46.gpu_shader_fp64.builtin.abs_dvec4
+KHR-GL46.gpu_shader_fp64.builtin.ceil_dvec2
+KHR-GL46.gpu_shader_fp64.builtin.clamp_against_scalar_dvec2
+KHR-GL46.gpu_shader_fp64.builtin.clamp_against_scalar_dvec3
+KHR-GL46.gpu_shader_fp64.builtin.clamp_dvec4
+KHR-GL46.gpu_shader_fp64.builtin.dot_dvec3
+KHR-GL46.gpu_shader_fp64.builtin.floor_dvec3
+KHR-GL46.gpu_shader_fp64.builtin.fma_dvec2
+KHR-GL46.gpu_shader_fp64.builtin.fma_dvec3
+KHR-GL46.gpu_shader_fp64.builtin.fract_double
+KHR-GL46.gpu_shader_fp64.builtin.fract_dvec3
+KHR-GL46.gpu_shader_fp64.builtin.frexp_double
+KHR-GL46.gpu_shader_fp64.builtin.frexp_dvec4
+KHR-GL46.gpu_shader_fp64.builtin.greaterthan_dvec4
+KHR-GL46.gpu_shader_fp64.builtin.inverse_dmat2
+KHR-GL46.gpu_shader_fp64.builtin.inversesqrt_double
+KHR-GL46.gpu_shader_fp64.builtin.ldexp_dvec4
+KHR-GL46.gpu_shader_fp64.builtin.length_dvec2
+KHR-GL46.gpu_shader_fp64.builtin.lessthanequal_dvec2
+KHR-GL46.gpu_shader_fp64.builtin.lessthanequal_dvec3
+KHR-GL46.gpu_shader_fp64.builtin.lessthanequal_dvec4
+KHR-GL46.gpu_shader_fp64.builtin.matrixcompmult_dmat4
+KHR-GL46.gpu_shader_fp64.builtin.max_double
+KHR-GL46.gpu_shader_fp64.builtin.max_dvec2
+KHR-GL46.gpu_shader_fp64.builtin.max_dvec4
+KHR-GL46.gpu_shader_fp64.builtin.min_against_scalar_dvec2
+KHR-GL46.gpu_shader_fp64.builtin.min_against_scalar_dvec3
+KHR-GL46.gpu_shader_fp64.builtin.min_double
+KHR-GL46.gpu_shader_fp64.builtin.min_dvec3
+KHR-GL46.gpu_shader_fp64.builtin.min_dvec4
+KHR-GL46.gpu_shader_fp64.builtin.mix_double
+KHR-GL46.gpu_shader_fp64.builtin.mix_dvec3
+KHR-GL46.gpu_shader_fp64.builtin.mix_dvec4
+KHR-GL46.gpu_shader_fp64.builtin.mod_against_scalar_dvec2
+KHR-GL46.gpu_shader_fp64.builtin.mod_against_scalar_dvec4
+KHR-GL46.gpu_shader_fp64.builtin.mod_double
+KHR-GL46.gpu_shader_fp64.builtin.mod_dvec2
+KHR-GL46.gpu_shader_fp64.builtin.normalize_dvec2
+KHR-GL46.gpu_shader_fp64.builtin.normalize_dvec4
+KHR-GL46.gpu_shader_fp64.builtin.notequal_dvec3
+KHR-GL46.gpu_shader_fp64.builtin.outerproduct_dmat2
+KHR-GL46.gpu_shader_fp64.builtin.outerproduct_dmat2x3
+KHR-GL46.gpu_shader_fp64.builtin.outerproduct_dmat2x4
+KHR-GL46.gpu_shader_fp64.builtin.outerproduct_dmat3
+KHR-GL46.gpu_shader_fp64.builtin.outerproduct_dmat3x4
+KHR-GL46.gpu_shader_fp64.builtin.outerproduct_dmat4x2
+KHR-GL46.gpu_shader_fp64.builtin.reflect_dvec3
+KHR-GL46.gpu_shader_fp64.builtin.smoothstep_double
+KHR-GL46.gpu_shader_fp64.builtin.smoothstep_dvec2
+KHR-GL46.gpu_shader_fp64.builtin.sqrt_dvec2
+KHR-GL46.gpu_shader_fp64.builtin.step_against_scalar_dvec2
+KHR-GL46.gpu_shader_fp64.builtin.step_dvec2
+KHR-GL46.gpu_shader_fp64.builtin.step_dvec3
+KHR-GL46.gpu_shader_fp64.builtin.step_dvec4
+KHR-GL46.gpu_shader_fp64.builtin.trunc_dvec2
+KHR-GL46.gpu_shader_fp64.builtin.unpackdouble2x32_double
+KHR-GL46.packed_pixels.pbo_rectangle.rg16f_format_rg
+KHR-GL46.packed_pixels.pbo_rectangle.rg8_snorm_format_rg
+KHR-GL46.packed_pixels.pbo_rectangle.rgb32f_format_rgb
+KHR-GL46.packed_pixels.pbo_rectangle.rgba16_format_bgra
+KHR-GL46.packed_pixels.varied_rectangle.rgb32f_format_rgb
+KHR-GL46.texture_buffer.texture_buffer_max_size
+KHR-GL46.texture_lod_bias.texture_lod_bias_all
+dEQP-GLES3.functional.occlusion_query.conservative_depth_write_stencil_clear
+dEQP-GLES31.functional.copy_image.mixed.viewclass_128_bits_mixed.rgba32i_rgba_astc_8x6_khr.texture2d_array_to_texture2d_array
+dEQP-GLES31.functional.copy_image.mixed.viewclass_128_bits_mixed.rgba32i_srgb8_alpha8_astc_8x6_khr.renderbuffer_to_texture2d_array
+dEQP-GLES31.functional.synchronization.inter_invocation.ssbo_atomic_alias_overwrite
+glx@glx-multithread-texture
+spec@arb_compute_variable_group_size@local-size
+spec@arb_point_parameters@arb_point_parameters-point-attenuation
+spec@arb_point_parameters@arb_point_parameters-point-attenuation@Aliased combinations
@@ -0,0 +1,22 @@
+# Too slow
+dEQP-VK.binding_model.descriptor_buffer.multiple.compute_comp_buffers32_sets1
+dEQP-VK.binding_model.descriptor_buffer.multiple.graphics_comp_buffers32_sets1
+dEQP-VK.binding_model.descriptor_buffer.multiple.graphics_frag_buffers32_sets1
+dEQP-VK.binding_model.descriptor_buffer.multiple.graphics_vert_buffers32_sets1
+dEQP-VK.draw.*.shader_layer.(vertex|tessellation)_shader_256
+dEQP-VK.memory.pipeline_barrier.transfer_src_transfer_dst.1048576
+dEQP-VK.reconvergence.(sub|work)group_uniform_control_flow_(ballot|elect).compute.nesting4.(0.40|7.10)
+
+# New timeouts in 1.4.1.0
+dEQP-VK.binding_model.descriptor_buffer.sparse_binding_buffer.multiple.compute_comp_buffers32_sets1
+dEQP-VK.binding_model.descriptor_buffer.sparse_binding_buffer.multiple.graphics_comp_buffers32_sets1
+dEQP-VK.binding_model.descriptor_buffer.sparse_binding_buffer.multiple.graphics_frag_buffers32_sets1
+dEQP-VK.binding_model.descriptor_buffer.sparse_binding_buffer.multiple.graphics_vert_buffers32_sets1
+dEQP-VK.binding_model.descriptor_buffer.sparse_residency_buffer.multiple.compute_comp_buffers32_sets1
+dEQP-VK.binding_model.descriptor_buffer.sparse_residency_buffer.multiple.graphics_comp_buffers32_sets1
+dEQP-VK.binding_model.descriptor_buffer.sparse_residency_buffer.multiple.graphics_frag_buffers32_sets1
+dEQP-VK.binding_model.descriptor_buffer.sparse_residency_buffer.multiple.graphics_vert_buffers32_sets1
+dEQP-VK.binding_model.descriptor_buffer.traditional_buffer.multiple.compute_comp_buffers32_sets1
+dEQP-VK.binding_model.descriptor_buffer.traditional_buffer.multiple.graphics_comp_buffers32_sets1
+dEQP-VK.binding_model.descriptor_buffer.traditional_buffer.multiple.graphics_frag_buffers32_sets1
+dEQP-VK.binding_model.descriptor_buffer.traditional_buffer.multiple.graphics_vert_buffers32_sets1
@@ -0,0 +1,20 @@
+# nouveau 0000:2d:00.0: gsp: mmu fault queued
+# nouveau 0000:2d:00.0: gsp: rc engn:00000001 chid:24 type:31 scope:1 part:233
+# nouveau 0000:2d:00.0: fifo:c00000:0003:0018:[d3d12[1056]] errored - disabling channel
+# nouveau 0000:2d:00.0: d3d12[1056]: channel 24 killed!
+test_uav_counter_null_behavior_dxbc,Crash
+test_uav_counter_null_behavior_dxil,Crash
+
+test_custom_border_color_limits,Fail
+test_derivative_hoisting_dxil,Fail
+
+test_undefined_structured_raw_alias_dxbc,Fail
+test_undefined_structured_raw_alias_dxil,Fail
+test_undefined_structured_raw_read_typed_dxbc,Fail
+test_undefined_structured_raw_read_typed_dxil,Fail
+
+test_large_buffer_descriptors,Crash
+
+test_line_tessellation_dxbc,Fail
+test_line_tessellation_dxil,Fail
+test_tessellation_read_tesslevel,Fail
@@ -0,0 +1,5 @@
+# To use this config on you editor, follow the instructions at:
+# http://editorconfig.org
+
+[*.rs]
+max_line_length = 80
@@ -0,0 +1,81 @@
+// Copyright © 2022 Collabora, Ltd.
+// SPDX-License-Identifier: MIT
+
+// Generated by random.org
+const INIT_DATA: [u16; 64] = [
+    8235, 45378, 61861, 23858, 42241, 27301, 23697, 15333, 52828, 39161, 46572,
+    7478, 11198, 4926, 21736, 23781, 37217, 20323, 32754, 61367, 12882, 35352,
+    29343, 26458, 62858, 44415, 35786, 58466, 45062, 30330, 30350, 35134,
+    28228, 36077, 26793, 27223, 52219, 52544, 3565, 56253, 40962, 40833, 35607,
+    46482, 64735, 31252, 44165, 40273, 46825, 15929, 27160, 1106, 59035, 48146,
+    47365, 19901, 37178, 35614, 29628, 23344, 11328, 60105, 28727, 344,
+];
+
+/// An implementation of the ACORN random number generator
+///
+/// See also <http://acorn.wikramaratna.org/concept.html>
+///
+/// This isn't cryptographically secure but it's easy and fast and good enough
+/// for generating test data.  More importantly, we own the implementation so it
+/// won't change with random library or compiler upgrades.
+pub struct Acorn {
+    y_n: [u64; Self::K],
+}
+
+impl Acorn {
+    const ORD: usize = 60;
+    const K: usize = 16;
+
+    pub fn new() -> Self {
+        let mut y_0 = [0_u64; Self::K];
+        for m in 0..Self::K {
+            y_0[m] |= u64::from(INIT_DATA[m * 4 + 0]) << 0;
+            y_0[m] |= u64::from(INIT_DATA[m * 4 + 1]) << 16;
+            y_0[m] |= u64::from(INIT_DATA[m * 4 + 2]) << 32;
+            y_0[m] |= u64::from(INIT_DATA[m * 4 + 3]) << 48;
+            y_0[m] &= (1u64 << Self::ORD) - 1;
+        }
+        // For good randomness, the seed should be odd
+        assert!(y_0[0] % 2 == 1);
+        Acorn { y_n: y_0 }
+    }
+
+    fn next_u60(&mut self) -> u64 {
+        // y_n[0] remains uncanged
+        for m in 1..Self::K {
+            // We go in increasing M so y_n[m] is y_(n-1)[m] when we read it but
+            // self.y_n[m - 1] is what it says it is
+            self.y_n[m] = (self.y_n[m] + self.y_n[m - 1]) % (1 << Self::ORD);
+        }
+        self.y_n[Self::K - 1]
+    }
+
+    pub fn get_uint(&mut self, bits: usize) -> u64 {
+        if bits <= Self::ORD {
+            self.next_u60() >> (Self::ORD - bits)
+        } else {
+            let hi = self.next_u60() << (bits - Self::ORD);
+            hi | self.get_uint(bits - Self::ORD)
+        }
+    }
+
+    pub fn get_bool(&mut self) -> bool {
+        self.get_uint(1) != 0
+    }
+
+    pub fn get_u8(&mut self) -> u8 {
+        self.get_uint(8) as u8
+    }
+
+    pub fn get_u16(&mut self) -> u16 {
+        self.get_uint(16) as u16
+    }
+
+    pub fn get_u32(&mut self) -> u32 {
+        self.get_uint(32) as u32
+    }
+
+    pub fn get_u64(&mut self) -> u64 {
+        self.get_uint(64)
+    }
+}
@@ -0,0 +1,209 @@
+#! /usr/bin/env python3
+#
+# Copyright © 2024 Collabora Ltd. and Red Hat Inc.
+# SPDX-License-Identifier: MIT
+
+# This script takes a list of Rust files, each of the form nvh_path_to_mod.rs
+# and constructs a lib.rs which puts each of them in ::path::to::mod.
+
+import argparse
+import csv
+import os
+import sys
+
+from mako import template
+
+TEMPLATE_RS = template.Template(text="""\
+// Copyright 2024 Red Hat Inc.
+// SPDX-License-Identifier: MIT
+
+// This file is generated by lat_rs_gen.py. DO NOT EDIT!
+#![allow(unused_variables)]
+
+const fn pred(has_pred: bool, a: u32, b: u32) -> u32 {
+    if has_pred {
+        a + b
+    } else {
+        b
+    }
+}
+
+% for reg_file, cats in file_cats.items():
+<% enum_name = to_camel(reg_file) + 'Latency' + sm.upper() %>
+#[derive(PartialEq)]
+pub enum ${enum_name} {
+% for category in cats[0].header.cats:
+   ${to_camel(category)},
+% endfor
+}
+
+impl ${to_camel(reg_file)}Latency${sm.upper()} {
+% for bigcat in cats:
+    pub fn ${bigcat.header.latcat}(
+        ${bigcat.header.cat0}: ${enum_name},
+        ${bigcat.header.cat1}: ${enum_name},
+        has_pred: bool
+    ) -> u32 {
+        use ${enum_name}::*;
+        match ${bigcat.header.cat1} {
+% for cat in bigcat.header.cats:
+            #[allow(clippy::match_single_binding)]
+            ${to_camel(cat)} => match ${bigcat.header.cat0} {
+            <% has_non = False %>
+% for cat2 in bigcat.header.cats:
+            % if bigcat.fields[loop.parent.index].flds[cat2].pred == True:
+                ${to_camel(cat2)} => pred(
+                    has_pred,
+                    ${bigcat.fields[loop.parent.index].flds[cat2].value},
+                    ${bigcat.fields[loop.parent.index].flds[cat2].pred_val}
+                ),
+            % elif bigcat.fields[loop.parent.index].flds[cat2].value != "none":
+                ${to_camel(cat2)} => ${bigcat.fields[loop.parent.index].flds[cat2].value},
+            % else:
+                <% has_none = True %>
+            % endif
+% endfor
+            % if has_none:
+              _ => panic!("Illegal ${bigcat.header.cat0} value in ${bigcat.header.latcat} for ${to_camel(cat)}"),
+            % endif
+         }
+% endfor
+      }
+   }
+% endfor
+}
+
+% endfor
+""")
+
+## A mere convenience to convert snake_case to CamelCase. Numbers are prefixed
+## with "_".
+def to_camel(snake_str):
+    result = ''.join(word.title() for word in snake_str.split('_'))
+    return result if not result[0].isdigit() else '_' + result
+
+def reader(csvfile):
+    """Wrapper around csv.reader that skips comments and blanks."""
+    # csv.reader actually reads the file one line at a time (it was designed to
+    # open excel generated sheets), so hold the file until all of the lines are
+    # read.
+    with open(csvfile, 'r') as f:
+        for line in csv.reader(f):
+            if line and not line[0].startswith('#'):
+                yield line
+
+class Fld(object):
+    def __init__(self, line):
+        if "none" in line:
+            self.valid = False
+        else:
+            self.valid = True
+        self.pred = False
+        if "+" in line:
+            self.pred = True
+            part = line.split("+")
+            self.value = part[0]
+            self.pred_val = part[1]
+        elif " & sb" in line:
+            self.scoreboard = True
+            self.value = line.removesuffix(" & sb");
+        else:
+            self.scoreboard = False
+            self.value = line.strip()
+
+class Header(object):
+    def __init__(self, line):
+        self.latcat = line[0].strip()
+        self.cats = line[1:]
+
+        if self.latcat == "raw":
+            self.cat0 = "writer"
+            self.cat1 = "reader"
+        elif self.latcat == "war":
+            self.cat0 = "reader"
+            self.cat1 = "writer"
+        elif self.latcat == "waw":
+            self.cat0 = "writer1"
+            self.cat1 = "writer2"
+
+
+class Fields(object):
+    def __init__(self, header, line):
+        self.fldcat = line[0].strip()
+        self.flds = {}
+        for index, cat in enumerate(header.cats):
+            self.flds[cat] = Fld(line[index + 1])
+
+class Category(object):
+    def __init__(self, header, fields):
+        self.header = header
+        self.fields = fields
+
+lattypes = ["reg", "ureg", "pred", "upred"]
+
+def emit_cats(dirname, f, sm, lat):
+    cats = []
+    for index, cat in enumerate(["raw", "war", "waw"]):
+        first_line = False
+        fields = []
+        for l in reader(dirname + "sm" + sm + "/" + lat + "_" + cat + ".csv"):
+            if first_line == False:
+                header = Header(l)
+                first_line = True
+            else:
+                fields.append(Fields(header, l))
+        cats.append(Category(header, fields))
+
+    try:
+            f.write(TEMPLATE_RS.render(lat=lat, sm=sm, cats=cats))
+    except Exception:
+        # In the event there's an error, this imports some helpers from mako
+        # to print a useful stack trace and prints it, then exits with
+        # status 1, if python is run with debug; otherwise it just raises
+        # the exception
+        import sys
+        from mako import exceptions
+        print(exceptions.text_error_template().render(), file=sys.stderr)
+        sys.exit(1)
+
+def emit_sm(dirname, f, sm):
+    for lat in lattypes:
+        emit_cats(dirname, f, sm, lat)
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-p', '--import-path', required=True)
+    parser.add_argument('--out-rs', required=True, help='Output Rust file.')
+    parser.add_argument('--sm', help='SM', required=True)
+    parser.add_argument('csv_files', metavar='FILE', nargs='*',
+                        action='append',
+                        help='Input CSV filename')
+
+    args = parser.parse_args()
+
+    sys.path.insert(0, args.import_path)
+    import util
+
+    file_cats = {}
+    for csv_file in args.csv_files[0]:
+        split = os.path.basename(csv_file).removesuffix('.csv').split('_')
+        assert len(split) == 2
+        reg_file = split[0]
+        latcat = split[1]
+
+        r = reader(csv_file)
+        header = Header(next(r))
+        fields = [Fields(header, l) for l in r]
+
+        cat = Category(header, fields)
+        file_cats.setdefault(reg_file, []).append(cat)
+
+    environment = dict(
+        sm=args.sm,
+        file_cats=file_cats,
+        to_camel=to_camel,
+    )
+    util.write_template_rs(args.out_rs, TEMPLATE_RS, environment)
+
+if __name__ == '__main__':
+    main()
@@ -0,0 +1,39 @@
+#! /usr/bin/env python3
+#
+# Copyright © 2024 Collabora Ltd. and Red Hat Inc.
+# SPDX-License-Identifier: MIT
+
+# This script takes a list of Rust files, each of the form nvh_path_to_mod.rs
+# and constructs a lib.rs which puts each of them in ::path::to::mod.
+
+import argparse
+import sys
+
+from mako.template import Template
+
+TEMPLATE_RS = Template("""\
+// Copyright © 2024 Collabora Ltd. and Red Hat Inc.
+// SPDX-License-Identifier: MIT
+
+// This file is generated by lib_rs_gen.py. DO NOT EDIT!
+
+% for mod in mods:
+pub mod ${mod};
+% endfor
+""")
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-p', '--import-path', required=True)
+    parser.add_argument('--out-rs', required=True, help='Output Rust file.')
+    parser.add_argument('mods', metavar='MOD', nargs='*',
+                        action='append', help='Submodule')
+    args = parser.parse_args()
+
+    sys.path.insert(0, args.import_path)
+    import util
+
+    util.write_template_rs(args.out_rs, TEMPLATE_RS, dict(mods=args.mods[0]))
+
+if __name__ == '__main__':
+    main()
@@ -0,0 +1,60 @@
+# Copyright © 2025 Collabora, Ltd.
+# SPDX-License-Identifier: MIT
+
+_nak_lat_sm100_files = [
+  'pred_raw.csv',
+  'pred_war.csv',
+  'pred_waw.csv',
+  'reg_raw.csv',
+  'reg_war.csv',
+  'reg_waw.csv',
+  'upred_raw.csv',
+  'upred_war.csv',
+  'upred_waw.csv',
+  'ureg_raw.csv',
+  'ureg_war.csv',
+  'ureg_waw.csv',
+]
+
+_nak_lat_sms = {
+  'sm100': _nak_lat_sm100_files,
+}
+
+_lat_rs_gen = files('lat_rs_gen.py')
+
+_lat_rs_generated = []
+foreach sm, csvs : _nak_lat_sms
+  csv_files = []
+  foreach i : range(csvs.length())
+    csv_files += files(sm + '/' + csvs[i])
+  endforeach
+
+  _lat_rs_generated += custom_target(
+    sm+'.rs',
+    input : [_lat_rs_gen, csv_files],
+    output : [sm+'.rs'],
+    command : [
+      prog_python, '@INPUT@', '-p', nouveau_util_py_path,
+      '--out-rs', '@OUTPUT0@', '--sm', sm,
+    ],
+    depend_files : nouveau_util_py,
+  )
+endforeach
+
+_nak_latencies_lib_rs = custom_target(
+  'lib.rs',
+  input : ['lib_rs_gen.py', _lat_rs_generated, nouveau_util_py],
+  output : ['lib.rs'],
+  command : [
+    prog_python, '@INPUT0@', '-p', nouveau_util_py_path,
+    '--out-rs', '@OUTPUT0@', _nak_lat_sms.keys()
+  ],
+)
+
+libnak_latencies_rs = static_library(
+  'nak_latencies',
+  _nak_latencies_lib_rs,
+  gnu_symbol_visibility : 'hidden',
+  rust_abi : 'rust',
+  rust_args : rust_global_args,
+)
@@ -0,0 +1,13 @@
+raw,disp_alu,disp_dual_alu,coupled,dualalu,r2p,r2ur,fma,fp16,hfma2_mma,redirected_fp64,decoupled,guard
+disp_alu,13,13,13,13,13,13,13,13,14,14,1 & sb,none
+disp_dual_alu,13,13,13,13,13,13,13,13,14,14,1 & sb,none
+coupled,4,4,4,4,5,8,5,5,6,6,1 & sb,none
+dualalu,4,4,4,4,5,8,5,5,6,6,1 & sb,none
+r2p,4,4,4,4,5,8,5,5,6,6,1 & sb,none
+r2ur,none,none,none,none,none,none,none,none,none,none,none,none
+fma,5,5,5,5,5,13,4,5,6,6,1 & sb,none
+fp16,13,13,13,13,13,13,13,5,14,14,1 & sb,none
+hfma2_mma,13,13,13,13,13,13,13,13,6,6,1 & sb,none
+redirected_fp64,13,13,13,13,13,13,13,13,6,6,1 & sb,none
+decoupled,13,13,13,13,13,13,13,13,14,14,1 & sb,none
+guard,13,13,13,13,13,13,13,13,14,14,1 & sb,none
@@ -0,0 +1,14 @@
+war,disp_alu,disp_dual_alu,coupled,dualalu,r2p,r2ur,fma,fp16,hfma2_mma,redirected_fp64,decoupled,guard
+disp_alu,1,1,1,1,1,none,1,1,1,1,1,1
+disp_dual_alu,1,1,1,1,1,none,1,1,1,1,1,1
+coupled,1,1,1,1,1,none,1,1,1,1,1,1
+dualalu,1,1,1,1,1,none,1,1,1,1,1,1
+r2p,1,1,1,1,1,none,1,1,1,1,1,1
+r2ur,1,1,1,1,1,none,1,1,1,1,1,1
+fma,1,1,1,1,1,none,1,1,1,1,1,1
+fp16,1,1,1,1,1,none,1,1,1,1,1,1
+hfma2_mma,1,1,1,1,1,none,1,1,1,1,1,1
+redirected_fp64,1,1,1,1,1,none,1,1,1,1,1,1
+decoupled,1,1,1,1,1,none,1,1,1,1,1,1
+guard,none,none,none,none,none,none,none,none,none,none,none,none
+
@@ -0,0 +1,13 @@
+waw,disp_alu,disp_dual_alu,coupled,dualalu,r2p,r2ur,fma,fp16,hfma2_mma,redirected_fp64,decoupled,guard
+disp_alu,1,1,1,1,1,1+7,1,1,2,2,1 & sb,none
+disp_dual_alu,1,1,1,1,1,1+7,1,1,2,2,1 & sb,none
+coupled,1,1,1,1,1,1+7,1,1,2,2,1 & sb,none
+dualalu,1,1,1,1,1,1+7,1,1,2,2,1 & sb,none
+r2p,1,1,1,1,1,1+7,1,1,2,2,1 & sb,none
+r2ur,1,1,1,1,1,1,1,1,2,2,1 & sb,none
+fma,1,1,1,1,1,1+8,1,1,2,2,1 & sb,none
+fp16,2+6,2+6,2+6,2+6,2+6,2+6,2+6,1,2+7,2+7,1 & sb,none
+hfma2_mma,2+5,2+5,2+5,2+5,2+5,2+5,2+5,2+5,1,1,1 & sb,none
+redirected_fp64,2+5,2+5,2+5,2+5,2+5,2+5,2+5,2+5,1,1,1 & sb,none
+decoupled,2+10,2+10,2+10,2+10,2+10,2+10,2+10,2+10,1+12,1+12,1 & sb,none
+guard,none,none,none,none,none,none,none,none,none,none,none,none
@@ -0,0 +1,23 @@
+raw,alu,dualalu,disp_64,fma,fma_alu,imad_wide_read_ab,imad_wide_read_cl,imad_wide_read_ch,imad_wide_write_dl,imad_wide_write_dh,fp16,fp16_alu,fp16_f32,hfma2_mma,redirected_fp64,imma,hmma,dmma,branch,decoupled,decoupled_agu
+alu,4,4,6,5,5,none,none,none,3,5,5,5,5,10,10,19,19,1 & sb,1 & sb,1 & sb,1 & sb
+dualalu,4,4,6,5,5,none,none,none,3,5,5,5,5,10,10,19,19,1 & sb,1 & sb,1 & sb,1 & sb
+disp_64,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none
+fma,5,5,6,4,4,none,none,none,2,4,5,5,5,10,10,19,19,1 & sb,1 & sb,1 & sb,1 & sb
+fma_alu,5,5,6,4,4,none,none,none,2,4,5,5,5,10,10,19,19,1 & sb,1 & sb,1 & sb,1 & sb
+imad_wide_read_ab,5,5,6,4,4,none,none,none,4,6,5,5,5,10,10,19,19,1 & sb,1 & sb,1 & sb,1 & sb
+imad_wide_read_cl,5,5,6,4,4,none,none,none,2,4,5,5,5,10,10,19,19,1 & sb,1 & sb,1 & sb,1 & sb
+imad_wide_read_ch,3,3,4,2,2,none,none,none,2,2,3,3,3,8,8,19,19,1 & sb,1 & sb,1 & sb,1 & sb
+imad_wide_write_dl,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none
+imad_wide_write_dh,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none
+fp16,5,5,6,5,5,none,none,none,3,5,4,5,5,10,10,19,19,1 & sb,1 & sb,1 & sb,1 & sb
+fp16_alu,5,5,6,5,5,none,none,none,3,5,5,4,5,10,10,19,19,1 & sb,1 & sb,1 & sb,1 & sb
+fp16_f32,5,5,6,5,5,none,none,none,3,5,5,5,5,10,10,19,19,1 & sb,1 & sb,1 & sb,1 & sb
+hfma2_mma,6,6,6,6,6,none,none,none,6,6,6,6,6,8,10,19,19,1 & sb,1 & sb,1 & sb,1 & sb
+redirected_fp64,6,6,6,6,6,none,none,none,6,6,6,6,6,8,8,19,19,1 & sb,1 & sb,1 & sb,1 & sb
+imma,7,7,7,7,7,none,none,none,7,7,7,7,7,11,11,20,20,1 & sb,1 & sb,1 & sb,1 & sb
+hmma,7,7,7,7,7,none,none,none,7,7,7,7,7,11,11,20,20,1 & sb,1 & sb,1 & sb,1 & sb
+dmma,7,7,7,7,7,none,none,none,7,7,7,7,7,11,11,20,20,1 & sb,1 & sb,1 & sb,1 & sb
+branch,4,4,4,4,4,none,none,none,4,4,4,4,4,6,6,19,19,1 & sb,1 & sb,1 & sb,1 & sb
+decoupled,4,4,4,4,4,none,none,none,4,4,4,4,4,6,6,19,19,1 & sb,1 & sb,1 & sb,1 & sb
+decoupled_agu,5,5,5,5,5,none,none,none,5,5,5,5,5,7,7,19,19,1 & sb,1 & sb,1 & sb,1 & sb
+
@@ -0,0 +1,23 @@
+war,alu,dualalu,disp_64,fma,fma_alu,imad_wide_read_ab,imad_wide_read_cl,imad_wide_read_ch,imad_wide_write_dl,imad_wide_write_dh,fp16,fp16_alu,fp16_f32,hfma2_mma,redirected_fp64,imma,hmma,dmma,branch,decoupled,decoupled_agu
+alu,1,1,1,1,1,1,1,1,none,none,1,1,1,1,1,1,1,1 & sb,1 & sb,1 & sb,1 & sb
+dualalu,1,1,1,1,1,1,1,1,none,none,1,1,1,1,1,1,1,1 & sb,1 & sb,1 & sb,1 & sb
+disp_64,1,1,1,1,1,1,1,1,none,none,1,1,1,1,1,1,1,1 & sb,1 & sb,1 & sb,1 & sb
+fma,1,1,1,1,1,1,1,1,none,none,1,1,1,1,1,1,1,1 & sb,1 & sb,1 & sb,1 & sb
+fma_alu,1,1,1,1,1,1,1,1,none,none,1,1,1,1,1,1,1,1 & sb,1 & sb,1 & sb,1 & sb
+imad_wide_read_ab,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none
+imad_wide_read_cl,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none
+imad_wide_read_ch,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none
+imad_wide_write_dl,1,1,1,1,1,1,1,1,none,none,1,1,1,1,1,1,1,1 & sb,1 & sb,1 & sb,1 & sb
+imad_wide_write_dh,1,1,1,1,1,1,1,1,none,none,1,1,1,1,1,1,1,1 & sb,1 & sb,1 & sb,1 & sb
+fp16,1,1,1,1,1,1,1,1,none,none,1,1,1,1,1,1,1,1 & sb,1 & sb,1 & sb,1 & sb
+fp16_alu,1,1,1,1,1,1,1,1,none,none,1,1,1,1,1,1,1,1 & sb,1 & sb,1 & sb,1 & sb
+fp16_f32,1,1,1,1,1,1,1,1,none,none,1,1,1,1,1,1,1,1 & sb,1 & sb,1 & sb,1 & sb
+hfma2_mma,1,1,1,1,1,1,1,1,none,none,1,1,1,1,1,1,1,1 & sb,1 & sb,1 & sb,1 & sb
+redirected_fp64,1,1,1,1,1,1,1,1,none,none,1,1,1,1,1,1,1,1 & sb,1 & sb,1 & sb,1 & sb
+imma,2,2,2,2,2,2,2,2,none,none,2,2,2,2,1,1,1,1 & sb,1 & sb,1 & sb,1 & sb
+hmma,2,2,2,2,2,2,2,2,none,none,2,2,2,2,1,1,1,1 & sb,1 & sb,1 & sb,1 & sb
+dmma,2,2,2,2,2,2,2,2,none,none,2,2,2,2,1,1,1,1 & sb,1 & sb,1 & sb,1 & sb
+branch,2,2,2,2,2,2,2,2,none,none,2,2,2,2,1,1,1,1 & sb,1 & sb,1 & sb,1 & sb
+decoupled,2,2,2,2,2,2,2,2,none,none,2,2,2,2,1,1,1,1 & sb,1 & sb,1 & sb,1 & sb
+decoupled_agu,2,2,2,2,2,2,2,2,none,none,2,2,2,2,1,1,1,1 & sb,1 & sb,1 & sb,1 & sb
+
@@ -0,0 +1,22 @@
+waw,alu,dualalu,disp_64,fma,fma_alu,imad_wide_read_ab,imad_wide_read_cl,imad_wide_read_ch,imad_wide_write_dl,imad_wide_write_dh,fp16,fp16_alu,fp16_f32,hfma2_mma,redirected_fp64,imma,hmma,dmma,branch,decoupled,decoupled_agu
+alu,1,1,1+1,1,1,none,none,none,1,1,1,1,1,3+3,3+3,14+2,14+2,1 & sb,1 & sb,1 & sb,1 & sb
+dualalu,1,1,1+1,1,1,none,none,none,1,1,1,1,1,3+3,3+3,14+2,14+2,1 & sb,1 & sb,1 & sb,1 & sb
+disp_64,1,1,1,1,1,none,none,none,1,1,1,1,1,3+1,3+1,14+1,14+1,1 & sb,1 & sb,1 & sb,1 & sb
+fma,1,1,1+1,1,1,none,none,none,1,1+1,1,1,1,3+3,3+3,14+3,14+3,1 & sb,1 & sb,1 & sb,1 & sb
+fma_alu,1,1,1+1,1,1,none,none,none,1,1+1,1,1,1,3+3,3+3,14+3,14+3,1 & sb,1 & sb,1 & sb,1 & sb
+imad_wide_read_ab,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none
+imad_wide_read_cl,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none
+imad_wide_read_ch,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none
+imad_wide_write_dl,1+2,1+2,1+3,1+1,1+1,none,none,none,1,1+1,1+2,1+2,1+2,5+3,5+3,14+3,14+3,1 & sb,1 & sb,1 & sb,1 & sb
+imad_wide_write_dh,1,1,1+1,1,1,none,none,none,1,1,1,1,1,5+1,5+1,14+3,14+3,1 & sb,1 & sb,1 & sb,1 & sb
+fp16,1,1,1+1,1,1,none,none,none,1,1,1,1,1,3+3,3+3,14+2,14+2,1 & sb,1 & sb,1 & sb,1 & sb
+fp16_alu,1,1,1+1,1,1,none,none,none,1,1,1,1,1,3+3,3+3,14+2,14+2,1 & sb,1 & sb,1 & sb,1 & sb
+fp16_f32,1,1,1,1,1,none,none,none,1,1,1,1,1,3+2,3+2,14+2,14+2,1 & sb,1 & sb,1 & sb,1 & sb
+hfma2_mma,1,1,1,1,1,none,none,none,1,1,1,1,1,1,3,14,14,1 & sb,1 & sb,1 & sb,1 & sb
+redirected_fp64,1,1,1,1,1,none,none,none,1,1,1,1,1,2,1,13,13,1 & sb,1 & sb,1 & sb,1 & sb
+imma,2,2,2,2,2,none,none,none,2,2,2,2,2,2,2,1,1,1 & sb,1 & sb,1 & sb,1 & sb
+hmma,2,2,2,2,2,none,none,none,2,2,2,2,2,2,2,1,1,1 & sb,1 & sb,1 & sb,1 & sb
+dmma,2+4,2+4,2+4,2+4,2+4,none,none,none,2+4,2+4,2+4,2+4,2+4,2+8,2+8,10+9,10+9,1 & sb,1 & sb,1 & sb,1 & sb
+branch,1+5,1+5,1+5,1+5,1+5,none,none,none,1+5,1+5,1+5,1+5,1+5,1+9,1+9,13+6,13+6,1 & sb,1 & sb,1 & sb,1 & sb
+decoupled,1+5,1+5,1+5,1+5,1+5,none,none,none,1+5,1+5,1+5,1+5,1+5,1+9,1+9,13+6,13+6,1 & sb,1 & sb,1 & sb,1 & sb
+decoupled_agu,1+5,1+5,1+5,1+5,1+5,none,none,none,1+5,1+5,1+5,1+5,1+5,1+9,1+9,13+6,13+6,1 & sb,1 & sb,1 & sb,1 & sb
@@ -0,0 +1,8 @@
+raw,coupled,udp,voteu,u_guard,bra_jmp,uldc_mma,usetmaxreg
+coupled,none,6,1,none,none,none,1 & sb
+udp,none,4,1,none,none,none,1 & sb
+voteu,none,none,none,none,none,none,none
+u_guard,none,11,5,none,none,none,1 & sb
+bra_jmp,none,9,2,none,none,none,1 & sb
+uldc_mma,none,11,5,none,none,none,1 & sb
+usetmaxreg,none,none,none,none,none,none,none
@@ -0,0 +1,9 @@
+war,coupled,udp,voteu,u_guard,bra_jmp,uldc_mma,usetmaxreg
+coupled,none,none,none,none,none,none,none
+udp,1,1,none,1,1,1,none
+voteu,2,2,none,1,1,1,none
+u_guard,none,none,none,none,none,none,none
+bra_jmp,none,none,none,none,none,none,none
+uldc_mma,none,none,none,none,none,none,none
+usetmaxreg,1,1,none,1,1,1,none
+
@@ -0,0 +1,8 @@
+waw,coupled,udp,voteu,u_guard,bra_jmp,uldc_mma,usetmaxreg
+coupled,none,none,none,none,none,none,none
+udp,none,1,1,none,none,none,1 & sb
+voteu,none,7,1,none,none,none,1 & sb
+u_guard,none,none,none,none,none,none,none
+bra_jmp,none,none,none,none,none,none,none
+uldc_mma,none,none,none,none,none,none,none
+usetmaxreg,none,8+2,8,none,none,none,1 & sb
@@ -0,0 +1,18 @@
+raw,coupled,coupled_mma,decoupled,branch,coupled_bindless,decoupled_bindless,hfma2_mma,to_ur,tex,tma,rpcmov_64,udp,uldc,umov,elect,r2ur,voteu
+coupled,none,none,none,none,none,none,none,1 & sb,none,none,none,6,2,2,2,13,2
+coupled_mma,none,none,none,none,none,none,none,1 & sb,none,none,none,6,2,2,2,13,2
+decoupled,none,none,none,none,none,none,none,1 & sb,none,none,none,9,2,2,2,13,2
+branch,none,none,none,none,none,none,none,1 & sb,none,none,none,10,3,3,3,13,3
+coupled_bindless,none,none,none,none,none,none,none,1 & sb,none,none,none,12,5,5,5,15,5
+decoupled_bindless,none,none,none,none,none,none,none,1 & sb,none,none,none,12,5,5,5,15,5
+hfma2_mma,none,none,none,none,none,none,none,1 & sb,none,none,none,9,2,2,2,13,2
+to_ur,none,none,none,none,none,none,none,1 & sb,none,none,none,12,5,5,5,13,5
+tex,none,none,none,none,none,none,none,1 & sb,none,none,none,9,2,2,2,13,2
+tma,none,none,none,none,none,none,none,1 & sb,none,none,none,9,2,2,2,13,2
+rpcmov_64,none,none,none,none,none,none,none,1 & sb,none,none,none,9,2,2,2,13,2
+udp,none,none,none,none,none,none,none,1 & sb,none,none,none,4,2,2,2,13,2
+uldc,none,none,none,none,none,none,none,1 & sb,none,none,none,12,5,5,5,15,5
+umov,none,none,none,none,none,none,none,1 & sb,none,none,none,7,2,2,2,13,2
+elect,none,none,none,none,none,none,none,1 & sb,none,none,none,8,2,2,2,13,2
+r2ur,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none
+voteu,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none
@@ -0,0 +1,18 @@
+war,coupled,coupled_mma,decoupled,branch,coupled_bindless,decoupled_bindless,hfma2_mma,to_ur,tex,tma,rpcmov_64,udp,uldc,umov,elect,r2ur,voteu
+coupled,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none
+coupled_mma,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none
+decoupled,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none
+branch,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none
+coupled_bindless,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none
+decoupled_bindless,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none
+hfma2_mma,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none
+to_ur,1,1,1,1,1,1,1,1 & sb,1,1 & sb,1,1,1,1,1,none,none
+tex,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none
+tma,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none
+rpcmov_64,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none
+udp,1,1,1,1,1,1,1,1 & sb,1,1 & sb,1,1,1,1,1,none,none
+uldc,1,1,1,1,1,1,1,1 & sb,1,1 & sb,1,3,1,1,1,none,none
+umov,1,1,1,1,1,1,1,1 & sb,1,1 & sb,1,3,1,1,1,none,none
+elect,1,1,1,1,1,1,1,1 & sb,1,1 & sb,1,3,1,1,1,none,none
+r2ur,1,1,1,1,1,1,1,1 & sb,1,1 & sb,1,1,1,1,1,none,none
+voteu,1,1,1,1,1,1,1,1 & sb,1,1 & sb,1,3,1,1,1,none,none
@@ -0,0 +1,19 @@
+waw,coupled,coupled_mma,decoupled,branch,coupled_bindless,decoupled_bindless,hfma2_mma,to_ur,tex,tma,rpcmov_64,udp,uldc,umov,elect,r2ur,voteu
+coupled,none,none,none,none,none,none,none,1 & sb,none,none,none,none,none,none,none,none,none
+coupled_mma,none,none,none,none,none,none,none,1 & sb,none,none,none,none,none,none,none,none,none
+decoupled,none,none,none,none,none,none,none,1 & sb,none,none,none,none,none,none,none,none,none
+branch,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none
+coupled_bindless,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none
+decoupled_bindless,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none
+hfma2_mma,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none
+to_ur,none,none,none,none,none,none,none,1 & sb,none,none,none,4+7,4,4,4,4+10,4
+tex,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none
+tma,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none
+rpcmov_64,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none,none
+udp,none,none,none,none,none,none,none,1 & sb,none,none,none,1,1,1,1,4+5,1
+uldc,none,none,none,none,none,none,none,1 & sb,none,none,none,7,1,1,1,10+1,1
+umov,none,none,none,none,none,none,none,1 & sb,none,none,none,7,1,1,1,10+1,1
+elect,none,none,none,none,none,none,none,1 & sb,none,none,none,7,1,1,1,10+1,1
+r2ur,none,none,none,none,none,none,none,1 & sb,none,none,none,7,1,1,1,1,1
+voteu,none,none,none,none,none,none,none,1 & sb,none,none,none,7,1,1,1,10+1,1
+
@@ -0,0 +1,184 @@
+# Copyright © 2022 Collabora, Ltd.
+# SPDX-License-Identifier: MIT
+
+subdir('latencies')
+
+dep_paste = dependency('paste',
+  version : '>= 1.0.14',
+  fallback : ['paste-1-rs', 'dep_paste'],
+  required : true,
+)
+
+dep_rustc_hash = dependency('rustc-hash',
+  version: '>= 2.0',
+  fallback: ['rustc-hash-2-rs', 'dep_rustc_hash'],
+  required: true,
+)
+
+libnak_c_files = files(
+  'nak.h',
+  'nak_nir.c',
+  'nak_nir_lower_cf.c',
+  'nak_nir_lower_cmat.c',
+  'nak_nir_lower_fs_inputs.c',
+  'nak_nir_lower_gs_intrinsics.c',
+  'nak_nir_lower_image_addrs.c',
+  'nak_nir_lower_mesh_intrinsics.c',
+  'nak_nir_lower_non_uniform_ldcx.c',
+  'nak_nir_lower_scan_reduce.c',
+  'nak_nir_lower_shared_atomics.c',
+  'nak_nir_lower_tex.c',
+  'nak_nir_lower_vtg_io.c',
+  'nak_nir_mark_lcssa_invariants.c',
+  'nak_nir_rematerialize_load_const.c',
+)
+
+_libacorn_rs = static_library(
+  'acorn',
+  files('acorn/lib.rs'),
+  gnu_symbol_visibility : 'hidden',
+  rust_abi : 'rust',
+  rust_args : nouveau_rust_args,
+)
+
+idep_acorn_rs = declare_dependency(
+  link_with : _libacorn_rs,
+)
+
+libnak_deps = [
+  idep_mesautil,
+  idep_nir_headers,
+  idep_nvidia_headers,
+  idep_nil_h,
+]
+
+_nak_bindings_rs = rust.bindgen(
+  input : ['nak_bindings.h'],
+  output : 'nak_bindings.rs',
+  c_args : [
+    pre_args,
+  ],
+  args : [
+    bindgen_output_args,
+    compiler_rs_bindgen_blocklist,
+    '--raw-line', 'use compiler::bindings::*;',
+    '--allowlist-type', 'drm.*',
+    '--allowlist-type', 'nak_.*',
+    '--allowlist-type', 'nouveau_ws_.*',
+    '--allowlist-var', 'DRM_.*',
+    '--allowlist-var', 'NAK_.*',
+    '--allowlist-var', 'NVIDIA_VENDOR_ID',
+    '--allowlist-function', 'drm.*',
+    '--allowlist-function', 'nak_.*',
+    '--allowlist-function', 'nouveau_ws_.*',
+    # provided through compiler::bindings::*
+    '--blocklist-type', 'glsl_.*',
+    '--no-prepend-enum-name',
+    '--with-derive-default',
+  ],
+  dependencies : [
+    dep_libdrm,
+    idep_nouveau_ws,
+    libnak_deps,
+  ],
+)
+
+_libnak_bindings_rs = static_library(
+  'nak_bindings',
+  _nak_bindings_rs,
+  gnu_symbol_visibility : 'hidden',
+  dependencies : [
+    idep_compiler_rs,
+  ],
+  rust_abi : 'rust',
+)
+
+_libnak_ir_proc_rs = rust.proc_macro(
+  'nak_ir_proc',
+  files('nak/ir_proc.rs'),
+  dependencies : [idep_compiler_proc_rs],
+  rust_args : rust_global_args,
+)
+
+_libnak_rs = static_library(
+  'nak_rs',
+  files('nak/lib.rs'),
+  gnu_symbol_visibility : 'hidden',
+  rust_abi : 'c',
+  rust_args : [
+    nouveau_rust_args,
+    # Otherwise, rustc trips up on -pthread
+    '-Clink-arg=-Wno-unused-command-line-argument',
+  ],
+  dependencies : [
+    dep_paste,
+    dep_rustc_hash,
+    idep_bitview_rs,
+    idep_compiler_rs,
+    idep_nvidia_headers_rs,
+  ],
+  link_with : [
+    _libnak_bindings_rs,
+    _libnak_ir_proc_rs,
+    libnak_latencies_rs,
+  ],
+)
+
+# TODO: Linking Rust executables (such as unit tests) doesn't play nicely
+# with the sanitizers because meson doesn't know to pass -fsanitize to the
+# Rust linker.  See also https://github.com/mesonbuild/meson/issues/11741
+#
+# Also, don't build on Android because we do cross-builds and, unlike most
+# tests, these depend on libdrm and that confuses the build system.
+if with_tests and get_option('b_sanitize') == 'none' and not with_platform_android
+  rust.test(
+    'nak',
+    _libnak_rs,
+    args : [
+      # Don't run HW or nvdisasm tests by default
+      '--skip', 'hw_tests::',
+      '--skip', 'nvdisasm_tests::',
+    ],
+    suite : ['nouveau'],
+    dependencies : [
+      dep_libdrm,
+      idep_nouveau_ws,
+      idep_compiler.partial_dependency(link_args : true, links : true),
+      idep_mesautil.partial_dependency(link_args : true, links : true),
+      idep_nv_push_rs,
+    ],
+    # This is needed to ensure we link against glibc
+    # See also https://gitlab.freedesktop.org/mesa/mesa/-/issues/11632
+    rust_args: ['-C', 'default-linker-libraries'],
+    link_with: [
+      _libacorn_rs,
+    ],
+  )
+endif
+
+nak_nir_algebraic_c = custom_target(
+  'nak_nir_algebraic.c',
+  input : 'nak_nir_algebraic.py',
+  output : 'nak_nir_algebraic.c',
+  command : [
+    prog_python, '@INPUT@',
+    '-p', dir_compiler_nir,
+    '--out', '@OUTPUT@',
+  ],
+  depend_files : nir_algebraic_depends,
+)
+
+_libnak = static_library(
+  'nak',
+  [libnak_c_files, nak_nir_algebraic_c],
+  include_directories : [inc_include, inc_src],
+  dependencies : libnak_deps,
+  link_with : [_libnak_rs],
+  c_args : [no_override_init_args],
+  gnu_symbol_visibility : 'hidden',
+)
+
+idep_nak = declare_dependency(
+  include_directories : include_directories('.'),
+  link_with : _libnak,
+)
@@ -0,0 +1,339 @@
+/*
+ * Copyright © 2022 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef NAK_H
+#define NAK_H
+
+#include "compiler/shader_enums.h"
+#include "nir_defines.h"
+
+#include <assert.h>
+#include <stdbool.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define NAK_SUBGROUP_SIZE 32
+
+struct nak_compiler;
+struct nv_device_info;
+
+struct nak_compiler *nak_compiler_create(const struct nv_device_info *dev);
+void nak_compiler_destroy(struct nak_compiler *nak);
+
+uint64_t nak_debug_flags(const struct nak_compiler *nak);
+
+const struct nir_shader_compiler_options *
+nak_nir_options(const struct nak_compiler *nak);
+
+void nak_preprocess_nir(nir_shader *nir, const struct nak_compiler *nak);
+
+bool nak_nir_lower_image_addrs(nir_shader *nir, const struct nak_compiler *nak);
+
+struct nak_sample_location {
+   uint8_t x_u4 : 4;
+   uint8_t y_u4 : 4;
+};
+static_assert(sizeof(struct nak_sample_location) == 1,
+              "This struct has no holes");
+
+struct nak_sample_mask {
+   uint16_t sample_mask;
+};
+static_assert(sizeof(struct nak_sample_mask) == 2,
+              "This struct has no holes");
+
+PRAGMA_DIAGNOSTIC_PUSH
+PRAGMA_DIAGNOSTIC_ERROR(-Wpadded)
+struct nak_fs_key {
+   bool zs_self_dep;
+
+   /** True if sample shading is forced on via an API knob such as
+    * VkPipelineMultisampleStateCreateInfo::minSampleShading
+    */
+   bool force_sample_shading;
+   bool uses_underestimate;
+
+   uint8_t pad;
+};
+PRAGMA_DIAGNOSTIC_POP
+static_assert(sizeof(struct nak_fs_key) == 4, "This struct has no holes");
+
+struct nak_constant_offset_info {
+   /**
+    * The constant buffer index and offset at which the sample locations and
+    * pass sample masks tables lives.
+    */
+   uint8_t sample_info_cb;
+
+   /**
+    * The offset into sample_info_cb at which the sample locations live.  The
+    * sample locations table is an array of nak_sample_location where each
+    * sample location is two 4-bit unorm values packed into an 8-bit value
+    * with the bottom 4 bits for x and the top 4 bits for y.
+    */
+   uint32_t sample_locations_offset;
+
+   /**
+    * The offset into sample_info_cb at which the sample masks table lives.
+    * The sample masks table is an array of nak_sample_mask where each entry
+    * represents the set of samples covered by that pass corresponding to that
+    * sample in a multi-pass fragment shader invocaiton.
+    */
+   uint32_t sample_masks_offset;
+
+   /**
+    * The constant buffer index at which the printf buffer pointer lives.
+    */
+   uint8_t printf_cb;
+
+   /**
+    * The offset into printf_cb for the printf buffer pointer.
+    */
+   uint32_t printf_buffer_offset;
+};
+const extern struct nak_constant_offset_info nak_const_offsets_base;
+const extern struct nak_constant_offset_info nak_const_offsets_turing_graphics;
+
+#define NAK_PRINTF_BUFFER_SIZE 0x40000
+
+#ifdef NDEBUG
+#define NAK_CAN_PRINTF false
+#else
+#define NAK_CAN_PRINTF true
+#endif
+
+void nak_postprocess_nir(nir_shader *nir, const struct nak_compiler *nak,
+                         nir_variable_mode robust2_modes,
+                         const struct nak_fs_key *fs_key,
+                         bool has_task_shader);
+
+enum ENUM_PACKED nak_ts_domain {
+   NAK_TS_DOMAIN_ISOLINE = 0,
+   NAK_TS_DOMAIN_TRIANGLE = 1,
+   NAK_TS_DOMAIN_QUAD = 2,
+};
+
+enum ENUM_PACKED nak_ts_spacing {
+   NAK_TS_SPACING_INTEGER = 0,
+   NAK_TS_SPACING_FRACT_ODD = 1,
+   NAK_TS_SPACING_FRACT_EVEN = 2,
+};
+
+enum PACKED nak_mesh_topology {
+   NAK_MESH_TOPOLOGY_POINTS = 0,
+   NAK_MESH_TOPOLOGY_LINES = 1,
+   NAK_MESH_TOPOLOGY_TRIANGLES = 4,
+};
+
+struct nak_xfb_info {
+   uint32_t stride[4];
+   uint8_t stream[4];
+   uint8_t attr_count[4];
+   uint8_t attr_index[4][128];
+};
+
+/* This struct MUST have explicit padding fields to ensure that all padding is
+ * zeroed and the zeros get properly copied, even across API boundaries.
+ */
+#pragma GCC diagnostic push
+#pragma GCC diagnostic error "-Wpadded"
+struct nak_shader_info {
+   mesa_shader_stage stage;
+
+   /** Shader model */
+   uint8_t sm;
+
+   /** Number of GPRs used */
+   uint8_t num_gprs;
+
+   /**
+    * Number of control barriers used
+    *
+    * These are barriers in the sense of glsl barrier(), not reconvergence
+    * barriers. In CUDA, these barriers have an index, but we currently
+    * only use index zero for vulkan, which means this will be at most 1.
+    */
+   uint8_t num_control_barriers;
+
+   uint8_t _pad0;
+
+   /** Maximum number of warps per SM based on static information */
+   uint32_t max_warps_per_sm;
+
+   /** Number of instructions used */
+   uint32_t num_instrs;
+
+   /** Number of cycles used by fixed-latency instructions */
+   uint64_t num_static_cycles;
+
+   /** Number of spills from GPRs to Memory */
+   uint32_t num_spills_to_mem;
+
+   /** Number of fills from Memory to GPRs */
+   uint32_t num_fills_from_mem;
+
+   /** Number of spills between register files */
+   uint32_t num_spills_to_reg;
+
+   /** Number of fills between register files */
+   uint32_t num_fills_from_reg;
+
+   /** Size of shader local (scratch) memory */
+   uint32_t slm_size;
+
+   /** Size of call/return stack in bytes/warp */
+   uint32_t crs_size;
+
+   union {
+      struct {
+         /* Local workgroup size */
+         uint16_t local_size[3];
+
+         /* Shared memory size */
+         uint16_t smem_size;
+
+         uint8_t _pad[132];
+      } cs;
+
+      struct {
+         bool writes_depth;
+         bool reads_sample_mask;
+         bool post_depth_coverage;
+         bool uses_sample_shading;
+         bool early_fragment_tests;
+
+         uint8_t _pad[135];
+      } fs;
+
+      struct {
+         enum nak_ts_domain domain;
+         enum nak_ts_spacing spacing;
+         bool ccw;
+         bool point_mode;
+
+         uint8_t _pad[136];
+      } ts;
+
+      struct {
+         uint32_t gs_hdr[32];
+         uint16_t max_primitives;
+         uint16_t max_vertices;
+         uint16_t local_size;
+         uint16_t smem_size;
+         enum nak_mesh_topology topology;
+
+         /** Shader header for GS stage when per primitive outputs are used */
+         bool has_gs_sph;
+         bool has_task_shader;
+
+         uint8_t _pad[1];
+      } mesh;
+
+      struct {
+         uint16_t local_size;
+         uint16_t payload_smem_size;
+         uint16_t smem_size;
+         uint8_t _pad[130];
+      } task;
+
+      /* Used to initialize the union for other stages */
+      uint8_t _pad[140];
+   };
+
+   struct {
+      bool writes_layer;
+      bool writes_point_size;
+      bool writes_vprs_table_index;
+      uint8_t clip_enable;
+      uint8_t cull_enable;
+
+      uint8_t _pad[3];
+
+      struct nak_xfb_info xfb;
+   } vtg;
+
+   uint8_t _pad1[4];
+
+   /** Shader header for 3D stages */
+   uint32_t hdr[32];
+};
+#pragma GCC diagnostic pop
+
+struct nak_shader_bin {
+   struct nak_shader_info info;
+
+   uint32_t code_size;
+   const void *code;
+
+   const char *asm_str;
+};
+
+void nak_shader_bin_destroy(struct nak_shader_bin *bin);
+
+struct nak_shader_bin *
+nak_compile_shader(nir_shader *nir, bool dump_asm,
+                   const struct nak_compiler *nak,
+                   nir_variable_mode robust2_modes,
+                   const struct nak_fs_key *fs_key,
+                   bool has_task_shader);
+
+struct nak_qmd_cbuf {
+   uint32_t index;
+   uint32_t size;
+   uint64_t addr;
+};
+
+struct nak_qmd_info {
+   uint64_t addr;
+
+   uint32_t smem_size;
+
+   uint32_t global_size[3];
+
+   uint32_t num_cbufs;
+   struct nak_qmd_cbuf cbufs[8];
+};
+
+#define NAK_QMD_ALIGN_B 256
+#define NAK_MAX_QMD_SIZE_B 384
+#define NAK_MAX_QMD_DWORDS (NAK_MAX_QMD_SIZE_B / 4)
+
+uint32_t nak_qmd_size_B(const struct nv_device_info *dev);
+
+void nak_fill_qmd(const struct nv_device_info *dev,
+                  const struct nak_shader_info *info,
+                  const struct nak_qmd_info *qmd_info,
+                  void *qmd_out, size_t qmd_size);
+
+struct nak_qmd_dispatch_size_layout {
+   uint16_t x_start, x_end;
+   uint16_t y_start, y_end;
+   uint16_t z_start, z_end;
+
+   uint16_t local_x_start, local_x_end;
+   uint16_t local_y_start, local_y_end;
+   uint16_t local_z_start, local_z_end;
+};
+
+struct nak_qmd_dispatch_size_layout
+nak_get_qmd_dispatch_size_layout(const struct nv_device_info *dev);
+
+struct nak_qmd_cbuf_desc_layout {
+   uint16_t addr_shift;
+   uint16_t addr_lo_start, addr_lo_end;
+   uint16_t addr_hi_start, addr_hi_end;
+};
+
+struct nak_qmd_cbuf_desc_layout
+nak_get_qmd_cbuf_desc_layout(const struct nv_device_info *dev, uint8_t idx);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* NAK_H */
@@ -0,0 +1,567 @@
+// Copyright © 2022 Collabora, Ltd.
+// SPDX-License-Identifier: MIT
+
+use crate::from_nir::*;
+use crate::ir::{
+    ShaderInfo, ShaderIoInfo, ShaderModel, ShaderModelInfo, ShaderStageInfo,
+};
+use crate::sph;
+
+use compiler::bindings::*;
+use nak_bindings::*;
+
+use std::cmp::max;
+use std::env;
+use std::ffi::{CStr, CString};
+use std::fmt::Write;
+use std::os::raw::c_void;
+use std::panic;
+use std::sync::OnceLock;
+
+#[repr(u8)]
+enum DebugFlags {
+    Panic,
+    Print,
+    Serial,
+    Spill,
+    Annotate,
+    NoUgpr,
+    Cycles,
+}
+
+pub struct Debug {
+    flags: u32,
+}
+
+impl Debug {
+    fn new() -> Debug {
+        let debug_var = "NAK_DEBUG";
+        let debug_str = match env::var(debug_var) {
+            Ok(s) => s,
+            Err(_) => {
+                return Debug { flags: 0 };
+            }
+        };
+
+        let mut flags = 0;
+        for flag in debug_str.split(',') {
+            match flag.trim() {
+                "panic" => flags |= 1 << DebugFlags::Panic as u8,
+                "print" => flags |= 1 << DebugFlags::Print as u8,
+                "serial" => flags |= 1 << DebugFlags::Serial as u8,
+                "spill" => flags |= 1 << DebugFlags::Spill as u8,
+                "annotate" => flags |= 1 << DebugFlags::Annotate as u8,
+                "nougpr" => flags |= 1 << DebugFlags::NoUgpr as u8,
+                "cycles" => flags |= 1 << DebugFlags::Cycles as u8,
+                unk => eprintln!("Unknown NAK_DEBUG flag \"{}\"", unk),
+            }
+        }
+        Debug { flags: flags }
+    }
+}
+
+pub trait GetDebugFlags {
+    fn debug_flags(&self) -> u32;
+
+    fn panic(&self) -> bool {
+        self.debug_flags() & (1 << DebugFlags::Panic as u8) != 0
+    }
+
+    fn print(&self) -> bool {
+        self.debug_flags() & (1 << DebugFlags::Print as u8) != 0
+    }
+
+    fn serial(&self) -> bool {
+        self.debug_flags() & (1 << DebugFlags::Serial as u8) != 0
+    }
+
+    fn spill(&self) -> bool {
+        self.debug_flags() & (1 << DebugFlags::Spill as u8) != 0
+    }
+
+    fn annotate(&self) -> bool {
+        self.debug_flags() & (1 << DebugFlags::Annotate as u8) != 0
+    }
+
+    fn no_ugpr(&self) -> bool {
+        self.debug_flags() & (1 << DebugFlags::NoUgpr as u8) != 0
+    }
+
+    fn cycles(&self) -> bool {
+        self.debug_flags() & (1 << DebugFlags::Cycles as u8) != 0
+    }
+}
+
+pub static DEBUG: OnceLock<Debug> = OnceLock::new();
+
+impl GetDebugFlags for OnceLock<Debug> {
+    fn debug_flags(&self) -> u32 {
+        self.get_or_init(Debug::new).flags
+    }
+}
+
+#[no_mangle]
+pub extern "C" fn nak_should_print_nir() -> bool {
+    DEBUG.print()
+}
+
+#[no_mangle]
+pub extern "C" fn nak_debug_no_ugpr() -> bool {
+    DEBUG.no_ugpr()
+}
+
+fn nir_options(dev: &nv_device_info) -> nir_shader_compiler_options {
+    nir_shader_compiler_options {
+        lower_fdiv: true,
+        float_mul_add16: nir_float_muladd_support_has_ffma
+            | nir_float_muladd_support_fuse,
+        float_mul_add32: nir_float_muladd_support_has_ffma
+            | nir_float_muladd_support_fuse,
+        float_mul_add64: nir_float_muladd_support_has_ffma
+            | nir_float_muladd_support_fuse,
+        lower_flrp16: true,
+        lower_flrp32: true,
+        lower_flrp64: true,
+        lower_fsqrt: dev.sm < 52,
+        lower_bitfield_extract: false,
+        lower_bitfield_extract8: true,
+        lower_bitfield_extract16: true,
+        lower_bitfield_insert: true,
+        lower_pack_half_2x16: true,
+        lower_pack_unorm_2x16: true,
+        lower_pack_snorm_2x16: true,
+        lower_pack_unorm_4x8: true,
+        lower_pack_snorm_4x8: true,
+        lower_unpack_half_2x16: true,
+        lower_unpack_unorm_2x16: true,
+        lower_unpack_snorm_2x16: true,
+        lower_unpack_unorm_4x8: true,
+        lower_unpack_snorm_4x8: true,
+        lower_insert_byte: true,
+        lower_insert_word: true,
+        lower_device_index_to_zero: true,
+        lower_isign: true,
+        lower_uadd_sat: dev.sm < 70,
+        lower_usub_sat: dev.sm < 70,
+        lower_iadd_sat: true, // TODO
+        lower_doubles_options: nir_lower_drcp
+            | nir_lower_dsqrt
+            | nir_lower_drsq
+            | nir_lower_dtrunc
+            | nir_lower_dfloor
+            | nir_lower_dceil
+            | nir_lower_dfract
+            | nir_lower_dround_even
+            | nir_lower_dsat
+            | if dev.sm >= 70 { nir_lower_dminmax } else { 0 },
+        lower_int64_options: !(nir_lower_icmp64
+            | nir_lower_iadd64
+            | nir_lower_ineg64
+            | nir_lower_shift64
+            | nir_lower_imul_2x32_64
+            | nir_lower_vote_ieq64
+            | nir_lower_conv64)
+            | if dev.sm < 70 { nir_lower_vote_ieq64 } else { 0 }
+            | if dev.sm < 32 { nir_lower_shift64 } else { 0 },
+        lower_fmod: true,
+        lower_ffract: true,
+        lower_fpow: true,
+        lower_scmp: true,
+        lower_uadd_carry: true,
+        lower_usub_borrow: true,
+        has_rotate32: dev.sm >= 32,
+        has_iadd3: dev.sm >= 70,
+        has_imad32: dev.sm >= 70,
+        has_sdot_4x8: dev.sm >= 70,
+        has_udot_4x8: dev.sm >= 70,
+        has_sudot_4x8: dev.sm >= 70,
+        // We set .ftz on f32 by default so we can support fmulz whenever the client
+        // doesn't explicitly request denorms.
+        has_fmulz_no_denorms: true,
+        has_ffmaz_no_denorms: true,
+        has_find_msb_rev: true,
+        has_pack_half_2x16_rtz: true,
+        has_bfm: dev.sm >= 70,
+        discard_is_demote: true,
+        has_load_global_bounded: dev.sm >= 73,
+        vectorize_vec2_16bit: true,
+
+        max_unroll_iterations: 32,
+        max_samples: 8,
+        scalarize_ddx: true,
+        ..Default::default()
+    }
+}
+
+#[no_mangle]
+pub extern "C" fn nak_compiler_create(
+    dev: *const nv_device_info,
+) -> *mut nak_compiler {
+    assert!(!dev.is_null());
+    let dev = unsafe { &*dev };
+
+    let nak = Box::new(nak_compiler {
+        sm: dev.sm,
+        warps_per_sm: dev.max_warps_per_mp,
+        nir_options: nir_options(dev),
+    });
+
+    Box::into_raw(nak)
+}
+
+#[no_mangle]
+pub extern "C" fn nak_compiler_destroy(nak: *mut nak_compiler) {
+    unsafe { drop(Box::from_raw(nak)) };
+}
+
+#[no_mangle]
+pub extern "C" fn nak_debug_flags(_nak: *const nak_compiler) -> u64 {
+    DEBUG.debug_flags().into()
+}
+
+#[no_mangle]
+pub extern "C" fn nak_nir_options(
+    nak: *const nak_compiler,
+) -> *const nir_shader_compiler_options {
+    assert!(!nak.is_null());
+    let nak = unsafe { &*nak };
+    &nak.nir_options
+}
+
+#[repr(C)]
+pub struct ShaderBin {
+    pub bin: nak_shader_bin,
+    code: Vec<u32>,
+    asm: CString,
+}
+
+impl ShaderBin {
+    pub fn new(
+        sm: &ShaderModelInfo,
+        info: &ShaderInfo,
+        fs_key: Option<&nak_fs_key>,
+        code: Vec<u32>,
+        asm: &str,
+    ) -> ShaderBin {
+        let asm = CString::new(asm)
+            .expect("NAK assembly has unexpected null characters");
+
+        let c_info = nak_shader_info {
+            stage: match info.stage {
+                ShaderStageInfo::Compute(_) => MESA_SHADER_COMPUTE,
+                ShaderStageInfo::Vertex(_) => MESA_SHADER_VERTEX,
+                ShaderStageInfo::Fragment(_) => MESA_SHADER_FRAGMENT,
+                ShaderStageInfo::Geometry(_) => MESA_SHADER_GEOMETRY,
+                ShaderStageInfo::TessellationInit(_) => MESA_SHADER_TESS_CTRL,
+                ShaderStageInfo::Tessellation(_) => MESA_SHADER_TESS_EVAL,
+                ShaderStageInfo::Task(_) => MESA_SHADER_TASK,
+                ShaderStageInfo::Mesh(_) => MESA_SHADER_MESH,
+            },
+            sm: sm.sm(),
+            num_gprs: {
+                max(4, info.num_gprs as u32 + sm.hw_reserved_gprs())
+                    .try_into()
+                    .unwrap()
+            },
+            num_control_barriers: info.num_control_barriers,
+            _pad0: Default::default(),
+            _pad1: Default::default(),
+            max_warps_per_sm: info.max_warps_per_sm,
+            num_instrs: info.num_instrs,
+            num_static_cycles: info.num_static_cycles,
+            num_spills_to_mem: info.num_spills_to_mem,
+            num_fills_from_mem: info.num_fills_from_mem,
+            num_spills_to_reg: info.num_spills_to_reg,
+            num_fills_from_reg: info.num_fills_from_reg,
+            slm_size: info.slm_size,
+            crs_size: sm.crs_size(info.max_crs_depth),
+            __bindgen_anon_1: match &info.stage {
+                ShaderStageInfo::Compute(cs_info) => {
+                    nak_shader_info__bindgen_ty_1 {
+                        cs: nak_shader_info__bindgen_ty_1__bindgen_ty_1 {
+                            local_size: [
+                                cs_info.local_size[0],
+                                cs_info.local_size[1],
+                                cs_info.local_size[2],
+                            ],
+                            smem_size: cs_info.smem_size,
+                            _pad: [0; 132],
+                        },
+                    }
+                }
+                ShaderStageInfo::Fragment(fs_info) => {
+                    let fs_io_info = match &info.io {
+                        ShaderIoInfo::Fragment(io) => io,
+                        _ => unreachable!(),
+                    };
+                    nak_shader_info__bindgen_ty_1 {
+                        fs: nak_shader_info__bindgen_ty_1__bindgen_ty_2 {
+                            writes_depth: fs_io_info.writes_depth,
+                            reads_sample_mask: fs_io_info.reads_sample_mask,
+                            post_depth_coverage: fs_info.post_depth_coverage,
+                            uses_sample_shading: fs_info.uses_sample_shading,
+                            early_fragment_tests: fs_info.early_fragment_tests,
+                            _pad: [0; 135],
+                        },
+                    }
+                }
+                ShaderStageInfo::TessellationInit(ts_info) => {
+                    nak_shader_info__bindgen_ty_1 {
+                        ts: nak_shader_info__bindgen_ty_1__bindgen_ty_3 {
+                            domain: 0,
+                            spacing: ts_info
+                                .common
+                                .spacing
+                                .map_or(0, |x| x as u8),
+                            ccw: ts_info.common.ccw,
+                            point_mode: ts_info.common.point_mode,
+                            _pad: [0; 136],
+                        },
+                    }
+                }
+                ShaderStageInfo::Tessellation(ts_info) => {
+                    nak_shader_info__bindgen_ty_1 {
+                        ts: nak_shader_info__bindgen_ty_1__bindgen_ty_3 {
+                            domain: ts_info.domain as u8,
+                            spacing: ts_info
+                                .common
+                                .spacing
+                                .map_or(0, |x| x as u8),
+                            ccw: ts_info.common.ccw,
+                            point_mode: ts_info.common.point_mode,
+                            _pad: [0; 136],
+                        },
+                    }
+                }
+                ShaderStageInfo::Task(task_info) => {
+                    nak_shader_info__bindgen_ty_1 {
+                        task: nak_shader_info__bindgen_ty_1__bindgen_ty_5 {
+                            local_size: task_info.local_size,
+                            payload_smem_size: task_info.payload_smem_size,
+                            smem_size: task_info.smem_size,
+                            _pad: [0; 130],
+                        },
+                    }
+                }
+                ShaderStageInfo::Mesh(mesh_info) => {
+                    nak_shader_info__bindgen_ty_1 {
+                        mesh: nak_shader_info__bindgen_ty_1__bindgen_ty_4 {
+                            gs_hdr: sph::encode_gs_mesh_header(
+                                sm.sm(),
+                                mesh_info,
+                            ),
+                            max_primitives: mesh_info.max_primitives,
+                            max_vertices: mesh_info.max_vertices,
+                            local_size: mesh_info.local_size,
+                            smem_size: mesh_info.smem_size,
+                            topology: mesh_info.output_topology,
+                            has_gs_sph: mesh_info.has_gs_sph,
+                            has_task_shader: mesh_info.has_task_shader,
+                            _pad: [0; 1],
+                        },
+                    }
+                }
+                _ => nak_shader_info__bindgen_ty_1 { _pad: [0; 140] },
+            },
+            vtg: match &info.io {
+                ShaderIoInfo::Vtg(io) => {
+                    let writes_layer;
+                    let writes_vprs_table_index;
+                    if let ShaderStageInfo::Mesh(mesh) = &info.stage {
+                        writes_layer = mesh
+                            .primitive_io
+                            .attr_written(NAK_ATTR_RT_ARRAY_INDEX);
+                        writes_vprs_table_index = mesh
+                            .primitive_io
+                            .attr_written(NAK_ATTR_VPRS_TABLE_INDEX);
+                    } else {
+                        writes_layer = io.attr_written(NAK_ATTR_RT_ARRAY_INDEX);
+                        writes_vprs_table_index =
+                            io.attr_written(NAK_ATTR_VPRS_TABLE_INDEX);
+                    }
+
+                    nak_shader_info__bindgen_ty_2 {
+                        writes_layer,
+                        writes_point_size: io.attr_written(NAK_ATTR_POINT_SIZE),
+                        writes_vprs_table_index,
+                        clip_enable: io.clip_enable,
+                        cull_enable: io.cull_enable,
+                        xfb: if let Some(xfb) = &io.xfb {
+                            **xfb
+                        } else {
+                            Default::default()
+                        },
+                        _pad: Default::default(),
+                    }
+                }
+                _ => Default::default(),
+            },
+            hdr: sph::encode_header(sm, info, fs_key),
+        };
+
+        if DEBUG.print() {
+            let stage_name = unsafe {
+                let c_name = _mesa_shader_stage_to_string(c_info.stage as u32);
+                CStr::from_ptr(c_name).to_str().expect("Invalid UTF-8")
+            };
+
+            eprintln!("Stage: {}", stage_name);
+            eprintln!("Instruction count: {}", c_info.num_instrs);
+            eprintln!("Static cycle count: {}", c_info.num_static_cycles);
+            eprintln!("Max warps/SM: {}", c_info.max_warps_per_sm);
+            eprintln!("Spills to mem: {}", c_info.num_spills_to_mem);
+            eprintln!("Spills to reg: {}", c_info.num_spills_to_reg);
+            eprintln!("Fills from mem: {}", c_info.num_fills_from_mem);
+            eprintln!("Fills from reg: {}", c_info.num_fills_from_reg);
+            eprintln!("Num GPRs: {}", c_info.num_gprs);
+            eprintln!("SLM size: {}", c_info.slm_size);
+
+            if c_info.stage != MESA_SHADER_COMPUTE {
+                eprint_hex("Header", &c_info.hdr);
+            }
+
+            eprint_hex("Encoded shader", &code);
+        }
+
+        let bin = nak_shader_bin {
+            info: c_info,
+            code_size: (code.len() * 4).try_into().unwrap(),
+            code: code.as_ptr() as *const c_void,
+            asm_str: if asm.is_empty() {
+                std::ptr::null()
+            } else {
+                asm.as_ptr()
+            },
+        };
+        ShaderBin {
+            bin: bin,
+            code: code,
+            asm: asm,
+        }
+    }
+}
+
+impl std::ops::Deref for ShaderBin {
+    type Target = nak_shader_bin;
+
+    fn deref(&self) -> &nak_shader_bin {
+        &self.bin
+    }
+}
+
+#[no_mangle]
+pub extern "C" fn nak_shader_bin_destroy(bin: *mut nak_shader_bin) {
+    unsafe {
+        _ = Box::from_raw(bin as *mut ShaderBin);
+    };
+}
+
+fn eprint_hex(label: &str, data: &[u32]) {
+    eprint!("{}:", label);
+    for i in 0..data.len() {
+        if (i % 8) == 0 {
+            eprintln!();
+            eprint!(" ");
+        }
+        eprint!(" {:08x}", data[i]);
+    }
+    eprintln!();
+}
+
+macro_rules! pass {
+    ($s: expr, $pass: ident) => {
+        $s.$pass();
+        if DEBUG.print() {
+            eprintln!("NAK IR after {}:\n{}", stringify!($pass), $s);
+        }
+    };
+}
+
+fn nak_compile_shader_internal(
+    nir: *mut nir_shader,
+    dump_asm: bool,
+    nak: *const nak_compiler,
+    robust2_modes: nir_variable_mode,
+    fs_key: *const nak_fs_key,
+    has_task_shader: bool,
+) -> *mut nak_shader_bin {
+    unsafe {
+        nak_postprocess_nir(nir, nak, robust2_modes, fs_key, has_task_shader)
+    };
+    let nak = unsafe { &*nak };
+    let nir = unsafe { &*nir };
+    let fs_key = if fs_key.is_null() {
+        None
+    } else {
+        Some(unsafe { &*fs_key })
+    };
+
+    let sm = ShaderModelInfo::new(nak.sm, nak.warps_per_sm);
+    let mut s = nak_shader_from_nir(nak, nir, &sm, has_task_shader);
+
+    if DEBUG.print() {
+        eprintln!("NAK IR:\n{}", &s);
+    }
+
+    pass!(s, opt_bar_prop);
+    pass!(s, opt_uniform_instrs);
+    pass!(s, opt_copy_prop);
+    pass!(s, opt_prmt);
+    pass!(s, opt_lop);
+    pass!(s, opt_copy_prop);
+    pass!(s, opt_dce);
+    pass!(s, opt_out);
+    pass!(s, legalize);
+    pass!(s, opt_dce);
+    pass!(s, opt_instr_sched_prepass);
+    pass!(s, assign_regs);
+    pass!(s, lower_par_copies);
+    pass!(s, lower_copy_swap);
+    if nak.sm >= 70 {
+        pass!(s, opt_jump_thread);
+    } else {
+        pass!(s, opt_crs);
+    }
+
+    s.remove_annotations();
+
+    pass!(s, opt_instr_sched_postpass);
+    pass!(s, calc_instr_deps);
+
+    s.gather_info();
+
+    let mut asm = String::new();
+    if dump_asm {
+        write!(asm, "{}", s).expect("Failed to dump assembly");
+    }
+
+    let code = sm.encode_shader(&s);
+    let bin = Box::new(ShaderBin::new(&sm, &s.info, fs_key, code, &asm));
+    Box::into_raw(bin) as *mut nak_shader_bin
+}
+
+#[no_mangle]
+pub extern "C" fn nak_compile_shader(
+    nir: *mut nir_shader,
+    dump_asm: bool,
+    nak: *const nak_compiler,
+    robust2_modes: nir_variable_mode,
+    fs_key: *const nak_fs_key,
+    has_task_shader: bool,
+) -> *mut nak_shader_bin {
+    let compile = || {
+        nak_compile_shader_internal(
+            nir,
+            dump_asm,
+            nak,
+            robust2_modes,
+            fs_key,
+            has_task_shader,
+        )
+    };
+    if DEBUG.panic() {
+        compile()
+    } else {
+        panic::catch_unwind(compile).unwrap_or(std::ptr::null_mut())
+    }
+}
@@ -0,0 +1,59 @@
+// Copyright © 2025 Collabora, Ltd.
+// SPDX-License-Identifier: MIT
+
+use crate::ir::*;
+
+use rustc_hash::FxHashMap;
+
+pub struct ConstTracker {
+    map: FxHashMap<SSAValue, SrcRef>,
+}
+
+/// A tracker struct for finding re-materializable constants
+///
+/// Anything which is an immediate, Zero, or a bound cbuf can trivially be
+/// re-materialized anywhere in the shader and it's probably cheaper to do so
+/// than to try and keep them around in GPRs forever.  This is just a helper
+/// struct for implementing this logic in compiler passes.
+impl ConstTracker {
+    pub fn new() -> Self {
+        ConstTracker {
+            map: Default::default(),
+        }
+    }
+
+    /// Registers a copy instruction
+    ///
+    /// If the source of the copy is a constant, the destination SSA value and
+    /// the constant value get stored as a key/value pair.
+    pub fn add_copy(&mut self, op: &OpCopy) {
+        let Some(dst) = op.dst.as_ssa() else {
+            return;
+        };
+        debug_assert!(dst.comps() == 1);
+        let dst = dst[0];
+
+        debug_assert!(op.src.is_unmodified());
+        let is_const = match &op.src.src_ref {
+            SrcRef::Zero | SrcRef::True | SrcRef::False | SrcRef::Imm32(_) => {
+                true
+            }
+            SrcRef::CBuf(cb) => matches!(cb.buf, CBuf::Binding(_)),
+            _ => false,
+        };
+
+        if is_const {
+            self.map.insert(dst, op.src.src_ref.clone());
+        }
+    }
+
+    /// Tests if the ConstTracker contains the given SSAValue
+    pub fn contains(&self, ssa: &SSAValue) -> bool {
+        self.map.contains_key(ssa)
+    }
+
+    /// Returns the SrcRef associated with this SSAValue, if any
+    pub fn get(&self, ssa: &SSAValue) -> Option<&SrcRef> {
+        self.map.get(ssa)
+    }
+}
@@ -0,0 +1,624 @@
+// Copyright © 2022 Collabora, Ltd.
+// SPDX-License-Identifier: MIT
+
+use compiler::bindings::*;
+use nak_bindings::*;
+use nv_push_rs::Push as NvPush;
+use nvidia_headers::classes::cla0c0::mthd as cla0c0;
+use nvidia_headers::classes::clb1c0::mthd as clb1c0;
+use nvidia_headers::classes::clb1c0::MAXWELL_COMPUTE_B;
+use nvidia_headers::classes::clc3c0::mthd as clc3c0;
+use nvidia_headers::classes::clc3c0::VOLTA_COMPUTE_A;
+use nvidia_headers::classes::clc6c0::mthd as clc6c0;
+use nvidia_headers::classes::clc6c0::AMPERE_COMPUTE_A;
+use nvidia_headers::classes::clcbc0::HOPPER_COMPUTE_A;
+
+use std::io;
+use std::ops::Deref;
+use std::ptr;
+use std::ptr::NonNull;
+use std::slice;
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::{Arc, Mutex};
+
+unsafe fn is_nvidia_device(dev: drmDevicePtr) -> bool {
+    unsafe {
+        match (*dev).bustype as u32 {
+            DRM_BUS_PCI => {
+                let pci = (*dev).deviceinfo.pci.as_ref().unwrap();
+                pci.vendor_id == (NVIDIA_VENDOR_ID as u16)
+            }
+            _ => false,
+        }
+    }
+}
+
+#[repr(C)]
+pub struct CB0 {
+    pub data_addr_lo: u32,
+    pub data_addr_hi: u32,
+    pub data_stride: u32,
+    pub invocations: u32,
+}
+
+struct DrmDevices {
+    num_devices: usize,
+    devices: [drmDevicePtr; 16],
+}
+
+impl DrmDevices {
+    fn get() -> io::Result<Self> {
+        unsafe {
+            let mut devices: [drmDevicePtr; 16] = [std::ptr::null_mut(); 16];
+            let num_devices = drmGetDevices(
+                devices.as_mut_ptr(),
+                devices.len().try_into().unwrap(),
+            );
+            if num_devices < 0 {
+                return Err(io::Error::last_os_error());
+            }
+            Ok(DrmDevices {
+                num_devices: num_devices.try_into().unwrap(),
+                devices,
+            })
+        }
+    }
+
+    fn iter(&self) -> slice::Iter<'_, drmDevicePtr> {
+        self.devices[..self.num_devices].iter()
+    }
+}
+
+impl Deref for DrmDevices {
+    type Target = [drmDevicePtr];
+
+    fn deref(&self) -> &[drmDevicePtr] {
+        &self.devices[..self.num_devices]
+    }
+}
+
+impl Drop for DrmDevices {
+    fn drop(&mut self) {
+        unsafe {
+            drmFreeDevices(
+                self.devices.as_mut_ptr(),
+                self.num_devices.try_into().unwrap(),
+            );
+        }
+    }
+}
+
+pub struct Device {
+    dev: NonNull<nouveau_ws_device>,
+    next_addr: AtomicU64,
+}
+
+impl Device {
+    pub fn new(dev_id: Option<usize>) -> io::Result<Arc<Self>> {
+        let drm_devices = DrmDevices::get()?;
+        unsafe {
+            let drm_dev = if let Some(dev_id) = dev_id {
+                if dev_id >= drm_devices.len() {
+                    return Err(io::Error::new(
+                        io::ErrorKind::NotFound,
+                        "Unknown device {dev_id}",
+                    ));
+                }
+                drm_devices[dev_id]
+            } else {
+                if let Some(dev) =
+                    drm_devices.iter().find(|dev| is_nvidia_device(**dev))
+                {
+                    *dev
+                } else {
+                    return Err(io::Error::new(
+                        io::ErrorKind::NotFound,
+                        "Failed to find an NVIDIA device",
+                    ));
+                }
+            };
+
+            let dev = nouveau_ws_device_new(drm_dev);
+            let Some(dev) = NonNull::new(dev) else {
+                return Err(io::Error::last_os_error());
+            };
+
+            Ok(Arc::new(Device {
+                dev,
+                next_addr: AtomicU64::new(1 << 16),
+            }))
+        }
+    }
+
+    pub fn dev_info(&self) -> &nv_device_info {
+        unsafe { &self.dev.as_ref().info }
+    }
+
+    fn fd(&self) -> i32 {
+        unsafe { self.dev.as_ref().fd }
+    }
+
+    fn ws_dev(&self) -> *mut nouveau_ws_device {
+        self.dev.as_ptr()
+    }
+}
+
+impl Drop for Device {
+    fn drop(&mut self) {
+        unsafe { nouveau_ws_device_destroy(self.ws_dev()) }
+    }
+}
+
+pub struct Context {
+    dev: Arc<Device>,
+    ctx: NonNull<nouveau_ws_context>,
+    syncobj: u32,
+    sync_value: Mutex<u64>,
+}
+
+impl Context {
+    pub fn new(dev: Arc<Device>) -> io::Result<Self> {
+        unsafe {
+            let mut ctx: *mut nouveau_ws_context = std::ptr::null_mut();
+            let err = nouveau_ws_context_create(
+                dev.ws_dev(),
+                NOUVEAU_WS_ENGINE_COMPUTE,
+                &mut ctx,
+            );
+            if err != 0 {
+                return Err(io::Error::last_os_error());
+            }
+            let ctx = NonNull::new(ctx).unwrap();
+
+            let mut syncobj = 0_u32;
+            let err = drmSyncobjCreate(dev.fd(), 0, &mut syncobj);
+            if err != 0 {
+                nouveau_ws_context_destroy(ctx.as_ptr());
+                return Err(io::Error::last_os_error());
+            }
+
+            Ok(Context {
+                dev,
+                ctx,
+                syncobj,
+                sync_value: Mutex::new(0),
+            })
+        }
+    }
+
+    pub fn exec(&self, addr: u64, len: u16) -> io::Result<()> {
+        let sync_value = unsafe {
+            let mut sync_value = self.sync_value.lock().unwrap();
+            *sync_value += 1;
+
+            let push = drm_nouveau_exec_push {
+                va: addr,
+                va_len: len.into(),
+                flags: 0,
+            };
+            let sig = drm_nouveau_sync {
+                flags: DRM_NOUVEAU_SYNC_TIMELINE_SYNCOBJ,
+                handle: self.syncobj,
+                timeline_value: *sync_value,
+            };
+            let exec = drm_nouveau_exec {
+                channel: self.ctx.as_ref().channel as u32,
+                wait_count: 0,
+                wait_ptr: 0,
+                push_count: 1,
+                push_ptr: &push as *const _ as u64,
+                sig_count: 1,
+                sig_ptr: &sig as *const _ as u64,
+            };
+            let err = drmIoctl(
+                self.dev.fd(),
+                DRM_RS_IOCTL_NOUVEAU_EXEC.into(),
+                &exec as *const _ as *mut std::os::raw::c_void,
+            );
+            if err != 0 {
+                return Err(io::Error::last_os_error());
+            }
+            *sync_value
+        };
+        // The close of this unsafe { } drops the lock
+
+        unsafe {
+            let err = drmSyncobjTimelineWait(
+                self.dev.fd(),
+                &self.syncobj as *const _ as *mut _,
+                &sync_value as *const _ as *mut _,
+                1,        // num_handles
+                i64::MAX, // timeout_nsec
+                DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
+                std::ptr::null_mut(),
+            );
+            if err != 0 {
+                return Err(io::Error::last_os_error());
+            }
+
+            // Exec again to check for errors
+            let mut exec = drm_nouveau_exec {
+                channel: self.ctx.as_ref().channel as u32,
+                wait_count: 0,
+                wait_ptr: 0,
+                push_count: 0,
+                push_ptr: 0,
+                sig_count: 0,
+                sig_ptr: 0,
+            };
+            let err = drmIoctl(
+                self.dev.fd(),
+                DRM_RS_IOCTL_NOUVEAU_EXEC.into(),
+                ptr::from_mut(&mut exec).cast(),
+            );
+            if err != 0 {
+                return Err(io::Error::last_os_error());
+            }
+        }
+
+        Ok(())
+    }
+}
+
+impl Drop for Context {
+    fn drop(&mut self) {
+        unsafe {
+            drmSyncobjDestroy(self.dev.fd(), self.syncobj);
+            nouveau_ws_context_destroy(self.ctx.as_ptr());
+        }
+    }
+}
+
+pub struct BO {
+    dev: Arc<Device>,
+    bo: NonNull<nouveau_ws_bo>,
+    pub addr: u64,
+    pub map: *mut std::os::raw::c_void,
+}
+
+impl BO {
+    pub fn new(dev: Arc<Device>, size: u64) -> io::Result<BO> {
+        let size = size.next_multiple_of(4096);
+
+        let mut map: *mut std::os::raw::c_void = std::ptr::null_mut();
+        let bo = unsafe {
+            nouveau_ws_bo_new_mapped(
+                dev.ws_dev(),
+                size,
+                0, // align
+                NOUVEAU_WS_BO_GART,
+                NOUVEAU_WS_BO_RDWR,
+                ptr::from_mut(&mut map),
+            )
+        };
+        let Some(bo) = NonNull::new(bo) else {
+            return Err(io::Error::last_os_error());
+        };
+        assert!(!map.is_null());
+
+        let addr = dev.next_addr.fetch_add(size, Ordering::Relaxed);
+        assert!(addr % 4096 == 0);
+
+        unsafe {
+            nouveau_ws_bo_bind_vma(
+                dev.ws_dev(),
+                bo.as_ptr(),
+                addr,
+                size,
+                0, // bo_offset
+                0, // pte_kind
+            );
+        }
+
+        Ok(BO { dev, bo, addr, map })
+    }
+}
+
+impl Drop for BO {
+    fn drop(&mut self) {
+        unsafe {
+            nouveau_ws_bo_unbind_vma(
+                self.dev.dev.as_ptr(),
+                self.addr,
+                self.bo.as_ref().size,
+            );
+            nouveau_ws_bo_destroy(self.bo.as_ptr());
+        }
+    }
+}
+
+struct QMDHeapImpl {
+    dev: Arc<Device>,
+    bos: Vec<BO>,
+    last_offset: u32,
+    free: Vec<(u64, *mut std::os::raw::c_void)>,
+}
+
+struct QMDHeap {
+    pub qmd_size_B: u32,
+    imp: Mutex<QMDHeapImpl>,
+}
+
+struct QMD<'a> {
+    heap: &'a QMDHeap,
+    pub addr: u64,
+    pub map: *mut std::os::raw::c_void,
+}
+
+impl Drop for QMD<'_> {
+    fn drop(&mut self) {
+        let mut heap = self.heap.imp.lock().unwrap();
+        heap.free.push((self.addr, self.map));
+    }
+}
+
+impl QMDHeap {
+    const BO_SIZE: u32 = 1 << 16;
+
+    fn new(dev: Arc<Device>) -> Self {
+        let qmd_size_B = unsafe { nak_qmd_size_B(dev.dev_info()) };
+        Self {
+            qmd_size_B,
+            imp: Mutex::new(QMDHeapImpl {
+                dev,
+                bos: Vec::new(),
+                last_offset: Self::BO_SIZE,
+                free: Vec::new(),
+            }),
+        }
+    }
+
+    fn alloc_qmd<'a>(&'a self) -> io::Result<QMD<'a>> {
+        let mut imp = self.imp.lock().unwrap();
+        if let Some((addr, map)) = imp.free.pop() {
+            return Ok(QMD {
+                heap: self,
+                addr,
+                map,
+            });
+        }
+
+        if imp.last_offset >= Self::BO_SIZE {
+            let dev = imp.dev.clone();
+            imp.bos.push(BO::new(dev, Self::BO_SIZE.into())?);
+            imp.last_offset = 0;
+        }
+
+        let bo = imp.bos.last().unwrap();
+        let addr = bo.addr + u64::from(imp.last_offset);
+        let map =
+            unsafe { bo.map.byte_offset(imp.last_offset.try_into().unwrap()) };
+        imp.last_offset += self.qmd_size_B.next_multiple_of(NAK_QMD_ALIGN_B);
+
+        Ok(QMD {
+            heap: self,
+            addr,
+            map,
+        })
+    }
+}
+
+pub struct Runner {
+    dev: Arc<Device>,
+    ctx: Context,
+    qmd_heap: QMDHeap,
+}
+
+impl Runner {
+    pub fn new(dev_id: Option<usize>) -> Runner {
+        let dev = Device::new(dev_id).expect("Failed to create nouveau device");
+        let ctx = Context::new(dev.clone()).expect("Failed to create context");
+        let qmd_heap = QMDHeap::new(dev.clone());
+        Runner { dev, ctx, qmd_heap }
+    }
+
+    pub fn device(&self) -> Arc<Device> {
+        self.dev.clone()
+    }
+
+    pub fn dev_info(&self) -> &nv_device_info {
+        self.dev.dev_info()
+    }
+
+    pub unsafe fn run_raw(
+        &self,
+        shader: &nak_shader_bin,
+        invocations: u32,
+        data_stride: u32,
+        data: *mut std::os::raw::c_void,
+        data_size: usize,
+    ) -> io::Result<()> {
+        let cs_info = unsafe {
+            assert!(shader.info.stage == MESA_SHADER_COMPUTE);
+            &shader.info.__bindgen_anon_1.cs
+        };
+        assert!(cs_info.local_size[1] == 1 && cs_info.local_size[2] == 1);
+        let local_size = cs_info.local_size[0];
+
+        // Compute the needed size of the buffer
+        let mut size = 0_usize;
+
+        const MAX_PUSH_DW: usize = 256;
+        let push_offset = size;
+        size = push_offset + 4 * MAX_PUSH_DW;
+
+        let shader_offset = size.next_multiple_of(0x80);
+        size = shader_offset + usize::try_from(shader.code_size).unwrap();
+
+        let cb0_offset = size.next_multiple_of(256);
+        size = cb0_offset + size_of::<CB0>();
+
+        let data_offset = size.next_multiple_of(256);
+        size = data_offset + data_size;
+
+        let bo = BO::new(self.dev.clone(), size.try_into().unwrap())?;
+
+        // Copy the data from the caller into our BO
+        let data_addr = bo.addr + u64::try_from(data_offset).unwrap();
+        unsafe {
+            let data_map = bo.map.byte_offset(data_offset.try_into().unwrap());
+            if data_size > 0 {
+                std::ptr::copy(data, data_map, data_size);
+            }
+        }
+
+        // Fill out cb0
+        let cb0_addr = bo.addr + u64::try_from(cb0_offset).unwrap();
+        unsafe {
+            let cb0_map = bo.map.byte_offset(cb0_offset.try_into().unwrap());
+            cb0_map.cast::<CB0>().write(CB0 {
+                data_addr_lo: data_addr as u32,
+                data_addr_hi: (data_addr >> 32) as u32,
+                data_stride,
+                invocations,
+            });
+        }
+
+        // Upload the shader
+        let shader_addr = bo.addr + u64::try_from(shader_offset).unwrap();
+        unsafe {
+            let shader_map =
+                bo.map.byte_offset(shader_offset.try_into().unwrap());
+            std::ptr::copy(
+                shader.code,
+                shader_map,
+                shader.code_size.try_into().unwrap(),
+            );
+        }
+
+        // Populate and upload the QMD
+        let mut qmd_cbufs: [nak_qmd_cbuf; 8] = Default::default();
+        qmd_cbufs[0] = nak_qmd_cbuf {
+            index: 0,
+            size: size_of::<CB0>().next_multiple_of(256).try_into().unwrap(),
+            addr: cb0_addr,
+        };
+        let qmd_info = nak_qmd_info {
+            // Pre-Volta, we set the program region to the start of the bo
+            addr: if self.dev_info().cls_compute < VOLTA_COMPUTE_A {
+                shader_offset.try_into().unwrap()
+            } else {
+                shader_addr
+            },
+            smem_size: unsafe { shader.info.__bindgen_anon_1.cs }
+                .smem_size
+                .into(),
+            global_size: [invocations.div_ceil(local_size.into()), 1, 1],
+            num_cbufs: 1,
+            cbufs: qmd_cbufs,
+        };
+
+        let qmd = self.qmd_heap.alloc_qmd()?;
+        unsafe {
+            nak_fill_qmd(
+                self.dev_info(),
+                &shader.info,
+                &qmd_info,
+                qmd.map,
+                self.qmd_heap.qmd_size_B.try_into().unwrap(),
+            );
+        }
+
+        // Fill out the pushbuf
+        let mut p = NvPush::new();
+
+        p.push_mthd(cla0c0::SetObject {
+            class_id: self.dev_info().cls_compute.into(),
+            engine_id: 0,
+        });
+        if self.dev_info().cls_compute < VOLTA_COMPUTE_A {
+            p.push_mthd(cla0c0::SetProgramRegionA {
+                address_upper: (bo.addr >> 32) as u32,
+            });
+            p.push_mthd(cla0c0::SetProgramRegionB {
+                address_lower: bo.addr as u32,
+            });
+        }
+
+        let mut smem_base_addr = 0xfe000000_u64;
+        if self.dev_info().cls_compute >= HOPPER_COMPUTE_A {
+            // This needs to be 4GB aligned on Hopper+
+            smem_base_addr <<= 8;
+        }
+
+        let lmem_base_addr = 0xff000000_u32;
+        if self.dev_info().cls_compute >= VOLTA_COMPUTE_A {
+            p.push_mthd(clc3c0::SetShaderSharedMemoryWindowA {
+                base_address_upper: (smem_base_addr >> 32) as u32,
+            });
+            p.push_mthd(clc3c0::SetShaderSharedMemoryWindowB {
+                base_address: smem_base_addr as u32,
+            });
+
+            p.push_mthd(clc3c0::SetShaderLocalMemoryWindowA {
+                base_address_upper: 0,
+            });
+            p.push_mthd(clc3c0::SetShaderLocalMemoryWindowB {
+                base_address: lmem_base_addr,
+            });
+        } else {
+            p.push_mthd(cla0c0::SetShaderSharedMemoryWindow {
+                base_address: smem_base_addr as u32,
+            });
+            p.push_mthd(cla0c0::SetShaderLocalMemoryWindow {
+                base_address: lmem_base_addr,
+            });
+        }
+
+        if self.dev_info().cls_compute >= MAXWELL_COMPUTE_B {
+            p.push_mthd(clb1c0::InvalidateSkedCaches { v: 0 });
+        }
+
+        p.push_mthd(cla0c0::SendPcasA {
+            qmd_address_shifted8: (qmd.addr >> 8) as u32,
+        });
+        if self.dev_info().cls_compute >= AMPERE_COMPUTE_A {
+            p.push_mthd(clc6c0::SendSignalingPcas2B {
+                pcas_action: clc6c0::SendSignalingPcas2BPcasAction::InvalidateCopySchedule,
+            });
+        } else {
+            p.push_mthd(cla0c0::SendSignalingPcasB {
+                invalidate: true,
+                schedule: true,
+            });
+        }
+
+        let push_addr = bo.addr + u64::try_from(push_offset).unwrap();
+        unsafe {
+            let push_map = bo.map.byte_offset(push_offset.try_into().unwrap());
+            std::ptr::copy(p.as_ptr(), push_map.cast(), p.len());
+        }
+
+        let res = self.ctx.exec(push_addr, (p.len() * 4).try_into().unwrap());
+
+        // Always copy the data back to the caller, even if exec fails
+        unsafe {
+            let data_map = bo.map.byte_offset(data_offset.try_into().unwrap());
+            if data_size > 0 {
+                std::ptr::copy(data_map, data, data_size);
+            }
+        }
+
+        res
+    }
+
+    pub fn run<T>(
+        &self,
+        shader: &nak_shader_bin,
+        data: &mut [T],
+    ) -> io::Result<()> {
+        unsafe {
+            let stride = size_of::<T>();
+            self.run_raw(
+                shader,
+                data.len().try_into().unwrap(),
+                stride.try_into().unwrap(),
+                data.as_mut_ptr().cast(),
+                size_of_val(data),
+            )
+        }
+    }
+}
+
+unsafe impl Sync for Runner {}
+unsafe impl Send for Runner {}
@@ -0,0 +1,156 @@
+// Copyright © 2023 Collabora, Ltd.
+// SPDX-License-Identifier: MIT
+
+extern crate proc_macro;
+extern crate proc_macro2;
+#[macro_use]
+extern crate quote;
+extern crate syn;
+
+use compiler_proc::as_slice::*;
+use proc_macro::TokenStream;
+use proc_macro2::TokenStream as TokenStream2;
+use syn::*;
+
+#[proc_macro_derive(SrcsAsSlice, attributes(src_type))]
+pub fn derive_srcs_as_slice(input: TokenStream) -> TokenStream {
+    let input2 = input.clone();
+    let DeriveInput { ident, data, .. } = parse_macro_input!(input2);
+
+    if let Data::Enum(e) = data {
+        let mut as_slice_cases = TokenStream2::new();
+        let mut as_mut_slice_cases = TokenStream2::new();
+        let mut types_cases = TokenStream2::new();
+        for v in e.variants {
+            let case = v.ident;
+            as_slice_cases.extend(quote! {
+                #ident::#case(x) => x.srcs_as_slice(),
+            });
+            as_mut_slice_cases.extend(quote! {
+                #ident::#case(x) => x.srcs_as_mut_slice(),
+            });
+            types_cases.extend(quote! {
+                #ident::#case(x) => x.src_types(),
+            });
+        }
+        quote! {
+            impl SrcsAsSlice for #ident {
+                fn srcs_as_slice(&self) -> &[Src] {
+                    match self {
+                        #as_slice_cases
+                    }
+                }
+
+                fn srcs_as_mut_slice(&mut self) -> &mut [Src] {
+                    match self {
+                        #as_mut_slice_cases
+                    }
+                }
+
+                fn src_types(&self) -> SrcTypeList {
+                    match self {
+                        #types_cases
+                    }
+                }
+            }
+        }
+        .into()
+    } else {
+        derive_as_slice(input, "Src", "src_type", "SrcType")
+    }
+}
+
+#[proc_macro_derive(DstsAsSlice, attributes(dst_type))]
+pub fn derive_dsts_as_slice(input: TokenStream) -> TokenStream {
+    let input2 = input.clone();
+    let DeriveInput { ident, data, .. } = parse_macro_input!(input2);
+
+    if let Data::Enum(e) = data {
+        let mut as_slice_cases = TokenStream2::new();
+        let mut as_mut_slice_cases = TokenStream2::new();
+        let mut types_cases = TokenStream2::new();
+        for v in e.variants {
+            let case = v.ident;
+            as_slice_cases.extend(quote! {
+                #ident::#case(x) => x.dsts_as_slice(),
+            });
+            as_mut_slice_cases.extend(quote! {
+                #ident::#case(x) => x.dsts_as_mut_slice(),
+            });
+            types_cases.extend(quote! {
+                #ident::#case(x) => x.dst_types(),
+            });
+        }
+        quote! {
+            impl DstsAsSlice for #ident {
+                fn dsts_as_slice(&self) -> &[Dst] {
+                    match self {
+                        #as_slice_cases
+                    }
+                }
+
+                fn dsts_as_mut_slice(&mut self) -> &mut [Dst] {
+                    match self {
+                        #as_mut_slice_cases
+                    }
+                }
+
+                fn dst_types(&self) -> DstTypeList {
+                    match self {
+                        #types_cases
+                    }
+                }
+            }
+        }
+        .into()
+    } else {
+        derive_as_slice(input, "Dst", "dst_type", "DstType")
+    }
+}
+
+#[proc_macro_derive(DisplayOp)]
+pub fn enum_derive_display_op(input: TokenStream) -> TokenStream {
+    let DeriveInput { ident, data, .. } = parse_macro_input!(input);
+
+    if let Data::Enum(e) = data {
+        let mut fmt_dsts_cases = TokenStream2::new();
+        let mut fmt_op_cases = TokenStream2::new();
+        for v in e.variants {
+            let case = v.ident;
+            fmt_dsts_cases.extend(quote! {
+                #ident::#case(x) => x.fmt_dsts(f),
+            });
+            fmt_op_cases.extend(quote! {
+                #ident::#case(x) => x.fmt_op(f),
+            });
+        }
+        quote! {
+            impl DisplayOp for #ident {
+                fn fmt_dsts(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+                    match self {
+                        #fmt_dsts_cases
+                    }
+                }
+
+                fn fmt_op(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+                    match self {
+                        #fmt_op_cases
+                    }
+                }
+            }
+        }
+        .into()
+    } else {
+        panic!("Not an enum type");
+    }
+}
+
+#[proc_macro_derive(FromVariants)]
+pub fn derive_from_variants(input: TokenStream) -> TokenStream {
+    compiler_proc::from_variants::derive_from_variants(input)
+}
+
+#[proc_macro_derive(EnumAsU8)]
+pub fn derive_enum_as_u8(input: TokenStream) -> TokenStream {
+    compiler_proc::enum_as_u8::derive_enum_as_u8(input)
+}
@@ -0,0 +1,620 @@
+// Copyright © 2022 Collabora, Ltd.
+// SPDX-License-Identifier: MIT
+
+use crate::api::{GetDebugFlags, DEBUG};
+use crate::const_tracker::ConstTracker;
+use crate::ir::*;
+use crate::liveness::{BlockLiveness, Liveness, SimpleLiveness};
+
+use rustc_hash::{FxHashMap, FxHashSet};
+
+pub fn src_is_upred_reg(src: &Src) -> bool {
+    match &src.src_ref {
+        SrcRef::True | SrcRef::False => false,
+        SrcRef::SSA(ssa) => {
+            assert!(ssa.comps() == 1);
+            match ssa[0].file() {
+                RegFile::Pred => false,
+                RegFile::UPred => true,
+                _ => panic!("Not a predicate source"),
+            }
+        }
+        SrcRef::Reg(_) => panic!("Not in SSA form"),
+        _ => panic!("Not a predicate source"),
+    }
+}
+
+pub fn src_is_reg(src: &Src, reg_file: RegFile) -> bool {
+    match &src.src_ref {
+        SrcRef::Zero | SrcRef::True | SrcRef::False => true,
+        SrcRef::SSA(ssa) => ssa.file() == reg_file,
+        SrcRef::Imm32(_) | SrcRef::CBuf(_) => false,
+        SrcRef::Reg(_) => panic!("Not in SSA form"),
+    }
+}
+
+pub fn swap_srcs_if_not_reg(
+    x: &mut Src,
+    y: &mut Src,
+    reg_file: RegFile,
+) -> bool {
+    if !src_is_reg(x, reg_file) && src_is_reg(y, reg_file) {
+        std::mem::swap(x, y);
+        true
+    } else {
+        false
+    }
+}
+
+fn src_is_imm(src: &Src) -> bool {
+    matches!(src.src_ref, SrcRef::Imm32(_))
+}
+
+pub enum PadValue {
+    Zero,
+    #[allow(dead_code)]
+    Undefined,
+}
+
+pub trait LegalizeBuildHelpers: SSABuilder {
+    fn copy_ssa(&mut self, ssa: &mut SSAValue, reg_file: RegFile) {
+        let tmp = self.alloc_ssa(reg_file);
+        self.copy_to(tmp.into(), (*ssa).into());
+        *ssa = tmp;
+    }
+
+    fn copy_ssa_ref(&mut self, vec: &mut SSARef, reg_file: RegFile) {
+        for ssa in &mut vec[..] {
+            self.copy_ssa(ssa, reg_file);
+        }
+    }
+
+    fn copy_pred_ssa_if_uniform(&mut self, ssa: &mut SSAValue) {
+        match ssa.file() {
+            RegFile::Pred => (),
+            RegFile::UPred => self.copy_ssa(ssa, RegFile::Pred),
+            _ => panic!("Not a predicate value"),
+        }
+    }
+
+    fn copy_pred_if_upred(&mut self, pred: &mut Pred) {
+        match &mut pred.pred_ref {
+            PredRef::None => (),
+            PredRef::SSA(ssa) => {
+                self.copy_pred_ssa_if_uniform(ssa);
+            }
+            PredRef::Reg(_) => panic!("Not in SSA form"),
+        }
+    }
+
+    fn copy_src_if_upred(&mut self, src: &mut Src) {
+        match &mut src.src_ref {
+            SrcRef::True | SrcRef::False => (),
+            SrcRef::SSA(ssa) => {
+                assert!(ssa.comps() == 1);
+                self.copy_pred_ssa_if_uniform(&mut ssa[0]);
+            }
+            SrcRef::Reg(_) => panic!("Not in SSA form"),
+            _ => panic!("Not a predicate source"),
+        }
+    }
+
+    fn copy_src_if_not_same_file(&mut self, src: &mut Src) {
+        let SrcRef::SSA(vec) = &mut src.src_ref else {
+            return;
+        };
+
+        if vec.comps() == 1 {
+            return;
+        }
+
+        let mut all_same = true;
+        let file = vec[0].file();
+        for i in 1..vec.comps() {
+            let c_file = vec[usize::from(i)].file();
+            if c_file != file {
+                debug_assert!(c_file.to_warp() == file.to_warp());
+                all_same = false;
+            }
+        }
+
+        if !all_same {
+            self.copy_ssa_ref(vec, file.to_warp());
+        }
+    }
+
+    fn align_reg(
+        &mut self,
+        src: &mut Src,
+        n_comps: usize,
+        pad_value: PadValue,
+    ) {
+        debug_assert!(!matches!(src.src_ref, SrcRef::Reg(_)));
+        let SrcRef::SSA(ref old_val) = src.src_ref else {
+            return;
+        };
+        assert!(old_val.len() <= n_comps);
+        assert!(src.is_unmodified());
+
+        let pad_fn = || {
+            Some(match pad_value {
+                PadValue::Zero => self.copy(0.into()),
+                PadValue::Undefined => self.undef(),
+            })
+        };
+
+        // Pad the given ssa_ref with either undefined or zero
+        let ssa_vals: Vec<_> = old_val
+            .iter()
+            .copied()
+            .chain(std::iter::from_fn(pad_fn))
+            .take(n_comps)
+            .collect();
+
+        // Collect it in a new ssa_ref and replace it with the original.
+        let val = SSARef::try_from(ssa_vals).expect("Cannot create SSARef");
+        src.src_ref = val.into();
+    }
+
+    fn copy_alu_src(
+        &mut self,
+        src: &mut Src,
+        reg_file: RegFile,
+        src_type: SrcType,
+    ) {
+        let val = match src_type {
+            SrcType::GPR
+            | SrcType::ALU
+            | SrcType::F32
+            | SrcType::F16
+            | SrcType::F16v2
+            | SrcType::I32
+            | SrcType::B32 => self.alloc_ssa_vec(reg_file, 1),
+            SrcType::F64 => self.alloc_ssa_vec(reg_file, 2),
+            SrcType::Pred => self.alloc_ssa_vec(reg_file, 1),
+            _ => panic!("Unknown source type"),
+        };
+
+        if DEBUG.annotate() {
+            self.push_instr(Instr::new(OpAnnotate {
+                annotation: "copy generated by legalizer".into(),
+            }));
+        }
+
+        let old_src_ref =
+            std::mem::replace(&mut src.src_ref, val.clone().into());
+        if val.comps() == 1 {
+            self.copy_to(val[0].into(), old_src_ref.into());
+        } else {
+            match old_src_ref {
+                SrcRef::Imm32(u) => {
+                    // Immediates go in the top bits
+                    self.copy_to(val[0].into(), 0.into());
+                    self.copy_to(val[1].into(), u.into());
+                }
+                SrcRef::CBuf(cb) => {
+                    // CBufs load 8B
+                    self.copy_to(val[0].into(), cb.clone().into());
+                    self.copy_to(val[1].into(), cb.offset(4).into());
+                }
+                SrcRef::SSA(vec) => {
+                    assert!(vec.comps() == 2);
+                    self.copy_to(val[0].into(), vec[0].into());
+                    self.copy_to(val[1].into(), vec[1].into());
+                }
+                _ => panic!("Invalid 64-bit SrcRef"),
+            }
+        }
+    }
+
+    fn copy_alu_src_if_not_reg(
+        &mut self,
+        src: &mut Src,
+        reg_file: RegFile,
+        src_type: SrcType,
+    ) {
+        if !src_is_reg(src, reg_file) {
+            self.copy_alu_src(src, reg_file, src_type);
+        }
+    }
+
+    fn copy_alu_src_if_not_reg_or_imm(
+        &mut self,
+        src: &mut Src,
+        reg_file: RegFile,
+        src_type: SrcType,
+    ) {
+        if !src_is_reg(src, reg_file)
+            && !matches!(&src.src_ref, SrcRef::Imm32(_))
+        {
+            self.copy_alu_src(src, reg_file, src_type);
+        }
+    }
+
+    fn copy_alu_src_if_imm(
+        &mut self,
+        src: &mut Src,
+        reg_file: RegFile,
+        src_type: SrcType,
+    ) {
+        if src_is_imm(src) {
+            self.copy_alu_src(src, reg_file, src_type);
+        }
+    }
+
+    fn copy_alu_src_if_ineg_imm(
+        &mut self,
+        src: &mut Src,
+        reg_file: RegFile,
+        src_type: SrcType,
+    ) {
+        assert!(src_type == SrcType::I32);
+        if src_is_imm(src) && src.src_mod.is_ineg() {
+            self.copy_alu_src(src, reg_file, src_type);
+        }
+    }
+
+    fn copy_alu_src_if_both_not_reg(
+        &mut self,
+        src1: &Src,
+        src2: &mut Src,
+        reg_file: RegFile,
+        src_type: SrcType,
+    ) {
+        if !src_is_reg(src1, reg_file) && !src_is_reg(src2, reg_file) {
+            self.copy_alu_src(src2, reg_file, src_type);
+        }
+    }
+
+    fn copy_alu_src_and_lower_fmod(
+        &mut self,
+        src: &mut Src,
+        reg_file: RegFile,
+        src_type: SrcType,
+    ) {
+        match src_type {
+            SrcType::F16 | SrcType::F16v2 => {
+                let val = self.alloc_ssa(reg_file);
+                let old_src = std::mem::replace(src, val.into());
+                self.push_op(OpHAdd2 {
+                    dst: val.into(),
+                    srcs: [Src::ZERO.fneg(), old_src],
+                    saturate: false,
+                    ftz: false,
+                    f32: false,
+                });
+            }
+            SrcType::F32 => {
+                let val = self.alloc_ssa(reg_file);
+                let old_src = std::mem::replace(src, val.into());
+                self.push_op(OpFAdd {
+                    dst: val.into(),
+                    srcs: [Src::ZERO.fneg(), old_src],
+                    saturate: false,
+                    rnd_mode: FRndMode::NearestEven,
+                    ftz: false,
+                });
+            }
+            SrcType::F64 => {
+                let val = self.alloc_ssa_vec(reg_file, 2);
+                let old_src = std::mem::replace(src, val.clone().into());
+                self.push_op(OpDAdd {
+                    dst: val.into(),
+                    srcs: [Src::ZERO.fneg(), old_src],
+                    rnd_mode: FRndMode::NearestEven,
+                });
+            }
+            _ => panic!("Invalid ffabs srouce type"),
+        }
+    }
+
+    fn copy_alu_src_and_lower_ineg(
+        &mut self,
+        src: &mut Src,
+        reg_file: RegFile,
+        src_type: SrcType,
+    ) {
+        assert!(src_type == SrcType::I32);
+        let val = self.alloc_ssa(reg_file);
+        let old_src = std::mem::replace(src, val.into());
+        if self.sm() >= 70 {
+            self.push_op(OpIAdd3 {
+                srcs: [Src::ZERO, old_src, Src::ZERO],
+                overflow: [Dst::None, Dst::None],
+                dst: val.into(),
+            });
+        } else {
+            self.push_op(OpIAdd2 {
+                dst: val.into(),
+                carry_out: Dst::None,
+                srcs: [Src::ZERO, old_src],
+            });
+        }
+    }
+
+    fn copy_alu_src_if_fabs(
+        &mut self,
+        src: &mut Src,
+        reg_file: RegFile,
+        src_type: SrcType,
+    ) {
+        if src.src_mod.has_fabs() {
+            self.copy_alu_src_and_lower_fmod(src, reg_file, src_type);
+        }
+    }
+
+    fn copy_alu_src_if_i20_overflow(
+        &mut self,
+        src: &mut Src,
+        reg_file: RegFile,
+        src_type: SrcType,
+    ) {
+        if src.as_imm_not_i20().is_some() {
+            self.copy_alu_src(src, reg_file, src_type);
+        }
+    }
+
+    fn copy_alu_src_if_f20_overflow(
+        &mut self,
+        src: &mut Src,
+        reg_file: RegFile,
+        src_type: SrcType,
+    ) {
+        if src.as_imm_not_f20().is_some() {
+            self.copy_alu_src(src, reg_file, src_type);
+        }
+    }
+
+    fn copy_ssa_ref_if_uniform(&mut self, ssa_ref: &mut SSARef) {
+        for ssa in &mut ssa_ref[..] {
+            if ssa.is_uniform() {
+                let warp = self.alloc_ssa(ssa.file().to_warp());
+                self.copy_to(warp.into(), (*ssa).into());
+                *ssa = warp;
+            }
+        }
+    }
+
+    fn copy_src_if_uniform(&mut self, src: &mut Src) {
+        match &mut src.src_ref {
+            SrcRef::SSA(ssa) => self.copy_ssa_ref_if_uniform(ssa),
+            SrcRef::Imm32(_) | SrcRef::False | SrcRef::True | SrcRef::Zero => {}
+            _ => panic!("Unsupported source reference"),
+        }
+    }
+}
+
+pub struct LegalizeBuilder<'a> {
+    b: SSAInstrBuilder<'a>,
+    const_tracker: &'a mut ConstTracker,
+}
+
+impl<'a> LegalizeBuilder<'a> {
+    fn new(
+        sm: &'a ShaderModelInfo,
+        alloc: &'a mut SSAValueAllocator,
+        const_tracker: &'a mut ConstTracker,
+    ) -> Self {
+        LegalizeBuilder {
+            b: SSAInstrBuilder::new(sm, alloc),
+            const_tracker,
+        }
+    }
+
+    pub fn into_vec(self) -> Vec<Instr> {
+        self.b.into_vec()
+    }
+
+    #[allow(dead_code)]
+    pub fn into_mapped_instrs(self) -> MappedInstrs {
+        self.b.into_mapped_instrs()
+    }
+}
+
+impl<'a> Builder for LegalizeBuilder<'a> {
+    fn push_instr(&mut self, instr: Instr) -> &mut Instr {
+        self.b.push_instr(instr)
+    }
+
+    fn sm(&self) -> u8 {
+        self.b.sm()
+    }
+
+    fn copy_to(&mut self, dst: Dst, mut src: Src) {
+        if let Some(ssa_ref) = src.as_ssa() {
+            if let &[ssa_value] = &ssa_ref[..] {
+                if let Some(new_src) = self.const_tracker.get(&ssa_value) {
+                    src = new_src.clone().into();
+                }
+            }
+        };
+        self.b.copy_to(dst, src);
+    }
+}
+
+impl<'a> SSABuilder for LegalizeBuilder<'a> {
+    fn alloc_ssa(&mut self, file: RegFile) -> SSAValue {
+        self.b.alloc_ssa(file)
+    }
+
+    fn alloc_ssa_vec(&mut self, file: RegFile, comps: u8) -> SSARef {
+        self.b.alloc_ssa_vec(file, comps)
+    }
+}
+
+impl LegalizeBuildHelpers for LegalizeBuilder<'_> {}
+
+fn legalize_instr(
+    sm: &ShaderModelInfo,
+    b: &mut LegalizeBuilder,
+    bl: &impl BlockLiveness,
+    block_uniform: bool,
+    pinned: &FxHashSet<SSARef>,
+    ip: usize,
+    instr: &mut Instr,
+) {
+    // Handle a few no-op cases up-front
+    match &instr.op {
+        Op::Annotate(_) => {
+            // OpAnnotate does nothing.  There's nothing to legalize.
+            return;
+        }
+        Op::Undef(_)
+        | Op::PhiSrcs(_)
+        | Op::PhiDsts(_)
+        | Op::Pin(_)
+        | Op::Unpin(_)
+        | Op::RegOut(_) => {
+            // These are implemented by RA and can take pretty much anything
+            // you can throw at them.
+            debug_assert!(instr.pred.is_true());
+            return;
+        }
+        Op::Copy(_) => {
+            // OpCopy is implemented in a lowering pass and can handle anything
+            return;
+        }
+        Op::SrcBar(_) => {
+            // This is turned into a nop by calc_instr_deps
+            return;
+        }
+        Op::Swap(_) | Op::ParCopy(_) => {
+            // These are generated by RA and should not exist yet
+            panic!("Unsupported instruction");
+        }
+        _ => (),
+    }
+
+    if !instr.is_uniform() {
+        b.copy_pred_if_upred(&mut instr.pred);
+    }
+
+    let src_types = instr.src_types();
+    for (i, src) in instr.srcs_mut().iter_mut().enumerate() {
+        if matches!(src.src_ref, SrcRef::Imm32(_)) {
+            // Fold modifiers on Imm32 sources whenever possible.  Not all
+            // instructions suppport modifiers and immediates at the same time.
+            // But leave Zero sources alone as we don't want to make things
+            // immediates that could just be rZ.
+            if let Some(u) = src.as_u32(src_types[i]) {
+                *src = u.into();
+            }
+        }
+        b.copy_src_if_not_same_file(src);
+
+        if !block_uniform {
+            // In non-uniform control-flow, we can't collect uniform vectors so
+            // we need to insert copies to warp regs which we can collect.
+            match &mut src.src_ref {
+                SrcRef::SSA(vec) => {
+                    if vec.is_uniform()
+                        && vec.comps() > 1
+                        && !pinned.contains(vec)
+                    {
+                        b.copy_ssa_ref(vec, vec.file().to_warp());
+                    }
+                }
+                SrcRef::CBuf(CBufRef {
+                    buf: CBuf::BindlessSSA(handle),
+                    ..
+                }) => assert!(pinned.contains(&SSARef::new(handle))),
+                _ => (),
+            }
+        }
+    }
+
+    // OpBreak and OpBSsy impose additional RA constraints
+    let mut legalize_break_bssy = |bar_in: &mut Src, bar_out: &mut Dst| {
+        let bar_in_ssa = bar_in.src_ref.as_ssa().unwrap();
+        if !bar_out.is_none() && bl.is_live_after_ip(&bar_in_ssa[0], ip) {
+            let gpr = b.bmov_to_gpr(bar_in.clone());
+            let tmp = b.bmov_to_bar(gpr.into());
+            *bar_in = tmp.into();
+        }
+    };
+    match &mut instr.op {
+        Op::Break(op) => legalize_break_bssy(&mut op.bar_in, &mut op.bar_out),
+        Op::BSSy(op) => legalize_break_bssy(&mut op.bar_in, &mut op.bar_out),
+        _ => (),
+    }
+
+    sm.legalize_op(b, &mut instr.op);
+
+    let mut vec_src_map: FxHashMap<SSARef, SSARef> = Default::default();
+    let mut vec_comps: FxHashSet<_> = Default::default();
+    for src in instr.srcs_mut() {
+        if let SrcRef::SSA(vec) = &src.src_ref {
+            if vec.comps() == 1 {
+                continue;
+            }
+
+            // If the same vector shows up twice in one instruction, that's
+            // okay. Just make it look the same as the previous source we
+            // fixed up.
+            if let Some(new_vec) = vec_src_map.get(vec) {
+                src.src_ref = new_vec.clone().into();
+                continue;
+            }
+
+            let mut new_vec = vec.clone();
+            for c in 0..vec.comps() {
+                let ssa = vec[usize::from(c)];
+                // If the same SSA value shows up in multiple non-identical
+                // vector sources or as multiple components in the same
+                // source, we need to make a copy so it can get assigned to
+                // multiple different registers.
+                if vec_comps.contains(&ssa) {
+                    let copy = b.alloc_ssa(ssa.file());
+                    b.copy_to(copy.into(), ssa.into());
+                    new_vec[usize::from(c)] = copy;
+                } else {
+                    vec_comps.insert(ssa);
+                }
+            }
+
+            vec_src_map.insert(vec.clone(), new_vec.clone());
+            src.src_ref = new_vec.into();
+        }
+    }
+}
+
+impl Shader<'_> {
+    pub fn legalize(&mut self) {
+        let sm = self.sm;
+        for f in &mut self.functions {
+            let live = SimpleLiveness::for_function(f);
+            let mut pinned: FxHashSet<_> = Default::default();
+            let mut const_tracker = ConstTracker::new();
+
+            for (bi, b) in f.blocks.iter_mut().enumerate() {
+                let bl = live.block_live(bi);
+                let bu = b.uniform;
+
+                let mut instrs = Vec::new();
+                for (ip, mut instr) in b.instrs.drain(..).enumerate() {
+                    match &instr.op {
+                        Op::Pin(pin) => {
+                            if let Dst::SSA(ssa) = &pin.dst {
+                                pinned.insert(ssa.clone());
+                            }
+                        }
+                        Op::Copy(copy) => {
+                            const_tracker.add_copy(copy);
+                        }
+                        _ => (),
+                    }
+
+                    let mut b = LegalizeBuilder::new(
+                        sm,
+                        &mut f.ssa_alloc,
+                        &mut const_tracker,
+                    );
+                    legalize_instr(sm, &mut b, bl, bu, &pinned, ip, &mut instr);
+                    b.push_instr(instr);
+                    instrs.append(&mut b.into_vec());
+                }
+                b.instrs = instrs;
+            }
+        }
+    }
+}
@@ -0,0 +1,52 @@
+// Copyright © 2023 Collabora, Ltd.
+// SPDX-License-Identifier: MIT
+
+mod api;
+mod assign_regs;
+mod builder;
+mod calc_instr_deps;
+mod const_tracker;
+mod from_nir;
+mod ir;
+mod legalize;
+mod liveness;
+mod lower_copy_swap;
+mod lower_par_copies;
+mod opt_bar_prop;
+mod opt_copy_prop;
+mod opt_crs;
+mod opt_dce;
+mod opt_instr_sched_common;
+mod opt_instr_sched_postpass;
+mod opt_instr_sched_prepass;
+mod opt_jump_thread;
+mod opt_lop;
+mod opt_out;
+mod opt_prmt;
+mod opt_uniform_instrs;
+mod qmd;
+mod reg_tracker;
+mod repair_ssa;
+mod sm120_instr_latencies;
+mod sm20;
+mod sm30_instr_latencies;
+mod sm32;
+mod sm50;
+mod sm70;
+mod sm70_encode;
+mod sm75_instr_latencies;
+mod sm80_instr_latencies;
+mod sph;
+mod spill_values;
+mod ssa_value;
+mod to_cssa;
+mod union_find;
+
+#[cfg(test)]
+mod hw_tests;
+
+#[cfg(test)]
+mod hw_runner;
+
+#[cfg(test)]
+mod nvdisasm_tests;
@@ -0,0 +1,606 @@
+// Copyright © 2022 Collabora, Ltd.
+// SPDX-License-Identifier: MIT
+
+use crate::ir::*;
+
+use compiler::bitset::BitSet;
+use compiler::dataflow::BackwardDataflow;
+use rustc_hash::{FxHashMap, FxHashSet};
+use std::cmp::{max, min, Ord, Ordering};
+
+#[derive(Clone, Default)]
+pub struct LiveSet {
+    live: PerRegFile<u32>,
+    set: FxHashSet<SSAValue>,
+}
+
+impl LiveSet {
+    pub fn new() -> LiveSet {
+        Default::default()
+    }
+
+    pub fn clear(&mut self) {
+        self.live = Default::default();
+        self.set.clear();
+    }
+
+    pub fn contains(&self, ssa: &SSAValue) -> bool {
+        self.set.contains(ssa)
+    }
+
+    pub fn count(&self, file: RegFile) -> u32 {
+        self.live[file]
+    }
+
+    pub fn insert(&mut self, ssa: SSAValue) -> bool {
+        if self.set.insert(ssa) {
+            self.live[ssa.file()] += 1;
+            true
+        } else {
+            false
+        }
+    }
+
+    pub fn iter(&self) -> impl Iterator<Item = &SSAValue> {
+        self.set.iter()
+    }
+
+    pub fn remove(&mut self, ssa: &SSAValue) -> bool {
+        if self.set.remove(ssa) {
+            self.live[ssa.file()] -= 1;
+            true
+        } else {
+            false
+        }
+    }
+
+    pub fn insert_instr_top_down<L: BlockLiveness>(
+        &mut self,
+        ip: usize,
+        instr: &Instr,
+        bl: &L,
+    ) -> PerRegFile<u32> {
+        // Vector destinations go live before sources are killed.  Even
+        // in the case where the destination is immediately killed, it
+        // still may contribute to pressure temporarily.
+        for dst in instr.dsts() {
+            if let Dst::SSA(vec) = dst {
+                if vec.comps() > 1 {
+                    for ssa in vec.iter() {
+                        self.insert(*ssa);
+                    }
+                }
+            }
+        }
+
+        let after_dsts_live = self.live;
+
+        instr.for_each_ssa_use(|ssa| {
+            if !bl.is_live_after_ip(ssa, ip) {
+                self.remove(ssa);
+            }
+        });
+
+        // Scalar destinations are allocated last
+        for dst in instr.dsts() {
+            if let Dst::SSA(vec) = dst {
+                if vec.comps() == 1 {
+                    self.insert(vec[0]);
+                }
+            }
+        }
+
+        let max_live = PerRegFile::new_with(|file| {
+            max(self.live[file], after_dsts_live[file])
+        });
+
+        // It's possible (but unlikely) that a destination is immediately
+        // killed. Remove any which are killed by this instruction.
+        instr.for_each_ssa_def(|ssa| {
+            debug_assert!(self.contains(ssa));
+            if !bl.is_live_after_ip(ssa, ip) {
+                self.remove(ssa);
+            }
+        });
+
+        max_live
+    }
+}
+
+impl FromIterator<SSAValue> for LiveSet {
+    fn from_iter<T: IntoIterator<Item = SSAValue>>(iter: T) -> Self {
+        let mut set = LiveSet::new();
+        let iter = iter.into_iter();
+        set.set.reserve(iter.size_hint().0);
+        for ssa in iter {
+            set.insert(ssa);
+        }
+        set
+    }
+}
+
+impl Extend<SSAValue> for LiveSet {
+    fn extend<T: IntoIterator<Item = SSAValue>>(&mut self, iter: T) {
+        let iter = iter.into_iter();
+        self.set.reserve(iter.size_hint().0);
+        for ssa in iter {
+            self.insert(ssa);
+        }
+    }
+}
+
+pub trait BlockLiveness {
+    /// Returns true if @val is still live after @ip
+    fn is_live_after_ip(&self, val: &SSAValue, ip: usize) -> bool;
+
+    /// Returns true if @val is live-in to this block
+    fn is_live_in(&self, val: &SSAValue) -> bool;
+
+    /// Returns true if @val is live-out of this block
+    fn is_live_out(&self, val: &SSAValue) -> bool;
+
+    fn get_instr_pressure(&self, ip: usize, instr: &Instr) -> PerRegFile<u8> {
+        let mut live = PerRegFile::new_with(|_| 0_i8);
+
+        // Vector destinations go live before sources are killed.
+        for dst in instr.dsts() {
+            if let Dst::SSA(vec) = dst {
+                if vec.comps() > 1 {
+                    for ssa in vec.iter() {
+                        live[ssa.file()] += 1;
+                    }
+                }
+            }
+        }
+
+        // This is the first high point
+        let vec_dst_live = live;
+
+        // Use a hash set because sources may occur more than once
+        let mut killed: FxHashSet<_> = Default::default();
+        instr.for_each_ssa_use(|ssa| {
+            if !self.is_live_after_ip(ssa, ip) {
+                killed.insert(*ssa);
+            }
+        });
+        for ssa in killed.drain() {
+            live[ssa.file()] -= 1;
+        }
+
+        // Scalar destinations are allocated last
+        for dst in instr.dsts() {
+            if let Dst::SSA(vec) = dst {
+                if vec.comps() == 1 {
+                    live[vec[0].file()] += 1;
+                }
+            }
+        }
+
+        PerRegFile::new_with(|file| {
+            max(0, max(vec_dst_live[file], live[file]))
+                .try_into()
+                .unwrap()
+        })
+    }
+}
+
+pub trait Liveness {
+    type PerBlock: BlockLiveness;
+
+    fn block_live(&self, idx: usize) -> &Self::PerBlock;
+
+    fn calc_max_live(&self, f: &Function) -> PerRegFile<u32> {
+        let mut max_live: PerRegFile<u32> = Default::default();
+        let mut block_live_out: Vec<LiveSet> = Vec::new();
+
+        for (bb_idx, bb) in f.blocks.iter().enumerate() {
+            let bl = self.block_live(bb_idx);
+
+            let mut live = LiveSet::new();
+
+            // Predecessors are added block order so we can just grab the first
+            // one (if any) and it will be a block we've processed.
+            if let Some(pred_idx) = f.blocks.pred_indices(bb_idx).first() {
+                let pred_out = &block_live_out[*pred_idx];
+                for ssa in pred_out.iter() {
+                    if bl.is_live_in(ssa) {
+                        live.insert(*ssa);
+                    }
+                }
+            }
+
+            for (ip, instr) in bb.instrs.iter().enumerate() {
+                let live_at_instr = live.insert_instr_top_down(ip, instr, bl);
+                max_live = PerRegFile::new_with(|file| {
+                    max(max_live[file], live_at_instr[file])
+                });
+
+                if let Op::RegOut(reg_out) = &instr.op {
+                    // This should be the last instruction.  Everything should
+                    // be dead once we've processed it.
+                    debug_assert!(live.count(RegFile::GPR) == 0);
+                    let num_gprs_out = reg_out.srcs.len().try_into().unwrap();
+                    max_live[RegFile::GPR] =
+                        max(max_live[RegFile::GPR], num_gprs_out);
+                }
+            }
+
+            assert!(block_live_out.len() == bb_idx);
+            block_live_out.push(live);
+        }
+
+        max_live
+    }
+}
+
+#[derive(Default)]
+pub struct SimpleBlockLiveness {
+    defs: BitSet<SSAValue>,
+    uses: BitSet<SSAValue>,
+    last_use: FxHashMap<SSAValue, usize>,
+    live_in: BitSet<SSAValue>,
+    live_out: BitSet<SSAValue>,
+}
+
+impl SimpleBlockLiveness {
+    fn new() -> Self {
+        Default::default()
+    }
+
+    fn add_def(&mut self, ssa: SSAValue) {
+        self.defs.insert(ssa);
+    }
+
+    fn add_use(&mut self, ssa: SSAValue, ip: usize) {
+        self.uses.insert(ssa);
+        self.last_use.insert(ssa, ip);
+    }
+}
+
+impl BlockLiveness for SimpleBlockLiveness {
+    fn is_live_after_ip(&self, val: &SSAValue, ip: usize) -> bool {
+        if self.live_out.contains(*val) {
+            true
+        } else if let Some(last_use_ip) = self.last_use.get(val) {
+            *last_use_ip > ip
+        } else {
+            false
+        }
+    }
+
+    fn is_live_in(&self, val: &SSAValue) -> bool {
+        self.live_in.contains(*val)
+    }
+
+    fn is_live_out(&self, val: &SSAValue) -> bool {
+        self.live_out.contains(*val)
+    }
+}
+
+pub struct SimpleLiveness {
+    ssa_block_ip: FxHashMap<SSAValue, (usize, usize)>,
+    blocks: Vec<SimpleBlockLiveness>,
+}
+
+impl SimpleLiveness {
+    pub fn for_function(func: &Function) -> SimpleLiveness {
+        let mut l = SimpleLiveness {
+            ssa_block_ip: Default::default(),
+            blocks: Vec::new(),
+        };
+
+        for (bi, b) in func.blocks.iter().enumerate() {
+            let mut bl = SimpleBlockLiveness::new();
+
+            for (ip, instr) in b.instrs.iter().enumerate() {
+                instr.for_each_ssa_use(|ssa| {
+                    bl.add_use(*ssa, ip);
+                });
+                instr.for_each_ssa_def(|ssa| {
+                    l.ssa_block_ip.insert(*ssa, (bi, ip));
+                    bl.add_def(*ssa);
+                });
+            }
+
+            l.blocks.push(bl);
+        }
+        assert!(l.blocks.len() == func.blocks.len());
+
+        let mut live_in: Vec<BitSet<SSAValue>> =
+            (0..func.blocks.len()).map(|_| Default::default()).collect();
+        let mut live_out: Vec<BitSet<SSAValue>> =
+            (0..func.blocks.len()).map(|_| Default::default()).collect();
+        BackwardDataflow {
+            cfg: &func.blocks,
+            block_in: &mut live_in[..],
+            block_out: &mut live_out[..],
+            transfer: |block_idx, _, live_in, live_out| {
+                let bl = &l.blocks[block_idx];
+                live_in.union_with(
+                    (live_out.s(..) | bl.uses.s(..)) - bl.defs.s(..),
+                )
+            },
+            join: |live_out, succ_live_in| {
+                *live_out |= succ_live_in.s(..);
+            },
+        }
+        .solve();
+
+        for ((bl, b_live_in), b_live_out) in l
+            .blocks
+            .iter_mut()
+            .zip(live_in.into_iter())
+            .zip(live_out.into_iter())
+        {
+            bl.live_in = b_live_in;
+            bl.live_out = b_live_out;
+        }
+
+        l
+    }
+}
+
+impl SimpleLiveness {
+    pub fn def_block_ip(&self, ssa: &SSAValue) -> (usize, usize) {
+        *self.ssa_block_ip.get(ssa).unwrap()
+    }
+
+    pub fn interferes(&self, a: &SSAValue, b: &SSAValue) -> bool {
+        let (ab, ai) = self.def_block_ip(a);
+        let (bb, bi) = self.def_block_ip(b);
+
+        match ab.cmp(&bb).then(ai.cmp(&bi)) {
+            Ordering::Equal => true,
+            Ordering::Less => self.block_live(bb).is_live_after_ip(a, bi),
+            Ordering::Greater => self.block_live(ab).is_live_after_ip(b, ai),
+        }
+    }
+}
+
+impl Liveness for SimpleLiveness {
+    type PerBlock = SimpleBlockLiveness;
+
+    fn block_live(&self, idx: usize) -> &SimpleBlockLiveness {
+        &self.blocks[idx]
+    }
+}
+
+struct SSAUseDef {
+    defined: bool,
+    uses: Vec<usize>,
+}
+
+impl SSAUseDef {
+    fn add_def(&mut self) {
+        self.defined = true;
+    }
+
+    fn add_in_block_use(&mut self, use_ip: usize) {
+        self.uses.push(use_ip);
+    }
+
+    fn add_successor_use(
+        &mut self,
+        num_block_instrs: usize,
+        use_ip: usize,
+    ) -> bool {
+        // IPs are relative to the start of their block
+        let use_ip = num_block_instrs + use_ip;
+
+        if let Some(last_use_ip) = self.uses.last_mut() {
+            if *last_use_ip < num_block_instrs {
+                // We've never seen a successor use before
+                self.uses.push(use_ip);
+                true
+            } else if *last_use_ip > use_ip {
+                // Otherwise, we want the minimum next use
+                *last_use_ip = use_ip;
+                true
+            } else {
+                false
+            }
+        } else {
+            self.uses.push(use_ip);
+            true
+        }
+    }
+}
+
+pub struct NextUseBlockLiveness {
+    num_instrs: usize,
+    ssa_map: FxHashMap<SSAValue, SSAUseDef>,
+}
+
+impl NextUseBlockLiveness {
+    fn new(num_instrs: usize) -> Self {
+        Self {
+            num_instrs: num_instrs,
+            ssa_map: Default::default(),
+        }
+    }
+
+    fn entry_mut(&mut self, ssa: SSAValue) -> &mut SSAUseDef {
+        self.ssa_map.entry(ssa).or_insert_with(|| SSAUseDef {
+            defined: false,
+            uses: Vec::new(),
+        })
+    }
+
+    fn add_def(&mut self, ssa: SSAValue) {
+        self.entry_mut(ssa).add_def();
+    }
+
+    fn add_use(&mut self, ssa: SSAValue, ip: usize) {
+        self.entry_mut(ssa).add_in_block_use(ip);
+    }
+
+    /// Returns an iterator over all the values which are live-in to this block
+    pub fn iter_live_in(&self) -> impl Iterator<Item = &SSAValue> {
+        self.ssa_map.iter().filter_map(|(ssa, entry)| {
+            if entry.defined || entry.uses.is_empty() {
+                None
+            } else {
+                Some(ssa)
+            }
+        })
+    }
+
+    /// Returns the IP of the first use of @val
+    ///
+    /// The returned IP is relative to the start of this block.  If the next use
+    /// is in some successor block, the returned IP is relative to the start of
+    /// this block.  If @val is not used in this block and is not live-out, None
+    /// is returned.
+    pub fn first_use(&self, val: &SSAValue) -> Option<usize> {
+        if let Some(entry) = self.ssa_map.get(val) {
+            entry.uses.first().cloned()
+        } else {
+            None
+        }
+    }
+
+    /// Returns the IP of the first use of @val which is greater than or equal
+    /// to @ip
+    ///
+    /// All IPs are relative to the start of the block.  If the next use is some
+    /// successor block, the returned IP is relative to the start of this block.
+    pub fn next_use_after_or_at_ip(
+        &self,
+        val: &SSAValue,
+        ip: usize,
+    ) -> Option<usize> {
+        if let Some(entry) = self.ssa_map.get(val) {
+            let i = entry.uses.partition_point(|u| *u < ip);
+            if i < entry.uses.len() {
+                Some(entry.uses[i])
+            } else {
+                None
+            }
+        } else {
+            None
+        }
+    }
+}
+
+impl BlockLiveness for NextUseBlockLiveness {
+    fn is_live_after_ip(&self, val: &SSAValue, ip: usize) -> bool {
+        if let Some(entry) = self.ssa_map.get(val) {
+            if let Some(last_use_ip) = entry.uses.last() {
+                *last_use_ip > ip
+            } else {
+                false
+            }
+        } else {
+            false
+        }
+    }
+
+    fn is_live_in(&self, val: &SSAValue) -> bool {
+        if let Some(entry) = self.ssa_map.get(val) {
+            !entry.defined && !entry.uses.is_empty()
+        } else {
+            false
+        }
+    }
+
+    fn is_live_out(&self, val: &SSAValue) -> bool {
+        if let Some(entry) = self.ssa_map.get(val) {
+            if let Some(last_use_ip) = entry.uses.last() {
+                *last_use_ip >= self.num_instrs
+            } else {
+                false
+            }
+        } else {
+            false
+        }
+    }
+}
+
+/// An implementation of Liveness that tracks next-use IPs for each SSAValue
+///
+/// Along with the usual liveness information, this tracks next-use IPs for each
+/// SSAValue.  Cross-block next-use IPs computed are as per the global next-use
+/// distance algorithm described in "Register Spilling and Live-Range Splitting
+/// for SSA-Form Programs" by Braun and Hack.
+pub struct NextUseLiveness {
+    blocks: Vec<NextUseBlockLiveness>,
+}
+
+impl NextUseLiveness {
+    pub fn for_function(
+        func: &Function,
+        files: &RegFileSet,
+    ) -> NextUseLiveness {
+        let mut blocks = Vec::new();
+        for (bi, b) in func.blocks.iter().enumerate() {
+            let mut bl = NextUseBlockLiveness::new(b.instrs.len());
+
+            for (ip, instr) in b.instrs.iter().enumerate() {
+                instr.for_each_ssa_use(|ssa| {
+                    if files.contains(ssa.file()) {
+                        bl.add_use(*ssa, ip);
+                    }
+                });
+
+                instr.for_each_ssa_def(|ssa| {
+                    if files.contains(ssa.file()) {
+                        bl.add_def(*ssa);
+                    }
+                });
+            }
+
+            debug_assert!(bi == blocks.len());
+            blocks.push(bl);
+        }
+
+        let mut live_out: Vec<FxHashMap<SSAValue, usize>> =
+            (0..func.blocks.len()).map(|_| Default::default()).collect();
+        BackwardDataflow {
+            cfg: &func.blocks,
+            block_in: &mut blocks[..],
+            block_out: &mut live_out[..],
+            transfer: |_block_idx, _block, live_in, live_out| {
+                let num_instrs = live_in.num_instrs;
+                let mut changed = false;
+
+                for (&ssa, &first_use_ip) in live_out.iter() {
+                    changed |= live_in
+                        .entry_mut(ssa)
+                        .add_successor_use(num_instrs, first_use_ip);
+                }
+                changed
+            },
+            join: |live_out, succ_live_in| {
+                if live_out.capacity() == 0 {
+                    live_out.reserve(succ_live_in.ssa_map.len());
+                }
+
+                for (&ssa, entry) in succ_live_in.ssa_map.iter() {
+                    if entry.defined {
+                        continue;
+                    }
+
+                    let Some(&first_use_ip) = entry.uses.first() else {
+                        continue;
+                    };
+
+                    live_out
+                        .entry(ssa)
+                        .and_modify(|val| *val = min(*val, first_use_ip))
+                        .or_insert(first_use_ip);
+                }
+            },
+        }
+        .solve();
+
+        NextUseLiveness { blocks }
+    }
+}
+
+impl Liveness for NextUseLiveness {
+    type PerBlock = NextUseBlockLiveness;
+
+    fn block_live(&self, idx: usize) -> &NextUseBlockLiveness {
+        &self.blocks[idx]
+    }
+}
@@ -0,0 +1,307 @@
+// Copyright © 2022 Collabora, Ltd.
+// SPDX-License-Identifier: MIT
+
+use crate::{
+    api::{GetDebugFlags, DEBUG},
+    ir::*,
+};
+
+use std::cmp::max;
+
+struct LowerCopySwap {
+    slm_start: u32,
+    slm_size: u32,
+}
+
+impl LowerCopySwap {
+    fn new(slm_size: u32) -> Self {
+        Self {
+            slm_start: slm_size,
+            slm_size: slm_size,
+        }
+    }
+
+    fn lower_copy(&mut self, b: &mut impl Builder, copy: OpCopy) {
+        let dst_reg = copy.dst.as_reg().unwrap();
+        assert!(dst_reg.comps() == 1);
+        assert!(copy.src.is_unmodified());
+        assert!(copy.src.is_uniform() || !dst_reg.is_uniform());
+
+        match dst_reg.file() {
+            RegFile::GPR | RegFile::UGPR => match copy.src.src_ref {
+                SrcRef::Zero | SrcRef::Imm32(_) => {
+                    b.push_op(OpMov {
+                        dst: copy.dst,
+                        src: copy.src,
+                        quad_lanes: 0xf,
+                    });
+                }
+                SrcRef::CBuf(_) => match dst_reg.file() {
+                    RegFile::GPR => {
+                        if b.sm() >= 100 {
+                            b.push_op(OpLdc {
+                                dst: copy.dst,
+                                cb: copy.src,
+                                offset: 0.into(),
+                                mode: LdcMode::Indexed,
+                                mem_type: MemType::B32,
+                            });
+                        } else {
+                            b.push_op(OpMov {
+                                dst: copy.dst,
+                                src: copy.src,
+                                quad_lanes: 0xf,
+                            });
+                        }
+                    }
+                    RegFile::UGPR => {
+                        b.push_op(OpLdc {
+                            dst: copy.dst,
+                            cb: copy.src,
+                            offset: 0.into(),
+                            mode: LdcMode::Indexed,
+                            mem_type: MemType::B32,
+                        });
+                    }
+                    _ => panic!("Invalid cbuf destination"),
+                },
+                SrcRef::True | SrcRef::False => {
+                    panic!("Cannot copy to GPR");
+                }
+                SrcRef::Reg(src_reg) => match src_reg.file() {
+                    RegFile::GPR | RegFile::UGPR => {
+                        b.push_op(OpMov {
+                            dst: copy.dst,
+                            src: copy.src,
+                            quad_lanes: 0xf,
+                        });
+                    }
+                    RegFile::Bar => {
+                        b.push_op(OpBMov {
+                            dst: copy.dst,
+                            src: copy.src,
+                            clear: false,
+                        });
+                    }
+                    RegFile::Mem => {
+                        let access = MemAccess {
+                            mem_type: MemType::B32,
+                            space: MemSpace::Local,
+                            order: MemOrder::Strong(MemScope::CTA),
+                            eviction_priority: MemEvictionPriority::Normal,
+                        };
+                        let addr = self.slm_start + src_reg.base_idx() * 4;
+                        self.slm_size = max(self.slm_size, addr + 4);
+                        b.push_op(OpLd {
+                            dst: copy.dst,
+                            addr: Src::ZERO,
+                            uniform_addr: Src::ZERO,
+                            pred: true.into(),
+                            offset: addr.try_into().unwrap(),
+                            stride: OffsetStride::X1,
+                            access: access,
+                        });
+                    }
+                    _ => panic!("Cannot copy to GPR"),
+                },
+                SrcRef::SSA(_) => panic!("Should be run after RA"),
+            },
+            RegFile::Pred | RegFile::UPred => match copy.src.src_ref {
+                SrcRef::Zero | SrcRef::Imm32(_) | SrcRef::CBuf(_) => {
+                    panic!("Cannot copy to Pred");
+                }
+                SrcRef::True => {
+                    b.lop2_to(
+                        copy.dst,
+                        LogicOp2::PassB,
+                        Src::new_imm_bool(true),
+                        Src::new_imm_bool(true),
+                    );
+                }
+                SrcRef::False => {
+                    b.lop2_to(
+                        copy.dst,
+                        LogicOp2::PassB,
+                        Src::new_imm_bool(true),
+                        Src::new_imm_bool(false),
+                    );
+                }
+                SrcRef::Reg(src_reg) => match src_reg.file() {
+                    RegFile::Pred => {
+                        b.lop2_to(
+                            copy.dst,
+                            LogicOp2::PassB,
+                            Src::new_imm_bool(true),
+                            copy.src,
+                        );
+                    }
+                    RegFile::UPred => {
+                        // PLOP3 supports a UPred in src[2]
+                        b.push_op(OpPLop3 {
+                            dsts: [copy.dst, Dst::None],
+                            srcs: [true.into(), true.into(), copy.src],
+                            ops: [
+                                LogicOp3::new_lut(&|_, _, z| z),
+                                LogicOp3::new_const(false),
+                            ],
+                        });
+                    }
+                    _ => panic!("Cannot copy to Pred"),
+                },
+                SrcRef::SSA(_) => panic!("Should be run after RA"),
+            },
+            RegFile::Bar => match copy.src.src_ref {
+                SrcRef::Reg(src_reg) => match src_reg.file() {
+                    RegFile::GPR | RegFile::UGPR => {
+                        b.push_op(OpBMov {
+                            dst: copy.dst,
+                            src: copy.src,
+                            clear: false,
+                        });
+                    }
+                    _ => panic!("Cannot copy to Bar"),
+                },
+                _ => panic!("Cannot copy to Bar"),
+            },
+            RegFile::Mem => match copy.src.src_ref {
+                SrcRef::Reg(src_reg) => match src_reg.file() {
+                    RegFile::GPR => {
+                        let access = MemAccess {
+                            mem_type: MemType::B32,
+                            space: MemSpace::Local,
+                            order: MemOrder::Strong(MemScope::CTA),
+                            eviction_priority: MemEvictionPriority::Normal,
+                        };
+                        let addr = self.slm_start + dst_reg.base_idx() * 4;
+                        self.slm_size = max(self.slm_size, addr + 4);
+                        b.push_op(OpSt {
+                            addr: Src::ZERO,
+                            uniform_addr: Src::ZERO,
+                            data: copy.src,
+                            offset: addr.try_into().unwrap(),
+                            stride: OffsetStride::X1,
+                            access: access,
+                        });
+                    }
+                    _ => panic!("Cannot copy to Mem"),
+                },
+                _ => panic!("Cannot copy to Mem"),
+            },
+            _ => panic!("Unhandled register file"),
+        }
+    }
+
+    fn lower_r2ur(&mut self, b: &mut impl Builder, r2ur: OpR2UR) {
+        assert!(r2ur.src.is_unmodified());
+        if r2ur.src.is_uniform() {
+            let copy = OpCopy {
+                dst: r2ur.dst,
+                src: r2ur.src,
+            };
+            self.lower_copy(b, copy);
+        } else {
+            let src_file = r2ur.src.src_ref.as_reg().unwrap().file();
+            let dst_file = r2ur.dst.as_reg().unwrap().file();
+            match src_file {
+                RegFile::GPR => {
+                    assert!(dst_file == RegFile::UGPR);
+                    b.push_op(r2ur);
+                }
+                RegFile::Pred => {
+                    assert!(dst_file == RegFile::UPred);
+                    // It doesn't matter what channel we take
+                    b.push_op(OpVote {
+                        op: VoteOp::Any,
+                        ballot: Dst::None,
+                        vote: r2ur.dst,
+                        pred: r2ur.src,
+                    });
+                }
+                _ => panic!("No matching uniform register file"),
+            }
+        }
+    }
+
+    fn lower_swap(&mut self, b: &mut impl Builder, swap: OpSwap) {
+        let x = *swap.dsts[0].as_reg().unwrap();
+        let y = *swap.dsts[1].as_reg().unwrap();
+
+        assert!(x.file() == y.file());
+        assert!(x.file() != RegFile::Mem);
+        assert!(x.comps() == 1 && y.comps() == 1);
+        assert!(swap.srcs[0].is_unmodified());
+        assert!(*swap.srcs[0].src_ref.as_reg().unwrap() == y);
+        assert!(swap.srcs[1].is_unmodified());
+        assert!(*swap.srcs[1].src_ref.as_reg().unwrap() == x);
+
+        if x == y {
+            // Nothing to do
+        } else if x.is_predicate() && b.sm() >= 70 {
+            b.push_op(OpPLop3 {
+                dsts: [x.into(), y.into()],
+                srcs: [x.into(), y.into(), Src::new_imm_bool(true)],
+                ops: [
+                    LogicOp3::new_lut(&|_, y, _| y),
+                    LogicOp3::new_lut(&|x, _, _| x),
+                ],
+            });
+        } else {
+            b.lop2_to(x.into(), LogicOp2::Xor, x.into(), y.into());
+            b.lop2_to(y.into(), LogicOp2::Xor, x.into(), y.into());
+            b.lop2_to(x.into(), LogicOp2::Xor, x.into(), y.into());
+        }
+    }
+
+    fn run(&mut self, s: &mut Shader) {
+        let sm = s.sm;
+        s.map_instrs(|instr: Instr, _| -> MappedInstrs {
+            match instr.op {
+                Op::R2UR(r2ur) => {
+                    debug_assert!(instr.pred.is_true());
+                    let mut b = InstrBuilder::new(sm);
+                    if DEBUG.annotate() {
+                        b.push_instr(Instr::new(OpAnnotate {
+                            annotation: "r2ur lowered by lower_copy_swap"
+                                .into(),
+                        }));
+                    }
+                    self.lower_r2ur(&mut b, *r2ur);
+                    b.into_mapped_instrs()
+                }
+                Op::Copy(copy) => {
+                    debug_assert!(instr.pred.is_true());
+                    let mut b = InstrBuilder::new(sm);
+                    if DEBUG.annotate() {
+                        b.push_instr(Instr::new(OpAnnotate {
+                            annotation: "copy lowered by lower_copy_swap"
+                                .into(),
+                        }));
+                    }
+                    self.lower_copy(&mut b, *copy);
+                    b.into_mapped_instrs()
+                }
+                Op::Swap(swap) => {
+                    debug_assert!(instr.pred.is_true());
+                    let mut b = InstrBuilder::new(sm);
+                    if DEBUG.annotate() {
+                        b.push_instr(Instr::new(OpAnnotate {
+                            annotation: "swap lowered by lower_copy_swap"
+                                .into(),
+                        }));
+                    }
+                    self.lower_swap(&mut b, *swap);
+                    b.into_mapped_instrs()
+                }
+                _ => [instr].into(),
+            }
+        });
+    }
+}
+
+impl Shader<'_> {
+    pub fn lower_copy_swap(&mut self) {
+        let mut pass = LowerCopySwap::new(self.info.slm_size);
+        pass.run(self);
+        self.info.slm_size = pass.slm_size;
+    }
+}
@@ -0,0 +1,275 @@
+// Copyright © 2022 Collabora, Ltd.
+// SPDX-License-Identifier: MIT
+
+use crate::{
+    api::{GetDebugFlags, DEBUG},
+    ir::*,
+};
+
+use rustc_hash::FxHashMap;
+
+struct CopyNode {
+    num_reads: usize,
+    src: isize,
+}
+
+struct CopyGraph {
+    nodes: Vec<CopyNode>,
+}
+
+impl CopyGraph {
+    pub fn new() -> CopyGraph {
+        CopyGraph { nodes: Vec::new() }
+    }
+
+    fn add_node(&mut self) -> usize {
+        let node_idx = self.nodes.len();
+        self.nodes.push(CopyNode {
+            num_reads: 0,
+            src: -1,
+        });
+        node_idx
+    }
+
+    fn num_reads(&self, node_idx: usize) -> usize {
+        self.nodes[node_idx].num_reads
+    }
+
+    fn src(&self, node_idx: usize) -> Option<usize> {
+        if self.nodes[node_idx].src < 0 {
+            None
+        } else {
+            Some(self.nodes[node_idx].src.try_into().unwrap())
+        }
+    }
+
+    fn add_edge(&mut self, dst_idx: usize, src_idx: usize) {
+        // Disallow self-loops
+        assert!(dst_idx != src_idx);
+
+        // Each node has in-degree at most 1
+        assert!(self.nodes[dst_idx].src == -1);
+        self.nodes[dst_idx].src = src_idx.try_into().unwrap();
+        self.nodes[src_idx].num_reads += 1;
+    }
+
+    fn del_edge(&mut self, dst_idx: usize, src_idx: usize) -> bool {
+        assert!(self.nodes[dst_idx].src >= 0);
+        self.nodes[dst_idx].src = -1;
+        self.nodes[src_idx].num_reads -= 1;
+        self.nodes[src_idx].num_reads == 0
+    }
+}
+
+fn copy_needs_tmp(dst: &RegRef, src: &SrcRef) -> bool {
+    if let Some(src_reg) = src.as_reg() {
+        (dst.file() == RegFile::Mem && src_reg.file() != RegFile::GPR)
+            || (dst.file() == RegFile::Bar && src_reg.file() == RegFile::Bar)
+    } else {
+        // Non-GPR to Mem copies need a temporary
+        dst.file() == RegFile::Mem
+    }
+}
+
+fn cycle_use_swap(pc: &OpParCopy, file: RegFile) -> bool {
+    match file {
+        RegFile::GPR => {
+            if let Some(tmp) = &pc.tmp {
+                debug_assert!(tmp.file() == RegFile::GPR);
+                false
+            } else {
+                true
+            }
+        }
+        RegFile::Bar | RegFile::Mem => {
+            let tmp = &pc.tmp.expect("This copy needs a temporary");
+            assert!(tmp.comps() >= 2, "Memory cycles need 2 temporaries");
+            false
+        }
+        _ => true,
+    }
+}
+
+fn lower_par_copy(pc: OpParCopy, sm: &ShaderModelInfo) -> MappedInstrs {
+    let mut graph = CopyGraph::new();
+    let mut vals = Vec::new();
+    let mut reg_to_idx = FxHashMap::default();
+
+    for (i, (dst, _)) in pc.dsts_srcs.iter().enumerate() {
+        // Destinations must be pairwise unique
+        let reg = dst.as_reg().unwrap();
+        assert!(!reg_to_idx.contains_key(reg));
+
+        // Everything must be scalar
+        assert!(reg.comps() == 1);
+
+        let node_idx = graph.add_node();
+        assert!(node_idx == i && vals.len() == i);
+        vals.push(SrcRef::from(*reg));
+        reg_to_idx.insert(*reg, i);
+    }
+
+    for (dst_idx, (_, src)) in pc.dsts_srcs.iter().enumerate() {
+        assert!(src.is_unmodified());
+        let src = &src.src_ref;
+
+        let src_idx = if let SrcRef::Reg(reg) = src {
+            // Everything must be scalar
+            assert!(reg.comps() == 1);
+
+            *reg_to_idx.entry(*reg).or_insert_with(|| {
+                let node_idx = graph.add_node();
+                assert!(node_idx == vals.len());
+                vals.push(src.clone());
+                node_idx
+            })
+        } else {
+            // We can't have bindless CBufs because we can't resolve cycles
+            // containing one.
+            assert!(src.get_reg().is_none());
+
+            let node_idx = graph.add_node();
+            assert!(node_idx == vals.len());
+            vals.push(src.clone());
+            node_idx
+        };
+
+        if dst_idx != src_idx {
+            graph.add_edge(dst_idx, src_idx);
+        }
+    }
+
+    let mut b = InstrBuilder::new(sm);
+
+    let mut ready = Vec::new();
+    for i in 0..pc.dsts_srcs.len() {
+        if graph.num_reads(i) == 0 {
+            ready.push(i);
+        }
+    }
+    while let Some(dst_idx) = ready.pop() {
+        if let Some(src_idx) = graph.src(dst_idx) {
+            let dst = *vals[dst_idx].as_reg().unwrap();
+            let src = vals[src_idx].clone();
+            if copy_needs_tmp(&dst, &src) {
+                let tmp = pc.tmp.expect("This copy needs a temporary").comp(0);
+                b.copy_to(tmp.into(), src.into());
+                b.copy_to(dst.into(), tmp.into());
+            } else {
+                b.copy_to(dst.into(), src.into());
+            }
+            if graph.del_edge(dst_idx, src_idx) {
+                ready.push(src_idx);
+            }
+        }
+    }
+
+    // At this point, all we are left with in the graph are isolated nodes
+    // (no edges) and cycles.
+    //
+    // Proof:
+    //
+    // Without loss of generality, we can assume that there are no isolated
+    // nodes in the graph.  By construction, no node has in-degree more than 1
+    // (that would indicate a duplicate destination).  The loop above ensures
+    // that there are no nodes with an in-degree of 1 and an out-degree of 0.
+    //
+    // Suppose that there were a node with an in-degree of 0.  Then, because no
+    // node has an in-degree greater than 1, the number of edges must be less
+    // than the number of nodes.  This implies that there is some node N with
+    // with out-degree of 0.  If N has an in-degree of 0 then it is isolated,
+    // which is a contradiction.  If N has an in-degree of 1 then it is a node
+    // with in-degree of 1 and out-degree of 0 which is also a contradiction.
+    // Therefore, there are no nodes with in-degree of 0 and all nodes have an
+    // in-degree of 1.
+    //
+    // Since all nodes have an in-degree of 1, no node has an out-degree of 0
+    // and, because the sum of all in-degrees equals the sum of all out-degrees
+    // (they both equal the number of edges), every node must also have an
+    // out-degree of 1.  Therefore, the graph only contains cycles.
+    //
+    // QED
+    for i in 0..pc.dsts_srcs.len() {
+        if graph.src(i).is_none() {
+            debug_assert!(graph.num_reads(i) == 0);
+            continue;
+        }
+
+        let file = vals[i].as_reg().unwrap().file();
+        if cycle_use_swap(&pc, file) {
+            loop {
+                let j = graph.src(i).unwrap();
+                // We're part of a cycle so j also has a source
+                let k = graph.src(j).unwrap();
+                let j_reg = vals[j].as_reg().unwrap();
+                let k_reg = vals[k].as_reg().unwrap();
+                b.swap(*j_reg, *k_reg);
+                graph.del_edge(i, j);
+                graph.del_edge(j, k);
+                if i == k {
+                    // This was our last swap
+                    break;
+                } else {
+                    graph.add_edge(i, k);
+                }
+            }
+        } else {
+            let pc_tmp = pc.tmp.expect("This copy needs a temporary");
+            let tmp = pc_tmp.comp(0);
+
+            let mut p = i;
+            let mut p_reg = *vals[p].as_reg().unwrap();
+            b.copy_to(tmp.into(), p_reg.into());
+
+            loop {
+                let j = graph.src(p).unwrap();
+                if j == i {
+                    break;
+                }
+
+                let j_reg = *vals[j].as_reg().unwrap();
+                debug_assert!(j_reg.file() == file);
+
+                if file == RegFile::Bar || file == RegFile::Mem {
+                    let copy_tmp = pc_tmp.comp(1);
+                    b.copy_to(copy_tmp.into(), j_reg.into());
+                    b.copy_to(p_reg.into(), copy_tmp.into());
+                } else {
+                    b.copy_to(p_reg.into(), j_reg.into());
+                }
+                graph.del_edge(p, j);
+
+                p = j;
+                p_reg = j_reg;
+            }
+
+            b.copy_to(p_reg.into(), tmp.into());
+            graph.del_edge(p, i);
+        }
+    }
+
+    b.into_mapped_instrs()
+}
+
+impl Shader<'_> {
+    pub fn lower_par_copies(&mut self) {
+        let sm = self.sm;
+        self.map_instrs(|instr, _| -> MappedInstrs {
+            match instr.op {
+                Op::ParCopy(pc) => {
+                    assert!(instr.pred.is_true());
+                    let mut instrs = MappedInstrs::new();
+                    if DEBUG.annotate() {
+                        instrs.push(Instr::new(OpAnnotate {
+                            annotation: "par_copy lowered by lower_par_copy"
+                                .into(),
+                        }));
+                    }
+                    instrs.extend(lower_par_copy(*pc, sm));
+                    instrs
+                }
+                _ => [instr].into(),
+            }
+        });
+    }
+}
@@ -0,0 +1,275 @@
+// Copyright © 2023 Collabora, Ltd.
+// SPDX-License-Identifier: MIT
+
+use crate::api::{GetDebugFlags, DEBUG};
+use crate::ir::*;
+
+use compiler::bitset::BitSet;
+use rustc_hash::FxHashMap;
+
+struct PhiMap {
+    phi_ssa: FxHashMap<Phi, Vec<SSAValue>>,
+    ssa_phi: FxHashMap<SSAValue, Phi>,
+}
+
+impl PhiMap {
+    pub fn new() -> PhiMap {
+        PhiMap {
+            ssa_phi: Default::default(),
+            phi_ssa: Default::default(),
+        }
+    }
+
+    fn add_phi_srcs(&mut self, op: &OpPhiSrcs) {
+        for (phi, src) in op.srcs.iter() {
+            if let SrcRef::SSA(ssa) = &src.src_ref {
+                assert!(ssa.comps() == 1);
+                let phi_srcs = self.phi_ssa.entry(*phi).or_default();
+                phi_srcs.push(ssa[0]);
+            }
+        }
+    }
+
+    fn add_phi_dsts(&mut self, op: &OpPhiDsts) {
+        for (phi, dst) in op.dsts.iter() {
+            if let Dst::SSA(ssa) = dst {
+                assert!(ssa.comps() == 1);
+                self.ssa_phi.insert(ssa[0], *phi);
+            }
+        }
+    }
+
+    fn get_phi(&self, ssa: &SSAValue) -> Option<&Phi> {
+        self.ssa_phi.get(ssa)
+    }
+
+    fn phi_srcs(&self, idx: &Phi) -> &[SSAValue] {
+        self.phi_ssa.get(idx).unwrap()
+    }
+}
+
+#[derive(Default)]
+struct BarPropPass {
+    ssa_map: FxHashMap<SSAValue, SSAValue>,
+    phi_is_bar: BitSet<Phi>,
+    phi_is_not_bar: BitSet<Phi>,
+}
+
+impl BarPropPass {
+    pub fn new() -> BarPropPass {
+        Default::default()
+    }
+
+    fn add_copy(&mut self, dst: SSAValue, src: SSAValue) {
+        assert!(dst.file() == RegFile::Bar || src.file() == RegFile::Bar);
+        self.ssa_map.insert(dst, src);
+    }
+
+    fn is_bar(&self, ssa: &SSAValue) -> bool {
+        ssa.file() == RegFile::Bar || self.ssa_map.contains_key(ssa)
+    }
+
+    fn map_bar<'a>(&'a self, ssa: &'a SSAValue) -> Option<&'a SSAValue> {
+        let mut ssa = ssa;
+        let mut last_bar = None;
+        loop {
+            let Some(mapped) = self.ssa_map.get(ssa) else {
+                break;
+            };
+
+            if mapped.file() == RegFile::Bar {
+                last_bar = Some(mapped);
+            }
+            ssa = mapped;
+        }
+
+        last_bar
+    }
+
+    fn phi_can_be_bar_recur(
+        &mut self,
+        phi_map: &PhiMap,
+        seen: &mut BitSet<Phi>,
+        phi: Phi,
+    ) -> bool {
+        if self.phi_is_not_bar.contains(phi) {
+            return false;
+        }
+
+        if seen.contains(phi) {
+            // If we've hit a cycle, that's not a fail
+            return true;
+        }
+
+        seen.insert(phi);
+
+        for src_ssa in phi_map.phi_srcs(&phi) {
+            if self.is_bar(src_ssa) {
+                continue;
+            }
+
+            if let Some(src_phi) = phi_map.get_phi(src_ssa) {
+                if self.phi_can_be_bar_recur(phi_map, seen, *src_phi) {
+                    continue;
+                }
+            }
+
+            self.phi_is_not_bar.insert(phi);
+            return false;
+        }
+
+        true
+    }
+
+    fn add_phi_recur(
+        &mut self,
+        ssa_alloc: &mut SSAValueAllocator,
+        phi_map: &PhiMap,
+        needs_bar: &mut BitSet<Phi>,
+        phi: Phi,
+        ssa: SSAValue,
+    ) {
+        if !needs_bar.contains(phi) {
+            return;
+        }
+
+        let bar = ssa_alloc.alloc(RegFile::Bar);
+        self.ssa_map.insert(ssa, bar);
+        self.phi_is_bar.insert(phi);
+        needs_bar.remove(phi);
+
+        for src_ssa in phi_map.phi_srcs(&phi) {
+            if let Some(src_phi) = phi_map.get_phi(src_ssa) {
+                self.add_phi_recur(
+                    ssa_alloc, phi_map, needs_bar, *src_phi, *src_ssa,
+                );
+            }
+        }
+    }
+
+    fn try_add_phi(
+        &mut self,
+        ssa_alloc: &mut SSAValueAllocator,
+        phi_map: &PhiMap,
+        phi: Phi,
+        ssa: SSAValue,
+    ) {
+        if self.phi_is_bar.contains(phi) {
+            return;
+        }
+
+        let mut seen = BitSet::new();
+        if self.phi_can_be_bar_recur(phi_map, &mut seen, phi) {
+            self.add_phi_recur(ssa_alloc, phi_map, &mut seen, phi, ssa);
+            assert!(seen.is_empty());
+        }
+    }
+
+    fn run(&mut self, f: &mut Function) {
+        let mut phi_map = PhiMap::new();
+        let mut phis_want_bar = Vec::new();
+        for b in &f.blocks {
+            for instr in &b.instrs {
+                match &instr.op {
+                    Op::PhiSrcs(op) => {
+                        phi_map.add_phi_srcs(op);
+                    }
+                    Op::PhiDsts(op) => {
+                        phi_map.add_phi_dsts(op);
+                    }
+                    Op::BMov(op) => {
+                        assert!(!op.clear);
+                        assert!(op.src.is_unmodified());
+                        let dst = op.dst.as_ssa().unwrap();
+                        let src = op.src.as_ssa().unwrap();
+                        assert!(dst.comps() == 1 && src.comps() == 1);
+
+                        self.add_copy(dst[0], src[0]);
+
+                        if let Some(phi) = phi_map.get_phi(&src[0]) {
+                            phis_want_bar.push((*phi, src[0]));
+                        }
+                    }
+                    _ => (),
+                }
+            }
+        }
+
+        for (phi, ssa) in phis_want_bar.drain(..) {
+            self.try_add_phi(&mut f.ssa_alloc, &phi_map, phi, ssa);
+        }
+
+        f.map_instrs(|mut instr, _| {
+            match &mut instr.op {
+                Op::PhiSrcs(op) => {
+                    for (idx, src) in op.srcs.iter_mut() {
+                        if self.phi_is_bar.contains(*idx) {
+                            // Barrier immediates don't exist
+                            let ssa = src.as_ssa().unwrap();
+                            let bar = *self.map_bar(&ssa[0]).unwrap();
+                            *src = bar.into();
+                        }
+                    }
+                    [instr].into()
+                }
+                Op::PhiDsts(op) => {
+                    let mut bmovs = Vec::new();
+                    for (idx, dst) in op.dsts.iter_mut() {
+                        if self.phi_is_bar.contains(*idx) {
+                            let ssa = dst.as_ssa().unwrap().clone();
+                            let bar = *self.ssa_map.get(&ssa[0]).unwrap();
+                            *dst = bar.into();
+
+                            // On the off chance that someone still wants the
+                            // GPR version of this barrier, insert an OpBMov to
+                            // copy into the GPR.  DCE will clean it up if it's
+                            // never used.
+                            bmovs.push(Instr::new(OpBMov {
+                                dst: ssa.into(),
+                                src: bar.into(),
+                                clear: false,
+                            }));
+                        }
+                    }
+                    if bmovs.is_empty() {
+                        [instr].into()
+                    } else {
+                        if DEBUG.annotate() {
+                            bmovs.insert(
+                                0,
+                                Instr::new(OpAnnotate {
+                                    annotation: "generated by opt_bar_prop"
+                                        .into(),
+                                }),
+                            );
+                        }
+                        bmovs.insert(1, instr);
+                        bmovs.into()
+                    }
+                }
+                _ => {
+                    let src_types = instr.src_types();
+                    for (i, src) in instr.srcs_mut().iter_mut().enumerate() {
+                        if src_types[i] != SrcType::Bar {
+                            continue;
+                        }
+                        if let SrcRef::SSA(ssa) = &src.src_ref {
+                            if let Some(bar) = self.map_bar(&ssa[0]) {
+                                *src = (*bar).into();
+                            }
+                        };
+                    }
+                    [instr].into()
+                }
+            }
+        });
+    }
+}
+
+impl Shader<'_> {
+    pub fn opt_bar_prop(&mut self) {
+        for f in &mut self.functions {
+            BarPropPass::new().run(f);
+        }
+    }
+}
@@ -0,0 +1,878 @@
+// Copyright © 2022 Collabora, Ltd.
+// SPDX-License-Identifier: MIT
+
+use crate::ir::*;
+
+use rustc_hash::FxHashMap;
+
+enum CBufRule {
+    Yes,
+    No,
+    BindlessRequiresBlock(usize),
+}
+
+impl CBufRule {
+    fn allows_src(&self, src_bi: usize, src: &Src) -> bool {
+        let SrcRef::CBuf(cb) = &src.src_ref else {
+            return true;
+        };
+
+        match self {
+            CBufRule::Yes => true,
+            CBufRule::No => false,
+            CBufRule::BindlessRequiresBlock(bi) => match cb.buf {
+                CBuf::Binding(_) => true,
+                CBuf::BindlessSSA(_) => src_bi == *bi,
+                CBuf::BindlessUGPR(_) => panic!("Not in SSA form"),
+            },
+        }
+    }
+}
+
+struct CopyEntry {
+    bi: usize,
+    src_type: SrcType,
+    src: Src,
+    ftz: bool,
+}
+
+struct PrmtEntry {
+    bi: usize,
+    sel: PrmtSel,
+    srcs: [Src; 2],
+}
+
+/// This entry tracks b2i conversions
+struct ConvBoolToInt {
+    src: Src,
+}
+
+enum CopyPropEntry {
+    Copy(CopyEntry),
+    Prmt(PrmtEntry),
+    ConvBoolToInt(ConvBoolToInt),
+}
+
+struct CopyPropPass<'a> {
+    sm: &'a ShaderModelInfo,
+    ssa_map: FxHashMap<SSAValue, CopyPropEntry>,
+}
+
+impl<'a> CopyPropPass<'a> {
+    pub fn new(sm: &'a ShaderModelInfo) -> Self {
+        CopyPropPass {
+            sm: sm,
+            ssa_map: Default::default(),
+        }
+    }
+
+    fn add_copy(
+        &mut self,
+        bi: usize,
+        dst: SSAValue,
+        src_type: SrcType,
+        src: Src,
+    ) {
+        self.add_copy_float(bi, dst, src_type, src, false);
+    }
+
+    fn add_copy_float(
+        &mut self,
+        bi: usize,
+        dst: SSAValue,
+        src_type: SrcType,
+        src: Src,
+        ftz: bool,
+    ) {
+        assert!(src.src_ref.get_reg().is_none());
+        self.ssa_map.insert(
+            dst,
+            CopyPropEntry::Copy(CopyEntry {
+                bi,
+                src_type,
+                src,
+                ftz,
+            }),
+        );
+    }
+
+    fn add_b2i(&mut self, _bi: usize, dst: SSAValue, src: Src) {
+        assert!(src.src_ref.get_reg().is_none());
+        assert!(src.is_predicate());
+        assert!(dst.is_gpr());
+        self.ssa_map
+            .insert(dst, CopyPropEntry::ConvBoolToInt(ConvBoolToInt { src }));
+    }
+
+    fn add_i2b(&mut self, bi: usize, dst: SSAValue, src: Src, inverted: bool) {
+        assert!(src.src_ref.get_reg().is_none());
+        assert!(dst.is_predicate());
+        // Fold i2b(b2i(x))
+        // if we find i2b(b2i(x)) replace that with a copy
+        let parent = src.as_ssa().and_then(|x| self.get_copy(&x[0]));
+        let Some(CopyPropEntry::ConvBoolToInt(par_entry)) = parent else {
+            return;
+        };
+        let mut copy_src = par_entry.src.clone().modify(src.src_mod);
+        if inverted {
+            copy_src = copy_src.bnot();
+        }
+
+        self.add_copy(bi, dst, SrcType::Pred, copy_src);
+    }
+
+    fn add_prmt(
+        &mut self,
+        bi: usize,
+        dst: SSAValue,
+        sel: PrmtSel,
+        srcs: [Src; 2],
+    ) {
+        assert!(
+            srcs[0].src_ref.get_reg().is_none()
+                && srcs[1].src_ref.get_reg().is_none()
+        );
+        self.ssa_map
+            .insert(dst, CopyPropEntry::Prmt(PrmtEntry { bi, sel, srcs }));
+    }
+
+    fn add_fp64_copy(&mut self, bi: usize, dst: &SSARef, src: Src) {
+        assert!(dst.comps() == 2);
+        match src.src_ref {
+            SrcRef::Zero | SrcRef::Imm32(_) => {
+                self.add_copy(bi, dst[0], SrcType::ALU, Src::ZERO);
+                self.add_copy(bi, dst[1], SrcType::F64, src);
+            }
+            SrcRef::CBuf(cb) => {
+                let lo32 = Src::from(SrcRef::CBuf(cb.clone()));
+                let hi32 = Src {
+                    src_ref: SrcRef::CBuf(cb.offset(4)),
+                    src_mod: src.src_mod,
+                    src_swizzle: src.src_swizzle,
+                };
+                self.add_copy(bi, dst[0], SrcType::ALU, lo32);
+                self.add_copy(bi, dst[1], SrcType::F64, hi32);
+            }
+            SrcRef::SSA(ssa) => {
+                assert!(ssa.comps() == 2);
+                let lo32 = Src::from(ssa[0]);
+                let hi32 = Src {
+                    src_ref: ssa[1].into(),
+                    src_mod: src.src_mod,
+                    src_swizzle: src.src_swizzle,
+                };
+                self.add_copy(bi, dst[0], SrcType::ALU, lo32);
+                self.add_copy(bi, dst[1], SrcType::F64, hi32);
+            }
+            _ => (),
+        }
+    }
+
+    fn get_copy(&self, dst: &SSAValue) -> Option<&CopyPropEntry> {
+        self.ssa_map.get(dst)
+    }
+
+    fn prop_to_pred(&self, pred: &mut Pred) {
+        loop {
+            let src_ssa = match &pred.pred_ref {
+                PredRef::SSA(ssa) => ssa,
+                _ => return,
+            };
+
+            let Some(CopyPropEntry::Copy(entry)) = self.get_copy(src_ssa)
+            else {
+                return;
+            };
+
+            match &entry.src.src_ref {
+                SrcRef::True => {
+                    pred.pred_ref = PredRef::None;
+                }
+                SrcRef::False => {
+                    pred.pred_ref = PredRef::None;
+                    pred.pred_inv = !pred.pred_inv;
+                }
+                SrcRef::SSA(ssa) => {
+                    assert!(ssa.comps() == 1);
+                    pred.pred_ref = PredRef::SSA(ssa[0]);
+                }
+                _ => return,
+            }
+
+            match entry.src.src_mod {
+                SrcMod::None => (),
+                SrcMod::BNot => {
+                    pred.pred_inv = !pred.pred_inv;
+                }
+                _ => panic!("Invalid predicate modifier"),
+            }
+        }
+    }
+
+    fn prop_to_ssa_values(
+        &self,
+        src_ssa: &mut [SSAValue],
+        same_file: bool,
+    ) -> bool {
+        let mut progress = false;
+
+        for c_ssa in src_ssa {
+            let Some(CopyPropEntry::Copy(entry)) = self.get_copy(c_ssa) else {
+                continue;
+            };
+
+            if entry.ftz {
+                continue;
+            }
+
+            if entry.src.is_unmodified() {
+                if let SrcRef::SSA(entry_ssa) = &entry.src.src_ref {
+                    assert!(entry_ssa.comps() == 1);
+
+                    if same_file && (c_ssa.file() != entry_ssa[0].file()) {
+                        continue;
+                    }
+
+                    *c_ssa = entry_ssa[0];
+                    progress = true;
+                }
+            }
+        }
+
+        progress
+    }
+
+    fn prop_to_ssa_ref(&self, src_ssa: &mut SSARef) -> bool {
+        self.prop_to_ssa_values(&mut src_ssa[..], false)
+    }
+
+    fn prop_to_cbuf_ref(&self, cbuf: &mut CBufRef) {
+        match cbuf.buf {
+            CBuf::BindlessSSA(ref mut ssa_values) => loop {
+                if !self.prop_to_ssa_values(&mut ssa_values[..], true) {
+                    break;
+                }
+            },
+            _ => (),
+        }
+    }
+
+    fn prop_to_ssa_src(&self, src: &mut Src) {
+        assert!(src.is_unmodified());
+        if let SrcRef::SSA(src_ssa) = &mut src.src_ref {
+            loop {
+                if !self.prop_to_ssa_ref(src_ssa) {
+                    break;
+                }
+            }
+        }
+    }
+
+    fn prop_to_gpr_src(&self, src: &mut Src) {
+        loop {
+            let src_ssa = match &mut src.src_ref {
+                SrcRef::SSA(ssa) => {
+                    // First, try to propagate SSA components
+                    if self.prop_to_ssa_ref(ssa) {
+                        continue;
+                    }
+                    ssa
+                }
+                _ => return,
+            };
+
+            for c in 0..usize::from(src_ssa.comps()) {
+                let Some(CopyPropEntry::Copy(entry)) =
+                    self.get_copy(&src_ssa[c])
+                else {
+                    return;
+                };
+
+                match entry.src.src_ref {
+                    SrcRef::Zero | SrcRef::Imm32(0) => (),
+                    _ => return,
+                }
+            }
+
+            // If we got here, all the components are zero
+            src.src_ref = SrcRef::Zero;
+        }
+    }
+
+    fn prop_to_scalar_src(
+        &self,
+        src_type: SrcType,
+        cbuf_rule: &CBufRule,
+        src: &mut Src,
+        instr_ftz: bool,
+    ) {
+        loop {
+            let src_ssa = match &src.src_ref {
+                SrcRef::SSA(ssa) => ssa,
+                _ => return,
+            };
+
+            assert!(src_ssa.comps() == 1);
+            let entry = match self.get_copy(&src_ssa[0]) {
+                Some(e) => e,
+                None => return,
+            };
+
+            match entry {
+                CopyPropEntry::Copy(entry) => {
+                    // If the original op flushes denorms, but not the
+                    // consumer, we skip the propagation
+                    if entry.ftz && !instr_ftz {
+                        return;
+                    }
+
+                    if !cbuf_rule.allows_src(entry.bi, &entry.src) {
+                        return;
+                    }
+
+                    // If there are modifiers or ftz enabled, the source types
+                    // have to match
+                    if (!entry.src.is_unmodified() || entry.ftz)
+                        && !entry.src_type.eq_ftz_mod(src_type)
+                    {
+                        return;
+                    }
+
+                    src.src_ref = entry.src.src_ref.clone();
+                    src.src_mod = entry.src.src_mod.modify(src.src_mod);
+                }
+                CopyPropEntry::Prmt(entry) => {
+                    // Turn the swizzle into a permute. For F16, we use Xx to
+                    // indicate that it only takes the bottom 16 bits.
+                    let swizzle_prmt: [u8; 4] = match src_type {
+                        SrcType::F16 => match src.src_swizzle {
+                            SrcSwizzle::None | SrcSwizzle::Xx => [0, 1, 0, 1],
+                            SrcSwizzle::Yy => [2, 3, 2, 3],
+                        },
+                        SrcType::F16v2 => match src.src_swizzle {
+                            SrcSwizzle::None => [0, 1, 2, 3],
+                            SrcSwizzle::Xx => [0, 1, 0, 1],
+                            SrcSwizzle::Yy => [2, 3, 2, 3],
+                        },
+                        _ => [0, 1, 2, 3],
+                    };
+
+                    let mut entry_src_idx = None;
+                    let mut combined = [0_u8; 4];
+
+                    for i in 0..4 {
+                        let prmt_byte = entry.sel.get(swizzle_prmt[i].into());
+
+                        // If we have a sign extension, we cannot simplify it.
+                        if prmt_byte.msb() {
+                            return;
+                        }
+
+                        // Ensure we are using the same source, we cannot
+                        // combine multiple sources.
+                        if entry_src_idx.is_none() {
+                            entry_src_idx = Some(prmt_byte.src());
+                        } else if entry_src_idx != Some(prmt_byte.src()) {
+                            return;
+                        }
+
+                        combined[i] = prmt_byte.byte().try_into().unwrap();
+                    }
+
+                    let entry_src_idx = entry_src_idx.unwrap();
+                    let entry_src = &entry.srcs[entry_src_idx];
+
+                    if !cbuf_rule.allows_src(entry.bi, entry_src) {
+                        return;
+                    }
+
+                    // See if that permute is a valid swizzle
+                    let new_swizzle = match src_type {
+                        SrcType::F16 => match combined {
+                            [0, 1, _, _] => SrcSwizzle::None,
+                            [2, 3, _, _] => SrcSwizzle::Yy,
+                            _ => return,
+                        },
+                        SrcType::F16v2 => match combined {
+                            [0, 1, 2, 3] => SrcSwizzle::None,
+                            [0, 1, 0, 1] => SrcSwizzle::Xx,
+                            [2, 3, 2, 3] => SrcSwizzle::Yy,
+                            _ => return,
+                        },
+                        _ => {
+                            if combined != [0, 1, 2, 3] {
+                                return;
+                            }
+                            SrcSwizzle::None
+                        }
+                    };
+
+                    src.src_ref = entry_src.src_ref.clone();
+                    src.src_mod = entry_src.src_mod.modify(src.src_mod);
+                    src.src_swizzle = new_swizzle;
+                }
+                CopyPropEntry::ConvBoolToInt(_) => {
+                    // b2i(i2b(x)) can't be easily optimized
+                    return;
+                }
+            }
+        }
+    }
+
+    fn prop_to_f64_src(&self, cbuf_rule: &CBufRule, src: &mut Src) {
+        loop {
+            let src_ssa = match &mut src.src_ref {
+                SrcRef::SSA(ssa) => ssa,
+                _ => return,
+            };
+
+            assert!(src_ssa.comps() == 2);
+
+            // First, try to propagate the two halves individually.  Source
+            // modifiers only apply to the high 32 bits so we have to reject
+            // any copies with source modifiers in the low bits and apply
+            // source modifiers as needed when propagating the high bits.
+            let lo_entry_or_none = self.get_copy(&src_ssa[0]);
+            if let Some(CopyPropEntry::Copy(lo_entry)) = lo_entry_or_none {
+                if lo_entry.src.is_unmodified() {
+                    if let SrcRef::SSA(lo_entry_ssa) = &lo_entry.src.src_ref {
+                        src_ssa[0] = lo_entry_ssa[0];
+                        continue;
+                    }
+                }
+            }
+
+            let hi_entry_or_none = self.get_copy(&src_ssa[1]);
+            if let Some(CopyPropEntry::Copy(hi_entry)) = hi_entry_or_none {
+                if hi_entry.src.is_unmodified()
+                    || hi_entry.src_type == SrcType::F64
+                {
+                    if let SrcRef::SSA(hi_entry_ssa) = &hi_entry.src.src_ref {
+                        src_ssa[1] = hi_entry_ssa[0];
+                        src.src_mod = hi_entry.src.src_mod.modify(src.src_mod);
+                        continue;
+                    }
+                }
+            }
+
+            let Some(CopyPropEntry::Copy(lo_entry)) = lo_entry_or_none else {
+                return;
+            };
+
+            let Some(CopyPropEntry::Copy(hi_entry)) = hi_entry_or_none else {
+                return;
+            };
+
+            if !lo_entry.src.is_unmodified() {
+                return;
+            }
+
+            if !hi_entry.src.is_unmodified()
+                && hi_entry.src_type != SrcType::F64
+            {
+                return;
+            }
+
+            if !cbuf_rule.allows_src(hi_entry.bi, &hi_entry.src)
+                || !cbuf_rule.allows_src(lo_entry.bi, &lo_entry.src)
+            {
+                return;
+            }
+
+            let new_src_ref = match &hi_entry.src.src_ref {
+                SrcRef::Zero => match &lo_entry.src.src_ref {
+                    SrcRef::Zero | SrcRef::Imm32(0) => SrcRef::Zero,
+                    _ => return,
+                },
+                SrcRef::Imm32(i) => {
+                    // 32-bit immediates for f64 srouces are the top 32 bits
+                    // with zero in the lower 32.
+                    match lo_entry.src.src_ref {
+                        SrcRef::Zero | SrcRef::Imm32(0) => SrcRef::Imm32(*i),
+                        _ => return,
+                    }
+                }
+                SrcRef::CBuf(hi_cb) => match &lo_entry.src.src_ref {
+                    SrcRef::CBuf(lo_cb) => {
+                        if hi_cb.buf != lo_cb.buf {
+                            return;
+                        }
+                        if lo_cb.offset % 8 != 0 {
+                            return;
+                        }
+                        if hi_cb.offset != lo_cb.offset + 4 {
+                            return;
+                        }
+                        SrcRef::CBuf(lo_cb.clone())
+                    }
+                    _ => return,
+                },
+                // SrcRef::SSA is already handled above
+                _ => return,
+            };
+
+            src.src_ref = new_src_ref;
+            src.src_mod = hi_entry.src.src_mod.modify(src.src_mod);
+        }
+    }
+
+    fn prop_to_src(
+        &self,
+        src_type: SrcType,
+        cbuf_rule: &CBufRule,
+        src: &mut Src,
+        instr_ftz: bool,
+    ) {
+        match src_type {
+            SrcType::SSA => {
+                self.prop_to_ssa_src(src);
+            }
+            SrcType::GPR => {
+                self.prop_to_gpr_src(src);
+            }
+            SrcType::ALU
+            | SrcType::F16
+            | SrcType::F16v2
+            | SrcType::F32
+            | SrcType::I32
+            | SrcType::B32
+            | SrcType::Pred => {
+                self.prop_to_scalar_src(src_type, cbuf_rule, src, instr_ftz);
+            }
+            SrcType::F64 => {
+                self.prop_to_f64_src(cbuf_rule, src);
+            }
+            SrcType::Carry | SrcType::Bar => (),
+        }
+
+        match &mut src.src_ref {
+            SrcRef::CBuf(cbuf) => {
+                self.prop_to_cbuf_ref(cbuf);
+            }
+            _ => (),
+        }
+    }
+
+    fn try_add_instr(&mut self, bi: usize, instr: &Instr) {
+        match &instr.op {
+            Op::HAdd2(add) => {
+                let dst = add.dst.as_ssa().unwrap();
+                assert!(dst.comps() == 1);
+                let dst = dst[0];
+
+                if !add.saturate {
+                    if add.srcs[0].is_fneg_zero(SrcType::F16v2) {
+                        self.add_copy_float(
+                            bi,
+                            dst,
+                            SrcType::F16v2,
+                            add.srcs[1].clone(),
+                            add.ftz,
+                        );
+                    } else if add.srcs[1].is_fneg_zero(SrcType::F16v2) {
+                        self.add_copy_float(
+                            bi,
+                            dst,
+                            SrcType::F16v2,
+                            add.srcs[0].clone(),
+                            add.ftz,
+                        );
+                    }
+                }
+            }
+            Op::FAdd(add) => {
+                let dst = add.dst.as_ssa().unwrap();
+                assert!(dst.comps() == 1);
+                let dst = dst[0];
+
+                if !add.saturate {
+                    if add.srcs[0].is_fneg_zero(SrcType::F32) {
+                        self.add_copy_float(
+                            bi,
+                            dst,
+                            SrcType::F32,
+                            add.srcs[1].clone(),
+                            add.ftz,
+                        );
+                    } else if add.srcs[1].is_fneg_zero(SrcType::F32) {
+                        self.add_copy_float(
+                            bi,
+                            dst,
+                            SrcType::F32,
+                            add.srcs[0].clone(),
+                            add.ftz,
+                        );
+                    }
+                }
+            }
+            Op::DAdd(add) => {
+                let dst = add.dst.as_ssa().unwrap();
+                if add.srcs[0].is_fneg_zero(SrcType::F64) {
+                    self.add_fp64_copy(bi, dst, add.srcs[1].clone());
+                } else if add.srcs[1].is_fneg_zero(SrcType::F64) {
+                    self.add_fp64_copy(bi, dst, add.srcs[0].clone());
+                }
+            }
+            Op::Lop3(lop) => {
+                let dst = lop.dst.as_ssa().unwrap();
+                assert!(dst.comps() == 1);
+                let dst = dst[0];
+
+                let op = lop.op;
+                if op.lut == 0 {
+                    self.add_copy(bi, dst, SrcType::ALU, SrcRef::Zero.into());
+                } else if op.lut == !0 {
+                    self.add_copy(
+                        bi,
+                        dst,
+                        SrcType::ALU,
+                        SrcRef::Imm32(u32::MAX).into(),
+                    );
+                } else {
+                    for s in 0..3 {
+                        if op.lut == LogicOp3::SRC_MASKS[s] {
+                            self.add_copy(
+                                bi,
+                                dst,
+                                SrcType::ALU,
+                                lop.srcs[s].clone(),
+                            );
+                        }
+                    }
+                }
+            }
+            Op::PLop3(lop) => {
+                for i in 0..2 {
+                    let dst = match &lop.dsts[i] {
+                        Dst::SSA(vec) => {
+                            assert!(vec.comps() == 1);
+                            vec[0]
+                        }
+                        _ => continue,
+                    };
+
+                    let op = lop.ops[i];
+                    if op.lut == 0 {
+                        self.add_copy(
+                            bi,
+                            dst,
+                            SrcType::Pred,
+                            SrcRef::False.into(),
+                        );
+                    } else if op.lut == !0 {
+                        self.add_copy(
+                            bi,
+                            dst,
+                            SrcType::Pred,
+                            SrcRef::True.into(),
+                        );
+                    } else {
+                        for s in 0..3 {
+                            if op.lut == LogicOp3::SRC_MASKS[s] {
+                                self.add_copy(
+                                    bi,
+                                    dst,
+                                    SrcType::Pred,
+                                    lop.srcs[s].clone(),
+                                );
+                            } else if op.lut == !LogicOp3::SRC_MASKS[s] {
+                                self.add_copy(
+                                    bi,
+                                    dst,
+                                    SrcType::Pred,
+                                    lop.srcs[s].clone().bnot(),
+                                );
+                            }
+                        }
+                    }
+                }
+            }
+            Op::Sel(sel) => {
+                let dst = sel.dst.as_ssa().unwrap();
+                assert!(dst.comps() == 1);
+                let dst = dst[0];
+
+                let src = match &sel.srcs {
+                    [z, u] if z.is_zero() && u.is_nonzero() => {
+                        sel.cond.clone().bnot()
+                    }
+                    [u, z] if z.is_zero() && u.is_nonzero() => sel.cond.clone(),
+                    _ => return,
+                };
+
+                self.add_b2i(bi, dst, src);
+            }
+            Op::ISetP(isetp) if isetp.set_op.is_trivial(&isetp.accum) => {
+                let dst = isetp.dst.as_ssa().unwrap();
+                assert!(dst.comps() == 1);
+                let dst = dst[0];
+
+                let src = match (&isetp.srcs[0], &isetp.srcs[1]) {
+                    (z, x) | (x, z) if z.is_zero() => x,
+                    _ => return,
+                };
+
+                // -0 = 0
+                // -x != 0 => x != 0
+                if !matches!(src.src_mod, SrcMod::None | SrcMod::INeg) {
+                    return;
+                }
+
+                // x op 0
+                let inverted = match isetp.cmp_op {
+                    IntCmpOp::Eq => true,
+                    IntCmpOp::Ne => false,
+                    _ => return,
+                };
+                self.add_i2b(bi, dst, src.clone(), inverted);
+            }
+            Op::IAdd2(add) => {
+                let dst = add.dst.as_ssa().unwrap();
+                assert!(dst.comps() == 1);
+                let dst = dst[0];
+
+                if add.srcs[0].is_zero() {
+                    self.add_copy(bi, dst, SrcType::I32, add.srcs[1].clone());
+                } else if add.srcs[1].is_zero() {
+                    self.add_copy(bi, dst, SrcType::I32, add.srcs[0].clone());
+                }
+            }
+            Op::IAdd3(add) => {
+                let dst = add.dst.as_ssa().unwrap();
+                assert!(dst.comps() == 1);
+                let dst = dst[0];
+
+                if add.srcs[0].is_zero() {
+                    if add.srcs[1].is_zero() {
+                        self.add_copy(
+                            bi,
+                            dst,
+                            SrcType::I32,
+                            add.srcs[2].clone(),
+                        );
+                    } else if add.srcs[2].is_zero() {
+                        self.add_copy(
+                            bi,
+                            dst,
+                            SrcType::I32,
+                            add.srcs[1].clone(),
+                        );
+                    }
+                } else if add.srcs[1].is_zero() && add.srcs[2].is_zero() {
+                    self.add_copy(bi, dst, SrcType::I32, add.srcs[0].clone());
+                }
+            }
+            Op::Prmt(prmt) => {
+                let dst = prmt.dst.as_ssa().unwrap();
+                assert!(dst.comps() == 1);
+                if let Some(sel) = prmt.get_sel() {
+                    if let Some(imm) = prmt.as_u32() {
+                        self.add_copy(bi, dst[0], SrcType::GPR, imm.into());
+                    } else if sel == PrmtSel(0x3210) {
+                        self.add_copy(
+                            bi,
+                            dst[0],
+                            SrcType::GPR,
+                            prmt.srcs[0].clone(),
+                        );
+                    } else if sel == PrmtSel(0x7654) {
+                        self.add_copy(
+                            bi,
+                            dst[0],
+                            SrcType::GPR,
+                            prmt.srcs[1].clone(),
+                        );
+                    } else {
+                        self.add_prmt(bi, dst[0], sel, prmt.srcs.clone());
+                    }
+                }
+            }
+            Op::R2UR(r2ur) => {
+                assert!(r2ur.src.is_unmodified());
+                if r2ur.src.is_uniform() {
+                    let dst = r2ur.dst.as_ssa().unwrap();
+                    assert!(dst.comps() == 1);
+                    self.add_copy(bi, dst[0], SrcType::GPR, r2ur.src.clone());
+                }
+            }
+            Op::Copy(copy) => {
+                let dst = copy.dst.as_ssa().unwrap();
+                assert!(dst.comps() == 1);
+                self.add_copy(bi, dst[0], SrcType::GPR, copy.src.clone());
+            }
+            Op::ParCopy(pcopy) => {
+                for (dst, src) in pcopy.dsts_srcs.iter() {
+                    let dst = dst.as_ssa().unwrap();
+                    assert!(dst.comps() == 1);
+                    self.add_copy(bi, dst[0], SrcType::GPR, src.clone());
+                }
+            }
+            _ => (),
+        }
+    }
+
+    pub fn run(&mut self, f: &mut Function) {
+        for (bi, b) in f.blocks.iter_mut().enumerate() {
+            let b_uniform = b.uniform;
+            for instr in &mut b.instrs {
+                self.try_add_instr(bi, instr);
+
+                self.prop_to_pred(&mut instr.pred);
+
+                let cbuf_rule = if self.sm.sm() >= 100 {
+                    // Blackwell+ doesn't allow cbufs directly in instruction
+                    // sources anymore and instead have to be explicitly loaded
+                    // with OpLdc.
+                    CBufRule::No
+                } else if instr.is_uniform() {
+                    CBufRule::No
+                } else if !b_uniform {
+                    CBufRule::BindlessRequiresBlock(bi)
+                } else {
+                    CBufRule::Yes
+                };
+
+                // Carry-out and overflow interact funny with SrcMod::INeg so we
+                // can only propagate with modifiers if no carry/overflow is
+                // written.
+                let force_alu_src_type = match &instr.op {
+                    Op::IAdd2(add) => !add.carry_out.is_none(),
+                    Op::IAdd2X(add) => !add.carry_out.is_none(),
+                    Op::IAdd3(add) => {
+                        !add.overflow[0].is_none() || !add.overflow[1].is_none()
+                    }
+                    Op::IAdd3X(add) => {
+                        !add.overflow[0].is_none() || !add.overflow[1].is_none()
+                    }
+                    Op::Lea(lea) => !lea.overflow.is_none(),
+                    Op::LeaX(lea) => !lea.overflow.is_none(),
+                    _ => false,
+                };
+
+                let src_types = instr.src_types();
+                let ftz = instr.ftz();
+                for (i, src) in instr.srcs_mut().iter_mut().enumerate() {
+                    let mut src_type = src_types[i];
+                    if force_alu_src_type {
+                        src_type = match src_type {
+                            SrcType::ALU | SrcType::B32 | SrcType::I32 => {
+                                SrcType::ALU
+                            }
+                            SrcType::Carry | SrcType::Pred => src_type,
+                            _ => panic!("Unhandled src_type"),
+                        };
+                    };
+                    self.prop_to_src(src_type, &cbuf_rule, src, ftz);
+                }
+            }
+        }
+    }
+}
+
+impl Shader<'_> {
+    pub fn opt_copy_prop(&mut self) {
+        for f in &mut self.functions {
+            CopyPropPass::new(self.sm).run(f);
+        }
+    }
+}
@@ -0,0 +1,45 @@
+// Copyright © 2022 Collabora, Ltd.
+// SPDX-License-Identifier: MIT
+
+use crate::ir::*;
+
+use rustc_hash::FxHashSet;
+
+fn opt_crs(f: &mut Function) {
+    let mut live_targets: FxHashSet<Label> = Default::default();
+    for b in f.blocks.iter() {
+        let Some(instr) = b.instrs.last() else {
+            continue;
+        };
+
+        match &instr.op {
+            Op::Sync(OpSync { target })
+            | Op::Brk(OpBrk { target })
+            | Op::Cont(OpCont { target }) => {
+                live_targets.insert(*target);
+            }
+            _ => (),
+        }
+    }
+
+    f.map_instrs(|instr, _| match &instr.op {
+        Op::SSy(OpSSy { target })
+        | Op::PBk(OpPBk { target })
+        | Op::PCnt(OpPCnt { target }) => {
+            if live_targets.contains(target) {
+                [instr].into()
+            } else {
+                [].into()
+            }
+        }
+        _ => [instr].into(),
+    });
+}
+
+impl Shader<'_> {
+    pub fn opt_crs(&mut self) {
+        for f in &mut self.functions {
+            opt_crs(f);
+        }
+    }
+}
@@ -0,0 +1,191 @@
+// Copyright © 2022 Collabora, Ltd.
+// SPDX-License-Identifier: MIT
+
+use crate::{
+    api::{GetDebugFlags, DEBUG},
+    ir::*,
+};
+use compiler::bitset::BitSet;
+
+struct DeadCodePass {
+    any_dead: bool,
+    new_live: bool,
+    live_ssa: BitSet<SSAValue>,
+    live_phi: BitSet<Phi>,
+}
+
+impl DeadCodePass {
+    pub fn new() -> DeadCodePass {
+        DeadCodePass {
+            any_dead: false,
+            new_live: false,
+            live_ssa: Default::default(),
+            live_phi: Default::default(),
+        }
+    }
+
+    fn mark_ssa_live(&mut self, ssa: &SSAValue) {
+        self.new_live |= self.live_ssa.insert(*ssa);
+    }
+
+    fn mark_src_live(&mut self, src: &Src) {
+        for ssa in src.iter_ssa() {
+            self.mark_ssa_live(ssa);
+        }
+    }
+
+    fn mark_phi_live(&mut self, phi: Phi) {
+        self.new_live |= self.live_phi.insert(phi);
+    }
+
+    fn is_dst_live(&self, dst: &Dst) -> bool {
+        match dst {
+            Dst::SSA(ssa) => {
+                for val in ssa.iter() {
+                    if self.live_ssa.contains(*val) {
+                        return true;
+                    }
+                }
+                false
+            }
+            Dst::None => false,
+            _ => panic!("Invalid SSA destination"),
+        }
+    }
+
+    fn is_phi_live(&self, phi: Phi) -> bool {
+        self.live_phi.contains(phi)
+    }
+
+    fn is_instr_live(&self, instr: &Instr) -> bool {
+        if instr.pred.is_false() {
+            return false;
+        }
+
+        if !instr.can_eliminate() {
+            return true;
+        }
+
+        for dst in instr.dsts() {
+            if self.is_dst_live(dst) {
+                return true;
+            }
+        }
+
+        false
+    }
+
+    fn mark_instr(&mut self, instr: &Instr) {
+        match &instr.op {
+            Op::PhiSrcs(phi) => {
+                assert!(instr.pred.is_true());
+                for (phi, src) in phi.srcs.iter() {
+                    if self.is_phi_live(*phi) {
+                        self.mark_src_live(src);
+                    } else {
+                        self.any_dead = true;
+                    }
+                }
+            }
+            Op::PhiDsts(phi) => {
+                assert!(instr.pred.is_true());
+                for (phi, dst) in phi.dsts.iter() {
+                    if self.is_dst_live(dst) {
+                        self.mark_phi_live(*phi);
+                    } else {
+                        self.any_dead = true;
+                    }
+                }
+            }
+            Op::ParCopy(pcopy) => {
+                assert!(instr.pred.is_true());
+                for (dst, src) in pcopy.dsts_srcs.iter() {
+                    if self.is_dst_live(dst) {
+                        self.mark_src_live(src);
+                    } else {
+                        self.any_dead = true;
+                    }
+                }
+            }
+            _ => {
+                if self.is_instr_live(instr) {
+                    if let PredRef::SSA(ssa) = &instr.pred.pred_ref {
+                        self.mark_ssa_live(ssa);
+                    }
+
+                    for src in instr.srcs() {
+                        self.mark_src_live(src);
+                    }
+                } else {
+                    self.any_dead = true;
+                }
+            }
+        }
+    }
+
+    fn map_instr(&self, mut instr: Instr) -> MappedInstrs {
+        let is_live = match &mut instr.op {
+            Op::PhiSrcs(phi) => {
+                phi.srcs.retain(|phi, _| self.is_phi_live(*phi));
+                !phi.srcs.is_empty()
+            }
+            Op::PhiDsts(phi) => {
+                phi.dsts.retain(|_, dst| self.is_dst_live(dst));
+                !phi.dsts.is_empty()
+            }
+            Op::ParCopy(pcopy) => {
+                pcopy.dsts_srcs.retain(|dst, _| self.is_dst_live(dst));
+                !pcopy.dsts_srcs.is_empty()
+            }
+            _ => self.is_instr_live(&instr),
+        };
+
+        if is_live {
+            [instr].into()
+        } else {
+            if DEBUG.annotate() {
+                let ann = OpAnnotate {
+                    annotation: "killed by dce".into(),
+                };
+                [Instr::new(ann)].into()
+            } else {
+                [].into()
+            }
+        }
+    }
+
+    pub fn run(&mut self, f: &mut Function) {
+        loop {
+            self.new_live = false;
+            self.any_dead = false;
+
+            for b in f.blocks.iter().rev() {
+                for instr in b.instrs.iter().rev() {
+                    self.mark_instr(instr);
+                }
+            }
+
+            if !self.new_live {
+                break;
+            }
+        }
+
+        if self.any_dead {
+            f.map_instrs(|instr, _| self.map_instr(instr));
+        }
+    }
+}
+
+impl Function {
+    pub fn opt_dce(&mut self) {
+        DeadCodePass::new().run(self);
+    }
+}
+
+impl Shader<'_> {
+    pub fn opt_dce(&mut self) {
+        for f in &mut self.functions {
+            f.opt_dce();
+        }
+    }
+}
@@ -0,0 +1,465 @@
+// Copyright © 2024 Valve Corporation
+// SPDX-License-Identifier: MIT
+
+use crate::ir::*;
+use compiler::cfg::CFG;
+use std::cmp::max;
+use std::cmp::Reverse;
+
+pub mod graph {
+    #[derive(Clone)]
+    pub struct Edge<EdgeLabel> {
+        pub label: EdgeLabel,
+        pub head_idx: usize,
+    }
+
+    #[derive(Clone)]
+    pub struct Node<NodeLabel, EdgeLabel> {
+        pub label: NodeLabel,
+        pub outgoing_edges: Vec<Edge<EdgeLabel>>,
+    }
+
+    #[derive(Clone)]
+    pub struct Graph<NodeLabel, EdgeLabel> {
+        pub nodes: Vec<Node<NodeLabel, EdgeLabel>>,
+    }
+
+    impl<NodeLabel, EdgeLabel> Graph<NodeLabel, EdgeLabel> {
+        pub fn new(node_labels: impl Iterator<Item = NodeLabel>) -> Self {
+            let nodes = node_labels
+                .map(|label| Node {
+                    label,
+                    outgoing_edges: Vec::new(),
+                })
+                .collect();
+
+            Graph { nodes }
+        }
+
+        pub fn add_edge(
+            &mut self,
+            tail_idx: usize,
+            head_idx: usize,
+            label: EdgeLabel,
+        ) {
+            assert!(head_idx < self.nodes.len());
+            self.nodes[tail_idx]
+                .outgoing_edges
+                .push(Edge { label, head_idx });
+        }
+
+        pub fn reverse(&mut self) {
+            let old_edges: Vec<_> = self
+                .nodes
+                .iter_mut()
+                .map(|node| std::mem::take(&mut node.outgoing_edges))
+                .collect();
+
+            for (tail_idx, edges) in old_edges.into_iter().enumerate() {
+                for e in edges.into_iter() {
+                    self.add_edge(e.head_idx, tail_idx, e.label);
+                }
+            }
+        }
+    }
+}
+
+#[derive(Eq, PartialEq)]
+pub enum SideEffect {
+    /// No side effect (ALU-like)
+    None,
+
+    /// Instruction reads or writes memory
+    ///
+    /// This will be serialized with respect to other
+    /// SideEffect::Memory instructions
+    Memory,
+
+    /// This instcuction is a full code motion barrier
+    ///
+    /// No other instruction will be re-ordered with respect to this one
+    Barrier,
+}
+
+pub fn side_effect_type(op: &Op) -> SideEffect {
+    match op {
+        // Float ALU
+        Op::F2FP(_)
+        | Op::FAdd(_)
+        | Op::FFma(_)
+        | Op::FMnMx(_)
+        | Op::FMul(_)
+        | Op::FSet(_)
+        | Op::FSetP(_)
+        | Op::HAdd2(_)
+        | Op::HFma2(_)
+        | Op::HMul2(_)
+        | Op::HSet2(_)
+        | Op::HSetP2(_)
+        | Op::HMnMx2(_)
+        | Op::FSwz(_)
+        | Op::FSwzAdd(_) => SideEffect::None,
+
+        // Multi-function unit
+        Op::Rro(_) | Op::MuFu(_) => SideEffect::None,
+
+        // Double-precision float ALU
+        Op::DAdd(_)
+        | Op::DFma(_)
+        | Op::DMnMx(_)
+        | Op::DMul(_)
+        | Op::DSetP(_) => SideEffect::None,
+
+        // Integer ALU
+        Op::BRev(_)
+        | Op::Flo(_)
+        | Op::PopC(_)
+        | Op::IMad(_)
+        | Op::IMul(_)
+        | Op::BMsk(_)
+        | Op::IAbs(_)
+        | Op::IAdd2(_)
+        | Op::IAdd2X(_)
+        | Op::IAdd3(_)
+        | Op::IAdd3X(_)
+        | Op::IDp4(_)
+        | Op::IMad64(_)
+        | Op::IMnMx(_)
+        | Op::ISetP(_)
+        | Op::Lea(_)
+        | Op::LeaX(_)
+        | Op::Lop2(_)
+        | Op::Lop3(_)
+        | Op::SuClamp(_)
+        | Op::SuBfm(_)
+        | Op::SuEau(_)
+        | Op::IMadSp(_)
+        | Op::Shf(_)
+        | Op::Shl(_)
+        | Op::Shr(_)
+        | Op::Bfe(_) => SideEffect::None,
+
+        // Conversions
+        Op::F2F(_) | Op::F2I(_) | Op::I2F(_) | Op::I2I(_) | Op::FRnd(_) => {
+            SideEffect::None
+        }
+
+        // Move ops
+        Op::Mov(_) | Op::Prmt(_) | Op::Sel(_) | Op::Sgxt(_) => SideEffect::None,
+        Op::Shfl(_) => SideEffect::None,
+
+        // Predicate ops
+        Op::PLop3(_) | Op::PSetP(_) => SideEffect::None,
+
+        // Uniform ops
+        Op::R2UR(_) | Op::Redux(_) => SideEffect::None,
+
+        // Texture ops
+        Op::Tex(_)
+        | Op::Tld(_)
+        | Op::Tld4(_)
+        | Op::Tmml(_)
+        | Op::Txd(_)
+        | Op::Txq(_) => SideEffect::Memory,
+
+        // Surface ops
+        Op::SuLd(_)
+        | Op::SuSt(_)
+        | Op::SuAtom(_)
+        | Op::SuLdGa(_)
+        | Op::SuStGa(_) => SideEffect::Memory,
+
+        // Memory ops
+        Op::Ipa(_) | Op::Ldc(_) => SideEffect::None,
+        Op::Ld(_)
+        | Op::Ldsm(_)
+        | Op::LdSharedLock(_)
+        | Op::St(_)
+        | Op::StSCheckUnlock(_)
+        | Op::Atom(_)
+        | Op::AL2P(_)
+        | Op::ALd(_)
+        | Op::ASt(_)
+        | Op::CCtl(_)
+        | Op::LdTram(_)
+        | Op::MemBar(_) => SideEffect::Memory,
+
+        // Matrix ops
+        Op::Imma(_) | Op::Hmma(_) | Op::Movm(_) => SideEffect::None,
+
+        // Control-flow ops
+        Op::BClear(_)
+        | Op::Break(_)
+        | Op::BSSy(_)
+        | Op::BSync(_)
+        | Op::SSy(_)
+        | Op::Sync(_)
+        | Op::Brk(_)
+        | Op::PBk(_)
+        | Op::Cont(_)
+        | Op::PCnt(_)
+        | Op::Bra(_)
+        | Op::Exit(_)
+        | Op::WarpSync(_) => SideEffect::Barrier,
+
+        // We don't model the barrier register yet, so serialize these
+        Op::BMov(_) => SideEffect::Memory,
+
+        // Geometry ops
+        Op::Out(_) | Op::OutFinal(_) => SideEffect::Barrier,
+
+        // Miscellaneous ops
+        Op::Bar(_)
+        | Op::TexDepBar(_)
+        | Op::CS2R(_)
+        | Op::Isberd(_)
+        | Op::Isbewr(_)
+        | Op::ViLd(_)
+        | Op::Kill(_)
+        | Op::S2R(_) => SideEffect::Barrier,
+        Op::PixLd(_) | Op::Vote(_) | Op::Match(_) => SideEffect::None,
+        Op::Nop(OpNop { label, .. }) => {
+            if label.is_none() {
+                SideEffect::None
+            } else {
+                SideEffect::Barrier
+            }
+        }
+
+        // Virtual ops
+        Op::Annotate(_)
+        | Op::ParCopy(_)
+        | Op::Swap(_)
+        | Op::Copy(_)
+        | Op::Undef(_) => SideEffect::None,
+
+        Op::SrcBar(_)
+        | Op::Pin(_)
+        | Op::Unpin(_)
+        | Op::PhiSrcs(_)
+        | Op::PhiDsts(_)
+        | Op::RegOut(_) => SideEffect::Barrier,
+    }
+}
+
+pub fn estimate_block_weight(cfg: &CFG<BasicBlock>, block_idx: usize) -> u64 {
+    let loop_depth = cfg.loop_depth(block_idx) as f32;
+    10_f32.powf((loop_depth + 1.0).log2()) as u64
+}
+
+/// Try to guess how many cycles a variable latency instruction will take
+///
+/// These values are based on the cycle estimates from ["Dissecting the NVidia
+/// Turing T4 GPU via Microbenchmarking"](https://arxiv.org/pdf/1903.07486).
+/// Memory instructions were copied from L1 data cache latencies.
+/// For instructions not mentioned in the paper, I made up numbers.
+/// This could probably be improved.
+pub fn estimate_variable_latency(sm: &ShaderModelInfo, op: &Op) -> u32 {
+    if !sm.op_needs_scoreboard(op) {
+        return 0;
+    }
+
+    match op {
+        // Multi-function unit
+        Op::Rro(_) | Op::MuFu(_) => 15,
+
+        // Double-precision float ALU
+        Op::DFma(_) | Op::DSetP(_) => 54,
+        Op::DAdd(_) | Op::DMnMx(_) | Op::DMul(_) => 48,
+
+        // Integer ALU
+        Op::BRev(_) | Op::Flo(_) | Op::PopC(_) => 15,
+        Op::IMad(_) | Op::IMul(_) => {
+            assert!(sm.sm() < 70);
+            86
+        }
+
+        // Conversions
+        Op::F2F(_) | Op::F2I(_) | Op::I2F(_) | Op::I2I(_) | Op::FRnd(_) => 15,
+
+        // Move ops
+        Op::Shfl(_) => 15,
+
+        // Uniform ops
+        Op::Redux(_) | Op::R2UR(_) => 15,
+
+        // Texture ops
+        Op::Tex(_)
+        | Op::Tld(_)
+        | Op::Tld4(_)
+        | Op::Tmml(_)
+        | Op::Txd(_)
+        | Op::Txq(_) => 32,
+
+        // Surface ops
+        Op::SuLd(_)
+        | Op::SuSt(_)
+        | Op::SuAtom(_)
+        | Op::SuLdGa(_)
+        | Op::SuStGa(_) => 32,
+
+        // Memory ops
+        Op::Ldc(_) => 4,
+
+        Op::Ld(_)
+        | Op::Ldsm(_)
+        | Op::Movm(_)
+        | Op::LdSharedLock(_)
+        | Op::St(_)
+        | Op::StSCheckUnlock(_)
+        | Op::Atom(_)
+        | Op::AL2P(_)
+        | Op::ALd(_)
+        | Op::ASt(_)
+        | Op::Ipa(_)
+        | Op::CCtl(_)
+        | Op::LdTram(_)
+        | Op::MemBar(_) => 32,
+
+        // Control-flow ops
+        Op::WarpSync(_) => 16,
+
+        // Barrier
+        Op::BMov(_) => 16,
+
+        // Geometry ops
+        Op::Out(_) | Op::OutFinal(_) => 2,
+
+        // Miscellaneous ops
+        Op::Bar(_)
+        | Op::TexDepBar(_)
+        | Op::CS2R(_)
+        | Op::Isberd(_)
+        | Op::Isbewr(_)
+        | Op::ViLd(_)
+        | Op::Kill(_)
+        | Op::PixLd(_)
+        | Op::S2R(_)
+        | Op::Match(_) => 16,
+
+        Op::Hmma(_) | Op::Imma(_) => 22,
+
+        _ => panic!("Unknown variable latency op {op}"),
+    }
+}
+
+#[derive(Default, Clone)]
+pub struct NodeLabel {
+    pub cycles_to_end: u32,
+    pub num_uses: u32,
+
+    /// The first cycle that the instruction can begin executing
+    pub ready_cycle: u32,
+
+    pub exec_latency: u32,
+}
+
+pub struct EdgeLabel {
+    pub latency: u32,
+}
+
+pub type DepGraph = graph::Graph<NodeLabel, EdgeLabel>;
+
+pub fn calc_statistics(g: &mut DepGraph) -> Vec<usize> {
+    let mut initial_ready_list = Vec::new();
+    for i in (0..g.nodes.len()).rev() {
+        let node = &g.nodes[i];
+        let mut max_delay = 0;
+        for edge in &node.outgoing_edges {
+            assert!(edge.head_idx > i);
+            max_delay = max(
+                max_delay,
+                g.nodes[edge.head_idx].label.cycles_to_end + edge.label.latency,
+            );
+        }
+        let node = &mut g.nodes[i];
+        node.label.cycles_to_end = max_delay;
+        node.label.num_uses = node.outgoing_edges.len().try_into().unwrap();
+        if node.label.num_uses == 0 {
+            initial_ready_list.push(i);
+        }
+    }
+
+    initial_ready_list
+}
+
+#[derive(Clone, PartialEq, Eq, PartialOrd, Ord)]
+pub struct ReadyInstr {
+    cycles_to_end: u32,
+
+    // We use the original instruction order as a final tie-breaker, the idea
+    // being that the original schedule is often not too bad. Since we're
+    // iterating in reverse order, that means scheduling the largest instruciton
+    // index first.
+    pub index: usize,
+}
+
+impl ReadyInstr {
+    pub fn new<E>(g: &graph::Graph<NodeLabel, E>, i: usize) -> Self {
+        let label = &g.nodes[i].label;
+        // NodeLabel::cycles_to_end is cycles from the beginning of the
+        // instruction to the top of the block, but we want from the end of the
+        // instruction to the top of the block
+        let cycles_to_end = label.cycles_to_end + label.exec_latency;
+        ReadyInstr {
+            cycles_to_end,
+            index: i,
+        }
+    }
+}
+
+#[derive(Clone, PartialEq, Eq, PartialOrd, Ord)]
+pub struct FutureReadyInstr {
+    /// The first cycle that the instruction can end executing
+    pub ready_cycle: Reverse<u32>,
+    pub index: usize,
+}
+
+impl FutureReadyInstr {
+    pub fn new<E>(g: &graph::Graph<NodeLabel, E>, i: usize) -> Self {
+        let label = &g.nodes[i].label;
+        // NodeLabel::ready_cycle is the earliest beginning cycle for the
+        // instruction, but we need the earliest end cycle for the instruction.
+        let ready_cycle = label.ready_cycle.saturating_sub(label.exec_latency);
+        FutureReadyInstr {
+            ready_cycle: Reverse(ready_cycle),
+            index: i,
+        }
+    }
+}
+
+#[allow(dead_code)]
+pub fn save_graphviz(
+    instrs: &[Box<Instr>],
+    g: &DepGraph,
+) -> std::io::Result<()> {
+    // dot /tmp/instr_dep_graph.dot -Tsvg > /tmp/instr_dep_graph.svg
+
+    use std::fs::File;
+    use std::io::{BufWriter, Write};
+
+    let file = File::create("/tmp/instr_dep_graph.dot")?;
+    let mut w = BufWriter::new(file);
+
+    writeln!(w, "digraph {{")?;
+    for (i, instr) in instrs.iter().enumerate() {
+        let l = &g.nodes[i].label;
+        writeln!(
+            w,
+            "    {i} [label=\"{}\\n{}, {}\"];",
+            instr, l.cycles_to_end, l.num_uses
+        )?;
+    }
+    for (i, node) in g.nodes.iter().enumerate() {
+        for j in &node.outgoing_edges {
+            writeln!(
+                w,
+                "    {i} -> {} [label=\"{}\"];",
+                j.head_idx, j.label.latency
+            )?;
+        }
+    }
+    writeln!(w, "}}")?;
+    w.flush()?;
+    Ok(())
+}
@@ -0,0 +1,256 @@
+// Copyright © 2024 Valve Corporation
+// SPDX-License-Identifier: MIT
+
+use crate::ir::*;
+use crate::opt_instr_sched_common::*;
+use crate::reg_tracker::RegRefIterable;
+use crate::reg_tracker::RegTracker;
+use std::cmp::max;
+use std::cmp::Reverse;
+use std::collections::BinaryHeap;
+
+struct RegUse<T: Clone> {
+    reads: Vec<T>,
+    write: Option<T>,
+}
+
+impl<T: Clone> RegUse<T> {
+    pub fn new() -> Self {
+        RegUse {
+            reads: Vec::new(),
+            write: None,
+        }
+    }
+
+    pub fn add_read(&mut self, dep: T) {
+        self.reads.push(dep);
+    }
+
+    pub fn set_write(&mut self, dep: T) {
+        self.write = Some(dep);
+        self.reads.clear();
+    }
+}
+
+fn generate_dep_graph(sm: &ShaderModelInfo, instrs: &[Instr]) -> DepGraph {
+    let mut g = DepGraph::new((0..instrs.len()).map(|_| Default::default()));
+
+    // Maps registers to RegUse<ip, src_dst_idx>.  Predicates are
+    // represented by src_idx = usize::MAX.
+    let mut uses: Box<RegTracker<RegUse<(usize, usize)>>> =
+        Box::new(RegTracker::new_with(&|| RegUse::new()));
+
+    let mut last_memory_op = None;
+    let mut last_barrier_op = None;
+
+    for ip in (0..instrs.len()).rev() {
+        let instr = &instrs[ip];
+
+        if let Some(bar_ip) = last_barrier_op {
+            g.add_edge(ip, bar_ip, EdgeLabel { latency: 0 });
+        }
+
+        match side_effect_type(&instr.op) {
+            SideEffect::None => (),
+            SideEffect::Barrier => {
+                let last_ip = last_barrier_op.unwrap_or(instrs.len());
+                for other_ip in (ip + 1)..last_ip {
+                    g.add_edge(ip, other_ip, EdgeLabel { latency: 0 });
+                }
+                last_barrier_op = Some(ip);
+            }
+            SideEffect::Memory => {
+                if let Some(mem_ip) = last_memory_op {
+                    g.add_edge(ip, mem_ip, EdgeLabel { latency: 0 });
+                }
+                last_memory_op = Some(ip);
+            }
+        }
+
+        uses.for_each_instr_dst_mut(instr, |i, u| {
+            if let Some((w_ip, w_dst_idx)) = u.write {
+                let mut latency = sm.waw_latency(
+                    &instr.op,
+                    i,
+                    !instr.pred.pred_ref.is_none(),
+                    &instrs[w_ip].op,
+                    w_dst_idx,
+                );
+                if sm.op_needs_scoreboard(&instr.op) {
+                    // Barriers take two cycles to become active
+                    latency = max(latency, 2);
+                }
+                g.add_edge(ip, w_ip, EdgeLabel { latency });
+            }
+
+            for &(r_ip, r_src_idx) in &u.reads {
+                let mut latency = if r_src_idx == usize::MAX {
+                    sm.paw_latency(&instr.op, i)
+                } else {
+                    sm.raw_latency(&instr.op, i, &instrs[r_ip].op, r_src_idx)
+                };
+
+                latency =
+                    max(latency, estimate_variable_latency(sm, &instr.op));
+
+                g.add_edge(ip, r_ip, EdgeLabel { latency });
+            }
+        });
+        uses.for_each_instr_src_mut(instr, |i, u| {
+            if let Some((w_ip, w_dst_idx)) = u.write {
+                let mut latency =
+                    sm.war_latency(&instr.op, i, &instrs[w_ip].op, w_dst_idx);
+                if sm.op_needs_scoreboard(&instr.op) {
+                    // Barriers take two cycles to become active
+                    latency = max(latency, 2);
+                }
+                g.add_edge(ip, w_ip, EdgeLabel { latency });
+            }
+        });
+
+        // We're iterating in reverse, so writes are logically first
+        uses.for_each_instr_dst_mut(instr, |i, c| {
+            c.set_write((ip, i));
+        });
+        uses.for_each_instr_pred_mut(instr, |c| {
+            c.add_read((ip, usize::MAX));
+        });
+        uses.for_each_instr_src_mut(instr, |i, c| {
+            c.add_read((ip, i));
+        });
+
+        // Initialize this node's distance to the end
+        let mut ready_cycle = (0..instr.dsts().len())
+            .map(|i| sm.worst_latency(&instr.op, i))
+            .max()
+            .unwrap_or(0);
+
+        let var_latency = estimate_variable_latency(sm, &instr.op)
+            + sm.exec_latency(&instrs[instrs.len() - 1].op);
+        ready_cycle = max(ready_cycle, var_latency);
+
+        let label = &mut g.nodes[ip].label;
+        label.exec_latency = sm.exec_latency(&instr.op);
+        label.ready_cycle = ready_cycle;
+    }
+
+    g
+}
+
+fn generate_order(
+    g: &mut DepGraph,
+    init_ready_list: Vec<usize>,
+) -> (Vec<usize>, u64) {
+    let mut ready_instrs: BinaryHeap<ReadyInstr> = BinaryHeap::new();
+    let mut future_ready_instrs: BinaryHeap<FutureReadyInstr> = init_ready_list
+        .into_iter()
+        .map(|i| FutureReadyInstr::new(g, i))
+        .collect();
+
+    let mut current_cycle = 0u32;
+    let mut instr_order = Vec::with_capacity(g.nodes.len());
+    loop {
+        // Move ready instructions to the ready list
+        loop {
+            let Some(fri) = future_ready_instrs.peek() else {
+                break;
+            };
+            if current_cycle < fri.ready_cycle.0 {
+                break;
+            }
+            ready_instrs.push(ReadyInstr::new(g, fri.index));
+            future_ready_instrs.pop();
+        }
+
+        // Pick a ready instruction
+        let next_idx = match ready_instrs.pop() {
+            None => match future_ready_instrs.peek() {
+                None => break, // Both lists are empty. We're done!
+                Some(&FutureReadyInstr {
+                    ready_cycle: Reverse(ready_cycle),
+                    ..
+                }) => {
+                    // Fast-forward time to when the next instr is ready
+                    assert!(ready_cycle > current_cycle);
+                    current_cycle = ready_cycle;
+                    continue;
+                }
+            },
+            Some(ReadyInstr { index, .. }) => index,
+        };
+
+        // Schedule the instuction
+        instr_order.push(next_idx);
+        current_cycle += g.nodes[next_idx].label.exec_latency;
+
+        let outgoing_edges =
+            std::mem::take(&mut g.nodes[next_idx].outgoing_edges);
+        for edge in outgoing_edges.into_iter() {
+            let dep_instr = &mut g.nodes[edge.head_idx].label;
+            dep_instr.ready_cycle =
+                max(dep_instr.ready_cycle, current_cycle + edge.label.latency);
+            dep_instr.num_uses -= 1;
+            if dep_instr.num_uses == 0 {
+                future_ready_instrs
+                    .push(FutureReadyInstr::new(g, edge.head_idx));
+            }
+        }
+    }
+
+    (instr_order, u64::from(current_cycle))
+}
+
+fn sched_buffer(
+    sm: &ShaderModelInfo,
+    instrs: Vec<Instr>,
+) -> (impl Iterator<Item = Instr> + use<>, u64) {
+    let mut g = generate_dep_graph(sm, &instrs);
+    let init_ready_list = calc_statistics(&mut g);
+    // save_graphviz(&instrs, &g).unwrap();
+    g.reverse();
+    let (new_order, cycle_count) = generate_order(&mut g, init_ready_list);
+
+    // Apply the new instruction order
+    let mut instrs: Vec<Option<Instr>> = instrs.into_iter().map(Some).collect();
+    let instrs = new_order.into_iter().rev().map(move |i| {
+        std::mem::take(&mut instrs[i]).expect("Instruction scheduled twice")
+    });
+    (instrs, cycle_count)
+}
+
+impl Function {
+    pub fn opt_instr_sched_postpass(&mut self, sm: &ShaderModelInfo) -> u64 {
+        let mut num_static_cycles = 0u64;
+        for i in 0..self.blocks.len() {
+            let block = &mut self.blocks[i];
+
+            let orig_instr_count = block.instrs.len();
+            let instrs = std::mem::take(&mut block.instrs);
+            let (instrs, cycle_count) = sched_buffer(sm, instrs);
+            block.instrs = instrs.collect();
+            assert_eq!(orig_instr_count, block.instrs.len());
+
+            let block_weight = estimate_block_weight(&self.blocks, i);
+            num_static_cycles = cycle_count
+                .checked_mul(block_weight)
+                .expect("Cycle count estimate overflow")
+                .checked_add(num_static_cycles)
+                .expect("Cycle count estimate overflow");
+        }
+        num_static_cycles
+    }
+}
+
+impl Shader<'_> {
+    /// Post-RA instruction scheduling
+    ///
+    /// Uses the popular latency-weighted-depth heuristic.
+    /// See eg. Cooper & Torczon's "Engineering A Compiler", 3rd ed.
+    /// Chapter 12.3 "Local scheduling"
+    pub fn opt_instr_sched_postpass(&mut self) {
+        self.info.num_static_cycles = 0;
+        for f in &mut self.functions {
+            self.info.num_static_cycles += f.opt_instr_sched_postpass(self.sm);
+        }
+    }
+}
@@ -0,0 +1,175 @@
+// Copyright © 2023 Mel Henning
+// SPDX-License-Identifier: MIT
+
+use crate::ir::*;
+
+use compiler::cfg::CFGBuilder;
+use rustc_hash::{FxBuildHasher, FxHashMap};
+
+fn clone_branch(op: &Op) -> Op {
+    match op {
+        Op::Bra(b) => {
+            assert!(b.cond.is_true());
+            Op::Bra(b.clone())
+        }
+        Op::Exit(e) => Op::Exit(e.clone()),
+        _ => unreachable!(),
+    }
+}
+
+fn jump_thread(func: &mut Function) -> bool {
+    // Let's call a basic block "trivial" if its only instruction is an
+    // unconditional branch. If a block is trivial, we can update all of its
+    // predecessors to jump to its sucessor.
+    //
+    // A single reverse pass over the basic blocks is enough to update all of
+    // the edges we're interested in. Roughly, if we assume that all loops in
+    // the shader can terminate, then loop heads are never trivial and we
+    // never replace a backward edge. Therefore, in each step we only need to
+    // make sure that later control flow has been replaced in order to update
+    // the current block as much as possible.
+    //
+    // We additionally try to update a branch-to-empty-block to point to the
+    // block's successor, which along with block dce/reordering can sometimes
+    // enable a later optimization that converts branches to fallthrough.
+    let mut progress = false;
+
+    // A branch to label can be replaced with Op
+    let mut replacements: FxHashMap<Label, Op> = Default::default();
+
+    // Invariant 1: At the end of each loop iteration,
+    //              every trivial block with an index in [i, blocks.len())
+    //              is represented in replacements.keys()
+    // Invariant 2: replacements.values() never contains
+    //              a branch to a trivial block
+    for i in (0..func.blocks.len()).rev() {
+        // Replace the branch if possible
+        if let Some(instr) = func.blocks[i].instrs.last_mut() {
+            if let Op::Bra(bra) = &mut instr.op {
+                if let Some(replacement) = replacements.get(&bra.target) {
+                    if bra.cond.is_true() {
+                        // Unconditional branches can just be replaced
+                        instr.op = clone_branch(replacement);
+                        progress = true;
+                    } else {
+                        // OpExit has a form that takes an input predicate but it
+                        // doesn't support upred so there's nothing we can do here.
+                        // TODO: however we could do something for the case OpBra has a
+                        //       non uniform input predicate.
+                        if let Op::Bra(replacement) = replacement {
+                            bra.target = replacement.target;
+                            progress = true;
+                        }
+                    }
+                }
+                // If the branch target was previously a trivial block then the
+                // branch was previously a forward edge (see above) and by
+                // invariants 1 and 2 we just updated the branch to target
+                // a nontrivial block
+            }
+        }
+
+        // Is this block trivial?
+        let block_label = func.blocks[i].label;
+        match &func.blocks[i].instrs[..] {
+            [instr] => {
+                if instr.is_branch_always_taken() {
+                    // Upholds invariant 2 because we updated the branch above
+                    replacements.insert(block_label, clone_branch(&instr.op));
+                }
+            }
+            [] => {
+                // Empty block - falls through
+                // Our successor might be trivial, so we need to
+                // apply the rewrite map to uphold invariant 2
+                let target_label = func.blocks[i + 1].label;
+                let replacement = replacements
+                    .get(&target_label)
+                    .map(clone_branch)
+                    .unwrap_or_else(|| {
+                        Op::Bra(
+                            OpBra {
+                                target: target_label,
+                                cond: true.into(),
+                            }
+                            .into(),
+                        )
+                    });
+                replacements.insert(block_label, replacement);
+            }
+            _ => (),
+        }
+    }
+
+    if progress {
+        // We don't update the CFG above, so rewrite it if we made progress
+        rewrite_cfg(func);
+    }
+
+    progress
+}
+
+fn rewrite_cfg(func: &mut Function) {
+    // CFGBuilder takes care of removing dead blocks for us
+    // We use the basic block's label to identify it
+    let mut builder = CFGBuilder::<_, _, FxBuildHasher>::new();
+
+    for i in 0..func.blocks.len() {
+        let block = &func.blocks[i];
+        // Note: fall-though must be first edge
+        if block.falls_through() {
+            let next_block = &func.blocks[i + 1];
+            builder.add_edge(block.label, next_block.label);
+        }
+        if let Some(control_flow) = block.branch() {
+            match &control_flow.op {
+                Op::Bra(bra) => {
+                    builder.add_edge(block.label, bra.target);
+                }
+                Op::Exit(_) => (),
+                _ => unreachable!(),
+            };
+        }
+    }
+
+    for block in func.blocks.drain() {
+        builder.add_node(block.label, block);
+    }
+    let _ = std::mem::replace(&mut func.blocks, builder.as_cfg());
+}
+
+/// Replace jumps to the following block with fall-through
+fn opt_fall_through(func: &mut Function) {
+    for i in 0..func.blocks.len() - 1 {
+        let remove_last_instr = match func.blocks[i].branch() {
+            Some(b) => match &b.op {
+                Op::Bra(bra) => bra.target == func.blocks[i + 1].label,
+                _ => false,
+            },
+            None => false,
+        };
+
+        if remove_last_instr {
+            func.blocks[i].instrs.pop();
+        }
+    }
+}
+
+impl Function {
+    pub fn opt_jump_thread(&mut self) {
+        if jump_thread(self) {
+            opt_fall_through(self);
+        }
+    }
+}
+
+impl Shader<'_> {
+    /// A simple jump threading pass
+    ///
+    /// Note that this can introduce critical edges, so it cannot be run before RA
+    pub fn opt_jump_thread(&mut self) {
+        for f in &mut self.functions {
+            f.opt_jump_thread();
+        }
+    }
+}
@@ -0,0 +1,266 @@
+// Copyright © 2022 Collabora, Ltd.
+// SPDX-License-Identifier: MIT
+
+use crate::ir::*;
+
+use rustc_hash::FxHashMap;
+use std::slice;
+
+struct LopEntry {
+    op: LogicOp3,
+    srcs_used: u8,
+    srcs: [Src; 3],
+}
+
+struct LopPass {
+    use_counts: FxHashMap<SSAValue, u32>,
+    ssa_lop: FxHashMap<SSAValue, LopEntry>,
+}
+
+fn src_as_bool(src: &Src) -> Option<bool> {
+    assert!(src.is_unmodified());
+    match src.src_ref {
+        SrcRef::Zero | SrcRef::False | SrcRef::Imm32(0) => Some(false),
+        SrcRef::True | SrcRef::Imm32(u32::MAX) => Some(true),
+        _ => None,
+    }
+}
+
+impl LopPass {
+    fn new(f: &Function) -> LopPass {
+        let mut use_counts = FxHashMap::default();
+        for b in &f.blocks {
+            for instr in &b.instrs {
+                if let PredRef::SSA(ssa) = instr.pred.pred_ref {
+                    use_counts.entry(ssa).and_modify(|e| *e += 1).or_insert(1);
+                }
+
+                for src in instr.srcs() {
+                    if let SrcRef::SSA(vec) = &src.src_ref {
+                        for ssa in vec.iter() {
+                            use_counts
+                                .entry(*ssa)
+                                .and_modify(|e| *e += 1)
+                                .or_insert(1);
+                        }
+                    }
+                }
+            }
+        }
+        LopPass {
+            use_counts: use_counts,
+            ssa_lop: Default::default(),
+        }
+    }
+
+    fn add_lop(&mut self, ssa: SSAValue, op: LogicOp3, srcs: [Src; 3]) {
+        let mut srcs_used = 0;
+        for i in 0..3 {
+            if op.src_used(i) {
+                srcs_used |= 1 << i;
+                assert!(src_as_bool(&srcs[i]).is_none());
+            }
+        }
+        let entry = LopEntry {
+            op: op,
+            srcs_used: srcs_used,
+            srcs: srcs,
+        };
+        self.ssa_lop.insert(ssa, entry);
+    }
+
+    fn dedup_srcs(&self, op: &mut LogicOp3, srcs: &[Src; 3]) {
+        for i in 0..2 {
+            for j in (i + 1)..3 {
+                if srcs[i].src_ref == srcs[j].src_ref {
+                    assert!(srcs[i].is_unmodified());
+                    assert!(srcs[j].is_unmodified());
+                    *op = LogicOp3::new_lut(&|x, y, z| {
+                        let mut s = [x, y, z];
+                        s[j] = s[i];
+                        op.eval(s[0], s[1], s[2])
+                    });
+                }
+            }
+        }
+    }
+
+    fn try_prop_to_src(
+        &self,
+        ops: &mut [LogicOp3],
+        srcs: &mut [Src; 3],
+        src_idx: usize,
+    ) {
+        loop {
+            assert!(srcs[src_idx].is_unmodified());
+            let ssa = match &srcs[src_idx].src_ref {
+                SrcRef::SSA(vec) => {
+                    assert!(vec.comps() == 1);
+                    vec[0]
+                }
+                _ => return,
+            };
+
+            let Some(entry) = self.ssa_lop.get(&ssa) else {
+                return;
+            };
+
+            let entry_use_count = *self.use_counts.get(&ssa).unwrap();
+            if entry.srcs_used.count_ones() > 1 && entry_use_count > 1 {
+                return;
+            }
+
+            let mut entry_srcs = [usize::MAX; 3];
+            let mut next_src = 0_usize;
+            for i in 0..3 {
+                if entry.srcs_used & (1 << i) == 0 {
+                    continue;
+                }
+
+                let mut found = false;
+                for j in 0..3 {
+                    if entry.srcs[i].src_ref == srcs[j].src_ref {
+                        entry_srcs[i] = j;
+                        found = true;
+                        break;
+                    }
+                }
+                if found {
+                    continue;
+                }
+
+                loop {
+                    if next_src >= srcs.len() {
+                        return;
+                    }
+
+                    // All callers of this function need to ensure that
+                    // constant sources are already folded so we know we
+                    // can always re-use them.
+                    if next_src == src_idx
+                        || src_as_bool(&srcs[next_src]).is_some()
+                    {
+                        entry_srcs[i] = next_src;
+                        next_src += 1;
+                        break;
+                    }
+                    next_src += 1;
+                }
+            }
+
+            // Clear out the propagated source. What we put here doesn't matter
+            // since it's no longer used.  It may be overwritten by one of the
+            // entry sources but there is no guarantee of this.
+            srcs[src_idx] = match ssa.file() {
+                RegFile::GPR | RegFile::UGPR => SrcRef::Zero.into(),
+                RegFile::Pred | RegFile::UPred => SrcRef::True.into(),
+                RegFile::Carry | RegFile::Bar | RegFile::Mem => {
+                    panic!("Not a normal register");
+                }
+            };
+
+            for i in 0..3 {
+                if entry_srcs[i] != usize::MAX {
+                    srcs[entry_srcs[i]] = entry.srcs[i].clone();
+                }
+            }
+            for op in ops.iter_mut() {
+                *op = LogicOp3::new_lut(&|x, y, z| {
+                    let mut s = [x, y, z];
+                    let mut es = [0; 3];
+                    for i in 0..3 {
+                        if entry_srcs[i] != usize::MAX {
+                            es[i] = s[entry_srcs[i]];
+                        }
+                    }
+                    let e = entry.op.eval(es[0], es[1], es[2]);
+                    s[src_idx] = e;
+                    op.eval(s[0], s[1], s[2])
+                });
+            }
+        }
+    }
+
+    fn opt_lop3(&mut self, op: &mut OpLop3) {
+        self.dedup_srcs(&mut op.op, &op.srcs);
+
+        for (i, src) in op.srcs.iter_mut().enumerate() {
+            assert!(src.is_unmodified());
+
+            if let Some(b) = src_as_bool(src) {
+                op.op.fix_src(i, b);
+            }
+
+            if !op.op.src_used(i) {
+                // Replace unused sources with RZ
+                *src = SrcRef::Zero.into();
+            }
+        }
+
+        for i in 0..3 {
+            self.try_prop_to_src(slice::from_mut(&mut op.op), &mut op.srcs, i);
+        }
+
+        if let Dst::SSA(ssa) = &op.dst {
+            assert!(ssa.comps() == 1);
+            self.add_lop(ssa[0], op.op, op.srcs.clone());
+        }
+    }
+
+    fn opt_plop3(&mut self, op: &mut OpPLop3) {
+        for (i, src) in op.srcs.iter_mut().enumerate() {
+            if src.src_mod.is_bnot() {
+                op.ops[0].invert_src(i);
+                op.ops[1].invert_src(i);
+                src.src_mod = SrcMod::None;
+            }
+        }
+
+        self.dedup_srcs(&mut op.ops[0], &op.srcs);
+        self.dedup_srcs(&mut op.ops[1], &op.srcs);
+
+        // Replace unused sources with PT
+        for (i, src) in op.srcs.iter_mut().enumerate() {
+            if let Some(b) = src_as_bool(src) {
+                op.ops[0].fix_src(i, b);
+                op.ops[1].fix_src(i, b);
+            }
+
+            if !op.ops[0].src_used(i) && !op.ops[1].src_used(i) {
+                *src = SrcRef::True.into();
+            }
+        }
+
+        for i in 0..3 {
+            self.try_prop_to_src(&mut op.ops, &mut op.srcs, i);
+        }
+
+        for i in 0..2 {
+            if let Dst::SSA(ssa) = &op.dsts[i] {
+                assert!(ssa.comps() == 1);
+                self.add_lop(ssa[0], op.ops[i], op.srcs.clone());
+            }
+        }
+    }
+
+    fn run(&mut self, f: &mut Function) {
+        for b in &mut f.blocks {
+            for instr in &mut b.instrs {
+                match &mut instr.op {
+                    Op::Lop3(op) => self.opt_lop3(op),
+                    Op::PLop3(op) => self.opt_plop3(op),
+                    _ => (),
+                }
+            }
+        }
+    }
+}
+
+impl Shader<'_> {
+    pub fn opt_lop(&mut self) {
+        for f in &mut self.functions {
+            let mut pass = LopPass::new(f);
+            pass.run(f);
+        }
+    }
+}
@@ -0,0 +1,66 @@
+// Copyright © 2023 Collabora, Ltd.
+// SPDX-License-Identifier: MIT
+
+use crate::{
+    api::{GetDebugFlags, DEBUG},
+    ir::*,
+};
+
+fn try_combine_outs(emit: &mut Instr, cut: &Instr) -> bool {
+    let Op::Out(emit) = &mut emit.op else {
+        return false;
+    };
+
+    let Op::Out(cut) = &cut.op else {
+        return false;
+    };
+
+    if emit.out_type != OutType::Emit || cut.out_type != OutType::Cut {
+        return false;
+    }
+
+    let Some(handle) = emit.dst.as_ssa() else {
+        return false;
+    };
+
+    if cut.handle.as_ssa() != Some(handle) {
+        return false;
+    }
+
+    if emit.stream != cut.stream {
+        return false;
+    }
+
+    emit.dst = cut.dst.clone();
+    emit.out_type = OutType::EmitThenCut;
+
+    true
+}
+
+impl Shader<'_> {
+    pub fn opt_out(&mut self) {
+        if !matches!(self.info.stage, ShaderStageInfo::Geometry(_)) {
+            return;
+        }
+
+        for f in &mut self.functions {
+            for b in &mut f.blocks {
+                let mut instrs = Vec::new();
+                for instr in b.instrs.drain(..) {
+                    if let Some(prev) = instrs.last_mut() {
+                        if try_combine_outs(prev, &instr) {
+                            if DEBUG.annotate() {
+                                instrs.push(Instr::new(OpAnnotate {
+                                    annotation: "combined by opt_out".into(),
+                                }));
+                            }
+                            continue;
+                        }
+                    }
+                    instrs.push(instr);
+                }
+                b.instrs = instrs;
+            }
+        }
+    }
+}
@@ -0,0 +1,284 @@
+/*
+ * Copyright © 2023 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+use crate::ir::*;
+
+use rustc_hash::FxHashMap;
+
+struct PrmtSrcs {
+    srcs: [SrcRef; 2],
+    num_srcs: usize,
+    imm_src: usize,
+    num_imm_bytes: usize,
+}
+
+impl PrmtSrcs {
+    fn new() -> PrmtSrcs {
+        PrmtSrcs {
+            srcs: [SrcRef::Zero, SrcRef::Zero],
+            num_srcs: 0,
+            imm_src: usize::MAX,
+            num_imm_bytes: 0,
+        }
+    }
+
+    fn try_add_src(&mut self, src: &SrcRef) -> Option<usize> {
+        for i in 0..self.num_srcs {
+            if self.srcs[i] == *src {
+                return Some(i);
+            }
+        }
+
+        if self.num_srcs < 2 {
+            let i = self.num_srcs;
+            self.num_srcs += 1;
+            self.srcs[i] = src.clone();
+            Some(i)
+        } else {
+            None
+        }
+    }
+
+    fn try_add_imm_u8(&mut self, u: u8) -> Option<usize> {
+        if self.imm_src == usize::MAX {
+            if self.num_srcs >= 2 {
+                return None;
+            }
+            self.imm_src = self.num_srcs;
+            self.num_srcs += 1;
+        }
+
+        match &mut self.srcs[self.imm_src] {
+            SrcRef::Zero => {
+                if u == 0 {
+                    // Common case, just leave it as a SrcRef::Zero
+                    debug_assert!(self.num_imm_bytes <= 1);
+                    self.num_imm_bytes = 1;
+                    Some(0)
+                } else {
+                    let b = self.num_imm_bytes;
+                    self.num_imm_bytes += 1;
+                    let imm = u32::from(u) << (b * 8);
+                    self.srcs[self.imm_src] = SrcRef::Imm32(imm);
+                    Some(b)
+                }
+            }
+            SrcRef::Imm32(imm) => {
+                let b = self.num_imm_bytes;
+                self.num_imm_bytes += 1;
+                *imm |= u32::from(u) << (b * 8);
+                Some(b)
+            }
+            _ => panic!("We said this was the imm src"),
+        }
+    }
+}
+
+struct PrmtEntry {
+    sel: PrmtSel,
+    srcs: [SrcRef; 2],
+}
+
+struct PrmtPass {
+    ssa_prmt: FxHashMap<SSAValue, PrmtEntry>,
+}
+
+impl PrmtPass {
+    fn new() -> PrmtPass {
+        PrmtPass {
+            ssa_prmt: Default::default(),
+        }
+    }
+
+    fn add_prmt(&mut self, op: &OpPrmt) {
+        let Dst::SSA(dst_ssa) = &op.dst else {
+            return;
+        };
+        debug_assert!(dst_ssa.comps() == 1);
+        let dst_ssa = dst_ssa[0];
+
+        let Some(sel) = op.get_sel() else {
+            return;
+        };
+
+        debug_assert!(op.srcs[0].is_unmodified());
+        debug_assert!(op.srcs[1].is_unmodified());
+        let srcs = [op.srcs[0].src_ref.clone(), op.srcs[1].src_ref.clone()];
+
+        self.ssa_prmt.insert(dst_ssa, PrmtEntry { sel, srcs });
+    }
+
+    fn get_prmt(&self, ssa: &SSAValue) -> Option<&PrmtEntry> {
+        self.ssa_prmt.get(ssa)
+    }
+
+    fn get_prmt_for_src(&self, src: &Src) -> Option<&PrmtEntry> {
+        debug_assert!(src.is_unmodified());
+        if let SrcRef::SSA(vec) = &src.src_ref {
+            debug_assert!(vec.comps() == 1);
+            self.get_prmt(&vec[0])
+        } else {
+            None
+        }
+    }
+
+    /// Try to optimize for the OpPrmt of OpPrmt case where only one source of
+    /// the inner OpPrmt is used
+    fn try_opt_prmt_src(&mut self, op: &mut OpPrmt, src_idx: usize) -> bool {
+        let Some(op_sel) = op.get_sel() else {
+            return false;
+        };
+
+        let Some(src_prmt) = self.get_prmt_for_src(&op.srcs[src_idx]) else {
+            return false;
+        };
+
+        let mut new_sel = [PrmtSelByte::INVALID; 4];
+        let mut src_prmt_src = usize::MAX;
+        for i in 0..4 {
+            let op_sel_byte = op_sel.get(i);
+            if op_sel_byte.src() != src_idx {
+                new_sel[i] = op_sel_byte;
+                continue;
+            }
+
+            let src_sel_byte = src_prmt.sel.get(op_sel_byte.byte());
+
+            if src_prmt_src != usize::MAX && src_prmt_src != src_sel_byte.src()
+            {
+                return false;
+            }
+            src_prmt_src = src_sel_byte.src();
+
+            new_sel[i] = PrmtSelByte::new(
+                src_idx,
+                src_sel_byte.byte(),
+                op_sel_byte.msb() | src_sel_byte.msb(),
+            );
+        }
+
+        let new_sel = PrmtSel::new(new_sel);
+
+        op.sel = new_sel.into();
+        if src_prmt_src == usize::MAX {
+            // This source is unused
+            op.srcs[src_idx] = 0.into();
+        } else {
+            op.srcs[src_idx] = src_prmt.srcs[src_prmt_src].clone().into();
+        }
+        true
+    }
+
+    /// Try to optimize for the OpPrmt of OpPrmt case as if we're considering a
+    /// full 4-way OpPrmt in which some sources may be duplicates
+    fn try_opt_prmt4(&mut self, op: &mut OpPrmt) -> bool {
+        let Some(op_sel) = op.get_sel() else {
+            return false;
+        };
+
+        let mut srcs = PrmtSrcs::new();
+        let mut new_sel = [PrmtSelByte::INVALID; 4];
+        for i in 0..4 {
+            let op_sel_byte = op_sel.get(i);
+            let src = &op.srcs[op_sel_byte.src()];
+
+            if let Some(src_prmt) = self.get_prmt_for_src(src) {
+                let src_sel_byte = src_prmt.sel.get(op_sel_byte.byte());
+                let src_prmt_src = &src_prmt.srcs[src_sel_byte.src()];
+                if let Some(u) = src_prmt_src.as_u32() {
+                    let mut imm_u8 = src_sel_byte.fold_u32(u);
+                    if op_sel_byte.msb() {
+                        imm_u8 = ((imm_u8 as i8) >> 7) as u8;
+                    }
+
+                    let Some(byte_idx) = srcs.try_add_imm_u8(imm_u8) else {
+                        return false;
+                    };
+
+                    new_sel[i] =
+                        PrmtSelByte::new(srcs.imm_src, byte_idx, false);
+                } else {
+                    let Some(src_idx) = srcs.try_add_src(src_prmt_src) else {
+                        return false;
+                    };
+
+                    new_sel[i] = PrmtSelByte::new(
+                        src_idx,
+                        src_sel_byte.byte(),
+                        op_sel_byte.msb() | src_sel_byte.msb(),
+                    );
+                }
+            } else if let Some(u) = src.as_u32(SrcType::ALU) {
+                let imm_u8 = op_sel_byte.fold_u32(u);
+                let Some(byte_idx) = srcs.try_add_imm_u8(imm_u8) else {
+                    return false;
+                };
+
+                new_sel[i] = PrmtSelByte::new(srcs.imm_src, byte_idx, false);
+            } else {
+                debug_assert!(src.is_unmodified());
+                let Some(src_idx) = srcs.try_add_src(&src.src_ref) else {
+                    return false;
+                };
+
+                new_sel[i] = PrmtSelByte::new(
+                    src_idx,
+                    op_sel_byte.byte(),
+                    op_sel_byte.msb(),
+                );
+            }
+        }
+
+        let new_sel = PrmtSel::new(new_sel);
+        if new_sel == op_sel
+            && srcs.srcs[0] == op.srcs[0].src_ref
+            && srcs.srcs[1] == op.srcs[1].src_ref
+        {
+            return false;
+        }
+
+        op.sel = new_sel.into();
+        let [srcs0, srcs1] = srcs.srcs;
+        op.srcs[0] = srcs0.into();
+        op.srcs[1] = srcs1.into();
+        true
+    }
+
+    fn opt_prmt(&mut self, op: &mut OpPrmt) {
+        for i in 0..2 {
+            loop {
+                if !self.try_opt_prmt_src(op, i) {
+                    break;
+                }
+            }
+        }
+
+        loop {
+            if !self.try_opt_prmt4(op) {
+                break;
+            }
+        }
+
+        self.add_prmt(op);
+    }
+
+    fn run(&mut self, f: &mut Function) {
+        for b in &mut f.blocks {
+            for instr in &mut b.instrs {
+                if let Op::Prmt(op) = &mut instr.op {
+                    self.opt_prmt(op);
+                }
+            }
+        }
+    }
+}
+
+impl Shader<'_> {
+    pub fn opt_prmt(&mut self) {
+        for f in &mut self.functions {
+            PrmtPass::new().run(f);
+        }
+    }
+}
@@ -0,0 +1,116 @@
+// Copyright © 2024 Collabora, Ltd.
+// SPDX-License-Identifier: MIT
+
+use crate::ir::*;
+
+use rustc_hash::FxHashMap;
+
+fn should_lower_to_warp(
+    sm: &ShaderModelInfo,
+    instr: &Instr,
+    r2ur: &FxHashMap<SSAValue, SSAValue>,
+) -> bool {
+    if !sm.op_can_be_uniform(&instr.op) {
+        return true;
+    }
+
+    let mut num_non_uniform_srcs = 0;
+    instr.for_each_ssa_use(|ssa| {
+        if !ssa.is_uniform() || r2ur.contains_key(ssa) {
+            num_non_uniform_srcs += 1;
+        }
+    });
+
+    num_non_uniform_srcs >= 2
+}
+
+fn propagate_r2ur(
+    instr: &mut Instr,
+    r2ur: &FxHashMap<SSAValue, SSAValue>,
+) -> bool {
+    let mut progress = false;
+
+    // We don't want Instr::for_each_ssa_use_mut() because it would treat
+    // bindless cbuf sources as SSA sources.
+    for src in instr.srcs_mut() {
+        if let SrcRef::SSA(vec) = &mut src.src_ref {
+            for ssa in &mut vec[..] {
+                if let Some(r) = r2ur.get(ssa) {
+                    progress = true;
+                    *ssa = *r;
+                }
+            }
+        }
+    }
+
+    progress
+}
+
+impl Shader<'_> {
+    pub fn opt_uniform_instrs(&mut self) {
+        let sm = self.sm;
+        let mut r2ur = Default::default();
+        let mut propagated_r2ur = false;
+        self.map_instrs(|mut instr, alloc| {
+            match &instr.op {
+                Op::Redux(_)
+                | Op::PhiDsts(_)
+                | Op::PhiSrcs(_)
+                | Op::Pin(_)
+                | Op::Unpin(_)
+                | Op::Vote(_) => [instr].into(),
+                Op::Bra(bra) if sm.sm() >= 80 => match &instr.pred.pred_ref {
+                    PredRef::SSA(ssa) if ssa.file() == RegFile::UPred => {
+                        let bra_u = OpBra {
+                            target: bra.target,
+                            cond: instr.pred.into(),
+                        };
+                        [Instr::new(bra_u)].into()
+                    }
+                    _ => [instr].into(),
+                },
+                _ if instr.is_uniform() => {
+                    let mut b = InstrBuilder::new(sm);
+                    if should_lower_to_warp(sm, &instr, &r2ur) {
+                        propagated_r2ur |= propagate_r2ur(&mut instr, &r2ur);
+                        instr.for_each_ssa_def_mut(|ssa| {
+                            let w = alloc.alloc(ssa.file().to_warp());
+                            r2ur.insert(*ssa, w);
+                            b.push_op(OpR2UR {
+                                dst: (*ssa).into(),
+                                src: w.into(),
+                            });
+                            *ssa = w;
+                        });
+                        let mut v = b.into_vec();
+                        v.insert(0, instr);
+                        v.into()
+                    } else {
+                        // We may have non-uniform sources
+                        instr.for_each_ssa_use_mut(|ssa| {
+                            let file = ssa.file();
+                            if !file.is_uniform() {
+                                let u = alloc.alloc(file.to_uniform().unwrap());
+                                b.push_op(OpR2UR {
+                                    dst: u.into(),
+                                    src: (*ssa).into(),
+                                });
+                                *ssa = u;
+                            }
+                        });
+                        b.push_instr(instr);
+                        b.into_mapped_instrs()
+                    }
+                }
+                _ => {
+                    propagated_r2ur |= propagate_r2ur(&mut instr, &r2ur);
+                    [instr].into()
+                }
+            }
+        });
+
+        if propagated_r2ur {
+            self.opt_dce();
+        }
+    }
+}
@@ -0,0 +1,778 @@
+// Copyright © 2024 Collabora, Ltd.
+// SPDX-License-Identifier: MIT
+
+extern crate nvidia_headers;
+
+use std::cmp::min;
+
+use compiler::bindings::*;
+use nak_bindings::*;
+use nvidia_headers::classes::{cla0c0, clc0c0, clc3c0, clc6c0, clcbc0, clcdc0};
+
+use bitview::*;
+use paste::paste;
+
+type QMDBitView<'a> = BitMutView<'a, [u32]>;
+
+trait QMD {
+    const GLOBAL_SIZE_LAYOUT: nak_qmd_dispatch_size_layout;
+
+    fn new() -> Self;
+    fn set_barrier_count(&mut self, barrier_count: u8);
+    fn set_cbuf(&mut self, idx: u8, addr: u64, size: u32);
+    fn cbuf_desc_layout(idx: u8) -> nak_qmd_cbuf_desc_layout;
+    fn set_global_size(&mut self, width: u32, height: u32, depth: u32);
+    fn set_local_size(&mut self, width: u16, height: u16, depth: u16);
+    fn set_prog_addr(&mut self, addr: u64);
+    fn set_register_count(&mut self, register_count: u8);
+    fn set_crs_size(&mut self, crs_size: u32);
+    fn set_slm_size(&mut self, slm_size: u32);
+    fn set_smem_size(
+        &mut self,
+        smem_size: u32,
+        dev: &nv_device_info,
+        info: &nak_shader_info,
+    );
+}
+
+macro_rules! set_enum {
+    ($bv:expr, $cls:ident, $strct:ident, $field:ident, $enum:ident) => {
+        $bv.set_field(
+            paste! {$cls::[<$strct _ $field>]},
+            paste! {$cls::[<$strct _ $field _ $enum>]},
+        )
+    };
+}
+
+macro_rules! set_field {
+    ($bv:expr, $cls:ident, $strct:ident, $field:ident, $val:expr) => {
+        $bv.set_field(paste! {$cls::[<$strct _ $field>]}, $val)
+    };
+}
+
+macro_rules! set_array {
+    ($bv:expr, $cls:ident, $strct:ident, $f:ident, $i:expr, $x:expr) => {
+        $bv.set_field(paste! {$cls::[<$strct _ $f>]}($i), $x)
+    };
+}
+
+macro_rules! qmd_init {
+    ($bv: expr, $c:ident, $s:ident, $mjv:expr, $mnv:expr) => {
+        set_field!($bv, $c, $s, QMD_MAJOR_VERSION, $mjv);
+        set_field!($bv, $c, $s, QMD_VERSION, $mnv);
+
+        set_enum!($bv, $c, $s, API_VISIBLE_CALL_LIMIT, NO_CHECK);
+        set_enum!($bv, $c, $s, SAMPLER_INDEX, INDEPENDENTLY);
+    };
+}
+
+macro_rules! qmd_impl_common {
+    ($c:ident, $s:ident) => {
+        fn set_barrier_count(&mut self, barrier_count: u8) {
+            let mut bv = QMDBitView::new(&mut self.qmd);
+            set_field!(bv, $c, $s, BARRIER_COUNT, barrier_count);
+        }
+
+        const GLOBAL_SIZE_LAYOUT: nak_qmd_dispatch_size_layout = {
+            assert!(paste! {$c::[<$s _MAX_BIT>]} < size_of::<Self>() * 8);
+            assert!(NAK_MAX_QMD_SIZE_B as usize >= size_of::<Self>());
+
+            let w = paste! {$c::[<$s _CTA_RASTER_WIDTH>]};
+            let h = paste! {$c::[<$s _CTA_RASTER_HEIGHT>]};
+            let d = paste! {$c::[<$s _CTA_RASTER_DEPTH>]};
+
+            let local_w = paste! {$c::[<$s _CTA_THREAD_DIMENSION0>]};
+            let local_h = paste! {$c::[<$s _CTA_THREAD_DIMENSION1>]};
+            let local_d = paste! {$c::[<$s _CTA_THREAD_DIMENSION2>]};
+
+            nak_qmd_dispatch_size_layout {
+                x_start: w.start as u16,
+                x_end: w.end as u16,
+                y_start: h.start as u16,
+                y_end: h.end as u16,
+                z_start: d.start as u16,
+                z_end: d.end as u16,
+                local_x_start: local_w.start as u16,
+                local_x_end: local_w.end as u16,
+                local_y_start: local_h.start as u16,
+                local_y_end: local_h.end as u16,
+                local_z_start: local_d.start as u16,
+                local_z_end: local_d.end as u16,
+            }
+        };
+
+        fn set_global_size(&mut self, width: u32, height: u32, depth: u32) {
+            let mut bv = QMDBitView::new(&mut self.qmd);
+            set_field!(bv, $c, $s, CTA_RASTER_WIDTH, width);
+            set_field!(bv, $c, $s, CTA_RASTER_HEIGHT, height);
+            set_field!(bv, $c, $s, CTA_RASTER_DEPTH, depth);
+        }
+
+        fn set_local_size(&mut self, width: u16, height: u16, depth: u16) {
+            let mut bv = QMDBitView::new(&mut self.qmd);
+            set_field!(bv, $c, $s, CTA_THREAD_DIMENSION0, width);
+            set_field!(bv, $c, $s, CTA_THREAD_DIMENSION1, height);
+            set_field!(bv, $c, $s, CTA_THREAD_DIMENSION2, depth);
+        }
+    };
+}
+
+macro_rules! qmd_impl_set_crs_size {
+    ($c:ident, $s:ident) => {
+        fn set_crs_size(&mut self, crs_size: u32) {
+            let mut bv = QMDBitView::new(&mut self.qmd);
+            let crs_size = crs_size.next_multiple_of(0x200);
+            set_field!(bv, $c, $s, SHADER_LOCAL_MEMORY_CRS_SIZE, crs_size);
+        }
+    };
+}
+
+const NONE_SHIFT: u8 = 0;
+const SHIFTED4_SHIFT: u8 = 4;
+const SHIFTED6_SHIFT: u8 = 6;
+
+macro_rules! suffix_shift {
+    ($suffix:ident) => {
+        paste! { [<$suffix _SHIFT>] }
+    };
+}
+
+macro_rules! suffix_field2 {
+    ($c:ident, $s:ident, $field:ident, $subfield:ident, NONE) => {
+        paste! { $c::[<$s _ $field _ $subfield>] }
+    };
+    ($c:ident, $s:ident, $field:ident, $subfield:ident, $suffix:ident) => {
+        paste! { $c::[<$s _ $field _ $subfield _ $suffix>] }
+    };
+}
+
+macro_rules! cbuf_suffix_field {
+    ($c:ident, $s:ident, $subfield:ident, $suffix:ident) => {
+        suffix_field2!($c, $s, CONSTANT_BUFFER, $subfield, $suffix)
+    };
+}
+
+macro_rules! qmd_impl_set_cbuf {
+    ($c:ident, $s:ident, $addr_suffix:ident, $size_suffix:ident) => {
+        fn set_cbuf(&mut self, idx: u8, addr: u64, size: u32) {
+            let mut bv = QMDBitView::new(&mut self.qmd);
+            let idx = idx.into();
+
+            let addr_shift = suffix_shift!($addr_suffix);
+            let addr_shifted = addr >> addr_shift;
+            assert!((addr_shifted << addr_shift) == addr);
+            bv.set_field(
+                cbuf_suffix_field!($c, $s, ADDR_LOWER, $addr_suffix)(idx),
+                addr_shifted as u32,
+            );
+            bv.set_field(
+                cbuf_suffix_field!($c, $s, ADDR_UPPER, $addr_suffix)(idx),
+                (addr_shifted >> 32) as u32,
+            );
+
+            let size_shift = suffix_shift!($size_suffix);
+            assert!(((size >> size_shift) << size_shift) == size);
+            bv.set_field(
+                cbuf_suffix_field!($c, $s, SIZE, $size_suffix)(idx),
+                size >> size_shift,
+            );
+
+            set_array!(bv, $c, $s, CONSTANT_BUFFER_VALID, idx, true);
+        }
+
+        fn cbuf_desc_layout(idx: u8) -> nak_qmd_cbuf_desc_layout {
+            let lo = cbuf_suffix_field!($c, $s, ADDR_LOWER, $addr_suffix)(
+                idx.into(),
+            );
+            let hi = cbuf_suffix_field!($c, $s, ADDR_UPPER, $addr_suffix)(
+                idx.into(),
+            );
+            nak_qmd_cbuf_desc_layout {
+                addr_shift: suffix_shift!($addr_suffix).into(),
+                addr_lo_start: lo.start as u16,
+                addr_lo_end: lo.end as u16,
+                addr_hi_start: hi.start as u16,
+                addr_hi_end: hi.end as u16,
+            }
+        }
+    };
+}
+
+macro_rules! qmd_impl_set_prog_addr_32 {
+    ($c:ident, $s:ident) => {
+        fn set_prog_addr(&mut self, addr: u64) {
+            let mut bv = QMDBitView::new(&mut self.qmd);
+            set_field!(bv, $c, $s, PROGRAM_OFFSET, addr);
+        }
+    };
+}
+
+macro_rules! prog_addr_suffix_field {
+    ($c:ident, $s:ident, $subfield:ident, $suffix:ident) => {
+        suffix_field2!($c, $s, PROGRAM_ADDRESS, $subfield, $suffix)
+    };
+}
+
+macro_rules! qmd_impl_set_prog_addr_64 {
+    ($c:ident, $s:ident, $addr_suffix:ident) => {
+        fn set_prog_addr(&mut self, addr: u64) {
+            let mut bv = QMDBitView::new(&mut self.qmd);
+
+            let addr_shift = suffix_shift!($addr_suffix);
+            let addr_shifted = addr >> addr_shift;
+            assert!((addr_shifted << addr_shift) == addr);
+
+            bv.set_field(
+                prog_addr_suffix_field!($c, $s, LOWER, $addr_suffix),
+                addr_shifted as u32,
+            );
+            bv.set_field(
+                prog_addr_suffix_field!($c, $s, UPPER, $addr_suffix),
+                (addr_shifted >> 32) as u32,
+            );
+        }
+    };
+}
+
+macro_rules! qmd_impl_set_register_count {
+    ($c:ident, $s:ident, $field:ident) => {
+        fn set_register_count(&mut self, register_count: u8) {
+            let mut bv = QMDBitView::new(&mut self.qmd);
+            set_field!(bv, $c, $s, $field, register_count);
+        }
+    };
+}
+
+macro_rules! qmd_impl_set_slm_size {
+    ($c:ident, $s:ident, $size_suffix:ident) => {
+        fn set_slm_size(&mut self, slm_size: u32) {
+            let mut bv = QMDBitView::new(&mut self.qmd);
+            let slm_size = slm_size.next_multiple_of(0x10);
+            let size_shift = suffix_shift!($size_suffix);
+            let slm_size_shifted = slm_size >> size_shift;
+            assert!(slm_size_shifted << size_shift == slm_size);
+            bv.set_field(
+                suffix_field2!(
+                    $c,
+                    $s,
+                    SHADER_LOCAL_MEMORY_HIGH,
+                    SIZE,
+                    $size_suffix
+                ),
+                0,
+            );
+            bv.set_field(
+                suffix_field2!(
+                    $c,
+                    $s,
+                    SHADER_LOCAL_MEMORY_LOW,
+                    SIZE,
+                    $size_suffix
+                ),
+                slm_size_shifted,
+            );
+        }
+    };
+}
+
+mod qmd_0_6 {
+    use crate::qmd::*;
+    use nvidia_headers::classes::cla0c0::qmd as cla0c0;
+
+    #[repr(transparent)]
+    pub struct Qmd0_6 {
+        qmd: [u32; 64],
+    }
+
+    impl QMD for Qmd0_6 {
+        fn new() -> Self {
+            let mut qmd = [0; 64];
+            let mut bv = QMDBitView::new(&mut qmd);
+            qmd_init!(bv, cla0c0, QMDV00_06, 0, 6);
+            set_field!(bv, cla0c0, QMDV00_06, SASS_VERSION, 0x30);
+            Self { qmd }
+        }
+
+        qmd_impl_common!(cla0c0, QMDV00_06);
+        qmd_impl_set_crs_size!(cla0c0, QMDV00_06);
+        qmd_impl_set_cbuf!(cla0c0, QMDV00_06, NONE, NONE);
+        qmd_impl_set_prog_addr_32!(cla0c0, QMDV00_06);
+        qmd_impl_set_register_count!(cla0c0, QMDV00_06, REGISTER_COUNT);
+        qmd_impl_set_slm_size!(cla0c0, QMDV00_06, NONE);
+
+        fn set_smem_size(
+            &mut self,
+            smem_size: u32,
+            _dev: &nv_device_info,
+            _info: &nak_shader_info,
+        ) {
+            let mut bv = QMDBitView::new(&mut self.qmd);
+
+            let smem_size = smem_size.next_multiple_of(0x100);
+            set_field!(bv, cla0c0, QMDV00_06, SHARED_MEMORY_SIZE, smem_size);
+
+            let l1_config = if smem_size <= (16 << 10) {
+                cla0c0::QMDV00_06_L1_CONFIGURATION_DIRECTLY_ADDRESSABLE_MEMORY_SIZE_16KB
+            } else if smem_size <= (32 << 10) {
+                cla0c0::QMDV00_06_L1_CONFIGURATION_DIRECTLY_ADDRESSABLE_MEMORY_SIZE_32KB
+            } else if smem_size <= (48 << 10) {
+                cla0c0::QMDV00_06_L1_CONFIGURATION_DIRECTLY_ADDRESSABLE_MEMORY_SIZE_48KB
+            } else {
+                panic!("Invalid shared memory size");
+            };
+            set_field!(bv, cla0c0, QMDV00_06, L1_CONFIGURATION, l1_config);
+        }
+    }
+}
+use qmd_0_6::Qmd0_6;
+
+mod qmd_2_1 {
+    use crate::qmd::*;
+    use nvidia_headers::classes::clc0c0::qmd as clc0c0;
+
+    #[repr(transparent)]
+    pub struct Qmd2_1 {
+        qmd: [u32; 64],
+    }
+
+    impl QMD for Qmd2_1 {
+        fn new() -> Self {
+            let mut qmd = [0; 64];
+            let mut bv = QMDBitView::new(&mut qmd);
+            qmd_init!(bv, clc0c0, QMDV02_01, 2, 1);
+            set_field!(bv, clc0c0, QMDV02_01, SM_GLOBAL_CACHING_ENABLE, true);
+            Self { qmd }
+        }
+
+        qmd_impl_common!(clc0c0, QMDV02_01);
+        qmd_impl_set_crs_size!(clc0c0, QMDV02_01);
+        qmd_impl_set_cbuf!(clc0c0, QMDV02_01, NONE, SHIFTED4);
+        qmd_impl_set_prog_addr_32!(clc0c0, QMDV02_01);
+        qmd_impl_set_register_count!(clc0c0, QMDV02_01, REGISTER_COUNT);
+        qmd_impl_set_slm_size!(clc0c0, QMDV02_01, NONE);
+
+        fn set_smem_size(
+            &mut self,
+            smem_size: u32,
+            _dev: &nv_device_info,
+            _info: &nak_shader_info,
+        ) {
+            let mut bv = QMDBitView::new(&mut self.qmd);
+
+            let smem_size = smem_size.next_multiple_of(0x100);
+            set_field!(bv, clc0c0, QMDV02_01, SHARED_MEMORY_SIZE, smem_size);
+        }
+    }
+}
+use qmd_2_1::Qmd2_1;
+
+fn gv100_smem_size_to_hw(size_kb: u16) -> u16 {
+    assert!(size_kb % 4 == 0);
+    (size_kb / 4) + 1
+}
+
+/// Returns the SM_CONFIG_SHARED_MEM_SIZE for the given workgroups and shared
+/// memory size.
+fn gv100_pick_smem_size_kb(
+    size: u32,
+    smem_sizes_kb: &[u16],
+    workgroups: u32,
+) -> u16 {
+    let highest = *smem_sizes_kb.last().unwrap();
+    // How many workgroups can we launch with the requested shared mem size?
+    let workgroups = min(workgroups, u32::from(highest) * 1024 / size.max(1));
+    assert!(
+        workgroups > 0,
+        "Requested shared memory not supported by the hw."
+    );
+
+    // Now that we know for how many workgroups we have shared memory, try to
+    // find the smallest allocation that fits.
+    *smem_sizes_kb
+        .iter()
+        .find(|&&val| u32::from(val) * 1024 >= size * workgroups)
+        .unwrap_or(&highest)
+}
+
+/// Returns the (MIN_, TARGET_, MAX_) SM_CONFIG_SHARED_MEM_SIZE values.
+fn gv100_get_hw_smem_sizes(
+    smem_size: u32,
+    dev: &nv_device_info,
+    info: &nak_shader_info,
+) -> (u16, u16, u16) {
+    let cs_info = unsafe {
+        assert!(info.stage == MESA_SHADER_COMPUTE);
+        &info.__bindgen_anon_1.cs
+    };
+
+    let smem_sizes = &dev.sm_smem_sizes_kB[0..dev.sm_smem_size_count.into()];
+    let threads =
+        cs_info.local_size[0] * cs_info.local_size[1] * cs_info.local_size[2];
+    let warps = threads.div_ceil(32);
+    let workgroups_per_sm = info.max_warps_per_sm / u32::from(warps);
+
+    let min = gv100_pick_smem_size_kb(smem_size, smem_sizes, 1);
+    let target =
+        gv100_pick_smem_size_kb(smem_size, smem_sizes, workgroups_per_sm);
+    let max = *smem_sizes.last().unwrap();
+
+    (
+        gv100_smem_size_to_hw(min),
+        gv100_smem_size_to_hw(target),
+        gv100_smem_size_to_hw(max),
+    )
+}
+
+macro_rules! qmd_impl_set_smem_size_bounded {
+    ($c:ident, $s:ident) => {
+        fn set_smem_size(
+            &mut self,
+            smem_size: u32,
+            dev: &nv_device_info,
+            info: &nak_shader_info,
+        ) {
+            let mut bv = QMDBitView::new(&mut self.qmd);
+
+            let smem_size = smem_size.next_multiple_of(0x100);
+            set_field!(bv, $c, $s, SHARED_MEMORY_SIZE, smem_size);
+
+            let (min, target, max) =
+                gv100_get_hw_smem_sizes(smem_size, dev, info);
+
+            set_field!(bv, $c, $s, MIN_SM_CONFIG_SHARED_MEM_SIZE, min);
+            set_field!(bv, $c, $s, MAX_SM_CONFIG_SHARED_MEM_SIZE, max);
+            set_field!(bv, $c, $s, TARGET_SM_CONFIG_SHARED_MEM_SIZE, target);
+        }
+    };
+}
+
+macro_rules! qmd_impl_set_smem_size_bounded_gb {
+    ($c:ident, $s:ident) => {
+        fn set_smem_size(
+            &mut self,
+            smem_size: u32,
+            dev: &nv_device_info,
+            info: &nak_shader_info,
+        ) {
+            let mut bv = QMDBitView::new(&mut self.qmd);
+
+            let smem_size = smem_size.next_multiple_of(0x100);
+            let size_shift = 7;
+            let smem_size_shifted = smem_size >> size_shift;
+            assert!((smem_size_shifted << size_shift) == smem_size);
+            set_field!(
+                bv,
+                $c,
+                $s,
+                SHARED_MEMORY_SIZE_SHIFTED7,
+                smem_size_shifted
+            );
+
+            let (min, target, max) =
+                gv100_get_hw_smem_sizes(smem_size, dev, info);
+
+            set_field!(bv, $c, $s, MIN_SM_CONFIG_SHARED_MEM_SIZE, min);
+            set_field!(bv, $c, $s, MAX_SM_CONFIG_SHARED_MEM_SIZE, max);
+            set_field!(bv, $c, $s, TARGET_SM_CONFIG_SHARED_MEM_SIZE, target);
+        }
+    };
+}
+
+mod qmd_2_2 {
+    use crate::qmd::*;
+    use nvidia_headers::classes::clc3c0::qmd as clc3c0;
+
+    #[repr(transparent)]
+    pub struct Qmd2_2 {
+        qmd: [u32; 64],
+    }
+
+    impl QMD for Qmd2_2 {
+        fn new() -> Self {
+            let mut qmd = [0; 64];
+            let mut bv = QMDBitView::new(&mut qmd);
+            qmd_init!(bv, clc3c0, QMDV02_02, 2, 2);
+            set_field!(bv, clc3c0, QMDV02_02, SM_GLOBAL_CACHING_ENABLE, true);
+            Self { qmd }
+        }
+
+        qmd_impl_common!(clc3c0, QMDV02_02);
+        qmd_impl_set_crs_size!(clc3c0, QMDV02_02);
+        qmd_impl_set_cbuf!(clc3c0, QMDV02_02, NONE, SHIFTED4);
+        qmd_impl_set_prog_addr_64!(clc3c0, QMDV02_02, NONE);
+        qmd_impl_set_register_count!(clc3c0, QMDV02_02, REGISTER_COUNT_V);
+        qmd_impl_set_smem_size_bounded!(clc3c0, QMDV02_02);
+        qmd_impl_set_slm_size!(clc3c0, QMDV02_02, NONE);
+    }
+}
+use qmd_2_2::Qmd2_2;
+
+mod qmd_3_0 {
+    use crate::qmd::*;
+    use nvidia_headers::classes::clc6c0::qmd as clc6c0;
+
+    #[repr(transparent)]
+    pub struct Qmd3_0 {
+        qmd: [u32; 64],
+    }
+
+    impl QMD for Qmd3_0 {
+        fn new() -> Self {
+            let mut qmd = [0; 64];
+            let mut bv = QMDBitView::new(&mut qmd);
+            qmd_init!(bv, clc6c0, QMDV03_00, 3, 0);
+            set_field!(bv, clc6c0, QMDV03_00, SM_GLOBAL_CACHING_ENABLE, true);
+            Self { qmd }
+        }
+
+        qmd_impl_common!(clc6c0, QMDV03_00);
+
+        fn set_crs_size(&mut self, crs_size: u32) {
+            assert!(crs_size == 0);
+        }
+
+        qmd_impl_set_cbuf!(clc6c0, QMDV03_00, NONE, SHIFTED4);
+        qmd_impl_set_prog_addr_64!(clc6c0, QMDV03_00, NONE);
+        qmd_impl_set_register_count!(clc6c0, QMDV03_00, REGISTER_COUNT_V);
+        qmd_impl_set_smem_size_bounded!(clc6c0, QMDV03_00);
+        qmd_impl_set_slm_size!(clc6c0, QMDV03_00, NONE);
+    }
+}
+use qmd_3_0::Qmd3_0;
+
+mod qmd_4_0 {
+    use crate::qmd::*;
+    mod clcbc0 {
+        pub use nvidia_headers::classes::clcbc0::qmd::*;
+
+        // Some renames we have to carry for Hopper
+        pub use QMDV04_00_GRID_DEPTH as QMDV04_00_CTA_RASTER_DEPTH;
+        pub use QMDV04_00_GRID_HEIGHT as QMDV04_00_CTA_RASTER_HEIGHT;
+        pub use QMDV04_00_GRID_WIDTH as QMDV04_00_CTA_RASTER_WIDTH;
+        pub use QMDV04_00_QMD_MINOR_VERSION as QMDV04_00_QMD_VERSION;
+    }
+
+    #[repr(transparent)]
+    pub struct Qmd4_0 {
+        qmd: [u32; 96],
+    }
+
+    impl QMD for Qmd4_0 {
+        fn new() -> Self {
+            let mut qmd = [0; 96];
+            let mut bv = QMDBitView::new(&mut qmd);
+            qmd_init!(bv, clcbc0, QMDV04_00, 4, 0);
+            Self { qmd }
+        }
+
+        qmd_impl_common!(clcbc0, QMDV04_00);
+
+        fn set_crs_size(&mut self, crs_size: u32) {
+            assert!(crs_size == 0);
+        }
+
+        qmd_impl_set_cbuf!(clcbc0, QMDV04_00, SHIFTED6, SHIFTED4);
+        qmd_impl_set_prog_addr_64!(clcbc0, QMDV04_00, NONE);
+        qmd_impl_set_register_count!(clcbc0, QMDV04_00, REGISTER_COUNT);
+        qmd_impl_set_smem_size_bounded!(clcbc0, QMDV04_00);
+        qmd_impl_set_slm_size!(clcbc0, QMDV04_00, NONE);
+    }
+}
+use qmd_4_0::Qmd4_0;
+
+mod qmd_5_0 {
+    use crate::qmd::*;
+    // Blackwell_A is CD97, Blackwell_B is CE97, however if Hopper is v4 and
+    // Blackwell_B is v5, then Blackwell_A also has to be v5 hence we'll use
+    // clcdc0qmd here
+    mod clcdc0 {
+        pub use nvidia_headers::classes::clcdc0::qmd::*;
+
+        // Some renames we may have to carry for Blackwell
+        pub use QMDV05_00_GRID_DEPTH as QMDV05_00_CTA_RASTER_DEPTH;
+        pub use QMDV05_00_GRID_HEIGHT as QMDV05_00_CTA_RASTER_HEIGHT;
+        pub use QMDV05_00_GRID_WIDTH as QMDV05_00_CTA_RASTER_WIDTH;
+        pub use QMDV05_00_QMD_MINOR_VERSION as QMDV05_00_QMD_VERSION;
+    }
+
+    pub struct Qmd5_0 {
+        qmd: [u32; 96],
+    }
+
+    impl QMD for Qmd5_0 {
+        fn new() -> Self {
+            let mut qmd = [0; 96];
+            let mut bv = QMDBitView::new(&mut qmd);
+            qmd_init!(bv, clcdc0, QMDV05_00, 5, 0);
+            set_field!(bv, clcdc0, QMDV05_00, QMD_TYPE, 0x2);
+            set_field!(bv, clcdc0, QMDV05_00, QMD_GROUP_ID, 0x1f);
+            Self { qmd }
+        }
+
+        qmd_impl_common!(clcdc0, QMDV05_00);
+
+        fn set_crs_size(&mut self, crs_size: u32) {
+            assert!(crs_size == 0);
+        }
+
+        qmd_impl_set_cbuf!(clcdc0, QMDV05_00, SHIFTED6, SHIFTED4);
+        qmd_impl_set_prog_addr_64!(clcdc0, QMDV05_00, SHIFTED4);
+        qmd_impl_set_register_count!(clcdc0, QMDV05_00, REGISTER_COUNT);
+        qmd_impl_set_smem_size_bounded_gb!(clcdc0, QMDV05_00);
+        qmd_impl_set_slm_size!(clcdc0, QMDV05_00, SHIFTED4);
+    }
+}
+use qmd_5_0::Qmd5_0;
+
+fn fill_qmd<Q: QMD>(
+    dev: &nv_device_info,
+    info: &nak_shader_info,
+    qmd_info: &nak_qmd_info,
+) -> Q {
+    let cs_info = unsafe {
+        assert!(info.stage == MESA_SHADER_COMPUTE);
+        &info.__bindgen_anon_1.cs
+    };
+
+    let mut qmd = Q::new();
+
+    qmd.set_barrier_count(info.num_control_barriers);
+    qmd.set_global_size(
+        qmd_info.global_size[0],
+        qmd_info.global_size[1],
+        qmd_info.global_size[2],
+    );
+    qmd.set_local_size(
+        cs_info.local_size[0],
+        cs_info.local_size[1],
+        cs_info.local_size[2],
+    );
+    qmd.set_prog_addr(qmd_info.addr);
+    qmd.set_register_count(info.num_gprs);
+    qmd.set_crs_size(info.crs_size);
+    qmd.set_slm_size(info.slm_size);
+
+    assert!(qmd_info.smem_size <= u32::from(dev.max_smem_per_wg_kB) * 1024);
+    qmd.set_smem_size(qmd_info.smem_size, dev, info);
+
+    for i in 0..qmd_info.num_cbufs {
+        let cb = &qmd_info.cbufs[usize::try_from(i).unwrap()];
+        if cb.size > 0 {
+            qmd.set_cbuf(cb.index.try_into().unwrap(), cb.addr, cb.size);
+        }
+    }
+
+    qmd
+}
+
+#[no_mangle]
+pub extern "C" fn nak_qmd_size_B(dev: &nv_device_info) -> u32 {
+    let size_B = if dev.cls_compute >= clcdc0::BLACKWELL_COMPUTE_A {
+        size_of::<Qmd5_0>().try_into().unwrap()
+    } else if dev.cls_compute >= clcbc0::HOPPER_COMPUTE_A {
+        size_of::<Qmd4_0>().try_into().unwrap()
+    } else if dev.cls_compute >= clc6c0::AMPERE_COMPUTE_A {
+        size_of::<Qmd3_0>().try_into().unwrap()
+    } else if dev.cls_compute >= clc3c0::VOLTA_COMPUTE_A {
+        size_of::<Qmd2_2>().try_into().unwrap()
+    } else if dev.cls_compute >= clc0c0::PASCAL_COMPUTE_A {
+        size_of::<Qmd2_1>().try_into().unwrap()
+    } else if dev.cls_compute >= cla0c0::KEPLER_COMPUTE_A {
+        size_of::<Qmd0_6>().try_into().unwrap()
+    } else {
+        panic!("Unknown shader model");
+    };
+    assert!(size_B <= NAK_MAX_QMD_SIZE_B);
+    size_B
+}
+
+#[no_mangle]
+pub extern "C" fn nak_fill_qmd(
+    dev: *const nv_device_info,
+    info: *const nak_shader_info,
+    qmd_info: *const nak_qmd_info,
+    qmd_out: *mut ::std::os::raw::c_void,
+    qmd_size: usize,
+) {
+    assert!(!dev.is_null());
+    let dev = unsafe { &*dev };
+
+    assert!(!info.is_null());
+    let info = unsafe { &*info };
+
+    assert!(!qmd_info.is_null());
+    let qmd_info = unsafe { &*qmd_info };
+
+    unsafe {
+        if dev.cls_compute >= clcdc0::BLACKWELL_COMPUTE_A {
+            let qmd_out = qmd_out as *mut Qmd5_0;
+            assert!(qmd_size == size_of_val(&*qmd_out));
+            qmd_out.write(fill_qmd(dev, info, qmd_info));
+        } else if dev.cls_compute >= clcbc0::HOPPER_COMPUTE_A {
+            let qmd_out = qmd_out as *mut Qmd4_0;
+            assert!(qmd_size == size_of_val(&*qmd_out));
+            qmd_out.write(fill_qmd(dev, info, qmd_info));
+        } else if dev.cls_compute >= clc6c0::AMPERE_COMPUTE_A {
+            let qmd_out = qmd_out as *mut Qmd3_0;
+            assert!(qmd_size == size_of_val(&*qmd_out));
+            qmd_out.write(fill_qmd(dev, info, qmd_info));
+        } else if dev.cls_compute >= clc3c0::VOLTA_COMPUTE_A {
+            let qmd_out = qmd_out as *mut Qmd2_2;
+            assert!(qmd_size == size_of_val(&*qmd_out));
+            qmd_out.write(fill_qmd(dev, info, qmd_info));
+        } else if dev.cls_compute >= clc0c0::PASCAL_COMPUTE_A {
+            let qmd_out = qmd_out as *mut Qmd2_1;
+            assert!(qmd_size == size_of_val(&*qmd_out));
+            qmd_out.write(fill_qmd(dev, info, qmd_info));
+        } else if dev.cls_compute >= cla0c0::KEPLER_COMPUTE_A {
+            let qmd_out = qmd_out as *mut Qmd0_6;
+            assert!(qmd_size == size_of_val(&*qmd_out));
+            qmd_out.write(fill_qmd(dev, info, qmd_info));
+        } else {
+            panic!("Unknown shader model");
+        }
+    }
+}
+
+#[no_mangle]
+pub extern "C" fn nak_get_qmd_dispatch_size_layout(
+    dev: &nv_device_info,
+) -> nak_qmd_dispatch_size_layout {
+    if dev.cls_compute >= clcdc0::BLACKWELL_COMPUTE_A {
+        Qmd5_0::GLOBAL_SIZE_LAYOUT
+    } else if dev.cls_compute >= clcbc0::HOPPER_COMPUTE_A {
+        Qmd4_0::GLOBAL_SIZE_LAYOUT
+    } else if dev.cls_compute >= clc6c0::AMPERE_COMPUTE_A {
+        Qmd3_0::GLOBAL_SIZE_LAYOUT
+    } else if dev.cls_compute >= clc3c0::VOLTA_COMPUTE_A {
+        Qmd2_2::GLOBAL_SIZE_LAYOUT
+    } else if dev.cls_compute >= clc0c0::PASCAL_COMPUTE_A {
+        Qmd2_1::GLOBAL_SIZE_LAYOUT
+    } else if dev.cls_compute >= cla0c0::KEPLER_COMPUTE_A {
+        Qmd0_6::GLOBAL_SIZE_LAYOUT
+    } else {
+        panic!("Unsupported shader model");
+    }
+}
+
+#[no_mangle]
+pub extern "C" fn nak_get_qmd_cbuf_desc_layout(
+    dev: &nv_device_info,
+    idx: u8,
+) -> nak_qmd_cbuf_desc_layout {
+    if dev.cls_compute >= clcdc0::BLACKWELL_COMPUTE_A {
+        Qmd5_0::cbuf_desc_layout(idx)
+    } else if dev.cls_compute >= clcbc0::HOPPER_COMPUTE_A {
+        Qmd4_0::cbuf_desc_layout(idx)
+    } else if dev.cls_compute >= clc6c0::AMPERE_COMPUTE_A {
+        Qmd3_0::cbuf_desc_layout(idx)
+    } else if dev.cls_compute >= clc3c0::VOLTA_COMPUTE_A {
+        Qmd2_2::cbuf_desc_layout(idx)
+    } else if dev.cls_compute >= clc0c0::PASCAL_COMPUTE_A {
+        Qmd2_1::cbuf_desc_layout(idx)
+    } else if dev.cls_compute >= cla0c0::KEPLER_COMPUTE_A {
+        Qmd0_6::cbuf_desc_layout(idx)
+    } else {
+        panic!("Unsupported shader model");
+    }
+}
@@ -0,0 +1,186 @@
+// Copyright © 2022 Collabora, Ltd.
+// SPDX-License-Identifier: MIT
+
+use rustc_hash::FxHashMap;
+
+use crate::ir::*;
+
+use std::ops::{Index, IndexMut, Range};
+
+pub struct RegTracker<T> {
+    reg: [T; 255],
+    ureg: [T; 80],
+    pred: [T; 7],
+    upred: [T; 7],
+    carry: [T; 1],
+}
+
+fn new_array_with<T, const N: usize>(f: &impl Fn() -> T) -> [T; N] {
+    let mut v = Vec::with_capacity(N);
+    for _ in 0..N {
+        v.push(f());
+    }
+    v.try_into()
+        .unwrap_or_else(|_| panic!("Array size mismatch"))
+}
+
+impl<T> RegTracker<T> {
+    pub fn new_with(f: &impl Fn() -> T) -> Self {
+        Self {
+            reg: new_array_with(f),
+            ureg: new_array_with(f),
+            pred: new_array_with(f),
+            upred: new_array_with(f),
+            carry: new_array_with(f),
+        }
+    }
+}
+
+impl<T> Index<RegRef> for RegTracker<T> {
+    type Output = [T];
+
+    fn index(&self, reg: RegRef) -> &[T] {
+        let range = reg.idx_range();
+        let range = Range {
+            start: usize::try_from(range.start).unwrap(),
+            end: usize::try_from(range.end).unwrap(),
+        };
+
+        match reg.file() {
+            RegFile::GPR => &self.reg[range],
+            RegFile::UGPR => &self.ureg[range],
+            RegFile::Pred => &self.pred[range],
+            RegFile::UPred => &self.upred[range],
+            RegFile::Carry => &self.carry[range],
+            RegFile::Bar => &[], // Barriers have a HW scoreboard
+            RegFile::Mem => panic!("Not a register"),
+        }
+    }
+}
+
+impl<T> IndexMut<RegRef> for RegTracker<T> {
+    fn index_mut(&mut self, reg: RegRef) -> &mut [T] {
+        let range = reg.idx_range();
+        let range = Range {
+            start: usize::try_from(range.start).unwrap(),
+            end: usize::try_from(range.end).unwrap(),
+        };
+
+        match reg.file() {
+            RegFile::GPR => &mut self.reg[range],
+            RegFile::UGPR => &mut self.ureg[range],
+            RegFile::Pred => &mut self.pred[range],
+            RegFile::UPred => &mut self.upred[range],
+            RegFile::Carry => &mut self.carry[range],
+            RegFile::Bar => &mut [], // Barriers have a HW scoreboard
+            RegFile::Mem => panic!("Not a register"),
+        }
+    }
+}
+
+/// Memory-light version of [RegTracker].
+///
+/// This version uses sparse hashmaps instead of dense arrays.
+#[derive(Clone, PartialEq, Eq, Default)]
+pub struct SparseRegTracker<T: Default> {
+    regs: FxHashMap<RegRef, T>,
+}
+
+impl<T: Default + Clone + Eq> SparseRegTracker<T> {
+    pub fn for_each_pred(&mut self, f: impl FnMut(&mut T)) {
+        self.for_each_ref_mut(RegRef::new(RegFile::Pred, 0, 7), f);
+    }
+
+    pub fn for_each_carry(&mut self, f: impl FnMut(&mut T)) {
+        self.for_each_ref_mut(RegRef::new(RegFile::Carry, 0, 1), f);
+    }
+
+    pub fn merge_with(&mut self, other: &Self, mut f: impl FnMut(&mut T, &T)) {
+        use std::collections::hash_map::Entry;
+
+        for (k, v) in other.regs.iter() {
+            match self.regs.entry(*k) {
+                Entry::Occupied(mut occupied_entry) => {
+                    f(occupied_entry.get_mut(), v);
+                }
+                Entry::Vacant(vacant_entry) => {
+                    vacant_entry.insert((*v).clone());
+                }
+            }
+        }
+    }
+
+    pub fn retain(&mut self, mut f: impl FnMut(&mut T) -> bool) {
+        self.regs.retain(|_k, v| f(v));
+    }
+}
+
+/// Common behavior for [RegTracker] and [SparseRegTracker]
+pub trait RegRefIterable<T> {
+    fn for_each_ref_mut(&mut self, reg: RegRef, f: impl FnMut(&mut T));
+
+    fn for_each_instr_pred_mut(
+        &mut self,
+        instr: &Instr,
+        mut f: impl FnMut(&mut T),
+    ) {
+        if let PredRef::Reg(reg) = &instr.pred.pred_ref {
+            self.for_each_ref_mut(*reg, |t| f(t));
+        }
+    }
+
+    fn for_each_instr_src_mut(
+        &mut self,
+        instr: &Instr,
+        mut f: impl FnMut(usize, &mut T),
+    ) {
+        for (i, src) in instr.srcs().iter().enumerate() {
+            match &src.src_ref {
+                SrcRef::Reg(reg) => {
+                    self.for_each_ref_mut(*reg, |t| f(i, t));
+                }
+                SrcRef::CBuf(CBufRef {
+                    buf: CBuf::BindlessUGPR(reg),
+                    ..
+                }) => {
+                    self.for_each_ref_mut(*reg, |t| f(i, t));
+                }
+                _ => (),
+            }
+        }
+    }
+
+    fn for_each_instr_dst_mut(
+        &mut self,
+        instr: &Instr,
+        mut f: impl FnMut(usize, &mut T),
+    ) {
+        for (i, dst) in instr.dsts().iter().enumerate() {
+            if let Dst::Reg(reg) = dst {
+                self.for_each_ref_mut(*reg, |t| f(i, t));
+            }
+        }
+    }
+}
+
+impl<T: Default> RegRefIterable<T> for SparseRegTracker<T> {
+    fn for_each_ref_mut(&mut self, reg: RegRef, mut f: impl FnMut(&mut T)) {
+        match reg.file() {
+            RegFile::Bar => return, // Barriers have a HW scoreboard
+            RegFile::Mem => panic!("Not a register"),
+            _ => {}
+        }
+
+        for i in 0..reg.comps() {
+            f(self.regs.entry(reg.comp(i)).or_default());
+        }
+    }
+}
+
+impl<T> RegRefIterable<T> for RegTracker<T> {
+    fn for_each_ref_mut(&mut self, reg: RegRef, mut f: impl FnMut(&mut T)) {
+        for entry in &mut self[reg] {
+            f(entry);
+        }
+    }
+}
@@ -0,0 +1,346 @@
+// Copyright © 2023 Collabora, Ltd.
+// SPDX-License-Identifier: MIT
+
+use crate::ir::*;
+use crate::union_find::UnionFind;
+
+use compiler::bitset::BitSet;
+use rustc_hash::{FxBuildHasher, FxHashMap};
+use std::cell::RefCell;
+use std::cmp::Reverse;
+use std::collections::BinaryHeap;
+
+struct PhiTracker {
+    phi: Phi,
+    orig: SSAValue,
+    dst: SSAValue,
+    srcs: FxHashMap<usize, SSAValue>,
+}
+
+struct DefTrackerBlock {
+    pred: Vec<usize>,
+    succ: Vec<usize>,
+    defs: RefCell<FxHashMap<SSAValue, SSAValue>>,
+    phis: RefCell<Vec<PhiTracker>>,
+}
+
+fn get_ssa_or_phi(
+    worklist: &mut BinaryHeap<Reverse<usize>>,
+    ssa_alloc: &mut SSAValueAllocator,
+    phi_alloc: &mut PhiAllocator,
+    blocks: &[DefTrackerBlock],
+    needs_src: &mut BitSet,
+    b_idx: usize,
+    ssa: SSAValue,
+) -> SSAValue {
+    // Annoyingly, Rust stack sizes get to be a problem so we don't want to use
+    // actual recursion here.  Instead we use a worklist in the form of a binary
+    // heap.  Using a binary heap ensures that we process the earliest blocks
+    // first.
+    debug_assert!(worklist.is_empty());
+    worklist.push(Reverse(b_idx));
+
+    loop {
+        let b_idx = worklist.peek().unwrap().0;
+        let b = &blocks[b_idx];
+
+        if let Some(&b_ssa) = b.defs.borrow().get(&ssa) {
+            // We already sorted this one out, pop the stack.
+            worklist.pop();
+            if worklist.is_empty() {
+                return b_ssa;
+            } else {
+                continue;
+            }
+        }
+
+        let mut pushed_pred = false;
+        let mut pred_ssa = None;
+        let mut all_same = true;
+        for &p_idx in &b.pred {
+            if p_idx >= b_idx {
+                // This is a loop back-edge, add a phi just in case.  We'll
+                // remove it later if it's not needed
+                all_same = false;
+            } else if let Some(&p_ssa) = blocks[p_idx].defs.borrow().get(&ssa) {
+                if *pred_ssa.get_or_insert(p_ssa) != p_ssa {
+                    all_same = false;
+                }
+            } else {
+                worklist.push(Reverse(p_idx));
+                pushed_pred = true;
+            }
+        }
+
+        // If we pushed any predecessors to the stack, loop again to sort them
+        // out before we try to sort out this block.
+        if pushed_pred {
+            continue;
+        }
+
+        // We now have everything we need to sort out this block
+        let b_ssa = if all_same {
+            pred_ssa.expect("Undefined value")
+        } else {
+            let phi = phi_alloc.alloc();
+            let phi_ssa = ssa_alloc.alloc(ssa.file());
+            let mut pt = PhiTracker {
+                phi,
+                orig: ssa,
+                dst: phi_ssa,
+                srcs: Default::default(),
+            };
+            for &p_idx in &b.pred {
+                if p_idx >= b_idx {
+                    needs_src.insert(p_idx);
+                    continue;
+                }
+                // Earlier iterations of the loop ensured this exists
+                let p_ssa = *blocks[p_idx].defs.borrow().get(&ssa).unwrap();
+                pt.srcs.insert(p_idx, p_ssa);
+            }
+            blocks[b_idx].phis.borrow_mut().push(pt);
+            phi_ssa
+        };
+
+        blocks[b_idx].defs.borrow_mut().insert(ssa, b_ssa);
+        worklist.pop();
+        if worklist.is_empty() {
+            return b_ssa;
+        }
+    }
+}
+
+fn get_or_insert_phi_dsts(bb: &mut BasicBlock) -> &mut OpPhiDsts {
+    let ip = if let Some(ip) = bb.phi_dsts_ip() {
+        ip
+    } else {
+        bb.instrs.insert(0, Instr::new(OpPhiDsts::new()));
+        0
+    };
+    match &mut bb.instrs[ip].op {
+        Op::PhiDsts(op) => op,
+        _ => panic!("Expected to find the phi we just inserted"),
+    }
+}
+
+fn get_or_insert_phi_srcs(bb: &mut BasicBlock) -> &mut OpPhiSrcs {
+    let ip = if let Some(ip) = bb.phi_srcs_ip() {
+        ip
+    } else if let Some(ip) = bb.branch_ip() {
+        bb.instrs.insert(ip, Instr::new(OpPhiSrcs::new()));
+        ip
+    } else {
+        bb.instrs.push(Instr::new(OpPhiSrcs::new()));
+        bb.instrs.len() - 1
+    };
+    match &mut bb.instrs[ip].op {
+        Op::PhiSrcs(op) => op,
+        _ => panic!("Expected to find the phi we just inserted"),
+    }
+}
+
+impl Function {
+    /// Repairs SSA form
+    ///
+    /// Certain passes such as register spilling may produce a program that is
+    /// no longer in SSA form.  This pass is able to repair SSA by inserting
+    /// phis as needed.  Even though we do not require dominance or that each
+    /// value be defined once we do require that, for every use of an SSAValue
+    /// and for every path from the start of the program to that use, there must
+    /// be some definition of the value along that path.
+    ///
+    /// The algorithm implemented here is based on the one in "Simple and
+    /// Efficient Construction of Static Single Assignment Form" by Braun, et.
+    /// al.  The primary difference between our implementation and the paper is
+    /// that we can't rewrite the IR on-the-fly.  Instead, we store everything
+    /// in hash tables and handle removing redundant phis with back-edges as a
+    /// separate pass between figuring out where phis are needed and actually
+    /// constructing the phi instructions.
+    pub fn repair_ssa(&mut self) {
+        // First, count the number of defs for each SSA value.  This will allow
+        // us to skip any SSA values which only have a single definition in
+        // later passes.
+        let mut has_mult_defs = false;
+        let mut num_defs = FxHashMap::default();
+        for b in &self.blocks {
+            for instr in &b.instrs {
+                instr.for_each_ssa_def(|ssa| {
+                    num_defs
+                        .entry(*ssa)
+                        .and_modify(|e| {
+                            has_mult_defs = true;
+                            *e += 1;
+                        })
+                        .or_insert(1);
+                });
+            }
+        }
+
+        if !has_mult_defs {
+            return;
+        }
+
+        let cfg = &mut self.blocks;
+        let ssa_alloc = &mut self.ssa_alloc;
+        let phi_alloc = &mut self.phi_alloc;
+
+        let mut blocks = Vec::new();
+        let mut needs_src = BitSet::new();
+        let mut ssa_or_phi_worklist = BinaryHeap::new();
+        for b_idx in 0..cfg.len() {
+            assert!(blocks.len() == b_idx);
+            blocks.push(DefTrackerBlock {
+                pred: cfg.pred_indices(b_idx).to_vec(),
+                succ: cfg.succ_indices(b_idx).to_vec(),
+                defs: RefCell::new(Default::default()),
+                phis: RefCell::new(Vec::new()),
+            });
+
+            for instr in &mut cfg[b_idx].instrs {
+                instr.for_each_ssa_use_mut(|ssa| {
+                    if num_defs.get(ssa).cloned().unwrap_or(0) > 1 {
+                        *ssa = get_ssa_or_phi(
+                            &mut ssa_or_phi_worklist,
+                            ssa_alloc,
+                            phi_alloc,
+                            &blocks,
+                            &mut needs_src,
+                            b_idx,
+                            *ssa,
+                        );
+                    }
+                });
+
+                instr.for_each_ssa_def_mut(|ssa| {
+                    if num_defs.get(ssa).cloned().unwrap_or(0) > 1 {
+                        let new_ssa = ssa_alloc.alloc(ssa.file());
+                        blocks[b_idx].defs.borrow_mut().insert(*ssa, new_ssa);
+                        *ssa = new_ssa;
+                    }
+                });
+            }
+        }
+
+        // Populate phi sources for any back-edges
+        loop {
+            let Some(b_idx) = needs_src.iter().next() else {
+                break;
+            };
+            needs_src.remove(b_idx);
+
+            for s_idx in &blocks[b_idx].succ {
+                if *s_idx <= b_idx {
+                    let s = &blocks[*s_idx];
+
+                    // We do a mutable borrow here.  The algorithm is recursive
+                    // and may insert phis into other blocks.  However, because
+                    // this is phi exists, its destination should be in the def
+                    // set for s and so no new phis should need to be added.
+                    // RefCell's dynamic borrow checks will assert this.
+                    for phi in s.phis.borrow_mut().iter_mut() {
+                        phi.srcs.entry(b_idx).or_insert_with(|| {
+                            get_ssa_or_phi(
+                                &mut ssa_or_phi_worklist,
+                                ssa_alloc,
+                                phi_alloc,
+                                &blocks,
+                                &mut needs_src,
+                                b_idx,
+                                phi.orig,
+                            )
+                        });
+                    }
+                }
+            }
+        }
+
+        // For loop back-edges, we inserted a phi whether we need one or not.
+        // We want to eliminate any redundant phis.
+        let mut ssa_map = UnionFind::<SSAValue, FxBuildHasher>::new();
+        if cfg.has_loop() {
+            let mut to_do = true;
+            while to_do {
+                to_do = false;
+                for b_idx in 0..cfg.len() {
+                    let b = &blocks[b_idx];
+                    b.phis.borrow_mut().retain_mut(|phi| {
+                        let mut ssa = None;
+                        for (_, p_ssa) in phi.srcs.iter_mut() {
+                            // Apply the remap to the phi sources so that we
+                            // pick up any remaps from previous loop iterations.
+                            *p_ssa = ssa_map.find(*p_ssa);
+
+                            if *p_ssa == phi.dst {
+                                continue;
+                            }
+                            if *ssa.get_or_insert(*p_ssa) != *p_ssa {
+                                // Multiple unique sources
+                                return true;
+                            }
+                        }
+
+                        // All sources are identical or the phi destination so
+                        // we can delete this phi and add it to the remap
+                        let ssa = ssa.expect("Circular SSA def");
+                        // union(a, b) ensures that the representative is the representative
+                        // for a.  This means union(ssa, phi.dst) ensures that phi.dst gets
+                        // mapped to ssa, not the other way around.
+                        ssa_map.union(ssa, phi.dst);
+                        to_do = true;
+                        false
+                    });
+                }
+            }
+        }
+
+        // Now we apply the remap to instruction sources and place the actual
+        // phis
+        for b_idx in 0..cfg.len() {
+            // Grab the successor index for inserting OpPhiSrc before we take a
+            // mutable reference to the CFG.  There are no critical edges so we
+            // can only have an OpPhiSrc if there is a single successor.
+            let succ = cfg.succ_indices(b_idx);
+            let s_idx = if succ.len() == 1 {
+                Some(succ[0])
+            } else {
+                for s_idx in succ {
+                    debug_assert!(blocks[*s_idx].phis.borrow().is_empty());
+                }
+                None
+            };
+
+            let bb = &mut cfg[b_idx];
+
+            // First we have phi destinations
+            let b_phis = blocks[b_idx].phis.borrow();
+            if !b_phis.is_empty() {
+                let phi_dst = get_or_insert_phi_dsts(bb);
+                for pt in b_phis.iter() {
+                    phi_dst.dsts.push(pt.phi, pt.dst.into());
+                }
+            }
+
+            // Fix up any remapped SSA values in sources
+            if !ssa_map.is_empty() {
+                for instr in &mut bb.instrs {
+                    instr.for_each_ssa_use_mut(|ssa| {
+                        *ssa = ssa_map.find(*ssa);
+                    });
+                }
+            }
+
+            if let Some(s_idx) = s_idx {
+                let s_phis = blocks[s_idx].phis.borrow();
+                if !s_phis.is_empty() {
+                    let phi_src = get_or_insert_phi_srcs(bb);
+                    for pt in s_phis.iter() {
+                        let mut ssa = *pt.srcs.get(&b_idx).unwrap();
+                        ssa = ssa_map.find(ssa);
+                        phi_src.srcs.push(pt.phi, ssa.into());
+                    }
+                }
+            }
+        }
+    }
+}
@@ -0,0 +1,561 @@
+// Copyright © 2025 Red Hat.
+// SPDX-License-Identifier: MIT
+#![allow(non_camel_case_types)]
+
+use crate::ir::*;
+
+use nak_latencies::sm100::*;
+
+// This contains the register scheduling information provided by NVIDIA.  This
+// file is for Blackwell only.
+//
+// These latencies come from B100 (SM100) and not the consumer RTX chips
+// (SM120).  We have to add some padding to get everything passing on the RTX
+// chips so that's done in this file while using the sm100 CSVs.
+
+// Coupled instructions are ones with fixed latencies, they need delays but not
+// scoreboards.  Decoupled instructions are ones with variable latencies, need
+// scoreboards but not delays.  There are also redirected instructions which
+// depending on the SM, can be coupled or Decoupled so both delays and
+// scoreboards needs to be provided.
+
+fn op_reg_latency(op: &Op, reader: bool, op_reg_idx: usize) -> RegLatencySM100 {
+    use RegLatencySM100::*;
+    match op {
+        // this will need updating if imad grows support for input predicates
+        Op::IMad(_) | Op::IMul(_) => Fma,
+        Op::IMad64(_) => {
+            if reader {
+                match op_reg_idx {
+                    0 | 1 => ImadWideReadAb,
+                    2 => ImadWideReadCl, // vs upper C operand - work it out
+                    _ => {
+                        panic!("Illegal field in imadwide")
+                    }
+                }
+            } else {
+                ImadWideWriteDh // as above this needs more work
+            }
+        }
+
+        Op::PopC(_) => Decoupled,
+        Op::IAdd3(_) | Op::IAdd3X(_) => Alu,
+
+        Op::BMsk(_) => Alu,
+        Op::Sgxt(_) => Alu,
+        Op::Lop3(_) => Alu,
+        Op::Flo(_) => Decoupled,
+        Op::ISetP(_) => Dualalu,
+        Op::IAbs(_) => Alu,
+        Op::Lea(_) => Alu,
+        Op::LeaX(_) => Alu,
+        Op::IMnMx(_) => Dualalu,
+        Op::I2I(_) => Alu,
+        // I2IP => alu
+        Op::Shf(_) => Alu,
+
+        Op::F2FP(_) => Alu,
+        Op::FFma(_) => Fma,
+        Op::FAdd(_) => Fma,
+        Op::FMul(_) => Fma,
+        Op::FMnMx(_) => Dualalu,
+        Op::FSwzAdd(_) => Fma,
+        Op::FSet(_) => Dualalu,
+        // FSel => Alu,
+        Op::FSetP(_) => Dualalu,
+        // FChk => Decoupled,
+        Op::DAdd(_) | Op::DFma(_) | Op::DMul(_) | Op::DSetP(_) => {
+            RedirectedFp64
+        }
+
+        Op::DMnMx(_) => RedirectedFp64, // not in docs
+
+        Op::HAdd2(hadd2) => {
+            if hadd2.f32 {
+                Fp16F32
+            } else {
+                Fp16
+            }
+        }
+        Op::HFma2(_) | Op::HMul2(_) => Fp16,
+
+        Op::HSet2(_) | Op::HSetP2(_) | Op::HMnMx2(_) => Fp16Alu,
+        Op::Hmma(_) => Hmma,
+        Op::Ipa(_) => DecoupledAgu,
+        Op::MuFu(_) => Decoupled,
+
+        // Conversion functions all Decoupled
+        Op::F2F(_) => Decoupled,
+        Op::F2I(_) => Decoupled,
+        Op::I2F(_) => Decoupled,
+        Op::FRnd(_) => Decoupled,
+        Op::AL2P(_) => Decoupled,
+
+        Op::Mov(_) => Dualalu,
+        Op::Movm(_) => DecoupledAgu,
+        Op::Sel(_) => Dualalu,
+        Op::BRev(_) => Decoupled,
+        // P2R => Alu,
+        // R2P => Alu,
+        Op::PLop3(_) => Alu,
+        Op::Prmt(_) => Alu,
+        Op::Nop(_) => Disp64,
+        Op::Vote(_) => Dualalu,
+        Op::Match(_) => Decoupled,
+        Op::S2R(_) => DecoupledAgu,
+        Op::R2UR(_) => Alu,
+        Op::Redux(_) => {
+            if reader {
+                Decoupled
+            } else {
+                panic!("Illegal R2UR");
+            }
+        }
+        Op::CS2R(cs2r) => {
+            if cs2r.dst.comps() == 2 {
+                Disp64
+            } else {
+                Dualalu
+            }
+        }
+        // B2R => DecoupledAgu,
+        // LEPC => Disp64
+        Op::BMov(bmov) => match bmov.dst {
+            Dst::Reg(_) => Branch,
+            _ => Branch,
+        },
+        // RPCMOV.32 => Alu,
+        // RPCMOV.64 => Disp64
+        // PMTRIG => Disp64
+        // CSMTEST =>  Alu,
+        Op::Bar(_) => DecoupledAgu,
+        Op::Imma(_) => Imma,
+        Op::IDp4(_) => Fma,
+        Op::BClear(_) => Decoupled,
+        Op::Bra(_) => Decoupled,
+        Op::BSSy(_) => Decoupled,
+        Op::Kill(_) => Decoupled,
+        Op::Exit(_) => Decoupled,
+        Op::BSync(_) => Decoupled,
+        Op::Tex(_) => Decoupled,
+        Op::Tld(_) => Decoupled,
+        Op::Tld4(_) => Decoupled,
+        Op::Tmml(_) => Decoupled,
+        Op::Txd(_) => Decoupled,
+        Op::Txq(_) => Decoupled,
+        Op::Ldc(_) => Decoupled,
+        Op::ALd(_) => DecoupledAgu,
+        Op::ASt(_) => DecoupledAgu,
+        Op::Out(_) => DecoupledAgu,
+        Op::OutFinal(_) => DecoupledAgu,
+        Op::Ld(_) => DecoupledAgu,
+        Op::St(_) => DecoupledAgu,
+        Op::Atom(_) => DecoupledAgu,
+        //CCtl.i,c are coupled
+        Op::CCtl(_) => DecoupledAgu,
+        Op::MemBar(_) => Decoupled,
+        Op::SuLd(_) => Decoupled,
+        Op::SuSt(_) => Decoupled,
+        Op::SuAtom(_) => Decoupled,
+        Op::PixLd(_) => DecoupledAgu,
+        Op::Isberd(_) => DecoupledAgu,
+        Op::Isbewr(_) => DecoupledAgu,
+        Op::LdTram(_) => DecoupledAgu,
+        Op::Shfl(_) => DecoupledAgu,
+        Op::Ldsm(_) => DecoupledAgu,
+        x => {
+            panic!("Illegal instuction in reg category {}", x);
+        }
+    }
+}
+
+fn op_pred_latency(op: &Op) -> PredLatencySM100 {
+    use PredLatencySM100::*;
+    match op {
+        Op::Atom(_) => Decoupled,
+        Op::Bra(_) => Decoupled,
+        Op::DSetP(_) => RedirectedFp64,
+        Op::FMnMx(_) | Op::FSetP(_) => Dualalu,
+        Op::HFma2(_) => Fp16,
+        Op::HMnMx2(_) => Fp16,
+        Op::HSetP2(_) => Fp16,
+        Op::IAdd3(_) => Coupled,
+        Op::IAdd3X(_) => Coupled,
+        Op::IMad(_) => Fma,
+        Op::IMad64(_) => Fma,
+        Op::IMnMx(_) => Dualalu,
+        Op::IMul(_) => Fma,
+        Op::Ipa(_) => Decoupled,
+        Op::ISetP(_) => Dualalu,
+
+        Op::Ld(_) => Decoupled,
+
+        Op::Lea(_) | Op::LeaX(_) => Coupled,
+        Op::PixLd(_) => Decoupled,
+        Op::PLop3(_) => Coupled,
+        Op::PSetP(_) => Coupled,
+        Op::R2UR(_) => R2Ur,
+        Op::Sel(_) => Dualalu,
+        Op::Shfl(_) => Decoupled,
+        Op::SuLd(_) => Decoupled,
+        Op::SuSt(_) => Decoupled,
+        Op::Tex(_) => Decoupled,
+        Op::Tld(_) => Decoupled,
+        Op::Tld4(_) => Decoupled,
+        Op::Tmml(_) => Decoupled,
+        Op::Txd(_) => Decoupled,
+        Op::Txq(_) => Decoupled,
+
+        Op::Vote(_) => DispDualAlu,
+        Op::Match(_) => Decoupled,
+        _ => {
+            panic!("Illegal op in sm120 pred latency {}", op);
+        }
+    }
+}
+
+fn op_ureg_latency(
+    op: &Op,
+    reader: bool,
+    op_reg_idx: usize,
+) -> UregLatencySM100 {
+    use UregLatencySM100::*;
+    // this decides between the category types for readers.
+    let bindless = reader && op.srcs_as_slice()[op_reg_idx].is_bindless_cbuf();
+
+    let coupled = if bindless { CoupledBindless } else { Coupled };
+    let decoupled = if bindless {
+        DecoupledBindless
+    } else {
+        Decoupled
+    };
+
+    // if this is a reader from a ureg, it could be a U* instruction or a
+    // regular instruction.
+    let uniform_op = op.is_uniform();
+
+    let coupled = if uniform_op { Udp } else { coupled };
+    let decoupled = if uniform_op { Udp } else { decoupled };
+
+    match op {
+        Op::BMsk(_) => coupled,
+        Op::BRev(_) => decoupled,
+        // uclea?
+        Op::Flo(_) => decoupled,
+        Op::IAdd3(_) | Op::IAdd3X(_) => coupled,
+        Op::IAbs(_) => coupled,
+        Op::IDp4(_) => coupled,
+        Op::IMnMx(_) => coupled,
+        Op::IMad(_) => coupled,
+
+        Op::IMad64(_) => coupled,
+        Op::ISetP(_) => coupled,
+        Op::ALd(_)
+        | Op::ASt(_)
+        | Op::Ld(_)
+        | Op::Ldsm(_)
+        | Op::St(_)
+        | Op::Atom(_) => decoupled,
+        Op::SuLd(_) | Op::SuSt(_) | Op::SuAtom(_) => decoupled,
+        Op::Ldc(_) => {
+            if uniform_op {
+                ToUr
+            } else {
+                decoupled
+            }
+        }
+        Op::Lea(_) => coupled,
+        Op::LeaX(_) => coupled,
+        Op::Lop2(_) | Op::Lop3(_) => coupled,
+
+        Op::MuFu(_) => decoupled,
+        Op::Mov(_) => {
+            if uniform_op {
+                Umov
+            } else {
+                coupled
+            }
+        }
+
+        // mov32i => uldc
+        // p2ur => udp,
+        Op::PLop3(_) => coupled,
+        Op::PopC(_) => {
+            if uniform_op {
+                coupled
+            } else {
+                decoupled
+            }
+        }
+        Op::Prmt(_) => coupled,
+        Op::PSetP(_) => coupled,
+        // UR2UP
+        Op::Sel(_) => coupled,
+        Op::Sgxt(_) => coupled,
+        Op::Shf(_) => coupled,
+        Op::Shfl(_) => decoupled,
+
+        Op::I2F(_) => decoupled,
+        Op::F2I(_) => decoupled,
+        Op::F2F(_) => decoupled,
+        Op::R2UR(_) => {
+            if !reader {
+                R2Ur
+            } else {
+                panic!("Illegal R2UR in ureg");
+            }
+        }
+        Op::Redux(_) => {
+            if !reader {
+                ToUr
+            } else {
+                panic!("Illegal R2UR in ureg");
+            }
+        }
+        Op::Vote(_) => Voteu,
+        Op::S2R(_) => ToUr,
+
+        Op::Tex(_) | Op::Tld(_) | Op::Tld4(_) | Op::Txq(_) => Tex,
+        Op::FRnd(_) => decoupled,
+        Op::F2FP(_)
+        | Op::FAdd(_)
+        | Op::FMul(_)
+        | Op::FFma(_)
+        | Op::FSet(_)
+        | Op::FSetP(_)
+        | Op::FMnMx(_)
+        | Op::HAdd2(_)
+        | Op::HMul2(_)
+        | Op::HSet2(_)
+        | Op::HFma2(_)
+        | Op::HMnMx2(_)
+        | Op::HSetP2(_) => coupled,
+        Op::DMul(_) | Op::DFma(_) | Op::DAdd(_) | Op::DSetP(_) => decoupled,
+        _ => {
+            panic!("Illegal instuction in ureg category {}", op);
+        }
+    }
+}
+
+fn op_upred_latency(op: &Op) -> UpredLatencySM100 {
+    use UpredLatencySM100::*;
+    let uniform_op = op.is_uniform();
+    match op {
+        Op::BMsk(_)
+        | Op::BRev(_)
+        | Op::Flo(_)
+        | Op::IAdd3(_)
+        | Op::IAdd3X(_)
+        | Op::IMad(_)
+        | Op::ISetP(_)
+        | Op::Lea(_)
+        | Op::LeaX(_)
+        | Op::Lop3(_)
+        | Op::Mov(_) => Udp,
+        Op::Bra(_) => BraJmp,
+        Op::Ldc(_) => UldcMma,
+        Op::PLop3(_) => {
+            if uniform_op {
+                Udp
+            } else {
+                Coupled
+            }
+        }
+        Op::PSetP(_) => {
+            if uniform_op {
+                Udp
+            } else {
+                Coupled
+            }
+        }
+        Op::Sel(_) => {
+            if uniform_op {
+                Udp
+            } else {
+                Coupled
+            }
+        }
+        Op::Vote(_) => {
+            if uniform_op {
+                Voteu
+            } else {
+                panic!("Illegal Vote in upred");
+            }
+        }
+        _ => {
+            panic!("Illegal instuction in upred category {}", op);
+        }
+    }
+}
+
+pub struct SM120Latency {}
+
+impl SM120Latency {
+    pub fn needs_scoreboards(op: &Op) -> bool {
+        if op.is_uniform() {
+            matches!(
+                op_ureg_latency(op, false, 0),
+                UregLatencySM100::Uldc
+                    | UregLatencySM100::ToUr
+                    | UregLatencySM100::Tex
+            )
+        } else {
+            matches!(
+                op_reg_latency(op, false, 0),
+                RegLatencySM100::Dmma
+                    | RegLatencySM100::Hmma
+                    | RegLatencySM100::RedirectedFp64
+                    | RegLatencySM100::Branch
+                    | RegLatencySM100::Decoupled
+                    | RegLatencySM100::DecoupledAgu
+            )
+        }
+    }
+
+    pub fn raw(
+        write: &Op,
+        dst_idx: usize,
+        read: Option<&Op>,
+        src_idx: usize,
+    ) -> u32 {
+        let Some(dst_file) = write.dsts_as_slice()[dst_idx].file() else {
+            return 0;
+        };
+
+        match dst_file {
+            RegFile::GPR => {
+                let write_latency = op_reg_latency(write, false, dst_idx);
+                let read_latency = match read {
+                    Some(op) => op_reg_latency(op, true, src_idx),
+                    None => RegLatencySM100::RedirectedFp64,
+                };
+                // The latencies are for SM100 docs, but some chips need large
+                // one just override here.
+                if write_latency == RegLatencySM100::Hmma
+                    || read_latency == RegLatencySM100::Hmma
+                {
+                    RegLatencySM100::raw(write_latency, read_latency, false) + 9
+                } else if write_latency == RegLatencySM100::Imma
+                    || read_latency == RegLatencySM100::Imma
+                {
+                    RegLatencySM100::raw(write_latency, read_latency, false) + 5
+                } else {
+                    RegLatencySM100::raw(write_latency, read_latency, false) + 1
+                }
+            }
+            RegFile::UGPR => {
+                let write_latency = op_ureg_latency(write, false, dst_idx);
+                let read_latency = match read {
+                    Some(op) => op_ureg_latency(op, true, src_idx),
+                    None => UregLatencySM100::Uldc,
+                };
+                UregLatencySM100::raw(write_latency, read_latency, false) + 1
+            }
+            RegFile::Pred => {
+                let write_latency = op_pred_latency(write);
+                let read_latency = match read {
+                    Some(op) => op_pred_latency(op),
+                    None => PredLatencySM100::RedirectedFp64,
+                };
+                PredLatencySM100::raw(write_latency, read_latency, false) + 1
+            }
+            RegFile::UPred => {
+                let write_latency = op_upred_latency(write);
+                let read_latency = match read {
+                    Some(op) => op_upred_latency(op),
+                    None => UpredLatencySM100::UGuard,
+                };
+                UpredLatencySM100::raw(write_latency, read_latency, false) + 1
+            }
+            RegFile::Bar => 0, // Barriers have a HW scoreboard
+            _ => panic!("Not a register"),
+        }
+    }
+
+    pub fn war(read: &Op, src_idx: usize, write: &Op, dst_idx: usize) -> u32 {
+        let Some(dst_file) = write.dsts_as_slice()[dst_idx].file() else {
+            return 0;
+        };
+
+        match dst_file {
+            RegFile::GPR => {
+                let write_latency = op_reg_latency(write, false, dst_idx);
+                let read_latency = op_reg_latency(read, true, src_idx);
+
+                if write_latency == RegLatencySM100::Hmma
+                    || read_latency == RegLatencySM100::Hmma
+                {
+                    RegLatencySM100::war(read_latency, write_latency, false) + 7
+                } else {
+                    RegLatencySM100::war(read_latency, write_latency, false)
+                }
+            }
+            RegFile::UGPR => {
+                let write_latency = op_ureg_latency(write, false, dst_idx);
+                let read_latency = op_ureg_latency(read, true, src_idx);
+                UregLatencySM100::war(read_latency, write_latency, false)
+            }
+            RegFile::Pred => {
+                let write_latency = op_pred_latency(write);
+                let read_latency = op_pred_latency(read);
+                PredLatencySM100::war(read_latency, write_latency, false)
+            }
+            RegFile::UPred => {
+                let write_latency = op_upred_latency(write);
+                let read_latency = op_upred_latency(read);
+                UpredLatencySM100::war(read_latency, write_latency, false)
+            }
+            _ => panic!("Not a register"),
+        }
+    }
+
+    pub fn waw(
+        a: &Op,
+        a_dst_idx: usize,
+        b: &Op,
+        b_dst_idx: usize,
+        a_op_pred: bool,
+    ) -> u32 {
+        let Some(dst_file) = a.dsts_as_slice()[a_dst_idx].file() else {
+            return 0;
+        };
+
+        match dst_file {
+            RegFile::GPR => {
+                let write1_latency = op_reg_latency(a, false, a_dst_idx);
+                let write2_latency = op_reg_latency(b, false, b_dst_idx);
+                if write1_latency == RegLatencySM100::Hmma
+                    || write2_latency == RegLatencySM100::Hmma
+                {
+                    RegLatencySM100::waw(
+                        write1_latency,
+                        write2_latency,
+                        a_op_pred,
+                    ) + 7
+                } else {
+                    RegLatencySM100::waw(
+                        write1_latency,
+                        write2_latency,
+                        a_op_pred,
+                    )
+                }
+            }
+            RegFile::UGPR => {
+                let write1_latency = op_ureg_latency(a, false, a_dst_idx);
+                let write2_latency = op_ureg_latency(b, false, b_dst_idx);
+                UregLatencySM100::waw(write1_latency, write2_latency, a_op_pred)
+            }
+            RegFile::Pred => {
+                let write1_latency = op_pred_latency(a);
+                let write2_latency = op_pred_latency(b);
+                PredLatencySM100::waw(write1_latency, write2_latency, a_op_pred)
+            }
+            RegFile::UPred => {
+                let write1_latency = op_upred_latency(a);
+                let write2_latency = op_upred_latency(b);
+                UpredLatencySM100::waw(write1_latency, write2_latency, false)
+            }
+            _ => panic!("Not a register"),
+        }
+    }
+}
@@ -0,0 +1,168 @@
+use bitview::{BitMutViewable, BitViewable, SetField};
+use rustc_hash::FxHashMap;
+
+use crate::ir::{Instr, InstrDeps, Label, Op, OpNop, Shader};
+
+pub fn instr_latency(_sm: u8, op: &Op, _dst_idx: usize) -> u32 {
+    if op.is_fp64() {
+        return 20;
+    }
+
+    match op {
+        Op::Ipa(_) => 15,
+        Op::Ld(_) => 24,
+        Op::ALd(_) => 24,
+        Op::IMul(_) => 15, // This does not apply to imad, right? right???
+        Op::ISetP(_) => 13,
+        Op::PSetP(_) => 13,
+        Op::IAdd2(o) if !o.carry_out.is_none() => 13,
+        Op::Tex(_)
+        | Op::Tld(_)
+        | Op::Tld4(_)
+        | Op::Tmml(_)
+        | Op::Txd(_)
+        | Op::Txq(_) => 17,
+        _ => 9,
+    }
+}
+
+pub fn latency_upper_bound() -> u32 {
+    24
+}
+
+pub fn instr_exec_latency(sm: u8, op: &Op) -> u32 {
+    let is_kepler_a = sm == 30;
+    match op {
+        Op::Tex(_)
+        | Op::Tld(_)
+        | Op::Tld4(_)
+        | Op::Tmml(_)
+        | Op::Txd(_)
+        | Op::Txq(_) => 17,
+        Op::MemBar(_) => 16,
+        Op::Cont(_) | Op::Brk(_) if is_kepler_a => 5,
+        Op::Exit(_) => 15,
+        _ => 1,
+    }
+}
+
+fn calc_instr_sched(prev_op: Option<&Op>, op: &Op, deps: &InstrDeps) -> u8 {
+    // Kepler is the first generation to lift scoreboarding from the
+    // hardware into the compiler. For each instruction we encode
+    // the delay but not all the other information necessary for newer
+    // architectures.
+    // The hardware still checks for data-hazard and, if present, it
+    // will delay the instruction by 32 cycles.
+    match op {
+        Op::TexDepBar(_) => 0xc2,
+        Op::Sync(_) => 0x00, // Wait 16 cycles
+        _ => {
+            // TODO: when we support dual-issue this should check for
+            // both previous ops
+            let base = match prev_op {
+                Some(Op::ASt(_)) => 0x40,
+                _ => 0x20,
+            };
+
+            let delay = deps.delay;
+            debug_assert!(delay >= 1 && delay <= 32);
+            base | (delay - 1)
+        }
+    }
+
+    // 0x00: wait for 16 cycles
+    // 0x04: dual-issue with next instruction
+    // 0xc2: if TEXBAR
+    // 0x20 | 0x40: suspend for N+1 cycles (N = bitmask 0x1f)
+    //              0x40 only if prev_op is attribute store
+    // Unsure:
+    // 0x80: global memory bit
+    //
+    // TODO:
+    // - Dual issue (0x04)
+    // - Functional Unit tracking
+}
+
+pub trait KeplerInstructionEncoder {
+    /// Encode the instruction and push it into the "encoded" vec
+    fn encode_instr(
+        &self,
+        instr: &Instr,
+        labels: &FxHashMap<Label, usize>,
+        encoded: &mut Vec<u32>,
+    );
+
+    /// Prepare the scheduling instruction opcode-field and return a
+    /// subset where the actual scheduling information will be written
+    fn prepare_sched_instr<'a>(
+        &self,
+        sched_instr: &'a mut [u32; 2],
+    ) -> impl BitMutViewable + 'a;
+}
+
+/// Helper function that encodes shaders for both KeplerA and KeplerB.
+/// Difference in the encoders are handled by KeplerInstructionEncoder.
+pub fn encode_kepler_shader<E>(encoder: &E, s: &Shader<'_>) -> Vec<u32>
+where
+    E: KeplerInstructionEncoder,
+{
+    const INSTR_LEN_BYTES: usize = 8;
+    assert!(s.functions.len() == 1);
+    let func = &s.functions[0];
+
+    // --- Compute label addresses ---
+    // We need a schedule instruction every 7 instructions, these don't
+    // define jump boundaries so we can have multible blocks in the same
+    // 7-instr group.
+    let mut ip = 0_usize;
+    let mut labels = FxHashMap::default();
+    for b in &func.blocks {
+        let num_sched = (ip / 7) + 1;
+        labels.insert(b.label, (ip + num_sched) * INSTR_LEN_BYTES);
+        ip += b.instrs.len();
+    }
+
+    // --- Real encoding ---
+    // Create an instruction iterator and iterate it in chunks of 7.
+    // fill the last chunk with a nop (it should never be executed).
+    let mut instr_iter =
+        func.blocks.iter().flat_map(|b| b.instrs.iter()).peekable();
+    let mut filling_instr = Instr {
+        pred: true.into(),
+        op: Op::Nop(OpNop { label: None }),
+        deps: InstrDeps::new(),
+    };
+    filling_instr.deps.set_delay(1);
+    let mut sched_chunk_gen = || {
+        if instr_iter.peek().is_none() {
+            return None;
+        }
+        Some([0; 7].map(|_| instr_iter.next().unwrap_or(&filling_instr)))
+    };
+
+    let mut encoded = Vec::new();
+    let mut prev_op = None;
+    while let Some(sched_chunk) = sched_chunk_gen() {
+        let sched_i = encoded.len();
+
+        let mut sched_instr = [0u32; 2];
+        encoded.extend(&sched_instr[..]); // Push now, will edit later
+        let mut bv = encoder.prepare_sched_instr(&mut sched_instr);
+        // There should be 8 bits for each instr in a scheduling block
+        debug_assert!(bv.bits() == 8 * 7);
+
+        for (i, instr) in sched_chunk.iter().enumerate() {
+            encoder.encode_instr(instr, &labels, &mut encoded);
+
+            let sched = calc_instr_sched(prev_op, &instr.op, &instr.deps);
+            bv.set_field(i * 8..(i + 1) * 8, sched);
+            prev_op = Some(&instr.op);
+        }
+
+        drop(bv);
+        encoded[sched_i] = sched_instr[0];
+        encoded[sched_i + 1] = sched_instr[1];
+    }
+
+    encoded
+}
@@ -0,0 +1,296 @@
+// Copyright © 2022 Collabora, Ltd.
+// SPDX-License-Identifier: MIT
+
+use crate::ir::*;
+use crate::legalize::LegalizeBuilder;
+use crate::sm120_instr_latencies::SM120Latency;
+use crate::sm70_encode::*;
+use crate::sm75_instr_latencies::SM75Latency;
+use crate::sm80_instr_latencies::SM80Latency;
+pub struct ShaderModel70 {
+    sm: u8,
+}
+
+impl ShaderModel70 {
+    pub fn new(sm: u8) -> Self {
+        assert!(sm >= 70);
+        Self { sm }
+    }
+
+    fn has_uniform_alu(&self) -> bool {
+        self.sm >= 73
+    }
+
+    fn instr_latency(&self, op: &Op, dst_idx: usize) -> u32 {
+        let file = match &op.dsts_as_slice()[dst_idx] {
+            Dst::None => return 0,
+            Dst::SSA(vec) => vec.file(),
+            Dst::Reg(reg) => reg.file(),
+        };
+
+        let (gpr_latency, pred_latency) = if self.sm < 80 {
+            match op {
+                // Double-precision float ALU
+                Op::DAdd(_)
+                | Op::DFma(_)
+                | Op::DMnMx(_)
+                | Op::DMul(_)
+                | Op::DSetP(_)
+                // Half-precision float ALU
+                | Op::HAdd2(_)
+                | Op::HFma2(_)
+                | Op::HMul2(_)
+                | Op::HSet2(_)
+                | Op::HSetP2(_)
+                | Op::HMnMx2(_) => if self.is_volta() {
+                    // Volta is even slower
+                    (13, 15)
+                } else {
+                    (13, 14)
+                }
+                _ => (6, 13)
+            }
+        } else {
+            (6, 13)
+        };
+
+        // This is BS and we know it
+        match file {
+            RegFile::GPR => gpr_latency,
+            RegFile::UGPR => 12,
+            RegFile::Pred => pred_latency,
+            RegFile::UPred => 11,
+            RegFile::Bar => 0, // Barriers have a HW scoreboard
+            RegFile::Carry => 6,
+            RegFile::Mem => panic!("Not a register"),
+        }
+    }
+}
+
+impl ShaderModel for ShaderModel70 {
+    fn sm(&self) -> u8 {
+        self.sm
+    }
+
+    fn num_regs(&self, file: RegFile) -> u32 {
+        match file {
+            RegFile::GPR => 255 - self.hw_reserved_gprs(),
+            RegFile::UGPR => {
+                if !self.has_uniform_alu() {
+                    0
+                } else if self.sm >= 100 {
+                    80
+                } else {
+                    63
+                }
+            }
+            RegFile::Pred => 7,
+            RegFile::UPred => {
+                if self.has_uniform_alu() {
+                    7
+                } else {
+                    0
+                }
+            }
+            RegFile::Carry => 0,
+            RegFile::Bar => 16,
+            RegFile::Mem => RegRef::MAX_IDX + 1,
+        }
+    }
+
+    fn hw_reserved_gprs(&self) -> u32 {
+        // On Volta+, 2 GPRs get burned for the program counter - see the
+        // footnote on table 2 of the volta whitepaper
+        // https://images.nvidia.com/content/volta-architecture/pdf/volta-architecture-whitepaper.pdf
+        2
+    }
+
+    fn crs_size(&self, max_crs_depth: u32) -> u32 {
+        assert!(max_crs_depth == 0);
+        0
+    }
+
+    fn op_can_be_uniform(&self, op: &Op) -> bool {
+        if !self.has_uniform_alu() {
+            return false;
+        }
+
+        match op {
+            Op::R2UR(_)
+            | Op::Redux(_)
+            | Op::S2R(_)
+            | Op::BMsk(_)
+            | Op::BRev(_)
+            | Op::Flo(_)
+            | Op::IAdd3(_)
+            | Op::IAdd3X(_)
+            | Op::IMad(_)
+            | Op::IMad64(_)
+            | Op::ISetP(_)
+            | Op::Lea(_)
+            | Op::LeaX(_)
+            | Op::Lop3(_)
+            | Op::Mov(_)
+            | Op::PLop3(_)
+            | Op::PopC(_)
+            | Op::Prmt(_)
+            | Op::PSetP(_)
+            | Op::Sel(_)
+            | Op::Sgxt(_)
+            | Op::Shf(_)
+            | Op::Shl(_)
+            | Op::Shr(_)
+            | Op::Vote(_)
+            | Op::Copy(_)
+            | Op::Pin(_)
+            | Op::Unpin(_) => true,
+            Op::Ldc(op) => op.offset.is_zero(),
+            // UCLEA  USHL  USHR
+            _ => false,
+        }
+    }
+
+    fn op_needs_scoreboard(&self, op: &Op) -> bool {
+        if op.no_scoreboard() {
+            return false;
+        }
+
+        if self.is_blackwell() {
+            SM120Latency::needs_scoreboards(op)
+        } else if self.is_ampere() || self.is_ada() {
+            SM80Latency::needs_scoreboards(op)
+        } else if self.is_turing() {
+            SM75Latency::needs_scoreboards(op)
+        } else {
+            !op.has_fixed_latency(self.sm())
+        }
+    }
+
+    fn exec_latency(&self, op: &Op) -> u32 {
+        match op {
+            Op::Bar(_) | Op::MemBar(_) => {
+                if self.sm >= 80 {
+                    6
+                } else {
+                    5
+                }
+            }
+            Op::CCtl(_op) => {
+                // CCTL.C needs 8, CCTL.I needs 11
+                11
+            }
+            // Op::DepBar(_) => 4,
+            _ => 1, // TODO: co-issue
+        }
+    }
+
+    fn raw_latency(
+        &self,
+        write: &Op,
+        dst_idx: usize,
+        read: &Op,
+        src_idx: usize,
+    ) -> u32 {
+        if self.is_blackwell() {
+            SM120Latency::raw(write, dst_idx, Some(read), src_idx)
+        } else if self.is_ampere() || self.is_ada() {
+            SM80Latency::raw(write, dst_idx, Some(read), src_idx)
+        } else if self.is_turing() {
+            SM75Latency::raw(write, dst_idx, Some(read), src_idx)
+        } else {
+            self.instr_latency(write, dst_idx)
+        }
+    }
+
+    fn war_latency(
+        &self,
+        read: &Op,
+        src_idx: usize,
+        write: &Op,
+        dst_idx: usize,
+    ) -> u32 {
+        if self.is_blackwell() {
+            SM120Latency::war(read, src_idx, write, dst_idx)
+        } else if self.is_ampere() || self.is_ada() {
+            SM80Latency::war(read, src_idx, write, dst_idx)
+        } else if self.is_turing() {
+            SM75Latency::war(read, src_idx, write, dst_idx)
+        } else {
+            // We assume the source gets read in the first 4 cycles.  We don't
+            // know how quickly the write will happen.  This is all a guess.
+            4
+        }
+    }
+
+    fn waw_latency(
+        &self,
+        a: &Op,
+        a_dst_idx: usize,
+        a_has_pred: bool,
+        b: &Op,
+        b_dst_idx: usize,
+    ) -> u32 {
+        if self.is_blackwell() {
+            SM120Latency::waw(a, a_dst_idx, b, b_dst_idx, a_has_pred)
+        } else if self.is_ampere() || self.is_ada() {
+            SM80Latency::waw(a, a_dst_idx, b, b_dst_idx, a_has_pred)
+        } else if self.is_turing() {
+            SM75Latency::waw(a, a_dst_idx, b, b_dst_idx, a_has_pred)
+        } else {
+            // We know our latencies are wrong so assume the wrote could happen
+            // anywhere between 0 and instr_latency(a) cycles
+            self.instr_latency(a, a_dst_idx)
+        }
+    }
+
+    fn paw_latency(&self, write: &Op, dst_idx: usize) -> u32 {
+        if self.is_blackwell() {
+            SM120Latency::raw(write, dst_idx, None, 0)
+        } else if self.is_ampere() || self.is_ada() {
+            SM80Latency::raw(write, dst_idx, None, 0)
+        } else if self.is_turing() {
+            SM75Latency::raw(write, dst_idx, None, 0)
+        } else if self.is_volta() {
+            match write {
+                Op::DSetP(_) | Op::HSetP2(_) => 15,
+                _ => 13,
+            }
+        } else {
+            13
+        }
+    }
+
+    fn latency_upper_bound(&self) -> u32 {
+        if self.is_blackwell() || self.is_ampere() || self.is_ada() {
+            30
+        } else if self.is_turing() {
+            25
+        } else {
+            15
+        }
+    }
+
+    fn worst_latency(&self, write: &Op, dst_idx: usize) -> u32 {
+        if self.is_blackwell() {
+            SM120Latency::raw(write, dst_idx, None, 0)
+        } else if self.is_ampere() || self.is_ada() {
+            SM80Latency::raw(write, dst_idx, None, 0)
+        } else if self.is_turing() {
+            SM75Latency::raw(write, dst_idx, None, 0)
+        } else {
+            self.instr_latency(write, dst_idx)
+        }
+    }
+
+    fn max_instr_delay(&self) -> u8 {
+        15
+    }
+
+    fn legalize_op(&self, b: &mut LegalizeBuilder, op: &mut Op) {
+        legalize_sm70_op(self, b, op);
+    }
+
+    fn encode_shader(&self, s: &Shader<'_>) -> Vec<u32> {
+        encode_sm70_shader(self, s)
+    }
+}
@@ -0,0 +1,609 @@
+// Copyright © 2023 Collabora, Ltd.
+// SPDX-License-Identifier: MIT
+
+extern crate bitview;
+extern crate nvidia_headers;
+
+use crate::ir::{
+    MeshShaderInfo, ShaderInfo, ShaderIoInfo, ShaderModel, ShaderModelInfo,
+    ShaderStageInfo, VtgIoInfo,
+};
+use bitview::{
+    BitMutView, BitMutViewable, BitView, BitViewable, SetBit, SetField,
+};
+use nak_bindings::*;
+use nvidia_headers::classes::cla097::sph::*;
+use std::ops::Range;
+
+pub const _SPHV3_SHADER_HEADER_SIZE: usize = 20;
+pub const SPHV4_SHADER_HEADER_SIZE: usize = 32;
+pub const CURRENT_MAX_SHADER_HEADER_SIZE: usize = SPHV4_SHADER_HEADER_SIZE;
+
+type SubSPHView<'a> = BitMutView<'a, [u32; CURRENT_MAX_SHADER_HEADER_SIZE]>;
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum ShaderType {
+    Vertex,
+    TessellationInit,
+    Tessellation,
+    Geometry,
+    Fragment,
+}
+
+impl From<&ShaderStageInfo> for ShaderType {
+    fn from(value: &ShaderStageInfo) -> Self {
+        match value {
+            ShaderStageInfo::Vertex(_) | ShaderStageInfo::Task(_) => {
+                ShaderType::Vertex
+            }
+            ShaderStageInfo::Fragment(_) => ShaderType::Fragment,
+            ShaderStageInfo::Geometry(_) => ShaderType::Geometry,
+            ShaderStageInfo::TessellationInit(_) => {
+                ShaderType::TessellationInit
+            }
+            ShaderStageInfo::Tessellation(_) => ShaderType::Tessellation,
+            ShaderStageInfo::Mesh(info) => {
+                if info.has_task_shader {
+                    ShaderType::Tessellation
+                } else {
+                    ShaderType::Vertex
+                }
+            }
+            _ => panic!("Invalid ShaderStageInfo {:?}", value),
+        }
+    }
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum OutputTopology {
+    PointList,
+    LineStrip,
+    TriangleStrip,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum PixelImap {
+    Unused,
+    Constant,
+    Perspective,
+    ScreenLinear,
+}
+
+impl From<PixelImap> for u8 {
+    fn from(value: PixelImap) -> u8 {
+        match value {
+            PixelImap::Unused => 0,
+            PixelImap::Constant => 1,
+            PixelImap::Perspective => 2,
+            PixelImap::ScreenLinear => 3,
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct ShaderProgramHeader {
+    pub data: [u32; CURRENT_MAX_SHADER_HEADER_SIZE],
+    shader_type: ShaderType,
+}
+
+impl BitViewable for ShaderProgramHeader {
+    fn bits(&self) -> usize {
+        BitView::new(&self.data).bits()
+    }
+
+    fn get_bit_range_u64(&self, range: Range<usize>) -> u64 {
+        BitView::new(&self.data).get_bit_range_u64(range)
+    }
+}
+
+impl BitMutViewable for ShaderProgramHeader {
+    fn set_bit_range_u64(&mut self, range: Range<usize>, val: u64) {
+        BitMutView::new(&mut self.data).set_bit_range_u64(range, val);
+    }
+}
+
+impl ShaderProgramHeader {
+    pub fn new(shader_type: ShaderType, sm: u8) -> Self {
+        let mut res = Self {
+            data: [0; CURRENT_MAX_SHADER_HEADER_SIZE],
+            shader_type,
+        };
+
+        let sph_type = if shader_type == ShaderType::Fragment {
+            SPHV3_T1_SPH_TYPE_TYPE_02_PS
+        } else {
+            SPHV3_T1_SPH_TYPE_TYPE_01_VTG
+        };
+
+        let sph_version = if sm >= 73 { 4 } else { 3 };
+        res.set_sph_type(sph_type, sph_version);
+        res.set_shader_type(shader_type);
+
+        res
+    }
+
+    #[inline]
+    fn imap_system_values_ab(&mut self) -> SubSPHView<'_> {
+        BitMutView::new_subset(&mut self.data, 160..192)
+    }
+
+    #[inline]
+    fn imap_g_vtg(&mut self) -> SubSPHView<'_> {
+        assert!(self.shader_type != ShaderType::Fragment);
+
+        BitMutView::new_subset(&mut self.data, 192..320)
+    }
+
+    #[inline]
+    fn imap_g_ps(&mut self) -> SubSPHView<'_> {
+        assert!(self.shader_type == ShaderType::Fragment);
+
+        BitMutView::new_subset(&mut self.data, 192..448)
+    }
+
+    #[inline]
+    fn imap_system_values_c(&mut self) -> SubSPHView<'_> {
+        if self.shader_type == ShaderType::Fragment {
+            BitMutView::new_subset(&mut self.data, 464..480)
+        } else {
+            BitMutView::new_subset(&mut self.data, 336..352)
+        }
+    }
+
+    #[inline]
+    fn imap_system_values_d_vtg(&mut self) -> SubSPHView<'_> {
+        assert!(self.shader_type != ShaderType::Fragment);
+        BitMutView::new_subset(&mut self.data, 392..400)
+    }
+
+    #[inline]
+    fn omap_system_values_ab(&mut self) -> SubSPHView<'_> {
+        assert!(self.shader_type != ShaderType::Fragment);
+        BitMutView::new_subset(&mut self.data, 400..432)
+    }
+
+    #[inline]
+    fn omap_g(&mut self) -> SubSPHView<'_> {
+        assert!(self.shader_type != ShaderType::Fragment);
+
+        BitMutView::new_subset(&mut self.data, 432..560)
+    }
+
+    #[inline]
+    fn omap_system_values_c(&mut self) -> SubSPHView<'_> {
+        assert!(self.shader_type != ShaderType::Fragment);
+        BitMutView::new_subset(&mut self.data, 576..592)
+    }
+
+    #[inline]
+    fn imap_system_values_d_ps(&mut self) -> SubSPHView<'_> {
+        assert!(self.shader_type == ShaderType::Fragment);
+        BitMutView::new_subset(&mut self.data, 560..576)
+    }
+
+    #[inline]
+    fn omap_target(&mut self) -> SubSPHView<'_> {
+        assert!(self.shader_type == ShaderType::Fragment);
+
+        BitMutView::new_subset(&mut self.data, 576..608)
+    }
+
+    #[inline]
+    fn omap_system_values_d_vtg(&mut self) -> SubSPHView<'_> {
+        assert!(self.shader_type != ShaderType::Fragment);
+        BitMutView::new_subset(&mut self.data, 632..640)
+    }
+
+    #[inline]
+    fn set_sph_type(&mut self, sph_type: u32, sph_version: u8) {
+        self.set_field(SPHV3_T1_SPH_TYPE, sph_type);
+        self.set_field(SPHV3_T1_VERSION, sph_version);
+    }
+
+    #[inline]
+    fn set_shader_type(&mut self, shader_type: ShaderType) {
+        self.set_field(
+            SPHV3_T1_SHADER_TYPE,
+            match shader_type {
+                ShaderType::Vertex => SPHV3_T1_SHADER_TYPE_VERTEX,
+                ShaderType::TessellationInit => {
+                    SPHV3_T1_SHADER_TYPE_TESSELLATION_INIT
+                }
+                ShaderType::Tessellation => SPHV3_T1_SHADER_TYPE_TESSELLATION,
+                ShaderType::Geometry => SPHV3_T1_SHADER_TYPE_GEOMETRY,
+                ShaderType::Fragment => SPHV3_T1_SHADER_TYPE_PIXEL,
+            },
+        );
+    }
+
+    #[inline]
+    pub fn set_multiple_render_target_enable(&mut self, mrt_enable: bool) {
+        self.set_field(SPHV3_T1_MRT_ENABLE, mrt_enable);
+    }
+
+    #[inline]
+    pub fn set_kills_pixels(&mut self, kills_pixels: bool) {
+        self.set_field(SPHV3_T1_KILLS_PIXELS, kills_pixels);
+    }
+
+    #[inline]
+    pub fn set_does_global_store(&mut self, does_global_store: bool) {
+        self.set_field(SPHV3_T1_DOES_GLOBAL_STORE, does_global_store);
+    }
+
+    #[inline]
+    pub fn set_sass_version(&mut self, sass_version: u8) {
+        self.set_field(SPHV3_T1_SASS_VERSION, sass_version);
+    }
+
+    #[inline]
+    pub fn set_gs_passthrough_enable(&mut self, gs_passthrough_enable: bool) {
+        assert!(self.shader_type == ShaderType::Geometry);
+        self.set_bit(24, gs_passthrough_enable);
+    }
+
+    #[inline]
+    pub fn set_isbe_space_sharing_enable(
+        &mut self,
+        isbe_space_sharing_enable: bool,
+    ) {
+        assert!(self.shader_type == ShaderType::Vertex);
+        self.set_bit(25, isbe_space_sharing_enable);
+    }
+
+    #[inline]
+    pub fn set_does_load_or_store(&mut self, does_load_or_store: bool) {
+        self.set_field(SPHV3_T1_DOES_LOAD_OR_STORE, does_load_or_store);
+    }
+
+    #[inline]
+    pub fn set_does_fp64(&mut self, does_fp64: bool) {
+        self.set_field(SPHV3_T1_DOES_FP64, does_fp64);
+    }
+
+    #[inline]
+    pub fn set_stream_out_mask(&mut self, stream_out_mask: u8) {
+        self.set_field(SPHV3_T1_STREAM_OUT_MASK, stream_out_mask);
+    }
+
+    #[inline]
+    pub fn set_shader_local_memory_size(
+        &mut self,
+        shader_local_memory_size: u64,
+    ) {
+        assert!(shader_local_memory_size <= 0xffffffffffff);
+        assert!(shader_local_memory_size % 0x10 == 0);
+
+        let low = (shader_local_memory_size & 0xffffff) as u32;
+        let high = ((shader_local_memory_size >> 32) & 0xffffff) as u32;
+
+        self.set_field(SPHV3_T1_SHADER_LOCAL_MEMORY_LOW_SIZE, low);
+        self.set_field(SPHV3_T1_SHADER_LOCAL_MEMORY_HIGH_SIZE, high);
+    }
+
+    #[inline]
+    pub fn set_per_patch_attribute_count(
+        &mut self,
+        per_patch_attribute_count: u8,
+    ) {
+        assert!(self.shader_type == ShaderType::TessellationInit);
+
+        self.set_field(
+            SPHV3_T1_PER_PATCH_ATTRIBUTE_COUNT,
+            per_patch_attribute_count,
+        );
+
+        // This is Kepler+
+        self.set_field(
+            SPHV3_T1_RESERVED_COMMON_B,
+            per_patch_attribute_count & 0xf,
+        );
+        self.set_field(148..152, per_patch_attribute_count >> 4);
+    }
+
+    #[inline]
+    pub fn set_threads_per_input_primitive(
+        &mut self,
+        threads_per_input_primitive: u8,
+    ) {
+        self.set_field(
+            SPHV3_T1_THREADS_PER_INPUT_PRIMITIVE,
+            threads_per_input_primitive,
+        );
+    }
+
+    #[inline]
+    pub fn set_shader_local_memory_crs_size(
+        &mut self,
+        shader_local_memory_crs_size: u32,
+    ) {
+        assert!(shader_local_memory_crs_size <= 0xffffff);
+        self.set_field(
+            SPHV3_T1_SHADER_LOCAL_MEMORY_CRS_SIZE,
+            shader_local_memory_crs_size,
+        );
+    }
+
+    #[inline]
+    pub fn set_output_topology(&mut self, output_topology: OutputTopology) {
+        self.set_field(
+            SPHV3_T1_OUTPUT_TOPOLOGY,
+            match output_topology {
+                OutputTopology::PointList => SPHV3_T1_OUTPUT_TOPOLOGY_POINTLIST,
+                OutputTopology::LineStrip => SPHV3_T1_OUTPUT_TOPOLOGY_LINESTRIP,
+                OutputTopology::TriangleStrip => {
+                    SPHV3_T1_OUTPUT_TOPOLOGY_TRIANGLESTRIP
+                }
+            },
+        );
+    }
+
+    #[inline]
+    pub fn set_max_output_vertex_count(
+        &mut self,
+        max_output_vertex_count: u16,
+    ) {
+        assert!(max_output_vertex_count <= 0xfff);
+        self.set_field(
+            SPHV3_T1_MAX_OUTPUT_VERTEX_COUNT,
+            max_output_vertex_count,
+        );
+    }
+
+    #[inline]
+    pub fn set_store_req_start(&mut self, store_req_start: u8) {
+        self.set_field(SPHV3_T1_STORE_REQ_START, store_req_start);
+    }
+
+    #[inline]
+    pub fn set_store_req_end(&mut self, store_req_end: u8) {
+        self.set_field(SPHV3_T1_STORE_REQ_END, store_req_end);
+    }
+
+    pub fn set_imap_system_values_ab(&mut self, val: u32) {
+        self.imap_system_values_ab().set_field(0..32, val);
+    }
+
+    pub fn set_imap_system_values_c(&mut self, val: u16) {
+        self.imap_system_values_c().set_field(0..16, val);
+    }
+
+    pub fn set_imap_system_values_d_vtg(&mut self, val: u8) {
+        assert!(self.shader_type != ShaderType::Fragment);
+        self.imap_system_values_d_vtg().set_field(0..8, val);
+    }
+
+    #[inline]
+    pub fn set_imap_vector_ps(&mut self, index: usize, value: PixelImap) {
+        assert!(index < 128);
+        assert!(self.shader_type == ShaderType::Fragment);
+
+        self.imap_g_ps()
+            .set_field(index * 2..(index + 1) * 2, u8::from(value));
+    }
+
+    #[inline]
+    pub fn set_imap_system_values_d_ps(
+        &mut self,
+        index: usize,
+        value: PixelImap,
+    ) {
+        assert!(index < 8);
+        assert!(self.shader_type == ShaderType::Fragment);
+
+        self.imap_system_values_d_ps()
+            .set_field(index * 2..(index + 1) * 2, u8::from(value));
+    }
+
+    #[inline]
+    pub fn set_imap_vector_vtg(&mut self, index: usize, value: u32) {
+        assert!(index < 4);
+        assert!(self.shader_type != ShaderType::Fragment);
+
+        self.imap_g_vtg()
+            .set_field(index * 32..(index + 1) * 32, value);
+    }
+
+    #[inline]
+    pub fn set_omap_system_values_ab(&mut self, val: u32) {
+        self.omap_system_values_ab().set_field(0..32, val);
+    }
+
+    #[inline]
+    pub fn set_omap_system_values_c(&mut self, val: u16) {
+        self.omap_system_values_c().set_field(0..16, val);
+    }
+
+    pub fn set_omap_system_values_d_vtg(&mut self, val: u8) {
+        assert!(self.shader_type != ShaderType::Fragment);
+        self.omap_system_values_d_vtg().set_field(0..8, val);
+    }
+
+    #[inline]
+    pub fn set_omap_vector(&mut self, index: usize, value: u32) {
+        assert!(index < 4);
+        assert!(self.shader_type != ShaderType::Fragment);
+
+        self.omap_g().set_field(index * 32..(index + 1) * 32, value);
+    }
+
+    #[inline]
+    pub fn set_omap_targets(&mut self, value: u32) {
+        self.omap_target().set_field(0..32, value)
+    }
+
+    #[inline]
+    pub fn set_omap_sample_mask(&mut self, sample_mask: bool) {
+        assert!(self.shader_type == ShaderType::Fragment);
+        self.set_field(SPHV3_T2_OMAP_SAMPLE_MASK, sample_mask);
+    }
+
+    #[inline]
+    pub fn set_omap_depth(&mut self, depth: bool) {
+        assert!(self.shader_type == ShaderType::Fragment);
+        self.set_field(SPHV3_T2_OMAP_DEPTH, depth);
+    }
+
+    #[inline]
+    pub fn set_does_interlock(&mut self, does_interlock: bool) {
+        assert!(self.shader_type == ShaderType::Fragment);
+        self.set_bit(610, does_interlock);
+    }
+
+    #[inline]
+    pub fn set_uses_underestimate(&mut self, uses_underestimate: bool) {
+        assert!(self.shader_type == ShaderType::Fragment);
+        self.set_bit(611, uses_underestimate);
+    }
+
+    #[inline]
+    fn pervertex_imap_vector_ps(&mut self) -> SubSPHView<'_> {
+        assert!(self.shader_type == ShaderType::Fragment);
+
+        BitMutView::new_subset(&mut self.data, 672..800)
+    }
+
+    #[inline]
+    pub fn set_pervertex_imap_vector(&mut self, index: usize, value: u32) {
+        assert!(index < 4);
+        assert!(self.shader_type == ShaderType::Fragment);
+
+        self.pervertex_imap_vector_ps()
+            .set_field(index * 32..(index + 1) * 32, value);
+    }
+}
+
+fn encode_vtg_io(sph: &mut ShaderProgramHeader, io: &VtgIoInfo) {
+    sph.set_imap_system_values_ab(io.sysvals_in.ab);
+    sph.set_imap_system_values_c(io.sysvals_in.c);
+    sph.set_imap_system_values_d_vtg(io.sysvals_in_d);
+
+    for (index, value) in io.attr_in.iter().enumerate() {
+        sph.set_imap_vector_vtg(index, *value);
+    }
+
+    for (index, value) in io.attr_out.iter().enumerate() {
+        sph.set_omap_vector(index, *value);
+    }
+
+    sph.set_store_req_start(io.store_req_start);
+    sph.set_store_req_end(io.store_req_end);
+
+    sph.set_omap_system_values_ab(io.sysvals_out.ab);
+    sph.set_omap_system_values_c(io.sysvals_out.c);
+    sph.set_omap_system_values_d_vtg(io.sysvals_out_d);
+}
+
+pub fn encode_header(
+    sm: &ShaderModelInfo,
+    shader_info: &ShaderInfo,
+    fs_key: Option<&nak_fs_key>,
+) -> [u32; CURRENT_MAX_SHADER_HEADER_SIZE] {
+    if let ShaderStageInfo::Compute(_) = shader_info.stage {
+        return [0_u32; CURRENT_MAX_SHADER_HEADER_SIZE];
+    }
+
+    let mut sph =
+        ShaderProgramHeader::new(ShaderType::from(&shader_info.stage), sm.sm());
+
+    let slm_size = shader_info.slm_size.next_multiple_of(16);
+    sph.set_sass_version(1);
+    sph.set_does_load_or_store(
+        shader_info.uses_global_mem || (sm.is_kepler() && slm_size > 0),
+    );
+    sph.set_does_global_store(shader_info.writes_global_mem);
+    sph.set_does_fp64(shader_info.uses_fp64);
+
+    sph.set_shader_local_memory_size(slm_size.into());
+    let crs_size = sm.crs_size(shader_info.max_crs_depth);
+    sph.set_shader_local_memory_crs_size(crs_size);
+
+    match &shader_info.io {
+        ShaderIoInfo::Vtg(io) => encode_vtg_io(&mut sph, io),
+        ShaderIoInfo::Fragment(io) => {
+            sph.set_imap_system_values_ab(io.sysvals_in.ab);
+            sph.set_imap_system_values_c(io.sysvals_in.c);
+
+            for (index, imap) in io.sysvals_in_d.iter().enumerate() {
+                sph.set_imap_system_values_d_ps(index, *imap);
+            }
+
+            for (index, imap) in io.attr_in.iter().enumerate() {
+                sph.set_imap_vector_ps(index, *imap);
+            }
+
+            let uses_underestimate =
+                fs_key.is_some_and(|key| key.uses_underestimate);
+
+            // This isn't so much a "Do we write multiple render targets?" bit
+            // as a "Should color0 be broadcast to all render targets?" bit. In
+            // other words, it's the gl_FragCoord behavior, not gl_FragData.
+            //
+            // For now, we always set it to true because Vulkan requires
+            // explicit fragment output locations.
+            sph.set_multiple_render_target_enable(true);
+
+            sph.set_omap_sample_mask(io.writes_sample_mask);
+            sph.set_omap_depth(io.writes_depth);
+            sph.set_omap_targets(io.writes_color);
+            sph.set_uses_underestimate(uses_underestimate);
+
+            for (index, value) in io.barycentric_attr_in.iter().enumerate() {
+                sph.set_pervertex_imap_vector(index, *value);
+            }
+        }
+        _ => {}
+    }
+
+    match &shader_info.stage {
+        ShaderStageInfo::Vertex(stage) => {
+            sph.set_isbe_space_sharing_enable(stage.isbe_space_sharing_enable);
+        }
+        ShaderStageInfo::Fragment(stage) => {
+            let zs_self_dep = fs_key.is_some_and(|key| key.zs_self_dep);
+            sph.set_kills_pixels(stage.uses_kill || zs_self_dep);
+            sph.set_does_interlock(stage.does_interlock);
+        }
+        ShaderStageInfo::Geometry(stage) => {
+            sph.set_gs_passthrough_enable(stage.passthrough_enable);
+            sph.set_stream_out_mask(stage.stream_out_mask);
+            sph.set_threads_per_input_primitive(
+                stage.threads_per_input_primitive,
+            );
+            sph.set_output_topology(stage.output_topology);
+            sph.set_max_output_vertex_count(stage.max_output_vertex_count);
+        }
+        ShaderStageInfo::TessellationInit(stage) => {
+            sph.set_per_patch_attribute_count(stage.per_patch_attribute_count);
+            sph.set_threads_per_input_primitive(stage.threads_per_patch);
+        }
+        ShaderStageInfo::Compute(_) => {
+            panic!("Compute shaders don't have a SPH!")
+        }
+        _ => {}
+    };
+
+    sph.data
+}
+
+pub fn encode_gs_mesh_header(
+    sm: u8,
+    mesh_info: &MeshShaderInfo,
+) -> [u32; CURRENT_MAX_SHADER_HEADER_SIZE] {
+    // If there is no usage of per primitive output, we can skip the GS stage.
+    if !mesh_info.has_gs_sph {
+        return [0_u32; CURRENT_MAX_SHADER_HEADER_SIZE];
+    }
+
+    let mut sph = ShaderProgramHeader::new(ShaderType::Geometry, sm);
+    sph.set_sass_version(1);
+    sph.set_gs_passthrough_enable(true);
+    sph.set_stream_out_mask(0);
+    sph.set_threads_per_input_primitive(1);
+    sph.set_output_topology(OutputTopology::PointList);
+    sph.set_max_output_vertex_count(1);
+
+    encode_vtg_io(&mut sph, &mesh_info.primitive_io);
+
+    sph.data
+}
@@ -0,0 +1,338 @@
+// Copyright © 2025 Collabora, Ltd.
+// SPDX-License-Identifier: MIT
+
+use crate::ir::{HasRegFile, RegFile};
+use compiler::bitset::IntoBitIndex;
+use compiler::lower_bounded::*;
+use std::fmt;
+use std::ops::{Deref, DerefMut};
+
+type SSAValueInner = LowerBoundedU32<{ SSAValue::IDX_DELTA }>;
+
+/// An SSA value
+///
+/// Each SSA in NAK represents a single 32-bit or 1-bit (if a predicate) value
+/// which must either be spilled to memory or allocated space in the specified
+/// register file.  Whenever more data is required such as a 64-bit memory
+/// address, double-precision float, or a vec4 texture result, multiple SSA
+/// values are used.
+///
+/// Each SSA value logically contains two things: an index and a register file.
+/// It is required that each index refers to a unique SSA value, regardless of
+/// register file.  This way the index can be used to index tightly-packed data
+/// structures such as bitsets without having to determine separate ranges for
+/// each register file.
+#[repr(transparent)]
+#[derive(Clone, Copy, Eq, Hash, PartialEq)]
+pub struct SSAValue {
+    packed: SSAValueInner,
+}
+
+impl SSAValue {
+    const IDX_DELTA: u32 = 17;
+
+    /// Returns an SSA value with the given register file and index
+    fn new(file: RegFile, idx: u32) -> SSAValue {
+        assert!(idx < (1 << 29) - SSAValue::IDX_DELTA);
+        let mut packed = idx + SSAValue::IDX_DELTA;
+        assert!(u8::from(file) < 8);
+        packed |= u32::from(u8::from(file)) << 29;
+        SSAValue {
+            packed: packed.try_into().unwrap(),
+        }
+    }
+
+    /// Returns the index of this SSA value
+    pub fn idx(&self) -> u32 {
+        (self.packed.get() & 0x1fffffff) - SSAValue::IDX_DELTA
+    }
+}
+
+impl HasRegFile for SSAValue {
+    /// Returns the register file of this SSA value
+    fn file(&self) -> RegFile {
+        RegFile::try_from((self.packed.get() >> 29) as u8).unwrap()
+    }
+}
+
+impl IntoBitIndex for SSAValue {
+    fn into_bit_index(self) -> usize {
+        // Indices are guaranteed unique by the allocator
+        self.idx().try_into().unwrap()
+    }
+}
+
+impl fmt::Display for SSAValue {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "%{}{}", self.file().fmt_prefix(), self.idx())
+    }
+}
+
+impl fmt::Debug for SSAValue {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fmt::Display::fmt(self, f)
+    }
+}
+
+type SSARefSmall =
+    LowerBoundedU32Array<{ SSAValue::IDX_DELTA }, { SSARef::SMALL_MAX_IDX }>;
+type SSARefLarge =
+    LowerBoundedU32Array<{ SSAValue::IDX_DELTA }, { SSARef::LARGE_MAX_IDX }>;
+
+#[derive(Clone, Eq, Hash, PartialEq)]
+enum SSARefInner {
+    Small(SSARefSmall),
+    Large(Box<SSARefLarge>),
+}
+
+impl SSARefInner {
+    fn push(&mut self, val: SSAValue) {
+        match self {
+            SSARefInner::Small(small) => {
+                if small.try_push(val.packed).is_ok() {
+                    return;
+                }
+
+                SSARef::cold();
+                let mut large: Box<SSARefLarge> = Default::default();
+                for s in small.into_iter() {
+                    large.try_push(s).expect("Small has to fit in large");
+                }
+                large.try_push(val.packed).unwrap();
+                *self = Self::Large(large);
+            }
+            SSARefInner::Large(arr) => {
+                SSARef::cold();
+                arr.try_push(val.packed)
+                    .expect("Too many vector components for SSARef");
+            }
+        }
+    }
+}
+
+/// A reference to one or more SSA values
+///
+/// Because each SSA value represents a single 1 or 32-bit scalar, we need a way
+/// to reference multiple SSA values for instructions which read or write
+/// multiple registers in the same source.  When the register allocator runs,
+/// all the SSA values in a given SSA ref will be placed in consecutive
+/// registers, with the base register aligned to the number of values, aligned
+/// to the next power of two.
+///
+/// An SSA reference can reference between 1 and 16 SSA values.  It dereferences
+/// to a slice for easy access to individual SSA values.  The structure is
+/// designed so that is always 16B, regardless of how many SSA values are
+/// referenced so it's easy and fairly cheap to clone and embed in other
+/// structures.
+#[derive(Clone, Eq, Hash, PartialEq)]
+pub struct SSARef {
+    v: SSARefInner,
+}
+
+#[cfg(target_arch = "x86_64")]
+const _: () = {
+    debug_assert!(size_of::<SSARef>() == 16);
+};
+
+impl SSARef {
+    const SMALL_MAX_IDX: usize = 3;
+    const LARGE_MAX_IDX: usize = 15;
+
+    /// Returns a new SSA reference.
+    ///
+    /// # Panics
+    ///
+    /// This method will panic if the number of SSA values in the slice do not
+    /// fit in an SSARef.
+    #[inline]
+    pub fn new(comps: &[SSAValue]) -> SSARef {
+        assert!(comps.len() > 0);
+        let mut v = SSARefInner::Small(Default::default());
+        for ssa in comps {
+            v.push(*ssa);
+        }
+        SSARef { v }
+    }
+
+    /// Constructs an SSA reference from an iterator of SSA values.
+    ///
+    /// # Panics
+    ///
+    /// This method will panic if the number of SSA values in the slice do not
+    /// fit in an SSARef.
+    fn from_iter(it: impl ExactSizeIterator<Item = SSAValue>) -> Self {
+        assert!(it.len() > 0);
+        let mut v = SSARefInner::Small(Default::default());
+        for ssa in it {
+            v.push(ssa);
+        }
+        SSARef { v }
+    }
+
+    /// Returns the number of components in this SSA reference.
+    pub fn comps(&self) -> u8 {
+        match &self.v {
+            SSARefInner::Small(x) => x.len() as u8,
+            SSARefInner::Large(x) => {
+                Self::cold();
+                x.len() as u8
+            }
+        }
+    }
+
+    #[cold]
+    #[inline]
+    fn cold() {}
+}
+
+impl Deref for SSARef {
+    type Target = [SSAValue];
+
+    fn deref(&self) -> &[SSAValue] {
+        let s = match &self.v {
+            SSARefInner::Small(x) => x.as_slice(),
+            SSARefInner::Large(x) => {
+                Self::cold();
+                x.as_slice()
+            }
+        };
+        // SAFETY: SSAValue is repr(transparent) of SSAValueInner
+        unsafe { std::mem::transmute(s) }
+    }
+}
+
+impl DerefMut for SSARef {
+    fn deref_mut(&mut self) -> &mut [SSAValue] {
+        let s = match &mut self.v {
+            SSARefInner::Small(x) => x.as_mut_slice(),
+            SSARefInner::Large(x) => {
+                Self::cold();
+                x.as_mut_slice()
+            }
+        };
+        // SAFETY: SSAValue is repr(transparent) of SSAValueInner
+        unsafe { std::mem::transmute(s) }
+    }
+}
+
+impl TryFrom<&[SSAValue]> for SSARef {
+    type Error = &'static str;
+
+    fn try_from(comps: &[SSAValue]) -> Result<Self, Self::Error> {
+        if comps.len() == 0 {
+            Err("Empty vector")
+        } else if comps.len() > SSARefLarge::MAX_LEN {
+            Err("Too many vector components")
+        } else {
+            Ok(SSARef::new(comps))
+        }
+    }
+}
+
+impl TryFrom<Vec<SSAValue>> for SSARef {
+    type Error = &'static str;
+
+    fn try_from(comps: Vec<SSAValue>) -> Result<Self, Self::Error> {
+        SSARef::try_from(&comps[..])
+    }
+}
+
+macro_rules! impl_ssa_ref_from_arr {
+    ($n: expr) => {
+        impl From<[SSAValue; $n]> for SSARef {
+            fn from(comps: [SSAValue; $n]) -> Self {
+                SSARef::new(&comps[..])
+            }
+        }
+    };
+}
+impl_ssa_ref_from_arr!(1);
+impl_ssa_ref_from_arr!(2);
+impl_ssa_ref_from_arr!(3);
+impl_ssa_ref_from_arr!(4);
+
+impl From<SSAValue> for SSARef {
+    fn from(val: SSAValue) -> Self {
+        [val].into()
+    }
+}
+
+impl fmt::Display for SSARef {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        if self.comps() == 1 {
+            write!(f, "{}", self[0])
+        } else {
+            write!(f, "{{")?;
+            for (i, v) in self.iter().enumerate() {
+                if i != 0 {
+                    write!(f, " ")?;
+                }
+                write!(f, "{}", v)?;
+            }
+            write!(f, "}}")
+        }
+    }
+}
+
+impl HasRegFile for SSARef {
+    fn file(&self) -> RegFile {
+        (&self[..]).file()
+    }
+}
+
+#[test]
+fn test_ssa_ref_round_trip() {
+    for len in 1..16 {
+        let vec: Vec<_> = (0..len)
+            .map(|i| SSAValue::new(RegFile::GPR, 1337 ^ i ^ len))
+            .collect();
+
+        let ssa_ref = SSARef::new(&vec);
+        assert_eq!(ssa_ref[..], vec[..]);
+    }
+}
+
+pub struct SSAValueAllocator {
+    count: u32,
+}
+
+/// An allocator for SSA values.
+///
+/// This is the only valid way to create SSAValues.  At most one SSA value
+/// allocator may exist per shader to ensure the invariant that SSA value
+/// indices are unique.
+impl SSAValueAllocator {
+    /// Creates a new SSA value allocator.
+    pub fn new() -> SSAValueAllocator {
+        SSAValueAllocator { count: 0 }
+    }
+
+    #[allow(dead_code)]
+    pub fn max_idx(&self) -> u32 {
+        self.count
+    }
+
+    /// Allocates an SSA value.
+    pub fn alloc(&mut self, file: RegFile) -> SSAValue {
+        let idx = self.count;
+        self.count += 1;
+        SSAValue::new(file, idx)
+    }
+
+    /// Allocates multiple SSA values and returns them as an SSA reference.
+    pub fn alloc_vec(&mut self, file: RegFile, comps: u8) -> SSARef {
+        SSARef::from_iter((0..comps).map(|_| self.alloc(file)))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_ssa_value_print() {
+        let ssa = SSAValue::new(RegFile::UPred, 42);
+        assert_eq!(format!("{}", ssa), "%up42");
+        assert_eq!(format!("{:?}", ssa), "%up42");
+    }
+}
@@ -0,0 +1,404 @@
+// Copyright © 2023 Collabora, Ltd.
+// SPDX-License-Identifier: MIT
+
+use crate::api::{GetDebugFlags, DEBUG};
+use crate::ir::*;
+use crate::liveness::{BlockLiveness, Liveness, SimpleLiveness};
+
+use compiler::cfg::CFG;
+use rustc_hash::FxHashMap;
+use std::iter::Peekable;
+
+struct MergedIter<I: Iterator> {
+    a: Peekable<I>,
+    b: Peekable<I>,
+}
+
+impl<I: Iterator> MergedIter<I> {
+    fn new(a: I, b: I) -> Self {
+        Self {
+            a: a.peekable(),
+            b: b.peekable(),
+        }
+    }
+}
+
+impl<I: Iterator> Iterator for MergedIter<I>
+where
+    <I as Iterator>::Item: Ord,
+{
+    type Item = <I as Iterator>::Item;
+
+    fn next(&mut self) -> Option<<I as Iterator>::Item> {
+        if let Some(a) = self.a.peek() {
+            if let Some(b) = self.b.peek() {
+                if a <= b {
+                    self.a.next()
+                } else {
+                    self.b.next()
+                }
+            } else {
+                self.a.next()
+            }
+        } else {
+            self.b.next()
+        }
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        let (a_max, a_size) = self.a.size_hint();
+        let (b_max, b_size) = self.b.size_hint();
+        (a_max + b_max, a_size.zip(b_size).map(|(a, b)| a + b))
+    }
+}
+
+enum CoalesceItem {
+    SSA(SSAValue),
+    Phi(Phi),
+}
+
+struct CoalesceNode {
+    set: usize,
+    block: usize,
+    ip_1: usize,
+    item: CoalesceItem,
+}
+
+struct CoalesceSet {
+    nodes: Vec<usize>,
+}
+
+struct CoalesceGraph<'a> {
+    live: &'a SimpleLiveness,
+    nodes: Vec<CoalesceNode>,
+    sets: Vec<CoalesceSet>,
+    ssa_node: FxHashMap<SSAValue, usize>,
+    phi_node_file: FxHashMap<Phi, (usize, RegFile)>,
+}
+
+impl<'a> CoalesceGraph<'a> {
+    fn new(live: &'a SimpleLiveness) -> Self {
+        Self {
+            live: live,
+            nodes: Vec::new(),
+            sets: Vec::new(),
+            ssa_node: Default::default(),
+            phi_node_file: Default::default(),
+        }
+    }
+
+    fn add_ssa(&mut self, ssa: SSAValue) {
+        debug_assert!(self.sets.is_empty());
+
+        // Set it to usize::MAX for now.  We'll update later
+        if self.ssa_node.insert(ssa, usize::MAX).is_none() {
+            let (block, ip) = self.live.def_block_ip(&ssa);
+            self.nodes.push(CoalesceNode {
+                set: usize::MAX,
+                block: block,
+                ip_1: ip + 1,
+                item: CoalesceItem::SSA(ssa),
+            });
+        }
+    }
+
+    fn add_phi_dst(&mut self, phi: Phi, file: RegFile, block: usize) {
+        debug_assert!(self.sets.is_empty());
+
+        // Record the register file now.  We'll set the node later
+        let old = self.phi_node_file.insert(phi, (usize::MAX, file));
+        debug_assert!(old.is_none());
+
+        self.nodes.push(CoalesceNode {
+            set: usize::MAX,
+            block: block,
+            ip_1: 0,
+            item: CoalesceItem::Phi(phi),
+        });
+    }
+
+    fn add_phi_src(&mut self, phi: Phi, block: usize) {
+        debug_assert!(self.sets.is_empty());
+
+        self.nodes.push(CoalesceNode {
+            set: usize::MAX,
+            block: block,
+            ip_1: usize::MAX,
+            item: CoalesceItem::Phi(phi),
+        });
+    }
+
+    fn init_sets<N>(&mut self, cfg: &CFG<N>) {
+        // Sort the nodes by dom_dfs_pre_index followed by ip+1.  Stash the
+        // dom_dfs_pre_index in the set for now.  We don't actually fill out
+        // the set field until later.
+        for n in self.nodes.iter_mut() {
+            n.set = cfg.dom_dfs_pre_index(n.block);
+        }
+        self.nodes
+            .sort_by(|a, b| a.set.cmp(&b.set).then(a.ip_1.cmp(&b.ip_1)));
+
+        for ni in 0..self.nodes.len() {
+            match &self.nodes[ni].item {
+                CoalesceItem::SSA(ssa) => {
+                    let old = self.ssa_node.insert(*ssa, ni);
+                    debug_assert!(old == Some(usize::MAX));
+
+                    self.nodes[ni].set = self.sets.len();
+                    self.sets.push(CoalesceSet { nodes: vec![ni] });
+                }
+                CoalesceItem::Phi(phi) => {
+                    let (pn, _) = self.phi_node_file.get_mut(phi).unwrap();
+
+                    // We only want one set per phi and phi_node contains the
+                    // index to any one of the nodes.
+                    if *pn == usize::MAX {
+                        self.nodes[ni].set = self.sets.len();
+                        self.sets.push(CoalesceSet { nodes: vec![ni] });
+                        *pn = ni;
+                    } else {
+                        let s = self.nodes[*pn].set;
+                        self.nodes[ni].set = s;
+                    }
+                }
+            }
+        }
+    }
+
+    fn node_dominates<N>(&self, p: usize, c: usize, cfg: &CFG<N>) -> bool {
+        if self.nodes[p].block == self.nodes[c].block {
+            self.nodes[p].ip_1 <= self.nodes[c].ip_1
+        } else {
+            cfg.dominates(self.nodes[p].block, self.nodes[c].block)
+        }
+    }
+
+    fn phi_ssa_interferes(&self, phi: &CoalesceNode, ssa: &SSAValue) -> bool {
+        if phi.ip_1 == 0 {
+            self.live.block_live(phi.block).is_live_in(ssa)
+        } else {
+            debug_assert!(phi.ip_1 == usize::MAX);
+            self.live.block_live(phi.block).is_live_out(ssa)
+        }
+    }
+
+    fn nodes_interfere(&self, a: usize, b: usize) -> bool {
+        let a = &self.nodes[a];
+        let b = &self.nodes[b];
+
+        match &a.item {
+            CoalesceItem::SSA(a_ssa) => match &b.item {
+                CoalesceItem::SSA(b_ssa) => self.live.interferes(a_ssa, b_ssa),
+                CoalesceItem::Phi(_) => self.phi_ssa_interferes(b, a_ssa),
+            },
+            CoalesceItem::Phi(_) => match &b.item {
+                CoalesceItem::SSA(b_ssa) => self.phi_ssa_interferes(a, b_ssa),
+                CoalesceItem::Phi(_) => {
+                    // Phi nodes represent the temporary SSA value made between
+                    // the parallel copy and the phi in the Boissinot algorithm
+                    // so they interfere if and only if they're in the same
+                    // block and both at the start or both at the end.
+                    a.block == b.block && a.ip_1 == b.ip_1
+                }
+            },
+        }
+    }
+
+    pub fn sets_interfere<N>(&self, a: usize, b: usize, cfg: &CFG<N>) -> bool {
+        let a = &self.sets[a];
+        let b = &self.sets[b];
+
+        // Stack of nodes which dominate the current node
+        let mut dom = Vec::new();
+
+        for n in MergedIter::new(a.nodes.iter(), b.nodes.iter()) {
+            while let Some(p) = dom.last() {
+                if !self.node_dominates(*p, *n, cfg) {
+                    dom.pop();
+                } else {
+                    break;
+                }
+            }
+
+            if let Some(p) = dom.last() {
+                if self.nodes_interfere(*n, *p) {
+                    return true;
+                }
+            }
+
+            dom.push(*n);
+        }
+
+        false
+    }
+
+    pub fn sets_merge(&mut self, a: usize, b: usize) -> usize {
+        let a_nodes = std::mem::take(&mut self.sets[a].nodes);
+        let b_nodes = std::mem::take(&mut self.sets[b].nodes);
+        let nodes = MergedIter::new(a_nodes.into_iter(), b_nodes.into_iter());
+
+        self.sets[a].nodes = nodes
+            .inspect(|&n| {
+                self.nodes[n].set = a;
+            })
+            .collect();
+
+        a
+    }
+
+    pub fn ssa_set(&self, ssa: &SSAValue) -> usize {
+        self.nodes[*self.ssa_node.get(ssa).unwrap()].set
+    }
+
+    pub fn phi_set_file(&self, phi: &Phi) -> (usize, RegFile) {
+        let (n, file) = self.phi_node_file.get(phi).unwrap();
+        (self.nodes[*n].set, *file)
+    }
+}
+
+impl Function {
+    /// Convert a function to CSSA (Conventional SSA) form
+    ///
+    /// In "Translating Out of Static Single Assignment Form" by Sreedhar, et.
+    /// al., they define CSSA form via what they call the Phi Congruence
+    /// Property:
+    ///
+    /// > The occurrences of all resources which belong to the same phi
+    /// > congruence class in a program can be replaced by a representative
+    /// > resource. After the replacement, the phi instruction can be
+    /// > eliminated without violating the semantics of the original program.
+    ///
+    /// A more compiler-theoretic definition of CSSA form is a version of SSA
+    /// form in which, for each phi, none of the SSA values involved in the phi
+    /// (either as a source or destination) interfere.  While most of the papers
+    /// discussing CSSA form do so in the context of out-of-SSA, this property
+    /// is also useful for SSA-based spilling and register allocation.
+    ///
+    /// Our implementation is based on the algorithm described in "Revisiting
+    /// Out-of-SSA Translation for Correctness, Code Quality, and Effciency" by
+    /// Boissinot et. al.  The primary difference between this algorithm and
+    /// the one in that paper is that we don't actually insert parallel copies
+    /// and remove redundant entries.  Instead, we treat OpPhiSrcs and OpPhiDsts
+    /// as as the parallel copies with the phi index standing in for all of the
+    /// SSA values used directly by the phi.  Then, instead of removing copies
+    /// where the source and destination don't interfere, we insert copies
+    /// whenever the source or destination and phi index do interfere.  This
+    /// lets us avoid inserting pointless instructions.
+    pub fn to_cssa(&mut self) {
+        let live = SimpleLiveness::for_function(self);
+
+        let mut cg = CoalesceGraph::new(&live);
+        for (bi, b) in self.blocks.iter().enumerate() {
+            if let Some(phi) = b.phi_dsts() {
+                for (idx, dst) in phi.dsts.iter() {
+                    let vec = dst.as_ssa().unwrap();
+                    debug_assert!(vec.comps() == 1);
+                    cg.add_ssa(vec[0]);
+                    cg.add_phi_dst(*idx, vec[0].file(), bi);
+                }
+            }
+
+            if let Some(phi) = b.phi_srcs() {
+                for (idx, src) in phi.srcs.iter() {
+                    if let SrcRef::SSA(vec) = &src.src_ref {
+                        debug_assert!(vec.comps() == 1);
+                        cg.add_ssa(vec[0]);
+                    }
+                    cg.add_phi_src(*idx, bi);
+                }
+            }
+        }
+        cg.init_sets(&self.blocks);
+
+        for bi in 0..self.blocks.len() {
+            let block_instrs = std::mem::take(&mut self.blocks[bi].instrs);
+
+            let mut instrs = Vec::new();
+            for mut instr in block_instrs.into_iter() {
+                match &mut instr.op {
+                    Op::PhiDsts(phi) => {
+                        let mut pcopy = OpParCopy::new();
+                        for (idx, dst) in phi.dsts.iter_mut() {
+                            let (ps, file) = cg.phi_set_file(idx);
+
+                            let vec = dst.as_ssa().unwrap();
+                            debug_assert!(vec.comps() == 1);
+                            debug_assert!(vec[0].file() == file);
+                            let ds = cg.ssa_set(&vec[0]);
+
+                            if !cg.sets_interfere(ps, ds, &self.blocks) {
+                                cg.sets_merge(ps, ds);
+                                continue;
+                            }
+
+                            let tmp = self.ssa_alloc.alloc(file);
+                            let old_dst = std::mem::replace(dst, tmp.into());
+                            pcopy.push(old_dst, tmp.into());
+                        }
+
+                        instrs.push(instr);
+                        if !pcopy.is_empty() {
+                            if DEBUG.annotate() {
+                                instrs.push(Instr::new(OpAnnotate {
+                                    annotation: "generated by to_cssa".into(),
+                                }));
+                            }
+                            instrs.push(Instr::new(pcopy));
+                        }
+                    }
+                    Op::PhiSrcs(phi) => {
+                        let mut pcopy = OpParCopy::new();
+                        for (idx, src) in phi.srcs.iter_mut() {
+                            let (ps, file) = cg.phi_set_file(idx);
+
+                            debug_assert!(src.is_unmodified());
+                            if let SrcRef::SSA(vec) = &src.src_ref {
+                                debug_assert!(vec.comps() == 1);
+                                if vec[0].file() == file {
+                                    let ss = cg.ssa_set(&vec[0]);
+                                    if cg.sets_interfere(ps, ss, &self.blocks) {
+                                        let tmp = self.ssa_alloc.alloc(file);
+                                        let old_src =
+                                            std::mem::replace(src, tmp.into());
+                                        pcopy.push(tmp.into(), old_src);
+                                    } else {
+                                        cg.sets_merge(ps, ss);
+                                    }
+                                    continue;
+                                }
+                            }
+
+                            // Non-SSA sources or sources with a different file
+                            // from the destination get an actual copy
+                            // instruction and are not considered part of the
+                            // parallel copy.
+                            let tmp = self.ssa_alloc.alloc(file);
+                            if DEBUG.annotate() {
+                                instrs.push(Instr::new(OpAnnotate {
+                                    annotation: "generated by to_cssa".into(),
+                                }));
+                            }
+                            let old_src = std::mem::replace(src, tmp.into());
+                            instrs.push(Instr::new(OpCopy {
+                                dst: tmp.into(),
+                                src: old_src,
+                            }));
+                        }
+
+                        if !pcopy.is_empty() {
+                            if DEBUG.annotate() {
+                                instrs.push(Instr::new(OpAnnotate {
+                                    annotation: "generated by to_cssa".into(),
+                                }));
+                            }
+                            instrs.push(Instr::new(pcopy));
+                        }
+                        instrs.push(instr);
+                    }
+                    _ => instrs.push(instr),
+                }
+            }
+            self.blocks[bi].instrs = instrs;
+        }
+    }
+}
@@ -0,0 +1,293 @@
+// Copyright © 2024 Mel Henning
+// SPDX-License-Identifier: MIT
+
+use std::collections::HashMap;
+use std::hash::{BuildHasher, Hash};
+
+#[derive(Copy, Clone)]
+struct Root<X: Copy> {
+    size: usize,
+    representative: X,
+}
+
+#[derive(Copy, Clone)]
+enum Node<X: Copy> {
+    Child { parent_idx: usize },
+    Root(Root<X>),
+}
+
+/// Union-find structure
+///
+/// This implementation follows Tarjan and van Leeuwen - specifically the
+/// "link by size" and "halving" variant.
+///
+/// Robert E. Tarjan and Jan van Leeuwen. 1984. Worst-case Analysis of Set
+///     Union Algorithms. J. ACM 31, 2 (April 1984), 245–281.
+///     <https://doi.org/10.1145/62.2160>
+pub struct UnionFind<X: Copy + Hash + Eq, H: BuildHasher + Default> {
+    idx_map: HashMap<X, usize, H>,
+    nodes: Vec<Node<X>>,
+}
+
+impl<X: Copy + Hash + Eq, H: BuildHasher + Default> UnionFind<X, H> {
+    /// Create a new union-find structure
+    ///
+    /// At initialization, each possible value is in its own set
+    pub fn new() -> Self {
+        UnionFind {
+            idx_map: Default::default(),
+            nodes: Vec::new(),
+        }
+    }
+
+    fn find_root(&mut self, mut idx: usize) -> (usize, Root<X>) {
+        loop {
+            match self.nodes[idx] {
+                Node::Child { parent_idx } => {
+                    match self.nodes[parent_idx] {
+                        Node::Child {
+                            parent_idx: grandparent_idx,
+                        } => {
+                            // "Halving" in Tarjan and van Leeuwen
+                            self.nodes[idx] = Node::Child {
+                                parent_idx: grandparent_idx,
+                            };
+                            idx = grandparent_idx;
+                        }
+                        Node::Root(parent_root) => {
+                            return (parent_idx, parent_root)
+                        }
+                    }
+                }
+                Node::Root(root) => return (idx, root),
+            }
+        }
+    }
+
+    /// Find the representative element for x
+    pub fn find(&mut self, x: X) -> X {
+        match self.idx_map.get(&x) {
+            Some(&idx) => {
+                let (_, Root { representative, .. }) = self.find_root(idx);
+                representative
+            }
+            None => x,
+        }
+    }
+
+    fn map_or_create(&mut self, x: X) -> usize {
+        *self.idx_map.entry(x).or_insert_with(|| {
+            self.nodes.push(Node::Root(Root {
+                size: 1,
+                representative: x,
+            }));
+            self.nodes.len() - 1
+        })
+    }
+
+    /// Union the sets containing a and b
+    ///
+    /// The representative for a will become the representative of
+    /// the combined set
+    pub fn union(&mut self, a: X, b: X) {
+        if a == b {
+            return;
+        }
+
+        let a_idx = self.map_or_create(a);
+        let b_idx = self.map_or_create(b);
+        let (a_root_idx, a_root) = self.find_root(a_idx);
+        let (b_root_idx, b_root) = self.find_root(b_idx);
+
+        if a_root_idx != b_root_idx {
+            // Keep the tree balanced
+            let (new_root_idx, new_child_idx) = if a_root.size >= b_root.size {
+                (a_root_idx, b_root_idx)
+            } else {
+                (b_root_idx, a_root_idx)
+            };
+
+            self.nodes[new_root_idx] = Node::Root(Root {
+                size: a_root.size + b_root.size,
+                representative: a_root.representative,
+            });
+            self.nodes[new_child_idx] = Node::Child {
+                parent_idx: new_root_idx,
+            };
+        }
+    }
+
+    /// Return true if find() is the identity mapping
+    pub fn is_empty(&self) -> bool {
+        self.nodes.is_empty()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::union_find::Node;
+    use crate::union_find::UnionFind;
+    use std::cmp::max;
+    use std::hash::BuildHasher;
+    use std::hash::Hash;
+    use std::hash::RandomState;
+
+    fn ceil_log2(x: usize) -> u32 {
+        assert!(x > 0);
+        usize::BITS - (x - 1).leading_zeros()
+    }
+
+    struct HeightInfo {
+        height: u32,
+        size: usize,
+    }
+
+    pub struct HeightCalc<'a, X: Copy + Hash + Eq, H: BuildHasher + Default> {
+        uf: &'a UnionFind<X, H>,
+        downward_edges: Vec<Vec<usize>>,
+    }
+
+    impl<'a, X: Copy + Hash + Eq, H: BuildHasher + Default> HeightCalc<'a, X, H> {
+        fn new(uf: &'a UnionFind<X, H>) -> Self {
+            let mut downward_edges: Vec<Vec<usize>> =
+                uf.nodes.iter().map(|_| Vec::new()).collect();
+            for (i, node) in uf.nodes.iter().enumerate() {
+                if let Node::Child { parent_idx } = node {
+                    downward_edges[*parent_idx].push(i);
+                }
+            }
+
+            HeightCalc { uf, downward_edges }
+        }
+
+        fn calc_info(&self, idx: usize) -> HeightInfo {
+            let mut result = HeightInfo { height: 0, size: 1 };
+            for child in &self.downward_edges[idx] {
+                let child_result = self.calc_info(*child);
+                result.height = max(result.height, child_result.height + 1);
+                result.size += child_result.size;
+            }
+            result
+        }
+
+        fn check_roots(&self) -> u32 {
+            let mut total_size = 0;
+            let mut max_height = 0;
+            for (i, node) in self.uf.nodes.iter().enumerate() {
+                if let Node::Root(root) = node {
+                    let info = self.calc_info(i);
+                    assert_eq!(root.size, info.size);
+
+                    total_size += info.size;
+                    max_height = max(max_height, info.height);
+
+                    let max_expected_height = ceil_log2(root.size + 1) - 1;
+                    if info.height > max_expected_height {
+                        eprintln!(
+                            "height {}\t max_expected_height {}\t size {}",
+                            info.height, max_expected_height, info.size
+                        );
+                    }
+                    assert!(info.height <= max_expected_height);
+                }
+            }
+            assert_eq!(total_size, self.uf.nodes.len());
+            assert_eq!(total_size, self.uf.idx_map.len());
+            max_height
+        }
+
+        pub fn check(uf: &'a UnionFind<X, H>) -> u32 {
+            HeightCalc::new(uf).check_roots()
+        }
+    }
+
+    #[test]
+    fn test_basic() {
+        let mut f = UnionFind::<u32, RandomState>::new();
+        assert_eq!(f.find(10), 10);
+        assert_eq!(f.find(12), 12);
+
+        f.union(10, 12);
+        f.union(11, 13);
+
+        HeightCalc::check(&f);
+
+        assert_eq!(f.find(13), 11);
+        assert_eq!(f.find(12), 10);
+        assert_eq!(f.find(11), 11);
+        assert_eq!(f.find(10), 10);
+
+        f.union(12, 13);
+
+        HeightCalc::check(&f);
+
+        assert_eq!(f.find(13), 10);
+        assert_eq!(f.find(12), 10);
+        assert_eq!(f.find(11), 10);
+        assert_eq!(f.find(10), 10);
+
+        assert_eq!(f.find(14), 14);
+
+        HeightCalc::check(&f);
+
+        // Union the set with itself
+        f.union(11, 10);
+
+        HeightCalc::check(&f);
+
+        assert_eq!(f.find(13), 10);
+        assert_eq!(f.find(12), 10);
+        assert_eq!(f.find(11), 10);
+        assert_eq!(f.find(10), 10);
+    }
+
+    #[test]
+    fn test_chain_a_height() {
+        let mut f = UnionFind::<u32, RandomState>::new();
+        for i in 0..1000 {
+            f.union(i, i + 1);
+            HeightCalc::check(&f);
+        }
+        assert_eq!(f.find(1000), 0);
+    }
+
+    #[test]
+    fn test_chain_b_height() {
+        let mut f = UnionFind::<u32, RandomState>::new();
+        for i in 0..1000 {
+            f.union(i + 1, i);
+            HeightCalc::check(&f);
+        }
+        assert_eq!(f.find(0), 1000);
+    }
+
+    #[test]
+    fn test_binary_tree_height() {
+        let height = 8;
+        let count = 1 << height;
+
+        let mut f = UnionFind::<usize, RandomState>::new();
+        for current_height in 0..height {
+            let stride = 1 << current_height;
+            for i in (0..count).step_by(2 * stride) {
+                f.union(i, i + stride);
+            }
+            let actual_height = HeightCalc::check(&f);
+
+            // actual_height can vary based on tiebreaker condition
+            assert!(
+                actual_height == current_height
+                    || actual_height == current_height + 1
+            );
+        }
+
+        // Check path halving
+        let actual_height_before = HeightCalc::check(&f);
+        for i in 0..count {
+            assert_eq!(f.find(i), 0);
+        }
+        let actual_height_after = HeightCalc::check(&f);
+
+        assert!(actual_height_after <= actual_height_before.div_ceil(2));
+    }
+}
@@ -0,0 +1,20 @@
+/*
+ * Copyright © 2022 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "nak_private.h"
+
+#include "nouveau_bo.h"
+#include "nouveau_context.h"
+#include "nouveau_device.h"
+
+#include <xf86drm.h>
+#include "drm-uapi/nouveau_drm.h"
+
+#define DRM_RS_IOCTL(FOO) \
+   DRM_RS_IOCTL_##FOO = DRM_IOCTL_##FOO
+
+enum ENUM_PACKED drm_rs_ioctls {
+   DRM_RS_IOCTL(NOUVEAU_EXEC),
+};
@@ -0,0 +1,117 @@
+# Copyright (C) 2023 Red Hat, Inc.
+# Copyright (C) 2021 Collabora, Ltd.
+# Copyright (C) 2016 Intel Corporation
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice (including the next
+# paragraph) shall be included in all copies or substantial portions of the
+# Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+import argparse
+import sys
+
+a = 'a'
+b = 'b'
+c = 'c'
+s = 's'
+
+# common conditions to improve readability
+volta = 'nak->sm >= 70 && nak->sm < 73'
+fp16_round_is_rtz = 'nir_is_rounding_mode_rtz(info->float_controls_execution_mode, 16)'
+
+algebraic_lowering = [
+    # Volta doesn't have `IMNMX`
+    (('imin', 'a', 'b'), ('bcsel', ('ilt', a, b), a, b), volta),
+    (('imax', 'a', 'b'), ('bcsel', ('ilt', a, b), b, a), volta),
+    (('umin', 'a', 'b'), ('bcsel', ('ult', a, b), a, b), volta),
+    (('umax', 'a', 'b'), ('bcsel', ('ult', a, b), b, a), volta),
+    (('iadd', 'a@64', ('ineg', 'b@64')), ('isub', a, b)),
+
+    (('iadd', ('iadd(is_used_once)', 'a(is_not_const)', '#b'), 'c(is_not_const)'),
+        ('iadd3', a, b, c), 'options->has_iadd3'),
+    (('iadd', ('iadd(is_used_once)', 'a(is_not_const)', 'b(is_not_const)'), '#c'),
+        ('iadd3', a, b, c), 'options->has_iadd3'),
+
+    (('iadd(is_used_by_non_ldc_nv)', 'a@32', ('ishl', 'b@32', '#s@32')),
+        ('lea_nv', a, b, s), 'nak->sm >= 70'),
+    (('iadd', 'a@64', ('ishl', 'b@64', '#s@32')),
+        ('lea_nv', a, b, s), 'nak->sm >= 70'),
+]
+
+for f2f16 in ['f2f16', 'f2f16_rtz', 'f2f16_rtne']:
+    algebraic_lowering += [
+        (('vec2', (f2f16 + '(is_used_once)', 'a@32'), (f2f16 + '(is_used_once)', 'b@32')), (f2f16, ('vec2', a, b)), 'nak->sm >= 86')
+    ]
+
+# If we find mufu surrounded by bit_size conversions, just do the op in the
+# original bit_size.
+# MUFU.F16 internally appears to operate with the same precision as F32 does
+# with the result being rounded towards zero to F16. EXP2 and RCP seem to be
+# off by one around Inf, so it's only safe if we can ignore inf for those.
+#
+# This was verified with the `hw_tests::test_op_mufu_f16_down`.
+
+# mufu.f16 for those is identical to mufu.f32 with rtz rounding except for results around infinity
+for op in ['fexp2', 'frcp']:
+    algebraic_lowering += [
+        (('f2f16_rtz(ninf)', (op + '(is_used_once)', ('f2f32', 'a@16'))), (op, a), 'nak->sm >= 73'),
+        (('f2f16(ninf)', (op + '(is_used_once)', ('f2f32', 'a@16'))),
+            (op, a),
+            'nak->sm >= 73 && ' + fp16_round_is_rtz),
+    ]
+
+# mufu.f16 for those is identical to mufu.f32 with rtz rounding
+for op in ['fcos_normalized_2_pi', 'flog2', 'frsq', 'fsin_normalized_2_pi', 'fsqrt']:
+    algebraic_lowering += [
+        (('f2f16_rtz', (op + '(is_used_once)', ('f2f32', 'a@16'))), (op, a), 'nak->sm >= 73'),
+        (('f2f16', (op + '(is_used_once)', ('f2f32', 'a@16'))),
+            (op, a),
+            'nak->sm >= 73 && ' + fp16_round_is_rtz),
+    ]
+
+# If contract is on we can always remove the conversions
+for op in ['fcos_normalized_2_pi', 'fexp2', 'flog2', 'frcp', 'frsq', 'fsin_normalized_2_pi', 'fsqrt']:
+    for f2f16 in ['f2f16_rtz', 'f2f16_rtne', 'f2f16']:
+        algebraic_lowering += [
+            ((f2f16 + '(contract)', (op + '(is_used_once)', ('f2f32', 'a@16'))),
+                (op, a), 'nak->sm >= 73'),
+            (('f2f32(contract)', (op + '(is_used_once)', (f2f16, 'a@32'))), (op, a)),
+    ]
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--out', required=True, help='Output file.')
+    parser.add_argument('-p', '--import-path', required=True)
+    args = parser.parse_args()
+    sys.path.insert(0, args.import_path)
+
+    import nir_algebraic  # pylint: disable=import-error
+
+    try:
+        with open(args.out, 'w', encoding='utf-8') as f:
+            f.write('#include "nak_private.h"')
+            f.write(nir_algebraic.AlgebraicPass(
+                "nak_nir_lower_algebraic_late",
+                algebraic_lowering,
+                [
+                    ("const struct nak_compiler *", "nak"),
+                ]).render())
+    except Exception:
+        sys.exit(1)
+
+if __name__ == '__main__':
+    main()
@@ -0,0 +1,539 @@
+/*
+ * Copyright © 2022 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "nak_private.h"
+#include "nir_builder.h"
+
+static void
+push_block(nir_builder *b, nir_block *block, bool divergent)
+{
+   assert(nir_cursors_equal(b->cursor, nir_after_impl(b->impl)));
+   block->divergent = divergent;
+   block->cf_node.parent = &b->impl->cf_node;
+   exec_list_push_tail(&b->impl->body, &block->cf_node.node);
+   b->cursor = nir_after_block(block);
+}
+
+enum scope_type {
+   SCOPE_TYPE_SHADER,
+   SCOPE_TYPE_IF_MERGE,
+   SCOPE_TYPE_LOOP_BREAK,
+   SCOPE_TYPE_LOOP_CONT,
+};
+
+struct scope {
+   enum scope_type type;
+
+   struct scope *parent;
+   uint32_t depth;
+
+   /**
+    * True if control-flow ever diverges within this scope, not accounting
+    * for divergence in child scopes.
+    */
+   bool divergent;
+
+   nir_block *merge;
+   nir_def *bar;
+
+   uint32_t escapes;
+};
+
+static struct scope
+push_scope(nir_builder *b,
+           enum scope_type scope_type,
+           struct scope *parent,
+           bool divergent,
+           bool needs_sync,
+           nir_block *merge_block)
+{
+   struct scope scope = {
+      .type = scope_type,
+      .parent = parent,
+      .depth = parent->depth + 1,
+      .divergent = parent->divergent || divergent,
+      .merge = merge_block,
+   };
+
+   if (needs_sync)
+      scope.bar = nir_bar_set_nv(b);
+
+   return scope;
+}
+
+static void
+pop_scope(nir_builder *b, nir_def *esc_reg, struct scope scope)
+{
+   if (scope.bar == NULL)
+      return;
+
+   nir_bar_sync_nv(b, scope.bar, scope.bar);
+
+   if (scope.escapes > 0) {
+      /* Find the nearest scope with a sync. */
+      nir_block *parent_merge = b->impl->end_block;
+      for (struct scope *p = scope.parent; p != NULL; p = p->parent) {
+         if (p->bar != NULL) {
+            parent_merge = p->merge;
+            break;
+         }
+      }
+
+      /* No escape is ~0, halt is 0, and we choose outer scope indices such
+       * that outer scopes always have lower indices than inner scopes.
+       */
+      nir_def *esc = nir_ult_imm(b, nir_load_reg(b, esc_reg), scope.depth);
+
+      /* We have to put the escape in its own block to avoid critical edges.
+       * If we just did goto_if, we would end up with multiple successors,
+       * including a jump to the parent's merge block which has multiple
+       * predecessors.
+       */
+      nir_block *esc_block = nir_block_create(b->shader);
+      nir_block *next_block = nir_block_create(b->shader);
+      nir_goto_if(b, esc_block, esc, next_block);
+      push_block(b, esc_block, false);
+      nir_goto(b, parent_merge);
+      push_block(b, next_block, scope.parent->divergent);
+   }
+}
+
+static enum scope_type
+jump_target_scope_type(nir_jump_type jump_type)
+{
+   switch (jump_type) {
+   case nir_jump_break:    return SCOPE_TYPE_LOOP_BREAK;
+   case nir_jump_continue: return SCOPE_TYPE_LOOP_CONT;
+   default:
+      UNREACHABLE("Unknown jump type");
+   }
+}
+
+static void
+break_scopes(nir_builder *b, nir_def *esc_reg,
+             struct scope *current_scope,
+             nir_jump_type jump_type)
+{
+   nir_block *first_sync = NULL;
+   uint32_t target_depth = UINT32_MAX;
+   enum scope_type target_scope_type = jump_target_scope_type(jump_type);
+   for (struct scope *scope = current_scope; scope; scope = scope->parent) {
+      if (first_sync == NULL && scope->bar != NULL)
+         first_sync = scope->merge;
+
+      if (scope->type == target_scope_type) {
+         if (first_sync == NULL) {
+            first_sync = scope->merge;
+         } else {
+            /* In order for our cascade to work, we need to have the invariant
+             * that anything which escapes any scope with a warp sync needs to
+             * target a scope with a warp sync.
+             */
+            assert(scope->bar != NULL);
+         }
+         target_depth = scope->depth;
+         break;
+      } else {
+         scope->escapes++;
+      }
+   }
+   assert(target_depth < UINT32_MAX);
+
+   nir_store_reg(b, nir_imm_int(b, target_depth), esc_reg);
+   nir_goto(b, first_sync);
+}
+
+static void
+normal_exit(nir_builder *b, nir_def *esc_reg, nir_block *merge_block)
+{
+   assert(nir_cursors_equal(b->cursor, nir_after_impl(b->impl)));
+   nir_block *block = nir_cursor_current_block(b->cursor);
+
+   if (!nir_block_ends_in_jump(block)) {
+      nir_store_reg(b, nir_imm_int(b, ~0), esc_reg);
+      nir_goto(b, merge_block);
+   }
+}
+
+/* This is a heuristic for what instructions are allowed before we sync.
+ * Annoyingly, we've gotten rid of phis so it's not as simple as "is it a
+ * phi?".
+ */
+static bool
+instr_is_allowed_before_sync(nir_instr *instr)
+{
+   switch (instr->type) {
+   case nir_instr_type_alu: {
+      nir_alu_instr *alu = nir_instr_as_alu(instr);
+      /* We could probably allow more ALU as long as it doesn't contain
+       * derivatives but let's be conservative and only allow mov for now.
+       */
+      return alu->op == nir_op_mov;
+   }
+
+   case nir_instr_type_intrinsic: {
+      nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
+      return intrin->intrinsic == nir_intrinsic_load_reg ||
+             intrin->intrinsic == nir_intrinsic_store_reg;
+   }
+
+   default:
+      return false;
+   }
+}
+
+/** Returns true if our successor will sync for us
+ *
+ * This is a bit of a heuristic
+ */
+static bool
+parent_scope_will_sync(nir_cf_node *node, struct scope *parent_scope)
+{
+   /* First search forward to see if there's anything non-trivial after this
+    * node within the parent scope.
+    */
+   nir_block *block = nir_cf_node_as_block(nir_cf_node_next(node));
+   nir_foreach_instr(instr, block) {
+      if (!instr_is_allowed_before_sync(instr))
+         return false;
+   }
+
+   /* There's another loop or if following and we didn't find a sync */
+   if (nir_cf_node_next(&block->cf_node))
+      return false;
+
+   /* See if the parent scope will sync for us. */
+   if (parent_scope->bar != NULL)
+      return true;
+
+   switch (parent_scope->type) {
+   case SCOPE_TYPE_SHADER:
+      return true;
+
+   case SCOPE_TYPE_IF_MERGE:
+      return parent_scope_will_sync(block->cf_node.parent,
+                                    parent_scope->parent);
+
+   case SCOPE_TYPE_LOOP_CONT:
+      /* In this case, the loop doesn't have a sync of its own so we're
+       * expected to be uniform before we hit the continue.
+       */
+      return false;
+
+   case SCOPE_TYPE_LOOP_BREAK:
+      UNREACHABLE("Loops must have a continue scope");
+
+   default:
+      UNREACHABLE("Unknown scope type");
+   }
+}
+
+static bool
+block_is_merge(const nir_block *block)
+{
+   /* If it's unreachable, there is no merge */
+   if (block->imm_dom == NULL)
+      return false;
+
+   unsigned num_preds = 0;
+   nir_foreach_pred(pred, block) {
+      /* We don't care about unreachable blocks */
+      if (pred->imm_dom == NULL)
+         continue;
+
+      num_preds++;
+   }
+
+   return num_preds > 1;
+}
+
+static void
+lower_cf_list(nir_builder *b, nir_def *esc_reg, struct scope *parent_scope,
+              struct exec_list *cf_list)
+{
+   foreach_list_typed_safe(nir_cf_node, node, node, cf_list) {
+      switch (node->type) {
+      case nir_cf_node_block: {
+         nir_block *block = nir_cf_node_as_block(node);
+         if (exec_list_is_empty(&block->instr_list))
+            break;
+
+         nir_cursor start = nir_before_block(block);
+         nir_cursor end = nir_after_block(block);
+
+         nir_jump_instr *jump = NULL;
+         nir_instr *last_instr = nir_block_last_instr(block);
+         if (last_instr->type == nir_instr_type_jump) {
+            jump = nir_instr_as_jump(last_instr);
+            end = nir_before_instr(&jump->instr);
+         }
+
+         nir_cf_list instrs;
+         nir_cf_extract(&instrs, start, end);
+         b->cursor = nir_cf_reinsert(&instrs, b->cursor);
+
+         if (jump != NULL) {
+            if (jump->type == nir_jump_halt) {
+               /* Halt instructions map to OpExit on NVIDIA hardware and
+                * exited lanes never block a bsync.
+                */
+               nir_instr_remove(&jump->instr);
+               nir_builder_instr_insert(b, &jump->instr);
+            } else {
+               /* Everything else needs a break cascade */
+               break_scopes(b, esc_reg, parent_scope, jump->type);
+            }
+         }
+         break;
+      }
+
+      case nir_cf_node_if: {
+         nir_if *nif = nir_cf_node_as_if(node);
+
+         nir_def *cond = nif->condition.ssa;
+         bool divergent = nir_src_is_divergent(&nif->condition);
+         nir_instr_clear_src(NULL, &nif->condition);
+
+         nir_block *then_block = nir_block_create(b->shader);
+         nir_block *else_block = nir_block_create(b->shader);
+         nir_block *merge_block = nir_block_create(b->shader);
+
+         const bool needs_sync = divergent &&
+            block_is_merge(nir_cf_node_as_block(nir_cf_node_next(node))) &&
+            !parent_scope_will_sync(&nif->cf_node, parent_scope);
+
+         struct scope scope = push_scope(b, SCOPE_TYPE_IF_MERGE,
+                                         parent_scope, divergent,
+                                         needs_sync, merge_block);
+
+         nir_goto_if(b, then_block, cond, else_block);
+
+         push_block(b, then_block, scope.divergent);
+         lower_cf_list(b, esc_reg, &scope, &nif->then_list);
+         normal_exit(b, esc_reg, merge_block);
+
+         push_block(b, else_block, scope.divergent);
+         lower_cf_list(b, esc_reg, &scope, &nif->else_list);
+         normal_exit(b, esc_reg, merge_block);
+
+         push_block(b, merge_block, parent_scope->divergent);
+         pop_scope(b, esc_reg, scope);
+
+         break;
+      }
+
+      case nir_cf_node_loop: {
+         nir_loop *loop = nir_cf_node_as_loop(node);
+
+         nir_block *head_block = nir_block_create(b->shader);
+         nir_block *break_block = nir_block_create(b->shader);
+         nir_block *cont_block = nir_block_create(b->shader);
+
+         /* TODO: We can potentially avoid the break sync for loops when the
+          * parent scope syncs for us.  However, we still need to handle the
+          * continue clause cascading to the break.  If there is a
+          * nir_jump_halt involved, then we have a real cascade where it needs
+          * to then jump to the next scope.  Getting all these cases right
+          * while avoiding an extra sync for the loop break is tricky at best.
+          */
+         struct scope break_scope = push_scope(b, SCOPE_TYPE_LOOP_BREAK,
+                                               parent_scope,
+                                               nir_loop_is_divergent(loop),
+                                               nir_loop_is_divergent(loop),
+                                               break_block);
+
+         nir_goto(b, head_block);
+         push_block(b, head_block, break_scope.divergent);
+
+         struct scope cont_scope = push_scope(b, SCOPE_TYPE_LOOP_CONT,
+                                              &break_scope,
+                                              nir_loop_is_divergent(loop),
+                                              nir_loop_is_divergent(loop),
+                                              cont_block);
+
+         lower_cf_list(b, esc_reg, &cont_scope, &loop->body);
+         normal_exit(b, esc_reg, cont_block);
+
+         push_block(b, cont_block, break_scope.divergent);
+
+         pop_scope(b, esc_reg, cont_scope);
+
+         lower_cf_list(b, esc_reg, &break_scope, &loop->continue_list);
+
+         nir_goto(b, head_block);
+         push_block(b, break_block, parent_scope->divergent);
+
+         pop_scope(b, esc_reg, break_scope);
+
+         break;
+      }
+
+      default:
+         UNREACHABLE("Unknown CF node type");
+      }
+   }
+}
+
+static void
+recompute_phi_divergence_impl(nir_function_impl *impl)
+{
+   bool progress;
+   do {
+      progress = false;
+      nir_foreach_block_unstructured(block, impl) {
+         nir_foreach_instr(instr, block) {
+            if (instr->type != nir_instr_type_phi)
+               break;
+
+            nir_phi_instr *phi = nir_instr_as_phi(instr);
+
+            bool divergent = false;
+            nir_foreach_phi_src(phi_src, phi) {
+               /* There is a tricky case we need to care about here where a
+                * convergent block has a divergent dominator.  This can happen
+                * if, for instance, you have the following loop:
+                *
+                *    loop {
+                *       if (div) {
+                *          %20 = load_ubo(0, 0);
+                *       } else {
+                *          terminate;
+                *       }
+                *    }
+                *    use(%20);
+                *
+                * In this case, the load_ubo() dominates the use() even though
+                * the load_ubo() exists in divergent control-flow.  In this
+                * case, we simply flag the whole phi divergent because we
+                * don't want to deal with inserting a r2ur somewhere.
+                */
+               if (phi_src->pred->divergent || phi_src->src.ssa->divergent ||
+                   nir_def_block(phi_src->src.ssa)->divergent) {
+                  divergent = true;
+                  break;
+               }
+            }
+
+            if (divergent != phi->def.divergent) {
+               phi->def.divergent = divergent;
+               progress = true;
+            }
+         }
+      }
+   } while(progress);
+
+   impl->valid_metadata |= nir_metadata_divergence;
+}
+
+static bool
+lower_cf_func(nir_function *func)
+{
+   if (func->impl == NULL)
+      return false;
+
+   if (exec_list_is_singular(&func->impl->body)) {
+      return nir_no_progress(func->impl);
+   }
+
+   nir_function_impl *old_impl = func->impl;
+
+   /* We use this in block_is_merge() */
+   nir_metadata_require(old_impl, nir_metadata_dominance | nir_metadata_divergence);
+
+   /* We get rid of single source phis, because they would be converted to phis
+    * with undef after lowering regs to SSA.
+    */
+   nir_foreach_block(block, old_impl) {
+      if (nir_block_num_preds(block) <= 1)
+         nir_remove_single_src_phis_block(block);
+   }
+
+   /* First, we temporarily get rid of SSA.  This will make all our block
+    * motion way easier.  Ask the pass to place reg writes directly in the
+    * immediate predecessors of the phis instead of trying to be clever.
+    * This will ensure that we never get a write to a uniform register from
+    * non-uniform control flow and makes our divergence reconstruction for
+    * phis more reliable.
+    */
+   nir_foreach_block(block, old_impl)
+      nir_lower_phis_to_regs_block(block, true);
+
+   /* We create a whole new nir_function_impl and copy the contents over */
+   func->impl = NULL;
+   nir_function_impl *new_impl = nir_function_impl_create(func);
+   new_impl->structured = false;
+
+   /* We copy defs from the old impl */
+   new_impl->ssa_alloc = old_impl->ssa_alloc;
+
+   nir_builder b = nir_builder_at(nir_before_impl(new_impl));
+   nir_def *esc_reg = nir_decl_reg(&b, 1, 32, 0);
+
+   /* Having a function scope makes everything easier */
+   struct scope scope = {
+      .type = SCOPE_TYPE_SHADER,
+      .merge = new_impl->end_block,
+   };
+   lower_cf_list(&b, esc_reg, &scope, &old_impl->body);
+   normal_exit(&b, esc_reg, new_impl->end_block);
+
+   /* Now sort by reverse PDFS and restore SSA
+    *
+    * Note: Since we created a new nir_function_impl, there is no metadata,
+    * dirty or otherwise, so we have no need to call nir_progress().
+    */
+   nir_sort_unstructured_blocks(new_impl);
+   nir_repair_ssa_impl(new_impl);
+   nir_lower_reg_intrinsics_to_ssa_impl(new_impl);
+   recompute_phi_divergence_impl(new_impl);
+
+   return true;
+}
+
+bool
+nak_nir_lower_cf(nir_shader *nir)
+{
+   bool progress = false;
+
+   nir_foreach_function(func, nir) {
+      if (lower_cf_func(func))
+         progress = true;
+   }
+
+   return progress;
+}
+
+bool
+nak_block_is_divergent(const nir_block *block)
+{
+   const nir_cf_node* node = &block->cf_node;
+
+   while (node) {
+      switch (node->type) {
+         case nir_cf_node_if: {
+            nir_if *nif = nir_cf_node_as_if(node);
+            if (nir_src_is_divergent(&nif->condition))
+               return true;
+            break;
+         }
+         case nir_cf_node_loop: {
+            nir_loop *loop = nir_cf_node_as_loop(node);
+            if (nir_loop_is_divergent(loop))
+               return true;
+            break;
+         }
+         case nir_cf_node_block: {
+            nir_block *block = nir_cf_node_as_block(node);
+            if (block->divergent)
+               return true;
+            break;
+         }
+         case nir_cf_node_function:
+            break;
+      }
+      node = node->parent;
+   }
+   return false;
+}
@@ -0,0 +1,412 @@
+/*
+ * Copyright © 2023 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "nak_private.h"
+#include "nir_builder.h"
+
+/** Load a flat FS input */
+static nir_def *
+load_fs_input(nir_builder *b, unsigned num_components, uint32_t addr,
+              UNUSED const struct nak_compiler *nak)
+{
+   const struct nak_nir_ipa_flags flags = {
+      .interp_mode = NAK_INTERP_MODE_CONSTANT,
+      .interp_freq = NAK_INTERP_FREQ_CONSTANT,
+      .interp_loc = NAK_INTERP_LOC_DEFAULT,
+   };
+
+   nir_def *comps[NIR_MAX_VEC_COMPONENTS];
+   for (unsigned c = 0; c < num_components; c++) {
+      comps[c] = nir_ipa_nv(b, nir_imm_float(b, 0), nir_imm_int(b, 0),
+                            .base = addr + c * 4, .flags = NAK_AS_U32(flags));
+   }
+   return nir_vec(b, comps, num_components);
+}
+
+static nir_def *
+load_frag_w(nir_builder *b, enum nak_interp_loc interp_loc, nir_def *offset,
+            const struct nak_compiler *nak)
+{
+   if (offset == NULL)
+      offset = nir_imm_int(b, 0);
+
+   const uint16_t w_addr =
+      nak_sysval_attr_addr(nak, SYSTEM_VALUE_FRAG_COORD) + 12;
+
+   const struct nak_nir_ipa_flags flags = {
+      .interp_mode = NAK_INTERP_MODE_SCREEN_LINEAR,
+      .interp_freq = NAK_INTERP_FREQ_PASS,
+      .interp_loc = interp_loc,
+   };
+
+   return nir_ipa_nv(b, nir_imm_float(b, 0), offset,
+                     .base = w_addr, .flags = NAK_AS_U32(flags));
+}
+
+static nir_def *
+interp_fs_input(nir_builder *b, unsigned num_components, uint32_t addr,
+                enum nak_interp_mode interp_mode,
+                enum nak_interp_loc interp_loc,
+                nir_def *inv_w, nir_def *offset,
+                const struct nak_compiler *nak)
+{
+   if (offset == NULL)
+      offset = nir_imm_int(b, 0);
+
+   if (nak->sm >= 70) {
+      const struct nak_nir_ipa_flags flags = {
+         .interp_mode = interp_mode,
+         .interp_freq = NAK_INTERP_FREQ_PASS,
+         .interp_loc = interp_loc,
+      };
+
+      nir_def *comps[NIR_MAX_VEC_COMPONENTS];
+      for (unsigned c = 0; c < num_components; c++) {
+         comps[c] = nir_ipa_nv(b, nir_imm_float(b, 0), offset,
+                               .base = addr + c * 4,
+                               .flags = NAK_AS_U32(flags));
+         if (interp_mode == NAK_INTERP_MODE_PERSPECTIVE) {
+            unsigned fp_math_ctrl = b->fp_math_ctrl;
+            b->fp_math_ctrl |= nir_fp_exact;
+            /* It seems critical that this is done as round to zero.
+             * The Surge 2 and Shadow of the Tomb Raider show artifacts if not.
+             */
+            comps[c] = nir_fmul_rtz(b, comps[c], inv_w);
+            b->fp_math_ctrl = fp_math_ctrl;
+         }
+      }
+      return nir_vec(b, comps, num_components);
+   } else if (nak->sm >= 20) {
+      struct nak_nir_ipa_flags flags = {
+         .interp_mode = interp_mode,
+         .interp_freq = NAK_INTERP_FREQ_PASS,
+         .interp_loc = interp_loc,
+      };
+
+      if (interp_mode == NAK_INTERP_MODE_PERSPECTIVE)
+         flags.interp_freq = NAK_INTERP_FREQ_PASS_MUL_W;
+      else
+         inv_w = nir_imm_float(b, 0);
+
+      nir_def *comps[NIR_MAX_VEC_COMPONENTS];
+      for (unsigned c = 0; c < num_components; c++) {
+         comps[c] = nir_ipa_nv(b, inv_w, offset,
+                               .base = addr + c * 4,
+                               .flags = NAK_AS_U32(flags));
+      }
+      return nir_vec(b, comps, num_components);
+   } else {
+      UNREACHABLE("Unsupported shader model");
+   }
+}
+
+static nir_def *
+load_sample_pos_u4_at(nir_builder *b, nir_def *sample_id,
+                      const struct nak_compiler* nak)
+{
+   const struct nak_constant_offset_info *info = nak_const_offsets(nak, true);
+   nir_def *loc = nir_ldc_nv(b, 1, 8,
+                             nir_imm_int(b, info->sample_info_cb),
+                             nir_iadd_imm(b, sample_id,
+                                          info->sample_locations_offset),
+                             .align_mul = 1, .align_offset = 0);
+
+   /* The rest of these calculations are in 32-bit */
+   loc = nir_u2u32(b, loc);
+   nir_def *loc_x_u4 = nir_iand_imm(b, loc, 0xf);
+   nir_def *loc_y_u4 = nir_iand_imm(b, nir_ushr_imm(b, loc, 4), 0xf);
+   return nir_vec2(b, loc_x_u4, loc_y_u4);
+}
+
+static nir_def *
+load_pass_sample_mask_at(nir_builder *b, nir_def *sample_id,
+                         const struct nak_compiler* nak)
+{
+   const struct nak_constant_offset_info *info = nak_const_offsets(nak, true);
+   nir_def *offset =
+      nir_imul_imm(b, sample_id, sizeof(struct nak_sample_mask));
+   offset = nir_iadd_imm(b, offset, info->sample_masks_offset);
+
+   return nir_ldc_nv(b, 1, 8 * sizeof(struct nak_sample_mask),
+                     nir_imm_int(b, info->sample_info_cb),
+                     offset,
+                     .align_mul = sizeof(struct nak_sample_mask),
+                     .align_offset = 0);
+}
+
+static nir_def *
+load_sample_pos_at(nir_builder *b, nir_def *sample_id,
+                   const struct nak_compiler* nak)
+{
+   nir_def *loc_u4 = load_sample_pos_u4_at(b, sample_id, nak);
+   nir_def *result = nir_fmul_imm(b, nir_i2f32(b, loc_u4), 1.0 / 16.0);
+
+   return result;
+}
+
+static nir_def *
+load_barycentric_offset(nir_builder *b, nir_intrinsic_instr *bary,
+                        const struct nak_compiler* nak)
+{
+   nir_def *offset_s12;
+
+   if (bary->intrinsic == nir_intrinsic_load_barycentric_coord_at_sample ||
+       bary->intrinsic == nir_intrinsic_load_barycentric_at_sample) {
+      nir_def *sample_id = bary->src[0].ssa;
+      nir_def *offset_u4 = load_sample_pos_u4_at(b, sample_id, nak);
+      /* The sample position we loaded is a u4 from the upper-left and the
+       * sample position wanted by ipa.offset is s12
+       */
+      offset_s12 = nir_iadd_imm(b, nir_ishl_imm(b, offset_u4, 8), -2048);
+   } else {
+      nir_def *offset_f = bary->src[0].ssa;
+
+      offset_f = nir_fclamp(b, offset_f, nir_imm_float(b, -0.5),
+                            nir_imm_float(b, 0.437500));
+      offset_s12 = nir_f2i32(b, nir_fmul_imm(b, offset_f, 4096.0));
+   }
+
+   return nir_prmt_nv(b, nir_imm_int(b, 0x5410),
+                         nir_channel(b, offset_s12, 0),
+                         nir_channel(b, offset_s12, 1));
+}
+
+struct lower_fs_input_ctx {
+   const struct nak_compiler *nak;
+   const struct nak_fs_key *fs_key;
+};
+
+static uint16_t
+fs_input_intrin_addr(nir_intrinsic_instr *intrin,
+                     const struct nak_compiler *nak)
+{
+   const nir_io_semantics sem = nir_intrinsic_io_semantics(intrin);
+   return nak_varying_attr_addr(nak, sem.location) +
+          nir_src_as_uint(*nir_get_io_offset_src(intrin)) * 16 +
+          nir_intrinsic_component(intrin) * 4;
+}
+
+static bool
+lower_fs_input_intrin(nir_builder *b, nir_intrinsic_instr *intrin, void *data)
+{
+   const struct lower_fs_input_ctx *ctx = data;
+   const struct nak_compiler *nak = ctx->nak;
+
+   b->cursor = nir_before_instr(&intrin->instr);
+
+   nir_def *res;
+   switch (intrin->intrinsic) {
+   case nir_intrinsic_load_barycentric_pixel: {
+      if (!(ctx->fs_key && ctx->fs_key->force_sample_shading))
+         return false;
+
+      intrin->intrinsic = nir_intrinsic_load_barycentric_sample;
+      return true;
+   }
+
+   case nir_intrinsic_load_frag_coord:
+   case nir_intrinsic_load_point_coord: {
+      const enum nak_interp_loc interp_loc =
+         b->shader->info.fs.uses_sample_shading ? NAK_INTERP_LOC_CENTROID
+                                                : NAK_INTERP_LOC_DEFAULT;
+      const uint32_t addr =
+         intrin->intrinsic == nir_intrinsic_load_point_coord ?
+         nak_sysval_attr_addr(nak, SYSTEM_VALUE_POINT_COORD) :
+         nak_sysval_attr_addr(nak, SYSTEM_VALUE_FRAG_COORD);
+
+      res = interp_fs_input(b, intrin->def.num_components, addr,
+                            NAK_INTERP_MODE_SCREEN_LINEAR,
+                            interp_loc, NULL, NULL,
+                            ctx->nak);
+      break;
+   }
+
+   case nir_intrinsic_load_front_face:
+   case nir_intrinsic_load_layer_id:
+   case nir_intrinsic_load_primitive_id: {
+      assert(b->shader->info.stage == MESA_SHADER_FRAGMENT);
+      const gl_system_value sysval =
+         nir_system_value_from_intrinsic(intrin->intrinsic);
+      const uint32_t addr = nak_sysval_attr_addr(nak, sysval);
+
+      res = load_fs_input(b, intrin->def.num_components, addr, ctx->nak);
+      if (intrin->def.bit_size == 1)
+         res = nir_i2b(b, res);
+      break;
+   }
+
+   case nir_intrinsic_load_input:
+   case nir_intrinsic_load_per_primitive_input: {
+      const uint16_t addr = fs_input_intrin_addr(intrin, ctx->nak);
+      res = load_fs_input(b, intrin->def.num_components, addr, ctx->nak);
+      break;
+   }
+
+   case nir_intrinsic_load_barycentric_coord_pixel:
+   case nir_intrinsic_load_barycentric_coord_centroid:
+   case nir_intrinsic_load_barycentric_coord_sample:
+   case nir_intrinsic_load_barycentric_coord_at_sample:
+   case nir_intrinsic_load_barycentric_coord_at_offset: {
+      uint32_t addr;
+      enum nak_interp_mode interp_mode;
+      if (nir_intrinsic_interp_mode(intrin) == INTERP_MODE_NOPERSPECTIVE) {
+         addr = NAK_ATTR_BARY_COORD_NO_PERSP;
+         interp_mode = NAK_INTERP_MODE_SCREEN_LINEAR;
+      } else {
+         addr = NAK_ATTR_BARY_COORD;
+         interp_mode = NAK_INTERP_MODE_PERSPECTIVE;
+      }
+
+      nir_def *offset = NULL;
+      enum nak_interp_loc interp_loc;
+      switch (intrin->intrinsic) {
+      case nir_intrinsic_load_barycentric_coord_at_sample:
+      case nir_intrinsic_load_barycentric_coord_at_offset:
+         interp_loc = NAK_INTERP_LOC_OFFSET;
+         offset = load_barycentric_offset(b, intrin, nak);
+         break;
+      case nir_intrinsic_load_barycentric_coord_centroid:
+      case nir_intrinsic_load_barycentric_coord_sample:
+         interp_loc = NAK_INTERP_LOC_CENTROID;
+         break;
+      case nir_intrinsic_load_barycentric_coord_pixel:
+         interp_loc = NAK_INTERP_LOC_DEFAULT;
+         break;
+      default:
+         UNREACHABLE("Unknown intrinsic");
+      }
+
+      nir_def *inv_w = NULL;
+      if (interp_mode == NAK_INTERP_MODE_PERSPECTIVE)
+         inv_w = nir_frcp(b, load_frag_w(b, interp_loc, offset, nak));
+
+      res = interp_fs_input(b, intrin->def.num_components,
+                            addr, interp_mode, interp_loc,
+                            inv_w, offset, ctx->nak);
+      break;
+   }
+
+   case nir_intrinsic_load_interpolated_input: {
+      const uint16_t addr = fs_input_intrin_addr(intrin, ctx->nak);
+      nir_intrinsic_instr *bary = nir_src_as_intrinsic(intrin->src[0]);
+
+      enum nak_interp_mode interp_mode;
+      if (nir_intrinsic_interp_mode(bary) == INTERP_MODE_SMOOTH ||
+          nir_intrinsic_interp_mode(bary) == INTERP_MODE_NONE)
+         interp_mode = NAK_INTERP_MODE_PERSPECTIVE;
+      else
+         interp_mode = NAK_INTERP_MODE_SCREEN_LINEAR;
+
+      nir_def *offset = NULL;
+      enum nak_interp_loc interp_loc;
+      switch (bary->intrinsic) {
+      case nir_intrinsic_load_barycentric_at_offset:
+      case nir_intrinsic_load_barycentric_at_sample: {
+         interp_loc = NAK_INTERP_LOC_OFFSET;
+         offset = load_barycentric_offset(b, bary, nak);
+         break;
+      }
+
+      case nir_intrinsic_load_barycentric_centroid:
+      case nir_intrinsic_load_barycentric_sample:
+         interp_loc = NAK_INTERP_LOC_CENTROID;
+         break;
+
+      case nir_intrinsic_load_barycentric_pixel:
+         interp_loc = NAK_INTERP_LOC_DEFAULT;
+         break;
+
+      default:
+         UNREACHABLE("Unsupported barycentric");
+      }
+
+      nir_def *inv_w = NULL;
+      if (interp_mode == NAK_INTERP_MODE_PERSPECTIVE)
+         inv_w = nir_frcp(b, load_frag_w(b, interp_loc, offset, nak));
+
+      res = interp_fs_input(b, intrin->def.num_components,
+                            addr, interp_mode, interp_loc,
+                            inv_w, offset, ctx->nak);
+      break;
+   }
+
+   case nir_intrinsic_load_sample_mask_in:
+      b->cursor = nir_after_instr(&intrin->instr);
+
+      /* pixld.covmask returns the coverage mask for the entire pixel being
+       * shaded, not the set of samples covered by the current FS invocation.
+       * We need to mask off excess samples in order to get the GL/Vulkan
+       * behavior.
+       */
+      if (b->shader->info.fs.uses_sample_shading) {
+         /* Mask off just the current sample */
+         nir_def *sample = nir_load_sample_id(b);
+         nir_def *mask = nir_ishl(b, nir_imm_int(b, 1), sample);
+         mask = nir_iand(b, &intrin->def, mask);
+         nir_def_rewrite_uses_after(&intrin->def, mask);
+
+         return true;
+      } else if (ctx->fs_key && ctx->fs_key->force_sample_shading) {
+         /* In this case we don't know up-front how many passes will be run so
+          * we need to take the per-pass sample mask from the driver and AND
+          * that with the coverage mask.
+          */
+         nir_def *sample = nir_load_sample_id(b);
+         nir_def *mask = load_pass_sample_mask_at(b, sample, nak);
+         mask = nir_iand(b, &intrin->def, nir_u2u32(b, mask));
+         nir_def_rewrite_uses_after(&intrin->def, mask);
+
+         return true;
+      } else {
+         /* We're always executing single-pass so just use the sample mask as
+          * given by the hardware.
+          */
+         return false;
+      }
+      break;
+
+   case nir_intrinsic_load_sample_pos:
+      res = load_sample_pos_at(b, nir_load_sample_id(b), nak);
+      break;
+
+   case nir_intrinsic_load_input_vertex: {
+      const uint16_t addr = fs_input_intrin_addr(intrin, ctx->nak);
+      unsigned vertex_id = nir_src_as_uint(intrin->src[0]);
+      assert(vertex_id < 3);
+
+      nir_def *comps[NIR_MAX_VEC_COMPONENTS];
+      for (unsigned c = 0; c < intrin->def.num_components; c++) {
+         nir_def *data = nir_ldtram_nv(b, .base = addr + c * 4,
+                                       .flags = vertex_id == 2);
+         comps[c] = nir_channel(b, data, vertex_id & 1);
+      }
+      res = nir_vec(b, comps, intrin->num_components);
+      break;
+   }
+
+   default:
+      return false;
+   }
+
+   nir_def_replace(&intrin->def, res);
+
+   return true;
+}
+
+bool
+nak_nir_lower_fs_inputs(nir_shader *nir,
+                        const struct nak_compiler *nak,
+                        const struct nak_fs_key *fs_key)
+{
+   const struct lower_fs_input_ctx fs_in_ctx = {
+      .nak = nak,
+      .fs_key = fs_key,
+   };
+   NIR_PASS(_, nir, nir_shader_intrinsics_pass, lower_fs_input_intrin,
+              nir_metadata_control_flow,
+              (void *)&fs_in_ctx);
+
+   return true;
+}
@@ -0,0 +1,124 @@
+/*
+ * Copyright © 2023 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "nak_private.h"
+#include "nir_builder.h"
+#include "nir_format_convert.h"
+
+
+struct state {
+   nir_builder *builder;
+   nir_variable *handle_var;
+   bool progress;
+};
+
+static void
+rewrite_emit_vertex(nir_intrinsic_instr *intr, struct state *state)
+{
+   nir_builder *b = state->builder;
+   const unsigned stream = nir_intrinsic_stream_id(intr);
+
+   b->cursor = nir_before_instr(&intr->instr);
+
+   nir_def *gs_handle = nir_load_var(b, state->handle_var);
+   nir_def *gs_handle_out = nir_emit_vertex_nv(b, 32, gs_handle, stream);
+
+   nir_store_var(b, state->handle_var, gs_handle_out, 0x1);
+
+   nir_instr_remove(&intr->instr);
+
+   state->progress = true;
+}
+
+static void
+rewrite_end_primitive(nir_intrinsic_instr *intr, struct state *state)
+{
+   nir_builder *b = state->builder;
+   const unsigned stream = nir_intrinsic_stream_id(intr);
+
+   b->cursor = nir_before_instr(&intr->instr);
+
+   nir_def *gs_handle = nir_load_var(b, state->handle_var);
+   nir_def *gs_handle_out = nir_end_primitive_nv(b, 32, gs_handle, stream);
+
+   nir_store_var(b, state->handle_var, gs_handle_out, 0x1);
+
+   nir_instr_remove(&intr->instr);
+
+   state->progress = true;
+}
+
+static void
+rewrite_ast_nv(nir_intrinsic_instr *intr, struct state *state)
+{
+   nir_builder *b = state->builder;
+
+   b->cursor = nir_before_instr(&intr->instr);
+
+   nir_src *vtx = &intr->src[1];
+   nir_def *gs_handle = nir_load_var(b, state->handle_var);
+   nir_src_rewrite(vtx, gs_handle);
+
+   state->progress = true;
+}
+
+static void
+append_final_primitive_nv(nir_block *end_block, struct state *state)
+{
+   nir_builder *b = state->builder;
+
+   nir_foreach_pred(pred, end_block) {
+      b->cursor = nir_after_block_before_jump(pred);
+
+      nir_def *gs_handle = nir_load_var(b, state->handle_var);
+      nir_final_primitive_nv(b, gs_handle);
+
+      state->progress = true;
+   }
+}
+
+bool
+nak_nir_lower_gs_intrinsics(nir_shader *nir)
+{
+   struct state state;
+
+   state.progress = false;
+   nir_function_impl *impl = nir_shader_get_entrypoint(nir);
+   nir_builder builder = nir_builder_at(nir_before_impl(impl));
+
+   state.builder = &builder;
+
+   state.handle_var = nir_local_variable_create(impl, glsl_uint_type(), "gs_handle");
+   nir_store_var(&builder, state.handle_var, nir_imm_int(&builder, 0), 0x1);
+
+   nir_foreach_block_safe(block, impl) {
+      nir_foreach_instr_safe(instr, block) {
+         if (instr->type != nir_instr_type_intrinsic)
+            continue;
+
+         nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
+
+         switch (intr->intrinsic) {
+         case nir_intrinsic_emit_vertex:
+            rewrite_emit_vertex(intr, &state);
+            break;
+         case nir_intrinsic_end_primitive:
+            rewrite_end_primitive(intr, &state);
+            break;
+         case nir_intrinsic_ast_nv:
+            rewrite_ast_nv(intr, &state);
+            break;
+         default:
+            break;
+         }
+      }
+   }
+
+   append_final_primitive_nv(impl->end_block, &state);
+
+   nir_progress(true, impl, nir_metadata_none);
+
+   return state.progress;
+}
@@ -0,0 +1,621 @@
+/*
+ * Copyright © 2023 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "nak.h"
+#include "nak_private.h"
+#include "nil.h"
+
+#include "nir_format_convert.h"
+
+static nir_def *
+build_load_su_info(nir_builder *b, nir_deref_instr *deref, uint32_t offset)
+{
+   return nir_image_deref_load_info_nv(b, 1, &deref->def, .base = offset);
+}
+
+#define load_su_info(b, d, field) \
+   build_load_su_info((b), (d), offsetof(struct nil_su_info, field))
+
+static enum pipe_format
+format_for_bits(unsigned bits)
+{
+   switch (bits) {
+   case 8:   return PIPE_FORMAT_R8_UINT;
+   case 16:  return PIPE_FORMAT_R16_UINT;
+   case 32:  return PIPE_FORMAT_R32_UINT;
+   case 64:  return PIPE_FORMAT_R32G32_UINT;
+   case 128: return PIPE_FORMAT_R32G32B32A32_UINT;
+   default: UNREACHABLE("Unknown number of image format bits");
+   }
+}
+
+static unsigned
+sampler_dim_len(enum glsl_sampler_dim dim)
+{
+   switch (dim) {
+   case GLSL_SAMPLER_DIM_1D:
+   case GLSL_SAMPLER_DIM_BUF:
+      return 1;
+   case GLSL_SAMPLER_DIM_CUBE:
+   case GLSL_SAMPLER_DIM_2D:
+   case GLSL_SAMPLER_DIM_RECT:
+   case GLSL_SAMPLER_DIM_MS:
+      return 2;
+   case GLSL_SAMPLER_DIM_3D:
+      return 3;
+   default:
+      UNREACHABLE("Unhandled sampler dim");
+      return 1;// Never reached
+   }
+}
+
+static nir_def *
+lower_formatted_image_load(nir_builder *b,
+                           nir_intrinsic_instr *intrin,
+                           enum pipe_format format)
+{
+   if (format == PIPE_FORMAT_NONE)
+      return NULL;
+
+   unsigned bits = util_format_get_blocksizebits(format);
+
+   assert(intrin->intrinsic == nir_intrinsic_suldga_nv);
+
+   nir_intrinsic_set_format(intrin, format_for_bits(bits));
+
+   const unsigned num_raw_components = DIV_ROUND_UP(bits, 32);
+   intrin->num_components = num_raw_components;
+   intrin->def.num_components = num_raw_components;
+
+   b->cursor = nir_after_instr(&intrin->instr);
+   nir_def *rgba = NULL;
+   switch (format) {
+   case PIPE_FORMAT_R64_UINT:
+   case PIPE_FORMAT_R64_SINT:
+      rgba = nir_vec4(b, nir_pack_64_2x32(b, &intrin->def),
+                         nir_imm_int64(b, 0),
+                         nir_imm_int64(b, 0),
+                         nir_imm_int64(b, 1));
+      break;
+   default:
+      rgba = nir_format_unpack_rgba(b, &intrin->def, format);
+      break;
+   }
+
+   return rgba;
+}
+
+static nir_def *
+load_su_info_clamp(nir_builder *b, nir_deref_instr *deref,
+                   unsigned xyz)
+{
+   /* Array length is always stored in clamp_z */
+   enum glsl_sampler_dim dim = glsl_get_sampler_dim(deref->type);
+   if (dim == GLSL_SAMPLER_DIM_1D && xyz == 1)
+      return load_su_info(b, deref, clamp_z);
+
+   switch (xyz) {
+   case 0: return load_su_info(b, deref, clamp_x);
+   case 1: return load_su_info(b, deref, clamp_y);
+   case 2: return load_su_info(b, deref, clamp_z);
+   default: UNREACHABLE("Invalid image dimension");
+   }
+}
+
+static nir_def *
+clamp_coord(nir_builder *b, nir_deref_instr *deref,
+            nir_def* coord, int xyz)
+{
+   nir_def *clamp = load_su_info_clamp(b, deref, xyz);
+
+   const struct nak_nir_suclamp_flags flags = {
+      .mode = NAK_SUCLAMP_MODE_STORED_DESCRIPTOR,
+      .round = NAK_SUCLAMP_ROUND_R1,
+      .is_s32 = false,
+      .is_2d = true,
+   };
+
+   nir_def *dst = nir_suclamp_nv(b, coord, clamp,
+                                 .flags = NAK_AS_U32(flags));
+   return nir_channel(b, dst, 0);
+}
+
+static void
+load_sample_size(nir_builder *b, nir_deref_instr *deref,
+                 nir_def **sample_width_log2,
+                 nir_def **sample_height_log2)
+{
+   /* MS width and height are stored in the lower 8 bits of pitch */
+   nir_def *pitch = load_su_info(b, deref, pitch);
+   *sample_width_log2 = nir_ubitfield_extract_imm(b, pitch, 24, 4);
+   *sample_height_log2 = nir_ubitfield_extract_imm(b, pitch, 28, 4);
+}
+
+// Kepler only supports suldga, sustga, so we need to compute
+// the raw address manually, this is done through a weird dance
+// and custom ops.
+// - each coordinate is clamped through suclamp
+// - compute block offset using gob coordinates (y * pitch + x)
+// - merge bitfields (output of suclamps)
+//   this will both merge the block coordinates, combine
+//   pixel coordinates (in GoB-space) and OR together the OOB
+//   predicate
+// - compute the effective upper address by combining block_offset,
+//   bitfield and base_address
+// - combine effective upper address and the combined bitfield that
+//   contains the lower 8 bits of global address
+// - pass the combined values into suldga/sustga
+//
+// Linear Layout support:
+// Shader cannot know at compile time if an image is in linear format
+// or in block format, thus it needs to support both modes using the
+// same opcodes, the only difference should be in descriptors.
+// The address for linear layout is computed with:
+// eff_addr = base_addr + (y * pitch + x)*el_size_B
+// Here we first compute off = (y * pitch + x) with imadsp
+// then we split what goes in the first 8 bits using subfm
+// (in 0..8 it loads off << el_size_B.log2, in 16..32 it
+// loads off >> el_size_B.log2)
+// For this we need to emit eau = sueau bf.x, bf.y, off
+// this would totally screw up block-linear calcs, but we can use
+// is_3d=false to skip the third argument only in block-linear.
+static void
+compute_image_address(nir_builder *b,
+                      nir_intrinsic_instr *intrin,
+                      enum glsl_sampler_dim sampler_dim,
+                      nir_def **addr,
+                      nir_def **is_oob)
+{
+   nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
+
+   const unsigned dim = sampler_dim_len(sampler_dim);
+   const bool is_cube = sampler_dim == GLSL_SAMPLER_DIM_CUBE;
+
+   // A CubeMap is a 6-array of 2D images, a CubeMap array is just
+   // multiple CubeMap concatenated together, so the cube access is alway
+   // treated as an array
+   const bool is_array = nir_intrinsic_image_array(intrin) || is_cube;
+
+   // Prevent read/write for null-descriptors
+   nir_def *base_addr = load_su_info(b, deref, addr_shifted8);
+   *is_oob = nir_ieq_imm(b, base_addr, 0);
+
+   nir_def *coords[3];
+   for (int i = 0; i < 3; i++)
+      coords[i] = nir_channel(b, intrin->src[1].ssa, i);
+
+   // Lower multi-sample coords
+   if (sampler_dim == GLSL_SAMPLER_DIM_MS) {
+      nir_def *s = intrin->src[2].ssa;
+      nir_def *sw_log2, *sh_log2;
+      load_sample_size(b, deref, &sw_log2, &sh_log2);
+      nir_def *s_map = load_su_info(b, deref, extra);// multi-sample table
+      nir_def *num_samples = nir_ishl(b, nir_imm_int(b, 1),
+                                      nir_iadd(b, sw_log2, sh_log2));
+
+      nir_def *s_xy = nir_ushr(b, s_map, nir_imul_imm(b, s, 4));
+      nir_def *sx = nir_ubitfield_extract_imm(b, s_xy, 0, 2);
+      nir_def *sy = nir_ubitfield_extract_imm(b, s_xy, 2, 2);
+
+      nir_def *sw = nir_ishl(b, nir_imm_int(b, 1), sw_log2);
+      nir_def *sh = nir_ishl(b, nir_imm_int(b, 1), sh_log2);
+
+      nir_def *x = nir_imad(b, coords[0], sw, sx);
+      nir_def *y = nir_imad(b, coords[1], sh, sy);
+
+      // Check if OOB
+      *is_oob = nir_ior(b, *is_oob, nir_uge(b, s, num_samples));
+
+      coords[0] = x;
+      coords[1] = y;
+   }
+
+   // Clamp coordinates
+   // This computes a bitfield with the following info inside:
+   // - Block coordinates
+   // - GoB coordinates
+   // - predicate for OOB access
+   // - Wether the clamp is using pitch_linear
+   // - n. of block tiles for the coordinate
+   nir_def *clamped_x = clamp_coord(b, deref, coords[0], 0);
+   nir_def *clamped_y = dim >= 2 ? clamp_coord(b, deref, coords[1], 1) : nir_imm_int(b, 0);
+   nir_def *clamped_z = dim >= 3 ? clamp_coord(b, deref, coords[2], 2) : nir_imm_int(b, 0);
+
+   // For arrays the clamp is "plain", no bitfield is computed
+   // just the OOB predicate.
+   nir_def *array_idx = NULL;
+   if (is_array) {
+      nir_def *clamp = load_su_info_clamp(b, deref, dim);
+      nir_def *coord = coords[dim];
+
+      const struct nak_nir_suclamp_flags flags = {
+         .mode = NAK_SUCLAMP_MODE_PITCH_LINEAR,
+         .round = NAK_SUCLAMP_ROUND_R1,
+         .is_s32 = false,
+         .is_2d = false,
+      };
+
+      nir_def *dst = nir_suclamp_nv(b, coord, clamp,
+                                    .flags = NAK_AS_U32(flags));
+      array_idx = nir_channel(b, dst, 0);
+      *is_oob = nir_ior(b, *is_oob, nir_ine_imm(b, nir_channel(b, dst, 1), 0));
+   }
+
+   // Compute offset
+   // for Block-Linear: GOB coordinates (offset that contributes *64)
+   // for Pitch-Linear: offset in pixels (y * pitch) + x
+   // for Buffer: offset = x
+   nir_def *off;
+   if (dim == 1) {
+      // This can be only 16 bits because it's only tile coordinates
+      // (actually it's 20 bits in pitch-linear mode, we don't support
+      // images that big)
+      off = nir_iand_imm(b, clamped_x, 0xffff);
+   } else if (dim == 2) {
+      // off = clamped.y * pitch + clamped.x
+      nir_def *pitch = load_su_info(b, deref, pitch);
+
+      const struct nak_nir_imadsp_flags flags = {
+         .src0 = NAK_IMAD_TYPE_U16_LO,
+         .src1 = NAK_IMAD_TYPE_U24,
+         .src2 = NAK_IMAD_TYPE_U16_LO,
+         .params_from_src1 = false,
+      };
+      off = nir_imadsp_nv(b, clamped_y, pitch, clamped_x,
+                          .flags = NAK_AS_U32(flags));
+   } else {
+      assert(dim == 3);
+      // off = (clamped.z * height + clamped.y) * pitch + clamped.x
+      // height is the height in blocks, we can compute this by doing a
+      // block linear clamp with the maximum value.
+      // (block shift-right is applied by suclamp)
+      nir_def *dim_y = load_su_info_clamp(b, deref, 1);
+      const struct nak_nir_suclamp_flags clamp_flags = {
+         .mode = NAK_SUCLAMP_MODE_BLOCK_LINEAR,
+         .round = NAK_SUCLAMP_ROUND_R1,
+         .is_s32 = false,
+         .is_2d = false,
+      };
+      nir_def *max_y = nir_suclamp_nv(b, nir_imm_int(b, -1), dim_y,
+                                      .flags=NAK_AS_U32(clamp_flags));
+
+      // max_y is still a bitfield, we can add 1 but we must only use the
+      // lower 16 bits of height_b.
+      nir_def *height_b = nir_iadd_imm(b, nir_channel(b, max_y, 0), 1);
+
+      const struct nak_nir_imadsp_flags flags_zy = {
+         .src0 = NAK_IMAD_TYPE_U16_LO,
+         .src1 = NAK_IMAD_TYPE_U16_LO,
+         .src2 = NAK_IMAD_TYPE_U16_LO,
+         .params_from_src1 = false,
+      };
+      nir_def *off_2d = nir_imadsp_nv(b, clamped_z, height_b, clamped_y,
+                                      .flags = NAK_AS_U32(flags_zy));
+
+      nir_def *pitch = load_su_info(b, deref, pitch);
+      const struct nak_nir_imadsp_flags flags = {
+         .src0 = NAK_IMAD_TYPE_U32,
+         .src1 = NAK_IMAD_TYPE_U24,
+         .src2 = NAK_IMAD_TYPE_U16_LO,
+         .params_from_src1 = false,
+      };
+      off = nir_imadsp_nv(b, off_2d, pitch, clamped_x,
+                          .flags = NAK_AS_U32(flags));
+   }
+
+   // Compute merged bitfield
+   nir_def *bf;
+   {
+      // bf, pred = subfm clamped.x, clamped.y, clamped.z
+      nir_def *bfz;
+      bool is_3d = dim >= 3;
+
+      if (dim == 2 && !is_array) {
+         // Special case for pitch_linear support, see comment above.
+         bfz = off;
+      } else {
+         bfz = clamped_z;
+      }
+
+      nir_def *combined = nir_subfm_nv(b, clamped_x, clamped_y, bfz,
+                                       .flags = is_3d);
+      bf = nir_channel(b, combined, 0);
+      nir_def *bfm_oob = nir_ine_imm(b, nir_channel(b, combined, 1), 0);
+      *is_oob = nir_ior(b, *is_oob, bfm_oob);
+   }
+
+   nir_def *eau = nir_sueau_nv(b, off, bf, base_addr);
+
+   // Apply array layer offset
+   if (is_array) {
+      nir_def *array_stride = load_su_info(b, deref, array_stride_shifted8);
+
+      // Note: this only works because array_idx has been plain-clamped
+      // so it's not a bitfield (and we can read more than u16)
+      const struct nak_nir_imadsp_flags flags = {
+         .src0 = NAK_IMAD_TYPE_U32,
+         .src1 = NAK_IMAD_TYPE_U24,
+         .src2 = NAK_IMAD_TYPE_U32,
+         .params_from_src1 = false,
+      };
+
+      eau = nir_imadsp_nv(b, array_idx, array_stride, eau,
+                          .flags = NAK_AS_U32(flags));
+   }
+
+   *addr = nir_vec2(b, bf, eau);
+}
+
+// Buffer address calculation is much simpler than images
+// In the end we need the global address to reach in two registers
+// The high register should have the highest 32-bit part of the address
+// The lower register should have the lowest 8-bit part
+// This computation can be summarized as:
+// res = addr + clamp(x) * num_comps
+// Unfortunately, given the weird register division, we need some weird
+// ops. We abuse some special functions from image addressing to reduce
+// the number of instructions.
+//
+// For null descriptors, we put bit 31 high in lower_addr,
+// this is then passed on bfm and sets the oob predicate high.
+// And that's how we get safe null descriptors for free.
+//
+// TODO: check if we can also use vector instructions, codegen used VSHL
+//       but it also didn't support sub-0x100 aligned buffers
+static void
+compute_buffer_address(nir_builder *b,
+                       nir_intrinsic_instr *intrin,
+                       enum glsl_sampler_dim sampler_dim,
+                       nir_def **addr,
+                       nir_def **is_oob)
+{
+   nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
+
+   nir_def *num_elems = load_su_info(b, deref, clamp_x);
+   nir_def *el_size_B = load_su_info(b, deref, pitch);
+   nir_def *lower_addr = load_su_info(b, deref, extra);
+
+   nir_def *raw_off = nir_channel(b, intrin->src[1].ssa, 0);
+
+   *is_oob = nir_uge(b, raw_off, num_elems);
+
+   nir_def *offset = nir_imad(b, raw_off, el_size_B, lower_addr);
+   nir_def *base_addr = load_su_info(b, deref, addr_shifted8);
+
+   *addr = nir_vec2(b, offset, base_addr);
+}
+
+static nir_def *
+compute_address_from_ga_offset(nir_builder *b, nir_def *addr,
+                               enum pipe_format format,
+                               enum nak_su_ga_offset_mode offset_mode)
+{
+   assert(offset_mode == NAK_SUGA_OFF_MODE_U32 ||
+          offset_mode == NAK_SUGA_OFF_MODE_U8);
+   // mode U8: addr_hi contains bits 8..40, addr_lo contains 0..8
+   // mode U32: addr_hi contains bits 8..40, addr_lo contains 0..32
+   //           and they should be added to addr_hi
+   nir_def *lo_8;
+   nir_def *hi_32 = nir_channel(b, addr, 1);
+
+   // With what should we fill the lower 8 bits?
+   if (offset_mode == NAK_SUGA_OFF_MODE_U8) {
+      lo_8 = nir_channel(b, addr, 0);
+   } else {
+      lo_8 = nir_imm_int(b, 0);
+   }
+
+   // Construct the 64-bit address (hi_32 << 8 | lo_8)
+   nir_def *low = nir_prmt_nv(b, nir_imm_int(b, 0x6540), lo_8, hi_32);
+   nir_def *high = nir_prmt_nv(b, nir_imm_int(b, 0x0007),
+                                 nir_imm_int(b, 0), hi_32);
+   nir_def *full_addr = nir_pack_64_2x32_split(b, low, high);
+
+   if (offset_mode == NAK_SUGA_OFF_MODE_U32) {
+      full_addr = nir_iadd(b, full_addr, nir_u2u64(b, nir_channel(b, addr, 0)));
+   }
+
+   return full_addr;
+}
+
+static void
+lower_image_access(nir_builder *b, nir_intrinsic_instr *intrin)
+{
+   b->cursor = nir_instr_remove(&intrin->instr);
+   nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
+
+   enum pipe_format format = nir_intrinsic_format(intrin);
+
+   if (format == PIPE_FORMAT_NONE)
+      format = nir_deref_instr_get_variable(deref)->data.image.format;
+
+   const unsigned int num_dst_components = intrin->def.num_components;
+   const enum glsl_sampler_dim sampler_dim = nir_intrinsic_image_dim(intrin);
+   enum nak_su_ga_offset_mode offset_mode;
+
+   nir_def *addr, *is_oob;
+   if (sampler_dim != GLSL_SAMPLER_DIM_BUF) {
+      compute_image_address(b, intrin, sampler_dim, &addr, &is_oob);
+      offset_mode = NAK_SUGA_OFF_MODE_U8;
+   } else {
+      compute_buffer_address(b, intrin, sampler_dim, &addr, &is_oob);
+      offset_mode = NAK_SUGA_OFF_MODE_U32;
+   }
+
+   switch (intrin->intrinsic) {
+   case nir_intrinsic_image_deref_load: {
+      // .format intrinsic index is only used to pass how many bits to store
+      nir_def *fmt = load_su_info(b, deref, format_info);
+      nir_def *new_ssa = nir_suldga_nv(b, num_dst_components, addr, fmt,
+                                       is_oob,
+                                       .format = format,
+                                       .access = nir_intrinsic_access(intrin),
+                                       .flags = offset_mode);
+
+      nir_intrinsic_instr *parent = nir_def_as_intrinsic(new_ssa);
+      new_ssa = lower_formatted_image_load(b, parent, format);
+      nir_def_rewrite_uses(&intrin->def, new_ssa);
+
+      break;
+   }
+   case nir_intrinsic_image_deref_store: {
+      nir_def *fmt = load_su_info(b, deref, format_info);
+      nir_sustga_nv(b, addr, fmt, is_oob, intrin->src[3].ssa,
+                    .access = nir_intrinsic_access(intrin),
+                    .flags = offset_mode);
+      break;
+   }
+   case nir_intrinsic_image_deref_atomic:
+   case nir_intrinsic_image_deref_atomic_swap: {
+      nir_atomic_op atomic_op = nir_intrinsic_atomic_op(intrin);
+
+      // suldga and sustga expect address as [low_8, high_32]
+      // while global_atomic expects a 64-bit address
+      nir_def *full_addr = compute_address_from_ga_offset(b, addr,
+                                                          format,
+                                                          offset_mode);
+
+      const unsigned bit_size = (format == PIPE_FORMAT_R64_UINT ||
+                                 format == PIPE_FORMAT_R64_SINT) ? 64 : 32;
+
+      nir_def *res, *res_ib, *res_oob;
+      nir_push_if(b, nir_inot(b, is_oob));
+      if (intrin->intrinsic == nir_intrinsic_image_deref_atomic) {
+         res_ib = nir_global_atomic(b, bit_size, full_addr,
+                                    intrin->src[3].ssa,
+                                    .atomic_op = atomic_op);
+      } else {
+         res_ib = nir_global_atomic_swap(b, bit_size, full_addr,
+                                         intrin->src[3].ssa,
+                                         intrin->src[4].ssa,
+                                         .atomic_op = atomic_op);
+      }
+      nir_push_else(b, NULL); {
+         res_oob = nir_imm_intN_t(b, 0, bit_size);
+      }
+      nir_pop_if(b, NULL);
+      res = nir_if_phi(b, res_ib, res_oob);
+
+      nir_def_rewrite_uses(&intrin->def, res);
+      break;
+   }
+   default:
+      UNREACHABLE("Unknown image intrinsic");
+   }
+}
+
+static void
+lower_image_size(nir_builder *b, nir_intrinsic_instr *intrin)
+{
+   b->cursor = nir_instr_remove(&intrin->instr);
+   nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
+
+   const unsigned dim = sampler_dim_len(nir_intrinsic_image_dim(intrin));
+
+   const bool is_array = nir_intrinsic_image_array(intrin);
+   const unsigned cdim = dim + is_array;
+   assert(cdim <= 3);
+   nir_def *comps[3];
+
+   nir_def *is_null = nir_ieq_imm(b, load_su_info(b, deref, addr_shifted8),
+                                    0);
+   nir_def *one_if_present = nir_bcsel(b, is_null, nir_imm_int(b, 0),
+                                                   nir_imm_int(b, 1));
+
+   // In descriptors we don't really have the size, but the clamp
+   // since it's inclusive, it is size - 1.
+   // Also, clamp is a bitfield, we need to extract the lower 16 bits.
+   // We can do both operations (extraction and addition) with an imadsp.
+   // To handle null descriptors, the addition accumulator of the imadsp
+   // is 1 only for non-null descriptors.
+   for (int i = 0; i < cdim; i++) {
+      nir_def *clamp = load_su_info_clamp(b, deref, i);
+      const struct nak_nir_imadsp_flags flags = {
+         .src0 = NAK_IMAD_TYPE_U16_LO,
+         .src1 = NAK_IMAD_TYPE_U24,
+         .src2 = NAK_IMAD_TYPE_U16_LO,
+         .params_from_src1 = false,
+      };
+      comps[i] = nir_imadsp_nv(b, clamp, one_if_present,
+                               one_if_present,
+                               .flags = NAK_AS_U32(flags));
+   }
+   // Clamp has multi-sampling already lowered, we need to de-lower it.
+   if (nir_intrinsic_image_dim(intrin) == GLSL_SAMPLER_DIM_MS) {
+      nir_def *ms_w_log2, *ms_h_log2;
+      load_sample_size(b, deref, &ms_w_log2, &ms_h_log2);
+
+      comps[0] = nir_ishr(b, comps[0], ms_w_log2);
+      comps[1] = nir_ishr(b, comps[1], ms_h_log2);
+   }
+
+   nir_def_rewrite_uses(&intrin->def, nir_vec(b, comps, cdim));
+}
+
+static void
+lower_buffer_size(nir_builder *b, nir_intrinsic_instr *intrin)
+{
+   b->cursor = nir_instr_remove(&intrin->instr);
+   nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
+   nir_def *num_elems = load_su_info(b, deref, clamp_x);
+
+   nir_def_rewrite_uses(&intrin->def, num_elems);
+}
+
+static void
+lower_image_samples(nir_builder *b, nir_intrinsic_instr *intrin)
+{
+   b->cursor = nir_instr_remove(&intrin->instr);
+   nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
+
+   nir_def *sw_log2, *sh_log2;
+   load_sample_size(b, deref, &sw_log2, &sh_log2);
+
+   nir_def *samples_log2 = nir_iadd(b, sw_log2, sh_log2);
+   nir_def *samples = nir_ishl(b, nir_imm_int(b, 1), samples_log2);
+
+   // Handle null descriptors
+   nir_def *addr = load_su_info(b, deref, addr_shifted8);
+   nir_def *is_null = nir_ieq_imm(b, addr, 0);
+   samples = nir_bcsel(b, is_null, nir_imm_int(b, 0), samples);
+
+   nir_def_rewrite_uses(&intrin->def, samples);
+}
+
+static bool
+lower_image_intrin(nir_builder *b,
+                   nir_intrinsic_instr *intrin,
+                   void *_data)
+{
+   switch (intrin->intrinsic) {
+   case nir_intrinsic_image_deref_load:
+   case nir_intrinsic_image_deref_store:
+   case nir_intrinsic_image_deref_atomic:
+   case nir_intrinsic_image_deref_atomic_swap:
+      lower_image_access(b, intrin);
+      return true;
+   case nir_intrinsic_image_deref_size:
+      if (nir_intrinsic_image_dim(intrin) == GLSL_SAMPLER_DIM_BUF) {
+         lower_buffer_size(b, intrin);
+      } else {
+         lower_image_size(b, intrin);
+      }
+      return true;
+   case nir_intrinsic_image_deref_samples:
+      lower_image_samples(b, intrin);
+      return true;
+   default:
+      return false;
+   }
+}
+
+bool
+nak_nir_lower_image_addrs(nir_shader *nir,
+                          const struct nak_compiler *nak)
+{
+   return nir_shader_intrinsics_pass(nir, lower_image_intrin,
+                                     nir_metadata_none,
+                                     (void *)nak);
+}
@@ -0,0 +1,586 @@
+/*
+ * Copyright © 2026 Valve Corporation.
+ * Copyright © 2023 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "nak_private.h"
+#include "nir_builder.h"
+
+static bool
+lower_mesh_io_intrin(nir_builder *b,
+                    nir_intrinsic_instr *intrin,
+                    const struct lower_mesh_intrinsics_ctx *ctx)
+{
+   b->cursor = nir_before_instr(&intrin->instr);
+
+   nir_def *vtx = NULL, *offset = NULL, *data = NULL;
+
+   switch (intrin->intrinsic) {
+   case nir_intrinsic_load_per_vertex_output:
+   case nir_intrinsic_load_per_primitive_output:
+      vtx = intrin->src[0].ssa;
+      offset = intrin->src[1].ssa;
+      break;
+
+   case nir_intrinsic_store_per_vertex_output:
+   case nir_intrinsic_store_per_primitive_output:
+      data = intrin->src[0].ssa;
+      vtx = intrin->src[1].ssa;
+      offset = intrin->src[2].ssa;
+      break;
+
+   default:
+      UNREACHABLE("unknown intrinsic");
+   }
+
+   const bool is_per_primitive = intrin->intrinsic == nir_intrinsic_load_per_primitive_output ||
+                                 intrin->intrinsic == nir_intrinsic_store_per_primitive_output;
+
+   const bool is_store = data != NULL;
+   nir_io_semantics sem = nir_intrinsic_io_semantics(intrin);
+
+   const bool is_primitive_indices = sem.location == VARYING_SLOT_PRIMITIVE_INDICES;
+
+   const struct nak_nir_isbe_flags flags = {
+      .access = is_primitive_indices ? NAK_ISBE_ACCESS_MAP : NAK_ISBE_ACCESS_ATTR,
+      .output = true,
+      .skew = !is_primitive_indices,
+      .per_primitive = is_per_primitive,
+   };
+
+   uint32_t base_addr =
+      nak_varying_mesh_skew_attr_addr(ctx->nak, sem.location) +
+      4 * nir_intrinsic_component(intrin);
+
+   uint32_t range;
+   if (nir_src_is_const(nir_src_for_ssa(offset))) {
+      uint32_t const_offset = nir_src_as_uint(nir_src_for_ssa(offset));
+      /* Tighten the range */
+      base_addr += const_offset * 16;
+      range = 4 * intrin->num_components;
+
+      if (const_offset != 0)
+         offset = nir_imm_int(b, 0);
+   } else {
+      /* Offsets from NIR are in vec4's */
+      offset = nir_imul_imm(b, offset, 16);
+      range = (sem.num_slots - 1) * 16 + intrin->num_components * 4;
+   }
+
+   nir_def *isbe_offset;
+   uint32_t stride;
+   if (is_primitive_indices) {
+      const uint32_t vertices_per_prim = mesa_vertices_per_prim(b->shader->info.mesh.primitive_type);
+
+      /* Indices are 8 bits on hardware */
+      isbe_offset = nir_iadd(b, offset, nir_iadd_imm(b, nir_imul_imm(b, vtx, vertices_per_prim), 4));
+      stride = 1;
+   } else {
+      uint16_t skew_attr_offset = nak_mesh_skew_offset(ctx, sem.location, base_addr, is_per_primitive);
+      nir_def *skew_start_offset;
+      uint16_t skew_group_size;
+
+      if (is_per_primitive) {
+         skew_start_offset = nir_imm_int(b, nak_mesh_skew_vert_total_size(ctx));
+         skew_group_size = nak_mesh_skew_prim_group_size(ctx);
+      } else {
+         skew_start_offset = nir_imm_int(b, 0);
+         skew_group_size = nak_mesh_skew_vert_group_size(ctx);
+      }
+
+      /* Readjust offset to take into account SKEW groups */
+      nir_def *offset_ajusted = nir_imul_imm(b, offset, NAK_MESH_SKEW_GROUP_COUNT);
+      skew_start_offset = nir_iadd(b, skew_start_offset, nir_imul_imm(b, nir_udiv_imm(b, vtx, 32), skew_group_size));
+
+      isbe_offset = nir_iadd(b, nir_iadd_imm(b, nir_iadd(b, nir_imul_imm(b, nir_imod_imm(b, vtx, 32), 4),
+                                                         skew_start_offset),
+                                             skew_attr_offset),
+                             offset_ajusted);
+      stride = 4 * NAK_MESH_SKEW_GROUP_COUNT;
+   }
+
+   if (is_store) {
+      u_foreach_bit(c, nir_intrinsic_write_mask(intrin)) {
+         nir_def *c_offset = nir_iadd_imm(b, isbe_offset, c * stride);
+         nir_def *c_data = nir_channel(b, data, c);
+
+         /* Handle indices conversion */
+         if (is_primitive_indices)
+            c_data = nir_u2u8(b, c_data);
+
+         nir_isbewr_nv(b, c_data, c_offset, .range_base = base_addr,
+                       .range = range, .flags = NAK_AS_U32(flags));
+      }
+   } else {
+      const uint8_t bit_size = is_primitive_indices ? 8 : intrin->def.bit_size;
+
+      nir_def *comps[NIR_MAX_VEC_COMPONENTS];
+      for (uint32_t c = 0; c < intrin->num_components; c++) {
+         nir_def *c_offset = nir_iadd_imm(b, isbe_offset, c * stride);
+         nir_def *c_data =
+            nir_isberd_nv(b, bit_size, c_offset, .range_base = base_addr,
+                          .range = range, .flags = NAK_AS_U32(flags));
+
+         /* Handle indices conversion */
+         if (is_primitive_indices)
+            c_data = nir_u2u32(b, c_data);
+
+         comps[c] = c_data;
+      }
+
+      nir_def *dst = nir_vec(b, comps, intrin->num_components);
+      nir_def_rewrite_uses(&intrin->def, dst);
+   }
+
+   nir_instr_remove(&intrin->instr);
+
+   return true;
+}
+
+static bool
+lower_set_vertex_and_primitive_count(nir_builder *b,
+                                     nir_intrinsic_instr *intrin)
+{
+   b->cursor = nir_before_instr(&intrin->instr);
+
+   nir_def *primitive_count = intrin->src[1].ssa;
+   nir_def *offset = nir_imm_int(b, 0x3);
+
+   const struct nak_nir_isbe_flags flags = {
+      .access = NAK_ISBE_ACCESS_MAP,
+      .output = true,
+      .skew = false,
+      .per_primitive = false,
+   };
+
+   nir_isbewr_nv(b, primitive_count, offset,
+                 .flags = NAK_AS_U32(flags));
+
+   nir_instr_remove(&intrin->instr);
+
+   return true;
+}
+
+static bool
+lower_load_workgroup_index(nir_builder *b,
+                           nir_intrinsic_instr *intrin,
+                           bool from_skew)
+{
+   nir_function_impl *impl = nir_shader_get_entrypoint(b->shader);
+
+   /* We need to make sure that this is read before any writes to allow ISBE
+    * space sharing optimisation to happen */
+   b->cursor = nir_before_impl(impl);
+
+   const struct nak_nir_isbe_flags flags = {
+      .access = NAK_ISBE_ACCESS_ATTR,
+      .output = false,
+      .skew = from_skew,
+      .per_primitive = false,
+   };
+
+   nir_def *dst =  nir_isberd_nv(b, 32, nir_imm_int(b, 0),
+                                 .range_base = NAK_ATTR_VERTEX_ID,
+                                 .range = 4,
+                                 .flags = NAK_AS_U32(flags));
+
+   nir_def_rewrite_uses(&intrin->def, dst);
+   nir_instr_remove(&intrin->instr);
+
+   return true;
+}
+
+static bool
+lower_load_num_workgroups(nir_builder *b, nir_intrinsic_instr *intrin)
+{
+   /* If we are here, we have a task shader */
+   b->cursor = nir_before_instr(&intrin->instr);
+
+   const struct nak_nir_isbe_flags flags = {
+      .access = NAK_ISBE_ACCESS_ATTR,
+      .output = false,
+      .skew = false,
+      .per_primitive = false,
+   };
+
+   nir_def *x =
+      nir_isberd_nv(b, 32, nir_imm_int(b, 0x8), .flags = NAK_AS_U32(flags),
+                    .access = ACCESS_CAN_REORDER);
+   nir_def *y =
+      nir_isberd_nv(b, 32, nir_imm_int(b, 0xC), .flags = NAK_AS_U32(flags),
+                    .access = ACCESS_CAN_REORDER);
+   nir_def *z =
+      nir_isberd_nv(b, 32, nir_imm_int(b, 0x10), .flags = NAK_AS_U32(flags),
+                    .access = ACCESS_CAN_REORDER);
+   nir_def *dst = nir_vec3(b, x, y, z);
+   nir_def_rewrite_uses(&intrin->def, dst);
+   nir_instr_remove(&intrin->instr);
+
+   return true;
+}
+
+static bool
+lower_load_shared(nir_builder *b, nir_intrinsic_instr *intrin,
+                  uint32_t base_offset)
+{
+   b->cursor = nir_before_instr(&intrin->instr);
+
+   nir_def *offset = intrin->src[0].ssa;
+
+   const uint8_t bit_size = intrin->def.bit_size;
+   assert(bit_size == 32 && intrin->def.num_components == 1);
+
+   const uint32_t base = nir_intrinsic_base(intrin);
+   const struct nak_nir_isbe_flags flags = {
+      .access = NAK_ISBE_ACCESS_ATTR,
+      .output = true,
+      .skew = false,
+      .per_primitive = false,
+   };
+
+   offset = nir_iadd_imm(b, offset, base_offset + base);
+   nir_def *dst = nir_isberd_nv(b, 32, offset, .flags = NAK_AS_U32(flags));
+   nir_def_rewrite_uses(&intrin->def, dst);
+   nir_instr_remove(&intrin->instr);
+
+   return true;
+}
+
+static bool
+lower_store_shared(nir_builder *b, nir_intrinsic_instr *intrin,
+                   uint32_t base_offset)
+{
+   b->cursor = nir_before_instr(&intrin->instr);
+
+   nir_def *value = intrin->src[0].ssa;
+   nir_def *offset = intrin->src[1].ssa;
+
+   const uint8_t bit_size = value->bit_size;
+   assert(bit_size == 32 &&
+          nir_intrinsic_write_mask(intrin) == nir_component_mask(1));
+
+   const uint32_t base = nir_intrinsic_base(intrin);
+   const struct nak_nir_isbe_flags flags = {
+      .access = NAK_ISBE_ACCESS_ATTR,
+      .output = true,
+      .skew = false,
+      .per_primitive = false,
+   };
+
+   offset = nir_iadd_imm(b, offset, base_offset + base);
+   nir_isbewr_nv(b, value, offset, .flags = NAK_AS_U32(flags));
+   nir_instr_remove(&intrin->instr);
+
+   return true;
+}
+
+static bool
+lower_load_task_payload(nir_builder *b, nir_intrinsic_instr *intrin)
+{
+   b->cursor = nir_before_instr(&intrin->instr);
+
+   nir_def *offset = intrin->src[0].ssa;
+
+   const uint8_t bit_size = intrin->def.bit_size;
+   assert(bit_size == 32 && intrin->def.num_components == 1);
+
+   const uint32_t base = nir_intrinsic_base(intrin);
+   const struct nak_nir_isbe_flags flags = {
+      .access = NAK_ISBE_ACCESS_ATTR,
+      .output = false,
+      .skew = false,
+      .per_primitive = false,
+   };
+
+   offset = nir_iadd_imm(b, offset, base);
+   nir_def *dst = nir_isberd_nv(b, 32, offset, .flags = NAK_AS_U32(flags),
+                                .access = ACCESS_CAN_REORDER);
+   nir_def_rewrite_uses(&intrin->def, dst);
+   nir_instr_remove(&intrin->instr);
+
+   return true;
+}
+
+static bool
+lower_mesh_intrin(nir_builder *b,
+                  nir_intrinsic_instr *intrin,
+                  void *cb_data)
+{
+   const struct lower_mesh_intrinsics_ctx *ctx = cb_data;
+
+   /* Shared memory is after attributes on mesh shaders */
+   const uint32_t shared_memory_base = nak_mesh_skew_total_size(ctx);
+   assert(shared_memory_base % 0x80 == 0);
+
+   switch (intrin->intrinsic) {
+   case nir_intrinsic_load_per_vertex_output:
+   case nir_intrinsic_load_per_primitive_output:
+   case nir_intrinsic_store_per_vertex_output:
+   case nir_intrinsic_store_per_primitive_output:
+      return lower_mesh_io_intrin(b, intrin, ctx);
+   case nir_intrinsic_set_vertex_and_primitive_count:
+      return lower_set_vertex_and_primitive_count(b, intrin);
+   case nir_intrinsic_load_workgroup_index:
+      return lower_load_workgroup_index(b, intrin, !ctx->has_task_shader);
+   case nir_intrinsic_load_num_workgroups:
+      return lower_load_num_workgroups(b, intrin);
+   case nir_intrinsic_load_shared:
+      return lower_load_shared(b, intrin, shared_memory_base);
+   case nir_intrinsic_store_shared:
+      return lower_store_shared(b, intrin, shared_memory_base);
+   case nir_intrinsic_load_task_payload:
+      return lower_load_task_payload(b, intrin);
+   case nir_intrinsic_shared_atomic:
+   case nir_intrinsic_shared_atomic_swap:
+      UNREACHABLE(
+         "Should have been lowered by nak_nir_lower_mesh_stages_shared_atomics");
+   default:
+      return false;
+   }
+}
+
+struct lower_emulated_attributes_state {
+   uint32_t viewport_shared_offset;
+   uint32_t cullprimitive_shared_offset;
+};
+
+static bool
+lower_emulated_attributes_intrin(nir_builder *b, nir_intrinsic_instr *intrin,
+                                 void *_data)
+{
+   const struct lower_emulated_attributes_state *state = _data;
+   nir_def *vtx = NULL, *offset = NULL;
+
+   switch (intrin->intrinsic) {
+   case nir_intrinsic_load_per_primitive_output:
+      vtx = intrin->src[0].ssa;
+      offset = intrin->src[1].ssa;
+      break;
+
+   case nir_intrinsic_store_per_primitive_output:
+      vtx = intrin->src[1].ssa;
+      offset = intrin->src[2].ssa;
+      break;
+
+   default:
+      return false;
+   }
+
+   nir_io_semantics sem = nir_intrinsic_io_semantics(intrin);
+
+   if (sem.location != VARYING_SLOT_VIEWPORT &&
+       sem.location != VARYING_SLOT_CULL_PRIMITIVE)
+      return false;
+
+   b->cursor = nir_before_instr(&intrin->instr);
+   nir_def *shared_offset = nir_iadd_imm(b, offset, nir_intrinsic_base(intrin));
+   shared_offset = nir_iadd(b, shared_offset, nir_imul_imm(b, vtx, 4));
+
+   if (sem.location == VARYING_SLOT_CULL_PRIMITIVE)
+      shared_offset =
+         nir_iadd_imm(b, shared_offset, state->cullprimitive_shared_offset);
+   else
+      shared_offset =
+         nir_iadd_imm(b, shared_offset, state->viewport_shared_offset);
+
+   if (intrin->intrinsic == nir_intrinsic_store_per_primitive_output) {
+      nir_def *data = intrin->src[0].ssa;
+      switch (sem.location) {
+      case VARYING_SLOT_VIEWPORT:
+         /* In case of Viewport, the data needs to be translated to a proper
+          * mask value to map to ViewportMask */
+         data = nir_ishl(b, nir_imm_int(b, 1), data);
+         break;
+      case VARYING_SLOT_CULL_PRIMITIVE:
+         /* In case of CullPrimitive, the data is already a 32-bit value so no
+          * translation is needed */
+         break;
+      default:
+         UNREACHABLE("Should never happen");
+      }
+
+      nir_store_shared(b, data, shared_offset);
+      nir_instr_remove(&intrin->instr);
+   } else {
+      /* Reading back isn't allowed by VK_EXT_mesh_shader but allowed by
+       * VK_NV_mesh_shader. We support readback for completeness and in case we
+       * add support for NV specific extension in the future */
+      nir_def *data = nir_load_shared(b, 1, 32, shared_offset);
+      switch (sem.location) {
+      case VARYING_SLOT_VIEWPORT:
+         /* In case of Viewport, find the first index that is set. */
+         data = nir_find_lsb(b, data);
+         break;
+      case VARYING_SLOT_CULL_PRIMITIVE:
+         /* In case of CullPrimitive, we check if no bits are set */
+         data = nir_ine_imm(b, data, 0);
+         break;
+      default:
+         UNREACHABLE("Should never happen");
+      }
+      nir_def_replace(&intrin->def, data);
+   }
+
+   return true;
+}
+
+bool
+nak_nir_lower_mesh_emulated_attributes(nir_shader *nir)
+{
+   if (nir->info.stage != MESA_SHADER_MESH)
+      return false;
+
+   /* Only apply this pass when really needed */
+   if ((nir->info.per_primitive_outputs &
+        (VARYING_BIT_CULL_PRIMITIVE | VARYING_BIT_VIEWPORT)) == 0)
+      return false;
+
+   /* If we are here, we need to emulate Viewport / CullPrimitive with the
+    * ViewportMask. This means if we need to always keep a shadow copy of the
+    * ViewportMask and CullPrimitive in shared memory and write the actual
+    * ViewportMask at the end of the shader. */
+   bool progress = false;
+
+   /* Reserve space for the Viewport and CullPrimitive shadow copies */
+   uint32_t shared_memory_offset = nir->info.shared_size;
+   nir->info.shared_size += 8 * nir->info.mesh.max_primitives_out;
+
+   struct lower_emulated_attributes_state state = {
+      .viewport_shared_offset = shared_memory_offset,
+      .cullprimitive_shared_offset =
+         shared_memory_offset + 4 * nir->info.mesh.max_primitives_out,
+   };
+
+   /* First we lower things to shared memory */
+   progress |= nir_shader_intrinsics_pass(nir, lower_emulated_attributes_intrin,
+                                          nir_metadata_control_flow, &state);
+
+   /* Finally, we ensure that the shared region is init at the start of the
+    * shader and we add primitive writes at the end of the shader to write the
+    * real value depending on the culling state.*/
+   if (progress) {
+      nir_function_impl *impl = nir_shader_get_entrypoint(nir);
+      nir_builder b = nir_builder_at(nir_before_impl(impl));
+      nir_def *zero = nir_imm_int(&b, 0);
+      nir_def *viewport_default = nir_imm_int(&b, 1 << 0);
+
+      nir_def *lane_id =
+         nak_nir_load_sysval(&b, NAK_SV_LANE_ID, ACCESS_CAN_REORDER);
+      nir_push_if(&b, nir_ieq(&b, lane_id, zero));
+      {
+         for (uint32_t i = 0; i < nir->info.mesh.max_primitives_out; i++) {
+            nir_store_shared(
+               &b, viewport_default,
+               nir_imm_int(&b, state.viewport_shared_offset + i * 4));
+            nir_store_shared(
+               &b, zero,
+               nir_imm_int(&b, state.cullprimitive_shared_offset + i * 4));
+         }
+      }
+      nir_pop_if(&b, NULL);
+      nir_barrier(&b, SCOPE_WORKGROUP, SCOPE_WORKGROUP, NIR_MEMORY_ACQ_REL,
+                  nir_var_mem_shared);
+
+      b = nir_builder_at(nir_after_impl(impl));
+      nir_barrier(&b, SCOPE_WORKGROUP, SCOPE_WORKGROUP, NIR_MEMORY_ACQ_REL,
+                  nir_var_mem_shared);
+      nir_push_if(&b, nir_ieq(&b, lane_id, zero));
+      {
+         for (uint32_t i = 0; i < nir->info.mesh.max_primitives_out; i++) {
+            nir_def *viewport_mask = nir_load_shared(
+               &b, 1, 32,
+               nir_imm_int(&b, state.viewport_shared_offset + i * 4));
+            nir_def *cull_primitive = nir_load_shared(
+               &b, 1, 32,
+               nir_imm_int(&b, state.cullprimitive_shared_offset + i * 4));
+
+            viewport_mask = nir_bcsel(&b, nir_ine_imm(&b, cull_primitive, 0),
+                                      zero, viewport_mask);
+            nir_store_per_primitive_output(
+               &b, viewport_mask, nir_imm_int(&b, i), zero, .base = 0,
+               .src_type = nir_type_uint32,
+               .io_semantics = (nir_io_semantics){
+                  .location = VARYING_SLOT_VIEWPORT_MASK,
+                  .num_slots = 1,
+               });
+         }
+      }
+      nir_pop_if(&b, NULL);
+   }
+
+   return progress;
+}
+
+bool
+nak_nir_lower_mesh_intrinsics(nir_shader *nir,
+                              struct lower_mesh_intrinsics_ctx *ctx)
+{
+   return nir_shader_intrinsics_pass(
+      nir, lower_mesh_intrin, nir_metadata_block_index | nir_metadata_dominance,
+      ctx);
+}
+
+static bool
+lower_launch_mesh_workgroups(nir_builder *b, nir_intrinsic_instr *intrin)
+{
+   b->cursor = nir_before_instr(&intrin->instr);
+
+   nir_def *dim = intrin->src[0].ssa;
+   nir_def *x = nir_channel(b, dim, 0);
+   nir_def *y = nir_channel(b, dim, 1);
+   nir_def *z = nir_channel(b, dim, 2);
+   nir_def *task_count = nir_imul(b, nir_imul(b, x, y), z);
+
+   const struct nak_nir_isbe_flags flags = {
+      .access = NAK_ISBE_ACCESS_ATTR,
+      .output = true,
+      .skew = false,
+      .per_primitive = false,
+   };
+
+   nir_isbewr_nv(b, task_count, nir_imm_int(b, 0x4),
+                 .flags = NAK_AS_U32(flags));
+   nir_isbewr_nv(b, x, nir_imm_int(b, 0x8), .flags = NAK_AS_U32(flags));
+   nir_isbewr_nv(b, y, nir_imm_int(b, 0xC), .flags = NAK_AS_U32(flags));
+   nir_isbewr_nv(b, z, nir_imm_int(b, 0x10), .flags = NAK_AS_U32(flags));
+   nir_instr_remove(&intrin->instr);
+   return true;
+}
+
+static bool
+lower_task_intrin(nir_builder *b,
+                  nir_intrinsic_instr *intrin,
+                  void *cb_data)
+{
+   switch (intrin->intrinsic) {
+   case nir_intrinsic_load_shared:
+      return lower_load_shared(b, intrin, 0);
+   case nir_intrinsic_store_shared:
+      return lower_store_shared(b, intrin, 0);
+   case nir_intrinsic_load_workgroup_index:
+      return lower_load_workgroup_index(b, intrin, true);
+   case nir_intrinsic_launch_mesh_workgroups:
+      return lower_launch_mesh_workgroups(b, intrin);
+   case nir_intrinsic_shared_atomic:
+   case nir_intrinsic_shared_atomic_swap:
+      UNREACHABLE(
+         "Should have been lowered by nak_nir_lower_mesh_stages_shared_atomics");
+   case nir_intrinsic_load_task_payload:
+   case nir_intrinsic_store_task_payload:
+   case nir_intrinsic_task_payload_atomic:
+   case nir_intrinsic_task_payload_atomic_swap:
+      UNREACHABLE("Should have been lowered by nvk_nir_lower_task_shader");
+   default:
+      return false;
+   }
+}
+
+bool
+nak_nir_lower_task_intrinsics(nir_shader *nir)
+{
+   return nir_shader_intrinsics_pass(nir, lower_task_intrin,
+                                     nir_metadata_block_index |
+                                     nir_metadata_dominance,
+                                     NULL);
+}
@@ -0,0 +1,518 @@
+/*
+ * Copyright © 2024 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "nak_private.h"
+#include "nir_builder.h"
+#include "nir_vla.h"
+
+/*
+ * This pass relies on nak_nir_mark_lcssa_invariants being run first, because it
+ * assumes that convergent values used in convergent control flow can be
+ * allocated to uniform registers. See the example in that file for more
+ * details.
+ */
+
+static void
+lower_ldcx_to_global(nir_builder *b, nir_intrinsic_instr *load,
+                     const struct nak_compiler *nak)
+{
+   assert(load->intrinsic == nir_intrinsic_ldcx_nv);
+
+   b->cursor = nir_before_instr(&load->instr);
+
+   nir_def *handle = load->src[0].ssa;
+   assert(handle->bit_size == 64);
+   nir_def *offset = load->src[1].ssa;
+   assert(offset->bit_size == 32);
+
+   nir_def *addr;
+   nir_def *size;
+   if (nak->sm >= 100) {
+      /* Base address shifted by 6 is the bottom 51 bits */
+      nir_def *addr_6_lo = nir_unpack_64_2x32_split_x(b, handle);
+      nir_def *addr_6_hi = nir_iand_imm(b, nir_unpack_64_2x32_split_y(b, handle),
+                                        BITFIELD_MASK(19));
+      nir_def *addr_6 = nir_pack_64_2x32_split(b, addr_6_lo, addr_6_hi);
+      addr = nir_ishl_imm(b, addr_6, 6);
+
+      /* Size shifted by 2 is the top 13 bits */
+      size = nir_unpack_64_2x32_split_y(b, handle);
+      size = nir_ishl_imm(b, nir_ushr_imm(b, size, 19), 4);
+   } else {
+      /* Base address shifted by 4 is the bottom 45 bits */
+      nir_def *addr_4_lo = nir_unpack_64_2x32_split_x(b, handle);
+      nir_def *addr_4_hi = nir_iand_imm(b, nir_unpack_64_2x32_split_y(b, handle),
+                                        BITFIELD_MASK(13));
+      nir_def *addr_4 = nir_pack_64_2x32_split(b, addr_4_lo, addr_4_hi);
+      addr =  nir_ishl_imm(b, addr_4, 4);
+
+      /* Size shifted by 2 is the top 19 bits */
+      size = nir_unpack_64_2x32_split_y(b, handle);
+      size = nir_ishl_imm(b, nir_ushr_imm(b, size, 13), 4);
+   }
+   /* At this point we can assume the offset is aligned so we only need a
+    * simple less-than check here.
+    */
+   nir_def *cond = nir_ilt(b, offset, size);
+   nir_def *zero_addr = nir_imm_zero(b, addr->num_components,
+                                        addr->bit_size);
+   nir_def *val = nir_load_global_nv(b,
+      load->def.num_components, load->def.bit_size,
+      nir_iadd(b, addr, nir_u2u64(b, offset)),
+      zero_addr, cond,
+      .align_mul = nir_intrinsic_align_mul(load),
+      .align_offset = nir_intrinsic_align_offset(load),
+      .access = ACCESS_CAN_REORDER,
+      .base = nir_intrinsic_base(load));
+
+   nir_def_replace(&load->def, val);
+}
+
+static bool
+lower_all_ldcx_to_global_intrin(nir_builder *b, nir_intrinsic_instr *load,
+                                void *data)
+{
+   if (load->intrinsic != nir_intrinsic_ldcx_nv)
+      return false;
+
+   lower_ldcx_to_global(b, load, data);
+   return true;
+}
+
+struct non_uniform_section {
+   nir_block *pred;
+   nir_block *succ;
+   struct hash_table *live_handles;
+};
+
+static void
+non_uniform_section_init(struct non_uniform_section *nus,
+                         nir_block *pred, nir_block *succ)
+{
+   *nus = (struct non_uniform_section) {
+      .pred = pred,
+      .succ = succ,
+      .live_handles = _mesa_pointer_hash_table_create(NULL),
+   };
+}
+
+static void
+non_uniform_section_finish(struct non_uniform_section *nus)
+{
+   _mesa_hash_table_destroy(nus->live_handles, NULL);
+}
+
+static void
+add_live_handle(nir_def *handle, struct non_uniform_section *nus)
+{
+   uint32_t hash = _mesa_hash_pointer(handle);
+   struct hash_entry *entry =
+      _mesa_hash_table_search_pre_hashed(nus->live_handles, hash, handle);
+   if (entry == NULL) {
+      _mesa_hash_table_insert_pre_hashed(nus->live_handles, hash, handle,
+                                         (void *)(uintptr_t)1);
+   } else {
+      entry->data = (void *)(((uintptr_t)entry->data) + 1);
+   }
+}
+
+static bool
+def_needs_hoist(nir_def *def, nir_block *target)
+{
+   return nir_def_block(def)->index > target->index;
+}
+
+static bool
+can_hoist_def(nir_def *def, nir_block *target)
+{
+   if (!def_needs_hoist(def, target))
+      return true;
+
+   nir_instr *instr = nir_def_instr(def);
+   switch (instr->type) {
+   case nir_instr_type_alu: {
+      nir_alu_instr *alu = nir_instr_as_alu(instr);
+      for (uint8_t i = 0; i < nir_op_infos[alu->op].num_inputs; i++) {
+         if (!can_hoist_def(alu->src[i].src.ssa, target))
+            return false;
+      }
+      return true;
+   }
+
+   case nir_instr_type_intrinsic: {
+      nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
+      if (intrin->intrinsic != nir_intrinsic_ldc_nv &&
+          intrin->intrinsic != nir_intrinsic_ldcx_nv)
+         return false;
+
+      for (uint8_t i = 0; i < 2; i++) {
+         if (!can_hoist_def(intrin->src[i].ssa, target))
+            return false;
+      }
+
+      return true;
+   }
+
+   case nir_instr_type_load_const:
+      return true;
+
+   default:
+      return false;
+   }
+}
+
+static bool
+hoist_def(nir_def *def, nir_block *target)
+{
+   if (!def_needs_hoist(def, target))
+      return false;
+
+   nir_instr *instr = nir_def_instr(def);
+   switch (instr->type) {
+   case nir_instr_type_alu: {
+      nir_alu_instr *alu = nir_instr_as_alu(instr);
+      for (uint8_t i = 0; i < nir_op_infos[alu->op].num_inputs; i++)
+         hoist_def(alu->src[i].src.ssa, target);
+      break;
+   }
+
+   case nir_instr_type_intrinsic: {
+      nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
+      if (intrin->intrinsic != nir_intrinsic_ldc_nv &&
+          intrin->intrinsic != nir_intrinsic_ldcx_nv)
+         break;
+
+      for (uint8_t i = 0; i < 2; i++)
+         hoist_def(intrin->src[i].ssa, target);
+      break;
+   }
+
+   case nir_instr_type_load_const:
+      break;
+
+   default:
+      UNREACHABLE("Cannot hoist instruction");
+   }
+
+   nir_instr_remove(instr);
+   nir_instr_insert(nir_after_block(target), instr);
+
+   return true;
+}
+
+static bool
+try_hoist_ldcx_handles_block(nir_block *block, struct non_uniform_section *nus)
+{
+   bool progress = false;
+
+   nir_foreach_instr_safe(instr, block) {
+      switch (instr->type) {
+      case nir_instr_type_alu: {
+         /* ALU instructions may directly consume cbuf values so mark
+          * any ldcx_nv sources as also used.
+          */
+         nir_alu_instr *alu = nir_instr_as_alu(instr);
+         for (uint8_t i = 0; i < nir_op_infos[alu->op].num_inputs; i++) {
+            nir_instr *src_instr = nir_def_instr(alu->src[i].src.ssa);
+            if (src_instr->type != nir_instr_type_intrinsic)
+               continue;
+
+            nir_intrinsic_instr *src_intrin =
+               nir_instr_as_intrinsic(src_instr);
+            if (src_intrin->intrinsic != nir_intrinsic_ldcx_nv)
+               continue;
+
+            if (nir_intrinsic_access(src_intrin) & ACCESS_NON_UNIFORM)
+               continue;
+
+            /* If it really is a uniform handle, it should have been
+             * hoistd by now.
+             */
+            nir_def *handle = src_intrin->src[0].ssa;
+            if (def_needs_hoist(handle, nus->pred))
+               continue;
+
+            add_live_handle(handle, nus);
+         }
+         break;
+      }
+
+      case nir_instr_type_intrinsic: {
+         nir_intrinsic_instr *load = nir_instr_as_intrinsic(instr);
+         if (load->intrinsic != nir_intrinsic_ldcx_nv)
+            break;
+
+         if (nir_intrinsic_access(load) & ACCESS_NON_UNIFORM)
+            break;
+
+         nir_def *handle = load->src[0].ssa;
+         if (can_hoist_def(handle, nus->pred)) {
+            progress |= hoist_def(handle, nus->pred);
+            add_live_handle(handle, nus);
+         }
+         break;
+      }
+
+      default:
+         break;
+      }
+   }
+
+   return progress;
+}
+
+struct handle {
+   nir_def *handle;
+   uint64_t sort_key;
+};
+
+static int
+cmp_handles(const void *_a, const void *_b)
+{
+   const struct handle *a = _a, *b = _b;
+
+   if (a->sort_key < b->sort_key)
+      return -1;
+   else if (a->sort_key > b->sort_key)
+      return 1;
+   else
+      return 0;
+}
+
+static bool
+sort_and_mark_live_handles(nir_builder *b, struct non_uniform_section *nus)
+{
+   /* There are 63 uniform registers and each handle takes 2 */
+   static const unsigned max_live_handles = 31;
+
+   const unsigned num_handles = nus->live_handles->entries;
+   if (num_handles == 0)
+      return false;
+
+   NIR_VLA(struct handle, handles, num_handles);
+
+   unsigned i = 0;
+   hash_table_foreach(nus->live_handles, entry) {
+      nir_def *handle = (nir_def *)entry->key;
+      uint64_t uses = (uintptr_t)entry->data;
+
+      handles[i++] = (struct handle) {
+         .handle = handle,
+         .sort_key = (uses << 32) | handle->index,
+      };
+   }
+   _mesa_hash_table_clear(nus->live_handles, NULL);
+
+   qsort(handles, num_handles, sizeof(*handles), cmp_handles);
+
+   b->cursor = nir_after_block(nus->pred);
+   for (unsigned i = 0; i < num_handles && i < max_live_handles; i++) {
+      nir_def *handle = handles[i].handle;
+      if (handle->divergent)
+         handle = nir_as_uniform(b, handles[i].handle);
+      nir_pin_cx_handle_nv(b, handle);
+
+      _mesa_hash_table_insert(nus->live_handles, handle, handle);
+      _mesa_hash_table_insert(nus->live_handles, handles[i].handle, handle);
+   }
+
+   b->cursor = nir_before_block_after_phis(nus->succ);
+   for (unsigned i = 0; i < num_handles && i < max_live_handles; i++)
+      nir_unpin_cx_handle_nv(b, handles[i].handle);
+
+   return true;
+}
+
+static bool
+try_remat_ldcx_alu_use(nir_builder *b, nir_alu_instr *alu, uint8_t src_idx,
+                       struct non_uniform_section *nus)
+{
+   nir_instr *src_instr = nir_def_instr(alu->src[src_idx].src.ssa);
+   if (src_instr->type != nir_instr_type_intrinsic)
+      return false;
+
+   nir_intrinsic_instr *load = nir_instr_as_intrinsic(src_instr);
+   if (load->intrinsic != nir_intrinsic_ldcx_nv)
+      return false;
+
+   /* If it's already in the same block, there's nothing to do */
+   if (load->instr.block == alu->instr.block)
+      return false;
+
+   nir_def *handle = load->src[0].ssa;
+
+   /* Only constant offset can be used directly by ALU ops */
+   if (!nir_src_is_const(load->src[1]))
+      return false;
+
+   struct hash_entry *entry =
+      _mesa_hash_table_search(nus->live_handles, handle);
+   if (entry == NULL)
+      return false;
+
+   nir_intrinsic_instr *new_load =
+      nir_instr_as_intrinsic(nir_instr_clone(b->shader, &load->instr));
+   nir_instr_insert(nir_before_instr(&alu->instr), &new_load->instr);
+
+   nir_foreach_use_safe(use, &load->def) {
+      if (nir_src_use_instr(use)->type == nir_instr_type_alu &&
+          nir_src_use_instr(use)->block == alu->instr.block)
+         nir_src_rewrite(use, &new_load->def);
+   }
+
+   return true;
+}
+
+static bool
+lower_ldcx_block(nir_builder *b, nir_block *block,
+                 struct non_uniform_section *nus,
+                 const struct nak_compiler *nak)
+{
+   bool progress = false;
+
+   nir_foreach_instr_safe(instr, block) {
+      switch (instr->type) {
+      case nir_instr_type_alu: {
+         nir_alu_instr *alu = nir_instr_as_alu(instr);
+         if (nus != NULL) {
+            /* In non-uniform control-flow, NAK can't copy-prop cbuf values
+             * across blocks.  Re-materialize cbufs that are used by ALU
+             * instructions in the same block.
+             */
+            for (uint8_t i = 0; i < nir_op_infos[alu->op].num_inputs; i++)
+               progress |= try_remat_ldcx_alu_use(b, alu, i, nus);
+         }
+         break;
+      }
+      case nir_instr_type_intrinsic: {
+         nir_intrinsic_instr *load = nir_instr_as_intrinsic(instr);
+         if (load->intrinsic != nir_intrinsic_ldcx_nv)
+            break;
+
+         nir_def *handle = load->src[0].ssa;
+         if (nus == NULL) {
+            /* Uniform control-flow */
+            if (nir_intrinsic_access(load) & ACCESS_NON_UNIFORM) {
+               lower_ldcx_to_global(b, load, nak);
+               progress = true;
+            } else if (handle->divergent) {
+               b->cursor = nir_before_instr(&load->instr);
+               nir_def *ugpr = nir_as_uniform(b, handle);
+               nir_src_rewrite(&load->src[0], ugpr);
+               progress = true;
+            }
+         } else {
+            struct hash_entry *entry =
+               _mesa_hash_table_search(nus->live_handles, handle);
+            if (entry != NULL) {
+               nir_def *ugpr = entry->data;
+               assert(!ugpr->divergent);
+               nir_src_rewrite(&load->src[0], ugpr);
+            } else {
+               lower_ldcx_to_global(b, load, nak);
+            }
+            progress = true;
+         }
+         break;
+      }
+
+      default:
+         break;
+      }
+   }
+
+   return progress;
+}
+
+static bool
+lower_non_uniform_cf_node(nir_builder *b, nir_cf_node *node,
+                          nir_block *pred, nir_block *succ,
+                          const struct nak_compiler *nak)
+{
+   bool progress = false;
+
+   struct non_uniform_section nus;
+   non_uniform_section_init(&nus, pred, succ);
+
+   /* This also marks any uniform handles it finds as live */
+   nir_foreach_block_in_cf_node(block, node)
+      progress |= try_hoist_ldcx_handles_block(block, &nus);
+
+   progress |= sort_and_mark_live_handles(b, &nus);
+
+   nir_foreach_block_in_cf_node_safe(block, node)
+      progress |= lower_ldcx_block(b, block, &nus, nak);
+
+   non_uniform_section_finish(&nus);
+
+   return progress;
+}
+
+static bool
+lower_cf_list(nir_builder *b, struct exec_list *cf_list,
+              const struct nak_compiler *nak)
+{
+   bool progress = false;
+
+   nir_block *block = NULL;
+   foreach_list_typed_safe(nir_cf_node, node, node, cf_list) {
+      switch (node->type) {
+      case nir_cf_node_block:
+         block = nir_cf_node_as_block(node);
+         progress |= lower_ldcx_block(b, block, NULL, nak);
+         continue;
+
+      case nir_cf_node_if: {
+         nir_if *nif = nir_cf_node_as_if(node);
+         if (nir_src_is_divergent(&nif->condition)) {
+            nir_block *succ = nir_cf_node_as_block(nir_cf_node_next(node));
+            progress |= lower_non_uniform_cf_node(b, node, block, succ, nak);
+         } else {
+            progress |= lower_cf_list(b, &nif->then_list, nak);
+            progress |= lower_cf_list(b, &nif->else_list, nak);
+         }
+         break;
+      }
+
+      case nir_cf_node_loop: {
+         nir_loop *loop = nir_cf_node_as_loop(node);
+         if (nir_loop_is_divergent(loop)) {
+            nir_block *succ = nir_cf_node_as_block(nir_cf_node_next(node));
+            progress |= lower_non_uniform_cf_node(b, node, block, succ, nak);
+         } else {
+            progress |= lower_cf_list(b, &loop->body, nak);
+            progress |= lower_cf_list(b, &loop->continue_list, nak);
+         }
+         break;
+      }
+
+      default:
+         UNREACHABLE("Unknown CF node type");
+      }
+   }
+
+   return progress;
+}
+
+bool
+nak_nir_lower_non_uniform_ldcx(nir_shader *nir,
+                               const struct nak_compiler *nak)
+{
+   /* If we don't have UGPRs, lower all of them. */
+   if (nak_debug_no_ugpr()) {
+      return nir_shader_intrinsics_pass(nir, lower_all_ldcx_to_global_intrin,
+                                        nir_metadata_none, (void *)nak);
+   }
+
+   /* Real functions are going to make hash of this */
+   nir_function_impl *impl = nir_shader_get_entrypoint(nir);
+   nir_builder b = nir_builder_create(impl);
+
+   /* We use block indices to determine when something is a predecessor */
+   nir_metadata_require(impl, nir_metadata_block_index | nir_metadata_divergence);
+
+   bool progress = lower_cf_list(&b, &impl->body, nak);
+   return nir_progress(progress, impl, nir_metadata_none);
+}
@@ -0,0 +1,272 @@
+/*
+ * Copyright © 2023 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "nak_private.h"
+#include "nir_builder.h"
+
+static nir_def *
+cluster_mask(nir_builder *b, unsigned cluster_size)
+{
+   nir_def *mask = nir_ballot(b, 1, 32, nir_imm_true(b));
+
+   if (cluster_size < 32) {
+      nir_def *idx = nir_load_subgroup_invocation(b);
+      nir_def *cluster = nir_iand_imm(b, idx, ~(uint64_t)(cluster_size - 1));
+
+      nir_def *cluster_mask = nir_imm_int(b, BITFIELD_MASK(cluster_size));
+      cluster_mask = nir_ishl(b, cluster_mask, cluster);
+
+      mask = nir_iand(b, mask, cluster_mask);
+   }
+
+   return mask;
+}
+
+static nir_def *
+build_scan_bool(nir_builder *b, nir_intrinsic_op op, nir_op red_op,
+                nir_def *data, unsigned cluster_size)
+{
+   /* Handle a couple of special cases first */
+   if (op == nir_intrinsic_reduce && cluster_size == 32) {
+      switch (red_op) {
+      case nir_op_iand:
+         return nir_vote_all(b, 1, data);
+      case nir_op_ior:
+         return nir_vote_any(b, 1, data);
+      case nir_op_ixor:
+         /* The generic path is fine */
+         break;
+      default:
+         UNREACHABLE("Unsupported boolean reduction op");
+      }
+   }
+
+   nir_def *mask = cluster_mask(b, cluster_size);
+   switch (op) {
+   case nir_intrinsic_exclusive_scan:
+      mask = nir_iand(b, mask, nir_load_subgroup_lt_mask(b, 1, 32));
+      break;
+   case nir_intrinsic_inclusive_scan:
+      mask = nir_iand(b, mask, nir_load_subgroup_le_mask(b, 1, 32));
+      break;
+   case nir_intrinsic_reduce:
+      break;
+   default:
+      UNREACHABLE("Unsupported scan/reduce op");
+   }
+
+   data = nir_ballot(b, 1, 32, data);
+
+   switch (red_op) {
+   case nir_op_iand:
+      return nir_ieq_imm(b, nir_iand(b, nir_inot(b, data), mask), 0);
+   case nir_op_ior:
+      return nir_ine_imm(b, nir_iand(b, data, mask), 0);
+   case nir_op_ixor: {
+      nir_def *count = nir_bit_count(b, nir_iand(b, data, mask));
+      return nir_ine_imm(b, nir_iand_imm(b, count, 1), 0);
+   }
+   default:
+      UNREACHABLE("Unsupported boolean reduction op");
+   }
+}
+
+static nir_def *
+build_identity(nir_builder *b, unsigned bit_size, nir_op op)
+{
+   nir_const_value ident_const = nir_alu_binop_identity(op, bit_size);
+   return nir_build_imm(b, 1, bit_size, &ident_const);
+}
+
+/* Implementation of scan/reduce that assumes a full subgroup */
+static nir_def *
+build_scan_full(nir_builder *b, nir_intrinsic_op op, nir_op red_op,
+                nir_def *data, unsigned cluster_size)
+{
+   switch (op) {
+   case nir_intrinsic_exclusive_scan:
+   case nir_intrinsic_inclusive_scan: {
+      for (unsigned i = 1; i < cluster_size; i *= 2) {
+         nir_def *idx = nir_load_subgroup_invocation(b);
+         nir_def *has_buddy = nir_ige_imm(b, idx, i);
+
+         nir_def *buddy_data = nir_shuffle_up(b, data, nir_imm_int(b, i));
+         nir_def *accum = nir_build_alu2(b, red_op, data, buddy_data);
+         data = nir_bcsel(b, has_buddy, accum, data);
+      }
+
+      if (op == nir_intrinsic_exclusive_scan) {
+         /* For exclusive scans, we need to shift one more time and fill in the
+          * bottom channel with identity.
+          */
+         assert(cluster_size == 32);
+         nir_def *idx = nir_load_subgroup_invocation(b);
+         nir_def *has_buddy = nir_ige_imm(b, idx, 1);
+
+         nir_def *buddy_data = nir_shuffle_up(b, data, nir_imm_int(b, 1));
+         nir_def *identity = build_identity(b, data->bit_size, red_op);
+         data = nir_bcsel(b, has_buddy, buddy_data, identity);
+      }
+
+      return data;
+   }
+
+   case nir_intrinsic_reduce: {
+      for (unsigned i = 1; i < cluster_size; i *= 2) {
+         nir_def *buddy_data = nir_shuffle_xor(b, data, nir_imm_int(b, i));
+         data = nir_build_alu2(b, red_op, data, buddy_data);
+      }
+      return data;
+   }
+
+   default:
+      UNREACHABLE("Unsupported scan/reduce op");
+   }
+}
+
+/* Fully generic implementation of scan/reduce that takes a mask */
+static nir_def *
+build_scan_reduce(nir_builder *b, nir_intrinsic_op op, nir_op red_op,
+                  nir_def *data, nir_def *mask, unsigned max_mask_bits)
+{
+   nir_def *lt_mask = nir_load_subgroup_lt_mask(b, 1, 32);
+
+   /* Mask of all channels whose values we need to accumulate.  Our own value
+    * is already in accum, if inclusive, thanks to the initialization above.
+    * We only need to consider lower indexed invocations.
+    */
+   nir_def *remaining = nir_iand(b, mask, lt_mask);
+
+   for (unsigned i = 1; i < max_mask_bits; i *= 2) {
+      /* At each step, our buddy channel is the first channel we have yet to
+       * take into account in the accumulator.
+       */
+      nir_def *has_buddy = nir_ine_imm(b, remaining, 0);
+      nir_def *buddy = nir_ufind_msb(b, remaining);
+
+      /* Accumulate with our buddy channel, if any */
+      nir_def *buddy_data = nir_shuffle(b, data, buddy);
+      nir_def *accum = nir_build_alu2(b, red_op, data, buddy_data);
+      data = nir_bcsel(b, has_buddy, accum, data);
+
+      /* We just took into account everything in our buddy's accumulator from
+       * the previous step.  The only things remaining are whatever channels
+       * were remaining for our buddy.
+       */
+      nir_def *buddy_remaining = nir_shuffle(b, remaining, buddy);
+      remaining = nir_bcsel(b, has_buddy, buddy_remaining, nir_imm_int(b, 0));
+   }
+
+   switch (op) {
+   case nir_intrinsic_exclusive_scan: {
+      /* For exclusive scans, we need to shift one more time and fill in the
+       * bottom channel with identity.
+       *
+       * Some of this will get CSE'd with the first step but that's okay. The
+       * code is cleaner this way.
+       */
+      nir_def *lower = nir_iand(b, mask, lt_mask);
+      nir_def *has_buddy = nir_ine_imm(b, lower, 0);
+      nir_def *buddy = nir_ufind_msb(b, lower);
+
+      nir_def *buddy_data = nir_shuffle(b, data, buddy);
+      nir_def *identity = build_identity(b, data->bit_size, red_op);
+      return nir_bcsel(b, has_buddy, buddy_data, identity);
+   }
+
+   case nir_intrinsic_inclusive_scan:
+      return data;
+
+   case nir_intrinsic_reduce: {
+      /* For reductions, we need to take the top value of the scan */
+      nir_def *idx = nir_ufind_msb(b, mask);
+      return nir_shuffle(b, data, idx);
+   }
+
+   default:
+      UNREACHABLE("Unsupported scan/reduce op");
+   }
+}
+
+static bool
+nak_nir_lower_scan_reduce_intrin(nir_builder *b,
+                                 nir_intrinsic_instr *intrin,
+                                 void *_nak)
+{
+   const struct nak_compiler *nak = (const struct nak_compiler *) _nak;
+
+   switch (intrin->intrinsic) {
+   case nir_intrinsic_exclusive_scan:
+   case nir_intrinsic_inclusive_scan:
+   case nir_intrinsic_reduce:
+      break;
+   default:
+      return false;
+   }
+
+   const nir_op red_op = nir_intrinsic_reduction_op(intrin);
+
+   /* Grab the cluster size, defaulting to 32 */
+   unsigned cluster_size = 32;
+   if (nir_intrinsic_has_cluster_size(intrin)) {
+      cluster_size = nir_intrinsic_cluster_size(intrin);
+      if (cluster_size == 0 || cluster_size > 32)
+         cluster_size = 32;
+   }
+
+   b->cursor = nir_before_instr(&intrin->instr);
+
+   nir_def *data;
+   if (cluster_size == 1) {
+      /* Simple case where we're not actually doing any reducing at all. */
+      assert(intrin->intrinsic == nir_intrinsic_reduce);
+      data = intrin->src[0].ssa;
+   } else if (intrin->intrinsic == nir_intrinsic_reduce &&
+              nak->sm >= 80 &&
+              red_op != nir_op_imul &&
+              nir_op_infos[red_op].output_type != nir_type_float &&
+              intrin->src[0].ssa->bit_size == 32 &&
+              cluster_size == 32 &&
+              !nak_block_is_divergent(intrin->instr.block)) {
+      /* TODO: We could probably also use REDUX for the non-uniform case if we
+       *       were allowed to write uregs from non-uniform control flow.
+       */
+      return false;
+   } else if (intrin->src[0].ssa->bit_size == 1) {
+      data = build_scan_bool(b, intrin->intrinsic, red_op,
+                             intrin->src[0].ssa, cluster_size);
+   } else {
+      /* First, we need a mask of all invocations to be included in the
+       * reduction or scan.  For trivial cluster sizes, that's just the mask
+       * of enabled channels.
+       */
+      nir_def *mask = cluster_mask(b, cluster_size);
+
+      nir_def *full, *partial;
+      nir_push_if(b, nir_ieq_imm(b, mask, -1));
+      {
+         full = build_scan_full(b, intrin->intrinsic, red_op,
+                                intrin->src[0].ssa, cluster_size);
+      }
+      nir_push_else(b, NULL);
+      {
+         partial = build_scan_reduce(b, intrin->intrinsic, red_op,
+                                     intrin->src[0].ssa, mask, cluster_size);
+      }
+      nir_pop_if(b, NULL);
+      data = nir_if_phi(b, full, partial);
+   }
+
+   nir_def_replace(&intrin->def, data);
+
+   return true;
+}
+
+bool
+nak_nir_lower_scan_reduce(nir_shader *nir, const struct nak_compiler *nak)
+{
+   return nir_shader_intrinsics_pass(nir, nak_nir_lower_scan_reduce_intrin,
+                                     nir_metadata_none, (void*) nak);
+}
@@ -0,0 +1,187 @@
+/*
+ * Copyright © 2026 Valve Corporation.
+ * Copyright © 2025 Lorenzo Rossi
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "nak_private.h"
+#include "nir.h"
+#include "nir_builder.h"
+
+/*
+ * Convert atomic arithmetic to regular arithmetic along with mutex locks.
+ *
+ * eg:
+ * atomicAdd(addr, 1) ->
+ *
+ * uint expected = a[0];
+ * bool success = false;
+ * do {
+ *    data, is_locked = load_locked(a[0])
+ *    if (is_locked) {
+ *       data = data + 1;
+ *       success = store_and_unlock(&a[0], data);
+ *    }
+ * } while (!success);
+ *
+ * special_case cmp_exc and exc.
+ */
+
+static nir_def *
+lower_atomic_op(nir_builder *b, nir_intrinsic_instr *intr, nir_def *loaded)
+{
+   // Assume we have the lock, the previous value is in loaded and we must
+   // compute the value to store in the address.
+   // to_store = op(loaded, data)
+   nir_def *data = intr->src[1].ssa;
+   nir_def *to_store;
+
+   switch (nir_intrinsic_atomic_op(intr)) {
+   case nir_atomic_op_imin:
+   case nir_atomic_op_umin:
+   case nir_atomic_op_imax:
+   case nir_atomic_op_umax:
+   case nir_atomic_op_iand:
+   case nir_atomic_op_ior:
+   case nir_atomic_op_ixor:
+   case nir_atomic_op_fadd:
+   case nir_atomic_op_fmin:
+   case nir_atomic_op_fmax:
+   case nir_atomic_op_iadd: {
+      to_store = nir_build_alu2(
+         b, nir_atomic_op_to_alu(nir_intrinsic_atomic_op(intr)), loaded, data);
+      nir_alu_instr *alu = nir_def_as_alu(to_store);
+      alu->fp_math_ctrl = nir_op_valid_fp_math_ctrl(alu->op, nir_fp_no_fast_math);
+      break;
+   }
+   case nir_atomic_op_xchg: {
+      // op(loaded, data) = data
+      to_store = data;
+      break;
+   }
+   case nir_atomic_op_cmpxchg: {
+      // op(loaded, src1, src2) = loaded == src1 ? src2 : loaded;
+      nir_def *new_data = intr->src[2].ssa;
+      to_store = nir_bcsel(b, nir_ieq(b, loaded, data), new_data, loaded);
+      break;
+   }
+   case nir_atomic_op_fcmpxchg: /* TODO: shared atomic floats */
+   default:
+      UNREACHABLE("Invalid intrinsic");
+   }
+
+   return to_store;
+}
+
+static nir_def *
+build_kepler_atomic(nir_builder *b, nir_intrinsic_instr *intr)
+{
+   // TODO: this is currently compiled down to ~20 instructions while
+   //       CUDA can optimize the same code to only ~5.
+   nir_def *loaded_data;
+   nir_def *addr = intr->src[0].ssa;
+
+   nir_loop *loop = nir_push_loop(b);
+   {
+      nir_def *load = nir_load_shared_lock_nv(b, intr->def.bit_size, addr);
+
+      loaded_data = nir_channel(b, load, 0);
+      nir_def *is_locked = nir_u2u32(b, nir_channel(b, load, 1));
+      nir_if *nif = nir_push_if(b, nir_ine_imm(b, is_locked, 0));
+      {
+         nir_def *new_data = lower_atomic_op(b, intr, loaded_data);
+         nir_def *success = nir_store_shared_unlock_nv(b, 32, new_data, addr);
+
+         nir_break_if(b, nir_ine_imm(b, success, 0));
+      }
+      nir_pop_if(b, nif);
+   }
+   nir_pop_loop(b, loop);
+   return loaded_data;
+}
+
+static bool
+nak_nir_lower_kepler_atomics_intrin(nir_builder *b,
+                                 nir_intrinsic_instr *intrin,
+                                 UNUSED void *_data)
+{
+   if (intrin->intrinsic != nir_intrinsic_shared_atomic &&
+       intrin->intrinsic != nir_intrinsic_shared_atomic_swap)
+      return false;
+
+   b->cursor = nir_before_instr(&intrin->instr);
+   nir_def_replace(&intrin->def, build_kepler_atomic(b, intrin));
+   return true;
+}
+
+bool
+nak_nir_lower_kepler_shared_atomics(nir_shader *nir)
+{
+   return nir_shader_intrinsics_pass(nir, nak_nir_lower_kepler_atomics_intrin,
+                                     nir_metadata_none, NULL);
+}
+
+static nir_def *
+build_mesh_atomic(nir_builder *b, nir_intrinsic_instr *intrin)
+{
+   nir_def *current_value;
+   nir_def *offset = intrin->src[0].ssa;
+   nir_def *current_invocation = nir_load_subgroup_invocation(b);
+
+   /* Basic spin lock implementation */
+   nir_loop *loop = nir_push_loop(b);
+   {
+      /* First we check what active threads match our offset value */
+      nir_def *active_thread_mask = nir_match_any_nv(b, 32, offset);
+
+      /* Then we elect a thread to work in this loop iteration */
+      nir_def *elected_thread = nir_ufind_msb(b, active_thread_mask);
+
+      /* Check if the current invocation won and if so do the operation */
+      nir_if *if_body =
+         nir_push_if(b, nir_ieq(b, elected_thread, current_invocation));
+      {
+         current_value = nir_load_shared(b, 1, intrin->def.bit_size, offset,
+                                         .base = nir_intrinsic_base(intrin));
+         nir_def *new_value = lower_atomic_op(b, intrin, current_value);
+         nir_store_shared(b, new_value, offset,
+                          .base = nir_intrinsic_base(intrin));
+         nir_jump(b, nir_jump_break);
+      }
+      nir_pop_if(b, if_body);
+   }
+   nir_pop_loop(b, loop);
+
+   return current_value;
+}
+
+static bool
+nak_nir_lower_mesh_stages_shared_atomics_intrin(nir_builder *b,
+                                                nir_intrinsic_instr *intrin,
+                                                UNUSED void *_data)
+{
+   if (intrin->intrinsic != nir_intrinsic_shared_atomic &&
+       intrin->intrinsic != nir_intrinsic_shared_atomic_swap)
+      return false;
+
+   b->cursor = nir_before_instr(&intrin->instr);
+   nir_def_replace(&intrin->def, build_mesh_atomic(b, intrin));
+   return true;
+}
+
+bool
+nak_nir_lower_mesh_stages_shared_atomics(nir_shader *nir)
+{
+   if (!mesa_shader_stage_is_mesh(nir->info.stage))
+      return false;
+
+   /* The local workgroup should have been lowered to a single subgroup. */
+   ASSERTED uint16_t wg_sz = nir->info.workgroup_size[0] *
+                             nir->info.workgroup_size[1] *
+                             nir->info.workgroup_size[2];
+   assert(wg_sz <= NAK_SUBGROUP_SIZE);
+
+   return nir_shader_intrinsics_pass(
+      nir, nak_nir_lower_mesh_stages_shared_atomics_intrin, nir_metadata_none,
+      NULL);
+}
@@ -0,0 +1,939 @@
+/*
+ * Copyright © 2023 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "nak_private.h"
+#include "nir_builder.h"
+#include "nir_format_convert.h"
+
+#include "util/u_math.h"
+
+static bool
+has_null_descriptors(const struct nak_compiler *nak) {
+   /* We only have "real" null descriptors on Volta+ */
+   return nak->sm >= 70;
+}
+
+static bool
+has_cbuf_tex(const struct nak_compiler *nak) {
+   /* TODO: Figure out how bound textures work on blackwell */
+   return nak->sm >= 70 && nak->sm < 100;
+}
+
+static bool
+tex_handle_as_cbuf(nir_def *tex_h, uint32_t *cbuf_out)
+{
+   if (!nir_def_is_intrinsic(tex_h))
+      return false;
+
+   nir_intrinsic_instr *intrin = nir_def_as_intrinsic(tex_h);
+   if (intrin->intrinsic != nir_intrinsic_ldc_nv)
+      return false;
+
+   if (!nir_src_is_const(intrin->src[1]))
+      return false;
+
+   uint32_t idx = nir_src_as_uint(intrin->src[0]);
+   uint32_t offset = nir_src_as_uint(intrin->src[1]);
+   assert(idx < (1 << 5) && offset < (1 << 16));
+   *cbuf_out = (idx << 16) | offset;
+
+   return true;
+}
+
+static nir_def *
+build_txq(nir_builder *b, nir_texop op, nir_def *img_h, nir_def *lod_idx,
+          bool can_speculate, const struct nak_compiler *nak)
+{
+   uint32_t texture_index = 0;
+   enum nak_nir_tex_ref_type ref_type = NAK_NIR_TEX_REF_TYPE_BINDLESS;
+   if (has_cbuf_tex(nak) && tex_handle_as_cbuf(img_h, &texture_index)) {
+      ref_type = NAK_NIR_TEX_REF_TYPE_CBUF;
+      img_h = NULL;
+   }
+
+   nir_tex_instr *txq = nir_tex_instr_create(b->shader, 1);
+   txq->op = op,
+
+   /* txq doesn't take any dimension parameters so we just smash in 2D because
+    * NIR needs us to set something.  And using the same dimensionality for
+    * everything gives us maximum CSE.
+    */
+   txq->sampler_dim = GLSL_SAMPLER_DIM_2D,
+   txq->is_array = false,
+   txq->dest_type = nir_type_int32;
+   txq->can_speculate = can_speculate;
+
+   nir_def *src[2] = { NULL, };
+   unsigned src_comps = 0;
+
+   if (img_h != NULL)
+      src[src_comps++] = img_h;
+   if (lod_idx != NULL)
+      src[src_comps++] = lod_idx;
+
+   if (src_comps == 0)
+      src[src_comps++] = nir_imm_int(b, 0);
+   nir_def *txq_src = nir_vec(b, src, src_comps);
+
+   txq->src[0] = (nir_tex_src) {
+      .src_type = nir_tex_src_backend1,
+      .src = nir_src_for_ssa(txq_src),
+   };
+   txq->texture_index = texture_index;
+
+   const struct nak_nir_tex_flags flags = {
+      .ref_type = ref_type,
+   };
+   txq->backend_flags = NAK_AS_U32(flags);
+
+   nir_def_init(&txq->instr, &txq->def, 4, 32);
+   nir_builder_instr_insert(b, &txq->instr);
+
+   return &txq->def;
+}
+
+static nir_def *
+build_txq_levels(nir_builder *b, nir_def *img_h, bool can_speculate,
+                 const struct nak_compiler *nak)
+{
+   nir_def *res = build_txq(b, nir_texop_hdr_dim_nv, img_h, nir_imm_int(b, 0),
+                            can_speculate, nak);
+   return nir_channel(b, res, 3);
+}
+
+static nir_def *
+build_img_is_null(nir_builder *b, nir_def *img_h, bool can_speculate,
+                  const struct nak_compiler *nak)
+{
+   /* Prior to Volta, we don't have real NULL descriptors but we can figure
+    * out if it's null based on the number of levels returned by
+    * txq.dimension.
+    */
+   return nir_ieq_imm(b, build_txq_levels(b, img_h, can_speculate, nak), 0);
+}
+
+static enum glsl_sampler_dim
+remap_sampler_dim(enum glsl_sampler_dim dim)
+{
+   switch (dim) {
+   case GLSL_SAMPLER_DIM_SUBPASS: return GLSL_SAMPLER_DIM_2D;
+   case GLSL_SAMPLER_DIM_SUBPASS_MS: return GLSL_SAMPLER_DIM_MS;
+   default: return dim;
+   }
+}
+
+static bool
+texop_supports_scalar(nir_tex_instr *tex, const struct nak_compiler *nak)
+{
+   if (nak->sm < 70)
+      return false;
+
+   switch (tex->op) {
+   case nir_texop_tex:
+   case nir_texop_tg4:
+   case nir_texop_txb:
+   case nir_texop_txf:
+   case nir_texop_txf_ms:
+   case nir_texop_txl:
+      return true;
+   default:
+      return false;
+   }
+}
+
+static bool
+try_scalarize_tex(nir_tex_instr *tex, const struct nak_compiler *nak,
+                  nir_def **src0, unsigned *src0_comps,
+                  nir_def **src1, unsigned *src1_comps)
+{
+   if (!texop_supports_scalar(tex, nak))
+      return false;
+
+   /* for SM70+ we simply rebalance the sources if there are 4 or less
+    * TODO: it might be worth to rebalance 3 + 1 or 1 + 3 combinations,
+    *       but so far this seems to be a net negative.
+    */
+   if (nak->sm >= 70 && *src1_comps == 0) {
+      assert(*src0_comps <= 4);
+      switch (*src0_comps) {
+      case 4:
+         src1[0] = src0[2];
+         src1[1] = src0[3];
+         *src0_comps = 2;
+         *src1_comps = 2;
+         return true;
+      case 3:
+         src1[0] = src0[2];
+         *src0_comps = 2;
+         *src1_comps = 1;
+         return true;
+      case 2:
+         src1[0] = src0[1];
+         *src0_comps = 1;
+         *src1_comps = 1;
+         return true;
+      case 1:
+         /* Nothing to do */
+         return true;
+      default:
+         UNREACHABLE("unexpected src0 comps");
+         return false;
+      }
+   }
+
+   return false;
+}
+
+static bool
+lower_tex(nir_builder *b, nir_tex_instr *tex, const struct nak_compiler *nak)
+{
+   b->cursor = nir_before_instr(&tex->instr);
+
+   nir_def *tex_h = NULL, *samp_h = NULL, *coord = NULL, *ms_idx = NULL;
+   nir_def *offset = NULL, *lod = NULL, *bias = NULL, *min_lod = NULL;
+   nir_def *ddx = NULL, *ddy = NULL, *z_cmpr = NULL;
+   for (unsigned i = 0; i < tex->num_srcs; i++) {
+      switch (tex->src[i].src_type) {
+      case nir_tex_src_texture_handle: tex_h =     tex->src[i].src.ssa; break;
+      case nir_tex_src_sampler_handle: samp_h =    tex->src[i].src.ssa; break;
+      case nir_tex_src_coord:          coord =     tex->src[i].src.ssa; break;
+      case nir_tex_src_ms_index:       ms_idx =    tex->src[i].src.ssa; break;
+      case nir_tex_src_comparator:     z_cmpr =    tex->src[i].src.ssa; break;
+      case nir_tex_src_offset:         offset =    tex->src[i].src.ssa; break;
+      case nir_tex_src_lod:            lod =       tex->src[i].src.ssa; break;
+      case nir_tex_src_bias:           bias =      tex->src[i].src.ssa; break;
+      case nir_tex_src_min_lod:        min_lod =   tex->src[i].src.ssa; break;
+      case nir_tex_src_ddx:            ddx =       tex->src[i].src.ssa; break;
+      case nir_tex_src_ddy:            ddy =       tex->src[i].src.ssa; break;
+      default:
+         UNREACHABLE("Unsupported texture source");
+      }
+      /* Remove sources as we walk them.  We'll add them back later */
+      nir_instr_clear_src(&tex->instr, &tex->src[i].src);
+   }
+   tex->num_srcs = 0;
+
+   /* Combine sampler and texture into one if needed */
+   if (samp_h != NULL && samp_h != tex_h) {
+      tex_h = nir_ior(b, nir_iand_imm(b, tex_h,  0x000fffff),
+                         nir_iand_imm(b, samp_h, 0xfff00000));
+   }
+
+   enum nak_nir_tex_ref_type ref_type = NAK_NIR_TEX_REF_TYPE_BINDLESS;
+   if (has_cbuf_tex(nak) && tex_handle_as_cbuf(tex_h, &tex->texture_index)) {
+      ref_type = NAK_NIR_TEX_REF_TYPE_CBUF;
+      tex_h = NULL;
+   }
+
+   /* Array index is treated separately, so pull it off if we have one. */
+   nir_def *arr_idx = NULL;
+   unsigned coord_components = tex->coord_components;
+   if (coord && tex->is_array) {
+      if (tex->op == nir_texop_lod) {
+         /* The HW wants an array index. Use zero. */
+         arr_idx = nir_imm_int(b, 0);
+      } else {
+         arr_idx = nir_channel(b, coord, --coord_components);
+
+         /* Everything but texelFetch takes a float index
+          *
+          * TODO: Use F2I.U32.RNE
+          */
+         if (tex->op != nir_texop_txf && tex->op != nir_texop_txf_ms) {
+            arr_idx = nir_fadd_imm(b, arr_idx, 0.5);
+
+            // TODO: Hardware seems to clamp negative values to zero for us
+            // in f2u, but we still need this fmax for constant folding.
+            arr_idx = nir_fmax(b, arr_idx, nir_imm_float(b, 0.0));
+
+            arr_idx = nir_f2u32(b, arr_idx);
+         }
+
+         arr_idx = nir_umin(b, arr_idx, nir_imm_int(b, UINT16_MAX));
+      }
+   }
+
+   enum nak_nir_lod_mode lod_mode = NAK_NIR_LOD_MODE_AUTO;
+   if (tex->op == nir_texop_txf_ms) {
+      /* Multisampled textures do not have miplevels */
+      lod_mode = NAK_NIR_LOD_MODE_ZERO;
+      lod = NULL; /* We don't need this */
+   } else if (lod != NULL) {
+      nir_scalar lod_s = { .def = lod, .comp = 0 };
+      if (nir_scalar_is_const(lod_s) &&
+          nir_scalar_as_uint(lod_s) == 0) {
+         lod_mode = NAK_NIR_LOD_MODE_ZERO;
+         lod = NULL; /* We don't need this */
+      } else {
+         lod_mode = NAK_NIR_LOD_MODE_LOD;
+      }
+   } else if (bias != NULL) {
+      lod_mode = NAK_NIR_LOD_MODE_BIAS;
+      lod = bias;
+   }
+
+   if (min_lod != NULL) {
+      switch (lod_mode) {
+      case NAK_NIR_LOD_MODE_AUTO:
+         lod_mode = NAK_NIR_LOD_MODE_CLAMP;
+         break;
+      case NAK_NIR_LOD_MODE_BIAS:
+         lod_mode = NAK_NIR_LOD_MODE_BIAS_CLAMP;
+         break;
+      default:
+         UNREACHABLE("Invalid min_lod");
+      }
+      min_lod = nir_f2u32(b, nir_fmax(b, nir_fmul_imm(b, min_lod, 256),
+                                         nir_imm_float(b, 16)));
+   }
+
+   enum nak_nir_offset_mode offset_mode = NAK_NIR_OFFSET_MODE_NONE;
+   if (offset != NULL) {
+      /* For TG4, offsets, are packed into a single 32-bit value with 8 bits
+       * per component.  For all other texture instructions, offsets are
+       * packed into a single at most 16-bit value with 8 bits per component.
+       */
+      static const unsigned bits4[] = { 4, 4, 4, 4 };
+      static const unsigned bits8[] = { 8, 8, 8, 8 };
+      const unsigned *bits = tex->op == nir_texop_tg4 ? bits8 : bits4;
+
+      offset = nir_pad_vector_imm_int(b, offset, 0, 4);
+      offset = nir_format_clamp_sint(b, offset, bits);
+      offset = nir_format_pack_uint(b, offset, bits, 4);
+      offset_mode = NAK_NIR_OFFSET_MODE_AOFFI;
+   } else if (nir_tex_instr_has_explicit_tg4_offsets(tex)) {
+      uint64_t off_u64 = 0;
+      for (uint8_t i = 0; i < 8; ++i) {
+         uint64_t off = (uint8_t)tex->tg4_offsets[i / 2][i % 2];
+         off_u64 |= off << (i * 8);
+      }
+      offset = nir_imm_ivec2(b, off_u64, off_u64 >> 32);
+      offset_mode = NAK_NIR_OFFSET_MODE_PER_PX;
+   }
+
+#define PUSH(a, x) do { \
+   nir_def *val = (x); \
+   assert(a##_comps < ARRAY_SIZE(a)); \
+   a[a##_comps++] = val; \
+} while(0)
+
+   bool scalar = false;
+   if (nak->sm >= 50) {
+      nir_def *src0[4] = { NULL, };
+      nir_def *src1[4] = { NULL, };
+      unsigned src0_comps = 0, src1_comps = 0;
+
+      if (tex->op == nir_texop_txd) {
+         if (tex_h != NULL)
+            PUSH(src0, tex_h);
+
+         for (uint32_t i = 0; i < coord_components; i++)
+            PUSH(src0, nir_channel(b, coord, i));
+
+         if (offset != NULL) {
+            nir_def *arr_idx_or_zero = arr_idx ? arr_idx : nir_imm_int(b, 0);
+            nir_def *arr_off = nir_prmt_nv(b, nir_imm_int(b, 0x1054),
+                                           offset, arr_idx_or_zero);
+            PUSH(src0, arr_off);
+         } else if (arr_idx != NULL) {
+            PUSH(src0, arr_idx);
+         }
+
+         assert(ddx->num_components == coord_components);
+         for (uint32_t i = 0; i < coord_components; i++) {
+            PUSH(src1, nir_channel(b, ddx, i));
+            PUSH(src1, nir_channel(b, ddy, i));
+         }
+      } else {
+         if (min_lod != NULL) {
+            nir_def *arr_idx_or_zero = arr_idx ? arr_idx : nir_imm_int(b, 0);
+            nir_def *arr_ml = nir_prmt_nv(b, nir_imm_int(b, 0x1054),
+                                          min_lod, arr_idx_or_zero);
+            PUSH(src0, arr_ml);
+         } else if (arr_idx != NULL) {
+            PUSH(src0, arr_idx);
+         }
+
+         for (uint32_t i = 0; i < coord_components; i++)
+            PUSH(src0, nir_channel(b, coord, i));
+
+         if (tex_h != NULL)
+            PUSH(src1, tex_h);
+         if (lod != NULL)
+            PUSH(src1, lod);
+         if (offset_mode == NAK_NIR_OFFSET_MODE_AOFFI) {
+            PUSH(src1, offset);
+         } else if (offset_mode == NAK_NIR_OFFSET_MODE_PER_PX) {
+            PUSH(src1, nir_channel(b, offset, 0));
+            PUSH(src1, nir_channel(b, offset, 1));
+         }
+         if (z_cmpr != NULL)
+            PUSH(src1, z_cmpr);
+         if (ms_idx != NULL)
+            PUSH(src1, ms_idx);
+      }
+
+      scalar = try_scalarize_tex(tex, nak, src0, &src0_comps, src1, &src1_comps);
+
+      nir_tex_instr_add_src(tex, nir_tex_src_backend1,
+                            nir_vec(b, src0, src0_comps));
+
+      if (src1_comps > 0) {
+         nir_tex_instr_add_src(tex, nir_tex_src_backend2,
+                               nir_vec(b, src1, src1_comps));
+      }
+   } else if (nak->sm >= 30) {
+      nir_def *src[8] = { NULL, };
+      unsigned src_comps = 0;
+
+      if (tex_h != NULL)
+         PUSH(src, tex_h);
+
+      if (offset != NULL && tex->op == nir_texop_txd) {
+         nir_def *arr_idx_or_zero = arr_idx ? arr_idx : nir_imm_int(b, 0);
+         // TODO: This may be backwards?
+         nir_def *arr_off = nir_prmt_nv(b, nir_imm_int(b, 0x1054),
+                                        offset, arr_idx_or_zero);
+         PUSH(src, arr_off);
+      } else if (arr_idx != NULL) {
+         PUSH(src, arr_idx);
+      }
+
+      for (uint32_t i = 0; i < coord_components; i++)
+         PUSH(src, nir_channel(b, coord, i));
+
+      if (ms_idx != NULL)
+         PUSH(src, ms_idx);
+      if (lod != NULL)
+         PUSH(src, lod);
+
+      if (tex->op != nir_texop_txd) {
+         if (offset_mode == NAK_NIR_OFFSET_MODE_AOFFI) {
+            PUSH(src, offset);
+         } else if (offset_mode == NAK_NIR_OFFSET_MODE_PER_PX) {
+            PUSH(src, nir_channel(b, offset, 0));
+            PUSH(src, nir_channel(b, offset, 1));
+         }
+      }
+
+      if (z_cmpr != NULL)
+         PUSH(src, z_cmpr);
+
+      if (tex->op == nir_texop_txd) {
+         assert(ddx->num_components == coord_components);
+         for (uint32_t i = 0; i < coord_components; i++) {
+            PUSH(src, nir_channel(b, ddx, i));
+            PUSH(src, nir_channel(b, ddy, i));
+         }
+      }
+
+      /* Both sources are vec4s so we need an even multiple of 4 */
+      while (src_comps % 4)
+         PUSH(src, nir_undef(b, 1, 32));
+
+      nir_tex_instr_add_src(tex, nir_tex_src_backend1,
+                            nir_vec(b, src, 4));
+      if (src_comps > 4) {
+         nir_tex_instr_add_src(tex, nir_tex_src_backend2,
+                               nir_vec(b, src + 4, 4));
+      }
+   } else {
+      UNREACHABLE("Unsupported shader model");
+   }
+
+   tex->sampler_dim = remap_sampler_dim(tex->sampler_dim);
+
+   struct nak_nir_tex_flags flags = {
+      .ref_type = ref_type,
+      .lod_mode = lod_mode,
+      .offset_mode = offset_mode,
+      .has_z_cmpr = tex->is_shadow,
+      .is_sparse = tex->is_sparse,
+      .nodep = tex->skip_helpers,
+      .scalar = scalar,
+   };
+   STATIC_ASSERT(sizeof(flags) == sizeof(tex->backend_flags));
+   memcpy(&tex->backend_flags, &flags, sizeof(flags));
+
+   if (tex->op == nir_texop_lod) {
+      b->cursor = nir_after_instr(&tex->instr);
+
+      /* The outputs are flipped compared to what NIR expects */
+      nir_def *abs = nir_channel(b, &tex->def, 1);
+      nir_def *rel = nir_channel(b, &tex->def, 0);
+
+      /* The returned values are not quite what we want:
+       * (a) convert from s16/u16 to f32
+       * (b) multiply by 1/256
+       *
+       * TODO: We can make this cheaper once we have 16-bit in NAK
+       */
+      abs = nir_u2f32(b, nir_iand_imm(b, abs, 0xffff));
+      nir_def *shift = nir_imm_int(b, 16);
+      rel = nir_i2f32(b, nir_ishr(b, nir_ishl(b, rel, shift), shift));
+
+      abs = nir_fmul_imm(b, abs, 1.0 / 256.0);
+      rel = nir_fmul_imm(b, rel, 1.0 / 256.0);
+
+      nir_def *res = nir_vec2(b, abs, rel);
+
+      if (!has_null_descriptors(nak)) {
+         nir_def *img_is_null =
+            build_img_is_null(b, tex_h, tex->can_speculate, nak);
+         res = nir_bcsel(b, img_is_null, nir_imm_int(b, 0), res);
+      }
+
+      nir_def_rewrite_uses_after(&tex->def, res);
+   }
+
+   return true;
+}
+
+static nir_def *
+build_txq_samples_raw(nir_builder *b, nir_def *img_h, bool can_speculate,
+                      const struct nak_compiler *nak)
+{
+   nir_def *res = build_txq(b, nir_texop_tex_type_nv, img_h, NULL,
+                            can_speculate, nak);
+   return nir_channel(b, res, 2);
+}
+
+static nir_def *
+build_txq_samples(nir_builder *b, nir_def *img_h, bool can_speculate,
+                  const struct nak_compiler *nak)
+{
+   nir_def *res = build_txq_samples_raw(b, img_h, can_speculate, nak);
+
+   if (!has_null_descriptors(nak)) {
+      res = nir_bcsel(b, build_img_is_null(b, img_h, can_speculate, nak),
+                      nir_imm_int(b, 0), res);
+   }
+
+   return res;
+}
+
+static nir_def *
+build_txq_size(nir_builder *b, unsigned num_components,
+               nir_def *img_h, nir_def *lod, bool can_speculate,
+               const struct nak_compiler *nak)
+{
+   if (lod == NULL)
+      lod = nir_imm_int(b, 0);
+
+   nir_def *res = build_txq(b, nir_texop_hdr_dim_nv, img_h, lod,
+                            can_speculate, nak);
+   res = nir_trim_vector(b, res, num_components);
+
+   if (!has_null_descriptors(nak)) {
+      res = nir_bcsel(b, build_img_is_null(b, img_h, can_speculate, nak),
+                      nir_imm_int(b, 0), res);
+   }
+
+   return res;
+}
+
+static bool
+lower_txq(nir_builder *b, nir_tex_instr *tex, const struct nak_compiler *nak)
+{
+   b->cursor = nir_before_instr(&tex->instr);
+
+   assert(!tex->is_sparse);
+
+   nir_def *tex_h = NULL, *lod = NULL;
+   for (unsigned i = 0; i < tex->num_srcs; i++) {
+      switch (tex->src[i].src_type) {
+      case nir_tex_src_texture_handle: tex_h = tex->src[i].src.ssa; break;
+      case nir_tex_src_sampler_handle: break; /* Ignored */
+      case nir_tex_src_lod:            lod = tex->src[i].src.ssa; break;
+      default:
+         UNREACHABLE("Unsupported texture source");
+      }
+   }
+
+   nir_def *res;
+   switch (tex->op) {
+   case nir_texop_txs:
+      res = build_txq_size(b, tex->def.num_components, tex_h, lod,
+                           tex->can_speculate, nak);
+      break;
+   case nir_texop_query_levels:
+      res = build_txq_levels(b, tex_h, tex->can_speculate, nak);
+      break;
+   case nir_texop_texture_samples:
+      res = build_txq_samples(b, tex_h, tex->can_speculate, nak);
+      break;
+   default:
+      UNREACHABLE("Invalid texture query op");
+   }
+
+   nir_def_replace(&tex->def, res);
+
+   return true;
+}
+
+static bool
+shrink_image_load(nir_builder *b, nir_intrinsic_instr *intrin,
+                  const struct nak_compiler *nak)
+{
+   enum pipe_format format = nir_intrinsic_format(intrin);
+   nir_component_mask_t color_comps_read =
+      nir_def_components_read(&intrin->def);
+
+   assert(intrin->intrinsic == nir_intrinsic_bindless_image_load ||
+          intrin->intrinsic == nir_intrinsic_bindless_image_sparse_load);
+
+   /* Pick off the sparse resident component (if any) before we do anything
+    * else.  This makes later logic easier.
+    */
+   bool is_sparse = false;
+   if (intrin->intrinsic == nir_intrinsic_bindless_image_sparse_load) {
+      unsigned resident_comp = intrin->def.num_components - 1;
+      if (color_comps_read & BITFIELD_BIT(resident_comp)) {
+         is_sparse = true;
+         color_comps_read &= ~BITFIELD_BIT(resident_comp);
+      } else {
+         /* If the sparse bit is never used, get rid of it */
+         intrin->intrinsic = nir_intrinsic_bindless_image_load;
+         intrin->num_components--;
+         intrin->def.num_components--;
+      }
+   }
+
+   if (intrin->def.bit_size == 64) {
+      assert(format == PIPE_FORMAT_NONE ||
+             format == PIPE_FORMAT_R64_UINT ||
+             format == PIPE_FORMAT_R64_SINT);
+
+      b->cursor = nir_after_instr(&intrin->instr);
+
+      nir_def *data_xy, *data_w, *resident = NULL;
+      if (color_comps_read & BITFIELD_BIT(3)) {
+         /* Thanks to descriptor indexing, we need to ensure that null
+          * descriptor behavior works properly.  In particular, normal zero
+          * reads will return (0, 0, 0, 1) whereas null descriptor reads need
+          * to return (0, 0, 0, 0).  This means we can't blindly extend with
+          * an alpha component of 1.  Instead, we need to trust the hardware
+          * to extend the original RG32 with z = 0 and w = 1 and copy the w
+          * value all the way out to 64-bit w value.
+          */
+         assert(intrin->num_components == 4 + is_sparse);
+         assert(intrin->def.num_components == 4 + is_sparse);
+         intrin->def.bit_size = 32;
+
+         data_xy = nir_channels(b, &intrin->def, 0x3);
+         data_w = nir_channels(b, &intrin->def, 0x8);
+         if (is_sparse)
+            resident = nir_channel(b, &intrin->def, 4);
+      } else {
+         intrin->num_components = 2 + is_sparse;
+         intrin->def.num_components = 2 + is_sparse;
+         intrin->def.bit_size = 32;
+
+         data_xy = nir_channels(b, &intrin->def, 0x3);
+         data_w = nir_imm_int(b, 0);
+         if (is_sparse)
+            resident = nir_channel(b, &intrin->def, 2);
+      }
+
+      nir_def *data;
+      if (is_sparse) {
+         data = nir_vec5(b, nir_pack_64_2x32(b, data_xy),
+                         nir_imm_zero(b, 1, 64),
+                         nir_imm_zero(b, 1, 64),
+                         nir_u2u64(b, data_w),
+                         nir_u2u64(b, resident));
+      } else {
+         data = nir_vec4(b, nir_pack_64_2x32(b, data_xy),
+                         nir_imm_zero(b, 1, 64),
+                         nir_imm_zero(b, 1, 64),
+                         nir_u2u64(b, data_w));
+      }
+
+      nir_def_rewrite_uses_after(&intrin->def, data);
+      return true;
+   }
+
+   if (format == PIPE_FORMAT_NONE)
+      return false;
+
+   /* In order for null descriptors to work properly, we don't want to shrink
+    * loads when the alpha channel is read even if we know the format has
+    * fewer channels.
+    */
+   if (color_comps_read & BITFIELD_BIT(3))
+      return false;
+
+   const unsigned old_comps = intrin->def.num_components;
+
+   unsigned new_comps = util_format_get_nr_components(format);
+   new_comps = util_next_power_of_two(new_comps);
+   if (color_comps_read <= BITFIELD_MASK(2))
+      new_comps = 2;
+   if (color_comps_read <= BITFIELD_MASK(1))
+      new_comps = 1;
+
+   if (new_comps + is_sparse >= intrin->num_components)
+      return false;
+
+   b->cursor = nir_after_instr(&intrin->instr);
+
+   intrin->num_components = new_comps + is_sparse;
+   intrin->def.num_components = new_comps + is_sparse;
+
+   assert(new_comps <= 4);
+   nir_def *comps[5];
+   for (unsigned c = 0; c < new_comps; c++)
+      comps[c] = nir_channel(b, &intrin->def, c);
+   for (unsigned c = new_comps; c < 3; c++)
+      comps[c] = nir_imm_intN_t(b, 0, intrin->def.bit_size);
+   if (new_comps < 4)
+      comps[3] = nir_imm_intN_t(b, 1, intrin->def.bit_size);
+
+   /* The resident bit always goes in the last channel */
+   if (is_sparse)
+      comps[old_comps - 1] = nir_channel(b, &intrin->def, new_comps);
+
+   nir_def *data = nir_vec(b, comps, old_comps);
+   nir_def_rewrite_uses_after(&intrin->def, data);
+   return true;
+}
+
+static bool
+shrink_image_store(nir_builder *b, nir_intrinsic_instr *intrin,
+                  const struct nak_compiler *nak)
+{
+   enum pipe_format format = nir_intrinsic_format(intrin);
+   nir_def *data = intrin->src[3].ssa;
+
+   if (data->bit_size == 64) {
+      assert(format == PIPE_FORMAT_NONE ||
+             format == PIPE_FORMAT_R64_UINT ||
+             format == PIPE_FORMAT_R64_SINT);
+
+      b->cursor = nir_before_instr(&intrin->instr);
+
+      /* For 64-bit image ops, we actually want a vec2 */
+      nir_def *data_vec2 = nir_unpack_64_2x32(b, nir_channel(b, data, 0));
+      nir_src_rewrite(&intrin->src[3], data_vec2);
+      intrin->num_components = 2;
+      return true;
+   }
+
+   if (format == PIPE_FORMAT_NONE)
+      return false;
+
+   unsigned new_comps = util_format_get_nr_components(format);
+   new_comps = util_next_power_of_two(new_comps);
+   if (new_comps >= intrin->num_components)
+      return false;
+
+   b->cursor = nir_before_instr(&intrin->instr);
+
+   nir_def *trimmed = nir_trim_vector(b, data, new_comps);
+   nir_src_rewrite(&intrin->src[3], trimmed);
+   intrin->num_components = new_comps;
+   return true;
+}
+
+static nir_def *
+build_px_size_sa_log2(nir_builder *b, nir_def *samples)
+{
+   nir_def *samples_log2 = nir_ufind_msb(b, samples);
+   /* Map from samples_log2 to pixels per sample (log2):
+    *
+    *  0 -> (0, 0)
+    *  1 -> (1, 0)
+    *  2 -> (1, 1)
+    *  3 -> (2, 1)
+    *  4 -> (2, 2)
+    *
+    * so
+    *
+    * h_log2 = samples_log2 / 2
+    * w_log2 = (samples_log2 + 1) / 2 = samples_log2 - h_log2
+    */
+   nir_def *h_log2 = nir_udiv_imm(b, samples_log2, 2);
+   nir_def *w_log2 = nir_isub(b, samples_log2, h_log2);
+   return nir_vec2(b, w_log2, h_log2);
+}
+
+static bool
+lower_msaa_image_access(nir_builder *b, nir_intrinsic_instr *intrin,
+                        const struct nak_compiler *nak)
+{
+   assert(nir_intrinsic_image_dim(intrin) == GLSL_SAMPLER_DIM_MS);
+
+   b->cursor = nir_before_instr(&intrin->instr);
+
+   bool can_speculate = nir_instr_can_speculate(&intrin->instr);
+   nir_def *img_h = intrin->src[0].ssa;
+   nir_def *x = nir_channel(b, intrin->src[1].ssa, 0);
+   nir_def *y = nir_channel(b, intrin->src[1].ssa, 1);
+   nir_def *z = nir_channel(b, intrin->src[1].ssa, 2);
+   nir_def *w = nir_channel(b, intrin->src[1].ssa, 3);
+   nir_def *s = intrin->src[2].ssa;
+
+   nir_def *samples = build_txq_samples_raw(b, img_h, can_speculate, nak);
+
+   nir_def *px_size_sa_log2 = build_px_size_sa_log2(b, samples);
+   nir_def *px_w_log2 = nir_channel(b, px_size_sa_log2, 0);
+   nir_def *px_h_log2 = nir_channel(b, px_size_sa_log2, 1);
+
+   /* Compute the x/y offsets
+    *
+    * txq.sampler_pos gives us the sample coordinates as a signed 4.12 fixed
+    * point with x in the bottom 16 bits and y in the top 16 bits.
+    */
+   nir_def *spos_sf = build_txq(b, nir_texop_sample_pos_nv, img_h, s,
+                                can_speculate, nak);
+   spos_sf = nir_trim_vector(b, spos_sf, 2);
+
+   /* Fortunately, the samples are laid out in the supersampled image the same
+    * as the sample locations, rounded to an integer sample offset.  So we
+    * just have to figure out which samples each of those hits in the 2D grid.
+    *
+    * Add 0x0800 to convert from signed 4.12 fixed-point centered around 0 to
+    * unsigned 4.12 fixed point.  Then shift by 12 - px_sz_log2 to divide off
+    * the extra, leaving an integer offset.  It's safe to do it all in one add
+    * because we know a priori that the low 8 bits of each sample position are
+    * zero so any overflow in the low 16 bits will just set a 1 in bit 16
+    * which will get shifted away.
+    */
+   nir_def *spos_uf = nir_iadd_imm(b, spos_sf, 0x08000800);
+   nir_def *sx = nir_ushr(b, nir_iand_imm(b, spos_uf, 0xffff),
+                             nir_isub_imm(b, 12, px_w_log2));
+   nir_def *sy = nir_ushr(b, spos_uf, nir_isub_imm(b, 28, px_h_log2));
+
+   /* Add in the sample offsets */
+   x = nir_iadd(b, nir_ishl(b, x, px_w_log2), sx);
+   y = nir_iadd(b, nir_ishl(b, y, px_h_log2), sy);
+
+   /* Smash x negative if s > samples to get OOB behavior */
+   x = nir_bcsel(b, nir_ult(b, s, samples), x, nir_imm_int(b, -1));
+
+   nir_intrinsic_set_image_dim(intrin, GLSL_SAMPLER_DIM_2D);
+   nir_src_rewrite(&intrin->src[1], nir_vec4(b, x, y, z, w));
+   nir_src_rewrite(&intrin->src[2], nir_undef(b, 1, 32));
+
+   return true;
+}
+
+static bool
+lower_image_txq(nir_builder *b, nir_intrinsic_instr *intrin,
+                const struct nak_compiler *nak)
+{
+   b->cursor = nir_before_instr(&intrin->instr);
+
+   nir_def *img_h = intrin->src[0].ssa;
+
+   bool can_speculate = nir_instr_can_speculate(&intrin->instr);
+   nir_def *res;
+
+   switch (intrin->intrinsic) {
+   case nir_intrinsic_bindless_image_size:
+      res = build_txq_size(b, intrin->def.num_components, img_h,
+                           intrin->src[1].ssa /* lod */, can_speculate, nak);
+
+      if (nir_intrinsic_image_dim(intrin) == GLSL_SAMPLER_DIM_MS) {
+         /* When NIL sets up the MSAA image descriptor, it uses a width and
+          * height in samples, rather than pixels because sust/ld/atom ignore
+          * the sample count and blindly bounds check whatever x/y coordinates
+          * they're given.  This means we need to divide back out the pixel
+          * size in order to get the size in pixels.
+          */
+         nir_def *samples = build_txq_samples_raw(b, img_h, can_speculate, nak);
+         nir_def *px_size_sa_log2 = build_px_size_sa_log2(b, samples);
+         /* The first two coordinates are X/Y. If we have more than that,
+          * it’s the array index and that isn’t scaled.
+          */
+         px_size_sa_log2 = nir_pad_vector_imm_int(b, px_size_sa_log2, 0,
+                                                  intrin->def.num_components);
+         res = nir_ushr(b, res, px_size_sa_log2);
+      }
+      break;
+   case nir_intrinsic_bindless_image_samples:
+      res = build_txq_samples(b, img_h, can_speculate, nak);
+      break;
+   default:
+      UNREACHABLE("Invalid image query op");
+   }
+
+   nir_def_replace(&intrin->def, res);
+
+   return true;
+}
+
+static bool
+lower_tex_instr(nir_builder *b, nir_instr *instr, void *_data)
+{
+   const struct nak_compiler *nak = _data;
+
+   switch (instr->type) {
+   case nir_instr_type_tex: {
+      nir_tex_instr *tex = nir_instr_as_tex(instr);
+      switch (tex->op) {
+      case nir_texop_tex:
+      case nir_texop_txb:
+      case nir_texop_txl:
+      case nir_texop_txd:
+      case nir_texop_txf:
+      case nir_texop_txf_ms:
+      case nir_texop_tg4:
+      case nir_texop_lod:
+         return lower_tex(b, tex, nak);
+      case nir_texop_txs:
+      case nir_texop_query_levels:
+      case nir_texop_texture_samples:
+         return lower_txq(b, tex, nak);
+      default:
+         UNREACHABLE("Unsupported texture instruction");
+      }
+   }
+   case nir_instr_type_intrinsic: {
+      nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
+      switch (intrin->intrinsic) {
+      case nir_intrinsic_bindless_image_load:
+      case nir_intrinsic_bindless_image_sparse_load: {
+         bool progress = false;
+         if (nir_intrinsic_image_dim(intrin) == GLSL_SAMPLER_DIM_MS)
+            progress |= lower_msaa_image_access(b, intrin, nak);
+         progress |= shrink_image_load(b, intrin, nak);
+         return progress;
+      }
+
+      case nir_intrinsic_bindless_image_store: {
+         bool progress = false;
+         if (nir_intrinsic_image_dim(intrin) == GLSL_SAMPLER_DIM_MS)
+            progress |= lower_msaa_image_access(b, intrin, nak);
+         progress |= shrink_image_store(b, intrin, nak);
+         return progress;
+      }
+
+      case nir_intrinsic_bindless_image_atomic:
+      case nir_intrinsic_bindless_image_atomic_swap:
+         if (nir_intrinsic_image_dim(intrin) == GLSL_SAMPLER_DIM_MS)
+            return lower_msaa_image_access(b, intrin, nak);
+         return false;
+
+      case nir_intrinsic_bindless_image_size:
+      case nir_intrinsic_bindless_image_samples:
+         return lower_image_txq(b, intrin, nak);
+
+      default:
+         return false;
+      }
+   }
+   default:
+      return false;
+   }
+}
+
+bool
+nak_nir_lower_tex(nir_shader *nir, const struct nak_compiler *nak)
+{
+   return nir_shader_instructions_pass(nir, lower_tex_instr,
+                                       nir_metadata_control_flow,
+                                       (void *)nak);
+}
@@ -0,0 +1,289 @@
+/*
+ * Copyright © 2023 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "nak_private.h"
+#include "nir_builder.h"
+
+static nir_def *
+tess_ctrl_output_vtx(nir_builder *b, nir_def *vtx)
+{
+   /* This is the pattern we see emitted by the blob driver:
+    *
+    *    S2R R0, SR_LANEID
+    *    S2R R6, SR_INVOCATION_ID
+    *    # R3 is our vertex index
+    *    SULD.P.2D.CTA.R.IGN R3, [R2], 0x1d, 0x0
+    *    IMAD.IADD R5, R0, 0x1, -R6
+    *    IMAD.SHL.U32 R0, R3, 0x4, RZ
+    *    LEA.HI.SX32 R4, R0, R5, 0x1e
+    *    ALD.O R4, a[0x88], R4
+    *
+    * Translating the MADs and re-naming registers, this is
+    *
+    *    %r0 = iadd %lane -%invoc
+    *    %r1 = imul %vtx 0x4
+    *    %r2 = lea.hi.sx32 %r1 %r0 0x1e
+    *    %out = ald.o a[%r2][0x88]
+    *
+    * But `lea.hi.sx32 %r1 %r0 0x1e` is just `(%r1 >> (32 - 0x1e)) + %r0`.
+    * Since %r1 is just `%vtx * 4` and 0x1e is 30, the whole bit on the left
+    * is `(%vtx * 4) >> 2 = %vtx`, assuming no overflow.  So, this means
+    *
+    *    %r0 = iadd %lane -%invoc
+    *    %r2 = iadd %vtx %r0
+    *    %out = ald.o a[%r2][0x88]
+    *
+    * In other words, the hardware actually indexes them by lane index with
+    * all of the invocations for a given TCS dispatch going in a sequential
+    * range of lanes.  We have to compute the lane index of the requested
+    * invocation from the invocation index.
+    */
+   nir_def *lane = nak_nir_load_sysval(b, NAK_SV_LANE_ID,
+                                       ACCESS_CAN_REORDER);
+   nir_def *invoc = nak_nir_load_sysval(b, NAK_SV_INVOCATION_ID,
+                                        ACCESS_CAN_REORDER);
+
+   return nir_iadd(b, lane, nir_iadd(b, vtx, nir_ineg(b, invoc)));
+}
+
+static bool
+lower_vtg_io_intrin(nir_builder *b,
+                    nir_intrinsic_instr *intrin,
+                    void *cb_data)
+{
+   const struct nak_compiler *nak = cb_data;
+   b->cursor = nir_before_instr(&intrin->instr);
+
+   nir_def *vtx = NULL, *offset = NULL, *data = NULL;
+   switch (intrin->intrinsic) {
+   case nir_intrinsic_load_primitive_id:
+   case nir_intrinsic_load_instance_id:
+   case nir_intrinsic_load_vertex_id: {
+      /* These still input loads but they're a special case */
+      const gl_system_value sysval =
+         nir_system_value_from_intrinsic(intrin->intrinsic);
+      const uint32_t addr = nak_sysval_attr_addr(nak, sysval);
+      nir_def *val = nir_ald_nv(b, 1, nir_imm_int(b, 0), nir_imm_int(b, 0),
+                                .base = addr, .flags = 0,
+                                .range_base = addr, .range = 4,
+                                .access = ACCESS_CAN_REORDER);
+      nir_def_rewrite_uses(&intrin->def, val);
+      return true;
+   }
+
+   case nir_intrinsic_load_input:
+   case nir_intrinsic_load_output:
+      offset = intrin->src[0].ssa;
+      break;
+
+   case nir_intrinsic_load_per_vertex_input:
+      vtx = intrin->src[0].ssa;
+      offset = intrin->src[1].ssa;
+      break;
+
+   case nir_intrinsic_load_per_vertex_output:
+      if (b->shader->info.stage == MESA_SHADER_TESS_CTRL)
+         vtx = tess_ctrl_output_vtx(b, intrin->src[0].ssa);
+      else
+         vtx = intrin->src[0].ssa;
+      offset = intrin->src[1].ssa;
+      break;
+
+   case nir_intrinsic_store_output:
+      data = intrin->src[0].ssa;
+      offset = intrin->src[1].ssa;
+      break;
+
+   case nir_intrinsic_store_per_vertex_output:
+      data = intrin->src[0].ssa;
+      vtx = intrin->src[1].ssa;
+      offset = intrin->src[2].ssa;
+      break;
+
+   default:
+      return false;
+   }
+   const bool offset_is_const = nir_src_is_const(nir_src_for_ssa(offset));
+
+   const bool is_store = data != NULL;
+
+   bool is_output;
+   switch (intrin->intrinsic) {
+   case nir_intrinsic_load_input:
+   case nir_intrinsic_load_per_vertex_input:
+      is_output = false;
+      break;
+
+   case nir_intrinsic_load_output:
+   case nir_intrinsic_load_per_vertex_output:
+   case nir_intrinsic_store_output:
+   case nir_intrinsic_store_per_vertex_output:
+      is_output = true;
+      break;
+
+   default:
+      UNREACHABLE("Unknown NIR I/O intrinsic");
+   }
+
+   bool is_patch;
+   switch (b->shader->info.stage) {
+   case MESA_SHADER_VERTEX:
+   case MESA_SHADER_GEOMETRY:
+      is_patch = false;
+      break;
+
+   case MESA_SHADER_TESS_CTRL:
+      is_patch = is_output && vtx == NULL;
+      break;
+
+   case MESA_SHADER_TESS_EVAL:
+      is_patch = !is_output && vtx == NULL;
+      break;
+
+   default:
+      UNREACHABLE("Unknown shader stage");
+   }
+
+   nir_component_mask_t mask;
+   if (is_store)
+      mask = nir_intrinsic_write_mask(intrin);
+   else
+      mask = nir_component_mask(intrin->num_components);
+
+   if (vtx != NULL && !is_output) {
+      if (nak->sm >= 50) {
+         nir_def *info = nak_nir_load_sysval(b, NAK_SV_INVOCATION_INFO,
+                                             ACCESS_CAN_REORDER);
+         nir_def *lo = nir_extract_u8_imm(b, info, 0);
+         nir_def *hi = nir_extract_u8_imm(b, info, 2);
+         nir_def *idx = nir_iadd(b, nir_imul(b, lo, hi), vtx);
+
+         const struct nak_nir_isbe_flags flags = {
+            .access = NAK_ISBE_ACCESS_MAP,
+            .output = false,
+            .skew = false,
+            .per_primitive = false,
+         };
+
+         vtx = nir_isberd_nv(b, 8, idx,
+                             .flags = NAK_AS_U32(flags),
+                             .access = ACCESS_CAN_REORDER);
+      } else {
+         vtx = nir_vild_nv(b, vtx);
+      }
+   }
+
+   if (vtx == NULL)
+      vtx = nir_imm_int(b, 0);
+
+   const nir_io_semantics sem = nir_intrinsic_io_semantics(intrin);
+   unsigned component = nir_intrinsic_component(intrin);
+
+   uint32_t base_addr;
+   if (b->shader->info.stage == MESA_SHADER_VERTEX && !is_output)
+      base_addr = nak_attribute_attr_addr(nak, sem.location);
+   else
+      base_addr = nak_varying_attr_addr(nak, sem.location);
+   base_addr += 4 * component;
+
+   uint32_t range;
+   if (offset_is_const) {
+      unsigned const_offset = nir_src_as_uint(nir_src_for_ssa(offset));
+
+      /* Tighten the range */
+      base_addr += const_offset * 16;
+      range = 4 * intrin->num_components;
+
+      if (const_offset != 0)
+         offset = nir_imm_int(b, 0);
+   } else {
+      /* Offsets from NIR are in vec4's */
+      offset = nir_imul_imm(b, offset, 16);
+      range = (sem.num_slots - 1) * 16 + intrin->num_components * 4;
+   }
+
+   const struct nak_nir_attr_io_flags flags = {
+      .output = is_output,
+      .patch = is_patch,
+      .phys = !offset_is_const && !is_patch,
+   };
+
+   nir_def *dst_comps[NIR_MAX_VEC_COMPONENTS];
+   while (mask) {
+      const unsigned c = ffs(mask) - 1;
+      unsigned comps = ffs(~(mask >> c)) - 1;
+      assert(comps > 0);
+
+      unsigned c_addr = base_addr + 4 * c;
+
+      /* vec2 has to be vec2 aligned, vec3/4 have to be vec4 aligned.  We
+       * don't have actual alignment information on these intrinsics but we
+       * can assume that the indirect offset (if any) is a multiple of 16 so
+       * we don't need to worry about that and can just look at c_addr.
+       */
+      comps = MIN2(comps, 4);
+      if (c_addr & 0xf)
+         comps = MIN2(comps, 2);
+      if (c_addr & 0x7)
+         comps = 1;
+      assert(!(c_addr & 0x3));
+
+      /* Vector load/store with non-constant offsets don't work pre-Maxwell.
+       * They encode fine and they don't throw any shader exceptions but the
+       * seem to get stuck in the hardware and we get context timeouts.
+       */
+      if (nak->sm < 50 && !offset_is_const)
+         comps = 1;
+
+      nir_def *c_offset = offset;
+      if (flags.phys) {
+         /* Physical addressing has to be scalar */
+         comps = 1;
+
+         /* Use al2p to compute a physical address */
+         c_offset = nir_al2p_nv(b, offset, .base = c_addr,
+                                .flags = NAK_AS_U32(flags));
+         c_addr = 0;
+      }
+
+      if (is_store) {
+         nir_def *c_data = nir_channels(b, data, BITFIELD_RANGE(c, comps));
+         nir_ast_nv(b, c_data, vtx, c_offset,
+                    .base = c_addr,
+                    .flags = NAK_AS_U32(flags),
+                    .range_base = base_addr,
+                    .range = range);
+      } else {
+         uint32_t access = flags.output ? 0 : ACCESS_CAN_REORDER;
+         nir_def *c_data = nir_ald_nv(b, comps, vtx, c_offset,
+                                      .base = c_addr,
+                                      .flags = NAK_AS_U32(flags),
+                                      .range_base = base_addr,
+                                      .range = range,
+                                      .access = access);
+         for (unsigned i = 0; i < comps; i++)
+            dst_comps[c + i] = nir_channel(b, c_data, i);
+      }
+
+      mask &= ~BITFIELD_RANGE(c, comps);
+   }
+
+   if (!is_store) {
+      nir_def *dst = nir_vec(b, dst_comps, intrin->num_components);
+      nir_def_rewrite_uses(&intrin->def, dst);
+   }
+
+   nir_instr_remove(&intrin->instr);
+
+   return true;
+}
+
+bool
+nak_nir_lower_vtg_io(nir_shader *nir, const struct nak_compiler *nak)
+{
+   return nir_shader_intrinsics_pass(nir, lower_vtg_io_intrin,
+                                     nir_metadata_control_flow,
+                                     (void *)nak);
+}
@@ -0,0 +1,115 @@
+/*
+ * Copyright 2024 Valve Corporation
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "nak_private.h"
+#include "nir_builder.h"
+
+/*
+ * Divergent loops can define convergent values that escape the loop. eg.
+ *
+ *    div loop {
+ *       con %72 = ...
+ *       ...
+ *    }
+ *    ... = @ldcx_nv (%72, 0x30)
+ *
+ * When that happens, we would like to have the opportunity to move the value
+ * into a uniform register once we reach uniform control flow. This pass allows
+ * that by inserting nir_intrinsic_as_uniform after the loop-closed phi, which
+ * becomes r2ur in the backend.
+ *
+ * With nir_to_lcssa and this pass, our example becomes something like:
+ *
+ *    div loop {
+ *       con %72 = ...
+ *       ...
+ *    }
+ *    con %73 = phi(%72, %72)
+ *    con %74 = @as_uniform(%73)
+ *    ... = @ldcx_nv (%74, 0x30)
+ *
+ * nir_to_lcssa(nir, false, false) and nir_divergence_analysis must be run
+ * before this pass.
+ */
+
+static bool
+lower_block(nir_builder *b, nir_block *block)
+{
+   bool progress = false;
+
+   b->cursor = nir_after_phis(block);
+   nir_foreach_phi_safe(phi, block) {
+      if (phi->def.divergent) {
+         continue;
+      }
+
+      nir_def* x = nir_as_uniform(b, &phi->def);
+      x->divergent = false;
+      nir_def_rewrite_uses_after(&phi->def, x);
+
+      progress = true;
+   }
+
+   return progress;
+}
+
+
+static bool
+lower_cf_list(nir_builder *b, struct exec_list *cf_list)
+{
+   bool progress = false;
+
+   foreach_list_typed_safe(nir_cf_node, node, node, cf_list) {
+      switch (node->type) {
+      case nir_cf_node_block:
+         break;
+
+      case nir_cf_node_if: {
+         nir_if *nif = nir_cf_node_as_if(node);
+         if (nir_src_is_divergent(&nif->condition)) {
+            nir_block *succ = nir_cf_node_as_block(nir_cf_node_next(node));
+            progress |= lower_block(b, succ);
+         } else {
+            progress |= lower_cf_list(b, &nif->then_list);
+            progress |= lower_cf_list(b, &nif->else_list);
+         }
+         break;
+      }
+
+      case nir_cf_node_loop: {
+         nir_loop *loop = nir_cf_node_as_loop(node);
+         if (nir_loop_is_divergent(loop)) {
+            nir_block *succ = nir_cf_node_as_block(nir_cf_node_next(node));
+            progress |= lower_block(b, succ);
+         } else {
+            progress |= lower_cf_list(b, &loop->body);
+            progress |= lower_cf_list(b, &loop->continue_list);
+         }
+         break;
+      }
+
+      default:
+         UNREACHABLE("Unknown CF node type");
+      }
+   }
+
+   return progress;
+}
+
+bool
+nak_nir_mark_lcssa_invariants(nir_shader *shader)
+{
+   bool progress = false;
+
+   nir_foreach_function_impl(impl, shader) {
+      nir_builder b = nir_builder_create(impl);
+      nir_metadata_require(impl, nir_metadata_divergence);
+
+      bool impl_progress = lower_cf_list(&b, &impl->body);
+      progress |= nir_progress(impl_progress, impl, nir_metadata_control_flow);
+   }
+
+   return progress;
+}
@@ -0,0 +1,103 @@
+/*
+ * Copyright © 2025 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "nak_private.h"
+#include "nir_builder.h"
+#include "util/hash_table.h"
+
+struct remat_ctx {
+   struct hash_table *remap;
+   nir_block *block;
+   nir_builder b;
+};
+
+static bool
+rematerialize_load_const(nir_src *src, void *_ctx)
+{
+   struct remat_ctx *ctx = _ctx;
+
+   if (!nir_src_is_const(*src))
+      return true;
+
+   struct hash_entry *entry = _mesa_hash_table_search(ctx->remap, src->ssa);
+   if (entry != NULL) {
+      nir_src_rewrite(src, entry->data);
+      return true;
+   }
+
+   nir_load_const_instr *old_lc =
+      nir_def_as_load_const(src->ssa);
+
+   nir_load_const_instr *new_lc =
+      nir_instr_as_load_const(nir_instr_clone(ctx->b.shader, &old_lc->instr));
+   nir_builder_instr_insert(&ctx->b, &new_lc->instr);
+
+   _mesa_hash_table_insert(ctx->remap, &old_lc->def, &new_lc->def);
+   nir_src_rewrite(src, &new_lc->def);
+
+   return true;
+}
+
+static bool
+rematerialize_load_const_impl(nir_function_impl *impl)
+{
+   struct remat_ctx ctx = {
+      .remap = _mesa_pointer_hash_table_create(NULL),
+      .b = nir_builder_create(impl),
+   };
+
+   bool progress = false;
+   nir_foreach_block(block, impl) {
+      _mesa_hash_table_clear(ctx.remap, NULL);
+      ctx.block = block;
+
+      nir_foreach_instr(instr, block) {
+         if (instr->type == nir_instr_type_phi)
+            continue;
+
+         ctx.b.cursor = nir_before_instr(instr);
+         nir_foreach_src(instr, rematerialize_load_const, &ctx);
+      }
+
+      ctx.b.cursor = nir_after_block_before_jump(block);
+
+      for (unsigned i = 0; i < ARRAY_SIZE(block->successors); i++) {
+         nir_block *succ = block->successors[i];
+         if (succ == NULL)
+            continue;
+
+         nir_foreach_instr(instr, succ) {
+            if (instr->type != nir_instr_type_phi)
+               break;
+
+            nir_phi_instr *phi = nir_instr_as_phi(instr);
+            nir_foreach_phi_src(src, phi) {
+               if (src->pred != block)
+                  continue;
+
+               rematerialize_load_const(&src->src, &ctx);
+            }
+         }
+      }
+
+      if (ctx.remap->entries > 0)
+         progress = true;
+   }
+
+   _mesa_hash_table_destroy(ctx.remap, NULL);
+
+   return nir_progress(progress, impl, nir_metadata_control_flow);
+}
+
+bool
+nak_nir_rematerialize_load_const(nir_shader *nir)
+{
+   bool progress = false;
+
+   nir_foreach_function_impl(impl, nir)
+      progress |= rematerialize_load_const_impl(impl);
+
+   return progress;
+}
@@ -0,0 +1,461 @@
+/*
+ * Copyright © 2022 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef NAK_PRIVATE_H
+#define NAK_PRIVATE_H
+
+#include "nak.h"
+#include "nir.h"
+#include "nv_device_info.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+bool nak_should_print_nir(void);
+bool nak_debug_no_ugpr(void);
+
+struct nak_compiler {
+   uint8_t sm;
+   uint8_t warps_per_sm;
+
+   struct nir_shader_compiler_options nir_options;
+};
+
+enum ENUM_PACKED nak_attr {
+   /* System values A */
+   NAK_ATTR_TESS_LOD_LEFT     = 0x000,
+   NAK_ATTR_TESS_LOD_RIGHT    = 0x004,
+   NAK_ATTR_TESS_LOD_BOTTOM   = 0x008,
+   NAK_ATTR_TESS_LOD_TOP      = 0x00c,
+   NAK_ATTR_TESS_LOD          = NAK_ATTR_TESS_LOD_LEFT,
+   NAK_ATTR_TESS_INTERRIOR_U  = 0x010,
+   NAK_ATTR_TESS_INTERRIOR_V  = 0x014,
+   NAK_ATTR_TESS_INTERRIOR    = NAK_ATTR_TESS_INTERRIOR_U,
+
+   /* Patch attributes */
+   NAK_ATTR_PATCH_START       = 0x020,
+
+   /* System values B? */
+   NAK_ATTR_VPRS_TABLE_INDEX  = 0x5c,
+
+   /* System values B */
+   NAK_ATTR_PRIMITIVE_ID      = 0x060,
+   NAK_ATTR_RT_ARRAY_INDEX    = 0x064,
+   NAK_ATTR_VIEWPORT_INDEX    = 0x068,
+   NAK_ATTR_POINT_SIZE        = 0x06c,
+   NAK_ATTR_POSITION_X        = 0x070,
+   NAK_ATTR_POSITION_Y        = 0x074,
+   NAK_ATTR_POSITION_Z        = 0x078,
+   NAK_ATTR_POSITION_W        = 0x07c,
+   NAK_ATTR_POSITION          = NAK_ATTR_POSITION_X,
+
+   /* Generic attributes */
+   NAK_ATTR_GENERIC_START     = 0x080,
+
+   /* System values C */
+   NAK_ATTR_CLIP_CULL_DIST_0  = 0x2c0,
+   NAK_ATTR_CLIP_CULL_DIST_1  = 0x2c4,
+   NAK_ATTR_CLIP_CULL_DIST_2  = 0x2c8,
+   NAK_ATTR_CLIP_CULL_DIST_3  = 0x2cc,
+   NAK_ATTR_CLIP_CULL_DIST_4  = 0x2d0,
+   NAK_ATTR_CLIP_CULL_DIST_5  = 0x2d4,
+   NAK_ATTR_CLIP_CULL_DIST_6  = 0x2d8,
+   NAK_ATTR_CLIP_CULL_DIST_7  = 0x2dc,
+   NAK_ATTR_CLIP_CULL_DIST    = NAK_ATTR_CLIP_CULL_DIST_0,
+   NAK_ATTR_POINT_SPRITE_S    = 0x2e0,
+   NAK_ATTR_POINT_SPRITE_T    = 0x2e4,
+   NAK_ATTR_POINT_SPRITE      = NAK_ATTR_POINT_SPRITE_S,
+   NAK_ATTR_FOG_COORD         = 0x2e8,
+   /* Reserved                  0x2ec */
+   NAK_ATTR_TESS_COORD_X      = 0x2f0,
+   NAK_ATTR_TESS_COORD_Y      = 0x2f4,
+   NAK_ATTR_TESS_COORD        = NAK_ATTR_TESS_COORD_X,
+   NAK_ATTR_INSTANCE_ID       = 0x2f8,
+   NAK_ATTR_VERTEX_ID         = 0x2fc,
+
+   /* System values D */
+   NAK_ATTR_VIEWPORT_MASK         = 0x3a0,
+   NAK_ATTR_BARY_COORD_NO_PERSP_X = 0x3a8,
+   NAK_ATTR_BARY_COORD_NO_PERSP_Y = 0x3ac,
+   NAK_ATTR_BARY_COORD_NO_PERSP_Z = 0x3b0,
+   NAK_ATTR_BARY_COORD_NO_PERSP = NAK_ATTR_BARY_COORD_NO_PERSP_X,
+
+   NAK_ATTR_BARY_COORD_X = 0x3b4,
+   NAK_ATTR_BARY_COORD_Y = 0x3b8,
+   NAK_ATTR_BARY_COORD_Z = 0x3bc,
+   NAK_ATTR_BARY_COORD   = NAK_ATTR_BARY_COORD_X,
+   NAK_ATTR_SPH_END      = NAK_ATTR_BARY_COORD_Z + 4,
+
+   /* Not in SPH */
+   NAK_ATTR_FRONT_FACE        = 0x3fc,
+};
+
+static inline uint16_t
+nak_attribute_attr_addr(UNUSED const struct nak_compiler *nak,
+                        gl_vert_attrib attrib)
+{
+   assert(attrib >= VERT_ATTRIB_GENERIC0);
+   return NAK_ATTR_GENERIC_START + (attrib - VERT_ATTRIB_GENERIC0) * 0x10;
+}
+
+uint16_t nak_varying_attr_addr(const struct nak_compiler *nak,
+                               gl_varying_slot slot);
+uint16_t nak_varying_mesh_skew_attr_addr(const struct nak_compiler *nak,
+                                         gl_varying_slot slot);
+uint16_t nak_sysval_attr_addr(const struct nak_compiler *nak,
+                              gl_system_value sysval);
+
+enum ENUM_PACKED nak_sv {
+   NAK_SV_LANE_ID          = 0x00,
+   NAK_SV_VIRTCFG          = 0x02,
+   NAK_SV_VIRTID           = 0x03,
+   NAK_SV_PRIM_TYPE        = 0x10,
+   NAK_SV_INVOCATION_ID    = 0x11,
+   NAK_SV_THREAD_KILL      = 0x13,
+   NAK_SV_INVOCATION_INFO  = 0x1d,
+   NAK_SV_COMBINED_TID     = 0x20,
+   NAK_SV_TID_X            = 0x21,
+   NAK_SV_TID_Y            = 0x22,
+   NAK_SV_TID_Z            = 0x23,
+   NAK_SV_TID              = NAK_SV_TID_X,
+   NAK_SV_CTAID_X          = 0x25,
+   NAK_SV_CTAID_Y          = 0x26,
+   NAK_SV_CTAID_Z          = 0x27,
+   NAK_SV_CTAID            = NAK_SV_CTAID_X,
+   NAK_SV_LANEMASK_EQ      = 0x38,
+   NAK_SV_LANEMASK_LT      = 0x39,
+   NAK_SV_LANEMASK_LE      = 0x3a,
+   NAK_SV_LANEMASK_GT      = 0x3b,
+   NAK_SV_LANEMASK_GE      = 0x3c,
+   NAK_SV_CLOCK_LO         = 0x50,
+   NAK_SV_CLOCK_HI         = 0x51,
+   NAK_SV_CLOCK            = NAK_SV_CLOCK_LO,
+   NAK_SV_VARIABLE_RATE    = 0x84,
+};
+
+bool nak_nir_workgroup_has_one_subgroup(const nir_shader *nir);
+
+nir_def *nak_nir_load_sysval(nir_builder *b, enum nak_sv idx,
+                             enum gl_access_qualifier access);
+
+struct nak_xfb_info
+nak_xfb_from_nir(const struct nak_compiler *nak,
+                 const struct nir_xfb_info *nir_xfb);
+
+enum nak_nir_tex_ref_type {
+   /** Indicates that this is a bindless texture */
+   NAK_NIR_TEX_REF_TYPE_BINDLESS,
+
+   /** Indicates that this is a bound texture
+    *
+    * The binding index provided in texture_index.
+    */
+   NAK_NIR_TEX_REF_TYPE_BOUND,
+
+   /** Indicates that this is a cbuf texture
+    *
+    * texture_index is (idx << 16) | offset.
+    */
+   NAK_NIR_TEX_REF_TYPE_CBUF,
+};
+
+enum nak_nir_lod_mode {
+   NAK_NIR_LOD_MODE_AUTO = 0,
+   NAK_NIR_LOD_MODE_ZERO,
+   NAK_NIR_LOD_MODE_BIAS,
+   NAK_NIR_LOD_MODE_LOD,
+   NAK_NIR_LOD_MODE_CLAMP,
+   NAK_NIR_LOD_MODE_BIAS_CLAMP,
+};
+
+enum nak_nir_offset_mode {
+   NAK_NIR_OFFSET_MODE_NONE = 0,
+   NAK_NIR_OFFSET_MODE_AOFFI,
+   NAK_NIR_OFFSET_MODE_PER_PX,
+};
+
+PRAGMA_DIAGNOSTIC_PUSH
+PRAGMA_DIAGNOSTIC_ERROR(-Wpadded)
+struct nak_nir_tex_flags {
+   enum nak_nir_tex_ref_type ref_type:2;
+   enum nak_nir_lod_mode lod_mode:3;
+   enum nak_nir_offset_mode offset_mode:2;
+   bool has_z_cmpr:1;
+   bool is_sparse:1;
+   bool nodep:1;
+   bool scalar:1;
+   uint32_t pad:21;
+};
+PRAGMA_DIAGNOSTIC_POP
+static_assert(sizeof(struct nak_nir_tex_flags) == 4,
+              "nak_nir_tex_flags has no holes");
+
+#define NAK_AS_U32(x) ({\
+   static_assert(sizeof(x) == 4, "x must be 4 bytes"); \
+   uint32_t _u; \
+   memcpy(&_u, &(x), 4); \
+   _u; \
+})
+
+bool nak_nir_lower_scan_reduce(nir_shader *shader, const struct nak_compiler *nak);
+bool nak_nir_lower_tex(nir_shader *nir, const struct nak_compiler *nak);
+bool nak_nir_lower_gs_intrinsics(nir_shader *shader);
+bool nak_nir_lower_algebraic_late(nir_shader *nir, const struct nak_compiler *nak);
+bool nak_nir_lower_kepler_shared_atomics(nir_shader *shader);
+bool nak_nir_lower_mesh_stages_shared_atomics(nir_shader *nir);
+
+struct nak_nir_attr_io_flags {
+   bool output : 1;
+   bool patch : 1;
+   bool phys : 1;
+   uint32_t pad:29;
+};
+
+enum nak_suclamp_mode {
+   NAK_SUCLAMP_MODE_STORED_DESCRIPTOR,
+   NAK_SUCLAMP_MODE_PITCH_LINEAR,
+   NAK_SUCLAMP_MODE_BLOCK_LINEAR,
+};
+
+enum nak_suclamp_round {
+   NAK_SUCLAMP_ROUND_R1,
+   NAK_SUCLAMP_ROUND_R2,
+   NAK_SUCLAMP_ROUND_R4,
+   NAK_SUCLAMP_ROUND_R8,
+   NAK_SUCLAMP_ROUND_R16,
+};
+
+struct nak_nir_suclamp_flags {
+   enum nak_suclamp_mode mode : 2;
+   enum nak_suclamp_round round : 3;
+   bool is_s32 : 1;
+   bool is_2d : 1;
+   uint32_t pad:25;
+};
+
+enum nak_su_ga_offset_mode {
+   NAK_SUGA_OFF_MODE_U32,
+   NAK_SUGA_OFF_MODE_S32,
+   NAK_SUGA_OFF_MODE_U8,
+   NAK_SUGA_OFF_MODE_S8,
+};
+
+enum nak_imad_src_type {// 3 bits
+   NAK_IMAD_TYPE_U32,
+   NAK_IMAD_TYPE_U24,
+   NAK_IMAD_TYPE_U16_LO,
+   NAK_IMAD_TYPE_U16_HI,
+   NAK_IMAD_TYPE_S32,
+   NAK_IMAD_TYPE_S24,
+   NAK_IMAD_TYPE_S16_LO,
+   NAK_IMAD_TYPE_S16_HI,
+};
+
+struct nak_nir_imadsp_flags {
+   enum nak_imad_src_type src0 : 3;
+   enum nak_imad_src_type src1 : 3;
+   enum nak_imad_src_type src2 : 3;
+   bool params_from_src1 : 1;
+   uint32_t pad:22;
+};
+
+bool nak_nir_lower_vtg_io(nir_shader *nir, const struct nak_compiler *nak);
+
+enum nak_isbe_access {
+   NAK_ISBE_ACCESS_MAP,
+   NAK_ISBE_ACCESS_PATCH,
+   NAK_ISBE_ACCESS_PRIM,
+   NAK_ISBE_ACCESS_ATTR,
+};
+
+struct nak_nir_isbe_flags {
+   enum nak_isbe_access access : 2;
+   bool output : 1;
+   bool skew : 1;
+   bool per_primitive : 1;
+   uint32_t pad : 27;
+};
+
+struct lower_mesh_intrinsics_ctx {
+   const struct nak_compiler *nak;
+
+   uint32_t max_vertices_out;
+   uint32_t max_primitives_out;
+   bool has_task_shader;
+
+   BITSET_DECLARE(skew_vert_attr_used, NAK_ATTR_SPH_END);
+   BITSET_DECLARE(skew_prim_attr_used, NAK_ATTR_SPH_END);
+};
+
+#define NAK_MESH_SKEW_GROUP_COUNT 32
+
+static uint32_t
+nak_mesh_skew_attr_used_index(uint32_t base_addr)
+{
+   assert(base_addr < NAK_ATTR_SPH_END);
+
+   return base_addr / 4;
+}
+
+static uint32_t
+nak_mesh_skew_vert_group_size(const struct lower_mesh_intrinsics_ctx *ctx)
+{
+   return BITSET_COUNT(ctx->skew_vert_attr_used) * 4 * NAK_MESH_SKEW_GROUP_COUNT;
+}
+
+static uint32_t
+nak_mesh_skew_vert_total_size(const struct lower_mesh_intrinsics_ctx *ctx)
+{
+   return nak_mesh_skew_vert_group_size(ctx) * DIV_ROUND_UP(ctx->max_vertices_out, NAK_MESH_SKEW_GROUP_COUNT);
+}
+
+static uint32_t
+nak_mesh_skew_prim_group_size(const struct lower_mesh_intrinsics_ctx *ctx)
+{
+   return BITSET_COUNT(ctx->skew_prim_attr_used) * 4 * NAK_MESH_SKEW_GROUP_COUNT;
+}
+
+static uint32_t
+nak_mesh_skew_prim_total_size(const struct lower_mesh_intrinsics_ctx *ctx)
+{
+   return nak_mesh_skew_prim_group_size(ctx) * DIV_ROUND_UP(ctx->max_primitives_out, NAK_MESH_SKEW_GROUP_COUNT);
+}
+
+static uint32_t
+nak_mesh_skew_total_size(const struct lower_mesh_intrinsics_ctx *ctx)
+{
+   return nak_mesh_skew_vert_total_size(ctx) + nak_mesh_skew_prim_total_size(ctx);
+}
+
+static uint32_t
+nak_mesh_skew_offset(const struct lower_mesh_intrinsics_ctx *ctx,
+                     gl_varying_slot slot,
+                     uint32_t base_addr,
+                     bool per_primitive)
+{
+   const uint32_t bit_idx = nak_mesh_skew_attr_used_index(base_addr);
+
+   uint32_t bit_count;
+
+   if (per_primitive)
+      bit_count = BITSET_PREFIX_SUM(ctx->skew_prim_attr_used, bit_idx);
+   else
+      bit_count = BITSET_PREFIX_SUM(ctx->skew_vert_attr_used, bit_idx);
+
+   uint32_t size = bit_count * 4 * NAK_MESH_SKEW_GROUP_COUNT;
+
+   return size;
+}
+
+bool nak_nir_lower_mesh_emulated_attributes(nir_shader *nir);
+bool nak_nir_lower_mesh_intrinsics(nir_shader *nir, struct lower_mesh_intrinsics_ctx *ctx);
+bool nak_nir_lower_task_intrinsics(nir_shader *nir);
+
+enum nak_interp_mode {
+   NAK_INTERP_MODE_PERSPECTIVE,
+   NAK_INTERP_MODE_SCREEN_LINEAR,
+   NAK_INTERP_MODE_CONSTANT,
+};
+
+enum nak_interp_freq {
+    NAK_INTERP_FREQ_PASS,
+    NAK_INTERP_FREQ_PASS_MUL_W,
+    NAK_INTERP_FREQ_CONSTANT,
+    NAK_INTERP_FREQ_STATE,
+};
+
+enum nak_interp_loc {
+   NAK_INTERP_LOC_DEFAULT,
+   NAK_INTERP_LOC_CENTROID,
+   NAK_INTERP_LOC_OFFSET,
+};
+
+struct nak_nir_ipa_flags {
+   enum nak_interp_mode interp_mode:2;
+   enum nak_interp_freq interp_freq:2;
+   enum nak_interp_loc interp_loc:2;
+   uint32_t pad:26;
+};
+
+enum nak_cmat_type {
+   NAK_CMAT_TYPE_M8N8K16_INT,
+   NAK_CMAT_TYPE_M16N8K16_INT,
+   NAK_CMAT_TYPE_M16N8K32_INT,
+
+   NAK_CMAT_TYPE_M16N8K8_FLOAT,
+   NAK_CMAT_TYPE_M16N8K16_FLOAT,
+
+   /* Software emulated cmat layouts
+    *
+    * Those aren't supported as a single native *MMA invocation on any hardware,
+    * so in order to support those we execute multiple *MMA instructions with a
+    * register layout defined by us.
+    */
+   NAK_CMAT_TYPE_M16N16K32_INT_SW,
+   NAK_CMAT_TYPE_M16N16K16_FLOAT_SW,
+};
+
+struct nak_nir_cmat_mul_add_flags {
+   enum nak_cmat_type cmat_type:3;
+   enum glsl_base_type a_type:5;
+   enum glsl_base_type b_type:5;
+   bool sat:1;
+   uint32_t pad:18;
+};
+
+bool nak_nir_lower_fs_inputs(nir_shader *nir,
+                             const struct nak_compiler *nak,
+                             const struct nak_fs_key *fs_key);
+
+enum nak_fs_out {
+   NAK_FS_OUT_COLOR0 = 0x00,
+   NAK_FS_OUT_COLOR1 = 0x10,
+   NAK_FS_OUT_COLOR2 = 0x20,
+   NAK_FS_OUT_COLOR3 = 0x30,
+   NAK_FS_OUT_COLOR4 = 0x40,
+   NAK_FS_OUT_COLOR5 = 0x50,
+   NAK_FS_OUT_COLOR6 = 0x60,
+   NAK_FS_OUT_COLOR7 = 0x70,
+   NAK_FS_OUT_SAMPLE_MASK = 0x80,
+   NAK_FS_OUT_DEPTH = 0x84,
+};
+
+#define NAK_FS_OUT_COLOR(n) (NAK_FS_OUT_COLOR0 + (n) * 16)
+
+static inline const struct nak_constant_offset_info*
+nak_const_offsets(const struct nak_compiler* nak, bool is_graphics)
+{
+   if (nak->sm >= 75 && is_graphics) {
+      return &nak_const_offsets_turing_graphics;
+   } else {
+      return &nak_const_offsets_base;
+   }
+}
+
+bool nak_nir_rematerialize_load_const(nir_shader *nir);
+bool nak_nir_mark_lcssa_invariants(nir_shader *nir);
+bool nak_nir_lower_non_uniform_ldcx(nir_shader *nir, const struct nak_compiler *nak);
+bool nak_nir_add_barriers(nir_shader *nir, const struct nak_compiler *nak);
+bool nak_nir_lower_cf(nir_shader *nir);
+bool nak_nir_lower_cmat(nir_shader *shader, const struct nak_compiler *nak);
+
+/**
+ * Check if, for nak's purposes, a block is divergent
+ *
+ * Note that this differs from block->divergent because in nir's terms, the
+ * start block of a loop with divergent continues can be non-divergent but nak
+ * will always consider this divergent. This matters because nak does not allow
+ * writing uregs from these blocks.
+ */
+bool nak_block_is_divergent(const nir_block *block);
+
+void nak_optimize_nir(nir_shader *nir, const struct nak_compiler *nak);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* NAK_PRIVATE */
@@ -0,0 +1,28 @@
+### nouveau_noop backend
+
+This implements the minimum of nouveau in order to make shader-db work.
+The submit ioctl is stubbed out to not execute anything.
+
+Export `MESA_LOADER_DRIVER_OVERRIDE=nouveau
+LD_PRELOAD=$prefix/lib/libnouveau_noop_drm_shim.so`.
+
+By default, GK110 is exposed.  The chip can be selected with an environment
+variable like `NOUVEAU_CHIPSET=f0`. Some chips of note with the appropriate
+chipset ids:
+
+| ID  | Chip Name | Sample Marketing Name | Comment |
+| --- | --------- | --------------------- | ------- |
+| 30  | NV30      | GeForce FX 5500       | DX9     |
+| 40  | NV40      | GeForce 6800          | DX9c, better control flow |
+| 50  | G80       | GeForce 8800 GTS      | SM 1.0, 128 regs |
+| 84  | G84       | GeForce 8600 GS       | SM 1.1  |
+| a0  | GT200     | GeForce GTX 280       | SM 1.2 + fp64 |
+| a3  | GT215     | GeForce GT 240        | DX10.1 ops |
+| c0  | GF100     | GeForce GTX 480       | SM 2.0, 64 regs |
+| e4  | GK104     | GeForce GTX 680       | SM 3.0, 64 regs |
+| f0  | GK110     | GeForce GTX 780       | SM 3.5, 256 regs |
+| 117 | GM107     | GeForce GTX 750       | SM 5.0  |
+| 124 | GM204     | GeForce GTX 980       | SM 5.2  |
+| 134 | GP104     | GeForce GTX 1080      | SM 6.0  |
+| 140 | GV100     | TITAN V               | SM 7.0  |
+| 162 | TU102     | GeForce RTX 2080      | SM 7.5  |
@@ -0,0 +1,12 @@
+# Copyright © 2021 Ilia Mirkin
+# SPDX-License-Identifier: MIT
+
+libnouveau_noop_drm_shim = shared_library(
+  'nouveau_noop_drm_shim',
+  'nouveau_noop.c',
+  include_directories: [inc_include, inc_src],
+  c_args: [ cc.get_supported_arguments('-Wno-gnu-variable-sized-type-not-at-end') ],
+  dependencies: dep_drm_shim,
+  gnu_symbol_visibility : 'hidden',
+  install : true,
+)
@@ -0,0 +1,492 @@
+/*
+ * Copyright © 2021 Ilia Mirkin
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <limits.h>
+#include <stdio.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <sys/ioctl.h>
+#include "drm-uapi/nouveau_drm.h"
+#include "nouveau/nvif/ioctl.h"
+#include "nouveau/nvif/cl0080.h"
+#include "drm-shim/drm_shim.h"
+#include "util/os_misc.h"
+#include "util/u_math.h"
+
+#include "../../gallium/drivers/nouveau/nv_object.xml.h"
+bool drm_shim_driver_prefers_first_render_node = true;
+
+struct nouveau_device {
+   uint64_t next_offset;
+};
+
+static struct nouveau_device nouveau = {
+   .next_offset = 0x1000,
+};
+
+struct nouveau_shim_bo {
+   struct shim_bo base;
+   uint64_t offset;
+};
+
+static struct nouveau_shim_bo *
+nouveau_shim_bo(struct shim_bo *bo)
+{
+   return (struct nouveau_shim_bo *)bo;
+}
+
+struct nouveau_device_info {
+   uint32_t chip_id;
+};
+
+static struct nouveau_device_info device_info;
+
+static int
+nouveau_ioctl_noop(int fd, unsigned long request, void *arg)
+{
+   return 0;
+}
+
+static int
+nouveau_ioctl_gem_new(int fd, unsigned long request, void *arg)
+{
+   struct shim_fd *shim_fd = drm_shim_fd_lookup(fd);
+   struct drm_nouveau_gem_new *create = arg;
+   struct nouveau_shim_bo *bo = calloc(1, sizeof(*bo));
+
+   drm_shim_bo_init(&bo->base, create->info.size);
+
+   assert(ULONG_MAX - nouveau.next_offset > create->info.size);
+
+   create->info.handle = drm_shim_bo_get_handle(shim_fd, &bo->base);
+   create->info.map_handle = drm_shim_bo_get_mmap_offset(shim_fd, &bo->base);
+
+   if (create->align != 0)
+      nouveau.next_offset = align64(nouveau.next_offset, create->align);
+   create->info.offset = nouveau.next_offset;
+   nouveau.next_offset += create->info.size;
+
+   bo->offset = create->info.offset;
+
+   drm_shim_bo_put(&bo->base);
+
+   return 0;
+}
+
+static int
+nouveau_ioctl_gem_info(int fd, unsigned long request, void *arg)
+{
+   struct shim_fd *shim_fd = drm_shim_fd_lookup(fd);
+   struct drm_nouveau_gem_info *info = arg;
+   struct nouveau_shim_bo *bo =
+      nouveau_shim_bo(drm_shim_bo_lookup(shim_fd, info->handle));
+   info->map_handle = drm_shim_bo_get_mmap_offset(shim_fd, &bo->base);
+   info->offset = bo->offset;
+   info->size = bo->base.size;
+
+   drm_shim_bo_put(&bo->base);
+
+   return 0;
+}
+
+static int
+nouveau_ioctl_gem_pushbuf(int fd, unsigned long request, void *arg)
+{
+   struct drm_nouveau_gem_pushbuf *submit = arg;
+   submit->vram_available = 3ULL << 30;
+   submit->gart_available = 1ULL << 40;
+   return 0;
+}
+
+static int
+nouveau_ioctl_channel_alloc(int fd, unsigned long request, void *arg)
+{
+   struct shim_fd *shim_fd = drm_shim_fd_lookup(fd);
+   struct drm_nouveau_channel_alloc *alloc = arg;
+   if (device_info.chip_id == 0x50 || device_info.chip_id >= 0x80)
+      alloc->pushbuf_domains = NOUVEAU_GEM_DOMAIN_VRAM | NOUVEAU_GEM_DOMAIN_GART;
+   else
+      alloc->pushbuf_domains = NOUVEAU_GEM_DOMAIN_GART;
+
+   /* NOTE: this will get leaked since we don't handle the channel
+    * free. However only one channel is created per screen, so impact should
+    * be limited. */
+   struct nouveau_shim_bo *notify = calloc(1, sizeof(*notify));
+   drm_shim_bo_init(&notify->base, 0x1000);
+   notify->offset = nouveau.next_offset;
+   nouveau.next_offset += 0x1000;
+   alloc->notifier_handle = drm_shim_bo_get_handle(shim_fd, &notify->base);
+
+   drm_shim_bo_put(&notify->base);
+
+   return 0;
+}
+
+static int
+nouveau_ioctl_get_param(int fd, unsigned long request, void *arg)
+{
+   struct drm_nouveau_getparam *gp = arg;
+
+   switch (gp->param) {
+   case NOUVEAU_GETPARAM_CHIPSET_ID:
+      gp->value = device_info.chip_id;
+      return 0;
+   case NOUVEAU_GETPARAM_PCI_VENDOR:
+      gp->value = 0x10de;
+      return 0;
+   case NOUVEAU_GETPARAM_PCI_DEVICE:
+      gp->value = 0x1004;
+      return 0;
+   case NOUVEAU_GETPARAM_BUS_TYPE:
+      gp->value = 2 /* NV_PCIE */;
+      return 0;
+   case NOUVEAU_GETPARAM_FB_SIZE:
+      gp->value = 3ULL << 30;
+      return 0;
+   case NOUVEAU_GETPARAM_AGP_SIZE:
+      gp->value = 1ULL << 40;
+      return 0;
+   case NOUVEAU_GETPARAM_PTIMER_TIME:
+      gp->value = 0;
+      return 0;
+   case NOUVEAU_GETPARAM_HAS_BO_USAGE:
+      gp->value = 1;
+      return 0;
+   case NOUVEAU_GETPARAM_GRAPH_UNITS:
+      gp->value = 0x01000101;
+      return 0;
+   case NOUVEAU_GETPARAM_EXEC_PUSH_MAX:
+      gp->value = 510;
+      return 0;
+   case NOUVEAU_GETPARAM_VRAM_BAR_SIZE:
+      gp->value = 1L << 34;
+      return 0;
+   case NOUVEAU_GETPARAM_VRAM_USED:
+      gp->value = 4096;
+      return 0;
+   case NOUVEAU_GETPARAM_HAS_VMA_TILEMODE:
+      gp->value = 1;
+      return 0;
+   default:
+      fprintf(stderr, "Unknown DRM_IOCTL_NOUVEAU_GETPARAM %llu\n",
+              (long long unsigned)gp->param);
+      return -1;
+   }
+}
+
+static int
+nouveau_ioctl_nvif(int fd, unsigned long request, void *arg)
+{
+   struct {
+      struct nvif_ioctl_v0 ioctl;
+   } *args = arg;
+
+   switch (args->ioctl.type) {
+   case NVIF_IOCTL_V0_MTHD: {
+      struct {
+         struct nvif_ioctl_v0 ioctl;
+         struct nvif_ioctl_mthd_v0 mthd;
+      } *mthd = (void *)args;
+      switch (mthd->mthd.method) {
+      case NV_DEVICE_V0_INFO: {
+         struct nv_device_info_v0 *info = (void *)&mthd->mthd.data;
+         info->chipset = device_info.chip_id;
+         info->platform = NV_DEVICE_INFO_V0_PCIE;
+
+         /* make something up */
+         info->ram_user = 3ULL << 30;
+         break;
+      }
+      default:
+         break;
+      }
+      break;
+   }
+   case NVIF_IOCTL_V0_SCLASS: {
+      struct {
+         struct nvif_ioctl_v0 ioctl;
+         struct nvif_ioctl_sclass_v0 sclass;
+      } *sclass = (void *)args;
+
+      if (sclass->sclass.count == 0) {
+         sclass->sclass.count = device_info.chip_id >= 0xe0 ? 4 : 3;
+         return 0;
+      }
+      int idx = 0;
+      /* m2mf */
+      switch (device_info.chip_id & ~0xf) {
+      case 0x170:
+      case 0x160:
+      case 0x140:
+      case 0x130:
+      case 0x120:
+      case 0x110:
+      case 0x100:
+      case 0xf0:
+         sclass->sclass.oclass[idx].oclass = NVF0_P2MF_CLASS;
+         break;
+      case 0xe0:
+         sclass->sclass.oclass[idx].oclass = NVE4_P2MF_CLASS;
+         break;
+      default:
+         sclass->sclass.oclass[idx].oclass = NVC0_M2MF_CLASS;
+         break;
+      }
+      sclass->sclass.oclass[idx].minver = -1;
+      sclass->sclass.oclass[idx].maxver = -1;
+      idx++;
+      if (device_info.chip_id >= 0xe0) {
+         switch (device_info.chip_id & ~0xf) {
+         case 0x170:
+            sclass->sclass.oclass[idx].oclass = AMPERE_DMA_COPY_A;
+            break;
+         case 0x160:
+            sclass->sclass.oclass[idx].oclass = TURING_DMA_COPY_A;
+            break;
+         case 0x140:
+            sclass->sclass.oclass[idx].oclass = VOLTA_DMA_COPY_A;
+            break;
+         case 0x130:
+            sclass->sclass.oclass[idx].oclass = PASCAL_DMA_COPY_A;
+            break;
+         case 0x120:
+         case 0x110:
+            sclass->sclass.oclass[idx].oclass = MAXWELL_DMA_COPY_A;
+            break;
+         case 0x100:
+         case 0xf0:
+         case 0xe0:
+            sclass->sclass.oclass[idx].oclass = KEPLER_DMA_COPY_A;
+            break;
+         }
+         sclass->sclass.oclass[idx].minver = -1;
+         sclass->sclass.oclass[idx].maxver = -1;
+         idx++;
+      }
+      /* 2d */
+      if (device_info.chip_id >= 0x50) {
+         if (device_info.chip_id <= 0xa0)
+            sclass->sclass.oclass[idx].oclass = NV50_2D_CLASS;
+         else
+            sclass->sclass.oclass[idx].oclass = NVC0_2D_CLASS;
+
+         sclass->sclass.oclass[idx].minver = -1;
+         sclass->sclass.oclass[idx].maxver = -1;
+         idx++;
+      }
+      /* 3d */
+      switch (device_info.chip_id & ~0xf) {
+      case 0x170:
+         sclass->sclass.oclass[idx].oclass = GA102_3D_CLASS;
+         break;
+      case 0x160:
+         sclass->sclass.oclass[idx].oclass = TU102_3D_CLASS;
+         break;
+      case 0x140:
+         sclass->sclass.oclass[idx].oclass = GV100_3D_CLASS;
+         break;
+      case 0x130:
+         switch (device_info.chip_id) {
+         case 0x130:
+         case 0x13b:
+            sclass->sclass.oclass[idx].oclass = GP100_3D_CLASS;
+            break;
+         default:
+            sclass->sclass.oclass[idx].oclass = GP102_3D_CLASS;
+            break;
+         }
+         break;
+      case 0x120:
+         sclass->sclass.oclass[idx].oclass = GM200_3D_CLASS;
+         break;
+      case 0x110:
+         sclass->sclass.oclass[idx].oclass = GM107_3D_CLASS;
+         break;
+      case 0x100:
+      case 0xf0:
+         sclass->sclass.oclass[idx].oclass = NVF0_3D_CLASS;
+         break;
+      case 0xe0:
+         switch (device_info.chip_id) {
+         case 0xea:
+            sclass->sclass.oclass[idx].oclass = NVEA_3D_CLASS;
+            break;
+         default:
+            sclass->sclass.oclass[idx].oclass = NVE4_3D_CLASS;
+            break;
+         }
+         break;
+      case 0xd0:
+         sclass->sclass.oclass[idx].oclass = NVC8_3D_CLASS;
+         break;
+      default:
+      case 0xc0:
+         switch (device_info.chip_id) {
+         case 0xc8:
+            sclass->sclass.oclass[idx].oclass = NVC8_3D_CLASS;
+            break;
+         case 0xc1:
+            sclass->sclass.oclass[idx].oclass = NVC1_3D_CLASS;
+            break;
+         default:
+            sclass->sclass.oclass[idx].oclass = NVC0_3D_CLASS;
+            break;
+         }
+         break;
+      }
+      sclass->sclass.oclass[idx].minver = -1;
+      sclass->sclass.oclass[idx].maxver = -1;
+      idx++;
+      switch (device_info.chip_id & ~0xf) {
+      case 0x170:
+         sclass->sclass.oclass[idx].oclass = GA102_COMPUTE_CLASS;
+         break;
+      case 0x160:
+         sclass->sclass.oclass[idx].oclass = TU102_COMPUTE_CLASS;
+         break;
+      case 0x140:
+         sclass->sclass.oclass[idx].oclass = GV100_COMPUTE_CLASS;
+         break;
+      case 0x130:
+         switch (device_info.chip_id) {
+         case 0x130:
+         case 0x13b:
+            sclass->sclass.oclass[idx].oclass = GP100_COMPUTE_CLASS;
+            break;
+         default:
+            sclass->sclass.oclass[idx].oclass = GP104_COMPUTE_CLASS;
+            break;
+         }
+         break;
+      case 0x120:
+         sclass->sclass.oclass[idx].oclass = GM200_COMPUTE_CLASS;
+         break;
+      case 0x110:
+         sclass->sclass.oclass[idx].oclass = GM107_COMPUTE_CLASS;
+         break;
+      case 0x100:
+      case 0xf0:
+         sclass->sclass.oclass[idx].oclass = NVF0_COMPUTE_CLASS;
+         break;
+      case 0xe0:
+         sclass->sclass.oclass[idx].oclass = NVE4_COMPUTE_CLASS;
+         break;
+      default:
+         sclass->sclass.oclass[idx].oclass = NVC0_COMPUTE_CLASS;
+         break;
+      }
+      sclass->sclass.oclass[idx].minver = -1;
+      sclass->sclass.oclass[idx].maxver = -1;
+      break;
+   }
+   default:
+      break;
+   }
+
+   return 0;
+}
+
+static ioctl_fn_t driver_ioctls[] = {
+   [DRM_NOUVEAU_GETPARAM] = nouveau_ioctl_get_param,
+   [DRM_NOUVEAU_NVIF] = nouveau_ioctl_nvif,
+   [DRM_NOUVEAU_CHANNEL_ALLOC] = nouveau_ioctl_channel_alloc,
+   [DRM_NOUVEAU_CHANNEL_FREE] = nouveau_ioctl_noop,
+   [DRM_NOUVEAU_GROBJ_ALLOC] = nouveau_ioctl_noop,
+   [DRM_NOUVEAU_NOTIFIEROBJ_ALLOC] = nouveau_ioctl_noop,
+   [DRM_NOUVEAU_GPUOBJ_FREE] = nouveau_ioctl_noop,
+   [DRM_NOUVEAU_GEM_NEW] = nouveau_ioctl_gem_new,
+   [DRM_NOUVEAU_GEM_PUSHBUF] = nouveau_ioctl_gem_pushbuf,
+   [DRM_NOUVEAU_GEM_CPU_PREP] = nouveau_ioctl_noop,
+   [DRM_NOUVEAU_GEM_INFO] = nouveau_ioctl_gem_info,
+   [DRM_NOUVEAU_GEM_CPU_FINI] = nouveau_ioctl_gem_info,
+   [DRM_NOUVEAU_VM_INIT] = nouveau_ioctl_noop,
+   [DRM_NOUVEAU_VM_BIND] = nouveau_ioctl_noop,
+   [DRM_NOUVEAU_EXEC] = nouveau_ioctl_noop,
+   [DRM_NOUVEAU_GET_ZCULL_INFO] = nouveau_ioctl_noop,
+};
+
+static void
+nouveau_driver_get_device_info(void)
+{
+   const char *env = os_get_option("NOUVEAU_CHIPSET");
+
+   if (!env) {
+      device_info.chip_id = 0xf0;
+      return;
+   }
+
+   device_info.chip_id = strtol(env, NULL, 16);
+}
+
+void
+drm_shim_driver_init(void)
+{
+   shim_device.bus_type = DRM_BUS_PCI;
+   shim_device.driver_name = "nouveau";
+   shim_device.driver_ioctls = driver_ioctls;
+   shim_device.driver_ioctl_count = ARRAY_SIZE(driver_ioctls);
+
+   shim_device.version_major = 1;
+   shim_device.version_minor = 3;
+   shim_device.version_patchlevel = 1;
+
+   nouveau_driver_get_device_info();
+
+   /* Ask userspace to consider all fences completed. */
+   os_set_option("NOUVEAU_DISABLE_FENCES", "true", true);
+
+   /* nothing looks at the pci id, so fix it to a GTX 780 */
+   static const char uevent_content[] =
+      "DRIVER=nouveau\n"
+      "PCI_CLASS=30000\n"
+      "PCI_ID=10de:1004\n"
+      "PCI_SUBSYS_ID=1028:075B\n"
+      "PCI_SLOT_NAME=0000:01:00.0\n"
+      "MODALIAS=pci:v000010ded00005916sv00001028sd0000075Bbc03sc00i00\n";
+   drm_shim_override_file(uevent_content,
+                          "/sys/dev/char/%d:%d/device/uevent",
+                          DRM_MAJOR, render_node_minor);
+   drm_shim_override_file("0x0\n",
+                          "/sys/dev/char/%d:%d/device/revision",
+                          DRM_MAJOR, render_node_minor);
+   drm_shim_override_file("0x10de",
+                          "/sys/dev/char/%d:%d/device/vendor",
+                          DRM_MAJOR, render_node_minor);
+   drm_shim_override_file("0x10de",
+                          "/sys/devices/pci0000:00/0000:01:00.0/vendor");
+   drm_shim_override_file("0x1004",
+                          "/sys/dev/char/%d:%d/device/device",
+                          DRM_MAJOR, render_node_minor);
+   drm_shim_override_file("0x1004",
+                          "/sys/devices/pci0000:00/0000:01:00.0/device");
+   drm_shim_override_file("0x1234",
+                          "/sys/dev/char/%d:%d/device/subsystem_vendor",
+                          DRM_MAJOR, render_node_minor);
+   drm_shim_override_file("0x1234",
+                          "/sys/devices/pci0000:00/0000:01:00.0/subsystem_vendor");
+   drm_shim_override_file("0x1234",
+                          "/sys/dev/char/%d:%d/device/subsystem_device",
+                          DRM_MAJOR, render_node_minor);
+   drm_shim_override_file("0x1234",
+                          "/sys/devices/pci0000:00/0000:01:00.0/subsystem_device");
+}
@@ -0,0 +1 @@
+inc_nouveau_drm = include_directories('.')
@@ -0,0 +1,45 @@
+#ifndef __NVIF_CL0080_H__
+#define __NVIF_CL0080_H__
+
+struct nv_device_v0 {
+	__u8  version;
+	__u8  pad01[7];
+	__u64 device;	/* device identifier, ~0 for client default */
+};
+
+#define NV_DEVICE_V0_INFO                                                  0x00
+#define NV_DEVICE_V0_TIME                                                  0x01
+
+struct nv_device_info_v0 {
+	__u8  version;
+#define NV_DEVICE_INFO_V0_IGP                                              0x00
+#define NV_DEVICE_INFO_V0_PCI                                              0x01
+#define NV_DEVICE_INFO_V0_AGP                                              0x02
+#define NV_DEVICE_INFO_V0_PCIE                                             0x03
+#define NV_DEVICE_INFO_V0_SOC                                              0x04
+	__u8  platform;
+	__u16 chipset;	/* from NV_PMC_BOOT_0 */
+	__u8  revision;	/* from NV_PMC_BOOT_0 */
+#define NV_DEVICE_INFO_V0_TNT                                              0x01
+#define NV_DEVICE_INFO_V0_CELSIUS                                          0x02
+#define NV_DEVICE_INFO_V0_KELVIN                                           0x03
+#define NV_DEVICE_INFO_V0_RANKINE                                          0x04
+#define NV_DEVICE_INFO_V0_CURIE                                            0x05
+#define NV_DEVICE_INFO_V0_TESLA                                            0x06
+#define NV_DEVICE_INFO_V0_FERMI                                            0x07
+#define NV_DEVICE_INFO_V0_KEPLER                                           0x08
+#define NV_DEVICE_INFO_V0_MAXWELL                                          0x09
+	__u8  family;
+	__u8  pad06[2];
+	__u64 ram_size;
+	__u64 ram_user;
+	char  chip[16];
+	char  name[64];
+};
+
+struct nv_device_time_v0 {
+	__u8  version;
+	__u8  pad01[7];
+	__u64 time;
+};
+#endif
@@ -0,0 +1,141 @@
+#ifndef __NVIF_CLASS_H__
+#define __NVIF_CLASS_H__
+
+/* these class numbers are made up by us, and not nvidia-assigned */
+#define NVIF_CLASS_CONTROL                                    /* if0001.h */ -1
+#define NVIF_CLASS_PERFMON                                    /* if0002.h */ -2
+#define NVIF_CLASS_PERFDOM                                    /* if0003.h */ -3
+#define NVIF_CLASS_SW_NV04                                    /* if0004.h */ -4
+#define NVIF_CLASS_SW_NV10                                    /* if0005.h */ -5
+#define NVIF_CLASS_SW_NV50                                    /* if0005.h */ -6
+#define NVIF_CLASS_SW_GF100                                   /* if0005.h */ -7
+
+/* the below match nvidia-assigned (either in hw, or sw) class numbers */
+#define NV_DEVICE                                     /* cl0080.h */ 0x00000080
+
+#define NV_DMA_FROM_MEMORY                            /* cl0002.h */ 0x00000002
+#define NV_DMA_TO_MEMORY                              /* cl0002.h */ 0x00000003
+#define NV_DMA_IN_MEMORY                              /* cl0002.h */ 0x0000003d
+
+#define FERMI_TWOD_A                                                 0x0000902d
+
+#define FERMI_MEMORY_TO_MEMORY_FORMAT_A                              0x00009039
+
+#define KEPLER_INLINE_TO_MEMORY_A                                    0x0000a040
+#define KEPLER_INLINE_TO_MEMORY_B                                    0x0000a140
+
+#define NV04_DISP                                     /* cl0046.h */ 0x00000046
+
+#define NV03_CHANNEL_DMA                              /* cl506b.h */ 0x0000006b
+#define NV10_CHANNEL_DMA                              /* cl506b.h */ 0x0000006e
+#define NV17_CHANNEL_DMA                              /* cl506b.h */ 0x0000176e
+#define NV40_CHANNEL_DMA                              /* cl506b.h */ 0x0000406e
+#define NV50_CHANNEL_DMA                              /* cl506e.h */ 0x0000506e
+#define G82_CHANNEL_DMA                               /* cl826e.h */ 0x0000826e
+
+#define NV50_CHANNEL_GPFIFO                           /* cl506f.h */ 0x0000506f
+#define G82_CHANNEL_GPFIFO                            /* cl826f.h */ 0x0000826f
+#define FERMI_CHANNEL_GPFIFO                          /* cl906f.h */ 0x0000906f
+#define KEPLER_CHANNEL_GPFIFO_A                       /* cla06f.h */ 0x0000a06f
+#define MAXWELL_CHANNEL_GPFIFO_A                      /* cla06f.h */ 0x0000b06f
+
+#define NV50_DISP                                     /* cl5070.h */ 0x00005070
+#define G82_DISP                                      /* cl5070.h */ 0x00008270
+#define GT200_DISP                                    /* cl5070.h */ 0x00008370
+#define GT214_DISP                                    /* cl5070.h */ 0x00008570
+#define GT206_DISP                                    /* cl5070.h */ 0x00008870
+#define GF110_DISP                                    /* cl5070.h */ 0x00009070
+#define GK104_DISP                                    /* cl5070.h */ 0x00009170
+#define GK110_DISP                                    /* cl5070.h */ 0x00009270
+#define GM107_DISP                                    /* cl5070.h */ 0x00009470
+#define GM204_DISP                                    /* cl5070.h */ 0x00009570
+
+#define NV31_MPEG                                                    0x00003174
+#define G82_MPEG                                                     0x00008274
+
+#define NV74_VP2                                                     0x00007476
+
+#define NV50_DISP_CURSOR                              /* cl507a.h */ 0x0000507a
+#define G82_DISP_CURSOR                               /* cl507a.h */ 0x0000827a
+#define GT214_DISP_CURSOR                             /* cl507a.h */ 0x0000857a
+#define GF110_DISP_CURSOR                             /* cl507a.h */ 0x0000907a
+#define GK104_DISP_CURSOR                             /* cl507a.h */ 0x0000917a
+
+#define NV50_DISP_OVERLAY                             /* cl507b.h */ 0x0000507b
+#define G82_DISP_OVERLAY                              /* cl507b.h */ 0x0000827b
+#define GT214_DISP_OVERLAY                            /* cl507b.h */ 0x0000857b
+#define GF110_DISP_OVERLAY                            /* cl507b.h */ 0x0000907b
+#define GK104_DISP_OVERLAY                            /* cl507b.h */ 0x0000917b
+
+#define NV50_DISP_BASE_CHANNEL_DMA                    /* cl507c.h */ 0x0000507c
+#define G82_DISP_BASE_CHANNEL_DMA                     /* cl507c.h */ 0x0000827c
+#define GT200_DISP_BASE_CHANNEL_DMA                   /* cl507c.h */ 0x0000837c
+#define GT214_DISP_BASE_CHANNEL_DMA                   /* cl507c.h */ 0x0000857c
+#define GF110_DISP_BASE_CHANNEL_DMA                   /* cl507c.h */ 0x0000907c
+#define GK104_DISP_BASE_CHANNEL_DMA                   /* cl507c.h */ 0x0000917c
+#define GK110_DISP_BASE_CHANNEL_DMA                   /* cl507c.h */ 0x0000927c
+
+#define NV50_DISP_CORE_CHANNEL_DMA                    /* cl507d.h */ 0x0000507d
+#define G82_DISP_CORE_CHANNEL_DMA                     /* cl507d.h */ 0x0000827d
+#define GT200_DISP_CORE_CHANNEL_DMA                   /* cl507d.h */ 0x0000837d
+#define GT214_DISP_CORE_CHANNEL_DMA                   /* cl507d.h */ 0x0000857d
+#define GT206_DISP_CORE_CHANNEL_DMA                   /* cl507d.h */ 0x0000887d
+#define GF110_DISP_CORE_CHANNEL_DMA                   /* cl507d.h */ 0x0000907d
+#define GK104_DISP_CORE_CHANNEL_DMA                   /* cl507d.h */ 0x0000917d
+#define GK110_DISP_CORE_CHANNEL_DMA                   /* cl507d.h */ 0x0000927d
+#define GM107_DISP_CORE_CHANNEL_DMA                   /* cl507d.h */ 0x0000947d
+#define GM204_DISP_CORE_CHANNEL_DMA                   /* cl507d.h */ 0x0000957d
+
+#define NV50_DISP_OVERLAY_CHANNEL_DMA                 /* cl507e.h */ 0x0000507e
+#define G82_DISP_OVERLAY_CHANNEL_DMA                  /* cl507e.h */ 0x0000827e
+#define GT200_DISP_OVERLAY_CHANNEL_DMA                /* cl507e.h */ 0x0000837e
+#define GT214_DISP_OVERLAY_CHANNEL_DMA                /* cl507e.h */ 0x0000857e
+#define GF110_DISP_OVERLAY_CONTROL_DMA                /* cl507e.h */ 0x0000907e
+#define GK104_DISP_OVERLAY_CONTROL_DMA                /* cl507e.h */ 0x0000917e
+
+#define FERMI_A                                       /* cl9097.h */ 0x00009097
+#define FERMI_B                                       /* cl9097.h */ 0x00009197
+#define FERMI_C                                       /* cl9097.h */ 0x00009297
+
+#define KEPLER_A                                      /* cl9097.h */ 0x0000a097
+#define KEPLER_B                                      /* cl9097.h */ 0x0000a197
+#define KEPLER_C                                      /* cl9097.h */ 0x0000a297
+
+#define MAXWELL_A                                     /* cl9097.h */ 0x0000b097
+#define MAXWELL_B                                     /* cl9097.h */ 0x0000b197
+
+#define NV74_BSP                                                     0x000074b0
+
+#define GT212_MSVLD                                                  0x000085b1
+#define IGT21A_MSVLD                                                 0x000086b1
+#define G98_MSVLD                                                    0x000088b1
+#define GF100_MSVLD                                                  0x000090b1
+#define GK104_MSVLD                                                  0x000095b1
+
+#define GT212_MSPDEC                                                 0x000085b2
+#define G98_MSPDEC                                                   0x000088b2
+#define GF100_MSPDEC                                                 0x000090b2
+#define GK104_MSPDEC                                                 0x000095b2
+
+#define GT212_MSPPP                                                  0x000085b3
+#define G98_MSPPP                                                    0x000088b3
+#define GF100_MSPPP                                                  0x000090b3
+
+#define G98_SEC                                                      0x000088b4
+
+#define GT212_DMA                                                    0x000085b5
+#define FERMI_DMA                                                    0x000090b5
+#define KEPLER_DMA_COPY_A                                            0x0000a0b5
+#define MAXWELL_DMA_COPY_A                                           0x0000b0b5
+
+#define FERMI_DECOMPRESS                                             0x000090b8
+
+#define FERMI_COMPUTE_A                                              0x000090c0
+#define FERMI_COMPUTE_B                                              0x000091c0
+#define KEPLER_COMPUTE_A                                             0x0000a0c0
+#define KEPLER_COMPUTE_B                                             0x0000a1c0
+#define MAXWELL_COMPUTE_A                                            0x0000b0c0
+#define MAXWELL_COMPUTE_B                                            0x0000b1c0
+
+#define NV74_CIPHER                                                  0x000074c1
+#endif
@@ -0,0 +1,132 @@
+#ifndef __NVIF_IOCTL_H__
+#define __NVIF_IOCTL_H__
+
+#define NVIF_VERSION_LATEST                               0x0000000000000000ULL
+
+struct nvif_ioctl_v0 {
+	__u8  version;
+#define NVIF_IOCTL_V0_NOP                                                  0x00
+#define NVIF_IOCTL_V0_SCLASS                                               0x01
+#define NVIF_IOCTL_V0_NEW                                                  0x02
+#define NVIF_IOCTL_V0_DEL                                                  0x03
+#define NVIF_IOCTL_V0_MTHD                                                 0x04
+#define NVIF_IOCTL_V0_RD                                                   0x05
+#define NVIF_IOCTL_V0_WR                                                   0x06
+#define NVIF_IOCTL_V0_MAP                                                  0x07
+#define NVIF_IOCTL_V0_UNMAP                                                0x08
+#define NVIF_IOCTL_V0_NTFY_NEW                                             0x09
+#define NVIF_IOCTL_V0_NTFY_DEL                                             0x0a
+#define NVIF_IOCTL_V0_NTFY_GET                                             0x0b
+#define NVIF_IOCTL_V0_NTFY_PUT                                             0x0c
+	__u8  type;
+	__u8  pad02[4];
+#define NVIF_IOCTL_V0_OWNER_NVIF                                           0x00
+#define NVIF_IOCTL_V0_OWNER_ANY                                            0xff
+	__u8  owner;
+#define NVIF_IOCTL_V0_ROUTE_NVIF                                           0x00
+#define NVIF_IOCTL_V0_ROUTE_HIDDEN                                         0xff
+	__u8  route;
+	__u64 token;
+	__u64 object;
+	__u8  data[];		/* ioctl data (below) */
+};
+
+struct nvif_ioctl_nop_v0 {
+	__u64 version;
+};
+
+struct nvif_ioctl_sclass_v0 {
+	/* nvif_ioctl ... */
+	__u8  version;
+	__u8  count;
+	__u8  pad02[6];
+	struct nvif_ioctl_sclass_oclass_v0 {
+		__s32 oclass;
+		__s16 minver;
+		__s16 maxver;
+	} oclass[];
+};
+
+struct nvif_ioctl_new_v0 {
+	/* nvif_ioctl ... */
+	__u8  version;
+	__u8  pad01[6];
+	__u8  route;
+	__u64 token;
+	__u64 object;
+	__u32 handle;
+	__s32 oclass;
+	__u8  data[];		/* class data (class.h) */
+};
+
+struct nvif_ioctl_del {
+};
+
+struct nvif_ioctl_rd_v0 {
+	/* nvif_ioctl ... */
+	__u8  version;
+	__u8  size;
+	__u8  pad02[2];
+	__u32 data;
+	__u64 addr;
+};
+
+struct nvif_ioctl_wr_v0 {
+	/* nvif_ioctl ... */
+	__u8  version;
+	__u8  size;
+	__u8  pad02[2];
+	__u32 data;
+	__u64 addr;
+};
+
+struct nvif_ioctl_map_v0 {
+	/* nvif_ioctl ... */
+	__u8  version;
+	__u8  pad01[3];
+	__u32 length;
+	__u64 handle;
+};
+
+struct nvif_ioctl_unmap {
+};
+
+struct nvif_ioctl_ntfy_new_v0 {
+	/* nvif_ioctl ... */
+	__u8  version;
+	__u8  event;
+	__u8  index;
+	__u8  pad03[5];
+	__u8  data[];		/* event request data (event.h) */
+};
+
+struct nvif_ioctl_ntfy_del_v0 {
+	/* nvif_ioctl ... */
+	__u8  version;
+	__u8  index;
+	__u8  pad02[6];
+};
+
+struct nvif_ioctl_ntfy_get_v0 {
+	/* nvif_ioctl ... */
+	__u8  version;
+	__u8  index;
+	__u8  pad02[6];
+};
+
+struct nvif_ioctl_ntfy_put_v0 {
+	/* nvif_ioctl ... */
+	__u8  version;
+	__u8  index;
+	__u8  pad02[6];
+};
+
+struct nvif_ioctl_mthd_v0 {
+	/* nvif_ioctl ... */
+	__u8  version;
+	__u8  method;
+	__u8  pad02[6];
+	__u8  data[];		/* method data (class.h) */
+};
+
+#endif