wip: crocus/rusticl/elk/hasvk local changes before h264 cherry-pick

src/gallium/drivers/crocus/crocus_program.c

@@ -714,6 +714,7 @@ static const char *surface_group_names[] = {
    [CROCUS_SURFACE_GROUP_UBO]                = "ubo",
    [CROCUS_SURFACE_GROUP_SSBO]               = "ssbo",
    [CROCUS_SURFACE_GROUP_IMAGE]              = "image",
+   [CROCUS_SURFACE_GROUP_GLOBAL]             = "global (IVB)",
 };
 
 static void
@@ -842,7 +843,8 @@ crocus_setup_binding_table(const struct intel_device_info *devinfo,
          bt->used_mask[CROCUS_SURFACE_GROUP_RENDER_TARGET_READ] =
             BITFIELD64_MASK(num_render_targets);
       }
-   } else if (info->stage == MESA_SHADER_COMPUTE) {
+   } else if (info->stage == MESA_SHADER_COMPUTE ||
+              info->stage == MESA_SHADER_KERNEL) {
       bt->sizes[CROCUS_SURFACE_GROUP_CS_WORK_GROUPS] = 1;
    } else if (info->stage == MESA_SHADER_GEOMETRY) {
       /* In gfx6 we reserve the first ELK_MAX_SOL_BINDINGS entries for transform
@@ -2504,6 +2506,335 @@ crocus_update_compiled_shaders(struct crocus_context *ice)
    return true;
 }
 
+/* IVB Gen7 doesn't support stateless global memory access (BTI 255 is only
+ * valid on Haswell+ / Gen7.5+). Instead of a dedicated global-memory BTI
+ * (limited to 64 slots, and unable to distinguish multiple __global buffer
+ * arguments since their kernel_input handles all resolve to address 0),
+ * redirect every load_global/store_global to load_ssbo/store_ssbo.
+ *
+ * The address of a global access is always kernel_input UBO offset N
+ * (optionally + a relative offset expression). N corresponds 1:1, in
+ * increasing order, to rusticl's start_slot ordering in
+ * crocus_set_global_binding() -- both are "the Nth __global buffer
+ * argument that's actually used", skipping dead/scalar arguments. So we
+ * assign SSBO index = rank of N among all distinct offsets seen.
+ */
+/* Pass 1 helper: find N in load_ubo(0,N), without building any new
+ * instructions (safe to call speculatively during discovery). Handles
+ * arbitrarily nested iadd chains, e.g. iadd(iadd(load_ubo(0,N), i*8), 4)
+ * which occurs for struct field access (out[i].b). */
+/* Recursively determines whether following iadd chains from `def`
+ * eventually reaches a load_ubo intrinsic. Unlike a shallow type check,
+ * this actually walks the chain so mul/bcsel/etc. on the *other* branch
+ * don't cause a false positive. */
+static bool
+crocus_def_leads_to_ubo(nir_def *def)
+{
+   nir_instr *instr = nir_def_instr(def);
+
+   if (instr->type == nir_instr_type_intrinsic &&
+       nir_instr_as_intrinsic(instr)->intrinsic == nir_intrinsic_load_ubo)
+      return true;
+
+   if (instr->type == nir_instr_type_phi) {
+      nir_phi_instr *phi = nir_instr_as_phi(instr);
+      nir_foreach_phi_src(phi_src, phi) {
+         if (!crocus_def_leads_to_ubo(phi_src->src.ssa))
+            return false;
+      }
+      return true;
+   }
+
+   if (instr->type != nir_instr_type_alu)
+      return false;
+
+   nir_alu_instr *alu = nir_instr_as_alu(instr);
+   if (alu->op != nir_op_iadd)
+      return false;
+
+   nir_def *lhs = alu->src[0].src.ssa;
+   nir_def *rhs = alu->src[1].src.ssa;
+
+   return crocus_def_leads_to_ubo(lhs) || crocus_def_leads_to_ubo(rhs);
+}
+
+static bool
+crocus_global_addr_find_ubo_offset(nir_src *addr_src, uint32_t *out_offset)
+{
+   nir_def *cur = addr_src->ssa;
+
+   for (;;) {
+      if (nir_def_instr(cur)->type == nir_instr_type_intrinsic &&
+          nir_instr_as_intrinsic(nir_def_instr(cur))->intrinsic ==
+             nir_intrinsic_load_ubo) {
+         nir_intrinsic_instr *ld_ubo = nir_instr_as_intrinsic(nir_def_instr(cur));
+         if (!nir_src_is_const(ld_ubo->src[1]))
+            return false;
+         *out_offset = nir_src_as_uint(ld_ubo->src[1]);
+         return true;
+      }
+
+      /* Control-flow merge: all phi sources must resolve to the
+       * same underlying UBO offset (i.e. the same buffer), only
+       * the runtime-computed relative offset may differ per branch. */
+      if (nir_def_instr(cur)->type == nir_instr_type_phi) {
+         nir_phi_instr *phi = nir_instr_as_phi(nir_def_instr(cur));
+         uint32_t phi_off;
+         bool first = true;
+         nir_foreach_phi_src(phi_src, phi) {
+            uint32_t src_off;
+            if (!crocus_global_addr_find_ubo_offset(&phi_src->src, &src_off))
+               return false;
+            if (first) {
+               phi_off = src_off;
+               first = false;
+            } else if (src_off != phi_off) {
+               return false;
+            }
+         }
+         *out_offset = phi_off;
+         return true;
+      }
+
+      if (nir_def_instr(cur)->type != nir_instr_type_alu)
+         return false;
+
+      nir_alu_instr *alu = nir_instr_as_alu(nir_def_instr(cur));
+      if (alu->op != nir_op_iadd)
+         return false;
+
+      nir_def *lhs = alu->src[0].src.ssa;
+      nir_def *rhs = alu->src[1].src.ssa;
+
+      if (crocus_def_leads_to_ubo(lhs))
+         cur = lhs;
+      else if (crocus_def_leads_to_ubo(rhs))
+         cur = rhs;
+      else
+         return false;
+   }
+}
+
+/* Pass 2 helper: rebuild the full relative offset expression (sum of
+ * everything peeled off while descending to the load_ubo leaf). Must
+ * only be called with b.cursor already positioned before the real
+ * instruction being rewritten. */
+static nir_def *
+crocus_global_addr_build_rel_offset(nir_builder *b, nir_src *addr_src)
+{
+   nir_def *cur = addr_src->ssa;
+   nir_def *sum = NULL;
+
+   for (;;) {
+      if (nir_def_instr(cur)->type == nir_instr_type_intrinsic &&
+          nir_instr_as_intrinsic(nir_def_instr(cur))->intrinsic ==
+             nir_intrinsic_load_ubo) {
+         return sum ? sum : nir_imm_int(b, 0);
+      }
+
+      if (nir_def_instr(cur)->type == nir_instr_type_phi) {
+         nir_phi_instr *phi = nir_instr_as_phi(nir_def_instr(cur));
+         nir_phi_instr *new_phi = nir_phi_instr_create(b->shader);
+         nir_def_init(&new_phi->instr, &new_phi->def, 1, 32);
+
+         nir_foreach_phi_src(phi_src, phi) {
+            nir_builder pb = *b;
+            pb.cursor = nir_after_block_before_jump(phi_src->pred);
+            nir_def *branch_off =
+               crocus_global_addr_build_rel_offset(&pb, &phi_src->src);
+            nir_phi_instr_add_src(new_phi, phi_src->pred, branch_off);
+         }
+
+         nir_instr_insert_before(&phi->instr, &new_phi->instr);
+         nir_def *phi_result = &new_phi->def;
+         return sum ? nir_iadd(b, sum, phi_result) : phi_result;
+      }
+
+      nir_alu_instr *alu = nir_instr_as_alu(nir_def_instr(cur));
+      nir_def *lhs = alu->src[0].src.ssa;
+      nir_def *rhs = alu->src[1].src.ssa;
+
+      bool follow_lhs = crocus_def_leads_to_ubo(lhs);
+      nir_def *peeled = follow_lhs ? rhs : lhs;
+      sum = sum ? nir_iadd(b, sum, peeled) : peeled;
+      cur = follow_lhs ? lhs : rhs;
+   }
+}
+
+static bool
+crocus_lower_cs_global_to_ssbo(nir_shader *nir)
+{
+   nir_function_impl *impl = nir_shader_get_entrypoint(nir);
+
+   uint32_t offsets[64];
+   unsigned num_offsets = 0;
+
+   /* Pass 1: discover every distinct kernel_input offset used as a
+    * load_global/store_global base. */
+   nir_foreach_block(block, impl) {
+      nir_foreach_instr(instr, block) {
+         if (instr->type != nir_instr_type_intrinsic)
+            continue;
+         nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
+
+         nir_src *addr_src;
+         switch (intrin->intrinsic) {
+         case nir_intrinsic_load_global:
+         case nir_intrinsic_load_global_constant:
+            addr_src = &intrin->src[0];
+            break;
+         case nir_intrinsic_store_global:
+            addr_src = &intrin->src[1];
+            break;
+         case nir_intrinsic_global_atomic:
+         case nir_intrinsic_global_atomic_swap:
+            addr_src = &intrin->src[0];
+            break;
+         default:
+            continue;
+         }
+
+        uint32_t off;
+      if (!crocus_global_addr_find_ubo_offset(addr_src, &off)) {
+      fprintf(stderr, "=== UNRECOGNIZED GLOBAL PATTERN (intrinsic: %s) ===\n",
+             nir_intrinsic_infos[intrin->intrinsic].name);
+      nir_print_instr(&intrin->instr, stderr);
+      fprintf(stderr, "\n--- addr_src def ---\n");
+      nir_print_instr(nir_def_instr(addr_src->ssa), stderr);
+         fprintf(stderr, "\n");
+       fflush(stderr);
+       assert(!"crocus: unrecognized __global address pattern on IVB");
+   }
+      
+
+         bool found = false;
+         for (unsigned i = 0; i < num_offsets; i++) {
+            if (offsets[i] == off) {
+               found = true;
+               break;
+            }
+         }
+         if (!found) {
+            assert(num_offsets < ARRAY_SIZE(offsets));
+            offsets[num_offsets++] = off;
+         }
+      }
+   }
+
+   if (num_offsets == 0)
+      return false;
+
+   /* Sort ascending: SSBO index = rank, matching rusticl's start_slot order */
+   for (unsigned i = 1; i < num_offsets; i++) {
+      uint32_t key = offsets[i];
+      int j = i - 1;
+      while (j >= 0 && offsets[j] > key) {
+         offsets[j + 1] = offsets[j];
+         j--;
+      }
+      offsets[j + 1] = key;
+   }
+
+   /* Pass 2: rewrite load_global/store_global -> load_ssbo/store_ssbo */
+   nir_builder b = nir_builder_create(impl);
+
+   nir_foreach_block_safe(block, impl) {
+      nir_foreach_instr_safe(instr, block) {
+         if (instr->type != nir_instr_type_intrinsic)
+            continue;
+         nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
+
+         nir_src *addr_src;
+         enum { OP_LOAD, OP_STORE, OP_ATOMIC, OP_ATOMIC_SWAP } kind;
+
+         switch (intrin->intrinsic) {
+         case nir_intrinsic_load_global:
+         case nir_intrinsic_load_global_constant:
+               kind = OP_LOAD;
+               addr_src = &intrin->src[0];
+              break;
+         case nir_intrinsic_store_global:
+               kind = OP_STORE;
+               addr_src = &intrin->src[1];
+              break;
+         case nir_intrinsic_global_atomic:
+               kind = OP_ATOMIC;
+               addr_src = &intrin->src[0];
+              break;
+         case nir_intrinsic_global_atomic_swap:
+               kind = OP_ATOMIC_SWAP;
+               addr_src = &intrin->src[0];
+              break;
+            default:
+               continue;
+         }
+
+   b.cursor = nir_before_instr(instr);
+
+   uint32_t off;
+   if (!crocus_global_addr_find_ubo_offset(addr_src, &off))
+      assert(!"crocus: unrecognized __global address pattern on IVB");
+
+   nir_def *rel = crocus_global_addr_build_rel_offset(&b, addr_src);
+
+   unsigned ssbo_idx = 0;
+   bool found = false;
+   for (unsigned i = 0; i < num_offsets; i++) {
+   if (offsets[i] == off) {
+      ssbo_idx = i;
+      found = true;
+      break;
+      }
+   }
+   assert(found);
+
+   nir_def *idx_ssa = nir_imm_int(&b, ssbo_idx);
+   nir_def *off_ssa = rel ? rel : nir_imm_int(&b, 0);
+
+   switch (kind) {
+   case OP_LOAD: {
+   nir_def *result =
+      nir_load_ssbo(&b, intrin->def.num_components,
+                    intrin->def.bit_size, idx_ssa, off_ssa,
+                    .access = nir_intrinsic_access(intrin),
+                    .align_mul = nir_intrinsic_align_mul(intrin),
+                    .align_offset = nir_intrinsic_align_offset(intrin));
+   nir_def_replace(&intrin->def, result);
+   break;
+   }
+   case OP_STORE:
+   nir_store_ssbo(&b, intrin->src[0].ssa, idx_ssa, off_ssa,
+                  .write_mask = nir_intrinsic_write_mask(intrin),
+                  .access = nir_intrinsic_access(intrin),
+                  .align_mul = nir_intrinsic_align_mul(intrin),
+                  .align_offset = nir_intrinsic_align_offset(intrin));
+   nir_instr_remove(instr);
+   break;
+   case OP_ATOMIC: {
+   nir_def *result =
+      nir_ssbo_atomic(&b, intrin->def.bit_size, idx_ssa, off_ssa,
+                      intrin->src[1].ssa,
+                      .atomic_op = nir_intrinsic_atomic_op(intrin));
+   nir_def_replace(&intrin->def, result);
+   break;
+   }
+      case OP_ATOMIC_SWAP: {
+         nir_def *result =
+            nir_ssbo_atomic_swap(&b, intrin->def.bit_size, idx_ssa, off_ssa,
+                                 intrin->src[1].ssa, intrin->src[2].ssa,
+                                 .atomic_op = nir_intrinsic_atomic_op(intrin));
+         nir_def_replace(&intrin->def, result);
+         break;
+      }
+      }
+   }
+   }
+
+   nir->info.num_ssbos = num_offsets;
+   nir_progress(true, impl, nir_metadata_none);
+   return true;
+}
+
 static struct crocus_compiled_shader *
 crocus_compile_cs(struct crocus_context *ice,
                   struct crocus_uncompiled_shader *ish,
@@ -2522,6 +2853,9 @@ crocus_compile_cs(struct crocus_context *ice,
 
    nir_shader *nir = nir_shader_clone(mem_ctx, ish->nir);
 
+   if (devinfo->ver < 8)
+      NIR_PASS(_, nir, crocus_lower_cs_global_to_ssbo);
+
    NIR_PASS(_, nir, elk_nir_lower_cs_intrinsics, devinfo, cs_prog_data);
 
    crocus_setup_uniforms(devinfo, mem_ctx, nir, prog_data, &system_values,
@@ -2620,11 +2954,14 @@ crocus_fill_cs_push_const_buffer(struct elk_cs_prog_data *cs_prog_data,
                                  uint32_t *dst)
 {
    assert(elk_cs_push_const_total_size(cs_prog_data, threads) > 0);
-   assert(cs_prog_data->push.cross_thread.size == 0);
-   assert(cs_prog_data->push.per_thread.dwords == 1);
-   assert(cs_prog_data->base.param[0] == ELK_PARAM_BUILTIN_SUBGROUP_ID);
-   for (unsigned t = 0; t < threads; t++)
-      dst[8 * t] = t;
+
+   /* per_thread: subgroup_id -- cross_thread dwords'den sonra yaz */
+   if (cs_prog_data->push.per_thread.dwords == 1 &&
+       cs_prog_data->base.param[0] == ELK_PARAM_BUILTIN_SUBGROUP_ID) {
+      unsigned cross_dwords = cs_prog_data->push.cross_thread.dwords;
+      for (unsigned t = 0; t < threads; t++)
+         dst[cross_dwords + 8 * t] = t;
+   }
 }
 
 /**
@@ -2906,6 +3243,9 @@ crocus_create_compute_state(struct pipe_context *ctx,
    struct crocus_uncompiled_shader *ish =
       crocus_create_uncompiled_shader(ctx, (void *) state->prog, NULL);
 
+   if (ish->nir->info.stage == MESA_SHADER_KERNEL)
+   ish->nir->info.stage = MESA_SHADER_COMPUTE;   
+
    ish->nos |= (1ull << CROCUS_NOS_TEXTURES);
    // XXX: disallow more than 64KB of shared variables
 
@@ -3112,11 +3452,14 @@ crocus_get_compute_state_info(struct pipe_context *ctx, void *state,
                               struct pipe_compute_state_object_info *info)
 {
    struct crocus_screen *screen = (void *) ctx->screen;
-   info->max_threads = screen->devinfo.max_cs_threads *
-                       screen->devinfo.subslice_total;
+   const struct intel_device_info *devinfo = &screen->devinfo;
+
+   const uint32_t simd_width = devinfo->ver >= 8 ? 32 : 8;
+
+   info->max_threads = simd_width * devinfo->max_cs_workgroup_threads;
    info->private_memory = 0;
-   info->preferred_simd_size = 16;
-   info->simd_sizes = 8 | 16;
+   info->preferred_simd_size = simd_width;
+   info->simd_sizes = devinfo->ver >= 8 ? (8 | 16 | 32) : (8 | 16);
 }
 
 void