wip: crocus/rusticl/elk/hasvk local changes before h264 cherry-pick

src/intel/compiler/elk/elk_compiler.c

@@ -172,6 +172,10 @@ elk_get_compiler_config_value(const struct elk_compiler *compiler)
 unsigned
 elk_prog_data_size(mesa_shader_stage stage)
 {
+   if (stage == MESA_SHADER_KERNEL) {
+      stage = MESA_SHADER_COMPUTE;
+   }
+
    static const size_t stage_sizes[] = {
       [MESA_SHADER_VERTEX]       = sizeof(struct elk_vs_prog_data),
       [MESA_SHADER_TESS_CTRL]    = sizeof(struct elk_tcs_prog_data),

src/intel/compiler/elk/elk_compiler.h

@@ -1053,6 +1053,7 @@ struct elk_cs_prog_data {
        * surface indices the CS-specific surfaces
        */
       uint32_t work_groups_start;
+      uint32_t global_start; /* IVB: BTI for full-GTT global surface */
       /** @} */
    } binding_table;
 };

src/intel/compiler/elk/elk_fs_nir.cpp

@@ -5114,6 +5114,31 @@ fs_nir_emit_intrinsic(nir_to_elk_state &ntb,
 
    case nir_intrinsic_load_global:
    case nir_intrinsic_load_global_constant: {
+      if (devinfo->ver < 8) {
+         elk_fs_reg dest = get_nir_def(ntb, instr->def);
+         const unsigned bit_size = instr->def.bit_size;
+         elk_fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS];
+         /* IVB: use dedicated full-GTT surface for global access */
+         const struct elk_cs_prog_data *cs_pd =
+            elk_cs_prog_data(ntb.s.prog_data);
+         srcs[SURFACE_LOGICAL_SRC_SURFACE] = elk_imm_ud(cs_pd->binding_table.global_start);
+         srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(ntb, instr->src[0]);
+         srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = elk_imm_ud(1);
+         srcs[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK] = elk_imm_ud(0);
+         dest.type = elk_reg_type_from_bit_size(bit_size, ELK_REGISTER_TYPE_UD);
+
+         if (bit_size == 32 && nir_intrinsic_align(instr) >= 4) {
+            srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = elk_imm_ud(instr->num_components);
+            elk_fs_inst *inst = bld.emit(ELK_SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL, dest, srcs, SURFACE_LOGICAL_NUM_SRCS);
+            inst->size_written = instr->num_components * s.dispatch_width * 4;
+         } else {
+            srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = elk_imm_ud(bit_size);
+            elk_fs_reg read_result = bld.vgrf(ELK_REGISTER_TYPE_UD);
+            bld.emit(ELK_SHADER_OPCODE_BYTE_SCATTERED_READ_LOGICAL, read_result, srcs, SURFACE_LOGICAL_NUM_SRCS);
+            bld.MOV(dest, subscript(read_result, dest.type, 0));
+         }
+         break;
+      }
       assert(devinfo->ver >= 8);
 
       assert(instr->def.bit_size <= 32);
@@ -5150,13 +5175,34 @@ fs_nir_emit_intrinsic(nir_to_elk_state &ntb,
    }
 
    case nir_intrinsic_store_global: {
+      if (devinfo->ver < 8) {
+         const unsigned bit_size = nir_src_bit_size(instr->src[0]);
+         elk_fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS];
+         /* IVB: use dedicated full-GTT surface for global access */
+         const struct elk_cs_prog_data *cs_pd =
+            elk_cs_prog_data(ntb.s.prog_data);
+         srcs[SURFACE_LOGICAL_SRC_SURFACE] = elk_imm_ud(cs_pd->binding_table.global_start);
+         srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(ntb, instr->src[1]);
+         srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = elk_imm_ud(1);
+         srcs[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK] = elk_imm_ud(1);
+
+         elk_fs_reg data = get_nir_src(ntb, instr->src[0]);
+         data.type = elk_reg_type_from_bit_size(bit_size, ELK_REGISTER_TYPE_UD);
+
+         if (bit_size == 32 && nir_intrinsic_align(instr) >= 4) {
+            srcs[SURFACE_LOGICAL_SRC_DATA] = data;
+            srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = elk_imm_ud(instr->num_components);
+            bld.emit(ELK_SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL, elk_fs_reg(), srcs, SURFACE_LOGICAL_NUM_SRCS);
+         } else {
+            srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = elk_imm_ud(bit_size);
+            srcs[SURFACE_LOGICAL_SRC_DATA] = bld.vgrf(ELK_REGISTER_TYPE_UD);
+            bld.MOV(srcs[SURFACE_LOGICAL_SRC_DATA], data);
+            bld.emit(ELK_SHADER_OPCODE_BYTE_SCATTERED_WRITE_LOGICAL, elk_fs_reg(), srcs, SURFACE_LOGICAL_NUM_SRCS);
+         }
+         break;
+      }
       assert(devinfo->ver >= 8);
-
-      assert(nir_src_bit_size(instr->src[0]) <= 32);
-      assert(nir_intrinsic_write_mask(instr) ==
-             (1u << instr->num_components) - 1);
-      assert(nir_intrinsic_align(instr) > 0);
-
+      
       elk_fs_reg srcs[A64_LOGICAL_NUM_SRCS];
       srcs[A64_LOGICAL_ADDRESS] = get_nir_src(ntb, instr->src[1]);
       srcs[A64_LOGICAL_ENABLE_HELPERS] =
@@ -5523,6 +5569,7 @@ fs_nir_emit_intrinsic(nir_to_elk_state &ntb,
          srcs[SURFACE_LOGICAL_SRC_SURFACE] =
             elk_imm_ud(GFX8_BTI_STATELESS_NON_COHERENT);
       } else {
+         /* IVB: use dedicated full-GTT surface instead of stateless BTI 255 */
          srcs[SURFACE_LOGICAL_SRC_SURFACE] = elk_imm_ud(ELK_BTI_STATELESS);
       }
 
@@ -6241,6 +6288,7 @@ static void
 fs_nir_emit_global_atomic(nir_to_elk_state &ntb, const fs_builder &bld,
                           nir_intrinsic_instr *instr)
 {
+   const intel_device_info *devinfo = ntb.devinfo;
    enum elk_lsc_opcode op = elk_lsc_aop_for_nir_intrinsic(instr);
    int num_data = lsc_op_num_data_values(op);
 
@@ -6262,6 +6310,20 @@ fs_nir_emit_global_atomic(nir_to_elk_state &ntb, const fs_builder &bld,
       data = tmp;
    }
 
+   if (devinfo->ver < 8) {
+      elk_fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS];
+      srcs[SURFACE_LOGICAL_SRC_SURFACE] = elk_imm_ud(ELK_BTI_STATELESS);
+      srcs[SURFACE_LOGICAL_SRC_ADDRESS] = addr;
+      srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = elk_imm_ud(1);
+      srcs[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK] = elk_imm_ud(1);
+      if (num_data >= 1)
+         srcs[SURFACE_LOGICAL_SRC_DATA] = data;
+      srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = elk_imm_ud(op);
+
+      bld.emit(ELK_SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL, dest, srcs, SURFACE_LOGICAL_NUM_SRCS);
+      return;
+   }
+
    elk_fs_reg srcs[A64_LOGICAL_NUM_SRCS];
    srcs[A64_LOGICAL_ADDRESS] = addr;
    srcs[A64_LOGICAL_SRC] = data;
@@ -6853,6 +6915,7 @@ fs_nir_emit_instr(nir_to_elk_state &ntb, nir_instr *instr)
          fs_nir_emit_fs_intrinsic(ntb, nir_instr_as_intrinsic(instr));
          break;
       case MESA_SHADER_COMPUTE:
+      case MESA_SHADER_KERNEL:
          fs_nir_emit_cs_intrinsic(ntb, nir_instr_as_intrinsic(instr));
          break;
       default:

src/intel/compiler/elk/elk_lower_logical_sends.cpp

@@ -1199,8 +1199,7 @@ lower_surface_logical_send(const fs_builder &bld, elk_fs_inst *inst)
    if (is_typed_access || is_stateless) {
       fs_builder ubld = bld.exec_all().group(8, 0);
       header = ubld.vgrf(ELK_REGISTER_TYPE_UD);
-      if (is_stateless) {
-         assert(!is_surface_access);
+      if (is_stateless && !is_surface_access) {
          ubld.emit(ELK_SHADER_OPCODE_SCRATCH_HEADER, header);
       } else {
          ubld.MOV(header, elk_imm_d(0));

src/intel/compiler/elk/elk_nir.c

@@ -766,9 +766,12 @@ elk_nir_optimize(nir_shader *nir, bool is_scalar,
    OPT(nir_remove_dead_variables, nir_var_function_temp, NULL);
 }
 
-static unsigned
-lower_bit_size_callback(const nir_instr *instr, UNUSED void *data)
+static unsigned 
+lower_bit_size_callback(const nir_instr *instr, void *data)
 {
+   const struct elk_compiler *compiler = data;
+   const struct intel_device_info *devinfo = compiler->devinfo;
+
    switch (instr->type) {
    case nir_instr_type_alu: {
       nir_alu_instr *alu = nir_instr_as_alu(instr);
@@ -785,9 +788,19 @@ lower_bit_size_callback(const nir_instr *instr, UNUSED void *data)
       default:
          break;
       }
-
-      if (alu->def.bit_size >= 32)
-         return 0;
+       if (alu->def.bit_size == 16 &&
+           devinfo->ver < 8 &&
+           nir_op_infos[alu->op].output_type == nir_type_float) {
+          /* Gen7 has no native FP16 ALU/register support at all (see the
+           * empty HF entry in gfx7_hw_3src_type[] / gfx7_hw_type[] in
+           * elk_reg_type.c) -- unlike >=Gfx8, it cannot represent a
+           * 16-bit float operand or result under ANY circumstance, not
+           * just when inputs are mixed-width. Lower unconditionally,
+           * regardless of the source bit sizes, or elk_reg_type_to_hw_type()
+           * will hit an unencodable HF register type and assert.
+           */
+          return 32;
+       }
 
       /* Note: nir_op_iabs and nir_op_ineg are not lowered here because the
        * 8-bit ABS or NEG instruction should eventually get copy propagated
@@ -805,7 +818,7 @@ lower_bit_size_callback(const nir_instr *instr, UNUSED void *data)
       case nir_op_ffract:
       case nir_op_fround_even:
       case nir_op_ftrunc:
-         return 32;
+         return alu->def.bit_size < 32 ? 32 : 0;
       case nir_op_frcp:
       case nir_op_frsq:
       case nir_op_fsqrt:
@@ -814,7 +827,13 @@ lower_bit_size_callback(const nir_instr *instr, UNUSED void *data)
       case nir_op_flog2:
       case nir_op_fsin:
       case nir_op_fcos:
-         return 32;
+         /* Same issue as the idiv/umod/fceil group above: these
+          * transcendental ops need >=32-bit precision on Gen7, but if
+          * the source is already >=32-bit, unconditionally returning
+          * 32 here trips the nir_lower_bit_size assert (src->bit_size
+          * < bit_size fails when equal or the source is 64-bit).
+          */
+         return alu->def.bit_size < 32 ? 32 : 0;
       case nir_op_isign:
          UNREACHABLE("Should have been lowered by nir_opt_algebraic.");
       default:
@@ -1310,8 +1329,26 @@ get_mem_access_size_align(nir_intrinsic_op intrin, uint8_t bytes,
                           bool offset_is_const, enum gl_access_qualifier access,
                           const void *cb_data)
 {
+   const struct intel_device_info *devinfo = cb_data;
    const uint32_t align = nir_combined_align(align_mul, align_offset);
 
+   /* IVB (Gen7, verx10=70) has no byte-scattered dataport message --
+    * elk_dp_byte_scattered_rw_desc() asserts verx10 >= 75 (Haswell+).
+    * This is a real HW limitation, not a missing BTI like the global
+    * stateless case. Never request a sub-dword access on this hardware;
+    * always request dword granularity. nir_lower_mem_access_bit_sizes
+    * turns the mismatch into a shift+extract on the load side, and an
+    * atomic iand/ior read-modify-write on the store side (both already
+    * generic in the pass, we just need to ask for them). */
+   if (devinfo->verx10 < 75 && (align < 4 || bytes < 4)) {
+      return (nir_mem_access_size_align) {
+         .bit_size = 32,
+         .num_components = 1,
+         .align = 4,
+         .shift = nir_mem_access_shift_method_scalar,
+      };
+   }
+
    switch (intrin) {
    case nir_intrinsic_load_ssbo:
    case nir_intrinsic_load_shared:
@@ -1401,6 +1438,7 @@ elk_vectorize_lower_mem_access(nir_shader *nir,
       OPT(nir_opt_load_store_vectorize, &options);
    }
 
+   const struct intel_device_info *devinfo = compiler->devinfo;
    nir_lower_mem_access_bit_sizes_options mem_access_options = {
       .modes = nir_var_mem_ssbo |
                nir_var_mem_constant |
@@ -1409,6 +1447,8 @@ elk_vectorize_lower_mem_access(nir_shader *nir,
                nir_var_mem_global |
                nir_var_mem_shared,
       .callback = get_mem_access_size_align,
+      .cb_data = devinfo,
+      .may_lower_unaligned_stores_to_atomics = devinfo->verx10 < 75,
    };
    OPT(nir_lower_mem_access_bit_sizes, &mem_access_options);
 

src/intel/compiler/elk/elk_reg_type.c

@@ -220,10 +220,16 @@ elk_reg_type_to_hw_type(const struct intel_device_info *devinfo,
    }
 
    if (file == ELK_IMMEDIATE_VALUE) {
-      assert(table[type].imm_type != (enum hw_imm_type)INVALID);
+      if (table[type].imm_type == (enum hw_imm_type)INVALID) {
+         fprintf(stderr, "elk_reg_type_to_hw_type: INVALID imm_type! type=%d\n", type);
+         assert(0);
+      }
       return table[type].imm_type;
    } else {
-      assert(table[type].reg_type != (enum hw_reg_type)INVALID);
+      if (table[type].reg_type == (enum hw_reg_type)INVALID) {
+         fprintf(stderr, "elk_reg_type_to_hw_type: INVALID reg_type! type=%d\n", type);
+         assert(0);
+      }
       return table[type].reg_type;
    }
 }