diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..fb043dc --- /dev/null +++ b/flake.lock @@ -0,0 +1,27 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1784007870, + "narHash": "sha256-djcLt/JJphyNt4eDY9XTly+/WbCK5lqWq9lSgCmJkkQ=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "18b9261cb3294b6d2a06d03f96872827b8fe2698", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix index c65f719..968b63f 100644 --- a/flake.nix +++ b/flake.nix @@ -1,58 +1,200 @@ { - inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; - flake-utils.url = "github:numtide/flake-utils"; - }; + description = "intel-hard-graphics / clvk development"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; outputs = + { self, nixpkgs }: + let + system = "x86_64-linux"; + pkgs = import nixpkgs { inherit system; }; + + # HD 4000 (Gen7) PCI ID: 8086:0166 + pciId = "8086:0166"; + defaultClvkBuildDir = "/home/elars/ProjectsHDD/Drivers/clvk/build"; + + commonBuildInputs = with pkgs; [ + llvmPackages_21.clang-unwrapped + llvmPackages_21.llvm.dev + llvmPackages_21.libclang + llvmPackages_21.bintools + python3Packages.packaging + python3Packages.pyyaml + python3Packages.mako + python3 + cargo + meson + ninja + pkg-config + spirv-llvm-translator + spirv-tools + rust-bindgen + rustc + vkmark + vulkan-headers + vulkan-loader + vulkan-tools + shaderc + libpciaccess + opencl-headers + libclc + bison + flex + libdrm + libelf + expat + zlib + zstd + libx11 + ocl-icd + libxcb + cmake + libxext + libxfixes + libxrandr + libxshmfence + wayland + wayland-protocols + wayland-scanner + xorgproto + xorg.libXxf86vm + xorg.xcbutilkeysyms + (python3.withPackages ( + ps: with ps; [ + pyopencl + opencv4 + numpy + ] + )) + clinfo + strace + clpeak + pocl + gdb + ]; + + commonShellVars = '' + export LIBRARY_PATH=${pkgs.llvmPackages_21.clang-unwrapped.lib}/lib:$LIBRARY_PATH + export LIBCLANG_PATH=${pkgs.llvmPackages_21.libclang.lib}/lib + export BINDGEN_EXTRA_CLANG_ARGS="-I${pkgs.llvmPackages_21.clang-unwrapped}/lib/clang/21/include" + export C_INCLUDE_PATH=${pkgs.opencl-headers}/include:$C_INCLUDE_PATH + export LIBRARY_PATH=${pkgs.ocl-icd}/lib:$LIBRARY_PATH + export LD_LIBRARY_PATH="${pkgs.vulkan-loader}/lib:$LD_LIBRARY_PATH" + ''; + + # 32-bit (i686) build için: L4D2 gibi 32-bit oyunlarla test amaçlı. + # clvk/rusticl/OpenCL burada yok — sadece hasvk (vulkan_hasvk) + crocus + # için gereken Vulkan/GL/WSI kütüphaneleri, i686 varyantlarıyla. + pkgs32 = pkgs.pkgsi686Linux; + + i686BuildInputs = with pkgs32; [ + libdrm + libelf + libpciaccess + expat + zlib + zstd + libx11 + libxcb + libxext + libxfixes + libxrandr + libxshmfence + wayland + wayland-protocols + xorgproto + xorg.libXxf86vm + xorg.xcbutilkeysyms + vulkan-headers + vulkan-loader + llvmPackages_21.llvm.dev + llvmPackages_21.clang-unwrapped + llvmPackages_21.libclang + spirv-llvm-translator + spirv-tools + libclc + # crocus + hasvk (with_driver_using_cl) libclc'ye hard-dependency, + # rusticl/opencl kapalı olsa bile gerekiyor + ]; + + # Build araçları (meson/ninja/python/bison/flex/pkg-config vs.) i686'da + # da bulunur ama host mimarisinde çalıştırılmaları yeterli; yine de + # pkgsi686Linux'tan almak pkg-config'in .pc arama yollarını tutarlı + # tutar. cmake/rust burada gerekmiyor (clvk 32-bit hedefi değil). + i686NativeBuildInputs = with pkgs32; [ + meson + ninja + pkg-config + python3 + python3Packages.packaging + python3Packages.pyyaml + python3Packages.mako + bison + flex + wayland-scanner + ]; + in { - self, - nixpkgs, - flake-utils, - }: - flake-utils.lib.eachDefaultSystem ( - system: - let - pkgs = nixpkgs.legacyPackages.${system}; - llvmPkgs = pkgs.llvmPackages_19; - in - { - devShells.default = pkgs.mkShell { - name = "mesa-crocus-opencl"; - - packages = with pkgs; [ - # build tools - meson - ninja - pkg-config - cmake - python3 - python3Packages.packaging - - # LLVM stack - llvmPkgs.llvm.dev - llvmPkgs.clang - llvmPkgs.libclc - - # Rust - rust-bindgen - rustc - cargo - - # Mesa deps - libdrm - expat - zlib - zstd - glslang - libffi - elfutils - ]; - + devShells.${system} = { + # nix develop = intel-hard-graphics shell + default = pkgs.mkShell { + buildInputs = commonBuildInputs; shellHook = '' - export PKG_CONFIG_PATH=${pkgs.llvmPackages_19.libclc}/share/pkgconfig:$PKG_CONFIG_PATH + ${commonShellVars} + export LD_LIBRARY_PATH=$PWD/build/src/gallium/targets/rusticl:$LD_LIBRARY_PATH + export RUSTICL_ENABLE=crocus + + mkdir -p $PWD/.icd-vendors + ln -sf $PWD/build/src/gallium/targets/rusticl/rusticl.icd $PWD/.icd-vendors/rusticl.icd + ln -sf ${pkgs.pocl}/etc/OpenCL/vendors/pocl.icd $PWD/.icd-vendors/pocl.icd + export OCL_ICD_VENDORS=$PWD/.icd-vendors + + echo "=== intel-hard-graphics ===" + echo "OCL_ICD_VENDORS=$OCL_ICD_VENDORS" + echo "RUSTICL_ENABLE=$RUSTICL_ENABLE" + echo "" ''; }; - } - ); + # nix develop .#clvk = clvk shell + clvk = pkgs.mkShell { + buildInputs = commonBuildInputs; + shellHook = '' + ${commonShellVars} + + CLVK_BUILD_DIR="''${CLVK_BUILD_DIR:-${defaultClvkBuildDir}}" + + if [ ! -d "$CLVK_BUILD_DIR" ]; then + echo "UYARI: clvk build dizini bulunamadi: $CLVK_BUILD_DIR" + echo "CLVK_BUILD_DIR ortam degiskeniyle dogru yolu ver, ornek:" + echo " CLVK_BUILD_DIR=/gercek/yol nix develop .#clvk" + else + export LD_LIBRARY_PATH="$CLVK_BUILD_DIR:$LD_LIBRARY_PATH" + fi + + export MESA_VK_DEVICE_SELECT="${pciId}" + + echo "=== clvk ===" + echo "CLVK_BUILD_DIR=$CLVK_BUILD_DIR" + echo "MESA_VK_DEVICE_SELECT=$MESA_VK_DEVICE_SELECT" + echo "" + ''; + }; + # nix develop .#i686 = 32-bit hasvk/crocus build shell (L4D2 testi için) + i686 = pkgs32.mkShell { + buildInputs = i686BuildInputs; + nativeBuildInputs = i686NativeBuildInputs; + shellHook = '' + export MESA_VK_DEVICE_SELECT="${pciId}" + + echo "=== intel-hard-graphics (i686 / 32-bit) ===" + echo "CC=$CC" + echo "meson setup build32 --buildtype=debugoptimized \\" + echo " -Dgallium-drivers=crocus -Dvulkan-drivers=intel_hasvk \\" + echo " -Dplatforms=x11 -Degl=disabled -Dglx=disabled -Dgbm=disabled \\" + echo " -Dopengl=false -Dgles1=disabled -Dgles2=disabled \\" + echo " -Dgallium-rusticl=false -Dmicrosoft-clc=disabled -Dllvm=enabled" + echo "" + ''; + }; + }; + }; } diff --git a/src/gallium/drivers/crocus/crocus_batch.c b/src/gallium/drivers/crocus/crocus_batch.c index 868a72b..d2db3c3 100644 --- a/src/gallium/drivers/crocus/crocus_batch.c +++ b/src/gallium/drivers/crocus/crocus_batch.c @@ -301,7 +301,7 @@ ensure_exec_obj_space(struct crocus_batch *batch, uint32_t count) } } -static struct drm_i915_gem_exec_object2 * +struct drm_i915_gem_exec_object2 * crocus_use_bo(struct crocus_batch *batch, struct crocus_bo *bo, bool writable) { assert(bo->bufmgr == batch->command.bo->bufmgr); @@ -458,7 +458,10 @@ recreate_growing_buffer(struct crocus_batch *batch, struct crocus_screen *screen = batch->screen; struct crocus_bufmgr *bufmgr = screen->bufmgr; grow->bo = crocus_bo_alloc(bufmgr, name, size); - grow->bo->kflags |= EXEC_OBJECT_CAPTURE; + static uint64_t next_grow_pin = 0x20000000ULL; /* 512MB */ + grow->bo->gtt_offset = next_grow_pin; + next_grow_pin += align64(size, 4096); + grow->bo->kflags |= EXEC_OBJECT_CAPTURE | EXEC_OBJECT_PINNED; grow->partial_bo = NULL; grow->partial_bo_map = NULL; grow->partial_bytes = 0; @@ -903,8 +906,9 @@ submit_batch(struct crocus_batch *batch) DBG("BO %d migrated: 0x%" PRIx64 " -> 0x%" PRIx64 "\n", bo->gem_handle, bo->gtt_offset, (uint64_t)batch->validation_list[i].offset); - assert(!(bo->kflags & EXEC_OBJECT_PINNED)); - bo->gtt_offset = batch->validation_list[i].offset; + if (!(bo->kflags & EXEC_OBJECT_PINNED)) { + bo->gtt_offset = batch->validation_list[i].offset; + } } } diff --git a/src/gallium/drivers/crocus/crocus_batch.h b/src/gallium/drivers/crocus/crocus_batch.h index ae2f8aa..4e1efcc 100644 --- a/src/gallium/drivers/crocus/crocus_batch.h +++ b/src/gallium/drivers/crocus/crocus_batch.h @@ -200,6 +200,7 @@ bool crocus_batch_prepare_noop(struct crocus_batch *batch, bool noop_enable); /* Inverted meaning, but using the same bit...emit_reloc will flip it. */ #define RELOC_32BIT EXEC_OBJECT_SUPPORTS_48B_ADDRESS +struct drm_i915_gem_exec_object2 *crocus_use_bo(struct crocus_batch *batch, struct crocus_bo *bo, bool writable); void crocus_use_pinned_bo(struct crocus_batch *batch, struct crocus_bo *bo, bool writable); uint64_t crocus_command_reloc(struct crocus_batch *batch, uint32_t batch_offset, diff --git a/src/gallium/drivers/crocus/crocus_context.h b/src/gallium/drivers/crocus/crocus_context.h index 3efc069..ff495b6 100644 --- a/src/gallium/drivers/crocus/crocus_context.h +++ b/src/gallium/drivers/crocus/crocus_context.h @@ -339,6 +339,7 @@ enum crocus_surface_group { CROCUS_SURFACE_GROUP_IMAGE, CROCUS_SURFACE_GROUP_UBO, CROCUS_SURFACE_GROUP_SSBO, + CROCUS_SURFACE_GROUP_GLOBAL, CROCUS_SURFACE_GROUP_COUNT, }; @@ -608,6 +609,7 @@ struct crocus_context { struct crocus_state_ref grid_size; /** Global bindings for compute */ struct pipe_resource *global_bindings[CROCUS_MAX_GLOBAL_BINDINGS]; + uint32_t *global_binding_handles[CROCUS_MAX_GLOBAL_BINDINGS]; /** * Array of aux usages for drawing, altered to account for any diff --git a/src/gallium/drivers/crocus/crocus_draw.c b/src/gallium/drivers/crocus/crocus_draw.c index 3b3d73d..9879682 100644 --- a/src/gallium/drivers/crocus/crocus_draw.c +++ b/src/gallium/drivers/crocus/crocus_draw.c @@ -517,9 +517,16 @@ crocus_launch_grid(struct pipe_context *ctx, const struct pipe_grid_info *grid) crocus_handle_always_flush_cache(batch); + for (unsigned i = 0; i < CROCUS_MAX_GLOBAL_BINDINGS; i++) { + if (ice->state.global_bindings[i]) { + struct crocus_resource *res = (struct crocus_resource *)ice->state.global_bindings[i]; + crocus_use_bo(batch, res->bo, true); + } + } + screen->vtbl.upload_compute_state(ice, batch, grid); - crocus_handle_always_flush_cache(batch); + crocus_flush_all_caches(batch); ice->state.dirty &= ~CROCUS_ALL_DIRTY_FOR_COMPUTE; ice->state.stage_dirty &= ~CROCUS_ALL_STAGE_DIRTY_FOR_COMPUTE; diff --git a/src/gallium/drivers/crocus/crocus_formats.c b/src/gallium/drivers/crocus/crocus_formats.c index 29b3232..aaebaba 100644 --- a/src/gallium/drivers/crocus/crocus_formats.c +++ b/src/gallium/drivers/crocus/crocus_formats.c @@ -509,10 +509,18 @@ crocus_is_format_supported(struct pipe_screen *pscreen, /* Dataport doesn't support compression, and we can't resolve an MCS * compressed surface. (Buffer images may have sample count of 0.) */ - supported &= sample_count == 0; + bool sc_ok = sample_count == 0; + bool tw_ok = isl_format_supports_typed_writes(devinfo, format); + bool ts_ok = isl_has_matching_typed_storage_image_format(devinfo, format); + supported &= sc_ok && tw_ok && ts_ok; - supported &= isl_format_supports_typed_writes(devinfo, format); - supported &= isl_has_matching_typed_storage_image_format(devinfo, format); + FILE *dbgf = fopen("/tmp/crocus_debug.log", "a"); + if (dbgf) { + fprintf(dbgf, "CROCUS_DBG: shader_image check pformat=%d target=%d sample_count=%u " + "isl_format=%d sc_ok=%d tw_ok=%d ts_ok=%d supported=%d\n", + pformat, target, sample_count, format, sc_ok, tw_ok, ts_ok, supported); + fclose(dbgf); + } } if (usage & PIPE_BIND_SAMPLER_VIEW) { diff --git a/src/gallium/drivers/crocus/crocus_program.c b/src/gallium/drivers/crocus/crocus_program.c index b17ed02..6209d5c 100644 --- a/src/gallium/drivers/crocus/crocus_program.c +++ b/src/gallium/drivers/crocus/crocus_program.c @@ -714,6 +714,7 @@ static const char *surface_group_names[] = { [CROCUS_SURFACE_GROUP_UBO] = "ubo", [CROCUS_SURFACE_GROUP_SSBO] = "ssbo", [CROCUS_SURFACE_GROUP_IMAGE] = "image", + [CROCUS_SURFACE_GROUP_GLOBAL] = "global (IVB)", }; static void @@ -842,7 +843,8 @@ crocus_setup_binding_table(const struct intel_device_info *devinfo, bt->used_mask[CROCUS_SURFACE_GROUP_RENDER_TARGET_READ] = BITFIELD64_MASK(num_render_targets); } - } else if (info->stage == MESA_SHADER_COMPUTE) { + } else if (info->stage == MESA_SHADER_COMPUTE || + info->stage == MESA_SHADER_KERNEL) { bt->sizes[CROCUS_SURFACE_GROUP_CS_WORK_GROUPS] = 1; } else if (info->stage == MESA_SHADER_GEOMETRY) { /* In gfx6 we reserve the first ELK_MAX_SOL_BINDINGS entries for transform @@ -2504,6 +2506,335 @@ crocus_update_compiled_shaders(struct crocus_context *ice) return true; } +/* IVB Gen7 doesn't support stateless global memory access (BTI 255 is only + * valid on Haswell+ / Gen7.5+). Instead of a dedicated global-memory BTI + * (limited to 64 slots, and unable to distinguish multiple __global buffer + * arguments since their kernel_input handles all resolve to address 0), + * redirect every load_global/store_global to load_ssbo/store_ssbo. + * + * The address of a global access is always kernel_input UBO offset N + * (optionally + a relative offset expression). N corresponds 1:1, in + * increasing order, to rusticl's start_slot ordering in + * crocus_set_global_binding() -- both are "the Nth __global buffer + * argument that's actually used", skipping dead/scalar arguments. So we + * assign SSBO index = rank of N among all distinct offsets seen. + */ +/* Pass 1 helper: find N in load_ubo(0,N), without building any new + * instructions (safe to call speculatively during discovery). Handles + * arbitrarily nested iadd chains, e.g. iadd(iadd(load_ubo(0,N), i*8), 4) + * which occurs for struct field access (out[i].b). */ +/* Recursively determines whether following iadd chains from `def` + * eventually reaches a load_ubo intrinsic. Unlike a shallow type check, + * this actually walks the chain so mul/bcsel/etc. on the *other* branch + * don't cause a false positive. */ +static bool +crocus_def_leads_to_ubo(nir_def *def) +{ + nir_instr *instr = nir_def_instr(def); + + if (instr->type == nir_instr_type_intrinsic && + nir_instr_as_intrinsic(instr)->intrinsic == nir_intrinsic_load_ubo) + return true; + + if (instr->type == nir_instr_type_phi) { + nir_phi_instr *phi = nir_instr_as_phi(instr); + nir_foreach_phi_src(phi_src, phi) { + if (!crocus_def_leads_to_ubo(phi_src->src.ssa)) + return false; + } + return true; + } + + if (instr->type != nir_instr_type_alu) + return false; + + nir_alu_instr *alu = nir_instr_as_alu(instr); + if (alu->op != nir_op_iadd) + return false; + + nir_def *lhs = alu->src[0].src.ssa; + nir_def *rhs = alu->src[1].src.ssa; + + return crocus_def_leads_to_ubo(lhs) || crocus_def_leads_to_ubo(rhs); +} + +static bool +crocus_global_addr_find_ubo_offset(nir_src *addr_src, uint32_t *out_offset) +{ + nir_def *cur = addr_src->ssa; + + for (;;) { + if (nir_def_instr(cur)->type == nir_instr_type_intrinsic && + nir_instr_as_intrinsic(nir_def_instr(cur))->intrinsic == + nir_intrinsic_load_ubo) { + nir_intrinsic_instr *ld_ubo = nir_instr_as_intrinsic(nir_def_instr(cur)); + if (!nir_src_is_const(ld_ubo->src[1])) + return false; + *out_offset = nir_src_as_uint(ld_ubo->src[1]); + return true; + } + + /* Control-flow merge: all phi sources must resolve to the + * same underlying UBO offset (i.e. the same buffer), only + * the runtime-computed relative offset may differ per branch. */ + if (nir_def_instr(cur)->type == nir_instr_type_phi) { + nir_phi_instr *phi = nir_instr_as_phi(nir_def_instr(cur)); + uint32_t phi_off; + bool first = true; + nir_foreach_phi_src(phi_src, phi) { + uint32_t src_off; + if (!crocus_global_addr_find_ubo_offset(&phi_src->src, &src_off)) + return false; + if (first) { + phi_off = src_off; + first = false; + } else if (src_off != phi_off) { + return false; + } + } + *out_offset = phi_off; + return true; + } + + if (nir_def_instr(cur)->type != nir_instr_type_alu) + return false; + + nir_alu_instr *alu = nir_instr_as_alu(nir_def_instr(cur)); + if (alu->op != nir_op_iadd) + return false; + + nir_def *lhs = alu->src[0].src.ssa; + nir_def *rhs = alu->src[1].src.ssa; + + if (crocus_def_leads_to_ubo(lhs)) + cur = lhs; + else if (crocus_def_leads_to_ubo(rhs)) + cur = rhs; + else + return false; + } +} + +/* Pass 2 helper: rebuild the full relative offset expression (sum of + * everything peeled off while descending to the load_ubo leaf). Must + * only be called with b.cursor already positioned before the real + * instruction being rewritten. */ +static nir_def * +crocus_global_addr_build_rel_offset(nir_builder *b, nir_src *addr_src) +{ + nir_def *cur = addr_src->ssa; + nir_def *sum = NULL; + + for (;;) { + if (nir_def_instr(cur)->type == nir_instr_type_intrinsic && + nir_instr_as_intrinsic(nir_def_instr(cur))->intrinsic == + nir_intrinsic_load_ubo) { + return sum ? sum : nir_imm_int(b, 0); + } + + if (nir_def_instr(cur)->type == nir_instr_type_phi) { + nir_phi_instr *phi = nir_instr_as_phi(nir_def_instr(cur)); + nir_phi_instr *new_phi = nir_phi_instr_create(b->shader); + nir_def_init(&new_phi->instr, &new_phi->def, 1, 32); + + nir_foreach_phi_src(phi_src, phi) { + nir_builder pb = *b; + pb.cursor = nir_after_block_before_jump(phi_src->pred); + nir_def *branch_off = + crocus_global_addr_build_rel_offset(&pb, &phi_src->src); + nir_phi_instr_add_src(new_phi, phi_src->pred, branch_off); + } + + nir_instr_insert_before(&phi->instr, &new_phi->instr); + nir_def *phi_result = &new_phi->def; + return sum ? nir_iadd(b, sum, phi_result) : phi_result; + } + + nir_alu_instr *alu = nir_instr_as_alu(nir_def_instr(cur)); + nir_def *lhs = alu->src[0].src.ssa; + nir_def *rhs = alu->src[1].src.ssa; + + bool follow_lhs = crocus_def_leads_to_ubo(lhs); + nir_def *peeled = follow_lhs ? rhs : lhs; + sum = sum ? nir_iadd(b, sum, peeled) : peeled; + cur = follow_lhs ? lhs : rhs; + } +} + +static bool +crocus_lower_cs_global_to_ssbo(nir_shader *nir) +{ + nir_function_impl *impl = nir_shader_get_entrypoint(nir); + + uint32_t offsets[64]; + unsigned num_offsets = 0; + + /* Pass 1: discover every distinct kernel_input offset used as a + * load_global/store_global base. */ + nir_foreach_block(block, impl) { + nir_foreach_instr(instr, block) { + if (instr->type != nir_instr_type_intrinsic) + continue; + nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); + + nir_src *addr_src; + switch (intrin->intrinsic) { + case nir_intrinsic_load_global: + case nir_intrinsic_load_global_constant: + addr_src = &intrin->src[0]; + break; + case nir_intrinsic_store_global: + addr_src = &intrin->src[1]; + break; + case nir_intrinsic_global_atomic: + case nir_intrinsic_global_atomic_swap: + addr_src = &intrin->src[0]; + break; + default: + continue; + } + + uint32_t off; + if (!crocus_global_addr_find_ubo_offset(addr_src, &off)) { + fprintf(stderr, "=== UNRECOGNIZED GLOBAL PATTERN (intrinsic: %s) ===\n", + nir_intrinsic_infos[intrin->intrinsic].name); + nir_print_instr(&intrin->instr, stderr); + fprintf(stderr, "\n--- addr_src def ---\n"); + nir_print_instr(nir_def_instr(addr_src->ssa), stderr); + fprintf(stderr, "\n"); + fflush(stderr); + assert(!"crocus: unrecognized __global address pattern on IVB"); + } + + + bool found = false; + for (unsigned i = 0; i < num_offsets; i++) { + if (offsets[i] == off) { + found = true; + break; + } + } + if (!found) { + assert(num_offsets < ARRAY_SIZE(offsets)); + offsets[num_offsets++] = off; + } + } + } + + if (num_offsets == 0) + return false; + + /* Sort ascending: SSBO index = rank, matching rusticl's start_slot order */ + for (unsigned i = 1; i < num_offsets; i++) { + uint32_t key = offsets[i]; + int j = i - 1; + while (j >= 0 && offsets[j] > key) { + offsets[j + 1] = offsets[j]; + j--; + } + offsets[j + 1] = key; + } + + /* Pass 2: rewrite load_global/store_global -> load_ssbo/store_ssbo */ + nir_builder b = nir_builder_create(impl); + + nir_foreach_block_safe(block, impl) { + nir_foreach_instr_safe(instr, block) { + if (instr->type != nir_instr_type_intrinsic) + continue; + nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); + + nir_src *addr_src; + enum { OP_LOAD, OP_STORE, OP_ATOMIC, OP_ATOMIC_SWAP } kind; + + switch (intrin->intrinsic) { + case nir_intrinsic_load_global: + case nir_intrinsic_load_global_constant: + kind = OP_LOAD; + addr_src = &intrin->src[0]; + break; + case nir_intrinsic_store_global: + kind = OP_STORE; + addr_src = &intrin->src[1]; + break; + case nir_intrinsic_global_atomic: + kind = OP_ATOMIC; + addr_src = &intrin->src[0]; + break; + case nir_intrinsic_global_atomic_swap: + kind = OP_ATOMIC_SWAP; + addr_src = &intrin->src[0]; + break; + default: + continue; + } + + b.cursor = nir_before_instr(instr); + + uint32_t off; + if (!crocus_global_addr_find_ubo_offset(addr_src, &off)) + assert(!"crocus: unrecognized __global address pattern on IVB"); + + nir_def *rel = crocus_global_addr_build_rel_offset(&b, addr_src); + + unsigned ssbo_idx = 0; + bool found = false; + for (unsigned i = 0; i < num_offsets; i++) { + if (offsets[i] == off) { + ssbo_idx = i; + found = true; + break; + } + } + assert(found); + + nir_def *idx_ssa = nir_imm_int(&b, ssbo_idx); + nir_def *off_ssa = rel ? rel : nir_imm_int(&b, 0); + + switch (kind) { + case OP_LOAD: { + nir_def *result = + nir_load_ssbo(&b, intrin->def.num_components, + intrin->def.bit_size, idx_ssa, off_ssa, + .access = nir_intrinsic_access(intrin), + .align_mul = nir_intrinsic_align_mul(intrin), + .align_offset = nir_intrinsic_align_offset(intrin)); + nir_def_replace(&intrin->def, result); + break; + } + case OP_STORE: + nir_store_ssbo(&b, intrin->src[0].ssa, idx_ssa, off_ssa, + .write_mask = nir_intrinsic_write_mask(intrin), + .access = nir_intrinsic_access(intrin), + .align_mul = nir_intrinsic_align_mul(intrin), + .align_offset = nir_intrinsic_align_offset(intrin)); + nir_instr_remove(instr); + break; + case OP_ATOMIC: { + nir_def *result = + nir_ssbo_atomic(&b, intrin->def.bit_size, idx_ssa, off_ssa, + intrin->src[1].ssa, + .atomic_op = nir_intrinsic_atomic_op(intrin)); + nir_def_replace(&intrin->def, result); + break; + } + case OP_ATOMIC_SWAP: { + nir_def *result = + nir_ssbo_atomic_swap(&b, intrin->def.bit_size, idx_ssa, off_ssa, + intrin->src[1].ssa, intrin->src[2].ssa, + .atomic_op = nir_intrinsic_atomic_op(intrin)); + nir_def_replace(&intrin->def, result); + break; + } + } + } + } + + nir->info.num_ssbos = num_offsets; + nir_progress(true, impl, nir_metadata_none); + return true; +} + static struct crocus_compiled_shader * crocus_compile_cs(struct crocus_context *ice, struct crocus_uncompiled_shader *ish, @@ -2522,6 +2853,9 @@ crocus_compile_cs(struct crocus_context *ice, nir_shader *nir = nir_shader_clone(mem_ctx, ish->nir); + if (devinfo->ver < 8) + NIR_PASS(_, nir, crocus_lower_cs_global_to_ssbo); + NIR_PASS(_, nir, elk_nir_lower_cs_intrinsics, devinfo, cs_prog_data); crocus_setup_uniforms(devinfo, mem_ctx, nir, prog_data, &system_values, @@ -2620,11 +2954,14 @@ crocus_fill_cs_push_const_buffer(struct elk_cs_prog_data *cs_prog_data, uint32_t *dst) { assert(elk_cs_push_const_total_size(cs_prog_data, threads) > 0); - assert(cs_prog_data->push.cross_thread.size == 0); - assert(cs_prog_data->push.per_thread.dwords == 1); - assert(cs_prog_data->base.param[0] == ELK_PARAM_BUILTIN_SUBGROUP_ID); - for (unsigned t = 0; t < threads; t++) - dst[8 * t] = t; + + /* per_thread: subgroup_id -- cross_thread dwords'den sonra yaz */ + if (cs_prog_data->push.per_thread.dwords == 1 && + cs_prog_data->base.param[0] == ELK_PARAM_BUILTIN_SUBGROUP_ID) { + unsigned cross_dwords = cs_prog_data->push.cross_thread.dwords; + for (unsigned t = 0; t < threads; t++) + dst[cross_dwords + 8 * t] = t; + } } /** @@ -2906,6 +3243,9 @@ crocus_create_compute_state(struct pipe_context *ctx, struct crocus_uncompiled_shader *ish = crocus_create_uncompiled_shader(ctx, (void *) state->prog, NULL); + if (ish->nir->info.stage == MESA_SHADER_KERNEL) + ish->nir->info.stage = MESA_SHADER_COMPUTE; + ish->nos |= (1ull << CROCUS_NOS_TEXTURES); // XXX: disallow more than 64KB of shared variables @@ -3112,11 +3452,14 @@ crocus_get_compute_state_info(struct pipe_context *ctx, void *state, struct pipe_compute_state_object_info *info) { struct crocus_screen *screen = (void *) ctx->screen; - info->max_threads = screen->devinfo.max_cs_threads * - screen->devinfo.subslice_total; + const struct intel_device_info *devinfo = &screen->devinfo; + + const uint32_t simd_width = devinfo->ver >= 8 ? 32 : 8; + + info->max_threads = simd_width * devinfo->max_cs_workgroup_threads; info->private_memory = 0; - info->preferred_simd_size = 16; - info->simd_sizes = 8 | 16; + info->preferred_simd_size = simd_width; + info->simd_sizes = devinfo->ver >= 8 ? (8 | 16 | 32) : (8 | 16); } void diff --git a/src/gallium/drivers/crocus/crocus_screen.c b/src/gallium/drivers/crocus/crocus_screen.c index 6c7022a..55adc6d 100644 --- a/src/gallium/drivers/crocus/crocus_screen.c +++ b/src/gallium/drivers/crocus/crocus_screen.c @@ -203,7 +203,12 @@ crocus_init_compute_caps(struct crocus_screen *screen) if (devinfo->ver < 7) return; - const uint32_t max_invocations = 32 * devinfo->max_cs_workgroup_threads; + /* Gen7 (IVB/BYT/HSW) compute dispatch max SIMD width is 8; + * Gfx9+ can reach SIMD32. Using 32 unconditionally overreports + * the work-group size on Gen7 (2048 instead of the real 512). + */ + const uint32_t simd_width = devinfo->ver >= 8 ? 32 : 8; + const uint32_t max_invocations = simd_width * devinfo->max_cs_workgroup_threads; caps->address_bits = 32; @@ -227,6 +232,22 @@ crocus_init_compute_caps(struct crocus_screen *screen) caps->subgroup_sizes = ELK_SUBGROUP_SIZE; caps->max_variable_threads_per_block = max_invocations; + + caps->max_compute_units = intel_device_info_eu_total(devinfo) > 0 + ? intel_device_info_eu_total(devinfo) + : (devinfo->gt == 2 ? 16 : 6); + + /* Gen7 supports up to 2GB (1 << 31) buffers via stateless access. + * Limit to 1GB to be safe with 32-bit addressing. + */ + caps->max_global_size = 1ULL << 30; /* 1 GB */ + caps->max_mem_alloc_size = 1ULL << 29; /* 512 MB — meets CL min for non-CUSTOM */ + + caps->max_clock_frequency = 1300; /* MHz, approximate for IVB GT2 */ + /* Gen7+ has a real, physically separate Shared Local Memory block + * (accessed via Binding Table Index 254), unlike architectures where + * "local" memory is just a region of global memory. */ + caps->has_dedicated_local_mem = true; } static void @@ -289,6 +310,7 @@ crocus_init_screen_caps(struct crocus_screen *screen) caps->gl_clamp = true; caps->legacy_math_rules = true; caps->native_fence_fd = true; + caps->cl_gl_sharing = true; caps->int64 = caps->shader_ballot = @@ -493,63 +515,7 @@ crocus_query_memory_info(struct pipe_screen *pscreen, info->avail_staging_memory = info->total_staging_memory; } -static int -crocus_get_compute_param(struct pipe_screen *pscreen, - enum pipe_shader_ir ir_type, - enum pipe_compute_cap param, - void *ret) -{ - struct crocus_screen *screen = (struct crocus_screen *) pscreen; - const struct intel_device_info *devinfo = &screen->devinfo; - switch (param) { - case PIPE_COMPUTE_CAP_MAX_COMPUTE_UNITS: - if (ret) { - uint32_t num_eu = devinfo->num_eu_total > 0 ? devinfo->num_eu_total : ((devinfo->gt == 2) ? 16 : 6); - ((uint32_t *)ret)[0] = num_eu; - } - return sizeof(uint32_t); - - case PIPE_COMPUTE_CAP_MAX_THREADS_PER_BLOCK: - if (ret) - ((uint64_t *)ret)[0] = 1024; - return sizeof(uint64_t); - - case PIPE_COMPUTE_CAP_MAX_GRID_SIZE: - if (ret) { - ((uint64_t *)ret)[0] = 65535; - ((uint64_t *)ret)[1] = 65535; - ((uint64_t *)ret)[2] = 65535; - } - return 3 * sizeof(uint64_t); - - case PIPE_COMPUTE_CAP_MAX_BLOCK_SIZE: - if (ret) { - ((uint64_t *)ret)[0] = 1024; - ((uint64_t *)ret)[1] = 1024; - ((uint64_t *)ret)[2] = 1024; - } - return 3 * sizeof(uint64_t); - - case PIPE_COMPUTE_CAP_PREFERRED_BLOCK_SIZE: - if (ret) - ((uint64_t *)ret)[0] = 256; - return sizeof(uint64_t); - - case PIPE_COMPUTE_CAP_IMAGES_SUPPORTED: - if (ret) - ((uint32_t *)ret)[0] = 1; - return sizeof(uint32_t); - - case PIPE_COMPUTE_CAP_SUBGROUP_SIZE: - if (ret) - ((uint32_t *)ret)[0] = devinfo->subgroup_size ? devinfo->subgroup_size : 16; - return sizeof(uint32_t); - - default: - return 0; - } -} static struct disk_cache * crocus_get_disk_shader_cache(struct pipe_screen *pscreen) @@ -720,8 +686,6 @@ crocus_screen_create(int fd, const struct pipe_screen_config *config) pscreen->query_memory_info = crocus_query_memory_info; pscreen->get_driver_query_group_info = crocus_get_monitor_group_info; pscreen->get_driver_query_info = crocus_get_monitor_info; - pscreen->get_compute_param = crocus_get_compute_param; - crocus_init_shader_caps(screen); crocus_init_compute_caps(screen); crocus_init_screen_caps(screen); diff --git a/src/gallium/drivers/crocus/crocus_state.c b/src/gallium/drivers/crocus/crocus_state.c index b47d38e..1cdf675 100644 --- a/src/gallium/drivers/crocus/crocus_state.c +++ b/src/gallium/drivers/crocus/crocus_state.c @@ -5513,6 +5513,35 @@ crocus_populate_binding_table(struct crocus_context *ice, s++; } +/* IVB Gen7: emit one surface per global binding using actual BO address. + * handle[] holds relative offset (0), surface base = BO gtt address. + * store_global(val, abs_addr) works because abs_addr = gtt_offset + rel, + * and surface base = gtt_offset, so hw writes to gtt_offset + rel. */ + foreach_surface_used(i, CROCUS_SURFACE_GROUP_GLOBAL) { + struct isl_device *isl_dev_g = &batch->screen->isl_dev; + uint32_t g_offset = 0; + uint32_t *g_surf = stream_state(batch, isl_dev_g->ss.size, + isl_dev_g->ss.align, &g_offset); + struct pipe_resource *global_res = ice->state.global_bindings[i]; + if (global_res) { + struct crocus_resource *gres = (struct crocus_resource *) global_res; + isl_buffer_fill_state(isl_dev_g, g_surf, + .address = crocus_state_reloc(batch, + g_offset + isl_dev_g->ss.addr_offset, + gres->bo, 0, + RELOC_32BIT | RELOC_WRITE), + .size_B = gres->bo->size, + .format = ISL_FORMAT_RAW, + .swizzle = ISL_SWIZZLE_IDENTITY, + .stride_B = 1, + .mocs = crocus_mocs(gres->bo, isl_dev_g)); + } else { + emit_null_surface(batch, &g_offset); + } + surf_offsets[s] = g_offset; + s++; + } + } /* ------------------------------------------------------------------- */ static uint32_t @@ -5544,6 +5573,7 @@ crocus_update_surface_base_address(struct crocus_batch *batch) crocus_emit_cmd(batch, GENX(STATE_BASE_ADDRESS), sba) { /* Set base addresses */ sba.GeneralStateBaseAddressModifyEnable = true; + sba.GeneralStateBaseAddress = ro_bo(batch->state.bo, 0); #if GFX_VER >= 6 sba.DynamicStateBaseAddressModifyEnable = true; @@ -5587,6 +5617,7 @@ crocus_update_surface_base_address(struct crocus_batch *batch) */ sba.DynamicStateAccessUpperBound = ro_bo(NULL, 0xfffff000); sba.DynamicStateAccessUpperBoundModifyEnable = true; + sba.GeneralStateAccessUpperBound = ro_bo(NULL, 0xfffff000); #else /* Same idea but using General State Base Address on Gen4-5 */ sba.GeneralStateAccessUpperBound = ro_bo(NULL, 0xfffff000); @@ -5598,7 +5629,10 @@ crocus_update_surface_base_address(struct crocus_batch *batch) * if you don't set the "Address Modify Enable" bit for the base. */ sba.GeneralStateMOCS = mocs; - sba.StatelessDataPortAccessMOCS = mocs; + sba.StatelessDataPortAccessMOCS = mocs ? mocs : 1; /* Try L3 cacheable if 0 */ +#if GFX_VERx10 == 70 + sba.StatelessDataPortAccessForceWriteThru = true; +#endif sba.DynamicStateMOCS = mocs; sba.IndirectObjectMOCS = mocs; sba.InstructionMOCS = mocs; @@ -8073,8 +8107,15 @@ crocus_upload_compute_state(struct crocus_context *ice, elk_cs_get_dispatch_info(devinfo, cs_prog_data, grid->block); crocus_update_surface_base_address(batch); - if ((stage_dirty & CROCUS_STAGE_DIRTY_CONSTANTS_CS) && shs->sysvals_need_upload) + if ((stage_dirty & CROCUS_STAGE_DIRTY_CONSTANTS_CS) && shs->sysvals_need_upload) { + for (unsigned i = 0; i < CROCUS_MAX_GLOBAL_BINDINGS; i++) { + if (ice->state.global_bindings[i]) { + struct crocus_resource *res = (struct crocus_resource *) ice->state.global_bindings[i]; + crocus_use_bo(batch, res->bo, true); + } + } upload_sysvals(ice, MESA_SHADER_COMPUTE); + } if (stage_dirty & CROCUS_STAGE_DIRTY_BINDINGS_CS) { crocus_populate_binding_table(ice, batch, MESA_SHADER_COMPUTE, false); @@ -8145,13 +8186,10 @@ crocus_upload_compute_state(struct crocus_context *ice, } } - /* TODO: Combine subgroup-id with cbuf0 so we can push regular uniforms */ + /* Push cross_thread (kernel args) + per_thread (subgroup_id) to CURBE */ if ((stage_dirty & CROCUS_STAGE_DIRTY_CS) || cs_prog_data->local_size[0] == 0 /* Variable local group size */) { uint32_t curbe_data_offset = 0; - assert(cs_prog_data->push.cross_thread.dwords == 0 && - cs_prog_data->push.per_thread.dwords == 1 && - cs_prog_data->base.param[0] == ELK_PARAM_BUILTIN_SUBGROUP_ID); const unsigned push_const_size = elk_cs_push_const_total_size(cs_prog_data, dispatch.threads); uint32_t *curbe_data_map = @@ -8160,6 +8198,24 @@ crocus_upload_compute_state(struct crocus_context *ice, &curbe_data_offset); assert(curbe_data_map); memset(curbe_data_map, 0x5a, align(push_const_size, 64)); + + if (cs_prog_data->push.cross_thread.size > 0) { + struct pipe_constant_buffer *cbuf = &shs->constbufs[0]; + if (cbuf->user_buffer) { + memcpy(curbe_data_map, cbuf->user_buffer, + cs_prog_data->push.cross_thread.size); + } else if (cbuf->buffer) { + struct crocus_resource *res = + (struct crocus_resource *) cbuf->buffer; + void *map = crocus_bo_map(NULL, res->bo, MAP_READ); + if (map) { + memcpy(curbe_data_map, + (uint8_t *)map + cbuf->buffer_offset, + cs_prog_data->push.cross_thread.size); + } + } + } + crocus_fill_cs_push_const_buffer(cs_prog_data, dispatch.threads, curbe_data_map); @@ -8184,7 +8240,8 @@ crocus_upload_compute_state(struct crocus_context *ice, idd.ConstantURBEntryReadLength = cs_prog_data->push.per_thread.regs; idd.BarrierEnable = cs_prog_data->uses_barrier; idd.SharedLocalMemorySize = intel_compute_slm_encode_size(GFX_VER, - prog_data->total_shared); + prog_data->total_shared + + grid->variable_shared_mem); #if GFX_VERx10 >= 75 idd.CrossThreadConstantDataReadLength = cs_prog_data->push.cross_thread.regs; #endif @@ -8279,6 +8336,11 @@ crocus_upload_compute_state(struct crocus_context *ice, crocus_emit_cmd(batch, GENX(MEDIA_STATE_FLUSH), msf); + crocus_emit_pipe_control_flush(batch, + "compute: flush DC after GPGPU_WALKER", + PIPE_CONTROL_DATA_CACHE_FLUSH | + PIPE_CONTROL_CS_STALL); + batch->contains_draw = true; } @@ -8286,23 +8348,40 @@ static void crocus_set_global_binding(struct pipe_context *ctx, unsigned start_slot, unsigned count, struct pipe_resource **resources, - uint32_t **handles) -{ + uint32_t **handles) { struct crocus_context *ice = (struct crocus_context *) ctx; assert(start_slot + count <= CROCUS_MAX_GLOBAL_BINDINGS); + for (unsigned i = 0; i < count; i++) { if (resources && resources[i]) { - pipe_resource_reference(&ice->state.global_bindings[start_slot + i], - resources[i]); - struct crocus_resource *res = (struct crocus_resource *) resources[i]; - assert(res->base.b.target == PIPE_BUFFER); - uint32_t addr = 0; - memcpy(&addr, handles[i], sizeof(addr)); - addr += res->bo->gtt_offset; - memcpy(handles[i], &addr, sizeof(addr)); + pipe_resource_reference(&ice->state.global_bindings[start_slot + i], + resources[i]); + struct crocus_resource *res = (struct crocus_resource *) resources[i]; + assert(res->base.b.target == PIPE_BUFFER); + /* IVB: handle = 0 (offset from BO base = 0). + * Surface base = BO real address via reloc. + * store_global(val, 0 + i*4) -> surface[base=real_addr][offset=i*4] */ + + uint32_t addr = 0; + memcpy(handles[i], &addr, sizeof(addr)); + ice->state.global_binding_handles[start_slot + i] = NULL; + + /* IVB: also bind as SSBO at the same index, for the load_global -> + * load_ssbo NIR redirect pass (crocus_lower_cs_global_to_ssbo). */ + struct crocus_shader_state *cs_shs = + &ice->state.shaders[MESA_SHADER_COMPUTE]; + struct pipe_shader_buffer *ssbo = &cs_shs->ssbo[start_slot + i]; + pipe_resource_reference(&ssbo->buffer, &res->base.b); + ssbo->buffer_offset = res->offset; + ssbo->buffer_size = res->bo->size - res->offset; + cs_shs->bound_ssbos |= 1u << (start_slot + i); + cs_shs->writable_ssbos |= 1u << (start_slot + i); + res->bind_history |= PIPE_BIND_SHADER_BUFFER; + res->bind_stages |= 1u << MESA_SHADER_COMPUTE; } else { pipe_resource_reference(&ice->state.global_bindings[start_slot + i], NULL); + ice->state.global_binding_handles[start_slot + i] = NULL; } } ice->state.stage_dirty |= CROCUS_STAGE_DIRTY_BINDINGS_CS; diff --git a/src/gallium/frontends/rusticl/api/context.rs b/src/gallium/frontends/rusticl/api/context.rs index 1f1db47..85c0f9f 100644 --- a/src/gallium/frontends/rusticl/api/context.rs +++ b/src/gallium/frontends/rusticl/api/context.rs @@ -20,6 +20,14 @@ use std::ffi::c_void; use std::ptr; use std::slice; +fn rusticl_dbg(msg: &str) { + use std::io::Write; + if let Ok(mut f) = std::fs::OpenOptions::new().create(true).append(true).open("/tmp/rusticl_debug.log") { + let _ = writeln!(f, "{}", msg); + let _ = f.flush(); + } +} + #[cl_info_entrypoint(clGetContextInfo)] unsafe impl CLInfo for cl_context { fn query(&self, q: cl_context_info, v: CLInfoValue) -> CLResult { @@ -99,7 +107,9 @@ pub fn get_gl_context_info_khr( } } + rusticl_dbg(&format!("RUSTICL_DBG: gl_context={:?} glx_display={:?} egl_display={:?}", gl_context, glx_display, egl_display)); let gl_ctx_manager = GLCtxManager::new(gl_context, glx_display, egl_display)?; + rusticl_dbg(&format!("RUSTICL_DBG: GLCtxManager::new result is_some={}", gl_ctx_manager.is_some())); unsafe { gl_ctx_manager .ok_or(CL_INVALID_GL_SHAREGROUP_REFERENCE_KHR)? @@ -175,28 +185,24 @@ fn create_context( let gl_ctx_manager = GLCtxManager::new(gl_context, glx_display, egl_display)?; if let Some(gl_ctx_manager) = &gl_ctx_manager { - // errcode_ret returns CL_INVALID_OPERATION if a context was specified as described above - // and any of the following conditions hold: - // ... - // Any of the devices specified in the devices argument cannot support OpenCL objects which - // share the data store of an OpenGL object. - let [dev] = devs.as_slice() else { + rusticl_dbg(&format!("RUSTICL_DBG: create_context fail at (B) - multiple devices with gl sharing")); return Err(CL_INVALID_OPERATION); }; if !dev.is_gl_sharing_supported() { + rusticl_dbg(&format!("RUSTICL_DBG: create_context fail at (C) - gl sharing not supported on this device")); return Err(CL_INVALID_OPERATION); } - // gl sharing is only supported on devices with an UUID, so we can simply unwrap it let dev_uuid: [c_char; UUID_SIZE] = dev .screen() .device_uuid() .unwrap_or_default() .map(|val| val as c_char); if gl_ctx_manager.interop_dev_info.device_uuid != dev_uuid { - // we only support gl_sharing on the same device + eprintln!("RUSTICL_DBG: create_context fail at (D) - uuid mismatch: gl_ctx={:?} dev={:?}", + gl_ctx_manager.interop_dev_info.device_uuid, dev_uuid); return Err(CL_INVALID_OPERATION); } } diff --git a/src/gallium/frontends/rusticl/api/device.rs b/src/gallium/frontends/rusticl/api/device.rs index d25dad9..9dde388 100644 --- a/src/gallium/frontends/rusticl/api/device.rs +++ b/src/gallium/frontends/rusticl/api/device.rs @@ -165,8 +165,14 @@ unsafe impl CLInfo for cl_device_id { } CL_DEVICE_LINKER_AVAILABLE => v.write::(true), CL_DEVICE_LOCAL_MEM_SIZE => v.write::(dev.local_mem_size()), - // TODO add query for CL_LOCAL vs CL_GLOBAL - CL_DEVICE_LOCAL_MEM_TYPE => v.write::(CL_GLOBAL), + CL_DEVICE_LOCAL_MEM_TYPE => { + let local_type = if dev.screen().compute_caps().has_dedicated_local_mem { + CL_LOCAL + } else { + CL_GLOBAL + }; + v.write::(local_type) + } CL_DEVICE_LUID => v.write::<[cl_uchar; CL_LUID_SIZE as usize]>( dev.screen().device_luid().unwrap_or_default(), ), diff --git a/src/gallium/frontends/rusticl/api/memory.rs b/src/gallium/frontends/rusticl/api/memory.rs index 8960db4..be71b25 100644 --- a/src/gallium/frontends/rusticl/api/memory.rs +++ b/src/gallium/frontends/rusticl/api/memory.rs @@ -916,6 +916,23 @@ fn create_image_with_properties( )?; let filtered_flags = filter_image_access_flags(flags); + { + use std::io::Write; + if let Ok(mut dbgf) = std::fs::OpenOptions::new().create(true).append(true).open("/tmp/rusticl_debug.log") { + let img_fmt = unsafe { *image_format }; + let _ = writeln!(dbgf, "RUSTICL_DBG: create_image format={:?} channel_order={} channel_data_type={} image_type={} filtered_flags={:#x}", + format, img_fmt.image_channel_order, img_fmt.image_channel_data_type, desc.image_type, filtered_flags); + for d in &c.devs { + match d.formats.get(format) { + None => { let _ = writeln!(dbgf, "RUSTICL_DBG: device has NO entry for this pipe_format at all"); } + Some(f) => { + let _ = writeln!(dbgf, "RUSTICL_DBG: device format map: {:?}", f); + } + } + } + let _ = dbgf.flush(); + } + } // CL_IMAGE_FORMAT_NOT_SUPPORTED if there are no devices in context that support image_format. c.devs .iter() diff --git a/src/gallium/frontends/rusticl/core/device.rs b/src/gallium/frontends/rusticl/core/device.rs index 551497e..056130a 100644 --- a/src/gallium/frontends/rusticl/core/device.rs +++ b/src/gallium/frontends/rusticl/core/device.rs @@ -935,12 +935,21 @@ impl DeviceBase { } pub fn is_gl_sharing_supported(&self) -> bool { - self.screen.caps().cl_gl_sharing - && self.screen.caps().dmabuf != 0 - && !self.is_device_software() - && self.screen.is_res_handle_supported() - && self.screen.device_uuid().is_some() - && self.caps.has_create_fence_fd + let c1 = self.screen.caps().cl_gl_sharing; + let c2 = self.screen.caps().dmabuf != 0; + let c3 = !self.is_device_software(); + let c4 = self.screen.is_res_handle_supported(); + let c5 = self.screen.device_uuid().is_some(); + let c6 = self.caps.has_create_fence_fd; + { + use std::io::Write; + if let Ok(mut f) = std::fs::OpenOptions::new().create(true).append(true).open("/tmp/rusticl_debug.log") { + let _ = writeln!(f, "RUSTICL_DBG: is_gl_sharing_supported: cl_gl_sharing={} dmabuf={} !is_sw={} res_handle={} uuid_some={} fence_fd={}", + c1, c2, c3, c4, c5, c6); + let _ = f.flush(); + } + } + c1 && c2 && c3 && c4 && c5 && c6 } pub fn is_device_software(&self) -> bool { diff --git a/src/gallium/frontends/rusticl/core/kernel.rs b/src/gallium/frontends/rusticl/core/kernel.rs index e608a8d..7365a30 100644 --- a/src/gallium/frontends/rusticl/core/kernel.rs +++ b/src/gallium/frontends/rusticl/core/kernel.rs @@ -1801,6 +1801,9 @@ impl Kernel { // subtract the shader local_size as we only request something on top of that. variable_local_size -= static_local_size; + let input_ptr_before = exec_builder.input().as_ptr(); + let input_len_before = exec_builder.input().len(); + let (resources, mut globals) = exec_builder.get_resources_and_globals(); ctx.bind_kernel(&nir_kernel_builds, variant)?; diff --git a/src/gallium/frontends/rusticl/core/queue.rs b/src/gallium/frontends/rusticl/core/queue.rs index 893a13e..4ee692f 100644 --- a/src/gallium/frontends/rusticl/core/queue.rs +++ b/src/gallium/frontends/rusticl/core/queue.rs @@ -168,6 +168,7 @@ impl<'c> QueueContextWithState<'c> { pub fn update_cb0(&self, data: &[u8]) -> CLResult<()> { // only update if we actually bind data if !data.is_empty() { + eprintln!("update_cb0: use_stream={} len={}", self.use_stream, data.len()); if self.use_stream { if !self.ctx.set_constant_buffer_stream(0, data) { return Err(CL_OUT_OF_RESOURCES); diff --git a/src/gallium/include/pipe/p_defines.h b/src/gallium/include/pipe/p_defines.h index 8854309..bc8d505 100644 --- a/src/gallium/include/pipe/p_defines.h +++ b/src/gallium/include/pipe/p_defines.h @@ -825,6 +825,7 @@ struct pipe_compute_caps { unsigned max_variable_threads_per_block; uint64_t max_mem_alloc_size; uint64_t max_global_size; + bool has_dedicated_local_mem; /* true if SLM is physically separate from global memory */ }; struct pipe_mesh_caps { diff --git a/src/intel/compiler/elk/elk_compiler.c b/src/intel/compiler/elk/elk_compiler.c index 17b250b..92d8c2c 100644 --- a/src/intel/compiler/elk/elk_compiler.c +++ b/src/intel/compiler/elk/elk_compiler.c @@ -172,6 +172,10 @@ elk_get_compiler_config_value(const struct elk_compiler *compiler) unsigned elk_prog_data_size(mesa_shader_stage stage) { + if (stage == MESA_SHADER_KERNEL) { + stage = MESA_SHADER_COMPUTE; + } + static const size_t stage_sizes[] = { [MESA_SHADER_VERTEX] = sizeof(struct elk_vs_prog_data), [MESA_SHADER_TESS_CTRL] = sizeof(struct elk_tcs_prog_data), diff --git a/src/intel/compiler/elk/elk_compiler.h b/src/intel/compiler/elk/elk_compiler.h index a8c75a4..6f9f9fe 100644 --- a/src/intel/compiler/elk/elk_compiler.h +++ b/src/intel/compiler/elk/elk_compiler.h @@ -1053,6 +1053,7 @@ struct elk_cs_prog_data { * surface indices the CS-specific surfaces */ uint32_t work_groups_start; + uint32_t global_start; /* IVB: BTI for full-GTT global surface */ /** @} */ } binding_table; }; diff --git a/src/intel/compiler/elk/elk_fs_nir.cpp b/src/intel/compiler/elk/elk_fs_nir.cpp index 474cd76..74b9ea1 100644 --- a/src/intel/compiler/elk/elk_fs_nir.cpp +++ b/src/intel/compiler/elk/elk_fs_nir.cpp @@ -5114,6 +5114,31 @@ fs_nir_emit_intrinsic(nir_to_elk_state &ntb, case nir_intrinsic_load_global: case nir_intrinsic_load_global_constant: { + if (devinfo->ver < 8) { + elk_fs_reg dest = get_nir_def(ntb, instr->def); + const unsigned bit_size = instr->def.bit_size; + elk_fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS]; + /* IVB: use dedicated full-GTT surface for global access */ + const struct elk_cs_prog_data *cs_pd = + elk_cs_prog_data(ntb.s.prog_data); + srcs[SURFACE_LOGICAL_SRC_SURFACE] = elk_imm_ud(cs_pd->binding_table.global_start); + srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(ntb, instr->src[0]); + srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = elk_imm_ud(1); + srcs[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK] = elk_imm_ud(0); + dest.type = elk_reg_type_from_bit_size(bit_size, ELK_REGISTER_TYPE_UD); + + if (bit_size == 32 && nir_intrinsic_align(instr) >= 4) { + srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = elk_imm_ud(instr->num_components); + elk_fs_inst *inst = bld.emit(ELK_SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL, dest, srcs, SURFACE_LOGICAL_NUM_SRCS); + inst->size_written = instr->num_components * s.dispatch_width * 4; + } else { + srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = elk_imm_ud(bit_size); + elk_fs_reg read_result = bld.vgrf(ELK_REGISTER_TYPE_UD); + bld.emit(ELK_SHADER_OPCODE_BYTE_SCATTERED_READ_LOGICAL, read_result, srcs, SURFACE_LOGICAL_NUM_SRCS); + bld.MOV(dest, subscript(read_result, dest.type, 0)); + } + break; + } assert(devinfo->ver >= 8); assert(instr->def.bit_size <= 32); @@ -5150,13 +5175,34 @@ fs_nir_emit_intrinsic(nir_to_elk_state &ntb, } case nir_intrinsic_store_global: { + if (devinfo->ver < 8) { + const unsigned bit_size = nir_src_bit_size(instr->src[0]); + elk_fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS]; + /* IVB: use dedicated full-GTT surface for global access */ + const struct elk_cs_prog_data *cs_pd = + elk_cs_prog_data(ntb.s.prog_data); + srcs[SURFACE_LOGICAL_SRC_SURFACE] = elk_imm_ud(cs_pd->binding_table.global_start); + srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(ntb, instr->src[1]); + srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = elk_imm_ud(1); + srcs[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK] = elk_imm_ud(1); + + elk_fs_reg data = get_nir_src(ntb, instr->src[0]); + data.type = elk_reg_type_from_bit_size(bit_size, ELK_REGISTER_TYPE_UD); + + if (bit_size == 32 && nir_intrinsic_align(instr) >= 4) { + srcs[SURFACE_LOGICAL_SRC_DATA] = data; + srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = elk_imm_ud(instr->num_components); + bld.emit(ELK_SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL, elk_fs_reg(), srcs, SURFACE_LOGICAL_NUM_SRCS); + } else { + srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = elk_imm_ud(bit_size); + srcs[SURFACE_LOGICAL_SRC_DATA] = bld.vgrf(ELK_REGISTER_TYPE_UD); + bld.MOV(srcs[SURFACE_LOGICAL_SRC_DATA], data); + bld.emit(ELK_SHADER_OPCODE_BYTE_SCATTERED_WRITE_LOGICAL, elk_fs_reg(), srcs, SURFACE_LOGICAL_NUM_SRCS); + } + break; + } assert(devinfo->ver >= 8); - - assert(nir_src_bit_size(instr->src[0]) <= 32); - assert(nir_intrinsic_write_mask(instr) == - (1u << instr->num_components) - 1); - assert(nir_intrinsic_align(instr) > 0); - + elk_fs_reg srcs[A64_LOGICAL_NUM_SRCS]; srcs[A64_LOGICAL_ADDRESS] = get_nir_src(ntb, instr->src[1]); srcs[A64_LOGICAL_ENABLE_HELPERS] = @@ -5523,6 +5569,7 @@ fs_nir_emit_intrinsic(nir_to_elk_state &ntb, srcs[SURFACE_LOGICAL_SRC_SURFACE] = elk_imm_ud(GFX8_BTI_STATELESS_NON_COHERENT); } else { + /* IVB: use dedicated full-GTT surface instead of stateless BTI 255 */ srcs[SURFACE_LOGICAL_SRC_SURFACE] = elk_imm_ud(ELK_BTI_STATELESS); } @@ -6241,6 +6288,7 @@ static void fs_nir_emit_global_atomic(nir_to_elk_state &ntb, const fs_builder &bld, nir_intrinsic_instr *instr) { + const intel_device_info *devinfo = ntb.devinfo; enum elk_lsc_opcode op = elk_lsc_aop_for_nir_intrinsic(instr); int num_data = lsc_op_num_data_values(op); @@ -6262,6 +6310,20 @@ fs_nir_emit_global_atomic(nir_to_elk_state &ntb, const fs_builder &bld, data = tmp; } + if (devinfo->ver < 8) { + elk_fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS]; + srcs[SURFACE_LOGICAL_SRC_SURFACE] = elk_imm_ud(ELK_BTI_STATELESS); + srcs[SURFACE_LOGICAL_SRC_ADDRESS] = addr; + srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = elk_imm_ud(1); + srcs[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK] = elk_imm_ud(1); + if (num_data >= 1) + srcs[SURFACE_LOGICAL_SRC_DATA] = data; + srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = elk_imm_ud(op); + + bld.emit(ELK_SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL, dest, srcs, SURFACE_LOGICAL_NUM_SRCS); + return; + } + elk_fs_reg srcs[A64_LOGICAL_NUM_SRCS]; srcs[A64_LOGICAL_ADDRESS] = addr; srcs[A64_LOGICAL_SRC] = data; @@ -6853,6 +6915,7 @@ fs_nir_emit_instr(nir_to_elk_state &ntb, nir_instr *instr) fs_nir_emit_fs_intrinsic(ntb, nir_instr_as_intrinsic(instr)); break; case MESA_SHADER_COMPUTE: + case MESA_SHADER_KERNEL: fs_nir_emit_cs_intrinsic(ntb, nir_instr_as_intrinsic(instr)); break; default: diff --git a/src/intel/compiler/elk/elk_lower_logical_sends.cpp b/src/intel/compiler/elk/elk_lower_logical_sends.cpp index 368c234..59b2d3b 100644 --- a/src/intel/compiler/elk/elk_lower_logical_sends.cpp +++ b/src/intel/compiler/elk/elk_lower_logical_sends.cpp @@ -1199,8 +1199,7 @@ lower_surface_logical_send(const fs_builder &bld, elk_fs_inst *inst) if (is_typed_access || is_stateless) { fs_builder ubld = bld.exec_all().group(8, 0); header = ubld.vgrf(ELK_REGISTER_TYPE_UD); - if (is_stateless) { - assert(!is_surface_access); + if (is_stateless && !is_surface_access) { ubld.emit(ELK_SHADER_OPCODE_SCRATCH_HEADER, header); } else { ubld.MOV(header, elk_imm_d(0)); diff --git a/src/intel/compiler/elk/elk_nir.c b/src/intel/compiler/elk/elk_nir.c index bfed62b..cef363b 100644 --- a/src/intel/compiler/elk/elk_nir.c +++ b/src/intel/compiler/elk/elk_nir.c @@ -766,9 +766,12 @@ elk_nir_optimize(nir_shader *nir, bool is_scalar, OPT(nir_remove_dead_variables, nir_var_function_temp, NULL); } -static unsigned -lower_bit_size_callback(const nir_instr *instr, UNUSED void *data) +static unsigned +lower_bit_size_callback(const nir_instr *instr, void *data) { + const struct elk_compiler *compiler = data; + const struct intel_device_info *devinfo = compiler->devinfo; + switch (instr->type) { case nir_instr_type_alu: { nir_alu_instr *alu = nir_instr_as_alu(instr); @@ -785,9 +788,19 @@ lower_bit_size_callback(const nir_instr *instr, UNUSED void *data) default: break; } - - if (alu->def.bit_size >= 32) - return 0; + if (alu->def.bit_size == 16 && + devinfo->ver < 8 && + nir_op_infos[alu->op].output_type == nir_type_float) { + /* Gen7 has no native FP16 ALU/register support at all (see the + * empty HF entry in gfx7_hw_3src_type[] / gfx7_hw_type[] in + * elk_reg_type.c) -- unlike >=Gfx8, it cannot represent a + * 16-bit float operand or result under ANY circumstance, not + * just when inputs are mixed-width. Lower unconditionally, + * regardless of the source bit sizes, or elk_reg_type_to_hw_type() + * will hit an unencodable HF register type and assert. + */ + return 32; + } /* Note: nir_op_iabs and nir_op_ineg are not lowered here because the * 8-bit ABS or NEG instruction should eventually get copy propagated @@ -805,7 +818,7 @@ lower_bit_size_callback(const nir_instr *instr, UNUSED void *data) case nir_op_ffract: case nir_op_fround_even: case nir_op_ftrunc: - return 32; + return alu->def.bit_size < 32 ? 32 : 0; case nir_op_frcp: case nir_op_frsq: case nir_op_fsqrt: @@ -814,7 +827,13 @@ lower_bit_size_callback(const nir_instr *instr, UNUSED void *data) case nir_op_flog2: case nir_op_fsin: case nir_op_fcos: - return 32; + /* Same issue as the idiv/umod/fceil group above: these + * transcendental ops need >=32-bit precision on Gen7, but if + * the source is already >=32-bit, unconditionally returning + * 32 here trips the nir_lower_bit_size assert (src->bit_size + * < bit_size fails when equal or the source is 64-bit). + */ + return alu->def.bit_size < 32 ? 32 : 0; case nir_op_isign: UNREACHABLE("Should have been lowered by nir_opt_algebraic."); default: @@ -1310,8 +1329,26 @@ get_mem_access_size_align(nir_intrinsic_op intrin, uint8_t bytes, bool offset_is_const, enum gl_access_qualifier access, const void *cb_data) { + const struct intel_device_info *devinfo = cb_data; const uint32_t align = nir_combined_align(align_mul, align_offset); + /* IVB (Gen7, verx10=70) has no byte-scattered dataport message -- + * elk_dp_byte_scattered_rw_desc() asserts verx10 >= 75 (Haswell+). + * This is a real HW limitation, not a missing BTI like the global + * stateless case. Never request a sub-dword access on this hardware; + * always request dword granularity. nir_lower_mem_access_bit_sizes + * turns the mismatch into a shift+extract on the load side, and an + * atomic iand/ior read-modify-write on the store side (both already + * generic in the pass, we just need to ask for them). */ + if (devinfo->verx10 < 75 && (align < 4 || bytes < 4)) { + return (nir_mem_access_size_align) { + .bit_size = 32, + .num_components = 1, + .align = 4, + .shift = nir_mem_access_shift_method_scalar, + }; + } + switch (intrin) { case nir_intrinsic_load_ssbo: case nir_intrinsic_load_shared: @@ -1401,6 +1438,7 @@ elk_vectorize_lower_mem_access(nir_shader *nir, OPT(nir_opt_load_store_vectorize, &options); } + const struct intel_device_info *devinfo = compiler->devinfo; nir_lower_mem_access_bit_sizes_options mem_access_options = { .modes = nir_var_mem_ssbo | nir_var_mem_constant | @@ -1409,6 +1447,8 @@ elk_vectorize_lower_mem_access(nir_shader *nir, nir_var_mem_global | nir_var_mem_shared, .callback = get_mem_access_size_align, + .cb_data = devinfo, + .may_lower_unaligned_stores_to_atomics = devinfo->verx10 < 75, }; OPT(nir_lower_mem_access_bit_sizes, &mem_access_options); diff --git a/src/intel/compiler/elk/elk_reg_type.c b/src/intel/compiler/elk/elk_reg_type.c index c484c5d..be68c91 100644 --- a/src/intel/compiler/elk/elk_reg_type.c +++ b/src/intel/compiler/elk/elk_reg_type.c @@ -220,10 +220,16 @@ elk_reg_type_to_hw_type(const struct intel_device_info *devinfo, } if (file == ELK_IMMEDIATE_VALUE) { - assert(table[type].imm_type != (enum hw_imm_type)INVALID); + if (table[type].imm_type == (enum hw_imm_type)INVALID) { + fprintf(stderr, "elk_reg_type_to_hw_type: INVALID imm_type! type=%d\n", type); + assert(0); + } return table[type].imm_type; } else { - assert(table[type].reg_type != (enum hw_reg_type)INVALID); + if (table[type].reg_type == (enum hw_reg_type)INVALID) { + fprintf(stderr, "elk_reg_type_to_hw_type: INVALID reg_type! type=%d\n", type); + assert(0); + } return table[type].reg_type; } } diff --git a/src/intel/dev/__pycache__/intel_device_info.cpython-313.pyc b/src/intel/dev/__pycache__/intel_device_info.cpython-313.pyc new file mode 100644 index 0000000..24513ef Binary files /dev/null and b/src/intel/dev/__pycache__/intel_device_info.cpython-313.pyc differ diff --git a/src/intel/dev/__pycache__/intel_device_info.cpython-314.pyc b/src/intel/dev/__pycache__/intel_device_info.cpython-314.pyc new file mode 100644 index 0000000..a174940 Binary files /dev/null and b/src/intel/dev/__pycache__/intel_device_info.cpython-314.pyc differ diff --git a/src/intel/genxml/__pycache__/intel_genxml.cpython-313.pyc b/src/intel/genxml/__pycache__/intel_genxml.cpython-313.pyc new file mode 100644 index 0000000..7bc86ec Binary files /dev/null and b/src/intel/genxml/__pycache__/intel_genxml.cpython-313.pyc differ diff --git a/src/intel/genxml/__pycache__/intel_genxml.cpython-314.pyc b/src/intel/genxml/__pycache__/intel_genxml.cpython-314.pyc new file mode 100644 index 0000000..2d716c5 Binary files /dev/null and b/src/intel/genxml/__pycache__/intel_genxml.cpython-314.pyc differ diff --git a/src/intel/genxml/__pycache__/util.cpython-313.pyc b/src/intel/genxml/__pycache__/util.cpython-313.pyc new file mode 100644 index 0000000..6fe91fd Binary files /dev/null and b/src/intel/genxml/__pycache__/util.cpython-313.pyc differ diff --git a/src/intel/genxml/__pycache__/util.cpython-314.pyc b/src/intel/genxml/__pycache__/util.cpython-314.pyc new file mode 100644 index 0000000..8bb3eda Binary files /dev/null and b/src/intel/genxml/__pycache__/util.cpython-314.pyc differ diff --git a/src/intel/vulkan_hasvk/anv_batch_chain.c b/src/intel/vulkan_hasvk/anv_batch_chain.c index 2ecb8a4..5fbc50e 100644 --- a/src/intel/vulkan_hasvk/anv_batch_chain.c +++ b/src/intel/vulkan_hasvk/anv_batch_chain.c @@ -274,9 +274,62 @@ anv_reloc_list_append(struct anv_reloc_list *list, return VK_SUCCESS; } -/*-----------------------------------------------------------------------* - * Functions related to anv_batch - *-----------------------------------------------------------------------*/ +/* Like anv_reloc_list_append(), but only copies/rebases the relocations + * whose offset falls at or after @skip_bytes -- i.e. the ones that belong + * to the suffix of @other starting at that byte, which is what + * anv_batch_emit_batch_suffix() actually copies into @batch. Relocations + * before @skip_bytes belong to a region we are deliberately NOT emitting + * and must not be added (their target offset would land outside what we + * copied, corrupting the batch). + */ +static VkResult +anv_reloc_list_append_range(struct anv_reloc_list *list, + const VkAllocationCallbacks *alloc, + struct anv_reloc_list *other, + uint32_t skip_bytes, uint32_t offset) +{ + uint32_t n = 0; + for (uint32_t i = 0; i < other->num_relocs; i++) { + if (other->relocs[i].offset >= skip_bytes) + n++; + } + + if (n == 0) { + /* Still need to propagate BO dependencies: shaders/state referenced + * only from the skipped prefix may not be otherwise declared as + * used by this command buffer submission if this is the very first + * time this pipeline's group is touched in it. + */ + anv_reloc_list_grow_deps(list, alloc, other->dep_words); + for (uint32_t w = 0; w < other->dep_words; w++) + list->deps[w] |= other->deps[w]; + return VK_SUCCESS; + } + + VkResult result = anv_reloc_list_grow(list, alloc, n); + if (result != VK_SUCCESS) + return result; + + uint32_t j = list->num_relocs; + for (uint32_t i = 0; i < other->num_relocs; i++) { + if (other->relocs[i].offset < skip_bytes) + continue; + list->relocs[j] = other->relocs[i]; + list->relocs[j].offset += offset - skip_bytes; + list->reloc_bos[j] = other->reloc_bos[i]; + j++; + } + assert(j == list->num_relocs + n); + list->num_relocs = j; + + anv_reloc_list_grow_deps(list, alloc, other->dep_words); + for (uint32_t w = 0; w < other->dep_words; w++) + list->deps[w] |= other->deps[w]; + + return VK_SUCCESS; +} + + void * anv_batch_emit_dwords(struct anv_batch *batch, int num_dwords) @@ -340,7 +393,45 @@ anv_batch_emit_batch(struct anv_batch *batch, struct anv_batch *other) batch->next += size; } -/*-----------------------------------------------------------------------* +void +anv_batch_emit_batch_suffix(struct anv_batch *batch, struct anv_batch *other, + uint32_t skip_bytes) +{ + uint32_t full_size = other->next - other->start; + assert(skip_bytes <= full_size); + assert(skip_bytes % 4 == 0); + uint32_t size = full_size - skip_bytes; + + if (size == 0) + return; + assert(size % 4 == 0); + + if (batch->next + size > batch->end) { + VkResult result = batch->extend_cb(batch, batch->user_data); + if (result != VK_SUCCESS) { + anv_batch_set_error(batch, result); + return; + } + } + + assert(batch->next + size <= batch->end); + + VG(VALGRIND_CHECK_MEM_IS_DEFINED(other->start + skip_bytes, size)); + memcpy(batch->next, other->start + skip_bytes, size); + + uint32_t offset = batch->next - batch->start; + VkResult result = anv_reloc_list_append_range(batch->relocs, batch->alloc, + other->relocs, + skip_bytes, offset); + if (result != VK_SUCCESS) { + anv_batch_set_error(batch, result); + return; + } + + batch->next += size; +} + +/*----------------------------------------------------------------------- * Functions related to anv_batch_bo *-----------------------------------------------------------------------*/ diff --git a/src/intel/vulkan_hasvk/anv_image.c b/src/intel/vulkan_hasvk/anv_image.c index 90b2309..e0c7b9f 100644 --- a/src/intel/vulkan_hasvk/anv_image.c +++ b/src/intel/vulkan_hasvk/anv_image.c @@ -569,17 +569,12 @@ add_aux_surface_if_supported(struct anv_device *device, return VK_SUCCESS; } - if (device->info->ver == 7) { - anv_perf_warn(VK_LOG_OBJS(&image->vk.base), "Implement gfx7 HiZ"); - return VK_SUCCESS; - } - if (image->vk.mip_levels > 1) { anv_perf_warn(VK_LOG_OBJS(&image->vk.base), "Enable multi-LOD HiZ"); return VK_SUCCESS; } - if (device->info->ver == 8 && image->vk.samples > 1) { + if (device->info->ver <= 8 && image->vk.samples > 1) { anv_perf_warn(VK_LOG_OBJS(&image->vk.base), "Enable gfx8 multisampled HiZ"); return VK_SUCCESS; diff --git a/src/intel/vulkan_hasvk/anv_private.h b/src/intel/vulkan_hasvk/anv_private.h index dad8b1a..d82fa9d 100644 --- a/src/intel/vulkan_hasvk/anv_private.h +++ b/src/intel/vulkan_hasvk/anv_private.h @@ -1339,6 +1339,17 @@ struct anv_batch { void *anv_batch_emit_dwords(struct anv_batch *batch, int num_dwords); void anv_batch_emit_batch(struct anv_batch *batch, struct anv_batch *other); + +/* Like anv_batch_emit_batch(), but only copies the suffix of @other + * starting at byte offset @skip_bytes (and only appends/rebases the + * relocations that fall within that suffix). Gen7-only: used to skip + * re-emitting a pipeline's VS-stage instruction group when it is + * byte-identical to what's already programmed for the previous bind + * (see cmd_buffer_flush_gfx_state). + */ +void anv_batch_emit_batch_suffix(struct anv_batch *batch, + struct anv_batch *other, + uint32_t skip_bytes); struct anv_address anv_batch_address(struct anv_batch *batch, void *batch_location); static inline void @@ -2399,6 +2410,30 @@ struct anv_cmd_graphics_state { struct anv_graphics_pipeline *pipeline; + /** Last emitted URB configuration (Gen7 stall optimization) */ + struct intel_urb_config last_urb_cfg; + bool urb_cfg_valid; + + /** + * Gen7 VS-stage instruction group cache. Tracks the VS/TCS/TES/GS + * shader binaries AND the effective URB config (see last_urb_cfg + * above) of the pipeline for which the group (URB + VS + HS + DS + + * GS, plus the PRM-mandated depth-stall PIPE_CONTROL before them) was + * last fully emitted into this command buffer, so a subsequent bind + * can skip re-emitting the whole group when both match the previous + * bind. Shader pointers alone are not sufficient -- URB layout also + * depends on l3_config. Always present (not #if-guarded) so this + * struct's layout doesn't depend on GFX_VER; only read/written by + * Gen7 code paths, inert everywhere else. + */ + struct anv_shader_bin *vs_stage_key[4]; + bool vs_stage_key_valid; + /** True once a stall satisfying the PRM's VS-associated-3DSTATE rule + * has been emitted for the *current* draw's pipeline flush. Used to + * decide whether a later VS-descriptor/push-constant update still + * needs its own stall (see cmd_buffer_flush_gfx_state). */ + bool vs_stage_stall_emitted; + VkRenderingFlags rendering_flags; VkRect2D render_area; uint32_t layer_count; @@ -2790,6 +2825,22 @@ struct anv_pipeline { struct anv_graphics_pipeline { struct anv_pipeline base; + /* URB configuration computed at pipeline creation time. + * Used to skip redundant URB re-emission and PIPE_CONTROL stalls on Gen7. + */ + struct intel_urb_config urb_cfg; + + /* Byte length, from the start of base.batch, of the VS-stage + * instruction group (3DSTATE_URB_* + 3DSTATE_VS + 3DSTATE_HS + + * 3DSTATE_DS + 3DSTATE_GS, plus the PRM-mandated depth-stall + * PIPE_CONTROL emitted just before them). Computed unconditionally in + * genX(graphics_pipeline_emit) so this struct's layout doesn't depend + * on GFX_VER; only meaningful/consumed on Gen7 (0 elsewhere, since the + * reordering that makes the group a contiguous prefix of base.batch + * only happens for GFX_VER <= 7). + */ + uint32_t vs_stage_batch_size; + /* Shaders */ struct anv_shader_bin * shaders[ANV_GRAPHICS_SHADER_STAGE_COUNT]; diff --git a/src/intel/vulkan_hasvk/genX_cmd_buffer.c b/src/intel/vulkan_hasvk/genX_cmd_buffer.c index 4abc736..3683705 100644 --- a/src/intel/vulkan_hasvk/genX_cmd_buffer.c +++ b/src/intel/vulkan_hasvk/genX_cmd_buffer.c @@ -3352,8 +3352,84 @@ genX(cmd_buffer_flush_gfx_state)(struct anv_cmd_buffer *cmd_buffer) } } +#if GFX_VER <= 7 + /* Reset for this flush cycle; set true below the moment a stall + * covering the PRM's VS-associated-3DSTATE rule has actually been + * emitted, whichever path ends up doing it. */ + cmd_buffer->state.gfx.vs_stage_stall_emitted = false; +#endif + if (cmd_buffer->state.gfx.dirty & ANV_CMD_DIRTY_PIPELINE) { +#if GFX_VER <= 7 + /* Skip re-emitting the VS-stage instruction group (URB + VS + HS + + * DS + GS, plus its PRM-mandated depth-stall PIPE_CONTROL, all kept + * contiguous at the front of the pipeline's static batch -- see + * genX(graphics_pipeline_emit)) when it is byte-identical to what's + * already programmed. This requires BOTH the same VS/TCS/TES/GS + * shader binaries AND the same effective URB config as the last + * pipeline for which we fully emitted the group -- shader pointers + * alone are not enough, since URB start/entry-count also depends on + * l3_config (see the urb_unchanged check below). Two different + * VkPipeline objects that share the same shaders and l3_config + * (e.g. VulkanMod's opaque vs. cutout terrain pipelines, which only + * differ in rasterizer cull mode) hit this path every switch. + */ + struct anv_cmd_graphics_state *gfx_state = &cmd_buffer->state.gfx; + struct anv_shader_bin *new_key[4] = { + pipeline->shaders[MESA_SHADER_VERTEX], + pipeline->shaders[MESA_SHADER_TESS_CTRL], + pipeline->shaders[MESA_SHADER_TESS_EVAL], + pipeline->shaders[MESA_SHADER_GEOMETRY], + }; + + /* Same shaders alone is NOT sufficient: the actual URB byte layout + * (VSURBStartingAddress / VSNumberofURBEntries) also depends on + * l3_config (e.g. render target/attachment count shifts L3 + * partitioning -- see intel_get_urb_config()), so two pipelines + * with byte-identical VS/HS/DS/GS shaders can still need a + * different URB layout. Must check both. + */ + bool urb_unchanged = gfx_state->urb_cfg_valid; + if (urb_unchanged) { + for (int s = MESA_SHADER_VERTEX; s <= MESA_SHADER_GEOMETRY; s++) { + if (intel_urb_setup_changed(&gfx_state->last_urb_cfg, + &pipeline->urb_cfg, s)) { + urb_unchanged = false; + break; + } + } + } + + bool vs_stage_unchanged = + gfx_state->vs_stage_key_valid && + memcmp(gfx_state->vs_stage_key, new_key, sizeof(new_key)) == 0 && + urb_unchanged; + vs_stage_unchanged = false; /* DIAGNOSTIC: force full emit, see script header */ + + if (vs_stage_unchanged) { + anv_batch_emit_batch_suffix(&cmd_buffer->batch, &pipeline->base.batch, + pipeline->vs_stage_batch_size); + /* We skipped the group -- no stall owed for URB/VS/HS/DS/GS + * themselves, but if descriptors/push-constants are ALSO dirty + * this flush cycle, 3DSTATE_CONSTANT_VS / + * 3DSTATE_BINDING_TABLE_POINTER_VS are still about to be emitted + * below and are covered by the SAME PRM rule -- so do NOT mark + * vs_stage_stall_emitted here; let the check below decide. + */ + } else { + anv_batch_emit_batch(&cmd_buffer->batch, &pipeline->base.batch); + memcpy(gfx_state->vs_stage_key, new_key, sizeof(new_key)); + gfx_state->vs_stage_key_valid = true; + gfx_state->last_urb_cfg = pipeline->urb_cfg; + gfx_state->urb_cfg_valid = true; + /* Full emit -- the PIPE_CONTROL baked into the pipeline's static + * batch (by emit_urb_setup) was actually sent, covering the + * whole PRM-mandated group for this flush cycle. */ + gfx_state->vs_stage_stall_emitted = true; + } +#else anv_batch_emit_batch(&cmd_buffer->batch, &pipeline->base.batch); +#endif /* If the pipeline changed, we may need to re-allocate push constant * space in the URB. @@ -3362,8 +3438,9 @@ genX(cmd_buffer_flush_gfx_state)(struct anv_cmd_buffer *cmd_buffer) } #if GFX_VER <= 7 - if (cmd_buffer->state.descriptors_dirty & VK_SHADER_STAGE_VERTEX_BIT || - cmd_buffer->state.push_constants_dirty & VK_SHADER_STAGE_VERTEX_BIT) { + if (!cmd_buffer->state.gfx.vs_stage_stall_emitted && + (cmd_buffer->state.descriptors_dirty & VK_SHADER_STAGE_VERTEX_BIT || + cmd_buffer->state.push_constants_dirty & VK_SHADER_STAGE_VERTEX_BIT)) { /* From the IVB PRM Vol. 2, Part 1, Section 3.2.1: * * "A PIPE_CONTROL with Post-Sync Operation set to 1h and a depth @@ -3380,6 +3457,7 @@ genX(cmd_buffer_flush_gfx_state)(struct anv_cmd_buffer *cmd_buffer) pc.Address = cmd_buffer->device->workaround_address; anv_debug_dump_pc(pc); } + cmd_buffer->state.gfx.vs_stage_stall_emitted = true; } #endif diff --git a/src/intel/vulkan_hasvk/genX_pipeline.c b/src/intel/vulkan_hasvk/genX_pipeline.c index f727f1c..d4a950c 100644 --- a/src/intel/vulkan_hasvk/genX_pipeline.c +++ b/src/intel/vulkan_hasvk/genX_pipeline.c @@ -325,6 +325,28 @@ emit_urb_setup(struct anv_graphics_pipeline *pipeline) entry_size[i] = prog_data ? prog_data->urb_entry_size : 1; } +#if GFX_VER <= 7 + /* Compute and cache the *actual* URB config -- not just the shaders' + * requested entry sizes, but the real per-stage start offsets and + * entry counts, which also depend on l3_config (e.g. attachment/RT + * count can shift L3 partitioning). This is the correct, complete key + * for cmd_buffer_flush_gfx_state() to decide whether the VS-stage + * instruction group can be skipped on a later bind: two pipelines + * with byte-identical VS/HS/DS/GS shaders can still require a + * different URB layout if their l3_config differs -- comparing shader + * pointers alone is not sufficient. + */ + pipeline->urb_cfg = (struct intel_urb_config) { + .size = { entry_size[0], entry_size[1], entry_size[2], entry_size[3] }, + }; + bool urb_constrained; + intel_get_urb_config(pipeline->base.device->info, pipeline->base.l3_config, + pipeline->active_stages & + VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT, + pipeline->active_stages & VK_SHADER_STAGE_GEOMETRY_BIT, + &pipeline->urb_cfg, &urb_constrained); +#endif + genX(emit_urb_setup)(pipeline->base.device, &pipeline->base.batch, pipeline->base.l3_config, pipeline->active_stages, entry_size); @@ -1827,6 +1849,16 @@ genX(graphics_pipeline_emit)(struct anv_graphics_pipeline *pipeline, { emit_urb_setup(pipeline); + /* DIAGNOSTIC v2: VS/HS/DS/GS reordering reverted -- back to vanilla + * emission position (after vertex_input, below). vs_stage_batch_size + * is set to 0 here since it's meaningless with the group no longer + * contiguous with URB; this is fine because diagnostic v1 already + * force-disabled the skip path that would have used it. + */ +#if GFX_VER <= 7 + pipeline->vs_stage_batch_size = 0; +#endif + assert(state->rs != NULL); emit_rs_state(pipeline, state->ia, state->rs, state->ms, state->rp); emit_ms_state(pipeline, state->ms); @@ -1856,6 +1888,8 @@ genX(graphics_pipeline_emit)(struct anv_graphics_pipeline *pipeline, emit_vertex_input(pipeline, state->vi); + /* DIAGNOSTIC v2: always emit here (both Gen7 and Gen8+), matching + * vanilla hasvk's original position exactly. */ emit_3dstate_vs(pipeline); emit_3dstate_hs_te_ds(pipeline, state->ts); emit_3dstate_gs(pipeline, state->rs); diff --git a/src/intel/vulkan_hasvk/gfx7_cmd_buffer.c b/src/intel/vulkan_hasvk/gfx7_cmd_buffer.c index ef653ab..0a13541 100644 --- a/src/intel/vulkan_hasvk/gfx7_cmd_buffer.c +++ b/src/intel/vulkan_hasvk/gfx7_cmd_buffer.c @@ -232,7 +232,8 @@ genX(cmd_buffer_flush_dynamic_state)(struct anv_cmd_buffer *cmd_buffer) */ if ((cmd_buffer->state.gfx.dirty & ANV_CMD_DIRTY_PIPELINE) || BITSET_TEST(dyn->dirty, MESA_VK_DYNAMIC_IA_PRIMITIVE_TOPOLOGY) || - BITSET_TEST(dyn->dirty, MESA_VK_DYNAMIC_CB_COLOR_WRITE_ENABLES)) { + BITSET_TEST(dyn->dirty, MESA_VK_DYNAMIC_CB_COLOR_WRITE_ENABLES) || + BITSET_TEST(dyn->dirty, MESA_VK_DYNAMIC_RS_LINE_STIPPLE)) { VkPolygonMode dynamic_raster_mode = genX(raster_polygon_mode)(cmd_buffer->state.gfx.pipeline, dyn->ia.primitive_topology); @@ -243,10 +244,13 @@ genX(cmd_buffer_flush_dynamic_state)(struct anv_cmd_buffer *cmd_buffer) .ThreadDispatchEnable = anv_pipeline_has_stage(pipeline, MESA_SHADER_FRAGMENT) && (pipeline->force_fragment_thread_dispatch || - !anv_cmd_buffer_all_color_write_masked(cmd_buffer)), + !anv_cmd_buffer_all_color_write_masked(cmd_buffer)), .MultisampleRasterizationMode = genX(ms_rasterization_mode)(pipeline, dynamic_raster_mode), + #if GFX_VERx10 == 70 + .LineStippleEnable = dyn->rs.line.stipple.enable, + #endif }; GENX(3DSTATE_WM_pack)(NULL, dwords, &wm);