intel-hard/intel-hard-graphics
2
0 Fork 0
mirror of https://github.com/elwrcl/intel-hard-graphics.git synced 2026-10-31 19:08:03 +03:00
Code Issues Packages Projects Releases Wiki Activity

init

Browse Source
This commit is contained in:
elwrcl committed 2026-06-06 13:25:15 +03:00
1 parent 04d4dc8b09
commit 818cab85dc
12511 files changed
+6156641
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/util/.editorconfig
+5
View File
@@ -0,0 +1,5 @@
[rb_tree.c]
indent_size = 4
[tests/rb_tree_test.cpp]
indent_size = 4
src/util/00-anv-defaults.conf
+234
View File
@@ -0,0 +1,234 @@
<?xml version="1.0"?>
<driconf>
<device driver="anv">
<application name="Aperture Desk Job" executable="deskjob">
<option name="anv_assume_full_subgroups" value="32" />
</application>
<application name="A Plague Tale : Requiem" executable="APlagueTaleRequiem_x64.exe">
<option name="intel_storage_cache_policy_wt" value="true" />
</application>
<application name="Breaking Limit" executable="GPUScoreVulkan">
<option name="anv_assume_full_subgroups_with_barrier" value="true" />
</application>
<application name="Detroit: Become Human" executable="DetroitBecomeHuman.exe">
<option name="anv_assume_full_subgroups" value="32" />
</application>
<application name="DOOMEternal" executable="DOOMEternalx64vk.exe">
<option name="anv_assume_full_subgroups" value="32" />
<option name="fp64_workaround_enabled" value="true" />
</application>
<application name="Drive Beyond Horizons" executable="DriveBeyondHorizons.exe">
<option name="force_vk_vendor" value="-1" />
</application>
<application name="Farming Simulator 2022" executable="FarmingSimulator2022Game.exe">
<option name="limit_trig_input_range" value="true" />
</application>
<application name="The Last Of Us Part I" executable="tlou-i.exe">
<option name="anv_large_workgroup_non_coherent_image_workaround" value="true" />
</application>
<application name="The Last Of Us Part II Remastered" executable="tlou-ii.exe">
<option name="anv_large_workgroup_non_coherent_image_workaround" value="true" />
</application>
<application name="RESIDENT EVIL 2" executable="re2.exe">
<option name="anv_assume_full_subgroups_with_shared_memory" value="true" />
</application>
<application name="Wolfenstein: Youngblood(x64vk)" executable="Youngblood_x64vk.exe">
<option name="limit_trig_input_range" value="true" />
</application>
<application name="Batman™: Arkham Knight" executable="BatmanAK.exe">
<option name="anv_sample_mask_out_opengl_behaviour" value="true"/>
</application>
<application name="Company of Heroes 3" executable="RelicCoH3.exe">
<option name="limit_trig_input_range" value="true"/>
</application>
<application name="Rise of the Tomb Raider" executable="RiseOfTheTombRaider">
<option name="limit_trig_input_range" value="true" />
</application>
<application name="Rise of the Tomb Raider" executable="ROTTR.exe">
<option name="limit_trig_input_range" value="true" />
</application>
<application name="Sky: Children of the Light" executable="Sky.exe">
<option name="vk_lower_terminate_to_discard" value="true" />
</application>
<application name="Monster Hunter Wilds" executable="MonsterHunterWilds.exe">
<option name="anv_disable_link_time_optimization" value="true" />
</application>
<application name="NieR Replicant ver.1.22474487139" executable="NieR Replicant ver.1.22474487139.exe">
<option name="limit_trig_input_range" value="true" />
</application>
<application name="NieR:Automata" executable="NieRAutomata.exe">
<option name="limit_trig_input_range" value="true" />
</application>
<application name="Valheim" executable="valheim.x86_64">
<option name="limit_trig_input_range" value="true" />
</application>
<application name="Cyberpunk 2077" executable="Cyberpunk2077.exe">
<!--
Cyberpunk 2077 uses a watchdog thread to terminate
the process in case the render thread hasn't responded within 2 minutes.
This option speeds up shader compilation.
See: https://gitlab.freedesktop.org/mesa/mesa/-/issues/9241
-->
<option name="shader_spilling_rate" value="15" />
</application>
<application name="Elden Ring" executable="eldenring.exe">
<option name="fake_sparse" value="true" />
</application>
<application name="Armored Core 6" executable="armoredcore6.exe">
<option name="fake_sparse" value="true" />
</application>
<engine engine_name_match="vkd3d|DXVK">
<option name="anv_force_filter_addr_rounding" value="true" />
<option name="anv_promote_cbv_to_push_buffers" value="true" />
<option name="anv_fake_image_compression_control_xe2_plus" value="false" />
</engine>
<!-- Needed to avoid XeSS code paths. -->
<application name="Marvel's Spider-Man Remastered" executable="Spider-Man.exe">
<option name="force_vk_vendor" value="-1" />
<option name="shader_spilling_rate" value="0" />
</application>
<application name="Hitman 3" executable="hitman3.exe">
<option name="force_vk_vendor" value="-1"/>
</application>
<application name="Hogwarts Legacy" executable="HogwartsLegacy.exe">
<option name="force_vk_vendor" value="-1" />
<option name="intel_storage_cache_policy_wt" value="true" />
</application>
<application name="DEATH STRANDING" executable="ds.exe">
<option name="force_vk_vendor" value="-1" />
</application>
<application name="Diablo IV" executable="Diablo IV.exe">
<option name="force_vk_vendor" value="-1" />
</application>
<application name="Faaast Penguin" executable="FaaastPenguinClient.exe">
<option name="force_vk_vendor" value="-1" />
</application>
<application name="Satisfactory Steam" executable="FactoryGameSteam-Win64-Shipping.exe">
<option name="force_vk_vendor" value="-1" />
</application>
<application name="Satisfactory EGS" executable="FactoryGameEGS-Win64-Shipping.exe">
<option name="force_vk_vendor" value="-1" />
</application>
<application name="Space Engineers 2" executable="SpaceEngineers2.exe">
<option name="intel_storage_cache_policy_wt" value="true" />
</application>
<application name="Dying Light 2" executable="DyingLightGame_x64_rwdi.exe">
<option name="force_vk_vendor" value="-1" />
</application>
<application name="Witcher3" executable="witcher3.exe">
<option name="force_vk_vendor" value="-1" />
</application>
<application name="Baldur's Gate 3" executable="bg3.exe">
<option name="anv_disable_fcv" value="true" />
</application>
<application name="Baldur's Gate 3" executable="bg3">
<option name="anv_brw_disable_subgroup_size_control" value="true" />
</application>
<application name="The Finals" executable="Discovery.exe">
<option name="force_vk_vendor" value="-1" />
</application>
<application name="Palworld2" executable="Palworld-Win64-Shipping.exe">
<option name="force_vk_vendor" value="-1" />
</application>
<application name="Red Dead Redemption 2" executable="RDR2.exe">
<option name="force_vk_vendor" value="-1" />
</application>
<application name="Shadow of the Tomb Raider" executable="SOTTR.exe">
<option name="force_vk_vendor" value="-1" />
</application>
<application name="Silent Hill 2" executable="SHProto-Win64-Shipping.exe">
<option name="force_vk_vendor" value="-1" />
</application>
<application name="Marvel Rivals" executable="Marvel-Win64-Shipping.exe">
<option name="force_vk_vendor" value="-1" />
</application>
<application name="Bellwright" executable="BellwrightGame-Win64-Shipping.exe">
<option name="force_vk_vendor" value="-1" />
</application>
<application name="A Game About Digging A Hole" executable="DiggingGame.exe">
<option name="force_vk_vendor" value="-1" />
</application>
<application name="Jusant" executable="ASC-Win64-Shipping.exe">
<option name="force_vk_vendor" value="-1"/>
</application>
<application name="DIRT 5" executable="DIRT5.exe">
<option name="fp64_workaround_enabled" value="true" />
</application>
<application name="X4 Foundations" executable="X4">
<option name="anv_upper_bound_descriptor_pool_sampler" value="true" />
</application>
<application name="Total War: WARHAMMER III" executable="TotalWarhammer3">
<option name="anv_fake_nonlocal_memory" value="true" />
<option name="anv_enable_opt_divergent_atomics" value="1" />
</application>
<application name="Total War: WARHAMMER III" executable="Warhammer3.exe">
<option name="anv_enable_opt_divergent_atomics" value="1" />
</application>
<application name="The Elder Scrolls IV: Oblivion Remastered" executable="OblivionRemastered-Win.exe">
<option name="anv_enable_opt_divergent_atomics" value="1" />
</application>
<application name="Call of Duty: Black Ops III" executable="BlackOps3.exe">
<option name="anv_enable_opt_divergent_atomics" value="1" />
</application>
<application name="Horizon Forbidden West" executable="HorizonForbiddenWest.exe">
<!-- Missing barrier between clear/init shader & write
shader.
-->
<option name="anv_barrier_post_untyped_clear_shader" value="true" />
</application>
<application name="Blender" executable="blender">
<!-- Virtual shadow mapping page tagging in compute shaders
benefits from divergent atomics fusion, while fragment
shaders doing the atomic raster step in general have worse
performance with this optimization turned on. -->
<option name="anv_enable_opt_divergent_atomics_compute_only" value="1" />
</application>
<!-- Source2 games seem confused by the CCS-only memory type being
restricted to images, so allow it for buffers. More details on
the issue:
https://gitlab.freedesktop.org/mesa/mesa/-/issues/11520 -->
<engine engine_name_match="Source2">
<option name="anv_enable_buffer_comp" value="true" />
</engine>
<!-- Many DX12 games get a speedup by enabling buffer compression. -->
<engine engine_name_match="vkd3d">
<option name="anv_enable_buffer_comp" value="true" />
</engine>
<engine engine_name_match="GTK">
<option name="anv_disable_drm_ccs_modifiers" value="true" />
</engine>
<engine engine_name_match="GTK" engine_versions="16777216:16859138">
<option name="vk_wsi_disable_unordered_submits" value="true" />
</engine>
<!--
Disable 16-bit feature on zink and angle so that GLES mediump doesn't
lower to our inefficent 16-bit shader support. No need to do so for
ANGLE, since it uses RelaxedPrecision decorations, which the intel
compiler ignores.
-->
<engine engine_name_match="mesa zink">
<option name="no_16bit" value="true" />
</engine>
<!-- Disable FCV optimization for Unreal Engine 5.1 workloads. -->
<engine engine_name_match="UnrealEngine5.1">
<option name="anv_disable_fcv" value="true" />
<option name="anv_assume_full_subgroups" value="16" />
</engine>
<!-- We trust vkd3d to not disable compression all the time. -->
<engine engine_name_match="vkd3d">
<option name="compression_control_enabled" value="true" />
<option name="anv_enable_fully_covered" value="true" />
</engine>
<engine engine_name_match="ANGLE">
<option name="custom_border_colors_without_format" value="true" />
</engine>
<!-- Only use binding table entries for render targets. -->
<engine engine_name_match="vkd3d">
<option name="anv_state_cache_perf_fix" value="true" />
</engine>
<application name="Split Fiction" executable="SplitFiction.exe">
<!-- Problematic shader: dxil:2b33f0608c02f754 (TODO: there could be others) -->
<option name="anv_fs_sampler_undef_derivatives_workaround" value="true" />
</application>
</device>
</driconf>
src/util/00-asahi-defaults.conf
+82
View File
@@ -0,0 +1,82 @@
<?xml version="1.0"?>
<driconf>
<device driver="asahi">
<!-- Override GL renderer for known web browsers since one or more
web security frameworks apparently block clients with "Apple"
in the the renderer string if the OS is not one of Apple's.
Affected web sites include paypal.com, etsy.com and wsj.com.
Reported as web-compat bug in
https://github.com/webcompat/web-bugs/issues/189524
-->
<application name="Firefox" executable="firefox">
<option name="force_gl_renderer" value="AGX G13/G14"/>
</application>
<application name="Firefox ESR" executable="firefox-esr">
<option name="force_gl_renderer" value="AGX G13/G14"/>
</application>
<application name="Chromium" executable="chromium">
<option name="force_gl_renderer" value="AGX G13/G14"/>
</application>
<application name="Chromium" executable="chromium-browser">
<option name="force_gl_renderer" value="AGX G13/G14"/>
</application>
<application name="Google Chrome" executable="chrome">
<option name="force_gl_renderer" value="AGX G13/G14"/>
</application>
<application name="vivaldi-bin" executable="vivaldi-bin">
<option name="force_gl_renderer" value="AGX G13/G14"/>
</application>
<application name="Iceweasel" executable="iceweasel">
<option name="force_gl_renderer" value="AGX G13/G14"/>
</application>
<application name="Epiphany" executable="epiphany">
<option name="force_gl_renderer" value="AGX G13/G14"/>
</application>
<application name="Konqueror" executable="konqueror">
<option name="force_gl_renderer" value="AGX G13/G14"/>
</application>
<application name="Falkon" executable="falkon">
<option name="force_gl_renderer" value="AGX G13/G14"/>
</application>
<application name="Seamonkey" executable="seamonkey">
<option name="force_gl_renderer" value="AGX G13/G14"/>
</application>
<application name="Waterfox" executable="waterfox">
<option name="force_gl_renderer" value="AGX G13/G14"/>
</application>
<!-- Disable fp16 support for browsers, since there is too much
broken WebGL out there that uses the wrong precision.
Bonus workaround for Firefox bug #1845309. -->
<application name="Firefox" executable="firefox">
<option name="asahi_no_fp16" value="true" />
</application>
<application name="Firefox ESR" executable="firefox-esr">
<option name="asahi_no_fp16" value="true" />
</application>
<application name="Chromium" executable="chromium">
<option name="asahi_no_fp16" value="true" />
</application>
<application name="Google Chrome" executable="chrome">
<option name="asahi_no_fp16" value="true" />
</application>
<application name="Iceweasel" executable="iceweasel">
<option name="asahi_no_fp16" value="true" />
</application>
<application name="Epiphany" executable="epiphany">
<option name="asahi_no_fp16" value="true" />
</application>
<application name="Konqueror" executable="konqueror">
<option name="asahi_no_fp16" value="true" />
</application>
<application name="Falkon" executable="falkon">
<option name="asahi_no_fp16" value="true" />
</application>
<application name="Seamonkey" executable="seamonkey">
<option name="asahi_no_fp16" value="true" />
</application>
<application name="Waterfox" executable="waterfox">
<option name="asahi_no_fp16" value="true" />
</application>
</device>
</driconf>
src/util/00-crocus-defaults.conf
+14
View File
@@ -0,0 +1,14 @@
<?xml version="1.0"?>
<driconf>
<device driver="crocus">
<application name="glmark2" executable="glmark2">
<option name="limit_trig_input_range" value="true" />
</application>
<application name="Counter-Strike: Global Offensive" executable="csgo_linux64">
<option name="force_gl_vendor" value="X.Org" />
</application>
<application name="Insurgency" executable="insurgency_linux">
<option name="force_gl_vendor" value="X.Org" />
</application>
</device>
</driconf>
src/util/00-d3d12-defaults.conf
+11
View File
@@ -0,0 +1,11 @@
<?xml version="1.0"?>
<driconf>
<device driver="d3d12">
<application name="Blender" executable="blender.exe">
<!-- Blender needs GL4.3, but that requires a D3D feature that not all in-market
drivers support. Turns out they're fine with just the parts of this extension
that don't need the D3D feature, so we're good to just flip it on. -->
<option name="mesa_extension_override" value="+GL_ARB_texture_view" />
</application>
</device>
</driconf>
src/util/00-dzn-defaults.conf
+22
View File
@@ -0,0 +1,22 @@
<?xml version="1.0"?>
<driconf>
<device driver="dzn">
<application name="DOOMEternal" executable="DOOMEternalx64vk.exe">
<option name="dzn_enable_8bit_loads_stores" value="true" />
<option name="dzn_claim_wide_lines" value="true" />
</application>
<application name="No Man's Sky" executable="NMS.exe">
<option name="dzn_claim_wide_lines" value="true" />
</application>
<!-- RDR2 will default to Vulkan despite having a D3D path. Ideally
dzn would be "just as efficient" as the native path, but in practice
it's currently broken. Just disable dzn and let the app use native. -->
<application name="Red Dead Redemption 2" executable="RDR2.exe">
<option name="dzn_disable" value="true" />
</application>
<!-- Similar to RDR2, but selects dzn instead of native drivers. -->
<application name="Baldur's Gate 3" executable="bg3.exe">
<option name="dzn_disable" value="true" />
</application>
</device>
</driconf>
src/util/00-hasvk-defaults.conf
+136
View File
@@ -0,0 +1,136 @@
<?xml version="1.0"?>
<driconf>
<device driver="hasvk">
<application name="Aperture Desk Job" executable="deskjob">
<option name="anv_assume_full_subgroups" value="32" />
</application>
<application name="Brawlhalla" executable="BrawlhallaGame.exe">
<option name="hasvk_report_vk_1_3_version" value="true" />
</application>
<application name="Detroit: Become Human" executable="DetroitBecomeHuman.exe">
<option name="anv_assume_full_subgroups" value="32" />
</application>
<application name="DOOMEternal" executable="DOOMEternalx64vk.exe">
<option name="anv_assume_full_subgroups" value="32" />
</application>
<application name="Drive Beyond Horizons" executable="DriveBeyondHorizons.exe">
<option name="force_vk_vendor" value="-1" />
</application>
<application name="Farming Simulator 2022" executable="FarmingSimulator2022Game.exe">
<option name="limit_trig_input_range" value="true" />
</application>
<application name="Wolfenstein: Youngblood(x64vk)" executable="Youngblood_x64vk.exe">
<option name="limit_trig_input_range" value="true" />
</application>
<application name="Batman™: Arkham Knight" executable="BatmanAK.exe">
<option name="anv_sample_mask_out_opengl_behaviour" value="true"/>
</application>
<application name="Company of Heroes 3" executable="RelicCoH3.exe">
<option name="limit_trig_input_range" value="true"/>
</application>
<application name="Rise of the Tomb Raider" executable="RiseOfTheTombRaider">
<option name="limit_trig_input_range" value="true" />
</application>
<application name="Rise of the Tomb Raider" executable="ROTTR.exe">
<option name="limit_trig_input_range" value="true" />
</application>
<application name="Sky: Children of the Light" executable="Sky.exe">
<option name="vk_lower_terminate_to_discard" value="true" />
</application>
<application name="Monster Hunter Wilds" executable="MonsterHunterWilds.exe">
<option name="anv_disable_link_time_optimization" value="true" />
</application>
<application name="NieR Replicant ver.1.22474487139" executable="NieR Replicant ver.1.22474487139.exe">
<option name="limit_trig_input_range" value="true" />
</application>
<application name="NieR:Automata" executable="NieRAutomata.exe">
<option name="limit_trig_input_range" value="true" />
</application>
<application name="Valheim" executable="valheim.x86_64">
<option name="limit_trig_input_range" value="true" />
</application>
<application name="Cyberpunk 2077" executable="Cyberpunk2077.exe">
<!--
Cyberpunk 2077 uses a watchdog thread to terminate
the process in case the render thread hasn't responded within 2 minutes.
This option speeds up shader compilation.
See: https://gitlab.freedesktop.org/mesa/mesa/-/issues/9241
-->
<option name="shader_spilling_rate" value="15" />
</application>
<!-- Needed to avoid XeSS code paths. -->
<application name="Marvel's Spider-Man Remastered" executable="Spider-Man.exe">
<option name="force_vk_vendor" value="-1" />
<option name="shader_spilling_rate" value="0" />
</application>
<application name="Hitman 3" executable="hitman3.exe">
<option name="force_vk_vendor" value="-1"/>
</application>
<application name="Hogwarts Legacy" executable="HogwartsLegacy.exe">
<option name="force_vk_vendor" value="-1" />
</application>
<application name="DEATH STRANDING" executable="ds.exe">
<option name="force_vk_vendor" value="-1" />
</application>
<application name="Diablo IV" executable="Diablo IV.exe">
<option name="force_vk_vendor" value="-1" />
</application>
<application name="Faaast Penguin" executable="FaaastPenguinClient.exe">
<option name="force_vk_vendor" value="-1" />
</application>
<application name="Satisfactory Steam" executable="FactoryGameSteam-Win64-Shipping.exe">
<option name="force_vk_vendor" value="-1" />
</application>
<application name="Satisfactory EGS" executable="FactoryGameEGS-Win64-Shipping.exe">
<option name="force_vk_vendor" value="-1" />
</application>
<application name="Dying Light 2" executable="DyingLightGame_x64_rwdi.exe">
<option name="force_vk_vendor" value="-1" />
</application>
<application name="Witcher3" executable="witcher3.exe">
<option name="force_vk_vendor" value="-1" />
</application>
<application name="The Finals" executable="Discovery.exe">
<option name="force_vk_vendor" value="-1" />
</application>
<application name="Palworld2" executable="Palworld-Win64-Shipping.exe">
<option name="force_vk_vendor" value="-1" />
</application>
<application name="Red Dead Redemption 2" executable="RDR2.exe">
<option name="force_vk_vendor" value="-1" />
</application>
<application name="Shadow of the Tomb Raider" executable="SOTTR.exe">
<option name="force_vk_vendor" value="-1" />
</application>
<application name="Silent Hill 2" executable="SHProto-Win64-Shipping.exe">
<option name="force_vk_vendor" value="-1" />
</application>
<application name="Marvel Rivals" executable="Marvel-Win64-Shipping.exe">
<option name="force_vk_vendor" value="-1" />
</application>
<application name="Bellwright" executable="BellwrightGame-Win64-Shipping.exe">
<option name="force_vk_vendor" value="-1" />
</application>
<application name="A Game About Digging A Hole" executable="DiggingGame.exe">
<option name="force_vk_vendor" value="-1" />
</application>
<application name="Jusant" executable="ASC-Win64-Shipping.exe">
<option name="force_vk_vendor" value="-1"/>
</application>
<engine engine_name_match="GTK">
<option name="anv_disable_drm_ccs_modifiers" value="true" />
</engine>
<engine engine_name_match="GTK" engine_versions="16777216:16859138">
<option name="vk_wsi_disable_unordered_submits" value="true" />
</engine>
<!--
Disable 16-bit feature on zink and angle so that GLES mediump doesn't
lower to our inefficent 16-bit shader support. No need to do so for
ANGLE, since it uses RelaxedPrecision decorations, which the intel
compiler ignores.
-->
<engine engine_name_match="mesa zink">
<option name="no_16bit" value="true" />
</engine>
</device>
</driconf>
src/util/00-hk-defaults.conf
+36
View File
@@ -0,0 +1,36 @@
<?xml version="1.0"?>
<driconf>
<device driver="hk">
<engine engine_name_match="DXVK|vkd3d">
<!--
The amount of bug reports we get about Honeykrisp being slow
outweighs the number of games that need custom border emulation.
I'm tired.
-->
<option name="hk_disable_border_emulation" value="true" />
<!--
Needed for FL11_1. While nominally conformant, CTS hits piles
of timeouts (== flakes & fails & other fun times), especially on
min-spec M1. Therefore the feature is hidden. I don't expect native
apps to use this.
-->
<option name="hk_enable_vertex_pipeline_stores_atomics" value="true" />
</engine>
<engine engine_name_match="vkd3d">
<!--
Apple hardware lacks min/max filtering, but this is required for
FL12_0. Fortunately, this is an obscure feature and we support
the rest of FL12_0, so we prefer to fake support and fail D3D
conformance rather than be limited to FL11.
-->
<option name="hk_fake_minmax" value="true" />
<!---
We need this for FL12_0, but we only implement it for vkd3d
because it's not the speediest thing.
-->
<option name="hk_image_view_min_lod" value="true" />
</engine>
</device>
</driconf>
src/util/00-iris-defaults.conf
+27
View File
@@ -0,0 +1,27 @@
<?xml version="1.0"?>
<driconf>
<device driver="iris">
<application name="Middle Earth: Shadow of Mordor" executable="ShadowOfMordor">
<option name="vs_position_always_invariant" value="true" />
</application>
<application name="glmark2" executable="glmark2">
<option name="limit_trig_input_range" value="true" />
</application>
<application name="Counter-Strike: Global Offensive" executable="csgo_linux64">
<option name="force_gl_vendor" value="X.Org" />
</application>
<application name="Insurgency" executable="insurgency_linux">
<option name="force_gl_vendor" value="X.Org" />
</application>
<application name="Amnesia Bunker" executable="AmnesiaTheBunker.exe">
<option name="intel_disable_threaded_context" value="true" />
</application>
<application name="Amnesia Bunker" executable="AmnesiaTheBunker_NoSteam.exe">
<option name="intel_disable_threaded_context" value="true" />
</application>
<application name="SPECviewperf13" executable="viewperf">
<!-- creo-03 needs this to compile shaders; we don't support some corner cases -->
<option name="mesa_extension_override" value="+GL_EXT_shader_image_load_store" />
</application>
</device>
</driconf>
src/util/00-mesa-defaults.conf
+721
View File
@@ -0,0 +1,721 @@
<?xml version="1.0"?>
<!-- This file describes the drirc options that apply to all drivers. -->
<driconf>
<device>
<application name="Akka Arrh" executable="Project-A.exe">
<option name="allow_glsl_extension_directive_midshader" value="true" />
</application>
<application name="Unigine Sanctuary" executable="Sanctuary">
<option name="force_glsl_extensions_warn" value="true" />
<option name="disable_blend_func_extended" value="true" />
<option name="disable_arb_gpu_shader5" value="true" />
</application>
<application name="Unigine Tropics" executable="Tropics">
<option name="force_glsl_extensions_warn" value="true" />
<option name="disable_blend_func_extended" value="true" />
<option name="disable_arb_gpu_shader5" value="true" />
</application>
<application name="Unigine Heaven (32-bit)" executable="heaven_x86">
<option name="allow_glsl_extension_directive_midshader" value="true" />
<!-- remove dual_color_blend_by_location if 4.1 ever comes out -->
<option name="dual_color_blend_by_location" value="true" />
</application>
<application name="Unigine Heaven (64-bit)" executable="heaven_x64">
<option name="allow_glsl_extension_directive_midshader" value="true" />
<!-- remove dual_color_blend_by_location if 4.1 ever comes out -->
<option name="dual_color_blend_by_location" value="true" />
</application>
<application name="Unigine Heaven (Windows)" executable="heaven.exe">
<option name="allow_glsl_extension_directive_midshader" value="true" />
<!-- remove dual_color_blend_by_location if 4.1 ever comes out -->
<option name="dual_color_blend_by_location" value="true" />
</application>
<application name="Unigine Valley (32-bit)" executable="valley_x86">
<option name="allow_glsl_extension_directive_midshader" value="true" />
<!-- remove dual_color_blend_by_location if 1.1 ever comes out -->
<option name="dual_color_blend_by_location" value="true" />
</application>
<application name="Unigine Valley (64-bit)" executable="valley_x64">
<option name="allow_glsl_extension_directive_midshader" value="true" />
<!-- remove dual_color_blend_by_location if 1.1 ever comes out -->
<option name="dual_color_blend_by_location" value="true" />
</application>
<application name="Unigine Valley (Windows)" executable="valley.exe">
<option name="allow_glsl_extension_directive_midshader" value="true" />
<!-- remove dual_color_blend_by_location if 1.1 ever comes out -->
<option name="dual_color_blend_by_location" value="true" />
</application>
<application name="Unigine OilRush (32-bit)" executable="OilRush_x86">
<option name="disable_blend_func_extended" value="true" />
<option name="allow_glsl_extension_directive_midshader" value="true" />
</application>
<application name="Unigine OilRush (64-bit)" executable="OilRush_x64">
<option name="disable_blend_func_extended" value="true" />
<option name="allow_glsl_extension_directive_midshader" value="true" />
</application>
<application name="Savage 2" executable="savage2.bin">
<option name="disable_glsl_line_continuations" value="true" />
</application>
<!-- For issues with the legacy OpenGL version of TF2 -->
<application name="Team Fortress 2" executable="tf_linux64">
<option name="disable_uniform_array_resize" value="true" />
</application>
<application name="Topogun (32-bit)" executable="topogun32">
<option name="always_have_depth_buffer" value="true" />
</application>
<application name="Topogun (64-bit)" executable="topogun64">
<option name="always_have_depth_buffer" value="true" />
</application>
<application name="Dead Island (incl. Definitive Edition)" executable="DeadIslandGame">
<option name="allow_glsl_extension_directive_midshader" value="true" />
<!-- For the Definitive Edition which shares the same executable name -->
<option name="allow_glsl_builtin_variable_redeclaration" value="true" />
</application>
<application name="Dead Island Riptide Definitive Edition" executable="DeadIslandRiptideGame">
<option name="allow_glsl_builtin_variable_redeclaration" value="true" />
</application>
<application name="Doom 3: BFG" executable="Doom3BFG.exe">
<option name="allow_glsl_builtin_variable_redeclaration" value="true" />
<option name="force_glsl_extensions_warn" value="true" />
</application>
<application name="Dune: Spice Wars" executable="D4X.exe">
<option name="disable_uniform_array_resize" value="true" />
</application>
<application name="The Chronicles of Riddick: Assault on Dark Athena" executable="DarkAthena.exe">
<option name="disable_uniform_array_resize" value="true" />
<option name="alias_shader_extension" value="GL_ATI_shader_texture_lod:GL_ARB_shader_texture_lod" />
<option name="allow_vertex_texture_bias" value="true" />
</application>
<application name="Cursemark" executable="Cursemark.exe">
<option name="glsl_zero_init" value="true"/>
</application>
<application name="Dying Light" executable="DyingLightGame">
<option name="allow_glsl_builtin_variable_redeclaration" value="true" />
<option name="dual_color_blend_by_location" value="true" />
</application>
<application name="Exanima" executable="Exanima.exe">
<option name="glsl_zero_init" value="true"/>
</application>
<application name="Full Bore" executable="fullbore">
<option name="allow_glsl_builtin_variable_redeclaration" value="true" />
</application>
<application name="RAGE (64-bit)" executable="Rage64.exe">
<option name="allow_glsl_builtin_variable_redeclaration" value="true" />
<option name="force_gl_map_buffer_synchronized" value="true" />
</application>
<application name="RAGE (32-bit)" executable="Rage.exe">
<option name="allow_glsl_builtin_variable_redeclaration" value="true" />
<option name="force_gl_map_buffer_synchronized" value="true" />
</application>
<application name="Second Life" executable="do-not-directly-run-secondlife-bin">
<option name="allow_glsl_extension_directive_midshader" value="true" />
</application>
<application name="TombRaider 4-5-6 Remastered" executable="tomb456.exe">
<option name="force_gl_depth_component_type_int" value="true" />
</application>
<application name="Warsow (32-bit)" executable="warsow.i386">
<option name="allow_glsl_extension_directive_midshader" value="true" />
</application>
<application name="Warsow (64-bit)" executable="warsow.x86_64">
<option name="allow_glsl_extension_directive_midshader" value="true" />
</application>
<application name="Rust" executable="rust">
<option name="glsl_zero_init" value="true"/>
</application>
<application name="Divinity: Original Sin Enhanced Edition" executable="EoCApp">
<option name="allow_glsl_extension_directive_midshader" value="true" />
<option name="glsl_correct_derivatives_after_discard" value="true" />
<option name="force_gl_vendor" value="ATI Technologies, Inc."/>
</application>
<application name="Metro 2033 Redux / Metro Last Night Redux" executable="metro">
<option name="allow_glsl_extension_directive_midshader" value="true" />
</application>
<application name="Worms W.M.D" executable="Worms W.M.Dx64">
<option name="allow_higher_compat_version" value="true" />
</application>
<application name="Crookz - The Big Heist" executable="Crookz">
<option name="allow_higher_compat_version" value="true" />
</application>
<application name="Tropico 5" executable="Tropico5">
<option name="allow_higher_compat_version" value="true" />
</application>
<application name="Faster than Light (32-bit)" executable="FTL.x86">
<option name="allow_higher_compat_version" value="true" />
</application>
<application name="Faster than Light (64-bit)" executable="FTL.amd64">
<option name="allow_higher_compat_version" value="true" />
</application>
<application name="Final Fantasy VIII: Remastered" executable="FFVIII.exe">
<option name="allow_higher_compat_version" value="true" />
</application>
<application name="Lethis - Path Of Progress" executable="Lethis-Path_Of_Progress.exe">
<option name="force_explicit_uniform_loc_zero" value="fQualityFactor" />
<option name="force_compat_profile" value="true" />
</application>
<application name="SNK HEROINES Tag Team Frenzy" executable="SNKHEROINES.exe">
<option name="force_compat_shaders" value="true" />
</application>
<application name="Metal Slug XX" executable="MSXX_main.exe">
<option name="force_compat_shaders" value="true" />
</application>
<application name="The Culling" executable="Victory">
<option name="force_glsl_version" value="440" />
</application>
<application name="Spec Ops: The Line (32-bit)" executable="specops.i386">
<option name="force_glsl_abs_sqrt" value="true" />
</application>
<application name="Spec Ops: The Line (64-bit)" executable="specops">
<option name="force_glsl_abs_sqrt" value="true" />
</application>
<application name="Interstellar Rift" executable="IR.exe">
<option name="force_compat_shaders" value="true" />
<option name="glx_clear_context_reset_isolation_bit" value="true" />
</application>
<application name="Jamestown+" executable="JamestownPlus.exe">
<option name="force_compat_profile" value="true" />
</application>
<application name="Kerbal Space Program (32-bit)" executable="KSP.x86">
<option name="glsl_zero_init" value="true"/>
</application>
<application name="Kerbal Space Program (64-bit)" executable="KSP.x86_64">
<option name="glsl_zero_init" value="true"/>
</application>
<application name="Luna Sky" executable="lunasky">
<option name="force_glsl_version" value="130"/>
<option name="glsl_ignore_write_to_readonly_var" value="true"/>
</application>
<application name="MDK2 HD" executable="mdk2hd.exe">
<option name="allow_glsl_embedded_structure_declarations" value="true"/>
</application>
<application name="Rocket League" executable="RocketLeague">
<option name="glsl_correct_derivatives_after_discard" value="true"/>
</application>
<application name="The Witcher 2" executable="witcher2">
<option name="glsl_correct_derivatives_after_discard" value="true"/>
</application>
<application name="Unreal 4 Editor" executable="UE4Editor">
<option name="allow_glsl_cross_stage_interpolation_mismatch" value="true"/>
</application>
<application name="Observer" executable="TheObserver-Linux-Shipping">
<option name="allow_glsl_cross_stage_interpolation_mismatch" value="true"/>
</application>
<!-- Pixel Game Maker MV workarounds. Games include: YuuYuu Jiteki no Yuukarin, Elfin Force, Eden Gamma, etc -->
<application name="Pixel Game Maker MV" executable="player.exe">
<option name="allow_glsl_120_subset_in_110" value="true" />
</application>
<application name="Eden Gamma" executable="EdenGammaGame-0.3.0.2.exe">
<option name="allow_glsl_120_subset_in_110" value="true" />
</application>
<application name="Steamroll" executable="Steamroll-Linux-Shipping">
<option name="allow_glsl_cross_stage_interpolation_mismatch" value="true"/>
</application>
<application name="Refunct" executable="Refunct-Linux-Shipping">
<option name="allow_glsl_cross_stage_interpolation_mismatch" value="true"/>
</application>
<application name="We Happy Few" executable="GlimpseGame">
<option name="allow_glsl_cross_stage_interpolation_mismatch" value="true"/>
</application>
<application name="Google Earth VR" executable="Earth.exe">
<option name="allow_glsl_builtin_const_expression" value="true"/>
<option name="allow_glsl_relaxed_es" value="true"/>
</application>
<application name="Champions of Regnum" executable="game">
<option name="allow_extra_pp_tokens" value="true" />
</application>
<application name="Wolfenstein The Old Blood" executable="WolfOldBlood_x64.exe">
<option name="force_compat_profile" value="true" />
</application>
<application name="ARMA 3" executable="arma3.x86_64">
<option name="glsl_correct_derivatives_after_discard" value="true"/>
</application>
<application name="Epic Games Launcher" executable="EpicGamesLauncher.exe">
<option name="force_compat_profile" value="true" />
</application>
<application name="GpuTest" executable="GpuTest">
<!-- GiMark benchmark from GpuTest does not initialize .w channel of
lightDir0 and lightDir1 varyings in the vertex shader while
uses it in the fragment shader. -->
<option name="glsl_zero_init" value="true"/>
</application>
<application name="Curse of the Dead Gods" executable="Curse of the Dead Gods.exe">
<!-- https://gitlab.freedesktop.org/mesa/mesa/-/issues/2598 -->
<option name="glsl_zero_init" value="true" />
</application>
<application name="GRID Autosport" executable="GridAutosport">
<!-- https://gitlab.freedesktop.org/mesa/mesa/-/issues/1258 -->
<option name="force_integer_tex_nearest" value="true" />
</application>
<application name="DIRT: Showdown" executable="dirt.i386">
<!-- https://gitlab.freedesktop.org/mesa/mesa/-/issues/1258 -->
<option name="force_integer_tex_nearest" value="true" />
</application>
<application name="DiRT Rally" executable="DirtRally">
<!-- https://gitlab.freedesktop.org/mesa/mesa/-/issues/5648 -->
<option name="vs_position_always_invariant" value="true" />
</application>
<application name="Foundation" executable="foundation.exe">
<!-- https://gitlab.freedesktop.org/mesa/mesa/-/issues/12882 -->
<option name="force_integer_tex_nearest" value="true" />
</application>
<application name="Homerun Clash" executable="com.haegin.homerunclash">
<!-- https://gitlab.freedesktop.org/mesa/mesa/-/issues/7199 -->
<option name="lower_depth_range_rate" value="0.8" />
</application>
<application name="The Spirit and The Mouse" executable="TheSpiritAndTheMouse.exe">
<!-- https://gitlab.freedesktop.org/mesa/mesa/-/issues/9738 -->
<option name="lower_depth_range_rate" value="0.8" />
</application>
<application name="Penumbra: Overture (Windows)" executable="Penumbra.exe">
<!-- https://gitlab.freedesktop.org/mesa/mesa/-/issues/8425 -->
<option name="allow_multisampled_copyteximage" value="true" />
<option name="vertex_program_default_out" value="true" />
</application>
<application name="Penumbra: Overture" executable="penumbra.bin">
<!-- https://gitlab.freedesktop.org/mesa/mesa/-/issues/8425 -->
<option name="allow_multisampled_copyteximage" value="true" />
<option name="vertex_program_default_out" value="true" />
</application>
<!-- Workarounds for SPECviewperf relying on invalid / non-conformant
OpenGL behavior. Older SPECviewperf versions might also need this.
-->
<application name="SPECviewperf13" executable="viewperf">
<option name="vblank_mode" value="0" />
<option name="glthread_nop_check_framebuffer_status" value="true" />
<option name="allow_glsl_extension_directive_midshader" value="true" />
<option name="allow_glsl_120_subset_in_110" value="true" />
<!-- This makes it use GL_NVX_gpu_memory_info to query memory info
instead of non-existent GLX_AMD_gpu_association. Yes, it checks
the vendor string, not the extension list. -->
<option name="force_gl_vendor" value="NVIDIA Corporation" />
<!-- creo-02 doesn't enable GL_EXT_shader_image_load_store in GLSL -->
<option name="force_glsl_extensions_warn" value="true" />
<option name="mesa_no_error" value="true" />
<!-- Creating 10-bit pbuffers fails in the X server and returns BadAlloc. -->
<option name="allow_rgb10_configs" value="false" />
<option name="allow_invalid_glx_destroy_window" value="true" />
<!-- 16-bit configs cause issues with viewset maya_06. -->
<option name="allow_rgb16_configs" value="false" />
</application>
<!-- Workaround for unsynchronized VBO updates on Dead Cells android
game. (Possibly also needed for desktop version?)
-->
<application name="Dead-Cells" executable="com.playdigious.deadcells.mobile">
<option name="ignore_map_unsynchronized" value="true" />
</application>
<application name="Teardown" executable="teardown.exe">
<!-- https://gitlab.freedesktop.org/mesa/mesa/-/issues/3714 -->
<option name="force_integer_tex_nearest" value="true" />
<option name="allow_glsl_extension_directive_midshader" value="true" />
</application>
<application name="Golf With Your Friends" executable="Golf With Your Friends.x86_64">
<!-- https://gitlab.freedesktop.org/mesa/mesa/-/issues/4547 -->
<option name="glsl_correct_derivatives_after_discard" value="true" />
</application>
<application name="Cossacks 3" executable="cossacks.exe">
<option name="allow_glsl_120_subset_in_110" value="true" />
</application>
<application name="Kaiju-A-Gogo" executable="kaiju.exe">
<option name="allow_glsl_120_subset_in_110" value="true" />
</application>
<application name="Captain Lycop: Invasion of the Heters (Wine)" executable="lycop.exe">
<option name="allow_glsl_120_subset_in_110" value="true" />
</application>
<application name="Captain Lycop: Invasion of the Heters" executable="lycop">
<option name="allow_glsl_120_subset_in_110" value="true" />
</application>
<application name="Joe Danger (Wine)" executable="JoeDanger.exe">
<option name="allow_glsl_120_subset_in_110" value="true" />
</application>
<application name="Joe Danger 2 (Wine)" executable="JoeDanger2.exe">
<option name="allow_glsl_120_subset_in_110" value="true" />
</application>
<application name="Investigation Stories : gunsound (Wine)" executable="gunsound.exe">
<option name="allow_glsl_120_subset_in_110" value="true" />
</application>
<application name="BETA CAE Systems - GL detect tool" executable="detect_opengl_tool">
<option name="mesa_extension_override" value="-GL_MESA_pack_invert -GL_MESA_framebuffer_flip_y -GL_MESA_window_pos" />
</application>
<application name="BETA CAE Systems - ANSA" executable="ansa_linux_x86_64">
<option name="mesa_extension_override" value="-GL_MESA_pack_invert -GL_MESA_framebuffer_flip_y -GL_MESA_window_pos" />
<option name="do_dce_before_clip_cull_analysis" value="true" />
</application>
<application name="BETA CAE Systems - META" executable="meta_post_x86_64">
<option name="mesa_extension_override" value="-GL_MESA_pack_invert -GL_MESA_framebuffer_flip_y -GL_MESA_window_pos" />
</application>
<!-- Workaround for Mari with executable name like Mari4.5v2, Mari4.7v4 -->
<application name="Mari" executable_regexp="Mari[0-9]+[.][0-9]+v[0-9]+">
<!-- Need ARB_gpu_shader5 for non-const sampler array index support -->
<option name="force_glsl_extensions_warn" value="true" />
</application>
<application name="Discovery Studio 2020" executable="DiscoveryStudio2020-bin">
<option name="force_direct_glx_context" value="true" />
</application>
<application name="Abaqus" executable="ABQcaeK">
<option name="keep_native_window_glx_drawable" value="true" />
<option name="allow_rgb10_configs" value="false" />
</application>
<application name="Maya" executable="maya.bin">
<!-- Needed otherwise the app doesn't try to use OpenCL -->
<option name="force_gl_vendor" value="ATI Technologies Inc." />
</application>
<application name="SD Gundam G Generation Cross Rays" executable="togg.exe">
<option name="force_compat_shaders" value="true" />
</application>
<application name="FINAL FANTASY XI" executable="pol.exe">
<!-- Causes invisible textures. -->
<!-- https://gitlab.freedesktop.org/mesa/mesa/-/issues/7562 -->
<option name="mesa_glthread_app_profile" value="0"/>
</application>
<application name="Talos Principle" executable="Talos">
<option name="vk_wsi_force_swapchain_to_current_extent" value="true"/>
<option name="vk_x11_ignore_suboptimal" value="true"/>
</application>
<application name="Talos Principle (Unrestricted)" executable="Talos_Unrestricted">
<option name="vk_wsi_force_swapchain_to_current_extent" value="true"/>
<option name="vk_x11_ignore_suboptimal" value="true"/>
</application>
<application name="Talos Principle VR" executable="Talos_VR">
<option name="vk_wsi_force_swapchain_to_current_extent" value="true"/>
<option name="vk_x11_ignore_suboptimal" value="true"/>
</application>
<application name="Talos Principle VR (Unrestricted)" executable="Talos_Unrestricted_VR">
<option name="vk_wsi_force_swapchain_to_current_extent" value="true"/>
<option name="vk_x11_ignore_suboptimal" value="true"/>
</application>
<application name="Serious Sam Fusion" executable="Sam2017">
<option name="vk_wsi_force_swapchain_to_current_extent" value="true"/>
<option name="vk_x11_ignore_suboptimal" value="true"/>
</application>
<application name="Serious Sam Fusion (Unrestricted)" executable="Sam2017_Unrestricted">
<option name="vk_wsi_force_swapchain_to_current_extent" value="true"/>
<option name="vk_x11_ignore_suboptimal" value="true"/>
</application>
<application name="DeusExMD" executable="DeusExMD">
<option name="mesa_glthread_app_profile" value="0"/>
</application>
<application name="F1 2015" executable="F12015">
<option name="mesa_glthread_app_profile" value="0"/>
</application>
<application name="KWin Wayland" executable="kwin_wayland">
<!-- This works around a cursor issue with glthread.
See: https://gitlab.freedesktop.org/mesa/mesa/-/issues/7351#note_1581105 -->
<option name="mesa_glthread_app_profile" value="0"/>
</application>
<!-- Adaptive sync denylist follows below: -->
<application name="gnome-shell" executable="gnome-shell">
<option name="adaptive_sync" value="false" />
</application>
<application name="Desktop — Plasma" executable="plasmashell">
<option name="adaptive_sync" value="false" />
</application>
<application name="budgie-wm" executable="budgie-wm">
<option name="adaptive_sync" value="false" />
</application>
<application name="kwin_x11" executable="kwin_x11">
<option name="adaptive_sync" value="false" />
</application>
<application name="ksmserver-logout-greeter" executable="ksmserver-logout-greeter">
<option name="adaptive_sync" value="false" />
</application>
<application name="ksmserver-switchuser-greeter" executable="ksmserver-switchuser-greeter">
<option name="adaptive_sync" value="false" />
</application>
<application name="kscreenlocker_greet" executable="kscreenlocker_greet">
<option name="adaptive_sync" value="false" />
</application>
<application name="startplasma" executable="startplasma">
<option name="adaptive_sync" value="false" />
</application>
<application name="sddm-greeter" executable="sddm-greeter">
<option name="adaptive_sync" value="false" />
</application>
<application name="krunner" executable="krunner">
<option name="adaptive_sync" value="false" />
</application>
<application name="spectacle" executable="spectacle">
<option name="adaptive_sync" value="false" />
</application>
<application name="marco" executable="marco">
<option name="adaptive_sync" value="false" />
</application>
<application name="compton" executable="compton">
<option name="adaptive_sync" value="false" />
</application>
<application name="picom" executable="picom">
<option name="adaptive_sync" value="false" />
</application>
<application name="xfwm4" executable="xfwm4">
<option name="adaptive_sync" value="false" />
</application>
<application name="Enlightenment" executable="enlightenment">
<option name="adaptive_sync" value="false" />
</application>
<application name="mutter" executable="mutter">
<option name="adaptive_sync" value="false" />
</application>
<application name="muffin" executable="muffin">
<option name="adaptive_sync" value="false" />
</application>
<application name="cinnamon" executable="cinnamon">
<option name="adaptive_sync" value="false" />
</application>
<application name="compiz" executable="compiz">
<option name="adaptive_sync" value="false" />
</application>
<application name="Firefox" executable="firefox">
<option name="adaptive_sync" value="false" />
</application>
<application name="Firefox ESR" executable="firefox-esr">
<option name="adaptive_sync" value="false" />
</application>
<application name="Chromium" executable="chromium">
<option name="adaptive_sync" value="false" />
</application>
<application name="Google Chrome" executable="chrome">
<option name="adaptive_sync" value="false" />
</application>
<application name="Iceweasel" executable="iceweasel">
<option name="adaptive_sync" value="false" />
</application>
<application name="Epiphany" executable="epiphany">
<option name="adaptive_sync" value="false" />
</application>
<application name="Konqueror" executable="konqueror">
<option name="adaptive_sync" value="false" />
</application>
<application name="Falkon" executable="falkon">
<option name="adaptive_sync" value="false" />
</application>
<application name="Seamonkey" executable="seamonkey">
<option name="adaptive_sync" value="false" />
</application>
<application name="Waterfox" executable="waterfox">
<option name="adaptive_sync" value="false" />
</application>
<application name="VLC Media Player" executable="vlc">
<option name="adaptive_sync" value="false" />
</application>
<application name="Totem" executable="totem">
<option name="adaptive_sync" value="false" />
</application>
<application name="Dragon Player" executable="dragon">
<option name="adaptive_sync" value="false" />
</application>
<application name="gfxbench" executable="testfw_app">
<option name="mesa_glthread_app_profile" value="0" />
<option name="vk_x11_override_min_image_count" value="2" />
<option name="vk_wsi_force_bgra8_unorm_first" value="true" />
</application>
<application name="Rainbow Six Siege (Vulkan)" executable="RainbowSix_Vulkan.exe">
<option name="vk_x11_override_min_image_count" value="2" />
<option name="vk_x11_strict_image_count" value="true" />
</application>
<application name="Rainbow Six Extraction (Wine)" executable="R6-Extraction.exe">
<option name="vk_x11_override_min_image_count" value="2" />
<option name="vk_x11_strict_image_count" value="true" />
</application>
<!-- Workaround for Hades: Vulkan backend of the game is not starting
if the implementation returns more than 3 swapchain images. -->
<application name="Hades" executable="Hades.exe">
<option name="vk_x11_override_min_image_count" value="3" />
<option name="vk_x11_strict_image_count" value="true" />
</application>
<application name="Brink" executable="brink.exe">
<option name="glx_extension_override" value="-GLX_OML_swap_method" />
<option name="allow_higher_compat_version" value="true" />
</application>
<application name="Enter The Gungeon (32 bits)" executable="EtG.x86">
<option name="glsl_correct_derivatives_after_discard" value="true" />
</application>
<application name="Enter The Gungeon (64 bits)" executable="EtG.x86_64">
<option name="glsl_correct_derivatives_after_discard" value="true" />
</application>
<application name="Assault Android Cactus (32-bit)" executable="cactus_demo.x86">
<option name="vs_position_always_precise" value="true" />
</application>
<application name="Assault Android Cactus (64-bit)" executable="cactus_demo.x86_64">
<option name="vs_position_always_precise" value="true" />
</application>
<application name="Assault Android Cactus (32-bit)" executable="cactus.x86">
<option name="vs_position_always_precise" value="true" />
</application>
<application name="Assault Android Cactus (64-bit)" executable="cactus.x86_64">
<option name="vs_position_always_precise" value="true" />
</application>
<application name="Limbo" executable="limbo">
<!-- issue 7976 -->
<option name="glsl_zero_init" value="true"/>
</application>
<application name="Genshin Impact" executable="com.miHoYo.GenshinImpact">
<option name="ignore_discard_framebuffer" value="true" />
</application>
<!-- Vulkan workarounds: -->
<!-- Works around the game not starting (does not deal with
the implementation returning more images than the minimum
specified by the application. -->
<engine engine_name_match="UnrealEngine4.*" engine_versions="0:23">
<option name="vk_x11_strict_image_count" value="true" />
</engine>
<!-- DOOM crashes at startup without this workaround if the adaptative sync mode is enabled. -->
<application name="DOOM" executable="DOOMx64vk.exe">
<option name="vk_x11_strict_image_count" value="true" />
</application>
<!-- DOOM Doom Eternal happily creates a swapchain with 2 images for
IMMEDIATE. This fixes a 10% performance issue with RADV. -->
<application name="DOOMEternal" executable="DOOMEternalx64vk.exe">
<option name="vk_x11_ensure_min_image_count" value="true" />
</application>
<application name="Path of Exile" executable="PathOfExile_x64Steam.exe">
<option name="vk_zero_vram" value="true" />
</application>
<application name="Path of Exile" executable="PathOfExileSteam.exe">
<option name="vk_zero_vram" value="true" />
</application>
<application name="X4 Foundations" executable="X4">
<option name="vk_zero_vram" value="true" />
</application>
<!-- Wolfenstein: Youngblood crashes at startup on xwayland without this workaround. -->
<application name="Wolfenstein: Youngblood(x64vk)" executable="Youngblood_x64vk.exe">
<option name="vk_x11_strict_image_count" value="true" />
</application>
<!-- Wolfenstein II: The New Colossus crashes with vsync turned off on xwayland without this workaround. -->
<application name="Wolfenstein II: The New Colossus" executable="NewColossus_x64vk.exe">
<option name="vk_x11_strict_image_count" value="true" />
</application>
<!-- Metro:Exodus crashes at startup on xwayland without this workaround. -->
<application name="Metro: Exodus" application_name_match="metroexodus">
<option name="vk_x11_strict_image_count" value="true" />
</application>
<!-- Atlas Fallen Vulkan crashes with vsync turned off on xwayland without this workaround. -->
<application name="Atlas Fallen" executable="AtlasFallen (VK).exe">
<option name="vk_x11_strict_image_count" value="true" />
</application>
<!-- Game does not consider larger image count in non-vsynced modes. -->
<application name="Detroit Become Human" application_name_match="DetroitBecomeHuman">
<option name="vk_x11_strict_image_count" value="true" />
</application>
<!-- Application exits with an error if contents do not show up to GDI -->
<application name="文香白板" executable="WxBoardMain.exe">
<option name="wgl_require_gdi_compat" value="true" />
</application>
</device>
</driconf>
src/util/00-msm-defaults.conf
+93
View File
@@ -0,0 +1,93 @@
<?xml version="1.0"?>
<driconf>
<!--
The android game hall of shame:
Workarounds for android games that are not aware of open source drivers.
These can be removed as games have their allowlists updated.
-->
<device driver="msm" device="FD618">
<application name="Call of Duty Mobile" executable="com.activision.callofduty.shooter">
<option name="force_gl_renderer" value="Adreno (TM) 618"/>
<option name="force_gl_vendor" value="Qualcomm"/>
</application>
<application name="Asphalt 8" executable="com.gameloft.android.ANMP.GloftA8HM">
<!-- Note: asphalt8's allowlist seems to not have been updated recently
enough to be aware of "Adreno (TM) 618", but we seem to otherwise
have enough headroom to opt into the higher settings. (Conversely
the allowlist seems to allow G72-MP3 into the higher tier, despite
not being able to maintain fps.. possibly it is not properly
differentiating between various core MPn configurations. So I'm
going to use that to justify claiming to be a618's *slightly*
bigger brother)
-->
<option name="force_gl_renderer" value="Adreno (TM) 630"/>
<option name="force_gl_vendor" value="Qualcomm"/>
</application>
<application name="Asphalt 9" executable="com.gameloft.android.ANMP.GloftA9HM">
<option name="force_gl_renderer" value="Adreno (TM) 618"/>
<option name="force_gl_vendor" value="Qualcomm"/>
</application>
<application name="PUBG Mobile" executable="com.tencent.ig">
<!-- Note: Maxed out a630 settings still limits max framerate, yet GPU
is below 45% utilization (mostly closer to 40%). If I claim to
be a640, that unlocks "Extreme" framerate (and "Ultra HD" graphics).
With "Extreme" + "HDR" we are comfortably at 60fps with plenty of
headroom. "Extreme" + "Ultra HD" (max settings for a640) we are
starting to approach the limit. I'll stick with a630 to be
conservative, but argueably we could go higher. (This flagship
gaming phone thing is a scam!)
-->
<option name="force_gl_renderer" value="Adreno (TM) 630"/>
<option name="force_gl_vendor" value="Qualcomm"/>
</application>
</device>
<device driver="msm">
<!--
ANGLE is catastrophically broken, see:
https://issues.angleproject.org/u/1/issues/431097618
This broken codepath is exposed when skia detects vendor
freedreno (as well as closed mali/img/adreno drivers.. maybe
the issue will actually get fixed when a new enough skia+
ANGLE hits the closed drivers on android).
Workaround the issue by spoofing the vendor string so
skia doesn't pick the broken ANGLE codepath. Other
electron apps will have the same problem, and should be
listed here as well.
-->
<application name="Chromium" executable="chromium">
<option name="force_gl_vendor" value="angleisbroken"/>
</application>
<application name="Chromium" executable="chromium-browser">
<option name="force_gl_vendor" value="angleisbroken"/>
</application>
<application name="Chromium" executable="chrome">
<option name="force_gl_vendor" value="angleisbroken"/>
</application>
<application name="VS Code" executable="code">
<option name="force_gl_vendor" value="angleisbroken"/>
</application>
<application name="element-desktop" executable="element-desktop">
<option name="force_gl_vendor" value="angleisbroken"/>
</application>
<application name="vivaldi-bin" executable="vivaldi-bin">
<option name="force_gl_vendor" value="angleisbroken"/>
</application>
<application name="code" executable="vivaldi-bin">
<option name="force_gl_vendor" value="angleisbroken"/>
</application>
<!--
Xwayland mixes implicit and explicit sync, depending on the
client app. This trips up the heuristic that switches to
explicit sync once it sees that the "app" is using explicit
sync, causing flickering for implicit sync clients
-->
<application name="Xwayland" executable="Xwayland">
<option name="disable_explicit_sync_heuristic" value="true"/>
</application>
</device>
</driconf>
src/util/00-nvk-defaults.conf
+21
View File
@@ -0,0 +1,21 @@
<?xml version="1.0"?>
<driconf>
<device driver="nvk">
<application name="Artifact Classic" executable="dcg">
<option name="force_vk_vendor" value="-1" />
</application>
<application name="Deep Rock Galactic" executable="FSD.exe">
<!-- Tries to use NvAPI_D3D11_CreateCubinComputeShaderWithName otherwise -->
<option name="force_vk_vendor" value="-1" />
</application>
<application name="Metro Exodus (Linux native)" application_name_match="metroexodus">
<option name="nvk_app_layer" value="metroexodus" />
</application>
<application name="No Man's Sky" executable="NMS.exe">
<option name="force_vk_vendor" value="-1" />
</application>
<application name="X4 Foundations" executable="X4">
<option name="force_vk_vendor" value="-1" />
</application>
</device>
</driconf>
src/util/00-panfrost-defaults.conf
+8
View File
@@ -0,0 +1,8 @@
<?xml version="1.0"?>
<driconf>
<device driver="panfrost">
<application name="all-default">
<option name="allow_rgb16_configs" value="false"/>
</application>
</device>
</driconf>
src/util/00-panvk-defaults.conf
+14
View File
@@ -0,0 +1,14 @@
<?xml version="1.0"?>
<driconf>
<device driver="panvk">
<engine engine_name_match="ANGLE">
<!--
ANGLE requires VK_EXT_transform_feedback or vertexPipelineStoresAndAtomics
to enable OpenGL 3.1 support. As we currently don't support this extension,
we enable support for vertexPipelineStoresAndAtomics to allow XFB emulation
on hardware without speculative behaviors around vertices.
-->
<option name="pan_enable_vertex_pipeline_stores_atomics" value="true" />
</engine>
</device>
</driconf>
src/util/00-r300-defaults.conf
+44
View File
@@ -0,0 +1,44 @@
<?xml version="1.0"?>
<driconf>
<device driver="r300">
<!-- Only one app can use Hyperz at a time. -->
<application name="X server" executable="X">
<option name="r300_nohiz" value="true" />
<option name="r300_nozmask" value="true" />
</application>
<application name="X server" executable="Xorg">
<option name="r300_nohiz" value="true" />
<option name="r300_nozmask" value="true" />
</application>
<application name="Compiz" executable="check_gl_texture_size">
<option name="r300_nohiz" value="true" />
<option name="r300_nozmask" value="true" />
</application>
<application name="Compiz" executable="Compiz">
<option name="r300_nohiz" value="true" />
<option name="r300_nozmask" value="true" />
</application>
<application name="GNOME Shell" executable="gnome-session-check-accelerated-helper">
<option name="r300_nohiz" value="true" />
<option name="r300_nozmask" value="true" />
</application>
<application name="GNOME Shell" executable="gnome-shell">
<option name="r300_nohiz" value="true" />
<option name="r300_nozmask" value="true" />
</application>
<application name="KWin" executable="kwin_opengl_test">
<option name="r300_nohiz" value="true" />
<option name="r300_nozmask" value="true" />
</application>
<application name="KWin" executable="kwin">
<option name="r300_nohiz" value="true" />
<option name="r300_nozmask" value="true" />
</application>
<application name="Unigine Tropics" executable="Tropics">
<option name="r300_ffmath" value="true" />
</application>
<application name="Oilrush" executable="Oilrush_x86">
<option name="r300_ffmath" value="true" />
</application>
</device>
</driconf>
src/util/00-r600-defaults.conf
+8
View File
@@ -0,0 +1,8 @@
<?xml version="1.0"?>
<driconf>
<device driver="r600">
<application name="Tomb Raider 2013" executable="TombRaider">
<option name="mesa_extension_override" value="-GL_ARB_shader_image_load_store" />
</application>
</device>
</driconf>
src/util/00-radeonsi-defaults.conf
+85
View File
@@ -0,0 +1,85 @@
<?xml version="1.0"?>
<driconf>
<device driver="radeonsi">
<application name="Alien Isolation" executable="AlienIsolation">
<option name="force_gl_vendor" value="X.Org" />
</application>
<application name="American Truck Simulator" executable="amtrucks">
<option name="radeonsi_zerovram" value="true" />
</application>
<application name="Counter-Strike Global Offensive" executable="csgo_linux64">
<option name="radeonsi_zerovram" value="true" />
<option name="radeonsi_clamp_div_by_zero" value="true" />
</application>
<application name="Exanima" executable="Exanima.exe">
<option name="radeonsi_zerovram" value="true"/>
</application>
<application name="Nowhere Patrol" executable="NowherePatrol.exe">
<option name="radeonsi_zerovram" value="true" />
</application>
<application name="Rocket League" executable="RocketLeague">
<option name="radeonsi_zerovram" value="true" />
</application>
<!-- This applies to all apps having "runner" as their executable, but that's OK.
Apps where this workaround is needed:
- Monolith demo5
- Memoranda
- Nuclear Throne
- Peace, Death!
- Riptale
- Turmoil
- Undertale
-->
<!-- YoYo game engine workarounds -->
<application name="any runner executable" executable="runner">
<option name="radeonsi_sync_compile" value="true" />
<option name="radeonsi_zerovram" value="true" />
</application>
<application name="Gun Godz" executable="GUNGODZ">
<option name="radeonsi_sync_compile" value="true" />
</application>
<!-- https://gitlab.freedesktop.org/mesa/mesa/-/issues/2647 -->
<application name="Kerbal Space Program" executable="KSP.x86_64">
<option name="radeonsi_no_infinite_interp" value="true" />
</application>
<!-- https://gitlab.freedesktop.org/mesa/mesa/-/issues/2686 -->
<application name="7 Days to Die" executable="7DaysToDie.x86_64">
<option name="radeonsi_zerovram" value="true" />
</application>
<application name="7 Days to Die" executable="7DaysToDie.x86">
<option name="radeonsi_zerovram" value="true" />
</application>
<application name="SPECviewperf13" executable="viewperf">
<option name="radeonsi_sync_compile" value="true" />
</application>
<application name="Road Redemption" executable="RoadRedemption.x86_64">
<option name="radeonsi_clamp_div_by_zero" value="true" />
</application>
<application name="Wasteland 2" executable="WL2">
<option name="radeonsi_clamp_div_by_zero" value="true" />
</application>
<application name="Teardown" executable="teardown.exe">
<!-- https://gitlab.freedesktop.org/mesa/mesa/-/issues/3714 -->
<option name="radeonsi_clamp_div_by_zero" value="true" />
<option name="radeonsi_no_infinite_interp" value="true" />
</application>
<application name="SpaceEngine" executable="SpaceEngine.exe">
<option name="force_gl_vendor" value="X.Org" />
</application>
<application name="BETA CAE Systems - GL detect tool" executable="detect_opengl_tool">
<option name="force_gl_vendor" value="ATI Technologies Inc." />
</application>
<application name="BETA CAE Systems - ANSA" executable="ansa_linux_x86_64">
<option name="force_gl_vendor" value="ATI Technologies Inc." />
</application>
<application name="BETA CAE Systems - META" executable="meta_post_x86_64">
<option name="force_gl_vendor" value="ATI Technologies Inc." />
<option name="radeonsi_force_use_fma32" value="true" />
</application>
<application name="Black Geyser: Couriers of Darkness" executable="BlackGeyser.x86_64">
<option name="radeonsi_zerovram" value="true" />
</application>
</device>
</driconf>
src/util/00-radv-defaults.conf
+327
View File
@@ -0,0 +1,327 @@
<?xml version="1.0"?>
<!--
============================================
Application bugs worked around in this file:
============================================
-->
<!-- NOTE: `executable` shouldn't be used for shader based drirc workarounds
because they aren't applied when replaying with Fossilize for precompilation. -->
<driconf>
<device driver="radv">
<!-- Engine workarounds -->
<engine engine_name_match="vkd3d">
<option name="vk_zero_vram" value="true" />
<option name="radv_disable_aniso_single_level" value="true" />
<option name="radv_disable_trunc_coord" value="true" />
<option name="radv_cooperative_matrix2_nv" value="true" />
</engine>
<engine engine_name_match="DXVK">
<option name="radv_disable_aniso_single_level" value="true" />
<option name="radv_disable_trunc_coord" value="true" />
</engine>
<engine engine_name_match="mesa zink">
<option name="radv_disable_aniso_single_level" value="true" />
</engine>
<engine engine_name_match="Quantic Dream Engine">
<option name="vk_zero_vram" value="true" />
<option name="radv_disable_tc_compat_htile_general" value="true" />
<option name="radv_app_layer" value="quanticdream" />
</engine>
<engine engine_name_match="^UnrealEngine"> <!-- UE4/5 Vulkan RHI -->
<option name="radv_flush_before_query_copy" value="true" />
<option name="radv_flush_before_timestamp_write" value="true" />
<option name="vk_zero_vram" value="true" />
</engine>
<engine engine_name_match="DXVK_NvRemix">
<option name="radv_tex_non_uniform" value="true" />
</engine>
<!-- VK_MAKE_VERSION() encode for 4.0.0 to 4.20.2 -->
<engine engine_name_match="GTK" engine_versions="16777216:16859138">
<option name="vk_wsi_disable_unordered_submits" value="true" />
</engine>
<!-- Game workarounds -->
<application name="Shadow Of The Tomb Raider (Native)" application_name_match="ShadowOfTheTomb">
<option name="radv_report_llvm9_version_string" value="true" />
<option name="radv_invariant_geom" value="true" />
<!-- The full WA hurts performance too much, use the partial one
that seems fine overall. -->
<option name="radv_gfx12_hiz_wa" value="partial" />
</application>
<application name="Shadow Of The Tomb Raider (DX11/DX12)" application_name_match="SOTTR.exe">
<!-- DXVK and vkd3d-proton both enable invariant position but it's
disabled for SotTR because it breaks NV more than it helps -->
<option name="radv_invariant_geom" value="true" />
<option name="radv_split_fma" value="true" />
</application>
<application name="RAGE 2" application_name_match="Rage 2">
<option name="radv_enable_mrt_output_nan_fixup" value="true" />
<option name="radv_app_layer" value="rage2" />
<option name="vk_zero_vram" value="true" />
</application>
<application name="The Surge 2" application_name_match="Fledge">
<option name="radv_disable_shrink_image_store" value="true" />
<option name="vk_zero_vram" value="true" />
</application>
<application name="World War Z (and World War Z: Aftermath)" application_name_match="WWZ|wwz">
<option name="radv_override_uniform_offset_alignment" value="16" />
<option name="radv_disable_shrink_image_store" value="true" />
<option name="radv_invariant_geom" value="true" />
</application>
<application name="DOOM VFR" application_name_match="DOOM_VFR">
<option name="radv_no_dynamic_bounds" value="true" />
</application>
<application name="DOOM Eternal" application_name_match="DOOMEternal">
<option name="vk_zero_vram" value="true" />
</application>
<application name="Indiana Jones: The Great Circle" application_name_match="TheGreatCircle">
<option name="vk_zero_vram" value="true" />
<option name="radv_disable_dcc_stores" value="true" />
<option name="radv_invariant_geom" value="true" />
<option name="radv_emulate_rt" value="true" />
<option name="radv_enable_float16_gfx8" value="true" />
<!-- The full WA hurts performance too much, use the partial one
that seems fine overall. -->
<option name="radv_gfx12_hiz_wa" value="partial" />
</application>
<application name="DOOM (2016)" application_name_match="DOOM$">
<option name="radv_disable_dcc" value="true" />
<option name="radv_invariant_geom" value="true" />
</application>
<application name="DOOM: The Dark Ages" application_name_match="DOOMTheDarkAges">
<option name="vk_zero_vram" value="true" />
<option name="radv_disable_dcc_stores" value="true" />
<option name="radv_no_implicit_varying_subgroup_size" value="true" />
</application>
<application name="Wolfenstein II" application_name_match="Wolfenstein II The New Colossus">
<option name="radv_disable_dcc" value="true" />
</application>
<application name="Grid Autosport" application_name_match="GRIDAutosport_avx.exe">
<option name="radv_disable_sinking_load_input_fs" value="true" />
</application>
<application name="Grid Autosport" application_name_match="GRIDAutosport.exe">
<option name="radv_disable_sinking_load_input_fs" value="true" />
</application>
<application name="Hammerting" application_name_match="TDTD">
<option name="vk_zero_vram" value="true" />
</application>
<application name="RDR2" application_name_match="Red Dead Redemption 2">
<option name="radv_enable_unified_heap_on_apu" value="true" />
<!-- Hide resizable (only when detected) and report a carveout of
256MiB to workaround bad VRAM memory management. -->
<option name="radv_hide_rebar_on_dgpu" value="true" />
<option name="vk_zero_vram" value="true" />
<!-- Disable DCC for mips to workaround rendering issues because
the game aliases two images incorrectly. -->
<option name="radv_disable_dcc_mips" value="true" />
<!-- The full WA hurts performance too much, use the partial one
that seems fine overall. -->
<option name="radv_gfx12_hiz_wa" value="partial" />
</application>
<application name="Metro Exodus (Linux native)" application_name_match="metroexodus">
<option name="radv_app_layer" value="metroexodus" />
</application>
<application name="Hellblade: Senua's Sacrifice" application_name_match="HellbladeGame-Win64-Shipping.exe">
<!-- RT shaders in Hellblade: Senua's Sacrifice cause misrenders because they assume full subgroups.
This is only guaranteed with wave64, so force wave64 on for this game. -->
<option name="radv_rt_wave64" value="true" />
</application>
<application name="War Thunder" application_name_match="WarThunder">
<option name="radv_invariant_geom" value="true"/>
</application>
<!-- Crysis 2/3 remastered are DX11 games that use Vulkan interop for RT
and the app/engine (nvpro-sample) seems too generic to apply the
drirc entry for. The executable name works just fine but that means
no pre-compilation for the RT shaders. -->
<application name="Crysis 2 Remastered" executable="Crysis2Remastered.exe">
<option name="radv_ssbo_non_uniform" value="true" />
</application>
<application name="Crysis 3 Remastered" executable="Crysis3Remastered.exe">
<option name="radv_ssbo_non_uniform" value="true" />
</application>
<application name="Persona 3 Reload" application_name_match="P3R.exe">
<option name="radv_disable_ngg_gs" value="true" />
</application>
<application name="WWE 2k23" application_name_match="WWE2K23_x64.exe">
<option name="radv_disable_dcc" value="true" />
</application>
<application name="GPUScore: Breaking Limit" application_name_match="GPUScoreVulkan">
<option name="vk_zero_vram" value="true" />
</application>
<application name="DiRT4" application_name_match="dirt4.exe">
<!-- The full WA hurts performance too much, use the partial one
that seems fine overall. -->
<option name="radv_gfx12_hiz_wa" value="partial" />
</application>
<application name="Strange Brigade (DX12)" application_name_match="StrangeBrigade_DX12.exe">
<!-- The full WA hurts performance too much, use the partial one
that seems fine overall. -->
<option name="radv_gfx12_hiz_wa" value="partial" />
</application>
<application name="Strange Brigade (VK)" application_name_match="Strange">
<!-- The full WA hurts performance too much, use the partial one
that seems fine overall. -->
<option name="radv_gfx12_hiz_wa" value="partial" />
<!-- To fix a hazard with present after write. -->
<option name="radv_app_layer" value="strange_brigade" />
</application>
<application name="Sniper Elite 5 (DX12)" application_name_match="sniper5_dx12.exe">
<!-- The full WA hurts performance too much, use the partial one
that seems fine overall. -->
<option name="radv_gfx12_hiz_wa" value="partial" />
</application>
<application name="Sniper Elite 5 (VK)" application_name_match="Sniper5">
<!-- The full WA hurts performance too much, use the partial one
that seems fine overall. -->
<option name="radv_gfx12_hiz_wa" value="partial" />
</application>
<application name="Counter Strike 2" application_name_match="csgo">
<!-- The full WA hurts performance too much, use the partial one
that seems fine overall. -->
<option name="radv_gfx12_hiz_wa" value="partial" />
</application>
<application name="Forza Horizon 5" application_name_match="ForzaHorizon5.exe">
<!-- The full WA hurts performance too much, use the partial one
that seems fine overall. -->
<option name="radv_gfx12_hiz_wa" value="partial" />
<!-- Force waiting for VM MAP updates at allocation time to
workaround an use-before-alloc which causes GPU hangs. -->
<option name="radv_wait_for_vm_map_updates" value="true" />
</application>
<application name="Forza Horizon 6" application_name_match="forzahorizon6.exe">
<!-- Force waiting for VM MAP updates at allocation time to
workaround an use-before-alloc which causes GPU hangs. -->
<option name="radv_wait_for_vm_map_updates" value="true" />
<!-- Using 64 byte descriptors lets vkd3d-proton work around
descriptor type mismatches.
TODO Remove this when VK_EXT_descriptor_heap is enabled by
default because it should not be necessary anymore. -->
<option name="radv_force_64_byte_sampled_image" value="true" />
</application>
<application name="Far Cry 2" application_name_match="FarCry2.exe">
<!-- The full WA hurts performance too much, use the partial one
that seems fine overall. -->
<option name="radv_gfx12_hiz_wa" value="partial" />
</application>
<application name="DiRT Rally 2.0" application_name_match="dirtrally2.exe">
<!-- The full WA hurts performance too much, use the partial one
that seems fine overall. -->
<option name="radv_gfx12_hiz_wa" value="partial" />
</application>
<application name="Valheim" application_name_match="Valheim">
<!-- The full WA hurts performance too much, use the partial one
that seems fine overall. -->
<option name="radv_gfx12_hiz_wa" value="partial" />
</application>
<application name="The Last of Us Part I" application_name_match="tlou-i.exe">
<!-- TLOU1 is much faster with 2D swizzle modes for 3D storage images. -->
<option name="radv_prefer_2d_swizzle_for_3d_storage" value="true" />
</application>
<!-- OpenGL Game workarounds (zink) -->
<application name="Black Geyser: Couriers of Darkness" executable="BlackGeyser.x86_64">
<option name="vk_zero_vram" value="true" />
</application>
<!-- https://gitlab.freedesktop.org/mesa/mesa/-/issues/2686 -->
<application name="7 Days to Die" executable="7DaysToDie.x86_64">
<option name="vk_zero_vram" value="true" />
</application>
<application name="7 Days to Die" executable="7DaysToDie.x86">
<option name="vk_zero_vram" value="true" />
</application>
<!-- This applies to all apps having "runner" as their executable, but that's OK.
Apps where this workaround is needed:
- Monolith demo5
-->
<application name="any runner executable" executable="runner">
<option name="vk_zero_vram" value="true" />
</application>
<application name="American Truck Simulator" executable="amtrucks">
<option name="vk_zero_vram" value="true" />
</application>
<application name="Counter-Strike Global Offensive" executable="csgo_linux64">
<option name="vk_zero_vram" value="true" />
</application>
<application name="Exanima" executable="Exanima.exe">
<option name="vk_zero_vram" value="true"/>
</application>
<application name="Rocket League" executable="RocketLeague">
<option name="vk_zero_vram" value="true" />
</application>
<application name="Crystal Project" executable="Crystal Project.bin.x86_64">
<option name="vk_zero_vram" value="true" />
</application>
<application name="Enshrouded" executable="enshrouded.exe">
<option name="vk_zero_vram" value="true"/>
</application>
<application name="No Man's Sky" application_name_match="No Man's Sky">
<option name="radv_app_layer" value="no_mans_sky" />
</application>
<application name="Crimson Desert" application_name_match="CrimsonDesert.exe">
<!-- Using 64 byte descriptors lets vkd3d-proton work around
descriptor type mismatches.
TODO Remove this when VK_EXT_descriptor_heap is enabled by
default because it should not be necessary anymore. -->
<option name="radv_force_64_byte_sampled_image" value="true" />
</application>
</device>
</driconf>
src/util/00-turnip-defaults.conf
+87
View File
@@ -0,0 +1,87 @@
<?xml version="1.0"?>
<driconf>
<device driver="turnip">
<!-- Up to DXVK version 2.3.0 -->
<engine engine_name_match="DXVK" engine_versions="0:8400896">
<!--
See: https://github.com/doitsujin/dxvk/issues/3861
The shader-defined array size is entirely meaningless in D3D.
-->
<option name="tu_allow_oob_indirect_ubo_loads" value="true" />
</engine>
<engine engine_name_match="DXVK|vkd3d">
<!--
DXVK and VKD3D-Proton use customBorderColorWithoutFormat and have most of
D24S8 images with USAGE_SAMPLED, but never sample it with border color.
-->
<option name="tu_enable_fast_border_color_for_undefined_formats" value="true" />
<!--
For sampling, Vulkan requires texture coordinates to be truncated, whereas
D3D requires them to be rounded to nearest even value. The former is used by
default, the latter is used through this option.
-->
<option name="tu_use_tex_coord_round_nearest_even_mode" value="true" />
<!--
The hardware algorithm for alpha-to-coverage seems to
consistently produce bad results compared to other vendors.
Always enabling emulation would potentially regress performance
when it is dynamically enabled and off, but both both DXVK and
vkd3d-proton are known to compile pipelines with
alpha-to-coverage statically disabled.
-->
<option name="tu_emulate_alpha_to_coverage" value="true" />
<!--
DX games almost always tend to prefer SYSMEM over GMEM, so we just hardcode
this for DXVK and VKD3D-Proton.
-->
<option name="tu_autotune_algorithm" value="prefer_sysmem" />
</engine>
<engine engine_name_match="vkd3d">
<option name="tu_enable_softfloat32" value="true" />
</engine>
<engine engine_name_match="ANGLE">
<!--
ANGLE uses customBorderColorWithoutFormat on D24S8 and doesn't check
depthStencilSwizzleOneSupport.
-->
<option name="tu_enable_d24s8_border_color_workaround" value="true" />
</engine>
<application name="Sons Of The Forest" executable="SonsOfTheForest.exe">
<option name="tu_ignore_frag_depth_direction" value="true" />
</application>
<application name="Yooka-Laylee" application_name_match="YookaLaylee.x86_64">
<!--
A vertex shader performs OOB UBO reads causing vertex corruption.
Disable UBO to const lowering to force bounds checking which
fixes the corruption.
-->
<option name="tu_allow_oob_indirect_ubo_loads" value="true" />
</application>
<application name="Creed: Rise to Glory" executable="Creed-Win64-Shipping.exe">
<option name="tu_ignore_frag_depth_direction" value="true" />
</application>
<application name="Half-Life: Alyx" application_name_match="hlvr">
<option name="tu_emulate_alpha_to_coverage" value="true" />
</application>
<application name='No Man&apos;s Sky' application_name_match='No Man&apos;s Sky'>
<!-- A lighting CS reducing 8x8 regions with fmin/fmax does a write once per
subgroup (assuming 64), or collects the results from two subgroups in the
32 subgroup size case. Thus, our 128 subgroup size results in vertical banding
when half the 8x8 regions don't get written.
-->
<option name="tu_restrict_subgroup_size_64" value="true" />
</application>
<application name="Forsaken Remastered" executable="ForsakenEx">
<!-- https://gitlab.freedesktop.org/mesa/mesa/-/issues/5437 -->
<option name="tu_dont_care_as_load" value="true" />
</application>
<application name="Spilled!" executable="Spilled!.exe">
<!-- Required for Turnip's GMEM mode running under Unity's Vulkan backend. -->
<option name="tu_dont_care_as_load" value="true" />
</application>
</device>
</driconf>
src/util/00-v3d-defaults.conf
+14
View File
@@ -0,0 +1,14 @@
<?xml version="1.0"?>
<driconf>
<device driver="v3d">
<application name="gnome-shell" executable="gnome-shell">
<option name="v3d_nonmsaa_texture_size_limit" value="true" />
</application>
<application name="mutter" executable="mutter">
<option name="v3d_nonmsaa_texture_size_limit" value="true" />
</application>
<application name="Xorg" executable="Xorg">
<option name="v3d_nonmsaa_texture_size_limit" value="true" />
</application>
</device>
</driconf>
src/util/00-virtio_gpu-defaults.conf
+8
View File
@@ -0,0 +1,8 @@
<?xml version="1.0"?>
<driconf>
<device driver="virtio_gpu">
<application name="Counter-Strike Global Offensive" executable="csgo_linux64">
<option name="format_l8_srgb_enable_readback" value="true" />
</application>
</device>
</driconf>
src/util/00-vmwgfx-defaults.conf
+15
View File
@@ -0,0 +1,15 @@
<?xml version="1.0"?>
<driconf>
<!-- vmwgfx doesn't like full buffer swaps and can't sync to vertical retraces.-->
<device driver="vmwgfx">
<application name="gnome-shell" executable="gnome-shell">
<option name="glx_extension_override" value="-GLX_OML_sync_control -GLX_SGI_video_sync" />
</application>
<application name="cinnamon" executable="cinnamon">
<option name="glx_extension_override" value="-GLX_OML_sync_control -GLX_SGI_video_sync" />
</application>
<application name="Compiz" executable="Compiz">
<option name="glx_extension_override" value="-GLX_OML_sync_control" />
</application>
</device>
</driconf>
src/util/00-zink-defaults.conf
+31
View File
@@ -0,0 +1,31 @@
<?xml version="1.0"?>
<driconf>
<device driver="zink">
<application name="Hyperdimension Neptunia Re;Birth1" executable="NeptuniaReBirth1.exe">
<!-- glthread uploads need too much vram and exceed 32bit VA limit -->
<!-- https://gitlab.freedesktop.org/mesa/mesa/-/issues/8333 -->
<option name="mesa_glthread_app_profile" value="0"/>
</application>
<application name="Borderlands 2 (Native, OpenGL, 32bit)" executable="Borderlands2">
<!-- https://gitlab.freedesktop.org/mesa/mesa/-/issues/9205 -->
<option name="glsl_correct_derivatives_after_discard" value="true" />
</application>
<application name="SPECviewperf13" executable="viewperf">
<option name="mesa_extension_override" value="+GL_EXT_shader_image_load_store" />
</application>
<!-- GL_POINT_SMOOTH is used in Quake II to render particles -->
<application name="Quake II" executable="quake2-engine">
<option name="zink_emulate_point_smooth" value="true"/>
</application>
<application name="Quake II (yamagi)" executable="yamagi-quake2">
<option name="zink_emulate_point_smooth" value="true"/>
</application>
<application name="Quake II (wine)" executable="quake2.exe">
<option name="zink_emulate_point_smooth" value="true"/>
</application>
<application name="Tomb Raider 2013" executable="TombRaider">
<option name="zink_shader_object_enable" value="true" />
</application>
</device>
</driconf>
src/util/anon_file.c
+264
View File
@@ -0,0 +1,264 @@
/*
* Copyright © 2012 Collabora, Ltd.
* Copyright (C) 2025 Arm Ltd.
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice (including the
* next paragraph) shall be included in all copies or substantial
* portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
/*
* Based on weston shared/os-compatibility.c
*/
#include "anon_file.h"
#include "detect_os.h"
#ifndef _WIN32
#include <unistd.h>
#include <fcntl.h>
#include <errno.h>
#include <stdlib.h>
#if defined(HAVE_MEMFD_CREATE) || defined(__FreeBSD__) || defined(__OpenBSD__)
#include <sys/mman.h>
#elif DETECT_OS_ANDROID
#include <sys/syscall.h>
#include <linux/memfd.h>
#endif
#if !(defined(__FreeBSD__) || DETECT_OS_ANDROID)
#include "log.h"
#include "os_misc.h"
#include <sys/stat.h>
#include <stdio.h>
#endif
#if !(defined(__FreeBSD__) || defined(HAVE_MKOSTEMP) || DETECT_OS_ANDROID)
static int
set_cloexec_or_close(int fd)
{
long flags;
if (fd == -1)
return -1;
flags = fcntl(fd, F_GETFD);
if (flags == -1)
goto err;
if (fcntl(fd, F_SETFD, flags | FD_CLOEXEC) == -1)
goto err;
return fd;
err:
close(fd);
return -1;
}
#endif
#if !(defined(__FreeBSD__) || DETECT_OS_ANDROID)
static int
create_tmpfile_cloexec(char *tmpname)
{
int fd;
#ifdef HAVE_MKOSTEMP
fd = mkostemp(tmpname, O_CLOEXEC);
#else
fd = mkstemp(tmpname);
#endif
if (fd < 0) {
return fd;
}
#ifndef HAVE_MKOSTEMP
fd = set_cloexec_or_close(fd);
#endif
unlink(tmpname);
return fd;
}
#endif
#if !(defined(__FreeBSD__) || DETECT_OS_ANDROID)
/*
* Gets the path to a suitable temporary directory for the current user.
* Prefers using the environment variable `XDG_RUNTIME_DIR` if set,
* otherwise falls back to creating or re-using a folder in `/tmp`.
* Returns a pointer to the path buffer, which the caller must free().
* Returns NULL if no suitable directory can found or created.
*/
static char*
get_or_create_user_temp_dir(void) {
const char* env;
char* buf;
struct stat st;
int uid = getuid();
int n;
env = os_get_option("XDG_RUNTIME_DIR");
if (env && env[0] != '\0') {
n = asprintf(&buf, "%s", env);
if (n < 0)
return NULL;
return buf;
}
n = asprintf(&buf, "/tmp/xdg-runtime-mesa-%ld", (long)getuid());
if (n < 0)
return NULL;
mesa_logd("%s: XDG_RUNTIME_DIR not set; falling back to temp dir %s",
__func__, buf);
if (stat(buf, &st) == 0) {
/* If already exists, confirm the owner/permissions */
if (!S_ISDIR(st.st_mode)) {
mesa_loge(
"%s: %s exists but is not a directory", __func__, buf);
free(buf);
return NULL;
}
if (st.st_uid != uid) {
mesa_loge(
"%s: %s exists but has wrong owner", __func__, buf);
free(buf);
return NULL;
}
return buf;
} else if (errno == ENOENT) {
/* Doesn't exist, try to create it */
if (mkdir(buf, 0700) != 0) {
mesa_loge("%s: mkdir %s failed: %s", __func__, buf,
strerror(errno));
free(buf);
return NULL;
}
return buf;
} else {
mesa_loge("%s: stat %s failed: %s", __func__, buf,
strerror(errno));
free(buf);
return NULL;
}
}
#endif
/*
* Create a new, unique, anonymous file of the given size, and
* return the file descriptor for it. The file descriptor is set
* CLOEXEC. The file is immediately suitable for mmap()'ing
* the given size at offset zero.
*
* An optional name for debugging can be provided as the second argument.
*
* The file should not have a permanent backing store like a disk,
* but may have if XDG_RUNTIME_DIR is not properly implemented in OS.
*
* If memfd or SHM_ANON is supported, the filesystem is not touched at all.
* Otherwise, the file name is deleted from the file system.
*
* The file is suitable for buffer sharing between processes by
* transmitting the file descriptor over Unix sockets using the
* SCM_RIGHTS methods.
*/
int
os_create_anonymous_file(int64_t size, const char *debug_name)
{
int fd = -1, ret;
/* First try using preferred APIs */
#if defined(HAVE_MEMFD_CREATE)
if (!debug_name)
debug_name = "mesa-shared";
fd = memfd_create(debug_name, MFD_CLOEXEC | MFD_ALLOW_SEALING);
#elif DETECT_OS_ANDROID
if (!debug_name)
debug_name = "mesa-shared";
fd = syscall(SYS_memfd_create, debug_name, MFD_CLOEXEC | MFD_ALLOW_SEALING);
#elif defined(__FreeBSD__)
fd = shm_open(SHM_ANON, O_CREAT | O_RDWR | O_CLOEXEC, 0600);
#elif defined(__OpenBSD__)
char template[] = "/tmp/mesa-XXXXXXXXXX";
fd = shm_mkstemp(template);
if (fd != -1)
shm_unlink(template);
#endif
/*
* If preferred API failed (or not included in this build),
* fall back to using a file in a temporary dir
*/
#if !(defined(__FreeBSD__) || DETECT_OS_ANDROID)
if (fd == -1) {
char *path;
char *name;
path = get_or_create_user_temp_dir();
if (!path) {
errno = ENOENT;
return -1;
}
int k;
if (debug_name)
k = asprintf(&name, "%s/mesa-shared-%s-XXXXXX", path, debug_name);
else
k = asprintf(&name, "%s/mesa-shared-XXXXXX", path);
if (k < 0 || !name) {
free(path);
return -1;
}
fd = create_tmpfile_cloexec(name);
free(path);
free(name);
}
#endif
if (fd < 0)
return -1;
ret = ftruncate(fd, (off_t)size);
if (ret < 0) {
close(fd);
return -1;
}
return fd;
}
#else
#include <windows.h>
#include <io.h>
int
os_create_anonymous_file(int64_t size, const char *debug_name)
{
(void)debug_name;
HANDLE h = CreateFileMappingW(INVALID_HANDLE_VALUE, NULL,
PAGE_READWRITE, (size >> 32), size & 0xFFFFFFFF, NULL);
return _open_osfhandle((intptr_t)h, 0);
}
#endif
src/util/anon_file.h
+42
View File
@@ -0,0 +1,42 @@
/*
* Copyright © 2012 Collabora, Ltd.
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice (including the
* next paragraph) shall be included in all copies or substantial
* portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#ifndef _ANON_FILE_H_
#define _ANON_FILE_H_
#include <stdint.h>
#ifdef __cplusplus
extern "C"{
#endif
/* On win32, off_t is only 32 bit, so always using 64 bit size */
int os_create_anonymous_file(int64_t size, const char *debug_name);
#ifdef __cplusplus
}
#endif
#endif
src/util/bfloat.h
+69
View File
@@ -0,0 +1,69 @@
/*
* Copyright © 2025 Intel Corporation
* SPDX-License-Identifier: MIT
*/
#pragma once
#include <assert.h>
#include <math.h>
#include <stdint.h>
#include "u_math.h"
/* When converting a Float NaN value to BFloat16 it is possible that the
* significand bits that make the value a NaN will be rounded/truncated off
* so ensure at least one significand bit is set.
*/
static inline uint16_t
_mesa_float_nan_to_bfloat_bits(union fi x)
{
assert(isnan(x.f));
return x.ui >> 16 | 1 << 6;
}
/* Round-towards-zero. */
static inline uint16_t
_mesa_float_to_bfloat16_bits_rtz(float f)
{
union fi x;
x.f = f;
if (isnan(f))
return _mesa_float_nan_to_bfloat_bits(x);
return x.ui >> 16;
}
/* Round-to-nearest-even. */
static inline uint16_t
_mesa_float_to_bfloat16_bits_rte(float f)
{
union fi x;
x.f = f;
if (isnan(f))
return _mesa_float_nan_to_bfloat_bits(x);
/* Use the tail part that is discarded to decide rounding,
* break the tie with the nearest even.
*
* Overflow of the significand value will turn to zero and
* increment the exponent. If exponent reaches 0xff, the
* value will correctly end up as +/- Inf.
*/
uint32_t result = x.ui >> 16;
const uint32_t tail = x.ui & 0xffff;
if (tail > 0x8000 || (tail == 0x8000 && (result & 1) == 1))
result++;
return result;
}
static inline float
_mesa_bfloat16_bits_to_float(uint16_t bf)
{
union fi x;
x.ui = (uint32_t)bf << 16;
return x.f;
}
src/util/bigmath.h
+112
View File
@@ -0,0 +1,112 @@
/*
* Copyright © 2018 Intel Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
#ifndef UTIL_BIGMATH_H
#define UTIL_BIGMATH_H
#include "macros.h"
#include <assert.h>
#include <stdint.h>
#include <string.h>
static inline bool
_ubm_add_u32arr(uint32_t *dst, unsigned dst_len,
uint32_t *a, unsigned a_len,
uint32_t *b, unsigned b_len)
{
uint32_t carry = 0;
for (unsigned i = 0; i < dst_len; i++) {
uint64_t sum = carry;
if (i < a_len)
sum += a[i];
if (i < b_len)
sum += b[i];
dst[i] = sum;
carry = sum >> 32;
}
/* Now compute overflow */
for (unsigned i = dst_len; i < a_len; i++) {
if (a[i])
return true;
}
for (unsigned i = dst_len; i < b_len; i++) {
if (b[i])
return true;
}
return carry;
}
#define ubm_add_u32arr(dst, a, b) \
_ubm_add_u32arr(dst, ARRAY_SIZE(dst), a, ARRAY_SIZE(a), b, ARRAY_SIZE(b))
static inline bool
_ubm_mul_u32arr(uint32_t *dst, unsigned dst_len,
uint32_t *a, unsigned a_len,
uint32_t *b, unsigned b_len)
{
memset(dst, 0, dst_len * sizeof(*dst));
bool overflow = false;
for (unsigned i = 0; i < a_len; i++) {
uint32_t carry = 0;
for (unsigned j = 0; j < b_len; j++) {
/* The maximum values of a[i] and b[i] are UINT32_MAX so the maximum
* value of tmp is UINT32_MAX * UINT32_MAX. The maximum value that
* will fit in tmp is
*
* UINT64_MAX = UINT32_MAX << 32 + UINT32_MAX
* = UINT32_MAX * (UINT32_MAX + 1) + UINT32_MAX
* = UINT32_MAX * UINT32_MAX + 2 * UINT32_MAX
*
* so we're guaranteed that we can add in two more 32-bit values
* without overflowing tmp.
*/
uint64_t tmp = (uint64_t)a[i] * (uint64_t)b[j];
tmp += carry;
if (i + j < dst_len) {
tmp += dst[i + j];
dst[i + j] = tmp;
carry = tmp >> 32;
} else {
/* We're trying to write a value that doesn't fit */
overflow = overflow || tmp > 0;
break;
}
}
if (i + b_len < dst_len)
dst[i + b_len] = carry;
else
overflow = overflow || carry > 0;
}
return overflow;
}
#define ubm_mul_u32arr(dst, a, b) \
_ubm_mul_u32arr(dst, ARRAY_SIZE(dst), a, ARRAY_SIZE(a), b, ARRAY_SIZE(b))
#endif /* UTIL_BIGMATH_H */
src/util/bitpack_helpers.h
+210
View File
@@ -0,0 +1,210 @@
/*
* Copyright (C) 2016 Intel Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
#ifndef UTIL_BITPACK_HELPERS_H
#define UTIL_BITPACK_HELPERS_H
#include <math.h>
#include <stdbool.h>
#include "util/macros.h"
#include "util/u_math.h"
#ifdef HAVE_VALGRIND
#include <valgrind.h>
#include <memcheck.h>
#ifndef NDEBUG
#define util_bitpack_validate_value(x) \
VALGRIND_CHECK_MEM_IS_DEFINED(&(x), sizeof(x))
#endif
#endif
#ifndef util_bitpack_validate_value
#define util_bitpack_validate_value(x)
#endif
ALWAYS_INLINE static uint64_t
util_bitpack_ones(uint32_t start, uint32_t end)
{
return (UINT64_MAX >> (64 - (end - start + 1))) << start;
}
ALWAYS_INLINE static uint64_t
util_bitpack_uint(uint64_t v, uint32_t start, UNUSED uint32_t end)
{
util_bitpack_validate_value(v);
#ifndef NDEBUG
const int bits = end - start + 1;
if (bits < 64) {
const uint64_t max = u_uintN_max(bits);
assert(v <= max);
}
#endif
return v << start;
}
ALWAYS_INLINE static uint64_t
util_bitpack_uint_nonzero(uint64_t v, uint32_t start, uint32_t end)
{
assert(v != 0ull);
return util_bitpack_uint(v, start, end);
}
ALWAYS_INLINE static uint64_t
util_bitpack_sint(int64_t v, uint32_t start, uint32_t end)
{
const int bits = end - start + 1;
util_bitpack_validate_value(v);
#ifndef NDEBUG
if (bits < 64) {
const int64_t min = u_intN_min(bits);
const int64_t max = u_intN_max(bits);
assert(min <= v && v <= max);
}
#endif
const uint64_t mask = BITFIELD64_MASK(bits);
return (v & mask) << start;
}
ALWAYS_INLINE static uint64_t
util_bitpack_sint_nonzero(int64_t v, uint32_t start, uint32_t end)
{
assert(v != 0ll);
return util_bitpack_sint(v, start, end);
}
ALWAYS_INLINE static uint32_t
util_bitpack_float(float v)
{
util_bitpack_validate_value(v);
union { float f; uint32_t dw; } x;
x.f = v;
return x.dw;
}
ALWAYS_INLINE static uint32_t
util_bitpack_float_nonzero(float v)
{
assert(v != 0.0f);
return util_bitpack_float(v);
}
ALWAYS_INLINE static uint64_t
util_bitpack_sfixed(float v, uint32_t start, uint32_t end,
uint32_t fract_bits)
{
util_bitpack_validate_value(v);
const float factor = (1 << fract_bits);
#ifndef NDEBUG
const int total_bits = end - start + 1;
const float min = u_intN_min(total_bits) / factor;
const float max = u_intN_max(total_bits) / factor;
assert(min <= v && v <= max);
#endif
const int64_t int_val = llroundf(v * factor);
const uint64_t mask = UINT64_MAX >> (64 - (end - start + 1));
return (int_val & mask) << start;
}
ALWAYS_INLINE static uint64_t
util_bitpack_sfixed_clamp(float v, uint32_t start, uint32_t end,
uint32_t fract_bits)
{
util_bitpack_validate_value(v);
const float factor = (1 << fract_bits);
const int total_bits = end - start + 1;
const float min = u_intN_min(total_bits) / factor;
const float max = u_intN_max(total_bits) / factor;
const int64_t int_val = llroundf(CLAMP(v, min, max) * factor);
const uint64_t mask = UINT64_MAX >> (64 - (end - start + 1));
return (int_val & mask) << start;
}
ALWAYS_INLINE static uint64_t
util_bitpack_sfixed_nonzero(float v, uint32_t start, uint32_t end,
uint32_t fract_bits)
{
assert(v != 0.0f);
return util_bitpack_sfixed(v, start, end, fract_bits);
}
ALWAYS_INLINE static uint64_t
util_bitpack_ufixed(float v, uint32_t start, ASSERTED uint32_t end,
uint32_t fract_bits)
{
util_bitpack_validate_value(v);
const float factor = (1 << fract_bits);
#ifndef NDEBUG
const int total_bits = end - start + 1;
const float min = 0.0f;
const float max = u_uintN_max(total_bits) / factor;
assert(min <= v && v <= max);
#endif
const uint64_t uint_val = llroundf(v * factor);
return uint_val << start;
}
ALWAYS_INLINE static uint64_t
util_bitpack_ufixed_clamp(float v, uint32_t start, ASSERTED uint32_t end,
uint32_t fract_bits)
{
util_bitpack_validate_value(v);
const float factor = (1 << fract_bits);
const int total_bits = end - start + 1;
const float min = 0.0f;
const float max = u_uintN_max(total_bits) / factor;
const uint64_t uint_val = llroundf(CLAMP(v, min, max) * factor);
return uint_val << start;
}
ALWAYS_INLINE static uint64_t
util_bitpack_ufixed_nonzero(float v, uint32_t start, uint32_t end,
uint32_t fract_bits)
{
assert(v != 0.0f);
return util_bitpack_ufixed(v, start, end, fract_bits);
}
#endif /* UTIL_BITPACK_HELPERS_H */
src/util/bitscan.c
+80
View File
@@ -0,0 +1,80 @@
/**************************************************************************
*
* Copyright 2008 VMware, Inc.
* All Rights Reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sub license, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice (including the
* next paragraph) shall be included in all copies or substantial portions
* of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
* IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
* ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*
**************************************************************************/
#include "bitscan.h"
#ifdef HAVE___BUILTIN_FFS
#elif defined(_MSC_VER) && (_M_IX86 || _M_ARM || _M_AMD64 || _M_IA64)
#else
int
ffs(int i)
{
int bit = 0;
if (!i)
return bit;
if (!(i & 0xffff)) {
bit += 16;
i >>= 16;
}
if (!(i & 0xff)) {
bit += 8;
i >>= 8;
}
if (!(i & 0xf)) {
bit += 4;
i >>= 4;
}
if (!(i & 0x3)) {
bit += 2;
i >>= 2;
}
if (!(i & 0x1))
bit += 1;
return bit + 1;
}
#endif
#ifdef HAVE___BUILTIN_FFSLL
#elif defined(_MSC_VER) && (_M_AMD64 || _M_ARM64 || _M_IA64)
#else
int
ffsll(long long int val)
{
int bit;
bit = ffs((unsigned) (val & 0xffffffff));
if (bit != 0)
return bit;
bit = ffs((unsigned) (val >> 32));
if (bit != 0)
return 32 + bit;
return 0;
}
#endif
src/util/bitscan.h
+438
View File
@@ -0,0 +1,438 @@
/**************************************************************************
*
* Copyright 2008 VMware, Inc.
* All Rights Reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sub license, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice (including the
* next paragraph) shall be included in all copies or substantial portions
* of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
* IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
* ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*
**************************************************************************/
#ifndef BITSCAN_H
#define BITSCAN_H
#include <assert.h>
#include <stdint.h>
#include <stdbool.h>
#include <string.h>
#if defined(_MSC_VER)
#include <intrin.h>
#endif
#if defined(__POPCNT__)
#include <popcntintrin.h>
#endif
#include "util/detect_arch.h"
#include "util/detect_cc.h"
#include "util/macros.h"
#ifdef __cplusplus
extern "C" {
#endif
/**
* Find first bit set in word. Least significant bit is 1.
* Return 0 if no bits set.
*/
#ifdef HAVE___BUILTIN_FFS
#define ffs __builtin_ffs
#elif defined(_MSC_VER) && (_M_IX86 || _M_ARM || _M_AMD64 || _M_IA64)
static inline
int ffs(int i)
{
unsigned long index;
if (_BitScanForward(&index, i))
return index + 1;
else
return 0;
}
#else
extern
int ffs(int i);
#endif
#ifdef HAVE___BUILTIN_FFSLL
#define ffsll __builtin_ffsll
#elif defined(_MSC_VER) && (_M_AMD64 || _M_ARM64 || _M_IA64)
static inline int
ffsll(long long int i)
{
unsigned long index;
if (_BitScanForward64(&index, i))
return index + 1;
else
return 0;
}
#else
extern int
ffsll(long long int val);
#endif
/* Destructively loop over all of the bits in a mask as in:
*
* while (mymask) {
* int i = u_bit_scan(&mymask);
* ... process element i
* }
*
*/
static inline int
u_bit_scan(unsigned *mask)
{
const int i = ffs(*mask) - 1;
*mask ^= (1u << i);
return i;
}
#define u_foreach_bit(b, dword) \
for (uint32_t __dword = (dword), b; \
((b) = ffs(__dword) - 1, __dword); \
/* See util_bitcount below. */ \
__dword &= __dword - 1)
static inline int
u_bit_scan64(uint64_t *mask)
{
const int i = ffsll(*mask) - 1;
*mask ^= (((uint64_t)1) << i);
return i;
}
#define u_foreach_bit64(b, dword) \
for (uint64_t __dword = (dword), b; \
((b) = ffsll(__dword) - 1, __dword); \
/* See util_bitcount below. */ \
__dword &= __dword - 1)
/* Given two bitmasks, loop over all bits of both of them.
* Bits of mask1 are: b = scan_bit(mask1);
* Bits of mask2 are: b = offset + scan_bit(mask2);
*/
#define u_foreach_bit64_two_masks(b, mask1, offset, mask2) \
for (uint64_t __mask1 = (mask1), __mask2 = (mask2), b; \
(__mask1 ? ((b) = ffsll(__mask1) - 1) \
: ((b) = ffsll(__mask2) - 1 + offset), __mask1 || __mask2); \
__mask1 ? (__mask1 &= ~(1ull << (b))) : (__mask2 &= ~(1ull << (b - offset))))
/* Determine if an uint32_t value is a power of two.
*
* \note
* Zero is treated as a power of two.
*/
static inline bool
util_is_power_of_two_or_zero(uint32_t v)
{
return IS_POT(v);
}
/* Determine if an uint64_t value is a power of two.
*
* \note
* Zero is treated as a power of two.
*/
static inline bool
util_is_power_of_two_or_zero64(uint64_t v)
{
return IS_POT(v);
}
/* Determine if an uint32_t value is a power of two.
*
* \note
* Zero is \b not treated as a power of two.
*/
static inline bool
util_is_power_of_two_nonzero(uint32_t v)
{
/* __POPCNT__ is different from HAVE___BUILTIN_POPCOUNT. The latter
* indicates the existence of the __builtin_popcount function. The former
* indicates that _mm_popcnt_u32 exists and is a native instruction.
*
* The other alternative is to use SSE 4.2 compile-time flags. This has
* two drawbacks. First, there is currently no build infrastructure for
* SSE 4.2 (only 4.1), so that would have to be added. Second, some AMD
* CPUs support POPCNT but not SSE 4.2 (e.g., Barcelona).
*/
#ifdef __POPCNT__
return _mm_popcnt_u32(v) == 1;
#else
return IS_POT_NONZERO(v);
#endif
}
/* Determine if an uint64_t value is a power of two.
*
* \note
* Zero is \b not treated as a power of two.
*/
static inline bool
util_is_power_of_two_nonzero64(uint64_t v)
{
return IS_POT_NONZERO(v);
}
/* Determine if an size_t/uintptr_t/intptr_t value is a power of two.
*
* \note
* Zero is \b not treated as a power of two.
*/
static inline bool
util_is_power_of_two_nonzero_uintptr(uintptr_t v)
{
return IS_POT_NONZERO(v);
}
/* For looping over a bitmask when you want to loop over consecutive bits
* manually, for example:
*
* while (mask) {
* int start, count, i;
*
* u_bit_scan_consecutive_range(&mask, &start, &count);
*
* for (i = 0; i < count; i++)
* ... process element (start+i)
* }
*/
static inline void
u_bit_scan_consecutive_range(unsigned *mask, int *start, int *count)
{
if (*mask == 0xffffffff) {
*start = 0;
*count = 32;
*mask = 0;
return;
}
*start = ffs(*mask) - 1;
*count = ffs(~(*mask >> *start)) - 1;
*mask &= ~(((1u << *count) - 1) << *start);
}
static inline void
u_bit_scan_consecutive_range64(uint64_t *mask, int *start, int *count)
{
if (*mask == UINT64_MAX) {
*start = 0;
*count = 64;
*mask = 0;
return;
}
*start = ffsll(*mask) - 1;
*count = ffsll(~(*mask >> *start)) - 1;
*mask &= ~(((((uint64_t)1) << *count) - 1) << *start);
}
/**
* Find last bit set in a word. The least significant bit is 1.
* Return 0 if no bits are set.
* Essentially ffs() in the reverse direction.
*/
static inline unsigned
util_last_bit(unsigned u)
{
#if defined(HAVE___BUILTIN_CLZ)
return u == 0 ? 0 : 32 - __builtin_clz(u);
#elif defined(_MSC_VER) && (_M_IX86 || _M_ARM || _M_AMD64 || _M_IA64)
unsigned long index;
if (_BitScanReverse(&index, u))
return index + 1;
else
return 0;
#else
unsigned r = 0;
while (u) {
r++;
u >>= 1;
}
return r;
#endif
}
/**
* Find last bit set in a word. The least significant bit is 1.
* Return 0 if no bits are set.
* Essentially ffsll() in the reverse direction.
*/
static inline unsigned
util_last_bit64(uint64_t u)
{
#if defined(HAVE___BUILTIN_CLZLL)
return u == 0 ? 0 : 64 - __builtin_clzll(u);
#elif defined(_MSC_VER) && (_M_AMD64 || _M_ARM64 || _M_IA64)
unsigned long index;
if (_BitScanReverse64(&index, u))
return index + 1;
else
return 0;
#else
unsigned r = 0;
while (u) {
r++;
u >>= 1;
}
return r;
#endif
}
/**
* Find last bit in a word that does not match the sign bit. The least
* significant bit is 1.
* Return 0 if no bits are set.
*/
static inline unsigned
util_last_bit_signed(int i)
{
if (i >= 0)
return util_last_bit(i);
else
return util_last_bit(~(unsigned)i);
}
/* Returns a bitfield in which the first count bits starting at start are
* set.
*/
static inline unsigned
u_bit_consecutive(unsigned start, unsigned count)
{
assert(start + count <= 32);
if (count == 32)
return ~0;
return ((1u << count) - 1) << start;
}
static inline uint64_t
u_bit_consecutive64(unsigned start, unsigned count)
{
assert(start + count <= 64);
if (count == 64)
return ~(uint64_t)0;
return (((uint64_t)1 << count) - 1) << start;
}
/**
* Return number of bits set in n.
*/
static inline unsigned
util_bitcount(unsigned n)
{
#if defined(HAVE___BUILTIN_POPCOUNT)
return __builtin_popcount(n);
#elif __OPENCL_VERSION__
return popcount(n);
#else
/* K&R classic bitcount.
*
* For each iteration, clear the LSB from the bitfield.
* Requires only one iteration per set bit, instead of
* one iteration per bit less than highest set bit.
*/
unsigned bits;
for (bits = 0; n; bits++) {
n &= n - 1;
}
return bits;
#endif
}
/**
* Return the number of bits set in n using the native popcnt instruction.
* The caller is responsible for ensuring that popcnt is supported by the CPU.
*
* gcc doesn't use it if -mpopcnt or -march= that has popcnt is missing.
*
*/
static inline unsigned
util_popcnt_inline_asm(unsigned n)
{
#if (DETECT_ARCH_X86 || DETECT_ARCH_X86_64) && DETECT_CC_GCC
uint32_t out;
__asm volatile("popcnt %1, %0" : "=r"(out) : "r"(n));
return out;
#else
/* We should never get here by accident, but I'm sure it'll happen. */
return util_bitcount(n);
#endif
}
static inline unsigned
util_bitcount64(uint64_t n)
{
#ifdef HAVE___BUILTIN_POPCOUNTLL
return __builtin_popcountll(n);
#elif __OPENCL_VERSION__
return popcount(n);
#else
return util_bitcount((unsigned)n) + util_bitcount((unsigned)(n >> 32));
#endif
}
/**
* Widens the given bit mask by a multiplier, meaning that it will
* replicate each bit by that amount.
*
* For example:
* 0b101 widened by 2 will become: 0b110011
*
* This is typically used in shader I/O to transform a 64-bit
* writemask to a 32-bit writemask.
*/
static inline uint32_t
util_widen_mask(uint32_t mask, unsigned multiplier)
{
uint32_t new_mask = 0;
u_foreach_bit(i, mask)
new_mask |= ((1u << multiplier) - 1u) << (i * multiplier);
return new_mask;
}
#ifdef __cplusplus
}
/* util_bitcount has large measurable overhead (~2%), so it's recommended to
* use the POPCNT instruction via inline assembly if the CPU supports it.
*/
enum util_popcnt {
POPCNT_NO,
POPCNT_YES,
POPCNT_INVALID,
};
/* Convenient function to select popcnt through a C++ template argument.
* This should be used as part of larger functions that are optimized
* as a whole.
*/
template<util_popcnt POPCNT> inline unsigned
util_bitcount_fast(unsigned n)
{
if (POPCNT == POPCNT_YES)
return util_popcnt_inline_asm(n);
else
return util_bitcount(n);
}
#endif /* __cplusplus */
#endif /* BITSCAN_H */
src/util/bitset.h
+633
View File
@@ -0,0 +1,633 @@
/*
* Mesa 3-D graphics library
*
* Copyright (C) 2006 Brian Paul All Rights Reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included
* in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
/**
* \file bitset.h
* \brief Bitset of arbitrary size definitions.
* \author Michal Krol
*/
#ifndef BITSET_H
#define BITSET_H
#include <stdlib.h>
#include "util/bitscan.h"
#include "util/macros.h"
#include "ralloc.h"
/****************************************************************************
* generic bitset implementation
*/
#define BITSET_WORD unsigned int
#define BITSET_WORDBITS (sizeof (BITSET_WORD) * 8)
/* bitset declarations
*/
#define BITSET_WORDS(bits) (((bits) + BITSET_WORDBITS - 1) / BITSET_WORDBITS)
#define BITSET_DECLARE(name, bits) BITSET_WORD name[BITSET_WORDS(bits)]
#define BITSET_BYTES(bits) (BITSET_WORDS(bits) * sizeof(BITSET_WORD))
/* bitset operations
*/
#define BITSET_COPY(x, y) memcpy( (x), (y), sizeof (x) )
#define BITSET_EQUAL(x, y) (memcmp( (x), (y), sizeof (x) ) == 0)
#define BITSET_ZERO(x) memset( (x), 0, sizeof (x) )
#define BITSET_ONES(x) memset( (x), 0xff, sizeof (x) )
#define BITSET_SIZE(x) (8 * sizeof(x)) // bitset size in bits
#define BITSET_BITWORD(b) ((b) / BITSET_WORDBITS)
#define BITSET_BIT(b) (1u << ((b) % BITSET_WORDBITS))
/* single bit operations
*/
#define BITSET_TEST(x, b) (((x)[BITSET_BITWORD(b)] & BITSET_BIT(b)) != 0)
#define BITSET_SET(x, b) ((x)[BITSET_BITWORD(b)] |= BITSET_BIT(b))
#define BITSET_CLEAR(x, b) ((x)[BITSET_BITWORD(b)] &= ~BITSET_BIT(b))
#define BITSET_MASK(b) (((b) % BITSET_WORDBITS == 0) ? ~0 : BITSET_BIT(b) - 1)
#define BITSET_RANGE(b, e) ((BITSET_MASK((e) + 1)) & ~(BITSET_BIT(b) - 1))
/* logic bit operations
*/
static inline void
__bitset_and(BITSET_WORD *r, const BITSET_WORD *x, const BITSET_WORD *y, unsigned n)
{
for (unsigned i = 0; i < n; i++)
r[i] = x[i] & y[i];
}
static inline void
__bitset_or(BITSET_WORD *r, const BITSET_WORD *x, const BITSET_WORD *y, unsigned n)
{
for (unsigned i = 0; i < n; i++)
r[i] = x[i] | y[i];
}
static inline void
__bitset_not(BITSET_WORD *x, unsigned n)
{
for (unsigned i = 0; i < n; i++)
x[i] = ~x[i];
}
static inline void
__bitset_andnot(BITSET_WORD *r, const BITSET_WORD *x, const BITSET_WORD *y, unsigned n)
{
for (unsigned i = 0; i < n; i++)
r[i] = x[i] & ~y[i];
}
#define BITSET_AND(r, x, y) \
do { \
STATIC_ASSERT(ARRAY_SIZE(r) == ARRAY_SIZE(x)); \
STATIC_ASSERT(ARRAY_SIZE(r) == ARRAY_SIZE(y)); \
__bitset_and(r, x, y, ARRAY_SIZE(r)); \
} while (0)
#define BITSET_OR(r, x, y) \
do { \
STATIC_ASSERT(ARRAY_SIZE(r) == ARRAY_SIZE(x)); \
STATIC_ASSERT(ARRAY_SIZE(r) == ARRAY_SIZE(y)); \
__bitset_or(r, x, y, ARRAY_SIZE(r)); \
} while (0)
#define BITSET_NOT(x) \
__bitset_not(x, ARRAY_SIZE(x))
#define BITSET_ANDNOT(r, x, y) \
do { \
assert(ARRAY_SIZE(r) == ARRAY_SIZE(x)); \
assert(ARRAY_SIZE(r) == ARRAY_SIZE(y)); \
__bitset_andnot(r, x, y, ARRAY_SIZE(r)); \
} while (0)
static inline void
__bitset_rotate_right(BITSET_WORD *x, unsigned amount, unsigned n)
{
assert(amount < BITSET_WORDBITS);
if (amount == 0)
return;
for (unsigned i = 0; i < n - 1; i++) {
x[i] = (x[i] >> amount) | (x[i + 1] << (BITSET_WORDBITS - amount));
}
x[n - 1] = x[n - 1] >> amount;
}
static inline void
__bitset_rotate_left(BITSET_WORD *x, unsigned amount, unsigned n)
{
assert(amount < BITSET_WORDBITS);
if (amount == 0)
return;
for (int i = n - 1; i > 0; i--) {
x[i] = (x[i] << amount) | (x[i - 1] >> (BITSET_WORDBITS - amount));
}
x[0] = x[0] << amount;
}
static inline void
__bitset_shr(BITSET_WORD *x, unsigned amount, unsigned n)
{
const unsigned int words = amount / BITSET_WORDBITS;
if (amount == 0)
return;
if (words) {
unsigned i;
for (i = 0; i < n - words; i++)
x[i] = x[i + words];
while (i < n)
x[i++] = 0;
amount %= BITSET_WORDBITS;
}
__bitset_rotate_right(x, amount, n);
}
static inline void
__bitset_shl(BITSET_WORD *x, unsigned amount, unsigned n)
{
const int words = amount / BITSET_WORDBITS;
if (amount == 0)
return;
if (words) {
int i;
for (i = n - 1; i >= words; i--) {
x[i] = x[i - words];
}
while (i >= 0) {
x[i--] = 0;
}
amount %= BITSET_WORDBITS;
}
__bitset_rotate_left(x, amount, n);
}
#define BITSET_SHR(x, n) \
__bitset_shr(x, n, ARRAY_SIZE(x));
#define BITSET_SHL(x, n) \
__bitset_shl(x, n, ARRAY_SIZE(x));
/* bit range operations (e=end is inclusive)
*/
#define BITSET_GET_RANGE_INSIDE_WORD(x, b, e) \
(BITSET_BITWORD(b) == BITSET_BITWORD(e) ? \
(((x)[BITSET_BITWORD(b)] >> (b % BITSET_WORDBITS)) & \
BITSET_MASK((e) - (b) + 1)) : \
(assert (!"BITSET_TEST_RANGE: bit range crosses word boundary"), 0))
#define BITSET_TEST_RANGE_INSIDE_WORD(x, b, e, mask) \
(BITSET_GET_RANGE_INSIDE_WORD(x, b, e) == (mask))
#define BITSET_SET_RANGE_INSIDE_WORD(x, b, e) \
(BITSET_BITWORD(b) == BITSET_BITWORD(e) ? \
((x)[BITSET_BITWORD(b)] |= BITSET_RANGE(b, e)) : \
(assert (!"BITSET_SET_RANGE_INSIDE_WORD: bit range crosses word boundary"), 0))
#define BITSET_CLEAR_RANGE_INSIDE_WORD(x, b, e) \
(BITSET_BITWORD(b) == BITSET_BITWORD(e) ? \
((x)[BITSET_BITWORD(b)] &= ~BITSET_RANGE(b, e)) : \
(assert (!"BITSET_CLEAR_RANGE: bit range crosses word boundary"), 0))
static inline bool
__bitset_test_range(const BITSET_WORD *r, int start, int end)
{
while (start <= end) {
unsigned start_mod = start % BITSET_WORDBITS;
unsigned size = MIN2(BITSET_WORDBITS - start_mod, end - start + 1u);
if (!BITSET_TEST_RANGE_INSIDE_WORD(r, start, start + size - 1u, 0))
return true;
start += size;
}
return false;
}
#define BITSET_TEST_RANGE(x, b, e) \
__bitset_test_range(x, b, e)
static inline void
__bitset_set_range(BITSET_WORD *r, int start, int end)
{
while (start <= end) {
unsigned start_mod = start % BITSET_WORDBITS;
unsigned size = MIN2(BITSET_WORDBITS - start_mod, end - start + 1u);
BITSET_SET_RANGE_INSIDE_WORD(r, start, start + size - 1u);
start += size;
}
}
#define BITSET_SET_RANGE(x, b, e) \
__bitset_set_range(x, b, e)
static inline void
__bitset_clear_range(BITSET_WORD *r, int start, int end)
{
while (start <= end) {
unsigned start_mod = start % BITSET_WORDBITS;
unsigned size = MIN2(BITSET_WORDBITS - start_mod, end - start + 1u);
BITSET_CLEAR_RANGE_INSIDE_WORD(r, start, start + size - 1u);
start += size;
}
}
#define BITSET_CLEAR_RANGE(x, b, e) \
__bitset_clear_range(x, b, e)
#define BITSET_CLEAR_COUNT(x, b, n) BITSET_CLEAR_RANGE(x, (b), (b) + (n) - 1)
#define BITSET_TEST_COUNT(x, b, n) BITSET_TEST_RANGE(x, (b), (b) + (n) - 1)
#define BITSET_SET_COUNT(x, b, n) BITSET_SET_RANGE(x, (b), (b) + (n) - 1)
static inline unsigned
__bitset_extract(const BITSET_WORD *r, unsigned start, unsigned count)
{
assert(count <= BITSET_WORDBITS);
unsigned shift = start % BITSET_WORDBITS;
BITSET_WORD lower = r[BITSET_BITWORD(start)] >> shift;
BITSET_WORD upper = shift ? r[BITSET_BITWORD(start + count - 1)] << (BITSET_WORDBITS - shift) : 0;
BITSET_WORD total = lower | upper;
return count != BITSET_WORDBITS ? total & ((1u << count) - 1u) : total;
}
#define BITSET_EXTRACT(x, s, c) \
__bitset_extract(x, s, c)
static inline unsigned
__bitset_prefix_sum(const BITSET_WORD *x, unsigned b, unsigned n)
{
unsigned prefix = 0;
for (unsigned i = 0; i < n; i++) {
if ((i + 1) * BITSET_WORDBITS <= b) {
prefix += util_bitcount(x[i]);
} else {
prefix += util_bitcount(x[i] & BITFIELD_MASK(b - i * BITSET_WORDBITS));
break;
}
}
return prefix;
}
/* Count set bits in the bitset (compute the size/cardinality of the bitset).
* This is a special case of prefix sum, but this convenience method is more
* natural when applicable.
*/
static inline unsigned
__bitset_count(const BITSET_WORD *x, unsigned n)
{
return __bitset_prefix_sum(x, ~0, n);
}
#define BITSET_PREFIX_SUM(x, b) \
__bitset_prefix_sum(x, b, ARRAY_SIZE(x))
#define BITSET_COUNT(x) \
__bitset_count(x, ARRAY_SIZE(x))
/* Return true if the bitset has no bits set.
*/
static inline bool
__bitset_is_empty(const BITSET_WORD *x, int n)
{
for (int i = 0; i < n; i++) {
if (x[i])
return false;
}
return true;
}
/* Get first bit set in a bitset.
*/
static inline int
__bitset_ffs(const BITSET_WORD *x, int n)
{
for (int i = 0; i < n; i++) {
if (x[i])
return ffs(x[i]) + BITSET_WORDBITS * i;
}
return 0;
}
/* Get the last bit set in a bitset.
*/
static inline int
__bitset_last_bit(const BITSET_WORD *x, int n)
{
for (int i = n - 1; i >= 0; i--) {
if (x[i])
return util_last_bit(x[i]) + BITSET_WORDBITS * i;
}
return 0;
}
/* Get the last bit set in a bitset before last_bit.
*/
static inline int
__bitset_last_bit_before(const BITSET_WORD *x, int last_bit)
{
int n = last_bit / BITSET_WORDBITS;
int reminder = last_bit % BITSET_WORDBITS;
if (reminder) {
BITSET_WORD last = x[n] & BITFIELD_MASK(reminder);
if (last)
return util_last_bit(last) + n * BITSET_WORDBITS;
}
return __bitset_last_bit(x, n);
}
#define BITSET_FFS(x) __bitset_ffs(x, ARRAY_SIZE(x))
#define BITSET_LAST_BIT(x) __bitset_last_bit(x, ARRAY_SIZE(x))
#define BITSET_LAST_BIT_SIZED(x, size) __bitset_last_bit(x, size)
#define BITSET_LAST_BIT_BEFORE(x, last_bit) __bitset_last_bit_before(x, last_bit)
#define BITSET_IS_EMPTY(x) __bitset_is_empty(x, ARRAY_SIZE(x))
static inline unsigned
__bitset_next_set(unsigned i, BITSET_WORD *tmp,
const BITSET_WORD *set, unsigned size)
{
unsigned bit, word;
/* NOTE: The initial conditions for this function are very specific. At
* the start of the loop, the tmp variable must be set to *set and the
* initial i value set to 0. This way, if there is a bit set in the first
* word, we ignore the i-value and just grab that bit (so 0 is ok, even
* though 0 may be returned). If the first word is 0, then the value of
* `word` will be 0 and we will go on to look at the second word.
*/
word = BITSET_BITWORD(i);
while (*tmp == 0) {
word++;
if (word >= BITSET_WORDS(size))
return size;
*tmp = set[word];
}
/* Find the next set bit in the non-zero word */
bit = ffs(*tmp) - 1;
/* Unset the bit */
*tmp &= ~(1ull << bit);
return word * BITSET_WORDBITS + bit;
}
/**
* Iterates over each set bit in a set
*
* @param __i iteration variable, bit number
* @param __set the bitset to iterate (will not be modified)
* @param __size number of bits in the set to consider
*/
#define BITSET_FOREACH_SET(__i, __set, __size) \
for (BITSET_WORD __tmp = (__size) == 0 ? 0 : *(__set), *__foo = &__tmp; __foo != NULL; __foo = NULL) \
for (__i = 0; \
(__i = __bitset_next_set(__i, &__tmp, __set, __size)) < (__size);)
static inline void
__bitset_next_range(unsigned *start, unsigned *end, const BITSET_WORD *set,
unsigned size)
{
/* To find the next start, start searching from end. In the first iteration
* it will be at 0, in every subsequent iteration it will be at the first
* 0-bit after the range.
*/
unsigned word = BITSET_BITWORD(*end);
if (word >= BITSET_WORDS(size)) {
*start = *end = size;
return;
}
BITSET_WORD tmp = set[word] & ~(BITSET_BIT(*end) - 1);
while (!tmp) {
word++;
if (word >= BITSET_WORDS(size)) {
*start = *end = size;
return;
}
tmp = set[word];
}
*start = word * BITSET_WORDBITS + ffs(tmp) - 1;
/* Now do the opposite to find end. Here we can start at start + 1, because
* we know that the bit at start is 1 and we're searching for the first
* 0-bit.
*/
word = BITSET_BITWORD(*start + 1);
if (word >= BITSET_WORDS(size)) {
*end = size;
return;
}
tmp = set[word] | (BITSET_BIT(*start + 1) - 1);
while (~tmp == 0) {
word++;
if (word >= BITSET_WORDS(size)) {
*end = size;
return;
}
tmp = set[word];
}
/* Cap "end" at "size" in case there are extra bits past "size" set in the
* word. This is only necessary for "end" because we terminate the loop if
* "start" goes past "size".
*/
*end = MIN2(word * BITSET_WORDBITS + ffs(~tmp) - 1, size);
}
/**
* Iterates over each contiguous range of set bits in a set
*
* @param __start the first 1 bit of the current range
* @param __end the bit after the last 1 bit of the current range
* @param __set the bitset to iterate (will not be modified)
* @param __size number of bits in the set to consider
*/
#define BITSET_FOREACH_RANGE(__start, __end, __set, __size) \
for (__start = 0, __end = 0, \
__bitset_next_range(&__start, &__end, __set, __size); \
__start < __size; \
__bitset_next_range(&__start, &__end, __set, __size))
static inline BITSET_WORD *
BITSET_CALLOC(unsigned size)
{
return (BITSET_WORD *) calloc(BITSET_WORDS(size), sizeof(BITSET_WORD));
}
static inline BITSET_WORD *
BITSET_RZALLOC(const void *memctx, unsigned size)
{
return (BITSET_WORD *)rzalloc_size(memctx, BITSET_BYTES(size));
}
static inline BITSET_WORD *
BITSET_LINEAR_ZALLOC(linear_ctx *memctx, unsigned size)
{
return (BITSET_WORD *)linear_zalloc_child(memctx, BITSET_BYTES(size));
}
#ifdef __cplusplus
/**
* Simple C++ wrapper of a bitset type of static size, with value semantics
* and basic bitwise arithmetic operators. The operators defined below are
* expected to have the same semantics as the same operator applied to other
* fundamental integer types. T is the name of the struct to instantiate
* it as, and N is the number of bits in the bitset.
*/
#define DECLARE_BITSET_T(T, N) struct T { \
explicit \
operator bool() const \
{ \
for (unsigned i = 0; i < BITSET_WORDS(N); i++) \
if (words[i]) \
return true; \
return false; \
} \
\
T & \
operator=(int x) \
{ \
const T c = {{ (BITSET_WORD)x }}; \
return *this = c; \
} \
\
friend bool \
operator==(const T &b, const T &c) \
{ \
return BITSET_EQUAL(b.words, c.words); \
} \
\
friend bool \
operator!=(const T &b, const T &c) \
{ \
return !(b == c); \
} \
\
friend bool \
operator==(const T &b, int x) \
{ \
const T c = {{ (BITSET_WORD)x }}; \
return b == c; \
} \
\
friend bool \
operator!=(const T &b, int x) \
{ \
return !(b == x); \
} \
\
friend T \
operator~(const T &b) \
{ \
T c; \
for (unsigned i = 0; i < BITSET_WORDS(N); i++) \
c.words[i] = ~b.words[i]; \
return c; \
} \
\
T & \
operator|=(const T &b) \
{ \
for (unsigned i = 0; i < BITSET_WORDS(N); i++) \
words[i] |= b.words[i]; \
return *this; \
} \
\
friend T \
operator|(const T &b, const T &c) \
{ \
T d = b; \
d |= c; \
return d; \
} \
\
T & \
operator&=(const T &b) \
{ \
for (unsigned i = 0; i < BITSET_WORDS(N); i++) \
words[i] &= b.words[i]; \
return *this; \
} \
\
friend T \
operator&(const T &b, const T &c) \
{ \
T d = b; \
d &= c; \
return d; \
} \
\
bool \
test(unsigned i) const \
{ \
return BITSET_TEST(words, i); \
} \
\
T & \
set(unsigned i) \
{ \
BITSET_SET(words, i); \
return *this; \
} \
\
T & \
clear(unsigned i) \
{ \
BITSET_CLEAR(words, i); \
return *this; \
} \
\
BITSET_WORD words[BITSET_WORDS(N)]; \
}
#endif
#endif
src/util/blake3/README
+18
View File
@@ -0,0 +1,18 @@
This folder contains a local copy of BLAKE3 cryptographic hash library, version 1.8.2.
Except for changes listed in the "Changes" section, this is a verbatim copy from
https://github.com/BLAKE3-team/BLAKE3, tag 1.8.2.
Files will be periodically synchronized with the upstream, and any local changes should
be clearly documented below.
Changes:
- Rename .asm files to .masm due to a Meson limitation (https://mesonbuild.com/Release-notes-for-0-64-0.html#new-languages-nasm-and-masm)
- Add non-typedef struct name to blake3_hasher.
- Add "static" to blake3_hash4_neon, to comply with -Werror=missing-prototypes.
- Add mesa_blake3_visibility.h and set symbol visibility to hidden for assembly sources.
- Drop BLAKE3_PRIVATE from blake3_compress_subtree_wide and blake3_compress_subtree_wide_join_tbb
src/util/blake3/blake3.c
+652
View File
@@ -0,0 +1,652 @@
#include <assert.h>
#include <stdbool.h>
#include <string.h>
#include "blake3.h"
#include "blake3_impl.h"
const char *blake3_version(void) { return BLAKE3_VERSION_STRING; }
INLINE void chunk_state_init(blake3_chunk_state *self, const uint32_t key[8],
uint8_t flags) {
memcpy(self->cv, key, BLAKE3_KEY_LEN);
self->chunk_counter = 0;
memset(self->buf, 0, BLAKE3_BLOCK_LEN);
self->buf_len = 0;
self->blocks_compressed = 0;
self->flags = flags;
}
INLINE void chunk_state_reset(blake3_chunk_state *self, const uint32_t key[8],
uint64_t chunk_counter) {
memcpy(self->cv, key, BLAKE3_KEY_LEN);
self->chunk_counter = chunk_counter;
self->blocks_compressed = 0;
memset(self->buf, 0, BLAKE3_BLOCK_LEN);
self->buf_len = 0;
}
INLINE size_t chunk_state_len(const blake3_chunk_state *self) {
return (BLAKE3_BLOCK_LEN * (size_t)self->blocks_compressed) +
((size_t)self->buf_len);
}
INLINE size_t chunk_state_fill_buf(blake3_chunk_state *self,
const uint8_t *input, size_t input_len) {
size_t take = BLAKE3_BLOCK_LEN - ((size_t)self->buf_len);
if (take > input_len) {
take = input_len;
}
uint8_t *dest = self->buf + ((size_t)self->buf_len);
memcpy(dest, input, take);
self->buf_len += (uint8_t)take;
return take;
}
INLINE uint8_t chunk_state_maybe_start_flag(const blake3_chunk_state *self) {
if (self->blocks_compressed == 0) {
return CHUNK_START;
} else {
return 0;
}
}
typedef struct {
uint32_t input_cv[8];
uint64_t counter;
uint8_t block[BLAKE3_BLOCK_LEN];
uint8_t block_len;
uint8_t flags;
} output_t;
INLINE output_t make_output(const uint32_t input_cv[8],
const uint8_t block[BLAKE3_BLOCK_LEN],
uint8_t block_len, uint64_t counter,
uint8_t flags) {
output_t ret;
memcpy(ret.input_cv, input_cv, 32);
memcpy(ret.block, block, BLAKE3_BLOCK_LEN);
ret.block_len = block_len;
ret.counter = counter;
ret.flags = flags;
return ret;
}
// Chaining values within a given chunk (specifically the compress_in_place
// interface) are represented as words. This avoids unnecessary bytes<->words
// conversion overhead in the portable implementation. However, the hash_many
// interface handles both user input and parent node blocks, so it accepts
// bytes. For that reason, chaining values in the CV stack are represented as
// bytes.
INLINE void output_chaining_value(const output_t *self, uint8_t cv[32]) {
uint32_t cv_words[8];
memcpy(cv_words, self->input_cv, 32);
blake3_compress_in_place(cv_words, self->block, self->block_len,
self->counter, self->flags);
store_cv_words(cv, cv_words);
}
INLINE void output_root_bytes(const output_t *self, uint64_t seek, uint8_t *out,
size_t out_len) {
if (out_len == 0) {
return;
}
uint64_t output_block_counter = seek / 64;
size_t offset_within_block = seek % 64;
uint8_t wide_buf[64];
if(offset_within_block) {
blake3_compress_xof(self->input_cv, self->block, self->block_len, output_block_counter, self->flags | ROOT, wide_buf);
const size_t available_bytes = 64 - offset_within_block;
const size_t bytes = out_len > available_bytes ? available_bytes : out_len;
memcpy(out, wide_buf + offset_within_block, bytes);
out += bytes;
out_len -= bytes;
output_block_counter += 1;
}
if(out_len / 64) {
blake3_xof_many(self->input_cv, self->block, self->block_len, output_block_counter, self->flags | ROOT, out, out_len / 64);
}
output_block_counter += out_len / 64;
out += out_len & -64;
out_len -= out_len & -64;
if(out_len) {
blake3_compress_xof(self->input_cv, self->block, self->block_len, output_block_counter, self->flags | ROOT, wide_buf);
memcpy(out, wide_buf, out_len);
}
}
INLINE void chunk_state_update(blake3_chunk_state *self, const uint8_t *input,
size_t input_len) {
if (self->buf_len > 0) {
size_t take = chunk_state_fill_buf(self, input, input_len);
input += take;
input_len -= take;
if (input_len > 0) {
blake3_compress_in_place(
self->cv, self->buf, BLAKE3_BLOCK_LEN, self->chunk_counter,
self->flags | chunk_state_maybe_start_flag(self));
self->blocks_compressed += 1;
self->buf_len = 0;
memset(self->buf, 0, BLAKE3_BLOCK_LEN);
}
}
while (input_len > BLAKE3_BLOCK_LEN) {
blake3_compress_in_place(self->cv, input, BLAKE3_BLOCK_LEN,
self->chunk_counter,
self->flags | chunk_state_maybe_start_flag(self));
self->blocks_compressed += 1;
input += BLAKE3_BLOCK_LEN;
input_len -= BLAKE3_BLOCK_LEN;
}
chunk_state_fill_buf(self, input, input_len);
}
INLINE output_t chunk_state_output(const blake3_chunk_state *self) {
uint8_t block_flags =
self->flags | chunk_state_maybe_start_flag(self) | CHUNK_END;
return make_output(self->cv, self->buf, self->buf_len, self->chunk_counter,
block_flags);
}
INLINE output_t parent_output(const uint8_t block[BLAKE3_BLOCK_LEN],
const uint32_t key[8], uint8_t flags) {
return make_output(key, block, BLAKE3_BLOCK_LEN, 0, flags | PARENT);
}
// Given some input larger than one chunk, return the number of bytes that
// should go in the left subtree. This is the largest power-of-2 number of
// chunks that leaves at least 1 byte for the right subtree.
INLINE size_t left_subtree_len(size_t input_len) {
// Subtract 1 to reserve at least one byte for the right side. input_len
// should always be greater than BLAKE3_CHUNK_LEN.
size_t full_chunks = (input_len - 1) / BLAKE3_CHUNK_LEN;
return round_down_to_power_of_2(full_chunks) * BLAKE3_CHUNK_LEN;
}
// Use SIMD parallelism to hash up to MAX_SIMD_DEGREE chunks at the same time
// on a single thread. Write out the chunk chaining values and return the
// number of chunks hashed. These chunks are never the root and never empty;
// those cases use a different codepath.
INLINE size_t compress_chunks_parallel(const uint8_t *input, size_t input_len,
const uint32_t key[8],
uint64_t chunk_counter, uint8_t flags,
uint8_t *out) {
#if defined(BLAKE3_TESTING)
assert(0 < input_len);
assert(input_len <= MAX_SIMD_DEGREE * BLAKE3_CHUNK_LEN);
#endif
const uint8_t *chunks_array[MAX_SIMD_DEGREE];
size_t input_position = 0;
size_t chunks_array_len = 0;
while (input_len - input_position >= BLAKE3_CHUNK_LEN) {
chunks_array[chunks_array_len] = &input[input_position];
input_position += BLAKE3_CHUNK_LEN;
chunks_array_len += 1;
}
blake3_hash_many(chunks_array, chunks_array_len,
BLAKE3_CHUNK_LEN / BLAKE3_BLOCK_LEN, key, chunk_counter,
true, flags, CHUNK_START, CHUNK_END, out);
// Hash the remaining partial chunk, if there is one. Note that the empty
// chunk (meaning the empty message) is a different codepath.
if (input_len > input_position) {
uint64_t counter = chunk_counter + (uint64_t)chunks_array_len;
blake3_chunk_state chunk_state;
chunk_state_init(&chunk_state, key, flags);
chunk_state.chunk_counter = counter;
chunk_state_update(&chunk_state, &input[input_position],
input_len - input_position);
output_t output = chunk_state_output(&chunk_state);
output_chaining_value(&output, &out[chunks_array_len * BLAKE3_OUT_LEN]);
return chunks_array_len + 1;
} else {
return chunks_array_len;
}
}
// Use SIMD parallelism to hash up to MAX_SIMD_DEGREE parents at the same time
// on a single thread. Write out the parent chaining values and return the
// number of parents hashed. (If there's an odd input chaining value left over,
// return it as an additional output.) These parents are never the root and
// never empty; those cases use a different codepath.
INLINE size_t compress_parents_parallel(const uint8_t *child_chaining_values,
size_t num_chaining_values,
const uint32_t key[8], uint8_t flags,
uint8_t *out) {
#if defined(BLAKE3_TESTING)
assert(2 <= num_chaining_values);
assert(num_chaining_values <= 2 * MAX_SIMD_DEGREE_OR_2);
#endif
const uint8_t *parents_array[MAX_SIMD_DEGREE_OR_2];
size_t parents_array_len = 0;
while (num_chaining_values - (2 * parents_array_len) >= 2) {
parents_array[parents_array_len] =
&child_chaining_values[2 * parents_array_len * BLAKE3_OUT_LEN];
parents_array_len += 1;
}
blake3_hash_many(parents_array, parents_array_len, 1, key,
0, // Parents always use counter 0.
false, flags | PARENT,
0, // Parents have no start flags.
0, // Parents have no end flags.
out);
// If there's an odd child left over, it becomes an output.
if (num_chaining_values > 2 * parents_array_len) {
memcpy(&out[parents_array_len * BLAKE3_OUT_LEN],
&child_chaining_values[2 * parents_array_len * BLAKE3_OUT_LEN],
BLAKE3_OUT_LEN);
return parents_array_len + 1;
} else {
return parents_array_len;
}
}
// The wide helper function returns (writes out) an array of chaining values
// and returns the length of that array. The number of chaining values returned
// is the dynamically detected SIMD degree, at most MAX_SIMD_DEGREE. Or fewer,
// if the input is shorter than that many chunks. The reason for maintaining a
// wide array of chaining values going back up the tree, is to allow the
// implementation to hash as many parents in parallel as possible.
//
// As a special case when the SIMD degree is 1, this function will still return
// at least 2 outputs. This guarantees that this function doesn't perform the
// root compression. (If it did, it would use the wrong flags, and also we
// wouldn't be able to implement extendable output.) Note that this function is
// not used when the whole input is only 1 chunk long; that's a different
// codepath.
//
// Why not just have the caller split the input on the first update(), instead
// of implementing this special rule? Because we don't want to limit SIMD or
// multi-threading parallelism for that update().
static size_t blake3_compress_subtree_wide(const uint8_t *input,
size_t input_len,
const uint32_t key[8],
uint64_t chunk_counter,
uint8_t flags, uint8_t *out,
bool use_tbb) {
// Note that the single chunk case does *not* bump the SIMD degree up to 2
// when it is 1. If this implementation adds multi-threading in the future,
// this gives us the option of multi-threading even the 2-chunk case, which
// can help performance on smaller platforms.
if (input_len <= blake3_simd_degree() * BLAKE3_CHUNK_LEN) {
return compress_chunks_parallel(input, input_len, key, chunk_counter, flags,
out);
}
// With more than simd_degree chunks, we need to recurse. Start by dividing
// the input into left and right subtrees. (Note that this is only optimal
// as long as the SIMD degree is a power of 2. If we ever get a SIMD degree
// of 3 or something, we'll need a more complicated strategy.)
size_t left_input_len = left_subtree_len(input_len);
size_t right_input_len = input_len - left_input_len;
const uint8_t *right_input = &input[left_input_len];
uint64_t right_chunk_counter =
chunk_counter + (uint64_t)(left_input_len / BLAKE3_CHUNK_LEN);
// Make space for the child outputs. Here we use MAX_SIMD_DEGREE_OR_2 to
// account for the special case of returning 2 outputs when the SIMD degree
// is 1.
uint8_t cv_array[2 * MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN];
size_t degree = blake3_simd_degree();
if (left_input_len > BLAKE3_CHUNK_LEN && degree == 1) {
// The special case: We always use a degree of at least two, to make
// sure there are two outputs. Except, as noted above, at the chunk
// level, where we allow degree=1. (Note that the 1-chunk-input case is
// a different codepath.)
degree = 2;
}
uint8_t *right_cvs = &cv_array[degree * BLAKE3_OUT_LEN];
// Recurse!
size_t left_n = -1;
size_t right_n = -1;
#if defined(BLAKE3_USE_TBB)
blake3_compress_subtree_wide_join_tbb(
key, flags, use_tbb,
// left-hand side
input, left_input_len, chunk_counter, cv_array, &left_n,
// right-hand side
right_input, right_input_len, right_chunk_counter, right_cvs, &right_n);
#else
left_n = blake3_compress_subtree_wide(
input, left_input_len, key, chunk_counter, flags, cv_array, use_tbb);
right_n = blake3_compress_subtree_wide(right_input, right_input_len, key,
right_chunk_counter, flags, right_cvs,
use_tbb);
#endif // BLAKE3_USE_TBB
// The special case again. If simd_degree=1, then we'll have left_n=1 and
// right_n=1. Rather than compressing them into a single output, return
// them directly, to make sure we always have at least two outputs.
if (left_n == 1) {
memcpy(out, cv_array, 2 * BLAKE3_OUT_LEN);
return 2;
}
// Otherwise, do one layer of parent node compression.
size_t num_chaining_values = left_n + right_n;
return compress_parents_parallel(cv_array, num_chaining_values, key, flags,
out);
}
// Hash a subtree with compress_subtree_wide(), and then condense the resulting
// list of chaining values down to a single parent node. Don't compress that
// last parent node, however. Instead, return its message bytes (the
// concatenated chaining values of its children). This is necessary when the
// first call to update() supplies a complete subtree, because the topmost
// parent node of that subtree could end up being the root. It's also necessary
// for extended output in the general case.
//
// As with compress_subtree_wide(), this function is not used on inputs of 1
// chunk or less. That's a different codepath.
INLINE void
compress_subtree_to_parent_node(const uint8_t *input, size_t input_len,
const uint32_t key[8], uint64_t chunk_counter,
uint8_t flags, uint8_t out[2 * BLAKE3_OUT_LEN],
bool use_tbb) {
#if defined(BLAKE3_TESTING)
assert(input_len > BLAKE3_CHUNK_LEN);
#endif
uint8_t cv_array[MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN];
size_t num_cvs = blake3_compress_subtree_wide(input, input_len, key,
chunk_counter, flags, cv_array, use_tbb);
assert(num_cvs <= MAX_SIMD_DEGREE_OR_2);
// The following loop never executes when MAX_SIMD_DEGREE_OR_2 is 2, because
// as we just asserted, num_cvs will always be <=2 in that case. But GCC
// (particularly GCC 8.5) can't tell that it never executes, and if NDEBUG is
// set then it emits incorrect warnings here. We tried a few different
// hacks to silence these, but in the end our hacks just produced different
// warnings (see https://github.com/BLAKE3-team/BLAKE3/pull/380). Out of
// desperation, we ifdef out this entire loop when we know it's not needed.
#if MAX_SIMD_DEGREE_OR_2 > 2
// If MAX_SIMD_DEGREE_OR_2 is greater than 2 and there's enough input,
// compress_subtree_wide() returns more than 2 chaining values. Condense
// them into 2 by forming parent nodes repeatedly.
uint8_t out_array[MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN / 2];
while (num_cvs > 2) {
num_cvs =
compress_parents_parallel(cv_array, num_cvs, key, flags, out_array);
memcpy(cv_array, out_array, num_cvs * BLAKE3_OUT_LEN);
}
#endif
memcpy(out, cv_array, 2 * BLAKE3_OUT_LEN);
}
INLINE void hasher_init_base(blake3_hasher *self, const uint32_t key[8],
uint8_t flags) {
memcpy(self->key, key, BLAKE3_KEY_LEN);
chunk_state_init(&self->chunk, key, flags);
self->cv_stack_len = 0;
}
void blake3_hasher_init(blake3_hasher *self) { hasher_init_base(self, IV, 0); }
void blake3_hasher_init_keyed(blake3_hasher *self,
const uint8_t key[BLAKE3_KEY_LEN]) {
uint32_t key_words[8];
load_key_words(key, key_words);
hasher_init_base(self, key_words, KEYED_HASH);
}
void blake3_hasher_init_derive_key_raw(blake3_hasher *self, const void *context,
size_t context_len) {
blake3_hasher context_hasher;
hasher_init_base(&context_hasher, IV, DERIVE_KEY_CONTEXT);
blake3_hasher_update(&context_hasher, context, context_len);
uint8_t context_key[BLAKE3_KEY_LEN];
blake3_hasher_finalize(&context_hasher, context_key, BLAKE3_KEY_LEN);
uint32_t context_key_words[8];
load_key_words(context_key, context_key_words);
hasher_init_base(self, context_key_words, DERIVE_KEY_MATERIAL);
}
void blake3_hasher_init_derive_key(blake3_hasher *self, const char *context) {
blake3_hasher_init_derive_key_raw(self, context, strlen(context));
}
// As described in hasher_push_cv() below, we do "lazy merging", delaying
// merges until right before the next CV is about to be added. This is
// different from the reference implementation. Another difference is that we
// aren't always merging 1 chunk at a time. Instead, each CV might represent
// any power-of-two number of chunks, as long as the smaller-above-larger stack
// order is maintained. Instead of the "count the trailing 0-bits" algorithm
// described in the spec, we use a "count the total number of 1-bits" variant
// that doesn't require us to retain the subtree size of the CV on top of the
// stack. The principle is the same: each CV that should remain in the stack is
// represented by a 1-bit in the total number of chunks (or bytes) so far.
INLINE void hasher_merge_cv_stack(blake3_hasher *self, uint64_t total_len) {
size_t post_merge_stack_len = (size_t)popcnt(total_len);
while (self->cv_stack_len > post_merge_stack_len) {
uint8_t *parent_node =
&self->cv_stack[(self->cv_stack_len - 2) * BLAKE3_OUT_LEN];
output_t output = parent_output(parent_node, self->key, self->chunk.flags);
output_chaining_value(&output, parent_node);
self->cv_stack_len -= 1;
}
}
// In reference_impl.rs, we merge the new CV with existing CVs from the stack
// before pushing it. We can do that because we know more input is coming, so
// we know none of the merges are root.
//
// This setting is different. We want to feed as much input as possible to
// compress_subtree_wide(), without setting aside anything for the chunk_state.
// If the user gives us 64 KiB, we want to parallelize over all 64 KiB at once
// as a single subtree, if at all possible.
//
// This leads to two problems:
// 1) This 64 KiB input might be the only call that ever gets made to update.
// In this case, the root node of the 64 KiB subtree would be the root node
// of the whole tree, and it would need to be ROOT finalized. We can't
// compress it until we know.
// 2) This 64 KiB input might complete a larger tree, whose root node is
// similarly going to be the root of the whole tree. For example, maybe
// we have 196 KiB (that is, 128 + 64) hashed so far. We can't compress the
// node at the root of the 256 KiB subtree until we know how to finalize it.
//
// The second problem is solved with "lazy merging". That is, when we're about
// to add a CV to the stack, we don't merge it with anything first, as the
// reference impl does. Instead we do merges using the *previous* CV that was
// added, which is sitting on top of the stack, and we put the new CV
// (unmerged) on top of the stack afterwards. This guarantees that we never
// merge the root node until finalize().
//
// Solving the first problem requires an additional tool,
// compress_subtree_to_parent_node(). That function always returns the top
// *two* chaining values of the subtree it's compressing. We then do lazy
// merging with each of them separately, so that the second CV will always
// remain unmerged. (That also helps us support extendable output when we're
// hashing an input all-at-once.)
INLINE void hasher_push_cv(blake3_hasher *self, uint8_t new_cv[BLAKE3_OUT_LEN],
uint64_t chunk_counter) {
hasher_merge_cv_stack(self, chunk_counter);
memcpy(&self->cv_stack[self->cv_stack_len * BLAKE3_OUT_LEN], new_cv,
BLAKE3_OUT_LEN);
self->cv_stack_len += 1;
}
INLINE void blake3_hasher_update_base(blake3_hasher *self, const void *input,
size_t input_len, bool use_tbb) {
// Explicitly checking for zero avoids causing UB by passing a null pointer
// to memcpy. This comes up in practice with things like:
// std::vector<uint8_t> v;
// blake3_hasher_update(&hasher, v.data(), v.size());
if (input_len == 0) {
return;
}
const uint8_t *input_bytes = (const uint8_t *)input;
// If we have some partial chunk bytes in the internal chunk_state, we need
// to finish that chunk first.
if (chunk_state_len(&self->chunk) > 0) {
size_t take = BLAKE3_CHUNK_LEN - chunk_state_len(&self->chunk);
if (take > input_len) {
take = input_len;
}
chunk_state_update(&self->chunk, input_bytes, take);
input_bytes += take;
input_len -= take;
// If we've filled the current chunk and there's more coming, finalize this
// chunk and proceed. In this case we know it's not the root.
if (input_len > 0) {
output_t output = chunk_state_output(&self->chunk);
uint8_t chunk_cv[32];
output_chaining_value(&output, chunk_cv);
hasher_push_cv(self, chunk_cv, self->chunk.chunk_counter);
chunk_state_reset(&self->chunk, self->key, self->chunk.chunk_counter + 1);
} else {
return;
}
}
// Now the chunk_state is clear, and we have more input. If there's more than
// a single chunk (so, definitely not the root chunk), hash the largest whole
// subtree we can, with the full benefits of SIMD (and maybe in the future,
// multi-threading) parallelism. Two restrictions:
// - The subtree has to be a power-of-2 number of chunks. Only subtrees along
// the right edge can be incomplete, and we don't know where the right edge
// is going to be until we get to finalize().
// - The subtree must evenly divide the total number of chunks up until this
// point (if total is not 0). If the current incomplete subtree is only
// waiting for 1 more chunk, we can't hash a subtree of 4 chunks. We have
// to complete the current subtree first.
// Because we might need to break up the input to form powers of 2, or to
// evenly divide what we already have, this part runs in a loop.
while (input_len > BLAKE3_CHUNK_LEN) {
size_t subtree_len = round_down_to_power_of_2(input_len);
uint64_t count_so_far = self->chunk.chunk_counter * BLAKE3_CHUNK_LEN;
// Shrink the subtree_len until it evenly divides the count so far. We know
// that subtree_len itself is a power of 2, so we can use a bitmasking
// trick instead of an actual remainder operation. (Note that if the caller
// consistently passes power-of-2 inputs of the same size, as is hopefully
// typical, this loop condition will always fail, and subtree_len will
// always be the full length of the input.)
//
// An aside: We don't have to shrink subtree_len quite this much. For
// example, if count_so_far is 1, we could pass 2 chunks to
// compress_subtree_to_parent_node. Since we'll get 2 CVs back, we'll still
// get the right answer in the end, and we might get to use 2-way SIMD
// parallelism. The problem with this optimization, is that it gets us
// stuck always hashing 2 chunks. The total number of chunks will remain
// odd, and we'll never graduate to higher degrees of parallelism. See
// https://github.com/BLAKE3-team/BLAKE3/issues/69.
while ((((uint64_t)(subtree_len - 1)) & count_so_far) != 0) {
subtree_len /= 2;
}
// The shrunken subtree_len might now be 1 chunk long. If so, hash that one
// chunk by itself. Otherwise, compress the subtree into a pair of CVs.
uint64_t subtree_chunks = subtree_len / BLAKE3_CHUNK_LEN;
if (subtree_len <= BLAKE3_CHUNK_LEN) {
blake3_chunk_state chunk_state;
chunk_state_init(&chunk_state, self->key, self->chunk.flags);
chunk_state.chunk_counter = self->chunk.chunk_counter;
chunk_state_update(&chunk_state, input_bytes, subtree_len);
output_t output = chunk_state_output(&chunk_state);
uint8_t cv[BLAKE3_OUT_LEN];
output_chaining_value(&output, cv);
hasher_push_cv(self, cv, chunk_state.chunk_counter);
} else {
// This is the high-performance happy path, though getting here depends
// on the caller giving us a long enough input.
uint8_t cv_pair[2 * BLAKE3_OUT_LEN];
compress_subtree_to_parent_node(input_bytes, subtree_len, self->key,
self->chunk.chunk_counter,
self->chunk.flags, cv_pair, use_tbb);
hasher_push_cv(self, cv_pair, self->chunk.chunk_counter);
hasher_push_cv(self, &cv_pair[BLAKE3_OUT_LEN],
self->chunk.chunk_counter + (subtree_chunks / 2));
}
self->chunk.chunk_counter += subtree_chunks;
input_bytes += subtree_len;
input_len -= subtree_len;
}
// If there's any remaining input less than a full chunk, add it to the chunk
// state. In that case, also do a final merge loop to make sure the subtree
// stack doesn't contain any unmerged pairs. The remaining input means we
// know these merges are non-root. This merge loop isn't strictly necessary
// here, because hasher_push_chunk_cv already does its own merge loop, but it
// simplifies blake3_hasher_finalize below.
if (input_len > 0) {
chunk_state_update(&self->chunk, input_bytes, input_len);
hasher_merge_cv_stack(self, self->chunk.chunk_counter);
}
}
void blake3_hasher_update(blake3_hasher *self, const void *input,
size_t input_len) {
bool use_tbb = false;
blake3_hasher_update_base(self, input, input_len, use_tbb);
}
#if defined(BLAKE3_USE_TBB)
void blake3_hasher_update_tbb(blake3_hasher *self, const void *input,
size_t input_len) {
bool use_tbb = true;
blake3_hasher_update_base(self, input, input_len, use_tbb);
}
#endif // BLAKE3_USE_TBB
void blake3_hasher_finalize(const blake3_hasher *self, uint8_t *out,
size_t out_len) {
blake3_hasher_finalize_seek(self, 0, out, out_len);
}
void blake3_hasher_finalize_seek(const blake3_hasher *self, uint64_t seek,
uint8_t *out, size_t out_len) {
// Explicitly checking for zero avoids causing UB by passing a null pointer
// to memcpy. This comes up in practice with things like:
// std::vector<uint8_t> v;
// blake3_hasher_finalize(&hasher, v.data(), v.size());
if (out_len == 0) {
return;
}
// If the subtree stack is empty, then the current chunk is the root.
if (self->cv_stack_len == 0) {
output_t output = chunk_state_output(&self->chunk);
output_root_bytes(&output, seek, out, out_len);
return;
}
// If there are any bytes in the chunk state, finalize that chunk and do a
// roll-up merge between that chunk hash and every subtree in the stack. In
// this case, the extra merge loop at the end of blake3_hasher_update
// guarantees that none of the subtrees in the stack need to be merged with
// each other first. Otherwise, if there are no bytes in the chunk state,
// then the top of the stack is a chunk hash, and we start the merge from
// that.
output_t output;
size_t cvs_remaining;
if (chunk_state_len(&self->chunk) > 0) {
cvs_remaining = self->cv_stack_len;
output = chunk_state_output(&self->chunk);
} else {
// There are always at least 2 CVs in the stack in this case.
cvs_remaining = self->cv_stack_len - 2;
output = parent_output(&self->cv_stack[cvs_remaining * 32], self->key,
self->chunk.flags);
}
while (cvs_remaining > 0) {
cvs_remaining -= 1;
uint8_t parent_block[BLAKE3_BLOCK_LEN];
memcpy(parent_block, &self->cv_stack[cvs_remaining * 32], 32);
output_chaining_value(&output, &parent_block[32]);
output = parent_output(parent_block, self->key, self->chunk.flags);
}
output_root_bytes(&output, seek, out, out_len);
}
void blake3_hasher_reset(blake3_hasher *self) {
chunk_state_reset(&self->chunk, self->key, 0);
self->cv_stack_len = 0;
}
src/util/blake3/blake3.h
+60
View File
@@ -0,0 +1,60 @@
#ifndef BLAKE3_H
#define BLAKE3_H
#include <stddef.h>
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
#define BLAKE3_VERSION_STRING "1.8.2"
#define BLAKE3_KEY_LEN 32
#define BLAKE3_OUT_LEN 32
#define BLAKE3_BLOCK_LEN 64
#define BLAKE3_CHUNK_LEN 1024
#define BLAKE3_MAX_DEPTH 54
// This struct is a private implementation detail. It has to be here because
// it's part of blake3_hasher below.
typedef struct {
uint32_t cv[8];
uint64_t chunk_counter;
uint8_t buf[BLAKE3_BLOCK_LEN];
uint8_t buf_len;
uint8_t blocks_compressed;
uint8_t flags;
} blake3_chunk_state;
typedef struct blake3_hasher {
uint32_t key[8];
blake3_chunk_state chunk;
uint8_t cv_stack_len;
// The stack size is MAX_DEPTH + 1 because we do lazy merging. For example,
// with 7 chunks, we have 3 entries in the stack. Adding an 8th chunk
// requires a 4th entry, rather than merging everything down to 1, because we
// don't know whether more input is coming. This is different from how the
// reference implementation does things.
uint8_t cv_stack[(BLAKE3_MAX_DEPTH + 1) * BLAKE3_OUT_LEN];
} blake3_hasher;
const char *blake3_version(void);
void blake3_hasher_init(blake3_hasher *self);
void blake3_hasher_init_keyed(blake3_hasher *self,
const uint8_t key[BLAKE3_KEY_LEN]);
void blake3_hasher_init_derive_key(blake3_hasher *self, const char *context);
void blake3_hasher_init_derive_key_raw(blake3_hasher *self, const void *context,
size_t context_len);
void blake3_hasher_update(blake3_hasher *self, const void *input,
size_t input_len);
void blake3_hasher_finalize(const blake3_hasher *self, uint8_t *out,
size_t out_len);
void blake3_hasher_finalize_seek(const blake3_hasher *self, uint64_t seek,
uint8_t *out, size_t out_len);
void blake3_hasher_reset(blake3_hasher *self);
#ifdef __cplusplus
}
#endif
#endif /* BLAKE3_H */
src/util/blake3/blake3_avx2.c
+326
View File
@@ -0,0 +1,326 @@
#include "blake3_impl.h"
#include <immintrin.h>
#define DEGREE 8
INLINE __m256i loadu(const uint8_t src[32]) {
return _mm256_loadu_si256((const __m256i *)src);
}
INLINE void storeu(__m256i src, uint8_t dest[16]) {
_mm256_storeu_si256((__m256i *)dest, src);
}
INLINE __m256i addv(__m256i a, __m256i b) { return _mm256_add_epi32(a, b); }
// Note that clang-format doesn't like the name "xor" for some reason.
INLINE __m256i xorv(__m256i a, __m256i b) { return _mm256_xor_si256(a, b); }
INLINE __m256i set1(uint32_t x) { return _mm256_set1_epi32((int32_t)x); }
INLINE __m256i rot16(__m256i x) {
return _mm256_shuffle_epi8(
x, _mm256_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2,
13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2));
}
INLINE __m256i rot12(__m256i x) {
return _mm256_or_si256(_mm256_srli_epi32(x, 12), _mm256_slli_epi32(x, 32 - 12));
}
INLINE __m256i rot8(__m256i x) {
return _mm256_shuffle_epi8(
x, _mm256_set_epi8(12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1,
12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1));
}
INLINE __m256i rot7(__m256i x) {
return _mm256_or_si256(_mm256_srli_epi32(x, 7), _mm256_slli_epi32(x, 32 - 7));
}
INLINE void round_fn(__m256i v[16], __m256i m[16], size_t r) {
v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][0]]);
v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][2]]);
v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][4]]);
v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][6]]);
v[0] = addv(v[0], v[4]);
v[1] = addv(v[1], v[5]);
v[2] = addv(v[2], v[6]);
v[3] = addv(v[3], v[7]);
v[12] = xorv(v[12], v[0]);
v[13] = xorv(v[13], v[1]);
v[14] = xorv(v[14], v[2]);
v[15] = xorv(v[15], v[3]);
v[12] = rot16(v[12]);
v[13] = rot16(v[13]);
v[14] = rot16(v[14]);
v[15] = rot16(v[15]);
v[8] = addv(v[8], v[12]);
v[9] = addv(v[9], v[13]);
v[10] = addv(v[10], v[14]);
v[11] = addv(v[11], v[15]);
v[4] = xorv(v[4], v[8]);
v[5] = xorv(v[5], v[9]);
v[6] = xorv(v[6], v[10]);
v[7] = xorv(v[7], v[11]);
v[4] = rot12(v[4]);
v[5] = rot12(v[5]);
v[6] = rot12(v[6]);
v[7] = rot12(v[7]);
v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][1]]);
v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][3]]);
v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][5]]);
v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][7]]);
v[0] = addv(v[0], v[4]);
v[1] = addv(v[1], v[5]);
v[2] = addv(v[2], v[6]);
v[3] = addv(v[3], v[7]);
v[12] = xorv(v[12], v[0]);
v[13] = xorv(v[13], v[1]);
v[14] = xorv(v[14], v[2]);
v[15] = xorv(v[15], v[3]);
v[12] = rot8(v[12]);
v[13] = rot8(v[13]);
v[14] = rot8(v[14]);
v[15] = rot8(v[15]);
v[8] = addv(v[8], v[12]);
v[9] = addv(v[9], v[13]);
v[10] = addv(v[10], v[14]);
v[11] = addv(v[11], v[15]);
v[4] = xorv(v[4], v[8]);
v[5] = xorv(v[5], v[9]);
v[6] = xorv(v[6], v[10]);
v[7] = xorv(v[7], v[11]);
v[4] = rot7(v[4]);
v[5] = rot7(v[5]);
v[6] = rot7(v[6]);
v[7] = rot7(v[7]);
v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][8]]);
v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][10]]);
v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][12]]);
v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][14]]);
v[0] = addv(v[0], v[5]);
v[1] = addv(v[1], v[6]);
v[2] = addv(v[2], v[7]);
v[3] = addv(v[3], v[4]);
v[15] = xorv(v[15], v[0]);
v[12] = xorv(v[12], v[1]);
v[13] = xorv(v[13], v[2]);
v[14] = xorv(v[14], v[3]);
v[15] = rot16(v[15]);
v[12] = rot16(v[12]);
v[13] = rot16(v[13]);
v[14] = rot16(v[14]);
v[10] = addv(v[10], v[15]);
v[11] = addv(v[11], v[12]);
v[8] = addv(v[8], v[13]);
v[9] = addv(v[9], v[14]);
v[5] = xorv(v[5], v[10]);
v[6] = xorv(v[6], v[11]);
v[7] = xorv(v[7], v[8]);
v[4] = xorv(v[4], v[9]);
v[5] = rot12(v[5]);
v[6] = rot12(v[6]);
v[7] = rot12(v[7]);
v[4] = rot12(v[4]);
v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][9]]);
v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][11]]);
v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][13]]);
v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][15]]);
v[0] = addv(v[0], v[5]);
v[1] = addv(v[1], v[6]);
v[2] = addv(v[2], v[7]);
v[3] = addv(v[3], v[4]);
v[15] = xorv(v[15], v[0]);
v[12] = xorv(v[12], v[1]);
v[13] = xorv(v[13], v[2]);
v[14] = xorv(v[14], v[3]);
v[15] = rot8(v[15]);
v[12] = rot8(v[12]);
v[13] = rot8(v[13]);
v[14] = rot8(v[14]);
v[10] = addv(v[10], v[15]);
v[11] = addv(v[11], v[12]);
v[8] = addv(v[8], v[13]);
v[9] = addv(v[9], v[14]);
v[5] = xorv(v[5], v[10]);
v[6] = xorv(v[6], v[11]);
v[7] = xorv(v[7], v[8]);
v[4] = xorv(v[4], v[9]);
v[5] = rot7(v[5]);
v[6] = rot7(v[6]);
v[7] = rot7(v[7]);
v[4] = rot7(v[4]);
}
INLINE void transpose_vecs(__m256i vecs[DEGREE]) {
// Interleave 32-bit lanes. The low unpack is lanes 00/11/44/55, and the high
// is 22/33/66/77.
__m256i ab_0145 = _mm256_unpacklo_epi32(vecs[0], vecs[1]);
__m256i ab_2367 = _mm256_unpackhi_epi32(vecs[0], vecs[1]);
__m256i cd_0145 = _mm256_unpacklo_epi32(vecs[2], vecs[3]);
__m256i cd_2367 = _mm256_unpackhi_epi32(vecs[2], vecs[3]);
__m256i ef_0145 = _mm256_unpacklo_epi32(vecs[4], vecs[5]);
__m256i ef_2367 = _mm256_unpackhi_epi32(vecs[4], vecs[5]);
__m256i gh_0145 = _mm256_unpacklo_epi32(vecs[6], vecs[7]);
__m256i gh_2367 = _mm256_unpackhi_epi32(vecs[6], vecs[7]);
// Interleave 64-bit lanes. The low unpack is lanes 00/22 and the high is
// 11/33.
__m256i abcd_04 = _mm256_unpacklo_epi64(ab_0145, cd_0145);
__m256i abcd_15 = _mm256_unpackhi_epi64(ab_0145, cd_0145);
__m256i abcd_26 = _mm256_unpacklo_epi64(ab_2367, cd_2367);
__m256i abcd_37 = _mm256_unpackhi_epi64(ab_2367, cd_2367);
__m256i efgh_04 = _mm256_unpacklo_epi64(ef_0145, gh_0145);
__m256i efgh_15 = _mm256_unpackhi_epi64(ef_0145, gh_0145);
__m256i efgh_26 = _mm256_unpacklo_epi64(ef_2367, gh_2367);
__m256i efgh_37 = _mm256_unpackhi_epi64(ef_2367, gh_2367);
// Interleave 128-bit lanes.
vecs[0] = _mm256_permute2x128_si256(abcd_04, efgh_04, 0x20);
vecs[1] = _mm256_permute2x128_si256(abcd_15, efgh_15, 0x20);
vecs[2] = _mm256_permute2x128_si256(abcd_26, efgh_26, 0x20);
vecs[3] = _mm256_permute2x128_si256(abcd_37, efgh_37, 0x20);
vecs[4] = _mm256_permute2x128_si256(abcd_04, efgh_04, 0x31);
vecs[5] = _mm256_permute2x128_si256(abcd_15, efgh_15, 0x31);
vecs[6] = _mm256_permute2x128_si256(abcd_26, efgh_26, 0x31);
vecs[7] = _mm256_permute2x128_si256(abcd_37, efgh_37, 0x31);
}
INLINE void transpose_msg_vecs(const uint8_t *const *inputs,
size_t block_offset, __m256i out[16]) {
out[0] = loadu(&inputs[0][block_offset + 0 * sizeof(__m256i)]);
out[1] = loadu(&inputs[1][block_offset + 0 * sizeof(__m256i)]);
out[2] = loadu(&inputs[2][block_offset + 0 * sizeof(__m256i)]);
out[3] = loadu(&inputs[3][block_offset + 0 * sizeof(__m256i)]);
out[4] = loadu(&inputs[4][block_offset + 0 * sizeof(__m256i)]);
out[5] = loadu(&inputs[5][block_offset + 0 * sizeof(__m256i)]);
out[6] = loadu(&inputs[6][block_offset + 0 * sizeof(__m256i)]);
out[7] = loadu(&inputs[7][block_offset + 0 * sizeof(__m256i)]);
out[8] = loadu(&inputs[0][block_offset + 1 * sizeof(__m256i)]);
out[9] = loadu(&inputs[1][block_offset + 1 * sizeof(__m256i)]);
out[10] = loadu(&inputs[2][block_offset + 1 * sizeof(__m256i)]);
out[11] = loadu(&inputs[3][block_offset + 1 * sizeof(__m256i)]);
out[12] = loadu(&inputs[4][block_offset + 1 * sizeof(__m256i)]);
out[13] = loadu(&inputs[5][block_offset + 1 * sizeof(__m256i)]);
out[14] = loadu(&inputs[6][block_offset + 1 * sizeof(__m256i)]);
out[15] = loadu(&inputs[7][block_offset + 1 * sizeof(__m256i)]);
for (size_t i = 0; i < 8; ++i) {
_mm_prefetch((const void *)&inputs[i][block_offset + 256], _MM_HINT_T0);
}
transpose_vecs(&out[0]);
transpose_vecs(&out[8]);
}
INLINE void load_counters(uint64_t counter, bool increment_counter,
__m256i *out_lo, __m256i *out_hi) {
const __m256i mask = _mm256_set1_epi32(-(int32_t)increment_counter);
const __m256i add0 = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
const __m256i add1 = _mm256_and_si256(mask, add0);
__m256i l = _mm256_add_epi32(_mm256_set1_epi32((int32_t)counter), add1);
__m256i carry = _mm256_cmpgt_epi32(_mm256_xor_si256(add1, _mm256_set1_epi32(0x80000000)),
_mm256_xor_si256( l, _mm256_set1_epi32(0x80000000)));
__m256i h = _mm256_sub_epi32(_mm256_set1_epi32((int32_t)(counter >> 32)), carry);
*out_lo = l;
*out_hi = h;
}
static
void blake3_hash8_avx2(const uint8_t *const *inputs, size_t blocks,
const uint32_t key[8], uint64_t counter,
bool increment_counter, uint8_t flags,
uint8_t flags_start, uint8_t flags_end, uint8_t *out) {
__m256i h_vecs[8] = {
set1(key[0]), set1(key[1]), set1(key[2]), set1(key[3]),
set1(key[4]), set1(key[5]), set1(key[6]), set1(key[7]),
};
__m256i counter_low_vec, counter_high_vec;
load_counters(counter, increment_counter, &counter_low_vec,
&counter_high_vec);
uint8_t block_flags = flags | flags_start;
for (size_t block = 0; block < blocks; block++) {
if (block + 1 == blocks) {
block_flags |= flags_end;
}
__m256i block_len_vec = set1(BLAKE3_BLOCK_LEN);
__m256i block_flags_vec = set1(block_flags);
__m256i msg_vecs[16];
transpose_msg_vecs(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs);
__m256i v[16] = {
h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3],
h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7],
set1(IV[0]), set1(IV[1]), set1(IV[2]), set1(IV[3]),
counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec,
};
round_fn(v, msg_vecs, 0);
round_fn(v, msg_vecs, 1);
round_fn(v, msg_vecs, 2);
round_fn(v, msg_vecs, 3);
round_fn(v, msg_vecs, 4);
round_fn(v, msg_vecs, 5);
round_fn(v, msg_vecs, 6);
h_vecs[0] = xorv(v[0], v[8]);
h_vecs[1] = xorv(v[1], v[9]);
h_vecs[2] = xorv(v[2], v[10]);
h_vecs[3] = xorv(v[3], v[11]);
h_vecs[4] = xorv(v[4], v[12]);
h_vecs[5] = xorv(v[5], v[13]);
h_vecs[6] = xorv(v[6], v[14]);
h_vecs[7] = xorv(v[7], v[15]);
block_flags = flags;
}
transpose_vecs(h_vecs);
storeu(h_vecs[0], &out[0 * sizeof(__m256i)]);
storeu(h_vecs[1], &out[1 * sizeof(__m256i)]);
storeu(h_vecs[2], &out[2 * sizeof(__m256i)]);
storeu(h_vecs[3], &out[3 * sizeof(__m256i)]);
storeu(h_vecs[4], &out[4 * sizeof(__m256i)]);
storeu(h_vecs[5], &out[5 * sizeof(__m256i)]);
storeu(h_vecs[6], &out[6 * sizeof(__m256i)]);
storeu(h_vecs[7], &out[7 * sizeof(__m256i)]);
}
#if !defined(BLAKE3_NO_SSE41)
void blake3_hash_many_sse41(const uint8_t *const *inputs, size_t num_inputs,
size_t blocks, const uint32_t key[8],
uint64_t counter, bool increment_counter,
uint8_t flags, uint8_t flags_start,
uint8_t flags_end, uint8_t *out);
#else
void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs,
size_t blocks, const uint32_t key[8],
uint64_t counter, bool increment_counter,
uint8_t flags, uint8_t flags_start,
uint8_t flags_end, uint8_t *out);
#endif
void blake3_hash_many_avx2(const uint8_t *const *inputs, size_t num_inputs,
size_t blocks, const uint32_t key[8],
uint64_t counter, bool increment_counter,
uint8_t flags, uint8_t flags_start,
uint8_t flags_end, uint8_t *out) {
while (num_inputs >= DEGREE) {
blake3_hash8_avx2(inputs, blocks, key, counter, increment_counter, flags,
flags_start, flags_end, out);
if (increment_counter) {
counter += DEGREE;
}
inputs += DEGREE;
num_inputs -= DEGREE;
out = &out[DEGREE * BLAKE3_OUT_LEN];
}
#if !defined(BLAKE3_NO_SSE41)
blake3_hash_many_sse41(inputs, num_inputs, blocks, key, counter,
increment_counter, flags, flags_start, flags_end, out);
#else
blake3_hash_many_portable(inputs, num_inputs, blocks, key, counter,
increment_counter, flags, flags_start, flags_end,
out);
#endif
}
src/util/blake3/blake3_avx2_x86-64_unix.S
+1820
View File
File diff suppressed because it is too large. Load diff
src/util/blake3/blake3_avx2_x86-64_windows_gnu.S
+1817
View File
File diff suppressed because it is too large. Load diff
src/util/blake3/blake3_avx2_x86-64_windows_msvc.masm
+1828
View File
File diff suppressed because it is too large. Load diff
src/util/blake3/blake3_avx512.c
+1388
View File
File diff suppressed because it is too large. Load diff
src/util/blake3/blake3_avx512_x86-64_unix.S
+4834
View File
File diff suppressed because it is too large. Load diff
src/util/blake3/blake3_avx512_x86-64_windows_gnu.S
+2615
View File
File diff suppressed because it is too large. Load diff
src/util/blake3/blake3_avx512_x86-64_windows_msvc.masm
+2634
View File
File diff suppressed because it is too large. Load diff
src/util/blake3/blake3_dispatch.c
+332
View File
@@ -0,0 +1,332 @@
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#include "blake3_impl.h"
#if defined(_MSC_VER)
#include <Windows.h>
#endif
#if defined(IS_X86)
#if defined(_MSC_VER)
#include <intrin.h>
#elif defined(__GNUC__)
#include <immintrin.h>
#else
#undef IS_X86 /* Unimplemented! */
#endif
#endif
#if !defined(BLAKE3_ATOMICS)
#if defined(__has_include)
#if __has_include(<stdatomic.h>) && !defined(_MSC_VER)
#define BLAKE3_ATOMICS 1
#else
#define BLAKE3_ATOMICS 0
#endif /* __has_include(<stdatomic.h>) && !defined(_MSC_VER) */
#else
#define BLAKE3_ATOMICS 0
#endif /* defined(__has_include) */
#endif /* BLAKE3_ATOMICS */
#if BLAKE3_ATOMICS
#define ATOMIC_INT _Atomic int
#define ATOMIC_LOAD(x) x
#define ATOMIC_STORE(x, y) x = y
#elif defined(_MSC_VER)
#define ATOMIC_INT LONG
#define ATOMIC_LOAD(x) InterlockedOr(&x, 0)
#define ATOMIC_STORE(x, y) InterlockedExchange(&x, y)
#else
#define ATOMIC_INT int
#define ATOMIC_LOAD(x) x
#define ATOMIC_STORE(x, y) x = y
#endif
#define MAYBE_UNUSED(x) (void)((x))
#if defined(IS_X86)
static uint64_t xgetbv(void) {
#if defined(_MSC_VER)
return _xgetbv(0);
#else
uint32_t eax = 0, edx = 0;
__asm__ __volatile__("xgetbv\n" : "=a"(eax), "=d"(edx) : "c"(0));
return ((uint64_t)edx << 32) | eax;
#endif
}
static void cpuid(uint32_t out[4], uint32_t id) {
#if defined(_MSC_VER)
__cpuid((int *)out, id);
#elif defined(__i386__) || defined(_M_IX86)
__asm__ __volatile__("movl %%ebx, %1\n"
"cpuid\n"
"xchgl %1, %%ebx\n"
: "=a"(out[0]), "=r"(out[1]), "=c"(out[2]), "=d"(out[3])
: "a"(id));
#else
__asm__ __volatile__("cpuid\n"
: "=a"(out[0]), "=b"(out[1]), "=c"(out[2]), "=d"(out[3])
: "a"(id));
#endif
}
static void cpuidex(uint32_t out[4], uint32_t id, uint32_t sid) {
#if defined(_MSC_VER)
__cpuidex((int *)out, id, sid);
#elif defined(__i386__) || defined(_M_IX86)
__asm__ __volatile__("movl %%ebx, %1\n"
"cpuid\n"
"xchgl %1, %%ebx\n"
: "=a"(out[0]), "=r"(out[1]), "=c"(out[2]), "=d"(out[3])
: "a"(id), "c"(sid));
#else
__asm__ __volatile__("cpuid\n"
: "=a"(out[0]), "=b"(out[1]), "=c"(out[2]), "=d"(out[3])
: "a"(id), "c"(sid));
#endif
}
#endif
enum cpu_feature {
SSE2 = 1 << 0,
SSSE3 = 1 << 1,
SSE41 = 1 << 2,
AVX = 1 << 3,
AVX2 = 1 << 4,
AVX512F = 1 << 5,
AVX512VL = 1 << 6,
/* ... */
UNDEFINED = 1 << 30
};
#if !defined(BLAKE3_TESTING)
static /* Allow the variable to be controlled manually for testing */
#endif
ATOMIC_INT g_cpu_features = UNDEFINED;
#if !defined(BLAKE3_TESTING)
static
#endif
enum cpu_feature
get_cpu_features(void) {
/* If TSAN detects a data race here, try compiling with -DBLAKE3_ATOMICS=1 */
enum cpu_feature features = ATOMIC_LOAD(g_cpu_features);
if (features != UNDEFINED) {
return features;
} else {
#if defined(IS_X86)
uint32_t regs[4] = {0};
uint32_t *eax = &regs[0], *ebx = &regs[1], *ecx = &regs[2], *edx = &regs[3];
(void)edx;
features = 0;
cpuid(regs, 0);
const int max_id = *eax;
cpuid(regs, 1);
#if defined(__amd64__) || defined(_M_X64)
features |= SSE2;
#else
if (*edx & (1UL << 26))
features |= SSE2;
#endif
if (*ecx & (1UL << 9))
features |= SSSE3;
if (*ecx & (1UL << 19))
features |= SSE41;
if (*ecx & (1UL << 27)) { // OSXSAVE
const uint64_t mask = xgetbv();
if ((mask & 6) == 6) { // SSE and AVX states
if (*ecx & (1UL << 28))
features |= AVX;
if (max_id >= 7) {
cpuidex(regs, 7, 0);
if (*ebx & (1UL << 5))
features |= AVX2;
if ((mask & 224) == 224) { // Opmask, ZMM_Hi256, Hi16_Zmm
if (*ebx & (1UL << 31))
features |= AVX512VL;
if (*ebx & (1UL << 16))
features |= AVX512F;
}
}
}
}
ATOMIC_STORE(g_cpu_features, features);
return features;
#else
/* How to detect NEON? */
return 0;
#endif
}
}
void blake3_compress_in_place(uint32_t cv[8],
const uint8_t block[BLAKE3_BLOCK_LEN],
uint8_t block_len, uint64_t counter,
uint8_t flags) {
#if defined(IS_X86)
const enum cpu_feature features = get_cpu_features();
MAYBE_UNUSED(features);
#if !defined(BLAKE3_NO_AVX512)
if (features & AVX512VL) {
blake3_compress_in_place_avx512(cv, block, block_len, counter, flags);
return;
}
#endif
#if !defined(BLAKE3_NO_SSE41)
if (features & SSE41) {
blake3_compress_in_place_sse41(cv, block, block_len, counter, flags);
return;
}
#endif
#if !defined(BLAKE3_NO_SSE2)
if (features & SSE2) {
blake3_compress_in_place_sse2(cv, block, block_len, counter, flags);
return;
}
#endif
#endif
blake3_compress_in_place_portable(cv, block, block_len, counter, flags);
}
void blake3_compress_xof(const uint32_t cv[8],
const uint8_t block[BLAKE3_BLOCK_LEN],
uint8_t block_len, uint64_t counter, uint8_t flags,
uint8_t out[64]) {
#if defined(IS_X86)
const enum cpu_feature features = get_cpu_features();
MAYBE_UNUSED(features);
#if !defined(BLAKE3_NO_AVX512)
if (features & AVX512VL) {
blake3_compress_xof_avx512(cv, block, block_len, counter, flags, out);
return;
}
#endif
#if !defined(BLAKE3_NO_SSE41)
if (features & SSE41) {
blake3_compress_xof_sse41(cv, block, block_len, counter, flags, out);
return;
}
#endif
#if !defined(BLAKE3_NO_SSE2)
if (features & SSE2) {
blake3_compress_xof_sse2(cv, block, block_len, counter, flags, out);
return;
}
#endif
#endif
blake3_compress_xof_portable(cv, block, block_len, counter, flags, out);
}
void blake3_xof_many(const uint32_t cv[8],
const uint8_t block[BLAKE3_BLOCK_LEN],
uint8_t block_len, uint64_t counter, uint8_t flags,
uint8_t out[64], size_t outblocks) {
if (outblocks == 0) {
// The current assembly implementation always outputs at least 1 block.
return;
}
#if defined(IS_X86)
const enum cpu_feature features = get_cpu_features();
MAYBE_UNUSED(features);
#if !defined(_WIN32) && !defined(BLAKE3_NO_AVX512)
if (features & AVX512VL) {
blake3_xof_many_avx512(cv, block, block_len, counter, flags, out, outblocks);
return;
}
#endif
#endif
for(size_t i = 0; i < outblocks; ++i) {
blake3_compress_xof(cv, block, block_len, counter + i, flags, out + 64*i);
}
}
void blake3_hash_many(const uint8_t *const *inputs, size_t num_inputs,
size_t blocks, const uint32_t key[8], uint64_t counter,
bool increment_counter, uint8_t flags,
uint8_t flags_start, uint8_t flags_end, uint8_t *out) {
#if defined(IS_X86)
const enum cpu_feature features = get_cpu_features();
MAYBE_UNUSED(features);
#if !defined(BLAKE3_NO_AVX512)
if ((features & (AVX512F|AVX512VL)) == (AVX512F|AVX512VL)) {
blake3_hash_many_avx512(inputs, num_inputs, blocks, key, counter,
increment_counter, flags, flags_start, flags_end,
out);
return;
}
#endif
#if !defined(BLAKE3_NO_AVX2)
if (features & AVX2) {
blake3_hash_many_avx2(inputs, num_inputs, blocks, key, counter,
increment_counter, flags, flags_start, flags_end,
out);
return;
}
#endif
#if !defined(BLAKE3_NO_SSE41)
if (features & SSE41) {
blake3_hash_many_sse41(inputs, num_inputs, blocks, key, counter,
increment_counter, flags, flags_start, flags_end,
out);
return;
}
#endif
#if !defined(BLAKE3_NO_SSE2)
if (features & SSE2) {
blake3_hash_many_sse2(inputs, num_inputs, blocks, key, counter,
increment_counter, flags, flags_start, flags_end,
out);
return;
}
#endif
#endif
#if BLAKE3_USE_NEON == 1
blake3_hash_many_neon(inputs, num_inputs, blocks, key, counter,
increment_counter, flags, flags_start, flags_end, out);
return;
#endif
blake3_hash_many_portable(inputs, num_inputs, blocks, key, counter,
increment_counter, flags, flags_start, flags_end,
out);
}
// The dynamically detected SIMD degree of the current platform.
size_t blake3_simd_degree(void) {
#if defined(IS_X86)
const enum cpu_feature features = get_cpu_features();
MAYBE_UNUSED(features);
#if !defined(BLAKE3_NO_AVX512)
if ((features & (AVX512F|AVX512VL)) == (AVX512F|AVX512VL)) {
return 16;
}
#endif
#if !defined(BLAKE3_NO_AVX2)
if (features & AVX2) {
return 8;
}
#endif
#if !defined(BLAKE3_NO_SSE41)
if (features & SSE41) {
return 4;
}
#endif
#if !defined(BLAKE3_NO_SSE2)
if (features & SSE2) {
return 4;
}
#endif
#endif
#if BLAKE3_USE_NEON == 1
return 4;
#endif
return 1;
}
src/util/blake3/blake3_impl.h
+333
View File
@@ -0,0 +1,333 @@
#ifndef BLAKE3_IMPL_H
#define BLAKE3_IMPL_H
#include <assert.h>
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#include <string.h>
#include "blake3.h"
#ifdef __cplusplus
extern "C" {
#endif
// internal flags
enum blake3_flags {
CHUNK_START = 1 << 0,
CHUNK_END = 1 << 1,
PARENT = 1 << 2,
ROOT = 1 << 3,
KEYED_HASH = 1 << 4,
DERIVE_KEY_CONTEXT = 1 << 5,
DERIVE_KEY_MATERIAL = 1 << 6,
};
// This C implementation tries to support recent versions of GCC, Clang, and
// MSVC.
#if defined(_MSC_VER)
#define INLINE static __forceinline
#else
#define INLINE static inline __attribute__((always_inline))
#endif
#ifdef __cplusplus
#define NOEXCEPT noexcept
#else
#define NOEXCEPT
#endif
#if (defined(__x86_64__) || defined(_M_X64)) && !defined(_M_ARM64EC)
#define IS_X86
#define IS_X86_64
#endif
#if defined(__i386__) || defined(_M_IX86)
#define IS_X86
#define IS_X86_32
#endif
#if defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC)
#define IS_AARCH64
#endif
#if defined(IS_X86)
#if defined(_MSC_VER)
#include <intrin.h>
#endif
#endif
#if !defined(BLAKE3_USE_NEON)
// If BLAKE3_USE_NEON not manually set, autodetect based on AArch64ness
#if defined(IS_AARCH64)
#if defined(__ARM_BIG_ENDIAN)
#define BLAKE3_USE_NEON 0
#else
#define BLAKE3_USE_NEON 1
#endif
#else
#define BLAKE3_USE_NEON 0
#endif
#endif
#if defined(IS_X86)
#define MAX_SIMD_DEGREE 16
#elif BLAKE3_USE_NEON == 1
#define MAX_SIMD_DEGREE 4
#else
#define MAX_SIMD_DEGREE 1
#endif
// There are some places where we want a static size that's equal to the
// MAX_SIMD_DEGREE, but also at least 2.
#define MAX_SIMD_DEGREE_OR_2 (MAX_SIMD_DEGREE > 2 ? MAX_SIMD_DEGREE : 2)
static const uint32_t IV[8] = {0x6A09E667UL, 0xBB67AE85UL, 0x3C6EF372UL,
0xA54FF53AUL, 0x510E527FUL, 0x9B05688CUL,
0x1F83D9ABUL, 0x5BE0CD19UL};
static const uint8_t MSG_SCHEDULE[7][16] = {
{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
{2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8},
{3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1},
{10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6},
{12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4},
{9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7},
{11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13},
};
/* Find index of the highest set bit */
/* x is assumed to be nonzero. */
static unsigned int highest_one(uint64_t x) {
#if defined(__GNUC__) || defined(__clang__)
return 63 ^ (unsigned int)__builtin_clzll(x);
#elif defined(_MSC_VER) && defined(IS_X86_64)
unsigned long index;
_BitScanReverse64(&index, x);
return index;
#elif defined(_MSC_VER) && defined(IS_X86_32)
if(x >> 32) {
unsigned long index;
_BitScanReverse(&index, (unsigned long)(x >> 32));
return 32 + index;
} else {
unsigned long index;
_BitScanReverse(&index, (unsigned long)x);
return index;
}
#else
unsigned int c = 0;
if(x & 0xffffffff00000000ULL) { x >>= 32; c += 32; }
if(x & 0x00000000ffff0000ULL) { x >>= 16; c += 16; }
if(x & 0x000000000000ff00ULL) { x >>= 8; c += 8; }
if(x & 0x00000000000000f0ULL) { x >>= 4; c += 4; }
if(x & 0x000000000000000cULL) { x >>= 2; c += 2; }
if(x & 0x0000000000000002ULL) { c += 1; }
return c;
#endif
}
// Count the number of 1 bits.
INLINE unsigned int popcnt(uint64_t x) {
#if defined(__GNUC__) || defined(__clang__)
return (unsigned int)__builtin_popcountll(x);
#else
unsigned int count = 0;
while (x != 0) {
count += 1;
x &= x - 1;
}
return count;
#endif
}
// Largest power of two less than or equal to x. As a special case, returns 1
// when x is 0.
INLINE uint64_t round_down_to_power_of_2(uint64_t x) {
return 1ULL << highest_one(x | 1);
}
INLINE uint32_t counter_low(uint64_t counter) { return (uint32_t)counter; }
INLINE uint32_t counter_high(uint64_t counter) {
return (uint32_t)(counter >> 32);
}
INLINE uint32_t load32(const void *src) {
const uint8_t *p = (const uint8_t *)src;
return ((uint32_t)(p[0]) << 0) | ((uint32_t)(p[1]) << 8) |
((uint32_t)(p[2]) << 16) | ((uint32_t)(p[3]) << 24);
}
INLINE void load_key_words(const uint8_t key[BLAKE3_KEY_LEN],
uint32_t key_words[8]) {
key_words[0] = load32(&key[0 * 4]);
key_words[1] = load32(&key[1 * 4]);
key_words[2] = load32(&key[2 * 4]);
key_words[3] = load32(&key[3 * 4]);
key_words[4] = load32(&key[4 * 4]);
key_words[5] = load32(&key[5 * 4]);
key_words[6] = load32(&key[6 * 4]);
key_words[7] = load32(&key[7 * 4]);
}
INLINE void load_block_words(const uint8_t block[BLAKE3_BLOCK_LEN],
uint32_t block_words[16]) {
for (size_t i = 0; i < 16; i++) {
block_words[i] = load32(&block[i * 4]);
}
}
INLINE void store32(void *dst, uint32_t w) {
uint8_t *p = (uint8_t *)dst;
p[0] = (uint8_t)(w >> 0);
p[1] = (uint8_t)(w >> 8);
p[2] = (uint8_t)(w >> 16);
p[3] = (uint8_t)(w >> 24);
}
INLINE void store_cv_words(uint8_t bytes_out[32], uint32_t cv_words[8]) {
store32(&bytes_out[0 * 4], cv_words[0]);
store32(&bytes_out[1 * 4], cv_words[1]);
store32(&bytes_out[2 * 4], cv_words[2]);
store32(&bytes_out[3 * 4], cv_words[3]);
store32(&bytes_out[4 * 4], cv_words[4]);
store32(&bytes_out[5 * 4], cv_words[5]);
store32(&bytes_out[6 * 4], cv_words[6]);
store32(&bytes_out[7 * 4], cv_words[7]);
}
void blake3_compress_in_place(uint32_t cv[8],
const uint8_t block[BLAKE3_BLOCK_LEN],
uint8_t block_len, uint64_t counter,
uint8_t flags);
void blake3_compress_xof(const uint32_t cv[8],
const uint8_t block[BLAKE3_BLOCK_LEN],
uint8_t block_len, uint64_t counter, uint8_t flags,
uint8_t out[64]);
void blake3_xof_many(const uint32_t cv[8],
const uint8_t block[BLAKE3_BLOCK_LEN],
uint8_t block_len, uint64_t counter, uint8_t flags,
uint8_t out[64], size_t outblocks);
void blake3_hash_many(const uint8_t *const *inputs, size_t num_inputs,
size_t blocks, const uint32_t key[8], uint64_t counter,
bool increment_counter, uint8_t flags,
uint8_t flags_start, uint8_t flags_end, uint8_t *out);
size_t blake3_simd_degree(void);
static size_t blake3_compress_subtree_wide(const uint8_t *input, size_t input_len,
const uint32_t key[8],
uint64_t chunk_counter, uint8_t flags,
uint8_t *out, bool use_tbb);
#if defined(BLAKE3_USE_TBB)
void blake3_compress_subtree_wide_join_tbb(
// shared params
const uint32_t key[8], uint8_t flags, bool use_tbb,
// left-hand side params
const uint8_t *l_input, size_t l_input_len, uint64_t l_chunk_counter,
uint8_t *l_cvs, size_t *l_n,
// right-hand side params
const uint8_t *r_input, size_t r_input_len, uint64_t r_chunk_counter,
uint8_t *r_cvs, size_t *r_n) NOEXCEPT;
#endif
// Declarations for implementation-specific functions.
void blake3_compress_in_place_portable(uint32_t cv[8],
const uint8_t block[BLAKE3_BLOCK_LEN],
uint8_t block_len, uint64_t counter,
uint8_t flags);
void blake3_compress_xof_portable(const uint32_t cv[8],
const uint8_t block[BLAKE3_BLOCK_LEN],
uint8_t block_len, uint64_t counter,
uint8_t flags, uint8_t out[64]);
void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs,
size_t blocks, const uint32_t key[8],
uint64_t counter, bool increment_counter,
uint8_t flags, uint8_t flags_start,
uint8_t flags_end, uint8_t *out);
#if defined(IS_X86)
#if !defined(BLAKE3_NO_SSE2)
void blake3_compress_in_place_sse2(uint32_t cv[8],
const uint8_t block[BLAKE3_BLOCK_LEN],
uint8_t block_len, uint64_t counter,
uint8_t flags);
void blake3_compress_xof_sse2(const uint32_t cv[8],
const uint8_t block[BLAKE3_BLOCK_LEN],
uint8_t block_len, uint64_t counter,
uint8_t flags, uint8_t out[64]);
void blake3_hash_many_sse2(const uint8_t *const *inputs, size_t num_inputs,
size_t blocks, const uint32_t key[8],
uint64_t counter, bool increment_counter,
uint8_t flags, uint8_t flags_start,
uint8_t flags_end, uint8_t *out);
#endif
#if !defined(BLAKE3_NO_SSE41)
void blake3_compress_in_place_sse41(uint32_t cv[8],
const uint8_t block[BLAKE3_BLOCK_LEN],
uint8_t block_len, uint64_t counter,
uint8_t flags);
void blake3_compress_xof_sse41(const uint32_t cv[8],
const uint8_t block[BLAKE3_BLOCK_LEN],
uint8_t block_len, uint64_t counter,
uint8_t flags, uint8_t out[64]);
void blake3_hash_many_sse41(const uint8_t *const *inputs, size_t num_inputs,
size_t blocks, const uint32_t key[8],
uint64_t counter, bool increment_counter,
uint8_t flags, uint8_t flags_start,
uint8_t flags_end, uint8_t *out);
#endif
#if !defined(BLAKE3_NO_AVX2)
void blake3_hash_many_avx2(const uint8_t *const *inputs, size_t num_inputs,
size_t blocks, const uint32_t key[8],
uint64_t counter, bool increment_counter,
uint8_t flags, uint8_t flags_start,
uint8_t flags_end, uint8_t *out);
#endif
#if !defined(BLAKE3_NO_AVX512)
void blake3_compress_in_place_avx512(uint32_t cv[8],
const uint8_t block[BLAKE3_BLOCK_LEN],
uint8_t block_len, uint64_t counter,
uint8_t flags);
void blake3_compress_xof_avx512(const uint32_t cv[8],
const uint8_t block[BLAKE3_BLOCK_LEN],
uint8_t block_len, uint64_t counter,
uint8_t flags, uint8_t out[64]);
void blake3_hash_many_avx512(const uint8_t *const *inputs, size_t num_inputs,
size_t blocks, const uint32_t key[8],
uint64_t counter, bool increment_counter,
uint8_t flags, uint8_t flags_start,
uint8_t flags_end, uint8_t *out);
#if !defined(_WIN32)
void blake3_xof_many_avx512(const uint32_t cv[8],
const uint8_t block[BLAKE3_BLOCK_LEN],
uint8_t block_len, uint64_t counter, uint8_t flags,
uint8_t* out, size_t outblocks);
#endif
#endif
#endif
#if BLAKE3_USE_NEON == 1
void blake3_hash_many_neon(const uint8_t *const *inputs, size_t num_inputs,
size_t blocks, const uint32_t key[8],
uint64_t counter, bool increment_counter,
uint8_t flags, uint8_t flags_start,
uint8_t flags_end, uint8_t *out);
#endif
#ifdef __cplusplus
}
#endif
#endif /* BLAKE3_IMPL_H */
src/util/blake3/blake3_neon.c
+366
View File
@@ -0,0 +1,366 @@
#include "blake3_impl.h"
#include <arm_neon.h>
#ifdef __ARM_BIG_ENDIAN
#error "This implementation only supports little-endian ARM."
// It might be that all we need for big-endian support here is to get the loads
// and stores right, but step zero would be finding a way to test it in CI.
#endif
INLINE uint32x4_t loadu_128(const uint8_t src[16]) {
// vld1q_u32 has alignment requirements. Don't use it.
return vreinterpretq_u32_u8(vld1q_u8(src));
}
INLINE void storeu_128(uint32x4_t src, uint8_t dest[16]) {
// vst1q_u32 has alignment requirements. Don't use it.
vst1q_u8(dest, vreinterpretq_u8_u32(src));
}
INLINE uint32x4_t add_128(uint32x4_t a, uint32x4_t b) {
return vaddq_u32(a, b);
}
INLINE uint32x4_t xor_128(uint32x4_t a, uint32x4_t b) {
return veorq_u32(a, b);
}
INLINE uint32x4_t set1_128(uint32_t x) { return vld1q_dup_u32(&x); }
INLINE uint32x4_t set4(uint32_t a, uint32_t b, uint32_t c, uint32_t d) {
uint32_t array[4] = {a, b, c, d};
return vld1q_u32(array);
}
INLINE uint32x4_t rot16_128(uint32x4_t x) {
// The straightforward implementation would be two shifts and an or, but that's
// slower on microarchitectures we've tested. See
// https://github.com/BLAKE3-team/BLAKE3/pull/319.
// return vorrq_u32(vshrq_n_u32(x, 16), vshlq_n_u32(x, 32 - 16));
return vreinterpretq_u32_u16(vrev32q_u16(vreinterpretq_u16_u32(x)));
}
INLINE uint32x4_t rot12_128(uint32x4_t x) {
// See comment in rot16_128.
// return vorrq_u32(vshrq_n_u32(x, 12), vshlq_n_u32(x, 32 - 12));
return vsriq_n_u32(vshlq_n_u32(x, 32-12), x, 12);
}
INLINE uint32x4_t rot8_128(uint32x4_t x) {
// See comment in rot16_128.
// return vorrq_u32(vshrq_n_u32(x, 8), vshlq_n_u32(x, 32 - 8));
#if defined(__clang__)
return vreinterpretq_u32_u8(__builtin_shufflevector(vreinterpretq_u8_u32(x), vreinterpretq_u8_u32(x), 1,2,3,0,5,6,7,4,9,10,11,8,13,14,15,12));
#elif __GNUC__ * 10000 + __GNUC_MINOR__ * 100 >=40700
static const uint8x16_t r8 = {1,2,3,0,5,6,7,4,9,10,11,8,13,14,15,12};
return vreinterpretq_u32_u8(__builtin_shuffle(vreinterpretq_u8_u32(x), vreinterpretq_u8_u32(x), r8));
#else
return vsriq_n_u32(vshlq_n_u32(x, 32-8), x, 8);
#endif
}
INLINE uint32x4_t rot7_128(uint32x4_t x) {
// See comment in rot16_128.
// return vorrq_u32(vshrq_n_u32(x, 7), vshlq_n_u32(x, 32 - 7));
return vsriq_n_u32(vshlq_n_u32(x, 32-7), x, 7);
}
// TODO: compress_neon
// TODO: hash2_neon
/*
* ----------------------------------------------------------------------------
* hash4_neon
* ----------------------------------------------------------------------------
*/
INLINE void round_fn4(uint32x4_t v[16], uint32x4_t m[16], size_t r) {
v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][0]]);
v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][2]]);
v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][4]]);
v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][6]]);
v[0] = add_128(v[0], v[4]);
v[1] = add_128(v[1], v[5]);
v[2] = add_128(v[2], v[6]);
v[3] = add_128(v[3], v[7]);
v[12] = xor_128(v[12], v[0]);
v[13] = xor_128(v[13], v[1]);
v[14] = xor_128(v[14], v[2]);
v[15] = xor_128(v[15], v[3]);
v[12] = rot16_128(v[12]);
v[13] = rot16_128(v[13]);
v[14] = rot16_128(v[14]);
v[15] = rot16_128(v[15]);
v[8] = add_128(v[8], v[12]);
v[9] = add_128(v[9], v[13]);
v[10] = add_128(v[10], v[14]);
v[11] = add_128(v[11], v[15]);
v[4] = xor_128(v[4], v[8]);
v[5] = xor_128(v[5], v[9]);
v[6] = xor_128(v[6], v[10]);
v[7] = xor_128(v[7], v[11]);
v[4] = rot12_128(v[4]);
v[5] = rot12_128(v[5]);
v[6] = rot12_128(v[6]);
v[7] = rot12_128(v[7]);
v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][1]]);
v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][3]]);
v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][5]]);
v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][7]]);
v[0] = add_128(v[0], v[4]);
v[1] = add_128(v[1], v[5]);
v[2] = add_128(v[2], v[6]);
v[3] = add_128(v[3], v[7]);
v[12] = xor_128(v[12], v[0]);
v[13] = xor_128(v[13], v[1]);
v[14] = xor_128(v[14], v[2]);
v[15] = xor_128(v[15], v[3]);
v[12] = rot8_128(v[12]);
v[13] = rot8_128(v[13]);
v[14] = rot8_128(v[14]);
v[15] = rot8_128(v[15]);
v[8] = add_128(v[8], v[12]);
v[9] = add_128(v[9], v[13]);
v[10] = add_128(v[10], v[14]);
v[11] = add_128(v[11], v[15]);
v[4] = xor_128(v[4], v[8]);
v[5] = xor_128(v[5], v[9]);
v[6] = xor_128(v[6], v[10]);
v[7] = xor_128(v[7], v[11]);
v[4] = rot7_128(v[4]);
v[5] = rot7_128(v[5]);
v[6] = rot7_128(v[6]);
v[7] = rot7_128(v[7]);
v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][8]]);
v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][10]]);
v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][12]]);
v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][14]]);
v[0] = add_128(v[0], v[5]);
v[1] = add_128(v[1], v[6]);
v[2] = add_128(v[2], v[7]);
v[3] = add_128(v[3], v[4]);
v[15] = xor_128(v[15], v[0]);
v[12] = xor_128(v[12], v[1]);
v[13] = xor_128(v[13], v[2]);
v[14] = xor_128(v[14], v[3]);
v[15] = rot16_128(v[15]);
v[12] = rot16_128(v[12]);
v[13] = rot16_128(v[13]);
v[14] = rot16_128(v[14]);
v[10] = add_128(v[10], v[15]);
v[11] = add_128(v[11], v[12]);
v[8] = add_128(v[8], v[13]);
v[9] = add_128(v[9], v[14]);
v[5] = xor_128(v[5], v[10]);
v[6] = xor_128(v[6], v[11]);
v[7] = xor_128(v[7], v[8]);
v[4] = xor_128(v[4], v[9]);
v[5] = rot12_128(v[5]);
v[6] = rot12_128(v[6]);
v[7] = rot12_128(v[7]);
v[4] = rot12_128(v[4]);
v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][9]]);
v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][11]]);
v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][13]]);
v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][15]]);
v[0] = add_128(v[0], v[5]);
v[1] = add_128(v[1], v[6]);
v[2] = add_128(v[2], v[7]);
v[3] = add_128(v[3], v[4]);
v[15] = xor_128(v[15], v[0]);
v[12] = xor_128(v[12], v[1]);
v[13] = xor_128(v[13], v[2]);
v[14] = xor_128(v[14], v[3]);
v[15] = rot8_128(v[15]);
v[12] = rot8_128(v[12]);
v[13] = rot8_128(v[13]);
v[14] = rot8_128(v[14]);
v[10] = add_128(v[10], v[15]);
v[11] = add_128(v[11], v[12]);
v[8] = add_128(v[8], v[13]);
v[9] = add_128(v[9], v[14]);
v[5] = xor_128(v[5], v[10]);
v[6] = xor_128(v[6], v[11]);
v[7] = xor_128(v[7], v[8]);
v[4] = xor_128(v[4], v[9]);
v[5] = rot7_128(v[5]);
v[6] = rot7_128(v[6]);
v[7] = rot7_128(v[7]);
v[4] = rot7_128(v[4]);
}
INLINE void transpose_vecs_128(uint32x4_t vecs[4]) {
// Individually transpose the four 2x2 sub-matrices in each corner.
uint32x4x2_t rows01 = vtrnq_u32(vecs[0], vecs[1]);
uint32x4x2_t rows23 = vtrnq_u32(vecs[2], vecs[3]);
// Swap the top-right and bottom-left 2x2s (which just got transposed).
vecs[0] =
vcombine_u32(vget_low_u32(rows01.val[0]), vget_low_u32(rows23.val[0]));
vecs[1] =
vcombine_u32(vget_low_u32(rows01.val[1]), vget_low_u32(rows23.val[1]));
vecs[2] =
vcombine_u32(vget_high_u32(rows01.val[0]), vget_high_u32(rows23.val[0]));
vecs[3] =
vcombine_u32(vget_high_u32(rows01.val[1]), vget_high_u32(rows23.val[1]));
}
INLINE void transpose_msg_vecs4(const uint8_t *const *inputs,
size_t block_offset, uint32x4_t out[16]) {
out[0] = loadu_128(&inputs[0][block_offset + 0 * sizeof(uint32x4_t)]);
out[1] = loadu_128(&inputs[1][block_offset + 0 * sizeof(uint32x4_t)]);
out[2] = loadu_128(&inputs[2][block_offset + 0 * sizeof(uint32x4_t)]);
out[3] = loadu_128(&inputs[3][block_offset + 0 * sizeof(uint32x4_t)]);
out[4] = loadu_128(&inputs[0][block_offset + 1 * sizeof(uint32x4_t)]);
out[5] = loadu_128(&inputs[1][block_offset + 1 * sizeof(uint32x4_t)]);
out[6] = loadu_128(&inputs[2][block_offset + 1 * sizeof(uint32x4_t)]);
out[7] = loadu_128(&inputs[3][block_offset + 1 * sizeof(uint32x4_t)]);
out[8] = loadu_128(&inputs[0][block_offset + 2 * sizeof(uint32x4_t)]);
out[9] = loadu_128(&inputs[1][block_offset + 2 * sizeof(uint32x4_t)]);
out[10] = loadu_128(&inputs[2][block_offset + 2 * sizeof(uint32x4_t)]);
out[11] = loadu_128(&inputs[3][block_offset + 2 * sizeof(uint32x4_t)]);
out[12] = loadu_128(&inputs[0][block_offset + 3 * sizeof(uint32x4_t)]);
out[13] = loadu_128(&inputs[1][block_offset + 3 * sizeof(uint32x4_t)]);
out[14] = loadu_128(&inputs[2][block_offset + 3 * sizeof(uint32x4_t)]);
out[15] = loadu_128(&inputs[3][block_offset + 3 * sizeof(uint32x4_t)]);
transpose_vecs_128(&out[0]);
transpose_vecs_128(&out[4]);
transpose_vecs_128(&out[8]);
transpose_vecs_128(&out[12]);
}
INLINE void load_counters4(uint64_t counter, bool increment_counter,
uint32x4_t *out_low, uint32x4_t *out_high) {
uint64_t mask = (increment_counter ? ~0 : 0);
*out_low = set4(
counter_low(counter + (mask & 0)), counter_low(counter + (mask & 1)),
counter_low(counter + (mask & 2)), counter_low(counter + (mask & 3)));
*out_high = set4(
counter_high(counter + (mask & 0)), counter_high(counter + (mask & 1)),
counter_high(counter + (mask & 2)), counter_high(counter + (mask & 3)));
}
static void blake3_hash4_neon(const uint8_t *const *inputs, size_t blocks,
const uint32_t key[8], uint64_t counter,
bool increment_counter, uint8_t flags,
uint8_t flags_start, uint8_t flags_end, uint8_t *out) {
uint32x4_t h_vecs[8] = {
set1_128(key[0]), set1_128(key[1]), set1_128(key[2]), set1_128(key[3]),
set1_128(key[4]), set1_128(key[5]), set1_128(key[6]), set1_128(key[7]),
};
uint32x4_t counter_low_vec, counter_high_vec;
load_counters4(counter, increment_counter, &counter_low_vec,
&counter_high_vec);
uint8_t block_flags = flags | flags_start;
for (size_t block = 0; block < blocks; block++) {
if (block + 1 == blocks) {
block_flags |= flags_end;
}
uint32x4_t block_len_vec = set1_128(BLAKE3_BLOCK_LEN);
uint32x4_t block_flags_vec = set1_128(block_flags);
uint32x4_t msg_vecs[16];
transpose_msg_vecs4(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs);
uint32x4_t v[16] = {
h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3],
h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7],
set1_128(IV[0]), set1_128(IV[1]), set1_128(IV[2]), set1_128(IV[3]),
counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec,
};
round_fn4(v, msg_vecs, 0);
round_fn4(v, msg_vecs, 1);
round_fn4(v, msg_vecs, 2);
round_fn4(v, msg_vecs, 3);
round_fn4(v, msg_vecs, 4);
round_fn4(v, msg_vecs, 5);
round_fn4(v, msg_vecs, 6);
h_vecs[0] = xor_128(v[0], v[8]);
h_vecs[1] = xor_128(v[1], v[9]);
h_vecs[2] = xor_128(v[2], v[10]);
h_vecs[3] = xor_128(v[3], v[11]);
h_vecs[4] = xor_128(v[4], v[12]);
h_vecs[5] = xor_128(v[5], v[13]);
h_vecs[6] = xor_128(v[6], v[14]);
h_vecs[7] = xor_128(v[7], v[15]);
block_flags = flags;
}
transpose_vecs_128(&h_vecs[0]);
transpose_vecs_128(&h_vecs[4]);
// The first four vecs now contain the first half of each output, and the
// second four vecs contain the second half of each output.
storeu_128(h_vecs[0], &out[0 * sizeof(uint32x4_t)]);
storeu_128(h_vecs[4], &out[1 * sizeof(uint32x4_t)]);
storeu_128(h_vecs[1], &out[2 * sizeof(uint32x4_t)]);
storeu_128(h_vecs[5], &out[3 * sizeof(uint32x4_t)]);
storeu_128(h_vecs[2], &out[4 * sizeof(uint32x4_t)]);
storeu_128(h_vecs[6], &out[5 * sizeof(uint32x4_t)]);
storeu_128(h_vecs[3], &out[6 * sizeof(uint32x4_t)]);
storeu_128(h_vecs[7], &out[7 * sizeof(uint32x4_t)]);
}
/*
* ----------------------------------------------------------------------------
* hash_many_neon
* ----------------------------------------------------------------------------
*/
void blake3_compress_in_place_portable(uint32_t cv[8],
const uint8_t block[BLAKE3_BLOCK_LEN],
uint8_t block_len, uint64_t counter,
uint8_t flags);
INLINE void hash_one_neon(const uint8_t *input, size_t blocks,
const uint32_t key[8], uint64_t counter,
uint8_t flags, uint8_t flags_start, uint8_t flags_end,
uint8_t out[BLAKE3_OUT_LEN]) {
uint32_t cv[8];
memcpy(cv, key, BLAKE3_KEY_LEN);
uint8_t block_flags = flags | flags_start;
while (blocks > 0) {
if (blocks == 1) {
block_flags |= flags_end;
}
// TODO: Implement compress_neon. However note that according to
// https://github.com/BLAKE2/BLAKE2/commit/7965d3e6e1b4193438b8d3a656787587d2579227,
// compress_neon might not be any faster than compress_portable.
blake3_compress_in_place_portable(cv, input, BLAKE3_BLOCK_LEN, counter,
block_flags);
input = &input[BLAKE3_BLOCK_LEN];
blocks -= 1;
block_flags = flags;
}
memcpy(out, cv, BLAKE3_OUT_LEN);
}
void blake3_hash_many_neon(const uint8_t *const *inputs, size_t num_inputs,
size_t blocks, const uint32_t key[8],
uint64_t counter, bool increment_counter,
uint8_t flags, uint8_t flags_start,
uint8_t flags_end, uint8_t *out) {
while (num_inputs >= 4) {
blake3_hash4_neon(inputs, blocks, key, counter, increment_counter, flags,
flags_start, flags_end, out);
if (increment_counter) {
counter += 4;
}
inputs += 4;
num_inputs -= 4;
out = &out[4 * BLAKE3_OUT_LEN];
}
while (num_inputs > 0) {
hash_one_neon(inputs[0], blocks, key, counter, flags, flags_start,
flags_end, out);
if (increment_counter) {
counter += 1;
}
inputs += 1;
num_inputs -= 1;
out = &out[BLAKE3_OUT_LEN];
}
}
src/util/blake3/blake3_portable.c
+160
View File
@@ -0,0 +1,160 @@
#include "blake3_impl.h"
#include <string.h>
INLINE uint32_t rotr32(uint32_t w, uint32_t c) {
return (w >> c) | (w << (32 - c));
}
INLINE void g(uint32_t *state, size_t a, size_t b, size_t c, size_t d,
uint32_t x, uint32_t y) {
state[a] = state[a] + state[b] + x;
state[d] = rotr32(state[d] ^ state[a], 16);
state[c] = state[c] + state[d];
state[b] = rotr32(state[b] ^ state[c], 12);
state[a] = state[a] + state[b] + y;
state[d] = rotr32(state[d] ^ state[a], 8);
state[c] = state[c] + state[d];
state[b] = rotr32(state[b] ^ state[c], 7);
}
INLINE void round_fn(uint32_t state[16], const uint32_t *msg, size_t round) {
// Select the message schedule based on the round.
const uint8_t *schedule = MSG_SCHEDULE[round];
// Mix the columns.
g(state, 0, 4, 8, 12, msg[schedule[0]], msg[schedule[1]]);
g(state, 1, 5, 9, 13, msg[schedule[2]], msg[schedule[3]]);
g(state, 2, 6, 10, 14, msg[schedule[4]], msg[schedule[5]]);
g(state, 3, 7, 11, 15, msg[schedule[6]], msg[schedule[7]]);
// Mix the rows.
g(state, 0, 5, 10, 15, msg[schedule[8]], msg[schedule[9]]);
g(state, 1, 6, 11, 12, msg[schedule[10]], msg[schedule[11]]);
g(state, 2, 7, 8, 13, msg[schedule[12]], msg[schedule[13]]);
g(state, 3, 4, 9, 14, msg[schedule[14]], msg[schedule[15]]);
}
INLINE void compress_pre(uint32_t state[16], const uint32_t cv[8],
const uint8_t block[BLAKE3_BLOCK_LEN],
uint8_t block_len, uint64_t counter, uint8_t flags) {
uint32_t block_words[16];
block_words[0] = load32(block + 4 * 0);
block_words[1] = load32(block + 4 * 1);
block_words[2] = load32(block + 4 * 2);
block_words[3] = load32(block + 4 * 3);
block_words[4] = load32(block + 4 * 4);
block_words[5] = load32(block + 4 * 5);
block_words[6] = load32(block + 4 * 6);
block_words[7] = load32(block + 4 * 7);
block_words[8] = load32(block + 4 * 8);
block_words[9] = load32(block + 4 * 9);
block_words[10] = load32(block + 4 * 10);
block_words[11] = load32(block + 4 * 11);
block_words[12] = load32(block + 4 * 12);
block_words[13] = load32(block + 4 * 13);
block_words[14] = load32(block + 4 * 14);
block_words[15] = load32(block + 4 * 15);
state[0] = cv[0];
state[1] = cv[1];
state[2] = cv[2];
state[3] = cv[3];
state[4] = cv[4];
state[5] = cv[5];
state[6] = cv[6];
state[7] = cv[7];
state[8] = IV[0];
state[9] = IV[1];
state[10] = IV[2];
state[11] = IV[3];
state[12] = counter_low(counter);
state[13] = counter_high(counter);
state[14] = (uint32_t)block_len;
state[15] = (uint32_t)flags;
round_fn(state, &block_words[0], 0);
round_fn(state, &block_words[0], 1);
round_fn(state, &block_words[0], 2);
round_fn(state, &block_words[0], 3);
round_fn(state, &block_words[0], 4);
round_fn(state, &block_words[0], 5);
round_fn(state, &block_words[0], 6);
}
void blake3_compress_in_place_portable(uint32_t cv[8],
const uint8_t block[BLAKE3_BLOCK_LEN],
uint8_t block_len, uint64_t counter,
uint8_t flags) {
uint32_t state[16];
compress_pre(state, cv, block, block_len, counter, flags);
cv[0] = state[0] ^ state[8];
cv[1] = state[1] ^ state[9];
cv[2] = state[2] ^ state[10];
cv[3] = state[3] ^ state[11];
cv[4] = state[4] ^ state[12];
cv[5] = state[5] ^ state[13];
cv[6] = state[6] ^ state[14];
cv[7] = state[7] ^ state[15];
}
void blake3_compress_xof_portable(const uint32_t cv[8],
const uint8_t block[BLAKE3_BLOCK_LEN],
uint8_t block_len, uint64_t counter,
uint8_t flags, uint8_t out[64]) {
uint32_t state[16];
compress_pre(state, cv, block, block_len, counter, flags);
store32(&out[0 * 4], state[0] ^ state[8]);
store32(&out[1 * 4], state[1] ^ state[9]);
store32(&out[2 * 4], state[2] ^ state[10]);
store32(&out[3 * 4], state[3] ^ state[11]);
store32(&out[4 * 4], state[4] ^ state[12]);
store32(&out[5 * 4], state[5] ^ state[13]);
store32(&out[6 * 4], state[6] ^ state[14]);
store32(&out[7 * 4], state[7] ^ state[15]);
store32(&out[8 * 4], state[8] ^ cv[0]);
store32(&out[9 * 4], state[9] ^ cv[1]);
store32(&out[10 * 4], state[10] ^ cv[2]);
store32(&out[11 * 4], state[11] ^ cv[3]);
store32(&out[12 * 4], state[12] ^ cv[4]);
store32(&out[13 * 4], state[13] ^ cv[5]);
store32(&out[14 * 4], state[14] ^ cv[6]);
store32(&out[15 * 4], state[15] ^ cv[7]);
}
INLINE void hash_one_portable(const uint8_t *input, size_t blocks,
const uint32_t key[8], uint64_t counter,
uint8_t flags, uint8_t flags_start,
uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN]) {
uint32_t cv[8];
memcpy(cv, key, BLAKE3_KEY_LEN);
uint8_t block_flags = flags | flags_start;
while (blocks > 0) {
if (blocks == 1) {
block_flags |= flags_end;
}
blake3_compress_in_place_portable(cv, input, BLAKE3_BLOCK_LEN, counter,
block_flags);
input = &input[BLAKE3_BLOCK_LEN];
blocks -= 1;
block_flags = flags;
}
store_cv_words(out, cv);
}
void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs,
size_t blocks, const uint32_t key[8],
uint64_t counter, bool increment_counter,
uint8_t flags, uint8_t flags_start,
uint8_t flags_end, uint8_t *out) {
while (num_inputs > 0) {
hash_one_portable(inputs[0], blocks, key, counter, flags, flags_start,
flags_end, out);
if (increment_counter) {
counter += 1;
}
inputs += 1;
num_inputs -= 1;
out = &out[BLAKE3_OUT_LEN];
}
}
src/util/blake3/blake3_sse2.c
+566
View File
@@ -0,0 +1,566 @@
#include "blake3_impl.h"
#include <immintrin.h>
#define DEGREE 4
#define _mm_shuffle_ps2(a, b, c) \
(_mm_castps_si128( \
_mm_shuffle_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b), (c))))
INLINE __m128i loadu(const uint8_t src[16]) {
return _mm_loadu_si128((const __m128i *)src);
}
INLINE void storeu(__m128i src, uint8_t dest[16]) {
_mm_storeu_si128((__m128i *)dest, src);
}
INLINE __m128i addv(__m128i a, __m128i b) { return _mm_add_epi32(a, b); }
// Note that clang-format doesn't like the name "xor" for some reason.
INLINE __m128i xorv(__m128i a, __m128i b) { return _mm_xor_si128(a, b); }
INLINE __m128i set1(uint32_t x) { return _mm_set1_epi32((int32_t)x); }
INLINE __m128i set4(uint32_t a, uint32_t b, uint32_t c, uint32_t d) {
return _mm_setr_epi32((int32_t)a, (int32_t)b, (int32_t)c, (int32_t)d);
}
INLINE __m128i rot16(__m128i x) {
return _mm_shufflehi_epi16(_mm_shufflelo_epi16(x, 0xB1), 0xB1);
}
INLINE __m128i rot12(__m128i x) {
return xorv(_mm_srli_epi32(x, 12), _mm_slli_epi32(x, 32 - 12));
}
INLINE __m128i rot8(__m128i x) {
return xorv(_mm_srli_epi32(x, 8), _mm_slli_epi32(x, 32 - 8));
}
INLINE __m128i rot7(__m128i x) {
return xorv(_mm_srli_epi32(x, 7), _mm_slli_epi32(x, 32 - 7));
}
INLINE void g1(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3,
__m128i m) {
*row0 = addv(addv(*row0, m), *row1);
*row3 = xorv(*row3, *row0);
*row3 = rot16(*row3);
*row2 = addv(*row2, *row3);
*row1 = xorv(*row1, *row2);
*row1 = rot12(*row1);
}
INLINE void g2(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3,
__m128i m) {
*row0 = addv(addv(*row0, m), *row1);
*row3 = xorv(*row3, *row0);
*row3 = rot8(*row3);
*row2 = addv(*row2, *row3);
*row1 = xorv(*row1, *row2);
*row1 = rot7(*row1);
}
// Note the optimization here of leaving row1 as the unrotated row, rather than
// row0. All the message loads below are adjusted to compensate for this. See
// discussion at https://github.com/sneves/blake2-avx2/pull/4
INLINE void diagonalize(__m128i *row0, __m128i *row2, __m128i *row3) {
*row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(2, 1, 0, 3));
*row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2));
*row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(0, 3, 2, 1));
}
INLINE void undiagonalize(__m128i *row0, __m128i *row2, __m128i *row3) {
*row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(0, 3, 2, 1));
*row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2));
*row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(2, 1, 0, 3));
}
INLINE __m128i blend_epi16(__m128i a, __m128i b, const int16_t imm8) {
const __m128i bits = _mm_set_epi16(0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01);
__m128i mask = _mm_set1_epi16(imm8);
mask = _mm_and_si128(mask, bits);
mask = _mm_cmpeq_epi16(mask, bits);
return _mm_or_si128(_mm_and_si128(mask, b), _mm_andnot_si128(mask, a));
}
INLINE void compress_pre(__m128i rows[4], const uint32_t cv[8],
const uint8_t block[BLAKE3_BLOCK_LEN],
uint8_t block_len, uint64_t counter, uint8_t flags) {
rows[0] = loadu((uint8_t *)&cv[0]);
rows[1] = loadu((uint8_t *)&cv[4]);
rows[2] = set4(IV[0], IV[1], IV[2], IV[3]);
rows[3] = set4(counter_low(counter), counter_high(counter),
(uint32_t)block_len, (uint32_t)flags);
__m128i m0 = loadu(&block[sizeof(__m128i) * 0]);
__m128i m1 = loadu(&block[sizeof(__m128i) * 1]);
__m128i m2 = loadu(&block[sizeof(__m128i) * 2]);
__m128i m3 = loadu(&block[sizeof(__m128i) * 3]);
__m128i t0, t1, t2, t3, tt;
// Round 1. The first round permutes the message words from the original
// input order, into the groups that get mixed in parallel.
t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(2, 0, 2, 0)); // 6 4 2 0
g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
t1 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 3, 1)); // 7 5 3 1
g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
diagonalize(&rows[0], &rows[2], &rows[3]);
t2 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(2, 0, 2, 0)); // 14 12 10 8
t2 = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2, 1, 0, 3)); // 12 10 8 14
g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
t3 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 1, 3, 1)); // 15 13 11 9
t3 = _mm_shuffle_epi32(t3, _MM_SHUFFLE(2, 1, 0, 3)); // 13 11 9 15
g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
undiagonalize(&rows[0], &rows[2], &rows[3]);
m0 = t0;
m1 = t1;
m2 = t2;
m3 = t3;
// Round 2. This round and all following rounds apply a fixed permutation
// to the message words from the round before.
t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
t1 = blend_epi16(tt, t1, 0xCC);
g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
diagonalize(&rows[0], &rows[2], &rows[3]);
t2 = _mm_unpacklo_epi64(m3, m1);
tt = blend_epi16(t2, m2, 0xC0);
t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
t3 = _mm_unpackhi_epi32(m1, m3);
tt = _mm_unpacklo_epi32(m2, t3);
t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
undiagonalize(&rows[0], &rows[2], &rows[3]);
m0 = t0;
m1 = t1;
m2 = t2;
m3 = t3;
// Round 3
t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
t1 = blend_epi16(tt, t1, 0xCC);
g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
diagonalize(&rows[0], &rows[2], &rows[3]);
t2 = _mm_unpacklo_epi64(m3, m1);
tt = blend_epi16(t2, m2, 0xC0);
t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
t3 = _mm_unpackhi_epi32(m1, m3);
tt = _mm_unpacklo_epi32(m2, t3);
t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
undiagonalize(&rows[0], &rows[2], &rows[3]);
m0 = t0;
m1 = t1;
m2 = t2;
m3 = t3;
// Round 4
t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
t1 = blend_epi16(tt, t1, 0xCC);
g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
diagonalize(&rows[0], &rows[2], &rows[3]);
t2 = _mm_unpacklo_epi64(m3, m1);
tt = blend_epi16(t2, m2, 0xC0);
t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
t3 = _mm_unpackhi_epi32(m1, m3);
tt = _mm_unpacklo_epi32(m2, t3);
t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
undiagonalize(&rows[0], &rows[2], &rows[3]);
m0 = t0;
m1 = t1;
m2 = t2;
m3 = t3;
// Round 5
t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
t1 = blend_epi16(tt, t1, 0xCC);
g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
diagonalize(&rows[0], &rows[2], &rows[3]);
t2 = _mm_unpacklo_epi64(m3, m1);
tt = blend_epi16(t2, m2, 0xC0);
t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
t3 = _mm_unpackhi_epi32(m1, m3);
tt = _mm_unpacklo_epi32(m2, t3);
t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
undiagonalize(&rows[0], &rows[2], &rows[3]);
m0 = t0;
m1 = t1;
m2 = t2;
m3 = t3;
// Round 6
t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
t1 = blend_epi16(tt, t1, 0xCC);
g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
diagonalize(&rows[0], &rows[2], &rows[3]);
t2 = _mm_unpacklo_epi64(m3, m1);
tt = blend_epi16(t2, m2, 0xC0);
t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
t3 = _mm_unpackhi_epi32(m1, m3);
tt = _mm_unpacklo_epi32(m2, t3);
t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
undiagonalize(&rows[0], &rows[2], &rows[3]);
m0 = t0;
m1 = t1;
m2 = t2;
m3 = t3;
// Round 7
t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
t1 = blend_epi16(tt, t1, 0xCC);
g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
diagonalize(&rows[0], &rows[2], &rows[3]);
t2 = _mm_unpacklo_epi64(m3, m1);
tt = blend_epi16(t2, m2, 0xC0);
t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
t3 = _mm_unpackhi_epi32(m1, m3);
tt = _mm_unpacklo_epi32(m2, t3);
t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
undiagonalize(&rows[0], &rows[2], &rows[3]);
}
void blake3_compress_in_place_sse2(uint32_t cv[8],
const uint8_t block[BLAKE3_BLOCK_LEN],
uint8_t block_len, uint64_t counter,
uint8_t flags) {
__m128i rows[4];
compress_pre(rows, cv, block, block_len, counter, flags);
storeu(xorv(rows[0], rows[2]), (uint8_t *)&cv[0]);
storeu(xorv(rows[1], rows[3]), (uint8_t *)&cv[4]);
}
void blake3_compress_xof_sse2(const uint32_t cv[8],
const uint8_t block[BLAKE3_BLOCK_LEN],
uint8_t block_len, uint64_t counter,
uint8_t flags, uint8_t out[64]) {
__m128i rows[4];
compress_pre(rows, cv, block, block_len, counter, flags);
storeu(xorv(rows[0], rows[2]), &out[0]);
storeu(xorv(rows[1], rows[3]), &out[16]);
storeu(xorv(rows[2], loadu((uint8_t *)&cv[0])), &out[32]);
storeu(xorv(rows[3], loadu((uint8_t *)&cv[4])), &out[48]);
}
INLINE void round_fn(__m128i v[16], __m128i m[16], size_t r) {
v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][0]]);
v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][2]]);
v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][4]]);
v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][6]]);
v[0] = addv(v[0], v[4]);
v[1] = addv(v[1], v[5]);
v[2] = addv(v[2], v[6]);
v[3] = addv(v[3], v[7]);
v[12] = xorv(v[12], v[0]);
v[13] = xorv(v[13], v[1]);
v[14] = xorv(v[14], v[2]);
v[15] = xorv(v[15], v[3]);
v[12] = rot16(v[12]);
v[13] = rot16(v[13]);
v[14] = rot16(v[14]);
v[15] = rot16(v[15]);
v[8] = addv(v[8], v[12]);
v[9] = addv(v[9], v[13]);
v[10] = addv(v[10], v[14]);
v[11] = addv(v[11], v[15]);
v[4] = xorv(v[4], v[8]);
v[5] = xorv(v[5], v[9]);
v[6] = xorv(v[6], v[10]);
v[7] = xorv(v[7], v[11]);
v[4] = rot12(v[4]);
v[5] = rot12(v[5]);
v[6] = rot12(v[6]);
v[7] = rot12(v[7]);
v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][1]]);
v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][3]]);
v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][5]]);
v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][7]]);
v[0] = addv(v[0], v[4]);
v[1] = addv(v[1], v[5]);
v[2] = addv(v[2], v[6]);
v[3] = addv(v[3], v[7]);
v[12] = xorv(v[12], v[0]);
v[13] = xorv(v[13], v[1]);
v[14] = xorv(v[14], v[2]);
v[15] = xorv(v[15], v[3]);
v[12] = rot8(v[12]);
v[13] = rot8(v[13]);
v[14] = rot8(v[14]);
v[15] = rot8(v[15]);
v[8] = addv(v[8], v[12]);
v[9] = addv(v[9], v[13]);
v[10] = addv(v[10], v[14]);
v[11] = addv(v[11], v[15]);
v[4] = xorv(v[4], v[8]);
v[5] = xorv(v[5], v[9]);
v[6] = xorv(v[6], v[10]);
v[7] = xorv(v[7], v[11]);
v[4] = rot7(v[4]);
v[5] = rot7(v[5]);
v[6] = rot7(v[6]);
v[7] = rot7(v[7]);
v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][8]]);
v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][10]]);
v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][12]]);
v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][14]]);
v[0] = addv(v[0], v[5]);
v[1] = addv(v[1], v[6]);
v[2] = addv(v[2], v[7]);
v[3] = addv(v[3], v[4]);
v[15] = xorv(v[15], v[0]);
v[12] = xorv(v[12], v[1]);
v[13] = xorv(v[13], v[2]);
v[14] = xorv(v[14], v[3]);
v[15] = rot16(v[15]);
v[12] = rot16(v[12]);
v[13] = rot16(v[13]);
v[14] = rot16(v[14]);
v[10] = addv(v[10], v[15]);
v[11] = addv(v[11], v[12]);
v[8] = addv(v[8], v[13]);
v[9] = addv(v[9], v[14]);
v[5] = xorv(v[5], v[10]);
v[6] = xorv(v[6], v[11]);
v[7] = xorv(v[7], v[8]);
v[4] = xorv(v[4], v[9]);
v[5] = rot12(v[5]);
v[6] = rot12(v[6]);
v[7] = rot12(v[7]);
v[4] = rot12(v[4]);
v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][9]]);
v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][11]]);
v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][13]]);
v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][15]]);
v[0] = addv(v[0], v[5]);
v[1] = addv(v[1], v[6]);
v[2] = addv(v[2], v[7]);
v[3] = addv(v[3], v[4]);
v[15] = xorv(v[15], v[0]);
v[12] = xorv(v[12], v[1]);
v[13] = xorv(v[13], v[2]);
v[14] = xorv(v[14], v[3]);
v[15] = rot8(v[15]);
v[12] = rot8(v[12]);
v[13] = rot8(v[13]);
v[14] = rot8(v[14]);
v[10] = addv(v[10], v[15]);
v[11] = addv(v[11], v[12]);
v[8] = addv(v[8], v[13]);
v[9] = addv(v[9], v[14]);
v[5] = xorv(v[5], v[10]);
v[6] = xorv(v[6], v[11]);
v[7] = xorv(v[7], v[8]);
v[4] = xorv(v[4], v[9]);
v[5] = rot7(v[5]);
v[6] = rot7(v[6]);
v[7] = rot7(v[7]);
v[4] = rot7(v[4]);
}
INLINE void transpose_vecs(__m128i vecs[DEGREE]) {
// Interleave 32-bit lanes. The low unpack is lanes 00/11 and the high is
// 22/33. Note that this doesn't split the vector into two lanes, as the
// AVX2 counterparts do.
__m128i ab_01 = _mm_unpacklo_epi32(vecs[0], vecs[1]);
__m128i ab_23 = _mm_unpackhi_epi32(vecs[0], vecs[1]);
__m128i cd_01 = _mm_unpacklo_epi32(vecs[2], vecs[3]);
__m128i cd_23 = _mm_unpackhi_epi32(vecs[2], vecs[3]);
// Interleave 64-bit lanes.
__m128i abcd_0 = _mm_unpacklo_epi64(ab_01, cd_01);
__m128i abcd_1 = _mm_unpackhi_epi64(ab_01, cd_01);
__m128i abcd_2 = _mm_unpacklo_epi64(ab_23, cd_23);
__m128i abcd_3 = _mm_unpackhi_epi64(ab_23, cd_23);
vecs[0] = abcd_0;
vecs[1] = abcd_1;
vecs[2] = abcd_2;
vecs[3] = abcd_3;
}
INLINE void transpose_msg_vecs(const uint8_t *const *inputs,
size_t block_offset, __m128i out[16]) {
out[0] = loadu(&inputs[0][block_offset + 0 * sizeof(__m128i)]);
out[1] = loadu(&inputs[1][block_offset + 0 * sizeof(__m128i)]);
out[2] = loadu(&inputs[2][block_offset + 0 * sizeof(__m128i)]);
out[3] = loadu(&inputs[3][block_offset + 0 * sizeof(__m128i)]);
out[4] = loadu(&inputs[0][block_offset + 1 * sizeof(__m128i)]);
out[5] = loadu(&inputs[1][block_offset + 1 * sizeof(__m128i)]);
out[6] = loadu(&inputs[2][block_offset + 1 * sizeof(__m128i)]);
out[7] = loadu(&inputs[3][block_offset + 1 * sizeof(__m128i)]);
out[8] = loadu(&inputs[0][block_offset + 2 * sizeof(__m128i)]);
out[9] = loadu(&inputs[1][block_offset + 2 * sizeof(__m128i)]);
out[10] = loadu(&inputs[2][block_offset + 2 * sizeof(__m128i)]);
out[11] = loadu(&inputs[3][block_offset + 2 * sizeof(__m128i)]);
out[12] = loadu(&inputs[0][block_offset + 3 * sizeof(__m128i)]);
out[13] = loadu(&inputs[1][block_offset + 3 * sizeof(__m128i)]);
out[14] = loadu(&inputs[2][block_offset + 3 * sizeof(__m128i)]);
out[15] = loadu(&inputs[3][block_offset + 3 * sizeof(__m128i)]);
for (size_t i = 0; i < 4; ++i) {
_mm_prefetch((const void *)&inputs[i][block_offset + 256], _MM_HINT_T0);
}
transpose_vecs(&out[0]);
transpose_vecs(&out[4]);
transpose_vecs(&out[8]);
transpose_vecs(&out[12]);
}
INLINE void load_counters(uint64_t counter, bool increment_counter,
__m128i *out_lo, __m128i *out_hi) {
const __m128i mask = _mm_set1_epi32(-(int32_t)increment_counter);
const __m128i add0 = _mm_set_epi32(3, 2, 1, 0);
const __m128i add1 = _mm_and_si128(mask, add0);
__m128i l = _mm_add_epi32(_mm_set1_epi32((int32_t)counter), add1);
__m128i carry = _mm_cmpgt_epi32(_mm_xor_si128(add1, _mm_set1_epi32(0x80000000)),
_mm_xor_si128( l, _mm_set1_epi32(0x80000000)));
__m128i h = _mm_sub_epi32(_mm_set1_epi32((int32_t)(counter >> 32)), carry);
*out_lo = l;
*out_hi = h;
}
static
void blake3_hash4_sse2(const uint8_t *const *inputs, size_t blocks,
const uint32_t key[8], uint64_t counter,
bool increment_counter, uint8_t flags,
uint8_t flags_start, uint8_t flags_end, uint8_t *out) {
__m128i h_vecs[8] = {
set1(key[0]), set1(key[1]), set1(key[2]), set1(key[3]),
set1(key[4]), set1(key[5]), set1(key[6]), set1(key[7]),
};
__m128i counter_low_vec, counter_high_vec;
load_counters(counter, increment_counter, &counter_low_vec,
&counter_high_vec);
uint8_t block_flags = flags | flags_start;
for (size_t block = 0; block < blocks; block++) {
if (block + 1 == blocks) {
block_flags |= flags_end;
}
__m128i block_len_vec = set1(BLAKE3_BLOCK_LEN);
__m128i block_flags_vec = set1(block_flags);
__m128i msg_vecs[16];
transpose_msg_vecs(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs);
__m128i v[16] = {
h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3],
h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7],
set1(IV[0]), set1(IV[1]), set1(IV[2]), set1(IV[3]),
counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec,
};
round_fn(v, msg_vecs, 0);
round_fn(v, msg_vecs, 1);
round_fn(v, msg_vecs, 2);
round_fn(v, msg_vecs, 3);
round_fn(v, msg_vecs, 4);
round_fn(v, msg_vecs, 5);
round_fn(v, msg_vecs, 6);
h_vecs[0] = xorv(v[0], v[8]);
h_vecs[1] = xorv(v[1], v[9]);
h_vecs[2] = xorv(v[2], v[10]);
h_vecs[3] = xorv(v[3], v[11]);
h_vecs[4] = xorv(v[4], v[12]);
h_vecs[5] = xorv(v[5], v[13]);
h_vecs[6] = xorv(v[6], v[14]);
h_vecs[7] = xorv(v[7], v[15]);
block_flags = flags;
}
transpose_vecs(&h_vecs[0]);
transpose_vecs(&h_vecs[4]);
// The first four vecs now contain the first half of each output, and the
// second four vecs contain the second half of each output.
storeu(h_vecs[0], &out[0 * sizeof(__m128i)]);
storeu(h_vecs[4], &out[1 * sizeof(__m128i)]);
storeu(h_vecs[1], &out[2 * sizeof(__m128i)]);
storeu(h_vecs[5], &out[3 * sizeof(__m128i)]);
storeu(h_vecs[2], &out[4 * sizeof(__m128i)]);
storeu(h_vecs[6], &out[5 * sizeof(__m128i)]);
storeu(h_vecs[3], &out[6 * sizeof(__m128i)]);
storeu(h_vecs[7], &out[7 * sizeof(__m128i)]);
}
INLINE void hash_one_sse2(const uint8_t *input, size_t blocks,
const uint32_t key[8], uint64_t counter,
uint8_t flags, uint8_t flags_start,
uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN]) {
uint32_t cv[8];
memcpy(cv, key, BLAKE3_KEY_LEN);
uint8_t block_flags = flags | flags_start;
while (blocks > 0) {
if (blocks == 1) {
block_flags |= flags_end;
}
blake3_compress_in_place_sse2(cv, input, BLAKE3_BLOCK_LEN, counter,
block_flags);
input = &input[BLAKE3_BLOCK_LEN];
blocks -= 1;
block_flags = flags;
}
memcpy(out, cv, BLAKE3_OUT_LEN);
}
void blake3_hash_many_sse2(const uint8_t *const *inputs, size_t num_inputs,
size_t blocks, const uint32_t key[8],
uint64_t counter, bool increment_counter,
uint8_t flags, uint8_t flags_start,
uint8_t flags_end, uint8_t *out) {
while (num_inputs >= DEGREE) {
blake3_hash4_sse2(inputs, blocks, key, counter, increment_counter, flags,
flags_start, flags_end, out);
if (increment_counter) {
counter += DEGREE;
}
inputs += DEGREE;
num_inputs -= DEGREE;
out = &out[DEGREE * BLAKE3_OUT_LEN];
}
while (num_inputs > 0) {
hash_one_sse2(inputs[0], blocks, key, counter, flags, flags_start,
flags_end, out);
if (increment_counter) {
counter += 1;
}
inputs += 1;
num_inputs -= 1;
out = &out[BLAKE3_OUT_LEN];
}
}
src/util/blake3/blake3_sse2_x86-64_unix.S
+2299
View File
File diff suppressed because it is too large. Load diff
src/util/blake3/blake3_sse2_x86-64_windows_gnu.S
+2332
View File
File diff suppressed because it is too large. Load diff
src/util/blake3/blake3_sse2_x86-64_windows_msvc.masm
+2350
View File
File diff suppressed because it is too large. Load diff
src/util/blake3/blake3_sse41.c
+560
View File
@@ -0,0 +1,560 @@
#include "blake3_impl.h"
#include <immintrin.h>
#define DEGREE 4
#define _mm_shuffle_ps2(a, b, c) \
(_mm_castps_si128( \
_mm_shuffle_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b), (c))))
INLINE __m128i loadu(const uint8_t src[16]) {
return _mm_loadu_si128((const __m128i *)src);
}
INLINE void storeu(__m128i src, uint8_t dest[16]) {
_mm_storeu_si128((__m128i *)dest, src);
}
INLINE __m128i addv(__m128i a, __m128i b) { return _mm_add_epi32(a, b); }
// Note that clang-format doesn't like the name "xor" for some reason.
INLINE __m128i xorv(__m128i a, __m128i b) { return _mm_xor_si128(a, b); }
INLINE __m128i set1(uint32_t x) { return _mm_set1_epi32((int32_t)x); }
INLINE __m128i set4(uint32_t a, uint32_t b, uint32_t c, uint32_t d) {
return _mm_setr_epi32((int32_t)a, (int32_t)b, (int32_t)c, (int32_t)d);
}
INLINE __m128i rot16(__m128i x) {
return _mm_shuffle_epi8(
x, _mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2));
}
INLINE __m128i rot12(__m128i x) {
return xorv(_mm_srli_epi32(x, 12), _mm_slli_epi32(x, 32 - 12));
}
INLINE __m128i rot8(__m128i x) {
return _mm_shuffle_epi8(
x, _mm_set_epi8(12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1));
}
INLINE __m128i rot7(__m128i x) {
return xorv(_mm_srli_epi32(x, 7), _mm_slli_epi32(x, 32 - 7));
}
INLINE void g1(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3,
__m128i m) {
*row0 = addv(addv(*row0, m), *row1);
*row3 = xorv(*row3, *row0);
*row3 = rot16(*row3);
*row2 = addv(*row2, *row3);
*row1 = xorv(*row1, *row2);
*row1 = rot12(*row1);
}
INLINE void g2(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3,
__m128i m) {
*row0 = addv(addv(*row0, m), *row1);
*row3 = xorv(*row3, *row0);
*row3 = rot8(*row3);
*row2 = addv(*row2, *row3);
*row1 = xorv(*row1, *row2);
*row1 = rot7(*row1);
}
// Note the optimization here of leaving row1 as the unrotated row, rather than
// row0. All the message loads below are adjusted to compensate for this. See
// discussion at https://github.com/sneves/blake2-avx2/pull/4
INLINE void diagonalize(__m128i *row0, __m128i *row2, __m128i *row3) {
*row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(2, 1, 0, 3));
*row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2));
*row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(0, 3, 2, 1));
}
INLINE void undiagonalize(__m128i *row0, __m128i *row2, __m128i *row3) {
*row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(0, 3, 2, 1));
*row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2));
*row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(2, 1, 0, 3));
}
INLINE void compress_pre(__m128i rows[4], const uint32_t cv[8],
const uint8_t block[BLAKE3_BLOCK_LEN],
uint8_t block_len, uint64_t counter, uint8_t flags) {
rows[0] = loadu((uint8_t *)&cv[0]);
rows[1] = loadu((uint8_t *)&cv[4]);
rows[2] = set4(IV[0], IV[1], IV[2], IV[3]);
rows[3] = set4(counter_low(counter), counter_high(counter),
(uint32_t)block_len, (uint32_t)flags);
__m128i m0 = loadu(&block[sizeof(__m128i) * 0]);
__m128i m1 = loadu(&block[sizeof(__m128i) * 1]);
__m128i m2 = loadu(&block[sizeof(__m128i) * 2]);
__m128i m3 = loadu(&block[sizeof(__m128i) * 3]);
__m128i t0, t1, t2, t3, tt;
// Round 1. The first round permutes the message words from the original
// input order, into the groups that get mixed in parallel.
t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(2, 0, 2, 0)); // 6 4 2 0
g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
t1 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 3, 1)); // 7 5 3 1
g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
diagonalize(&rows[0], &rows[2], &rows[3]);
t2 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(2, 0, 2, 0)); // 14 12 10 8
t2 = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2, 1, 0, 3)); // 12 10 8 14
g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
t3 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 1, 3, 1)); // 15 13 11 9
t3 = _mm_shuffle_epi32(t3, _MM_SHUFFLE(2, 1, 0, 3)); // 13 11 9 15
g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
undiagonalize(&rows[0], &rows[2], &rows[3]);
m0 = t0;
m1 = t1;
m2 = t2;
m3 = t3;
// Round 2. This round and all following rounds apply a fixed permutation
// to the message words from the round before.
t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
t1 = _mm_blend_epi16(tt, t1, 0xCC);
g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
diagonalize(&rows[0], &rows[2], &rows[3]);
t2 = _mm_unpacklo_epi64(m3, m1);
tt = _mm_blend_epi16(t2, m2, 0xC0);
t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
t3 = _mm_unpackhi_epi32(m1, m3);
tt = _mm_unpacklo_epi32(m2, t3);
t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
undiagonalize(&rows[0], &rows[2], &rows[3]);
m0 = t0;
m1 = t1;
m2 = t2;
m3 = t3;
// Round 3
t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
t1 = _mm_blend_epi16(tt, t1, 0xCC);
g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
diagonalize(&rows[0], &rows[2], &rows[3]);
t2 = _mm_unpacklo_epi64(m3, m1);
tt = _mm_blend_epi16(t2, m2, 0xC0);
t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
t3 = _mm_unpackhi_epi32(m1, m3);
tt = _mm_unpacklo_epi32(m2, t3);
t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
undiagonalize(&rows[0], &rows[2], &rows[3]);
m0 = t0;
m1 = t1;
m2 = t2;
m3 = t3;
// Round 4
t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
t1 = _mm_blend_epi16(tt, t1, 0xCC);
g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
diagonalize(&rows[0], &rows[2], &rows[3]);
t2 = _mm_unpacklo_epi64(m3, m1);
tt = _mm_blend_epi16(t2, m2, 0xC0);
t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
t3 = _mm_unpackhi_epi32(m1, m3);
tt = _mm_unpacklo_epi32(m2, t3);
t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
undiagonalize(&rows[0], &rows[2], &rows[3]);
m0 = t0;
m1 = t1;
m2 = t2;
m3 = t3;
// Round 5
t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
t1 = _mm_blend_epi16(tt, t1, 0xCC);
g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
diagonalize(&rows[0], &rows[2], &rows[3]);
t2 = _mm_unpacklo_epi64(m3, m1);
tt = _mm_blend_epi16(t2, m2, 0xC0);
t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
t3 = _mm_unpackhi_epi32(m1, m3);
tt = _mm_unpacklo_epi32(m2, t3);
t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
undiagonalize(&rows[0], &rows[2], &rows[3]);
m0 = t0;
m1 = t1;
m2 = t2;
m3 = t3;
// Round 6
t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
t1 = _mm_blend_epi16(tt, t1, 0xCC);
g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
diagonalize(&rows[0], &rows[2], &rows[3]);
t2 = _mm_unpacklo_epi64(m3, m1);
tt = _mm_blend_epi16(t2, m2, 0xC0);
t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
t3 = _mm_unpackhi_epi32(m1, m3);
tt = _mm_unpacklo_epi32(m2, t3);
t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
undiagonalize(&rows[0], &rows[2], &rows[3]);
m0 = t0;
m1 = t1;
m2 = t2;
m3 = t3;
// Round 7
t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
t1 = _mm_blend_epi16(tt, t1, 0xCC);
g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
diagonalize(&rows[0], &rows[2], &rows[3]);
t2 = _mm_unpacklo_epi64(m3, m1);
tt = _mm_blend_epi16(t2, m2, 0xC0);
t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
t3 = _mm_unpackhi_epi32(m1, m3);
tt = _mm_unpacklo_epi32(m2, t3);
t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
undiagonalize(&rows[0], &rows[2], &rows[3]);
}
void blake3_compress_in_place_sse41(uint32_t cv[8],
const uint8_t block[BLAKE3_BLOCK_LEN],
uint8_t block_len, uint64_t counter,
uint8_t flags) {
__m128i rows[4];
compress_pre(rows, cv, block, block_len, counter, flags);
storeu(xorv(rows[0], rows[2]), (uint8_t *)&cv[0]);
storeu(xorv(rows[1], rows[3]), (uint8_t *)&cv[4]);
}
void blake3_compress_xof_sse41(const uint32_t cv[8],
const uint8_t block[BLAKE3_BLOCK_LEN],
uint8_t block_len, uint64_t counter,
uint8_t flags, uint8_t out[64]) {
__m128i rows[4];
compress_pre(rows, cv, block, block_len, counter, flags);
storeu(xorv(rows[0], rows[2]), &out[0]);
storeu(xorv(rows[1], rows[3]), &out[16]);
storeu(xorv(rows[2], loadu((uint8_t *)&cv[0])), &out[32]);
storeu(xorv(rows[3], loadu((uint8_t *)&cv[4])), &out[48]);
}
INLINE void round_fn(__m128i v[16], __m128i m[16], size_t r) {
v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][0]]);
v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][2]]);
v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][4]]);
v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][6]]);
v[0] = addv(v[0], v[4]);
v[1] = addv(v[1], v[5]);
v[2] = addv(v[2], v[6]);
v[3] = addv(v[3], v[7]);
v[12] = xorv(v[12], v[0]);
v[13] = xorv(v[13], v[1]);
v[14] = xorv(v[14], v[2]);
v[15] = xorv(v[15], v[3]);
v[12] = rot16(v[12]);
v[13] = rot16(v[13]);
v[14] = rot16(v[14]);
v[15] = rot16(v[15]);
v[8] = addv(v[8], v[12]);
v[9] = addv(v[9], v[13]);
v[10] = addv(v[10], v[14]);
v[11] = addv(v[11], v[15]);
v[4] = xorv(v[4], v[8]);
v[5] = xorv(v[5], v[9]);
v[6] = xorv(v[6], v[10]);
v[7] = xorv(v[7], v[11]);
v[4] = rot12(v[4]);
v[5] = rot12(v[5]);
v[6] = rot12(v[6]);
v[7] = rot12(v[7]);
v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][1]]);
v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][3]]);
v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][5]]);
v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][7]]);
v[0] = addv(v[0], v[4]);
v[1] = addv(v[1], v[5]);
v[2] = addv(v[2], v[6]);
v[3] = addv(v[3], v[7]);
v[12] = xorv(v[12], v[0]);
v[13] = xorv(v[13], v[1]);
v[14] = xorv(v[14], v[2]);
v[15] = xorv(v[15], v[3]);
v[12] = rot8(v[12]);
v[13] = rot8(v[13]);
v[14] = rot8(v[14]);
v[15] = rot8(v[15]);
v[8] = addv(v[8], v[12]);
v[9] = addv(v[9], v[13]);
v[10] = addv(v[10], v[14]);
v[11] = addv(v[11], v[15]);
v[4] = xorv(v[4], v[8]);
v[5] = xorv(v[5], v[9]);
v[6] = xorv(v[6], v[10]);
v[7] = xorv(v[7], v[11]);
v[4] = rot7(v[4]);
v[5] = rot7(v[5]);
v[6] = rot7(v[6]);
v[7] = rot7(v[7]);
v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][8]]);
v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][10]]);
v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][12]]);
v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][14]]);
v[0] = addv(v[0], v[5]);
v[1] = addv(v[1], v[6]);
v[2] = addv(v[2], v[7]);
v[3] = addv(v[3], v[4]);
v[15] = xorv(v[15], v[0]);
v[12] = xorv(v[12], v[1]);
v[13] = xorv(v[13], v[2]);
v[14] = xorv(v[14], v[3]);
v[15] = rot16(v[15]);
v[12] = rot16(v[12]);
v[13] = rot16(v[13]);
v[14] = rot16(v[14]);
v[10] = addv(v[10], v[15]);
v[11] = addv(v[11], v[12]);
v[8] = addv(v[8], v[13]);
v[9] = addv(v[9], v[14]);
v[5] = xorv(v[5], v[10]);
v[6] = xorv(v[6], v[11]);
v[7] = xorv(v[7], v[8]);
v[4] = xorv(v[4], v[9]);
v[5] = rot12(v[5]);
v[6] = rot12(v[6]);
v[7] = rot12(v[7]);
v[4] = rot12(v[4]);
v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][9]]);
v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][11]]);
v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][13]]);
v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][15]]);
v[0] = addv(v[0], v[5]);
v[1] = addv(v[1], v[6]);
v[2] = addv(v[2], v[7]);
v[3] = addv(v[3], v[4]);
v[15] = xorv(v[15], v[0]);
v[12] = xorv(v[12], v[1]);
v[13] = xorv(v[13], v[2]);
v[14] = xorv(v[14], v[3]);
v[15] = rot8(v[15]);
v[12] = rot8(v[12]);
v[13] = rot8(v[13]);
v[14] = rot8(v[14]);
v[10] = addv(v[10], v[15]);
v[11] = addv(v[11], v[12]);
v[8] = addv(v[8], v[13]);
v[9] = addv(v[9], v[14]);
v[5] = xorv(v[5], v[10]);
v[6] = xorv(v[6], v[11]);
v[7] = xorv(v[7], v[8]);
v[4] = xorv(v[4], v[9]);
v[5] = rot7(v[5]);
v[6] = rot7(v[6]);
v[7] = rot7(v[7]);
v[4] = rot7(v[4]);
}
INLINE void transpose_vecs(__m128i vecs[DEGREE]) {
// Interleave 32-bit lanes. The low unpack is lanes 00/11 and the high is
// 22/33. Note that this doesn't split the vector into two lanes, as the
// AVX2 counterparts do.
__m128i ab_01 = _mm_unpacklo_epi32(vecs[0], vecs[1]);
__m128i ab_23 = _mm_unpackhi_epi32(vecs[0], vecs[1]);
__m128i cd_01 = _mm_unpacklo_epi32(vecs[2], vecs[3]);
__m128i cd_23 = _mm_unpackhi_epi32(vecs[2], vecs[3]);
// Interleave 64-bit lanes.
__m128i abcd_0 = _mm_unpacklo_epi64(ab_01, cd_01);
__m128i abcd_1 = _mm_unpackhi_epi64(ab_01, cd_01);
__m128i abcd_2 = _mm_unpacklo_epi64(ab_23, cd_23);
__m128i abcd_3 = _mm_unpackhi_epi64(ab_23, cd_23);
vecs[0] = abcd_0;
vecs[1] = abcd_1;
vecs[2] = abcd_2;
vecs[3] = abcd_3;
}
INLINE void transpose_msg_vecs(const uint8_t *const *inputs,
size_t block_offset, __m128i out[16]) {
out[0] = loadu(&inputs[0][block_offset + 0 * sizeof(__m128i)]);
out[1] = loadu(&inputs[1][block_offset + 0 * sizeof(__m128i)]);
out[2] = loadu(&inputs[2][block_offset + 0 * sizeof(__m128i)]);
out[3] = loadu(&inputs[3][block_offset + 0 * sizeof(__m128i)]);
out[4] = loadu(&inputs[0][block_offset + 1 * sizeof(__m128i)]);
out[5] = loadu(&inputs[1][block_offset + 1 * sizeof(__m128i)]);
out[6] = loadu(&inputs[2][block_offset + 1 * sizeof(__m128i)]);
out[7] = loadu(&inputs[3][block_offset + 1 * sizeof(__m128i)]);
out[8] = loadu(&inputs[0][block_offset + 2 * sizeof(__m128i)]);
out[9] = loadu(&inputs[1][block_offset + 2 * sizeof(__m128i)]);
out[10] = loadu(&inputs[2][block_offset + 2 * sizeof(__m128i)]);
out[11] = loadu(&inputs[3][block_offset + 2 * sizeof(__m128i)]);
out[12] = loadu(&inputs[0][block_offset + 3 * sizeof(__m128i)]);
out[13] = loadu(&inputs[1][block_offset + 3 * sizeof(__m128i)]);
out[14] = loadu(&inputs[2][block_offset + 3 * sizeof(__m128i)]);
out[15] = loadu(&inputs[3][block_offset + 3 * sizeof(__m128i)]);
for (size_t i = 0; i < 4; ++i) {
_mm_prefetch((const void *)&inputs[i][block_offset + 256], _MM_HINT_T0);
}
transpose_vecs(&out[0]);
transpose_vecs(&out[4]);
transpose_vecs(&out[8]);
transpose_vecs(&out[12]);
}
INLINE void load_counters(uint64_t counter, bool increment_counter,
__m128i *out_lo, __m128i *out_hi) {
const __m128i mask = _mm_set1_epi32(-(int32_t)increment_counter);
const __m128i add0 = _mm_set_epi32(3, 2, 1, 0);
const __m128i add1 = _mm_and_si128(mask, add0);
__m128i l = _mm_add_epi32(_mm_set1_epi32((int32_t)counter), add1);
__m128i carry = _mm_cmpgt_epi32(_mm_xor_si128(add1, _mm_set1_epi32(0x80000000)),
_mm_xor_si128( l, _mm_set1_epi32(0x80000000)));
__m128i h = _mm_sub_epi32(_mm_set1_epi32((int32_t)(counter >> 32)), carry);
*out_lo = l;
*out_hi = h;
}
static
void blake3_hash4_sse41(const uint8_t *const *inputs, size_t blocks,
const uint32_t key[8], uint64_t counter,
bool increment_counter, uint8_t flags,
uint8_t flags_start, uint8_t flags_end, uint8_t *out) {
__m128i h_vecs[8] = {
set1(key[0]), set1(key[1]), set1(key[2]), set1(key[3]),
set1(key[4]), set1(key[5]), set1(key[6]), set1(key[7]),
};
__m128i counter_low_vec, counter_high_vec;
load_counters(counter, increment_counter, &counter_low_vec,
&counter_high_vec);
uint8_t block_flags = flags | flags_start;
for (size_t block = 0; block < blocks; block++) {
if (block + 1 == blocks) {
block_flags |= flags_end;
}
__m128i block_len_vec = set1(BLAKE3_BLOCK_LEN);
__m128i block_flags_vec = set1(block_flags);
__m128i msg_vecs[16];
transpose_msg_vecs(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs);
__m128i v[16] = {
h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3],
h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7],
set1(IV[0]), set1(IV[1]), set1(IV[2]), set1(IV[3]),
counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec,
};
round_fn(v, msg_vecs, 0);
round_fn(v, msg_vecs, 1);
round_fn(v, msg_vecs, 2);
round_fn(v, msg_vecs, 3);
round_fn(v, msg_vecs, 4);
round_fn(v, msg_vecs, 5);
round_fn(v, msg_vecs, 6);
h_vecs[0] = xorv(v[0], v[8]);
h_vecs[1] = xorv(v[1], v[9]);
h_vecs[2] = xorv(v[2], v[10]);
h_vecs[3] = xorv(v[3], v[11]);
h_vecs[4] = xorv(v[4], v[12]);
h_vecs[5] = xorv(v[5], v[13]);
h_vecs[6] = xorv(v[6], v[14]);
h_vecs[7] = xorv(v[7], v[15]);
block_flags = flags;
}
transpose_vecs(&h_vecs[0]);
transpose_vecs(&h_vecs[4]);
// The first four vecs now contain the first half of each output, and the
// second four vecs contain the second half of each output.
storeu(h_vecs[0], &out[0 * sizeof(__m128i)]);
storeu(h_vecs[4], &out[1 * sizeof(__m128i)]);
storeu(h_vecs[1], &out[2 * sizeof(__m128i)]);
storeu(h_vecs[5], &out[3 * sizeof(__m128i)]);
storeu(h_vecs[2], &out[4 * sizeof(__m128i)]);
storeu(h_vecs[6], &out[5 * sizeof(__m128i)]);
storeu(h_vecs[3], &out[6 * sizeof(__m128i)]);
storeu(h_vecs[7], &out[7 * sizeof(__m128i)]);
}
INLINE void hash_one_sse41(const uint8_t *input, size_t blocks,
const uint32_t key[8], uint64_t counter,
uint8_t flags, uint8_t flags_start,
uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN]) {
uint32_t cv[8];
memcpy(cv, key, BLAKE3_KEY_LEN);
uint8_t block_flags = flags | flags_start;
while (blocks > 0) {
if (blocks == 1) {
block_flags |= flags_end;
}
blake3_compress_in_place_sse41(cv, input, BLAKE3_BLOCK_LEN, counter,
block_flags);
input = &input[BLAKE3_BLOCK_LEN];
blocks -= 1;
block_flags = flags;
}
memcpy(out, cv, BLAKE3_OUT_LEN);
}
void blake3_hash_many_sse41(const uint8_t *const *inputs, size_t num_inputs,
size_t blocks, const uint32_t key[8],
uint64_t counter, bool increment_counter,
uint8_t flags, uint8_t flags_start,
uint8_t flags_end, uint8_t *out) {
while (num_inputs >= DEGREE) {
blake3_hash4_sse41(inputs, blocks, key, counter, increment_counter, flags,
flags_start, flags_end, out);
if (increment_counter) {
counter += DEGREE;
}
inputs += DEGREE;
num_inputs -= DEGREE;
out = &out[DEGREE * BLAKE3_OUT_LEN];
}
while (num_inputs > 0) {
hash_one_sse41(inputs[0], blocks, key, counter, flags, flags_start,
flags_end, out);
if (increment_counter) {
counter += 1;
}
inputs += 1;
num_inputs -= 1;
out = &out[BLAKE3_OUT_LEN];
}
}
src/util/blake3/blake3_sse41_x86-64_unix.S
+2036
View File
File diff suppressed because it is too large. Load diff
src/util/blake3/blake3_sse41_x86-64_windows_gnu.S
+2069
View File
File diff suppressed because it is too large. Load diff
src/util/blake3/blake3_sse41_x86-64_windows_msvc.masm
+2089
View File
File diff suppressed because it is too large. Load diff
src/util/blake3/mesa_blake3_visibility.h
+5
View File
@@ -0,0 +1,5 @@
#ifdef __APPLE__
#define HIDDEN .private_extern
#else
#define HIDDEN .hidden
#endif
src/util/blake3/meson.build
+52
View File
@@ -0,0 +1,52 @@
files_blake3 = [
'blake3.c',
'blake3_dispatch.c',
'blake3_portable.c'
]
blake3_defs = []
is_windows = host_machine.system() == 'windows'
is_msvc = meson.get_compiler('c').get_id() == 'msvc'
cpu_family = host_machine.cpu_family()
blake3_x86_no_simd_defs = ['-DBLAKE3_NO_SSE2', '-DBLAKE3_NO_SSE41', '-DBLAKE3_NO_AVX2', '-DBLAKE3_NO_AVX512']
if cpu_family == 'x86_64'
if is_windows
if is_msvc
# An up-to-date version of Meson, not using the VS backend is needed.
# See https://github.com/mesonbuild/meson/issues/11653
if meson.backend() == 'ninja' and add_languages('masm', required : false)
files_blake3 += ['blake3_sse2_x86-64_windows_msvc.masm', 'blake3_sse41_x86-64_windows_msvc.masm', 'blake3_avx2_x86-64_windows_msvc.masm', 'blake3_avx512_x86-64_windows_msvc.masm']
else
blake3_defs += blake3_x86_no_simd_defs
endif
else
files_blake3 += ['blake3_sse2_x86-64_windows_gnu.S', 'blake3_sse41_x86-64_windows_gnu.S', 'blake3_avx2_x86-64_windows_gnu.S', 'blake3_avx512_x86-64_windows_gnu.S']
endif
# Disable blake assembly for x32, x86-64 with 32-bit pointers.
# GNU triplet is x86_64-linux-gnux32
elif cc.sizeof('void *') == 4
blake3_defs += blake3_x86_no_simd_defs
else
files_blake3 += ['blake3_sse2_x86-64_unix.S', 'blake3_sse41_x86-64_unix.S', 'blake3_avx2_x86-64_unix.S', 'blake3_avx512_x86-64_unix.S']
endif
elif cpu_family == 'x86'
# There are no assembly versions for 32-bit x86. Compiling the C versions require a different compilation flag per
# file, which is not well supported by Meson. Leave SIMD support out for now.
blake3_defs += blake3_x86_no_simd_defs
elif cpu_family == 'aarch64' and target_machine.endian() == 'little'
files_blake3 += ['blake3_neon.c']
endif
blake3 = static_library(
'blake3',
files_blake3,
c_args : blake3_defs + cc.get_supported_arguments('-Wno-error'),
gnu_symbol_visibility : 'hidden',
build_by_default : false,
)
idep_blake3 = declare_dependency(
link_with : blake3,
)
src/util/blend.h
+197
View File
@@ -0,0 +1,197 @@
/*
* Copyright 2023 Valve Corporation
* Copyright 2007 VMware, Inc.
* SPDX-License-Identifier: MIT
*/
#ifndef UTIL_BLEND_H
#define UTIL_BLEND_H
#include <stdbool.h>
#include "util/macros.h"
#define PIPE_BLENDFACTOR_INVERT_BIT (0x10)
enum pipe_blendfactor {
PIPE_BLENDFACTOR_ONE = 1,
PIPE_BLENDFACTOR_SRC_COLOR,
PIPE_BLENDFACTOR_SRC_ALPHA,
PIPE_BLENDFACTOR_DST_ALPHA,
PIPE_BLENDFACTOR_DST_COLOR,
PIPE_BLENDFACTOR_SRC_ALPHA_SATURATE,
PIPE_BLENDFACTOR_CONST_COLOR,
PIPE_BLENDFACTOR_CONST_ALPHA,
PIPE_BLENDFACTOR_SRC1_COLOR,
PIPE_BLENDFACTOR_SRC1_ALPHA,
PIPE_BLENDFACTOR_ZERO = PIPE_BLENDFACTOR_INVERT_BIT | PIPE_BLENDFACTOR_ONE,
PIPE_BLENDFACTOR_INV_SRC_COLOR,
PIPE_BLENDFACTOR_INV_SRC_ALPHA,
PIPE_BLENDFACTOR_INV_DST_ALPHA,
PIPE_BLENDFACTOR_INV_DST_COLOR,
/* Intentionally weird wrapping due to Gallium trace parsing this file */
PIPE_BLENDFACTOR_INV_CONST_COLOR = PIPE_BLENDFACTOR_INVERT_BIT
| PIPE_BLENDFACTOR_CONST_COLOR,
PIPE_BLENDFACTOR_INV_CONST_ALPHA,
PIPE_BLENDFACTOR_INV_SRC1_COLOR,
PIPE_BLENDFACTOR_INV_SRC1_ALPHA,
};
static inline bool
util_blendfactor_is_inverted(enum pipe_blendfactor factor)
{
/* By construction of the enum */
return (factor & PIPE_BLENDFACTOR_INVERT_BIT);
}
static inline enum pipe_blendfactor
util_blendfactor_without_invert(enum pipe_blendfactor factor)
{
/* By construction of the enum */
return (enum pipe_blendfactor)(factor & ~PIPE_BLENDFACTOR_INVERT_BIT);
}
/**
* Color blend factor enums are treated as their alpha counterpart when used
* as the alpha blend factor. This helper just converts from color to alpha.
*/
static inline enum pipe_blendfactor
util_blendfactor_to_alpha(enum pipe_blendfactor factor)
{
switch (factor) {
#define UTIL_BLENDFACTOR_TO_ALPHA_CASE(x) \
case PIPE_BLENDFACTOR_##x##_COLOR: \
return PIPE_BLENDFACTOR_##x##_ALPHA; \
case PIPE_BLENDFACTOR_INV_##x##_COLOR: \
return PIPE_BLENDFACTOR_INV_##x##_ALPHA;
UTIL_BLENDFACTOR_TO_ALPHA_CASE(SRC)
UTIL_BLENDFACTOR_TO_ALPHA_CASE(DST)
UTIL_BLENDFACTOR_TO_ALPHA_CASE(CONST)
UTIL_BLENDFACTOR_TO_ALPHA_CASE(SRC1)
#undef UTIL_BLENDFACTOR_TO_ALPHA_CASE
case PIPE_BLENDFACTOR_ONE:
case PIPE_BLENDFACTOR_SRC_ALPHA:
case PIPE_BLENDFACTOR_DST_ALPHA:
case PIPE_BLENDFACTOR_SRC_ALPHA_SATURATE:
case PIPE_BLENDFACTOR_CONST_ALPHA:
case PIPE_BLENDFACTOR_SRC1_ALPHA:
case PIPE_BLENDFACTOR_ZERO:
case PIPE_BLENDFACTOR_INV_SRC_ALPHA:
case PIPE_BLENDFACTOR_INV_DST_ALPHA:
case PIPE_BLENDFACTOR_INV_CONST_ALPHA:
case PIPE_BLENDFACTOR_INV_SRC1_ALPHA:
return factor;
}
UNREACHABLE("Invalid blend factor");
}
enum pipe_blend_func {
PIPE_BLEND_ADD,
PIPE_BLEND_SUBTRACT,
PIPE_BLEND_REVERSE_SUBTRACT,
PIPE_BLEND_MIN,
PIPE_BLEND_MAX,
};
enum pipe_logicop {
PIPE_LOGICOP_CLEAR,
PIPE_LOGICOP_NOR,
PIPE_LOGICOP_AND_INVERTED,
PIPE_LOGICOP_COPY_INVERTED,
PIPE_LOGICOP_AND_REVERSE,
PIPE_LOGICOP_INVERT,
PIPE_LOGICOP_XOR,
PIPE_LOGICOP_NAND,
PIPE_LOGICOP_AND,
PIPE_LOGICOP_EQUIV,
PIPE_LOGICOP_NOOP,
PIPE_LOGICOP_OR_INVERTED,
PIPE_LOGICOP_COPY,
PIPE_LOGICOP_OR_REVERSE,
PIPE_LOGICOP_OR,
PIPE_LOGICOP_SET,
};
enum pipe_advanced_blend_mode {
PIPE_ADVANCED_BLEND_NONE = 0,
PIPE_ADVANCED_BLEND_MULTIPLY,
PIPE_ADVANCED_BLEND_SCREEN,
PIPE_ADVANCED_BLEND_OVERLAY,
PIPE_ADVANCED_BLEND_DARKEN,
PIPE_ADVANCED_BLEND_LIGHTEN,
PIPE_ADVANCED_BLEND_COLORDODGE,
PIPE_ADVANCED_BLEND_COLORBURN,
PIPE_ADVANCED_BLEND_HARDLIGHT,
PIPE_ADVANCED_BLEND_SOFTLIGHT,
PIPE_ADVANCED_BLEND_DIFFERENCE,
PIPE_ADVANCED_BLEND_EXCLUSION,
PIPE_ADVANCED_BLEND_HSL_HUE,
PIPE_ADVANCED_BLEND_HSL_SATURATION,
PIPE_ADVANCED_BLEND_HSL_COLOR,
PIPE_ADVANCED_BLEND_HSL_LUMINOSITY,
/* Extended blend modes */
PIPE_ADVANCED_BLEND_SRC,
PIPE_ADVANCED_BLEND_DST,
PIPE_ADVANCED_BLEND_SRC_OVER,
PIPE_ADVANCED_BLEND_DST_OVER,
PIPE_ADVANCED_BLEND_SRC_IN,
PIPE_ADVANCED_BLEND_DST_IN,
PIPE_ADVANCED_BLEND_SRC_OUT,
PIPE_ADVANCED_BLEND_DST_OUT,
PIPE_ADVANCED_BLEND_SRC_ATOP,
PIPE_ADVANCED_BLEND_DST_ATOP,
PIPE_ADVANCED_BLEND_XOR,
PIPE_ADVANCED_BLEND_INVERT,
PIPE_ADVANCED_BLEND_INVERT_RGB,
PIPE_ADVANCED_BLEND_LINEARDODGE,
PIPE_ADVANCED_BLEND_LINEARBURN,
PIPE_ADVANCED_BLEND_VIVIDLIGHT,
PIPE_ADVANCED_BLEND_LINEARLIGHT,
PIPE_ADVANCED_BLEND_PINLIGHT,
PIPE_ADVANCED_BLEND_HARDMIX,
PIPE_ADVANCED_BLEND_PLUS,
PIPE_ADVANCED_BLEND_PLUS_CLAMPED,
PIPE_ADVANCED_BLEND_PLUS_CLAMPED_ALPHA,
PIPE_ADVANCED_BLEND_PLUS_DARKER,
PIPE_ADVANCED_BLEND_MINUS,
PIPE_ADVANCED_BLEND_MINUS_CLAMPED,
PIPE_ADVANCED_BLEND_CONTRAST,
PIPE_ADVANCED_BLEND_INVERT_OVG,
PIPE_ADVANCED_BLEND_RED,
PIPE_ADVANCED_BLEND_GREEN,
PIPE_ADVANCED_BLEND_BLUE,
};
enum pipe_blend_overlap_mode {
PIPE_BLEND_OVERLAP_UNCORRELATED = 0,
PIPE_BLEND_OVERLAP_CONJOINT = 1,
PIPE_BLEND_OVERLAP_DISJOINT = 2,
};
/**
* When faking RGBX render target formats with RGBA ones, the blender is still
* supposed to treat the destination's alpha channel as 1 instead of the
* garbage that's there. Return a blend factor that will take that into
* account.
*/
static inline enum pipe_blendfactor
util_blend_dst_alpha_to_one(enum pipe_blendfactor factor)
{
switch (factor) {
case PIPE_BLENDFACTOR_DST_ALPHA:
return PIPE_BLENDFACTOR_ONE;
case PIPE_BLENDFACTOR_INV_DST_ALPHA:
return PIPE_BLENDFACTOR_ZERO;
default:
return factor;
}
}
#endif
src/util/blob.c
+360
View File
@@ -0,0 +1,360 @@
/*
* Copyright © 2014 Intel Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
#include <string.h>
#include "blob.h"
#include "u_math.h"
#ifdef HAVE_VALGRIND
#include <valgrind.h>
#include <memcheck.h>
#define VG(x) x
#else
#define VG(x)
#endif
#define BLOB_INITIAL_SIZE 4096
/* Ensure that \blob will be able to fit an additional object of size
* \additional. The growing (if any) will occur by doubling the existing
* allocation.
*/
static bool
grow_to_fit(struct blob *blob, size_t additional)
{
size_t to_allocate;
uint8_t *new_data;
if (blob->out_of_memory)
return false;
if (blob->size + additional <= blob->allocated)
return true;
if (blob->fixed_allocation) {
blob->out_of_memory = true;
return false;
}
if (blob->allocated == 0)
to_allocate = BLOB_INITIAL_SIZE;
else
to_allocate = blob->allocated * 2;
to_allocate = MAX2(to_allocate, blob->allocated + additional);
new_data = realloc(blob->data, to_allocate);
if (new_data == NULL) {
blob->out_of_memory = true;
return false;
}
blob->data = new_data;
blob->allocated = to_allocate;
return true;
}
/* Align the blob->size so that reading or writing a value at (blob->data +
* blob->size) will result in an access aligned to a granularity of \alignment
* bytes.
*
* \return True unless allocation fails
*/
bool
blob_align(struct blob *blob, size_t alignment)
{
const size_t new_size = align_uintptr(blob->size, alignment);
if (blob->size < new_size) {
if (!grow_to_fit(blob, new_size - blob->size))
return false;
if (blob->data)
memset(blob->data + blob->size, 0, new_size - blob->size);
blob->size = new_size;
}
return true;
}
void
blob_reader_align(struct blob_reader *blob, size_t alignment)
{
blob->current = blob->data + align_uintptr(blob->current - blob->data, alignment);
}
void
blob_init(struct blob *blob)
{
blob->data = NULL;
blob->allocated = 0;
blob->size = 0;
blob->fixed_allocation = false;
blob->out_of_memory = false;
}
void
blob_init_fixed(struct blob *blob, void *data, size_t size)
{
blob->data = data;
blob->allocated = size;
blob->size = 0;
blob->fixed_allocation = true;
blob->out_of_memory = false;
}
void
blob_finish_get_buffer(struct blob *blob, void **buffer, size_t *size)
{
*buffer = blob->data;
*size = blob->size;
blob->data = NULL;
/* Trim the buffer. */
*buffer = realloc(*buffer, *size);
}
bool
blob_overwrite_bytes(struct blob *blob,
size_t offset,
const void *bytes,
size_t to_write)
{
/* Detect an attempt to overwrite data out of bounds. */
if (offset + to_write < offset || blob->size < offset + to_write)
return false;
VG(VALGRIND_CHECK_MEM_IS_DEFINED(bytes, to_write));
if (blob->data)
memcpy(blob->data + offset, bytes, to_write);
return true;
}
bool
blob_write_bytes(struct blob *blob, const void *bytes, size_t to_write)
{
if (! grow_to_fit(blob, to_write))
return false;
if (blob->data && to_write > 0) {
VG(VALGRIND_CHECK_MEM_IS_DEFINED(bytes, to_write));
memcpy(blob->data + blob->size, bytes, to_write);
}
blob->size += to_write;
return true;
}
intptr_t
blob_reserve_bytes(struct blob *blob, size_t to_write)
{
intptr_t ret;
if (! grow_to_fit (blob, to_write))
return -1;
ret = blob->size;
blob->size += to_write;
return ret;
}
intptr_t
blob_reserve_uint32(struct blob *blob)
{
blob_align(blob, sizeof(uint32_t));
return blob_reserve_bytes(blob, sizeof(uint32_t));
}
intptr_t
blob_reserve_intptr(struct blob *blob)
{
blob_align(blob, sizeof(intptr_t));
return blob_reserve_bytes(blob, sizeof(intptr_t));
}
#define BLOB_WRITE_TYPE(name, type) \
bool \
name(struct blob *blob, type value) \
{ \
blob_align(blob, sizeof(value)); \
return blob_write_bytes(blob, &value, sizeof(value)); \
}
BLOB_WRITE_TYPE(blob_write_uint8, uint8_t)
BLOB_WRITE_TYPE(blob_write_uint16, uint16_t)
BLOB_WRITE_TYPE(blob_write_uint32, uint32_t)
BLOB_WRITE_TYPE(blob_write_uint64, uint64_t)
BLOB_WRITE_TYPE(blob_write_intptr, intptr_t)
#define ASSERT_ALIGNED(_offset, _align) \
assert(align_uintptr((_offset), (_align)) == (_offset))
bool
blob_overwrite_uint8 (struct blob *blob,
size_t offset,
uint8_t value)
{
ASSERT_ALIGNED(offset, sizeof(value));
return blob_overwrite_bytes(blob, offset, &value, sizeof(value));
}
bool
blob_overwrite_uint32 (struct blob *blob,
size_t offset,
uint32_t value)
{
ASSERT_ALIGNED(offset, sizeof(value));
return blob_overwrite_bytes(blob, offset, &value, sizeof(value));
}
bool
blob_overwrite_intptr (struct blob *blob,
size_t offset,
intptr_t value)
{
ASSERT_ALIGNED(offset, sizeof(value));
return blob_overwrite_bytes(blob, offset, &value, sizeof(value));
}
bool
blob_write_string(struct blob *blob, const char *str)
{
return blob_write_bytes(blob, str, strlen(str) + 1);
}
void
blob_reader_init(struct blob_reader *blob, const void *data, size_t size)
{
blob->data = data;
blob->end = blob->data + size;
blob->current = data;
blob->overrun = false;
}
/* Check that an object of size \size can be read from this blob.
*
* If not, set blob->overrun to indicate that we attempted to read too far.
*/
static bool
ensure_can_read(struct blob_reader *blob, size_t size)
{
if (blob->overrun)
return false;
if (blob->current <= blob->end && blob->end - blob->current >= size)
return true;
blob->overrun = true;
return false;
}
const void *
blob_read_bytes(struct blob_reader *blob, size_t size)
{
const void *ret;
if (! ensure_can_read (blob, size))
return NULL;
ret = blob->current;
blob->current += size;
return ret;
}
void
blob_copy_bytes(struct blob_reader *blob, void *dest, size_t size)
{
const void *bytes;
bytes = blob_read_bytes(blob, size);
if (bytes == NULL || size == 0)
return;
memcpy(dest, bytes, size);
}
void
blob_skip_bytes(struct blob_reader *blob, size_t size)
{
if (ensure_can_read (blob, size))
blob->current += size;
}
#define BLOB_READ_TYPE(name, type) \
type \
name(struct blob_reader *blob) \
{ \
type ret = 0; \
int size = sizeof(ret); \
blob_reader_align(blob, size); \
blob_copy_bytes(blob, &ret, size); \
return ret; \
}
BLOB_READ_TYPE(blob_read_uint8, uint8_t)
BLOB_READ_TYPE(blob_read_uint16, uint16_t)
BLOB_READ_TYPE(blob_read_uint32, uint32_t)
BLOB_READ_TYPE(blob_read_uint64, uint64_t)
BLOB_READ_TYPE(blob_read_intptr, intptr_t)
char *
blob_read_string(struct blob_reader *blob)
{
int size;
char *ret;
const uint8_t *nul;
/* If we're already at the end, then this is an overrun. */
if (blob->current >= blob->end) {
blob->overrun = true;
return NULL;
}
/* Similarly, if there is no zero byte in the data remaining in this blob,
* we also consider that an overrun.
*/
nul = memchr(blob->current, 0, blob->end - blob->current);
if (nul == NULL) {
blob->overrun = true;
return NULL;
}
size = nul - blob->current + 1;
assert(ensure_can_read(blob, size));
ret = (char *) blob->current;
blob->current += size;
return ret;
}
src/util/blob.h
+442
View File
@@ -0,0 +1,442 @@
/*
* Copyright © 2014 Intel Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
#ifndef BLOB_H
#define BLOB_H
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#include <stdlib.h>
#ifdef __cplusplus
extern "C" {
#endif
/* The blob functions implement a simple, low-level API for serializing and
* deserializing.
*
* All objects written to a blob will be serialized directly, (without any
* additional meta-data to describe the data written). Therefore, it is the
* caller's responsibility to ensure that any data can be read later, (either
* by knowing exactly what data is expected, or by writing to the blob
* sufficient meta-data to describe what has been written).
*
* A blob is efficient in that it dynamically grows by doubling in size, so
* allocation costs are logarithmic.
*/
struct blob {
/* The data actually written to the blob. Never read or write this directly
* when serializing, use blob_reserve_* and blob_overwrite_* instead which
* check for out_of_memory and handle fixed-size blobs correctly.
*/
uint8_t *data;
/** Number of bytes that have been allocated for \c data. */
size_t allocated;
/** The number of bytes that have actual data written to them. */
size_t size;
/** True if \c data a fixed allocation that we cannot resize
*
* \see blob_init_fixed
*/
bool fixed_allocation;
/**
* True if we've ever failed to realloc or if we go past the end of a fixed
* allocation blob.
*/
bool out_of_memory;
};
/* When done reading, the caller can ensure that everything was consumed by
* checking the following:
*
* 1. blob->current should be equal to blob->end, (if not, too little was
* read).
*
* 2. blob->overrun should be false, (otherwise, too much was read).
*/
struct blob_reader {
const uint8_t *data;
const uint8_t *end;
const uint8_t *current;
bool overrun;
};
/**
* Init a new, empty blob.
*/
void
blob_init(struct blob *blob);
/**
* Init a new, fixed-size blob.
*
* A fixed-size blob has a fixed block of data that will not be freed on
* blob_finish and will never be grown. If we hit the end, we simply start
* returning false from the write functions.
*
* If a fixed-size blob has a NULL data pointer then the data is written but
* it otherwise operates normally. This can be used to determine the size
* that will be required to write a given data structure.
*/
void
blob_init_fixed(struct blob *blob, void *data, size_t size);
/**
* Finish a blob and free its memory.
*
* If \blob was initialized with blob_init_fixed, the data pointer is
* considered to be owned by the user and will not be freed.
*/
static inline void
blob_finish(struct blob *blob)
{
if (!blob->fixed_allocation)
free(blob->data);
}
void
blob_finish_get_buffer(struct blob *blob, void **buffer, size_t *size);
/**
* Aligns the blob to the given alignment.
*
* \see blob_reader_align
*
* \return True unless allocation fails
*/
bool
blob_align(struct blob *blob, size_t alignment);
/**
* Add some unstructured, fixed-size data to a blob.
*
* \return True unless allocation failed.
*/
bool
blob_write_bytes(struct blob *blob, const void *bytes, size_t to_write);
/**
* Reserve space in \blob for a number of bytes.
*
* Space will be allocated within the blob for these byes, but the bytes will
* be left uninitialized. The caller is expected to use \sa
* blob_overwrite_bytes to write to these bytes.
*
* \return An offset to space allocated within \blob to which \to_write bytes
* can be written, (or -1 in case of any allocation error).
*/
intptr_t
blob_reserve_bytes(struct blob *blob, size_t to_write);
/**
* Similar to \sa blob_reserve_bytes, but only reserves an uint32_t worth of
* space. Note that this must be used if later reading with \sa
* blob_read_uint32, since it aligns the offset correctly.
*/
intptr_t
blob_reserve_uint32(struct blob *blob);
/**
* Similar to \sa blob_reserve_bytes, but only reserves an intptr_t worth of
* space. Note that this must be used if later reading with \sa
* blob_read_intptr, since it aligns the offset correctly.
*/
intptr_t
blob_reserve_intptr(struct blob *blob);
/**
* Overwrite some data previously written to the blob.
*
* Writes data to an existing portion of the blob at an offset of \offset.
* This data range must have previously been written to the blob by one of the
* blob_write_* calls.
*
* For example usage, see blob_overwrite_uint32
*
* \return True unless the requested offset or offset+to_write lie outside
* the current blob's size.
*/
bool
blob_overwrite_bytes(struct blob *blob,
size_t offset,
const void *bytes,
size_t to_write);
/**
* Add a uint8_t to a blob.
*
* \return True unless allocation failed.
*/
bool
blob_write_uint8(struct blob *blob, uint8_t value);
/**
* Overwrite a uint8_t previously written to the blob.
*
* Writes a uint8_t value to an existing portion of the blob at an offset of
* \offset. This data range must have previously been written to the blob by
* one of the blob_write_* calls.
*
* \return True unless the requested position or position+to_write lie outside
* the current blob's size.
*/
bool
blob_overwrite_uint8(struct blob *blob,
size_t offset,
uint8_t value);
/**
* Add a uint16_t to a blob.
*
* \note This function will only write to a uint16_t-aligned offset from the
* beginning of the blob's data, so some padding bytes may be added to the
* blob if this write follows some unaligned write (such as
* blob_write_string).
*
* \return True unless allocation failed.
*/
bool
blob_write_uint16(struct blob *blob, uint16_t value);
/**
* Add a uint32_t to a blob.
*
* \note This function will only write to a uint32_t-aligned offset from the
* beginning of the blob's data, so some padding bytes may be added to the
* blob if this write follows some unaligned write (such as
* blob_write_string).
*
* \return True unless allocation failed.
*/
bool
blob_write_uint32(struct blob *blob, uint32_t value);
/**
* Overwrite a uint32_t previously written to the blob.
*
* Writes a uint32_t value to an existing portion of the blob at an offset of
* \offset. This data range must have previously been written to the blob by
* one of the blob_write_* calls.
*
*
* The expected usage is something like the following pattern:
*
* size_t offset;
*
* offset = blob_reserve_uint32(blob);
* ... various blob write calls, writing N items ...
* blob_overwrite_uint32 (blob, offset, N);
*
* \return True unless the requested position or position+to_write lie outside
* the current blob's size.
*/
bool
blob_overwrite_uint32(struct blob *blob,
size_t offset,
uint32_t value);
/**
* Add a uint64_t to a blob.
*
* \note This function will only write to a uint64_t-aligned offset from the
* beginning of the blob's data, so some padding bytes may be added to the
* blob if this write follows some unaligned write (such as
* blob_write_string).
*
* \return True unless allocation failed.
*/
bool
blob_write_uint64(struct blob *blob, uint64_t value);
/**
* Add an intptr_t to a blob.
*
* \note This function will only write to an intptr_t-aligned offset from the
* beginning of the blob's data, so some padding bytes may be added to the
* blob if this write follows some unaligned write (such as
* blob_write_string).
*
* \return True unless allocation failed.
*/
bool
blob_write_intptr(struct blob *blob, intptr_t value);
/**
* Overwrite an intptr_t previously written to the blob.
*
* Writes a intptr_t value to an existing portion of the blob at an offset of
* \offset. This data range must have previously been written to the blob by
* one of the blob_write_* calls.
*
* For example usage, see blob_overwrite_uint32
*
* \return True unless the requested position or position+to_write lie outside
* the current blob's size.
*/
bool
blob_overwrite_intptr(struct blob *blob,
size_t offset,
intptr_t value);
/**
* Add a NULL-terminated string to a blob, (including the NULL terminator).
*
* \return True unless allocation failed.
*/
bool
blob_write_string(struct blob *blob, const char *str);
/**
* Start reading a blob, (initializing the contents of \blob for reading).
*
* After this call, the caller can use the various blob_read_* functions to
* read elements from the data array.
*
* For all of the blob_read_* functions, if there is insufficient data
* remaining, the functions will do nothing, (perhaps returning default values
* such as 0). The caller can detect this by noting that the blob_reader's
* current value is unchanged before and after the call.
*/
void
blob_reader_init(struct blob_reader *blob, const void *data, size_t size);
/**
* Align the current offset of the blob reader to the given alignment.
*
* This may be useful if you need the result of blob_read_bytes to have a
* particular alignment. Note that this only aligns relative to blob->data
* and the alignment of the resulting pointer is only guaranteed if blob->data
* is also aligned to the requested alignment.
*/
void
blob_reader_align(struct blob_reader *blob, size_t alignment);
/**
* Read some unstructured, fixed-size data from the current location, (and
* update the current location to just past this data).
*
* \note The memory returned belongs to the data underlying the blob reader. The
* caller must copy the data in order to use it after the lifetime of the data
* underlying the blob reader.
*
* \return The bytes read (see note above about memory lifetime).
*/
const void *
blob_read_bytes(struct blob_reader *blob, size_t size);
/**
* Read some unstructured, fixed-size data from the current location, copying
* it to \dest (and update the current location to just past this data)
*/
void
blob_copy_bytes(struct blob_reader *blob, void *dest, size_t size);
/**
* Skip \size bytes within the blob.
*/
void
blob_skip_bytes(struct blob_reader *blob, size_t size);
/**
* Read a uint8_t from the current location, (and update the current location
* to just past this uint8_t).
*
* \return The uint8_t read
*/
uint8_t
blob_read_uint8(struct blob_reader *blob);
/**
* Read a uint16_t from the current location, (and update the current location
* to just past this uint16_t).
*
* \note This function will only read from a uint16_t-aligned offset from the
* beginning of the blob's data, so some padding bytes may be skipped.
*
* \return The uint16_t read
*/
uint16_t
blob_read_uint16(struct blob_reader *blob);
/**
* Read a uint32_t from the current location, (and update the current location
* to just past this uint32_t).
*
* \note This function will only read from a uint32_t-aligned offset from the
* beginning of the blob's data, so some padding bytes may be skipped.
*
* \return The uint32_t read
*/
uint32_t
blob_read_uint32(struct blob_reader *blob);
/**
* Read a uint64_t from the current location, (and update the current location
* to just past this uint64_t).
*
* \note This function will only read from a uint64_t-aligned offset from the
* beginning of the blob's data, so some padding bytes may be skipped.
*
* \return The uint64_t read
*/
uint64_t
blob_read_uint64(struct blob_reader *blob);
/**
* Read an intptr_t value from the current location, (and update the
* current location to just past this intptr_t).
*
* \note This function will only read from an intptr_t-aligned offset from the
* beginning of the blob's data, so some padding bytes may be skipped.
*
* \return The intptr_t read
*/
intptr_t
blob_read_intptr(struct blob_reader *blob);
/**
* Read a NULL-terminated string from the current location, (and update the
* current location to just past this string).
*
* \note The memory returned belongs to the data underlying the blob reader. The
* caller must copy the string in order to use the string after the lifetime
* of the data underlying the blob reader.
*
* \return The string read (see note above about memory lifetime). However, if
* there is no NULL byte remaining within the blob, this function returns
* NULL.
*/
char *
blob_read_string(struct blob_reader *blob);
#ifdef __cplusplus
}
#endif
#endif /* BLOB_H */
src/util/box.h
+380
View File
@@ -0,0 +1,380 @@
#ifndef UTIL_BOX_H
#define UTIL_BOX_H
#include "util/u_math.h"
#include "util/format/u_format.h"
/**
* Subregion of 1D/2D/3D image resource.
*/
struct pipe_box
{
/* Only "x" and "width" are used to represent buffer ranges.
* The maximum representable texture size is ANY x ANY x 16K.
*/
int32_t x;
int32_t width;
int32_t y;
int32_t height;
int16_t z;
int16_t depth;
};
static inline void
u_box_1d(unsigned x, unsigned w, struct pipe_box *box)
{
box->x = x;
box->y = 0;
box->z = 0;
box->width = w;
box->height = 1;
box->depth = 1;
}
static inline void
u_box_2d(unsigned x,unsigned y, unsigned w, unsigned h, struct pipe_box *box)
{
box->x = x;
box->y = y;
box->z = 0;
box->width = w;
box->height = h;
box->depth = 1;
}
static inline void
u_box_origin_2d(unsigned w, unsigned h, struct pipe_box *box)
{
box->x = 0;
box->y = 0;
box->z = 0;
box->width = w;
box->height = h;
box->depth = 1;
}
static inline void
u_box_2d_zslice(unsigned x, unsigned y, unsigned z,
unsigned w, unsigned h, struct pipe_box *box)
{
box->x = x;
box->y = y;
box->z = (int16_t)z;
box->width = w;
box->height = h;
box->depth = 1;
}
static inline void
u_box_3d(unsigned x, unsigned y, unsigned z,
unsigned w, unsigned h, unsigned d,
struct pipe_box *box)
{
box->x = x;
box->y = y;
box->z = (int16_t)z;
box->width = w;
box->height = h;
box->depth = (int16_t)d;
}
/* Clips @dst to width @w and height @h.
* Returns -1 if the resulting box would be empty (then @dst is left unchanged).
* 0 if nothing has been reduced.
* 1 if width has been reduced.
* 2 if height has been reduced.
* 3 if both width and height have been reduced.
* Aliasing permitted.
*/
static inline int
u_box_clip_2d(struct pipe_box *dst,
const struct pipe_box *box, int w, int h)
{
unsigned i;
int a[2], b[2], dim[2];
int *start, *end;
int res = 0;
if (!box->width || !box->height)
return -1;
dim[0] = w;
dim[1] = h;
a[0] = box->x;
a[1] = box->y;
b[0] = box->x + box->width;
b[1] = box->y + box->height;
for (i = 0; i < 2; ++i) {
start = (a[i] <= b[i]) ? &a[i] : &b[i];
end = (a[i] <= b[i]) ? &b[i] : &a[i];
if (*end < 0 || *start >= dim[i])
return -1;
if (*start < 0) {
*start = 0;
res |= (1 << i);
}
if (*end > dim[i]) {
*end = dim[i];
res |= (1 << i);
}
}
if (res) {
dst->x = a[0];
dst->y = a[1];
dst->width = b[0] - a[0];
dst->height = b[1] - a[1];
}
return res;
}
static inline int64_t
u_box_volume_3d(const struct pipe_box *box)
{
return (int64_t)box->width * box->height * box->depth;
}
/* Aliasing of @dst permitted. Supports empty width */
static inline void
u_box_union_1d(struct pipe_box *dst,
const struct pipe_box *a, const struct pipe_box *b)
{
int x, width;
if (a->width == 0) {
x = b->x;
width = b->width;
} else if (b->width == 0) {
x = a->x;
width = a->width;
} else {
x = MIN2(a->x, b->x);
width = MAX2(a->x + a->width, b->x + b->width) - x;
}
dst->x = x;
dst->width = width;
}
/* Aliasing of @dst permitted. */
static inline void
u_box_intersect_1d(struct pipe_box *dst,
const struct pipe_box *a, const struct pipe_box *b)
{
int x;
x = MAX2(a->x, b->x);
dst->width = MIN2(a->x + a->width, b->x + b->width) - x;
dst->x = x;
if (dst->width <= 0) {
dst->x = 0;
dst->width = 0;
}
}
/* Aliasing of @dst permitted. */
static inline void
u_box_union_2d(struct pipe_box *dst,
const struct pipe_box *a, const struct pipe_box *b)
{
int x, y;
x = MIN2(a->x, b->x);
y = MIN2(a->y, b->y);
dst->width = MAX2(a->x + a->width, b->x + b->width) - x;
dst->height = MAX2(a->y + a->height, b->y + b->height) - y;
dst->x = x;
dst->y = y;
}
/* Aliasing of @dst permitted. */
static inline void
u_box_intersect_2d(struct pipe_box *dst,
const struct pipe_box *a, const struct pipe_box *b)
{
int x, y;
x = MAX2(a->x, b->x);
y = MAX2(a->y, b->y);
dst->width = MIN2(a->x + a->width, b->x + b->width) - x;
dst->x = x;
if (dst->width <= 0) {
dst->x = 0;
dst->width = 0;
}
dst->height = MIN2(a->y + a->height, b->y + b->height) - y;
dst->y = y;
if (dst->height <= 0) {
dst->y = 0;
dst->height = 0;
}
}
/* Aliasing of @dst permitted. */
static inline void
u_box_union_3d(struct pipe_box *dst,
const struct pipe_box *a, const struct pipe_box *b)
{
int x, y, z;
x = MIN2(a->x, b->x);
y = MIN2(a->y, b->y);
z = MIN2(a->z, b->z);
dst->width = MAX2(a->x + a->width, b->x + b->width) - x;
dst->height = MAX2(a->y + a->height, b->y + b->height) - y;
dst->depth = (int16_t) (MAX2(a->z + a->depth, b->z + b->depth) - z);
dst->x = x;
dst->y = y;
dst->z = (int16_t)z;
}
static inline bool
u_box_test_intersection_1d(const struct pipe_box *a,
const struct pipe_box *b)
{
int ax[2], bx[2];
ax[0] = MIN2(a->x, a->x + a->width);
ax[1] = MAX2(a->x, a->x + a->width - 1);
bx[0] = MIN2(b->x, b->x + b->width);
bx[1] = MAX2(b->x, b->x + b->width - 1);
return ax[1] >= bx[0] && bx[1] >= ax[0];
}
static inline bool
u_box_test_intersection_2d(const struct pipe_box *a,
const struct pipe_box *b)
{
int ax[2], ay[2], bx[2], by[2];
ax[0] = MIN2(a->x, a->x + a->width);
ax[1] = MAX2(a->x, a->x + a->width - 1);
ay[0] = MIN2(a->y, a->y + a->height);
ay[1] = MAX2(a->y, a->y + a->height - 1);
bx[0] = MIN2(b->x, b->x + b->width);
bx[1] = MAX2(b->x, b->x + b->width - 1);
by[0] = MIN2(b->y, b->y + b->height);
by[1] = MAX2(b->y, b->y + b->height - 1);
return ax[1] >= bx[0] && bx[1] >= ax[0] &&
ay[1] >= by[0] && by[1] >= ay[0];
}
static inline bool
u_box_test_intersection_3d(const struct pipe_box *a,
const struct pipe_box *b)
{
int ax[2], ay[2], ad[2], bx[2], by[2], bd[2];
ax[0] = MIN2(a->x, a->x + a->width);
ax[1] = MAX2(a->x, a->x + a->width - 1);
ay[0] = MIN2(a->y, a->y + a->height);
ay[1] = MAX2(a->y, a->y + a->height - 1);
ad[0] = MIN2(a->z, a->z + a->depth);
ad[1] = MAX2(a->z, a->z + a->depth - 1);
bx[0] = MIN2(b->x, b->x + b->width);
bx[1] = MAX2(b->x, b->x + b->width - 1);
by[0] = MIN2(b->y, b->y + b->height);
by[1] = MAX2(b->y, b->y + b->height - 1);
bd[0] = MIN2(b->z, b->z + b->depth);
bd[1] = MAX2(b->z, b->z + b->depth - 1);
return ax[1] >= bx[0] && bx[1] >= ax[0] &&
ay[1] >= by[0] && by[1] >= ay[0] &&
ad[1] >= bd[0] && bd[1] >= ad[0];
}
static inline void
u_box_minify_2d(struct pipe_box *dst,
const struct pipe_box *src, unsigned l)
{
dst->x = src->x >> l;
dst->y = src->y >> l;
dst->width = MAX2(src->width >> l, 1);
dst->height = MAX2(src->height >> l, 1);
}
static inline void
u_box_minify_3d(struct pipe_box *dst,
const struct pipe_box *src, unsigned l)
{
dst->x = src->x >> l;
dst->y = src->y >> l;
dst->z = src->z >> l;
dst->width = MAX2(src->width >> l, 1);
dst->height = MAX2(src->height >> l, 1);
dst->depth = MAX2(src->depth >> l, 1);
}
/* Converts a box specified in pixels to an equivalent box specified
* in blocks, where the boxes represent a region-of-interest of an image with
* the given format. This is trivial (a copy) for uncompressed formats.
*/
static inline void
u_box_pixels_to_blocks(struct pipe_box *blocks,
const struct pipe_box *pixels, enum pipe_format format)
{
u_box_3d(
pixels->x / util_format_get_blockwidth(format),
pixels->y / util_format_get_blockheight(format),
pixels->z,
DIV_ROUND_UP(pixels->width, util_format_get_blockwidth(format)),
DIV_ROUND_UP(pixels->height, util_format_get_blockheight(format)),
pixels->depth,
blocks);
}
static inline bool
util_is_box_sint16(const struct pipe_box *box)
{
return util_is_sint16(box->x) && util_is_sint16(box->y) &&
util_is_sint16(box->z) && util_is_sint16(box->width) &&
util_is_sint16(box->height) && util_is_sint16(box->depth) &&
util_is_sint16(box->x + box->width) &&
util_is_sint16(box->y + box->height) &&
util_is_sint16(box->z + box->depth);
}
static inline bool
util_is_box_out_of_bounds(const struct pipe_box *src_box, unsigned coord_mask,
unsigned width, unsigned height, unsigned mip_level)
{
int src_width = u_minify(width, mip_level);
int src_height = u_minify(height, mip_level);
struct pipe_box box = *src_box;
/* Eliminate negative width/height/depth. */
if (box.width < 0) {
box.x += box.width;
box.width *= -1;
}
if (box.height < 0) {
box.y += box.height;
box.height *= -1;
}
bool x_in_bounds = box.x >= 0 && box.x < src_width &&
box.x + box.width > 0 && box.x + box.width <= src_width;
bool y_in_bounds = box.y >= 0 && box.y < src_height &&
box.y + box.height > 0 && box.y + box.height <= src_height;
/* Return if the box is not in bounds. */
if (coord_mask & BITFIELD_BIT(0) && !x_in_bounds)
return true;
if (coord_mask & BITFIELD_BIT(1) && !y_in_bounds)
return true;
return false;
}
#endif
src/util/build_id.c
+209
View File
@@ -0,0 +1,209 @@
/*
* Copyright © 2016 Intel Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
#include "build_id.h"
#include "hex.h"
#if HAVE_BUILD_ID
#include <dlfcn.h>
#include <stddef.h>
#include <string.h>
#include "macros.h"
#if DETECT_OS_APPLE
#include <mach-o/loader.h>
struct build_id_note {
const struct uuid_command uuid_cmd;
};
#else
#include <link.h>
#ifndef NT_GNU_BUILD_ID
#define NT_GNU_BUILD_ID 3
#endif
#ifndef ElfW
#define ElfW(type) Elf_##type
#endif
struct build_id_note {
ElfW(Nhdr) nhdr;
char name[4]; /* Note name for build-id is "GNU\0" */
uint8_t build_id[0];
};
#endif /* DETECT_OS_APPLE */
struct callback_data {
/* Base address of shared object, taken from Dl_info::dli_fbase */
const void *dli_fbase;
struct build_id_note *note;
};
#if DETECT_OS_APPLE
static int
build_id_find_uuid_command(void *data_)
{
struct callback_data *data = data_;
const struct mach_header *header =
(const struct mach_header *)data->dli_fbase;
const struct load_command *cmd = NULL;
/* Headers' sizes differ based on architecture */
if (header->magic == MH_MAGIC_64 || header->magic == MH_CIGAM_64) {
cmd =
(const struct load_command *)((const struct mach_header_64 *)header +
1);
} else if (header->magic == MH_MAGIC || header->magic == MH_CIGAM) {
cmd = (const struct load_command *)(header + 1);
} else {
return 0;
}
uint32_t ncmds = header->ncmds;
for (uint32_t i = 0; i < ncmds; i++) {
if (cmd->cmd == LC_UUID) {
data->note = (struct build_id_note *)cmd;
return 1;
}
cmd = (const struct load_command *)((const char *)cmd + cmd->cmdsize);
}
return 0;
}
#else
static int
build_id_find_nhdr_callback(struct dl_phdr_info *info, size_t size, void *data_)
{
struct callback_data *data = data_;
/* Calculate address where shared object is mapped into the process space.
* (Using the base address and the virtual address of the first LOAD segment)
*/
void *map_start = NULL;
for (unsigned i = 0; i < info->dlpi_phnum; i++) {
if (info->dlpi_phdr[i].p_type == PT_LOAD) {
map_start = (void *)(info->dlpi_addr + info->dlpi_phdr[i].p_vaddr);
break;
}
}
if (map_start != data->dli_fbase)
return 0;
for (unsigned i = 0; i < info->dlpi_phnum; i++) {
if (info->dlpi_phdr[i].p_type != PT_NOTE)
continue;
struct build_id_note *note = (void *)(info->dlpi_addr +
info->dlpi_phdr[i].p_vaddr);
ptrdiff_t len = info->dlpi_phdr[i].p_filesz;
while (len >= sizeof(struct build_id_note)) {
if (note->nhdr.n_type == NT_GNU_BUILD_ID &&
note->nhdr.n_descsz != 0 &&
note->nhdr.n_namesz == 4 &&
memcmp(note->name, "GNU", 4) == 0) {
data->note = note;
return 1;
}
size_t offset = sizeof(ElfW(Nhdr)) +
ALIGN_POT(note->nhdr.n_namesz, 4) +
ALIGN_POT(note->nhdr.n_descsz, 4);
note = (struct build_id_note *)((char *)note + offset);
len -= offset;
}
}
return 0;
}
#endif /* DETECT_OS_APPLE */
const struct build_id_note *
build_id_find_nhdr_for_addr(const void *addr)
{
Dl_info info;
if (!dladdr(addr, &info))
return NULL;
if (!info.dli_fbase)
return NULL;
struct callback_data data = {
.dli_fbase = info.dli_fbase,
.note = NULL,
};
#if DETECT_OS_APPLE
if (!build_id_find_uuid_command(&data))
return NULL;
#else
if (!dl_iterate_phdr(build_id_find_nhdr_callback, &data))
return NULL;
#endif /* DETECT_OS_APPLE */
return data.note;
}
unsigned
build_id_length(const struct build_id_note *note)
{
#if DETECT_OS_APPLE
return sizeof(note->uuid_cmd.uuid);
#else
return note->nhdr.n_descsz;
#endif /* DETECT_OS_APPLE */
}
const uint8_t *
build_id_data(const struct build_id_note *note)
{
#if DETECT_OS_APPLE
return note->uuid_cmd.uuid;
#else
return note->build_id;
#endif /* DETECT_OS_APPLE */
}
void
copy_build_id_to_sha1(uint8_t sha1[BLAKE3_KEY_LEN],
const struct build_id_note *note)
{
unsigned length = build_id_length(note);
assert(length <= BLAKE3_KEY_LEN);
memcpy(sha1, build_id_data(note), length);
memset(sha1 + length, 0, BLAKE3_KEY_LEN - length);
}
#endif
void
_mesa_sha1_format(char *buf, const unsigned char *sha1)
{
mesa_bytes_to_hex(buf, sha1, BLAKE3_KEY_LEN);
}
src/util/build_id.h
+68
View File
@@ -0,0 +1,68 @@
/*
* Copyright © 2016 Intel Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
#ifndef BUILD_ID_H
#define BUILD_ID_H
#include "detect_os.h"
#if defined(HAVE_DL_ITERATE_PHDR) || DETECT_OS_APPLE
#define HAVE_BUILD_ID 1
#else
#define HAVE_BUILD_ID 0
#endif /* defined(HAVE_DL_ITERATE_PHDR) || DETECT_OS_APPLE */
#ifdef __cplusplus
extern "C" {
#endif
#include "mesa-blake3.h"
#if HAVE_BUILD_ID
#define BUILD_ID_EXPECTED_HASH_LENGTH 20 /* sha1 */
struct build_id_note;
const struct build_id_note *
build_id_find_nhdr_for_addr(const void *addr);
unsigned
build_id_length(const struct build_id_note *note);
const uint8_t *
build_id_data(const struct build_id_note *note);
void
copy_build_id_to_sha1(uint8_t sha1[BLAKE3_KEY_LEN],
const struct build_id_note *note);
#endif
void
_mesa_sha1_format(char *buf, const unsigned char *sha1);
#ifdef __cplusplus
}
#endif
#endif /* BUILD_ID_H */
src/util/cache_ops.h
+122
View File
@@ -0,0 +1,122 @@
/*
* Copyright © 2025 Collabora Ltd.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
#ifndef UTIL_CACHE_OPS_H
#define UTIL_CACHE_OPS_H
#include <stdbool.h>
#include <stddef.h>
#include "detect_arch.h"
#include "u_cpu_detect.h"
#ifdef __cplusplus
extern "C" {
#endif
/** Returns true if we have cache operations available */
static inline bool
util_has_cache_ops(void)
{
/* TODO: Port to MSVC if and when we have Windows hardware drivers that
* need cache flushing ops.
*/
#if defined(_MSC_VER)
return false;
#endif
#if DETECT_ARCH_X86
return util_get_cpu_caps()->has_sse2;
#elif DETECT_ARCH_X86_64 || DETECT_ARCH_AARCH64
return true;
#else
return false;
#endif
}
/** Returns the cache granularity
*
* This is the maximum number of bytes that may be overwritten as the result
* of a cache flush or cache line eviction. On big.LITTLE platforms, the
* cache flush helpers may sometimes operate at a smaller granularity but may
* also round up to at most util_cache_granularity().
*
* Vulkan drivers should return this as nonCoherentAtomSize.
*/
size_t util_cache_granularity(void);
/** Flushes a range to main memory */
void util_flush_range(void *start, size_t size);
/** Flushes a range to main memory and invalidates those cache lines */
void util_flush_inval_range(void *start, size_t size);
/** Flushes a range to main memory without fencing
*
* This is for the case where you have a lot of ranges to flush and want to
* avoid unnecessary fencing. In this case, call
*
* util_pre_flush_fence()
* util_flush_range_no_fence()
* util_flush_range_no_fence()
* util_post_flush_fence()
*/
void util_flush_range_no_fence(void *start, size_t size);
/** Flushes a range to main memory and invalidates those cache lines without
* fencing
*
* This is for the case where you have a lot of ranges to flush and invalidate
* and want to avoid unnecessary fencing. In this case, call
*
* util_pre_flush_fence()
* util_flush_inval_range_no_fence()
* util_flush_range_no_fence()
* util_flush_inval_range_no_fence()
* util_post_flush_inval_fence()
*/
void util_flush_inval_range_no_fence(void *start, size_t size);
/** Fence between memory access and cache flush operations
*
* see util_flush_range_no_fence()
*/
void util_pre_flush_fence(void);
/** Fence between cache flush operations and memory access
*
* see util_flush_range_no_fence()
*/
void util_post_flush_fence(void);
/** Fence between cache invalidate operations and memory access
*
* see util_flush_inval_range_no_fence()
*/
void util_post_flush_inval_fence(void);
#ifdef __cplusplus
}
#endif
#endif /* UTIL_CACHE_OPS_H */
src/util/cache_ops_aarch64.c
+228
View File
@@ -0,0 +1,228 @@
/*
* Copyright © 2025 Collabora Ltd. and Igalia S.L.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
#include "cache_ops.h"
#include "util/macros.h"
#include "util/u_atomic.h"
static uint32_t
get_ctr_el0(void)
{
uint32_t ctr_el0;
__asm("mrs\t%x0, ctr_el0" : "=r"(ctr_el0));
return ctr_el0;
}
static uint32_t
get_ctr_cwg(void)
{
return (get_ctr_el0() >> 24) & 0xf;
}
size_t
util_cache_granularity(void)
{
static uint32_t cached_size = 0;
uint32_t size = p_atomic_read(&cached_size);
if (likely(size > 0))
return size;
/* We use CTR_EL0.CWG as the cache granularity. According to Arm:
*
* "CWG, [27:24]
*
* Cache write-back granule. Log2 of the number of words of the maximum
* size of memory that can be overwritten as a result of the eviction of
* a cache entry that has had a memory location in it modified"
*
* On big.LITTLE CPUs, Linux will trap on fetching CTR_EL0 and take the
* maximum across all CPU cores so this should really be the maximum that
* drivers and clients can assume.
*/
size = 4 << ((get_ctr_el0() >> 24) & 0xf);
p_atomic_set(&cached_size, size);
return size;
}
static size_t
get_dmin_line(void)
{
static uint32_t cached_size = 0;
uint32_t size = p_atomic_read(&cached_size);
if (likely(size > 0))
return size;
/* For walking cache lines, we want to use CTR_EL0.DminLine as the step
* size. According to Arm:
*
* "DminLine, [19:16]
*
* Log2 of the number of words in the smallest cache line of all the
* data and unified caches that the core controls"
*
* On big.LITTLE CPUs, Linux will trap on fetching CTR_EL0 and take the
* minimum across all CPU cores so this should be safe no matter what core
* we happen to be living on.
*/
size = 4 << ((get_ctr_el0() >> 16) & 0xf);
p_atomic_set(&cached_size, size);
return size;
}
static void
flush_l1_cacheline(UNUSED void *p)
{
/* Clean data cache. */
__asm volatile("dc cvac, %0" : : "r" (p) : "memory");
}
static void
flush_inval_l1_cacheline(UNUSED void *p)
{
/* Clean and Invalidate data cache, there is no separate Invalidate. */
__asm volatile("dc civac, %0" : : "r" (p) : "memory");
}
static void
data_sync_bar(void)
{
__asm volatile("dsb sy");
}
void
util_flush_range_no_fence(void *start, size_t size)
{
uintptr_t l1_cacheline_size = get_dmin_line();
char *p = (char *) (((uintptr_t) start) & ~(l1_cacheline_size - 1));
char *end = ((char *) start) + size;
while (p < end) {
flush_l1_cacheline(p);
p += l1_cacheline_size;
}
}
void
util_flush_inval_range_no_fence(void *start, size_t size)
{
uintptr_t l1_cacheline_size = get_dmin_line();
char *p = (char *) (((uintptr_t) start) & ~(l1_cacheline_size - 1));
char *end = ((char *) start) + size;
while (p < end) {
flush_inval_l1_cacheline(p);
p += l1_cacheline_size;
}
}
void
util_flush_range(void *p, size_t size)
{
if (size == 0)
return;
util_pre_flush_fence();
util_flush_range_no_fence(p, size);
util_post_flush_fence();
}
void
util_flush_inval_range(void *p, size_t size)
{
if (size == 0)
return;
util_pre_flush_fence();
util_flush_inval_range_no_fence(p, size);
util_post_flush_inval_fence();
}
void
util_pre_flush_fence(void)
{
/* From the Arm ® Architecture Reference Manual (revision L.b):
*
* "All data cache instructions, other than DC ZVA, DC GVA, and DC GZVA
* that specify an address: [...] Execute in program order relative to
* other data cache instructions, other than DC ZVA, DC GVA, and DC GZVA
* that specify an address within the same cache line of minimum size,
* as indicated by CTR_EL0.DMinLine."
*
* So cache flush operations are properly ordered against memory accesses
* and there's nothing we need to do to ensure that prior writes land
* before the cache flush operations flush the data.
*
* In the case where this pre_flush_fence() is called before a flush/inval
* used for a GPU -> CPU barrier, there is also nothing to do because it's
* the responsibility of the GPU to ensure that all memory writes have
* landed before we see this on the CPU side.
*/
}
void
util_post_flush_fence(void)
{
/* From the Arm ® Architecture Reference Manual (revision L.b):
*
* "A cache maintenance instruction can complete at any time after it is
* executed, but is only guaranteed to be complete, and its effects
* visible to other observers, following a DSB instruction executed by
* the PE that executed the cache maintenance instruction."
*
* In order to ensure that the GPU sees data flushed by pror cache flushes,
* we need to execute a DSB to ensure the flushes land.
*/
data_sync_bar();
}
void
util_post_flush_inval_fence(void)
{
/* From the Arm ® Architecture Reference Manual (revision L.b):
*
* "All data cache instructions, other than DC ZVA, DC GVA, and DC GZVA
* that specify an address: [...] Execute in program order relative to
* other data cache instructions, other than DC ZVA, DC GVA, and DC GZVA
* that specify an address within the same cache line of minimum size,
* as indicated by CTR_EL0.DMinLine."
*
* This seems to imply that memory access that happens after the cache
* flush/invalidate operation would be properly ordered with respect to it.
* However, the manual also says:
*
* "A cache maintenance instruction can complete at any time after it is
* executed, but is only guaranteed to be complete, and its effects
* visible to other observers, following a DSB instruction executed by
* the PE that executed the cache maintenance instruction."
*
* In practice, it appears that the ordering guarantees only really apply
* to the queue order in the data cache and not the order in which
* operations complete. In other words, a read which is queued after the
* invalidate may still use the stale cache line unless we explicitly
* insert a DSB between them.
*/
data_sync_bar();
}
src/util/cache_ops_null.c
+70
View File
@@ -0,0 +1,70 @@
/*
* Copyright © 2025 Collabora Ltd.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
#include "cache_ops.h"
#include "util/macros.h"
size_t
util_cache_granularity()
{
return 0;
}
void
util_flush_range(void *start, size_t size)
{
UNREACHABLE("Cache ops are not implemented on this platform");
}
void
util_flush_inval_range(void *start, size_t size)
{
UNREACHABLE("Cache ops are not implemented on this platform");
}
void
util_flush_range_no_fence(void *start, size_t size)
{
UNREACHABLE("Cache ops are not implemented on this platform");
}
void
util_flush_inval_range_no_fence(void *start, size_t size)
{
UNREACHABLE("Cache ops are not implemented on this platform");
}
void util_pre_flush_fence(void)
{
UNREACHABLE("Cache ops are not implemented on this platform");
}
void util_post_flush_fence(void)
{
UNREACHABLE("Cache ops are not implemented on this platform");
}
void util_post_flush_inval_fence(void)
{
UNREACHABLE("Cache ops are not implemented on this platform");
}
src/util/cache_ops_x86.c
+133
View File
@@ -0,0 +1,133 @@
/*
* Copyright © 2017 Intel Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
#include "cache_ops.h"
#include "u_cpu_detect.h"
#define CACHELINE_SIZE 64
#define CACHELINE_MASK 63
size_t
util_cache_granularity(void)
{
return util_get_cpu_caps()->cacheline;
}
/* Defined in cache_ops_x86_clflushopt.c */
#ifdef HAVE___BUILTIN_IA32_CLFLUSHOPT
void util_clflushopt_range(void *start, size_t size);
#endif
static void
util_clflush_range(void *start, size_t size)
{
char *p = (char *) (((uintptr_t) start) & ~CACHELINE_MASK);
char *end = ((char *) start) + size;
while (p < end) {
__builtin_ia32_clflush(p);
p += CACHELINE_SIZE;
}
}
void
util_flush_range_no_fence(void *start, size_t size)
{
#ifdef HAVE___BUILTIN_IA32_CLFLUSHOPT
if (util_get_cpu_caps()->has_clflushopt) {
util_clflushopt_range(start, size);
return;
}
#endif
util_clflush_range(start, size);
}
void
util_flush_range(void *start, size_t size)
{
__builtin_ia32_mfence();
#ifdef HAVE___BUILTIN_IA32_CLFLUSHOPT
if (util_get_cpu_caps()->has_clflushopt) {
util_clflushopt_range(start, size);
/* clflushopt doesn't include an mfence like clflush */
__builtin_ia32_mfence();
return;
}
#endif
util_clflush_range(start, size);
/* The last clflush acts as an mfence */
}
void
util_flush_inval_range_no_fence(void *start, size_t size)
{
if (size == 0)
return;
/* Modern Atom CPUs (Baytrail+) have issues with clflush serialization,
* where mfence is not a sufficient synchronization barrier. We must
* double clflush the last cacheline. This guarantees it will be ordered
* after the preceding clflushes, and then the mfence guards against
* prefetches crossing the clflush boundary.
*
* See kernel commit 396f5d62d1a5fd99421855a08ffdef8edb43c76e
* ("drm: Restore double clflush on the last partial cacheline")
* and https://bugs.freedesktop.org/show_bug.cgi?id=92845.
*/
#ifdef HAVE___BUILTIN_IA32_CLFLUSHOPT
if (util_get_cpu_caps()->has_clflushopt) {
util_clflushopt_range(start, size);
/* clflushopt doesn't include an mfence like clflush */
__builtin_ia32_mfence();
util_clflushopt_range((char *)start + size - 1, 1);
return;
}
#endif
util_clflush_range(start, size);
__builtin_ia32_clflush((char *)start + size - 1);
}
void
util_flush_inval_range(void *start, size_t size)
{
util_flush_inval_range_no_fence(start, size);
__builtin_ia32_mfence();
}
void
util_pre_flush_fence(void)
{
__builtin_ia32_mfence();
}
void
util_post_flush_fence(void)
{
__builtin_ia32_mfence();
}
void
util_post_flush_inval_fence(void)
{
__builtin_ia32_mfence();
}
src/util/cache_ops_x86_clflushopt.c
+47
View File
@@ -0,0 +1,47 @@
/*
* Copyright © 2017 Intel Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
#include <stdint.h>
#include "util/u_cpu_detect.h"
#ifndef HAVE___BUILTIN_IA32_CLFLUSHOPT
#error "Compiler doesn't support clflushopt!"
#endif
void util_clflushopt_range(void *start, size_t size);
void
util_clflushopt_range(void *start, size_t size)
{
const struct util_cpu_caps_t *cpu_caps = util_get_cpu_caps();
assert(cpu_caps->has_clflushopt);
assert(cpu_caps->cacheline > 0);
uint8_t *p = (uint8_t *) (((uintptr_t) start) &
~((uintptr_t)cpu_caps->cacheline - 1));
uint8_t *end = (uint8_t *)start + size;
while (p < end) {
__builtin_ia32_clflushopt(p);
p += cpu_caps->cacheline;
}
}
src/util/cnd_monotonic.c
+156
View File
@@ -0,0 +1,156 @@
/**************************************************************************
*
* Copyright 2020 Lag Free Games, LLC
* All Rights Reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sub license, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice (including the
* next paragraph) shall be included in all copies or substantial portions
* of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
* IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
* ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*
**************************************************************************/
#include "cnd_monotonic.h"
#include "util/os_time.h"
#include "util/timespec.h"
#include <assert.h>
#ifdef _WIN32
#include <windows.h>
static_assert(sizeof(struct u_cnd_monotonic) == sizeof(CONDITION_VARIABLE),
"The size of u_cnd_monotonic must equal to CONDITION_VARIABLE");
static_assert(sizeof(mtx_t) == sizeof(CRITICAL_SECTION),
"The size of mtx_t must equal to CRITICAL_SECTION");
#endif
int
u_cnd_monotonic_init(struct u_cnd_monotonic *cond)
{
assert(cond != NULL);
#ifdef _WIN32
InitializeConditionVariable((PCONDITION_VARIABLE)cond);
return thrd_success;
#else
int ret = thrd_error;
pthread_condattr_t condattr;
if (pthread_condattr_init(&condattr) == 0) {
if (
// pthread_condattr_setclock is not supported on Apple platforms.
// Instead, they use a relative deadline. See u_cnd_monotonic_timedwait.
#ifndef __APPLE__
(pthread_condattr_setclock(&condattr, CLOCK_MONOTONIC) == 0) &&
#endif
(pthread_cond_init(&cond->cond, &condattr) == 0)) {
ret = thrd_success;
}
pthread_condattr_destroy(&condattr);
}
return ret;
#endif
}
void
u_cnd_monotonic_destroy(struct u_cnd_monotonic *cond)
{
assert(cond != NULL);
#ifdef _WIN32
// Do nothing
#else
pthread_cond_destroy(&cond->cond);
#endif
}
int
u_cnd_monotonic_broadcast(struct u_cnd_monotonic *cond)
{
assert(cond != NULL);
#ifdef _WIN32
WakeAllConditionVariable((PCONDITION_VARIABLE)cond);
return thrd_success;
#else
return (pthread_cond_broadcast(&cond->cond) == 0) ? thrd_success : thrd_error;
#endif
}
int
u_cnd_monotonic_signal(struct u_cnd_monotonic *cond)
{
assert(cond != NULL);
#ifdef _WIN32
WakeConditionVariable((PCONDITION_VARIABLE)cond);
return thrd_success;
#else
return (pthread_cond_signal(&cond->cond) == 0) ? thrd_success : thrd_error;
#endif
}
int
u_cnd_monotonic_timedwait(struct u_cnd_monotonic *cond, mtx_t *mtx,
const struct timespec *abs_time)
{
assert(cond != NULL);
assert(mtx != NULL);
assert(abs_time != NULL);
#ifdef _WIN32
const uint64_t future = (abs_time->tv_sec * 1000) + (abs_time->tv_nsec / 1000000);
const uint64_t now = os_time_get_nano() / 1000000;
const DWORD timeout = (future > now) ? (DWORD)(future - now) : 0;
if (SleepConditionVariableCS((PCONDITION_VARIABLE)cond,
(PCRITICAL_SECTION)mtx, timeout))
return thrd_success;
return (GetLastError() == ERROR_TIMEOUT) ? thrd_timedout : thrd_error;
#else
#ifdef __APPLE__
// Convert to a relative wait as we can't use CLOCK_MONOTONIC deadlines on macOS.
struct timespec now_time;
timespec_get(&now_time, TIME_MONOTONIC);
struct timespec rel_time;
timespec_sub_saturate(&rel_time, abs_time, &now_time);
int rt = pthread_cond_timedwait_relative_np(&cond->cond, mtx, &rel_time);
#else
int rt = pthread_cond_timedwait(&cond->cond, mtx, abs_time);
#endif
if (rt == ETIMEDOUT)
return thrd_timedout;
return (rt == 0) ? thrd_success : thrd_error;
#endif
}
int
u_cnd_monotonic_wait(struct u_cnd_monotonic *cond, mtx_t *mtx)
{
assert(cond != NULL);
assert(mtx != NULL);
#ifdef _WIN32
SleepConditionVariableCS((PCONDITION_VARIABLE)cond,
(PCRITICAL_SECTION)mtx, INFINITE);
return thrd_success;
#else
return (pthread_cond_wait(&cond->cond, mtx) == 0) ? thrd_success : thrd_error;
#endif
}
src/util/cnd_monotonic.h
+62
View File
@@ -0,0 +1,62 @@
/**************************************************************************
*
* Copyright 2020 Lag Free Games, LLC
* All Rights Reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sub license, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice (including the
* next paragraph) shall be included in all copies or substantial portions
* of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
* IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
* ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*
**************************************************************************/
#ifndef CND_MONOTONIC_H
#define CND_MONOTONIC_H
#include "c11/threads.h"
#include "util/os_time.h"
#include <assert.h>
#ifdef __cplusplus
extern "C" {
#endif
struct u_cnd_monotonic
{
#ifdef _WIN32
void *Ptr;
#else
pthread_cond_t cond;
#endif
};
int u_cnd_monotonic_init(struct u_cnd_monotonic *cond);
void u_cnd_monotonic_destroy(struct u_cnd_monotonic *cond);
int u_cnd_monotonic_broadcast(struct u_cnd_monotonic *cond);
int u_cnd_monotonic_signal(struct u_cnd_monotonic *cond);
int u_cnd_monotonic_timedwait(struct u_cnd_monotonic *cond, mtx_t *mtx,
const struct timespec *abs_time);
int u_cnd_monotonic_wait(struct u_cnd_monotonic *cond, mtx_t *mtx);
#ifdef __cplusplus
}
#endif
#endif
src/util/compiler.h
+124
View File
@@ -0,0 +1,124 @@
/*
* Mesa 3-D graphics library
*
* Copyright (C) 1999-2008 Brian Paul All Rights Reserved.
* Copyright (C) 2009 VMware, Inc. All Rights Reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included
* in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
/**
* \file compiler.h
* Compiler-related stuff.
*/
#ifndef COMPILER_H
#define COMPILER_H
#include <assert.h>
#include <limits.h>
#include <stdarg.h>
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include "util/detect.h"
#include "util/macros.h"
/**
* Define CPU_TO_LE32
* Do not use these unless absolutely necessary!
* Try to use a runtime test instead.
* For now, only used by some DRI hardware drivers for color/texel packing.
*/
#if UTIL_ARCH_BIG_ENDIAN
#if defined(__linux__)
#include <byteswap.h>
#define CPU_TO_LE32( x ) bswap_32( x )
#elif defined(__APPLE__)
#include <CoreFoundation/CFByteOrder.h>
#define CPU_TO_LE32( x ) CFSwapInt32HostToLittle( x )
#elif defined(__OpenBSD__)
#include <sys/types.h>
#define CPU_TO_LE32( x ) htole32( x )
#else /*__linux__ */
#include <sys/endian.h>
#define CPU_TO_LE32( x ) bswap32( x )
#endif /*__linux__*/
#else
#define CPU_TO_LE32( x ) ( x )
#endif
#define LE32_TO_CPU( x ) CPU_TO_LE32( x )
/* Macro for stack alignment. */
#if defined(__GNUC__) && DETECT_ARCH_X86
#define UTIL_ALIGN_STACK __attribute__((force_align_arg_pointer))
#else
#define UTIL_ALIGN_STACK
#endif
#define IEEE_ONE 0x3f800000
#ifndef __has_attribute
# define __has_attribute(x) 0
#endif
#if defined(__has_cpp_attribute) && defined(__clang__)
/* We do not do the same trick as __has_attribute because parsing
* clang::fallthrough in the preprocessor fails in GCC. */
# define HAS_CLANG_FALLTHROUGH __has_cpp_attribute(clang::fallthrough)
#else
# define HAS_CLANG_FALLTHROUGH 0
#endif
#if (defined(__cplusplus) && (__cplusplus >= 201703L)) || \
(defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201710L))
/* Standard C++17/C23 attribute */
#define FALLTHROUGH [[fallthrough]]
#elif HAS_CLANG_FALLTHROUGH
/* Clang++ specific */
#define FALLTHROUGH [[clang::fallthrough]]
#elif __has_attribute(fallthrough)
/* Non-standard but supported by at least gcc and clang */
#define FALLTHROUGH __attribute__((fallthrough))
#else
#define FALLTHROUGH do { } while(0)
#endif
#ifdef __cplusplus
extern "C" {
#endif
#if !defined(__HAIKU__) && !defined(__USE_MISC)
#if !DETECT_OS_ANDROID
typedef unsigned int uint;
#endif
#endif
#if defined(__cplusplus)
}
#endif
#endif /* COMPILER_H */
src/util/compress.c
+169
View File
@@ -0,0 +1,169 @@
/*
* Copyright © 2021 Valve Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
#ifdef HAVE_COMPRESSION
#include <assert.h>
/* Ensure that zlib uses 'const' in 'z_const' declarations. */
#ifndef ZLIB_CONST
#define ZLIB_CONST
#endif
#ifdef HAVE_ZLIB
#include "zlib.h"
#endif
#ifdef HAVE_ZSTD
#include "zstd.h"
#endif
#include "util/compress.h"
#include "util/perf/cpu_trace.h"
#include "macros.h"
/* 3 is the recomended level, with 22 as the absolute maximum */
#define ZSTD_COMPRESSION_LEVEL 3
size_t
util_compress_max_compressed_len(size_t in_data_size)
{
#ifdef HAVE_ZSTD
/* from the zstd docs (https://facebook.github.io/zstd/zstd_manual.html):
* compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`.
*/
return ZSTD_compressBound(in_data_size);
#elif defined(HAVE_ZLIB)
/* From https://zlib.net/zlib_tech.html:
*
* "In the worst possible case, where the other block types would expand
* the data, deflation falls back to stored (uncompressed) blocks. Thus
* for the default settings used by deflateInit(), compress(), and
* compress2(), the only expansion is an overhead of five bytes per 16 KB
* block (about 0.03%), plus a one-time overhead of six bytes for the
* entire stream."
*/
size_t num_blocks = (in_data_size + 16383) / 16384; /* round up blocks */
return in_data_size + 6 + (num_blocks * 5);
#else
STATIC_ASSERT(false);
#endif
}
/* Compress data and return the size of the compressed data */
size_t
util_compress_deflate(const uint8_t *in_data, size_t in_data_size,
uint8_t *out_data, size_t out_buff_size)
{
MESA_TRACE_FUNC();
#ifdef HAVE_ZSTD
size_t ret = ZSTD_compress(out_data, out_buff_size, in_data, in_data_size,
ZSTD_COMPRESSION_LEVEL);
if (ZSTD_isError(ret))
return 0;
return ret;
#elif defined(HAVE_ZLIB)
size_t compressed_size = 0;
/* allocate deflate state */
z_stream strm;
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;
strm.next_in = in_data;
strm.next_out = out_data;
strm.avail_in = in_data_size;
strm.avail_out = out_buff_size;
int ret = deflateInit(&strm, Z_BEST_COMPRESSION);
if (ret != Z_OK) {
(void) deflateEnd(&strm);
return 0;
}
/* compress until end of in_data */
ret = deflate(&strm, Z_FINISH);
/* stream should be complete */
assert(ret == Z_STREAM_END);
if (ret == Z_STREAM_END) {
compressed_size = strm.total_out;
}
/* clean up and return */
(void) deflateEnd(&strm);
return compressed_size;
#else
STATIC_ASSERT(false);
# endif
}
/**
* Decompresses data, returns true if successful.
*/
bool
util_compress_inflate(const uint8_t *in_data, size_t in_data_size,
uint8_t *out_data, size_t out_data_size)
{
MESA_TRACE_FUNC();
#ifdef HAVE_ZSTD
size_t ret = ZSTD_decompress(out_data, out_data_size, in_data, in_data_size);
return !ZSTD_isError(ret);
#elif defined(HAVE_ZLIB)
z_stream strm;
/* allocate inflate state */
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;
strm.next_in = in_data;
strm.avail_in = in_data_size;
strm.next_out = out_data;
strm.avail_out = out_data_size;
int ret = inflateInit(&strm);
if (ret != Z_OK)
return false;
ret = inflate(&strm, Z_NO_FLUSH);
assert(ret != Z_STREAM_ERROR); /* state not clobbered */
/* Unless there was an error we should have decompressed everything in one
* go as we know the uncompressed file size.
*/
if (ret != Z_STREAM_END) {
(void)inflateEnd(&strm);
return false;
}
assert(strm.avail_out == 0);
/* clean up and return */
(void)inflateEnd(&strm);
return true;
#else
STATIC_ASSERT(false);
#endif
}
#endif
src/util/compress.h
+40
View File
@@ -0,0 +1,40 @@
/*
* Copyright © 2021 Valve Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
#ifdef HAVE_COMPRESSION
#include <stdbool.h>
#include <inttypes.h>
size_t
util_compress_max_compressed_len(size_t in_data_size);
bool
util_compress_inflate(const uint8_t *in_data, size_t in_data_size,
uint8_t *out_data, size_t out_data_size);
size_t
util_compress_deflate(const uint8_t *in_data, size_t in_data_size,
uint8_t *out_data, size_t out_buff_size);
#endif
src/util/crc32.c
+134
View File
@@ -0,0 +1,134 @@
/**************************************************************************
*
* Copyright 2008 VMware, Inc.
* All Rights Reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sub license, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice (including the
* next paragraph) shall be included in all copies or substantial portions
* of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
* IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
* ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*
**************************************************************************/
/**
* @file
* CRC32 implementation.
*
* @author Jose Fonseca
*/
#ifdef HAVE_ZLIB
#include <zlib.h>
#endif
#include "crc32.h"
static const uint32_t
util_crc32_table[256] = {
0x00000000, 0x77073096, 0xee0e612c, 0x990951ba,
0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3,
0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988,
0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91,
0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de,
0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7,
0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec,
0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5,
0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172,
0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b,
0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940,
0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59,
0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116,
0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f,
0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924,
0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d,
0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a,
0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433,
0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818,
0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01,
0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e,
0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457,
0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c,
0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65,
0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2,
0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb,
0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0,
0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9,
0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086,
0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f,
0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4,
0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad,
0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a,
0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683,
0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8,
0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1,
0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe,
0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7,
0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc,
0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5,
0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252,
0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b,
0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60,
0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79,
0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236,
0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f,
0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04,
0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d,
0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a,
0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713,
0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38,
0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21,
0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e,
0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777,
0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c,
0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45,
0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2,
0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db,
0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0,
0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9,
0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6,
0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf,
0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94,
0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d
};
/**
* @sa http://www.w3.org/TR/PNG/#D-CRCAppendix
*/
uint32_t
util_hash_crc32(const void *data, size_t size)
{
const uint8_t *p = data;
uint32_t crc = 0xffffffff;
#ifdef HAVE_ZLIB
/* Prefer zlib's implementation for better performance.
* zlib's uInt is always "unsigned int" while size_t can be 64bit.
* Since 1.2.9 there's crc32_z that takes size_t, but use the more
* available function to avoid build system complications.
*/
if ((uInt)size == size)
return ~crc32(0, data, size);
#endif
while (size--)
crc = util_crc32_table[(crc ^ *p++) & 0xff] ^ (crc >> 8);
return crc;
}
src/util/crc32.h
+55
View File
@@ -0,0 +1,55 @@
/**************************************************************************
*
* Copyright 2008 VMware, Inc.
* All Rights Reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sub license, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice (including the
* next paragraph) shall be included in all copies or substantial portions
* of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
* IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
* ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*
**************************************************************************/
/**
* @file
* CRC32 function.
*
* @author Jose Fonseca <[email protected]>
*/
#ifndef CRC32_H_
#define CRC32_H_
#include <stdlib.h>
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
uint32_t
util_hash_crc32(const void *data, size_t size);
#ifdef __cplusplus
}
#endif
#endif /* CRC32_H_ */
src/util/dag.c
+286
View File
@@ -0,0 +1,286 @@
/*
* Copyright © 2019 Broadcom
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
#include "util/set.h"
#include "util/dag.h"
#include <stdio.h>
static void
append_edge(struct dag_node *parent, struct dag_node *child, uintptr_t data)
{
/* Remove the child as a DAG head. */
list_delinit(&child->link);
struct dag_edge edge = {
.child = child,
.data = data,
};
util_dynarray_append(&parent->edges, edge);
child->parent_count++;
}
/**
* Adds a directed edge from the parent node to the child.
*
* Both nodes should have been initialized with dag_init_node(). The edge
* list may contain multiple edges to the same child with different data.
*/
void
dag_add_edge(struct dag_node *parent, struct dag_node *child, uintptr_t data)
{
util_dynarray_foreach(&parent->edges, struct dag_edge, edge) {
if (edge->child == child && edge->data == data)
return;
}
append_edge(parent, child, data);
}
/**
* Adds a directed edge from the parent node to the child.
*
* Both nodes should have been initialized with dag_init_node(). If there is
* already an existing edge, the data is updated to the maximum of the
* previous data and the new data. This is useful if the data represents a
* delay.
*/
void
dag_add_edge_max_data(struct dag_node *parent, struct dag_node *child,
uintptr_t data)
{
util_dynarray_foreach(&parent->edges, struct dag_edge, edge) {
if (edge->child == child) {
edge->data = MAX2(edge->data, data);
return;
}
}
append_edge(parent, child, data);
}
/* Removes a single edge from the graph, promoting the child to a DAG head.
*
* Note that calling this other than through dag_prune_head() means that you
* need to be careful when iterating the edges of remaining nodes for NULL
* children.
*/
void
dag_remove_edge(struct dag *dag, struct dag_edge *edge)
{
if (!edge->child)
return;
struct dag_node *child = edge->child;
child->parent_count--;
if (child->parent_count == 0)
list_addtail(&child->link, &dag->heads);
edge->child = NULL;
edge->data = 0;
}
/**
* Removes a DAG head from the graph, and moves any new dag heads into the
* heads list.
*/
void
dag_prune_head(struct dag *dag, struct dag_node *node)
{
assert(!node->parent_count);
list_delinit(&node->link);
util_dynarray_foreach(&node->edges, struct dag_edge, edge) {
dag_remove_edge(dag, edge);
}
}
/**
* Initializes DAG node (probably embedded in some other datastructure in the
* user).
*/
void
dag_init_node(struct dag *dag, struct dag_node *node)
{
util_dynarray_init(&node->edges, dag);
list_addtail(&node->link, &dag->heads);
}
struct dag_traverse_bottom_up_state {
struct set *seen;
void (*cb)(struct dag_node *node, void *data);
void *data;
};
static void
dag_traverse_bottom_up_node(struct dag_node *node,
struct dag_traverse_bottom_up_state *state)
{
if (_mesa_set_search(state->seen, node))
return;
struct util_dynarray stack = UTIL_DYNARRAY_INIT;
do {
assert(node);
while (node->edges.size != 0) {
util_dynarray_append(&stack, node);
/* Push unprocessed children onto stack in reverse order. Note that
* it's possible for any of the children nodes to already be on the
* stack.
*/
util_dynarray_foreach_reverse(&node->edges, struct dag_edge, edge) {
if (!_mesa_set_search(state->seen, edge->child)) {
util_dynarray_append(&stack, edge->child);
}
}
/* Get last element pushed: either left-most child or current node.
* If it's the current node, that means that we've processed all its
* children already.
*/
struct dag_node *top = util_dynarray_pop(&stack, struct dag_node *);
if (top == node)
break;
node = top;
}
/* Process the node */
state->cb(node, state->data);
_mesa_set_add(state->seen, node);
/* Find the next unprocessed node in the stack */
do {
node = NULL;
if (stack.size == 0)
break;
node = util_dynarray_pop(&stack, struct dag_node *);
} while (_mesa_set_search(state->seen, node));
} while (node);
util_dynarray_fini(&stack);
}
/**
* Walks the DAG from leaves to the root, ensuring that each node is only seen
* once its children have been, and each node is only traversed once.
*/
void
dag_traverse_bottom_up(struct dag *dag, void (*cb)(struct dag_node *node,
void *data), void *data)
{
struct dag_traverse_bottom_up_state state = {
.seen = _mesa_pointer_set_create(NULL),
.data = data,
.cb = cb,
};
list_for_each_entry(struct dag_node, node, &dag->heads, link) {
dag_traverse_bottom_up_node(node, &state);
}
ralloc_free(state.seen);
}
/**
* Creates an empty DAG datastructure.
*/
struct dag *
dag_create(void *mem_ctx)
{
struct dag *dag = rzalloc(mem_ctx, struct dag);
list_inithead(&dag->heads);
return dag;
}
struct dag_validate_state {
struct util_dynarray stack;
struct set *stack_set;
struct set *seen;
void (*cb)(const struct dag_node *node, void *data);
void *data;
};
static void
dag_validate_node(struct dag_node *node,
struct dag_validate_state *state)
{
if (_mesa_set_search(state->stack_set, node)) {
fprintf(stderr, "DAG validation failed at:\n");
fprintf(stderr, " %p: ", node);
state->cb(node, state->data);
fprintf(stderr, "\n");
fprintf(stderr, "Nodes in stack:\n");
util_dynarray_foreach(&state->stack, struct dag_node *, nodep) {
struct dag_node *node = *nodep;
fprintf(stderr, " %p: ", node);
state->cb(node, state->data);
fprintf(stderr, "\n");
}
abort();
}
if (_mesa_set_search(state->seen, node))
return;
_mesa_set_add(state->stack_set, node);
_mesa_set_add(state->seen, node);
util_dynarray_append(&state->stack, node);
util_dynarray_foreach(&node->edges, struct dag_edge, edge) {
dag_validate_node(edge->child, state);
}
(void)util_dynarray_pop(&state->stack, struct dag_node *);
_mesa_set_remove_key(state->stack_set, node);
}
void
dag_validate(struct dag *dag, void (*cb)(const struct dag_node *node,
void *data),
void *data)
{
void *mem_ctx = ralloc_context(NULL);
struct dag_validate_state state = {
.stack_set = _mesa_pointer_set_create(mem_ctx),
.seen = _mesa_pointer_set_create(mem_ctx),
.cb = cb,
.data = data,
};
util_dynarray_init(&state.stack, mem_ctx);
list_validate(&dag->heads);
list_for_each_entry(struct dag_node, node, &dag->heads, link) {
dag_validate_node(node, &state);
}
ralloc_free(mem_ctx);
}
src/util/dag.h
+68
View File
@@ -0,0 +1,68 @@
/*
* Copyright © 2019 Broadcom
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
#ifndef DAG_H
#define DAG_H
#include <stdint.h>
#include "util/list.h"
#include "util/u_dynarray.h"
#ifdef __cplusplus
extern "C" {
#endif
struct dag_edge {
struct dag_node *child;
/* User-defined data associated with the edge. */
uintptr_t data;
};
struct dag_node {
/* Position in the DAG heads list (or a self-link) */
struct list_head link;
/* Array struct edge to the children. */
struct util_dynarray edges;
uint32_t parent_count;
};
struct dag {
struct list_head heads;
};
struct dag *dag_create(void *mem_ctx);
void dag_init_node(struct dag *dag, struct dag_node *node);
void dag_add_edge(struct dag_node *parent, struct dag_node *child, uintptr_t data);
void dag_add_edge_max_data(struct dag_node *parent, struct dag_node *child, uintptr_t data);
void dag_remove_edge(struct dag *dag, struct dag_edge *edge);
void dag_traverse_bottom_up(struct dag *dag, void (*cb)(struct dag_node *node,
void *data), void *data);
void dag_prune_head(struct dag *dag, struct dag_node *node);
void dag_validate(struct dag *dag, void (*cb)(const struct dag_node *node,
void *data), void *data);
#ifdef __cplusplus
}
#endif
#endif
src/util/detect.h
+68
View File
@@ -0,0 +1,68 @@
/**************************************************************************
*
* Copyright 2022 Yonggang Luo
* All Rights Reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sub license, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice (including the
* next paragraph) shall be included in all copies or substantial portions
* of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
* IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
* ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*
**************************************************************************/
/**
* @file
* Mesa configuration defines.
*
* This header file sets several defines based on the compiler, processor
* architecture, and operating system being used. These defines are comes
* from corresponding headers.
*
* See:
* - http://gcc.gnu.org/onlinedocs/cpp/Common-Predefined-Macros.html
* - echo | gcc -dM -E - | sort
* - http://msdn.microsoft.com/en-us/library/b0084kay.aspx
* - https://sourceforge.net/p/predef/wiki/Home/
*/
#ifndef UTIL_DETECT_H_
#define UTIL_DETECT_H_
#include <limits.h>
/*
* Compiler detection
*/
#include "util/detect_cc.h"
/*
* Processor architecture detection
*/
#include "util/detect_arch.h"
/*
* Endian detection detection
*/
#include "util/u_endian.h"
/*
* Operating system family detection
*/
#include "util/detect_os.h"
#endif /* UTIL_DETECT_H_ */
src/util/detect_arch.h
+201
View File
@@ -0,0 +1,201 @@
/**************************************************************************
*
* Copyright 2008 VMware, Inc.
* All Rights Reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sub license, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice (including the
* next paragraph) shall be included in all copies or substantial portions
* of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
* IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
* ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*
**************************************************************************/
/**
* @file
* Gallium configuration defines.
*
* This header file sets several defines based on the compiler, processor
* architecture, and operating system being used. These defines should be used
* throughout the code to facilitate porting to new platforms. It is likely that
* this file is auto-generated by an autoconf-like tool at some point, as some
* things cannot be determined by pre-defined environment alone.
*
* See also:
* - http://gcc.gnu.org/onlinedocs/cpp/Common-Predefined-Macros.html
* - echo | gcc -dM -E - | sort
* - http://msdn.microsoft.com/en-us/library/b0084kay.aspx
*
* @author José Fonseca <[email protected]>
*/
#ifndef UTIL_DETECT_ARCH_H_
#define UTIL_DETECT_ARCH_H_
#include <limits.h>
#include "util/detect_cc.h"
/*
* Processor architecture
*/
#if defined(__i386__) /* gcc */ || defined(_M_IX86) /* msvc */ || defined(_X86_) || defined(__386__) || defined(i386) || defined(__i386) /* Sun cc */
#define DETECT_ARCH_X86 1
#endif
#if defined(__x86_64__) /* gcc */ || \
((defined(_M_X64) || defined(_M_AMD64)) && !defined(_M_ARM64EC)) /* msvc */ || \
defined(__x86_64) /* Sun cc */
#define DETECT_ARCH_X86_64 1
#endif
#if DETECT_ARCH_X86 || DETECT_ARCH_X86_64
#if DETECT_CC_GCC && !defined(__SSE2__)
/* #warning SSE2 support requires -msse -msse2 compiler options */
#else
#define DETECT_ARCH_SSE 1
#endif
#endif
#if defined(__ppc__) || defined(__ppc64__) || defined(__PPC__) || defined(__PPC64__)
#define DETECT_ARCH_PPC 1
#if defined(__ppc64__) || defined(__PPC64__)
#define DETECT_ARCH_PPC_64 1
#endif
#endif
#if defined(__s390x__)
#define DETECT_ARCH_S390 1
#endif
#if defined(__arm__)
#define DETECT_ARCH_ARM 1
#endif
#if defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC)
#define DETECT_ARCH_AARCH64 1
#endif
#if defined(_M_ARM64EC)
#define DETECT_ARCH_ARM64EC 1
#endif
#if defined(__mips64) && defined(__LP64__)
#define DETECT_ARCH_MIPS64 1
#endif
#if defined(__mips__)
#define DETECT_ARCH_MIPS 1
#endif
#if defined(__hppa__)
#define DETECT_ARCH_HPPA 1
#endif
#if defined(__riscv)
#define DETECT_ARCH_RISCV 1
#if __riscv_xlen == 64
#define DETECT_ARCH_RISCV64 1
#elif __riscv_xlen == 32
#define DETECT_ARCH_RISCV32 1
#else
#error "detect_arch: unknown target riscv xlen"
#endif
#endif
#if defined(__loongarch__)
#ifdef __loongarch_lp64
#define DETECT_ARCH_LOONGARCH64 1
#else
#error "detect_arch: unknown target loongarch base ABI type"
#endif
#endif
#if defined(__sparc__) || defined(__sparc)
#define DETECT_ARCH_SPARC64 1
#endif
#ifndef DETECT_ARCH_X86
#define DETECT_ARCH_X86 0
#endif
#ifndef DETECT_ARCH_X86_64
#define DETECT_ARCH_X86_64 0
#endif
#ifndef DETECT_ARCH_SSE
#define DETECT_ARCH_SSE 0
#endif
#ifndef DETECT_ARCH_PPC
#define DETECT_ARCH_PPC 0
#endif
#ifndef DETECT_ARCH_PPC_64
#define DETECT_ARCH_PPC_64 0
#endif
#ifndef DETECT_ARCH_S390
#define DETECT_ARCH_S390 0
#endif
#ifndef DETECT_ARCH_ARM
#define DETECT_ARCH_ARM 0
#endif
#ifndef DETECT_ARCH_AARCH64
#define DETECT_ARCH_AARCH64 0
#endif
#ifndef DETECT_ARCH_ARM64EC
#define DETECT_ARCH_ARM64EC 0
#endif
#ifndef DETECT_ARCH_MIPS64
#define DETECT_ARCH_MIPS64 0
#endif
#ifndef DETECT_ARCH_MIPS
#define DETECT_ARCH_MIPS 0
#endif
#ifndef DETECT_ARCH_HPPA
#define DETECT_ARCH_HPPA 0
#endif
#ifndef DETECT_ARCH_RISCV
#define DETECT_ARCH_RISCV 0
#endif
#ifndef DETECT_ARCH_RISCV32
#define DETECT_ARCH_RISCV32 0
#endif
#ifndef DETECT_ARCH_RISCV64
#define DETECT_ARCH_RISCV64 0
#endif
#ifndef DETECT_ARCH_LOONGARCH64
#define DETECT_ARCH_LOONGARCH64 0
#endif
#ifndef DETECT_ARCH_SPARC64
#define DETECT_ARCH_SPARC64 0
#endif
#endif /* UTIL_DETECT_ARCH_H_ */
src/util/detect_cc.h
+88
View File
@@ -0,0 +1,88 @@
/**************************************************************************
*
* Copyright 2022 Yonggang Luo
* All Rights Reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sub license, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice (including the
* next paragraph) shall be included in all copies or substantial portions
* of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
* IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
* ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*
**************************************************************************/
/**
* @file
* Compiler configuration defines.
*
* See:
* - http://gcc.gnu.org/onlinedocs/cpp/Common-Predefined-Macros.html
* - echo | gcc -dM -E - | sort
* - http://msdn.microsoft.com/en-us/library/b0084kay.aspx
* - https://sourceforge.net/p/predef/wiki/Home/
*/
#ifndef UTIL_DETECT_CC_H_
#define UTIL_DETECT_CC_H_
/*
* Compiler
*/
#if defined(__GNUC__)
#define DETECT_CC_GCC 1
#define DETECT_CC_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
#endif
/*
* Meaning of _MSC_VER value:
* - 1800: Visual Studio 2013
* - 1700: Visual Studio 2012
* - 1600: Visual Studio 2010
* - 1500: Visual Studio 2008
* - 1400: Visual C++ 2005
* - 1310: Visual C++ .NET 2003
* - 1300: Visual C++ .NET 2002
*
* __MSC__ seems to be an old macro -- it is not pre-defined on recent MSVC
* versions.
*/
#if defined(_MSC_VER) || defined(__MSC__)
#define DETECT_CC_MSVC 1
#endif
#if defined(__ICL)
#define DETECT_CC_ICL 1
#endif
#ifndef DETECT_CC_GCC
#define DETECT_CC_GCC 0
#endif
#ifndef DETECT_CC_GCC_VERSION
#define DETECT_CC_GCC_VERSION 0
#endif
#ifndef DETECT_CC_MSVC
#define DETECT_CC_MSVC 0
#endif
#ifndef DETECT_CC_ICL
#define DETECT_CC_ICL 0
#endif
#endif /* UTIL_DETECT_CC_H_ */
src/util/detect_os.h
+150
View File
@@ -0,0 +1,150 @@
/* SPDX-License-Identifier: MIT */
/* Copyright 2008 VMware, Inc. */
/**
* Auto-detect the operating system family.
*
* See also:
* - http://gcc.gnu.org/onlinedocs/cpp/Common-Predefined-Macros.html
* - echo | gcc -dM -E - | sort
* - http://msdn.microsoft.com/en-us/library/b0084kay.aspx
*
* @author José Fonseca <[email protected]>
*/
#ifndef DETECT_OS_H
#define DETECT_OS_H
#if defined(__linux__)
#define DETECT_OS_LINUX 1
#define DETECT_OS_POSIX 1
#endif
/*
* Android defines __linux__, so DETECT_OS_LINUX and DETECT_OS_POSIX will
* also be defined.
*/
#if defined(__ANDROID__)
#define DETECT_OS_ANDROID 1
#endif
#if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
#define DETECT_OS_FREEBSD 1
#define DETECT_OS_BSD 1
#define DETECT_OS_POSIX 1
#endif
#if defined(__OpenBSD__)
#define DETECT_OS_OPENBSD 1
#define DETECT_OS_BSD 1
#define DETECT_OS_POSIX 1
#endif
#if defined(__NetBSD__)
#define DETECT_OS_NETBSD 1
#define DETECT_OS_BSD 1
#define DETECT_OS_POSIX 1
#endif
#if defined(__DragonFly__)
#define DETECT_OS_DRAGONFLY 1
#define DETECT_OS_BSD 1
#define DETECT_OS_POSIX 1
#endif
#if defined(__Fuchsia__)
#define DETECT_OS_FUCHSIA 1
#define DETECT_OS_POSIX_LITE 1
#endif
#if defined(__GNU__)
#define DETECT_OS_HURD 1
#define DETECT_OS_POSIX 1
#endif
#if defined(__sun)
#define DETECT_OS_SOLARIS 1
#define DETECT_OS_POSIX 1
#endif
#if defined(__APPLE__)
#define DETECT_OS_APPLE 1
#define DETECT_OS_POSIX 1
#endif
#if defined(_WIN32)
#define DETECT_OS_WINDOWS 1
#endif
#if defined(__HAIKU__)
#define DETECT_OS_HAIKU 1
#define DETECT_OS_POSIX 1
#endif
#if defined(__CYGWIN__)
#define DETECT_OS_CYGWIN 1
#define DETECT_OS_POSIX 1
#endif
#if defined(__managarm__)
#define DETECT_OS_MANAGARM 1
#define DETECT_OS_POSIX 1
#endif
/*
* Make sure DETECT_OS_* are always defined, so that they can be used with #if
*/
#ifndef DETECT_OS_ANDROID
#define DETECT_OS_ANDROID 0
#endif
#ifndef DETECT_OS_APPLE
#define DETECT_OS_APPLE 0
#endif
#ifndef DETECT_OS_BSD
#define DETECT_OS_BSD 0
#endif
#ifndef DETECT_OS_CYGWIN
#define DETECT_OS_CYGWIN 0
#endif
#ifndef DETECT_OS_DRAGONFLY
#define DETECT_OS_DRAGONFLY 0
#endif
#ifndef DETECT_OS_FREEBSD
#define DETECT_OS_FREEBSD 0
#endif
#ifndef DETECT_OS_HAIKU
#define DETECT_OS_HAIKU 0
#endif
#ifndef DETECT_OS_FUCHSIA
#define DETECT_OS_FUCHSIA 0
#endif
#ifndef DETECT_OS_HURD
#define DETECT_OS_HURD 0
#endif
#ifndef DETECT_OS_LINUX
#define DETECT_OS_LINUX 0
#endif
#ifndef DETECT_OS_NETBSD
#define DETECT_OS_NETBSD 0
#endif
#ifndef DETECT_OS_OPENBSD
#define DETECT_OS_OPENBSD 0
#endif
#ifndef DETECT_OS_SOLARIS
#define DETECT_OS_SOLARIS 0
#endif
#ifndef DETECT_OS_POSIX
#define DETECT_OS_POSIX 0
#endif
#ifndef DETECT_OS_POSIX_LITE
#define DETECT_OS_POSIX_LITE DETECT_OS_POSIX
#endif
#ifndef DETECT_OS_WINDOWS
#define DETECT_OS_WINDOWS 0
#endif
#ifndef DETECT_OS_MANAGARM
#define DETECT_OS_MANAGARM 0
#endif
#endif /* DETECT_OS_H */
src/util/disk_cache.c
+705
View File
@@ -0,0 +1,705 @@
/*
* Copyright © 2014 Intel Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
#ifdef ENABLE_SHADER_CACHE
#include <ctype.h>
#include <ftw.h>
#include <string.h>
#include <stdlib.h>
#include <stdio.h>
#include <sys/file.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/mman.h>
#include <fcntl.h>
#include <errno.h>
#include <dirent.h>
#include <inttypes.h>
#include "util/compress.h"
#include "util/crc32.h"
#include "util/u_debug.h"
#include "util/rand_xor.h"
#include "util/u_atomic.h"
#include "util/mesa-blake3.h"
#include "util/perf/cpu_trace.h"
#include "util/ralloc.h"
#include "util/compiler.h"
#include "util/log.h"
#include "disk_cache.h"
#include "disk_cache_os.h"
/* The cache version should be bumped whenever a change is made to the
* structure of cache entries or the index. This will give any 3rd party
* applications reading the cache entries a chance to adjust to the changes.
*
* - The cache version is checked internally when reading a cache entry. If we
* ever have a mismatch we are in big trouble as this means we had a cache
* collision. In case of such an event please check the skys for giant
* asteroids and that the entire Mesa team hasn't been eaten by wolves.
*
* - There is no strict requirement that cache versions be backwards
* compatible but effort should be taken to limit disruption where possible.
*/
#define CACHE_VERSION 1
#define DRV_KEY_CPY(_dst, _src, _src_size) \
do { \
memcpy(_dst, _src, _src_size); \
_dst += _src_size; \
} while (0);
static bool
disk_cache_init_queue(struct disk_cache *cache)
{
if (util_queue_is_initialized(&cache->cache_queue))
return true;
/* 4 threads were chosen below because just about all modern CPUs currently
* available that run Mesa have *at least* 4 cores. For these CPUs allowing
* more threads can result in the queue being processed faster, thus
* avoiding excessive memory use due to a backlog of cache entrys building
* up in the queue. Since we set the UTIL_QUEUE_INIT_USE_MINIMUM_PRIORITY
* flag this should have little negative impact on low core systems.
*
* The queue will resize automatically when it's full, so adding new jobs
* doesn't stall.
*/
return util_queue_init(&cache->cache_queue, "disk$", 32, 4,
UTIL_QUEUE_INIT_RESIZE_IF_FULL |
UTIL_QUEUE_INIT_USE_MINIMUM_PRIORITY |
UTIL_QUEUE_INIT_SET_FULL_THREAD_AFFINITY, NULL);
}
static struct disk_cache *
disk_cache_type_create(const char *gpu_name,
const char *driver_id,
const char *cache_dir_name,
uint64_t driver_flags,
enum disk_cache_type cache_type,
uint64_t max_size)
{
void *local;
struct disk_cache *cache = NULL;
uint8_t cache_version = CACHE_VERSION;
size_t cv_size = sizeof(cache_version);
/* A ralloc context for transient data during this invocation. */
local = ralloc_context(NULL);
if (local == NULL)
goto fail;
cache = rzalloc(NULL, struct disk_cache);
if (cache == NULL)
goto fail;
/* Assume failure. */
cache->path_init_failed = true;
cache->type = DISK_CACHE_NONE;
if (!disk_cache_enabled())
goto path_fail;
const char *path =
disk_cache_generate_cache_dir(local, gpu_name, driver_id, cache_dir_name, cache_type, true);
if (!path)
goto path_fail;
cache->path = ralloc_strdup(cache, path);
if (cache->path == NULL)
goto path_fail;
/* Cache tests that want to have a disabled cache compression are using
* the "make_check_uncompressed" for the driver_id name. Hence here we
* disable disk cache compression when mesa's build tests require it.
*/
if (strcmp(driver_id, "make_check_uncompressed") == 0)
cache->compression_disabled = true;
if (cache_type == DISK_CACHE_SINGLE_FILE) {
if (!disk_cache_load_cache_index_foz(local, cache))
goto path_fail;
} else if (cache_type == DISK_CACHE_DATABASE) {
if (!disk_cache_db_load_cache_index(local, cache))
goto path_fail;
}
if (!os_get_option("MESA_SHADER_CACHE_DIR") && !os_get_option("MESA_GLSL_CACHE_DIR"))
disk_cache_touch_cache_user_marker(cache->path);
cache->type = cache_type;
cache->stats.enabled = debug_get_bool_option("MESA_SHADER_CACHE_SHOW_STATS",
false);
if (!disk_cache_mmap_cache_index(local, cache))
goto path_fail;
cache->max_size = max_size;
if (cache->type == DISK_CACHE_DATABASE)
mesa_cache_db_multipart_set_size_limit(&cache->cache_db, cache->max_size);
if (!disk_cache_init_queue(cache))
goto fail;
cache->path_init_failed = false;
path_fail:
cache->driver_keys_blob_size = cv_size;
/* Create driver id keys */
size_t id_size = strlen(driver_id) + 1;
size_t gpu_name_size = strlen(gpu_name) + 1;
cache->driver_keys_blob_size += id_size;
cache->driver_keys_blob_size += gpu_name_size;
/* We sometimes store entire structs that contains a pointers in the cache,
* use pointer size as a key to avoid hard to debug issues.
*/
uint8_t ptr_size = sizeof(void *);
size_t ptr_size_size = sizeof(ptr_size);
cache->driver_keys_blob_size += ptr_size_size;
size_t driver_flags_size = sizeof(driver_flags);
cache->driver_keys_blob_size += driver_flags_size;
cache->driver_keys_blob =
ralloc_size(cache, cache->driver_keys_blob_size);
if (!cache->driver_keys_blob)
goto fail;
uint8_t *drv_key_blob = cache->driver_keys_blob;
DRV_KEY_CPY(drv_key_blob, &cache_version, cv_size)
DRV_KEY_CPY(drv_key_blob, driver_id, id_size)
DRV_KEY_CPY(drv_key_blob, gpu_name, gpu_name_size)
DRV_KEY_CPY(drv_key_blob, &ptr_size, ptr_size_size)
DRV_KEY_CPY(drv_key_blob, &driver_flags, driver_flags_size)
/* Seed our rand function */
s_rand_xorshift128plus(cache->seed_xorshift128plus, true);
ralloc_free(local);
return cache;
fail:
ralloc_free(cache);
ralloc_free(local);
return NULL;
}
struct disk_cache *
disk_cache_create(const char *gpu_name, const char *driver_id,
uint64_t driver_flags)
{
enum disk_cache_type cache_type;
struct disk_cache *cache;
uint64_t max_size = 0;
const char *max_size_str;
if (debug_get_bool_option("MESA_DISK_CACHE_SINGLE_FILE", false)) {
cache_type = DISK_CACHE_SINGLE_FILE;
} else if (debug_get_bool_option("MESA_DISK_CACHE_DATABASE", false)) {
cache_type = DISK_CACHE_DATABASE;
/* Since switching the default cache to <mesa_shader_cache_db>, remove the
* old cache folder if it hasn't been modified for more than 7 days.
*/
if (!os_get_option("MESA_SHADER_CACHE_DIR") && !os_get_option("MESA_GLSL_CACHE_DIR") &&
disk_cache_enabled())
disk_cache_delete_old_cache();
} else if (debug_get_bool_option("MESA_DISK_CACHE_MULTI_FILE", true)) {
cache_type = DISK_CACHE_MULTI_FILE;
} else {
return NULL;
}
max_size_str = os_get_option("MESA_SHADER_CACHE_MAX_SIZE");
if (!max_size_str) {
max_size_str = os_get_option("MESA_GLSL_CACHE_MAX_SIZE");
if (max_size_str)
fprintf(stderr,
"*** MESA_GLSL_CACHE_MAX_SIZE is deprecated; "
"use MESA_SHADER_CACHE_MAX_SIZE instead ***\n");
}
#ifdef MESA_SHADER_CACHE_MAX_SIZE
if (!max_size_str) {
max_size_str = MESA_SHADER_CACHE_MAX_SIZE;
}
#endif
if (max_size_str) {
char *end;
max_size = strtoul(max_size_str, &end, 10);
if (end == max_size_str) {
max_size = 0;
} else {
switch (*end) {
case 'K':
case 'k':
max_size *= 1024;
break;
case 'M':
case 'm':
max_size *= 1024*1024;
break;
case '\0':
case 'G':
case 'g':
default:
max_size *= 1024*1024*1024;
break;
}
}
}
/* Default to 1GB for maximum cache size. */
if (max_size == 0) {
max_size = 1024*1024*1024;
}
/* Create main writable cache. */
cache = disk_cache_type_create(gpu_name, driver_id, NULL, driver_flags,
cache_type, max_size);
if (!cache)
return NULL;
/* If MESA_DISK_CACHE_SINGLE_FILE is unset and MESA_DISK_CACHE_COMBINE_RW_WITH_RO_FOZ
* is set, then enable additional Fossilize RO caches together with the RW
* cache. At first we will check cache entry presence in the RO caches and
* if entry isn't found there, then we'll fall back to the RW cache.
*/
if (cache_type != DISK_CACHE_SINGLE_FILE && !cache->path_init_failed &&
debug_get_bool_option("MESA_DISK_CACHE_COMBINE_RW_WITH_RO_FOZ", false)) {
/* Create read-only cache used for sharing prebuilt shaders.
* If cache entry will be found in this cache, then the main cache
* will be bypassed.
*/
cache->foz_ro_cache = disk_cache_type_create(gpu_name, driver_id, NULL,
driver_flags,
DISK_CACHE_SINGLE_FILE,
max_size);
}
return cache;
}
struct disk_cache *
disk_cache_create_custom(const char *gpu_name, const char *driver_id,
uint64_t driver_flags, const char *cache_dir_name,
uint32_t max_size)
{
return disk_cache_type_create(gpu_name, driver_id, cache_dir_name, 0,
DISK_CACHE_DATABASE, max_size);
}
void
disk_cache_destroy(struct disk_cache *cache)
{
if (unlikely(cache && cache->stats.enabled)) {
mesa_logi("disk shader cache: hits = %u, misses = %u\n",
cache->stats.hits,
cache->stats.misses);
}
if (cache && util_queue_is_initialized(&cache->cache_queue)) {
util_queue_finish(&cache->cache_queue);
util_queue_destroy(&cache->cache_queue);
if (cache->foz_ro_cache)
disk_cache_destroy(cache->foz_ro_cache);
if (cache->type == DISK_CACHE_SINGLE_FILE)
foz_destroy(&cache->foz_db);
if (cache->type == DISK_CACHE_DATABASE)
mesa_cache_db_multipart_close(&cache->cache_db);
disk_cache_destroy_mmap(cache);
}
ralloc_free(cache);
}
void
disk_cache_wait_for_idle(struct disk_cache *cache)
{
util_queue_finish(&cache->cache_queue);
}
void
disk_cache_remove(struct disk_cache *cache, const cache_key key)
{
if (cache->type == DISK_CACHE_DATABASE) {
mesa_cache_db_multipart_entry_remove(&cache->cache_db, key);
return;
}
char *filename = disk_cache_get_cache_filename(cache, key);
if (filename == NULL) {
return;
}
disk_cache_evict_item(cache, filename);
}
static struct disk_cache_put_job *
create_put_job(struct disk_cache *cache, const cache_key key,
void *data, size_t size,
struct cache_item_metadata *cache_item_metadata,
bool take_ownership)
{
struct disk_cache_put_job *dc_job = (struct disk_cache_put_job *)
malloc(sizeof(struct disk_cache_put_job) + (take_ownership ? 0 : size));
if (dc_job) {
dc_job->cache = cache;
memcpy(dc_job->key, key, sizeof(cache_key));
if (take_ownership) {
dc_job->data = data;
} else {
dc_job->data = dc_job + 1;
memcpy(dc_job->data, data, size);
}
dc_job->size = size;
/* Copy the cache item metadata */
if (cache_item_metadata) {
dc_job->cache_item_metadata.type = cache_item_metadata->type;
if (cache_item_metadata->type == CACHE_ITEM_TYPE_GLSL) {
dc_job->cache_item_metadata.num_keys =
cache_item_metadata->num_keys;
dc_job->cache_item_metadata.keys = (cache_key *)
malloc(cache_item_metadata->num_keys * sizeof(cache_key));
if (!dc_job->cache_item_metadata.keys)
goto fail;
memcpy(dc_job->cache_item_metadata.keys,
cache_item_metadata->keys,
sizeof(cache_key) * cache_item_metadata->num_keys);
}
} else {
dc_job->cache_item_metadata.type = CACHE_ITEM_TYPE_UNKNOWN;
dc_job->cache_item_metadata.keys = NULL;
}
}
return dc_job;
fail:
free(dc_job);
return NULL;
}
static void
destroy_put_job(void *job, void *gdata, int thread_index)
{
if (job) {
struct disk_cache_put_job *dc_job = (struct disk_cache_put_job *) job;
free(dc_job->cache_item_metadata.keys);
free(job);
}
}
static void
destroy_put_job_nocopy(void *job, void *gdata, int thread_index)
{
struct disk_cache_put_job *dc_job = (struct disk_cache_put_job *) job;
free(dc_job->data);
destroy_put_job(job, gdata, thread_index);
}
static void
blob_put_compressed(struct disk_cache *cache, const cache_key key,
const void *data, size_t size);
static void
cache_put(void *job, void *gdata, int thread_index)
{
assert(job);
unsigned i = 0;
char *filename = NULL;
struct disk_cache_put_job *dc_job = (struct disk_cache_put_job *) job;
if (dc_job->cache->blob_put_cb) {
blob_put_compressed(dc_job->cache, dc_job->key, dc_job->data, dc_job->size);
} else if (dc_job->cache->type == DISK_CACHE_SINGLE_FILE) {
disk_cache_write_item_to_disk_foz(dc_job);
} else if (dc_job->cache->type == DISK_CACHE_DATABASE) {
disk_cache_db_write_item_to_disk(dc_job);
} else if (dc_job->cache->type == DISK_CACHE_MULTI_FILE) {
filename = disk_cache_get_cache_filename(dc_job->cache, dc_job->key);
if (filename == NULL)
goto done;
/* If the cache is too large, evict something else first. */
while (p_atomic_read_relaxed(&dc_job->cache->size->value) + dc_job->size > dc_job->cache->max_size &&
i < 8) {
disk_cache_evict_lru_item(dc_job->cache);
i++;
}
disk_cache_write_item_to_disk(dc_job, filename);
done:
free(filename);
}
}
struct blob_cache_entry {
uint32_t uncompressed_size;
uint8_t compressed_data[];
};
static void
blob_put_compressed(struct disk_cache *cache, const cache_key key,
const void *data, size_t size)
{
MESA_TRACE_FUNC();
size_t max_buf = util_compress_max_compressed_len(size);
struct blob_cache_entry *entry = malloc(max_buf + sizeof(*entry));
if (!entry)
goto out;
entry->uncompressed_size = size;
size_t compressed_size =
util_compress_deflate(data, size, entry->compressed_data, max_buf);
if (!compressed_size)
goto out;
unsigned entry_size = compressed_size + sizeof(*entry);
// The curly brackets are here to only trace the blob_put_cb call
{
MESA_TRACE_SCOPE("blob_put");
cache->blob_put_cb(key, CACHE_KEY_SIZE, entry, entry_size);
}
out:
free(entry);
}
static void *
blob_get_compressed(struct disk_cache *cache, const cache_key key,
size_t *size)
{
MESA_TRACE_FUNC();
/* This is what Android EGL defines as the maxValueSize in egl_cache_t
* class implementation.
*/
const signed long max_blob_size = 64 * 1024;
struct blob_cache_entry *entry = malloc(max_blob_size);
if (!entry)
return NULL;
signed long entry_size;
// The curly brackets are here to only trace the blob_get_cb call
{
MESA_TRACE_SCOPE("blob_get");
entry_size = cache->blob_get_cb(key, CACHE_KEY_SIZE, entry, max_blob_size);
}
if (!entry_size) {
free(entry);
return NULL;
}
void *data = malloc(entry->uncompressed_size);
if (!data) {
free(entry);
return NULL;
}
unsigned compressed_size = entry_size - sizeof(*entry);
bool ret = util_compress_inflate(entry->compressed_data, compressed_size,
data, entry->uncompressed_size);
if (!ret) {
free(data);
free(entry);
return NULL;
}
if (size)
*size = entry->uncompressed_size;
free(entry);
return data;
}
void
disk_cache_put(struct disk_cache *cache, const cache_key key,
const void *data, size_t size,
struct cache_item_metadata *cache_item_metadata)
{
if (!util_queue_is_initialized(&cache->cache_queue))
return;
struct disk_cache_put_job *dc_job =
create_put_job(cache, key, (void*)data, size, cache_item_metadata, false);
if (dc_job) {
util_queue_fence_init(&dc_job->fence);
util_queue_add_job(&cache->cache_queue, dc_job, &dc_job->fence,
cache_put, destroy_put_job, dc_job->size);
}
}
void
disk_cache_put_nocopy(struct disk_cache *cache, const cache_key key,
void *data, size_t size,
struct cache_item_metadata *cache_item_metadata)
{
if (!util_queue_is_initialized(&cache->cache_queue)) {
free(data);
return;
}
struct disk_cache_put_job *dc_job =
create_put_job(cache, key, data, size, cache_item_metadata, true);
if (dc_job) {
util_queue_fence_init(&dc_job->fence);
util_queue_add_job(&cache->cache_queue, dc_job, &dc_job->fence,
cache_put, destroy_put_job_nocopy, dc_job->size);
}
}
void *
disk_cache_get(struct disk_cache *cache, const cache_key key, size_t *size)
{
void *buf = NULL;
if (size)
*size = 0;
if (cache->foz_ro_cache)
buf = disk_cache_load_item_foz(cache->foz_ro_cache, key, size);
if (!buf) {
if (cache->blob_get_cb) {
buf = blob_get_compressed(cache, key, size);
} else if (cache->type == DISK_CACHE_SINGLE_FILE) {
buf = disk_cache_load_item_foz(cache, key, size);
} else if (cache->type == DISK_CACHE_DATABASE) {
buf = disk_cache_db_load_item(cache, key, size);
} else if (cache->type == DISK_CACHE_MULTI_FILE) {
char *filename = disk_cache_get_cache_filename(cache, key);
if (filename)
buf = disk_cache_load_item(cache, filename, size);
}
}
if (unlikely(cache->stats.enabled)) {
if (buf)
p_atomic_inc(&cache->stats.hits);
else
p_atomic_inc(&cache->stats.misses);
}
return buf;
}
void
disk_cache_put_key(struct disk_cache *cache, const cache_key key)
{
const uint32_t *key_chunk = (const uint32_t *) key;
int i = CPU_TO_LE32(*key_chunk) & CACHE_INDEX_KEY_MASK;
unsigned char *entry;
if (cache->blob_put_cb) {
cache->blob_put_cb(key, CACHE_KEY_SIZE, key_chunk, sizeof(uint32_t));
return;
}
if (cache->path_init_failed)
return;
entry = &cache->stored_keys[i * CACHE_KEY_SIZE];
memcpy(entry, key, CACHE_KEY_SIZE);
}
/* This function lets us test whether a given key was previously
* stored in the cache with disk_cache_put_key(). The implement is
* efficient by not using syscalls or hitting the disk. It's not
* race-free, but the races are benign. If we race with someone else
* calling disk_cache_put_key, then that's just an extra cache miss and an
* extra recompile.
*/
bool
disk_cache_has_key(struct disk_cache *cache, const cache_key key)
{
const uint32_t *key_chunk = (const uint32_t *) key;
int i = CPU_TO_LE32(*key_chunk) & CACHE_INDEX_KEY_MASK;
unsigned char *entry;
if (cache->blob_get_cb) {
uint32_t blob;
return cache->blob_get_cb(key, CACHE_KEY_SIZE, &blob, sizeof(uint32_t));
}
if (cache->path_init_failed)
return false;
entry = &cache->stored_keys[i * CACHE_KEY_SIZE];
return memcmp(entry, key, CACHE_KEY_SIZE) == 0;
}
void
disk_cache_compute_key(struct disk_cache *cache, const void *data, size_t size,
cache_key key)
{
blake3_hasher ctx;
_mesa_blake3_init(&ctx);
_mesa_blake3_update(&ctx, cache->driver_keys_blob,
cache->driver_keys_blob_size);
_mesa_blake3_update(&ctx, data, size);
_mesa_blake3_final(&ctx, key);
}
void
disk_cache_set_callbacks(struct disk_cache *cache, disk_cache_put_cb put,
disk_cache_get_cb get)
{
cache->blob_put_cb = put;
cache->blob_get_cb = get;
disk_cache_init_queue(cache);
}
#endif /* ENABLE_SHADER_CACHE */
src/util/disk_cache.h
+349
View File
@@ -0,0 +1,349 @@
/*
* Copyright © 2014 Intel Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
#ifndef DISK_CACHE_H
#define DISK_CACHE_H
#ifdef HAVE_DLFCN_H
#include <dlfcn.h>
#include <stdio.h>
#include "util/build_id.h"
#endif
#include <assert.h>
#include <stdint.h>
#include <stdbool.h>
#include <sys/stat.h>
#include "util/mesa-blake3.h"
#include "util/detect_os.h"
#ifdef __cplusplus
extern "C" {
#endif
/* Size of cache keys in bytes. */
#define CACHE_KEY_SIZE BLAKE3_KEY_LEN
#define CACHE_DIR_NAME "mesa_shader_cache"
#define CACHE_DIR_NAME_SF "mesa_shader_cache_sf"
#define CACHE_DIR_NAME_DB "mesa_shader_cache_db"
typedef uint8_t cache_key[CACHE_KEY_SIZE];
/* WARNING: 3rd party applications might be reading the cache item metadata.
* Do not change these values without making the change widely known.
* Please contact Valve developers and make them aware of this change.
*/
#define CACHE_ITEM_TYPE_UNKNOWN 0x0
#define CACHE_ITEM_TYPE_GLSL 0x1
typedef void
(*disk_cache_put_cb) (const void *key, signed long keySize,
const void *value, signed long valueSize);
typedef signed long
(*disk_cache_get_cb) (const void *key, signed long keySize,
void *value, signed long valueSize);
struct cache_item_metadata {
/**
* The cache item type. This could be used to identify a GLSL cache item,
* a certain type of IR (tgsi, nir, etc), or signal that it is the final
* binary form of the shader.
*/
uint32_t type;
/** GLSL cache item metadata */
cache_key *keys; /* blake3 list of shaders that make up the cache item */
uint32_t num_keys;
};
struct disk_cache;
#ifdef HAVE_DLADDR
static inline bool
disk_cache_get_function_timestamp(void *ptr, uint32_t* timestamp)
{
Dl_info info;
struct stat st;
if (!dladdr(ptr, &info) || !info.dli_fname) {
return false;
}
if (stat(info.dli_fname, &st)) {
return false;
}
if (!st.st_mtime) {
fprintf(stderr, "Mesa: The provided filesystem timestamp for the cache "
"is bogus! Disabling On-disk cache.\n");
return false;
}
*timestamp = st.st_mtime;
return true;
}
static inline bool
disk_cache_get_function_identifier(void *ptr, blake3_hasher *ctx)
{
uint32_t timestamp;
#if HAVE_BUILD_ID
const struct build_id_note *note = NULL;
if ((note = build_id_find_nhdr_for_addr(ptr))) {
_mesa_blake3_update(ctx, build_id_data(note), build_id_length(note));
} else
#endif
if (disk_cache_get_function_timestamp(ptr, &timestamp)) {
_mesa_blake3_update(ctx, &timestamp, sizeof(timestamp));
} else
return false;
return true;
}
#elif DETECT_OS_WINDOWS
bool
disk_cache_get_function_identifier(void *ptr, blake3_hasher *ctx);
#else
static inline bool
disk_cache_get_function_identifier(void *ptr, blake3_hasher *ctx)
{
return false;
}
#endif
/* Provide inlined stub functions if the shader cache is disabled. */
#ifdef ENABLE_SHADER_CACHE
/**
* Create a new cache object.
*
* This function creates the handle necessary for all subsequent cache_*
* functions.
*
* This cache provides two distinct operations:
*
* o Storage and retrieval of arbitrary objects by cryptographic
* name (or "key"). This is provided via disk_cache_put() and
* disk_cache_get().
*
* o The ability to store a key alone and check later whether the
* key was previously stored. This is provided via disk_cache_put_key()
* and disk_cache_has_key().
*
* The put_key()/has_key() operations are conceptually identical to
* put()/get() with no data, but are provided separately to allow for
* a more efficient implementation.
*
* In all cases, the keys are sequences of 20 bytes. It is anticipated
* that callers will compute appropriate BLAKE3 signatures for keys,
* (though nothing in this implementation directly relies on how the
* names are computed). See mesa-blake3.h and _mesa_blake3_compute for
* assistance in computing BLAKE3 signatures.
*/
struct disk_cache *
disk_cache_create(const char *gpu_name, const char *timestamp,
uint64_t driver_flags);
struct disk_cache *
disk_cache_create_custom(const char *gpu_name, const char *driver_id,
uint64_t driver_flags, const char *cache_dir_name,
uint32_t max_size);
/**
* Destroy a cache object, (freeing all associated resources).
*/
void
disk_cache_destroy(struct disk_cache *cache);
/* Wait for all previous disk_cache_put() calls to be processed (used for unit
* testing).
*/
void
disk_cache_wait_for_idle(struct disk_cache *cache);
/**
* Remove the item in the cache under the name \key.
*/
void
disk_cache_remove(struct disk_cache *cache, const cache_key key);
/**
* Store an item in the cache under the name \key.
*
* The item can be retrieved later with disk_cache_get(), (unless the item has
* been evicted in the interim).
*
* Any call to disk_cache_put() may cause an existing, random item to be
* evicted from the cache.
*/
void
disk_cache_put(struct disk_cache *cache, const cache_key key,
const void *data, size_t size,
struct cache_item_metadata *cache_item_metadata);
/**
* Store an item in the cache under the name \key without copying the data param.
*
* The item can be retrieved later with disk_cache_get(), (unless the item has
* been evicted in the interim).
*
* Any call to disk_cache_put() may cause an existing, random item to be
* evicted from the cache.
*
* @p data will be freed
*/
void
disk_cache_put_nocopy(struct disk_cache *cache, const cache_key key,
void *data, size_t size,
struct cache_item_metadata *cache_item_metadata);
/**
* Retrieve an item previously stored in the cache with the name <key>.
*
* The item must have been previously stored with a call to disk_cache_put().
*
* If \size is non-NULL, then, on successful return, it will be set to the
* size of the object.
*
* \return A pointer to the stored object if found. NULL if the object
* is not found, or if any error occurs, (memory allocation failure,
* filesystem error, etc.). The returned data is malloc'ed so the
* caller should call free() it when finished.
*/
void *
disk_cache_get(struct disk_cache *cache, const cache_key key, size_t *size);
/**
* Store the name \key within the cache, (without any associated data).
*
* Later this key can be checked with disk_cache_has_key(), (unless the key
* has been evicted in the interim).
*
* Any call to disk_cache_put_key() may cause an existing, random key to be
* evicted from the cache.
*/
void
disk_cache_put_key(struct disk_cache *cache, const cache_key key);
/**
* Test whether the name \key was previously recorded in the cache.
*
* Return value: True if disk_cache_put_key() was previously called with
* \key, (and the key was not evicted in the interim).
*
* Note: disk_cache_has_key() will only return true for keys passed to
* disk_cache_put_key(). Specifically, a call to disk_cache_put() will not cause
* disk_cache_has_key() to return true for the same key.
*/
bool
disk_cache_has_key(struct disk_cache *cache, const cache_key key);
/**
* Compute the name \key from \data of given \size.
*/
void
disk_cache_compute_key(struct disk_cache *cache, const void *data, size_t size,
cache_key key);
void
disk_cache_set_callbacks(struct disk_cache *cache, disk_cache_put_cb put,
disk_cache_get_cb get);
#else
static inline struct disk_cache *
disk_cache_create(const char *gpu_name, const char *timestamp,
uint64_t driver_flags)
{
return NULL;
}
static inline struct disk_cache *
disk_cache_create_custom(const char *gpu_name, const char *driver_id,
uint64_t driver_flags, const char *cache_dir_name,
uint32_t max_size)
{
return NULL;
}
static inline void
disk_cache_destroy(struct disk_cache *cache)
{
}
static inline void
disk_cache_put(struct disk_cache *cache, const cache_key key,
const void *data, size_t size,
struct cache_item_metadata *cache_item_metadata)
{
}
static inline void
disk_cache_put_nocopy(struct disk_cache *cache, const cache_key key,
void *data, size_t size,
struct cache_item_metadata *cache_item_metadata)
{
}
static inline void
disk_cache_remove(struct disk_cache *cache, const cache_key key)
{
}
static inline uint8_t *
disk_cache_get(struct disk_cache *cache, const cache_key key, size_t *size)
{
return NULL;
}
static inline void
disk_cache_put_key(struct disk_cache *cache, const cache_key key)
{
}
static inline bool
disk_cache_has_key(struct disk_cache *cache, const cache_key key)
{
return false;
}
static inline void
disk_cache_compute_key(struct disk_cache *cache, const void *data, size_t size,
cache_key key)
{
}
static inline void
disk_cache_set_callbacks(struct disk_cache *cache, disk_cache_put_cb put,
disk_cache_get_cb get)
{
}
#endif /* ENABLE_SHADER_CACHE */
#ifdef __cplusplus
}
#endif
#endif /* CACHE_H */
src/util/disk_cache_os.c
+1290
View File
File diff suppressed because it is too large. Load diff
src/util/disk_cache_os.h
+195
View File
@@ -0,0 +1,195 @@
/*
* Copyright © 2014 Intel Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
#ifndef DISK_CACHE_OS_H
#define DISK_CACHE_OS_H
#include "util/u_queue.h"
#include "util/disk_cache.h"
#if DETECT_OS_WINDOWS
/* TODO: implement disk cache support on windows */
#else
#include "util/fossilize_db.h"
#include "util/mesa_cache_db.h"
#include "util/mesa_cache_db_multipart.h"
#ifdef __cplusplus
extern "C" {
#endif
/* Number of bits to mask off from a cache key to get an index. */
#define CACHE_INDEX_KEY_BITS 16
/* Mask for computing an index from a key. */
#define CACHE_INDEX_KEY_MASK ((1 << CACHE_INDEX_KEY_BITS) - 1)
/* The number of keys that can be stored in the index. */
#define CACHE_INDEX_MAX_KEYS (1 << CACHE_INDEX_KEY_BITS)
enum disk_cache_type {
DISK_CACHE_NONE,
DISK_CACHE_MULTI_FILE,
DISK_CACHE_SINGLE_FILE,
DISK_CACHE_DATABASE,
};
struct disk_cache {
/* The path to the cache directory. */
char *path;
bool path_init_failed;
/* Thread queue for compressing and writing cache entries to disk */
struct util_queue cache_queue;
struct foz_db foz_db;
struct mesa_cache_db_multipart cache_db;
enum disk_cache_type type;
/* Seed for rand, which is used to pick a random directory */
uint64_t seed_xorshift128plus[2];
/* A pointer to the mmapped index file within the cache directory. */
uint8_t *index_mmap;
size_t index_mmap_size;
/* Pointer to total size of all objects in cache (within index_mmap) */
p_atomic_uint64_t *size;
/* Pointer to stored keys, (within index_mmap). */
uint8_t *stored_keys;
/* Maximum size of all cached objects (in bytes). */
uint64_t max_size;
/* Driver cache keys. */
uint8_t *driver_keys_blob;
size_t driver_keys_blob_size;
disk_cache_put_cb blob_put_cb;
disk_cache_get_cb blob_get_cb;
/* Don't compress cached data. This is for testing purposes only. */
bool compression_disabled;
struct {
bool enabled;
unsigned hits;
unsigned misses;
} stats;
/* Internal RO FOZ cache for combined use of RO and RW caches. */
struct disk_cache *foz_ro_cache;
};
struct cache_entry_file_data {
uint32_t crc32;
uint32_t uncompressed_size;
};
struct disk_cache_put_job {
struct util_queue_fence fence;
struct disk_cache *cache;
cache_key key;
/* Copy of cache data to be compressed and written. */
void *data;
/* Size of data to be compressed and written. */
size_t size;
struct cache_item_metadata cache_item_metadata;
};
const char *
disk_cache_generate_cache_dir(void *mem_ctx, const char *gpu_name,
const char *driver_id,
const char *cache_dir_name_custom,
enum disk_cache_type cache_type,
bool mkdir);
void
disk_cache_evict_lru_item(struct disk_cache *cache);
void
disk_cache_evict_item(struct disk_cache *cache, char *filename);
void *
disk_cache_load_item_foz(struct disk_cache *cache, const cache_key key,
size_t *size);
void *
disk_cache_load_item(struct disk_cache *cache, char *filename, size_t *size);
char *
disk_cache_get_cache_filename(struct disk_cache *cache, const cache_key key);
bool
disk_cache_write_item_to_disk_foz(struct disk_cache_put_job *dc_job);
void
disk_cache_write_item_to_disk(struct disk_cache_put_job *dc_job,
char *filename);
bool
disk_cache_enabled(void);
bool
disk_cache_load_cache_index_foz(void *mem_ctx, struct disk_cache *cache);
void
disk_cache_touch_cache_user_marker(char *path);
bool
disk_cache_mmap_cache_index(void *mem_ctx, struct disk_cache *cache);
void
disk_cache_destroy_mmap(struct disk_cache *cache);
void *
disk_cache_db_load_item(struct disk_cache *cache, const cache_key key,
size_t *size);
bool
disk_cache_db_write_item_to_disk(struct disk_cache_put_job *dc_job);
bool
disk_cache_db_load_cache_index(void *mem_ctx, struct disk_cache *cache);
void
disk_cache_delete_old_cache(void);
#ifdef __cplusplus
}
#endif
#endif
#endif /* DISK_CACHE_OS_H */
src/util/double.c
+38
View File
@@ -0,0 +1,38 @@
/*
* Mesa 3-D graphics library
*
* Copyright (C) 2018-2019 Intel Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included
* in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
#include "softfloat.h"
#include "double.h"
float
_mesa_double_to_float(double val)
{
return _mesa_double_to_f32(val, false);
}
float
_mesa_double_to_float_rtz(double val)
{
return _mesa_double_to_f32(val, true);
}
src/util/double.h
+54
View File
@@ -0,0 +1,54 @@
/*
* Mesa 3-D graphics library
*
* Copyright (C) 2018-2019 Intel Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included
* in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
#ifndef _DOUBLE_H_
#define _DOUBLE_H_
#include "u_math.h"
#ifdef __cplusplus
extern "C" {
#endif
/*
* This API is no more than a wrapper to the counterpart softfloat.h
* calls. Still, softfloat.h conversion API is meant to be kept private. In
* other words, only use the API published here, instead of calling directly
* the softfloat.h one.
*/
float _mesa_double_to_float(double val);
float _mesa_double_to_float_rtz(double val);
static inline float
_mesa_double_to_float_rtne(double val)
{
return _mesa_double_to_float(val);
}
#ifdef __cplusplus
} /* extern C */
#endif
#endif /* _DOUBLE_H_ */
src/util/driconf.dtd
+23
View File
@@ -0,0 +1,23 @@
<!ELEMENT driconf (device+)>
<!ELEMENT device (application | engine)+>
<!ATTLIST device driver CDATA #IMPLIED
device CDATA #IMPLIED>
<!ELEMENT application (option+)>
<!ATTLIST application name CDATA #REQUIRED
executable CDATA #IMPLIED
executable_regexp CDATA #IMPLIED
sha1 CDATA #IMPLIED
application_name_match CDATA #IMPLIED
application_versions CDATA #IMPLIED>
<!ELEMENT engine (option+)>
<!-- engine_name_match: A regexp matching the engine name -->
<!-- engine_versions: A version in range format
(version 1 to 4 : "1:4") -->
<!ATTLIST engine engine_name_match CDATA #REQUIRED
engine_versions CDATA #IMPLIED>
<!ELEMENT option EMPTY>
<!ATTLIST option name CDATA #REQUIRED
value CDATA #REQUIRED>
src/util/driconf.h
+676
View File
@@ -0,0 +1,676 @@
/*
* XML DRI client-side driver configuration
* Copyright (C) 2003 Felix Kuehling
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included
* in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* FELIX KUEHLING, OR ANY OTHER CONTRIBUTORS BE LIABLE FOR ANY CLAIM,
* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
* OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
* OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*
*/
/**
* \file driconf.h
* \brief Pool of common options
* \author Felix Kuehling
*
* This file defines macros that can be used to construct
* driConfigOptions in the drivers.
*/
#ifndef __DRICONF_H
#define __DRICONF_H
#include "xmlconfig.h"
/*
* generic macros
*/
/** \brief Names a section of related options to follow */
#define DRI_CONF_SECTION(text) { .desc = text, .info = { .type = DRI_SECTION } },
#define DRI_CONF_SECTION_END
/** \brief End an option description */
#define DRI_CONF_OPT_END },
/** \brief A verbal description (empty version) */
#define DRI_CONF_DESC(text) .desc = text,
/** \brief A verbal description of an enum value */
#define DRI_CONF_ENUM(_value,text) { .value = _value, .desc = text },
#define DRI_CONF_RANGE_I(min, max) \
.range = { \
.start = { ._int = min }, \
.end = { ._int = max }, \
} \
#define DRI_CONF_RANGE_U64(min, max) \
.range = { \
.start = { ._uint64 = min }, \
.end = { ._uint64 = max }, \
}
#define DRI_CONF_RANGE_F(min, max) \
.range = { \
.start = { ._float = min }, \
.end = { ._float = max }, \
} \
/**
* \brief A boolean option definition, with the default value passed in as a
* string
*/
#define DRI_CONF_OPT_B(_name, def, _desc) { \
.desc = _desc, \
.info = { \
.name = #_name, \
.type = DRI_BOOL, \
}, \
.value = { ._bool = def }, \
},
#define DRI_CONF_OPT_I(_name, def, min, max, _desc) { \
.desc = _desc, \
.info = { \
.name = #_name, \
.type = DRI_INT, \
DRI_CONF_RANGE_I(min, max), \
}, \
.value = { ._int = def }, \
},
#define DRI_CONF_OPT_U64(_name, def, min, max, _desc) { \
.desc = _desc, \
.info = { \
.name = #_name, \
.type = DRI_UINT64, \
DRI_CONF_RANGE_U64(min, max), \
}, \
.value = { ._uint64 = def }, \
},
#define DRI_CONF_OPT_F(_name, def, min, max, _desc) { \
.desc = _desc, \
.info = { \
.name = #_name, \
.type = DRI_FLOAT, \
DRI_CONF_RANGE_F(min, max), \
}, \
.value = { ._float = def }, \
},
#define DRI_CONF_OPT_E(_name, def, min, max, _desc, values) { \
.desc = _desc, \
.info = { \
.name = #_name, \
.type = DRI_ENUM, \
DRI_CONF_RANGE_I(min, max), \
}, \
.value = { ._int = def }, \
.enums = { values }, \
},
#define DRI_CONF_OPT_S(_name, def, _desc) { \
.desc = _desc, \
.info = { \
.name = #_name, \
.type = DRI_STRING, \
}, \
.value = { ._string = #def }, \
},
#define DRI_CONF_OPT_S_NODEF(_name, _desc) { \
.desc = _desc, \
.info = { \
.name = #_name, \
.type = DRI_STRING, \
}, \
.value = { ._string = "" }, \
},
/**
* \brief Debugging options
*/
#define DRI_CONF_SECTION_DEBUG DRI_CONF_SECTION("Debugging")
#define DRI_CONF_ALWAYS_FLUSH_CACHE(def) \
DRI_CONF_OPT_B(always_flush_cache, def, \
"Enable flushing GPU caches with each draw call")
#define DRI_CONF_DISABLE_THROTTLING(def) \
DRI_CONF_OPT_B(disable_throttling, def, \
"Disable throttling on first batch after flush")
#define DRI_CONF_FORCE_GLSL_EXTENSIONS_WARN(def) \
DRI_CONF_OPT_B(force_glsl_extensions_warn, def, \
"Force GLSL extension default behavior to 'warn'")
#define DRI_CONF_DISABLE_BLEND_FUNC_EXTENDED(def) \
DRI_CONF_OPT_B(disable_blend_func_extended, def, \
"Disable dual source blending")
#define DRI_CONF_DISABLE_ARB_GPU_SHADER5(def) \
DRI_CONF_OPT_B(disable_arb_gpu_shader5, def, \
"Disable GL_ARB_gpu_shader5")
#define DRI_CONF_DUAL_COLOR_BLEND_BY_LOCATION(def) \
DRI_CONF_OPT_B(dual_color_blend_by_location, def, \
"Identify dual color blending sources by location rather than index")
#define DRI_CONF_DISABLE_GLSL_LINE_CONTINUATIONS(def) \
DRI_CONF_OPT_B(disable_glsl_line_continuations, def, \
"Disable backslash-based line continuations in GLSL source")
#define DRI_CONF_DISABLE_UNIFORM_ARRAY_RESIZE(def) \
DRI_CONF_OPT_B(disable_uniform_array_resize, def, \
"Disable the glsl optimisation that resizes uniform arrays")
#define DRI_CONF_ALIAS_SHADER_EXTENSION() \
DRI_CONF_OPT_S_NODEF(alias_shader_extension, "Allow alias for shader extensions")
#define DRI_CONF_ALLOW_VERTEX_TEXTURE_BIAS(def) \
DRI_CONF_OPT_B(allow_vertex_texture_bias, def, \
"Allow GL2 vertex shaders to have access to texture2D/textureCube with bias variants")
#define DRI_CONF_FORCE_GLSL_VERSION(def) \
DRI_CONF_OPT_I(force_glsl_version, def, 0, 999, \
"Force a default GLSL version for shaders that lack an explicit #version line")
#define DRI_CONF_ALLOW_EXTRA_PP_TOKENS(def) \
DRI_CONF_OPT_B(allow_extra_pp_tokens, def, \
"Allow extra tokens at end of preprocessor directives.")
#define DRI_CONF_ALLOW_GLSL_EXTENSION_DIRECTIVE_MIDSHADER(def) \
DRI_CONF_OPT_B(allow_glsl_extension_directive_midshader, def, \
"Allow GLSL #extension directives in the middle of shaders")
#define DRI_CONF_ALLOW_GLSL_120_SUBSET_IN_110(def) \
DRI_CONF_OPT_B(allow_glsl_120_subset_in_110, def, \
"Allow a subset of GLSL 1.20 in GLSL 1.10 as needed by SPECviewperf13")
#define DRI_CONF_ALLOW_GLSL_EMBEDDED_STRUCTURE_DECLARATIONS(def) \
DRI_CONF_OPT_B(allow_glsl_embedded_structure_declarations, def, \
"Allow embedded structure declarations again in GLSL 1.20+")
#define DRI_CONF_ALLOW_GLSL_BUILTIN_CONST_EXPRESSION(def) \
DRI_CONF_OPT_B(allow_glsl_builtin_const_expression, def, \
"Allow builtins as part of constant expressions")
#define DRI_CONF_ALLOW_GLSL_RELAXED_ES(def) \
DRI_CONF_OPT_B(allow_glsl_relaxed_es, def, \
"Allow some relaxation of GLSL ES shader restrictions")
#define DRI_CONF_ALLOW_GLSL_BUILTIN_VARIABLE_REDECLARATION(def) \
DRI_CONF_OPT_B(allow_glsl_builtin_variable_redeclaration, def, \
"Allow GLSL built-in variables to be redeclared verbatim")
#define DRI_CONF_ALLOW_HIGHER_COMPAT_VERSION(def) \
DRI_CONF_OPT_B(allow_higher_compat_version, def, \
"Allow a higher compat profile (version 3.1+) for apps that request it")
#define DRI_CONF_ALLOW_GLSL_COMPAT_SHADERS(def) \
DRI_CONF_OPT_B(allow_glsl_compat_shaders, def, \
"Allow in GLSL: #version xxx compatibility")
#define DRI_CONF_FORCE_GLSL_ABS_SQRT(def) \
DRI_CONF_OPT_B(force_glsl_abs_sqrt, def, \
"Force computing the absolute value for sqrt() and inversesqrt()")
#define DRI_CONF_GLSL_CORRECT_DERIVATIVES_AFTER_DISCARD(def) \
DRI_CONF_OPT_B(glsl_correct_derivatives_after_discard, def, \
"Implicit and explicit derivatives after a discard behave as if the discard didn't happen")
#define DRI_CONF_GLSL_IGNORE_WRITE_TO_READONLY_VAR(def) \
DRI_CONF_OPT_B(glsl_ignore_write_to_readonly_var, def, \
"Forces the GLSL compiler to ignore writes to readonly vars rather than throwing an error")
#define DRI_CONF_ALLOW_GLSL_CROSS_STAGE_INTERPOLATION_MISMATCH(def) \
DRI_CONF_OPT_B(allow_glsl_cross_stage_interpolation_mismatch, def, \
"Allow interpolation qualifier mismatch across shader stages")
#define DRI_CONF_DO_DCE_BEFORE_CLIP_CULL_ANALYSIS(def) \
DRI_CONF_OPT_B(do_dce_before_clip_cull_analysis, def, \
"Use dead code elimitation before checking for invalid Clip*/CullDistance variables usage.")
#define DRI_CONF_ALLOW_DRAW_OUT_OF_ORDER(def) \
DRI_CONF_OPT_B(allow_draw_out_of_order, def, \
"Allow out-of-order draw optimizations. Set when Z fighting doesn't have to be accurate.")
#define DRI_CONF_GLTHREAD_NOP_CHECK_FRAMEBUFFER_STATUS(def) \
DRI_CONF_OPT_B(glthread_nop_check_framebuffer_status, def, \
"glthread always returns GL_FRAMEBUFFER_COMPLETE to prevent synchronization.")
#define DRI_CONF_FORCE_EXPLICIT_UNIFORM_LOC_ZERO() \
DRI_CONF_OPT_S_NODEF(force_explicit_uniform_loc_zero, "Forces an explicit uniform location of zero for the uniform.")
#define DRI_CONF_FORCE_GL_VENDOR() \
DRI_CONF_OPT_S_NODEF(force_gl_vendor, "Override GPU vendor string.")
#define DRI_CONF_FORCE_GL_RENDERER() \
DRI_CONF_OPT_S_NODEF(force_gl_renderer, "Override GPU renderer string.")
#define DRI_CONF_FORCE_COMPAT_PROFILE(def) \
DRI_CONF_OPT_B(force_compat_profile, def, \
"Force an OpenGL compatibility context")
#define DRI_CONF_FORCE_COMPAT_SHADERS(def) \
DRI_CONF_OPT_B(force_compat_shaders, def, \
"Force OpenGL compatibility shaders")
#define DRI_CONF_FORCE_DIRECT_GLX_CONTEXT(def) \
DRI_CONF_OPT_B(force_direct_glx_context, def, \
"Force direct GLX context (even if indirect is requested)")
#define DRI_CONF_ALLOW_INVALID_GLX_DESTROY_WINDOW(def) \
DRI_CONF_OPT_B(allow_invalid_glx_destroy_window, def, \
"Allow passing an invalid window into glXDestroyWindow")
#define DRI_CONF_KEEP_NATIVE_WINDOW_GLX_DRAWABLE(def) \
DRI_CONF_OPT_B(keep_native_window_glx_drawable, def, \
"Keep GLX drawable created from native window when switch context")
#define DRI_CONF_OVERRIDE_VRAM_SIZE() \
DRI_CONF_OPT_I(override_vram_size, -1, -1, 2147483647, \
"Override the VRAM size advertised to the application in MiB (-1 = default)")
#define DRI_CONF_FORCE_GL_MAP_BUFFER_SYNCHRONIZED(def) \
DRI_CONF_OPT_B(force_gl_map_buffer_synchronized, def, "Override GL_MAP_UNSYNCHRONIZED_BIT.")
#define DRI_CONF_FORCE_GL_DEPTH_COMPONENT_TYPE_INT(def) \
DRI_CONF_OPT_B(force_gl_depth_component_type_int, def, "Override GL_DEPTH_COMPONENT type from unsigned short to unsigned int")
#define DRI_CONF_TRANSCODE_ETC(def) \
DRI_CONF_OPT_B(transcode_etc, def, "Transcode ETC formats to DXTC if unsupported")
#define DRI_CONF_TRANSCODE_ASTC(def) \
DRI_CONF_OPT_B(transcode_astc, def, "Transcode ASTC formats to DXTC if unsupported")
#define DRI_CONF_ALLOW_COMPRESSED_FALLBACK(def) \
DRI_CONF_OPT_B(allow_compressed_fallback, def, "Allow fallback to uncompressed formats for unsupported compressed formats")
#define DRI_CONF_MESA_EXTENSION_OVERRIDE() \
DRI_CONF_OPT_S_NODEF(mesa_extension_override, \
"Allow enabling/disabling a list of extensions")
#define DRI_CONF_GLX_EXTENSION_OVERRIDE() \
DRI_CONF_OPT_S_NODEF(glx_extension_override, \
"Allow enabling/disabling a list of GLX extensions")
#define DRI_CONF_GLX_CLEAR_CONTEXT_RESET_ISOLATION_BIT(def) \
DRI_CONF_OPT_B(glx_clear_context_reset_isolation_bit, def, "Clear context reset isolation bit before creating context")
#define DRI_CONF_INDIRECT_GL_EXTENSION_OVERRIDE() \
DRI_CONF_OPT_S_NODEF(indirect_gl_extension_override, \
"Allow enabling/disabling a list of indirect-GL extensions")
#define DRI_CONF_FORCE_PROTECTED_CONTENT_CHECK(def) \
DRI_CONF_OPT_B(force_protected_content_check, def, \
"Reject image import if protected_content attribute doesn't match")
#define DRI_CONF_IGNORE_MAP_UNSYNCHRONIZED(def) \
DRI_CONF_OPT_B(ignore_map_unsynchronized, def, \
"Ignore GL_MAP_UNSYNCHRONIZED_BIT, workaround for games that use it incorrectly")
#define DRI_CONF_VK_LOWER_TERMINATE_TO_DISCARD(def) \
DRI_CONF_OPT_B(vk_lower_terminate_to_discard, def, \
"Lower terminate to discard (which is implicitly demote)")
#define DRI_CONF_LIMIT_TRIG_INPUT_RANGE(def) \
DRI_CONF_OPT_B(limit_trig_input_range, def, \
"Limit trig input range to [-2p : 2p] to improve sin/cos calculation precision on Intel")
#define DRI_CONF_NO_16BIT(def) \
DRI_CONF_OPT_B(no_16bit, def, \
"Disable 16-bit instructions")
#define DRI_CONF_IGNORE_DISCARD_FRAMEBUFFER(def) \
DRI_CONF_OPT_B(ignore_discard_framebuffer, def, \
"Ignore glDiscardFramebuffer/glInvalidateFramebuffer, workaround for games that use it incorrectly")
#define DRI_CONF_FORCE_VK_VENDOR() \
DRI_CONF_OPT_I(force_vk_vendor, 0, -1, 2147483647, "Override GPU vendor id")
#define DRI_CONFIG_INTEL_TBIMR(def) \
DRI_CONF_OPT_B(intel_tbimr, def, "Enable TBIMR tiled rendering")
#define DRI_CONFIG_INTEL_FORCE_COMPUTE_SURFACE_PREFETCH(def) \
DRI_CONF_OPT_B(intel_force_compute_surface_prefetch, def, \
"Enable binding table surface prefteching for compute shaders")
#define DRI_CONFIG_INTEL_FORCE_SAMPLER_PREFETCH(def) \
DRI_CONF_OPT_B(intel_force_sampler_prefetch, def, \
"Enable binding table sampler prefteching")
#define DRI_CONFIG_INTEL_VF_DISTRIBUTION(def) \
DRI_CONF_OPT_B(intel_vf_distribution, def, "Enable geometry distribution")
#define DRI_CONFIG_INTEL_TE_DISTRIBUTION(def) \
DRI_CONF_OPT_B(intel_te_distribution, def, "Enable tesselation distribution")
#define DRI_CONFIG_INTEL_STORAGE_CACHE_POLICY_WT(def) \
DRI_CONF_OPT_B(intel_storage_cache_policy_wt, def, "Enable write-through cache policy for storage buffers/images.")
#define DRI_CONF_INTEL_ENABLE_WA_14018912822(def) \
DRI_CONF_OPT_B(intel_enable_wa_14018912822, def, \
"Intel workaround for using zero blend constants")
#define DRI_CONF_INTEL_ENABLE_WA_14024015672_MSAA(def) \
DRI_CONF_OPT_B(intel_enable_wa_14024015672_msaa, def, \
"Intel workaround for RHWO MSAA")
#define DRI_CONF_INTEL_SAMPLER_ROUTE_TO_LSC(def) \
DRI_CONF_OPT_B(intel_sampler_route_to_lsc, def, \
"Intel specific toggle to enable sampler route to LSC")
#define DRI_CONF_INTEL_DISABLE_THREADED_CONTEXT(def) \
DRI_CONF_OPT_B(intel_disable_threaded_context, def, "Disable threaded context")
#define DRI_CONF_VK_REQUIRE_ASTC(def) \
DRI_CONF_OPT_B(vk_require_astc, def, \
"Implement emulated ASTC on HW that does not support it")
/**
* \brief Image quality-related options
*/
#define DRI_CONF_SECTION_QUALITY DRI_CONF_SECTION("Image Quality")
#define DRI_CONF_PP_LOWER_DEPTH_RANGE_RATE() \
DRI_CONF_OPT_F(lower_depth_range_rate, 1.0, 0.0, 1.0, \
"Lower depth range for fixing misrendering issues due to z coordinate float point interpolation accuracy")
/**
* \brief Performance-related options
*/
#define DRI_CONF_SECTION_PERFORMANCE DRI_CONF_SECTION("Performance")
#define DRI_CONF_VBLANK_NEVER 0
#define DRI_CONF_VBLANK_DEF_INTERVAL_0 1
#define DRI_CONF_VBLANK_DEF_INTERVAL_1 2
#define DRI_CONF_VBLANK_ALWAYS_SYNC 3
#define DRI_CONF_VBLANK_MODE(def) \
DRI_CONF_OPT_E(vblank_mode, def, 0, 3, \
"Synchronization with vertical refresh (swap intervals)", \
DRI_CONF_ENUM(0,"Never synchronize with vertical refresh, ignore application's choice") \
DRI_CONF_ENUM(1,"Initial swap interval 0, obey application's choice") \
DRI_CONF_ENUM(2,"Initial swap interval 1, obey application's choice") \
DRI_CONF_ENUM(3,"Always synchronize with vertical refresh, application chooses the minimum swap interval"))
#define DRI_CONF_ADAPTIVE_SYNC(def) \
DRI_CONF_OPT_B(adaptive_sync,def, \
"Adapt the monitor sync to the application performance (when possible)")
#define DRI_CONF_BLOCK_ON_DEPLETED_BUFFERS(def) \
DRI_CONF_OPT_B(block_on_depleted_buffers, def, \
"Block clients using buffer backpressure until new buffer is available to reduce latency")
#define DRI_CONF_VK_WSI_FORCE_BGRA8_UNORM_FIRST(def) \
DRI_CONF_OPT_B(vk_wsi_force_bgra8_unorm_first, def, \
"Force vkGetPhysicalDeviceSurfaceFormatsKHR to return VK_FORMAT_B8G8R8A8_UNORM as the first format")
#define DRI_CONF_VK_WSI_FORCE_SWAPCHAIN_TO_CURRENT_EXTENT(def) \
DRI_CONF_OPT_B(vk_wsi_force_swapchain_to_current_extent, def, \
"Force VkSwapchainCreateInfoKHR::imageExtent to be VkSurfaceCapabilities2KHR::currentExtent")
#define DRI_CONF_VK_WSI_DISABLE_UNORDERED_SUBMITS(def) \
DRI_CONF_OPT_B(vk_wsi_disable_unordered_submits, def, \
"Disable unordered WSI submits to workaround application synchronization bugs")
#define DRI_CONF_VK_X11_OVERRIDE_MIN_IMAGE_COUNT(def) \
DRI_CONF_OPT_I(vk_x11_override_min_image_count, def, 0, 999, \
"Override the VkSurfaceCapabilitiesKHR::minImageCount (0 = no override)")
#define DRI_CONF_VK_X11_STRICT_IMAGE_COUNT(def) \
DRI_CONF_OPT_B(vk_x11_strict_image_count, def, \
"Force the X11 WSI to create exactly the number of image specified by the application in VkSwapchainCreateInfoKHR::minImageCount")
#define DRI_CONF_VK_X11_ENSURE_MIN_IMAGE_COUNT(def) \
DRI_CONF_OPT_B(vk_x11_ensure_min_image_count, def, \
"Force the X11 WSI to create at least the number of image specified by the driver in VkSurfaceCapabilitiesKHR::minImageCount")
#define DRI_CONF_VK_X11_IGNORE_SUBOPTIMAL(def) \
DRI_CONF_OPT_B(vk_x11_ignore_suboptimal, def, \
"Force the X11 WSI to never report VK_SUBOPTIMAL_KHR")
#define DRI_CONF_VK_XWAYLAND_WAIT_READY(def) \
DRI_CONF_OPT_B(vk_xwayland_wait_ready, def, \
"Wait for fences before submitting buffers to Xwayland")
#define DRI_CONF_MESA_GLTHREAD_DRIVER(def) \
DRI_CONF_OPT_B(mesa_glthread_driver, def, \
"Enable offloading GL driver work to a separate thread")
#define DRI_CONF_MESA_NO_ERROR(def) \
DRI_CONF_OPT_B(mesa_no_error, def, \
"Disable GL driver error checking")
#define DRI_CONF_SHADER_SPILLING_RATE(def) \
DRI_CONF_OPT_I(shader_spilling_rate, def, 0, 100, \
"Speed up shader compilation by increasing number of spilled registers after ra_allocate failure")
/**
* \brief Miscellaneous configuration options
*/
#define DRI_CONF_SECTION_MISCELLANEOUS DRI_CONF_SECTION("Miscellaneous")
#define DRI_CONF_ALWAYS_HAVE_DEPTH_BUFFER(def) \
DRI_CONF_OPT_B(always_have_depth_buffer, def, \
"Create all visuals with a depth buffer")
#define DRI_CONF_GLSL_ZERO_INIT(def) \
DRI_CONF_OPT_B(glsl_zero_init, def, \
"Force uninitialized variables to default to zero")
#define DRI_CONF_VS_POSITION_ALWAYS_INVARIANT(def) \
DRI_CONF_OPT_B(vs_position_always_invariant, def, \
"Force the vertex shader's gl_Position output to be considered 'invariant'")
#define DRI_CONF_VS_POSITION_ALWAYS_PRECISE(def) \
DRI_CONF_OPT_B(vs_position_always_precise, def, \
"Force the vertex shader's gl_Position output to be considered 'precise'")
#define DRI_CONF_ALLOW_RGB10_CONFIGS(def) \
DRI_CONF_OPT_B(allow_rgb10_configs, def, \
"Allow exposure of visuals and fbconfigs with rgb10a2 formats")
#define DRI_CONF_ALLOW_RGB16_CONFIGS(def) \
DRI_CONF_OPT_B(allow_rgb16_configs, def, \
"Allow exposure of visuals and fbconfigs with rgb16 and rgba16 formats")
#define DRI_CONF_FORCE_INTEGER_TEX_NEAREST(def) \
DRI_CONF_OPT_B(force_integer_tex_nearest, def, \
"Force integer textures to use nearest filtering")
/* The GL spec does not allow this but wine has translation bug:
https://bugs.winehq.org/show_bug.cgi?id=54787
*/
#define DRI_CONF_ALLOW_MULTISAMPLED_COPYTEXIMAGE(def) \
DRI_CONF_OPT_B(allow_multisampled_copyteximage, def, \
"Allow CopyTexSubImage and other to copy sampled framebuffer")
#define DRI_CONF_VERTEX_PROGRAM_DEFAULT_OUT(def) \
DRI_CONF_OPT_B(vertex_program_default_out, def, \
"Initialize outputs of vertex program to a default value vec4(0, 0, 0, 1)")
#define DRI_CONF_HEAP_MEMORY_PERCENT(def) \
DRI_CONF_OPT_F(heap_memory_percent, def, 0.0, 1.0, \
"Percentage of total system memory to report as gpu heap memory (0 = driver default)")
#define DRI_CONF_VK_ZERO_VRAM(def) \
DRI_CONF_OPT_B(vk_zero_vram, def, \
"Initialize to zero all VRAM allocations")
/**
* \brief Initialization configuration options
*/
#define DRI_CONF_SECTION_INITIALIZATION DRI_CONF_SECTION("Initialization")
#define DRI_CONF_DEVICE_ID_PATH_TAG() \
DRI_CONF_OPT_S_NODEF(device_id, "Define the graphic device to use if possible")
#define DRI_CONF_DRI_DRIVER() \
DRI_CONF_OPT_S_NODEF(dri_driver, "Override the DRI driver to load")
#define DRI_CONF_V3D_NONMSAA_TEXTURE_SIZE_LIMIT(def) \
DRI_CONF_OPT_B(v3d_nonmsaa_texture_size_limit, def, \
"Report the non-MSAA-only texture size limit")
/**
* \brief wgl specific configuration options
*/
#define DRI_CONF_WGL_FRAME_LATENCY(def) \
DRI_CONF_OPT_I(wgl_frame_latency, def, 1, 16, \
"Override default maximum frame latency")
#define DRI_CONF_WGL_SWAP_INTERVAL(def) \
DRI_CONF_OPT_I(wgl_swap_interval, def, 1, 4, \
"Override default swap interval")
#define DRI_CONF_WGL_REQUIRE_GDI_COMPAT(def) \
DRI_CONF_OPT_B(wgl_require_gdi_compat, def, \
"Require all pixel formats to have PFD_SUPPORT_GDI flag")
/**
* \brief virgl specific configuration options
*/
#define DRI_CONF_GLES_EMULATE_BGRA(def) \
DRI_CONF_OPT_B(gles_emulate_bgra, def, \
"On GLES emulate BGRA formats by using a swizzled RGBA format")
#define DRI_CONF_GLES_APPLY_BGRA_DEST_SWIZZLE(def) \
DRI_CONF_OPT_B(gles_apply_bgra_dest_swizzle, def, \
"When the BGRA formats are emulated by using swizzled RGBA formats on GLES apply the swizzle when writing")
#define DRI_CONF_GLES_SAMPLES_PASSED_VALUE(def, minimum, maximum) \
DRI_CONF_OPT_I(gles_samples_passed_value, def, minimum, maximum, \
"GL_SAMPLES_PASSED value when emulated by GL_ANY_SAMPLES_PASSED")
#define DRI_CONF_FORMAT_L8_SRGB_ENABLE_READBACK(def) \
DRI_CONF_OPT_B(format_l8_srgb_enable_readback, def, \
"Force-enable reading back L8_SRGB textures")
#define DRI_CONF_VIRGL_SHADER_SYNC(def) \
DRI_CONF_OPT_B(virgl_shader_sync, def, \
"Make shader compilation synchronous")
/**
* \brief freedreno specific configuration options
*/
#define DRI_CONF_DISABLE_CONSERVATIVE_LRZ(def) \
DRI_CONF_OPT_B(disable_conservative_lrz, def, \
"Disable conservative LRZ")
#define DRI_CONF_DISABLE_EXPLICIT_SYNC_HEURISTIC(def) \
DRI_CONF_OPT_B(disable_explicit_sync_heuristic, def, \
"Disable Explicit-sync heuristic")
/**
* \brief panfrost specific configuration options
*/
#define DRI_CONF_PAN_COMPUTE_CORE_MASK(def) \
DRI_CONF_OPT_U64(pan_compute_core_mask, def, 0, UINT64_MAX, \
"Bitmask of shader cores that may be used for compute jobs. If unset, defaults to scheduling across all available cores.")
#define DRI_CONF_PAN_FRAGMENT_CORE_MASK(def) \
DRI_CONF_OPT_U64(pan_fragment_core_mask, def, 0, UINT64_MAX, \
"Bitmask of shader cores that may be used for fragment jobs. If unset, defaults to scheduling across all available cores.")
#define DRI_CONF_PAN_ENABLE_VERTEX_PIPELINE_STORES_ATOMICS(def) \
DRI_CONF_OPT_B(pan_enable_vertex_pipeline_stores_atomics, def, \
"Enable vertexPipelineStoresAndAtomics on v13+ (This cannot work on older generation because of speculative behaviors around vertices)")
#define DRI_CONF_PAN_FORCE_ENABLE_SHADER_ATOMICS(def) \
DRI_CONF_OPT_B(pan_force_enable_shader_atomics, def, \
"Enable fragmentStoresAndAtomics and vertexPipelineStoresAndAtomics on any architecture. (This may not work reliably and is for debug purposes only!)")
/**
* \brief Honeykrisp specific configuration options
*/
#define DRI_CONF_HK_DISABLE_BORDER_EMULATION(def) \
DRI_CONF_OPT_B(hk_disable_border_emulation, def, \
"Disable custom border colour emulation")
#define DRI_CONF_HK_FAKE_MINMAX(def) \
DRI_CONF_OPT_B(hk_fake_minmax, def, \
"Fake support for min/max filtering")
#define DRI_CONF_HK_IMAGE_VIEW_MIN_LOD(def) \
DRI_CONF_OPT_B(hk_image_view_min_lod, def, \
"Emulate VK_EXT_image_view_min_lod (conformant but slower)")
#define DRI_CONF_HK_ENABLE_VERTEX_PIPELINE_STORES_ATOMICS(def) \
DRI_CONF_OPT_B(hk_enable_vertex_pipeline_stores_atomics, def, \
"Enable vertexPipelineStoresAndAtomics")
/**
* \brief venus specific configuration options
*/
#define DRI_CONF_VENUS_IMPLICIT_FENCING(def) \
DRI_CONF_OPT_B(venus_implicit_fencing, def, \
"Assume the virtio-gpu kernel driver supports implicit fencing")
#define DRI_CONF_VENUS_WSI_MULTI_PLANE_MODIFIERS(def) \
DRI_CONF_OPT_B(venus_wsi_multi_plane_modifiers, def, \
"Enable support of multi-plane format modifiers for wsi images")
/**
* \brief HASVK specific configuration options
*/
#define DRI_CONF_HASVK_ASSUME_FULL_SUBGROUPS(def) \
DRI_CONF_OPT_I(anv_assume_full_subgroups, def, 0, 32, \
"Allow assuming full subgroups requirement even when it's not specified explicitly and set the given size")
#define DRI_CONF_HASVK_OVERRIDE_API_VERSION(def) \
DRI_CONF_OPT_B(hasvk_report_vk_1_3_version, def, \
"Override intel_hasvk API version")
#define DRI_CONF_HASVK_SAMPLE_MASK_OUT_OPENGL_BEHAVIOUR(def) \
DRI_CONF_OPT_B(anv_sample_mask_out_opengl_behaviour, def, \
"Ignore sample mask out when having single sampled target")
/**
* \brief DZN specific configuration options
*/
#define DRI_CONF_DZN_CLAIM_WIDE_LINES(def) \
DRI_CONF_OPT_B(dzn_claim_wide_lines, def, "Claim wide line support")
#define DRI_CONF_DZN_ENABLE_8BIT_LOADS_STORES(def) \
DRI_CONF_OPT_B(dzn_enable_8bit_loads_stores, def, "Enable VK_KHR_8bit_loads_stores")
#define DRI_CONF_DZN_DISABLE(def) \
DRI_CONF_OPT_B(dzn_disable, def, "Fail instance creation")
/**
* \brief NVK specific configuration options
*/
#define DRI_CONF_NVK_APP_LAYER() DRI_CONF_OPT_S_NODEF(nvk_app_layer, "Select an application layer.")
/**
* \brief Asahi specific configuration options
*/
#define DRI_CONF_ASAHI_NO_FP16(def) \
DRI_CONF_OPT_B(asahi_no_fp16, def, \
"Disable 16-bit float support")
#endif
src/util/driconf_static.py
+242
View File
@@ -0,0 +1,242 @@
#
# Copyright (C) 2021 Google, Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice (including the next
# paragraph) shall be included in all copies or substantial portions of the
# Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
from mako.template import Template
from xml.etree import ElementTree
import argparse
import os
def dbg(str):
if False:
print(str)
cnt = 0
def cname(name):
global cnt
cnt = cnt + 1
return name + '_' + str(cnt)
class Option(object):
def __init__(self, xml):
self.cname = cname('option')
self.name = xml.attrib['name']
self.value = xml.attrib['value']
class Application(object):
def __init__(self, xml):
self.cname = cname('application')
self.name = xml.attrib['name']
self.executable = xml.attrib.get('executable', None)
self.executable_regexp = xml.attrib.get('executable_regexp', None)
self.blake3 = xml.attrib.get('blake3', None)
self.application_name_match = xml.attrib.get('application_name_match', None)
self.application_versions = xml.attrib.get('application_versions', None)
self.options = []
for option in xml.findall('option'):
self.options.append(Option(option))
class Engine(object):
def __init__(self, xml):
self.cname = cname('engine')
self.engine_name_match = xml.attrib['engine_name_match']
self.engine_versions = xml.attrib.get('engine_versions', None)
self.options = []
for option in xml.findall('option'):
self.options.append(Option(option))
class Device(object):
def __init__(self, xml):
self.cname = cname('device')
self.driver = xml.attrib.get('driver', None)
self.device = xml.attrib.get('device', None)
self.applications = []
self.engines = []
for application in xml.findall('application'):
self.applications.append(Application(application))
for engine in xml.findall('engine'):
self.engines.append(Engine(engine))
class DriConf(object):
def __init__(self, xmlpaths):
self.devices = []
for xmlpath in xmlpaths:
root = ElementTree.parse(xmlpath).getroot()
for device in root.findall('device'):
self.devices.append(Device(device))
template = """\
/* Copyright (C) 2021 Google, Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
struct driconf_option {
const char *name;
const char *value;
};
struct driconf_application {
const char *name;
const char *executable;
const char *executable_regexp;
const char *blake3;
const char *application_name_match;
const char *application_versions;
unsigned num_options;
const struct driconf_option *options;
};
struct driconf_engine {
const char *engine_name_match;
const char *engine_versions;
unsigned num_options;
const struct driconf_option *options;
};
struct driconf_device {
const char *driver;
const char *device;
unsigned num_engines;
const struct driconf_engine *engines;
unsigned num_applications;
const struct driconf_application *applications;
};
<%def name="render_options(cname, options)">
static const struct driconf_option ${cname}[] = {
% for option in options:
{ .name = "${option.name}", .value = "${option.value}" },
% endfor
};
</%def>
%for device in driconf.devices:
% for engine in device.engines:
${render_options(engine.cname + '_options', engine.options)}
% endfor
%if len(device.engines) > 0:
static const struct driconf_engine ${device.cname}_engines[] = {
% for engine in device.engines:
{ .engine_name_match = "${engine.engine_name_match}",
% if engine.engine_versions:
.engine_versions = "${engine.engine_versions}",
% endif
.num_options = ${len(engine.options)},
.options = ${engine.cname + '_options'},
},
% endfor
};
%endif
% for application in device.applications:
${render_options(application.cname + '_options', application.options)}
% endfor
%if len(device.applications) > 0:
static const struct driconf_application ${device.cname}_applications[] = {
% for application in device.applications:
{ .name = "${application.name}",
% if application.executable:
.executable = "${application.executable}",
% endif
% if application.executable_regexp:
.executable_regexp = "${application.executable_regexp}",
% endif
% if application.blake3:
.blake3 = "${application.blake3}",
% endif
% if application.application_name_match:
.application_name_match = "${application.application_name_match}",
% endif
% if application.application_versions:
.application_versions = "${application.application_versions}",
% endif
.num_options = ${len(application.options)},
.options = ${application.cname + '_options'},
},
% endfor
};
%endif
static const struct driconf_device ${device.cname} = {
% if device.driver:
.driver = "${device.driver}",
% endif
% if device.device:
.device = "${device.device}",
% endif
.num_engines = ${len(device.engines)},
% if len(device.engines) > 0:
.engines = ${device.cname}_engines,
% endif
.num_applications = ${len(device.applications)},
% if len(device.applications) > 0:
.applications = ${device.cname}_applications,
% endif
};
%endfor
static const struct driconf_device *driconf[] = {
%for device in driconf.devices:
&${device.cname},
%endfor
};
"""
parser = argparse.ArgumentParser()
parser.add_argument('drirc',
nargs=argparse.ONE_OR_MORE,
help='drirc *.conf file(s) to statically include')
parser.add_argument('header',
help='C header file to output the static configuration to')
args = parser.parse_args()
xml = args.drirc
dst = args.header
with open(dst, 'w', encoding='utf-8') as f:
f.write(Template(template).render(driconf=DriConf(xml)))
src/util/drirc_gen.py
+292
View File
@@ -0,0 +1,292 @@
#!/usr/bin/env python3
# Copyright (C) 2026 Valve Corporation
# SPDX-License-Identifier: MIT
# Generates the C header and source files for parsing a driver's list of
# supported DRI config options and their default values into a struct of values
# for the config options. This generates the old DRI_CONF_SECTION /
# DRI_CONF_OPT_* macros internally, and supports validation that the driver
# config files don't refer to options that the driver doesn't use (e.g. due to
# typos).
#
# To use it, call <driver>_parse_dri_options() with the appropriate
# driConfigFileParseParams, and destroy it at teardown with
# driDestroyOptionCache() and driDestroyOptionInfo().
import argparse
import os
import re
import sys
import xml.etree.ElementTree as ET
from enum import Enum
from mako.template import Template
class DrircOptionType(Enum):
BOOL = 0,
INT = 1,
UINT64 = 2,
FLOAT = 3,
STRING = 4,
STRING_NODEF = 5,
ENUM = 6,
type_to_ctype: dict[DrircOptionType, str] = {
DrircOptionType.BOOL: "bool",
DrircOptionType.INT: "int",
DrircOptionType.UINT64: "uint64_t",
DrircOptionType.FLOAT: "float",
DrircOptionType.STRING: "char *",
DrircOptionType.STRING_NODEF: "char *",
DrircOptionType.ENUM: "int",
}
type_to_macro: dict[DrircOptionType, str] = {
DrircOptionType.BOOL: "DRI_CONF_OPT_B",
DrircOptionType.INT: "DRI_CONF_OPT_I",
DrircOptionType.UINT64: "DRI_CONF_OPT_U64",
DrircOptionType.FLOAT: "DRI_CONF_OPT_F",
DrircOptionType.STRING: "DRI_CONF_OPT_S",
DrircOptionType.STRING_NODEF: "DRI_CONF_OPT_S_NODEF",
DrircOptionType.ENUM: "DRI_CONF_OPT_E",
}
type_to_queryfn: dict[DrircOptionType, str] = {
DrircOptionType.BOOL: "driQueryOptionb",
DrircOptionType.INT: "driQueryOptioni",
DrircOptionType.UINT64: "driQueryOptionu64",
DrircOptionType.FLOAT: "driQueryOptionf",
DrircOptionType.STRING: "driQueryOptionstr",
DrircOptionType.STRING_NODEF: "driQueryOptionstr",
DrircOptionType.ENUM: "driQueryOptioni",
}
class DrircOption(object):
def __init__(self, dtype, name, description, c_name):
self.dtype = dtype
self.name = name
self.description = description
self.c_name = c_name
self.c_args = []
class DrircBool(DrircOption):
def __init__(self, name, value, description="", c_name=None):
super().__init__(DrircOptionType.BOOL, name, description, c_name)
self.value = value
self.c_args = ["true" if value else "false", f"\"{self.description}\""]
class DrircInt(DrircOption):
def __init__(self, name, value, min_value, max_value, description="", c_name=None):
super().__init__(DrircOptionType.INT, name, description, c_name)
self.value = value
self.min_value = min_value
self.max_value = max_value
self.c_args = [f"{value}", f"{min_value}", f"{max_value}", f"\"{self.description}\""]
class DrircUint64(DrircOption):
def __init__(self, name, value, min_value, max_value, description="", c_name=None):
super().__init__(DrircOptionType.UINT64, name, description, c_name)
self.value = value
self.min_value = min_value
self.max_value = max_value
self.c_args = [f"{value}", f"{min_value}", f"{max_value}", f"\"{self.description}\""]
class DrircFloat(DrircOption):
def __init__(self, name, value, min_value, max_value, description="", c_name=None):
super().__init__(DrircOptionType.FLOAT, name, description, c_name)
self.value = value
self.min_value = min_value
self.max_value = max_value
self.c_args = [f"{value}", f"{min_value}", f"{max_value}", f"\"{self.description}\""]
class DrircString(DrircOption):
def __init__(self, name, value=None, description="", c_name=None):
dtype = DrircOptionType.STRING if value is not None else DrircOptionType.STRING_NODEF
super().__init__(dtype, name, description, c_name)
self.value = value
if value is not None:
self.c_args = [f'"{value}"', f'"{self.description}"']
else:
self.c_args = [f'"{self.description}"']
class DrircEnumValue(object):
def __init__(self, value, description=""):
self.value = value
self.description = description
class DrircEnum(DrircOption):
def __init__(self, name, value, min_value, max_value, values, description="", c_name=None):
super().__init__(DrircOptionType.ENUM, name, description, c_name)
self.values = values
self.value = value
self.min_value = min_value
self.max_value = max_value
self.c_args = [f"{value}", f"{min_value}", f"{min_value}", f"\"{self.description}\""]
vals = []
for v in values:
vals.append(f"DRI_CONF_ENUM({v.value}, \"{v.description}\")")
self.c_args.append(''.join(vals))
class DrircSection(object):
"""Class that represents a DRIRC section
"""
def __init__(self, name, options=[], c_name=None):
"""Parameters:
- options: list of DrircOption
"""
self.name = name
self.options = options
self.c_name = c_name if c_name is not None else name.lower()
TEMPLATE_H = """\
/* This file is autogenerated. Do not edit. */
#ifndef ${driver_prefix.upper()}_DRIRC_H
#define ${driver_prefix.upper()}_DRIRC_H
#ifdef __cplusplus
extern "C" {
#endif
#include "util/driconf.h"
struct ${driver_prefix}_drirc {
struct driOptionCache options;
struct driOptionCache available_options;
% for section in sections:
struct {
% for option in section.options:
% if option.c_name is not None:
${type_to_ctype(option.dtype)} ${option.c_name};
% endif
% endfor
} ${section.c_name};
% endfor
};
void ${driver_prefix}_parse_dri_options(struct ${driver_prefix}_drirc *drirc,
const driConfigFileParseParams *params);
#ifdef __cplusplus
}
#endif
#endif /* ${driver_prefix.upper()}_DRIRC_H */
"""
TEMPLATE_C = """\
/* This file is autogenerated. Do not edit. */
#include "${include_file}"
static const driOptionDescription dri_options[] = {
% for section in sections:
DRI_CONF_SECTION("${section.name}")
% for option in section.options:
${type_to_macro(option.dtype)}(${option.name}
% for arg in option.c_args:
, ${arg}
% endfor
)
% endfor
DRI_CONF_SECTION_END
% endfor
};
void
${driver_prefix}_parse_dri_options(struct ${driver_prefix}_drirc *drirc,
const driConfigFileParseParams *params)
{
driParseOptionInfo(&drirc->available_options, dri_options, ARRAY_SIZE(dri_options));
driParseConfigFiles(&drirc->options, &drirc->available_options, params);
% for section in sections:
% for option in section.options:
% if option.c_name is not None:
drirc->${section.c_name}.${option.c_name} = ${type_to_queryfn(option.dtype)}(&drirc->options, "${option.name}");
% endif
% endfor
% endfor
}
"""
def drirc_validate(conf_paths, sections, driver=None):
declared = {opt.name for section in sections for opt in section.options}
conf_names = set()
for conf_path in conf_paths:
tree = ET.parse(conf_path)
if driver is None:
for option in tree.iter('option'):
name = option.get('name')
if name:
conf_names.add(name)
else:
for device in tree.iter('device'):
if device.get('driver') != driver:
continue
for option in device.iter('option'):
name = option.get('name')
if name:
conf_names.add(name)
missing = conf_names - declared
if missing:
print('ERROR: options used in conf but not declared:', file=sys.stderr)
for name in sorted(missing):
print(f' {name}', file=sys.stderr)
sys.exit(1)
def drirc_generate(cpath, hpath, driver_prefix, sections):
environment = {
'driver_prefix': driver_prefix,
'sections': sections,
'type_to_ctype': lambda dtype: type_to_ctype[dtype],
'type_to_macro': lambda dtype: type_to_macro[dtype],
'type_to_queryfn': lambda dtype: type_to_queryfn[dtype],
'include_file': os.path.basename(hpath),
}
with open(cpath, 'w', encoding='utf-8') as f:
try:
result = f.write(Template(TEMPLATE_C).render(**environment))
except Exception:
from mako import exceptions
print(exceptions.text_error_template().render(), file=sys.stderr)
sys.exit(1)
with open(hpath, 'w', encoding='utf-8') as f:
try:
result = f.write(Template(TEMPLATE_H).render(**environment))
except Exception:
from mako import exceptions
print(exceptions.text_error_template().render(), file=sys.stderr)
sys.exit(1)
def add_common_vk_wsi_options(debug_options, performance_options):
B = DrircBool
I = DrircInt
performance_options.extend([
B("adaptive_sync", True,
"Adapt the monitor sync to the application performance (when possible)"),
I("vk_x11_override_min_image_count", 0, 0, 999,
"Override the VkSurfaceCapabilitiesKHR::minImageCount (0 = no override)"),
B("vk_x11_strict_image_count", False,
"Force the X11 WSI to create exactly the number of image specified by the application in VkSwapchainCreateInfoKHR::minImageCount"),
B("vk_x11_ensure_min_image_count", False,
"Force the X11 WSI to create at least the number of image specified by the driver in VkSurfaceCapabilitiesKHR::minImageCount"),
B("vk_xwayland_wait_ready", False,
"Wait for fences before submitting buffers to Xwayland"),
])
debug_options.extend([
B("vk_wsi_force_bgra8_unorm_first", False,
"Force vkGetPhysicalDeviceSurfaceFormatsKHR to return VK_FORMAT_B8G8R8A8_UNORM as the first format"),
B("vk_wsi_force_swapchain_to_current_extent", False,
"Force VkSwapchainCreateInfoKHR::imageExtent to be VkSurfaceCapabilities2KHR::currentExtent"),
B("vk_wsi_disable_unordered_submits", False,
"Disable unordered WSI submits to workaround application synchronization bugs"),
B("vk_x11_ignore_suboptimal", False,
"Force the X11 WSI to never report VK_SUBOPTIMAL_KHR"),
])
src/util/drm_is_nouveau.h
+18
View File
@@ -0,0 +1,18 @@
#ifndef DRM_IS_NOUVEAU
#define DRM_IS_NOUVEAU
#include "util/libdrm.h"
static inline bool
drm_fd_is_nouveau(int fd)
{
drmVersionPtr ver = drmGetVersion(fd);
if (!ver) {
return false;
}
const bool is_nouveau = !strncmp("nouveau", ver->name, ver->name_len);
drmFreeVersion(ver);
return is_nouveau;
}
#endif
src/util/enum_operators.h
+74
View File
@@ -0,0 +1,74 @@
/*
* Copyright © 2020 Red Hat Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
#if defined(__cplusplus) && !defined(__OPENCL_VERSION__)
#include "macros.h"
#include <type_traits>
// some enum helpers
#define MESA_DEFINE_CPP_ENUM_BINARY_OPERATOR(Enum, op) \
extern "C++" { \
UNUSED static constexpr \
Enum operator op (Enum a, Enum b) \
{ \
using IntType = std::underlying_type_t<Enum>; \
IntType ua = static_cast<IntType>(a); \
IntType ub = static_cast<IntType>(b); \
return static_cast<Enum>(ua op ub); \
} \
\
UNUSED static constexpr \
Enum& operator op##= (Enum &a, Enum b) \
{ \
using IntType = std::underlying_type_t<Enum>; \
IntType ua = static_cast<IntType>(a); \
IntType ub = static_cast<IntType>(b); \
ua op##= ub; \
a = static_cast<Enum>(ua); \
return a; \
} \
}
#define MESA_DEFINE_CPP_ENUM_UNARY_OPERATOR(Enum, op) \
extern "C++" { \
UNUSED static constexpr \
Enum operator op (Enum a) \
{ \
using IntType = std::underlying_type_t<Enum>; \
IntType ua = static_cast<IntType>(a); \
return static_cast<Enum>(op ua); \
} \
}
#define MESA_DEFINE_CPP_ENUM_BITFIELD_OPERATORS(Enum) \
MESA_DEFINE_CPP_ENUM_BINARY_OPERATOR(Enum, |) \
MESA_DEFINE_CPP_ENUM_BINARY_OPERATOR(Enum, &) \
MESA_DEFINE_CPP_ENUM_BINARY_OPERATOR(Enum, ^) \
MESA_DEFINE_CPP_ENUM_UNARY_OPERATOR(Enum, ~)
#else
#define MESA_DEFINE_CPP_ENUM_BITFIELD_OPERATORS(Enum)
#endif
src/util/fast_idiv_by_const.c
+237
View File
@@ -0,0 +1,237 @@
/*
* Copyright © 2018 Advanced Micro Devices, Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
/* Imported from:
* https://raw.githubusercontent.com/ridiculousfish/libdivide/master/divide_by_constants_codegen_reference.c
* Paper:
* http://ridiculousfish.com/files/faster_unsigned_division_by_constants.pdf
*
* The author, ridiculous_fish, wrote:
*
* ''Reference implementations of computing and using the "magic number"
* approach to dividing by constants, including codegen instructions.
* The unsigned division incorporates the "round down" optimization per
* ridiculous_fish.
*
* This is free and unencumbered software. Any copyright is dedicated
* to the Public Domain.''
*/
#include "fast_idiv_by_const.h"
#include "u_math.h"
#include "util/macros.h"
#include <limits.h>
#include <assert.h>
struct util_fast_udiv_info
util_compute_fast_udiv_info(uint64_t D, unsigned num_bits, unsigned UINT_BITS)
{
/* The numerator must fit in a uint64_t */
assert(num_bits > 0 && num_bits <= UINT_BITS);
assert(D != 0);
/* The eventual result */
struct util_fast_udiv_info result;
if (util_is_power_of_two_or_zero64(D)) {
unsigned div_shift = util_logbase2_64(D);
if (div_shift) {
/* Dividing by a power of two. */
result.multiplier = 1ull << (UINT_BITS - div_shift);
result.pre_shift = 0;
result.post_shift = 0;
result.increment = 0;
return result;
} else {
/* Dividing by 1. */
/* Assuming: floor((num + 1) * (2^32 - 1) / 2^32) = num */
result.multiplier = u_uintN_max(UINT_BITS);
result.pre_shift = 0;
result.post_shift = 0;
result.increment = 1;
return result;
}
}
/* The extra shift implicit in the difference between UINT_BITS and num_bits
*/
const unsigned extra_shift = UINT_BITS - num_bits;
/* The initial power of 2 is one less than the first one that can possibly
* work.
*/
const uint64_t initial_power_of_2 = (uint64_t)1 << (UINT_BITS-1);
/* The remainder and quotient of our power of 2 divided by d */
uint64_t quotient = initial_power_of_2 / D;
uint64_t remainder = initial_power_of_2 % D;
/* ceil(log_2 D) */
unsigned ceil_log_2_D;
/* The magic info for the variant "round down" algorithm */
uint64_t down_multiplier = 0;
unsigned down_exponent = 0;
int has_magic_down = 0;
/* Compute ceil(log_2 D) */
ceil_log_2_D = 0;
uint64_t tmp;
for (tmp = D; tmp > 0; tmp >>= 1)
ceil_log_2_D += 1;
/* Begin a loop that increments the exponent, until we find a power of 2
* that works.
*/
unsigned exponent;
for (exponent = 0; ; exponent++) {
/* Quotient and remainder is from previous exponent; compute it for this
* exponent.
*/
if (remainder >= D - remainder) {
/* Doubling remainder will wrap around D */
quotient = quotient * 2 + 1;
remainder = remainder * 2 - D;
} else {
/* Remainder will not wrap */
quotient = quotient * 2;
remainder = remainder * 2;
}
/* We're done if this exponent works for the round_up algorithm.
* Note that exponent may be larger than the maximum shift supported,
* so the check for >= ceil_log_2_D is critical.
*/
if ((exponent + extra_shift >= ceil_log_2_D) ||
(D - remainder) <= ((uint64_t)1 << (exponent + extra_shift)))
break;
/* Set magic_down if we have not set it yet and this exponent works for
* the round_down algorithm
*/
if (!has_magic_down &&
remainder <= ((uint64_t)1 << (exponent + extra_shift))) {
has_magic_down = 1;
down_multiplier = quotient;
down_exponent = exponent;
}
}
if (exponent < ceil_log_2_D) {
/* magic_up is efficient */
result.multiplier = quotient + 1;
result.pre_shift = 0;
result.post_shift = exponent;
result.increment = 0;
} else if (D & 1) {
/* Odd divisor, so use magic_down, which must have been set */
assert(has_magic_down);
result.multiplier = down_multiplier;
result.pre_shift = 0;
result.post_shift = down_exponent;
result.increment = 1;
} else {
/* Even divisor, so use a prefix-shifted dividend */
unsigned pre_shift = 0;
uint64_t shifted_D = D;
while ((shifted_D & 1) == 0) {
shifted_D >>= 1;
pre_shift += 1;
}
result = util_compute_fast_udiv_info(shifted_D, num_bits - pre_shift,
UINT_BITS);
/* expect no increment or pre_shift in this path */
assert(result.increment == 0 && result.pre_shift == 0);
result.pre_shift = pre_shift;
}
return result;
}
struct util_fast_sdiv_info
util_compute_fast_sdiv_info(int64_t D, unsigned SINT_BITS)
{
/* D must not be zero. */
assert(D != 0);
/* The result is not correct for these divisors. */
assert(D != 1 && D != -1);
/* Our result */
struct util_fast_sdiv_info result;
/* Absolute value of D (we know D is not the most negative value since
* that's a power of 2)
*/
const uint64_t abs_d = (D < 0 ? -D : D);
/* The initial power of 2 is one less than the first one that can possibly
* work */
/* "two31" in Warren */
unsigned exponent = SINT_BITS - 1;
const uint64_t initial_power_of_2 = (uint64_t)1 << exponent;
/* Compute the absolute value of our "test numerator,"
* which is the largest dividend whose remainder with d is d-1.
* This is called anc in Warren.
*/
const uint64_t tmp = initial_power_of_2 + (D < 0);
const uint64_t abs_test_numer = tmp - 1 - tmp % abs_d;
/* Initialize our quotients and remainders (q1, r1, q2, r2 in Warren) */
uint64_t quotient1 = initial_power_of_2 / abs_test_numer;
uint64_t remainder1 = initial_power_of_2 % abs_test_numer;
uint64_t quotient2 = initial_power_of_2 / abs_d;
uint64_t remainder2 = initial_power_of_2 % abs_d;
uint64_t delta;
/* Begin our loop */
do {
/* Update the exponent */
exponent++;
/* Update quotient1 and remainder1 */
quotient1 *= 2;
remainder1 *= 2;
if (remainder1 >= abs_test_numer) {
quotient1 += 1;
remainder1 -= abs_test_numer;
}
/* Update quotient2 and remainder2 */
quotient2 *= 2;
remainder2 *= 2;
if (remainder2 >= abs_d) {
quotient2 += 1;
remainder2 -= abs_d;
}
/* Keep going as long as (2**exponent) / abs_d <= delta */
delta = abs_d - remainder2;
} while (quotient1 < delta || (quotient1 == delta && remainder1 == 0));
result.multiplier = util_sign_extend(quotient2 + 1, SINT_BITS);
if (D < 0) result.multiplier = -result.multiplier;
result.shift = exponent - SINT_BITS;
return result;
}
src/util/fast_idiv_by_const.h
+183
View File
@@ -0,0 +1,183 @@
/*
* Copyright © 2018 Advanced Micro Devices, Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
#ifndef FAST_IDIV_BY_CONST_H
#define FAST_IDIV_BY_CONST_H
/* Imported from:
* https://raw.githubusercontent.com/ridiculousfish/libdivide/master/divide_by_constants_codegen_reference.c
*/
#include <inttypes.h>
#include <limits.h>
#include <assert.h>
#ifdef __cplusplus
extern "C" {
#endif
/* Computes "magic info" for performing signed division by a fixed integer D.
* The type 'sint_t' is assumed to be defined as a signed integer type large
* enough to hold both the dividend and the divisor.
* Here >> is arithmetic (signed) shift, and >>> is logical shift.
*
* To emit code for n/d, rounding towards zero, use the following sequence:
*
* m = compute_signed_magic_info(D)
* emit("result = (m.multiplier * n) >> SINT_BITS");
* if d > 0 and m.multiplier < 0: emit("result += n")
* if d < 0 and m.multiplier > 0: emit("result -= n")
* if m.post_shift > 0: emit("result >>= m.shift")
* emit("result += (result < 0)")
*
* The shifts by SINT_BITS may be "free" if the high half of the full multiply
* is put in a separate register.
*
* The final add can of course be implemented via the sign bit, e.g.
* result += (result >>> (SINT_BITS - 1))
* or
* result -= (result >> (SINT_BITS - 1))
*
* This code is heavily indebted to Hacker's Delight by Henry Warren.
* See http://www.hackersdelight.org/HDcode/magic.c.txt
* Used with permission from http://www.hackersdelight.org/permissions.htm
*/
struct util_fast_sdiv_info {
int64_t multiplier; /* the "magic number" multiplier */
unsigned shift; /* shift for the dividend after multiplying */
};
struct util_fast_sdiv_info
util_compute_fast_sdiv_info(int64_t D, unsigned SINT_BITS);
/* Computes "magic info" for performing unsigned division by a fixed positive
* integer D. UINT_BITS is the bit size at which the final "magic"
* calculation will be performed; it is assumed to be large enough to hold
* both the dividand and the divisor. num_bits can be set appropriately if n
* is known to be smaller than calc_bits; if this is not known then UINT_BITS
* for num_bits.
*
* Assume we have a hardware register of width UINT_BITS, a known constant D
* which is not zero and not a power of 2, and a variable n of width num_bits
* (which may be up to UINT_BITS). To emit code for n/d, use one of the two
* following sequences (here >>> refers to a logical bitshift):
*
* m = compute_unsigned_magic_info(D, num_bits)
* if m.pre_shift > 0: emit("n >>>= m.pre_shift")
* if m.increment: emit("n = saturated_increment(n)")
* emit("result = (m.multiplier * n) >>> UINT_BITS")
* if m.post_shift > 0: emit("result >>>= m.post_shift")
*
* or
*
* m = compute_unsigned_magic_info(D, num_bits)
* if m.pre_shift > 0: emit("n >>>= m.pre_shift")
* emit("result = m.multiplier * n")
* if m.increment: emit("result = result + m.multiplier")
* emit("result >>>= UINT_BITS")
* if m.post_shift > 0: emit("result >>>= m.post_shift")
*
* This second version works even if D is 1. The shifts by UINT_BITS may be
* "free" if the high half of the full multiply is put in a separate register.
*
* saturated_increment(n) means "increment n unless it would wrap to 0," i.e.
* if n == (1 << UINT_BITS)-1: result = n
* else: result = n+1
* A common way to implement this is with the carry bit. For example, on x86:
* add 1
* sbb 0
*
* Some invariants:
* 1: At least one of pre_shift and increment is zero
* 2: multiplier is never zero
*
* This code incorporates the "round down" optimization per ridiculous_fish.
*/
struct util_fast_udiv_info {
uint64_t multiplier; /* the "magic number" multiplier */
unsigned pre_shift; /* shift for the dividend before multiplying */
unsigned post_shift; /* shift for the dividend after multiplying */
int increment; /* 0 or 1; if set then increment the numerator, using one of
the two strategies */
};
struct util_fast_udiv_info
util_compute_fast_udiv_info(uint64_t D, unsigned num_bits, unsigned UINT_BITS);
/* Below are possible options for dividing by a uniform in a shader where
* the divisor is constant but not known at compile time.
*/
/* Full version. */
static inline uint32_t
util_fast_udiv32(uint32_t n, struct util_fast_udiv_info info)
{
n = n >> info.pre_shift;
/* If the divisor is not 1, you can instead use a 32-bit ADD that clamps
* to UINT_MAX. Dividing by 1 needs the full 64-bit ADD.
*
* If you have unsigned 64-bit MAD with 32-bit inputs, you can do:
* increment = increment ? multiplier : 0; // on the CPU
* (n * multiplier + increment) // on the GPU using unsigned 64-bit MAD
*/
n = (((uint64_t)n + info.increment) * info.multiplier) >> 32;
n = n >> info.post_shift;
return n;
}
/* A little more efficient version if n != UINT_MAX, i.e. no unsigned
* wraparound in the computation.
*/
static inline uint32_t
util_fast_udiv32_nuw(uint32_t n, struct util_fast_udiv_info info)
{
assert(n != UINT32_MAX);
n = n >> info.pre_shift;
n = n + info.increment;
n = ((uint64_t)n * info.multiplier) >> 32;
n = n >> info.post_shift;
return n;
}
/* Even faster version but both operands must be 31-bit unsigned integers
* and the divisor must be greater than 1.
*
* info must be computed with num_bits == 31.
*/
static inline uint32_t
util_fast_udiv32_u31_d_not_one(uint32_t n, struct util_fast_udiv_info info)
{
assert(info.pre_shift == 0);
assert(info.increment == 0);
n = ((uint64_t)n * info.multiplier) >> 32;
n = n >> info.post_shift;
return n;
}
#ifdef __cplusplus
} /* extern C */
#endif
#endif /* FAST_IDIV_BY_CONST_H */
src/util/fast_urem_by_const.h
+76
View File
@@ -0,0 +1,76 @@
/*
* Copyright © 2010 Valve Software
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
#include <stdint.h>
/*
* Code for fast 32-bit unsigned remainder, based off of "Faster Remainder by
* Direct Computation: Applications to Compilers and Software Libraries,"
* available at https://arxiv.org/pdf/1902.01961.pdf.
*
* util_fast_urem32(n, d, REMAINDER_MAGIC(d)) returns the same thing as
* n % d for any unsigned n and d, however it compiles down to only a few
* multiplications, so it should be faster than plain uint32_t modulo if the
* same divisor is used many times.
*/
#define REMAINDER_MAGIC(divisor) \
((uint64_t) ~0ull / (divisor) + 1)
/*
* Get bits 64-96 of a 32x64-bit multiply. If __int128_t is available, we use
* it, which usually compiles down to one instruction on 64-bit architectures.
* Otherwise on 32-bit architectures we usually get four instructions (one
* 32x32->64 multiply, one 32x32->32 multiply, and one 64-bit add).
*/
static inline uint32_t
_mul32by64_hi(uint32_t a, uint64_t b)
{
#ifdef HAVE_UINT128
return ((__uint128_t) b * a) >> 64;
#else
/*
* Let b = b0 + 2^32 * b1. Then a * b = a * b0 + 2^32 * a * b1. We would
* have to do a 96-bit addition to get the full result, except that only
* one term has non-zero lower 32 bits, which means that to get the high 32
* bits, we only have to add the high 64 bits of each term. Unfortunately,
* we have to do the 64-bit addition in case the low 32 bits overflow.
*/
uint32_t b0 = (uint32_t) b;
uint32_t b1 = b >> 32;
return ((((uint64_t) a * b0) >> 32) + (uint64_t) a * b1) >> 32;
#endif
}
static inline uint32_t
util_fast_urem32(uint32_t n, uint32_t d, uint64_t magic)
{
uint64_t lowbits = magic * n;
uint32_t result = _mul32by64_hi(d, lowbits);
#if MESA_DEBUG
assert(result == n % d);
#endif
return result;
}
src/util/float8.c
+192
View File
@@ -0,0 +1,192 @@
/*
* Copyright 2025 Valve Corporation
* SPDX-License-Identifier: MIT
*/
#include "float8.h"
#include <assert.h>
#include <math.h>
#include "bitscan.h"
#include "u_math.h"
#define E4M3FN_NAN 0xff
#define E4M3FN_MAX 0x7e
#define E5M2_NAN 0xfe
#define E5M2_MAX 0x7b
#define E5M2_INF 0x7c
uint8_t
_mesa_float_to_e4m3fn(float val)
{
/* This is a finite only format, out of range values (after rounding)
* are converted to NaN.
*/
if (fabs(val) > 464.0f || isnan(val))
return E4M3FN_NAN;
bool s = fui(val) & 0x80000000;
int e = ((fui(val) >> 23) & 0xff) - 127 + 7;
uint32_t m = fui(val) & 0x7fffff;
uint8_t res = s ? 0x80 : 0;
/* Zero, underflow. */
if (e < -3)
return res;
bool is_denorm = e <= 0;
bool round_up = false;
if (is_denorm) {
unsigned offset = 1 - e;
round_up |= m & ((1 << offset) - 1);
m = (m | 0x800000) >> offset;
}
round_up |= m & 0x17ffff;
if ((m & 0x080000) && round_up) {
m += 0x100000;
if (m & 0x800000) {
m = 0;
e += 1;
}
}
if (!is_denorm)
res |= (e << 3);
res |= (m >> 20);
return res;
}
uint8_t
_mesa_float_to_e4m3fn_sat(float val)
{
if (val > 448.0f)
return E4M3FN_MAX;
else if (val < -448.0f)
return 0x80 | E4M3FN_MAX;
else
return _mesa_float_to_e4m3fn(val);
}
float
_mesa_e4m3fn_to_float(uint8_t val)
{
bool s = val & 0x80;
uint32_t e = (val >> 3) & 0xf;
uint32_t m = val & 0x7;
if (e == 0xf && m == 0x7)
return uif(0xffc00000);
uint32_t res = s ? 0x80000000 : 0;
if (e == 0 && m == 0) {
/* Zero. */
} else if (e == 0) {
/* Denorm. */
unsigned shift = (4 - util_last_bit(m));
res |= (127 - 6 - shift) << 23;
res |= ((m << shift) & 0x7) << (23 - 3);
} else {
res |= (e + (127 - 7)) << 23;
res |= m << (23 - 3);
}
return uif(res);
}
uint8_t
_mesa_float_to_e5m2(float val)
{
bool s = fui(val) & 0x80000000;
uint8_t res = s ? 0x80 : 0;
if (isnan(val))
return E5M2_NAN;
else if (fabs(val) >= 61440.0f)
return res | E5M2_INF;
int e = ((fui(val) >> 23) & 0xff) - 127 + 15;
uint32_t m = fui(val) & 0x7fffff;
/* Zero, underflow. */
if (e < -2)
return res;
bool is_denorm = e <= 0;
bool round_up = false;
if (is_denorm) {
unsigned offset = 1 - e;
round_up |= m & ((1 << offset) - 1);
m = (m | 0x800000) >> offset;
}
round_up |= m & 0x2fffff;
if ((m & 0x100000) && round_up) {
m += 0x200000;
if (m & 0x800000) {
m = 0;
e += 1;
}
}
if (!is_denorm)
res |= (e << 2);
res |= (m >> 21);
return res;
}
uint8_t
_mesa_float_to_e5m2_sat(float val)
{
if (val > 57344.0f)
return E5M2_MAX;
else if (val < -57344.0f)
return 0x80 | E5M2_MAX;
else
return _mesa_float_to_e5m2(val);
}
float
_mesa_e5m2_to_float(uint8_t val)
{
bool s = val & 0x80;
uint32_t e = (val >> 2) & 0x1f;
uint32_t m = val & 0x3;
if (e == 0x1f && m != 0)
return uif(0xffc00000);
uint32_t res = s ? 0x80000000 : 0;
if (e == 0x1f) {
/* Infinity. */
res |= 0x7f800000;
} else if (e == 0 && m == 0) {
/* Zero. */
} else if (e == 0) {
/* Denorm. */
unsigned shift = (3 - util_last_bit(m));
res |= (127 - 14 - shift) << 23;
res |= ((m << shift) & 0x3) << (23 - 2);
} else {
res |= (e + (127 - 15)) << 23;
res |= m << (23 - 2);
}
return uif(res);
}
src/util/float8.h
+24
View File
@@ -0,0 +1,24 @@
/*
* Copyright 2025 Valve Corporation
* SPDX-License-Identifier: MIT
*/
#pragma once
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
uint8_t _mesa_float_to_e4m3fn(float val);
uint8_t _mesa_float_to_e4m3fn_sat(float val);
float _mesa_e4m3fn_to_float(uint8_t val);
uint8_t _mesa_float_to_e5m2(float val);
uint8_t _mesa_float_to_e5m2_sat(float val);
float _mesa_e5m2_to_float(uint8_t val);
#ifdef __cplusplus
} /* extern C */
#endif
src/util/format/format_utils.h
+210
View File
@@ -0,0 +1,210 @@
/**
* \file format_utils.h
* A collection of format conversion utility functions.
*/
/*
* Mesa 3-D graphics library
*
* Copyright (C) 1999-2006 Brian Paul All Rights Reserved.
* Copyright (C) 2014 Intel Corporation All Rights Reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included
* in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
#ifndef UTIL_FORMAT_UTILS_H
#define UTIL_FORMAT_UTILS_H
#include "util/half_float.h"
#include "util/rounding.h"
/* Extends an integer of size SRC_BITS to one of size DST_BITS linearly */
#define EXTEND_NORMALIZED_INT(X, SRC_BITS, DST_BITS) \
(((X) * (int)((unsigned)u_uintN_max(DST_BITS) / (unsigned)u_uintN_max(SRC_BITS))) + \
(((DST_BITS) % (SRC_BITS)) ? ((X) >> ((SRC_BITS) - (DST_BITS) % (SRC_BITS))) : 0))
static inline float
_mesa_unorm_to_float(unsigned x, unsigned src_bits)
{
return x * (1.0f / (float)u_uintN_max(src_bits));
}
static inline float
_mesa_snorm_to_float(int x, unsigned src_bits)
{
if (x <= -u_intN_max(src_bits))
return -1.0f;
else
return x * (1.0f / (float)u_intN_max(src_bits));
}
static inline uint16_t
_mesa_unorm_to_half(unsigned x, unsigned src_bits)
{
return _mesa_float_to_half(_mesa_unorm_to_float(x, src_bits));
}
static inline uint16_t
_mesa_snorm_to_half(int x, unsigned src_bits)
{
return _mesa_float_to_half(_mesa_snorm_to_float(x, src_bits));
}
static inline unsigned
_mesa_float_to_unorm(float x, unsigned dst_bits)
{
if (x < 0.0f)
return 0;
else if (x > 1.0f)
return u_uintN_max(dst_bits);
else
return _mesa_i64roundevenf(x * u_uintN_max(dst_bits));
}
static inline unsigned
_mesa_half_to_unorm(uint16_t x, unsigned dst_bits)
{
return _mesa_float_to_unorm(_mesa_half_to_float(x), dst_bits);
}
static inline unsigned
_mesa_unorm_to_unorm(unsigned x, unsigned src_bits, unsigned dst_bits)
{
if (src_bits < dst_bits) {
assert(dst_bits <= sizeof(unsigned) * 8);
return EXTEND_NORMALIZED_INT(x, src_bits, dst_bits);
} else if (src_bits > dst_bits) {
unsigned src_half = (1u << (src_bits - 1)) - 1;
unsigned max_dst = (unsigned)u_uintN_max(dst_bits);
unsigned max_src = (unsigned)u_uintN_max(src_bits);
if (src_bits + dst_bits > sizeof(x) * 8) {
assert(src_bits + dst_bits <= sizeof(uint64_t) * 8);
return ((uint64_t) x * max_dst + src_half) / max_src;
} else {
return (x * max_dst + src_half) / max_src;
}
} else {
return x;
}
}
static inline unsigned
_mesa_snorm_to_unorm(int x, unsigned src_bits, unsigned dst_bits)
{
if (x < 0)
return 0;
else
return _mesa_unorm_to_unorm(x, src_bits - 1, dst_bits);
}
static inline int
_mesa_float_to_snorm(float x, unsigned dst_bits)
{
if (x < -1.0f)
return -u_intN_max(dst_bits);
else if (x > 1.0f)
return u_intN_max(dst_bits);
else
return _mesa_lroundevenf(x * u_intN_max(dst_bits));
}
static inline int
_mesa_half_to_snorm(uint16_t x, unsigned dst_bits)
{
return _mesa_float_to_snorm(_mesa_half_to_float(x), dst_bits);
}
static inline int
_mesa_unorm_to_snorm(unsigned x, unsigned src_bits, unsigned dst_bits)
{
return _mesa_unorm_to_unorm(x, src_bits, dst_bits - 1);
}
static inline int
_mesa_snorm_to_snorm(int x, unsigned src_bits, unsigned dst_bits)
{
if (x < -u_intN_max(src_bits))
return -u_intN_max(dst_bits);
else if (src_bits < dst_bits){
assert(dst_bits <= sizeof(unsigned) * 8);
return EXTEND_NORMALIZED_INT(x, src_bits - 1, dst_bits - 1);
} else
return x >> (src_bits - dst_bits);
}
static inline unsigned
_mesa_unsigned_to_unsigned(unsigned src, unsigned dst_size)
{
return MIN2(src, u_uintN_max(dst_size));
}
static inline int
_mesa_unsigned_to_signed(unsigned src, unsigned dst_size)
{
return MIN2(src, (unsigned)u_intN_max(dst_size));
}
static inline int
_mesa_signed_to_signed(int src, unsigned dst_size)
{
return CLAMP(src, u_intN_min(dst_size), u_intN_max(dst_size));
}
static inline unsigned
_mesa_signed_to_unsigned(int src, unsigned dst_size)
{
return CLAMP(src, 0, u_uintN_max(dst_size));
}
static inline unsigned
_mesa_float_to_unsigned(float src, unsigned dst_bits)
{
if (src < 0.0f)
return 0;
if (src > (float)u_uintN_max(dst_bits))
return u_uintN_max(dst_bits);
return _mesa_signed_to_unsigned(src, dst_bits);
}
static inline unsigned
_mesa_float_to_signed(float src, unsigned dst_bits)
{
if (src < (float)(-u_intN_max(dst_bits)))
return -u_intN_max(dst_bits);
if (src > (float)u_intN_max(dst_bits))
return u_intN_max(dst_bits);
return _mesa_signed_to_signed(src, dst_bits);
}
static inline unsigned
_mesa_half_to_unsigned(uint16_t src, unsigned dst_bits)
{
if (_mesa_half_is_negative(src))
return 0;
return _mesa_unsigned_to_unsigned(_mesa_float_to_half(src), dst_bits);
}
static inline unsigned
_mesa_half_to_signed(uint16_t src, unsigned dst_bits)
{
return _mesa_float_to_signed(_mesa_half_to_float(src), dst_bits);
}
#endif /* UTIL_FORMAT_UTILS_H */
src/util/format/meson.build
+48
View File
@@ -0,0 +1,48 @@
# Copyright © 2019 Google LLC
# SPDX-License-Identifier: MIT
files_mesa_format = files(
'u_format.c',
'u_format_bptc.c',
'u_format_etc.c',
'u_format_fxt1.c',
'u_format_latc.c',
'u_format_other.c',
'u_format_rgtc.c',
'u_format_s3tc.c',
'u_format_tests.c',
'u_format_unpack_neon.c',
'u_format_yuv.c',
'u_format_zs.c',
)
u_format_gen_h = custom_target(
'u_format_gen.h',
input : ['u_format_table.py', 'u_format.yaml'],
output : 'u_format_gen.h',
command : [prog_python, '@INPUT@', '--enums'],
depend_files : files('u_format_pack.py', 'u_format_parse.py'),
capture : true,
)
u_format_pack_h = custom_target(
'u_format_pack.h',
input : ['u_format_table.py', 'u_format.yaml'],
output : 'u_format_pack.h',
command : [prog_python, '@INPUT@', '--header'],
depend_files : files('u_format_pack.py', 'u_format_parse.py'),
capture : true,
)
u_format_table_c = custom_target(
'u_format_table.c',
input : ['u_format_table.py', 'u_format.yaml'],
output : 'u_format_table.c',
command : [prog_python, '@INPUT@'],
depend_files : files('u_format_pack.py', 'u_format_parse.py'),
capture : true,
)
idep_mesautilformat = declare_dependency(sources: u_format_gen_h)
files_mesa_format += [u_format_gen_h, u_format_pack_h, u_format_table_c]
src/util/format/texcompress_bptc_tmp.h
+1861
View File
File diff suppressed because it is too large. Load diff
Loaded 100 of 12511 files, more files were not shown because too many files have changed in this diff. Show more