mirror of
https://github.com/elwrcl/intel-hard-graphics.git
synced 2026-10-31 19:08:03 +03:00
init
This commit is contained in:
12511 files changed
+6156641
No files matched your search
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"file_format_version": "1.2.1",
|
||||
"layer": {
|
||||
"name": "VK_LAYER_MESA_anti_lag",
|
||||
"type": "GLOBAL",
|
||||
"library_path": "@library_path@",
|
||||
"api_version": "1.4.303",
|
||||
"implementation_version": "1",
|
||||
"description": "Open-source implementation of the VK_AMD_anti_lag extension.",
|
||||
"functions": {
|
||||
"vkNegotiateLoaderLayerInterfaceVersion": "anti_lag_NegotiateLoaderLayerInterfaceVersion"
|
||||
},
|
||||
"device_extensions": [
|
||||
{
|
||||
"name": "VK_AMD_anti_lag",
|
||||
"spec_version": "1",
|
||||
"entrypoints": [
|
||||
"vkAntiLagUpdateAMD"
|
||||
]
|
||||
}
|
||||
],
|
||||
"disable_environment": {
|
||||
"DISABLE_LAYER_MESA_ANTI_LAG": "1"
|
||||
},
|
||||
"enable_environment": {
|
||||
"ENABLE_LAYER_MESA_ANTI_LAG": "1"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,740 @@
|
||||
/*
|
||||
* Copyright © 2025 Valve Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*/
|
||||
|
||||
#include "anti_lag_layer.h"
|
||||
#include "util/os_time.h"
|
||||
#include "util/simple_mtx.h"
|
||||
#include "util/u_atomic.h"
|
||||
#include "vulkan/vulkan_core.h"
|
||||
#include "ringbuffer.h"
|
||||
#include "vk_alloc.h"
|
||||
#include "vk_util.h"
|
||||
|
||||
#define ONE_MS INT64_C(1000000)
|
||||
|
||||
static bool
|
||||
evaluate_frame(device_context *ctx, frame *frame, bool force_wait)
|
||||
{
|
||||
/* This frame is not finished yet. */
|
||||
if (frame->state != FRAME_PRESENT)
|
||||
return false;
|
||||
|
||||
const uint32_t frame_idx = ringbuffer_index(ctx->frames, frame);
|
||||
|
||||
/* Before we commit to completing a frame, all submits on all queues must have completed. */
|
||||
for (unsigned i = 0; i < ctx->num_queues; i++) {
|
||||
queue_context *queue_ctx = &ctx->queues[i];
|
||||
ringbuffer_lock(queue_ctx->queries);
|
||||
uint64_t expected_signal_value = queue_ctx->semaphore_value - queue_ctx->queries.size / 2 +
|
||||
queue_ctx->submissions_per_frame[frame_idx] / 2;
|
||||
|
||||
ringbuffer_unlock(queue_ctx->queries);
|
||||
|
||||
if (force_wait) {
|
||||
/* Wait for the timeline semaphore of the frame to be signaled. */
|
||||
struct VkSemaphoreWaitInfo wait_info = {
|
||||
.sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO,
|
||||
.semaphoreCount = 1,
|
||||
.pSemaphores = &queue_ctx->semaphore,
|
||||
.pValues = &expected_signal_value,
|
||||
};
|
||||
ctx->vtable.WaitSemaphores(ctx->device, &wait_info, 0);
|
||||
} else {
|
||||
/* Return early if the last timeline semaphore of the frame has not been signaled yet. */
|
||||
uint64_t signal_value;
|
||||
ctx->vtable.GetSemaphoreCounterValue(ctx->device, queue_ctx->semaphore, &signal_value);
|
||||
if (signal_value < expected_signal_value)
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
uint64_t cpu_start_time = UINT64_MAX;
|
||||
uint64_t gpu_start_time = UINT64_MAX;
|
||||
uint64_t gpu_end_time = 0;
|
||||
int64_t min_delay = INT64_MAX;
|
||||
|
||||
/* For each queue, retrieve timestamp query results. */
|
||||
for (unsigned i = 0; i < ctx->num_queues; i++) {
|
||||
queue_context *queue_ctx = &ctx->queues[i];
|
||||
|
||||
/* As we hold a global mtx and this is the only place where queries are free'd,
|
||||
* we don't need to lock the query ringbuffer here in order to read the first entry.
|
||||
*/
|
||||
struct query *query = ringbuffer_first(queue_ctx->queries);
|
||||
uint32_t query_idx = ringbuffer_index(queue_ctx->queries, query);
|
||||
int num_timestamps =
|
||||
MIN2(queue_ctx->submissions_per_frame[frame_idx], MAX_QUERIES - query_idx);
|
||||
|
||||
while (num_timestamps > 0) {
|
||||
/* Retreive timestamp results from this queue. */
|
||||
ctx->vtable.GetQueryPoolResults(ctx->device, queue_ctx->queryPool, query_idx,
|
||||
num_timestamps, sizeof(struct query) * num_timestamps,
|
||||
&query->gpu_ts, sizeof(struct query),
|
||||
VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WAIT_BIT);
|
||||
|
||||
ringbuffer_lock(queue_ctx->queries);
|
||||
gpu_start_time = MIN2(gpu_start_time, query->gpu_ts);
|
||||
cpu_start_time = MIN2(cpu_start_time, query->submit_ts);
|
||||
|
||||
for (unsigned j = 0; j < num_timestamps; j++) {
|
||||
/* Even timestamps mark the begin and odd timestamps mark the end
|
||||
* of each submission.
|
||||
*/
|
||||
if (j % 2 == 0) {
|
||||
/* Calibrate device timestamps. */
|
||||
query->gpu_ts = ctx->calibration.delta +
|
||||
(uint64_t)(query->gpu_ts * ctx->calibration.timestamp_period);
|
||||
|
||||
/* This value might be negative due to timestamp inaccuracies. */
|
||||
int64_t submission_delay = query->gpu_ts - query->submit_ts;
|
||||
min_delay = MIN2(min_delay, submission_delay);
|
||||
} else {
|
||||
gpu_end_time = MAX2(gpu_end_time, query->gpu_ts);
|
||||
}
|
||||
|
||||
/* Check if we can reset half of the query pool at once. */
|
||||
uint32_t next_idx = ringbuffer_index(queue_ctx->queries, query) + 1;
|
||||
const bool reset = next_idx == MAX_QUERIES || next_idx == MAX_QUERIES / 2;
|
||||
if (reset) {
|
||||
ringbuffer_unlock(queue_ctx->queries);
|
||||
ctx->vtable.ResetQueryPool(ctx->device, queue_ctx->queryPool,
|
||||
next_idx - MAX_QUERIES / 2, MAX_QUERIES / 2);
|
||||
ringbuffer_lock(queue_ctx->queries);
|
||||
}
|
||||
|
||||
/* Free query. */
|
||||
ringbuffer_free(queue_ctx->queries, query);
|
||||
queue_ctx->submissions_per_frame[frame_idx]--;
|
||||
|
||||
query = ringbuffer_first(queue_ctx->queries);
|
||||
}
|
||||
|
||||
/* Ensure that the total number of queries across all frames is correct. */
|
||||
ASSERTED uint32_t count = 0;
|
||||
for (unsigned i = 0; i < MAX_FRAMES; i++)
|
||||
count += queue_ctx->submissions_per_frame[i];
|
||||
assert(count == queue_ctx->queries.size);
|
||||
|
||||
query_idx = ringbuffer_index(queue_ctx->queries, query);
|
||||
num_timestamps =
|
||||
MIN2(queue_ctx->submissions_per_frame[frame_idx], MAX_QUERIES - query_idx);
|
||||
|
||||
ringbuffer_unlock(queue_ctx->queries);
|
||||
}
|
||||
}
|
||||
|
||||
if (gpu_end_time) {
|
||||
frame->frame_time = (gpu_end_time - gpu_start_time) * ctx->calibration.timestamp_period;
|
||||
frame->min_delay = min_delay;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool
|
||||
calibrate_timestamps(device_context *ctx)
|
||||
{
|
||||
uint64_t ts[2];
|
||||
uint64_t deviation;
|
||||
|
||||
VkCalibratedTimestampInfoKHR info[2] = {
|
||||
{
|
||||
.sType = VK_STRUCTURE_TYPE_CALIBRATED_TIMESTAMP_INFO_KHR,
|
||||
.timeDomain = VK_TIME_DOMAIN_CLOCK_MONOTONIC_KHR,
|
||||
},
|
||||
{
|
||||
.sType = VK_STRUCTURE_TYPE_CALIBRATED_TIMESTAMP_INFO_KHR,
|
||||
.timeDomain = VK_TIME_DOMAIN_DEVICE_KHR,
|
||||
},
|
||||
};
|
||||
|
||||
VkResult result = ctx->vtable.GetCalibratedTimestampsKHR(ctx->device, 2, info, ts, &deviation);
|
||||
if (result == VK_SUCCESS) {
|
||||
/* We take a moving average in order to avoid variance. */
|
||||
int64_t new_delta = ts[0] - (int64_t)(ts[1] * ctx->calibration.timestamp_period);
|
||||
|
||||
if (ctx->calibration.delta == 0) {
|
||||
ctx->calibration.delta = new_delta;
|
||||
} else {
|
||||
int64_t diff = new_delta - ctx->calibration.delta;
|
||||
ctx->calibration.delta += diff / 8;
|
||||
}
|
||||
|
||||
/* Take a new calibrated timestamp every second. */
|
||||
ctx->calibration.recalibrate_when = ts[0] + ONE_SECOND_IN_NS;
|
||||
}
|
||||
|
||||
return result == VK_SUCCESS;
|
||||
}
|
||||
|
||||
static void
|
||||
begin_next_frame(device_context *ctx)
|
||||
{
|
||||
frame *next_frame;
|
||||
if (ctx->active_frame) {
|
||||
assert(ctx->active_frame->state == FRAME_SUBMIT);
|
||||
ctx->active_frame->state = FRAME_PRESENT;
|
||||
next_frame = ringbuffer_next(ctx->frames, ctx->active_frame);
|
||||
} else {
|
||||
next_frame = ringbuffer_last(ctx->frames);
|
||||
}
|
||||
|
||||
/* If there is a frame ready, it becomes active. */
|
||||
if (next_frame->state == FRAME_INPUT) {
|
||||
next_frame->state = FRAME_SUBMIT;
|
||||
ctx->active_frame = next_frame;
|
||||
} else {
|
||||
ctx->active_frame = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
static void
|
||||
anti_lag_disable(device_context *ctx)
|
||||
{
|
||||
ctx->avg_frame_time = ONE_MS;
|
||||
|
||||
ringbuffer_lock(ctx->frames);
|
||||
{
|
||||
while (ctx->frames.size) {
|
||||
/* Set force-wait=true, so that all pending timestamp queries get completed. */
|
||||
begin_next_frame(ctx);
|
||||
frame *frame = ringbuffer_first(ctx->frames);
|
||||
evaluate_frame(ctx, frame, true);
|
||||
frame->state = FRAME_INVALID;
|
||||
ringbuffer_free(ctx->frames, frame);
|
||||
}
|
||||
|
||||
assert(!ctx->active_frame);
|
||||
ctx->enabled = false;
|
||||
}
|
||||
ringbuffer_unlock(ctx->frames);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the amount of time that we want the next frame to be delayed.
|
||||
*
|
||||
* The algorithm used by this function is very simplistic and only aims
|
||||
* to minimize the delay between calls to vkQueueSubmit or vkQueueSubmit2
|
||||
* and the begin of the execution of the submission.
|
||||
*/
|
||||
static uint64_t
|
||||
get_wait_time(device_context *ctx)
|
||||
{
|
||||
/* If we have at least 3 frames, then 2 frames must already have been
|
||||
* submitted to the GPU. It is safe to wait for one to complete.
|
||||
*/
|
||||
frame *next_frame = ringbuffer_first(ctx->frames);
|
||||
bool force_wait = ctx->frames.size >= 3;
|
||||
|
||||
while (evaluate_frame(ctx, next_frame, force_wait)) {
|
||||
|
||||
if (next_frame->frame_time) {
|
||||
/* Calculate average absolute deviation as exponential moving average with alpha = 0.125. */
|
||||
int64_t delta = ctx->queuing_delay + ctx->avg_frame_time - next_frame->min_delay -
|
||||
next_frame->frame_time;
|
||||
int64_t diff = (delta < 0 ? -delta : delta) - ctx->delta;
|
||||
ctx->delta += diff / 8;
|
||||
|
||||
/* Update frame time: Use an exponential moving average with alpha = 0.5. */
|
||||
ctx->avg_frame_time = (ctx->avg_frame_time + next_frame->frame_time) / 2;
|
||||
}
|
||||
|
||||
/* Update queuing delay: Just delay the next frame by half of it. */
|
||||
ctx->queuing_delay = next_frame->min_delay / 2;
|
||||
|
||||
ringbuffer_lock(ctx->frames);
|
||||
{
|
||||
next_frame->state = FRAME_INVALID;
|
||||
ringbuffer_free(ctx->frames, next_frame);
|
||||
next_frame = ringbuffer_first(ctx->frames);
|
||||
}
|
||||
ringbuffer_unlock(ctx->frames);
|
||||
|
||||
/* If we successfully evaluated one frame, only attempt to evaluate the next one,
|
||||
* if the completion is clearly expected before the next deadline.
|
||||
* Also, don't use force-wait in this case.
|
||||
*/
|
||||
force_wait = false;
|
||||
if (ctx->frames.size <= 2)
|
||||
break;
|
||||
}
|
||||
|
||||
/* Start the next input sampling after the period of one frame.
|
||||
* We add half the queuing delay as that's what we try to minimize.
|
||||
* Subtract the average frame time deviation as slack time, so that
|
||||
* we don't drain the GPU from work.
|
||||
*/
|
||||
int64_t delay = ctx->avg_frame_time + ctx->queuing_delay - MIN2(ctx->delta, ONE_MS);
|
||||
|
||||
if (delay > 100 * ONE_MS) {
|
||||
/* This corresponds to <10 FPS. Something might have gone wrong. */
|
||||
calibrate_timestamps(ctx);
|
||||
delay = 0;
|
||||
}
|
||||
|
||||
return MAX2(0, delay);
|
||||
}
|
||||
|
||||
static void
|
||||
reset_frame(frame *frame)
|
||||
{
|
||||
assert(frame->state == FRAME_INVALID);
|
||||
frame->frame_time = 0;
|
||||
frame->min_delay = 0;
|
||||
frame->state = FRAME_INPUT;
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
anti_lag_AntiLagUpdateAMD(VkDevice device, const VkAntiLagDataAMD *pData)
|
||||
{
|
||||
if (pData == NULL)
|
||||
return;
|
||||
|
||||
device_context *ctx = get_device_context(device);
|
||||
if (pData->mode == VK_ANTI_LAG_MODE_OFF_AMD) {
|
||||
/* Application request to disable Anti-Lag. */
|
||||
simple_mtx_lock(&ctx->mtx);
|
||||
anti_lag_disable(ctx);
|
||||
simple_mtx_unlock(&ctx->mtx);
|
||||
return;
|
||||
}
|
||||
|
||||
UNUSED uint64_t frame_idx = 0;
|
||||
p_atomic_set(&ctx->enabled, true);
|
||||
|
||||
if (pData->pPresentationInfo) {
|
||||
/* The same frameIndex value should be used with VK_ANTI_LAG_STAGE_INPUT_AMD before
|
||||
* the frame begins and with VK_ANTI_LAG_STAGE_PRESENT_AMD when the frame ends.
|
||||
*/
|
||||
frame_idx = pData->pPresentationInfo->frameIndex;
|
||||
|
||||
/* This marks the end of the input stage of the current frame. */
|
||||
// TODO: If available, this can be used to calculate the simulation time
|
||||
if (pData->pPresentationInfo->stage == VK_ANTI_LAG_STAGE_PRESENT_AMD)
|
||||
return;
|
||||
}
|
||||
|
||||
/* Lock this function, in order to avoid race conditions on frame evaluation. */
|
||||
simple_mtx_lock(&ctx->mtx);
|
||||
|
||||
/* VK_ANTI_LAG_STAGE_INPUT_AMD: This marks the begin of a new frame.
|
||||
* Evaluate previous frames in order to determine the wait time.
|
||||
*/
|
||||
uint64_t delay = get_wait_time(ctx);
|
||||
|
||||
/* Ensure maxFPS adherence. */
|
||||
if (pData->maxFPS) {
|
||||
uint64_t frametime_period = ONE_SECOND_IN_NS / pData->maxFPS;
|
||||
delay = MAX2(delay, frametime_period);
|
||||
}
|
||||
|
||||
uint64_t next_deadline = ctx->prev_input_begin + delay;
|
||||
uint64_t now = os_time_get_nano();
|
||||
|
||||
/* Recalibrate every now and then. */
|
||||
if (next_deadline > ctx->calibration.recalibrate_when)
|
||||
calibrate_timestamps(ctx);
|
||||
|
||||
/* Sleep until deadline is met. */
|
||||
os_time_nanosleep_until(next_deadline);
|
||||
|
||||
ctx->prev_input_begin = MAX2(now, next_deadline);
|
||||
|
||||
simple_mtx_unlock(&ctx->mtx);
|
||||
}
|
||||
|
||||
static queue_context *
|
||||
get_queue_context(device_context *ctx, VkQueue queue)
|
||||
{
|
||||
for (unsigned i = 0; i < ctx->num_queues; i++) {
|
||||
if (ctx->queues[i].queue == queue)
|
||||
return &ctx->queues[i];
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static struct query *
|
||||
allocate_query(queue_context *queue_ctx, uint32_t frame_idx)
|
||||
{
|
||||
/* Allow for a single frame to use at most half of the query pool. */
|
||||
if (queue_ctx->submissions_per_frame[frame_idx] > MAX_QUERIES / 2)
|
||||
return NULL;
|
||||
|
||||
/* Check that the next query index has been reset properly:
|
||||
*
|
||||
* We use some double-buffering here in order to reduce the number of
|
||||
* VkResetQueryPool commands.
|
||||
* Return false if the next query-index allocation crosses into the half
|
||||
* which still contains active queries,
|
||||
*/
|
||||
if (queue_ctx->queries.size > MAX_QUERIES / 2) {
|
||||
struct query *last_query = ringbuffer_last(queue_ctx->queries);
|
||||
uint32_t next_idx = ringbuffer_index(queue_ctx->queries, last_query) + 1;
|
||||
if (next_idx == MAX_QUERIES || next_idx == MAX_QUERIES / 2)
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return ringbuffer_alloc(queue_ctx->queries);
|
||||
}
|
||||
|
||||
static bool
|
||||
get_commandbuffer(device_context *ctx, queue_context *queue_ctx, VkCommandBuffer *timestamp_begin,
|
||||
VkCommandBuffer *timestamp_end, bool has_command_buffer,
|
||||
bool has_wait_before_cmdbuffer, bool *early_submit)
|
||||
{
|
||||
uint64_t now = os_time_get_nano();
|
||||
|
||||
/* Begin critical section. */
|
||||
ringbuffer_lock(ctx->frames);
|
||||
ringbuffer_lock(queue_ctx->queries);
|
||||
|
||||
/* Don't record timestamps for queues that are not deemed sensitive to latency. */
|
||||
bool need_query = ctx->active_frame && p_atomic_read(&queue_ctx->latency_sensitive);
|
||||
uint32_t frame_idx;
|
||||
struct query *query = NULL;
|
||||
|
||||
if (need_query) {
|
||||
assert(ctx->active_frame->state == FRAME_SUBMIT);
|
||||
frame_idx = ringbuffer_index(ctx->frames, ctx->active_frame);
|
||||
|
||||
/* For the very first submissions in a frame (until we observe real GPU work happening),
|
||||
* we would want to submit a timestamp before anything else, including waits.
|
||||
* This allows us to detect a sensitive queue going idle before we can submit work to it.
|
||||
* If the queue in question depends on semaphores from other unrelated queues,
|
||||
* we may not easily be able to detect that situation without adding a lot more complexity.
|
||||
*/
|
||||
*early_submit = has_wait_before_cmdbuffer && queue_ctx->submissions_per_frame[frame_idx] == 0;
|
||||
if (has_command_buffer || *early_submit)
|
||||
query = allocate_query(queue_ctx, frame_idx);
|
||||
}
|
||||
|
||||
if (query == NULL) {
|
||||
ringbuffer_unlock(queue_ctx->queries);
|
||||
ringbuffer_unlock(ctx->frames);
|
||||
return false;
|
||||
}
|
||||
|
||||
query->submit_ts = now;
|
||||
|
||||
/* Assign commandBuffer for timestamp. */
|
||||
*timestamp_begin = query->cmdbuffer;
|
||||
|
||||
/* Allocate a second query to mark the end of GPU work. */
|
||||
query = allocate_query(queue_ctx, frame_idx);
|
||||
assert(query);
|
||||
query->submit_ts = now;
|
||||
*timestamp_end = query->cmdbuffer;
|
||||
|
||||
/* Increment timeline semaphore count. */
|
||||
queue_ctx->semaphore_value++;
|
||||
|
||||
/* Add new submission entry for the current frame */
|
||||
queue_ctx->submissions_per_frame[frame_idx] += 2;
|
||||
|
||||
ringbuffer_unlock(queue_ctx->queries);
|
||||
ringbuffer_unlock(ctx->frames);
|
||||
return true;
|
||||
}
|
||||
|
||||
static VkResult
|
||||
queue_submit2(device_context *ctx, VkQueue queue, uint32_t submitCount,
|
||||
const VkSubmitInfo2 *pSubmits, VkFence fence, PFN_vkQueueSubmit2 queueSubmit2)
|
||||
{
|
||||
queue_context *queue_ctx = get_queue_context(ctx, queue);
|
||||
if (!ctx->active_frame || !queue_ctx || !submitCount)
|
||||
return queueSubmit2(queue, submitCount, pSubmits, fence);
|
||||
|
||||
bool has_wait_before_cmdbuffer = false;
|
||||
int first = -1;
|
||||
VkCommandBuffer timestamp_begin;
|
||||
VkCommandBuffer timestamp_end;
|
||||
|
||||
/* Check if any submission contains commandbuffers. */
|
||||
for (unsigned i = 0; i < submitCount; i++) {
|
||||
if (pSubmits[i].waitSemaphoreInfoCount != 0)
|
||||
has_wait_before_cmdbuffer = true;
|
||||
|
||||
if (pSubmits[i].commandBufferInfoCount) {
|
||||
first = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/* Get timestamp commandbuffer. */
|
||||
bool early_submit;
|
||||
if (!get_commandbuffer(ctx, queue_ctx, ×tamp_begin, ×tamp_end, first >= 0,
|
||||
has_wait_before_cmdbuffer, &early_submit)) {
|
||||
return queueSubmit2(queue, submitCount, pSubmits, fence);
|
||||
}
|
||||
|
||||
VkSubmitInfo2 *submits;
|
||||
VkCommandBufferSubmitInfo *cmdbuffers_begin;
|
||||
VkCommandBufferSubmitInfo *cmdbuffers_end;
|
||||
VkSemaphoreSubmitInfo *semaphores;
|
||||
VK_MULTIALLOC(ma);
|
||||
|
||||
if (early_submit) {
|
||||
vk_multialloc_add(&ma, &submits, VkSubmitInfo2, submitCount + 1);
|
||||
vk_multialloc_add(&ma, &cmdbuffers_begin, VkCommandBufferSubmitInfo, 1);
|
||||
vk_multialloc_add(&ma, &cmdbuffers_end, VkCommandBufferSubmitInfo,
|
||||
pSubmits[submitCount - 1].commandBufferInfoCount + 1);
|
||||
first = 0;
|
||||
} else {
|
||||
vk_multialloc_add(&ma, &submits, VkSubmitInfo2, submitCount);
|
||||
if (first == submitCount - 1) {
|
||||
vk_multialloc_add(&ma, &cmdbuffers_begin, VkCommandBufferSubmitInfo,
|
||||
pSubmits[first].commandBufferInfoCount + 2);
|
||||
} else {
|
||||
vk_multialloc_add(&ma, &cmdbuffers_begin, VkCommandBufferSubmitInfo,
|
||||
pSubmits[first].commandBufferInfoCount + 1);
|
||||
vk_multialloc_add(&ma, &cmdbuffers_end, VkCommandBufferSubmitInfo,
|
||||
pSubmits[submitCount - 1].commandBufferInfoCount + 1);
|
||||
}
|
||||
}
|
||||
vk_multialloc_add(&ma, &semaphores, VkSemaphoreSubmitInfo,
|
||||
pSubmits[submitCount - 1].signalSemaphoreInfoCount + 1);
|
||||
|
||||
void *buf = vk_multialloc_zalloc(&ma, &ctx->alloc, VK_SYSTEM_ALLOCATION_SCOPE_COMMAND);
|
||||
if (!buf)
|
||||
return VK_ERROR_OUT_OF_HOST_MEMORY;
|
||||
|
||||
if (early_submit) {
|
||||
memcpy(submits + 1, pSubmits, sizeof(VkSubmitInfo2) * submitCount);
|
||||
submits[0] = (VkSubmitInfo2){.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2};
|
||||
submitCount++;
|
||||
} else {
|
||||
memcpy(submits, pSubmits, sizeof(VkSubmitInfo2) * submitCount);
|
||||
}
|
||||
|
||||
VkSubmitInfo2 *submit_info = &submits[first];
|
||||
|
||||
/* Add commandbuffer to submission. */
|
||||
cmdbuffers_begin[0] = (VkCommandBufferSubmitInfo){
|
||||
.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO,
|
||||
.commandBuffer = timestamp_begin,
|
||||
};
|
||||
memcpy(&cmdbuffers_begin[1], submit_info->pCommandBufferInfos,
|
||||
sizeof(VkCommandBufferSubmitInfo) * submit_info->commandBufferInfoCount);
|
||||
submit_info->pCommandBufferInfos = cmdbuffers_begin;
|
||||
submit_info->commandBufferInfoCount++;
|
||||
|
||||
/* Add commandbuffer to submission. */
|
||||
submit_info = &submits[submitCount - 1];
|
||||
if (first == submitCount - 1) {
|
||||
cmdbuffers_begin[submit_info->commandBufferInfoCount] = (VkCommandBufferSubmitInfo){
|
||||
.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO,
|
||||
.commandBuffer = timestamp_end,
|
||||
};
|
||||
} else {
|
||||
memcpy(&cmdbuffers_end[0], submit_info->pCommandBufferInfos,
|
||||
sizeof(VkCommandBufferSubmitInfo) * submit_info->commandBufferInfoCount);
|
||||
cmdbuffers_end[submit_info->commandBufferInfoCount] = (VkCommandBufferSubmitInfo){
|
||||
.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO,
|
||||
.commandBuffer = timestamp_end,
|
||||
};
|
||||
submit_info->pCommandBufferInfos = cmdbuffers_end;
|
||||
}
|
||||
submit_info->commandBufferInfoCount++;
|
||||
|
||||
/* Add timeline semaphore to last submission. */
|
||||
memcpy(semaphores, submit_info->pSignalSemaphoreInfos,
|
||||
sizeof(VkSemaphoreSubmitInfo) * submit_info->signalSemaphoreInfoCount);
|
||||
semaphores[submit_info->signalSemaphoreInfoCount] = (VkSemaphoreSubmitInfo){
|
||||
.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO,
|
||||
.semaphore = queue_ctx->semaphore,
|
||||
.value = queue_ctx->semaphore_value,
|
||||
.stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
|
||||
};
|
||||
submit_info->pSignalSemaphoreInfos = semaphores;
|
||||
submit_info->signalSemaphoreInfoCount++;
|
||||
|
||||
/* Submit with added timestamp query commandbuffer. */
|
||||
VkResult res = queueSubmit2(queue, submitCount, submits, fence);
|
||||
vk_free(&ctx->alloc, submits);
|
||||
return res;
|
||||
}
|
||||
|
||||
VKAPI_ATTR VkResult VKAPI_CALL
|
||||
anti_lag_QueueSubmit2KHR(VkQueue queue, uint32_t submitCount, const VkSubmitInfo2 *pSubmits,
|
||||
VkFence fence)
|
||||
{
|
||||
device_context *ctx = get_device_context(queue);
|
||||
return queue_submit2(ctx, queue, submitCount, pSubmits, fence, ctx->vtable.QueueSubmit2KHR);
|
||||
}
|
||||
|
||||
VKAPI_ATTR VkResult VKAPI_CALL
|
||||
anti_lag_QueueSubmit2(VkQueue queue, uint32_t submitCount, const VkSubmitInfo2 *pSubmits,
|
||||
VkFence fence)
|
||||
{
|
||||
device_context *ctx = get_device_context(queue);
|
||||
return queue_submit2(ctx, queue, submitCount, pSubmits, fence, ctx->vtable.QueueSubmit2);
|
||||
}
|
||||
|
||||
VKAPI_ATTR VkResult VKAPI_CALL
|
||||
anti_lag_QueueSubmit(VkQueue queue, uint32_t submitCount, const VkSubmitInfo *pSubmits,
|
||||
VkFence fence)
|
||||
{
|
||||
device_context *ctx = get_device_context(queue);
|
||||
queue_context *queue_ctx = get_queue_context(ctx, queue);
|
||||
if (!ctx->active_frame || !queue_ctx || !submitCount)
|
||||
return ctx->vtable.QueueSubmit(queue, submitCount, pSubmits, fence);
|
||||
|
||||
bool has_wait_before_cmdbuffer = false;
|
||||
int first = -1;
|
||||
VkCommandBuffer timestamp_begin;
|
||||
VkCommandBuffer timestamp_end;
|
||||
|
||||
/* Check if any submission contains commandbuffers or waits before those. */
|
||||
for (unsigned i = 0; i < submitCount; i++) {
|
||||
if (pSubmits[i].waitSemaphoreCount != 0)
|
||||
has_wait_before_cmdbuffer = true;
|
||||
|
||||
if (pSubmits[i].commandBufferCount) {
|
||||
first = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/* Get timestamp commandbuffer. */
|
||||
bool early_submit;
|
||||
if (!get_commandbuffer(ctx, queue_ctx, ×tamp_begin, ×tamp_end, first >= 0,
|
||||
has_wait_before_cmdbuffer, &early_submit)) {
|
||||
return ctx->vtable.QueueSubmit(queue, submitCount, pSubmits, fence);
|
||||
}
|
||||
|
||||
VkSubmitInfo *submits;
|
||||
VkCommandBuffer *cmdbuffers_begin;
|
||||
VkCommandBuffer *cmdbuffers_end;
|
||||
VkSemaphore *semaphores;
|
||||
VkTimelineSemaphoreSubmitInfo *semaphore_info;
|
||||
uint64_t *semaphore_values;
|
||||
VK_MULTIALLOC(ma);
|
||||
|
||||
if (early_submit) {
|
||||
vk_multialloc_add(&ma, &submits, VkSubmitInfo, submitCount + 1);
|
||||
vk_multialloc_add(&ma, &cmdbuffers_begin, VkCommandBuffer, 1);
|
||||
vk_multialloc_add(&ma, &cmdbuffers_end, VkCommandBuffer,
|
||||
pSubmits[submitCount - 1].commandBufferCount + 1);
|
||||
first = 0;
|
||||
} else {
|
||||
vk_multialloc_add(&ma, &submits, VkSubmitInfo, submitCount);
|
||||
if (first == submitCount - 1) {
|
||||
vk_multialloc_add(&ma, &cmdbuffers_begin, VkCommandBuffer,
|
||||
pSubmits[first].commandBufferCount + 2);
|
||||
} else {
|
||||
vk_multialloc_add(&ma, &cmdbuffers_begin, VkCommandBuffer,
|
||||
pSubmits[first].commandBufferCount + 1);
|
||||
vk_multialloc_add(&ma, &cmdbuffers_end, VkCommandBuffer,
|
||||
pSubmits[submitCount - 1].commandBufferCount + 1);
|
||||
}
|
||||
}
|
||||
vk_multialloc_add(&ma, &semaphores, VkSemaphore,
|
||||
pSubmits[submitCount - 1].signalSemaphoreCount + 1);
|
||||
vk_multialloc_add(&ma, &semaphore_info, VkTimelineSemaphoreSubmitInfo, 1);
|
||||
vk_multialloc_add(&ma, &semaphore_values, uint64_t,
|
||||
pSubmits[submitCount - 1].signalSemaphoreCount + 1);
|
||||
|
||||
void *buf = vk_multialloc_zalloc(&ma, &ctx->alloc, VK_SYSTEM_ALLOCATION_SCOPE_COMMAND);
|
||||
if (!buf)
|
||||
return VK_ERROR_OUT_OF_HOST_MEMORY;
|
||||
|
||||
if (early_submit) {
|
||||
memcpy(submits + 1, pSubmits, sizeof(VkSubmitInfo) * submitCount);
|
||||
submits[0] = (VkSubmitInfo){.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO};
|
||||
submitCount++;
|
||||
} else {
|
||||
memcpy(submits, pSubmits, sizeof(VkSubmitInfo) * submitCount);
|
||||
}
|
||||
|
||||
VkSubmitInfo *submit_info = &submits[first];
|
||||
|
||||
/* Add commandbuffer to submission. */
|
||||
cmdbuffers_begin[0] = timestamp_begin;
|
||||
memcpy(&cmdbuffers_begin[1], submit_info->pCommandBuffers,
|
||||
sizeof(VkCommandBuffer) * submit_info->commandBufferCount);
|
||||
submit_info->pCommandBuffers = cmdbuffers_begin;
|
||||
submit_info->commandBufferCount++;
|
||||
|
||||
submit_info = &submits[submitCount - 1];
|
||||
|
||||
/* Add commandbuffer to submission. */
|
||||
if (first == submitCount - 1) {
|
||||
cmdbuffers_begin[submit_info->commandBufferCount] = timestamp_end;
|
||||
} else {
|
||||
memcpy(&cmdbuffers_end[0], submit_info->pCommandBuffers,
|
||||
sizeof(VkCommandBuffer) * submit_info->commandBufferCount);
|
||||
cmdbuffers_end[submit_info->commandBufferCount] = timestamp_end;
|
||||
submit_info->pCommandBuffers = cmdbuffers_end;
|
||||
}
|
||||
submit_info->commandBufferCount++;
|
||||
|
||||
/* Add timeline semaphore to submission. */
|
||||
const VkTimelineSemaphoreSubmitInfo *tlssi =
|
||||
vk_find_struct_const(submit_info->pNext, TIMELINE_SEMAPHORE_SUBMIT_INFO);
|
||||
memcpy(&semaphores[0], submit_info->pSignalSemaphores,
|
||||
sizeof(VkSemaphore) * submit_info->signalSemaphoreCount);
|
||||
semaphores[submit_info->signalSemaphoreCount] = queue_ctx->semaphore;
|
||||
semaphore_values[submit_info->signalSemaphoreCount] = queue_ctx->semaphore_value;
|
||||
submit_info->pSignalSemaphores = semaphores;
|
||||
submit_info->signalSemaphoreCount++;
|
||||
if (tlssi) {
|
||||
*semaphore_info = *tlssi; /* save original values */
|
||||
memcpy(&semaphore_values[1], tlssi->pSignalSemaphoreValues,
|
||||
sizeof(uint64_t) * tlssi->signalSemaphoreValueCount);
|
||||
((VkTimelineSemaphoreSubmitInfo *)tlssi)->pSignalSemaphoreValues = semaphore_values;
|
||||
((VkTimelineSemaphoreSubmitInfo *)tlssi)->signalSemaphoreValueCount =
|
||||
submit_info->signalSemaphoreCount;
|
||||
} else {
|
||||
*semaphore_info = (VkTimelineSemaphoreSubmitInfo){
|
||||
.sType = VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO,
|
||||
.pNext = submit_info->pNext,
|
||||
.signalSemaphoreValueCount = submit_info->signalSemaphoreCount,
|
||||
.pSignalSemaphoreValues = semaphore_values,
|
||||
};
|
||||
submit_info->pNext = semaphore_info;
|
||||
}
|
||||
|
||||
/* Submit with added timestamp query commandbuffer. */
|
||||
VkResult res = ctx->vtable.QueueSubmit(queue, submitCount, submits, fence);
|
||||
if (tlssi)
|
||||
*(VkTimelineSemaphoreSubmitInfo *)tlssi = *semaphore_info; /* restore */
|
||||
vk_free(&ctx->alloc, buf);
|
||||
return res;
|
||||
}
|
||||
|
||||
VKAPI_ATTR VkResult VKAPI_CALL
|
||||
anti_lag_QueuePresentKHR(VkQueue queue, const VkPresentInfoKHR *pPresentInfo)
|
||||
{
|
||||
|
||||
device_context *ctx = get_device_context(queue);
|
||||
|
||||
if (ctx->enabled) {
|
||||
/* When multiple queues are in flight, the min-delay approach
|
||||
* has problems. An async compute queue could be submitted to
|
||||
* with very low delay while the main graphics queue would be swamped with work.
|
||||
* If we take a global min-delay over all queues, the algorithm would
|
||||
* assume that there is very low delay and thus sleeps are disabled, but
|
||||
* unless the graphics work depends directly on the async compute work,
|
||||
* this is a false assumption.
|
||||
*/
|
||||
queue_context *queue_ctx = get_queue_context(ctx, queue);
|
||||
p_atomic_set(&queue_ctx->latency_sensitive, true);
|
||||
|
||||
/* Initialize new frame. */
|
||||
ringbuffer_lock(ctx->frames);
|
||||
{
|
||||
if (ctx->enabled) {
|
||||
frame *new_frame = ringbuffer_alloc(ctx->frames);
|
||||
reset_frame(new_frame);
|
||||
begin_next_frame(ctx);
|
||||
}
|
||||
}
|
||||
ringbuffer_unlock(ctx->frames);
|
||||
}
|
||||
|
||||
return ctx->vtable.QueuePresentKHR(queue, pPresentInfo);
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
/*
|
||||
* Copyright © 2025 Valve Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*/
|
||||
|
||||
#ifndef ANTI_LAG_LAYER_H
|
||||
#define ANTI_LAG_LAYER_H
|
||||
|
||||
#include "util/simple_mtx.h"
|
||||
#include "vulkan/vk_layer.h"
|
||||
#include "vulkan/vulkan_core.h"
|
||||
#include "ringbuffer.h"
|
||||
|
||||
#define MAX_FRAMES 8
|
||||
#define MAX_QUERIES 256
|
||||
|
||||
enum frame_state {
|
||||
FRAME_INVALID = 0,
|
||||
FRAME_INPUT, /* Frame is in input stage. */
|
||||
FRAME_SUBMIT, /* All current queueSubmit calls are associated with this frame. */
|
||||
FRAME_PRESENT, /* Frame is in present stage and latencies can be evaluated. */
|
||||
};
|
||||
|
||||
typedef struct frame {
|
||||
uint64_t frame_time;
|
||||
int64_t min_delay;
|
||||
|
||||
enum frame_state state;
|
||||
} frame;
|
||||
|
||||
struct query {
|
||||
uint64_t gpu_ts;
|
||||
uint64_t submit_ts;
|
||||
VkCommandBuffer cmdbuffer;
|
||||
};
|
||||
|
||||
typedef struct queue_context {
|
||||
VkQueue queue;
|
||||
uint32_t queue_family_idx;
|
||||
bool latency_sensitive;
|
||||
VkCommandPool cmdPool;
|
||||
VkQueryPool queryPool;
|
||||
VkSemaphore semaphore;
|
||||
uint64_t semaphore_value;
|
||||
uint8_t submissions_per_frame[MAX_FRAMES];
|
||||
RINGBUFFER_DECLARE(queries, struct query, MAX_QUERIES);
|
||||
} queue_context;
|
||||
|
||||
typedef struct device_context {
|
||||
|
||||
struct DeviceDispatchTable {
|
||||
#define DECLARE_HOOK(fn) PFN_vk##fn fn
|
||||
DECLARE_HOOK(GetDeviceProcAddr);
|
||||
DECLARE_HOOK(SetDeviceLoaderData);
|
||||
DECLARE_HOOK(DestroyDevice);
|
||||
DECLARE_HOOK(QueueSubmit);
|
||||
DECLARE_HOOK(QueueSubmit2);
|
||||
DECLARE_HOOK(QueueSubmit2KHR);
|
||||
DECLARE_HOOK(GetDeviceQueue);
|
||||
DECLARE_HOOK(CreateCommandPool);
|
||||
DECLARE_HOOK(DestroyCommandPool);
|
||||
DECLARE_HOOK(CreateQueryPool);
|
||||
DECLARE_HOOK(ResetQueryPool);
|
||||
DECLARE_HOOK(DestroyQueryPool);
|
||||
DECLARE_HOOK(GetQueryPoolResults);
|
||||
DECLARE_HOOK(AllocateCommandBuffers);
|
||||
DECLARE_HOOK(FreeCommandBuffers);
|
||||
DECLARE_HOOK(BeginCommandBuffer);
|
||||
DECLARE_HOOK(EndCommandBuffer);
|
||||
DECLARE_HOOK(GetCalibratedTimestampsKHR);
|
||||
DECLARE_HOOK(CmdWriteTimestamp);
|
||||
DECLARE_HOOK(CreateSemaphore);
|
||||
DECLARE_HOOK(DestroySemaphore);
|
||||
DECLARE_HOOK(GetSemaphoreCounterValue);
|
||||
DECLARE_HOOK(WaitSemaphores);
|
||||
DECLARE_HOOK(QueuePresentKHR);
|
||||
#undef DECLARE_HOOK
|
||||
} vtable;
|
||||
|
||||
VkDevice device;
|
||||
VkAllocationCallbacks alloc;
|
||||
simple_mtx_t mtx;
|
||||
bool enabled;
|
||||
|
||||
struct {
|
||||
int64_t delta;
|
||||
uint64_t recalibrate_when;
|
||||
float timestamp_period;
|
||||
} calibration;
|
||||
|
||||
RINGBUFFER_DECLARE(frames, frame, MAX_FRAMES);
|
||||
frame *active_frame;
|
||||
|
||||
uint64_t avg_frame_time;
|
||||
uint64_t delta;
|
||||
uint64_t prev_input_begin;
|
||||
int64_t queuing_delay;
|
||||
|
||||
unsigned num_queues;
|
||||
queue_context queues[];
|
||||
} device_context;
|
||||
|
||||
device_context *get_device_context(const void *object);
|
||||
|
||||
void anti_lag_AntiLagUpdateAMD(VkDevice device, const VkAntiLagDataAMD *pData);
|
||||
VkResult anti_lag_QueueSubmit2KHR(VkQueue queue, uint32_t submitCount,
|
||||
const VkSubmitInfo2 *pSubmits, VkFence fence);
|
||||
VkResult anti_lag_QueueSubmit2(VkQueue queue, uint32_t submitCount, const VkSubmitInfo2 *pSubmits,
|
||||
VkFence fence);
|
||||
VkResult anti_lag_QueueSubmit(VkQueue queue, uint32_t submitCount, const VkSubmitInfo *pSubmits,
|
||||
VkFence fence);
|
||||
VkResult anti_lag_QueuePresentKHR(VkQueue queue, const VkPresentInfoKHR *pPresentInfo);
|
||||
|
||||
VkResult anti_lag_NegotiateLoaderLayerInterfaceVersion(VkNegotiateLayerInterface *pVersionStruct);
|
||||
|
||||
#endif /* ANTI_LAG_LAYER_H */
|
||||
@@ -0,0 +1,909 @@
|
||||
/*
|
||||
* Copyright © 2025 Valve Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*/
|
||||
|
||||
#include "util/simple_mtx.h"
|
||||
#include "vulkan/vk_layer.h"
|
||||
#include "vulkan/vulkan_core.h"
|
||||
#include "anti_lag_layer.h"
|
||||
#include "vk_alloc.h"
|
||||
#include "vk_util.h"
|
||||
|
||||
static uintptr_t
|
||||
object_to_key(const void *object)
|
||||
{
|
||||
return (uintptr_t)*(uintptr_t *)object;
|
||||
}
|
||||
|
||||
typedef struct instance_data {
|
||||
struct InstanceDispatchTable {
|
||||
#define DECLARE_HOOK(fn) PFN_vk##fn fn
|
||||
DECLARE_HOOK(GetInstanceProcAddr);
|
||||
DECLARE_HOOK(CreateInstance);
|
||||
DECLARE_HOOK(DestroyInstance);
|
||||
DECLARE_HOOK(CreateDevice);
|
||||
DECLARE_HOOK(EnumerateDeviceExtensionProperties);
|
||||
DECLARE_HOOK(GetPhysicalDeviceFeatures2KHR);
|
||||
DECLARE_HOOK(GetPhysicalDeviceFeatures2);
|
||||
DECLARE_HOOK(GetPhysicalDeviceProperties);
|
||||
DECLARE_HOOK(GetPhysicalDeviceCalibrateableTimeDomainsEXT);
|
||||
DECLARE_HOOK(GetPhysicalDeviceCalibrateableTimeDomainsKHR);
|
||||
DECLARE_HOOK(GetPhysicalDeviceQueueFamilyProperties);
|
||||
#undef DECLARE_HOOK
|
||||
} vtable;
|
||||
|
||||
VkInstance instance;
|
||||
uint32_t apiVersion;
|
||||
VkAllocationCallbacks alloc;
|
||||
struct instance_data *next;
|
||||
} instance_data;
|
||||
|
||||
static void
|
||||
init_instance_vtable(instance_data *ctx, PFN_vkGetInstanceProcAddr gpa)
|
||||
{
|
||||
ctx->vtable.GetInstanceProcAddr = gpa;
|
||||
#define INIT_HOOK(fn) ctx->vtable.fn = (PFN_vk##fn)gpa(ctx->instance, "vk" #fn)
|
||||
INIT_HOOK(CreateInstance);
|
||||
INIT_HOOK(DestroyInstance);
|
||||
INIT_HOOK(CreateDevice);
|
||||
INIT_HOOK(EnumerateDeviceExtensionProperties);
|
||||
INIT_HOOK(GetPhysicalDeviceFeatures2KHR);
|
||||
INIT_HOOK(GetPhysicalDeviceFeatures2);
|
||||
INIT_HOOK(GetPhysicalDeviceProperties);
|
||||
INIT_HOOK(GetPhysicalDeviceCalibrateableTimeDomainsEXT);
|
||||
INIT_HOOK(GetPhysicalDeviceCalibrateableTimeDomainsKHR);
|
||||
INIT_HOOK(GetPhysicalDeviceQueueFamilyProperties);
|
||||
#undef INIT_HOOK
|
||||
}
|
||||
|
||||
static simple_mtx_t instance_mtx = SIMPLE_MTX_INITIALIZER;
|
||||
static instance_data *instance_list = NULL;
|
||||
|
||||
static void
|
||||
add_instance(instance_data *instance)
|
||||
{
|
||||
simple_mtx_lock(&instance_mtx);
|
||||
instance_data **ptr = &instance_list;
|
||||
while (*ptr != NULL)
|
||||
ptr = &(*ptr)->next;
|
||||
*ptr = instance;
|
||||
simple_mtx_unlock(&instance_mtx);
|
||||
}
|
||||
|
||||
static instance_data *
|
||||
remove_instance(const void *object)
|
||||
{
|
||||
uintptr_t key = object_to_key(object);
|
||||
simple_mtx_lock(&instance_mtx);
|
||||
instance_data **ptr = &instance_list;
|
||||
while (*ptr && key != object_to_key((*ptr)->instance))
|
||||
ptr = &(*ptr)->next;
|
||||
|
||||
instance_data *ctx = *ptr;
|
||||
*ptr = ctx ? ctx->next : NULL;
|
||||
simple_mtx_unlock(&instance_mtx);
|
||||
return ctx;
|
||||
}
|
||||
|
||||
static instance_data *
|
||||
get_instance_data(const void *object)
|
||||
{
|
||||
uintptr_t key = object_to_key(object);
|
||||
simple_mtx_lock(&instance_mtx);
|
||||
instance_data *ctx = instance_list;
|
||||
while (ctx && key != object_to_key(ctx->instance))
|
||||
ctx = ctx->next;
|
||||
simple_mtx_unlock(&instance_mtx);
|
||||
return ctx;
|
||||
}
|
||||
|
||||
static VKAPI_ATTR VkResult VKAPI_CALL
|
||||
anti_lag_CreateInstance(const VkInstanceCreateInfo *pCreateInfo,
|
||||
const VkAllocationCallbacks *pAllocator, VkInstance *pInstance)
|
||||
{
|
||||
VkLayerInstanceCreateInfo *chain_info = (VkLayerInstanceCreateInfo *)(pCreateInfo->pNext);
|
||||
while (chain_info && !(chain_info->sType == VK_STRUCTURE_TYPE_LOADER_INSTANCE_CREATE_INFO &&
|
||||
chain_info->function == VK_LAYER_LINK_INFO)) {
|
||||
chain_info = (VkLayerInstanceCreateInfo *)(chain_info->pNext);
|
||||
}
|
||||
|
||||
assert(chain_info && chain_info->u.pLayerInfo);
|
||||
PFN_vkGetInstanceProcAddr fpGetInstanceProcAddr =
|
||||
chain_info->u.pLayerInfo->pfnNextGetInstanceProcAddr;
|
||||
PFN_vkCreateInstance fpCreateInstance =
|
||||
(PFN_vkCreateInstance)fpGetInstanceProcAddr(NULL, "vkCreateInstance");
|
||||
if (fpCreateInstance == NULL)
|
||||
return VK_ERROR_INITIALIZATION_FAILED;
|
||||
|
||||
/* Advance the link info for the next element on the chain. */
|
||||
chain_info->u.pLayerInfo = chain_info->u.pLayerInfo->pNext;
|
||||
|
||||
/* Create Instance. */
|
||||
VkResult result = fpCreateInstance(pCreateInfo, pAllocator, pInstance);
|
||||
if (result != VK_SUCCESS)
|
||||
return result;
|
||||
|
||||
/* Create Instance context. */
|
||||
const VkAllocationCallbacks *alloc = pAllocator ? pAllocator : vk_default_allocator();
|
||||
void *buf = vk_alloc(alloc, sizeof(instance_data), alignof(instance_data),
|
||||
VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE);
|
||||
if (!buf) {
|
||||
PFN_vkDestroyInstance fpDestroyInstance =
|
||||
(PFN_vkDestroyInstance)fpGetInstanceProcAddr(*pInstance, "vkDestroyInstance");
|
||||
fpDestroyInstance(*pInstance, alloc);
|
||||
return VK_ERROR_OUT_OF_HOST_MEMORY;
|
||||
}
|
||||
instance_data *ctx = (instance_data *)buf;
|
||||
ctx->apiVersion = pCreateInfo->pApplicationInfo && pCreateInfo->pApplicationInfo->apiVersion
|
||||
? pCreateInfo->pApplicationInfo->apiVersion
|
||||
: VK_API_VERSION_1_0;
|
||||
ctx->instance = *pInstance;
|
||||
ctx->alloc = *alloc;
|
||||
ctx->next = NULL;
|
||||
init_instance_vtable(ctx, fpGetInstanceProcAddr);
|
||||
add_instance(ctx);
|
||||
|
||||
return VK_SUCCESS;
|
||||
}
|
||||
|
||||
static VKAPI_ATTR void VKAPI_CALL
|
||||
anti_lag_DestroyInstance(VkInstance instance, const VkAllocationCallbacks *pAllocator)
|
||||
{
|
||||
instance_data *ctx = remove_instance(instance);
|
||||
if (ctx) {
|
||||
ctx->vtable.DestroyInstance(instance, pAllocator);
|
||||
vk_free(&ctx->alloc, ctx);
|
||||
}
|
||||
}
|
||||
|
||||
typedef struct device_data {
|
||||
VkDevice device;
|
||||
PFN_vkGetDeviceProcAddr GetDeviceProcAddr;
|
||||
device_context *ctx; /* NULL if anti-lag ext is not enabled. */
|
||||
struct device_data *next;
|
||||
} device_data;
|
||||
|
||||
static void
|
||||
init_device_vtable(device_context *ctx, PFN_vkGetDeviceProcAddr gpa, PFN_vkSetDeviceLoaderData sld,
|
||||
bool calibrated_timestamps_khr, bool host_query_reset_ext,
|
||||
bool timeline_semaphore_khr)
|
||||
{
|
||||
ctx->vtable.GetDeviceProcAddr = gpa;
|
||||
ctx->vtable.SetDeviceLoaderData = sld;
|
||||
#define INIT_HOOK(fn) ctx->vtable.fn = (PFN_vk##fn)gpa(ctx->device, "vk" #fn)
|
||||
#define INIT_HOOK_ALIAS(fn, alias, cond) \
|
||||
ctx->vtable.fn = (PFN_vk##fn)gpa(ctx->device, cond ? "vk" #alias : "vk" #fn)
|
||||
INIT_HOOK(DestroyDevice);
|
||||
INIT_HOOK(QueueSubmit);
|
||||
INIT_HOOK(QueueSubmit2);
|
||||
INIT_HOOK(QueueSubmit2KHR);
|
||||
INIT_HOOK(GetDeviceQueue);
|
||||
INIT_HOOK(CreateCommandPool);
|
||||
INIT_HOOK(DestroyCommandPool);
|
||||
INIT_HOOK(CreateQueryPool);
|
||||
INIT_HOOK_ALIAS(ResetQueryPool, ResetQueryPoolEXT, host_query_reset_ext);
|
||||
INIT_HOOK(DestroyQueryPool);
|
||||
INIT_HOOK(GetQueryPoolResults);
|
||||
INIT_HOOK(AllocateCommandBuffers);
|
||||
INIT_HOOK(FreeCommandBuffers);
|
||||
INIT_HOOK(BeginCommandBuffer);
|
||||
INIT_HOOK(EndCommandBuffer);
|
||||
INIT_HOOK_ALIAS(GetCalibratedTimestampsKHR, GetCalibratedTimestampsEXT, !calibrated_timestamps_khr);
|
||||
INIT_HOOK(CmdWriteTimestamp);
|
||||
INIT_HOOK(CreateSemaphore);
|
||||
INIT_HOOK(DestroySemaphore);
|
||||
INIT_HOOK(QueuePresentKHR);
|
||||
INIT_HOOK_ALIAS(GetSemaphoreCounterValue, GetSemaphoreCounterValueKHR, timeline_semaphore_khr);
|
||||
INIT_HOOK_ALIAS(WaitSemaphores, WaitSemaphoresKHR, timeline_semaphore_khr);
|
||||
#undef INIT_HOOK
|
||||
#undef INIT_HOOK_ALIAS
|
||||
}
|
||||
|
||||
static simple_mtx_t device_mtx = SIMPLE_MTX_INITIALIZER;
|
||||
static device_data *device_list = NULL;
|
||||
|
||||
static void
|
||||
add_device(device_data *device)
|
||||
{
|
||||
simple_mtx_lock(&device_mtx);
|
||||
device_data **ptr = &device_list;
|
||||
while (*ptr != NULL)
|
||||
ptr = &(*ptr)->next;
|
||||
*ptr = device;
|
||||
simple_mtx_unlock(&device_mtx);
|
||||
}
|
||||
|
||||
static device_data *
|
||||
remove_device(const void *object)
|
||||
{
|
||||
uintptr_t key = object_to_key(object);
|
||||
simple_mtx_lock(&device_mtx);
|
||||
device_data **ptr = &device_list;
|
||||
while (*ptr && key != object_to_key((*ptr)->device))
|
||||
ptr = &(*ptr)->next;
|
||||
|
||||
device_data *ctx = *ptr;
|
||||
*ptr = ctx ? ctx->next : NULL;
|
||||
simple_mtx_unlock(&device_mtx);
|
||||
return ctx;
|
||||
}
|
||||
|
||||
static device_data *
|
||||
get_device_data(const void *object)
|
||||
{
|
||||
uintptr_t key = object_to_key(object);
|
||||
simple_mtx_lock(&device_mtx);
|
||||
device_data *ctx = device_list;
|
||||
while (ctx && key != object_to_key(ctx->device))
|
||||
ctx = ctx->next;
|
||||
simple_mtx_unlock(&device_mtx);
|
||||
return ctx;
|
||||
}
|
||||
|
||||
device_context *
|
||||
get_device_context(const void *object)
|
||||
{
|
||||
device_data *data = get_device_data(object);
|
||||
assert(data && data->ctx);
|
||||
return data->ctx;
|
||||
}
|
||||
|
||||
static VkLayerDeviceCreateInfo *
|
||||
get_device_chain_info(const VkDeviceCreateInfo *pCreateInfo, VkLayerFunction func)
|
||||
{
|
||||
vk_foreach_struct_const (item, pCreateInfo->pNext) {
|
||||
if (item->sType == VK_STRUCTURE_TYPE_LOADER_DEVICE_CREATE_INFO &&
|
||||
((VkLayerDeviceCreateInfo *)item)->function == func)
|
||||
return (VkLayerDeviceCreateInfo *)item;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static bool
|
||||
should_enable_layer(instance_data *ctx, VkPhysicalDevice physicalDevice,
|
||||
VkPhysicalDeviceAntiLagFeaturesAMD ext_feature)
|
||||
{
|
||||
/* The extension is not requested by the application. */
|
||||
if (!ext_feature.antiLag)
|
||||
return false;
|
||||
|
||||
/* Ensure that the underlying implementation does not expose VK_AMD_anti_lag itself. */
|
||||
ext_feature.antiLag = false;
|
||||
|
||||
/* Don't clobber unrelated pNext chains passed in at device creation time. */
|
||||
ext_feature.pNext = NULL;
|
||||
|
||||
VkPhysicalDeviceFeatures2 features = {
|
||||
.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2,
|
||||
.pNext = &ext_feature,
|
||||
};
|
||||
|
||||
if (ctx->vtable.GetPhysicalDeviceFeatures2KHR) {
|
||||
ctx->vtable.GetPhysicalDeviceFeatures2KHR(physicalDevice, &features);
|
||||
return !ext_feature.antiLag;
|
||||
}
|
||||
|
||||
if (ctx->vtable.GetPhysicalDeviceFeatures2) {
|
||||
ctx->vtable.GetPhysicalDeviceFeatures2(physicalDevice, &features);
|
||||
return !ext_feature.antiLag;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool
|
||||
check_calibrated_timestamps(instance_data *data, VkPhysicalDevice physicalDevice, bool *has_khr)
|
||||
{
|
||||
VkResult res;
|
||||
uint32_t count = 0;
|
||||
res = data->vtable.EnumerateDeviceExtensionProperties(physicalDevice, NULL, &count, NULL);
|
||||
VkExtensionProperties *extensions =
|
||||
vk_alloc(&data->alloc, count * sizeof(VkExtensionProperties), alignof(VkExtensionProperties),
|
||||
VK_SYSTEM_ALLOCATION_SCOPE_COMMAND);
|
||||
if (!extensions)
|
||||
return false;
|
||||
|
||||
res |= data->vtable.EnumerateDeviceExtensionProperties(physicalDevice, NULL, &count, extensions);
|
||||
|
||||
*has_khr = false;
|
||||
bool has_ext = false;
|
||||
if (res == VK_SUCCESS) {
|
||||
for (unsigned i = 0; i < count; i++) {
|
||||
if (strcmp(extensions[i].extensionName, VK_KHR_CALIBRATED_TIMESTAMPS_EXTENSION_NAME) == 0)
|
||||
*has_khr = true;
|
||||
if (strcmp(extensions[i].extensionName, VK_EXT_CALIBRATED_TIMESTAMPS_EXTENSION_NAME) == 0)
|
||||
has_ext = true;
|
||||
}
|
||||
}
|
||||
|
||||
vk_free(&data->alloc, extensions);
|
||||
return *has_khr || has_ext;
|
||||
}
|
||||
|
||||
/* Initialize per-queue context:
|
||||
*
|
||||
* This includes creating one CommandPool and one QueryPool per Queue as well as
|
||||
* recording one CommandBuffer per timestamp query.
|
||||
*/
|
||||
static VkResult
|
||||
init_queue_context(device_context *ctx, queue_context *queue_ctx)
|
||||
{
|
||||
#define CHECK_RESULT(res, label) \
|
||||
if (res != VK_SUCCESS) { \
|
||||
goto label; \
|
||||
}
|
||||
|
||||
VkResult result;
|
||||
|
||||
/* Create command pool */
|
||||
struct VkCommandPoolCreateInfo pool_info = {
|
||||
.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,
|
||||
.pNext = NULL,
|
||||
.flags = 0,
|
||||
.queueFamilyIndex = queue_ctx->queue_family_idx,
|
||||
};
|
||||
result =
|
||||
ctx->vtable.CreateCommandPool(ctx->device, &pool_info, &ctx->alloc, &queue_ctx->cmdPool);
|
||||
CHECK_RESULT(result, fail_cmdpool)
|
||||
|
||||
/* Create query pool */
|
||||
VkQueryPoolCreateInfo query_pool_info = {
|
||||
.sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO,
|
||||
.queryType = VK_QUERY_TYPE_TIMESTAMP,
|
||||
.queryCount = MAX_QUERIES,
|
||||
};
|
||||
result = ctx->vtable.CreateQueryPool(ctx->device, &query_pool_info, &ctx->alloc,
|
||||
&queue_ctx->queryPool);
|
||||
CHECK_RESULT(result, fail_querypool)
|
||||
ctx->vtable.ResetQueryPool(ctx->device, queue_ctx->queryPool, 0, MAX_QUERIES);
|
||||
ringbuffer_init(queue_ctx->queries);
|
||||
|
||||
/* Create timeline semaphore */
|
||||
VkSemaphoreTypeCreateInfo timelineCreateInfo = {
|
||||
.sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO,
|
||||
.pNext = NULL,
|
||||
.semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE,
|
||||
.initialValue = 0,
|
||||
};
|
||||
VkSemaphoreCreateInfo createInfo = {
|
||||
.sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,
|
||||
.pNext = &timelineCreateInfo,
|
||||
.flags = 0,
|
||||
};
|
||||
result =
|
||||
ctx->vtable.CreateSemaphore(ctx->device, &createInfo, &ctx->alloc, &queue_ctx->semaphore);
|
||||
CHECK_RESULT(result, fail_semaphore);
|
||||
|
||||
for (unsigned j = 0; j < MAX_QUERIES; j++) {
|
||||
struct query *query = &queue_ctx->queries.data[j];
|
||||
|
||||
/* Allocate commandBuffer for timestamp. */
|
||||
VkCommandBufferAllocateInfo buffer_info = {
|
||||
.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO,
|
||||
.commandPool = queue_ctx->cmdPool,
|
||||
.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY,
|
||||
.commandBufferCount = 1,
|
||||
};
|
||||
result = ctx->vtable.AllocateCommandBuffers(ctx->device, &buffer_info, &query->cmdbuffer);
|
||||
CHECK_RESULT(result, fail)
|
||||
result = ctx->vtable.SetDeviceLoaderData(ctx->device, query->cmdbuffer);
|
||||
CHECK_RESULT(result, fail)
|
||||
|
||||
/* Record commandbuffer. */
|
||||
VkCommandBufferBeginInfo beginInfo = {
|
||||
.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
|
||||
};
|
||||
|
||||
result = ctx->vtable.BeginCommandBuffer(query->cmdbuffer, &beginInfo);
|
||||
CHECK_RESULT(result, fail)
|
||||
if (j % 2 == 0)
|
||||
ctx->vtable.CmdWriteTimestamp(query->cmdbuffer, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
|
||||
queue_ctx->queryPool, j);
|
||||
else
|
||||
ctx->vtable.CmdWriteTimestamp(query->cmdbuffer, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
|
||||
queue_ctx->queryPool, j);
|
||||
result = ctx->vtable.EndCommandBuffer(query->cmdbuffer);
|
||||
CHECK_RESULT(result, fail)
|
||||
}
|
||||
|
||||
#undef CHECK_RESULT
|
||||
return result;
|
||||
|
||||
fail:
|
||||
ctx->vtable.DestroySemaphore(ctx->device, queue_ctx->semaphore, &ctx->alloc);
|
||||
fail_semaphore:
|
||||
ctx->vtable.DestroyQueryPool(ctx->device, queue_ctx->queryPool, &ctx->alloc);
|
||||
fail_querypool:
|
||||
ctx->vtable.DestroyCommandPool(ctx->device, queue_ctx->cmdPool, &ctx->alloc);
|
||||
fail_cmdpool:
|
||||
for (queue_context *qctx = ctx->queues; qctx != queue_ctx; qctx++) {
|
||||
ctx->vtable.DestroyQueryPool(ctx->device, qctx->queryPool, &ctx->alloc);
|
||||
ctx->vtable.DestroyCommandPool(ctx->device, qctx->cmdPool, &ctx->alloc);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
static VKAPI_ATTR VkResult VKAPI_CALL
|
||||
anti_lag_CreateDevice(VkPhysicalDevice physicalDevice, const VkDeviceCreateInfo *pCreateInfo,
|
||||
const VkAllocationCallbacks *pAllocator, VkDevice *pDevice)
|
||||
{
|
||||
instance_data *instance_ctx = get_instance_data(physicalDevice);
|
||||
VkLayerDeviceCreateInfo *chain_info = get_device_chain_info(pCreateInfo, VK_LAYER_LINK_INFO);
|
||||
PFN_vkGetDeviceProcAddr fpGetDeviceProcAddr = chain_info->u.pLayerInfo->pfnNextGetDeviceProcAddr;
|
||||
PFN_vkGetInstanceProcAddr fpGetInstanceProcAddr =
|
||||
chain_info->u.pLayerInfo->pfnNextGetInstanceProcAddr;
|
||||
PFN_vkCreateDevice fpCreateDevice =
|
||||
(PFN_vkCreateDevice)fpGetInstanceProcAddr(instance_ctx->instance, "vkCreateDevice");
|
||||
if (fpCreateDevice == NULL)
|
||||
return VK_ERROR_INITIALIZATION_FAILED;
|
||||
|
||||
/* Advance the link info for the next element on the chain. */
|
||||
chain_info->u.pLayerInfo = chain_info->u.pLayerInfo->pNext;
|
||||
|
||||
const VkAllocationCallbacks *alloc = pAllocator ? pAllocator : &instance_ctx->alloc;
|
||||
device_data *data;
|
||||
VkResult result;
|
||||
|
||||
/* Only allocate a context and add to dispatch if the extension is enabled. */
|
||||
const VkPhysicalDeviceAntiLagFeaturesAMD *ext_features =
|
||||
vk_find_struct_const(pCreateInfo->pNext, PHYSICAL_DEVICE_ANTI_LAG_FEATURES_AMD);
|
||||
bool enable = ext_features && should_enable_layer(instance_ctx, physicalDevice, *ext_features);
|
||||
if (enable) {
|
||||
/* Count queues with sufficient timestamp valid bits. */
|
||||
// TODO: make it work with less than 64 valid bits
|
||||
unsigned num_queue_families = 0;
|
||||
unsigned num_queues = 0;
|
||||
for (unsigned i = 0; i < pCreateInfo->queueCreateInfoCount; i++)
|
||||
num_queue_families =
|
||||
MAX2(num_queue_families, pCreateInfo->pQueueCreateInfos[i].queueFamilyIndex + 1);
|
||||
VkQueueFamilyProperties *queue_family_props =
|
||||
vk_alloc(alloc, num_queue_families * sizeof(VkQueueFamilyProperties),
|
||||
alignof(VkQueueFamilyProperties), VK_SYSTEM_ALLOCATION_SCOPE_COMMAND);
|
||||
if (!queue_family_props)
|
||||
return VK_ERROR_OUT_OF_HOST_MEMORY;
|
||||
|
||||
instance_ctx->vtable.GetPhysicalDeviceQueueFamilyProperties(
|
||||
physicalDevice, &num_queue_families, queue_family_props);
|
||||
for (unsigned i = 0; i < pCreateInfo->queueCreateInfoCount; i++) {
|
||||
uint32_t queue_family_idx = pCreateInfo->pQueueCreateInfos[i].queueFamilyIndex;
|
||||
if (queue_family_props[queue_family_idx].timestampValidBits == 64 &&
|
||||
(queue_family_props[queue_family_idx].queueFlags &
|
||||
(VK_QUEUE_GRAPHICS_BIT | VK_QUEUE_COMPUTE_BIT))) {
|
||||
num_queues += pCreateInfo->pQueueCreateInfos[i].queueCount;
|
||||
}
|
||||
}
|
||||
|
||||
/* Allocate the context. */
|
||||
device_context *ctx;
|
||||
queue_context *queues;
|
||||
VK_MULTIALLOC(ma);
|
||||
vk_multialloc_add(&ma, &data, device_data, 1);
|
||||
vk_multialloc_add(&ma, &ctx, struct device_context, 1);
|
||||
vk_multialloc_add(&ma, &queues, queue_context, num_queues);
|
||||
void *buf = vk_multialloc_zalloc(&ma, alloc, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
|
||||
if (!buf) {
|
||||
vk_free(alloc, queue_family_props);
|
||||
return VK_ERROR_OUT_OF_HOST_MEMORY;
|
||||
}
|
||||
|
||||
VkPhysicalDeviceProperties properties;
|
||||
instance_ctx->vtable.GetPhysicalDeviceProperties(physicalDevice, &properties);
|
||||
|
||||
/* Ensure that calibrated timestamps and host query reset extensions are enabled. */
|
||||
bool has_calibrated_timestamps = false;
|
||||
bool has_calibrated_timestamps_khr = false;
|
||||
bool has_vk12 = instance_ctx->apiVersion >= VK_API_VERSION_1_2 &&
|
||||
properties.apiVersion >= VK_API_VERSION_1_2;
|
||||
bool has_host_query_reset = has_vk12;
|
||||
bool has_host_query_reset_ext = false;
|
||||
bool has_timeline_semaphore = has_vk12;
|
||||
bool has_timeline_semaphore_khr = false;
|
||||
for (unsigned i = 0; i < pCreateInfo->enabledExtensionCount; i++) {
|
||||
if (strcmp(pCreateInfo->ppEnabledExtensionNames[i],
|
||||
VK_KHR_CALIBRATED_TIMESTAMPS_EXTENSION_NAME) == 0)
|
||||
has_calibrated_timestamps = has_calibrated_timestamps_khr = true;
|
||||
if (strcmp(pCreateInfo->ppEnabledExtensionNames[i],
|
||||
VK_EXT_CALIBRATED_TIMESTAMPS_EXTENSION_NAME) == 0)
|
||||
has_calibrated_timestamps = true;
|
||||
if (strcmp(pCreateInfo->ppEnabledExtensionNames[i],
|
||||
VK_EXT_HOST_QUERY_RESET_EXTENSION_NAME) == 0)
|
||||
has_host_query_reset = has_host_query_reset_ext = true;
|
||||
if (strcmp(pCreateInfo->ppEnabledExtensionNames[i],
|
||||
VK_KHR_TIMELINE_SEMAPHORE_EXTENSION_NAME) == 0)
|
||||
has_timeline_semaphore = has_timeline_semaphore_khr = true;
|
||||
}
|
||||
|
||||
/* Add missing extensions. */
|
||||
VkDeviceCreateInfo create_info = *pCreateInfo;
|
||||
const char **ext_names = NULL;
|
||||
uint32_t num_extra_extensions =
|
||||
!has_calibrated_timestamps + !has_host_query_reset + !has_timeline_semaphore;
|
||||
if (num_extra_extensions) {
|
||||
ext_names = vk_alloc(
|
||||
alloc, (pCreateInfo->enabledExtensionCount + num_extra_extensions) * sizeof(char *),
|
||||
alignof(char *), VK_SYSTEM_ALLOCATION_SCOPE_COMMAND);
|
||||
if (!ext_names) {
|
||||
result = VK_ERROR_OUT_OF_HOST_MEMORY;
|
||||
goto fail;
|
||||
}
|
||||
|
||||
memcpy(ext_names, pCreateInfo->ppEnabledExtensionNames,
|
||||
sizeof(char *) * pCreateInfo->enabledExtensionCount);
|
||||
|
||||
if (!has_timeline_semaphore) {
|
||||
has_timeline_semaphore_khr = true;
|
||||
ext_names[create_info.enabledExtensionCount++] =
|
||||
VK_KHR_TIMELINE_SEMAPHORE_EXTENSION_NAME;
|
||||
}
|
||||
if (!has_host_query_reset) {
|
||||
has_host_query_reset_ext = true;
|
||||
ext_names[create_info.enabledExtensionCount++] = VK_EXT_HOST_QUERY_RESET_EXTENSION_NAME;
|
||||
}
|
||||
if (!has_calibrated_timestamps) {
|
||||
check_calibrated_timestamps(instance_ctx, physicalDevice,
|
||||
&has_calibrated_timestamps_khr);
|
||||
ext_names[create_info.enabledExtensionCount++] =
|
||||
has_calibrated_timestamps_khr ? VK_KHR_CALIBRATED_TIMESTAMPS_EXTENSION_NAME
|
||||
: VK_EXT_CALIBRATED_TIMESTAMPS_EXTENSION_NAME;
|
||||
}
|
||||
create_info.ppEnabledExtensionNames = ext_names;
|
||||
}
|
||||
|
||||
/* Ensure that hostQueryReset feature is enabled. */
|
||||
const VkPhysicalDeviceVulkan12Features *vk12 =
|
||||
vk_find_struct_const(pCreateInfo->pNext, PHYSICAL_DEVICE_VULKAN_1_2_FEATURES);
|
||||
const VkPhysicalDeviceHostQueryResetFeatures *query_reset =
|
||||
vk_find_struct_const(pCreateInfo->pNext, PHYSICAL_DEVICE_HOST_QUERY_RESET_FEATURES);
|
||||
const VkPhysicalDeviceTimelineSemaphoreFeatures *timeline_semaphore =
|
||||
vk_find_struct_const(pCreateInfo->pNext, PHYSICAL_DEVICE_TIMELINE_SEMAPHORE_FEATURES);
|
||||
uint32_t prev_hostQueryReset;
|
||||
uint32_t prev_timelineSemaphore;
|
||||
if (vk12) {
|
||||
prev_hostQueryReset = vk12->hostQueryReset;
|
||||
prev_timelineSemaphore = vk12->timelineSemaphore;
|
||||
((VkPhysicalDeviceVulkan12Features *)vk12)->hostQueryReset = VK_TRUE;
|
||||
((VkPhysicalDeviceVulkan12Features *)vk12)->timelineSemaphore = VK_TRUE;
|
||||
} else {
|
||||
if (query_reset) {
|
||||
prev_hostQueryReset = query_reset->hostQueryReset;
|
||||
((VkPhysicalDeviceHostQueryResetFeatures *)query_reset)->hostQueryReset = VK_TRUE;
|
||||
} else {
|
||||
VkPhysicalDeviceHostQueryResetFeatures *feat =
|
||||
alloca(sizeof(VkPhysicalDeviceHostQueryResetFeatures));
|
||||
*feat = (VkPhysicalDeviceHostQueryResetFeatures){
|
||||
.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_HOST_QUERY_RESET_FEATURES,
|
||||
.pNext = (void *)create_info.pNext,
|
||||
.hostQueryReset = VK_TRUE,
|
||||
};
|
||||
create_info.pNext = feat;
|
||||
}
|
||||
if (timeline_semaphore) {
|
||||
prev_timelineSemaphore = timeline_semaphore->timelineSemaphore;
|
||||
((VkPhysicalDeviceTimelineSemaphoreFeatures *)timeline_semaphore)->timelineSemaphore =
|
||||
VK_TRUE;
|
||||
} else {
|
||||
VkPhysicalDeviceTimelineSemaphoreFeatures *feat =
|
||||
alloca(sizeof(VkPhysicalDeviceTimelineSemaphoreFeatures));
|
||||
*feat = (VkPhysicalDeviceTimelineSemaphoreFeatures){
|
||||
.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TIMELINE_SEMAPHORE_FEATURES,
|
||||
.pNext = (void *)create_info.pNext,
|
||||
.timelineSemaphore = VK_TRUE,
|
||||
};
|
||||
create_info.pNext = feat;
|
||||
}
|
||||
}
|
||||
|
||||
/* Create Device. */
|
||||
result = fpCreateDevice(physicalDevice, &create_info, pAllocator, pDevice);
|
||||
|
||||
if (vk12) {
|
||||
((VkPhysicalDeviceVulkan12Features *)vk12)->hostQueryReset = prev_hostQueryReset;
|
||||
((VkPhysicalDeviceVulkan12Features *)vk12)->timelineSemaphore = prev_timelineSemaphore;
|
||||
} else {
|
||||
if (query_reset)
|
||||
((VkPhysicalDeviceHostQueryResetFeatures *)query_reset)->hostQueryReset =
|
||||
prev_hostQueryReset;
|
||||
if (timeline_semaphore)
|
||||
((VkPhysicalDeviceTimelineSemaphoreFeatures *)timeline_semaphore)->timelineSemaphore =
|
||||
prev_timelineSemaphore;
|
||||
}
|
||||
if (ext_names)
|
||||
vk_free(alloc, ext_names);
|
||||
|
||||
if (result != VK_SUCCESS)
|
||||
goto fail;
|
||||
|
||||
/* Initialize Context. */
|
||||
data->ctx = ctx;
|
||||
ctx->device = *pDevice;
|
||||
chain_info = get_device_chain_info(pCreateInfo, VK_LOADER_DATA_CALLBACK);
|
||||
PFN_vkSetDeviceLoaderData fpSetDeviceLoaderData =
|
||||
(PFN_vkSetDeviceLoaderData)chain_info->u.pfnSetDeviceLoaderData;
|
||||
init_device_vtable(ctx, fpGetDeviceProcAddr, fpSetDeviceLoaderData,
|
||||
has_calibrated_timestamps_khr, has_host_query_reset_ext,
|
||||
has_timeline_semaphore_khr);
|
||||
simple_mtx_init(&ctx->mtx, mtx_plain);
|
||||
ctx->num_queues = num_queues;
|
||||
ctx->alloc = *alloc;
|
||||
ctx->calibration.timestamp_period = properties.limits.timestampPeriod;
|
||||
ringbuffer_init(ctx->frames);
|
||||
|
||||
/* Initialize Queue contexts. */
|
||||
unsigned idx = 0;
|
||||
for (unsigned i = 0; i < pCreateInfo->queueCreateInfoCount; i++) {
|
||||
/* Skip queue families without sufficient timestamp valid bits.
|
||||
* Also skip queue families which cannot do GRAPHICS or COMPUTE since they
|
||||
* always heavily async in nature (DMA transfers and sparse for example).
|
||||
* Video is also irrelvant here since it should never be a critical path
|
||||
* in a game that wants anti-lag. */
|
||||
uint32_t queue_family_idx = pCreateInfo->pQueueCreateInfos[i].queueFamilyIndex;
|
||||
if (queue_family_props[queue_family_idx].timestampValidBits != 64 ||
|
||||
!(queue_family_props[queue_family_idx].queueFlags &
|
||||
(VK_QUEUE_GRAPHICS_BIT | VK_QUEUE_COMPUTE_BIT)))
|
||||
continue;
|
||||
|
||||
for (unsigned j = 0; j < pCreateInfo->pQueueCreateInfos[i].queueCount; j++) {
|
||||
VkQueue queue;
|
||||
ctx->vtable.GetDeviceQueue(*pDevice, queue_family_idx, j, &queue);
|
||||
ctx->queues[idx].queue = queue;
|
||||
ctx->queues[idx].queue_family_idx = queue_family_idx;
|
||||
result = init_queue_context(ctx, &ctx->queues[idx]);
|
||||
idx++;
|
||||
if (result != VK_SUCCESS)
|
||||
goto fail;
|
||||
}
|
||||
}
|
||||
assert(idx == num_queues);
|
||||
fail:
|
||||
vk_free(alloc, queue_family_props);
|
||||
} else {
|
||||
data = (device_data *)vk_alloc(alloc, sizeof(device_data), alignof(device_data),
|
||||
VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
|
||||
if (!data)
|
||||
return VK_ERROR_OUT_OF_HOST_MEMORY;
|
||||
result = fpCreateDevice(physicalDevice, pCreateInfo, pAllocator, pDevice);
|
||||
data->ctx = NULL;
|
||||
}
|
||||
|
||||
if (result == VK_SUCCESS) {
|
||||
data->device = *pDevice;
|
||||
data->GetDeviceProcAddr = fpGetDeviceProcAddr;
|
||||
data->next = NULL;
|
||||
add_device(data);
|
||||
} else {
|
||||
vk_free(alloc, data);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
static VKAPI_ATTR void VKAPI_CALL
|
||||
anti_lag_DestroyDevice(VkDevice pDevice, const VkAllocationCallbacks *pAllocator)
|
||||
{
|
||||
device_data *data = remove_device(pDevice);
|
||||
assert(data && data->ctx);
|
||||
device_context *ctx = data->ctx;
|
||||
|
||||
/* Destroy per-queue context.
|
||||
* The application must ensure that no work is active on the device.
|
||||
*/
|
||||
for (unsigned i = 0; i < ctx->num_queues; i++) {
|
||||
queue_context *queue_ctx = &ctx->queues[i];
|
||||
ctx->vtable.DestroyQueryPool(ctx->device, queue_ctx->queryPool, &ctx->alloc);
|
||||
ctx->vtable.DestroyCommandPool(ctx->device, queue_ctx->cmdPool, &ctx->alloc);
|
||||
ctx->vtable.DestroySemaphore(ctx->device, queue_ctx->semaphore, &ctx->alloc);
|
||||
}
|
||||
|
||||
ctx->vtable.DestroyDevice(pDevice, pAllocator);
|
||||
vk_free(&ctx->alloc, data);
|
||||
}
|
||||
|
||||
static bool
|
||||
is_anti_lag_supported(VkPhysicalDevice physicalDevice)
|
||||
{
|
||||
instance_data *data = get_instance_data(physicalDevice);
|
||||
VkPhysicalDeviceProperties properties;
|
||||
data->vtable.GetPhysicalDeviceProperties(physicalDevice, &properties);
|
||||
if (properties.limits.timestampPeriod == 0.0 || !properties.limits.timestampComputeAndGraphics)
|
||||
return false;
|
||||
|
||||
/* Check whether calibrated timestamps are supported. */
|
||||
bool has_khr;
|
||||
if (!check_calibrated_timestamps(data, physicalDevice, &has_khr))
|
||||
return false;
|
||||
|
||||
/* Check whether timeline semaphores and host query reset are supported. */
|
||||
VkPhysicalDeviceTimelineSemaphoreFeatures timeline_semaphore = {
|
||||
.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TIMELINE_SEMAPHORE_FEATURES,
|
||||
.timelineSemaphore = VK_FALSE,
|
||||
};
|
||||
VkPhysicalDeviceHostQueryResetFeatures query_reset = {
|
||||
.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_HOST_QUERY_RESET_FEATURES,
|
||||
.pNext = &timeline_semaphore,
|
||||
.hostQueryReset = VK_FALSE,
|
||||
};
|
||||
VkPhysicalDeviceFeatures2 features = {
|
||||
.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2,
|
||||
.pNext = &query_reset,
|
||||
};
|
||||
if (data->vtable.GetPhysicalDeviceFeatures2KHR)
|
||||
data->vtable.GetPhysicalDeviceFeatures2KHR(physicalDevice, &features);
|
||||
else if (data->vtable.GetPhysicalDeviceFeatures2)
|
||||
data->vtable.GetPhysicalDeviceFeatures2(physicalDevice, &features);
|
||||
if (!timeline_semaphore.timelineSemaphore || !query_reset.hostQueryReset)
|
||||
return false;
|
||||
|
||||
/* Check that DEVICE and CLOCK_MONOTONIC time domains are available. */
|
||||
VkResult res;
|
||||
uint32_t count = 0;
|
||||
PFN_vkGetPhysicalDeviceCalibrateableTimeDomainsKHR ctd =
|
||||
has_khr ? data->vtable.GetPhysicalDeviceCalibrateableTimeDomainsKHR
|
||||
: data->vtable.GetPhysicalDeviceCalibrateableTimeDomainsEXT;
|
||||
res = ctd(physicalDevice, &count, NULL);
|
||||
VkTimeDomainKHR *time_domains = alloca(count * sizeof(VkTimeDomainKHR));
|
||||
res |= ctd(physicalDevice, &count, time_domains);
|
||||
if (res != VK_SUCCESS)
|
||||
return false;
|
||||
|
||||
bool has_device_domain = false;
|
||||
bool has_host_domain = false;
|
||||
for (unsigned i = 0; i < count; i++) {
|
||||
has_device_domain |= time_domains[i] == VK_TIME_DOMAIN_DEVICE_KHR;
|
||||
has_host_domain |= time_domains[i] == VK_TIME_DOMAIN_CLOCK_MONOTONIC_KHR;
|
||||
}
|
||||
|
||||
return has_device_domain && has_host_domain;
|
||||
}
|
||||
|
||||
static VKAPI_ATTR VkResult VKAPI_CALL
|
||||
anti_lag_EnumerateDeviceExtensionProperties(VkPhysicalDevice physicalDevice, const char *pLayerName,
|
||||
uint32_t *pPropertyCount,
|
||||
VkExtensionProperties *pProperties)
|
||||
{
|
||||
instance_data *instance_data = get_instance_data(physicalDevice);
|
||||
|
||||
if (pLayerName && strcmp(pLayerName, "VK_LAYER_MESA_anti_lag") == 0) {
|
||||
if (!is_anti_lag_supported(physicalDevice)) {
|
||||
*pPropertyCount = 0;
|
||||
return VK_SUCCESS;
|
||||
}
|
||||
|
||||
VK_OUTARRAY_MAKE_TYPED(VkExtensionProperties, out, pProperties, pPropertyCount);
|
||||
vk_outarray_append_typed(VkExtensionProperties, &out, prop)
|
||||
{
|
||||
*prop =
|
||||
(VkExtensionProperties){VK_AMD_ANTI_LAG_EXTENSION_NAME, VK_AMD_ANTI_LAG_SPEC_VERSION};
|
||||
}
|
||||
return vk_outarray_status(&out);
|
||||
}
|
||||
|
||||
return instance_data->vtable.EnumerateDeviceExtensionProperties(physicalDevice, pLayerName,
|
||||
pPropertyCount, pProperties);
|
||||
}
|
||||
|
||||
static VKAPI_ATTR void VKAPI_CALL
|
||||
anti_lag_GetPhysicalDeviceFeatures2(VkPhysicalDevice physicalDevice,
|
||||
VkPhysicalDeviceFeatures2 *pFeatures)
|
||||
{
|
||||
instance_data *ctx = get_instance_data(physicalDevice);
|
||||
ctx->vtable.GetPhysicalDeviceFeatures2(physicalDevice, pFeatures);
|
||||
VkPhysicalDeviceAntiLagFeaturesAMD *anti_lag_features =
|
||||
vk_find_struct(pFeatures->pNext, PHYSICAL_DEVICE_ANTI_LAG_FEATURES_AMD);
|
||||
|
||||
if (anti_lag_features) {
|
||||
anti_lag_features->antiLag |= is_anti_lag_supported(physicalDevice);
|
||||
}
|
||||
}
|
||||
|
||||
static VKAPI_ATTR void VKAPI_CALL
|
||||
anti_lag_GetPhysicalDeviceFeatures2KHR(VkPhysicalDevice physicalDevice,
|
||||
VkPhysicalDeviceFeatures2 *pFeatures)
|
||||
{
|
||||
instance_data *ctx = get_instance_data(physicalDevice);
|
||||
ctx->vtable.GetPhysicalDeviceFeatures2KHR(physicalDevice, pFeatures);
|
||||
VkPhysicalDeviceAntiLagFeaturesAMD *anti_lag_features =
|
||||
vk_find_struct(pFeatures->pNext, PHYSICAL_DEVICE_ANTI_LAG_FEATURES_AMD);
|
||||
|
||||
if (anti_lag_features) {
|
||||
anti_lag_features->antiLag |= is_anti_lag_supported(physicalDevice);
|
||||
}
|
||||
}
|
||||
|
||||
static VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL
|
||||
anti_lag_GetInstanceProcAddr(VkInstance instance, const char *pName);
|
||||
|
||||
static VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL
|
||||
anti_lag_GetDeviceProcAddr(VkDevice device, const char *pName);
|
||||
|
||||
#define ADD_HOOK(fn) {"vk" #fn, (PFN_vkVoidFunction)anti_lag_##fn}
|
||||
static const struct {
|
||||
const char *name;
|
||||
PFN_vkVoidFunction ptr;
|
||||
} instance_funcptr_map[] = {
|
||||
ADD_HOOK(GetInstanceProcAddr),
|
||||
ADD_HOOK(CreateInstance),
|
||||
ADD_HOOK(DestroyInstance),
|
||||
ADD_HOOK(EnumerateDeviceExtensionProperties),
|
||||
ADD_HOOK(CreateDevice),
|
||||
ADD_HOOK(GetPhysicalDeviceFeatures2),
|
||||
ADD_HOOK(GetPhysicalDeviceFeatures2KHR),
|
||||
};
|
||||
|
||||
static const struct {
|
||||
const char *name;
|
||||
PFN_vkVoidFunction ptr;
|
||||
} device_funcptr_map[] = {
|
||||
ADD_HOOK(GetDeviceProcAddr),
|
||||
ADD_HOOK(DestroyDevice),
|
||||
ADD_HOOK(AntiLagUpdateAMD),
|
||||
ADD_HOOK(QueueSubmit),
|
||||
ADD_HOOK(QueueSubmit2),
|
||||
ADD_HOOK(QueueSubmit2KHR),
|
||||
ADD_HOOK(QueuePresentKHR),
|
||||
};
|
||||
#undef ADD_HOOK
|
||||
|
||||
static VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL
|
||||
anti_lag_GetInstanceProcAddr(VkInstance instance, const char *pName)
|
||||
{
|
||||
if (!pName)
|
||||
return NULL;
|
||||
|
||||
PFN_vkVoidFunction result = NULL;
|
||||
if (instance) {
|
||||
instance_data *ctx = get_instance_data(instance);
|
||||
if (ctx)
|
||||
result = ctx->vtable.GetInstanceProcAddr(instance, pName);
|
||||
}
|
||||
|
||||
/* Only hook instance functions which are exposed by the underlying impl.
|
||||
* Ignore instance parameter for vkCreateInstance and vkCreateDevice.
|
||||
*/
|
||||
if (result || strcmp(pName, "vkCreateInstance") == 0 || strcmp(pName, "vkCreateDevice") == 0) {
|
||||
for (uint32_t i = 0; i < ARRAY_SIZE(instance_funcptr_map); i++) {
|
||||
if (strcmp(pName, instance_funcptr_map[i].name) == 0)
|
||||
return instance_funcptr_map[i].ptr;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
static VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL
|
||||
anti_lag_GetDeviceProcAddr(VkDevice device, const char *pName)
|
||||
{
|
||||
if (!pName || !device)
|
||||
return NULL;
|
||||
|
||||
device_data *data = get_device_data(device);
|
||||
PFN_vkVoidFunction result = data->GetDeviceProcAddr(device, pName);
|
||||
|
||||
/* Only hook device functions if the Layer extension is enabled. */
|
||||
if (data->ctx && (result || strcmp(pName, "vkAntiLagUpdateAMD") == 0)) {
|
||||
for (uint32_t i = 0; i < ARRAY_SIZE(device_funcptr_map); i++) {
|
||||
if (strcmp(pName, device_funcptr_map[i].name) == 0)
|
||||
return device_funcptr_map[i].ptr;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
PUBLIC VKAPI_ATTR VkResult VKAPI_CALL
|
||||
anti_lag_NegotiateLoaderLayerInterfaceVersion(VkNegotiateLayerInterface *pVersionStruct)
|
||||
{
|
||||
assert(pVersionStruct != NULL);
|
||||
assert(pVersionStruct->sType == LAYER_NEGOTIATE_INTERFACE_STRUCT);
|
||||
|
||||
if (pVersionStruct->loaderLayerInterfaceVersion >= 2) {
|
||||
pVersionStruct->loaderLayerInterfaceVersion = 2;
|
||||
pVersionStruct->pfnGetInstanceProcAddr = anti_lag_GetInstanceProcAddr;
|
||||
pVersionStruct->pfnGetDeviceProcAddr = anti_lag_GetDeviceProcAddr;
|
||||
pVersionStruct->pfnGetPhysicalDeviceProcAddr = NULL;
|
||||
}
|
||||
|
||||
return VK_SUCCESS;
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Copyright © 2025 Valve Corporation
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
vklayer_files = files(
|
||||
'anti_lag_layer.c',
|
||||
'anti_lag_layer_interface.c',
|
||||
)
|
||||
|
||||
vklayer_mesa_anti_lag = shared_library(
|
||||
'VkLayer_MESA_anti_lag',
|
||||
vklayer_files,
|
||||
c_args : [no_override_init_args],
|
||||
gnu_symbol_visibility : 'hidden',
|
||||
dependencies : [
|
||||
idep_vulkan_util, idep_mesautil,
|
||||
],
|
||||
include_directories : [inc_include, inc_util, inc_src],
|
||||
link_args : cc.get_supported_link_arguments(['-Wl,-Bsymbolic-functions', '-Wl,-z,relro']),
|
||||
install : true
|
||||
)
|
||||
|
||||
configure_file(
|
||||
configuration : {'library_path' : 'libVkLayer_MESA_anti_lag.so'},
|
||||
input : 'VkLayer_MESA_anti_lag.json.in',
|
||||
output : '@BASENAME@',
|
||||
install : true,
|
||||
install_dir : join_paths(get_option('datadir'), 'vulkan', 'implicit_layer.d'),
|
||||
install_tag : 'runtime',
|
||||
)
|
||||
|
||||
configure_file(
|
||||
configuration : {'library_path' : vklayer_mesa_anti_lag.full_path()},
|
||||
input : 'VkLayer_MESA_anti_lag.json.in',
|
||||
output : '@PLAINNAME@_dev.json',
|
||||
)
|
||||
devenv.append('VK_ADD_IMPLICIT_LAYER_PATH', meson.current_build_dir())
|
||||
@@ -0,0 +1,58 @@
|
||||
/*
|
||||
* Copyright © 2025 Valve Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*/
|
||||
|
||||
#ifndef RINGBUFFER_H
|
||||
#define RINGBUFFER_H
|
||||
|
||||
#include "util/macros.h"
|
||||
|
||||
#define RINGBUFFER_DECLARE(name, type, N) \
|
||||
struct { \
|
||||
type data[N]; \
|
||||
uint32_t head; \
|
||||
uint32_t tail; \
|
||||
uint32_t size; \
|
||||
simple_mtx_t mtx; \
|
||||
} name
|
||||
|
||||
#define ringbuffer_init(buffer) \
|
||||
(buffer.head = buffer.tail = buffer.size = 0, simple_mtx_init(&buffer.mtx, mtx_plain))
|
||||
|
||||
#define ringbuffer_lock(buffer) simple_mtx_lock(&buffer.mtx)
|
||||
#define ringbuffer_unlock(buffer) simple_mtx_unlock(&buffer.mtx)
|
||||
|
||||
static inline uint32_t
|
||||
__ringbuffer_add_wrap(uint32_t *val, uint32_t *size, uint32_t N)
|
||||
{
|
||||
uint32_t prev = *val;
|
||||
*val = (*val + 1) % N;
|
||||
*size = *size + 1;
|
||||
assert(*size <= N);
|
||||
return prev;
|
||||
}
|
||||
|
||||
#define ringbuffer_alloc(buffer) \
|
||||
(buffer.size == ARRAY_SIZE(buffer.data) \
|
||||
? NULL \
|
||||
: &buffer.data[__ringbuffer_add_wrap(&buffer.head, &buffer.size, ARRAY_SIZE(buffer.data))])
|
||||
|
||||
#define ringbuffer_free(buffer, elem) \
|
||||
assert(elem == NULL || elem == &buffer.data[buffer.tail]); \
|
||||
buffer.size--; \
|
||||
assert(buffer.size < ARRAY_SIZE(buffer.data)); \
|
||||
buffer.tail = (buffer.tail + 1) % ARRAY_SIZE(buffer.data)
|
||||
|
||||
#define ringbuffer_first(buffer) (&buffer.data[buffer.tail])
|
||||
|
||||
#define ringbuffer_last(buffer) \
|
||||
(&buffer.data[(buffer.head + ARRAY_SIZE(buffer.data) - 1) % ARRAY_SIZE(buffer.data)])
|
||||
|
||||
#define ringbuffer_index(buffer, elem) (elem - buffer.data)
|
||||
|
||||
#define ringbuffer_next(buffer, elem) \
|
||||
(&buffer.data[(ringbuffer_index(buffer, elem) + 1) % ARRAY_SIZE(buffer.data)])
|
||||
|
||||
#endif /* RINGBUFFER_H */
|
||||
@@ -0,0 +1,4 @@
|
||||
BasedOnStyle: InheritParentConfig
|
||||
DisableFormat: false
|
||||
|
||||
ColumnLimit: 100
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"file_format_version" : "1.0.0",
|
||||
"layer" : {
|
||||
"name": "VK_LAYER_MESA_device_select",
|
||||
"type": "GLOBAL",
|
||||
"library_path": "@library_path@",
|
||||
"api_version": "1.4.303",
|
||||
"implementation_version": "1",
|
||||
"description": "Linux device selection layer",
|
||||
"instance_extensions": [
|
||||
{
|
||||
"name": "VK_EXT_layer_settings",
|
||||
"spec_version": "2"
|
||||
}
|
||||
],
|
||||
"functions": {
|
||||
"vkNegotiateLoaderLayerInterfaceVersion": "vkNegotiateLoaderLayerInterfaceVersion"
|
||||
},
|
||||
"disable_environment": {
|
||||
"NODEVICE_SELECT": "1"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,416 @@
|
||||
/*
|
||||
* Copyright © 2017 Google
|
||||
* Copyright © 2019 Red Hat
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
/* Rules for device selection.
|
||||
* Is there an X or wayland connection open (or DISPLAY set).
|
||||
* If no - try and find which device was the boot_vga device.
|
||||
* If yes - try and work out which device is the connection primary,
|
||||
* DRI_PRIME tagged overrides only work if bus info, =1 will just pick an alternate.
|
||||
*/
|
||||
|
||||
#include <fcntl.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include "util/macros.h"
|
||||
#include "device_select.h"
|
||||
|
||||
static bool
|
||||
fill_drm_device_info(const struct instance_info *info, struct device_pci_info *drm_device,
|
||||
VkPhysicalDevice device)
|
||||
{
|
||||
VkPhysicalDevicePCIBusInfoPropertiesEXT ext_pci_properties =
|
||||
(VkPhysicalDevicePCIBusInfoPropertiesEXT){
|
||||
.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PCI_BUS_INFO_PROPERTIES_EXT};
|
||||
|
||||
VkPhysicalDeviceProperties2 properties =
|
||||
(VkPhysicalDeviceProperties2){.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2};
|
||||
|
||||
if (info->has_vulkan11 && info->has_pci_bus)
|
||||
properties.pNext = &ext_pci_properties;
|
||||
device_select_get_properties(info, device, &properties);
|
||||
|
||||
drm_device->device_type = properties.properties.deviceType;
|
||||
drm_device->dev_info.vendor_id = properties.properties.vendorID;
|
||||
drm_device->dev_info.device_id = properties.properties.deviceID;
|
||||
if (info->has_vulkan11 && info->has_pci_bus) {
|
||||
drm_device->has_bus_info = true;
|
||||
drm_device->bus_info.domain = ext_pci_properties.pciDomain;
|
||||
drm_device->bus_info.bus = ext_pci_properties.pciBus;
|
||||
drm_device->bus_info.dev = ext_pci_properties.pciDevice;
|
||||
drm_device->bus_info.func = ext_pci_properties.pciFunction;
|
||||
}
|
||||
return drm_device->device_type == VK_PHYSICAL_DEVICE_TYPE_CPU;
|
||||
}
|
||||
|
||||
static int
|
||||
device_select_find_explicit_default(struct device_pci_info *pci_infos, uint32_t device_count,
|
||||
const char *selection)
|
||||
{
|
||||
int default_idx = -1;
|
||||
unsigned vendor_id, device_id;
|
||||
int matched = sscanf(selection, "%x:%x", &vendor_id, &device_id);
|
||||
if (matched != 2)
|
||||
return default_idx;
|
||||
|
||||
for (unsigned i = 0; i < device_count; ++i) {
|
||||
if (pci_infos[i].dev_info.vendor_id == vendor_id &&
|
||||
pci_infos[i].dev_info.device_id == device_id)
|
||||
default_idx = i;
|
||||
}
|
||||
return default_idx;
|
||||
}
|
||||
|
||||
static int
|
||||
device_select_find_typed_default(struct device_pci_info *pci_infos, uint32_t device_count,
|
||||
const char *selection)
|
||||
{
|
||||
static struct {
|
||||
const char *name;
|
||||
VkPhysicalDeviceType type;
|
||||
} names[] = {
|
||||
{"other", VK_PHYSICAL_DEVICE_TYPE_OTHER},
|
||||
{"integrated", VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU},
|
||||
{"igpu", VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU},
|
||||
{"discrete", VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU},
|
||||
{"dgpu", VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU},
|
||||
{"virtual", VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU},
|
||||
{"vgpu", VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU},
|
||||
{"cpu", VK_PHYSICAL_DEVICE_TYPE_CPU},
|
||||
};
|
||||
|
||||
for (unsigned i = 0; i < ARRAY_SIZE(names); ++i) {
|
||||
if (strncasecmp(names[i].name, selection, strlen(names[i].name)) == 0) {
|
||||
for (unsigned j = 0; j < device_count; ++j) {
|
||||
if (pci_infos[j].device_type == names[i].type)
|
||||
return j;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
static int
|
||||
device_select_find_dri_prime_tag_default(struct device_pci_info *pci_infos, uint32_t device_count,
|
||||
const char *dri_prime)
|
||||
{
|
||||
int default_idx = -1;
|
||||
|
||||
/* Drop the trailing '!' if present. */
|
||||
int ref = strlen("pci-xxxx_yy_zz_w");
|
||||
int n = strlen(dri_prime);
|
||||
if (n < ref)
|
||||
return default_idx;
|
||||
if (n == ref + 1 && dri_prime[n - 1] == '!')
|
||||
n--;
|
||||
|
||||
for (unsigned i = 0; i < device_count; ++i) {
|
||||
char *tag = NULL;
|
||||
if (asprintf(&tag, "pci-%04x_%02x_%02x_%1u", pci_infos[i].bus_info.domain,
|
||||
pci_infos[i].bus_info.bus, pci_infos[i].bus_info.dev,
|
||||
pci_infos[i].bus_info.func) >= 0) {
|
||||
if (strncmp(dri_prime, tag, n) == 0)
|
||||
default_idx = i;
|
||||
}
|
||||
free(tag);
|
||||
}
|
||||
return default_idx;
|
||||
}
|
||||
|
||||
static int
|
||||
device_select_find_boot_vga_vid_did(struct device_pci_info *pci_infos, uint32_t device_count)
|
||||
{
|
||||
char path[1024];
|
||||
int fd;
|
||||
int default_idx = -1;
|
||||
uint8_t boot_vga = 0;
|
||||
ssize_t size_ret;
|
||||
#pragma pack(push, 1)
|
||||
struct id {
|
||||
uint16_t vid;
|
||||
uint16_t did;
|
||||
} id;
|
||||
#pragma pack(pop)
|
||||
|
||||
for (unsigned i = 0; i < 64; i++) {
|
||||
snprintf(path, 1023, "/sys/class/drm/card%d/device/boot_vga", i);
|
||||
fd = open(path, O_RDONLY);
|
||||
if (fd != -1) {
|
||||
uint8_t val;
|
||||
size_ret = read(fd, &val, 1);
|
||||
close(fd);
|
||||
if (size_ret == 1 && val == '1')
|
||||
boot_vga = 1;
|
||||
} else {
|
||||
return default_idx;
|
||||
}
|
||||
|
||||
if (boot_vga) {
|
||||
snprintf(path, 1023, "/sys/class/drm/card%d/device/config", i);
|
||||
fd = open(path, O_RDONLY);
|
||||
if (fd != -1) {
|
||||
size_ret = read(fd, &id, 4);
|
||||
close(fd);
|
||||
if (size_ret != 4)
|
||||
return default_idx;
|
||||
} else {
|
||||
return default_idx;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!boot_vga)
|
||||
return default_idx;
|
||||
|
||||
for (unsigned i = 0; i < device_count; ++i) {
|
||||
if (id.vid == pci_infos[i].dev_info.vendor_id && id.did == pci_infos[i].dev_info.device_id) {
|
||||
default_idx = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return default_idx;
|
||||
}
|
||||
|
||||
static int
|
||||
device_select_find_boot_vga_default(struct device_pci_info *pci_infos, uint32_t device_count)
|
||||
{
|
||||
char boot_vga_path[1024];
|
||||
int default_idx = -1;
|
||||
for (unsigned i = 0; i < device_count; ++i) {
|
||||
/* fallback to probing the pci bus boot_vga device. */
|
||||
snprintf(boot_vga_path, 1023, "/sys/bus/pci/devices/%04x:%02x:%02x.%x/boot_vga",
|
||||
pci_infos[i].bus_info.domain, pci_infos[i].bus_info.bus, pci_infos[i].bus_info.dev,
|
||||
pci_infos[i].bus_info.func);
|
||||
int fd = open(boot_vga_path, O_RDONLY);
|
||||
if (fd != -1) {
|
||||
uint8_t val;
|
||||
if (read(fd, &val, 1) == 1) {
|
||||
if (val == '1')
|
||||
default_idx = i;
|
||||
}
|
||||
close(fd);
|
||||
}
|
||||
if (default_idx != -1)
|
||||
break;
|
||||
}
|
||||
return default_idx;
|
||||
}
|
||||
|
||||
static int
|
||||
device_select_find_non_cpu(struct device_pci_info *pci_infos, uint32_t device_count)
|
||||
{
|
||||
int default_idx = -1;
|
||||
|
||||
/* pick first GPU device */
|
||||
for (unsigned i = 0; i < device_count; ++i) {
|
||||
if (pci_infos[i].device_type != VK_PHYSICAL_DEVICE_TYPE_CPU) {
|
||||
default_idx = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return default_idx;
|
||||
}
|
||||
|
||||
static int
|
||||
find_non_cpu_skip(struct device_pci_info *pci_infos, uint32_t device_count, int skip_idx,
|
||||
int skip_count)
|
||||
{
|
||||
for (unsigned i = 0; i < device_count; ++i) {
|
||||
if (i == skip_idx)
|
||||
continue;
|
||||
if (pci_infos[i].device_type == VK_PHYSICAL_DEVICE_TYPE_CPU)
|
||||
continue;
|
||||
skip_count--;
|
||||
if (skip_count > 0)
|
||||
continue;
|
||||
|
||||
return i;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
static bool
|
||||
ends_with_exclamation_mark(const char *str)
|
||||
{
|
||||
size_t n = strlen(str);
|
||||
return n > 1 && str[n - 1] == '!';
|
||||
}
|
||||
|
||||
static int
|
||||
get_selected(const struct instance_info *info, uint32_t count, struct device_pci_info *pci_infos,
|
||||
const char *dri_prime, bool *expose_only_one_dev)
|
||||
{
|
||||
int default_idx = -1;
|
||||
|
||||
if (info->selection)
|
||||
default_idx = device_select_find_typed_default(pci_infos, count, info->selection);
|
||||
if (default_idx != -1) {
|
||||
if (info->debug)
|
||||
fprintf(stderr,
|
||||
"device-select: device_select_find_typed_default for MESA_VK_DEVICE_SELECT "
|
||||
"selected %i\n",
|
||||
default_idx);
|
||||
*expose_only_one_dev = ends_with_exclamation_mark(info->selection);
|
||||
return default_idx;
|
||||
}
|
||||
|
||||
if (info->selection)
|
||||
default_idx = device_select_find_explicit_default(pci_infos, count, info->selection);
|
||||
if (default_idx != -1) {
|
||||
if (info->debug)
|
||||
fprintf(stderr,
|
||||
"device-select: device_select_find_explicit_default for MESA_VK_DEVICE_SELECT "
|
||||
"selected %i\n",
|
||||
default_idx);
|
||||
*expose_only_one_dev = ends_with_exclamation_mark(info->selection);
|
||||
return default_idx;
|
||||
}
|
||||
|
||||
if (!dri_prime)
|
||||
return default_idx;
|
||||
|
||||
/* Try DRI_PRIME=vendor_id:device_id */
|
||||
default_idx = device_select_find_explicit_default(pci_infos, count, dri_prime);
|
||||
if (default_idx != -1) {
|
||||
if (info->debug)
|
||||
fprintf(stderr, "device-select: device_select_find_explicit_default selected %i\n",
|
||||
default_idx);
|
||||
*expose_only_one_dev = ends_with_exclamation_mark(dri_prime);
|
||||
return default_idx;
|
||||
}
|
||||
|
||||
/* Try DRI_PRIME=pci-xxxx_yy_zz_w */
|
||||
if (!info->has_vulkan11 && !info->has_pci_bus)
|
||||
fprintf(stderr, "device-select: cannot correctly use DRI_PRIME tag\n");
|
||||
else
|
||||
default_idx = device_select_find_dri_prime_tag_default(pci_infos, count, dri_prime);
|
||||
|
||||
if (default_idx != -1) {
|
||||
if (info->debug)
|
||||
fprintf(stderr, "device-select: device_select_find_dri_prime_tag_default selected %i\n",
|
||||
default_idx);
|
||||
*expose_only_one_dev = ends_with_exclamation_mark(dri_prime);
|
||||
}
|
||||
|
||||
return default_idx;
|
||||
}
|
||||
|
||||
static int
|
||||
get_default(const struct instance_info *info, uint32_t count, struct device_pci_info *pci_infos)
|
||||
{
|
||||
int default_idx = -1;
|
||||
|
||||
bool has_cpu = false;
|
||||
for (unsigned i = 0; i < count; ++i)
|
||||
has_cpu |= pci_infos[i].device_type == VK_PHYSICAL_DEVICE_TYPE_CPU;
|
||||
|
||||
if (default_idx == -1 && info->has_wayland) {
|
||||
default_idx = device_select_find_wayland_pci_default(pci_infos, count);
|
||||
if (info->debug && default_idx != -1)
|
||||
fprintf(stderr, "device-select: device_select_find_wayland_pci_default selected %i\n",
|
||||
default_idx);
|
||||
}
|
||||
|
||||
if (default_idx == -1 && info->has_xcb) {
|
||||
default_idx = device_select_find_xcb_pci_default(pci_infos, count);
|
||||
if (info->debug && default_idx != -1)
|
||||
fprintf(stderr, "device-select: device_select_find_xcb_pci_default selected %i\n",
|
||||
default_idx);
|
||||
}
|
||||
|
||||
if (default_idx == -1) {
|
||||
if (info->has_vulkan11 && info->has_pci_bus)
|
||||
default_idx = device_select_find_boot_vga_default(pci_infos, count);
|
||||
else
|
||||
default_idx = device_select_find_boot_vga_vid_did(pci_infos, count);
|
||||
if (info->debug && default_idx != -1)
|
||||
fprintf(stderr, "device-select: device_select_find_boot_vga selected %i\n", default_idx);
|
||||
}
|
||||
|
||||
/* If no GPU has been selected so far, select the first non-CPU device. If none are available,
|
||||
* pick the first CPU device.
|
||||
*/
|
||||
if (default_idx == -1) {
|
||||
default_idx = device_select_find_non_cpu(pci_infos, count);
|
||||
if (info->debug && default_idx != -1)
|
||||
fprintf(stderr, "device-select: device_select_find_non_cpu selected %i\n", default_idx);
|
||||
}
|
||||
|
||||
if (default_idx == -1 && has_cpu)
|
||||
default_idx = 0;
|
||||
|
||||
return default_idx;
|
||||
}
|
||||
|
||||
uint32_t
|
||||
device_select_get_first(const struct instance_info *info, uint32_t physical_device_count,
|
||||
VkPhysicalDevice *pPhysicalDevices, bool *expose_only_one_dev)
|
||||
{
|
||||
int default_idx = -1;
|
||||
int dri_prime_as_int = -1;
|
||||
int cpu_count = 0;
|
||||
if (info->dri_prime) {
|
||||
if (strchr(info->dri_prime, ':') == NULL)
|
||||
dri_prime_as_int = atoi(info->dri_prime);
|
||||
|
||||
if (dri_prime_as_int < 0)
|
||||
dri_prime_as_int = 0;
|
||||
}
|
||||
|
||||
struct device_pci_info *pci_infos =
|
||||
(struct device_pci_info *)calloc(physical_device_count, sizeof(struct device_pci_info));
|
||||
if (!pci_infos)
|
||||
return 0;
|
||||
|
||||
for (unsigned i = 0; i < physical_device_count; ++i)
|
||||
cpu_count += fill_drm_device_info(info, &pci_infos[i], pPhysicalDevices[i]) ? 1 : 0;
|
||||
|
||||
default_idx = get_selected(info, physical_device_count, pci_infos,
|
||||
dri_prime_as_int ? NULL : info->dri_prime, expose_only_one_dev);
|
||||
if (default_idx == -1)
|
||||
default_idx = get_default(info, physical_device_count, pci_infos);
|
||||
|
||||
/* DRI_PRIME=n handling - pick any other device than default. */
|
||||
if (dri_prime_as_int > 0 && info->debug)
|
||||
fprintf(stderr, "device-select: DRI_PRIME=%d, default_idx so far: %i\n", dri_prime_as_int,
|
||||
default_idx);
|
||||
if (dri_prime_as_int > 0 && (physical_device_count - cpu_count) > 1 &&
|
||||
(default_idx == 0 || default_idx == 1)) {
|
||||
default_idx =
|
||||
find_non_cpu_skip(pci_infos, physical_device_count, default_idx, dri_prime_as_int);
|
||||
if (default_idx != -1) {
|
||||
if (info->debug)
|
||||
fprintf(stderr, "device-select: find_non_cpu_skip selected %i\n", default_idx);
|
||||
*expose_only_one_dev = ends_with_exclamation_mark(info->dri_prime);
|
||||
}
|
||||
}
|
||||
|
||||
free(pci_infos);
|
||||
return default_idx == -1 ? 0 : default_idx;
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
/*
|
||||
* Copyright © 2019 Red Hat
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
#ifndef DEVICE_SELECT_H
|
||||
#define DEVICE_SELECT_H
|
||||
|
||||
#include <vulkan/vulkan.h>
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
#include "xf86drm.h"
|
||||
|
||||
struct instance_info {
|
||||
PFN_vkDestroyInstance DestroyInstance;
|
||||
PFN_vkEnumeratePhysicalDevices EnumeratePhysicalDevices;
|
||||
PFN_vkEnumeratePhysicalDeviceGroups EnumeratePhysicalDeviceGroups;
|
||||
PFN_vkGetInstanceProcAddr GetInstanceProcAddr;
|
||||
PFN_vkEnumerateDeviceExtensionProperties EnumerateDeviceExtensionProperties;
|
||||
PFN_vkGetPhysicalDeviceProperties GetPhysicalDeviceProperties;
|
||||
PFN_vkGetPhysicalDeviceProperties2 GetPhysicalDeviceProperties2;
|
||||
bool has_pci_bus, has_vulkan11;
|
||||
bool has_wayland, has_xcb;
|
||||
bool xserver;
|
||||
|
||||
/* don't do device selection */
|
||||
bool bypass_device_select;
|
||||
bool debug;
|
||||
|
||||
char *selection;
|
||||
char *dri_prime;
|
||||
bool force_default_device;
|
||||
};
|
||||
|
||||
/* We don't use `drmPciDeviceInfo` because it uses 16-bit ids,
|
||||
* instead of Vulkan's 32-bit ones. */
|
||||
struct device_info {
|
||||
uint32_t vendor_id;
|
||||
uint32_t device_id;
|
||||
};
|
||||
|
||||
struct device_pci_info {
|
||||
struct device_info dev_info;
|
||||
drmPciBusInfo bus_info;
|
||||
bool has_bus_info;
|
||||
VkPhysicalDeviceType device_type;
|
||||
};
|
||||
|
||||
#ifdef VK_USE_PLATFORM_XCB_KHR
|
||||
int device_select_find_xcb_pci_default(struct device_pci_info *devices, uint32_t device_count);
|
||||
#else
|
||||
static inline int
|
||||
device_select_find_xcb_pci_default(struct device_pci_info *devices, uint32_t device_count)
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef VK_USE_PLATFORM_WAYLAND_KHR
|
||||
int device_select_find_wayland_pci_default(struct device_pci_info *devices, uint32_t device_count);
|
||||
#else
|
||||
static inline int
|
||||
device_select_find_wayland_pci_default(struct device_pci_info *devices, uint32_t device_count)
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
#endif
|
||||
|
||||
void device_select_get_properties(const struct instance_info *info, VkPhysicalDevice device,
|
||||
VkPhysicalDeviceProperties2 *properties);
|
||||
|
||||
uint32_t device_select_get_first(const struct instance_info *info, uint32_t physical_device_count,
|
||||
VkPhysicalDevice *pPhysicalDevices, bool *expose_only_one_dev);
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,463 @@
|
||||
/*
|
||||
* Copyright © 2017 Google
|
||||
* Copyright © 2019 Red Hat
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
/* Rules for device selection.
|
||||
* Is there an X or wayland connection open (or DISPLAY set).
|
||||
* If no - try and find which device was the boot_vga device.
|
||||
* If yes - try and work out which device is the connection primary,
|
||||
* DRI_PRIME tagged overrides only work if bus info, =1 will just pick an alternate.
|
||||
*/
|
||||
|
||||
#include <vulkan/vk_layer.h>
|
||||
#include <vulkan/vulkan.h>
|
||||
|
||||
#include <assert.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "util/hash_table.h"
|
||||
#include "util/simple_mtx.h"
|
||||
#include "util/u_debug.h"
|
||||
#include "device_select.h"
|
||||
#include "vk_util.h"
|
||||
|
||||
static struct hash_table *device_select_instance_ht = NULL;
|
||||
static simple_mtx_t device_select_mutex = SIMPLE_MTX_INITIALIZER;
|
||||
|
||||
static void
|
||||
device_select_layer_add_instance(VkInstance instance, struct instance_info *info)
|
||||
{
|
||||
simple_mtx_lock(&device_select_mutex);
|
||||
|
||||
if (!device_select_instance_ht)
|
||||
device_select_instance_ht = _mesa_pointer_hash_table_create(NULL);
|
||||
_mesa_hash_table_insert(device_select_instance_ht, instance, info);
|
||||
|
||||
simple_mtx_unlock(&device_select_mutex);
|
||||
}
|
||||
|
||||
static struct instance_info *
|
||||
device_select_layer_get_instance(VkInstance instance)
|
||||
{
|
||||
struct hash_entry *entry;
|
||||
struct instance_info *info = NULL;
|
||||
simple_mtx_lock(&device_select_mutex);
|
||||
entry = _mesa_hash_table_search(device_select_instance_ht, (void *)instance);
|
||||
if (entry)
|
||||
info = (struct instance_info *)entry->data;
|
||||
simple_mtx_unlock(&device_select_mutex);
|
||||
return info;
|
||||
}
|
||||
|
||||
static void
|
||||
device_select_layer_remove_instance(VkInstance instance)
|
||||
{
|
||||
simple_mtx_lock(&device_select_mutex);
|
||||
|
||||
_mesa_hash_table_remove_key(device_select_instance_ht, instance);
|
||||
|
||||
if (_mesa_hash_table_num_entries(device_select_instance_ht) == 0) {
|
||||
_mesa_hash_table_destroy(device_select_instance_ht, NULL);
|
||||
device_select_instance_ht = NULL;
|
||||
}
|
||||
|
||||
simple_mtx_unlock(&device_select_mutex);
|
||||
}
|
||||
|
||||
static VkResult
|
||||
device_select_CreateInstance(const VkInstanceCreateInfo *pCreateInfo,
|
||||
const VkAllocationCallbacks *pAllocator, VkInstance *pInstance)
|
||||
{
|
||||
VkLayerInstanceCreateInfo *chain_info = NULL;
|
||||
bool bypass_device_select = false;
|
||||
vk_foreach_struct_const(s, pCreateInfo->pNext) {
|
||||
switch (s->sType) {
|
||||
case VK_STRUCTURE_TYPE_LOADER_INSTANCE_CREATE_INFO: {
|
||||
const VkLayerInstanceCreateInfo *this_info = (const void *)s;
|
||||
if (this_info->function == VK_LAYER_LINK_INFO)
|
||||
chain_info = (VkLayerInstanceCreateInfo *)this_info; /* loses const */
|
||||
break;
|
||||
}
|
||||
case VK_STRUCTURE_TYPE_LAYER_SETTINGS_CREATE_INFO_EXT: {
|
||||
const VkLayerSettingsCreateInfoEXT *lsci = (const void *)s;
|
||||
for (unsigned i = 0; i < lsci->settingCount; i++) {
|
||||
const VkLayerSettingEXT *ls = &lsci->pSettings[i];
|
||||
if (!strcmp(ls->pLayerName, "MESA_device_select")) {
|
||||
if (!strcmp(ls->pSettingName, "no_device_select")) {
|
||||
assert(ls->type == VK_LAYER_SETTING_TYPE_BOOL32_EXT);
|
||||
uint32_t *values = (uint32_t *)ls->pValues;
|
||||
bypass_device_select = values[0];
|
||||
}
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
assert(chain_info->u.pLayerInfo);
|
||||
|
||||
PFN_vkGetInstanceProcAddr GetInstanceProcAddr =
|
||||
chain_info->u.pLayerInfo->pfnNextGetInstanceProcAddr;
|
||||
PFN_vkCreateInstance fpCreateInstance =
|
||||
(PFN_vkCreateInstance)GetInstanceProcAddr(NULL, "vkCreateInstance");
|
||||
if (fpCreateInstance == NULL) {
|
||||
return VK_ERROR_INITIALIZATION_FAILED;
|
||||
}
|
||||
|
||||
chain_info->u.pLayerInfo = chain_info->u.pLayerInfo->pNext;
|
||||
|
||||
const char *engineName =
|
||||
pCreateInfo->pApplicationInfo && pCreateInfo->pApplicationInfo->pEngineName
|
||||
? pCreateInfo->pApplicationInfo->pEngineName
|
||||
: "";
|
||||
const char *applicationName =
|
||||
pCreateInfo->pApplicationInfo && pCreateInfo->pApplicationInfo->pApplicationName
|
||||
? pCreateInfo->pApplicationInfo->pApplicationName
|
||||
: "";
|
||||
VkResult result = fpCreateInstance(pCreateInfo, pAllocator, pInstance);
|
||||
if (result != VK_SUCCESS) {
|
||||
return result;
|
||||
}
|
||||
|
||||
bool zink = !strcmp(engineName, "mesa zink");
|
||||
bool xwayland = !strcmp(applicationName, "Xwayland");
|
||||
struct instance_info *info = (struct instance_info *)calloc(1, sizeof(struct instance_info));
|
||||
info->GetInstanceProcAddr = GetInstanceProcAddr;
|
||||
info->xserver = !strcmp(applicationName, "Xorg") || !strcmp(applicationName, "Xephyr");
|
||||
|
||||
#ifdef VK_USE_PLATFORM_WAYLAND_KHR
|
||||
bool has_wayland = os_get_option("WAYLAND_DISPLAY") || os_get_option("WAYLAND_SOCKET");
|
||||
#endif
|
||||
#ifdef VK_USE_PLATFORM_XCB_KHR
|
||||
bool has_xcb = !!os_get_option("DISPLAY");
|
||||
#endif
|
||||
|
||||
for (unsigned i = 0; i < pCreateInfo->enabledExtensionCount; i++) {
|
||||
#ifdef VK_USE_PLATFORM_WAYLAND_KHR
|
||||
if (!strcmp(pCreateInfo->ppEnabledExtensionNames[i], VK_KHR_WAYLAND_SURFACE_EXTENSION_NAME) &&
|
||||
has_wayland)
|
||||
info->has_wayland = true;
|
||||
#endif
|
||||
#ifdef VK_USE_PLATFORM_XCB_KHR
|
||||
if (!strcmp(pCreateInfo->ppEnabledExtensionNames[i], VK_KHR_XCB_SURFACE_EXTENSION_NAME) &&
|
||||
has_xcb)
|
||||
info->has_xcb = !info->xserver || !zink;
|
||||
#endif
|
||||
}
|
||||
|
||||
if (zink && xwayland)
|
||||
bypass_device_select = true;
|
||||
/*
|
||||
* The loader is currently not able to handle GetPhysicalDeviceProperties2KHR calls in
|
||||
* EnumeratePhysicalDevices when there are other layers present. To avoid mysterious crashes
|
||||
* for users just use only the vulkan version for now.
|
||||
*/
|
||||
info->bypass_device_select = bypass_device_select;
|
||||
info->has_vulkan11 = pCreateInfo->pApplicationInfo &&
|
||||
pCreateInfo->pApplicationInfo->apiVersion >= VK_MAKE_VERSION(1, 1, 0);
|
||||
|
||||
info->debug = debug_get_bool_option("MESA_VK_DEVICE_SELECT_DEBUG", false) ||
|
||||
debug_get_bool_option("DRI_PRIME_DEBUG", false);
|
||||
info->selection = os_get_option_dup("MESA_VK_DEVICE_SELECT");
|
||||
info->dri_prime = os_get_option_dup("DRI_PRIME");
|
||||
info->force_default_device =
|
||||
debug_get_bool_option("MESA_VK_DEVICE_SELECT_FORCE_DEFAULT_DEVICE", false);
|
||||
|
||||
#define DEVSEL_GET_CB(func) \
|
||||
info->func = (PFN_vk##func)info->GetInstanceProcAddr(*pInstance, "vk" #func)
|
||||
DEVSEL_GET_CB(DestroyInstance);
|
||||
DEVSEL_GET_CB(EnumeratePhysicalDevices);
|
||||
DEVSEL_GET_CB(EnumeratePhysicalDeviceGroups);
|
||||
DEVSEL_GET_CB(GetPhysicalDeviceProperties);
|
||||
DEVSEL_GET_CB(EnumerateDeviceExtensionProperties);
|
||||
if (info->has_vulkan11)
|
||||
DEVSEL_GET_CB(GetPhysicalDeviceProperties2);
|
||||
#undef DEVSEL_GET_CB
|
||||
|
||||
device_select_layer_add_instance(*pInstance, info);
|
||||
|
||||
return VK_SUCCESS;
|
||||
}
|
||||
|
||||
static void
|
||||
device_select_DestroyInstance(VkInstance instance, const VkAllocationCallbacks *pAllocator)
|
||||
{
|
||||
struct instance_info *info = device_select_layer_get_instance(instance);
|
||||
|
||||
device_select_layer_remove_instance(instance);
|
||||
info->DestroyInstance(instance, pAllocator);
|
||||
free(info->dri_prime);
|
||||
free(info->selection);
|
||||
free(info);
|
||||
}
|
||||
|
||||
void
|
||||
device_select_get_properties(const struct instance_info *info, VkPhysicalDevice device,
|
||||
VkPhysicalDeviceProperties2 *properties)
|
||||
{
|
||||
if (info->GetPhysicalDeviceProperties2 && info->has_vulkan11)
|
||||
info->GetPhysicalDeviceProperties2(device, properties);
|
||||
else
|
||||
info->GetPhysicalDeviceProperties(device, &properties->properties);
|
||||
}
|
||||
|
||||
static void
|
||||
print_gpu(const struct instance_info *info, unsigned index, VkPhysicalDevice device)
|
||||
{
|
||||
const char *type = "";
|
||||
VkPhysicalDevicePCIBusInfoPropertiesEXT ext_pci_properties =
|
||||
(VkPhysicalDevicePCIBusInfoPropertiesEXT){
|
||||
.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PCI_BUS_INFO_PROPERTIES_EXT,
|
||||
};
|
||||
VkPhysicalDeviceProperties2 properties = (VkPhysicalDeviceProperties2){
|
||||
.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2,
|
||||
};
|
||||
if (info->has_vulkan11 && info->has_pci_bus)
|
||||
properties.pNext = &ext_pci_properties;
|
||||
device_select_get_properties(info, device, &properties);
|
||||
|
||||
switch (properties.properties.deviceType) {
|
||||
case VK_PHYSICAL_DEVICE_TYPE_OTHER:
|
||||
default:
|
||||
type = "other";
|
||||
break;
|
||||
case VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU:
|
||||
type = "integrated GPU";
|
||||
break;
|
||||
case VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU:
|
||||
type = "discrete GPU";
|
||||
break;
|
||||
case VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU:
|
||||
type = "virtual GPU";
|
||||
break;
|
||||
case VK_PHYSICAL_DEVICE_TYPE_CPU:
|
||||
type = "CPU";
|
||||
break;
|
||||
}
|
||||
fprintf(stderr, " GPU %d: %x:%x \"%s\" %s", index, properties.properties.vendorID,
|
||||
properties.properties.deviceID, properties.properties.deviceName, type);
|
||||
if (info->has_vulkan11 && info->has_pci_bus)
|
||||
fprintf(stderr, " %04x:%02x:%02x.%x", ext_pci_properties.pciDomain, ext_pci_properties.pciBus,
|
||||
ext_pci_properties.pciDevice, ext_pci_properties.pciFunction);
|
||||
fprintf(stderr, "\n");
|
||||
}
|
||||
|
||||
static VkResult
|
||||
device_select_EnumeratePhysicalDevices(VkInstance instance, uint32_t *pPhysicalDeviceCount,
|
||||
VkPhysicalDevice *pPhysicalDevices)
|
||||
{
|
||||
struct instance_info *info = device_select_layer_get_instance(instance);
|
||||
uint32_t physical_device_count = 0;
|
||||
uint32_t selected_physical_device_count = 0;
|
||||
bool expose_only_one_dev = false;
|
||||
if (info->bypass_device_select)
|
||||
return info->EnumeratePhysicalDevices(instance, pPhysicalDeviceCount, pPhysicalDevices);
|
||||
VkResult result = info->EnumeratePhysicalDevices(instance, &physical_device_count, NULL);
|
||||
VK_OUTARRAY_MAKE_TYPED(VkPhysicalDevice, out, pPhysicalDevices, pPhysicalDeviceCount);
|
||||
if (result != VK_SUCCESS)
|
||||
return result;
|
||||
|
||||
VkPhysicalDevice *physical_devices =
|
||||
(VkPhysicalDevice *)calloc(sizeof(VkPhysicalDevice), physical_device_count);
|
||||
VkPhysicalDevice *selected_physical_devices =
|
||||
(VkPhysicalDevice *)calloc(sizeof(VkPhysicalDevice), physical_device_count);
|
||||
|
||||
if (!physical_devices || !selected_physical_devices) {
|
||||
result = VK_ERROR_OUT_OF_HOST_MEMORY;
|
||||
goto out;
|
||||
}
|
||||
|
||||
result = info->EnumeratePhysicalDevices(instance, &physical_device_count, physical_devices);
|
||||
if (result != VK_SUCCESS)
|
||||
goto out;
|
||||
|
||||
for (unsigned i = 0; i < physical_device_count; i++) {
|
||||
uint32_t count;
|
||||
info->EnumerateDeviceExtensionProperties(physical_devices[i], NULL, &count, NULL);
|
||||
if (count > 0) {
|
||||
VkExtensionProperties *extensions = calloc(count, sizeof(VkExtensionProperties));
|
||||
if (info->EnumerateDeviceExtensionProperties(physical_devices[i], NULL, &count,
|
||||
extensions) == VK_SUCCESS) {
|
||||
for (unsigned j = 0; j < count; j++) {
|
||||
if (!strcmp(extensions[j].extensionName, VK_EXT_PCI_BUS_INFO_EXTENSION_NAME))
|
||||
info->has_pci_bus = true;
|
||||
}
|
||||
}
|
||||
free(extensions);
|
||||
}
|
||||
}
|
||||
if (info->debug || (info->selection && strcmp(info->selection, "list") == 0)) {
|
||||
fprintf(stderr, "selectable devices:\n");
|
||||
for (unsigned i = 0; i < physical_device_count; ++i)
|
||||
print_gpu(info, i, physical_devices[i]);
|
||||
|
||||
if (info->selection && strcmp(info->selection, "list") == 0)
|
||||
exit(0);
|
||||
}
|
||||
|
||||
unsigned selected_index =
|
||||
device_select_get_first(info, physical_device_count, physical_devices, &expose_only_one_dev);
|
||||
selected_physical_device_count = physical_device_count;
|
||||
selected_physical_devices[0] = physical_devices[selected_index];
|
||||
for (unsigned i = 0; i < physical_device_count - 1; ++i) {
|
||||
unsigned this_idx = i < selected_index ? i : i + 1;
|
||||
selected_physical_devices[i + 1] = physical_devices[this_idx];
|
||||
}
|
||||
|
||||
if (selected_physical_device_count == 0) {
|
||||
fprintf(stderr, "WARNING: selected no devices with MESA_VK_DEVICE_SELECT\n");
|
||||
}
|
||||
|
||||
assert(result == VK_SUCCESS);
|
||||
|
||||
/* do not give multiple device option to app if force default device */
|
||||
if (info->force_default_device && selected_physical_device_count != 0)
|
||||
expose_only_one_dev = true;
|
||||
|
||||
if (expose_only_one_dev)
|
||||
selected_physical_device_count = 1;
|
||||
|
||||
for (unsigned i = 0; i < selected_physical_device_count; i++) {
|
||||
vk_outarray_append_typed(VkPhysicalDevice, &out, ent)
|
||||
{
|
||||
*ent = selected_physical_devices[i];
|
||||
}
|
||||
}
|
||||
result = vk_outarray_status(&out);
|
||||
out:
|
||||
free(physical_devices);
|
||||
free(selected_physical_devices);
|
||||
return result;
|
||||
}
|
||||
|
||||
static VkResult
|
||||
device_select_EnumeratePhysicalDeviceGroups(VkInstance instance,
|
||||
uint32_t *pPhysicalDeviceGroupCount,
|
||||
VkPhysicalDeviceGroupProperties *pPhysicalDeviceGroups)
|
||||
{
|
||||
struct instance_info *info = device_select_layer_get_instance(instance);
|
||||
uint32_t physical_device_group_count = 0;
|
||||
uint32_t selected_physical_device_group_count = 0;
|
||||
if (info->bypass_device_select)
|
||||
return info->EnumeratePhysicalDeviceGroups(instance, pPhysicalDeviceGroupCount,
|
||||
pPhysicalDeviceGroups);
|
||||
VkResult result =
|
||||
info->EnumeratePhysicalDeviceGroups(instance, &physical_device_group_count, NULL);
|
||||
VK_OUTARRAY_MAKE_TYPED(VkPhysicalDeviceGroupProperties, out, pPhysicalDeviceGroups,
|
||||
pPhysicalDeviceGroupCount);
|
||||
|
||||
if (result != VK_SUCCESS)
|
||||
return result;
|
||||
|
||||
VkPhysicalDeviceGroupProperties *physical_device_groups =
|
||||
(VkPhysicalDeviceGroupProperties *)calloc(sizeof(VkPhysicalDeviceGroupProperties),
|
||||
physical_device_group_count);
|
||||
VkPhysicalDeviceGroupProperties *selected_physical_device_groups =
|
||||
(VkPhysicalDeviceGroupProperties *)calloc(sizeof(VkPhysicalDeviceGroupProperties),
|
||||
physical_device_group_count);
|
||||
|
||||
if (!physical_device_groups || !selected_physical_device_groups) {
|
||||
result = VK_ERROR_OUT_OF_HOST_MEMORY;
|
||||
goto out;
|
||||
}
|
||||
|
||||
for (unsigned i = 0; i < physical_device_group_count; i++)
|
||||
physical_device_groups[i].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES;
|
||||
|
||||
result = info->EnumeratePhysicalDeviceGroups(instance, &physical_device_group_count,
|
||||
physical_device_groups);
|
||||
if (result != VK_SUCCESS)
|
||||
goto out;
|
||||
|
||||
/* just sort groups with CPU devices to the end? - assume nobody will mix these */
|
||||
int num_gpu_groups = 0;
|
||||
int num_cpu_groups = 0;
|
||||
selected_physical_device_group_count = physical_device_group_count;
|
||||
for (unsigned i = 0; i < physical_device_group_count; i++) {
|
||||
bool group_has_cpu_device = false;
|
||||
for (unsigned j = 0; j < physical_device_groups[i].physicalDeviceCount; j++) {
|
||||
VkPhysicalDevice physical_device = physical_device_groups[i].physicalDevices[j];
|
||||
VkPhysicalDeviceProperties2 properties = (VkPhysicalDeviceProperties2){
|
||||
.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2,
|
||||
};
|
||||
info->GetPhysicalDeviceProperties(physical_device, &properties.properties);
|
||||
group_has_cpu_device = properties.properties.deviceType == VK_PHYSICAL_DEVICE_TYPE_CPU;
|
||||
}
|
||||
|
||||
if (group_has_cpu_device) {
|
||||
selected_physical_device_groups[physical_device_group_count - num_cpu_groups - 1] =
|
||||
physical_device_groups[i];
|
||||
num_cpu_groups++;
|
||||
} else {
|
||||
selected_physical_device_groups[num_gpu_groups] = physical_device_groups[i];
|
||||
num_gpu_groups++;
|
||||
}
|
||||
}
|
||||
|
||||
assert(result == VK_SUCCESS);
|
||||
|
||||
for (unsigned i = 0; i < selected_physical_device_group_count; i++) {
|
||||
vk_outarray_append_typed(VkPhysicalDeviceGroupProperties, &out, ent)
|
||||
{
|
||||
*ent = selected_physical_device_groups[i];
|
||||
}
|
||||
}
|
||||
result = vk_outarray_status(&out);
|
||||
out:
|
||||
free(physical_device_groups);
|
||||
free(selected_physical_device_groups);
|
||||
return result;
|
||||
}
|
||||
|
||||
static void (*get_instance_proc_addr(VkInstance instance, const char *name))()
|
||||
{
|
||||
if (strcmp(name, "vkGetInstanceProcAddr") == 0)
|
||||
return (void (*)())get_instance_proc_addr;
|
||||
if (strcmp(name, "vkCreateInstance") == 0)
|
||||
return (void (*)())device_select_CreateInstance;
|
||||
if (strcmp(name, "vkDestroyInstance") == 0)
|
||||
return (void (*)())device_select_DestroyInstance;
|
||||
if (strcmp(name, "vkEnumeratePhysicalDevices") == 0)
|
||||
return (void (*)())device_select_EnumeratePhysicalDevices;
|
||||
if (strcmp(name, "vkEnumeratePhysicalDeviceGroups") == 0)
|
||||
return (void (*)())device_select_EnumeratePhysicalDeviceGroups;
|
||||
|
||||
struct instance_info *info = device_select_layer_get_instance(instance);
|
||||
return info->GetInstanceProcAddr(instance, name);
|
||||
}
|
||||
|
||||
PUBLIC VkResult
|
||||
vkNegotiateLoaderLayerInterfaceVersion(VkNegotiateLayerInterface *pVersionStruct)
|
||||
{
|
||||
if (pVersionStruct->loaderLayerInterfaceVersion < 2)
|
||||
return VK_ERROR_INITIALIZATION_FAILED;
|
||||
pVersionStruct->loaderLayerInterfaceVersion = 2;
|
||||
|
||||
pVersionStruct->pfnGetInstanceProcAddr = get_instance_proc_addr;
|
||||
|
||||
return VK_SUCCESS;
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
/*
|
||||
* Copyright © 2019 Red Hat
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
#include "util/macros.h"
|
||||
#include <wayland-client.h>
|
||||
#ifdef HAVE_BIND_WL_DISPLAY
|
||||
#include "wayland-drm-client-protocol.h"
|
||||
#endif
|
||||
#include <fcntl.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
#include <xf86drm.h>
|
||||
#include "device_select.h"
|
||||
#include "linux-dmabuf-unstable-v1-client-protocol.h"
|
||||
|
||||
struct device_select_wayland_info {
|
||||
#ifdef HAVE_BIND_WL_DISPLAY
|
||||
struct wl_drm *wl_drm;
|
||||
drmDevicePtr drm_dev_info;
|
||||
#endif
|
||||
|
||||
struct zwp_linux_dmabuf_v1 *wl_dmabuf;
|
||||
struct zwp_linux_dmabuf_feedback_v1 *wl_dmabuf_feedback;
|
||||
struct wl_fixes *wl_fixes;
|
||||
drmDevicePtr dmabuf_dev_info;
|
||||
};
|
||||
|
||||
#ifdef HAVE_BIND_WL_DISPLAY
|
||||
static void
|
||||
device_select_drm_handle_device(void *data, struct wl_drm *drm, const char *device)
|
||||
{
|
||||
struct device_select_wayland_info *info = data;
|
||||
|
||||
int fd = open(device, O_RDWR | O_CLOEXEC);
|
||||
if (fd == -1)
|
||||
return;
|
||||
|
||||
drmGetDevice2(fd, 0, &info->drm_dev_info);
|
||||
close(fd);
|
||||
}
|
||||
|
||||
static void
|
||||
device_select_drm_handle_format(void *data, struct wl_drm *drm, uint32_t format)
|
||||
{
|
||||
/* ignore this event */
|
||||
}
|
||||
|
||||
static void
|
||||
device_select_drm_handle_authenticated(void *data, struct wl_drm *drm)
|
||||
{
|
||||
/* ignore this event */
|
||||
}
|
||||
|
||||
static void
|
||||
device_select_drm_handle_capabilities(void *data, struct wl_drm *drm, uint32_t value)
|
||||
{
|
||||
/* ignore this event */
|
||||
}
|
||||
|
||||
static const struct wl_drm_listener ds_drm_listener = {
|
||||
.device = device_select_drm_handle_device,
|
||||
.format = device_select_drm_handle_format,
|
||||
.authenticated = device_select_drm_handle_authenticated,
|
||||
.capabilities = device_select_drm_handle_capabilities,
|
||||
};
|
||||
#endif
|
||||
|
||||
static void
|
||||
default_dmabuf_feedback_format_table(
|
||||
void *data, struct zwp_linux_dmabuf_feedback_v1 *zwp_linux_dmabuf_feedback_v1, int32_t fd,
|
||||
uint32_t size)
|
||||
{
|
||||
|
||||
/* ignore this event */
|
||||
close(fd);
|
||||
}
|
||||
|
||||
static void
|
||||
default_dmabuf_feedback_main_device(void *data,
|
||||
struct zwp_linux_dmabuf_feedback_v1 *dmabuf_feedback,
|
||||
struct wl_array *device)
|
||||
{
|
||||
|
||||
struct device_select_wayland_info *info = data;
|
||||
|
||||
dev_t dev_id;
|
||||
assert(device->size == sizeof(dev_id));
|
||||
memcpy(&dev_id, device->data, device->size);
|
||||
|
||||
drmGetDeviceFromDevId(dev_id, 0, &info->dmabuf_dev_info);
|
||||
return;
|
||||
}
|
||||
|
||||
static void
|
||||
default_dmabuf_feedback_tranche_target_device(void *data,
|
||||
struct zwp_linux_dmabuf_feedback_v1 *dmabuf_feedback,
|
||||
struct wl_array *device)
|
||||
{
|
||||
/* ignore this event */
|
||||
}
|
||||
|
||||
static void
|
||||
default_dmabuf_feedback_tranche_flags(void *data,
|
||||
struct zwp_linux_dmabuf_feedback_v1 *dmabuf_feedback,
|
||||
uint32_t flags)
|
||||
{
|
||||
/* ignore this event */
|
||||
}
|
||||
|
||||
static void
|
||||
default_dmabuf_feedback_tranche_formats(void *data,
|
||||
struct zwp_linux_dmabuf_feedback_v1 *dmabuf_feedback,
|
||||
struct wl_array *indices)
|
||||
{
|
||||
/* ignore this event */
|
||||
}
|
||||
|
||||
static void
|
||||
default_dmabuf_feedback_tranche_done(void *data,
|
||||
struct zwp_linux_dmabuf_feedback_v1 *dmabuf_feedback)
|
||||
{
|
||||
/* ignore this event */
|
||||
}
|
||||
|
||||
static void
|
||||
default_dmabuf_feedback_done(void *data, struct zwp_linux_dmabuf_feedback_v1 *dmabuf_feedback)
|
||||
{
|
||||
/* ignore this event */
|
||||
}
|
||||
|
||||
static const struct zwp_linux_dmabuf_feedback_v1_listener dmabuf_feedback_listener = {
|
||||
.format_table = default_dmabuf_feedback_format_table,
|
||||
.main_device = default_dmabuf_feedback_main_device,
|
||||
.tranche_target_device = default_dmabuf_feedback_tranche_target_device,
|
||||
.tranche_flags = default_dmabuf_feedback_tranche_flags,
|
||||
.tranche_formats = default_dmabuf_feedback_tranche_formats,
|
||||
.tranche_done = default_dmabuf_feedback_tranche_done,
|
||||
.done = default_dmabuf_feedback_done,
|
||||
};
|
||||
|
||||
static void
|
||||
device_select_registry_global(void *data, struct wl_registry *registry, uint32_t name,
|
||||
const char *interface, uint32_t version)
|
||||
{
|
||||
struct device_select_wayland_info *info = data;
|
||||
#if HAVE_BIND_WL_DISPLAY
|
||||
if (strcmp(interface, wl_drm_interface.name) == 0) {
|
||||
info->wl_drm = wl_registry_bind(registry, name, &wl_drm_interface, MIN2(version, 2));
|
||||
wl_drm_add_listener(info->wl_drm, &ds_drm_listener, data);
|
||||
} else
|
||||
#endif
|
||||
if (strcmp(interface, zwp_linux_dmabuf_v1_interface.name) == 0 &&
|
||||
version >= ZWP_LINUX_DMABUF_V1_GET_DEFAULT_FEEDBACK_SINCE_VERSION) {
|
||||
info->wl_dmabuf = wl_registry_bind(registry, name, &zwp_linux_dmabuf_v1_interface,
|
||||
ZWP_LINUX_DMABUF_V1_GET_DEFAULT_FEEDBACK_SINCE_VERSION);
|
||||
info->wl_dmabuf_feedback = zwp_linux_dmabuf_v1_get_default_feedback(info->wl_dmabuf);
|
||||
zwp_linux_dmabuf_feedback_v1_add_listener(info->wl_dmabuf_feedback, &dmabuf_feedback_listener,
|
||||
data);
|
||||
#ifdef WL_FIXES_INTERFACE
|
||||
} else if (strcmp(interface, wl_fixes_interface.name) == 0) {
|
||||
info->wl_fixes = wl_registry_bind(registry, name, &wl_fixes_interface, 1);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
static void
|
||||
device_select_registry_global_remove_cb(void *data, struct wl_registry *registry, uint32_t name)
|
||||
{
|
||||
}
|
||||
|
||||
int
|
||||
device_select_find_wayland_pci_default(struct device_pci_info *devices, uint32_t device_count)
|
||||
{
|
||||
struct wl_display *display;
|
||||
struct wl_registry *registry = NULL;
|
||||
unsigned default_idx = -1;
|
||||
struct device_select_wayland_info info = {};
|
||||
|
||||
display = wl_display_connect(NULL);
|
||||
if (!display)
|
||||
return -1;
|
||||
|
||||
registry = wl_display_get_registry(display);
|
||||
if (!registry) {
|
||||
wl_display_disconnect(display);
|
||||
return -1;
|
||||
}
|
||||
|
||||
static const struct wl_registry_listener registry_listener = {
|
||||
device_select_registry_global, device_select_registry_global_remove_cb};
|
||||
|
||||
wl_registry_add_listener(registry, ®istry_listener, &info);
|
||||
wl_display_dispatch(display);
|
||||
wl_display_roundtrip(display);
|
||||
|
||||
drmDevicePtr target = NULL;
|
||||
if (info.dmabuf_dev_info != NULL) {
|
||||
target = info.dmabuf_dev_info;
|
||||
}
|
||||
#ifdef HAVE_BIND_WL_DISPLAY
|
||||
if (target == NULL && info.drm_dev_info != NULL) {
|
||||
target = info.drm_dev_info;
|
||||
}
|
||||
#endif
|
||||
if (target == NULL) {
|
||||
goto done;
|
||||
}
|
||||
|
||||
for (unsigned i = 0; i < device_count; i++) {
|
||||
if (devices[i].has_bus_info) {
|
||||
if (target->businfo.pci->domain == devices[i].bus_info.domain &&
|
||||
target->businfo.pci->bus == devices[i].bus_info.bus &&
|
||||
target->businfo.pci->dev == devices[i].bus_info.dev &&
|
||||
target->businfo.pci->func == devices[i].bus_info.func) {
|
||||
default_idx = i;
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
if (target->deviceinfo.pci->vendor_id == devices[i].dev_info.vendor_id &&
|
||||
target->deviceinfo.pci->device_id == devices[i].dev_info.device_id) {
|
||||
default_idx = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
done:
|
||||
if (info.dmabuf_dev_info != NULL)
|
||||
drmFreeDevice(&info.dmabuf_dev_info);
|
||||
#ifdef HAVE_BIND_WL_DISPLAY
|
||||
if (info.drm_dev_info != NULL)
|
||||
drmFreeDevice(&info.drm_dev_info);
|
||||
#endif
|
||||
|
||||
if (info.wl_dmabuf_feedback)
|
||||
zwp_linux_dmabuf_feedback_v1_destroy(info.wl_dmabuf_feedback);
|
||||
if (info.wl_dmabuf)
|
||||
zwp_linux_dmabuf_v1_destroy(info.wl_dmabuf);
|
||||
#ifdef HAVE_BIND_WL_DISPLAY
|
||||
if (info.wl_drm)
|
||||
wl_drm_destroy(info.wl_drm);
|
||||
#endif
|
||||
#ifdef WL_FIXES_INTERFACE
|
||||
if (info.wl_fixes) {
|
||||
wl_fixes_destroy_registry(info.wl_fixes, registry);
|
||||
wl_fixes_destroy(info.wl_fixes);
|
||||
}
|
||||
#endif
|
||||
|
||||
wl_registry_destroy(registry);
|
||||
wl_display_disconnect(display);
|
||||
return default_idx;
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
/*
|
||||
* Copyright © 2019 Red Hat
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
/* connect to an X server and work out the default device. */
|
||||
|
||||
#include <fcntl.h>
|
||||
#include <stdlib.h>
|
||||
#include <unistd.h>
|
||||
#include <xf86drm.h>
|
||||
#include <xcb/dri3.h>
|
||||
#include <xcb/xcb.h>
|
||||
|
||||
#include "device_select.h"
|
||||
static int
|
||||
ds_dri3_open(xcb_connection_t *conn, xcb_window_t root, uint32_t provider)
|
||||
{
|
||||
xcb_dri3_open_cookie_t cookie;
|
||||
xcb_dri3_open_reply_t *reply;
|
||||
int fd;
|
||||
|
||||
cookie = xcb_dri3_open(conn, root, provider);
|
||||
|
||||
reply = xcb_dri3_open_reply(conn, cookie, NULL);
|
||||
if (!reply)
|
||||
return -1;
|
||||
|
||||
if (reply->nfd != 1) {
|
||||
free(reply);
|
||||
return -1;
|
||||
}
|
||||
|
||||
fd = xcb_dri3_open_reply_fds(conn, reply)[0];
|
||||
free(reply);
|
||||
fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC);
|
||||
|
||||
return fd;
|
||||
}
|
||||
|
||||
int
|
||||
device_select_find_xcb_pci_default(struct device_pci_info *devices, uint32_t device_count)
|
||||
{
|
||||
const xcb_setup_t *setup;
|
||||
xcb_screen_iterator_t iter;
|
||||
int scrn;
|
||||
xcb_connection_t *conn;
|
||||
int default_idx = -1;
|
||||
drmDevicePtr xdev = NULL;
|
||||
|
||||
conn = xcb_connect(NULL, &scrn);
|
||||
if (xcb_connection_has_error(conn)) {
|
||||
xcb_disconnect(conn);
|
||||
return -1;
|
||||
}
|
||||
|
||||
xcb_query_extension_cookie_t dri3_cookie;
|
||||
xcb_query_extension_reply_t *dri3_reply = NULL;
|
||||
|
||||
dri3_cookie = xcb_query_extension(conn, 4, "DRI3");
|
||||
dri3_reply = xcb_query_extension_reply(conn, dri3_cookie, NULL);
|
||||
|
||||
if (!dri3_reply)
|
||||
goto out;
|
||||
|
||||
if (dri3_reply->present == 0)
|
||||
goto out;
|
||||
|
||||
setup = xcb_get_setup(conn);
|
||||
iter = xcb_setup_roots_iterator(setup);
|
||||
|
||||
xcb_screen_t *screen = iter.data;
|
||||
|
||||
int dri3_fd = ds_dri3_open(conn, screen->root, 0);
|
||||
if (dri3_fd == -1)
|
||||
goto out;
|
||||
|
||||
int ret = drmGetDevice2(dri3_fd, 0, &xdev);
|
||||
close(dri3_fd);
|
||||
if (ret < 0)
|
||||
goto out;
|
||||
|
||||
for (unsigned i = 0; i < device_count; i++) {
|
||||
if (devices[i].has_bus_info) {
|
||||
if (xdev->businfo.pci->domain == devices[i].bus_info.domain &&
|
||||
xdev->businfo.pci->bus == devices[i].bus_info.bus &&
|
||||
xdev->businfo.pci->dev == devices[i].bus_info.dev &&
|
||||
xdev->businfo.pci->func == devices[i].bus_info.func) {
|
||||
default_idx = i;
|
||||
}
|
||||
} else {
|
||||
if (xdev->deviceinfo.pci->vendor_id == devices[i].dev_info.vendor_id &&
|
||||
xdev->deviceinfo.pci->device_id == devices[i].dev_info.device_id)
|
||||
default_idx = i;
|
||||
}
|
||||
if (default_idx != -1)
|
||||
break;
|
||||
}
|
||||
|
||||
out:
|
||||
free(dri3_reply);
|
||||
drmFreeDevice(&xdev); /* Is NULL pointer safe. */
|
||||
xcb_disconnect(conn);
|
||||
return default_idx;
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
# Copyright © 2019 Intel Corporation
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
vklayer_files = files(
|
||||
'device_select.c',
|
||||
'device_select_layer.c',
|
||||
)
|
||||
|
||||
vklayer_deps = []
|
||||
vklayer_flags = []
|
||||
|
||||
if with_platform_x11
|
||||
vklayer_files += files('device_select_x11.c')
|
||||
vklayer_deps += dep_xcb_dri3
|
||||
endif
|
||||
|
||||
if with_platform_wayland
|
||||
vklayer_files += files('device_select_wayland.c')
|
||||
if with_wayland_bind_display
|
||||
vklayer_files += [ wayland_drm_client_protocol_h, wayland_drm_protocol_c ]
|
||||
vklayer_flags += [ '-DHAVE_BIND_WL_DISPLAY' ]
|
||||
endif
|
||||
vklayer_files += wp_files['linux-dmabuf-unstable-v1']
|
||||
vklayer_deps += dep_wayland_client
|
||||
endif
|
||||
|
||||
vklayer_mesa_device_select = shared_library(
|
||||
'VkLayer_MESA_device_select',
|
||||
vklayer_files,
|
||||
c_args : [no_override_init_args, vklayer_flags],
|
||||
gnu_symbol_visibility : 'hidden',
|
||||
dependencies : [
|
||||
idep_vulkan_util, idep_vulkan_wsi_headers, idep_mesautil, vklayer_deps,
|
||||
dep_libdrm, dep_dl
|
||||
],
|
||||
include_directories : [inc_include, inc_util, inc_src],
|
||||
link_args : cc.get_supported_link_arguments(['-Wl,-Bsymbolic-functions', '-Wl,-z,relro']),
|
||||
install : true
|
||||
)
|
||||
|
||||
configure_file(
|
||||
configuration : {'library_path' : 'libVkLayer_MESA_device_select.so'},
|
||||
input : 'VkLayer_MESA_device_select.json.in',
|
||||
output : '@BASENAME@',
|
||||
install : true,
|
||||
install_dir : join_paths(get_option('datadir'), 'vulkan', 'implicit_layer.d'),
|
||||
install_tag : 'runtime',
|
||||
)
|
||||
|
||||
configure_file(
|
||||
configuration : {'library_path' : vklayer_mesa_device_select.full_path()},
|
||||
input : 'VkLayer_MESA_device_select.json.in',
|
||||
output : '@PLAINNAME@_dev.json',
|
||||
)
|
||||
devenv.append('VK_ADD_IMPLICIT_LAYER_PATH', meson.current_build_dir())
|
||||
@@ -0,0 +1,103 @@
|
||||
# Copyright © 2017 Intel Corporation
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
vk_api_xml = files('registry/vk.xml')
|
||||
|
||||
vulkan_icd_link_args = []
|
||||
vulkan_icd_link_depends = []
|
||||
|
||||
if with_platform_android
|
||||
vulkan_icd_symbols = files('vulkan-icd-android-symbols.txt')
|
||||
if with_ld_version_script
|
||||
vulkan_icd_link_args += ['-Wl,--version-script', join_paths(meson.current_source_dir(), 'vulkan-android.sym')]
|
||||
vulkan_icd_link_depends += files('vulkan-android.sym')
|
||||
endif
|
||||
else
|
||||
vulkan_icd_symbols = files('vulkan-icd-symbols.txt')
|
||||
if with_ld_version_script
|
||||
vulkan_icd_link_args += ['-Wl,--version-script', join_paths(meson.current_source_dir(), 'vulkan.sym')]
|
||||
vulkan_icd_link_depends += files('vulkan.sym')
|
||||
endif
|
||||
endif
|
||||
|
||||
inc_vulkan_util = include_directories('util')
|
||||
|
||||
vulkan_wsi_deps = []
|
||||
vulkan_wsi_list = []
|
||||
|
||||
vulkan_api_def = custom_target(
|
||||
'vulkan_api.def',
|
||||
input: 'vulkan_api.def.in',
|
||||
output : 'vulkan_api.def',
|
||||
command : gen_vs_module_defs_normal_command,
|
||||
)
|
||||
|
||||
if with_platform_x11
|
||||
vulkan_wsi_deps += [
|
||||
dep_xcb,
|
||||
dep_x11_xcb,
|
||||
dep_xcb_dri3,
|
||||
dep_xcb_present,
|
||||
dep_xcb_xfixes,
|
||||
dep_xcb_sync,
|
||||
dep_xcb_xrandr,
|
||||
dep_xcb_shm,
|
||||
dep_xshmfence,
|
||||
]
|
||||
vulkan_wsi_list += '-DVK_USE_PLATFORM_XCB_KHR'
|
||||
vulkan_wsi_list += '-DVK_USE_PLATFORM_XLIB_KHR'
|
||||
if with_xcb_keysyms
|
||||
vulkan_wsi_deps += dep_xcb_keysyms
|
||||
endif
|
||||
endif
|
||||
if with_platform_wayland
|
||||
vulkan_wsi_deps += dep_wayland_client
|
||||
vulkan_wsi_list += '-DVK_USE_PLATFORM_WAYLAND_KHR'
|
||||
endif
|
||||
if system_has_kms_drm and not with_platform_android
|
||||
vulkan_wsi_deps += [dep_libdrm, dep_display_info]
|
||||
vulkan_wsi_list += '-DVK_USE_PLATFORM_DISPLAY_KHR'
|
||||
endif
|
||||
if with_xlib_lease
|
||||
vulkan_wsi_deps += [dep_xlib_xrandr]
|
||||
vulkan_wsi_list += '-DVK_USE_PLATFORM_XLIB_XRANDR_EXT'
|
||||
endif
|
||||
if with_platform_android
|
||||
vulkan_wsi_list += '-DVK_USE_PLATFORM_ANDROID_KHR'
|
||||
endif
|
||||
if with_platform_windows
|
||||
vulkan_wsi_list += '-DVK_USE_PLATFORM_WIN32_KHR'
|
||||
endif
|
||||
if with_platform_macos
|
||||
vulkan_wsi_list += '-DVK_USE_PLATFORM_MACOS_MVK'
|
||||
vulkan_wsi_list += '-DVK_USE_PLATFORM_METAL_EXT'
|
||||
vulkan_wsi_deps += dependency(
|
||||
'appleframeworks',
|
||||
modules : ['QuartzCore', 'Metal']
|
||||
)
|
||||
endif
|
||||
|
||||
idep_vulkan_wsi_defines = declare_dependency(
|
||||
compile_args : vulkan_wsi_list,
|
||||
dependencies : vulkan_wsi_deps,
|
||||
)
|
||||
vulkan_wsi_deps += idep_vulkan_wsi_defines
|
||||
|
||||
subdir('util')
|
||||
subdir('runtime')
|
||||
subdir('wsi')
|
||||
if with_vulkan_overlay_layer
|
||||
subdir('overlay-layer')
|
||||
endif
|
||||
if with_vulkan_device_select_layer
|
||||
subdir('device-select-layer')
|
||||
endif
|
||||
if with_vulkan_screenshot_layer
|
||||
subdir('screenshot-layer')
|
||||
endif
|
||||
if with_vulkan_vram_report_limit_layer
|
||||
subdir('vram-report-limit-layer')
|
||||
endif
|
||||
if with_vulkan_anti_lag_layer
|
||||
subdir('anti-lag-layer')
|
||||
endif
|
||||
@@ -0,0 +1,100 @@
|
||||
A Vulkan layer to display information about the running application using an overlay.
|
||||
|
||||
Building
|
||||
=======
|
||||
|
||||
The overlay layer will be built if :code:`overlay` is passed as a :code:`vulkan-layers` argument. For example:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
meson -Dvulkan-layers=device-select,overlay builddir/
|
||||
ninja -C builddir/
|
||||
sudo ninja -C builddir/ install
|
||||
|
||||
See `docs/install.rst <https://gitlab.freedesktop.org/mesa/mesa/-/blob/master/docs/install.rst>`__ for more information.
|
||||
|
||||
Basic Usage
|
||||
=======
|
||||
|
||||
Turn on the layer:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
VK_LOADER_LAYERS_ENABLE=VK_LAYER_MESA_overlay /path/to/my_vulkan_app
|
||||
|
||||
|
||||
List the available statistics:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
VK_LOADER_LAYERS_ENABLE=VK_LAYER_MESA_overlay VK_LAYER_MESA_OVERLAY_CONFIG=help /path/to/my_vulkan_app
|
||||
|
||||
|
||||
Turn on some statistics:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
VK_LOADER_LAYERS_ENABLE=VK_LAYER_MESA_overlay VK_LAYER_MESA_OVERLAY_CONFIG=submit,draw,pipeline_graphics /path/to/my_vulkan_app
|
||||
|
||||
Position the overlay:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
VK_LOADER_LAYERS_ENABLE=VK_LAYER_MESA_overlay VK_LAYER_MESA_OVERLAY_CONFIG=submit,draw,pipeline_graphics,position=top-right /path/to/my_vulkan_app
|
||||
|
||||
Logging Statistics
|
||||
=======
|
||||
|
||||
Log statistics to a file:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
VK_LOADER_LAYERS_ENABLE=VK_LAYER_MESA_overlay VK_LAYER_MESA_OVERLAY_CONFIG=output_file=/tmp/output.txt /path/to/my_vulkan_app
|
||||
|
||||
Logging is enabled for the entire lifecycle of the process unless a control socket is specified (see below).
|
||||
|
||||
**Note:** some statistics (e.g. :code:`frame_timing` and :code:`gpu_timing`) log values for the entire sample interval instead of per-frame.
|
||||
For these statistics, logging the :code:`frame` statistic allows one to compute per-frame statistics after capture.
|
||||
|
||||
Log statistics to a file, controlling when such statistics will start to be captured:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
VK_LOADER_LAYERS_ENABLE=VK_LAYER_MESA_overlay VK_LAYER_MESA_OVERLAY_CONFIG=output_file=/tmp/output.txt,control=mesa_overlay /path/to/my_vulkan_app
|
||||
|
||||
The command above will open a Unix socket with the abstract path :code:`mesa_overlay`. When a control socket is specified,
|
||||
logging must be explicitly enabled through the control socket. :code:`mesa-overlay-control.py` provides a convenient CLI:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
mesa-overlay-control.py start-capture
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
mesa-overlay-control.py stop-capture
|
||||
|
||||
Direct Socket Control
|
||||
------
|
||||
|
||||
The Unix socket may be used directly if needed. Once a client connects to the socket, the overlay layer will immediately
|
||||
send the following commands to the client:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
:MesaOverlayControlVersion=1;
|
||||
:DeviceName=<device name>;
|
||||
:MesaVersion=<mesa version>;
|
||||
|
||||
The client connected to the overlay layer can enable statistics capturing by sending the command:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
:capture=1;
|
||||
|
||||
And disable it by sending:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
:capture=0;
|
||||
|
||||
.. _docs/install.rst: ../../docs/install.rst
|
||||
@@ -0,0 +1,3 @@
|
||||
- cpu load
|
||||
- human readable numbers (some numbers like fragment invocations are
|
||||
in the millions range)
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"file_format_version" : "1.0.0",
|
||||
"layer" : {
|
||||
"name": "VK_LAYER_MESA_overlay",
|
||||
"type": "GLOBAL",
|
||||
"library_path": "@library_path@",
|
||||
"api_version": "1.4.303",
|
||||
"implementation_version": "1",
|
||||
"description": "Mesa Overlay layer"
|
||||
}
|
||||
}
|
||||
+199
@@ -0,0 +1,199 @@
|
||||
#!/usr/bin/env python3
|
||||
import socket
|
||||
import sys
|
||||
import select
|
||||
from select import EPOLLIN, EPOLLPRI, EPOLLERR
|
||||
import time
|
||||
import argparse
|
||||
|
||||
TIMEOUT = 1.0 # seconds
|
||||
|
||||
VERSION_HEADER = bytearray('MesaOverlayControlVersion', 'utf-8')
|
||||
DEVICE_NAME_HEADER = bytearray('DeviceName', 'utf-8')
|
||||
MESA_VERSION_HEADER = bytearray('MesaVersion', 'utf-8')
|
||||
|
||||
DEFAULT_SERVER_ADDRESS = "\0mesa_overlay"
|
||||
|
||||
class Connection:
|
||||
def __init__(self, path):
|
||||
# Create a Unix Domain socket and connect
|
||||
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
try:
|
||||
sock.connect(path)
|
||||
except socket.error as msg:
|
||||
print(msg)
|
||||
sys.exit(1)
|
||||
|
||||
self.sock = sock
|
||||
|
||||
# initialize poll interface and register socket
|
||||
epoll = select.epoll()
|
||||
epoll.register(sock, EPOLLIN | EPOLLPRI | EPOLLERR)
|
||||
self.epoll = epoll
|
||||
|
||||
def recv(self, timeout):
|
||||
'''
|
||||
timeout as float in seconds
|
||||
returns:
|
||||
- None on error or disconnection
|
||||
- bytes() (empty) on timeout
|
||||
'''
|
||||
|
||||
events = self.epoll.poll(timeout)
|
||||
for ev in events:
|
||||
(fd, event) = ev
|
||||
if fd != self.sock.fileno():
|
||||
continue
|
||||
|
||||
# check for socket error
|
||||
if event & EPOLLERR:
|
||||
return None
|
||||
|
||||
# EPOLLIN or EPOLLPRI, just read the message
|
||||
msg = self.sock.recv(4096)
|
||||
|
||||
# socket disconnected
|
||||
if len(msg) == 0:
|
||||
return None
|
||||
|
||||
return msg
|
||||
|
||||
return bytes()
|
||||
|
||||
def send(self, msg):
|
||||
self.sock.send(msg)
|
||||
|
||||
class MsgParser:
|
||||
MSGBEGIN = bytes(':', 'utf-8')[0]
|
||||
MSGEND = bytes(';', 'utf-8')[0]
|
||||
MSGSEP = bytes('=', 'utf-8')[0]
|
||||
|
||||
def __init__(self, conn):
|
||||
self.cmdpos = 0
|
||||
self.parampos = 0
|
||||
self.bufferpos = 0
|
||||
self.reading_cmd = False
|
||||
self.reading_param = False
|
||||
self.buffer = None
|
||||
self.cmd = bytearray(4096)
|
||||
self.param = bytearray(4096)
|
||||
|
||||
self.conn = conn
|
||||
|
||||
def readCmd(self, ncmds, timeout=TIMEOUT):
|
||||
'''
|
||||
returns:
|
||||
- None on error or disconnection
|
||||
- bytes() (empty) on timeout
|
||||
'''
|
||||
|
||||
parsed = []
|
||||
|
||||
remaining = timeout
|
||||
|
||||
while remaining > 0 and ncmds > 0:
|
||||
now = time.monotonic()
|
||||
|
||||
if self.buffer is None:
|
||||
self.buffer = self.conn.recv(remaining)
|
||||
self.bufferpos = 0
|
||||
|
||||
# disconnected or error
|
||||
if self.buffer is None:
|
||||
return None
|
||||
|
||||
for i in range(self.bufferpos, len(self.buffer)):
|
||||
c = self.buffer[i]
|
||||
self.bufferpos += 1
|
||||
if c == self.MSGBEGIN:
|
||||
self.cmdpos = 0
|
||||
self.parampos = 0
|
||||
self.reading_cmd = True
|
||||
self.reading_param = False
|
||||
elif c == self.MSGEND:
|
||||
if not self.reading_cmd:
|
||||
continue
|
||||
self.reading_cmd = False
|
||||
self.reading_param = False
|
||||
|
||||
cmd = self.cmd[0:self.cmdpos]
|
||||
param = self.param[0:self.parampos]
|
||||
self.reading_cmd = False
|
||||
self.reading_param = False
|
||||
|
||||
parsed.append((cmd, param))
|
||||
ncmds -= 1
|
||||
if ncmds == 0:
|
||||
break
|
||||
elif c == self.MSGSEP:
|
||||
if self.reading_cmd:
|
||||
self.reading_param = True
|
||||
else:
|
||||
if self.reading_param:
|
||||
self.param[self.parampos] = c
|
||||
self.parampos += 1
|
||||
elif self.reading_cmd:
|
||||
self.cmd[self.cmdpos] = c
|
||||
self.cmdpos += 1
|
||||
|
||||
# if we read the entire buffer and didn't finish the command,
|
||||
# throw it away
|
||||
self.buffer = None
|
||||
|
||||
# check if we have time for another iteration
|
||||
elapsed = time.monotonic() - now
|
||||
remaining = max(0, remaining - elapsed)
|
||||
|
||||
# timeout
|
||||
return parsed
|
||||
|
||||
def control(args):
|
||||
if args.socket:
|
||||
address = '\0' + args.socket
|
||||
else:
|
||||
address = DEFAULT_SERVER_ADDRESS
|
||||
|
||||
conn = Connection(address)
|
||||
msgparser = MsgParser(conn)
|
||||
|
||||
version = None
|
||||
name = None
|
||||
mesa_version = None
|
||||
|
||||
msgs = msgparser.readCmd(3)
|
||||
|
||||
for m in msgs:
|
||||
cmd, param = m
|
||||
if cmd == VERSION_HEADER:
|
||||
version = int(param)
|
||||
elif cmd == DEVICE_NAME_HEADER:
|
||||
name = param.decode('utf-8')
|
||||
elif cmd == MESA_VERSION_HEADER:
|
||||
mesa_version = param.decode('utf-8')
|
||||
|
||||
if version != 1 or name is None or mesa_version is None:
|
||||
print('ERROR: invalid protocol')
|
||||
sys.exit(1)
|
||||
|
||||
if args.info:
|
||||
print(f"Protocol Version: {version}")
|
||||
print(f"Device Name: {name}")
|
||||
print(f"Mesa Version: {mesa_version}")
|
||||
|
||||
if args.cmd == 'start-capture':
|
||||
conn.send(bytearray(':capture=1;', 'utf-8'))
|
||||
elif args.cmd == 'stop-capture':
|
||||
conn.send(bytearray(':capture=0;', 'utf-8'))
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description='MESA_overlay control client')
|
||||
parser.add_argument('--info', action='store_true', help='Print info from socket')
|
||||
parser.add_argument('--socket', '-s', type=str, help='Path to socket')
|
||||
|
||||
commands = parser.add_subparsers(help='commands to run', dest='cmd')
|
||||
commands.add_parser('start-capture')
|
||||
commands.add_parser('stop-capture')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
control(args)
|
||||
@@ -0,0 +1,59 @@
|
||||
# Copyright © 2019 Intel Corporation
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
overlay_shaders = [
|
||||
'overlay.frag',
|
||||
'overlay.vert',
|
||||
]
|
||||
overlay_spv = []
|
||||
foreach s : ['overlay.frag', 'overlay.vert']
|
||||
_name = f'@[email protected]'
|
||||
overlay_spv += custom_target(
|
||||
_name,
|
||||
input : s,
|
||||
output : _name,
|
||||
command : [
|
||||
prog_glslang, '-V', '-x', '-o', '@OUTPUT@', '@INPUT@', glslang_quiet,
|
||||
glslang_depfile,
|
||||
],
|
||||
depfile : f'@[email protected]',
|
||||
)
|
||||
endforeach
|
||||
|
||||
vklayer_files = files(
|
||||
'overlay.cpp',
|
||||
'overlay_params.c',
|
||||
)
|
||||
|
||||
vklayer_mesa_overlay = shared_library(
|
||||
'VkLayer_MESA_overlay',
|
||||
vklayer_files, overlay_spv, sha1_h,
|
||||
c_args : no_override_init_args + cc.get_supported_arguments('-Wno-error=stringop-overflow'),
|
||||
gnu_symbol_visibility : 'hidden',
|
||||
dependencies : [idep_vulkan_util, idep_mesautil, vulkan_wsi_deps, libimgui_core_dep, dep_dl],
|
||||
include_directories : [inc_include, inc_src],
|
||||
link_args : cc.get_supported_link_arguments(['-Wl,-Bsymbolic-functions', '-Wl,-z,relro']),
|
||||
install : true
|
||||
)
|
||||
|
||||
configure_file(
|
||||
configuration : {'library_path' : 'libVkLayer_MESA_overlay.so'},
|
||||
input : 'VkLayer_MESA_overlay.json.in',
|
||||
output : '@BASENAME@',
|
||||
install : true,
|
||||
install_dir : join_paths(get_option('datadir'), 'vulkan', 'explicit_layer.d'),
|
||||
install_tag : 'runtime',
|
||||
)
|
||||
|
||||
configure_file(
|
||||
configuration : {'library_path' : vklayer_mesa_overlay.full_path()},
|
||||
input : 'VkLayer_MESA_overlay.json.in',
|
||||
output : '@PLAINNAME@_dev.json',
|
||||
)
|
||||
devenv.append('VK_ADD_LAYER_PATH', meson.current_build_dir())
|
||||
|
||||
install_data(
|
||||
'mesa-overlay-control.py',
|
||||
install_dir : get_option('bindir'),
|
||||
install_mode : 'r-xr-xr-x',
|
||||
)
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,14 @@
|
||||
#version 450 core
|
||||
layout(location = 0) out vec4 fColor;
|
||||
|
||||
layout(set=0, binding=0) uniform sampler2D sTexture;
|
||||
|
||||
layout(location = 0) in struct{
|
||||
vec4 Color;
|
||||
vec2 UV;
|
||||
} In;
|
||||
|
||||
void main()
|
||||
{
|
||||
fColor = In.Color * texture(sTexture, In.UV.st);
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
#version 450 core
|
||||
layout(location = 0) in vec2 aPos;
|
||||
layout(location = 1) in vec2 aUV;
|
||||
layout(location = 2) in vec4 aColor;
|
||||
|
||||
layout(push_constant) uniform uPushConstant{
|
||||
vec2 uScale;
|
||||
vec2 uTranslate;
|
||||
} pc;
|
||||
|
||||
out gl_PerVertex{
|
||||
vec4 gl_Position;
|
||||
};
|
||||
|
||||
layout(location = 0) out struct{
|
||||
vec4 Color;
|
||||
vec2 UV;
|
||||
} Out;
|
||||
|
||||
void main()
|
||||
{
|
||||
Out.Color = aColor;
|
||||
Out.UV = aUV;
|
||||
gl_Position = vec4(aPos*pc.uScale+pc.uTranslate, 0, 1);
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
/*
|
||||
* Copyright © 2019 Intel Corporation
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <errno.h>
|
||||
|
||||
#include "overlay_params.h"
|
||||
|
||||
#include "util/os_socket.h"
|
||||
|
||||
static enum overlay_param_position
|
||||
parse_position(const char *str)
|
||||
{
|
||||
if (!str || !strcmp(str, "top-left"))
|
||||
return LAYER_POSITION_TOP_LEFT;
|
||||
if (!strcmp(str, "top-right"))
|
||||
return LAYER_POSITION_TOP_RIGHT;
|
||||
if (!strcmp(str, "bottom-left"))
|
||||
return LAYER_POSITION_BOTTOM_LEFT;
|
||||
if (!strcmp(str, "bottom-right"))
|
||||
return LAYER_POSITION_BOTTOM_RIGHT;
|
||||
return LAYER_POSITION_TOP_LEFT;
|
||||
}
|
||||
|
||||
static const char *
|
||||
parse_output_file(const char *str)
|
||||
{
|
||||
return strdup(str);
|
||||
}
|
||||
|
||||
static const char *
|
||||
parse_control(const char *str)
|
||||
{
|
||||
return strdup(str);
|
||||
}
|
||||
|
||||
static uint32_t
|
||||
parse_fps_sampling_period(const char *str)
|
||||
{
|
||||
return strtol(str, NULL, 0) * 1000;
|
||||
}
|
||||
|
||||
static bool
|
||||
parse_no_display(const char *str)
|
||||
{
|
||||
return strtol(str, NULL, 0) != 0;
|
||||
}
|
||||
|
||||
static unsigned
|
||||
parse_unsigned(const char *str)
|
||||
{
|
||||
return strtol(str, NULL, 0);
|
||||
}
|
||||
|
||||
#define parse_width(s) parse_unsigned(s)
|
||||
#define parse_height(s) parse_unsigned(s)
|
||||
|
||||
static bool
|
||||
parse_help(const char *str)
|
||||
{
|
||||
fprintf(stderr, "Layer params using VK_LAYER_MESA_OVERLAY_CONFIG=\n");
|
||||
#define OVERLAY_PARAM_BOOL(name) \
|
||||
fprintf(stderr, "\t%s=0|1\n", #name);
|
||||
#define OVERLAY_PARAM_CUSTOM(name)
|
||||
OVERLAY_PARAMS
|
||||
#undef OVERLAY_PARAM_BOOL
|
||||
#undef OVERLAY_PARAM_CUSTOM
|
||||
fprintf(stderr, "\tposition=top-left|top-right|bottom-left|bottom-right\n");
|
||||
fprintf(stderr, "\tfps_sampling_period=number-of-milliseconds\n");
|
||||
fprintf(stderr, "\tno_display=0|1\n");
|
||||
fprintf(stderr, "\toutput_file=/path/to/output.txt\n");
|
||||
fprintf(stderr, "\twidth=width-in-pixels\n");
|
||||
fprintf(stderr, "\theight=height-in-pixels\n");
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool is_delimiter(char c)
|
||||
{
|
||||
return c == 0 || c == ',' || c == ':' || c == ';' || c == '=';
|
||||
}
|
||||
|
||||
static int
|
||||
parse_string(const char *s, char *out_param, char *out_value)
|
||||
{
|
||||
int i = 0;
|
||||
|
||||
for (; !is_delimiter(*s); s++, out_param++, i++)
|
||||
*out_param = *s;
|
||||
|
||||
*out_param = 0;
|
||||
|
||||
if (*s == '=') {
|
||||
s++;
|
||||
i++;
|
||||
for (; !is_delimiter(*s); s++, out_value++, i++)
|
||||
*out_value = *s;
|
||||
} else
|
||||
*(out_value++) = '1';
|
||||
*out_value = 0;
|
||||
|
||||
if (*s && is_delimiter(*s)) {
|
||||
s++;
|
||||
i++;
|
||||
}
|
||||
|
||||
if (*s && !i) {
|
||||
fprintf(stderr, "mesa-overlay: syntax error: unexpected '%c' (%i) while "
|
||||
"parsing a string\n", *s, *s);
|
||||
fflush(stderr);
|
||||
}
|
||||
|
||||
return i;
|
||||
}
|
||||
|
||||
const char *overlay_param_names[] = {
|
||||
#define OVERLAY_PARAM_BOOL(name) #name,
|
||||
#define OVERLAY_PARAM_CUSTOM(name)
|
||||
OVERLAY_PARAMS
|
||||
#undef OVERLAY_PARAM_BOOL
|
||||
#undef OVERLAY_PARAM_CUSTOM
|
||||
};
|
||||
|
||||
void
|
||||
parse_overlay_env(struct overlay_params *params,
|
||||
const char *env)
|
||||
{
|
||||
uint32_t num;
|
||||
char key[256], value[256];
|
||||
|
||||
memset(params, 0, sizeof(*params));
|
||||
|
||||
/* Visible by default */
|
||||
params->enabled[OVERLAY_PARAM_ENABLED_fps] = true;
|
||||
params->enabled[OVERLAY_PARAM_ENABLED_frame_timing] = true;
|
||||
params->enabled[OVERLAY_PARAM_ENABLED_device] = true;
|
||||
params->enabled[OVERLAY_PARAM_ENABLED_format] = true;
|
||||
params->fps_sampling_period = 500000; /* 500ms */
|
||||
params->width = params->height = 300;
|
||||
params->control = NULL;
|
||||
|
||||
if (!env)
|
||||
return;
|
||||
|
||||
while ((num = parse_string(env, key, value)) != 0) {
|
||||
env += num;
|
||||
|
||||
#define OVERLAY_PARAM_BOOL(name) \
|
||||
if (!strcmp(#name, key)) { \
|
||||
params->enabled[OVERLAY_PARAM_ENABLED_##name] = \
|
||||
strtol(value, NULL, 0); \
|
||||
continue; \
|
||||
}
|
||||
#define OVERLAY_PARAM_CUSTOM(name) \
|
||||
if (!strcmp(#name, key)) { \
|
||||
params->name = parse_##name(value); \
|
||||
continue; \
|
||||
}
|
||||
OVERLAY_PARAMS
|
||||
#undef OVERLAY_PARAM_BOOL
|
||||
#undef OVERLAY_PARAM_CUSTOM
|
||||
fprintf(stderr, "Unknown option '%s'\n", key);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
/*
|
||||
* Copyright © 2019 Intel Corporation
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#ifndef OVERLAY_PARAMS_H
|
||||
#define OVERLAY_PARAMS_H
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdint.h>
|
||||
#include <stdbool.h>
|
||||
|
||||
#define OVERLAY_PARAMS \
|
||||
OVERLAY_PARAM_BOOL(device) \
|
||||
OVERLAY_PARAM_BOOL(format) \
|
||||
OVERLAY_PARAM_BOOL(fps) \
|
||||
OVERLAY_PARAM_BOOL(frame) \
|
||||
OVERLAY_PARAM_BOOL(frame_timing) \
|
||||
OVERLAY_PARAM_BOOL(submit) \
|
||||
OVERLAY_PARAM_BOOL(draw) \
|
||||
OVERLAY_PARAM_BOOL(draw_indexed) \
|
||||
OVERLAY_PARAM_BOOL(draw_indirect) \
|
||||
OVERLAY_PARAM_BOOL(draw_indexed_indirect) \
|
||||
OVERLAY_PARAM_BOOL(draw_indirect_count) \
|
||||
OVERLAY_PARAM_BOOL(draw_indexed_indirect_count) \
|
||||
OVERLAY_PARAM_BOOL(dispatch) \
|
||||
OVERLAY_PARAM_BOOL(dispatch_indirect) \
|
||||
OVERLAY_PARAM_BOOL(pipeline_graphics) \
|
||||
OVERLAY_PARAM_BOOL(pipeline_compute) \
|
||||
OVERLAY_PARAM_BOOL(pipeline_raytracing) \
|
||||
OVERLAY_PARAM_BOOL(acquire) \
|
||||
OVERLAY_PARAM_BOOL(acquire_timing) \
|
||||
OVERLAY_PARAM_BOOL(present_timing) \
|
||||
OVERLAY_PARAM_BOOL(vertices) \
|
||||
OVERLAY_PARAM_BOOL(primitives) \
|
||||
OVERLAY_PARAM_BOOL(vert_invocations) \
|
||||
OVERLAY_PARAM_BOOL(geom_invocations) \
|
||||
OVERLAY_PARAM_BOOL(geom_primitives) \
|
||||
OVERLAY_PARAM_BOOL(clip_invocations) \
|
||||
OVERLAY_PARAM_BOOL(clip_primitives) \
|
||||
OVERLAY_PARAM_BOOL(frag_invocations) \
|
||||
OVERLAY_PARAM_BOOL(tess_ctrl_patches) \
|
||||
OVERLAY_PARAM_BOOL(tess_eval_invocations) \
|
||||
OVERLAY_PARAM_BOOL(compute_invocations) \
|
||||
OVERLAY_PARAM_BOOL(gpu_timing) \
|
||||
OVERLAY_PARAM_CUSTOM(fps_sampling_period) \
|
||||
OVERLAY_PARAM_CUSTOM(output_file) \
|
||||
OVERLAY_PARAM_CUSTOM(position) \
|
||||
OVERLAY_PARAM_CUSTOM(width) \
|
||||
OVERLAY_PARAM_CUSTOM(height) \
|
||||
OVERLAY_PARAM_CUSTOM(no_display) \
|
||||
OVERLAY_PARAM_CUSTOM(control) \
|
||||
OVERLAY_PARAM_CUSTOM(help)
|
||||
|
||||
enum overlay_param_position {
|
||||
LAYER_POSITION_TOP_LEFT,
|
||||
LAYER_POSITION_TOP_RIGHT,
|
||||
LAYER_POSITION_BOTTOM_LEFT,
|
||||
LAYER_POSITION_BOTTOM_RIGHT,
|
||||
};
|
||||
|
||||
enum overlay_param_enabled {
|
||||
#define OVERLAY_PARAM_BOOL(name) OVERLAY_PARAM_ENABLED_##name,
|
||||
#define OVERLAY_PARAM_CUSTOM(name)
|
||||
OVERLAY_PARAMS
|
||||
#undef OVERLAY_PARAM_BOOL
|
||||
#undef OVERLAY_PARAM_CUSTOM
|
||||
OVERLAY_PARAM_ENABLED_MAX
|
||||
};
|
||||
|
||||
struct overlay_params {
|
||||
bool enabled[OVERLAY_PARAM_ENABLED_MAX];
|
||||
enum overlay_param_position position;
|
||||
const char *output_file;
|
||||
const char *control;
|
||||
uint32_t fps_sampling_period; /* us */
|
||||
bool help;
|
||||
bool no_display;
|
||||
unsigned width;
|
||||
unsigned height;
|
||||
};
|
||||
|
||||
const extern char *overlay_param_names[];
|
||||
|
||||
void parse_overlay_env(struct overlay_params *params,
|
||||
const char *env);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* OVERLAY_PARAMS_H */
|
||||
Executable
+191
@@ -0,0 +1,191 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Check for and replace aliases with their new names from vk.xml
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import pathlib
|
||||
import subprocess
|
||||
import sys
|
||||
import xml.etree.ElementTree as et
|
||||
|
||||
THIS_FILE = pathlib.Path(__file__)
|
||||
CWD = pathlib.Path.cwd()
|
||||
|
||||
VK_XML = THIS_FILE.parent / 'vk.xml'
|
||||
EXCLUDE_PATHS = [
|
||||
VK_XML.relative_to(CWD).as_posix(),
|
||||
|
||||
# These files come from other repos, there's no point checking and
|
||||
# fixing them here as that would be overwritten in the next sync.
|
||||
'src/amd/vulkan/radix_sort/',
|
||||
'src/virtio/venus-protocol/',
|
||||
]
|
||||
|
||||
|
||||
def get_aliases(xml_file: pathlib.Path):
|
||||
"""
|
||||
Get all the aliases defined in vk.xml
|
||||
"""
|
||||
xml = et.parse(xml_file)
|
||||
|
||||
for node in ([]
|
||||
+ xml.findall('.//enum[@alias]')
|
||||
+ xml.findall('.//type[@alias]')
|
||||
+ xml.findall('.//command[@alias]')
|
||||
):
|
||||
# Some renames only apply to some APIs
|
||||
if 'api' in node.attrib and 'vulkan' not in node.attrib['api'].split(','):
|
||||
continue
|
||||
|
||||
yield node.attrib['name'], node.attrib['alias']
|
||||
|
||||
|
||||
def remove_prefix(string: str, prefix: str):
|
||||
"""
|
||||
Remove prefix if string starts with it, and return the full string
|
||||
otherwise.
|
||||
"""
|
||||
if not string.startswith(prefix):
|
||||
return string
|
||||
return string[len(prefix):]
|
||||
|
||||
|
||||
# Function from https://stackoverflow.com/a/312464
|
||||
def chunks(lst: list, n: int):
|
||||
"""
|
||||
Yield successive n-sized chunks from lst.
|
||||
"""
|
||||
for i in range(0, len(lst), n):
|
||||
yield lst[i:i + n]
|
||||
|
||||
|
||||
def main(paths: list[str]):
|
||||
"""
|
||||
Entrypoint; perform the search for all the aliases and replace them.
|
||||
"""
|
||||
def prepare_identifier(identifier: str) -> str:
|
||||
prefixes_seen = []
|
||||
for prefix in [
|
||||
# Various macros prepend these, so they will not appear in the code using them.
|
||||
# List generated using this command:
|
||||
# $ prefixes=$(git grep -woiE 'VK_\w+_' -- src/ ':!src/vulkan/registry/' | cut -d: -f2 | sort -u)
|
||||
# $ for prefix in $prefixes; do grep -q $prefix src/vulkan/registry/vk.xml && echo "'$prefix',"; done
|
||||
# (the second part eliminates prefixes used only in mesa code and not upstream)
|
||||
'VK_BLEND_FACTOR_',
|
||||
'VK_BLEND_OP_',
|
||||
'VK_BORDER_COLOR_',
|
||||
'VK_COMMAND_BUFFER_RESET_',
|
||||
'VK_COMMAND_POOL_RESET_',
|
||||
'VK_COMPARE_OP_',
|
||||
'VK_COMPONENT_SWIZZLE_',
|
||||
'VK_DESCRIPTOR_TYPE_',
|
||||
'VK_DRIVER_ID_',
|
||||
'VK_DYNAMIC_STATE_',
|
||||
'VK_FORMAT_',
|
||||
'VK_IMAGE_ASPECT_MEMORY_PLANE_',
|
||||
'VK_IMAGE_ASPECT_PLANE_',
|
||||
'VK_IMAGE_USAGE_',
|
||||
'VK_NV_',
|
||||
'VK_PERFORMANCE_COUNTER_UNIT_',
|
||||
'VK_PIPELINE_BIND_POINT_',
|
||||
'VK_SAMPLER_ADDRESS_MODE_',
|
||||
'VK_SHADER_STAGE_TESSELLATION_',
|
||||
'VK_SHADER_STAGE_',
|
||||
'VK_STENCIL_OP_',
|
||||
'VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_',
|
||||
'VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_',
|
||||
'VK_STRUCTURE_TYPE_',
|
||||
'VK_USE_PLATFORM_',
|
||||
'VK_VERSION_',
|
||||
|
||||
# Many places use the identifier without the `vk` prefix
|
||||
# (eg. with the driver name as a prefix instead)
|
||||
'VK_',
|
||||
'Vk',
|
||||
'vk',
|
||||
]:
|
||||
# The order matters! A shorter substring will match before a longer
|
||||
# one, hiding its matches.
|
||||
for prefix_seen in prefixes_seen:
|
||||
assert not prefix.startswith(prefix_seen), f'{prefix_seen} must come before {prefix}'
|
||||
prefixes_seen.append(prefix)
|
||||
|
||||
identifier = remove_prefix(identifier, prefix)
|
||||
|
||||
return identifier
|
||||
|
||||
aliases = {}
|
||||
for old_name, alias_for in get_aliases(VK_XML):
|
||||
old_name = prepare_identifier(old_name)
|
||||
alias_for = prepare_identifier(alias_for)
|
||||
aliases[old_name] = alias_for
|
||||
|
||||
print(f'Found {len(aliases)} aliases in {VK_XML.name}')
|
||||
|
||||
# Some aliases have aliases
|
||||
recursion_needs_checking = True
|
||||
while recursion_needs_checking:
|
||||
recursion_needs_checking = False
|
||||
for old, new in aliases.items():
|
||||
if new in aliases:
|
||||
aliases[old] = aliases[new]
|
||||
recursion_needs_checking = True
|
||||
|
||||
# Doing the whole search in a single command breaks grep, so only
|
||||
# look for 500 aliases at a time. Searching them one at a time would
|
||||
# be extremely slow.
|
||||
files_with_aliases = set()
|
||||
for aliases_chunk in chunks([*aliases], 500):
|
||||
grep_cmd = [
|
||||
'git',
|
||||
'grep',
|
||||
'-rlP',
|
||||
'|'.join(aliases_chunk),
|
||||
] + paths
|
||||
search_output = subprocess.run(
|
||||
grep_cmd,
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
).stdout.decode()
|
||||
files_with_aliases.update(search_output.splitlines())
|
||||
|
||||
|
||||
def file_matches_path(file: str, path: str) -> bool:
|
||||
# if path is a folder; match any file within
|
||||
if path.endswith('/') and file.startswith(path):
|
||||
return True
|
||||
return file == path
|
||||
|
||||
for excluded_path in EXCLUDE_PATHS:
|
||||
files_with_aliases = {
|
||||
file for file in files_with_aliases
|
||||
if not file_matches_path(file, excluded_path)
|
||||
}
|
||||
|
||||
if not files_with_aliases:
|
||||
print('No alias found in any file.')
|
||||
sys.exit(0)
|
||||
|
||||
print(f'{len(files_with_aliases)} files contain aliases:')
|
||||
print('\n'.join(f'- {file}' for file in sorted(files_with_aliases)))
|
||||
|
||||
command = [
|
||||
'sed',
|
||||
'-i',
|
||||
";".join([f's/{old}/{new}/g' for old, new in aliases.items()]),
|
||||
]
|
||||
command += files_with_aliases
|
||||
subprocess.check_call(command, stderr=subprocess.DEVNULL)
|
||||
print('All aliases have been replaced')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('paths',
|
||||
nargs=argparse.ZERO_OR_MORE,
|
||||
default=['src/'],
|
||||
help='Limit script to these paths (default: `src/`)')
|
||||
args = parser.parse_args()
|
||||
main(**vars(args))
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,259 @@
|
||||
/*
|
||||
* Copyright © 2025 Valve Corporation
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#version 460
|
||||
|
||||
#include "vk_build_interface.h"
|
||||
#include "vk_debug.h"
|
||||
|
||||
layout(local_size_x_id = SUBGROUP_SIZE_ID, local_size_y = 1, local_size_z = 1) in;
|
||||
|
||||
layout(push_constant) uniform CONSTS
|
||||
{
|
||||
hploc_args args;
|
||||
};
|
||||
|
||||
uint32_t
|
||||
delta(uint32_t index)
|
||||
{
|
||||
uint32_t left_index = index;
|
||||
uint32_t right_index = index + 1;
|
||||
|
||||
if (VK_BUILD_FLAG(VK_BUILD_FLAG_64BIT_KEYS)) {
|
||||
uint64_t left_key = packUint2x32(u32vec2(
|
||||
DEREF(INDEX(key64_id_pair, args.ids, left_index)).key_lo,
|
||||
DEREF(INDEX(key64_id_pair, args.ids, left_index)).key_hi));
|
||||
uint64_t right_key = packUint2x32(u32vec2(
|
||||
DEREF(INDEX(key64_id_pair, args.ids, right_index)).key_lo,
|
||||
DEREF(INDEX(key64_id_pair, args.ids, right_index)).key_hi));
|
||||
return left_key != right_key ? (32 + uint32_t(findMSB(left_key ^ right_key))) : uint32_t(findMSB(left_index ^ right_index));
|
||||
} else {
|
||||
uint32_t left_key = DEREF(INDEX(key32_id_pair, args.ids, left_index)).key;
|
||||
uint32_t right_key = DEREF(INDEX(key32_id_pair, args.ids, right_index)).key;
|
||||
|
||||
return left_key != right_key ? (32 + findMSB(left_key ^ right_key)) : findMSB(left_index ^ right_index);
|
||||
}
|
||||
}
|
||||
|
||||
#define SEARCH_RADIUS 16
|
||||
|
||||
shared uint32_t node_ids[SUBGROUP_SIZE];
|
||||
shared vk_aabb node_aabbs[SUBGROUP_SIZE];
|
||||
shared uint32_t candidate_infos[SUBGROUP_SIZE];
|
||||
|
||||
void
|
||||
main(void)
|
||||
{
|
||||
uint32_t global_id = gl_GlobalInvocationID.x;
|
||||
REF(vk_ir_header) header = REF(vk_ir_header)(args.header);
|
||||
uint32_t active_leaf_count = DEREF(header).active_leaf_count;
|
||||
|
||||
if (active_leaf_count <= 1) {
|
||||
if (global_id > 0)
|
||||
return;
|
||||
|
||||
DEREF(header).ir_internal_node_count = 1;
|
||||
|
||||
uint32_t child_id = VK_BVH_INVALID_NODE;
|
||||
vk_ir_node child = vk_ir_node(vk_aabb(vec3(0.0), vec3(0.0)));
|
||||
if (active_leaf_count > 0) {
|
||||
if (VK_BUILD_FLAG(VK_BUILD_FLAG_64BIT_KEYS)) {
|
||||
REF(key64_id_pair) key_id = INDEX(key64_id_pair, args.ids, global_id);
|
||||
child_id = DEREF(key_id).id;
|
||||
} else {
|
||||
REF(key32_id_pair) key_id = INDEX(key32_id_pair, args.ids, global_id);
|
||||
child_id = DEREF(key_id).id;
|
||||
}
|
||||
child = DEREF(REF(vk_ir_node)(OFFSET(args.bvh, ir_id_to_offset(child_id))));
|
||||
}
|
||||
|
||||
REF(vk_ir_box_node) node = REF(vk_ir_box_node)(OFFSET(args.bvh, args.internal_node_base));
|
||||
DEREF(node).base.aabb = child.aabb;
|
||||
DEREF(node).children[0] = child_id;
|
||||
DEREF(node).children[1] = VK_BVH_INVALID_NODE;
|
||||
DEREF(node).bvh_offset = VK_UNKNOWN_BVH_OFFSET;
|
||||
if (VK_BUILD_FLAG(VK_BUILD_FLAG_PROPAGATE_CULL_FLAGS) && active_leaf_count > 0)
|
||||
DEREF(node).flags = fetch_child_flags(args.bvh, child_id);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
/* Start at the leaf nodes which cover only one primitive => start=end. */
|
||||
uint32_t range_start = global_id;
|
||||
uint32_t range_end = global_id;
|
||||
|
||||
uint32_t internal_node_count = active_leaf_count - 1;
|
||||
|
||||
bool is_active = global_id < active_leaf_count;
|
||||
|
||||
while (subgroupAny(is_active)) {
|
||||
uint32_t parent_index = 0xffffffff;
|
||||
if (is_active) {
|
||||
/* The parent node has either the index range_start-1 or range_end. Avoid indexing -1 or active_leaf_count. */
|
||||
bool use_right_parent = range_start == 0 || (range_end < internal_node_count && delta(range_end) < delta(range_start - 1));
|
||||
|
||||
parent_index = use_right_parent ? range_end : (range_start - 1);
|
||||
if (parent_index == internal_node_count) {
|
||||
is_active = false;
|
||||
} else {
|
||||
uint32_t prev_range = atomicExchange(DEREF(INDEX(uint32_t, args.ranges, parent_index)), use_right_parent ? range_start : range_end);
|
||||
if (prev_range == 0xffffffff) {
|
||||
is_active = false;
|
||||
} else {
|
||||
if (use_right_parent)
|
||||
range_end = prev_range;
|
||||
else
|
||||
range_start = prev_range;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Merging phase for every invocation that has a range with more than SUBGROUP_SIZE / 2 nodes.
|
||||
* The nodes are merged until the number of nodes is below SUBGROUP_SIZE / 2 which ensures that
|
||||
* the invocation handling the parent node can load its child nodes.
|
||||
*/
|
||||
const uint32_t cluster_threshold = SUBGROUP_SIZE / 2;
|
||||
uint32_t range_size = range_end - range_start + 1;
|
||||
bool range_is_root = subgroupAny(range_size == active_leaf_count);
|
||||
uint64_t cluster_mask = packUint2x32(subgroupBallot(is_active && (range_is_root || range_size > cluster_threshold)).xy);
|
||||
while (cluster_mask != 0) {
|
||||
uint32_t cluster_invoc = uint32_t(findLSB(cluster_mask));
|
||||
/* Clear the LSB. */
|
||||
cluster_mask &= cluster_mask - 1;
|
||||
|
||||
uint32_t start = subgroupShuffle(range_start, cluster_invoc);
|
||||
uint32_t split = subgroupShuffle(parent_index, cluster_invoc);
|
||||
uint32_t end = subgroupShuffle(range_end, cluster_invoc);
|
||||
|
||||
uint32_t load_index, load_base, index_range;
|
||||
if (gl_SubgroupInvocationID >= cluster_threshold) {
|
||||
load_index = gl_SubgroupInvocationID - cluster_threshold;
|
||||
load_base = split + 1;
|
||||
index_range = end - split;
|
||||
} else {
|
||||
load_index = gl_SubgroupInvocationID;
|
||||
load_base = start;
|
||||
index_range = split + 1 - start;
|
||||
}
|
||||
uint32_t node_id_index = load_base + load_index;
|
||||
uint32_t node_id = VK_BVH_INVALID_NODE;
|
||||
if (load_index < index_range) {
|
||||
if (VK_BUILD_FLAG(VK_BUILD_FLAG_64BIT_KEYS)) {
|
||||
node_id = DEREF(INDEX(key64_id_pair, args.ids, node_id_index)).id;
|
||||
} else {
|
||||
node_id = DEREF(INDEX(key32_id_pair, args.ids, node_id_index)).id;
|
||||
}
|
||||
}
|
||||
|
||||
uvec4 node_valid_mask = subgroupBallot(node_id != VK_BVH_INVALID_NODE);
|
||||
uint32_t node_prefix_sum = subgroupBallotExclusiveBitCount(node_valid_mask);
|
||||
uint32_t node_count = subgroupBallotBitCount(node_valid_mask);
|
||||
if (node_id != VK_BVH_INVALID_NODE) {
|
||||
node_ids[node_prefix_sum] = node_id;
|
||||
node_aabbs[node_prefix_sum] = DEREF(REF(vk_ir_node)(OFFSET(args.bvh, ir_id_to_offset(node_id)))).aabb;
|
||||
}
|
||||
|
||||
while (node_count > (range_is_root ? 1 : cluster_threshold)) {
|
||||
node_id = VK_BVH_INVALID_NODE;
|
||||
vk_aabb node_aabb = node_aabbs[gl_SubgroupInvocationID];
|
||||
if (gl_SubgroupInvocationID < node_count) {
|
||||
candidate_infos[gl_SubgroupInvocationID] = 0xffffffff;
|
||||
uint32_t best_candidate = 0xffffffff;
|
||||
for (uint32_t i = 1; i <= SEARCH_RADIUS; i++) {
|
||||
int32_t index = int32_t(gl_SubgroupInvocationID) - int(i);
|
||||
|
||||
vk_aabb shared_bounds;
|
||||
shared_bounds.min = min(node_aabbs[index].min, node_aabb.min);
|
||||
shared_bounds.max = max(node_aabbs[index].max, node_aabb.max);
|
||||
|
||||
uint32_t shared_sa = (floatBitsToUint(aabb_surface_area(shared_bounds)) << 1u) & (~(SUBGROUP_SIZE - 1));
|
||||
if (index >= 0) {
|
||||
uint32_t candidate = shared_sa | index;
|
||||
best_candidate = min(best_candidate, candidate);
|
||||
|
||||
candidate = shared_sa | gl_SubgroupInvocationID;
|
||||
atomicMin(candidate_infos[index], candidate);
|
||||
}
|
||||
}
|
||||
|
||||
best_candidate = min(best_candidate, candidate_infos[gl_SubgroupInvocationID]);
|
||||
uint32_t best_index = best_candidate & (SUBGROUP_SIZE - 1);
|
||||
uint32_t other_node_id = node_ids[best_index];
|
||||
|
||||
vk_aabb shared_bounds;
|
||||
shared_bounds.min = min(node_aabbs[best_index].min, node_aabb.min);
|
||||
shared_bounds.max = max(node_aabbs[best_index].max, node_aabb.max);
|
||||
|
||||
/* There is always at least on pair of invocations that can be merged because there is a finite number of pairs and
|
||||
* one of them therefore has a minimum surface area. If more than two nodes have the exact same surface area, the
|
||||
* neighbor search prioritizes lower invocation indices.
|
||||
*/
|
||||
bool merge = best_index < SUBGROUP_SIZE && subgroupShuffle(best_index, best_index) == gl_SubgroupInvocationID;
|
||||
|
||||
node_id = node_ids[gl_SubgroupInvocationID];
|
||||
|
||||
if (merge) {
|
||||
if (gl_SubgroupInvocationID < best_index) {
|
||||
uint32_t dst_index = atomicAdd(DEREF(header).ir_internal_node_count, 1);
|
||||
uint32_t dst_offset = args.internal_node_base + dst_index * SIZEOF(vk_ir_box_node);
|
||||
|
||||
node_aabb = shared_bounds;
|
||||
|
||||
REF(vk_ir_box_node) node = REF(vk_ir_box_node)(OFFSET(args.bvh, dst_offset));
|
||||
DEREF(node).base.aabb = shared_bounds;
|
||||
DEREF(node).children[0] = node_id;
|
||||
DEREF(node).children[1] = other_node_id;
|
||||
DEREF(node).bvh_offset = VK_UNKNOWN_BVH_OFFSET;
|
||||
if (VK_BUILD_FLAG(VK_BUILD_FLAG_PROPAGATE_CULL_FLAGS)) {
|
||||
DEREF(node).flags = fetch_child_flags(args.bvh, node_id) & fetch_child_flags(args.bvh, other_node_id);
|
||||
}
|
||||
|
||||
node_id = pack_ir_node_id(dst_offset, vk_ir_node_internal);
|
||||
} else {
|
||||
node_id = VK_BVH_INVALID_NODE;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
node_count = subgroupBallotBitCount(subgroupBallot(node_id != VK_BVH_INVALID_NODE));
|
||||
uint32_t node_prefix_sum = subgroupBallotExclusiveBitCount(subgroupBallot(node_id != VK_BVH_INVALID_NODE));
|
||||
if (node_id != VK_BVH_INVALID_NODE) {
|
||||
node_ids[node_prefix_sum] = node_id;
|
||||
node_aabbs[node_prefix_sum] = node_aabb;
|
||||
}
|
||||
}
|
||||
|
||||
if (gl_SubgroupInvocationID < min(end - start + 1, cluster_threshold)) {
|
||||
uint32_t dst_node = gl_SubgroupInvocationID < node_count ? node_ids[gl_SubgroupInvocationID] : VK_BVH_INVALID_NODE;
|
||||
if (VK_BUILD_FLAG(VK_BUILD_FLAG_64BIT_KEYS)) {
|
||||
DEREF(INDEX(key64_id_pair, args.ids, start + gl_SubgroupInvocationID)).id = dst_node;
|
||||
} else {
|
||||
DEREF(INDEX(key32_id_pair, args.ids, start + gl_SubgroupInvocationID)).id = dst_node;
|
||||
}
|
||||
}
|
||||
|
||||
memoryBarrier(gl_ScopeDevice, gl_StorageSemanticsBuffer,
|
||||
gl_SemanticsAcquireRelease | gl_SemanticsMakeAvailable | gl_SemanticsMakeVisible);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
/*
|
||||
* Copyright © 2022 Bas Nieuwenhuizen
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#version 460
|
||||
|
||||
#include "vk_build_interface.h"
|
||||
|
||||
layout(local_size_x_id = SUBGROUP_SIZE_ID, local_size_y = 1, local_size_z = 1) in;
|
||||
|
||||
TYPE(lbvh_node_info, 4);
|
||||
|
||||
layout(push_constant) uniform CONSTS
|
||||
{
|
||||
lbvh_generate_ir_args args;
|
||||
};
|
||||
|
||||
void
|
||||
main(void)
|
||||
{
|
||||
uint32_t global_id = gl_GlobalInvocationID.x;
|
||||
|
||||
uint32_t internal_node_count = max(DEREF(args.header).active_leaf_count, 2) - 1;
|
||||
if (global_id >= internal_node_count)
|
||||
return;
|
||||
|
||||
uint32_t idx = global_id;
|
||||
|
||||
uint32_t previous_id = VK_BVH_INVALID_NODE;
|
||||
vk_aabb previous_bounds;
|
||||
previous_bounds.min = vec3(INFINITY);
|
||||
previous_bounds.max = vec3(-INFINITY);
|
||||
uint32_t previous_flags;
|
||||
|
||||
for (;;) {
|
||||
uint32_t count = 0;
|
||||
|
||||
/* Check if all children have been processed. As this is an atomic the last path coming from
|
||||
* a child will pass here, while earlier paths break.
|
||||
*/
|
||||
count = atomicAdd(
|
||||
DEREF(INDEX(lbvh_node_info, args.node_info, idx)).path_count, 1, gl_ScopeDevice,
|
||||
gl_StorageSemanticsBuffer,
|
||||
gl_SemanticsAcquireRelease | gl_SemanticsMakeAvailable | gl_SemanticsMakeVisible);
|
||||
if (count != 2)
|
||||
break;
|
||||
|
||||
/* We allocate nodes on demand with the atomic here to ensure children come before their
|
||||
* parents, which is a requirement of the encoder.
|
||||
*/
|
||||
uint32_t dst_idx = atomicAdd(DEREF(args.header).ir_internal_node_count, 1);
|
||||
|
||||
uint32_t current_offset = args.internal_node_base + dst_idx * SIZEOF(vk_ir_box_node);
|
||||
uint32_t current_id = pack_ir_node_id(current_offset, vk_ir_node_internal);
|
||||
|
||||
REF(vk_ir_box_node) node = REF(vk_ir_box_node)(OFFSET(args.bvh, current_offset));
|
||||
vk_aabb bounds = previous_bounds;
|
||||
uint32_t flags = VK_BVH_BOX_FLAG_ONLY_OPAQUE | VK_BVH_BOX_FLAG_NO_OPAQUE;
|
||||
|
||||
lbvh_node_info info = DEREF(INDEX(lbvh_node_info, args.node_info, idx));
|
||||
|
||||
uint32_t children[2] = info.children;
|
||||
|
||||
/* Try using the cached previous_bounds instead of fetching the bounds twice. */
|
||||
int32_t previous_child_index = -1;
|
||||
if (previous_id == children[0])
|
||||
previous_child_index = 0;
|
||||
else if (previous_id == children[1])
|
||||
previous_child_index = 1;
|
||||
|
||||
if (previous_child_index == -1) {
|
||||
if (children[0] != VK_BVH_INVALID_NODE) {
|
||||
uint32_t child_offset = ir_id_to_offset(children[0]);
|
||||
REF(vk_ir_node) child = REF(vk_ir_node)(OFFSET(args.bvh, child_offset));
|
||||
vk_aabb child_bounds = DEREF(child).aabb;
|
||||
bounds.min = min(bounds.min, child_bounds.min);
|
||||
bounds.max = max(bounds.max, child_bounds.max);
|
||||
|
||||
if (VK_BUILD_FLAG(VK_BUILD_FLAG_PROPAGATE_CULL_FLAGS))
|
||||
flags &= fetch_child_flags(args.bvh, children[0]);
|
||||
}
|
||||
previous_child_index = 0;
|
||||
} else {
|
||||
if (VK_BUILD_FLAG(VK_BUILD_FLAG_PROPAGATE_CULL_FLAGS))
|
||||
flags &= previous_flags;
|
||||
}
|
||||
|
||||
/* Fetch the non-cached child */
|
||||
if (children[1 - previous_child_index] != VK_BVH_INVALID_NODE) {
|
||||
uint32_t child_offset = ir_id_to_offset(children[1 - previous_child_index]);
|
||||
REF(vk_ir_node) child = REF(vk_ir_node)(OFFSET(args.bvh, child_offset));
|
||||
vk_aabb child_bounds = DEREF(child).aabb;
|
||||
bounds.min = min(bounds.min, child_bounds.min);
|
||||
bounds.max = max(bounds.max, child_bounds.max);
|
||||
|
||||
if (VK_BUILD_FLAG(VK_BUILD_FLAG_PROPAGATE_CULL_FLAGS))
|
||||
flags &= fetch_child_flags(args.bvh, children[1 - previous_child_index]);
|
||||
}
|
||||
|
||||
|
||||
vk_ir_box_node node_value;
|
||||
|
||||
node_value.base.aabb = bounds;
|
||||
node_value.bvh_offset = VK_UNKNOWN_BVH_OFFSET;
|
||||
node_value.children = children;
|
||||
|
||||
if (VK_BUILD_FLAG(VK_BUILD_FLAG_PROPAGATE_CULL_FLAGS))
|
||||
node_value.flags = flags;
|
||||
|
||||
DEREF(node) = node_value;
|
||||
|
||||
if (info.parent == VK_BVH_INVALID_NODE)
|
||||
break;
|
||||
|
||||
idx = info.parent & ~LBVH_RIGHT_CHILD_BIT;
|
||||
|
||||
DEREF(INDEX(lbvh_node_info, args.node_info, idx))
|
||||
.children[(info.parent >> LBVH_RIGHT_CHILD_BIT_SHIFT) & 1] = current_id;
|
||||
|
||||
previous_id = current_id;
|
||||
previous_bounds = bounds;
|
||||
previous_flags = flags;
|
||||
|
||||
memoryBarrierBuffer();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
/*
|
||||
* Copyright © 2022 Bas Nieuwenhuizen
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#version 460
|
||||
|
||||
#include "vk_build_interface.h"
|
||||
|
||||
layout(local_size_x_id = SUBGROUP_SIZE_ID, local_size_y = 1, local_size_z = 1) in;
|
||||
|
||||
TYPE(lbvh_node_info, 4);
|
||||
|
||||
layout(push_constant) uniform CONSTS
|
||||
{
|
||||
lbvh_main_args args;
|
||||
};
|
||||
|
||||
int32_t
|
||||
longest_common_prefix(int32_t i, uint32_t key_i_lo, uint32_t key_i_hi, int32_t j, uint32_t active_leaf_count)
|
||||
{
|
||||
if (j < 0 || j >= active_leaf_count)
|
||||
return -1;
|
||||
|
||||
if (VK_BUILD_FLAG(VK_BUILD_FLAG_64BIT_KEYS)) {
|
||||
uint64_t key_i = packUint2x32(u32vec2(key_i_lo, key_i_hi));
|
||||
key64_id_pair key_id_j = DEREF(INDEX(key64_id_pair, args.src_ids, j));
|
||||
uint64_t key_j = packUint2x32(u32vec2(key_id_j.key_lo, key_id_j.key_hi));
|
||||
|
||||
uint64_t diff = key_i ^ key_j;
|
||||
int32_t ret = 0;
|
||||
if (key_i == key_j) {
|
||||
ret += 64;
|
||||
diff = i ^ j;
|
||||
}
|
||||
|
||||
return ret + 63 - int32_t(findMSB(diff));
|
||||
} else {
|
||||
uint32_t key_i = key_i_lo;
|
||||
uint32_t key_j = DEREF(INDEX(key32_id_pair, args.src_ids, j)).key;
|
||||
|
||||
uint32_t diff = key_i ^ key_j;
|
||||
int32_t ret = 0;
|
||||
if (key_i == key_j) {
|
||||
ret += 32;
|
||||
diff = i ^ j;
|
||||
}
|
||||
|
||||
return ret + 31 - findMSB(diff);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* The LBVH algorithm constructs a radix tree of the sorted nodes according to their key.
|
||||
*
|
||||
* We do this by making the following decision:
|
||||
*
|
||||
* Node N always either starts or ends at leaf N.
|
||||
*
|
||||
* From there it follows that we always have to extend it into the direction which has
|
||||
* a longer common prefix with the direct neighbour. Then we try to make the node cover
|
||||
* as many leaves as possible without including the other neighbour.
|
||||
*
|
||||
* For finding the split point we compute the longest common prefix of all the leaves within the
|
||||
* node, and look for the first leaf the same length common prefix with leaf N as that.
|
||||
*
|
||||
* To give an example: leaves=[000,001,010,011,100,101,110,111], node=5 (with value 101)
|
||||
*
|
||||
* lcp(101, 100) = 2 and lcp(101, 110) = 1, so we extend down.
|
||||
* lcp(101, 011) = 0, so the range of the node is [4,5] with values [100, 101]
|
||||
*
|
||||
* the lcp of all the leaves in the range is the same as the lcp of the first and last leaf, in this
|
||||
* case that is lcp(101, 100) = 2. Then we have lcp(101, 101) = 3 and lcp(101, 100) = 2, so the first
|
||||
* leaf that has a longer lcp is 4. Hence the two children of this node have range [4,4] and [5,5]
|
||||
*/
|
||||
void
|
||||
main()
|
||||
{
|
||||
uint32_t active_leaf_count = DEREF(args.header).active_leaf_count;
|
||||
|
||||
if (active_leaf_count <= 1) {
|
||||
REF(lbvh_node_info) dst = REF(lbvh_node_info)(args.node_info);
|
||||
DEREF(dst).parent = VK_BVH_INVALID_NODE;
|
||||
DEREF(dst).path_count = 2;
|
||||
uint32_t id = VK_BUILD_FLAG(VK_BUILD_FLAG_64BIT_KEYS) ? DEREF(INDEX(key64_id_pair, args.src_ids, 0)).id
|
||||
: DEREF(INDEX(key32_id_pair, args.src_ids, 0)).id;
|
||||
DEREF(dst).children[0] = active_leaf_count == 1 ? id : VK_BVH_INVALID_NODE;
|
||||
DEREF(dst).children[1] = VK_BVH_INVALID_NODE;
|
||||
return;
|
||||
}
|
||||
|
||||
int32_t id = int32_t(gl_GlobalInvocationID.x);
|
||||
uint32_t internal_node_count = max(active_leaf_count, 2) - 1;
|
||||
if (id >= internal_node_count)
|
||||
return;
|
||||
|
||||
uint32_t key_lo;
|
||||
uint32_t key_hi;
|
||||
if (VK_BUILD_FLAG(VK_BUILD_FLAG_64BIT_KEYS)) {
|
||||
key64_id_pair key_id = DEREF(INDEX(key64_id_pair, args.src_ids, id));
|
||||
key_lo = key_id.key_lo;
|
||||
key_hi = key_id.key_hi;
|
||||
} else {
|
||||
key_lo = DEREF(INDEX(key32_id_pair, args.src_ids, id)).key;
|
||||
}
|
||||
|
||||
int32_t left_lcp = longest_common_prefix(id, key_lo, key_hi, id - 1, active_leaf_count);
|
||||
int32_t right_lcp = longest_common_prefix(id, key_lo, key_hi, id + 1, active_leaf_count);
|
||||
int32_t dir = right_lcp > left_lcp ? 1 : -1;
|
||||
int32_t lcp_min = min(left_lcp, right_lcp);
|
||||
|
||||
/* Determine the bounds for the binary search for the length of the range that
|
||||
* this subtree is going to own.
|
||||
*/
|
||||
int32_t lmax = 128;
|
||||
while (longest_common_prefix(id, key_lo, key_hi, id + dir * lmax, active_leaf_count) > lcp_min) {
|
||||
lmax *= 2;
|
||||
}
|
||||
|
||||
int32_t length = 0;
|
||||
for (int32_t t = lmax / 2; t >= 1; t /= 2) {
|
||||
if (longest_common_prefix(id, key_lo, key_hi, id + (length + t) * dir, active_leaf_count) > lcp_min)
|
||||
length += t;
|
||||
}
|
||||
int32_t other_end = id + length * dir;
|
||||
|
||||
/* The number of bits in the prefix that is the same for all elements in the
|
||||
* range.
|
||||
*/
|
||||
int32_t lcp_node = longest_common_prefix(id, key_lo, key_hi, other_end, active_leaf_count);
|
||||
int32_t child_range = 0;
|
||||
for (int32_t diff = 2; diff < 2 * length; diff *= 2) {
|
||||
int32_t t = DIV_ROUND_UP(length, diff);
|
||||
if (longest_common_prefix(id, key_lo, key_hi, id + (child_range + t) * dir, active_leaf_count) > lcp_node)
|
||||
child_range += t;
|
||||
}
|
||||
|
||||
int32_t child_split = id + child_range * dir;
|
||||
|
||||
/* If dir = -1, right = child_split */
|
||||
int32_t left = child_split + min(dir, 0);
|
||||
int32_t right = left + 1;
|
||||
|
||||
/* if the number of leaves covered by a child is 1, we can use the leaf directly */
|
||||
bool left_leaf = min(id, other_end) == left;
|
||||
bool right_leaf = max(id, other_end) == right;
|
||||
|
||||
if (!left_leaf)
|
||||
DEREF(INDEX(lbvh_node_info, args.node_info, left)).parent = id;
|
||||
if (!right_leaf)
|
||||
DEREF(INDEX(lbvh_node_info, args.node_info, right)).parent = LBVH_RIGHT_CHILD_BIT | id;
|
||||
|
||||
REF(lbvh_node_info) dst = INDEX(lbvh_node_info, args.node_info, id);
|
||||
DEREF(dst).path_count = (left_leaf ? 1 : 0) + (right_leaf ? 1 : 0);
|
||||
if (VK_BUILD_FLAG(VK_BUILD_FLAG_64BIT_KEYS)) {
|
||||
DEREF(dst).children[0] = DEREF(INDEX(key64_id_pair, args.src_ids, left)).id;
|
||||
DEREF(dst).children[1] = DEREF(INDEX(key64_id_pair, args.src_ids, right)).id;
|
||||
} else {
|
||||
DEREF(dst).children[0] = DEREF(INDEX(key32_id_pair, args.src_ids, left)).id;
|
||||
DEREF(dst).children[1] = DEREF(INDEX(key32_id_pair, args.src_ids, right)).id;
|
||||
}
|
||||
if (id == 0)
|
||||
DEREF(dst).parent = VK_BVH_INVALID_NODE;
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
/*
|
||||
* Copyright © 2022 Konstantin Seurer
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#version 460
|
||||
|
||||
#include "leaf.h"
|
||||
@@ -0,0 +1,261 @@
|
||||
/*
|
||||
* Copyright © 2022 Konstantin Seurer
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "vk_build_interface.h"
|
||||
|
||||
layout(local_size_x_id = SUBGROUP_SIZE_ID, local_size_y = 1, local_size_z = 1) in;
|
||||
|
||||
layout(push_constant) uniform CONSTS {
|
||||
leaf_args args;
|
||||
};
|
||||
|
||||
bool
|
||||
build_triangle(inout vk_aabb bounds, VOID_REF dst_ptr, vk_bvh_geometry_data geom_data, uint32_t global_id)
|
||||
{
|
||||
bool is_valid = true;
|
||||
triangle_indices indices = load_indices(geom_data.indices, geom_data.index_format, global_id);
|
||||
|
||||
triangle_vertices vertices = load_vertices(geom_data.data, indices, geom_data.vertex_format, geom_data.stride);
|
||||
|
||||
if (geom_data.transform != NULL) {
|
||||
mat4 transform = mat4(1.0);
|
||||
|
||||
for (uint32_t col = 0; col < 4; col++)
|
||||
for (uint32_t row = 0; row < 3; row++)
|
||||
transform[col][row] = DEREF(INDEX(float, geom_data.transform, col + row * 4));
|
||||
|
||||
for (uint32_t i = 0; i < 3; i++)
|
||||
vertices.vertex[i] = transform * vertices.vertex[i];
|
||||
}
|
||||
|
||||
/* An inactive triangle is one for which the first (X) component of any vertex is NaN. If any
|
||||
* other vertex component is NaN, and the first is not, the behavior is undefined. We treat those
|
||||
* undefined cases as inactive to filter out NaNs. If the vertex format does not have a NaN
|
||||
* representation, then all triangles are considered active.
|
||||
*/
|
||||
if (any(isnan(vertices.vertex[0])) || any(isnan(vertices.vertex[1])) || any(isnan(vertices.vertex[2]))) {
|
||||
if (!VK_BUILD_FLAG(VK_BUILD_FLAG_ALWAYS_ACTIVE))
|
||||
return false;
|
||||
|
||||
is_valid = false;
|
||||
vertices.vertex[0] = vec4(0);
|
||||
vertices.vertex[1] = vec4(0);
|
||||
vertices.vertex[2] = vec4(0);
|
||||
}
|
||||
|
||||
REF(vk_ir_triangle_node) node = REF(vk_ir_triangle_node)(dst_ptr);
|
||||
|
||||
bounds.min = vec3(INFINITY);
|
||||
bounds.max = vec3(-INFINITY);
|
||||
|
||||
for (uint32_t coord = 0; coord < 3; coord++)
|
||||
for (uint32_t comp = 0; comp < 3; comp++) {
|
||||
DEREF(node).coords[coord][comp] = vertices.vertex[coord][comp];
|
||||
bounds.min[comp] = min(bounds.min[comp], vertices.vertex[coord][comp]);
|
||||
bounds.max[comp] = max(bounds.max[comp], vertices.vertex[coord][comp]);
|
||||
}
|
||||
|
||||
DEREF(node).base.aabb = bounds;
|
||||
DEREF(node).triangle_id = global_id;
|
||||
DEREF(node).geometry_id_and_flags = geom_data.geometry_id;
|
||||
|
||||
return is_valid;
|
||||
}
|
||||
|
||||
bool
|
||||
build_aabb(inout vk_aabb bounds, VOID_REF src_ptr, VOID_REF dst_ptr, uint32_t geometry_id, uint32_t global_id)
|
||||
{
|
||||
bool is_valid = true;
|
||||
REF(vk_ir_aabb_node) node = REF(vk_ir_aabb_node)(dst_ptr);
|
||||
|
||||
for (uint32_t vec = 0; vec < 2; vec++)
|
||||
for (uint32_t comp = 0; comp < 3; comp++) {
|
||||
float coord = DEREF(INDEX(float, src_ptr, comp + vec * 3));
|
||||
|
||||
if (vec == 0)
|
||||
bounds.min[comp] = coord;
|
||||
else
|
||||
bounds.max[comp] = coord;
|
||||
}
|
||||
|
||||
/* An inactive AABB is one for which the minimum X coordinate is NaN. If any other component is
|
||||
* NaN, and the first is not, the behavior is undefined. We treat those undefined cases as inactive
|
||||
* to filter out NaNs.
|
||||
*/
|
||||
if (any(isnan(bounds.min)) || any(isnan(bounds.max))) {
|
||||
if (!VK_BUILD_FLAG(VK_BUILD_FLAG_ALWAYS_ACTIVE))
|
||||
return false;
|
||||
|
||||
is_valid = false;
|
||||
bounds.min = vec3(0);
|
||||
bounds.max = vec3(0);
|
||||
}
|
||||
|
||||
DEREF(node).base.aabb = bounds;
|
||||
DEREF(node).primitive_id = global_id;
|
||||
DEREF(node).geometry_id_and_flags = geometry_id;
|
||||
|
||||
return is_valid;
|
||||
}
|
||||
|
||||
vk_aabb
|
||||
calculate_instance_node_bounds(vk_aabb blas_aabb, mat3x4 otw_matrix)
|
||||
{
|
||||
vk_aabb aabb;
|
||||
|
||||
for (uint32_t comp = 0; comp < 3; ++comp) {
|
||||
aabb.min[comp] = otw_matrix[comp][3];
|
||||
aabb.max[comp] = otw_matrix[comp][3];
|
||||
for (uint32_t col = 0; col < 3; ++col) {
|
||||
aabb.min[comp] +=
|
||||
min(otw_matrix[comp][col] * blas_aabb.min[col], otw_matrix[comp][col] * blas_aabb.max[col]);
|
||||
aabb.max[comp] +=
|
||||
max(otw_matrix[comp][col] * blas_aabb.min[col], otw_matrix[comp][col] * blas_aabb.max[col]);
|
||||
}
|
||||
}
|
||||
return aabb;
|
||||
}
|
||||
|
||||
bool
|
||||
build_instance(inout vk_aabb bounds, VOID_REF src_ptr, VOID_REF dst_ptr, uint32_t global_id)
|
||||
{
|
||||
REF(vk_ir_instance_node) node = REF(vk_ir_instance_node)(dst_ptr);
|
||||
|
||||
AccelerationStructureInstance instance = DEREF(REF(AccelerationStructureInstance)(src_ptr));
|
||||
|
||||
/* An inactive instance is one whose acceleration structure handle is VK_NULL_HANDLE. Since the active terminology is
|
||||
* only relevant for BVH updates, which we do not implement, we can also skip instances with mask == 0.
|
||||
*/
|
||||
if (instance.accelerationStructureReference == 0 || instance.custom_instance_and_mask < (1u << 24u))
|
||||
return false;
|
||||
|
||||
DEREF(node).base_ptr = instance.accelerationStructureReference;
|
||||
|
||||
mat4 transform = mat4(instance.transform);
|
||||
DEREF(node).otw_matrix = mat3x4(transform);
|
||||
|
||||
vk_aabb blas_aabb = DEREF(REF(vk_aabb)(instance.accelerationStructureReference + BVH_BOUNDS_OFFSET));
|
||||
|
||||
bounds = calculate_instance_node_bounds(blas_aabb, mat3x4(transform));
|
||||
|
||||
#ifdef CALCULATE_FINE_INSTANCE_NODE_BOUNDS
|
||||
vec3 blas_aabb_extent = blas_aabb.max - blas_aabb.min;
|
||||
float blas_aabb_volume = blas_aabb_extent.x * blas_aabb_extent.y * blas_aabb_extent.z;
|
||||
blas_aabb_volume *= abs(determinant(mat3(transform)));
|
||||
|
||||
vec3 bounds_extent = bounds.max - bounds.min;
|
||||
float bounds_volume = bounds_extent.x * bounds_extent.y * bounds_extent.z;
|
||||
|
||||
/* Only try calculating finer-grained instance node bounds if the volume of the transformed
|
||||
* instance AABB is significantly higher than the volume of the BLAS without transformations
|
||||
* applied. Otherwise, the finer-grained bounds won't be much smaller and the additional overhead
|
||||
* wouldn't be worth it.
|
||||
*/
|
||||
if (bounds_volume > 1.4f * blas_aabb_volume)
|
||||
bounds = CALCULATE_FINE_INSTANCE_NODE_BOUNDS(instance.accelerationStructureReference, mat3x4(transform));
|
||||
#endif
|
||||
|
||||
if (any(isnan(bounds.min)) || any(isnan(bounds.max)))
|
||||
return false;
|
||||
|
||||
DEREF(node).base.aabb = bounds;
|
||||
DEREF(node).custom_instance_and_mask = instance.custom_instance_and_mask;
|
||||
DEREF(node).sbt_offset_and_flags = instance.sbt_offset_and_flags;
|
||||
DEREF(node).instance_id = global_id;
|
||||
|
||||
if (!VK_BUILD_FLAG(VK_BUILD_FLAG_PROPAGATE_CULL_FLAGS))
|
||||
return true;
|
||||
|
||||
uint32_t root_flags = 0;
|
||||
if ((instance.sbt_offset_and_flags & (VK_GEOMETRY_INSTANCE_FORCE_OPAQUE_BIT_KHR << 24)) != 0)
|
||||
root_flags = VK_BVH_BOX_FLAG_ONLY_OPAQUE;
|
||||
else if ((instance.sbt_offset_and_flags & (VK_GEOMETRY_INSTANCE_FORCE_NO_OPAQUE_BIT_KHR << 24)) != 0)
|
||||
root_flags = VK_BVH_BOX_FLAG_NO_OPAQUE;
|
||||
else
|
||||
root_flags = DEREF(REF(uint32_t)(instance.accelerationStructureReference + ROOT_FLAGS_OFFSET));
|
||||
DEREF(node).root_flags = root_flags;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void
|
||||
main(void)
|
||||
{
|
||||
uint32_t global_id = gl_GlobalInvocationID.x;
|
||||
uint32_t primitive_id = args.geom_data.first_id + global_id;
|
||||
|
||||
uint32_t src_offset = global_id * args.geom_data.stride;
|
||||
|
||||
uint32_t dst_stride;
|
||||
uint32_t node_type;
|
||||
if (args.geom_data.geometry_type == VK_GEOMETRY_TYPE_TRIANGLES_KHR) {
|
||||
dst_stride = SIZEOF(vk_ir_triangle_node);
|
||||
node_type = vk_ir_node_triangle;
|
||||
} else if (args.geom_data.geometry_type == VK_GEOMETRY_TYPE_AABBS_KHR) {
|
||||
dst_stride = SIZEOF(vk_ir_aabb_node);
|
||||
node_type = vk_ir_node_aabb;
|
||||
} else {
|
||||
dst_stride = SIZEOF(vk_ir_instance_node);
|
||||
node_type = vk_ir_node_instance;
|
||||
}
|
||||
|
||||
uint32_t dst_offset = primitive_id * dst_stride;
|
||||
VOID_REF dst_ptr = OFFSET(args.bvh, dst_offset);
|
||||
|
||||
vk_aabb bounds;
|
||||
bool is_active;
|
||||
if (args.geom_data.geometry_type == VK_GEOMETRY_TYPE_TRIANGLES_KHR) {
|
||||
is_active = build_triangle(bounds, dst_ptr, args.geom_data, global_id);
|
||||
} else if (args.geom_data.geometry_type == VK_GEOMETRY_TYPE_AABBS_KHR) {
|
||||
VOID_REF src_ptr = OFFSET(args.geom_data.data, src_offset);
|
||||
is_active = build_aabb(bounds, src_ptr, dst_ptr, args.geom_data.geometry_id, global_id);
|
||||
} else {
|
||||
VOID_REF src_ptr = OFFSET(args.geom_data.data, src_offset);
|
||||
/* arrayOfPointers */
|
||||
if (args.geom_data.stride == 8) {
|
||||
src_ptr = DEREF(REF(VOID_REF)(src_ptr));
|
||||
}
|
||||
|
||||
is_active = build_instance(bounds, src_ptr, dst_ptr, global_id);
|
||||
}
|
||||
|
||||
if (VK_BUILD_FLAG(VK_BUILD_FLAG_ALWAYS_ACTIVE))
|
||||
is_active = true;
|
||||
|
||||
uint32_t id = is_active ? pack_ir_node_id(dst_offset, node_type) : VK_BVH_INVALID_NODE;
|
||||
if (VK_BUILD_FLAG(VK_BUILD_FLAG_64BIT_KEYS))
|
||||
DEREF(INDEX(key64_id_pair, args.ids, primitive_id)).id = id;
|
||||
else
|
||||
DEREF(INDEX(key32_id_pair, args.ids, primitive_id)).id = id;
|
||||
|
||||
uvec4 ballot = subgroupBallot(is_active);
|
||||
if (subgroupElect())
|
||||
atomicAdd(DEREF(args.header).active_leaf_count, subgroupBallotBitCount(ballot));
|
||||
|
||||
atomicMin(DEREF(args.header).min_bounds[0], to_emulated_float(bounds.min.x));
|
||||
atomicMin(DEREF(args.header).min_bounds[1], to_emulated_float(bounds.min.y));
|
||||
atomicMin(DEREF(args.header).min_bounds[2], to_emulated_float(bounds.min.z));
|
||||
atomicMax(DEREF(args.header).max_bounds[0], to_emulated_float(bounds.max.x));
|
||||
atomicMax(DEREF(args.header).max_bounds[1], to_emulated_float(bounds.max.y));
|
||||
atomicMax(DEREF(args.header).max_bounds[2], to_emulated_float(bounds.max.z));
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
# Copyright © 2022 Konstantin Seurer
|
||||
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
# source file, output name, defines
|
||||
bvh_shaders = [
|
||||
[
|
||||
'lbvh_generate_ir.comp',
|
||||
'lbvh_generate_ir',
|
||||
],
|
||||
[
|
||||
'lbvh_main.comp',
|
||||
'lbvh_main',
|
||||
],
|
||||
[
|
||||
'leaf.comp',
|
||||
'leaf',
|
||||
],
|
||||
[
|
||||
'morton.comp',
|
||||
'morton',
|
||||
],
|
||||
[
|
||||
'ploc_internal.comp',
|
||||
'ploc_internal',
|
||||
],
|
||||
[
|
||||
'hploc_internal.comp',
|
||||
'hploc_internal',
|
||||
],
|
||||
]
|
||||
|
||||
spirv_include_dir = dir_source_root + '/src/compiler/spirv'
|
||||
vk_bvh_include_dir = dir_source_root + '/src/vulkan/runtime/bvh'
|
||||
|
||||
vk_bvh_includes = files(
|
||||
'leaf.h',
|
||||
'vk_build_helpers.h',
|
||||
'vk_build_interface.h',
|
||||
'vk_bvh.h',
|
||||
'vk_debug.h',
|
||||
spirv_include_dir + '/spirv_internal_exts.h',
|
||||
)
|
||||
|
||||
vk_glsl_shader_extensions = [
|
||||
'GL_GOOGLE_include_directive',
|
||||
'GL_EXT_shader_explicit_arithmetic_types_int8',
|
||||
'GL_EXT_shader_explicit_arithmetic_types_int16',
|
||||
'GL_EXT_shader_explicit_arithmetic_types_int32',
|
||||
'GL_EXT_shader_explicit_arithmetic_types_int64',
|
||||
'GL_EXT_shader_explicit_arithmetic_types_float16',
|
||||
'GL_EXT_scalar_block_layout',
|
||||
'GL_EXT_buffer_reference',
|
||||
'GL_EXT_buffer_reference2',
|
||||
'GL_KHR_memory_scope_semantics',
|
||||
'GL_KHR_shader_subgroup_arithmetic',
|
||||
'GL_KHR_shader_subgroup_basic',
|
||||
'GL_KHR_shader_subgroup_shuffle',
|
||||
'GL_KHR_shader_subgroup_ballot',
|
||||
'GL_KHR_shader_subgroup_clustered',
|
||||
'GL_KHR_shader_subgroup_vote',
|
||||
'GL_EXT_shader_atomic_int64',
|
||||
'GL_EXT_spirv_intrinsics',
|
||||
]
|
||||
|
||||
vk_glsl_shader_preamble = []
|
||||
foreach ext : vk_glsl_shader_extensions
|
||||
vk_glsl_shader_preamble += ['-P#extension ' + ext + ' : require']
|
||||
endforeach
|
||||
|
||||
bvh_spv = []
|
||||
foreach s : bvh_shaders
|
||||
command = [
|
||||
prog_glslang, '-V', '-I' + vk_bvh_include_dir, '-I' + spirv_include_dir, '--target-env', 'spirv1.5', '-x', '-o', '@OUTPUT@', '@INPUT@'
|
||||
] + (with_mesa_debug ? ['-g'] : [])
|
||||
command += glslang_quiet
|
||||
command += vk_glsl_shader_preamble
|
||||
|
||||
bvh_spv += custom_target(
|
||||
s[1] + '.spv.h',
|
||||
input : s[0],
|
||||
output : s[1] + '.spv.h',
|
||||
command : command,
|
||||
depend_files: vk_bvh_includes
|
||||
)
|
||||
endforeach
|
||||
@@ -0,0 +1,124 @@
|
||||
/*
|
||||
* Copyright © 2022 Konstantin Seurer
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#version 460
|
||||
|
||||
#include "vk_build_interface.h"
|
||||
|
||||
layout(local_size_x_id = SUBGROUP_SIZE_ID, local_size_y = 1, local_size_z = 1) in;
|
||||
|
||||
layout(push_constant) uniform CONSTS {
|
||||
morton_args args;
|
||||
};
|
||||
|
||||
uint32_t
|
||||
morton_component(uint32_t x)
|
||||
{
|
||||
x = (x * 0x00010001u) & 0xFF0000FFu;
|
||||
x = (x * 0x00000101u) & 0x0F00F00Fu;
|
||||
x = (x * 0x00000011u) & 0xC30C30C3u;
|
||||
x = (x * 0x00000005u) & 0x49249249u;
|
||||
return x;
|
||||
}
|
||||
|
||||
uint64_t
|
||||
morton_component64(uint64_t x)
|
||||
{
|
||||
x = (x | x << 32) & 0x1f00000000fffful;
|
||||
x = (x | x << 16) & 0x1f0000ff0000fful;
|
||||
x = (x | x << 8) & 0x100f00f00f00f00ful;
|
||||
x = (x | x << 4) & 0x10c30c30c30c30c3ul;
|
||||
x = (x | x << 2) & 0x1249249249249249ul;
|
||||
return x;
|
||||
}
|
||||
|
||||
uint32_t
|
||||
morton_code(uint32_t x, uint32_t y, uint32_t z)
|
||||
{
|
||||
return (morton_component(x) << 2) | (morton_component(y) << 1) | morton_component(z);
|
||||
}
|
||||
|
||||
uint64_t
|
||||
morton_code64(uint64_t x, uint64_t y, uint64_t z)
|
||||
{
|
||||
return (morton_component64(x) << 2) | (morton_component64(y) << 1) | morton_component64(z);
|
||||
}
|
||||
|
||||
uint32_t
|
||||
lbvh_key(float x01, float y01, float z01)
|
||||
{
|
||||
return morton_code(uint32_t(x01 * 255.0), uint32_t(y01 * 255.0), uint32_t(z01 * 255.0)) << 8;
|
||||
}
|
||||
|
||||
uint64_t
|
||||
lbvh_key64(float x01, float y01, float z01)
|
||||
{
|
||||
return morton_code64(uint64_t(x01 * float(0x1fffff)), uint64_t(y01 * float(0x1fffff)), uint64_t(z01 * float(0x1fffff)));
|
||||
}
|
||||
|
||||
void
|
||||
main(void)
|
||||
{
|
||||
uint32_t global_id = gl_GlobalInvocationID.x;
|
||||
|
||||
uint32_t id;
|
||||
if (VK_BUILD_FLAG(VK_BUILD_FLAG_64BIT_KEYS))
|
||||
id = DEREF(INDEX(key64_id_pair, args.ids, global_id)).id;
|
||||
else
|
||||
id = DEREF(INDEX(key32_id_pair, args.ids, global_id)).id;
|
||||
|
||||
uint32_t key;
|
||||
uint64_t key64;
|
||||
if (id != VK_BVH_INVALID_NODE) {
|
||||
vk_aabb bounds = DEREF(REF(vk_ir_node)OFFSET(args.bvh, ir_id_to_offset(id))).aabb;
|
||||
vec3 center = (bounds.min + bounds.max) * 0.5;
|
||||
|
||||
vk_aabb bvh_bounds;
|
||||
bvh_bounds.min.x = from_emulated_float(DEREF(args.header).min_bounds[0]);
|
||||
bvh_bounds.min.y = from_emulated_float(DEREF(args.header).min_bounds[1]);
|
||||
bvh_bounds.min.z = from_emulated_float(DEREF(args.header).min_bounds[2]);
|
||||
bvh_bounds.max.x = from_emulated_float(DEREF(args.header).max_bounds[0]);
|
||||
bvh_bounds.max.y = from_emulated_float(DEREF(args.header).max_bounds[1]);
|
||||
bvh_bounds.max.z = from_emulated_float(DEREF(args.header).max_bounds[2]);
|
||||
|
||||
vec3 normalized_center = (center - bvh_bounds.min) / (bvh_bounds.max - bvh_bounds.min);
|
||||
|
||||
if (VK_BUILD_FLAG(VK_BUILD_FLAG_64BIT_KEYS))
|
||||
key64 = lbvh_key64(normalized_center.x, normalized_center.y, normalized_center.z);
|
||||
else
|
||||
key = lbvh_key(normalized_center.x, normalized_center.y, normalized_center.z);
|
||||
} else {
|
||||
/* Move null instances to the end to avoid mixing null instances with active instances. This
|
||||
* way, we can skip early during traversal.
|
||||
*/
|
||||
key = 0xFFFFFFFFu;
|
||||
key64 = 0xFFFFFFFFFFFFFFFFul;
|
||||
}
|
||||
if (VK_BUILD_FLAG(VK_BUILD_FLAG_64BIT_KEYS)) {
|
||||
REF(key64_id_pair) key_id = INDEX(key64_id_pair, args.ids, global_id);
|
||||
DEREF(key_id).key_lo = unpackUint2x32(key64).x;
|
||||
DEREF(key_id).key_hi = unpackUint2x32(key64).y;
|
||||
} else {
|
||||
DEREF(INDEX(key32_id_pair, args.ids, global_id)).key = key;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,446 @@
|
||||
/*
|
||||
* Copyright © 2022 Bas Nieuwenhuizen
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#version 460
|
||||
|
||||
layout(local_size_x = 1024, local_size_y = 1, local_size_z = 1) in;
|
||||
|
||||
#define USE_GLOBAL_SYNC
|
||||
#include "vk_build_interface.h"
|
||||
|
||||
TYPE(ploc_prefix_scan_partition, 4);
|
||||
|
||||
layout(push_constant) uniform CONSTS
|
||||
{
|
||||
ploc_args args;
|
||||
};
|
||||
|
||||
shared uint32_t exclusive_prefix_sum;
|
||||
shared uint32_t aggregate_sums[PLOC_SUBGROUPS_PER_WORKGROUP];
|
||||
shared uint32_t aggregate_sums2[PLOC_SUBGROUPS_PER_WORKGROUP];
|
||||
|
||||
/*
|
||||
* Global prefix scan over all workgroups to find out the index of the collapsed node to write.
|
||||
* See https://research.nvidia.com/sites/default/files/publications/nvr-2016-002.pdf
|
||||
* One partition = one workgroup in this case.
|
||||
*/
|
||||
uint32_t
|
||||
prefix_scan(uvec4 ballot, REF(ploc_prefix_scan_partition) partitions, uint32_t task_index)
|
||||
{
|
||||
if (gl_LocalInvocationIndex == 0) {
|
||||
exclusive_prefix_sum = 0;
|
||||
if (task_index >= gl_WorkGroupSize.x) {
|
||||
REF(ploc_prefix_scan_partition) current_partition =
|
||||
REF(ploc_prefix_scan_partition)(INDEX(ploc_prefix_scan_partition, partitions, task_index / gl_WorkGroupSize.x));
|
||||
|
||||
REF(ploc_prefix_scan_partition) previous_partition = current_partition - 1;
|
||||
|
||||
while (true) {
|
||||
/* See if this previous workgroup already set their inclusive sum */
|
||||
if (atomicLoad(DEREF(previous_partition).inclusive_sum, gl_ScopeDevice,
|
||||
gl_StorageSemanticsBuffer,
|
||||
gl_SemanticsAcquire | gl_SemanticsMakeVisible) != 0xFFFFFFFF) {
|
||||
atomicAdd(exclusive_prefix_sum, DEREF(previous_partition).inclusive_sum);
|
||||
break;
|
||||
} else {
|
||||
atomicAdd(exclusive_prefix_sum, DEREF(previous_partition).aggregate);
|
||||
previous_partition -= 1;
|
||||
}
|
||||
}
|
||||
/* Set the inclusive sum for the next workgroups */
|
||||
atomicStore(DEREF(current_partition).inclusive_sum,
|
||||
DEREF(current_partition).aggregate + exclusive_prefix_sum, gl_ScopeDevice,
|
||||
gl_StorageSemanticsBuffer, gl_SemanticsRelease | gl_SemanticsMakeAvailable);
|
||||
}
|
||||
}
|
||||
|
||||
if (subgroupElect())
|
||||
aggregate_sums[gl_SubgroupID] = subgroupBallotBitCount(ballot);
|
||||
barrier();
|
||||
|
||||
if (PLOC_SUBGROUPS_PER_WORKGROUP <= SUBGROUP_SIZE) {
|
||||
if (gl_LocalInvocationID.x < PLOC_SUBGROUPS_PER_WORKGROUP) {
|
||||
aggregate_sums[gl_LocalInvocationID.x] =
|
||||
exclusive_prefix_sum + subgroupExclusiveAdd(aggregate_sums[gl_LocalInvocationID.x]);
|
||||
}
|
||||
} else {
|
||||
/* If the length of aggregate_sums[] is larger than SUBGROUP_SIZE,
|
||||
* the prefix scan can't be done simply by subgroupExclusiveAdd.
|
||||
*/
|
||||
if (gl_LocalInvocationID.x < PLOC_SUBGROUPS_PER_WORKGROUP)
|
||||
aggregate_sums2[gl_LocalInvocationID.x] = aggregate_sums[gl_LocalInvocationID.x];
|
||||
barrier();
|
||||
|
||||
/* Hillis Steele inclusive scan on aggregate_sums2 */
|
||||
for (uint32_t stride = 1; stride < PLOC_SUBGROUPS_PER_WORKGROUP; stride *= 2) {
|
||||
uint32_t value = 0;
|
||||
if (gl_LocalInvocationID.x >= stride && gl_LocalInvocationID.x < PLOC_SUBGROUPS_PER_WORKGROUP)
|
||||
value = aggregate_sums2[gl_LocalInvocationID.x - stride];
|
||||
barrier();
|
||||
if (gl_LocalInvocationID.x < PLOC_SUBGROUPS_PER_WORKGROUP)
|
||||
aggregate_sums2[gl_LocalInvocationID.x] += value;
|
||||
barrier();
|
||||
}
|
||||
|
||||
/* Adapt to exclusive and add the prefix_sum from previous workgroups */
|
||||
if (gl_LocalInvocationID.x < PLOC_SUBGROUPS_PER_WORKGROUP) {
|
||||
if (gl_LocalInvocationID.x == 0)
|
||||
aggregate_sums[gl_LocalInvocationID.x] = exclusive_prefix_sum;
|
||||
else
|
||||
aggregate_sums[gl_LocalInvocationID.x] = exclusive_prefix_sum + aggregate_sums2[gl_LocalInvocationID.x - 1];
|
||||
}
|
||||
}
|
||||
barrier();
|
||||
|
||||
return aggregate_sums[gl_SubgroupID] + subgroupBallotExclusiveBitCount(ballot);
|
||||
}
|
||||
|
||||
/* Relative cost of increasing the BVH depth. Deep BVHs will require more backtracking. */
|
||||
#define BVH_LEVEL_COST 0.2
|
||||
|
||||
uint32_t
|
||||
push_node(uint32_t children[2], vk_aabb bounds[2])
|
||||
{
|
||||
uint32_t internal_node_index = atomicAdd(DEREF(args.header).ir_internal_node_count, 1);
|
||||
uint32_t dst_offset = args.internal_node_offset + internal_node_index * SIZEOF(vk_ir_box_node);
|
||||
uint32_t dst_id = pack_ir_node_id(dst_offset, vk_ir_node_internal);
|
||||
REF(vk_ir_box_node) dst_node = REF(vk_ir_box_node)(OFFSET(args.bvh, dst_offset));
|
||||
|
||||
vk_aabb total_bounds;
|
||||
total_bounds.min = vec3(INFINITY);
|
||||
total_bounds.max = vec3(-INFINITY);
|
||||
|
||||
uint32_t dst_flags = VK_BVH_BOX_FLAG_ONLY_OPAQUE | VK_BVH_BOX_FLAG_NO_OPAQUE;
|
||||
|
||||
for (uint i = 0; i < 2; ++i) {
|
||||
total_bounds.min = min(total_bounds.min, bounds[i].min);
|
||||
total_bounds.max = max(total_bounds.max, bounds[i].max);
|
||||
|
||||
DEREF(dst_node).children[i] = children[i];
|
||||
|
||||
if (VK_BUILD_FLAG(VK_BUILD_FLAG_PROPAGATE_CULL_FLAGS))
|
||||
dst_flags &= fetch_child_flags(args.bvh, children[i]);
|
||||
}
|
||||
|
||||
DEREF(dst_node).base.aabb = total_bounds;
|
||||
DEREF(dst_node).bvh_offset = VK_UNKNOWN_BVH_OFFSET;
|
||||
|
||||
if (VK_BUILD_FLAG(VK_BUILD_FLAG_PROPAGATE_CULL_FLAGS))
|
||||
DEREF(dst_node).flags = dst_flags;
|
||||
|
||||
return dst_id;
|
||||
}
|
||||
|
||||
#define PLOC_NEIGHBOURHOOD 16
|
||||
#define PLOC_OFFSET_MASK ((1 << 5) - 1)
|
||||
|
||||
uint32_t
|
||||
encode_neighbour_offset(float sah, uint32_t i, uint32_t j)
|
||||
{
|
||||
int32_t offset = int32_t(j) - int32_t(i);
|
||||
uint32_t encoded_offset = offset + PLOC_NEIGHBOURHOOD - (offset > 0 ? 1 : 0);
|
||||
return (floatBitsToUint(sah) & (~PLOC_OFFSET_MASK)) | encoded_offset;
|
||||
}
|
||||
|
||||
int32_t
|
||||
decode_neighbour_offset(uint32_t encoded_offset)
|
||||
{
|
||||
int32_t offset = int32_t(encoded_offset & PLOC_OFFSET_MASK) - PLOC_NEIGHBOURHOOD;
|
||||
return offset + (offset >= 0 ? 1 : 0);
|
||||
}
|
||||
|
||||
#define NUM_PLOC_LDS_ITEMS PLOC_WORKGROUP_SIZE + 4 * PLOC_NEIGHBOURHOOD
|
||||
|
||||
shared vk_aabb shared_bounds[NUM_PLOC_LDS_ITEMS];
|
||||
shared uint32_t nearest_neighbour_indices[NUM_PLOC_LDS_ITEMS];
|
||||
|
||||
uint32_t
|
||||
load_id(VOID_REF ids, uint32_t iter, uint32_t index)
|
||||
{
|
||||
if (iter == 0) {
|
||||
if (VK_BUILD_FLAG(VK_BUILD_FLAG_64BIT_KEYS))
|
||||
return DEREF(REF(key64_id_pair)(INDEX(key64_id_pair, ids, index))).id;
|
||||
else
|
||||
return DEREF(REF(key32_id_pair)(INDEX(key32_id_pair, ids, index))).id;
|
||||
} else {
|
||||
return DEREF(REF(uint32_t)(INDEX(uint32_t, ids, index)));
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
load_bounds(VOID_REF ids, uint32_t iter, uint32_t task_index, uint32_t lds_base,
|
||||
uint32_t neighbourhood_overlap, uint32_t search_bound)
|
||||
{
|
||||
for (uint32_t i = task_index - 2 * neighbourhood_overlap; i < search_bound;
|
||||
i += gl_WorkGroupSize.x) {
|
||||
uint32_t id = load_id(ids, iter, i);
|
||||
if (id == VK_BVH_INVALID_NODE)
|
||||
continue;
|
||||
|
||||
VOID_REF addr = OFFSET(args.bvh, ir_id_to_offset(id));
|
||||
REF(vk_ir_node) node = REF(vk_ir_node)(addr);
|
||||
|
||||
shared_bounds[i - lds_base] = DEREF(node).aabb;
|
||||
}
|
||||
}
|
||||
|
||||
float
|
||||
combined_node_cost(uint32_t lds_base, uint32_t i, uint32_t j)
|
||||
{
|
||||
vk_aabb combined_bounds;
|
||||
combined_bounds.min = min(shared_bounds[i - lds_base].min, shared_bounds[j - lds_base].min);
|
||||
combined_bounds.max = max(shared_bounds[i - lds_base].max, shared_bounds[j - lds_base].max);
|
||||
return aabb_surface_area(combined_bounds);
|
||||
}
|
||||
|
||||
shared uint32_t shared_aggregate_sum;
|
||||
|
||||
void
|
||||
main(void)
|
||||
{
|
||||
VOID_REF src_ids = args.ids_0;
|
||||
VOID_REF dst_ids = args.ids_1;
|
||||
|
||||
/* We try to use LBVH for BVHs where we know there will be less than 5 leaves,
|
||||
* but sometimes all leaves might be inactive */
|
||||
if (DEREF(args.header).active_leaf_count <= 2) {
|
||||
if (gl_GlobalInvocationID.x == 0) {
|
||||
uint32_t internal_node_index = atomicAdd(DEREF(args.header).ir_internal_node_count, 1);
|
||||
uint32_t dst_offset = args.internal_node_offset + internal_node_index * SIZEOF(vk_ir_box_node);
|
||||
REF(vk_ir_box_node) dst_node = REF(vk_ir_box_node)(OFFSET(args.bvh, dst_offset));
|
||||
|
||||
vk_aabb total_bounds;
|
||||
total_bounds.min = vec3(INFINITY);
|
||||
total_bounds.max = vec3(-INFINITY);
|
||||
|
||||
uint32_t i = 0;
|
||||
for (; i < DEREF(args.header).active_leaf_count; i++) {
|
||||
uint32_t child_id;
|
||||
if (VK_BUILD_FLAG(VK_BUILD_FLAG_64BIT_KEYS))
|
||||
child_id = DEREF(INDEX(key64_id_pair, src_ids, i)).id;
|
||||
else
|
||||
child_id = DEREF(INDEX(key32_id_pair, src_ids, i)).id;
|
||||
|
||||
if (child_id != VK_BVH_INVALID_NODE) {
|
||||
VOID_REF node = OFFSET(args.bvh, ir_id_to_offset(child_id));
|
||||
REF(vk_ir_node) child = REF(vk_ir_node)(node);
|
||||
vk_aabb bounds = DEREF(child).aabb;
|
||||
|
||||
total_bounds.min = min(total_bounds.min, bounds.min);
|
||||
total_bounds.max = max(total_bounds.max, bounds.max);
|
||||
}
|
||||
|
||||
DEREF(dst_node).children[i] = child_id;
|
||||
}
|
||||
for (; i < 2; i++)
|
||||
DEREF(dst_node).children[i] = VK_BVH_INVALID_NODE;
|
||||
|
||||
DEREF(dst_node).base.aabb = total_bounds;
|
||||
DEREF(dst_node).bvh_offset = VK_UNKNOWN_BVH_OFFSET;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
/* Only initialize sync_data once per workgroup. For intra-workgroup synchronization,
|
||||
* fetch_task contains a workgroup-scoped control+memory barrier.
|
||||
*/
|
||||
if (gl_LocalInvocationIndex == 0) {
|
||||
atomicCompSwap(DEREF(args.header).sync_data.task_counts[0], 0xFFFFFFFF,
|
||||
DEREF(args.header).active_leaf_count);
|
||||
atomicCompSwap(DEREF(args.header).sync_data.current_phase_end_counter, 0xFFFFFFFF,
|
||||
DIV_ROUND_UP(DEREF(args.header).active_leaf_count, gl_WorkGroupSize.x));
|
||||
}
|
||||
|
||||
REF(ploc_prefix_scan_partition)
|
||||
partitions = REF(ploc_prefix_scan_partition)(args.prefix_scan_partitions);
|
||||
|
||||
uint32_t task_index = fetch_task(args.header, false);
|
||||
|
||||
for (uint iter = 0;; ++iter) {
|
||||
if (task_index == TASK_INDEX_INVALID)
|
||||
break;
|
||||
|
||||
/* Find preferred partners and merge them */
|
||||
PHASE (args.header) {
|
||||
uint32_t current_task_count = task_count(args.header);
|
||||
uint32_t base_index = task_index - gl_LocalInvocationID.x;
|
||||
uint32_t neighbourhood_overlap = min(PLOC_NEIGHBOURHOOD, base_index);
|
||||
uint32_t double_neighbourhood_overlap = min(2 * PLOC_NEIGHBOURHOOD, base_index);
|
||||
/* Upper bound to where valid nearest node indices are written. */
|
||||
uint32_t write_bound =
|
||||
min(current_task_count, base_index + gl_WorkGroupSize.x + PLOC_NEIGHBOURHOOD);
|
||||
/* Upper bound to where valid nearest node indices are searched. */
|
||||
uint32_t search_bound =
|
||||
min(current_task_count, base_index + gl_WorkGroupSize.x + 2 * PLOC_NEIGHBOURHOOD);
|
||||
uint32_t lds_base = base_index - double_neighbourhood_overlap;
|
||||
|
||||
load_bounds(src_ids, iter, task_index, lds_base, neighbourhood_overlap, search_bound);
|
||||
|
||||
for (uint32_t i = gl_LocalInvocationID.x; i < NUM_PLOC_LDS_ITEMS; i += gl_WorkGroupSize.x)
|
||||
nearest_neighbour_indices[i] = 0xFFFFFFFF;
|
||||
barrier();
|
||||
|
||||
for (uint32_t i = task_index - double_neighbourhood_overlap; i < write_bound;
|
||||
i += gl_WorkGroupSize.x) {
|
||||
uint32_t right_bound = min(search_bound - 1 - i, PLOC_NEIGHBOURHOOD);
|
||||
|
||||
uint32_t fallback_pair = i == 0 ? (i + 1) : (i - 1);
|
||||
uint32_t min_offset = encode_neighbour_offset(INFINITY, i, fallback_pair);
|
||||
|
||||
for (uint32_t j = max(i + 1, base_index - neighbourhood_overlap); j <= i + right_bound;
|
||||
++j) {
|
||||
|
||||
float sah = combined_node_cost(lds_base, i, j);
|
||||
uint32_t i_encoded_offset = encode_neighbour_offset(sah, i, j);
|
||||
uint32_t j_encoded_offset = encode_neighbour_offset(sah, j, i);
|
||||
min_offset = min(min_offset, i_encoded_offset);
|
||||
atomicMin(nearest_neighbour_indices[j - lds_base], j_encoded_offset);
|
||||
}
|
||||
if (i >= base_index - neighbourhood_overlap)
|
||||
atomicMin(nearest_neighbour_indices[i - lds_base], min_offset);
|
||||
}
|
||||
|
||||
if (gl_LocalInvocationID.x == 0)
|
||||
shared_aggregate_sum = 0;
|
||||
barrier();
|
||||
|
||||
for (uint32_t i = task_index - neighbourhood_overlap; i < write_bound;
|
||||
i += gl_WorkGroupSize.x) {
|
||||
uint32_t left_bound = min(i, PLOC_NEIGHBOURHOOD);
|
||||
uint32_t right_bound = min(search_bound - 1 - i, PLOC_NEIGHBOURHOOD);
|
||||
/*
|
||||
* Workaround for a worst-case scenario in PLOC: If the combined area of
|
||||
* all nodes (in the neighbourhood) is the same, then the chosen nearest
|
||||
* neighbour is the first neighbour. However, this means that no nodes
|
||||
* except the first two will find each other as nearest neighbour. Therefore,
|
||||
* only one node is contained in each BVH level. By first testing if the immediate
|
||||
* neighbour on one side is the nearest, all immediate neighbours will be merged
|
||||
* on every step.
|
||||
*/
|
||||
uint32_t preferred_pair;
|
||||
if ((i & 1) != 0)
|
||||
preferred_pair = i - min(left_bound, 1);
|
||||
else
|
||||
preferred_pair = i + min(right_bound, 1);
|
||||
|
||||
if (preferred_pair != i) {
|
||||
uint32_t encoded_min_sah =
|
||||
nearest_neighbour_indices[i - lds_base] & (~PLOC_OFFSET_MASK);
|
||||
float sah = combined_node_cost(lds_base, i, preferred_pair);
|
||||
uint32_t encoded_sah = floatBitsToUint(sah) & (~PLOC_OFFSET_MASK);
|
||||
uint32_t encoded_offset = encode_neighbour_offset(sah, i, preferred_pair);
|
||||
if (encoded_sah <= encoded_min_sah) {
|
||||
nearest_neighbour_indices[i - lds_base] = encoded_offset;
|
||||
}
|
||||
}
|
||||
}
|
||||
barrier();
|
||||
|
||||
bool has_valid_node = true;
|
||||
|
||||
if (task_index < current_task_count) {
|
||||
uint32_t base_index = task_index - gl_LocalInvocationID.x;
|
||||
|
||||
uint32_t neighbour_index =
|
||||
task_index +
|
||||
decode_neighbour_offset(nearest_neighbour_indices[task_index - lds_base]);
|
||||
uint32_t other_neighbour_index =
|
||||
neighbour_index +
|
||||
decode_neighbour_offset(nearest_neighbour_indices[neighbour_index - lds_base]);
|
||||
uint32_t id = load_id(src_ids, iter, task_index);
|
||||
if (other_neighbour_index == task_index) {
|
||||
if (task_index < neighbour_index) {
|
||||
uint32_t neighbour_id = load_id(src_ids, iter, neighbour_index);
|
||||
uint32_t children[2] = {id, neighbour_id};
|
||||
vk_aabb bounds[2] = {shared_bounds[task_index - lds_base], shared_bounds[neighbour_index - lds_base]};
|
||||
|
||||
DEREF(REF(uint32_t)(INDEX(uint32_t, dst_ids, task_index))) = push_node(children, bounds);
|
||||
DEREF(REF(uint32_t)(INDEX(uint32_t, dst_ids, neighbour_index))) =
|
||||
VK_BVH_INVALID_NODE;
|
||||
} else {
|
||||
/* We still store in the other case so we don't destroy the node id needed to
|
||||
* create the internal node */
|
||||
has_valid_node = false;
|
||||
}
|
||||
} else {
|
||||
DEREF(REF(uint32_t)(INDEX(uint32_t, dst_ids, task_index))) = id;
|
||||
}
|
||||
|
||||
/* Compact - prepare prefix scan */
|
||||
uvec4 ballot = subgroupBallot(has_valid_node);
|
||||
|
||||
uint32_t aggregate_sum = subgroupBallotBitCount(ballot);
|
||||
if (subgroupElect())
|
||||
atomicAdd(shared_aggregate_sum, aggregate_sum);
|
||||
}
|
||||
|
||||
barrier();
|
||||
/*
|
||||
* The paper proposes initializing all partitions to an invalid state
|
||||
* and only computing aggregates afterwards. We skip that step and
|
||||
* initialize the partitions to a valid state. This also simplifies
|
||||
* the look-back, as there will never be any blocking due to invalid
|
||||
* partitions.
|
||||
*/
|
||||
if (gl_LocalInvocationIndex == 0) {
|
||||
REF(ploc_prefix_scan_partition)
|
||||
current_partition = REF(ploc_prefix_scan_partition)(
|
||||
INDEX(ploc_prefix_scan_partition, partitions, task_index / gl_WorkGroupSize.x));
|
||||
DEREF(current_partition).aggregate = shared_aggregate_sum;
|
||||
if (task_index < gl_WorkGroupSize.x) {
|
||||
DEREF(current_partition).inclusive_sum = shared_aggregate_sum;
|
||||
} else {
|
||||
DEREF(current_partition).inclusive_sum = 0xFFFFFFFF;
|
||||
}
|
||||
}
|
||||
|
||||
if (task_index == 0)
|
||||
set_next_task_count(args.header, task_count(args.header));
|
||||
}
|
||||
|
||||
/* Compact - prefix scan and update */
|
||||
PHASE (args.header) {
|
||||
uint32_t current_task_count = task_count(args.header);
|
||||
|
||||
uint32_t id = task_index < current_task_count
|
||||
? DEREF(REF(uint32_t)(INDEX(uint32_t, dst_ids, task_index)))
|
||||
: VK_BVH_INVALID_NODE;
|
||||
uvec4 ballot = subgroupBallot(id != VK_BVH_INVALID_NODE);
|
||||
|
||||
uint32_t new_offset = prefix_scan(ballot, partitions, task_index);
|
||||
if (task_index >= current_task_count)
|
||||
continue;
|
||||
|
||||
if (id != VK_BVH_INVALID_NODE) {
|
||||
DEREF(REF(uint32_t)(INDEX(uint32_t, src_ids, new_offset))) = id;
|
||||
++new_offset;
|
||||
}
|
||||
|
||||
if (task_index == current_task_count - 1) {
|
||||
set_next_task_count(args.header, new_offset);
|
||||
if (new_offset == 1)
|
||||
DEREF(args.header).sync_data.next_phase_exit_flag = 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,553 @@
|
||||
/*
|
||||
* Copyright © 2022 Konstantin Seurer
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#ifndef VK_BVH_BUILD_HELPERS_H
|
||||
#define VK_BVH_BUILD_HELPERS_H
|
||||
|
||||
#include "vk_bvh.h"
|
||||
|
||||
#define VK_FORMAT_UNDEFINED 0
|
||||
#define VK_FORMAT_R4G4_UNORM_PACK8 1
|
||||
#define VK_FORMAT_R4G4B4A4_UNORM_PACK16 2
|
||||
#define VK_FORMAT_B4G4R4A4_UNORM_PACK16 3
|
||||
#define VK_FORMAT_R5G6B5_UNORM_PACK16 4
|
||||
#define VK_FORMAT_B5G6R5_UNORM_PACK16 5
|
||||
#define VK_FORMAT_R5G5B5A1_UNORM_PACK16 6
|
||||
#define VK_FORMAT_B5G5R5A1_UNORM_PACK16 7
|
||||
#define VK_FORMAT_A1R5G5B5_UNORM_PACK16 8
|
||||
#define VK_FORMAT_R8_UNORM 9
|
||||
#define VK_FORMAT_R8_SNORM 10
|
||||
#define VK_FORMAT_R8_USCALED 11
|
||||
#define VK_FORMAT_R8_SSCALED 12
|
||||
#define VK_FORMAT_R8_UINT 13
|
||||
#define VK_FORMAT_R8_SINT 14
|
||||
#define VK_FORMAT_R8_SRGB 15
|
||||
#define VK_FORMAT_R8G8_UNORM 16
|
||||
#define VK_FORMAT_R8G8_SNORM 17
|
||||
#define VK_FORMAT_R8G8_USCALED 18
|
||||
#define VK_FORMAT_R8G8_SSCALED 19
|
||||
#define VK_FORMAT_R8G8_UINT 20
|
||||
#define VK_FORMAT_R8G8_SINT 21
|
||||
#define VK_FORMAT_R8G8_SRGB 22
|
||||
#define VK_FORMAT_R8G8B8_UNORM 23
|
||||
#define VK_FORMAT_R8G8B8_SNORM 24
|
||||
#define VK_FORMAT_R8G8B8_USCALED 25
|
||||
#define VK_FORMAT_R8G8B8_SSCALED 26
|
||||
#define VK_FORMAT_R8G8B8_UINT 27
|
||||
#define VK_FORMAT_R8G8B8_SINT 28
|
||||
#define VK_FORMAT_R8G8B8_SRGB 29
|
||||
#define VK_FORMAT_B8G8R8_UNORM 30
|
||||
#define VK_FORMAT_B8G8R8_SNORM 31
|
||||
#define VK_FORMAT_B8G8R8_USCALED 32
|
||||
#define VK_FORMAT_B8G8R8_SSCALED 33
|
||||
#define VK_FORMAT_B8G8R8_UINT 34
|
||||
#define VK_FORMAT_B8G8R8_SINT 35
|
||||
#define VK_FORMAT_B8G8R8_SRGB 36
|
||||
#define VK_FORMAT_R8G8B8A8_UNORM 37
|
||||
#define VK_FORMAT_R8G8B8A8_SNORM 38
|
||||
#define VK_FORMAT_R8G8B8A8_USCALED 39
|
||||
#define VK_FORMAT_R8G8B8A8_SSCALED 40
|
||||
#define VK_FORMAT_R8G8B8A8_UINT 41
|
||||
#define VK_FORMAT_R8G8B8A8_SINT 42
|
||||
#define VK_FORMAT_R8G8B8A8_SRGB 43
|
||||
#define VK_FORMAT_B8G8R8A8_UNORM 44
|
||||
#define VK_FORMAT_B8G8R8A8_SNORM 45
|
||||
#define VK_FORMAT_B8G8R8A8_USCALED 46
|
||||
#define VK_FORMAT_B8G8R8A8_SSCALED 47
|
||||
#define VK_FORMAT_B8G8R8A8_UINT 48
|
||||
#define VK_FORMAT_B8G8R8A8_SINT 49
|
||||
#define VK_FORMAT_B8G8R8A8_SRGB 50
|
||||
#define VK_FORMAT_A8B8G8R8_UNORM_PACK32 51
|
||||
#define VK_FORMAT_A8B8G8R8_SNORM_PACK32 52
|
||||
#define VK_FORMAT_A8B8G8R8_USCALED_PACK32 53
|
||||
#define VK_FORMAT_A8B8G8R8_SSCALED_PACK32 54
|
||||
#define VK_FORMAT_A8B8G8R8_UINT_PACK32 55
|
||||
#define VK_FORMAT_A8B8G8R8_SINT_PACK32 56
|
||||
#define VK_FORMAT_A8B8G8R8_SRGB_PACK32 57
|
||||
#define VK_FORMAT_A2R10G10B10_UNORM_PACK32 58
|
||||
#define VK_FORMAT_A2R10G10B10_SNORM_PACK32 59
|
||||
#define VK_FORMAT_A2R10G10B10_USCALED_PACK32 60
|
||||
#define VK_FORMAT_A2R10G10B10_SSCALED_PACK32 61
|
||||
#define VK_FORMAT_A2R10G10B10_UINT_PACK32 62
|
||||
#define VK_FORMAT_A2R10G10B10_SINT_PACK32 63
|
||||
#define VK_FORMAT_A2B10G10R10_UNORM_PACK32 64
|
||||
#define VK_FORMAT_A2B10G10R10_SNORM_PACK32 65
|
||||
#define VK_FORMAT_A2B10G10R10_USCALED_PACK32 66
|
||||
#define VK_FORMAT_A2B10G10R10_SSCALED_PACK32 67
|
||||
#define VK_FORMAT_A2B10G10R10_UINT_PACK32 68
|
||||
#define VK_FORMAT_A2B10G10R10_SINT_PACK32 69
|
||||
#define VK_FORMAT_R16_UNORM 70
|
||||
#define VK_FORMAT_R16_SNORM 71
|
||||
#define VK_FORMAT_R16_USCALED 72
|
||||
#define VK_FORMAT_R16_SSCALED 73
|
||||
#define VK_FORMAT_R16_UINT 74
|
||||
#define VK_FORMAT_R16_SINT 75
|
||||
#define VK_FORMAT_R16_SFLOAT 76
|
||||
#define VK_FORMAT_R16G16_UNORM 77
|
||||
#define VK_FORMAT_R16G16_SNORM 78
|
||||
#define VK_FORMAT_R16G16_USCALED 79
|
||||
#define VK_FORMAT_R16G16_SSCALED 80
|
||||
#define VK_FORMAT_R16G16_UINT 81
|
||||
#define VK_FORMAT_R16G16_SINT 82
|
||||
#define VK_FORMAT_R16G16_SFLOAT 83
|
||||
#define VK_FORMAT_R16G16B16_UNORM 84
|
||||
#define VK_FORMAT_R16G16B16_SNORM 85
|
||||
#define VK_FORMAT_R16G16B16_USCALED 86
|
||||
#define VK_FORMAT_R16G16B16_SSCALED 87
|
||||
#define VK_FORMAT_R16G16B16_UINT 88
|
||||
#define VK_FORMAT_R16G16B16_SINT 89
|
||||
#define VK_FORMAT_R16G16B16_SFLOAT 90
|
||||
#define VK_FORMAT_R16G16B16A16_UNORM 91
|
||||
#define VK_FORMAT_R16G16B16A16_SNORM 92
|
||||
#define VK_FORMAT_R16G16B16A16_USCALED 93
|
||||
#define VK_FORMAT_R16G16B16A16_SSCALED 94
|
||||
#define VK_FORMAT_R16G16B16A16_UINT 95
|
||||
#define VK_FORMAT_R16G16B16A16_SINT 96
|
||||
#define VK_FORMAT_R16G16B16A16_SFLOAT 97
|
||||
#define VK_FORMAT_R32_UINT 98
|
||||
#define VK_FORMAT_R32_SINT 99
|
||||
#define VK_FORMAT_R32_SFLOAT 100
|
||||
#define VK_FORMAT_R32G32_UINT 101
|
||||
#define VK_FORMAT_R32G32_SINT 102
|
||||
#define VK_FORMAT_R32G32_SFLOAT 103
|
||||
#define VK_FORMAT_R32G32B32_UINT 104
|
||||
#define VK_FORMAT_R32G32B32_SINT 105
|
||||
#define VK_FORMAT_R32G32B32_SFLOAT 106
|
||||
#define VK_FORMAT_R32G32B32A32_UINT 107
|
||||
#define VK_FORMAT_R32G32B32A32_SINT 108
|
||||
#define VK_FORMAT_R32G32B32A32_SFLOAT 109
|
||||
#define VK_FORMAT_R64_UINT 110
|
||||
#define VK_FORMAT_R64_SINT 111
|
||||
#define VK_FORMAT_R64_SFLOAT 112
|
||||
#define VK_FORMAT_R64G64_UINT 113
|
||||
#define VK_FORMAT_R64G64_SINT 114
|
||||
#define VK_FORMAT_R64G64_SFLOAT 115
|
||||
#define VK_FORMAT_R64G64B64_UINT 116
|
||||
#define VK_FORMAT_R64G64B64_SINT 117
|
||||
#define VK_FORMAT_R64G64B64_SFLOAT 118
|
||||
#define VK_FORMAT_R64G64B64A64_UINT 119
|
||||
#define VK_FORMAT_R64G64B64A64_SINT 120
|
||||
#define VK_FORMAT_R64G64B64A64_SFLOAT 121
|
||||
|
||||
#define VK_INDEX_TYPE_UINT16 0
|
||||
#define VK_INDEX_TYPE_UINT32 1
|
||||
#define VK_INDEX_TYPE_NONE_KHR 1000165000
|
||||
#define VK_INDEX_TYPE_UINT8_EXT 1000265000
|
||||
|
||||
#define VK_GEOMETRY_TYPE_TRIANGLES_KHR 0
|
||||
#define VK_GEOMETRY_TYPE_AABBS_KHR 1
|
||||
#define VK_GEOMETRY_TYPE_INSTANCES_KHR 2
|
||||
|
||||
#define VK_GEOMETRY_INSTANCE_TRIANGLE_FACING_CULL_DISABLE_BIT_KHR 1
|
||||
#define VK_GEOMETRY_INSTANCE_TRIANGLE_FLIP_FACING_BIT_KHR 2
|
||||
#define VK_GEOMETRY_INSTANCE_FORCE_OPAQUE_BIT_KHR 4
|
||||
#define VK_GEOMETRY_INSTANCE_FORCE_NO_OPAQUE_BIT_KHR 8
|
||||
|
||||
#define TYPE(type, align) \
|
||||
layout(buffer_reference, buffer_reference_align = align, scalar) buffer type##_ref \
|
||||
{ \
|
||||
type value; \
|
||||
};
|
||||
|
||||
#define REF(type) type##_ref
|
||||
#define VOID_REF uint64_t
|
||||
#define NULL 0
|
||||
#define DEREF(var) var.value
|
||||
|
||||
#define SIZEOF(type) uint32_t(uint64_t(REF(type)(uint64_t(0)) + 1))
|
||||
|
||||
#define OFFSET(ptr, offset) (uint64_t(ptr) + offset)
|
||||
|
||||
#define INFINITY (1.0 / 0.0)
|
||||
#define NAN (0.0 / 0.0)
|
||||
#define NAN_F16 (0.0hf / 0.0hf)
|
||||
|
||||
#define INDEX(type, ptr, index) REF(type)(OFFSET(ptr, (index)*SIZEOF(type)))
|
||||
|
||||
TYPE(int8_t, 1);
|
||||
TYPE(uint8_t, 1);
|
||||
TYPE(int16_t, 2);
|
||||
TYPE(uint16_t, 2);
|
||||
TYPE(int32_t, 4);
|
||||
TYPE(uint32_t, 4);
|
||||
TYPE(int64_t, 8);
|
||||
TYPE(uint64_t, 8);
|
||||
|
||||
TYPE(float, 4);
|
||||
|
||||
TYPE(vec2, 4);
|
||||
TYPE(vec3, 4);
|
||||
TYPE(vec4, 4);
|
||||
|
||||
TYPE(uvec4, 16);
|
||||
|
||||
TYPE(VOID_REF, 8);
|
||||
|
||||
/* A GLSL-adapted copy of VkAccelerationStructureInstanceKHR. */
|
||||
struct AccelerationStructureInstance {
|
||||
mat3x4 transform;
|
||||
uint32_t custom_instance_and_mask;
|
||||
uint32_t sbt_offset_and_flags;
|
||||
uint64_t accelerationStructureReference;
|
||||
};
|
||||
TYPE(AccelerationStructureInstance, 8);
|
||||
|
||||
/* copied from u_math.h */
|
||||
uint32_t
|
||||
align(uint32_t value, uint32_t alignment)
|
||||
{
|
||||
return (value + alignment - 1) & ~(alignment - 1);
|
||||
}
|
||||
|
||||
int32_t
|
||||
to_emulated_float(float f)
|
||||
{
|
||||
int32_t bits = floatBitsToInt(f);
|
||||
return f < 0 ? -2147483648 - bits : bits;
|
||||
}
|
||||
|
||||
float
|
||||
from_emulated_float(int32_t bits)
|
||||
{
|
||||
return intBitsToFloat(bits < 0 ? -2147483648 - bits : bits);
|
||||
}
|
||||
|
||||
TYPE(vk_aabb, 4);
|
||||
|
||||
TYPE(key32_id_pair, 4);
|
||||
TYPE(key64_id_pair, 4);
|
||||
|
||||
TYPE(vk_accel_struct_serialization_header, 8);
|
||||
|
||||
TYPE(vk_ir_header, 4);
|
||||
TYPE(vk_ir_node, 4);
|
||||
TYPE(vk_ir_box_node, 4);
|
||||
TYPE(vk_ir_triangle_node, 4);
|
||||
TYPE(vk_ir_aabb_node, 4);
|
||||
TYPE(vk_ir_instance_node, 8);
|
||||
|
||||
TYPE(vk_global_sync_data, 4);
|
||||
|
||||
TYPE(vk_bvh_geometry_data, 8);
|
||||
|
||||
uint32_t
|
||||
ir_id_to_offset(uint32_t id)
|
||||
{
|
||||
return id & (~3u);
|
||||
}
|
||||
|
||||
uint32_t
|
||||
ir_id_to_type(uint32_t id)
|
||||
{
|
||||
return id & 3u;
|
||||
}
|
||||
|
||||
uint32_t
|
||||
pack_ir_node_id(uint32_t offset, uint32_t type)
|
||||
{
|
||||
return offset | type;
|
||||
}
|
||||
|
||||
float
|
||||
aabb_surface_area(vk_aabb aabb)
|
||||
{
|
||||
vec3 diagonal = aabb.max - aabb.min;
|
||||
return 2 * diagonal.x * diagonal.y + 2 * diagonal.y * diagonal.z + 2 * diagonal.x * diagonal.z;
|
||||
}
|
||||
|
||||
/* Just a wrapper for 3 uints. */
|
||||
struct triangle_indices {
|
||||
uint32_t index[3];
|
||||
};
|
||||
|
||||
triangle_indices
|
||||
load_indices(VOID_REF indices, uint32_t index_format, uint32_t global_id)
|
||||
{
|
||||
triangle_indices result;
|
||||
|
||||
uint32_t index_base = global_id * 3;
|
||||
|
||||
switch (index_format) {
|
||||
case VK_INDEX_TYPE_UINT16: {
|
||||
result.index[0] = DEREF(INDEX(uint16_t, indices, index_base + 0));
|
||||
result.index[1] = DEREF(INDEX(uint16_t, indices, index_base + 1));
|
||||
result.index[2] = DEREF(INDEX(uint16_t, indices, index_base + 2));
|
||||
break;
|
||||
}
|
||||
case VK_INDEX_TYPE_UINT32: {
|
||||
result.index[0] = DEREF(INDEX(uint32_t, indices, index_base + 0));
|
||||
result.index[1] = DEREF(INDEX(uint32_t, indices, index_base + 1));
|
||||
result.index[2] = DEREF(INDEX(uint32_t, indices, index_base + 2));
|
||||
break;
|
||||
}
|
||||
case VK_INDEX_TYPE_NONE_KHR: {
|
||||
result.index[0] = index_base + 0;
|
||||
result.index[1] = index_base + 1;
|
||||
result.index[2] = index_base + 2;
|
||||
break;
|
||||
}
|
||||
case VK_INDEX_TYPE_UINT8_EXT: {
|
||||
result.index[0] = DEREF(INDEX(uint8_t, indices, index_base + 0));
|
||||
result.index[1] = DEREF(INDEX(uint8_t, indices, index_base + 1));
|
||||
result.index[2] = DEREF(INDEX(uint8_t, indices, index_base + 2));
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/* Just a wrapper for 3 vec4s. */
|
||||
struct triangle_vertices {
|
||||
vec4 vertex[3];
|
||||
};
|
||||
|
||||
TYPE(float16_t, 2);
|
||||
|
||||
triangle_vertices
|
||||
load_vertices(VOID_REF vertices, triangle_indices indices, uint32_t vertex_format, uint32_t stride)
|
||||
{
|
||||
triangle_vertices result;
|
||||
|
||||
for (uint32_t i = 0; i < 3; i++) {
|
||||
VOID_REF vertex_ptr = OFFSET(vertices, indices.index[i] * stride);
|
||||
vec4 vertex = vec4(0.0, 0.0, 0.0, 1.0);
|
||||
|
||||
switch (vertex_format) {
|
||||
case VK_FORMAT_R32G32_SFLOAT:
|
||||
vertex.x = DEREF(INDEX(float, vertex_ptr, 0));
|
||||
vertex.y = DEREF(INDEX(float, vertex_ptr, 1));
|
||||
break;
|
||||
case VK_FORMAT_R32G32B32_SFLOAT:
|
||||
case VK_FORMAT_R32G32B32A32_SFLOAT:
|
||||
vertex.x = DEREF(INDEX(float, vertex_ptr, 0));
|
||||
vertex.y = DEREF(INDEX(float, vertex_ptr, 1));
|
||||
vertex.z = DEREF(INDEX(float, vertex_ptr, 2));
|
||||
break;
|
||||
case VK_FORMAT_R16G16_SFLOAT:
|
||||
vertex.x = DEREF(INDEX(float16_t, vertex_ptr, 0));
|
||||
vertex.y = DEREF(INDEX(float16_t, vertex_ptr, 1));
|
||||
break;
|
||||
case VK_FORMAT_R16G16B16_SFLOAT:
|
||||
case VK_FORMAT_R16G16B16A16_SFLOAT:
|
||||
vertex.x = DEREF(INDEX(float16_t, vertex_ptr, 0));
|
||||
vertex.y = DEREF(INDEX(float16_t, vertex_ptr, 1));
|
||||
vertex.z = DEREF(INDEX(float16_t, vertex_ptr, 2));
|
||||
break;
|
||||
case VK_FORMAT_R16G16_SNORM:
|
||||
vertex.x = max(-1.0, DEREF(INDEX(int16_t, vertex_ptr, 0)) / float(0x7FFF));
|
||||
vertex.y = max(-1.0, DEREF(INDEX(int16_t, vertex_ptr, 1)) / float(0x7FFF));
|
||||
break;
|
||||
case VK_FORMAT_R16G16B16A16_SNORM:
|
||||
vertex.x = max(-1.0, DEREF(INDEX(int16_t, vertex_ptr, 0)) / float(0x7FFF));
|
||||
vertex.y = max(-1.0, DEREF(INDEX(int16_t, vertex_ptr, 1)) / float(0x7FFF));
|
||||
vertex.z = max(-1.0, DEREF(INDEX(int16_t, vertex_ptr, 2)) / float(0x7FFF));
|
||||
break;
|
||||
case VK_FORMAT_R8G8_SNORM:
|
||||
vertex.x = max(-1.0, DEREF(INDEX(int8_t, vertex_ptr, 0)) / float(0x7F));
|
||||
vertex.y = max(-1.0, DEREF(INDEX(int8_t, vertex_ptr, 1)) / float(0x7F));
|
||||
break;
|
||||
case VK_FORMAT_R8G8B8A8_SNORM:
|
||||
vertex.x = max(-1.0, DEREF(INDEX(int8_t, vertex_ptr, 0)) / float(0x7F));
|
||||
vertex.y = max(-1.0, DEREF(INDEX(int8_t, vertex_ptr, 1)) / float(0x7F));
|
||||
vertex.z = max(-1.0, DEREF(INDEX(int8_t, vertex_ptr, 2)) / float(0x7F));
|
||||
break;
|
||||
case VK_FORMAT_R16G16_UNORM:
|
||||
vertex.x = DEREF(INDEX(uint16_t, vertex_ptr, 0)) / float(0xFFFF);
|
||||
vertex.y = DEREF(INDEX(uint16_t, vertex_ptr, 1)) / float(0xFFFF);
|
||||
break;
|
||||
case VK_FORMAT_R16G16B16A16_UNORM:
|
||||
vertex.x = DEREF(INDEX(uint16_t, vertex_ptr, 0)) / float(0xFFFF);
|
||||
vertex.y = DEREF(INDEX(uint16_t, vertex_ptr, 1)) / float(0xFFFF);
|
||||
vertex.z = DEREF(INDEX(uint16_t, vertex_ptr, 2)) / float(0xFFFF);
|
||||
break;
|
||||
case VK_FORMAT_R8G8_UNORM:
|
||||
vertex.x = DEREF(INDEX(uint8_t, vertex_ptr, 0)) / float(0xFF);
|
||||
vertex.y = DEREF(INDEX(uint8_t, vertex_ptr, 1)) / float(0xFF);
|
||||
break;
|
||||
case VK_FORMAT_R8G8B8A8_UNORM:
|
||||
vertex.x = DEREF(INDEX(uint8_t, vertex_ptr, 0)) / float(0xFF);
|
||||
vertex.y = DEREF(INDEX(uint8_t, vertex_ptr, 1)) / float(0xFF);
|
||||
vertex.z = DEREF(INDEX(uint8_t, vertex_ptr, 2)) / float(0xFF);
|
||||
break;
|
||||
case VK_FORMAT_A2B10G10R10_UNORM_PACK32: {
|
||||
uint32_t data = DEREF(REF(uint32_t)(vertex_ptr));
|
||||
vertex.x = float(data & 0x3FF) / 0x3FF;
|
||||
vertex.y = float((data >> 10) & 0x3FF) / 0x3FF;
|
||||
vertex.z = float((data >> 20) & 0x3FF) / 0x3FF;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
result.vertex[i] = vertex;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/* Fetch the flags of child nodes used to determine whether all/no children are opaque. */
|
||||
uint32_t fetch_child_flags(VOID_REF bvh, uint32_t node_ptr)
|
||||
{
|
||||
VOID_REF node = OFFSET(bvh, ir_id_to_offset(node_ptr));
|
||||
switch (ir_id_to_type(node_ptr)) {
|
||||
case vk_ir_node_triangle:
|
||||
return (DEREF(REF(vk_ir_triangle_node)(node)).geometry_id_and_flags & VK_GEOMETRY_OPAQUE) != 0
|
||||
? VK_BVH_BOX_FLAG_ONLY_OPAQUE
|
||||
: VK_BVH_BOX_FLAG_NO_OPAQUE;
|
||||
case vk_ir_node_internal:
|
||||
return DEREF(REF(vk_ir_box_node)(node)).flags;
|
||||
case vk_ir_node_instance:
|
||||
return DEREF(REF(vk_ir_instance_node)(node)).root_flags;
|
||||
case vk_ir_node_aabb:
|
||||
return (DEREF(REF(vk_ir_aabb_node)(node)).geometry_id_and_flags & VK_GEOMETRY_OPAQUE) != 0
|
||||
? VK_BVH_BOX_FLAG_ONLY_OPAQUE
|
||||
: VK_BVH_BOX_FLAG_NO_OPAQUE;
|
||||
default:
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/** Compute ceiling of integer quotient of A divided by B.
|
||||
From macros.h */
|
||||
#define DIV_ROUND_UP(A, B) (((A) + (B)-1) / (B))
|
||||
|
||||
#ifdef USE_GLOBAL_SYNC
|
||||
|
||||
/* There might be more invocations available than tasks to do.
|
||||
* In that case, the fetched task index is greater than the
|
||||
* counter offset for the next phase. To avoid out-of-bounds
|
||||
* accessing, phases will be skipped until the task index is
|
||||
* is in-bounds again. */
|
||||
uint32_t num_tasks_to_skip = 0;
|
||||
uint32_t phase_index = 0;
|
||||
bool should_skip = false;
|
||||
shared uint32_t global_task_index;
|
||||
|
||||
shared uint32_t shared_phase_index;
|
||||
|
||||
uint32_t
|
||||
task_count(REF(vk_ir_header) header)
|
||||
{
|
||||
uint32_t phase_index = DEREF(header).sync_data.phase_index;
|
||||
return DEREF(header).sync_data.task_counts[phase_index & 1];
|
||||
}
|
||||
|
||||
/* Sets the task count for the next phase. */
|
||||
void
|
||||
set_next_task_count(REF(vk_ir_header) header, uint32_t new_count)
|
||||
{
|
||||
uint32_t phase_index = DEREF(header).sync_data.phase_index;
|
||||
DEREF(header).sync_data.task_counts[(phase_index + 1) & 1] = new_count;
|
||||
}
|
||||
|
||||
/*
|
||||
* This function has two main objectives:
|
||||
* Firstly, it partitions pending work among free invocations.
|
||||
* Secondly, it guarantees global synchronization between different phases.
|
||||
*
|
||||
* After every call to fetch_task, a new task index is returned.
|
||||
* fetch_task will also set num_tasks_to_skip. Use should_execute_phase
|
||||
* to determine if the current phase should be executed or skipped.
|
||||
*
|
||||
* Since tasks are assigned per-workgroup, there is a possibility of the task index being
|
||||
* greater than the total task count.
|
||||
*/
|
||||
uint32_t
|
||||
fetch_task(REF(vk_ir_header) header, bool did_work)
|
||||
{
|
||||
/* Perform a memory + control barrier for all buffer writes for the entire workgroup.
|
||||
* This guarantees that once the workgroup leaves the PHASE loop, all invocations have finished
|
||||
* and their results are written to memory. */
|
||||
controlBarrier(gl_ScopeWorkgroup, gl_ScopeDevice, gl_StorageSemanticsBuffer,
|
||||
gl_SemanticsAcquireRelease | gl_SemanticsMakeAvailable | gl_SemanticsMakeVisible);
|
||||
if (gl_LocalInvocationIndex == 0) {
|
||||
if (did_work)
|
||||
atomicAdd(DEREF(header).sync_data.task_done_counter, 1);
|
||||
global_task_index = atomicAdd(DEREF(header).sync_data.task_started_counter, 1);
|
||||
|
||||
do {
|
||||
/* Perform a memory barrier to refresh the current phase's end counter, in case
|
||||
* another workgroup changed it. */
|
||||
memoryBarrier(gl_ScopeDevice, gl_StorageSemanticsBuffer,
|
||||
gl_SemanticsAcquireRelease | gl_SemanticsMakeAvailable | gl_SemanticsMakeVisible);
|
||||
|
||||
/* The first invocation of the first workgroup in a new phase is responsible to initiate the
|
||||
* switch to a new phase. It is only possible to switch to a new phase if all tasks of the
|
||||
* previous phase have been completed. Switching to a new phase and incrementing the phase
|
||||
* end counter in turn notifies all invocations for that phase that it is safe to execute.
|
||||
*/
|
||||
if (global_task_index == DEREF(header).sync_data.current_phase_end_counter &&
|
||||
DEREF(header).sync_data.task_done_counter == DEREF(header).sync_data.current_phase_end_counter) {
|
||||
if (DEREF(header).sync_data.next_phase_exit_flag != 0) {
|
||||
DEREF(header).sync_data.phase_index = TASK_INDEX_INVALID;
|
||||
memoryBarrier(gl_ScopeDevice, gl_StorageSemanticsBuffer,
|
||||
gl_SemanticsAcquireRelease | gl_SemanticsMakeAvailable | gl_SemanticsMakeVisible);
|
||||
} else {
|
||||
atomicAdd(DEREF(header).sync_data.phase_index, 1);
|
||||
DEREF(header).sync_data.current_phase_start_counter = DEREF(header).sync_data.current_phase_end_counter;
|
||||
/* Ensure the changes to the phase index and start/end counter are visible for other
|
||||
* workgroup waiting in the loop. */
|
||||
memoryBarrier(gl_ScopeDevice, gl_StorageSemanticsBuffer,
|
||||
gl_SemanticsAcquireRelease | gl_SemanticsMakeAvailable | gl_SemanticsMakeVisible);
|
||||
atomicAdd(DEREF(header).sync_data.current_phase_end_counter,
|
||||
DIV_ROUND_UP(task_count(header), gl_WorkGroupSize.x));
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
/* If other invocations have finished all nodes, break out; there is no work to do */
|
||||
if (DEREF(header).sync_data.phase_index == TASK_INDEX_INVALID) {
|
||||
break;
|
||||
}
|
||||
} while (global_task_index >= DEREF(header).sync_data.current_phase_end_counter);
|
||||
|
||||
shared_phase_index = DEREF(header).sync_data.phase_index;
|
||||
}
|
||||
|
||||
barrier();
|
||||
if (DEREF(header).sync_data.phase_index == TASK_INDEX_INVALID)
|
||||
return TASK_INDEX_INVALID;
|
||||
|
||||
num_tasks_to_skip = shared_phase_index - phase_index;
|
||||
|
||||
uint32_t local_task_index = global_task_index - DEREF(header).sync_data.current_phase_start_counter;
|
||||
return local_task_index * gl_WorkGroupSize.x + gl_LocalInvocationID.x;
|
||||
}
|
||||
|
||||
bool
|
||||
should_execute_phase()
|
||||
{
|
||||
if (num_tasks_to_skip > 0) {
|
||||
/* Skip to next phase. */
|
||||
++phase_index;
|
||||
--num_tasks_to_skip;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
#define PHASE(header) \
|
||||
for (; task_index != TASK_INDEX_INVALID && should_execute_phase(); task_index = fetch_task(header, true))
|
||||
#endif
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,122 @@
|
||||
/*
|
||||
* Copyright © 2022 Konstantin Seurer
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#ifndef VK_BVH_BUILD_INTERFACE_H
|
||||
#define VK_BVH_BUILD_INTERFACE_H
|
||||
|
||||
#ifdef VULKAN
|
||||
#include "vk_build_helpers.h"
|
||||
#else
|
||||
#include <stdint.h>
|
||||
#include "vk_bvh.h"
|
||||
#define REF(type) uint64_t
|
||||
#define VOID_REF uint64_t
|
||||
#endif
|
||||
|
||||
#define SUBGROUP_SIZE_ID 0
|
||||
#define BVH_BOUNDS_OFFSET_ID 1
|
||||
#define BUILD_FLAGS_ID 2
|
||||
#define ROOT_FLAGS_OFFSET_ID 3
|
||||
#ifdef VULKAN
|
||||
layout (constant_id = SUBGROUP_SIZE_ID) const int SUBGROUP_SIZE = 64;
|
||||
layout (constant_id = BVH_BOUNDS_OFFSET_ID) const int BVH_BOUNDS_OFFSET = 0;
|
||||
layout (constant_id = BUILD_FLAGS_ID) const int BUILD_FLAGS = 0;
|
||||
layout (constant_id = ROOT_FLAGS_OFFSET_ID) const int ROOT_FLAGS_OFFSET = -1;
|
||||
#endif
|
||||
|
||||
#define VK_BUILD_FLAG_ALWAYS_ACTIVE (1u << 0)
|
||||
#define VK_BUILD_FLAG_PROPAGATE_CULL_FLAGS (1u << 1)
|
||||
#define VK_BUILD_FLAG_64BIT_KEYS (1u << 2)
|
||||
#define VK_BUILD_FLAG_COUNT 3
|
||||
|
||||
#define VK_BUILD_FLAG(flag) ((BUILD_FLAGS & flag) != 0)
|
||||
|
||||
struct leaf_args {
|
||||
VOID_REF bvh;
|
||||
REF(vk_ir_header) header;
|
||||
VOID_REF ids;
|
||||
|
||||
vk_bvh_geometry_data geom_data;
|
||||
};
|
||||
|
||||
struct morton_args {
|
||||
VOID_REF bvh;
|
||||
REF(vk_ir_header) header;
|
||||
VOID_REF ids;
|
||||
};
|
||||
|
||||
#define LBVH_RIGHT_CHILD_BIT_SHIFT 29
|
||||
#define LBVH_RIGHT_CHILD_BIT (1 << LBVH_RIGHT_CHILD_BIT_SHIFT)
|
||||
|
||||
struct lbvh_node_info {
|
||||
/* Number of children that have been processed (or are invalid/leaves) in
|
||||
* the lbvh_generate_ir pass.
|
||||
*/
|
||||
uint32_t path_count;
|
||||
|
||||
uint32_t children[2];
|
||||
uint32_t parent;
|
||||
};
|
||||
|
||||
struct lbvh_main_args {
|
||||
VOID_REF bvh;
|
||||
VOID_REF src_ids;
|
||||
VOID_REF node_info;
|
||||
REF(vk_ir_header) header;
|
||||
uint32_t internal_node_base;
|
||||
};
|
||||
|
||||
struct lbvh_generate_ir_args {
|
||||
VOID_REF bvh;
|
||||
VOID_REF node_info;
|
||||
REF(vk_ir_header) header;
|
||||
uint32_t internal_node_base;
|
||||
};
|
||||
|
||||
struct ploc_prefix_scan_partition {
|
||||
uint32_t aggregate;
|
||||
uint32_t inclusive_sum;
|
||||
};
|
||||
|
||||
#define PLOC_WORKGROUP_SIZE 1024
|
||||
#define PLOC_SUBGROUPS_PER_WORKGROUP \
|
||||
(DIV_ROUND_UP(PLOC_WORKGROUP_SIZE, SUBGROUP_SIZE))
|
||||
|
||||
struct ploc_args {
|
||||
VOID_REF bvh;
|
||||
VOID_REF prefix_scan_partitions;
|
||||
REF(vk_ir_header) header;
|
||||
VOID_REF ids_0;
|
||||
VOID_REF ids_1;
|
||||
uint32_t internal_node_offset;
|
||||
};
|
||||
|
||||
struct hploc_args {
|
||||
REF(vk_ir_header) header;
|
||||
VOID_REF bvh;
|
||||
REF(key32_id_pair) ids;
|
||||
VOID_REF ranges;
|
||||
uint32_t internal_node_base;
|
||||
};
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,183 @@
|
||||
/*
|
||||
* Copyright © 2021 Bas Nieuwenhuizen
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#ifndef BVH_VK_BVH_H
|
||||
#define BVH_VK_BVH_H
|
||||
|
||||
#define vk_ir_node_triangle 0
|
||||
#define vk_ir_node_internal 1
|
||||
#define vk_ir_node_instance 2
|
||||
#define vk_ir_node_aabb 3
|
||||
|
||||
#define VK_GEOMETRY_OPAQUE (1u << 31)
|
||||
|
||||
#ifdef VULKAN
|
||||
#define VK_UUID_SIZE 16
|
||||
#else
|
||||
#include <vulkan/vulkan.h>
|
||||
typedef struct vk_ir_node vk_ir_node;
|
||||
typedef struct vk_global_sync_data vk_global_sync_data;
|
||||
typedef struct vk_bvh_geometry_data vk_bvh_geometry_data;
|
||||
|
||||
typedef struct {
|
||||
float values[3][4];
|
||||
} mat3x4;
|
||||
|
||||
typedef struct {
|
||||
float x;
|
||||
float y;
|
||||
float z;
|
||||
} vec3;
|
||||
|
||||
typedef struct vk_aabb vk_aabb;
|
||||
#endif
|
||||
|
||||
struct vk_aabb {
|
||||
vec3 min;
|
||||
vec3 max;
|
||||
};
|
||||
|
||||
/* This is the header structure for serialized acceleration structures, as
|
||||
* defined by the Vulkan spec.
|
||||
*/
|
||||
struct vk_accel_struct_serialization_header {
|
||||
uint8_t driver_uuid[VK_UUID_SIZE];
|
||||
uint8_t accel_struct_compat[VK_UUID_SIZE];
|
||||
uint64_t serialization_size;
|
||||
uint64_t deserialization_size;
|
||||
uint64_t instance_count;
|
||||
#ifndef VULKAN
|
||||
uint64_t instances[];
|
||||
#endif
|
||||
};
|
||||
|
||||
struct vk_global_sync_data {
|
||||
uint32_t task_counts[2];
|
||||
uint32_t task_started_counter;
|
||||
uint32_t task_done_counter;
|
||||
uint32_t current_phase_start_counter;
|
||||
uint32_t current_phase_end_counter;
|
||||
uint32_t phase_index;
|
||||
/* If this flag is set, the shader should exit
|
||||
* instead of executing another phase */
|
||||
uint32_t next_phase_exit_flag;
|
||||
};
|
||||
|
||||
struct vk_ir_header {
|
||||
int32_t min_bounds[3];
|
||||
int32_t max_bounds[3];
|
||||
uint32_t active_leaf_count;
|
||||
/* Indirect dispatch dimensions for the encoder.
|
||||
* ir_internal_node_count is the thread count in the X dimension,
|
||||
* while Y and Z are always set to 1. */
|
||||
uint32_t ir_internal_node_count;
|
||||
uint32_t dispatch_size_y;
|
||||
uint32_t dispatch_size_z;
|
||||
vk_global_sync_data sync_data;
|
||||
/* Generic offset used by the driver during encoding
|
||||
* to write HW nodes in a compact way.
|
||||
*/
|
||||
uint32_t dst_node_offset;
|
||||
/* Same as dst_node_offset but only useful if the driver
|
||||
* uses a separate memory section for leaf nodes.
|
||||
*/
|
||||
uint32_t dst_leaf_node_offset;
|
||||
/* Additional fields for the driver to use during encode. */
|
||||
uint32_t driver_internal[6];
|
||||
};
|
||||
|
||||
struct vk_ir_node {
|
||||
vk_aabb aabb;
|
||||
};
|
||||
|
||||
#define VK_UNKNOWN_BVH_OFFSET 0xFFFFFFFF
|
||||
#define VK_NULL_BVH_OFFSET 0xFFFFFFFE
|
||||
|
||||
/* Box node contains only opaque leaves */
|
||||
#define VK_BVH_BOX_FLAG_ONLY_OPAQUE 0x1
|
||||
/* Box node contains no opaque leaves */
|
||||
#define VK_BVH_BOX_FLAG_NO_OPAQUE 0x2
|
||||
|
||||
struct vk_ir_box_node {
|
||||
vk_ir_node base;
|
||||
uint32_t children[2];
|
||||
uint32_t bvh_offset;
|
||||
uint32_t flags;
|
||||
};
|
||||
|
||||
struct vk_ir_aabb_node {
|
||||
vk_ir_node base;
|
||||
uint32_t primitive_id;
|
||||
uint32_t geometry_id_and_flags;
|
||||
};
|
||||
|
||||
struct vk_ir_triangle_node {
|
||||
vk_ir_node base;
|
||||
float coords[3][3];
|
||||
uint32_t triangle_id;
|
||||
uint32_t geometry_id_and_flags;
|
||||
};
|
||||
|
||||
struct vk_ir_instance_node {
|
||||
vk_ir_node base;
|
||||
/* See radv_bvh_instance_node */
|
||||
uint64_t base_ptr;
|
||||
uint32_t custom_instance_and_mask;
|
||||
uint32_t sbt_offset_and_flags;
|
||||
mat3x4 otw_matrix;
|
||||
uint32_t instance_id;
|
||||
/* The root node's flags. */
|
||||
uint32_t root_flags;
|
||||
};
|
||||
|
||||
#define VK_BVH_INVALID_NODE 0xFFFFFFFF
|
||||
|
||||
/* If the task index is set to this value, there is no
|
||||
* more work to do. */
|
||||
#define TASK_INDEX_INVALID 0xFFFFFFFF
|
||||
|
||||
struct vk_bvh_geometry_data {
|
||||
uint64_t data;
|
||||
uint64_t indices;
|
||||
uint64_t transform;
|
||||
|
||||
uint32_t geometry_id;
|
||||
uint32_t geometry_type;
|
||||
uint32_t first_id;
|
||||
uint32_t stride;
|
||||
uint32_t vertex_format;
|
||||
uint32_t index_format;
|
||||
};
|
||||
|
||||
struct key32_id_pair {
|
||||
uint32_t id;
|
||||
uint32_t key;
|
||||
};
|
||||
|
||||
struct key64_id_pair {
|
||||
uint32_t id;
|
||||
uint32_t key_lo;
|
||||
uint32_t key_hi;
|
||||
};
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,19 @@
|
||||
|
||||
#ifndef BVH_VK_DEBUG_H
|
||||
#define BVH_VK_DEBUG_H
|
||||
|
||||
#ifndef NDEBUG
|
||||
|
||||
#extension GL_EXT_debug_printf : enable
|
||||
|
||||
#define printf debugPrintfEXT
|
||||
#define assert(expr, message) do { if (!(expr)) printf(message); } while (false)
|
||||
|
||||
#else
|
||||
|
||||
#define printf
|
||||
#define assert(expr, message)
|
||||
|
||||
#endif
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,341 @@
|
||||
# Copyright © 2017 Intel Corporation
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
# Mesa-local imports in the Python files must be declared here for correct
|
||||
# dependency tracking.
|
||||
|
||||
vulkan_lite_runtime_files = files(
|
||||
'rmv/vk_rmv_common.c',
|
||||
'rmv/vk_rmv_exporter.c',
|
||||
'vk_blend.c',
|
||||
'vk_buffer.c',
|
||||
'vk_buffer_view.c',
|
||||
'vk_cmd_copy.c',
|
||||
'vk_cmd_enqueue.c',
|
||||
'vk_command_buffer.c',
|
||||
'vk_command_pool.c',
|
||||
'vk_debug_report.c',
|
||||
'vk_debug_utils.c',
|
||||
'vk_deferred_operation.c',
|
||||
'vk_descriptor_set_layout.c',
|
||||
'vk_descriptors.c',
|
||||
'vk_descriptor_update_template.c',
|
||||
'vk_device.c',
|
||||
'vk_device_generated_commands.c',
|
||||
'vk_device_memory.c',
|
||||
'vk_fence.c',
|
||||
'vk_framebuffer.c',
|
||||
'vk_graphics_state.c',
|
||||
'vk_image.c',
|
||||
'vk_log.c',
|
||||
'vk_meta_object_list.c',
|
||||
'vk_object.c',
|
||||
'vk_physical_device.c',
|
||||
'vk_pipeline_layout.c',
|
||||
'vk_query_pool.c',
|
||||
'vk_queue.c',
|
||||
'vk_render_pass.c',
|
||||
'vk_sampler.c',
|
||||
'vk_semaphore.c',
|
||||
'vk_standard_sample_locations.c',
|
||||
'vk_sync.c',
|
||||
'vk_sync_binary.c',
|
||||
'vk_sync_dummy.c',
|
||||
'vk_sync_timeline.c',
|
||||
'vk_synchronization.c',
|
||||
'vk_video.c',
|
||||
'vk_ycbcr_conversion.c',
|
||||
)
|
||||
|
||||
vulkan_lite_runtime_deps = [
|
||||
vulkan_wsi_deps,
|
||||
idep_mesautil,
|
||||
idep_nir_headers,
|
||||
idep_vulkan_util,
|
||||
]
|
||||
|
||||
if dep_libdrm.found()
|
||||
vulkan_lite_runtime_files += files('vk_drm_syncobj.c')
|
||||
vulkan_lite_runtime_deps += dep_libdrm
|
||||
endif
|
||||
|
||||
if with_platform_android
|
||||
vulkan_lite_runtime_files += files('vk_android.c')
|
||||
vulkan_lite_runtime_deps += [dep_android, idep_u_gralloc]
|
||||
endif
|
||||
|
||||
vk_common_entrypoints = custom_target(
|
||||
'vk_common_entrypoints',
|
||||
input : [vk_entrypoints_gen, vk_api_xml],
|
||||
output : ['vk_common_entrypoints.h', 'vk_common_entrypoints.c'],
|
||||
command : [
|
||||
prog_python, '@INPUT0@', '--xml', '@INPUT1@', '--proto', '--weak',
|
||||
'--out-h', '@OUTPUT0@', '--out-c', '@OUTPUT1@', '--prefix', 'vk_common',
|
||||
'--beta', with_vulkan_beta.to_string()
|
||||
],
|
||||
depend_files : vk_entrypoints_gen_depend_files,
|
||||
)
|
||||
|
||||
vk_cmd_queue = custom_target(
|
||||
'vk_cmd_queue',
|
||||
input : [vk_cmd_queue_gen, vk_api_xml],
|
||||
output : ['vk_cmd_queue.c', 'vk_cmd_queue.h'],
|
||||
command : [
|
||||
prog_python, '@INPUT0@', '--xml', '@INPUT1@',
|
||||
'--out-c', '@OUTPUT0@', '--out-h', '@OUTPUT1@',
|
||||
'--beta', with_vulkan_beta.to_string()
|
||||
],
|
||||
depend_files : vk_cmd_queue_gen_depend_files,
|
||||
)
|
||||
|
||||
vk_cmd_enqueue_entrypoints = custom_target(
|
||||
'vk_cmd_enqueue_entrypoints',
|
||||
input : [vk_entrypoints_gen, vk_api_xml],
|
||||
output : ['vk_cmd_enqueue_entrypoints.h', 'vk_cmd_enqueue_entrypoints.c'],
|
||||
command : [
|
||||
prog_python, '@INPUT0@', '--xml', '@INPUT1@', '--proto', '--weak',
|
||||
'--out-h', '@OUTPUT0@', '--out-c', '@OUTPUT1@',
|
||||
'--prefix', 'vk_cmd_enqueue', '--prefix', 'vk_cmd_enqueue_unless_primary',
|
||||
'--beta', with_vulkan_beta.to_string()
|
||||
],
|
||||
depend_files : vk_entrypoints_gen_depend_files,
|
||||
)
|
||||
|
||||
vk_dispatch_trampolines = custom_target(
|
||||
'vk_dispatch_trampolines',
|
||||
input : [vk_dispatch_trampolines_gen, vk_api_xml],
|
||||
output : ['vk_dispatch_trampolines.c', 'vk_dispatch_trampolines.h'],
|
||||
command : [
|
||||
prog_python, '@INPUT0@', '--xml', '@INPUT1@',
|
||||
'--out-c', '@OUTPUT0@', '--out-h', '@OUTPUT1@',
|
||||
'--beta', with_vulkan_beta.to_string()
|
||||
],
|
||||
depend_files : vk_dispatch_trampolines_gen_depend_files,
|
||||
)
|
||||
|
||||
vk_physical_device_features = custom_target(
|
||||
'vk_physical_device_features',
|
||||
input : [vk_physical_device_features_gen, vk_api_xml],
|
||||
output : ['vk_physical_device_features.c', 'vk_physical_device_features.h'],
|
||||
command : [
|
||||
prog_python, '@INPUT0@', '--xml', '@INPUT1@',
|
||||
'--out-c', '@OUTPUT0@', '--out-h', '@OUTPUT1@',
|
||||
'--beta', with_vulkan_beta.to_string()
|
||||
],
|
||||
depend_files : vk_physical_device_features_gen_depend_files,
|
||||
)
|
||||
|
||||
vk_physical_device_properties = custom_target(
|
||||
'vk_physical_device_properties',
|
||||
input : [vk_physical_device_properties_gen, vk_api_xml],
|
||||
output : ['vk_physical_device_properties.c', 'vk_physical_device_properties.h'],
|
||||
command : [
|
||||
prog_python, '@INPUT0@', '--xml', '@INPUT1@',
|
||||
'--out-c', '@OUTPUT0@', '--out-h', '@OUTPUT1@',
|
||||
'--beta', with_vulkan_beta.to_string()
|
||||
],
|
||||
depend_files : vk_physical_device_properties_gen_depend_files,
|
||||
)
|
||||
|
||||
vk_physical_device_spirv_caps = custom_target(
|
||||
'vk_physical_device_spirv_caps',
|
||||
input : [vk_physical_device_spirv_caps_gen, vk_api_xml],
|
||||
output : 'vk_physical_device_spirv_caps.c',
|
||||
command : [
|
||||
prog_python, '@INPUT0@', '--xml', '@INPUT1@',
|
||||
'--out-c', '@OUTPUT0@', '--beta', with_vulkan_beta.to_string()
|
||||
],
|
||||
depend_files : vk_physical_device_spirv_caps_gen_depend_files,
|
||||
)
|
||||
|
||||
vk_synchronization_helpers = custom_target(
|
||||
'vk_synchronization_helpers',
|
||||
input : [vk_synchronization_helpers_gen, vk_api_xml],
|
||||
output : 'vk_synchronization_helpers.c',
|
||||
command : [
|
||||
prog_python, '@INPUT0@', '--xml', '@INPUT1@',
|
||||
'--out-c', '@OUTPUT0@',
|
||||
'--beta', with_vulkan_beta.to_string()
|
||||
],
|
||||
depend_files : vk_synchronization_helpers_gen_depend_files,
|
||||
)
|
||||
|
||||
vk_format_info = custom_target(
|
||||
'vk_format_info',
|
||||
input : ['vk_format_info_gen.py', vk_api_xml],
|
||||
output : ['vk_format_info.c', 'vk_format_info.h'],
|
||||
command : [
|
||||
prog_python, '@INPUT0@', '--xml', '@INPUT1@',
|
||||
'--out-c', '@OUTPUT0@', '--out-h', '@OUTPUT1@'
|
||||
],
|
||||
)
|
||||
|
||||
vulkan_lite_runtime_files += [
|
||||
vk_cmd_enqueue_entrypoints,
|
||||
vk_cmd_queue,
|
||||
vk_common_entrypoints,
|
||||
vk_dispatch_trampolines,
|
||||
vk_format_info,
|
||||
vk_physical_device_features,
|
||||
vk_physical_device_properties,
|
||||
vk_physical_device_spirv_caps,
|
||||
vk_synchronization_helpers,
|
||||
]
|
||||
|
||||
# as a runtime library dep to ensure header gen order
|
||||
vulkan_lite_runtime_header_gen_deps = declare_dependency(
|
||||
sources : [
|
||||
spirv_info_h,
|
||||
vk_cmd_enqueue_entrypoints[0],
|
||||
vk_cmd_queue[1],
|
||||
vk_common_entrypoints[0],
|
||||
vk_dispatch_trampolines[1],
|
||||
vk_format_info[1],
|
||||
vk_physical_device_features[1],
|
||||
vk_physical_device_properties[1],
|
||||
],
|
||||
)
|
||||
|
||||
vulkan_lite_runtime_deps += vulkan_lite_runtime_header_gen_deps
|
||||
|
||||
libvulkan_lite_runtime = static_library(
|
||||
'vulkan_lite_runtime',
|
||||
vulkan_lite_runtime_files,
|
||||
include_directories : [inc_include, inc_src],
|
||||
dependencies : vulkan_lite_runtime_deps,
|
||||
c_args : [c_msvc_compat_args, soong_compat_c_args, '-DMESA_VK_LOG=@0@'.format(get_option('debug').to_int())],
|
||||
gnu_symbol_visibility : 'hidden',
|
||||
build_by_default : false,
|
||||
)
|
||||
|
||||
libvulkan_lite_instance = static_library(
|
||||
'vulkan_lite_instance',
|
||||
['vk_instance.c'],
|
||||
include_directories : [inc_include, inc_src],
|
||||
dependencies : vulkan_lite_runtime_deps,
|
||||
c_args : ['-DVK_LITE_RUNTIME_INSTANCE=1', c_msvc_compat_args, soong_compat_c_args],
|
||||
gnu_symbol_visibility : 'hidden',
|
||||
build_by_default : false,
|
||||
)
|
||||
|
||||
# The sources part is to ensure those generated headers used externally are
|
||||
# indeed generated before being compiled with, as long as either one of below
|
||||
# is included as a dependency:
|
||||
# - idep_vulkan_lite_runtime_headers
|
||||
# - idep_vulkan_lite_runtime
|
||||
# - idep_vulkan_runtime_headers
|
||||
# - idep_vulkan_runtime
|
||||
idep_vulkan_lite_runtime_headers = declare_dependency(
|
||||
sources : [
|
||||
spirv_info_h,
|
||||
vk_cmd_enqueue_entrypoints[0],
|
||||
vk_cmd_queue[1],
|
||||
vk_common_entrypoints[0],
|
||||
vk_physical_device_features[1],
|
||||
vk_physical_device_properties[1],
|
||||
],
|
||||
include_directories : include_directories('.'),
|
||||
)
|
||||
|
||||
# This is likely a bug in the Meson VS backend, as MSVC with ninja works fine.
|
||||
# See this discussion here:
|
||||
# https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/10506
|
||||
if get_option('backend').startswith('vs')
|
||||
idep_vulkan_lite_runtime = declare_dependency(
|
||||
link_with : [libvulkan_lite_runtime, libvulkan_lite_instance],
|
||||
dependencies : idep_vulkan_lite_runtime_headers
|
||||
)
|
||||
else
|
||||
idep_vulkan_lite_runtime = declare_dependency(
|
||||
# Instruct users of this library to link with --whole-archive. Otherwise,
|
||||
# our weak function overloads may not resolve properly.
|
||||
link_whole : [libvulkan_lite_runtime, libvulkan_lite_instance],
|
||||
dependencies : idep_vulkan_lite_runtime_headers
|
||||
)
|
||||
endif
|
||||
|
||||
vulkan_runtime_files = files(
|
||||
'vk_meta.c',
|
||||
'vk_meta_blit_resolve.c',
|
||||
'vk_meta_clear.c',
|
||||
'vk_meta_copy_fill_update.c',
|
||||
'vk_meta_draw_rects.c',
|
||||
'vk_nir.c',
|
||||
'vk_nir_convert_ycbcr.c',
|
||||
'vk_nir_lower_descriptor_heaps.c',
|
||||
'vk_pipeline.c',
|
||||
'vk_pipeline_cache.c',
|
||||
'vk_shader.c',
|
||||
'vk_shader_module.c',
|
||||
'vk_texcompress_etc2.c',
|
||||
)
|
||||
|
||||
vulkan_runtime_deps = [
|
||||
vulkan_lite_runtime_deps,
|
||||
idep_nir,
|
||||
idep_vtn,
|
||||
]
|
||||
|
||||
if prog_glslang.found()
|
||||
if with_bvh
|
||||
subdir('radix_sort')
|
||||
vulkan_runtime_files += radix_sort_files
|
||||
vulkan_runtime_files += radix_sort_spv
|
||||
subdir('bvh')
|
||||
vulkan_runtime_files += files('vk_acceleration_structure.c')
|
||||
vulkan_runtime_files += bvh_spv
|
||||
endif
|
||||
vulkan_runtime_files += files('vk_texcompress_astc.c')
|
||||
vulkan_runtime_files += custom_target(
|
||||
'astc_spv.h',
|
||||
input : astc_decoder_glsl_file,
|
||||
output : 'astc_spv.h',
|
||||
command : [
|
||||
prog_glslang, '-V', '-S', 'comp', '-x', '-o', '@OUTPUT@', '@INPUT@',
|
||||
glslang_quiet, glslang_depfile,
|
||||
],
|
||||
depfile : 'astc_spv.h.d',
|
||||
)
|
||||
endif
|
||||
|
||||
libvulkan_runtime = static_library(
|
||||
'vulkan_runtime',
|
||||
[vulkan_runtime_files],
|
||||
include_directories : [inc_include, inc_src],
|
||||
dependencies : vulkan_runtime_deps,
|
||||
c_args : [c_msvc_compat_args, soong_compat_c_args],
|
||||
gnu_symbol_visibility : 'hidden',
|
||||
build_by_default : false,
|
||||
)
|
||||
|
||||
libvulkan_instance = static_library(
|
||||
'vulkan_instance',
|
||||
['vk_instance.c'],
|
||||
include_directories : [inc_include, inc_src],
|
||||
dependencies : vulkan_runtime_deps,
|
||||
c_args : ['-DVK_LITE_RUNTIME_INSTANCE=0', c_msvc_compat_args, soong_compat_c_args],
|
||||
gnu_symbol_visibility : 'hidden',
|
||||
build_by_default : false,
|
||||
)
|
||||
|
||||
if get_option('backend').startswith('vs')
|
||||
idep_vulkan_runtime_body = declare_dependency(
|
||||
link_with : [libvulkan_lite_runtime, libvulkan_runtime, libvulkan_instance],
|
||||
)
|
||||
else
|
||||
idep_vulkan_runtime_body = declare_dependency(
|
||||
link_whole : [libvulkan_lite_runtime, libvulkan_runtime, libvulkan_instance],
|
||||
)
|
||||
endif
|
||||
|
||||
idep_vulkan_runtime_headers = [idep_vulkan_lite_runtime_headers]
|
||||
idep_vulkan_runtime_headers += declare_dependency(
|
||||
include_directories : include_directories('bvh'),
|
||||
)
|
||||
|
||||
idep_vulkan_runtime = declare_dependency(
|
||||
dependencies : [
|
||||
idep_vulkan_runtime_headers,
|
||||
idep_vulkan_runtime_body,
|
||||
]
|
||||
)
|
||||
@@ -0,0 +1,24 @@
|
||||
Copyright 2019 The Fuchsia Authors.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
@@ -0,0 +1,112 @@
|
||||
// Copyright 2019 The Fuchsia Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
#ifndef SRC_GRAPHICS_LIB_COMPUTE_COMMON_MACROS_H_
|
||||
#define SRC_GRAPHICS_LIB_COMPUTE_COMMON_MACROS_H_
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#include <assert.h>
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
//
|
||||
// clang-format off
|
||||
//
|
||||
|
||||
#define ARRAY_LENGTH_MACRO(x_) (sizeof(x_)/sizeof(x_[0]))
|
||||
#define OFFSETOF_MACRO(t_,m_) offsetof(t_,m_)
|
||||
#define MEMBER_SIZE_MACRO(t_,m_) sizeof(((t_*)0)->m_)
|
||||
|
||||
//
|
||||
// FIXME(allanmac):
|
||||
//
|
||||
// Consider providing typed min/max() functions:
|
||||
//
|
||||
// <type> [min|max]_<type>(a,b) { ; }
|
||||
//
|
||||
// But note we still need preprocessor-time min/max().
|
||||
//
|
||||
|
||||
#define MAX_MACRO(t_,a_,b_) (((a_) > (b_)) ? (a_) : (b_))
|
||||
#define MIN_MACRO(t_,a_,b_) (((a_) < (b_)) ? (a_) : (b_))
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#define BITS_TO_MASK_MACRO(n_) (((uint32_t)1<<(n_))-1)
|
||||
#define BITS_TO_MASK_64_MACRO(n_) (((uint64_t)1<<(n_))-1)
|
||||
|
||||
#define BITS_TO_MASK_AT_MACRO(n_,b_) (BITS_TO_MASK_MACRO(n_) <<(b_))
|
||||
#define BITS_TO_MASK_AT_64_MACRO(n_,b_) (BITS_TO_MASK_64_MACRO(n_)<<(b_))
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#define STRINGIFY_MACRO_2(a_) #a_
|
||||
#define STRINGIFY_MACRO(a_) STRINGIFY_MACRO_2(a_)
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#define CONCAT_MACRO_2(a_,b_) a_ ## b_
|
||||
#define CONCAT_MACRO(a_,b_) CONCAT_MACRO_2(a_,b_)
|
||||
|
||||
//
|
||||
// Round up/down
|
||||
//
|
||||
|
||||
#define ROUND_DOWN_MACRO(v_,q_) (((v_) / (q_)) * (q_))
|
||||
#define ROUND_UP_MACRO(v_,q_) ((((v_) + (q_) - 1) / (q_)) * (q_))
|
||||
|
||||
//
|
||||
// Round up/down when q is a power-of-two.
|
||||
//
|
||||
|
||||
#define ROUND_DOWN_POW2_MACRO(v_,q_) ((v_) & ~((q_) - 1))
|
||||
#define ROUND_UP_POW2_MACRO(v_,q_) ROUND_DOWN_POW2_MACRO((v_) + (q_) - 1, q_)
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#if defined (_MSC_VER) && !defined (__clang__)
|
||||
#define STATIC_ASSERT_MACRO(c_,m_) static_assert(c_,m_)
|
||||
#else
|
||||
#define STATIC_ASSERT_MACRO(c_,m_) _Static_assert(c_,m_)
|
||||
#endif
|
||||
|
||||
#define STATIC_ASSERT_MACRO_1(c_) STATIC_ASSERT_MACRO(c_,#c_)
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#if defined (_MSC_VER) && !defined (__clang__)
|
||||
#define POPCOUNT_MACRO(...) __popcnt(__VA_ARGS__)
|
||||
#else
|
||||
#define POPCOUNT_MACRO(...) __builtin_popcount(__VA_ARGS__)
|
||||
#endif
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#if defined (_MSC_VER) && !defined (__clang__)
|
||||
#define ALIGN_MACRO(bytes_) __declspec(align(bytes_)) // only accepts integer as arg
|
||||
#else
|
||||
#include <stdalign.h>
|
||||
#define ALIGN_MACRO(bytes_) alignas(bytes_)
|
||||
#endif
|
||||
|
||||
//
|
||||
// clang-format on
|
||||
//
|
||||
|
||||
#endif // SRC_GRAPHICS_LIB_COMPUTE_COMMON_MACROS_H_
|
||||
@@ -0,0 +1,90 @@
|
||||
// Copyright 2019 The Fuchsia Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
#include "util.h"
|
||||
|
||||
#include <assert.h>
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#if defined(_MSC_VER) && !defined(__clang__)
|
||||
|
||||
#include <intrin.h>
|
||||
|
||||
#endif
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
bool
|
||||
is_pow2_u32(uint32_t n)
|
||||
{
|
||||
return n && !(n & (n - 1));
|
||||
}
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
uint32_t
|
||||
pow2_ru_u32(uint32_t n)
|
||||
{
|
||||
assert(n <= 0x80000000U);
|
||||
|
||||
n--;
|
||||
n |= n >> 1;
|
||||
n |= n >> 2;
|
||||
n |= n >> 4;
|
||||
n |= n >> 8;
|
||||
n |= n >> 16;
|
||||
n++;
|
||||
|
||||
return n;
|
||||
}
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
uint32_t
|
||||
pow2_rd_u32(uint32_t n)
|
||||
{
|
||||
assert(n > 0);
|
||||
|
||||
return 1u << msb_idx_u32(n);
|
||||
}
|
||||
|
||||
//
|
||||
// ASSUMES NON-ZERO
|
||||
//
|
||||
|
||||
uint32_t
|
||||
msb_idx_u32(uint32_t n)
|
||||
{
|
||||
assert(n > 0);
|
||||
#if defined(_MSC_VER) && !defined(__clang__)
|
||||
|
||||
uint32_t index;
|
||||
|
||||
_BitScanReverse((unsigned long *)&index, n);
|
||||
|
||||
return index;
|
||||
|
||||
#elif defined(__GNUC__)
|
||||
|
||||
return __builtin_clz(n) ^ 31;
|
||||
|
||||
#else
|
||||
|
||||
#error "No msb_index()"
|
||||
|
||||
#endif
|
||||
}
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
@@ -0,0 +1,59 @@
|
||||
// Copyright 2019 The Fuchsia Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
#ifndef SRC_GRAPHICS_LIB_COMPUTE_COMMON_UTIL_H_
|
||||
#define SRC_GRAPHICS_LIB_COMPUTE_COMMON_UTIL_H_
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
// Return true iff |n| is a power of 2.
|
||||
bool
|
||||
is_pow2_u32(uint32_t n);
|
||||
|
||||
// Return |n| rounded-up to the nearest power of 2.
|
||||
// If |n| is zero then return 0.
|
||||
// REQUIRES: |n <= 0x80000000|.
|
||||
uint32_t
|
||||
pow2_ru_u32(uint32_t n);
|
||||
|
||||
// Return |n| rounded-down to the nearest power of 2.
|
||||
// REQUIRES: |n > 0|.
|
||||
uint32_t
|
||||
pow2_rd_u32(uint32_t n);
|
||||
|
||||
// Return the most-significant bit position for |n|.
|
||||
// REQUIRES: |n > 0|.
|
||||
uint32_t
|
||||
msb_idx_u32(uint32_t n); // 0-based bit position
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#endif // SRC_GRAPHICS_LIB_COMPUTE_COMMON_UTIL_H_
|
||||
@@ -0,0 +1,347 @@
|
||||
// Copyright 2019 The Fuchsia Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#include "barrier.h"
|
||||
#include "vulkan/runtime/vk_device.h"
|
||||
#include "vulkan/runtime/vk_command_buffer.h"
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
void
|
||||
vk_barrier_compute_w_to_compute_r(VkCommandBuffer cb)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, cb);
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
|
||||
static VkMemoryBarrier const mb = {
|
||||
|
||||
.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
|
||||
.pNext = NULL,
|
||||
.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT,
|
||||
.dstAccessMask = VK_ACCESS_SHADER_READ_BIT
|
||||
};
|
||||
|
||||
disp->CmdPipelineBarrier(cb,
|
||||
VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
|
||||
VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
|
||||
0,
|
||||
1,
|
||||
&mb,
|
||||
0,
|
||||
NULL,
|
||||
0,
|
||||
NULL);
|
||||
}
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
void
|
||||
vk_barrier_compute_w_to_transfer_r(VkCommandBuffer cb)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, cb);
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
|
||||
static VkMemoryBarrier const mb = {
|
||||
|
||||
.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
|
||||
.pNext = NULL,
|
||||
.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT,
|
||||
.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT
|
||||
};
|
||||
|
||||
disp->CmdPipelineBarrier(cb,
|
||||
VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
|
||||
VK_PIPELINE_STAGE_TRANSFER_BIT,
|
||||
0,
|
||||
1,
|
||||
&mb,
|
||||
0,
|
||||
NULL,
|
||||
0,
|
||||
NULL);
|
||||
}
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
void
|
||||
vk_barrier_transfer_w_to_compute_r(VkCommandBuffer cb)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, cb);
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
|
||||
static VkMemoryBarrier const mb = {
|
||||
|
||||
.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
|
||||
.pNext = NULL,
|
||||
.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
|
||||
.dstAccessMask = VK_ACCESS_SHADER_READ_BIT
|
||||
};
|
||||
|
||||
disp->CmdPipelineBarrier(cb,
|
||||
VK_PIPELINE_STAGE_TRANSFER_BIT,
|
||||
VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
|
||||
0,
|
||||
1,
|
||||
&mb,
|
||||
0,
|
||||
NULL,
|
||||
0,
|
||||
NULL);
|
||||
}
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
void
|
||||
vk_barrier_transfer_w_to_compute_w(VkCommandBuffer cb)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, cb);
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
|
||||
static VkMemoryBarrier const mb = {
|
||||
|
||||
.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
|
||||
.pNext = NULL,
|
||||
.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
|
||||
.dstAccessMask = VK_ACCESS_SHADER_WRITE_BIT
|
||||
};
|
||||
|
||||
disp->CmdPipelineBarrier(cb,
|
||||
VK_PIPELINE_STAGE_TRANSFER_BIT,
|
||||
VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
|
||||
0,
|
||||
1,
|
||||
&mb,
|
||||
0,
|
||||
NULL,
|
||||
0,
|
||||
NULL);
|
||||
}
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
void
|
||||
vk_barrier_compute_w_to_indirect_compute_r(VkCommandBuffer cb)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, cb);
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
|
||||
static VkMemoryBarrier const mb = {
|
||||
|
||||
.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
|
||||
.pNext = NULL,
|
||||
.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT,
|
||||
.dstAccessMask = VK_ACCESS_INDIRECT_COMMAND_READ_BIT | //
|
||||
VK_ACCESS_SHADER_READ_BIT
|
||||
};
|
||||
|
||||
disp->CmdPipelineBarrier(cb,
|
||||
VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
|
||||
VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
|
||||
0,
|
||||
1,
|
||||
&mb,
|
||||
0,
|
||||
NULL,
|
||||
0,
|
||||
NULL);
|
||||
}
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
void
|
||||
vk_barrier_transfer_w_compute_w_to_transfer_r(VkCommandBuffer cb)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, cb);
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
|
||||
static VkMemoryBarrier const mb = {
|
||||
|
||||
.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
|
||||
.pNext = NULL,
|
||||
.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT | //
|
||||
VK_ACCESS_SHADER_WRITE_BIT,
|
||||
.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT
|
||||
};
|
||||
|
||||
disp->CmdPipelineBarrier(cb,
|
||||
VK_PIPELINE_STAGE_TRANSFER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
|
||||
VK_PIPELINE_STAGE_TRANSFER_BIT,
|
||||
0,
|
||||
1,
|
||||
&mb,
|
||||
0,
|
||||
NULL,
|
||||
0,
|
||||
NULL);
|
||||
}
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
void
|
||||
vk_barrier_compute_w_to_host_r(VkCommandBuffer cb)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, cb);
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
|
||||
static VkMemoryBarrier const mb = {
|
||||
|
||||
.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
|
||||
.pNext = NULL,
|
||||
.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT,
|
||||
.dstAccessMask = VK_ACCESS_HOST_READ_BIT
|
||||
};
|
||||
|
||||
disp->CmdPipelineBarrier(cb,
|
||||
VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
|
||||
VK_PIPELINE_STAGE_HOST_BIT,
|
||||
0,
|
||||
1,
|
||||
&mb,
|
||||
0,
|
||||
NULL,
|
||||
0,
|
||||
NULL);
|
||||
}
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
void
|
||||
vk_barrier_transfer_w_to_host_r(VkCommandBuffer cb)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, cb);
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
|
||||
static VkMemoryBarrier const mb = {
|
||||
|
||||
.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
|
||||
.pNext = NULL,
|
||||
.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
|
||||
.dstAccessMask = VK_ACCESS_HOST_READ_BIT
|
||||
};
|
||||
|
||||
disp->CmdPipelineBarrier(cb,
|
||||
VK_PIPELINE_STAGE_TRANSFER_BIT,
|
||||
VK_PIPELINE_STAGE_HOST_BIT,
|
||||
0,
|
||||
1,
|
||||
&mb,
|
||||
0,
|
||||
NULL,
|
||||
0,
|
||||
NULL);
|
||||
}
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
void
|
||||
vk_memory_barrier(VkCommandBuffer cb,
|
||||
VkPipelineStageFlags src_stage,
|
||||
VkAccessFlags src_mask,
|
||||
VkPipelineStageFlags dst_stage,
|
||||
VkAccessFlags dst_mask)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, cb);
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
|
||||
VkMemoryBarrier const mb = { .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
|
||||
.pNext = NULL,
|
||||
.srcAccessMask = src_mask,
|
||||
.dstAccessMask = dst_mask };
|
||||
|
||||
disp->CmdPipelineBarrier(cb, src_stage, dst_stage, 0, 1, &mb, 0, NULL, 0, NULL);
|
||||
}
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
void
|
||||
vk_barrier_debug(VkCommandBuffer cb)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, cb);
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
|
||||
static VkMemoryBarrier const mb = {
|
||||
|
||||
.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
|
||||
.pNext = NULL,
|
||||
.srcAccessMask = VK_ACCESS_INDIRECT_COMMAND_READ_BIT | //
|
||||
VK_ACCESS_INDEX_READ_BIT | //
|
||||
VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT | //
|
||||
VK_ACCESS_UNIFORM_READ_BIT | //
|
||||
VK_ACCESS_INPUT_ATTACHMENT_READ_BIT | //
|
||||
VK_ACCESS_SHADER_READ_BIT | //
|
||||
VK_ACCESS_SHADER_WRITE_BIT | //
|
||||
VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | //
|
||||
VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | //
|
||||
VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT | //
|
||||
VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | //
|
||||
VK_ACCESS_TRANSFER_READ_BIT | //
|
||||
VK_ACCESS_TRANSFER_WRITE_BIT | //
|
||||
VK_ACCESS_HOST_READ_BIT | //
|
||||
VK_ACCESS_HOST_WRITE_BIT,
|
||||
.dstAccessMask = VK_ACCESS_INDIRECT_COMMAND_READ_BIT | //
|
||||
VK_ACCESS_INDEX_READ_BIT | //
|
||||
VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT | //
|
||||
VK_ACCESS_UNIFORM_READ_BIT | //
|
||||
VK_ACCESS_INPUT_ATTACHMENT_READ_BIT | //
|
||||
VK_ACCESS_SHADER_READ_BIT | //
|
||||
VK_ACCESS_SHADER_WRITE_BIT | //
|
||||
VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | //
|
||||
VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | //
|
||||
VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT | //
|
||||
VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | //
|
||||
VK_ACCESS_TRANSFER_READ_BIT | //
|
||||
VK_ACCESS_TRANSFER_WRITE_BIT | //
|
||||
VK_ACCESS_HOST_READ_BIT | //
|
||||
VK_ACCESS_HOST_WRITE_BIT
|
||||
};
|
||||
|
||||
disp->CmdPipelineBarrier(cb,
|
||||
VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
|
||||
VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
|
||||
0,
|
||||
1,
|
||||
&mb,
|
||||
0,
|
||||
NULL,
|
||||
0,
|
||||
NULL);
|
||||
}
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
@@ -0,0 +1,72 @@
|
||||
// Copyright 2019 The Fuchsia Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
#ifndef SRC_GRAPHICS_LIB_COMPUTE_COMMON_VK_BARRIER_H_
|
||||
#define SRC_GRAPHICS_LIB_COMPUTE_COMMON_VK_BARRIER_H_
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#include <vulkan/vulkan_core.h>
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
void
|
||||
vk_barrier_compute_w_to_compute_r(VkCommandBuffer cb);
|
||||
|
||||
void
|
||||
vk_barrier_compute_w_to_transfer_r(VkCommandBuffer cb);
|
||||
|
||||
void
|
||||
vk_barrier_transfer_w_to_compute_r(VkCommandBuffer cb);
|
||||
|
||||
void
|
||||
vk_barrier_transfer_w_to_compute_w(VkCommandBuffer cb);
|
||||
|
||||
void
|
||||
vk_barrier_compute_w_to_indirect_compute_r(VkCommandBuffer cb);
|
||||
|
||||
void
|
||||
vk_barrier_transfer_w_compute_w_to_transfer_r(VkCommandBuffer cb);
|
||||
|
||||
void
|
||||
vk_barrier_compute_w_to_host_r(VkCommandBuffer cb);
|
||||
|
||||
void
|
||||
vk_barrier_transfer_w_to_host_r(VkCommandBuffer cb);
|
||||
|
||||
void
|
||||
vk_memory_barrier(VkCommandBuffer cb,
|
||||
VkPipelineStageFlags src_stage,
|
||||
VkAccessFlags src_mask,
|
||||
VkPipelineStageFlags dst_stage,
|
||||
VkAccessFlags dst_mask);
|
||||
|
||||
void
|
||||
vk_barrier_debug(VkCommandBuffer cb);
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#endif // SRC_GRAPHICS_LIB_COMPUTE_COMMON_VK_BARRIER_H_
|
||||
@@ -0,0 +1,38 @@
|
||||
# Copyright © 2022 Konstantin Seurer
|
||||
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
subdir('shaders')
|
||||
|
||||
radix_sort_files = files(
|
||||
'common/vk/barrier.c',
|
||||
'common/vk/barrier.h',
|
||||
'common/macros.h',
|
||||
'common/util.c',
|
||||
'common/util.h',
|
||||
'shaders/push.h',
|
||||
'radix_sort_u64.c',
|
||||
'radix_sort_u64.h',
|
||||
'radix_sort_u96.c',
|
||||
'radix_sort_vk_devaddr.h',
|
||||
'radix_sort_vk_ext.h',
|
||||
'radix_sort_vk.c',
|
||||
'radix_sort_vk.h',
|
||||
'target.h'
|
||||
)
|
||||
@@ -0,0 +1,59 @@
|
||||
/*
|
||||
* Copyright © 2024 Valve Corporation
|
||||
* SPDX-License-Identifier: MIT
|
||||
*/
|
||||
|
||||
#include "radix_sort_u64.h"
|
||||
#include <assert.h>
|
||||
|
||||
static const uint32_t init_spv[] = {
|
||||
#include "radix_sort/shaders/u64_init.spv.h"
|
||||
};
|
||||
|
||||
static const uint32_t fill_spv[] = {
|
||||
#include "radix_sort/shaders/fill.spv.h"
|
||||
};
|
||||
|
||||
static const uint32_t histogram_spv[] = {
|
||||
#include "radix_sort/shaders/u64_histogram.spv.h"
|
||||
};
|
||||
|
||||
static const uint32_t prefix_spv[] = {
|
||||
#include "radix_sort/shaders/u64_prefix.spv.h"
|
||||
};
|
||||
|
||||
static const uint32_t scatter_0_even_spv[] = {
|
||||
#include "radix_sort/shaders/u64_scatter_0_even.spv.h"
|
||||
};
|
||||
|
||||
static const uint32_t scatter_0_odd_spv[] = {
|
||||
#include "radix_sort/shaders/u64_scatter_0_odd.spv.h"
|
||||
};
|
||||
|
||||
static const uint32_t scatter_1_even_spv[] = {
|
||||
#include "radix_sort/shaders/u64_scatter_1_even.spv.h"
|
||||
};
|
||||
|
||||
static const uint32_t scatter_1_odd_spv[] = {
|
||||
#include "radix_sort/shaders/u64_scatter_1_odd.spv.h"
|
||||
};
|
||||
|
||||
|
||||
radix_sort_vk_t *
|
||||
vk_create_radix_sort_u64(VkDevice device, VkAllocationCallbacks const *ac,
|
||||
VkPipelineCache pc,
|
||||
struct radix_sort_vk_target_config config)
|
||||
{
|
||||
assert(config.keyval_dwords == 2);
|
||||
|
||||
const uint32_t *spv[8] = {
|
||||
init_spv, fill_spv, histogram_spv, prefix_spv,
|
||||
scatter_0_even_spv, scatter_0_odd_spv, scatter_1_even_spv, scatter_1_odd_spv,
|
||||
};
|
||||
const uint32_t spv_sizes[8] = {
|
||||
sizeof(init_spv), sizeof(fill_spv), sizeof(histogram_spv), sizeof(prefix_spv),
|
||||
sizeof(scatter_0_even_spv), sizeof(scatter_0_odd_spv), sizeof(scatter_1_even_spv), sizeof(scatter_1_odd_spv),
|
||||
};
|
||||
return radix_sort_vk_create(device, ac, pc, spv, spv_sizes, config);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
/*
|
||||
* Copyright © 2024 Valve Corporation
|
||||
* SPDX-License-Identifier: MIT
|
||||
*/
|
||||
|
||||
#ifndef VK_RADIX_SORT_U64
|
||||
#define VK_RADIX_SORT_U64
|
||||
|
||||
#include "radix_sort_vk.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
radix_sort_vk_t *
|
||||
vk_create_radix_sort_u64(VkDevice device, VkAllocationCallbacks const *ac,
|
||||
VkPipelineCache pc,
|
||||
struct radix_sort_vk_target_config config);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,69 @@
|
||||
/*
|
||||
* Copyright © 2024 Valve Corporation
|
||||
* SPDX-License-Identifier: MIT
|
||||
*/
|
||||
|
||||
#include "radix_sort_u96.h"
|
||||
#include <assert.h>
|
||||
|
||||
static const uint32_t init_spv[] = {
|
||||
#include "radix_sort/shaders/u96_init.spv.h"
|
||||
};
|
||||
|
||||
static const uint32_t fill_spv[] = {
|
||||
#include "radix_sort/shaders/fill.spv.h"
|
||||
};
|
||||
|
||||
static const uint32_t histogram_spv[] = {
|
||||
#include "radix_sort/shaders/u96_histogram.spv.h"
|
||||
};
|
||||
|
||||
static const uint32_t prefix_spv[] = {
|
||||
#include "radix_sort/shaders/u96_prefix.spv.h"
|
||||
};
|
||||
|
||||
static const uint32_t scatter_0_even_spv[] = {
|
||||
#include "radix_sort/shaders/u96_scatter_0_even.spv.h"
|
||||
};
|
||||
|
||||
static const uint32_t scatter_0_odd_spv[] = {
|
||||
#include "radix_sort/shaders/u96_scatter_0_odd.spv.h"
|
||||
};
|
||||
|
||||
static const uint32_t scatter_1_even_spv[] = {
|
||||
#include "radix_sort/shaders/u96_scatter_1_even.spv.h"
|
||||
};
|
||||
|
||||
static const uint32_t scatter_1_odd_spv[] = {
|
||||
#include "radix_sort/shaders/u96_scatter_1_odd.spv.h"
|
||||
};
|
||||
|
||||
static const uint32_t scatter_2_even_spv[] = {
|
||||
#include "radix_sort/shaders/u96_scatter_2_even.spv.h"
|
||||
};
|
||||
|
||||
static const uint32_t scatter_2_odd_spv[] = {
|
||||
#include "radix_sort/shaders/u96_scatter_2_odd.spv.h"
|
||||
};
|
||||
|
||||
|
||||
radix_sort_vk_t *
|
||||
vk_create_radix_sort_u96(VkDevice device, VkAllocationCallbacks const *ac,
|
||||
VkPipelineCache pc,
|
||||
struct radix_sort_vk_target_config config)
|
||||
{
|
||||
assert(config.keyval_dwords == 3);
|
||||
|
||||
const uint32_t *spv[10] = {
|
||||
init_spv, fill_spv, histogram_spv, prefix_spv,
|
||||
scatter_0_even_spv, scatter_0_odd_spv, scatter_1_even_spv, scatter_1_odd_spv,
|
||||
scatter_2_even_spv, scatter_2_odd_spv,
|
||||
};
|
||||
const uint32_t spv_sizes[10] = {
|
||||
sizeof(init_spv), sizeof(fill_spv), sizeof(histogram_spv), sizeof(prefix_spv),
|
||||
sizeof(scatter_0_even_spv), sizeof(scatter_0_odd_spv), sizeof(scatter_1_even_spv), sizeof(scatter_1_odd_spv),
|
||||
sizeof(scatter_2_even_spv), sizeof(scatter_2_odd_spv),
|
||||
};
|
||||
return radix_sort_vk_create(device, ac, pc, spv, spv_sizes, config);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
/*
|
||||
* Copyright © 2024 Valve Corporation
|
||||
* SPDX-License-Identifier: MIT
|
||||
*/
|
||||
|
||||
#ifndef VK_RADIX_SORT_U96
|
||||
#define VK_RADIX_SORT_U96
|
||||
|
||||
#include "radix_sort_vk.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
radix_sort_vk_t *
|
||||
vk_create_radix_sort_u96(VkDevice device, VkAllocationCallbacks const *ac,
|
||||
VkPipelineCache pc,
|
||||
struct radix_sort_vk_target_config config);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,384 @@
|
||||
// Copyright 2021 The Fuchsia Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
#ifndef SRC_GRAPHICS_LIB_COMPUTE_RADIX_SORT_PLATFORMS_VK_INCLUDE_RADIX_SORT_PLATFORMS_VK_RADIX_SORT_VK_H_
|
||||
#define SRC_GRAPHICS_LIB_COMPUTE_RADIX_SORT_PLATFORMS_VK_INCLUDE_RADIX_SORT_PLATFORMS_VK_RADIX_SORT_VK_H_
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#include <vulkan/vulkan_core.h>
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#include "target.h"
|
||||
|
||||
//
|
||||
// Radix Sort Vk is a high-performance sorting library for Vulkan 1.2.
|
||||
//
|
||||
// The sorting function is both directly and indirectly dispatchable.
|
||||
//
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
//
|
||||
// Get a Radix Sort target's Vulkan requirements.
|
||||
//
|
||||
// A Radix Sort target is a binary image containing configuration parameters and
|
||||
// a bundle of SPIR-V modules.
|
||||
//
|
||||
// Targets are prebuilt and specific to a particular device vendor, architecture
|
||||
// and key-val configuration.
|
||||
//
|
||||
// A Radix Sort instance can only be created with a VkDevice that is initialized
|
||||
// with all of the target's required extensions and features.
|
||||
//
|
||||
// The `radix_sort_vk_target_get_requirements()` function yields the extensions
|
||||
// and initialized feature flags required by a Radix Sort target.
|
||||
//
|
||||
// These requirements can be merged with other Vulkan library requirements
|
||||
// before VkDevice creation.
|
||||
//
|
||||
// If the `.ext_names` member is NULL, the `.ext_name_count` member will be
|
||||
// initialized.
|
||||
//
|
||||
// Returns `false` if:
|
||||
//
|
||||
// * The .ext_names field is NULL and the number of required extensions is
|
||||
// greater than zero.
|
||||
// * The .ext_name_count is less than the number of required extensions is
|
||||
// greater than zero.
|
||||
// * Any of the .pdf, .pdf11 or .pdf12 members are NULL.
|
||||
//
|
||||
// Otherwise, returns true.
|
||||
//
|
||||
typedef struct radix_sort_vk_target radix_sort_vk_target_t;
|
||||
|
||||
//
|
||||
// NOTE: The library currently supports uint32_t and uint64_t keyvals.
|
||||
//
|
||||
|
||||
#define RS_KV_DWORDS_MAX 3
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
struct rs_pipeline_layout_scatter
|
||||
{
|
||||
VkPipelineLayout even;
|
||||
VkPipelineLayout odd;
|
||||
};
|
||||
|
||||
struct rs_pipeline_scatter
|
||||
{
|
||||
VkPipeline even;
|
||||
VkPipeline odd;
|
||||
};
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
struct rs_pipeline_layouts_named
|
||||
{
|
||||
VkPipelineLayout init;
|
||||
VkPipelineLayout fill;
|
||||
VkPipelineLayout histogram;
|
||||
VkPipelineLayout prefix;
|
||||
struct rs_pipeline_layout_scatter scatter[RS_KV_DWORDS_MAX];
|
||||
};
|
||||
|
||||
struct rs_pipelines_named
|
||||
{
|
||||
VkPipeline init;
|
||||
VkPipeline fill;
|
||||
VkPipeline histogram;
|
||||
VkPipeline prefix;
|
||||
struct rs_pipeline_scatter scatter[RS_KV_DWORDS_MAX];
|
||||
};
|
||||
|
||||
// clang-format off
|
||||
#define RS_PIPELINE_LAYOUTS_HANDLES (sizeof(struct rs_pipeline_layouts_named) / sizeof(VkPipelineLayout))
|
||||
#define RS_PIPELINES_HANDLES (sizeof(struct rs_pipelines_named) / sizeof(VkPipeline))
|
||||
// clang-format on
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
struct radix_sort_vk
|
||||
{
|
||||
struct radix_sort_vk_target_config config;
|
||||
|
||||
union
|
||||
{
|
||||
struct rs_pipeline_layouts_named named;
|
||||
VkPipelineLayout handles[RS_PIPELINE_LAYOUTS_HANDLES];
|
||||
} pipeline_layouts;
|
||||
|
||||
union
|
||||
{
|
||||
struct rs_pipelines_named named;
|
||||
VkPipeline handles[RS_PIPELINES_HANDLES];
|
||||
} pipelines;
|
||||
|
||||
struct
|
||||
{
|
||||
struct
|
||||
{
|
||||
VkDeviceSize offset;
|
||||
VkDeviceSize range;
|
||||
} histograms;
|
||||
|
||||
struct
|
||||
{
|
||||
VkDeviceSize offset;
|
||||
} partitions;
|
||||
|
||||
} internal;
|
||||
};
|
||||
|
||||
//
|
||||
// Create a Radix Sort instance for a target.(VkCommandBuffer cb,
|
||||
//
|
||||
// Keyval size is implicitly determined by the target.
|
||||
//
|
||||
// Returns NULL on failure.
|
||||
//
|
||||
typedef struct radix_sort_vk radix_sort_vk_t;
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
radix_sort_vk_t *
|
||||
radix_sort_vk_create(VkDevice device,
|
||||
VkAllocationCallbacks const * ac,
|
||||
VkPipelineCache pc,
|
||||
const uint32_t* const* spv,
|
||||
const uint32_t* spv_sizes,
|
||||
struct radix_sort_vk_target_config config);
|
||||
|
||||
//
|
||||
// Destroy the Radix Sort instance using the same device and allocator used at
|
||||
// creation.
|
||||
//
|
||||
void
|
||||
radix_sort_vk_destroy(radix_sort_vk_t * rs, //
|
||||
VkDevice d,
|
||||
VkAllocationCallbacks const * ac);
|
||||
|
||||
//
|
||||
// Returns the buffer size and alignment requirements for a maximum number of
|
||||
// keyvals.
|
||||
//
|
||||
// The radix sort implementation is not an in-place sorting algorithm so two
|
||||
// non-overlapping keyval buffers are required that are at least
|
||||
// `.keyvals_size`.
|
||||
//
|
||||
// The radix sort instance also requires an `internal` buffer during sorting.
|
||||
//
|
||||
// If the indirect dispatch sorting function is used, then an `indirect` buffer
|
||||
// is also required.
|
||||
//
|
||||
// The alignment requirements for the keyval, internal, and indirect buffers
|
||||
// must be honored. All alignments are power of 2.
|
||||
//
|
||||
// Input:
|
||||
// count : Maximum number of keyvals
|
||||
//
|
||||
// Outputs:
|
||||
// keyval_size : Size of a single keyval
|
||||
//
|
||||
// keyvals_size : Minimum size of the even and odd keyval buffers
|
||||
// keyvals_alignment : Alignment of each keyval buffer
|
||||
//
|
||||
// internal_size : Minimum size of internal buffer
|
||||
// internal_alignment : Alignment of the internal buffer
|
||||
//
|
||||
// indirect_size : Minimum size of indirect buffer
|
||||
// indirect_alignment : Alignment of the indirect buffer
|
||||
//
|
||||
// .keyvals_even/odd
|
||||
// -----------------
|
||||
// VK_BUFFER_USAGE_STORAGE_BUFFER_BIT
|
||||
// VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT
|
||||
//
|
||||
// .internal
|
||||
// ---------
|
||||
// VK_BUFFER_USAGE_STORAGE_BUFFER_BIT
|
||||
// VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT
|
||||
// VK_BUFFER_USAGE_TRANSFER_DST_BIT ("direct" mode only)
|
||||
//
|
||||
// .indirect
|
||||
// ---------
|
||||
// VK_BUFFER_USAGE_STORAGE_BUFFER_BIT
|
||||
// VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT
|
||||
// VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT
|
||||
//
|
||||
typedef struct radix_sort_vk_memory_requirements
|
||||
{
|
||||
VkDeviceSize keyval_size;
|
||||
|
||||
VkDeviceSize keyvals_size;
|
||||
VkDeviceSize keyvals_alignment;
|
||||
|
||||
VkDeviceSize internal_size;
|
||||
VkDeviceSize internal_alignment;
|
||||
|
||||
VkDeviceSize indirect_size;
|
||||
VkDeviceSize indirect_alignment;
|
||||
} radix_sort_vk_memory_requirements_t;
|
||||
|
||||
void
|
||||
radix_sort_vk_get_memory_requirements(radix_sort_vk_t const * rs,
|
||||
uint32_t count,
|
||||
radix_sort_vk_memory_requirements_t * mr);
|
||||
|
||||
//
|
||||
// Direct dispatch sorting
|
||||
// -----------------------
|
||||
//
|
||||
// Using a key size of `key_bits`, sort `count` keyvals found in the
|
||||
// `.devaddr_keyvals_even` buffer.
|
||||
//
|
||||
// Each internal sorting pass copies the keyvals from one keyvals buffer to the
|
||||
// other.
|
||||
//
|
||||
// The number of internal sorting passes is determined by `.key_bits`.
|
||||
//
|
||||
// If an even number of internal sorting passes is required, the sorted keyvals
|
||||
// will be found in the "even" keyvals buffer. Otherwise, the sorted keyvals
|
||||
// will be found in the "odd" keyvals buffer.
|
||||
//
|
||||
// Which buffer has the sorted keyvals is returned in `keyvals_sorted`.
|
||||
//
|
||||
// A keyval's `key_bits` are the most significant bits of a keyval.
|
||||
//
|
||||
// The maximum number of key bits is determined by the keyval size.
|
||||
//
|
||||
// The keyval count must be less than (1 << 30) as well as be less than or equal
|
||||
// to the count used to obtain the the memory requirements.
|
||||
//
|
||||
// The info struct's `ext` member must be NULL.
|
||||
//
|
||||
// This function appends push constants, dispatch commands, and barriers.
|
||||
//
|
||||
// Pipeline barriers should be applied as necessary, both before and after
|
||||
// invoking this function.
|
||||
//
|
||||
// The sort begins with either a TRANSFER/WRITE or a COMPUTE/READ to the
|
||||
// `internal` and `keyvals_even` buffers.
|
||||
//
|
||||
// The sort ends with a COMPUTE/WRITE to the `internal` and `keyvals_sorted`
|
||||
// buffers.
|
||||
//
|
||||
|
||||
//
|
||||
// Direct dispatch sorting using VkDescriptorBufferInfo structures
|
||||
// ---------------------------------------------------------------
|
||||
//
|
||||
typedef struct radix_sort_vk_sort_info
|
||||
{
|
||||
void * ext;
|
||||
uint32_t key_bits;
|
||||
uint32_t count;
|
||||
VkDescriptorBufferInfo keyvals_even;
|
||||
VkDescriptorBufferInfo keyvals_odd;
|
||||
VkDescriptorBufferInfo internal;
|
||||
} radix_sort_vk_sort_info_t;
|
||||
|
||||
void
|
||||
radix_sort_vk_sort(radix_sort_vk_t const * rs,
|
||||
radix_sort_vk_sort_info_t const * info,
|
||||
VkDevice device,
|
||||
VkCommandBuffer cb,
|
||||
VkDescriptorBufferInfo * keyvals_sorted);
|
||||
|
||||
//
|
||||
// Indirect dispatch sorting
|
||||
// -------------------------
|
||||
//
|
||||
// Using a key size of `key_bits`, at pipeline execution time, load keyvals
|
||||
// count from `devaddr_count` and sorts the keyvals in `.devaddr_keyvals_even`.
|
||||
//
|
||||
// Each internal sorting pass copies the keyvals from one keyvals buffer to the
|
||||
// other.
|
||||
//
|
||||
// The number of internal sorting passes is determined by `.key_bits`.
|
||||
//
|
||||
// If an even number of internal sorting passes is required, the sorted keyvals
|
||||
// will be found in the "even" keyvals buffer. Otherwise, the sorted keyvals
|
||||
// will be found in the "odd" keyvals buffer.
|
||||
//
|
||||
// Which buffer has the sorted keyvals is returned in `keyvals_sorted`.
|
||||
//
|
||||
// A keyval's `key_bits` are the most significant bits of a keyval.
|
||||
//
|
||||
// The keyval count must be less than (1 << 30) as well as be less than or equal
|
||||
// to the count used to obtain the the memory requirements.
|
||||
//
|
||||
// The info struct's `ext` member must be NULL.
|
||||
//
|
||||
// This function appends push constants, dispatch commands, and barriers.
|
||||
//
|
||||
// Pipeline barriers should be applied as necessary, both before and after
|
||||
// invoking this function.
|
||||
//
|
||||
// The indirect radix sort begins with a COMPUTE/READ from the `count` buffer
|
||||
// and ends with a COMPUTE/WRITE to the `internal` and the `keyvals_sorted`
|
||||
// buffers.
|
||||
//
|
||||
// The `indirect` buffer must support USAGE_INDIRECT.
|
||||
//
|
||||
// The `count` buffer must be at least 4 bytes and 4-byte aligned.
|
||||
//
|
||||
|
||||
//
|
||||
// Indirect dispatch sorting using VkDescriptorBufferInfo structures
|
||||
// -----------------------------------------------------------------
|
||||
//
|
||||
typedef struct radix_sort_vk_sort_indirect_info
|
||||
{
|
||||
void * ext;
|
||||
uint32_t key_bits;
|
||||
VkDescriptorBufferInfo count;
|
||||
VkDescriptorBufferInfo keyvals_even;
|
||||
VkDescriptorBufferInfo keyvals_odd;
|
||||
VkDescriptorBufferInfo internal;
|
||||
VkDescriptorBufferInfo indirect;
|
||||
} radix_sort_vk_sort_indirect_info_t;
|
||||
|
||||
void
|
||||
radix_sort_vk_sort_indirect(radix_sort_vk_t const * rs,
|
||||
radix_sort_vk_sort_indirect_info_t const * info,
|
||||
VkDevice device,
|
||||
VkCommandBuffer cb,
|
||||
VkDescriptorBufferInfo * keyvals_sorted);
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#endif // SRC_GRAPHICS_LIB_COMPUTE_RADIX_SORT_PLATFORMS_VK_INCLUDE_RADIX_SORT_PLATFORMS_VK_RADIX_SORT_VK_H_
|
||||
@@ -0,0 +1,104 @@
|
||||
// Copyright 2021 The Fuchsia Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
#ifndef SRC_GRAPHICS_LIB_COMPUTE_RADIX_SORT_PLATFORMS_VK_INCLUDE_RADIX_SORT_PLATFORMS_VK_RADIX_SORT_VK_DEVADDR_H_
|
||||
#define SRC_GRAPHICS_LIB_COMPUTE_RADIX_SORT_PLATFORMS_VK_INCLUDE_RADIX_SORT_PLATFORMS_VK_RADIX_SORT_VK_DEVADDR_H_
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#include "radix_sort_vk.h"
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
//
|
||||
// Structure that enables integration with Vulkan drivers.
|
||||
//
|
||||
typedef struct radix_sort_vk_buffer_info
|
||||
{
|
||||
VkBuffer buffer;
|
||||
VkDeviceSize offset;
|
||||
VkDeviceAddress devaddr;
|
||||
} radix_sort_vk_buffer_info_t;
|
||||
|
||||
//
|
||||
// Function prototypes
|
||||
//
|
||||
typedef void (*radix_sort_vk_fill_buffer_pfn)(VkCommandBuffer cb,
|
||||
radix_sort_vk_buffer_info_t const * buffer_info,
|
||||
VkDeviceSize offset,
|
||||
VkDeviceSize size,
|
||||
uint32_t data);
|
||||
|
||||
typedef void (*radix_sort_vk_dispatch_indirect_pfn)(VkCommandBuffer cb,
|
||||
radix_sort_vk_buffer_info_t const * buffer_info,
|
||||
VkDeviceSize offset);
|
||||
|
||||
//
|
||||
// Direct dispatch sorting using buffer device addresses
|
||||
// -----------------------------------------------------
|
||||
//
|
||||
typedef struct radix_sort_vk_sort_devaddr_info
|
||||
{
|
||||
void * ext;
|
||||
uint32_t key_bits;
|
||||
uint32_t count;
|
||||
radix_sort_vk_buffer_info_t keyvals_even;
|
||||
VkDeviceAddress keyvals_odd;
|
||||
radix_sort_vk_buffer_info_t internal;
|
||||
radix_sort_vk_fill_buffer_pfn fill_buffer;
|
||||
} radix_sort_vk_sort_devaddr_info_t;
|
||||
|
||||
void
|
||||
radix_sort_vk_sort_devaddr(radix_sort_vk_t const * rs,
|
||||
radix_sort_vk_sort_devaddr_info_t const * info,
|
||||
VkDevice device,
|
||||
VkCommandBuffer cb,
|
||||
VkDeviceAddress * keyvals_sorted);
|
||||
|
||||
//
|
||||
// Indirect dispatch sorting using buffer device addresses
|
||||
// -------------------------------------------------------
|
||||
//
|
||||
// clang-format off
|
||||
//
|
||||
typedef struct radix_sort_vk_sort_indirect_devaddr_info
|
||||
{
|
||||
void * ext;
|
||||
uint32_t key_bits;
|
||||
VkDeviceAddress count;
|
||||
VkDeviceAddress keyvals_even;
|
||||
VkDeviceAddress keyvals_odd;
|
||||
VkDeviceAddress internal;
|
||||
radix_sort_vk_buffer_info_t indirect;
|
||||
radix_sort_vk_dispatch_indirect_pfn dispatch_indirect;
|
||||
} radix_sort_vk_sort_indirect_devaddr_info_t;
|
||||
|
||||
void
|
||||
radix_sort_vk_sort_indirect_devaddr(radix_sort_vk_t const * rs,
|
||||
radix_sort_vk_sort_indirect_devaddr_info_t const * info,
|
||||
VkDevice device,
|
||||
VkCommandBuffer cb,
|
||||
VkDeviceAddress * keyvals_sorted);
|
||||
|
||||
//
|
||||
// clang-format on
|
||||
//
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#endif // SRC_GRAPHICS_LIB_COMPUTE_RADIX_SORT_PLATFORMS_VK_INCLUDE_RADIX_SORT_PLATFORMS_VK_RADIX_SORT_VK_DEVADDR_H_
|
||||
@@ -0,0 +1,77 @@
|
||||
// Copyright 2021 The Fuchsia Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
#ifndef SRC_GRAPHICS_LIB_COMPUTE_RADIX_SORT_PLATFORMS_VK_RADIX_SORT_VK_EXT_H_
|
||||
#define SRC_GRAPHICS_LIB_COMPUTE_RADIX_SORT_PLATFORMS_VK_RADIX_SORT_VK_EXT_H_
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#include <vulkan/vulkan_core.h>
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
//
|
||||
// Radix sort extensions
|
||||
// ---------------------
|
||||
//
|
||||
#ifndef RADIX_SORT_VK_DISABLE_EXTENSIONS
|
||||
|
||||
//
|
||||
// Extension types
|
||||
//
|
||||
enum radix_sort_vk_ext_type
|
||||
{
|
||||
RADIX_SORT_VK_EXT_TIMESTAMPS
|
||||
};
|
||||
|
||||
//
|
||||
// Timestamp each logical step of the algorithm
|
||||
//
|
||||
// Number of timestamps is: 5 + (number of subpasses)
|
||||
//
|
||||
// * direct dispatch: 4 + subpass count
|
||||
// * indirect dispatch: 5 + subpass count
|
||||
//
|
||||
// Indirect / 32-bit keyvals: 9
|
||||
// Indirect / 64-bit keyvals: 13
|
||||
//
|
||||
struct radix_sort_vk_ext_timestamps
|
||||
{
|
||||
void * ext;
|
||||
enum radix_sort_vk_ext_type type;
|
||||
uint32_t timestamp_count;
|
||||
VkQueryPool timestamps;
|
||||
uint32_t timestamps_set;
|
||||
};
|
||||
|
||||
#endif
|
||||
|
||||
//
|
||||
//
|
||||
//dsc
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#endif // SRC_GRAPHICS_LIB_COMPUTE_RADIX_SORT_PLATFORMS_VK_RADIX_SORT_VK_EXT_H_
|
||||
@@ -0,0 +1,151 @@
|
||||
// Copyright 2021 The Fuchsia Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
#ifndef SRC_GRAPHICS_LIB_COMPUTE_RADIX_SORT_PLATFORMS_VK_SHADERS_BUFREF_H_
|
||||
#define SRC_GRAPHICS_LIB_COMPUTE_RADIX_SORT_PLATFORMS_VK_SHADERS_BUFREF_H_
|
||||
|
||||
//
|
||||
// GLSL
|
||||
//
|
||||
|
||||
#ifdef VULKAN // defined by GLSL/VK compiler
|
||||
|
||||
#extension GL_EXT_shader_explicit_arithmetic_types : require
|
||||
|
||||
//
|
||||
// If the target does not support VkPhysicalDeviceFeatures.shaderInt64
|
||||
// then:
|
||||
//
|
||||
// #define RS_DISABLE_SHADER_INT64
|
||||
//
|
||||
// clang-format off
|
||||
#ifdef RS_DISABLE_SHADER_INT64
|
||||
#extension GL_EXT_buffer_reference_uvec2 : require
|
||||
#else
|
||||
#extension GL_EXT_buffer_reference2 : require
|
||||
#endif
|
||||
// clang-format on
|
||||
|
||||
//
|
||||
// Restrict shouldn't have any noticeable impact on these kernels and
|
||||
// benchmarks appear to prove that true but it's correct to include
|
||||
// the qualifier.
|
||||
//
|
||||
#define RS_RESTRICT restrict
|
||||
|
||||
//
|
||||
// If the device doesn't support .shaderInt64 then the buffer reference address
|
||||
// is a uvec2.
|
||||
//
|
||||
#ifdef RS_DISABLE_SHADER_INT64
|
||||
#define RS_DEVADDR u32vec2
|
||||
#else
|
||||
#define RS_DEVADDR uint64_t
|
||||
#endif
|
||||
|
||||
//
|
||||
// Define a buffer reference.
|
||||
//
|
||||
#define RS_BUFREF_DEFINE(_layout, _name, _devaddr) RS_RESTRICT _layout _name = _layout(_devaddr)
|
||||
|
||||
//
|
||||
// Define a buffer reference at a UINT32 offset.
|
||||
//
|
||||
#ifdef RS_DISABLE_SHADER_INT64
|
||||
#define RS_BUFREF_DEFINE_AT_OFFSET_UINT32(_layout, _name, _devaddr_u32vec2, _offset) \
|
||||
RS_RESTRICT _layout _name; \
|
||||
{ \
|
||||
u32vec2 devaddr; \
|
||||
uint32_t carry; \
|
||||
\
|
||||
devaddr.x = uaddCarry(_devaddr_u32vec2.x, _offset, carry); \
|
||||
devaddr.y = _devaddr_u32vec2.y + carry; \
|
||||
\
|
||||
_name = _layout(devaddr); \
|
||||
}
|
||||
#else
|
||||
#define RS_BUFREF_DEFINE_AT_OFFSET_UINT32(_layout, _name, _devaddr, _offset) \
|
||||
RS_RESTRICT _layout _name = _layout(_devaddr + _offset)
|
||||
#endif
|
||||
|
||||
//
|
||||
// Define a buffer reference at a packed UINT64 offset.
|
||||
//
|
||||
#ifdef RS_DISABLE_SHADER_INT64
|
||||
#define RS_BUFREF_DEFINE_AT_OFFSET_U32VEC2(_layout, _name, _devaddr_u32vec2, _offset_u32vec2) \
|
||||
RS_RESTRICT _layout _name; \
|
||||
{ \
|
||||
u32vec2 devaddr; \
|
||||
uint32_t carry; \
|
||||
\
|
||||
devaddr.x = uaddCarry(_devaddr_u32vec2.x, _offset_u32vec2.x, carry); \
|
||||
devaddr.y = _devaddr_u32vec2.y + _offset_u32vec2.y + carry; \
|
||||
\
|
||||
_name = _layout(devaddr); \
|
||||
}
|
||||
#else
|
||||
#define RS_BUFREF_DEFINE_AT_OFFSET_U32VEC2(_layout, _name, _devaddr, _offset_u32vec2) \
|
||||
RS_RESTRICT _layout _name = _layout(_devaddr + pack64(_offset_u32vec2))
|
||||
#endif
|
||||
|
||||
//
|
||||
// Increment the buffer reference by a UINT32 offset.
|
||||
//
|
||||
#ifdef RS_DISABLE_SHADER_INT64
|
||||
#define RS_BUFREF_INC_UINT32(_layout, _name, _inc) \
|
||||
{ \
|
||||
u32vec2 devaddr = u32vec2(_name); \
|
||||
uint32_t carry; \
|
||||
\
|
||||
devaddr.x = uaddCarry(devaddr.x, _inc, carry); \
|
||||
devaddr.y = devaddr.y + carry; \
|
||||
\
|
||||
_name = _layout(devaddr); \
|
||||
}
|
||||
#else
|
||||
#define RS_BUFREF_INC_UINT32(_layout, _name, _inc) _name = _layout(uint64_t(_name) + _inc)
|
||||
#endif
|
||||
|
||||
//
|
||||
// Increment the buffer reference by a packed UINT64 offset.
|
||||
//
|
||||
#ifdef RS_DISABLE_SHADER_INT64
|
||||
#define RS_BUFREF_INC_U32VEC2(_layout, _name, _inc_u32vec2) \
|
||||
{ \
|
||||
u32vec2 devaddr = u32vec2(_name); \
|
||||
uint32_t carry; \
|
||||
\
|
||||
devaddr.x = uaddCarry(devaddr.x, _inc_u32vec2.x, carry); \
|
||||
devaddr.y = devaddr.y + _inc_u32vec2.y + carry; \
|
||||
\
|
||||
_name = _layout(devaddr); \
|
||||
}
|
||||
#else
|
||||
#define RS_BUFREF_INC_U32VEC2(_layout, _name, _inc_u32vec2) \
|
||||
_name = _layout(uint64_t(_name) + pack64(_inc_u32vec2))
|
||||
#endif
|
||||
|
||||
//
|
||||
// Increment the buffer reference by the product of two UINT32 factors.
|
||||
//
|
||||
#define RS_BUFREF_INC_UINT32_UINT32(_layout, _name, _inc_a, _inc_b) \
|
||||
{ \
|
||||
u32vec2 inc; \
|
||||
\
|
||||
umulExtended(_inc_a, _inc_b, inc.y, inc.x); \
|
||||
\
|
||||
RS_BUFREF_INC_U32VEC2(_layout, _name, inc); \
|
||||
}
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#endif
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#endif // SRC_GRAPHICS_LIB_COMPUTE_RADIX_SORT_PLATFORMS_VK_SHADERS_BUFREF_H_
|
||||
@@ -0,0 +1,33 @@
|
||||
// Copyright 2024 Valve Corporation
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
#ifdef VULKAN
|
||||
#define CONFIG(_name, _id, default_val) layout (constant_id = _id) const int _name = default_val;
|
||||
#else
|
||||
enum rs_config {
|
||||
#define CONFIG(_name, _id, default_val) _name = _id,
|
||||
#endif
|
||||
|
||||
#define RS_FILL_WORKGROUP_SIZE_ID 0
|
||||
CONFIG(RS_FILL_WORKGROUP_SIZE, RS_FILL_WORKGROUP_SIZE_ID, 7)
|
||||
CONFIG(RS_FILL_BLOCK_ROWS, 1, 8)
|
||||
|
||||
#define RS_HISTOGRAM_WORKGROUP_SIZE_ID 2
|
||||
CONFIG(RS_HISTOGRAM_WORKGROUP_SIZE, RS_HISTOGRAM_WORKGROUP_SIZE_ID, 7)
|
||||
CONFIG(RS_HISTOGRAM_SUBGROUP_SIZE_LOG2, 3, 7)
|
||||
CONFIG(RS_HISTOGRAM_BLOCK_ROWS, 4, 8)
|
||||
|
||||
#define RS_PREFIX_WORKGROUP_SIZE_ID 5
|
||||
CONFIG(RS_PREFIX_WORKGROUP_SIZE, RS_PREFIX_WORKGROUP_SIZE_ID, 8)
|
||||
CONFIG(RS_PREFIX_SUBGROUP_SIZE_LOG2, 6, 6)
|
||||
|
||||
#define RS_SCATTER_WORKGROUP_SIZE_ID 7
|
||||
CONFIG(RS_SCATTER_WORKGROUP_SIZE, RS_SCATTER_WORKGROUP_SIZE_ID, 8)
|
||||
CONFIG(RS_SCATTER_SUBGROUP_SIZE_LOG2, 8, 6)
|
||||
CONFIG(RS_SCATTER_BLOCK_ROWS, 9, 14)
|
||||
|
||||
CONFIG(RS_SCATTER_NONSEQUENTIAL_DISPATCH, 10, 0)
|
||||
|
||||
#ifndef VULKAN
|
||||
};
|
||||
#endif
|
||||
@@ -0,0 +1,131 @@
|
||||
// Copyright 2021 The Fuchsia Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
#version 460
|
||||
|
||||
//
|
||||
// Each workgroup fills up to RS_BLOCK_KEYVALS
|
||||
//
|
||||
|
||||
// clang-format off
|
||||
#extension GL_GOOGLE_include_directive : require
|
||||
#extension GL_EXT_control_flow_attributes : require
|
||||
// clang-format on
|
||||
|
||||
//
|
||||
// Load arch/keyval configuration
|
||||
//
|
||||
#include "config.h"
|
||||
|
||||
//
|
||||
// Buffer reference macros and push constants
|
||||
//
|
||||
#include "bufref.h"
|
||||
#include "push.h"
|
||||
|
||||
//
|
||||
// Subgroup uniform support
|
||||
//
|
||||
#if defined(RS_SCATTER_SUBGROUP_UNIFORM_DISABLE) && defined(GL_EXT_subgroupuniform_qualifier)
|
||||
#extension GL_EXT_subgroupuniform_qualifier : required
|
||||
#define RS_SUBGROUP_UNIFORM subgroupuniformEXT
|
||||
#else
|
||||
#define RS_SUBGROUP_UNIFORM
|
||||
#endif
|
||||
|
||||
//
|
||||
// Declare the push constants
|
||||
//
|
||||
RS_STRUCT_PUSH_FILL();
|
||||
|
||||
layout(push_constant) uniform block_push
|
||||
{
|
||||
rs_push_fill push;
|
||||
};
|
||||
|
||||
//
|
||||
// The "init" shader configures the fill info structure.
|
||||
//
|
||||
RS_STRUCT_INDIRECT_INFO_FILL();
|
||||
|
||||
//
|
||||
// Local macros
|
||||
//
|
||||
// clang-format off
|
||||
#define RS_WORKGROUP_SIZE (RS_FILL_WORKGROUP_SIZE)
|
||||
#define RS_BLOCK_DWORDS (RS_FILL_BLOCK_ROWS * RS_WORKGROUP_SIZE)
|
||||
#define RS_RADIX_MASK ((1 << RS_RADIX_LOG2) - 1)
|
||||
// clang-format on
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
layout(local_size_x_id = RS_FILL_WORKGROUP_SIZE_ID) in;
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
layout(buffer_reference, std430) buffer buffer_rs_indirect_info_fill
|
||||
{
|
||||
rs_indirect_info_fill info;
|
||||
};
|
||||
|
||||
layout(buffer_reference, std430) buffer buffer_rs_dwords
|
||||
{
|
||||
uint32_t extent[];
|
||||
};
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
void
|
||||
main()
|
||||
{
|
||||
//
|
||||
// Define indirect info bufref for the fill
|
||||
//
|
||||
readonly RS_BUFREF_DEFINE(buffer_rs_indirect_info_fill, rs_info, push.devaddr_info);
|
||||
|
||||
RS_SUBGROUP_UNIFORM const rs_indirect_info_fill info = rs_info.info;
|
||||
|
||||
//
|
||||
// Define dwords bufref
|
||||
//
|
||||
// Assumes less than 2^32-1 keys and then extended multiplies it by
|
||||
// the keyval size.
|
||||
//
|
||||
// Assumes push.devaddr_dwords_base is suitably aligned to
|
||||
// RS_BLOCK_DWORDS -- at a subgroup or transaction size is fine.
|
||||
//
|
||||
const uint32_t dwords_idx =
|
||||
(info.block_offset + gl_WorkGroupID.x) * RS_BLOCK_DWORDS + gl_LocalInvocationID.x;
|
||||
|
||||
u32vec2 dwords_offset;
|
||||
|
||||
umulExtended(dwords_idx, 4, dwords_offset.y, dwords_offset.x);
|
||||
|
||||
writeonly RS_BUFREF_DEFINE_AT_OFFSET_U32VEC2(buffer_rs_dwords,
|
||||
rs_dwords,
|
||||
push.devaddr_dwords,
|
||||
dwords_offset);
|
||||
|
||||
//
|
||||
// Fills are always aligned to RS_BLOCK_KEYVALS
|
||||
//
|
||||
// ((v >= min) && (v < max)) == ((v - min) < (max - min))
|
||||
//
|
||||
const uint32_t row_idx = dwords_idx - info.dword_offset_min;
|
||||
|
||||
for (uint32_t ii = 0; ii < RS_FILL_BLOCK_ROWS; ii++)
|
||||
{
|
||||
if (row_idx + (ii * RS_WORKGROUP_SIZE) < info.dword_offset_max_minus_min)
|
||||
{
|
||||
rs_dwords.extent[ii * RS_WORKGROUP_SIZE] = push.dword;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
@@ -0,0 +1,441 @@
|
||||
// Copyright 2021 The Fuchsia Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
#version 460
|
||||
|
||||
//
|
||||
// Produce multiple radix size histograms from the keyvals.
|
||||
//
|
||||
|
||||
// clang-format off
|
||||
#extension GL_GOOGLE_include_directive : require
|
||||
#extension GL_EXT_control_flow_attributes : require
|
||||
#extension GL_KHR_shader_subgroup_basic : require
|
||||
#extension GL_EXT_scalar_block_layout : require
|
||||
// clang-format on
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
#include "config.h"
|
||||
|
||||
//
|
||||
// Optional switches:
|
||||
//
|
||||
// #define RS_HISTOGRAM_ENABLE_BITFIELD_EXTRACT
|
||||
// #define RS_HISTOGRAM_DISABLE_SMEM_HISTOGRAM
|
||||
//
|
||||
|
||||
//
|
||||
// Buffer reference macros and push constants
|
||||
//
|
||||
#include "bufref.h"
|
||||
#include "push.h"
|
||||
|
||||
//
|
||||
// Push constants for histogram shader
|
||||
//
|
||||
RS_STRUCT_PUSH_HISTOGRAM();
|
||||
|
||||
layout(push_constant) uniform block_push
|
||||
{
|
||||
rs_push_histogram push;
|
||||
};
|
||||
|
||||
//
|
||||
// Subgroup uniform support
|
||||
//
|
||||
#if defined(RS_HISTOGRAM_SUBGROUP_UNIFORM_DISABLE) && defined(GL_EXT_subgroupuniform_qualifier)
|
||||
#extension GL_EXT_subgroupuniform_qualifier : required
|
||||
#define RS_SUBGROUP_UNIFORM subgroupuniformEXT
|
||||
#else
|
||||
#define RS_SUBGROUP_UNIFORM
|
||||
#endif
|
||||
|
||||
//
|
||||
// Check all switches are defined
|
||||
//
|
||||
|
||||
// What's the size of the keyval?
|
||||
#ifndef RS_KEYVAL_DWORDS
|
||||
#error "Undefined: RS_KEYVAL_DWORDS"
|
||||
#endif
|
||||
|
||||
//
|
||||
// Local macros
|
||||
//
|
||||
// clang-format off
|
||||
#define RS_WORKGROUP_SIZE (RS_HISTOGRAM_WORKGROUP_SIZE)
|
||||
#define RS_SUBGROUP_SIZE (1 << RS_HISTOGRAM_SUBGROUP_SIZE_LOG2)
|
||||
#define RS_WORKGROUP_SUBGROUPS (RS_WORKGROUP_SIZE / RS_SUBGROUP_SIZE)
|
||||
#define RS_BLOCK_KEYVALS (RS_HISTOGRAM_BLOCK_ROWS * RS_WORKGROUP_SIZE)
|
||||
#define RS_KEYVAL_SIZE (RS_KEYVAL_DWORDS * 4)
|
||||
#define RS_RADIX_MASK ((1 << RS_RADIX_LOG2) - 1)
|
||||
// clang-format on
|
||||
|
||||
//
|
||||
// Keyval type
|
||||
//
|
||||
#if (RS_KEYVAL_DWORDS == 1)
|
||||
#define RS_KEYVAL_TYPE uint32_t
|
||||
#elif (RS_KEYVAL_DWORDS == 2)
|
||||
#define RS_KEYVAL_TYPE u32vec2
|
||||
#elif (RS_KEYVAL_DWORDS == 3)
|
||||
#define RS_KEYVAL_TYPE u32vec3
|
||||
#else
|
||||
#error "Unsupported RS_KEYVAL_DWORDS"
|
||||
#endif
|
||||
|
||||
//
|
||||
// Histogram offset depends on number of workgroups.
|
||||
//
|
||||
#define RS_HISTOGRAM_BASE(pass_) ((RS_RADIX_SIZE * 4) * pass_)
|
||||
|
||||
#define RS_HISTOGRAM_OFFSET(pass_) \
|
||||
RS_HISTOGRAM_BASE(pass_) + (RS_WORKGROUP_SUBGROUPS == 1 ? gl_SubgroupInvocationID : gl_LocalInvocationID.x) * 4
|
||||
|
||||
//
|
||||
// Assumes (RS_RADIX_LOG2 == 8)
|
||||
//
|
||||
// Error if this ever changes
|
||||
//
|
||||
#if (RS_RADIX_LOG2 != 8)
|
||||
#error "(RS_RADIX_LOG2 != 8)"
|
||||
#endif
|
||||
|
||||
//
|
||||
// Is bitfield extract faster?
|
||||
//
|
||||
#ifdef RS_HISTOGRAM_ENABLE_BITFIELD_EXTRACT
|
||||
//----------------------------------------------------------------------
|
||||
|
||||
//
|
||||
// Extract a keyval digit
|
||||
//
|
||||
#if (RS_KEYVAL_DWORDS == 1)
|
||||
#define RS_KV_EXTRACT_DIGIT(kv_, pass_) bitfieldExtract(kv_, pass_ * RS_RADIX_LOG2, RS_RADIX_LOG2)
|
||||
#else
|
||||
#define RS_KV_EXTRACT_DIGIT(kv_, pass_) \
|
||||
bitfieldExtract(kv_[pass_ / 4], (pass_ & 3) * RS_RADIX_LOG2, RS_RADIX_LOG2)
|
||||
#endif
|
||||
//----------------------------------------------------------------------
|
||||
#else
|
||||
//----------------------------------------------------------------------
|
||||
|
||||
//
|
||||
// Extract a keyval digit
|
||||
//
|
||||
#if (RS_KEYVAL_DWORDS == 1)
|
||||
#define RS_KV_EXTRACT_DIGIT(kv_, pass_) ((kv_ >> (pass_ * RS_RADIX_LOG2)) & RS_RADIX_MASK)
|
||||
#else
|
||||
#define RS_KV_EXTRACT_DIGIT(kv_, pass_) \
|
||||
((kv_[pass_ / 4] >> ((pass_ & 3) * RS_RADIX_LOG2)) & RS_RADIX_MASK)
|
||||
#endif
|
||||
//----------------------------------------------------------------------
|
||||
#endif
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
#ifndef RS_HISTOGRAM_DISABLE_SMEM_HISTOGRAM
|
||||
|
||||
struct rs_histogram_smem
|
||||
{
|
||||
uint32_t histogram[RS_RADIX_SIZE];
|
||||
};
|
||||
|
||||
shared rs_histogram_smem smem;
|
||||
|
||||
#endif
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
layout(local_size_x_id = RS_HISTOGRAM_WORKGROUP_SIZE_ID) in;
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
layout(buffer_reference, scalar) buffer buffer_rs_kv
|
||||
{
|
||||
RS_KEYVAL_TYPE extent[];
|
||||
};
|
||||
|
||||
layout(buffer_reference, std430) buffer buffer_rs_histograms
|
||||
{
|
||||
uint32_t extent[];
|
||||
};
|
||||
|
||||
//
|
||||
// Shared memory functions
|
||||
//
|
||||
#ifndef RS_HISTOGRAM_DISABLE_SMEM_HISTOGRAM
|
||||
|
||||
//
|
||||
// NOTE: Must use same access pattern as rs_histogram_zero()
|
||||
//
|
||||
void
|
||||
rs_histogram_zero()
|
||||
{
|
||||
//
|
||||
// Zero SMEM histogram
|
||||
//
|
||||
if (RS_WORKGROUP_SUBGROUPS == 1)
|
||||
{
|
||||
const uint32_t smem_offset = gl_SubgroupInvocationID;
|
||||
|
||||
for (RS_SUBGROUP_UNIFORM uint32_t ii = 0; ii < RS_RADIX_SIZE; ii += RS_SUBGROUP_SIZE)
|
||||
{
|
||||
smem.histogram[smem_offset + ii] = 0;
|
||||
}
|
||||
}
|
||||
else if (RS_WORKGROUP_SIZE < RS_RADIX_SIZE)
|
||||
{
|
||||
const uint32_t smem_offset = gl_LocalInvocationID.x;
|
||||
|
||||
for (uint32_t ii = 0; ii < RS_RADIX_SIZE; ii += RS_WORKGROUP_SIZE)
|
||||
{
|
||||
smem.histogram[smem_offset + ii] = 0;
|
||||
}
|
||||
|
||||
const uint32_t smem_idx = smem_offset + ((RS_RADIX_SIZE / RS_WORKGROUP_SIZE) * RS_WORKGROUP_SIZE);
|
||||
|
||||
if (smem_idx < RS_RADIX_SIZE)
|
||||
{
|
||||
smem.histogram[smem_idx] = 0;
|
||||
}
|
||||
}
|
||||
else if (RS_WORKGROUP_SIZE >= RS_RADIX_SIZE)
|
||||
{
|
||||
if (RS_WORKGROUP_SIZE == RS_RADIX_SIZE || gl_LocalInvocationID.x < RS_RADIX_SIZE)
|
||||
{
|
||||
smem.histogram[gl_LocalInvocationID.x] = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// NOTE: Must use same access pattern as rs_histogram_zero()
|
||||
//
|
||||
void
|
||||
rs_histogram_global_store(restrict buffer_rs_histograms rs_histograms)
|
||||
{
|
||||
//
|
||||
// Store to GMEM
|
||||
//
|
||||
if (RS_WORKGROUP_SUBGROUPS == 1)
|
||||
{
|
||||
const uint32_t smem_offset = gl_SubgroupInvocationID;
|
||||
|
||||
for (RS_SUBGROUP_UNIFORM uint32_t ii = 0; ii < RS_RADIX_SIZE; ii += RS_SUBGROUP_SIZE)
|
||||
{
|
||||
const uint32_t count = smem.histogram[smem_offset + ii];
|
||||
|
||||
atomicAdd(rs_histograms.extent[ii], count);
|
||||
}
|
||||
}
|
||||
else if (RS_WORKGROUP_SIZE < RS_RADIX_SIZE)
|
||||
{
|
||||
const uint32_t smem_offset = gl_LocalInvocationID.x;
|
||||
|
||||
for (uint32_t ii = 0; ii < RS_RADIX_SIZE; ii += RS_WORKGROUP_SIZE)
|
||||
{
|
||||
const uint32_t count = smem.histogram[smem_offset + ii];
|
||||
|
||||
atomicAdd(rs_histograms.extent[ii], count);
|
||||
}
|
||||
|
||||
const uint32_t smem_idx = smem_offset + ((RS_RADIX_SIZE / RS_WORKGROUP_SIZE) * RS_WORKGROUP_SIZE);
|
||||
|
||||
if (smem_idx < RS_RADIX_SIZE)
|
||||
{
|
||||
const uint32_t count = smem.histogram[smem_idx];
|
||||
|
||||
atomicAdd(rs_histograms.extent[((RS_RADIX_SIZE / RS_WORKGROUP_SIZE) * RS_WORKGROUP_SIZE)],
|
||||
count);
|
||||
}
|
||||
}
|
||||
else if (RS_WORKGROUP_SIZE >= RS_RADIX_SIZE)
|
||||
{
|
||||
if (RS_WORKGROUP_SIZE == RS_RADIX_SIZE || gl_LocalInvocationID.x < RS_RADIX_SIZE)
|
||||
{
|
||||
const uint32_t count = smem.histogram[gl_LocalInvocationID.x];
|
||||
|
||||
atomicAdd(rs_histograms.extent[0], count);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
#ifndef RS_HISTOGRAM_DISABLE_SMEM_HISTOGRAM
|
||||
|
||||
void
|
||||
rs_histogram_atomic_after_write()
|
||||
{
|
||||
if (RS_WORKGROUP_SUBGROUPS == 1)
|
||||
subgroupMemoryBarrierShared();
|
||||
else
|
||||
barrier();
|
||||
}
|
||||
|
||||
void
|
||||
rs_histogram_read_after_atomic()
|
||||
{
|
||||
if (RS_WORKGROUP_SUBGROUPS == 1)
|
||||
subgroupMemoryBarrierShared();
|
||||
else
|
||||
barrier();
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
void
|
||||
main()
|
||||
{
|
||||
//
|
||||
// Which subgroups have work?
|
||||
//
|
||||
RS_KEYVAL_TYPE kv[RS_HISTOGRAM_BLOCK_ROWS];
|
||||
|
||||
//
|
||||
// Define kv_in bufref
|
||||
//
|
||||
// Assumes less than 2^30-1 keys and then extended multiplies it
|
||||
// by the keyval size.
|
||||
//
|
||||
u32vec2 kv_in_offset;
|
||||
|
||||
umulExtended(gl_WorkGroupID.x * RS_BLOCK_KEYVALS + gl_LocalInvocationID.x,
|
||||
RS_KEYVAL_SIZE,
|
||||
kv_in_offset.y, // msb
|
||||
kv_in_offset.x); // lsb
|
||||
|
||||
readonly RS_BUFREF_DEFINE_AT_OFFSET_U32VEC2(buffer_rs_kv,
|
||||
rs_kv_in,
|
||||
push.devaddr_keyvals,
|
||||
kv_in_offset);
|
||||
|
||||
//
|
||||
// Load keyvals
|
||||
//
|
||||
for (RS_SUBGROUP_UNIFORM uint32_t ii = 0; ii < RS_HISTOGRAM_BLOCK_ROWS; ii++)
|
||||
{
|
||||
kv[ii] = rs_kv_in.extent[ii * RS_WORKGROUP_SIZE];
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// Accumulate and store histograms for passes
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// MACRO EXPANSION VARIANT
|
||||
//
|
||||
// NOTE: THIS ALSO SERVES AS A MALI R24+ WORKAROUND: EXPLICITLY
|
||||
// EXPAND THE FOR/LOOP PASSES
|
||||
//
|
||||
#ifndef RS_HISTOGRAM_DISABLE_SMEM_HISTOGRAM
|
||||
|
||||
#define RS_HISTOGRAM_PASS(pass_) \
|
||||
rs_histogram_zero(); \
|
||||
\
|
||||
rs_histogram_atomic_after_write(); \
|
||||
\
|
||||
for (RS_SUBGROUP_UNIFORM uint32_t jj = 0; jj < RS_HISTOGRAM_BLOCK_ROWS; jj++) \
|
||||
{ \
|
||||
const uint32_t digit = RS_KV_EXTRACT_DIGIT(kv[jj], pass_); \
|
||||
\
|
||||
atomicAdd(smem.histogram[digit], 1); \
|
||||
} \
|
||||
\
|
||||
rs_histogram_read_after_atomic(); \
|
||||
\
|
||||
{ \
|
||||
const uint32_t rs_histogram_offset = RS_HISTOGRAM_OFFSET(pass_); \
|
||||
\
|
||||
RS_BUFREF_DEFINE_AT_OFFSET_UINT32(buffer_rs_histograms, \
|
||||
rs_histograms, \
|
||||
push.devaddr_histograms, \
|
||||
rs_histogram_offset); \
|
||||
\
|
||||
rs_histogram_global_store(rs_histograms); \
|
||||
} \
|
||||
\
|
||||
if (push.passes == (RS_KEYVAL_SIZE - pass_)) \
|
||||
return;
|
||||
|
||||
#else // NO SHARED MEMORY
|
||||
|
||||
#define RS_HISTOGRAM_PASS(pass_) \
|
||||
{ \
|
||||
const uint32_t rs_histogram_base = RS_HISTOGRAM_BASE(pass_); \
|
||||
\
|
||||
RS_BUFREF_DEFINE_AT_OFFSET_UINT32(buffer_rs_histograms, \
|
||||
rs_histograms, \
|
||||
push.devaddr_histograms, \
|
||||
rs_histogram_base); \
|
||||
\
|
||||
for (RS_SUBGROUP_UNIFORM uint32_t jj = 0; jj < RS_HISTOGRAM_BLOCK_ROWS; jj++) \
|
||||
{ \
|
||||
const uint32_t digit = RS_KV_EXTRACT_DIGIT(kv[jj], pass_); \
|
||||
\
|
||||
atomicAdd(rs_histograms.extent[digit], 1); \
|
||||
} \
|
||||
} \
|
||||
\
|
||||
if (push.passes == (RS_KEYVAL_SIZE - pass_)) \
|
||||
return;
|
||||
|
||||
#endif
|
||||
|
||||
#if (RS_KEYVAL_DWORDS == 1)
|
||||
|
||||
RS_HISTOGRAM_PASS(3)
|
||||
RS_HISTOGRAM_PASS(2)
|
||||
RS_HISTOGRAM_PASS(1)
|
||||
RS_HISTOGRAM_PASS(0)
|
||||
|
||||
#elif (RS_KEYVAL_DWORDS == 2)
|
||||
|
||||
RS_HISTOGRAM_PASS(7)
|
||||
RS_HISTOGRAM_PASS(6)
|
||||
RS_HISTOGRAM_PASS(5)
|
||||
RS_HISTOGRAM_PASS(4)
|
||||
RS_HISTOGRAM_PASS(3)
|
||||
RS_HISTOGRAM_PASS(2)
|
||||
RS_HISTOGRAM_PASS(1)
|
||||
RS_HISTOGRAM_PASS(0)
|
||||
|
||||
#elif (RS_KEYVAL_DWORDS == 3)
|
||||
|
||||
RS_HISTOGRAM_PASS(11)
|
||||
RS_HISTOGRAM_PASS(10)
|
||||
RS_HISTOGRAM_PASS(9)
|
||||
RS_HISTOGRAM_PASS(8)
|
||||
RS_HISTOGRAM_PASS(7)
|
||||
RS_HISTOGRAM_PASS(6)
|
||||
RS_HISTOGRAM_PASS(5)
|
||||
RS_HISTOGRAM_PASS(4)
|
||||
RS_HISTOGRAM_PASS(3)
|
||||
RS_HISTOGRAM_PASS(2)
|
||||
RS_HISTOGRAM_PASS(1)
|
||||
RS_HISTOGRAM_PASS(0)
|
||||
|
||||
#else
|
||||
#error "Error: (RS_KEYVAL_DWORDS >= 4) not implemented."
|
||||
#endif
|
||||
}
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
@@ -0,0 +1,190 @@
|
||||
// Copyright 2021 The Fuchsia Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
#version 460
|
||||
|
||||
//
|
||||
// Initialize the `rs_indirect_info` struct
|
||||
//
|
||||
|
||||
// clang-format off
|
||||
#extension GL_GOOGLE_include_directive : require
|
||||
#extension GL_EXT_control_flow_attributes : require
|
||||
// clang-format on
|
||||
|
||||
//
|
||||
// Load arch/keyval configuration
|
||||
//
|
||||
#include "config.h"
|
||||
|
||||
//
|
||||
// Buffer reference macros and push constants
|
||||
//
|
||||
#include "bufref.h"
|
||||
#include "push.h"
|
||||
|
||||
//
|
||||
// Subgroup uniform support
|
||||
//
|
||||
#if defined(RS_SCATTER_SUBGROUP_UNIFORM_DISABLE) && defined(GL_EXT_subgroupuniform_qualifier)
|
||||
#extension GL_EXT_subgroupuniform_qualifier : required
|
||||
#define RS_SUBGROUP_UNIFORM subgroupuniformEXT
|
||||
#else
|
||||
#define RS_SUBGROUP_UNIFORM
|
||||
#endif
|
||||
|
||||
//
|
||||
// Declare the push constants
|
||||
//
|
||||
RS_STRUCT_PUSH_INIT();
|
||||
|
||||
layout(push_constant) uniform block_push
|
||||
{
|
||||
rs_push_init push;
|
||||
};
|
||||
|
||||
//
|
||||
// The "init" shader configures the fill info structure.
|
||||
//
|
||||
RS_STRUCT_INDIRECT_INFO();
|
||||
|
||||
//
|
||||
// Local macros
|
||||
//
|
||||
// clang-format off
|
||||
#define RS_FILL_WORKGROUP_SIZE (RS_FILL_WORKGROUP_SIZE)
|
||||
#define RS_SCATTER_WORKGROUP_SIZE (RS_SCATTER_WORKGROUP_SIZE)
|
||||
#define RS_HISTOGRAM_WORKGROUP_SIZE (RS_HISTOGRAM_WORKGROUP_SIZE)
|
||||
|
||||
#define RS_FILL_BLOCK_DWORDS (RS_FILL_BLOCK_ROWS * RS_FILL_WORKGROUP_SIZE)
|
||||
#define RS_SCATTER_BLOCK_KEYVALS (RS_SCATTER_BLOCK_ROWS * RS_SCATTER_WORKGROUP_SIZE)
|
||||
#define RS_HISTOGRAM_BLOCK_KEYVALS (RS_HISTOGRAM_BLOCK_ROWS * RS_HISTOGRAM_WORKGROUP_SIZE)
|
||||
// clang-format on
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
layout(local_size_x = 1) in;
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
layout(buffer_reference, std430) buffer buffer_rs_count
|
||||
{
|
||||
uint32_t count;
|
||||
};
|
||||
|
||||
layout(buffer_reference, std430) buffer buffer_rs_indirect_info
|
||||
{
|
||||
rs_indirect_info info;
|
||||
};
|
||||
|
||||
//
|
||||
// Helper macros
|
||||
//
|
||||
// RU = Round Up
|
||||
// RD = Round Down
|
||||
//
|
||||
#define RS_COUNT_RU_BLOCKS(count_, block_size_) ((count_ + block_size_ - 1) / (block_size_))
|
||||
#define RS_COUNT_RD_BLOCKS(count_, block_size_) ((count_) / (block_size_))
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
void
|
||||
main()
|
||||
{
|
||||
//
|
||||
// Load the keyval count
|
||||
//
|
||||
readonly RS_BUFREF_DEFINE(buffer_rs_count, rs_count, push.devaddr_count);
|
||||
|
||||
RS_SUBGROUP_UNIFORM const uint32_t count = rs_count.count;
|
||||
|
||||
//
|
||||
// Define the init struct bufref
|
||||
//
|
||||
writeonly RS_BUFREF_DEFINE(buffer_rs_indirect_info, rs_indirect_info, push.devaddr_info);
|
||||
|
||||
//
|
||||
// Size and set scatter dispatch
|
||||
//
|
||||
const uint32_t scatter_ru_blocks = RS_COUNT_RU_BLOCKS(count, RS_SCATTER_BLOCK_KEYVALS);
|
||||
const uint32_t count_ru_scatter = scatter_ru_blocks * RS_SCATTER_BLOCK_KEYVALS;
|
||||
|
||||
rs_indirect_info.info.dispatch.scatter = u32vec4(scatter_ru_blocks, 1, 1, 0);
|
||||
|
||||
//
|
||||
// Size and set histogram dispatch
|
||||
//
|
||||
const uint32_t histo_ru_blocks = RS_COUNT_RU_BLOCKS(count_ru_scatter, RS_HISTOGRAM_BLOCK_KEYVALS);
|
||||
const uint32_t count_ru_histo = histo_ru_blocks * RS_HISTOGRAM_BLOCK_KEYVALS;
|
||||
|
||||
rs_indirect_info.info.dispatch.histogram = u32vec4(histo_ru_blocks, 1, 1, 0);
|
||||
|
||||
//
|
||||
// Size and set pad fill and dispatch
|
||||
//
|
||||
const uint32_t count_dwords = count * RS_KEYVAL_DWORDS;
|
||||
const uint32_t pad_rd_blocks = RS_COUNT_RD_BLOCKS(count_dwords, RS_FILL_BLOCK_DWORDS);
|
||||
const uint32_t count_rd_pad = pad_rd_blocks * RS_FILL_BLOCK_DWORDS;
|
||||
const uint32_t count_ru_histo_dwords = count_ru_histo * RS_KEYVAL_DWORDS;
|
||||
const uint32_t pad_dwords = count_ru_histo_dwords - count_rd_pad;
|
||||
const uint32_t pad_ru_blocks = RS_COUNT_RU_BLOCKS(pad_dwords, RS_FILL_BLOCK_DWORDS);
|
||||
|
||||
rs_indirect_info_fill pad;
|
||||
|
||||
pad.block_offset = pad_rd_blocks;
|
||||
pad.dword_offset_min = count_dwords;
|
||||
pad.dword_offset_max_minus_min = count_ru_histo_dwords - count_dwords;
|
||||
|
||||
rs_indirect_info.info.pad = pad;
|
||||
rs_indirect_info.info.dispatch.pad = u32vec4(pad_ru_blocks, 1, 1, 0);
|
||||
|
||||
//
|
||||
// Size and set zero fill and dispatch
|
||||
//
|
||||
// NOTE(allanmac): We could zero the histogram passes on the host
|
||||
// since the number of passes is known ahead of time but since the
|
||||
// 256-dword partitions directly follow the 256-dword histograms, we
|
||||
// can dispatch just one FILL.
|
||||
//
|
||||
// The "internal" memory map looks like this:
|
||||
//
|
||||
// +---------------------------------+ <-- 0
|
||||
// | histograms[keyval_size] |
|
||||
// +---------------------------------+ <-- keyval_size * histo_dwords
|
||||
// | partitions[scatter_blocks_ru-1] |
|
||||
// +---------------------------------+ <-- (keyval_size + scatter_blocks_ru - 1) * histo_dwords
|
||||
// | workgroup_ids[keyval_size] |
|
||||
// +---------------------------------+ <-- (keyval_size + scatter_blocks_ru - 1) * histo_dwords + keyval_size
|
||||
//
|
||||
// NOTE(allanmac): The `.block_offset` and `.dword_offset_min`
|
||||
// parameters are zeroes because the host can offset the buffer
|
||||
// device address since the number of passes is known by the host.
|
||||
// If we ever wanted to supported an indirect number of "key" bits
|
||||
// in the sort, then this would need to change.
|
||||
//
|
||||
// NOTE(allanmac): The `.workgroup_ids[]` are only used if
|
||||
// nonsequential dispatch isn't supported by the device.
|
||||
//
|
||||
rs_indirect_info_fill zero;
|
||||
|
||||
zero.block_offset = 0;
|
||||
zero.dword_offset_min = 0;
|
||||
zero.dword_offset_max_minus_min = (push.passes + scatter_ru_blocks - 1) * RS_RADIX_SIZE;
|
||||
|
||||
if (RS_SCATTER_NONSEQUENTIAL_DISPATCH != 0)
|
||||
zero.dword_offset_max_minus_min += (RS_KEYVAL_DWORDS * 4); // one pass per byte
|
||||
|
||||
const uint32_t zero_ru_blocks =
|
||||
RS_COUNT_RU_BLOCKS(zero.dword_offset_max_minus_min, RS_FILL_BLOCK_DWORDS);
|
||||
|
||||
rs_indirect_info.info.zero = zero;
|
||||
rs_indirect_info.info.dispatch.zero = u32vec4(zero_ru_blocks, 1, 1, 0);
|
||||
}
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
@@ -0,0 +1,128 @@
|
||||
# Copyright © 2022 Konstantin Seurer
|
||||
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
radix_sort_shaders = [
|
||||
[
|
||||
'init.comp',
|
||||
'u64_init',
|
||||
['-DRS_KEYVAL_DWORDS=2'],
|
||||
],
|
||||
[
|
||||
'init.comp',
|
||||
'u96_init',
|
||||
['-DRS_KEYVAL_DWORDS=3'],
|
||||
],
|
||||
[
|
||||
'fill.comp',
|
||||
'fill',
|
||||
[],
|
||||
],
|
||||
[
|
||||
'histogram.comp',
|
||||
'u64_histogram',
|
||||
['-DRS_KEYVAL_DWORDS=2'],
|
||||
],
|
||||
[
|
||||
'histogram.comp',
|
||||
'u96_histogram',
|
||||
['-DRS_KEYVAL_DWORDS=3'],
|
||||
],
|
||||
[
|
||||
'prefix.comp',
|
||||
'u64_prefix',
|
||||
['-DRS_KEYVAL_DWORDS=2'],
|
||||
],
|
||||
[
|
||||
'prefix.comp',
|
||||
'u96_prefix',
|
||||
['-DRS_KEYVAL_DWORDS=3'],
|
||||
],
|
||||
[
|
||||
'scatter_0_even.comp',
|
||||
'u64_scatter_0_even',
|
||||
['-DRS_KEYVAL_DWORDS=2'],
|
||||
],
|
||||
[
|
||||
'scatter_0_even.comp',
|
||||
'u96_scatter_0_even',
|
||||
['-DRS_KEYVAL_DWORDS=3'],
|
||||
],
|
||||
[
|
||||
'scatter_0_odd.comp',
|
||||
'u64_scatter_0_odd',
|
||||
['-DRS_KEYVAL_DWORDS=2'],
|
||||
],
|
||||
[
|
||||
'scatter_0_odd.comp',
|
||||
'u96_scatter_0_odd',
|
||||
['-DRS_KEYVAL_DWORDS=3'],
|
||||
],
|
||||
[
|
||||
'scatter_1_even.comp',
|
||||
'u64_scatter_1_even',
|
||||
['-DRS_KEYVAL_DWORDS=2'],
|
||||
],
|
||||
[
|
||||
'scatter_1_even.comp',
|
||||
'u96_scatter_1_even',
|
||||
['-DRS_KEYVAL_DWORDS=3'],
|
||||
],
|
||||
[
|
||||
'scatter_1_odd.comp',
|
||||
'u64_scatter_1_odd',
|
||||
['-DRS_KEYVAL_DWORDS=2'],
|
||||
],
|
||||
[
|
||||
'scatter_1_odd.comp',
|
||||
'u96_scatter_1_odd',
|
||||
['-DRS_KEYVAL_DWORDS=3'],
|
||||
],
|
||||
[
|
||||
'scatter_2_even.comp',
|
||||
'u96_scatter_2_even',
|
||||
['-DRS_KEYVAL_DWORDS=3'],
|
||||
],
|
||||
[
|
||||
'scatter_2_odd.comp',
|
||||
'u96_scatter_2_odd',
|
||||
['-DRS_KEYVAL_DWORDS=3'],
|
||||
],
|
||||
]
|
||||
|
||||
shader_include_files = files(
|
||||
'bufref.h',
|
||||
'prefix_limits.h',
|
||||
'prefix.h',
|
||||
'push.h',
|
||||
'scatter.glsl',
|
||||
'config.h',
|
||||
)
|
||||
|
||||
radix_sort_spv = []
|
||||
foreach s : radix_sort_shaders
|
||||
radix_sort_spv += custom_target(
|
||||
s[1] + '.spv.h',
|
||||
input : s[0],
|
||||
output : s[1] + '.spv.h',
|
||||
command : [
|
||||
prog_glslang, '-V', '--target-env', 'spirv1.5', '-x', '-o', '@OUTPUT@', '@INPUT@'
|
||||
] + s[2] + glslang_quiet + (with_mesa_debug ? ['-g'] : []),
|
||||
depend_files: shader_include_files)
|
||||
endforeach
|
||||
@@ -0,0 +1,146 @@
|
||||
// Copyright 2021 The Fuchsia Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
#version 460
|
||||
|
||||
//
|
||||
// Prefix sum the coarse histograms.
|
||||
//
|
||||
|
||||
// clang-format off
|
||||
#extension GL_GOOGLE_include_directive : require
|
||||
#extension GL_EXT_control_flow_attributes : require
|
||||
#extension GL_KHR_shader_subgroup_arithmetic : require
|
||||
#extension GL_KHR_shader_subgroup_ballot : require
|
||||
// clang-format on
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
#include "config.h"
|
||||
|
||||
//
|
||||
// Buffer reference macros and push constants
|
||||
//
|
||||
#include "bufref.h"
|
||||
#include "push.h"
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
RS_STRUCT_PUSH_PREFIX();
|
||||
|
||||
layout(push_constant) uniform block_push
|
||||
{
|
||||
rs_push_prefix push;
|
||||
};
|
||||
|
||||
//
|
||||
// Subgroup uniform support
|
||||
//
|
||||
#if defined(RS_HISTOGRAM_SUBGROUP_UNIFORM_DISABLE) && defined(GL_EXT_subgroupuniform_qualifier)
|
||||
#extension GL_EXT_subgroupuniform_qualifier : required
|
||||
#define RS_SUBGROUP_UNIFORM subgroupuniformEXT
|
||||
#else
|
||||
#define RS_SUBGROUP_UNIFORM
|
||||
#endif
|
||||
|
||||
//
|
||||
// Local macros
|
||||
//
|
||||
// clang-format off
|
||||
#define RS_KEYVAL_SIZE (RS_KEYVAL_DWORDS * 4)
|
||||
#define RS_WORKGROUP_SIZE (RS_PREFIX_WORKGROUP_SIZE)
|
||||
#define RS_SUBGROUP_SIZE (1 << RS_PREFIX_SUBGROUP_SIZE_LOG2)
|
||||
#define RS_WORKGROUP_SUBGROUPS (RS_WORKGROUP_SIZE / RS_SUBGROUP_SIZE)
|
||||
// clang-format on
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
layout(local_size_x_id = RS_PREFIX_WORKGROUP_SIZE_ID) in;
|
||||
|
||||
//
|
||||
// Histogram buffer reference
|
||||
//
|
||||
layout(buffer_reference, std430) buffer buffer_rs_histograms
|
||||
{
|
||||
uint32_t extent[];
|
||||
};
|
||||
|
||||
//
|
||||
// Load prefix limits before loading function
|
||||
//
|
||||
#include "prefix_limits.h"
|
||||
|
||||
//----------------------------------------
|
||||
shared uint32_t smem_sweep0[RS_SWEEP_0_SIZE];
|
||||
|
||||
#define RS_PREFIX_SWEEP0(idx_) smem_sweep0[idx_]
|
||||
//----------------------------------------
|
||||
|
||||
//----------------------------------------
|
||||
shared uint32_t smem_sweep1[RS_SWEEP_1_SIZE];
|
||||
|
||||
#define RS_PREFIX_SWEEP1(idx_) smem_sweep1[idx_]
|
||||
//----------------------------------------
|
||||
|
||||
//----------------------------------------
|
||||
shared uint32_t smem_sweep2[RS_SWEEP_2_SIZE];
|
||||
|
||||
#define RS_PREFIX_SWEEP2(idx_) smem_sweep2[idx_]
|
||||
//----------------------------------------
|
||||
|
||||
//
|
||||
// Define function arguments
|
||||
//
|
||||
#define RS_PREFIX_ARGS buffer_rs_histograms rs_histograms
|
||||
|
||||
//
|
||||
// Define load/store functions
|
||||
//
|
||||
// clang-format off
|
||||
#define RS_PREFIX_LOAD(idx_) rs_histograms.extent[idx_]
|
||||
#define RS_PREFIX_STORE(idx_) rs_histograms.extent[idx_]
|
||||
// clang-format on
|
||||
|
||||
//
|
||||
// Load prefix function
|
||||
//
|
||||
#include "prefix.h"
|
||||
|
||||
//
|
||||
// Exclusive prefix of uint32_t[256]
|
||||
//
|
||||
void
|
||||
main()
|
||||
{
|
||||
//
|
||||
// Define buffer reference to read histograms
|
||||
//
|
||||
// NOTE(allanmac): The histogram buffer reference could be adjusted
|
||||
// on the host to save a couple instructions at the cost of added
|
||||
// complexity.
|
||||
//
|
||||
const uint32_t invocation_id = RS_WORKGROUP_SUBGROUPS == 1 ? gl_SubgroupInvocationID : gl_LocalInvocationID.x;
|
||||
|
||||
RS_SUBGROUP_UNIFORM
|
||||
const uint32_t histograms_base = ((RS_KEYVAL_SIZE - 1 - gl_WorkGroupID.x) * RS_RADIX_SIZE);
|
||||
const uint32_t histograms_offset = (histograms_base + invocation_id) * 4;
|
||||
|
||||
RS_BUFREF_DEFINE_AT_OFFSET_UINT32(buffer_rs_histograms,
|
||||
rs_histograms,
|
||||
push.devaddr_histograms,
|
||||
histograms_offset);
|
||||
|
||||
|
||||
//
|
||||
// Compute exclusive prefix of uint32_t[256]
|
||||
//
|
||||
rs_prefix(rs_histograms);
|
||||
}
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
@@ -0,0 +1,356 @@
|
||||
// Copyright 2021 The Fuchsia Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
#ifndef SRC_GRAPHICS_LIB_COMPUTE_RADIX_SORT_PLATFORMS_VK_SHADERS_PREFIX_H_
|
||||
#define SRC_GRAPHICS_LIB_COMPUTE_RADIX_SORT_PLATFORMS_VK_SHADERS_PREFIX_H_
|
||||
|
||||
//
|
||||
// Requires several defines
|
||||
//
|
||||
#ifndef RS_PREFIX_LIMITS
|
||||
#error "Error: \"prefix_limits.h\" not loaded"
|
||||
#endif
|
||||
|
||||
#ifndef RS_PREFIX_ARGS
|
||||
#error "Error: RS_PREFIX_ARGS undefined"
|
||||
#endif
|
||||
|
||||
#ifndef RS_PREFIX_LOAD
|
||||
#error "Error: RS_PREFIX_LOAD undefined"
|
||||
#endif
|
||||
|
||||
#ifndef RS_PREFIX_STORE
|
||||
#error "Error: RS_PREFIX_STORE undefined"
|
||||
#endif
|
||||
|
||||
//
|
||||
// Optional switches:
|
||||
//
|
||||
// * Disable holding original inclusively scanned histogram values in registers.
|
||||
//
|
||||
// #define RS_PREFIX_DISABLE_COMPONENTS_IN_REGISTERS
|
||||
//
|
||||
|
||||
//
|
||||
// Compute exclusive prefix of uint32_t[256]
|
||||
//
|
||||
void
|
||||
rs_prefix(RS_PREFIX_ARGS)
|
||||
{
|
||||
if (RS_WORKGROUP_SUBGROUPS == 1)
|
||||
{
|
||||
//
|
||||
// Workgroup is a single subgroup so no shared memory is required.
|
||||
//
|
||||
|
||||
//
|
||||
// Exclusive scan-add the histogram
|
||||
//
|
||||
const uint32_t h0 = RS_PREFIX_LOAD(0);
|
||||
const uint32_t h0_inc = subgroupInclusiveAdd(h0);
|
||||
RS_SUBGROUP_UNIFORM uint32_t h_last = subgroupBroadcast(h0_inc, RS_SUBGROUP_SIZE - 1);
|
||||
|
||||
RS_PREFIX_STORE(0) = h0_inc - h0; // exclusive
|
||||
|
||||
//
|
||||
// Each iteration is dependent on the previous so no unrolling. The
|
||||
// compiler is free to hoist the loads upward though.
|
||||
//
|
||||
for (RS_SUBGROUP_UNIFORM uint32_t ii = RS_SUBGROUP_SIZE; //
|
||||
ii < RS_RADIX_SIZE;
|
||||
ii += RS_SUBGROUP_SIZE)
|
||||
{
|
||||
const uint32_t h = RS_PREFIX_LOAD(ii);
|
||||
const uint32_t h_inc = subgroupInclusiveAdd(h) + h_last;
|
||||
h_last = subgroupBroadcast(h_inc, RS_SUBGROUP_SIZE - 1);
|
||||
|
||||
RS_PREFIX_STORE(ii) = h_inc - h; // exclusive
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
//
|
||||
// Workgroup is multiple subgroups and uses shared memory to store
|
||||
// the scan's intermediate results.
|
||||
//
|
||||
// Assumes a power-of-two subgroup, workgroup and radix size.
|
||||
//
|
||||
// Downsweep: Repeatedly scan reductions until they fit in a single
|
||||
// subgroup.
|
||||
//
|
||||
// Upsweep: Then uniformly apply reductions to each subgroup.
|
||||
//
|
||||
//
|
||||
// Subgroup Size | 4 | 8 | 16 | 32 | 64 | 128 |
|
||||
// --------------+----+----+----+----+----+-----+
|
||||
// Sweep 0 | 64 | 32 | 16 | 8 | 4 | 2 | sweep_0[]
|
||||
// Sweep 1 | 16 | 4 | - | - | - | - | sweep_1[]
|
||||
// Sweep 2 | 4 | - | - | - | - | - | sweep_2[]
|
||||
// --------------+----+----+----+----+----+-----+
|
||||
// Total dwords | 84 | 36 | 16 | 8 | 4 | 2 |
|
||||
// --------------+----+----+----+----+----+-----+
|
||||
//
|
||||
#ifndef RS_PREFIX_DISABLE_COMPONENTS_IN_REGISTERS
|
||||
uint32_t h_exc[RS_H_COMPONENTS];
|
||||
#endif
|
||||
|
||||
//
|
||||
// Downsweep 0
|
||||
//
|
||||
for (uint32_t ii = 0; ii < RS_H_COMPONENTS; ii++)
|
||||
{
|
||||
const uint32_t h = RS_PREFIX_LOAD(ii * RS_WORKGROUP_SIZE);
|
||||
|
||||
const uint32_t h_inc = subgroupInclusiveAdd(h);
|
||||
|
||||
const uint32_t smem_idx = (ii * RS_WORKGROUP_SUBGROUPS) + gl_SubgroupID;
|
||||
|
||||
RS_PREFIX_SWEEP0(smem_idx) = subgroupBroadcast(h_inc, RS_SUBGROUP_SIZE - 1);
|
||||
|
||||
//
|
||||
#ifndef RS_PREFIX_DISABLE_COMPONENTS_IN_REGISTERS
|
||||
h_exc[ii] = h_inc - h;
|
||||
#else
|
||||
RS_PREFIX_STORE(ii * RS_WORKGROUP_SIZE) = h_inc - h;
|
||||
#endif
|
||||
}
|
||||
|
||||
barrier();
|
||||
|
||||
//
|
||||
// Skip generalizing these sweeps for all possible subgroups -- just
|
||||
// write them directly.
|
||||
//
|
||||
if (RS_SUBGROUP_SIZE == 128)
|
||||
{
|
||||
// There are only two elements in SWEEP0 per subgroup. The scan is
|
||||
// trivial so we fold it into the upsweep.
|
||||
}
|
||||
else if (RS_SUBGROUP_SIZE >= 16)
|
||||
{
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// Scan 0
|
||||
//
|
||||
if (RS_SWEEP_0_SIZE != RS_WORKGROUP_SIZE && // workgroup has inactive components
|
||||
gl_LocalInvocationID.x < RS_SWEEP_0_SIZE)
|
||||
{
|
||||
const uint32_t h0_red = RS_PREFIX_SWEEP0(gl_LocalInvocationID.x);
|
||||
const uint32_t h0_inc = subgroupInclusiveAdd(h0_red);
|
||||
|
||||
RS_PREFIX_SWEEP0(gl_LocalInvocationID.x) = h0_inc - h0_red;
|
||||
}
|
||||
|
||||
barrier();
|
||||
}
|
||||
else if (RS_SUBGROUP_SIZE == 8)
|
||||
{
|
||||
if (RS_SWEEP_0_SIZE < RS_WORKGROUP_SIZE)
|
||||
{
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// Scan 0 and Downsweep 1
|
||||
//
|
||||
if (gl_LocalInvocationID.x < RS_SWEEP_0_SIZE) // 32 invocations
|
||||
{
|
||||
const uint32_t h0_red = RS_PREFIX_SWEEP0(gl_LocalInvocationID.x);
|
||||
const uint32_t h0_inc = subgroupInclusiveAdd(h0_red);
|
||||
|
||||
RS_PREFIX_SWEEP0(gl_LocalInvocationID.x) = h0_inc - h0_red;
|
||||
RS_PREFIX_SWEEP1(gl_SubgroupID) = subgroupBroadcast(h0_inc, RS_SUBGROUP_SIZE - 1);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// Scan 0 and Downsweep 1
|
||||
//
|
||||
for (uint32_t ii = 0; ii < RS_S0_PASSES; ii++) // 32 invocations
|
||||
{
|
||||
const uint32_t idx0 = (ii * RS_WORKGROUP_SIZE) + gl_LocalInvocationID.x;
|
||||
const uint32_t idx1 = (ii * RS_WORKGROUP_SUBGROUPS) + gl_SubgroupID;
|
||||
|
||||
const uint32_t h0_red = RS_PREFIX_SWEEP0(idx0);
|
||||
const uint32_t h0_inc = subgroupInclusiveAdd(h0_red);
|
||||
|
||||
RS_PREFIX_SWEEP0(idx0) = h0_inc - h0_red;
|
||||
RS_PREFIX_SWEEP1(idx1) = subgroupBroadcast(h0_inc, RS_SUBGROUP_SIZE - 1);
|
||||
}
|
||||
}
|
||||
|
||||
barrier();
|
||||
|
||||
//
|
||||
// Scan 1
|
||||
//
|
||||
if (gl_LocalInvocationID.x < RS_SWEEP_1_SIZE) // 4 invocations
|
||||
{
|
||||
const uint32_t h1_red = RS_PREFIX_SWEEP1(gl_LocalInvocationID.x);
|
||||
const uint32_t h1_inc = subgroupInclusiveAdd(h1_red);
|
||||
|
||||
RS_PREFIX_SWEEP1(gl_LocalInvocationID.x) = h1_inc - h1_red;
|
||||
}
|
||||
|
||||
barrier();
|
||||
}
|
||||
else if (RS_SUBGROUP_SIZE == 4)
|
||||
{
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// Scan 0 and Downsweep 1
|
||||
//
|
||||
if (RS_SWEEP_0_SIZE < RS_WORKGROUP_SIZE)
|
||||
{
|
||||
if (gl_LocalInvocationID.x < RS_SWEEP_0_SIZE) // 64 invocations
|
||||
{
|
||||
const uint32_t h0_red = RS_PREFIX_SWEEP0(gl_LocalInvocationID.x);
|
||||
const uint32_t h0_inc = subgroupInclusiveAdd(h0_red);
|
||||
|
||||
RS_PREFIX_SWEEP0(gl_LocalInvocationID.x) = h0_inc - h0_red;
|
||||
RS_PREFIX_SWEEP1(gl_SubgroupID) = subgroupBroadcast(h0_inc, RS_SUBGROUP_SIZE - 1);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
for (uint32_t ii = 0; ii < RS_S0_PASSES; ii++) // 64 invocations
|
||||
{
|
||||
const uint32_t idx0 = (ii * RS_WORKGROUP_SIZE) + gl_LocalInvocationID.x;
|
||||
const uint32_t idx1 = (ii * RS_WORKGROUP_SUBGROUPS) + gl_SubgroupID;
|
||||
|
||||
const uint32_t h0_red = RS_PREFIX_SWEEP0(idx0);
|
||||
const uint32_t h0_inc = subgroupInclusiveAdd(h0_red);
|
||||
|
||||
RS_PREFIX_SWEEP0(idx0) = h0_inc - h0_red;
|
||||
RS_PREFIX_SWEEP1(idx1) = subgroupBroadcast(h0_inc, RS_SUBGROUP_SIZE - 1);
|
||||
}
|
||||
}
|
||||
|
||||
barrier();
|
||||
|
||||
//
|
||||
// Scan 1 and Downsweep 2
|
||||
//
|
||||
if (RS_SWEEP_1_SIZE < RS_WORKGROUP_SIZE)
|
||||
{
|
||||
if (gl_LocalInvocationID.x < RS_SWEEP_1_SIZE) // 16 invocations
|
||||
{
|
||||
const uint32_t h1_red = RS_PREFIX_SWEEP1(gl_LocalInvocationID.x);
|
||||
const uint32_t h1_inc = subgroupInclusiveAdd(h1_red);
|
||||
|
||||
RS_PREFIX_SWEEP1(gl_LocalInvocationID.x) = h1_inc - h1_red;
|
||||
RS_PREFIX_SWEEP2(gl_SubgroupID) = subgroupBroadcast(h1_inc, RS_SUBGROUP_SIZE - 1);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
for (uint32_t ii = 0; ii < RS_S1_PASSES; ii++) // 16 invocations
|
||||
{
|
||||
const uint32_t idx1 = (ii * RS_WORKGROUP_SIZE) + gl_LocalInvocationID.x;
|
||||
const uint32_t idx2 = (ii * RS_WORKGROUP_SUBGROUPS) + gl_SubgroupID;
|
||||
|
||||
const uint32_t h1_red = RS_PREFIX_SWEEP1(idx1);
|
||||
const uint32_t h1_inc = subgroupInclusiveAdd(h1_red);
|
||||
|
||||
RS_PREFIX_SWEEP1(idx1) = h1_inc - h1_red;
|
||||
RS_PREFIX_SWEEP2(idx2) = subgroupBroadcast(h1_inc, RS_SUBGROUP_SIZE - 1);
|
||||
}
|
||||
}
|
||||
|
||||
barrier();
|
||||
|
||||
//
|
||||
// Scan 2
|
||||
//
|
||||
// 4 invocations
|
||||
//
|
||||
if (gl_LocalInvocationID.x < RS_SWEEP_2_SIZE)
|
||||
{
|
||||
const uint32_t h2_red = RS_PREFIX_SWEEP2(gl_LocalInvocationID.x);
|
||||
const uint32_t h2_inc = subgroupInclusiveAdd(h2_red);
|
||||
|
||||
RS_PREFIX_SWEEP2(gl_LocalInvocationID.x) = h2_inc - h2_red;
|
||||
}
|
||||
|
||||
barrier();
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// Final upsweep 0
|
||||
//
|
||||
if (RS_SUBGROUP_SIZE == 128)
|
||||
{
|
||||
// There must be more than one subgroup per workgroup, but the maximum
|
||||
// workgroup size is 256 so there must be exactly two subgroups per
|
||||
// workgroup and RS_H_COMPONENTS must be 1.
|
||||
#ifndef RS_PREFIX_DISABLE_COMPONENTS_IN_REGISTERS
|
||||
RS_PREFIX_STORE(0) = h_exc[0] + (gl_SubgroupID > 0 ? RS_PREFIX_SWEEP0(0) : 0);
|
||||
#else
|
||||
const uint32_t h_exc = RS_PREFIX_LOAD(0);
|
||||
|
||||
RS_PREFIX_STORE(0) = h_exc + (gl_SubgroupID > 0 ? RS_PREFIX_SWEEP0(0) : 0);
|
||||
#endif
|
||||
}
|
||||
else if (RS_SUBGROUP_SIZE >= 16)
|
||||
{
|
||||
for (uint32_t ii = 0; ii < RS_H_COMPONENTS; ii++)
|
||||
{
|
||||
const uint32_t idx0 = (ii * RS_WORKGROUP_SUBGROUPS) + gl_SubgroupID;
|
||||
|
||||
// clang format issue
|
||||
#ifndef RS_PREFIX_DISABLE_COMPONENTS_IN_REGISTERS
|
||||
RS_PREFIX_STORE(ii * RS_WORKGROUP_SIZE) = h_exc[ii] + RS_PREFIX_SWEEP0(idx0);
|
||||
#else
|
||||
const uint32_t h_exc = RS_PREFIX_LOAD(ii * RS_WORKGROUP_SIZE);
|
||||
|
||||
RS_PREFIX_STORE(ii * RS_WORKGROUP_SIZE) = h_exc + RS_PREFIX_SWEEP0(idx0);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
else if (RS_SUBGROUP_SIZE == 8)
|
||||
{
|
||||
for (uint32_t ii = 0; ii < RS_H_COMPONENTS; ii++)
|
||||
{
|
||||
const uint32_t idx0 = (ii * RS_WORKGROUP_SUBGROUPS) + gl_SubgroupID;
|
||||
const uint32_t idx1 = idx0 / RS_SUBGROUP_SIZE;
|
||||
|
||||
#ifndef RS_PREFIX_DISABLE_COMPONENTS_IN_REGISTERS
|
||||
RS_PREFIX_STORE(ii * RS_WORKGROUP_SIZE) =
|
||||
h_exc[ii] + RS_PREFIX_SWEEP0(idx0) + RS_PREFIX_SWEEP1(idx1);
|
||||
#else
|
||||
const uint32_t h_exc = RS_PREFIX_LOAD(ii * RS_WORKGROUP_SIZE);
|
||||
|
||||
RS_PREFIX_STORE(ii * RS_WORKGROUP_SIZE) =
|
||||
h_exc + RS_PREFIX_SWEEP0(idx0) + RS_PREFIX_SWEEP1(idx1);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
else if (RS_SUBGROUP_SIZE == 4)
|
||||
{
|
||||
for (uint32_t ii = 0; ii < RS_H_COMPONENTS; ii++)
|
||||
{
|
||||
const uint32_t idx0 = (ii * RS_WORKGROUP_SUBGROUPS) + gl_SubgroupID;
|
||||
const uint32_t idx1 = idx0 / RS_SUBGROUP_SIZE;
|
||||
const uint32_t idx2 = idx1 / RS_SUBGROUP_SIZE;
|
||||
|
||||
#ifndef RS_PREFIX_DISABLE_COMPONENTS_IN_REGISTERS
|
||||
RS_PREFIX_STORE(ii * RS_WORKGROUP_SIZE) =
|
||||
h_exc[ii] + (RS_PREFIX_SWEEP0(idx0) + RS_PREFIX_SWEEP1(idx1) + RS_PREFIX_SWEEP2(idx2));
|
||||
#else
|
||||
const uint32_t h_exc = RS_PREFIX_LOAD(ii * RS_WORKGROUP_SIZE);
|
||||
|
||||
RS_PREFIX_STORE(ii * RS_WORKGROUP_SIZE) =
|
||||
h_exc + (RS_PREFIX_SWEEP0(idx0) + RS_PREFIX_SWEEP1(idx1) + RS_PREFIX_SWEEP2(idx2));
|
||||
#endif
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#endif // SRC_GRAPHICS_LIB_COMPUTE_RADIX_SORT_PLATFORMS_VK_SHADERS_PREFIX_H_
|
||||
@@ -0,0 +1,34 @@
|
||||
// Copyright 2021 The Fuchsia Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
#ifndef SRC_GRAPHICS_LIB_COMPUTE_RADIX_SORT_PLATFORMS_VK_SHADERS_PREFIX_LIMITS_H_
|
||||
#define SRC_GRAPHICS_LIB_COMPUTE_RADIX_SORT_PLATFORMS_VK_SHADERS_PREFIX_LIMITS_H_
|
||||
|
||||
//
|
||||
// Define various prefix limits
|
||||
//
|
||||
#define RS_PREFIX_LIMITS
|
||||
|
||||
// clang-format off
|
||||
#define RS_H_COMPONENTS (RS_RADIX_SIZE / RS_WORKGROUP_SIZE)
|
||||
|
||||
#define RS_SWEEP_0_SIZE (RS_WORKGROUP_SUBGROUPS == 1 ? 0 : (RS_RADIX_SIZE / RS_SUBGROUP_SIZE))
|
||||
#define RS_SWEEP_1_SIZE (RS_WORKGROUP_SUBGROUPS == 1 ? 0 : (RS_SWEEP_0_SIZE / RS_SUBGROUP_SIZE))
|
||||
#define RS_SWEEP_2_SIZE (RS_WORKGROUP_SUBGROUPS == 1 ? 0 : (RS_SWEEP_1_SIZE / RS_SUBGROUP_SIZE))
|
||||
|
||||
#define RS_SWEEP_SIZE (RS_SWEEP_0_SIZE + RS_SWEEP_1_SIZE + RS_SWEEP_2_SIZE)
|
||||
|
||||
#define RS_S0_PASSES (RS_SWEEP_0_SIZE / RS_WORKGROUP_SIZE)
|
||||
#define RS_S1_PASSES (RS_SWEEP_1_SIZE / RS_WORKGROUP_SIZE)
|
||||
|
||||
#define RS_SWEEP_0_OFFSET 0
|
||||
#define RS_SWEEP_1_OFFSET (RS_SWEEP_0_OFFSET + RS_SWEEP_0_SIZE)
|
||||
#define RS_SWEEP_2_OFFSET (RS_SWEEP_1_OFFSET + RS_SWEEP_1_SIZE)
|
||||
// clang-format on
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#endif // SRC_GRAPHICS_LIB_COMPUTE_RADIX_SORT_PLATFORMS_VK_SHADERS_PREFIX_LIMITS_H_
|
||||
@@ -0,0 +1,263 @@
|
||||
// Copyright 2021 The Fuchsia Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
#ifndef SRC_GRAPHICS_LIB_COMPUTE_RADIX_SORT_PLATFORMS_VK_SHADERS_PUSH_H_
|
||||
#define SRC_GRAPHICS_LIB_COMPUTE_RADIX_SORT_PLATFORMS_VK_SHADERS_PUSH_H_
|
||||
|
||||
//
|
||||
// There is a limit to the maximum number of keyvals that can be sorted because
|
||||
// the top 2 bits in the atomic lookback counters are used as tag bits.
|
||||
//
|
||||
#define RS_MAX_KEYVALS ((1 << 30) - 1)
|
||||
|
||||
//
|
||||
// Right now, the entire implementation is very much dependent on an 8-bit radix
|
||||
// size. Most of the shaders attempt to honor this defined size but there are
|
||||
// still a number of places where 256 is assumed.
|
||||
//
|
||||
#define RS_RADIX_LOG2 8
|
||||
#define RS_RADIX_SIZE (1 << RS_RADIX_LOG2)
|
||||
|
||||
//
|
||||
// LOOKBACK STATUS FLAGS
|
||||
//
|
||||
// The decoupled lookback status flags are stored in the two
|
||||
// high bits of the count:
|
||||
//
|
||||
// 0 31
|
||||
// | REDUCTION OR PREFIX COUNT | STATUS |
|
||||
// +---------------------------+--------+
|
||||
// | 30 | 2 |
|
||||
//
|
||||
// This limits the keyval extent size to (2^30-1).
|
||||
//
|
||||
// Valid status flags are:
|
||||
//
|
||||
// EVEN PASS ODD PASS
|
||||
// ----------------------- -----------------------
|
||||
// 0 : invalid 0 : prefix available
|
||||
// 1 : reduction available 1 : ---
|
||||
// 2 : prefix available 2 : invalid
|
||||
// 3 : --- 3 : reduction available
|
||||
//
|
||||
// Atomically adding +1 to a "reduction available" status results in a "prefix
|
||||
// available" status.
|
||||
//
|
||||
// clang-format off
|
||||
#define RS_PARTITION_STATUS_EVEN_INVALID (0u)
|
||||
#define RS_PARTITION_STATUS_EVEN_REDUCTION (1u)
|
||||
#define RS_PARTITION_STATUS_EVEN_PREFIX (2u)
|
||||
|
||||
#define RS_PARTITION_STATUS_ODD_INVALID (2u)
|
||||
#define RS_PARTITION_STATUS_ODD_REDUCTION (3u)
|
||||
#define RS_PARTITION_STATUS_ODD_PREFIX (0u)
|
||||
// clang-format on
|
||||
|
||||
//
|
||||
// Arguments to indirectly launched shaders.
|
||||
//
|
||||
// struct rs_indirect_info_dispatch
|
||||
// {
|
||||
// u32vec4 pad;
|
||||
// u32vec4 zero;
|
||||
// u32vec4 histogram;
|
||||
// u32vec4 scatter;
|
||||
// };
|
||||
//
|
||||
// struct rs_indirect_info_fill
|
||||
// {
|
||||
// uint32_t block_offset;
|
||||
// uint32_t dword_offset_min;
|
||||
// uint32_t dword_offset_max_minus_min;
|
||||
// uint32_t reserved; // padding for 16 bytes
|
||||
// };
|
||||
//
|
||||
// struct rs_indirect_info
|
||||
// {
|
||||
// rs_indirect_info_fill pad;
|
||||
// rs_indirect_info_fill zero;
|
||||
// rs_indirect_info_dispatch dispatch;
|
||||
// };
|
||||
//
|
||||
#define RS_STRUCT_INDIRECT_INFO_DISPATCH() \
|
||||
struct rs_indirect_info_dispatch \
|
||||
{ \
|
||||
RS_STRUCT_MEMBER_STRUCT(u32vec4, pad) \
|
||||
RS_STRUCT_MEMBER_STRUCT(u32vec4, zero) \
|
||||
RS_STRUCT_MEMBER_STRUCT(u32vec4, histogram) \
|
||||
RS_STRUCT_MEMBER_STRUCT(u32vec4, scatter) \
|
||||
}
|
||||
|
||||
#define RS_STRUCT_INDIRECT_INFO_FILL() \
|
||||
struct rs_indirect_info_fill \
|
||||
{ \
|
||||
RS_STRUCT_MEMBER(uint32_t, block_offset) \
|
||||
RS_STRUCT_MEMBER(uint32_t, dword_offset_min) \
|
||||
RS_STRUCT_MEMBER(uint32_t, dword_offset_max_minus_min) \
|
||||
RS_STRUCT_MEMBER(uint32_t, reserved) \
|
||||
}
|
||||
|
||||
#define RS_STRUCT_INDIRECT_INFO() \
|
||||
RS_STRUCT_INDIRECT_INFO_DISPATCH(); \
|
||||
RS_STRUCT_INDIRECT_INFO_FILL(); \
|
||||
struct rs_indirect_info \
|
||||
{ \
|
||||
RS_STRUCT_MEMBER_STRUCT(rs_indirect_info_fill, pad) \
|
||||
RS_STRUCT_MEMBER_STRUCT(rs_indirect_info_fill, zero) \
|
||||
RS_STRUCT_MEMBER_STRUCT(rs_indirect_info_dispatch, dispatch) \
|
||||
}
|
||||
|
||||
//
|
||||
// Define the push constant structures shared by the host and device.
|
||||
//
|
||||
// INIT
|
||||
// ----
|
||||
// struct rs_push_init
|
||||
// {
|
||||
// uint64_t devaddr_count; // address of count buffer
|
||||
// uint64_t devaddr_indirect; // address of indirect info buffer
|
||||
// };
|
||||
//
|
||||
// FILL
|
||||
// ----
|
||||
// struct rs_push_fill
|
||||
// {
|
||||
// uint64_t devaddr_info; // address of indirect info for fill shader
|
||||
// uint64_t devaddr_dwords; // address of dwords extent
|
||||
// uint32_t dword; // dword value used to fill the dwords extent
|
||||
// };
|
||||
//
|
||||
// HISTOGRAM
|
||||
// ---------
|
||||
// struct rs_push_histogram
|
||||
// {
|
||||
// uint64_t devaddr_histograms; // address of histograms extent
|
||||
// uint64_t devaddr_keyvals; // address of keyvals extent
|
||||
// uint32_t passes; // number of passes
|
||||
// };
|
||||
//
|
||||
// PREFIX
|
||||
// ------
|
||||
// struct rs_push_prefix
|
||||
// {
|
||||
// uint64_t devaddr_histograms; // address of histograms extent
|
||||
// };
|
||||
//
|
||||
// SCATTER
|
||||
// -------
|
||||
// struct rs_push_scatter
|
||||
// {
|
||||
// uint64_t devaddr_keyvals_in; // address of input keyvals
|
||||
// uint64_t devaddr_keyvals_out; // address of output keyvals
|
||||
// uint64_t devaddr_partitions // address of partitions
|
||||
// uint64_t devaddr_histogram; // address of pass histogram
|
||||
// uint32_t pass_offset; // keyval pass offset
|
||||
// };
|
||||
//
|
||||
#define RS_STRUCT_PUSH_INIT() \
|
||||
struct rs_push_init \
|
||||
{ \
|
||||
RS_STRUCT_MEMBER(RS_DEVADDR, devaddr_info) \
|
||||
RS_STRUCT_MEMBER(RS_DEVADDR, devaddr_count) \
|
||||
RS_STRUCT_MEMBER(uint32_t, passes) \
|
||||
}
|
||||
|
||||
#define RS_STRUCT_PUSH_FILL() \
|
||||
struct rs_push_fill \
|
||||
{ \
|
||||
RS_STRUCT_MEMBER(RS_DEVADDR, devaddr_info) \
|
||||
RS_STRUCT_MEMBER(RS_DEVADDR, devaddr_dwords) \
|
||||
RS_STRUCT_MEMBER(uint32_t, dword) \
|
||||
}
|
||||
|
||||
#define RS_STRUCT_PUSH_HISTOGRAM() \
|
||||
struct rs_push_histogram \
|
||||
{ \
|
||||
RS_STRUCT_MEMBER(RS_DEVADDR, devaddr_histograms) \
|
||||
RS_STRUCT_MEMBER(RS_DEVADDR, devaddr_keyvals) \
|
||||
RS_STRUCT_MEMBER(uint32_t, passes) \
|
||||
}
|
||||
|
||||
#define RS_STRUCT_PUSH_PREFIX() \
|
||||
struct rs_push_prefix \
|
||||
{ \
|
||||
RS_STRUCT_MEMBER(RS_DEVADDR, devaddr_histograms) \
|
||||
}
|
||||
|
||||
#define RS_STRUCT_PUSH_SCATTER() \
|
||||
struct rs_push_scatter \
|
||||
{ \
|
||||
RS_STRUCT_MEMBER(RS_DEVADDR, devaddr_keyvals_even) \
|
||||
RS_STRUCT_MEMBER(RS_DEVADDR, devaddr_keyvals_odd) \
|
||||
RS_STRUCT_MEMBER(RS_DEVADDR, devaddr_partitions) \
|
||||
RS_STRUCT_MEMBER(RS_DEVADDR, devaddr_histograms) \
|
||||
RS_STRUCT_MEMBER(uint32_t, pass_offset) \
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// GLSL
|
||||
//
|
||||
#ifdef VULKAN // defined by GLSL/VK compiler
|
||||
|
||||
// clang-format off
|
||||
#define RS_STRUCT_MEMBER(type_, name_) type_ name_;
|
||||
#define RS_STRUCT_MEMBER_FARRAY(type_, len_, name_) type_ name_[len_];
|
||||
#define RS_STRUCT_MEMBER_STRUCT(type_, name_) type_ name_;
|
||||
// clang-format on
|
||||
|
||||
////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// C/C++
|
||||
//
|
||||
#else
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
struct u32vec4
|
||||
{
|
||||
uint32_t x;
|
||||
uint32_t y;
|
||||
uint32_t z;
|
||||
uint32_t w;
|
||||
};
|
||||
|
||||
// clang-format off
|
||||
#define RS_DEVADDR uint64_t
|
||||
#define RS_STRUCT_MEMBER(type_, name_) type_ name_;
|
||||
#define RS_STRUCT_MEMBER_FARRAY(type_, len_, name_) type_ name_[len_];
|
||||
#define RS_STRUCT_MEMBER_STRUCT(type_, name_) struct type_ name_;
|
||||
// clang-format on
|
||||
|
||||
RS_STRUCT_PUSH_INIT();
|
||||
RS_STRUCT_PUSH_FILL();
|
||||
RS_STRUCT_PUSH_HISTOGRAM();
|
||||
RS_STRUCT_PUSH_PREFIX();
|
||||
RS_STRUCT_PUSH_SCATTER();
|
||||
|
||||
RS_STRUCT_INDIRECT_INFO();
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#endif // SRC_GRAPHICS_LIB_COMPUTE_RADIX_SORT_PLATFORMS_VK_SHADERS_PUSH_H_
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,36 @@
|
||||
// Copyright 2021 The Fuchsia Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
#version 460
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
// clang-format off
|
||||
#define RS_SCATTER_KEYVAL_DWORD_BASE 0
|
||||
|
||||
#define RS_PARTITION_STATUS_INVALID RS_PARTITION_STATUS_EVEN_INVALID
|
||||
#define RS_PARTITION_STATUS_REDUCTION RS_PARTITION_STATUS_EVEN_REDUCTION
|
||||
#define RS_PARTITION_STATUS_PREFIX RS_PARTITION_STATUS_EVEN_PREFIX
|
||||
|
||||
#define RS_DEVADDR_KEYVALS_IN(push_) push_.devaddr_keyvals_even
|
||||
#define RS_DEVADDR_KEYVALS_OUT(push_) push_.devaddr_keyvals_odd
|
||||
// clang-format on
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#extension GL_GOOGLE_include_directive : require
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#include "scatter.glsl"
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
@@ -0,0 +1,36 @@
|
||||
// Copyright 2021 The Fuchsia Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
#version 460
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
// clang-format off
|
||||
#define RS_SCATTER_KEYVAL_DWORD_BASE 0
|
||||
|
||||
#define RS_PARTITION_STATUS_INVALID RS_PARTITION_STATUS_ODD_INVALID
|
||||
#define RS_PARTITION_STATUS_REDUCTION RS_PARTITION_STATUS_ODD_REDUCTION
|
||||
#define RS_PARTITION_STATUS_PREFIX RS_PARTITION_STATUS_ODD_PREFIX
|
||||
|
||||
#define RS_DEVADDR_KEYVALS_IN(push_) push_.devaddr_keyvals_odd
|
||||
#define RS_DEVADDR_KEYVALS_OUT(push_) push_.devaddr_keyvals_even
|
||||
// clang-format on
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#extension GL_GOOGLE_include_directive : require
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#include "scatter.glsl"
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
@@ -0,0 +1,36 @@
|
||||
// Copyright 2021 The Fuchsia Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
#version 460
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
// clang-format off
|
||||
#define RS_SCATTER_KEYVAL_DWORD_BASE 1
|
||||
|
||||
#define RS_PARTITION_STATUS_INVALID RS_PARTITION_STATUS_EVEN_INVALID
|
||||
#define RS_PARTITION_STATUS_REDUCTION RS_PARTITION_STATUS_EVEN_REDUCTION
|
||||
#define RS_PARTITION_STATUS_PREFIX RS_PARTITION_STATUS_EVEN_PREFIX
|
||||
|
||||
#define RS_DEVADDR_KEYVALS_IN(push_) push_.devaddr_keyvals_even
|
||||
#define RS_DEVADDR_KEYVALS_OUT(push_) push_.devaddr_keyvals_odd
|
||||
// clang-format on
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#extension GL_GOOGLE_include_directive : require
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#include "scatter.glsl"
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
@@ -0,0 +1,36 @@
|
||||
// Copyright 2021 The Fuchsia Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
#version 460
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
// clang-format off
|
||||
#define RS_SCATTER_KEYVAL_DWORD_BASE 1
|
||||
|
||||
#define RS_PARTITION_STATUS_INVALID RS_PARTITION_STATUS_ODD_INVALID
|
||||
#define RS_PARTITION_STATUS_REDUCTION RS_PARTITION_STATUS_ODD_REDUCTION
|
||||
#define RS_PARTITION_STATUS_PREFIX RS_PARTITION_STATUS_ODD_PREFIX
|
||||
|
||||
#define RS_DEVADDR_KEYVALS_IN(push_) push_.devaddr_keyvals_odd
|
||||
#define RS_DEVADDR_KEYVALS_OUT(push_) push_.devaddr_keyvals_even
|
||||
// clang-format on
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#extension GL_GOOGLE_include_directive : require
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#include "scatter.glsl"
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
@@ -0,0 +1,36 @@
|
||||
// Copyright 2021 The Fuchsia Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
#version 460
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
// clang-format off
|
||||
#define RS_SCATTER_KEYVAL_DWORD_BASE 2
|
||||
|
||||
#define RS_PARTITION_STATUS_INVALID RS_PARTITION_STATUS_EVEN_INVALID
|
||||
#define RS_PARTITION_STATUS_REDUCTION RS_PARTITION_STATUS_EVEN_REDUCTION
|
||||
#define RS_PARTITION_STATUS_PREFIX RS_PARTITION_STATUS_EVEN_PREFIX
|
||||
|
||||
#define RS_DEVADDR_KEYVALS_IN(push_) push_.devaddr_keyvals_even
|
||||
#define RS_DEVADDR_KEYVALS_OUT(push_) push_.devaddr_keyvals_odd
|
||||
// clang-format on
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#extension GL_GOOGLE_include_directive : require
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#include "scatter.glsl"
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
@@ -0,0 +1,36 @@
|
||||
// Copyright 2021 The Fuchsia Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
#version 460
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
// clang-format off
|
||||
#define RS_SCATTER_KEYVAL_DWORD_BASE 2
|
||||
|
||||
#define RS_PARTITION_STATUS_INVALID RS_PARTITION_STATUS_ODD_INVALID
|
||||
#define RS_PARTITION_STATUS_REDUCTION RS_PARTITION_STATUS_ODD_REDUCTION
|
||||
#define RS_PARTITION_STATUS_PREFIX RS_PARTITION_STATUS_ODD_PREFIX
|
||||
|
||||
#define RS_DEVADDR_KEYVALS_IN(push_) push_.devaddr_keyvals_odd
|
||||
#define RS_DEVADDR_KEYVALS_OUT(push_) push_.devaddr_keyvals_even
|
||||
// clang-format on
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#extension GL_GOOGLE_include_directive : require
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#include "scatter.glsl"
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
@@ -0,0 +1,60 @@
|
||||
// Copyright 2021 The Fuchsia Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
#ifndef SRC_GRAPHICS_LIB_COMPUTE_RADIX_SORT_PLATFORMS_VK_TARGET_H_
|
||||
#define SRC_GRAPHICS_LIB_COMPUTE_RADIX_SORT_PLATFORMS_VK_TARGET_H_
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
//
|
||||
// This structure packages target-specific configuration parameters.
|
||||
//
|
||||
|
||||
struct radix_sort_vk_target_config
|
||||
{
|
||||
uint32_t keyval_dwords;
|
||||
|
||||
struct
|
||||
{
|
||||
uint32_t workgroup_size_log2;
|
||||
} init;
|
||||
|
||||
struct
|
||||
{
|
||||
uint32_t workgroup_size_log2;
|
||||
uint32_t block_rows;
|
||||
} fill;
|
||||
|
||||
struct
|
||||
{
|
||||
uint32_t workgroup_size_log2;
|
||||
uint32_t subgroup_size_log2;
|
||||
uint32_t block_rows;
|
||||
} histogram;
|
||||
|
||||
struct
|
||||
{
|
||||
uint32_t workgroup_size_log2;
|
||||
uint32_t subgroup_size_log2;
|
||||
} prefix;
|
||||
|
||||
struct
|
||||
{
|
||||
uint32_t workgroup_size_log2;
|
||||
uint32_t subgroup_size_log2;
|
||||
uint32_t block_rows;
|
||||
} scatter;
|
||||
|
||||
bool nonsequential_dispatch;
|
||||
};
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
#endif // SRC_GRAPHICS_LIB_COMPUTE_RADIX_SORT_PLATFORMS_VK_TARGET_H_
|
||||
@@ -0,0 +1,144 @@
|
||||
/*
|
||||
* Copyright © 2022 Friedrich Vock
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "vk_rmv_common.h"
|
||||
#include "vk_buffer.h"
|
||||
#include "vk_device.h"
|
||||
|
||||
void
|
||||
vk_memory_trace_init(struct vk_device *device, const struct vk_rmv_device_info *device_info)
|
||||
{
|
||||
device->memory_trace_data.device_info = *device_info;
|
||||
device->memory_trace_data.is_enabled = true;
|
||||
device->memory_trace_data.tokens = UTIL_DYNARRAY_INIT;
|
||||
simple_mtx_init(&device->memory_trace_data.token_mtx, mtx_plain);
|
||||
|
||||
device->memory_trace_data.next_resource_id = 1;
|
||||
device->memory_trace_data.handle_table = _mesa_hash_table_u64_create(NULL);
|
||||
}
|
||||
|
||||
void
|
||||
vk_memory_trace_finish(struct vk_device *device)
|
||||
{
|
||||
if (!device->memory_trace_data.is_enabled)
|
||||
return;
|
||||
util_dynarray_foreach (&device->memory_trace_data.tokens, struct vk_rmv_token, token) {
|
||||
switch (token->type) {
|
||||
case VK_RMV_TOKEN_TYPE_RESOURCE_CREATE: {
|
||||
struct vk_rmv_resource_create_token *create_token = &token->data.resource_create;
|
||||
if (create_token->type == VK_RMV_RESOURCE_TYPE_DESCRIPTOR_POOL) {
|
||||
free(create_token->descriptor_pool.pool_sizes);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case VK_RMV_TOKEN_TYPE_USERDATA:
|
||||
free(token->data.userdata.name);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
util_dynarray_fini(&device->memory_trace_data.tokens);
|
||||
if (_mesa_hash_table_num_entries(&device->memory_trace_data.handle_table->table))
|
||||
fprintf(stderr,
|
||||
"mesa: Unfreed resources detected at device destroy, there may be memory leaks!\n");
|
||||
_mesa_hash_table_u64_destroy(device->memory_trace_data.handle_table);
|
||||
device->memory_trace_data.is_enabled = false;
|
||||
}
|
||||
|
||||
void
|
||||
vk_rmv_emit_token(struct vk_memory_trace_data *data, enum vk_rmv_token_type type, void *token_data)
|
||||
{
|
||||
struct vk_rmv_token token;
|
||||
token.type = type;
|
||||
token.timestamp = (uint64_t)os_time_get_nano();
|
||||
memcpy(&token.data, token_data, vk_rmv_token_size_from_type(type));
|
||||
util_dynarray_append(&data->tokens, token);
|
||||
}
|
||||
|
||||
uint32_t
|
||||
vk_rmv_get_resource_id_locked(struct vk_device *device, uint64_t handle)
|
||||
{
|
||||
void *entry = _mesa_hash_table_u64_search(device->memory_trace_data.handle_table, handle);
|
||||
if (!entry) {
|
||||
uint32_t id = device->memory_trace_data.next_resource_id++;
|
||||
_mesa_hash_table_u64_insert(device->memory_trace_data.handle_table, handle,
|
||||
(void *)(uintptr_t)id);
|
||||
return id;
|
||||
}
|
||||
return (uint32_t)(uintptr_t)entry;
|
||||
}
|
||||
|
||||
void
|
||||
vk_rmv_destroy_resource_id_locked(struct vk_device *device, uint64_t handle)
|
||||
{
|
||||
_mesa_hash_table_u64_remove(device->memory_trace_data.handle_table, handle);
|
||||
}
|
||||
|
||||
void
|
||||
vk_rmv_log_buffer_create(struct vk_device *device, bool is_internal, VkBuffer _buffer)
|
||||
{
|
||||
if (!device->memory_trace_data.is_enabled)
|
||||
return;
|
||||
|
||||
VK_FROM_HANDLE(vk_buffer, buffer, _buffer);
|
||||
simple_mtx_lock(&device->memory_trace_data.token_mtx);
|
||||
struct vk_rmv_resource_create_token token = {0};
|
||||
token.is_driver_internal = is_internal;
|
||||
token.resource_id = vk_rmv_get_resource_id_locked(device, (uint64_t)_buffer);
|
||||
token.type = VK_RMV_RESOURCE_TYPE_BUFFER;
|
||||
token.buffer.create_flags = buffer->create_flags;
|
||||
token.buffer.size = buffer->size;
|
||||
token.buffer.usage_flags = buffer->usage;
|
||||
|
||||
vk_rmv_emit_token(&device->memory_trace_data, VK_RMV_TOKEN_TYPE_RESOURCE_CREATE, &token);
|
||||
simple_mtx_unlock(&device->memory_trace_data.token_mtx);
|
||||
}
|
||||
|
||||
void
|
||||
vk_rmv_log_cpu_map(struct vk_device *device, uint64_t va, bool is_unmap)
|
||||
{
|
||||
if (!device->memory_trace_data.is_enabled)
|
||||
return;
|
||||
|
||||
struct vk_rmv_cpu_map_token map_token;
|
||||
map_token.address = va;
|
||||
map_token.unmapped = is_unmap;
|
||||
|
||||
simple_mtx_lock(&device->memory_trace_data.token_mtx);
|
||||
vk_rmv_emit_token(&device->memory_trace_data, VK_RMV_TOKEN_TYPE_CPU_MAP, &map_token);
|
||||
simple_mtx_unlock(&device->memory_trace_data.token_mtx);
|
||||
}
|
||||
|
||||
void
|
||||
vk_rmv_log_misc_token(struct vk_device *device, enum vk_rmv_misc_event_type type)
|
||||
{
|
||||
if (!device->memory_trace_data.is_enabled)
|
||||
return;
|
||||
|
||||
simple_mtx_lock(&device->memory_trace_data.token_mtx);
|
||||
struct vk_rmv_misc_token token;
|
||||
token.type = type;
|
||||
vk_rmv_emit_token(&device->memory_trace_data, VK_RMV_TOKEN_TYPE_MISC, &token);
|
||||
simple_mtx_unlock(&device->memory_trace_data.token_mtx);
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
/*
|
||||
* Copyright © 2022 Friedrich Vock
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#ifndef VK_RMV_COMMON_H
|
||||
#define VK_RMV_COMMON_H
|
||||
|
||||
#include <stdbool.h>
|
||||
#include "util/hash_table.h"
|
||||
#include "util/simple_mtx.h"
|
||||
#include "util/u_debug.h"
|
||||
#include "util/u_dynarray.h"
|
||||
#include <vulkan/vulkan_core.h>
|
||||
#include "vk_rmv_tokens.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
struct vk_memory_trace_data;
|
||||
|
||||
/*
|
||||
* The different memory domains RMV supports.
|
||||
*/
|
||||
enum vk_rmv_memory_location {
|
||||
/* DEVICE_LOCAL | HOST_VISIBLE */
|
||||
VK_RMV_MEMORY_LOCATION_DEVICE,
|
||||
/* DEVICE_LOCAL */
|
||||
VK_RMV_MEMORY_LOCATION_DEVICE_INVISIBLE,
|
||||
/* HOST_VISIBLE | HOST_COHERENT */
|
||||
VK_RMV_MEMORY_LOCATION_HOST,
|
||||
|
||||
/* add above here */
|
||||
VK_RMV_MEMORY_LOCATION_COUNT
|
||||
};
|
||||
|
||||
/*
|
||||
* Information about a memory domain.
|
||||
*/
|
||||
struct vk_rmv_memory_info {
|
||||
uint64_t size;
|
||||
uint64_t physical_base_address;
|
||||
};
|
||||
|
||||
enum vk_rmv_memory_type {
|
||||
VK_RMV_MEMORY_TYPE_UNKNOWN,
|
||||
VK_RMV_MEMORY_TYPE_DDR2,
|
||||
VK_RMV_MEMORY_TYPE_DDR3,
|
||||
VK_RMV_MEMORY_TYPE_DDR4,
|
||||
VK_RMV_MEMORY_TYPE_GDDR5,
|
||||
VK_RMV_MEMORY_TYPE_GDDR6,
|
||||
VK_RMV_MEMORY_TYPE_HBM,
|
||||
VK_RMV_MEMORY_TYPE_HBM2,
|
||||
VK_RMV_MEMORY_TYPE_HBM3,
|
||||
VK_RMV_MEMORY_TYPE_LPDDR4,
|
||||
VK_RMV_MEMORY_TYPE_LPDDR5,
|
||||
VK_RMV_MEMORY_TYPE_DDR5
|
||||
};
|
||||
|
||||
/*
|
||||
* Device information for RMV traces.
|
||||
*/
|
||||
struct vk_rmv_device_info {
|
||||
struct vk_rmv_memory_info memory_infos[VK_RMV_MEMORY_LOCATION_COUNT];
|
||||
|
||||
/* The memory type of dedicated VRAM. */
|
||||
enum vk_rmv_memory_type vram_type;
|
||||
|
||||
char device_name[128];
|
||||
|
||||
uint32_t pcie_family_id;
|
||||
uint32_t pcie_revision_id;
|
||||
uint32_t pcie_device_id;
|
||||
/* The minimum shader clock, in MHz. */
|
||||
uint32_t minimum_shader_clock;
|
||||
/* The maximum shader clock, in MHz. */
|
||||
uint32_t maximum_shader_clock;
|
||||
uint32_t vram_operations_per_clock;
|
||||
uint32_t vram_bus_width;
|
||||
/* The VRAM bandwidth, in GB/s (1 GB/s = 1000 MB/s). */
|
||||
uint32_t vram_bandwidth;
|
||||
/* The minimum memory clock, in MHz. */
|
||||
uint32_t minimum_memory_clock;
|
||||
/* The maximum memory clock, in MHz. */
|
||||
uint32_t maximum_memory_clock;
|
||||
};
|
||||
|
||||
struct vk_device;
|
||||
|
||||
struct vk_memory_trace_data {
|
||||
struct util_dynarray tokens;
|
||||
simple_mtx_t token_mtx;
|
||||
|
||||
bool is_enabled;
|
||||
|
||||
struct vk_rmv_device_info device_info;
|
||||
|
||||
struct hash_table_u64 *handle_table;
|
||||
uint32_t next_resource_id;
|
||||
};
|
||||
|
||||
struct vk_device;
|
||||
|
||||
void vk_memory_trace_init(struct vk_device *device, const struct vk_rmv_device_info *device_info);
|
||||
|
||||
void vk_memory_trace_finish(struct vk_device *device);
|
||||
|
||||
int vk_dump_rmv_capture(struct vk_memory_trace_data *data);
|
||||
|
||||
void vk_rmv_emit_token(struct vk_memory_trace_data *data, enum vk_rmv_token_type type,
|
||||
void *token_data);
|
||||
void vk_rmv_log_buffer_create(struct vk_device *device, bool is_internal, VkBuffer _buffer);
|
||||
void vk_rmv_log_cpu_map(struct vk_device *device, uint64_t va, bool is_unmap);
|
||||
void vk_rmv_log_misc_token(struct vk_device *device, enum vk_rmv_misc_event_type type);
|
||||
|
||||
/* Retrieves the unique resource id for the resource specified by handle.
|
||||
* Allocates a new id if none exists already.
|
||||
* The memory trace mutex should be locked when entering this function. */
|
||||
uint32_t vk_rmv_get_resource_id_locked(struct vk_device *device, uint64_t handle);
|
||||
/* Destroys a resource id. If the same handle is allocated again, a new resource
|
||||
* id is given to it.
|
||||
* The memory trace mutex should be locked when entering this function. */
|
||||
void vk_rmv_destroy_resource_id_locked(struct vk_device *device, uint64_t handle);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,304 @@
|
||||
/*
|
||||
* Copyright © 2022 Friedrich Vock
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#ifndef VK_RMV_TOKENS_H
|
||||
#define VK_RMV_TOKENS_H
|
||||
|
||||
#include <stdint.h>
|
||||
#include <string.h>
|
||||
#include "util/os_time.h"
|
||||
#include <vulkan/vulkan_core.h>
|
||||
|
||||
/*
|
||||
* Implemented types of tokens.
|
||||
*/
|
||||
enum vk_rmv_token_type {
|
||||
VK_RMV_TOKEN_TYPE_USERDATA,
|
||||
VK_RMV_TOKEN_TYPE_MISC,
|
||||
VK_RMV_TOKEN_TYPE_RESOURCE_BIND,
|
||||
VK_RMV_TOKEN_TYPE_RESOURCE_REFERENCE,
|
||||
VK_RMV_TOKEN_TYPE_PAGE_TABLE_UPDATE,
|
||||
VK_RMV_TOKEN_TYPE_CPU_MAP,
|
||||
VK_RMV_TOKEN_TYPE_VIRTUAL_FREE,
|
||||
VK_RMV_TOKEN_TYPE_VIRTUAL_ALLOCATE,
|
||||
VK_RMV_TOKEN_TYPE_RESOURCE_CREATE,
|
||||
VK_RMV_TOKEN_TYPE_RESOURCE_DESTROY
|
||||
};
|
||||
|
||||
/*
|
||||
* The type of miscellaneous event reported through a MISC token.
|
||||
*/
|
||||
enum vk_rmv_misc_event_type {
|
||||
VK_RMV_MISC_EVENT_TYPE_SUBMIT_GRAPHICS,
|
||||
VK_RMV_MISC_EVENT_TYPE_SUBMIT_COMPUTE,
|
||||
VK_RMV_MISC_EVENT_TYPE_SUBMIT_COPY,
|
||||
VK_RMV_MISC_EVENT_TYPE_PRESENT,
|
||||
VK_RMV_MISC_EVENT_TYPE_INVALIDATE_RANGES,
|
||||
VK_RMV_MISC_EVENT_TYPE_FLUSH_MAPPED_RANGE,
|
||||
VK_RMV_MISC_EVENT_TYPE_TRIM_MEMORY
|
||||
};
|
||||
|
||||
enum vk_rmv_resource_type {
|
||||
VK_RMV_RESOURCE_TYPE_IMAGE,
|
||||
VK_RMV_RESOURCE_TYPE_BUFFER,
|
||||
VK_RMV_RESOURCE_TYPE_GPU_EVENT,
|
||||
VK_RMV_RESOURCE_TYPE_BORDER_COLOR_PALETTE,
|
||||
VK_RMV_RESOURCE_TYPE_INDIRECT_CMD_GENERATOR,
|
||||
VK_RMV_RESOURCE_TYPE_MOTION_ESTIMATOR,
|
||||
VK_RMV_RESOURCE_TYPE_PERF_EXPERIMENT,
|
||||
VK_RMV_RESOURCE_TYPE_QUERY_HEAP,
|
||||
VK_RMV_RESOURCE_TYPE_VIDEO_DECODER,
|
||||
VK_RMV_RESOURCE_TYPE_VIDEO_ENCODER,
|
||||
VK_RMV_RESOURCE_TYPE_TIMESTAMP,
|
||||
VK_RMV_RESOURCE_TYPE_HEAP,
|
||||
VK_RMV_RESOURCE_TYPE_PIPELINE,
|
||||
VK_RMV_RESOURCE_TYPE_DESCRIPTOR_HEAP,
|
||||
VK_RMV_RESOURCE_TYPE_DESCRIPTOR_POOL,
|
||||
VK_RMV_RESOURCE_TYPE_COMMAND_ALLOCATOR,
|
||||
VK_RMV_RESOURCE_TYPE_MISC_INTERNAL
|
||||
};
|
||||
|
||||
/*
|
||||
* Token data for all types of tokens.
|
||||
*/
|
||||
|
||||
struct vk_rmv_timestamp_token {
|
||||
uint64_t value;
|
||||
};
|
||||
|
||||
struct vk_rmv_userdata_token {
|
||||
char *name;
|
||||
uint32_t resource_id;
|
||||
};
|
||||
|
||||
struct vk_rmv_misc_token {
|
||||
enum vk_rmv_misc_event_type type;
|
||||
};
|
||||
|
||||
struct vk_rmv_resource_bind_token {
|
||||
uint64_t address;
|
||||
uint64_t size;
|
||||
bool is_system_memory;
|
||||
uint32_t resource_id;
|
||||
};
|
||||
|
||||
struct vk_rmv_resource_reference_token {
|
||||
uint64_t virtual_address;
|
||||
bool residency_removed;
|
||||
};
|
||||
|
||||
enum vk_rmv_page_table_update_type {
|
||||
VK_RMV_PAGE_TABLE_UPDATE_TYPE_DISCARD,
|
||||
VK_RMV_PAGE_TABLE_UPDATE_TYPE_UPDATE,
|
||||
VK_RMV_PAGE_TABLE_UPDATE_TYPE_TRANSFER
|
||||
};
|
||||
|
||||
struct vk_rmv_page_table_update_token {
|
||||
uint64_t virtual_address;
|
||||
uint64_t physical_address;
|
||||
uint64_t page_count;
|
||||
uint32_t page_size;
|
||||
int pid;
|
||||
bool is_unmap;
|
||||
enum vk_rmv_page_table_update_type type;
|
||||
};
|
||||
|
||||
struct vk_rmv_cpu_map_token {
|
||||
uint64_t address;
|
||||
bool unmapped;
|
||||
};
|
||||
|
||||
struct vk_rmv_virtual_free_token {
|
||||
uint64_t address;
|
||||
};
|
||||
|
||||
enum vk_rmv_kernel_memory_domain {
|
||||
VK_RMV_KERNEL_MEMORY_DOMAIN_CPU = 0x1,
|
||||
VK_RMV_KERNEL_MEMORY_DOMAIN_GTT = 0x2,
|
||||
VK_RMV_KERNEL_MEMORY_DOMAIN_VRAM = 0x4
|
||||
};
|
||||
|
||||
struct vk_rmv_virtual_allocate_token {
|
||||
uint32_t page_count;
|
||||
bool is_driver_internal;
|
||||
bool is_in_invisible_vram;
|
||||
uint64_t address;
|
||||
enum vk_rmv_kernel_memory_domain preferred_domains;
|
||||
};
|
||||
|
||||
struct vk_rmv_image_description {
|
||||
VkImageCreateFlags create_flags;
|
||||
VkImageUsageFlags usage_flags;
|
||||
VkImageType type;
|
||||
VkExtent3D extent;
|
||||
VkFormat format;
|
||||
uint32_t num_mips;
|
||||
uint32_t num_slices;
|
||||
VkImageTiling tiling;
|
||||
|
||||
uint32_t log2_samples;
|
||||
uint32_t log2_storage_samples;
|
||||
|
||||
uint32_t alignment_log2;
|
||||
uint32_t metadata_alignment_log2;
|
||||
uint32_t image_alignment_log2;
|
||||
|
||||
uint64_t size;
|
||||
uint64_t metadata_size;
|
||||
uint64_t metadata_header_size;
|
||||
|
||||
uint64_t metadata_offset;
|
||||
uint64_t metadata_header_offset;
|
||||
|
||||
bool presentable;
|
||||
};
|
||||
|
||||
struct vk_rmv_event_description {
|
||||
VkEventCreateFlags flags;
|
||||
};
|
||||
|
||||
struct vk_rmv_border_color_palette_description {
|
||||
uint8_t num_entries;
|
||||
};
|
||||
|
||||
struct vk_rmv_buffer_description {
|
||||
VkBufferCreateFlags create_flags;
|
||||
VkBufferUsageFlags usage_flags;
|
||||
uint64_t size;
|
||||
};
|
||||
|
||||
struct vk_rmv_query_pool_description {
|
||||
VkQueryType type;
|
||||
bool has_cpu_access;
|
||||
};
|
||||
|
||||
/* The heap description refers to a VkDeviceMemory resource. */
|
||||
struct vk_rmv_heap_description {
|
||||
VkMemoryAllocateFlags alloc_flags;
|
||||
uint64_t size;
|
||||
uint32_t alignment;
|
||||
uint32_t heap_index;
|
||||
};
|
||||
|
||||
struct vk_rmv_pipeline_description {
|
||||
bool is_internal;
|
||||
uint64_t hash_lo;
|
||||
uint64_t hash_hi;
|
||||
VkShaderStageFlags shader_stages;
|
||||
bool is_ngg;
|
||||
};
|
||||
|
||||
struct vk_rmv_descriptor_pool_description {
|
||||
uint32_t max_sets;
|
||||
uint32_t pool_size_count;
|
||||
VkDescriptorPoolSize *pool_sizes;
|
||||
};
|
||||
|
||||
struct vk_rmv_command_buffer_description {
|
||||
enum vk_rmv_kernel_memory_domain preferred_domain;
|
||||
uint64_t executable_size;
|
||||
uint64_t app_available_executable_size;
|
||||
uint64_t embedded_data_size;
|
||||
uint64_t app_available_embedded_data_size;
|
||||
uint64_t scratch_size;
|
||||
uint64_t app_available_scratch_size;
|
||||
};
|
||||
|
||||
enum vk_rmv_misc_internal_type {
|
||||
VK_RMV_MISC_INTERNAL_TYPE_PADDING,
|
||||
};
|
||||
|
||||
struct vk_rmv_misc_internal_description {
|
||||
enum vk_rmv_misc_internal_type type;
|
||||
};
|
||||
|
||||
struct vk_rmv_resource_create_token {
|
||||
uint32_t resource_id;
|
||||
bool is_driver_internal;
|
||||
enum vk_rmv_resource_type type;
|
||||
union {
|
||||
struct vk_rmv_event_description event;
|
||||
struct vk_rmv_border_color_palette_description border_color_palette;
|
||||
struct vk_rmv_image_description image;
|
||||
struct vk_rmv_buffer_description buffer;
|
||||
struct vk_rmv_query_pool_description query_pool;
|
||||
struct vk_rmv_heap_description heap;
|
||||
struct vk_rmv_pipeline_description pipeline;
|
||||
struct vk_rmv_descriptor_pool_description descriptor_pool;
|
||||
struct vk_rmv_command_buffer_description command_buffer;
|
||||
struct vk_rmv_misc_internal_description misc_internal;
|
||||
};
|
||||
};
|
||||
|
||||
struct vk_rmv_resource_destroy_token {
|
||||
uint32_t resource_id;
|
||||
};
|
||||
|
||||
struct vk_rmv_token {
|
||||
enum vk_rmv_token_type type;
|
||||
uint64_t timestamp;
|
||||
union {
|
||||
struct vk_rmv_timestamp_token timestamp;
|
||||
struct vk_rmv_userdata_token userdata;
|
||||
struct vk_rmv_misc_token misc;
|
||||
struct vk_rmv_resource_bind_token resource_bind;
|
||||
struct vk_rmv_resource_reference_token resource_reference;
|
||||
struct vk_rmv_page_table_update_token page_table_update;
|
||||
struct vk_rmv_cpu_map_token cpu_map;
|
||||
struct vk_rmv_virtual_free_token virtual_free;
|
||||
struct vk_rmv_virtual_allocate_token virtual_allocate;
|
||||
struct vk_rmv_resource_create_token resource_create;
|
||||
struct vk_rmv_resource_destroy_token resource_destroy;
|
||||
} data;
|
||||
};
|
||||
|
||||
static inline size_t
|
||||
vk_rmv_token_size_from_type(enum vk_rmv_token_type type)
|
||||
{
|
||||
switch (type) {
|
||||
case VK_RMV_TOKEN_TYPE_USERDATA:
|
||||
return sizeof(struct vk_rmv_userdata_token);
|
||||
case VK_RMV_TOKEN_TYPE_MISC:
|
||||
return sizeof(struct vk_rmv_misc_token);
|
||||
case VK_RMV_TOKEN_TYPE_RESOURCE_BIND:
|
||||
return sizeof(struct vk_rmv_resource_bind_token);
|
||||
case VK_RMV_TOKEN_TYPE_RESOURCE_REFERENCE:
|
||||
return sizeof(struct vk_rmv_resource_reference_token);
|
||||
case VK_RMV_TOKEN_TYPE_PAGE_TABLE_UPDATE:
|
||||
return sizeof(struct vk_rmv_page_table_update_token);
|
||||
case VK_RMV_TOKEN_TYPE_CPU_MAP:
|
||||
return sizeof(struct vk_rmv_cpu_map_token);
|
||||
case VK_RMV_TOKEN_TYPE_VIRTUAL_FREE:
|
||||
return sizeof(struct vk_rmv_virtual_free_token);
|
||||
case VK_RMV_TOKEN_TYPE_VIRTUAL_ALLOCATE:
|
||||
return sizeof(struct vk_rmv_virtual_allocate_token);
|
||||
case VK_RMV_TOKEN_TYPE_RESOURCE_CREATE:
|
||||
return sizeof(struct vk_rmv_resource_create_token);
|
||||
case VK_RMV_TOKEN_TYPE_RESOURCE_DESTROY:
|
||||
return sizeof(struct vk_rmv_resource_destroy_token);
|
||||
default:
|
||||
UNREACHABLE("invalid token type");
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,244 @@
|
||||
/*
|
||||
* Copyright © 2021 Bas Nieuwenhuizen
|
||||
* Copyright © 2023 Valve Corporation
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#ifndef VK_ACCELERATION_STRUCTURE_H
|
||||
#define VK_ACCELERATION_STRUCTURE_H
|
||||
|
||||
#include "vk_buffer.h"
|
||||
#include "vk_meta.h"
|
||||
#include "vk_object.h"
|
||||
#include "radix_sort/radix_sort_vk.h"
|
||||
#include "radix_sort/common/vk/barrier.h"
|
||||
#include "radix_sort/shaders/push.h"
|
||||
|
||||
#include "bvh/vk_bvh.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
enum vk_acceleration_structure_build_step {
|
||||
VK_ACCELERATION_STRUCTURE_BUILD_STEP_TOP,
|
||||
VK_ACCELERATION_STRUCTURE_BUILD_STEP_BUILD_LEAVES,
|
||||
VK_ACCELERATION_STRUCTURE_BUILD_STEP_MORTON_GENERATE,
|
||||
VK_ACCELERATION_STRUCTURE_BUILD_STEP_MORTON_SORT,
|
||||
VK_ACCELERATION_STRUCTURE_BUILD_STEP_LBVH_BUILD_INTERNAL,
|
||||
VK_ACCELERATION_STRUCTURE_BUILD_STEP_PLOC_BUILD_INTERNAL,
|
||||
VK_ACCELERATION_STRUCTURE_BUILD_STEP_HPLOC_BUILD_INTERNAL,
|
||||
VK_ACCELERATION_STRUCTURE_BUILD_STEP_ENCODE,
|
||||
VK_ACCELERATION_STRUCTURE_BUILD_STEP_UPDATE,
|
||||
};
|
||||
|
||||
struct vk_acceleration_structure_build_marker {
|
||||
enum vk_acceleration_structure_build_step step;
|
||||
union {
|
||||
struct {
|
||||
uint32_t blas_count;
|
||||
uint32_t tlas_count;
|
||||
} top; /* Used for VK_ACCELERATION_STRUCTURE_BUILD_STEP_TOP */
|
||||
struct {
|
||||
uint32_t pass;
|
||||
uint32_t key;
|
||||
uint32_t leaf_node_count;
|
||||
uint32_t internal_node_count;
|
||||
} encode; /* Used for VK_ACCELERATION_STRUCTURE_BUILD_STEP_ENCODE, VK_ACCELERATION_STRUCTURE_BUILD_STEP_UPDATE */
|
||||
};
|
||||
};
|
||||
|
||||
struct vk_acceleration_structure {
|
||||
struct vk_object_base base;
|
||||
|
||||
struct vk_buffer *buffer;
|
||||
|
||||
uint64_t addr;
|
||||
uint64_t offset;
|
||||
uint64_t size;
|
||||
};
|
||||
|
||||
static inline VkDeviceAddress
|
||||
vk_acceleration_structure_get_va(const struct vk_acceleration_structure *accel_struct)
|
||||
{
|
||||
if (accel_struct->buffer) {
|
||||
return vk_buffer_address(accel_struct->buffer, accel_struct->offset);
|
||||
} else {
|
||||
assert(accel_struct->addr != 0);
|
||||
return accel_struct->addr;
|
||||
}
|
||||
}
|
||||
|
||||
VK_DEFINE_NONDISP_HANDLE_CASTS(vk_acceleration_structure, base, VkAccelerationStructureKHR,
|
||||
VK_OBJECT_TYPE_ACCELERATION_STRUCTURE_KHR)
|
||||
|
||||
#define MAX_ENCODE_PASSES 5
|
||||
#define MAX_UPDATE_PASSES 2
|
||||
|
||||
enum vk_internal_build_type {
|
||||
VK_INTERNAL_BUILD_TYPE_LBVH,
|
||||
VK_INTERNAL_BUILD_TYPE_PLOC,
|
||||
VK_INTERNAL_BUILD_TYPE_HPLOC,
|
||||
VK_INTERNAL_BUILD_TYPE_UPDATE,
|
||||
};
|
||||
|
||||
struct vk_build_config {
|
||||
enum vk_internal_build_type internal_type;
|
||||
bool updateable;
|
||||
bool u64_keys;
|
||||
uint32_t encode_key[MAX_ENCODE_PASSES];
|
||||
uint32_t update_key[MAX_ENCODE_PASSES];
|
||||
};
|
||||
|
||||
struct vk_scratch_layout {
|
||||
uint32_t size;
|
||||
uint32_t update_size;
|
||||
|
||||
uint32_t header_offset;
|
||||
|
||||
/* Used for BUILD only. */
|
||||
|
||||
uint32_t sort_buffer_offset[2];
|
||||
uint32_t sort_internal_offset;
|
||||
|
||||
uint32_t ploc_prefix_sum_partition_offset;
|
||||
uint32_t lbvh_node_offset;
|
||||
uint32_t hploc_ranges_offset;
|
||||
|
||||
uint32_t ir_offset;
|
||||
uint32_t internal_node_offset;
|
||||
};
|
||||
|
||||
struct vk_acceleration_structure_build_state {
|
||||
const VkAccelerationStructureBuildGeometryInfoKHR *build_info;
|
||||
const VkAccelerationStructureBuildRangeInfoKHR *build_range_infos;
|
||||
uint32_t leaf_node_count;
|
||||
uint32_t internal_node_count;
|
||||
struct vk_scratch_layout scratch;
|
||||
struct vk_build_config config;
|
||||
|
||||
/* Internal state of vk_acceleration_structure.c */
|
||||
uint32_t build_flags;
|
||||
|
||||
uint32_t scratch_offset;
|
||||
|
||||
/* Radix sort state */
|
||||
uint32_t scatter_blocks;
|
||||
uint32_t count_ru_scatter;
|
||||
uint32_t histo_blocks;
|
||||
uint32_t count_ru_histo;
|
||||
struct rs_push_scatter push_scatter;
|
||||
|
||||
uint32_t last_encode_pass;
|
||||
};
|
||||
|
||||
struct vk_acceleration_structure_build_ops {
|
||||
void (*begin_debug_marker)(VkCommandBuffer commandBuffer,
|
||||
struct vk_acceleration_structure_build_marker *marker);
|
||||
void (*end_debug_marker)(VkCommandBuffer commandBuffer,
|
||||
struct vk_acceleration_structure_build_marker *marker);
|
||||
|
||||
void (*get_build_config)(VkDevice device, struct vk_acceleration_structure_build_state *state);
|
||||
|
||||
VkDeviceSize (*get_as_size)(VkDevice device, const struct vk_acceleration_structure_build_state *state);
|
||||
VkDeviceSize (*get_encode_scratch_size)(VkDevice device, const struct vk_acceleration_structure_build_state *state);
|
||||
VkDeviceSize (*get_update_scratch_size)(VkDevice device, const struct vk_acceleration_structure_build_state *state);
|
||||
|
||||
VkResult (*encode_prepare[MAX_ENCODE_PASSES])(VkCommandBuffer cmd_buffer, const struct vk_acceleration_structure_build_state *state);
|
||||
void (*encode_as[MAX_ENCODE_PASSES])(VkCommandBuffer cmd_buffer, const struct vk_acceleration_structure_build_state *state);
|
||||
|
||||
void (*init_update_scratch)(VkCommandBuffer cmd_buffer, const struct vk_acceleration_structure_build_state *states, uint32_t build_count);
|
||||
void (*update_prepare[MAX_ENCODE_PASSES])(VkCommandBuffer cmd_buffer, const struct vk_acceleration_structure_build_state *state,
|
||||
bool flushed_cp_after_init_update_scratch, bool flushed_compute_after_init_update_scratch);
|
||||
void (*update_as[MAX_ENCODE_PASSES])(VkCommandBuffer cmd_buffer, const struct vk_acceleration_structure_build_state *state);
|
||||
|
||||
const uint32_t *leaf_spirv_override;
|
||||
size_t leaf_spirv_override_size;
|
||||
};
|
||||
|
||||
struct vk_acceleration_structure_build_args {
|
||||
uint32_t subgroup_size;
|
||||
uint32_t bvh_bounds_offset;
|
||||
uint32_t root_flags_offset;
|
||||
bool propagate_cull_flags;
|
||||
bool emit_markers;
|
||||
const radix_sort_vk_t *radix_sort_64;
|
||||
const radix_sort_vk_t *radix_sort_96;
|
||||
};
|
||||
|
||||
VkResult vk_get_bvh_build_pipeline_layout(struct vk_device *device, struct vk_meta_device *meta,
|
||||
unsigned push_constant_size, VkPipelineLayout *layout);
|
||||
|
||||
VkResult vk_get_bvh_build_pipeline_spv(struct vk_device *device, struct vk_meta_device *meta,
|
||||
enum vk_meta_object_key_type type, const uint32_t *spv,
|
||||
uint32_t spv_size, unsigned push_constant_size,
|
||||
const struct vk_acceleration_structure_build_args *args,
|
||||
uint32_t flags, VkPipeline *pipeline,
|
||||
bool unaligned_dispatch);
|
||||
|
||||
void vk_cmd_build_acceleration_structures(VkCommandBuffer cmdbuf,
|
||||
struct vk_device *device,
|
||||
struct vk_meta_device *meta,
|
||||
uint32_t info_count,
|
||||
const VkAccelerationStructureBuildGeometryInfoKHR *pInfos,
|
||||
const VkAccelerationStructureBuildRangeInfoKHR *const *ppBuildRangeInfos,
|
||||
const struct vk_acceleration_structure_build_args *args);
|
||||
|
||||
void vk_get_as_build_sizes(VkDevice _device, VkAccelerationStructureBuildTypeKHR buildType,
|
||||
const VkAccelerationStructureBuildGeometryInfoKHR *pBuildInfo,
|
||||
const uint32_t *pMaxPrimitiveCounts,
|
||||
VkAccelerationStructureBuildSizesInfoKHR *pSizeInfo,
|
||||
const struct vk_acceleration_structure_build_args *args);
|
||||
|
||||
bool vk_acceleration_struct_vtx_format_supported(VkFormat format);
|
||||
|
||||
static inline VkGeometryTypeKHR
|
||||
vk_get_as_geometry_type(const VkAccelerationStructureBuildGeometryInfoKHR *build_info)
|
||||
{
|
||||
if (build_info->geometryCount) {
|
||||
if (build_info->pGeometries)
|
||||
return build_info->pGeometries[0].geometryType;
|
||||
else
|
||||
return build_info->ppGeometries[0]->geometryType;
|
||||
}
|
||||
|
||||
/* If there are no geometries, the geometry type shouldn't matter, but
|
||||
* return something.
|
||||
*/
|
||||
return VK_GEOMETRY_TYPE_TRIANGLES_KHR;
|
||||
}
|
||||
|
||||
struct vk_bvh_geometry_data
|
||||
vk_fill_geometry_data(VkAccelerationStructureTypeKHR type, uint32_t first_id, uint32_t geom_index,
|
||||
const VkAccelerationStructureGeometryKHR *geometry,
|
||||
const VkAccelerationStructureBuildRangeInfoKHR *build_range_info);
|
||||
|
||||
void vk_accel_struct_cmd_begin_debug_marker(VkCommandBuffer commandBuffer,
|
||||
struct vk_acceleration_structure_build_marker *marker);
|
||||
|
||||
void vk_accel_struct_cmd_end_debug_marker(VkCommandBuffer commandBuffer,
|
||||
struct vk_acceleration_structure_build_marker *marker);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,208 @@
|
||||
/*
|
||||
* Copyright © 2023 Collabora, Ltd.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
#ifndef VK_ANDROID_H
|
||||
#define VK_ANDROID_H
|
||||
|
||||
#include <stdbool.h>
|
||||
|
||||
#include "vulkan/vulkan_core.h"
|
||||
#include "vulkan/vk_android_native_buffer.h"
|
||||
|
||||
#include "util/detect_os.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
struct u_gralloc;
|
||||
struct vk_device;
|
||||
struct vk_image;
|
||||
|
||||
#ifdef VK_USE_PLATFORM_ANDROID_KHR
|
||||
|
||||
struct u_gralloc *vk_android_get_ugralloc(void);
|
||||
|
||||
VkResult vk_android_import_anb(struct vk_device *device,
|
||||
const VkImageCreateInfo *pCreateInfo,
|
||||
const VkAllocationCallbacks *alloc,
|
||||
struct vk_image *image);
|
||||
|
||||
VkResult vk_android_import_anb_memory(struct vk_device *device,
|
||||
struct vk_image *image,
|
||||
const VkNativeBufferANDROID *anb,
|
||||
const VkAllocationCallbacks *alloc);
|
||||
|
||||
VkResult vk_android_get_anb_layout(
|
||||
const VkImageCreateInfo *pCreateInfo,
|
||||
VkImageDrmFormatModifierExplicitCreateInfoEXT *out,
|
||||
VkSubresourceLayout *out_layouts, int max_planes);
|
||||
|
||||
VkResult vk_android_init_deferred_image(struct vk_device *device,
|
||||
struct vk_image *image,
|
||||
const VkImageCreateInfo *pCreateInfo,
|
||||
const VkAllocationCallbacks *pAllocator);
|
||||
|
||||
#else
|
||||
|
||||
static inline struct u_gralloc *
|
||||
vk_android_get_ugralloc(void)
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static inline VkResult
|
||||
vk_android_import_anb(struct vk_device *device,
|
||||
const VkImageCreateInfo *pCreateInfo,
|
||||
const VkAllocationCallbacks *alloc,
|
||||
struct vk_image *image)
|
||||
{
|
||||
return VK_ERROR_FEATURE_NOT_PRESENT;
|
||||
}
|
||||
|
||||
static inline VkResult
|
||||
vk_android_get_anb_layout(
|
||||
const VkImageCreateInfo *pCreateInfo,
|
||||
VkImageDrmFormatModifierExplicitCreateInfoEXT *out,
|
||||
VkSubresourceLayout *out_layouts, int max_planes)
|
||||
{
|
||||
return VK_ERROR_FEATURE_NOT_PRESENT;
|
||||
}
|
||||
|
||||
static inline VkResult
|
||||
vk_android_init_deferred_image(struct vk_device *device,
|
||||
struct vk_image *image,
|
||||
const VkImageCreateInfo *pCreateInfo,
|
||||
const VkAllocationCallbacks *pAllocator)
|
||||
{
|
||||
return VK_ERROR_FEATURE_NOT_PRESENT;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
#if defined(VK_USE_PLATFORM_ANDROID_KHR) && ANDROID_API_LEVEL >= 26
|
||||
|
||||
struct AHardwareBuffer;
|
||||
|
||||
uint64_t vk_android_get_front_buffer_usage(void);
|
||||
|
||||
VkFormat vk_ahb_format_to_image_format(uint32_t ahb_format);
|
||||
|
||||
uint32_t vk_image_format_to_ahb_format(VkFormat vk_format);
|
||||
|
||||
uint64_t vk_image_usage_to_ahb_usage(const VkImageCreateFlags vk_create,
|
||||
const VkImageUsageFlags vk_usage);
|
||||
|
||||
struct AHardwareBuffer *vk_alloc_ahardware_buffer(
|
||||
const VkMemoryAllocateInfo *pAllocateInfo);
|
||||
|
||||
VkResult vk_android_get_ahb_layout(
|
||||
struct AHardwareBuffer *ahardware_buffer,
|
||||
VkImageDrmFormatModifierExplicitCreateInfoEXT *out,
|
||||
VkSubresourceLayout *out_layouts, int max_planes);
|
||||
|
||||
VkResult vk_android_get_ahb_image_properties(
|
||||
VkPhysicalDevice pdev_handle,
|
||||
const VkPhysicalDeviceImageFormatInfo2 *info,
|
||||
VkImageFormatProperties2 *props);
|
||||
|
||||
void vk_android_get_ahb_buffer_properties(
|
||||
VkPhysicalDevice pdev_handle,
|
||||
const VkPhysicalDeviceExternalBufferInfo *info,
|
||||
VkExternalBufferProperties *props);
|
||||
|
||||
bool vk_android_rp_attachment_has_external_format(
|
||||
const VkAttachmentDescription2 *desc);
|
||||
|
||||
#else /* defined(VK_USE_PLATFORM_ANDROID_KHR) && ANDROID_API_LEVEL >= 26 */
|
||||
|
||||
static inline uint64_t
|
||||
vk_android_get_front_buffer_usage(void)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline VkFormat
|
||||
vk_ahb_format_to_image_format(uint32_t ahb_format)
|
||||
{
|
||||
return VK_FORMAT_UNDEFINED;
|
||||
}
|
||||
|
||||
static inline uint32_t
|
||||
vk_image_format_to_ahb_format(VkFormat vk_format)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline uint64_t
|
||||
vk_image_usage_to_ahb_usage(const VkImageCreateFlags vk_create,
|
||||
const VkImageUsageFlags vk_usage)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline struct AHardwareBuffer *
|
||||
vk_alloc_ahardware_buffer(const VkMemoryAllocateInfo *pAllocateInfo)
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static inline VkResult
|
||||
vk_android_get_ahb_layout(
|
||||
struct AHardwareBuffer *ahardware_buffer,
|
||||
VkImageDrmFormatModifierExplicitCreateInfoEXT *out,
|
||||
VkSubresourceLayout *out_layouts, int max_planes)
|
||||
{
|
||||
return VK_ERROR_FEATURE_NOT_PRESENT;
|
||||
}
|
||||
|
||||
static inline VkResult
|
||||
vk_android_get_ahb_image_properties(
|
||||
VkPhysicalDevice pdev_handle,
|
||||
const VkPhysicalDeviceImageFormatInfo2 *info,
|
||||
VkImageFormatProperties2 *props)
|
||||
{
|
||||
return VK_ERROR_FORMAT_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
static inline void
|
||||
vk_android_get_ahb_buffer_properties(
|
||||
VkPhysicalDevice pdev_handle,
|
||||
const VkPhysicalDeviceExternalBufferInfo *info,
|
||||
VkExternalBufferProperties *props)
|
||||
{
|
||||
}
|
||||
|
||||
static bool
|
||||
vk_android_rp_attachment_has_external_format(
|
||||
const VkAttachmentDescription2 *desc)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
#endif /* ANDROID_API_LEVEL >= 26 */
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* VK_ANDROID_H */
|
||||
@@ -0,0 +1,232 @@
|
||||
/*
|
||||
* Copyright 2023 Valve Corporation
|
||||
* Copyright 2021 Collabora Ltd.
|
||||
* SPDX-License-Identifier: MIT
|
||||
*/
|
||||
|
||||
#include "vk_blend.h"
|
||||
#include "util/macros.h"
|
||||
#include "vulkan/vulkan_core.h"
|
||||
|
||||
enum pipe_logicop
|
||||
vk_logic_op_to_pipe(VkLogicOp in)
|
||||
{
|
||||
switch (in) {
|
||||
case VK_LOGIC_OP_CLEAR:
|
||||
return PIPE_LOGICOP_CLEAR;
|
||||
case VK_LOGIC_OP_AND:
|
||||
return PIPE_LOGICOP_AND;
|
||||
case VK_LOGIC_OP_AND_REVERSE:
|
||||
return PIPE_LOGICOP_AND_REVERSE;
|
||||
case VK_LOGIC_OP_COPY:
|
||||
return PIPE_LOGICOP_COPY;
|
||||
case VK_LOGIC_OP_AND_INVERTED:
|
||||
return PIPE_LOGICOP_AND_INVERTED;
|
||||
case VK_LOGIC_OP_NO_OP:
|
||||
return PIPE_LOGICOP_NOOP;
|
||||
case VK_LOGIC_OP_XOR:
|
||||
return PIPE_LOGICOP_XOR;
|
||||
case VK_LOGIC_OP_OR:
|
||||
return PIPE_LOGICOP_OR;
|
||||
case VK_LOGIC_OP_NOR:
|
||||
return PIPE_LOGICOP_NOR;
|
||||
case VK_LOGIC_OP_EQUIVALENT:
|
||||
return PIPE_LOGICOP_EQUIV;
|
||||
case VK_LOGIC_OP_INVERT:
|
||||
return PIPE_LOGICOP_INVERT;
|
||||
case VK_LOGIC_OP_OR_REVERSE:
|
||||
return PIPE_LOGICOP_OR_REVERSE;
|
||||
case VK_LOGIC_OP_COPY_INVERTED:
|
||||
return PIPE_LOGICOP_COPY_INVERTED;
|
||||
case VK_LOGIC_OP_OR_INVERTED:
|
||||
return PIPE_LOGICOP_OR_INVERTED;
|
||||
case VK_LOGIC_OP_NAND:
|
||||
return PIPE_LOGICOP_NAND;
|
||||
case VK_LOGIC_OP_SET:
|
||||
return PIPE_LOGICOP_SET;
|
||||
default:
|
||||
UNREACHABLE("Invalid logicop");
|
||||
}
|
||||
}
|
||||
|
||||
enum pipe_blend_func
|
||||
vk_blend_op_to_pipe(VkBlendOp in)
|
||||
{
|
||||
switch (in) {
|
||||
case VK_BLEND_OP_ADD:
|
||||
return PIPE_BLEND_ADD;
|
||||
case VK_BLEND_OP_SUBTRACT:
|
||||
return PIPE_BLEND_SUBTRACT;
|
||||
case VK_BLEND_OP_REVERSE_SUBTRACT:
|
||||
return PIPE_BLEND_REVERSE_SUBTRACT;
|
||||
case VK_BLEND_OP_MIN:
|
||||
return PIPE_BLEND_MIN;
|
||||
case VK_BLEND_OP_MAX:
|
||||
return PIPE_BLEND_MAX;
|
||||
default:
|
||||
UNREACHABLE("Invalid blend op");
|
||||
}
|
||||
}
|
||||
|
||||
enum pipe_blendfactor
|
||||
vk_blend_factor_to_pipe(VkBlendFactor vk_factor)
|
||||
{
|
||||
switch (vk_factor) {
|
||||
case VK_BLEND_FACTOR_ZERO:
|
||||
return PIPE_BLENDFACTOR_ZERO;
|
||||
case VK_BLEND_FACTOR_ONE:
|
||||
return PIPE_BLENDFACTOR_ONE;
|
||||
case VK_BLEND_FACTOR_SRC_COLOR:
|
||||
return PIPE_BLENDFACTOR_SRC_COLOR;
|
||||
case VK_BLEND_FACTOR_ONE_MINUS_SRC_COLOR:
|
||||
return PIPE_BLENDFACTOR_INV_SRC_COLOR;
|
||||
case VK_BLEND_FACTOR_DST_COLOR:
|
||||
return PIPE_BLENDFACTOR_DST_COLOR;
|
||||
case VK_BLEND_FACTOR_ONE_MINUS_DST_COLOR:
|
||||
return PIPE_BLENDFACTOR_INV_DST_COLOR;
|
||||
case VK_BLEND_FACTOR_SRC_ALPHA:
|
||||
return PIPE_BLENDFACTOR_SRC_ALPHA;
|
||||
case VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA:
|
||||
return PIPE_BLENDFACTOR_INV_SRC_ALPHA;
|
||||
case VK_BLEND_FACTOR_DST_ALPHA:
|
||||
return PIPE_BLENDFACTOR_DST_ALPHA;
|
||||
case VK_BLEND_FACTOR_ONE_MINUS_DST_ALPHA:
|
||||
return PIPE_BLENDFACTOR_INV_DST_ALPHA;
|
||||
case VK_BLEND_FACTOR_CONSTANT_COLOR:
|
||||
return PIPE_BLENDFACTOR_CONST_COLOR;
|
||||
case VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_COLOR:
|
||||
return PIPE_BLENDFACTOR_INV_CONST_COLOR;
|
||||
case VK_BLEND_FACTOR_CONSTANT_ALPHA:
|
||||
return PIPE_BLENDFACTOR_CONST_ALPHA;
|
||||
case VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_ALPHA:
|
||||
return PIPE_BLENDFACTOR_INV_CONST_ALPHA;
|
||||
case VK_BLEND_FACTOR_SRC1_COLOR:
|
||||
return PIPE_BLENDFACTOR_SRC1_COLOR;
|
||||
case VK_BLEND_FACTOR_ONE_MINUS_SRC1_COLOR:
|
||||
return PIPE_BLENDFACTOR_INV_SRC1_COLOR;
|
||||
case VK_BLEND_FACTOR_SRC1_ALPHA:
|
||||
return PIPE_BLENDFACTOR_SRC1_ALPHA;
|
||||
case VK_BLEND_FACTOR_ONE_MINUS_SRC1_ALPHA:
|
||||
return PIPE_BLENDFACTOR_INV_SRC1_ALPHA;
|
||||
case VK_BLEND_FACTOR_SRC_ALPHA_SATURATE:
|
||||
return PIPE_BLENDFACTOR_SRC_ALPHA_SATURATE;
|
||||
default:
|
||||
UNREACHABLE("Invalid blend factor");
|
||||
}
|
||||
}
|
||||
|
||||
enum pipe_advanced_blend_mode
|
||||
vk_advanced_blend_op_to_pipe(VkBlendOp in)
|
||||
{
|
||||
switch (in) {
|
||||
case VK_BLEND_OP_ZERO_EXT:
|
||||
return PIPE_ADVANCED_BLEND_NONE;
|
||||
case VK_BLEND_OP_MULTIPLY_EXT:
|
||||
return PIPE_ADVANCED_BLEND_MULTIPLY;
|
||||
case VK_BLEND_OP_SCREEN_EXT:
|
||||
return PIPE_ADVANCED_BLEND_SCREEN;
|
||||
case VK_BLEND_OP_OVERLAY_EXT:
|
||||
return PIPE_ADVANCED_BLEND_OVERLAY;
|
||||
case VK_BLEND_OP_DARKEN_EXT:
|
||||
return PIPE_ADVANCED_BLEND_DARKEN;
|
||||
case VK_BLEND_OP_LIGHTEN_EXT:
|
||||
return PIPE_ADVANCED_BLEND_LIGHTEN;
|
||||
case VK_BLEND_OP_COLORDODGE_EXT:
|
||||
return PIPE_ADVANCED_BLEND_COLORDODGE;
|
||||
case VK_BLEND_OP_COLORBURN_EXT:
|
||||
return PIPE_ADVANCED_BLEND_COLORBURN;
|
||||
case VK_BLEND_OP_HARDLIGHT_EXT:
|
||||
return PIPE_ADVANCED_BLEND_HARDLIGHT;
|
||||
case VK_BLEND_OP_SOFTLIGHT_EXT:
|
||||
return PIPE_ADVANCED_BLEND_SOFTLIGHT;
|
||||
case VK_BLEND_OP_DIFFERENCE_EXT:
|
||||
return PIPE_ADVANCED_BLEND_DIFFERENCE;
|
||||
case VK_BLEND_OP_EXCLUSION_EXT:
|
||||
return PIPE_ADVANCED_BLEND_EXCLUSION;
|
||||
case VK_BLEND_OP_HSL_HUE_EXT:
|
||||
return PIPE_ADVANCED_BLEND_HSL_HUE;
|
||||
case VK_BLEND_OP_HSL_SATURATION_EXT:
|
||||
return PIPE_ADVANCED_BLEND_HSL_SATURATION;
|
||||
case VK_BLEND_OP_HSL_COLOR_EXT:
|
||||
return PIPE_ADVANCED_BLEND_HSL_COLOR;
|
||||
case VK_BLEND_OP_HSL_LUMINOSITY_EXT:
|
||||
return PIPE_ADVANCED_BLEND_HSL_LUMINOSITY;
|
||||
case VK_BLEND_OP_SRC_EXT:
|
||||
return PIPE_ADVANCED_BLEND_SRC;
|
||||
case VK_BLEND_OP_DST_EXT:
|
||||
return PIPE_ADVANCED_BLEND_DST;
|
||||
case VK_BLEND_OP_SRC_OVER_EXT:
|
||||
return PIPE_ADVANCED_BLEND_SRC_OVER;
|
||||
case VK_BLEND_OP_DST_OVER_EXT:
|
||||
return PIPE_ADVANCED_BLEND_DST_OVER;
|
||||
case VK_BLEND_OP_SRC_IN_EXT:
|
||||
return PIPE_ADVANCED_BLEND_SRC_IN;
|
||||
case VK_BLEND_OP_DST_IN_EXT:
|
||||
return PIPE_ADVANCED_BLEND_DST_IN;
|
||||
case VK_BLEND_OP_SRC_OUT_EXT:
|
||||
return PIPE_ADVANCED_BLEND_SRC_OUT;
|
||||
case VK_BLEND_OP_DST_OUT_EXT:
|
||||
return PIPE_ADVANCED_BLEND_DST_OUT;
|
||||
case VK_BLEND_OP_SRC_ATOP_EXT:
|
||||
return PIPE_ADVANCED_BLEND_SRC_ATOP;
|
||||
case VK_BLEND_OP_DST_ATOP_EXT:
|
||||
return PIPE_ADVANCED_BLEND_DST_ATOP;
|
||||
case VK_BLEND_OP_XOR_EXT:
|
||||
return PIPE_ADVANCED_BLEND_XOR;
|
||||
case VK_BLEND_OP_INVERT_EXT:
|
||||
return PIPE_ADVANCED_BLEND_INVERT;
|
||||
case VK_BLEND_OP_INVERT_RGB_EXT:
|
||||
return PIPE_ADVANCED_BLEND_INVERT_RGB;
|
||||
case VK_BLEND_OP_LINEARDODGE_EXT:
|
||||
return PIPE_ADVANCED_BLEND_LINEARDODGE;
|
||||
case VK_BLEND_OP_LINEARBURN_EXT:
|
||||
return PIPE_ADVANCED_BLEND_LINEARBURN;
|
||||
case VK_BLEND_OP_VIVIDLIGHT_EXT:
|
||||
return PIPE_ADVANCED_BLEND_VIVIDLIGHT;
|
||||
case VK_BLEND_OP_LINEARLIGHT_EXT:
|
||||
return PIPE_ADVANCED_BLEND_LINEARLIGHT;
|
||||
case VK_BLEND_OP_PINLIGHT_EXT:
|
||||
return PIPE_ADVANCED_BLEND_PINLIGHT;
|
||||
case VK_BLEND_OP_HARDMIX_EXT:
|
||||
return PIPE_ADVANCED_BLEND_HARDMIX;
|
||||
case VK_BLEND_OP_PLUS_EXT:
|
||||
return PIPE_ADVANCED_BLEND_PLUS;
|
||||
case VK_BLEND_OP_PLUS_CLAMPED_EXT:
|
||||
return PIPE_ADVANCED_BLEND_PLUS_CLAMPED;
|
||||
case VK_BLEND_OP_PLUS_CLAMPED_ALPHA_EXT:
|
||||
return PIPE_ADVANCED_BLEND_PLUS_CLAMPED_ALPHA;
|
||||
case VK_BLEND_OP_PLUS_DARKER_EXT:
|
||||
return PIPE_ADVANCED_BLEND_PLUS_DARKER;
|
||||
case VK_BLEND_OP_MINUS_EXT:
|
||||
return PIPE_ADVANCED_BLEND_MINUS;
|
||||
case VK_BLEND_OP_MINUS_CLAMPED_EXT:
|
||||
return PIPE_ADVANCED_BLEND_MINUS_CLAMPED;
|
||||
case VK_BLEND_OP_CONTRAST_EXT:
|
||||
return PIPE_ADVANCED_BLEND_CONTRAST;
|
||||
case VK_BLEND_OP_INVERT_OVG_EXT:
|
||||
return PIPE_ADVANCED_BLEND_INVERT_OVG;
|
||||
case VK_BLEND_OP_RED_EXT:
|
||||
return PIPE_ADVANCED_BLEND_RED;
|
||||
case VK_BLEND_OP_GREEN_EXT:
|
||||
return PIPE_ADVANCED_BLEND_GREEN;
|
||||
case VK_BLEND_OP_BLUE_EXT:
|
||||
return PIPE_ADVANCED_BLEND_BLUE;
|
||||
default:
|
||||
UNREACHABLE("Invalid advanced blend op");
|
||||
}
|
||||
}
|
||||
|
||||
enum pipe_blend_overlap_mode
|
||||
vk_blend_overlap_to_pipe(VkBlendOverlapEXT in)
|
||||
{
|
||||
switch (in) {
|
||||
case VK_BLEND_OVERLAP_UNCORRELATED_EXT:
|
||||
return PIPE_BLEND_OVERLAP_UNCORRELATED;
|
||||
case VK_BLEND_OVERLAP_DISJOINT_EXT:
|
||||
return PIPE_BLEND_OVERLAP_DISJOINT;
|
||||
case VK_BLEND_OVERLAP_CONJOINT_EXT:
|
||||
return PIPE_BLEND_OVERLAP_CONJOINT;
|
||||
default:
|
||||
UNREACHABLE("Invalid blend overlap");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
/*
|
||||
* Copyright 2023 Valve Corporation
|
||||
* SPDX-License-Identifier: MIT
|
||||
*/
|
||||
|
||||
#ifndef VK_BLEND_H
|
||||
#define VK_BLEND_H
|
||||
|
||||
#include <stdbool.h>
|
||||
#include "util/blend.h"
|
||||
#include "vulkan/vulkan_core.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
enum pipe_logicop vk_logic_op_to_pipe(VkLogicOp in);
|
||||
enum pipe_blend_func vk_blend_op_to_pipe(VkBlendOp in);
|
||||
enum pipe_blendfactor vk_blend_factor_to_pipe(VkBlendFactor in);
|
||||
enum pipe_advanced_blend_mode vk_advanced_blend_op_to_pipe(VkBlendOp in);
|
||||
enum pipe_blend_overlap_mode vk_blend_overlap_to_pipe(VkBlendOverlapEXT in);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,176 @@
|
||||
/*
|
||||
* Copyright © 2022 Collabora, Ltd.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "vk_buffer.h"
|
||||
|
||||
#include "vk_common_entrypoints.h"
|
||||
#include "vk_alloc.h"
|
||||
#include "vk_device.h"
|
||||
#include "vk_util.h"
|
||||
|
||||
void
|
||||
vk_buffer_init(struct vk_device *device,
|
||||
struct vk_buffer *buffer,
|
||||
const VkBufferCreateInfo *pCreateInfo)
|
||||
{
|
||||
vk_object_base_init(device, &buffer->base, VK_OBJECT_TYPE_BUFFER);
|
||||
|
||||
assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO);
|
||||
assert(pCreateInfo->size > 0);
|
||||
|
||||
buffer->create_flags = pCreateInfo->flags;
|
||||
buffer->size = pCreateInfo->size;
|
||||
buffer->usage = pCreateInfo->usage;
|
||||
buffer->device_address = 0;
|
||||
|
||||
const VkBufferUsageFlags2CreateInfoKHR *usage2_info =
|
||||
vk_find_struct_const(pCreateInfo->pNext,
|
||||
BUFFER_USAGE_FLAGS_2_CREATE_INFO_KHR);
|
||||
if (usage2_info != NULL)
|
||||
buffer->usage = usage2_info->usage;
|
||||
|
||||
buffer->address_flags =
|
||||
((buffer->create_flags & VK_BUFFER_CREATE_PROTECTED_BIT) ?
|
||||
VK_ADDRESS_COMMAND_PROTECTED_BIT_KHR : 0) |
|
||||
((buffer->usage & VK_BUFFER_USAGE_STORAGE_BUFFER_BIT) ?
|
||||
VK_ADDRESS_COMMAND_STORAGE_BUFFER_USAGE_BIT_KHR : 0) |
|
||||
((buffer->usage & VK_BUFFER_USAGE_TRANSFORM_FEEDBACK_BUFFER_BIT_EXT) ?
|
||||
VK_ADDRESS_COMMAND_TRANSFORM_FEEDBACK_BUFFER_USAGE_BIT_KHR : 0) |
|
||||
((buffer->usage & VK_BUFFER_CREATE_SPARSE_BINDING_BIT) == 0 ?
|
||||
VK_ADDRESS_COMMAND_FULLY_BOUND_BIT_KHR : 0);
|
||||
|
||||
buffer->copy_flags =
|
||||
((buffer->create_flags & VK_BUFFER_CREATE_PROTECTED_BIT) ?
|
||||
VK_ADDRESS_COPY_PROTECTED_BIT_KHR : 0) |
|
||||
((buffer->create_flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT) ?
|
||||
VK_ADDRESS_COPY_SPARSE_BIT_KHR : 0);
|
||||
}
|
||||
|
||||
void *
|
||||
vk_buffer_create(struct vk_device *device,
|
||||
const VkBufferCreateInfo *pCreateInfo,
|
||||
const VkAllocationCallbacks *alloc,
|
||||
size_t size)
|
||||
{
|
||||
struct vk_buffer *buffer =
|
||||
vk_zalloc2(&device->alloc, alloc, size, 8,
|
||||
VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
|
||||
if (buffer == NULL)
|
||||
return NULL;
|
||||
|
||||
vk_buffer_init(device, buffer, pCreateInfo);
|
||||
|
||||
return buffer;
|
||||
}
|
||||
|
||||
void
|
||||
vk_buffer_finish(struct vk_buffer *buffer)
|
||||
{
|
||||
vk_object_base_finish(&buffer->base);
|
||||
}
|
||||
|
||||
void
|
||||
vk_buffer_destroy(struct vk_device *device,
|
||||
const VkAllocationCallbacks *alloc,
|
||||
struct vk_buffer *buffer)
|
||||
{
|
||||
vk_object_free(device, alloc, buffer);
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_GetBufferMemoryRequirements(VkDevice _device,
|
||||
VkBuffer buffer,
|
||||
VkMemoryRequirements *pMemoryRequirements)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_device, device, _device);
|
||||
|
||||
VkBufferMemoryRequirementsInfo2 info = {
|
||||
.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_REQUIREMENTS_INFO_2,
|
||||
.buffer = buffer,
|
||||
};
|
||||
VkMemoryRequirements2 reqs = {
|
||||
.sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2,
|
||||
};
|
||||
device->dispatch_table.GetBufferMemoryRequirements2(_device, &info, &reqs);
|
||||
|
||||
*pMemoryRequirements = reqs.memoryRequirements;
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_GetBufferMemoryRequirements2(VkDevice _device,
|
||||
const VkBufferMemoryRequirementsInfo2 *pInfo,
|
||||
VkMemoryRequirements2 *pMemoryRequirements)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_device, device, _device);
|
||||
VK_FROM_HANDLE(vk_buffer, buffer, pInfo->buffer);
|
||||
|
||||
VkBufferUsageFlags2CreateInfoKHR usage2_info = {
|
||||
.sType = VK_STRUCTURE_TYPE_BUFFER_USAGE_FLAGS_2_CREATE_INFO_KHR,
|
||||
.pNext = NULL,
|
||||
.usage = buffer->usage,
|
||||
};
|
||||
VkBufferCreateInfo pCreateInfo = {
|
||||
.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,
|
||||
.pNext = &usage2_info,
|
||||
.usage = buffer->usage,
|
||||
.size = buffer->size,
|
||||
.flags = buffer->create_flags,
|
||||
.sharingMode = VK_SHARING_MODE_EXCLUSIVE,
|
||||
.queueFamilyIndexCount = 0,
|
||||
.pQueueFamilyIndices = NULL,
|
||||
};
|
||||
VkDeviceBufferMemoryRequirements info = {
|
||||
.sType = VK_STRUCTURE_TYPE_DEVICE_BUFFER_MEMORY_REQUIREMENTS,
|
||||
.pNext = NULL,
|
||||
.pCreateInfo = &pCreateInfo,
|
||||
};
|
||||
|
||||
device->dispatch_table.GetDeviceBufferMemoryRequirements(_device, &info, pMemoryRequirements);
|
||||
}
|
||||
|
||||
VKAPI_ATTR VkResult VKAPI_CALL
|
||||
vk_common_BindBufferMemory(VkDevice _device,
|
||||
VkBuffer buffer,
|
||||
VkDeviceMemory memory,
|
||||
VkDeviceSize memoryOffset)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_device, device, _device);
|
||||
|
||||
VkBindBufferMemoryInfo bind = {
|
||||
.sType = VK_STRUCTURE_TYPE_BIND_BUFFER_MEMORY_INFO,
|
||||
.buffer = buffer,
|
||||
.memory = memory,
|
||||
.memoryOffset = memoryOffset,
|
||||
};
|
||||
|
||||
return device->dispatch_table.BindBufferMemory2(_device, 1, &bind);
|
||||
}
|
||||
|
||||
VKAPI_ATTR VkDeviceAddress VKAPI_CALL
|
||||
vk_common_GetBufferDeviceAddress(UNUSED VkDevice device,
|
||||
const VkBufferDeviceAddressInfo *pInfo)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_buffer, buffer, pInfo->buffer);
|
||||
|
||||
return buffer->device_address;
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
/*
|
||||
* Copyright © 2022 Collabora, Ltd.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
#ifndef VK_BUFFER_H
|
||||
#define VK_BUFFER_H
|
||||
|
||||
#include "vk_object.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
struct vk_buffer {
|
||||
struct vk_object_base base;
|
||||
|
||||
/** VkBufferCreateInfo::flags */
|
||||
VkBufferCreateFlags create_flags;
|
||||
|
||||
/** VkBufferCreateInfo::size */
|
||||
VkDeviceSize size;
|
||||
|
||||
/** VkBufferCreateInfo::usage or VkBufferUsageFlags2CreateInfoKHR::usage */
|
||||
VkBufferUsageFlags2KHR usage;
|
||||
|
||||
/** Set by the implementation
|
||||
*
|
||||
* The implementation must set this at creation for sparse buffers or can
|
||||
* delay as far as the bind for non-sparse buffers.
|
||||
*/
|
||||
VkDeviceAddress device_address;
|
||||
|
||||
/** Inferred address flags from create_flags */
|
||||
VkAddressCommandFlagsKHR address_flags;
|
||||
|
||||
/** Inferred copy flags from create_flags */
|
||||
VkAddressCopyFlagsKHR copy_flags;
|
||||
};
|
||||
VK_DEFINE_NONDISP_HANDLE_CASTS(vk_buffer, base, VkBuffer,
|
||||
VK_OBJECT_TYPE_BUFFER);
|
||||
|
||||
void vk_buffer_init(struct vk_device *device,
|
||||
struct vk_buffer *buffer,
|
||||
const VkBufferCreateInfo *pCreateInfo);
|
||||
void vk_buffer_finish(struct vk_buffer *buffer);
|
||||
|
||||
void *vk_buffer_create(struct vk_device *device,
|
||||
const VkBufferCreateInfo *pCreateInfo,
|
||||
const VkAllocationCallbacks *alloc,
|
||||
size_t size);
|
||||
void vk_buffer_destroy(struct vk_device *device,
|
||||
const VkAllocationCallbacks *alloc,
|
||||
struct vk_buffer *buffer);
|
||||
|
||||
static inline VkDeviceAddress
|
||||
vk_buffer_address(const struct vk_buffer *buffer,
|
||||
VkDeviceSize offset)
|
||||
{
|
||||
assert(buffer->size == 0 || buffer->device_address != 0);
|
||||
return buffer->device_address + offset;
|
||||
}
|
||||
|
||||
static inline uint64_t
|
||||
vk_buffer_range(const struct vk_buffer *buffer,
|
||||
uint64_t offset, uint64_t range)
|
||||
{
|
||||
assert(offset <= buffer->size);
|
||||
if (range == VK_WHOLE_SIZE) {
|
||||
return buffer->size - offset;
|
||||
} else {
|
||||
assert(range + offset >= range);
|
||||
assert(range + offset <= buffer->size);
|
||||
return range;
|
||||
}
|
||||
}
|
||||
|
||||
static inline VkDeviceAddressRangeKHR
|
||||
vk_device_address_range(const struct vk_buffer *buffer,
|
||||
VkDeviceSize offset,
|
||||
VkDeviceSize range)
|
||||
{
|
||||
VkDeviceAddressRangeKHR addr_range = { 0 };
|
||||
|
||||
if (buffer) {
|
||||
addr_range.address = vk_buffer_address(buffer, offset);
|
||||
addr_range.size = vk_buffer_range(buffer, offset, range);
|
||||
}
|
||||
|
||||
return addr_range;
|
||||
}
|
||||
|
||||
static inline VkStridedDeviceAddressRangeKHR
|
||||
vk_strided_device_address_range(const struct vk_buffer *buffer,
|
||||
VkDeviceSize offset,
|
||||
VkDeviceSize range,
|
||||
VkDeviceSize stride)
|
||||
{
|
||||
VkStridedDeviceAddressRangeKHR addr_range = { 0 };
|
||||
|
||||
if (buffer) {
|
||||
addr_range.address = vk_buffer_address(buffer, offset);
|
||||
addr_range.size = vk_buffer_range(buffer, offset, range);
|
||||
addr_range.stride = stride;
|
||||
}
|
||||
|
||||
return addr_range;
|
||||
}
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* VK_BUFFER_H */
|
||||
@@ -0,0 +1,83 @@
|
||||
/*
|
||||
* Copyright © 2022 Collabora, Ltd.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "vk_buffer_view.h"
|
||||
|
||||
#include "vk_alloc.h"
|
||||
#include "vk_buffer.h"
|
||||
#include "vk_device.h"
|
||||
#include "vk_format.h"
|
||||
|
||||
void
|
||||
vk_buffer_view_init(struct vk_device *device,
|
||||
struct vk_buffer_view *buffer_view,
|
||||
const VkBufferViewCreateInfo *pCreateInfo)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_buffer, buffer, pCreateInfo->buffer);
|
||||
|
||||
vk_object_base_init(device, &buffer_view->base, VK_OBJECT_TYPE_BUFFER_VIEW);
|
||||
|
||||
assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO);
|
||||
assert(pCreateInfo->flags == 0);
|
||||
assert(pCreateInfo->range > 0);
|
||||
|
||||
buffer_view->buffer = buffer;
|
||||
buffer_view->format = pCreateInfo->format;
|
||||
buffer_view->offset = pCreateInfo->offset;
|
||||
buffer_view->range = vk_buffer_range(buffer, pCreateInfo->offset,
|
||||
pCreateInfo->range);
|
||||
buffer_view->elements = buffer_view->range /
|
||||
vk_format_get_blocksize(buffer_view->format);
|
||||
}
|
||||
|
||||
void *
|
||||
vk_buffer_view_create(struct vk_device *device,
|
||||
const VkBufferViewCreateInfo *pCreateInfo,
|
||||
const VkAllocationCallbacks *alloc,
|
||||
size_t size)
|
||||
{
|
||||
struct vk_buffer_view *buffer_view;
|
||||
|
||||
buffer_view = vk_zalloc2(&device->alloc, alloc, size, 8,
|
||||
VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
|
||||
if (!buffer_view)
|
||||
return NULL;
|
||||
|
||||
vk_buffer_view_init(device, buffer_view, pCreateInfo);
|
||||
|
||||
return buffer_view;
|
||||
}
|
||||
|
||||
void
|
||||
vk_buffer_view_finish(struct vk_buffer_view *buffer_view)
|
||||
{
|
||||
vk_object_base_finish(&buffer_view->base);
|
||||
}
|
||||
|
||||
void
|
||||
vk_buffer_view_destroy(struct vk_device *device,
|
||||
const VkAllocationCallbacks *alloc,
|
||||
struct vk_buffer_view *buffer_view)
|
||||
{
|
||||
vk_object_free(device, alloc, buffer_view);
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
/*
|
||||
* Copyright © 2022 Collabora, Ltd.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
#ifndef VK_BUFFER_VIEW_H
|
||||
#define VK_BUFFER_VIEW_H
|
||||
|
||||
#include "vk_object.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
struct vk_buffer_view {
|
||||
struct vk_object_base base;
|
||||
|
||||
/** VkBufferViewCreateInfo::buffer */
|
||||
struct vk_buffer *buffer;
|
||||
|
||||
/** VkBufferViewCreateInfo::format */
|
||||
VkFormat format;
|
||||
|
||||
/** VkBufferViewCreateInfo::offset */
|
||||
VkDeviceSize offset;
|
||||
|
||||
/** VkBufferViewCreateInfo::range
|
||||
*
|
||||
* This is asserted to be in-range for the attached buffer and will never
|
||||
* be VK_WHOLE_SIZE.
|
||||
*/
|
||||
VkDeviceSize range;
|
||||
|
||||
/* Number of elements in the buffer. This is range divided by the size of
|
||||
* format, rounded down.
|
||||
*/
|
||||
VkDeviceSize elements;
|
||||
};
|
||||
VK_DEFINE_NONDISP_HANDLE_CASTS(vk_buffer_view, base, VkBufferView,
|
||||
VK_OBJECT_TYPE_BUFFER_VIEW);
|
||||
|
||||
void vk_buffer_view_init(struct vk_device *device,
|
||||
struct vk_buffer_view *buffer_view,
|
||||
const VkBufferViewCreateInfo *pCreateInfo);
|
||||
void vk_buffer_view_finish(struct vk_buffer_view *buffer_view);
|
||||
void *vk_buffer_view_create(struct vk_device *device,
|
||||
const VkBufferViewCreateInfo *pCreateInfo,
|
||||
const VkAllocationCallbacks *alloc,
|
||||
size_t size);
|
||||
void vk_buffer_view_destroy(struct vk_device *device,
|
||||
const VkAllocationCallbacks *alloc,
|
||||
struct vk_buffer_view *buffer_view);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* VK_BUFFER_VIEW_H */
|
||||
@@ -0,0 +1,277 @@
|
||||
/*
|
||||
* Copyright © 2021 Intel Corporation
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "vk_common_entrypoints.h"
|
||||
#include "vk_device.h"
|
||||
#include "vk_util.h"
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdCopyBuffer(VkCommandBuffer commandBuffer,
|
||||
VkBuffer srcBuffer,
|
||||
VkBuffer dstBuffer,
|
||||
uint32_t regionCount,
|
||||
const VkBufferCopy *pRegions)
|
||||
{
|
||||
/* We don't have a vk_command_buffer object but we can assume, since we're
|
||||
* using common dispatch, that it's a vk_object of some sort.
|
||||
*/
|
||||
struct vk_object_base *disp = (struct vk_object_base *)commandBuffer;
|
||||
|
||||
STACK_ARRAY(VkBufferCopy2, region2s, regionCount);
|
||||
|
||||
for (uint32_t r = 0; r < regionCount; r++) {
|
||||
region2s[r] = (VkBufferCopy2) {
|
||||
.sType = VK_STRUCTURE_TYPE_BUFFER_COPY_2,
|
||||
.srcOffset = pRegions[r].srcOffset,
|
||||
.dstOffset = pRegions[r].dstOffset,
|
||||
.size = pRegions[r].size,
|
||||
};
|
||||
}
|
||||
|
||||
VkCopyBufferInfo2 info = {
|
||||
.sType = VK_STRUCTURE_TYPE_COPY_BUFFER_INFO_2,
|
||||
.srcBuffer = srcBuffer,
|
||||
.dstBuffer = dstBuffer,
|
||||
.regionCount = regionCount,
|
||||
.pRegions = region2s,
|
||||
};
|
||||
|
||||
disp->device->dispatch_table.CmdCopyBuffer2(commandBuffer, &info);
|
||||
|
||||
STACK_ARRAY_FINISH(region2s);
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdCopyImage(VkCommandBuffer commandBuffer,
|
||||
VkImage srcImage,
|
||||
VkImageLayout srcImageLayout,
|
||||
VkImage dstImage,
|
||||
VkImageLayout dstImageLayout,
|
||||
uint32_t regionCount,
|
||||
const VkImageCopy *pRegions)
|
||||
{
|
||||
/* We don't have a vk_command_buffer object but we can assume, since we're
|
||||
* using common dispatch, that it's a vk_object of some sort.
|
||||
*/
|
||||
struct vk_object_base *disp = (struct vk_object_base *)commandBuffer;
|
||||
|
||||
STACK_ARRAY(VkImageCopy2, region2s, regionCount);
|
||||
|
||||
for (uint32_t r = 0; r < regionCount; r++) {
|
||||
region2s[r] = (VkImageCopy2) {
|
||||
.sType = VK_STRUCTURE_TYPE_IMAGE_COPY_2,
|
||||
.srcSubresource = pRegions[r].srcSubresource,
|
||||
.srcOffset = pRegions[r].srcOffset,
|
||||
.dstSubresource = pRegions[r].dstSubresource,
|
||||
.dstOffset = pRegions[r].dstOffset,
|
||||
.extent = pRegions[r].extent,
|
||||
};
|
||||
}
|
||||
|
||||
VkCopyImageInfo2 info = {
|
||||
.sType = VK_STRUCTURE_TYPE_COPY_IMAGE_INFO_2,
|
||||
.srcImage = srcImage,
|
||||
.srcImageLayout = srcImageLayout,
|
||||
.dstImage = dstImage,
|
||||
.dstImageLayout = dstImageLayout,
|
||||
.regionCount = regionCount,
|
||||
.pRegions = region2s,
|
||||
};
|
||||
|
||||
disp->device->dispatch_table.CmdCopyImage2(commandBuffer, &info);
|
||||
|
||||
STACK_ARRAY_FINISH(region2s);
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdCopyBufferToImage(VkCommandBuffer commandBuffer,
|
||||
VkBuffer srcBuffer,
|
||||
VkImage dstImage,
|
||||
VkImageLayout dstImageLayout,
|
||||
uint32_t regionCount,
|
||||
const VkBufferImageCopy *pRegions)
|
||||
{
|
||||
/* We don't have a vk_command_buffer object but we can assume, since we're
|
||||
* using common dispatch, that it's a vk_object of some sort.
|
||||
*/
|
||||
struct vk_object_base *disp = (struct vk_object_base *)commandBuffer;
|
||||
|
||||
STACK_ARRAY(VkBufferImageCopy2, region2s, regionCount);
|
||||
|
||||
for (uint32_t r = 0; r < regionCount; r++) {
|
||||
region2s[r] = (VkBufferImageCopy2) {
|
||||
.sType = VK_STRUCTURE_TYPE_BUFFER_IMAGE_COPY_2,
|
||||
.bufferOffset = pRegions[r].bufferOffset,
|
||||
.bufferRowLength = pRegions[r].bufferRowLength,
|
||||
.bufferImageHeight = pRegions[r].bufferImageHeight,
|
||||
.imageSubresource = pRegions[r].imageSubresource,
|
||||
.imageOffset = pRegions[r].imageOffset,
|
||||
.imageExtent = pRegions[r].imageExtent,
|
||||
};
|
||||
}
|
||||
|
||||
VkCopyBufferToImageInfo2 info = {
|
||||
.sType = VK_STRUCTURE_TYPE_COPY_BUFFER_TO_IMAGE_INFO_2,
|
||||
.srcBuffer = srcBuffer,
|
||||
.dstImage = dstImage,
|
||||
.dstImageLayout = dstImageLayout,
|
||||
.regionCount = regionCount,
|
||||
.pRegions = region2s,
|
||||
};
|
||||
|
||||
disp->device->dispatch_table.CmdCopyBufferToImage2(commandBuffer, &info);
|
||||
|
||||
STACK_ARRAY_FINISH(region2s);
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdCopyImageToBuffer(VkCommandBuffer commandBuffer,
|
||||
VkImage srcImage,
|
||||
VkImageLayout srcImageLayout,
|
||||
VkBuffer dstBuffer,
|
||||
uint32_t regionCount,
|
||||
const VkBufferImageCopy *pRegions)
|
||||
{
|
||||
/* We don't have a vk_command_buffer object but we can assume, since we're
|
||||
* using common dispatch, that it's a vk_object of some sort.
|
||||
*/
|
||||
struct vk_object_base *disp = (struct vk_object_base *)commandBuffer;
|
||||
|
||||
STACK_ARRAY(VkBufferImageCopy2, region2s, regionCount);
|
||||
|
||||
for (uint32_t r = 0; r < regionCount; r++) {
|
||||
region2s[r] = (VkBufferImageCopy2) {
|
||||
.sType = VK_STRUCTURE_TYPE_BUFFER_IMAGE_COPY_2,
|
||||
.bufferOffset = pRegions[r].bufferOffset,
|
||||
.bufferRowLength = pRegions[r].bufferRowLength,
|
||||
.bufferImageHeight = pRegions[r].bufferImageHeight,
|
||||
.imageSubresource = pRegions[r].imageSubresource,
|
||||
.imageOffset = pRegions[r].imageOffset,
|
||||
.imageExtent = pRegions[r].imageExtent,
|
||||
};
|
||||
}
|
||||
|
||||
VkCopyImageToBufferInfo2 info = {
|
||||
.sType = VK_STRUCTURE_TYPE_COPY_IMAGE_TO_BUFFER_INFO_2,
|
||||
.srcImage = srcImage,
|
||||
.srcImageLayout = srcImageLayout,
|
||||
.dstBuffer = dstBuffer,
|
||||
.regionCount = regionCount,
|
||||
.pRegions = region2s,
|
||||
};
|
||||
|
||||
disp->device->dispatch_table.CmdCopyImageToBuffer2(commandBuffer, &info);
|
||||
|
||||
STACK_ARRAY_FINISH(region2s);
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdBlitImage(VkCommandBuffer commandBuffer,
|
||||
VkImage srcImage,
|
||||
VkImageLayout srcImageLayout,
|
||||
VkImage dstImage,
|
||||
VkImageLayout dstImageLayout,
|
||||
uint32_t regionCount,
|
||||
const VkImageBlit *pRegions,
|
||||
VkFilter filter)
|
||||
{
|
||||
/* We don't have a vk_command_buffer object but we can assume, since we're
|
||||
* using common dispatch, that it's a vk_object of some sort.
|
||||
*/
|
||||
struct vk_object_base *disp = (struct vk_object_base *)commandBuffer;
|
||||
|
||||
STACK_ARRAY(VkImageBlit2, region2s, regionCount);
|
||||
|
||||
for (uint32_t r = 0; r < regionCount; r++) {
|
||||
region2s[r] = (VkImageBlit2) {
|
||||
.sType = VK_STRUCTURE_TYPE_IMAGE_BLIT_2,
|
||||
.srcSubresource = pRegions[r].srcSubresource,
|
||||
.srcOffsets = {
|
||||
pRegions[r].srcOffsets[0],
|
||||
pRegions[r].srcOffsets[1],
|
||||
},
|
||||
.dstSubresource = pRegions[r].dstSubresource,
|
||||
.dstOffsets = {
|
||||
pRegions[r].dstOffsets[0],
|
||||
pRegions[r].dstOffsets[1],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
VkBlitImageInfo2 info = {
|
||||
.sType = VK_STRUCTURE_TYPE_BLIT_IMAGE_INFO_2,
|
||||
.srcImage = srcImage,
|
||||
.srcImageLayout = srcImageLayout,
|
||||
.dstImage = dstImage,
|
||||
.dstImageLayout = dstImageLayout,
|
||||
.regionCount = regionCount,
|
||||
.pRegions = region2s,
|
||||
.filter = filter,
|
||||
};
|
||||
|
||||
disp->device->dispatch_table.CmdBlitImage2(commandBuffer, &info);
|
||||
|
||||
STACK_ARRAY_FINISH(region2s);
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdResolveImage(VkCommandBuffer commandBuffer,
|
||||
VkImage srcImage,
|
||||
VkImageLayout srcImageLayout,
|
||||
VkImage dstImage,
|
||||
VkImageLayout dstImageLayout,
|
||||
uint32_t regionCount,
|
||||
const VkImageResolve *pRegions)
|
||||
{
|
||||
/* We don't have a vk_command_buffer object but we can assume, since we're
|
||||
* using common dispatch, that it's a vk_object of some sort.
|
||||
*/
|
||||
struct vk_object_base *disp = (struct vk_object_base *)commandBuffer;
|
||||
|
||||
STACK_ARRAY(VkImageResolve2, region2s, regionCount);
|
||||
|
||||
for (uint32_t r = 0; r < regionCount; r++) {
|
||||
region2s[r] = (VkImageResolve2) {
|
||||
.sType = VK_STRUCTURE_TYPE_IMAGE_RESOLVE_2,
|
||||
.srcSubresource = pRegions[r].srcSubresource,
|
||||
.srcOffset = pRegions[r].srcOffset,
|
||||
.dstSubresource = pRegions[r].dstSubresource,
|
||||
.dstOffset = pRegions[r].dstOffset,
|
||||
.extent = pRegions[r].extent,
|
||||
};
|
||||
}
|
||||
|
||||
VkResolveImageInfo2 info = {
|
||||
.sType = VK_STRUCTURE_TYPE_RESOLVE_IMAGE_INFO_2,
|
||||
.srcImage = srcImage,
|
||||
.srcImageLayout = srcImageLayout,
|
||||
.dstImage = dstImage,
|
||||
.dstImageLayout = dstImageLayout,
|
||||
.regionCount = regionCount,
|
||||
.pRegions = region2s,
|
||||
};
|
||||
|
||||
disp->device->dispatch_table.CmdResolveImage2(commandBuffer, &info);
|
||||
|
||||
STACK_ARRAY_FINISH(region2s);
|
||||
}
|
||||
@@ -0,0 +1,249 @@
|
||||
/*
|
||||
* Copyright © 2019 Red Hat.
|
||||
* Copyright © 2022 Collabora, LTD
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "vk_alloc.h"
|
||||
#include "vk_cmd_enqueue_entrypoints.h"
|
||||
#include "vk_command_buffer.h"
|
||||
#include "vk_device.h"
|
||||
#include "vk_util.h"
|
||||
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_cmd_enqueue_CmdDrawMultiEXT(VkCommandBuffer commandBuffer,
|
||||
uint32_t drawCount,
|
||||
const VkMultiDrawInfoEXT *pVertexInfo,
|
||||
uint32_t instanceCount,
|
||||
uint32_t firstInstance,
|
||||
uint32_t stride)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
|
||||
|
||||
struct vk_cmd_queue_entry *cmd =
|
||||
linear_zalloc_child(cmd_buffer->cmd_queue.ctx, sizeof(*cmd));
|
||||
if (!cmd)
|
||||
return;
|
||||
|
||||
cmd->type = VK_CMD_DRAW_MULTI_EXT;
|
||||
list_addtail(&cmd->cmd_link, &cmd_buffer->cmd_queue.cmds);
|
||||
|
||||
cmd->u.draw_multi_ext.draw_count = drawCount;
|
||||
if (pVertexInfo) {
|
||||
unsigned i = 0;
|
||||
cmd->u.draw_multi_ext.vertex_info =
|
||||
linear_alloc_child(cmd_buffer->cmd_queue.ctx,
|
||||
sizeof(*cmd->u.draw_multi_ext.vertex_info) * drawCount);
|
||||
|
||||
vk_foreach_multi_draw(draw, i, pVertexInfo, drawCount, stride) {
|
||||
memcpy(&cmd->u.draw_multi_ext.vertex_info[i], draw,
|
||||
sizeof(*cmd->u.draw_multi_ext.vertex_info));
|
||||
}
|
||||
}
|
||||
cmd->u.draw_multi_ext.instance_count = instanceCount;
|
||||
cmd->u.draw_multi_ext.first_instance = firstInstance;
|
||||
cmd->u.draw_multi_ext.stride = sizeof(*cmd->u.draw_multi_ext.vertex_info);
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_cmd_enqueue_CmdDrawMultiIndexedEXT(VkCommandBuffer commandBuffer,
|
||||
uint32_t drawCount,
|
||||
const VkMultiDrawIndexedInfoEXT *pIndexInfo,
|
||||
uint32_t instanceCount,
|
||||
uint32_t firstInstance,
|
||||
uint32_t stride,
|
||||
const int32_t *pVertexOffset)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
|
||||
|
||||
struct vk_cmd_queue_entry *cmd =
|
||||
linear_zalloc_child(cmd_buffer->cmd_queue.ctx, sizeof(*cmd));
|
||||
if (!cmd)
|
||||
return;
|
||||
|
||||
cmd->type = VK_CMD_DRAW_MULTI_INDEXED_EXT;
|
||||
list_addtail(&cmd->cmd_link, &cmd_buffer->cmd_queue.cmds);
|
||||
|
||||
cmd->u.draw_multi_indexed_ext.draw_count = drawCount;
|
||||
|
||||
if (pIndexInfo) {
|
||||
unsigned i = 0;
|
||||
cmd->u.draw_multi_indexed_ext.index_info =
|
||||
linear_alloc_child(cmd_buffer->cmd_queue.ctx,
|
||||
sizeof(*cmd->u.draw_multi_indexed_ext.index_info) * drawCount);
|
||||
|
||||
vk_foreach_multi_draw_indexed(draw, i, pIndexInfo, drawCount, stride) {
|
||||
cmd->u.draw_multi_indexed_ext.index_info[i].firstIndex = draw->firstIndex;
|
||||
cmd->u.draw_multi_indexed_ext.index_info[i].indexCount = draw->indexCount;
|
||||
if (pVertexOffset == NULL)
|
||||
cmd->u.draw_multi_indexed_ext.index_info[i].vertexOffset = draw->vertexOffset;
|
||||
}
|
||||
}
|
||||
|
||||
cmd->u.draw_multi_indexed_ext.instance_count = instanceCount;
|
||||
cmd->u.draw_multi_indexed_ext.first_instance = firstInstance;
|
||||
cmd->u.draw_multi_indexed_ext.stride = sizeof(*cmd->u.draw_multi_indexed_ext.index_info);
|
||||
|
||||
if (pVertexOffset) {
|
||||
cmd->u.draw_multi_indexed_ext.vertex_offset =
|
||||
linear_alloc_child(cmd_buffer->cmd_queue.ctx,
|
||||
sizeof(*cmd->u.draw_multi_indexed_ext.vertex_offset));
|
||||
|
||||
memcpy(cmd->u.draw_multi_indexed_ext.vertex_offset, pVertexOffset,
|
||||
sizeof(*cmd->u.draw_multi_indexed_ext.vertex_offset));
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef VK_ENABLE_BETA_EXTENSIONS
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_cmd_enqueue_CmdDispatchGraphAMDX(VkCommandBuffer commandBuffer, VkDeviceAddress scratch,
|
||||
VkDeviceSize scratchSize,
|
||||
const VkDispatchGraphCountInfoAMDX *pCountInfo)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
|
||||
|
||||
if (vk_command_buffer_has_error(cmd_buffer))
|
||||
return;
|
||||
|
||||
VkResult result = VK_SUCCESS;
|
||||
linear_ctx *ctx = cmd_buffer->cmd_queue.ctx;
|
||||
|
||||
struct vk_cmd_queue_entry *cmd = linear_zalloc_child(ctx, sizeof(struct vk_cmd_queue_entry));
|
||||
if (!cmd) {
|
||||
result = VK_ERROR_OUT_OF_HOST_MEMORY;
|
||||
goto finish;
|
||||
}
|
||||
|
||||
cmd->type = VK_CMD_DISPATCH_GRAPH_AMDX;
|
||||
|
||||
cmd->u.dispatch_graph_amdx.scratch = scratch;
|
||||
cmd->u.dispatch_graph_amdx.scratch_size = scratchSize;
|
||||
|
||||
cmd->u.dispatch_graph_amdx.count_info = linear_alloc_child(ctx, sizeof(VkDispatchGraphCountInfoAMDX));
|
||||
if (cmd->u.dispatch_graph_amdx.count_info == NULL) {
|
||||
result = VK_ERROR_OUT_OF_HOST_MEMORY;
|
||||
goto finish;
|
||||
}
|
||||
|
||||
memcpy((void *)cmd->u.dispatch_graph_amdx.count_info, pCountInfo,
|
||||
sizeof(VkDispatchGraphCountInfoAMDX));
|
||||
|
||||
uint32_t infos_size = pCountInfo->count * pCountInfo->stride;
|
||||
void *infos = linear_alloc_child(ctx, infos_size);
|
||||
cmd->u.dispatch_graph_amdx.count_info->infos.hostAddress = infos;
|
||||
memcpy(infos, pCountInfo->infos.hostAddress, infos_size);
|
||||
|
||||
for (uint32_t i = 0; i < pCountInfo->count; i++) {
|
||||
VkDispatchGraphInfoAMDX *info = (void *)((const uint8_t *)infos + i * pCountInfo->stride);
|
||||
|
||||
uint32_t payloads_size = info->payloadCount * info->payloadStride;
|
||||
void *dst_payload = linear_alloc_child(ctx, payloads_size);
|
||||
memcpy(dst_payload, info->payloads.hostAddress, payloads_size);
|
||||
info->payloads.hostAddress = dst_payload;
|
||||
}
|
||||
|
||||
list_addtail(&cmd->cmd_link, &cmd_buffer->cmd_queue.cmds);
|
||||
finish:
|
||||
if (unlikely(result != VK_SUCCESS))
|
||||
vk_command_buffer_set_error(cmd_buffer, result);
|
||||
}
|
||||
#endif
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_cmd_enqueue_CmdBuildAccelerationStructuresKHR(
|
||||
VkCommandBuffer commandBuffer, uint32_t infoCount,
|
||||
const VkAccelerationStructureBuildGeometryInfoKHR *pInfos,
|
||||
const VkAccelerationStructureBuildRangeInfoKHR *const *ppBuildRangeInfos)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
|
||||
|
||||
if (vk_command_buffer_has_error(cmd_buffer))
|
||||
return;
|
||||
|
||||
struct vk_cmd_queue *queue = &cmd_buffer->cmd_queue;
|
||||
|
||||
struct vk_cmd_queue_entry *cmd =
|
||||
linear_zalloc_child(queue->ctx, vk_cmd_queue_type_sizes[VK_CMD_BUILD_ACCELERATION_STRUCTURES_KHR]);
|
||||
if (!cmd)
|
||||
goto err;
|
||||
|
||||
cmd->type = VK_CMD_BUILD_ACCELERATION_STRUCTURES_KHR;
|
||||
|
||||
struct vk_cmd_build_acceleration_structures_khr *build =
|
||||
&cmd->u.build_acceleration_structures_khr;
|
||||
|
||||
build->info_count = infoCount;
|
||||
if (pInfos) {
|
||||
build->infos = linear_alloc_child(queue->ctx, sizeof(*build->infos) * infoCount);
|
||||
if (!build->infos)
|
||||
goto err;
|
||||
|
||||
memcpy((VkAccelerationStructureBuildGeometryInfoKHR *)build->infos, pInfos,
|
||||
sizeof(*build->infos) * (infoCount));
|
||||
|
||||
for (uint32_t i = 0; i < infoCount; i++) {
|
||||
uint32_t geometries_size =
|
||||
build->infos[i].geometryCount * sizeof(VkAccelerationStructureGeometryKHR);
|
||||
VkAccelerationStructureGeometryKHR *geometries = linear_alloc_child(queue->ctx, geometries_size);
|
||||
if (!geometries)
|
||||
goto err;
|
||||
|
||||
if (pInfos[i].pGeometries) {
|
||||
memcpy(geometries, pInfos[i].pGeometries, geometries_size);
|
||||
} else {
|
||||
for (uint32_t j = 0; j < build->infos[i].geometryCount; j++)
|
||||
memcpy(&geometries[j], pInfos[i].ppGeometries[j], sizeof(VkAccelerationStructureGeometryKHR));
|
||||
}
|
||||
|
||||
build->infos[i].pGeometries = geometries;
|
||||
}
|
||||
}
|
||||
if (ppBuildRangeInfos) {
|
||||
build->pp_build_range_infos =
|
||||
linear_alloc_child(queue->ctx, sizeof(*build->pp_build_range_infos) * infoCount);
|
||||
if (!build->pp_build_range_infos)
|
||||
goto err;
|
||||
|
||||
VkAccelerationStructureBuildRangeInfoKHR **pp_build_range_infos =
|
||||
(void *)build->pp_build_range_infos;
|
||||
|
||||
for (uint32_t i = 0; i < infoCount; i++) {
|
||||
uint32_t build_range_size =
|
||||
build->infos[i].geometryCount * sizeof(VkAccelerationStructureBuildRangeInfoKHR);
|
||||
VkAccelerationStructureBuildRangeInfoKHR *p_build_range_infos =
|
||||
linear_alloc_child(queue->ctx, build_range_size);
|
||||
if (!p_build_range_infos)
|
||||
goto err;
|
||||
|
||||
memcpy(p_build_range_infos, ppBuildRangeInfos[i], build_range_size);
|
||||
|
||||
pp_build_range_infos[i] = p_build_range_infos;
|
||||
}
|
||||
}
|
||||
|
||||
list_addtail(&cmd->cmd_link, &queue->cmds);
|
||||
return;
|
||||
|
||||
err:
|
||||
vk_command_buffer_set_error(cmd_buffer, VK_ERROR_OUT_OF_HOST_MEMORY);
|
||||
}
|
||||
@@ -0,0 +1,953 @@
|
||||
/*
|
||||
* Copyright © 2021 Intel Corporation
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "vk_command_buffer.h"
|
||||
|
||||
#include "vk_alloc.h"
|
||||
#include "vk_buffer.h"
|
||||
#include "vk_command_pool.h"
|
||||
#include "vk_common_entrypoints.h"
|
||||
#include "vk_device.h"
|
||||
|
||||
#include "vk_util.h"
|
||||
|
||||
VkResult
|
||||
vk_command_buffer_init(struct vk_command_pool *pool,
|
||||
struct vk_command_buffer *command_buffer,
|
||||
const struct vk_command_buffer_ops *ops,
|
||||
VkCommandBufferLevel level)
|
||||
{
|
||||
memset(command_buffer, 0, sizeof(*command_buffer));
|
||||
vk_object_base_init(pool->base.device, &command_buffer->base,
|
||||
VK_OBJECT_TYPE_COMMAND_BUFFER);
|
||||
|
||||
command_buffer->pool = pool;
|
||||
command_buffer->level = level;
|
||||
command_buffer->ops = ops;
|
||||
vk_dynamic_graphics_state_init(&command_buffer->dynamic_graphics_state);
|
||||
command_buffer->state = MESA_VK_COMMAND_BUFFER_STATE_INITIAL;
|
||||
command_buffer->record_result = VK_SUCCESS;
|
||||
vk_cmd_queue_init(&command_buffer->cmd_queue);
|
||||
vk_meta_object_list_init(&command_buffer->meta_objects);
|
||||
command_buffer->labels = UTIL_DYNARRAY_INIT;
|
||||
command_buffer->region_begin = true;
|
||||
|
||||
list_add(&command_buffer->pool_link, &pool->command_buffers);
|
||||
|
||||
return VK_SUCCESS;
|
||||
}
|
||||
|
||||
void
|
||||
vk_command_buffer_reset(struct vk_command_buffer *command_buffer)
|
||||
{
|
||||
vk_dynamic_graphics_state_clear(&command_buffer->dynamic_graphics_state);
|
||||
command_buffer->state = MESA_VK_COMMAND_BUFFER_STATE_INITIAL;
|
||||
command_buffer->record_result = VK_SUCCESS;
|
||||
vk_command_buffer_reset_render_pass(command_buffer);
|
||||
vk_cmd_queue_reset(&command_buffer->cmd_queue);
|
||||
vk_meta_object_list_reset(command_buffer->base.device,
|
||||
&command_buffer->meta_objects);
|
||||
util_dynarray_foreach (&command_buffer->labels, VkDebugUtilsLabelEXT, label)
|
||||
vk_free(&command_buffer->base.device->alloc, (void *)label->pLabelName);
|
||||
util_dynarray_clear(&command_buffer->labels);
|
||||
command_buffer->region_begin = true;
|
||||
}
|
||||
|
||||
void
|
||||
vk_command_buffer_begin(struct vk_command_buffer *command_buffer,
|
||||
const VkCommandBufferBeginInfo *pBeginInfo)
|
||||
{
|
||||
if (command_buffer->state != MESA_VK_COMMAND_BUFFER_STATE_INITIAL &&
|
||||
command_buffer->ops->reset != NULL)
|
||||
command_buffer->ops->reset(command_buffer, 0);
|
||||
|
||||
command_buffer->state = MESA_VK_COMMAND_BUFFER_STATE_RECORDING;
|
||||
}
|
||||
|
||||
VkResult
|
||||
vk_command_buffer_end(struct vk_command_buffer *command_buffer)
|
||||
{
|
||||
assert(command_buffer->state == MESA_VK_COMMAND_BUFFER_STATE_RECORDING);
|
||||
|
||||
if (vk_command_buffer_has_error(command_buffer))
|
||||
command_buffer->state = MESA_VK_COMMAND_BUFFER_STATE_INVALID;
|
||||
else
|
||||
command_buffer->state = MESA_VK_COMMAND_BUFFER_STATE_EXECUTABLE;
|
||||
|
||||
return vk_command_buffer_get_record_result(command_buffer);
|
||||
}
|
||||
|
||||
void
|
||||
vk_command_buffer_finish(struct vk_command_buffer *command_buffer)
|
||||
{
|
||||
list_del(&command_buffer->pool_link);
|
||||
vk_command_buffer_reset_render_pass(command_buffer);
|
||||
vk_cmd_queue_finish(&command_buffer->cmd_queue);
|
||||
util_dynarray_foreach (&command_buffer->labels, VkDebugUtilsLabelEXT, label)
|
||||
vk_free(&command_buffer->base.device->alloc, (void *)label->pLabelName);
|
||||
util_dynarray_fini(&command_buffer->labels);
|
||||
vk_meta_object_list_finish(command_buffer->base.device,
|
||||
&command_buffer->meta_objects);
|
||||
vk_object_base_finish(&command_buffer->base);
|
||||
}
|
||||
|
||||
void
|
||||
vk_command_buffer_recycle(struct vk_command_buffer *cmd_buffer)
|
||||
{
|
||||
/* Reset, returning resources to the pool. The command buffer object
|
||||
* itself will be recycled but, if the driver supports returning other
|
||||
* resources such as batch buffers to the pool, it should do so so they're
|
||||
* not tied up in recycled command buffer objects.
|
||||
*/
|
||||
cmd_buffer->ops->reset(cmd_buffer,
|
||||
VK_COMMAND_BUFFER_RESET_RELEASE_RESOURCES_BIT);
|
||||
|
||||
vk_object_base_recycle(&cmd_buffer->base);
|
||||
}
|
||||
|
||||
VKAPI_ATTR VkResult VKAPI_CALL
|
||||
vk_common_ResetCommandBuffer(VkCommandBuffer commandBuffer,
|
||||
VkCommandBufferResetFlags flags)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
|
||||
|
||||
if (cmd_buffer->state != MESA_VK_COMMAND_BUFFER_STATE_INITIAL)
|
||||
cmd_buffer->ops->reset(cmd_buffer, flags);
|
||||
|
||||
return VK_SUCCESS;
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdExecuteCommands(VkCommandBuffer commandBuffer,
|
||||
uint32_t commandBufferCount,
|
||||
const VkCommandBuffer *pCommandBuffers)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, primary, commandBuffer);
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
primary->base.device->command_dispatch_table;
|
||||
|
||||
for (uint32_t i = 0; i < commandBufferCount; i++) {
|
||||
VK_FROM_HANDLE(vk_command_buffer, secondary, pCommandBuffers[i]);
|
||||
|
||||
vk_cmd_queue_execute(&secondary->cmd_queue, commandBuffer, disp);
|
||||
}
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdBindVertexBuffers(VkCommandBuffer commandBuffer,
|
||||
uint32_t firstBinding,
|
||||
uint32_t bindingCount,
|
||||
const VkBuffer *pBuffers,
|
||||
const VkDeviceSize *pOffsets)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
|
||||
disp->CmdBindVertexBuffers2(commandBuffer, firstBinding, bindingCount,
|
||||
pBuffers, pOffsets, NULL, NULL);
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdBindIndexBuffer(
|
||||
VkCommandBuffer commandBuffer,
|
||||
VkBuffer buffer,
|
||||
VkDeviceSize offset,
|
||||
VkIndexType indexType)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
|
||||
disp->CmdBindIndexBuffer2KHR(commandBuffer, buffer, offset,
|
||||
VK_WHOLE_SIZE, indexType);
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdDispatch(VkCommandBuffer commandBuffer,
|
||||
uint32_t groupCountX,
|
||||
uint32_t groupCountY,
|
||||
uint32_t groupCountZ)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
|
||||
disp->CmdDispatchBase(commandBuffer, 0, 0, 0,
|
||||
groupCountX, groupCountY, groupCountZ);
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdSetDeviceMask(VkCommandBuffer commandBuffer, uint32_t deviceMask)
|
||||
{
|
||||
/* Nothing to do here since we only support a single device */
|
||||
assert(deviceMask == 0x1);
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdBindDescriptorSets(
|
||||
VkCommandBuffer commandBuffer,
|
||||
VkPipelineBindPoint pipelineBindPoint,
|
||||
VkPipelineLayout layout,
|
||||
uint32_t firstSet,
|
||||
uint32_t descriptorSetCount,
|
||||
const VkDescriptorSet* pDescriptorSets,
|
||||
uint32_t dynamicOffsetCount,
|
||||
const uint32_t* pDynamicOffsets)
|
||||
{
|
||||
const VkBindDescriptorSetsInfoKHR two = {
|
||||
.sType = VK_STRUCTURE_TYPE_BIND_DESCRIPTOR_SETS_INFO_KHR,
|
||||
.stageFlags = vk_shader_stages_from_bind_point(pipelineBindPoint),
|
||||
.layout = layout,
|
||||
.firstSet = firstSet,
|
||||
.descriptorSetCount = descriptorSetCount,
|
||||
.pDescriptorSets = pDescriptorSets,
|
||||
.dynamicOffsetCount = dynamicOffsetCount,
|
||||
.pDynamicOffsets = pDynamicOffsets
|
||||
};
|
||||
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
|
||||
disp->CmdBindDescriptorSets2KHR(commandBuffer, &two);
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdPushConstants(
|
||||
VkCommandBuffer commandBuffer,
|
||||
VkPipelineLayout layout,
|
||||
VkShaderStageFlags stageFlags,
|
||||
uint32_t offset,
|
||||
uint32_t size,
|
||||
const void* pValues)
|
||||
{
|
||||
const VkPushConstantsInfoKHR two = {
|
||||
.sType = VK_STRUCTURE_TYPE_PUSH_CONSTANTS_INFO_KHR,
|
||||
.layout = layout,
|
||||
.stageFlags = stageFlags,
|
||||
.offset = offset,
|
||||
.size = size,
|
||||
.pValues = pValues,
|
||||
};
|
||||
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
|
||||
disp->CmdPushConstants2KHR(commandBuffer, &two);
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdPushDescriptorSetKHR(
|
||||
VkCommandBuffer commandBuffer,
|
||||
VkPipelineBindPoint pipelineBindPoint,
|
||||
VkPipelineLayout layout,
|
||||
uint32_t set,
|
||||
uint32_t descriptorWriteCount,
|
||||
const VkWriteDescriptorSet* pDescriptorWrites)
|
||||
{
|
||||
const VkPushDescriptorSetInfoKHR two = {
|
||||
.sType = VK_STRUCTURE_TYPE_PUSH_CONSTANTS_INFO_KHR,
|
||||
.stageFlags = vk_shader_stages_from_bind_point(pipelineBindPoint),
|
||||
.layout = layout,
|
||||
.set = set,
|
||||
.descriptorWriteCount = descriptorWriteCount,
|
||||
.pDescriptorWrites = pDescriptorWrites,
|
||||
};
|
||||
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
|
||||
disp->CmdPushDescriptorSet2KHR(commandBuffer, &two);
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdPushDescriptorSetWithTemplateKHR(
|
||||
VkCommandBuffer commandBuffer,
|
||||
VkDescriptorUpdateTemplate descriptorUpdateTemplate,
|
||||
VkPipelineLayout layout,
|
||||
uint32_t set,
|
||||
const void* pData)
|
||||
{
|
||||
const VkPushDescriptorSetWithTemplateInfoKHR two = {
|
||||
.sType = VK_STRUCTURE_TYPE_PUSH_DESCRIPTOR_SET_WITH_TEMPLATE_INFO_KHR,
|
||||
.descriptorUpdateTemplate = descriptorUpdateTemplate,
|
||||
.layout = layout,
|
||||
.set = set,
|
||||
.pData = pData,
|
||||
};
|
||||
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
|
||||
disp->CmdPushDescriptorSetWithTemplate2KHR(commandBuffer, &two);
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdSetDescriptorBufferOffsetsEXT(
|
||||
VkCommandBuffer commandBuffer,
|
||||
VkPipelineBindPoint pipelineBindPoint,
|
||||
VkPipelineLayout layout,
|
||||
uint32_t firstSet,
|
||||
uint32_t setCount,
|
||||
const uint32_t* pBufferIndices,
|
||||
const VkDeviceSize* pOffsets)
|
||||
{
|
||||
const VkSetDescriptorBufferOffsetsInfoEXT two = {
|
||||
.sType = VK_STRUCTURE_TYPE_SET_DESCRIPTOR_BUFFER_OFFSETS_INFO_EXT,
|
||||
.stageFlags = vk_shader_stages_from_bind_point(pipelineBindPoint),
|
||||
.layout = layout,
|
||||
.firstSet = firstSet,
|
||||
.setCount = setCount,
|
||||
.pBufferIndices = pBufferIndices,
|
||||
.pOffsets = pOffsets
|
||||
};
|
||||
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
|
||||
disp->CmdSetDescriptorBufferOffsets2EXT(commandBuffer, &two);
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdBindDescriptorBufferEmbeddedSamplersEXT(
|
||||
VkCommandBuffer commandBuffer,
|
||||
VkPipelineBindPoint pipelineBindPoint,
|
||||
VkPipelineLayout layout,
|
||||
uint32_t set)
|
||||
{
|
||||
const VkBindDescriptorBufferEmbeddedSamplersInfoEXT two = {
|
||||
.sType = VK_STRUCTURE_TYPE_BIND_DESCRIPTOR_BUFFER_EMBEDDED_SAMPLERS_INFO_EXT,
|
||||
.stageFlags = vk_shader_stages_from_bind_point(pipelineBindPoint),
|
||||
.layout = layout,
|
||||
.set = set
|
||||
};
|
||||
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
|
||||
disp->CmdBindDescriptorBufferEmbeddedSamplers2EXT(commandBuffer, &two);
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdBindIndexBuffer2KHR(
|
||||
VkCommandBuffer commandBuffer,
|
||||
VkBuffer _buffer,
|
||||
VkDeviceSize offset,
|
||||
VkDeviceSize size,
|
||||
VkIndexType indexType)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
|
||||
VK_FROM_HANDLE(vk_buffer, buffer, _buffer);
|
||||
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
disp->CmdBindIndexBuffer3KHR(
|
||||
commandBuffer,
|
||||
&(VkBindIndexBuffer3InfoKHR) {
|
||||
.sType = VK_STRUCTURE_TYPE_BIND_INDEX_BUFFER_3_INFO_KHR,
|
||||
.addressRange = vk_device_address_range(
|
||||
buffer, offset, size),
|
||||
.addressFlags = buffer ? buffer->address_flags : 0,
|
||||
.indexType = indexType,
|
||||
});
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdBindVertexBuffers2EXT(
|
||||
VkCommandBuffer commandBuffer,
|
||||
uint32_t firstBinding,
|
||||
uint32_t bindingCount,
|
||||
const VkBuffer* pBuffers,
|
||||
const VkDeviceSize* pOffsets,
|
||||
const VkDeviceSize* pSizes,
|
||||
const VkDeviceSize* pStrides)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
|
||||
|
||||
STACK_ARRAY(VkBindVertexBuffer3InfoKHR, bindings, bindingCount);
|
||||
|
||||
for (uint32_t b = 0; b < bindingCount; b++) {
|
||||
VK_FROM_HANDLE(vk_buffer, buffer, pBuffers[b]);
|
||||
|
||||
bindings[b] = (VkBindVertexBuffer3InfoKHR) {
|
||||
.sType = VK_STRUCTURE_TYPE_BIND_VERTEX_BUFFER_3_INFO_KHR,
|
||||
.addressRange = vk_strided_device_address_range(
|
||||
buffer, pOffsets[b],
|
||||
pSizes != NULL ? pSizes[b] : VK_WHOLE_SIZE,
|
||||
pStrides != NULL ? pStrides[b] : 0),
|
||||
.addressFlags = buffer ? buffer->address_flags : 0,
|
||||
.setStride = pStrides != NULL,
|
||||
};
|
||||
}
|
||||
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
disp->CmdBindVertexBuffers3KHR(
|
||||
commandBuffer, firstBinding, bindingCount,
|
||||
bindingCount > 0 ? bindings : NULL);
|
||||
|
||||
STACK_ARRAY_FINISH(bindings);
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdDrawIndirect(
|
||||
VkCommandBuffer commandBuffer,
|
||||
VkBuffer _buffer,
|
||||
VkDeviceSize offset,
|
||||
uint32_t drawCount,
|
||||
uint32_t stride)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
|
||||
VK_FROM_HANDLE(vk_buffer, buffer, _buffer);
|
||||
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
disp->CmdDrawIndirect2KHR(commandBuffer,
|
||||
&(VkDrawIndirect2InfoKHR) {
|
||||
.sType = VK_STRUCTURE_TYPE_DRAW_INDIRECT_2_INFO_KHR,
|
||||
.addressRange = vk_strided_device_address_range(
|
||||
buffer, offset, VK_WHOLE_SIZE, stride),
|
||||
.addressFlags = buffer->address_flags,
|
||||
.drawCount = drawCount,
|
||||
});
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdDrawIndexedIndirect(
|
||||
VkCommandBuffer commandBuffer,
|
||||
VkBuffer _buffer,
|
||||
VkDeviceSize offset,
|
||||
uint32_t drawCount,
|
||||
uint32_t stride)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
|
||||
VK_FROM_HANDLE(vk_buffer, buffer, _buffer);
|
||||
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
disp->CmdDrawIndexedIndirect2KHR(
|
||||
commandBuffer,
|
||||
&(VkDrawIndirect2InfoKHR) {
|
||||
.sType = VK_STRUCTURE_TYPE_DRAW_INDIRECT_2_INFO_KHR,
|
||||
.addressRange = vk_strided_device_address_range(
|
||||
buffer, offset, VK_WHOLE_SIZE, stride),
|
||||
.addressFlags = buffer->address_flags,
|
||||
.drawCount = drawCount,
|
||||
});
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdDrawIndirectCount(
|
||||
VkCommandBuffer commandBuffer,
|
||||
VkBuffer _buffer,
|
||||
VkDeviceSize offset,
|
||||
VkBuffer countBuffer,
|
||||
VkDeviceSize countBufferOffset,
|
||||
uint32_t maxDrawCount,
|
||||
uint32_t stride)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
|
||||
VK_FROM_HANDLE(vk_buffer, buffer, _buffer);
|
||||
VK_FROM_HANDLE(vk_buffer, count_buffer, countBuffer);
|
||||
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
disp->CmdDrawIndirectCount2KHR(
|
||||
commandBuffer,
|
||||
&(VkDrawIndirectCount2InfoKHR) {
|
||||
.sType = VK_STRUCTURE_TYPE_DRAW_INDIRECT_COUNT_2_INFO_KHR,
|
||||
.addressRange = vk_strided_device_address_range(
|
||||
buffer, offset, VK_WHOLE_SIZE, stride),
|
||||
.addressFlags = buffer->address_flags,
|
||||
.countAddressRange = vk_device_address_range(
|
||||
count_buffer, countBufferOffset, VK_WHOLE_SIZE),
|
||||
.countAddressFlags = count_buffer->address_flags,
|
||||
.maxDrawCount = maxDrawCount,
|
||||
});
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdDrawIndexedIndirectCount(
|
||||
VkCommandBuffer commandBuffer,
|
||||
VkBuffer _buffer,
|
||||
VkDeviceSize offset,
|
||||
VkBuffer countBuffer,
|
||||
VkDeviceSize countBufferOffset,
|
||||
uint32_t maxDrawCount,
|
||||
uint32_t stride)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
|
||||
VK_FROM_HANDLE(vk_buffer, buffer, _buffer);
|
||||
VK_FROM_HANDLE(vk_buffer, count_buffer, countBuffer);
|
||||
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
disp->CmdDrawIndexedIndirectCount2KHR(
|
||||
commandBuffer,
|
||||
&(VkDrawIndirectCount2InfoKHR) {
|
||||
.sType = VK_STRUCTURE_TYPE_DRAW_INDIRECT_COUNT_2_INFO_KHR,
|
||||
.addressRange = vk_strided_device_address_range(
|
||||
buffer, offset, VK_WHOLE_SIZE, stride),
|
||||
.addressFlags = buffer->address_flags,
|
||||
.countAddressRange = vk_device_address_range(
|
||||
count_buffer, countBufferOffset, VK_WHOLE_SIZE),
|
||||
.countAddressFlags = count_buffer->address_flags,
|
||||
.maxDrawCount = maxDrawCount,
|
||||
});
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdBeginConditionalRenderingEXT(
|
||||
VkCommandBuffer commandBuffer,
|
||||
const VkConditionalRenderingBeginInfoEXT* pConditionalRenderingBegin)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
|
||||
VK_FROM_HANDLE(vk_buffer, buffer, pConditionalRenderingBegin->buffer);
|
||||
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
disp->CmdBeginConditionalRendering2EXT(
|
||||
commandBuffer,
|
||||
&(VkConditionalRenderingBeginInfo2EXT) {
|
||||
.sType = VK_STRUCTURE_TYPE_CONDITIONAL_RENDERING_BEGIN_INFO_2_EXT,
|
||||
.addressRange = vk_device_address_range(
|
||||
buffer, pConditionalRenderingBegin->offset, VK_WHOLE_SIZE),
|
||||
.flags = pConditionalRenderingBegin->flags,
|
||||
});
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdBindTransformFeedbackBuffersEXT(
|
||||
VkCommandBuffer commandBuffer,
|
||||
uint32_t firstBinding,
|
||||
uint32_t bindingCount,
|
||||
const VkBuffer* pBuffers,
|
||||
const VkDeviceSize* pOffsets,
|
||||
const VkDeviceSize* pSizes)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
|
||||
|
||||
STACK_ARRAY(VkBindTransformFeedbackBuffer2InfoEXT, bindings, bindingCount);
|
||||
|
||||
for (uint32_t b = 0; b < bindingCount; b++) {
|
||||
VK_FROM_HANDLE(vk_buffer, buffer, pBuffers[b]);
|
||||
|
||||
bindings[b] = (VkBindTransformFeedbackBuffer2InfoEXT) {
|
||||
.sType = VK_STRUCTURE_TYPE_BIND_TRANSFORM_FEEDBACK_BUFFER_2_INFO_EXT,
|
||||
.addressRange = vk_device_address_range(
|
||||
buffer, pOffsets ? pOffsets[b] : 0,
|
||||
pSizes ? pSizes[b] : VK_WHOLE_SIZE),
|
||||
.addressFlags = buffer ? buffer->address_flags : 0,
|
||||
};
|
||||
}
|
||||
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
disp->CmdBindTransformFeedbackBuffers2EXT(
|
||||
commandBuffer, firstBinding, bindingCount,
|
||||
bindingCount > 0 ? bindings : NULL);
|
||||
|
||||
STACK_ARRAY_FINISH(bindings);
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdBeginTransformFeedbackEXT(
|
||||
VkCommandBuffer commandBuffer,
|
||||
uint32_t firstCounterBuffer,
|
||||
uint32_t counterBufferCount,
|
||||
const VkBuffer* pCounterBuffers,
|
||||
const VkDeviceSize* pCounterBufferOffsets)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
|
||||
|
||||
STACK_ARRAY(VkBindTransformFeedbackBuffer2InfoEXT, buffers, counterBufferCount);
|
||||
|
||||
for (uint32_t b = 0; pCounterBuffers && b < counterBufferCount; b++) {
|
||||
VK_FROM_HANDLE(vk_buffer, buffer, pCounterBuffers[b]);
|
||||
|
||||
buffers[b] = (VkBindTransformFeedbackBuffer2InfoEXT) {
|
||||
.sType = VK_STRUCTURE_TYPE_BIND_TRANSFORM_FEEDBACK_BUFFER_2_INFO_EXT,
|
||||
.addressRange = vk_device_address_range(
|
||||
buffer, pCounterBufferOffsets ? pCounterBufferOffsets[b] : 0,
|
||||
VK_WHOLE_SIZE),
|
||||
.addressFlags = buffer ? buffer->address_flags : 0,
|
||||
};
|
||||
}
|
||||
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
disp->CmdBeginTransformFeedback2EXT(
|
||||
commandBuffer, firstCounterBuffer, counterBufferCount,
|
||||
counterBufferCount > 0 && pCounterBuffers ? buffers : NULL);
|
||||
|
||||
STACK_ARRAY_FINISH(buffers);
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdEndTransformFeedbackEXT(
|
||||
VkCommandBuffer commandBuffer,
|
||||
uint32_t firstCounterBuffer,
|
||||
uint32_t counterBufferCount,
|
||||
const VkBuffer* pCounterBuffers,
|
||||
const VkDeviceSize* pCounterBufferOffsets)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
|
||||
|
||||
STACK_ARRAY(VkBindTransformFeedbackBuffer2InfoEXT, buffers, counterBufferCount);
|
||||
|
||||
for (uint32_t b = 0; pCounterBuffers && b < counterBufferCount; b++) {
|
||||
VK_FROM_HANDLE(vk_buffer, buffer, pCounterBuffers[b]);
|
||||
|
||||
buffers[b] = (VkBindTransformFeedbackBuffer2InfoEXT) {
|
||||
.sType = VK_STRUCTURE_TYPE_BIND_TRANSFORM_FEEDBACK_BUFFER_2_INFO_EXT,
|
||||
.addressRange = vk_device_address_range(
|
||||
buffer, pCounterBufferOffsets ? pCounterBufferOffsets[b] : 0,
|
||||
VK_WHOLE_SIZE),
|
||||
.addressFlags = buffer ? buffer->address_flags : 0,
|
||||
};
|
||||
}
|
||||
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
disp->CmdEndTransformFeedback2EXT(
|
||||
commandBuffer, firstCounterBuffer, counterBufferCount,
|
||||
counterBufferCount > 0 && pCounterBuffers ? buffers : NULL);
|
||||
|
||||
STACK_ARRAY_FINISH(buffers);
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdDrawIndirectByteCountEXT(
|
||||
VkCommandBuffer commandBuffer,
|
||||
uint32_t instanceCount,
|
||||
uint32_t firstInstance,
|
||||
VkBuffer counterBuffer,
|
||||
VkDeviceSize counterBufferOffset,
|
||||
uint32_t counterOffset,
|
||||
uint32_t vertexStride)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
|
||||
VK_FROM_HANDLE(vk_buffer, buffer, counterBuffer);
|
||||
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
disp->CmdDrawIndirectByteCount2EXT(
|
||||
commandBuffer, instanceCount, firstInstance,
|
||||
&(VkBindTransformFeedbackBuffer2InfoEXT) {
|
||||
.addressRange = vk_device_address_range(
|
||||
buffer, counterBufferOffset, VK_WHOLE_SIZE),
|
||||
.addressFlags = buffer->address_flags,
|
||||
},
|
||||
counterOffset,
|
||||
vertexStride);
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdDrawMeshTasksIndirectEXT(
|
||||
VkCommandBuffer commandBuffer,
|
||||
VkBuffer _buffer,
|
||||
VkDeviceSize offset,
|
||||
uint32_t drawCount,
|
||||
uint32_t stride)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
|
||||
VK_FROM_HANDLE(vk_buffer, buffer, _buffer);
|
||||
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
disp->CmdDrawMeshTasksIndirect2EXT(
|
||||
commandBuffer,
|
||||
&(VkDrawIndirect2InfoKHR) {
|
||||
.addressRange = vk_strided_device_address_range(
|
||||
buffer, offset, VK_WHOLE_SIZE, stride),
|
||||
.addressFlags = buffer->address_flags,
|
||||
.drawCount = drawCount,
|
||||
});
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdDrawMeshTasksIndirectCountEXT(
|
||||
VkCommandBuffer commandBuffer,
|
||||
VkBuffer _buffer,
|
||||
VkDeviceSize offset,
|
||||
VkBuffer countBuffer,
|
||||
VkDeviceSize countBufferOffset,
|
||||
uint32_t maxDrawCount,
|
||||
uint32_t stride)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
|
||||
VK_FROM_HANDLE(vk_buffer, buffer, _buffer);
|
||||
VK_FROM_HANDLE(vk_buffer, count_buffer, countBuffer);
|
||||
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
disp->CmdDrawMeshTasksIndirectCount2EXT(
|
||||
commandBuffer,
|
||||
&(VkDrawIndirectCount2InfoKHR) {
|
||||
.addressRange = vk_strided_device_address_range(
|
||||
buffer, offset, VK_WHOLE_SIZE, stride),
|
||||
.addressFlags = buffer->address_flags,
|
||||
.countAddressRange = vk_device_address_range(
|
||||
count_buffer, countBufferOffset, VK_WHOLE_SIZE),
|
||||
.countAddressFlags = count_buffer->address_flags,
|
||||
.maxDrawCount = maxDrawCount,
|
||||
});
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdWriteBufferMarker2AMD(
|
||||
VkCommandBuffer commandBuffer,
|
||||
VkPipelineStageFlags2 stage,
|
||||
VkBuffer dstBuffer,
|
||||
VkDeviceSize dstOffset,
|
||||
uint32_t marker)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
|
||||
VK_FROM_HANDLE(vk_buffer, buffer, dstBuffer);
|
||||
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
disp->CmdWriteMarkerToMemoryAMD(
|
||||
commandBuffer,
|
||||
&(VkMemoryMarkerInfoAMD) {
|
||||
.sType = VK_STRUCTURE_TYPE_MEMORY_MARKER_INFO_AMD,
|
||||
.stage = stage,
|
||||
.dstRange = vk_device_address_range(
|
||||
buffer, dstOffset, VK_WHOLE_SIZE),
|
||||
.dstFlags = buffer->address_flags,
|
||||
.marker = marker,
|
||||
});
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdDispatchIndirect(
|
||||
VkCommandBuffer commandBuffer,
|
||||
VkBuffer _buffer,
|
||||
VkDeviceSize offset)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
|
||||
VK_FROM_HANDLE(vk_buffer, buffer, _buffer);
|
||||
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
disp->CmdDispatchIndirect2KHR(
|
||||
commandBuffer,
|
||||
&(VkDispatchIndirect2InfoKHR) {
|
||||
.sType = VK_STRUCTURE_TYPE_DISPATCH_INDIRECT_2_INFO_KHR,
|
||||
.addressRange = vk_device_address_range(
|
||||
buffer, offset, VK_WHOLE_SIZE),
|
||||
.addressFlags = buffer->address_flags,
|
||||
});
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdCopyBuffer2(
|
||||
VkCommandBuffer commandBuffer,
|
||||
const VkCopyBufferInfo2* pCopyBufferInfo)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
|
||||
VK_FROM_HANDLE(vk_buffer, src_buffer, pCopyBufferInfo->srcBuffer);
|
||||
VK_FROM_HANDLE(vk_buffer, dst_buffer, pCopyBufferInfo->dstBuffer);
|
||||
|
||||
STACK_ARRAY(VkDeviceMemoryCopyKHR, regions, pCopyBufferInfo->regionCount);
|
||||
|
||||
for (uint32_t r = 0; r < pCopyBufferInfo->regionCount; r++) {
|
||||
regions[r] = (VkDeviceMemoryCopyKHR) {
|
||||
.srcRange = vk_device_address_range(
|
||||
src_buffer,
|
||||
pCopyBufferInfo->pRegions[r].srcOffset,
|
||||
pCopyBufferInfo->pRegions[r].size),
|
||||
.srcFlags = src_buffer->address_flags,
|
||||
.dstRange = vk_device_address_range(
|
||||
dst_buffer,
|
||||
pCopyBufferInfo->pRegions[r].dstOffset,
|
||||
pCopyBufferInfo->pRegions[r].size),
|
||||
.dstFlags = dst_buffer->address_flags,
|
||||
};
|
||||
}
|
||||
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
disp->CmdCopyMemoryKHR(
|
||||
commandBuffer,
|
||||
&(VkCopyDeviceMemoryInfoKHR) {
|
||||
.sType = VK_STRUCTURE_TYPE_COPY_DEVICE_MEMORY_INFO_KHR,
|
||||
.regionCount = pCopyBufferInfo->regionCount,
|
||||
.pRegions = pCopyBufferInfo->regionCount > 0 ? regions : NULL,
|
||||
});
|
||||
|
||||
STACK_ARRAY_FINISH(regions);
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdCopyBufferToImage2(
|
||||
VkCommandBuffer commandBuffer,
|
||||
const VkCopyBufferToImageInfo2* pCopyBufferToImageInfo)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
|
||||
VK_FROM_HANDLE(vk_buffer, buffer, pCopyBufferToImageInfo->srcBuffer);
|
||||
|
||||
STACK_ARRAY(VkDeviceMemoryImageCopyKHR, regions,
|
||||
pCopyBufferToImageInfo->regionCount);
|
||||
|
||||
for (uint32_t r = 0; r < pCopyBufferToImageInfo->regionCount; r++) {
|
||||
regions[r] = (VkDeviceMemoryImageCopyKHR) {
|
||||
.sType = VK_STRUCTURE_TYPE_DEVICE_MEMORY_IMAGE_COPY_KHR,
|
||||
.addressRange = vk_device_address_range(
|
||||
buffer, pCopyBufferToImageInfo->pRegions[r].bufferOffset,
|
||||
VK_WHOLE_SIZE),
|
||||
.addressFlags = buffer->address_flags,
|
||||
.addressRowLength = pCopyBufferToImageInfo->pRegions[r].bufferRowLength,
|
||||
.addressImageHeight = pCopyBufferToImageInfo->pRegions[r].bufferImageHeight,
|
||||
.imageSubresource = pCopyBufferToImageInfo->pRegions[r].imageSubresource,
|
||||
.imageLayout = pCopyBufferToImageInfo->dstImageLayout,
|
||||
.imageOffset = pCopyBufferToImageInfo->pRegions[r].imageOffset,
|
||||
.imageExtent = pCopyBufferToImageInfo->pRegions[r].imageExtent,
|
||||
};
|
||||
}
|
||||
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
disp->CmdCopyMemoryToImageKHR(
|
||||
commandBuffer,
|
||||
&(VkCopyDeviceMemoryImageInfoKHR) {
|
||||
.sType = VK_STRUCTURE_TYPE_COPY_DEVICE_MEMORY_IMAGE_INFO_KHR,
|
||||
.image = pCopyBufferToImageInfo->dstImage,
|
||||
.regionCount = pCopyBufferToImageInfo->regionCount,
|
||||
.pRegions = pCopyBufferToImageInfo->regionCount > 0 ? regions : NULL,
|
||||
});
|
||||
|
||||
STACK_ARRAY_FINISH(regions);
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdCopyImageToBuffer2(
|
||||
VkCommandBuffer commandBuffer,
|
||||
const VkCopyImageToBufferInfo2* pCopyImageToBufferInfo)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
|
||||
VK_FROM_HANDLE(vk_buffer, buffer, pCopyImageToBufferInfo->dstBuffer);
|
||||
|
||||
STACK_ARRAY(VkDeviceMemoryImageCopyKHR, regions,
|
||||
pCopyImageToBufferInfo->regionCount);
|
||||
|
||||
for (uint32_t r = 0; r < pCopyImageToBufferInfo->regionCount; r++) {
|
||||
regions[r] = (VkDeviceMemoryImageCopyKHR) {
|
||||
.sType = VK_STRUCTURE_TYPE_DEVICE_MEMORY_IMAGE_COPY_KHR,
|
||||
.addressRange = vk_device_address_range(
|
||||
buffer, pCopyImageToBufferInfo->pRegions[r].bufferOffset,
|
||||
VK_WHOLE_SIZE),
|
||||
.addressFlags = buffer->address_flags,
|
||||
.addressRowLength = pCopyImageToBufferInfo->pRegions[r].bufferRowLength,
|
||||
.addressImageHeight = pCopyImageToBufferInfo->pRegions[r].bufferImageHeight,
|
||||
.imageSubresource = pCopyImageToBufferInfo->pRegions[r].imageSubresource,
|
||||
.imageLayout = pCopyImageToBufferInfo->srcImageLayout,
|
||||
.imageOffset = pCopyImageToBufferInfo->pRegions[r].imageOffset,
|
||||
.imageExtent = pCopyImageToBufferInfo->pRegions[r].imageExtent,
|
||||
};
|
||||
}
|
||||
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
disp->CmdCopyImageToMemoryKHR(
|
||||
commandBuffer,
|
||||
&(VkCopyDeviceMemoryImageInfoKHR) {
|
||||
.sType = VK_STRUCTURE_TYPE_COPY_DEVICE_MEMORY_IMAGE_INFO_KHR,
|
||||
.image = pCopyImageToBufferInfo->srcImage,
|
||||
.regionCount = pCopyImageToBufferInfo->regionCount,
|
||||
.pRegions = pCopyImageToBufferInfo->regionCount > 0 ? regions : NULL,
|
||||
});
|
||||
|
||||
STACK_ARRAY_FINISH(regions);
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdUpdateBuffer(
|
||||
VkCommandBuffer commandBuffer,
|
||||
VkBuffer dstBuffer,
|
||||
VkDeviceSize dstOffset,
|
||||
VkDeviceSize dataSize,
|
||||
const void* pData)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
|
||||
VK_FROM_HANDLE(vk_buffer, buffer, dstBuffer);
|
||||
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
const VkDeviceAddressRangeKHR addr_range =
|
||||
vk_device_address_range(buffer, dstOffset, dataSize);
|
||||
disp->CmdUpdateMemoryKHR(commandBuffer, &addr_range,
|
||||
buffer->address_flags, dataSize, pData);
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdFillBuffer(
|
||||
VkCommandBuffer commandBuffer,
|
||||
VkBuffer dstBuffer,
|
||||
VkDeviceSize dstOffset,
|
||||
VkDeviceSize size,
|
||||
uint32_t data)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
|
||||
VK_FROM_HANDLE(vk_buffer, buffer, dstBuffer);
|
||||
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
const VkDeviceAddressRangeKHR addr_range =
|
||||
vk_device_address_range(buffer, dstOffset, size);
|
||||
disp->CmdFillMemoryKHR(commandBuffer, &addr_range,
|
||||
buffer->address_flags, data);
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdCopyQueryPoolResults(
|
||||
VkCommandBuffer commandBuffer,
|
||||
VkQueryPool queryPool,
|
||||
uint32_t firstQuery,
|
||||
uint32_t queryCount,
|
||||
VkBuffer dstBuffer,
|
||||
VkDeviceSize dstOffset,
|
||||
VkDeviceSize stride,
|
||||
VkQueryResultFlags flags)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
|
||||
VK_FROM_HANDLE(vk_buffer, buffer, dstBuffer);
|
||||
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&cmd_buffer->base.device->dispatch_table;
|
||||
|
||||
const VkStridedDeviceAddressRangeKHR addr_range =
|
||||
vk_strided_device_address_range(
|
||||
buffer, dstOffset, VK_WHOLE_SIZE, stride);
|
||||
|
||||
disp->CmdCopyQueryPoolResultsToMemoryKHR(commandBuffer, queryPool,
|
||||
firstQuery, queryCount,
|
||||
&addr_range,
|
||||
buffer->address_flags, flags);
|
||||
}
|
||||
@@ -0,0 +1,249 @@
|
||||
/*
|
||||
* Copyright © 2021 Intel Corporation
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#ifndef VK_COMMAND_BUFFER_H
|
||||
#define VK_COMMAND_BUFFER_H
|
||||
|
||||
#include "vk_cmd_queue.h"
|
||||
#include "vk_graphics_state.h"
|
||||
#include "vk_log.h"
|
||||
#include "vk_meta_object_list.h"
|
||||
#include "vk_object.h"
|
||||
#include "util/list.h"
|
||||
#include "util/u_dynarray.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
struct vk_command_pool;
|
||||
struct vk_framebuffer;
|
||||
struct vk_image_view;
|
||||
struct vk_render_pass;
|
||||
|
||||
struct vk_attachment_view_state {
|
||||
VkImageLayout layout;
|
||||
VkImageLayout stencil_layout;
|
||||
const VkSampleLocationsInfoEXT *sample_locations;
|
||||
};
|
||||
|
||||
struct vk_attachment_state {
|
||||
struct vk_image_view *image_view;
|
||||
|
||||
/** A running tally of which views have been loaded */
|
||||
uint32_t views_loaded;
|
||||
|
||||
/** Per-view state */
|
||||
struct vk_attachment_view_state views[MESA_VK_MAX_MULTIVIEW_VIEW_COUNT];
|
||||
|
||||
/** VkRenderPassBeginInfo::pClearValues[i] */
|
||||
VkClearValue clear_value;
|
||||
};
|
||||
|
||||
/** Command buffer ops */
|
||||
struct vk_command_buffer_ops {
|
||||
/** Creates a command buffer
|
||||
*
|
||||
* Used by the common command pool implementation. This function MUST
|
||||
* call `vk_command_buffer_finish()`. Notably, this function does not
|
||||
* receive any additional parameters such as the level. The level will be
|
||||
* set by `vk_common_AllocateCommandBuffers()` and the driver must not rely
|
||||
* on it until `vkBeginCommandBuffer()` time.
|
||||
*/
|
||||
VkResult (*create)(struct vk_command_pool *, VkCommandBufferLevel,
|
||||
struct vk_command_buffer **);
|
||||
|
||||
/** Resets the command buffer
|
||||
*
|
||||
* Used by the common command pool implementation. This function MUST
|
||||
* call `vk_command_buffer_reset()`. Unlike `vkResetCommandBuffer()`,
|
||||
* this function does not have a return value because it may be called on
|
||||
* destruction paths.
|
||||
*/
|
||||
void (*reset)(struct vk_command_buffer *, VkCommandBufferResetFlags);
|
||||
|
||||
/** Destroys the command buffer
|
||||
*
|
||||
* Used by the common command pool implementation. This function MUST
|
||||
* call `vk_command_buffer_finish()`.
|
||||
*/
|
||||
void (*destroy)(struct vk_command_buffer *);
|
||||
};
|
||||
|
||||
enum mesa_vk_command_buffer_state {
|
||||
MESA_VK_COMMAND_BUFFER_STATE_INVALID,
|
||||
MESA_VK_COMMAND_BUFFER_STATE_INITIAL,
|
||||
MESA_VK_COMMAND_BUFFER_STATE_RECORDING,
|
||||
MESA_VK_COMMAND_BUFFER_STATE_EXECUTABLE,
|
||||
MESA_VK_COMMAND_BUFFER_STATE_PENDING,
|
||||
};
|
||||
|
||||
struct vk_command_buffer {
|
||||
struct vk_object_base base;
|
||||
|
||||
struct vk_command_pool *pool;
|
||||
|
||||
/** VkCommandBufferAllocateInfo::level */
|
||||
VkCommandBufferLevel level;
|
||||
|
||||
const struct vk_command_buffer_ops *ops;
|
||||
|
||||
struct vk_dynamic_graphics_state dynamic_graphics_state;
|
||||
|
||||
/** State of the command buffer */
|
||||
enum mesa_vk_command_buffer_state state;
|
||||
|
||||
/** Command buffer recording error state. */
|
||||
VkResult record_result;
|
||||
|
||||
/** Link in vk_command_pool::command_buffers if pool != NULL */
|
||||
struct list_head pool_link;
|
||||
|
||||
/** Command list for emulated secondary command buffers */
|
||||
struct vk_cmd_queue cmd_queue;
|
||||
|
||||
/** Object list for meta objects */
|
||||
struct vk_meta_object_list meta_objects;
|
||||
|
||||
/**
|
||||
* VK_EXT_debug_utils
|
||||
*
|
||||
* The next two fields represent debug labels storage.
|
||||
*
|
||||
* VK_EXT_debug_utils spec requires that upon triggering a debug message
|
||||
* with a command buffer attached to it, all "active" labels will also be
|
||||
* provided to the callback. The spec describes two distinct ways of
|
||||
* attaching a debug label to the command buffer: opening a label region
|
||||
* and inserting a single label.
|
||||
*
|
||||
* Label region is active between the corresponding `*BeginDebugUtilsLabel`
|
||||
* and `*EndDebugUtilsLabel` calls. The spec doesn't mention any limits on
|
||||
* nestedness of label regions. This implementation assumes that there
|
||||
* aren't any.
|
||||
*
|
||||
* The spec, however, doesn't explain the lifetime of a label submitted by
|
||||
* an `*InsertDebugUtilsLabel` call. The LunarG whitepaper [1] (pp 12-15)
|
||||
* provides a more detailed explanation along with some examples. According
|
||||
* to those, such label remains active until the next `*DebugUtilsLabel`
|
||||
* call. This means that there can be no more than one such label at a
|
||||
* time.
|
||||
*
|
||||
* ``labels`` contains all active labels at this point in order of
|
||||
* submission ``region_begin`` denotes whether the most recent label opens
|
||||
* a new region If ``labels`` is empty ``region_begin`` must be true.
|
||||
*
|
||||
* Anytime we modify labels, we first check for ``region_begin``. If it's
|
||||
* false, it means that the most recent label was submitted by
|
||||
* `*InsertDebugUtilsLabel` and we need to remove it before doing anything
|
||||
* else.
|
||||
*
|
||||
* See the discussion here:
|
||||
* https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/10318#note_1061317
|
||||
*
|
||||
* [1] https://www.lunarg.com/wp-content/uploads/2018/05/Vulkan-Debug-Utils_05_18_v1.pdf
|
||||
*/
|
||||
struct util_dynarray labels;
|
||||
bool region_begin;
|
||||
|
||||
struct vk_render_pass *render_pass;
|
||||
uint32_t subpass_idx;
|
||||
struct vk_framebuffer *framebuffer;
|
||||
VkRect2D render_area;
|
||||
|
||||
/**
|
||||
* True if we are currently inside a CmdPipelineBarrier() is inserted by
|
||||
* the runtime's vk_render_pass.c
|
||||
*/
|
||||
bool runtime_rp_barrier;
|
||||
|
||||
/* This uses the same trick as STACK_ARRAY */
|
||||
struct vk_attachment_state *attachments;
|
||||
struct vk_attachment_state _attachments[8];
|
||||
|
||||
VkRenderPassSampleLocationsBeginInfoEXT *pass_sample_locations;
|
||||
|
||||
/**
|
||||
* Bitmask of shader stages bound via a vk_pipeline since the last call to
|
||||
* vkBindShadersEXT().
|
||||
*
|
||||
* Used by the common vk_pipeline implementation
|
||||
*/
|
||||
VkShaderStageFlags pipeline_shader_stages;
|
||||
};
|
||||
|
||||
VK_DEFINE_HANDLE_CASTS(vk_command_buffer, base, VkCommandBuffer,
|
||||
VK_OBJECT_TYPE_COMMAND_BUFFER)
|
||||
|
||||
VkResult MUST_CHECK
|
||||
vk_command_buffer_init(struct vk_command_pool *pool,
|
||||
struct vk_command_buffer *command_buffer,
|
||||
const struct vk_command_buffer_ops *ops,
|
||||
VkCommandBufferLevel level);
|
||||
|
||||
void
|
||||
vk_command_buffer_reset_render_pass(struct vk_command_buffer *cmd_buffer);
|
||||
|
||||
void
|
||||
vk_command_buffer_reset(struct vk_command_buffer *command_buffer);
|
||||
|
||||
void
|
||||
vk_command_buffer_recycle(struct vk_command_buffer *command_buffer);
|
||||
|
||||
void
|
||||
vk_command_buffer_begin(struct vk_command_buffer *command_buffer,
|
||||
const VkCommandBufferBeginInfo *pBeginInfo);
|
||||
|
||||
VkResult
|
||||
vk_command_buffer_end(struct vk_command_buffer *command_buffer);
|
||||
|
||||
void
|
||||
vk_command_buffer_finish(struct vk_command_buffer *command_buffer);
|
||||
|
||||
static inline VkResult
|
||||
__vk_command_buffer_set_error(struct vk_command_buffer *command_buffer,
|
||||
VkResult error, const char *file, int line)
|
||||
{
|
||||
assert(error != VK_SUCCESS);
|
||||
error = __vk_errorf(command_buffer, error, file, line, NULL);
|
||||
if (command_buffer->record_result == VK_SUCCESS)
|
||||
command_buffer->record_result = error;
|
||||
return error;
|
||||
}
|
||||
|
||||
#define vk_command_buffer_set_error(command_buffer, error) \
|
||||
__vk_command_buffer_set_error(command_buffer, error, __FILE__, __LINE__)
|
||||
|
||||
static inline VkResult
|
||||
vk_command_buffer_get_record_result(struct vk_command_buffer *command_buffer)
|
||||
{
|
||||
return command_buffer->record_result;
|
||||
}
|
||||
|
||||
#define vk_command_buffer_has_error(command_buffer) \
|
||||
unlikely((command_buffer)->record_result != VK_SUCCESS)
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* VK_COMMAND_BUFFER_H */
|
||||
@@ -0,0 +1,274 @@
|
||||
/*
|
||||
* Copyright © 2015 Intel Corporation
|
||||
* Copyright © 2022 Collabora, Ltd
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "vk_command_pool.h"
|
||||
|
||||
#include "vk_alloc.h"
|
||||
#include "vk_command_buffer.h"
|
||||
#include "vk_common_entrypoints.h"
|
||||
#include "vk_device.h"
|
||||
#include "vk_log.h"
|
||||
|
||||
static bool
|
||||
should_recycle_command_buffers(struct vk_device *device)
|
||||
{
|
||||
/* They have to be using the common allocation implementation, otherwise
|
||||
* the recycled command buffers will never actually get re-used
|
||||
*/
|
||||
const struct vk_device_dispatch_table *disp = &device->dispatch_table;
|
||||
if (disp->AllocateCommandBuffers != vk_common_AllocateCommandBuffers)
|
||||
return false;
|
||||
|
||||
/* We need to be able to reset command buffers */
|
||||
if (device->command_buffer_ops->reset == NULL)
|
||||
return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static void
|
||||
destroy_free_command_buffers(struct vk_command_pool *pool)
|
||||
{
|
||||
for (uint32_t i = 0; i < ARRAY_SIZE(pool->free_command_buffers); i++) {
|
||||
list_for_each_entry_safe(struct vk_command_buffer, cmd_buffer,
|
||||
&pool->free_command_buffers[i], pool_link) {
|
||||
cmd_buffer->ops->destroy(cmd_buffer);
|
||||
}
|
||||
assert(list_is_empty(&pool->free_command_buffers[i]));
|
||||
}
|
||||
}
|
||||
|
||||
VkResult MUST_CHECK
|
||||
vk_command_pool_init(struct vk_device *device,
|
||||
struct vk_command_pool *pool,
|
||||
const VkCommandPoolCreateInfo *pCreateInfo,
|
||||
const VkAllocationCallbacks *pAllocator)
|
||||
{
|
||||
memset(pool, 0, sizeof(*pool));
|
||||
vk_object_base_init(device, &pool->base,
|
||||
VK_OBJECT_TYPE_COMMAND_POOL);
|
||||
|
||||
pool->flags = pCreateInfo->flags;
|
||||
pool->queue_family_index = pCreateInfo->queueFamilyIndex;
|
||||
pool->alloc = pAllocator ? *pAllocator : device->alloc;
|
||||
pool->command_buffer_ops = device->command_buffer_ops;
|
||||
pool->recycle_command_buffers = should_recycle_command_buffers(device);
|
||||
list_inithead(&pool->command_buffers);
|
||||
|
||||
for (uint32_t i = 0; i < ARRAY_SIZE(pool->free_command_buffers); i++)
|
||||
list_inithead(&pool->free_command_buffers[i]);
|
||||
|
||||
return VK_SUCCESS;
|
||||
}
|
||||
|
||||
void
|
||||
vk_command_pool_finish(struct vk_command_pool *pool)
|
||||
{
|
||||
list_for_each_entry_safe(struct vk_command_buffer, cmd_buffer,
|
||||
&pool->command_buffers, pool_link) {
|
||||
cmd_buffer->ops->destroy(cmd_buffer);
|
||||
}
|
||||
assert(list_is_empty(&pool->command_buffers));
|
||||
|
||||
destroy_free_command_buffers(pool);
|
||||
|
||||
vk_object_base_finish(&pool->base);
|
||||
}
|
||||
|
||||
VKAPI_ATTR VkResult VKAPI_CALL
|
||||
vk_common_CreateCommandPool(VkDevice _device,
|
||||
const VkCommandPoolCreateInfo *pCreateInfo,
|
||||
const VkAllocationCallbacks *pAllocator,
|
||||
VkCommandPool *pCommandPool)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_device, device, _device);
|
||||
struct vk_command_pool *pool;
|
||||
VkResult result;
|
||||
|
||||
pool = vk_alloc2(&device->alloc, pAllocator, sizeof(*pool), 8,
|
||||
VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
|
||||
if (pool == NULL)
|
||||
return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
|
||||
|
||||
result = vk_command_pool_init(device, pool, pCreateInfo, pAllocator);
|
||||
if (unlikely(result != VK_SUCCESS)) {
|
||||
vk_free2(&device->alloc, pAllocator, pool);
|
||||
return result;
|
||||
}
|
||||
|
||||
*pCommandPool = vk_command_pool_to_handle(pool);
|
||||
|
||||
return VK_SUCCESS;
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_DestroyCommandPool(VkDevice _device,
|
||||
VkCommandPool commandPool,
|
||||
const VkAllocationCallbacks *pAllocator)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_device, device, _device);
|
||||
VK_FROM_HANDLE(vk_command_pool, pool, commandPool);
|
||||
|
||||
if (pool == NULL)
|
||||
return;
|
||||
|
||||
vk_command_pool_finish(pool);
|
||||
vk_free2(&device->alloc, pAllocator, pool);
|
||||
}
|
||||
|
||||
VKAPI_ATTR VkResult VKAPI_CALL
|
||||
vk_common_ResetCommandPool(VkDevice device,
|
||||
VkCommandPool commandPool,
|
||||
VkCommandPoolResetFlags flags)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_pool, pool, commandPool);
|
||||
const struct vk_device_dispatch_table *disp =
|
||||
&pool->base.device->dispatch_table;
|
||||
|
||||
#define COPY_FLAG(flag) \
|
||||
if (flags & VK_COMMAND_POOL_RESET_##flag) \
|
||||
cb_flags |= VK_COMMAND_BUFFER_RESET_##flag
|
||||
|
||||
VkCommandBufferResetFlags cb_flags = 0;
|
||||
COPY_FLAG(RELEASE_RESOURCES_BIT);
|
||||
|
||||
#undef COPY_FLAG
|
||||
|
||||
list_for_each_entry_safe(struct vk_command_buffer, cmd_buffer,
|
||||
&pool->command_buffers, pool_link) {
|
||||
VkResult result =
|
||||
disp->ResetCommandBuffer(vk_command_buffer_to_handle(cmd_buffer),
|
||||
cb_flags);
|
||||
if (result != VK_SUCCESS)
|
||||
return result;
|
||||
}
|
||||
|
||||
return VK_SUCCESS;
|
||||
}
|
||||
|
||||
static void
|
||||
vk_command_buffer_recycle_or_destroy(struct vk_command_pool *pool,
|
||||
struct vk_command_buffer *cmd_buffer)
|
||||
{
|
||||
assert(pool == cmd_buffer->pool);
|
||||
|
||||
if (pool->recycle_command_buffers) {
|
||||
vk_command_buffer_recycle(cmd_buffer);
|
||||
|
||||
list_del(&cmd_buffer->pool_link);
|
||||
list_add(&cmd_buffer->pool_link, &pool->free_command_buffers[cmd_buffer->level]);
|
||||
} else {
|
||||
cmd_buffer->ops->destroy(cmd_buffer);
|
||||
}
|
||||
}
|
||||
|
||||
static struct vk_command_buffer *
|
||||
vk_command_pool_find_free(struct vk_command_pool *pool,
|
||||
VkCommandBufferLevel level)
|
||||
{
|
||||
if (list_is_empty(&pool->free_command_buffers[level]))
|
||||
return NULL;
|
||||
|
||||
struct vk_command_buffer *cmd_buffer =
|
||||
list_first_entry(&pool->free_command_buffers[level],
|
||||
struct vk_command_buffer, pool_link);
|
||||
|
||||
list_del(&cmd_buffer->pool_link);
|
||||
list_addtail(&cmd_buffer->pool_link, &pool->command_buffers);
|
||||
|
||||
return cmd_buffer;
|
||||
}
|
||||
|
||||
VKAPI_ATTR VkResult VKAPI_CALL
|
||||
vk_common_AllocateCommandBuffers(VkDevice device,
|
||||
const VkCommandBufferAllocateInfo *pAllocateInfo,
|
||||
VkCommandBuffer *pCommandBuffers)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_pool, pool, pAllocateInfo->commandPool);
|
||||
VkResult result;
|
||||
uint32_t i;
|
||||
|
||||
assert(device == vk_device_to_handle(pool->base.device));
|
||||
|
||||
for (i = 0; i < pAllocateInfo->commandBufferCount; i++) {
|
||||
struct vk_command_buffer *cmd_buffer =
|
||||
vk_command_pool_find_free(pool, pAllocateInfo->level);
|
||||
if (cmd_buffer == NULL) {
|
||||
result = pool->command_buffer_ops->create(pool, pAllocateInfo->level, &cmd_buffer);
|
||||
if (unlikely(result != VK_SUCCESS))
|
||||
goto fail;
|
||||
}
|
||||
|
||||
cmd_buffer->level = pAllocateInfo->level;
|
||||
|
||||
pCommandBuffers[i] = vk_command_buffer_to_handle(cmd_buffer);
|
||||
}
|
||||
|
||||
return VK_SUCCESS;
|
||||
|
||||
fail:
|
||||
while (i--) {
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, pCommandBuffers[i]);
|
||||
vk_command_buffer_recycle_or_destroy(pool, cmd_buffer);
|
||||
}
|
||||
for (i = 0; i < pAllocateInfo->commandBufferCount; i++)
|
||||
pCommandBuffers[i] = VK_NULL_HANDLE;
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_FreeCommandBuffers(VkDevice device,
|
||||
VkCommandPool commandPool,
|
||||
uint32_t commandBufferCount,
|
||||
const VkCommandBuffer *pCommandBuffers)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_pool, pool, commandPool);
|
||||
|
||||
for (uint32_t i = 0; i < commandBufferCount; i++) {
|
||||
VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, pCommandBuffers[i]);
|
||||
|
||||
if (cmd_buffer == NULL)
|
||||
continue;
|
||||
|
||||
vk_command_buffer_recycle_or_destroy(pool, cmd_buffer);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
vk_command_pool_trim(struct vk_command_pool *pool,
|
||||
VkCommandPoolTrimFlags flags)
|
||||
{
|
||||
destroy_free_command_buffers(pool);
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_TrimCommandPool(VkDevice device,
|
||||
VkCommandPool commandPool,
|
||||
VkCommandPoolTrimFlags flags)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_pool, pool, commandPool);
|
||||
|
||||
vk_command_pool_trim(pool, flags);
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
/*
|
||||
* Copyright © 2022 Collabora, Ltd
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#ifndef VK_COMMAND_POOL_H
|
||||
#define VK_COMMAND_POOL_H
|
||||
|
||||
#include "vk_object.h"
|
||||
#include "util/list.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/** Base object for implementin VkCommandPool */
|
||||
struct vk_command_pool {
|
||||
struct vk_object_base base;
|
||||
|
||||
/** VkCommandPoolCreateInfo::flags */
|
||||
VkCommandPoolCreateFlags flags;
|
||||
|
||||
/** VkCommandPoolCreateInfo::queueFamilyIndex */
|
||||
uint32_t queue_family_index;
|
||||
|
||||
/** Allocator passed to vkCreateCommandPool() */
|
||||
VkAllocationCallbacks alloc;
|
||||
|
||||
/** Command buffer vtable for command buffers allocated from this pool */
|
||||
const struct vk_command_buffer_ops *command_buffer_ops;
|
||||
|
||||
/** True if we should recycle command buffers */
|
||||
bool recycle_command_buffers;
|
||||
|
||||
/** List of all command buffers */
|
||||
struct list_head command_buffers;
|
||||
|
||||
/** List of freed command buffers for trimming. */
|
||||
struct list_head free_command_buffers[2];
|
||||
};
|
||||
|
||||
VK_DEFINE_NONDISP_HANDLE_CASTS(vk_command_pool, base, VkCommandPool,
|
||||
VK_OBJECT_TYPE_COMMAND_POOL);
|
||||
|
||||
/** Initialize a vk_command_pool
|
||||
*
|
||||
* :param device: |in| The Vulkan device
|
||||
* :param pool: |out| The command pool to initialize
|
||||
* :param pCreateInfo: |in| VkCommandPoolCreateInfo pointer passed to
|
||||
* `vkCreateCommandPool()`
|
||||
* :param pAllocator: |in| Allocation callbacks passed to
|
||||
* `vkCreateCommandPool()`
|
||||
*/
|
||||
VkResult MUST_CHECK
|
||||
vk_command_pool_init(struct vk_device *device,
|
||||
struct vk_command_pool *pool,
|
||||
const VkCommandPoolCreateInfo *pCreateInfo,
|
||||
const VkAllocationCallbacks *pAllocator);
|
||||
|
||||
/** Tear down a vk_command_pool
|
||||
*
|
||||
* :param pool: |inout| The command pool to tear down
|
||||
*/
|
||||
void
|
||||
vk_command_pool_finish(struct vk_command_pool *pool);
|
||||
|
||||
/** Trim a vk_command_pool
|
||||
*
|
||||
* This discards any resources that may be cached by the common
|
||||
* vk_command_pool code. For driver-implemented command pools, drivers should
|
||||
* call this function inside their `vkTrimCommandPool()` implementation. This
|
||||
* should be called before doing any driver-specific trimming in case it ends
|
||||
* up returning driver-internal resources to the pool.
|
||||
*
|
||||
* :param pool: |inout| The command pool to trim
|
||||
* :param flags: |in| Flags controling the trim operation
|
||||
*/
|
||||
void
|
||||
vk_command_pool_trim(struct vk_command_pool *pool,
|
||||
VkCommandPoolTrimFlags flags);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* VK_COMMAND_POOL_H */
|
||||
@@ -0,0 +1,159 @@
|
||||
/*
|
||||
* Copyright © 2017 Intel Corporation
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "vk_debug_report.h"
|
||||
|
||||
#include "vk_alloc.h"
|
||||
#include "vk_common_entrypoints.h"
|
||||
#include "vk_instance.h"
|
||||
#include "vk_util.h"
|
||||
|
||||
struct vk_debug_report_callback {
|
||||
struct vk_object_base base;
|
||||
|
||||
/* Link in the 'callbacks' list in anv_instance struct. */
|
||||
struct list_head link;
|
||||
VkDebugReportFlagsEXT flags;
|
||||
PFN_vkDebugReportCallbackEXT callback;
|
||||
void * data;
|
||||
};
|
||||
|
||||
VK_DEFINE_NONDISP_HANDLE_CASTS(vk_debug_report_callback, base,
|
||||
VkDebugReportCallbackEXT,
|
||||
VK_OBJECT_TYPE_DEBUG_REPORT_CALLBACK_EXT)
|
||||
|
||||
VKAPI_ATTR VkResult VKAPI_CALL
|
||||
vk_common_CreateDebugReportCallbackEXT(VkInstance _instance,
|
||||
const VkDebugReportCallbackCreateInfoEXT *pCreateInfo,
|
||||
const VkAllocationCallbacks *pAllocator,
|
||||
VkDebugReportCallbackEXT *pCallback)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_instance, instance, _instance);
|
||||
|
||||
struct vk_debug_report_callback *cb =
|
||||
vk_alloc2(&instance->alloc, pAllocator,
|
||||
sizeof(struct vk_debug_report_callback), 8,
|
||||
VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
|
||||
|
||||
if (!cb)
|
||||
return VK_ERROR_OUT_OF_HOST_MEMORY;
|
||||
|
||||
vk_object_base_instance_init(instance, &cb->base,
|
||||
VK_OBJECT_TYPE_DEBUG_REPORT_CALLBACK_EXT);
|
||||
|
||||
cb->flags = pCreateInfo->flags;
|
||||
cb->callback = pCreateInfo->pfnCallback;
|
||||
cb->data = pCreateInfo->pUserData;
|
||||
|
||||
mtx_lock(&instance->debug_report.callbacks_mutex);
|
||||
list_addtail(&cb->link, &instance->debug_report.callbacks);
|
||||
mtx_unlock(&instance->debug_report.callbacks_mutex);
|
||||
|
||||
*pCallback = vk_debug_report_callback_to_handle(cb);
|
||||
|
||||
return VK_SUCCESS;
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_DestroyDebugReportCallbackEXT(VkInstance _instance,
|
||||
VkDebugReportCallbackEXT _callback,
|
||||
const VkAllocationCallbacks *pAllocator)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_instance, instance, _instance);
|
||||
VK_FROM_HANDLE(vk_debug_report_callback, callback, _callback);
|
||||
|
||||
if (callback == NULL)
|
||||
return;
|
||||
|
||||
/* Remove from list and destroy given callback. */
|
||||
mtx_lock(&instance->debug_report.callbacks_mutex);
|
||||
list_del(&callback->link);
|
||||
vk_object_base_finish(&callback->base);
|
||||
vk_free2(&instance->alloc, pAllocator, callback);
|
||||
mtx_unlock(&instance->debug_report.callbacks_mutex);
|
||||
}
|
||||
|
||||
static void
|
||||
debug_report_message(struct vk_debug_report *debug_report,
|
||||
VkDebugReportFlagsEXT flags,
|
||||
VkDebugReportObjectTypeEXT object_type,
|
||||
uint64_t handle,
|
||||
size_t location,
|
||||
int32_t messageCode,
|
||||
const char* pLayerPrefix,
|
||||
const char *pMessage)
|
||||
{
|
||||
/* Return if no callbacks registered. */
|
||||
if (list_is_empty(&debug_report->callbacks))
|
||||
return;
|
||||
|
||||
mtx_lock(&debug_report->callbacks_mutex);
|
||||
|
||||
/* Section 33.2 of the Vulkan 1.0.59 spec says:
|
||||
*
|
||||
* "callback is an externally synchronized object and must not be
|
||||
* used on more than one thread at a time. This means that
|
||||
* vkDestroyDebugReportCallbackEXT must not be called when a callback
|
||||
* is active."
|
||||
*/
|
||||
list_for_each_entry(struct vk_debug_report_callback, cb,
|
||||
&debug_report->callbacks, link) {
|
||||
if (cb->flags & flags)
|
||||
cb->callback(flags, object_type, handle, location, messageCode,
|
||||
pLayerPrefix, pMessage, cb->data);
|
||||
}
|
||||
|
||||
mtx_unlock(&debug_report->callbacks_mutex);
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_DebugReportMessageEXT(VkInstance _instance,
|
||||
VkDebugReportFlagsEXT flags,
|
||||
VkDebugReportObjectTypeEXT objectType,
|
||||
uint64_t object,
|
||||
size_t location,
|
||||
int32_t messageCode,
|
||||
const char* pLayerPrefix,
|
||||
const char* pMessage)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_instance, instance, _instance);
|
||||
debug_report_message(&instance->debug_report, flags, objectType,
|
||||
object, location, messageCode, pLayerPrefix, pMessage);
|
||||
}
|
||||
|
||||
void
|
||||
vk_debug_report(struct vk_debug_report *debug_report,
|
||||
VkDebugReportFlagsEXT flags,
|
||||
const struct vk_object_base *object,
|
||||
size_t location,
|
||||
int32_t messageCode,
|
||||
const char* pLayerPrefix,
|
||||
const char *pMessage)
|
||||
{
|
||||
VkObjectType object_type =
|
||||
object ? object->type : VK_OBJECT_TYPE_UNKNOWN;
|
||||
|
||||
debug_report_message(debug_report, flags, (VkDebugReportObjectTypeEXT)object_type,
|
||||
(uint64_t)(uintptr_t)object, location, messageCode,
|
||||
pLayerPrefix, pMessage);
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
/*
|
||||
* Copyright © 2018, Google Inc.
|
||||
*
|
||||
* based on the anv driver which is:
|
||||
* Copyright © 2017 Intel Corporation
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
#ifndef VK_DEBUG_REPORT_H
|
||||
#define VK_DEBUG_REPORT_H
|
||||
|
||||
#include "vk_object.h"
|
||||
|
||||
#include "util/simple_mtx.h"
|
||||
#include "util/list.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
struct vk_debug_report {
|
||||
mtx_t callbacks_mutex;
|
||||
struct list_head callbacks;
|
||||
};
|
||||
|
||||
void
|
||||
vk_debug_report(struct vk_debug_report *debug_report,
|
||||
VkDebugReportFlagsEXT flags,
|
||||
const struct vk_object_base *object,
|
||||
size_t location,
|
||||
int32_t messageCode,
|
||||
const char* pLayerPrefix,
|
||||
const char *pMessage);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* VK_DEBUG_REPORT_H */
|
||||
@@ -0,0 +1,603 @@
|
||||
/*
|
||||
* Copyright © 2021 Intel Corporation
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "vk_debug_utils.h"
|
||||
|
||||
#include "vk_common_entrypoints.h"
|
||||
#include "vk_command_buffer.h"
|
||||
#include "vk_device.h"
|
||||
#include "vk_queue.h"
|
||||
#include "vk_object.h"
|
||||
#include "vk_alloc.h"
|
||||
#include "vk_util.h"
|
||||
#include "stdarg.h"
|
||||
#include "util/u_dynarray.h"
|
||||
#include "util/u_printf.h"
|
||||
#include "vulkan/vulkan_core.h"
|
||||
|
||||
void
|
||||
vk_debug_message(struct vk_instance *instance,
|
||||
VkDebugUtilsMessageSeverityFlagBitsEXT severity,
|
||||
VkDebugUtilsMessageTypeFlagsEXT types,
|
||||
const VkDebugUtilsMessengerCallbackDataEXT *pCallbackData)
|
||||
{
|
||||
mtx_lock(&instance->debug_utils.callbacks_mutex);
|
||||
|
||||
list_for_each_entry(struct vk_debug_utils_messenger, messenger,
|
||||
&instance->debug_utils.callbacks, link) {
|
||||
if ((messenger->severity & severity) &&
|
||||
(messenger->type & types))
|
||||
messenger->callback(severity, types, pCallbackData, messenger->data);
|
||||
}
|
||||
|
||||
mtx_unlock(&instance->debug_utils.callbacks_mutex);
|
||||
}
|
||||
|
||||
/* This function intended to be used by the drivers to report a
|
||||
* message to the special messenger, provided in the pNext chain while
|
||||
* creating an instance. It's only meant to be used during
|
||||
* vkCreateInstance or vkDestroyInstance calls.
|
||||
*/
|
||||
void
|
||||
vk_debug_message_instance(struct vk_instance *instance,
|
||||
VkDebugUtilsMessageSeverityFlagBitsEXT severity,
|
||||
VkDebugUtilsMessageTypeFlagsEXT types,
|
||||
const char *pMessageIdName,
|
||||
int32_t messageIdNumber,
|
||||
const char *pMessage)
|
||||
{
|
||||
if (list_is_empty(&instance->debug_utils.instance_callbacks))
|
||||
return;
|
||||
|
||||
const VkDebugUtilsMessengerCallbackDataEXT cbData = {
|
||||
.sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_MESSENGER_CALLBACK_DATA_EXT,
|
||||
.pMessageIdName = pMessageIdName,
|
||||
.messageIdNumber = messageIdNumber,
|
||||
.pMessage = pMessage,
|
||||
};
|
||||
|
||||
list_for_each_entry(struct vk_debug_utils_messenger, messenger,
|
||||
&instance->debug_utils.instance_callbacks, link) {
|
||||
if ((messenger->severity & severity) &&
|
||||
(messenger->type & types))
|
||||
messenger->callback(severity, types, &cbData, messenger->data);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
vk_address_binding_report(struct vk_instance *instance,
|
||||
struct vk_object_base *object,
|
||||
uint64_t base_address,
|
||||
uint64_t size,
|
||||
VkDeviceAddressBindingTypeEXT type)
|
||||
{
|
||||
if (list_is_empty(&instance->debug_utils.callbacks))
|
||||
return;
|
||||
|
||||
VkDeviceAddressBindingCallbackDataEXT addr_binding = {
|
||||
.sType = VK_STRUCTURE_TYPE_DEVICE_ADDRESS_BINDING_CALLBACK_DATA_EXT,
|
||||
.flags = object->client_visible ? 0 : VK_DEVICE_ADDRESS_BINDING_INTERNAL_OBJECT_BIT_EXT,
|
||||
.baseAddress = base_address,
|
||||
.size = size,
|
||||
.bindingType = type,
|
||||
};
|
||||
|
||||
VkDebugUtilsObjectNameInfoEXT object_name_info = {
|
||||
.sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_OBJECT_NAME_INFO_EXT,
|
||||
.pNext = NULL,
|
||||
.objectType = object->type,
|
||||
.objectHandle = (uint64_t)(uintptr_t)object,
|
||||
.pObjectName = object->object_name,
|
||||
};
|
||||
|
||||
VkDebugUtilsMessengerCallbackDataEXT cb_data = {
|
||||
.sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_MESSENGER_CALLBACK_DATA_EXT,
|
||||
.pNext = &addr_binding,
|
||||
.objectCount = 1,
|
||||
.pObjects = &object_name_info,
|
||||
};
|
||||
|
||||
vk_debug_message(instance, VK_DEBUG_UTILS_MESSAGE_SEVERITY_INFO_BIT_EXT,
|
||||
VK_DEBUG_UTILS_MESSAGE_TYPE_DEVICE_ADDRESS_BINDING_BIT_EXT,
|
||||
&cb_data);
|
||||
}
|
||||
|
||||
void
|
||||
vk_emit_device_memory_report(struct vk_device* device,
|
||||
VkDeviceMemoryReportEventTypeEXT type,
|
||||
uint64_t mem_obj_id,
|
||||
VkDeviceSize size,
|
||||
VkObjectType obj_type,
|
||||
uint64_t obj_handle,
|
||||
uint32_t heap_index)
|
||||
{
|
||||
assert(device->memory_reports);
|
||||
|
||||
const VkDeviceMemoryReportCallbackDataEXT report = {
|
||||
.sType = VK_STRUCTURE_TYPE_DEVICE_MEMORY_REPORT_CALLBACK_DATA_EXT,
|
||||
.type = type,
|
||||
.memoryObjectId = mem_obj_id,
|
||||
.size = size,
|
||||
.objectType = obj_type,
|
||||
.objectHandle = obj_handle,
|
||||
.heapIndex = heap_index,
|
||||
};
|
||||
|
||||
for (uint32_t i = 0; i < device->memory_report_count; i++)
|
||||
device->memory_reports[i].callback(&report, device->memory_reports[i].data);
|
||||
}
|
||||
|
||||
VKAPI_ATTR VkResult VKAPI_CALL
|
||||
vk_common_CreateDebugUtilsMessengerEXT(
|
||||
VkInstance _instance,
|
||||
const VkDebugUtilsMessengerCreateInfoEXT *pCreateInfo,
|
||||
const VkAllocationCallbacks *pAllocator,
|
||||
VkDebugUtilsMessengerEXT *pMessenger)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_instance, instance, _instance);
|
||||
|
||||
struct vk_debug_utils_messenger *messenger =
|
||||
vk_alloc2(&instance->alloc, pAllocator,
|
||||
sizeof(struct vk_debug_utils_messenger), 8,
|
||||
VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
|
||||
|
||||
if (!messenger)
|
||||
return VK_ERROR_OUT_OF_HOST_MEMORY;
|
||||
|
||||
if (pAllocator)
|
||||
messenger->alloc = *pAllocator;
|
||||
else
|
||||
messenger->alloc = instance->alloc;
|
||||
|
||||
vk_object_base_init(NULL, &messenger->base,
|
||||
VK_OBJECT_TYPE_DEBUG_UTILS_MESSENGER_EXT);
|
||||
|
||||
messenger->severity = pCreateInfo->messageSeverity;
|
||||
messenger->type = pCreateInfo->messageType;
|
||||
messenger->callback = pCreateInfo->pfnUserCallback;
|
||||
messenger->data = pCreateInfo->pUserData;
|
||||
|
||||
mtx_lock(&instance->debug_utils.callbacks_mutex);
|
||||
list_addtail(&messenger->link, &instance->debug_utils.callbacks);
|
||||
mtx_unlock(&instance->debug_utils.callbacks_mutex);
|
||||
|
||||
*pMessenger = vk_debug_utils_messenger_to_handle(messenger);
|
||||
|
||||
return VK_SUCCESS;
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_SubmitDebugUtilsMessageEXT(
|
||||
VkInstance _instance,
|
||||
VkDebugUtilsMessageSeverityFlagBitsEXT messageSeverity,
|
||||
VkDebugUtilsMessageTypeFlagsEXT messageTypes,
|
||||
const VkDebugUtilsMessengerCallbackDataEXT *pCallbackData)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_instance, instance, _instance);
|
||||
|
||||
vk_debug_message(instance, messageSeverity, messageTypes, pCallbackData);
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_DestroyDebugUtilsMessengerEXT(
|
||||
VkInstance _instance,
|
||||
VkDebugUtilsMessengerEXT _messenger,
|
||||
const VkAllocationCallbacks *pAllocator)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_instance, instance, _instance);
|
||||
VK_FROM_HANDLE(vk_debug_utils_messenger, messenger, _messenger);
|
||||
|
||||
if (messenger == NULL)
|
||||
return;
|
||||
|
||||
mtx_lock(&instance->debug_utils.callbacks_mutex);
|
||||
list_del(&messenger->link);
|
||||
mtx_unlock(&instance->debug_utils.callbacks_mutex);
|
||||
|
||||
vk_object_base_finish(&messenger->base);
|
||||
vk_free2(&instance->alloc, pAllocator, messenger);
|
||||
}
|
||||
|
||||
static VkResult
|
||||
vk_common_set_object_name_locked(
|
||||
struct vk_device *device,
|
||||
const VkDebugUtilsObjectNameInfoEXT *pNameInfo)
|
||||
{
|
||||
if (unlikely(device->swapchain_name == NULL)) {
|
||||
/* Even though VkSwapchain/Surface are non-dispatchable objects, we know
|
||||
* a priori that these are actually pointers so we can use
|
||||
* the pointer hash table for them.
|
||||
*/
|
||||
device->swapchain_name = _mesa_pointer_hash_table_create(NULL);
|
||||
if (device->swapchain_name == NULL)
|
||||
return VK_ERROR_OUT_OF_HOST_MEMORY;
|
||||
}
|
||||
|
||||
char *object_name = vk_strdup(&device->alloc, pNameInfo->pObjectName,
|
||||
VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
|
||||
if (object_name == NULL)
|
||||
return VK_ERROR_OUT_OF_HOST_MEMORY;
|
||||
struct hash_entry *entry =
|
||||
_mesa_hash_table_search(device->swapchain_name,
|
||||
(void *)(uintptr_t)pNameInfo->objectHandle);
|
||||
if (unlikely(entry == NULL)) {
|
||||
entry = _mesa_hash_table_insert(device->swapchain_name,
|
||||
(void *)(uintptr_t)pNameInfo->objectHandle,
|
||||
object_name);
|
||||
if (entry == NULL) {
|
||||
vk_free(&device->alloc, object_name);
|
||||
return VK_ERROR_OUT_OF_HOST_MEMORY;
|
||||
}
|
||||
} else {
|
||||
vk_free(&device->alloc, entry->data);
|
||||
entry->data = object_name;
|
||||
}
|
||||
return VK_SUCCESS;
|
||||
}
|
||||
|
||||
VKAPI_ATTR VkResult VKAPI_CALL
|
||||
vk_common_DebugMarkerSetObjectNameEXT(
|
||||
VkDevice _device,
|
||||
const VkDebugMarkerObjectNameInfoEXT *pNameInfo)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_device, device, _device);
|
||||
|
||||
VkObjectType object_type;
|
||||
switch (pNameInfo->objectType) {
|
||||
case VK_DEBUG_REPORT_OBJECT_TYPE_SURFACE_KHR_EXT:
|
||||
object_type = VK_OBJECT_TYPE_SURFACE_KHR;
|
||||
break;
|
||||
case VK_DEBUG_REPORT_OBJECT_TYPE_SWAPCHAIN_KHR_EXT:
|
||||
object_type = VK_OBJECT_TYPE_SWAPCHAIN_KHR;
|
||||
break;
|
||||
case VK_DEBUG_REPORT_OBJECT_TYPE_DEBUG_REPORT_CALLBACK_EXT_EXT:
|
||||
object_type = VK_OBJECT_TYPE_DEBUG_REPORT_CALLBACK_EXT;
|
||||
break;
|
||||
case VK_DEBUG_REPORT_OBJECT_TYPE_DISPLAY_KHR_EXT:
|
||||
object_type = VK_OBJECT_TYPE_DISPLAY_KHR;
|
||||
break;
|
||||
case VK_DEBUG_REPORT_OBJECT_TYPE_DISPLAY_MODE_KHR_EXT:
|
||||
object_type = VK_OBJECT_TYPE_DISPLAY_MODE_KHR;
|
||||
break;
|
||||
case VK_DEBUG_REPORT_OBJECT_TYPE_VALIDATION_CACHE_EXT_EXT:
|
||||
object_type = VK_OBJECT_TYPE_VALIDATION_CACHE_EXT;
|
||||
break;
|
||||
default:
|
||||
object_type = (VkObjectType)pNameInfo->objectType;
|
||||
break;
|
||||
}
|
||||
|
||||
VkDebugUtilsObjectNameInfoEXT name_info = {
|
||||
.sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_OBJECT_NAME_INFO_EXT,
|
||||
.objectType = object_type,
|
||||
.objectHandle = pNameInfo->object,
|
||||
.pObjectName = pNameInfo->pObjectName,
|
||||
};
|
||||
|
||||
return device->dispatch_table.SetDebugUtilsObjectNameEXT(_device, &name_info);
|
||||
}
|
||||
|
||||
VKAPI_ATTR VkResult VKAPI_CALL
|
||||
vk_common_DebugMarkerSetObjectTagEXT(
|
||||
VkDevice _device,
|
||||
const VkDebugMarkerObjectTagInfoEXT *pTagInfo)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_device, device, _device);
|
||||
|
||||
VkObjectType object_type;
|
||||
switch (pTagInfo->objectType) {
|
||||
case VK_DEBUG_REPORT_OBJECT_TYPE_SURFACE_KHR_EXT:
|
||||
object_type = VK_OBJECT_TYPE_SURFACE_KHR;
|
||||
break;
|
||||
case VK_DEBUG_REPORT_OBJECT_TYPE_SWAPCHAIN_KHR_EXT:
|
||||
object_type = VK_OBJECT_TYPE_SWAPCHAIN_KHR;
|
||||
break;
|
||||
case VK_DEBUG_REPORT_OBJECT_TYPE_DEBUG_REPORT_CALLBACK_EXT_EXT:
|
||||
object_type = VK_OBJECT_TYPE_DEBUG_REPORT_CALLBACK_EXT;
|
||||
break;
|
||||
case VK_DEBUG_REPORT_OBJECT_TYPE_DISPLAY_KHR_EXT:
|
||||
object_type = VK_OBJECT_TYPE_DISPLAY_KHR;
|
||||
break;
|
||||
case VK_DEBUG_REPORT_OBJECT_TYPE_DISPLAY_MODE_KHR_EXT:
|
||||
object_type = VK_OBJECT_TYPE_DISPLAY_MODE_KHR;
|
||||
break;
|
||||
case VK_DEBUG_REPORT_OBJECT_TYPE_VALIDATION_CACHE_EXT_EXT:
|
||||
object_type = VK_OBJECT_TYPE_VALIDATION_CACHE_EXT;
|
||||
break;
|
||||
default:
|
||||
object_type = (VkObjectType)pTagInfo->objectType;
|
||||
break;
|
||||
}
|
||||
|
||||
VkDebugUtilsObjectTagInfoEXT tag_info = {
|
||||
.sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_OBJECT_NAME_INFO_EXT,
|
||||
.objectType = object_type,
|
||||
.objectHandle = pTagInfo->object,
|
||||
.tagName = pTagInfo->tagName,
|
||||
.tagSize = pTagInfo->tagSize,
|
||||
.pTag = pTagInfo->pTag
|
||||
};
|
||||
|
||||
return device->dispatch_table.SetDebugUtilsObjectTagEXT(_device, &tag_info);
|
||||
}
|
||||
|
||||
VKAPI_ATTR VkResult VKAPI_CALL
|
||||
vk_common_SetDebugUtilsObjectNameEXT(
|
||||
VkDevice _device,
|
||||
const VkDebugUtilsObjectNameInfoEXT *pNameInfo)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_device, device, _device);
|
||||
|
||||
#ifdef VK_USE_PLATFORM_ANDROID_KHR
|
||||
if (pNameInfo->objectType == VK_OBJECT_TYPE_SWAPCHAIN_KHR ||
|
||||
pNameInfo->objectType == VK_OBJECT_TYPE_SURFACE_KHR) {
|
||||
#else
|
||||
if (pNameInfo->objectType == VK_OBJECT_TYPE_SURFACE_KHR) {
|
||||
#endif
|
||||
mtx_lock(&device->swapchain_name_mtx);
|
||||
VkResult res = vk_common_set_object_name_locked(device, pNameInfo);
|
||||
mtx_unlock(&device->swapchain_name_mtx);
|
||||
return res;
|
||||
}
|
||||
|
||||
struct vk_object_base *object =
|
||||
vk_object_base_from_u64_handle(pNameInfo->objectHandle,
|
||||
pNameInfo->objectType);
|
||||
|
||||
assert(object->device != NULL || object->instance != NULL);
|
||||
VkAllocationCallbacks *alloc = object->device != NULL ?
|
||||
&object->device->alloc : &object->instance->alloc;
|
||||
if (object->object_name) {
|
||||
vk_free(alloc, object->object_name);
|
||||
object->object_name = NULL;
|
||||
}
|
||||
if (pNameInfo->pObjectName != NULL) {
|
||||
object->object_name = vk_strdup(alloc, pNameInfo->pObjectName,
|
||||
VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
|
||||
if (!object->object_name)
|
||||
return VK_ERROR_OUT_OF_HOST_MEMORY;
|
||||
}
|
||||
|
||||
return VK_SUCCESS;
|
||||
}
|
||||
|
||||
VKAPI_ATTR VkResult VKAPI_CALL
|
||||
vk_common_SetDebugUtilsObjectTagEXT(
|
||||
VkDevice _device,
|
||||
const VkDebugUtilsObjectTagInfoEXT *pTagInfo)
|
||||
{
|
||||
/* no-op */
|
||||
return VK_SUCCESS;
|
||||
}
|
||||
|
||||
static void
|
||||
vk_common_append_debug_label(struct vk_device *device,
|
||||
struct util_dynarray *labels,
|
||||
const VkDebugUtilsLabelEXT *pLabelInfo)
|
||||
{
|
||||
util_dynarray_append(labels, *pLabelInfo);
|
||||
VkDebugUtilsLabelEXT *current_label =
|
||||
util_dynarray_top_ptr(labels, VkDebugUtilsLabelEXT);
|
||||
current_label->pLabelName =
|
||||
vk_strdup(&device->alloc, current_label->pLabelName,
|
||||
VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
|
||||
}
|
||||
|
||||
static void
|
||||
vk_common_pop_debug_label(struct vk_device *device,
|
||||
struct util_dynarray *labels)
|
||||
{
|
||||
if (labels->size == 0)
|
||||
return;
|
||||
|
||||
VkDebugUtilsLabelEXT previous_label =
|
||||
util_dynarray_pop(labels, VkDebugUtilsLabelEXT);
|
||||
vk_free(&device->alloc, (void *)previous_label.pLabelName);
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdBeginDebugUtilsLabelEXT(
|
||||
VkCommandBuffer _commandBuffer,
|
||||
const VkDebugUtilsLabelEXT *pLabelInfo)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, command_buffer, _commandBuffer);
|
||||
|
||||
/* If the latest label was submitted by CmdInsertDebugUtilsLabelEXT, we
|
||||
* should remove it first.
|
||||
*/
|
||||
if (!command_buffer->region_begin) {
|
||||
vk_common_pop_debug_label(command_buffer->base.device,
|
||||
&command_buffer->labels);
|
||||
}
|
||||
|
||||
vk_common_append_debug_label(command_buffer->base.device,
|
||||
&command_buffer->labels,
|
||||
pLabelInfo);
|
||||
command_buffer->region_begin = true;
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdEndDebugUtilsLabelEXT(VkCommandBuffer _commandBuffer)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, command_buffer, _commandBuffer);
|
||||
|
||||
/* If the latest label was submitted by CmdInsertDebugUtilsLabelEXT, we
|
||||
* should remove it first.
|
||||
*/
|
||||
if (!command_buffer->region_begin) {
|
||||
vk_common_pop_debug_label(command_buffer->base.device,
|
||||
&command_buffer->labels);
|
||||
}
|
||||
|
||||
vk_common_pop_debug_label(command_buffer->base.device,
|
||||
&command_buffer->labels);
|
||||
command_buffer->region_begin = true;
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdInsertDebugUtilsLabelEXT(
|
||||
VkCommandBuffer _commandBuffer,
|
||||
const VkDebugUtilsLabelEXT *pLabelInfo)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_command_buffer, command_buffer, _commandBuffer);
|
||||
|
||||
/* If the latest label was submitted by CmdInsertDebugUtilsLabelEXT, we
|
||||
* should remove it first.
|
||||
*/
|
||||
if (!command_buffer->region_begin) {
|
||||
vk_common_append_debug_label(command_buffer->base.device,
|
||||
&command_buffer->labels,
|
||||
pLabelInfo);
|
||||
}
|
||||
|
||||
vk_common_append_debug_label(command_buffer->base.device,
|
||||
&command_buffer->labels,
|
||||
pLabelInfo);
|
||||
command_buffer->region_begin = false;
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdDebugMarkerBeginEXT(
|
||||
VkCommandBuffer _commandBuffer,
|
||||
const VkDebugMarkerMarkerInfoEXT *pMarkerInfo)
|
||||
{
|
||||
VkDebugUtilsLabelEXT label_info = {
|
||||
.sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_LABEL_EXT,
|
||||
.pNext = pMarkerInfo->pNext,
|
||||
.pLabelName = pMarkerInfo->pMarkerName,
|
||||
};
|
||||
memcpy(label_info.color, pMarkerInfo->color, sizeof(label_info.color));
|
||||
|
||||
vk_common_CmdBeginDebugUtilsLabelEXT(_commandBuffer, &label_info);
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdDebugMarkerEndEXT(VkCommandBuffer _commandBuffer)
|
||||
{
|
||||
vk_common_CmdEndDebugUtilsLabelEXT(_commandBuffer);
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_CmdDebugMarkerInsertEXT(
|
||||
VkCommandBuffer _commandBuffer,
|
||||
const VkDebugMarkerMarkerInfoEXT *pMarkerInfo)
|
||||
{
|
||||
VkDebugUtilsLabelEXT label_info = {
|
||||
.sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_LABEL_EXT,
|
||||
.pNext = pMarkerInfo->pNext,
|
||||
.pLabelName = pMarkerInfo->pMarkerName,
|
||||
};
|
||||
memcpy(label_info.color, pMarkerInfo->color, sizeof(label_info.color));
|
||||
|
||||
vk_common_CmdInsertDebugUtilsLabelEXT(_commandBuffer, &label_info);
|
||||
}
|
||||
|
||||
void
|
||||
vk_queue_begin_debug_utils_label(struct vk_queue *queue,
|
||||
const VkDebugUtilsLabelEXT *pLabelInfo)
|
||||
{
|
||||
/* If the latest label was submitted by QueueInsertDebugUtilsLabelEXT, we
|
||||
* should remove it first.
|
||||
*/
|
||||
if (!queue->region_begin)
|
||||
vk_common_pop_debug_label(queue->base.device, &queue->labels);
|
||||
|
||||
vk_common_append_debug_label(queue->base.device,
|
||||
&queue->labels,
|
||||
pLabelInfo);
|
||||
queue->region_begin = true;
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_QueueBeginDebugUtilsLabelEXT(
|
||||
VkQueue _queue,
|
||||
const VkDebugUtilsLabelEXT *pLabelInfo)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_queue, queue, _queue);
|
||||
|
||||
vk_queue_lock(queue);
|
||||
vk_queue_begin_debug_utils_label(queue, pLabelInfo);
|
||||
vk_queue_unlock(queue);
|
||||
}
|
||||
|
||||
void
|
||||
vk_queue_end_debug_utils_label(struct vk_queue *queue)
|
||||
{
|
||||
/* If the latest label was submitted by QueueInsertDebugUtilsLabelEXT, we
|
||||
* should remove it first.
|
||||
*/
|
||||
if (!queue->region_begin)
|
||||
vk_common_pop_debug_label(queue->base.device, &queue->labels);
|
||||
|
||||
vk_common_pop_debug_label(queue->base.device, &queue->labels);
|
||||
queue->region_begin = true;
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_QueueEndDebugUtilsLabelEXT(VkQueue _queue)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_queue, queue, _queue);
|
||||
|
||||
vk_queue_lock(queue);
|
||||
vk_queue_end_debug_utils_label(queue);
|
||||
vk_queue_unlock(queue);
|
||||
}
|
||||
|
||||
static void
|
||||
vk_queue_insert_debug_utils_label(struct vk_queue *queue,
|
||||
const VkDebugUtilsLabelEXT *pLabelInfo)
|
||||
{
|
||||
/* If the latest label was submitted by QueueInsertDebugUtilsLabelEXT, we
|
||||
* should remove it first.
|
||||
*/
|
||||
if (!queue->region_begin)
|
||||
vk_common_pop_debug_label(queue->base.device, &queue->labels);
|
||||
|
||||
vk_common_append_debug_label(queue->base.device,
|
||||
&queue->labels,
|
||||
pLabelInfo);
|
||||
queue->region_begin = false;
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_QueueInsertDebugUtilsLabelEXT(
|
||||
VkQueue _queue,
|
||||
const VkDebugUtilsLabelEXT *pLabelInfo)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_queue, queue, _queue);
|
||||
|
||||
vk_queue_lock(queue);
|
||||
vk_queue_insert_debug_utils_label(queue, pLabelInfo);
|
||||
vk_queue_unlock(queue);
|
||||
}
|
||||
|
||||
VkResult
|
||||
vk_check_printf_status(struct vk_device *dev, struct u_printf_ctx *ctx)
|
||||
{
|
||||
if (u_printf_check_abort(stdout, ctx)) {
|
||||
vk_device_set_lost(dev, "GPU abort.");
|
||||
return VK_ERROR_DEVICE_LOST;
|
||||
} else {
|
||||
return VK_SUCCESS;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
/*
|
||||
* Copyright © 2021 Intel Corporation
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#ifndef VK_DEBUG_UTILS_H
|
||||
#define VK_DEBUG_UTILS_H
|
||||
|
||||
#include "vk_device.h"
|
||||
#include "vk_instance.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
struct vk_queue;
|
||||
|
||||
struct vk_debug_utils_messenger {
|
||||
struct vk_object_base base;
|
||||
VkAllocationCallbacks alloc;
|
||||
|
||||
struct list_head link;
|
||||
|
||||
VkDebugUtilsMessageSeverityFlagsEXT severity;
|
||||
VkDebugUtilsMessageTypeFlagsEXT type;
|
||||
PFN_vkDebugUtilsMessengerCallbackEXT callback;
|
||||
void *data;
|
||||
};
|
||||
|
||||
VK_DEFINE_NONDISP_HANDLE_CASTS(vk_debug_utils_messenger, base,
|
||||
VkDebugUtilsMessengerEXT,
|
||||
VK_OBJECT_TYPE_DEBUG_UTILS_MESSENGER_EXT)
|
||||
|
||||
void
|
||||
vk_debug_message(struct vk_instance *instance,
|
||||
VkDebugUtilsMessageSeverityFlagBitsEXT severity,
|
||||
VkDebugUtilsMessageTypeFlagsEXT types,
|
||||
const VkDebugUtilsMessengerCallbackDataEXT *pCallbackData);
|
||||
|
||||
void
|
||||
vk_debug_message_instance(struct vk_instance *instance,
|
||||
VkDebugUtilsMessageSeverityFlagBitsEXT severity,
|
||||
VkDebugUtilsMessageTypeFlagsEXT types,
|
||||
const char *pMessageIdName,
|
||||
int32_t messageIdNumber,
|
||||
const char *pMessage);
|
||||
|
||||
void
|
||||
vk_address_binding_report(struct vk_instance *instance,
|
||||
struct vk_object_base *object,
|
||||
uint64_t base_address,
|
||||
uint64_t size,
|
||||
VkDeviceAddressBindingTypeEXT type);
|
||||
|
||||
void
|
||||
vk_emit_device_memory_report(struct vk_device* device,
|
||||
VkDeviceMemoryReportEventTypeEXT type,
|
||||
uint64_t mem_obj_id,
|
||||
VkDeviceSize size,
|
||||
VkObjectType obj_type,
|
||||
uint64_t obj_handle,
|
||||
uint32_t heap_index);
|
||||
|
||||
void
|
||||
vk_queue_begin_debug_utils_label(struct vk_queue *queue,
|
||||
const VkDebugUtilsLabelEXT *pLabelInfo);
|
||||
|
||||
void
|
||||
vk_queue_end_debug_utils_label(struct vk_queue *queue);
|
||||
|
||||
|
||||
struct u_printf_ctx;
|
||||
|
||||
VkResult
|
||||
vk_check_printf_status(struct vk_device *dev, struct u_printf_ctx *ctx);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* VK_DEBUG_UTILS_H */
|
||||
@@ -0,0 +1,85 @@
|
||||
/*
|
||||
* Copyright © 2020 Intel Corporation
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "vk_deferred_operation.h"
|
||||
|
||||
#include "vk_alloc.h"
|
||||
#include "vk_common_entrypoints.h"
|
||||
#include "vk_device.h"
|
||||
|
||||
VKAPI_ATTR VkResult VKAPI_CALL
|
||||
vk_common_CreateDeferredOperationKHR(VkDevice _device,
|
||||
const VkAllocationCallbacks *pAllocator,
|
||||
VkDeferredOperationKHR *pDeferredOperation)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_device, device, _device);
|
||||
|
||||
struct vk_deferred_operation *op =
|
||||
vk_alloc2(&device->alloc, pAllocator, sizeof(*op), 8,
|
||||
VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
|
||||
if (op == NULL)
|
||||
return VK_ERROR_OUT_OF_HOST_MEMORY;
|
||||
|
||||
vk_object_base_init(device, &op->base,
|
||||
VK_OBJECT_TYPE_DEFERRED_OPERATION_KHR);
|
||||
|
||||
*pDeferredOperation = vk_deferred_operation_to_handle(op);
|
||||
|
||||
return VK_SUCCESS;
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_DestroyDeferredOperationKHR(VkDevice _device,
|
||||
VkDeferredOperationKHR operation,
|
||||
const VkAllocationCallbacks *pAllocator)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_device, device, _device);
|
||||
VK_FROM_HANDLE(vk_deferred_operation, op, operation);
|
||||
|
||||
if (op == NULL)
|
||||
return;
|
||||
|
||||
vk_object_base_finish(&op->base);
|
||||
vk_free2(&device->alloc, pAllocator, op);
|
||||
}
|
||||
|
||||
VKAPI_ATTR uint32_t VKAPI_CALL
|
||||
vk_common_GetDeferredOperationMaxConcurrencyKHR(UNUSED VkDevice device,
|
||||
UNUSED VkDeferredOperationKHR operation)
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
|
||||
VKAPI_ATTR VkResult VKAPI_CALL
|
||||
vk_common_GetDeferredOperationResultKHR(UNUSED VkDevice device,
|
||||
UNUSED VkDeferredOperationKHR operation)
|
||||
{
|
||||
return VK_SUCCESS;
|
||||
}
|
||||
|
||||
VKAPI_ATTR VkResult VKAPI_CALL
|
||||
vk_common_DeferredOperationJoinKHR(UNUSED VkDevice device,
|
||||
UNUSED VkDeferredOperationKHR operation)
|
||||
{
|
||||
return VK_SUCCESS;
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
/*
|
||||
* Copyright © 2020 Intel Corporation
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
#ifndef VK_DEFERRED_OPERATION_H
|
||||
#define VK_DEFERRED_OPERATION_H
|
||||
|
||||
#include "vk_object.h"
|
||||
|
||||
#include "c11/threads.h"
|
||||
#include "util/list.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
struct vk_deferred_operation {
|
||||
struct vk_object_base base;
|
||||
};
|
||||
|
||||
VK_DEFINE_NONDISP_HANDLE_CASTS(vk_deferred_operation, base,
|
||||
VkDeferredOperationKHR,
|
||||
VK_OBJECT_TYPE_DEFERRED_OPERATION_KHR)
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* VK_DEFERRED_OPERATION_H */
|
||||
@@ -0,0 +1,100 @@
|
||||
/*
|
||||
* Copyright © 2022 Collabora Ltd
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including the next
|
||||
* paragraph) shall be included in all copies or substantial portions of the
|
||||
* Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "vk_descriptor_set_layout.h"
|
||||
|
||||
#include "vk_alloc.h"
|
||||
#include "vk_common_entrypoints.h"
|
||||
#include "vk_device.h"
|
||||
|
||||
static void
|
||||
vk_descriptor_set_layout_init(struct vk_device *device,
|
||||
struct vk_descriptor_set_layout *layout,
|
||||
const VkDescriptorSetLayoutCreateInfo *pCreateInfo)
|
||||
{
|
||||
vk_object_base_init(device, &layout->base,
|
||||
VK_OBJECT_TYPE_DESCRIPTOR_SET_LAYOUT);
|
||||
|
||||
layout->ref_cnt = 1;
|
||||
layout->flags = pCreateInfo->flags;
|
||||
layout->destroy = vk_descriptor_set_layout_destroy;
|
||||
}
|
||||
|
||||
void *
|
||||
vk_descriptor_set_layout_zalloc(struct vk_device *device, size_t size,
|
||||
const VkDescriptorSetLayoutCreateInfo *pCreateInfo)
|
||||
{
|
||||
/* Because we're reference counting and lifetimes may not be what the
|
||||
* client expects, these have to be allocated off the device and not as
|
||||
* their own object.
|
||||
*/
|
||||
struct vk_descriptor_set_layout *layout =
|
||||
vk_zalloc(&device->alloc, size, 8, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
|
||||
if (!layout)
|
||||
return NULL;
|
||||
|
||||
vk_descriptor_set_layout_init(device, layout, pCreateInfo);
|
||||
|
||||
return layout;
|
||||
}
|
||||
|
||||
void *
|
||||
vk_descriptor_set_layout_multizalloc(struct vk_device *device,
|
||||
struct vk_multialloc *ma,
|
||||
const VkDescriptorSetLayoutCreateInfo *pCreateInfo)
|
||||
{
|
||||
/* Because we're reference counting and lifetimes may not be what the
|
||||
* client expects, these have to be allocated off the device and not as
|
||||
* their own object.
|
||||
*/
|
||||
struct vk_descriptor_set_layout *layout =
|
||||
vk_multialloc_zalloc(ma, &device->alloc,
|
||||
VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
|
||||
if (!layout)
|
||||
return NULL;
|
||||
|
||||
vk_descriptor_set_layout_init(device, layout, pCreateInfo);
|
||||
|
||||
return layout;
|
||||
}
|
||||
|
||||
void
|
||||
vk_descriptor_set_layout_destroy(struct vk_device *device,
|
||||
struct vk_descriptor_set_layout *layout)
|
||||
{
|
||||
vk_object_free(device, NULL, layout);
|
||||
}
|
||||
|
||||
VKAPI_ATTR void VKAPI_CALL
|
||||
vk_common_DestroyDescriptorSetLayout(VkDevice _device,
|
||||
VkDescriptorSetLayout descriptorSetLayout,
|
||||
UNUSED const VkAllocationCallbacks *pAllocator)
|
||||
{
|
||||
VK_FROM_HANDLE(vk_device, device, _device);
|
||||
VK_FROM_HANDLE(vk_descriptor_set_layout, layout, descriptorSetLayout);
|
||||
|
||||
if (layout == NULL)
|
||||
return;
|
||||
|
||||
vk_descriptor_set_layout_unref(device, layout);
|
||||
}
|
||||
Loaded 100 of 12511 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user