init

src/gallium/drivers/asahi/.clang-format

@@ -0,0 +1 @@
+../../../asahi/.clang-format

src/gallium/drivers/asahi/agx_blit.c

@@ -0,0 +1,664 @@
+/*
+ * Copyright 2021 Alyssa Rosenzweig
+ * Copyright 2020-2021 Collabora, Ltd.
+ * Copyright 2019 Sonny Jiang <[email protected]>
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ * Copyright 2014 Broadcom
+ * SPDX-License-Identifier: MIT
+ */
+
+#include <stdint.h>
+#include "asahi/compiler/agx_nir_texture.h"
+#include "asahi/layout/layout.h"
+#include "compiler/nir/nir_builder.h"
+#include "compiler/nir/nir_format_convert.h"
+#include "gallium/auxiliary/util/u_blitter.h"
+#include "gallium/auxiliary/util/u_dump.h"
+#include "nir/pipe_nir.h"
+#include "pipe/p_context.h"
+#include "pipe/p_defines.h"
+#include "pipe/p_state.h"
+#include "util/format/u_format.h"
+#include "util/format/u_formats.h"
+#include "util/hash_table.h"
+#include "util/macros.h"
+#include "util/ralloc.h"
+#include "util/u_sampler.h"
+#include "util/u_surface.h"
+#include "agx_state.h"
+#include "glsl_types.h"
+#include "nir.h"
+#include "nir_builder_opcodes.h"
+#include "shader_enums.h"
+
+/* For block based blit kernels, we hardcode the maximum tile size which we can
+ * always achieve. This simplifies our life.
+ */
+#define TILE_WIDTH  32
+#define TILE_HEIGHT 32
+
+static enum pipe_format
+effective_format(enum pipe_format format)
+{
+   switch (format) {
+   case PIPE_FORMAT_Z32_FLOAT:
+   case PIPE_FORMAT_Z24X8_UNORM:
+      return PIPE_FORMAT_R32_FLOAT;
+   case PIPE_FORMAT_Z16_UNORM:
+      return PIPE_FORMAT_R16_UNORM;
+   case PIPE_FORMAT_S8_UINT:
+      return PIPE_FORMAT_R8_UINT;
+   default:
+      return format;
+   }
+}
+
+static void *
+asahi_blit_compute_shader(struct pipe_context *ctx, struct asahi_blit_key *key)
+{
+   const nir_shader_compiler_options *options =
+      ctx->screen->nir_options[MESA_SHADER_COMPUTE];
+
+   nir_builder b_ =
+      nir_builder_init_simple_shader(MESA_SHADER_COMPUTE, options, "blit_cs");
+   nir_builder *b = &b_;
+   b->shader->info.workgroup_size[0] = TILE_WIDTH;
+   b->shader->info.workgroup_size[1] = TILE_HEIGHT;
+   b->shader->info.num_ubos = 1;
+
+   BITSET_SET(b->shader->info.textures_used, 0);
+   BITSET_SET(b->shader->info.samplers_used, 0);
+   BITSET_SET(b->shader->info.images_used, 0);
+
+   nir_def *zero = nir_imm_int(b, 0);
+
+   nir_def *params[4];
+   b->shader->num_uniforms = ARRAY_SIZE(params);
+   for (unsigned i = 0; i < b->shader->num_uniforms; ++i) {
+      params[i] = nir_load_ubo(b, 2, 32, zero, nir_imm_int(b, i * 8),
+                               .align_mul = 4, .range = ~0);
+   }
+
+   nir_def *trans_offs = params[0];
+   nir_def *trans_scale = params[1];
+   nir_def *dst_offs_2d = params[2];
+   nir_def *dimensions_el_2d = params[3];
+
+   nir_def *phys_id_el_nd = nir_trim_vector(
+      b, nir_load_global_invocation_id(b, 32), key->array ? 3 : 2);
+   nir_def *phys_id_el_2d = nir_trim_vector(b, phys_id_el_nd, 2);
+   nir_def *layer = key->array ? nir_channel(b, phys_id_el_nd, 2) : NULL;
+
+   /* Offset within the tile. We're dispatched for the entire tile but the
+    * beginning might be out-of-bounds, so fix up.
+    */
+   nir_def *offs_in_tile_el_2d = nir_iand_imm(b, dst_offs_2d, 31);
+   nir_def *logical_id_el_2d = nir_isub(b, phys_id_el_2d, offs_in_tile_el_2d);
+
+   nir_def *image_pos_2d = nir_iadd(b, logical_id_el_2d, dst_offs_2d);
+   nir_def *image_pos_nd = image_pos_2d;
+   if (layer) {
+      image_pos_nd =
+         nir_vector_insert_imm(b, nir_pad_vector(b, image_pos_nd, 3), layer, 2);
+   }
+
+   nir_def *in_bounds;
+   if (key->aligned) {
+      in_bounds = nir_imm_true(b);
+   } else {
+      in_bounds = nir_ige(b, logical_id_el_2d, nir_imm_ivec2(b, 0, 0));
+      in_bounds =
+         nir_iand(b, in_bounds, nir_ilt(b, logical_id_el_2d, dimensions_el_2d));
+   }
+
+   unsigned bit_size = 32;
+   nir_alu_type dst_type = nir_type_uint32;
+   if (util_format_is_float16(key->dst_format)) {
+      bit_size = 16;
+      dst_type = nir_type_float16;
+   }
+
+   nir_def *colour0, *colour1;
+   nir_push_if(b, nir_ball(b, in_bounds));
+   {
+      /* For pixels within the copy area, texture from the source */
+      nir_def *coords_el_2d =
+         nir_ffma_weak(b, nir_u2f32(b, logical_id_el_2d), trans_scale, trans_offs);
+
+      nir_def *coords_el_nd = coords_el_2d;
+      if (layer) {
+         coords_el_nd = nir_vector_insert_imm(
+            b, nir_pad_vector(b, coords_el_nd, 3), nir_u2f32(b, layer), 2);
+      }
+
+      colour0 = nir_tex(b, coords_el_nd, .texture_index = 0, .sampler_index = 0,
+                        .backend_flags = AGX_TEXTURE_FLAG_NO_CLAMP,
+                        .dim = GLSL_SAMPLER_DIM_2D, .is_array = key->array,
+                        .dest_type = dst_type);
+   }
+   nir_push_else(b, NULL);
+   {
+      /* For out-of-bounds pixels, copy in the destination */
+      colour1 = nir_image_load(
+         b, 4, bit_size, nir_imm_int(b, 0), nir_pad_vec4(b, image_pos_nd), zero, zero,
+         .image_array = key->array, .image_dim = GLSL_SAMPLER_DIM_2D,
+         .access = ACCESS_IN_BOUNDS, .dest_type = dst_type);
+   }
+   nir_pop_if(b, NULL);
+   nir_def *color = nir_if_phi(b, colour0, colour1);
+
+   enum asahi_blit_clamp clamp = ASAHI_BLIT_CLAMP_NONE;
+   bool src_sint = util_format_is_pure_sint(key->src_format);
+   bool dst_sint = util_format_is_pure_sint(key->dst_format);
+   if (util_format_is_pure_integer(key->src_format) &&
+       util_format_is_pure_integer(key->dst_format)) {
+
+      if (src_sint && !dst_sint)
+         clamp = ASAHI_BLIT_CLAMP_SINT_TO_UINT;
+      else if (!src_sint && dst_sint)
+         clamp = ASAHI_BLIT_CLAMP_UINT_TO_SINT;
+   }
+
+   if (clamp == ASAHI_BLIT_CLAMP_SINT_TO_UINT)
+      color = nir_imax(b, color, nir_imm_int(b, 0));
+   else if (clamp == ASAHI_BLIT_CLAMP_UINT_TO_SINT)
+      color = nir_umin(b, color, nir_imm_int(b, INT32_MAX));
+
+   nir_def *local_offset = nir_imm_int(b, 0);
+   nir_def *lid = nir_trim_vector(b, nir_load_local_invocation_id(b), 2);
+   lid = nir_u2u16(b, lid);
+
+   /* Pure integer formatss need to be clamped in software, at least in some
+    * cases. We do so on store. Piglit gl-3.0-render-integer checks this, as
+    * does KHR-GL33.packed_pixels.*.
+    *
+    * TODO: Make this common code somehow.
+    */
+   const struct util_format_description *desc =
+      util_format_description(key->dst_format);
+   unsigned c = util_format_get_first_non_void_channel(key->dst_format);
+
+   if (desc->channel[c].size <= 16 &&
+       util_format_is_pure_integer(key->dst_format)) {
+
+      unsigned bits[4] = {
+         desc->channel[0].size ?: desc->channel[0].size,
+         desc->channel[1].size ?: desc->channel[0].size,
+         desc->channel[2].size ?: desc->channel[0].size,
+         desc->channel[3].size ?: desc->channel[0].size,
+      };
+
+      if (util_format_is_pure_sint(key->dst_format))
+         color = nir_format_clamp_sint(b, color, bits);
+      else
+         color = nir_format_clamp_uint(b, color, bits);
+
+      color = nir_u2u16(b, color);
+   }
+
+   /* The source texel has been converted into a 32-bit value. We need to
+    * convert it to a tilebuffer format that can then be converted to the
+    * destination format in the PBE hardware. That's the renderable format for
+    * the destination format, which must exist along this path. This mirrors the
+    * flow of fragment and end-of-tile shaders.
+    */
+   enum pipe_format tib_format =
+      ail_pixel_format[effective_format(key->dst_format)].renderable;
+
+   nir_store_local_pixel_agx(b, color, nir_imm_intN_t(b, 1, 16), lid, .base = 0,
+                             .write_mask = 0xf, .format = tib_format,
+                             .explicit_coord = true);
+
+   nir_barrier(b, .execution_scope = SCOPE_WORKGROUP);
+
+   nir_push_if(b, nir_ball(b, nir_ieq_imm(b, lid, 0)));
+   {
+      nir_def *pbe_index = nir_imm_intN_t(b, 2, 16);
+      nir_image_store_block_agx(
+         b, pbe_index, local_offset, image_pos_nd, .format = tib_format,
+         .image_dim = GLSL_SAMPLER_DIM_2D, .image_array = key->array,
+         .explicit_coord = true);
+   }
+   nir_pop_if(b, NULL);
+   b->shader->info.cs.image_block_size_per_thread_agx =
+      util_format_get_blocksize(key->dst_format);
+
+   return pipe_shader_from_nir(ctx, b->shader);
+}
+
+static bool
+asahi_compute_blit_supported(const struct pipe_blit_info *info)
+{
+   return (info->src.box.depth == info->dst.box.depth) && !info->alpha_blend &&
+          !info->num_window_rectangles && !info->sample0_only &&
+          !info->scissor_enable && !info->window_rectangle_include &&
+          !info->swizzle_enable && info->src.resource->nr_samples <= 1 &&
+          info->dst.resource->nr_samples <= 1 &&
+          !util_format_is_depth_and_stencil(info->src.format) &&
+          !util_format_is_depth_and_stencil(info->dst.format) &&
+          info->src.box.depth >= 0 &&
+          info->mask == util_format_get_mask(info->src.format) &&
+          /* XXX: texsubimage pbo failing otherwise, needs investigation */
+          info->dst.format != PIPE_FORMAT_B5G6R5_UNORM &&
+          info->dst.format != PIPE_FORMAT_B5G5R5A1_UNORM &&
+          info->dst.format != PIPE_FORMAT_B5G5R5X1_UNORM &&
+          info->dst.format != PIPE_FORMAT_R5G6B5_UNORM &&
+          info->dst.format != PIPE_FORMAT_R5G5B5A1_UNORM &&
+          info->dst.format != PIPE_FORMAT_R5G5B5X1_UNORM;
+}
+
+static void
+asahi_compute_save(struct agx_context *ctx)
+{
+   struct asahi_blitter *blitter = &ctx->compute_blitter;
+   struct agx_stage *stage = &ctx->stage[MESA_SHADER_COMPUTE];
+
+   assert(!blitter->active && "recursion detected, driver bug");
+
+   pipe_resource_reference(&blitter->saved_cb.buffer, stage->cb[0].buffer);
+   memcpy(&blitter->saved_cb, &stage->cb[0],
+          sizeof(struct pipe_constant_buffer));
+
+   blitter->has_saved_image = stage->image_mask & BITFIELD_BIT(0);
+   if (blitter->has_saved_image) {
+      pipe_resource_reference(&blitter->saved_image.resource,
+                              stage->images[0].resource);
+      memcpy(&blitter->saved_image, &stage->images[0],
+             sizeof(struct pipe_image_view));
+   }
+
+   pipe_sampler_view_reference(&blitter->saved_sampler_view,
+                               &stage->textures[0]->base);
+
+   blitter->saved_num_sampler_states = stage->sampler_count;
+   memcpy(blitter->saved_sampler_states, stage->samplers,
+          stage->sampler_count * sizeof(void *));
+
+   blitter->saved_cs = stage->shader;
+   blitter->active = true;
+}
+
+static void
+asahi_compute_restore(struct agx_context *ctx)
+{
+   struct pipe_context *pctx = &ctx->base;
+   struct asahi_blitter *blitter = &ctx->compute_blitter;
+
+   if (blitter->has_saved_image) {
+      pctx->set_shader_images(pctx, MESA_SHADER_COMPUTE, 0, 1, 0,
+                              &blitter->saved_image);
+      pipe_resource_reference(&blitter->saved_image.resource, NULL);
+   }
+
+   /* take_ownership=true so do not unreference */
+   pctx->set_constant_buffer(pctx, MESA_SHADER_COMPUTE, 0, &blitter->saved_cb);
+   blitter->saved_cb.buffer = NULL;
+
+   if (blitter->saved_sampler_view) {
+      pctx->set_sampler_views(pctx, MESA_SHADER_COMPUTE, 0, 1, 0,
+                              &blitter->saved_sampler_view);
+
+      blitter->saved_sampler_view = NULL;
+   }
+
+   if (blitter->saved_num_sampler_states) {
+      pctx->bind_sampler_states(pctx, MESA_SHADER_COMPUTE, 0,
+                                blitter->saved_num_sampler_states,
+                                blitter->saved_sampler_states);
+   }
+
+   pctx->bind_compute_state(pctx, blitter->saved_cs);
+   blitter->saved_cs = NULL;
+   blitter->active = false;
+}
+
+static void
+asahi_compute_blit(struct pipe_context *ctx, const struct pipe_blit_info *info,
+                   struct asahi_blitter *blitter)
+{
+   if (info->src.box.width == 0 || info->src.box.height == 0 ||
+       info->dst.box.width == 0 || info->dst.box.height == 0)
+      return;
+
+   assert(asahi_compute_blit_supported(info));
+   asahi_compute_save(agx_context(ctx));
+
+   unsigned depth = info->dst.box.depth;
+   bool array = depth > 1;
+
+   struct pipe_resource *src = info->src.resource;
+   struct pipe_resource *dst = info->dst.resource;
+   struct pipe_sampler_view src_templ = {0}, *src_view;
+
+   float src_width = (float)u_minify(src->width0, info->src.level);
+   float src_height = (float)u_minify(src->height0, info->src.level);
+
+   float x_scale =
+      (info->src.box.width / (float)info->dst.box.width) / src_width;
+
+   float y_scale =
+      (info->src.box.height / (float)info->dst.box.height) / src_height;
+
+   /* Expand the grid so destinations are in tiles */
+   unsigned expanded_x0 = info->dst.box.x & ~(TILE_WIDTH - 1);
+   unsigned expanded_y0 = info->dst.box.y & ~(TILE_HEIGHT - 1);
+   unsigned expanded_x1 =
+      align(info->dst.box.x + info->dst.box.width, TILE_WIDTH);
+   unsigned expanded_y1 =
+      align(info->dst.box.y + info->dst.box.height, TILE_HEIGHT);
+
+   /* But clamp to the destination size to save some redundant threads */
+   expanded_x1 =
+      MIN2(expanded_x1, u_minify(info->dst.resource->width0, info->dst.level));
+   expanded_y1 =
+      MIN2(expanded_y1, u_minify(info->dst.resource->height0, info->dst.level));
+
+   /* Calculate the width/height based on the expanded grid */
+   unsigned width = expanded_x1 - expanded_x0;
+   unsigned height = expanded_y1 - expanded_y0;
+
+   unsigned data[] = {
+      fui(0.5f * x_scale + (float)info->src.box.x / src_width),
+      fui(0.5f * y_scale + (float)info->src.box.y / src_height),
+      fui(x_scale),
+      fui(y_scale),
+      info->dst.box.x,
+      info->dst.box.y,
+      info->dst.box.width,
+      info->dst.box.height,
+   };
+
+   struct pipe_constant_buffer cb = {
+      .buffer_size = sizeof(data),
+      .user_buffer = data,
+   };
+   ctx->set_constant_buffer(ctx, MESA_SHADER_COMPUTE, 0, &cb);
+
+   struct pipe_image_view image = {
+      .resource = dst,
+      .access = PIPE_IMAGE_ACCESS_WRITE | PIPE_IMAGE_ACCESS_DRIVER_INTERNAL,
+      .shader_access = PIPE_IMAGE_ACCESS_WRITE,
+      .format = info->dst.format,
+      .u.tex.level = info->dst.level,
+      .u.tex.first_layer = info->dst.box.z,
+      .u.tex.last_layer = info->dst.box.z + depth - 1,
+      .u.tex.single_layer_view = !array,
+   };
+   ctx->set_shader_images(ctx, MESA_SHADER_COMPUTE, 0, 1, 0, &image);
+
+   if (!blitter->sampler[info->filter]) {
+      struct pipe_sampler_state sampler_state = {
+         .wrap_s = PIPE_TEX_WRAP_CLAMP_TO_EDGE,
+         .wrap_t = PIPE_TEX_WRAP_CLAMP_TO_EDGE,
+         .wrap_r = PIPE_TEX_WRAP_CLAMP_TO_EDGE,
+         .min_img_filter = info->filter,
+         .mag_img_filter = info->filter,
+         .compare_func = PIPE_FUNC_ALWAYS,
+         .seamless_cube_map = true,
+         .max_lod = 31.0f,
+      };
+
+      blitter->sampler[info->filter] =
+         ctx->create_sampler_state(ctx, &sampler_state);
+   }
+
+   ctx->bind_sampler_states(ctx, MESA_SHADER_COMPUTE, 0, 1,
+                            &blitter->sampler[info->filter]);
+
+   /* Initialize the sampler view. */
+   u_sampler_view_default_template(&src_templ, src, src->format);
+   src_templ.format = info->src.format;
+   src_templ.target = array ? PIPE_TEXTURE_2D_ARRAY : PIPE_TEXTURE_2D;
+   src_templ.swizzle_r = PIPE_SWIZZLE_X;
+   src_templ.swizzle_g = PIPE_SWIZZLE_Y;
+   src_templ.swizzle_b = PIPE_SWIZZLE_Z;
+   src_templ.swizzle_a = PIPE_SWIZZLE_W;
+   src_templ.u.tex.first_layer = info->src.box.z;
+   src_templ.u.tex.last_layer = info->src.box.z + depth - 1;
+   src_templ.u.tex.first_level = info->src.level;
+   src_templ.u.tex.last_level = info->src.level;
+   src_view = ctx->create_sampler_view(ctx, src, &src_templ);
+   ctx->set_sampler_views(ctx, MESA_SHADER_COMPUTE, 0, 1, 0, &src_view);
+   ctx->sampler_view_release(ctx, src_view);
+
+   struct asahi_blit_key key = {
+      .src_format = info->src.format,
+      .dst_format = info->dst.format,
+      .array = array,
+      .aligned = info->dst.box.width == width && info->dst.box.height == height,
+   };
+   struct hash_entry *ent = _mesa_hash_table_search(blitter->blit_cs, &key);
+   void *cs = NULL;
+
+   if (ent) {
+      cs = ent->data;
+   } else {
+      cs = asahi_blit_compute_shader(ctx, &key);
+      _mesa_hash_table_insert(
+         blitter->blit_cs, ralloc_memdup(blitter->blit_cs, &key, sizeof(key)),
+         cs);
+   }
+
+   assert(cs != NULL);
+   ctx->bind_compute_state(ctx, cs);
+
+   struct pipe_grid_info grid_info = {
+      .block = {TILE_WIDTH, TILE_HEIGHT, 1},
+      .last_block = {width % TILE_WIDTH, height % TILE_HEIGHT, 1},
+      .grid =
+         {
+            DIV_ROUND_UP(width, TILE_WIDTH),
+            DIV_ROUND_UP(height, TILE_HEIGHT),
+            depth,
+         },
+   };
+   ctx->launch_grid(ctx, &grid_info);
+   ctx->set_shader_images(ctx, MESA_SHADER_COMPUTE, 0, 0, 1, NULL);
+   ctx->set_constant_buffer(ctx, MESA_SHADER_COMPUTE, 0, NULL);
+   ctx->set_sampler_views(ctx, MESA_SHADER_COMPUTE, 0, 0, 1, NULL);
+
+   asahi_compute_restore(agx_context(ctx));
+}
+
+void
+agx_blitter_save(struct agx_context *ctx, struct blitter_context *blitter,
+                 enum asahi_blitter_op op)
+{
+   util_blitter_save_vertex_shader(blitter,
+                                   ctx->stage[MESA_SHADER_VERTEX].shader);
+   util_blitter_save_tessctrl_shader(blitter,
+                                     ctx->stage[MESA_SHADER_TESS_CTRL].shader);
+   util_blitter_save_tesseval_shader(blitter,
+                                     ctx->stage[MESA_SHADER_TESS_EVAL].shader);
+   util_blitter_save_geometry_shader(blitter,
+                                     ctx->stage[MESA_SHADER_GEOMETRY].shader);
+   util_blitter_save_so_targets(blitter, ctx->streamout.num_targets,
+                                ctx->streamout.targets, MESA_PRIM_UNKNOWN);
+   util_blitter_save_vertex_buffers(blitter, ctx->vertex_buffers,
+                                    util_last_bit(ctx->vb_mask));
+   util_blitter_save_vertex_elements(blitter, ctx->attributes);
+   util_blitter_save_rasterizer(blitter, ctx->rast);
+   util_blitter_save_viewport(blitter, &ctx->viewport[0]);
+
+   if (op & ASAHI_SAVE_FRAGMENT_STATE) {
+      util_blitter_save_blend(blitter, ctx->blend);
+      util_blitter_save_depth_stencil_alpha(blitter, ctx->zs);
+      util_blitter_save_stencil_ref(blitter, &ctx->stencil_ref);
+      util_blitter_save_sample_mask(blitter, ctx->sample_mask, 0);
+
+      util_blitter_save_scissor(blitter, &ctx->scissor[0]);
+      util_blitter_save_fragment_shader(
+         blitter, ctx->stage[MESA_SHADER_FRAGMENT].shader);
+
+      if (op & ASAHI_SAVE_FRAGMENT_CONSTANT) {
+         util_blitter_save_fragment_constant_buffer_slot(
+            blitter, ctx->stage[MESA_SHADER_FRAGMENT].cb);
+      }
+   }
+
+   if (op & ASAHI_SAVE_FRAMEBUFFER) {
+      util_blitter_save_framebuffer(blitter, &ctx->framebuffer);
+   }
+
+   if (op & ASAHI_SAVE_TEXTURES) {
+      util_blitter_save_fragment_sampler_states(
+         blitter, ctx->stage[MESA_SHADER_FRAGMENT].sampler_count,
+         (void **)(ctx->stage[MESA_SHADER_FRAGMENT].samplers));
+
+      util_blitter_save_fragment_sampler_views(
+         blitter, ctx->stage[MESA_SHADER_FRAGMENT].texture_count,
+         (struct pipe_sampler_view **)ctx->stage[MESA_SHADER_FRAGMENT].textures);
+   }
+
+   if (!(op & ASAHI_DISABLE_RENDER_COND)) {
+      util_blitter_save_render_condition(blitter,
+                                         (struct pipe_query *)ctx->cond_query,
+                                         ctx->cond_cond, ctx->cond_mode);
+   }
+}
+
+void
+agx_blit(struct pipe_context *pipe, const struct pipe_blit_info *info)
+{
+   struct agx_context *ctx = agx_context(pipe);
+
+   if (info->render_condition_enable && !agx_render_condition_check(ctx))
+      return;
+
+   /* Legalize compression /before/ calling into u_blitter to avoid recursion.
+    * u_blitter bans recursive usage.
+    */
+   agx_legalize_compression(ctx, agx_resource(info->dst.resource),
+                            info->dst.format);
+
+   agx_legalize_compression(ctx, agx_resource(info->src.resource),
+                            info->src.format);
+
+   if (asahi_compute_blit_supported(info)) {
+      asahi_compute_blit(pipe, info, &ctx->compute_blitter);
+      return;
+   }
+
+   if (!util_blitter_is_blit_supported(ctx->blitter, info)) {
+      fprintf(stderr, "\n");
+      util_dump_blit_info(stderr, info);
+      fprintf(stderr, "\n\n");
+      UNREACHABLE("Unsupported blit");
+   }
+
+   /* Handle self-blits */
+   agx_flush_writer(ctx, agx_resource(info->dst.resource), "Blit");
+
+   agx_blitter_save(
+      ctx, ctx->blitter,
+      ASAHI_BLIT |
+         (info->render_condition_enable ? 0 : ASAHI_DISABLE_RENDER_COND));
+   util_blitter_blit(ctx->blitter, info, NULL);
+}
+
+static bool
+try_copy_via_blit(struct pipe_context *pctx, struct pipe_resource *dst,
+                  unsigned dst_level, unsigned dstx, unsigned dsty,
+                  unsigned dstz, struct pipe_resource *src, unsigned src_level,
+                  const struct pipe_box *src_box)
+{
+   struct agx_context *ctx = agx_context(pctx);
+
+   if (dst->target == PIPE_BUFFER)
+      return false;
+
+   /* TODO: Handle these for rusticl copies */
+   if (dst->target != src->target)
+      return false;
+
+   struct pipe_blit_info info = {
+      .dst =
+         {
+            .resource = dst,
+            .level = dst_level,
+            .box.x = dstx,
+            .box.y = dsty,
+            .box.z = dstz,
+            .box.width = src_box->width,
+            .box.height = src_box->height,
+            .box.depth = src_box->depth,
+            .format = dst->format,
+         },
+      .src =
+         {
+            .resource = src,
+            .level = src_level,
+            .box = *src_box,
+            .format = src->format,
+         },
+      .mask = util_format_get_mask(src->format),
+      .filter = PIPE_TEX_FILTER_NEAREST,
+      .scissor_enable = 0,
+      .swizzle_enable = 0,
+   };
+
+   /* snorm formats don't round trip, so don't use them for copies */
+   if (util_format_is_snorm(info.dst.format))
+      info.dst.format = util_format_snorm_to_sint(info.dst.format);
+
+   if (util_format_is_snorm(info.src.format))
+      info.src.format = util_format_snorm_to_sint(info.src.format);
+
+   if (util_blitter_is_blit_supported(ctx->blitter, &info) &&
+       info.dst.format == info.src.format) {
+
+      agx_blit(pctx, &info);
+      return true;
+   } else {
+      return false;
+   }
+}
+
+void
+agx_resource_copy_region(struct pipe_context *pctx, struct pipe_resource *dst,
+                         unsigned dst_level, unsigned dstx, unsigned dsty,
+                         unsigned dstz, struct pipe_resource *src,
+                         unsigned src_level, const struct pipe_box *src_box)
+{
+   if (dst->target == PIPE_BUFFER && src->target == PIPE_BUFFER) {
+      struct agx_batch *batch = agx_get_compute_batch(agx_context(pctx));
+      agx_batch_init_state(batch);
+      assert(dst->format == src->format);
+      unsigned bs = util_format_get_blocksize(dst->format);
+      unsigned size = bs * src_box->width;
+      unsigned dst_offset = dstx * bs;
+      uint64_t dst_addr = agx_map_gpu(agx_resource(dst)) + dst_offset;
+      uint64_t src_addr = agx_map_gpu(agx_resource(src)) + src_box->x * bs;
+
+      agx_batch_reads(batch, agx_resource(src));
+      agx_batch_writes_range(batch, agx_resource(dst), dst_offset, size);
+      /* Use vectorized copies for as much of the buffer as possible. This requires
+       * that dst, src, and size are all properly aligned. Failing to check for
+       * alignment on the buffers causes subtle and hard-to-debug issues!
+       */
+      if (size >= 16 && (dst_addr & 0xf) == 0 && (src_addr & 0xf) == 0) {
+         unsigned uint4s = size / 16;
+         unsigned bytes = uint4s * 16;
+
+         libagx_copy_uint4(batch, agx_1d(uint4s), AGX_BARRIER_ALL, dst_addr, src_addr);
+
+         dst_addr += bytes;
+         src_addr += bytes;
+         size -= bytes;
+      }
+
+      if (size) {
+         libagx_copy_uchar(batch, agx_1d(size), AGX_BARRIER_ALL, dst_addr, src_addr);
+      }
+
+      return;
+   }
+
+   if (try_copy_via_blit(pctx, dst, dst_level, dstx, dsty, dstz, src, src_level,
+                         src_box))
+      return;
+
+   /* CPU fallback */
+   util_resource_copy_region(pctx, dst, dst_level, dstx, dsty, dstz, src,
+                             src_level, src_box);
+}

src/gallium/drivers/asahi/agx_disk_cache.c

@@ -0,0 +1,224 @@
+/*
+ * Copyright 2023 Rose Hudson
+ * Copyright 2022 Amazon.com, Inc. or its affiliates.
+ * Copyright 2018 Intel Corporation
+ * SPDX-License-Identifier: MIT
+ */
+
+#include <assert.h>
+#include <stdio.h>
+
+#include "asahi/compiler/agx_debug.h"
+#include "compiler/shader_enums.h"
+#include "util/blob.h"
+#include "util/build_id.h"
+#include "util/disk_cache.h"
+#include "util/mesa-blake3.h"
+#include "agx_bo.h"
+#include "agx_device.h"
+#include "agx_disk_cache.h"
+#include "agx_state.h"
+
+/* Flags that are allowed and do not disable the disk cache */
+#define ALLOWED_FLAGS (AGX_DBG_NO16)
+
+/**
+ * Compute a disk cache key for the given uncompiled shader and shader key.
+ */
+static void
+agx_disk_cache_compute_key(struct disk_cache *cache,
+                           const struct agx_uncompiled_shader *uncompiled,
+                           const union asahi_shader_key *shader_key,
+                           cache_key cache_key)
+{
+   uint8_t data[sizeof(uncompiled->nir_blake3) + sizeof(*shader_key)];
+   int hash_size = sizeof(uncompiled->nir_blake3);
+   int key_size;
+   if (uncompiled->type == MESA_SHADER_VERTEX ||
+       uncompiled->type == MESA_SHADER_TESS_EVAL)
+      key_size = sizeof(shader_key->vs);
+   else if (uncompiled->type == MESA_SHADER_FRAGMENT)
+      key_size = sizeof(shader_key->fs);
+   else
+      key_size = 0;
+
+   memcpy(data, uncompiled->nir_blake3, hash_size);
+
+   if (key_size)
+      memcpy(data + hash_size, shader_key, key_size);
+
+   disk_cache_compute_key(cache, data, hash_size + key_size, cache_key);
+}
+
+static void
+write_shader(struct blob *blob, const struct agx_compiled_shader *binary,
+             bool is_root_gs)
+{
+   blob_write_bytes(blob, &binary->b.info, sizeof(binary->b.info));
+
+   if (binary->b.info.binary_size) {
+      blob_write_bytes(blob, binary->b.binary, binary->b.info.binary_size);
+   }
+
+   blob_write_bytes(blob, &binary->uvs, sizeof(binary->uvs));
+   blob_write_bytes(blob, &binary->attrib_components_read,
+                    sizeof(binary->attrib_components_read));
+   blob_write_bytes(blob, &binary->epilog_key, sizeof(binary->epilog_key));
+   blob_write_uint32(blob, binary->push_range_count);
+   blob_write_bytes(blob, binary->push,
+                    sizeof(binary->push[0]) * binary->push_range_count);
+
+   if (is_root_gs) {
+      blob_write_bytes(blob, &binary->gs, sizeof(binary->gs));
+      write_shader(blob, binary->pre_gs, false);
+
+      blob_write_uint8(blob, binary->gs_copy != NULL);
+      if (binary->gs_copy)
+         write_shader(blob, binary->gs_copy, false);
+
+      blob_write_uint8(blob, binary->gs_count != NULL);
+      if (binary->gs_count)
+         write_shader(blob, binary->gs_count, false);
+   }
+}
+
+static struct agx_compiled_shader *
+read_shader(struct agx_screen *screen, struct blob_reader *blob,
+            const struct agx_uncompiled_shader *uncompiled, bool is_root)
+{
+   struct agx_compiled_shader *binary = CALLOC_STRUCT(agx_compiled_shader);
+   binary->stage = uncompiled->type;
+   binary->so = uncompiled;
+
+   blob_copy_bytes(blob, &binary->b.info, sizeof(binary->b.info));
+   size_t size = binary->b.info.binary_size;
+
+   if (uncompiled->type == MESA_SHADER_VERTEX ||
+       uncompiled->type == MESA_SHADER_TESS_EVAL ||
+       uncompiled->type == MESA_SHADER_FRAGMENT) {
+
+      binary->b.binary = malloc(size);
+      blob_copy_bytes(blob, binary->b.binary, size);
+
+      if (size) {
+         binary->bo = agx_bo_create(&screen->dev, size, 0,
+                                    AGX_BO_EXEC | AGX_BO_LOW_VA, "Executable");
+         memcpy(agx_bo_map(binary->bo), binary->b.binary, size);
+      }
+   } else if (size) {
+      binary->bo = agx_bo_create(&screen->dev, size, 0,
+                                 AGX_BO_EXEC | AGX_BO_LOW_VA, "Executable");
+      blob_copy_bytes(blob, agx_bo_map(binary->bo), size);
+   }
+
+   blob_copy_bytes(blob, &binary->uvs, sizeof(binary->uvs));
+   blob_copy_bytes(blob, &binary->attrib_components_read,
+                   sizeof(binary->attrib_components_read));
+   blob_copy_bytes(blob, &binary->epilog_key, sizeof(binary->epilog_key));
+   binary->push_range_count = blob_read_uint32(blob);
+   blob_copy_bytes(blob, binary->push,
+                   sizeof(binary->push[0]) * binary->push_range_count);
+
+   if (is_root && uncompiled->type == MESA_SHADER_GEOMETRY) {
+      blob_copy_bytes(blob, &binary->gs, sizeof(binary->gs));
+      binary->pre_gs = read_shader(screen, blob, uncompiled, false);
+
+      if (blob_read_uint8(blob))
+         binary->gs_copy = read_shader(screen, blob, uncompiled, false);
+
+      if (blob_read_uint8(blob))
+         binary->gs_count = read_shader(screen, blob, uncompiled, false);
+   }
+
+   return binary;
+}
+
+/**
+ * Store the given compiled shader in the disk cache.
+ *
+ * This should only be called on newly compiled shaders.  No checking is
+ * done to prevent repeated stores of the same shader.
+ */
+void
+agx_disk_cache_store(struct disk_cache *cache,
+                     const struct agx_uncompiled_shader *uncompiled,
+                     const union asahi_shader_key *key,
+                     const struct agx_compiled_shader *binary)
+{
+#ifdef ENABLE_SHADER_CACHE
+   if (!cache)
+      return;
+
+   cache_key cache_key;
+   agx_disk_cache_compute_key(cache, uncompiled, key, cache_key);
+
+   struct blob blob;
+   blob_init(&blob);
+
+   write_shader(&blob, binary, uncompiled->type == MESA_SHADER_GEOMETRY);
+
+   disk_cache_put(cache, cache_key, blob.data, blob.size, NULL);
+   blob_finish(&blob);
+#endif
+}
+
+/**
+ * Search for a compiled shader in the disk cache.
+ */
+struct agx_compiled_shader *
+agx_disk_cache_retrieve(struct agx_screen *screen,
+                        const struct agx_uncompiled_shader *uncompiled,
+                        const union asahi_shader_key *key)
+{
+#ifdef ENABLE_SHADER_CACHE
+   struct disk_cache *cache = screen->disk_cache;
+   if (!cache)
+      return NULL;
+
+   cache_key cache_key;
+   agx_disk_cache_compute_key(cache, uncompiled, key, cache_key);
+
+   size_t size;
+   void *buffer = disk_cache_get(cache, cache_key, &size);
+   if (!buffer)
+      return NULL;
+
+   struct blob_reader blob;
+   blob_reader_init(&blob, buffer, size);
+
+   struct agx_compiled_shader *binary =
+      read_shader(screen, &blob, uncompiled, true);
+
+   free(buffer);
+   return binary;
+#else
+   return NULL;
+#endif
+}
+
+/**
+ * Initialise the on-disk shader cache.
+ */
+void
+agx_disk_cache_init(struct agx_screen *screen)
+{
+#ifdef ENABLE_SHADER_CACHE
+   if (agx_get_compiler_debug() || (screen->dev.debug & ~ALLOWED_FLAGS))
+      return;
+
+   const char *renderer = screen->pscreen.get_name(&screen->pscreen);
+
+   const struct build_id_note *note =
+      build_id_find_nhdr_for_addr(agx_disk_cache_init);
+   assert(note && build_id_length(note) == BUILD_ID_EXPECTED_HASH_LENGTH);
+
+   const uint8_t *id_sha1 = build_id_data(note);
+   assert(id_sha1);
+
+   char timestamp[BLAKE3_HEX_LEN];
+   _mesa_sha1_format(timestamp, id_sha1);
+
+   uint64_t driver_flags = screen->dev.debug;
+   screen->disk_cache = disk_cache_create(renderer, timestamp, driver_flags);
+#endif
+}

src/gallium/drivers/asahi/agx_disk_cache.h

@@ -0,0 +1,23 @@
+/*
+ * Copyright 2023 Rose Hudson
+ * Copyright 2018 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#pragma once
+
+#include "util/disk_cache.h"
+#include "agx_state.h"
+
+void agx_disk_cache_store(struct disk_cache *cache,
+                          const struct agx_uncompiled_shader *uncompiled,
+                          const union asahi_shader_key *key,
+                          const struct agx_compiled_shader *binary);
+
+struct agx_compiled_shader *
+agx_disk_cache_retrieve(struct agx_screen *screen,
+                        const struct agx_uncompiled_shader *uncompiled,
+                        const union asahi_shader_key *key);
+
+void agx_disk_cache_init(struct agx_screen *screen);

src/gallium/drivers/asahi/agx_fence.c

@@ -0,0 +1,163 @@
+/*
+ * Based on panfrost/pan_fence.c:
+ *
+ * Copyright 2022 Amazon.com, Inc. or its affiliates.
+ * Copyright 2008 VMware, Inc.
+ * Copyright 2014 Broadcom
+ * Copyright 2018 Alyssa Rosenzweig
+ * Copyright 2019 Collabora, Ltd.
+ * Copyright 2012 Rob Clark
+ * SPDX-License-Identifier: MIT
+ */
+
+#include <xf86drm.h>
+
+#include "agx_fence.h"
+#include "agx_state.h"
+
+#include "util/libsync.h"
+#include "util/os_time.h"
+#include "util/u_inlines.h"
+
+void
+agx_fence_reference(struct pipe_screen *pscreen, struct pipe_fence_handle **ptr,
+                    struct pipe_fence_handle *fence)
+{
+   struct agx_device *dev = agx_device(pscreen);
+   struct pipe_fence_handle *old = *ptr;
+
+   if (pipe_reference(old ? &old->reference : NULL,
+                      fence ? &fence->reference : NULL)) {
+      drmSyncobjDestroy(dev->fd, old->syncobj);
+      free(old);
+   }
+
+   *ptr = fence;
+}
+
+bool
+agx_fence_finish(struct pipe_screen *pscreen, struct pipe_context *ctx,
+                 struct pipe_fence_handle *fence, uint64_t timeout)
+{
+   struct agx_device *dev = agx_device(pscreen);
+   int ret;
+
+   if (fence->signaled)
+      return true;
+
+   uint64_t abs_timeout = os_time_get_absolute_timeout(timeout);
+   if (abs_timeout == OS_TIMEOUT_INFINITE)
+      abs_timeout = INT64_MAX;
+
+   ret = drmSyncobjWait(dev->fd, &fence->syncobj, 1, abs_timeout,
+                        DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL, NULL);
+
+   assert(ret >= 0 || ret == -ETIME);
+   fence->signaled = (ret >= 0);
+   return fence->signaled;
+}
+
+int
+agx_fence_get_fd(struct pipe_screen *screen, struct pipe_fence_handle *f)
+{
+   struct agx_device *dev = agx_device(screen);
+   int fd = -1;
+
+   int ret = drmSyncobjExportSyncFile(dev->fd, f->syncobj, &fd);
+   assert(ret >= 0);
+   assert(fd >= 0);
+
+   return fd;
+}
+
+struct pipe_fence_handle *
+agx_fence_from_fd(struct agx_context *ctx, int fd, enum pipe_fd_type type)
+{
+   struct agx_device *dev = agx_device(ctx->base.screen);
+   int ret;
+
+   struct pipe_fence_handle *f = calloc(1, sizeof(*f));
+   if (!f)
+      return NULL;
+
+   if (type == PIPE_FD_TYPE_NATIVE_SYNC) {
+      ret = drmSyncobjCreate(dev->fd, 0, &f->syncobj);
+      if (ret) {
+         agx_msg("create syncobj failed\n");
+         goto err_free_fence;
+      }
+
+      ret = drmSyncobjImportSyncFile(dev->fd, f->syncobj, fd);
+      if (ret) {
+         agx_msg("import syncfile failed\n");
+         goto err_destroy_syncobj;
+      }
+   } else {
+      assert(type == PIPE_FD_TYPE_SYNCOBJ);
+      ret = drmSyncobjFDToHandle(dev->fd, fd, &f->syncobj);
+      if (ret) {
+         agx_msg("import syncobj FD failed\n");
+         goto err_free_fence;
+      }
+   }
+
+   pipe_reference_init(&f->reference, 1);
+
+   return f;
+
+err_destroy_syncobj:
+   drmSyncobjDestroy(dev->fd, f->syncobj);
+err_free_fence:
+   free(f);
+   return NULL;
+}
+
+struct pipe_fence_handle *
+agx_fence_create(struct agx_context *ctx)
+{
+   struct agx_device *dev = agx_device(ctx->base.screen);
+   int fd = -1, ret;
+
+   /* Snapshot the last rendering out fence. We'd rather have another
+    * syncobj instead of a sync file, but this is all we get.
+    * (HandleToFD/FDToHandle just gives you another syncobj ID for the
+    * same syncobj).
+    */
+   ret = drmSyncobjExportSyncFile(dev->fd, ctx->syncobj, &fd);
+   assert(ret >= 0 && fd != -1 && "export failed");
+   if (ret || fd == -1) {
+      agx_msg("export failed\n");
+      return NULL;
+   }
+
+   struct pipe_fence_handle *f =
+      agx_fence_from_fd(ctx, fd, PIPE_FD_TYPE_NATIVE_SYNC);
+
+   close(fd);
+
+   return f;
+}
+
+void
+agx_create_fence_fd(struct pipe_context *pctx,
+                    struct pipe_fence_handle **pfence, int fd,
+                    enum pipe_fd_type type)
+{
+   *pfence = agx_fence_from_fd(agx_context(pctx), fd, type);
+}
+
+void
+agx_fence_server_sync(struct pipe_context *pctx, struct pipe_fence_handle *f,
+                      uint64_t value)
+{
+   struct agx_device *dev = agx_device(pctx->screen);
+   struct agx_context *ctx = agx_context(pctx);
+   int fd = -1, ret;
+   assert(!value);
+
+   ret = drmSyncobjExportSyncFile(dev->fd, f->syncobj, &fd);
+   assert(!ret);
+
+   sync_accumulate("asahi", &ctx->in_sync_fd, fd);
+   close(fd);
+}

src/gallium/drivers/asahi/agx_fence.h

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2022 Amazon.com, Inc. or its affiliates.
+ * Copyright 2018-2019 Alyssa Rosenzweig
+ * Copyright 2018-2019 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#pragma once
+
+#include "pipe/p_state.h"
+
+struct agx_context;
+
+struct pipe_fence_handle {
+   struct pipe_reference reference;
+   uint32_t syncobj;
+   bool signaled;
+};
+
+void agx_fence_reference(struct pipe_screen *pscreen,
+                         struct pipe_fence_handle **ptr,
+                         struct pipe_fence_handle *fence);
+
+bool agx_fence_finish(struct pipe_screen *pscreen, struct pipe_context *ctx,
+                      struct pipe_fence_handle *fence, uint64_t timeout);
+
+int agx_fence_get_fd(struct pipe_screen *screen, struct pipe_fence_handle *f);
+
+struct pipe_fence_handle *agx_fence_from_fd(struct agx_context *ctx, int fd,
+                                            enum pipe_fd_type type);
+
+struct pipe_fence_handle *agx_fence_create(struct agx_context *ctx);
+
+void agx_create_fence_fd(struct pipe_context *pctx,
+                         struct pipe_fence_handle **pfence, int fd,
+                         enum pipe_fd_type type);
+
+void agx_fence_server_sync(struct pipe_context *pctx,
+                           struct pipe_fence_handle *f, uint64_t value);

src/gallium/drivers/asahi/agx_nir_lower_bindings.c

@@ -0,0 +1,160 @@
+/*
+ * Copyright 2023 Valve Corporation
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "asahi/compiler/agx_nir.h"
+#include "compiler/nir/nir_builder.h"
+#include "util/bitset.h"
+#include "agx_state.h"
+#include "nir.h"
+#include "nir_builder_opcodes.h"
+#include "nir_intrinsics.h"
+#include "nir_intrinsics_indices.h"
+
+/*
+ * Lower binding table textures and images to texture state registers and (if
+ * necessary) bindless access into an internal table mapped like additional
+ * texture state registers. The following layout is used:
+ *
+ *    1. Textures
+ *    2. Images (read/write interleaved)
+ */
+
+static bool
+lower_sampler(nir_builder *b, nir_tex_instr *tex)
+{
+   if (!nir_tex_instr_need_sampler(tex))
+      return false;
+
+   nir_def *index = nir_steal_tex_src(tex, nir_tex_src_sampler_offset);
+   if (!index)
+      index = nir_imm_int(b, tex->sampler_index);
+
+   nir_tex_instr_add_src(tex, nir_tex_src_sampler_handle,
+                         nir_load_sampler_handle_agx(b, index));
+   return true;
+}
+
+static bool
+lower(nir_builder *b, nir_instr *instr, void *data)
+{
+   bool *uses_bindless_samplers = data;
+   bool progress = false;
+   bool force_bindless = agx_nir_needs_texture_crawl(instr);
+   b->cursor = nir_before_instr(instr);
+
+   if (instr->type == nir_instr_type_intrinsic) {
+      nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
+
+      switch (intr->intrinsic) {
+      case nir_intrinsic_image_load:
+      case nir_intrinsic_image_store:
+      case nir_intrinsic_image_size:
+      case nir_intrinsic_image_samples:
+      case nir_intrinsic_image_atomic:
+      case nir_intrinsic_image_atomic_swap:
+         break;
+      default:
+         return false;
+      }
+
+      nir_def *index = intr->src[0].ssa;
+      nir_scalar index_scalar = nir_scalar_resolved(index, 0);
+
+      /* Remap according to the driver layout */
+      unsigned offset = BITSET_LAST_BIT(b->shader->info.textures_used);
+
+      /* For reads and queries, we use the texture descriptor which is first.
+       * Writes and atomics use the PBE descriptor.
+       */
+      if (intr->intrinsic != nir_intrinsic_image_load &&
+          intr->intrinsic != nir_intrinsic_image_size &&
+          intr->intrinsic != nir_intrinsic_image_samples)
+         offset++;
+
+      /* If we can determine statically that the image fits in texture state
+       * registers, avoid lowering to bindless access.
+       */
+      if (nir_scalar_is_const(index_scalar) && !force_bindless) {
+         unsigned idx = (nir_scalar_as_uint(index_scalar) * 2) + offset;
+
+         if (idx < AGX_NUM_TEXTURE_STATE_REGS) {
+            nir_src_rewrite(&intr->src[0], nir_imm_intN_t(b, idx, 16));
+            return true;
+         }
+      }
+
+      /* Otherwise, lower to bindless...
+       *
+       * The driver uploads enough null texture/PBE descriptors for robustness
+       * given the shader limit, but we still need to clamp since we're lowering
+       * to bindless so the hardware doesn't know the limit.
+       *
+       * The GL spec says out-of-bounds image indexing is undefined, but
+       * faulting is not acceptable for robustness.
+       */
+      index = nir_umin(
+         b, index,
+         nir_imm_intN_t(b, b->shader->info.num_images - 1, index->bit_size));
+
+      index = nir_iadd_imm(b, nir_imul_imm(b, index, 2), offset);
+
+      nir_rewrite_image_intrinsic(intr, nir_load_texture_handle_agx(b, index),
+                                  nir_image_intrinsic_type_bindless);
+   } else if (instr->type == nir_instr_type_tex) {
+      nir_tex_instr *tex = nir_instr_as_tex(instr);
+
+      if (((BITSET_COUNT(b->shader->info.samplers_used) > 16) &&
+           (nir_tex_instr_src_index(tex, nir_tex_src_sampler_offset) >= 0 ||
+            tex->sampler_index >= 16)) &&
+          lower_sampler(b, tex)) {
+         progress = true;
+         *uses_bindless_samplers = true;
+      }
+
+      /* Nothing to do for "real" bindless */
+      if (nir_tex_instr_src_index(tex, nir_tex_src_texture_handle) >= 0)
+         return progress;
+
+      /* Textures are mapped 1:1, so if we can prove it fits in a texture state
+       * register, use the texture state register.
+       */
+      if (tex->texture_index < AGX_NUM_TEXTURE_STATE_REGS &&
+          nir_tex_instr_src_index(tex, nir_tex_src_texture_offset) == -1 &&
+          !force_bindless)
+         return progress;
+
+      /* Otherwise, lower to bindless. Could be optimized. */
+      nir_def *index = nir_steal_tex_src(tex, nir_tex_src_texture_offset);
+      if (!index)
+         index = nir_imm_int(b, tex->texture_index);
+
+      /* As above */
+      index = nir_umin(
+         b, index,
+         nir_imm_intN_t(b, b->shader->info.num_textures - 1, index->bit_size));
+
+      nir_tex_instr_add_src(tex, nir_tex_src_texture_handle,
+                            nir_load_texture_handle_agx(b, index));
+   }
+
+   return true;
+}
+
+bool
+agx_nir_lower_bindings(nir_shader *shader, bool *uses_bindless_samplers)
+{
+   /* First lower index to offset so we can lower more naturally */
+   bool progress = nir_lower_tex(
+      shader, &(nir_lower_tex_options){.lower_index_to_offset = true});
+
+   /* Next run constant folding so the constant optimizations above have a
+    * chance.
+    */
+   progress |= nir_opt_constant_folding(shader);
+
+   progress |= nir_shader_instructions_pass(
+      shader, lower, nir_metadata_control_flow, uses_bindless_samplers);
+   return progress;
+}

src/gallium/drivers/asahi/agx_nir_lower_point_size.c

@@ -0,0 +1,62 @@
+/*
+ * Copyright 2023 Valve Corporation
+ * SPDX-License-Identifier: MIT
+ */
+#include "agx_state.h"
+#include "nir.h"
+#include "nir_builder.h"
+#include "nir_builder_opcodes.h"
+
+/*
+ * gl_PointSize lowering. This runs late on a vertex shader. By this time, I/O
+ * has been lowered, and transform feedback has been written. Point size will
+ * thus only get consumed by the rasterizer, so we can clamp/replace. We do
+ * this instead of the mesa/st lowerings to avoid the variant. I wouldn't mind
+ * ripping this out some day...
+ */
+
+static bool
+pass(nir_builder *b, nir_intrinsic_instr *intr, void *data)
+{
+   b->cursor = nir_before_instr(&intr->instr);
+
+   if ((intr->intrinsic != nir_intrinsic_store_output) ||
+       (nir_intrinsic_io_semantics(intr).location != VARYING_SLOT_PSIZ))
+      return false;
+
+   /* The size we write must be clamped */
+   nir_def *size = nir_fmax(b, intr->src[0].ssa, nir_imm_float(b, 1.0f));
+
+   /* Override it if the API requires */
+   nir_def *fixed_size = nir_load_fixed_point_size_agx(b);
+   size = nir_bcsel(b, nir_fgt_imm(b, fixed_size, 0.0), fixed_size, size);
+
+   nir_src_rewrite(&intr->src[0], size);
+   return true;
+}
+
+bool
+agx_nir_lower_point_size(nir_shader *nir, bool insert_write)
+{
+   /* Lower existing point size write */
+   if (nir_shader_intrinsics_pass(nir, pass, nir_metadata_control_flow, NULL))
+      return true;
+
+   if (!insert_write)
+      return false;
+
+   /* If there's no existing point size write, insert one. This assumes there
+    * was a fixed point size set in the API. If not, GL allows undefined
+    * behaviour, which we implement by writing garbage.
+    */
+   nir_builder b =
+      nir_builder_at(nir_after_impl(nir_shader_get_entrypoint(nir)));
+
+   nir_store_output(
+      &b, nir_load_fixed_point_size_agx(&b), nir_imm_int(&b, 0),
+      .io_semantics.location = VARYING_SLOT_PSIZ, .io_semantics.num_slots = 1,
+      .write_mask = nir_component_mask(1), .src_type = nir_type_float32);
+
+   nir->info.outputs_written |= VARYING_BIT_PSIZ;
+   return nir_progress(true, b.impl, nir_metadata_control_flow);
+}

src/gallium/drivers/asahi/agx_nir_lower_sysvals.c

@@ -0,0 +1,525 @@
+/*
+ * Copyright 2022 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "compiler/nir/nir_builder.h"
+#include "pipe/p_defines.h"
+#include "poly/nir/poly_nir.h"
+#include "util/bitset.h"
+#include "util/u_dynarray.h"
+#include "agx_abi.h"
+#include "agx_state.h"
+#include "nir.h"
+#include "nir_builder_opcodes.h"
+#include "nir_intrinsics.h"
+#include "nir_intrinsics_indices.h"
+#include "shader_enums.h"
+
+#define AGX_TEXTURE_DESC_STRIDE 24
+
+/*
+ * Lower all system values to uniform loads. This pass tries to compact ranges
+ * of contiguous uploaded uniforms to reduce the draw-time overhead of uploading
+ * many tiny ranges. To do so, it works in 4 steps:
+ *
+ * 1. Lower NIR sysvals to loads from the system value buffers.
+ * 2. Walk the NIR, recording loads from system value buffers.
+ * 2. Walk the ranges of uniforms needed, compacting into contiguous ranges.
+ * 3. Fill in the load_preamble instructions with the real uniforms.
+ */
+
+#define MAX_TABLE_SIZE sizeof(struct agx_stage_uniforms)
+static_assert(sizeof(struct agx_draw_uniforms) <= MAX_TABLE_SIZE, "packed");
+
+struct table_state {
+   /* Bitset of 16-bit uniforms pushed */
+   BITSET_DECLARE(pushed, MAX_TABLE_SIZE / 2);
+
+   /* Element size in 16-bit units, so we may split ranges of different sizes
+    * to guarantee natural alignment.
+    */
+   uint8_t element_size[MAX_TABLE_SIZE / 2];
+};
+
+struct state {
+   mesa_shader_stage stage, hw_stage;
+
+   /* Array of nir_intrinsic_instr's to fix up at the end */
+   struct util_dynarray loads;
+
+   struct table_state tables[AGX_NUM_SYSVAL_TABLES];
+};
+
+static nir_def *
+load_sysval(nir_builder *b, unsigned dim, unsigned bitsize, uint8_t table,
+            uint16_t offset)
+{
+   return nir_load_sysval_agx(b, dim, bitsize, .desc_set = table,
+                              .binding = offset);
+}
+
+static nir_def *
+load_sysval_root(nir_builder *b, unsigned dim, unsigned bitsize, void *ptr)
+{
+   return load_sysval(b, dim, bitsize, AGX_SYSVAL_TABLE_ROOT, (uintptr_t)ptr);
+}
+
+static nir_def *
+load_sysval_indirect(nir_builder *b, unsigned dim, unsigned bitsize,
+                     uint8_t table, void *base, nir_def *offset_el)
+{
+   nir_scalar scalar = {offset_el, 0};
+   unsigned stride = (dim * bitsize) / 8;
+
+   if (nir_scalar_is_const(scalar)) {
+      /* Load the sysval directly */
+      return load_sysval(
+         b, dim, bitsize, table,
+         (uintptr_t)base + (nir_scalar_as_uint(scalar) * stride));
+   } else {
+      /* Load the base address of the table */
+      struct agx_draw_uniforms *u = NULL;
+      nir_def *table_base = load_sysval_root(b, 1, 64, &u->tables[table]);
+
+      /* Load address of the array in the table */
+      nir_def *array_base = nir_iadd_imm(b, table_base, (uintptr_t)base);
+
+      /* Index into the table and load */
+      nir_def *address = nir_iadd(
+         b, array_base, nir_u2u64(b, nir_imul_imm(b, offset_el, stride)));
+      return nir_load_global_constant(b, dim, bitsize, address);
+   }
+}
+
+static unsigned
+stage_table(nir_builder *b)
+{
+   mesa_shader_stage stage = b->shader->info.stage;
+   if (stage == MESA_SHADER_VERTEX && b->shader->info.vs.tes_poly)
+      stage = MESA_SHADER_TESS_EVAL;
+
+   assert(stage < MESA_SHADER_STAGES);
+   return AGX_SYSVAL_STAGE(stage);
+}
+
+static nir_def *
+load_ubo(nir_builder *b, nir_intrinsic_instr *intr, void *bases)
+{
+   nir_def *base =
+      load_sysval_indirect(b, 1, 64, stage_table(b), bases, intr->src[0].ssa);
+
+   nir_def *address = nir_iadd(b, base, nir_u2u64(b, intr->src[1].ssa));
+
+   return nir_load_global_constant(b, intr->num_components, intr->def.bit_size,
+                                   address,
+                                   .align_mul = nir_intrinsic_align(intr));
+}
+
+static nir_def *
+load_texture_handle(nir_builder *b, nir_intrinsic_instr *intr, void *base)
+{
+   nir_def *offs_B =
+      nir_imul_imm(b, nir_u2u32(b, intr->src[0].ssa), AGX_TEXTURE_DESC_STRIDE);
+
+   nir_load_sysval_agx(b, 1, 64, .desc_set = stage_table(b),
+                       .binding = (uintptr_t)base, .flags = ~0);
+   return nir_bindless_image_agx(b, offs_B);
+}
+
+static nir_def *
+load_provoking_vtx(nir_builder *b)
+{
+   struct agx_draw_uniforms *u = NULL;
+   return load_sysval_root(b, 1, 16, &u->provoking_vertex);
+}
+
+static nir_def *
+lower_intrinsic(nir_builder *b, nir_intrinsic_instr *intr,
+                bool lower_draw_params)
+{
+   struct agx_draw_uniforms *u = NULL;
+   struct agx_stage_uniforms *s = NULL;
+
+   switch (intr->intrinsic) {
+   case nir_intrinsic_load_ubo:
+      return load_ubo(b, intr, s->ubo_base);
+   case nir_intrinsic_load_texture_handle_agx:
+      return load_texture_handle(b, intr, &s->texture_base);
+   case nir_intrinsic_load_sampler_handle_agx:
+      return load_sysval_indirect(b, 1, 16, stage_table(b), &s->sampler_handle,
+                                  intr->src[0].ssa);
+   case nir_intrinsic_load_vbo_base_agx:
+      return load_sysval_indirect(b, 1, 64, AGX_SYSVAL_TABLE_ROOT,
+                                  &u->attrib_base, intr->src[0].ssa);
+   case nir_intrinsic_load_attrib_clamp_agx:
+      return load_sysval_indirect(b, 1, 32, AGX_SYSVAL_TABLE_ROOT,
+                                  &u->attrib_clamp, intr->src[0].ssa);
+   case nir_intrinsic_load_blend_const_color_r_float:
+      return load_sysval_root(b, 1, 32, &u->blend_constant[0]);
+   case nir_intrinsic_load_blend_const_color_g_float:
+      return load_sysval_root(b, 1, 32, &u->blend_constant[1]);
+   case nir_intrinsic_load_blend_const_color_b_float:
+      return load_sysval_root(b, 1, 32, &u->blend_constant[2]);
+   case nir_intrinsic_load_blend_const_color_a_float:
+      return load_sysval_root(b, 1, 32, &u->blend_constant[3]);
+   case nir_intrinsic_load_api_sample_mask_agx:
+      return load_sysval_root(b, 1, 16, &u->sample_mask);
+   case nir_intrinsic_load_sample_positions_agx:
+      return load_sysval_root(b, 1, 32, &u->ppp_multisamplectl);
+   case nir_intrinsic_load_stat_query_address_poly:
+      return load_sysval_root(
+         b, 1, 64, &u->pipeline_statistics[nir_intrinsic_base(intr)]);
+   case nir_intrinsic_load_ssbo_address:
+      assert(nir_src_as_uint(intr->src[1]) == 0);
+      return load_sysval_indirect(b, 1, 64, stage_table(b), &s->ssbo_base,
+                                  intr->src[0].ssa);
+   case nir_intrinsic_get_ubo_size:
+      return load_sysval_indirect(b, 1, 32, stage_table(b), &s->ubo_size,
+                                  intr->src[0].ssa);
+   case nir_intrinsic_get_ssbo_size:
+      return load_sysval_indirect(b, 1, 32, stage_table(b), &s->ssbo_size,
+                                  intr->src[0].ssa);
+   case nir_intrinsic_load_vertex_param_buffer_poly:
+      return load_sysval_root(b, 1, 64, &u->vertex_params);
+   case nir_intrinsic_load_geometry_param_buffer_poly:
+      return load_sysval_root(b, 1, 64, &u->geometry_params);
+   case nir_intrinsic_load_vs_outputs_poly:
+      return load_sysval_root(b, 1, 64, &u->vertex_outputs);
+   case nir_intrinsic_load_tess_param_buffer_poly:
+      return load_sysval_root(b, 1, 64, &u->tess_params);
+   case nir_intrinsic_load_fixed_point_size_agx:
+      return load_sysval_root(b, 1, 32, &u->fixed_point_size);
+   case nir_intrinsic_load_tex_sprite_mask_agx:
+      return load_sysval_root(b, 1, 16, &u->sprite_mask);
+   case nir_intrinsic_load_shader_part_tests_zs_agx:
+      return load_sysval_root(b, 1, 16, &u->no_epilog_discard);
+   case nir_intrinsic_load_clip_z_coeff:
+      return nir_f2f32(b, load_sysval_root(b, 1, 16, &u->clip_z_coeff));
+   case nir_intrinsic_load_rasterization_stream:
+      return nir_imm_int(b, 0);
+   case nir_intrinsic_load_depth_never_agx:
+      /* TODO: Do we need this workaround for anything in GL? */
+      return nir_imm_intN_t(b, 0, 16);
+   case nir_intrinsic_load_uvs_index_agx:
+      return load_sysval_root(
+         b, 1, 16, &u->uvs_index[nir_intrinsic_io_semantics(intr).location]);
+   case nir_intrinsic_load_is_first_fan_agx:
+      return nir_ieq_imm(b, load_provoking_vtx(b), 1);
+   case nir_intrinsic_load_provoking_last:
+      return nir_b2b32(b, nir_ieq_imm(b, load_provoking_vtx(b), 2));
+   default:
+      break;
+   }
+
+   if (!lower_draw_params)
+      return NULL;
+
+   switch (intr->intrinsic) {
+   case nir_intrinsic_load_num_workgroups:
+      return load_sysval(b, 3, 32, AGX_SYSVAL_TABLE_GRID, 0);
+   case nir_intrinsic_load_first_vertex:
+      return load_sysval(b, 1, 32, AGX_SYSVAL_TABLE_PARAMS, 0);
+   case nir_intrinsic_load_base_instance:
+      return load_sysval(b, 1, 32, AGX_SYSVAL_TABLE_PARAMS, 4);
+   case nir_intrinsic_load_base_vertex:
+      /* first vertex if indexed, 0 otherwise. More efficient for our hw than
+       * the lowering in NIR.
+       */
+      return nir_bcsel(
+         b, nir_i2b(b, load_sysval_root(b, 1, 16, &u->is_indexed_draw)),
+         load_sysval(b, 1, 32, AGX_SYSVAL_TABLE_PARAMS, 0), nir_imm_int(b, 0));
+   case nir_intrinsic_load_draw_id:
+      return load_sysval_root(b, 1, 32, &u->draw_id);
+   default:
+      return NULL;
+   }
+}
+
+/* Step 1. Lower NIR sysvals */
+static bool
+lower_sysvals(nir_builder *b, nir_instr *instr, void *data)
+{
+   bool *lower_draw_params = data;
+   b->cursor = nir_before_instr(instr);
+   nir_def *old;
+   nir_def *replacement = NULL;
+
+   if (instr->type == nir_instr_type_intrinsic) {
+      nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
+      old = &intr->def;
+      replacement = lower_intrinsic(b, intr, *lower_draw_params);
+   } else if (instr->type == nir_instr_type_tex) {
+      nir_tex_instr *tex = nir_instr_as_tex(instr);
+      old = &tex->def;
+
+      if (tex->op != nir_texop_lod_bias)
+         return false;
+
+      struct agx_stage_uniforms *s = NULL;
+
+      int src_idx = nir_tex_instr_src_index(tex, nir_tex_src_texture_offset);
+      if (src_idx >= 0) {
+         replacement = load_sysval_indirect(
+            b, 1, 16, stage_table(b), s->lod_bias, tex->src[src_idx].src.ssa);
+      } else {
+         replacement = load_sysval(b, 1, 16, stage_table(b),
+                                   (uintptr_t)&s->lod_bias[tex->sampler_index]);
+      }
+   }
+
+   if (replacement != NULL) {
+      nir_def_rewrite_uses(old, replacement);
+      return true;
+   } else {
+      return false;
+   }
+}
+
+/* Step 2: Record system value loads */
+static bool
+record_loads(nir_builder *b, nir_intrinsic_instr *intr, void *data)
+{
+   if (intr->intrinsic != nir_intrinsic_load_sysval_agx)
+      return false;
+
+   assert(intr->def.bit_size >= 16 && "no 8-bit sysvals");
+   unsigned dim = intr->def.num_components;
+   unsigned element_size = intr->def.bit_size / 16;
+   unsigned length = dim * element_size;
+
+   struct state *state = data;
+   struct table_state *table = &state->tables[nir_intrinsic_desc_set(intr)];
+   unsigned offset = nir_intrinsic_binding(intr);
+   assert((offset % 2) == 0 && "all entries are aligned by ABI");
+
+   BITSET_SET_COUNT(table->pushed, (offset / 2), length);
+
+   for (unsigned i = 0; i < length; ++i) {
+      if (table->element_size[(offset / 2) + i])
+         assert((table->element_size[(offset / 2) + i]) == element_size);
+      else
+         table->element_size[(offset / 2) + i] = element_size;
+   }
+
+   util_dynarray_append(&state->loads, intr);
+   return false;
+}
+
+/* Step 3: Decide where to push the system values */
+static struct agx_push_range *
+find_push_range_containing(struct agx_compiled_shader *shader, uint8_t table,
+                           uint16_t offset)
+{
+   for (unsigned i = 0; i < shader->push_range_count; ++i) {
+      struct agx_push_range *range = &shader->push[i];
+
+      if (range->table != table)
+         continue;
+
+      /* range->length is 16-bit words, need to convert. offset is bytes. */
+      uint16_t length_B = range->length * 2;
+
+      if (range->offset <= offset && offset < (range->offset + length_B))
+         return range;
+   }
+
+   UNREACHABLE("no containing range");
+}
+
+static unsigned
+lay_out_table(struct agx_compiled_shader *shader, struct table_state *state,
+              unsigned table_index, unsigned uniform)
+{
+   unsigned start, end;
+   BITSET_FOREACH_RANGE(start, end, state->pushed, sizeof(state->pushed) * 8) {
+      unsigned range_start = start;
+
+      do {
+         uint8_t size = state->element_size[range_start];
+
+         /* Find a range of constant element size. [range_start, range_end).
+          * Ranges may be at most 64 halfs.
+          */
+         unsigned range_end;
+         for (range_end = range_start + 1;
+              range_end < end && state->element_size[range_end] == size &&
+              range_end < range_start + 64;
+              ++range_end)
+            ;
+
+         /* Now make the range with the given size (naturally aligned) */
+         uniform = ALIGN_POT(uniform, size);
+
+         assert((shader->push_range_count < ARRAY_SIZE(shader->push)) &&
+                "AGX_MAX_PUSH_RANGES must be an upper bound");
+
+         /* Offsets must be aligned to 4 bytes, this may require pushing a
+          * little more than intended (otherwise we would need extra copies)
+          */
+         range_start = ROUND_DOWN_TO(range_start, 4 / 2);
+
+         shader->push[shader->push_range_count++] = (struct agx_push_range){
+            .uniform = uniform,
+            .table = table_index,
+            .offset = range_start * 2 /* bytes, not elements */,
+            .length = (range_end - range_start),
+         };
+
+         uniform += (range_end - range_start);
+         range_start = range_end;
+      } while (range_start < end);
+   }
+
+   return uniform;
+}
+
+static unsigned
+lay_out_uniforms(struct agx_compiled_shader *shader, struct state *state)
+{
+   unsigned uniform = 0;
+
+   if (state->stage == MESA_SHADER_VERTEX ||
+       state->stage == MESA_SHADER_TESS_EVAL) {
+      unsigned count =
+         DIV_ROUND_UP(BITSET_LAST_BIT(shader->attrib_components_read), 4);
+
+      struct agx_draw_uniforms *u = NULL;
+
+      if (count) {
+         shader->push[shader->push_range_count++] = (struct agx_push_range){
+            .uniform = AGX_ABI_VUNI_VBO_BASE(0),
+            .table = AGX_SYSVAL_TABLE_ROOT,
+            .offset = (uintptr_t)&u->attrib_base,
+            .length = 4 * count,
+         };
+
+         shader->push[shader->push_range_count++] = (struct agx_push_range){
+            .uniform = AGX_ABI_VUNI_VBO_CLAMP(count, 0),
+            .table = AGX_SYSVAL_TABLE_ROOT,
+            .offset = (uintptr_t)&u->attrib_clamp,
+            .length = 2 * count,
+         };
+      }
+
+      shader->push[shader->push_range_count++] = (struct agx_push_range){
+         .uniform = AGX_ABI_VUNI_FIRST_VERTEX(count),
+         .table = AGX_SYSVAL_TABLE_PARAMS,
+         .offset = 0,
+         .length = 4,
+      };
+
+      bool sw = state->hw_stage == MESA_SHADER_COMPUTE;
+      if (sw) {
+         shader->push[shader->push_range_count++] = (struct agx_push_range){
+            .uniform = AGX_ABI_VUNI_VERTEX_PARAMS(count),
+            .table = AGX_SYSVAL_TABLE_ROOT,
+            .offset = (uintptr_t)&u->vertex_params,
+            .length = 4,
+         };
+      }
+
+      uniform = AGX_ABI_VUNI_COUNT_GL(count, sw);
+   } else if (state->stage == MESA_SHADER_FRAGMENT) {
+      struct agx_draw_uniforms *u = NULL;
+      struct agx_stage_uniforms *s = NULL;
+      shader->push[shader->push_range_count++] = (struct agx_push_range){
+         .uniform = AGX_ABI_FUNI_EMRT_HEAP,
+         .table = AGX_SYSVAL_TABLE_FS,
+         .offset = (uintptr_t)&s->texture_base,
+         .length = 4,
+      };
+
+      shader->push[shader->push_range_count++] = (struct agx_push_range){
+         .uniform = AGX_ABI_FUNI_BLEND_R,
+         .table = AGX_SYSVAL_TABLE_ROOT,
+         .offset = (uintptr_t)&u->blend_constant,
+         .length = 8,
+      };
+
+      shader->push[shader->push_range_count++] = (struct agx_push_range){
+         .uniform = AGX_ABI_FUNI_ROOT,
+         .table = AGX_SYSVAL_TABLE_ROOT,
+         .offset = (uintptr_t)&u->tables[AGX_SYSVAL_TABLE_ROOT],
+         .length = 4,
+      };
+
+      uniform = AGX_ABI_FUNI_COUNT;
+   }
+
+   /* Lay out each system value table. We do this backwards to ensure the first
+    * uniform goes to the bindless texture base.
+    */
+   for (int t = AGX_NUM_SYSVAL_TABLES - 1; t >= 0; --t)
+      uniform = lay_out_table(shader, &state->tables[t], t, uniform);
+
+   /* Step 4: Fill in the loads */
+   util_dynarray_foreach(&state->loads, nir_intrinsic_instr *, intr_) {
+      nir_intrinsic_instr *intr = *intr_;
+      uint8_t table = nir_intrinsic_desc_set(intr);
+      uint16_t offset = nir_intrinsic_binding(intr);
+      bool bindless_image = nir_intrinsic_flags(intr);
+
+      struct agx_push_range *range =
+         find_push_range_containing(shader, table, offset);
+      unsigned base = range->uniform + ((offset - range->offset) / 2);
+
+      nir_builder b = nir_builder_at(nir_before_instr(&intr->instr));
+
+      if (bindless_image) {
+         nir_instr *next = nir_instr_next(&intr->instr);
+         assert(next->type == nir_instr_type_intrinsic);
+
+         nir_intrinsic_instr *nintr = nir_instr_as_intrinsic(next);
+         assert(nintr->intrinsic == nir_intrinsic_bindless_image_agx);
+
+         nir_intrinsic_set_desc_set(nintr, base);
+      } else {
+         nir_def *repl = nir_load_preamble(&b, intr->def.num_components,
+                                           intr->def.bit_size, .base = base);
+         nir_def_replace(&intr->def, repl);
+      }
+   }
+
+   return uniform;
+}
+
+bool
+agx_nir_lower_sysvals(nir_shader *shader, mesa_shader_stage desc_stage,
+                      bool lower_draw_params)
+{
+   /* override stage for the duration on the pass. XXX: should refactor, but
+    * it's annoying!
+    */
+   mesa_shader_stage phys_stage = shader->info.stage;
+   shader->info.stage = desc_stage;
+
+   bool progress = nir_shader_instructions_pass(
+      shader, lower_sysvals, nir_metadata_control_flow, &lower_draw_params);
+
+   shader->info.stage = phys_stage;
+   return progress;
+}
+
+bool
+agx_nir_layout_uniforms(nir_shader *shader,
+                        struct agx_compiled_shader *compiled,
+                        unsigned *push_size)
+{
+   struct state state = {
+      .stage = compiled->stage,
+      .hw_stage = shader->info.stage,
+   };
+
+   nir_shader_intrinsics_pass(shader, record_loads, nir_metadata_control_flow,
+                              &state);
+
+   *push_size = lay_out_uniforms(compiled, &state);
+
+   util_dynarray_fini(&state.loads);
+
+   /* Make sure texture handles have constants associated */
+   nir_opt_constant_folding(shader);
+
+   return true;
+}

src/gallium/drivers/asahi/agx_public.h

@@ -0,0 +1,19 @@
+/*
+ * Copyright 2010 Red Hat Inc.
+ * SPDX-License-Identifier: MIT
+ */
+#pragma once
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct pipe_screen;
+struct renderonly;
+
+struct pipe_screen *agx_screen_create(int fd, struct renderonly *ro,
+                                      const struct pipe_screen_config *config);
+
+#ifdef __cplusplus
+}
+#endif

src/gallium/drivers/asahi/agx_query.c

@@ -0,0 +1,641 @@
+/*
+ * Copyright 2022 Alyssa Rosenzweig
+ * Copyright 2019-2020 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include <stdint.h>
+#include "pipe/p_defines.h"
+#include "util/bitset.h"
+#include "util/macros.h"
+#include "util/ralloc.h"
+#include "util/u_dump.h"
+#include "util/u_inlines.h"
+#include "agx_abi.h"
+#include "agx_bo.h"
+#include "agx_device.h"
+#include "agx_state.h"
+#include "libagx.h"
+#include "libagx_dgc.h"
+#include "libagx_shaders.h"
+
+static bool
+is_occlusion(struct agx_query *query)
+{
+   switch (query->type) {
+   case PIPE_QUERY_OCCLUSION_COUNTER:
+   case PIPE_QUERY_OCCLUSION_PREDICATE:
+   case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE:
+      return true;
+   default:
+      return false;
+   }
+}
+
+static bool
+is_timer(struct agx_query *query)
+{
+   switch (query->type) {
+   case PIPE_QUERY_TIMESTAMP:
+   case PIPE_QUERY_TIME_ELAPSED:
+      return true;
+   default:
+      return false;
+   }
+}
+
+struct agx_oq_heap {
+   /* The GPU allocation itself */
+   struct agx_device *dev;
+   struct agx_bo *bo;
+
+   /* Bitset of query indices that are in use */
+   BITSET_DECLARE(available, AGX_MAX_OCCLUSION_QUERIES);
+};
+
+static void
+agx_destroy_oq_heap(void *heap_)
+{
+   struct agx_oq_heap *heap = heap_;
+   agx_bo_unreference(heap->dev, heap->bo);
+}
+
+static struct agx_oq_heap *
+agx_alloc_oq_heap(struct agx_context *ctx)
+{
+   struct agx_oq_heap *heap = rzalloc(ctx, struct agx_oq_heap);
+   ralloc_set_destructor(heap, agx_destroy_oq_heap);
+
+   heap->dev = agx_device(ctx->base.screen);
+   heap->bo =
+      agx_bo_create(heap->dev, AGX_MAX_OCCLUSION_QUERIES * sizeof(uint64_t), 0,
+                    AGX_BO_WRITEBACK, "Occlusion query heap");
+
+   /* At the start, everything is available */
+   BITSET_ONES(heap->available);
+
+   return heap;
+}
+
+static struct agx_oq_heap *
+agx_get_oq_heap(struct agx_context *ctx)
+{
+   if (!ctx->oq)
+      ctx->oq = agx_alloc_oq_heap(ctx);
+
+   return ctx->oq;
+}
+
+static struct agx_ptr
+agx_alloc_oq(struct agx_context *ctx)
+{
+   struct agx_oq_heap *heap = agx_get_oq_heap(ctx);
+
+   /* Find first available */
+   int ffs = BITSET_FFS(heap->available);
+   if (!ffs)
+      return (struct agx_ptr){NULL, 0};
+
+   /* Allocate it */
+   unsigned index = ffs - 1;
+   BITSET_CLEAR(heap->available, index);
+
+   unsigned offset = index * sizeof(uint64_t);
+
+   return (struct agx_ptr){
+      (uint8_t *)agx_bo_map(heap->bo) + offset,
+      heap->bo->va->addr + offset,
+   };
+}
+
+static unsigned
+agx_oq_index(struct agx_context *ctx, struct agx_query *q)
+{
+   assert(is_occlusion(q));
+
+   return (q->ptr.gpu - ctx->oq->bo->va->addr) / sizeof(uint64_t);
+}
+
+static void
+agx_free_oq(struct agx_context *ctx, struct agx_query *q)
+{
+   struct agx_oq_heap *heap = agx_get_oq_heap(ctx);
+   unsigned index = agx_oq_index(ctx, q);
+
+   assert(index < AGX_MAX_OCCLUSION_QUERIES);
+   assert(!BITSET_TEST(heap->available, index));
+
+   BITSET_SET(heap->available, index);
+}
+
+uint64_t
+agx_get_occlusion_heap(struct agx_batch *batch)
+{
+   if (!batch->ctx->oq)
+      return 0;
+
+   struct agx_bo *bo = batch->ctx->oq->bo;
+
+   if (agx_batch_uses_bo(batch, bo))
+      return bo->va->addr;
+   else
+      return 0;
+}
+
+static struct pipe_query *
+agx_create_query(struct pipe_context *ctx, unsigned query_type, unsigned index)
+{
+   struct agx_query *query = calloc(1, sizeof(struct agx_query));
+
+   query->type = query_type;
+   query->index = index;
+
+   /* Set all writer generations to a sentinel that will always compare as
+    * false, since nothing writes to no queries.
+    */
+   for (unsigned i = 0; i < ARRAY_SIZE(query->writer_generation); ++i) {
+      query->writer_generation[i] = UINT64_MAX;
+   }
+
+   if (is_occlusion(query)) {
+      query->ptr = agx_alloc_oq(agx_context(ctx));
+   } else {
+      /* TODO: a BO for the query is wasteful, but we benefit from BO list
+       * tracking / reference counting to deal with lifetimes.
+       */
+      query->bo = agx_bo_create(agx_device(ctx->screen), sizeof(uint64_t) * 2,
+                                0, AGX_BO_WRITEBACK, "Query");
+      query->ptr = (struct agx_ptr){
+         .gpu = query->bo->va->addr,
+         .cpu = agx_bo_map(query->bo),
+      };
+   }
+
+   if (!query->ptr.gpu) {
+      free(query);
+      return NULL;
+   }
+
+   return (struct pipe_query *)query;
+}
+
+static void
+flush_query_writers(struct agx_context *ctx, struct agx_query *query,
+                    const char *reason)
+{
+   STATIC_ASSERT(ARRAY_SIZE(ctx->batches.generation) == AGX_MAX_BATCHES);
+   STATIC_ASSERT(ARRAY_SIZE(ctx->batches.slots) == AGX_MAX_BATCHES);
+   STATIC_ASSERT(ARRAY_SIZE(query->writer_generation) == AGX_MAX_BATCHES);
+
+   for (unsigned i = 0; i < AGX_MAX_BATCHES; ++i) {
+      if (query->writer_generation[i] == ctx->batches.generation[i])
+         agx_flush_batch_for_reason(ctx, &ctx->batches.slots[i], reason);
+   }
+}
+
+static void
+sync_query_writers(struct agx_context *ctx, struct agx_query *query,
+                   const char *reason)
+{
+   for (unsigned i = 0; i < AGX_MAX_BATCHES; ++i) {
+      if (query->writer_generation[i] == ctx->batches.generation[i])
+         agx_sync_batch_for_reason(ctx, &ctx->batches.slots[i], reason);
+   }
+}
+
+static bool
+is_query_busy(struct agx_context *ctx, struct agx_query *query)
+{
+   for (unsigned i = 0; i < AGX_MAX_BATCHES; ++i) {
+      if (query->writer_generation[i] == ctx->batches.generation[i])
+         return true;
+   }
+
+   return false;
+}
+
+static void
+agx_destroy_query(struct pipe_context *pctx, struct pipe_query *pquery)
+{
+   struct agx_context *ctx = agx_context(pctx);
+   struct agx_query *query = (struct agx_query *)pquery;
+   struct agx_device *dev = agx_device(pctx->screen);
+
+   /* We don't reference count the occlusion query allocations, so we need to
+    * sync writers when destroying so we can freely write from the CPU after
+    * it's destroyed, since the driver will assume an available query is idle.
+    *
+    * For other queries, the BO itself is reference counted after the pipe_query
+    * is destroyed so we don't need to flush.
+    */
+   if (is_occlusion(query)) {
+      sync_query_writers(ctx, query, "Occlusion query destroy");
+      agx_free_oq(ctx, query);
+   } else {
+      agx_bo_unreference(dev, query->bo);
+   }
+
+   free(pquery);
+}
+
+static bool
+agx_begin_query(struct pipe_context *pctx, struct pipe_query *pquery)
+{
+   struct agx_context *ctx = agx_context(pctx);
+   struct agx_query *query = (struct agx_query *)pquery;
+
+   ctx->dirty |= AGX_DIRTY_QUERY;
+
+   switch (query->type) {
+   case PIPE_QUERY_OCCLUSION_COUNTER:
+   case PIPE_QUERY_OCCLUSION_PREDICATE:
+   case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE:
+      ctx->occlusion_query = query;
+      break;
+
+   case PIPE_QUERY_PRIMITIVES_GENERATED:
+      ctx->prims_generated[query->index] = query;
+      break;
+
+   case PIPE_QUERY_PRIMITIVES_EMITTED:
+      ctx->tf_prims_generated[query->index] = query;
+      break;
+
+   case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
+      ctx->tf_overflow[query->index] = query;
+      break;
+
+   case PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE:
+      ctx->tf_any_overflow = query;
+      break;
+
+   case PIPE_QUERY_TIME_ELAPSED:
+      ctx->time_elapsed = query;
+      break;
+
+   case PIPE_QUERY_TIMESTAMP:
+      /* No-op */
+      break;
+
+   case PIPE_QUERY_PIPELINE_STATISTICS_SINGLE:
+      assert(query->index < ARRAY_SIZE(ctx->pipeline_statistics));
+      ctx->pipeline_statistics[query->index] = query;
+      break;
+
+   default:
+      return false;
+   }
+
+   /* begin_query zeroes, sync so we can do that write from the CPU */
+   sync_query_writers(ctx, query, "Query overwritten");
+
+   uint64_t *ptr = query->ptr.cpu;
+   ptr[0] = 0;
+
+   if (query->type == PIPE_QUERY_TIME_ELAPSED) {
+      /* Timestamp begin in second record, the timestamp end in the first */
+      ptr[1] = UINT64_MAX;
+   }
+
+   return true;
+}
+
+static bool
+agx_end_query(struct pipe_context *pctx, struct pipe_query *pquery)
+{
+   struct agx_context *ctx = agx_context(pctx);
+   struct agx_device *dev = agx_device(pctx->screen);
+   struct agx_query *query = (struct agx_query *)pquery;
+
+   ctx->dirty |= AGX_DIRTY_QUERY;
+
+   switch (query->type) {
+   case PIPE_QUERY_OCCLUSION_COUNTER:
+   case PIPE_QUERY_OCCLUSION_PREDICATE:
+   case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE:
+      ctx->occlusion_query = NULL;
+      return true;
+   case PIPE_QUERY_PRIMITIVES_GENERATED:
+      ctx->prims_generated[query->index] = NULL;
+      return true;
+   case PIPE_QUERY_PRIMITIVES_EMITTED:
+      ctx->tf_prims_generated[query->index] = NULL;
+      return true;
+   case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
+      ctx->tf_overflow[query->index] = NULL;
+      return true;
+   case PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE:
+      ctx->tf_any_overflow = NULL;
+      return true;
+   case PIPE_QUERY_TIME_ELAPSED:
+      ctx->time_elapsed = NULL;
+      return true;
+   case PIPE_QUERY_PIPELINE_STATISTICS_SINGLE:
+      assert(query->index < ARRAY_SIZE(ctx->pipeline_statistics));
+      ctx->pipeline_statistics[query->index] = NULL;
+      return true;
+   case PIPE_QUERY_TIMESTAMP: {
+      /* Timestamp logically written now, set up batches to MAX their finish
+       * time in. If there are no batches, it's just the current time stamp.
+       */
+      agx_add_timestamp_end_query(ctx, query);
+
+      uint64_t *value = query->ptr.cpu;
+      *value = agx_get_gpu_timestamp(dev);
+
+      return true;
+   }
+   default:
+      return false;
+   }
+}
+
+enum query_copy_type {
+   QUERY_COPY_NORMAL,
+   QUERY_COPY_BOOL32,
+   QUERY_COPY_BOOL64,
+   QUERY_COPY_TIMESTAMP,
+   QUERY_COPY_TIME_ELAPSED,
+};
+
+static enum query_copy_type
+classify_query_type(enum pipe_query_type type)
+{
+   switch (type) {
+   case PIPE_QUERY_OCCLUSION_PREDICATE:
+   case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE:
+      return QUERY_COPY_BOOL32;
+
+   case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
+   case PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE:
+      return QUERY_COPY_BOOL64;
+
+   case PIPE_QUERY_TIMESTAMP:
+      return QUERY_COPY_TIMESTAMP;
+
+   case PIPE_QUERY_TIME_ELAPSED:
+      return QUERY_COPY_TIME_ELAPSED;
+
+   default:
+      return QUERY_COPY_NORMAL;
+   }
+}
+
+static bool
+agx_get_query_result(struct pipe_context *pctx, struct pipe_query *pquery,
+                     bool wait, union pipe_query_result *vresult)
+{
+   struct agx_query *query = (struct agx_query *)pquery;
+   struct agx_context *ctx = agx_context(pctx);
+   struct agx_device *dev = agx_device(pctx->screen);
+
+   /* TODO: Honour `wait` */
+   sync_query_writers(ctx, query, "Reading query results");
+
+   uint64_t *ptr = query->ptr.cpu;
+   uint64_t value = *ptr;
+
+   switch (classify_query_type(query->type)) {
+   case QUERY_COPY_BOOL32:
+      vresult->u64 = ((uint32_t)value) != 0;
+      return true;
+
+   case QUERY_COPY_BOOL64:
+      vresult->u64 = value != 0;
+      return true;
+
+   case QUERY_COPY_NORMAL:
+      vresult->u64 = value;
+      return true;
+
+   case QUERY_COPY_TIMESTAMP:
+      vresult->u64 = agx_gpu_timestamp_to_ns(dev, value);
+      return true;
+
+   case QUERY_COPY_TIME_ELAPSED:
+      /* end - begin */
+      vresult->u64 = agx_gpu_timestamp_to_ns(dev, ptr[0] - ptr[1]);
+      return true;
+
+   default:
+      UNREACHABLE("Other queries not yet supported");
+   }
+}
+
+static unsigned
+result_type_size(enum pipe_query_value_type result_type)
+{
+   return (result_type <= PIPE_QUERY_TYPE_U32) ? 4 : 8;
+}
+
+static void
+agx_get_query_result_resource_cpu(struct agx_context *ctx,
+                                  struct agx_query *query,
+                                  enum pipe_query_flags flags,
+                                  enum pipe_query_value_type result_type,
+                                  int index, struct pipe_resource *resource,
+                                  unsigned offset)
+{
+   union pipe_query_result result;
+   if (index < 0) {
+      /* availability */
+      result.u64 = !is_query_busy(ctx, query);
+   } else {
+      bool ready =
+         agx_get_query_result(&ctx->base, (void *)query, true, &result);
+
+      assert(ready);
+   }
+
+   /* Clamp to type, arb_query_buffer_object-qbo tests */
+   if (result_type == PIPE_QUERY_TYPE_U32) {
+      result.u32 = MIN2(result.u64, u_uintN_max(32));
+   } else if (result_type == PIPE_QUERY_TYPE_I32) {
+      int64_t x = result.u64;
+      x = MAX2(MIN2(x, u_intN_max(32)), u_intN_min(32));
+      result.u32 = x;
+   }
+
+   pipe_buffer_write(&ctx->base, resource, offset,
+                     result_type_size(result_type), &result.u64);
+}
+
+static bool
+agx_get_query_result_resource_gpu(struct agx_context *ctx,
+                                  struct agx_query *query,
+                                  enum pipe_query_flags flags,
+                                  enum pipe_query_value_type result_type,
+                                  int index, struct pipe_resource *prsrc,
+                                  unsigned offset)
+{
+   /* Handle availability queries on CPU */
+   if (index < 0)
+      return false;
+
+   /* TODO: timer queries on GPU */
+   if (query->type == PIPE_QUERY_TIMESTAMP ||
+       query->type == PIPE_QUERY_TIME_ELAPSED)
+      return false;
+
+   flush_query_writers(ctx, query, util_str_query_type(query->type, true));
+
+   struct agx_resource *rsrc = agx_resource(prsrc);
+   enum query_copy_type copy_type = classify_query_type(query->type);
+
+   struct agx_batch *batch = agx_get_compute_batch(ctx);
+   agx_batch_init_state(batch);
+   agx_dirty_all(ctx);
+
+   /* Set params */
+   agx_batch_writes_range(batch, rsrc, offset, result_type_size(result_type));
+
+   unsigned bool_size = copy_type == QUERY_COPY_BOOL64   ? 8
+                        : copy_type == QUERY_COPY_BOOL32 ? 4
+                                                         : 0;
+
+   libagx_copy_query_gl(batch, agx_1d(1), AGX_BARRIER_ALL, query->ptr.gpu,
+                        agx_map_gpu(rsrc) + offset, result_type, bool_size);
+   return true;
+}
+
+static void
+agx_get_query_result_resource(struct pipe_context *pipe, struct pipe_query *q,
+                              enum pipe_query_flags flags,
+                              enum pipe_query_value_type result_type, int index,
+                              struct pipe_resource *resource, unsigned offset)
+{
+   struct agx_query *query = (struct agx_query *)q;
+   struct agx_context *ctx = agx_context(pipe);
+
+   /* Try to copy on the GPU */
+   if (!agx_get_query_result_resource_gpu(ctx, query, flags, result_type, index,
+                                          resource, offset)) {
+
+      /* Else, fallback to CPU */
+      agx_get_query_result_resource_cpu(ctx, query, flags, result_type, index,
+                                        resource, offset);
+   }
+}
+
+static void
+agx_set_active_query_state(struct pipe_context *pipe, bool enable)
+{
+   struct agx_context *ctx = agx_context(pipe);
+
+   ctx->active_queries = enable;
+   ctx->dirty |= AGX_DIRTY_QUERY;
+}
+
+static void
+agx_add_query_to_batch(struct agx_batch *batch, struct agx_query *query)
+{
+   unsigned idx = agx_batch_idx(batch);
+   struct agx_bo *bo = is_occlusion(query) ? batch->ctx->oq->bo : query->bo;
+
+   agx_batch_add_bo(batch, bo);
+   query->writer_generation[idx] = batch->ctx->batches.generation[idx];
+}
+
+void
+agx_batch_add_timestamp_query(struct agx_batch *batch, struct agx_query *q)
+{
+   if (q) {
+      agx_add_query_to_batch(batch, q);
+      util_dynarray_append(&batch->timestamps, q->ptr);
+   }
+}
+
+uint16_t
+agx_get_oq_index(struct agx_batch *batch, struct agx_query *query)
+{
+   agx_add_query_to_batch(batch, query);
+   return agx_oq_index(batch->ctx, query);
+}
+
+uint64_t
+agx_get_query_address(struct agx_batch *batch, struct agx_query *query)
+{
+   if (query) {
+      agx_add_query_to_batch(batch, query);
+      return query->ptr.gpu;
+   } else {
+      return AGX_SCRATCH_PAGE_ADDRESS;
+   }
+}
+
+void
+agx_finish_batch_queries(struct agx_batch *batch, uint64_t begin_ts,
+                         uint64_t end_ts)
+{
+   /* Remove the batch as write from all queries by incrementing the generation
+    * of the batch.
+    */
+   batch->ctx->batches.generation[agx_batch_idx(batch)]++;
+
+   /* Write out timestamps */
+   util_dynarray_foreach(&batch->timestamps, struct agx_ptr, it) {
+      uint64_t *ptr = it->cpu;
+
+      ptr[0] = MAX2(ptr[0], end_ts);
+      ptr[1] = MIN2(ptr[1], begin_ts);
+   }
+}
+
+void
+agx_query_increment_cpu(struct agx_context *ctx, struct agx_query *query,
+                        uint64_t increment)
+{
+   if (!query)
+      return;
+
+   sync_query_writers(ctx, query, "CPU query increment");
+
+   uint64_t *value = query->ptr.cpu;
+   *value += increment;
+}
+
+static void
+agx_render_condition(struct pipe_context *pipe, struct pipe_query *query,
+                     bool condition, enum pipe_render_cond_flag mode)
+{
+   struct agx_context *ctx = agx_context(pipe);
+
+   ctx->cond_query = query;
+   ctx->cond_cond = condition;
+   ctx->cond_mode = mode;
+}
+
+bool
+agx_render_condition_check_inner(struct agx_context *ctx)
+{
+   assert(ctx->cond_query != NULL && "precondition");
+
+   perf_debug_ctx(ctx, "Implementing conditional rendering on the CPU");
+
+   union pipe_query_result res = {0};
+   bool wait = ctx->cond_mode != PIPE_RENDER_COND_NO_WAIT &&
+               ctx->cond_mode != PIPE_RENDER_COND_BY_REGION_NO_WAIT;
+
+   struct pipe_query *pq = (struct pipe_query *)ctx->cond_query;
+
+   if (agx_get_query_result(&ctx->base, pq, wait, &res))
+      return res.u64 != ctx->cond_cond;
+
+   return true;
+}
+
+void
+agx_init_query_functions(struct pipe_context *pctx)
+{
+   pctx->create_query = agx_create_query;
+   pctx->destroy_query = agx_destroy_query;
+   pctx->begin_query = agx_begin_query;
+   pctx->end_query = agx_end_query;
+   pctx->get_query_result = agx_get_query_result;
+   pctx->get_query_result_resource = agx_get_query_result_resource;
+   pctx->set_active_query_state = agx_set_active_query_state;
+   pctx->render_condition = agx_render_condition;
+
+   /* By default queries are active */
+   agx_context(pctx)->active_queries = true;
+}

src/gallium/drivers/asahi/agx_streamout.c

@@ -0,0 +1,183 @@
+/*
+ * Copyright 2023 Alyssa Rosenzweig
+ * Copyright 2022 Collabora Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "compiler/nir/nir_builder.h"
+#include "compiler/nir/nir_xfb_info.h"
+#include "pipe/p_defines.h"
+#include "util/u_draw.h"
+#include "util/u_dump.h"
+#include "util/u_inlines.h"
+#include "util/u_prim.h"
+#include "agx_state.h"
+
+static struct pipe_stream_output_target *
+agx_create_stream_output_target(struct pipe_context *pctx,
+                                struct pipe_resource *prsc,
+                                unsigned buffer_offset, unsigned buffer_size)
+{
+   struct agx_streamout_target *target =
+      rzalloc(pctx, struct agx_streamout_target);
+
+   if (!target)
+      return NULL;
+
+   pipe_reference_init(&target->base.reference, 1);
+   pipe_resource_reference(&target->base.buffer, prsc);
+
+   target->base.context = pctx;
+   target->base.buffer_offset = buffer_offset;
+   target->base.buffer_size = buffer_size;
+
+   uint32_t zero = 0;
+   target->offset = pipe_buffer_create_with_data(pctx, PIPE_BIND_GLOBAL,
+                                                 PIPE_USAGE_DEFAULT, 4, &zero);
+
+   return &target->base;
+}
+
+static void
+agx_stream_output_target_destroy(struct pipe_context *pctx,
+                                 struct pipe_stream_output_target *target)
+{
+   struct agx_streamout_target *tgt = agx_so_target(target);
+
+   pipe_resource_reference(&tgt->base.buffer, NULL);
+   pipe_resource_reference(&tgt->offset, NULL);
+   ralloc_free(target);
+}
+
+static void
+agx_set_stream_output_targets(struct pipe_context *pctx, unsigned num_targets,
+                              struct pipe_stream_output_target **targets,
+                              const unsigned *offsets,
+                              enum mesa_prim output_prim)
+{
+   struct agx_context *ctx = agx_context(pctx);
+   struct agx_streamout *so = &ctx->streamout;
+
+   assert(num_targets <= ARRAY_SIZE(so->targets));
+
+   for (unsigned i = 0; i < num_targets; i++) {
+      /* From the Gallium documentation:
+       *
+       *    -1 means the buffer should be appended to, and everything else sets
+       *    the internal offset.
+       *
+       * We append regardless, so just check for != -1. Yes, using a negative
+       * sentinel value with an unsigned type is bananas. But it's in the
+       * Gallium contract and it will work out fine. Probably should be
+       * redefined to be ~0 instead of -1 but it doesn't really matter.
+       */
+      if (offsets[i] != -1 && targets[i] != NULL) {
+         pipe_buffer_write(pctx, agx_so_target(targets[i])->offset, 0, 4,
+                           &offsets[i]);
+      }
+
+      pipe_so_target_reference(&so->targets[i], targets[i]);
+   }
+
+   for (unsigned i = num_targets; i < so->num_targets; i++)
+      pipe_so_target_reference(&so->targets[i], NULL);
+
+   so->num_targets = num_targets;
+}
+
+static struct pipe_stream_output_target *
+get_target(struct agx_context *ctx, unsigned buffer)
+{
+   if (buffer < ctx->streamout.num_targets)
+      return ctx->streamout.targets[buffer];
+   else
+      return NULL;
+}
+
+/*
+ * Return the address of the indexed streamout buffer. This will be
+ * pushed into the streamout shader.
+ */
+uint64_t
+agx_batch_get_so_address(struct agx_batch *batch, unsigned buffer,
+                         uint32_t *size)
+{
+   struct pipe_stream_output_target *target = get_target(batch->ctx, buffer);
+
+   /* If there's no target, don't write anything */
+   if (!target) {
+      *size = 0;
+      return 0;
+   }
+
+   /* Otherwise, write the target */
+   struct agx_resource *rsrc = agx_resource(target->buffer);
+   agx_batch_writes_range(batch, rsrc, target->buffer_offset,
+                          target->buffer_size);
+
+   *size = target->buffer_size;
+   return agx_map_gpu(rsrc) + target->buffer_offset;
+}
+
+void
+agx_draw_vbo_from_xfb(struct pipe_context *pctx,
+                      const struct pipe_draw_info *info, unsigned drawid_offset,
+                      const struct pipe_draw_indirect_info *indirect)
+{
+   perf_debug_ctx(agx_context(pctx), "draw auto");
+
+   struct agx_streamout_target *so =
+      agx_so_target(indirect->count_from_stream_output);
+
+   unsigned offset_B = 0;
+   pipe_buffer_read(pctx, so->offset, 0, 4, &offset_B);
+
+   unsigned count = offset_B / so->stride;
+
+   struct pipe_draw_start_count_bias draw = {
+      .start = 0,
+      .count = count,
+   };
+
+   pctx->draw_vbo(pctx, info, drawid_offset, NULL, &draw, 1);
+}
+
+static uint32_t
+xfb_prims_for_vertices(enum mesa_prim mode, unsigned verts)
+{
+   uint32_t prims = u_decomposed_prims_for_vertices(mode, verts);
+
+   /* The GL spec isn't super clear about this, but it implies that quads are
+    * supposed to be tessellated into primitives and piglit
+    * (ext_transform_feedback-tessellation quads) checks this.
+    */
+   if (u_decomposed_prim(mode) == MESA_PRIM_QUADS)
+      prims *= 2;
+
+   return prims;
+}
+
+/*
+ * Count generated primitives on the CPU for transform feedback. This only works
+ * in the absence of indirect draws, geometry shaders, or tessellation.
+ */
+void
+agx_primitives_update_direct(struct agx_context *ctx,
+                             const struct pipe_draw_info *info,
+                             const struct pipe_draw_start_count_bias *draw)
+{
+   assert(ctx->active_queries && ctx->prims_generated[0] && "precondition");
+   assert(!ctx->stage[MESA_SHADER_GEOMETRY].shader &&
+          "Geometry shaders use their own counting");
+
+   agx_query_increment_cpu(ctx, ctx->prims_generated[0],
+                           xfb_prims_for_vertices(info->mode, draw->count));
+}
+
+void
+agx_init_streamout_functions(struct pipe_context *ctx)
+{
+   ctx->create_stream_output_target = agx_create_stream_output_target;
+   ctx->stream_output_target_destroy = agx_stream_output_target_destroy;
+   ctx->set_stream_output_targets = agx_set_stream_output_targets;
+}

src/gallium/drivers/asahi/agx_uniforms.c

@@ -0,0 +1,152 @@
+/*
+ * Copyright 2021 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+#include <stdio.h>
+#include "pipe/p_state.h"
+#include "util/half_float.h"
+#include "util/macros.h"
+#include "agx_device.h"
+#include "agx_state.h"
+#include "pool.h"
+
+static uint64_t
+agx_const_buffer_ptr(struct agx_batch *batch, struct pipe_constant_buffer *cb)
+{
+   if (cb->buffer) {
+      struct agx_resource *rsrc = agx_resource(cb->buffer);
+      agx_batch_reads(batch, rsrc);
+      return agx_map_gpu(rsrc) + cb->buffer_offset;
+   } else {
+      return 0;
+   }
+}
+
+void
+agx_upload_vbos(struct agx_batch *batch)
+{
+   struct agx_context *ctx = batch->ctx;
+   struct agx_vertex_elements *attribs = ctx->attributes;
+   uint64_t buffers[PIPE_MAX_ATTRIBS] = {0};
+   size_t buf_sizes[PIPE_MAX_ATTRIBS] = {0};
+
+   u_foreach_bit(vbo, ctx->vb_mask) {
+      struct pipe_vertex_buffer vb = ctx->vertex_buffers[vbo];
+      assert(!vb.is_user_buffer);
+
+      if (vb.buffer.resource) {
+         struct agx_resource *rsrc = agx_resource(vb.buffer.resource);
+         agx_batch_reads(batch, rsrc);
+
+         buffers[vbo] = agx_map_gpu(rsrc) + vb.buffer_offset;
+         buf_sizes[vbo] = rsrc->layout.size_B - vb.buffer_offset -
+                          rsrc->layout.level_offsets_B[0];
+      }
+   }
+
+   for (unsigned i = 0; i < PIPE_MAX_ATTRIBS; ++i) {
+      unsigned buf = attribs->buffers[i];
+      uint64_t addr;
+
+      batch->uniforms.attrib_clamp[i] = agx_calculate_vbo_clamp(
+         buffers[buf], attribs->key[i].format, buf_sizes[buf],
+         attribs->key[i].stride, attribs->src_offsets[i], &addr);
+
+      batch->uniforms.attrib_base[i] = addr;
+   }
+}
+
+void
+agx_upload_uniforms(struct agx_batch *batch)
+{
+   struct agx_context *ctx = batch->ctx;
+
+   struct agx_ptr root_ptr = agx_pool_alloc_aligned(
+      &batch->pool, sizeof(struct agx_draw_uniforms), 16);
+
+   batch->uniforms.tables[AGX_SYSVAL_TABLE_ROOT] = root_ptr.gpu;
+   batch->uniforms.sample_mask = ctx->sample_mask;
+
+   assert(_mesa_float_to_half(0.5) == 0x3800);
+   batch->uniforms.clip_z_coeff =
+      (ctx->rast && !ctx->rast->base.clip_halfz) ? 0x3800 : 0x0;
+
+   batch->uniforms.sprite_mask =
+      (batch->reduced_prim == MESA_PRIM_POINTS && ctx->rast)
+         ? ctx->rast->base.sprite_coord_enable
+         : 0;
+
+   memcpy(root_ptr.cpu, &batch->uniforms, sizeof(batch->uniforms));
+}
+
+void
+agx_set_sampler_uniforms(struct agx_batch *batch, mesa_shader_stage stage)
+{
+   struct agx_context *ctx = batch->ctx;
+   struct agx_stage *st = &ctx->stage[stage];
+   struct agx_stage_uniforms *unif = &batch->stage_uniforms[stage];
+   struct agx_device *dev = agx_device(ctx->base.screen);
+
+   u_foreach_bit(s, st->valid_samplers) {
+      unif->lod_bias[s] = st->samplers[s]->lod_bias_as_fp16;
+   }
+
+   /* If we use bindless samplers, insert sampler into the heap */
+   if (st->shader && st->shader->uses_bindless_samplers) {
+      u_foreach_bit(s, st->valid_samplers) {
+         unif->sampler_handle[s] =
+            28 +
+            agx_sampler_heap_add(dev, &batch->sampler_heap,
+                                 &st->samplers[s]->desc_without_custom_border);
+      }
+   }
+}
+
+void
+agx_set_cbuf_uniforms(struct agx_batch *batch, mesa_shader_stage stage)
+{
+   struct agx_stage *st = &batch->ctx->stage[stage];
+   struct agx_stage_uniforms *unif = &batch->stage_uniforms[stage];
+
+   u_foreach_bit(cb, st->cb_mask) {
+      unif->ubo_base[cb] = agx_const_buffer_ptr(batch, &st->cb[cb]);
+      unif->ubo_size[cb] = st->cb[cb].buffer_size;
+   }
+}
+
+void
+agx_set_ssbo_uniforms(struct agx_batch *batch, mesa_shader_stage stage)
+{
+   struct agx_stage *st = &batch->ctx->stage[stage];
+   struct agx_stage_uniforms *unif = &batch->stage_uniforms[stage];
+
+   /* Single element sink. TODO: Optimize with soft fault. */
+   uint32_t zeroes[4] = {0};
+   uint64_t sink = agx_pool_upload_aligned(&batch->pool, &zeroes, 16, 16);
+
+   /* Consider all shader buffers, needed to avoid faults with
+    * e.g. arb_shader_storage_buffer_object-array-ssbo-binding.
+    */
+   for (unsigned cb = 0; cb < PIPE_MAX_SHADER_BUFFERS; ++cb) {
+      struct pipe_shader_buffer *sb = &st->ssbo[cb];
+
+      if (sb->buffer && st->ssbo[cb].buffer_size) {
+         struct agx_resource *rsrc = agx_resource(sb->buffer);
+
+         if (st->ssbo_writable_mask & BITFIELD_BIT(cb)) {
+            agx_batch_writes_range(batch, rsrc, sb->buffer_offset,
+                                   sb->buffer_size);
+            batch->incoherent_writes = true;
+         } else {
+            agx_batch_reads(batch, rsrc);
+         }
+
+         unif->ssbo_base[cb] = agx_map_gpu(rsrc) + sb->buffer_offset;
+         unif->ssbo_size[cb] = st->ssbo[cb].buffer_size;
+      } else {
+         /* Invalid, so use the sink */
+         unif->ssbo_base[cb] = sink;
+         unif->ssbo_size[cb] = 0;
+      }
+   }
+}

src/gallium/drivers/asahi/driinfo_asahi.h

@@ -0,0 +1,7 @@
+/* asahi specific driconf options */
+
+/* clang-format off */
+DRI_CONF_SECTION_MISCELLANEOUS
+   DRI_CONF_HEAP_MEMORY_PERCENT(1.0f)
+   DRI_CONF_ASAHI_NO_FP16(false)
+DRI_CONF_SECTION_END

src/gallium/drivers/asahi/meson.build

@@ -0,0 +1,32 @@
+# Copyright 2017 Intel Corporation
+# SPDX-License-Identifier: MIT
+
+files_asahi = files(
+  'driinfo_asahi.h',
+  'agx_batch.c',
+  'agx_blit.c',
+  'agx_disk_cache.c',
+  'agx_fence.c',
+  'agx_pipe.c',
+  'agx_nir_lower_sysvals.c',
+  'agx_nir_lower_bindings.c',
+  'agx_nir_lower_point_size.c',
+  'agx_query.c',
+  'agx_state.c',
+  'agx_streamout.c',
+  'agx_uniforms.c',
+)
+
+libasahi = static_library(
+  'asahi',
+  [files_asahi],
+  include_directories : [inc_gallium_aux, inc_gallium, inc_include, inc_src, inc_asahi, inc_virtio_gpu, inc_virtio_vdrm],
+  c_args : [c_msvc_compat_args, '-Wno-c2x-extensions'],
+  gnu_symbol_visibility : 'hidden',
+  dependencies : [idep_nir, idep_mesautil, idep_agx_pack, dep_libdrm, idep_libagx, idep_libagx_shaders_h],
+)
+
+driver_asahi = declare_dependency(
+  compile_args : '-DGALLIUM_ASAHI',
+  link_with : [libasahi, libasahiwinsys, libasahi_compiler, libasahi_lib, libasahi_layout, libasahi_decode]
+)

src/gallium/drivers/crocus/ci/crocus-g41-fails.txt

@@ -0,0 +1,257 @@
+KHR-GLES2.core.internalformat.texture2d.depth_component_unsigned_int_depth_component16,Fail
+KHR-GLES2.core.internalformat.texture2d.depth_component_unsigned_int_depth_component24,Fail
+KHR-GLES2.core.internalformat.texture2d.depth_component_unsigned_short_depth_component16,Fail
+KHR-GLES2.core.internalformat.texture2d.rgb_float_rgb32f,Fail
+KHR-GLES2.core.internalformat.texture2d.rgb_float_rgb32f_linear,Fail
+KHR-GLES2.core.internalformat.texture2d.rgba_float_rgba32f,Fail
+KHR-GLES2.core.internalformat.texture2d.rgba_float_rgba32f_linear,Fail
+
+# Missing the red triangle
+dEQP-GLES2.functional.clip_control.depth_mode_zero_to_one,Fail
+
+# Clipped wide primitives not rendered when the ceneter is offscreen
+dEQP-GLES2.functional.clipping.line.wide_line_clip_viewport_center,Fail
+dEQP-GLES2.functional.clipping.line.wide_line_clip_viewport_corner,Fail
+dEQP-GLES2.functional.clipping.point.wide_point_clip,Fail
+dEQP-GLES2.functional.clipping.point.wide_point_clip_viewport_center,Fail
+dEQP-GLES2.functional.clipping.point.wide_point_clip_viewport_corner,Fail
+
+# Bad slope of clipped triangles
+dEQP-GLES2.functional.clipping.triangle_vertex.clip_two.clip_neg_x_pos_z_and_neg_x_neg_y_neg_z,Fail
+dEQP-GLES2.functional.clipping.triangle_vertex.clip_two.clip_neg_y_neg_z_and_neg_x_neg_y_pos_z,Fail
+
+# RGB/RGBA half float should be incomplete, tested as complete.
+dEQP-GLES2.functional.fbo.completeness.renderable.texture.color0.rgb_half_float_oes,Fail
+
+dEQP-GLES2.functional.shaders.texture_functions.vertex.texturecubelod,Fail
+
+# Subpixel precision errors
+dEQP-GLES2.functional.texture.filtering.2d.linear_nearest_clamp_l8_npot,Fail
+dEQP-GLES2.functional.texture.filtering.2d.linear_nearest_clamp_rgb888_npot,Fail
+dEQP-GLES2.functional.texture.filtering.2d.linear_nearest_clamp_rgba4444_npot,Fail
+dEQP-GLES2.functional.texture.filtering.2d.linear_nearest_clamp_rgba8888_npot,Fail
+dEQP-GLES2.functional.texture.filtering.2d.nearest_linear_clamp_l8_npot,Fail
+dEQP-GLES2.functional.texture.filtering.2d.nearest_linear_clamp_rgb888_npot,Fail
+dEQP-GLES2.functional.texture.filtering.2d.nearest_linear_clamp_rgba4444_npot,Fail
+dEQP-GLES2.functional.texture.filtering.2d.nearest_linear_clamp_rgba8888_npot,Fail
+dEQP-GLES2.functional.texture.filtering.cube.linear_nearest_clamp_l8_npot,Fail
+dEQP-GLES2.functional.texture.filtering.cube.linear_nearest_clamp_rgb888_npot,Fail
+dEQP-GLES2.functional.texture.filtering.cube.linear_nearest_clamp_rgba4444_npot,Fail
+dEQP-GLES2.functional.texture.filtering.cube.linear_nearest_clamp_rgba8888_npot,Fail
+dEQP-GLES2.functional.texture.filtering.cube.nearest_linear_clamp_l8_npot,Fail
+dEQP-GLES2.functional.texture.filtering.cube.nearest_linear_clamp_rgb888_npot,Fail
+dEQP-GLES2.functional.texture.filtering.cube.nearest_linear_clamp_rgba4444_npot,Fail
+dEQP-GLES2.functional.texture.filtering.cube.nearest_linear_clamp_rgba8888_npot,Fail
+dEQP-GLES2.functional.texture.mipmap.cube.basic.linear_linear,Fail
+dEQP-GLES2.functional.texture.mipmap.cube.basic.linear_nearest,Fail
+dEQP-GLES2.functional.texture.mipmap.cube.bias.linear_linear,Fail
+dEQP-GLES2.functional.texture.mipmap.cube.bias.linear_nearest,Fail
+dEQP-GLES2.functional.texture.mipmap.cube.projected.linear_linear,Fail
+dEQP-GLES2.functional.texture.mipmap.cube.projected.linear_nearest,Fail
+dEQP-GLES2.functional.texture.vertex.cube.filtering.linear_mipmap_linear_linear_clamp,Fail
+dEQP-GLES2.functional.texture.vertex.cube.filtering.linear_mipmap_linear_linear_mirror,Fail
+dEQP-GLES2.functional.texture.vertex.cube.filtering.linear_mipmap_linear_nearest_clamp,Fail
+dEQP-GLES2.functional.texture.vertex.cube.filtering.linear_mipmap_linear_nearest_mirror,Fail
+dEQP-GLES2.functional.texture.vertex.cube.filtering.nearest_mipmap_linear_linear_clamp,Fail
+dEQP-GLES2.functional.texture.vertex.cube.filtering.nearest_mipmap_linear_linear_mirror,Fail
+dEQP-GLES2.functional.texture.vertex.cube.filtering.nearest_mipmap_linear_linear_repeat,Fail
+dEQP-GLES2.functional.texture.vertex.cube.filtering.nearest_mipmap_nearest_linear_clamp,Fail
+dEQP-GLES2.functional.texture.vertex.cube.filtering.nearest_mipmap_nearest_linear_mirror,Fail
+dEQP-GLES2.functional.texture.vertex.cube.filtering.nearest_mipmap_nearest_linear_repeat,Fail
+dEQP-GLES2.functional.texture.vertex.cube.wrap.clamp_clamp,Fail
+dEQP-GLES2.functional.texture.vertex.cube.wrap.clamp_mirror,Fail
+dEQP-GLES2.functional.texture.vertex.cube.wrap.clamp_repeat,Fail
+dEQP-GLES2.functional.texture.vertex.cube.wrap.mirror_clamp,Fail
+dEQP-GLES2.functional.texture.vertex.cube.wrap.mirror_mirror,Fail
+dEQP-GLES2.functional.texture.vertex.cube.wrap.mirror_repeat,Fail
+
+# No info, just piglit fail result.
+shaders@glsl-bug-110796,Fail
+
+# "glsl-routing: ../src/intel/compiler/brw_fs_visitor.cpp:155: fs_reg fs_visitor::interp_reg(int, int): Assertion `prog_data->urb_setup[location] >= 0' failed."
+shaders@glsl-routing,Crash
+
+# frontbuffer reading failures?
+spec@!opengl [email protected],Fail
+
+spec@!opengl 1.1@line-aa-width,Fail
+spec@!opengl 1.1@linestipple,Fail
+spec@!opengl 1.1@linestipple@Factor 2x,Fail
+spec@!opengl 1.1@linestipple@Factor 3x,Fail
+spec@!opengl 1.1@linestipple@Line loop,Fail
+spec@!opengl 1.1@linestipple@Line strip,Fail
+
+# abort()s submitting batchbuffer on context close, after some GL_OOMs.
+spec@!opengl 1.1@max-texture-size,Crash
+
+spec@!opengl [email protected],Fail
+
+# Regression from i965
+# "data not copied"
+spec@arb_copy_buffer@targets,Fail
+
+# "../src/mesa/state_tracker/st_cb_copyimage.c:309:swizzled_copy: Assertion `src_desc->block.bits == dst_desc->block.bits' failed."
+spec@arb_copy_image@arb_copy_image-formats,Crash
+
+# Regression from i965
+spec@arb_depth_buffer_float@fbo-generatemipmap-formats,Fail
+spec@arb_depth_buffer_float@fbo-generatemipmap-formats@GL_DEPTH_COMPONENT32F,Fail
+spec@arb_depth_buffer_float@fbo-generatemipmap-formats@GL_DEPTH_COMPONENT32F NPOT,Fail
+
+spec@arb_depth_buffer_float@texwrap formats bordercolor,Fail
+spec@arb_depth_buffer_float@texwrap formats bordercolor-swizzled,Fail
+spec@arb_depth_buffer_float@texwrap formats bordercolor-swizzled@GL_DEPTH32F_STENCIL8- swizzled- border color only,Fail
+spec@arb_depth_buffer_float@texwrap formats bordercolor-swizzled@GL_DEPTH_COMPONENT32F- swizzled- border color only,Fail
+spec@arb_depth_buffer_float@texwrap formats bordercolor@GL_DEPTH32F_STENCIL8- border color only,Fail
+spec@arb_depth_buffer_float@texwrap formats bordercolor@GL_DEPTH_COMPONENT32F- border color only,Fail
+
+# test bug: "Failed to compile fragment shader: 0:1(10): error: GLSL 1.30 is not supported. Supported versions are: 1.10, 1.20, and 1.00 ES"
+spec@arb_framebuffer_object@fbo-drawbuffers-none use_frag_out,Fail
+
+# "Left: textureCube, Right: textureCubeGradARB
+#  Probe color at (80,78)
+#    Left: 0.678431 0.333333 0.000000 1.000000
+#    Right: 0.647059 0.349020 0.000000 1.000000"
+spec@arb_shader_texture_lod@execution@arb_shader_texture_lod-texgradcube,Fail
+
+# Regression from i965
+spec@arb_texture_cube_map@copyteximage cube,Fail
+
+# Regressions from i965: i965 would demote these 32-bit internalformats to
+# 16-bit so that texture filtering worked, since GL 2.1 let you do that.
+# Crocus instead decided to follow r300 and other GL2 hardware in using the
+# requested size for storage even if the HW can't filter textures with it.
+spec@arb_texture_rg@fbo-blending-formats-float,Fail
+spec@arb_texture_rg@fbo-blending-formats-float@GL_RG32F,Fail
+spec@arb_texture_float@texwrap formats bordercolor,Fail
+spec@arb_texture_float@texwrap formats bordercolor-swizzled,Fail
+spec@arb_texture_float@texwrap formats bordercolor-swizzled@GL_INTENSITY32F_ARB- swizzled- border color only,Fail
+spec@arb_texture_float@texwrap formats bordercolor-swizzled@GL_LUMINANCE32F_ARB- swizzled- border color only,Fail
+spec@arb_texture_float@texwrap formats bordercolor-swizzled@GL_LUMINANCE_ALPHA32F_ARB- swizzled- border color only,Fail
+spec@arb_texture_float@texwrap formats bordercolor-swizzled@GL_RGB32F- swizzled- border color only,Fail
+spec@arb_texture_float@texwrap formats bordercolor-swizzled@GL_RGBA32F- swizzled- border color only,Fail
+spec@arb_texture_float@texwrap formats bordercolor@GL_INTENSITY32F_ARB- border color only,Fail
+spec@arb_texture_float@texwrap formats bordercolor@GL_LUMINANCE32F_ARB- border color only,Fail
+spec@arb_texture_float@texwrap formats bordercolor@GL_LUMINANCE_ALPHA32F_ARB- border color only,Fail
+spec@arb_texture_float@texwrap formats bordercolor@GL_RGB32F- border color only,Fail
+spec@arb_texture_float@texwrap formats bordercolor@GL_RGBA32F- border color only,Fail
+spec@arb_texture_rg@texwrap formats-float bordercolor,Fail
+spec@arb_texture_rg@texwrap formats-float bordercolor-swizzled,Fail
+spec@arb_texture_rg@texwrap formats-float bordercolor-swizzled@GL_R32F- swizzled- border color only,Fail
+spec@arb_texture_rg@texwrap formats-float bordercolor-swizzled@GL_RG32F- swizzled- border color only,Fail
+spec@arb_texture_rg@texwrap formats-float bordercolor@GL_R32F- border color only,Fail
+spec@arb_texture_rg@texwrap formats-float bordercolor@GL_RG32F- border color only,Fail
+
+# test bug: "ERROR - Duplicate subtest found, marking test failed: cube array texture"
+spec@arb_texture_storage@texture-storage@cube array texture,Fail
+
+# Regression from i965
+spec@arb_vertex_buffer_object@vbo-subdata-many drawarrays,Fail
+spec@arb_vertex_buffer_object@vbo-subdata-many drawelements,Fail
+
+spec@egl 1.4@eglterminate then unbind context,Fail
+
+spec@egl_ext_protected_content@conformance,Fail
+
+# "piglit: error: failed to get EGLConfig: EGL_NOT_INITIALIZED(0x3001)"
+spec@egl_khr_gl_image@egl_khr_gl_renderbuffer_image-clear-shared-image gl_depth_component24,Fail
+spec@egl_khr_gl_image@egl_khr_gl_renderbuffer_image-clear-shared-image gl_rgba,Fail
+
+# "eglInitialize() failed"
+spec@egl_khr_surfaceless_context@viewport,Fail
+
+# Regression from i965
+spec@ext_draw_buffers2@fbo-drawbuffers2-blend,Fail
+
+# "Failed blit src(0,0;1,1) - dst(0,0;2147483647,2147483647)
+#  Failed blit src(0,0;40,40) - dst(0,0;134217727,134217727)
+#  Failed blit src(0,0;40,40) - dst(0,0;2147483647,2147483647)
+#  Failed blit src(0,0;80,80) - dst(0,0;134217727,134217727)
+#  Failed blit src(0,0;80,80) - dst(0,0;2147483647,2147483647)
+#  Failed blit src(0,0;160,160) - dst(0,0;134217727,134217727)
+#  Failed blit src(0,0;160,160) - dst(0,0;2147483647,2147483647)
+#  Failed blit src(0,0;2147483647,2147483647) - dst(0,0;2147483647,2147483647)"
+spec@ext_framebuffer_blit@fbo-blit-check-limits,Fail
+
+# "../src/mesa/state_tracker/st_atom_blend.c:156:blend_per_rt: Assertion `st->has_indep_blend_func' failed."
+spec@ext_framebuffer_object@fbo-blending-format-quirks,Crash
+
+# "piglit: error: Failed at level 1, slice 1"
+spec@ext_gpu_shader4@execution@texelfetch@fs-texelfetch-isampler3d,Fail
+spec@ext_gpu_shader4@execution@texelfetch@fs-texelfetch-sampler3d,Fail
+spec@ext_gpu_shader4@execution@texelfetch@fs-texelfetch-usampler3d,Fail
+spec@ext_gpu_shader4@execution@texelfetch@vs-texelfetch-isampler3d,Fail
+spec@ext_gpu_shader4@execution@texelfetch@vs-texelfetch-sampler3d,Fail
+spec@ext_gpu_shader4@execution@texelfetch@vs-texelfetch-usampler3d,Fail
+spec@ext_gpu_shader4@execution@texelfetchoffset@fs-texelfetch-isampler3d,Fail
+spec@ext_gpu_shader4@execution@texelfetchoffset@fs-texelfetch-sampler3d,Fail
+spec@ext_gpu_shader4@execution@texelfetchoffset@fs-texelfetch-usampler3d,Fail
+spec@ext_gpu_shader4@execution@texelfetchoffset@vs-texelfetch-isampler3d,Fail
+spec@ext_gpu_shader4@execution@texelfetchoffset@vs-texelfetch-sampler3d,Fail
+spec@ext_gpu_shader4@execution@texelfetchoffset@vs-texelfetch-usampler3d,Fail
+
+# "Stopping after 100 failures
+#  Summary: -1/100 passed"
+spec@ext_gpu_shader4@tex-miplevel-selection gpu4texture() 1darrayshadow,Fail
+spec@ext_gpu_shader4@tex-miplevel-selection gpu4texture() 2darrayshadow,Fail
+spec@ext_gpu_shader4@tex-miplevel-selection gpu4texture() cubeshadow,Fail
+spec@ext_gpu_shader4@tex-miplevel-selection gpu4texture(bias) 1darrayshadow,Fail
+spec@ext_gpu_shader4@tex-miplevel-selection gpu4texture(bias) 2darrayshadow,Fail
+spec@ext_gpu_shader4@tex-miplevel-selection gpu4texture(bias) cubeshadow,Fail
+spec@ext_gpu_shader4@tex-miplevel-selection gpu4texturegrad 1darrayshadow,Fail
+spec@ext_gpu_shader4@tex-miplevel-selection gpu4texturegrad 2darray,Crash
+spec@ext_gpu_shader4@tex-miplevel-selection gpu4texturegrad 2darrayshadow,Fail
+spec@ext_gpu_shader4@tex-miplevel-selection gpu4texturegrad cubeshadow,Fail
+spec@ext_gpu_shader4@tex-miplevel-selection gpu4texturegradoffset 1darrayshadow,Fail
+spec@ext_gpu_shader4@tex-miplevel-selection gpu4texturegradoffset 2darray,Crash
+spec@ext_gpu_shader4@tex-miplevel-selection gpu4texturegradoffset 2darrayshadow,Fail
+spec@ext_gpu_shader4@tex-miplevel-selection gpu4texturelod 1darrayshadow,Fail
+spec@ext_gpu_shader4@tex-miplevel-selection gpu4texturelodoffset 1darrayshadow,Fail
+spec@ext_gpu_shader4@tex-miplevel-selection gpu4textureoffset 1darrayshadow,Fail
+spec@ext_gpu_shader4@tex-miplevel-selection gpu4textureoffset 2darrayshadow,Fail
+
+# Regression from i965
+# "Testing GL_LUMINANCE16F_ARB
+#  Expected: (1, 1, 1, 0)
+#  Actual: (1, 1, 1, 1)"
+spec@ext_packed_float@query-rgba-signed-components,Fail
+
+# "Unexpected GL error: GL_NO_ERROR 0x0
+#  (Error at /home/anholt/src/piglit/tests/spec/ext_texture_format_bgra8888/api-errors.c:63)
+#  Expected GL error: GL_INVALID_OPERATION 0x502
+#  Unexpected GL error: GL_NO_ERROR 0x0
+#  (Error at /home/anholt/src/piglit/tests/spec/ext_texture_format_bgra8888/api-errors.c:69)
+#  Expected GL error: GL_INVALID_OPERATION 0x502
+#  Unexpected GL error: GL_NO_ERROR 0x0
+#  (Error at /home/anholt/src/piglit/tests/spec/ext_texture_format_bgra8888/api-errors.c:103)
+#  Expected GL error: GL_INVALID_OPERATION 0x502"
+spec@ext_texture_format_bgra8888@api-errors,Fail
+
+# no info, just fails
+spec@intel_performance_query@intel_performance_query-issue_2235,Fail
+
+# "Unexpected GL error: GL_INVALID_ENUM 0x500
+#  (Error at tests/spec/khr_texture_compression_astc/khr_compressed_astc-basic.c:349)
+#  Expected GL error: GL_INVALID_OPERATION 0x502"
+spec@khr_texture_compression_astc@basic-gles,Fail
+
+spec@khr_texture_compression_astc@miptree-gl srgb-fp,Fail
+spec@khr_texture_compression_astc@miptree-gl srgb-fp@sRGB decode full precision,Fail
+spec@khr_texture_compression_astc@miptree-gles srgb,Fail
+spec@khr_texture_compression_astc@miptree-gles srgb-fp,Fail
+
+# new failures, needs debugging
+dEQP-GLES2.functional.polygon_offset.default_displacement_with_units,Fail
+dEQP-GLES2.functional.polygon_offset.default_render_with_factor,Fail
+dEQP-GLES2.functional.polygon_offset.fixed16_displacement_with_units,Fail
+dEQP-GLES2.functional.polygon_offset.fixed16_render_with_factor,Fail
+spec@!opengl 1.1@polygon-mode-offset,Fail
+spec@!opengl 1.1@polygon-mode-offset@config 0: Expected white pixel on bottom edge,Fail
+spec@!opengl 1.1@polygon-mode-offset@config 0: Expected white pixel on left edge,Fail
+spec@!opengl 1.1@polygon-mode-offset@config 0: Expected white pixel on right edge,Fail
+spec@!opengl 1.1@polygon-mode-offset@config 0: Expected white pixel on top edge,Fail
+spec@!opengl 1.1@polygon-mode-offset@config 3: Expected white pixel on bottom edge,Fail
+spec@!opengl 1.1@polygon-mode-offset@config 3: Expected white pixel on left edge,Fail
+spec@!opengl 1.1@polygon-mode-offset@config 3: Expected white pixel on right edge,Fail
+spec@!opengl 1.1@polygon-mode-offset@config 3: Expected white pixel on top edge,Fail
+spec@amd_performance_monitor@measure,Crash

src/gallium/drivers/crocus/ci/crocus-g41-flakes.txt

@@ -0,0 +1,35 @@
+# These seem stable on their own (or when running their whole test group),
+# but intermittently fail when mixed in with other tests.
+dEQP-GLES2.functional.shaders.functions.control_flow.return_after_loop_sequence_fragment
+dEQP-GLES2.functional.shaders.loops.do_while_uniform_iterations.nested_sequence_fragment
+dEQP-GLES2.functional.shaders.loops.while_uniform_iterations.nested_tricky_dataflow_2_fragment
+dEQP-GLES2.functional.shaders.loops.for_dynamic_iterations.nested_tricky_dataflow_2_fragment
+dEQP-GLES2.functional.shaders.loops.for_dynamic_iterations.nested_sequence_fragment
+dEQP-GLES2.functional.shaders.loops.for_uniform_iterations.basic_mediump_int_fragment
+dEQP-GLES2.functional.shaders.loops.for_uniform_iterations.nested_sequence_fragment
+
+KHR-GLES2.texture_3d.filtering.formats.rgba8_linear
+KHR-GLES2.texture_3d.filtering.formats.rgba8_linear_mipmap_linear
+KHR-GLES2.texture_3d.filtering.formats.rgba8_linear_mipmap_nearest
+KHR-GLES2.texture_3d.filtering.formats.rgba8_nearest
+
+spec@!opengl 2.0@attribs
+
+spec@arb_map_buffer_range@copybuffersubdata decrement-offset
+spec@arb_map_buffer_range@copybuffersubdata increment-offset
+spec@arb_map_buffer_range@copybuffersubdata offset=0
+spec@arb_texture_cube_map@copyteximage cube
+spec@arb_vertex_buffer_object@vbo-subdata-many drawarrays
+spec@arb_vertex_buffer_object@vbo-subdata-many drawelements
+spec@arb_vertex_buffer_object@vbo-subdata-many drawrangeelements
+spec@arb_vertex_type_2_10_10_10_rev@attribs
+
+spec@ext_framebuffer_multisample@accuracy all_samples depth_resolve small depthstencil
+spec@ext_gpu_shader4@tex-miplevel-selection gpu4texture() 2darray
+
+# GL_MAX_VARYING_FLOATS = 64
+# Probe color at (2,2)
+#   Expected: 0 255 0
+#   Observed: 255 255 255
+#   Failure with 1 vec4 varyings used in varying index 0
+shaders@glsl-max-varyings

src/gallium/drivers/crocus/ci/crocus-g41-skips.txt

@@ -0,0 +1,8 @@
+# Fails or timeouts, many involving a segfault.
+KHR-GLES2.texture_3d.filtering.combinations.*
+KHR-GLES2.texture_3d.filtering.sizes.*
+
+# We aren't running an X server.
+glx.*
+
+max-texture-size

src/gallium/drivers/crocus/ci/crocus-hsw-fails.txt

@@ -0,0 +1,958 @@
+spec@!opengl 1.0@depth-clear-precision-check,Fail
+spec@!opengl 1.0@depth-clear-precision-check@depth16,Fail
+spec@!opengl 1.0@depth-clear-precision-check@depth32,Fail
+
+spec@!opengl [email protected],Fail
+
+# Compat mode failure
+spec@!opengl 1.0@rasterpos,Fail
+
+spec@!opengl 1.1@line-smooth-coverage,Fail
+spec@!opengl 1.1@line-smooth-stipple,Fail
+
+spec@!opengl 1.1@linestipple,Fail
+spec@!opengl 1.1@linestipple@Factor 2x,Fail
+spec@!opengl 1.1@linestipple@Factor 3x,Fail
+spec@!opengl 1.1@linestipple@Line loop,Fail
+spec@!opengl 1.1@linestipple@Line strip,Fail
+
+spec@!opengl [email protected] cull-back pv-first,Fail
+spec@!opengl [email protected] cull-front pv-first,Fail
+spec@!opengl [email protected] line cull-back pv-first,Fail
+spec@!opengl [email protected] line cull-front pv-first,Fail
+spec@!opengl [email protected] pv-first,Fail
+
+spec@arb_direct_state_access@gettextureimage-formats init-by-rendering,Fail
+
+spec@arb_indirect_parameters@conditional-render,Fail
+spec@arb_indirect_parameters@conditional-render@Query result: Fail. Inverted mode: No.  Points drawn: 6,Fail
+spec@arb_indirect_parameters@conditional-render@Query result: Pass. Inverted mode: Yes.  Points drawn: 6,Fail
+spec@arb_indirect_parameters@conditional-render@Query result: Fail. Inverted mode: Yes.  Points drawn: 0,Fail
+spec@arb_indirect_parameters@conditional-render@Query result: Pass. Inverted mode: No.  Points drawn: 0,Fail
+
+spec@arb_program_interface_query@arb_program_interface_query-getprogramresourceindex,Fail
+spec@arb_program_interface_query@arb_program_interface_query-getprogramresourceindex@'vs_input2[1][0]' on GL_PROGRAM_INPUT,Fail
+
+spec@arb_query_buffer_object@qbo,Fail
+spec@arb_query_buffer_object@qbo@query-GL_TIMESTAMP-ASYNC_CPU_READ_BEFORE-GL_INT,Fail
+spec@arb_query_buffer_object@qbo@query-GL_TIMESTAMP-ASYNC_CPU_READ_BEFORE-GL_UNSIGNED_INT,Fail
+spec@arb_query_buffer_object@qbo@query-GL_TIMESTAMP-SYNC_CPU_READ_AFTER_CACHE_TEST-GL_INT,Fail
+spec@arb_query_buffer_object@qbo@query-GL_TIMESTAMP-SYNC_CPU_READ_AFTER_CACHE_TEST-GL_UNSIGNED_INT,Fail
+spec@arb_query_buffer_object@qbo@query-GL_TIMESTAMP-SYNC_CPU_READ_AFTER_CACHE_TEST-GL_UNSIGNED_INT64_ARB,Fail
+spec@arb_query_buffer_object@qbo@query-GL_TIME_ELAPSED-SYNC_CPU_READ_AFTER_CACHE_TEST-GL_INT,Fail
+spec@arb_query_buffer_object@qbo@query-GL_TIME_ELAPSED-SYNC_CPU_READ_AFTER_CACHE_TEST-GL_UNSIGNED_INT,Fail
+spec@arb_query_buffer_object@qbo@query-GL_TIME_ELAPSED-SYNC_CPU_READ_AFTER_CACHE_TEST-GL_UNSIGNED_INT64_ARB,Fail
+
+spec@arb_sample_shading@samplemask 2,Fail
+spec@arb_sample_shading@samplemask 2 all,Fail
+spec@arb_sample_shading@samplemask 2 [email protected] mask_in_one,Fail
+spec@arb_sample_shading@samplemask 2 [email protected] mask_in_one,Fail
+spec@arb_sample_shading@samplemask 2 [email protected] mask_in_one,Fail
+spec@arb_sample_shading@samplemask 2 all@noms mask_in_one,Fail
+spec@arb_sample_shading@samplemask 2 all@sample mask_in_one,Fail
+spec@arb_sample_shading@samplemask [email protected] mask_in_one,Fail
+spec@arb_sample_shading@samplemask [email protected] mask_in_one,Fail
+spec@arb_sample_shading@samplemask [email protected] mask_in_one,Fail
+spec@arb_sample_shading@samplemask 2@noms mask_in_one,Fail
+spec@arb_sample_shading@samplemask 2@sample mask_in_one,Fail
+spec@arb_sample_shading@samplemask 4,Fail
+spec@arb_sample_shading@samplemask 4 all,Fail
+spec@arb_sample_shading@samplemask 4 [email protected] mask_in_one,Fail
+spec@arb_sample_shading@samplemask 4 [email protected] mask_in_one,Fail
+spec@arb_sample_shading@samplemask 4 [email protected] mask_in_one,Fail
+spec@arb_sample_shading@samplemask 4 all@noms mask_in_one,Fail
+spec@arb_sample_shading@samplemask 4 all@sample mask_in_one,Fail
+spec@arb_sample_shading@samplemask [email protected] mask_in_one,Fail
+spec@arb_sample_shading@samplemask [email protected] mask_in_one,Fail
+spec@arb_sample_shading@samplemask [email protected] mask_in_one,Fail
+spec@arb_sample_shading@samplemask 4@noms mask_in_one,Fail
+spec@arb_sample_shading@samplemask 4@sample mask_in_one,Fail
+spec@arb_sample_shading@samplemask 6,Fail
+spec@arb_sample_shading@samplemask 6 all,Fail
+spec@arb_sample_shading@samplemask 6 [email protected] mask_in_one,Fail
+spec@arb_sample_shading@samplemask 6 [email protected] mask_in_one,Fail
+spec@arb_sample_shading@samplemask 6 [email protected] mask_in_one,Fail
+spec@arb_sample_shading@samplemask 6 [email protected] mask_in_one,Fail
+spec@arb_sample_shading@samplemask 6 all@noms mask_in_one,Fail
+spec@arb_sample_shading@samplemask 6 all@sample mask_in_one,Fail
+spec@arb_sample_shading@samplemask [email protected] mask_in_one,Fail
+spec@arb_sample_shading@samplemask [email protected] mask_in_one,Fail
+spec@arb_sample_shading@samplemask [email protected] mask_in_one,Fail
+spec@arb_sample_shading@samplemask [email protected] mask_in_one,Fail
+spec@arb_sample_shading@samplemask 6@noms mask_in_one,Fail
+spec@arb_sample_shading@samplemask 6@sample mask_in_one,Fail
+spec@arb_sample_shading@samplemask 8,Fail
+spec@arb_sample_shading@samplemask 8 all,Fail
+spec@arb_sample_shading@samplemask 8 [email protected] mask_in_one,Fail
+spec@arb_sample_shading@samplemask 8 [email protected] mask_in_one,Fail
+spec@arb_sample_shading@samplemask 8 [email protected] mask_in_one,Fail
+spec@arb_sample_shading@samplemask 8 [email protected] mask_in_one,Fail
+spec@arb_sample_shading@samplemask 8 all@noms mask_in_one,Fail
+spec@arb_sample_shading@samplemask 8 all@sample mask_in_one,Fail
+spec@arb_sample_shading@samplemask [email protected] mask_in_one,Fail
+spec@arb_sample_shading@samplemask [email protected] mask_in_one,Fail
+spec@arb_sample_shading@samplemask [email protected] mask_in_one,Fail
+spec@arb_sample_shading@samplemask [email protected] mask_in_one,Fail
+spec@arb_sample_shading@samplemask 8@noms mask_in_one,Fail
+spec@arb_sample_shading@samplemask 8@sample mask_in_one,Fail
+
+spec@arb_shader_image_load_store@early-z,Fail
+spec@arb_shader_image_load_store@early-z@occlusion query test/early-z pass,Fail
+
+spec@arb_shader_texture_lod@execution@arb_shader_texture_lod-texgradcube,Fail
+
+spec@arb_texture_buffer_object@formats (fs- arb),Crash
+spec@arb_texture_buffer_object@formats (vs- arb),Crash
+
+spec@arb_texture_view@rendering-formats,Fail
+spec@arb_texture_view@rendering-formats@clear GL_RGB10_A2 as GL_R32F,Fail
+spec@arb_texture_view@rendering-formats@clear GL_RGB10_A2 as GL_RG16F,Fail
+spec@arb_texture_view@rendering-formats@clear GL_RGB10_A2 as GL_RG16I,Fail
+spec@arb_texture_view@rendering-formats@clear GL_RGB10_A2 as GL_RG16_SNORM,Fail
+spec@arb_texture_view@rendering-formats@clear GL_RGB10_A2 as GL_RGBA8I,Fail
+spec@arb_texture_view@rendering-formats@clear GL_RGB10_A2 as GL_RGBA8_SNORM,Fail
+
+spec@egl_ext_protected_content@conformance,Fail
+
+spec@egl 1.4@eglterminate then unbind context,Fail
+
+spec@egl_khr_gl_image@egl_khr_gl_renderbuffer_image-clear-shared-image gl_depth_component24,Fail
+
+# "eglInitialize() failed"
+spec@egl_khr_surfaceless_context@viewport,Fail
+
+spec@ext_external_objects@vk-semaphores,Fail
+
+spec@ext_framebuffer_blit@fbo-blit-check-limits,Fail
+
+spec@ext_image_dma_buf_import@ext_image_dma_buf_import-export-tex,Fail
+# "Testing GL_LUMINANCE8_SNORM
+#  Expected: (1, 1, 1, 0)
+#  Actual: (1, 1, 1, 1)"
+spec@ext_packed_float@query-rgba-signed-components,Fail
+
+spec@ext_transform_feedback@tessellation triangle_fan flat_first,Fail
+
+# due to point size clamping in shaders since hw doesn't support it later.
+spec@ext_transform_feedback@builtin-varyings gl_pointsize,Fail
+
+[email protected]@execution@geometry@primitive-types gl_line_loop,Fail
+
+# Compat mode failures
+[email protected]@execution@primitive-id-no-gs-quads,Fail
+[email protected]@execution@primitive-id-no-gs-quad-strip,Fail
+
+spec@khr_texture_compression_astc@miptree-gl srgb-fp,Fail
+spec@khr_texture_compression_astc@miptree-gl srgb-fp@sRGB decode full precision,Fail
+spec@khr_texture_compression_astc@miptree-gles srgb-fp,Fail
+spec@khr_texture_compression_astc@miptree-gles srgb-fp@sRGB decode full precision,Fail
+spec@khr_texture_compression_astc@sliced-3d-miptree-gl srgb-fp,Fail
+spec@khr_texture_compression_astc@sliced-3d-miptree-gl srgb-fp@sRGB decode full precision,Fail
+spec@khr_texture_compression_astc@sliced-3d-miptree-gles srgb-fp,Fail
+spec@khr_texture_compression_astc@sliced-3d-miptree-gles srgb-fp@sRGB decode full precision,Fail
+
+spec@oes_texture_view@rendering-formats,Fail
+spec@oes_texture_view@rendering-formats@clear GL_RGB10_A2 as GL_R32F,Fail
+spec@oes_texture_view@rendering-formats@clear GL_RGB10_A2 as GL_RG16F,Fail
+spec@oes_texture_view@rendering-formats@clear GL_RGB10_A2 as GL_RG16I,Fail
+spec@oes_texture_view@rendering-formats@clear GL_RGB10_A2 as GL_RG16_SNORM,Fail
+spec@oes_texture_view@rendering-formats@clear GL_RGB10_A2 as GL_RGBA8I,Fail
+spec@oes_texture_view@rendering-formats@clear GL_RGB10_A2 as GL_RGBA8_SNORM,Fail
+
+KHR-GL46.direct_state_access.renderbuffers_storage_multisample,Fail
+KHR-GL46.direct_state_access.textures_storage_multisample_2d_r16i,Fail
+KHR-GL46.direct_state_access.textures_storage_multisample_2d_r32i,Fail
+KHR-GL46.direct_state_access.textures_storage_multisample_2d_r8i,Fail
+KHR-GL46.direct_state_access.textures_storage_multisample_2d_rg16i,Fail
+KHR-GL46.direct_state_access.textures_storage_multisample_2d_rg32i,Fail
+KHR-GL46.direct_state_access.textures_storage_multisample_2d_rg8i,Fail
+KHR-GL46.direct_state_access.textures_storage_multisample_2d_rgb32i,Fail
+KHR-GL46.direct_state_access.textures_storage_multisample_2d_rgba16i,Fail
+KHR-GL46.direct_state_access.textures_storage_multisample_2d_rgba32i,Fail
+KHR-GL46.direct_state_access.textures_storage_multisample_2d_rgba8i,Fail
+KHR-GL46.direct_state_access.textures_storage_multisample_3d_r16i,Fail
+KHR-GL46.direct_state_access.textures_storage_multisample_3d_r32i,Fail
+KHR-GL46.direct_state_access.textures_storage_multisample_3d_r8i,Fail
+KHR-GL46.direct_state_access.textures_storage_multisample_3d_rg16i,Fail
+KHR-GL46.direct_state_access.textures_storage_multisample_3d_rg32i,Fail
+KHR-GL46.direct_state_access.textures_storage_multisample_3d_rg8i,Fail
+KHR-GL46.direct_state_access.textures_storage_multisample_3d_rgb32i,Fail
+KHR-GL46.direct_state_access.textures_storage_multisample_3d_rgba16i,Fail
+KHR-GL46.direct_state_access.textures_storage_multisample_3d_rgba32i,Fail
+KHR-GL46.direct_state_access.textures_storage_multisample_3d_rgba8i,Fail
+
+# glcts: ../src/compiler/nir/nir.h:127: nir_component_mask: Assertion `nir_num_components_valid(num_components)' failed.
+KHR-GL46.gpu_shader_fp64.fp64.state_query,Crash
+
+KHR-GL46.sample_shading.render.rgba8i.full,Fail
+KHR-GL46.sample_shading.render.rgba8i.half,Fail
+KHR-GL46.sample_shading.render.rgba8i.none,Fail
+KHR-GL46.sample_variables.mask.rgba8i.samples_1.mask_0,Fail
+KHR-GL46.sample_variables.mask.rgba8i.samples_1.mask_1,Fail
+KHR-GL46.sample_variables.mask.rgba8i.samples_1.mask_2,Fail
+KHR-GL46.sample_variables.mask.rgba8i.samples_1.mask_3,Fail
+KHR-GL46.sample_variables.mask.rgba8i.samples_1.mask_4,Fail
+KHR-GL46.sample_variables.mask.rgba8i.samples_1.mask_5,Fail
+KHR-GL46.sample_variables.mask.rgba8i.samples_1.mask_6,Fail
+KHR-GL46.sample_variables.mask.rgba8i.samples_1.mask_7,Fail
+KHR-GL46.sample_variables.mask.rgba8i.samples_1.mask_zero,Fail
+
+# https://gitlab.freedesktop.org/mesa/mesa/-/issues/8900
+KHR-GL46.shader_texture_image_samples_tests.texture_functional_test,Fail
+
+KHR-GL46.shader_multisample_interpolation.render.base.rgba8i.samples_1,Fail
+KHR-GL46.shader_multisample_interpolation.render.centroid.rgba8i.samples_1,Fail
+KHR-GL46.shader_multisample_interpolation.render.interpolate_at_centroid.rgba8i.samples_1,Fail
+KHR-GL46.shader_multisample_interpolation.render.interpolate_at_centroid_check.rgba8i.samples_1,Fail
+KHR-GL46.shader_multisample_interpolation.render.interpolate_at_offset.rgba8i.samples_1,Fail
+KHR-GL46.shader_multisample_interpolation.render.interpolate_at_offset_check.rgba8i.samples_1,Fail
+KHR-GL46.shader_multisample_interpolation.render.interpolate_at_sample.rgba8i.samples_1,Fail
+KHR-GL46.shader_multisample_interpolation.render.interpolate_at_sample_check.rgba8i.samples_1,Fail
+KHR-GL46.shader_multisample_interpolation.render.sample.rgba8i.samples_1,Fail
+
+# "glcts: ../src/intel/compiler/brw_vec4_generator.cpp:2254: void generate_code(brw_codegen*, const brw_compiler*, void*, const nir_shader*, brw_vue_prog_data*, const cfg_t*, const brw::performance&, brw_compile_stats*, bool): Assertion `validated' failed."
+KHR-GL46.shading_language_420pack.binding_samplers,Crash
+
+KHR-GL46.texture_size_promotion.functional,Fail
+KHR-GLES31.core.sample_shading.render.rgba8i.full,Fail
+KHR-GLES31.core.sample_shading.render.rgba8i.half,Fail
+KHR-GLES31.core.sample_shading.render.rgba8i.none,Fail
+KHR-GLES31.core.sample_variables.mask.rgba8i.samples_1.mask_0,Fail
+KHR-GLES31.core.sample_variables.mask.rgba8i.samples_1.mask_1,Fail
+KHR-GLES31.core.sample_variables.mask.rgba8i.samples_1.mask_2,Fail
+KHR-GLES31.core.sample_variables.mask.rgba8i.samples_1.mask_3,Fail
+KHR-GLES31.core.sample_variables.mask.rgba8i.samples_1.mask_4,Fail
+KHR-GLES31.core.sample_variables.mask.rgba8i.samples_1.mask_5,Fail
+KHR-GLES31.core.sample_variables.mask.rgba8i.samples_1.mask_6,Fail
+KHR-GLES31.core.sample_variables.mask.rgba8i.samples_1.mask_7,Fail
+KHR-GLES31.core.sample_variables.mask.rgba8i.samples_1.mask_zero,Fail
+KHR-GLES31.core.shader_multisample_interpolation.render.base.rgba8i.samples_1,Fail
+KHR-GLES31.core.shader_multisample_interpolation.render.centroid.rgba8i.samples_1,Fail
+KHR-GLES31.core.shader_multisample_interpolation.render.interpolate_at_centroid.rgba8i.samples_1,Fail
+KHR-GLES31.core.shader_multisample_interpolation.render.interpolate_at_centroid_check.rgba8i.samples_1,Fail
+KHR-GLES31.core.shader_multisample_interpolation.render.interpolate_at_offset.rgba8i.samples_1,Fail
+KHR-GLES31.core.shader_multisample_interpolation.render.interpolate_at_offset_check.rgba8i.samples_1,Fail
+KHR-GLES31.core.shader_multisample_interpolation.render.interpolate_at_sample.rgba8i.samples_1,Fail
+KHR-GLES31.core.shader_multisample_interpolation.render.interpolate_at_sample_check.rgba8i.samples_1,Fail
+KHR-GLES31.core.shader_multisample_interpolation.render.sample.rgba8i.samples_1,Fail
+KHR-GLES31.core.texture_storage_multisample.APIDependencies.renderbuffer_storage_multisample_invalid_samples_argument_for_integer_internalformats,Fail
+dEQP-GLES31.functional.shaders.builtin_functions.texture_size.samples_1_texture_int_2d,Fail
+dEQP-GLES31.functional.shaders.builtin_functions.texture_size.samples_1_texture_int_2d_array,Fail
+dEQP-GLES31.functional.texture.multisample.samples_1.use_texture_int_2d,Fail
+dEQP-GLES31.functional.texture.multisample.samples_1.use_texture_int_2d_array,Fail
+
+spec@arb_copy_image@arb_copy_image-formats --samples=2,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_R16/Destination: GL_R16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_R16/Destination: GL_RG8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_R16I/Destination: GL_R16,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_R16I/Destination: GL_R16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_R16I/Destination: GL_R16UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_R16I/Destination: GL_R16_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_R16I/Destination: GL_RG8,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_R16I/Destination: GL_RG8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_R16I/Destination: GL_RG8UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_R16I/Destination: GL_RG8_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_R16UI/Destination: GL_R16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_R16UI/Destination: GL_RG8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_R16_SNORM/Destination: GL_R16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_R16_SNORM/Destination: GL_RG8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_R32F/Destination: GL_RG16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_R32F/Destination: GL_RGBA8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_R8/Destination: GL_R8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_R8I/Destination: GL_R8,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_R8I/Destination: GL_R8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_R8I/Destination: GL_R8UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_R8I/Destination: GL_R8_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_R8UI/Destination: GL_R8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_R8_SNORM/Destination: GL_R8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG16/Destination: GL_RG16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG16/Destination: GL_RGBA8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG16I/Destination: GL_R32F,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG16I/Destination: GL_RG16,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG16I/Destination: GL_RG16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG16I/Destination: GL_RG16UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG16I/Destination: GL_RG16_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG16I/Destination: GL_RGBA8,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG16I/Destination: GL_RGBA8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG16I/Destination: GL_RGBA8UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG16I/Destination: GL_RGBA8_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG16UI/Destination: GL_RG16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG16UI/Destination: GL_RGBA8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG16_SNORM/Destination: GL_RG16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG16_SNORM/Destination: GL_RGBA8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG32F/Destination: GL_RG32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG32F/Destination: GL_RGBA16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG32I/Destination: GL_RG32F,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG32I/Destination: GL_RG32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG32I/Destination: GL_RG32UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG32I/Destination: GL_RGBA16,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG32I/Destination: GL_RGBA16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG32I/Destination: GL_RGBA16UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG32I/Destination: GL_RGBA16_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG32UI/Destination: GL_RG32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG32UI/Destination: GL_RGBA16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG8/Destination: GL_R16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG8/Destination: GL_RG8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG8I/Destination: GL_R16,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG8I/Destination: GL_R16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG8I/Destination: GL_R16UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG8I/Destination: GL_R16_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG8I/Destination: GL_RG8,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG8I/Destination: GL_RG8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG8I/Destination: GL_RG8UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG8I/Destination: GL_RG8_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG8UI/Destination: GL_R16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG8UI/Destination: GL_RG8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG8_SNORM/Destination: GL_R16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RG8_SNORM/Destination: GL_RG8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGB16/Destination: GL_RGB16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGB16I/Destination: GL_RGB16,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGB16I/Destination: GL_RGB16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGB16I/Destination: GL_RGB16UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGB16I/Destination: GL_RGB16_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGB16UI/Destination: GL_RGB16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGB16_SNORM/Destination: GL_RGB16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGB32F/Destination: GL_RGB32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGB32I/Destination: GL_RGB32F,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGB32I/Destination: GL_RGB32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGB32I/Destination: GL_RGB32UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGB32UI/Destination: GL_RGB32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGB8/Destination: GL_RGB8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGB8I/Destination: GL_RGB8,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGB8I/Destination: GL_RGB8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGB8I/Destination: GL_RGB8UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGB8I/Destination: GL_RGB8_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGB8UI/Destination: GL_RGB8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGB8_SNORM/Destination: GL_RGB8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGBA16/Destination: GL_RG32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGBA16/Destination: GL_RGBA16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGBA16I/Destination: GL_RG32F,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGBA16I/Destination: GL_RG32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGBA16I/Destination: GL_RG32UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGBA16I/Destination: GL_RGBA16,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGBA16I/Destination: GL_RGBA16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGBA16I/Destination: GL_RGBA16UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGBA16I/Destination: GL_RGBA16_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGBA16UI/Destination: GL_RG32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGBA16UI/Destination: GL_RGBA16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGBA16_SNORM/Destination: GL_RG32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGBA16_SNORM/Destination: GL_RGBA16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGBA32F/Destination: GL_RGBA32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGBA32I/Destination: GL_RGBA32F,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGBA32I/Destination: GL_RGBA32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGBA32I/Destination: GL_RGBA32UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGBA32UI/Destination: GL_RGBA32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGBA8/Destination: GL_RG16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGBA8/Destination: GL_RGBA8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGBA8I/Destination: GL_R32F,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGBA8I/Destination: GL_RG16,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGBA8I/Destination: GL_RG16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGBA8I/Destination: GL_RG16UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGBA8I/Destination: GL_RG16_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGBA8I/Destination: GL_RGBA8,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGBA8I/Destination: GL_RGBA8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGBA8I/Destination: GL_RGBA8UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGBA8I/Destination: GL_RGBA8_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGBA8UI/Destination: GL_RG16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGBA8UI/Destination: GL_RGBA8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGBA8_SNORM/Destination: GL_RG16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=2@Source: GL_RGBA8_SNORM/Destination: GL_RGBA8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_R16/Destination: GL_R16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_R16/Destination: GL_RG8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_R16I/Destination: GL_R16,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_R16I/Destination: GL_R16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_R16I/Destination: GL_R16UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_R16I/Destination: GL_R16_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_R16I/Destination: GL_RG8,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_R16I/Destination: GL_RG8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_R16I/Destination: GL_RG8UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_R16I/Destination: GL_RG8_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_R16UI/Destination: GL_R16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_R16UI/Destination: GL_RG8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_R16_SNORM/Destination: GL_R16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_R16_SNORM/Destination: GL_RG8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_R32F/Destination: GL_RG16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_R32F/Destination: GL_RGBA8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_R8/Destination: GL_R8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_R8I/Destination: GL_R8,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_R8I/Destination: GL_R8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_R8I/Destination: GL_R8UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_R8I/Destination: GL_R8_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_R8UI/Destination: GL_R8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_R8_SNORM/Destination: GL_R8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG16/Destination: GL_RG16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG16/Destination: GL_RGBA8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG16I/Destination: GL_R32F,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG16I/Destination: GL_RG16,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG16I/Destination: GL_RG16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG16I/Destination: GL_RG16UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG16I/Destination: GL_RG16_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG16I/Destination: GL_RGBA8,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG16I/Destination: GL_RGBA8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG16I/Destination: GL_RGBA8UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG16I/Destination: GL_RGBA8_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG16UI/Destination: GL_RG16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG16UI/Destination: GL_RGBA8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG16_SNORM/Destination: GL_RG16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG16_SNORM/Destination: GL_RGBA8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG32F/Destination: GL_RG32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG32F/Destination: GL_RGBA16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG32I/Destination: GL_RG32F,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG32I/Destination: GL_RG32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG32I/Destination: GL_RG32UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG32I/Destination: GL_RGBA16,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG32I/Destination: GL_RGBA16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG32I/Destination: GL_RGBA16UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG32I/Destination: GL_RGBA16_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG32UI/Destination: GL_RG32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG32UI/Destination: GL_RGBA16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG8/Destination: GL_R16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG8/Destination: GL_RG8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG8I/Destination: GL_R16,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG8I/Destination: GL_R16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG8I/Destination: GL_R16UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG8I/Destination: GL_R16_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG8I/Destination: GL_RG8,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG8I/Destination: GL_RG8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG8I/Destination: GL_RG8UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG8I/Destination: GL_RG8_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG8UI/Destination: GL_R16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG8UI/Destination: GL_RG8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG8_SNORM/Destination: GL_R16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RG8_SNORM/Destination: GL_RG8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGB16/Destination: GL_RGB16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGB16I/Destination: GL_RGB16,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGB16I/Destination: GL_RGB16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGB16I/Destination: GL_RGB16UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGB16I/Destination: GL_RGB16_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGB16UI/Destination: GL_RGB16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGB16_SNORM/Destination: GL_RGB16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGB32F/Destination: GL_RGB32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGB32I/Destination: GL_RGB32F,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGB32I/Destination: GL_RGB32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGB32I/Destination: GL_RGB32UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGB32UI/Destination: GL_RGB32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGB8/Destination: GL_RGB8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGB8I/Destination: GL_RGB8,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGB8I/Destination: GL_RGB8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGB8I/Destination: GL_RGB8UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGB8I/Destination: GL_RGB8_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGB8UI/Destination: GL_RGB8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGB8_SNORM/Destination: GL_RGB8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGBA16/Destination: GL_RG32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGBA16/Destination: GL_RGBA16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGBA16I/Destination: GL_RG32F,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGBA16I/Destination: GL_RG32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGBA16I/Destination: GL_RG32UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGBA16I/Destination: GL_RGBA16,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGBA16I/Destination: GL_RGBA16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGBA16I/Destination: GL_RGBA16UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGBA16I/Destination: GL_RGBA16_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGBA16UI/Destination: GL_RG32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGBA16UI/Destination: GL_RGBA16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGBA16_SNORM/Destination: GL_RG32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGBA16_SNORM/Destination: GL_RGBA16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGBA32F/Destination: GL_RGBA32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGBA32I/Destination: GL_RGBA32F,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGBA32I/Destination: GL_RGBA32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGBA32I/Destination: GL_RGBA32UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGBA32UI/Destination: GL_RGBA32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGBA8/Destination: GL_RG16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGBA8/Destination: GL_RGBA8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGBA8I/Destination: GL_R32F,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGBA8I/Destination: GL_RG16,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGBA8I/Destination: GL_RG16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGBA8I/Destination: GL_RG16UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGBA8I/Destination: GL_RG16_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGBA8I/Destination: GL_RGBA8,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGBA8I/Destination: GL_RGBA8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGBA8I/Destination: GL_RGBA8UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGBA8I/Destination: GL_RGBA8_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGBA8UI/Destination: GL_RG16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGBA8UI/Destination: GL_RGBA8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGBA8_SNORM/Destination: GL_RG16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=4@Source: GL_RGBA8_SNORM/Destination: GL_RGBA8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_R16/Destination: GL_R16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_R16/Destination: GL_RG8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_R16I/Destination: GL_R16,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_R16I/Destination: GL_R16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_R16I/Destination: GL_R16UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_R16I/Destination: GL_R16_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_R16I/Destination: GL_RG8,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_R16I/Destination: GL_RG8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_R16I/Destination: GL_RG8UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_R16I/Destination: GL_RG8_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_R16UI/Destination: GL_R16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_R16UI/Destination: GL_RG8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_R16_SNORM/Destination: GL_R16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_R16_SNORM/Destination: GL_RG8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_R32F/Destination: GL_RG16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_R32F/Destination: GL_RGBA8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_R8/Destination: GL_R8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_R8I/Destination: GL_R8,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_R8I/Destination: GL_R8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_R8I/Destination: GL_R8UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_R8I/Destination: GL_R8_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_R8UI/Destination: GL_R8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_R8_SNORM/Destination: GL_R8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG16/Destination: GL_RG16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG16/Destination: GL_RGBA8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG16I/Destination: GL_R32F,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG16I/Destination: GL_RG16,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG16I/Destination: GL_RG16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG16I/Destination: GL_RG16UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG16I/Destination: GL_RG16_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG16I/Destination: GL_RGBA8,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG16I/Destination: GL_RGBA8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG16I/Destination: GL_RGBA8UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG16I/Destination: GL_RGBA8_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG16UI/Destination: GL_RG16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG16UI/Destination: GL_RGBA8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG16_SNORM/Destination: GL_RG16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG16_SNORM/Destination: GL_RGBA8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG32F/Destination: GL_RG32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG32F/Destination: GL_RGBA16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG32I/Destination: GL_RG32F,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG32I/Destination: GL_RG32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG32I/Destination: GL_RG32UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG32I/Destination: GL_RGBA16,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG32I/Destination: GL_RGBA16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG32I/Destination: GL_RGBA16UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG32I/Destination: GL_RGBA16_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG32UI/Destination: GL_RG32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG32UI/Destination: GL_RGBA16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG8/Destination: GL_R16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG8/Destination: GL_RG8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG8I/Destination: GL_R16,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG8I/Destination: GL_R16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG8I/Destination: GL_R16UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG8I/Destination: GL_R16_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG8I/Destination: GL_RG8,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG8I/Destination: GL_RG8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG8I/Destination: GL_RG8UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG8I/Destination: GL_RG8_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG8UI/Destination: GL_R16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG8UI/Destination: GL_RG8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG8_SNORM/Destination: GL_R16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RG8_SNORM/Destination: GL_RG8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGB16/Destination: GL_RGB16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGB16I/Destination: GL_RGB16,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGB16I/Destination: GL_RGB16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGB16I/Destination: GL_RGB16UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGB16I/Destination: GL_RGB16_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGB16UI/Destination: GL_RGB16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGB16_SNORM/Destination: GL_RGB16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGB32F/Destination: GL_RGB32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGB32I/Destination: GL_RGB32F,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGB32I/Destination: GL_RGB32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGB32I/Destination: GL_RGB32UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGB32UI/Destination: GL_RGB32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGB8/Destination: GL_RGB8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGB8I/Destination: GL_RGB8,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGB8I/Destination: GL_RGB8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGB8I/Destination: GL_RGB8UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGB8I/Destination: GL_RGB8_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGB8UI/Destination: GL_RGB8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGB8_SNORM/Destination: GL_RGB8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGBA16/Destination: GL_RG32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGBA16/Destination: GL_RGBA16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGBA16I/Destination: GL_RG32F,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGBA16I/Destination: GL_RG32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGBA16I/Destination: GL_RG32UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGBA16I/Destination: GL_RGBA16,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGBA16I/Destination: GL_RGBA16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGBA16I/Destination: GL_RGBA16UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGBA16I/Destination: GL_RGBA16_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGBA16UI/Destination: GL_RG32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGBA16UI/Destination: GL_RGBA16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGBA16_SNORM/Destination: GL_RG32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGBA16_SNORM/Destination: GL_RGBA16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGBA32F/Destination: GL_RGBA32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGBA32I/Destination: GL_RGBA32F,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGBA32I/Destination: GL_RGBA32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGBA32I/Destination: GL_RGBA32UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGBA32UI/Destination: GL_RGBA32I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGBA8/Destination: GL_RG16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGBA8/Destination: GL_RGBA8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGBA8I/Destination: GL_R32F,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGBA8I/Destination: GL_RG16,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGBA8I/Destination: GL_RG16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGBA8I/Destination: GL_RG16UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGBA8I/Destination: GL_RG16_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGBA8I/Destination: GL_RGBA8,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGBA8I/Destination: GL_RGBA8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGBA8I/Destination: GL_RGBA8UI,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGBA8I/Destination: GL_RGBA8_SNORM,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGBA8UI/Destination: GL_RG16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGBA8UI/Destination: GL_RGBA8I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGBA8_SNORM/Destination: GL_RG16I,Fail
+spec@arb_copy_image@arb_copy_image-formats --samples=8@Source: GL_RGBA8_SNORM/Destination: GL_RGBA8I,Fail
+spec@arb_texture_multisample@texturesize@fs-texturesize-isampler2dms,Fail
+spec@arb_texture_multisample@texturesize@fs-texturesize-isampler2dmsarray,Fail
+spec@arb_texture_multisample@texturesize@gs-texturesize-isampler2dms,Fail
+spec@arb_texture_multisample@texturesize@gs-texturesize-isampler2dmsarray,Fail
+spec@arb_texture_multisample@texturesize@tes-texturesize-isampler2dms,Fail
+spec@arb_texture_multisample@texturesize@tes-texturesize-isampler2dmsarray,Fail
+spec@arb_texture_multisample@texturesize@vs-texturesize-isampler2dms,Fail
+spec@arb_texture_multisample@texturesize@vs-texturesize-isampler2dmsarray,Fail
+spec@arb_texture_rg@multisample-fast-clear gl_arb_texture_rg-int,Fail
+spec@ext_shader_framebuffer_fetch_non_coherent@execution@gles3@integer-ms2,Fail
+spec@ext_shader_framebuffer_fetch_non_coherent@execution@gles3@integer-ms8,Fail
+spec@ext_texture_integer@multisample-fast-clear gl_ext_texture_integer,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_R16/Destination: GL_R16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_R16/Destination: GL_RG8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_R16I/Destination: GL_R16,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_R16I/Destination: GL_R16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_R16I/Destination: GL_R16UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_R16I/Destination: GL_R16_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_R16I/Destination: GL_RG8,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_R16I/Destination: GL_RG8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_R16I/Destination: GL_RG8UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_R16I/Destination: GL_RG8_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_R16UI/Destination: GL_R16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_R16UI/Destination: GL_RG8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_R16_SNORM/Destination: GL_R16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_R16_SNORM/Destination: GL_RG8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_R32F/Destination: GL_RG16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_R32F/Destination: GL_RGBA8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_R8/Destination: GL_R8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_R8I/Destination: GL_R8,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_R8I/Destination: GL_R8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_R8I/Destination: GL_R8UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_R8I/Destination: GL_R8_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_R8UI/Destination: GL_R8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_R8_SNORM/Destination: GL_R8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG16/Destination: GL_RG16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG16/Destination: GL_RGBA8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG16I/Destination: GL_R32F,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG16I/Destination: GL_RG16,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG16I/Destination: GL_RG16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG16I/Destination: GL_RG16UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG16I/Destination: GL_RG16_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG16I/Destination: GL_RGBA8,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG16I/Destination: GL_RGBA8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG16I/Destination: GL_RGBA8UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG16I/Destination: GL_RGBA8_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG16UI/Destination: GL_RG16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG16UI/Destination: GL_RGBA8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG16_SNORM/Destination: GL_RG16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG16_SNORM/Destination: GL_RGBA8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG32F/Destination: GL_RG32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG32F/Destination: GL_RGBA16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG32I/Destination: GL_RG32F,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG32I/Destination: GL_RG32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG32I/Destination: GL_RG32UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG32I/Destination: GL_RGBA16,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG32I/Destination: GL_RGBA16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG32I/Destination: GL_RGBA16UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG32I/Destination: GL_RGBA16_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG32UI/Destination: GL_RG32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG32UI/Destination: GL_RGBA16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG8/Destination: GL_R16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG8/Destination: GL_RG8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG8I/Destination: GL_R16,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG8I/Destination: GL_R16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG8I/Destination: GL_R16UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG8I/Destination: GL_R16_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG8I/Destination: GL_RG8,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG8I/Destination: GL_RG8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG8I/Destination: GL_RG8UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG8I/Destination: GL_RG8_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG8UI/Destination: GL_R16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG8UI/Destination: GL_RG8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG8_SNORM/Destination: GL_R16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RG8_SNORM/Destination: GL_RG8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGB16/Destination: GL_RGB16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGB16I/Destination: GL_RGB16,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGB16I/Destination: GL_RGB16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGB16I/Destination: GL_RGB16UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGB16I/Destination: GL_RGB16_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGB16UI/Destination: GL_RGB16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGB16_SNORM/Destination: GL_RGB16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGB32F/Destination: GL_RGB32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGB32I/Destination: GL_RGB32F,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGB32I/Destination: GL_RGB32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGB32I/Destination: GL_RGB32UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGB32UI/Destination: GL_RGB32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGB8/Destination: GL_RGB8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGB8I/Destination: GL_RGB8,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGB8I/Destination: GL_RGB8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGB8I/Destination: GL_RGB8UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGB8I/Destination: GL_RGB8_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGB8UI/Destination: GL_RGB8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGB8_SNORM/Destination: GL_RGB8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGBA16/Destination: GL_RG32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGBA16/Destination: GL_RGBA16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGBA16I/Destination: GL_RG32F,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGBA16I/Destination: GL_RG32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGBA16I/Destination: GL_RG32UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGBA16I/Destination: GL_RGBA16,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGBA16I/Destination: GL_RGBA16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGBA16I/Destination: GL_RGBA16UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGBA16I/Destination: GL_RGBA16_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGBA16UI/Destination: GL_RG32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGBA16UI/Destination: GL_RGBA16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGBA16_SNORM/Destination: GL_RG32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGBA16_SNORM/Destination: GL_RGBA16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGBA32F/Destination: GL_RGBA32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGBA32I/Destination: GL_RGBA32F,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGBA32I/Destination: GL_RGBA32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGBA32I/Destination: GL_RGBA32UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGBA32UI/Destination: GL_RGBA32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGBA8/Destination: GL_RG16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGBA8/Destination: GL_RGBA8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGBA8I/Destination: GL_R32F,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGBA8I/Destination: GL_RG16,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGBA8I/Destination: GL_RG16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGBA8I/Destination: GL_RG16UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGBA8I/Destination: GL_RG16_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGBA8I/Destination: GL_RGBA8,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGBA8I/Destination: GL_RGBA8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGBA8I/Destination: GL_RGBA8UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGBA8I/Destination: GL_RGBA8_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGBA8UI/Destination: GL_RG16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGBA8UI/Destination: GL_RGBA8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGBA8_SNORM/Destination: GL_RG16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=2@Source: GL_RGBA8_SNORM/Destination: GL_RGBA8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_R16/Destination: GL_R16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_R16/Destination: GL_RG8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_R16I/Destination: GL_R16,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_R16I/Destination: GL_R16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_R16I/Destination: GL_R16UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_R16I/Destination: GL_R16_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_R16I/Destination: GL_RG8,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_R16I/Destination: GL_RG8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_R16I/Destination: GL_RG8UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_R16I/Destination: GL_RG8_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_R16UI/Destination: GL_R16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_R16UI/Destination: GL_RG8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_R16_SNORM/Destination: GL_R16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_R16_SNORM/Destination: GL_RG8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_R32F/Destination: GL_RG16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_R32F/Destination: GL_RGBA8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_R8/Destination: GL_R8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_R8I/Destination: GL_R8,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_R8I/Destination: GL_R8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_R8I/Destination: GL_R8UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_R8I/Destination: GL_R8_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_R8UI/Destination: GL_R8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_R8_SNORM/Destination: GL_R8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG16/Destination: GL_RG16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG16/Destination: GL_RGBA8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG16I/Destination: GL_R32F,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG16I/Destination: GL_RG16,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG16I/Destination: GL_RG16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG16I/Destination: GL_RG16UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG16I/Destination: GL_RG16_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG16I/Destination: GL_RGBA8,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG16I/Destination: GL_RGBA8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG16I/Destination: GL_RGBA8UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG16I/Destination: GL_RGBA8_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG16UI/Destination: GL_RG16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG16UI/Destination: GL_RGBA8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG16_SNORM/Destination: GL_RG16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG16_SNORM/Destination: GL_RGBA8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG32F/Destination: GL_RG32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG32F/Destination: GL_RGBA16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG32I/Destination: GL_RG32F,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG32I/Destination: GL_RG32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG32I/Destination: GL_RG32UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG32I/Destination: GL_RGBA16,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG32I/Destination: GL_RGBA16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG32I/Destination: GL_RGBA16UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG32I/Destination: GL_RGBA16_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG32UI/Destination: GL_RG32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG32UI/Destination: GL_RGBA16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG8/Destination: GL_R16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG8/Destination: GL_RG8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG8I/Destination: GL_R16,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG8I/Destination: GL_R16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG8I/Destination: GL_R16UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG8I/Destination: GL_R16_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG8I/Destination: GL_RG8,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG8I/Destination: GL_RG8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG8I/Destination: GL_RG8UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG8I/Destination: GL_RG8_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG8UI/Destination: GL_R16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG8UI/Destination: GL_RG8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG8_SNORM/Destination: GL_R16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RG8_SNORM/Destination: GL_RG8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGB16/Destination: GL_RGB16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGB16I/Destination: GL_RGB16,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGB16I/Destination: GL_RGB16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGB16I/Destination: GL_RGB16UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGB16I/Destination: GL_RGB16_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGB16UI/Destination: GL_RGB16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGB16_SNORM/Destination: GL_RGB16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGB32F/Destination: GL_RGB32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGB32I/Destination: GL_RGB32F,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGB32I/Destination: GL_RGB32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGB32I/Destination: GL_RGB32UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGB32UI/Destination: GL_RGB32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGB8/Destination: GL_RGB8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGB8I/Destination: GL_RGB8,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGB8I/Destination: GL_RGB8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGB8I/Destination: GL_RGB8UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGB8I/Destination: GL_RGB8_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGB8UI/Destination: GL_RGB8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGB8_SNORM/Destination: GL_RGB8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGBA16/Destination: GL_RG32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGBA16/Destination: GL_RGBA16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGBA16I/Destination: GL_RG32F,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGBA16I/Destination: GL_RG32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGBA16I/Destination: GL_RG32UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGBA16I/Destination: GL_RGBA16,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGBA16I/Destination: GL_RGBA16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGBA16I/Destination: GL_RGBA16UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGBA16I/Destination: GL_RGBA16_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGBA16UI/Destination: GL_RG32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGBA16UI/Destination: GL_RGBA16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGBA16_SNORM/Destination: GL_RG32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGBA16_SNORM/Destination: GL_RGBA16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGBA32F/Destination: GL_RGBA32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGBA32I/Destination: GL_RGBA32F,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGBA32I/Destination: GL_RGBA32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGBA32I/Destination: GL_RGBA32UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGBA32UI/Destination: GL_RGBA32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGBA8/Destination: GL_RG16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGBA8/Destination: GL_RGBA8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGBA8I/Destination: GL_R32F,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGBA8I/Destination: GL_RG16,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGBA8I/Destination: GL_RG16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGBA8I/Destination: GL_RG16UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGBA8I/Destination: GL_RG16_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGBA8I/Destination: GL_RGBA8,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGBA8I/Destination: GL_RGBA8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGBA8I/Destination: GL_RGBA8UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGBA8I/Destination: GL_RGBA8_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGBA8UI/Destination: GL_RG16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGBA8UI/Destination: GL_RGBA8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGBA8_SNORM/Destination: GL_RG16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=4@Source: GL_RGBA8_SNORM/Destination: GL_RGBA8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_R16/Destination: GL_R16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_R16/Destination: GL_RG8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_R16I/Destination: GL_R16,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_R16I/Destination: GL_R16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_R16I/Destination: GL_R16UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_R16I/Destination: GL_R16_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_R16I/Destination: GL_RG8,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_R16I/Destination: GL_RG8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_R16I/Destination: GL_RG8UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_R16I/Destination: GL_RG8_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_R16UI/Destination: GL_R16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_R16UI/Destination: GL_RG8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_R16_SNORM/Destination: GL_R16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_R16_SNORM/Destination: GL_RG8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_R32F/Destination: GL_RG16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_R32F/Destination: GL_RGBA8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_R8/Destination: GL_R8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_R8I/Destination: GL_R8,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_R8I/Destination: GL_R8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_R8I/Destination: GL_R8UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_R8I/Destination: GL_R8_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_R8UI/Destination: GL_R8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_R8_SNORM/Destination: GL_R8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG16/Destination: GL_RG16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG16/Destination: GL_RGBA8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG16I/Destination: GL_R32F,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG16I/Destination: GL_RG16,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG16I/Destination: GL_RG16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG16I/Destination: GL_RG16UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG16I/Destination: GL_RG16_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG16I/Destination: GL_RGBA8,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG16I/Destination: GL_RGBA8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG16I/Destination: GL_RGBA8UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG16I/Destination: GL_RGBA8_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG16UI/Destination: GL_RG16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG16UI/Destination: GL_RGBA8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG16_SNORM/Destination: GL_RG16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG16_SNORM/Destination: GL_RGBA8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG32F/Destination: GL_RG32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG32F/Destination: GL_RGBA16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG32I/Destination: GL_RG32F,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG32I/Destination: GL_RG32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG32I/Destination: GL_RG32UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG32I/Destination: GL_RGBA16,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG32I/Destination: GL_RGBA16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG32I/Destination: GL_RGBA16UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG32I/Destination: GL_RGBA16_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG32UI/Destination: GL_RG32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG32UI/Destination: GL_RGBA16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG8/Destination: GL_R16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG8/Destination: GL_RG8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG8I/Destination: GL_R16,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG8I/Destination: GL_R16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG8I/Destination: GL_R16UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG8I/Destination: GL_R16_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG8I/Destination: GL_RG8,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG8I/Destination: GL_RG8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG8I/Destination: GL_RG8UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG8I/Destination: GL_RG8_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG8UI/Destination: GL_R16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG8UI/Destination: GL_RG8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG8_SNORM/Destination: GL_R16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RG8_SNORM/Destination: GL_RG8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGB16/Destination: GL_RGB16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGB16I/Destination: GL_RGB16,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGB16I/Destination: GL_RGB16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGB16I/Destination: GL_RGB16UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGB16I/Destination: GL_RGB16_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGB16UI/Destination: GL_RGB16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGB16_SNORM/Destination: GL_RGB16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGB32F/Destination: GL_RGB32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGB32I/Destination: GL_RGB32F,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGB32I/Destination: GL_RGB32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGB32I/Destination: GL_RGB32UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGB32UI/Destination: GL_RGB32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGB8/Destination: GL_RGB8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGB8I/Destination: GL_RGB8,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGB8I/Destination: GL_RGB8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGB8I/Destination: GL_RGB8UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGB8I/Destination: GL_RGB8_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGB8UI/Destination: GL_RGB8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGB8_SNORM/Destination: GL_RGB8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGBA16/Destination: GL_RG32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGBA16/Destination: GL_RGBA16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGBA16I/Destination: GL_RG32F,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGBA16I/Destination: GL_RG32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGBA16I/Destination: GL_RG32UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGBA16I/Destination: GL_RGBA16,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGBA16I/Destination: GL_RGBA16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGBA16I/Destination: GL_RGBA16UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGBA16I/Destination: GL_RGBA16_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGBA16UI/Destination: GL_RG32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGBA16UI/Destination: GL_RGBA16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGBA16_SNORM/Destination: GL_RG32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGBA16_SNORM/Destination: GL_RGBA16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGBA32F/Destination: GL_RGBA32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGBA32I/Destination: GL_RGBA32F,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGBA32I/Destination: GL_RGBA32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGBA32I/Destination: GL_RGBA32UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGBA32UI/Destination: GL_RGBA32I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGBA8/Destination: GL_RG16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGBA8/Destination: GL_RGBA8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGBA8I/Destination: GL_R32F,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGBA8I/Destination: GL_RG16,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGBA8I/Destination: GL_RG16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGBA8I/Destination: GL_RG16UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGBA8I/Destination: GL_RG16_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGBA8I/Destination: GL_RGBA8,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGBA8I/Destination: GL_RGBA8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGBA8I/Destination: GL_RGBA8UI,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGBA8I/Destination: GL_RGBA8_SNORM,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGBA8UI/Destination: GL_RG16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGBA8UI/Destination: GL_RGBA8I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGBA8_SNORM/Destination: GL_RG16I,Fail
+spec@nv_copy_image@nv_copy_image-formats --samples=8@Source: GL_RGBA8_SNORM/Destination: GL_RGBA8I,Fail
+
+# https://gitlab.freedesktop.org/mesa/mesa/-/issues/8348
+KHR-GL46.tessellation_shader.tessellation_control_to_tessellation_evaluation.gl_tessLevel,Fail
+KHR-GL46.tessellation_shader.tessellation_shader_tessellation.TES,Fail
+spec@arb_tessellation_shader@execution@vs-tes-tessinner-tessouter-inputs-quads,Fail
+spec@arb_tessellation_shader@execution@vs-tes-tessinner-tessouter-inputs-quads-dlist,Fail
+spec@arb_tessellation_shader@execution@vs-tes-tessinner-tessouter-inputs-tris,Fail

src/gallium/drivers/crocus/ci/crocus-hsw-flakes.txt

@@ -0,0 +1,13 @@
+KHR-GL46.texture_barrier.overlapping-texels
+KHR-GL46.texture_barrier_ARB.overlapping-texels
+KHR-GL46.texture_barrier_ARB.same-texel-rw-multipass
+KHR-GL46.texture_barrier.same-texel-rw-multipass
+KHR-GL46.tessellation_shader.tessellation_shader_tc_barriers.barrier_guarded_read_write_calls
+KHR-GLES31.core.tessellation_shader.tessellation_shader_tc_barriers.barrier_guarded_read_write_calls
+spec@arb_query_buffer_object@qbo
+spec@arb_shader_storage_buffer_object@execution@ssbo-atomicexchange-int
+spec@amd_performance_monitor@measure
+spec@amd_performance_monitor@measure@change counters while active
+spec@amd_performance_monitor@measure@counters in range
+
+spec@!opengl [email protected] 100000 varray .*

src/gallium/drivers/crocus/ci/crocus-hsw-skips.txt

@@ -0,0 +1,10 @@
+# We aren't running an X server.
+glx.*
+
+# Very slow tests, potentially timing out
+KHR-GL46.copy_image.functional
+KHR-GL46.texture_swizzle.functional
+KHR-GL46.texture_swizzle.smoke
+KHR-GL46.direct_state_access.framebuffers_texture_layer_attachment
+shaders@glsl-max-varyings
+spec@amd_performance_monitor@measure

src/gallium/drivers/crocus/ci/deqp-crocus-g41.toml

@@ -0,0 +1,30 @@
+[[deqp]]
+deqp = "/deqp-gles/modules/gles2/deqp-gles2"
+caselists = ["/deqp-gles/mustpass/gles2-main.txt"]
+deqp_args = [
+    "--deqp-surface-width=256",
+    "--deqp-surface-height=256",
+    "--deqp-surface-type=pbuffer",
+    "--deqp-gl-config-name=rgba8888d24s8ms0",
+    "--deqp-visibility=hidden"
+]
+version_check = "GL ES 2.0.*git"
+renderer_check = "Intel.*G41"
+
+[[deqp]]
+deqp = "/deqp-gles/external/openglcts/modules/glcts"
+caselists = ["/deqp-gles/mustpass/gles2-khr-main.txt"]
+deqp_args = [
+    "--deqp-surface-width=256",
+    "--deqp-surface-height=256",
+    "--deqp-surface-type=pbuffer",
+    "--deqp-gl-config-name=rgba8888d24s8ms0",
+    "--deqp-visibility=hidden"
+]
+
+[[piglit]]
+piglit_folder = "/piglit"
+# We use "gpu" to skip running glslparser tests, with the assumption that
+# they're covered by the llvmpipe run.
+profile = "gpu"
+process_isolation = true

src/gallium/drivers/crocus/ci/deqp-crocus-hsw.toml

@@ -0,0 +1,69 @@
+[[deqp]]
+deqp = "/deqp-gles/modules/gles2/deqp-gles2"
+caselists = ["/deqp-gles/mustpass/gles2-main.txt"]
+deqp_args = [
+    "--deqp-surface-width=256",
+    "--deqp-surface-height=256",
+    "--deqp-surface-type=pbuffer",
+    "--deqp-gl-config-name=rgba8888d24s8ms0",
+    "--deqp-visibility=hidden"
+]
+version_check = "GL ES 3.2.*git"
+renderer_check = "Intel.*HSW GT2"
+
+[[deqp]]
+deqp = "/deqp-gles/modules/gles3/deqp-gles3"
+caselists = ["/deqp-gles/mustpass/gles3-main.txt"]
+deqp_args = [
+    "--deqp-surface-width=256",
+    "--deqp-surface-height=256",
+    "--deqp-surface-type=pbuffer",
+    "--deqp-gl-config-name=rgba8888d24s8ms0",
+    "--deqp-visibility=hidden"
+]
+version_check = "GL ES 3.2.*git"
+renderer_check = "Intel.*HSW GT2"
+
+[[deqp]]
+deqp = "/deqp-gles/modules/gles31/deqp-gles31"
+caselists = ["/deqp-gles/mustpass/gles31-main.txt"]
+deqp_args = [
+    "--deqp-surface-width=256",
+    "--deqp-surface-height=256",
+    "--deqp-surface-type=pbuffer",
+    "--deqp-gl-config-name=rgba8888d24s8ms0",
+    "--deqp-visibility=hidden"
+]
+version_check = "GL ES 3.2.*git"
+renderer_check = "Intel.*HSW GT2"
+
+[[deqp]]
+deqp = "/deqp-gles/external/openglcts/modules/glcts"
+caselists = [
+        "/deqp-gles/mustpass/gles2-khr-main.txt",
+        "/deqp-gles/mustpass/gles3-khr-main.txt",
+        "/deqp-gles/mustpass/gles31-khr-main.txt"
+]
+deqp_args = [
+    "--deqp-surface-width=256",
+    "--deqp-surface-height=256",
+    "--deqp-surface-type=pbuffer",
+    "--deqp-gl-config-name=rgba8888d24s8ms0",
+    "--deqp-visibility=hidden"
+]
+
+[[deqp]]
+deqp = "/deqp-gl/external/openglcts/modules/glcts"
+caselists = ["/deqp-gl/mustpass/gl46-main.txt"]
+deqp_args = [
+    "--deqp-surface-width=256", "--deqp-surface-height=256",
+    "--deqp-surface-type=pbuffer", "--deqp-visibility=hidden",
+    "--deqp-gl-config-name=rgba8888d24s8ms0",
+]
+
+[[piglit]]
+piglit_folder = "/piglit"
+# We use "gpu" to skip running glslparser tests, with the assumption that
+# they're covered by the llvmpipe run.
+profile = "gpu"
+process_isolation = true

src/gallium/drivers/crocus/ci/gitlab-ci-inc.yml

@@ -0,0 +1,37 @@
+# Manual test rules for using anholt's shared g41.
+.anholt-g41-test:
+  extends:
+    - .crocus-manual-rules
+  tags:
+    - anholt-g41
+  variables:
+    GPU_VERSION: crocus-g41
+    FLAKES_CHANNEL: "#intel-ci"
+
+# Manual test rules for using anholt's shared hsw.
+.anholt-hsw-test:
+  tags:
+    - anholt-hsw
+  variables:
+    FDO_CI_CONCURRENT: 12
+
+# System is dead at the moment. needs maintenance.
+.crocus-g41:
+  extends:
+    - .anholt-g41-test
+    - .test-gl
+    - .test-piglit
+    - .deqp-test
+  variables:
+    DEQP_SUITE: crocus-g41
+    PIGLIT_PLATFORM: gbm
+
+.crocus-g41-traces:
+  extends:
+    - .anholt-g41-test
+    - .test-gl
+    - .piglit-traces-test
+  variables:
+    EGL_PLATFORM: "surfaceless"
+    PIGLIT_TRACES_FILE: traces-crocus.yml
+    PIGLIT_REPLAY_DEVICE_NAME: "crocus-g41"

src/gallium/drivers/crocus/ci/gitlab-ci.yml

@@ -0,0 +1,25 @@
+include:
+  - local: 'src/gallium/drivers/crocus/ci/gitlab-ci-inc.yml'
+
+.crocus-hsw:
+  extends:
+    - .anholt-hsw-test
+    - .test-gl
+    - .deqp-test
+    - .test-piglit
+    - .crocus-manual-rules
+  variables:
+    GPU_VERSION: crocus-hsw
+    DEQP_SUITE: crocus-hsw
+    PIGLIT_PLATFORM: gbm
+
+.crocus-hsw-traces:
+  extends:
+    - .anholt-hsw-test
+    - .test-gl
+    - .piglit-traces-test
+    - .crocus-manual-rules
+  variables:
+    EGL_PLATFORM: "surfaceless"
+    PIGLIT_TRACES_FILE: traces-crocus.yml
+    PIGLIT_REPLAY_DEVICE_NAME: "crocus-hsw"

src/gallium/drivers/crocus/ci/traces-crocus.yml

@@ -0,0 +1,176 @@
+%YAML 1.2
+---
+traces-db:
+  download-url: "https://s3.freedesktop.org/mesa-tracie-public/"
+
+traces:
+  0ad/0ad-v2.trace:
+    crocus-g41:
+      checksum: f4f723a73fde41b8a00a2a6518190adf
+    crocus-hsw:
+      checksum: 3ba17078e281e3d73a0e2d5d72ebf869
+  behdad-glyphy/glyphy-v2.trace:
+    crocus-g41:
+      label: [skip, flakes, broken]
+    crocus-hsw:
+      checksum: 49c62d99a21de18a3e184a57bf655849
+  glxgears/glxgears-2-v2.trace:
+    crocus-g41:
+      checksum: 18455018dcfa97f69c967f118551ae30
+    crocus-hsw:
+      checksum: f53ac20e17da91c0359c31f2fa3f401e
+  gputest/furmark-v2.trace:
+    crocus-g41:
+      checksum: c7c3df989f88852e88bef101b5c1dc22
+    crocus-hsw:
+      checksum: bb84373a69751942956d2320cff6dd1d
+# gimark requires an environment var to work around a bug, disable for now
+#  gputest/gimark-v2.trace:
+#    crocus-g41:
+#      label: [unsupported]
+#    crocus-hsw:
+#      label: [skip, flakes]
+#      text: lighting comes in from a different direction than on freedreno?
+  gputest/pixmark-julia-fp32-v2.trace:
+    crocus-g41:
+      checksum: 03776a063ee7049256acd414239a7f13
+    crocus-hsw:
+      checksum: dea71222007af1669cff54ce5e0b8c8d
+  gputest/pixmark-julia-fp64-v2.trace:
+    crocus-g41:
+      label: [unsupported]
+    crocus-hsw:
+      checksum: 7e4948c361528a7873dad9712f629770
+  gputest/pixmark-volplosion-v2.trace:
+    crocus-g41:
+      checksum: 6fc94a69b9e1f38693ea3ed4f530c7da
+    crocus-hsw:
+      checksum: b96cddb97eb0cde8a388dad76d8dfef6
+  humus/AmbientAperture-v2.trace:
+    crocus-g41:
+      label: [unsupported]
+    crocus-hsw:
+      checksum: 5686175a74f9ef6a3b13d30f1bd32b16
+  humus/CelShading-v2.trace:
+    crocus-g41:
+      checksum: 9b6e185ac8e8e67fc13d84f0ad6c3e3b
+    crocus-hsw:
+      checksum: 0f03835fd89b615542ece4bbb4712e48
+  humus/DynamicBranching3-v2.trace:
+    crocus-g41:
+      label: [unsupported]
+    crocus-hsw:
+      checksum: 16e3458485d1ba75fe80f61f59d124be
+  humus/HDR-v2.trace:
+    crocus-g41:
+      label: [unsupported]
+    crocus-hsw:
+      checksum: 0dfbc65adca38721831ffddf0ede2af6
+  humus/Portals-v2.trace:
+    crocus-g41:
+      checksum: b090e734ed332e259664851576925d7b
+    crocus-hsw:
+      checksum: 61ea77324581e86cf8d313fe3fb27270
+  humus/RaytracedShadows-v2.trace:
+    crocus-g41:
+      checksum: dbef340424f4f85eb8dc053ad3a56ae1
+    crocus-hsw:
+      checksum: bac350df35b9262efca6722faf5c4eaa
+  humus/VolumetricFogging2-v2.trace:
+    crocus-g41:
+      label: [unsupported]
+    crocus-hsw:
+      checksum: a4f444b6ca337901ae080d8f2af812b4
+  neverball/neverball-v2.trace:
+    crocus-g41:
+      checksum: 0f14a88f511cd886c920238c80017e5d
+    crocus-hsw:
+      checksum: d77b3697a9b1a47611cde275aa6cb912
+      text: "https://gitlab.freedesktop.org/mesa/mesa/-/issues/9159"
+  paraview/pv-manyspheres-v2.trace:
+    crocus-hsw:
+      checksum: 73e5d34fb13268b0c7126297bfc34965
+  paraview/pv-waveletcontour-v2.trace:
+    crocus-hsw:
+      checksum: b384e3e81e4843fbd1d50e0c542e600a
+  paraview/pv-waveletvolume-v2.trace:
+    crocus-hsw:
+      checksum: 6ad068ac9f390bf42ab9e2db880ad07b
+  pathfinder/canvas_moire-v2.trace:
+    crocus-hsw:
+      checksum: 21bccd42f2deb6416cf6591cd6a99258
+  pathfinder/canvas_text_v2-v2.trace:
+    crocus-hsw:
+      checksum: 448886e3b24f6408e013ea13f7c96d28
+  pathfinder/demo-v2.trace:
+    crocus-hsw:
+      checksum: d9b33f0a2efe17c21b7933242afd9ec7
+  supertuxkart/supertuxkart-mansion-egl-gles-v2.trace:
+    crocus-hsw:
+      checksum: c59dc93744d2dfcf172aa3f26ce2a62e
+  valve/counterstrike-v2.trace:
+    crocus-g41:
+      checksum: 90c01e73b3544cb477ac6109bc61bef1
+    crocus-hsw:
+      checksum: 0e1e52aaccd94a261cf9d8675039c41c
+  valve/counterstrike-source-v2.trace:
+    crocus-g41:
+      checksum: c51e21aada8e79fada36048e7d8de74a
+    crocus-hsw:
+      checksum: a78be6d590dd6c48bc6ad6090d88d9b3
+  valve/half-life-2-v2.trace:
+    crocus-g41:
+      checksum: c67b47eeefb2b7a3b6667844758d4a95
+    crocus-hsw:
+      checksum: 96dcdfa2fc7e64b8bd2da0ea7b854f6c
+  valve/portal-2-v2.trace:
+    crocus-g41:
+      checksum: fff536c4fb7753d5cf1896c4ad4c4452
+    crocus-hsw:
+      checksum: 3c9cea7f98d8e5ab93d69c4834b570c9
+  xonotic/xonotic-keybench-high-v2.trace:
+    crocus-g41:
+      label: [unsupported]
+    crocus-hsw:
+      label: [skip, flakes]
+  ror/ror-default.trace:
+    crocus-g41:
+      label: [unsupported]
+    crocus-hsw:
+      checksum: 197eba64053d9ef096a40950163adcce
+  nheko/nheko-colors.trace:
+    crocus-g41:
+      checksum: 3f4bfa1516e50194e4e86978420852c4
+    crocus-hsw:
+      checksum: 7a8b329f50653480eda50f53d715ad27
+  blender/blender-demo-cube_diorama.trace:
+    crocus-g41:
+      label: [unsupported]
+    crocus-hsw:
+      label: [skip, flakes]
+      checksum: f8cc19569c0c5a43cc4f16414167d234
+      text: Inconsistent rendering.
+  blender/blender-demo-ellie_pose.trace:
+    crocus-g41:
+      label: [unsupported]
+    crocus-hsw:
+      checksum: c07530897dc5a6882cbf35737334336b
+  unvanquished/unvanquished-lowest.trace:
+    crocus-g41:
+      label: [unsupported]
+    crocus-hsw:
+      label: [skip, flakes]
+      checksum: 884096b6dd40e2f500d0837501287fac
+      text: "https://gitlab.freedesktop.org/mesa/mesa/-/issues/7158"
+  unvanquished/unvanquished-ultra.trace:
+    crocus-g41:
+      label: [unsupported]
+    crocus-hsw:
+      label: [skip, flakes]
+      checksum: 99b3ec85e4b9335c3cffc49a63c0c62c
+      text: "Text glyphs render incorrectly sometimes."
+  warzone2100/warzone2100-default.trace:
+    crocus-g41:
+      label: [unsupported]
+    crocus-hsw:
+      checksum: 2548cf2f42ac7bbdf5d1b26fc2f1c5f5

src/gallium/drivers/crocus/crocus_batch.h

@@ -0,0 +1,325 @@
+/*
+ * Copyright © 2017 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifndef CROCUS_BATCH_DOT_H
+#define CROCUS_BATCH_DOT_H
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <string.h>
+
+#include "util/u_dynarray.h"
+
+#include "decoder/intel_decoder.h"
+#include "drm-uapi/i915_drm.h"
+
+#include "crocus_fence.h"
+#include "crocus_fine_fence.h"
+
+#include "crocus_bufmgr.h"
+/* The kernel assumes batchbuffers are smaller than 256kB. */
+#define MAX_BATCH_SIZE (256 * 1024)
+
+/* 3DSTATE_BINDING_TABLE_POINTERS has a U16 offset from Surface State Base
+ * Address, which means that we can't put binding tables beyond 64kB.  This
+ * effectively limits the maximum statebuffer size to 64kB.
+ */
+#define MAX_STATE_SIZE (64 * 1024)
+
+/* Our target batch size - flush approximately at this point. */
+#define BATCH_SZ (20 * 1024)
+#define STATE_SZ (16 * 1024)
+
+enum crocus_batch_name {
+   CROCUS_BATCH_RENDER,
+   CROCUS_BATCH_COMPUTE,
+};
+
+#define CROCUS_BATCH_COUNT 2
+
+struct crocus_address {
+   struct crocus_bo *bo;
+   int32_t offset;
+   uint32_t reloc_flags;
+};
+
+struct crocus_reloc_list {
+   struct drm_i915_gem_relocation_entry *relocs;
+   int reloc_count;
+   int reloc_array_size;
+};
+
+struct crocus_growing_bo {
+   struct crocus_bo *bo;
+   void *map;
+   void *map_next;
+   struct crocus_bo *partial_bo;
+   void *partial_bo_map;
+   unsigned partial_bytes;
+   struct crocus_reloc_list relocs;
+   unsigned used;
+};
+
+struct crocus_batch {
+   struct crocus_context *ice;
+   struct crocus_screen *screen;
+   struct util_debug_callback *dbg;
+   struct pipe_device_reset_callback *reset;
+
+   /** What batch is this? (e.g. CROCUS_BATCH_RENDER/COMPUTE) */
+   enum crocus_batch_name name;
+
+   /** buffers: command, state */
+   struct crocus_growing_bo command, state;
+
+   /** Size of the primary batch if we've moved on to a secondary. */
+   unsigned primary_batch_size;
+
+   bool state_base_address_emitted;
+   uint8_t pipe_controls_since_last_cs_stall;
+
+   uint32_t hw_ctx_id;
+
+   uint32_t valid_reloc_flags;
+
+   bool use_shadow_copy;
+   bool no_wrap;
+
+   /** The validation list */
+   struct drm_i915_gem_exec_object2 *validation_list;
+   struct crocus_bo **exec_bos;
+   int exec_count;
+   int exec_array_size;
+
+   /** Whether INTEL_BLACKHOLE_RENDER is enabled in the batch (aka first
+    * instruction is a MI_BATCH_BUFFER_END).
+    */
+   bool noop_enabled;
+
+   /**
+    * A list of crocus_syncobjs associated with this batch.
+    *
+    * The first list entry will always be a signalling sync-point, indicating
+    * that this batch has completed.  The others are likely to be sync-points
+    * to wait on before executing the batch.
+    */
+   struct util_dynarray syncobjs;
+
+   /** A list of drm_i915_exec_fences to have execbuf signal or wait on */
+   struct util_dynarray exec_fences;
+
+   /** The amount of aperture space (in bytes) used by all exec_bos */
+   int aperture_space;
+
+   struct {
+      /** Uploader to use for sequence numbers */
+      struct u_upload_mgr *uploader;
+
+      /** GPU buffer and CPU map where our seqno's will be written. */
+      struct crocus_state_ref ref;
+      uint32_t *map;
+
+      /** The sequence number to write the next time we add a fence. */
+      uint32_t next;
+   } fine_fences;
+
+   /** A seqno (and syncobj) for the last batch that was submitted. */
+   struct crocus_fine_fence *last_fence;
+
+   /** List of other batches which we might need to flush to use a BO */
+   struct crocus_batch *other_batches[CROCUS_BATCH_COUNT - 1];
+
+   struct {
+      /**
+       * Set of struct brw_bo * that have been rendered to within this
+       * batchbuffer and would need flushing before being used from another
+       * cache domain that isn't coherent with it (i.e. the sampler).
+       */
+      struct hash_table *render;
+
+      /**
+       * Set of struct brw_bo * that have been used as a depth buffer within
+       * this batchbuffer and would need flushing before being used from
+       * another cache domain that isn't coherent with it (i.e. the sampler).
+       */
+      struct set *depth;
+   } cache;
+
+   struct intel_batch_decode_ctx decoder;
+   struct hash_table_u64 *state_sizes;
+
+   /** Have we emitted any draw calls to this batch? */
+   bool contains_draw;
+
+   /** Batch contains fence signal operation. */
+   bool contains_fence_signal;
+};
+
+static inline bool
+batch_has_fine_fence(struct crocus_batch *batch)
+{
+   return !!batch->fine_fences.uploader;
+}
+
+#define BATCH_HAS_FINE_FENCES(batch) (!!(batch)->fine_fences.uploader)
+void crocus_init_batch(struct crocus_context *ctx,
+                       enum crocus_batch_name name,
+                       int priority);
+void crocus_batch_free(struct crocus_batch *batch);
+void crocus_batch_maybe_flush(struct crocus_batch *batch, unsigned estimate);
+
+void _crocus_batch_flush(struct crocus_batch *batch, const char *file, int line);
+#define crocus_batch_flush(batch) _crocus_batch_flush((batch), __FILE__, __LINE__)
+
+bool crocus_batch_references(struct crocus_batch *batch, struct crocus_bo *bo);
+
+bool crocus_batch_prepare_noop(struct crocus_batch *batch, bool noop_enable);
+
+#define RELOC_WRITE EXEC_OBJECT_WRITE
+#define RELOC_NEEDS_GGTT EXEC_OBJECT_NEEDS_GTT
+/* Inverted meaning, but using the same bit...emit_reloc will flip it. */
+#define RELOC_32BIT EXEC_OBJECT_SUPPORTS_48B_ADDRESS
+
+void crocus_use_pinned_bo(struct crocus_batch *batch, struct crocus_bo *bo,
+                          bool writable);
+uint64_t crocus_command_reloc(struct crocus_batch *batch, uint32_t batch_offset,
+                              struct crocus_bo *target, uint32_t target_offset,
+                              unsigned int reloc_flags);
+uint64_t crocus_state_reloc(struct crocus_batch *batch, uint32_t batch_offset,
+                            struct crocus_bo *target, uint32_t target_offset,
+                            unsigned int reloc_flags);
+
+enum pipe_reset_status crocus_batch_check_for_reset(struct crocus_batch *batch);
+
+void crocus_grow_buffer(struct crocus_batch *batch, bool grow_state,
+                        unsigned used, unsigned new_size);
+
+static inline unsigned
+crocus_batch_bytes_used(struct crocus_batch *batch)
+{
+   return batch->command.map_next - batch->command.map;
+}
+
+/**
+ * Ensure the current command buffer has \param size bytes of space
+ * remaining.  If not, this creates a secondary batch buffer and emits
+ * a jump from the primary batch to the start of the secondary.
+ *
+ * Most callers want crocus_get_command_space() instead.
+ */
+static inline void
+crocus_require_command_space(struct crocus_batch *batch, unsigned size)
+{
+   const unsigned required_bytes = crocus_batch_bytes_used(batch) + size;
+   unsigned used = crocus_batch_bytes_used(batch);
+   if (required_bytes >= BATCH_SZ && !batch->no_wrap) {
+      crocus_batch_flush(batch);
+   } else if (used + size >= batch->command.bo->size) {
+      const unsigned new_size =
+         MIN2(batch->command.bo->size + batch->command.bo->size / 2,
+              MAX_BATCH_SIZE);
+
+      crocus_grow_buffer(batch, false, used, new_size);
+      batch->command.map_next = (void *)batch->command.map + used;
+      assert(crocus_batch_bytes_used(batch) + size < batch->command.bo->size);
+   }
+}
+
+/**
+ * Allocate space in the current command buffer, and return a pointer
+ * to the mapped area so the caller can write commands there.
+ *
+ * This should be called whenever emitting commands.
+ */
+static inline void *
+crocus_get_command_space(struct crocus_batch *batch, unsigned bytes)
+{
+   crocus_require_command_space(batch, bytes);
+   void *map = batch->command.map_next;
+   batch->command.map_next += bytes;
+   return map;
+}
+
+/**
+ * Helper to emit GPU commands - allocates space, copies them there.
+ */
+static inline void
+crocus_batch_emit(struct crocus_batch *batch, const void *data, unsigned size)
+{
+   void *map = crocus_get_command_space(batch, size);
+   memcpy(map, data, size);
+}
+
+/**
+ * Get a pointer to the batch's signalling syncobj.  Does not refcount.
+ */
+static inline struct crocus_syncobj *
+crocus_batch_get_signal_syncobj(struct crocus_batch *batch)
+{
+   /* The signalling syncobj is the first one in the list. */
+   struct crocus_syncobj *syncobj =
+      ((struct crocus_syncobj **)util_dynarray_begin(&batch->syncobjs))[0];
+   return syncobj;
+}
+
+/**
+ * Take a reference to the batch's signalling syncobj.
+ *
+ * Callers can use this to wait for the the current batch under construction
+ * to complete (after flushing it).
+ */
+static inline void
+crocus_batch_reference_signal_syncobj(struct crocus_batch *batch,
+                                      struct crocus_syncobj **out_syncobj)
+{
+   struct crocus_syncobj *syncobj = crocus_batch_get_signal_syncobj(batch);
+   crocus_syncobj_reference(batch->screen, out_syncobj, syncobj);
+}
+
+/**
+ * Record the size of a piece of state for use in INTEL_DEBUG=bat printing.
+ */
+static inline void
+crocus_record_state_size(struct hash_table_u64 *ht, uint32_t offset_from_base,
+                         uint32_t size)
+{
+   if (ht) {
+      _mesa_hash_table_u64_insert(ht, offset_from_base,
+                                  (void *)(uintptr_t)size);
+   }
+}
+
+static inline bool
+crocus_ptr_in_state_buffer(struct crocus_batch *batch, void *p)
+{
+   return (char *)p >= (char *)batch->state.map &&
+          (char *)p < (char *)batch->state.map + batch->state.bo->size;
+}
+
+static inline void
+crocus_require_statebuffer_space(struct crocus_batch *batch, int size)
+{
+   if (batch->state.used + size >= STATE_SZ)
+      crocus_batch_flush(batch);
+}
+#endif

src/gallium/drivers/crocus/crocus_blit.c

@@ -0,0 +1,803 @@
+/*
+ * Copyright © 2017 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <stdio.h>
+#include "pipe/p_defines.h"
+#include "pipe/p_state.h"
+#include "pipe/p_context.h"
+#include "pipe/p_screen.h"
+#include "util/format/u_format.h"
+#include "util/u_inlines.h"
+#include "util/u_surface.h"
+#include "util/ralloc.h"
+#include "intel/blorp/blorp.h"
+#include "crocus_context.h"
+#include "crocus_resource.h"
+#include "crocus_screen.h"
+
+void crocus_blitter_begin(struct crocus_context *ice, enum crocus_blitter_op op, bool render_cond)
+{
+   util_blitter_save_vertex_shader(ice->blitter, ice->shaders.uncompiled[MESA_SHADER_VERTEX]);
+   util_blitter_save_tessctrl_shader(ice->blitter, ice->shaders.uncompiled[MESA_SHADER_TESS_CTRL]);
+   util_blitter_save_tesseval_shader(ice->blitter, ice->shaders.uncompiled[MESA_SHADER_TESS_EVAL]);
+   util_blitter_save_geometry_shader(ice->blitter, ice->shaders.uncompiled[MESA_SHADER_GEOMETRY]);
+   util_blitter_save_so_targets(ice->blitter, ice->state.so_targets,
+                                (struct pipe_stream_output_target**)ice->state.so_target, MESA_PRIM_UNKNOWN);
+   util_blitter_save_vertex_buffers(ice->blitter, ice->state.vertex_buffers,
+                                    util_last_bit(ice->state.bound_vertex_buffers));
+   util_blitter_save_vertex_elements(ice->blitter, (void *)ice->state.cso_vertex_elements);
+   if (op & CROCUS_SAVE_FRAGMENT_STATE) {
+      util_blitter_save_blend(ice->blitter, ice->state.cso_blend);
+      util_blitter_save_depth_stencil_alpha(ice->blitter, ice->state.cso_zsa);
+      util_blitter_save_stencil_ref(ice->blitter, &ice->state.stencil_ref);
+      util_blitter_save_fragment_shader(ice->blitter, ice->shaders.uncompiled[MESA_SHADER_FRAGMENT]);
+      util_blitter_save_sample_mask(ice->blitter, ice->state.sample_mask, 0);
+      util_blitter_save_rasterizer(ice->blitter, ice->state.cso_rast);
+      util_blitter_save_scissor(ice->blitter, &(struct pipe_scissor_state) {
+         ice->state.scissors[0].minx, ice->state.scissors[0].miny,
+         ice->state.scissors[0].maxx, ice->state.scissors[0].maxy });
+      util_blitter_save_viewport(ice->blitter, &ice->state.viewports[0]);
+      util_blitter_save_fragment_constant_buffer_slot(ice->blitter, &ice->state.shaders[MESA_SHADER_FRAGMENT].constbufs[0]);
+   }
+
+   if (!render_cond)
+      util_blitter_save_render_condition(ice->blitter,
+                                         (struct pipe_query *)ice->condition.query,
+                                         ice->condition.condition,
+                                         ice->condition.mode);
+
+//   util_blitter_save_scissor(ice->blitter, &ice->scissors[0]);
+   if (op & CROCUS_SAVE_FRAMEBUFFER)
+      util_blitter_save_framebuffer(ice->blitter, &ice->state.framebuffer);
+
+   if (op & CROCUS_SAVE_TEXTURES) {
+      util_blitter_save_fragment_sampler_states(ice->blitter, 1, (void **)ice->state.shaders[MESA_SHADER_FRAGMENT].samplers);
+      util_blitter_save_fragment_sampler_views(ice->blitter, 1, (struct pipe_sampler_view **)ice->state.shaders[MESA_SHADER_FRAGMENT].textures);
+   }
+}
+
+/**
+ * Helper function for handling mirror image blits.
+ *
+ * If coord0 > coord1, swap them and return "true" (mirrored).
+ */
+static bool
+apply_mirror(float *coord0, float *coord1)
+{
+   if (*coord0 > *coord1) {
+      float tmp = *coord0;
+      *coord0 = *coord1;
+      *coord1 = tmp;
+      return true;
+   }
+   return false;
+}
+
+/**
+ * Compute the number of pixels to clip for each side of a rect
+ *
+ * \param x0 The rect's left coordinate
+ * \param y0 The rect's bottom coordinate
+ * \param x1 The rect's right coordinate
+ * \param y1 The rect's top coordinate
+ * \param min_x The clipping region's left coordinate
+ * \param min_y The clipping region's bottom coordinate
+ * \param max_x The clipping region's right coordinate
+ * \param max_y The clipping region's top coordinate
+ * \param clipped_x0 The number of pixels to clip from the left side
+ * \param clipped_y0 The number of pixels to clip from the bottom side
+ * \param clipped_x1 The number of pixels to clip from the right side
+ * \param clipped_y1 The number of pixels to clip from the top side
+ *
+ * \return false if we clip everything away, true otherwise
+ */
+static inline bool
+compute_pixels_clipped(float x0, float y0, float x1, float y1,
+                       float min_x, float min_y, float max_x, float max_y,
+                       float *clipped_x0, float *clipped_y0,
+                       float *clipped_x1, float *clipped_y1)
+{
+   /* If we are going to clip everything away, stop. */
+   if (!(min_x <= max_x &&
+         min_y <= max_y &&
+         x0 <= max_x &&
+         y0 <= max_y &&
+         min_x <= x1 &&
+         min_y <= y1 &&
+         x0 <= x1 &&
+         y0 <= y1)) {
+      return false;
+   }
+
+   if (x0 < min_x)
+      *clipped_x0 = min_x - x0;
+   else
+      *clipped_x0 = 0;
+   if (max_x < x1)
+      *clipped_x1 = x1 - max_x;
+   else
+      *clipped_x1 = 0;
+
+   if (y0 < min_y)
+      *clipped_y0 = min_y - y0;
+   else
+      *clipped_y0 = 0;
+   if (max_y < y1)
+      *clipped_y1 = y1 - max_y;
+   else
+      *clipped_y1 = 0;
+
+   return true;
+}
+
+/**
+ * Clips a coordinate (left, right, top or bottom) for the src or dst rect
+ * (whichever requires the largest clip) and adjusts the coordinate
+ * for the other rect accordingly.
+ *
+ * \param mirror true if mirroring is required
+ * \param src the source rect coordinate (for example src_x0)
+ * \param dst0 the dst rect coordinate (for example dst_x0)
+ * \param dst1 the opposite dst rect coordinate (for example dst_x1)
+ * \param clipped_dst0 number of pixels to clip from the dst coordinate
+ * \param clipped_dst1 number of pixels to clip from the opposite dst coordinate
+ * \param scale the src vs dst scale involved for that coordinate
+ * \param is_left_or_bottom true if we are clipping the left or bottom sides
+ *        of the rect.
+ */
+static void
+clip_coordinates(bool mirror,
+                 float *src, float *dst0, float *dst1,
+                 float clipped_dst0,
+                 float clipped_dst1,
+                 float scale,
+                 bool is_left_or_bottom)
+{
+   /* When clipping we need to add or subtract pixels from the original
+    * coordinates depending on whether we are acting on the left/bottom
+    * or right/top sides of the rect respectively. We assume we have to
+    * add them in the code below, and multiply by -1 when we should
+    * subtract.
+    */
+   int mult = is_left_or_bottom ? 1 : -1;
+
+   if (!mirror) {
+      *dst0 += clipped_dst0 * mult;
+      *src += clipped_dst0 * scale * mult;
+   } else {
+      *dst1 -= clipped_dst1 * mult;
+      *src += clipped_dst1 * scale * mult;
+   }
+}
+
+/**
+ * Apply a scissor rectangle to blit coordinates.
+ *
+ * Returns true if the blit was entirely scissored away.
+ */
+static bool
+apply_blit_scissor(const struct pipe_scissor_state *scissor,
+                   float *src_x0, float *src_y0,
+                   float *src_x1, float *src_y1,
+                   float *dst_x0, float *dst_y0,
+                   float *dst_x1, float *dst_y1,
+                   bool mirror_x, bool mirror_y)
+{
+   float clip_dst_x0, clip_dst_x1, clip_dst_y0, clip_dst_y1;
+
+   /* Compute number of pixels to scissor away. */
+   if (!compute_pixels_clipped(*dst_x0, *dst_y0, *dst_x1, *dst_y1,
+                               scissor->minx, scissor->miny,
+                               scissor->maxx, scissor->maxy,
+                               &clip_dst_x0, &clip_dst_y0,
+                               &clip_dst_x1, &clip_dst_y1))
+      return true;
+
+   // XXX: comments assume source clipping, which we don't do
+
+   /* When clipping any of the two rects we need to adjust the coordinates
+    * in the other rect considering the scaling factor involved.  To obtain
+    * the best precision we want to make sure that we only clip once per
+    * side to avoid accumulating errors due to the scaling adjustment.
+    *
+    * For example, if src_x0 and dst_x0 need both to be clipped we want to
+    * avoid the situation where we clip src_x0 first, then adjust dst_x0
+    * accordingly but then we realize that the resulting dst_x0 still needs
+    * to be clipped, so we clip dst_x0 and adjust src_x0 again.  Because we are
+    * applying scaling factors to adjust the coordinates in each clipping
+    * pass we lose some precision and that can affect the results of the
+    * blorp blit operation slightly.  What we want to do here is detect the
+    * rect that we should clip first for each side so that when we adjust
+    * the other rect we ensure the resulting coordinate does not need to be
+    * clipped again.
+    *
+    * The code below implements this by comparing the number of pixels that
+    * we need to clip for each side of both rects considering the scales
+    * involved.  For example, clip_src_x0 represents the number of pixels
+    * to be clipped for the src rect's left side, so if clip_src_x0 = 5,
+    * clip_dst_x0 = 4 and scale_x = 2 it means that we are clipping more
+    * from the dst rect so we should clip dst_x0 only and adjust src_x0.
+    * This is because clipping 4 pixels in the dst is equivalent to
+    * clipping 4 * 2 = 8 > 5 in the src.
+    */
+
+   if (*src_x0 == *src_x1 || *src_y0 == *src_y1
+       || *dst_x0 == *dst_x1 || *dst_y0 == *dst_y1)
+      return true;
+
+   float scale_x = (float) (*src_x1 - *src_x0) / (*dst_x1 - *dst_x0);
+   float scale_y = (float) (*src_y1 - *src_y0) / (*dst_y1 - *dst_y0);
+
+   /* Clip left side */
+   clip_coordinates(mirror_x, src_x0, dst_x0, dst_x1,
+                    clip_dst_x0, clip_dst_x1, scale_x, true);
+
+   /* Clip right side */
+   clip_coordinates(mirror_x, src_x1, dst_x1, dst_x0,
+                    clip_dst_x1, clip_dst_x0, scale_x, false);
+
+   /* Clip bottom side */
+   clip_coordinates(mirror_y, src_y0, dst_y0, dst_y1,
+                    clip_dst_y0, clip_dst_y1, scale_y, true);
+
+   /* Clip top side */
+   clip_coordinates(mirror_y, src_y1, dst_y1, dst_y0,
+                    clip_dst_y1, clip_dst_y0, scale_y, false);
+
+   /* Check for invalid bounds
+    * Can't blit for 0-dimensions
+    */
+   return *src_x0 == *src_x1 || *src_y0 == *src_y1
+      || *dst_x0 == *dst_x1 || *dst_y0 == *dst_y1;
+}
+
+void
+crocus_blorp_surf_for_resource(struct crocus_vtable *vtbl,
+                               struct isl_device *isl_dev,
+                               struct blorp_surf *surf,
+                               struct pipe_resource *p_res,
+                               enum isl_aux_usage aux_usage,
+                               unsigned level,
+                               bool is_render_target)
+{
+   struct crocus_resource *res = (void *) p_res;
+
+   if (isl_aux_usage_has_hiz(aux_usage) &&
+       !crocus_resource_level_has_hiz(res, level))
+      aux_usage = ISL_AUX_USAGE_NONE;
+
+   *surf = (struct blorp_surf) {
+      .surf = &res->surf,
+      .addr = (struct blorp_address) {
+         .buffer = res->bo,
+         .offset = res->offset,
+         .reloc_flags = is_render_target ? EXEC_OBJECT_WRITE : 0,
+         .mocs = crocus_mocs(res->bo, isl_dev),
+      },
+      .aux_usage = aux_usage,
+   };
+
+   if (aux_usage != ISL_AUX_USAGE_NONE) {
+      surf->aux_surf = &res->aux.surf;
+      surf->aux_addr = (struct blorp_address) {
+         .buffer = res->aux.bo,
+         .offset = res->aux.offset,
+         .reloc_flags = is_render_target ? EXEC_OBJECT_WRITE : 0,
+         .mocs = crocus_mocs(res->bo, isl_dev),
+      };
+      surf->clear_color =
+         crocus_resource_get_clear_color(res);
+   }
+}
+
+static void
+tex_cache_flush_hack(struct crocus_batch *batch,
+                     enum isl_format view_format,
+                     enum isl_format surf_format)
+{
+   /* The WaSamplerCacheFlushBetweenRedescribedSurfaceReads workaround says:
+    *
+    *    "Currently Sampler assumes that a surface would not have two
+    *     different format associate with it.  It will not properly cache
+    *     the different views in the MT cache, causing a data corruption."
+    *
+    * We may need to handle this for texture views in general someday, but
+    * for now we handle it here, as it hurts copies and blits particularly
+    * badly because they ofter reinterpret formats.
+    *
+    * If the BO hasn't been referenced yet this batch, we assume that the
+    * texture cache doesn't contain any relevant data nor need flushing.
+    *
+    * Icelake (Gen11+) claims to fix this issue, but seems to still have
+    * issues with ASTC formats.
+    */
+   bool need_flush = view_format != surf_format;
+   if (!need_flush)
+      return;
+
+   const char *reason =
+      "workaround: WaSamplerCacheFlushBetweenRedescribedSurfaceReads";
+
+   crocus_emit_pipe_control_flush(batch, reason, PIPE_CONTROL_CS_STALL);
+   crocus_emit_pipe_control_flush(batch, reason,
+                                  PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
+}
+
+static struct crocus_resource *
+crocus_resource_for_aspect(const struct intel_device_info *devinfo,
+                           struct pipe_resource *p_res, unsigned pipe_mask)
+{
+   if (pipe_mask == PIPE_MASK_S) {
+      struct crocus_resource *junk, *s_res;
+      crocus_get_depth_stencil_resources(devinfo, p_res, &junk, &s_res);
+      return s_res;
+   } else {
+      return (struct crocus_resource *)p_res;
+   }
+}
+
+static enum pipe_format
+pipe_format_for_aspect(enum pipe_format format, unsigned pipe_mask)
+{
+   if (pipe_mask == PIPE_MASK_S) {
+      return util_format_stencil_only(format);
+   } else if (pipe_mask == PIPE_MASK_Z) {
+      return util_format_get_depth_only(format);
+   } else {
+      return format;
+   }
+}
+
+static void
+crocus_u_blitter(struct crocus_context *ice,
+                 const struct pipe_blit_info *info)
+{
+   struct pipe_blit_info dinfo = *info;
+   if (!util_format_has_alpha(dinfo.dst.resource->format))
+      dinfo.mask &= ~PIPE_MASK_A;
+   crocus_blitter_begin(ice, CROCUS_SAVE_FRAMEBUFFER | CROCUS_SAVE_TEXTURES | CROCUS_SAVE_FRAGMENT_STATE, info->render_condition_enable);
+   util_blitter_blit(ice->blitter, &dinfo, NULL);
+}
+
+/**
+ * The pipe->blit() driver hook.
+ *
+ * This performs a blit between two surfaces, which copies data but may
+ * also perform format conversion, scaling, flipping, and so on.
+ */
+static void
+crocus_blit(struct pipe_context *ctx, const struct pipe_blit_info *info)
+{
+   struct crocus_context *ice = (void *) ctx;
+   struct crocus_screen *screen = (struct crocus_screen *)ctx->screen;
+   const struct intel_device_info *devinfo = &screen->devinfo;
+   struct crocus_batch *batch = &ice->batches[CROCUS_BATCH_RENDER];
+   enum blorp_batch_flags blorp_flags = 0;
+
+   /* We don't support color masking. */
+   assert((info->mask & PIPE_MASK_RGBA) == PIPE_MASK_RGBA ||
+          (info->mask & PIPE_MASK_RGBA) == 0);
+
+   if (info->render_condition_enable)
+      if (!crocus_check_conditional_render(ice))
+         return;
+
+   if (devinfo->ver <= 5) {
+      if (!screen->vtbl.blit_blt(batch, info)) {
+
+         if (!util_format_is_depth_or_stencil(info->src.resource->format) &&
+             info->dst.resource->target != PIPE_TEXTURE_3D)
+            goto use_blorp;
+
+         if (!util_blitter_is_blit_supported(ice->blitter, info)) {
+            if (util_format_is_depth_or_stencil(info->src.resource->format)) {
+
+               struct pipe_blit_info depth_blit = *info;
+               depth_blit.mask = PIPE_MASK_Z;
+               crocus_blitter_begin(ice, CROCUS_SAVE_FRAMEBUFFER | CROCUS_SAVE_TEXTURES | CROCUS_SAVE_FRAGMENT_STATE, info->render_condition_enable);
+               util_blitter_blit(ice->blitter, &depth_blit, NULL);
+
+               struct pipe_surface dst_templ;
+               util_blitter_default_dst_texture(&dst_templ, info->dst.resource, info->dst.level, info->dst.box.z);
+
+               crocus_blitter_begin(ice, CROCUS_SAVE_FRAMEBUFFER | CROCUS_SAVE_TEXTURES | CROCUS_SAVE_FRAGMENT_STATE, info->render_condition_enable);
+
+               util_blitter_clear_depth_stencil(ice->blitter, &dst_templ, PIPE_CLEAR_STENCIL,
+                                                0, 0, info->dst.box.x, info->dst.box.y,
+                                                info->dst.box.width, info->dst.box.height);
+               crocus_blitter_begin(ice, CROCUS_SAVE_FRAMEBUFFER | CROCUS_SAVE_TEXTURES | CROCUS_SAVE_FRAGMENT_STATE, info->render_condition_enable);
+               util_blitter_stencil_fallback(ice->blitter,
+                                             info->dst.resource,
+                                             info->dst.level,
+                                             &info->dst.box,
+                                             info->src.resource,
+                                             info->src.level,
+                                             &info->src.box, NULL);
+            }
+            return;
+         }
+
+         crocus_u_blitter(ice, info);
+      }
+      return;
+   }
+
+   if (devinfo->ver == 6) {
+      if (info->src.resource->target == PIPE_TEXTURE_3D &&
+          info->dst.resource->target == PIPE_TEXTURE_3D) {
+         crocus_u_blitter(ice, info);
+         return;
+      }
+   }
+
+use_blorp:
+   if (info->render_condition_enable) {
+      if (ice->state.predicate == CROCUS_PREDICATE_STATE_USE_BIT)
+         blorp_flags |= BLORP_BATCH_PREDICATE_ENABLE;
+   }
+
+   float src_x0 = info->src.box.x;
+   float src_x1 = info->src.box.x + info->src.box.width;
+   float src_y0 = info->src.box.y;
+   float src_y1 = info->src.box.y + info->src.box.height;
+   float dst_x0 = info->dst.box.x;
+   float dst_x1 = info->dst.box.x + info->dst.box.width;
+   float dst_y0 = info->dst.box.y;
+   float dst_y1 = info->dst.box.y + info->dst.box.height;
+   bool mirror_x = apply_mirror(&src_x0, &src_x1);
+   bool mirror_y = apply_mirror(&src_y0, &src_y1);
+   enum blorp_filter filter;
+
+   if (info->scissor_enable) {
+      bool noop = apply_blit_scissor(&info->scissor,
+                                     &src_x0, &src_y0, &src_x1, &src_y1,
+                                     &dst_x0, &dst_y0, &dst_x1, &dst_y1,
+                                     mirror_x, mirror_y);
+      if (noop)
+         return;
+   }
+
+   if (abs(info->dst.box.width) == abs(info->src.box.width) &&
+       abs(info->dst.box.height) == abs(info->src.box.height)) {
+      if (info->src.resource->nr_samples > 1 &&
+          info->dst.resource->nr_samples <= 1) {
+         /* The OpenGL ES 3.2 specification, section 16.2.1, says:
+          *
+          *    "If the read framebuffer is multisampled (its effective
+          *     value of SAMPLE_BUFFERS is one) and the draw framebuffer
+          *     is not (its value of SAMPLE_BUFFERS is zero), the samples
+          *     corresponding to each pixel location in the source are
+          *     converted to a single sample before being written to the
+          *     destination.  The filter parameter is ignored.  If the
+          *     source formats are integer types or stencil values, a
+          *     single sample’s value is selected for each pixel.  If the
+          *     source formats are floating-point or normalized types,
+          *     the sample values for each pixel are resolved in an
+          *     implementation-dependent manner.  If the source formats
+          *     are depth values, sample values are resolved in an
+          *     implementation-dependent manner where the result will be
+          *     between the minimum and maximum depth values in the pixel."
+          *
+          * When selecting a single sample, we always choose sample 0.
+          */
+         if (util_format_is_depth_or_stencil(info->src.format) ||
+             util_format_is_pure_integer(info->src.format)) {
+            filter = BLORP_FILTER_SAMPLE_0;
+         } else {
+            filter = BLORP_FILTER_AVERAGE;
+         }
+      } else {
+         /* The OpenGL 4.6 specification, section 18.3.1, says:
+          *
+          *    "If the source and destination dimensions are identical,
+          *     no filtering is applied."
+          *
+          * Using BLORP_FILTER_NONE will also handle the upsample case by
+          * replicating the one value in the source to all values in the
+          * destination.
+          */
+         filter = BLORP_FILTER_NONE;
+      }
+   } else if (info->filter == PIPE_TEX_FILTER_LINEAR) {
+      filter = BLORP_FILTER_BILINEAR;
+   } else {
+      filter = BLORP_FILTER_NEAREST;
+   }
+
+   struct blorp_batch blorp_batch;
+   blorp_batch_init(&ice->blorp, &blorp_batch, batch, blorp_flags);
+
+   float src_z_step = (float)info->src.box.depth / (float)info->dst.box.depth;
+
+   /* There is no interpolation to the pixel center during rendering, so
+    * add the 0.5 offset ourselves here.
+    */
+   float depth_center_offset = 0;
+   if (info->src.resource->target == PIPE_TEXTURE_3D)
+      depth_center_offset = 0.5 / info->dst.box.depth * info->src.box.depth;
+
+   /* Perform a blit for each aspect requested by the caller. PIPE_MASK_R is
+    * used to represent the color aspect. */
+   unsigned aspect_mask = info->mask & (PIPE_MASK_R | PIPE_MASK_ZS);
+   while (aspect_mask) {
+      unsigned aspect = 1 << u_bit_scan(&aspect_mask);
+
+      struct crocus_resource *src_res =
+         crocus_resource_for_aspect(devinfo, info->src.resource, aspect);
+      struct crocus_resource *dst_res =
+         crocus_resource_for_aspect(devinfo, info->dst.resource, aspect);
+
+      enum pipe_format src_pfmt =
+         pipe_format_for_aspect(info->src.format, aspect);
+      enum pipe_format dst_pfmt =
+         pipe_format_for_aspect(info->dst.format, aspect);
+
+      struct crocus_format_info src_fmt =
+         crocus_format_for_usage(devinfo, src_pfmt, ISL_SURF_USAGE_TEXTURE_BIT);
+      enum isl_aux_usage src_aux_usage =
+         crocus_resource_texture_aux_usage(src_res);
+
+      crocus_resource_prepare_texture(ice, src_res, src_fmt.fmt,
+                                      info->src.level, 1, info->src.box.z,
+                                      info->src.box.depth);
+      //      crocus_emit_buffer_barrier_for(batch, src_res->bo,
+      //                                   CROCUS_DOMAIN_OTHER_READ);
+
+      bool dst_aux_disable = false;
+      /* on SNB blorp will use render target instead of depth
+       * so disable HiZ.
+       */
+      if (devinfo->ver <= 6 && util_format_is_depth_or_stencil(dst_pfmt))
+         dst_aux_disable = true;
+      struct crocus_format_info dst_fmt =
+         crocus_format_for_usage(devinfo, dst_pfmt,
+                                 ISL_SURF_USAGE_RENDER_TARGET_BIT);
+      enum isl_aux_usage dst_aux_usage =
+         crocus_resource_render_aux_usage(ice, dst_res, info->dst.level,
+                                          dst_fmt.fmt, dst_aux_disable);
+
+      struct blorp_surf src_surf, dst_surf;
+      crocus_blorp_surf_for_resource(&screen->vtbl, &screen->isl_dev, &src_surf,
+                                     &src_res->base.b, src_aux_usage,
+                                     info->src.level, false);
+      crocus_blorp_surf_for_resource(&screen->vtbl, &screen->isl_dev, &dst_surf,
+                                     &dst_res->base.b, dst_aux_usage,
+                                     info->dst.level, true);
+
+      crocus_resource_prepare_render(ice, dst_res, info->dst.level,
+                                     info->dst.box.z, info->dst.box.depth,
+                                     dst_aux_usage);
+      //      crocus_emit_buffer_barrier_for(batch, dst_res->bo,
+      //                                   CROCUS_DOMAIN_RENDER_WRITE);
+
+      if (crocus_batch_references(batch, src_res->bo))
+         tex_cache_flush_hack(batch, src_fmt.fmt, src_res->surf.format);
+
+      if (dst_res->base.b.target == PIPE_BUFFER) {
+         util_range_add(&dst_res->base.b, &dst_res->valid_buffer_range,
+                        dst_x0, dst_x1);
+      }
+
+      struct isl_swizzle src_swiz = pipe_to_isl_swizzles(src_fmt.swizzles);
+      struct isl_swizzle dst_swiz = pipe_to_isl_swizzles(dst_fmt.swizzles);
+
+      for (int slice = 0; slice < info->dst.box.depth; slice++) {
+         unsigned dst_z = info->dst.box.z + slice;
+         float src_z = info->src.box.z + slice * src_z_step +
+            depth_center_offset;
+
+         crocus_batch_maybe_flush(batch, 1500);
+
+         blorp_blit(&blorp_batch,
+                    &src_surf, info->src.level, src_z,
+                    src_fmt.fmt, src_swiz,
+                    &dst_surf, info->dst.level, dst_z,
+                    dst_fmt.fmt, dst_swiz,
+                    src_x0, src_y0, src_x1, src_y1,
+                    dst_x0, dst_y0, dst_x1, dst_y1,
+                    filter, mirror_x, mirror_y);
+
+      }
+
+      tex_cache_flush_hack(batch, src_fmt.fmt, src_res->surf.format);
+
+      crocus_resource_finish_render(ice, dst_res, info->dst.level,
+                                    info->dst.box.z, info->dst.box.depth,
+                                    dst_aux_usage);
+   }
+
+   blorp_batch_finish(&blorp_batch);
+
+   crocus_flush_and_dirty_for_history(ice, batch, (struct crocus_resource *)
+                                      info->dst.resource,
+                                      PIPE_CONTROL_RENDER_TARGET_FLUSH,
+                                      "cache history: post-blit");
+}
+
+static void
+get_copy_region_aux_settings(struct crocus_resource *res,
+                             enum isl_aux_usage *out_aux_usage,
+                             bool is_render_target)
+{
+   switch (res->aux.usage) {
+   case ISL_AUX_USAGE_MCS:
+      /* A stencil resolve operation must be performed prior to doing resource
+       * copies or used by CPU.
+       * (see HSD 1209978162)
+       */
+      if (is_render_target && isl_surf_usage_is_stencil(res->surf.usage)) {
+         *out_aux_usage = ISL_AUX_USAGE_NONE;
+      } else {
+         *out_aux_usage = res->aux.usage;
+      }
+      break;
+   default:
+      *out_aux_usage = ISL_AUX_USAGE_NONE;
+      break;
+   }
+}
+
+/**
+ * Perform a GPU-based raw memory copy between compatible view classes.
+ *
+ * Does not perform any flushing - the new data may still be left in the
+ * render cache, and old data may remain in other caches.
+ *
+ * Wraps blorp_copy() and blorp_buffer_copy().
+ */
+void
+crocus_copy_region(struct blorp_context *blorp,
+                   struct crocus_batch *batch,
+                   struct pipe_resource *dst,
+                   unsigned dst_level,
+                   unsigned dstx, unsigned dsty, unsigned dstz,
+                   struct pipe_resource *src,
+                   unsigned src_level,
+                   const struct pipe_box *src_box)
+{
+   struct blorp_batch blorp_batch;
+   struct crocus_context *ice = blorp->driver_ctx;
+   struct crocus_screen *screen = (void *) ice->ctx.screen;
+   const struct intel_device_info *devinfo = &screen->devinfo;
+   struct crocus_resource *src_res = (void *) src;
+   struct crocus_resource *dst_res = (void *) dst;
+
+   if (devinfo->ver <= 5) {
+      if (screen->vtbl.copy_region_blt(batch, dst_res,
+                                       dst_level, dstx, dsty, dstz,
+                                       src_res, src_level, src_box))
+         return;
+   }
+   enum isl_aux_usage src_aux_usage, dst_aux_usage;
+   get_copy_region_aux_settings(src_res, &src_aux_usage,
+                                false);
+   get_copy_region_aux_settings(dst_res, &dst_aux_usage,
+                                true);
+
+   if (crocus_batch_references(batch, src_res->bo))
+      tex_cache_flush_hack(batch, ISL_FORMAT_UNSUPPORTED, src_res->surf.format);
+
+   if (dst->target == PIPE_BUFFER)
+      util_range_add(&dst_res->base.b, &dst_res->valid_buffer_range, dstx, dstx + src_box->width);
+
+   if (dst->target == PIPE_BUFFER && src->target == PIPE_BUFFER) {
+      struct blorp_address src_addr = {
+         .buffer = crocus_resource_bo(src), .offset = src_box->x,
+         .mocs = crocus_mocs(src_res->bo, &screen->isl_dev),
+      };
+      struct blorp_address dst_addr = {
+         .buffer = crocus_resource_bo(dst), .offset = dstx,
+         .mocs = crocus_mocs(dst_res->bo, &screen->isl_dev),
+         .reloc_flags = EXEC_OBJECT_WRITE,
+      };
+
+      crocus_batch_maybe_flush(batch, 1500);
+
+      blorp_batch_init(&ice->blorp, &blorp_batch, batch, 0);
+      blorp_buffer_copy(&blorp_batch, src_addr, dst_addr, src_box->width);
+      blorp_batch_finish(&blorp_batch);
+   } else {
+      // XXX: what about one surface being a buffer and not the other?
+
+      struct blorp_surf src_surf, dst_surf;
+      crocus_blorp_surf_for_resource(&screen->vtbl, &screen->isl_dev, &src_surf,
+                                     src, src_aux_usage, src_level, false);
+      crocus_blorp_surf_for_resource(&screen->vtbl, &screen->isl_dev, &dst_surf,
+                                     dst, dst_aux_usage, dst_level, true);
+
+      crocus_resource_prepare_access(ice, src_res, src_level, 1,
+                                     src_box->z, src_box->depth,
+                                     src_aux_usage, false);
+      crocus_resource_prepare_access(ice, dst_res, dst_level, 1,
+                                     dstz, src_box->depth,
+                                     dst_aux_usage, false);
+
+      blorp_batch_init(&ice->blorp, &blorp_batch, batch, 0);
+
+      for (int slice = 0; slice < src_box->depth; slice++) {
+         crocus_batch_maybe_flush(batch, 1500);
+
+         blorp_copy(&blorp_batch, &src_surf, src_level, src_box->z + slice,
+                    &dst_surf, dst_level, dstz + slice,
+                    src_box->x, src_box->y, dstx, dsty,
+                    src_box->width, src_box->height);
+      }
+      blorp_batch_finish(&blorp_batch);
+
+      crocus_resource_finish_write(ice, dst_res, dst_level, dstz,
+                                   src_box->depth, dst_aux_usage);
+   }
+
+   tex_cache_flush_hack(batch, ISL_FORMAT_UNSUPPORTED, src_res->surf.format);
+}
+
+/**
+ * The pipe->resource_copy_region() driver hook.
+ *
+ * This implements ARB_copy_image semantics - a raw memory copy between
+ * compatible view classes.
+ */
+static void
+crocus_resource_copy_region(struct pipe_context *ctx,
+                            struct pipe_resource *p_dst,
+                            unsigned dst_level,
+                            unsigned dstx, unsigned dsty, unsigned dstz,
+                            struct pipe_resource *p_src,
+                            unsigned src_level,
+                            const struct pipe_box *src_box)
+{
+   struct crocus_context *ice = (void *) ctx;
+   struct crocus_batch *batch = &ice->batches[CROCUS_BATCH_RENDER];
+   struct crocus_screen *screen = (struct crocus_screen *)ctx->screen;
+   const struct intel_device_info *devinfo = &screen->devinfo;
+   struct crocus_resource *dst = (void *) p_dst;
+
+   if (devinfo->ver < 6 && util_format_is_depth_or_stencil(p_dst->format)) {
+      util_resource_copy_region(ctx, p_dst, dst_level, dstx, dsty, dstz,
+                                p_src, src_level, src_box);
+      return;
+   }
+   crocus_copy_region(&ice->blorp, batch, p_dst, dst_level, dstx, dsty, dstz,
+                      p_src, src_level, src_box);
+
+   if (util_format_is_depth_and_stencil(p_dst->format) &&
+       util_format_has_stencil(util_format_description(p_src->format)) &&
+       devinfo->ver >= 6) {
+      struct crocus_resource *junk, *s_src_res, *s_dst_res;
+      crocus_get_depth_stencil_resources(devinfo, p_src, &junk, &s_src_res);
+      crocus_get_depth_stencil_resources(devinfo, p_dst, &junk, &s_dst_res);
+
+      crocus_copy_region(&ice->blorp, batch, &s_dst_res->base.b, dst_level, dstx,
+                         dsty, dstz, &s_src_res->base.b, src_level, src_box);
+   }
+
+   crocus_flush_and_dirty_for_history(ice, batch, dst,
+                                      PIPE_CONTROL_RENDER_TARGET_FLUSH,
+                                      "cache history: post copy_region");
+}
+
+void
+crocus_init_blit_functions(struct pipe_context *ctx)
+{
+   ctx->blit = crocus_blit;
+   ctx->resource_copy_region = crocus_resource_copy_region;
+}

src/gallium/drivers/crocus/crocus_blorp.c

@@ -0,0 +1,452 @@
+/*
+ * Copyright © 2018 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+/**
+ * @file crocus_blorp.c
+ *
+ * ============================= GENXML CODE =============================
+ *              [This file is compiled once per generation.]
+ * =======================================================================
+ *
+ * GenX specific code for working with BLORP (blitting, resolves, clears
+ * on the 3D engine).  This provides the driver-specific hooks needed to
+ * implement the BLORP API.
+ *
+ * See crocus_blit.c, crocus_clear.c, and so on.
+ */
+
+#include <assert.h>
+
+#include "crocus_batch.h"
+#include "crocus_resource.h"
+#include "crocus_context.h"
+
+#include "util/u_upload_mgr.h"
+#include "intel/common/intel_l3_config.h"
+
+#include "blorp/blorp_genX_exec_elk.h"
+
+#if GFX_VER <= 5
+#include "gen4_blorp_exec.h"
+#endif
+
+static uint32_t *
+stream_state(struct crocus_batch *batch,
+             unsigned size,
+             unsigned alignment,
+             uint32_t *out_offset,
+             struct crocus_bo **out_bo)
+{
+   uint32_t offset = align(batch->state.used, alignment);
+
+   if (offset + size >= STATE_SZ && !batch->no_wrap) {
+      crocus_batch_flush(batch);
+      offset = align(batch->state.used, alignment);
+   } else if (offset + size >= batch->state.bo->size) {
+      const unsigned new_size =
+         MIN2(batch->state.bo->size + batch->state.bo->size / 2,
+              MAX_STATE_SIZE);
+      crocus_grow_buffer(batch, true, batch->state.used, new_size);
+      assert(offset + size < batch->state.bo->size);
+   }
+
+   crocus_record_state_size(batch->state_sizes, offset, size);
+
+   batch->state.used = offset + size;
+   *out_offset = offset;
+
+   /* If the caller has asked for a BO, we leave them the responsibility of
+    * adding bo->gtt_offset (say, by handing an address to genxml).  If not,
+    * we assume they want the offset from a base address.
+    */
+   if (out_bo)
+      *out_bo = batch->state.bo;
+
+   return (uint32_t *)batch->state.map + (offset >> 2);
+}
+
+static void *
+blorp_emit_dwords(struct blorp_batch *blorp_batch, unsigned n)
+{
+   struct crocus_batch *batch = blorp_batch->driver_batch;
+   return crocus_get_command_space(batch, n * sizeof(uint32_t));
+}
+
+static uint64_t
+blorp_emit_reloc(struct blorp_batch *blorp_batch, UNUSED void *location,
+                 struct blorp_address addr, uint32_t delta)
+{
+   struct crocus_batch *batch = blorp_batch->driver_batch;
+   uint32_t offset;
+
+   if (GFX_VER < 6 && crocus_ptr_in_state_buffer(batch, location)) {
+      offset = (char *)location - (char *)batch->state.map;
+      return crocus_state_reloc(batch, offset,
+                                addr.buffer, addr.offset + delta,
+                                addr.reloc_flags);
+   }
+
+   assert(!crocus_ptr_in_state_buffer(batch, location));
+
+   offset = (char *)location - (char *)batch->command.map;
+   return crocus_command_reloc(batch, offset,
+                               addr.buffer, addr.offset + delta,
+                               addr.reloc_flags);
+}
+
+static void
+blorp_surface_reloc(struct blorp_batch *blorp_batch, uint32_t ss_offset,
+                    struct blorp_address addr, uint32_t delta)
+{
+   struct crocus_batch *batch = blorp_batch->driver_batch;
+   struct crocus_bo *bo = addr.buffer;
+
+   uint64_t reloc_val =
+      crocus_state_reloc(batch, ss_offset, bo, addr.offset + delta,
+                         addr.reloc_flags);
+
+   void *reloc_ptr = (void *)batch->state.map + ss_offset;
+   *(uint32_t *)reloc_ptr = reloc_val;
+}
+
+static uint64_t
+blorp_get_surface_address(struct blorp_batch *blorp_batch,
+                          struct blorp_address addr)
+{
+   /* We'll let blorp_surface_reloc write the address. */
+   return 0ull;
+}
+
+#if GFX_VER >= 7
+static struct blorp_address
+blorp_get_surface_base_address(struct blorp_batch *blorp_batch)
+{
+   struct crocus_batch *batch = blorp_batch->driver_batch;
+   return (struct blorp_address) {
+      .buffer = batch->state.bo,
+      .offset = 0
+   };
+}
+#endif
+
+static void *
+blorp_alloc_dynamic_state(struct blorp_batch *blorp_batch,
+                          uint32_t size,
+                          uint32_t alignment,
+                          uint32_t *offset)
+{
+   struct crocus_batch *batch = blorp_batch->driver_batch;
+
+   return stream_state(batch, size, alignment, offset, NULL);
+}
+
+UNUSED static void *
+blorp_alloc_general_state(struct blorp_batch *blorp_batch,
+                          uint32_t size,
+                          uint32_t alignment,
+                          uint32_t *offset)
+{
+   /* Use dynamic state range for general state on crocus. */
+   return blorp_alloc_dynamic_state(blorp_batch, size, alignment, offset);
+}
+
+static bool
+blorp_alloc_binding_table(struct blorp_batch *blorp_batch,
+                          unsigned num_entries,
+                          unsigned state_size,
+                          unsigned state_alignment,
+                          uint32_t *bt_offset,
+                          uint32_t *surface_offsets,
+                          void **surface_maps)
+{
+   struct crocus_batch *batch = blorp_batch->driver_batch;
+   uint32_t *bt_map = stream_state(batch, num_entries * sizeof(uint32_t), 32,
+                                   bt_offset, NULL);
+
+   for (unsigned i = 0; i < num_entries; i++) {
+      surface_maps[i] = stream_state(batch,
+                                     state_size, state_alignment,
+                                     &(surface_offsets)[i], NULL);
+      bt_map[i] = surface_offsets[i];
+   }
+
+   return true;
+}
+
+static uint32_t
+blorp_binding_table_offset_to_pointer(struct blorp_batch *batch,
+                                      uint32_t offset)
+{
+   return offset;
+}
+
+static void *
+blorp_alloc_vertex_buffer(struct blorp_batch *blorp_batch,
+                          uint32_t size,
+                          struct blorp_address *addr)
+{
+   struct crocus_batch *batch = blorp_batch->driver_batch;
+   struct crocus_bo *bo;
+   uint32_t offset;
+
+   void *map = stream_state(batch, size, 64,
+                            &offset, &bo);
+
+   *addr = (struct blorp_address) {
+      .buffer = bo,
+      .offset = offset,
+      .reloc_flags = RELOC_32BIT,
+#if GFX_VER >= 7
+      .mocs = crocus_mocs(bo, &batch->screen->isl_dev),
+#endif
+   };
+
+   return map;
+}
+
+/**
+ */
+static void
+blorp_vf_invalidate_for_vb_48b_transitions(struct blorp_batch *blorp_batch,
+                                           const struct blorp_address *addrs,
+                                           UNUSED uint32_t *sizes,
+                                           unsigned num_vbs)
+{
+}
+
+static struct blorp_address
+blorp_get_workaround_address(struct blorp_batch *blorp_batch)
+{
+   struct crocus_batch *batch = blorp_batch->driver_batch;
+
+   return (struct blorp_address) {
+      .buffer = batch->ice->workaround_bo,
+      .offset = batch->ice->workaround_offset,
+   };
+}
+
+static void
+blorp_flush_range(UNUSED struct blorp_batch *blorp_batch,
+                  UNUSED void *start,
+                  UNUSED size_t size)
+{
+   /* All allocated states come from the batch which we will flush before we
+    * submit it.  There's nothing for us to do here.
+    */
+}
+
+#if GFX_VER >= 7
+static const struct intel_l3_config *
+blorp_get_l3_config(struct blorp_batch *blorp_batch)
+{
+   struct crocus_batch *batch = blorp_batch->driver_batch;
+   return batch->screen->l3_config_3d;
+}
+#endif
+
+static void
+blorp_pre_emit_urb_config(struct blorp_batch *blorp_batch,
+                          struct intel_urb_config *urb_cfg)
+{
+   /* Dummy. */
+}
+
+static void
+blorp_emit_urb_config(struct blorp_batch *blorp_batch,
+                      struct intel_urb_config *urb_cfg)
+{
+#if GFX_VER < 7
+   struct crocus_batch *batch = blorp_batch->driver_batch;
+#if GFX_VER <= 5
+   batch->screen->vtbl.calculate_urb_fence(batch, 0,
+                                           urb_cfg->size[MESA_SHADER_VERTEX],
+                                           urb_cfg->size[MESA_SHADER_FRAGMENT]);
+#else
+   genX(crocus_upload_urb)(batch, urb_cfg->size[MESA_SHADER_VERTEX], false,
+                           urb_cfg->size[MESA_SHADER_VERTEX]);
+#endif
+#endif
+}
+
+static void
+crocus_blorp_exec(struct blorp_batch *blorp_batch,
+                  const struct blorp_params *params)
+{
+   struct crocus_context *ice = blorp_batch->blorp->driver_ctx;
+   struct crocus_batch *batch = blorp_batch->driver_batch;
+
+   /* Flush the sampler and render caches.  We definitely need to flush the
+    * sampler cache so that we get updated contents from the render cache for
+    * the glBlitFramebuffer() source.  Also, we are sometimes warned in the
+    * docs to flush the cache between reinterpretations of the same surface
+    * data with different formats, which blorp does for stencil and depth
+    * data.
+    */
+   if (params->src.enabled)
+      crocus_cache_flush_for_read(batch, params->src.addr.buffer);
+   if (params->dst.enabled) {
+      crocus_cache_flush_for_render(batch, params->dst.addr.buffer,
+                                    params->dst.view.format,
+                                    params->dst.aux_usage);
+   }
+   if (params->depth.enabled)
+      crocus_cache_flush_for_depth(batch, params->depth.addr.buffer);
+   if (params->stencil.enabled)
+      crocus_cache_flush_for_depth(batch, params->stencil.addr.buffer);
+
+   crocus_require_command_space(batch, 1400);
+   crocus_require_statebuffer_space(batch, 600);
+   batch->no_wrap = true;
+
+#if GFX_VER == 8
+   genX(crocus_update_pma_fix)(ice, batch, false);
+#endif
+
+#if GFX_VER == 6
+   /* Emit workaround flushes when we switch from drawing to blorping. */
+   crocus_emit_post_sync_nonzero_flush(batch);
+#endif
+
+#if GFX_VER >= 6
+   crocus_emit_depth_stall_flushes(batch);
+#endif
+
+   blorp_emit(blorp_batch, GENX(3DSTATE_DRAWING_RECTANGLE), rect) {
+      rect.ClippedDrawingRectangleXMax = MAX2(params->x1, params->x0) - 1;
+      rect.ClippedDrawingRectangleYMax = MAX2(params->y1, params->y0) - 1;
+   }
+
+   batch->screen->vtbl.update_surface_base_address(batch);
+   crocus_handle_always_flush_cache(batch);
+
+   batch->contains_draw = true;
+   blorp_exec(blorp_batch, params);
+
+   batch->no_wrap = false;
+   crocus_handle_always_flush_cache(batch);
+
+   /* We've smashed all state compared to what the normal 3D pipeline
+    * rendering tracks for GL.
+    */
+
+   uint64_t skip_bits = (CROCUS_DIRTY_POLYGON_STIPPLE |
+                         CROCUS_DIRTY_GEN7_SO_BUFFERS |
+                         CROCUS_DIRTY_SO_DECL_LIST |
+                         CROCUS_DIRTY_LINE_STIPPLE |
+                         CROCUS_ALL_DIRTY_FOR_COMPUTE |
+                         CROCUS_DIRTY_GEN6_SCISSOR_RECT |
+                         CROCUS_DIRTY_GEN75_VF |
+                         CROCUS_DIRTY_SF_CL_VIEWPORT);
+
+   uint64_t skip_stage_bits = (CROCUS_ALL_STAGE_DIRTY_FOR_COMPUTE |
+                               CROCUS_STAGE_DIRTY_UNCOMPILED_VS |
+                               CROCUS_STAGE_DIRTY_UNCOMPILED_TCS |
+                               CROCUS_STAGE_DIRTY_UNCOMPILED_TES |
+                               CROCUS_STAGE_DIRTY_UNCOMPILED_GS |
+                               CROCUS_STAGE_DIRTY_UNCOMPILED_FS |
+                               CROCUS_STAGE_DIRTY_SAMPLER_STATES_VS |
+                               CROCUS_STAGE_DIRTY_SAMPLER_STATES_TCS |
+                               CROCUS_STAGE_DIRTY_SAMPLER_STATES_TES |
+                               CROCUS_STAGE_DIRTY_SAMPLER_STATES_GS);
+
+   if (!ice->shaders.uncompiled[MESA_SHADER_TESS_EVAL]) {
+      /* BLORP disabled tessellation, that's fine for the next draw */
+     skip_stage_bits |= CROCUS_STAGE_DIRTY_TCS |
+                        CROCUS_STAGE_DIRTY_TES |
+                        CROCUS_STAGE_DIRTY_CONSTANTS_TCS |
+                        CROCUS_STAGE_DIRTY_CONSTANTS_TES |
+                        CROCUS_STAGE_DIRTY_BINDINGS_TCS |
+                        CROCUS_STAGE_DIRTY_BINDINGS_TES;
+   }
+
+   if (!ice->shaders.uncompiled[MESA_SHADER_GEOMETRY]) {
+      /* BLORP disabled geometry shaders, that's fine for the next draw */
+     skip_stage_bits |= CROCUS_STAGE_DIRTY_GS |
+                        CROCUS_STAGE_DIRTY_CONSTANTS_GS |
+                        CROCUS_STAGE_DIRTY_BINDINGS_GS;
+   }
+
+   /* we can skip flagging CROCUS_DIRTY_DEPTH_BUFFER, if
+    * BLORP_BATCH_NO_EMIT_DEPTH_STENCIL is set.
+    */
+   if (blorp_batch->flags & BLORP_BATCH_NO_EMIT_DEPTH_STENCIL)
+      skip_bits |= CROCUS_DIRTY_DEPTH_BUFFER;
+
+   if (!params->fs_prog_data)
+      skip_bits |= CROCUS_DIRTY_GEN6_BLEND_STATE;
+
+   ice->state.dirty |= ~skip_bits;
+   ice->state.stage_dirty |= ~skip_stage_bits;
+
+   ice->urb.vsize = 0;
+   ice->urb.gs_present = false;
+   ice->urb.gsize = 0;
+   ice->urb.tess_present = false;
+   ice->urb.hsize = 0;
+   ice->urb.dsize = 0;
+
+   if (params->dst.enabled) {
+      crocus_render_cache_add_bo(batch, params->dst.addr.buffer,
+                                 params->dst.view.format,
+                                 params->dst.aux_usage);
+   }
+   if (params->depth.enabled)
+      crocus_depth_cache_add_bo(batch, params->depth.addr.buffer);
+   if (params->stencil.enabled)
+      crocus_depth_cache_add_bo(batch, params->stencil.addr.buffer);
+}
+
+static void
+blorp_measure_start(struct blorp_batch *blorp_batch,
+                    const struct blorp_params *params)
+{
+}
+
+static void
+blorp_measure_end(struct blorp_batch *blorp_batch,
+                  const struct blorp_params *params)
+{
+}
+
+static void
+blorp_emit_pre_draw(struct blorp_batch *batch, const struct blorp_params *params)
+{
+   /* "Not implemented" */
+}
+
+static void
+blorp_emit_post_draw(struct blorp_batch *batch, const struct blorp_params *params)
+{
+   /* "Not implemented" */
+}
+
+void
+genX(crocus_init_blorp)(struct crocus_context *ice)
+{
+   struct crocus_screen *screen = (struct crocus_screen *)ice->ctx.screen;
+
+   blorp_init_elk(&ice->blorp, ice, &screen->isl_dev, screen->compiler, NULL);
+   ice->blorp.lookup_shader = crocus_blorp_lookup_shader;
+   ice->blorp.upload_shader = crocus_blorp_upload_shader;
+   ice->blorp.get_surface_address = blorp_get_surface_address;
+   ice->blorp.exec = crocus_blorp_exec;
+}

src/gallium/drivers/crocus/crocus_blt.c

@@ -0,0 +1,420 @@
+/*
+ * Copyright © 2018 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+/* blt command encoding for gen4/5 */
+#include "crocus_context.h"
+
+#include "crocus_genx_macros.h"
+#include "crocus_genx_protos.h"
+#include "crocus_resource.h"
+
+#define FILE_DEBUG_FLAG DEBUG_BLIT
+
+#if GFX_VER <= 5
+
+static uint32_t
+color_depth_for_cpp(int cpp)
+{
+   switch (cpp) {
+   case 4: return COLOR_DEPTH_32bit;
+   case 2: return COLOR_DEPTH_565;
+   case 1: return COLOR_DEPTH_8bit;
+   default:
+      UNREACHABLE("not reached");
+   }
+}
+
+static void
+blt_set_alpha_to_one(struct crocus_batch *batch,
+		     struct crocus_resource *dst,
+		     int x, int y, int width, int height)
+{
+   const struct isl_format_layout *fmtl = isl_format_get_layout(dst->surf.format);
+   unsigned cpp = fmtl->bpb / 8;
+   uint32_t pitch = dst->surf.row_pitch_B;
+
+   if (dst->surf.tiling != ISL_TILING_LINEAR)
+      pitch /= 4;
+   /* We need to split the blit into chunks that each fit within the blitter's
+    * restrictions.  We can't use a chunk size of 32768 because we need to
+    * ensure that src_tile_x + chunk_size fits.  We choose 16384 because it's
+    * a nice round power of two, big enough that performance won't suffer, and
+    * small enough to guarantee everything fits.
+    */
+   const uint32_t max_chunk_size = 16384;
+
+   for (uint32_t chunk_x = 0; chunk_x < width; chunk_x += max_chunk_size) {
+      for (uint32_t chunk_y = 0; chunk_y < height; chunk_y += max_chunk_size) {
+         const uint32_t chunk_w = MIN2(max_chunk_size, width - chunk_x);
+         const uint32_t chunk_h = MIN2(max_chunk_size, height - chunk_y);
+         uint32_t tile_x, tile_y;
+         uint64_t offset_B;
+         ASSERTED uint32_t z_offset_el, array_offset;
+         isl_tiling_get_intratile_offset_el(dst->surf.tiling, dst->surf.dim,
+                                            dst->surf.msaa_layout,
+                                            cpp * 8, dst->surf.samples,
+                                            dst->surf.row_pitch_B,
+                                            dst->surf.array_pitch_el_rows,
+                                            chunk_x, chunk_y, 0, 0,
+                                            &offset_B,
+                                            &tile_x, &tile_y,
+                                            &z_offset_el, &array_offset);
+         assert(z_offset_el == 0);
+         assert(array_offset == 0);
+	 crocus_emit_cmd(batch, GENX(XY_COLOR_BLT), xyblt) {
+            xyblt.TilingEnable = dst->surf.tiling != ISL_TILING_LINEAR;
+            xyblt.ColorDepth = color_depth_for_cpp(cpp);
+            xyblt.RasterOperation = 0xF0;
+            xyblt.DestinationPitch = pitch;
+            xyblt._32bppByteMask = 2;
+            xyblt.DestinationBaseAddress = rw_bo(dst->bo, offset_B);
+            xyblt.DestinationX1Coordinate = tile_x;
+            xyblt.DestinationY1Coordinate = tile_y;
+            xyblt.DestinationX2Coordinate = tile_x + chunk_w;
+            xyblt.DestinationY2Coordinate = tile_y + chunk_h;
+            xyblt.SolidPatternColor = 0xffffffff;
+	 }
+      }
+   }
+}
+
+static bool validate_blit_for_blt(struct crocus_batch *batch,
+                                  const struct pipe_blit_info *info)
+{
+   /* If the source and destination are the same size with no mirroring,
+    * the rectangles are within the size of the texture and there is no
+    * scissor, then we can probably use the blit engine.
+    */
+   if (info->dst.box.width != info->src.box.width ||
+       info->dst.box.height != info->src.box.height)
+      return false;
+
+   if (info->scissor_enable)
+      return false;
+
+   if (info->dst.box.height < 0 || info->src.box.height < 0)
+      return false;
+
+   if (info->dst.box.depth > 1 || info->src.box.depth > 1)
+      return false;
+
+   const struct util_format_description *desc =
+      util_format_description(info->src.format);
+   int i = util_format_get_first_non_void_channel(info->src.format);
+   if (i == -1)
+      return false;
+
+   /* can't do the alpha to 1 setting for these. */
+   if ((util_format_has_alpha1(info->src.format) &&
+        util_format_has_alpha(info->dst.format) &&
+        desc->channel[i].size > 8))
+      return false;
+   return true;
+}
+
+static inline int crocus_resource_blt_pitch(struct crocus_resource *res)
+{
+   int pitch = res->surf.row_pitch_B;
+   if (res->surf.tiling != ISL_TILING_LINEAR)
+      pitch /= 4;
+   return pitch;
+}
+
+
+static bool emit_copy_blt(struct crocus_batch *batch,
+                          struct crocus_resource *src,
+                          struct crocus_resource *dst,
+                          unsigned cpp,
+                          int32_t src_pitch,
+                          unsigned src_offset,
+                          int32_t dst_pitch,
+                          unsigned dst_offset,
+                          uint16_t src_x, uint16_t src_y,
+                          uint16_t dst_x, uint16_t dst_y,
+                          uint16_t w, uint16_t h)
+
+{
+   uint32_t src_tile_w, src_tile_h;
+   uint32_t dst_tile_w, dst_tile_h;
+   int dst_y2 = dst_y + h;
+   int dst_x2 = dst_x + w;
+
+   DBG("%s src:buf(%p)/%d+%d %d,%d dst:buf(%p)/%d+%d %d,%d sz:%dx%d\n",
+       __func__,
+       src, src_pitch, src_offset, src_x, src_y,
+       dst, dst_pitch, dst_offset, dst_x, dst_y, w, h);
+
+   isl_get_tile_dims(src->surf.tiling, cpp, &src_tile_w, &src_tile_h);
+   isl_get_tile_dims(dst->surf.tiling, cpp, &dst_tile_w, &dst_tile_h);
+
+   /* For Tiled surfaces, the pitch has to be a multiple of the Tile width
+    * (X direction width of the Tile). This is ensured while allocating the
+    * buffer object.
+    */
+   assert(src->surf.tiling == ISL_TILING_LINEAR || (src_pitch % src_tile_w) == 0);
+   assert(dst->surf.tiling == ISL_TILING_LINEAR || (dst_pitch % dst_tile_w) == 0);
+
+   /* For big formats (such as floating point), do the copy using 16 or
+    * 32bpp and multiply the coordinates.
+    */
+   if (cpp > 4) {
+      if (cpp % 4 == 2) {
+         dst_x *= cpp / 2;
+         dst_x2 *= cpp / 2;
+         src_x *= cpp / 2;
+         cpp = 2;
+      } else {
+         assert(cpp % 4 == 0);
+         dst_x *= cpp / 4;
+         dst_x2 *= cpp / 4;
+         src_x *= cpp / 4;
+         cpp = 4;
+      }
+   }
+
+   /* Blit pitch must be dword-aligned.  Otherwise, the hardware appears to drop
+    * the low bits.  Offsets must be naturally aligned.
+    */
+   if (src_pitch % 4 != 0 || src_offset % cpp != 0 ||
+       dst_pitch % 4 != 0 || dst_offset % cpp != 0)
+     return false;
+
+   /* For tiled source and destination, pitch value should be specified
+    * as a number of Dwords.
+    */
+   if (dst->surf.tiling != ISL_TILING_LINEAR)
+      dst_pitch /= 4;
+
+   if (src->surf.tiling != ISL_TILING_LINEAR)
+      src_pitch /= 4;
+
+   assert(cpp <= 4);
+   crocus_emit_cmd(batch, GENX(XY_SRC_COPY_BLT), xyblt) {
+      xyblt.RasterOperation = 0xCC;
+      xyblt.DestinationTilingEnable = dst->surf.tiling != ISL_TILING_LINEAR;
+      xyblt.SourceTilingEnable = src->surf.tiling != ISL_TILING_LINEAR;
+      xyblt.SourceBaseAddress = ro_bo(src->bo, src_offset);
+      xyblt.DestinationBaseAddress = rw_bo(dst->bo, dst_offset);
+      xyblt.ColorDepth = color_depth_for_cpp(cpp);
+      xyblt._32bppByteMask = cpp == 4 ? 0x3 : 0x1;
+      xyblt.DestinationX1Coordinate = dst_x;
+      xyblt.DestinationY1Coordinate = dst_y;
+      xyblt.DestinationX2Coordinate = dst_x2;
+      xyblt.DestinationY2Coordinate = dst_y2;
+      xyblt.DestinationPitch = dst_pitch;
+      xyblt.SourceX1Coordinate = src_x;
+      xyblt.SourceY1Coordinate = src_y;
+      xyblt.SourcePitch = src_pitch;
+   };
+
+   crocus_emit_mi_flush(batch);
+   return true;
+}
+
+static bool crocus_emit_blt(struct crocus_batch *batch,
+                            struct crocus_resource *src,
+                            struct crocus_resource *dst,
+                            unsigned dst_level,
+                            unsigned dst_x, unsigned dst_y,
+                            unsigned dst_z,
+                            unsigned src_level,
+                            const struct pipe_box *src_box)
+{
+   const struct isl_format_layout *src_fmtl = isl_format_get_layout(src->surf.format);
+   unsigned src_cpp = src_fmtl->bpb / 8;
+   const struct isl_format_layout *dst_fmtl = isl_format_get_layout(dst->surf.format);
+   const unsigned dst_cpp = dst_fmtl->bpb / 8;
+   uint16_t src_x, src_y;
+   uint32_t src_image_x, src_image_y, dst_image_x, dst_image_y;
+   uint32_t src_width = src_box->width, src_height = src_box->height;
+
+   /* gen4/5 can't handle Y tiled blits. */
+   if (src->surf.tiling == ISL_TILING_Y0 || dst->surf.tiling == ISL_TILING_Y0)
+      return false;
+
+   if (src->surf.format != dst->surf.format)
+      return false;
+
+   if (src_cpp != dst_cpp)
+      return false;
+
+   src_x = src_box->x;
+   src_y = src_box->y;
+
+   assert(src_cpp == dst_cpp);
+
+   crocus_resource_get_image_offset(src, src_level, src_box->z, &src_image_x,
+                                    &src_image_y);
+   if (util_format_is_compressed(src->base.b.format)) {
+      int bw = util_format_get_blockwidth(src->base.b.format);
+      int bh = util_format_get_blockheight(src->base.b.format);
+      assert(src_x % bw == 0);
+      assert(src_y % bh == 0);
+      src_x /= (int)bw;
+      src_y /= (int)bh;
+      src_width = DIV_ROUND_UP(src_width, (int)bw);
+      src_height = DIV_ROUND_UP(src_height, (int)bh);
+   }
+
+   crocus_resource_get_image_offset(dst, dst_level, dst_z, &dst_image_x,
+                                    &dst_image_y);
+   if (util_format_is_compressed(dst->base.b.format)) {
+      int bw = util_format_get_blockwidth(dst->base.b.format);
+      int bh = util_format_get_blockheight(dst->base.b.format);
+      assert(dst_x % bw == 0);
+      assert(dst_y % bh == 0);
+      dst_x /= (int)bw;
+      dst_y /= (int)bh;
+   }
+   src_x += src_image_x;
+   src_y += src_image_y;
+   dst_x += dst_image_x;
+   dst_y += dst_image_y;
+
+   /* According to the Ivy Bridge PRM, Vol1 Part4, section 1.2.1.2 (Graphics
+    * Data Size Limitations):
+    *
+    *    The BLT engine is capable of transferring very large quantities of
+    *    graphics data. Any graphics data read from and written to the
+    *    destination is permitted to represent a number of pixels that
+    *    occupies up to 65,536 scan lines and up to 32,768 bytes per scan line
+    *    at the destination. The maximum number of pixels that may be
+    *    represented per scan line’s worth of graphics data depends on the
+    *    color depth.
+    *
+    * The blitter's pitch is a signed 16-bit integer, but measured in bytes
+    * for linear surfaces and DWords for tiled surfaces.  So the maximum
+    * pitch is 32k linear and 128k tiled.
+    */
+   if (crocus_resource_blt_pitch(src) >= 32768 ||
+       crocus_resource_blt_pitch(dst) >= 32768) {
+      return false;
+   }
+
+   /* We need to split the blit into chunks that each fit within the blitter's
+    * restrictions.  We can't use a chunk size of 32768 because we need to
+    * ensure that src_tile_x + chunk_size fits.  We choose 16384 because it's
+    * a nice round power of two, big enough that performance won't suffer, and
+    * small enough to guarantee everything fits.
+    */
+   const uint32_t max_chunk_size = 16384;
+
+   for (uint32_t chunk_x = 0; chunk_x < src_width; chunk_x += max_chunk_size) {
+      for (uint32_t chunk_y = 0; chunk_y < src_height; chunk_y += max_chunk_size) {
+         const uint32_t chunk_w = MIN2(max_chunk_size, src_width - chunk_x);
+         const uint32_t chunk_h = MIN2(max_chunk_size, src_height - chunk_y);
+
+         uint64_t src_offset;
+         uint32_t src_tile_x, src_tile_y;
+         ASSERTED uint32_t z_offset_el, array_offset;
+         isl_tiling_get_intratile_offset_el(src->surf.tiling, src->surf.dim,
+                                            src->surf.msaa_layout,
+                                            src_cpp * 8, src->surf.samples,
+                                            src->surf.row_pitch_B,
+                                            src->surf.array_pitch_el_rows,
+                                            src_x + chunk_x, src_y + chunk_y, 0, 0,
+                                            &src_offset,
+                                            &src_tile_x, &src_tile_y,
+                                            &z_offset_el, &array_offset);
+         assert(z_offset_el == 0);
+         assert(array_offset == 0);
+
+         uint64_t dst_offset;
+         uint32_t dst_tile_x, dst_tile_y;
+         isl_tiling_get_intratile_offset_el(dst->surf.tiling, dst->surf.dim,
+                                            dst->surf.msaa_layout,
+                                            dst_cpp * 8, dst->surf.samples,
+                                            dst->surf.row_pitch_B,
+                                            dst->surf.array_pitch_el_rows,
+                                            dst_x + chunk_x, dst_y + chunk_y, 0, 0,
+                                            &dst_offset,
+                                            &dst_tile_x, &dst_tile_y,
+                                            &z_offset_el, &array_offset);
+         assert(z_offset_el == 0);
+         assert(array_offset == 0);
+         if (!emit_copy_blt(batch, src, dst,
+                            src_cpp, src->surf.row_pitch_B,
+                            src_offset,
+                            dst->surf.row_pitch_B, dst_offset,
+                            src_tile_x, src_tile_y,
+                            dst_tile_x, dst_tile_y,
+                            chunk_w, chunk_h)) {
+            return false;
+         }
+      }
+   }
+
+   if (util_format_has_alpha1(src->base.b.format) &&
+       util_format_has_alpha(dst->base.b.format))
+      blt_set_alpha_to_one(batch, dst, 0, 0, src_width, src_height);
+   return true;
+}
+
+static bool crocus_blit_blt(struct crocus_batch *batch,
+                            const struct pipe_blit_info *info)
+{
+   if (!validate_blit_for_blt(batch, info))
+      return false;
+
+   return crocus_emit_blt(batch,
+                          (struct crocus_resource *)info->src.resource,
+                          (struct crocus_resource *)info->dst.resource,
+                          info->dst.level,
+                          info->dst.box.x,
+                          info->dst.box.y,
+                          info->dst.box.z,
+                          info->src.level,
+                          &info->src.box);
+}
+
+
+static bool crocus_copy_region_blt(struct crocus_batch *batch,
+                                   struct crocus_resource *dst,
+                                   unsigned dst_level,
+                                   unsigned dstx, unsigned dsty, unsigned dstz,
+                                   struct crocus_resource *src,
+                                   unsigned src_level,
+                                   const struct pipe_box *src_box)
+{
+   if (dst->base.b.target == PIPE_BUFFER || src->base.b.target == PIPE_BUFFER)
+      return false;
+   return crocus_emit_blt(batch,
+                          src,
+                          dst,
+                          dst_level,
+                          dstx, dsty, dstz,
+                          src_level,
+                          src_box);
+}
+#endif
+
+void
+genX(crocus_init_blt)(struct crocus_screen *screen)
+{
+#if GFX_VER <= 5
+   screen->vtbl.blit_blt = crocus_blit_blt;
+   screen->vtbl.copy_region_blt = crocus_copy_region_blt;
+#else
+   screen->vtbl.blit_blt = NULL;
+   screen->vtbl.copy_region_blt = NULL;
+#endif
+}

src/gallium/drivers/crocus/crocus_bufmgr.h

@@ -0,0 +1,356 @@
+/*
+ * Copyright © 2017 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifndef CROCUS_BUFMGR_H
+#define CROCUS_BUFMGR_H
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include "util/macros.h"
+#include "util/u_atomic.h"
+#include "util/list.h"
+#include "pipe/p_defines.h"
+
+struct crocus_batch;
+struct intel_device_info;
+struct util_debug_callback;
+
+#define CROCUS_BINDER_SIZE (64 * 1024)
+#define CROCUS_MAX_BINDERS 100
+
+struct crocus_bo {
+   /**
+    * Size in bytes of the buffer object.
+    *
+    * The size may be larger than the size originally requested for the
+    * allocation, such as being aligned to page size.
+    */
+   uint64_t size;
+
+   /** Buffer manager context associated with this buffer object */
+   struct crocus_bufmgr *bufmgr;
+
+   /** The GEM handle for this buffer object. */
+   uint32_t gem_handle;
+
+   /**
+    * Virtual address of the buffer inside the PPGTT (Per-Process Graphics
+    * Translation Table).
+    *
+    * Although each hardware context has its own VMA, we assign BO's to the
+    * same address in all contexts, for simplicity.
+    */
+   uint64_t gtt_offset;
+
+   /**
+    * The validation list index for this buffer, or -1 when not in a batch.
+    * Note that a single buffer may be in multiple batches (contexts), and
+    * this is a global field, which refers to the last batch using the BO.
+    * It should not be considered authoritative, but can be used to avoid a
+    * linear walk of the validation list in the common case by guessing that
+    * exec_bos[bo->index] == bo and confirming whether that's the case.
+    *
+    * XXX: this is not ideal now that we have more than one batch per context,
+    * XXX: as the index will flop back and forth between the render index and
+    * XXX: compute index...
+    */
+   unsigned index;
+
+   /**
+    * Boolean of whether the GPU is definitely not accessing the buffer.
+    *
+    * This is only valid when reusable, since non-reusable
+    * buffers are those that have been shared with other
+    * processes, so we don't know their state.
+    */
+   bool idle;
+
+   int refcount;
+   const char *name;
+
+   uint64_t kflags;
+
+   /**
+    * Kenel-assigned global name for this object
+    *
+    * List contains both flink named and prime fd'd objects
+    */
+   unsigned global_name;
+
+   /**
+    * Current tiling mode
+    */
+   uint32_t tiling_mode;
+   uint32_t swizzle_mode;
+   uint32_t stride;
+
+   time_t free_time;
+
+   /** Mapped address for the buffer, saved across map/unmap cycles */
+   void *map_cpu;
+   /** GTT virtual address for the buffer, saved across map/unmap cycles */
+   void *map_gtt;
+   /** WC CPU address for the buffer, saved across map/unmap cycles */
+   void *map_wc;
+
+   /** BO cache list */
+   struct list_head head;
+
+   /** List of GEM handle exports of this buffer (bo_export) */
+   struct list_head exports;
+
+   /**
+    * Boolean of whether this buffer can be re-used
+    */
+   bool reusable;
+
+   /**
+    * Boolean of whether this buffer has been shared with an external client.
+    */
+   bool external;
+
+   /**
+    * Boolean of whether this buffer is cache coherent
+    */
+   bool cache_coherent;
+
+   /**
+    * Boolean of whether this buffer points into user memory
+    */
+   bool userptr;
+
+   /**
+    * Boolean of if this is used for scanout.
+    */
+   bool scanout;
+
+   /** Pre-computed hash using _mesa_hash_pointer for cache tracking sets */
+   uint32_t hash;
+};
+
+#define BO_ALLOC_ZEROED   (1 << 0)
+#define BO_ALLOC_COHERENT (1 << 1)
+#define BO_ALLOC_SCANOUT  (1 << 2)
+
+/**
+ * Allocate a buffer object.
+ *
+ * Buffer objects are not necessarily initially mapped into CPU virtual
+ * address space or graphics device aperture.  They must be mapped
+ * using crocus_bo_map() to be used by the CPU.
+ */
+struct crocus_bo *crocus_bo_alloc(struct crocus_bufmgr *bufmgr,
+                                  const char *name, uint64_t size);
+
+/**
+ * Allocate a tiled buffer object.
+ *
+ * Alignment for tiled objects is set automatically; the 'flags'
+ * argument provides a hint about how the object will be used initially.
+ *
+ * Valid tiling formats are:
+ *  I915_TILING_NONE
+ *  I915_TILING_X
+ *  I915_TILING_Y
+ */
+struct crocus_bo *crocus_bo_alloc_tiled(struct crocus_bufmgr *bufmgr,
+                                        const char *name, uint64_t size,
+                                        uint32_t alignment,
+                                        uint32_t tiling_mode, uint32_t pitch,
+                                        unsigned flags);
+
+struct crocus_bo *crocus_bo_create_userptr(struct crocus_bufmgr *bufmgr,
+                                           const char *name, void *ptr,
+                                           size_t size);
+
+/** Takes a reference on a buffer object */
+static inline void
+crocus_bo_reference(struct crocus_bo *bo)
+{
+   p_atomic_inc(&bo->refcount);
+}
+
+static inline int
+atomic_add_unless(int *v, int add, int unless)
+{
+   int c, old;
+   c = p_atomic_read(v);
+   while (c != unless && (old = p_atomic_cmpxchg(v, c, c + add)) != c)
+      c = old;
+   return c == unless;
+}
+
+void __crocus_bo_unreference(struct crocus_bo *bo);
+
+/**
+ * Releases a reference on a buffer object, freeing the data if
+ * no references remain.
+ */
+static inline void crocus_bo_unreference(struct crocus_bo *bo)
+{
+   if (bo == NULL)
+      return;
+
+   assert(p_atomic_read(&bo->refcount) > 0);
+
+   if (atomic_add_unless(&bo->refcount, -1, 1)) {
+      __crocus_bo_unreference(bo);
+   }
+}
+
+#define MAP_READ          PIPE_MAP_READ
+#define MAP_WRITE         PIPE_MAP_WRITE
+#define MAP_ASYNC         PIPE_MAP_UNSYNCHRONIZED
+#define MAP_PERSISTENT    PIPE_MAP_PERSISTENT
+#define MAP_COHERENT      PIPE_MAP_COHERENT
+/* internal */
+#define MAP_INTERNAL_MASK (0xff << 24)
+#define MAP_RAW           (0x01 << 24)
+
+#define MAP_FLAGS         (MAP_READ | MAP_WRITE | MAP_ASYNC | \
+                           MAP_PERSISTENT | MAP_COHERENT | MAP_INTERNAL_MASK)
+
+/**
+ * Maps the buffer into userspace.
+ *
+ * This function will block waiting for any existing execution on the
+ * buffer to complete, first.  The resulting mapping is returned.
+ */
+MUST_CHECK void *crocus_bo_map(struct util_debug_callback *dbg,
+                             struct crocus_bo *bo, unsigned flags);
+
+/**
+ * Reduces the refcount on the userspace mapping of the buffer
+ * object.
+ */
+static inline int crocus_bo_unmap(struct crocus_bo *bo) { return 0; }
+
+/**
+ * Waits for rendering to an object by the GPU to have completed.
+ *
+ * This is not required for any access to the BO by bo_map,
+ * bo_subdata, etc.  It is merely a way for the driver to implement
+ * glFinish.
+ */
+void crocus_bo_wait_rendering(struct crocus_bo *bo);
+
+/**
+ * Unref a buffer manager instance.
+ */
+void crocus_bufmgr_unref(struct crocus_bufmgr *bufmgr);
+
+/**
+ * Get the current tiling (and resulting swizzling) mode for the bo.
+ *
+ * \param buf Buffer to get tiling mode for
+ * \param tiling_mode returned tiling mode
+ * \param swizzle_mode returned swizzling mode
+ */
+int crocus_bo_get_tiling(struct crocus_bo *bo, uint32_t *tiling_mode,
+                         uint32_t *swizzle_mode);
+
+/**
+ * Create a visible name for a buffer which can be used by other apps
+ *
+ * \param buf Buffer to create a name for
+ * \param name Returned name
+ */
+int crocus_bo_flink(struct crocus_bo *bo, uint32_t *name);
+
+/**
+ * Is this buffer shared with external clients (exported)?
+ */
+static inline bool
+crocus_bo_is_external(const struct crocus_bo *bo)
+{
+   return bo->external;
+}
+
+/**
+ * Returns 1 if mapping the buffer for write could cause the process
+ * to block, due to the object being active in the GPU.
+ */
+int crocus_bo_busy(struct crocus_bo *bo);
+
+/**
+ * Specify the volatility of the buffer.
+ * \param bo Buffer to create a name for
+ * \param madv The purgeable status
+ *
+ * Use I915_MADV_DONTNEED to mark the buffer as purgeable, and it will be
+ * reclaimed under memory pressure. If you subsequently require the buffer,
+ * then you must pass I915_MADV_WILLNEED to mark the buffer as required.
+ *
+ * Returns 1 if the buffer was retained, or 0 if it was discarded whilst
+ * marked as I915_MADV_DONTNEED.
+ */
+int crocus_bo_madvise(struct crocus_bo *bo, int madv);
+
+struct crocus_bufmgr *
+crocus_bufmgr_get_for_fd(struct intel_device_info *devinfo, int fd,
+                         bool bo_reuse);
+int crocus_bufmgr_get_fd(struct crocus_bufmgr *bufmgr);
+
+struct crocus_bo *crocus_bo_gem_create_from_name(struct crocus_bufmgr *bufmgr,
+                                                 const char *name,
+                                                 unsigned handle);
+
+int crocus_bo_wait(struct crocus_bo *bo, int64_t timeout_ns);
+
+uint32_t crocus_create_hw_context(struct crocus_bufmgr *bufmgr);
+uint32_t crocus_clone_hw_context(struct crocus_bufmgr *bufmgr, uint32_t ctx_id);
+
+#define CROCUS_CONTEXT_LOW_PRIORITY    ((I915_CONTEXT_MIN_USER_PRIORITY - 1) / 2)
+#define CROCUS_CONTEXT_MEDIUM_PRIORITY (I915_CONTEXT_DEFAULT_PRIORITY)
+#define CROCUS_CONTEXT_HIGH_PRIORITY   ((I915_CONTEXT_MAX_USER_PRIORITY + 1) / 2)
+
+int crocus_hw_context_set_priority(struct crocus_bufmgr *bufmgr,
+                                   uint32_t ctx_id, int priority);
+
+void crocus_destroy_hw_context(struct crocus_bufmgr *bufmgr, uint32_t ctx_id);
+
+int crocus_bo_export_dmabuf(struct crocus_bo *bo, int *prime_fd);
+struct crocus_bo *crocus_bo_import_dmabuf(struct crocus_bufmgr *bufmgr,
+                                          int prime_fd, uint64_t modifier);
+struct crocus_bo *crocus_bo_import_dmabuf_no_mods(struct crocus_bufmgr *bufmgr,
+                                                  int prime_fd);
+
+/**
+ * Exports a bo as a GEM handle into a given DRM file descriptor
+ * \param bo Buffer to export
+ * \param drm_fd File descriptor where the new handle is created
+ * \param out_handle Pointer to store the new handle
+ *
+ * Returns 0 if the buffer was successfully exported, a non zero error code
+ * otherwise.
+ */
+int crocus_bo_export_gem_handle_for_device(struct crocus_bo *bo, int drm_fd,
+                                           uint32_t *out_handle);
+
+uint32_t crocus_bo_export_gem_handle(struct crocus_bo *bo);
+
+int drm_ioctl(int fd, unsigned long request, void *arg);
+
+#endif /* CROCUS_BUFMGR_H */

src/gallium/drivers/crocus/crocus_clear.c

@@ -0,0 +1,829 @@
+/*
+ * Copyright © 2017 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <stdio.h>
+#include <errno.h>
+#include "pipe/p_defines.h"
+#include "pipe/p_state.h"
+#include "pipe/p_context.h"
+#include "pipe/p_screen.h"
+#include "util/u_inlines.h"
+#include "util/u_surface.h"
+#include "util/format/u_format.h"
+#include "util/u_upload_mgr.h"
+#include "util/ralloc.h"
+#include "crocus_context.h"
+#include "crocus_resource.h"
+#include "crocus_screen.h"
+#include "intel/compiler/elk/elk_compiler.h"
+#include "util/format_srgb.h"
+
+static bool
+crocus_is_color_fast_clear_compatible(struct crocus_context *ice,
+                                      enum isl_format format,
+                                      const union isl_color_value color)
+{
+   if (isl_format_has_int_channel(format)) {
+      perf_debug(&ice->dbg, "Integer fast clear not enabled for %s",
+                 isl_format_get_name(format));
+      return false;
+   }
+
+   for (int i = 0; i < 4; i++) {
+      if (!isl_format_has_color_component(format, i)) {
+         continue;
+      }
+
+      if (color.f32[i] != 0.0f && color.f32[i] != 1.0f) {
+         return false;
+      }
+   }
+
+   return true;
+}
+
+static bool
+can_fast_clear_color(struct crocus_context *ice,
+                     struct pipe_resource *p_res,
+                     unsigned level,
+                     const struct pipe_box *box,
+                     bool render_condition_enabled,
+                     enum isl_format format,
+                     enum isl_format render_format,
+                     union isl_color_value color)
+{
+   struct crocus_resource *res = (void *) p_res;
+
+   if (INTEL_DEBUG(DEBUG_NO_FAST_CLEAR))
+      return false;
+
+   if (!isl_aux_usage_has_fast_clears(res->aux.usage))
+      return false;
+
+   /* Check for partial clear */
+   if (box->x > 0 || box->y > 0 ||
+       box->width < u_minify(p_res->width0, level) ||
+       box->height < u_minify(p_res->height0, level)) {
+      return false;
+   }
+
+   /* Avoid conditional fast clears to maintain correct tracking of the aux
+    * state (see iris_resource_finish_write for more info). Note that partial
+    * fast clears (if they existed) would not pose a problem with conditional
+    * rendering.
+    */
+   if (render_condition_enabled &&
+       ice->state.predicate == CROCUS_PREDICATE_STATE_USE_BIT) {
+      return false;
+   }
+
+   /* We store clear colors as floats or uints as needed.  If there are
+    * texture views in play, the formats will not properly be respected
+    * during resolves because the resolve operations only know about the
+    * resource and not the renderbuffer.
+    */
+   if (!crocus_render_formats_color_compatible(render_format, res->surf.format,
+                                             color)) {
+      return false;
+   }
+
+   /* XXX: if (irb->mt->supports_fast_clear)
+    * see intel_miptree_create_for_dri_image()
+    */
+
+   if (!crocus_is_color_fast_clear_compatible(ice, format, color))
+      return false;
+
+   return true;
+}
+
+static union isl_color_value
+convert_fast_clear_color(struct crocus_context *ice,
+                         struct crocus_resource *res,
+                         enum isl_format render_format,
+                         const union isl_color_value color)
+{
+   union isl_color_value override_color = color;
+   struct pipe_resource *p_res = (void *) res;
+
+   const enum pipe_format format = p_res->format;
+   const struct util_format_description *desc =
+      util_format_description(format);
+   unsigned colormask = util_format_colormask(desc);
+
+   if (util_format_is_intensity(format) ||
+       util_format_is_luminance(format) ||
+       util_format_is_luminance_alpha(format)) {
+      override_color.u32[1] = override_color.u32[0];
+      override_color.u32[2] = override_color.u32[0];
+      if (util_format_is_intensity(format))
+         override_color.u32[3] = override_color.u32[0];
+   } else {
+      for (int chan = 0; chan < 3; chan++) {
+         if (!(colormask & (1 << chan)))
+            override_color.u32[chan] = 0;
+      }
+   }
+
+   if (util_format_is_unorm(format)) {
+      for (int i = 0; i < 4; i++)
+         override_color.f32[i] = CLAMP(override_color.f32[i], 0.0f, 1.0f);
+   } else if (util_format_is_snorm(format)) {
+      for (int i = 0; i < 4; i++)
+         override_color.f32[i] = CLAMP(override_color.f32[i], -1.0f, 1.0f);
+   } else if (util_format_is_pure_uint(format)) {
+      for (int i = 0; i < 4; i++) {
+         unsigned bits = util_format_get_component_bits(
+            format, UTIL_FORMAT_COLORSPACE_RGB, i);
+         if (bits < 32) {
+            uint32_t max = (1u << bits) - 1;
+            override_color.u32[i] = MIN2(override_color.u32[i], max);
+         }
+      }
+   } else if (util_format_is_pure_sint(format)) {
+      for (int i = 0; i < 4; i++) {
+         unsigned bits = util_format_get_component_bits(
+            format, UTIL_FORMAT_COLORSPACE_RGB, i);
+         if (bits < 32) {
+            int32_t max = (1 << (bits - 1)) - 1;
+            int32_t min = -(1 << (bits - 1));
+            override_color.i32[i] = CLAMP(override_color.i32[i], min, max);
+         }
+      }
+   } else if (format == PIPE_FORMAT_R11G11B10_FLOAT ||
+              format == PIPE_FORMAT_R9G9B9E5_FLOAT) {
+      /* these packed float formats only store unsigned values */
+      for (int i = 0; i < 4; i++)
+         override_color.f32[i] = MAX2(override_color.f32[i], 0.0f);
+   }
+
+   if (!(colormask & 1 << 3)) {
+      if (util_format_is_pure_integer(format))
+         override_color.u32[3] = 1;
+      else
+         override_color.f32[3] = 1.0f;
+   }
+
+   /* Handle linear to SRGB conversion */
+   if (isl_format_is_srgb(render_format)) {
+      for (int i = 0; i < 3; i++) {
+         override_color.f32[i] =
+            util_format_linear_to_srgb_float(override_color.f32[i]);
+      }
+   }
+
+   return override_color;
+}
+
+static void
+fast_clear_color(struct crocus_context *ice,
+                 struct crocus_resource *res,
+                 unsigned level,
+                 const struct pipe_box *box,
+                 enum isl_format format,
+                 union isl_color_value color,
+                 enum blorp_batch_flags blorp_flags)
+{
+   struct crocus_batch *batch = &ice->batches[CROCUS_BATCH_RENDER];
+   struct crocus_screen *screen = batch->screen;
+   struct pipe_resource *p_res = (void *) res;
+
+   color = convert_fast_clear_color(ice, res, format, color);
+
+   bool color_changed = !!memcmp(&res->aux.clear_color, &color,
+                                 sizeof(color));
+
+   if (color_changed) {
+      /* If we are clearing to a new clear value, we need to resolve fast
+       * clears from other levels/layers first, since we can't have different
+       * levels/layers with different fast clear colors.
+       */
+      for (unsigned res_lvl = 0; res_lvl < res->surf.levels; res_lvl++) {
+         const unsigned level_layers =
+            crocus_get_num_logical_layers(res, res_lvl);
+         for (unsigned layer = 0; layer < level_layers; layer++) {
+            if (res_lvl == level &&
+                layer >= box->z &&
+                layer < box->z + box->depth) {
+               /* We're going to clear this layer anyway.  Leave it alone. */
+               continue;
+            }
+
+            enum isl_aux_state aux_state =
+               crocus_resource_get_aux_state(res, res_lvl, layer);
+
+            if (aux_state != ISL_AUX_STATE_CLEAR &&
+                aux_state != ISL_AUX_STATE_PARTIAL_CLEAR &&
+                aux_state != ISL_AUX_STATE_COMPRESSED_CLEAR) {
+               /* This slice doesn't have any fast-cleared bits. */
+               continue;
+            }
+
+            /* If we got here, then the level may have fast-clear bits that use
+             * the old clear value.  We need to do a color resolve to get rid
+             * of their use of the clear color before we can change it.
+             * Fortunately, few applications ever change their clear color at
+             * different levels/layers, so this shouldn't happen often.
+             */
+            crocus_resource_prepare_access(ice, res,
+                                           res_lvl, 1, layer, 1,
+                                           res->aux.usage,
+                                           false);
+            perf_debug(&ice->dbg,
+                       "Resolving resource (%p) level %d, layer %d: color changing from "
+                       "(%0.2f, %0.2f, %0.2f, %0.2f) to "
+                       "(%0.2f, %0.2f, %0.2f, %0.2f)\n",
+                       res, res_lvl, layer,
+                       res->aux.clear_color.f32[0],
+                       res->aux.clear_color.f32[1],
+                       res->aux.clear_color.f32[2],
+                       res->aux.clear_color.f32[3],
+                       color.f32[0], color.f32[1], color.f32[2], color.f32[3]);
+         }
+      }
+   }
+
+   crocus_resource_set_clear_color(ice, res, color);
+
+   /* If the buffer is already in ISL_AUX_STATE_CLEAR, and the color hasn't
+    * changed, the clear is redundant and can be skipped.
+    */
+   const enum isl_aux_state aux_state =
+      crocus_resource_get_aux_state(res, level, box->z);
+   if (!color_changed && box->depth == 1 && aux_state == ISL_AUX_STATE_CLEAR)
+      return;
+
+   /* Ivybrigde PRM Vol 2, Part 1, "11.7 MCS Buffer for Render Target(s)":
+    *
+    *    "Any transition from any value in {Clear, Render, Resolve} to a
+    *    different value in {Clear, Render, Resolve} requires end of pipe
+    *    synchronization."
+    *
+    * In other words, fast clear ops are not properly synchronized with
+    * other drawing.  We need to use a PIPE_CONTROL to ensure that the
+    * contents of the previous draw hit the render target before we resolve
+    * and again afterwards to ensure that the resolve is complete before we
+    * do any more regular drawing.
+    */
+   crocus_emit_end_of_pipe_sync(batch,
+                                "fast clear: pre-flush",
+                                PIPE_CONTROL_RENDER_TARGET_FLUSH);
+
+   struct blorp_batch blorp_batch;
+   blorp_batch_init(&ice->blorp, &blorp_batch, batch, blorp_flags);
+
+   struct blorp_surf surf;
+   crocus_blorp_surf_for_resource(&screen->vtbl, &batch->screen->isl_dev, &surf,
+                                  p_res, res->aux.usage, level, true);
+
+   /* In newer gens (> 9), the hardware will do a linear -> sRGB conversion of
+    * the clear color during the fast clear, if the surface format is of sRGB
+    * type. We use the linear version of the surface format here to prevent
+    * that from happening, since we already do our own linear -> sRGB
+    * conversion in convert_fast_clear_color().
+    */
+   blorp_fast_clear(&blorp_batch, &surf, isl_format_srgb_to_linear(format),
+                    ISL_SWIZZLE_IDENTITY,
+                    level, box->z, box->depth,
+                    box->x, box->y, box->x + box->width,
+                    box->y + box->height);
+   blorp_batch_finish(&blorp_batch);
+   crocus_emit_end_of_pipe_sync(batch,
+                                "fast clear: post flush",
+                                PIPE_CONTROL_RENDER_TARGET_FLUSH);
+
+   crocus_resource_set_aux_state(ice, res, level, box->z,
+                                 box->depth, ISL_AUX_STATE_CLEAR);
+   ice->state.stage_dirty |= CROCUS_ALL_STAGE_DIRTY_BINDINGS;
+   return;
+}
+
+static void
+clear_color(struct crocus_context *ice,
+            struct pipe_resource *p_res,
+            unsigned level,
+            const struct pipe_box *box,
+            bool render_condition_enabled,
+            enum isl_format format,
+            struct isl_swizzle swizzle,
+            union isl_color_value color)
+{
+   struct crocus_resource *res = (void *) p_res;
+   struct crocus_batch *batch = &ice->batches[CROCUS_BATCH_RENDER];
+   struct crocus_screen *screen = batch->screen;
+   const struct intel_device_info *devinfo = &batch->screen->devinfo;
+   enum blorp_batch_flags blorp_flags = 0;
+
+   if (render_condition_enabled) {
+      if (!crocus_check_conditional_render(ice))
+         return;
+
+      if (ice->state.predicate == CROCUS_PREDICATE_STATE_USE_BIT)
+         blorp_flags |= BLORP_BATCH_PREDICATE_ENABLE;
+   }
+
+   if (p_res->target == PIPE_BUFFER)
+      util_range_add(&res->base.b, &res->valid_buffer_range, box->x, box->x + box->width);
+
+   crocus_batch_maybe_flush(batch, 1500);
+
+   bool can_fast_clear = can_fast_clear_color(ice, p_res, level, box,
+                                              render_condition_enabled,
+                                              res->surf.format, format, color);
+   if (can_fast_clear) {
+      fast_clear_color(ice, res, level, box, format, color,
+                       blorp_flags);
+      return;
+   }
+
+   enum isl_aux_usage aux_usage =
+      crocus_resource_render_aux_usage(ice, res, level, format, false);
+
+   crocus_resource_prepare_render(ice, res, level,
+                                  box->z, box->depth, aux_usage);
+
+   struct blorp_surf surf;
+   crocus_blorp_surf_for_resource(&screen->vtbl, &batch->screen->isl_dev, &surf,
+                                  p_res, aux_usage, level, true);
+
+   struct blorp_batch blorp_batch;
+   blorp_batch_init(&ice->blorp, &blorp_batch, batch, blorp_flags);
+
+   if (!isl_format_supports_rendering(devinfo, format) &&
+       isl_format_is_rgbx(format))
+      format = isl_format_rgbx_to_rgba(format);
+
+   blorp_clear(&blorp_batch, &surf, format, swizzle,
+               level, box->z, box->depth, box->x, box->y,
+               box->x + box->width, box->y + box->height,
+               color, 0 /* color_write_disable */);
+
+   blorp_batch_finish(&blorp_batch);
+   crocus_flush_and_dirty_for_history(ice, batch, res,
+                                      PIPE_CONTROL_RENDER_TARGET_FLUSH,
+                                      "cache history: post color clear");
+
+   crocus_resource_finish_render(ice, res, level,
+                                 box->z, box->depth, aux_usage);
+}
+
+static bool
+can_fast_clear_depth(struct crocus_context *ice,
+                     struct crocus_resource *res,
+                     unsigned level,
+                     const struct pipe_box *box,
+                     bool render_condition_enabled,
+                     float depth)
+{
+   struct pipe_resource *p_res = (void *) res;
+   struct pipe_context *ctx = (void *) ice;
+   struct crocus_screen *screen = (void *) ctx->screen;
+   const struct intel_device_info *devinfo = &screen->devinfo;
+
+   if (devinfo->ver < 6)
+      return false;
+
+   if (INTEL_DEBUG(DEBUG_NO_FAST_CLEAR))
+      return false;
+
+   /* Check for partial clears */
+   if (box->x > 0 || box->y > 0 ||
+       box->width < u_minify(p_res->width0, level) ||
+       box->height < u_minify(p_res->height0, level)) {
+      return false;
+   }
+
+   /* Avoid conditional fast clears to maintain correct tracking of the aux
+    * state (see iris_resource_finish_write for more info). Note that partial
+    * fast clears would not pose a problem with conditional rendering.
+    */
+   if (render_condition_enabled &&
+       ice->state.predicate == CROCUS_PREDICATE_STATE_USE_BIT) {
+      return false;
+   }
+
+   if (!crocus_resource_level_has_hiz(res, level))
+      return false;
+
+   if (res->base.b.format == PIPE_FORMAT_Z16_UNORM) {
+      /* From the Sandy Bridge PRM, volume 2 part 1, page 314:
+       *
+       *     "[DevSNB+]: Several cases exist where Depth Buffer Clear cannot be
+       *      enabled (the legacy method of clearing must be performed):
+       *
+       *      - DevSNB{W/A}]: When depth buffer format is D16_UNORM and the
+       *        width of the map (LOD0) is not multiple of 16, fast clear
+       *        optimization must be disabled.
+       */
+      if (devinfo->ver == 6 &&
+          (u_minify(res->surf.phys_level0_sa.width,
+                    level) % 16) != 0)
+         return false;
+   }
+   return true;
+}
+
+static void
+fast_clear_depth(struct crocus_context *ice,
+                 struct crocus_resource *res,
+                 unsigned level,
+                 const struct pipe_box *box,
+                 float depth)
+{
+   struct crocus_batch *batch = &ice->batches[CROCUS_BATCH_RENDER];
+
+   /* If we're clearing to a new clear value, then we need to resolve any clear
+    * flags out of the HiZ buffer into the real depth buffer.
+    */
+   if (res->aux.clear_color.f32[0] != depth) {
+      for (unsigned res_level = 0; res_level < res->surf.levels; res_level++) {
+         if (!crocus_resource_level_has_hiz(res, res_level))
+            continue;
+
+         const unsigned level_layers =
+            crocus_get_num_logical_layers(res, res_level);
+         for (unsigned layer = 0; layer < level_layers; layer++) {
+            if (res_level == level &&
+                layer >= box->z &&
+                layer < box->z + box->depth) {
+               /* We're going to clear this layer anyway.  Leave it alone. */
+               continue;
+            }
+
+            enum isl_aux_state aux_state =
+               crocus_resource_get_aux_state(res, res_level, layer);
+
+            if (aux_state != ISL_AUX_STATE_CLEAR &&
+                aux_state != ISL_AUX_STATE_COMPRESSED_CLEAR) {
+               /* This slice doesn't have any fast-cleared bits. */
+               continue;
+            }
+
+            /* If we got here, then the level may have fast-clear bits that
+             * use the old clear value.  We need to do a depth resolve to get
+             * rid of their use of the clear value before we can change it.
+             * Fortunately, few applications ever change their depth clear
+             * value so this shouldn't happen often.
+             */
+            crocus_hiz_exec(ice, batch, res, res_level, layer, 1,
+                            ISL_AUX_OP_FULL_RESOLVE);
+            crocus_resource_set_aux_state(ice, res, res_level, layer, 1,
+                                          ISL_AUX_STATE_RESOLVED);
+         }
+      }
+      const union isl_color_value clear_value = { .f32 = {depth, } };
+      crocus_resource_set_clear_color(ice, res, clear_value);
+   }
+
+   for (unsigned l = 0; l < box->depth; l++) {
+      enum isl_aux_state aux_state =
+         crocus_resource_level_has_hiz(res, level) ?
+         crocus_resource_get_aux_state(res, level, box->z + l) :
+         ISL_AUX_STATE_AUX_INVALID;
+      if (aux_state != ISL_AUX_STATE_CLEAR) {
+         crocus_hiz_exec(ice, batch, res, level,
+                         box->z + l, 1, ISL_AUX_OP_FAST_CLEAR);
+      }
+   }
+
+   crocus_resource_set_aux_state(ice, res, level, box->z, box->depth,
+                                 ISL_AUX_STATE_CLEAR);
+   ice->state.dirty |= CROCUS_DIRTY_DEPTH_BUFFER;
+}
+
+static void
+clear_depth_stencil(struct crocus_context *ice,
+                    struct pipe_resource *p_res,
+                    unsigned level,
+                    const struct pipe_box *box,
+                    bool render_condition_enabled,
+                    bool clear_depth,
+                    bool clear_stencil,
+                    float depth,
+                    uint8_t stencil)
+{
+   struct crocus_resource *res = (void *) p_res;
+   struct crocus_batch *batch = &ice->batches[CROCUS_BATCH_RENDER];
+   struct crocus_screen *screen = batch->screen;
+   enum blorp_batch_flags blorp_flags = 0;
+
+   if (render_condition_enabled) {
+      if (!crocus_check_conditional_render(ice))
+         return;
+
+      if (ice->state.predicate == CROCUS_PREDICATE_STATE_USE_BIT)
+         blorp_flags |= BLORP_BATCH_PREDICATE_ENABLE;
+   }
+
+   crocus_batch_maybe_flush(batch, 1500);
+
+   struct crocus_resource *z_res;
+   struct crocus_resource *stencil_res;
+   struct blorp_surf z_surf;
+   struct blorp_surf stencil_surf;
+
+   crocus_get_depth_stencil_resources(&batch->screen->devinfo, p_res, &z_res, &stencil_res);
+   if (z_res && clear_depth &&
+       can_fast_clear_depth(ice, z_res, level, box, render_condition_enabled,
+                            depth)) {
+      fast_clear_depth(ice, z_res, level, box, depth);
+      crocus_flush_and_dirty_for_history(ice, batch, res, 0,
+                                         "cache history: post fast Z clear");
+      clear_depth = false;
+      z_res = NULL;
+   }
+
+   /* At this point, we might have fast cleared the depth buffer. So if there's
+    * no stencil clear pending, return early.
+    */
+   if (!(clear_depth || (clear_stencil && stencil_res))) {
+      return;
+   }
+
+   if (clear_depth && z_res) {
+      const enum isl_aux_usage aux_usage =
+         crocus_resource_render_aux_usage(ice, z_res, level, z_res->surf.format,
+                                          false);
+      crocus_resource_prepare_render(ice, z_res, level, box->z, box->depth,
+                                     aux_usage);
+      crocus_blorp_surf_for_resource(&screen->vtbl, &batch->screen->isl_dev,
+                                     &z_surf, &z_res->base.b, aux_usage,
+                                     level, true);
+   }
+
+   struct blorp_batch blorp_batch;
+   blorp_batch_init(&ice->blorp, &blorp_batch, batch, blorp_flags);
+
+   uint8_t stencil_mask = clear_stencil && stencil_res ? 0xff : 0;
+   if (stencil_mask) {
+      crocus_resource_prepare_access(ice, stencil_res, level, 1, box->z,
+                                     box->depth, stencil_res->aux.usage, false);
+      crocus_blorp_surf_for_resource(&screen->vtbl, &batch->screen->isl_dev,
+                                     &stencil_surf, &stencil_res->base.b,
+                                     stencil_res->aux.usage, level, true);
+   }
+
+   blorp_clear_depth_stencil(&blorp_batch, &z_surf, &stencil_surf,
+                             level, box->z, box->depth,
+                             box->x, box->y,
+                             box->x + box->width,
+                             box->y + box->height,
+                             clear_depth && z_res, depth,
+                             stencil_mask, stencil);
+
+   blorp_batch_finish(&blorp_batch);
+   crocus_flush_and_dirty_for_history(ice, batch, res, 0,
+                                      "cache history: post slow ZS clear");
+
+   if (clear_depth && z_res) {
+      crocus_resource_finish_render(ice, z_res, level,
+                                    box->z, box->depth, z_surf.aux_usage);
+   }
+
+   if (stencil_mask) {
+      crocus_resource_finish_write(ice, stencil_res, level, box->z, box->depth,
+                                   stencil_res->aux.usage);
+   }
+}
+
+/**
+ * The pipe->clear() driver hook.
+ *
+ * This clears buffers attached to the current draw framebuffer.
+ */
+static void
+crocus_clear(struct pipe_context *ctx,
+             unsigned buffers,
+             uint32_t color_clear_mask,
+             uint8_t stencil_clear_mask,
+             const struct pipe_scissor_state *scissor_state,
+             const union pipe_color_union *p_color,
+             double depth,
+             unsigned stencil)
+{
+   struct crocus_context *ice = (void *) ctx;
+   struct pipe_framebuffer_state *cso_fb = &ice->state.framebuffer;
+   struct crocus_screen *screen = (void *) ctx->screen;
+   const struct intel_device_info *devinfo = &screen->devinfo;
+   assert(buffers != 0);
+
+   struct pipe_box box = {
+      .width = cso_fb->width,
+      .height = cso_fb->height,
+   };
+
+   if (scissor_state) {
+      box.x = scissor_state->minx;
+      box.y = scissor_state->miny;
+      box.width = MIN2(box.width, scissor_state->maxx - scissor_state->minx);
+      box.height = MIN2(box.height, scissor_state->maxy - scissor_state->miny);
+   }
+
+   if (buffers & PIPE_CLEAR_DEPTHSTENCIL) {
+      if (devinfo->ver < 6) {
+         crocus_blitter_begin(ice, CROCUS_SAVE_FRAGMENT_STATE, true);
+         util_blitter_clear(ice->blitter, cso_fb->width, cso_fb->height,
+                            util_framebuffer_get_num_layers(cso_fb),
+                            buffers & PIPE_CLEAR_DEPTHSTENCIL, p_color, depth, stencil, false);
+      } else {
+         const struct pipe_surface *psurf = &cso_fb->zsbuf;
+         box.depth = psurf->last_layer - psurf->first_layer + 1;
+         box.z = psurf->first_layer;
+
+         clear_depth_stencil(ice, psurf->texture, psurf->level, &box, true,
+                             buffers & PIPE_CLEAR_DEPTH,
+                             buffers & PIPE_CLEAR_STENCIL,
+                             depth, stencil);
+      }
+      buffers &= ~PIPE_CLEAR_DEPTHSTENCIL;
+   }
+
+   if (buffers & PIPE_CLEAR_COLOR) {
+      /* pipe_color_union and isl_color_value are interchangeable */
+      union isl_color_value *color = (void *) p_color;
+
+      for (unsigned i = 0; i < cso_fb->nr_cbufs; i++) {
+         if (buffers & (PIPE_CLEAR_COLOR0 << i)) {
+            struct pipe_surface *psurf = ice->state.fb_cbufs[i];
+            struct crocus_surface *isurf = (void *) psurf;
+            box.depth = psurf->last_layer - psurf->first_layer + 1,
+            box.z = psurf->first_layer,
+
+            clear_color(ice, psurf->texture, psurf->level, &box,
+                        true, isurf->view.format, isurf->view.swizzle,
+                        *color);
+         }
+      }
+   }
+}
+
+/**
+ * The pipe->clear_texture() driver hook.
+ *
+ * This clears the given texture resource.
+ */
+static void
+crocus_clear_texture(struct pipe_context *ctx,
+                     struct pipe_resource *p_res,
+                     unsigned level,
+                     const struct pipe_box *box,
+                     const void *data)
+{
+   struct crocus_context *ice = (void *) ctx;
+   struct crocus_screen *screen = (void *) ctx->screen;
+   const struct intel_device_info *devinfo = &screen->devinfo;
+
+   if (devinfo->ver < 6) {
+      u_default_clear_texture(ctx, p_res, level, box, data);
+      return;
+   }
+
+   if (util_format_is_depth_or_stencil(p_res->format)) {
+      const struct util_format_unpack_description *fmt_unpack =
+         util_format_unpack_description(p_res->format);
+
+      float depth = 0.0;
+      uint8_t stencil = 0;
+
+      if (fmt_unpack->unpack_z_float)
+         fmt_unpack->unpack_z_float(&depth, 0, data, 0, 1, 1);
+
+      if (fmt_unpack->unpack_s_8uint)
+         fmt_unpack->unpack_s_8uint(&stencil, 0, data, 0, 1, 1);
+
+      clear_depth_stencil(ice, p_res, level, box, true, true, true,
+                          depth, stencil);
+   } else {
+      union isl_color_value color;
+      struct crocus_resource *res = (void *) p_res;
+      enum isl_format format = res->surf.format;
+
+      if (!isl_format_supports_rendering(devinfo, format)) {
+         const struct isl_format_layout *fmtl = isl_format_get_layout(format);
+         // XXX: actually just get_copy_format_for_bpb from BLORP
+         // XXX: don't cut and paste this
+         switch (fmtl->bpb) {
+         case 8:   format = ISL_FORMAT_R8_UINT;           break;
+         case 16:  format = ISL_FORMAT_R8G8_UINT;         break;
+         case 24:  format = ISL_FORMAT_R8G8B8_UINT;       break;
+         case 32:  format = ISL_FORMAT_R8G8B8A8_UINT;     break;
+         case 48:  format = ISL_FORMAT_R16G16B16_UINT;    break;
+         case 64:  format = ISL_FORMAT_R16G16B16A16_UINT; break;
+         case 96:  format = ISL_FORMAT_R32G32B32_UINT;    break;
+         case 128: format = ISL_FORMAT_R32G32B32A32_UINT; break;
+         default:
+            UNREACHABLE("Unknown format bpb");
+         }
+
+         /* No aux surfaces for non-renderable surfaces */
+         assert(res->aux.usage == ISL_AUX_USAGE_NONE);
+      }
+
+      isl_color_value_unpack(&color, format, data);
+
+      clear_color(ice, p_res, level, box, true, format,
+                  ISL_SWIZZLE_IDENTITY, color);
+   }
+}
+
+/**
+ * The pipe->clear_render_target() driver hook.
+ *
+ * This clears the given render target surface.
+ */
+static void
+crocus_clear_render_target(struct pipe_context *ctx,
+                           struct pipe_surface *psurf,
+                           const union pipe_color_union *p_color,
+                           unsigned dst_x, unsigned dst_y,
+                           unsigned width, unsigned height,
+                           bool render_condition_enabled)
+{
+   struct crocus_context *ice = (void *) ctx;
+   struct crocus_surface *isurf = (void *) psurf;
+   struct pipe_box box = {
+      .x = dst_x,
+      .y = dst_y,
+      .z = psurf->first_layer,
+      .width = width,
+      .height = height,
+      .depth = psurf->last_layer - psurf->first_layer + 1
+   };
+
+   /* pipe_color_union and isl_color_value are interchangeable */
+   union isl_color_value *color = (void *) p_color;
+
+   clear_color(ice, psurf->texture, psurf->level, &box,
+               render_condition_enabled,
+               isurf->view.format, isurf->view.swizzle, *color);
+}
+
+/**
+ * The pipe->clear_depth_stencil() driver hook.
+ *
+ * This clears the given depth/stencil surface.
+ */
+static void
+crocus_clear_depth_stencil(struct pipe_context *ctx,
+                           struct pipe_surface *psurf,
+                           unsigned flags,
+                           double depth,
+                           unsigned stencil,
+                           unsigned dst_x, unsigned dst_y,
+                           unsigned width, unsigned height,
+                           bool render_condition_enabled)
+{
+   return;
+#if 0
+   struct crocus_context *ice = (void *) ctx;
+   struct pipe_box box = {
+      .x = dst_x,
+      .y = dst_y,
+      .z = psurf->first_layer,
+      .width = width,
+      .height = height,
+      .depth = psurf->last_layer - psurf->first_layer + 1
+   };
+   uint32_t blit_flags = 0;
+
+   assert(util_format_is_depth_or_stencil(psurf->texture->format));
+
+   crocus_blitter_begin(ice, CROCUS_SAVE_FRAGMENT_STATE);
+   util_blitter_clear(ice->blitter, width, height,
+                      1, flags, NULL, depth, stencil, render_condition_enabled);
+#if 0
+   clear_depth_stencil(ice, psurf->texture, psurf->level, &box,
+                       render_condition_enabled,
+                       flags & PIPE_CLEAR_DEPTH, flags & PIPE_CLEAR_STENCIL,
+                       depth, stencil);
+#endif
+#endif
+}
+
+void
+crocus_init_clear_functions(struct pipe_context *ctx)
+{
+   ctx->clear = crocus_clear;
+   ctx->clear_texture = crocus_clear_texture;
+   ctx->clear_render_target = crocus_clear_render_target;
+   ctx->clear_depth_stencil = crocus_clear_depth_stencil;
+}

src/gallium/drivers/crocus/crocus_context.c

@@ -0,0 +1,365 @@
+/*
+ * Copyright © 2017 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <stdio.h>
+#include <time.h>
+#include "pipe/p_defines.h"
+#include "pipe/p_state.h"
+#include "util/ralloc.h"
+#include "util/u_inlines.h"
+#include "util/format/u_format.h"
+#include "util/u_upload_mgr.h"
+#include "drm-uapi/i915_drm.h"
+#include "crocus_context.h"
+#include "crocus_perf.h"
+#include "crocus_resource.h"
+#include "crocus_screen.h"
+#include "common/i915/intel_defines.h"
+#include "common/intel_debug_identifier.h"
+#include "common/intel_sample_positions.h"
+
+/**
+ * The pipe->set_debug_callback() driver hook.
+ */
+static void
+crocus_set_debug_callback(struct pipe_context *ctx,
+                          const struct util_debug_callback *cb)
+{
+   struct crocus_context *ice = (struct crocus_context *)ctx;
+
+   if (cb)
+      ice->dbg = *cb;
+   else
+      memset(&ice->dbg, 0, sizeof(ice->dbg));
+}
+
+static bool
+crocus_init_identifier_bo(struct crocus_context *ice)
+{
+   void *bo_map;
+
+   bo_map = crocus_bo_map(NULL, ice->workaround_bo, MAP_READ | MAP_WRITE);
+   if (!bo_map)
+      return false;
+
+   ice->workaround_bo->kflags |= EXEC_OBJECT_CAPTURE;
+   ice->workaround_offset = align(
+      intel_debug_write_identifiers(bo_map, 4096, "Crocus"), 32);
+
+   crocus_bo_unmap(ice->workaround_bo);
+
+   return true;
+}
+
+/**
+ * Called from the batch module when it detects a GPU hang.
+ *
+ * In this case, we've lost our GEM context, and can't rely on any existing
+ * state on the GPU.  We must mark everything dirty and wipe away any saved
+ * assumptions about the last known state of the GPU.
+ */
+void
+crocus_lost_context_state(struct crocus_batch *batch)
+{
+   /* The batch module doesn't have an crocus_context, because we want to
+    * avoid introducing lots of layering violations.  Unfortunately, here
+    * we do need to inform the context of batch catastrophe.  We know the
+    * batch is one of our context's, so hackily claw our way back.
+    */
+   struct crocus_context *ice = batch->ice;
+   struct crocus_screen *screen = batch->screen;
+   if (batch->name == CROCUS_BATCH_RENDER) {
+      screen->vtbl.init_render_context(batch);
+   } else if (batch->name == CROCUS_BATCH_COMPUTE) {
+      screen->vtbl.init_compute_context(batch);
+   } else {
+      UNREACHABLE("unhandled batch reset");
+   }
+
+   ice->state.dirty = ~0ull;
+   memset(ice->state.last_grid, 0, sizeof(ice->state.last_grid));
+   batch->state_base_address_emitted = false;
+   screen->vtbl.lost_genx_state(ice, batch);
+}
+
+static enum pipe_reset_status
+crocus_get_device_reset_status(struct pipe_context *ctx)
+{
+   struct crocus_context *ice = (struct crocus_context *)ctx;
+
+   enum pipe_reset_status worst_reset = PIPE_NO_RESET;
+
+   /* Check the reset status of each batch's hardware context, and take the
+    * worst status (if one was guilty, proclaim guilt).
+    */
+   for (int i = 0; i < ice->batch_count; i++) {
+      /* This will also recreate the hardware contexts as necessary, so any
+       * future queries will show no resets.  We only want to report once.
+       */
+      enum pipe_reset_status batch_reset =
+         crocus_batch_check_for_reset(&ice->batches[i]);
+
+      if (batch_reset == PIPE_NO_RESET)
+         continue;
+
+      if (worst_reset == PIPE_NO_RESET) {
+         worst_reset = batch_reset;
+      } else {
+         /* GUILTY < INNOCENT < UNKNOWN */
+         worst_reset = MIN2(worst_reset, batch_reset);
+      }
+   }
+
+   if (worst_reset != PIPE_NO_RESET && ice->reset.reset)
+      ice->reset.reset(ice->reset.data, worst_reset);
+
+   return worst_reset;
+}
+
+static void
+crocus_set_device_reset_callback(struct pipe_context *ctx,
+                                 const struct pipe_device_reset_callback *cb)
+{
+   struct crocus_context *ice = (struct crocus_context *)ctx;
+
+   if (cb)
+      ice->reset = *cb;
+   else
+      memset(&ice->reset, 0, sizeof(ice->reset));
+}
+
+static void
+crocus_get_sample_position(struct pipe_context *ctx,
+                           unsigned sample_count,
+                           unsigned sample_index,
+                           float *out_value)
+{
+   union {
+      struct {
+         float x[8];
+         float y[8];
+      } a;
+      struct {
+         float  _0XOffset,  _1XOffset,  _2XOffset,  _3XOffset,
+                _4XOffset,  _5XOffset,  _6XOffset,  _7XOffset;
+         float  _0YOffset,  _1YOffset,  _2YOffset,  _3YOffset,
+                _4YOffset,  _5YOffset,  _6YOffset,  _7YOffset;
+      } v;
+   } u;
+   switch (sample_count) {
+   case 1:  INTEL_SAMPLE_POS_1X(u.v._);  break;
+   case 2:  INTEL_SAMPLE_POS_2X(u.v._);  break;
+   case 4:  INTEL_SAMPLE_POS_4X(u.v._);  break;
+   case 8:  INTEL_SAMPLE_POS_8X(u.v._);  break;
+   default: UNREACHABLE("invalid sample count");
+   }
+
+   out_value[0] = u.a.x[sample_index];
+   out_value[1] = u.a.y[sample_index];
+}
+
+#define genX_call(devinfo, func, ...)                   \
+   switch ((devinfo)->verx10) {                         \
+   case 80:                                             \
+      gfx8_##func(__VA_ARGS__);                         \
+      break;                                            \
+   case 75:                                             \
+      gfx75_##func(__VA_ARGS__);                        \
+      break;                                            \
+   case 70:                                             \
+      gfx7_##func(__VA_ARGS__);                         \
+      break;                                            \
+   case 60:                                             \
+      gfx6_##func(__VA_ARGS__);                         \
+      break;                                            \
+   case 50:                                             \
+      gfx5_##func(__VA_ARGS__);                         \
+      break;                                            \
+   case 45:                                             \
+      gfx45_##func(__VA_ARGS__);                        \
+      break;                                            \
+   case 40:                                             \
+      gfx4_##func(__VA_ARGS__);                         \
+      break;                                            \
+   default:                                             \
+      UNREACHABLE("Unknown hardware generation");       \
+   }
+
+/**
+ * Destroy a context, freeing any associated memory.
+ */
+static void
+crocus_destroy_context(struct pipe_context *ctx)
+{
+   struct crocus_context *ice = (struct crocus_context *)ctx;
+   struct crocus_screen *screen = (struct crocus_screen *)ctx->screen;
+   const struct intel_device_info *devinfo = &screen->devinfo;
+
+   blorp_finish(&ice->blorp);
+
+   intel_perf_free_context(ice->perf_ctx);
+   if (ctx->stream_uploader)
+      u_upload_destroy(ctx->stream_uploader);
+
+   if (ice->blitter)
+      util_blitter_destroy(ice->blitter);
+   screen->vtbl.destroy_state(ice);
+
+   genX_call(devinfo, crocus_framebuffer_init, ctx, NULL, ice->state.fb_cbufs, &ice->state.fb_zsbuf);
+   util_unreference_framebuffer_state(&ice->state.framebuffer);
+
+   for (unsigned i = 0; i < ARRAY_SIZE(ice->shaders.scratch_bos); i++) {
+      for (unsigned j = 0; j < ARRAY_SIZE(ice->shaders.scratch_bos[i]); j++)
+         crocus_bo_unreference(ice->shaders.scratch_bos[i][j]);
+   }
+
+   crocus_destroy_program_cache(ice);
+   u_upload_destroy(ice->query_buffer_uploader);
+
+   crocus_bo_unreference(ice->workaround_bo);
+
+   slab_destroy_child(&ice->transfer_pool);
+   slab_destroy_child(&ice->transfer_pool_unsync);
+
+   crocus_batch_free(&ice->batches[CROCUS_BATCH_RENDER]);
+   if (ice->batches[CROCUS_BATCH_COMPUTE].ice)
+      crocus_batch_free(&ice->batches[CROCUS_BATCH_COMPUTE]);
+
+   ralloc_free(ice);
+}
+
+/**
+ * Create a context.
+ *
+ * This is where each context begins.
+ */
+struct pipe_context *
+crocus_create_context(struct pipe_screen *pscreen, void *priv, unsigned flags)
+{
+   struct crocus_screen *screen = (struct crocus_screen*)pscreen;
+   const struct intel_device_info *devinfo = &screen->devinfo;
+   struct crocus_context *ice = rzalloc(NULL, struct crocus_context);
+
+   if (!ice)
+      return NULL;
+
+   struct pipe_context *ctx = &ice->ctx;
+
+   ctx->screen = pscreen;
+   ctx->priv = priv;
+
+   ctx->stream_uploader = u_upload_create_default(ctx);
+   if (!ctx->stream_uploader) {
+      ralloc_free(ice);
+      return NULL;
+   }
+   ctx->const_uploader = ctx->stream_uploader;
+
+   ctx->destroy = crocus_destroy_context;
+   ctx->set_debug_callback = crocus_set_debug_callback;
+   ctx->set_device_reset_callback = crocus_set_device_reset_callback;
+   ctx->get_device_reset_status = crocus_get_device_reset_status;
+   ctx->get_sample_position = crocus_get_sample_position;
+
+   ice->shaders.urb_size = devinfo->urb.size;
+
+   crocus_init_context_fence_functions(ctx);
+   crocus_init_blit_functions(ctx);
+   crocus_init_clear_functions(ctx);
+   crocus_init_program_functions(ctx);
+   crocus_init_resource_functions(ctx);
+   crocus_init_flush_functions(ctx);
+   crocus_init_perfquery_functions(ctx);
+
+   crocus_init_program_cache(ice);
+
+   slab_create_child(&ice->transfer_pool, &screen->transfer_pool);
+   slab_create_child(&ice->transfer_pool_unsync, &screen->transfer_pool);
+
+   ice->query_buffer_uploader =
+      u_upload_create(ctx, 4096, PIPE_BIND_CUSTOM, PIPE_USAGE_STAGING,
+                      0);
+
+   ice->workaround_bo =
+      crocus_bo_alloc(screen->bufmgr, "workaround", 4096);
+   if (!ice->workaround_bo)
+      return NULL;
+
+   if (!crocus_init_identifier_bo(ice))
+      return NULL;
+
+   genX_call(devinfo, crocus_init_state, ice);
+   genX_call(devinfo, crocus_init_blorp, ice);
+   genX_call(devinfo, crocus_init_query, ice);
+
+   ice->blitter = util_blitter_create(&ice->ctx);
+   if (ice->blitter == NULL)
+      return NULL;
+   int priority = 0;
+   if (flags & PIPE_CONTEXT_HIGH_PRIORITY)
+      priority = INTEL_CONTEXT_HIGH_PRIORITY;
+   if (flags & PIPE_CONTEXT_LOW_PRIORITY)
+      priority = INTEL_CONTEXT_LOW_PRIORITY;
+
+   ice->batch_count = devinfo->ver >= 7 ? CROCUS_BATCH_COUNT : 1;
+   for (int i = 0; i < ice->batch_count; i++) {
+      crocus_init_batch(ice, (enum crocus_batch_name) i,
+                        priority);
+   }
+
+   ice->urb.size = devinfo->urb.size;
+   screen->vtbl.init_render_context(&ice->batches[CROCUS_BATCH_RENDER]);
+   if (ice->batch_count > 1)
+      screen->vtbl.init_compute_context(&ice->batches[CROCUS_BATCH_COMPUTE]);
+
+   if (!(flags & PIPE_CONTEXT_PREFER_THREADED)) {
+      fprintf(stderr, "CROCUS unthreaded context: get_compute_state_info=%p, set_global_binding=%p\n",
+              ctx->get_compute_state_info, ctx->set_global_binding);
+      return ctx;
+   }
+
+   struct pipe_context *tc = threaded_context_create(ctx, &screen->transfer_pool,
+                                  crocus_replace_buffer_storage,
+                                  NULL, /* TODO: asynchronous flushes? */
+                                  &ice->thrctx);
+   fprintf(stderr, "CROCUS threaded context: get_compute_state_info=%p, set_global_binding=%p\n",
+           tc->get_compute_state_info, tc->set_global_binding);
+   return tc;
+}
+
+bool
+crocus_sw_check_cond_render(struct crocus_context *ice)
+{
+   struct crocus_query *q = ice->condition.query;
+   union pipe_query_result result;
+
+   bool wait = ice->condition.mode == PIPE_RENDER_COND_WAIT ||
+      ice->condition.mode == PIPE_RENDER_COND_BY_REGION_WAIT;
+   if (!q)
+      return true;
+
+   bool ret = ice->ctx.get_query_result(&ice->ctx, (void *)q, wait, &result);
+   if (!ret)
+      return true;
+
+   return ice->condition.condition ? result.u64 == 0 : result.u64 != 0;
+}

src/gallium/drivers/crocus/crocus_defines.h

@@ -0,0 +1,58 @@
+/*
+ * Copyright © 2018 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * on the rights to use, copy, modify, merge, publish, distribute, sub
+ * license, and/or sell copies of the Software, and to permit persons to whom
+ * the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef CROCUS_DEFINES_H
+#define CROCUS_DEFINES_H
+
+/**
+ * @file crocus_defines.h
+ *
+ * Random hardware #defines that we're not using GENXML for.
+ */
+
+#define MI_PREDICATE                         (0xC << 23)
+# define MI_PREDICATE_LOADOP_KEEP            (0 << 6)
+# define MI_PREDICATE_LOADOP_LOAD            (2 << 6)
+# define MI_PREDICATE_LOADOP_LOADINV         (3 << 6)
+# define MI_PREDICATE_COMBINEOP_SET          (0 << 3)
+# define MI_PREDICATE_COMBINEOP_AND          (1 << 3)
+# define MI_PREDICATE_COMBINEOP_OR           (2 << 3)
+# define MI_PREDICATE_COMBINEOP_XOR          (3 << 3)
+# define MI_PREDICATE_COMPAREOP_TRUE         (0 << 0)
+# define MI_PREDICATE_COMPAREOP_FALSE        (1 << 0)
+# define MI_PREDICATE_COMPAREOP_SRCS_EQUAL   (2 << 0)
+# define MI_PREDICATE_COMPAREOP_DELTAS_EQUAL (3 << 0)
+
+/* Predicate registers */
+#define MI_PREDICATE_SRC0                    0x2400
+#define MI_PREDICATE_SRC1                    0x2408
+#define MI_PREDICATE_DATA                    0x2410
+#define MI_PREDICATE_RESULT                  0x2418
+#define MI_PREDICATE_RESULT_1                0x241C
+#define MI_PREDICATE_RESULT_2                0x2214
+
+#define CS_GPR(n) (0x2600 + (n) * 8)
+
+/* The number of bits in our TIMESTAMP queries. */
+#define TIMESTAMP_BITS 36
+
+#endif

src/gallium/drivers/crocus/crocus_disk_cache.c

@@ -0,0 +1,260 @@
+/*
+ * Copyright © 2018 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+/**
+ * @file crocus_disk_cache.c
+ *
+ * Functions for interacting with the on-disk shader cache.
+ */
+
+#include <stdio.h>
+#include <stdint.h>
+#include <assert.h>
+#include <string.h>
+
+#include "compiler/nir/nir.h"
+#include "util/blob.h"
+#include "util/build_id.h"
+#include "util/disk_cache.h"
+#include "util/mesa-blake3.h"
+
+#include "crocus_context.h"
+
+static bool debug = false;
+
+/**
+ * Compute a disk cache key for the given uncompiled shader and NOS key.
+ */
+static void
+crocus_disk_cache_compute_key(struct disk_cache *cache,
+                              const struct crocus_uncompiled_shader *ish,
+                              const void *orig_prog_key,
+                              uint32_t prog_key_size,
+                              cache_key cache_key)
+{
+   /* Create a copy of the program key with program_string_id zeroed out.
+    * It's essentially random data which we don't want to include in our
+    * hashing and comparisons.  We'll set a proper value on a cache hit.
+    */
+   union elk_any_prog_key prog_key;
+   memcpy(&prog_key, orig_prog_key, prog_key_size);
+   prog_key.base.program_string_id = 0;
+
+   uint8_t data[sizeof(prog_key) + sizeof(ish->nir_blake3)];
+   uint32_t data_size = prog_key_size + sizeof(ish->nir_blake3);
+
+   memcpy(data, ish->nir_blake3, sizeof(ish->nir_blake3));
+   memcpy(data + sizeof(ish->nir_blake3), &prog_key, prog_key_size);
+
+   disk_cache_compute_key(cache, data, data_size, cache_key);
+}
+
+/**
+ * Store the given compiled shader in the disk cache.
+ *
+ * This should only be called on newly compiled shaders.  No checking is
+ * done to prevent repeated stores of the same shader.
+ */
+void
+crocus_disk_cache_store(struct disk_cache *cache,
+                        const struct crocus_uncompiled_shader *ish,
+                        const struct crocus_compiled_shader *shader,
+                        void *map,
+                        const void *prog_key,
+                        uint32_t prog_key_size)
+{
+#ifdef ENABLE_SHADER_CACHE
+   if (!cache)
+      return;
+
+   mesa_shader_stage stage = ish->nir->info.stage;
+   const struct elk_stage_prog_data *prog_data = shader->prog_data;
+
+   cache_key cache_key;
+   crocus_disk_cache_compute_key(cache, ish, prog_key, prog_key_size, cache_key);
+
+   if (debug) {
+      char blake3[BLAKE3_HEX_LEN];
+      _mesa_blake3_format(blake3, cache_key);
+      fprintf(stderr, "[mesa disk cache] storing %s\n", blake3);
+   }
+
+   struct blob blob;
+   blob_init(&blob);
+
+   /* We write the following data to the cache blob:
+    *
+    * 1. Prog data (must come first because it has the assembly size)
+    * 2. Assembly code
+    * 3. Number of entries in the system value array
+    * 4. System value array
+    * 5. Legacy param array (only used for compute workgroup ID)
+    * 6. Binding table
+    */
+   blob_write_bytes(&blob, shader->prog_data, elk_prog_data_size(stage));
+   blob_write_bytes(&blob, map + shader->offset, shader->prog_data->program_size);
+   blob_write_bytes(&blob, &shader->num_system_values, sizeof(unsigned));
+   blob_write_bytes(&blob, shader->system_values,
+                    shader->num_system_values * sizeof(enum elk_param_builtin));
+   blob_write_bytes(&blob, prog_data->param,
+                    prog_data->nr_params * sizeof(uint32_t));
+   blob_write_bytes(&blob, &shader->bt, sizeof(shader->bt));
+
+   disk_cache_put(cache, cache_key, blob.data, blob.size, NULL);
+   blob_finish(&blob);
+#endif
+}
+
+/**
+ * Search for a compiled shader in the disk cache.  If found, upload it
+ * to the in-memory program cache so we can use it.
+ */
+struct crocus_compiled_shader *
+crocus_disk_cache_retrieve(struct crocus_context *ice,
+                           const struct crocus_uncompiled_shader *ish,
+                           const void *prog_key,
+                           uint32_t key_size)
+{
+#ifdef ENABLE_SHADER_CACHE
+   struct crocus_screen *screen = (void *) ice->ctx.screen;
+   struct disk_cache *cache = screen->disk_cache;
+   mesa_shader_stage stage = ish->nir->info.stage;
+
+   if (!cache)
+      return NULL;
+
+   cache_key cache_key;
+   crocus_disk_cache_compute_key(cache, ish, prog_key, key_size, cache_key);
+
+   if (debug) {
+      char blake3[BLAKE3_HEX_LEN];
+      _mesa_blake3_format(blake3, cache_key);
+      fprintf(stderr, "[mesa disk cache] retrieving %s: ", blake3);
+   }
+
+   size_t size;
+   void *buffer = disk_cache_get(screen->disk_cache, cache_key, &size);
+
+   if (debug)
+      fprintf(stderr, "%s\n", buffer ? "found" : "missing");
+
+   if (!buffer)
+      return NULL;
+
+   const uint32_t prog_data_size = elk_prog_data_size(stage);
+
+   struct elk_stage_prog_data *prog_data = ralloc_size(NULL, prog_data_size);
+   const void *assembly;
+   uint32_t num_system_values;
+   uint32_t *system_values = NULL;
+   uint32_t *so_decls = NULL;
+
+   struct blob_reader blob;
+   blob_reader_init(&blob, buffer, size);
+   blob_copy_bytes(&blob, prog_data, prog_data_size);
+   assembly = blob_read_bytes(&blob, prog_data->program_size);
+   num_system_values = blob_read_uint32(&blob);
+   if (num_system_values) {
+      system_values =
+         ralloc_array(NULL, enum elk_param_builtin, num_system_values);
+      blob_copy_bytes(&blob, system_values,
+                      num_system_values * sizeof(enum elk_param_builtin));
+   }
+
+   prog_data->param = NULL;
+   if (prog_data->nr_params) {
+      prog_data->param = ralloc_array(NULL, uint32_t, prog_data->nr_params);
+      blob_copy_bytes(&blob, prog_data->param,
+                      prog_data->nr_params * sizeof(uint32_t));
+   }
+
+   struct crocus_binding_table bt;
+   blob_copy_bytes(&blob, &bt, sizeof(bt));
+
+   if ((stage == MESA_SHADER_VERTEX ||
+        stage == MESA_SHADER_TESS_EVAL ||
+        stage == MESA_SHADER_GEOMETRY) && screen->devinfo.ver > 6) {
+      struct elk_vue_prog_data *vue_prog_data = (void *) prog_data;
+      so_decls = screen->vtbl.create_so_decl_list(&ish->stream_output,
+                                                  &vue_prog_data->vue_map);
+   }
+
+   /* System values and uniforms are stored in constant buffer 0, the
+    * user-facing UBOs are indexed by one.  So if any constant buffer is
+    * needed, the constant buffer 0 will be needed, so account for it.
+    */
+   unsigned num_cbufs = ish->nir->info.num_ubos;
+
+   if (num_cbufs || ish->nir->num_uniforms)
+      num_cbufs++;
+
+   if (num_system_values)
+      num_cbufs++;
+
+   /* Upload our newly read shader to the in-memory program cache and
+    * return it to the caller.
+    */
+   struct crocus_compiled_shader *shader =
+      crocus_upload_shader(ice, stage, key_size, prog_key, assembly,
+                           prog_data->program_size,
+                           prog_data, prog_data_size, so_decls, system_values,
+                           num_system_values, num_cbufs, &bt);
+
+   free(buffer);
+
+   return shader;
+#else
+   return NULL;
+#endif
+}
+
+/**
+ * Initialize the on-disk shader cache.
+ */
+void
+crocus_disk_cache_init(struct crocus_screen *screen)
+{
+#ifdef ENABLE_SHADER_CACHE
+   if (INTEL_DEBUG(DEBUG_DISK_CACHE_DISABLE_MASK))
+      return;
+
+   /* array length = print length + nul char + 1 extra to verify it's unused */
+   char renderer[13];
+   UNUSED int len =
+      snprintf(renderer, sizeof(renderer), "crocus_%04x", screen->pci_id);
+   assert(len == sizeof(renderer) - 2);
+
+   const struct build_id_note *note =
+      build_id_find_nhdr_for_addr(crocus_disk_cache_init);
+   assert(note && build_id_length(note) == BUILD_ID_EXPECTED_HASH_LENGTH); /* sha1 */
+
+   const uint8_t *id_sha1 = build_id_data(note);
+   assert(id_sha1);
+
+   char timestamp[BLAKE3_HEX_LEN];
+   _mesa_sha1_format(timestamp, id_sha1);
+
+   const uint64_t driver_flags =
+      elk_get_compiler_config_value(screen->compiler);
+   screen->disk_cache = disk_cache_create(renderer, timestamp, driver_flags);
+#endif
+}

src/gallium/drivers/crocus/crocus_draw.c

@@ -0,0 +1,530 @@
+/*
+ * Copyright © 2017 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+/**
+ * @file crocus_draw.c
+ *
+ * The main driver hooks for drawing and launching compute shaders.
+ */
+
+#include <stdio.h>
+#include <errno.h>
+#include "pipe/p_defines.h"
+#include "pipe/p_state.h"
+#include "pipe/p_context.h"
+#include "pipe/p_screen.h"
+#include "util/u_draw.h"
+#include "util/u_inlines.h"
+#include "util/u_transfer.h"
+#include "util/u_upload_mgr.h"
+#include "intel/compiler/elk/elk_compiler.h"
+#include "intel/compiler/elk/elk_eu_defines.h"
+#include "compiler/shader_info.h"
+#include "crocus_context.h"
+#include "crocus_defines.h"
+#include "util/u_prim_restart.h"
+#include "util/u_prim.h"
+
+static bool
+prim_is_points_or_lines(enum mesa_prim mode)
+{
+   /* We don't need to worry about adjacency - it can only be used with
+    * geometry shaders, and we don't care about this info when GS is on.
+    */
+   return mode == MESA_PRIM_POINTS ||
+          mode == MESA_PRIM_LINES ||
+          mode == MESA_PRIM_LINE_LOOP ||
+          mode == MESA_PRIM_LINE_STRIP;
+}
+
+static bool
+can_cut_index_handle_restart_index(struct crocus_context *ice,
+                                   const struct pipe_draw_info *draw)
+{
+   switch (draw->index_size) {
+   case 1:
+      return draw->restart_index == 0xff;
+   case 2:
+      return draw->restart_index == 0xffff;
+   case 4:
+      return draw->restart_index == 0xffffffff;
+   default:
+      UNREACHABLE("illegal index size\n");
+   }
+
+   return false;
+}
+
+static bool
+can_cut_index_handle_prim(struct crocus_context *ice,
+                          const struct pipe_draw_info *draw)
+{
+   struct crocus_screen *screen = (struct crocus_screen*)ice->ctx.screen;
+   const struct intel_device_info *devinfo = &screen->devinfo;
+
+   /* Haswell can do it all. */
+   if (devinfo->verx10 >= 75)
+      return true;
+
+   if (!can_cut_index_handle_restart_index(ice, draw))
+      return false;
+
+   switch (draw->mode) {
+   case MESA_PRIM_POINTS:
+   case MESA_PRIM_LINES:
+   case MESA_PRIM_LINE_STRIP:
+   case MESA_PRIM_TRIANGLES:
+   case MESA_PRIM_TRIANGLE_STRIP:
+   case MESA_PRIM_LINES_ADJACENCY:
+   case MESA_PRIM_LINE_STRIP_ADJACENCY:
+   case MESA_PRIM_TRIANGLES_ADJACENCY:
+   case MESA_PRIM_TRIANGLE_STRIP_ADJACENCY:
+      return true;
+   default:
+      break;
+   }
+   return false;
+}
+
+/**
+ * Record the current primitive mode and restart information, flagging
+ * related packets as dirty if necessary.
+ *
+ * This must be called before updating compiled shaders, because the patch
+ * information informs the TCS key.
+ */
+static void
+crocus_update_draw_info(struct crocus_context *ice,
+                        const struct pipe_draw_info *info,
+                        const struct pipe_draw_start_count_bias *draw)
+{
+   struct crocus_screen *screen = (struct crocus_screen *)ice->ctx.screen;
+   enum mesa_prim mode = info->mode;
+
+   if (screen->devinfo.ver < 6) {
+      /* Slight optimization to avoid the GS program when not needed:
+       */
+      struct pipe_rasterizer_state *rs_state = crocus_get_rast_state(ice);
+      if (mode == MESA_PRIM_QUAD_STRIP && !rs_state->flatshade &&
+          rs_state->fill_front == PIPE_POLYGON_MODE_FILL &&
+          rs_state->fill_back == PIPE_POLYGON_MODE_FILL)
+         mode = MESA_PRIM_TRIANGLE_STRIP;
+      if (mode == MESA_PRIM_QUADS &&
+          draw->count == 4 &&
+          !rs_state->flatshade &&
+          rs_state->fill_front == PIPE_POLYGON_MODE_FILL &&
+          rs_state->fill_back == PIPE_POLYGON_MODE_FILL)
+         mode = MESA_PRIM_TRIANGLE_FAN;
+   }
+
+   if (ice->state.prim_mode != mode) {
+      ice->state.prim_mode = mode;
+
+      enum mesa_prim reduced = u_reduced_prim(mode);
+      if (ice->state.reduced_prim_mode != reduced) {
+         if (screen->devinfo.ver < 6)
+            ice->state.dirty |= CROCUS_DIRTY_GEN4_CLIP_PROG | CROCUS_DIRTY_GEN4_SF_PROG;
+         /* if the reduced prim changes the WM needs updating. */
+         ice->state.stage_dirty |= CROCUS_STAGE_DIRTY_UNCOMPILED_FS;
+         ice->state.reduced_prim_mode = reduced;
+      }
+
+      if (screen->devinfo.ver == 8)
+         ice->state.dirty |= CROCUS_DIRTY_GEN8_VF_TOPOLOGY;
+
+      if (screen->devinfo.ver <= 6)
+         ice->state.dirty |= CROCUS_DIRTY_GEN4_FF_GS_PROG;
+
+      if (screen->devinfo.ver >= 7)
+         ice->state.dirty |= CROCUS_DIRTY_GEN7_SBE;
+
+      /* For XY Clip enables */
+      bool points_or_lines = prim_is_points_or_lines(mode);
+      if (points_or_lines != ice->state.prim_is_points_or_lines) {
+         ice->state.prim_is_points_or_lines = points_or_lines;
+         ice->state.dirty |= CROCUS_DIRTY_CLIP;
+      }
+   }
+
+   if (info->mode == MESA_PRIM_PATCHES &&
+       ice->state.vertices_per_patch != ice->state.patch_vertices) {
+      ice->state.vertices_per_patch = ice->state.patch_vertices;
+
+      if (screen->devinfo.ver == 8)
+         ice->state.dirty |= CROCUS_DIRTY_GEN8_VF_TOPOLOGY;
+      /* This is needed for key->input_vertices */
+      ice->state.stage_dirty |= CROCUS_STAGE_DIRTY_UNCOMPILED_TCS;
+
+      /* Flag constants dirty for gl_PatchVerticesIn if needed. */
+      const struct shader_info *tcs_info =
+         crocus_get_shader_info(ice, MESA_SHADER_TESS_CTRL);
+      if (tcs_info &&
+          BITSET_TEST(tcs_info->system_values_read, SYSTEM_VALUE_VERTICES_IN)) {
+         ice->state.stage_dirty |= CROCUS_STAGE_DIRTY_CONSTANTS_TCS;
+         ice->state.shaders[MESA_SHADER_TESS_CTRL].sysvals_need_upload = true;
+      }
+   }
+
+   const unsigned cut_index = info->primitive_restart ? info->restart_index :
+                                                        ice->state.cut_index;
+   if (ice->state.primitive_restart != info->primitive_restart ||
+       ice->state.cut_index != cut_index) {
+      if (screen->devinfo.verx10 >= 75)
+         ice->state.dirty |= CROCUS_DIRTY_GEN75_VF;
+      ice->state.primitive_restart = info->primitive_restart;
+      ice->state.cut_index = info->restart_index;
+   }
+}
+
+/**
+ * Update shader draw parameters, flagging VF packets as dirty if necessary.
+ */
+static void
+crocus_update_draw_parameters(struct crocus_context *ice,
+                              const struct pipe_draw_info *info,
+                              unsigned drawid_offset,
+                              const struct pipe_draw_indirect_info *indirect,
+                              const struct pipe_draw_start_count_bias *draw)
+{
+   bool changed = false;
+
+   if (ice->state.vs_uses_draw_params) {
+      struct crocus_state_ref *draw_params = &ice->draw.draw_params;
+
+      if (indirect && indirect->buffer) {
+         pipe_resource_reference(&draw_params->res, indirect->buffer);
+         draw_params->offset =
+            indirect->offset + (info->index_size ? 12 : 8);
+
+         changed = true;
+         ice->draw.params_valid = false;
+      } else {
+         int firstvertex = info->index_size ? draw->index_bias : draw->start;
+
+         if (!ice->draw.params_valid ||
+             ice->draw.params.firstvertex != firstvertex ||
+             ice->draw.params.baseinstance != info->start_instance) {
+
+            changed = true;
+            ice->draw.params.firstvertex = firstvertex;
+            ice->draw.params.baseinstance = info->start_instance;
+            ice->draw.params_valid = true;
+
+            u_upload_data_ref(ice->ctx.stream_uploader, 0,
+                          sizeof(ice->draw.params), 4, &ice->draw.params,
+                          &draw_params->offset, &draw_params->res);
+         }
+      }
+   }
+
+   if (ice->state.vs_uses_derived_draw_params) {
+      struct crocus_state_ref *derived_params = &ice->draw.derived_draw_params;
+      int is_indexed_draw = info->index_size ? -1 : 0;
+
+      if (ice->draw.derived_params.drawid != drawid_offset ||
+          ice->draw.derived_params.is_indexed_draw != is_indexed_draw) {
+
+         changed = true;
+         ice->draw.derived_params.drawid = drawid_offset;
+         ice->draw.derived_params.is_indexed_draw = is_indexed_draw;
+
+         u_upload_data_ref(ice->ctx.stream_uploader, 0,
+                       sizeof(ice->draw.derived_params), 4,
+                       &ice->draw.derived_params, &derived_params->offset,
+                       &derived_params->res);
+      }
+   }
+
+   if (changed) {
+      struct crocus_screen *screen = (struct crocus_screen *)ice->ctx.screen;
+      ice->state.dirty |= CROCUS_DIRTY_VERTEX_BUFFERS |
+                          CROCUS_DIRTY_VERTEX_ELEMENTS;
+      if (screen->devinfo.ver == 8)
+         ice->state.dirty |= CROCUS_DIRTY_GEN8_VF_SGVS;
+   }
+}
+
+static void
+crocus_indirect_draw_vbo(struct crocus_context *ice,
+                         const struct pipe_draw_info *dinfo,
+                         unsigned drawid_offset,
+                         const struct pipe_draw_indirect_info *dindirect,
+                         const struct pipe_draw_start_count_bias *draws)
+{
+   struct crocus_batch *batch = &ice->batches[CROCUS_BATCH_RENDER];
+   struct crocus_screen *screen = batch->screen;
+   struct pipe_draw_info info = *dinfo;
+   struct pipe_draw_indirect_info indirect = *dindirect;
+   const struct intel_device_info *devinfo = &batch->screen->devinfo;
+
+   if (devinfo->verx10 >= 75 && indirect.indirect_draw_count &&
+       ice->state.predicate == CROCUS_PREDICATE_STATE_USE_BIT) {
+      /* Upload MI_PREDICATE_RESULT to GPR15.*/
+      screen->vtbl.load_register_reg64(batch, CS_GPR(15), MI_PREDICATE_RESULT);
+   }
+
+   uint64_t orig_dirty = ice->state.dirty;
+   uint64_t orig_stage_dirty = ice->state.stage_dirty;
+
+   for (int i = 0; i < indirect.draw_count; i++) {
+      crocus_batch_maybe_flush(batch, 1500);
+      crocus_require_statebuffer_space(batch, 2400);
+
+      if (ice->state.vs_uses_draw_params ||
+	  ice->state.vs_uses_derived_draw_params)
+         crocus_update_draw_parameters(ice, &info, drawid_offset + i, &indirect, draws);
+
+      screen->vtbl.upload_render_state(ice, batch, &info, drawid_offset + i, &indirect, draws);
+
+      ice->state.dirty &= ~CROCUS_ALL_DIRTY_FOR_RENDER;
+      ice->state.stage_dirty &= ~CROCUS_ALL_STAGE_DIRTY_FOR_RENDER;
+
+      indirect.offset += indirect.stride;
+   }
+
+   if (devinfo->verx10 >= 75 && indirect.indirect_draw_count &&
+       ice->state.predicate == CROCUS_PREDICATE_STATE_USE_BIT) {
+      /* Restore MI_PREDICATE_RESULT. */
+      screen->vtbl.load_register_reg64(batch, MI_PREDICATE_RESULT, CS_GPR(15));
+   }
+
+   /* Put this back for post-draw resolves, we'll clear it again after. */
+   ice->state.dirty = orig_dirty;
+   ice->state.stage_dirty = orig_stage_dirty;
+}
+
+static void
+crocus_simple_draw_vbo(struct crocus_context *ice,
+                       const struct pipe_draw_info *draw,
+                       unsigned drawid_offset,
+                       const struct pipe_draw_indirect_info *indirect,
+                       const struct pipe_draw_start_count_bias *sc)
+{
+   struct crocus_batch *batch = &ice->batches[CROCUS_BATCH_RENDER];
+   struct crocus_screen *screen = batch->screen;
+
+   crocus_batch_maybe_flush(batch, 1500);
+   crocus_require_statebuffer_space(batch, 2400);
+
+   if (ice->state.vs_uses_draw_params ||
+       ice->state.vs_uses_derived_draw_params)
+      crocus_update_draw_parameters(ice, draw, drawid_offset, indirect, sc);
+
+   screen->vtbl.upload_render_state(ice, batch, draw, drawid_offset, indirect, sc);
+}
+
+static void
+crocus_draw_vbo_get_vertex_count(struct pipe_context *ctx,
+                                 const struct pipe_draw_info *info_in,
+                                 unsigned drawid_offset,
+                                 const struct pipe_draw_indirect_info *indirect)
+{
+   struct crocus_screen *screen = (struct crocus_screen *)ctx->screen;
+   struct pipe_draw_info info = *info_in;
+   struct pipe_draw_start_count_bias draw;
+
+   uint32_t val = screen->vtbl.get_so_offset(indirect->count_from_stream_output);
+
+   draw.start = 0;
+   draw.count = val;
+   ctx->draw_vbo(ctx, &info, drawid_offset, NULL, &draw, 1);
+}
+
+/**
+ * The pipe->draw_vbo() driver hook.  Performs a draw on the GPU.
+ */
+void
+crocus_draw_vbo(struct pipe_context *ctx,
+                const struct pipe_draw_info *info,
+                unsigned drawid_offset,
+                const struct pipe_draw_indirect_info *indirect,
+                const struct pipe_draw_start_count_bias *draws,
+                unsigned num_draws)
+{
+   if (num_draws > 1) {
+      util_draw_multi(ctx, info, drawid_offset, indirect, draws, num_draws);
+      return;
+   }
+
+   if (!indirect && (!draws[0].count || !info->instance_count))
+      return;
+
+   struct crocus_context *ice = (struct crocus_context *) ctx;
+   struct crocus_screen *screen = (struct crocus_screen*)ice->ctx.screen;
+   struct crocus_batch *batch = &ice->batches[CROCUS_BATCH_RENDER];
+
+   if (!crocus_check_conditional_render(ice))
+      return;
+
+   if (info->primitive_restart && !can_cut_index_handle_prim(ice, info)) {
+      util_draw_vbo_without_prim_restart(ctx, info, drawid_offset,
+                                         indirect, draws);
+      return;
+   }
+
+   if (screen->devinfo.verx10 < 75 &&
+       indirect && indirect->count_from_stream_output) {
+      crocus_draw_vbo_get_vertex_count(ctx, info, drawid_offset, indirect);
+      return;
+   }
+
+   /**
+    * The hardware is capable of removing dangling vertices on its own; however,
+    * prior to Gen6, we sometimes convert quads into trifans (and quad strips
+    * into tristrips), since pre-Gen6 hardware requires a GS to render quads.
+    * This function manually trims dangling vertices from a draw call involving
+    * quads so that those dangling vertices won't get drawn when we convert to
+    * trifans/tristrips.
+    */
+   if (screen->devinfo.ver < 6) {
+      if (info->mode == MESA_PRIM_QUADS || info->mode == MESA_PRIM_QUAD_STRIP) {
+         bool trim = u_trim_pipe_prim(info->mode, (unsigned *)&draws[0].count);
+         if (!trim)
+            return;
+      }
+   }
+
+   /* We can't safely re-emit 3DSTATE_SO_BUFFERS because it may zero the
+    * write offsets, changing the behavior.
+    */
+   if (INTEL_DEBUG(DEBUG_REEMIT)) {
+      ice->state.dirty |= CROCUS_ALL_DIRTY_FOR_RENDER & ~(CROCUS_DIRTY_GEN7_SO_BUFFERS | CROCUS_DIRTY_GEN6_SVBI);
+      ice->state.stage_dirty |= CROCUS_ALL_STAGE_DIRTY_FOR_RENDER;
+   }
+
+   /* Emit Sandybridge workaround flushes on every primitive, for safety. */
+   if (screen->devinfo.ver == 6)
+      crocus_emit_post_sync_nonzero_flush(batch);
+
+   crocus_update_draw_info(ice, info, draws);
+
+   if (!crocus_update_compiled_shaders(ice))
+      return;
+
+   if (ice->state.dirty & CROCUS_DIRTY_RENDER_RESOLVES_AND_FLUSHES) {
+      bool draw_aux_buffer_disabled[ELK_MAX_DRAW_BUFFERS] = { };
+      for (mesa_shader_stage stage = 0; stage < MESA_SHADER_COMPUTE; stage++) {
+         if (ice->shaders.prog[stage])
+            crocus_predraw_resolve_inputs(ice, batch, draw_aux_buffer_disabled,
+                                          stage, true);
+      }
+      crocus_predraw_resolve_framebuffer(ice, batch, draw_aux_buffer_disabled);
+   }
+
+   crocus_handle_always_flush_cache(batch);
+
+   if (indirect && indirect->buffer)
+      crocus_indirect_draw_vbo(ice, info, drawid_offset, indirect, draws);
+   else
+      crocus_simple_draw_vbo(ice, info, drawid_offset, indirect, draws);
+
+   crocus_handle_always_flush_cache(batch);
+
+   crocus_postdraw_update_resolve_tracking(ice, batch);
+
+   ice->state.dirty &= ~CROCUS_ALL_DIRTY_FOR_RENDER;
+   ice->state.stage_dirty &= ~CROCUS_ALL_STAGE_DIRTY_FOR_RENDER;
+}
+
+static void
+crocus_update_grid_size_resource(struct crocus_context *ice,
+                                 const struct pipe_grid_info *grid)
+{
+   struct crocus_state_ref *grid_ref = &ice->state.grid_size;
+   const struct crocus_compiled_shader *shader = ice->shaders.prog[MESA_SHADER_COMPUTE];
+   bool grid_needs_surface = shader->bt.used_mask[CROCUS_SURFACE_GROUP_CS_WORK_GROUPS];
+
+   if (grid->indirect) {
+      pipe_resource_reference(&grid_ref->res, grid->indirect);
+      grid_ref->offset = grid->indirect_offset;
+
+      /* Zero out the grid size so that the next non-indirect grid launch will
+       * re-upload it properly.
+       */
+      memset(ice->state.last_grid, 0, sizeof(ice->state.last_grid));
+   } else if (memcmp(ice->state.last_grid, grid->grid, sizeof(grid->grid)) != 0) {
+      memcpy(ice->state.last_grid, grid->grid, sizeof(grid->grid));
+      u_upload_data_ref(ice->ctx.const_uploader, 0, sizeof(grid->grid), 4,
+                    grid->grid, &grid_ref->offset, &grid_ref->res);
+   }
+
+   /* Skip surface upload if we don't need it or we already have one */
+   if (!grid_needs_surface)
+      return;
+
+   ice->state.stage_dirty |= CROCUS_STAGE_DIRTY_BINDINGS_CS;
+}
+
+
+void
+crocus_launch_grid(struct pipe_context *ctx, const struct pipe_grid_info *grid)
+{
+   struct crocus_context *ice = (struct crocus_context *) ctx;
+   struct crocus_batch *batch = &ice->batches[CROCUS_BATCH_COMPUTE];
+   struct crocus_screen *screen = batch->screen;
+
+   if (!crocus_check_conditional_render(ice))
+      return;
+
+   if (INTEL_DEBUG(DEBUG_REEMIT)) {
+      ice->state.dirty |= CROCUS_ALL_DIRTY_FOR_COMPUTE;
+      ice->state.stage_dirty |= CROCUS_ALL_STAGE_DIRTY_FOR_COMPUTE;
+   }
+
+   /* We can't do resolves on the compute engine, so awkwardly, we have to
+    * do them on the render batch...
+    */
+   if (ice->state.dirty & CROCUS_DIRTY_COMPUTE_RESOLVES_AND_FLUSHES) {
+      crocus_predraw_resolve_inputs(ice, &ice->batches[CROCUS_BATCH_RENDER], NULL,
+                                    MESA_SHADER_COMPUTE, false);
+   }
+
+   crocus_batch_maybe_flush(batch, 1500);
+   crocus_require_statebuffer_space(batch, 2500);
+   crocus_update_compiled_compute_shader(ice);
+
+   if (memcmp(ice->state.last_block, grid->block, sizeof(grid->block)) != 0) {
+      memcpy(ice->state.last_block, grid->block, sizeof(grid->block));
+      ice->state.stage_dirty |= CROCUS_STAGE_DIRTY_CONSTANTS_CS;
+      ice->state.shaders[MESA_SHADER_COMPUTE].sysvals_need_upload = true;
+   }
+
+   crocus_update_grid_size_resource(ice, grid);
+
+   if (ice->state.compute_predicate) {
+      screen->vtbl.emit_compute_predicate(batch);
+      ice->state.compute_predicate = NULL;
+   }
+
+   crocus_handle_always_flush_cache(batch);
+
+   screen->vtbl.upload_compute_state(ice, batch, grid);
+
+   crocus_handle_always_flush_cache(batch);
+
+   ice->state.dirty &= ~CROCUS_ALL_DIRTY_FOR_COMPUTE;
+   ice->state.stage_dirty &= ~CROCUS_ALL_STAGE_DIRTY_FOR_COMPUTE;
+
+   /* Note: since compute shaders can't access the framebuffer, there's
+    * no need to call crocus_postdraw_update_resolve_tracking.
+    */
+}

src/gallium/drivers/crocus/crocus_fence.c

@@ -0,0 +1,576 @@
+/*
+ * Copyright © 2018 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+/**
+ * @file crocus_fence.c
+ *
+ * Fences for driver and IPC serialisation, scheduling and synchronisation.
+ */
+
+#include "util/u_inlines.h"
+#include "intel/common/intel_gem.h"
+
+#include "crocus_batch.h"
+#include "crocus_bufmgr.h"
+#include "crocus_context.h"
+#include "crocus_fence.h"
+#include "crocus_screen.h"
+
+static uint32_t
+gem_syncobj_create(int fd, uint32_t flags)
+{
+   struct drm_syncobj_create args = {
+      .flags = flags,
+   };
+
+   intel_ioctl(fd, DRM_IOCTL_SYNCOBJ_CREATE, &args);
+
+   return args.handle;
+}
+
+static void
+gem_syncobj_destroy(int fd, uint32_t handle)
+{
+   struct drm_syncobj_destroy args = {
+      .handle = handle,
+   };
+
+   intel_ioctl(fd, DRM_IOCTL_SYNCOBJ_DESTROY, &args);
+}
+
+/**
+ * Make a new sync-point.
+ */
+struct crocus_syncobj *
+crocus_create_syncobj(struct crocus_screen *screen)
+{
+   struct crocus_syncobj *syncobj = malloc(sizeof(*syncobj));
+
+   if (!syncobj)
+      return NULL;
+
+   syncobj->handle = gem_syncobj_create(screen->fd, 0);
+   assert(syncobj->handle);
+
+   pipe_reference_init(&syncobj->ref, 1);
+
+   return syncobj;
+}
+
+void
+crocus_syncobj_destroy(struct crocus_screen *screen,
+                       struct crocus_syncobj *syncobj)
+{
+   gem_syncobj_destroy(screen->fd, syncobj->handle);
+   free(syncobj);
+}
+
+/**
+ * Add a sync-point to the batch, with the given flags.
+ *
+ * \p flags   One of I915_EXEC_FENCE_WAIT or I915_EXEC_FENCE_SIGNAL.
+ */
+void
+crocus_batch_add_syncobj(struct crocus_batch *batch,
+                         struct crocus_syncobj *syncobj, unsigned flags)
+{
+   struct drm_i915_gem_exec_fence *fence =
+      util_dynarray_grow(&batch->exec_fences, struct drm_i915_gem_exec_fence, 1);
+
+   *fence = (struct drm_i915_gem_exec_fence){
+      .handle = syncobj->handle,
+      .flags = flags,
+   };
+
+   struct crocus_syncobj **store =
+      util_dynarray_grow(&batch->syncobjs, struct crocus_syncobj *, 1);
+
+   *store = NULL;
+   crocus_syncobj_reference(batch->screen, store, syncobj);
+}
+
+/**
+ * Walk through a batch's dependencies (any I915_EXEC_FENCE_WAIT syncobjs)
+ * and unreference any which have already passed.
+ *
+ * Sometimes the compute batch is seldom used, and accumulates references
+ * to stale render batches that are no longer of interest, so we can free
+ * those up.
+ */
+static void
+clear_stale_syncobjs(struct crocus_batch *batch)
+{
+   struct crocus_screen *screen = batch->screen;
+
+   int n = util_dynarray_num_elements(&batch->syncobjs, struct crocus_syncobj *);
+
+   assert(n == util_dynarray_num_elements(&batch->exec_fences,
+                                          struct drm_i915_gem_exec_fence));
+
+   /* Skip the first syncobj, as it's the signalling one. */
+   for (int i = n - 1; i > 0; i--) {
+      struct crocus_syncobj **syncobj =
+         util_dynarray_element(&batch->syncobjs, struct crocus_syncobj *, i);
+      struct drm_i915_gem_exec_fence *fence =
+         util_dynarray_element(&batch->exec_fences,
+                               struct drm_i915_gem_exec_fence, i);
+      assert(fence->flags & I915_EXEC_FENCE_WAIT);
+
+      if (crocus_wait_syncobj(&screen->base, *syncobj, 0))
+         continue;
+
+      /* This sync object has already passed, there's no need to continue
+       * marking it as a dependency; we can stop holding on to the reference.
+       */
+      crocus_syncobj_reference(screen, syncobj, NULL);
+
+      /* Remove it from the lists; move the last element here. */
+      struct crocus_syncobj **nth_syncobj =
+         util_dynarray_pop_ptr(&batch->syncobjs, struct crocus_syncobj *);
+      struct drm_i915_gem_exec_fence *nth_fence =
+         util_dynarray_pop_ptr(&batch->exec_fences,
+                               struct drm_i915_gem_exec_fence);
+
+      if (syncobj != nth_syncobj) {
+         *syncobj = *nth_syncobj;
+         memcpy(fence, nth_fence, sizeof(*fence));
+      }
+   }
+}
+
+/* ------------------------------------------------------------------- */
+
+struct pipe_fence_handle {
+   struct pipe_reference ref;
+
+   struct pipe_context *unflushed_ctx;
+
+   struct crocus_fine_fence *fine[CROCUS_BATCH_COUNT];
+};
+
+static void
+crocus_fence_destroy(struct pipe_screen *p_screen,
+                     struct pipe_fence_handle *fence)
+{
+   struct crocus_screen *screen = (struct crocus_screen *)p_screen;
+
+   for (unsigned i = 0; i < ARRAY_SIZE(fence->fine); i++)
+      crocus_fine_fence_reference(screen, &fence->fine[i], NULL);
+
+   free(fence);
+}
+
+static void
+crocus_fence_reference(struct pipe_screen *p_screen,
+                       struct pipe_fence_handle **dst,
+                       struct pipe_fence_handle *src)
+{
+   if (pipe_reference(&(*dst)->ref, &src->ref))
+      crocus_fence_destroy(p_screen, *dst);
+
+   *dst = src;
+}
+
+bool
+crocus_wait_syncobj(struct pipe_screen *p_screen,
+                    struct crocus_syncobj *syncobj, int64_t timeout_nsec)
+{
+   if (!syncobj)
+      return false;
+
+   struct crocus_screen *screen = (struct crocus_screen *)p_screen;
+   struct drm_syncobj_wait args = {
+      .handles = (uintptr_t)&syncobj->handle,
+      .count_handles = 1,
+      .timeout_nsec = timeout_nsec,
+   };
+   return intel_ioctl(screen->fd, DRM_IOCTL_SYNCOBJ_WAIT, &args);
+}
+
+static void
+crocus_fence_flush(struct pipe_context *ctx,
+                   struct pipe_fence_handle **out_fence, unsigned flags)
+{
+   struct crocus_screen *screen = (void *)ctx->screen;
+   struct crocus_context *ice = (struct crocus_context *)ctx;
+
+   const bool deferred = flags & PIPE_FLUSH_DEFERRED;
+
+   if (!deferred) {
+      for (unsigned i = 0; i < ice->batch_count; i++)
+         crocus_batch_flush(&ice->batches[i]);
+   }
+
+   if (!out_fence)
+      return;
+
+   struct pipe_fence_handle *fence = calloc(1, sizeof(*fence));
+   if (!fence)
+      return;
+
+   pipe_reference_init(&fence->ref, 1);
+
+   if (deferred)
+      fence->unflushed_ctx = ctx;
+
+   for (unsigned b = 0; b < ice->batch_count; b++) {
+      struct crocus_batch *batch = &ice->batches[b];
+
+      if (deferred && crocus_batch_bytes_used(batch) > 0) {
+         struct crocus_fine_fence *fine =
+            crocus_fine_fence_new(batch, CROCUS_FENCE_BOTTOM_OF_PIPE);
+         crocus_fine_fence_reference(screen, &fence->fine[b], fine);
+         crocus_fine_fence_reference(screen, &fine, NULL);
+      } else {
+         /* This batch has no commands queued up (perhaps we just flushed,
+          * or all the commands are on the other batch).  Wait for the last
+          * syncobj on this engine - unless it's already finished by now.
+          */
+         if (crocus_fine_fence_signaled(batch->last_fence))
+            continue;
+
+         crocus_fine_fence_reference(screen, &fence->fine[b],
+                                     batch->last_fence);
+      }
+   }
+
+   crocus_fence_reference(ctx->screen, out_fence, NULL);
+   *out_fence = fence;
+}
+
+static void
+crocus_fence_await(struct pipe_context *ctx, struct pipe_fence_handle *fence, uint64_t value)
+{
+   struct crocus_context *ice = (struct crocus_context *)ctx;
+   assert(!value);
+
+   /* Unflushed fences from the same context are no-ops. */
+   if (ctx && ctx == fence->unflushed_ctx)
+      return;
+
+   for (unsigned i = 0; i < ARRAY_SIZE(fence->fine); i++) {
+      struct crocus_fine_fence *fine = fence->fine[i];
+
+      if (crocus_fine_fence_signaled(fine))
+         continue;
+
+      for (unsigned b = 0; b < ice->batch_count; b++) {
+         struct crocus_batch *batch = &ice->batches[b];
+
+         /* We're going to make any future work in this batch wait for our
+          * fence to have gone by.  But any currently queued work doesn't
+          * need to wait.  Flush the batch now, so it can happen sooner.
+          */
+         crocus_batch_flush(batch);
+
+         /* Before adding a new reference, clean out any stale ones. */
+         clear_stale_syncobjs(batch);
+
+         crocus_batch_add_syncobj(batch, fine->syncobj, I915_EXEC_FENCE_WAIT);
+      }
+   }
+}
+
+#define NSEC_PER_SEC (1000 * USEC_PER_SEC)
+#define USEC_PER_SEC (1000 * MSEC_PER_SEC)
+#define MSEC_PER_SEC (1000)
+
+static uint64_t
+gettime_ns(void)
+{
+   struct timespec current;
+   clock_gettime(CLOCK_MONOTONIC, &current);
+   return (uint64_t)current.tv_sec * NSEC_PER_SEC + current.tv_nsec;
+}
+
+static uint64_t
+rel2abs(uint64_t timeout)
+{
+   if (timeout == 0)
+      return 0;
+
+   uint64_t current_time = gettime_ns();
+   uint64_t max_timeout = (uint64_t)INT64_MAX - current_time;
+
+   timeout = MIN2(max_timeout, timeout);
+
+   return current_time + timeout;
+}
+
+static bool
+crocus_fence_finish(struct pipe_screen *p_screen, struct pipe_context *ctx,
+                    struct pipe_fence_handle *fence, uint64_t timeout)
+{
+   ctx = threaded_context_unwrap_sync(ctx);
+   struct crocus_context *ice = (struct crocus_context *)ctx;
+   struct crocus_screen *screen = (struct crocus_screen *)p_screen;
+
+   /* If we created the fence with PIPE_FLUSH_DEFERRED, we may not have
+    * flushed yet.  Check if our syncobj is the current batch's signalling
+    * syncobj - if so, we haven't flushed and need to now.
+    *
+    * The Gallium docs mention that a flush will occur if \p ctx matches
+    * the context the fence was created with.  It may be NULL, so we check
+    * that it matches first.
+    */
+   if (ctx && ctx == fence->unflushed_ctx) {
+      for (unsigned i = 0; i < ice->batch_count; i++) {
+         struct crocus_fine_fence *fine = fence->fine[i];
+
+         if (crocus_fine_fence_signaled(fine))
+            continue;
+
+         if (fine->syncobj == crocus_batch_get_signal_syncobj(&ice->batches[i]))
+            crocus_batch_flush(&ice->batches[i]);
+      }
+
+      /* The fence is no longer deferred. */
+      fence->unflushed_ctx = NULL;
+   }
+
+   unsigned int handle_count = 0;
+   uint32_t handles[ARRAY_SIZE(fence->fine)];
+   for (unsigned i = 0; i < ARRAY_SIZE(fence->fine); i++) {
+      struct crocus_fine_fence *fine = fence->fine[i];
+
+      if (crocus_fine_fence_signaled(fine))
+         continue;
+
+      handles[handle_count++] = fine->syncobj->handle;
+   }
+
+   if (handle_count == 0)
+      return true;
+
+   struct drm_syncobj_wait args = {
+      .handles = (uintptr_t)handles,
+      .count_handles = handle_count,
+      .timeout_nsec = rel2abs(timeout),
+      .flags = DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL
+   };
+   if (fence->unflushed_ctx) {
+      /* This fence had a deferred flush from another context.  We can't
+       * safely flush it here, because the context might be bound to a
+       * different thread, and poking at its internals wouldn't be safe.
+       *
+       * Instead, use the WAIT_FOR_SUBMIT flag to block and hope that
+       * another thread submits the work.
+       */
+      args.flags |= DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT;
+   }
+   return intel_ioctl(screen->fd, DRM_IOCTL_SYNCOBJ_WAIT, &args) == 0;
+}
+
+#ifndef SYNC_IOC_MAGIC
+/* duplicated from linux/sync_file.h to avoid build-time dependency
+ * on new (v4.7) kernel headers.  Once distro's are mostly using
+ * something newer than v4.7 drop this and #include <linux/sync_file.h>
+ * instead.
+ */
+struct sync_merge_data {
+   char name[32];
+   __s32 fd2;
+   __s32 fence;
+   __u32 flags;
+   __u32 pad;
+};
+
+#define SYNC_IOC_MAGIC '>'
+#define SYNC_IOC_MERGE _IOWR(SYNC_IOC_MAGIC, 3, struct sync_merge_data)
+#endif
+
+static int
+sync_merge_fd(int sync_fd, int new_fd)
+{
+   if (sync_fd == -1)
+      return new_fd;
+
+   if (new_fd == -1)
+      return sync_fd;
+
+   struct sync_merge_data args = {
+      .name = "crocus fence",
+      .fd2 = new_fd,
+      .fence = -1,
+   };
+
+   intel_ioctl(sync_fd, SYNC_IOC_MERGE, &args);
+   close(new_fd);
+   close(sync_fd);
+
+   return args.fence;
+}
+
+static int
+crocus_fence_get_fd(struct pipe_screen *p_screen,
+                    struct pipe_fence_handle *fence)
+{
+   struct crocus_screen *screen = (struct crocus_screen *)p_screen;
+   int fd = -1;
+
+   /* Deferred fences aren't supported. */
+   if (fence->unflushed_ctx)
+      return -1;
+
+   for (unsigned i = 0; i < ARRAY_SIZE(fence->fine); i++) {
+      struct crocus_fine_fence *fine = fence->fine[i];
+
+      if (crocus_fine_fence_signaled(fine))
+         continue;
+
+      struct drm_syncobj_handle args = {
+         .handle = fine->syncobj->handle,
+         .flags = DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE,
+         .fd = -1,
+      };
+
+      intel_ioctl(screen->fd, DRM_IOCTL_SYNCOBJ_HANDLE_TO_FD, &args);
+      fd = sync_merge_fd(fd, args.fd);
+   }
+
+   if (fd == -1) {
+      /* Our fence has no syncobj's recorded.  This means that all of the
+       * batches had already completed, their syncobj's had been signalled,
+       * and so we didn't bother to record them.  But we're being asked to
+       * export such a fence.  So export a dummy already-signalled syncobj.
+       */
+      struct drm_syncobj_handle args = {
+         .flags = DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE,
+         .fd = -1,
+      };
+
+      args.handle = gem_syncobj_create(screen->fd, DRM_SYNCOBJ_CREATE_SIGNALED);
+      intel_ioctl(screen->fd, DRM_IOCTL_SYNCOBJ_HANDLE_TO_FD, &args);
+      gem_syncobj_destroy(screen->fd, args.handle);
+      return args.fd;
+   }
+
+   return fd;
+}
+
+static void
+crocus_fence_create_fd(struct pipe_context *ctx, struct pipe_fence_handle **out,
+                       int fd, enum pipe_fd_type type)
+{
+   assert(type == PIPE_FD_TYPE_NATIVE_SYNC || type == PIPE_FD_TYPE_SYNCOBJ);
+
+   struct crocus_screen *screen = (struct crocus_screen *)ctx->screen;
+   struct drm_syncobj_handle args = {
+      .fd = fd,
+   };
+
+   if (type == PIPE_FD_TYPE_NATIVE_SYNC) {
+      args.flags = DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE;
+      args.handle = gem_syncobj_create(screen->fd, DRM_SYNCOBJ_CREATE_SIGNALED);
+   }
+
+   if (intel_ioctl(screen->fd, DRM_IOCTL_SYNCOBJ_FD_TO_HANDLE, &args) == -1) {
+      fprintf(stderr, "DRM_IOCTL_SYNCOBJ_FD_TO_HANDLE failed: %s\n",
+              strerror(errno));
+      if (type == PIPE_FD_TYPE_NATIVE_SYNC)
+         gem_syncobj_destroy(screen->fd, args.handle);
+      *out = NULL;
+      return;
+   }
+
+   struct crocus_syncobj *syncobj = malloc(sizeof(*syncobj));
+   if (!syncobj) {
+      *out = NULL;
+      return;
+   }
+   syncobj->handle = args.handle;
+   pipe_reference_init(&syncobj->ref, 1);
+
+   struct crocus_fine_fence *fine = calloc(1, sizeof(*fine));
+   if (!fine) {
+      free(syncobj);
+      *out = NULL;
+      return;
+   }
+
+   static const uint32_t zero = 0;
+
+   /* Fences work in terms of crocus_fine_fence, but we don't actually have a
+    * seqno for an imported fence.  So, create a fake one which always
+    * returns as 'not signaled' so we fall back to using the sync object.
+    */
+   fine->seqno = UINT32_MAX;
+   fine->map = &zero;
+   fine->syncobj = syncobj;
+   fine->flags = CROCUS_FENCE_END;
+   pipe_reference_init(&fine->reference, 1);
+
+   struct pipe_fence_handle *fence = calloc(1, sizeof(*fence));
+   if (!fence) {
+      free(fine);
+      free(syncobj);
+      *out = NULL;
+      return;
+   }
+   pipe_reference_init(&fence->ref, 1);
+   fence->fine[0] = fine;
+
+   *out = fence;
+}
+
+static void
+crocus_fence_signal(struct pipe_context *ctx, struct pipe_fence_handle *fence, uint64_t value)
+{
+   struct crocus_context *ice = (struct crocus_context *)ctx;
+   assert(!value);
+
+   if (ctx == fence->unflushed_ctx)
+      return;
+
+   for (unsigned b = 0; b < ice->batch_count; b++) {
+      for (unsigned i = 0; i < ARRAY_SIZE(fence->fine); i++) {
+         struct crocus_fine_fence *fine = fence->fine[i];
+
+         /* already signaled fence skipped */
+         if (crocus_fine_fence_signaled(fine))
+            continue;
+
+         ice->batches[b].contains_fence_signal = true;
+         crocus_batch_add_syncobj(&ice->batches[b], fine->syncobj,
+                                  I915_EXEC_FENCE_SIGNAL);
+      }
+      if (ice->batches[b].contains_fence_signal)
+         crocus_batch_flush(&ice->batches[b]);
+   }
+}
+
+void
+crocus_init_screen_fence_functions(struct pipe_screen *screen)
+{
+   screen->fence_reference = crocus_fence_reference;
+   screen->fence_finish = crocus_fence_finish;
+   screen->fence_get_fd = crocus_fence_get_fd;
+}
+
+void
+crocus_init_context_fence_functions(struct pipe_context *ctx)
+{
+   ctx->flush = crocus_fence_flush;
+   ctx->create_fence_fd = crocus_fence_create_fd;
+   ctx->fence_server_sync = crocus_fence_await;
+   ctx->fence_server_signal = crocus_fence_signal;
+}

src/gallium/drivers/crocus/crocus_fence.h

@@ -0,0 +1,60 @@
+/*
+ * Copyright © 2018 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifndef CROCUS_FENCE_H
+#define CROCUS_FENCE_H
+
+#include "util/u_inlines.h"
+
+struct pipe_screen;
+struct crocus_screen;
+struct crocus_batch;
+
+struct crocus_syncobj {
+   struct pipe_reference ref;
+   uint32_t handle;
+};
+
+void crocus_init_context_fence_functions(struct pipe_context *ctx);
+void crocus_init_screen_fence_functions(struct pipe_screen *screen);
+
+struct crocus_syncobj *crocus_create_syncobj(struct crocus_screen *screen);
+void crocus_syncobj_destroy(struct crocus_screen *, struct crocus_syncobj *);
+void crocus_batch_add_syncobj(struct crocus_batch *batch,
+                              struct crocus_syncobj *syncobj,
+                              unsigned flags);
+bool crocus_wait_syncobj(struct pipe_screen *screen,
+                         struct crocus_syncobj *syncobj,
+                         int64_t timeout_nsec);
+static inline void
+crocus_syncobj_reference(struct crocus_screen *screen,
+                         struct crocus_syncobj **dst,
+                         struct crocus_syncobj *src)
+{
+   if (pipe_reference(&(*dst)->ref, &src->ref))
+      crocus_syncobj_destroy(screen, *dst);
+
+   *dst = src;
+}
+
+#endif

src/gallium/drivers/crocus/crocus_fine_fence.c

@@ -0,0 +1,85 @@
+#include "crocus_context.h"
+#include "crocus_fine_fence.h"
+#include "util/u_upload_mgr.h"
+
+static void
+crocus_fine_fence_reset(struct crocus_batch *batch)
+{
+   u_upload_alloc_ref(batch->fine_fences.uploader,
+                  0, sizeof(uint64_t), sizeof(uint64_t),
+                  &batch->fine_fences.ref.offset, &batch->fine_fences.ref.res,
+                  (void **)&batch->fine_fences.map);
+   WRITE_ONCE(*batch->fine_fences.map, 0);
+   batch->fine_fences.next++;
+}
+
+void
+crocus_fine_fence_init(struct crocus_batch *batch)
+{
+   batch->fine_fences.ref.res = NULL;
+   batch->fine_fences.next = 0;
+   if (batch_has_fine_fence(batch))
+      crocus_fine_fence_reset(batch);
+}
+
+static uint32_t
+crocus_fine_fence_next(struct crocus_batch *batch)
+{
+   if (!batch_has_fine_fence(batch))
+      return UINT32_MAX;
+
+   uint32_t seqno = batch->fine_fences.next++;
+
+   if (batch->fine_fences.next == 0)
+      crocus_fine_fence_reset(batch);
+
+   return seqno;
+}
+
+void
+crocus_fine_fence_destroy(struct crocus_screen *screen,
+                          struct crocus_fine_fence *fine)
+{
+   crocus_syncobj_reference(screen, &fine->syncobj, NULL);
+   pipe_resource_reference(&fine->ref.res, NULL);
+   free(fine);
+}
+
+struct crocus_fine_fence *
+crocus_fine_fence_new(struct crocus_batch *batch, unsigned flags)
+{
+   struct crocus_fine_fence *fine = calloc(1, sizeof(*fine));
+   if (!fine)
+      return NULL;
+
+   pipe_reference_init(&fine->reference, 1);
+
+   fine->seqno = crocus_fine_fence_next(batch);
+
+   crocus_syncobj_reference(batch->screen, &fine->syncobj,
+                            crocus_batch_get_signal_syncobj(batch));
+
+   if (!batch_has_fine_fence(batch))
+      return fine;
+   pipe_resource_reference(&fine->ref.res, batch->fine_fences.ref.res);
+   fine->ref.offset = batch->fine_fences.ref.offset;
+   fine->map = batch->fine_fences.map;
+   fine->flags = flags;
+
+   unsigned pc;
+   if (flags & CROCUS_FENCE_TOP_OF_PIPE) {
+      pc = PIPE_CONTROL_WRITE_IMMEDIATE | PIPE_CONTROL_CS_STALL;
+   } else {
+      pc = PIPE_CONTROL_WRITE_IMMEDIATE |
+           PIPE_CONTROL_RENDER_TARGET_FLUSH |
+           PIPE_CONTROL_TILE_CACHE_FLUSH |
+           PIPE_CONTROL_DEPTH_CACHE_FLUSH |
+           PIPE_CONTROL_DATA_CACHE_FLUSH;
+   }
+   crocus_emit_pipe_control_write(batch, "fence: fine", pc,
+                                  crocus_resource_bo(fine->ref.res),
+                                  fine->ref.offset,
+                                  fine->seqno);
+
+   return fine;
+}

src/gallium/drivers/crocus/crocus_fine_fence.h

@@ -0,0 +1,109 @@
+/*
+ * Copyright © 2020 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifndef CROCUS_FINE_FENCE_DOT_H
+#define CROCUS_FINE_FENCE_DOT_H
+
+#include <stdbool.h>
+#include <stdint.h>
+
+#include "crocus_screen.h"
+#include "crocus_resource.h"
+
+/**
+ * A lightweight sequence number fence.
+ *
+ * We emit PIPE_CONTROLs inside a batch (possibly in the middle)
+ * which update a monotonically increasing, 32-bit counter.  We
+ * can then check if that moment has passed by either:
+ *
+ * 1. Checking on the CPU by snooping on the DWord via a coherent map
+ *
+ * 2. Blocking on the GPU with MI_SEMAPHORE_WAIT from a second batch
+ *    (relying on mid-batch preemption to switch GPU execution to the
+ *    batch that writes it).
+ */
+struct crocus_fine_fence {
+   struct pipe_reference reference;
+
+   /** Buffer where the seqno lives */
+   struct crocus_state_ref ref;
+
+   /** Coherent CPU map of the buffer containing the seqno DWord. */
+   const uint32_t *map;
+
+   /**
+    * A drm_syncobj pointing which will be signaled at the end of the
+    * batch which writes this seqno.  This can be used to block until
+    * the seqno has definitely passed (but may wait longer than necessary).
+    */
+   struct crocus_syncobj *syncobj;
+
+#define CROCUS_FENCE_BOTTOM_OF_PIPE 0x0 /**< Written by bottom-of-pipe flush */
+#define CROCUS_FENCE_TOP_OF_PIPE    0x1 /**< Written by top-of-pipe flush */
+#define CROCUS_FENCE_END            0x2 /**< Written at the end of a batch */
+
+   /** Information about the type of flush involved (see CROCUS_FENCE_*) */
+   uint32_t flags;
+
+   /**
+    * Sequence number expected to be written by the flush we inserted
+    * when creating this fence.  The crocus_fine_fence is 'signaled' when *@map
+    * (written by the flush on the GPU) is greater-than-or-equal to @seqno.
+    */
+   uint32_t seqno;
+};
+
+void crocus_fine_fence_init(struct crocus_batch *batch);
+
+struct crocus_fine_fence *crocus_fine_fence_new(struct crocus_batch *batch,
+                                                unsigned flags);
+
+void crocus_fine_fence_destroy(struct crocus_screen *screen,
+                               struct crocus_fine_fence *sq);
+
+static inline void
+crocus_fine_fence_reference(struct crocus_screen *screen,
+                            struct crocus_fine_fence **dst,
+                            struct crocus_fine_fence *src)
+{
+   if (pipe_reference(&(*dst)->reference, &src->reference))
+      crocus_fine_fence_destroy(screen, *dst);
+
+   *dst = src;
+}
+
+/**
+ * Return true if this seqno has passed.
+ *
+ * NULL is considered signaled.
+ */
+static inline bool
+crocus_fine_fence_signaled(const struct crocus_fine_fence *sq)
+{
+   if (sq && !sq->map)
+      return false;
+   return !sq || (READ_ONCE(*sq->map) >= sq->seqno);
+}
+
+#endif

src/gallium/drivers/crocus/crocus_formats.c

@@ -0,0 +1,585 @@
+/*
+ * Copyright © 2017 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+/**
+ * @file crocus_formats.c
+ *
+ * Converts Gallium formats (PIPE_FORMAT_*) to hardware ones (ISL_FORMAT_*).
+ * Provides information about which formats support what features.
+ */
+
+#include "util/bitscan.h"
+#include "util/macros.h"
+#include "util/format/u_format.h"
+
+#include "crocus_resource.h"
+#include "crocus_screen.h"
+
+static enum isl_format
+crocus_isl_format_for_pipe_format(enum pipe_format pf)
+{
+   static const enum isl_format table[PIPE_FORMAT_COUNT] = {
+      [0 ... PIPE_FORMAT_COUNT-1] = ISL_FORMAT_UNSUPPORTED,
+
+      [PIPE_FORMAT_B8G8R8A8_UNORM]          = ISL_FORMAT_B8G8R8A8_UNORM,
+      [PIPE_FORMAT_B8G8R8X8_UNORM]          = ISL_FORMAT_B8G8R8X8_UNORM,
+      [PIPE_FORMAT_B5G5R5A1_UNORM]          = ISL_FORMAT_B5G5R5A1_UNORM,
+      [PIPE_FORMAT_B4G4R4A4_UNORM]          = ISL_FORMAT_B4G4R4A4_UNORM,
+      [PIPE_FORMAT_B5G6R5_UNORM]            = ISL_FORMAT_B5G6R5_UNORM,
+      [PIPE_FORMAT_R10G10B10A2_UNORM]       = ISL_FORMAT_R10G10B10A2_UNORM,
+
+      [PIPE_FORMAT_Z16_UNORM]               = ISL_FORMAT_R16_UNORM,
+      [PIPE_FORMAT_Z32_UNORM]               = ISL_FORMAT_R32_UNORM,
+      [PIPE_FORMAT_Z32_FLOAT]               = ISL_FORMAT_R32_FLOAT,
+
+      /* We translate the combined depth/stencil formats to depth only here */
+      [PIPE_FORMAT_Z24_UNORM_S8_UINT]       = ISL_FORMAT_R24_UNORM_X8_TYPELESS,
+      [PIPE_FORMAT_Z24X8_UNORM]             = ISL_FORMAT_R24_UNORM_X8_TYPELESS,
+      [PIPE_FORMAT_Z32_FLOAT_S8X24_UINT]    = ISL_FORMAT_R32_FLOAT,
+
+      [PIPE_FORMAT_S8_UINT]                 = ISL_FORMAT_R8_UINT,
+      [PIPE_FORMAT_X24S8_UINT]              = ISL_FORMAT_R8_UINT,
+      [PIPE_FORMAT_X32_S8X24_UINT]          = ISL_FORMAT_R8_UINT,
+
+      [PIPE_FORMAT_R64_FLOAT]               = ISL_FORMAT_R64_FLOAT,
+      [PIPE_FORMAT_R64G64_FLOAT]            = ISL_FORMAT_R64G64_FLOAT,
+      [PIPE_FORMAT_R64G64B64_FLOAT]         = ISL_FORMAT_R64G64B64_FLOAT,
+      [PIPE_FORMAT_R64G64B64A64_FLOAT]      = ISL_FORMAT_R64G64B64A64_FLOAT,
+      [PIPE_FORMAT_R32_FLOAT]               = ISL_FORMAT_R32_FLOAT,
+      [PIPE_FORMAT_R32G32_FLOAT]            = ISL_FORMAT_R32G32_FLOAT,
+      [PIPE_FORMAT_R32G32B32_FLOAT]         = ISL_FORMAT_R32G32B32_FLOAT,
+      [PIPE_FORMAT_R32G32B32A32_FLOAT]      = ISL_FORMAT_R32G32B32A32_FLOAT,
+      [PIPE_FORMAT_R32_UNORM]               = ISL_FORMAT_R32_UNORM,
+      [PIPE_FORMAT_R32G32_UNORM]            = ISL_FORMAT_R32G32_UNORM,
+      [PIPE_FORMAT_R32G32B32_UNORM]         = ISL_FORMAT_R32G32B32_UNORM,
+      [PIPE_FORMAT_R32G32B32A32_UNORM]      = ISL_FORMAT_R32G32B32A32_UNORM,
+      [PIPE_FORMAT_R32_USCALED]             = ISL_FORMAT_R32_USCALED,
+      [PIPE_FORMAT_R32G32_USCALED]          = ISL_FORMAT_R32G32_USCALED,
+      [PIPE_FORMAT_R32G32B32_USCALED]       = ISL_FORMAT_R32G32B32_USCALED,
+      [PIPE_FORMAT_R32G32B32A32_USCALED]    = ISL_FORMAT_R32G32B32A32_USCALED,
+      [PIPE_FORMAT_R32_SNORM]               = ISL_FORMAT_R32_SNORM,
+      [PIPE_FORMAT_R32G32_SNORM]            = ISL_FORMAT_R32G32_SNORM,
+      [PIPE_FORMAT_R32G32B32_SNORM]         = ISL_FORMAT_R32G32B32_SNORM,
+      [PIPE_FORMAT_R32G32B32A32_SNORM]      = ISL_FORMAT_R32G32B32A32_SNORM,
+      [PIPE_FORMAT_R32_SSCALED]             = ISL_FORMAT_R32_SSCALED,
+      [PIPE_FORMAT_R32G32_SSCALED]          = ISL_FORMAT_R32G32_SSCALED,
+      [PIPE_FORMAT_R32G32B32_SSCALED]       = ISL_FORMAT_R32G32B32_SSCALED,
+      [PIPE_FORMAT_R32G32B32A32_SSCALED]    = ISL_FORMAT_R32G32B32A32_SSCALED,
+      [PIPE_FORMAT_R16_UNORM]               = ISL_FORMAT_R16_UNORM,
+      [PIPE_FORMAT_R16G16_UNORM]            = ISL_FORMAT_R16G16_UNORM,
+      [PIPE_FORMAT_R16G16B16_UNORM]         = ISL_FORMAT_R16G16B16_UNORM,
+      [PIPE_FORMAT_R16G16B16A16_UNORM]      = ISL_FORMAT_R16G16B16A16_UNORM,
+      [PIPE_FORMAT_R16_USCALED]             = ISL_FORMAT_R16_USCALED,
+      [PIPE_FORMAT_R16G16_USCALED]          = ISL_FORMAT_R16G16_USCALED,
+      [PIPE_FORMAT_R16G16B16_USCALED]       = ISL_FORMAT_R16G16B16_USCALED,
+      [PIPE_FORMAT_R16G16B16A16_USCALED]    = ISL_FORMAT_R16G16B16A16_USCALED,
+      [PIPE_FORMAT_R16_SNORM]               = ISL_FORMAT_R16_SNORM,
+      [PIPE_FORMAT_R16G16_SNORM]            = ISL_FORMAT_R16G16_SNORM,
+      [PIPE_FORMAT_R16G16B16_SNORM]         = ISL_FORMAT_R16G16B16_SNORM,
+      [PIPE_FORMAT_R16G16B16A16_SNORM]      = ISL_FORMAT_R16G16B16A16_SNORM,
+      [PIPE_FORMAT_R16_SSCALED]             = ISL_FORMAT_R16_SSCALED,
+      [PIPE_FORMAT_R16G16_SSCALED]          = ISL_FORMAT_R16G16_SSCALED,
+      [PIPE_FORMAT_R16G16B16_SSCALED]       = ISL_FORMAT_R16G16B16_SSCALED,
+      [PIPE_FORMAT_R16G16B16A16_SSCALED]    = ISL_FORMAT_R16G16B16A16_SSCALED,
+      [PIPE_FORMAT_R8_UNORM]                = ISL_FORMAT_R8_UNORM,
+      [PIPE_FORMAT_R8G8_UNORM]              = ISL_FORMAT_R8G8_UNORM,
+      [PIPE_FORMAT_R8G8B8_UNORM]            = ISL_FORMAT_R8G8B8_UNORM,
+      [PIPE_FORMAT_R8G8B8A8_UNORM]          = ISL_FORMAT_R8G8B8A8_UNORM,
+      [PIPE_FORMAT_R8_USCALED]              = ISL_FORMAT_R8_USCALED,
+      [PIPE_FORMAT_R8G8_USCALED]            = ISL_FORMAT_R8G8_USCALED,
+      [PIPE_FORMAT_R8G8B8_USCALED]          = ISL_FORMAT_R8G8B8_USCALED,
+      [PIPE_FORMAT_R8G8B8A8_USCALED]        = ISL_FORMAT_R8G8B8A8_USCALED,
+      [PIPE_FORMAT_R8_SNORM]                = ISL_FORMAT_R8_SNORM,
+      [PIPE_FORMAT_R8G8_SNORM]              = ISL_FORMAT_R8G8_SNORM,
+      [PIPE_FORMAT_R8G8B8_SNORM]            = ISL_FORMAT_R8G8B8_SNORM,
+      [PIPE_FORMAT_R8G8B8A8_SNORM]          = ISL_FORMAT_R8G8B8A8_SNORM,
+      [PIPE_FORMAT_R8_SSCALED]              = ISL_FORMAT_R8_SSCALED,
+      [PIPE_FORMAT_R8G8_SSCALED]            = ISL_FORMAT_R8G8_SSCALED,
+      [PIPE_FORMAT_R8G8B8_SSCALED]          = ISL_FORMAT_R8G8B8_SSCALED,
+      [PIPE_FORMAT_R8G8B8A8_SSCALED]        = ISL_FORMAT_R8G8B8A8_SSCALED,
+      [PIPE_FORMAT_R32_FIXED]               = ISL_FORMAT_R32_SFIXED,
+      [PIPE_FORMAT_R32G32_FIXED]            = ISL_FORMAT_R32G32_SFIXED,
+      [PIPE_FORMAT_R32G32B32_FIXED]         = ISL_FORMAT_R32G32B32_SFIXED,
+      [PIPE_FORMAT_R32G32B32A32_FIXED]      = ISL_FORMAT_R32G32B32A32_SFIXED,
+      [PIPE_FORMAT_R16_FLOAT]               = ISL_FORMAT_R16_FLOAT,
+      [PIPE_FORMAT_R16G16_FLOAT]            = ISL_FORMAT_R16G16_FLOAT,
+      [PIPE_FORMAT_R16G16B16_FLOAT]         = ISL_FORMAT_R16G16B16_FLOAT,
+      [PIPE_FORMAT_R16G16B16A16_FLOAT]      = ISL_FORMAT_R16G16B16A16_FLOAT,
+
+      [PIPE_FORMAT_R8G8B8_SRGB]             = ISL_FORMAT_R8G8B8_UNORM_SRGB,
+      [PIPE_FORMAT_B8G8R8A8_SRGB]           = ISL_FORMAT_B8G8R8A8_UNORM_SRGB,
+      [PIPE_FORMAT_B8G8R8X8_SRGB]           = ISL_FORMAT_B8G8R8X8_UNORM_SRGB,
+      [PIPE_FORMAT_R8G8B8A8_SRGB]           = ISL_FORMAT_R8G8B8A8_UNORM_SRGB,
+
+      [PIPE_FORMAT_DXT1_RGB]                = ISL_FORMAT_BC1_UNORM,
+      [PIPE_FORMAT_DXT1_RGBA]               = ISL_FORMAT_BC1_UNORM,
+      [PIPE_FORMAT_DXT3_RGBA]               = ISL_FORMAT_BC2_UNORM,
+      [PIPE_FORMAT_DXT5_RGBA]               = ISL_FORMAT_BC3_UNORM,
+
+      [PIPE_FORMAT_DXT1_SRGB]               = ISL_FORMAT_BC1_UNORM_SRGB,
+      [PIPE_FORMAT_DXT1_SRGBA]              = ISL_FORMAT_BC1_UNORM_SRGB,
+      [PIPE_FORMAT_DXT3_SRGBA]              = ISL_FORMAT_BC2_UNORM_SRGB,
+      [PIPE_FORMAT_DXT5_SRGBA]              = ISL_FORMAT_BC3_UNORM_SRGB,
+
+      [PIPE_FORMAT_RGTC1_UNORM]             = ISL_FORMAT_BC4_UNORM,
+      [PIPE_FORMAT_RGTC1_SNORM]             = ISL_FORMAT_BC4_SNORM,
+      [PIPE_FORMAT_RGTC2_UNORM]             = ISL_FORMAT_BC5_UNORM,
+      [PIPE_FORMAT_RGTC2_SNORM]             = ISL_FORMAT_BC5_SNORM,
+
+      [PIPE_FORMAT_R10G10B10A2_USCALED]     = ISL_FORMAT_R10G10B10A2_USCALED,
+      [PIPE_FORMAT_R11G11B10_FLOAT]         = ISL_FORMAT_R11G11B10_FLOAT,
+      [PIPE_FORMAT_R9G9B9E5_FLOAT]          = ISL_FORMAT_R9G9B9E5_SHAREDEXP,
+      [PIPE_FORMAT_R1_UNORM]                = ISL_FORMAT_R1_UNORM,
+      [PIPE_FORMAT_R10G10B10X2_USCALED]     = ISL_FORMAT_R10G10B10X2_USCALED,
+      [PIPE_FORMAT_B10G10R10A2_UNORM]       = ISL_FORMAT_B10G10R10A2_UNORM,
+      [PIPE_FORMAT_R8G8B8X8_UNORM]          = ISL_FORMAT_R8G8B8X8_UNORM,
+
+      [PIPE_FORMAT_I8_UNORM]                = ISL_FORMAT_R8_UNORM,
+      [PIPE_FORMAT_I16_UNORM]               = ISL_FORMAT_R16_UNORM,
+      [PIPE_FORMAT_I8_SNORM]                = ISL_FORMAT_R8_SNORM,
+      [PIPE_FORMAT_I16_SNORM]               = ISL_FORMAT_R16_SNORM,
+      [PIPE_FORMAT_I16_FLOAT]               = ISL_FORMAT_R16_FLOAT,
+      [PIPE_FORMAT_I32_FLOAT]               = ISL_FORMAT_R32_FLOAT,
+
+      [PIPE_FORMAT_L8_UINT]                 = ISL_FORMAT_L8_UINT,
+      [PIPE_FORMAT_L8_UNORM]                = ISL_FORMAT_L8_UNORM,
+      [PIPE_FORMAT_L8_SNORM]                = ISL_FORMAT_R8_SNORM,
+      [PIPE_FORMAT_L8_SINT]                 = ISL_FORMAT_L8_SINT,
+      [PIPE_FORMAT_L16_UNORM]               = ISL_FORMAT_L16_UNORM,
+      [PIPE_FORMAT_L16_SNORM]               = ISL_FORMAT_R16_SNORM,
+      [PIPE_FORMAT_L16_FLOAT]               = ISL_FORMAT_L16_FLOAT,
+      [PIPE_FORMAT_L32_FLOAT]               = ISL_FORMAT_L32_FLOAT,
+
+      [PIPE_FORMAT_A8_UNORM]                = ISL_FORMAT_A8_UNORM,
+      [PIPE_FORMAT_A16_UNORM]               = ISL_FORMAT_A16_UNORM,
+      [PIPE_FORMAT_A16_FLOAT]               = ISL_FORMAT_A16_FLOAT,
+      [PIPE_FORMAT_A32_FLOAT]               = ISL_FORMAT_A32_FLOAT,
+
+      [PIPE_FORMAT_L8A8_UNORM]              = ISL_FORMAT_L8A8_UNORM,
+      [PIPE_FORMAT_L16A16_UNORM]            = ISL_FORMAT_L16A16_UNORM,
+      [PIPE_FORMAT_L16A16_FLOAT]            = ISL_FORMAT_L16A16_FLOAT,
+      [PIPE_FORMAT_L32A32_FLOAT]            = ISL_FORMAT_L32A32_FLOAT,
+
+      /* Sadly, we have to use luminance[-alpha] formats for sRGB decoding. */
+      [PIPE_FORMAT_R8_SRGB]                 = ISL_FORMAT_L8_UNORM_SRGB,
+      [PIPE_FORMAT_L8_SRGB]                 = ISL_FORMAT_L8_UNORM_SRGB,
+      [PIPE_FORMAT_L8A8_SRGB]               = ISL_FORMAT_L8A8_UNORM_SRGB,
+
+      [PIPE_FORMAT_R10G10B10A2_SSCALED]     = ISL_FORMAT_R10G10B10A2_SSCALED,
+      [PIPE_FORMAT_R10G10B10A2_SNORM]       = ISL_FORMAT_R10G10B10A2_SNORM,
+
+      [PIPE_FORMAT_B10G10R10A2_USCALED]     = ISL_FORMAT_B10G10R10A2_USCALED,
+      [PIPE_FORMAT_B10G10R10A2_SSCALED]     = ISL_FORMAT_B10G10R10A2_SSCALED,
+      [PIPE_FORMAT_B10G10R10A2_SNORM]       = ISL_FORMAT_B10G10R10A2_SNORM,
+
+      [PIPE_FORMAT_R8_UINT]                 = ISL_FORMAT_R8_UINT,
+      [PIPE_FORMAT_R8G8_UINT]               = ISL_FORMAT_R8G8_UINT,
+      [PIPE_FORMAT_R8G8B8_UINT]             = ISL_FORMAT_R8G8B8_UINT,
+      [PIPE_FORMAT_R8G8B8A8_UINT]           = ISL_FORMAT_R8G8B8A8_UINT,
+
+      [PIPE_FORMAT_R8_SINT]                 = ISL_FORMAT_R8_SINT,
+      [PIPE_FORMAT_R8G8_SINT]               = ISL_FORMAT_R8G8_SINT,
+      [PIPE_FORMAT_R8G8B8_SINT]             = ISL_FORMAT_R8G8B8_SINT,
+      [PIPE_FORMAT_R8G8B8A8_SINT]           = ISL_FORMAT_R8G8B8A8_SINT,
+
+      [PIPE_FORMAT_R16_UINT]                = ISL_FORMAT_R16_UINT,
+      [PIPE_FORMAT_R16G16_UINT]             = ISL_FORMAT_R16G16_UINT,
+      [PIPE_FORMAT_R16G16B16_UINT]          = ISL_FORMAT_R16G16B16_UINT,
+      [PIPE_FORMAT_R16G16B16A16_UINT]       = ISL_FORMAT_R16G16B16A16_UINT,
+
+      [PIPE_FORMAT_R16_SINT]                = ISL_FORMAT_R16_SINT,
+      [PIPE_FORMAT_R16G16_SINT]             = ISL_FORMAT_R16G16_SINT,
+      [PIPE_FORMAT_R16G16B16_SINT]          = ISL_FORMAT_R16G16B16_SINT,
+      [PIPE_FORMAT_R16G16B16A16_SINT]       = ISL_FORMAT_R16G16B16A16_SINT,
+
+      [PIPE_FORMAT_R32_UINT]                = ISL_FORMAT_R32_UINT,
+      [PIPE_FORMAT_R32G32_UINT]             = ISL_FORMAT_R32G32_UINT,
+      [PIPE_FORMAT_R32G32B32_UINT]          = ISL_FORMAT_R32G32B32_UINT,
+      [PIPE_FORMAT_R32G32B32A32_UINT]       = ISL_FORMAT_R32G32B32A32_UINT,
+
+      [PIPE_FORMAT_R32_SINT]                = ISL_FORMAT_R32_SINT,
+      [PIPE_FORMAT_R32G32_SINT]             = ISL_FORMAT_R32G32_SINT,
+      [PIPE_FORMAT_R32G32B32_SINT]          = ISL_FORMAT_R32G32B32_SINT,
+      [PIPE_FORMAT_R32G32B32A32_SINT]       = ISL_FORMAT_R32G32B32A32_SINT,
+
+      [PIPE_FORMAT_B10G10R10A2_UINT]        = ISL_FORMAT_B10G10R10A2_UINT,
+
+      [PIPE_FORMAT_ETC1_RGB8]               = ISL_FORMAT_ETC1_RGB8,
+
+      [PIPE_FORMAT_R8G8B8X8_SRGB]           = ISL_FORMAT_R8G8B8X8_UNORM_SRGB,
+      [PIPE_FORMAT_B10G10R10X2_UNORM]       = ISL_FORMAT_B10G10R10X2_UNORM,
+      [PIPE_FORMAT_R16G16B16X16_UNORM]      = ISL_FORMAT_R16G16B16X16_UNORM,
+      [PIPE_FORMAT_R16G16B16X16_FLOAT]      = ISL_FORMAT_R16G16B16X16_FLOAT,
+      [PIPE_FORMAT_R32G32B32X32_FLOAT]      = ISL_FORMAT_R32G32B32X32_FLOAT,
+
+      [PIPE_FORMAT_R10G10B10A2_UINT]        = ISL_FORMAT_R10G10B10A2_UINT,
+
+      [PIPE_FORMAT_B5G6R5_SRGB]             = ISL_FORMAT_B5G6R5_UNORM_SRGB,
+
+      [PIPE_FORMAT_BPTC_RGBA_UNORM]         = ISL_FORMAT_BC7_UNORM,
+      [PIPE_FORMAT_BPTC_SRGBA]              = ISL_FORMAT_BC7_UNORM_SRGB,
+      [PIPE_FORMAT_BPTC_RGB_FLOAT]          = ISL_FORMAT_BC6H_SF16,
+      [PIPE_FORMAT_BPTC_RGB_UFLOAT]         = ISL_FORMAT_BC6H_UF16,
+
+      [PIPE_FORMAT_ETC2_RGB8]               = ISL_FORMAT_ETC2_RGB8,
+      [PIPE_FORMAT_ETC2_SRGB8]              = ISL_FORMAT_ETC2_SRGB8,
+      [PIPE_FORMAT_ETC2_RGB8A1]             = ISL_FORMAT_ETC2_RGB8_PTA,
+      [PIPE_FORMAT_ETC2_SRGB8A1]            = ISL_FORMAT_ETC2_SRGB8_PTA,
+      [PIPE_FORMAT_ETC2_RGBA8]              = ISL_FORMAT_ETC2_EAC_RGBA8,
+      [PIPE_FORMAT_ETC2_SRGBA8]             = ISL_FORMAT_ETC2_EAC_SRGB8_A8,
+      [PIPE_FORMAT_ETC2_R11_UNORM]          = ISL_FORMAT_EAC_R11,
+      [PIPE_FORMAT_ETC2_R11_SNORM]          = ISL_FORMAT_EAC_SIGNED_R11,
+      [PIPE_FORMAT_ETC2_RG11_UNORM]         = ISL_FORMAT_EAC_RG11,
+      [PIPE_FORMAT_ETC2_RG11_SNORM]         = ISL_FORMAT_EAC_SIGNED_RG11,
+
+      [PIPE_FORMAT_FXT1_RGB]                = ISL_FORMAT_FXT1,
+      [PIPE_FORMAT_FXT1_RGBA]               = ISL_FORMAT_FXT1,
+
+      [PIPE_FORMAT_ASTC_4x4]                = ISL_FORMAT_ASTC_LDR_2D_4X4_FLT16,
+      [PIPE_FORMAT_ASTC_5x4]                = ISL_FORMAT_ASTC_LDR_2D_5X4_FLT16,
+      [PIPE_FORMAT_ASTC_5x5]                = ISL_FORMAT_ASTC_LDR_2D_5X5_FLT16,
+      [PIPE_FORMAT_ASTC_6x5]                = ISL_FORMAT_ASTC_LDR_2D_6X5_FLT16,
+      [PIPE_FORMAT_ASTC_6x6]                = ISL_FORMAT_ASTC_LDR_2D_6X6_FLT16,
+      [PIPE_FORMAT_ASTC_8x5]                = ISL_FORMAT_ASTC_LDR_2D_8X5_FLT16,
+      [PIPE_FORMAT_ASTC_8x6]                = ISL_FORMAT_ASTC_LDR_2D_8X6_FLT16,
+      [PIPE_FORMAT_ASTC_8x8]                = ISL_FORMAT_ASTC_LDR_2D_8X8_FLT16,
+      [PIPE_FORMAT_ASTC_10x5]               = ISL_FORMAT_ASTC_LDR_2D_10X5_FLT16,
+      [PIPE_FORMAT_ASTC_10x6]               = ISL_FORMAT_ASTC_LDR_2D_10X6_FLT16,
+      [PIPE_FORMAT_ASTC_10x8]               = ISL_FORMAT_ASTC_LDR_2D_10X8_FLT16,
+      [PIPE_FORMAT_ASTC_10x10]              = ISL_FORMAT_ASTC_LDR_2D_10X10_FLT16,
+      [PIPE_FORMAT_ASTC_12x10]              = ISL_FORMAT_ASTC_LDR_2D_12X10_FLT16,
+      [PIPE_FORMAT_ASTC_12x12]              = ISL_FORMAT_ASTC_LDR_2D_12X12_FLT16,
+
+      [PIPE_FORMAT_ASTC_4x4_SRGB]           = ISL_FORMAT_ASTC_LDR_2D_4X4_U8SRGB,
+      [PIPE_FORMAT_ASTC_5x4_SRGB]           = ISL_FORMAT_ASTC_LDR_2D_5X4_U8SRGB,
+      [PIPE_FORMAT_ASTC_5x5_SRGB]           = ISL_FORMAT_ASTC_LDR_2D_5X5_U8SRGB,
+      [PIPE_FORMAT_ASTC_6x5_SRGB]           = ISL_FORMAT_ASTC_LDR_2D_6X5_U8SRGB,
+      [PIPE_FORMAT_ASTC_6x6_SRGB]           = ISL_FORMAT_ASTC_LDR_2D_6X6_U8SRGB,
+      [PIPE_FORMAT_ASTC_8x5_SRGB]           = ISL_FORMAT_ASTC_LDR_2D_8X5_U8SRGB,
+      [PIPE_FORMAT_ASTC_8x6_SRGB]           = ISL_FORMAT_ASTC_LDR_2D_8X6_U8SRGB,
+      [PIPE_FORMAT_ASTC_8x8_SRGB]           = ISL_FORMAT_ASTC_LDR_2D_8X8_U8SRGB,
+      [PIPE_FORMAT_ASTC_10x5_SRGB]          = ISL_FORMAT_ASTC_LDR_2D_10X5_U8SRGB,
+      [PIPE_FORMAT_ASTC_10x6_SRGB]          = ISL_FORMAT_ASTC_LDR_2D_10X6_U8SRGB,
+      [PIPE_FORMAT_ASTC_10x8_SRGB]          = ISL_FORMAT_ASTC_LDR_2D_10X8_U8SRGB,
+      [PIPE_FORMAT_ASTC_10x10_SRGB]         = ISL_FORMAT_ASTC_LDR_2D_10X10_U8SRGB,
+      [PIPE_FORMAT_ASTC_12x10_SRGB]         = ISL_FORMAT_ASTC_LDR_2D_12X10_U8SRGB,
+      [PIPE_FORMAT_ASTC_12x12_SRGB]         = ISL_FORMAT_ASTC_LDR_2D_12X12_U8SRGB,
+
+      [PIPE_FORMAT_A1B5G5R5_UNORM]          = ISL_FORMAT_A1B5G5R5_UNORM,
+
+      /* We support these so that we know the API expects no alpha channel.
+       * Otherwise, the state tracker would just give us a format with alpha
+       * and we wouldn't know to override the swizzle to 1.
+       */
+      [PIPE_FORMAT_R16G16B16X16_UINT]       = ISL_FORMAT_R16G16B16A16_UINT,
+      [PIPE_FORMAT_R16G16B16X16_SINT]       = ISL_FORMAT_R16G16B16A16_SINT,
+      [PIPE_FORMAT_R32G32B32X32_UINT]       = ISL_FORMAT_R32G32B32A32_UINT,
+      [PIPE_FORMAT_R32G32B32X32_SINT]       = ISL_FORMAT_R32G32B32A32_SINT,
+      [PIPE_FORMAT_R10G10B10X2_SNORM]       = ISL_FORMAT_R10G10B10A2_SNORM,
+   };
+   assert(pf < PIPE_FORMAT_COUNT);
+   return table[pf];
+}
+
+static enum isl_format
+get_render_format(enum pipe_format pformat, enum isl_format def_format)
+{
+   switch (pformat) {
+   case PIPE_FORMAT_A16_UNORM:            return ISL_FORMAT_R16_UNORM;
+   case PIPE_FORMAT_A16_FLOAT:            return ISL_FORMAT_R16_FLOAT;
+   case PIPE_FORMAT_A32_FLOAT:            return ISL_FORMAT_R32_FLOAT;
+
+   case PIPE_FORMAT_I8_UNORM:             return ISL_FORMAT_R8_UNORM;
+   case PIPE_FORMAT_I16_UNORM:            return ISL_FORMAT_R16_UNORM;
+   case PIPE_FORMAT_I16_FLOAT:            return ISL_FORMAT_R16_FLOAT;
+   case PIPE_FORMAT_I32_FLOAT:            return ISL_FORMAT_R32_FLOAT;
+
+   case PIPE_FORMAT_L8_UNORM:             return ISL_FORMAT_R8_UNORM;
+   case PIPE_FORMAT_L8_UINT:              return ISL_FORMAT_R8_UINT;
+   case PIPE_FORMAT_L8_SINT:              return ISL_FORMAT_R8_SINT;
+   case PIPE_FORMAT_L16_UNORM:            return ISL_FORMAT_R16_UNORM;
+   case PIPE_FORMAT_L16_FLOAT:            return ISL_FORMAT_R16_FLOAT;
+   case PIPE_FORMAT_L32_FLOAT:            return ISL_FORMAT_R32_FLOAT;
+
+   case PIPE_FORMAT_L8A8_UNORM:           return ISL_FORMAT_R8G8_UNORM;
+   case PIPE_FORMAT_L16A16_UNORM:         return ISL_FORMAT_R16G16_UNORM;
+   case PIPE_FORMAT_L16A16_FLOAT:         return ISL_FORMAT_R16G16_FLOAT;
+   case PIPE_FORMAT_L32A32_FLOAT:         return ISL_FORMAT_R32G32_FLOAT;
+
+   default:
+      return def_format;
+   }
+}
+
+struct crocus_format_info
+crocus_format_for_usage(const struct intel_device_info *devinfo,
+                        enum pipe_format pformat,
+                        isl_surf_usage_flags_t usage)
+{
+   struct crocus_format_info info = { crocus_isl_format_for_pipe_format(pformat),
+                                      { PIPE_SWIZZLE_X, PIPE_SWIZZLE_Y, PIPE_SWIZZLE_Z, PIPE_SWIZZLE_W } };
+
+   if (info.fmt == ISL_FORMAT_UNSUPPORTED)
+      return info;
+
+   if (pformat == PIPE_FORMAT_A8_UNORM) {
+      info.fmt = ISL_FORMAT_A8_UNORM;
+   }
+
+   if (usage & ISL_SURF_USAGE_RENDER_TARGET_BIT)
+      info.fmt = get_render_format(pformat, info.fmt);
+   if (devinfo->ver < 6) {
+      if (pformat == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT)
+         info.fmt = ISL_FORMAT_R32_FLOAT_X8X24_TYPELESS;
+      if (pformat == PIPE_FORMAT_X32_S8X24_UINT)
+         info.fmt = ISL_FORMAT_X32_TYPELESS_G8X24_UINT;
+      if (pformat == PIPE_FORMAT_X24S8_UINT)
+         info.fmt = ISL_FORMAT_X24_TYPELESS_G8_UINT;
+   }
+
+   const struct isl_format_layout *fmtl = isl_format_get_layout(info.fmt);
+
+   if (util_format_is_snorm(pformat)) {
+      if (util_format_is_intensity(pformat)) {
+         info.swizzles[0] = PIPE_SWIZZLE_X;
+         info.swizzles[1] = PIPE_SWIZZLE_X;
+         info.swizzles[2] = PIPE_SWIZZLE_X;
+         info.swizzles[3] = PIPE_SWIZZLE_X;
+      } else if (util_format_is_luminance(pformat)) {
+         info.swizzles[0] = PIPE_SWIZZLE_X;
+         info.swizzles[1] = PIPE_SWIZZLE_X;
+         info.swizzles[2] = PIPE_SWIZZLE_X;
+         info.swizzles[3] = PIPE_SWIZZLE_1;
+      } else if (util_format_is_luminance_alpha(pformat)) {
+         info.swizzles[0] = PIPE_SWIZZLE_X;
+         info.swizzles[1] = PIPE_SWIZZLE_X;
+         info.swizzles[2] = PIPE_SWIZZLE_X;
+         info.swizzles[3] = PIPE_SWIZZLE_Y;
+      } else if (util_format_is_alpha(pformat)) {
+         info.swizzles[0] = PIPE_SWIZZLE_0;
+         info.swizzles[1] = PIPE_SWIZZLE_0;
+         info.swizzles[2] = PIPE_SWIZZLE_0;
+         info.swizzles[3] = PIPE_SWIZZLE_X;
+      }
+   }
+
+   /* When faking RGBX pipe formats with RGBA ISL formats, override alpha. */
+   if (!util_format_has_alpha(pformat) && fmtl->channels.a.type != ISL_VOID) {
+      info.swizzles[0] = PIPE_SWIZZLE_X;
+      info.swizzles[1] = PIPE_SWIZZLE_Y;
+      info.swizzles[2] = PIPE_SWIZZLE_Z;
+      info.swizzles[3] = PIPE_SWIZZLE_1;
+   }
+
+   /* We choose RGBA over RGBX for rendering the hardware doesn't support
+    * rendering to RGBX. However, when this internal override is used on Gen9+,
+    * fast clears don't work correctly.
+    *
+    * i965 fixes this by pretending to not support RGBX formats, and the higher
+    * layers of Mesa pick the RGBA format instead. Gallium doesn't work that
+    * way, and might choose a different format, like BGRX instead of RGBX,
+    * which will also cause problems when sampling from a surface fast cleared
+    * as RGBX. So we always choose RGBA instead of RGBX explicitly
+    * here.
+    */
+   if (isl_format_is_rgbx(info.fmt) &&
+       !isl_format_supports_rendering(devinfo, info.fmt) &&
+       (usage & ISL_SURF_USAGE_RENDER_TARGET_BIT)) {
+      info.fmt = isl_format_rgbx_to_rgba(info.fmt);
+      info.swizzles[0] = PIPE_SWIZZLE_X;
+      info.swizzles[1] = PIPE_SWIZZLE_Y;
+      info.swizzles[2] = PIPE_SWIZZLE_Z;
+      info.swizzles[3] = PIPE_SWIZZLE_1;
+   }
+
+   return info;
+}
+
+/**
+ * The pscreen->is_format_supported() driver hook.
+ *
+ * Returns true if the given format is supported for the given usage
+ * (PIPE_BIND_*) and sample count.
+ */
+bool
+crocus_is_format_supported(struct pipe_screen *pscreen,
+                           enum pipe_format pformat,
+                           enum pipe_texture_target target,
+                           unsigned sample_count, unsigned storage_sample_count,
+                           unsigned usage)
+{
+   struct crocus_screen *screen = (struct crocus_screen *)pscreen;
+   const struct intel_device_info *devinfo = &screen->devinfo;
+
+   if (!util_is_power_of_two_or_zero(sample_count))
+      return false;
+   if (devinfo->ver >= 7) {
+      if (sample_count > 8 || sample_count == 2)
+         return false;
+   } else if (devinfo->ver == 6) {
+      if (sample_count > 4 || sample_count == 2)
+         return false;
+   } else if (sample_count > 1) {
+      return false;
+   }
+
+   if (pformat == PIPE_FORMAT_NONE)
+      return true;
+
+   enum isl_format format = crocus_isl_format_for_pipe_format(pformat);
+
+   if (format == ISL_FORMAT_UNSUPPORTED)
+      return false;
+
+   /* no stencil texturing prior to haswell */
+   if (devinfo->verx10 < 75) {
+      if (pformat == PIPE_FORMAT_S8_UINT ||
+          pformat == PIPE_FORMAT_X24S8_UINT ||
+          pformat == PIPE_FORMAT_S8X24_UINT ||
+          pformat == PIPE_FORMAT_X32_S8X24_UINT)
+         return false;
+   }
+
+   const struct isl_format_layout *fmtl = isl_format_get_layout(format);
+   const bool is_integer = isl_format_has_int_channel(format);
+   bool supported = true;
+
+   if (sample_count > 1)
+      supported &= isl_format_supports_multisampling(devinfo, format);
+
+   if (usage & PIPE_BIND_DEPTH_STENCIL) {
+      bool depth_fmts = format == ISL_FORMAT_R32_FLOAT_X8X24_TYPELESS ||
+         format == ISL_FORMAT_R32_FLOAT ||
+         format == ISL_FORMAT_R24_UNORM_X8_TYPELESS ||
+         format == ISL_FORMAT_R8_UINT;
+
+      /* Z16 is disabled here as on pre-GEN8 it's slower. */
+      if (devinfo->ver == 8)
+         depth_fmts |= format == ISL_FORMAT_R16_UNORM;
+      supported &= depth_fmts;
+   }
+
+   if (usage & PIPE_BIND_RENDER_TARGET) {
+      /* Alpha and luminance-alpha formats other than A8_UNORM are not
+       * renderable.
+       *
+       * For BLORP, we can apply the swizzle in the shader.  But for
+       * general rendering, this would mean recompiling the shader, which
+       * we'd like to avoid doing.  So we mark these formats non-renderable.
+       *
+       * We do support A8_UNORM as it's required and is renderable.
+       */
+      if (pformat != PIPE_FORMAT_A8_UNORM &&
+          (util_format_is_alpha(pformat) ||
+           util_format_is_luminance_alpha(pformat)))
+         supported = false;
+
+      enum isl_format rt_format = format;
+
+      if (isl_format_is_rgbx(format) &&
+          !isl_format_supports_rendering(devinfo, format))
+         rt_format = isl_format_rgbx_to_rgba(format);
+
+      supported &= isl_format_supports_rendering(devinfo, rt_format);
+
+      if (!is_integer)
+         supported &= isl_format_supports_alpha_blending(devinfo, rt_format);
+   }
+
+   if (usage & PIPE_BIND_SHADER_IMAGE) {
+      /* Dataport doesn't support compression, and we can't resolve an MCS
+       * compressed surface.  (Buffer images may have sample count of 0.)
+       */
+      supported &= sample_count == 0;
+
+      supported &= isl_format_supports_typed_writes(devinfo, format);
+      supported &= isl_has_matching_typed_storage_image_format(devinfo, format);
+   }
+
+   if (usage & PIPE_BIND_SAMPLER_VIEW) {
+      supported &= isl_format_supports_sampling(devinfo, format);
+
+      /* disable Z16 unorm depth textures pre gen8 */
+      if (devinfo->ver < 8 && pformat == PIPE_FORMAT_Z16_UNORM)
+         supported = false;
+
+      bool ignore_filtering = false;
+
+      if (is_integer)
+         ignore_filtering = true;
+
+      /* I said them, but I lied them. */
+      if (devinfo->ver < 5 && (format == ISL_FORMAT_R32G32B32A32_FLOAT ||
+                               format == ISL_FORMAT_R24_UNORM_X8_TYPELESS ||
+                               format == ISL_FORMAT_R32_FLOAT ||
+                               format == ISL_FORMAT_R32_FLOAT_X8X24_TYPELESS))
+         ignore_filtering = true;
+      if (!ignore_filtering)
+         supported &= isl_format_supports_filtering(devinfo, format);
+
+      /* Don't advertise 3-component RGB formats for non-buffer textures.
+       * This ensures that they are renderable from an API perspective since
+       * the state tracker will fall back to RGBA or RGBX, which are
+       * renderable.  We want to render internally for copies and blits,
+       * even if the application doesn't.
+       *
+       * Buffer textures don't need to be renderable, so we support real RGB.
+       * This is useful for PBO upload, and 32-bit RGB support is mandatory.
+       */
+      if (target != PIPE_BUFFER)
+         supported &= fmtl->bpb != 24 && fmtl->bpb != 48 && fmtl->bpb != 96;
+   }
+
+   if (usage & PIPE_BIND_VERTEX_BUFFER) {
+      supported &= isl_format_supports_vertex_fetch(devinfo, format);
+
+      if (devinfo->verx10 < 75) {
+         /* W/A: Pre-Haswell, the hardware doesn't really support the formats
+          * we'd like to use here, so upload everything as UINT and fix it in
+          * the shader
+          */
+         if (format == ISL_FORMAT_R10G10B10A2_UNORM ||
+             format == ISL_FORMAT_B10G10R10A2_UNORM ||
+             format == ISL_FORMAT_R10G10B10A2_SNORM ||
+             format == ISL_FORMAT_B10G10R10A2_SNORM ||
+             format == ISL_FORMAT_R10G10B10A2_USCALED ||
+             format == ISL_FORMAT_B10G10R10A2_USCALED ||
+             format == ISL_FORMAT_R10G10B10A2_SSCALED ||
+             format == ISL_FORMAT_B10G10R10A2_SSCALED)
+            supported = true;
+
+         if (format == ISL_FORMAT_R8G8B8_SINT ||
+             format == ISL_FORMAT_R8G8B8_UINT ||
+             format == ISL_FORMAT_R16G16B16_SINT ||
+             format == ISL_FORMAT_R16G16B16_UINT)
+            supported = true;
+      }
+   }
+
+   if (usage & PIPE_BIND_INDEX_BUFFER) {
+      supported &= format == ISL_FORMAT_R8_UINT ||
+                   format == ISL_FORMAT_R16_UINT ||
+                   format == ISL_FORMAT_R32_UINT;
+   }
+
+   return supported;
+}

src/gallium/drivers/crocus/crocus_genx_macros.h

@@ -0,0 +1,170 @@
+/*
+ * Copyright © 2019 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+/**
+ * Macro and function definitions needed in order to use genxml.
+ *
+ * This should only be included in sources compiled per-generation.
+ */
+
+#include "crocus_batch.h"
+
+#include "genxml/gen_macros.h"
+
+#define __gen_address_type struct crocus_address
+#define __gen_user_data struct crocus_batch
+#define __gen_combine_address crocus_combine_address
+
+static inline void *
+__gen_get_batch_dwords(struct crocus_batch *batch, unsigned dwords)
+{
+   return crocus_get_command_space(batch, dwords * sizeof(uint32_t));
+}
+
+static inline struct crocus_address
+__gen_address_offset(struct crocus_address addr, uint64_t offset)
+{
+   addr.offset += offset;
+   return addr;
+}
+
+static uint64_t
+__gen_combine_address(struct crocus_batch *batch, void *location,
+                      struct crocus_address addr, uint32_t delta)
+{
+   uint32_t offset = (char *)location - (char *)batch->command.map;
+
+   if (addr.bo == NULL) {
+      return addr.offset + delta;
+   } else {
+      if (GFX_VER < 6 && crocus_ptr_in_state_buffer(batch, location)) {
+         offset = (char *) location - (char *) batch->state.map;
+         return crocus_state_reloc(batch, offset, addr.bo,
+                                   addr.offset + delta,
+                                   addr.reloc_flags);
+      }
+
+      assert(!crocus_ptr_in_state_buffer(batch, location));
+
+      offset = (char *) location - (char *) batch->command.map;
+      return crocus_command_reloc(batch, offset, addr.bo,
+                                  addr.offset + delta,
+                                  addr.reloc_flags);
+   }
+}
+
+static inline struct crocus_address
+__gen_get_batch_address(struct crocus_batch *batch, void *location)
+{
+   UNREACHABLE("Not supported by crocus");
+}
+
+#define __gen_address_type struct crocus_address
+#define __gen_user_data struct crocus_batch
+
+#define __genxml_cmd_length(cmd) cmd ## _length
+#define __genxml_cmd_length_bias(cmd) cmd ## _length_bias
+#define __genxml_cmd_header(cmd) cmd ## _header
+#define __genxml_cmd_pack(cmd) cmd ## _pack
+#define __genxml_reg_num(cmd) cmd ## _num
+
+#include "genxml/genX_pack.h"
+#include "genxml/gen_macros.h"
+#include "genxml/genX_bits.h"
+
+/* CS_GPR(15) is reserved for combining conditional rendering predicates
+ * with GL_ARB_indirect_parameters draw number predicates.
+ */
+#define MI_BUILDER_NUM_ALLOC_GPRS 15
+#include "common/mi_builder.h"
+
+#define _crocus_pack_command(batch, cmd, dst, name)                 \
+   for (struct cmd name = { __genxml_cmd_header(cmd) },           \
+        *_dst = (void *)(dst); __builtin_expect(_dst != NULL, 1); \
+        ({ __genxml_cmd_pack(cmd)(batch, (void *)_dst, &name);    \
+           _dst = NULL;                                           \
+           }))
+
+#define crocus_pack_command(cmd, dst, name) \
+   _crocus_pack_command(NULL, cmd, dst, name)
+
+#define _crocus_pack_state(batch, cmd, dst, name)                   \
+   for (struct cmd name = {},                                     \
+        *_dst = (void *)(dst); __builtin_expect(_dst != NULL, 1); \
+        __genxml_cmd_pack(cmd)(batch, (void *)_dst, &name),       \
+        _dst = NULL)
+
+#define crocus_pack_state(cmd, dst, name)                           \
+   _crocus_pack_state(NULL, cmd, dst, name)
+
+#define crocus_emit_cmd(batch, cmd, name) \
+   _crocus_pack_command(batch, cmd, __gen_get_batch_dwords(batch, __genxml_cmd_length(cmd)), name)
+
+#define crocus_emit_merge(batch, dwords0, dwords1, num_dwords)    \
+   do {                                                         \
+      uint32_t *dw = __gen_get_batch_dwords(batch, num_dwords); \
+      for (uint32_t i = 0; i < num_dwords; i++)                 \
+         dw[i] = (dwords0)[i] | (dwords1)[i];                   \
+      VG(VALGRIND_CHECK_MEM_IS_DEFINED(dw, num_dwords));        \
+   } while (0)
+
+#define crocus_emit_reg(batch, reg, name)                                 \
+   for (struct reg name = {}, *_cont = (struct reg *)1; _cont != NULL;  \
+        ({                                                              \
+            uint32_t _dw[__genxml_cmd_length(reg)];                     \
+            __genxml_cmd_pack(reg)(NULL, _dw, &name);                   \
+            for (unsigned i = 0; i < __genxml_cmd_length(reg); i++) {   \
+               crocus_emit_cmd(batch, GENX(MI_LOAD_REGISTER_IMM), lri) {  \
+                  lri.RegisterOffset   = __genxml_reg_num(reg);         \
+                  lri.DataDWord        = _dw[i];                        \
+               }                                                        \
+            }                                                           \
+           _cont = NULL;                                                \
+         }))
+
+
+/**
+ * crocus_address constructor helpers:
+ *
+ * When using these to construct a CSO, pass NULL for \p bo, and manually
+ * pin the BO later.  Otherwise, genxml's address handling will add the
+ * BO to the current batch's validation list at CSO creation time, rather
+ * than at draw time as desired.
+ */
+
+UNUSED static struct crocus_address
+ro_bo(struct crocus_bo *bo, uint64_t offset)
+{
+   return (struct crocus_address) { .bo = bo, .offset = offset, .reloc_flags = RELOC_32BIT };
+}
+
+UNUSED static struct crocus_address
+rw_bo(struct crocus_bo *bo, uint64_t offset)
+{
+   return (struct crocus_address) { .bo = bo, .offset = offset, .reloc_flags = RELOC_32BIT | RELOC_WRITE };
+}
+
+UNUSED static struct crocus_address
+ggtt_bo(struct crocus_bo *bo, uint64_t offset)
+{
+   return (struct crocus_address) { .bo = bo, .offset = offset, .reloc_flags = RELOC_WRITE | RELOC_NEEDS_GGTT };
+}

src/gallium/drivers/crocus/crocus_genx_protos.h

@@ -0,0 +1,51 @@
+/*
+ * Copyright © 2019 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+/* GenX-specific function declarations.
+ *
+ * Don't include this directly, it will be included by crocus_context.h.
+ *
+ * NOTE: This header can be included multiple times, from the same file.
+ */
+
+/* crocus_state.c */
+void genX(crocus_init_state)(struct crocus_context *ice);
+void genX(crocus_init_screen_state)(struct crocus_screen *screen);
+void genX(crocus_upload_urb)(struct crocus_batch *batch,
+                      unsigned vs_size,
+                      bool gs_present,
+                      unsigned gs_size);
+void genX(crocus_update_pma_fix)(struct crocus_context *ice,
+                                 struct crocus_batch *batch,
+                                 bool enable);
+void
+genX(crocus_framebuffer_init)(struct pipe_context *pctx, const struct pipe_framebuffer_state *fb, struct pipe_surface **cbufs, struct pipe_surface **zsbuf);
+
+/* crocus_blorp.c */
+void genX(crocus_init_blorp)(struct crocus_context *ice);
+
+/* crocus_query.c */
+void genX(crocus_init_query)(struct crocus_context *ice);
+void genX(crocus_init_screen_query)(struct crocus_screen *screen);
+
+/* crocus_blt.c */
+void genX(crocus_init_blt)(struct crocus_screen *screen);

src/gallium/drivers/crocus/crocus_monitor.c

@@ -0,0 +1,337 @@
+/*
+ * Copyright © 2019 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include "crocus_monitor.h"
+
+#include <xf86drm.h>
+
+#include "crocus_screen.h"
+#include "crocus_context.h"
+#include "crocus_perf.h"
+
+#include "perf/intel_perf.h"
+#include "perf/intel_perf_query.h"
+#include "perf/intel_perf_regs.h"
+
+struct crocus_monitor_object {
+   int num_active_counters;
+   int *active_counters;
+
+   size_t result_size;
+   unsigned char *result_buffer;
+
+   struct intel_perf_query_object *query;
+};
+
+int
+crocus_get_monitor_info(struct pipe_screen *pscreen, unsigned index,
+                        struct pipe_driver_query_info *info)
+{
+   const struct crocus_screen *screen = (struct crocus_screen *)pscreen;
+   struct intel_perf_config *perf_cfg = screen->perf_cfg;
+   assert(perf_cfg);
+   if (!perf_cfg)
+      return 0;
+
+   if (!info) {
+      /* return the number of metrics */
+      return perf_cfg->n_counters;
+   }
+
+   struct intel_perf_query_counter_info *counter_info = &perf_cfg->counter_infos[index];
+   struct intel_perf_query_info *query_info =
+      &perf_cfg->queries[intel_perf_query_counter_info_first_query(counter_info)];
+   struct intel_perf_query_counter *counter = counter_info->counter;
+   struct intel_perf_query_result results;
+
+   intel_perf_query_result_clear(&results);
+
+   info->group_id = counter_info->location.group_idx;
+   info->name = counter->name;
+   info->query_type = PIPE_QUERY_DRIVER_SPECIFIC + index;
+
+   if (counter->type == INTEL_PERF_COUNTER_TYPE_THROUGHPUT)
+      info->result_type = PIPE_DRIVER_QUERY_RESULT_TYPE_AVERAGE;
+   else
+      info->result_type = PIPE_DRIVER_QUERY_RESULT_TYPE_CUMULATIVE;
+   switch (counter->data_type) {
+   case INTEL_PERF_COUNTER_DATA_TYPE_BOOL32:
+   case INTEL_PERF_COUNTER_DATA_TYPE_UINT32:
+      info->type = PIPE_DRIVER_QUERY_TYPE_UINT;
+      uint64_t val =
+         counter->oa_counter_max_uint64 ?
+         counter->oa_counter_max_uint64(perf_cfg, query_info, &results) : 0;
+      info->max_value.u32 = (uint32_t)val;
+      break;
+   case INTEL_PERF_COUNTER_DATA_TYPE_UINT64:
+      info->type = PIPE_DRIVER_QUERY_TYPE_UINT64;
+      info->max_value.u64 =
+         counter->oa_counter_max_uint64 ?
+         counter->oa_counter_max_uint64(perf_cfg, query_info, &results) : 0;
+      break;
+   case INTEL_PERF_COUNTER_DATA_TYPE_FLOAT:
+   case INTEL_PERF_COUNTER_DATA_TYPE_DOUBLE:
+      info->type = PIPE_DRIVER_QUERY_TYPE_FLOAT;
+      info->max_value.f =
+         counter->oa_counter_max_float ?
+         counter->oa_counter_max_float(perf_cfg, query_info, &results) : 0.0f;
+      break;
+   default:
+      assert(false);
+      break;
+   }
+
+   /* indicates that this is an OA query, not a pipeline statistics query */
+   info->flags = PIPE_DRIVER_QUERY_FLAG_BATCH;
+   return 1;
+}
+
+static bool
+crocus_monitor_init_metrics(struct crocus_screen *screen)
+{
+   struct intel_perf_config *perf_cfg = intel_perf_new(screen);
+   if (unlikely(!perf_cfg))
+      return false;
+
+   screen->perf_cfg = perf_cfg;
+
+   crocus_perf_init_vtbl(perf_cfg);
+
+   intel_perf_init_metrics(perf_cfg, &screen->devinfo, screen->fd, true, true);
+
+   return perf_cfg->n_counters > 0;
+}
+
+int
+crocus_get_monitor_group_info(struct pipe_screen *pscreen,
+                              unsigned group_index,
+                              struct pipe_driver_query_group_info *info)
+{
+   struct crocus_screen *screen = (struct crocus_screen *)pscreen;
+   if (!screen->perf_cfg) {
+      if (!crocus_monitor_init_metrics(screen))
+         return 0;
+   }
+
+   const struct intel_perf_config *perf_cfg = screen->perf_cfg;
+
+   if (!info) {
+      /* return the count that can be queried */
+      return perf_cfg->n_queries;
+   }
+
+   if (group_index >= perf_cfg->n_queries) {
+      /* out of range */
+      return 0;
+   }
+
+   struct intel_perf_query_info *query = &perf_cfg->queries[group_index];
+
+   info->name = query->name;
+   info->max_active_queries = query->n_counters;
+   info->num_queries = query->n_counters;
+
+   return 1;
+}
+
+static void
+crocus_init_monitor_ctx(struct crocus_context *ice)
+{
+   struct crocus_screen *screen = (struct crocus_screen *) ice->ctx.screen;
+
+   ice->perf_ctx = intel_perf_new_context(ice);
+   if (unlikely(!ice->perf_ctx))
+      return;
+
+   struct intel_perf_context *perf_ctx = ice->perf_ctx;
+   struct intel_perf_config *perf_cfg = screen->perf_cfg;
+   intel_perf_init_context(perf_ctx,
+                           perf_cfg,
+                           ice,
+                           ice,
+                           screen->bufmgr,
+                           &screen->devinfo,
+                           ice->batches[CROCUS_BATCH_RENDER].hw_ctx_id,
+                           screen->fd);
+}
+
+/* entry point for GenPerfMonitorsAMD */
+struct crocus_monitor_object *
+crocus_create_monitor_object(struct crocus_context *ice,
+                             unsigned num_queries,
+                             unsigned *query_types)
+{
+   struct crocus_screen *screen = (struct crocus_screen *) ice->ctx.screen;
+   struct intel_perf_config *perf_cfg = screen->perf_cfg;
+   struct intel_perf_query_object *query_obj = NULL;
+
+   /* initialize perf context if this has not already been done.  This
+    * function is the first entry point that carries the gl context.
+    */
+   if (ice->perf_ctx == NULL) {
+      crocus_init_monitor_ctx(ice);
+   }
+   struct intel_perf_context *perf_ctx = ice->perf_ctx;
+
+   assert(num_queries > 0);
+   int query_index = query_types[0] - PIPE_QUERY_DRIVER_SPECIFIC;
+   assert(query_index <= perf_cfg->n_counters);
+   const int group = perf_cfg->counter_infos[query_index].location.group_idx;
+
+   struct crocus_monitor_object *monitor =
+      calloc(1, sizeof(struct crocus_monitor_object));
+   if (unlikely(!monitor))
+      goto allocation_failure;
+
+   monitor->num_active_counters = num_queries;
+   monitor->active_counters = calloc(num_queries, sizeof(int));
+   if (unlikely(!monitor->active_counters))
+      goto allocation_failure;
+
+   for (int i = 0; i < num_queries; ++i) {
+      unsigned current_query = query_types[i];
+      unsigned current_query_index = current_query - PIPE_QUERY_DRIVER_SPECIFIC;
+
+      /* all queries must be in the same group */
+      assert(current_query_index <= perf_cfg->n_counters);
+      assert(perf_cfg->counter_infos[current_query_index].location.group_idx == group);
+      monitor->active_counters[i] =
+         perf_cfg->counter_infos[current_query_index].location.counter_idx;
+   }
+
+   /* create the intel_perf_query */
+   query_obj = intel_perf_new_query(perf_ctx, group);
+   if (unlikely(!query_obj))
+      goto allocation_failure;
+
+   monitor->query = query_obj;
+   monitor->result_size = perf_cfg->queries[group].data_size;
+   monitor->result_buffer = calloc(1, monitor->result_size);
+   if (unlikely(!monitor->result_buffer))
+      goto allocation_failure;
+
+   return monitor;
+
+allocation_failure:
+   if (monitor) {
+      free(monitor->active_counters);
+      free(monitor->result_buffer);
+   }
+   free(query_obj);
+   free(monitor);
+   return NULL;
+}
+
+void
+crocus_destroy_monitor_object(struct pipe_context *ctx,
+                              struct crocus_monitor_object *monitor)
+{
+   struct crocus_context *ice = (struct crocus_context *)ctx;
+
+   intel_perf_delete_query(ice->perf_ctx, monitor->query);
+   free(monitor->result_buffer);
+   monitor->result_buffer = NULL;
+   free(monitor->active_counters);
+   monitor->active_counters = NULL;
+   free(monitor);
+}
+
+bool
+crocus_begin_monitor(struct pipe_context *ctx,
+                     struct crocus_monitor_object *monitor)
+{
+   struct crocus_context *ice = (void *) ctx;
+   struct intel_perf_context *perf_ctx = ice->perf_ctx;
+
+   return intel_perf_begin_query(perf_ctx, monitor->query);
+}
+
+bool
+crocus_end_monitor(struct pipe_context *ctx,
+                   struct crocus_monitor_object *monitor)
+{
+   struct crocus_context *ice = (void *) ctx;
+   struct intel_perf_context *perf_ctx = ice->perf_ctx;
+
+   intel_perf_end_query(perf_ctx, monitor->query);
+   return true;
+}
+
+bool
+crocus_get_monitor_result(struct pipe_context *ctx,
+                          struct crocus_monitor_object *monitor,
+                          bool wait,
+                          union pipe_numeric_type_union *result)
+{
+   struct crocus_context *ice = (void *) ctx;
+   struct intel_perf_context *perf_ctx = ice->perf_ctx;
+   struct crocus_batch *batch = &ice->batches[CROCUS_BATCH_RENDER];
+
+   bool monitor_ready =
+      intel_perf_is_query_ready(perf_ctx, monitor->query, batch);
+
+   if (!monitor_ready) {
+      if (!wait)
+         return false;
+      intel_perf_wait_query(perf_ctx, monitor->query, batch);
+   }
+
+   assert(intel_perf_is_query_ready(perf_ctx, monitor->query, batch));
+
+   unsigned bytes_written;
+   intel_perf_get_query_data(perf_ctx, monitor->query, batch,
+                             monitor->result_size,
+                             (unsigned*) monitor->result_buffer,
+                             &bytes_written);
+   if (bytes_written != monitor->result_size)
+      return false;
+
+   /* copy metrics into the batch result */
+   for (int i = 0; i < monitor->num_active_counters; ++i) {
+      int current_counter = monitor->active_counters[i];
+      const struct intel_perf_query_info *info =
+         intel_perf_query_info(monitor->query);
+      const struct intel_perf_query_counter *counter =
+         &info->counters[current_counter];
+      assert(intel_perf_query_counter_get_size(counter));
+      switch (counter->data_type) {
+      case INTEL_PERF_COUNTER_DATA_TYPE_UINT64:
+         result[i].u64 = *(uint64_t*)(monitor->result_buffer + counter->offset);
+         break;
+      case INTEL_PERF_COUNTER_DATA_TYPE_FLOAT:
+         result[i].f = *(float*)(monitor->result_buffer + counter->offset);
+         break;
+      case INTEL_PERF_COUNTER_DATA_TYPE_UINT32:
+      case INTEL_PERF_COUNTER_DATA_TYPE_BOOL32:
+         result[i].u64 = *(uint32_t*)(monitor->result_buffer + counter->offset);
+         break;
+      case INTEL_PERF_COUNTER_DATA_TYPE_DOUBLE: {
+         double v = *(double*)(monitor->result_buffer + counter->offset);
+         result[i].f = v;
+         break;
+      }
+      default:
+         UNREACHABLE("unexpected counter data type");
+      }
+   }
+   return true;
+}

src/gallium/drivers/crocus/crocus_monitor.h

@@ -0,0 +1,59 @@
+/*
+ * Copyright © 2019 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef CROCUS_MONITOR_H
+#define CROCUS_MONITOR_H
+
+#include "pipe/p_screen.h"
+
+int crocus_get_monitor_info(struct pipe_screen *pscreen, unsigned index,
+                            struct pipe_driver_query_info *info);
+int crocus_get_monitor_group_info(struct pipe_screen *pscreen,
+                                  unsigned index,
+                                  struct pipe_driver_query_group_info *info);
+
+struct crocus_context;
+struct crocus_screen;
+
+struct crocus_monitor_object *
+crocus_create_monitor_object(struct crocus_context *ice,
+                             unsigned num_queries,
+                             unsigned *query_types);
+
+struct pipe_query;
+void crocus_destroy_monitor_object(struct pipe_context *ctx,
+                                   struct crocus_monitor_object *monitor);
+
+bool
+crocus_begin_monitor(struct pipe_context *ctx,
+                     struct crocus_monitor_object *monitor);
+bool
+crocus_end_monitor(struct pipe_context *ctx,
+                   struct crocus_monitor_object *monitor);
+
+bool
+crocus_get_monitor_result(struct pipe_context *ctx,
+                          struct crocus_monitor_object *monitor,
+                          bool wait,
+                          union pipe_numeric_type_union *result);
+
+#endif

src/gallium/drivers/crocus/crocus_perf.c

@@ -0,0 +1,104 @@
+/*
+ * Copyright © 2019 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include "crocus_perf.h"
+#include "crocus_context.h"
+
+static void *
+crocus_oa_bo_alloc(void *bufmgr, const char *name, uint64_t size)
+{
+   return crocus_bo_alloc(bufmgr, name, size);
+}
+
+static void
+crocus_perf_emit_stall_at_pixel_scoreboard(struct crocus_context *ice)
+{
+   crocus_emit_end_of_pipe_sync(&ice->batches[CROCUS_BATCH_RENDER],
+				"OA metrics",
+				PIPE_CONTROL_STALL_AT_SCOREBOARD);
+}
+
+static void
+crocus_perf_emit_mi_report_perf_count(void *c,
+                                       void *bo,
+                                       uint32_t offset_in_bytes,
+                                       uint32_t report_id)
+{
+   struct crocus_context *ice = c;
+   struct crocus_batch *batch = &ice->batches[CROCUS_BATCH_RENDER];
+   batch->screen->vtbl.emit_mi_report_perf_count(batch, bo, offset_in_bytes, report_id);
+}
+
+static void
+crocus_perf_batchbuffer_flush(void *c, const char *file, int line)
+{
+   struct crocus_context *ice = c;
+   _crocus_batch_flush(&ice->batches[CROCUS_BATCH_RENDER], __FILE__, __LINE__);
+}
+
+static void
+crocus_perf_store_register_mem(void *ctx, void *bo,
+                             uint32_t reg, uint32_t reg_size,
+                             uint32_t offset)
+{
+   struct crocus_context *ice = ctx;
+   struct crocus_batch *batch = &ice->batches[CROCUS_BATCH_RENDER];
+   if (reg_size == 8) {
+      batch->screen->vtbl.store_register_mem64(batch, reg, bo, offset, false);
+   } else {
+      assert(reg_size == 4);
+      batch->screen->vtbl.store_register_mem32(batch, reg, bo, offset, false);
+   }
+}
+
+typedef void (*bo_unreference_t)(void *);
+typedef void *(*bo_map_t)(void *, void *, unsigned flags);
+typedef void (*bo_unmap_t)(void *);
+typedef void (*emit_mi_report_t)(void *, void *, uint32_t, uint32_t);
+typedef void (*emit_mi_flush_t)(void *);
+typedef void (*store_register_mem_t)(void *ctx, void *bo,
+                                     uint32_t reg, uint32_t reg_size,
+                                     uint32_t offset);
+typedef bool (*batch_references_t)(void *batch, void *bo);
+typedef void (*bo_wait_rendering_t)(void *bo);
+typedef int (*bo_busy_t)(void *bo);
+
+void
+crocus_perf_init_vtbl(struct intel_perf_config *perf_cfg)
+{
+   perf_cfg->vtbl.bo_alloc = crocus_oa_bo_alloc;
+   perf_cfg->vtbl.bo_unreference = (bo_unreference_t)crocus_bo_unreference;
+   perf_cfg->vtbl.bo_map = (bo_map_t)crocus_bo_map;
+   perf_cfg->vtbl.bo_unmap = (bo_unmap_t)crocus_bo_unmap;
+   perf_cfg->vtbl.emit_stall_at_pixel_scoreboard =
+      (emit_mi_flush_t)crocus_perf_emit_stall_at_pixel_scoreboard;
+
+   perf_cfg->vtbl.emit_mi_report_perf_count =
+      (emit_mi_report_t)crocus_perf_emit_mi_report_perf_count;
+   perf_cfg->vtbl.batchbuffer_flush = crocus_perf_batchbuffer_flush;
+   perf_cfg->vtbl.store_register_mem =
+      (store_register_mem_t) crocus_perf_store_register_mem;
+   perf_cfg->vtbl.batch_references = (batch_references_t)crocus_batch_references;
+   perf_cfg->vtbl.bo_wait_rendering =
+      (bo_wait_rendering_t)crocus_bo_wait_rendering;
+   perf_cfg->vtbl.bo_busy = (bo_busy_t)crocus_bo_busy;
+}

src/gallium/drivers/crocus/crocus_perf.h

@@ -0,0 +1,31 @@
+/*
+ * Copyright © 2019 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef CROCUS_PERF_H
+#define CROCUS_PERF_H
+
+#include "perf/intel_perf.h"
+#include "perf/intel_perf_query.h"
+
+void crocus_perf_init_vtbl(struct intel_perf_config *cfg);
+
+#endif /* CROCUS_PERF_H */

src/gallium/drivers/crocus/crocus_performance_query.c

@@ -0,0 +1,263 @@
+/*
+ * Copyright © 2019 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <xf86drm.h>
+
+#include "crocus_context.h"
+#include "crocus_perf.h"
+
+#include "main/mtypes.h"
+
+struct crocus_perf_query {
+   struct gl_perf_query_object base;
+   struct intel_perf_query_object *query;
+   bool begin_succeeded;
+};
+
+static unsigned
+crocus_init_perf_query_info(struct pipe_context *pipe)
+{
+   struct crocus_context *ice = (void *) pipe;
+   struct crocus_screen *screen = (struct crocus_screen *) ice->ctx.screen;
+   struct intel_perf_config *perf_cfg = NULL;
+
+   /* make sure pipe perf counter type/data-type enums are matched with intel_perf's */
+   STATIC_ASSERT(PIPE_PERF_COUNTER_TYPE_EVENT == (enum pipe_perf_counter_type)INTEL_PERF_COUNTER_TYPE_EVENT);
+   STATIC_ASSERT(PIPE_PERF_COUNTER_TYPE_DURATION_NORM == (enum pipe_perf_counter_type)INTEL_PERF_COUNTER_TYPE_DURATION_NORM);
+   STATIC_ASSERT(PIPE_PERF_COUNTER_TYPE_DURATION_RAW == (enum pipe_perf_counter_type)INTEL_PERF_COUNTER_TYPE_DURATION_RAW);
+   STATIC_ASSERT(PIPE_PERF_COUNTER_TYPE_THROUGHPUT == (enum pipe_perf_counter_type)INTEL_PERF_COUNTER_TYPE_THROUGHPUT);
+   STATIC_ASSERT(PIPE_PERF_COUNTER_TYPE_RAW == (enum pipe_perf_counter_type)INTEL_PERF_COUNTER_TYPE_RAW);
+
+   STATIC_ASSERT(PIPE_PERF_COUNTER_DATA_TYPE_BOOL32 == (enum pipe_perf_counter_data_type)INTEL_PERF_COUNTER_DATA_TYPE_BOOL32);
+   STATIC_ASSERT(PIPE_PERF_COUNTER_DATA_TYPE_UINT32 == (enum pipe_perf_counter_data_type)INTEL_PERF_COUNTER_DATA_TYPE_UINT32);
+   STATIC_ASSERT(PIPE_PERF_COUNTER_DATA_TYPE_UINT64 == (enum pipe_perf_counter_data_type)INTEL_PERF_COUNTER_DATA_TYPE_UINT64);
+   STATIC_ASSERT(PIPE_PERF_COUNTER_DATA_TYPE_FLOAT == (enum pipe_perf_counter_data_type)INTEL_PERF_COUNTER_DATA_TYPE_FLOAT);
+   STATIC_ASSERT(PIPE_PERF_COUNTER_DATA_TYPE_DOUBLE == (enum pipe_perf_counter_data_type)INTEL_PERF_COUNTER_DATA_TYPE_DOUBLE);
+
+   if (!ice->perf_ctx)
+      ice->perf_ctx = intel_perf_new_context(ice);
+
+   if (unlikely(!ice->perf_ctx))
+      return 0;
+
+   perf_cfg = intel_perf_config(ice->perf_ctx);
+
+   if (perf_cfg)
+      return perf_cfg->n_queries;
+
+   perf_cfg = intel_perf_new(ice->perf_ctx);
+
+   crocus_perf_init_vtbl(perf_cfg);
+
+   intel_perf_init_metrics(perf_cfg, &screen->devinfo, screen->fd,
+                           true /* pipeline_statistics */,
+                           true /* register snapshots */);
+
+   intel_perf_init_context(ice->perf_ctx,
+                         perf_cfg,
+                         ice,
+                         ice,
+                         screen->bufmgr,
+                         &screen->devinfo,
+                         ice->batches[CROCUS_BATCH_RENDER].hw_ctx_id,
+                         screen->fd);
+
+   return perf_cfg->n_queries;
+}
+
+static struct pipe_query *
+crocus_new_perf_query_obj(struct pipe_context *pipe, unsigned query_index)
+{
+   struct crocus_context *ice = (void *) pipe;
+   struct intel_perf_context *perf_ctx = ice->perf_ctx;
+   struct intel_perf_query_object * obj =
+      intel_perf_new_query(perf_ctx, query_index);
+   if (unlikely(!obj))
+      return NULL;
+
+   struct crocus_perf_query *q = calloc(1, sizeof(struct crocus_perf_query));
+   if (unlikely(!q)) {
+      intel_perf_delete_query(perf_ctx, obj);
+      return NULL;
+   }
+
+   q->query = obj;
+   return (struct pipe_query *)&q->base;
+}
+
+static bool
+crocus_begin_perf_query(struct pipe_context *pipe, struct pipe_query *q)
+{
+   struct crocus_context *ice = (void *) pipe;
+   struct crocus_perf_query *perf_query= (struct crocus_perf_query *) q;
+   struct intel_perf_query_object *obj = perf_query->query;
+   struct intel_perf_context *perf_ctx = ice->perf_ctx;
+
+   return (perf_query->begin_succeeded = intel_perf_begin_query(perf_ctx, obj));
+}
+
+static void
+crocus_end_perf_query(struct pipe_context *pipe, struct pipe_query *q)
+{
+   struct crocus_context *ice = (void *) pipe;
+   struct crocus_perf_query *perf_query = (struct crocus_perf_query *) q;
+   struct intel_perf_query_object *obj = perf_query->query;
+   struct intel_perf_context *perf_ctx = ice->perf_ctx;
+
+   if (perf_query->begin_succeeded)
+      intel_perf_end_query(perf_ctx, obj);
+}
+
+static void
+crocus_delete_perf_query(struct pipe_context *pipe, struct pipe_query *q)
+{
+   struct crocus_context *ice = (void *) pipe;
+   struct crocus_perf_query *perf_query = (struct crocus_perf_query *) q;
+   struct intel_perf_query_object *obj = perf_query->query;
+   struct intel_perf_context *perf_ctx = ice->perf_ctx;
+
+   intel_perf_delete_query(perf_ctx, obj);
+   free(q);
+}
+
+static void
+crocus_get_perf_query_info(struct pipe_context *pipe,
+                         unsigned query_index,
+                         const char **name,
+                         uint32_t *data_size,
+                         uint32_t *n_counters,
+                         uint32_t *n_active)
+{
+   struct crocus_context *ice = (void *) pipe;
+   struct intel_perf_context *perf_ctx = ice->perf_ctx;
+   struct intel_perf_config *perf_cfg = intel_perf_config(perf_ctx);
+   const struct intel_perf_query_info *info = &perf_cfg->queries[query_index];
+
+   *name = info->name;
+   *data_size = info->data_size;
+   *n_counters = info->n_counters;
+   *n_active = intel_perf_active_queries(perf_ctx, info);
+}
+
+static void
+crocus_get_perf_counter_info(struct pipe_context *pipe,
+                           unsigned query_index,
+                           unsigned counter_index,
+                           const char **name,
+                           const char **desc,
+                           uint32_t *offset,
+                           uint32_t *data_size,
+                           uint32_t *type_enum,
+                           uint32_t *data_type_enum,
+                           uint64_t *raw_max)
+{
+   struct crocus_context *ice = (void *) pipe;
+   struct intel_perf_context *perf_ctx = ice->perf_ctx;
+   struct intel_perf_config *perf_cfg = intel_perf_config(perf_ctx);
+   const struct intel_perf_query_info *info = &perf_cfg->queries[query_index];
+   const struct intel_perf_query_counter *counter =
+      &info->counters[counter_index];
+   struct intel_perf_query_result results;
+
+   intel_perf_query_result_clear(&results);
+
+   *name = counter->name;
+   *desc = counter->desc;
+   *offset = counter->offset;
+   *data_size = intel_perf_query_counter_get_size(counter);
+   *type_enum = counter->type;
+   *data_type_enum = counter->data_type;
+
+   if (counter->oa_counter_max_uint64) {
+      if (counter->data_type == INTEL_PERF_COUNTER_DATA_TYPE_FLOAT ||
+          counter->data_type == INTEL_PERF_COUNTER_DATA_TYPE_DOUBLE)
+         *raw_max = counter->oa_counter_max_float(perf_cfg, info, &results);
+      else
+         *raw_max = counter->oa_counter_max_uint64(perf_cfg, info, &results);
+   } else {
+      *raw_max = 0;
+   }
+}
+
+static void
+crocus_wait_perf_query(struct pipe_context *pipe, struct pipe_query *q)
+{
+   struct crocus_context *ice = (void *) pipe;
+   struct crocus_perf_query *perf_query = (struct crocus_perf_query *) q;
+   struct intel_perf_query_object *obj = perf_query->query;
+   struct intel_perf_context *perf_ctx = ice->perf_ctx;
+
+   if (perf_query->begin_succeeded)
+      intel_perf_wait_query(perf_ctx, obj, &ice->batches[CROCUS_BATCH_RENDER]);
+}
+
+static bool
+crocus_is_perf_query_ready(struct pipe_context *pipe, struct pipe_query *q)
+{
+   struct crocus_context *ice = (void *) pipe;
+   struct crocus_perf_query *perf_query = (struct crocus_perf_query *) q;
+   struct intel_perf_query_object *obj = perf_query->query;
+   struct intel_perf_context *perf_ctx = ice->perf_ctx;
+
+   if (perf_query->base.Ready)
+      return true;
+   if (!perf_query->begin_succeeded)
+      return true;
+
+   return intel_perf_is_query_ready(perf_ctx, obj,
+                                    &ice->batches[CROCUS_BATCH_RENDER]);
+}
+
+static bool
+crocus_get_perf_query_data(struct pipe_context *pipe,
+                         struct pipe_query *q,
+                         size_t data_size,
+                         uint32_t *data,
+                         uint32_t *bytes_written)
+{
+   struct crocus_context *ice = (void *) pipe;
+   struct crocus_perf_query *perf_query = (struct crocus_perf_query *) q;
+   struct intel_perf_query_object *obj = perf_query->query;
+   struct intel_perf_context *perf_ctx = ice->perf_ctx;
+
+   if (perf_query->begin_succeeded) {
+      intel_perf_get_query_data(perf_ctx, obj, &ice->batches[CROCUS_BATCH_RENDER],
+            data_size, data, bytes_written);
+   }
+
+   return perf_query->begin_succeeded;
+}
+
+void
+crocus_init_perfquery_functions(struct pipe_context *ctx)
+{
+   ctx->init_intel_perf_query_info = crocus_init_perf_query_info;
+   ctx->get_intel_perf_query_info = crocus_get_perf_query_info;
+   ctx->get_intel_perf_query_counter_info = crocus_get_perf_counter_info;
+   ctx->new_intel_perf_query_obj = crocus_new_perf_query_obj;
+   ctx->begin_intel_perf_query = crocus_begin_perf_query;
+   ctx->end_intel_perf_query = crocus_end_perf_query;
+   ctx->delete_intel_perf_query = crocus_delete_perf_query;
+   ctx->wait_intel_perf_query = crocus_wait_perf_query;
+   ctx->is_intel_perf_query_ready = crocus_is_perf_query_ready;
+   ctx->get_intel_perf_query_data = crocus_get_perf_query_data;
+}

src/gallium/drivers/crocus/crocus_pipe.h

@@ -0,0 +1,58 @@
+/*
+ * Copyright © 2017 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * on the rights to use, copy, modify, merge, publish, distribute, sub
+ * license, and/or sell copies of the Software, and to permit persons to whom
+ * the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef CROCUS_PIPE_H
+#define CROCUS_PIPE_H
+
+#include "pipe/p_defines.h"
+#include "compiler/shader_enums.h"
+
+static inline mesa_shader_stage
+stage_from_pipe(mesa_shader_stage pstage)
+{
+   return (mesa_shader_stage)pstage;
+}
+
+static inline mesa_shader_stage
+stage_to_pipe(mesa_shader_stage stage)
+{
+   return (mesa_shader_stage)stage;
+}
+
+/**
+ * Convert an swizzle enumeration (i.e. PIPE_SWIZZLE_X) to one of the HW's
+ * "Shader Channel Select" enumerations (i.e. SCS_RED).  The mappings are
+ *
+ * SWIZZLE_X, SWIZZLE_Y, SWIZZLE_Z, SWIZZLE_W, SWIZZLE_ZERO, SWIZZLE_ONE
+ *         0          1          2          3             4            5
+ *         4          5          6          7             0            1
+ *   SCS_RED, SCS_GREEN,  SCS_BLUE, SCS_ALPHA,     SCS_ZERO,     SCS_ONE
+ *
+ * which is simply adding 4 then modding by 8 (or anding with 7).
+ */
+static inline enum isl_channel_select
+pipe_swizzle_to_isl_channel(enum pipe_swizzle swizzle)
+{
+   return (swizzle + 4) & 7;
+}
+
+#endif

src/gallium/drivers/crocus/crocus_pipe_control.c

@@ -0,0 +1,376 @@
+/*
+ * Copyright © 2017 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+/**
+ * @file crocus_pipe_control.c
+ *
+ * PIPE_CONTROL is the main flushing and synchronization primitive on Intel
+ * GPUs.  It can invalidate caches, stall until rendering reaches various
+ * stages of completion, write to memory, and other things.  In a way, it's
+ * a swiss army knife command - it has all kinds of capabilities, but some
+ * significant limitations as well.
+ *
+ * Unfortunately, it's notoriously complicated and difficult to use.  Many
+ * sub-commands can't be used together.  Some are meant to be used at the
+ * top of the pipeline (invalidating caches before drawing), while some are
+ * meant to be used at the end (stalling or flushing after drawing).
+ *
+ * Also, there's a list of restrictions a mile long, which vary by generation.
+ * Do this before doing that, or suffer the consequences (usually a GPU hang).
+ *
+ * This file contains helpers for emitting them safely.  You can simply call
+ * crocus_emit_pipe_control_flush() with the desired operations (as logical
+ * PIPE_CONTROL_* bits), and it will take care of splitting it into multiple
+ * PIPE_CONTROL commands as necessary.  The per-generation workarounds are
+ * applied in crocus_emit_raw_pipe_control() in crocus_state.c.
+ */
+
+#include "crocus_context.h"
+#include "util/hash_table.h"
+#include "util/set.h"
+
+/**
+ * Emit a PIPE_CONTROL with various flushing flags.
+ *
+ * The caller is responsible for deciding what flags are appropriate for the
+ * given generation.
+ */
+void
+crocus_emit_pipe_control_flush(struct crocus_batch *batch,
+                               const char *reason,
+                               uint32_t flags)
+{
+   const struct intel_device_info *devinfo = &batch->screen->devinfo;
+
+   if (devinfo->ver >= 6 &&
+       (flags & PIPE_CONTROL_CACHE_FLUSH_BITS) &&
+       (flags & PIPE_CONTROL_CACHE_INVALIDATE_BITS)) {
+      /* A pipe control command with flush and invalidate bits set
+       * simultaneously is an inherently racy operation on Gen6+ if the
+       * contents of the flushed caches were intended to become visible from
+       * any of the invalidated caches.  Split it in two PIPE_CONTROLs, the
+       * first one should stall the pipeline to make sure that the flushed R/W
+       * caches are coherent with memory once the specified R/O caches are
+       * invalidated.  On pre-Gen6 hardware the (implicit) R/O cache
+       * invalidation seems to happen at the bottom of the pipeline together
+       * with any write cache flush, so this shouldn't be a concern.  In order
+       * to ensure a full stall, we do an end-of-pipe sync.
+       */
+      crocus_emit_end_of_pipe_sync(batch, reason,
+                                   flags & PIPE_CONTROL_CACHE_FLUSH_BITS);
+      flags &= ~(PIPE_CONTROL_CACHE_FLUSH_BITS | PIPE_CONTROL_CS_STALL);
+   }
+
+   batch->screen->vtbl.emit_raw_pipe_control(batch, reason, flags, NULL, 0, 0);
+}
+
+/**
+ * Emit a PIPE_CONTROL that writes to a buffer object.
+ *
+ * \p flags should contain one of the following items:
+ *  - PIPE_CONTROL_WRITE_IMMEDIATE
+ *  - PIPE_CONTROL_WRITE_TIMESTAMP
+ *  - PIPE_CONTROL_WRITE_DEPTH_COUNT
+ */
+void
+crocus_emit_pipe_control_write(struct crocus_batch *batch,
+                               const char *reason, uint32_t flags,
+                               struct crocus_bo *bo, uint32_t offset,
+                               uint64_t imm)
+{
+   batch->screen->vtbl.emit_raw_pipe_control(batch, reason, flags, bo, offset, imm);
+}
+
+/**
+ * Restriction [DevSNB, DevIVB]:
+ *
+ * Prior to changing Depth/Stencil Buffer state (i.e. any combination of
+ * 3DSTATE_DEPTH_BUFFER, 3DSTATE_CLEAR_PARAMS, 3DSTATE_STENCIL_BUFFER,
+ * 3DSTATE_HIER_DEPTH_BUFFER) SW must first issue a pipelined depth stall
+ * (PIPE_CONTROL with Depth Stall bit set), followed by a pipelined depth
+ * cache flush (PIPE_CONTROL with Depth Flush Bit set), followed by
+ * another pipelined depth stall (PIPE_CONTROL with Depth Stall bit set),
+ * unless SW can otherwise guarantee that the pipeline from WM onwards is
+ * already flushed (e.g., via a preceding MI_FLUSH).
+ */
+void
+crocus_emit_depth_stall_flushes(struct crocus_batch *batch)
+{
+   UNUSED const struct intel_device_info *devinfo = &batch->screen->devinfo;
+
+   assert(devinfo->ver >= 6);
+
+   /* Starting on BDW, these pipe controls are unnecessary.
+    *
+    *   WM HW will internally manage the draining pipe and flushing of the caches
+    *   when this command is issued. The PIPE_CONTROL restrictions are removed.
+    */
+   if (devinfo->ver >= 8)
+      return;
+
+   crocus_emit_pipe_control_flush(batch, "depth stall", PIPE_CONTROL_DEPTH_STALL);
+   crocus_emit_pipe_control_flush(batch, "depth stall", PIPE_CONTROL_DEPTH_CACHE_FLUSH);
+   crocus_emit_pipe_control_flush(batch, "depth stall", PIPE_CONTROL_DEPTH_STALL);
+}
+
+/*
+ * From Sandybridge PRM, volume 2, "1.7.2 End-of-Pipe Synchronization":
+ *
+ *  Write synchronization is a special case of end-of-pipe
+ *  synchronization that requires that the render cache and/or depth
+ *  related caches are flushed to memory, where the data will become
+ *  globally visible. This type of synchronization is required prior to
+ *  SW (CPU) actually reading the result data from memory, or initiating
+ *  an operation that will use as a read surface (such as a texture
+ *  surface) a previous render target and/or depth/stencil buffer
+ *
+ * From Haswell PRM, volume 2, part 1, "End-of-Pipe Synchronization":
+ *
+ *  Exercising the write cache flush bits (Render Target Cache Flush
+ *  Enable, Depth Cache Flush Enable, DC Flush) in PIPE_CONTROL only
+ *  ensures the write caches are flushed and doesn't guarantee the data
+ *  is globally visible.
+ *
+ *  SW can track the completion of the end-of-pipe-synchronization by
+ *  using "Notify Enable" and "PostSync Operation - Write Immediate
+ *  Data" in the PIPE_CONTROL command.
+ */
+void
+crocus_emit_end_of_pipe_sync(struct crocus_batch *batch,
+                             const char *reason, uint32_t flags)
+{
+   const struct intel_device_info *devinfo = &batch->screen->devinfo;
+
+   if (devinfo->ver >= 6) {
+      /* From Sandybridge PRM, volume 2, "1.7.3.1 Writing a Value to Memory":
+       *
+       *    "The most common action to perform upon reaching a synchronization
+       *    point is to write a value out to memory. An immediate value
+       *    (included with the synchronization command) may be written."
+       *
+       * From Broadwell PRM, volume 7, "End-of-Pipe Synchronization":
+       *
+       *    "In case the data flushed out by the render engine is to be read
+       *    back in to the render engine in coherent manner, then the render
+       *    engine has to wait for the fence completion before accessing the
+       *    flushed data. This can be achieved by following means on various
+       *    products: PIPE_CONTROL command with CS Stall and the required
+       *    write caches flushed with Post-Sync-Operation as Write Immediate
+       *    Data.
+       *
+       *    Example:
+       *       - Workload-1 (3D/GPGPU/MEDIA)
+       *       - PIPE_CONTROL (CS Stall, Post-Sync-Operation Write Immediate
+       *         Data, Required Write Cache Flush bits set)
+       *       - Workload-2 (Can use the data produce or output by Workload-1)
+       */
+      crocus_emit_pipe_control_write(batch, reason,
+                                     flags | PIPE_CONTROL_CS_STALL |
+                                     PIPE_CONTROL_WRITE_IMMEDIATE,
+                                     batch->ice->workaround_bo,
+                                     batch->ice->workaround_offset, 0);
+
+      if (batch->screen->devinfo.platform == INTEL_PLATFORM_HSW) {
+#define GEN7_3DPRIM_START_INSTANCE      0x243C
+         batch->screen->vtbl.load_register_mem32(batch, GEN7_3DPRIM_START_INSTANCE,
+                                                 batch->ice->workaround_bo,
+                                                 batch->ice->workaround_offset);
+      }
+   } else {
+      /* On gen4-5, a regular pipe control seems to suffice. */
+      crocus_emit_pipe_control_flush(batch, reason, flags);
+   }
+}
+
+/* Emit a pipelined flush to either flush render and texture cache for
+ * reading from a FBO-drawn texture, or flush so that frontbuffer
+ * render appears on the screen in DRI1.
+ *
+ * This is also used for the always_flush_cache driconf debug option.
+ */
+void
+crocus_emit_mi_flush(struct crocus_batch *batch)
+{
+   const struct intel_device_info *devinfo = &batch->screen->devinfo;
+   int flags = PIPE_CONTROL_RENDER_TARGET_FLUSH;
+   if (devinfo->ver >= 6) {
+      flags |= PIPE_CONTROL_INSTRUCTION_INVALIDATE |
+               PIPE_CONTROL_CONST_CACHE_INVALIDATE |
+               PIPE_CONTROL_DATA_CACHE_FLUSH |
+               PIPE_CONTROL_DEPTH_CACHE_FLUSH |
+               PIPE_CONTROL_VF_CACHE_INVALIDATE |
+               PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
+               PIPE_CONTROL_CS_STALL;
+   }
+   crocus_emit_pipe_control_flush(batch, "mi flush", flags);
+}
+
+/**
+ * Emits a PIPE_CONTROL with a non-zero post-sync operation, for
+ * implementing two workarounds on gen6.  From section 1.4.7.1
+ * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
+ *
+ * [DevSNB-C+{W/A}] Before any depth stall flush (including those
+ * produced by non-pipelined state commands), software needs to first
+ * send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
+ * 0.
+ *
+ * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
+ * =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
+ *
+ * And the workaround for these two requires this workaround first:
+ *
+ * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
+ * BEFORE the pipe-control with a post-sync op and no write-cache
+ * flushes.
+ *
+ * And this last workaround is tricky because of the requirements on
+ * that bit.  From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
+ * volume 2 part 1:
+ *
+ *     "1 of the following must also be set:
+ *      - Render Target Cache Flush Enable ([12] of DW1)
+ *      - Depth Cache Flush Enable ([0] of DW1)
+ *      - Stall at Pixel Scoreboard ([1] of DW1)
+ *      - Depth Stall ([13] of DW1)
+ *      - Post-Sync Operation ([13] of DW1)
+ *      - Notify Enable ([8] of DW1)"
+ *
+ * The cache flushes require the workaround flush that triggered this
+ * one, so we can't use it.  Depth stall would trigger the same.
+ * Post-sync nonzero is what triggered this second workaround, so we
+ * can't use that one either.  Notify enable is IRQs, which aren't
+ * really our business.  That leaves only stall at scoreboard.
+ */
+void
+crocus_emit_post_sync_nonzero_flush(struct crocus_batch *batch)
+{
+   crocus_emit_pipe_control_flush(batch, "nonzero",
+                                  PIPE_CONTROL_CS_STALL |
+                                  PIPE_CONTROL_STALL_AT_SCOREBOARD);
+
+   crocus_emit_pipe_control_write(batch, "nonzero",
+                                  PIPE_CONTROL_WRITE_IMMEDIATE,
+                                  batch->ice->workaround_bo,
+                                  batch->ice->workaround_offset, 0);
+}
+
+/**
+ * Flush and invalidate all caches (for debugging purposes).
+ */
+void
+crocus_flush_all_caches(struct crocus_batch *batch)
+{
+   crocus_emit_pipe_control_flush(batch, "debug: flush all caches",
+                                  PIPE_CONTROL_CS_STALL |
+                                  PIPE_CONTROL_DATA_CACHE_FLUSH |
+                                  PIPE_CONTROL_DEPTH_CACHE_FLUSH |
+                                  PIPE_CONTROL_RENDER_TARGET_FLUSH |
+                                  PIPE_CONTROL_VF_CACHE_INVALIDATE |
+                                  PIPE_CONTROL_INSTRUCTION_INVALIDATE |
+                                  PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
+                                  PIPE_CONTROL_CONST_CACHE_INVALIDATE |
+                                  PIPE_CONTROL_STATE_CACHE_INVALIDATE);
+}
+
+static void
+crocus_texture_barrier(struct pipe_context *ctx, unsigned flags)
+{
+   struct crocus_context *ice = (void *) ctx;
+   struct crocus_batch *render_batch = &ice->batches[CROCUS_BATCH_RENDER];
+   struct crocus_batch *compute_batch = &ice->batches[CROCUS_BATCH_COMPUTE];
+   const struct intel_device_info *devinfo = &render_batch->screen->devinfo;
+
+   if (devinfo->ver < 6) {
+      crocus_emit_mi_flush(render_batch);
+      return;
+   }
+
+   if (render_batch->contains_draw) {
+      crocus_batch_maybe_flush(render_batch, 48);
+      crocus_emit_pipe_control_flush(render_batch,
+                                     "API: texture barrier (1/2)",
+                                     (flags == 1 ? PIPE_CONTROL_DEPTH_CACHE_FLUSH  : 0) |
+                                     PIPE_CONTROL_RENDER_TARGET_FLUSH |
+                                     PIPE_CONTROL_CS_STALL);
+      crocus_emit_pipe_control_flush(render_batch,
+                                     "API: texture barrier (2/2)",
+                                     PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
+   }
+
+   if (compute_batch->contains_draw) {
+      crocus_batch_maybe_flush(compute_batch, 48);
+      crocus_emit_pipe_control_flush(compute_batch,
+                                     "API: texture barrier (1/2)",
+                                     PIPE_CONTROL_CS_STALL);
+      crocus_emit_pipe_control_flush(compute_batch,
+                                     "API: texture barrier (2/2)",
+                                     PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
+   }
+}
+
+static void
+crocus_memory_barrier(struct pipe_context *ctx, unsigned flags)
+{
+   struct crocus_context *ice = (void *) ctx;
+   unsigned bits = PIPE_CONTROL_DATA_CACHE_FLUSH | PIPE_CONTROL_CS_STALL;
+   const struct intel_device_info *devinfo = &ice->batches[0].screen->devinfo;
+
+   assert(devinfo->ver >= 7);
+
+   if (flags & (PIPE_BARRIER_VERTEX_BUFFER |
+                PIPE_BARRIER_INDEX_BUFFER |
+                PIPE_BARRIER_INDIRECT_BUFFER)) {
+      bits |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
+   }
+
+   if (flags & PIPE_BARRIER_CONSTANT_BUFFER) {
+      bits |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
+              PIPE_CONTROL_CONST_CACHE_INVALIDATE;
+   }
+
+   if (flags & (PIPE_BARRIER_TEXTURE | PIPE_BARRIER_FRAMEBUFFER)) {
+      bits |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
+              PIPE_CONTROL_RENDER_TARGET_FLUSH;
+   }
+
+   /* Typed surface messages are handled by the render cache on IVB, so we
+    * need to flush it too.
+    */
+   if (devinfo->verx10 < 75)
+      bits |= PIPE_CONTROL_RENDER_TARGET_FLUSH;
+
+   for (int i = 0; i < ice->batch_count; i++) {
+      if (ice->batches[i].contains_draw) {
+         crocus_batch_maybe_flush(&ice->batches[i], 24);
+         crocus_emit_pipe_control_flush(&ice->batches[i], "API: memory barrier",
+                                        bits);
+      }
+   }
+}
+
+void
+crocus_init_flush_functions(struct pipe_context *ctx)
+{
+   ctx->memory_barrier = crocus_memory_barrier;
+   ctx->texture_barrier = crocus_texture_barrier;
+}

src/gallium/drivers/crocus/crocus_program_cache.c

@@ -0,0 +1,346 @@
+/*
+ * Copyright © 2017 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+/**
+ * @file crocus_program_cache.c
+ *
+ * The in-memory program cache.  This is basically a hash table mapping
+ * API-specified shaders and a state key to a compiled variant.  It also
+ * takes care of uploading shader assembly into a BO for use on the GPU.
+ */
+
+#include <stdio.h>
+#include <errno.h>
+#include "pipe/p_defines.h"
+#include "pipe/p_state.h"
+#include "pipe/p_context.h"
+#include "pipe/p_screen.h"
+#include "util/u_atomic.h"
+#include "util/u_upload_mgr.h"
+#include "compiler/nir/nir.h"
+#include "compiler/nir/nir_builder.h"
+#include "intel/compiler/elk/elk_compiler.h"
+#include "intel/compiler/elk/elk_disasm.h"
+#include "intel/compiler/elk/elk_nir.h"
+#include "crocus_context.h"
+#include "crocus_resource.h"
+
+struct keybox {
+   uint16_t size;
+   enum crocus_program_cache_id cache_id;
+   uint8_t data[0];
+};
+
+static struct keybox *
+make_keybox(void *mem_ctx, enum crocus_program_cache_id cache_id,
+            const void *key, uint32_t key_size)
+{
+   struct keybox *keybox =
+      ralloc_size(mem_ctx, sizeof(struct keybox) + key_size);
+
+   keybox->cache_id = cache_id;
+   keybox->size = key_size;
+   memcpy(keybox->data, key, key_size);
+
+   return keybox;
+}
+
+static uint32_t
+keybox_hash(const void *void_key)
+{
+   const struct keybox *key = void_key;
+   return _mesa_hash_data(&key->cache_id, key->size + sizeof(key->cache_id));
+}
+
+static bool
+keybox_equals(const void *void_a, const void *void_b)
+{
+   const struct keybox *a = void_a, *b = void_b;
+   if (a->size != b->size)
+      return false;
+
+   return memcmp(a->data, b->data, a->size) == 0;
+}
+
+struct crocus_compiled_shader *
+crocus_find_cached_shader(struct crocus_context *ice,
+                          enum crocus_program_cache_id cache_id,
+                          uint32_t key_size, const void *key)
+{
+   struct keybox *keybox = make_keybox(NULL, cache_id, key, key_size);
+   struct hash_entry *entry =
+      _mesa_hash_table_search(ice->shaders.cache, keybox);
+
+   ralloc_free(keybox);
+
+   return entry ? entry->data : NULL;
+}
+
+const void *
+crocus_find_previous_compile(const struct crocus_context *ice,
+                             enum crocus_program_cache_id cache_id,
+                             unsigned program_string_id)
+{
+   hash_table_foreach(ice->shaders.cache, entry) {
+      const struct keybox *keybox = entry->key;
+      const struct elk_base_prog_key *key = (const void *)keybox->data;
+      if (keybox->cache_id == cache_id &&
+          key->program_string_id == program_string_id) {
+         return keybox->data;
+      }
+   }
+
+   return NULL;
+}
+
+/**
+ * Look for an existing entry in the cache that has identical assembly code.
+ *
+ * This is useful for programs generating shaders at runtime, where multiple
+ * distinct shaders (from an API perspective) may compile to the same assembly
+ * in our backend.  This saves space in the program cache buffer.
+ */
+static const struct crocus_compiled_shader *
+find_existing_assembly(struct hash_table *cache, void *map,
+                       const void *assembly, unsigned assembly_size)
+{
+   hash_table_foreach (cache, entry) {
+      const struct crocus_compiled_shader *existing = entry->data;
+
+      if (existing->map_size != assembly_size)
+         continue;
+
+      if (memcmp(map + existing->offset, assembly, assembly_size) == 0)
+         return existing;
+   }
+   return NULL;
+}
+
+static void
+crocus_cache_new_bo(struct crocus_context *ice,
+                    uint32_t new_size)
+{
+   struct crocus_screen *screen = (struct crocus_screen *)ice->ctx.screen;
+   struct crocus_bo *new_bo;
+   new_bo = crocus_bo_alloc(screen->bufmgr, "program cache", new_size);
+
+   void *map = crocus_bo_map(NULL, new_bo, MAP_READ | MAP_WRITE |
+                             MAP_ASYNC | MAP_PERSISTENT);
+
+   if (ice->shaders.cache_next_offset != 0) {
+      memcpy(map, ice->shaders.cache_bo_map, ice->shaders.cache_next_offset);
+   }
+
+   crocus_bo_unmap(ice->shaders.cache_bo);
+   crocus_bo_unreference(ice->shaders.cache_bo);
+   ice->shaders.cache_bo = new_bo;
+   ice->shaders.cache_bo_map = map;
+
+   if (screen->devinfo.ver <= 5) {
+      /* reemit all shaders on GEN4 only. */
+      ice->state.dirty |= CROCUS_DIRTY_CLIP | CROCUS_DIRTY_RASTER |
+         CROCUS_DIRTY_WM;
+      ice->state.stage_dirty |= CROCUS_STAGE_DIRTY_VS;
+   }
+   ice->batches[CROCUS_BATCH_RENDER].state_base_address_emitted = false;
+   ice->batches[CROCUS_BATCH_COMPUTE].state_base_address_emitted = false;
+   /* unset state base address */
+}
+
+static uint32_t
+crocus_alloc_item_data(struct crocus_context *ice, uint32_t size)
+{
+   if (ice->shaders.cache_next_offset + size > ice->shaders.cache_bo->size) {
+      uint32_t new_size = ice->shaders.cache_bo->size * 2;
+      while (ice->shaders.cache_next_offset + size > new_size)
+         new_size *= 2;
+
+      crocus_cache_new_bo(ice, new_size);
+   }
+   uint32_t offset = ice->shaders.cache_next_offset;
+
+   /* Programs are always 64-byte aligned, so set up the next one now */
+   ice->shaders.cache_next_offset = align(offset + size, 64);
+   return offset;
+}
+
+struct crocus_compiled_shader *
+crocus_upload_shader(struct crocus_context *ice,
+                     enum crocus_program_cache_id cache_id, uint32_t key_size,
+                     const void *key, const void *assembly, uint32_t asm_size,
+                     struct elk_stage_prog_data *prog_data,
+                     uint32_t prog_data_size, uint32_t *streamout,
+                     enum elk_param_builtin *system_values,
+                     unsigned num_system_values, unsigned num_cbufs,
+                     const struct crocus_binding_table *bt)
+{
+   struct hash_table *cache = ice->shaders.cache;
+   struct crocus_compiled_shader *shader =
+      rzalloc_size(cache, sizeof(struct crocus_compiled_shader));
+   const struct crocus_compiled_shader *existing = find_existing_assembly(
+      cache, ice->shaders.cache_bo_map, assembly, asm_size);
+
+   /* If we can find a matching prog in the cache already, then reuse the
+    * existing stuff without creating new copy into the underlying buffer
+    * object.  This is notably useful for programs generating shaders at
+    * runtime, where multiple shaders may compile to the same thing in our
+    * backend.
+    */
+   if (existing) {
+      shader->offset = existing->offset;
+      shader->map_size = existing->map_size;
+   } else {
+      shader->offset = crocus_alloc_item_data(ice, asm_size);
+      shader->map_size = asm_size;
+
+      memcpy(ice->shaders.cache_bo_map + shader->offset, assembly, asm_size);
+   }
+
+   shader->prog_data = prog_data;
+   shader->prog_data_size = prog_data_size;
+   shader->streamout = streamout;
+   shader->system_values = system_values;
+   shader->num_system_values = num_system_values;
+   shader->num_cbufs = num_cbufs;
+   shader->bt = *bt;
+
+   ralloc_steal(shader, shader->prog_data);
+   if (prog_data_size > 16)
+      ralloc_steal(shader->prog_data, prog_data->param);
+   ralloc_steal(shader, shader->streamout);
+   ralloc_steal(shader, shader->system_values);
+
+   struct keybox *keybox = make_keybox(shader, cache_id, key, key_size);
+   _mesa_hash_table_insert(ice->shaders.cache, keybox, shader);
+
+   return shader;
+}
+
+bool
+crocus_blorp_lookup_shader(struct blorp_batch *blorp_batch, const void *key,
+                           uint32_t key_size, uint32_t *kernel_out,
+                           void *prog_data_out)
+{
+   struct blorp_context *blorp = blorp_batch->blorp;
+   struct crocus_context *ice = blorp->driver_ctx;
+   struct crocus_compiled_shader *shader =
+      crocus_find_cached_shader(ice, CROCUS_CACHE_BLORP, key_size, key);
+
+   if (!shader)
+      return false;
+
+   *kernel_out = shader->offset;
+   *((void **)prog_data_out) = shader->prog_data;
+
+   return true;
+}
+
+bool
+crocus_blorp_upload_shader(struct blorp_batch *blorp_batch, uint32_t stage,
+                           const void *key, uint32_t key_size,
+                           const void *kernel, uint32_t kernel_size,
+                           const void *prog_data_templ,
+                           uint32_t prog_data_size, uint32_t *kernel_out,
+                           void *prog_data_out)
+{
+   struct blorp_context *blorp = blorp_batch->blorp;
+   struct crocus_context *ice = blorp->driver_ctx;
+
+   struct elk_stage_prog_data *prog_data = ralloc_size(NULL, prog_data_size);
+   memcpy(prog_data, prog_data_templ, prog_data_size);
+
+   struct crocus_binding_table bt;
+   memset(&bt, 0, sizeof(bt));
+
+   struct crocus_compiled_shader *shader = crocus_upload_shader(
+      ice, CROCUS_CACHE_BLORP, key_size, key, kernel, kernel_size, prog_data,
+      prog_data_size, NULL, NULL, 0, 0, &bt);
+
+   *kernel_out = shader->offset;
+   *((void **)prog_data_out) = shader->prog_data;
+
+   return true;
+}
+
+void
+crocus_init_program_cache(struct crocus_context *ice)
+{
+   struct crocus_screen *screen = (struct crocus_screen *)ice->ctx.screen;
+   ice->shaders.cache =
+      _mesa_hash_table_create(ice, keybox_hash, keybox_equals);
+
+   ice->shaders.cache_bo =
+      crocus_bo_alloc(screen->bufmgr, "program_cache", 16384);
+   ice->shaders.cache_bo_map =
+      crocus_bo_map(NULL, ice->shaders.cache_bo,
+                    MAP_READ | MAP_WRITE | MAP_ASYNC | MAP_PERSISTENT);
+}
+
+void
+crocus_destroy_program_cache(struct crocus_context *ice)
+{
+   for (int i = 0; i < MESA_SHADER_STAGES; i++) {
+      ice->shaders.prog[i] = NULL;
+   }
+
+   if (ice->shaders.cache_bo) {
+      crocus_bo_unmap(ice->shaders.cache_bo);
+      crocus_bo_unreference(ice->shaders.cache_bo);
+      ice->shaders.cache_bo_map = NULL;
+      ice->shaders.cache_bo = NULL;
+   }
+
+   ralloc_free(ice->shaders.cache);
+}
+
+static const char *
+cache_name(enum crocus_program_cache_id cache_id)
+{
+   if (cache_id == CROCUS_CACHE_BLORP)
+      return "BLORP";
+
+   if (cache_id == CROCUS_CACHE_SF)
+      return "SF";
+
+   if (cache_id == CROCUS_CACHE_CLIP)
+      return "CLIP";
+
+   if (cache_id == CROCUS_CACHE_FF_GS)
+      return "FF_GS";
+
+   return _mesa_shader_stage_to_string(cache_id);
+}
+
+void
+crocus_print_program_cache(struct crocus_context *ice)
+{
+   struct crocus_screen *screen = (struct crocus_screen *)ice->ctx.screen;
+   const struct elk_isa_info *isa = &screen->compiler->isa;
+
+   hash_table_foreach(ice->shaders.cache, entry) {
+      const struct keybox *keybox = entry->key;
+      struct crocus_compiled_shader *shader = entry->data;
+      fprintf(stderr, "%s:\n", cache_name(keybox->cache_id));
+      elk_disassemble(isa, ice->shaders.cache_bo_map + shader->offset, 0,
+                      shader->prog_data->program_size, NULL, stderr);
+   }
+}

src/gallium/drivers/crocus/crocus_resource.h

@@ -0,0 +1,534 @@
+/*
+ * Copyright 2017 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * on the rights to use, copy, modify, merge, publish, distribute, sub
+ * license, and/or sell copies of the Software, and to permit persons to whom
+ * the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef CROCUS_RESOURCE_H
+#define CROCUS_RESOURCE_H
+
+#include "pipe/p_state.h"
+#include "util/u_inlines.h"
+#include "util/u_range.h"
+#include "util/u_threaded_context.h"
+#include "intel/isl/isl.h"
+#include "intel/dev/intel_device_info.h"
+#include "crocus_bufmgr.h"
+
+struct crocus_batch;
+struct crocus_context;
+
+#define CROCUS_MAX_MIPLEVELS 15
+
+struct crocus_format_info {
+   enum isl_format fmt;
+   enum pipe_swizzle swizzles[4];
+};
+
+static inline enum isl_channel_select
+pipe_to_isl_swizzle(const enum pipe_swizzle pswz, bool green_to_blue)
+{
+   unsigned swz = (pswz + 4) & 7;
+
+   return (green_to_blue && swz == ISL_CHANNEL_SELECT_GREEN) ? ISL_CHANNEL_SELECT_BLUE : swz;
+}
+
+static inline struct isl_swizzle
+pipe_to_isl_swizzles(const enum pipe_swizzle pswz[4])
+{
+   struct isl_swizzle swz;
+   swz.r = pipe_to_isl_swizzle(pswz[0], false);
+   swz.g = pipe_to_isl_swizzle(pswz[1], false);
+   swz.b = pipe_to_isl_swizzle(pswz[2], false);
+   swz.a = pipe_to_isl_swizzle(pswz[3], false);
+   return swz;
+}
+
+static inline void
+crocus_combine_swizzle(enum pipe_swizzle outswz[4],
+                       const enum pipe_swizzle fswz[4],
+                       const enum pipe_swizzle vswz[4])
+{
+   for (unsigned i = 0; i < 4; i++) {
+      switch (vswz[i]) {
+      case PIPE_SWIZZLE_X: outswz[i] = fswz[0]; break;
+      case PIPE_SWIZZLE_Y: outswz[i] = fswz[1]; break;
+      case PIPE_SWIZZLE_Z: outswz[i] = fswz[2]; break;
+      case PIPE_SWIZZLE_W: outswz[i] = fswz[3]; break;
+      case PIPE_SWIZZLE_1: outswz[i] = PIPE_SWIZZLE_1; break;
+      case PIPE_SWIZZLE_0: outswz[i] = PIPE_SWIZZLE_0; break;
+      default: UNREACHABLE("invalid swizzle");
+      }
+   }
+}
+
+/**
+ * Resources represent a GPU buffer object or image (mipmap tree).
+ *
+ * They contain the storage (BO) and layout information (ISL surface).
+ */
+struct crocus_resource {
+   struct threaded_resource base;
+   enum pipe_format internal_format;
+
+   /**
+    * The ISL surface layout information for this resource.
+    *
+    * This is not filled out for PIPE_BUFFER resources, but is guaranteed
+    * to be zeroed.  Note that this also guarantees that res->surf.tiling
+    * will be ISL_TILING_LINEAR, so it's safe to check that.
+    */
+   struct isl_surf surf;
+
+   /** Backing storage for the resource */
+   struct crocus_bo *bo;
+
+   /** offset at which data starts in the BO */
+   uint64_t offset;
+
+   /**
+    * A bitfield of PIPE_BIND_* indicating how this resource was bound
+    * in the past.  Only meaningful for PIPE_BUFFER; used for flushing.
+    */
+   unsigned bind_history;
+
+   /**
+    * A bitfield of MESA_SHADER_* stages indicating where this resource
+    * was bound.
+    */
+   unsigned bind_stages;
+
+   /**
+    * For PIPE_BUFFER resources, a range which may contain valid data.
+    *
+    * This is a conservative estimate of what part of the buffer contains
+    * valid data that we have to preserve.  The rest of the buffer is
+    * considered invalid, and we can promote writes to that region to
+    * be unsynchronized writes, avoiding blit copies.
+    */
+   struct util_range valid_buffer_range;
+
+   /**
+    * Auxiliary buffer information (CCS, MCS, or HiZ).
+    */
+   struct {
+      /** The surface layout for the auxiliary buffer. */
+      struct isl_surf surf;
+
+      /** The buffer object containing the auxiliary data. */
+      struct crocus_bo *bo;
+
+      /** Offset into 'bo' where the auxiliary surface starts. */
+      uint32_t offset;
+
+      /**
+       * Fast clear color for this surface.  For depth surfaces, the clear
+       * value is stored as a float32 in the red component.
+       */
+      union isl_color_value clear_color;
+
+      /**
+       * \brief The type of auxiliary compression used by this resource.
+       *
+       * This describes the type of auxiliary compression that is intended to
+       * be used by this resource.  An aux usage of ISL_AUX_USAGE_NONE means
+       * that auxiliary compression is permanently disabled.  An aux usage
+       * other than ISL_AUX_USAGE_NONE does not imply that auxiliary
+       * compression will always be enabled for this surface.
+       */
+      enum isl_aux_usage usage;
+
+      /**
+       * \brief Maps miptree slices to their current aux state.
+       *
+       * This two-dimensional array is indexed as [level][layer] and stores an
+       * aux state for each slice.
+       */
+      enum isl_aux_state **state;
+
+      /**
+       * If (1 << level) is set, HiZ is enabled for that miplevel.
+       */
+      uint16_t has_hiz;
+   } aux;
+
+   /**
+    * \brief Shadow miptree for sampling when the main isn't supported by HW.
+    *
+    * To workaround various sampler bugs and limitations, we blit the main
+    * texture into a new texture that can be sampled.
+    *
+    * This miptree may be used for:
+    * - Stencil texturing (pre-BDW) as required by GL_ARB_stencil_texturing.
+    */
+   struct crocus_resource *shadow;
+   bool shadow_needs_update;
+
+   /**
+    * For external surfaces, this is format that was used to create or import
+    * the surface. For internal surfaces, this will always be
+    * PIPE_FORMAT_NONE.
+    */
+   enum pipe_format external_format;
+
+   /**
+    * For external surfaces, this is DRM format modifier that was used to
+    * create or import the surface.  For internal surfaces, this will always
+    * be DRM_FORMAT_MOD_INVALID.
+    */
+   const struct isl_drm_modifier_info *mod_info;
+
+   /**
+    * The screen the resource was originally created with, stored for refcounting.
+    */
+   struct pipe_screen *orig_screen;
+};
+
+/**
+ * A simple <resource, offset> tuple for storing a reference to a
+ * piece of state stored in a GPU buffer object.
+ */
+struct crocus_state_ref {
+   struct pipe_resource *res;
+   uint32_t offset;
+};
+
+/**
+ * Gallium CSO for sampler views (texture views).
+ *
+ * In addition to the normal pipe_resource, this adds an ISL view
+ * which may reinterpret the format or restrict levels/layers.
+ *
+ * These can also be linear texture buffers.
+ */
+struct crocus_sampler_view {
+   struct pipe_sampler_view base;
+   struct isl_view view;
+   struct isl_view gather_view;
+
+   enum pipe_swizzle swizzle[4];
+   union isl_color_value clear_color;
+
+   /* A short-cut (not a reference) to the actual resource being viewed.
+    * Multi-planar (or depth+stencil) images may have multiple resources
+    * chained together; this skips having to traverse base->texture->*.
+    */
+   struct crocus_resource *res;
+};
+
+/**
+ * Image view representation.
+ */
+struct crocus_image_view {
+   struct pipe_image_view base;
+   struct isl_view view;
+};
+
+/**
+ * Gallium CSO for surfaces (framebuffer attachments).
+ *
+ * A view of a surface that can be bound to a color render target or
+ * depth/stencil attachment.
+ */
+struct crocus_surface {
+   struct pipe_surface base;
+   struct isl_view view;
+   struct isl_view read_view;
+   struct isl_surf surf;
+   union isl_color_value clear_color;
+
+   struct pipe_resource *align_res;
+};
+
+/**
+ * Transfer object - information about a buffer mapping.
+ */
+struct crocus_transfer {
+   struct threaded_transfer base;
+   struct util_debug_callback *dbg;
+   void *buffer;
+   void *ptr;
+
+   /** A linear staging resource for GPU-based copy_region transfers. */
+   struct pipe_resource *staging;
+   struct blorp_context *blorp;
+   struct crocus_batch *batch;
+
+   bool dest_had_defined_contents;
+   bool has_swizzling;
+
+   void (*unmap)(struct crocus_transfer *);
+};
+
+/**
+ * Memory Object
+ */
+struct crocus_memory_object {
+   struct pipe_memory_object b;
+   struct crocus_bo *bo;
+   uint64_t format;
+   unsigned stride;
+};
+
+/**
+ * Unwrap a pipe_resource to get the underlying crocus_bo (for convenience).
+ */
+static inline struct crocus_bo *
+crocus_resource_bo(struct pipe_resource *p_res)
+{
+   struct crocus_resource *res = (void *) p_res;
+   return res->bo;
+}
+
+static inline uint32_t
+crocus_mocs(const struct crocus_bo *bo,
+            const struct isl_device *dev)
+{
+   return isl_mocs(dev, 0, bo && crocus_bo_is_external(bo));
+}
+
+struct crocus_format_info crocus_format_for_usage(const struct intel_device_info *,
+                                                  enum pipe_format pf,
+                                                  isl_surf_usage_flags_t usage);
+
+static inline struct pipe_resource *
+_crocus_resource_get_separate_stencil(struct pipe_resource *p_res)
+{
+   /* For packed depth-stencil, we treat depth as the primary resource
+    * and store S8 as the "second plane" resource.
+    */
+   if (p_res->next && p_res->next->format == PIPE_FORMAT_S8_UINT)
+      return p_res->next;
+
+   return NULL;
+
+}
+static inline void
+crocus_get_depth_stencil_resources(const struct intel_device_info *devinfo,
+                                   struct pipe_resource *res,
+                                   struct crocus_resource **out_z,
+                                   struct crocus_resource **out_s)
+{
+   /* gen4/5 only supports packed ds */
+   if (devinfo->ver < 6) {
+      *out_z = (void *)res;
+      *out_s = (void *)res;
+      return;
+   }
+
+   if (res && res->format != PIPE_FORMAT_S8_UINT) {
+      *out_z = (void *) res;
+      *out_s = (void *) _crocus_resource_get_separate_stencil(res);
+   } else {
+      *out_z = NULL;
+      *out_s = (void *) res;
+   }
+}
+
+
+bool crocus_resource_set_clear_color(struct crocus_context *ice,
+                                     struct crocus_resource *res,
+                                     union isl_color_value color);
+union isl_color_value
+crocus_resource_get_clear_color(const struct crocus_resource *res);
+
+void
+crocus_replace_buffer_storage(struct pipe_context *ctx,
+                              struct pipe_resource *p_dst,
+                              struct pipe_resource *p_src,
+                              unsigned num_rebinds,
+                              uint32_t rebind_mask,
+                              uint32_t delete_buffer_id);
+
+void crocus_init_screen_resource_functions(struct pipe_screen *pscreen);
+
+void crocus_dirty_for_history(struct crocus_context *ice,
+                              struct crocus_resource *res);
+uint32_t crocus_flush_bits_for_history(struct crocus_resource *res);
+
+void crocus_flush_and_dirty_for_history(struct crocus_context *ice,
+                                        struct crocus_batch *batch,
+                                        struct crocus_resource *res,
+                                        uint32_t extra_flags,
+                                        const char *reason);
+
+unsigned crocus_get_num_logical_layers(const struct crocus_resource *res,
+                                       unsigned level);
+
+void crocus_resource_disable_aux(struct crocus_resource *res);
+
+#define INTEL_REMAINING_LAYERS UINT32_MAX
+#define INTEL_REMAINING_LEVELS UINT32_MAX
+
+void
+crocus_hiz_exec(struct crocus_context *ice,
+                struct crocus_batch *batch,
+                struct crocus_resource *res,
+                unsigned int level, unsigned int start_layer,
+                unsigned int num_layers, enum isl_aux_op op);
+
+/**
+ * Prepare a miptree for access
+ *
+ * This function should be called prior to any access to miptree in order to
+ * perform any needed resolves.
+ *
+ * \param[in]  start_level    The first mip level to be accessed
+ *
+ * \param[in]  num_levels     The number of miplevels to be accessed or
+ *                            INTEL_REMAINING_LEVELS to indicate every level
+ *                            above start_level will be accessed
+ *
+ * \param[in]  start_layer    The first array slice or 3D layer to be accessed
+ *
+ * \param[in]  num_layers     The number of array slices or 3D layers be
+ *                            accessed or INTEL_REMAINING_LAYERS to indicate
+ *                            every layer above start_layer will be accessed
+ *
+ * \param[in]  aux_supported  Whether or not the access will support the
+ *                            miptree's auxiliary compression format;  this
+ *                            must be false for uncompressed miptrees
+ *
+ * \param[in]  fast_clear_supported Whether or not the access will support
+ *                                  fast clears in the miptree's auxiliary
+ *                                  compression format
+ */
+void
+crocus_resource_prepare_access(struct crocus_context *ice,
+                               struct crocus_resource *res,
+                               uint32_t start_level, uint32_t num_levels,
+                               uint32_t start_layer, uint32_t num_layers,
+                               enum isl_aux_usage aux_usage,
+                               bool fast_clear_supported);
+
+/**
+ * Complete a write operation
+ *
+ * This function should be called after any operation writes to a miptree.
+ * This will update the miptree's compression state so that future resolves
+ * happen correctly.  Technically, this function can be called before the
+ * write occurs but the caller must ensure that they don't interlace
+ * crocus_resource_prepare_access and crocus_resource_finish_write calls to
+ * overlapping layer/level ranges.
+ *
+ * \param[in]  level             The mip level that was written
+ *
+ * \param[in]  start_layer       The first array slice or 3D layer written
+ *
+ * \param[in]  num_layers        The number of array slices or 3D layers
+ *                               written or INTEL_REMAINING_LAYERS to indicate
+ *                               every layer above start_layer was written
+ *
+ * \param[in]  written_with_aux  Whether or not the write was done with
+ *                               auxiliary compression enabled
+ */
+void
+crocus_resource_finish_write(struct crocus_context *ice,
+                             struct crocus_resource *res, uint32_t level,
+                             uint32_t start_layer, uint32_t num_layers,
+                             enum isl_aux_usage aux_usage);
+
+/** Get the auxiliary compression state of a miptree slice */
+enum isl_aux_state
+crocus_resource_get_aux_state(const struct crocus_resource *res,
+                              uint32_t level, uint32_t layer);
+
+/**
+ * Set the auxiliary compression state of a miptree slice range
+ *
+ * This function directly sets the auxiliary compression state of a slice
+ * range of a miptree.  It only modifies data structures and does not do any
+ * resolves.  This should only be called by code which directly performs
+ * compression operations such as fast clears and resolves.  Most code should
+ * use crocus_resource_prepare_access or crocus_resource_finish_write.
+ */
+void
+crocus_resource_set_aux_state(struct crocus_context *ice,
+                              struct crocus_resource *res, uint32_t level,
+                              uint32_t start_layer, uint32_t num_layers,
+                              enum isl_aux_state aux_state);
+
+/**
+ * Prepare a miptree for raw access
+ *
+ * This helper prepares the miptree for access that knows nothing about any
+ * sort of compression whatsoever.  This is useful when mapping the surface or
+ * using it with the blitter.
+ */
+static inline void
+crocus_resource_access_raw(struct crocus_context *ice,
+                           struct crocus_resource *res,
+                           uint32_t level, uint32_t layer,
+                           uint32_t num_layers,
+                           bool write)
+{
+   crocus_resource_prepare_access(ice, res, level, 1, layer, num_layers,
+                                  ISL_AUX_USAGE_NONE, false);
+   if (write) {
+      crocus_resource_finish_write(ice, res, level, layer, num_layers,
+                                   ISL_AUX_USAGE_NONE);
+   }
+}
+
+void
+crocus_resource_get_image_offset(struct crocus_resource *res,
+                                 uint32_t level, uint32_t z,
+                                 uint32_t *x, uint32_t *y);
+static inline enum isl_aux_usage
+crocus_resource_texture_aux_usage(const struct crocus_resource *res)
+{
+   return res->aux.usage == ISL_AUX_USAGE_MCS ? ISL_AUX_USAGE_MCS : ISL_AUX_USAGE_NONE;
+}
+
+void crocus_resource_prepare_texture(struct crocus_context *ice,
+                                     struct crocus_resource *res,
+                                     enum isl_format view_format,
+                                     uint32_t start_level, uint32_t num_levels,
+                                     uint32_t start_layer, uint32_t num_layers);
+
+bool crocus_has_invalid_primary(const struct crocus_resource *res,
+                                unsigned start_level, unsigned num_levels,
+                                unsigned start_layer, unsigned num_layers);
+
+void crocus_resource_check_level_layer(const struct crocus_resource *res,
+                                       uint32_t level, uint32_t layer);
+
+bool crocus_resource_level_has_hiz(const struct crocus_resource *res,
+                                   uint32_t level);
+bool crocus_has_color_unresolved(const struct crocus_resource *res,
+                                 unsigned start_level, unsigned num_levels,
+                                 unsigned start_layer, unsigned num_layers);
+
+bool crocus_render_formats_color_compatible(enum isl_format a,
+                                          enum isl_format b,
+                                          union isl_color_value color);
+enum isl_aux_usage crocus_resource_render_aux_usage(struct crocus_context *ice,
+                                                    struct crocus_resource *res,
+                                                    uint32_t level,
+                                                    enum isl_format render_fmt,
+                                                    bool draw_aux_disabled);
+void crocus_resource_prepare_render(struct crocus_context *ice,
+                                    struct crocus_resource *res, uint32_t level,
+                                    uint32_t start_layer, uint32_t layer_count,
+                                    enum isl_aux_usage aux_usage);
+void crocus_resource_finish_render(struct crocus_context *ice,
+                                   struct crocus_resource *res, uint32_t level,
+                                   uint32_t start_layer, uint32_t layer_count,
+                                   enum isl_aux_usage aux_usage);
+#endif

src/gallium/drivers/crocus/crocus_screen.c

@@ -0,0 +1,723 @@
+/*
+ * Copyright © 2017 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+/**
+ * @file crocus_screen.c
+ *
+ * Screen related driver hooks and capability lists.
+ *
+ * A program may use multiple rendering contexts (crocus_context), but
+ * they all share a common screen (crocus_screen).  Global driver state
+ * can be stored in the screen; it may be accessed by multiple threads.
+ */
+
+#include <stdio.h>
+#include <errno.h>
+#include <sys/ioctl.h>
+#include "pipe/p_defines.h"
+#include "pipe/p_state.h"
+#include "pipe/p_context.h"
+#include "pipe/p_screen.h"
+#include "util/os_misc.h"
+#include "util/u_debug.h"
+#include "util/u_inlines.h"
+#include "util/format/u_format.h"
+#include "util/u_transfer_helper.h"
+#include "util/u_upload_mgr.h"
+#include "util/ralloc.h"
+#include "util/xmlconfig.h"
+#include "drm-uapi/i915_drm.h"
+#include "crocus_context.h"
+#include "crocus_defines.h"
+#include "crocus_fence.h"
+#include "crocus_perf.h"
+#include "crocus_pipe.h"
+#include "crocus_resource.h"
+#include "crocus_screen.h"
+#include "intel/compiler/elk/elk_compiler.h"
+#include "intel/common/intel_debug_identifier.h"
+#include "intel/common/intel_gem.h"
+#include "intel/common/intel_l3_config.h"
+#include "intel/common/intel_uuid.h"
+#include "crocus_monitor.h"
+
+#define genX_call(devinfo, func, ...)                   \
+   switch ((devinfo)->verx10) {                         \
+   case 80:                                             \
+      gfx8_##func(__VA_ARGS__);                         \
+      break;                                            \
+   case 75:                                             \
+      gfx75_##func(__VA_ARGS__);                        \
+      break;                                            \
+   case 70:                                             \
+      gfx7_##func(__VA_ARGS__);                         \
+      break;                                            \
+   case 60:                                             \
+      gfx6_##func(__VA_ARGS__);                         \
+      break;                                            \
+   case 50:                                             \
+      gfx5_##func(__VA_ARGS__);                         \
+      break;                                            \
+   case 45:                                             \
+      gfx45_##func(__VA_ARGS__);                        \
+      break;                                            \
+   case 40:                                             \
+      gfx4_##func(__VA_ARGS__);                         \
+      break;                                            \
+   default:                                             \
+      UNREACHABLE("Unknown hardware generation");       \
+   }
+
+static const char *
+crocus_get_vendor(struct pipe_screen *pscreen)
+{
+   return "Intel";
+}
+
+static const char *
+crocus_get_device_vendor(struct pipe_screen *pscreen)
+{
+   return "Intel";
+}
+
+static void
+crocus_get_device_uuid(struct pipe_screen *pscreen, char *uuid)
+{
+   struct crocus_screen *screen = (struct crocus_screen *)pscreen;
+
+   intel_uuid_compute_device_id((uint8_t *)uuid, &screen->devinfo, PIPE_UUID_SIZE);
+}
+
+static void
+crocus_get_driver_uuid(struct pipe_screen *pscreen, char *uuid)
+{
+   struct crocus_screen *screen = (struct crocus_screen *)pscreen;
+   const struct intel_device_info *devinfo = &screen->devinfo;
+
+   intel_uuid_compute_driver_id((uint8_t *)uuid, devinfo, PIPE_UUID_SIZE);
+}
+
+static const char *
+crocus_get_name(struct pipe_screen *pscreen)
+{
+   struct crocus_screen *screen = (struct crocus_screen *)pscreen;
+   const struct intel_device_info *devinfo = &screen->devinfo;
+   static char buf[128];
+
+   snprintf(buf, sizeof(buf), "Mesa %s", devinfo->name);
+   return buf;
+}
+
+static uint64_t
+get_aperture_size(int fd)
+{
+   struct drm_i915_gem_get_aperture aperture = {};
+   intel_ioctl(fd, DRM_IOCTL_I915_GEM_GET_APERTURE, &aperture);
+   return aperture.aper_size;
+}
+
+static void
+crocus_init_shader_caps(struct crocus_screen *screen)
+{
+   const struct intel_device_info *devinfo = &screen->devinfo;
+
+   for (unsigned i = 0; i <= MESA_SHADER_COMPUTE; i++) {
+      struct pipe_shader_caps *caps =
+         (struct pipe_shader_caps *)&screen->base.shader_caps[i];
+
+      if (devinfo->ver < 6 &&
+          i != MESA_SHADER_VERTEX &&
+          i != MESA_SHADER_FRAGMENT)
+         continue;
+
+      if (devinfo->ver == 6 &&
+          i != MESA_SHADER_VERTEX &&
+          i != MESA_SHADER_FRAGMENT &&
+          i != MESA_SHADER_GEOMETRY)
+         continue;
+
+      caps->max_instructions = i == MESA_SHADER_FRAGMENT ? 1024 : 16384;
+      caps->max_alu_instructions =
+      caps->max_tex_instructions =
+      caps->max_tex_indirections = i == MESA_SHADER_FRAGMENT ? 1024 : 0;
+
+      caps->max_control_flow_depth = UINT_MAX;
+
+      /* Gen7 vec4 geom backend */
+      caps->max_inputs = i == MESA_SHADER_VERTEX || i == MESA_SHADER_GEOMETRY ? 16 : 32;
+      caps->max_outputs = 32;
+      caps->max_const_buffer0_size = 16 * 1024 * sizeof(float);
+      caps->max_const_buffers = devinfo->ver >= 6 ? 16 : 1;
+      caps->max_temps = 256; /* GL_MAX_PROGRAM_TEMPORARIES_ARB */
+
+      /* Lie about these to avoid st/mesa's GLSL IR lowering of indirects,
+       * which we don't want.  Our compiler backend will check elk_compiler's
+       * options and call nir_lower_indirect_derefs_to_if_else_trees
+       * appropriately anyway.
+       */
+      caps->indirect_temp_addr = true;
+      caps->indirect_const_addr = true;
+
+      caps->integers = true;
+      caps->max_texture_samplers =
+      caps->max_sampler_views =
+         (devinfo->ver >= 7) ? CROCUS_MAX_TEXTURE_SAMPLERS : 16;
+
+      if (devinfo->ver >= 7 &&
+          (i == MESA_SHADER_FRAGMENT || i == MESA_SHADER_COMPUTE))
+         caps->max_shader_images = CROCUS_MAX_TEXTURE_SAMPLERS;
+
+      caps->max_shader_buffers =
+         devinfo->ver >= 7 ? (CROCUS_MAX_ABOS + CROCUS_MAX_SSBOS) : 0;
+
+      caps->supported_irs = 1 << PIPE_SHADER_IR_NIR;
+   }
+}
+
+static void
+crocus_init_compute_caps(struct crocus_screen *screen)
+{
+   struct pipe_compute_caps *caps =
+      (struct pipe_compute_caps *)&screen->base.compute_caps;
+   const struct intel_device_info *devinfo = &screen->devinfo;
+
+   if (devinfo->ver < 7)
+      return;
+
+   const uint32_t max_invocations = 32 * devinfo->max_cs_workgroup_threads;
+
+   caps->address_bits = 32;
+
+   caps->grid_dimension = 3;
+
+   caps->max_grid_size[0] =
+   caps->max_grid_size[1] =
+   caps->max_grid_size[2] = 65535;
+
+   /* MaxComputeWorkGroupSize[0..2] */
+   caps->max_block_size[0] =
+   caps->max_block_size[1] =
+   caps->max_block_size[2] = max_invocations;
+
+   /* MaxComputeWorkGroupInvocations */
+   caps->max_threads_per_block = max_invocations;
+
+   /* MaxComputeSharedMemorySize */
+   caps->max_local_size = 64 * 1024;
+
+   caps->subgroup_sizes = ELK_SUBGROUP_SIZE;
+
+   caps->max_variable_threads_per_block = max_invocations;
+}
+
+static void
+crocus_init_screen_caps(struct crocus_screen *screen)
+{
+   const struct intel_device_info *devinfo = &screen->devinfo;
+   struct pipe_caps *caps = (struct pipe_caps *)&screen->base.caps;
+
+   u_init_pipe_screen_caps(&screen->base, 1);
+
+   caps->prefer_real_buffer_in_constbuf0 = true;
+   caps->npot_textures = true;
+   caps->anisotropic_filter = true;
+   caps->occlusion_query = true;
+   caps->texture_swizzle = true;
+   caps->texture_mirror_clamp_to_edge = true;
+   caps->blend_equation_separate = true;
+   caps->fragment_shader_texture_lod = true;
+   caps->fragment_shader_derivatives = true;
+   caps->primitive_restart = true;
+   caps->primitive_restart_fixed_index = true;
+   caps->indep_blend_enable = true;
+   caps->fs_coord_origin_upper_left = true;
+   caps->fs_coord_pixel_center_integer = true;
+   caps->depth_clip_disable = true;
+   caps->vs_instanceid = true;
+   caps->vertex_element_instance_divisor = true;
+   caps->seamless_cube_map = true;
+   caps->seamless_cube_map_per_texture = true;
+   caps->conditional_render = true;
+   caps->texture_barrier = true;
+   caps->vertex_color_unclamped = true;
+   caps->start_instance = true;
+   caps->force_persample_interp = true;
+   caps->mixed_framebuffer_sizes = true;
+   caps->vs_layer_viewport = true;
+   caps->tes_layer_viewport = true;
+   caps->uma = true;
+   caps->clip_halfz = true;
+   caps->tgsi_texcoord = true;
+   caps->device_reset_status_query = true;
+   caps->copy_between_compressed_and_plain_formats = true;
+   caps->signed_vertex_buffer_offset = true;
+   caps->texture_float_linear = true;
+   caps->texture_half_float_linear = true;
+   caps->polygon_offset_clamp = true;
+   caps->multisample_z_resolve = true;
+   caps->shader_group_vote = true;
+   caps->vs_window_space_position = true;
+   caps->texture_gather_sm5 = true;
+   caps->shader_array_components = true;
+   caps->glsl_tess_levels_as_inputs = true;
+   caps->fs_position_is_sysval = true;
+   caps->fs_face_is_integer_sysval = true;
+   caps->invalidate_buffer = true;
+   caps->surface_no_compress = true;
+   caps->surface_reinterpret_blocks = true;
+   caps->fence_signal = true;
+   caps->demote_to_helper_invocation = true;
+   caps->gl_clamp = true;
+   caps->legacy_math_rules = true;
+   caps->native_fence_fd = true;
+
+   caps->int64 =
+   caps->shader_ballot =
+   caps->packed_uniforms = devinfo->ver == 8;
+
+   caps->quads_follow_provoking_vertex_convention = devinfo->ver <= 5;
+
+   caps->texture_query_lod =
+   caps->query_time_elapsed = devinfo->ver >= 5;
+
+   caps->draw_indirect =
+   caps->multi_draw_indirect =
+   caps->multi_draw_indirect_params =
+   caps->framebuffer_no_attachment =
+   caps->fs_fine_derivative =
+   caps->stream_output_interleave_buffers =
+   caps->shader_clock =
+   caps->texture_query_samples =
+   caps->compute =
+   caps->sampler_view_target =
+   caps->shader_samples_identical =
+   caps->shader_pack_half_float =
+   caps->gl_spirv =
+   caps->gl_spirv_variable_pointers =
+   caps->compute_shader_derivatives =
+   caps->doubles =
+   caps->memobj =
+   caps->image_store_formatted =
+   caps->alpha_to_coverage_dither_control = devinfo->ver >= 7;
+
+   caps->query_buffer_object =
+   caps->robust_buffer_access_behavior = devinfo->ver >= 7;
+
+   caps->cull_distance =
+   caps->query_pipeline_statistics_single =
+   caps->stream_output_pause_resume =
+   caps->sample_shading =
+   caps->cube_map_array =
+   caps->query_so_overflow =
+   caps->texture_multisample =
+   caps->conditional_render_inverted =
+   caps->query_timestamp =
+   caps->texture_buffer_objects =
+   caps->indep_blend_func =
+   caps->texture_shadow_lod =
+   caps->load_constbuf =
+   caps->draw_parameters =
+   caps->clear_scissored = devinfo->ver >= 6;
+
+   caps->fbfetch = devinfo->verx10 >= 45 ? ELK_MAX_DRAW_BUFFERS : 0;
+   /* in theory CL (965gm) can do this */
+   caps->max_dual_source_render_targets = devinfo->verx10 >= 45 ? 1 : 0;
+   caps->max_render_targets = ELK_MAX_DRAW_BUFFERS;
+   caps->max_texture_2d_size = devinfo->ver >= 7 ? 16384 : 8192;
+   caps->max_texture_cube_levels = devinfo->ver >= 7 ?
+      CROCUS_MAX_MIPLEVELS /* 16384x16384 */ : CROCUS_MAX_MIPLEVELS - 1; /* 8192x8192 */
+   caps->max_texture_3d_levels = 12; /* 2048x2048 */
+   caps->max_stream_output_buffers = (devinfo->ver >= 6) ? 4 : 0;
+   caps->max_texture_array_layers = devinfo->ver >= 7 ? 2048 : 512;
+   caps->max_stream_output_separate_components =
+      ELK_MAX_SOL_BINDINGS / CROCUS_MAX_SOL_BUFFERS;
+   caps->max_stream_output_interleaved_components = ELK_MAX_SOL_BINDINGS;
+   caps->glsl_feature_level_compatibility =
+   caps->glsl_feature_level = devinfo->ver >= 7 ? 460 :
+      (devinfo->ver >= 6 ? 330 : 140);
+   caps->clip_planes = devinfo->verx10 < 45 ? 6 : 1; // defaults to MAX (8)
+   /* 3DSTATE_CONSTANT_XS requires the start of UBOs to be 32B aligned */
+   caps->constant_buffer_offset_alignment = 32;
+   caps->min_map_buffer_alignment = CROCUS_MAP_BUFFER_ALIGNMENT;
+   caps->shader_buffer_offset_alignment = devinfo->ver >= 7 ? 4 : 0;
+   caps->max_shader_buffer_size = devinfo->ver >= 7 ? (1 << 27) : 0;
+   caps->texture_buffer_offset_alignment = 16; // XXX: u_screen says 256 is the minimum value...
+   caps->texture_transfer_modes = PIPE_TEXTURE_TRANSFER_BLIT;
+   caps->max_texel_buffer_elements = CROCUS_MAX_TEXTURE_BUFFER_SIZE;
+   caps->max_viewports = devinfo->ver >= 6 ? 16 : 1;
+   caps->max_geometry_output_vertices = devinfo->ver >= 6 ? 256 : 0;
+   caps->max_geometry_total_output_components = devinfo->ver >= 6 ? 1024 : 0;
+   caps->max_gs_invocations = devinfo->ver >= 7 ? 32 : 1;
+   caps->max_texture_gather_components = devinfo->ver >= 7 ? 4 :
+      (devinfo->ver == 6 ? 1 : 0);
+   caps->min_texture_gather_offset = devinfo->ver >= 7 ? -32 :
+      (devinfo->ver == 6 ? -8 : 0);
+   caps->max_texture_gather_offset = devinfo->ver >= 7 ? 31 :
+      (devinfo->ver == 6 ? 7 : 0);
+   caps->max_vertex_streams = devinfo->ver >= 7 ? 4 : 1;
+   caps->vendor_id = 0x8086;
+   caps->device_id = screen->pci_id;
+
+   /* Once a batch uses more than 75% of the maximum mappable size, we
+    * assume that there's some fragmentation, and we start doing extra
+    * flushing, etc.  That's the big cliff apps will care about.
+    */
+   const unsigned gpu_mappable_megabytes =
+      (screen->aperture_threshold) / (1024 * 1024);
+
+   uint64_t system_memory_bytes =
+      os_get_gpu_heap_size(screen->driconf.heap_memory_percent, NULL);
+   if (!system_memory_bytes) {
+      caps->video_memory = -1;
+   } else {
+      const unsigned system_memory_megabytes =
+         (unsigned) (system_memory_bytes / (1024 * 1024));
+
+      caps->video_memory = MIN2(system_memory_megabytes, gpu_mappable_megabytes);
+   }
+
+   caps->max_shader_patch_varyings =
+   caps->max_varyings = (screen->devinfo.ver >= 6) ? 32 : 16;
+   /* AMD_pinned_memory assumes the flexibility of using client memory
+    * for any buffer (incl. vertex buffers) which rules out the prospect
+    * of using snooped buffers, as using snooped buffers without
+    * cogniscience is likely to be detrimental to performance and require
+    * extensive checking in the driver for correctness, e.g. to prevent
+    * illegal snoop <-> snoop transfers.
+    */
+   caps->resource_from_user_memory = devinfo->has_llc && devinfo->has_userptr_uapi;
+   caps->throttle = !screen->driconf.disable_throttling;
+
+   caps->context_priority_mask =
+      PIPE_CONTEXT_PRIORITY_LOW |
+      PIPE_CONTEXT_PRIORITY_MEDIUM |
+      PIPE_CONTEXT_PRIORITY_HIGH;
+
+   /* Let mesa/st lower for us */
+   caps->flatshade = false;
+
+   caps->frontend_noop = true;
+   // XXX: don't hardcode 00:00:02.0 PCI here
+   caps->pci_group = 0;
+   caps->pci_bus = 0;
+   caps->pci_device = 2;
+   caps->pci_function = 0;
+
+   caps->hardware_gl_select = false;
+
+   caps->timer_resolution = DIV_ROUND_UP(1000000000ull, devinfo->timestamp_frequency);
+
+   caps->min_line_width =
+   caps->min_line_width_aa =
+   caps->min_point_size =
+   caps->min_point_size_aa = 1;
+
+   caps->point_size_granularity =
+   caps->line_width_granularity = 0.1;
+
+   caps->max_line_width =
+   caps->max_line_width_aa = devinfo->ver >= 6 ? 7.375f : 7.0f;
+
+   caps->max_point_size =
+   caps->max_point_size_aa = 255.0f;
+
+   caps->max_texture_anisotropy = 16.0f;
+   caps->max_texture_lod_bias = 15.0f;
+}
+
+static uint64_t
+crocus_get_timestamp(struct pipe_screen *pscreen)
+{
+   struct crocus_screen *screen = (struct crocus_screen *) pscreen;
+   uint64_t result;
+
+   if (!intel_gem_read_render_timestamp(crocus_bufmgr_get_fd(screen->bufmgr),
+                                        screen->devinfo.kmd_type, &result))
+      return 0;
+
+   result = intel_device_info_timebase_scale(&screen->devinfo, result);
+   result &= (1ull << TIMESTAMP_BITS) - 1;
+
+   return result;
+}
+
+void
+crocus_screen_destroy(struct crocus_screen *screen)
+{
+   intel_perf_free(screen->perf_cfg);
+   u_transfer_helper_destroy(screen->base.transfer_helper);
+   crocus_bufmgr_unref(screen->bufmgr);
+   disk_cache_destroy(screen->disk_cache);
+   intel_virtio_unref_fd(screen->winsys_fd);
+   close(screen->winsys_fd);
+   ralloc_free(screen);
+}
+
+static void
+crocus_screen_unref(struct pipe_screen *pscreen)
+{
+   crocus_pscreen_unref(pscreen);
+}
+
+static void
+crocus_query_memory_info(struct pipe_screen *pscreen,
+                         struct pipe_memory_info *info)
+{
+}
+
+static int
+crocus_get_compute_param(struct pipe_screen *pscreen,
+                         enum pipe_shader_ir ir_type,
+                         enum pipe_compute_cap param,
+                         void *ret)
+{
+   struct crocus_screen *screen = (struct crocus_screen *) pscreen;
+   const struct intel_device_info *devinfo = &screen->devinfo;
+
+   switch (param) {
+   case PIPE_COMPUTE_CAP_MAX_COMPUTE_UNITS:
+      if (ret) {
+         uint32_t num_eu = devinfo->num_eu_total > 0 ? devinfo->num_eu_total : ((devinfo->gt == 2) ? 16 : 6);
+         ((uint32_t *)ret)[0] = num_eu;
+      }
+      return sizeof(uint32_t);
+
+   case PIPE_COMPUTE_CAP_MAX_THREADS_PER_BLOCK:
+      if (ret)
+         ((uint64_t *)ret)[0] = 1024;
+      return sizeof(uint64_t);
+
+   case PIPE_COMPUTE_CAP_MAX_GRID_SIZE:
+      if (ret) {
+         ((uint64_t *)ret)[0] = 65535;
+         ((uint64_t *)ret)[1] = 65535;
+         ((uint64_t *)ret)[2] = 65535;
+      }
+      return 3 * sizeof(uint64_t);
+
+   case PIPE_COMPUTE_CAP_MAX_BLOCK_SIZE:
+      if (ret) {
+         ((uint64_t *)ret)[0] = 1024;
+         ((uint64_t *)ret)[1] = 1024;
+         ((uint64_t *)ret)[2] = 1024;
+      }
+      return 3 * sizeof(uint64_t);
+
+   case PIPE_COMPUTE_CAP_PREFERRED_BLOCK_SIZE:
+      if (ret)
+         ((uint64_t *)ret)[0] = 256;
+      return sizeof(uint64_t);
+
+   case PIPE_COMPUTE_CAP_IMAGES_SUPPORTED:
+      if (ret)
+         ((uint32_t *)ret)[0] = 1;
+      return sizeof(uint32_t);
+
+   case PIPE_COMPUTE_CAP_SUBGROUP_SIZE:
+      if (ret)
+         ((uint32_t *)ret)[0] = devinfo->subgroup_size ? devinfo->subgroup_size : 16;
+      return sizeof(uint32_t);
+
+   default:
+      return 0;
+   }
+}
+
+static struct disk_cache *
+crocus_get_disk_shader_cache(struct pipe_screen *pscreen)
+{
+   struct crocus_screen *screen = (struct crocus_screen *) pscreen;
+   return screen->disk_cache;
+}
+
+static const struct intel_l3_config *
+crocus_get_default_l3_config(const struct intel_device_info *devinfo,
+                             bool compute)
+{
+   bool wants_dc_cache = true;
+   bool has_slm = compute;
+   const struct intel_l3_weights w =
+      intel_get_default_l3_weights(devinfo, wants_dc_cache, has_slm);
+   return intel_get_l3_config(devinfo, w);
+}
+
+static void
+crocus_shader_debug_log(void *data, unsigned *id, const char *fmt, ...)
+{
+   struct util_debug_callback *dbg = data;
+   va_list args;
+
+   if (!dbg->debug_message)
+      return;
+
+   va_start(args, fmt);
+   dbg->debug_message(dbg->data, id, UTIL_DEBUG_TYPE_SHADER_INFO, fmt, args);
+   va_end(args);
+}
+
+static void
+crocus_shader_perf_log(void *data, unsigned *id, const char *fmt, ...)
+{
+   struct util_debug_callback *dbg = data;
+   va_list args;
+   va_start(args, fmt);
+
+   if (INTEL_DEBUG(DEBUG_PERF)) {
+      va_list args_copy;
+      va_copy(args_copy, args);
+      vfprintf(stderr, fmt, args_copy);
+      va_end(args_copy);
+   }
+
+   if (dbg->debug_message) {
+      dbg->debug_message(dbg->data, id, UTIL_DEBUG_TYPE_PERF_INFO, fmt, args);
+   }
+
+   va_end(args);
+}
+
+static int
+crocus_screen_get_fd(struct pipe_screen *pscreen)
+{
+   struct crocus_screen *screen = (struct crocus_screen *)pscreen;
+
+   return screen->winsys_fd;
+}
+
+struct pipe_screen *
+crocus_screen_create(int fd, const struct pipe_screen_config *config)
+{
+   struct crocus_screen *screen = rzalloc(NULL, struct crocus_screen);
+   if (!screen)
+      return NULL;
+
+   if (intel_virtio_init_fd(fd) < 0)
+      return NULL;
+
+   if (!intel_get_device_info_from_fd(fd, &screen->devinfo, 4, 8))
+      return NULL;
+   screen->pci_id = screen->devinfo.pci_device_id;
+
+   if (screen->devinfo.ver > 8)
+      return NULL;
+
+   if (screen->devinfo.ver == 8) {
+      /* bind to cherryview or bdw if forced */
+      if (screen->devinfo.platform != INTEL_PLATFORM_CHV &&
+          !os_get_option("CROCUS_GEN8"))
+         return NULL;
+   }
+
+   p_atomic_set(&screen->refcount, 1);
+
+   screen->aperture_bytes = get_aperture_size(fd);
+   screen->aperture_threshold = screen->aperture_bytes * 3 / 4;
+
+   driParseConfigFiles(config->options, config->options_info,
+                       &(driConfigFileParseParams) { .driverName = "crocus" });
+
+   bool bo_reuse = false;
+   int bo_reuse_mode = driQueryOptioni(config->options, "bo_reuse");
+   switch (bo_reuse_mode) {
+   case DRI_CONF_BO_REUSE_DISABLED:
+      break;
+   case DRI_CONF_BO_REUSE_ALL:
+      bo_reuse = true;
+      break;
+   }
+
+   screen->bufmgr = crocus_bufmgr_get_for_fd(&screen->devinfo, fd, bo_reuse);
+   if (!screen->bufmgr)
+      return NULL;
+   screen->fd = crocus_bufmgr_get_fd(screen->bufmgr);
+   screen->winsys_fd = fd;
+
+   process_intel_debug_variable();
+
+   screen->driconf.dual_color_blend_by_location =
+      driQueryOptionb(config->options, "dual_color_blend_by_location");
+   screen->driconf.disable_throttling =
+      driQueryOptionb(config->options, "disable_throttling");
+   screen->driconf.always_flush_cache =
+      driQueryOptionb(config->options, "always_flush_cache");
+   screen->driconf.limit_trig_input_range =
+      driQueryOptionb(config->options, "limit_trig_input_range");
+   screen->driconf.lower_depth_range_rate =
+      driQueryOptionf(config->options, "lower_depth_range_rate");
+
+   screen->driconf.heap_memory_percent =
+      driQueryOptionf(config->options, "heap_memory_percent");
+   if (screen->driconf.heap_memory_percent == OS_GPU_HEAP_SIZE_HEURISTIC)
+      screen->driconf.heap_memory_percent = 1.0f;
+
+   screen->precompile = debug_get_bool_option("shader_precompile", true);
+
+   isl_device_init(&screen->isl_dev, &screen->devinfo);
+
+   screen->compiler = elk_compiler_create(screen, &screen->devinfo);
+   screen->compiler->shader_debug_log = crocus_shader_debug_log;
+   screen->compiler->shader_perf_log = crocus_shader_perf_log;
+   screen->compiler->supports_shader_constants = false;
+   screen->compiler->constant_buffer_0_is_relative = true;
+
+   if (screen->devinfo.ver >= 7) {
+      screen->l3_config_3d = crocus_get_default_l3_config(&screen->devinfo, false);
+      screen->l3_config_cs = crocus_get_default_l3_config(&screen->devinfo, true);
+   }
+
+   crocus_disk_cache_init(screen);
+
+   slab_create_parent(&screen->transfer_pool,
+                      sizeof(struct crocus_transfer), 64);
+
+   struct pipe_screen *pscreen = &screen->base;
+
+   crocus_init_screen_fence_functions(pscreen);
+   crocus_init_screen_resource_functions(pscreen);
+
+   for (unsigned i = 0; i <= MESA_SHADER_COMPUTE; i++)
+      pscreen->nir_options[i] = screen->compiler->nir_options[i];
+
+   pscreen->destroy = crocus_screen_unref;
+   pscreen->get_name = crocus_get_name;
+   pscreen->get_vendor = crocus_get_vendor;
+   pscreen->get_device_vendor = crocus_get_device_vendor;
+   pscreen->get_screen_fd = crocus_screen_get_fd;
+   pscreen->get_device_uuid = crocus_get_device_uuid;
+   pscreen->get_driver_uuid = crocus_get_driver_uuid;
+   pscreen->get_disk_shader_cache = crocus_get_disk_shader_cache;
+   pscreen->is_format_supported = crocus_is_format_supported;
+   pscreen->context_create = crocus_create_context;
+   pscreen->get_timestamp = crocus_get_timestamp;
+   pscreen->query_memory_info = crocus_query_memory_info;
+   pscreen->get_driver_query_group_info = crocus_get_monitor_group_info;
+   pscreen->get_driver_query_info = crocus_get_monitor_info;
+   pscreen->get_compute_param = crocus_get_compute_param;
+
+   crocus_init_shader_caps(screen);
+   crocus_init_compute_caps(screen);
+   crocus_init_screen_caps(screen);
+
+   genX_call(&screen->devinfo, crocus_init_screen_state, screen);
+   genX_call(&screen->devinfo, crocus_init_screen_query, screen);
+   return pscreen;
+}

src/gallium/drivers/crocus/crocus_screen.h

@@ -0,0 +1,255 @@
+/*
+ * Copyright © 2017 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * on the rights to use, copy, modify, merge, publish, distribute, sub
+ * license, and/or sell copies of the Software, and to permit persons to whom
+ * the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef CROCUS_SCREEN_H
+#define CROCUS_SCREEN_H
+
+#include "pipe/p_screen.h"
+#include "pipe/p_state.h"
+#include "frontend/drm_driver.h"
+#include "util/disk_cache.h"
+#include "util/slab.h"
+#include "util/u_screen.h"
+#include "intel/dev/intel_device_info.h"
+#include "intel/dev/virtio/intel_virtio.h"
+#include "intel/isl/isl.h"
+#include "crocus_bufmgr.h"
+#include "compiler/shader_enums.h"
+
+struct crocus_resource;
+struct crocus_context;
+struct crocus_sampler_state;
+struct intel_vue_map;
+struct elk_tcs_prog_key;
+struct elk_tes_prog_key;
+struct elk_cs_prog_key;
+struct elk_fs_prog_key;
+struct elk_vs_prog_key;
+struct elk_gs_prog_key;
+struct shader_info;
+
+#define READ_ONCE(x) (*(volatile __typeof__(x) *)&(x))
+#define WRITE_ONCE(x, v) *(volatile __typeof__(x) *)&(x) = (v)
+
+#define CROCUS_MAX_TEXTURE_SAMPLERS 32
+#define CROCUS_MAX_SOL_BUFFERS 4
+#define CROCUS_MAP_BUFFER_ALIGNMENT 64
+
+
+/**
+ * Virtual table for generation-specific (genxml) function calls.
+ */
+struct crocus_vtable {
+   void (*destroy_state)(struct crocus_context *ice);
+   void (*init_render_context)(struct crocus_batch *batch);
+   void (*init_compute_context)(struct crocus_batch *batch);
+   void (*upload_render_state)(struct crocus_context *ice,
+                               struct crocus_batch *batch,
+                               const struct pipe_draw_info *draw,
+                               unsigned drawid_offset,
+                               const struct pipe_draw_indirect_info *indirect,
+                               const struct pipe_draw_start_count_bias *sc);
+   void (*update_surface_base_address)(struct crocus_batch *batch);
+
+   void (*upload_compute_state)(struct crocus_context *ice,
+                                struct crocus_batch *batch,
+                                const struct pipe_grid_info *grid);
+   void (*rebind_buffer)(struct crocus_context *ice,
+                         struct crocus_resource *res);
+   void (*resolve_conditional_render)(struct crocus_context *ice);
+   void (*emit_compute_predicate)(struct crocus_batch *batch);
+   void (*load_register_reg32)(struct crocus_batch *batch, uint32_t dst,
+                               uint32_t src);
+   void (*load_register_reg64)(struct crocus_batch *batch, uint32_t dst,
+                               uint32_t src);
+   void (*load_register_imm32)(struct crocus_batch *batch, uint32_t reg,
+                               uint32_t val);
+   void (*load_register_imm64)(struct crocus_batch *batch, uint32_t reg,
+                               uint64_t val);
+   void (*load_register_mem32)(struct crocus_batch *batch, uint32_t reg,
+                               struct crocus_bo *bo, uint32_t offset);
+   void (*load_register_mem64)(struct crocus_batch *batch, uint32_t reg,
+                               struct crocus_bo *bo, uint32_t offset);
+   void (*store_register_mem32)(struct crocus_batch *batch, uint32_t reg,
+                                struct crocus_bo *bo, uint32_t offset,
+                                bool predicated);
+   void (*store_register_mem64)(struct crocus_batch *batch, uint32_t reg,
+                                struct crocus_bo *bo, uint32_t offset,
+                                bool predicated);
+   void (*store_data_imm32)(struct crocus_batch *batch,
+                            struct crocus_bo *bo, uint32_t offset,
+                            uint32_t value);
+   void (*store_data_imm64)(struct crocus_batch *batch,
+                            struct crocus_bo *bo, uint32_t offset,
+                            uint64_t value);
+   void (*copy_mem_mem)(struct crocus_batch *batch,
+                        struct crocus_bo *dst_bo, uint32_t dst_offset,
+                        struct crocus_bo *src_bo, uint32_t src_offset,
+                        unsigned bytes);
+   void (*emit_raw_pipe_control)(struct crocus_batch *batch,
+                                 const char *reason, uint32_t flags,
+                                 struct crocus_bo *bo, uint32_t offset,
+                                 uint64_t imm);
+
+   void (*emit_mi_report_perf_count)(struct crocus_batch *batch,
+                                     struct crocus_bo *bo,
+                                     uint32_t offset_in_bytes,
+                                     uint32_t report_id);
+
+   uint32_t *(*create_so_decl_list)(const struct pipe_stream_output_info *sol,
+                                    const struct intel_vue_map *vue_map);
+   void (*populate_vs_key)(const struct crocus_context *ice,
+                           const struct shader_info *info,
+                           mesa_shader_stage last_stage,
+                           struct elk_vs_prog_key *key);
+   void (*populate_tcs_key)(const struct crocus_context *ice,
+                            struct elk_tcs_prog_key *key);
+   void (*populate_tes_key)(const struct crocus_context *ice,
+                            const struct shader_info *info,
+                            mesa_shader_stage last_stage,
+                            struct elk_tes_prog_key *key);
+   void (*populate_gs_key)(const struct crocus_context *ice,
+                           const struct shader_info *info,
+                           mesa_shader_stage last_stage,
+                           struct elk_gs_prog_key *key);
+   void (*populate_fs_key)(const struct crocus_context *ice,
+                           const struct shader_info *info,
+                           struct elk_fs_prog_key *key);
+   void (*populate_cs_key)(const struct crocus_context *ice,
+                           struct elk_cs_prog_key *key);
+   void (*fill_clamp_mask)(const struct crocus_sampler_state *state,
+                           int s,
+                           uint32_t *clamp_mask);
+   void (*lost_genx_state)(struct crocus_context *ice, struct crocus_batch *batch);
+
+   void (*finish_batch)(struct crocus_batch *batch); /* haswell only */
+
+   void (*upload_urb_fence)(struct crocus_batch *batch); /* gen4/5 only */
+
+   bool (*blit_blt)(struct crocus_batch *batch,
+                    const struct pipe_blit_info *info);
+   bool (*copy_region_blt)(struct crocus_batch *batch,
+                           struct crocus_resource *dst,
+                           unsigned dst_level,
+                           unsigned dstx, unsigned dsty, unsigned dstz,
+                           struct crocus_resource *src,
+                           unsigned src_level,
+                           const struct pipe_box *src_box);
+   bool (*calculate_urb_fence)(struct crocus_batch *batch, unsigned csize,
+                               unsigned vsize, unsigned sfsize);
+   void (*batch_reset_dirty)(struct crocus_batch *batch);
+   unsigned (*translate_prim_type)(enum mesa_prim prim, uint8_t verts_per_patch);
+
+   void (*update_so_strides)(struct crocus_context *ice,
+                             uint16_t *strides);
+
+   uint32_t (*get_so_offset)(struct pipe_stream_output_target *tgt);
+};
+
+struct crocus_screen {
+   struct pipe_screen base;
+
+   uint32_t refcount;
+
+   /** Global slab allocator for crocus_transfer_map objects */
+   struct slab_parent_pool transfer_pool;
+
+   /** drm device file descriptor, shared with bufmgr, do not close. */
+   int fd;
+
+   /**
+    * drm device file descriptor to used for window system integration, owned
+    * by iris_screen, can be a different DRM instance than fd.
+    */
+   int winsys_fd;
+
+   /** PCI ID for our GPU device */
+   int pci_id;
+
+   struct crocus_vtable vtbl;
+
+   /** Global program_string_id counter (see get_program_string_id()) */
+   unsigned program_id;
+
+   /** Precompile shaders at link time?  (Can be disabled for debugging.) */
+   bool precompile;
+
+   /** driconf options and application workarounds */
+   struct {
+      /** Dual color blend by location instead of index (for broken apps) */
+      bool dual_color_blend_by_location;
+      bool disable_throttling;
+      bool always_flush_cache;
+      bool limit_trig_input_range;
+      float lower_depth_range_rate;
+      float heap_memory_percent;
+   } driconf;
+
+   uint64_t aperture_bytes;
+   uint64_t aperture_threshold;
+
+   struct intel_device_info devinfo;
+   struct isl_device isl_dev;
+   struct crocus_bufmgr *bufmgr;
+   struct elk_compiler *compiler;
+   struct intel_perf_config *perf_cfg;
+
+   const struct intel_l3_config *l3_config_3d;
+   const struct intel_l3_config *l3_config_cs;
+
+   struct disk_cache *disk_cache;
+};
+
+struct pipe_screen *
+crocus_screen_create(int fd, const struct pipe_screen_config *config);
+
+void crocus_screen_destroy(struct crocus_screen *screen);
+
+UNUSED static inline struct pipe_screen *
+crocus_pscreen_ref(struct pipe_screen *pscreen)
+{
+   struct crocus_screen *screen = (struct crocus_screen *) pscreen;
+
+   p_atomic_inc(&screen->refcount);
+   return pscreen;
+}
+
+UNUSED static inline void
+crocus_pscreen_unref(struct pipe_screen *pscreen)
+{
+   struct crocus_screen *screen = (struct crocus_screen *) pscreen;
+
+   if (p_atomic_dec_zero(&screen->refcount))
+      crocus_screen_destroy(screen);
+}
+
+bool
+crocus_is_format_supported(struct pipe_screen *pscreen,
+                           enum pipe_format format,
+                           enum pipe_texture_target target,
+                           unsigned sample_count,
+                           unsigned storage_sample_count,
+                           unsigned usage);
+
+void crocus_disk_cache_init(struct crocus_screen *screen);
+
+#endif

src/gallium/drivers/crocus/crocus_todo.txt

@@ -0,0 +1,16 @@
+Quick TODO list from what I can see:
+
+General:
+Re-emit SURFACE_STATE_BASE_ADDRESS at the top of every batch
+
+Gen4:
+rgb32 issue
+
+Gen5:
+rgb32 issue
+
+Gen6:
+vec4 push constants
+
+Gen7:
+

src/gallium/drivers/crocus/driinfo_crocus.h

@@ -0,0 +1,20 @@
+// crocus specific driconf options
+
+DRI_CONF_SECTION_DEBUG
+   DRI_CONF_DUAL_COLOR_BLEND_BY_LOCATION(false)
+   DRI_CONF_DISABLE_THROTTLING(false)
+   DRI_CONF_ALWAYS_FLUSH_CACHE(false)
+   DRI_CONF_LIMIT_TRIG_INPUT_RANGE(false)
+DRI_CONF_SECTION_END
+
+DRI_CONF_SECTION_PERFORMANCE
+   DRI_CONF_OPT_E(bo_reuse, 1, 0, 1, "Buffer object reuse",)
+DRI_CONF_SECTION_END
+
+DRI_CONF_SECTION_QUALITY
+   DRI_CONF_PP_LOWER_DEPTH_RANGE_RATE()
+DRI_CONF_SECTION_END
+
+DRI_CONF_SECTION_MISCELLANEOUS
+   DRI_CONF_HEAP_MEMORY_PERCENT(1.0f)
+DRI_CONF_SECTION_END

src/gallium/drivers/crocus/gen4_blorp_exec.h

@@ -0,0 +1,190 @@
+/*
+ * Copyright © 2016 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+static inline struct blorp_address
+dynamic_state_address(struct blorp_batch *blorp_batch, uint32_t offset)
+{
+   struct crocus_batch *batch = blorp_batch->driver_batch;
+
+   return (struct blorp_address) {
+      .buffer = batch->state.bo,
+      .offset = offset,
+   };
+
+}
+
+static inline struct blorp_address
+instruction_state_address(struct blorp_batch *blorp_batch, uint32_t offset)
+{
+   struct crocus_batch *batch = blorp_batch->driver_batch;
+
+   return (struct blorp_address) {
+      .buffer = batch->ice->shaders.cache_bo,
+      .offset = offset,
+   };
+}
+
+static struct blorp_address
+blorp_emit_vs_state(struct blorp_batch *blorp_batch)
+{
+   struct crocus_batch *batch = blorp_batch->driver_batch;
+
+   uint32_t offset;
+   blorp_emit_dynamic(blorp_batch, GENX(VS_STATE), vs, 64, &offset) {
+      vs.Enable = false;
+      vs.URBEntryAllocationSize = batch->ice->urb.vsize - 1;
+#if GFX_VER == 5
+      vs.NumberofURBEntries = batch->ice->urb.nr_vs_entries >> 2;
+#else
+      vs.NumberofURBEntries = batch->ice->urb.nr_vs_entries;
+#endif
+   }
+
+   return dynamic_state_address(blorp_batch, offset);
+}
+
+static struct blorp_address
+blorp_emit_sf_state(struct blorp_batch *blorp_batch,
+                    const struct blorp_params *params)
+{
+   struct crocus_batch *batch = blorp_batch->driver_batch;
+   const struct elk_sf_prog_data *prog_data = params->sf_prog_data;
+
+   uint32_t offset;
+   blorp_emit_dynamic(blorp_batch, GENX(SF_STATE), sf, 64, &offset) {
+#if GFX_VER == 4
+      sf.KernelStartPointer =
+         instruction_state_address(blorp_batch, params->sf_prog_kernel);
+#else
+      sf.KernelStartPointer = params->sf_prog_kernel;
+#endif
+      sf.GRFRegisterCount = DIV_ROUND_UP(prog_data->total_grf, 16) - 1;
+      sf.VertexURBEntryReadLength = prog_data->urb_read_length;
+      sf.VertexURBEntryReadOffset = ELK_SF_URB_ENTRY_READ_OFFSET;
+      sf.DispatchGRFStartRegisterForURBData = 3;
+      sf.URBEntryAllocationSize = batch->ice->urb.sfsize - 1;
+      sf.NumberofURBEntries = batch->ice->urb.nr_sf_entries;
+
+#if GFX_VER == 5
+      sf.MaximumNumberofThreads = MIN2(48, batch->ice->urb.nr_sf_entries) - 1;
+#else
+      sf.MaximumNumberofThreads = MIN2(24, batch->ice->urb.nr_sf_entries) - 1;
+#endif
+      sf.ViewportTransformEnable = false;
+
+      sf.CullMode = CULLMODE_NONE;
+   }
+
+   return dynamic_state_address(blorp_batch, offset);
+}
+
+static struct blorp_address
+blorp_emit_wm_state(struct blorp_batch *blorp_batch,
+                    const struct blorp_params *params)
+{
+   const struct elk_fs_prog_data *prog_data = params->fs_prog_data;
+
+   uint32_t offset;
+   blorp_emit_dynamic(blorp_batch, GENX(WM_STATE), wm, 64, &offset) {
+      if (params->src.enabled) {
+         /* Iron Lake can't do sampler prefetch */
+         wm.SamplerCount = (GFX_VER != 5);
+         wm.BindingTableEntryCount = 2;
+         uint32_t sampler = blorp_emit_sampler_state(blorp_batch);
+         wm.SamplerStatePointer = dynamic_state_address(blorp_batch, sampler);
+      }
+
+      if (prog_data) {
+         wm.DispatchGRFStartRegisterForConstantSetupData0 =
+            prog_data->base.dispatch_grf_start_reg;
+         wm.SetupURBEntryReadLength = prog_data->num_varying_inputs * 2;
+         wm.SetupURBEntryReadOffset = 0;
+
+         wm.DepthCoefficientURBReadOffset = 1;
+         wm.PixelShaderKillsPixel = prog_data->uses_kill;
+         wm.ThreadDispatchEnable = true;
+         wm.EarlyDepthTestEnable = true;
+
+         wm._8PixelDispatchEnable = prog_data->dispatch_8;
+         wm._16PixelDispatchEnable = prog_data->dispatch_16;
+         wm._32PixelDispatchEnable = prog_data->dispatch_32;
+
+#if GFX_VER == 4
+         wm.KernelStartPointer0 =
+            instruction_state_address(blorp_batch, params->wm_prog_kernel);
+         wm.GRFRegisterCount0 = elk_fs_prog_data_reg_blocks(prog_data, wm, 0);
+#else
+         wm.KernelStartPointer0 = params->wm_prog_kernel +
+                                  elk_fs_prog_data_prog_offset(prog_data, wm, 0);
+         wm.KernelStartPointer1 = params->wm_prog_kernel +
+                                  elk_fs_prog_data_prog_offset(prog_data, wm, 1);
+         wm.KernelStartPointer2 = params->wm_prog_kernel +
+                                  elk_fs_prog_data_prog_offset(prog_data, wm, 2);
+         wm.GRFRegisterCount0 = elk_fs_prog_data_reg_blocks(prog_data, wm, 0);
+         wm.GRFRegisterCount1 = elk_fs_prog_data_reg_blocks(prog_data, wm, 1);
+         wm.GRFRegisterCount2 = elk_fs_prog_data_reg_blocks(prog_data, wm, 2);
+#endif
+      }
+
+      wm.MaximumNumberofThreads =
+         blorp_batch->blorp->compiler->elk->devinfo->max_wm_threads - 1;
+   }
+
+   return dynamic_state_address(blorp_batch, offset);
+}
+
+static struct blorp_address
+blorp_emit_color_calc_state(struct blorp_batch *blorp_batch)
+{
+   uint32_t cc_viewport = blorp_emit_cc_viewport(blorp_batch);
+
+   uint32_t offset;
+   blorp_emit_dynamic(blorp_batch, GENX(COLOR_CALC_STATE), cc, 64, &offset) {
+      cc.CCViewportStatePointer = dynamic_state_address(blorp_batch, cc_viewport);
+   }
+
+   return dynamic_state_address(blorp_batch, offset);
+}
+
+static void
+blorp_emit_pipeline(struct blorp_batch *blorp_batch,
+                    const struct blorp_params *params)
+{
+   struct crocus_batch *batch = blorp_batch->driver_batch;
+
+   emit_urb_config(blorp_batch, params);
+
+   blorp_emit(blorp_batch, GENX(3DSTATE_PIPELINED_POINTERS), pp) {
+      pp.PointertoVSState = blorp_emit_vs_state(blorp_batch);
+      pp.GSEnable = false;
+      pp.ClipEnable = false;
+      pp.PointertoSFState = blorp_emit_sf_state(blorp_batch, params);
+      pp.PointertoWMState = blorp_emit_wm_state(blorp_batch, params);
+      pp.PointertoColorCalcState = blorp_emit_color_calc_state(blorp_batch);
+   }
+
+   batch->screen->vtbl.upload_urb_fence(batch);
+
+   blorp_emit(blorp_batch, GENX(CS_URB_STATE), curb);
+   blorp_emit(blorp_batch, GENX(CONSTANT_BUFFER), curb);
+}

src/gallium/drivers/crocus/meson.build

@@ -0,0 +1,81 @@
+# Copyright © 2017-2019 Intel Corporation
+# SPDX-License-Identifier: MIT
+
+files_libcrocus = files(
+  'gen4_blorp_exec.h',
+  'driinfo_crocus.h',
+  'crocus_batch.c',
+  'crocus_batch.h',
+  'crocus_blit.c',
+  'crocus_bufmgr.c',
+  'crocus_bufmgr.h',
+  'crocus_clear.c',
+  'crocus_context.c',
+  'crocus_context.h',
+  'crocus_draw.c',
+  'crocus_fence.c',
+  'crocus_fence.h',
+  'crocus_fine_fence.c',
+  'crocus_fine_fence.h',
+  'crocus_formats.c',
+  'crocus_genx_macros.h',
+  'crocus_genx_protos.h',
+  'crocus_monitor.c',
+  'crocus_perf.c',
+  'crocus_perf.h',
+  'crocus_performance_query.c',
+  'crocus_pipe.h',
+  'crocus_pipe_control.c',
+  'crocus_program.c',
+  'crocus_program_cache.c',
+  'crocus_resolve.c',
+  'crocus_resource.c',
+  'crocus_resource.h',
+  'crocus_screen.c',
+  'crocus_screen.h',
+  'crocus_disk_cache.c',
+)
+
+crocus_per_hw_ver_libs = []
+foreach v : ['40', '45', '50', '60', '70', '75', '80']
+  crocus_per_hw_ver_libs += static_library(
+    'crocus_per_hw_ver@0@'.format(v),
+    ['crocus_blorp.c', 'crocus_query.c', 'crocus_state.c', 'crocus_blt.c', gen_xml_pack],
+    include_directories : [inc_include, inc_src, inc_mesa, inc_gallium, inc_gallium_aux, inc_intel],
+    c_args : [
+      no_override_init_args, sse2_args,
+      '-DGFX_VERx10=@0@'.format(v),
+    ],
+    gnu_symbol_visibility : 'hidden',
+    dependencies : [dep_libdrm, dep_valgrind, idep_mesautil, idep_genxml, idep_nir_headers, idep_intel_dev, idep_intel_blorp_elk],
+  )
+endforeach
+
+libcrocus = static_library(
+  'crocus',
+  [files_libcrocus, gen_xml_pack],
+  include_directories : [
+    inc_src, inc_include, inc_gallium, inc_gallium_aux, inc_intel,
+    inc_gallium_drivers,
+    # these should not be necessary, but main/macros.h...
+    inc_mesa
+  ],
+  c_args : [sse2_args],
+  cpp_args : [sse2_args],
+  gnu_symbol_visibility : 'hidden',
+  dependencies : [
+    dep_libdrm, dep_valgrind, idep_genxml, idep_mesautil,
+    idep_libintel_common, idep_nir_headers,
+    idep_intel_dev, idep_intel_blorp_elk, idep_intel_decoder_elk,
+    idep_intel_compiler_elk,
+  ],
+  link_with : [
+    crocus_per_hw_ver_libs, libisl,
+    libintel_perf
+  ],
+)
+
+driver_crocus = declare_dependency(
+  compile_args : '-DGALLIUM_CROCUS',
+  link_with : [libcrocus, libcrocuswinsys],
+)

src/gallium/drivers/d3d12/.clang-format

@@ -0,0 +1,63 @@
+
+BasedOnStyle: InheritParentConfig
+DisableFormat: false
+
+ColumnLimit: 120
+SortIncludes: Never
+
+AlignConsecutiveAssignments: None
+AlignConsecutiveDeclarations: None
+AlignTrailingComments: true
+
+PenaltyExcessCharacter: 75
+
+AlwaysBreakTemplateDeclarations: Yes
+BreakConstructorInitializers: BeforeColon
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth: 3
+IndentCaseLabels: true
+
+BreakBeforeTernaryOperators: false 
+
+BreakBeforeBraces: Custom
+BraceWrapping:
+  AfterCaseLabel: true
+  AfterClass: true
+  AfterEnum: true
+  AfterNamespace: true
+  AfterObjCDeclaration: true
+  AfterStruct: true
+  AfterUnion: true
+  AfterExternBlock: false
+  BeforeCatch: false
+  BeforeElse: false
+  IndentBraces: false
+  SplitEmptyFunction: false
+  SplitEmptyRecord: false
+  SplitEmptyNamespace: false
+
+FixNamespaceComments: true
+KeepEmptyLinesAtTheStartOfBlocks: true
+
+AlwaysBreakAfterReturnType: TopLevel
+
+BinPackArguments: false
+BinPackParameters: false
+Cpp11BracedListStyle: false
+
+SpaceAfterCStyleCast: true
+SpaceBeforeCpp11BracedList: true
+SpaceBeforeCtorInitializerColon: true
+SpacesInContainerLiterals: true
+SpaceBeforeParens: ControlStatements
+SpacesBeforeTrailingComments: 3
+SpacesInAngles: Never
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+SpacesInCStyleCastParentheses: false
+SpaceAfterLogicalNot: false
+SpaceBeforeAssignmentOperators: true
+SpaceInEmptyParentheses: false
+SpaceInEmptyBlock: true
+
+MaxEmptyLinesToKeep: 3

src/gallium/drivers/d3d12/ci/d3d12-flakes.txt

@@ -0,0 +1,10 @@
+# Tessellation isoline tests are flaky on WARP, probably a WARP bug
+isoline
+isoline-no-tcs
+
+# If the test machine are busy, these can spuriously fail
+arb_timer_query
+ext_timer_query
+
+# assertion failure in https://gitlab.freedesktop.org/mesa/mesa/-/jobs/48717501
+spec@arb_compute_shader@execution@simple-barrier-atomics

src/gallium/drivers/d3d12/ci/d3d12-quick_gl.toml

@@ -0,0 +1,5 @@
+[[piglit]]
+piglit_folder = "C:/piglit"
+profile = "quick_gl"
+process_isolation = true
+timeout = 240

src/gallium/drivers/d3d12/ci/d3d12-quick_gl.txt

@@ -0,0 +1,410 @@
+# from https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/25931
+spec@arb_internalformat_query2@image_format_compatibility_type pname checks,Fail
+spec@arb_internalformat_query2@image_format_compatibility_type pname checks@GL_IMAGE_FORMAT_COMPATIBILITY_TYPE,Fail
+spec@arb_internalformat_query2@max dimensions related pname checks,Fail
+spec@arb_internalformat_query2@max dimensions related pname checks@GL_MAX_COMBINED_DIMENSIONS,Fail
+spec@arb_internalformat_query2@max dimensions related pname checks@GL_MAX_DEPTH,Fail
+spec@arb_internalformat_query2@max dimensions related pname checks@GL_MAX_HEIGHT,Fail
+spec@arb_internalformat_query2@max dimensions related pname checks@GL_MAX_LAYERS,Fail
+spec@arb_internalformat_query2@max dimensions related pname checks@GL_MAX_WIDTH,Fail
+
+spec@!opengl [email protected],Fail
+spec@!opengl [email protected]@glFlush,Fail
+spec@!opengl [email protected],Fail
+spec@!opengl [email protected],Fail
+spec@!opengl 1.0@rasterpos,Fail
+spec@!opengl 1.0@rasterpos@glsl_vs_gs_linked,Fail
+spec@!opengl 1.0@rasterpos@glsl_vs_tes_linked,Fail
+spec@!opengl 1.1@line-smooth-coverage,Fail
+spec@!opengl 1.1@line-smooth-stipple,Fail
+spec@!opengl 1.1@linestipple,Fail
+spec@!opengl 1.1@linestipple@Factor 2x,Fail
+spec@!opengl 1.1@linestipple@Factor 3x,Fail
+spec@!opengl 1.1@linestipple@Line loop,Fail
+spec@!opengl 1.1@linestipple@Line strip,Fail
+spec@!opengl 1.1@linestipple@Restarting lines within a single Begin-End block,Fail
+spec@!opengl 1.1@polygon-mode-offset,Fail
+spec@!opengl 1.1@polygon-mode-offset@config 4: Expected white pixel on bottom edge,Fail
+spec@!opengl 1.1@polygon-mode-offset@config 4: Expected white pixel on left edge,Fail
+spec@!opengl 1.1@polygon-mode-offset@config 4: Expected white pixel on right edge,Fail
+spec@!opengl 1.1@polygon-mode-offset@config 4: Expected white pixel on top edge,Fail
+spec@!opengl 1.1@polygon-offset,Fail
+spec@!opengl 1.1@windowoverlap,Missing
+spec@!opengl 3.0@sampler-cube-shadow,Fail
+spec@!opengl 3.1@default-vao,Fail
+spec@!opengl 3.1@genned-names,Fail
+spec@!opengl [email protected],Fail
+spec@!opengl [email protected],Fail
+spec@!opengl [email protected] cull-back pv-last,Fail
+spec@!opengl [email protected] cull-front pv-last,Fail
+spec@!opengl [email protected] line cull-back pv-last,Fail
+spec@!opengl [email protected] line cull-front pv-last,Fail
+spec@!opengl [email protected] pv-last,Fail
+spec@arb_draw_indirect@arb_draw_indirect-api-errors,Crash
+spec@arb_framebuffer_object@fbo-blit-scaled-linear,Fail
+spec@arb_get_program_binary@restore-sso-program,Fail
+spec@arb_program_interface_query@arb_program_interface_query-getprogramresourceindex,Fail
+spec@arb_program_interface_query@arb_program_interface_query-getprogramresourceindex@'vs_input2' on GL_PROGRAM_INPUT,Fail
+spec@arb_program_interface_query@arb_program_interface_query-getprogramresourceindex@'vs_input2[1][0]' on GL_PROGRAM_INPUT,Fail
+spec@arb_sample_shading@arb_sample_shading-builtin-gl-sample-mask-mrt-alpha-to-coverage,Fail
+spec@arb_seamless_cube_map@arb_seamless_cubemap,Fail
+spec@arb_separate_shader_objects@rendezvous by location (5 stages),Crash
+spec@arb_shader_atomic_counters@semantics,Fail
+spec@arb_shader_atomic_counters@semantics@Tessellation control shader atomic built-in semantics,Fail
+spec@arb_texture_cube_map_array@arb_texture_cube_map_array-sampler-cube-array-shadow,Fail
+spec@arb_texture_rg@texwrap formats-int,Fail
+spec@arb_texture_rg@texwrap formats-int offset,Fail
+spec@arb_texture_rg@texwrap formats-int offset@GL_R16I,Fail
+spec@arb_texture_rg@texwrap formats-int offset@GL_R16I- NPOT,Fail
+spec@arb_texture_rg@texwrap formats-int offset@GL_R16I- swizzled,Fail
+spec@arb_texture_rg@texwrap formats-int offset@GL_R16UI,Fail
+spec@arb_texture_rg@texwrap formats-int offset@GL_R16UI- NPOT,Fail
+spec@arb_texture_rg@texwrap formats-int offset@GL_R16UI- swizzled,Fail
+spec@arb_texture_rg@texwrap formats-int offset@GL_R32I,Fail
+spec@arb_texture_rg@texwrap formats-int offset@GL_R32I- NPOT,Fail
+spec@arb_texture_rg@texwrap formats-int offset@GL_R32I- swizzled,Fail
+spec@arb_texture_rg@texwrap formats-int offset@GL_R32UI,Fail
+spec@arb_texture_rg@texwrap formats-int offset@GL_R32UI- NPOT,Fail
+spec@arb_texture_rg@texwrap formats-int offset@GL_R32UI- swizzled,Fail
+spec@arb_texture_rg@texwrap formats-int offset@GL_R8I,Fail
+spec@arb_texture_rg@texwrap formats-int offset@GL_R8I- NPOT,Fail
+spec@arb_texture_rg@texwrap formats-int offset@GL_R8I- swizzled,Fail
+spec@arb_texture_rg@texwrap formats-int offset@GL_R8UI,Fail
+spec@arb_texture_rg@texwrap formats-int offset@GL_R8UI- NPOT,Fail
+spec@arb_texture_rg@texwrap formats-int offset@GL_R8UI- swizzled,Fail
+spec@arb_texture_rg@texwrap formats-int offset@GL_RG16I,Fail
+spec@arb_texture_rg@texwrap formats-int offset@GL_RG16I- NPOT,Fail
+spec@arb_texture_rg@texwrap formats-int offset@GL_RG16I- swizzled,Fail
+spec@arb_texture_rg@texwrap formats-int offset@GL_RG16UI,Fail
+spec@arb_texture_rg@texwrap formats-int offset@GL_RG16UI- NPOT,Fail
+spec@arb_texture_rg@texwrap formats-int offset@GL_RG16UI- swizzled,Fail
+spec@arb_texture_rg@texwrap formats-int offset@GL_RG32I,Fail
+spec@arb_texture_rg@texwrap formats-int offset@GL_RG32I- NPOT,Fail
+spec@arb_texture_rg@texwrap formats-int offset@GL_RG32I- swizzled,Fail
+spec@arb_texture_rg@texwrap formats-int offset@GL_RG32UI,Fail
+spec@arb_texture_rg@texwrap formats-int offset@GL_RG32UI- NPOT,Fail
+spec@arb_texture_rg@texwrap formats-int offset@GL_RG32UI- swizzled,Fail
+spec@arb_texture_rg@texwrap formats-int offset@GL_RG8I,Fail
+spec@arb_texture_rg@texwrap formats-int offset@GL_RG8I- NPOT,Fail
+spec@arb_texture_rg@texwrap formats-int offset@GL_RG8I- swizzled,Fail
+spec@arb_texture_rg@texwrap formats-int offset@GL_RG8UI,Fail
+spec@arb_texture_rg@texwrap formats-int offset@GL_RG8UI- NPOT,Fail
+spec@arb_texture_rg@texwrap formats-int offset@GL_RG8UI- swizzled,Fail
+spec@arb_texture_rg@texwrap formats-int@GL_R16I,Fail
+spec@arb_texture_rg@texwrap formats-int@GL_R16I- NPOT,Fail
+spec@arb_texture_rg@texwrap formats-int@GL_R16I- swizzled,Fail
+spec@arb_texture_rg@texwrap formats-int@GL_R16UI,Fail
+spec@arb_texture_rg@texwrap formats-int@GL_R16UI- NPOT,Fail
+spec@arb_texture_rg@texwrap formats-int@GL_R16UI- swizzled,Fail
+spec@arb_texture_rg@texwrap formats-int@GL_R32I,Fail
+spec@arb_texture_rg@texwrap formats-int@GL_R32I- NPOT,Fail
+spec@arb_texture_rg@texwrap formats-int@GL_R32I- swizzled,Fail
+spec@arb_texture_rg@texwrap formats-int@GL_R32UI,Fail
+spec@arb_texture_rg@texwrap formats-int@GL_R32UI- NPOT,Fail
+spec@arb_texture_rg@texwrap formats-int@GL_R32UI- swizzled,Fail
+spec@arb_texture_rg@texwrap formats-int@GL_R8I,Fail
+spec@arb_texture_rg@texwrap formats-int@GL_R8I- NPOT,Fail
+spec@arb_texture_rg@texwrap formats-int@GL_R8I- swizzled,Fail
+spec@arb_texture_rg@texwrap formats-int@GL_R8UI,Fail
+spec@arb_texture_rg@texwrap formats-int@GL_R8UI- NPOT,Fail
+spec@arb_texture_rg@texwrap formats-int@GL_R8UI- swizzled,Fail
+spec@arb_texture_rg@texwrap formats-int@GL_RG16I,Fail
+spec@arb_texture_rg@texwrap formats-int@GL_RG16I- NPOT,Fail
+spec@arb_texture_rg@texwrap formats-int@GL_RG16I- swizzled,Fail
+spec@arb_texture_rg@texwrap formats-int@GL_RG16UI,Fail
+spec@arb_texture_rg@texwrap formats-int@GL_RG16UI- NPOT,Fail
+spec@arb_texture_rg@texwrap formats-int@GL_RG16UI- swizzled,Fail
+spec@arb_texture_rg@texwrap formats-int@GL_RG32I,Fail
+spec@arb_texture_rg@texwrap formats-int@GL_RG32I- NPOT,Fail
+spec@arb_texture_rg@texwrap formats-int@GL_RG32I- swizzled,Fail
+spec@arb_texture_rg@texwrap formats-int@GL_RG32UI,Fail
+spec@arb_texture_rg@texwrap formats-int@GL_RG32UI- NPOT,Fail
+spec@arb_texture_rg@texwrap formats-int@GL_RG32UI- swizzled,Fail
+spec@arb_texture_rg@texwrap formats-int@GL_RG8I,Fail
+spec@arb_texture_rg@texwrap formats-int@GL_RG8I- NPOT,Fail
+spec@arb_texture_rg@texwrap formats-int@GL_RG8I- swizzled,Fail
+spec@arb_texture_rg@texwrap formats-int@GL_RG8UI,Fail
+spec@arb_texture_rg@texwrap formats-int@GL_RG8UI- NPOT,Fail
+spec@arb_texture_rg@texwrap formats-int@GL_RG8UI- swizzled,Fail
+spec@arb_texture_rgb10_a2ui@texwrap formats,Fail
+spec@arb_texture_rgb10_a2ui@texwrap formats offset,Fail
+spec@arb_texture_rgb10_a2ui@texwrap formats offset@GL_RGB10_A2UI,Fail
+spec@arb_texture_rgb10_a2ui@texwrap formats offset@GL_RGB10_A2UI- NPOT,Fail
+spec@arb_texture_rgb10_a2ui@texwrap formats offset@GL_RGB10_A2UI- swizzled,Fail
+spec@arb_texture_rgb10_a2ui@texwrap formats@GL_RGB10_A2UI,Fail
+spec@arb_texture_rgb10_a2ui@texwrap formats@GL_RGB10_A2UI- NPOT,Fail
+spec@arb_texture_rgb10_a2ui@texwrap formats@GL_RGB10_A2UI- swizzled,Fail
+spec@arb_texture_stencil8@texwrap formats,Fail
+spec@arb_texture_stencil8@texwrap formats offset,Fail
+spec@arb_texture_stencil8@texwrap formats offset@GL_STENCIL_INDEX8,Fail
+spec@arb_texture_stencil8@texwrap formats offset@GL_STENCIL_INDEX8- NPOT,Fail
+spec@arb_texture_stencil8@texwrap formats offset@GL_STENCIL_INDEX8- swizzled,Fail
+spec@arb_texture_stencil8@texwrap formats@GL_STENCIL_INDEX8,Fail
+spec@arb_texture_stencil8@texwrap formats@GL_STENCIL_INDEX8- NPOT,Fail
+spec@arb_texture_stencil8@texwrap formats@GL_STENCIL_INDEX8- swizzled,Fail
+spec@arb_transform_feedback_instanced@draw-auto instanced,Fail
+spec@arb_vertex_program@arb_vertex_program-matrix-property-bindings,Fail
+spec@ext_framebuffer_multisample@accuracy 16 srgb depthstencil,Fail
+spec@ext_framebuffer_multisample@accuracy 16 srgb depthstencil linear,Fail
+spec@ext_framebuffer_multisample@accuracy 16 srgb small depthstencil,Fail
+spec@ext_framebuffer_multisample@accuracy 16 srgb small depthstencil linear,Fail
+spec@ext_framebuffer_multisample@accuracy all_samples srgb depthstencil,Fail
+spec@ext_framebuffer_multisample@accuracy all_samples srgb depthstencil linear,Fail
+spec@ext_framebuffer_multisample@accuracy all_samples srgb small depthstencil,Fail
+spec@ext_framebuffer_multisample@accuracy all_samples srgb small depthstencil linear,Fail
+spec@ext_framebuffer_multisample@alpha-to-one-dual-src-blend 16,Fail
+spec@ext_framebuffer_multisample@alpha-to-one-dual-src-blend 2,Fail
+spec@ext_framebuffer_multisample@alpha-to-one-dual-src-blend 4,Fail
+spec@ext_framebuffer_multisample@alpha-to-one-dual-src-blend 6,Fail
+spec@ext_framebuffer_multisample@alpha-to-one-dual-src-blend 8,Fail
+spec@ext_framebuffer_multisample@draw-buffers-alpha-to-one 16,Fail
+spec@ext_framebuffer_multisample@draw-buffers-alpha-to-one 2,Fail
+spec@ext_framebuffer_multisample@draw-buffers-alpha-to-one 4,Fail
+spec@ext_framebuffer_multisample@draw-buffers-alpha-to-one 6,Fail
+spec@ext_framebuffer_multisample@draw-buffers-alpha-to-one 8,Fail
+spec@ext_framebuffer_multisample@sample-alpha-to-one 16,Fail
+spec@ext_framebuffer_multisample@sample-alpha-to-one 2,Fail
+spec@ext_framebuffer_multisample@sample-alpha-to-one 4,Fail
+spec@ext_framebuffer_multisample@sample-alpha-to-one 6,Fail
+spec@ext_framebuffer_multisample@sample-alpha-to-one 8,Fail
+spec@ext_texture_integer@texwrap formats,Fail
+spec@ext_texture_integer@texwrap formats offset,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_ALPHA16I_EXT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_ALPHA16I_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_ALPHA16I_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_ALPHA16UI_EXT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_ALPHA16UI_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_ALPHA16UI_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_ALPHA32I_EXT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_ALPHA32I_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_ALPHA32I_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_ALPHA32UI_EXT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_ALPHA32UI_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_ALPHA32UI_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_ALPHA8I_EXT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_ALPHA8I_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_ALPHA8I_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_ALPHA8UI_EXT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_ALPHA8UI_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_ALPHA8UI_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_INTENSITY16I_EXT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_INTENSITY16I_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_INTENSITY16I_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_INTENSITY16UI_EXT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_INTENSITY16UI_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_INTENSITY16UI_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_INTENSITY32I_EXT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_INTENSITY32I_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_INTENSITY32I_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_INTENSITY32UI_EXT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_INTENSITY32UI_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_INTENSITY32UI_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_INTENSITY8I_EXT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_INTENSITY8I_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_INTENSITY8I_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_INTENSITY8UI_EXT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_INTENSITY8UI_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_INTENSITY8UI_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_LUMINANCE16I_EXT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_LUMINANCE16I_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_LUMINANCE16I_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_LUMINANCE16UI_EXT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_LUMINANCE16UI_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_LUMINANCE16UI_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_LUMINANCE32I_EXT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_LUMINANCE32I_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_LUMINANCE32I_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_LUMINANCE32UI_EXT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_LUMINANCE32UI_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_LUMINANCE32UI_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_LUMINANCE8I_EXT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_LUMINANCE8I_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_LUMINANCE8I_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_LUMINANCE8UI_EXT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_LUMINANCE8UI_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_LUMINANCE8UI_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_LUMINANCE_ALPHA16I_EXT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_LUMINANCE_ALPHA16I_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_LUMINANCE_ALPHA16I_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_LUMINANCE_ALPHA16UI_EXT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_LUMINANCE_ALPHA16UI_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_LUMINANCE_ALPHA16UI_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_LUMINANCE_ALPHA32I_EXT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_LUMINANCE_ALPHA32I_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_LUMINANCE_ALPHA32I_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_LUMINANCE_ALPHA32UI_EXT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_LUMINANCE_ALPHA32UI_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_LUMINANCE_ALPHA32UI_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_LUMINANCE_ALPHA8I_EXT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_LUMINANCE_ALPHA8I_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_LUMINANCE_ALPHA8I_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_LUMINANCE_ALPHA8UI_EXT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_LUMINANCE_ALPHA8UI_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_LUMINANCE_ALPHA8UI_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_RGB16I,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_RGB16I- NPOT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_RGB16I- swizzled,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_RGB16UI,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_RGB16UI- NPOT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_RGB16UI- swizzled,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_RGB32I,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_RGB32I- NPOT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_RGB32I- swizzled,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_RGB32UI,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_RGB32UI- NPOT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_RGB32UI- swizzled,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_RGB8I,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_RGB8I- NPOT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_RGB8I- swizzled,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_RGB8UI,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_RGB8UI- NPOT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_RGB8UI- swizzled,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_RGBA16I,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_RGBA16I- NPOT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_RGBA16I- swizzled,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_RGBA16UI,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_RGBA16UI- NPOT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_RGBA16UI- swizzled,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_RGBA32I,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_RGBA32I- NPOT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_RGBA32I- swizzled,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_RGBA32UI,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_RGBA32UI- NPOT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_RGBA32UI- swizzled,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_RGBA8I,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_RGBA8I- NPOT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_RGBA8I- swizzled,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_RGBA8UI,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_RGBA8UI- NPOT,Fail
+spec@ext_texture_integer@texwrap formats offset@GL_RGBA8UI- swizzled,Fail
+spec@ext_texture_integer@texwrap formats@GL_ALPHA16I_EXT,Fail
+spec@ext_texture_integer@texwrap formats@GL_ALPHA16I_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats@GL_ALPHA16I_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats@GL_ALPHA16UI_EXT,Fail
+spec@ext_texture_integer@texwrap formats@GL_ALPHA16UI_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats@GL_ALPHA16UI_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats@GL_ALPHA32I_EXT,Fail
+spec@ext_texture_integer@texwrap formats@GL_ALPHA32I_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats@GL_ALPHA32I_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats@GL_ALPHA32UI_EXT,Fail
+spec@ext_texture_integer@texwrap formats@GL_ALPHA32UI_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats@GL_ALPHA32UI_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats@GL_ALPHA8I_EXT,Fail
+spec@ext_texture_integer@texwrap formats@GL_ALPHA8I_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats@GL_ALPHA8I_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats@GL_ALPHA8UI_EXT,Fail
+spec@ext_texture_integer@texwrap formats@GL_ALPHA8UI_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats@GL_ALPHA8UI_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats@GL_INTENSITY16I_EXT,Fail
+spec@ext_texture_integer@texwrap formats@GL_INTENSITY16I_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats@GL_INTENSITY16I_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats@GL_INTENSITY16UI_EXT,Fail
+spec@ext_texture_integer@texwrap formats@GL_INTENSITY16UI_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats@GL_INTENSITY16UI_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats@GL_INTENSITY32I_EXT,Fail
+spec@ext_texture_integer@texwrap formats@GL_INTENSITY32I_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats@GL_INTENSITY32I_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats@GL_INTENSITY32UI_EXT,Fail
+spec@ext_texture_integer@texwrap formats@GL_INTENSITY32UI_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats@GL_INTENSITY32UI_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats@GL_INTENSITY8I_EXT,Fail
+spec@ext_texture_integer@texwrap formats@GL_INTENSITY8I_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats@GL_INTENSITY8I_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats@GL_INTENSITY8UI_EXT,Fail
+spec@ext_texture_integer@texwrap formats@GL_INTENSITY8UI_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats@GL_INTENSITY8UI_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats@GL_LUMINANCE16I_EXT,Fail
+spec@ext_texture_integer@texwrap formats@GL_LUMINANCE16I_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats@GL_LUMINANCE16I_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats@GL_LUMINANCE16UI_EXT,Fail
+spec@ext_texture_integer@texwrap formats@GL_LUMINANCE16UI_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats@GL_LUMINANCE16UI_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats@GL_LUMINANCE32I_EXT,Fail
+spec@ext_texture_integer@texwrap formats@GL_LUMINANCE32I_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats@GL_LUMINANCE32I_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats@GL_LUMINANCE32UI_EXT,Fail
+spec@ext_texture_integer@texwrap formats@GL_LUMINANCE32UI_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats@GL_LUMINANCE32UI_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats@GL_LUMINANCE8I_EXT,Fail
+spec@ext_texture_integer@texwrap formats@GL_LUMINANCE8I_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats@GL_LUMINANCE8I_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats@GL_LUMINANCE8UI_EXT,Fail
+spec@ext_texture_integer@texwrap formats@GL_LUMINANCE8UI_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats@GL_LUMINANCE8UI_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats@GL_LUMINANCE_ALPHA16I_EXT,Fail
+spec@ext_texture_integer@texwrap formats@GL_LUMINANCE_ALPHA16I_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats@GL_LUMINANCE_ALPHA16I_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats@GL_LUMINANCE_ALPHA16UI_EXT,Fail
+spec@ext_texture_integer@texwrap formats@GL_LUMINANCE_ALPHA16UI_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats@GL_LUMINANCE_ALPHA16UI_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats@GL_LUMINANCE_ALPHA32I_EXT,Fail
+spec@ext_texture_integer@texwrap formats@GL_LUMINANCE_ALPHA32I_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats@GL_LUMINANCE_ALPHA32I_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats@GL_LUMINANCE_ALPHA32UI_EXT,Fail
+spec@ext_texture_integer@texwrap formats@GL_LUMINANCE_ALPHA32UI_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats@GL_LUMINANCE_ALPHA32UI_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats@GL_LUMINANCE_ALPHA8I_EXT,Fail
+spec@ext_texture_integer@texwrap formats@GL_LUMINANCE_ALPHA8I_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats@GL_LUMINANCE_ALPHA8I_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats@GL_LUMINANCE_ALPHA8UI_EXT,Fail
+spec@ext_texture_integer@texwrap formats@GL_LUMINANCE_ALPHA8UI_EXT- NPOT,Fail
+spec@ext_texture_integer@texwrap formats@GL_LUMINANCE_ALPHA8UI_EXT- swizzled,Fail
+spec@ext_texture_integer@texwrap formats@GL_RGB16I,Fail
+spec@ext_texture_integer@texwrap formats@GL_RGB16I- NPOT,Fail
+spec@ext_texture_integer@texwrap formats@GL_RGB16I- swizzled,Fail
+spec@ext_texture_integer@texwrap formats@GL_RGB16UI,Fail
+spec@ext_texture_integer@texwrap formats@GL_RGB16UI- NPOT,Fail
+spec@ext_texture_integer@texwrap formats@GL_RGB16UI- swizzled,Fail
+spec@ext_texture_integer@texwrap formats@GL_RGB32I,Fail
+spec@ext_texture_integer@texwrap formats@GL_RGB32I- NPOT,Fail
+spec@ext_texture_integer@texwrap formats@GL_RGB32I- swizzled,Fail
+spec@ext_texture_integer@texwrap formats@GL_RGB32UI,Fail
+spec@ext_texture_integer@texwrap formats@GL_RGB32UI- NPOT,Fail
+spec@ext_texture_integer@texwrap formats@GL_RGB32UI- swizzled,Fail
+spec@ext_texture_integer@texwrap formats@GL_RGB8I,Fail
+spec@ext_texture_integer@texwrap formats@GL_RGB8I- NPOT,Fail
+spec@ext_texture_integer@texwrap formats@GL_RGB8I- swizzled,Fail
+spec@ext_texture_integer@texwrap formats@GL_RGB8UI,Fail
+spec@ext_texture_integer@texwrap formats@GL_RGB8UI- NPOT,Fail
+spec@ext_texture_integer@texwrap formats@GL_RGB8UI- swizzled,Fail
+spec@ext_texture_integer@texwrap formats@GL_RGBA16I,Fail
+spec@ext_texture_integer@texwrap formats@GL_RGBA16I- NPOT,Fail
+spec@ext_texture_integer@texwrap formats@GL_RGBA16I- swizzled,Fail
+spec@ext_texture_integer@texwrap formats@GL_RGBA16UI,Fail
+spec@ext_texture_integer@texwrap formats@GL_RGBA16UI- NPOT,Fail
+spec@ext_texture_integer@texwrap formats@GL_RGBA16UI- swizzled,Fail
+spec@ext_texture_integer@texwrap formats@GL_RGBA32I,Fail
+spec@ext_texture_integer@texwrap formats@GL_RGBA32I- NPOT,Fail
+spec@ext_texture_integer@texwrap formats@GL_RGBA32I- swizzled,Fail
+spec@ext_texture_integer@texwrap formats@GL_RGBA32UI,Fail
+spec@ext_texture_integer@texwrap formats@GL_RGBA32UI- NPOT,Fail
+spec@ext_texture_integer@texwrap formats@GL_RGBA32UI- swizzled,Fail
+spec@ext_texture_integer@texwrap formats@GL_RGBA8I,Fail
+spec@ext_texture_integer@texwrap formats@GL_RGBA8I- NPOT,Fail
+spec@ext_texture_integer@texwrap formats@GL_RGBA8I- swizzled,Fail
+spec@ext_texture_integer@texwrap formats@GL_RGBA8UI,Fail
+spec@ext_texture_integer@texwrap formats@GL_RGBA8UI- NPOT,Fail
+spec@ext_texture_integer@texwrap formats@GL_RGBA8UI- swizzled,Fail
+spec@ext_transform_feedback@points-large,Fail
+spec@ext_transform_feedback@tessellation polygon wireframe,Fail
+spec@ext_transform_feedback@tessellation quad_strip wireframe,Fail
+spec@ext_transform_feedback@tessellation quads wireframe,Fail
+spec@ext_transform_feedback@tessellation triangle_strip flat_last,Fail
+[email protected]@[email protected],Fail
+[email protected]@execution@geometry@primitive-types gl_triangle_strip,Fail
+[email protected]@execution@geometry@primitive-types gl_triangle_strip_adjacency,Fail
+[email protected]@execution@geometry@tri-strip-ordering-with-prim-restart gl_triangle_strip ffs,Fail
+[email protected]@execution@geometry@tri-strip-ordering-with-prim-restart gl_triangle_strip other,Fail
+[email protected]@execution@geometry@tri-strip-ordering-with-prim-restart gl_triangle_strip_adjacency ffs,Fail
+[email protected]@execution@geometry@tri-strip-ordering-with-prim-restart gl_triangle_strip_adjacency other,Fail
+[email protected]@gs-max-output,Crash
+spec@khr_texture_compression_astc@miptree-gl srgb-fp,Fail
+spec@khr_texture_compression_astc@miptree-gl srgb-fp@sRGB decode full precision,Fail
+spec@khr_texture_compression_astc@sliced-3d-miptree-gl srgb-fp,Fail
+spec@khr_texture_compression_astc@sliced-3d-miptree-gl srgb-fp@sRGB decode full precision,Fail
+spec@nv_copy_depth_to_color@nv_copy_depth_to_color,Fail
+spec@nv_copy_depth_to_color@nv_copy_depth_to_color 0 0x223344ff,Fail
+spec@nv_copy_depth_to_color@nv_copy_depth_to_color 0 0x76356278,Fail
+spec@nv_copy_depth_to_color@nv_copy_depth_to_color 1 0x223344ff,Fail
+spec@nv_copy_depth_to_color@nv_copy_depth_to_color 1 0x76356278,Fail
+wgl@wgl-multi-context-single-window,Fail

src/gallium/drivers/d3d12/ci/d3d12-quick_shader.toml

@@ -0,0 +1,5 @@
+[[piglit]]
+piglit_folder = "C:/piglit"
+profile = "quick_shader"
+process_isolation = true
+timeout = 240

src/gallium/drivers/d3d12/ci/d3d12-quick_shader.txt

@@ -0,0 +1,25 @@
+spec@arb_gpu_shader_int64@execution@indirect-array-two-accesses,Fail
+spec@arb_shader_texture_lod@execution@glsl-fs-shadow2dgradarb-07,Fail
+spec@arb_shader_texture_lod@execution@glsl-fs-shadow2dgradarb-cumulative,Fail
+spec@arb_tessellation_shader@execution@gs-primitiveid-instanced,Fail
+spec@arb_tessellation_shader@execution@tcs-output-unmatched,Fail
+spec@arb_tessellation_shader@execution@tes-primitiveid,Fail
+spec@arb_tessellation_shader@execution@variable-indexing@tcs-patch-output-array-vec4-index-wr,Crash
+spec@arb_tessellation_shader@execution@variable-indexing@tcs-patch-vec4-index-wr,Fail
+spec@arb_tessellation_shader@execution@variable-indexing@tcs-patch-vec4-swiz-index-wr,Fail
+[email protected]@execution@geometry@primitive-id-out,Fail
+[email protected]@execution@primitive-id-no-gs-quad-strip,Fail
+[email protected]@execution@primitive-id-no-gs-quads,Fail
+[email protected]@execution@primitive-id-no-gs-point,Fail
+[email protected]@execution@variable-indexing@gs-output-array-vec4-index-wr,Crash
+
+# These tests use a TCS output variable only as temporary storage. Since the output
+# is unused by the TES, we remove it.
+spec@arb_tessellation_shader@execution@barrier-patch,Crash
+spec@arb_tessellation_shader@execution@variable-indexing@tcs-output-array-float-index-wr-before-barrier,Fail
+spec@arb_tessellation_shader@execution@variable-indexing@tcs-output-array-vec2-index-rd-after-barrier,Fail
+spec@arb_tessellation_shader@execution@variable-indexing@tcs-output-array-vec2-index-wr-before-barrier,Fail
+spec@arb_tessellation_shader@execution@variable-indexing@tcs-output-array-vec3-index-rd-after-barrier,Fail
+spec@arb_tessellation_shader@execution@variable-indexing@tcs-output-array-vec3-index-wr-before-barrier,Fail
+spec@arb_tessellation_shader@execution@variable-indexing@tcs-output-array-vec4-index-rd-after-barrier,Fail
+spec@arb_tessellation_shader@execution@variable-indexing@tcs-output-array-vec4-index-wr-before-barrier,Fail

src/gallium/drivers/d3d12/ci/d3d12-skips.txt

@@ -0,0 +1,37 @@
+# The Windows piglit build doesn't produce any EGL/GLES tests
+gles
+opengl es
+egl
+glsl-es
+spec@oes
+
+# These are GLES extensions
+ext_clear_texture
+ext_disjoint_timer_query
+ext_draw_instanced
+ext_occlusion_query_boolean
+ext_render_snorm
+ext_texture_format_bgra8888
+ext_texture_norm16
+ext_unpack_subimage
+
+# These are Vulkan tests which aren't built on Windows either
+ext_external_objects@vk
+
+# Linux-specific extension
+ext_image_dma_buf_import
+
+# Requires EGL/GLES
+shaders@glsl-bug-110796
+amd_compressed_atc_texture
+intel_performance_query
+
+# This test will timeout and slows down the whole job, so just skip it
+spec@arb_tessellation_shader@arb_tessellation_shader-tes-gs-max-output
+[email protected]@gs-max-output
+
+# The > symbol in a test name causes the log to be unable to be written
+>
+
+# This test requires glut
+windowoverlap

src/gallium/drivers/d3d12/ci/gitlab-ci-inc.yml

@@ -0,0 +1,43 @@
+.d3d12-test:
+  extends:
+    - .use-windows_test_msvc
+    - .glon12-test-rules
+  stage: layered-backends
+  dependencies:
+    - windows-msvc
+  needs:
+    - job: windows-msvc
+    - job: windows_test_msvc
+      optional: true
+  variables:
+    GIT_STRATEGY: none # testing doesn't build anything from source
+    GALLIUM_DRIVER: d3d12
+    DRIVER_NAME: d3d12
+    WGL_DISABLE_ERROR_DIALOGS: "true"
+    D3D12_AGILITY_RELATIVE_PATH: "D3D12\\"
+    D3D12_AGILITY_SDK_VERSION: 619
+
+.d3d12-test-piglit:
+  extends:
+    - .d3d12-test
+    - .test-piglit
+  script:
+    - . _install/piglit_run.ps1
+  # temporarily restrict to GSt runners until we discover why GL testing is
+  # so slow on the Microsoft runners
+  tags:
+    - windows
+    - docker
+    - "2022"
+    - mesa
+    - gstreamer-windows
+  timeout: 12m
+  artifacts:
+    when: on_failure
+    name: "${CI_PROJECT_NAME}_${CI_JOB_NAME_SLUG}"
+    paths:
+      - logs/
+  variables:
+    PIGLIT_SKIPS: d3d12-skips.txt
+    PIGLIT_FLAKES: d3d12-flakes.txt
+

src/gallium/drivers/d3d12/ci/gitlab-ci.yml

@@ -0,0 +1,22 @@
+include:
+  - local: 'src/gallium/drivers/d3d12/ci/gitlab-ci-inc.yml'
+
+test-d3d12-quick_gl:
+  extends:
+    - .d3d12-test-piglit
+  variables:
+    PIGLIT_SUITE: d3d12-quick_gl.toml
+    PIGLIT_BASELINE: d3d12-quick_gl.txt
+
+test-d3d12-quick_shader:
+  extends:
+    - .d3d12-test-piglit
+  variables:
+    PIGLIT_SUITE: d3d12-quick_shader.toml
+    PIGLIT_BASELINE: d3d12-quick_shader.txt
+
+test-d3d12-vainfo:
+  extends:
+    - .d3d12-test
+  script:
+    - . _install/vainfo_run.ps1

src/gallium/drivers/d3d12/d3d12_batch.cpp

@@ -0,0 +1,529 @@
+/*
+ * Copyright © Microsoft Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#include "d3d12_batch.h"
+#include "d3d12_context.h"
+#include "d3d12_fence.h"
+#include "d3d12_query.h"
+#include "d3d12_residency.h"
+#include "d3d12_resource.h"
+#include "d3d12_resource_state.h"
+#include "d3d12_screen.h"
+#include "d3d12_surface.h"
+
+#include "util/hash_table.h"
+#include "util/set.h"
+#include "util/u_inlines.h"
+
+#include <dxguids/dxguids.h>
+
+
+unsigned d3d12_sampler_desc_table_key_hash(const void* key)
+{
+   const d3d12_sampler_desc_table_key* table = (d3d12_sampler_desc_table_key*)key;
+
+   return _mesa_hash_data(table->descs, sizeof(table->descs[0]) * table->count);
+}
+bool d3d12_sampler_desc_table_key_equals(const void* a, const void* b)
+{
+   const d3d12_sampler_desc_table_key* table_a = (d3d12_sampler_desc_table_key*)a;
+   const d3d12_sampler_desc_table_key* table_b = (d3d12_sampler_desc_table_key*)b;
+   return table_a->count == table_b->count && memcmp(table_a->descs, table_b->descs, sizeof(table_a->descs[0]) * table_a->count) == 0;
+}
+
+bool
+d3d12_init_batch(struct d3d12_context *ctx, struct d3d12_batch *batch)
+{
+   struct d3d12_screen *screen = d3d12_screen(ctx->base.screen);
+
+   batch->bos = _mesa_hash_table_create(NULL, _mesa_hash_pointer,
+                                        _mesa_key_pointer_equal);
+
+   batch->local_bos = UTIL_DYNARRAY_INIT;
+   batch->local_bo_pending = UTIL_DYNARRAY_INIT;
+   batch->bos_pending = UTIL_DYNARRAY_INIT;
+
+#ifdef HAVE_GALLIUM_D3D12_GRAPHICS
+   batch->surfaces = _mesa_set_create(NULL, _mesa_hash_pointer,
+                                      _mesa_key_pointer_equal);
+#endif // HAVE_GALLIUM_D3D12_GRAPHICS
+
+   batch->objects = _mesa_set_create(NULL,
+                                     _mesa_hash_pointer,
+                                     _mesa_key_pointer_equal);
+
+if (!batch->bos || !batch->objects)
+      return false;
+
+#ifdef HAVE_GALLIUM_D3D12_GRAPHICS
+   if (!batch->surfaces)
+      return false;
+#endif // HAVE_GALLIUM_D3D12_GRAPHICS
+
+#ifdef HAVE_GALLIUM_D3D12_GRAPHICS
+   if (screen->max_feature_level >= D3D_FEATURE_LEVEL_11_0) {
+      batch->queries = _mesa_set_create(NULL, _mesa_hash_pointer,
+                                        _mesa_key_pointer_equal);
+
+      batch->view_heap =
+         d3d12_descriptor_heap_new(screen->dev,
+                                    D3D12_DESCRIPTOR_HEAP_TYPE_CBV_SRV_UAV,
+                                    D3D12_DESCRIPTOR_HEAP_FLAG_SHADER_VISIBLE,
+                                    8096);
+
+      batch->sampler_tables = _mesa_hash_table_create(NULL, d3d12_sampler_desc_table_key_hash,
+                                                      d3d12_sampler_desc_table_key_equals);
+      batch->sampler_views = _mesa_set_create(NULL, _mesa_hash_pointer,
+                                             _mesa_key_pointer_equal);
+
+      if (!batch->sampler_tables || !batch->sampler_views || !batch->view_heap || !batch->queries)
+         return false;
+
+      batch->zombie_samplers = UTIL_DYNARRAY_INIT;
+
+      batch->sampler_heap =
+         d3d12_descriptor_heap_new(screen->dev,
+                                 D3D12_DESCRIPTOR_HEAP_TYPE_SAMPLER,
+                                 D3D12_DESCRIPTOR_HEAP_FLAG_SHADER_VISIBLE,
+                                 1024);
+
+      if (!batch->sampler_heap)
+         return false;
+   }
+#endif // HAVE_GALLIUM_D3D12_GRAPHICS
+
+   if (FAILED(screen->dev->CreateCommandAllocator(screen->queue_type,
+                                                  IID_PPV_ARGS(&batch->cmdalloc))))
+      return false;
+
+   return true;
+}
+
+static inline void
+delete_bo(d3d12_bo *bo)
+{
+   d3d12_bo_unreference(bo);
+}
+static void
+delete_bo_entry(hash_entry *entry)
+{
+   struct d3d12_bo *bo = (struct d3d12_bo *)entry->key;
+   d3d12_bo_unreference(bo);
+}
+
+static void
+delete_sampler_view_table(hash_entry *entry)
+{
+   FREE((void*)entry->key);
+   FREE(entry->data);
+}
+
+static void
+delete_sampler_view(set_entry *entry)
+{
+   struct pipe_sampler_view *pres = (struct pipe_sampler_view *)entry->key;
+   pipe_sampler_view_reference(&pres, NULL);
+}
+
+static void
+delete_object(set_entry *entry)
+{
+   ID3D12Object *object = (ID3D12Object *)entry->key;
+   object->Release();
+}
+
+#ifdef HAVE_GALLIUM_D3D12_GRAPHICS
+static void
+delete_query(set_entry *entry)
+{
+   struct d3d12_query *query = (struct d3d12_query *)entry->key;
+   if (pipe_reference(&query->reference, nullptr))
+      d3d12_destroy_query(query);
+}
+#endif // HAVE_GALLIUM_D3D12_GRAPHICS
+
+bool
+d3d12_reset_batch(struct d3d12_context *ctx, struct d3d12_batch *batch, uint64_t timeout_ns)
+{
+   // batch hasn't been submitted before
+   if (!batch->fence && !batch->has_errors)
+      return true;
+
+   if (batch->fence) {
+      if (!d3d12_fence_finish(batch->fence, timeout_ns))
+         return false;
+      d3d12_fence_reference(&batch->fence, NULL);
+   }
+
+   _mesa_hash_table_clear(batch->bos, delete_bo_entry);
+
+   util_dynarray_foreach(&batch->bos_pending, struct d3d12_pending_free_entry*, entry)
+      FREE(*entry);
+   util_dynarray_clear(&batch->bos_pending);
+
+#ifdef HAVE_GALLIUM_D3D12_GRAPHICS
+   set_foreach_remove(batch->surfaces, entry) {
+      struct pipe_surface *surf = (struct pipe_surface *)entry->key;
+      pipe_surface_reference(&surf, NULL, &ctx->base, d3d12_surface_destroy);
+   }
+#endif // HAVE_GALLIUM_D3D12_GRAPHICS
+
+   _mesa_set_clear(batch->objects, delete_object);
+
+   util_dynarray_foreach(&batch->local_bos, d3d12_bo*, bo) {
+      (*bo)->local_reference_mask[batch->ctx_id] &= ~(1 << batch->ctx_index);
+      delete_bo(*bo);
+   }
+   util_dynarray_clear(&batch->local_bos);
+
+   util_dynarray_foreach(&batch->local_bo_pending, struct d3d12_pending_free_entry*, entry)
+      FREE(*entry);
+   util_dynarray_clear(&batch->local_bo_pending);
+
+#ifdef HAVE_GALLIUM_D3D12_GRAPHICS
+   if (d3d12_screen(ctx->base.screen)->max_feature_level >= D3D_FEATURE_LEVEL_11_0) {
+      _mesa_hash_table_clear(batch->sampler_tables, delete_sampler_view_table);
+      _mesa_set_clear(batch->sampler_views, delete_sampler_view);
+
+      _mesa_set_clear(batch->queries, delete_query);
+      util_dynarray_foreach(&batch->zombie_samplers, d3d12_descriptor_handle, handle)
+         d3d12_descriptor_handle_free(handle);
+      util_dynarray_clear(&batch->zombie_samplers);
+      d3d12_descriptor_heap_clear(batch->view_heap);
+      d3d12_descriptor_heap_clear(batch->sampler_heap);
+   }
+#endif // HAVE_GALLIUM_D3D12_GRAPHICS
+
+   if (FAILED(batch->cmdalloc->Reset())) {
+      debug_printf("D3D12: resetting ID3D12CommandAllocator failed\n");
+      return false;
+   }
+   batch->has_errors = false;
+   batch->pending_memory_barrier = false;
+   return true;
+}
+
+void
+d3d12_destroy_batch(struct d3d12_context *ctx, struct d3d12_batch *batch)
+{
+   d3d12_reset_batch(ctx, batch, OS_TIMEOUT_INFINITE);
+   batch->cmdalloc->Release();
+   _mesa_hash_table_destroy(batch->bos, NULL);
+
+#ifdef HAVE_GALLIUM_D3D12_GRAPHICS
+   if (d3d12_screen(ctx->base.screen)->max_feature_level >= D3D_FEATURE_LEVEL_11_0) {
+      d3d12_descriptor_heap_free(batch->sampler_heap);
+      d3d12_descriptor_heap_free(batch->view_heap);
+      _mesa_hash_table_destroy(batch->sampler_tables, NULL);
+      _mesa_set_destroy(batch->sampler_views, NULL);
+      _mesa_set_destroy(batch->queries, NULL);
+      util_dynarray_fini(&batch->zombie_samplers);
+   }
+   _mesa_set_destroy(batch->surfaces, NULL);
+#endif // HAVE_GALLIUM_D3D12_GRAPHICS
+
+   _mesa_set_destroy(batch->objects, NULL);
+   util_dynarray_fini(&batch->local_bos);
+   util_dynarray_fini(&batch->local_bo_pending);
+   util_dynarray_fini(&batch->bos_pending);
+}
+
+void
+d3d12_start_batch(struct d3d12_context *ctx, struct d3d12_batch *batch)
+{
+   struct d3d12_screen *screen = d3d12_screen(ctx->base.screen);
+   d3d12_reset_batch(ctx, batch, OS_TIMEOUT_INFINITE);
+
+   /* Create or reset global command list */
+   if (ctx->cmdlist) {
+      if (FAILED(ctx->cmdlist->Reset(batch->cmdalloc, NULL))) {
+         debug_printf("D3D12: resetting ID3D12GraphicsCommandList failed\n");
+         batch->has_errors = true;
+         return;
+      }
+   } else {
+      if (FAILED(screen->dev->CreateCommandList(0, screen->queue_type,
+                                                batch->cmdalloc, NULL,
+                                                IID_PPV_ARGS(&ctx->cmdlist)))) {
+         debug_printf("D3D12: creating ID3D12GraphicsCommandList failed\n");
+         batch->has_errors = true;
+         return;
+      }
+      if (FAILED(ctx->cmdlist->QueryInterface(IID_PPV_ARGS(&ctx->cmdlist2)))) {
+         ctx->cmdlist2 = nullptr;
+      }
+      if (FAILED(ctx->cmdlist->QueryInterface(IID_PPV_ARGS(&ctx->cmdlist8)))) {
+         ctx->cmdlist8 = nullptr;
+      }
+   }
+
+#ifdef HAVE_GALLIUM_D3D12_GRAPHICS
+   if (screen->max_feature_level >= D3D_FEATURE_LEVEL_11_0) {
+      ID3D12DescriptorHeap* heaps[2] = { d3d12_descriptor_heap_get(batch->view_heap),
+                                       d3d12_descriptor_heap_get(batch->sampler_heap) };
+      ctx->cmdlist->SetDescriptorHeaps(2, heaps);
+
+      ctx->cmdlist_dirty = ~0;
+      for (int i = 0; i < MESA_SHADER_STAGES; ++i)
+         ctx->shader_dirty[i] = ~0;
+   
+      if (!ctx->queries_disabled)
+         d3d12_resume_queries(ctx);
+      if (ctx->current_predication)
+         d3d12_enable_predication(ctx);
+   }
+#endif // HAVE_GALLIUM_D3D12_GRAPHICS
+
+   batch->submit_id = ++ctx->submit_id;
+}
+
+void
+d3d12_end_batch(struct d3d12_context *ctx, struct d3d12_batch *batch)
+{
+   struct d3d12_screen *screen = d3d12_screen(ctx->base.screen);
+
+#ifdef HAVE_GALLIUM_D3D12_GRAPHICS
+   if (!ctx->queries_disabled)
+      d3d12_suspend_queries(ctx);
+#endif // HAVE_GALLIUM_D3D12_GRAPHICS
+
+   if (FAILED(ctx->cmdlist->Close())) {
+      debug_printf("D3D12: closing ID3D12GraphicsCommandList failed\n");
+      batch->has_errors = true;
+      return;
+   }
+
+   mtx_lock(&screen->submit_mutex);
+
+#ifndef _GAMING_XBOX
+   d3d12_process_batch_residency(screen, batch);
+#endif
+
+   bool has_state_fixup = d3d12_context_state_resolve_submission(ctx, batch);
+
+   ID3D12CommandList *cmdlists[] = { ctx->state_fixup_cmdlist, ctx->cmdlist };
+   ID3D12CommandList **to_execute = cmdlists;
+   UINT count_to_execute = ARRAY_SIZE(cmdlists);
+   if (!has_state_fixup) {
+      to_execute++;
+      count_to_execute--;
+   }
+   screen->cmdqueue->ExecuteCommandLists(count_to_execute, to_execute);
+
+   batch->fence = d3d12_create_fence(screen, true);
+
+   if (batch->fence) {
+      uint64_t fv = batch->fence->value;
+
+      unsigned n_local = util_dynarray_num_elements(&batch->local_bo_pending,
+                                                    struct d3d12_pending_free_entry*);
+      struct d3d12_pending_free_entry **local_entries =
+         (struct d3d12_pending_free_entry **)util_dynarray_begin(&batch->local_bo_pending);
+      bool local_ok = n_local > 0;
+      for (unsigned i = 0; i < n_local; ++i) {
+         if (!local_entries[i]) {
+            local_ok = false;
+            break;
+         }
+      }
+
+      unsigned n_shared = util_dynarray_num_elements(&batch->bos_pending,
+                                                     struct d3d12_pending_free_entry*);
+      struct d3d12_pending_free_entry **shared_entries =
+         (struct d3d12_pending_free_entry **)util_dynarray_begin(&batch->bos_pending);
+      bool shared_ok = n_shared > 0;
+      for (unsigned i = 0; i < n_shared; ++i) {
+         if (!shared_entries[i]) {
+            shared_ok = false;
+            break;
+         }
+      }
+
+      if (local_ok || shared_ok) {
+         mtx_lock(&screen->pending_free_lock);
+         if (local_ok) {
+            for (unsigned i = 0; i < n_local; ++i) {
+               local_entries[i]->fence_value = fv;
+               list_addtail(&local_entries[i]->link, &screen->pending_free_list);
+            }
+         }
+         if (shared_ok) {
+            for (unsigned i = 0; i < n_shared; ++i) {
+               shared_entries[i]->fence_value = fv;
+               list_addtail(&shared_entries[i]->link, &screen->pending_free_list);
+            }
+         }
+         mtx_unlock(&screen->pending_free_lock);
+      }
+
+      if (local_ok) {
+         struct d3d12_bo **bos =
+            (struct d3d12_bo **)util_dynarray_begin(&batch->local_bos);
+         for (unsigned i = 0; i < n_local; ++i)
+            bos[i]->local_reference_mask[batch->ctx_id] &= ~(1 << batch->ctx_index);
+         util_dynarray_clear(&batch->local_bos);
+         util_dynarray_clear(&batch->local_bo_pending);
+      }
+
+      if (shared_ok) {
+         _mesa_hash_table_clear(batch->bos, NULL);
+         util_dynarray_clear(&batch->bos_pending);
+      }
+   }
+
+#ifdef HAVE_GALLIUM_D3D12_GRAPHICS
+   /* batch->queries is NULL when no grfx supported */
+   if (screen->max_feature_level >= D3D_FEATURE_LEVEL_11_0) {
+      set_foreach_remove(batch->queries, entry) {
+         d3d12_query *query = (struct d3d12_query *)entry->key;
+         if (pipe_reference(&query->reference, nullptr))
+            d3d12_destroy_query(query);
+         else
+            query->fence_value = screen->fence_value;
+      }
+   }
+#endif // HAVE_GALLIUM_D3D12_GRAPHICS
+
+   mtx_unlock(&screen->submit_mutex);
+
+   d3d12_screen_reclaim_completed(screen);
+}
+
+
+inline uint8_t*
+d3d12_batch_get_reference(struct d3d12_batch *batch,
+                          struct d3d12_bo *bo)
+{
+   if (batch->ctx_id != D3D12_CONTEXT_NO_ID) {
+      if ((bo->local_reference_mask[batch->ctx_id] & (1 << batch->ctx_index)) != 0) {
+         return &bo->local_reference_state[batch->ctx_id][batch->ctx_index];
+      }
+      else
+         return NULL;
+   }
+   else {
+      hash_entry* entry = _mesa_hash_table_search(batch->bos, bo);
+      if (entry == NULL)
+         return NULL;
+      else
+         return (uint8_t*)&entry->data;
+   }
+}
+
+inline uint8_t*
+d3d12_batch_acquire_reference(struct d3d12_batch *batch,
+                          struct d3d12_bo *bo)
+{
+   if (batch->ctx_id != D3D12_CONTEXT_NO_ID) {
+      if ((bo->local_reference_mask[batch->ctx_id] & (1 << batch->ctx_index)) == 0) {
+         d3d12_bo_reference(bo);
+         util_dynarray_append(&batch->local_bos, bo);
+         struct d3d12_pending_free_entry *entry = MALLOC_STRUCT(d3d12_pending_free_entry);
+         if (entry)
+            entry->bo = bo;
+         util_dynarray_append(&batch->local_bo_pending, entry);
+         bo->local_reference_mask[batch->ctx_id] |= (1 << batch->ctx_index);
+         bo->local_reference_state[batch->ctx_id][batch->ctx_index] = batch_bo_reference_none;
+      }
+      return &bo->local_reference_state[batch->ctx_id][batch->ctx_index];
+   }
+   else {
+      hash_entry* entry = _mesa_hash_table_search(batch->bos, bo);
+      if (entry == NULL) {
+         d3d12_bo_reference(bo);
+         entry = _mesa_hash_table_insert(batch->bos, bo, NULL);
+         struct d3d12_pending_free_entry *pe = MALLOC_STRUCT(d3d12_pending_free_entry);
+         if (pe)
+            pe->bo = bo;
+         util_dynarray_append(&batch->bos_pending, pe);
+      }
+
+      return (uint8_t*)&entry->data;
+   }
+}
+
+bool
+d3d12_batch_has_references(struct d3d12_batch *batch,
+                           struct d3d12_bo *bo,
+                           bool want_to_write)
+{
+   uint8_t*state = d3d12_batch_get_reference(batch, bo);
+   if (state == NULL)
+      return false;
+   bool resource_was_written = ((batch_bo_reference_state)(size_t)*state & batch_bo_reference_written) != 0;
+   return want_to_write || resource_was_written;
+}
+
+void
+d3d12_batch_reference_resource(struct d3d12_batch *batch,
+                               struct d3d12_resource *res,
+                               bool write)
+{
+   uint8_t*state = d3d12_batch_acquire_reference(batch, res->bo);
+
+   uint8_t new_data = write ? batch_bo_reference_written : batch_bo_reference_read;
+   uint8_t old_data = (uint8_t)*state;
+   *state = (old_data | new_data);
+}
+
+void
+d3d12_batch_reference_sampler_view(struct d3d12_batch *batch,
+                                   struct d3d12_sampler_view *sv)
+{
+   struct set_entry *entry = _mesa_set_search(batch->sampler_views, sv);
+   if (!entry) {
+      entry = _mesa_set_add(batch->sampler_views, sv);
+      pipe_reference(NULL, &sv->base.reference);
+
+      d3d12_batch_reference_resource(batch, d3d12_resource(sv->base.texture), false);
+   }
+}
+
+void
+d3d12_batch_reference_surface_texture(struct d3d12_batch *batch,
+                                      struct d3d12_surface *surf)
+{
+   d3d12_batch_reference_resource(batch, d3d12_resource(surf->base.texture), true);
+}
+
+void
+d3d12_batch_reference_object(struct d3d12_batch *batch,
+                             ID3D12Object *object)
+{
+   struct set_entry *entry = _mesa_set_search(batch->objects, object);
+   if (!entry) {
+      entry = _mesa_set_add(batch->objects, object);
+      object->AddRef();
+   }
+}
+
+void
+d3d12_batch_reference_query(struct d3d12_batch *batch,
+                            struct d3d12_query *query)
+{
+   struct set_entry *entry = _mesa_set_search(batch->queries, query);
+   if (!entry) {
+      entry = _mesa_set_add(batch->queries, query);
+      pipe_reference(NULL, &query->reference);
+   }
+}

src/gallium/drivers/d3d12/d3d12_batch.h

@@ -0,0 +1,113 @@
+/*
+ * Copyright © Microsoft Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifndef D3D12_BATCH_H
+#define D3D12_BATCH_H
+
+#include "util/u_dynarray.h"
+#include "util/hash_table.h"
+#include "pipe/p_state.h"
+#include <stdint.h>
+
+#include "d3d12_common.h"
+
+struct d3d12_bo;
+struct d3d12_descriptor_heap;
+struct d3d12_fence;
+
+
+struct d3d12_sampler_desc_table_key
+{
+   D3D12_CPU_DESCRIPTOR_HANDLE descs[PIPE_MAX_SHADER_SAMPLER_VIEWS];
+   unsigned count;
+};
+
+struct d3d12_batch {
+   struct d3d12_fence *fence;
+
+   struct hash_table *bos;
+   struct util_dynarray local_bos;
+   struct util_dynarray local_bo_pending;
+   struct util_dynarray bos_pending;
+   struct hash_table *sampler_tables;
+   struct set *sampler_views;
+#ifdef HAVE_GALLIUM_D3D12_GRAPHICS
+   struct set *surfaces;
+#endif // HAVE_GALLIUM_D3D12_GRAPHICS
+   struct set *objects;
+   struct set *queries;
+
+   struct util_dynarray zombie_samplers;
+
+   ID3D12CommandAllocator *cmdalloc;
+   struct d3d12_descriptor_heap *sampler_heap;
+   struct d3d12_descriptor_heap *view_heap;
+   bool has_errors;
+   bool pending_memory_barrier;
+
+   uint64_t submit_id;
+   uint32_t ctx_id, ctx_index;
+};
+
+bool
+d3d12_init_batch(struct d3d12_context *ctx, struct d3d12_batch *batch);
+
+void
+d3d12_destroy_batch(struct d3d12_context *ctx, struct d3d12_batch *batch);
+
+void
+d3d12_start_batch(struct d3d12_context *ctx, struct d3d12_batch *batch);
+
+void
+d3d12_end_batch(struct d3d12_context *ctx, struct d3d12_batch *batch);
+
+bool
+d3d12_reset_batch(struct d3d12_context *ctx, struct d3d12_batch *batch, uint64_t timeout_ns);
+
+bool
+d3d12_batch_has_references(struct d3d12_batch *batch,
+                           struct d3d12_bo *bo,
+                           bool want_to_write);
+
+void
+d3d12_batch_reference_resource(struct d3d12_batch *batch,
+                               struct d3d12_resource *res,
+                               bool write);
+
+void
+d3d12_batch_reference_sampler_view(struct d3d12_batch *batch,
+                                   struct d3d12_sampler_view *sv);
+
+void
+d3d12_batch_reference_surface_texture(struct d3d12_batch *batch,
+                              struct d3d12_surface *surf);
+
+void
+d3d12_batch_reference_object(struct d3d12_batch *batch,
+                             ID3D12Object *object);
+
+void
+d3d12_batch_reference_query(struct d3d12_batch *batch,
+                            struct d3d12_query *query);
+
+#endif

src/gallium/drivers/d3d12/d3d12_blit.cpp

@@ -0,0 +1,702 @@
+/*
+ * Copyright © Microsoft Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#include "d3d12_context.h"
+#include "d3d12_compiler.h"
+#include "d3d12_debug.h"
+#include "d3d12_blit.h"
+#include "d3d12_format.h"
+#include "d3d12_query.h"
+#include "d3d12_resource.h"
+#include "d3d12_screen.h"
+
+#include "util/u_blitter.h"
+#include "util/format/u_format.h"
+
+#include "nir_to_dxil.h"
+#include "nir_builder.h"
+
+static bool
+is_resolve(const struct pipe_blit_info *info)
+{
+   return info->src.resource->nr_samples > 1 &&
+          info->dst.resource->nr_samples <= 1;
+}
+
+static bool
+resolve_supported(const struct pipe_blit_info *info)
+{
+   assert(is_resolve(info));
+
+   // check for unsupported operations
+   if (util_format_is_depth_or_stencil(info->src.format) &&
+       info->mask != PIPE_MASK_Z) {
+      return false;
+   } else {
+      if (util_format_get_mask(info->dst.format) != info->mask ||
+          util_format_get_mask(info->src.format) != info->mask ||
+          util_format_has_alpha1(info->src.format))
+         return false;
+   }
+
+   if (info->filter != PIPE_TEX_FILTER_NEAREST ||
+       info->scissor_enable ||
+       info->swizzle_enable ||
+       info->num_window_rectangles > 0 ||
+       info->alpha_blend)
+      return false;
+
+   // formats need to match
+   struct d3d12_resource *src = d3d12_resource(info->src.resource);
+   struct d3d12_resource *dst = d3d12_resource(info->dst.resource);
+   if (src->dxgi_format != dst->dxgi_format)
+      return false;
+
+   if (util_format_is_pure_integer(src->base.b.format))
+      return false;
+
+   // sizes needs to match
+   if (info->src.box.width != info->dst.box.width ||
+       info->src.box.height != info->dst.box.height)
+      return false;
+
+   // can only resolve full subresource
+   if (info->src.box.width != (int)u_minify(info->src.resource->width0,
+                                            info->src.level) ||
+       info->src.box.height != (int)u_minify(info->src.resource->height0,
+                                             info->src.level) ||
+       info->dst.box.width != (int)u_minify(info->dst.resource->width0,
+                                            info->dst.level) ||
+       info->dst.box.height != (int)u_minify(info->dst.resource->height0,
+                                             info->dst.level))
+      return false;
+
+   return true;
+}
+
+static void
+blit_resolve(struct d3d12_context *ctx, const struct pipe_blit_info *info)
+{
+   struct d3d12_batch *batch = d3d12_current_batch(ctx);
+   struct d3d12_resource *src = d3d12_resource(info->src.resource);
+   struct d3d12_resource *dst = d3d12_resource(info->dst.resource);
+
+   d3d12_transition_resource_state(ctx, src,
+                                   D3D12_RESOURCE_STATE_RESOLVE_SOURCE,
+                                   D3D12_TRANSITION_FLAG_INVALIDATE_BINDINGS);
+   d3d12_transition_resource_state(ctx, dst,
+                                   D3D12_RESOURCE_STATE_RESOLVE_DEST,
+                                   D3D12_TRANSITION_FLAG_INVALIDATE_BINDINGS);
+
+   d3d12_apply_resource_states(ctx, false);
+
+   d3d12_batch_reference_resource(batch, src, false);
+   d3d12_batch_reference_resource(batch, dst, true);
+
+   DXGI_FORMAT dxgi_format = d3d12_get_resource_srv_format(src->base.b.format, src->base.b.target);
+
+   assert(src->dxgi_format == dst->dxgi_format);
+   ctx->cmdlist->ResolveSubresource(
+      d3d12_resource_resource(dst), info->dst.level,
+      d3d12_resource_resource(src), info->src.level,
+      dxgi_format);
+   ctx->has_commands = true;
+}
+
+static bool
+formats_are_copy_compatible(enum pipe_format src, enum pipe_format dst)
+{
+   if (src == dst)
+      return true;
+
+   /* We can skip the stencil copy */
+   if (util_format_get_depth_only(src) == dst ||
+       util_format_get_depth_only(dst) == src)
+      return true;
+
+   return false;
+}
+
+static bool
+box_fits(const struct pipe_box *box, const struct pipe_resource *res, int level)
+{
+   unsigned lwidth = u_minify(res->width0, level);
+   unsigned lheight= u_minify(res->height0, level);
+   unsigned ldepth = res->target == PIPE_TEXTURE_3D ? u_minify(res->depth0, level) :
+                                                      res->array_size;
+
+   unsigned wb = box->x;
+   unsigned we = box->x + box->width;
+
+   unsigned hb = box->y;
+   unsigned he = box->y + box->height;
+
+   unsigned db = box->z;
+   unsigned de = box->z + box->depth;
+
+   return (wb <= lwidth && we <= lwidth &&
+           hb <= lheight && he <= lheight &&
+           db <= ldepth && de <= ldepth);
+}
+
+static bool
+direct_copy_supported(struct d3d12_screen *screen,
+                      const struct pipe_blit_info *info,
+                      bool have_predication)
+{
+   if (info->scissor_enable || info->alpha_blend || info->swizzle_enable ||
+       (have_predication && info->render_condition_enable) ||
+       MAX2(info->src.resource->nr_samples, 1) != MAX2(info->dst.resource->nr_samples, 1)) {
+      return false;
+   }
+
+   if (!formats_are_copy_compatible(info->src.format, info->dst.format))
+      return false;
+
+   if (info->src.format != info->src.resource->format ||
+       info->dst.format != info->dst.resource->format)
+      return false;
+
+   if (util_format_is_depth_or_stencil(info->src.format) && !(info->mask & PIPE_MASK_ZS)) {
+      return false;
+   }
+
+   if (!util_format_is_depth_or_stencil(info->src.format)) {
+      if (util_format_get_mask(info->dst.format) != info->mask ||
+          util_format_get_mask(info->src.format) != info->mask)
+         return false;
+   }
+
+   if (abs(info->src.box.height) != info->dst.box.height) {
+      return false;
+   }
+
+   if (info->src.box.height != info->dst.box.height &&
+       (!util_format_is_depth_or_stencil(info->src.format) ||
+        screen->opts2.ProgrammableSamplePositionsTier ==
+        D3D12_PROGRAMMABLE_SAMPLE_POSITIONS_TIER_NOT_SUPPORTED)) {
+      return false;
+   }
+
+   if (!box_fits(&info->dst.box, info->dst.resource, info->dst.level)) {
+      return false;
+   }
+   if (!box_fits(&info->src.box, info->src.resource, info->src.level)) {
+      return false;
+   }
+
+   if (info->src.box.width != info->dst.box.width) {
+      return false;
+   }
+
+   if (info->src.box.depth != info->dst.box.depth) {
+      return false;
+   }
+
+   if ((screen->opts2.ProgrammableSamplePositionsTier ==
+        D3D12_PROGRAMMABLE_SAMPLE_POSITIONS_TIER_NOT_SUPPORTED &&
+        (info->src.resource->bind & PIPE_BIND_DEPTH_STENCIL ||
+         info->dst.resource->bind & PIPE_BIND_DEPTH_STENCIL)) ||
+        info->src.resource->nr_samples != info->dst.resource->nr_samples) {
+
+      if (info->dst.box.x != 0 ||
+          info->dst.box.y != 0 ||
+          info->dst.box.z != 0)
+         return false;
+
+      if (info->src.box.x != 0 ||
+          info->src.box.y != 0 ||
+          info->src.box.z != 0 ||
+          info->src.box.width != (int)u_minify(info->src.resource->width0,
+                                               info->src.level) ||
+          info->src.box.height != (int)u_minify(info->src.resource->height0,
+                                                info->src.level) ||
+          info->src.box.depth != (int)u_minify(info->src.resource->depth0,
+                                               info->src.level))
+         return false;
+   }
+
+   return true;
+}
+
+static bool
+is_same_resource(const struct pipe_blit_info *info)
+{
+   return d3d12_resource_resource(d3d12_resource(info->src.resource)) ==
+             d3d12_resource_resource(d3d12_resource(info->dst.resource)) &&
+          info->src.level == info->dst.level;
+}
+
+static void
+blit_same_resource(struct d3d12_context *ctx,
+                   const struct pipe_blit_info *info)
+{
+   struct pipe_blit_info dst_info = *info;
+
+   dst_info.src.level = 0;
+   dst_info.src.resource = create_staging_resource(ctx, d3d12_resource(info->src.resource),
+                                                   info->src.level,
+                                                   &info->src.box,
+                                                   &dst_info.src.box, PIPE_MASK_RGBAZS);
+   ctx->base.blit(&ctx->base, &dst_info);
+   pipe_resource_reference(&dst_info.src.resource, NULL);
+}
+
+static void
+util_blit_save_state(struct d3d12_context *ctx)
+{
+   util_blitter_save_blend(ctx->blitter, ctx->gfx_pipeline_state.blend);
+   util_blitter_save_depth_stencil_alpha(ctx->blitter, ctx->gfx_pipeline_state.zsa);
+   util_blitter_save_vertex_elements(ctx->blitter, ctx->gfx_pipeline_state.ves);
+   util_blitter_save_stencil_ref(ctx->blitter, &ctx->stencil_ref);
+   util_blitter_save_rasterizer(ctx->blitter, ctx->gfx_pipeline_state.rast);
+   util_blitter_save_fragment_shader(ctx->blitter, ctx->gfx_stages[MESA_SHADER_FRAGMENT]);
+   util_blitter_save_vertex_shader(ctx->blitter, ctx->gfx_stages[MESA_SHADER_VERTEX]);
+   util_blitter_save_geometry_shader(ctx->blitter, ctx->gfx_stages[MESA_SHADER_GEOMETRY]);
+   util_blitter_save_tessctrl_shader(ctx->blitter, ctx->gfx_stages[MESA_SHADER_TESS_CTRL]);
+   util_blitter_save_tesseval_shader(ctx->blitter, ctx->gfx_stages[MESA_SHADER_TESS_EVAL]);
+
+   util_blitter_save_framebuffer(ctx->blitter, &ctx->fb);
+   util_blitter_save_viewport(ctx->blitter, ctx->viewport_states);
+   util_blitter_save_scissor(ctx->blitter, ctx->scissor_states);
+   util_blitter_save_fragment_sampler_states(ctx->blitter,
+                                             ctx->num_samplers[MESA_SHADER_FRAGMENT],
+                                             (void **)ctx->samplers[MESA_SHADER_FRAGMENT]);
+   util_blitter_save_fragment_sampler_views(ctx->blitter,
+                                            ctx->num_sampler_views[MESA_SHADER_FRAGMENT],
+                                            ctx->sampler_views[MESA_SHADER_FRAGMENT]);
+   util_blitter_save_fragment_constant_buffer_slot(ctx->blitter, ctx->cbufs[MESA_SHADER_FRAGMENT]);
+   util_blitter_save_vertex_buffers(ctx->blitter, ctx->vbs, ctx->num_vbs);
+   util_blitter_save_sample_mask(ctx->blitter, ctx->gfx_pipeline_state.sample_mask, 0);
+   util_blitter_save_so_targets(ctx->blitter, ctx->gfx_pipeline_state.num_so_targets, ctx->so_targets,
+                                MESA_PRIM_UNKNOWN);
+}
+
+static void
+util_blit(struct d3d12_context *ctx,
+          const struct pipe_blit_info *info)
+{
+   util_blit_save_state(ctx);
+
+   util_blitter_blit(ctx->blitter, info, NULL);
+}
+
+static bool
+resolve_stencil_supported(struct d3d12_context *ctx,
+                          const struct pipe_blit_info *info)
+{
+   assert(is_resolve(info));
+
+   if (!util_format_is_depth_or_stencil(info->src.format) ||
+       !(info->mask & PIPE_MASK_S))
+      return false;
+
+   if (info->mask & PIPE_MASK_Z) {
+      struct pipe_blit_info new_info = *info;
+      new_info.mask = PIPE_MASK_Z;
+      if (!resolve_supported(&new_info) &&
+          !util_blitter_is_blit_supported(ctx->blitter, &new_info))
+         return false;
+   }
+
+   struct pipe_blit_info new_info = *info;
+   new_info.dst.format = PIPE_FORMAT_R8_UINT;
+   return util_blitter_is_blit_supported(ctx->blitter, &new_info);
+}
+
+static struct pipe_resource *
+create_tmp_resource(struct pipe_screen *screen,
+                    const struct pipe_blit_info *info)
+{
+   struct pipe_resource tpl = {};
+   tpl.width0 = info->dst.box.width;
+   tpl.height0 = static_cast<uint16_t>(info->dst.box.height);
+   tpl.depth0 = info->dst.box.depth;
+   tpl.array_size = 1;
+   tpl.format = PIPE_FORMAT_R8_UINT;
+   tpl.target = info->dst.resource->target;
+   tpl.nr_samples = info->dst.resource->nr_samples;
+   tpl.nr_storage_samples = info->dst.resource->nr_storage_samples;
+   tpl.usage = PIPE_USAGE_STREAM;
+   tpl.bind = PIPE_BIND_RENDER_TARGET | PIPE_BIND_SAMPLER_VIEW;
+   return screen->resource_create(screen, &tpl);
+}
+
+static void *
+get_stencil_resolve_vs(struct d3d12_context *ctx)
+{
+   if (ctx->stencil_resolve_vs)
+      return ctx->stencil_resolve_vs;
+
+   nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_VERTEX,
+                                                  &d3d12_screen(ctx->base.screen)->nir_options,
+                                                  "linear_blit_vs");
+
+   const struct glsl_type *vec4 = glsl_vec4_type();
+   nir_variable *pos_in = nir_variable_create(b.shader, nir_var_shader_in,
+                                              vec4, "pos");
+
+   nir_variable *pos_out = nir_variable_create(b.shader, nir_var_shader_out,
+                                               vec4, "gl_Position");
+   pos_out->data.location = VARYING_SLOT_POS;
+
+   nir_store_var(&b, pos_out, nir_load_var(&b, pos_in), 0xf);
+
+   struct pipe_shader_state state = {};
+   state.type = PIPE_SHADER_IR_NIR;
+   state.ir.nir = b.shader;
+   ctx->stencil_resolve_vs = ctx->base.create_vs_state(&ctx->base, &state);
+
+   return ctx->stencil_resolve_vs;
+}
+
+static void *
+get_stencil_resolve_fs(struct d3d12_context *ctx, bool no_flip)
+{
+   if (!no_flip && ctx->stencil_resolve_fs)
+      return ctx->stencil_resolve_fs;
+
+   if (no_flip && ctx->stencil_resolve_fs_no_flip)
+      return ctx->stencil_resolve_fs_no_flip;
+
+   nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_FRAGMENT,
+                                                  &d3d12_screen(ctx->base.screen)->nir_options,
+                                                  no_flip ? "stencil_resolve_fs_no_flip" : "stencil_resolve_fs");
+
+   nir_variable *stencil_out = nir_variable_create(b.shader,
+                                                   nir_var_shader_out,
+                                                   glsl_uint_type(),
+                                                   "stencil_out");
+   stencil_out->data.location = FRAG_RESULT_COLOR;
+
+   const struct glsl_type *sampler_type =
+      glsl_sampler_type(GLSL_SAMPLER_DIM_MS, false, false, GLSL_TYPE_UINT);
+   nir_variable *sampler = nir_variable_create(b.shader, nir_var_uniform,
+                                               sampler_type, "stencil_tex");
+   sampler->data.binding = 0;
+   sampler->data.explicit_binding = true;
+
+   nir_def *tex_deref = &nir_build_deref_var(&b, sampler)->def;
+
+   nir_variable *pos_in = nir_variable_create(b.shader, nir_var_shader_in,
+                                              glsl_vec4_type(), "pos");
+   pos_in->data.location = VARYING_SLOT_POS; // VARYING_SLOT_VAR0?
+   nir_def *pos = nir_load_var(&b, pos_in);
+
+   nir_def *pos_src;
+
+   if (no_flip)
+      pos_src = pos;
+   else {
+      nir_tex_instr *txs = nir_tex_instr_create(b.shader, 1);
+      txs->op = nir_texop_txs;
+      txs->sampler_dim = GLSL_SAMPLER_DIM_MS;
+      txs->src[0] = nir_tex_src_for_ssa(nir_tex_src_texture_deref, tex_deref);
+      txs->is_array = false;
+      txs->dest_type = nir_type_int;
+
+      nir_def_init(&txs->instr, &txs->def, 2, 32);
+      nir_builder_instr_insert(&b, &txs->instr);
+
+      pos_src = nir_vec4(&b,
+                         nir_channel(&b, pos, 0),
+                         /*Height - pos_dest.y - 1*/
+                         nir_fsub(&b,
+                                  nir_fsub(&b,
+                                           nir_channel(&b, nir_i2f32(&b, &txs->def), 1),
+                                           nir_channel(&b, pos, 1)),
+                                  nir_imm_float(&b, 1.0)),
+                         nir_channel(&b, pos, 2),
+                         nir_channel(&b, pos, 3));
+   }
+
+   nir_tex_instr *tex = nir_tex_instr_create(b.shader, 3);
+   tex->sampler_dim = GLSL_SAMPLER_DIM_MS;
+   tex->op = nir_texop_txf_ms;
+   tex->src[0] = nir_tex_src_for_ssa(nir_tex_src_coord,
+                                     nir_trim_vector(&b, nir_f2i32(&b, pos_src), 2));
+   tex->src[1] = nir_tex_src_for_ssa(nir_tex_src_ms_index, nir_imm_int(&b, 0)); /* just use first sample */
+   tex->src[2] = nir_tex_src_for_ssa(nir_tex_src_texture_deref, tex_deref);
+   tex->dest_type = nir_type_uint32;
+   tex->is_array = false;
+   tex->coord_components = 2;
+
+   nir_def_init(&tex->instr, &tex->def, 4, 32);
+   nir_builder_instr_insert(&b, &tex->instr);
+
+   nir_store_var(&b, stencil_out, nir_channel(&b, &tex->def, 1), 0x1);
+
+   struct pipe_shader_state state = {};
+   state.type = PIPE_SHADER_IR_NIR;
+   state.ir.nir = b.shader;
+   void *result;
+   if (no_flip) {
+      result = ctx->base.create_fs_state(&ctx->base, &state);
+      ctx->stencil_resolve_fs_no_flip = result;
+   } else {
+      result = ctx->base.create_fs_state(&ctx->base, &state);
+      ctx->stencil_resolve_fs = result;
+   }
+
+   return result;
+}
+
+static void *
+get_sampler_state(struct d3d12_context *ctx)
+{
+   if (ctx->sampler_state)
+      return ctx->sampler_state;
+
+   struct pipe_sampler_state state;
+   memset(&state, 0, sizeof(state));
+   state.wrap_s = PIPE_TEX_WRAP_CLAMP_TO_EDGE;
+   state.wrap_t = PIPE_TEX_WRAP_CLAMP_TO_EDGE;
+   state.wrap_r = PIPE_TEX_WRAP_CLAMP_TO_EDGE;
+
+   return ctx->sampler_state = ctx->base.create_sampler_state(&ctx->base, &state);
+}
+
+static struct pipe_resource *
+resolve_stencil_to_temp(struct d3d12_context *ctx,
+                        const struct pipe_blit_info *info)
+{
+   struct pipe_context *pctx = &ctx->base;
+   struct pipe_resource *tmp = create_tmp_resource(pctx->screen, info);
+   if (!tmp) {
+      debug_printf("D3D12: failed to create stencil-resolve temp-resource\n");
+      return NULL;
+   }
+   assert(tmp->nr_samples < 2);
+
+   /* resolve stencil into tmp */
+   struct pipe_surface dst_tmpl;
+   util_blitter_default_dst_texture(&dst_tmpl, tmp, 0, 0);
+   dst_tmpl.format = tmp->format;
+
+   struct pipe_sampler_view src_templ, *src_view;
+   util_blitter_default_src_texture(ctx->blitter, &src_templ,
+                                    info->src.resource, info->src.level);
+   src_templ.format = util_format_stencil_only(info->src.format);
+   src_view = pctx->create_sampler_view(pctx, info->src.resource, &src_templ);
+
+   void *sampler_state = get_sampler_state(ctx);
+
+   util_blit_save_state(ctx);
+   pctx->set_sampler_views(pctx, MESA_SHADER_FRAGMENT, 0, 1, 0, &src_view);
+   pctx->bind_sampler_states(pctx, MESA_SHADER_FRAGMENT, 0, 1, &sampler_state);
+   util_blitter_custom_shader(ctx->blitter, &dst_tmpl,
+                              (uint16_t)pipe_surface_width(&dst_tmpl),
+                              (uint16_t)pipe_surface_height(&dst_tmpl),
+                              get_stencil_resolve_vs(ctx),
+                              get_stencil_resolve_fs(ctx, info->src.box.height == info->dst.box.height));
+   util_blitter_restore_textures(ctx->blitter);
+   pipe_sampler_view_reference(&src_view, NULL);
+   return tmp;
+}
+
+static void
+blit_resolve_stencil(struct d3d12_context *ctx,
+                     const struct pipe_blit_info *info)
+{
+   assert(info->mask & PIPE_MASK_S);
+
+   if (D3D12_DEBUG_BLIT & d3d12_debug)
+      debug_printf("D3D12 BLIT: blit_resolve_stencil\n");
+
+   if (info->mask & PIPE_MASK_Z) {
+      /* resolve depth into dst */
+      struct pipe_blit_info new_info = *info;
+      new_info.mask = PIPE_MASK_Z;
+
+      if (resolve_supported(&new_info))
+         blit_resolve(ctx, &new_info);
+      else
+         util_blit(ctx, &new_info);
+   }
+
+   struct pipe_resource *tmp = resolve_stencil_to_temp(ctx, info);
+
+
+   /* copy resolved stencil into dst */
+   struct d3d12_resource *dst = d3d12_resource(info->dst.resource);
+   d3d12_transition_subresources_state(ctx, d3d12_resource(tmp),
+                                       0, 1, 0, 1, 0, 1,
+                                       D3D12_RESOURCE_STATE_COPY_SOURCE,
+                                       D3D12_TRANSITION_FLAG_NONE);
+   d3d12_transition_subresources_state(ctx, dst,
+                                       0, 1, 0, 1, 1, 1,
+                                       D3D12_RESOURCE_STATE_COPY_DEST,
+                                       D3D12_TRANSITION_FLAG_INVALIDATE_BINDINGS);
+   d3d12_apply_resource_states(ctx, false);
+
+   struct d3d12_batch *batch = d3d12_current_batch(ctx);
+   d3d12_batch_reference_resource(batch, d3d12_resource(tmp), false);
+   d3d12_batch_reference_resource(batch, dst, true);
+
+   D3D12_BOX src_box;
+   src_box.left = src_box.top = src_box.front = 0;
+   src_box.right = tmp->width0;
+   src_box.bottom = tmp->height0;
+   src_box.back = tmp->depth0;
+
+   D3D12_TEXTURE_COPY_LOCATION src_loc;
+   src_loc.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX;
+   src_loc.SubresourceIndex = 0;
+   src_loc.pResource = d3d12_resource_resource(d3d12_resource(tmp));
+
+   D3D12_TEXTURE_COPY_LOCATION dst_loc;
+   dst_loc.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX;
+   dst_loc.SubresourceIndex = 1;
+   dst_loc.pResource = d3d12_resource_resource(dst);
+
+   ctx->cmdlist->CopyTextureRegion(&dst_loc, info->dst.box.x,
+                                   info->dst.box.y, info->dst.box.z,
+                                   &src_loc, &src_box);
+   ctx->has_commands = true;
+
+   pipe_resource_reference(&tmp, NULL);
+}
+
+static bool
+replicate_stencil_supported(struct d3d12_context *ctx,
+                            const struct pipe_blit_info *info)
+{
+   if (!util_format_is_depth_or_stencil(info->src.format) ||
+       !(info->mask & PIPE_MASK_S))
+      return false;
+
+   if (info->mask & PIPE_MASK_Z) {
+      struct pipe_blit_info new_info = *info;
+      new_info.mask = PIPE_MASK_Z;
+      if (!util_blitter_is_blit_supported(ctx->blitter, &new_info))
+         return false;
+   }
+
+   return true;
+}
+
+static void
+blit_replicate_stencil(struct d3d12_context *ctx,
+                       const struct pipe_blit_info *info)
+{
+   assert(info->mask & PIPE_MASK_S);
+
+   if (D3D12_DEBUG_BLIT & d3d12_debug)
+      debug_printf("D3D12 BLIT: blit_replicate_stencil\n");
+
+   if (info->mask & PIPE_MASK_Z) {
+      /* resolve depth into dst */
+      struct pipe_blit_info new_info = *info;
+      new_info.mask = PIPE_MASK_Z;
+      util_blit(ctx, &new_info);
+   }
+
+   struct pipe_surface dst_templ;
+   util_blitter_default_dst_texture(&dst_templ, info->dst.resource,
+                           info->dst.level, info->dst.box.z);
+
+   util_blit_save_state(ctx);
+   util_blitter_clear_depth_stencil(ctx->blitter, &dst_templ, PIPE_CLEAR_STENCIL,
+                                    0, 0, info->dst.box.x, info->dst.box.y,
+                                    info->dst.box.width, info->dst.box.height);
+   util_blit_save_state(ctx);
+   util_blitter_stencil_fallback(ctx->blitter, info->dst.resource,
+                                 info->dst.level,
+                                 &info->dst.box,
+                                 info->src.resource,
+                                 info->src.level,
+                                 &info->src.box,
+                                 info->scissor_enable ? &info->scissor : NULL);
+}
+
+void
+d3d12_blit(struct pipe_context *pctx,
+           const struct pipe_blit_info *info)
+{
+   struct d3d12_context *ctx = d3d12_context(pctx);
+
+   if (!info->render_condition_enable && ctx->current_predication) {
+      if (D3D12_DEBUG_BLIT & d3d12_debug)
+         debug_printf("D3D12 BLIT: Disable predication\n");
+      ctx->cmdlist->SetPredication(nullptr, 0, D3D12_PREDICATION_OP_EQUAL_ZERO);
+   }
+
+   if (D3D12_DEBUG_BLIT & d3d12_debug) {
+      debug_printf("D3D12 BLIT: from %s@%d msaa:%d %dx%dx%d + %dx%dx%d\n",
+                   util_format_name(info->src.format), info->src.level,
+                   info->src.resource->nr_samples,
+                   info->src.box.x, info->src.box.y, info->src.box.z,
+                   info->src.box.width, info->src.box.height, info->src.box.depth);
+      debug_printf("            to   %s@%d msaa:%d %dx%dx%d + %dx%dx%d ",
+                   util_format_name(info->dst.format), info->dst.level,
+                   info->dst.resource->nr_samples,
+                   info->dst.box.x, info->dst.box.y, info->dst.box.z,
+                   info->dst.box.width, info->dst.box.height, info->dst.box.depth);
+      debug_printf("| flags %s%s%s\n",
+                   info->render_condition_enable ? "cond " : "",
+                   info->scissor_enable ? "scissor " : "",
+                   info->alpha_blend ? "blend" : "");
+   }
+
+   if (is_same_resource(info))
+      blit_same_resource(ctx, info);
+   else if (is_resolve(info)) {
+      if (resolve_supported(info))
+         blit_resolve(ctx, info);
+      else if (util_blitter_is_blit_supported(ctx->blitter, info))
+         util_blit(ctx, info);
+      else if (resolve_stencil_supported(ctx, info))
+         blit_resolve_stencil(ctx, info);
+      else
+         debug_printf("D3D12: resolve unsupported %s -> %s\n",
+                    util_format_short_name(info->src.resource->format),
+                    util_format_short_name(info->dst.resource->format));
+   } else if (direct_copy_supported(d3d12_screen(pctx->screen), info,
+                                    ctx->current_predication != nullptr))
+      d3d12_direct_copy(ctx, d3d12_resource(info->dst.resource),
+                        info->dst.level, &info->dst.box,
+                        d3d12_resource(info->src.resource),
+                        info->src.level, &info->src.box, info->mask);
+   else if (util_blitter_is_blit_supported(ctx->blitter, info))
+      util_blit(ctx, info);
+   else if (replicate_stencil_supported(ctx, info))
+      blit_replicate_stencil(ctx, info);
+   else
+      debug_printf("D3D12: blit unsupported %s -> %s\n",
+                 util_format_short_name(info->src.resource->format),
+                 util_format_short_name(info->dst.resource->format));
+
+   if (d3d12_screen(pctx->screen)->max_feature_level >= D3D_FEATURE_LEVEL_11_0) {
+      if (!info->render_condition_enable && ctx->current_predication) {
+         d3d12_enable_predication(ctx);
+         if (D3D12_DEBUG_BLIT & d3d12_debug)
+            debug_printf("D3D12 BLIT: Re-enable predication\n");
+      }
+   }
+}
+
+void
+d3d12_context_blit_init(struct pipe_context *ctx)
+{
+   ctx->blit = d3d12_blit;
+}

src/gallium/drivers/d3d12/d3d12_blit.h

@@ -0,0 +1,54 @@
+/*
+ * Copyright © Microsoft Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifndef D3D12_BLIT_H
+#define D3D12_BLIT_H
+
+struct d3d12_context;
+struct d3d12_resource;
+struct pipe_box;
+
+void
+d3d12_context_blit_init(struct pipe_context *ctx);
+
+void
+d3d12_context_copy_init(struct pipe_context *ctx);
+
+void
+d3d12_direct_copy(struct d3d12_context *ctx,
+                  struct d3d12_resource *dst,
+                  unsigned dst_level,
+                  const struct pipe_box *pdst_box,
+                  struct d3d12_resource *src,
+                  unsigned src_level,
+                  const struct pipe_box *psrc_box, unsigned mask);
+
+struct pipe_resource *
+create_staging_resource(struct d3d12_context *ctx,
+                        struct d3d12_resource *src,
+                        unsigned src_level,
+                        const struct pipe_box *src_box,
+                        struct pipe_box *dst_box,
+                        unsigned mask);
+
+#endif // D3D12_BLIT_H

src/gallium/drivers/d3d12/d3d12_bufmgr.cpp

@@ -0,0 +1,472 @@
+/*
+ * Copyright © Microsoft Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#include "d3d12_bufmgr.h"
+#include "d3d12_context.h"
+#include "d3d12_fence.h"
+#include "d3d12_format.h"
+#include "d3d12_screen.h"
+
+#include "pipebuffer/pb_buffer.h"
+#include "pipebuffer/pb_bufmgr.h"
+
+#include "util/format/u_format.h"
+#include "util/set.h"
+#include "util/u_memory.h"
+
+#include <dxguids/dxguids.h>
+
+struct d3d12_bufmgr {
+   struct pb_manager base;
+
+   struct d3d12_screen *screen;
+};
+
+extern const struct pb_vtbl d3d12_buffer_vtbl;
+
+static inline struct d3d12_bufmgr *
+d3d12_bufmgr(struct pb_manager *mgr)
+{
+   assert(mgr);
+
+   return (struct d3d12_bufmgr *)mgr;
+}
+
+static void
+describe_direct_bo(char *buf, struct d3d12_bo *ptr)
+{
+   sprintf(buf, "d3d12_bo<direct,%p,0x%x>", ptr->res, (unsigned)ptr->estimated_size);
+}
+
+static void
+describe_suballoc_bo(char *buf, struct d3d12_bo *ptr)
+{
+   char res[128];
+   uint64_t offset;
+   d3d12_bo *base = d3d12_bo_get_base(ptr, &offset);
+   describe_direct_bo(res, base);
+   sprintf(buf, "d3d12_bo<suballoc<%s>,0x%x,0x%x>", res,
+           (unsigned)ptr->buffer->base.size, (unsigned)offset);
+}
+
+void
+d3d12_debug_describe_bo(char *buf, struct d3d12_bo *ptr)
+{
+   if (ptr->buffer)
+      describe_suballoc_bo(buf, ptr);
+   else
+      describe_direct_bo(buf, ptr);
+}
+
+struct d3d12_bo *
+d3d12_bo_wrap_res(struct d3d12_screen *screen, ID3D12Resource *res, enum d3d12_residency_status residency)
+{
+   struct d3d12_bo *bo;
+
+   bo = MALLOC_STRUCT(d3d12_bo);
+   if (!bo)
+      return NULL;
+   memset(bo, 0, offsetof(d3d12_bo, local_context_states));
+
+   D3D12_RESOURCE_DESC desc = GetDesc(res);
+   unsigned array_size = desc.Dimension == D3D12_RESOURCE_DIMENSION_TEXTURE3D ? 1 : desc.DepthOrArraySize;
+   unsigned total_subresources = desc.MipLevels * array_size * d3d12_non_opaque_plane_count(desc.Format);
+   bool supports_simultaneous_access = d3d12_resource_supports_simultaneous_access(&desc);
+
+   pipe_reference_init(&bo->reference, 1);
+   bo->screen = screen;
+   bo->res = res;
+   bo->unique_id = p_atomic_inc_return(&screen->resource_id_generator);
+   if (!supports_simultaneous_access)
+      d3d12_resource_state_init(&bo->global_state, total_subresources, false);
+
+   bo->residency_status = residency;
+   bo->last_used_timestamp = 0;
+   desc.Flags &= ~D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS;
+   bo->estimated_size = GetResourceAllocationInfo(screen->dev, 0, 1, &desc).SizeInBytes;
+   if (residency == d3d12_resident) {
+      mtx_lock(&screen->submit_mutex);
+      list_add(&bo->residency_list_entry, &screen->residency_list);
+      mtx_unlock(&screen->submit_mutex);
+   }
+
+   return bo;
+}
+
+struct d3d12_bo *
+d3d12_bo_new(struct d3d12_screen *screen, uint64_t size, const pb_desc *pb_desc)
+{
+   ID3D12Device *dev = screen->dev;
+   ID3D12Resource *res;
+
+   D3D12_RESOURCE_DESC res_desc;
+   res_desc.Dimension = D3D12_RESOURCE_DIMENSION_BUFFER;
+   res_desc.Format = DXGI_FORMAT_UNKNOWN;
+   res_desc.Alignment = 0;
+   res_desc.Width = size;
+   res_desc.Height = 1;
+   res_desc.DepthOrArraySize = 1;
+   res_desc.MipLevels = 1;
+   res_desc.SampleDesc.Count = 1;
+   res_desc.SampleDesc.Quality = 0;
+   res_desc.Flags = (screen->max_feature_level >= D3D_FEATURE_LEVEL_11_0) ? D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS : D3D12_RESOURCE_FLAG_NONE;
+   res_desc.Layout = D3D12_TEXTURE_LAYOUT_ROW_MAJOR;
+
+   D3D12_HEAP_TYPE heap_type = D3D12_HEAP_TYPE_DEFAULT;
+   if (pb_desc->usage & PB_USAGE_CPU_READ)
+      heap_type = D3D12_HEAP_TYPE_READBACK;
+   else if (pb_desc->usage & PB_USAGE_CPU_WRITE)
+      heap_type = D3D12_HEAP_TYPE_UPLOAD;
+
+   D3D12_HEAP_FLAGS heap_flags = screen->support_create_not_resident ?
+      D3D12_HEAP_FLAG_CREATE_NOT_RESIDENT : D3D12_HEAP_FLAG_NONE;
+   enum d3d12_residency_status init_residency = screen->support_create_not_resident ?
+      d3d12_evicted : d3d12_resident;
+
+   D3D12_HEAP_PROPERTIES heap_pris = GetCustomHeapProperties(dev, heap_type);
+   d3d12_screen_reclaim_completed(screen);
+   HRESULT hres;
+   do {
+      hres = dev->CreateCommittedResource(&heap_pris,
+                                          heap_flags,
+                                          &res_desc,
+                                          D3D12_RESOURCE_STATE_COMMON,
+                                          NULL,
+                                          IID_PPV_ARGS(&res));
+   } while (hres == E_OUTOFMEMORY && d3d12_screen_reclaim_one(screen));
+
+   if (FAILED(hres))
+      return NULL;
+
+   return d3d12_bo_wrap_res(screen, res, init_residency);
+}
+
+struct d3d12_bo *
+d3d12_bo_wrap_buffer(struct d3d12_screen *screen, struct pb_buffer *buf)
+{
+   struct d3d12_bo *bo;
+
+   bo = MALLOC_STRUCT(d3d12_bo);
+   if (!bo)
+      return NULL;
+   memset(bo, 0, offsetof(d3d12_bo, local_context_states));
+
+   pipe_reference_init(&bo->reference, 1);
+   bo->screen = screen;
+   bo->buffer = buf;
+   bo->unique_id = p_atomic_inc_return(&screen->resource_id_generator);
+   bo->residency_status = d3d12_evicted;
+
+   return bo;
+}
+
+void
+d3d12_bo_unreference(struct d3d12_bo *bo)
+{
+   if (bo == NULL)
+      return;
+
+   assert(pipe_is_referenced(&bo->reference));
+
+   if (pipe_reference_described(&bo->reference, NULL,
+                                (debug_reference_descriptor)
+                                d3d12_debug_describe_bo)) {
+      if (bo->buffer)
+         pb_reference(&bo->buffer, NULL);
+
+      mtx_lock(&bo->screen->submit_mutex);
+
+      if (bo->residency_status == d3d12_resident)
+         list_del(&bo->residency_list_entry);
+
+      /* MSVC's offsetof fails when the name is ambiguous between struct and function */
+      typedef struct d3d12_context d3d12_context_type;
+      list_for_each_entry(d3d12_context_type, ctx, &bo->screen->context_list, context_list_entry) {
+         if (ctx->id == D3D12_CONTEXT_NO_ID) {
+            util_dynarray_append_typed(&ctx->recently_destroyed_bos, uint64_t, bo->unique_id);
+         } else if (bo->local_context_state_mask & (1u << ctx->id)) {
+            _mesa_set_remove_key(ctx->local_state_bos, bo);
+         }
+      }
+
+      mtx_unlock(&bo->screen->submit_mutex);
+
+      d3d12_resource_state_cleanup(&bo->global_state);
+      if (bo->res)
+         bo->res->Release();
+
+      uint64_t mask = bo->local_context_state_mask;
+      while (mask) {
+         int ctxid = u_bit_scan64(&mask);
+         d3d12_destroy_context_state_table_entry(&bo->local_context_states[ctxid]);
+      }
+
+      FREE(bo);
+   }
+}
+
+bool
+d3d12_screen_reclaim_completed(struct d3d12_screen *screen)
+{
+   uint64_t completed = screen->fence->GetCompletedValue();
+   struct list_head retired;
+   list_inithead(&retired);
+
+   mtx_lock(&screen->pending_free_lock);
+   list_for_each_entry_safe(struct d3d12_pending_free_entry, entry,
+                            &screen->pending_free_list, link) {
+      if (entry->fence_value > completed)
+         break;
+      list_del(&entry->link);
+      list_addtail(&entry->link, &retired);
+   }
+   mtx_unlock(&screen->pending_free_lock);
+
+   bool dropped = !list_is_empty(&retired);
+   list_for_each_entry_safe(struct d3d12_pending_free_entry, entry, &retired, link) {
+      d3d12_bo_unreference(entry->bo);
+      FREE(entry);
+   }
+   return dropped;
+}
+
+bool
+d3d12_screen_reclaim_one(struct d3d12_screen *screen)
+{
+   uint64_t target = 0;
+   bool have_target = false;
+
+   mtx_lock(&screen->pending_free_lock);
+   if (!list_is_empty(&screen->pending_free_list)) {
+      struct d3d12_pending_free_entry *head =
+         list_first_entry(&screen->pending_free_list,
+                          struct d3d12_pending_free_entry, link);
+      target = head->fence_value;
+      have_target = true;
+   }
+   mtx_unlock(&screen->pending_free_lock);
+
+   if (!have_target)
+      return false;
+
+   if (screen->fence->GetCompletedValue() < target)
+      screen->fence->SetEventOnCompletion(target, nullptr);
+
+   d3d12_screen_reclaim_completed(screen);
+   return true;
+}
+
+void *
+d3d12_bo_map(struct d3d12_bo *bo, D3D12_RANGE *range)
+{
+   struct d3d12_bo *base_bo;
+   D3D12_RANGE offset_range = {0, 0};
+   uint64_t offset;
+   void *ptr;
+
+   base_bo = d3d12_bo_get_base(bo, &offset);
+
+   if (!range || range->Begin >= range->End) {
+      offset_range.Begin = static_cast<size_t>(offset);
+      offset_range.End = static_cast<size_t>(offset + d3d12_bo_get_size(bo));
+      range = &offset_range;
+   } else {
+      offset_range.Begin = static_cast<size_t>(range->Begin + offset);
+      offset_range.End = static_cast<size_t>(range->End + offset);
+      range = &offset_range;
+   }
+
+   if (FAILED(base_bo->res->Map(0, range, &ptr)))
+      return NULL;
+
+   return (uint8_t *)ptr + (range ? range->Begin : 0);
+}
+
+void
+d3d12_bo_unmap(struct d3d12_bo *bo, D3D12_RANGE *range)
+{
+   struct d3d12_bo *base_bo;
+   D3D12_RANGE offset_range = {0, 0};
+   uint64_t offset;
+
+   base_bo = d3d12_bo_get_base(bo, &offset);
+
+   if (!range || range->Begin >= range->End) {
+      offset_range.Begin = static_cast<size_t>(offset);
+      offset_range.End = static_cast<size_t>(offset + d3d12_bo_get_size(bo));
+      range = &offset_range;
+   } else {
+      offset_range.Begin = static_cast<size_t>(range->Begin + offset);
+      offset_range.End = static_cast<size_t>(range->End + offset);
+      range = &offset_range;
+   }
+
+   base_bo->res->Unmap(0, range);
+}
+
+static void
+d3d12_buffer_destroy(void *winsys, struct pb_buffer *pbuf)
+{
+   struct d3d12_buffer *buf = d3d12_buffer(pbuf);
+
+   if (buf->map)
+      d3d12_bo_unmap(buf->bo, &buf->range);
+   d3d12_bo_unreference(buf->bo);
+   FREE(buf);
+}
+
+static void *
+d3d12_buffer_map(struct pb_buffer *pbuf,
+                 enum pb_usage_flags flags,
+                 void *flush_ctx)
+{
+   return d3d12_buffer(pbuf)->map;
+}
+
+static void
+d3d12_buffer_unmap(struct pb_buffer *pbuf)
+{
+}
+
+static void
+d3d12_buffer_get_base_buffer(struct pb_buffer *buf,
+                             struct pb_buffer **base_buf,
+                             pb_size *offset)
+{
+   *base_buf = buf;
+   *offset = 0;
+}
+
+static enum pipe_error
+d3d12_buffer_validate(struct pb_buffer *pbuf,
+                      struct pb_validate *vl,
+                      enum pb_usage_flags flags )
+{
+   /* Always pinned */
+   return PIPE_OK;
+}
+
+static void
+d3d12_buffer_fence(struct pb_buffer *pbuf,
+                   struct pipe_fence_handle *fence)
+{
+   if (!fence)
+      return;
+   struct d3d12_buffer *buf = d3d12_buffer(pbuf);
+   struct d3d12_fence *f = d3d12_fence(fence);
+   uint64_t offset;
+   struct d3d12_bo *base = d3d12_bo_get_base(buf->bo, &offset);
+   if (f->cmdqueue_fence != base->screen->fence)
+      return;
+   if (f->value > base->last_used_fence)
+      base->last_used_fence = f->value;
+}
+
+const struct pb_vtbl d3d12_buffer_vtbl = {
+   d3d12_buffer_destroy,
+   d3d12_buffer_map,
+   d3d12_buffer_unmap,
+   d3d12_buffer_validate,
+   d3d12_buffer_fence,
+   d3d12_buffer_get_base_buffer
+};
+
+static struct pb_buffer *
+d3d12_bufmgr_create_buffer(struct pb_manager *pmgr,
+                           pb_size size,
+                           const struct pb_desc *pb_desc)
+{
+   struct d3d12_bufmgr *mgr = d3d12_bufmgr(pmgr);
+   struct d3d12_buffer *buf;
+
+   buf = CALLOC_STRUCT(d3d12_buffer);
+   if (!buf)
+      return NULL;
+
+   pipe_reference_init(&buf->base.base.reference, 1);
+   buf->base.base.alignment_log2 = static_cast<uint8_t>(util_logbase2(pb_desc->alignment));
+   buf->base.base.usage = static_cast<uint16_t>(pb_desc->usage);
+   buf->base.vtbl = &d3d12_buffer_vtbl;
+   buf->base.base.size = size;
+   buf->range.Begin = 0;
+   buf->range.End = static_cast<size_t>(size);
+
+   buf->bo = d3d12_bo_new(mgr->screen, size, pb_desc);
+   if (!buf->bo) {
+      FREE(buf);
+      return NULL;
+   }
+
+   if (pb_desc->usage & PB_USAGE_CPU_READ_WRITE) {
+      buf->map = d3d12_bo_map(buf->bo, &buf->range);
+      if (!buf->map) {
+         d3d12_bo_unreference(buf->bo);
+         FREE(buf);
+         return NULL;
+      }
+   }
+
+   return &buf->base;
+}
+
+static void
+d3d12_bufmgr_flush(struct pb_manager *mgr)
+{
+   /* No-op */
+}
+
+static void
+d3d12_bufmgr_destroy(struct pb_manager *_mgr)
+{
+   struct d3d12_bufmgr *mgr = d3d12_bufmgr(_mgr);
+   FREE(mgr);
+}
+
+static bool
+d3d12_bufmgr_is_buffer_busy(struct pb_manager *_mgr, struct pb_buffer *_buf)
+{
+   struct d3d12_bufmgr *mgr = d3d12_bufmgr(_mgr);
+   struct d3d12_buffer *buf = d3d12_buffer(_buf);
+   uint64_t offset;
+   struct d3d12_bo *base = d3d12_bo_get_base(buf->bo, &offset);
+   return base->last_used_fence > mgr->screen->fence->GetCompletedValue();
+}
+
+struct pb_manager *
+d3d12_bufmgr_create(struct d3d12_screen *screen)
+{
+   struct d3d12_bufmgr *mgr;
+
+   mgr = CALLOC_STRUCT(d3d12_bufmgr);
+   if (!mgr)
+      return NULL;
+
+   mgr->base.destroy = d3d12_bufmgr_destroy;
+   mgr->base.create_buffer = d3d12_bufmgr_create_buffer;
+   mgr->base.flush = d3d12_bufmgr_flush;
+   mgr->base.is_buffer_busy = d3d12_bufmgr_is_buffer_busy;
+
+   mgr->screen = screen;
+
+   return &mgr->base;
+}

src/gallium/drivers/d3d12/d3d12_bufmgr.h

@@ -0,0 +1,176 @@
+/*
+ * Copyright © Microsoft Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifndef D3D12_BUFMGR_H
+#define D3D12_BUFMGR_H
+
+#include "pipebuffer/pb_buffer.h"
+#include "util/u_atomic.h"
+#include "util/list.h"
+
+#include "d3d12_common.h"
+#include "d3d12_resource_state.h"
+
+struct d3d12_bufmgr;
+struct d3d12_screen;
+struct pb_manager;
+
+enum d3d12_residency_status {
+   d3d12_evicted,
+   d3d12_resident,
+   d3d12_permanently_resident,
+};
+
+enum batch_bo_reference_state : uint8_t {
+   batch_bo_reference_none = 0,
+   batch_bo_reference_read = (1 << 0),
+   batch_bo_reference_written = (1 << 1),
+};
+
+struct d3d12_bo {
+   struct pipe_reference reference;
+   struct d3d12_screen *screen;
+   ID3D12Resource *res;
+   struct pb_buffer *buffer;
+   struct d3d12_resource_state global_state;
+
+   /* Used as a key in per-context resource state maps,
+    * to avoid needing to lock them for single-threaded lookups to
+    * protect against resource destruction.
+    */
+   uint64_t unique_id;
+
+#ifndef NDEBUG
+   bool is_front_buffer;
+#endif
+
+   struct list_head residency_list_entry;
+   uint64_t estimated_size;
+   int64_t last_used_timestamp;
+   uint64_t last_used_fence;
+   uint64_t last_used_periodic_notification_index;
+   enum d3d12_residency_status residency_status;
+   uint16_t local_needs_resolve_state;
+
+   unsigned local_context_state_mask;
+   uint8_t local_reference_mask[16];
+
+   d3d12_context_state_table_entry local_context_states[16];
+   uint8_t local_reference_state[16][8];
+};
+
+struct d3d12_buffer {
+   struct pb_buffer base;
+
+   struct d3d12_bo *bo;
+   D3D12_RANGE range;
+   void *map;
+};
+
+static inline struct d3d12_buffer *
+d3d12_buffer(struct pb_buffer *buf)
+{
+   assert(buf);
+   return (struct d3d12_buffer *)buf;
+}
+
+static inline struct d3d12_bo *
+d3d12_bo_get_base(struct d3d12_bo *bo, uint64_t *offset)
+{
+   if (bo->buffer) {
+      struct pb_buffer *base_buffer;
+      pb_get_base_buffer(bo->buffer, &base_buffer, offset);
+      return d3d12_buffer(base_buffer)->bo;
+   } else {
+      *offset = 0;
+      return bo;
+   }
+}
+
+static inline uint64_t
+d3d12_bo_get_size(struct d3d12_bo *bo)
+{
+   if (bo->buffer)
+      return bo->buffer->base.size;
+   else
+      return GetDesc(bo->res).Width;
+}
+
+static inline bool
+d3d12_bo_is_suballocated(struct d3d12_bo *bo)
+{
+   struct d3d12_bo *base_bo;
+   uint64_t offset;
+
+   if (!bo->buffer)
+      return false;
+
+   base_bo = d3d12_bo_get_base(bo, &offset);
+   return d3d12_bo_get_size(base_bo) != d3d12_bo_get_size(bo);
+}
+
+struct d3d12_bo *
+d3d12_bo_new(struct d3d12_screen *screen, uint64_t size, uint64_t alignment);
+
+struct d3d12_bo *
+d3d12_bo_wrap_res(struct d3d12_screen *screen, ID3D12Resource *res, enum d3d12_residency_status residency);
+
+struct d3d12_bo *
+d3d12_bo_wrap_buffer(struct d3d12_screen *screen, struct pb_buffer *buf);
+
+void
+d3d12_debug_describe_bo(char* buf, struct d3d12_bo* ptr);
+
+static inline void
+d3d12_bo_reference(struct d3d12_bo *bo)
+{
+   pipe_reference_described(NULL, &bo->reference,
+                            (debug_reference_descriptor)
+                            d3d12_debug_describe_bo);
+}
+
+void
+d3d12_bo_unreference(struct d3d12_bo *bo);
+
+void *
+d3d12_bo_map(struct d3d12_bo *bo, D3D12_RANGE *range);
+
+void
+d3d12_bo_unmap(struct d3d12_bo *bo, D3D12_RANGE *range);
+
+struct d3d12_pending_free_entry {
+   struct list_head link;
+   struct d3d12_bo *bo;
+   uint64_t fence_value;
+};
+
+bool
+d3d12_screen_reclaim_completed(struct d3d12_screen *screen);
+
+bool
+d3d12_screen_reclaim_one(struct d3d12_screen *screen);
+
+struct pb_manager *
+d3d12_bufmgr_create(struct d3d12_screen *screen);
+
+#endif

src/gallium/drivers/d3d12/d3d12_cmd_signature.cpp

@@ -0,0 +1,122 @@
+/*
+ * Copyright © Microsoft Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#include "d3d12_cmd_signature.h"
+#include "d3d12_compiler.h"
+#include "d3d12_screen.h"
+
+#include "util/u_memory.h"
+
+#include <dxguids/dxguids.h>
+
+struct d3d12_cmd_signature {
+   struct d3d12_cmd_signature_key key;
+   ID3D12CommandSignature *sig;
+};
+
+static ID3D12CommandSignature *
+create_cmd_signature(struct d3d12_context *ctx, const struct d3d12_cmd_signature_key *key)
+{
+   D3D12_COMMAND_SIGNATURE_DESC cmd_sig_desc = {};
+   D3D12_INDIRECT_ARGUMENT_DESC indirect_args[2] = {};
+
+   unsigned num_args = 0;
+   if (key->draw_or_dispatch_params) {
+      indirect_args[num_args].Type = D3D12_INDIRECT_ARGUMENT_TYPE_CONSTANT;
+      indirect_args[num_args].Constant.RootParameterIndex = key->params_root_const_param;
+      indirect_args[num_args].Constant.DestOffsetIn32BitValues = key->params_root_const_offset;
+      indirect_args[num_args++].Constant.Num32BitValuesToSet = key->compute ? 3 : 4;
+   }
+
+   indirect_args[num_args++].Type = key->compute ? D3D12_INDIRECT_ARGUMENT_TYPE_DISPATCH :
+      key->indexed ? D3D12_INDIRECT_ARGUMENT_TYPE_DRAW_INDEXED :
+      D3D12_INDIRECT_ARGUMENT_TYPE_DRAW;
+   cmd_sig_desc.ByteStride = key->multi_draw_stride;
+   cmd_sig_desc.NumArgumentDescs = num_args;
+   cmd_sig_desc.pArgumentDescs = indirect_args;
+
+   ID3D12CommandSignature *ret = nullptr;
+   d3d12_screen(ctx->base.screen)->dev->CreateCommandSignature(&cmd_sig_desc, key->root_sig,
+      IID_PPV_ARGS(&ret));
+   return ret;
+}
+
+ID3D12CommandSignature *
+d3d12_get_cmd_signature(struct d3d12_context *ctx,
+                            const struct d3d12_cmd_signature_key *key)
+{
+   struct hash_entry *entry = _mesa_hash_table_search(ctx->cmd_signature_cache, &key);
+   
+   if (!entry) {
+      struct d3d12_cmd_signature *data =
+         (struct d3d12_cmd_signature *)MALLOC(sizeof(struct d3d12_cmd_signature));
+      if (!data)
+         return NULL;
+
+      memcpy(&data->key, key, sizeof(*key));
+      data->sig = create_cmd_signature(ctx, key);
+      if (!data->sig) {
+         FREE(data);
+         return NULL;
+      }
+
+      entry = _mesa_hash_table_insert(ctx->cmd_signature_cache, &data->key, data);
+      assert(entry);
+   }
+
+   return ((struct d3d12_cmd_signature *)entry->data)->sig;
+}
+
+static uint32_t
+hash_cmd_signature_key(const void *key)
+{
+   return _mesa_hash_data(key, sizeof(struct d3d12_cmd_signature_key));
+}
+
+static bool
+equals_cmd_signature_key(const void *a, const void *b)
+{
+   return memcmp(a, b, sizeof(struct d3d12_cmd_signature_key)) == 0;
+}
+
+void
+d3d12_cmd_signature_cache_init(struct d3d12_context *ctx)
+{
+   ctx->cmd_signature_cache = _mesa_hash_table_create(NULL,
+                                                      hash_cmd_signature_key,
+                                                      equals_cmd_signature_key);
+}
+
+static void
+delete_entry(struct hash_entry *entry)
+{
+   struct d3d12_cmd_signature *data = (struct d3d12_cmd_signature *)entry->data;
+   data->sig->Release();
+   FREE(data);
+}
+
+void
+d3d12_cmd_signature_cache_destroy(struct d3d12_context *ctx)
+{
+   _mesa_hash_table_destroy(ctx->cmd_signature_cache, delete_entry);
+}

src/gallium/drivers/d3d12/d3d12_cmd_signature.h

@@ -0,0 +1,54 @@
+/*
+ * Copyright © Microsoft Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifndef D3D12_CMD_SIGNATURE_H
+#define D3D12_CMD_SIGNATURE_H
+
+#include "d3d12_context.h"
+
+struct d3d12_cmd_signature_key {
+   uint8_t compute:1;
+   uint8_t indexed:1;
+   uint8_t draw_or_dispatch_params:1;
+   /* 5 bits padding */
+   uint8_t params_root_const_param;
+   uint8_t params_root_const_offset;
+   /* 8 bits padding */
+
+   unsigned multi_draw_stride;
+
+   /* Will be zero if no constant/resource updates in the arg buffer */
+   ID3D12RootSignature *root_sig;
+};
+
+void
+d3d12_cmd_signature_cache_init(struct d3d12_context *ctx);
+
+void
+d3d12_cmd_signature_cache_destroy(struct d3d12_context *ctx);
+
+ID3D12CommandSignature *
+d3d12_get_cmd_signature(struct d3d12_context *ctx,
+                        const struct d3d12_cmd_signature_key *key);
+
+#endif

src/gallium/drivers/d3d12/d3d12_common.h

@@ -0,0 +1,156 @@
+/*
+ * Copyright © Yonggang Luo
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifndef D3D12_COMMON_STATE_H
+#define D3D12_COMMON_STATE_H
+
+#pragma once
+
+#include <wsl/winadapter.h>
+
+#define D3D12_IGNORE_SDK_LAYERS
+#ifndef _GAMING_XBOX
+#include <directx/d3d12.h>
+#include <directx/d3d12video.h>
+#elif defined(__cplusplus)
+#ifdef _GAMING_XBOX_SCARLETT
+#include <d3d12_xs.h>
+#include <d3d12video_xs.h>
+#else
+#include <d3d12_x.h>
+#include <d3d12video_x.h>
+#endif
+#ifdef IID_PPV_ARGS
+#undef IID_PPV_ARGS
+#endif
+#define IID_PPV_ARGS IID_GRAPHICS_PPV_ARGS
+#define D3D12_HEAP_FLAG_CREATE_NOT_RESIDENT (D3D12_HEAP_FLAGS) 0x800
+#endif /* _GAMING_XBOX */
+
+#ifndef D3D12_TEXTURE_DATA_PITCH_ALIGNMENT
+#define D3D12_TEXTURE_DATA_PITCH_ALIGNMENT D3D12XBOX_TEXTURE_DATA_PITCH_ALIGNMENT
+#endif /* D3D12_TEXTURE_DATA_PITCH_ALIGNMENT */
+
+#if defined(__cplusplus)
+#if !defined(_WIN32) || defined(_MSC_VER)
+inline D3D12_CPU_DESCRIPTOR_HANDLE
+GetCPUDescriptorHandleForHeapStart(ID3D12DescriptorHeap *heap)
+{
+   return heap->GetCPUDescriptorHandleForHeapStart();
+}
+inline D3D12_GPU_DESCRIPTOR_HANDLE
+GetGPUDescriptorHandleForHeapStart(ID3D12DescriptorHeap *heap)
+{
+   return heap->GetGPUDescriptorHandleForHeapStart();
+}
+D3D12_HEAP_DESC
+inline GetDesc(ID3D12Heap* heap)
+{
+   return heap->GetDesc();
+}
+inline D3D12_RESOURCE_DESC
+GetDesc(ID3D12Resource *res)
+{
+   return res->GetDesc();
+}
+inline D3D12_HEAP_PROPERTIES
+GetCustomHeapProperties(ID3D12Device *dev, D3D12_HEAP_TYPE type)
+{
+   return dev->GetCustomHeapProperties(0, type);
+}
+inline LUID
+GetAdapterLuid(ID3D12Device *dev)
+{
+   return dev->GetAdapterLuid();
+}
+inline D3D12_VIDEO_PROCESS_OUTPUT_STREAM_DESC
+GetOutputStreamDesc(ID3D12VideoProcessor *proc)
+{
+   return proc->GetOutputStreamDesc();
+}
+inline D3D12_RESOURCE_ALLOCATION_INFO
+GetResourceAllocationInfo(ID3D12Device *dev, UINT visibleMask, UINT numResourceDescs, const D3D12_RESOURCE_DESC *pResourceDescs)
+{
+   return dev->GetResourceAllocationInfo(visibleMask, numResourceDescs, pResourceDescs);
+}
+#else
+inline D3D12_CPU_DESCRIPTOR_HANDLE
+GetCPUDescriptorHandleForHeapStart(ID3D12DescriptorHeap *heap)
+{
+   D3D12_CPU_DESCRIPTOR_HANDLE ret;
+   heap->GetCPUDescriptorHandleForHeapStart(&ret);
+   return ret;
+}
+inline D3D12_GPU_DESCRIPTOR_HANDLE
+GetGPUDescriptorHandleForHeapStart(ID3D12DescriptorHeap *heap)
+{
+   D3D12_GPU_DESCRIPTOR_HANDLE ret;
+   heap->GetGPUDescriptorHandleForHeapStart(&ret);
+   return ret;
+}
+D3D12_HEAP_DESC
+inline GetDesc(ID3D12Heap* heap)
+{
+   D3D12_HEAP_DESC ret;
+   heap->GetDesc(&ret);
+   return ret;
+}
+inline D3D12_RESOURCE_DESC
+GetDesc(ID3D12Resource *res)
+{
+   D3D12_RESOURCE_DESC ret;
+   res->GetDesc(&ret);
+   return ret;
+}
+inline D3D12_HEAP_PROPERTIES
+GetCustomHeapProperties(ID3D12Device *dev, D3D12_HEAP_TYPE type)
+{
+   D3D12_HEAP_PROPERTIES ret;
+   dev->GetCustomHeapProperties(&ret, 0, type);
+   return ret;
+}
+inline LUID
+GetAdapterLuid(ID3D12Device *dev)
+{
+   LUID ret;
+   dev->GetAdapterLuid(&ret);
+   return ret;
+}
+inline D3D12_VIDEO_PROCESS_OUTPUT_STREAM_DESC
+GetOutputStreamDesc(ID3D12VideoProcessor *proc)
+{
+   D3D12_VIDEO_PROCESS_OUTPUT_STREAM_DESC ret;
+   proc->GetOutputStreamDesc(&ret);
+   return ret;
+}
+inline D3D12_RESOURCE_ALLOCATION_INFO
+GetResourceAllocationInfo(ID3D12Device *dev, UINT visibleMask, UINT numResourceDescs, const D3D12_RESOURCE_DESC *pResourceDescs)
+{
+   D3D12_RESOURCE_ALLOCATION_INFO ret;
+   dev->GetResourceAllocationInfo(&ret, visibleMask, numResourceDescs, pResourceDescs);
+   return ret;
+}
+#endif
+#endif
+
+#endif

src/gallium/drivers/d3d12/d3d12_compiler.h

@@ -0,0 +1,334 @@
+/*
+ * Copyright © Microsoft Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifndef D3D12_COMPILER_H
+#define D3D12_COMPILER_H
+
+#include "d3d12_common.h"
+
+#include "dxil_nir_lower_int_samplers.h"
+
+#include "pipe/p_defines.h"
+#include "pipe/p_state.h"
+
+#include "compiler/shader_info.h"
+
+#include "nir.h"
+
+struct pipe_screen;
+struct d3d12_context;
+struct d3d12_screen;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum d3d12_state_var {
+   D3D12_STATE_VAR_Y_FLIP = 0,
+   D3D12_STATE_VAR_PT_SPRITE,
+   D3D12_STATE_VAR_DRAW_PARAMS,
+   D3D12_STATE_VAR_DEPTH_TRANSFORM,
+   D3D12_STATE_VAR_DEFAULT_INNER_TESS_LEVEL,
+   D3D12_STATE_VAR_DEFAULT_OUTER_TESS_LEVEL,
+   D3D12_STATE_VAR_PATCH_VERTICES_IN,
+   D3D12_MAX_GRAPHICS_STATE_VARS,
+
+   D3D12_STATE_VAR_NUM_WORKGROUPS = 0,
+   D3D12_STATE_VAR_TRANSFORM_GENERIC0,
+   D3D12_STATE_VAR_TRANSFORM_GENERIC1,
+   D3D12_MAX_COMPUTE_STATE_VARS,
+
+   D3D12_MAX_STATE_VARS = MAX2(D3D12_MAX_GRAPHICS_STATE_VARS, D3D12_MAX_COMPUTE_STATE_VARS)
+};
+
+#define D3D12_MAX_POINT_SIZE 255.0f
+
+
+void
+d3d12_varying_cache_init(struct d3d12_screen *ctx);
+
+void
+d3d12_varying_cache_destroy(struct d3d12_screen *ctx);
+
+
+struct d3d12_varying_info {
+   struct {
+      const struct glsl_type *types[4];
+      uint8_t location_frac_mask:4;
+      uint8_t patch:1;
+      struct {
+         unsigned interpolation:3;   // INTERP_MODE_COUNT = 5
+         unsigned driver_location:6; // VARYING_SLOT_MAX = 64
+         unsigned compact:1;
+         unsigned always_active_io:1;
+      } vars[4];
+   } slots[VARYING_SLOT_MAX];
+   uint64_t mask;
+   uint32_t hash;
+   uint32_t max;
+};
+
+struct d3d12_image_format_conversion_info {
+   enum pipe_format view_format, emulated_format;
+};
+struct d3d12_image_format_conversion_info_arr {
+   unsigned n_images;
+   struct d3d12_image_format_conversion_info* image_format_conversion;
+};
+
+struct d3d12_shader_key {
+   uint32_t hash;
+   mesa_shader_stage stage;
+
+   uint64_t next_varying_inputs;
+   uint64_t prev_varying_outputs;
+   BITSET_WORD *next_varying_frac_inputs;
+   BITSET_WORD *prev_varying_frac_outputs;
+   union {
+      struct {
+         unsigned last_vertex_processing_stage : 1;
+         unsigned invert_depth : 16;
+         unsigned halfz : 1;
+         unsigned samples_int_textures : 1;
+         unsigned input_clip_size : 4;
+         unsigned next_has_frac_inputs : 1;
+         unsigned prev_has_frac_outputs : 1;
+      };
+      uint32_t common_all;
+   };
+   unsigned tex_saturate_s : PIPE_MAX_SAMPLERS;
+   unsigned tex_saturate_r : PIPE_MAX_SAMPLERS;
+   unsigned tex_saturate_t : PIPE_MAX_SAMPLERS;
+   union {
+      struct {
+         unsigned needs_format_emulation:1;
+         enum pipe_format format_conversion[PIPE_MAX_ATTRIBS];
+      } vs;
+
+      union {
+         struct {
+            unsigned sprite_coord_enable:24;
+            unsigned sprite_origin_upper_left:1;
+            unsigned point_pos_stream_out:1;
+            unsigned writes_psize:1;
+            unsigned point_size_per_vertex:1;
+            unsigned aa_point:1;
+            unsigned stream_output_factor:3;
+            unsigned primitive_id:1;
+            unsigned triangle_strip:1;
+         };
+         uint64_t all;
+      } gs;
+
+      struct {
+         union {
+            struct {
+               uint32_t next_patch_inputs;
+               unsigned primitive_mode:2;
+               unsigned ccw:1;
+               unsigned point_mode:1;
+               unsigned spacing:2;
+               unsigned patch_vertices_in:5;
+            };
+            uint64_t all;
+         };
+      } hs;
+
+      struct {
+         unsigned tcs_vertices_out;
+         uint32_t prev_patch_outputs;
+      } ds;
+
+      union {
+         struct {
+            unsigned missing_dual_src_outputs : 2;
+            unsigned frag_result_color_lowering : 4;
+            unsigned cast_to_uint : 1;
+            unsigned cast_to_int : 1;
+            unsigned provoking_vertex : 2;
+            unsigned manual_depth_range : 1;
+            unsigned polygon_stipple : 1;
+            unsigned remap_front_facing : 1;
+            unsigned multisample_disabled : 1;
+         };
+         unsigned short all;
+      } fs;
+
+      struct {
+         unsigned workgroup_size[3];
+      } cs;
+   };
+
+   int n_texture_states;
+   dxil_wrap_sampler_state *tex_wrap_states;
+   dxil_texture_swizzle_state swizzle_state[PIPE_MAX_SHADER_SAMPLER_VIEWS];
+   enum compare_func sampler_compare_funcs[PIPE_MAX_SHADER_SAMPLER_VIEWS];
+
+   unsigned n_images;
+   struct d3d12_image_format_conversion_info image_format_conversion[PIPE_MAX_SHADER_IMAGES];
+};
+
+struct d3d12_shader {
+   void *bytecode;
+   size_t bytecode_length;
+
+   nir_shader *nir;
+
+   /* UBOs can be sparse, if there's no uniforms then ubo0 is unused, and state vars are an internal ubo */
+   uint32_t begin_ubo_binding;
+   uint32_t end_ubo_binding;
+
+   struct {
+      enum d3d12_state_var var;
+      unsigned offset;
+   } state_vars[D3D12_MAX_STATE_VARS];
+   unsigned num_state_vars;
+   size_t state_vars_size;
+   bool state_vars_used;
+
+   /* Samplers/textures can be sparse for some internal shaders */
+   struct {
+      uint32_t dimension;
+   } srv_bindings[PIPE_MAX_SHADER_SAMPLER_VIEWS];
+   uint32_t begin_srv_binding;
+   uint32_t end_srv_binding;
+
+   /* Images and SSBOs are never sparse */
+   struct {
+      uint32_t dimension;
+   } uav_bindings[PIPE_MAX_SHADER_IMAGES];
+
+   unsigned pstipple_binding;
+
+   struct d3d12_shader_key key;
+   struct d3d12_shader *next_variant;
+};
+
+struct d3d12_gs_variant_key
+{
+   union {
+      struct {
+         unsigned passthrough:1;
+         unsigned provoking_vertex:3;
+         unsigned alternate_tri:1;
+         unsigned fill_mode:2;
+         unsigned cull_mode:2;
+         unsigned has_front_face:1;
+         unsigned front_ccw:1;
+         unsigned edge_flag_fix:1;
+         unsigned flatshade_first:1;
+      };
+      uint64_t all;
+   };
+   uint64_t flat_varyings;
+   struct d3d12_varying_info *varyings;
+};
+
+struct d3d12_tcs_variant_key
+{
+   unsigned vertices_out;
+   struct d3d12_varying_info *varyings;
+};
+
+struct d3d12_shader_selector {
+   mesa_shader_stage stage;
+   const nir_shader *initial;
+   struct d3d12_varying_info *initial_output_vars;
+   struct d3d12_varying_info *initial_input_vars;
+
+   struct d3d12_shader *first;
+   struct d3d12_shader *current;
+
+   struct pipe_stream_output_info so_info;
+
+   unsigned samples_int_textures:1;
+   unsigned compare_with_lod_bias_grad:1;
+   unsigned workgroup_size_variable:1;
+   unsigned has_frac_inputs:1;
+   unsigned has_frac_outputs:1;
+
+   BITSET_DECLARE(varying_frac_inputs, 64 * 4);
+   BITSET_DECLARE(varying_frac_outputs, 64 * 4);
+
+   bool is_variant;
+   union {
+      struct d3d12_gs_variant_key gs_key;
+      struct d3d12_tcs_variant_key tcs_key;
+   };
+};
+
+struct d3d12_shader_selector *
+d3d12_create_shader(struct d3d12_context *ctx,
+                    mesa_shader_stage stage,
+                    const struct pipe_shader_state *shader);
+
+struct d3d12_shader_selector *
+d3d12_create_compute_shader(struct d3d12_context *ctx,
+                            const struct pipe_compute_state *shader);
+
+void
+d3d12_shader_free(struct d3d12_shader_selector *shader);
+
+void
+d3d12_select_shader_variants(struct d3d12_context *ctx,
+                             const struct pipe_draw_info *dinfo);
+
+void
+d3d12_select_compute_shader_variants(struct d3d12_context *ctx,
+                                     const struct pipe_grid_info *info);
+
+void
+d3d12_gs_variant_cache_init(struct d3d12_context *ctx);
+
+void
+d3d12_gs_variant_cache_destroy(struct d3d12_context *ctx);
+
+struct d3d12_shader_selector *
+d3d12_get_gs_variant(struct d3d12_context *ctx, struct d3d12_gs_variant_key *key);
+
+void
+d3d12_tcs_variant_cache_init(struct d3d12_context *ctx);
+
+void
+d3d12_tcs_variant_cache_destroy(struct d3d12_context *ctx);
+
+struct d3d12_shader_selector *
+d3d12_get_tcs_variant(struct d3d12_context *ctx, struct d3d12_tcs_variant_key *key);
+
+unsigned
+missing_dual_src_outputs(struct d3d12_context* ctx);
+
+bool
+has_flat_varyings(struct d3d12_context* ctx);
+
+bool
+d3d12_compare_varying_info(const struct d3d12_varying_info *expect, const struct d3d12_varying_info *have);
+
+bool
+manual_depth_range(struct d3d12_context* ctx);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/gallium/drivers/d3d12/d3d12_compute_transforms.cpp

@@ -0,0 +1,513 @@
+/*
+ * Copyright © Microsoft Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#include "d3d12_compute_transforms.h"
+#include "d3d12_nir_passes.h"
+#include "d3d12_query.h"
+#include "d3d12_screen.h"
+
+#include "nir.h"
+#include "nir_builder.h"
+
+#include "util/u_memory.h"
+
+nir_shader *
+get_indirect_draw_base_vertex_transform(const nir_shader_compiler_options *options, const d3d12_compute_transform_key *args)
+{
+   nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_COMPUTE, options, "TransformIndirectDrawBaseVertex");
+
+   if (args->base_vertex.dynamic_count) {
+      nir_variable *count_ubo = nir_variable_create(b.shader, nir_var_mem_ubo,
+         glsl_uint_type(), "in_count");
+      count_ubo->data.driver_location = 0;
+   }
+
+   nir_variable *input_ssbo = nir_variable_create(b.shader, nir_var_mem_ssbo,
+      glsl_array_type(glsl_uint_type(), 0, 0), "input");
+   nir_variable *output_ssbo = nir_variable_create(b.shader, nir_var_mem_ssbo,
+      input_ssbo->type, "output");
+   input_ssbo->data.driver_location = 0;
+   output_ssbo->data.driver_location = 1;
+
+   nir_def *draw_id = nir_channel(&b, nir_load_global_invocation_id(&b, 32), 0);
+   if (args->base_vertex.dynamic_count) {
+      nir_def *count = nir_load_ubo(&b, 1, 32, nir_imm_int(&b, 1), nir_imm_int(&b, 0),
+         (gl_access_qualifier)0, 4, 0, 0, 4);
+      nir_push_if(&b, nir_ilt(&b, draw_id, count));
+   }
+
+   nir_variable *stride_ubo = NULL;
+   nir_def *in_stride_offset_and_base_drawid = d3d12_get_state_var(&b, D3D12_STATE_VAR_TRANSFORM_GENERIC0, "d3d12_Stride",
+      glsl_uvec4_type(), &stride_ubo);
+   nir_def *in_offset = nir_iadd(&b, nir_channel(&b, in_stride_offset_and_base_drawid, 1),
+      nir_imul(&b, nir_channel(&b, in_stride_offset_and_base_drawid, 0), draw_id));
+   nir_def *in_data0 = nir_load_ssbo(&b, 4, 32, nir_imm_int(&b, 0), in_offset, (gl_access_qualifier)0, 4, 0);
+
+   nir_def *in_data1 = NULL;
+   nir_def *base_vertex = NULL, *base_instance = NULL;
+   if (args->base_vertex.indexed) {
+      nir_def *in_offset1 = nir_iadd(&b, in_offset, nir_imm_int(&b, 16));
+      in_data1 = nir_load_ssbo(&b, 1, 32, nir_imm_int(&b, 0), in_offset1, (gl_access_qualifier)0, 4, 0);
+      base_vertex = nir_channel(&b, in_data0, 3);
+      base_instance = in_data1;
+   } else {
+      base_vertex = nir_channel(&b, in_data0, 2);
+      base_instance = nir_channel(&b, in_data0, 3);
+   }
+
+   /* 4 additional uints for base vertex, base instance, draw ID, and a bool for indexed draw */
+   unsigned out_stride = sizeof(uint32_t) * ((args->base_vertex.indexed ? 5 : 4) + 4);
+
+   nir_def *out_offset = nir_imul(&b, draw_id, nir_imm_int(&b, out_stride));
+   nir_def *out_data0 = nir_vec4(&b, base_vertex, base_instance,
+      nir_iadd(&b, draw_id, nir_channel(&b, in_stride_offset_and_base_drawid, 2)),
+      nir_imm_int(&b, args->base_vertex.indexed ? -1 : 0));
+   nir_def *out_data1 = in_data0;
+
+   nir_store_ssbo(&b, out_data0, nir_imm_int(&b, 1), out_offset, 0xf, (gl_access_qualifier)0, 4, 0);
+   nir_store_ssbo(&b, out_data1, nir_imm_int(&b, 1), nir_iadd(&b, out_offset, nir_imm_int(&b, 16)),
+      (1u << out_data1->num_components) - 1, (gl_access_qualifier)0, 4, 0);
+   if (args->base_vertex.indexed)
+      nir_store_ssbo(&b, in_data1, nir_imm_int(&b, 1), nir_iadd(&b, out_offset, nir_imm_int(&b, 32)), 1, (gl_access_qualifier)0, 4, 0);
+
+   if (args->base_vertex.dynamic_count)
+      nir_pop_if(&b, NULL);
+
+   nir_validate_shader(b.shader, "creation");
+   b.shader->info.num_ssbos = 2;
+   b.shader->info.num_ubos = (args->base_vertex.dynamic_count ? 1 : 0);
+
+   return b.shader;
+}
+
+static struct nir_shader *
+get_fake_so_buffer_copy_back(const nir_shader_compiler_options *options, const d3d12_compute_transform_key *key)
+{
+   nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_COMPUTE, options, "FakeSOBufferCopyBack");
+
+   nir_variable *output_so_data_var = nir_variable_create(b.shader, nir_var_mem_ssbo,
+      glsl_array_type(glsl_uint_type(), 0, 0), "output_data");
+   nir_variable *input_so_data_var = nir_variable_create(b.shader, nir_var_mem_ssbo, output_so_data_var->type, "input_data");
+   output_so_data_var->data.driver_location = 0;
+   input_so_data_var->data.driver_location = 1;
+
+   /* UBO is [fake SO filled size, fake SO vertex count, 1, 1, original SO filled size] */
+   nir_variable *input_ubo = nir_variable_create(b.shader, nir_var_mem_ubo,
+      glsl_array_type(glsl_uint_type(), 5, 0), "input_ubo");
+   input_ubo->data.driver_location = 0;
+
+   nir_def *original_so_filled_size = nir_load_ubo(&b, 1, 32, nir_imm_int(&b, 0), nir_imm_int(&b, 4 * sizeof(uint32_t)),
+      (gl_access_qualifier)0, 4, 0, 4 * sizeof(uint32_t), 4);
+
+   nir_variable *state_var = nullptr;
+   nir_def *fake_so_multiplier = d3d12_get_state_var(&b, D3D12_STATE_VAR_TRANSFORM_GENERIC0, "fake_so_multiplier", glsl_uint_type(), &state_var);
+
+   nir_def *vertex_offset = nir_imul(&b, nir_imm_int(&b, key->fake_so_buffer_copy_back.stride),
+      nir_channel(&b, nir_load_global_invocation_id(&b, 32), 0));
+
+   nir_def *output_offset_base = nir_iadd(&b, original_so_filled_size, vertex_offset);
+   nir_def *input_offset_base = nir_imul(&b, vertex_offset, fake_so_multiplier);
+
+   for (unsigned i = 0; i < key->fake_so_buffer_copy_back.num_ranges; ++i) {
+      auto& output = key->fake_so_buffer_copy_back.ranges[i];
+      assert(output.size % 4 == 0 && output.offset % 4 == 0);
+      nir_def *field_offset = nir_imm_int(&b, output.offset);
+      nir_def *output_offset = nir_iadd(&b, output_offset_base, field_offset);
+      nir_def *input_offset = nir_iadd(&b, input_offset_base, field_offset);
+
+      for (unsigned loaded = 0; loaded < output.size; loaded += 16) {
+         unsigned to_load = MIN2(output.size, 16);
+         unsigned components = to_load / 4;
+         nir_def *loaded_data = nir_load_ssbo(&b, components, 32, nir_imm_int(&b, 1),
+            nir_iadd(&b, input_offset, nir_imm_int(&b, loaded)), (gl_access_qualifier)0, 4, 0);
+         nir_store_ssbo(&b, loaded_data, nir_imm_int(&b, 0),
+            nir_iadd(&b, output_offset, nir_imm_int(&b, loaded)), (1u << components) - 1, (gl_access_qualifier)0, 4, 0);
+      }
+   }
+
+   nir_validate_shader(b.shader, "creation");
+   b.shader->info.num_ssbos = 2;
+   b.shader->info.num_ubos = 1;
+
+   return b.shader;
+}
+
+static struct nir_shader *
+get_fake_so_buffer_vertex_count(const nir_shader_compiler_options *options)
+{
+   nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_COMPUTE, options, "FakeSOBufferVertexCount");
+
+   nir_variable_create(b.shader, nir_var_mem_ssbo, glsl_array_type(glsl_uint_type(), 0, 0), "fake_so");
+   nir_def *fake_buffer_filled_size = nir_load_ssbo(&b, 1, 32, nir_imm_int(&b, 0), nir_imm_int(&b, 0), (gl_access_qualifier)0, 4, 0);
+
+   nir_variable *real_so_var = nir_variable_create(b.shader, nir_var_mem_ssbo,
+      glsl_array_type(glsl_uint_type(), 0, 0), "real_so");
+   real_so_var->data.driver_location = 1;
+   nir_def *real_buffer_filled_size = nir_load_ssbo(&b, 1, 32, nir_imm_int(&b, 1), nir_imm_int(&b, 0), (gl_access_qualifier)0, 4, 0);
+
+   nir_variable *state_var = nullptr;
+   nir_def *state_var_data = d3d12_get_state_var(&b, D3D12_STATE_VAR_TRANSFORM_GENERIC0, "state_var", glsl_uvec4_type(), &state_var);
+   nir_def *stride = nir_channel(&b, state_var_data, 0);
+   nir_def *fake_so_multiplier = nir_channel(&b, state_var_data, 1);
+
+   nir_def *real_so_bytes_added = nir_idiv(&b, fake_buffer_filled_size, fake_so_multiplier);
+   nir_def *vertex_count = nir_idiv(&b, real_so_bytes_added, stride);
+   nir_def *to_write_to_fake_buffer = nir_vec4(&b, vertex_count, nir_imm_int(&b, 1), nir_imm_int(&b, 1), real_buffer_filled_size);
+   nir_store_ssbo(&b, to_write_to_fake_buffer, nir_imm_int(&b, 0), nir_imm_int(&b, 4), 0xf, (gl_access_qualifier)0, 4, 0);
+
+   nir_def *updated_filled_size = nir_iadd(&b, real_buffer_filled_size, real_so_bytes_added);
+   nir_store_ssbo(&b, updated_filled_size, nir_imm_int(&b, 1), nir_imm_int(&b, 0), 1, (gl_access_qualifier)0, 4, 0);
+
+   nir_validate_shader(b.shader, "creation");
+   b.shader->info.num_ssbos = 2;
+   b.shader->info.num_ubos = 0;
+
+   return b.shader;
+}
+
+static struct nir_shader *
+get_draw_auto(const nir_shader_compiler_options *options)
+{
+   nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_COMPUTE, options, "DrawAuto");
+
+   nir_variable_create(b.shader, nir_var_mem_ssbo, glsl_array_type(glsl_uint_type(), 0, 0), "ssbo");
+   nir_def *buffer_filled_size = nir_load_ssbo(&b, 1, 32, nir_imm_int(&b, 0), nir_imm_int(&b, 0), (gl_access_qualifier)0, 4, 0);
+
+   nir_variable *state_var = nullptr;
+   nir_def *state_var_data = d3d12_get_state_var(&b, D3D12_STATE_VAR_TRANSFORM_GENERIC0, "state_var", glsl_uvec4_type(), &state_var);
+   nir_def *stride = nir_channel(&b, state_var_data, 0);
+   nir_def *vb_offset = nir_channel(&b, state_var_data, 1);
+
+   nir_def *vb_bytes = nir_bcsel(&b, nir_ilt(&b, vb_offset, buffer_filled_size),
+      nir_isub(&b, buffer_filled_size, vb_offset), nir_imm_int(&b, 0));
+
+   nir_def *vertex_count = nir_idiv(&b, vb_bytes, stride);
+   nir_def *to_write = nir_vec4(&b, vertex_count, nir_imm_int(&b, 1), nir_imm_int(&b, 0), nir_imm_int(&b, 0));
+   nir_store_ssbo(&b, to_write, nir_imm_int(&b, 0), nir_imm_int(&b, 4), 0xf, (gl_access_qualifier)0, 4, 0);
+
+   nir_validate_shader(b.shader, "creation");
+   b.shader->info.num_ssbos = 1;
+   b.shader->info.num_ubos = 0;
+
+   return b.shader;
+}
+
+static struct nir_shader *
+get_query_resolve(const nir_shader_compiler_options *options, const d3d12_compute_transform_key *key)
+{
+   nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_COMPUTE, options, "QueryResolve");
+
+   uint32_t bit_size = key->query_resolve.is_64bit ? 64 : 32;
+   const struct glsl_type *value_type = glsl_uintN_t_type(bit_size);
+
+   assert(!key->query_resolve.is_resolve_in_place ||
+          (key->query_resolve.is_64bit && key->query_resolve.num_subqueries == 1));
+   assert(key->query_resolve.num_subqueries == 1 ||
+          key->query_resolve.pipe_query_type == PIPE_QUERY_PRIMITIVES_GENERATED ||
+          key->query_resolve.pipe_query_type == PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE);
+   assert(key->query_resolve.num_subqueries <= 4);
+
+   nir_variable *inputs[4];
+   for (uint32_t i = 0; i < key->query_resolve.num_subqueries; ++i) {
+      /* Inputs are always 64-bit */
+      inputs[i] = nir_variable_create(b.shader, nir_var_mem_ssbo, glsl_array_type(glsl_uint64_t_type(), 0, 8), "input");
+      inputs[i]->data.binding = i;
+   }
+   nir_variable *output = inputs[0];
+   if (!key->query_resolve.is_resolve_in_place) {
+      output = nir_variable_create(b.shader, nir_var_mem_ssbo, glsl_array_type(value_type, 0, bit_size / 8), "output");
+      output->data.binding = key->query_resolve.num_subqueries;
+   }
+
+   /* How many entries in each sub-query is passed via root constants */
+   nir_variable *state_var = nullptr, *state_var1 = nullptr;
+   nir_def *state_var_data = d3d12_get_state_var(&b, D3D12_STATE_VAR_TRANSFORM_GENERIC0, "state_var", glsl_uvec4_type(), &state_var);
+   nir_def *state_var_data1 = d3d12_get_state_var(&b, D3D12_STATE_VAR_TRANSFORM_GENERIC1, "state_var1", glsl_uvec4_type(), &state_var1);
+
+   /* For in-place resolves, we resolve each field of the query. Otherwise, resolve one field into the dest */
+   nir_variable *results[sizeof(D3D12_QUERY_DATA_PIPELINE_STATISTICS) / sizeof(UINT64)];
+   uint32_t num_result_values = 1;
+
+   if (key->query_resolve.is_resolve_in_place) {
+      if (key->query_resolve.pipe_query_type == PIPE_QUERY_PIPELINE_STATISTICS)
+         num_result_values = sizeof(D3D12_QUERY_DATA_PIPELINE_STATISTICS) / sizeof(UINT64);
+      else if (key->query_resolve.pipe_query_type == PIPE_QUERY_SO_STATISTICS)
+         num_result_values = sizeof(D3D12_QUERY_DATA_SO_STATISTICS) / sizeof(UINT64);
+   }
+   
+   uint32_t var_bit_size = key->query_resolve.pipe_query_type == PIPE_QUERY_TIME_ELAPSED ||
+                           key->query_resolve.pipe_query_type == PIPE_QUERY_TIMESTAMP ? 64 : bit_size;
+   for (uint32_t i = 0; i < num_result_values; ++i) {
+      results[i] = nir_local_variable_create(b.impl, glsl_uintN_t_type(var_bit_size), "result");
+      nir_store_var(&b, results[i], nir_imm_intN_t(&b, 0, var_bit_size), 1);
+   }
+
+   /* For each subquery... */
+   for (uint32_t i = 0; i < key->query_resolve.num_subqueries; ++i) {
+      nir_def *num_results = nir_channel(&b, state_var_data, i);
+
+      uint32_t subquery_index = key->query_resolve.num_subqueries == 1 ?
+         key->query_resolve.single_subquery_index : i;
+      uint32_t base_offset = 0;
+      uint32_t stride = 0;
+      switch (key->query_resolve.pipe_query_type) {
+      case PIPE_QUERY_OCCLUSION_COUNTER:
+      case PIPE_QUERY_OCCLUSION_PREDICATE:
+      case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE:
+      case PIPE_QUERY_TIMESTAMP:
+         stride = 1;
+         break;
+      case PIPE_QUERY_TIME_ELAPSED:
+         stride = 2;
+         break;
+      case PIPE_QUERY_SO_STATISTICS:
+      case PIPE_QUERY_PRIMITIVES_EMITTED:
+      case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
+      case PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE:
+         stride = sizeof(D3D12_QUERY_DATA_SO_STATISTICS) / sizeof(UINT64);
+         break;
+      case PIPE_QUERY_PRIMITIVES_GENERATED:
+         if (subquery_index == 0)
+            stride = sizeof(D3D12_QUERY_DATA_SO_STATISTICS) / sizeof(UINT64);
+         else
+            stride = sizeof(D3D12_QUERY_DATA_PIPELINE_STATISTICS) / sizeof(UINT64);
+         if (!key->query_resolve.is_resolve_in_place) {
+            if (subquery_index == 1)
+               base_offset = offsetof(D3D12_QUERY_DATA_PIPELINE_STATISTICS, GSPrimitives) / sizeof(UINT64);
+            else if (subquery_index == 2)
+               base_offset = offsetof(D3D12_QUERY_DATA_PIPELINE_STATISTICS, IAPrimitives) / sizeof(UINT64);
+         }
+         break;
+      case PIPE_QUERY_PIPELINE_STATISTICS:
+         stride = sizeof(D3D12_QUERY_DATA_PIPELINE_STATISTICS) / sizeof(UINT64);
+         break;
+      default:
+         UNREACHABLE("Unhandled query resolve");
+      }
+
+      if (!key->query_resolve.is_resolve_in_place && key->query_resolve.num_subqueries == 1)
+         base_offset = key->query_resolve.single_result_field_offset;
+
+      nir_def *base_array_index = nir_imm_int(&b, base_offset);
+
+      /* For each query result in this subquery... */
+      nir_variable *loop_counter = nir_local_variable_create(b.impl, glsl_uint_type(), "loop_counter");
+      nir_store_var(&b, loop_counter, nir_imm_int(&b, 0), 1);
+      nir_loop *loop = nir_push_loop(&b);
+
+      nir_def *loop_counter_value = nir_load_var(&b, loop_counter);
+      nir_break_if(&b, nir_ieq(&b, loop_counter_value, num_results));
+
+      /* For each field in the query result, accumulate */
+      nir_def *array_index = nir_iadd(&b, nir_imul_imm(&b, loop_counter_value, stride), base_array_index);
+      for (uint32_t j = 0; j < num_result_values; ++j) {
+         nir_def *new_value;
+         if (key->query_resolve.pipe_query_type == PIPE_QUERY_TIME_ELAPSED) {
+            assert(j == 0 && i == 0);
+            nir_def *start = nir_load_ssbo(&b, 1, 64, nir_imm_int(&b, i), nir_imul_imm(&b, array_index, 8));
+            nir_def *end = nir_load_ssbo(&b, 1, 64, nir_imm_int(&b, i), nir_imul_imm(&b, nir_iadd_imm(&b, array_index, 1), 8));
+            new_value = nir_iadd(&b, nir_load_var(&b, results[j]), nir_isub(&b, end, start));
+         } else if (key->query_resolve.pipe_query_type == PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE ||
+                    key->query_resolve.pipe_query_type == PIPE_QUERY_SO_OVERFLOW_PREDICATE) {
+            /* These predicates are true if the primitives emitted != primitives stored */
+            assert(j == 0);
+            nir_def *val_a = nir_load_ssbo(&b, 1, 64, nir_imm_int(&b, i), nir_imul_imm(&b, array_index, 8));
+            nir_def *val_b = nir_load_ssbo(&b, 1, 64, nir_imm_int(&b, i), nir_imul_imm(&b, nir_iadd_imm(&b, array_index, 1), 8));
+            new_value = nir_ior(&b, nir_load_var(&b, results[j]), nir_u2uN(&b, nir_ine(&b, val_a, val_b), var_bit_size));
+         } else {
+            new_value = nir_u2uN(&b, nir_load_ssbo(&b, 1, 64, nir_imm_int(&b, i), nir_imul_imm(&b, nir_iadd_imm(&b, array_index, j), 8)), var_bit_size);
+            new_value = nir_iadd(&b, nir_load_var(&b, results[j]), new_value);
+         }
+         nir_store_var(&b, results[j], new_value, 1);
+      }
+      
+      nir_store_var(&b, loop_counter, nir_iadd_imm(&b, loop_counter_value, 1), 1);
+      nir_pop_loop(&b, loop);
+   }
+
+   /* Results are accumulated, now store the final values */
+   nir_def *output_base_index = nir_channel(&b, state_var_data1, 0);
+   for (uint32_t i = 0; i < num_result_values; ++i) {
+      /* When resolving in-place, resolve each field, otherwise just write the one result */
+      uint32_t field_offset = key->query_resolve.is_resolve_in_place ? i : 0;
+
+      /* When resolving time elapsed in-place, write [0, time], as the only special case */
+      if (key->query_resolve.is_resolve_in_place &&
+          key->query_resolve.pipe_query_type == PIPE_QUERY_TIME_ELAPSED) {
+         nir_store_ssbo(&b, nir_imm_int64(&b, 0), nir_imm_int(&b, output->data.binding),
+                        nir_imul_imm(&b, output_base_index, bit_size / 8), 1, (gl_access_qualifier)0, bit_size / 8, 0);
+         field_offset++;
+      }
+      nir_def *result_val = nir_load_var(&b, results[i]);
+      if (!key->query_resolve.is_resolve_in_place &&
+          (key->query_resolve.pipe_query_type == PIPE_QUERY_TIME_ELAPSED ||
+           key->query_resolve.pipe_query_type == PIPE_QUERY_TIMESTAMP)) {
+         result_val = nir_f2u64(&b, nir_fmul_imm(&b, nir_u2f32(&b, result_val), key->query_resolve.timestamp_multiplier));
+
+         if (!key->query_resolve.is_64bit) {
+            nir_alu_type rounding_type = key->query_resolve.is_signed ? nir_type_int : nir_type_uint;
+            nir_alu_type src_round = (nir_alu_type)(rounding_type | 64);
+            nir_alu_type dst_round = (nir_alu_type)(rounding_type | bit_size);
+            result_val = nir_convert_alu_types(&b, bit_size, result_val, src_round, dst_round, nir_rounding_mode_undef, true);
+         }
+      }
+      nir_store_ssbo(&b, result_val, nir_imm_int(&b, output->data.binding),
+                     nir_imul_imm(&b, nir_iadd_imm(&b, output_base_index, field_offset), bit_size / 8),
+                     1, (gl_access_qualifier)0, bit_size / 8, 0);
+   }
+
+   nir_validate_shader(b.shader, "creation");
+   b.shader->info.num_ssbos = key->query_resolve.num_subqueries + !key->query_resolve.is_resolve_in_place;
+   b.shader->info.num_ubos = 0;
+
+   NIR_PASS(_, b.shader, nir_lower_convert_alu_types, NULL);
+
+   return b.shader;
+}
+
+static struct nir_shader *
+create_compute_transform(const nir_shader_compiler_options *options, const d3d12_compute_transform_key *key)
+{
+   switch (key->type) {
+   case d3d12_compute_transform_type::base_vertex:
+      return get_indirect_draw_base_vertex_transform(options, key);
+   case d3d12_compute_transform_type::fake_so_buffer_copy_back:
+      return get_fake_so_buffer_copy_back(options, key);
+   case d3d12_compute_transform_type::fake_so_buffer_vertex_count:
+      return get_fake_so_buffer_vertex_count(options);
+   case d3d12_compute_transform_type::draw_auto:
+      return get_draw_auto(options);
+   case d3d12_compute_transform_type::query_resolve:
+      return get_query_resolve(options, key);
+   default:
+      UNREACHABLE("Invalid transform");
+   }
+}
+
+struct compute_transform
+{
+   d3d12_compute_transform_key key;
+   d3d12_shader_selector *shader;
+};
+
+d3d12_shader_selector *
+d3d12_get_compute_transform(struct d3d12_context *ctx, const d3d12_compute_transform_key *key)
+{
+   struct hash_entry *entry = _mesa_hash_table_search(ctx->compute_transform_cache, key);
+   if (!entry) {
+      compute_transform *data = (compute_transform *)MALLOC(sizeof(compute_transform));
+      if (!data)
+         return NULL;
+      
+      const nir_shader_compiler_options *options = &d3d12_screen(ctx->base.screen)->nir_options;
+
+      memcpy(&data->key, key, sizeof(*key));
+      nir_shader *s = create_compute_transform(options, key);
+      if (!s) {
+         FREE(data);
+         return NULL;
+      }
+      struct pipe_compute_state shader_args = { PIPE_SHADER_IR_NIR, s };
+      data->shader = d3d12_create_compute_shader(ctx, &shader_args);
+      if (!data->shader) {
+         ralloc_free(s);
+         FREE(data);
+         return NULL;
+      }
+
+      data->shader->is_variant = true;
+      entry = _mesa_hash_table_insert(ctx->compute_transform_cache, &data->key, data);
+      assert(entry);
+   }
+
+   return ((struct compute_transform *)entry->data)->shader;
+}
+
+static uint32_t
+hash_compute_transform_key(const void *key)
+{
+   return _mesa_hash_data(key, sizeof(struct d3d12_compute_transform_key));
+}
+
+static bool
+equals_compute_transform_key(const void *a, const void *b)
+{
+   return memcmp(a, b, sizeof(struct d3d12_compute_transform_key)) == 0;
+}
+
+void
+d3d12_compute_transform_cache_init(struct d3d12_context *ctx)
+{
+   ctx->compute_transform_cache = _mesa_hash_table_create(NULL,
+                                                          hash_compute_transform_key,
+                                                          equals_compute_transform_key);
+}
+
+static void
+delete_entry(struct hash_entry *entry)
+{
+   struct compute_transform *data = (struct compute_transform *)entry->data;
+   d3d12_shader_free(data->shader);
+   FREE(data);
+}
+
+void
+d3d12_compute_transform_cache_destroy(struct d3d12_context *ctx)
+{
+   _mesa_hash_table_destroy(ctx->compute_transform_cache, delete_entry);
+}
+
+void
+d3d12_save_compute_transform_state(struct d3d12_context *ctx, d3d12_compute_transform_save_restore *save)
+{
+   if (ctx->current_predication)
+      ctx->cmdlist->SetPredication(nullptr, 0, D3D12_PREDICATION_OP_EQUAL_ZERO);
+
+   memset(save, 0, sizeof(*save));
+   save->cs = ctx->compute_state;
+
+   pipe_resource_reference(&save->cbuf0.buffer, ctx->cbufs[MESA_SHADER_COMPUTE][1].buffer);
+   save->cbuf0 = ctx->cbufs[MESA_SHADER_COMPUTE][1];
+
+   for (unsigned i = 0; i < ARRAY_SIZE(save->ssbos); ++i) {
+      pipe_resource_reference(&save->ssbos[i].buffer, ctx->ssbo_views[MESA_SHADER_COMPUTE][i].buffer);
+      save->ssbos[i] = ctx->ssbo_views[MESA_SHADER_COMPUTE][i];
+   }
+
+   save->queries_disabled = ctx->queries_disabled;
+   ctx->base.set_active_query_state(&ctx->base, false);
+}
+
+void
+d3d12_restore_compute_transform_state(struct d3d12_context *ctx, d3d12_compute_transform_save_restore *save)
+{
+   ctx->base.set_active_query_state(&ctx->base, !save->queries_disabled);
+
+   ctx->base.bind_compute_state(&ctx->base, save->cs);
+
+   ctx->base.set_constant_buffer(&ctx->base, MESA_SHADER_COMPUTE, 1, &save->cbuf0);
+   ctx->base.set_shader_buffers(&ctx->base, MESA_SHADER_COMPUTE, 0, ARRAY_SIZE(save->ssbos), save->ssbos, (1u << ARRAY_SIZE(save->ssbos)) - 1);
+
+   if (ctx->current_predication)
+      d3d12_enable_predication(ctx);
+}

src/gallium/drivers/d3d12/d3d12_compute_transforms.h

@@ -0,0 +1,115 @@
+/*
+ * Copyright © Microsoft Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifndef D3D12_COMPUTE_TRANSFORMS_H
+#define D3D12_COMPUTE_TRANSFORMS_H
+
+#include "d3d12_context.h"
+#include "d3d12_compiler.h"
+
+enum class d3d12_compute_transform_type
+{
+   /* Extract vertex shader draw params (base vertex, instance, draw ID) from
+    * a stream of indirect draw (indexed) params
+    */
+   base_vertex,
+   /* Given an SO buffer's declaration in the key, copy filled items from a fake (multiplied)
+    * buffer into the original SO buffer, after the original filled size (loaded from the indirect
+    * arg buffer double-bound as a UBO), making sure to skip gaps
+    */
+   fake_so_buffer_copy_back,
+   /* Append a fake SO buffer filed size with (vertex count, 1, 1, original filled size)
+    * for an indirect dispatch of the fake_so_buffer_copy_back transform, and also update
+    * the original filled size with the fake filled size
+    */
+   fake_so_buffer_vertex_count,
+   /* Append a buffer filled size with (vertex count, 1, 0, 0) */
+   draw_auto,
+   /* Accumulate queries together and write a 32-bit or 64-bit result */
+   query_resolve,
+   max,
+};
+
+struct d3d12_compute_transform_key
+{
+   d3d12_compute_transform_type type;
+
+   union
+   {
+      struct {
+         unsigned indexed : 1;
+         unsigned dynamic_count : 1;
+      } base_vertex;
+
+      struct {
+         uint16_t stride;
+         uint16_t num_ranges;
+         struct {
+            uint16_t offset;
+            uint16_t size;
+         } ranges[PIPE_MAX_SO_OUTPUTS];
+      } fake_so_buffer_copy_back;
+
+      struct {
+         /* true means the accumulation should be done as uint64, else uint32. */
+         uint8_t is_64bit : 1;
+         /* Indicates how many subqueries to accumulate together into a final result. When
+          * set to 1, single_subquery_index determines where the data comes from. */
+         uint8_t num_subqueries : 3;
+         uint8_t pipe_query_type : 4;
+         /* true means output is written where input[0] was, else output is a separate buffer.
+          * true also means all fields are accumulated, else single_result_field_offset determines
+          * which field is resolved. Implies num_subqueries == 1. */
+         uint8_t is_resolve_in_place : 1;
+         uint8_t single_subquery_index : 2;
+         uint8_t single_result_field_offset : 4;
+         uint8_t is_signed : 1;
+         float timestamp_multiplier;
+      } query_resolve;
+   };
+};
+
+d3d12_shader_selector *
+d3d12_get_compute_transform(struct d3d12_context *ctx, const d3d12_compute_transform_key *key);
+
+void
+d3d12_compute_transform_cache_init(struct d3d12_context *ctx);
+
+void
+d3d12_compute_transform_cache_destroy(struct d3d12_context *ctx);
+
+struct d3d12_compute_transform_save_restore
+{
+   struct d3d12_shader_selector *cs;
+   struct pipe_constant_buffer cbuf0;
+   struct pipe_shader_buffer ssbos[5];
+   bool queries_disabled;
+};
+
+void
+d3d12_save_compute_transform_state(struct d3d12_context *ctx, d3d12_compute_transform_save_restore *save);
+
+void
+d3d12_restore_compute_transform_state(struct d3d12_context *ctx, d3d12_compute_transform_save_restore *save);
+
+#endif

src/gallium/drivers/d3d12/d3d12_context.h

@@ -0,0 +1,430 @@
+/*
+ * Copyright © Microsoft Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifndef D3D12_CONTEXT_H
+#define D3D12_CONTEXT_H
+
+#include "d3d12_batch.h"
+#include "d3d12_descriptor_pool.h"
+#include "d3d12_pipeline_state.h"
+
+#ifdef HAVE_GALLIUM_D3D12_GRAPHICS
+#include "dxil_nir_lower_int_samplers.h"
+#endif // HAVE_GALLIUM_D3D12_GRAPHICS
+
+#include "pipe/p_context.h"
+#include "pipe/p_state.h"
+#include "util/list.h"
+#include "util/slab.h"
+#include "util/u_framebuffer.h"
+#include "util/u_suballoc.h"
+#include "util/u_threaded_context.h"
+struct d3d12_context_queue_priority_manager;
+
+#define D3D12_GFX_SHADER_STAGES (MESA_SHADER_STAGES - 1)
+
+enum d3d12_dirty_flags
+{
+   D3D12_DIRTY_NONE             = 0,
+   D3D12_DIRTY_BLEND            = (1 << 0),
+   D3D12_DIRTY_RASTERIZER       = (1 << 1),
+   D3D12_DIRTY_ZSA              = (1 << 2),
+   D3D12_DIRTY_VERTEX_ELEMENTS  = (1 << 3),
+   D3D12_DIRTY_BLEND_COLOR      = (1 << 4),
+   D3D12_DIRTY_STENCIL_REF      = (1 << 5),
+   D3D12_DIRTY_SAMPLE_MASK      = (1 << 6),
+   D3D12_DIRTY_VIEWPORT         = (1 << 7),
+   D3D12_DIRTY_FRAMEBUFFER      = (1 << 8),
+   D3D12_DIRTY_SCISSOR          = (1 << 9),
+   D3D12_DIRTY_VERTEX_BUFFERS   = (1 << 10),
+   D3D12_DIRTY_INDEX_BUFFER     = (1 << 11),
+   D3D12_DIRTY_PRIM_MODE        = (1 << 12),
+   D3D12_DIRTY_SHADER           = (1 << 13),
+   D3D12_DIRTY_ROOT_SIGNATURE   = (1 << 14),
+   D3D12_DIRTY_STREAM_OUTPUT    = (1 << 15),
+   D3D12_DIRTY_STRIP_CUT_VALUE  = (1 << 16),
+   D3D12_DIRTY_COMPUTE_SHADER   = (1 << 17),
+   D3D12_DIRTY_COMPUTE_ROOT_SIGNATURE = (1 << 18),
+};
+
+enum d3d12_shader_dirty_flags
+{
+   D3D12_SHADER_DIRTY_CONSTBUF      = (1 << 0),
+   D3D12_SHADER_DIRTY_SAMPLER_VIEWS = (1 << 1),
+   D3D12_SHADER_DIRTY_SAMPLERS      = (1 << 2),
+   D3D12_SHADER_DIRTY_SSBO          = (1 << 3),
+   D3D12_SHADER_DIRTY_IMAGE         = (1 << 4),
+};
+
+#define D3D12_DIRTY_GFX_PSO (D3D12_DIRTY_BLEND | D3D12_DIRTY_RASTERIZER | D3D12_DIRTY_ZSA | \
+                             D3D12_DIRTY_FRAMEBUFFER | D3D12_DIRTY_SAMPLE_MASK | \
+                             D3D12_DIRTY_VERTEX_ELEMENTS | D3D12_DIRTY_PRIM_MODE | \
+                             D3D12_DIRTY_SHADER | D3D12_DIRTY_ROOT_SIGNATURE | \
+                             D3D12_DIRTY_STRIP_CUT_VALUE | D3D12_DIRTY_STREAM_OUTPUT)
+#define D3D12_DIRTY_COMPUTE_PSO (D3D12_DIRTY_COMPUTE_SHADER | D3D12_DIRTY_COMPUTE_ROOT_SIGNATURE)
+
+#define D3D12_DIRTY_COMPUTE_MASK (D3D12_DIRTY_COMPUTE_SHADER | D3D12_DIRTY_COMPUTE_ROOT_SIGNATURE)
+#define D3D12_DIRTY_GFX_MASK ~D3D12_DIRTY_COMPUTE_MASK
+
+
+#define D3D12_SHADER_DIRTY_ALL (D3D12_SHADER_DIRTY_CONSTBUF | D3D12_SHADER_DIRTY_SAMPLER_VIEWS | \
+                                D3D12_SHADER_DIRTY_SAMPLERS | D3D12_SHADER_DIRTY_SSBO | \
+                                D3D12_SHADER_DIRTY_IMAGE)
+
+enum d3d12_binding_type {
+   D3D12_BINDING_CONSTANT_BUFFER,
+   D3D12_BINDING_SHADER_RESOURCE_VIEW,
+   D3D12_BINDING_SAMPLER,
+   D3D12_BINDING_STATE_VARS,
+   D3D12_BINDING_SSBO,
+   D3D12_BINDING_IMAGE,
+   D3D12_NUM_BINDING_TYPES
+};
+
+struct d3d12_sampler_state {
+   struct d3d12_descriptor_handle handle, handle_without_shadow;
+   bool is_integer_texture;
+   bool is_shadow_sampler;
+   enum pipe_tex_wrap wrap_r;
+   enum pipe_tex_wrap wrap_s;
+   enum pipe_tex_wrap wrap_t;
+   enum pipe_tex_filter filter;
+   float lod_bias;
+   float min_lod, max_lod;
+   float border_color[4];
+   enum pipe_compare_func compare_func;
+};
+
+enum d3d12_blend_factor_flags {
+   D3D12_BLEND_FACTOR_NONE  = 0,
+   D3D12_BLEND_FACTOR_COLOR = 1 << 0,
+   D3D12_BLEND_FACTOR_ALPHA = 1 << 1,
+   D3D12_BLEND_FACTOR_ANY   = 1 << 2,
+};
+
+struct d3d12_sampler_view {
+   struct pipe_sampler_view base;
+   struct d3d12_descriptor_handle handle;
+   unsigned mip_levels;
+   unsigned array_size;
+   unsigned texture_generation_id;
+   unsigned swizzle_override_r:3;         /**< PIPE_SWIZZLE_x for red component */
+   unsigned swizzle_override_g:3;         /**< PIPE_SWIZZLE_x for green component */
+   unsigned swizzle_override_b:3;         /**< PIPE_SWIZZLE_x for blue component */
+   unsigned swizzle_override_a:3;         /**< PIPE_SWIZZLE_x for alpha component */
+};
+
+static inline struct d3d12_sampler_view *
+d3d12_sampler_view(struct pipe_sampler_view *pview)
+{
+   return (struct d3d12_sampler_view *)pview;
+}
+
+struct d3d12_stream_output_target {
+   struct pipe_stream_output_target base;
+   struct pipe_resource *fill_buffer;
+   unsigned fill_buffer_offset;
+};
+
+struct d3d12_shader_state {
+   struct d3d12_shader *current;
+   unsigned state_dirty;
+};
+
+struct blitter_context;
+struct primconvert_context;
+
+#ifdef _WIN32
+struct dxil_validator;
+#endif
+
+#ifdef __cplusplus
+class ResourceStateManager;
+#endif
+
+#define D3D12_CONTEXT_NO_ID 0xffffffff
+
+struct d3d12_context {
+   struct pipe_context base;
+
+   unsigned id;
+   struct slab_child_pool transfer_pool;
+   struct slab_child_pool transfer_pool_unsync;
+   struct list_head context_list_entry;
+   struct threaded_context *threaded_context;
+   struct d3d12_batch batches[8];
+   unsigned current_batch_idx;
+   struct util_dynarray recently_destroyed_bos;
+   struct util_dynarray barrier_scratch;
+   struct set *pending_barriers_bos;
+   struct util_dynarray local_pending_barriers_bos;
+   uint64_t submit_id;
+   ID3D12GraphicsCommandList *cmdlist;
+   ID3D12GraphicsCommandList2 *cmdlist2;
+   ID3D12GraphicsCommandList8 *cmdlist8;
+   ID3D12GraphicsCommandList *state_fixup_cmdlist;
+   struct hash_table_u64 *bo_state_table;
+   struct set *local_state_bos;
+   struct blitter_context *blitter;
+   uint flags;
+   bool queries_disabled;
+   bool has_commands;
+
+#ifdef __cplusplus
+   ResourceStateManager *resource_state_manager;
+#else
+   void *resource_state_manager; /* opaque pointer; we don't know about classes in C */
+#endif
+
+#ifdef HAVE_GALLIUM_D3D12_GRAPHICS
+   struct primconvert_context *primconvert;
+   struct u_suballocator query_allocator;
+   struct u_suballocator so_allocator;
+   struct hash_table *pso_cache;
+   struct hash_table *compute_pso_cache;
+   struct hash_table *root_signature_cache;
+   struct hash_table *cmd_signature_cache;
+   struct hash_table *gs_variant_cache;
+   struct hash_table *tcs_variant_cache;
+   struct hash_table *compute_transform_cache;
+
+   struct pipe_constant_buffer cbufs[MESA_SHADER_STAGES][PIPE_MAX_CONSTANT_BUFFERS];
+   struct d3d12_surface *fb_cbufs[PIPE_MAX_COLOR_BUFS];
+   struct d3d12_surface *fb_zsbuf;
+   struct pipe_framebuffer_state fb;
+   struct pipe_vertex_buffer vbs[PIPE_MAX_ATTRIBS];
+   D3D12_VERTEX_BUFFER_VIEW vbvs[PIPE_MAX_ATTRIBS];
+   unsigned num_vbs;
+   float flip_y;
+   bool need_zero_one_depth_range;
+   enum mesa_prim initial_api_prim;
+   struct pipe_viewport_state viewport_states[PIPE_MAX_VIEWPORTS];
+   D3D12_VIEWPORT viewports[PIPE_MAX_VIEWPORTS];
+   unsigned num_viewports;
+   struct pipe_scissor_state scissor_states[PIPE_MAX_VIEWPORTS];
+   D3D12_RECT scissors[PIPE_MAX_VIEWPORTS];
+   float blend_factor[4];
+   struct pipe_stencil_ref stencil_ref;
+   struct pipe_sampler_view *sampler_views[MESA_SHADER_STAGES][PIPE_MAX_SHADER_SAMPLER_VIEWS];
+   unsigned num_sampler_views[MESA_SHADER_STAGES];
+   unsigned has_int_samplers;
+   struct pipe_shader_buffer ssbo_views[MESA_SHADER_STAGES][PIPE_MAX_SHADER_BUFFERS];
+   unsigned num_ssbo_views[MESA_SHADER_STAGES];
+   struct pipe_image_view image_views[MESA_SHADER_STAGES][PIPE_MAX_SHADER_IMAGES];
+   enum pipe_format image_view_emulation_formats[MESA_SHADER_STAGES][PIPE_MAX_SHADER_IMAGES];
+   unsigned num_image_views[MESA_SHADER_STAGES];
+   struct d3d12_sampler_state *samplers[MESA_SHADER_STAGES][PIPE_MAX_SAMPLERS];
+   unsigned num_samplers[MESA_SHADER_STAGES];
+   D3D12_INDEX_BUFFER_VIEW ibv;
+
+   dxil_wrap_sampler_state tex_wrap_states[MESA_SHADER_STAGES][PIPE_MAX_SHADER_SAMPLER_VIEWS];
+   dxil_wrap_sampler_state tex_wrap_states_shader_key[PIPE_MAX_SHADER_SAMPLER_VIEWS];
+
+   dxil_texture_swizzle_state tex_swizzle_state[MESA_SHADER_STAGES][PIPE_MAX_SHADER_SAMPLER_VIEWS];
+   enum compare_func tex_compare_func[MESA_SHADER_STAGES][PIPE_MAX_SHADER_SAMPLER_VIEWS];
+
+   struct {
+      bool enabled;
+      uint32_t pattern[32];
+      struct pipe_resource *texture;
+      struct pipe_sampler_view *sampler_view;
+      struct d3d12_sampler_state *sampler_cso;
+   } pstipple;
+
+   struct pipe_stream_output_target *so_targets[PIPE_MAX_SO_BUFFERS];
+   D3D12_STREAM_OUTPUT_BUFFER_VIEW so_buffer_views[PIPE_MAX_SO_BUFFERS];
+   struct pipe_stream_output_target *fake_so_targets[PIPE_MAX_SO_BUFFERS];
+   D3D12_STREAM_OUTPUT_BUFFER_VIEW fake_so_buffer_views[PIPE_MAX_SO_BUFFERS];
+   unsigned fake_so_buffer_factor;
+   uint8_t patch_vertices;
+   float default_outer_tess_factor[4];
+   float default_inner_tess_factor[2];
+
+   struct d3d12_shader_selector *gfx_stages[D3D12_GFX_SHADER_STAGES];
+   struct d3d12_shader_selector *compute_state;
+
+   bool has_flat_varyings;
+   bool missing_dual_src_outputs;
+   bool manual_depth_range;
+
+   struct d3d12_gfx_pipeline_state gfx_pipeline_state;
+   struct d3d12_compute_pipeline_state compute_pipeline_state;
+   unsigned shader_dirty[MESA_SHADER_STAGES];
+   unsigned state_dirty;
+   unsigned cmdlist_dirty;
+   ID3D12PipelineState *current_gfx_pso;
+   ID3D12PipelineState *current_compute_pso;
+   uint16_t reverse_depth_range;
+
+   struct list_head active_queries;
+
+   struct d3d12_descriptor_pool *sampler_pool;
+   struct d3d12_descriptor_handle null_sampler;
+
+   PFN_D3D12_SERIALIZE_VERSIONED_ROOT_SIGNATURE D3D12SerializeVersionedRootSignature;
+#ifndef _GAMING_XBOX
+   ID3D12DeviceConfiguration *dev_config;
+#endif
+#ifdef _WIN32
+   struct dxil_validator *dxil_validator;
+#endif
+
+   struct d3d12_resource *current_predication;
+   bool predication_condition;
+   bool queries_suspended;
+
+   uint32_t transform_state_vars[8];
+   struct pipe_query *timestamp_query;
+
+   /* used by d3d12_blit.cpp */
+   void *stencil_resolve_vs, *stencil_resolve_fs, *stencil_resolve_fs_no_flip, *sampler_state;
+#endif // HAVE_GALLIUM_D3D12_GRAPHICS
+
+   struct d3d12_context_queue_priority_manager* priority_manager; // Object passed and managed by frontend
+   mtx_t priority_manager_lock; // Mutex to protect access to priority_manager
+
+   uint32_t max_video_encoding_async_depth = 0u;
+};
+
+static inline struct d3d12_context *
+d3d12_context(struct pipe_context *context)
+{
+   return (struct d3d12_context *)context;
+}
+
+static inline struct d3d12_batch *
+d3d12_current_batch(struct d3d12_context *ctx)
+{
+   assert(ctx->current_batch_idx < ARRAY_SIZE(ctx->batches));
+   return ctx->batches + ctx->current_batch_idx;
+}
+
+#define d3d12_foreach_submitted_batch(ctx, batch) \
+   unsigned oldest = (ctx->current_batch_idx + 1) % ARRAY_SIZE(ctx->batches); \
+   while (ctx->batches[oldest].fence == NULL && oldest != ctx->current_batch_idx) \
+      oldest = (oldest + 1) % ARRAY_SIZE(ctx->batches); \
+   struct d3d12_batch *batch = &ctx->batches[oldest]; \
+   for (; oldest != ctx->current_batch_idx; \
+        oldest = (oldest + 1) % ARRAY_SIZE(ctx->batches), \
+        batch = &ctx->batches[oldest])
+
+struct pipe_context *
+d3d12_context_create(struct pipe_screen *pscreen, void *priv, unsigned flags);
+
+int
+d3d12_context_set_queue_priority_manager(struct pipe_context *ctx, struct d3d12_context_queue_priority_manager *priority_manager);
+
+#ifdef HAVE_GALLIUM_D3D12_VIDEO
+int
+d3d12_video_encoder_set_max_async_queue_depth(struct pipe_context *ctx, uint32_t max_async_depth);
+
+int
+d3d12_video_encoder_get_last_slice_completion_fence(struct pipe_video_codec *codec,
+                                                    void *feedback,
+                                                    pipe_fence_handle **last_slice_completion_fence);
+#endif // HAVE_GALLIUM_D3D12_VIDEO
+
+bool
+d3d12_enable_fake_so_buffers(struct d3d12_context *ctx, unsigned factor);
+
+bool
+d3d12_disable_fake_so_buffers(struct d3d12_context *ctx);
+
+bool
+d3d12_flush_cmdlist(struct d3d12_context *ctx);
+
+void
+d3d12_flush_cmdlist_and_wait(struct d3d12_context *ctx);
+
+
+enum d3d12_transition_flags {
+   D3D12_TRANSITION_FLAG_NONE = 0,
+   D3D12_TRANSITION_FLAG_INVALIDATE_BINDINGS = 1,
+   D3D12_TRANSITION_FLAG_ACCUMULATE_STATE = 2,
+   D3D12_TRANSITION_FLAG_PENDING_MEMORY_BARRIER = 4,
+};
+
+void
+d3d12_transition_resource_state(struct d3d12_context* ctx,
+                                struct d3d12_resource* res,
+                                D3D12_RESOURCE_STATES state,
+                                d3d12_transition_flags flags);
+
+void
+d3d12_transition_subresources_state(struct d3d12_context *ctx,
+                                    struct d3d12_resource *res,
+                                    unsigned start_level, unsigned num_levels,
+                                    unsigned start_layer, unsigned num_layers,
+                                    unsigned start_plane, unsigned num_planes,
+                                    D3D12_RESOURCE_STATES state,
+                                    d3d12_transition_flags flags);
+
+void
+d3d12_apply_resource_states(struct d3d12_context* ctx, bool is_implicit_dispatch);
+
+void
+d3d12_draw_vbo(struct pipe_context *pctx,
+               const struct pipe_draw_info *dinfo,
+               unsigned drawid_offset,
+               const struct pipe_draw_indirect_info *indirect,
+               const struct pipe_draw_start_count_bias *draws,
+               unsigned num_draws);
+
+void
+d3d12_launch_grid(struct pipe_context *pctx,
+                  const struct pipe_grid_info *info);
+
+void
+d3d12_blit(struct pipe_context *pctx,
+           const struct pipe_blit_info *info);
+
+void
+d3d12_context_query_init(struct pipe_context *pctx);
+
+bool
+d3d12_need_zero_one_depth_range(struct d3d12_context *ctx);
+
+void
+d3d12_init_sampler_view_descriptor(struct d3d12_sampler_view *sampler_view);
+
+void
+d3d12_invalidate_context_bindings(struct d3d12_context *ctx, struct d3d12_resource *res);
+
+void
+d3d12_rebind_buffer(struct d3d12_context *ctx, struct d3d12_resource *res);
+
+void
+d3d12_init_null_sampler(struct d3d12_context *ctx);
+
+bool
+d3d12_init_polygon_stipple(struct pipe_context *pctx);
+
+void
+d3d12_init_graphics_context_functions(struct d3d12_context *ctx);
+
+#ifdef HAVE_GALLIUM_D3D12_VIDEO
+struct pipe_video_codec* d3d12_video_create_codec( struct pipe_context *context,
+                                                const struct pipe_video_codec *t);
+#endif
+
+#endif

src/gallium/drivers/d3d12/d3d12_context_common.cpp

@@ -0,0 +1,689 @@
+/*
+ * Copyright © Microsoft Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#include "d3d12_blit.h"
+#include "d3d12_cmd_signature.h"
+#include "d3d12_context.h"
+#ifdef HAVE_GALLIUM_D3D12_GRAPHICS
+#include "d3d12_compiler.h"
+#include "d3d12_compute_transforms.h"
+#include "nir_to_dxil.h"
+#ifdef _WIN32
+#include "dxil_validator.h"
+#endif
+#endif // HAVE_GALLIUM_D3D12_GRAPHICS
+#include "d3d12_debug.h"
+#include "d3d12_fence.h"
+#include "d3d12_format.h"
+#include "d3d12_query.h"
+#include "d3d12_resource.h"
+#include "d3d12_root_signature.h"
+#include "d3d12_screen.h"
+#include "d3d12_surface.h"
+#ifdef HAVE_GALLIUM_D3D12_VIDEO
+#include "d3d12_video_dec.h"
+#include "d3d12_video_enc.h"
+#include "d3d12_video_proc.h"
+#include "d3d12_video_buffer.h"
+#endif
+#include "indices/u_primconvert.h"
+#include "util/u_atomic.h"
+#include "util/u_blitter.h"
+#include "util/set.h"
+#include "util/u_dual_blend.h"
+#include "util/u_framebuffer.h"
+#include "util/u_helpers.h"
+#include "util/u_inlines.h"
+#include "util/u_memory.h"
+#include "util/u_upload_mgr.h"
+#include "util/u_pstipple.h"
+#include "util/u_sample_positions.h"
+#include "util/u_dl.h"
+#include <dxguids/dxguids.h>
+#include <string.h>
+#include "d3d12_interop_public.h"
+
+#ifndef _GAMING_XBOX
+#include <wrl/client.h>
+using Microsoft::WRL::ComPtr;
+#endif
+
+static void
+d3d12_context_destroy(struct pipe_context *pctx)
+{
+   struct d3d12_context *ctx = d3d12_context(pctx);
+
+   if (ctx->priority_manager)
+   {
+      struct d3d12_screen *screen = d3d12_screen(pctx->screen);
+      if (ctx->priority_manager->unregister_work_queue(ctx->priority_manager, screen->cmdqueue) != 0)
+      {
+         debug_printf("D3D12: Failed to unregister command queue with frontend priority manager\n");
+         assert(false);
+      }
+
+      mtx_destroy(&ctx->priority_manager_lock);
+   }
+
+   struct d3d12_screen *screen = d3d12_screen(pctx->screen);
+
+#ifdef HAVE_GALLIUM_D3D12_GRAPHICS
+   if ((screen->max_feature_level >= D3D_FEATURE_LEVEL_11_0) && !(ctx->flags & PIPE_CONTEXT_MEDIA_ONLY)) {
+      util_blitter_destroy(ctx->blitter); // Must be called before d3d12_destroy_batch
+   }
+#endif // HAVE_GALLIUM_D3D12_GRAPHICS
+
+   // Batch must be destroyed before the rest of the state objects below
+   d3d12_end_batch(ctx, d3d12_current_batch(ctx));
+   for (unsigned i = 0; i < ARRAY_SIZE(ctx->batches); ++i)
+      d3d12_destroy_batch(ctx, &ctx->batches[i]);
+   ctx->cmdlist->Release();
+   if (ctx->cmdlist2)
+      ctx->cmdlist2->Release();
+   if (ctx->cmdlist8)
+      ctx->cmdlist8->Release();
+
+   mtx_lock(&screen->submit_mutex);
+   list_del(&ctx->context_list_entry);
+   if (ctx->id != D3D12_CONTEXT_NO_ID) {
+      unsigned context_bit = 1u << ctx->id;
+      set_foreach(ctx->local_state_bos, entry) {
+         struct d3d12_bo *bo = (struct d3d12_bo *)entry->key;
+         if (bo->local_context_state_mask & context_bit) {
+            d3d12_destroy_context_state_table_entry(&bo->local_context_states[ctx->id]);
+            bo->local_context_state_mask &= ~context_bit;
+         }
+      }
+      _mesa_set_clear(ctx->local_state_bos, nullptr);
+      screen->context_id_list[screen->context_id_count++] = ctx->id;
+   }
+   mtx_unlock(&screen->submit_mutex);
+
+#ifdef HAVE_GALLIUM_D3D12_GRAPHICS
+   if ((screen->max_feature_level >= D3D_FEATURE_LEVEL_11_0) && !(ctx->flags & PIPE_CONTEXT_MEDIA_ONLY)) {
+#ifdef _WIN32
+      dxil_destroy_validator(ctx->dxil_validator);
+#endif // _WIN32
+
+#ifndef _GAMING_XBOX
+      if (ctx->dev_config)
+         ctx->dev_config->Release();
+#endif // _GAMING_XBOX
+
+      if (ctx->timestamp_query)
+         pctx->destroy_query(pctx, ctx->timestamp_query);
+
+      util_unreference_framebuffer_state(&ctx->fb);
+      d3d12_compute_pipeline_state_cache_destroy(ctx);
+      d3d12_root_signature_cache_destroy(ctx);
+      d3d12_cmd_signature_cache_destroy(ctx);
+      d3d12_compute_transform_cache_destroy(ctx);
+      d3d12_descriptor_pool_free(ctx->sampler_pool);
+      d3d12_gs_variant_cache_destroy(ctx);
+      d3d12_tcs_variant_cache_destroy(ctx);
+      d3d12_gfx_pipeline_state_cache_destroy(ctx);
+      util_primconvert_destroy(ctx->primconvert);
+      pipe_resource_reference(&ctx->pstipple.texture, nullptr);
+      pipe_sampler_view_reference(&ctx->pstipple.sampler_view, nullptr);
+      util_dynarray_fini(&ctx->recently_destroyed_bos);
+      FREE(ctx->pstipple.sampler_cso);
+      if (pctx->stream_uploader)
+         u_upload_destroy(pctx->stream_uploader);
+      if (pctx->const_uploader)
+         u_upload_destroy(pctx->const_uploader);
+      if (!ctx->queries_disabled) {
+         u_suballocator_destroy(&ctx->query_allocator);
+      }  
+   }
+#endif // HAVE_GALLIUM_D3D12_GRAPHICS
+
+   slab_destroy_child(&ctx->transfer_pool);
+   slab_destroy_child(&ctx->transfer_pool_unsync);
+   d3d12_context_state_table_destroy(ctx);
+
+   FREE(ctx);
+}
+
+bool
+d3d12_flush_cmdlist(struct d3d12_context *ctx)
+{
+   if (!ctx->has_commands)
+      return false;
+
+   d3d12_end_batch(ctx, d3d12_current_batch(ctx));
+
+   ctx->current_batch_idx++;
+   if (ctx->current_batch_idx == ARRAY_SIZE(ctx->batches))
+      ctx->current_batch_idx = 0;
+
+   d3d12_start_batch(ctx, d3d12_current_batch(ctx));
+   ctx->has_commands = false;
+   return true;
+}
+
+void
+d3d12_flush_cmdlist_and_wait(struct d3d12_context *ctx)
+{
+   struct d3d12_batch *batch = d3d12_current_batch(ctx);
+
+   d3d12_foreach_submitted_batch(ctx, old_batch)
+      d3d12_reset_batch(ctx, old_batch, OS_TIMEOUT_INFINITE);
+   if (d3d12_flush_cmdlist(ctx))
+      d3d12_reset_batch(ctx, batch, OS_TIMEOUT_INFINITE);
+}
+
+static void
+d3d12_flush(struct pipe_context *pipe,
+            struct pipe_fence_handle **fence,
+            unsigned flags)
+{
+   struct d3d12_context *ctx = d3d12_context(pipe);
+   struct d3d12_batch *batch = d3d12_current_batch(ctx);
+
+   bool flushed = d3d12_flush_cmdlist(ctx);
+
+   if (fence) {
+      if (flushed)
+         d3d12_fence_reference((struct d3d12_fence **)fence, batch->fence);
+      else
+         *fence = (pipe_fence_handle *)d3d12_create_fence(d3d12_screen(ctx->base.screen), false);
+   }
+}
+
+static void
+d3d12_flush_resource(struct pipe_context *pctx,
+                     struct pipe_resource *pres)
+{
+   struct d3d12_context *ctx = d3d12_context(pctx);
+   struct d3d12_resource *res = d3d12_resource(pres);
+
+   d3d12_transition_resource_state(ctx, res,
+                                   D3D12_RESOURCE_STATE_COMMON,
+                                   D3D12_TRANSITION_FLAG_INVALIDATE_BINDINGS);
+   d3d12_apply_resource_states(ctx, false);
+   d3d12_batch_reference_resource(d3d12_current_batch(ctx), res, true);
+   ctx->has_commands = true;
+}
+
+static void
+d3d12_signal(struct pipe_context *pipe,
+             struct pipe_fence_handle *pfence,
+             uint64_t value)
+{
+   struct d3d12_screen *screen = d3d12_screen(pipe->screen);
+   struct d3d12_fence *fence = d3d12_fence(pfence);
+   d3d12_flush_cmdlist(d3d12_context(pipe));
+   d3d12_fence_signal_impl(fence, screen->cmdqueue, value);
+}
+
+static void
+d3d12_wait(struct pipe_context *pipe, struct pipe_fence_handle *pfence, uint64_t value)
+{
+   struct d3d12_screen *screen = d3d12_screen(pipe->screen);
+   struct d3d12_fence *fence = d3d12_fence(pfence);
+   d3d12_flush_cmdlist(d3d12_context(pipe));
+   d3d12_fence_wait_impl(fence, screen->cmdqueue, value);
+}
+
+static void
+d3d12_replace_buffer_storage(struct pipe_context *pctx,
+   struct pipe_resource *pdst,
+   struct pipe_resource *psrc,
+   unsigned minimum_num_rebinds,
+   uint32_t rebind_mask,
+   uint32_t delete_buffer_id)
+{
+   struct d3d12_resource *dst = d3d12_resource(pdst);
+   struct d3d12_resource *src = d3d12_resource(psrc);
+
+   struct d3d12_bo *old_bo = dst->bo;
+   d3d12_bo_reference(src->bo);
+   dst->bo = src->bo;
+   p_atomic_inc(&dst->generation_id);
+#ifdef HAVE_GALLIUM_D3D12_GRAPHICS
+   struct d3d12_context *ctx = d3d12_context(pctx);
+   if ((d3d12_screen(pctx->screen)->max_feature_level >= D3D_FEATURE_LEVEL_11_0)
+      && !(ctx->flags & PIPE_CONTEXT_MEDIA_ONLY))
+      d3d12_rebind_buffer(ctx, dst);
+#endif // HAVE_GALLIUM_D3D12_GRAPHICS
+   d3d12_bo_unreference(old_bo);
+}
+
+static void
+d3d12_memory_barrier(struct pipe_context *pctx, unsigned flags)
+{
+#ifdef HAVE_GALLIUM_D3D12_GRAPHICS
+   struct d3d12_context *ctx = d3d12_context(pctx);
+   if (flags & PIPE_BARRIER_VERTEX_BUFFER)
+      ctx->state_dirty |= D3D12_DIRTY_VERTEX_BUFFERS;
+   if (flags & PIPE_BARRIER_INDEX_BUFFER)
+      ctx->state_dirty |= D3D12_DIRTY_INDEX_BUFFER;
+   if (flags & PIPE_BARRIER_FRAMEBUFFER)
+      ctx->state_dirty |= D3D12_DIRTY_FRAMEBUFFER;
+   if (flags & PIPE_BARRIER_STREAMOUT_BUFFER)
+      ctx->state_dirty |= D3D12_DIRTY_STREAM_OUTPUT;
+
+   /* TODO:
+    * PIPE_BARRIER_INDIRECT_BUFFER
+    */
+
+   for (unsigned i = 0; i < D3D12_GFX_SHADER_STAGES; ++i) {
+      if (flags & PIPE_BARRIER_CONSTANT_BUFFER)
+         ctx->shader_dirty[i] |= D3D12_SHADER_DIRTY_CONSTBUF;
+      if (flags & PIPE_BARRIER_TEXTURE)
+         ctx->shader_dirty[i] |= D3D12_SHADER_DIRTY_SAMPLER_VIEWS;
+      if (flags & PIPE_BARRIER_SHADER_BUFFER)
+         ctx->shader_dirty[i] |= D3D12_SHADER_DIRTY_SSBO;
+      if (flags & PIPE_BARRIER_IMAGE)
+         ctx->shader_dirty[i] |= D3D12_SHADER_DIRTY_IMAGE;
+   }
+   
+   /* Indicate that UAVs shouldn't override transitions. Ignore barriers that are only
+    * for UAVs or other fixed-function state that doesn't need a draw to resolve.
+    */
+   const unsigned ignored_barrier_flags =
+      PIPE_BARRIER_IMAGE |
+      PIPE_BARRIER_SHADER_BUFFER |
+      PIPE_BARRIER_UPDATE |
+      PIPE_BARRIER_MAPPED_BUFFER |
+      PIPE_BARRIER_QUERY_BUFFER;
+   d3d12_current_batch(ctx)->pending_memory_barrier = (flags & ~ignored_barrier_flags) != 0;
+
+   if (flags & (PIPE_BARRIER_IMAGE | PIPE_BARRIER_SHADER_BUFFER)) {
+      D3D12_RESOURCE_BARRIER uavBarrier;
+      uavBarrier.Type = D3D12_RESOURCE_BARRIER_TYPE_UAV;
+      uavBarrier.Flags = D3D12_RESOURCE_BARRIER_FLAG_NONE;
+      uavBarrier.UAV.pResource = nullptr;
+      ctx->cmdlist->ResourceBarrier(1, &uavBarrier);
+      ctx->has_commands = true;
+   }
+#endif // HAVE_GALLIUM_D3D12_GRAPHICS
+}
+
+static void
+d3d12_texture_barrier(struct pipe_context *pctx, unsigned flags)
+{
+   struct d3d12_context *ctx = d3d12_context(pctx);
+
+   /* D3D doesn't really have an equivalent in the legacy barrier model. When using enhanced barriers,
+    * this could be a more specific global barrier. But for now, just flush the world with an aliasing barrier. */
+   D3D12_RESOURCE_BARRIER aliasingBarrier;
+   aliasingBarrier.Type = D3D12_RESOURCE_BARRIER_TYPE_ALIASING;
+   aliasingBarrier.Flags = D3D12_RESOURCE_BARRIER_FLAG_NONE;
+   aliasingBarrier.Aliasing.pResourceBefore = nullptr;
+   aliasingBarrier.Aliasing.pResourceAfter = nullptr;
+   ctx->cmdlist->ResourceBarrier(1, &aliasingBarrier);
+   ctx->has_commands = true;
+}
+
+static enum pipe_reset_status
+d3d12_get_reset_status(struct pipe_context *pctx)
+{
+   struct d3d12_screen *screen = d3d12_screen(pctx->screen);
+   HRESULT hr = screen->dev->GetDeviceRemovedReason();
+   switch (hr) {
+   case DXGI_ERROR_DEVICE_HUNG:
+   case DXGI_ERROR_INVALID_CALL:
+      return PIPE_GUILTY_CONTEXT_RESET;
+   case DXGI_ERROR_DEVICE_RESET:
+      return PIPE_INNOCENT_CONTEXT_RESET;
+   default:
+      return SUCCEEDED(hr) ? PIPE_NO_RESET : PIPE_UNKNOWN_CONTEXT_RESET;
+   }
+}
+
+#ifdef HAVE_GALLIUM_D3D12_VIDEO
+struct pipe_video_codec*
+d3d12_video_create_codec(struct pipe_context *context,
+                         const struct pipe_video_codec *templat)
+{
+    if (templat->entrypoint == PIPE_VIDEO_ENTRYPOINT_ENCODE) {
+        return d3d12_video_encoder_create_encoder(context, templat);
+    } else if (templat->entrypoint == PIPE_VIDEO_ENTRYPOINT_BITSTREAM) {
+        return d3d12_video_create_decoder(context, templat);
+    } else if (templat->entrypoint == PIPE_VIDEO_ENTRYPOINT_PROCESSING) {
+        return d3d12_video_processor_create(context, templat);
+    } else {
+        debug_printf("D3D12: Unsupported video codec entrypoint %d\n", templat->entrypoint);
+        return nullptr;
+    }
+}
+#endif
+
+int
+d3d12_context_set_queue_priority(struct d3d12_context_queue_priority_manager* manager,
+                                     ID3D12CommandQueue *d3d12_queue,
+                                     const uint32_t* global_priority,
+                                     const uint32_t* local_priority)
+{
+   if (!global_priority || !local_priority)
+      return -1;
+
+   struct d3d12_context* ctx12 = d3d12_context(manager->context);
+   mtx_lock(&ctx12->priority_manager_lock);
+   {
+      // Set the queue priority
+      ComPtr<ID3D12CommandQueue1> prio_iface;
+      if(FAILED(d3d12_queue->QueryInterface(IID_PPV_ARGS(&prio_iface))))
+      {
+         mtx_unlock(&ctx12->priority_manager_lock);
+         return -1;
+      }
+
+      D3D12_COMMAND_QUEUE_GLOBAL_PRIORITY global_priority_val = static_cast<D3D12_COMMAND_QUEUE_GLOBAL_PRIORITY>(*global_priority);
+      if(FAILED(prio_iface->SetGlobalPriority(global_priority_val)))
+      {
+         mtx_unlock(&ctx12->priority_manager_lock);
+         return -1;
+      }
+
+      D3D12_COMMAND_QUEUE_PROCESS_PRIORITY local_priority_val = static_cast<D3D12_COMMAND_QUEUE_PROCESS_PRIORITY>(*local_priority);
+      if(FAILED(prio_iface->SetProcessPriority(local_priority_val)))
+      {
+         mtx_unlock(&ctx12->priority_manager_lock);
+         return -1;
+      }
+   }
+
+   mtx_unlock(&ctx12->priority_manager_lock);
+   return 0;
+}
+
+int
+d3d12_context_get_queue_priority(struct d3d12_context_queue_priority_manager* manager,
+                                     ID3D12CommandQueue *d3d12_queue,
+                                     uint32_t *global_priority,
+                                     uint32_t *local_priority)
+{
+   struct d3d12_context* ctx12 = d3d12_context(manager->context);
+
+   mtx_lock(&ctx12->priority_manager_lock);
+   {
+      ComPtr<ID3D12CommandQueue1> prio_iface;
+      if (FAILED(d3d12_queue->QueryInterface(IID_PPV_ARGS(&prio_iface))))
+      {
+         mtx_unlock(&ctx12->priority_manager_lock);
+         return -1;
+      }
+
+      if (global_priority)
+      {
+         D3D12_COMMAND_QUEUE_GLOBAL_PRIORITY global_priority_val = {};
+         if (FAILED(prio_iface->GetGlobalPriority(&global_priority_val)))
+         {
+            mtx_unlock(&ctx12->priority_manager_lock);
+            return -1;
+         }
+         *global_priority = static_cast<uint32_t>(global_priority_val);
+      }
+
+      if (local_priority)
+      {
+         D3D12_COMMAND_QUEUE_PROCESS_PRIORITY local_priority_val = {};
+         if (FAILED(prio_iface->GetProcessPriority(&local_priority_val)))
+         {
+            mtx_unlock(&ctx12->priority_manager_lock);
+            return -1;
+         }
+         *local_priority = static_cast<uint32_t>(local_priority_val);
+      }
+   }
+   mtx_unlock(&ctx12->priority_manager_lock);
+   return 0;
+}
+
+int
+d3d12_context_set_queue_priority_manager(struct pipe_context *ctx, struct d3d12_context_queue_priority_manager *priority_manager)
+{
+   if (ctx && priority_manager)
+   {
+      struct d3d12_context *d3d12_ctx = d3d12_context(ctx);
+      d3d12_ctx->priority_manager = (struct d3d12_context_queue_priority_manager*) priority_manager;
+
+      // Validate that the frontend has set all required function pointers
+      assert ( d3d12_ctx->priority_manager->register_work_queue );
+      assert ( d3d12_ctx->priority_manager->unregister_work_queue );
+
+      // Initialize the priority manager lock
+      if (thrd_success != mtx_init(&d3d12_ctx->priority_manager_lock, mtx_plain))
+      {
+         debug_printf("D3D12: Failed to initialize context priority manager lock\n");
+         return 1;
+      }
+
+      //
+      // Fill the function pointers for the context queue priority manager that the frontend expects
+      //
+
+      d3d12_ctx->priority_manager->context = ctx;
+      d3d12_ctx->priority_manager->set_queue_priority = d3d12_context_set_queue_priority;
+      d3d12_ctx->priority_manager->get_queue_priority = d3d12_context_get_queue_priority;
+
+      // Register the context's command queue with the frontend's priority manager
+      struct d3d12_screen *screen = d3d12_screen(ctx->screen);
+      if (d3d12_ctx->priority_manager->register_work_queue(d3d12_ctx->priority_manager, screen->cmdqueue) != 0)
+      {
+         debug_printf("D3D12: Failed to register command queue with frontend priority manager\n");
+         return 1;
+      }
+   }
+
+   return 0;
+}
+
+#ifdef HAVE_GALLIUM_D3D12_VIDEO
+int
+d3d12_video_encoder_set_max_async_queue_depth(struct pipe_context *ctx, uint32_t max_async_depth)
+{
+   if (max_async_depth > 8) {
+      debug_printf("d3d12_video_encoder_set_max_async_queue_depth: max_async_depth must be between 1 and 8\n");
+      return -1;
+   }
+
+   struct d3d12_context *d3d12_ctx = d3d12_context(ctx);
+   d3d12_ctx->max_video_encoding_async_depth = max_async_depth;
+   return 0;
+}
+#endif // HAVE_GALLIUM_D3D12_VIDEO
+
+struct pipe_context *
+d3d12_context_create(struct pipe_screen *pscreen, void *priv, unsigned flags)
+{
+   struct d3d12_screen *screen = d3d12_screen(pscreen);
+   if (FAILED(screen->dev->GetDeviceRemovedReason())) {
+      /* Attempt recovery, but this may fail */
+      screen->deinit(screen);
+      if (!screen->init(screen)) {
+         debug_printf("D3D12: failed to reset screen\n");
+         return nullptr;
+      }
+   }
+
+   if ((screen->max_feature_level < D3D_FEATURE_LEVEL_11_0) &&
+      !(flags & PIPE_CONTEXT_MEDIA_ONLY))
+   {
+      debug_printf("D3D12: Underlying screen maximum supported feature level is lower than D3D_FEATURE_LEVEL_11_0. The caller to context_create must pass PIPE_CONTEXT_MEDIA_ONLY in flags.\n");
+      return NULL;
+   }
+
+#ifndef HAVE_GALLIUM_D3D12_VIDEO
+   if (flags & PIPE_CONTEXT_MEDIA_ONLY)
+   {
+      debug_printf("D3D12: context_create passed PIPE_CONTEXT_MEDIA_ONLY in flags but no media support found.\n");
+      return NULL;
+   }
+#endif // ifndef HAVE_GALLIUM_D3D12_VIDEO
+
+   struct d3d12_context *ctx = CALLOC_STRUCT(d3d12_context);
+   if (!ctx)
+      return NULL;
+
+   ctx->max_video_encoding_async_depth = static_cast<uint32_t>(debug_get_num_option("D3D12_VIDEO_ENC_ASYNC_DEPTH", 8));
+   ctx->base.screen = pscreen;
+   ctx->base.priv = priv;
+
+   ctx->base.destroy = d3d12_context_destroy;
+   ctx->base.flush = d3d12_flush;
+   ctx->base.flush_resource = d3d12_flush_resource;
+   ctx->base.fence_server_signal = d3d12_signal;
+   ctx->base.fence_server_sync = d3d12_wait;
+   ctx->base.memory_barrier = d3d12_memory_barrier;
+   ctx->base.texture_barrier = d3d12_texture_barrier;
+
+   ctx->base.get_device_reset_status = d3d12_get_reset_status;
+   ctx->flags = flags;
+   d3d12_context_resource_init(&ctx->base);
+   d3d12_context_copy_init(&ctx->base);
+
+#ifdef HAVE_GALLIUM_D3D12_VIDEO
+   ctx->base.create_video_codec = d3d12_video_create_codec;
+   ctx->base.create_video_buffer = d3d12_video_buffer_create;
+   ctx->base.video_buffer_from_handle = d3d12_video_buffer_from_handle;
+#endif
+
+   slab_create_child(&ctx->transfer_pool, &d3d12_screen(pscreen)->transfer_pool);
+   slab_create_child(&ctx->transfer_pool_unsync, &d3d12_screen(pscreen)->transfer_pool);
+
+   d3d12_context_state_table_init(ctx);
+
+   ctx->queries_disabled = true; // Disabled by default, re-enable if supported FL below
+
+#ifdef HAVE_GALLIUM_D3D12_GRAPHICS
+   if ((screen->max_feature_level >= D3D_FEATURE_LEVEL_11_0) && !(flags & PIPE_CONTEXT_MEDIA_ONLY)) {
+#ifndef _GAMING_XBOX
+      (void)screen->dev->QueryInterface(&ctx->dev_config);
+#endif
+
+      d3d12_context_blit_init(&ctx->base);
+
+      u_suballocator_init(&ctx->so_allocator, &ctx->base, 4096, 0,
+                     PIPE_USAGE_DEFAULT,
+                     0, false);
+
+      ctx->has_flat_varyings = false;
+      ctx->missing_dual_src_outputs = false;
+      ctx->manual_depth_range = false;
+      
+      d3d12_compute_pipeline_state_cache_init(ctx);
+      d3d12_root_signature_cache_init(ctx);
+      d3d12_cmd_signature_cache_init(ctx);
+      d3d12_compute_transform_cache_init(ctx);
+
+      ctx->D3D12SerializeVersionedRootSignature =
+         (PFN_D3D12_SERIALIZE_VERSIONED_ROOT_SIGNATURE)util_dl_get_proc_address(screen->d3d12_mod, "D3D12SerializeVersionedRootSignature");
+
+      ctx->base.stream_uploader = u_upload_create_default(&ctx->base);
+      ctx->base.const_uploader = u_upload_create_default(&ctx->base);
+
+      ctx->base.get_sample_position = u_default_get_sample_position;
+      ctx->base.get_sample_position = u_default_get_sample_position;
+
+     d3d12_init_graphics_context_functions(ctx);
+
+      ctx->gfx_pipeline_state.sample_mask = ~0;
+
+      d3d12_context_query_init(&ctx->base);
+      ctx->queries_disabled = false;
+
+      struct primconvert_config cfg = {};
+      cfg.primtypes_mask = 1 << MESA_PRIM_POINTS |
+                           1 << MESA_PRIM_LINES |
+                           1 << MESA_PRIM_LINE_STRIP |
+                           1 << MESA_PRIM_TRIANGLES |
+                           1 << MESA_PRIM_TRIANGLE_STRIP;
+      cfg.restart_primtypes_mask = cfg.primtypes_mask;
+      cfg.fixed_prim_restart = true;
+      ctx->primconvert = util_primconvert_create_config(&ctx->base, &cfg);
+      if (!ctx->primconvert) {
+         debug_printf("D3D12: failed to create primconvert\n");
+         return NULL;
+      }
+
+      d3d12_gfx_pipeline_state_cache_init(ctx);
+      d3d12_gs_variant_cache_init(ctx);
+      d3d12_tcs_variant_cache_init(ctx);
+
+      ctx->sampler_pool = d3d12_descriptor_pool_new(screen,
+                                                   D3D12_DESCRIPTOR_HEAP_TYPE_SAMPLER,
+                                                   64);
+      if (!ctx->sampler_pool) {
+         FREE(ctx);
+         return NULL;
+      }
+      d3d12_init_null_sampler(ctx);
+
+      ctx->blitter = util_blitter_create(&ctx->base);
+      if (!ctx->blitter)
+         return NULL;
+
+      if (!d3d12_init_polygon_stipple(&ctx->base)) {
+         debug_printf("D3D12: failed to initialize polygon stipple resources\n");
+         FREE(ctx);
+         return NULL;
+      }
+#ifdef _WIN32
+         if (!(d3d12_debug & D3D12_DEBUG_EXPERIMENTAL) ||
+            (d3d12_debug & D3D12_DEBUG_DISASS))
+            ctx->dxil_validator = dxil_create_validator(NULL);
+#endif
+   }
+#endif // HAVE_GALLIUM_D3D12_GRAPHICS
+
+   ctx->submit_id = (uint64_t)p_atomic_add_return(&screen->ctx_count, 1) << 32ull;
+
+   for (unsigned i = 0; i < ARRAY_SIZE(ctx->batches); ++i) {
+      if (!d3d12_init_batch(ctx, &ctx->batches[i])) {
+         FREE(ctx);
+         return NULL;
+      }
+   }
+   d3d12_start_batch(ctx, &ctx->batches[0]);
+
+   mtx_lock(&screen->submit_mutex);
+   list_addtail(&ctx->context_list_entry, &screen->context_list);
+   if (screen->context_id_count > 0)
+      ctx->id = screen->context_id_list[--screen->context_id_count];
+   else
+      ctx->id = D3D12_CONTEXT_NO_ID;
+   mtx_unlock(&screen->submit_mutex);
+
+   for (unsigned i = 0; i < ARRAY_SIZE(ctx->batches); ++i) {
+      ctx->batches[i].ctx_id = ctx->id;
+      ctx->batches[i].ctx_index = i;
+   }
+
+   if (flags & PIPE_CONTEXT_PREFER_THREADED) {
+      struct pipe_context *ret = threaded_context_create(&ctx->base,
+         &screen->transfer_pool,
+         d3d12_replace_buffer_storage,
+         NULL,
+         &ctx->threaded_context);
+      if (ctx->threaded_context) {
+         ctx->threaded_context->bytes_replaced_limit = 1024 * 1024 * 1024; /* 1GiB */
+         threaded_context_init_bytes_mapped_limit(ctx->threaded_context, 4);
+      }
+      return ret;
+   }
+
+   return &ctx->base;
+}

src/gallium/drivers/d3d12/d3d12_copy.cpp

@@ -0,0 +1,363 @@
+/*
+ * Copyright © Microsoft Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#include "d3d12_context.h"
+#include "d3d12_debug.h"
+#include "d3d12_format.h"
+#include "d3d12_query.h"
+#include "d3d12_resource.h"
+#include "d3d12_screen.h"
+
+#include "util/u_blitter.h"
+#include "util/format/u_format.h"
+
+static void
+copy_buffer_region_no_barriers(struct d3d12_context *ctx,
+                               struct d3d12_resource *dst,
+                               uint64_t dst_offset,
+                               struct d3d12_resource *src,
+                               uint64_t src_offset,
+                               uint64_t size)
+{
+   uint64_t dst_off, src_off;
+   ID3D12Resource *dst_buf = d3d12_resource_underlying(dst, &dst_off);
+   ID3D12Resource *src_buf = d3d12_resource_underlying(src, &src_off);
+
+   ctx->cmdlist->CopyBufferRegion(dst_buf, dst_offset + dst_off,
+                                  src_buf, src_offset + src_off,
+                                  size);
+   ctx->has_commands = true;
+}
+
+inline static unsigned
+get_subresource_id(enum pipe_texture_target target, unsigned subres, unsigned stride,
+                   unsigned z, unsigned *updated_z, unsigned array_size, unsigned plane_slice)
+{
+   if (d3d12_subresource_id_uses_layer(target)) {
+      subres += stride * z;
+      if (updated_z)
+         *updated_z = 0;
+   }
+   return subres + plane_slice * array_size * stride;
+}
+
+static void
+copy_subregion_no_barriers(struct d3d12_context *ctx,
+                           struct d3d12_resource *dst,
+                           unsigned dst_level,
+                           unsigned dstx, unsigned dsty, unsigned dstz,
+                           struct d3d12_resource *src,
+                           unsigned src_level,
+                           const struct pipe_box *psrc_box,
+                           unsigned mask)
+{
+   UNUSED struct d3d12_screen *screen = d3d12_screen(ctx->base.screen);
+   D3D12_TEXTURE_COPY_LOCATION src_loc, dst_loc;
+   unsigned src_z = psrc_box->z;
+
+   int src_subres_stride = src->base.b.last_level + 1;
+   int dst_subres_stride = dst->base.b.last_level + 1;
+
+   int src_array_size = src->base.b.array_size;
+   int dst_array_size = dst->base.b.array_size;
+
+   int stencil_src_res_offset = 1;
+   int stencil_dst_res_offset = 1;
+
+   int src_nres = 1;
+   int dst_nres = 1;
+
+   if (dst->base.b.format == PIPE_FORMAT_Z24_UNORM_S8_UINT ||
+       dst->base.b.format == PIPE_FORMAT_S8_UINT_Z24_UNORM ||
+       dst->base.b.format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT) {
+      stencil_dst_res_offset = dst_subres_stride * dst_array_size;
+      src_nres = 2;
+   }
+
+   if (src->base.b.format == PIPE_FORMAT_Z24_UNORM_S8_UINT ||
+       src->base.b.format == PIPE_FORMAT_S8_UINT_Z24_UNORM ||
+       dst->base.b.format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT) {
+      stencil_src_res_offset = src_subres_stride * src_array_size;
+      dst_nres = 2;
+   }
+
+   static_assert(PIPE_MASK_S == 0x20 && PIPE_MASK_Z == 0x10, "unexpected ZS format mask");
+   int nsubres = MIN2(src_nres, dst_nres);
+   unsigned subresource_copy_mask = nsubres > 1 ? mask >> 4 : 1;
+
+   for (int subres = 0; subres < nsubres; ++subres) {
+
+      if (!(subresource_copy_mask & (1 << subres)))
+         continue;
+
+      src_loc.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX;
+      src_loc.SubresourceIndex = get_subresource_id(src->base.b.target, src_level, src_subres_stride, src_z, &src_z, src_array_size, src->plane_slice) +
+                                 subres * stencil_src_res_offset;
+      src_loc.pResource = d3d12_resource_resource(src);
+
+      dst_loc.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX;
+      dst_loc.SubresourceIndex = get_subresource_id(dst->base.b.target, dst_level, dst_subres_stride, dstz, &dstz, dst_array_size, dst->plane_slice) +
+                                 subres * stencil_dst_res_offset;
+      dst_loc.pResource = d3d12_resource_resource(dst);
+
+      if (psrc_box->x == 0 && psrc_box->y == 0 && psrc_box->z == 0 &&
+          psrc_box->width == (int)u_minify(src->base.b.width0, src_level) &&
+          psrc_box->height == (int)u_minify(src->base.b.height0, src_level) &&
+          psrc_box->depth == (int)u_minify(src->base.b.depth0, src_level)) {
+
+         assert((dstx == 0 && dsty == 0 && dstz == 0) ||
+                screen->opts2.ProgrammableSamplePositionsTier !=
+                D3D12_PROGRAMMABLE_SAMPLE_POSITIONS_TIER_NOT_SUPPORTED ||
+                (!util_format_is_depth_or_stencil(dst->base.b.format) &&
+                 !util_format_is_depth_or_stencil(src->base.b.format) &&
+                  dst->base.b.nr_samples == src->base.b.nr_samples));
+
+         ctx->cmdlist->CopyTextureRegion(&dst_loc, dstx, dsty, dstz,
+                                         &src_loc, NULL);
+         ctx->has_commands = true;
+
+      } else {
+         D3D12_BOX src_box;
+         src_box.left = psrc_box->x;
+         src_box.right = MIN2(psrc_box->x + psrc_box->width, (int)u_minify(src->base.b.width0, src_level));
+         src_box.top = psrc_box->y;
+         src_box.bottom = MIN2(psrc_box->y + psrc_box->height, (int)u_minify(src->base.b.height0, src_level));
+         src_box.front = src_z;
+         src_box.back = src_z + psrc_box->depth;
+
+         assert((screen->opts2.ProgrammableSamplePositionsTier !=
+                 D3D12_PROGRAMMABLE_SAMPLE_POSITIONS_TIER_NOT_SUPPORTED ||
+                 (!util_format_is_depth_or_stencil(dst->base.b.format) &&
+                  !util_format_is_depth_or_stencil(src->base.b.format))) &&
+                dst->base.b.nr_samples == src->base.b.nr_samples);
+
+         ctx->cmdlist->CopyTextureRegion(&dst_loc, dstx, dsty, dstz,
+                                         &src_loc, &src_box);
+         ctx->has_commands = true;
+      }
+   }
+}
+
+static void
+copy_resource_y_flipped_no_barriers(struct d3d12_context *ctx,
+                                    struct d3d12_resource *dst,
+                                    unsigned dst_level,
+                                    const struct pipe_box *pdst_box,
+                                    struct d3d12_resource *src,
+                                    unsigned src_level,
+                                    const struct pipe_box *psrc_box,
+                                    unsigned mask)
+{
+   if (D3D12_DEBUG_BLIT & d3d12_debug) {
+      debug_printf("D3D12 BLIT as COPY: from %s@%d %dx%dx%d + %dx%dx%d\n",
+                   util_format_name(src->base.b.format), src_level,
+                   psrc_box->x, psrc_box->y, psrc_box->z,
+                   psrc_box->width, psrc_box->height, psrc_box->depth);
+      debug_printf("      to   %s@%d %dx%dx%d\n",
+                   util_format_name(dst->base.b.format), dst_level,
+                   pdst_box->x, pdst_box->y, pdst_box->z);
+   }
+
+   struct pipe_box src_box = *psrc_box;
+   int src_inc = psrc_box->height > 0 ? 1 : -1;
+   int dst_inc = pdst_box->height > 0 ? 1 : -1;
+   src_box.height = 1;
+   int rows_to_copy = abs(psrc_box->height);
+
+   if (psrc_box->height < 0)
+      --src_box.y;
+
+   for (int y = 0, dest_y = pdst_box->y; y < rows_to_copy;
+        ++y, src_box.y += src_inc, dest_y += dst_inc) {
+      copy_subregion_no_barriers(ctx, dst, dst_level,
+                                 pdst_box->x, dest_y, pdst_box->z,
+                                 src, src_level, &src_box, mask);
+   }
+}
+
+void
+d3d12_direct_copy(struct d3d12_context *ctx,
+                  struct d3d12_resource *dst,
+                  unsigned dst_level,
+                  const struct pipe_box *pdst_box,
+                  struct d3d12_resource *src,
+                  unsigned src_level,
+                  const struct pipe_box *psrc_box,
+                  unsigned mask)
+{
+   struct d3d12_batch *batch = d3d12_current_batch(ctx);
+
+   unsigned src_subres = get_subresource_id(src->base.b.target, src_level, src->base.b.last_level + 1,
+                                            psrc_box->z, nullptr, src->base.b.array_size, src->plane_slice);
+   unsigned dst_subres = get_subresource_id(dst->base.b.target, dst_level, dst->base.b.last_level + 1,
+                                            pdst_box->z, nullptr, dst->base.b.array_size, dst->plane_slice);
+
+   if (D3D12_DEBUG_BLIT & d3d12_debug)
+      debug_printf("BLIT: Direct copy from subres %d to subres  %d\n",
+                   src_subres, dst_subres);
+
+   d3d12_transition_subresources_state(ctx, src, src_subres, 1, 0, 1,
+                                       d3d12_get_format_start_plane(src->base.b.format),
+                                       d3d12_get_format_num_planes(src->base.b.format),
+                                       D3D12_RESOURCE_STATE_COPY_SOURCE,
+                                       D3D12_TRANSITION_FLAG_INVALIDATE_BINDINGS);
+
+   d3d12_transition_subresources_state(ctx, dst, dst_subres, 1, 0, 1,
+                                       d3d12_get_format_start_plane(dst->base.b.format),
+                                       d3d12_get_format_num_planes(dst->base.b.format),
+                                       D3D12_RESOURCE_STATE_COPY_DEST,
+                                       D3D12_TRANSITION_FLAG_INVALIDATE_BINDINGS);
+
+   d3d12_apply_resource_states(ctx, false);
+
+   d3d12_batch_reference_resource(batch, src, false);
+   d3d12_batch_reference_resource(batch, dst, true);
+
+   if (src->base.b.target == PIPE_BUFFER) {
+      copy_buffer_region_no_barriers(ctx, dst, pdst_box->x,
+                                     src, psrc_box->x, psrc_box->width);
+   } else if (psrc_box->height == pdst_box->height) {
+      /* No flipping, we can forward this directly to resource_copy_region */
+      copy_subregion_no_barriers(ctx, dst, dst_level,
+                                 pdst_box->x, pdst_box->y, pdst_box->z,
+                                 src, src_level, psrc_box, mask);
+   } else {
+      assert(psrc_box->height == -pdst_box->height);
+      copy_resource_y_flipped_no_barriers(ctx, dst, dst_level, pdst_box,
+                                          src, src_level, psrc_box, mask);
+   }
+}
+
+struct pipe_resource *
+create_staging_resource(struct d3d12_context *ctx,
+                        struct d3d12_resource *src,
+                        unsigned src_level,
+                        const struct pipe_box *src_box,
+                        struct pipe_box *dst_box,
+                        unsigned mask)
+
+{
+   struct pipe_resource templ = {};
+   struct pipe_resource *staging_res;
+   struct pipe_box copy_src;
+
+   u_box_3d(MIN2(src_box->x, src_box->x + src_box->width),
+            MIN2(src_box->y, src_box->y + src_box->height),
+            MIN2(src_box->z, src_box->z + src_box->depth),
+            abs(src_box->width), abs(src_box->height), abs(src_box->depth),
+            &copy_src);
+
+   templ.format = src->base.b.format;
+   templ.width0 = copy_src.width;
+   templ.height0 = static_cast<uint16_t>(copy_src.height);
+   templ.depth0 = copy_src.depth;
+   templ.array_size = 1;
+   templ.nr_samples = src->base.b.nr_samples;
+   templ.nr_storage_samples = src->base.b.nr_storage_samples;
+   templ.usage = PIPE_USAGE_STAGING;
+   templ.bind = util_format_is_depth_or_stencil(templ.format) ? PIPE_BIND_DEPTH_STENCIL :
+      util_format_is_compressed(templ.format) ? 0 : PIPE_BIND_RENDER_TARGET;
+   templ.target = src->base.b.target;
+
+   staging_res = ctx->base.screen->resource_create(ctx->base.screen, &templ);
+
+   dst_box->x = 0;
+   dst_box->y = 0;
+   dst_box->z = 0;
+   dst_box->width = copy_src.width;
+   dst_box->height = copy_src.height;
+   dst_box->depth = copy_src.depth;
+
+   d3d12_direct_copy(ctx, d3d12_resource(staging_res), 0, dst_box,
+                     src, src_level, &copy_src, mask);
+
+   if (src_box->width < 0) {
+      dst_box->x = dst_box->width;
+      dst_box->width = src_box->width;
+   }
+
+   if (src_box->height < 0) {
+      dst_box->y = dst_box->height;
+      dst_box->height = src_box->height;
+   }
+
+   if (src_box->depth < 0) {
+      dst_box->z = dst_box->depth;
+      dst_box->depth = src_box->depth;
+   }
+   return staging_res;
+}
+
+static void
+d3d12_resource_copy_region(struct pipe_context *pctx,
+                           struct pipe_resource *pdst,
+                           unsigned dst_level,
+                           unsigned dstx, unsigned dsty, unsigned dstz,
+                           struct pipe_resource *psrc,
+                           unsigned src_level,
+                           const struct pipe_box *psrc_box)
+{
+   struct d3d12_context *ctx = d3d12_context(pctx);
+   struct d3d12_resource *dst = d3d12_resource(pdst);
+   struct d3d12_resource *src = d3d12_resource(psrc);
+   struct pipe_resource *staging_res = NULL;
+   const struct pipe_box *src_box = psrc_box;
+   struct pipe_box staging_box, dst_box;
+
+   if (D3D12_DEBUG_BLIT & d3d12_debug) {
+      debug_printf("D3D12 COPY: from %s@%d msaa:%d mips:%d %dx%dx%d + %dx%dx%d\n",
+                   util_format_name(psrc->format), src_level, psrc->nr_samples,
+                   psrc->last_level,
+                   psrc_box->x, psrc_box->y, psrc_box->z,
+                   psrc_box->width, psrc_box->height, psrc_box->depth);
+      debug_printf("            to   %s@%d msaa:%d mips:%d %dx%dx%d\n",
+                   util_format_name(pdst->format), dst_level, psrc->nr_samples,
+                   psrc->last_level, dstx, dsty, dstz);
+   }
+
+   /* Use an intermediate resource if copying from/to the same subresource */
+   if (d3d12_resource_resource(dst) == d3d12_resource_resource(src) && dst_level == src_level) {
+      staging_res = create_staging_resource(ctx, src, src_level, psrc_box, &staging_box, PIPE_MASK_RGBAZS);
+      src = d3d12_resource(staging_res);
+      src_level = 0;
+      src_box = &staging_box;
+   }
+
+   dst_box.x = dstx;
+   dst_box.y = dsty;
+   dst_box.z = static_cast<int16_t>(dstz);
+   dst_box.width = psrc_box->width;
+   dst_box.height = psrc_box->height;
+
+   d3d12_direct_copy(ctx, dst, dst_level, &dst_box,
+                     src, src_level, src_box, PIPE_MASK_RGBAZS);
+
+   if (staging_res)
+      pipe_resource_reference(&staging_res, NULL);
+}
+
+void
+d3d12_context_copy_init(struct pipe_context *ctx)
+{
+   ctx->resource_copy_region = d3d12_resource_copy_region;
+}

src/gallium/drivers/d3d12/d3d12_debug.h

@@ -0,0 +1,50 @@
+/*
+ * Copyright © Microsoft Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifndef D3D12_DEBUG_H
+#define D3D12_DEBUG_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+
+#define D3D12_DEBUG_VERBOSE       (1 << 0)
+#define D3D12_DEBUG_EXPERIMENTAL  (1 << 1)
+#define D3D12_DEBUG_DXIL          (1 << 2)
+#define D3D12_DEBUG_DISASS        (1 << 3)
+#define D3D12_DEBUG_BLIT          (1 << 4)
+#define D3D12_DEBUG_RESOURCE      (1 << 5)
+#define D3D12_DEBUG_DEBUG_LAYER   (1 << 6)
+#define D3D12_DEBUG_GPU_VALIDATOR (1 << 7)
+#define D3D12_DEBUG_SINGLETON     (1 << 8)
+#define D3D12_DEBUG_PIX           (1 << 9)
+
+extern uint32_t d3d12_debug;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/gallium/drivers/d3d12/d3d12_descriptor_pool.cpp

@@ -0,0 +1,243 @@
+/*
+ * Copyright © Microsoft Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#include "d3d12_context.h"
+#include "d3d12_descriptor_pool.h"
+#include "d3d12_screen.h"
+
+#include "pipe/p_context.h"
+#include "pipe/p_state.h"
+
+#include "util/list.h"
+#include "util/u_dynarray.h"
+#include "util/u_memory.h"
+
+#include <dxguids/dxguids.h>
+
+struct d3d12_descriptor_pool {
+   ID3D12Device *dev;
+   D3D12_DESCRIPTOR_HEAP_TYPE type;
+   uint32_t num_descriptors;
+   list_head heaps;
+};
+
+struct d3d12_descriptor_heap {
+   struct d3d12_descriptor_pool *pool;
+
+   D3D12_DESCRIPTOR_HEAP_DESC desc;
+   ID3D12Device *dev;
+   ID3D12DescriptorHeap *heap;
+   uint32_t desc_size;
+   size_t cpu_base;
+   uint64_t gpu_base;
+   uint32_t size;
+   uint32_t next;
+   util_dynarray free_list;
+   list_head link;
+};
+
+struct d3d12_descriptor_heap*
+d3d12_descriptor_heap_new(ID3D12Device *dev,
+                          D3D12_DESCRIPTOR_HEAP_TYPE type,
+                          D3D12_DESCRIPTOR_HEAP_FLAGS flags,
+                          uint32_t num_descriptors)
+{
+   struct d3d12_descriptor_heap *heap = CALLOC_STRUCT(d3d12_descriptor_heap);
+
+   heap->desc.NumDescriptors = num_descriptors;
+   heap->desc.Type = type;
+   heap->desc.Flags = flags;
+   if (FAILED(dev->CreateDescriptorHeap(&heap->desc,
+                                        IID_PPV_ARGS(&heap->heap)))) {
+      FREE(heap);
+      return NULL;
+   }
+
+   heap->dev = dev;
+   heap->desc_size = dev->GetDescriptorHandleIncrementSize(type);
+   heap->size = num_descriptors * heap->desc_size;
+   heap->cpu_base = GetCPUDescriptorHandleForHeapStart(heap->heap).ptr;
+   if (flags & D3D12_DESCRIPTOR_HEAP_FLAG_SHADER_VISIBLE)
+      heap->gpu_base = GetGPUDescriptorHandleForHeapStart(heap->heap).ptr;
+   heap->free_list = UTIL_DYNARRAY_INIT;
+
+   return heap;
+}
+
+void
+d3d12_descriptor_heap_free(struct d3d12_descriptor_heap *heap)
+{
+   heap->heap->Release();
+   util_dynarray_fini(&heap->free_list);
+   FREE(heap);
+}
+
+ID3D12DescriptorHeap*
+d3d12_descriptor_heap_get(struct d3d12_descriptor_heap *heap)
+{
+   return heap->heap;
+}
+
+static uint32_t
+d3d12_descriptor_heap_is_online(struct d3d12_descriptor_heap *heap)
+{
+   return (heap->desc.Flags & D3D12_DESCRIPTOR_HEAP_FLAG_SHADER_VISIBLE) ? 1 : 0;
+}
+
+static uint32_t
+d3d12_descriptor_heap_can_allocate(struct d3d12_descriptor_heap *heap)
+{
+   return (heap->free_list.size > 0 ||
+           heap->size >= heap->next + heap->desc_size);
+}
+
+uint32_t
+d3d12_descriptor_heap_get_remaining_handles(struct d3d12_descriptor_heap *heap)
+{
+   return (heap->size - heap->next) / heap->desc_size;
+}
+
+void
+d2d12_descriptor_heap_get_next_handle(struct d3d12_descriptor_heap *heap,
+                                      struct d3d12_descriptor_handle *handle)
+{
+   handle->heap = heap;
+   handle->cpu_handle.ptr = heap->cpu_base + heap->next;
+   handle->gpu_handle.ptr = d3d12_descriptor_heap_is_online(heap) ?
+                            heap->gpu_base + heap->next : 0;
+}
+
+uint32_t
+d3d12_descriptor_heap_alloc_handle(struct d3d12_descriptor_heap *heap,
+                                   struct d3d12_descriptor_handle *handle)
+{
+   uint32_t offset = 0;
+
+   assert(handle != NULL);
+
+   if (heap->free_list.size > 0) {
+      offset = util_dynarray_pop(&heap->free_list, uint32_t);
+   } else if (heap->size >= heap->next + heap->desc_size) {
+      offset = heap->next;
+      heap->next += heap->desc_size;
+   } else {
+      /* Todo: we should add a new descriptor heap to get more handles */
+      assert(0 && "No handles available in descriptor heap");
+      return 0;
+   }
+
+   handle->heap = heap;
+   handle->cpu_handle.ptr = heap->cpu_base + offset;
+   handle->gpu_handle.ptr = d3d12_descriptor_heap_is_online(heap) ?
+                            heap->gpu_base + offset : 0;
+
+   return 1;
+}
+
+void
+d3d12_descriptor_handle_free(struct d3d12_descriptor_handle *handle)
+{
+   const uint32_t index = static_cast<uint32_t>(handle->cpu_handle.ptr - handle->heap->cpu_base);
+   if (index + handle->heap->desc_size == handle->heap->next) {
+      handle->heap->next = index;
+   } else {
+      util_dynarray_append(&handle->heap->free_list, index);
+   }
+
+   handle->heap = NULL;
+   handle->cpu_handle.ptr = 0;
+   handle->gpu_handle.ptr = 0;
+}
+
+void
+d3d12_descriptor_heap_append_handles(struct d3d12_descriptor_heap *heap,
+                                     D3D12_CPU_DESCRIPTOR_HANDLE *handles,
+                                     unsigned num_handles)
+{
+   D3D12_CPU_DESCRIPTOR_HANDLE dst;
+
+   assert(heap->next + (num_handles * heap->desc_size) <= heap->size);
+   dst.ptr = heap->cpu_base + heap->next;
+   heap->dev->CopyDescriptors(1, &dst, &num_handles,
+                              num_handles, handles, NULL,
+                              heap->desc.Type);
+   heap->next += num_handles * heap->desc_size;
+}
+
+void
+d3d12_descriptor_heap_clear(struct d3d12_descriptor_heap *heap)
+{
+   heap->next = 0;
+   util_dynarray_clear(&heap->free_list);
+}
+
+struct d3d12_descriptor_pool*
+d3d12_descriptor_pool_new(struct d3d12_screen *screen,
+                          D3D12_DESCRIPTOR_HEAP_TYPE type,
+                          uint32_t num_descriptors)
+{
+   struct d3d12_descriptor_pool *pool = CALLOC_STRUCT(d3d12_descriptor_pool);
+   if (!pool)
+      return NULL;
+
+   pool->dev = screen->dev;
+   pool->type = type;
+   pool->num_descriptors = num_descriptors;
+   list_inithead(&pool->heaps);
+
+   return pool;
+}
+
+void
+d3d12_descriptor_pool_free(struct d3d12_descriptor_pool *pool)
+{
+   list_for_each_entry_safe(struct d3d12_descriptor_heap, heap, &pool->heaps, link) {
+      list_del(&heap->link);
+      d3d12_descriptor_heap_free(heap);
+   }
+   FREE(pool);
+}
+
+uint32_t
+d3d12_descriptor_pool_alloc_handle(struct d3d12_descriptor_pool *pool,
+                                   struct d3d12_descriptor_handle *handle)
+{
+   struct d3d12_descriptor_heap *valid_heap = NULL;
+
+   list_for_each_entry(struct d3d12_descriptor_heap, heap, &pool->heaps, link) {
+      if (d3d12_descriptor_heap_can_allocate(heap)) {
+         valid_heap = heap;
+         break;
+      }
+   }
+
+   if (!valid_heap) {
+      valid_heap = d3d12_descriptor_heap_new(pool->dev,
+                                             pool->type,
+                                             D3D12_DESCRIPTOR_HEAP_FLAG_NONE,
+                                             pool->num_descriptors);
+      list_addtail(&valid_heap->link, &pool->heaps);
+   }
+
+   return d3d12_descriptor_heap_alloc_handle(valid_heap, handle);
+}

src/gallium/drivers/d3d12/d3d12_descriptor_pool.h

@@ -0,0 +1,95 @@
+/*
+ * Copyright © Microsoft Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifndef D3D12_DESCRIPTOR_POOL_H
+#define D3D12_DESCRIPTOR_POOL_H
+
+#include "d3d12_common.h"
+
+struct d3d12_descriptor_pool;
+struct d3d12_descriptor_heap;
+
+struct d3d12_descriptor_handle {
+    D3D12_CPU_DESCRIPTOR_HANDLE cpu_handle;
+    D3D12_GPU_DESCRIPTOR_HANDLE gpu_handle;
+    struct d3d12_descriptor_heap *heap;
+};
+
+inline bool
+d3d12_descriptor_handle_is_allocated(struct d3d12_descriptor_handle *handle)
+{
+    return (handle->heap != NULL);
+}
+
+void
+d3d12_descriptor_handle_free(struct d3d12_descriptor_handle *handle);
+
+/* Offline Descriptor Pool */
+
+struct d3d12_descriptor_pool*
+d3d12_descriptor_pool_new(struct d3d12_screen *screen,
+                          D3D12_DESCRIPTOR_HEAP_TYPE type,
+                          uint32_t num_descriptors);
+
+void
+d3d12_descriptor_pool_free(struct d3d12_descriptor_pool *pool);
+
+uint32_t
+d3d12_descriptor_pool_alloc_handle(struct d3d12_descriptor_pool *pool,
+                                   struct d3d12_descriptor_handle *handle);
+
+
+/* Online/Offline Descriptor Heaps */
+
+struct d3d12_descriptor_heap*
+d3d12_descriptor_heap_new(ID3D12Device *device,
+                          D3D12_DESCRIPTOR_HEAP_TYPE type,
+                          D3D12_DESCRIPTOR_HEAP_FLAGS flags,
+                          uint32_t num_descriptors);
+
+void
+d3d12_descriptor_heap_free(struct d3d12_descriptor_heap *heap);
+
+ID3D12DescriptorHeap*
+d3d12_descriptor_heap_get(struct d3d12_descriptor_heap *heap);
+
+void
+d2d12_descriptor_heap_get_next_handle(struct d3d12_descriptor_heap *heap,
+                                      struct d3d12_descriptor_handle *handle);
+
+uint32_t
+d3d12_descriptor_heap_get_remaining_handles(struct d3d12_descriptor_heap *heap);
+
+uint32_t
+d3d12_descriptor_heap_alloc_handle(struct d3d12_descriptor_heap *heap,
+                                   struct d3d12_descriptor_handle *handle);
+
+void
+d3d12_descriptor_heap_append_handles(struct d3d12_descriptor_heap *heap,
+                                     D3D12_CPU_DESCRIPTOR_HANDLE *handles,
+                                     unsigned num_handles);
+
+void
+d3d12_descriptor_heap_clear(struct d3d12_descriptor_heap *heap);
+
+#endif