init

src/poly/.clang-format

@@ -0,0 +1,8 @@
+
+BasedOnStyle: InheritParentConfig
+DisableFormat: false
+
+AlignConsecutiveBitFields: Consecutive
+ColumnLimit: 80
+BreakStringLiterals: false
+SpaceBeforeParens: ControlStatementsExceptControlMacros

src/poly/cl/geometry.cl

@@ -0,0 +1,377 @@
+/*
+ * Copyright 2023 Alyssa Rosenzweig
+ * Copyright 2023 Valve Corporation
+ * Copyright 2025 Collabora Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "compiler/libcl/libcl_vk.h"
+#include "poly/geometry.h"
+#include "poly/prim.h"
+#include "poly/tessellator.h"
+#include "util/macros.h"
+#include "util/u_math.h"
+
+uint64_t nir_ro_to_rw_poly(uint64_t address);
+
+static inline uint
+xfb_prim(uint id, uint n, uint copy)
+{
+   return sub_sat(id, n - 1u) + copy;
+}
+
+/*
+ * Determine whether an output vertex has an n'th copy in the transform feedback
+ * buffer. This is written weirdly to let constant folding remove unnecessary
+ * stores when length is known statically.
+ */
+bool
+poly_xfb_vertex_copy_in_strip(uint n, uint id, uint length, uint copy)
+{
+   uint prim = xfb_prim(id, n, copy);
+
+   int num_prims = length - (n - 1);
+   return copy == 0 || (prim < num_prims && id >= copy && copy < num_prims);
+}
+
+uint
+poly_xfb_vertex_offset(uint n, uint invocation_base_prim, uint strip_base_prim,
+                       uint id_in_strip, uint copy, bool flatshade_first)
+{
+   uint prim = xfb_prim(id_in_strip, n, copy);
+   uint vert_0 = min(id_in_strip, n - 1);
+   uint vert = vert_0 - copy;
+
+   if (n == 3) {
+      vert = poly_map_vertex_in_tri_strip(prim, vert, flatshade_first);
+   }
+
+   /* Tally up in the whole buffer */
+   uint base_prim = invocation_base_prim + strip_base_prim;
+   uint base_vertex = base_prim * n;
+   return base_vertex + (prim * n) + vert;
+}
+
+uint64_t
+poly_xfb_vertex_address(constant struct poly_geometry_params *p, uint index,
+                        uint buffer, uint stride, uint output_offset)
+{
+   uint xfb_offset = (index * stride) + output_offset;
+
+   return (uintptr_t)(p->xfb_base[buffer]) + xfb_offset;
+}
+
+uint
+poly_vertex_id_for_line_class(enum mesa_prim mode, uint prim, uint vert,
+                              uint num_prims)
+{
+   /* Line list, line strip, or line loop */
+   if (mode == MESA_PRIM_LINE_LOOP && prim == (num_prims - 1) && vert == 1)
+      return 0;
+
+   if (mode == MESA_PRIM_LINES)
+      prim *= 2;
+
+   return prim + vert;
+}
+
+uint
+poly_vertex_id_for_tri_class(enum mesa_prim mode, uint prim, uint vert,
+                             bool flatshade_first)
+{
+   if (flatshade_first && mode == MESA_PRIM_TRIANGLE_FAN) {
+      vert = vert + 1;
+      vert = (vert == 3) ? 0 : vert;
+   }
+
+   if (mode == MESA_PRIM_TRIANGLE_FAN && vert == 0)
+      return 0;
+
+   if (mode == MESA_PRIM_TRIANGLES)
+      prim *= 3;
+
+   /* Triangle list, triangle strip, or triangle fan */
+   if (mode == MESA_PRIM_TRIANGLE_STRIP) {
+      unsigned pv = flatshade_first ? 0 : 2;
+
+      bool even = (prim & 1) == 0;
+      bool provoking = vert == pv;
+
+      vert = ((provoking || even) ? vert : ((3 - pv) - vert));
+   }
+
+   return prim + vert;
+}
+
+uint
+poly_vertex_id_for_line_adj_class(enum mesa_prim mode, uint prim, uint vert)
+{
+   /* Line list adj or line strip adj */
+   if (mode == MESA_PRIM_LINES_ADJACENCY)
+      prim *= 4;
+
+   return prim + vert;
+}
+
+uint
+poly_vertex_id_for_tri_adj_class(enum mesa_prim mode, uint prim, uint vert,
+                                 uint nr, bool flatshade_first)
+{
+   /* Tri adj list or tri adj strip */
+   if (mode == MESA_PRIM_TRIANGLE_STRIP_ADJACENCY) {
+      return poly_vertex_id_for_tri_strip_adj(prim, vert, nr, flatshade_first);
+   } else {
+      return (6 * prim) + vert;
+   }
+}
+
+uint
+poly_map_to_line_adj(uint id)
+{
+   /* Sequence (1, 2), (5, 6), (9, 10), ... */
+   return ((id & ~1) * 2) + (id & 1) + 1;
+}
+
+uint
+poly_map_to_line_strip_adj(uint id)
+{
+   /* Sequence (1, 2), (2, 3), (4, 5), .. */
+   uint prim = id / 2;
+   uint vert = id & 1;
+   return prim + vert + 1;
+}
+
+uint
+poly_map_to_tri_strip_adj(uint id)
+{
+   /* Sequence (0, 2, 4), (2, 6, 4), (4, 6, 8), (6, 10, 8)
+    *
+    * Although tri strips with adjacency have 6 cases in general, after
+    * disregarding the vertices only available in a geometry shader, there are
+    * only even/odd cases. In other words, it's just a triangle strip subject to
+    * extra padding.
+    *
+    * Dividing through by two, the sequence is:
+    *
+    *   (0, 1, 2), (1, 3, 2), (2, 3, 4), (3, 5, 4)
+    */
+   uint prim = id / 3;
+   uint vtx = id % 3;
+
+   /* Flip the winding order of odd triangles */
+   if ((prim % 2) == 1) {
+      if (vtx == 1)
+         vtx = 2;
+      else if (vtx == 2)
+         vtx = 1;
+   }
+
+   return 2 * (prim + vtx);
+}
+
+uint
+poly_load_index_buffer(constant struct poly_vertex_params *p, uint id,
+                       uint index_size)
+{
+   return poly_load_index(p->index_buffer, p->index_buffer_range_el, id,
+                          index_size);
+}
+
+static uint
+setup_xfb_buffer(global struct poly_geometry_params *p, uint i, uint stride,
+                 uint max_output_end, uint vertices_per_prim)
+{
+   uint xfb_offset = *(p->xfb_offs_ptrs[i]);
+   p->xfb_base[i] = p->xfb_base_original[i] + xfb_offset;
+
+   /* Let output_end = output_offset + output_size.
+    *
+    * Primitive P will write up to (but not including) offset:
+    *
+    *    xfb_offset + ((P - 1) * (verts_per_prim * stride))
+    *               + ((verts_per_prim - 1) * stride)
+    *               + output_end
+    *
+    * To fit all outputs for P, that value must be less than the XFB
+    * buffer size for the output with maximal output_end, as everything
+    * else is constant here across outputs within a buffer/primitive:
+    *
+    *    floor(P) <= (stride + size - xfb_offset - output_end)
+    *                 // (stride * verts_per_prim)
+    */
+   int numer_s = p->xfb_size[i] + (stride - max_output_end) - xfb_offset;
+   uint numer = max(numer_s, 0);
+   return numer / (stride * vertices_per_prim);
+}
+
+void
+poly_write_strip(GLOBAL uint32_t *index_buffer, uint32_t inv_index_offset,
+                 uint32_t prim_index_offset, uint32_t vertex_offset,
+                 uint32_t verts_in_prim, uint3 info)
+{
+   _poly_write_strip(index_buffer, inv_index_offset + prim_index_offset,
+                     vertex_offset, verts_in_prim, info.x, info.y, info.z);
+}
+
+void
+poly_pad_index_gs(global int *index_buffer, uint inv_index_offset,
+                  uint nr_indices, uint alloc)
+{
+   for (uint i = nr_indices; i < alloc; ++i) {
+      index_buffer[inv_index_offset + i] = -1;
+   }
+}
+
+uintptr_t
+poly_vertex_output_buffer(constant struct poly_vertex_params *p)
+{
+   return p->output_buffer;
+}
+
+uint64_t
+poly_vertex_outputs(constant struct poly_vertex_params *p)
+{
+   return p->outputs;
+}
+
+uintptr_t
+poly_vertex_output_address(constant struct poly_vertex_params *p,
+                           uint64_t mask, uint vtx, gl_varying_slot location)
+{
+   /* Written like this to let address arithmetic work */
+   return p->output_buffer +
+      ((uintptr_t)poly_tcs_in_offs_el(vtx, location, mask)) * 16;
+}
+
+unsigned
+poly_index_size(constant struct poly_vertex_params *p)
+{
+   return p->index_size_B;
+}
+
+unsigned
+poly_input_vertices(constant struct poly_vertex_params *p)
+{
+   return p->verts_per_instance;
+}
+
+global uint *
+poly_load_xfb_count_address(constant struct poly_geometry_params *p, int index,
+                            int count_words, uint unrolled_id)
+{
+   return &p->count_buffer[(unrolled_id * count_words) + index];
+}
+
+uint
+poly_previous_xfb_primitives(global struct poly_geometry_params *p,
+                             int static_count, int count_index, int count_words,
+                             bool prefix_sum, uint unrolled_id)
+{
+   if (static_count >= 0) {
+      /* If the number of outputted vertices per invocation is known statically,
+       * we can calculate the base.
+       */
+      return unrolled_id * static_count;
+   } else {
+      /* Otherwise, load from the count buffer buffer. Note that the sums are
+       * inclusive, so index 0 is nonzero. This requires a little fixup here. We
+       * use a saturating unsigned subtraction so we don't read out-of-bounds.
+       *
+       * If we didn't prefix sum, there's only one element.
+       */
+      uint prim_minus_1 = prefix_sum ? sub_sat(unrolled_id, 1u) : 0;
+      uint count = p->count_buffer[(prim_minus_1 * count_words) + count_index];
+
+      return unrolled_id == 0 ? 0 : count;
+   }
+}
+
+/* Like u_foreach_bit, specialized for XFB to enable loop unrolling */
+#define poly_foreach_xfb(word, index)                                          \
+   for (uint i = 0; i < 4; ++i)                                                \
+      if (word & BITFIELD_BIT(i))
+
+void
+poly_pre_gs(global struct poly_geometry_params *p, uint streams,
+            uint buffers_written, uint4 buffer_to_stream, int4 count_index,
+            uint4 stride, uint4 output_end, int4 static_count, uint invocations,
+            uint vertices_per_prim, global uint *gs_invocations,
+            global uint *gs_primitives, global uint *c_primitives,
+            global uint *c_invocations)
+{
+   unsigned count_words = !!(count_index[0] >= 0) + !!(count_index[1] >= 0) +
+                          !!(count_index[2] >= 0) + !!(count_index[3] >= 0);
+   bool prefix_sum = count_words && buffers_written;
+   uint unrolled_in_prims = p->input_primitives;
+
+   /* Determine the number of primitives generated in each stream */
+   uint4 in_prims = 0;
+   poly_foreach_xfb(streams, i) {
+      in_prims[i] = poly_previous_xfb_primitives(p, static_count[i],
+                                                 count_index[i], count_words,
+                                                 prefix_sum, unrolled_in_prims);
+
+      *(p->prims_generated_counter[i]) += in_prims[i];
+   }
+
+   uint4 prims = in_prims;
+   uint emitted_prims = prims[0] + prims[1] + prims[2] + prims[3];
+
+   if (buffers_written) {
+      poly_foreach_xfb(buffers_written, i) {
+         uint max_prims =
+            setup_xfb_buffer(p, i, stride[i], output_end[i], vertices_per_prim);
+
+         unsigned stream = buffer_to_stream[i];
+         prims[stream] = min(prims[stream], max_prims);
+      }
+
+      int4 overflow = prims < in_prims;
+
+      poly_foreach_xfb(streams, i) {
+         p->xfb_verts[i] = prims[i] * vertices_per_prim;
+
+         *(p->xfb_overflow[i]) += (bool)overflow[i];
+         *(p->xfb_prims_generated_counter[i]) += prims[i];
+      }
+
+      *(p->xfb_any_overflow) += any(overflow);
+
+      /* Update XFB counters */
+      poly_foreach_xfb(buffers_written, i) {
+         uint32_t prim_stride_B = stride[i] * vertices_per_prim;
+         unsigned stream = buffer_to_stream[i];
+
+         global uint *ptr = p->xfb_offs_ptrs[i];
+
+         ptr = (global uint *)nir_ro_to_rw_poly((uint64_t)ptr);
+         *ptr += prims[stream] * prim_stride_B;
+      }
+   }
+
+   /* The geometry shader is invoked once per primitive (after unrolling
+    * primitive restart). From the spec:
+    *
+    *    In case of instanced geometry shaders (see section 11.3.4.2) the
+    *    geometry shader invocations count is incremented for each separate
+    *    instanced invocation.
+    */
+   *gs_invocations += unrolled_in_prims * invocations;
+   *gs_primitives += emitted_prims;
+
+   /* Clipper queries are not well-defined, so we can emulate them in lots of
+    * silly ways. We need the hardware counters to implement them properly. For
+    * now, just consider all primitives emitted as passing through the clipper.
+    * This satisfies spec text:
+    *
+    *    The number of primitives that reach the primitive clipping stage.
+    *
+    * and
+    *
+    *    If at least one vertex of the primitive lies inside the clipping
+    *    volume, the counter is incremented by one or more. Otherwise, the
+    *    counter is incremented by zero or more.
+    */
+   *c_primitives += emitted_prims;
+   *c_invocations += emitted_prims;
+}

src/poly/cl/meson.build

@@ -0,0 +1,35 @@
+# Copyright 2024 Valve Corporation
+# Copyright © 2025 Collabora Ltd.
+# SPDX-License-Identifier: MIT
+
+libpoly_shader_files = files(
+  'geometry.cl',
+  'tessellation.cl',
+)
+
+libpoly_shaders_spv = custom_target(
+  input : libpoly_shader_files,
+  output : 'libpoly.spv',
+  command : [
+    prog_mesa_clc, '-o', '@OUTPUT@', '--depfile', '@DEPFILE@',
+    libpoly_shader_files, '--',
+    '-I' + join_paths(meson.project_source_root(), 'include'),
+    '-I' + join_paths(meson.project_source_root(), 'src/compiler/libcl'),
+    '-I' + join_paths(meson.current_source_dir(), '.'),
+    '-I' + join_paths(meson.current_source_dir(), '../../'),
+    cl_args,
+    ],
+  depends : [],
+  depfile : 'libpoly_shaders.h.d',
+)
+
+libpoly_shaders = custom_target(
+  input : libpoly_shaders_spv,
+  output : ['libpoly.cpp', 'libpoly.h'],
+  command : [prog_vtn_bindgen2, libpoly_shaders_spv, '@OUTPUT0@', '@OUTPUT1@'],
+)
+
+idep_libpoly = declare_dependency(
+  sources : [libpoly_shaders],
+  include_directories : include_directories('.'),
+)

src/poly/cl/restart.h

@@ -0,0 +1,307 @@
+/*
+ * Copyright 2023 Alyssa Rosenzweig
+ * Copyright 2023 Valve Corporation
+ * Copyright 2025 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "compiler/libcl/libcl.h"
+#include "poly/geometry.h"
+#include "poly/prim.h"
+
+#define POLY_DECL_UNROLL_RESTART_SCRATCH(__scratch, __wg_size) \
+   local uchar __scratch[MAX2(__wg_size / 8, sizeof(void *))]
+
+static inline void
+poly_store_local_ballot_arr(local void *dst, uint idx, uint4 ballot)
+{
+   const uint sg_size = get_sub_group_size();
+
+   if (sg_size == 8)
+      ((uchar *)dst)[idx] = ballot.x;
+   else if (sg_size == 16)
+      ((ushort *)dst)[idx] = ballot.x;
+   else if (sg_size == 32)
+      ((uint *)dst)[idx] = ballot.x;
+   else if (sg_size == 64)
+      ((uint2 *)dst)[idx] = ballot.xy;
+   else if (sg_size == 128)
+      ((uint4 *)dst)[idx] = ballot;
+}
+
+static inline uint4
+poly_load_local_ballot_arr(local void *src, uint idx)
+{
+   const uint sg_size = get_sub_group_size();
+
+   uint4 ballot = (uint4)(0);
+   if (sg_size == 8)
+      ballot.x = ((uchar *)src)[idx];
+   else if (sg_size == 16)
+      ballot.x = ((ushort *)src)[idx];
+   else if (sg_size == 32)
+      ballot.x = ((uint *)src)[idx];
+   else if (sg_size == 64)
+      ballot.xy = ((uint2 *)src)[idx];
+   else if (sg_size == 128)
+      ballot = ((uint4 *)src)[idx];
+
+   return ballot;
+}
+
+/* sub_group_ballot_find_lsb() doesn't have a defined return value when the
+ * ballot is empty so we need our own helper.
+ */
+static uint
+poly_ballot_ctz(uint4 ballot)
+{
+   const uint sg_size = get_sub_group_size();
+
+   if (ballot.x)
+      return ctz(ballot.x);
+   if (sg_size > 32 && ballot.y)
+      return 32 + ctz(ballot.y);
+   if (sg_size > 64 && ballot.z)
+      return 64 + ctz(ballot.z);
+   if (sg_size > 96 && ballot.w)
+      return 96 + ctz(ballot.w);
+
+   return sg_size;
+}
+
+static inline uint4
+poly_sub_group_broadcast_uint4(uint4 val, uint lane)
+{
+   uint4 bval;
+   bval.x = sub_group_broadcast(val.x, lane);
+   bval.y = sub_group_broadcast(val.y, lane);
+   bval.z = sub_group_broadcast(val.z, lane);
+   bval.w = sub_group_broadcast(val.w, lane);
+   return bval;
+}
+
+/*
+ * Return the ID of the first thread in the workgroup where cond is true, or
+ * a value greater than or equal to the workgroup size if cond is false across
+ * the workgroup.
+ */
+static inline uint
+poly_work_group_first_true(bool cond, local void *scratch)
+{
+   const uint sg_size = get_sub_group_size();
+   const uint num_sg = get_num_sub_groups();
+
+   uint4 ballot = sub_group_ballot(cond);
+   if (num_sg == 1)
+      return poly_ballot_ctz(ballot);
+
+   barrier(CLK_LOCAL_MEM_FENCE);
+
+   if (get_sub_group_local_id() == 0)
+      poly_store_local_ballot_arr(scratch, get_sub_group_id(), ballot);
+
+   barrier(CLK_LOCAL_MEM_FENCE);
+
+   for (uint32_t i = 0; i < num_sg; i += sg_size) {
+      /* Read one subgroup worth per invocation */
+      uint src_sg_id = i + get_sub_group_local_id();
+
+      /* Clamp src_sg_id so we don't read OOB if the number of sugroups is not
+       * a multiple of the subgroup size.  It's safe to repeat the top index
+       * because the top indices will all be the same and we'll always take
+       * the first one.
+       */
+      src_sg_id = min(src_sg_id, num_sg - 1);
+
+      ballot = poly_load_local_ballot_arr(scratch, src_sg_id);
+      uint4 wide_ballot = sub_group_ballot(any(ballot != (uint4)(0)));
+      if (all(wide_ballot == (uint4)(0)))
+         continue;
+
+      uint first_sg = poly_ballot_ctz(wide_ballot);
+      uint4 first_ballot = poly_sub_group_broadcast_uint4(ballot, first_sg);
+      return (i + first_sg) * sg_size + poly_ballot_ctz(first_ballot);
+   }
+
+   return num_sg * sg_size;
+}
+
+/*
+ * When unrolling the index buffer for a draw, we translate the old indirect
+ * draws to new indirect draws. This routine allocates the new index buffer and
+ * sets up most of the new draw descriptor.
+ */
+static inline global void *
+poly_setup_unroll_for_draw(global struct poly_heap *heap,
+                           constant uint *in_draw, global uint *out_draw,
+                           enum mesa_prim mode, uint in_index_size_B,
+                           uint out_index_size_B)
+{
+   /* Determine an upper bound on the memory required for the index buffer.
+    * Restarts only decrease the unrolled index buffer size, so the maximum size
+    * is the unrolled size when the input has no restarts.
+    */
+   uint max_prims = u_decomposed_prims_for_vertices(mode, in_draw[0]);
+   uint max_verts = max_prims * mesa_vertices_per_prim(mode);
+   uint alloc_size = max_verts * out_index_size_B;
+
+   /* Allocate unrolled index buffer.
+    *
+    * TODO: For multidraw, should be atomic. But multidraw+unroll isn't
+    * currently wired up in any driver.
+    */
+   uint old_heap_bottom_B = poly_heap_alloc_offs(heap, alloc_size);
+
+   /* Setup most of the descriptor. Count will be determined after unroll. */
+   uint vertex_offset_idx = in_index_size_B > 0 ? 3 : 2;
+   out_draw[1] = in_draw[1];                           /* instance count */
+   out_draw[2] = old_heap_bottom_B / out_index_size_B; /* index offset */
+   out_draw[3] = in_draw[vertex_offset_idx];           /* index bias */
+   out_draw[4] = in_draw[vertex_offset_idx + 1];       /* base instance */
+
+   /* Return the index buffer we allocated */
+   return (global uchar *)heap->base + old_heap_bottom_B;
+}
+
+static inline void
+poly_unroll_geometry(global uint32_t *out_draw,
+                     global struct poly_heap *heap,
+                     constant uint *in_draw,
+                     uint64_t index_buffer,
+                     uint32_t index_buffer_range_el,
+                     uint32_t in_index_size_B,
+                     uint32_t out_index_size_B,
+                     uint32_t restart_index,
+                     uint32_t flatshade_first,
+                     enum mesa_prim mode,
+                     local void *scratch)
+{
+   uint tid = cl_local_id.x;
+   uint count = in_draw[0];
+
+   uintptr_t out_ptr;
+   if (tid == 0) {
+      out_ptr = (uintptr_t)poly_setup_unroll_for_draw(heap, in_draw, out_draw,
+                                                      mode, in_index_size_B,
+                                                      out_index_size_B);
+      if (get_num_sub_groups() > 1)
+         *(uintptr_t *)scratch = out_ptr;
+   }
+
+   if (get_num_sub_groups() > 1) {
+      barrier(CLK_LOCAL_MEM_FENCE);
+      out_ptr = *(uintptr_t *)scratch;
+   } else {
+      out_ptr = sub_group_broadcast(out_ptr, 0);
+   }
+
+   uintptr_t in_ptr = (uintptr_t)(poly_index_buffer(
+      index_buffer, index_buffer_range_el, in_draw[2], in_index_size_B));
+   uint in_range_el = poly_index_buffer_range_el(
+      index_buffer_range_el, in_draw[2]);
+
+   uint out_prims = 0;
+   uint needle = 0;
+   uint per_prim = mesa_vertices_per_prim(mode);
+   while (needle < count) {
+      /* Search for next restart or the end. Lanes load in parallel. */
+      uint next_restart = needle;
+      for (;;) {
+         uint idx = next_restart + tid;
+         bool restart =
+            idx >= count || poly_load_index(in_ptr, in_range_el, idx,
+                                            in_index_size_B) == restart_index;
+
+         uint next_offs = poly_work_group_first_true(restart, scratch);
+
+         next_restart += next_offs;
+         if (next_offs < cl_local_size.x)
+            break;
+      }
+
+      /* Emit up to the next restart. Lanes output in parallel */
+      uint subcount = next_restart - needle;
+      uint subprims = u_decomposed_prims_for_vertices(mode, subcount);
+      uint out_prims_base = out_prims;
+      for (uint i = tid; i < subprims; i += cl_local_size.x) {
+         for (uint vtx = 0; vtx < per_prim; ++vtx) {
+            uint id = poly_vertex_id_for_topology(mode, flatshade_first, i,
+                                                  vtx, subprims);
+            uint offset = needle + id;
+
+            uint x = ((out_prims_base + i) * per_prim) + vtx;
+            uint y = poly_load_index(in_ptr, in_range_el, offset,
+                                     in_index_size_B);
+
+            poly_store_index(out_ptr, out_index_size_B, x, y);
+         }
+      }
+
+      out_prims += subprims;
+      needle = next_restart + 1;
+   }
+
+   if (tid == 0)
+      out_draw[0] = out_prims * per_prim;
+}
+
+static inline void
+poly_unroll_restart(global uint32_t *out_draw,
+                    global struct poly_heap *heap,
+                    constant uint *in_draw,
+                    uint64_t index_buffer,
+                    uint32_t index_buffer_range_el,
+                    uint32_t index_size_B,
+                    uint32_t restart_index,
+                    uint32_t flatshade_first,
+                    enum mesa_prim mode,
+                    local void *scratch)
+{
+   poly_unroll_geometry(out_draw, heap, in_draw, index_buffer,
+                        index_buffer_range_el, index_size_B, index_size_B,
+                        restart_index, flatshade_first, mode, scratch);
+}
+
+static inline uint32_t
+poly_count_restart_prims(constant uint *in_draw,
+                         uint64_t index_buffer,
+                         uint32_t index_buffer_range_el,
+                         uint32_t index_size_B,
+                         uint32_t restart_index,
+                         enum mesa_prim mode,
+                         local void *scratch)
+{
+   uint tid = cl_local_id.x;
+   uint count = in_draw[0];
+
+   uintptr_t in_ptr = (uintptr_t)(poly_index_buffer(
+      index_buffer, index_buffer_range_el, in_draw[2], index_size_B));
+   uint in_range_el = poly_index_buffer_range_el(
+      index_buffer_range_el, in_draw[2]);
+
+   uint out_prims = 0;
+   uint needle = 0;
+   uint per_prim = mesa_vertices_per_prim(mode);
+   while (needle < count) {
+      /* Search for next restart or the end. Lanes load in parallel. */
+      uint next_restart = needle;
+      for (;;) {
+         uint idx = next_restart + tid;
+         bool restart =
+            idx >= count || poly_load_index(in_ptr, in_range_el, idx,
+                                            index_size_B) == restart_index;
+
+         uint next_offs = poly_work_group_first_true(restart, scratch);
+
+         next_restart += next_offs;
+         if (next_offs < cl_local_size.x)
+            break;
+      }
+
+      uint subcount = next_restart - needle;
+      out_prims += u_decomposed_prims_for_vertices(mode, subcount);
+      needle = next_restart + 1;
+   }
+
+   return out_prims;
+}

src/poly/cl/tessellation.cl

@@ -0,0 +1,127 @@
+/*
+ * Copyright 2023 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "poly/geometry.h"
+#include "poly/tessellator.h"
+
+uint
+poly_tcs_patch_vertices_in(constant struct poly_tess_params *p)
+{
+   return p->input_patch_size;
+}
+
+uint
+poly_tes_patch_vertices_in(constant struct poly_tess_params *p)
+{
+   return p->output_patch_size;
+}
+
+uint
+poly_tcs_unrolled_id(constant struct poly_tess_params *p, uint3 wg_id)
+{
+   return (wg_id.y * p->patches_per_instance) + wg_id.x;
+}
+
+/*
+ * Helper to lower indexing for a tess eval shader ran as a compute shader. This
+ * handles the tess+geom case. This is simpler than the general input assembly
+ * lowering, as we know:
+ *
+ * 1. the index buffer is U32
+ * 2. the index is in bounds
+ *
+ * Therefore we do a simple load. No bounds checking needed.
+ */
+uint32_t
+poly_load_tes_index(constant struct poly_tess_params *p, uint32_t index)
+{
+   /* Swap second and third vertices of each triangle to flip winding order
+    * dynamically if needed.
+    */
+   if (p->ccw) {
+      uint id = index % 3;
+
+      if (id == 1)
+         index++;
+      else if (id == 2)
+         index--;
+   }
+
+   return p->index_buffer[index];
+}
+
+uintptr_t
+poly_tcs_out_address(constant struct poly_tess_params *p, uint patch_id,
+                     uint vtx_id, gl_varying_slot location, uint nr_patch_out,
+                     uint out_patch_size, uint64_t vtx_out_mask)
+{
+   uint stride_el =
+      poly_tcs_out_stride_el(nr_patch_out, out_patch_size, vtx_out_mask);
+
+   uint offs_el =
+      poly_tcs_out_offs_el(vtx_id, location, nr_patch_out, vtx_out_mask);
+
+   offs_el += patch_id * stride_el;
+
+   /* Written to match the AGX addressing mode */
+   return (uintptr_t)(p->tcs_buffer) + (((uintptr_t)offs_el) << 2);
+}
+
+static uint
+tes_unrolled_patch_id(uint raw_id)
+{
+   return raw_id / POLY_TES_PATCH_ID_STRIDE;
+}
+
+uint
+poly_tes_patch_id(constant struct poly_tess_params *p, uint raw_id)
+{
+   return tes_unrolled_patch_id(raw_id) % p->patches_per_instance;
+}
+
+static uint
+tes_vertex_id_in_patch(uint raw_id)
+{
+   return raw_id % POLY_TES_PATCH_ID_STRIDE;
+}
+
+float2
+poly_load_tess_coord(constant struct poly_tess_params *p, uint raw_id)
+{
+   uint patch = tes_unrolled_patch_id(raw_id);
+   uint vtx = tes_vertex_id_in_patch(raw_id);
+
+   global struct poly_tess_point *t =
+      &p->patch_coord_buffer[p->coord_allocs[patch] + vtx];
+
+   /* Written weirdly because NIR struggles with loads of structs */
+   uint2 fixed = *((global uint2 *)t);
+
+   /* Convert fixed point to float */
+   return convert_float2(fixed) / (1u << 16);
+}
+
+uintptr_t
+poly_tes_in_address(constant struct poly_tess_params *p, uint raw_id, uint vtx_id,
+                    gl_varying_slot location)
+{
+   uint patch = tes_unrolled_patch_id(raw_id);
+
+   return poly_tcs_out_address(p, patch, vtx_id, location,
+                               p->tcs_patch_constants, p->output_patch_size,
+                               p->tcs_per_vertex_outputs);
+}
+
+float4
+poly_tess_level_outer_default(constant struct poly_tess_params *p)
+{
+   return vload4(0, p->tess_level_outer_default);
+}
+
+float2
+poly_tess_level_inner_default(constant struct poly_tess_params *p)
+{
+   return vload2(0, p->tess_level_inner_default);
+}

src/poly/geometry.h

@@ -0,0 +1,707 @@
+/*
+ * Copyright 2023 Alyssa Rosenzweig
+ * Copyright 2023 Valve Corporation
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "compiler/libcl/libcl.h"
+#include "compiler/shader_enums.h"
+
+#include "util/bitscan.h"
+#include "util/u_math.h"
+
+#ifdef __OPENCL_VERSION__
+#include "compiler/libcl/libcl_vk.h"
+#endif
+
+#pragma once
+
+#define POLY_MAX_SO_BUFFERS     4
+#define POLY_MAX_VERTEX_STREAMS 4
+
+enum poly_gs_shape {
+   /* Indexed, where indices are encoded as:
+    *
+    *    round_to_pot(max_indices) * round_to_pot(input_primitives) *
+    *                              * instance_count
+    *
+    * invoked for max_indices * input_primitives * instance_count indices.
+    *
+    * This is used with any dynamic topology. No hardware instancing used.
+    */
+   POLY_GS_SHAPE_DYNAMIC_INDEXED,
+
+   /* Indexed with a static index buffer. Indices ranges up to max_indices.
+    * Hardware instance count = input_primitives * software instance count.
+    */
+   POLY_GS_SHAPE_STATIC_INDEXED,
+
+   /* Non-indexed. Dispatched as:
+    *
+    *    (max_indices, input_primitives * instance count).
+    */
+   POLY_GS_SHAPE_STATIC_PER_PRIM,
+
+   /* Non-indexed. Dispatched as:
+    *
+    *    (max_indices * input_primitives, instance count).
+    */
+   POLY_GS_SHAPE_STATIC_PER_INSTANCE,
+};
+
+static inline unsigned
+poly_gs_rast_vertices(enum poly_gs_shape shape, unsigned max_indices,
+                      unsigned input_primitives, unsigned instance_count)
+{
+   switch (shape) {
+   case POLY_GS_SHAPE_DYNAMIC_INDEXED:
+      return max_indices * input_primitives * instance_count;
+
+   case POLY_GS_SHAPE_STATIC_INDEXED:
+   case POLY_GS_SHAPE_STATIC_PER_PRIM:
+      return max_indices;
+
+   case POLY_GS_SHAPE_STATIC_PER_INSTANCE:
+      return max_indices * input_primitives;
+   }
+
+   UNREACHABLE("invalid shape");
+}
+
+static inline unsigned
+poly_gs_rast_instances(enum poly_gs_shape shape, unsigned max_indices,
+                       unsigned input_primitives, unsigned instance_count)
+{
+   switch (shape) {
+   case POLY_GS_SHAPE_DYNAMIC_INDEXED:
+      return 1;
+
+   case POLY_GS_SHAPE_STATIC_INDEXED:
+   case POLY_GS_SHAPE_STATIC_PER_PRIM:
+      return input_primitives * instance_count;
+
+   case POLY_GS_SHAPE_STATIC_PER_INSTANCE:
+      return instance_count;
+   }
+
+   UNREACHABLE("invalid shape");
+}
+
+static inline bool
+poly_gs_indexed(enum poly_gs_shape shape)
+{
+   return shape == POLY_GS_SHAPE_DYNAMIC_INDEXED ||
+          shape == POLY_GS_SHAPE_STATIC_INDEXED;
+}
+
+static inline unsigned
+poly_gs_index_size(enum poly_gs_shape shape)
+{
+   switch (shape) {
+   case POLY_GS_SHAPE_DYNAMIC_INDEXED:
+      return 4;
+   case POLY_GS_SHAPE_STATIC_INDEXED:
+      return 1;
+   default:
+      return 0;
+   }
+}
+
+/* Heap to allocate from. */
+struct poly_heap {
+   DEVICE(uchar) base;
+   uint32_t bottom, size;
+} PACKED;
+static_assert(sizeof(struct poly_heap) == 4 * 4,
+              "struct poly_heap must be 4 words");
+
+#ifdef __OPENCL_VERSION__
+static inline uint
+poly_heap_alloc_offs(global struct poly_heap *heap, uint size_B)
+{
+   size_B = align(size_B, 16);
+
+   uint offs =
+      atomic_fetch_add((volatile atomic_uint *)(&heap->bottom), size_B);
+
+   /* Use printf+abort because assert is stripped from release builds. */
+   if (heap->bottom >= heap->size) {
+      printf(
+         "FATAL: GPU heap overflow, allocating size %u, at offset %u, heap size %u!",
+         size_B, offs, heap->size);
+
+      abort();
+   }
+
+   return offs;
+}
+
+static inline global void *
+poly_heap_alloc(global struct poly_heap *heap, uint size_B)
+{
+   return heap->base + poly_heap_alloc_offs(heap, size_B);
+}
+
+uint64_t nir_load_ro_sink_address_poly(void);
+
+static inline uint64_t
+poly_index_buffer(uint64_t index_buffer, uint size_el, uint offset_el,
+                  uint elsize_B)
+{
+   if (offset_el < size_el)
+      return index_buffer + (offset_el * elsize_B);
+   else
+      return nir_load_ro_sink_address_poly();
+}
+#endif
+
+/** Parameters that feed a vertex (or tessellation evaluation) shader.
+ *
+ * From the perspective of libpoly, vertex and tessellation evaluation shaders
+ * are identical.  One just fets fed by the hardware's input assmebly (which
+ * may be emulated by the driver) and the other gets fed from the tessellator.
+ * However, from the perspective of a geometry dispatch, they are identical.
+ */
+struct poly_vertex_params {
+   /* Index buffer if present. */
+   uint64_t index_buffer;
+
+   /* Size of an index in the index buffer, in bytes */
+   uint32_t index_size_B;
+
+   /* Size of the bound index buffer for bounds checking */
+   uint32_t index_buffer_range_el;
+
+   /* Number of vertices per instance. Written by CPU for direct draw, indirect
+    * setup kernel for indirect. This is used for VS->GS and VS->TCS indexing.
+    */
+   uint32_t verts_per_instance;
+
+   /* Within an indirect VS draw, the grids used to dispatch the VS written
+    * out by the GS indirect setup kernel or the CPU for a direct draw. This is
+    * the "indirect local" format: first 3 is in threads, second 3 is in grid
+    * blocks. This lets us use nontrivial workgroups with indirect draws without
+    * needing any predication.
+    */
+   uint32_t grid[6];
+
+   uint32_t _pad;
+
+   /* Output buffer for vertex data */
+   uint64_t output_buffer;
+
+   /* Mask of outputs present in the output buffer */
+   uint64_t outputs;
+} PACKED;
+static_assert(sizeof(struct poly_vertex_params) == 16 * 4,
+              "struct poly_vertex_params must be 16 words");
+
+static inline void
+poly_vertex_params_init(struct poly_vertex_params *p,
+                        uint64_t outputs, const uint32_t wg_size[3])
+{
+   *p = (struct poly_vertex_params) {
+      .outputs = outputs,
+      .grid = {
+         0, 0, 1, /* x/y are set by poly_vertex_params_set_draw() */
+         wg_size[0], wg_size[1], wg_size[2],
+      },
+   };
+}
+
+static inline void
+poly_vertex_params_set_draw(struct poly_vertex_params *p,
+                            uint32_t vertex_count, uint32_t instance_count)
+{
+   /* Invoke VS as (vertices, instances) */
+   p->verts_per_instance = vertex_count;
+   p->grid[0] = vertex_count;
+   p->grid[1] = instance_count;
+}
+
+static inline uint
+poly_index_buffer_range_el(uint size_el, uint offset_el)
+{
+   return offset_el < size_el ? (size_el - offset_el) : 0;
+}
+
+/* This must match VkDraw[Indexed]IndirectCommand
+ *
+ * The vertex/index_count and first_vertex/index fields line up, as does
+ * instance_count.  The only ones that don't are vertexOffset and
+ * firstInstance but we always set those to zero.
+ */
+struct poly_indirect_draw {
+   union {
+      uint32_t vertex_count;
+      uint32_t index_count;
+   };
+   uint32_t instance_count;
+   union {
+      uint32_t first_vertex;
+      uint32_t first_index;
+   };
+   uint32_t zeros[2];
+};
+static_assert(sizeof(struct poly_indirect_draw) == 5 * 4,
+              "struct poly_indirect_draw must be 5 words");
+
+struct poly_geometry_params {
+   /* Address of count buffer. For an indirect draw, this will be written by the
+    * indirect setup kernel.
+    */
+   DEVICE(uint) count_buffer;
+
+   /* Address of the primitives generated counters */
+   DEVICE(uint) prims_generated_counter[POLY_MAX_VERTEX_STREAMS];
+   DEVICE(uint) xfb_prims_generated_counter[POLY_MAX_VERTEX_STREAMS];
+   DEVICE(uint) xfb_overflow[POLY_MAX_VERTEX_STREAMS];
+   DEVICE(uint) xfb_any_overflow;
+
+   /* Pointers to transform feedback buffer offsets in bytes */
+   DEVICE(uint) xfb_offs_ptrs[POLY_MAX_SO_BUFFERS];
+
+   /* Output index buffer, allocated by pre-GS. */
+   DEVICE(uint) output_index_buffer;
+
+   /* Address of transform feedback buffer in general, supplied by the CPU. */
+   DEVICE(uchar) xfb_base_original[POLY_MAX_SO_BUFFERS];
+
+   /* Address of transform feedback for the current primitive. Written by pre-GS
+    * program.
+    */
+   DEVICE(uchar) xfb_base[POLY_MAX_SO_BUFFERS];
+
+   /* Location-indexed mask of flat outputs, used for lowering GL edge flags. */
+   uint64_t flat_outputs;
+
+   uint32_t xfb_size[POLY_MAX_SO_BUFFERS];
+
+   /* Number of vertices emitted by transform feedback per stream. Written by
+    * the pre-GS program.
+    */
+   uint32_t xfb_verts[POLY_MAX_VERTEX_STREAMS];
+
+   /* Within an indirect GS draw, the grids used to dispatch the GS written
+    * out by the GS indirect setup kernel or the CPU for a direct draw. This is
+    * the "indirect local" format: first 3 is in threads, second 3 is in grid
+    * blocks. This lets us use nontrivial workgroups with indirect draws without
+    * needing any predication.
+    */
+   uint32_t grid[6];
+
+   /* Indirect draw command */
+   struct poly_indirect_draw draw;
+
+   /* Number of input primitives across all instances, calculated by the CPU for
+    * a direct draw or the GS indirect setup kernel for an indirect draw.
+    */
+   uint32_t input_primitives;
+
+   /* Number of input primitives per instance, rounded up to a power-of-two and
+    * with the base-2 log taken. This is used to partition the output vertex IDs
+    * efficiently.
+    */
+   uint32_t primitives_log2;
+
+   /* Number of bytes output by the GS count shader per input primitive (may be
+    * 0), written by CPU and consumed by indirect draw setup shader for
+    * allocating counts.
+    */
+   uint32_t count_buffer_stride;
+
+   /* Dynamic input topology. Must be compatible with the geometry shader's
+    * layout() declared input class.
+    */
+   uint32_t input_topology;
+} PACKED;
+static_assert(sizeof(struct poly_geometry_params) == 79 * 4,
+              "struct poly_geometry_params must be 79 words");
+
+static inline void
+poly_geometry_params_init(struct poly_geometry_params *p,
+                          enum mesa_prim prim, const uint32_t wg_size[3])
+{
+   *p = (struct poly_geometry_params) {
+      .input_topology = prim,
+      .grid = {
+         0, 0, 1, /* x/y are set by poly_geometry_params_set_draw() */
+         wg_size[0], wg_size[1], wg_size[2],
+      },
+   };
+}
+
+static inline void
+poly_geometry_params_set_draw(struct poly_geometry_params *p,
+                              enum mesa_prim prim,
+                              enum poly_gs_shape shape, uint32_t max_indices,
+                              uint32_t vertex_count, uint32_t instance_count)
+{
+   /* Calculate number of primitives input into the GS */
+   const uint32_t prim_per_instance =
+      u_decomposed_prims_for_vertices(prim, vertex_count);
+
+   /* Invoke GS as (primitives, instances) */
+   p->grid[0] = prim_per_instance;
+   p->grid[1] = instance_count;
+
+   p->input_primitives = prim_per_instance * instance_count;
+   p->primitives_log2 = util_logbase2_ceil(prim_per_instance);
+
+   p->draw.index_count = poly_gs_rast_vertices(
+      shape, max_indices, prim_per_instance, instance_count);
+   p->draw.instance_count = poly_gs_rast_instances(
+      shape, max_indices, prim_per_instance, instance_count);
+}
+
+/* TCS shared memory layout:
+ *
+ *    vec4 vs_outputs[VERTICES_IN_INPUT_PATCH][TOTAL_VERTEX_OUTPUTS];
+ *
+ * TODO: compact.
+ */
+static inline uint
+poly_tcs_in_offs_el(uint vtx, gl_varying_slot location,
+                    uint64_t crosslane_vs_out_mask)
+{
+   uint base = vtx * util_bitcount64(crosslane_vs_out_mask);
+   uint offs = util_bitcount64(crosslane_vs_out_mask &
+                               (((uint64_t)(1) << location) - 1));
+
+   return base + offs;
+}
+
+static inline uint
+poly_tcs_in_size(uint32_t vertices_in_patch, uint64_t crosslane_vs_out_mask)
+{
+   return vertices_in_patch * util_bitcount64(crosslane_vs_out_mask) * 16;
+}
+
+/*
+ * TCS out buffer layout, per-patch:
+ *
+ *    float tess_level_outer[4];
+ *    float tess_level_inner[2];
+ *    vec4 patch_out[MAX_PATCH_OUTPUTS];
+ *    vec4 vtx_out[OUT_PATCH_SIZE][TOTAL_VERTEX_OUTPUTS];
+ *
+ * Vertex out are compacted based on the mask of written out. Patch
+ * out are used as-is.
+ *
+ * Bounding boxes are ignored.
+ */
+static inline uint
+poly_tcs_out_offs_el(uint vtx_id, gl_varying_slot location, uint nr_patch_out,
+                     uint64_t vtx_out_mask)
+{
+   uint off = 0;
+   if (location == VARYING_SLOT_TESS_LEVEL_OUTER)
+      return off;
+
+   off += 4;
+   if (location == VARYING_SLOT_TESS_LEVEL_INNER)
+      return off;
+
+   off += 2;
+   if (location >= VARYING_SLOT_PATCH0)
+      return off + (4 * (location - VARYING_SLOT_PATCH0));
+
+   /* Anything else is a per-vtx output */
+   off += 4 * nr_patch_out;
+   off += 4 * vtx_id * util_bitcount64(vtx_out_mask);
+
+   uint idx = util_bitcount64(vtx_out_mask & (((uint64_t)(1) << location) - 1));
+   return off + (4 * idx);
+}
+
+static inline uint
+poly_tcs_out_stride_el(uint nr_patch_out, uint out_patch_size,
+                       uint64_t vtx_out_mask)
+{
+   return poly_tcs_out_offs_el(out_patch_size, VARYING_SLOT_POS, nr_patch_out,
+                               vtx_out_mask);
+}
+
+static inline uint
+poly_tcs_out_stride(uint nr_patch_out, uint out_patch_size,
+                    uint64_t vtx_out_mask)
+{
+   return poly_tcs_out_stride_el(nr_patch_out, out_patch_size, vtx_out_mask) *
+          4;
+}
+
+/* In a tess eval shader, stride for hw vertex ID */
+#define POLY_TES_PATCH_ID_STRIDE 8192
+
+static inline uint
+poly_compact_prim(enum mesa_prim prim)
+{
+   static_assert(MESA_PRIM_QUAD_STRIP == MESA_PRIM_QUADS + 1,
+                 "MESA_PRIM_QUAD_STRIP must be immediately after MESA_PRIM_QUADS");
+   static_assert(MESA_PRIM_POLYGON == MESA_PRIM_QUADS + 2,
+                 "MESA_PRIM_POLYGON must be immediately after MESA_PRIM_QUAD_STRIP");
+
+#ifndef __OPENCL_VERSION__
+   assert(prim != MESA_PRIM_QUADS);
+   assert(prim != MESA_PRIM_QUAD_STRIP);
+   assert(prim != MESA_PRIM_POLYGON);
+   assert(prim != MESA_PRIM_PATCHES);
+#endif
+
+   return (prim >= MESA_PRIM_QUADS) ? (prim - 3) : prim;
+}
+
+static inline enum mesa_prim
+poly_uncompact_prim(uint packed)
+{
+   if (packed >= MESA_PRIM_QUADS)
+      return (enum mesa_prim)(packed + 3);
+
+   return (enum mesa_prim)packed;
+}
+
+/*
+ * Write a strip into a 32-bit index buffer. This is the sequence:
+ *
+ *    (b, b + 1, b + 2, ..., b + n - 1, -1) where -1 is the restart index
+ *
+ * For points, we write index buffers without restart just for remapping.
+ */
+static inline void
+_poly_write_strip(GLOBAL uint32_t *index_buffer, uint32_t index_offset,
+                  uint32_t vertex_offset, uint32_t verts_in_prim,
+                  uint32_t stream, uint32_t stream_multiplier, uint32_t n)
+{
+   bool restart = n > 1;
+   if (verts_in_prim < n)
+      return;
+
+   GLOBAL uint32_t *out = &index_buffer[index_offset];
+
+   /* Write out indices for the strip */
+   for (uint32_t i = 0; i < verts_in_prim; ++i) {
+      out[i] = (vertex_offset + i) * stream_multiplier + stream;
+   }
+
+   if (restart)
+      out[verts_in_prim] = -1;
+}
+
+static inline unsigned
+poly_decomposed_prims_for_vertices_with_tess(enum mesa_prim prim, int vertices,
+                                             unsigned verts_per_patch)
+{
+   if (prim >= MESA_PRIM_PATCHES) {
+      return vertices / verts_per_patch;
+   } else {
+      return u_decomposed_prims_for_vertices(prim, vertices);
+   }
+}
+
+#ifdef __OPENCL_VERSION__
+/*
+ * Returns (work_group_scan_inclusive_add(x), work_group_sum(x)). Implemented
+ * manually with subgroup ops and local memory since Mesa doesn't do those
+ * lowerings yet.
+ */
+static inline uint2
+poly_work_group_scan_inclusive_add(uint x, local uint *scratch)
+{
+   uint sg_id = get_sub_group_id();
+
+   /* Partial prefix sum of the subgroup */
+   uint sg = sub_group_scan_inclusive_add(x);
+
+   /* Reduction (sum) for the subgroup */
+   uint sg_sum = sub_group_broadcast(sg, 31);
+
+   /* Write out all the subgroups sums */
+   barrier(CLK_LOCAL_MEM_FENCE);
+   scratch[sg_id] = sg_sum;
+   barrier(CLK_LOCAL_MEM_FENCE);
+
+   /* Read all the subgroup sums. Thread T in subgroup G reads the sum of all
+    * threads in subgroup T.
+    */
+   uint other_sum = scratch[get_sub_group_local_id()];
+
+   /* Exclusive sum the subgroup sums to get the total before the current group,
+    * which can be added to the total for the current group.
+    */
+   uint other_sums = sub_group_scan_exclusive_add(other_sum);
+   uint base = sub_group_broadcast(other_sums, sg_id);
+   uint prefix = base + sg;
+
+   /* Reduce the workgroup using the prefix sum we already did */
+   uint reduction = sub_group_broadcast(other_sums + other_sum, 31);
+
+   return (uint2)(prefix, reduction);
+}
+
+static inline void
+poly_prefix_sum(local uint *scratch, global uint *buffer, uint len, uint words,
+                uint word, uint wg_count)
+{
+   uint tid = cl_local_id.x;
+
+   /* Main loop: complete workgroups processing multiple values at once */
+   uint i, count = 0;
+   uint len_remainder = len % wg_count;
+   uint len_rounded_down = len - len_remainder;
+
+   for (i = tid; i < len_rounded_down; i += wg_count) {
+      global uint *ptr = &buffer[(i * words) + word];
+      uint value = *ptr;
+      uint2 sums = poly_work_group_scan_inclusive_add(value, scratch);
+
+      *ptr = count + sums[0];
+      count += sums[1];
+   }
+
+   /* The last iteration is special since we won't have a full subgroup unless
+    * the length is divisible by the subgroup size, and we don't advance count.
+    */
+   global uint *ptr = &buffer[(i * words) + word];
+   uint value = (tid < len_remainder) ? *ptr : 0;
+   uint scan = poly_work_group_scan_inclusive_add(value, scratch)[0];
+
+   if (tid < len_remainder) {
+      *ptr = count + scan;
+   }
+}
+
+static inline void
+poly_increment_counters(global uint32_t *a, global uint32_t *b,
+                        global uint32_t *c, uint count)
+{
+   global uint32_t *ptr[] = {a, b, c};
+
+   for (uint i = 0; i < 3; ++i) {
+      if (ptr[i]) {
+         *(ptr[i]) += count;
+      }
+   }
+}
+
+static inline void
+poly_increment_ia(global uint32_t *ia_vertices, global uint32_t *ia_primitives,
+                  global uint32_t *vs_invocations, global uint32_t *c_prims,
+                  global uint32_t *c_invs, constant uint32_t *draw,
+                  enum mesa_prim prim, unsigned verts_per_patch)
+{
+   poly_increment_counters(ia_vertices, vs_invocations, NULL,
+                           draw[0] * draw[1]);
+
+   uint prims = poly_decomposed_prims_for_vertices_with_tess(prim, draw[0],
+                                                             verts_per_patch) *
+                draw[1];
+
+   poly_increment_counters(ia_primitives, c_prims, c_invs, prims);
+}
+
+static inline void
+poly_gs_setup_indirect(uint64_t index_buffer, constant uint *draw,
+                       global struct poly_vertex_params *vp /* output */,
+                       global struct poly_geometry_params *p /* output */,
+                       global struct poly_heap *heap,
+                       uint64_t vs_outputs /* Vertex (TES) output mask */,
+                       uint32_t index_size_B /* 0 if no index bffer */,
+                       uint32_t index_buffer_range_el,
+                       uint32_t prim /* Input primitive type, enum mesa_prim */,
+                       int is_prefix_summing, uint max_indices,
+                       enum poly_gs_shape shape)
+{
+   /* Determine the (primitives, instances) grid size. */
+   uint vertex_count = draw[0];
+   uint instance_count = draw[1];
+
+   poly_vertex_params_set_draw(vp, vertex_count, instance_count);
+   poly_geometry_params_set_draw(p, prim, shape, max_indices,
+                                 vertex_count, instance_count);
+
+   /* If indexing is enabled, the third word is the offset into the index buffer
+    * in elements. Apply that offset now that we have it. For a hardware
+    * indirect draw, the hardware would do this for us, but for software input
+    * assembly we need to do it ourselves.
+    */
+   if (index_size_B) {
+      vp->index_buffer = poly_index_buffer(index_buffer, index_buffer_range_el,
+                                           draw[2], index_size_B);
+
+      vp->index_buffer_range_el =
+         poly_index_buffer_range_el(index_buffer_range_el, draw[2]);
+   }
+
+   /* We need to allocate VS and GS count buffers, do so now */
+   uint vertex_buffer_size =
+      poly_tcs_in_size(vertex_count * instance_count, vs_outputs);
+
+   if (is_prefix_summing) {
+      p->count_buffer = poly_heap_alloc(
+         heap, p->input_primitives * p->count_buffer_stride);
+   }
+
+   vp->output_buffer = (uintptr_t)poly_heap_alloc(heap, vertex_buffer_size);
+
+   vp->outputs = vs_outputs;
+
+   if (shape == POLY_GS_SHAPE_DYNAMIC_INDEXED) {
+      const uint32_t index_offset =
+         poly_heap_alloc_offs(heap, p->draw.index_count * 4);
+      p->draw.first_index = index_offset / 4;
+      p->output_index_buffer = (global uint *)(heap->base + index_offset);
+   }
+}
+
+static uint
+poly_load_index(uintptr_t index_buffer, uint32_t index_buffer_range_el, uint id,
+                uint index_size)
+{
+   if (index_size == 0) {
+      /* No index buffer, return vertex ID */
+      return id;
+   }
+
+   bool oob = id >= index_buffer_range_el;
+
+   /* If the load would be out-of-bounds, load the first element which is
+    * assumed valid. If the application index buffer is empty with robustness2,
+    * index_buffer will point to a zero sink where only the first is valid.
+    */
+   if (oob) {
+      id = 0;
+   }
+
+   uint el;
+   if (index_size == 1) {
+      el = ((constant uint8_t *)index_buffer)[id];
+   } else if (index_size == 2) {
+      el = ((constant uint16_t *)index_buffer)[id];
+   } else {
+      el = ((constant uint32_t *)index_buffer)[id];
+   }
+
+   /* D3D robustness semantics. TODO: Optimize? */
+   if (oob) {
+      el = 0;
+   }
+
+   return el;
+}
+
+static void
+poly_store_index(uintptr_t index_buffer, uint index_size_B, uint id, uint value)
+{
+   global uint32_t *out_32 = (global uint32_t *)index_buffer;
+   global uint16_t *out_16 = (global uint16_t *)index_buffer;
+   global uint8_t *out_8 = (global uint8_t *)index_buffer;
+
+   if (index_size_B == 4)
+      out_32[id] = value;
+   else if (index_size_B == 2)
+      out_16[id] = value;
+   else
+      out_8[id] = value;
+}
+
+#endif

src/poly/meson.build

@@ -0,0 +1,9 @@
+# Copyright © 2025 Collabora Ltd.
+# SPDX-License-Identifier: MIT
+
+inc_poly = include_directories([
+   '.', 'nir'
+])
+
+subdir('cl')
+subdir('nir')

src/poly/nir/meson.build

@@ -0,0 +1,19 @@
+# Copyright © 2025 Collabora Ltd.
+# SPDX-License-Identifier: MIT
+
+libpoly_nir_files = files(
+  'poly_nir_lower_gs.c',
+  'poly_nir_lower_sysvals.c',
+  'poly_nir_lower_tess.c',
+  'poly_nir_lower_vs.c',
+)
+
+libpoly_nir = static_library(
+  'libpoly_nir',
+  [libpoly_nir_files],
+  include_directories : [inc_poly],
+  c_args : [no_override_init_args, '-Wno-c2x-extensions'],
+  gnu_symbol_visibility : 'hidden',
+  dependencies: [idep_nir, idep_mesautil, idep_libpoly],
+  build_by_default : false,
+)

src/poly/nir/poly_nir.h

@@ -0,0 +1,62 @@
+/*
+ * Copyright 2023 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#pragma once
+
+#include <stdbool.h>
+#include <stdint.h>
+#include "poly/geometry.h"
+#include "nir.h"
+#include "shader_enums.h"
+
+struct nir_def *poly_load_per_vertex_input(struct nir_builder *b,
+                                           nir_intrinsic_instr *intr,
+                                           struct nir_def *vertex);
+
+nir_def *poly_nir_load_vertex_id(struct nir_builder *b, nir_def *id);
+
+bool poly_nir_lower_sw_vs(struct nir_shader *s);
+
+bool poly_nir_lower_vs_before_gs(struct nir_shader *vs);
+
+struct poly_gs_info {
+   /* Output primitive mode for geometry shaders */
+   enum mesa_prim mode;
+
+   /* Number of words per primitive in the count buffer */
+   unsigned count_words;
+
+   /* Per-input primitive stride of the output index buffer */
+   unsigned max_indices;
+
+   /* Whether the GS includes transform feedback at a compile-time level */
+   bool xfb;
+
+   /* Whether a prefix sum is required on the count outputs. Implies xfb */
+   bool prefix_sum;
+
+   /* Whether the GS writes to a stream other than stream #0 */
+   bool multistream;
+
+   /* Shape of the rasterization draw, named by the instance ID */
+   enum poly_gs_shape shape;
+
+   /* Static topology used if shape = POLY_GS_SHAPE_STATIC_INDEXED */
+   uint8_t topology[64];
+};
+
+bool poly_nir_lower_gs(struct nir_shader *gs, struct nir_shader **gs_count,
+                       struct nir_shader **gs_copy, struct nir_shader **pre_gs,
+                       struct poly_gs_info *info);
+
+bool poly_nir_lower_tcs(struct nir_shader *tcs);
+
+bool poly_nir_lower_tes(struct nir_shader *tes, bool to_hw_vs);
+
+uint64_t poly_tcs_per_vertex_outputs(const struct nir_shader *nir);
+
+unsigned poly_tcs_output_stride(const struct nir_shader *nir);
+
+bool poly_nir_lower_sysvals(struct nir_shader *nir);

src/poly/nir/poly_nir_lower_sysvals.c

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2023 Valve Corporation
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "nir_builder.h"
+#include "poly/cl/libpoly.h"
+#include "poly/nir/poly_nir.h"
+
+static bool
+lower_sysvals_intr(nir_builder *b, nir_intrinsic_instr *intr, void *data)
+{
+   switch (intr->intrinsic) {
+   case nir_intrinsic_load_index_size_poly: {
+      b->cursor = nir_before_instr(&intr->instr);
+      nir_def *vp = nir_load_vertex_param_buffer_poly(b);
+      nir_def_replace(&intr->def, poly_index_size(b, vp));
+      return true;
+   }
+
+   case nir_intrinsic_load_vs_outputs_poly: {
+      b->cursor = nir_before_instr(&intr->instr);
+      nir_def *vp = nir_load_vertex_param_buffer_poly(b);
+      nir_def_replace(&intr->def, poly_vertex_outputs(b, vp));
+      return true;
+   }
+
+   default:
+      return false;
+   }
+}
+
+bool
+poly_nir_lower_sysvals(struct nir_shader *nir)
+{
+   return nir_shader_intrinsics_pass(nir, lower_sysvals_intr,
+                                     nir_metadata_control_flow, NULL);
+}

src/poly/nir/poly_nir_lower_tess.c

@@ -0,0 +1,270 @@
+/*
+ * Copyright 2023 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "poly/cl/libpoly.h"
+#include "poly/geometry.h"
+#include "poly/nir/poly_nir.h"
+#include "util/bitscan.h"
+#include "util/macros.h"
+#include "nir.h"
+#include "nir_builder.h"
+#include "nir_builder_opcodes.h"
+#include "nir_intrinsics.h"
+#include "nir_intrinsics_indices.h"
+#include "shader_enums.h"
+
+static nir_def *
+tcs_unrolled_id(nir_builder *b)
+{
+   return poly_tcs_unrolled_id(b, nir_load_tess_param_buffer_poly(b),
+                               nir_load_workgroup_id(b));
+}
+
+uint64_t
+poly_tcs_per_vertex_outputs(const nir_shader *nir)
+{
+   return nir->info.outputs_written &
+          ~(VARYING_BIT_TESS_LEVEL_INNER | VARYING_BIT_TESS_LEVEL_OUTER |
+            VARYING_BIT_BOUNDING_BOX0 | VARYING_BIT_BOUNDING_BOX1);
+}
+
+unsigned
+poly_tcs_output_stride(const nir_shader *nir)
+{
+   return poly_tcs_out_stride(util_last_bit(nir->info.patch_outputs_written),
+                              nir->info.tess.tcs_vertices_out,
+                              poly_tcs_per_vertex_outputs(nir));
+}
+
+static nir_def *
+tcs_out_addr(nir_builder *b, nir_intrinsic_instr *intr, nir_def *vertex_id)
+{
+   nir_io_semantics sem = nir_intrinsic_io_semantics(intr);
+
+   nir_def *offset = nir_get_io_offset_src(intr)->ssa;
+   nir_def *addr = poly_tcs_out_address(
+      b, nir_load_tess_param_buffer_poly(b), tcs_unrolled_id(b), vertex_id,
+      nir_iadd_imm(b, offset, sem.location),
+      nir_imm_int(b, util_last_bit(b->shader->info.patch_outputs_written)),
+      nir_imm_int(b, b->shader->info.tess.tcs_vertices_out),
+      nir_imm_int64(b, poly_tcs_per_vertex_outputs(b->shader)));
+
+   addr = nir_iadd_imm(b, addr, nir_intrinsic_component(intr) * 4);
+
+   return addr;
+}
+
+static nir_def *
+lower_tes_load(nir_builder *b, nir_intrinsic_instr *intr)
+{
+   gl_varying_slot location = nir_intrinsic_io_semantics(intr).location;
+   nir_src *offset_src = nir_get_io_offset_src(intr);
+
+   nir_def *vertex = nir_imm_int(b, 0);
+   nir_def *offset = offset_src ? offset_src->ssa : nir_imm_int(b, 0);
+
+   if (intr->intrinsic == nir_intrinsic_load_per_vertex_input)
+      vertex = intr->src[0].ssa;
+
+   nir_def *addr = poly_tes_in_address(b, nir_load_tess_param_buffer_poly(b),
+                                       nir_load_vertex_id(b), vertex,
+                                       nir_iadd_imm(b, offset, location));
+
+   if (nir_intrinsic_has_component(intr))
+      addr = nir_iadd_imm(b, addr, nir_intrinsic_component(intr) * 4);
+
+   return nir_load_global_constant(b, intr->def.num_components,
+                                   intr->def.bit_size, addr, .align_mul = 4);
+}
+
+static nir_def *
+tcs_load_input(nir_builder *b, nir_intrinsic_instr *intr)
+{
+   nir_def *base = nir_imul(
+      b, tcs_unrolled_id(b),
+      poly_tcs_patch_vertices_in(b, nir_load_tess_param_buffer_poly(b)));
+   nir_def *vertex = nir_iadd(b, base, intr->src[0].ssa);
+
+   return poly_load_per_vertex_input(b, intr, vertex);
+}
+
+static nir_def *
+lower_tcs_impl(nir_builder *b, nir_intrinsic_instr *intr)
+{
+   switch (intr->intrinsic) {
+   case nir_intrinsic_barrier:
+      /* A patch fits in a subgroup, so the barrier is unnecessary. */
+      return NIR_LOWER_INSTR_PROGRESS_REPLACE;
+
+   case nir_intrinsic_load_primitive_id:
+      return nir_channel(b, nir_load_workgroup_id(b), 0);
+
+   case nir_intrinsic_load_instance_id:
+      return nir_channel(b, nir_load_workgroup_id(b), 1);
+
+   case nir_intrinsic_load_invocation_id:
+      if (b->shader->info.tess.tcs_vertices_out == 1)
+         return nir_imm_int(b, 0);
+      else
+         return nir_channel(b, nir_load_local_invocation_id(b), 0);
+
+   case nir_intrinsic_load_per_vertex_input:
+      return tcs_load_input(b, intr);
+
+   case nir_intrinsic_load_patch_vertices_in:
+      return poly_tcs_patch_vertices_in(b, nir_load_tess_param_buffer_poly(b));
+
+   case nir_intrinsic_load_tess_level_outer_default:
+      return poly_tess_level_outer_default(b,
+                                           nir_load_tess_param_buffer_poly(b));
+
+   case nir_intrinsic_load_tess_level_inner_default:
+      return poly_tess_level_inner_default(b,
+                                           nir_load_tess_param_buffer_poly(b));
+
+   case nir_intrinsic_load_output: {
+      nir_def *addr = tcs_out_addr(b, intr, nir_undef(b, 1, 32));
+      return nir_load_global(b, intr->def.num_components, intr->def.bit_size,
+                             addr, .align_mul = 4);
+   }
+
+   case nir_intrinsic_load_per_vertex_output: {
+      nir_def *addr = tcs_out_addr(b, intr, intr->src[0].ssa);
+      return nir_load_global(b, intr->def.num_components, intr->def.bit_size,
+                             addr, .align_mul = 4);
+   }
+
+   case nir_intrinsic_store_output: {
+      /* Only vec2, make sure we can't overwrite */
+      assert(intr->src[0].ssa->num_components <= 2 ||
+             nir_intrinsic_io_semantics(intr).location !=
+                VARYING_SLOT_TESS_LEVEL_INNER);
+
+      nir_store_global(b, intr->src[0].ssa,
+                       tcs_out_addr(b, intr, nir_undef(b, 1, 32)),
+                       .write_mask = nir_intrinsic_write_mask(intr));
+      return NIR_LOWER_INSTR_PROGRESS_REPLACE;
+   }
+
+   case nir_intrinsic_store_per_vertex_output: {
+      nir_store_global(b, intr->src[0].ssa,
+                       tcs_out_addr(b, intr, intr->src[1].ssa),
+                       .write_mask = nir_intrinsic_write_mask(intr));
+      return NIR_LOWER_INSTR_PROGRESS_REPLACE;
+   }
+
+   default:
+      return NULL;
+   }
+}
+
+static bool
+lower_tcs(nir_builder *b, nir_intrinsic_instr *intr, void *data)
+{
+   b->cursor = nir_before_instr(&intr->instr);
+
+   nir_def *repl = lower_tcs_impl(b, intr);
+   if (!repl)
+      return false;
+
+   if (repl != NIR_LOWER_INSTR_PROGRESS_REPLACE)
+      nir_def_rewrite_uses(&intr->def, repl);
+
+   nir_instr_remove(&intr->instr);
+   return true;
+}
+
+bool
+poly_nir_lower_tcs(nir_shader *tcs)
+{
+   return nir_shader_intrinsics_pass(tcs, lower_tcs, nir_metadata_control_flow,
+                                     NULL);
+}
+
+static nir_def *
+lower_tes_impl(nir_builder *b, nir_intrinsic_instr *intr, void *data)
+{
+   switch (intr->intrinsic) {
+   case nir_intrinsic_load_tess_coord_xy:
+      return poly_load_tess_coord(b, nir_load_tess_param_buffer_poly(b),
+                                  nir_load_vertex_id(b));
+
+   case nir_intrinsic_load_primitive_id:
+      return poly_tes_patch_id(b, nir_load_tess_param_buffer_poly(b),
+                               nir_load_vertex_id(b));
+
+   case nir_intrinsic_load_input:
+   case nir_intrinsic_load_per_vertex_input:
+   case nir_intrinsic_load_tess_level_inner:
+   case nir_intrinsic_load_tess_level_outer:
+      return lower_tes_load(b, intr);
+
+   case nir_intrinsic_load_patch_vertices_in:
+      return poly_tes_patch_vertices_in(b, nir_load_tess_param_buffer_poly(b));
+
+   default:
+      return NULL;
+   }
+}
+
+static bool
+lower_tes(nir_builder *b, nir_intrinsic_instr *intr, void *data)
+{
+   b->cursor = nir_before_instr(&intr->instr);
+   nir_def *repl = lower_tes_impl(b, intr, data);
+
+   if (repl) {
+      nir_def_replace(&intr->def, repl);
+      return true;
+   } else {
+      return false;
+   }
+}
+
+static bool
+lower_tes_indexing(nir_builder *b, nir_intrinsic_instr *intr, void *data)
+{
+   if (intr->intrinsic != nir_intrinsic_load_vertex_id)
+      return false;
+
+   b->cursor = nir_before_instr(&intr->instr);
+   nir_def *p = nir_load_tess_param_buffer_poly(b);
+   nir_def *id = nir_channel(b, nir_load_global_invocation_id(b, 32), 0);
+   nir_def_replace(&intr->def, poly_load_tes_index(b, p, id));
+   return true;
+}
+
+bool
+poly_nir_lower_tes(nir_shader *tes, bool to_hw_vs)
+{
+   nir_lower_tess_coord_z(
+      tes, tes->info.tess._primitive_mode == TESS_PRIMITIVE_TRIANGLES);
+
+   nir_shader_intrinsics_pass(tes, lower_tes, nir_metadata_control_flow, NULL);
+
+   /* Points mode renders as points, make sure we write point size for the HW */
+   if (tes->info.tess.point_mode && to_hw_vs) {
+      nir_lower_default_point_size(tes);
+   }
+
+   if (to_hw_vs) {
+      /* We lower to a HW VS, so update the shader info so the compiler does the
+       * right thing.
+       */
+      tes->info.stage = MESA_SHADER_VERTEX;
+      memset(&tes->info.vs, 0, sizeof(tes->info.vs));
+      tes->info.vs.tes_poly = true;
+   } else {
+      /* If we're running as a compute shader, we need to load from the index
+       * buffer manually. Fortunately, this doesn't require a shader key:
+       * tess-as-compute always use U32 index buffers.
+       */
+      nir_shader_intrinsics_pass(tes, lower_tes_indexing,
+                                 nir_metadata_control_flow, NULL);
+   }
+
+   nir_lower_idiv(tes, &(nir_lower_idiv_options){.allow_fp16 = true});
+   return nir_progress(true, nir_shader_get_entrypoint(tes), nir_metadata_none);
+}

src/poly/nir/poly_nir_lower_vs.c

@@ -0,0 +1,61 @@
+/*
+ * Copyright 2023 Valve Corporation
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "compiler/nir/nir_builder.h"
+#include "poly/cl/libpoly.h"
+#include "poly/nir/poly_nir.h"
+#include "poly/geometry.h"
+#include "nir.h"
+
+/*
+ * This file implements basic input assembly in software. It runs on software
+ * vertex shaders, as part of geometry/tessellation lowering. It does not apply
+ * the topology, which happens in the geometry shader.
+ */
+nir_def *
+poly_nir_load_vertex_id(nir_builder *b, nir_def *id)
+{
+   /* If drawing with an index buffer, pull the vertex ID. Otherwise, the
+    * vertex ID is just the index as-is.
+    */
+   nir_def *index_size = nir_load_index_size_poly(b);
+   nir_def *index_buffer_id;
+   nir_if *index_size_present = nir_push_if(b, nir_ine_imm(b, index_size, 0));
+   {
+      nir_def *p = nir_load_vertex_param_buffer_poly(b);
+      index_buffer_id = poly_load_index_buffer(b, p, id, index_size);
+   }
+   nir_pop_if(b, index_size_present);
+   nir_def *effective_id = nir_if_phi(b, index_buffer_id, id);
+
+   /* Add the "start", either an index bias or a base vertex. This must happen
+    * after indexing for proper index bias behaviour.
+    */
+   return nir_iadd(b, effective_id, nir_load_first_vertex(b));
+}
+
+static bool
+lower(nir_builder *b, nir_intrinsic_instr *intr, void *data)
+{
+   b->cursor = nir_before_instr(&intr->instr);
+
+   if (intr->intrinsic == nir_intrinsic_load_vertex_id) {
+      nir_def *id = nir_channel(b, nir_load_global_invocation_id(b, 32), 0);
+      nir_def_replace(&intr->def, poly_nir_load_vertex_id(b, id));
+      return true;
+   } else if (intr->intrinsic == nir_intrinsic_load_instance_id) {
+      nir_def_replace(&intr->def,
+                      nir_channel(b, nir_load_global_invocation_id(b, 32), 1));
+      return true;
+   }
+
+   return false;
+}
+
+bool
+poly_nir_lower_sw_vs(nir_shader *s)
+{
+   return nir_shader_intrinsics_pass(s, lower, nir_metadata_control_flow, NULL);
+}

src/poly/prim.h

@@ -0,0 +1,147 @@
+/*
+ * Copyright 2023 Alyssa Rosenzweig
+ * Copyright 2023 Valve Corporation
+ * Copyright 2025 Collabora Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "compiler/libcl/libcl.h"
+#include "compiler/shader_enums.h"
+
+#pragma once
+
+/* Swap the two non-provoking vertices in odd triangles. This generates a vertex
+ * ID list with a consistent winding order.
+ *
+ * Holding prim and flatshade_first constant, the map : [0, 1, 2] -> [0, 1, 2]
+ * is its own inverse. It is hence used both vertex fetch and transform
+ * feedback.
+ */
+static inline uint32_t
+poly_map_vertex_in_tri_strip(uint32_t prim, uint32_t vert, bool flatshade_first)
+{
+   unsigned pv = flatshade_first ? 0 : 2;
+
+   bool even = (prim & 1) == 0;
+   bool provoking = vert == pv;
+
+   return (provoking || even) ? vert : ((3 - pv) - vert);
+}
+
+static inline uint32_t
+poly_vertex_id_for_line_loop(uint32_t prim, uint32_t vert, uint32_t num_prims)
+{
+   /* (0, 1), (1, 2), (2, 0) */
+   if (prim == (num_prims - 1) && vert == 1)
+      return 0;
+   else
+      return prim + vert;
+}
+
+static inline uint32_t
+poly_vertex_id_for_tri_fan(uint32_t prim, uint32_t vert, bool flatshade_first)
+{
+   /* Vulkan spec section 20.1.7 gives (i + 1, i + 2, 0) for a provoking
+    * first. OpenGL instead wants (0, i + 1, i + 2) with a provoking last.
+    * Piglit clipflat expects us to switch between these orders depending on
+    * provoking vertex, to avoid trivializing the fan.
+    *
+    * Rotate accordingly.
+    */
+   if (flatshade_first) {
+      vert = (vert == 2) ? 0 : (vert + 1);
+   }
+
+   /* The simpler form assuming last is provoking. */
+   return (vert == 0) ? 0 : prim + vert;
+}
+
+static inline uint32_t
+poly_vertex_id_for_tri_strip_adj(uint32_t prim, uint32_t vert,
+                                 uint32_t num_prims, bool flatshade_first)
+{
+   /* See Vulkan spec section 20.1.11 "Triangle Strips With Adjancency".
+    *
+    * There are different cases for first/middle/last/only primitives and for
+    * odd/even primitives.  Determine which case we're in.
+    */
+   bool last = prim == (num_prims - 1);
+   bool first = prim == 0;
+   bool even = (prim & 1) == 0;
+   bool even_or_first = even || first;
+
+   /* When the last vertex is provoking, we rotate the primitives
+    * accordingly. This seems required for OpenGL.
+    */
+   if (!flatshade_first && !even_or_first) {
+      vert = (vert + 4u) % 6u;
+   }
+
+   /* Offsets per the spec. The spec lists 6 cases with 6 offsets. Luckily,
+    * there are lots of patterns we can exploit, avoiding a full 6x6 LUT.
+    *
+    * Here we assume the first vertex is provoking, the Vulkan default.
+    */
+   uint32_t offsets[6] = {
+      0,
+      first ? 1 : (even ? -2 : 3),
+      even_or_first ? 2 : 4,
+      last ? 5 : 6,
+      even_or_first ? 4 : 2,
+      even_or_first ? 3 : -2,
+   };
+
+   /* Ensure NIR can see thru the local array */
+   uint32_t offset = 0;
+   for (uint32_t i = 1; i < 6; ++i) {
+      if (i == vert)
+         offset = offsets[i];
+   }
+
+   /* Finally add to the base of the primitive */
+   return (prim * 2) + offset;
+}
+
+static inline uint32_t
+poly_vertex_id_for_topology(enum mesa_prim mode, bool flatshade_first,
+                            uint32_t prim, uint32_t vert, uint32_t num_prims)
+{
+   switch (mode) {
+   case MESA_PRIM_POINTS:
+   case MESA_PRIM_LINES:
+   case MESA_PRIM_TRIANGLES:
+   case MESA_PRIM_LINES_ADJACENCY:
+   case MESA_PRIM_TRIANGLES_ADJACENCY:
+      /* Regular primitive: every N vertices defines a primitive */
+      return (prim * mesa_vertices_per_prim(mode)) + vert;
+
+   case MESA_PRIM_LINE_LOOP:
+      return poly_vertex_id_for_line_loop(prim, vert, num_prims);
+
+   case MESA_PRIM_LINE_STRIP:
+   case MESA_PRIM_LINE_STRIP_ADJACENCY:
+      /* (i, i + 1) or (i, ..., i + 3) */
+      return prim + vert;
+
+   case MESA_PRIM_TRIANGLE_STRIP: {
+      /* Order depends on the provoking vert.
+       *
+       * First: (0, 1, 2), (1, 3, 2), (2, 3, 4).
+       * Last:  (0, 1, 2), (2, 1, 3), (2, 3, 4).
+       *
+       * Pull the (maybe swapped) vert from the corresponding primitive
+       */
+      return prim + poly_map_vertex_in_tri_strip(prim, vert, flatshade_first);
+   }
+
+   case MESA_PRIM_TRIANGLE_FAN:
+      return poly_vertex_id_for_tri_fan(prim, vert, flatshade_first);
+
+   case MESA_PRIM_TRIANGLE_STRIP_ADJACENCY:
+      return poly_vertex_id_for_tri_strip_adj(prim, vert, num_prims,
+                                              flatshade_first);
+
+   default:
+      return 0;
+   }
+}

src/poly/tessellator.h

@@ -0,0 +1,105 @@
+/*
+ * Copyright 2024 Valve Corporation
+ * SPDX-License-Identifier: MIT
+ */
+
+#pragma once
+
+#include "compiler/libcl/libcl.h"
+
+enum poly_tess_partitioning {
+   POLY_TESS_PARTITIONING_FRACTIONAL_ODD,
+   POLY_TESS_PARTITIONING_FRACTIONAL_EVEN,
+   POLY_TESS_PARTITIONING_INTEGER,
+};
+
+enum poly_tess_mode {
+   /* Do not actually tessellate, just write the index counts */
+   POLY_TESS_MODE_COUNT,
+
+   /* Tessellate using the count buffers to allocate indices */
+   POLY_TESS_MODE_WITH_COUNTS,
+};
+
+struct poly_tess_point {
+   uint32_t u;
+   uint32_t v;
+};
+static_assert(sizeof(struct poly_tess_point) == 8,
+              "struct poly_tess_point must be 8 bytes");
+
+struct poly_tess_params {
+   /* Heap to allocate tessellator outputs in */
+   DEVICE(struct poly_heap) heap;
+
+   /* Patch coordinate buffer, indexed as:
+    *
+    *    coord_allocs[patch_ID] + vertex_in_patch
+    */
+   DEVICE(struct poly_tess_point) patch_coord_buffer;
+
+   /* Per-patch index within the heap for the tess coords, written by the
+    * tessellator based on the allocated memory.
+    */
+   DEVICE(uint32_t) coord_allocs;
+
+   /* Space for output draws from the tessellator. API draw calls. */
+   DEVICE(uint32_t) out_draws;
+
+   /* Tessellation control shader output buffer. */
+   DEVICE(float) tcs_buffer;
+
+   /* Count buffer. # of indices per patch written here, then prefix summed. */
+   DEVICE(uint32_t) counts;
+
+   /* Allocated index buffer for all patches, if we're prefix summing counts */
+   DEVICE(uint32_t) index_buffer;
+
+   /* Address of the tess eval invocation counter for implementing pipeline
+    * statistics, if active. Zero if inactive. Incremented by tessellator.
+    */
+   DEVICE(uint32_t) statistic;
+
+   /* Bitfield of TCS per-vertex outputs */
+   uint64_t tcs_per_vertex_outputs;
+
+   /* Default tess levels used in OpenGL when there is no TCS in the pipeline.
+    * Unused in Vulkan and OpenGL ES.
+    */
+   float tess_level_outer_default[4];
+   float tess_level_inner_default[2];
+
+   /* Number of vertices in the input patch */
+   uint32_t input_patch_size;
+
+   /* Number of vertices in the TCS output patch */
+   uint32_t output_patch_size;
+
+   /* Number of patch constants written by TCS */
+   uint32_t tcs_patch_constants;
+
+   /* Number of input patches per instance of the VS/TCS */
+   uint32_t patches_per_instance;
+
+   /* Stride between tessellation facotrs in the TCS output buffer. */
+   uint32_t tcs_stride_el;
+
+   /* Number of patches being tessellated */
+   uint32_t nr_patches;
+
+   /* Partitioning and points mode. These affect per-patch setup code but not
+    * the hot tessellation loop so we make them dynamic to reduce tessellator
+    * variants.
+    */
+   enum poly_tess_partitioning partitioning;
+   uint32_t points_mode;
+   uint32_t isolines;
+
+   /* When fed into a geometry shader, triangles should be counter-clockwise.
+    * The tessellator always produces clockwise triangles, but we can swap
+    * dynamically in the TES.
+    */
+   uint32_t ccw;
+} PACKED;
+static_assert(sizeof(struct poly_tess_params) == 34 * 4,
+              "struct poly_tess_params must be 34 words");