init

src/panfrost/shared/meson.build

@@ -0,0 +1,40 @@
+# Copyright © 2018 Rob Clark
+# Copyright © 2019 Collabora
+# SPDX-License-Identifier: MIT
+
+libpanfrost_shared_files = files(
+  'pan_minmax_cache.c',
+  'pan_tiling.c',
+
+  'pan_minmax_cache.h',
+  'pan_tiling.h',
+)
+
+libpanfrost_shared = static_library(
+  'panfrost_shared',
+  [libpanfrost_shared_files],
+  include_directories : [inc_include, inc_src],
+  dependencies : idep_mesautil,
+  c_args : [no_override_init_args, soong_compat_c_args, '-O3'],
+  gnu_symbol_visibility : 'hidden',
+  build_by_default : false,
+)
+
+if with_tests
+  test(
+    'panfrost_tiling',
+    executable(
+      'panfrost_tiling',
+      files(
+        'test/test-tiling.cpp',
+      ),
+      cpp_args : cpp_msvc_compat_args,
+      gnu_symbol_visibility : 'hidden',
+      include_directories : [inc_include, inc_src, inc_panfrost],
+      dependencies: [idep_gtest, idep_mesautil],
+      link_with : [libpanfrost_shared],
+    ),
+    suite : ['panfrost'],
+    protocol : 'gtest',
+  )
+endif

src/panfrost/shared/pan_minmax_cache.c

@@ -0,0 +1,155 @@
+/*
+ * Copyright (c) 2020 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+/* Index buffer min/max cache. We need to calculate the min/max for arbitrary
+ * slices (start, start + count) of the index buffer at drawtime. As this can
+ * be quite expensive, we cache. Conceptually, we just use a hash table mapping
+ * the key (start, count) to the value (min, max). In practice, mesa's hash
+ * table implementation is higher overhead than we would like and makes
+ * handling memory usage a little complicated. So we use this data structure
+ * instead. Searching is O(n) to the size, but the size is capped at the
+ * PANFROST_MINMAX_SIZE constant (so this is a tradeoff between cache hit/miss
+ * ratio and cache search speed). Note that keys are adjacent so we get cache
+ * line alignment benefits. Insertion is O(1) and in-order until the cache
+ * fills up, after that it evicts the oldest cached value in a ring facilitated
+ * by index.
+ */
+
+#include "pan_minmax_cache.h"
+#include "util/macros.h"
+
+/*
+ * note: a count of 0 would be an empty range, which we don't have to
+ * cache; so returning a 0 to indicate "do not cache" is sensible
+ * otherwise create a key that encodes the start, count, and index size
+ */
+static uint64_t
+pan_calc_cache_key(struct pan_minmax_cache *cache, unsigned index_size,
+                   unsigned start, unsigned count)
+{
+   uint64_t ht_key;
+
+   if (!cache)
+      return 0; /* do not cache if no cache! */
+
+   /* we're going to store the item size in the upper bits of the count;
+    * if the count is too big to do this safely, bail and do not use
+    * the cache (this case is going to be horrible no matter what we do,
+    * and is highly unlikely)
+    */
+   if (count > 0x3FFFFFFF)
+      return 0;  /* do not cache */
+
+   /* find log2(index_size) or die tryin' */
+   switch (index_size) {
+   case 1:
+      index_size = 0;
+      break;
+   case 2:
+      index_size = 1;
+      break;
+   case 4:
+      index_size = 2;
+      break;
+   default:
+      UNREACHABLE("unknown index size");
+   }
+   count = count | (index_size << 30);
+   ht_key = ((uint64_t)count << 32) | start;
+   return ht_key;
+}
+
+bool
+pan_minmax_cache_get(struct pan_minmax_cache *cache, unsigned index_size,
+                     unsigned start, unsigned count, unsigned *min_index,
+                     unsigned *max_index)
+{
+   uint64_t ht_key = pan_calc_cache_key(cache, index_size, start, count);
+   bool found = false;
+
+   if (!ht_key)
+      return false;
+
+
+   for (unsigned i = 0; i < cache->size; ++i) {
+      if (cache->keys[i] == ht_key) {
+         uint64_t hit = cache->values[i];
+
+         *min_index = hit & 0xffffffff;
+         *max_index = hit >> 32;
+         found = true;
+         break;
+      }
+   }
+
+   return found;
+}
+
+void
+pan_minmax_cache_add(struct pan_minmax_cache *cache, unsigned index_size,
+                     unsigned start, unsigned count, unsigned min_index,
+                     unsigned max_index)
+{
+   uint64_t ht_key = pan_calc_cache_key(cache, index_size, start, count);
+   uint64_t value = min_index | (((uint64_t)max_index) << 32);
+   unsigned index = 0;
+
+   if (!ht_key)
+      return;
+
+   if (cache->size == PANFROST_MINMAX_SIZE) {
+      index = cache->index++;
+      cache->index = cache->index % PANFROST_MINMAX_SIZE;
+   } else {
+      index = cache->size++;
+   }
+
+   cache->keys[index] = ht_key;
+   cache->values[index] = value;
+}
+
+/* If we've been caching min/max indices and we update the index
+ * buffer, that may invalidate the min/max. Check what's been cached vs
+ * what we've written, and throw out invalid entries. */
+
+void
+pan_minmax_cache_invalidate(struct pan_minmax_cache *cache, unsigned index_size,
+                            size_t offset, size_t size)
+{
+   /* Ensure there is a cache to invalidate and a write */
+   if (!cache)
+      return;
+
+   /* convert offset and size to bytes, so that if we
+      update a region using a different item size we
+      still invalidate it */
+   offset *= index_size;
+   size *= index_size;
+   unsigned valid_count = 0;
+
+   for (unsigned i = 0; i < cache->size; ++i) {
+      uint64_t key = cache->keys[i];
+
+      /* the item size is in the upper 2 bits of the key
+       * as above, convert size and count to bytes to make
+       * region comparison agnostic to item size
+       */
+      uint32_t key_index_size = (key >> 62);
+      size_t count = ((key >> 32) & 0x3fffffff) << key_index_size;
+      size_t start = (key & 0xffffffff) << key_index_size;
+
+      /* 1D range intersection */
+      bool invalid = MAX2(offset, start) <
+                     MIN2(offset + size, start + count);
+      if (!invalid) {
+         cache->keys[valid_count] = key;
+         cache->values[valid_count] = cache->values[i];
+         valid_count++;
+      }
+   }
+
+   cache->size = valid_count;
+   cache->index = 0;
+}

src/panfrost/shared/pan_minmax_cache.h

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2020 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef H_PAN_MINMAX_CACHE
+#define H_PAN_MINMAX_CACHE
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <stddef.h>
+
+#define PANFROST_MINMAX_SIZE 64
+
+struct pan_minmax_cache {
+   uint64_t keys[PANFROST_MINMAX_SIZE];
+   uint64_t values[PANFROST_MINMAX_SIZE];
+   unsigned size;
+   unsigned index;
+};
+
+bool pan_minmax_cache_get(struct pan_minmax_cache *cache, unsigned index_size,
+                          unsigned start, unsigned count, unsigned *min_index,
+                          unsigned *max_index);
+
+void pan_minmax_cache_add(struct pan_minmax_cache *cache, unsigned index_size,
+                          unsigned start, unsigned count, unsigned min_index,
+                          unsigned max_index);
+
+void pan_minmax_cache_invalidate(struct pan_minmax_cache *cache,
+                                 unsigned index_size, size_t offset,
+                                 size_t size);
+
+#endif

src/panfrost/shared/pan_tiling.c

@@ -0,0 +1,525 @@
+/*
+ * Copyright (c) 2011-2013 Luc Verhaegen <[email protected]>
+ * Copyright (c) 2018 Alyssa Rosenzweig
+ * Copyright (c) 2018 Vasily Khoruzhick <[email protected]>
+ * Copyright (c) 2019 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "pan_tiling.h"
+#include <math.h>
+#include <stdbool.h>
+#include "util/macros.h"
+#include "util/ralloc.h"
+
+/*
+ * This file implements software encode/decode of u-interleaved textures.
+ * See docs/drivers/panfrost.rst for details on the format.
+ *
+ * The tricky bit is ordering along the space-filling curve:
+ *
+ *    | y3 | (x3 ^ y3) | y2 | (y2 ^ x2) | y1 | (y1 ^ x1) | y0 | (y0 ^ x0) |
+ *
+ * While interleaving bits is trivial in hardware, it is nontrivial in software.
+ * The trick is to divide the pattern up:
+ *
+ *    | y3 | y3 | y2 | y2 | y1 | y1 | y0 | y0 |
+ *  ^ |  0 | x3 |  0 | x2 |  0 | x1 |  0 | x0 |
+ *
+ * That is, duplicate the bits of the Y and space out the bits of the X. The top
+ * line is a function only of Y, so it can be calculated once per row and stored
+ * in a register. The bottom line is simply X with the bits spaced out. Spacing
+ * out the X is easy enough with a LUT, or by subtracting+ANDing the mask
+ * pattern (abusing carry bits).
+ *
+ */
+
+/*
+ * Given the lower 4-bits of the Y coordinate, we would like to
+ * duplicate every bit over. So instead of 0b1010, we would like
+ * 0b11001100. The idea is that for the bits in the solely Y place, we
+ * get a Y place, and the bits in the XOR place *also* get a Y.
+ */
+/* clang-format off */
+const uint32_t bit_duplication[16] = {
+   0b00000000,
+   0b00000011,
+   0b00001100,
+   0b00001111,
+   0b00110000,
+   0b00110011,
+   0b00111100,
+   0b00111111,
+   0b11000000,
+   0b11000011,
+   0b11001100,
+   0b11001111,
+   0b11110000,
+   0b11110011,
+   0b11111100,
+   0b11111111,
+};
+/* clang-format on */
+
+/*
+ * Space the bits out of a 4-bit nibble
+ */
+/* clang-format off */
+const unsigned space_4[16] = {
+   0b0000000,
+   0b0000001,
+   0b0000100,
+   0b0000101,
+   0b0010000,
+   0b0010001,
+   0b0010100,
+   0b0010101,
+   0b1000000,
+   0b1000001,
+   0b1000100,
+   0b1000101,
+   0b1010000,
+   0b1010001,
+   0b1010100,
+   0b1010101
+};
+/* clang-format on */
+
+/* The scheme uses 16x16 tiles */
+
+#define TILE_WIDTH      16
+#define TILE_HEIGHT     16
+#define PIXELS_PER_TILE (TILE_WIDTH * TILE_HEIGHT)
+
+enum pan_interleave_zs
+pan_get_interleave_zs(enum pipe_format format, bool depth, bool stencil)
+{
+   if (format == PIPE_FORMAT_Z24_UNORM_S8_UINT) {
+      if (depth && stencil) {
+         return PAN_INTERLEAVE_NONE;
+      } else if (depth && !stencil) {
+         return PAN_INTERLEAVE_DEPTH;
+      } else if (!depth && stencil) {
+         return PAN_INTERLEAVE_STENCIL;
+      } else if (!depth && !stencil) {
+         UNREACHABLE("at least one aspect must be specified");
+      }
+   }
+   return PAN_INTERLEAVE_NONE;
+}
+
+/* Optimized routine to tile an aligned (w & 0xF == 0) texture. Explanation:
+ *
+ * dest_start precomputes the offset to the beginning of the first horizontal
+ * tile we're writing to, knowing that x is 16-aligned. Tiles themselves are
+ * stored linearly, so we get the X tile number by shifting and then multiply
+ * by the bytes per tile .
+ *
+ * We iterate across the pixels we're trying to store in source-order. For each
+ * row in the destination image, we figure out which row of 16x16 block we're
+ * in, by slicing off the lower 4-bits (block_y).
+ *
+ * dest then precomputes the location of the top-left corner of the block the
+ * row starts in. In pixel coordinates (where the origin is the top-left),
+ * (block_y, 0) is the top-left corner of the leftmost tile in this row.  While
+ * pixels are reordered within a block, the blocks themselves are stored
+ * linearly, so multiplying block_y by the pixel stride of the destination
+ * image equals the byte offset of that top-left corner of the block this row
+ * is in.
+ *
+ * On the other hand, the source is linear so we compute the locations of the
+ * start and end of the row in the source by a simple linear addressing.
+ *
+ * For indexing within the tile, we need to XOR with the [y3 y3 y2 y2 y1 y1 y0
+ * y0] value. Since this is constant across a row, we look it up per-row and
+ * store in expanded_y.
+ *
+ * Finally, we iterate each row in source order. In the outer loop, we iterate
+ * each 16 pixel tile. Within each tile, we iterate the 16 pixels (this should
+ * be unrolled), calculating the index within the tile and writing.
+ */
+
+static ALWAYS_INLINE void
+pan_access_tiled_image_aligned(
+   void *dst, void *src,
+   unsigned dst_pixel_size, unsigned src_pixel_size,
+   unsigned shift,
+   uint16_t sx, uint16_t sy,
+   uint16_t w, uint16_t h,
+   uint32_t dst_stride, uint32_t src_stride,
+   enum pan_interleave_zs interleave,
+   bool is_store)
+{
+   uint8_t *dest_start = dst + ((sx >> 4) * PIXELS_PER_TILE * dst_pixel_size);
+   for (int y = sy, src_y = 0; src_y < h; ++y, ++src_y) {
+      uint8_t *dest = (uint8_t *)(dest_start + ((y >> 4) * dst_stride));
+      void *source = src + (src_y * src_stride);
+      void *source_end = source + w * src_pixel_size;
+      unsigned expanded_y = bit_duplication[y & 0xF] << shift;
+      for (; source < source_end; dest += (PIXELS_PER_TILE << shift)) {
+         for (uint8_t i = 0; i < 16; ++i) {
+            unsigned index = expanded_y ^ (space_4[i] << shift);
+            pan_access_image_pixel(dest + index, source, dst_pixel_size,
+                                   interleave, is_store);
+            source += src_pixel_size;
+         }
+      }
+   }
+}
+
+static ALWAYS_INLINE void
+pan_access_tiled_image_unaligned(
+   void *dst, void *src,
+   unsigned dst_pixel_size, unsigned src_pixel_size,
+   unsigned tile_shift,
+   uint16_t sx, uint16_t sy,
+   uint16_t w, uint16_t h,
+   uint32_t dst_stride, uint32_t src_stride,
+   enum pan_interleave_zs interleave,
+   bool is_store)
+{
+   const unsigned mask = (1 << tile_shift) - 1;
+   for (int y = sy, src_y = 0; src_y < h; ++y, ++src_y) {
+      unsigned block_start_s = (y >> tile_shift) * dst_stride;
+      unsigned source_start = src_y * src_stride;
+      unsigned expanded_y = bit_duplication[y & mask];
+
+      for (int x = sx, src_x = 0; src_x < w; ++x, ++src_x) {
+         unsigned block_x_s = (x >> tile_shift) * (1 << (tile_shift * 2));
+         unsigned index = expanded_y ^ space_4[x & mask];
+         uint8_t *source = src + source_start + src_pixel_size * src_x;
+         uint8_t *dest =
+            dst + block_start_s + dst_pixel_size * (block_x_s + index);
+         pan_access_image_pixel(dest, source, dst_pixel_size, interleave,
+                                is_store);
+      }
+   }
+}
+
+#define TILED_UNALIGNED_VARIANT(dst_bpp, src_bpp, interleave, store, shift)    \
+   pan_access_tiled_image_unaligned(dst, src, (dst_bpp) / 8, (src_bpp) / 8,    \
+                                    shift, sx, sy, w, h,                       \
+                                    dst_stride, src_stride, interleave, store)
+
+/* We have a separate call for each configuration, to ensure that the inlined
+ * function is specialized */
+#define TILED_UNALIGNED_VARIANTS(store, shift)                                 \
+   {                                                                           \
+      if (bpp == 8)                                                            \
+         TILED_UNALIGNED_VARIANT(8, 8, PAN_INTERLEAVE_NONE, store, shift);     \
+      else if (bpp == 16)                                                      \
+         TILED_UNALIGNED_VARIANT(16, 16, PAN_INTERLEAVE_NONE, store, shift);   \
+      else if (bpp == 24)                                                      \
+         TILED_UNALIGNED_VARIANT(24, 24, PAN_INTERLEAVE_NONE, store, shift);   \
+      else if (bpp == 32 && interleave == PAN_INTERLEAVE_NONE)                 \
+         TILED_UNALIGNED_VARIANT(32, 32, PAN_INTERLEAVE_NONE, store, shift);   \
+      else if (bpp == 32 && interleave == PAN_INTERLEAVE_DEPTH)                \
+         TILED_UNALIGNED_VARIANT(32, 32, PAN_INTERLEAVE_DEPTH, store, shift);  \
+      else if (bpp == 32 && interleave == PAN_INTERLEAVE_STENCIL)              \
+         TILED_UNALIGNED_VARIANT(32, 8, PAN_INTERLEAVE_STENCIL, store, shift); \
+      else if (bpp == 48)                                                      \
+         TILED_UNALIGNED_VARIANT(48, 48, PAN_INTERLEAVE_NONE, store, shift);   \
+      else if (bpp == 64)                                                      \
+         TILED_UNALIGNED_VARIANT(64, 64, PAN_INTERLEAVE_NONE, store, shift);   \
+      else if (bpp == 96)                                                      \
+         TILED_UNALIGNED_VARIANT(96, 96, PAN_INTERLEAVE_NONE, store, shift);   \
+      else if (bpp == 128)                                                     \
+         TILED_UNALIGNED_VARIANT(128, 128, PAN_INTERLEAVE_NONE, store, shift); \
+   }
+
+/*
+ * Perform a generic access to a tiled image with a given format. This works
+ * even for block-compressed images on entire blocks at a time. sx/sy/w/h are
+ * specified in pixels, not blocks, but our internal routines work in blocks,
+ * so we divide here. Alignment is assumed.
+ */
+static void
+pan_access_tiled_image_generic(void *dst, void *src, unsigned sx, unsigned sy,
+                               unsigned w, unsigned h, uint32_t dst_stride,
+                               uint32_t src_stride,
+                               const struct util_format_description *desc,
+                               enum pan_interleave_zs interleave,
+                               bool _is_store)
+{
+   unsigned bpp = desc->block.bits;
+
+   /* Convert units */
+   sx /= desc->block.width;
+   sy /= desc->block.height;
+   w = DIV_ROUND_UP(w, desc->block.width);
+   h = DIV_ROUND_UP(h, desc->block.height);
+
+   if (desc->block.width > 1) {
+      if (_is_store)
+         TILED_UNALIGNED_VARIANTS(true, 2)
+      else
+         TILED_UNALIGNED_VARIANTS(false, 2)
+   } else {
+      if (_is_store)
+         TILED_UNALIGNED_VARIANTS(true, 4)
+      else
+         TILED_UNALIGNED_VARIANTS(false, 4)
+   }
+}
+
+#define TILED_ALIGNED_VARIANT(interleave, store, dst_bpp, src_bpp, shift)      \
+   pan_access_tiled_image_aligned(dst, src, (dst_bpp) / 8, (src_bpp) / 8,      \
+                                  shift, sx, sy, w, h,                         \
+                                  dst_stride, src_stride, interleave, store)
+
+#define TILED_ALIGNED_VARIANTS(store)                                          \
+   {                                                                           \
+      if (bpp == 8)                                                            \
+         TILED_ALIGNED_VARIANT(PAN_INTERLEAVE_NONE, store, 8, 8, 0);           \
+      else if (bpp == 16)                                                      \
+         TILED_ALIGNED_VARIANT(PAN_INTERLEAVE_NONE, store, 16, 16, 1);         \
+      else if (bpp == 32 && interleave == PAN_INTERLEAVE_NONE)                 \
+         TILED_ALIGNED_VARIANT(PAN_INTERLEAVE_NONE, store, 32, 32, 2);         \
+      else if (bpp == 32 && interleave == PAN_INTERLEAVE_DEPTH)                \
+         TILED_ALIGNED_VARIANT(PAN_INTERLEAVE_DEPTH, store, 32, 32, 2);        \
+      else if (bpp == 32 && interleave == PAN_INTERLEAVE_STENCIL)              \
+         TILED_ALIGNED_VARIANT(PAN_INTERLEAVE_STENCIL, store, 32, 8, 2);       \
+      else if (bpp == 64)                                                      \
+         TILED_ALIGNED_VARIANT(PAN_INTERLEAVE_NONE, store, 64, 64, 3);         \
+      else if (bpp == 128)                                                     \
+         TILED_ALIGNED_VARIANT(PAN_INTERLEAVE_NONE, store, 128, 128, 4);       \
+   }
+
+/* Optimized variant of pan_access_tiled_image_generic except that requires
+ * sx/sy/w/h to be tile-aligned, and bpp to be a power of two */
+static void
+pan_access_tiled_image_generic_aligned(
+   void *dst, void *src, unsigned sx, unsigned sy, unsigned w, unsigned h,
+   uint32_t dst_stride, uint32_t src_stride,
+   const struct util_format_description *desc,
+   enum pan_interleave_zs interleave, bool is_store)
+{
+   unsigned bpp = desc->block.bits;
+
+   assert(sx % TILE_WIDTH == 0);
+   assert(sy % TILE_HEIGHT == 0);
+   assert(w % TILE_WIDTH == 0);
+   assert(h % TILE_HEIGHT == 0);
+   assert(util_is_power_of_two_nonzero(bpp));
+
+   if (is_store)
+      TILED_ALIGNED_VARIANTS(true)
+   else
+      TILED_ALIGNED_VARIANTS(false)
+}
+
+
+#define OFFSET(src, _x, _y)                                                    \
+   (void *)((uint8_t *)src + ((_y)-orig_y) * src_stride +                      \
+            (((_x)-orig_x) * (bpp / 8)))
+
+static ALWAYS_INLINE void
+pan_access_tiled_image(void *dst, void *src, unsigned x, unsigned y, unsigned w,
+                       unsigned h, uint32_t dst_stride, uint32_t src_stride,
+                       enum pipe_format format,
+                       enum pan_interleave_zs interleave, bool is_store)
+{
+   if (interleave != PAN_INTERLEAVE_NONE)
+      assert(format == PIPE_FORMAT_Z24_UNORM_S8_UINT);
+
+   const struct util_format_description *desc = util_format_description(format);
+   unsigned bpp = desc->block.bits;
+
+   /* Our optimized routines cannot handle unaligned blocks (without depending
+    * on platform-specific behaviour), and there is no good reason to do so. If
+    * these assertions fail, there is either a driver bug or a non-portable unit
+    * test.
+    */
+   assert((dst_stride % (bpp / 8)) == 0 && "unaligned destination stride");
+   assert((src_stride % (bpp / 8)) == 0 && "unaligned source stride");
+
+   if (desc->block.width > 1 ||
+       !util_is_power_of_two_nonzero(desc->block.bits)) {
+      pan_access_tiled_image_generic(dst, (void *)src, x, y, w, h, dst_stride,
+                                     src_stride, desc, interleave, is_store);
+
+      return;
+   }
+
+   unsigned first_full_tile_x = DIV_ROUND_UP(x, TILE_WIDTH) * TILE_WIDTH;
+   unsigned first_full_tile_y = DIV_ROUND_UP(y, TILE_HEIGHT) * TILE_HEIGHT;
+   unsigned last_full_tile_x = ((x + w) / TILE_WIDTH) * TILE_WIDTH;
+   unsigned last_full_tile_y = ((y + h) / TILE_HEIGHT) * TILE_HEIGHT;
+
+   /* First, tile the top portion */
+
+   unsigned orig_x = x, orig_y = y;
+
+   if (first_full_tile_y != y) {
+      unsigned dist = MIN2(first_full_tile_y - y, h);
+
+      pan_access_tiled_image_generic(dst, OFFSET(src, x, y), x, y, w, dist,
+                                     dst_stride, src_stride, desc, interleave,
+                                     is_store);
+
+      if (dist == h)
+         return;
+
+      y += dist;
+      h -= dist;
+   }
+
+   /* Next, the bottom portion */
+   if (last_full_tile_y != (y + h)) {
+      unsigned dist = (y + h) - last_full_tile_y;
+
+      pan_access_tiled_image_generic(dst, OFFSET(src, x, last_full_tile_y), x,
+                                     last_full_tile_y, w, dist, dst_stride,
+                                     src_stride, desc, interleave, is_store);
+
+      h -= dist;
+   }
+
+   /* The left portion */
+   if (first_full_tile_x != x) {
+      unsigned dist = MIN2(first_full_tile_x - x, w);
+
+      pan_access_tiled_image_generic(dst, OFFSET(src, x, y), x, y, dist, h,
+                                     dst_stride, src_stride, desc, interleave,
+                                     is_store);
+
+      if (dist == w)
+         return;
+
+      x += dist;
+      w -= dist;
+   }
+
+   /* Finally, the right portion */
+   if (last_full_tile_x != (x + w)) {
+      unsigned dist = (x + w) - last_full_tile_x;
+
+      pan_access_tiled_image_generic(dst, OFFSET(src, last_full_tile_x, y),
+                                     last_full_tile_x, y, dist, h, dst_stride,
+                                     src_stride, desc, interleave, is_store);
+
+      w -= dist;
+   }
+
+   pan_access_tiled_image_generic_aligned(dst, OFFSET(src, x, y), x, y, w,
+                                          h, dst_stride, src_stride, desc,
+                                          interleave, is_store);
+}
+
+/**
+ * Access a tiled image (load or store). Note: the region of interest (x, y, w,
+ * h) is specified in pixels, not blocks. It is expected that these quantities
+ * are aligned to the block size.
+ */
+void
+pan_store_tiled_image(void *dst, const void *src, unsigned x, unsigned y,
+                      unsigned w, unsigned h, uint32_t dst_stride,
+                      uint32_t src_stride, enum pipe_format format,
+                      enum pan_interleave_zs interleave)
+{
+   pan_access_tiled_image(dst, (void *)src, x, y, w, h, dst_stride, src_stride,
+                          format, interleave, true);
+}
+
+void
+pan_load_tiled_image(void *dst, const void *src, unsigned x, unsigned y,
+                     unsigned w, unsigned h, uint32_t dst_stride,
+                     uint32_t src_stride, enum pipe_format format,
+                     enum pan_interleave_zs interleave)
+{
+   pan_access_tiled_image((void *)src, dst, x, y, w, h, src_stride, dst_stride,
+                          format, interleave, false);
+}
+
+void
+pan_copy_tiled_image(void *dst, const void *src, unsigned dst_x, unsigned dst_y,
+                     unsigned src_x, unsigned src_y, unsigned w, unsigned h,
+                     uint32_t dst_stride, uint32_t src_stride,
+                     enum pipe_format format)
+{
+   const struct util_format_description *desc = util_format_description(format);
+   unsigned block_size_B = desc->block.bits / 8;
+
+   /* If both the src and dst region are tile-aligned, we can just memcpy
+    * whole tiles without any (de)tiling */
+   if (src_x % TILE_WIDTH == 0 && src_y % TILE_HEIGHT == 0 &&
+       dst_x % TILE_WIDTH == 0 && dst_y % TILE_HEIGHT == 0 &&
+       w % TILE_WIDTH == 0 && h % TILE_HEIGHT == 0) {
+
+      unsigned tile_size_B = block_size_B * PIXELS_PER_TILE;
+
+      unsigned w_t = w / TILE_WIDTH;
+      unsigned h_t = h / TILE_HEIGHT;
+      unsigned src_x_t = src_x / TILE_WIDTH;
+      unsigned src_y_t = src_y / TILE_HEIGHT;
+      unsigned dst_x_t = dst_x / TILE_WIDTH;
+      unsigned dst_y_t = dst_y / TILE_HEIGHT;
+
+      for (unsigned y_t = 0; y_t < h_t; y_t++) {
+         void *dst_tile_row = dst +
+            (y_t + dst_y_t) * dst_stride +
+            dst_x_t * tile_size_B;
+         const void *src_tile_row = src +
+            (y_t + src_y_t) * src_stride +
+            src_x_t * tile_size_B;
+         memcpy(dst_tile_row, src_tile_row, tile_size_B * w_t);
+      }
+
+      return;
+   }
+
+   /* Otherwise, we copy by working across the copy region in 64KiB chunks.
+    * For each chunk, we detile part of the src into a linear tempoaray
+    * buffer, then tile to the dst */
+
+   /* This could fit on the stack easily on glibc, but it's dicier on musl,
+    * which has a 128KiB stack size */
+   const size_t chunk_size_B = 65536;
+   void *chunk = ralloc_size(NULL, chunk_size_B);
+
+   /* Choose pixel dimensions of the chunk. These should be tile aligned,
+    * maximize used space in the buffer, and be close to a square. */
+   unsigned chunk_size_bl = chunk_size_B / block_size_B;
+   unsigned chunk_width_bl = (unsigned) sqrtf((float) (chunk_size_bl));
+   chunk_width_bl = (chunk_width_bl / TILE_WIDTH) * TILE_WIDTH;
+   unsigned chunk_height_bl = chunk_size_bl / chunk_width_bl;
+   chunk_height_bl = (chunk_height_bl / TILE_HEIGHT) * TILE_HEIGHT;
+
+   unsigned chunk_width_px = chunk_width_bl * desc->block.width;
+   unsigned chunk_height_px = chunk_height_bl * desc->block.height;
+
+   unsigned chunk_row_stride_B = chunk_width_bl * block_size_B;
+
+   /* Align chunk copy regions to src tiles, to optimize detiling. We can't
+    * get tile alignment on both src and dst, but one is better than nothing. */
+   unsigned src_first_tile_x = (src_x / TILE_WIDTH) * TILE_WIDTH;
+   unsigned src_first_tile_y = (src_y / TILE_HEIGHT) * TILE_HEIGHT;
+
+   for (unsigned x = src_first_tile_x; x < src_x + w; x += chunk_width_px) {
+      for (unsigned y = src_first_tile_y; y < src_y + h; y += chunk_height_px) {
+         /* x/y are tile-aligned, but because the actual copy region is not,
+          * we may need to start at an offset position on the left/top edges */
+         unsigned src_chunk_x = MAX2(src_x, x);
+         unsigned src_chunk_y = MAX2(src_y, y);
+         unsigned dst_chunk_x = dst_x + (src_chunk_x - src_x);
+         unsigned dst_chunk_y = dst_y + (src_chunk_y - src_y);
+
+         /* Similarly, right/bottom edges may not need a whole chunk */
+         unsigned src_chunk_right = MIN2(src_chunk_x + chunk_width_px,
+                                         src_x + w);
+         unsigned src_chunk_bottom = MIN2(src_chunk_y + chunk_height_px,
+                                          src_y + h);
+         unsigned width = src_chunk_right - src_chunk_x;
+         unsigned height = src_chunk_bottom - src_chunk_y;
+
+         pan_load_tiled_image(
+            chunk, src, src_chunk_x, src_chunk_y, width, height,
+            chunk_row_stride_B, src_stride, format, PAN_INTERLEAVE_NONE);
+         pan_store_tiled_image(
+            dst, chunk, dst_chunk_x, dst_chunk_y, width, height, dst_stride,
+            chunk_row_stride_B, format, PAN_INTERLEAVE_NONE);
+      }
+   }
+
+   ralloc_free(chunk);
+}

src/panfrost/shared/pan_tiling.h

@@ -0,0 +1,164 @@
+/*
+ * Copyright (c) 2011-2013 Luc Verhaegen <[email protected]>
+ * Copyright (c) 2018 Alyssa Rosenzweig
+ * Copyright (c) 2018 Vasily Khoruzhick <[email protected]>
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef H_PANFROST_TILING
+#define H_PANFROST_TILING
+
+#include <stdint.h>
+#include "util/bitscan.h"
+#include <util/format/u_format.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* The depth and stencil aspects of a Z24_UNORM_S8_UINT image are interleaved,
+ * where the bottom 24 bits are depth and the top 8 bits are stencil. When
+ * copying to/from a Z24S8 tiled image, the pan_interleave_zs enum specifies
+ * whether to (de)interleave the depth/stencil aspects */
+enum pan_interleave_zs {
+   /* Copy all aspects, no interleaving */
+   PAN_INTERLEAVE_NONE,
+   /* Copy only the depth aspect of a Z24S8 tiled image to/from linear Z24X8 */
+   PAN_INTERLEAVE_DEPTH,
+   /* Copy only the stencil aspect of a Z24S8 tiled image to/from linear S8 */
+   PAN_INTERLEAVE_STENCIL,
+};
+
+/**
+ * Get the appropriate pan_interleave_zs mode for copying to/from a given
+ * format.
+ *
+ * @depth Whether to copy the depth aspect
+ * @stencil Whether to copy the stencil aspect
+ */
+enum pan_interleave_zs
+pan_get_interleave_zs(enum pipe_format format, bool depth, bool stencil);
+
+/**
+ * Load a rectangular region from a tiled image to a linear staging image.
+ *
+ * @dst Linear destination
+ * @src Tiled source
+ * @x Region of interest of source in pixels, aligned to block size
+ * @y Region of interest of source in pixels, aligned to block size
+ * @w Region of interest of source in pixels, aligned to block size
+ * @h Region of interest of source in pixels, aligned to block size
+ * @dst_stride Stride in bytes of linear destination
+ * @src_stride Number of bytes between adjacent rows of tiles in source.
+ * @format Format of the source and destination image
+ * @interleave How to deinterleave ZS aspects from the tiled image
+ */
+void pan_load_tiled_image(void *dst, const void *src, unsigned x, unsigned y,
+                          unsigned w, unsigned h, uint32_t dst_stride,
+                          uint32_t src_stride, enum pipe_format format,
+                          enum pan_interleave_zs interleave);
+
+/**
+ * Store a linear staging image to a rectangular region of a tiled image.
+ *
+ * @dst Tiled destination
+ * @src Linear source
+ * @x Region of interest of destination in pixels, aligned to block size
+ * @y Region of interest of destination in pixels, aligned to block size
+ * @w Region of interest of destination in pixels, aligned to block size
+ * @h Region of interest of destination in pixels, aligned to block size
+ * @dst_stride Number of bytes between adjacent rows of tiles in destination.
+ * @src_stride Stride in bytes of linear source
+ * @format Format of the source and destination image
+ * @interleave How to interleave a ZS aspects to the tiled image
+ */
+void pan_store_tiled_image(void *dst, const void *src, unsigned x, unsigned y,
+                           unsigned w, unsigned h, uint32_t dst_stride,
+                           uint32_t src_stride, enum pipe_format format,
+                           enum pan_interleave_zs interleave);
+
+/**
+ * Copy a rectangular region from one tiled image to another.
+ *
+ * @dst Tiled destination
+ * @src Tiled source
+ * @dst_x Region of interest of destination in pixels, aligned to block size
+ * @dst_y Region of interest of destination in pixels, aligned to block size
+ * @src_x Region of interest of source in pixels, aligned to block size
+ * @src_y Region of interest of source in pixels, aligned to block size
+ * @w Size of region of interest in pixels, aligned to block size
+ * @h Size of region of interest in pixels, aligned to block size
+ * @dst_stride Number of bytes between adjacent rows of tiles in destination.
+ * @src_stride Number of bytes between adjacent rows of tiles in source.
+ * @format Format of the source and destination image
+ */
+void pan_copy_tiled_image(void *dst, const void *src, unsigned dst_x,
+                          unsigned dst_y, unsigned src_x, unsigned src_y,
+                          unsigned w, unsigned h, uint32_t dst_stride,
+                          uint32_t src_stride, enum pipe_format format);
+
+static inline
+void *assume_pixel_aligned(void *ptr, const unsigned pixel_size)
+{
+    if (pixel_size == 1)
+        return __builtin_assume_aligned(ptr, 1);
+    else if (pixel_size == 2)
+        return __builtin_assume_aligned(ptr, 2);
+    else if (pixel_size == 4)
+        return __builtin_assume_aligned(ptr, 4);
+    else if (pixel_size == 8)
+        return __builtin_assume_aligned(ptr, 8);
+    else if (pixel_size == 16)
+        return __builtin_assume_aligned(ptr, 16);
+    else
+        UNREACHABLE("unexpected pixel size");
+}
+
+static inline
+void pan_access_image_pixel(void *dst, void *src, const unsigned pixel_size,
+                            enum pan_interleave_zs interleave, bool is_store)
+{
+   if (util_is_power_of_two_nonzero(pixel_size)) {
+      src = assume_pixel_aligned(src, pixel_size);
+      if (interleave != PAN_INTERLEAVE_STENCIL)
+         dst = assume_pixel_aligned(dst, pixel_size);
+   }
+
+   switch (interleave) {
+      case PAN_INTERLEAVE_NONE:
+         if (is_store)
+            memcpy(dst, src, pixel_size);
+         else
+            memcpy(src, dst, pixel_size);
+         break;
+      case PAN_INTERLEAVE_DEPTH:
+         /* interleave only applies to Z24S8 */
+         assert(pixel_size == 4);
+         if (is_store) {
+            uint32_t src_pixel = *(uint32_t *) src;
+            *(uint16_t *) dst = src_pixel & 0xffff;
+            *((uint8_t *) dst + 2) = (src_pixel >> 16) & 0xff;
+         } else {
+            /* The top 8 bits of Z24X8 are unused, so we can overwrite them
+             * with zeros in a single 32B write, instead of needing separate
+             * 16B and 8B writes */
+            *(uint32_t *) src = *(uint32_t *) dst & 0xffffff;
+         }
+         break;
+      case PAN_INTERLEAVE_STENCIL:
+         /* interleave only applies to Z24S8 */
+         assert(pixel_size == 4);
+         if (is_store)
+            *((uint8_t *) dst + 3) = *(uint8_t *) src;
+         else
+            *(uint8_t *) src = *((uint8_t *) dst + 3);
+         break;
+   }
+}
+
+
+#ifdef __cplusplus
+} /* extern C */
+#endif
+
+#endif

src/panfrost/shared/test/test-tiling.cpp

@@ -0,0 +1,265 @@
+/*
+ * Copyright (C) 2022 Collabora, Ltd.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "pan_tiling.h"
+
+#include <gtest/gtest.h>
+
+/*
+ * Reference tiling algorithm, written for clarity rather than performance. See
+ * docs/drivers/panfrost.rst for details on the format.
+ */
+
+static unsigned
+u_order(unsigned x, unsigned y)
+{
+   assert(x < 16 && y < 16);
+
+   unsigned xy0 = ((x ^ y) & 1) ? 1 : 0;
+   unsigned xy1 = ((x ^ y) & 2) ? 1 : 0;
+   unsigned xy2 = ((x ^ y) & 4) ? 1 : 0;
+   unsigned xy3 = ((x ^ y) & 8) ? 1 : 0;
+
+   unsigned y0 = (y & 1) ? 1 : 0;
+   unsigned y1 = (y & 2) ? 1 : 0;
+   unsigned y2 = (y & 4) ? 1 : 0;
+   unsigned y3 = (y & 8) ? 1 : 0;
+
+   return (xy0 << 0) | (y0 << 1) | (xy1 << 2) | (y1 << 3) | (xy2 << 4) |
+          (y2 << 5) | (xy3 << 6) | (y3 << 7);
+}
+
+/* x/y are in blocks */
+static unsigned
+tiled_offset(unsigned x, unsigned y, unsigned stride, unsigned tilesize,
+             unsigned blocksize)
+{
+   unsigned tile_x = x / tilesize;
+   unsigned tile_y = y / tilesize;
+
+   unsigned x_in_tile = x % tilesize;
+   unsigned y_in_tile = y % tilesize;
+
+   unsigned index_in_tile = u_order(x_in_tile, y_in_tile);
+
+   unsigned row_offset = tile_y * stride;
+   unsigned col_offset = (tile_x * tilesize * tilesize) * blocksize;
+   unsigned block_offset = index_in_tile * blocksize;
+
+   return row_offset + col_offset + block_offset;
+}
+
+static unsigned
+linear_offset(unsigned x, unsigned y, unsigned stride, unsigned blocksize)
+{
+   return (stride * y) + (x * blocksize);
+}
+
+static void
+ref_access_tiled(void *dst, const void *src, unsigned region_x,
+                 unsigned region_y, unsigned w, unsigned h, uint32_t dst_stride,
+                 uint32_t src_stride, enum pipe_format format,
+                 bool dst_is_tiled)
+{
+   const struct util_format_description *desc = util_format_description(format);
+   ;
+
+   unsigned tilesize = (desc->block.width > 1) ? 4 : 16;
+   unsigned blocksize = (desc->block.bits / 8);
+
+   unsigned w_block = w / desc->block.width;
+   unsigned h_block = h / desc->block.height;
+
+   unsigned region_x_block = region_x / desc->block.width;
+   unsigned region_y_block = region_y / desc->block.height;
+
+   for (unsigned linear_y_block = 0; linear_y_block < h_block;
+        ++linear_y_block) {
+      for (unsigned linear_x_block = 0; linear_x_block < w_block;
+           ++linear_x_block) {
+
+         unsigned tiled_x_block = region_x_block + linear_x_block;
+         unsigned tiled_y_block = region_y_block + linear_y_block;
+
+         unsigned dst_offset, src_offset;
+
+         if (dst_is_tiled) {
+            dst_offset = tiled_offset(tiled_x_block, tiled_y_block, dst_stride,
+                                      tilesize, blocksize);
+            src_offset = linear_offset(linear_x_block, linear_y_block,
+                                       src_stride, blocksize);
+         } else {
+            dst_offset = linear_offset(linear_x_block, linear_y_block,
+                                       dst_stride, blocksize);
+            src_offset = tiled_offset(tiled_x_block, tiled_y_block, src_stride,
+                                      tilesize, blocksize);
+         }
+
+         memcpy((uint8_t *)dst + dst_offset, (const uint8_t *)src + src_offset,
+                desc->block.bits / 8);
+      }
+   }
+}
+
+/*
+ * Helper to build test cases for tiled texture access. This test suite compares
+ * the above reference tiling algorithm to the optimized algorithm used in
+ * production.
+ */
+static void
+test(unsigned width, unsigned height, unsigned rx, unsigned ry, unsigned rw,
+     unsigned rh, unsigned linear_stride, enum pipe_format format, bool store)
+{
+   unsigned bpp = util_format_get_blocksize(format);
+   unsigned tile_height = util_format_is_compressed(format) ? 4 : 16;
+
+   unsigned tiled_width = ALIGN_POT(width, 16);
+   unsigned tiled_height = ALIGN_POT(height, 16);
+   unsigned tiled_stride = tiled_width * tile_height * bpp;
+
+   unsigned dst_stride = store ? tiled_stride : linear_stride;
+   unsigned src_stride = store ? linear_stride : tiled_stride;
+
+   void *tiled = calloc(bpp, tiled_width * tiled_height);
+   void *linear = calloc(bpp, rw * linear_stride);
+   void *ref =
+      calloc(bpp, store ? (tiled_width * tiled_height) : (rw * linear_stride));
+
+   if (store) {
+      for (unsigned i = 0; i < bpp * rw * linear_stride; ++i) {
+         ((uint8_t *)linear)[i] = (i & 0xFF);
+      }
+
+      pan_store_tiled_image(tiled, linear, rx, ry, rw, rh, dst_stride,
+                            src_stride, format, PAN_INTERLEAVE_NONE);
+   } else {
+      for (unsigned i = 0; i < bpp * tiled_width * tiled_height; ++i) {
+         ((uint8_t *)tiled)[i] = (i & 0xFF);
+      }
+
+      pan_load_tiled_image(linear, tiled, rx, ry, rw, rh, dst_stride,
+                           src_stride, format, PAN_INTERLEAVE_NONE);
+   }
+
+   ref_access_tiled(ref, store ? linear : tiled, rx, ry, rw, rh, dst_stride,
+                    src_stride, format, store);
+
+   if (store)
+      EXPECT_EQ(memcmp(ref, tiled, bpp * tiled_width * tiled_height), 0);
+   else
+      EXPECT_EQ(memcmp(ref, linear, bpp * rw * linear_stride), 0);
+
+   free(ref);
+   free(tiled);
+   free(linear);
+}
+
+static void
+test_ldst(unsigned width, unsigned height, unsigned rx, unsigned ry,
+          unsigned rw, unsigned rh, unsigned linear_stride,
+          enum pipe_format format)
+{
+   test(width, height, rx, ry, rw, rh, linear_stride, format, true);
+   test(width, height, rx, ry, rw, rh, linear_stride, format, false);
+}
+
+TEST(UInterleavedTiling, RegulatFormats)
+{
+   /* 8-bit */
+   test_ldst(23, 17, 0, 0, 23, 17, 23, PIPE_FORMAT_R8_UINT);
+
+   /* 16-bit */
+   test_ldst(23, 17, 0, 0, 23, 17, 23 * 2, PIPE_FORMAT_R8G8_UINT);
+
+   /* 24-bit */
+   test_ldst(23, 17, 0, 0, 23, 17, 23 * 3, PIPE_FORMAT_R8G8B8_UINT);
+
+   /* 32-bit */
+   test_ldst(23, 17, 0, 0, 23, 17, 23 * 4, PIPE_FORMAT_R32_UINT);
+
+   /* 48-bit */
+   test_ldst(23, 17, 0, 0, 23, 17, 23 * 6, PIPE_FORMAT_R16G16B16_UINT);
+
+   /* 64-bit */
+   test_ldst(23, 17, 0, 0, 23, 17, 23 * 8, PIPE_FORMAT_R32G32_UINT);
+
+   /* 96-bit */
+   test_ldst(23, 17, 0, 0, 23, 17, 23 * 12, PIPE_FORMAT_R32G32B32_UINT);
+
+   /* 128-bit */
+   test_ldst(23, 17, 0, 0, 23, 17, 23 * 16, PIPE_FORMAT_R32G32B32A32_UINT);
+}
+
+TEST(UInterleavedTiling, UnpackedStrides)
+{
+   test_ldst(23, 17, 0, 0, 23, 17, 369 * 1, PIPE_FORMAT_R8_SINT);
+   test_ldst(23, 17, 0, 0, 23, 17, 369 * 2, PIPE_FORMAT_R8G8_SINT);
+   test_ldst(23, 17, 0, 0, 23, 17, 369 * 3, PIPE_FORMAT_R8G8B8_SINT);
+   test_ldst(23, 17, 0, 0, 23, 17, 369 * 4, PIPE_FORMAT_R32_SINT);
+   test_ldst(23, 17, 0, 0, 23, 17, 369 * 6, PIPE_FORMAT_R16G16B16_SINT);
+   test_ldst(23, 17, 0, 0, 23, 17, 369 * 8, PIPE_FORMAT_R32G32_SINT);
+   test_ldst(23, 17, 0, 0, 23, 17, 369 * 12, PIPE_FORMAT_R32G32B32_SINT);
+   test_ldst(23, 17, 0, 0, 23, 17, 369 * 16, PIPE_FORMAT_R32G32B32A32_SINT);
+}
+
+TEST(UInterleavedTiling, PartialAccess)
+{
+   test_ldst(23, 17, 3, 1, 13, 7, 369 * 1, PIPE_FORMAT_R8_UNORM);
+   test_ldst(23, 17, 3, 1, 13, 7, 369 * 2, PIPE_FORMAT_R8G8_UNORM);
+   test_ldst(23, 17, 3, 1, 13, 7, 369 * 3, PIPE_FORMAT_R8G8B8_UNORM);
+   test_ldst(23, 17, 3, 1, 13, 7, 369 * 4, PIPE_FORMAT_R32_UNORM);
+   test_ldst(23, 17, 3, 1, 13, 7, 369 * 6, PIPE_FORMAT_R16G16B16_UNORM);
+   test_ldst(23, 17, 3, 1, 13, 7, 369 * 8, PIPE_FORMAT_R32G32_UNORM);
+   test_ldst(23, 17, 3, 1, 13, 7, 369 * 12, PIPE_FORMAT_R32G32B32_UNORM);
+   test_ldst(23, 17, 3, 1, 13, 7, 369 * 16, PIPE_FORMAT_R32G32B32A32_UNORM);
+}
+
+TEST(UInterleavedTiling, ETC)
+{
+   /* Block alignment assumed */
+   test_ldst(32, 32, 0, 0, 32, 32, 512, PIPE_FORMAT_ETC1_RGB8);
+   test_ldst(32, 32, 0, 0, 32, 32, 512, PIPE_FORMAT_ETC2_RGB8A1);
+   test_ldst(32, 32, 0, 0, 32, 32, 512, PIPE_FORMAT_ETC2_RG11_SNORM);
+}
+
+TEST(UInterleavedTiling, PartialETC)
+{
+   /* Block alignment assumed */
+   test_ldst(32, 32, 4, 8, 16, 12, 512, PIPE_FORMAT_ETC1_RGB8);
+   test_ldst(32, 32, 4, 8, 16, 12, 512, PIPE_FORMAT_ETC2_RGB8A1);
+   test_ldst(32, 32, 4, 8, 16, 12, 512, PIPE_FORMAT_ETC2_RG11_SNORM);
+}
+
+TEST(UInterleavedTiling, DXT)
+{
+   /* Block alignment assumed */
+   test_ldst(32, 32, 0, 0, 32, 32, 512, PIPE_FORMAT_DXT1_RGB);
+   test_ldst(32, 32, 0, 0, 32, 32, 512, PIPE_FORMAT_DXT3_RGBA);
+   test_ldst(32, 32, 0, 0, 32, 32, 512, PIPE_FORMAT_DXT5_RGBA);
+}
+
+TEST(UInterleavedTiling, PartialDXT)
+{
+   /* Block alignment assumed */
+   test_ldst(32, 32, 4, 8, 16, 12, 512, PIPE_FORMAT_DXT1_RGB);
+   test_ldst(32, 32, 4, 8, 16, 12, 512, PIPE_FORMAT_DXT3_RGBA);
+   test_ldst(32, 32, 4, 8, 16, 12, 512, PIPE_FORMAT_DXT5_RGBA);
+}
+
+TEST(UInterleavedTiling, ASTC)
+{
+   /* Block alignment assumed */
+   test_ldst(40, 40, 0, 0, 40, 40, 512, PIPE_FORMAT_ASTC_4x4);
+   test_ldst(50, 40, 0, 0, 50, 40, 512, PIPE_FORMAT_ASTC_5x4);
+   test_ldst(50, 50, 0, 0, 50, 50, 512, PIPE_FORMAT_ASTC_5x5);
+}
+
+TEST(UInterleavedTiling, PartialASTC)
+{
+   /* Block alignment assumed */
+   test_ldst(40, 40, 4, 4, 16, 8, 512, PIPE_FORMAT_ASTC_4x4);
+   test_ldst(50, 40, 5, 4, 10, 8, 512, PIPE_FORMAT_ASTC_5x4);
+   test_ldst(50, 50, 5, 5, 10, 10, 512, PIPE_FORMAT_ASTC_5x5);
+}