hasvk: Rework H.264 decode into long mode.

This is why the previously added slice header parser exists.

Assisted-by: Claude

src/intel/vulkan_hasvk/anv_device.c

@@ -3209,10 +3209,9 @@ VkResult anv_AllocateMemory(
 
       mem->dedicated_image = image;
 
-      /* Some legacy (non-modifiers) consumers need the tiling to be set on
+      if (image->vk.wsi_legacy_scanout ||
-       * the BO.  In this case, we have a dedicated allocation.
+          (image->vk.usage & (VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR |
-       */
+                              VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR))) {
-      if (image->vk.wsi_legacy_scanout) {
          const struct isl_surf *surf = &image->planes[0].primary_surface.isl;
          result = anv_device_set_bo_tiling(device, mem->bo,
                                            surf->row_pitch_B,

src/intel/vulkan_hasvk/anv_image.c

@@ -221,6 +221,9 @@ choose_isl_surf_usage(VkImageCreateFlags vk_create_flags,
    if (vk_create_flags & VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT)
       isl_usage |= ISL_SURF_USAGE_CUBE_BIT;
 
+   if (vk_usage & VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR || vk_usage & VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR)
+      isl_usage |= ISL_SURF_USAGE_VIDEO_DECODE_BIT;
+
    /* Even if we're only using it for transfer operations, clears to depth and
     * stencil images happen as depth and stencil so they need the right ISL
     * usage bits or else things will fall apart.
@@ -690,23 +693,46 @@ add_video_buffers(struct anv_device *device,
                   struct anv_image *image,
                   const struct VkVideoProfileListInfoKHR *profile_list)
 {
-   ASSERTED bool ok;
+   VkResult result;
    unsigned size = 0;
 
    for (unsigned i = 0; i < profile_list->profileCount; i++) {
-      if (profile_list->pProfiles[i].videoCodecOperation == VK_VIDEO_CODEC_OPERATION_DECODE_H264_BIT_KHR) {
+      if (profile_list->pProfiles[i].videoCodecOperation ==
+          VK_VIDEO_CODEC_OPERATION_DECODE_H264_BIT_KHR) {
          unsigned w_mb = DIV_ROUND_UP(image->vk.extent.width, ANV_MB_WIDTH);
          unsigned h_mb = DIV_ROUND_UP(image->vk.extent.height, ANV_MB_HEIGHT);
+
+         /* On Ivy Bridge, the PRM specifies that the DMV buffer size is
+          * 557,056 bytes for a 1920x1088 frame (128x68 MBs). The hardware
+          * assumes a fixed frame width of 128 MBs regardless of actual frame
+          * width, but scales with height. This works out to 64 bytes per MB,
+          * or 8192 bytes per MB row (128 MBs * 64 bytes).
+          */
+         if (device->info->ver == 7) {
+            /* 128 MBs/row * h_mb rows * 64 bytes/MB = h_mb * 8192 */
+            size = h_mb * 8192;
+         } else {
+            /* XXX: Is this correct for Broadwell? */
             size = w_mb * h_mb * 128;
          }
       }
+   }
 
    if (size == 0)
       return VK_SUCCESS;
 
-   ok = image_binding_grow(device, image, ANV_IMAGE_MEMORY_BINDING_PRIVATE,
+   result =
-                           ANV_OFFSET_IMPLICIT, size, 65536, &image->vid_dmv_top_surface);
+      image_binding_grow(device, image, ANV_IMAGE_MEMORY_BINDING_PRIVATE,
-   return ok;
+                         ANV_OFFSET_IMPLICIT, size, 65536,
+                         &image->vid_dmv_top_surface);
+   if (result != VK_SUCCESS)
+      return result;
+
+   result =
+      image_binding_grow(device, image, ANV_IMAGE_MEMORY_BINDING_PRIVATE,
+                         ANV_OFFSET_IMPLICIT, size, 65536,
+                         &image->vid_dmv_bottom_surface);
+   return result;
 }
 
 /**
@@ -838,11 +864,12 @@ check_memory_bindings(const struct anv_device *device,
          ? ANV_IMAGE_MEMORY_BINDING_PLANE_0 + p
          : ANV_IMAGE_MEMORY_BINDING_MAIN;
 
-      /* Aliasing is incompatible with the private binding because it does not
+      /* Aliasing is generally incompatible with the private binding because
-       * live in a VkDeviceMemory.  The one exception is swapchain images.
+       * it does not live in a VkDeviceMemory.
        */
       assert(!(image->vk.create_flags & VK_IMAGE_CREATE_ALIAS_BIT) ||
              image->from_wsi ||
+             (plane->primary_surface.isl.usage & ISL_SURF_USAGE_VIDEO_DECODE_BIT) ||
              image->bindings[ANV_IMAGE_MEMORY_BINDING_PRIVATE].memory_range.size == 0);
 
       /* Check primary surface */
@@ -1278,6 +1305,7 @@ anv_image_init(struct anv_device *device, struct anv_image *image,
    const VkVideoProfileListInfoKHR *video_profile =
       vk_find_struct_const(pCreateInfo->pNext,
                            VIDEO_PROFILE_LIST_INFO_KHR);
+
    if (video_profile) {
       r = add_video_buffers(device, image, video_profile);
       if (r != VK_SUCCESS)
@@ -1342,6 +1370,21 @@ anv_image_init_from_create_info(struct anv_device *device,
       return anv_image_init_from_gralloc(device, image, pCreateInfo,
                                          gralloc_info);
 
+   const VkVideoProfileListInfoKHR *video_profile =
+      vk_find_struct_const(pCreateInfo->pNext, VIDEO_PROFILE_LIST_INFO_KHR);
+
+   /* For video surfaces, we need to strip VK_IMAGE_CREATE_BLOCK_TEXEL_VIEW_COMPATIBLE_BIT
+    * as it may cause unwanted linear tiling fallback for some video operations.
+    */
+   VkImageCreateInfo modified_create_info;
+   if (video_profile) {
+      modified_create_info = *pCreateInfo;
+      modified_create_info.flags &=
+         ~VK_IMAGE_CREATE_BLOCK_TEXEL_VIEW_COMPATIBLE_BIT;
+
+      pCreateInfo = &modified_create_info;
+   }
+
    struct anv_image_create_info create_info = {
       .vk_info = pCreateInfo,
    };
@@ -1358,6 +1401,10 @@ anv_image_init_from_create_info(struct anv_device *device,
        !isl_drm_modifier_has_aux(mod_explicit_info->drmFormatModifier))
       create_info.isl_extra_usage_flags |= ISL_SURF_USAGE_DISABLE_AUX_BIT;
 
+   if (video_profile && pCreateInfo->tiling == VK_IMAGE_TILING_OPTIMAL) {
+      create_info.isl_tiling_flags = ISL_TILING_Y0_BIT;
+   }
+
    return anv_image_init(device, image, &create_info);
 }
 
@@ -1730,6 +1777,20 @@ anv_bind_image_memory(struct anv_device *device,
       did_bind = true;
    }
 
+   if (image->vk.usage & (VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR |
+                          VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR)) {
+      const struct isl_surf *surf = &image->planes[0].primary_surface.isl;
+      struct anv_bo *bo = image->bindings[ANV_IMAGE_MEMORY_BINDING_MAIN].address.bo;
+
+      if (bo && surf->tiling != ISL_TILING_LINEAR) {
+         VkResult result = anv_device_set_bo_tiling(device, bo,
+                                                    surf->row_pitch_B, surf->tiling);
+         if (result != VK_SUCCESS) {
+            return result;
+         }
+      }
+   }
+
    if (bind_status)
       *bind_status->pResult = VK_SUCCESS;
 

src/intel/vulkan_hasvk/anv_private.h

@@ -3283,6 +3283,7 @@ struct anv_image {
    } planes[3];
 
    struct anv_image_memory_range vid_dmv_top_surface;
+   struct anv_image_memory_range vid_dmv_bottom_surface;
 };
 
 static inline bool

src/intel/vulkan_hasvk/genX_cmd_video.c

@@ -23,34 +23,98 @@
 
 #include "anv_private.h"
 
+#include <inttypes.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "util/vl_zscan_data.h"
+
 #include "genxml/gen_macros.h"
 #include "genxml/genX_video_pack.h"
 
+#include "anv_video_h264_slice.h"
+
+static uint32_t
+anv_h264_find_slice_nal(const uint8_t *buf, uint32_t off, uint32_t limit)
+{
+   for (int pass = 0; pass < 2; pass++) {
+      if (off + 2 >= limit)
+         break;
+
+      uint32_t sc;
+      if (buf[off] == 0 && buf[off + 1] == 0 && buf[off + 2] == 1)
+         sc = 3;
+      else if (off + 3 < limit &&
+               buf[off] == 0 && buf[off + 1] == 0 &&
+               buf[off + 2] == 0 && buf[off + 3] == 1)
+         sc = 4;
+      else
+         break;
+
+      if (off + sc >= limit)
+         break;
+
+      /* Skip AUD and continue to scan. */
+      if ((buf[off + sc] & 0x1f) == 9) {
+         off += sc + 1;
+         while (off + 2 < limit) {
+            if (buf[off] == 0 && buf[off + 1] == 0 &&
+                (buf[off + 2] == 1 ||
+                 (off + 3 < limit &&
+                  buf[off + 2] == 0 && buf[off + 3] == 1)))
+               break;
+            off++;
+         }
+         continue;
+      }
+
+      return off + sc;
+   }
+
+   return off + 3;
+}
+
 void
-genX(CmdBeginVideoCodingKHR)(VkCommandBuffer commandBuffer,
+genX(CmdBeginVideoCodingKHR) (VkCommandBuffer commandBuffer,
-                             const VkVideoBeginCodingInfoKHR *pBeginInfo)
+                              const VkVideoBeginCodingInfoKHR * pBeginInfo)
 {
    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
    ANV_FROM_HANDLE(anv_video_session, vid, pBeginInfo->videoSession);
-   ANV_FROM_HANDLE(anv_video_session_params, params, pBeginInfo->videoSessionParameters);
+   ANV_FROM_HANDLE(anv_video_session_params, params,
+                   pBeginInfo->videoSessionParameters);
 
    cmd_buffer->video.vid = vid;
    cmd_buffer->video.params = params;
 }
 
 void
-genX(CmdControlVideoCodingKHR)(VkCommandBuffer commandBuffer,
+genX(CmdControlVideoCodingKHR) (VkCommandBuffer commandBuffer,
-                               const VkVideoCodingControlInfoKHR *pCodingControlInfo)
+                                const VkVideoCodingControlInfoKHR *
+                                pCodingControlInfo)
 {
 
 }
 
-void
+void genX(CmdEndVideoCodingKHR) (VkCommandBuffer commandBuffer,
-genX(CmdEndVideoCodingKHR)(VkCommandBuffer commandBuffer,
+                                 const VkVideoEndCodingInfoKHR *
-                           const VkVideoEndCodingInfoKHR *pEndCodingInfo)
+                                 pEndCodingInfo)
 {
    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 
+#if GFX_VER == 8
+   anv_batch_emit(&cmd_buffer->batch, GENX(MI_FLUSH_DW), flush) {
+      flush.PostSyncOperation = 0;
+   }
+#elif GFX_VER <= 75
+   anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
+      pc.CommandStreamerStallEnable = 1;
+      pc.StallAtPixelScoreboard = 1;
+   }
+   anv_batch_emit(&cmd_buffer->batch, GENX(MFX_WAIT), wait) {
+      wait.MFXSyncControlFlag = 1;
+   }
+#endif
+
    cmd_buffer->video.vid = NULL;
    cmd_buffer->video.params = NULL;
 }
@@ -59,54 +123,134 @@ static void
 anv_h264_decode_video(struct anv_cmd_buffer *cmd_buffer,
                       const VkVideoDecodeInfoKHR *frame_info)
 {
+   /*
+    * Build a remapping table from Vulkan DPB slot index to hardware slot index
+    * (0-15). Some streams assign DPB slot indices >= 16 but the hardware only
+    * supports 16 frame-store entries, so we transparently redirect high slot
+    * indices onto unused low ones.
+    */
+   int8_t slot_to_hw[64];
+   memset(slot_to_hw, -1, sizeof(slot_to_hw));
+   {
+      bool used[16] = {};
+      for (uint32_t i = 0; i < frame_info->referenceSlotCount; i++) {
+         int s = frame_info->pReferenceSlots[i].slotIndex;
+         if (s >= 0 && s < 16) {
+            used[s] = true;
+            slot_to_hw[s] = (int8_t)s;
+         }
+      }
+      int next_free = 0;
+      for (uint32_t i = 0; i < frame_info->referenceSlotCount; i++) {
+         int s = frame_info->pReferenceSlots[i].slotIndex;
+         if (s >= 16 && s < 64) {
+            while (next_free < 16 && used[next_free])
+               next_free++;
+            if (next_free < 16) {
+               slot_to_hw[s] = (int8_t)next_free;
+               used[next_free++] = true;
+            } else {
+               mesa_logw_once("hasvk/video: Too many simultaneous H.264 DPB references - dropping.");
+            }
+         }
+      }
+   }
+
    ANV_FROM_HANDLE(anv_buffer, src_buffer, frame_info->srcBuffer);
    struct anv_video_session *vid = cmd_buffer->video.vid;
    struct anv_video_session_params *params = cmd_buffer->video.params;
    const struct VkVideoDecodeH264PictureInfoKHR *h264_pic_info =
       vk_find_struct_const(frame_info->pNext, VIDEO_DECODE_H264_PICTURE_INFO_KHR);
-   const StdVideoH264SequenceParameterSet *sps = vk_video_find_h264_dec_std_sps(&params->vk, h264_pic_info->pStdPictureInfo->seq_parameter_set_id);
+   const StdVideoH264SequenceParameterSet *sps;
-   const StdVideoH264PictureParameterSet *pps = vk_video_find_h264_dec_std_pps(&params->vk, h264_pic_info->pStdPictureInfo->pic_parameter_set_id);
+   const StdVideoH264PictureParameterSet *pps;
+   vk_video_get_h264_parameters(&vid->vk, params ? &params->vk : NULL, frame_info, h264_pic_info, &sps, &pps);
 
    anv_batch_emit(&cmd_buffer->batch, GENX(MI_FLUSH_DW), flush) {
-      flush.DWordLength = 2;
       flush.VideoPipelineCacheInvalidate = 1;
    };
 
+   const bool is_reference = frame_info->pSetupReferenceSlot != NULL;
+
    anv_batch_emit(&cmd_buffer->batch, GENX(MFX_PIPE_MODE_SELECT), sel) {
       sel.StandardSelect = SS_AVC;
       sel.CodecSelect = Decode;
-      sel.DecoderShortFormatMode = ShortFormatDriverInterface;
+      sel.DecoderShortFormatMode = LongFormatDriverInterface;
-      sel.DecoderModeSelect = VLDMode; // Hardcoded
+      sel.PreDeblockingOutputEnable = !is_reference;
-
+      sel.PostDeblockingOutputEnable = is_reference;
-      sel.PreDeblockingOutputEnable = 0;
+      sel.DecoderModeSelect = VLDMode;
-      sel.PostDeblockingOutputEnable = 1;
    }
 
    const struct anv_image_view *iv = anv_image_view_from_handle(frame_info->dstPictureResource.imageViewBinding);
    const struct anv_image *img = iv->image;
+
+   uint32_t frame_width_mbs  = sps->pic_width_in_mbs_minus1 + 1;
+   uint32_t frame_height_mbs = sps->pic_height_in_map_units_minus1 + 1;
+   if (!sps->flags.frame_mbs_only_flag)
+      frame_height_mbs *= 2;
+   if (frame_width_mbs > 256 || frame_height_mbs > 255)
+      mesa_logw_once("hasvk/video: frame %ux%u MBs exceeds hardware limits - clamping.",
+                     frame_width_mbs, frame_height_mbs);
+   frame_width_mbs  = MIN2(frame_width_mbs,  256u);
+   frame_height_mbs = MIN2(frame_height_mbs, 255u);
+
+   uint32_t y_cb_offset =
+      (img->planes[1].primary_surface.memory_range.offset -
+       img->planes[0].primary_surface.memory_range.offset) /
+       img->planes[0].primary_surface.isl.row_pitch_B;
+
+   if (INTEL_DEBUG(DEBUG_PERF)) {
+       fprintf(stderr, "hasvk/video: dst extent=%ux%u tiling=%u pitch=%u "
+               "plane0_off=%"PRIu64" plane1_off=%"PRIu64" phys_h=%u y_cb_offset=%u\n",
+               img->vk.extent.width, img->vk.extent.height,
+               img->planes[0].primary_surface.isl.tiling,
+               img->planes[0].primary_surface.isl.row_pitch_B,
+               img->planes[0].primary_surface.memory_range.offset,
+               img->planes[1].primary_surface.memory_range.offset,
+               img->planes[0].primary_surface.isl.phys_level0_sa.h,
+               y_cb_offset);
+   }
+
    anv_batch_emit(&cmd_buffer->batch, GENX(MFX_SURFACE_STATE), ss) {
       ss.Width = img->vk.extent.width - 1;
-      ss.Height = img->vk.extent.height - 1;
+      ss.Height = frame_height_mbs * 16 - 1;
-      ss.SurfaceFormat = PLANAR_420_8; // assert on this?
+      ss.SurfaceFormat = PLANAR_420_8;
       ss.InterleaveChroma = 1;
       ss.SurfacePitch = img->planes[0].primary_surface.isl.row_pitch_B - 1;
       ss.TiledSurface = img->planes[0].primary_surface.isl.tiling != ISL_TILING_LINEAR;
       ss.TileWalk = TW_YMAJOR;
-
+      ss.YOffsetforUCb = MIN2(y_cb_offset, 32767u);
-      ss.YOffsetforUCb = align(img->vk.extent.height, 32);
+      ss.YOffsetforVCr = 0;
-      ss.YOffsetforVCr = align(img->vk.extent.height, 32);
    }
 
    anv_batch_emit(&cmd_buffer->batch, GENX(MFX_PIPE_BUF_ADDR_STATE), buf) {
-      bool use_pre_deblock = false;
+      memset(buf.ReferencePictureAddress, 0, sizeof(buf.ReferencePictureAddress));
-      if (use_pre_deblock) {
+#if GFX_VERx10 == 70
-         buf.PreDeblockingDestinationAddress = anv_image_address(img,
+      memset(buf.ReferencePictureArbitrationPriorityControl, 0, sizeof(buf.ReferencePictureArbitrationPriorityControl));
+      memset(buf.ReferencePictureGraphicsDataType, 0, sizeof(buf.ReferencePictureGraphicsDataType));
+      memset(buf.ReferencePictureCacheabilityControl, 0, sizeof(buf.ReferencePictureCacheabilityControl));
+#elif GFX_VERx10 == 75
+      memset(buf.ReferencePictureMOCS, 0, sizeof(buf.ReferencePictureMOCS));
+#endif
+      memset(&buf, 0, sizeof(buf));
+      buf.DWordLength = GENX(MFX_PIPE_BUF_ADDR_STATE_length) - 2;
+      buf.SubOpcodeB = 2;
+      buf.Pipeline = 2;
+      buf.CommandType = 3;
+
+      struct anv_address dst_addr = anv_image_address(img,
                                                       &img->planes[0].primary_surface.memory_range);
+#if GFX_VERx10 == 70
+      if (is_reference) {
+         buf.PostDeblockingDestinationAddress = dst_addr;
+         buf.PostDeblockingDestinationCacheabilityControl = 2;
       } else {
-         buf.PostDeblockingDestinationAddress = anv_image_address(img,
+         buf.PreDeblockingDestinationAddress = dst_addr;
-                                                                  &img->planes[0].primary_surface.memory_range);
       }
-#if GFX_VERx10 >= 75
+#else
+      if (is_reference)
+         buf.PostDeblockingDestinationAddress = dst_addr;
+      else
+         buf.PreDeblockingDestinationAddress = dst_addr;
       buf.PreDeblockingDestinationMOCS = anv_mocs(cmd_buffer->device, buf.PreDeblockingDestinationAddress.bo, 0);
       buf.PostDeblockingDestinationMOCS = anv_mocs(cmd_buffer->device, buf.PostDeblockingDestinationAddress.bo, 0);
       buf.OriginalUncompressedPictureSourceMOCS = anv_mocs(cmd_buffer->device, NULL, 0);
@@ -117,33 +261,49 @@ anv_h264_decode_video(struct anv_cmd_buffer *cmd_buffer,
       buf.IntraRowStoreScratchBufferAddressHigh = (struct anv_address) { vid->vid_mem[ANV_VID_MEM_H264_INTRA_ROW_STORE].mem->bo,
          vid->vid_mem[ANV_VID_MEM_H264_INTRA_ROW_STORE].offset };
       buf.IntraRowStoreScratchBufferMOCS = anv_mocs(cmd_buffer->device, buf.IntraRowStoreScratchBufferAddressHigh.bo, 0);
-      buf.DeblockingFilterRowStoreScratchAddressHigh = (struct anv_address) { vid->vid_mem[ANV_VID_MEM_H264_DEBLOCK_FILTER_ROW_STORE].mem->bo, vid->vid_mem[ANV_VID_MEM_H264_DEBLOCK_FILTER_ROW_STORE].offset };
+      buf.DeblockingFilterRowStoreScratchAddressHigh = (struct anv_address) { vid->vid_mem[ANV_VID_MEM_H264_DEBLOCK_FILTER_ROW_STORE].mem->bo,
+         vid->vid_mem[ANV_VID_MEM_H264_DEBLOCK_FILTER_ROW_STORE].offset };
+      buf.DeblockingFilterRowStoreScratchMOCS = anv_mocs(cmd_buffer->device, buf.DeblockingFilterRowStoreScratchAddressHigh.bo, 0);
+      buf.MBStatusBufferMOCS = anv_mocs(cmd_buffer->device, NULL, 0);
+      buf.MBILDBStreamOutBufferMOCS = anv_mocs(cmd_buffer->device, NULL, 0);
 #else
       buf.IntraRowStoreScratchBufferAddress = (struct anv_address) { vid->vid_mem[ANV_VID_MEM_H264_INTRA_ROW_STORE].mem->bo,
                                                                      vid->vid_mem[ANV_VID_MEM_H264_INTRA_ROW_STORE].offset };
-#if GFX_VERx10 >= 75
+#if GFX_VERx10 == 70
+      buf.IntraRowStoreScratchBufferCacheabilityControl = 2;
+#elif GFX_VERx10 >= 75
       buf.IntraRowStoreScratchBufferMOCS = anv_mocs(cmd_buffer->device, vid->vid_mem[ANV_VID_MEM_H264_INTRA_ROW_STORE].mem->bo, 0);
 #endif
 #if GFX_VERx10 == 70
-      buf.DeblockingFilterRowStoreScratchBufferAddress = (struct anv_address) { vid->vid_mem[ANV_VID_MEM_H264_DEBLOCK_FILTER_ROW_STORE].mem->bo, vid->vid_mem[ANV_VID_MEM_H264_DEBLOCK_FILTER_ROW_STORE].offset };
+      buf.DeblockingFilterRowStoreScratchBufferAddress =
+         (struct anv_address) { vid->vid_mem[ANV_VID_MEM_H264_DEBLOCK_FILTER_ROW_STORE].mem->bo, vid->vid_mem[ANV_VID_MEM_H264_DEBLOCK_FILTER_ROW_STORE].offset };
+      buf.DeblockingFilterRowStoreScratchBufferCacheabilityControl = 2;
 #else
-      buf.DeblockingFilterRowStoreScratchAddress = (struct anv_address) { vid->vid_mem[ANV_VID_MEM_H264_DEBLOCK_FILTER_ROW_STORE].mem->bo, vid->vid_mem[ANV_VID_MEM_H264_DEBLOCK_FILTER_ROW_STORE].offset };
+      buf.DeblockingFilterRowStoreScratchAddress =
-#endif
+         (struct anv_address) { vid->vid_mem[ANV_VID_MEM_H264_DEBLOCK_FILTER_ROW_STORE].mem->bo, vid->vid_mem[ANV_VID_MEM_H264_DEBLOCK_FILTER_ROW_STORE].offset };
-#endif
-#if GFX_VERx10 == 75
       buf.DeblockingFilterRowStoreScratchMOCS = anv_mocs(cmd_buffer->device, vid->vid_mem[ANV_VID_MEM_H264_DEBLOCK_FILTER_ROW_STORE].mem->bo, 0);
       buf.MBStatusBufferMOCS = anv_mocs(cmd_buffer->device, NULL, 0);
       buf.MBILDBStreamOutBufferMOCS = anv_mocs(cmd_buffer->device, NULL, 0);
 #endif
+#endif
 
 #if GFX_VERx10 == 80
       struct anv_bo *ref_bo = NULL;
 #endif
       for (unsigned i = 0; i < frame_info->referenceSlotCount; i++) {
+         int s = frame_info->pReferenceSlots[i].slotIndex;
+         int hw = (s >= 0 && s < 64) ? (int)slot_to_hw[s] : -1;
+         if (hw < 0)
+            continue;
          const struct anv_image_view *ref_iv = anv_image_view_from_handle(frame_info->pReferenceSlots[i].pPictureResource->imageViewBinding);
-         int idx = frame_info->pReferenceSlots[i].slotIndex;
+         struct anv_address ref_addr = anv_image_address(ref_iv->image,
-         buf.ReferencePictureAddress[idx] = anv_image_address(ref_iv->image,
                                                          &ref_iv->image->planes[0].primary_surface.memory_range);
+         buf.ReferencePictureAddress[hw] = ref_addr;
+#if GFX_VERx10 == 70
+         buf.ReferencePictureCacheabilityControl[hw] = 2;
+#elif GFX_VERx10 == 75
+         buf.ReferencePictureMOCS[hw] = anv_mocs(cmd_buffer->device, ref_addr.bo, 0);
+#endif
 #if GFX_VERx10 == 80
          if (i == 0)
             ref_bo = ref_iv->image->bindings[0].address.bo;
@@ -157,7 +317,9 @@ anv_h264_decode_video(struct anv_cmd_buffer *cmd_buffer,
    anv_batch_emit(&cmd_buffer->batch, GENX(MFX_IND_OBJ_BASE_ADDR_STATE), index_obj) {
       index_obj.MFXIndirectBitstreamObjectAddress = anv_address_add(src_buffer->address,
                                                                     frame_info->srcBufferOffset & ~4095);
-#if GFX_VERx10 == 75
+#if GFX_VERx10 == 70
+      index_obj.MFXIndirectBitstreamObjectCacheabilityControl = 2;
+#elif GFX_VERx10 == 75
       index_obj.MFXIndirectBitstreamObjectMOCS = anv_mocs(cmd_buffer->device, src_buffer->address.bo,
                                                           0);
       index_obj.MFXIndirectMVObjectMOCS = anv_mocs(cmd_buffer->device, NULL, 0);
@@ -173,49 +335,33 @@ anv_h264_decode_video(struct anv_cmd_buffer *cmd_buffer,
    anv_batch_emit(&cmd_buffer->batch, GENX(MFX_BSP_BUF_BASE_ADDR_STATE), bsp) {
       bsp.BSDMPCRowStoreScratchBufferAddress = (struct anv_address) { vid->vid_mem[ANV_VID_MEM_H264_BSD_MPC_ROW_SCRATCH].mem->bo,
          vid->vid_mem[ANV_VID_MEM_H264_BSD_MPC_ROW_SCRATCH].offset };
-#if GFX_VERx10 == 75
+#if GFX_VERx10 == 70
+      bsp.BSDMPCRowStoreScratchBufferCacheabilityControl = 2;
+#elif GFX_VERx10 == 75
       bsp.BSDMPCRowStoreScratchBufferMOCS = anv_mocs(cmd_buffer->device, vid->vid_mem[ANV_VID_MEM_H264_BSD_MPC_ROW_SCRATCH].mem->bo, 0);
 #endif
 
       bsp.MPRRowStoreScratchBufferAddress = (struct anv_address) { vid->vid_mem[ANV_VID_MEM_H264_MPR_ROW_SCRATCH].mem->bo,
-         vid->vid_mem[ANV_VID_MEM_H264_BSD_MPC_ROW_SCRATCH].offset };
+         vid->vid_mem[ANV_VID_MEM_H264_MPR_ROW_SCRATCH].offset };
 
-#if GFX_VERx10 == 75
+#if GFX_VERx10 == 70
+      bsp.MPRRowStoreScratchBufferCacheabilityControl = 2;
+#elif GFX_VERx10 == 75
       bsp.MPRRowStoreScratchBufferMOCS = anv_mocs(cmd_buffer->device,  vid->vid_mem[ANV_VID_MEM_H264_MPR_ROW_SCRATCH].mem->bo, 0);
       bsp.BitplaneReadBufferMOCS = anv_mocs(cmd_buffer->device, NULL, 0);
 #endif
    }
 
-   anv_batch_emit(&cmd_buffer->batch, GENX(MFD_AVC_DPB_STATE), avc_dpb) {
-      for (unsigned i = 0; i < frame_info->referenceSlotCount; i++) {
-         const struct VkVideoDecodeH264DpbSlotInfoKHR *dpb_slot =
-            vk_find_struct_const(frame_info->pReferenceSlots[i].pNext, VIDEO_DECODE_H264_DPB_SLOT_INFO_KHR);
-         const StdVideoDecodeH264ReferenceInfo *ref_info = dpb_slot->pStdReferenceInfo;
-         int idx = frame_info->pReferenceSlots[i].slotIndex;
-         avc_dpb.NonExistingFrame[idx] = ref_info->flags.is_non_existing;
-         avc_dpb.LongTermFrame[idx] = ref_info->flags.used_for_long_term_reference;
-         if (!ref_info->flags.top_field_flag && !ref_info->flags.bottom_field_flag)
-            avc_dpb.UsedforReference[idx] = 3;
-         else
-            avc_dpb.UsedforReference[idx] = ref_info->flags.top_field_flag | (ref_info->flags.bottom_field_flag << 1);
-         avc_dpb.LTSTFrameNumberList[idx] = ref_info->FrameNum;
-      }
-   }
-
 #if GFX_VERx10 >= 75
    anv_batch_emit(&cmd_buffer->batch, GENX(MFD_AVC_PICID_STATE), picid) {
       picid.PictureIDRemappingDisable = true;
    }
 #endif
 
-   uint32_t pic_height = sps->pic_height_in_map_units_minus1 + 1;
-   if (!sps->flags.frame_mbs_only_flag)
-      pic_height *= 2;
-
    anv_batch_emit(&cmd_buffer->batch, GENX(MFX_AVC_IMG_STATE), avc_img) {
-      avc_img.FrameWidth = sps->pic_width_in_mbs_minus1;
+      avc_img.FrameWidth = MIN2(frame_width_mbs - 1, 255u);
-      avc_img.FrameHeight = pic_height - 1;
+      avc_img.FrameHeight = MIN2(frame_height_mbs - 1, 254u);
-      avc_img.FrameSize = (sps->pic_width_in_mbs_minus1 + 1) * pic_height;
+      avc_img.FrameSize = frame_width_mbs * frame_height_mbs;
 
       if (!h264_pic_info->pStdPictureInfo->flags.field_pic_flag)
          avc_img.ImageStructure = FramePicture;
@@ -253,139 +399,108 @@ anv_h264_decode_video(struct anv_cmd_buffer *cmd_buffer,
       avc_img.CurrentPictureFrameNumber = h264_pic_info->pStdPictureInfo->frame_num;
    }
 
-   if (pps->flags.pic_scaling_matrix_present_flag) {
+   StdVideoH264ScalingLists scaling_lists;
+   vk_video_derive_h264_scaling_list(sps, pps, &scaling_lists);
    anv_batch_emit(&cmd_buffer->batch, GENX(MFX_QM_STATE), qm) {
       qm.DWordLength = 16;
       qm.AVC = AVC_4x4_Intra_MATRIX;
       for (unsigned m = 0; m < 3; m++)
          for (unsigned q = 0; q < 16; q++)
-               qm.ForwardQuantizerMatrix[m * 16 + q] = pps->pScalingLists->ScalingList4x4[m][q];
+            qm.ForwardQuantizerMatrix[m * 16 + vl_zscan_normal_16[q]] =
+               scaling_lists.ScalingList4x4[m][q];
    }
    anv_batch_emit(&cmd_buffer->batch, GENX(MFX_QM_STATE), qm) {
       qm.DWordLength = 16;
       qm.AVC = AVC_4x4_Inter_MATRIX;
       for (unsigned m = 0; m < 3; m++)
          for (unsigned q = 0; q < 16; q++)
-               qm.ForwardQuantizerMatrix[m * 16 + q] = pps->pScalingLists->ScalingList4x4[m + 3][q];
+            qm.ForwardQuantizerMatrix[m * 16 + vl_zscan_normal_16[q]] =
+               scaling_lists.ScalingList4x4[m + 3][q];
    }
    if (pps->flags.transform_8x8_mode_flag) {
       anv_batch_emit(&cmd_buffer->batch, GENX(MFX_QM_STATE), qm) {
          qm.DWordLength = 16;
          qm.AVC = AVC_8x8_Intra_MATRIX;
          for (unsigned q = 0; q < 64; q++)
-               qm.ForwardQuantizerMatrix[q] = pps->pScalingLists->ScalingList8x8[0][q];
+            qm.ForwardQuantizerMatrix[vl_zscan_normal[q]] =
+               scaling_lists.ScalingList8x8[0][q];
       }
       anv_batch_emit(&cmd_buffer->batch, GENX(MFX_QM_STATE), qm) {
          qm.DWordLength = 16;
          qm.AVC = AVC_8x8_Inter_MATRIX;
          for (unsigned q = 0; q < 64; q++)
-               qm.ForwardQuantizerMatrix[q] = pps->pScalingLists->ScalingList8x8[3][q];
+            qm.ForwardQuantizerMatrix[vl_zscan_normal[q]] =
-         }
+               scaling_lists.ScalingList8x8[1][q];
-      }
-   } else if (sps->flags.seq_scaling_matrix_present_flag) {
-      anv_batch_emit(&cmd_buffer->batch, GENX(MFX_QM_STATE), qm) {
-         qm.DWordLength = 16;
-         qm.AVC = AVC_4x4_Intra_MATRIX;
-         for (unsigned m = 0; m < 3; m++)
-            for (unsigned q = 0; q < 16; q++)
-               qm.ForwardQuantizerMatrix[m * 16 + q] = sps->pScalingLists->ScalingList4x4[m][q];
-      }
-      anv_batch_emit(&cmd_buffer->batch, GENX(MFX_QM_STATE), qm) {
-         qm.DWordLength = 16;
-         qm.AVC = AVC_4x4_Inter_MATRIX;
-         for (unsigned m = 0; m < 3; m++)
-            for (unsigned q = 0; q < 16; q++)
-               qm.ForwardQuantizerMatrix[m * 16 + q] = sps->pScalingLists->ScalingList4x4[m + 3][q];
-      }
-      if (pps->flags.transform_8x8_mode_flag) {
-         anv_batch_emit(&cmd_buffer->batch, GENX(MFX_QM_STATE), qm) {
-            qm.DWordLength = 16;
-            qm.AVC = AVC_8x8_Intra_MATRIX;
-            for (unsigned q = 0; q < 64; q++)
-               qm.ForwardQuantizerMatrix[q] = sps->pScalingLists->ScalingList8x8[0][q];
-         }
-         anv_batch_emit(&cmd_buffer->batch, GENX(MFX_QM_STATE), qm) {
-            qm.DWordLength = 16;
-            qm.AVC = AVC_8x8_Inter_MATRIX;
-            for (unsigned q = 0; q < 64; q++)
-               qm.ForwardQuantizerMatrix[q] = sps->pScalingLists->ScalingList8x8[3][q];
-         }
-      }
-   } else {
-      anv_batch_emit(&cmd_buffer->batch, GENX(MFX_QM_STATE), qm) {
-         qm.DWordLength = 16;
-         qm.AVC = AVC_4x4_Intra_MATRIX;
-         for (unsigned q = 0; q < 3 * 16; q++)
-            qm.ForwardQuantizerMatrix[q] = 0x10;
-      }
-      anv_batch_emit(&cmd_buffer->batch, GENX(MFX_QM_STATE), qm) {
-         qm.DWordLength = 16;
-         qm.AVC = AVC_4x4_Inter_MATRIX;
-         for (unsigned q = 0; q < 3 * 16; q++)
-            qm.ForwardQuantizerMatrix[q] = 0x10;
-      }
-      if (pps->flags.transform_8x8_mode_flag) {
-         anv_batch_emit(&cmd_buffer->batch, GENX(MFX_QM_STATE), qm) {
-            qm.DWordLength = 16;
-            qm.AVC = AVC_8x8_Intra_MATRIX;
-            for (unsigned q = 0; q < 64; q++)
-               qm.ForwardQuantizerMatrix[q] = 0x10;
-         }
-         anv_batch_emit(&cmd_buffer->batch, GENX(MFX_QM_STATE), qm) {
-            qm.DWordLength = 16;
-            qm.AVC = AVC_8x8_Inter_MATRIX;
-            for (unsigned q = 0; q < 64; q++)
-               qm.ForwardQuantizerMatrix[q] = 0x10;
-         }
       }
    }
 
-   anv_batch_emit(&cmd_buffer->batch, GENX(MFX_AVC_DIRECTMODE_STATE), avc_directmode) {
+#pragma GCC diagnostic push
-      /* bind reference frame DMV */
+#pragma GCC diagnostic ignored "-Wunused-but-set-variable"
    struct anv_bo *dmv_bo = NULL;
-      for (unsigned i = 0; i < frame_info->referenceSlotCount; i++) {
+#pragma GCC diagnostic pop
-         int idx = frame_info->pReferenceSlots[i].slotIndex;
-         const struct VkVideoDecodeH264DpbSlotInfoKHR *dpb_slot =
-            vk_find_struct_const(frame_info->pReferenceSlots[i].pNext, VIDEO_DECODE_H264_DPB_SLOT_INFO_KHR);
-         const struct anv_image_view *ref_iv = anv_image_view_from_handle(frame_info->pReferenceSlots[i].pPictureResource->imageViewBinding);
-         const StdVideoDecodeH264ReferenceInfo *ref_info = dpb_slot->pStdReferenceInfo;
-         if (i == 0) {
-            dmv_bo = anv_image_address(ref_iv->image,
-                                       &ref_iv->image->vid_dmv_top_surface).bo;
-         }
-#if GFX_VERx10 == 70
-         avc_directmode.DirectMVBufferAddress[idx * 2] = anv_image_address(ref_iv->image,
-                                                                           &ref_iv->image->vid_dmv_top_surface);
-         avc_directmode.DirectMVBufferAddress[idx * 2 + 1] = anv_image_address(ref_iv->image,
-                                                                           &ref_iv->image->vid_dmv_top_surface);
-#endif
 
-#if GFX_VERx10 == 75
+   const uint8_t *bs_map = src_buffer->address.bo->map;
-         if (idx == 0)
+   bool bs_mapped_here = false;
-            avc_directmode.DirectMVBuffer0Address = anv_image_address(ref_iv->image,
+   if (!bs_map) {
-                                                                          &ref_iv->image->vid_dmv_top_surface);
+      void *tmp = NULL;
-         else
+      if (anv_device_map_bo(cmd_buffer->device,
-            avc_directmode.DirectMVBufferAddress1[idx - 1] = anv_image_address(ref_iv->image,
+                            src_buffer->address.bo,
-                                                                          &ref_iv->image->vid_dmv_top_surface);
+                            0, src_buffer->address.bo->size,
-#endif
+                            0, &tmp) == VK_SUCCESS) {
-         avc_directmode.POCList[2 * idx] = ref_info->PicOrderCnt[0];
+         bs_map = (const uint8_t *)tmp;
-         avc_directmode.POCList[2 * idx + 1] = ref_info->PicOrderCnt[1];
+         bs_mapped_here = true;
+      }
+   }
+
+   uint32_t mb_width = sps->pic_width_in_mbs_minus1 + 1;
+   uint32_t pic_height_in_map_units = sps->pic_height_in_map_units_minus1 + 1;
+   uint32_t mb_height = sps->flags.frame_mbs_only_flag ?
+                        pic_height_in_map_units : pic_height_in_map_units * 2;
+
+   struct anv_h264_slice_params *sp =
+      calloc(h264_pic_info->sliceCount, sizeof(*sp));
+   if (sp) {
+      for (unsigned s = 0; s < h264_pic_info->sliceCount; s++) {
+         uint32_t slice_off = h264_pic_info->pSliceOffsets[s];
+         uint32_t buf_end = (s + 1 < h264_pic_info->sliceCount) ?
+                             h264_pic_info->pSliceOffsets[s + 1] :
+                             frame_info->srcBufferRange;
+
+         uint32_t bo_base = frame_info->srcBufferOffset & ~4095u;
+         uint32_t page_off = frame_info->srcBufferOffset & 4095u;
+
+         uint32_t bo_nalu = bs_map ?
+            anv_h264_find_slice_nal(bs_map + bo_base,
+                                    page_off + slice_off,
+                                    page_off + buf_end) :
+            page_off + slice_off + 3;
+         uint32_t nalu_size = (page_off + buf_end > bo_nalu) ?
+                              (page_off + buf_end - bo_nalu) : 0;
+
+         bool ok = false;
+         if (bs_map && nalu_size > 0 &&
+             bo_base + bo_nalu + nalu_size <= src_buffer->address.bo->size) {
+            ok = anv_h264_parse_slice_header(
+                    bs_map + bo_base, bo_nalu, nalu_size,
+                    sps, pps, h264_pic_info,
+                    frame_info->referenceSlotCount,
+                    frame_info->pReferenceSlots,
+                    &sp[s]);
+         }
+         if (!ok) {
+            sp[s].slice_type = ANV_H264_SLICE_I;
+            sp[s].first_mb_in_slice = 0;
+            sp[s].num_ref_idx_l0_active_minus1 =
+               pps->num_ref_idx_l0_default_active_minus1;
+            sp[s].num_ref_idx_l1_active_minus1 =
+               pps->num_ref_idx_l1_default_active_minus1;
+            memset(sp[s].ref_list0, ANV_H264_INVALID_SLOT,
+                   sizeof(sp[s].ref_list0));
+            memset(sp[s].ref_list1, ANV_H264_INVALID_SLOT,
+                   sizeof(sp[s].ref_list1));
+         }
+         sp[s].parse_ok = ok;
       }
-#if GFX_VERx10 == 70
-      avc_directmode.DirectMVBufferWriteAddress[0] = anv_image_address(img,
-                                                                       &img->vid_dmv_top_surface);
-      avc_directmode.DirectMVBufferWriteAddress[1] = anv_image_address(img,
-                                                                       &img->vid_dmv_top_surface);
-#else
-      avc_directmode.DirectMVBufferWriteAddress = anv_image_address(img,
-                                                                    &img->vid_dmv_top_surface);
-#if GFX_VERx10 == 75
-      avc_directmode.DirectMVBufferMOCS = anv_mocs(cmd_buffer->device, dmv_bo, 0);
-      avc_directmode.DirectMVBufferWriteMOCS = anv_mocs(cmd_buffer->device, avc_directmode.DirectMVBufferWriteAddress.bo, 0);
-#endif
-#endif
-      avc_directmode.POCList[32] = h264_pic_info->pStdPictureInfo->PicOrderCnt[0];
-      avc_directmode.POCList[33] = h264_pic_info->pStdPictureInfo->PicOrderCnt[1];
    }
 
    uint32_t buffer_offset = frame_info->srcBufferOffset & 4095;
@@ -395,51 +510,275 @@ anv_h264_decode_video(struct anv_cmd_buffer *cmd_buffer,
       uint32_t current_offset = h264_pic_info->pSliceOffsets[s];
       uint32_t this_end;
       if (!last_slice) {
-         uint32_t next_offset = h264_pic_info->pSliceOffsets[s + 1];
+         this_end = h264_pic_info->pSliceOffsets[s + 1];
-         uint32_t next_end = h264_pic_info->pSliceOffsets[s + 2];
+      } else {
-         if (s == h264_pic_info->sliceCount - 2)
-            next_end = frame_info->srcBufferRange;
-         anv_batch_emit(&cmd_buffer->batch, GENX(MFD_AVC_SLICEADDR), sliceaddr) {
-            sliceaddr.IndirectBSDDataLength = next_end - next_offset - HEADER_OFFSET;
-            /* start decoding after the 3-byte header. */
-            sliceaddr.IndirectBSDDataStartAddress = buffer_offset + next_offset + HEADER_OFFSET;
-         };
-         this_end = next_offset;
-      } else
          this_end = frame_info->srcBufferRange;
+      }
+
+      anv_batch_emit(&cmd_buffer->batch, GENX(MFX_AVC_DIRECTMODE_STATE), avc_directmode) {
+         for (unsigned i = 0; i < frame_info->referenceSlotCount; i++) {
+            int s = frame_info->pReferenceSlots[i].slotIndex;
+            int hw = (s >= 0 && s < 64) ? (int)slot_to_hw[s] : -1;
+            if (hw < 0)
+               continue;
+            const struct VkVideoDecodeH264DpbSlotInfoKHR *dpb_slot =
+               vk_find_struct_const(frame_info->pReferenceSlots[i].pNext, VIDEO_DECODE_H264_DPB_SLOT_INFO_KHR);
+            const struct anv_image_view *ref_iv = anv_image_view_from_handle(frame_info->pReferenceSlots[i].pPictureResource->imageViewBinding);
+            const StdVideoDecodeH264ReferenceInfo *ref_info = dpb_slot->pStdReferenceInfo;
+            if (i == 0)
+               dmv_bo = anv_image_address(ref_iv->image, &ref_iv->image->vid_dmv_top_surface).bo;
+#if GFX_VERx10 == 70
+            avc_directmode.DirectMVBufferAddress[hw * 2] = anv_image_address(ref_iv->image, &ref_iv->image->vid_dmv_top_surface);
+            avc_directmode.DirectMVBufferAddress[hw * 2 + 1] = anv_image_address(ref_iv->image, &ref_iv->image->vid_dmv_bottom_surface);
+            avc_directmode.DirectMVBufferCacheabilityControl[hw * 2] = 2;
+            avc_directmode.DirectMVBufferCacheabilityControl[hw * 2 + 1] = 2;
+#elif GFX_VERx10 == 75
+            {
+               struct anv_address dmv_addr = anv_image_address(ref_iv->image, &ref_iv->image->vid_dmv_top_surface);
+               if (hw == 0) {
+                  avc_directmode.DirectMVBuffer0Address = dmv_addr;
+                  avc_directmode.DirectMVBufferMOCS = anv_mocs(cmd_buffer->device, dmv_addr.bo, 0);
+               } else {
+                  avc_directmode.DirectMVBufferAddress1[hw - 1] = dmv_addr;
+                  avc_directmode.DirectMVBufferMOCS1[hw - 1] = anv_mocs(cmd_buffer->device, dmv_addr.bo, 0);
+               }
+            }
+#elif GFX_VERx10 == 80
+            avc_directmode.DirectMVBufferAddress[hw] = anv_image_address(ref_iv->image,
+                                                                         &ref_iv->image->vid_dmv_top_surface);
+#endif
+            avc_directmode.POCList[2 * hw] = ref_info->PicOrderCnt[0];
+            avc_directmode.POCList[2 * hw + 1] = ref_info->PicOrderCnt[1];
+         }
+#if GFX_VERx10 == 70
+         avc_directmode.DirectMVBufferWriteAddress[0] = anv_image_address(img, &img->vid_dmv_top_surface);
+         avc_directmode.DirectMVBufferWriteAddress[1] = anv_image_address(img, &img->vid_dmv_bottom_surface);
+         avc_directmode.DirectMVBufferWriteCacheabilityControl[0] = 2;
+         avc_directmode.DirectMVBufferWriteCacheabilityControl[1] = 2;
+#elif GFX_VERx10 == 75
+         avc_directmode.DirectMVBufferWriteAddress = anv_image_address(img, &img->vid_dmv_top_surface);
+         avc_directmode.DirectMVBufferWriteMOCS = anv_mocs(cmd_buffer->device, avc_directmode.DirectMVBufferWriteAddress.bo, 0);
+         if (!avc_directmode.DirectMVBufferMOCS)
+            avc_directmode.DirectMVBufferMOCS = anv_mocs(cmd_buffer->device, avc_directmode.DirectMVBufferWriteAddress.bo, 0);
+#elif GFX_VERx10 == 80
+         avc_directmode.DirectMVBufferAttributes.TargetCache = 2; /* LLC/eLLC */
+         avc_directmode.DirectMVBufferWriteAddress = anv_image_address(img, &img->vid_dmv_top_surface);
+         avc_directmode.DirectMVBufferWriteAttributes.TargetCache = 2;
+#endif
+         avc_directmode.POCList[32] = h264_pic_info->pStdPictureInfo->PicOrderCnt[0];
+         avc_directmode.POCList[33] = h264_pic_info->pStdPictureInfo->PicOrderCnt[1];
+      }
+
+      const struct anv_h264_slice_params *cur = sp ? &sp[s] : NULL;
+
+      for (unsigned list = 0; list <= 1; list++) {
+         anv_batch_emit(&cmd_buffer->batch, GENX(MFX_AVC_REF_IDX_STATE), ref_idx) {
+            ref_idx.ReferencePictureListSelect = list;
+            const uint8_t *rl = (list == 0) ?
+               (cur ? cur->ref_list0 : NULL) :
+               (cur ? cur->ref_list1 : NULL);
+            for (unsigned e = 0; e < 32; e++) {
+               uint8_t slot = (rl && e < ANV_H264_MAX_REF_FRAMES) ?
+                              rl[e] : ANV_H264_INVALID_SLOT;
+               if (slot == ANV_H264_INVALID_SLOT) {
+                  ref_idx.ReferenceListEntry[e] = 0xFF;
+               } else {
+                  int8_t hw = (slot < 64) ? slot_to_hw[slot] : -1;
+                  if (hw < 0) {
+                     ref_idx.ReferenceListEntry[e] = 0xFF;
+                  } else {
+                     bool is_long = false;
+                     for (uint32_t i = 0; i < frame_info->referenceSlotCount; i++) {
+                        if (frame_info->pReferenceSlots[i].slotIndex == slot) {
+                           const struct VkVideoDecodeH264DpbSlotInfoKHR *d =
+                              vk_find_struct_const(frame_info->pReferenceSlots[i].pNext,
+                                                   VIDEO_DECODE_H264_DPB_SLOT_INFO_KHR);
+                           if (d && d->pStdReferenceInfo)
+                              is_long = d->pStdReferenceInfo->flags.used_for_long_term_reference;
+                           break;
+                        }
+                     }
+                     ref_idx.ReferenceListEntry[e] =
+                        (is_long ? 0x60u : 0x20u) | ((uint8_t)hw << 1);
+                  }
+               }
+            }
+         }
+      }
+
+      bool need_w_l0 = (pps->flags.weighted_pred_flag &&
+                        cur && (cur->slice_type == ANV_H264_SLICE_P ||
+                                cur->slice_type == ANV_H264_SLICE_SP));
+      bool need_w_l1 = (pps->weighted_bipred_idc == 1 &&
+                        cur && cur->slice_type == ANV_H264_SLICE_B);
+      if (need_w_l0 || need_w_l1) {
+         for (unsigned list = 0; list <= 1; list++) {
+            if (list == 0 && !need_w_l0) continue;
+            if (list == 1 && !need_w_l1) continue;
+            anv_batch_emit(&cmd_buffer->batch, GENX(MFX_AVC_WEIGHTOFFSET_STATE), wo) {
+               wo.WeightandOffsetSelect = list;
+               int active = (list == 0) ?
+                  cur->num_ref_idx_l0_active_minus1 + 1 :
+                  cur->num_ref_idx_l1_active_minus1 + 1;
+               if (active > ANV_H264_MAX_REF_FRAMES)
+                  active = ANV_H264_MAX_REF_FRAMES;
+               for (int i = 0; i < 32; i++) {
+                  int16_t lw = (i < active) ? ((list == 0) ? cur->luma_weight_l0[i] : cur->luma_weight_l1[i]) : 0;
+                  int16_t lo = (i < active) ? ((list == 0) ? cur->luma_offset_l0[i] : cur->luma_offset_l1[i]) : 0;
+                  int16_t cw0 = (i < active) ? ((list == 0) ? cur->chroma_weight_l0[i][0] : cur->chroma_weight_l1[i][0]) : 0;
+                  int16_t co0 = (i < active) ? ((list == 0) ? cur->chroma_offset_l0[i][0] : cur->chroma_offset_l1[i][0]) : 0;
+                  int16_t cw1 = (i < active) ? ((list == 0) ? cur->chroma_weight_l0[i][1] : cur->chroma_weight_l1[i][1]) : 0;
+                  int16_t co1 = (i < active) ? ((list == 0) ? cur->chroma_offset_l0[i][1] : cur->chroma_offset_l1[i][1]) : 0;
+                  wo.WeightOffset[i * 3 + 0] = ((uint32_t)(uint16_t)lo  << 16) | (uint16_t)lw;
+                  wo.WeightOffset[i * 3 + 1] = ((uint32_t)(uint16_t)co0 << 16) | (uint16_t)cw0;
+                  wo.WeightOffset[i * 3 + 2] = ((uint32_t)(uint16_t)co1 << 16) | (uint16_t)cw1;
+               }
+            }
+         }
+      }
+
+      /* The "phantom slice" for error concealment. */
+      if (s == 0 && cur && cur->first_mb_in_slice != 0) {
+         uint32_t first_hor = cur->first_mb_in_slice % mb_width;
+         uint32_t first_ver = cur->first_mb_in_slice / mb_width;
+         anv_batch_emit(&cmd_buffer->batch, GENX(MFX_AVC_SLICE_STATE), phantom) {
+            phantom.SliceType = ISlice;
+            phantom.SliceStartMBNumber = 0;
+            phantom.SliceHorizontalPosition = 0;
+            phantom.SliceVerticalPosition = 0;
+            phantom.NextSliceHorizontalPosition = first_hor;
+            phantom.NextSliceVerticalPosition = first_ver;
+            phantom.SliceQuantizationParameter = (uint32_t)(pps->pic_init_qp_minus26 + 26);
+            phantom.LastSliceGroup = false;
+         }
+         anv_batch_emit(&cmd_buffer->batch, GENX(MFD_AVC_BSD_OBJECT), phantom_bsd) {
+            phantom_bsd.IndirectBSDDataLength = 0;
+            phantom_bsd.IndirectBSDDataStartAddress = 0;
+            phantom_bsd.InlineData.LastSlice = false;
+            phantom_bsd.InlineData.FixPrevMBSkipped = 1;
+#if GFX_VERx10 >= 75
+            phantom_bsd.InlineData.IntraPredictionErrorControl = 1;
+            phantom_bsd.InlineData.Intra8x84x4PredictionErrorConcealmentControl = 1;
+            phantom_bsd.InlineData.ISliceConcealmentMode = 1;
+#endif
+         }
+      }
+
+      {
+         int st = cur ? cur->slice_type : ANV_H264_SLICE_I;
+         uint32_t next_fmb = (uint32_t)(mb_width * mb_height);
+         if (!last_slice && sp)
+            next_fmb = (uint32_t)sp[s + 1].first_mb_in_slice;
+         uint32_t cur_fmb = cur ? (uint32_t)cur->first_mb_in_slice : 0;
+         uint32_t cur_hor = cur_fmb % mb_width;
+         uint32_t cur_ver = cur_fmb / mb_width;
+         uint32_t next_hor = next_fmb % mb_width;
+         uint32_t next_ver = next_fmb / mb_width;
+         if (next_ver >= mb_height) { next_hor = 0; next_ver = mb_height; }
+
+         anv_batch_emit(&cmd_buffer->batch, GENX(MFX_AVC_SLICE_STATE), ss) {
+            ss.SliceType =
+               (st == ANV_H264_SLICE_P || st == ANV_H264_SLICE_SP) ? PSlice :
+               (st == ANV_H264_SLICE_B)                            ? BSlice :
+                                                                   ISlice;
+
+            bool implicit_bipred = (st == ANV_H264_SLICE_B &&
+                                    pps->weighted_bipred_idc == 2);
+            ss.Log2WeightDenominatorLuma = implicit_bipred ? 5 :
+                                           (cur ? CLAMP(cur->luma_log2_weight_denom, 0, 7) : 0);
+            ss.Log2WeightDenominatorChroma = implicit_bipred ? 5 :
+                                             (cur ? CLAMP(cur->chroma_log2_weight_denom, 0, 7) : 0);
+            ss.NumberofReferencePicturesinInterpredictionList0 =
+               (cur && st != ANV_H264_SLICE_I && st != ANV_H264_SLICE_SI) ?
+               MIN2((uint32_t)(cur->num_ref_idx_l0_active_minus1 + 1),
+                    (uint32_t)ANV_H264_MAX_REF_FRAMES) : 0;
+            ss.NumberofReferencePicturesinInterpredictionList1 =
+               (cur && st == ANV_H264_SLICE_B) ?
+               MIN2((uint32_t)(cur->num_ref_idx_l1_active_minus1 + 1),
+                    (uint32_t)ANV_H264_MAX_REF_FRAMES) : 0;
+            ss.SliceAlphaC0OffsetDiv2 = cur ? cur->slice_alpha_c0_offset_div2 : 0;
+            ss.SliceBetaOffsetDiv2 = cur ? cur->slice_beta_offset_div2 : 0;
+            ss.SliceQuantizationParameter =
+               (uint32_t)CLAMP((int)(pps->pic_init_qp_minus26 + 26 +
+               (cur ? cur->slice_qp_delta : 0)), 0, 51);
+            ss.CABACInitIDC = cur ? MIN2((uint32_t)cur->cabac_init_idc, 2u) : 0;
+            ss.DisableDeblockingFilterIndicator =
+               cur ? MIN2((uint32_t)cur->disable_deblocking_filter_idc, 2u) : 0;
+            ss.DirectPredictionType =
+               (cur && cur->direct_spatial_mv_pred_flag) ? Spatial : Temporal;
+            ss.WeightedPredictionIndicator =
+               (st == ANV_H264_SLICE_B) ? (uint32_t)pps->weighted_bipred_idc :
+               need_w_l0 ? 1u : 0u;
+            ss.SliceStartMBNumber = MIN2(cur_fmb, 32767u);
+            ss.SliceHorizontalPosition = MIN2(cur_hor, 255u);
+            ss.SliceVerticalPosition = MIN2(cur_ver, 255u);
+            ss.NextSliceHorizontalPosition = MIN2(next_hor, 255u);
+            ss.NextSliceVerticalPosition = MIN2(next_ver, 255u);
+            ss.LastSliceGroup = last_slice;
+            ss.SliceID = s & (uint32_t)((1u << 4) - 1);
+         }
+      }
+
+      uint32_t bsd_nal = bs_map ?
+         anv_h264_find_slice_nal(
+            bs_map + (frame_info->srcBufferOffset & ~4095u),
+            buffer_offset + current_offset,
+            buffer_offset + this_end) :
+         buffer_offset + current_offset + HEADER_OFFSET;
       anv_batch_emit(&cmd_buffer->batch, GENX(MFD_AVC_BSD_OBJECT), avc_bsd) {
-         avc_bsd.IndirectBSDDataLength = this_end - current_offset - HEADER_OFFSET;
+#if GFX_VERx10 == 70
-         /* start decoding after the 3-byte header. */
+         avc_bsd.IndirectBSDDataLength =
-         avc_bsd.IndirectBSDDataStartAddress = buffer_offset + current_offset + HEADER_OFFSET;
+            buffer_offset + frame_info->srcBufferRange - bsd_nal;
+#else
+         avc_bsd.IndirectBSDDataLength = buffer_offset + this_end - bsd_nal;
+#endif
+         avc_bsd.IndirectBSDDataStartAddress = bsd_nal;
          avc_bsd.InlineData.LastSlice = last_slice;
          avc_bsd.InlineData.FixPrevMBSkipped = 1;
+         avc_bsd.InlineData.MBHeaderErrorHandling = 1;
+         avc_bsd.InlineData.EntropyErrorHandling = 1;
+         avc_bsd.InlineData.MPRErrorHandling = 1;
+         avc_bsd.InlineData.BSDPrematureCompleteErrorHandling = 1;
+         avc_bsd.InlineData.IntraPredMode4x48x8LumaErrorControl = 1;
+         /* Initially I thought Ivy Bridge was incapable of doing short mode
+          * and loved to draw rainbows all day, but the more likely scenario
+          * was a fundamental breakdown here.
+          */
+         if (cur) {
+            avc_bsd.InlineData.FirstMBByteOffsetofSliceDataorSliceHeader =
+               cur->first_mb_byte_offset;
+            avc_bsd.InlineData.FirstMBBitOffset = cur->first_mb_bit_offset;
+         }
 #if GFX_VERx10 >= 75
          avc_bsd.InlineData.IntraPredictionErrorControl = 1;
          avc_bsd.InlineData.Intra8x84x4PredictionErrorConcealmentControl = 1;
          avc_bsd.InlineData.ISliceConcealmentMode = 1;
 #endif
-      };
       }
+   }
+
+   free(sp);
+   if (bs_mapped_here)
+      anv_device_unmap_bo(cmd_buffer->device,
+                          src_buffer->address.bo,
+                          (void *)bs_map,
+                          src_buffer->address.bo->size);
 }
 
 void
-genX(CmdDecodeVideoKHR)(VkCommandBuffer commandBuffer,
+genX(CmdDecodeVideoKHR) (VkCommandBuffer commandBuffer,
-                        const VkVideoDecodeInfoKHR *frame_info)
+                         const VkVideoDecodeInfoKHR * frame_info)
 {
    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
+
+   if (!cmd_buffer->video.vid)
+      abort();
+
    switch (cmd_buffer->video.vid->vk.op) {
    case VK_VIDEO_CODEC_OPERATION_DECODE_H264_BIT_KHR:
       anv_h264_decode_video(cmd_buffer, frame_info);
       break;
    default:
-      assert(0);
+      UNREACHABLE("Unsupported video codec operation!");
    }
 }
-
-#ifdef VK_ENABLE_BETA_EXTENSIONS
-void
-genX(CmdEncodeVideoKHR)(VkCommandBuffer commandBuffer,
-                        const VkVideoEncodeInfoKHR *pEncodeInfo)
-{
-}
-#endif