hasvk: Add open-loop CBR and VBR rate control for gen7.

Only constant QP was offered.  Add the two bitrate modes, with the quantizer
derived once per picture from the requested bitrate and frame rate.

There is no closed loop and there cannot be one here: a picture's size is
known only after the GPU encodes it, and vkCmdEncodeVideoKHR merely records
commands.  The VA-API encoder for this hardware runs its bit rate control
after each frame completes, which that API's synchronisation allows and this
one does not, and gen7 has no macroblock-level rate control the hardware
drives on its own -- the VA-API driver leaves those MFX_AVC_IMG_STATE fields
zero on this generation.  Vulkan defines rate control as best effort, and
this is the effort.

The quantizer comes from a measured curve rather than a guess: sweeping QP 14
to 50 over a 176x144 clip gives 361 bits per macroblock at QP 26 for an I
picture and 166 for a P one, halving every 12.5 steps of the quantizer for I
and every 10.3 for P.  Measured against the target across a 15x range of
bitrates, I pictures land between 0.82x and 1.30x, P pictures between 0.59x
and 1.56x -- the spread on P is content dependent and nothing open loop can
narrow it.  A target below about 90 bits per macroblock cannot be met at all,
since that is where the curve flattens.

dEQP-VK.video.encode.h264 goes from 84 to 92 of 120 with no failures.  The
remaining 28 are quantization maps, which gen7 has no hardware path for, and
B frames.

Co-Authored-By: Claude Opus 5 <[email protected]>

src/intel/vulkan_hasvk/anv_private.h

@@ -2677,6 +2677,16 @@ struct anv_cmd_buffer {
        * the encode command knows the final bitstream offset/size and does the
        * actual write. NULL bo means no active feedback query. */
       struct anv_address                         encode_feedback_query_addr;
+
+      /* ENCODE — rate control, from VkVideoEncodeRateControlInfoKHR on either
+       * vkCmdBeginVideoCodingKHR or vkCmdControlVideoCodingKHR.  DISABLED means
+       * the client's own constant QP is used instead. */
+      VkVideoEncodeRateControlModeFlagBitsKHR    rc_mode;
+      uint64_t                                   rc_average_bitrate;
+      uint32_t                                   rc_frame_rate_num;
+      uint32_t                                   rc_frame_rate_den;
+      uint32_t                                   rc_min_qp;
+      uint32_t                                   rc_max_qp;
    } video;
 };
 

src/intel/vulkan_hasvk/anv_video.c

@@ -215,11 +215,17 @@ anv_GetPhysicalDeviceVideoCapabilitiesKHR(VkPhysicalDevice physicalDevice,
          vk_find_struct(pCapabilities->pNext, VIDEO_ENCODE_CAPABILITIES_KHR);
       if (enc_caps) {
          enc_caps->flags = 0;
-         /* Constant QP only; no CBR/VBR rate control. */
+         /* CBR and VBR are open loop: the quantizer is derived from the
+          * bitrate target once per picture and nothing measures the result,
+          * so both behave the same and neither tracks the target closely.
+          * See anv_video_rate_control_qp(). */
          enc_caps->rateControlModes =
-            VK_VIDEO_ENCODE_RATE_CONTROL_MODE_DISABLED_BIT_KHR;
+            VK_VIDEO_ENCODE_RATE_CONTROL_MODE_DISABLED_BIT_KHR |
+            VK_VIDEO_ENCODE_RATE_CONTROL_MODE_CBR_BIT_KHR |
+            VK_VIDEO_ENCODE_RATE_CONTROL_MODE_VBR_BIT_KHR;
          enc_caps->maxRateControlLayers = 1;
-         enc_caps->maxBitrate = 0;
+         /* Level 4.1's ceiling, the highest level the encoder reports. */
+         enc_caps->maxBitrate = 62500000;
          enc_caps->maxQualityLevels = 1;
          enc_caps->encodeInputPictureGranularity.width = ANV_MB_WIDTH;
          enc_caps->encodeInputPictureGranularity.height = ANV_MB_HEIGHT;

src/intel/vulkan_hasvk/genX_cmd_video.c

@@ -111,6 +111,88 @@ anv_video_init_neutral_chroma(struct anv_device *device,
    mut_img->vid_neutral_chroma_init = true;
 }
 
+/* Open-loop rate control: derive a quantizer for the picture from the bitrate
+ * target.  There is no closed loop.  A picture's size is only known once the
+ * GPU has encoded it, and vkCmdEncodeVideoKHR merely records commands, so
+ * nothing here can react to what the previous picture actually cost -- the
+ * VA-API encoder for this hardware adjusts the quantizer after each frame
+ * completes, which that API's synchronisation makes possible and this one does
+ * not.  Vulkan defines rate control as best effort; this is the effort.  The
+ * stream lands near the requested bitrate rather than tracking it, and a burst
+ * of detail will overshoot until the next picture.
+ *
+ * The curve is measured on this encoder, sweeping QP 14 to 50 over a 176x144
+ * clip: at QP 26 a macroblock costs 361 bits in an I picture and 166 in a P
+ * one, and the cost halves every 12.5 steps of the quantizer for I pictures
+ * and every 10.3 for P.  It flattens out near 90 bits per macroblock, so a
+ * target below that cannot be met however high the quantizer goes and the
+ * picture simply overshoots.
+ */
+static uint32_t
+anv_video_rate_control_qp(const struct anv_cmd_buffer *cmd_buffer,
+                          uint32_t total_mbs, bool is_p)
+{
+   uint64_t bitrate = cmd_buffer->video.rc_average_bitrate;
+   uint32_t rate_num = cmd_buffer->video.rc_frame_rate_num;
+   uint32_t rate_den = cmd_buffer->video.rc_frame_rate_den;
+   uint32_t min_qp = cmd_buffer->video.rc_min_qp;
+   uint32_t max_qp = cmd_buffer->video.rc_max_qp ?
+                     cmd_buffer->video.rc_max_qp : 51;
+
+   if (bitrate == 0 || rate_num == 0 || total_mbs == 0)
+      return CLAMP(26, min_qp, max_qp);
+
+   uint64_t bits_per_frame = bitrate * rate_den / rate_num;
+   uint64_t bits_per_mb = MAX2(bits_per_frame / total_mbs, 1);
+   float anchor = is_p ? 166.0f : 361.0f;
+   float steps_per_halving = is_p ? 10.3f : 12.5f;
+   int qp = (int)(26.0f + steps_per_halving *
+                  log2f(anchor / (float)bits_per_mb) + 0.5f);
+
+   return CLAMP(qp, (int)min_qp, (int)max_qp);
+}
+
+/* Rate control arrives on either vkCmdBeginVideoCodingKHR or
+ * vkCmdControlVideoCodingKHR, so both funnel through here. */
+static void
+anv_video_set_rate_control(struct anv_cmd_buffer *cmd_buffer, const void *pnext)
+{
+   /* Command buffers come back from the pool with this struct untouched, so
+    * clear it whether or not the caller asked for rate control. */
+   cmd_buffer->video.rc_mode = VK_VIDEO_ENCODE_RATE_CONTROL_MODE_DEFAULT_KHR;
+   cmd_buffer->video.rc_average_bitrate = 0;
+   cmd_buffer->video.rc_frame_rate_num = 0;
+   cmd_buffer->video.rc_frame_rate_den = 1;
+   cmd_buffer->video.rc_min_qp = 0;
+   cmd_buffer->video.rc_max_qp = 0;
+
+   const VkVideoEncodeRateControlInfoKHR *rc =
+      vk_find_struct_const(pnext, VIDEO_ENCODE_RATE_CONTROL_INFO_KHR);
+   if (!rc)
+      return;
+
+   cmd_buffer->video.rc_mode = rc->rateControlMode;
+
+   /* Only one rate control layer is advertised, so only the first is read. */
+   if (rc->layerCount > 0 && rc->pLayers) {
+      const VkVideoEncodeRateControlLayerInfoKHR *layer = &rc->pLayers[0];
+      cmd_buffer->video.rc_average_bitrate = layer->averageBitrate;
+      cmd_buffer->video.rc_frame_rate_num = layer->frameRateNumerator;
+      cmd_buffer->video.rc_frame_rate_den =
+         layer->frameRateDenominator ? layer->frameRateDenominator : 1;
+
+      const VkVideoEncodeH264RateControlLayerInfoKHR *h264_layer =
+         vk_find_struct_const(layer->pNext,
+                              VIDEO_ENCODE_H264_RATE_CONTROL_LAYER_INFO_KHR);
+      if (h264_layer) {
+         if (h264_layer->useMinQp)
+            cmd_buffer->video.rc_min_qp = h264_layer->minQp.qpP;
+         if (h264_layer->useMaxQp)
+            cmd_buffer->video.rc_max_qp = h264_layer->maxQp.qpP;
+      }
+   }
+}
+
 void
 genX(CmdBeginVideoCodingKHR) (VkCommandBuffer commandBuffer,
                               const VkVideoBeginCodingInfoKHR * pBeginInfo)
@@ -127,10 +209,16 @@ genX(CmdBeginVideoCodingKHR) (VkCommandBuffer commandBuffer,
     * from a previous use. Clear it here; otherwise CmdEncodeVideoKHR would try
     * to write to that stale address even when no feedback query is active. */
    cmd_buffer->video.encode_feedback_query_addr = ANV_NULL_ADDRESS;
+   anv_video_set_rate_control(cmd_buffer, pBeginInfo->pNext);
 
    if (getenv("HASVK_ENC_LOG"))
-      mesa_logi("ENC CmdBeginVideoCodingKHR op=%u refSlots=%u",
-                vid->vk.op, pBeginInfo->referenceSlotCount);
+      mesa_logi("ENC CmdBeginVideoCodingKHR op=%u refSlots=%u rc_mode=%u "
+                "bitrate=%llu fps=%u/%u",
+                vid->vk.op, pBeginInfo->referenceSlotCount,
+                cmd_buffer->video.rc_mode,
+                (unsigned long long)cmd_buffer->video.rc_average_bitrate,
+                cmd_buffer->video.rc_frame_rate_num,
+                cmd_buffer->video.rc_frame_rate_den);
 }
 
 void
@@ -138,6 +226,12 @@ genX(CmdControlVideoCodingKHR) (VkCommandBuffer commandBuffer,
                                 const VkVideoCodingControlInfoKHR *
                                 pCodingControlInfo)
 {
+   ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
+
+   if (pCodingControlInfo->flags &
+       VK_VIDEO_CODING_CONTROL_ENCODE_RATE_CONTROL_BIT_KHR)
+      anv_video_set_rate_control(cmd_buffer, pCodingControlInfo->pNext);
+
    if (getenv("HASVK_ENC_LOG"))
       mesa_logi("ENC CmdControlVideoCodingKHR flags=0x%x", pCodingControlInfo->flags);
 }
@@ -1215,11 +1309,19 @@ anv_h264_encode_video(struct anv_cmd_buffer *cmd_buffer,
       StdVideoEncodeH264SliceHeader sh_fixed = *sh;
       sh_fixed.first_mb_in_slice = slice_start_mb;
 
+      /* Under rate control the client leaves constantQp at zero and the
+       * quantizer is the driver's to pick. */
+      uint32_t slice_qp =
+         cmd_buffer->video.rc_mode == VK_VIDEO_ENCODE_RATE_CONTROL_MODE_DISABLED_BIT_KHR ||
+         cmd_buffer->video.rc_mode == VK_VIDEO_ENCODE_RATE_CONTROL_MODE_DEFAULT_KHR ?
+         (uint32_t)entry->constantQp :
+         anv_video_rate_control_qp(cmd_buffer, total_mbs, is_p);
+
       genX(hasvk_mfc_emit_slice)(cmd_buffer, sps, pps,
                                  h264_pic_info->pStdPictureInfo, &sh_fixed,
                                  slice_start_mb, slice_end_mb,
                                  width_in_mbs, height_in_mbs,
-                                 (uint32_t)entry->constantQp,
+                                 slice_qp,
                                  s + 1 == slice_count, dst_page_off,
                                  intra_refresh_ptr);
    }