From 809d0ca65c7089e2ddfd66c591ff53b00af9fca0 Mon Sep 17 00:00:00 2001
From: chengzongkai <chengzongkai@outlook.com>
Date: Mon, 13 Jul 2026 15:32:13 +0800
Subject: [PATCH] =?UTF-8?q?fix:=20=E4=B8=BA=20VAAPI=20=E6=B7=BB=E5=8A=A0?=
 =?UTF-8?q?=20CQP=20=E7=A0=81=E7=8E=87=E6=8E=A7=E5=88=B6=E5=9B=9E=E9=80=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 libs/hwcodec/cpp/common/util.cpp              | 20 ++++++++++
 .../cpp/ffmpeg_ram/ffmpeg_ram_encode.cpp      | 40 ++++++++++++++++---
 2 files changed, 55 insertions(+), 5 deletions(-)

diff --git a/libs/hwcodec/cpp/common/util.cpp b/libs/hwcodec/cpp/common/util.cpp
index 08e4f87a..7eba59bd 100644
--- a/libs/hwcodec/cpp/common/util.cpp
+++ b/libs/hwcodec/cpp/common/util.cpp
@@ -368,6 +368,26 @@ struct CodecOptions {
 
 bool set_rate_control(AVCodecContext *c, const std::string &name, int rc,
                       int q) {
+  if (name.find("vaapi") != std::string::npos && rc == RC_CQ) {
+    // Used only after the normal bitrate-based VAAPI initialization fails.
+    // Some drivers, including Intel iHD on Jasper Lake, expose CQP as their
+    // only compatible rate-control mode.
+    c->bit_rate = 0;
+    c->rc_min_rate = 0;
+    c->rc_max_rate = 0;
+    c->rc_buffer_size = 0;
+    c->rc_initial_buffer_occupancy = 0;
+
+    const int qp = q > 0 ? q : 23;
+    const int ret = av_opt_set_int(c->priv_data, "qp", qp, 0);
+    if (ret < 0) {
+      LOG_ERROR(std::string("vaapi set qp failed, ret = ") +
+                av_err2str(ret));
+      return false;
+    }
+    return true;
+  }
+
   if (name.find("qsv") != std::string::npos) {
     // https://github.com/LizardByte/Sunshine/blob/3e47cd3cc8fd37a7a88be82444ff4f3c0022856b/src/video.cpp#L1635
     c->strict_std_compliance = FF_COMPLIANCE_UNOFFICIAL;
diff --git a/libs/hwcodec/cpp/ffmpeg_ram/ffmpeg_ram_encode.cpp b/libs/hwcodec/cpp/ffmpeg_ram/ffmpeg_ram_encode.cpp
index c4507a54..7ccef090 100644
--- a/libs/hwcodec/cpp/ffmpeg_ram/ffmpeg_ram_encode.cpp
+++ b/libs/hwcodec/cpp/ffmpeg_ram/ffmpeg_ram_encode.cpp
@@ -273,7 +273,10 @@ public:
       LOG_ERROR(std::string("set_quality failed, name: ") + name_);
       return false;
     }
-    util_encode::set_rate_control(c_, name_, rc_, q_);
+    if (!util_encode::set_rate_control(c_, name_, rc_, q_)) {
+      LOG_ERROR(std::string("set_rate_control failed, name: ") + name_);
+      return false;
+    }
     util_encode::set_gpu(c_->priv_data, name_, gpu_);
     util_encode::force_hw(c_->priv_data, name_);
     util_encode::set_others(c_->priv_data, name_);
@@ -694,11 +697,38 @@ ffmpeg_ram_new_encoder(const char *name, const char *mc_name, int width,
                        RamEncodeCallback callback) {
   FFmpegRamEncoder *encoder = NULL;
   try {
-    encoder = new FFmpegRamEncoder(name, mc_name, width, height, pixfmt, align,
-                                   fps, gop, rc, quality, kbs, q, thread_count,
-                                   gpu, callback);
+    auto try_create = [&](int attempt_rc, int attempt_kbs) {
+      FFmpegRamEncoder *candidate = new FFmpegRamEncoder(
+          name, mc_name, width, height, pixfmt, align, fps, gop, attempt_rc,
+          quality, attempt_kbs, q, thread_count, gpu, callback);
+      if (candidate && candidate->init(linesize, offset, length)) {
+        return candidate;
+      }
+
+      if (candidate) {
+        candidate->free_encoder();
+        delete candidate;
+      }
+      return static_cast<FFmpegRamEncoder *>(NULL);
+    };
+
+    // Preserve the existing VAAPI path first. With a target bitrate and no
+    // explicit QP, FFmpeg negotiates a supported bitrate-based mode such as
+    // AVBR, VBR or CBR with the driver.
+    encoder = try_create(rc, kbs);
     if (encoder) {
-      if (encoder->init(linesize, offset, length)) {
+      return encoder;
+    }
+
+    // Retry only VAAPI with a fresh context in explicit-QP mode. This avoids
+    // changing rate control on hardware which already supports CBR/VBR while
+    // allowing CQP-only drivers to pass probing and normal encoder creation.
+    if (name && std::string(name).find("vaapi") != std::string::npos &&
+        rc != RC_CQ) {
+      LOG_WARN(std::string("VAAPI bitrate-based rate control failed for ") +
+               name + ", retrying with CQP");
+      encoder = try_create(RC_CQ, 0);
+      if (encoder) {
         return encoder;
       }
     }