feat(video): 事务化切换与前端统一编排,增强视频输入格式支持

- 后端:切换事务+transition_id,/stream/mode 返回 switching/transition_id 与实际 codec

- 事件:新增 mode_switching/mode_ready,config/webrtc_ready/mode_changed 关联事务

- 编码/格式:扩展 NV21/NV16/NV24/RGB/BGR 输入与转换链路,RKMPP direct input 优化

- 前端:useVideoSession 统一切换,失败回退真实切回 MJPEG,菜单格式同步修复

- 清理:useVideoStream 降级为 MJPEG-only
This commit is contained in:
mofeng-git
2026-01-11 10:41:57 +08:00
parent 9feb74b72c
commit 206594e292
110 changed files with 3955 additions and 2251 deletions

View File

@@ -99,8 +99,18 @@ pub enum H264InputFormat {
Yuv420p,
/// NV12 - Y plane + interleaved UV plane (optimal for VAAPI)
Nv12,
/// NV21 - Y plane + interleaved VU plane
Nv21,
/// NV16 - Y plane + interleaved UV plane (4:2:2)
Nv16,
/// NV24 - Y plane + interleaved UV plane (4:4:4)
Nv24,
/// YUYV422 - packed YUV 4:2:2 format (optimal for RKMPP direct input)
Yuyv422,
/// RGB24 - packed RGB format (RKMPP direct input)
Rgb24,
/// BGR24 - packed BGR format (RKMPP direct input)
Bgr24,
}
impl Default for H264InputFormat {
@@ -202,7 +212,7 @@ pub fn get_available_encoders(width: u32, height: u32) -> Vec<CodecInfo> {
fps: 30,
gop: 30,
rc: RateControl::RC_CBR,
quality: Quality::Quality_Low, // Use low quality preset for fastest encoding (ultrafast)
quality: Quality::Quality_Low, // Use low quality preset for fastest encoding (ultrafast)
kbs: 2000,
q: 23,
thread_count: 4,
@@ -270,9 +280,8 @@ impl H264Encoder {
// Detect best encoder
let (_encoder_type, codec_name) = detect_best_encoder(width, height);
let codec_name = codec_name.ok_or_else(|| {
AppError::VideoError("No H.264 encoder available".to_string())
})?;
let codec_name = codec_name
.ok_or_else(|| AppError::VideoError("No H.264 encoder available".to_string()))?;
Self::with_codec(config, &codec_name)
}
@@ -287,8 +296,13 @@ impl H264Encoder {
// Select pixel format based on config
let pixfmt = match config.input_format {
H264InputFormat::Nv12 => AVPixelFormat::AV_PIX_FMT_NV12,
H264InputFormat::Nv21 => AVPixelFormat::AV_PIX_FMT_NV21,
H264InputFormat::Nv16 => AVPixelFormat::AV_PIX_FMT_NV16,
H264InputFormat::Nv24 => AVPixelFormat::AV_PIX_FMT_NV24,
H264InputFormat::Yuv420p => AVPixelFormat::AV_PIX_FMT_YUV420P,
H264InputFormat::Yuyv422 => AVPixelFormat::AV_PIX_FMT_YUYV422,
H264InputFormat::Rgb24 => AVPixelFormat::AV_PIX_FMT_RGB24,
H264InputFormat::Bgr24 => AVPixelFormat::AV_PIX_FMT_BGR24,
};
info!(
@@ -306,10 +320,10 @@ impl H264Encoder {
fps: config.fps as i32,
gop: config.gop_size as i32,
rc: RateControl::RC_CBR,
quality: Quality::Quality_Low, // Use low quality preset for fastest encoding (lowest latency)
quality: Quality::Quality_Low, // Use low quality preset for fastest encoding (lowest latency)
kbs: config.bitrate_kbps as i32,
q: 23,
thread_count: 4, // Use 4 threads for better performance
thread_count: 4, // Use 4 threads for better performance
};
let inner = HwEncoder::new(ctx).map_err(|_| {
@@ -353,9 +367,9 @@ impl H264Encoder {
/// Update bitrate dynamically
pub fn set_bitrate(&mut self, bitrate_kbps: u32) -> Result<()> {
self.inner.set_bitrate(bitrate_kbps as i32).map_err(|_| {
AppError::VideoError("Failed to set bitrate".to_string())
})?;
self.inner
.set_bitrate(bitrate_kbps as i32)
.map_err(|_| AppError::VideoError("Failed to set bitrate".to_string()))?;
self.config.bitrate_kbps = bitrate_kbps;
debug!("Bitrate updated to {} kbps", bitrate_kbps);
Ok(())
@@ -394,16 +408,7 @@ impl H264Encoder {
Ok(owned_frames)
}
Err(e) => {
// For the first ~30 frames, x264 may fail due to initialization
// Log as warning instead of error to avoid alarming users
if self.frame_count <= 30 {
warn!(
"Encode failed during initialization (frame {}): {} - this is normal for x264",
self.frame_count, e
);
} else {
error!("Encode failed: {}", e);
}
error!("Encode failed: {}", e);
Err(AppError::VideoError(format!("Encode failed: {}", e)))
}
}
@@ -458,7 +463,9 @@ impl Encoder for H264Encoder {
if frames.is_empty() {
// Encoder needs more frames (shouldn't happen with our config)
warn!("Encoder returned no frames");
return Err(AppError::VideoError("Encoder returned no frames".to_string()));
return Err(AppError::VideoError(
"Encoder returned no frames".to_string(),
));
}
// Take ownership of the first frame (zero-copy)
@@ -493,8 +500,13 @@ impl Encoder for H264Encoder {
// Check if the format matches our configured input format
match self.config.input_format {
H264InputFormat::Nv12 => matches!(format, PixelFormat::Nv12),
H264InputFormat::Nv21 => matches!(format, PixelFormat::Nv21),
H264InputFormat::Nv16 => matches!(format, PixelFormat::Nv16),
H264InputFormat::Nv24 => matches!(format, PixelFormat::Nv24),
H264InputFormat::Yuv420p => matches!(format, PixelFormat::Yuv420),
H264InputFormat::Yuyv422 => matches!(format, PixelFormat::Yuyv),
H264InputFormat::Rgb24 => matches!(format, PixelFormat::Rgb24),
H264InputFormat::Bgr24 => matches!(format, PixelFormat::Bgr24),
}
}
}
@@ -538,7 +550,11 @@ mod tests {
let config = H264Config::low_latency(Resolution::HD720, 2000);
match H264Encoder::new(config) {
Ok(encoder) => {
println!("Created encoder: {} ({})", encoder.codec_name(), encoder.encoder_type());
println!(
"Created encoder: {} ({})",
encoder.codec_name(),
encoder.encoder_type()
);
}
Err(e) => {
println!("Failed to create encoder: {}", e);

View File

@@ -92,8 +92,18 @@ pub enum H265InputFormat {
Yuv420p,
/// NV12 - Y plane + interleaved UV plane (optimal for hardware encoders)
Nv12,
/// NV21 - Y plane + interleaved VU plane
Nv21,
/// NV16 - Y plane + interleaved UV plane (4:2:2)
Nv16,
/// NV24 - Y plane + interleaved UV plane (4:4:4)
Nv24,
/// YUYV422 - packed YUV 4:2:2 format (optimal for RKMPP direct input)
Yuyv422,
/// RGB24 - packed RGB format (RKMPP direct input)
Rgb24,
/// BGR24 - packed BGR format (RKMPP direct input)
Bgr24,
}
impl Default for H265InputFormat {
@@ -252,10 +262,7 @@ pub fn detect_best_h265_encoder(width: u32, height: u32) -> (H265EncoderType, Op
H265EncoderType::Software // Default to software for unknown
};
info!(
"Selected H.265 encoder: {} ({})",
codec.name, encoder_type
);
info!("Selected H.265 encoder: {} ({})", codec.name, encoder_type);
(encoder_type, Some(codec.name.clone()))
}
@@ -304,7 +311,8 @@ impl H265Encoder {
if encoder_type == H265EncoderType::None {
return Err(AppError::VideoError(
"No H.265 encoder available. Please ensure FFmpeg is built with libx265 support.".to_string(),
"No H.265 encoder available. Please ensure FFmpeg is built with libx265 support."
.to_string(),
));
}
@@ -336,8 +344,17 @@ impl H265Encoder {
} else {
match config.input_format {
H265InputFormat::Nv12 => (AVPixelFormat::AV_PIX_FMT_NV12, H265InputFormat::Nv12),
H265InputFormat::Yuv420p => (AVPixelFormat::AV_PIX_FMT_YUV420P, H265InputFormat::Yuv420p),
H265InputFormat::Yuyv422 => (AVPixelFormat::AV_PIX_FMT_YUYV422, H265InputFormat::Yuyv422),
H265InputFormat::Nv21 => (AVPixelFormat::AV_PIX_FMT_NV21, H265InputFormat::Nv21),
H265InputFormat::Nv16 => (AVPixelFormat::AV_PIX_FMT_NV16, H265InputFormat::Nv16),
H265InputFormat::Nv24 => (AVPixelFormat::AV_PIX_FMT_NV24, H265InputFormat::Nv24),
H265InputFormat::Yuv420p => {
(AVPixelFormat::AV_PIX_FMT_YUV420P, H265InputFormat::Yuv420p)
}
H265InputFormat::Yuyv422 => {
(AVPixelFormat::AV_PIX_FMT_YUYV422, H265InputFormat::Yuyv422)
}
H265InputFormat::Rgb24 => (AVPixelFormat::AV_PIX_FMT_RGB24, H265InputFormat::Rgb24),
H265InputFormat::Bgr24 => (AVPixelFormat::AV_PIX_FMT_BGR24, H265InputFormat::Bgr24),
}
};
@@ -407,9 +424,9 @@ impl H265Encoder {
/// Update bitrate dynamically
pub fn set_bitrate(&mut self, bitrate_kbps: u32) -> Result<()> {
self.inner.set_bitrate(bitrate_kbps as i32).map_err(|_| {
AppError::VideoError("Failed to set H.265 bitrate".to_string())
})?;
self.inner
.set_bitrate(bitrate_kbps as i32)
.map_err(|_| AppError::VideoError("Failed to set H.265 bitrate".to_string()))?;
self.config.bitrate_kbps = bitrate_kbps;
debug!("H.265 bitrate updated to {} kbps", bitrate_kbps);
Ok(())
@@ -464,7 +481,10 @@ impl H265Encoder {
if keyframe || self.frame_count % 30 == 1 {
debug!(
"[H265] Encoded frame #{}: output_size={}, keyframe={}, frame_count={}",
self.frame_count, total_size, keyframe, owned_frames.len()
self.frame_count,
total_size,
keyframe,
owned_frames.len()
);
// Log first few bytes of keyframe for debugging
@@ -477,7 +497,10 @@ impl H265Encoder {
}
}
} else {
warn!("[H265] Encoder returned empty frame list for frame #{}", self.frame_count);
warn!(
"[H265] Encoder returned empty frame list for frame #{}",
self.frame_count
);
}
Ok(owned_frames)
@@ -567,8 +590,13 @@ impl Encoder for H265Encoder {
fn supports_format(&self, format: PixelFormat) -> bool {
match self.config.input_format {
H265InputFormat::Nv12 => matches!(format, PixelFormat::Nv12),
H265InputFormat::Nv21 => matches!(format, PixelFormat::Nv21),
H265InputFormat::Nv16 => matches!(format, PixelFormat::Nv16),
H265InputFormat::Nv24 => matches!(format, PixelFormat::Nv24),
H265InputFormat::Yuv420p => matches!(format, PixelFormat::Yuv420),
H265InputFormat::Yuyv422 => matches!(format, PixelFormat::Yuyv),
H265InputFormat::Rgb24 => matches!(format, PixelFormat::Rgb24),
H265InputFormat::Bgr24 => matches!(format, PixelFormat::Bgr24),
}
}
}
@@ -580,7 +608,10 @@ mod tests {
#[test]
fn test_detect_h265_encoder() {
let (encoder_type, codec_name) = detect_best_h265_encoder(1280, 720);
println!("Detected H.265 encoder: {:?} ({:?})", encoder_type, codec_name);
println!(
"Detected H.265 encoder: {:?} ({:?})",
encoder_type, codec_name
);
}
#[test]

View File

@@ -35,10 +35,12 @@ impl JpegEncoder {
// I420: Y = width*height, U = width*height/4, V = width*height/4
let i420_size = width * height * 3 / 2;
let mut compressor = turbojpeg::Compressor::new()
.map_err(|e| AppError::VideoError(format!("Failed to create turbojpeg compressor: {}", e)))?;
let mut compressor = turbojpeg::Compressor::new().map_err(|e| {
AppError::VideoError(format!("Failed to create turbojpeg compressor: {}", e))
})?;
compressor.set_quality(config.quality.min(100) as i32)
compressor
.set_quality(config.quality.min(100) as i32)
.map_err(|e| AppError::VideoError(format!("Failed to set JPEG quality: {}", e)))?;
Ok(Self {
@@ -56,7 +58,8 @@ impl JpegEncoder {
/// Set JPEG quality (1-100)
pub fn set_quality(&mut self, quality: u32) -> Result<()> {
self.compressor.set_quality(quality.min(100) as i32)
self.compressor
.set_quality(quality.min(100) as i32)
.map_err(|e| AppError::VideoError(format!("Failed to set JPEG quality: {}", e)))?;
self.config.quality = quality;
Ok(())
@@ -73,12 +76,14 @@ impl JpegEncoder {
pixels: self.i420_buffer.as_slice(),
width,
height,
align: 1, // No padding between rows
align: 1, // No padding between rows
subsamp: turbojpeg::Subsamp::Sub2x2, // YUV 4:2:0
};
// Compress YUV directly to JPEG (skips color space conversion!)
let jpeg_data = self.compressor.compress_yuv_to_vec(yuv_image)
let jpeg_data = self
.compressor
.compress_yuv_to_vec(yuv_image)
.map_err(|e| AppError::VideoError(format!("JPEG compression failed: {}", e)))?;
Ok(EncodedFrame::jpeg(

View File

@@ -19,7 +19,9 @@ pub mod vp8;
pub mod vp9;
// Core traits and types
pub use traits::{BitratePreset, EncodedFormat, EncodedFrame, Encoder, EncoderConfig, EncoderFactory};
pub use traits::{
BitratePreset, EncodedFormat, EncodedFrame, Encoder, EncoderConfig, EncoderFactory,
};
// WebRTC codec abstraction
pub use codec::{CodecFrame, VideoCodec, VideoCodecConfig, VideoCodecFactory, VideoCodecType};

View File

@@ -264,10 +264,7 @@ impl EncoderRegistry {
if let Some(encoder) = AvailableEncoder::from_codec_info(codec_info) {
debug!(
"Detected encoder: {} ({}) - {} priority={}",
encoder.codec_name,
encoder.format,
encoder.backend,
encoder.priority
encoder.codec_name, encoder.format, encoder.backend, encoder.priority
);
self.encoders
@@ -336,13 +333,15 @@ impl EncoderRegistry {
format: VideoEncoderType,
hardware_only: bool,
) -> Option<&AvailableEncoder> {
self.encoders.get(&format)?.iter().find(|e| {
if hardware_only {
e.is_hardware
} else {
true
}
})
self.encoders.get(&format)?.iter().find(
|e| {
if hardware_only {
e.is_hardware
} else {
true
}
},
)
}
/// Get all encoders for a format
@@ -523,9 +522,6 @@ mod tests {
// Should have detected at least H264 (software fallback available)
println!("Available formats: {:?}", registry.available_formats(false));
println!(
"Selectable formats: {:?}",
registry.selectable_formats()
);
println!("Selectable formats: {:?}", registry.selectable_formats());
}
}

View File

@@ -5,8 +5,8 @@ use serde::{Deserialize, Serialize};
use std::time::Instant;
use typeshare::typeshare;
use crate::video::format::{PixelFormat, Resolution};
use crate::error::Result;
use crate::video::format::{PixelFormat, Resolution};
/// Bitrate preset for video encoding
///
@@ -46,10 +46,10 @@ impl BitratePreset {
/// Quality preset uses longer GOP for better compression efficiency.
pub fn gop_size(&self, fps: u32) -> u32 {
match self {
Self::Speed => (fps / 2).max(15), // 0.5 second, minimum 15 frames
Self::Balanced => fps, // 1 second
Self::Quality => fps * 2, // 2 seconds
Self::Custom(_) => fps, // Default 1 second for custom
Self::Speed => (fps / 2).max(15), // 0.5 second, minimum 15 frames
Self::Balanced => fps, // 1 second
Self::Quality => fps * 2, // 2 seconds
Self::Custom(_) => fps, // Default 1 second for custom
}
}

View File

@@ -186,10 +186,7 @@ pub fn detect_best_vp8_encoder(width: u32, height: u32) -> (VP8EncoderType, Opti
VP8EncoderType::Software // Default to software for unknown
};
info!(
"Selected VP8 encoder: {} ({})",
codec.name, encoder_type
);
info!("Selected VP8 encoder: {} ({})", codec.name, encoder_type);
(encoder_type, Some(codec.name.clone()))
}
@@ -238,7 +235,8 @@ impl VP8Encoder {
if encoder_type == VP8EncoderType::None {
return Err(AppError::VideoError(
"No VP8 encoder available. Please ensure FFmpeg is built with libvpx support.".to_string(),
"No VP8 encoder available. Please ensure FFmpeg is built with libvpx support."
.to_string(),
));
}
@@ -270,7 +268,9 @@ impl VP8Encoder {
} else {
match config.input_format {
VP8InputFormat::Nv12 => (AVPixelFormat::AV_PIX_FMT_NV12, VP8InputFormat::Nv12),
VP8InputFormat::Yuv420p => (AVPixelFormat::AV_PIX_FMT_YUV420P, VP8InputFormat::Yuv420p),
VP8InputFormat::Yuv420p => {
(AVPixelFormat::AV_PIX_FMT_YUV420P, VP8InputFormat::Yuv420p)
}
}
};
@@ -340,9 +340,9 @@ impl VP8Encoder {
/// Update bitrate dynamically
pub fn set_bitrate(&mut self, bitrate_kbps: u32) -> Result<()> {
self.inner.set_bitrate(bitrate_kbps as i32).map_err(|_| {
AppError::VideoError("Failed to set VP8 bitrate".to_string())
})?;
self.inner
.set_bitrate(bitrate_kbps as i32)
.map_err(|_| AppError::VideoError("Failed to set VP8 bitrate".to_string()))?;
self.config.bitrate_kbps = bitrate_kbps;
debug!("VP8 bitrate updated to {} kbps", bitrate_kbps);
Ok(())
@@ -470,7 +470,10 @@ mod tests {
#[test]
fn test_detect_vp8_encoder() {
let (encoder_type, codec_name) = detect_best_vp8_encoder(1280, 720);
println!("Detected VP8 encoder: {:?} ({:?})", encoder_type, codec_name);
println!(
"Detected VP8 encoder: {:?} ({:?})",
encoder_type, codec_name
);
}
#[test]

View File

@@ -186,10 +186,7 @@ pub fn detect_best_vp9_encoder(width: u32, height: u32) -> (VP9EncoderType, Opti
VP9EncoderType::Software // Default to software for unknown
};
info!(
"Selected VP9 encoder: {} ({})",
codec.name, encoder_type
);
info!("Selected VP9 encoder: {} ({})", codec.name, encoder_type);
(encoder_type, Some(codec.name.clone()))
}
@@ -238,7 +235,8 @@ impl VP9Encoder {
if encoder_type == VP9EncoderType::None {
return Err(AppError::VideoError(
"No VP9 encoder available. Please ensure FFmpeg is built with libvpx support.".to_string(),
"No VP9 encoder available. Please ensure FFmpeg is built with libvpx support."
.to_string(),
));
}
@@ -270,7 +268,9 @@ impl VP9Encoder {
} else {
match config.input_format {
VP9InputFormat::Nv12 => (AVPixelFormat::AV_PIX_FMT_NV12, VP9InputFormat::Nv12),
VP9InputFormat::Yuv420p => (AVPixelFormat::AV_PIX_FMT_YUV420P, VP9InputFormat::Yuv420p),
VP9InputFormat::Yuv420p => {
(AVPixelFormat::AV_PIX_FMT_YUV420P, VP9InputFormat::Yuv420p)
}
}
};
@@ -340,9 +340,9 @@ impl VP9Encoder {
/// Update bitrate dynamically
pub fn set_bitrate(&mut self, bitrate_kbps: u32) -> Result<()> {
self.inner.set_bitrate(bitrate_kbps as i32).map_err(|_| {
AppError::VideoError("Failed to set VP9 bitrate".to_string())
})?;
self.inner
.set_bitrate(bitrate_kbps as i32)
.map_err(|_| AppError::VideoError("Failed to set VP9 bitrate".to_string()))?;
self.config.bitrate_kbps = bitrate_kbps;
debug!("VP9 bitrate updated to {} kbps", bitrate_kbps);
Ok(())
@@ -470,7 +470,10 @@ mod tests {
#[test]
fn test_detect_vp9_encoder() {
let (encoder_type, codec_name) = detect_best_vp9_encoder(1280, 720);
println!("Detected VP9 encoder: {:?} ({:?})", encoder_type, codec_name);
println!(
"Detected VP9 encoder: {:?} ({:?})",
encoder_type, codec_name
);
}
#[test]