feat: 新增安卓平台支持

This commit is contained in:
mofeng-git
2026-05-24 08:37:19 +00:00
parent dc6475776e
commit b31aae284d
105 changed files with 7900 additions and 473 deletions

View File

@@ -0,0 +1,122 @@
//! Android FFmpeg/MediaCodec encoder glue.
use bytes::Bytes;
use hwcodec::common::{Quality, RateControl};
use hwcodec::ffmpeg::{resolve_pixel_format, AVPixelFormat};
use hwcodec::ffmpeg_ram::encode::{EncodeContext, Encoder as HwEncoder};
use crate::error::{AppError, Result};
use crate::video::format::{PixelFormat, Resolution};
pub struct AndroidMediaCodecH264Encoder {
inner: HwEncoder,
resolution: Resolution,
input_format: PixelFormat,
bitrate_kbps: u32,
}
impl AndroidMediaCodecH264Encoder {
pub fn new(
resolution: Resolution,
input_format: PixelFormat,
fps: u32,
bitrate_kbps: u32,
) -> Result<Self> {
let pixfmt = match input_format {
PixelFormat::Nv12 => resolve_pixel_format("nv12", AVPixelFormat::AV_PIX_FMT_NV12),
PixelFormat::Yuv420 => {
resolve_pixel_format("yuv420p", AVPixelFormat::AV_PIX_FMT_YUV420P)
}
other => {
return Err(AppError::VideoError(format!(
"FFmpeg h264_mediacodec accepts NV12/YUV420P memory frames; {other} requires conversion first"
)))
}
};
let ctx = EncodeContext {
name: "h264_mediacodec".to_string(),
mc_name: None,
width: resolution.width as i32,
height: resolution.height as i32,
pixfmt,
align: 1,
fps: fps.max(1) as i32,
gop: fps.max(1) as i32,
rc: RateControl::RC_CBR,
quality: Quality::Quality_Low,
kbs: bitrate_kbps.max(1) as i32,
q: 23,
thread_count: 1,
};
let inner = HwEncoder::new(ctx).map_err(|_| {
AppError::VideoError("Failed to create FFmpeg h264_mediacodec encoder".to_string())
})?;
Ok(Self {
inner,
resolution,
input_format,
bitrate_kbps: bitrate_kbps.max(1),
})
}
pub fn encode_raw(&mut self, data: &[u8], pts_ms: i64) -> Result<Vec<AndroidH264Packet>> {
let min_len = self
.input_format
.frame_size(self.resolution)
.ok_or_else(|| AppError::VideoError("MediaCodec input must be raw YUV".to_string()))?;
if data.len() < min_len {
return Err(AppError::VideoError(format!(
"MediaCodec {} frame too small: {} < {}",
self.input_format,
data.len(),
min_len
)));
}
let packets = self
.inner
.encode_bytes(data, pts_ms)
.map_err(|err| AppError::VideoError(format!("h264_mediacodec encode failed: {err}")))?;
Ok(packets
.into_iter()
.map(|packet| AndroidH264Packet {
data: packet.data,
pts: packet.pts,
key_frame: packet.key == 1,
})
.collect())
}
pub fn set_bitrate(&mut self, bitrate_kbps: u32) -> Result<()> {
self.inner
.set_bitrate(bitrate_kbps.max(1) as i32)
.map_err(|_| AppError::VideoError("Failed to set MediaCodec bitrate".to_string()))?;
self.bitrate_kbps = bitrate_kbps.max(1);
Ok(())
}
pub fn request_keyframe(&mut self) {
self.inner.request_keyframe();
}
pub fn codec_name(&self) -> &str {
"h264_mediacodec"
}
pub fn input_format(&self) -> PixelFormat {
self.input_format
}
}
unsafe impl Send for AndroidMediaCodecH264Encoder {}
#[derive(Debug, Clone)]
pub struct AndroidH264Packet {
pub data: Bytes,
pub pts: i64,
pub key_frame: bool,
}

View File

@@ -0,0 +1,137 @@
//! Android FFmpeg/MediaCodec MJPEG decoder glue.
use hwcodec::ffmpeg::AVPixelFormat;
use hwcodec::ffmpeg_ram::decode::{DecodeContext, Decoder};
use tracing::{info, warn};
use crate::error::{AppError, Result};
use crate::video::codec::convert::Nv12Converter;
use crate::video::format::{PixelFormat, Resolution};
pub struct AndroidMediaCodecMjpegDecoder {
decoder: Decoder,
resolution: Resolution,
nv12_converter: Option<Nv12Converter>,
last_output_format: Option<PixelFormat>,
pending_frames: u32,
}
impl AndroidMediaCodecMjpegDecoder {
pub fn new(resolution: Resolution) -> Result<Self> {
let ctx = DecodeContext {
name: "mjpeg_mediacodec".to_string(),
width: resolution.width as i32,
height: resolution.height as i32,
sw_pixfmt: AVPixelFormat::AV_PIX_FMT_NV12,
thread_count: 1,
};
let decoder = Decoder::new(ctx).map_err(|_| {
AppError::VideoError("Failed to create FFmpeg mjpeg_mediacodec decoder".to_string())
})?;
Ok(Self {
decoder,
resolution,
nv12_converter: None,
last_output_format: None,
pending_frames: 0,
})
}
pub fn decode_to_nv12(&mut self, mjpeg: &[u8]) -> Result<Vec<u8>> {
let frames = match self.decoder.decode(mjpeg) {
Ok(frames) => frames,
Err(err) if err == -11 => {
self.pending_frames += 1;
if self.pending_frames <= 3 {
return Err(AppError::VideoError(
"mjpeg_mediacodec decode needs more input".to_string(),
));
}
return Err(AppError::VideoError(
"mjpeg_mediacodec decoder did not output after 3 frames".to_string(),
));
}
Err(err) => {
return Err(AppError::VideoError(format!(
"mjpeg_mediacodec decode failed: {err}"
)));
}
};
if frames.is_empty() {
self.pending_frames += 1;
if self.pending_frames <= 3 {
return Err(AppError::VideoError(
"mjpeg_mediacodec decode needs more input".to_string(),
));
}
return Err(AppError::VideoError(
"mjpeg_mediacodec decoder did not output after 3 frames".to_string(),
));
}
self.pending_frames = 0;
if frames.len() > 1 {
warn!(
"mjpeg_mediacodec decode returned {} frames, using last",
frames.len()
);
}
let frame = frames.pop().ok_or_else(|| {
AppError::VideoError("mjpeg_mediacodec decode returned empty".to_string())
})?;
if frame.width as u32 != self.resolution.width
|| frame.height as u32 != self.resolution.height
{
warn!(
"mjpeg_mediacodec output size {}x{} differs from expected {}x{}",
frame.width, frame.height, self.resolution.width, self.resolution.height
);
}
let output_format = pixel_format_from_av(frame.pixfmt).ok_or_else(|| {
AppError::VideoError(format!(
"mjpeg_mediacodec output pixfmt {:?} is not supported",
frame.pixfmt
))
})?;
if self.last_output_format != Some(output_format) {
info!("mjpeg_mediacodec output format: {}", output_format);
self.last_output_format = Some(output_format);
}
match output_format {
PixelFormat::Nv12 => Ok(frame.data),
PixelFormat::Nv21 => {
let converter = self
.nv12_converter
.get_or_insert_with(|| Nv12Converter::nv21_to_nv12(self.resolution));
Ok(converter.convert(&frame.data)?.to_vec())
}
PixelFormat::Yuv420 => {
let converter = self
.nv12_converter
.get_or_insert_with(|| Nv12Converter::yuv420_to_nv12(self.resolution));
Ok(converter.convert(&frame.data)?.to_vec())
}
other => Err(AppError::VideoError(format!(
"mjpeg_mediacodec output {} cannot be converted to NV12",
other
))),
}
}
}
fn pixel_format_from_av(format: AVPixelFormat) -> Option<PixelFormat> {
match format {
AVPixelFormat::AV_PIX_FMT_NV12 => Some(PixelFormat::Nv12),
AVPixelFormat::AV_PIX_FMT_NV21 => Some(PixelFormat::Nv21),
AVPixelFormat::AV_PIX_FMT_YUV420P | AVPixelFormat::AV_PIX_FMT_YUVJ420P => {
Some(PixelFormat::Yuv420)
}
_ => None,
}
}
unsafe impl Send for AndroidMediaCodecMjpegDecoder {}

View File

@@ -539,8 +539,46 @@ pub struct Nv12Converter {
resolution: Resolution,
/// Output buffer (reused across conversions)
output_buffer: Nv12Buffer,
/// Optional I420 buffer for intermediate conversions
i420_buffer: Option<Yuv420pBuffer>,
}
/// MJPEG decoder that writes NV12 directly using libyuv.
pub struct MjpegToNv12Decoder {
resolution: Resolution,
output_buffer: Nv12Buffer,
size_checked: bool,
}
impl MjpegToNv12Decoder {
pub fn new(resolution: Resolution) -> Self {
Self {
resolution,
output_buffer: Nv12Buffer::new(resolution),
size_checked: false,
}
}
pub fn decode(&mut self, input: &[u8]) -> Result<&[u8]> {
let width = self.resolution.width as i32;
let height = self.resolution.height as i32;
if !self.size_checked {
let (src_width, src_height) = libyuv::mjpg_size(input).map_err(|e| {
AppError::VideoError(format!("libyuv MJPEG header read failed: {}", e))
})?;
if src_width != width || src_height != height {
return Err(AppError::VideoError(format!(
"libyuv MJPEG size mismatch: {}x{} (expected {}x{})",
src_width, src_height, width, height
)));
}
self.size_checked = true;
}
libyuv::mjpg_to_nv12(input, self.output_buffer.as_bytes_mut(), width, height)
.map_err(|e| AppError::VideoError(format!("libyuv MJPEG->NV12 failed: {}", e)))?;
Ok(self.output_buffer.as_bytes())
}
}
impl Nv12Converter {
@@ -550,7 +588,6 @@ impl Nv12Converter {
src_format: PixelFormat::Bgr24,
resolution,
output_buffer: Nv12Buffer::new(resolution),
i420_buffer: None,
}
}
@@ -560,7 +597,6 @@ impl Nv12Converter {
src_format: PixelFormat::Rgb24,
resolution,
output_buffer: Nv12Buffer::new(resolution),
i420_buffer: None,
}
}
@@ -570,7 +606,6 @@ impl Nv12Converter {
src_format: PixelFormat::Yuyv,
resolution,
output_buffer: Nv12Buffer::new(resolution),
i420_buffer: None,
}
}
@@ -580,7 +615,6 @@ impl Nv12Converter {
src_format: PixelFormat::Yuv420,
resolution,
output_buffer: Nv12Buffer::new(resolution),
i420_buffer: None,
}
}
@@ -590,7 +624,6 @@ impl Nv12Converter {
src_format: PixelFormat::Nv21,
resolution,
output_buffer: Nv12Buffer::new(resolution),
i420_buffer: Some(Yuv420pBuffer::new(resolution)),
}
}
@@ -600,7 +633,6 @@ impl Nv12Converter {
src_format: PixelFormat::Nv16,
resolution,
output_buffer: Nv12Buffer::new(resolution),
i420_buffer: None,
}
}
@@ -610,7 +642,6 @@ impl Nv12Converter {
src_format: PixelFormat::Nv24,
resolution,
output_buffer: Nv12Buffer::new(resolution),
i420_buffer: None,
}
}
@@ -621,23 +652,6 @@ impl Nv12Converter {
// Handle formats that need custom conversion without holding dst borrow
match self.src_format {
PixelFormat::Nv21 => {
let mut i420 = self.i420_buffer.take().ok_or_else(|| {
AppError::VideoError("NV21 I420 buffer not initialized".to_string())
})?;
{
let dst = self.output_buffer.as_bytes_mut();
Self::convert_nv21_to_nv12_with_dims(
self.resolution.width as usize,
self.resolution.height as usize,
input,
dst,
&mut i420,
)?;
}
self.i420_buffer = Some(i420);
return Ok(self.output_buffer.as_bytes());
}
PixelFormat::Nv16 => {
let dst = self.output_buffer.as_bytes_mut();
Self::convert_nv16_to_nv12_with_dims(
@@ -667,6 +681,7 @@ impl Nv12Converter {
PixelFormat::Rgb24 => libyuv::rgb24_to_nv12(input, dst, width, height),
PixelFormat::Yuyv => libyuv::yuy2_to_nv12(input, dst, width, height),
PixelFormat::Yuv420 => libyuv::i420_to_nv12(input, dst, width, height),
PixelFormat::Nv21 => libyuv::nv21_to_nv12(input, dst, width, height),
_ => {
return Err(AppError::VideoError(format!(
"Unsupported conversion to NV12: {}",
@@ -680,21 +695,6 @@ impl Nv12Converter {
Ok(self.output_buffer.as_bytes())
}
fn convert_nv21_to_nv12_with_dims(
width: usize,
height: usize,
input: &[u8],
dst: &mut [u8],
yuv: &mut Yuv420pBuffer,
) -> Result<()> {
libyuv::nv21_to_i420(input, yuv.as_bytes_mut(), width as i32, height as i32)
.map_err(|e| AppError::VideoError(format!("libyuv NV21->I420 failed: {}", e)))?;
libyuv::i420_to_nv12(yuv.as_bytes(), dst, width as i32, height as i32)
.map_err(|e| AppError::VideoError(format!("libyuv I420->NV12 failed: {}", e)))?;
Ok(())
}
fn convert_nv16_to_nv12_with_dims(
width: usize,
height: usize,

View File

@@ -48,6 +48,8 @@ pub enum H264EncoderType {
Rkmpp,
/// V4L2 M2M (ARM generic) - requires hwcodec extension
V4l2M2m,
/// Android MediaCodec via FFmpeg
MediaCodec,
/// Software encoding (libx264/openh264)
Software,
/// No encoder available
@@ -64,6 +66,7 @@ impl std::fmt::Display for H264EncoderType {
H264EncoderType::Vaapi => write!(f, "VAAPI"),
H264EncoderType::Rkmpp => write!(f, "RKMPP"),
H264EncoderType::V4l2M2m => write!(f, "V4L2 M2M"),
H264EncoderType::MediaCodec => write!(f, "MediaCodec"),
H264EncoderType::Software => write!(f, "Software"),
H264EncoderType::None => write!(f, "None"),
}
@@ -80,6 +83,7 @@ impl From<EncoderBackend> for H264EncoderType {
EncoderBackend::Vaapi => H264EncoderType::Vaapi,
EncoderBackend::Rkmpp => H264EncoderType::Rkmpp,
EncoderBackend::V4l2m2m => H264EncoderType::V4l2M2m,
EncoderBackend::MediaCodec => H264EncoderType::MediaCodec,
EncoderBackend::Software => H264EncoderType::Software,
}
}
@@ -224,10 +228,10 @@ pub fn detect_best_encoder(width: u32, height: u32) -> (H264EncoderType, Option<
}
}
/// Encoded frame from hwcodec (cloned for ownership)
/// Encoded frame from hwcodec.
#[derive(Debug, Clone)]
pub struct HwEncodeFrame {
pub data: Vec<u8>,
pub data: Bytes,
pub pts: i64,
pub key: i32,
}
@@ -372,14 +376,12 @@ impl H264Encoder {
self.frame_count += 1;
match self.inner.encode(data, pts_ms) {
match self.inner.encode_bytes(data, pts_ms) {
Ok(frames) => {
// Zero-copy: drain frames from hwcodec buffer instead of cloning
// hwcodec returns &mut Vec, so we can take ownership via drain
let owned_frames: Vec<HwEncodeFrame> = frames
.drain(..)
.into_iter()
.map(|f| HwEncodeFrame {
data: f.data, // Move, not clone
data: f.data,
pts: f.pts,
key: f.key,
})

View File

@@ -45,6 +45,8 @@ pub enum H265EncoderType {
Rkmpp,
/// V4L2 M2M (ARM generic)
V4l2M2m,
/// Android MediaCodec via FFmpeg
MediaCodec,
/// Software encoder (libx265)
Software,
/// No encoder available
@@ -61,6 +63,7 @@ impl std::fmt::Display for H265EncoderType {
H265EncoderType::Vaapi => write!(f, "VAAPI"),
H265EncoderType::Rkmpp => write!(f, "RKMPP"),
H265EncoderType::V4l2M2m => write!(f, "V4L2 M2M"),
H265EncoderType::MediaCodec => write!(f, "MediaCodec"),
H265EncoderType::Software => write!(f, "Software"),
H265EncoderType::None => write!(f, "None"),
}
@@ -76,6 +79,7 @@ impl From<EncoderBackend> for H265EncoderType {
EncoderBackend::Vaapi => H265EncoderType::Vaapi,
EncoderBackend::Rkmpp => H265EncoderType::Rkmpp,
EncoderBackend::V4l2m2m => H265EncoderType::V4l2M2m,
EncoderBackend::MediaCodec => H265EncoderType::MediaCodec,
EncoderBackend::Software => H265EncoderType::Software,
}
}
@@ -243,10 +247,10 @@ pub fn is_h265_available() -> bool {
registry.is_codec_available(VideoEncoderType::H265)
}
/// Encoded frame from hwcodec (cloned for ownership)
/// Encoded frame from hwcodec.
#[derive(Debug, Clone)]
pub struct HwEncodeFrame {
pub data: Vec<u8>,
pub data: Bytes,
pub pts: i64,
pub key: i32,
}
@@ -465,13 +469,12 @@ impl H265Encoder {
);
}
match self.inner.encode(data, pts_ms) {
match self.inner.encode_bytes(data, pts_ms) {
Ok(frames) => {
// Zero-copy: drain frames from hwcodec buffer instead of cloning
let owned_frames: Vec<HwEncodeFrame> = frames
.drain(..)
.into_iter()
.map(|f| HwEncodeFrame {
data: f.data, // Move, not clone
data: f.data,
pts: f.pts,
key: f.key,
})

View File

@@ -1,54 +0,0 @@
//! MJPEG decoder using TurboJPEG (software) -> RGB24.
use turbojpeg::{Decompressor, Image, PixelFormat as TJPixelFormat};
use crate::error::{AppError, Result};
use crate::video::format::Resolution;
pub struct MjpegTurboDecoder {
decompressor: Decompressor,
resolution: Resolution,
}
impl MjpegTurboDecoder {
pub fn new(resolution: Resolution) -> Result<Self> {
let decompressor = Decompressor::new().map_err(|e| {
AppError::VideoError(format!("Failed to create turbojpeg decoder: {}", e))
})?;
Ok(Self {
decompressor,
resolution,
})
}
pub fn decode_to_rgb(&mut self, mjpeg: &[u8]) -> Result<Vec<u8>> {
let header = self
.decompressor
.read_header(mjpeg)
.map_err(|e| AppError::VideoError(format!("turbojpeg read_header failed: {}", e)))?;
if header.width as u32 != self.resolution.width
|| header.height as u32 != self.resolution.height
{
return Err(AppError::VideoError(format!(
"turbojpeg size mismatch: {}x{} (expected {}x{})",
header.width, header.height, self.resolution.width, self.resolution.height
)));
}
let pitch = header.width * 3;
let mut image = Image {
pixels: vec![0u8; header.height * pitch],
width: header.width,
pitch,
height: header.height,
format: TJPixelFormat::RGB,
};
self.decompressor
.decompress(mjpeg, image.as_deref_mut())
.map_err(|e| AppError::VideoError(format!("turbojpeg decode failed: {}", e)))?;
Ok(image.pixels)
}
}

View File

@@ -3,6 +3,10 @@
use hwcodec::common::DataFormat;
use hwcodec::ffmpeg_ram::CodecInfo;
#[cfg(feature = "android-mediacodec")]
pub mod android_mediacodec;
#[cfg(feature = "android-mediacodec")]
pub mod android_mjpeg;
pub mod convert;
pub mod h264;
@@ -16,16 +20,17 @@ pub mod video_codec;
pub mod vp8;
pub mod vp9;
pub mod mjpeg_turbo;
#[cfg(any(target_arch = "aarch64", target_arch = "arm"))]
#[cfg(all(feature = "desktop", any(target_arch = "aarch64", target_arch = "arm")))]
pub mod mjpeg_rkmpp;
pub use convert::{PixelConverter, Yuv420pBuffer};
#[cfg(feature = "android-mediacodec")]
pub use android_mediacodec::{AndroidH264Packet, AndroidMediaCodecH264Encoder};
#[cfg(feature = "android-mediacodec")]
pub use android_mjpeg::AndroidMediaCodecMjpegDecoder;
pub use convert::{MjpegToNv12Decoder, PixelConverter, Yuv420pBuffer};
pub use h264::{H264Config, H264Encoder, H264EncoderType, H264InputFormat};
pub use h265::{H265Config, H265Encoder, H265EncoderType, H265InputFormat};
pub use jpeg::JpegEncoder;
pub use mjpeg_turbo::MjpegTurboDecoder;
pub use registry::{AvailableEncoder, EncoderBackend, EncoderRegistry, VideoEncoderType};
pub use self_check::{
build_hardware_self_check_runtime_error, run_hardware_self_check, VideoEncoderSelfCheckCell,

View File

@@ -96,6 +96,8 @@ pub enum EncoderBackend {
Rkmpp,
/// V4L2 Memory-to-Memory (ARM)
V4l2m2m,
/// Android MediaCodec via FFmpeg
MediaCodec,
/// Software encoding (libx264, libx265, libvpx)
Software,
}
@@ -115,6 +117,8 @@ impl EncoderBackend {
EncoderBackend::Rkmpp
} else if name.contains("v4l2m2m") {
EncoderBackend::V4l2m2m
} else if name.contains("mediacodec") {
EncoderBackend::MediaCodec
} else {
EncoderBackend::Software
}
@@ -134,6 +138,7 @@ impl EncoderBackend {
EncoderBackend::Amf => "AMF",
EncoderBackend::Rkmpp => "RKMPP",
EncoderBackend::V4l2m2m => "V4L2 M2M",
EncoderBackend::MediaCodec => "MediaCodec",
EncoderBackend::Software => "Software",
}
}
@@ -148,6 +153,7 @@ impl EncoderBackend {
"amf" => Some(EncoderBackend::Amf),
"rkmpp" => Some(EncoderBackend::Rkmpp),
"v4l2m2m" | "v4l2" => Some(EncoderBackend::V4l2m2m),
"mediacodec" | "android-mediacodec" => Some(EncoderBackend::MediaCodec),
"software" | "cpu" => Some(EncoderBackend::Software),
_ => None,
}

View File

@@ -2,6 +2,8 @@ use serde::Serialize;
use std::sync::mpsc;
use std::time::{Duration, Instant};
#[cfg(feature = "android-mediacodec")]
use super::AndroidMediaCodecH264Encoder;
use super::{
EncoderRegistry, H264Config, H264Encoder, H265Config, H265Encoder, VP8Config, VP8Encoder,
VP9Config, VP9Encoder, VideoEncoderType,
@@ -235,6 +237,32 @@ fn run_smoke_test(
}
fn run_h264_smoke_test(resolution: Resolution, codec_name_ffmpeg: &str) -> Result<()> {
#[cfg(feature = "android-mediacodec")]
if codec_name_ffmpeg == "h264_mediacodec" {
let mut encoder = AndroidMediaCodecH264Encoder::new(
resolution,
PixelFormat::Nv12,
30,
bitrate_kbps_for_resolution(resolution),
)?;
encoder.request_keyframe();
let frame = build_nv12_test_frame(
resolution,
PixelFormat::Nv12.frame_size(resolution).unwrap_or(0),
);
for sequence in 0..SELF_CHECK_FRAME_ATTEMPTS {
let frames = encoder.encode_raw(&frame, pts_ms(sequence))?;
if frames.iter().any(|frame| !frame.data.is_empty()) {
return Ok(());
}
}
return Err(AppError::VideoError(
"Encoder produced no output after multiple frames".to_string(),
));
}
let mut encoder = H264Encoder::with_codec(
H264Config::low_latency(resolution, bitrate_kbps_for_resolution(resolution)),
codec_name_ffmpeg,

View File

@@ -898,6 +898,17 @@ pub fn enumerate_devices() -> Result<Vec<VideoDeviceInfo>> {
candidates.push(path);
}
if candidates.is_empty() {
let sysfs_entries = video_node_names("/sys/class/video4linux");
let dev_entries = video_node_names("/dev");
warn!(
"No video probe candidates after sysfs filter; /dev={:?}, /sys/class/video4linux={:?}",
dev_entries, sysfs_entries
);
} else {
debug!("Video probe candidates: {:?}", candidates);
}
collapse_rkcif_probe_candidates(&mut candidates);
// Second pass: probe the remaining candidates in parallel. Each probe
@@ -952,11 +963,35 @@ pub fn enumerate_devices() -> Result<Vec<VideoDeviceInfo>> {
// for a single MIPI CSI pipeline. Keep only the highest-priority node per
// (driver, bus_info) group so users see one device instead of ~11.
dedup_platform_subdevices(&mut devices);
devices.retain(|device| {
let hide = should_hide_android_platform_node(device);
if hide {
debug!(
"Hiding Android platform video node: {} ({}) {}",
device.name,
device.driver,
device.path.display()
);
}
!hide
});
info!("Found {} video capture devices", devices.len());
Ok(devices)
}
fn video_node_names(dir: &str) -> Vec<String> {
let mut names: Vec<String> = std::fs::read_dir(dir)
.ok()
.into_iter()
.flat_map(|entries| entries.filter_map(|entry| entry.ok()))
.filter_map(|entry| entry.file_name().to_str().map(str::to_owned))
.filter(|name| name.starts_with("video"))
.collect();
names.sort();
names
}
pub fn select_recovery_device(
devices: &[VideoDeviceInfo],
hint: &VideoDeviceRecoveryHint,
@@ -1020,6 +1055,33 @@ fn dedup_platform_subdevices(devices: &mut Vec<VideoDeviceInfo>) {
});
}
fn should_hide_android_platform_node(device: &VideoDeviceInfo) -> bool {
if !cfg!(feature = "android") {
return false;
}
let driver = device.driver.to_ascii_lowercase();
let name = device.name.to_ascii_lowercase();
let card = device.card.to_ascii_lowercase();
let usb_device = driver == "uvcvideo" || device.bus_info.starts_with("usb-");
let known_bridge =
driver.contains("rkcif") || driver.contains("rk_hdmirx") || driver.contains("tc358743");
if usb_device || known_bridge {
return false;
}
matches!(
driver.as_str(),
"ionvideo" | "amlvideo" | "amlvideo2" | "videosync"
) || matches!(
name.as_str(),
"ionvideo" | "amlvideo" | "amlvideo2" | "videosync"
) || matches!(
card.as_str(),
"ionvideo" | "amlvideo" | "amlvideo2" | "videosync"
)
}
/// rkcif registers many `/dev/video*` queues; probing all in parallel can
/// contend and time out. Keep one node per board (lowest `videoN`).
fn collapse_rkcif_probe_candidates(candidates: &mut Vec<PathBuf>) {
@@ -1123,6 +1185,20 @@ fn sysfs_maybe_capture(path: &Path) -> bool {
.to_lowercase();
let driver = extract_uevent_value(&uevent, "driver");
if cfg!(feature = "android") {
let platform_skip = ["ionvideo", "amlvideo", "amlvideo2", "videosync"];
let driver_skip = driver
.as_ref()
.is_some_and(|driver| platform_skip.iter().any(|hint| driver == hint));
if driver_skip || platform_skip.iter().any(|hint| sysfs_name == *hint) {
debug!(
"Skipping Android platform video node {:?}: {}",
path, sysfs_name
);
return false;
}
}
let mut maybe_capture = false;
let capture_hints = [
"capture",

View File

@@ -6,7 +6,10 @@ mod linux;
mod windows;
#[cfg(unix)]
pub use linux::*;
pub use linux::{
enumerate_devices, find_best_device, select_recovery_device, VideoDevice, VideoDeviceInfo,
VideoDeviceRecoveryHint,
};
#[cfg(windows)]
pub use windows::*;
@@ -33,3 +36,6 @@ pub(crate) fn is_rkcif_driver(driver: &str) -> bool {
pub(crate) fn is_csi_hdmi_bridge(device: &VideoDeviceInfo) -> bool {
is_rk_hdmirx_device(device) || is_rkcif_driver(&device.driver)
}
#[cfg(unix)]
pub(crate) use linux::parse_bridge_kind;

View File

@@ -8,13 +8,21 @@ pub mod codec_constraints;
pub mod device;
pub mod format;
pub mod frame;
#[cfg(any(feature = "android", feature = "desktop"))]
pub mod pipeline;
pub mod signal;
#[cfg(any(feature = "android", feature = "desktop"))]
pub mod stream_manager;
#[cfg(any(feature = "android", feature = "desktop"))]
pub mod streamer;
#[cfg(any(feature = "android", feature = "desktop"))]
pub mod traits;
#[cfg(any(feature = "android", feature = "desktop"))]
pub mod types;
pub use capture::{CaptureMeta, CaptureStream};
#[cfg(feature = "android-mediacodec")]
pub use codec::{AndroidH264Packet, AndroidMediaCodecH264Encoder};
pub use codec::{H264Encoder, H264EncoderType, JpegEncoder, PixelConverter, Yuv420pBuffer};
pub use device::{VideoDevice, VideoDeviceInfo};
pub use format::PixelFormat;

View File

@@ -1,14 +1,21 @@
use crate::error::{AppError, Result};
use crate::video::codec::convert::{Nv12Converter, PixelConverter};
use crate::video::codec::convert::{MjpegToNv12Decoder, Nv12Converter, PixelConverter};
use crate::video::codec::h264::{H264Config, H264Encoder, H264InputFormat};
use crate::video::codec::h265::{H265Config, H265Encoder, H265InputFormat};
use crate::video::codec::registry::{EncoderBackend, EncoderRegistry, VideoEncoderType};
use crate::video::codec::traits::EncoderConfig;
use crate::video::codec::vp8::{VP8Config, VP8Encoder};
use crate::video::codec::vp9::{VP9Config, VP9Encoder};
use crate::video::codec::MjpegTurboDecoder;
#[cfg(feature = "android-mediacodec")]
use crate::video::codec::AndroidMediaCodecH264Encoder;
#[cfg(feature = "android-mediacodec")]
use crate::video::codec::AndroidMediaCodecMjpegDecoder;
use crate::video::format::{PixelFormat, Resolution};
#[cfg(any(target_arch = "aarch64", target_arch = "arm"))]
use bytes::Bytes;
#[cfg(all(
any(target_arch = "aarch64", target_arch = "arm"),
not(target_os = "android")
))]
use hwcodec::ffmpeg_hw::{
last_error_message as ffmpeg_hw_last_error, HwMjpegH26xConfig, HwMjpegH26xPipeline,
};
@@ -22,9 +29,15 @@ pub(super) struct EncoderThreadState {
pub(super) nv12_converter: Option<Nv12Converter>,
pub(super) yuv420p_converter: Option<PixelConverter>,
pub(super) encoder_needs_yuv420p: bool,
#[cfg(any(target_arch = "aarch64", target_arch = "arm"))]
#[cfg(all(
any(target_arch = "aarch64", target_arch = "arm"),
not(target_os = "android")
))]
pub(super) ffmpeg_hw_pipeline: Option<HwMjpegH26xPipeline>,
#[cfg(any(target_arch = "aarch64", target_arch = "arm"))]
#[cfg(all(
any(target_arch = "aarch64", target_arch = "arm"),
not(target_os = "android")
))]
pub(super) ffmpeg_hw_enabled: bool,
pub(super) fps: u32,
pub(super) codec: VideoEncoderType,
@@ -39,7 +52,7 @@ pub(super) trait VideoEncoderTrait: Send {
}
pub(super) struct EncodedFrame {
pub(super) data: Vec<u8>,
pub(super) data: Bytes,
pub(super) key: i32,
}
@@ -70,6 +83,25 @@ impl VideoEncoderTrait for H264EncoderWrapper {
}
}
fn create_h264_encoder(
config: &SharedVideoPipelineConfig,
input_format: H264InputFormat,
codec_name: &str,
) -> Result<Box<dyn VideoEncoderTrait + Send>> {
let encoder = H264Encoder::with_codec(
H264Config {
base: EncoderConfig::h264(config.resolution, config.bitrate_kbps()),
bitrate_kbps: config.bitrate_kbps(),
gop_size: config.gop_size(),
fps: config.fps,
input_format,
},
codec_name,
)?;
info!("Created H264 encoder: {}", encoder.codec_name());
Ok(Box::new(H264EncoderWrapper(encoder)))
}
struct H265EncoderWrapper(H265Encoder);
impl VideoEncoderTrait for H265EncoderWrapper {
@@ -97,6 +129,35 @@ impl VideoEncoderTrait for H265EncoderWrapper {
}
}
#[cfg(feature = "android-mediacodec")]
struct AndroidMediaCodecH264EncoderWrapper(AndroidMediaCodecH264Encoder);
#[cfg(feature = "android-mediacodec")]
impl VideoEncoderTrait for AndroidMediaCodecH264EncoderWrapper {
fn encode_raw(&mut self, data: &[u8], pts_ms: i64) -> Result<Vec<EncodedFrame>> {
let frames = self.0.encode_raw(data, pts_ms)?;
Ok(frames
.into_iter()
.map(|f| EncodedFrame {
data: f.data,
key: if f.key_frame { 1 } else { 0 },
})
.collect())
}
fn set_bitrate(&mut self, bitrate_kbps: u32) -> Result<()> {
self.0.set_bitrate(bitrate_kbps)
}
fn codec_name(&self) -> &str {
self.0.codec_name()
}
fn request_keyframe(&mut self) {
self.0.request_keyframe()
}
}
struct VP8EncoderWrapper(VP8Encoder);
impl VideoEncoderTrait for VP8EncoderWrapper {
@@ -105,7 +166,7 @@ impl VideoEncoderTrait for VP8EncoderWrapper {
Ok(frames
.into_iter()
.map(|f| EncodedFrame {
data: f.data,
data: f.data.into(),
key: f.key,
})
.collect())
@@ -130,7 +191,7 @@ impl VideoEncoderTrait for VP9EncoderWrapper {
Ok(frames
.into_iter()
.map(|f| EncodedFrame {
data: f.data,
data: f.data.into(),
key: f.key,
})
.collect())
@@ -148,17 +209,100 @@ impl VideoEncoderTrait for VP9EncoderWrapper {
}
pub(super) enum MjpegDecoderKind {
Turbo(MjpegTurboDecoder),
#[cfg(feature = "android-mediacodec")]
AndroidMediaCodec {
decoder: AndroidMediaCodecMjpegDecoder,
fallback: Box<MjpegDecoderKind>,
fallback_active: bool,
output: Vec<u8>,
},
Libyuv {
decoder: MjpegToNv12Decoder,
},
}
impl MjpegDecoderKind {
pub(super) fn decode(&mut self, data: &[u8]) -> Result<Vec<u8>> {
pub(super) fn decode(&mut self, data: &[u8]) -> Result<&[u8]> {
match self {
MjpegDecoderKind::Turbo(decoder) => decoder.decode_to_rgb(data),
#[cfg(feature = "android-mediacodec")]
MjpegDecoderKind::AndroidMediaCodec {
decoder,
fallback,
fallback_active,
output,
} => {
if !*fallback_active {
match decoder.decode_to_nv12(data) {
Ok(decoded) => {
*output = decoded;
return Ok(output.as_slice());
}
Err(AppError::VideoError(message))
if message.contains("needs more input") =>
{
return Err(AppError::VideoError(message));
}
Err(err) => {
tracing::warn!(
"Android MediaCodec MJPEG decode failed; falling back to libyuv MJPEG->NV12: {}",
err
);
*fallback_active = true;
}
}
}
fallback.decode(data)
}
MjpegDecoderKind::Libyuv { decoder } => decoder.decode(data),
}
}
}
fn libyuv_mjpeg_decoder(resolution: Resolution) -> MjpegDecoderKind {
MjpegDecoderKind::Libyuv {
decoder: MjpegToNv12Decoder::new(resolution),
}
}
fn create_mjpeg_decoder(resolution: Resolution) -> Result<(MjpegDecoderKind, PixelFormat)> {
#[cfg(feature = "android-mediacodec")]
{
if std::env::var_os("ONE_KVM_ANDROID_MJPEG_MEDIACODEC").is_none() {
info!("MJPEG input detected, using libyuv decoder (MJPEG -> NV12)");
return Ok((libyuv_mjpeg_decoder(resolution), PixelFormat::Nv12));
}
info!("MJPEG input detected, trying Android MediaCodec decoder (MJPEG -> NV12)");
match AndroidMediaCodecMjpegDecoder::new(resolution) {
Ok(decoder) => {
info!("Using Android MediaCodec MJPEG decoder");
return Ok((
MjpegDecoderKind::AndroidMediaCodec {
decoder,
fallback: Box::new(libyuv_mjpeg_decoder(resolution)),
fallback_active: false,
output: Vec::with_capacity(
PixelFormat::Nv12
.frame_size(resolution)
.unwrap_or((resolution.width * resolution.height * 3 / 2) as usize),
),
},
PixelFormat::Nv12,
));
}
Err(err) => {
tracing::warn!(
"Android MediaCodec MJPEG decoder unavailable; using libyuv MJPEG->NV12: {}",
err
);
}
}
}
info!("MJPEG input detected, using libyuv decoder (MJPEG -> NV12)");
Ok((libyuv_mjpeg_decoder(resolution), PixelFormat::Nv12))
}
pub(super) fn build_encoder_state(
config: &SharedVideoPipelineConfig,
) -> Result<EncoderThreadState> {
@@ -256,9 +400,15 @@ pub(super) fn build_encoder_state(
}
};
#[cfg(any(target_arch = "aarch64", target_arch = "arm"))]
#[cfg(all(
any(target_arch = "aarch64", target_arch = "arm"),
not(target_os = "android")
))]
let is_rkmpp_encoder = selected_codec_name.contains("rkmpp");
#[cfg(any(target_arch = "aarch64", target_arch = "arm"))]
#[cfg(all(
any(target_arch = "aarch64", target_arch = "arm"),
not(target_os = "android")
))]
if needs_mjpeg_decode
&& is_rkmpp_encoder
&& matches!(
@@ -298,9 +448,15 @@ pub(super) fn build_encoder_state(
nv12_converter: None,
yuv420p_converter: None,
encoder_needs_yuv420p: false,
#[cfg(any(target_arch = "aarch64", target_arch = "arm"))]
#[cfg(all(
any(target_arch = "aarch64", target_arch = "arm"),
not(target_os = "android")
))]
ffmpeg_hw_pipeline: Some(pipeline),
#[cfg(any(target_arch = "aarch64", target_arch = "arm"))]
#[cfg(all(
any(target_arch = "aarch64", target_arch = "arm"),
not(target_os = "android")
))]
ffmpeg_hw_enabled: true,
fps: config.fps,
codec: config.output_codec,
@@ -309,16 +465,8 @@ pub(super) fn build_encoder_state(
}
let (mjpeg_decoder, pipeline_input_format) = if needs_mjpeg_decode {
info!(
"MJPEG input detected, using TurboJPEG decoder ({} -> RGB24)",
config.input_format
);
(
Some(MjpegDecoderKind::Turbo(MjpegTurboDecoder::new(
config.resolution,
)?)),
PixelFormat::Rgb24,
)
let (decoder, format) = create_mjpeg_decoder(config.resolution)?;
(Some(decoder), format)
} else {
(None, config.input_format)
};
@@ -347,18 +495,40 @@ pub(super) fn build_encoder_state(
);
}
let encoder = H264Encoder::with_codec(
H264Config {
base: EncoderConfig::h264(config.resolution, config.bitrate_kbps()),
bitrate_kbps: config.bitrate_kbps(),
gop_size: config.gop_size(),
fps: config.fps,
input_format,
},
&codec_name,
)?;
info!("Created H264 encoder: {}", encoder.codec_name());
Box::new(H264EncoderWrapper(encoder))
#[cfg(feature = "android-mediacodec")]
{
if codec_name == "h264_mediacodec" {
info!(
"Creating Android MediaCodec H264 encoder for {:?} input",
input_format
);
let pixel_format = match input_format {
H264InputFormat::Nv12 => PixelFormat::Nv12,
H264InputFormat::Yuv420p => PixelFormat::Yuv420,
other => {
return Err(AppError::VideoError(format!(
"Android MediaCodec H264 does not support {:?} direct input",
other
)));
}
};
let encoder = AndroidMediaCodecH264Encoder::new(
config.resolution,
pixel_format,
config.fps,
config.bitrate_kbps(),
)?;
info!("Created Android MediaCodec H264 encoder");
Box::new(AndroidMediaCodecH264EncoderWrapper(encoder))
} else {
create_h264_encoder(config, input_format, &codec_name)?
}
}
#[cfg(not(feature = "android-mediacodec"))]
{
create_h264_encoder(config, input_format, &codec_name)?
}
}
VideoEncoderType::H265 => {
let codec_name = selected_codec_name.clone();
@@ -452,6 +622,11 @@ pub(super) fn build_encoder_state(
pipeline_input_format,
PixelFormat::Nv12 | PixelFormat::Nv16 | PixelFormat::Nv21 | PixelFormat::Yuv420
)
} else if codec_name.contains("mediacodec") {
matches!(
pipeline_input_format,
PixelFormat::Nv12 | PixelFormat::Yuv420
)
} else {
false
};
@@ -501,9 +676,15 @@ pub(super) fn build_encoder_state(
nv12_converter,
yuv420p_converter,
encoder_needs_yuv420p: needs_yuv420p,
#[cfg(any(target_arch = "aarch64", target_arch = "arm"))]
#[cfg(all(
any(target_arch = "aarch64", target_arch = "arm"),
not(target_os = "android")
))]
ffmpeg_hw_pipeline: None,
#[cfg(any(target_arch = "aarch64", target_arch = "arm"))]
#[cfg(all(
any(target_arch = "aarch64", target_arch = "arm"),
not(target_os = "android")
))]
ffmpeg_hw_enabled: false,
fps: config.fps,
codec: config.output_codec,
@@ -527,6 +708,12 @@ fn h264_direct_input_format(
PixelFormat::Nv24 => Some(H264InputFormat::Nv24),
_ => None,
}
} else if codec_name.contains("mediacodec") {
match input_format {
PixelFormat::Nv12 => Some(H264InputFormat::Nv12),
PixelFormat::Yuv420 => Some(H264InputFormat::Yuv420p),
_ => None,
}
} else if codec_name.contains("libx264") {
match input_format {
PixelFormat::Nv12 => Some(H264InputFormat::Nv12),

View File

@@ -38,6 +38,7 @@ const CSI_BRIDGE_NOSIGNAL_INTERVAL_MS: u64 = 500;
const NOSIGNAL_POLL_MAX: Duration = Duration::from_secs(20);
/// Throttle repeated encoding errors to avoid log flooding
const ENCODE_ERROR_THROTTLE_SECS: u64 = 5;
const INVALID_MJPEG_LOG_THROTTLE_SECS: u64 = 5;
static PROCESS_START: std::sync::OnceLock<Instant> = std::sync::OnceLock::new();
@@ -60,7 +61,10 @@ use crate::video::frame::{FrameBuffer, FrameBufferPool, VideoFrame};
use crate::video::signal::SignalStatus;
const MIN_CAPTURE_FRAME_SIZE: usize = 128;
#[cfg(any(target_arch = "aarch64", target_arch = "arm"))]
#[cfg(all(
any(target_arch = "aarch64", target_arch = "arm"),
not(target_os = "android")
))]
use hwcodec::ffmpeg_hw::last_error_message as ffmpeg_hw_last_error;
/// Encoded video frame for distribution
@@ -480,9 +484,15 @@ impl SharedVideoPipeline {
fn apply_cmd(&self, state: &mut EncoderThreadState, cmd: PipelineCmd) -> Result<()> {
match cmd {
PipelineCmd::SetBitrate { bitrate_kbps, gop } => {
#[cfg(not(any(target_arch = "aarch64", target_arch = "arm")))]
#[cfg(any(
not(any(target_arch = "aarch64", target_arch = "arm")),
target_os = "android"
))]
let _ = gop;
#[cfg(any(target_arch = "aarch64", target_arch = "arm"))]
#[cfg(all(
any(target_arch = "aarch64", target_arch = "arm"),
not(target_os = "android")
))]
if state.ffmpeg_hw_enabled {
if let Some(ref mut pipeline) = state.ffmpeg_hw_pipeline {
pipeline
@@ -649,12 +659,14 @@ impl SharedVideoPipeline {
*guard = Some(cmd_tx);
}
// Encoder loop (runs on tokio, consumes latest frame)
// Encoder loop uses a dedicated OS thread because FFmpeg/MediaCodec work is synchronous.
{
let pipeline = pipeline.clone();
let latest_frame = latest_frame.clone();
tokio::spawn(async move {
let mut frame_count: u64 = 0;
let handle = tokio::runtime::Handle::current();
std::thread::spawn(move || {
let mut input_frame_count: u64 = 0;
let mut encoded_frame_count: u64 = 0;
let mut last_fps_time = Instant::now();
let mut fps_frame_count: u64 = 0;
let mut last_seq = *frame_seq_rx.borrow();
@@ -662,7 +674,7 @@ impl SharedVideoPipeline {
let mut suppressed_encode_errors: HashMap<String, u64> = HashMap::new();
while pipeline.running_flag.load(Ordering::Acquire) {
if frame_seq_rx.changed().await.is_err() {
if handle.block_on(frame_seq_rx.changed()).is_err() {
break;
}
if !pipeline.running_flag.load(Ordering::Acquire) {
@@ -694,15 +706,19 @@ impl SharedVideoPipeline {
None => continue,
};
match pipeline.encode_frame_sync(&mut encoder_state, &frame, frame_count) {
Ok(Some(encoded_frame)) => {
let encoded_arc = Arc::new(encoded_frame);
pipeline.broadcast_encoded(encoded_arc).await;
input_frame_count = input_frame_count.wrapping_add(1);
frame_count += 1;
fps_frame_count += 1;
match pipeline.encode_frame_sync(&mut encoder_state, &frame, input_frame_count)
{
Ok(encoded_frames) => {
for encoded_frame in encoded_frames {
let encoded_arc = Arc::new(encoded_frame);
handle.block_on(pipeline.broadcast_encoded(encoded_arc));
encoded_frame_count = encoded_frame_count.wrapping_add(1);
fps_frame_count += 1;
}
}
Ok(None) => {}
Err(e) => {
log_encoding_error(
&encode_error_throttler,
@@ -718,8 +734,15 @@ impl SharedVideoPipeline {
fps_frame_count = 0;
last_fps_time = Instant::now();
let mut s = pipeline.stats.lock().await;
s.current_fps = current_fps;
handle.block_on(async {
let mut s = pipeline.stats.lock().await;
s.current_fps = current_fps;
});
trace!(
"Shared pipeline processed {} input frames, emitted {} encoded frames",
input_frame_count,
encoded_frame_count
);
}
}
@@ -847,6 +870,8 @@ impl SharedVideoPipeline {
let mut sequence: u64 = 0;
let mut consecutive_timeouts: u32 = 0;
let capture_error_throttler = LogThrottler::with_secs(5);
let invalid_mjpeg_throttler =
LogThrottler::with_secs(INVALID_MJPEG_LOG_THROTTLE_SECS);
let mut suppressed_capture_errors: HashMap<String, u64> = HashMap::new();
while pipeline.running_flag.load(Ordering::Acquire) {
@@ -1207,6 +1232,20 @@ impl SharedVideoPipeline {
}
owned.truncate(frame_size);
if pixel_format.is_compressed() && !VideoFrame::is_valid_jpeg_bytes(&owned) {
if invalid_mjpeg_throttler.should_log("invalid_mjpeg_capture_frame") {
let b0 = owned.first().copied().unwrap_or_default();
let b1 = owned.get(1).copied().unwrap_or_default();
warn!(
"Dropping invalid MJPEG capture frame: size={}, starts with 0x{:02x} 0x{:02x}",
owned.len(),
b0,
b1
);
}
continue;
}
// Notify streaming only after frame validation passes —
// stale/warm-up frames from V4L2 kernel queues can cause
// DQBUF Ok with invalid data, which would prematurely
@@ -1244,7 +1283,7 @@ impl SharedVideoPipeline {
state: &mut EncoderThreadState,
frame: &VideoFrame,
frame_count: u64,
) -> Result<Option<EncodedVideoFrame>> {
) -> Result<Vec<EncodedVideoFrame>> {
let fps = state.fps;
let codec = state.codec;
let input_format = state.input_format;
@@ -1268,7 +1307,10 @@ impl SharedVideoPipeline {
current_ts_us.saturating_sub(start_ts_us) / 1000
};
#[cfg(any(target_arch = "aarch64", target_arch = "arm"))]
#[cfg(all(
any(target_arch = "aarch64", target_arch = "arm"),
not(target_os = "android")
))]
if state.ffmpeg_hw_enabled {
if input_format != PixelFormat::Mjpeg {
return Err(AppError::VideoError(
@@ -1295,17 +1337,17 @@ impl SharedVideoPipeline {
if let Some((data, is_keyframe)) = packet {
let sequence = self.sequence.fetch_add(1, Ordering::Relaxed) + 1;
return Ok(Some(EncodedVideoFrame {
return Ok(vec![EncodedVideoFrame {
data: Bytes::from(data),
pts_ms,
is_keyframe,
sequence,
duration: Duration::from_millis(1000 / fps as u64),
codec,
}));
}]);
}
return Ok(None);
return Ok(Vec::new());
}
let decoded_buf = if input_format.is_compressed() {
@@ -1313,12 +1355,26 @@ impl SharedVideoPipeline {
.mjpeg_decoder
.as_mut()
.ok_or_else(|| AppError::VideoError("MJPEG decoder not initialized".to_string()))?;
let decoded = decoder.decode(raw_frame)?;
let decoded = match decoder.decode(raw_frame) {
Ok(decoded) => decoded,
Err(err) => {
warn!("Dropping undecodable MJPEG frame before encode: {}", err);
return Ok(Vec::new());
}
};
Some(decoded)
} else {
None
};
let raw_frame = decoded_buf.as_deref().unwrap_or(raw_frame);
let compacted_buf = if decoded_buf.is_none() {
compact_strided_frame_for_encoder(frame, raw_frame)?
} else {
None
};
let raw_frame = decoded_buf
.as_deref()
.or(compacted_buf.as_deref())
.unwrap_or(raw_frame);
// Debug log for H265
if codec == VideoEncoderType::H265 && frame_count % 30 == 1 {
@@ -1365,8 +1421,24 @@ impl SharedVideoPipeline {
match encode_result {
Ok(frames) => {
if !frames.is_empty() {
let encoded = frames.into_iter().next().unwrap();
if frames.is_empty() {
if codec == VideoEncoderType::H265 {
warn!(
"[Pipeline-H265] Encoder returned no frames for frame #{}",
frame_count
);
} else {
trace!(
"Encoder returned no frames for input frame #{} ({})",
frame_count,
codec
);
}
return Ok(Vec::new());
}
let mut encoded_frames = Vec::with_capacity(frames.len());
for encoded in frames {
let is_keyframe = encoded.key == 1;
let sequence = self.sequence.fetch_add(1, Ordering::Relaxed) + 1;
if codec == VideoEncoderType::H264 {
@@ -1390,23 +1462,17 @@ impl SharedVideoPipeline {
}
}
Ok(Some(EncodedVideoFrame {
data: Bytes::from(encoded.data),
encoded_frames.push(EncodedVideoFrame {
data: encoded.data,
pts_ms,
is_keyframe,
sequence,
duration: Duration::from_millis(1000 / fps as u64),
codec,
}))
} else {
if codec == VideoEncoderType::H265 {
warn!(
"[Pipeline-H265] Encoder returned no frames for frame #{}",
frame_count
);
}
Ok(None)
});
}
Ok(encoded_frames)
}
Err(e) => {
if codec == VideoEncoderType::H265 {
@@ -1490,6 +1556,174 @@ impl SharedVideoPipeline {
}
}
fn compact_strided_frame_for_encoder(frame: &VideoFrame, data: &[u8]) -> Result<Option<Vec<u8>>> {
let width = frame.resolution.width as usize;
let height = frame.resolution.height as usize;
let stride = frame.stride as usize;
if width == 0 || height == 0 || stride == 0 || frame.format.is_compressed() {
return Ok(None);
}
let compact_size = match frame.format {
PixelFormat::Nv12 | PixelFormat::Nv21 | PixelFormat::Yuv420 | PixelFormat::Yvu420 => {
width * height * 3 / 2
}
PixelFormat::Nv16 | PixelFormat::Yuyv | PixelFormat::Yvyu | PixelFormat::Uyvy => {
width * height * 2
}
PixelFormat::Nv24 | PixelFormat::Rgb24 | PixelFormat::Bgr24 => width * height * 3,
PixelFormat::Rgb565 => width * height * 2,
PixelFormat::Grey => width * height,
PixelFormat::Mjpeg | PixelFormat::Jpeg => return Ok(None),
};
if data.len() == compact_size {
return Ok(None);
}
let mut out = vec![0u8; compact_size];
match frame.format {
PixelFormat::Nv12 | PixelFormat::Nv21 => {
let src_y_size = stride * height;
let src_uv_size = stride * height / 2;
require_len(data, src_y_size + src_uv_size, frame.format, stride)?;
copy_rows(data, 0, stride, &mut out, 0, width, width, height);
copy_rows(
data,
src_y_size,
stride,
&mut out,
width * height,
width,
width,
height / 2,
);
}
PixelFormat::Yuv420 | PixelFormat::Yvu420 => {
let src_y_size = stride * height;
let src_chroma_stride = stride / 2;
let src_chroma_size = src_chroma_stride * height / 2;
let dst_y_size = width * height;
let dst_chroma_stride = width / 2;
let dst_chroma_size = dst_chroma_stride * height / 2;
require_len(data, src_y_size + src_chroma_size * 2, frame.format, stride)?;
copy_rows(data, 0, stride, &mut out, 0, width, width, height);
copy_rows(
data,
src_y_size,
src_chroma_stride,
&mut out,
dst_y_size,
dst_chroma_stride,
dst_chroma_stride,
height / 2,
);
copy_rows(
data,
src_y_size + src_chroma_size,
src_chroma_stride,
&mut out,
dst_y_size + dst_chroma_size,
dst_chroma_stride,
dst_chroma_stride,
height / 2,
);
}
PixelFormat::Nv16 => {
let src_y_size = stride * height;
require_len(data, src_y_size + stride * height, frame.format, stride)?;
copy_rows(data, 0, stride, &mut out, 0, width, width, height);
copy_rows(
data,
src_y_size,
stride,
&mut out,
width * height,
width,
width,
height,
);
}
PixelFormat::Nv24 => {
let src_y_size = stride * height;
let src_uv_stride = stride * 2;
require_len(
data,
src_y_size + src_uv_stride * height,
frame.format,
stride,
)?;
copy_rows(data, 0, stride, &mut out, 0, width, width, height);
copy_rows(
data,
src_y_size,
src_uv_stride,
&mut out,
width * height,
width * 2,
width * 2,
height,
);
}
PixelFormat::Yuyv | PixelFormat::Yvyu | PixelFormat::Uyvy | PixelFormat::Rgb565 => {
let row_bytes = width * 2;
require_len(data, stride * height, frame.format, stride)?;
copy_rows(data, 0, stride, &mut out, 0, row_bytes, row_bytes, height);
}
PixelFormat::Rgb24 | PixelFormat::Bgr24 => {
let row_bytes = width * 3;
require_len(data, stride * height, frame.format, stride)?;
copy_rows(data, 0, stride, &mut out, 0, row_bytes, row_bytes, height);
}
PixelFormat::Grey => {
require_len(data, stride * height, frame.format, stride)?;
copy_rows(data, 0, stride, &mut out, 0, width, width, height);
}
PixelFormat::Mjpeg | PixelFormat::Jpeg => return Ok(None),
}
trace!(
"Compacted strided {} frame for encoder: {} -> {} bytes (stride={}, width={})",
frame.format,
data.len(),
out.len(),
stride,
width
);
Ok(Some(out))
}
fn require_len(data: &[u8], required: usize, format: PixelFormat, stride: usize) -> Result<()> {
if data.len() < required {
return Err(AppError::VideoError(format!(
"{} frame too small for stride compaction: {} < {} (stride={})",
format,
data.len(),
required,
stride
)));
}
Ok(())
}
fn copy_rows(
src: &[u8],
src_offset: usize,
src_stride: usize,
dst: &mut [u8],
dst_offset: usize,
dst_stride: usize,
row_bytes: usize,
rows: usize,
) {
for row in 0..rows {
let src_start = src_offset + row * src_stride;
let dst_start = dst_offset + row * dst_stride;
dst[dst_start..dst_start + row_bytes]
.copy_from_slice(&src[src_start..src_start + row_bytes]);
}
}
impl Drop for SharedVideoPipeline {
fn drop(&mut self) {
let _ = self.running.send(false);