perf(video): 优化视频流水线并统一使用 vendored OpenSSL

使用固定采集、解码和编码线程及同步最新帧槽,减少同步异步跨界和无效 MJPEG 解码。

编码前预留消费者容量,以完整批次投递编码结果并隔离慢客户端,避免丢失 H.264 参考帧;保留 PTS 修正,不增加 IDR 节流。

复用 Annex-B 缓冲并增加低频发送分项统计,默认启用 vendored OpenSSL SRTP AES-CM 后端。

HTTP 客户端、HTTPS 服务端和本地 DTLS 证书验证迁移至 OpenSSL,移除 rustls TLS 引擎依赖,保留证书验证、媒体加密和原有协议协商。
This commit is contained in:
mofeng-git
2026-10-01 18:34:21 +08:00
parent 48cd06c7a4
commit 318b58b26a
115 changed files with 18160 additions and 273 deletions

View File

@@ -69,63 +69,54 @@ pub fn strip_aud_nal_units(data: &[u8]) -> Vec<u8> {
result
}
pub fn extract_sps_pps(data: &[u8]) -> (Option<Vec<u8>>, Option<Vec<u8>>) {
let mut sps: Option<Vec<u8>> = None;
let mut pps: Option<Vec<u8>> = None;
let mut i = 0;
#[derive(Debug, Default)]
pub struct H264AccessUnit<'data> {
pub sps: Option<&'data [u8]>,
pub pps: Option<&'data [u8]>,
pub is_idr: bool,
}
while i < data.len() {
let start_code_len = if i + 4 <= data.len()
&& data[i] == 0
&& data[i + 1] == 0
&& data[i + 2] == 0
&& data[i + 3] == 1
{
4
} else if i + 3 <= data.len() && data[i] == 0 && data[i + 1] == 0 && data[i + 2] == 1 {
3
} else {
i += 1;
continue;
};
let nal_start = i + start_code_len;
if nal_start >= data.len() {
break;
}
let nal_type = data[nal_start] & 0x1F;
let mut nal_end = data.len();
let mut j = nal_start + 1;
while j + 3 <= data.len() {
if (data[j] == 0 && data[j + 1] == 0 && data[j + 2] == 1)
|| (j + 4 <= data.len()
&& data[j] == 0
&& data[j + 1] == 0
&& data[j + 2] == 0
&& data[j + 3] == 1)
{
nal_end = j;
break;
fn next_start_code(data: &[u8], mut offset: usize) -> Option<(usize, usize)> {
while offset + 3 <= data.len() {
if data[offset] == 0 && data[offset + 1] == 0 {
if data[offset + 2] == 1 {
return Some((offset, 3));
}
j += 1;
}
match nal_type {
7 => {
sps = Some(data[nal_start..nal_end].to_vec());
if offset + 4 <= data.len() && data[offset + 2] == 0 && data[offset + 3] == 1 {
return Some((offset, 4));
}
8 => {
pps = Some(data[nal_start..nal_end].to_vec());
}
_ => {}
}
i = nal_end;
offset += 1;
}
None
}
(sps, pps)
pub fn inspect_annex_b(data: &[u8]) -> H264AccessUnit<'_> {
let mut inspection = H264AccessUnit::default();
let mut start_code = next_start_code(data, 0);
while let Some((offset, prefix_length)) = start_code {
let nal_start = offset + prefix_length;
let next = next_start_code(data, nal_start);
let nal_end = next.map_or(data.len(), |(next_offset, _)| next_offset);
if nal_start < nal_end {
match data[nal_start] & 0x1f {
5 => inspection.is_idr = true,
7 => inspection.sps = Some(&data[nal_start..nal_end]),
8 => inspection.pps = Some(&data[nal_start..nal_end]),
_ => {}
}
}
start_code = next;
}
inspection
}
pub fn extract_sps_pps(data: &[u8]) -> (Option<Vec<u8>>, Option<Vec<u8>>) {
let inspection = inspect_annex_b(data);
(
inspection.sps.map(<[u8]>::to_vec),
inspection.pps.map(<[u8]>::to_vec),
)
}
pub fn has_sps_pps(data: &[u8]) -> bool {
@@ -215,8 +206,9 @@ pub fn parse_profile_level_id_from_sps(sps: &[u8]) -> Option<String> {
}
pub fn extract_profile_level_id(data: &[u8]) -> Option<String> {
let (sps, _) = extract_sps_pps(data);
sps.and_then(|sps_data| parse_profile_level_id_from_sps(&sps_data))
inspect_annex_b(data)
.sps
.and_then(parse_profile_level_id_from_sps)
}
pub fn is_annex_b(data: &[u8]) -> bool {
@@ -292,6 +284,58 @@ pub fn normalize_for_webrtc(data: &[u8]) -> Vec<u8> {
mod tests {
use super::*;
#[test]
fn inspects_parameter_sets_and_idr_in_one_pass() {
let data = [
0, 0, 0, 1, 0x67, 0x42, 0xe0, 0x1f, 0, 0, 1, 0x68, 0xce, 0, 0, 0, 1, 0x65, 0x88,
];
let inspection = inspect_annex_b(&data);
assert_eq!(inspection.sps, Some(&data[4..8]));
assert_eq!(inspection.pps, Some(&data[11..13]));
assert!(inspection.is_idr);
assert!(std::ptr::eq(
inspection.sps.unwrap().as_ptr(),
data[4..].as_ptr()
));
assert_eq!(extract_profile_level_id(&data).as_deref(), Some("42e01f"));
}
#[test]
fn predicted_packet_has_no_parameter_sets_or_idr() {
let data = [0, 0, 0, 1, 0x41, 0xc0];
let inspection = inspect_annex_b(&data);
assert!(inspection.sps.is_none());
assert!(inspection.pps.is_none());
assert!(!inspection.is_idr);
}
#[test]
fn empty_and_truncated_nal_units_are_ignored() {
for data in [
&[][..],
&[0][..],
&[0, 0][..],
&[0, 0, 1][..],
&[0, 0, 0, 1][..],
&[0, 0, 1, 0, 0, 0, 1][..],
] {
let inspection = inspect_annex_b(data);
assert!(inspection.sps.is_none());
assert!(inspection.pps.is_none());
assert!(!inspection.is_idr);
}
}
#[test]
fn inspection_keeps_last_parameter_sets() {
let data = [
0, 0, 1, 0x67, 0x42, 0xc0, 0x1e, 0, 0, 0, 1, 0x67, 0x42, 0xe0, 0x1f,
];
let inspection = inspect_annex_b(&data);
assert_eq!(inspection.sps, Some(&data[11..15]));
assert_eq!(extract_profile_level_id(&data).as_deref(), Some("42e01f"));
}
#[test]
fn detects_h264_keyframes() {
let idr_frame = vec![0x00, 0x00, 0x00, 0x01, 0x65];

View File

@@ -0,0 +1,100 @@
use std::collections::VecDeque;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::Arc;
use std::time::Duration;
use parking_lot::Mutex;
use tokio::sync::mpsc;
use super::frame_mailbox::FrameMailbox;
use super::EncodedVideoFrame;
use crate::video::frame::VideoFrame;
pub(super) type EncodedBatch = Arc<[Arc<EncodedVideoFrame>]>;
pub(super) type FrameMailboxHandle = Arc<Mutex<Option<Arc<FrameMailbox<Arc<VideoFrame>>>>>>;
#[derive(Default)]
pub(super) struct DeliveryStats {
pub sent_frames: AtomicU64,
pub send_time_ns: AtomicU64,
pub send_errors: AtomicU64,
}
pub(super) struct VideoSubscriber {
pub sender: mpsc::Sender<EncodedBatch>,
pub needs_keyframe: AtomicBool,
pub recovery_requested: AtomicBool,
}
pub(super) struct VideoFrameReservation {
pub subscriber: Arc<VideoSubscriber>,
pub permit: mpsc::OwnedPermit<EncodedBatch>,
}
pub struct EncodedVideoFrameReceiver {
receiver: mpsc::Receiver<EncodedBatch>,
pending: VecDeque<Arc<EncodedVideoFrame>>,
mailbox: FrameMailboxHandle,
stats: Arc<DeliveryStats>,
}
impl EncodedVideoFrameReceiver {
pub(super) fn new(
receiver: mpsc::Receiver<EncodedBatch>,
mailbox: FrameMailboxHandle,
stats: Arc<DeliveryStats>,
) -> Self {
Self {
receiver,
pending: VecDeque::new(),
mailbox,
stats,
}
}
fn notify_capacity(&self) {
if let Some(mailbox) = self.mailbox.lock().as_ref() {
mailbox.notify();
}
}
pub async fn recv(&mut self) -> Option<Arc<EncodedVideoFrame>> {
loop {
if let Some(frame) = self.pending.pop_front() {
return Some(frame);
}
let batch = self.receiver.recv().await?;
self.pending.extend(batch.iter().cloned());
self.notify_capacity();
}
}
pub fn try_recv(&mut self) -> Result<Arc<EncodedVideoFrame>, mpsc::error::TryRecvError> {
loop {
if let Some(frame) = self.pending.pop_front() {
return Ok(frame);
}
let batch = self.receiver.try_recv()?;
self.pending.extend(batch.iter().cloned());
self.notify_capacity();
}
}
pub(crate) fn record_send(&self, elapsed: Duration, succeeded: bool) {
self.stats
.send_time_ns
.fetch_add(elapsed.as_nanos() as u64, Ordering::Relaxed);
if succeeded {
self.stats.sent_frames.fetch_add(1, Ordering::Relaxed);
} else {
self.stats.send_errors.fetch_add(1, Ordering::Relaxed);
}
}
}
impl Drop for EncodedVideoFrameReceiver {
fn drop(&mut self) {
self.receiver.close();
self.notify_capacity();
}
}

View File

@@ -256,60 +256,80 @@ fn run(
}
let capture = active.as_mut().expect("opened capture");
let result = match capture.encoder.as_mut().expect("active encoder") {
CaptureEncoder::Dma(encoder) => {
let pts = pipeline.pts_ms();
capture
let reservations = pipeline.reserve_outputs();
let result = if reservations.as_ref().is_none_or(Vec::is_empty) {
match capture.encoder.as_mut().expect("active encoder") {
CaptureEncoder::Dma(_) => capture
.stream
.with_next_dmabuf(|index, bytes_used, fresh_fd| {
let keyframe = pipeline.keyframe_requested.swap(false, Ordering::AcqRel);
// The callback holds the dequeue lease until native encode
// completes (or destroys MPP on error), before QBUF.
unsafe { encoder.encode(index, bytes_used, fresh_fd, pts, keyframe) }
})
.map(|(_, packet)| {
packet
.map(|packet| {
let data = Bytes::from(packet);
let is_keyframe = match config.output_codec {
VideoEncoderType::H264 => h264_bitstream::is_keyframe(&data),
VideoEncoderType::H265 => h265_bitstream::is_keyframe(&data),
_ => false,
};
let (data, is_keyframe) = pipeline.inspect_and_parameterize_packet(
config.output_codec,
data,
is_keyframe,
);
if config.output_codec == VideoEncoderType::H264 {
pipeline.update_h264_profile_level_id(&data);
}
vec![EncodedVideoFrame {
data,
pts_ms: pts,
is_keyframe,
sequence: pipeline.sequence.fetch_add(1, Ordering::Relaxed) + 1,
duration: Duration::from_micros(
1_000_000 / config.fps.max(1) as u64,
),
codec: config.output_codec,
}]
})
.map_err(AppError::VideoError)
})
.with_next_dmabuf(|_, _, _| ())
.map(|_| Ok(Vec::new())),
CaptureEncoder::Copy(_) => {
let mut raw = buffer_pool.take(0);
let result = capture.stream.next_into(&mut raw).map(|_| Ok(Vec::new()));
buffer_pool.put(raw);
result
}
}
CaptureEncoder::Copy(encoder) => {
let mut raw = buffer_pool.take(0);
capture.stream.next_into(&mut raw).map(|meta| {
let frame = VideoFrame::from_pooled(
Arc::new(FrameBuffer::new(raw, Some(buffer_pool.clone()))),
config.resolution,
config.input_format,
capture.stream.stride(),
meta.sequence,
);
pipeline.encode_frame_sync(encoder, &frame)
})
} else {
match capture.encoder.as_mut().expect("active encoder") {
CaptureEncoder::Dma(encoder) => {
let pts = pipeline.pts_ms();
capture
.stream
.with_next_dmabuf(|index, bytes_used, fresh_fd| {
let keyframe =
pipeline.keyframe_requested.swap(false, Ordering::AcqRel);
// The callback holds the dequeue lease until native encode
// completes (or destroys MPP on error), before QBUF.
unsafe { encoder.encode(index, bytes_used, fresh_fd, pts, keyframe) }
})
.map(|(_, packet)| {
packet
.map(|packet| {
let data = Bytes::from(packet);
let is_keyframe = match config.output_codec {
VideoEncoderType::H264 => {
h264_bitstream::is_keyframe(&data)
}
VideoEncoderType::H265 => {
h265_bitstream::is_keyframe(&data)
}
_ => false,
};
let (data, is_keyframe) = pipeline
.inspect_and_parameterize_packet(
config.output_codec,
data,
is_keyframe,
);
vec![EncodedVideoFrame {
data,
pts_ms: pts,
is_keyframe,
sequence: pipeline.sequence.fetch_add(1, Ordering::Relaxed)
+ 1,
duration: Duration::from_micros(
1_000_000 / config.fps.max(1) as u64,
),
codec: config.output_codec,
}]
})
.map_err(AppError::VideoError)
})
}
CaptureEncoder::Copy(encoder) => {
let mut raw = buffer_pool.take(0);
capture.stream.next_into(&mut raw).map(|meta| {
let frame = VideoFrame::from_pooled(
Arc::new(FrameBuffer::new(raw, Some(buffer_pool.clone()))),
config.resolution,
config.input_format,
capture.stream.stride(),
meta.sequence,
);
pipeline.encode_frame_sync(encoder, &frame)
})
}
}
};
@@ -321,10 +341,11 @@ fn run(
config.input_format,
config.fps,
));
for frame in frames {
pipeline.broadcast_encoded(Arc::new(frame));
fps_frames += 1;
}
fps_frames += frames.len() as u32;
pipeline.broadcast_encoded_batch(
frames.into_iter().map(Arc::new).collect::<Vec<_>>().into(),
reservations.unwrap_or_default(),
);
}
Ok(Err(error)) => {
if matches!(capture.encoder, Some(CaptureEncoder::Dma(_))) {
@@ -380,7 +401,7 @@ fn run(
}
}
if fps_start.elapsed() >= Duration::from_secs(1) {
pipeline.stats.blocking_lock().current_fps =
pipeline.stats.lock().current_fps =
fps_frames as f32 / fps_start.elapsed().as_secs_f32();
fps_frames = 0;
fps_start = Instant::now();

View File

@@ -41,6 +41,11 @@ pub(super) trait VideoEncoderTrait: Send {
pub(super) struct EncodedFrame {
pub(super) data: Bytes,
pub(super) key: i32,
/// Packet pts in milliseconds, stamped by the encoder when the input
/// frame was accepted. With a pipelined encoder a packet can be handed
/// back one call later than its input frame, so consumers must use this
/// instead of the wall clock time of the current call.
pub(super) pts: i64,
}
struct H264EncoderWrapper(H264Encoder);
@@ -53,6 +58,7 @@ impl VideoEncoderTrait for H264EncoderWrapper {
.map(|f| EncodedFrame {
data: f.data,
key: f.key,
pts: f.pts,
})
.collect())
}
@@ -99,6 +105,7 @@ impl VideoEncoderTrait for H265EncoderWrapper {
.map(|f| EncodedFrame {
data: f.data,
key: f.key,
pts: f.pts,
})
.collect())
}
@@ -126,6 +133,7 @@ impl VideoEncoderTrait for VP8EncoderWrapper {
.map(|f| EncodedFrame {
data: f.data.into(),
key: f.key,
pts: f.pts,
})
.collect())
}
@@ -151,6 +159,7 @@ impl VideoEncoderTrait for VP9EncoderWrapper {
.map(|f| EncodedFrame {
data: f.data.into(),
key: f.key,
pts: f.pts,
})
.collect())
}

View File

@@ -0,0 +1,178 @@
use std::sync::{Condvar, Mutex};
#[derive(Debug, PartialEq, Eq)]
pub(super) enum PublishResult {
Published,
Replaced,
Rejected,
}
struct FrameState<Frame> {
pending: Option<Frame>,
last_sequence: Option<u64>,
closed: bool,
}
pub(super) struct FrameMailbox<Frame> {
state: Mutex<FrameState<Frame>>,
available: Condvar,
}
impl<Frame> FrameMailbox<Frame> {
pub(super) fn new() -> Self {
Self {
state: Mutex::new(FrameState {
pending: None,
last_sequence: None,
closed: false,
}),
available: Condvar::new(),
}
}
pub(super) fn publish(&self, sequence: u64, frame: Frame) -> PublishResult {
let mut state = self.state.lock().expect("frame mailbox lock poisoned");
if state.closed || state.last_sequence.is_some_and(|last| sequence <= last) {
return PublishResult::Rejected;
}
state.last_sequence = Some(sequence);
let outcome = if state.pending.replace(frame).is_some() {
PublishResult::Replaced
} else {
PublishResult::Published
};
self.available.notify_one();
outcome
}
#[cfg(test)]
pub(super) fn receive(&self) -> Option<Frame> {
self.receive_when(|| Some(())).map(|(frame, ())| frame)
}
pub(super) fn receive_when<Ready>(
&self,
mut ready: impl FnMut() -> Option<Ready>,
) -> Option<(Frame, Ready)> {
let mut state = self.state.lock().expect("frame mailbox lock poisoned");
loop {
if state.closed {
return None;
}
if state.pending.is_some() {
if let Some(ready) = ready() {
return state.pending.take().map(|frame| (frame, ready));
}
}
state = self
.available
.wait(state)
.expect("frame mailbox lock poisoned");
}
}
pub(super) fn notify(&self) {
let _state = self.state.lock().expect("frame mailbox lock poisoned");
self.available.notify_one();
}
pub(super) fn has_pending(&self) -> bool {
self.state
.lock()
.expect("frame mailbox lock poisoned")
.pending
.is_some()
}
pub(super) fn close(&self) {
let mut state = self.state.lock().expect("frame mailbox lock poisoned");
state.closed = true;
state.pending.take();
self.available.notify_all();
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::{mpsc, Arc};
use std::time::Duration;
#[test]
fn pending_frame_is_replaced_by_latest() {
let mailbox = FrameMailbox::new();
assert_eq!(mailbox.publish(1, "first"), PublishResult::Published);
assert!(mailbox.has_pending());
assert_eq!(mailbox.publish(2, "latest"), PublishResult::Replaced);
assert_eq!(mailbox.receive(), Some("latest"));
assert!(!mailbox.has_pending());
}
#[test]
fn consumed_sequence_still_rejects_old_and_duplicate_frames() {
let mailbox = FrameMailbox::new();
assert_eq!(mailbox.publish(10, "current"), PublishResult::Published);
assert_eq!(mailbox.receive(), Some("current"));
assert_eq!(mailbox.publish(9, "old"), PublishResult::Rejected);
assert_eq!(mailbox.publish(10, "duplicate"), PublishResult::Rejected);
assert!(!mailbox.has_pending());
assert_eq!(mailbox.publish(11, "next"), PublishResult::Published);
assert_eq!(mailbox.receive(), Some("next"));
}
#[test]
fn close_discards_pending_and_rejects_future_frames() {
let mailbox = FrameMailbox::new();
mailbox.publish(1, "pending");
mailbox.close();
assert!(!mailbox.has_pending());
assert_eq!(mailbox.receive(), None);
assert_eq!(mailbox.publish(2, "late"), PublishResult::Rejected);
}
#[test]
fn receiver_is_woken_by_publish() {
for sequence in 0..64 {
let mailbox = Arc::new(FrameMailbox::new());
let receiver = mailbox.clone();
let (ready_sender, ready_receiver) = mpsc::channel();
let (result_sender, result_receiver) = mpsc::channel();
let worker = std::thread::spawn(move || {
ready_sender.send(()).unwrap();
result_sender.send(receiver.receive()).unwrap();
});
ready_receiver.recv().unwrap();
mailbox.publish(sequence, sequence);
assert_eq!(
result_receiver
.recv_timeout(Duration::from_secs(2))
.unwrap(),
Some(sequence)
);
worker.join().unwrap();
}
}
#[test]
fn close_wakes_idle_receiver() {
for _ in 0..64 {
let mailbox = Arc::new(FrameMailbox::<u64>::new());
let receiver = mailbox.clone();
let (ready_sender, ready_receiver) = mpsc::channel();
let (result_sender, result_receiver) = mpsc::channel();
let worker = std::thread::spawn(move || {
ready_sender.send(()).unwrap();
result_sender.send(receiver.receive()).unwrap();
});
ready_receiver.recv().unwrap();
mailbox.close();
assert_eq!(
result_receiver
.recv_timeout(Duration::from_secs(2))
.unwrap(),
None
);
worker.join().unwrap();
}
}
}

View File

@@ -1,8 +1,11 @@
//! Video processing pipelines.
mod delivery;
mod encoder_state;
mod frame_mailbox;
mod shared;
pub use delivery::EncodedVideoFrameReceiver;
pub use shared::{
EncodedVideoFrame, PipelineAppliedConfig, PipelineLifecycle, PipelineStateNotification,
SharedVideoPipeline, SharedVideoPipelineConfig, SharedVideoPipelineStats,

File diff suppressed because it is too large Load Diff

View File

@@ -697,9 +697,7 @@ impl VideoStreamManager {
/// Returns None if video capture cannot be started or pipeline creation fails.
pub async fn subscribe_encoded_frames(
&self,
) -> Option<
tokio::sync::mpsc::Receiver<std::sync::Arc<crate::video::pipeline::EncodedVideoFrame>>,
> {
) -> Option<crate::video::pipeline::EncodedVideoFrameReceiver> {
// 1. Ensure video capture is initialized (for config discovery)
if self.streamer.state().await == StreamerState::Uninitialized {
tracing::info!("Initializing video capture for encoded frame subscription");