perf(video): 优化视频流水线并统一使用 vendored OpenSSL

使用固定采集、解码和编码线程及同步最新帧槽,减少同步异步跨界和无效 MJPEG 解码。

编码前预留消费者容量,以完整批次投递编码结果并隔离慢客户端,避免丢失 H.264 参考帧;保留 PTS 修正,不增加 IDR 节流。

复用 Annex-B 缓冲并增加低频发送分项统计,默认启用 vendored OpenSSL SRTP AES-CM 后端。

HTTP 客户端、HTTPS 服务端和本地 DTLS 证书验证迁移至 OpenSSL,移除 rustls TLS 引擎依赖,保留证书验证、媒体加密和原有协议协商。
This commit is contained in:
mofeng-git
2026-10-01 18:34:21 +08:00
parent 48cd06c7a4
commit 318b58b26a
115 changed files with 18160 additions and 273 deletions

View File

@@ -995,7 +995,6 @@ mod tests {
use super::*;
fn test_provider(endpoint: &str) -> OpenAiComputerProvider {
let _ = rustls::crypto::ring::default_provider().install_default();
OpenAiComputerProvider::new(
"secret".to_string(),
endpoint.to_string(),

View File

@@ -4,10 +4,9 @@ use std::io::Write;
use std::net::{IpAddr, SocketAddr};
use std::path::PathBuf;
use axum_server::tls_rustls::RustlsConfig;
use axum_server::tls_openssl::OpenSSLAcceptor;
use clap::{Args, Parser, Subcommand, ValueEnum};
use futures::{stream::FuturesUnordered, StreamExt};
use rustls::crypto::{ring, CryptoProvider};
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
use one_kvm::auth::{SessionStore, TwoFactorService, UserStore};
@@ -17,6 +16,7 @@ use one_kvm::platform::PlatformCapabilities;
use one_kvm::runtime::{RuntimeBuilder, WebConfigOverrides};
use one_kvm::state::ShutdownAction;
use one_kvm::utils::bind_tcp_listener;
use one_kvm::utils::tls::server_config_from_pem_file;
#[derive(Debug, Clone, Copy, Default, ValueEnum)]
enum LogLevel {
@@ -100,9 +100,6 @@ async fn main() -> anyhow::Result<()> {
init_logging(args.log_level);
CryptoProvider::install_default(ring::default_provider())
.expect("Failed to install rustls crypto provider");
tracing::info!("Starting One-KVM v{}", env!("CARGO_PKG_VERSION"));
let platform = PlatformCapabilities::current();
tracing::info!(
@@ -187,7 +184,7 @@ async fn main() -> anyhow::Result<()> {
let tls_config = if let (Some(cert_path), Some(key_path)) =
(&config.web.ssl_cert_path, &config.web.ssl_key_path)
{
RustlsConfig::from_pem_file(cert_path, key_path).await?
server_config_from_pem_file(cert_path, key_path).await?
} else {
let cert_dir = data_dir.join("certs");
let cert_path = cert_dir.join("server.crt");
@@ -203,7 +200,7 @@ async fn main() -> anyhow::Result<()> {
tracing::info!("Using existing TLS certificate from {}", cert_dir.display());
}
RustlsConfig::from_pem_file(&cert_path, &key_path).await?
server_config_from_pem_file(&cert_path, &key_path).await?
};
let servers = FuturesUnordered::new();
@@ -211,7 +208,8 @@ async fn main() -> anyhow::Result<()> {
let local_addr = listener.local_addr()?;
tracing::info!("Starting HTTPS server on {}", local_addr);
let server = axum_server::from_tcp_rustls(listener, tls_config.clone())?
let server = axum_server::from_tcp(listener)?
.acceptor(OpenSSLAcceptor::new(tls_config.clone()))
.serve(app.clone().into_make_service());
servers.push(server);
}

View File

@@ -9,6 +9,7 @@ pub mod net;
pub mod net;
pub mod serial;
pub mod throttle;
pub mod tls;
pub use fs::{list_dir_names, read_trimmed};
pub use host::{hostname_from_etc, hostname_uname};

119
src/utils/tls.rs Normal file
View File

@@ -0,0 +1,119 @@
use axum_server::tls_openssl::OpenSSLConfig;
use openssl::pkey::PKey;
use openssl::ssl::{SslAcceptor, SslMethod, SslVersion};
use openssl::x509::X509;
use std::io;
use std::path::Path;
pub fn server_config_from_pem(cert_pem: &[u8], key_pem: &[u8]) -> io::Result<OpenSSLConfig> {
let mut certificates = X509::stack_from_pem(cert_pem)
.map_err(io::Error::other)?
.into_iter();
let certificate = certificates.next().ok_or_else(|| {
io::Error::new(io::ErrorKind::InvalidData, "TLS certificate chain is empty")
})?;
let private_key = PKey::private_key_from_pem(key_pem).map_err(io::Error::other)?;
let mut acceptor =
SslAcceptor::mozilla_intermediate_v5(SslMethod::tls_server()).map_err(io::Error::other)?;
acceptor
.set_min_proto_version(Some(SslVersion::TLS1_2))
.map_err(io::Error::other)?;
acceptor
.set_certificate(&certificate)
.map_err(io::Error::other)?;
acceptor
.set_private_key(&private_key)
.map_err(io::Error::other)?;
for certificate in certificates {
acceptor
.add_extra_chain_cert(certificate)
.map_err(io::Error::other)?;
}
OpenSSLConfig::try_from(acceptor).map_err(io::Error::other)
}
pub async fn server_config_from_pem_file(
cert_path: impl AsRef<Path>,
key_path: impl AsRef<Path>,
) -> io::Result<OpenSSLConfig> {
let cert_pem = tokio::fs::read(cert_path).await?;
let key_pem = tokio::fs::read(key_path).await?;
server_config_from_pem(&cert_pem, &key_pem)
}
#[cfg(test)]
mod tests {
use super::*;
use axum_server::tls_openssl::OpenSSLAcceptor;
#[test]
fn preserves_certificate_chain() {
let leaf = rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap();
let intermediate = rcgen::generate_simple_self_signed(vec!["issuer".into()]).unwrap();
let chain = format!("{}{}", leaf.cert.pem(), intermediate.cert.pem());
let config = server_config_from_pem(
chain.as_bytes(),
leaf.signing_key.serialize_pem().as_bytes(),
)
.unwrap();
assert_eq!(config.get_inner().context().extra_chain_certs().len(), 1);
}
#[test]
fn rejects_empty_invalid_and_mismatched_credentials() {
let first = rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap();
let second = rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap();
let cert = first.cert.pem();
let key = first.signing_key.serialize_pem();
assert!(server_config_from_pem(b"", key.as_bytes()).is_err());
assert!(server_config_from_pem(b"invalid", key.as_bytes()).is_err());
assert!(server_config_from_pem(cert.as_bytes(), b"invalid").is_err());
assert!(server_config_from_pem(
cert.as_bytes(),
second.signing_key.serialize_pem().as_bytes(),
)
.is_err());
}
#[tokio::test]
async fn https_client_verifies_certificates_and_supports_tls12() {
let cert = rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap();
let cert_pem = cert.cert.pem();
let config = server_config_from_pem(
cert_pem.as_bytes(),
cert.signing_key.serialize_pem().as_bytes(),
)
.unwrap();
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
listener.set_nonblocking(true).unwrap();
let address = listener.local_addr().unwrap();
let handle = axum_server::Handle::new();
let app = axum::Router::new().route("/", axum::routing::get(|| async { "tls-ok" }));
let server = axum_server::from_tcp(listener)
.unwrap()
.acceptor(OpenSSLAcceptor::new(config))
.handle(handle.clone())
.serve(app.into_make_service());
let task = tokio::spawn(server);
handle.listening().await.unwrap();
let url = format!("https://localhost:{}/", address.port());
let root = reqwest::Certificate::from_pem(cert_pem.as_bytes()).unwrap();
for tls12_only in [false, true] {
let mut client = reqwest::Client::builder()
.add_root_certificate(root.clone())
.timeout(std::time::Duration::from_secs(5));
if tls12_only {
client = client.max_tls_version(reqwest::tls::Version::TLS_1_2);
}
let response = client.build().unwrap().get(&url).send().await.unwrap();
assert_eq!(response.text().await.unwrap(), "tls-ok");
}
let untrusted = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(5))
.build()
.unwrap();
assert!(untrusted.get(&url).send().await.is_err());
handle.shutdown();
task.await.unwrap().unwrap();
}
}

View File

@@ -69,63 +69,54 @@ pub fn strip_aud_nal_units(data: &[u8]) -> Vec<u8> {
result
}
pub fn extract_sps_pps(data: &[u8]) -> (Option<Vec<u8>>, Option<Vec<u8>>) {
let mut sps: Option<Vec<u8>> = None;
let mut pps: Option<Vec<u8>> = None;
let mut i = 0;
#[derive(Debug, Default)]
pub struct H264AccessUnit<'data> {
pub sps: Option<&'data [u8]>,
pub pps: Option<&'data [u8]>,
pub is_idr: bool,
}
while i < data.len() {
let start_code_len = if i + 4 <= data.len()
&& data[i] == 0
&& data[i + 1] == 0
&& data[i + 2] == 0
&& data[i + 3] == 1
{
4
} else if i + 3 <= data.len() && data[i] == 0 && data[i + 1] == 0 && data[i + 2] == 1 {
3
} else {
i += 1;
continue;
};
let nal_start = i + start_code_len;
if nal_start >= data.len() {
break;
}
let nal_type = data[nal_start] & 0x1F;
let mut nal_end = data.len();
let mut j = nal_start + 1;
while j + 3 <= data.len() {
if (data[j] == 0 && data[j + 1] == 0 && data[j + 2] == 1)
|| (j + 4 <= data.len()
&& data[j] == 0
&& data[j + 1] == 0
&& data[j + 2] == 0
&& data[j + 3] == 1)
{
nal_end = j;
break;
fn next_start_code(data: &[u8], mut offset: usize) -> Option<(usize, usize)> {
while offset + 3 <= data.len() {
if data[offset] == 0 && data[offset + 1] == 0 {
if data[offset + 2] == 1 {
return Some((offset, 3));
}
j += 1;
}
match nal_type {
7 => {
sps = Some(data[nal_start..nal_end].to_vec());
if offset + 4 <= data.len() && data[offset + 2] == 0 && data[offset + 3] == 1 {
return Some((offset, 4));
}
8 => {
pps = Some(data[nal_start..nal_end].to_vec());
}
_ => {}
}
i = nal_end;
offset += 1;
}
None
}
(sps, pps)
pub fn inspect_annex_b(data: &[u8]) -> H264AccessUnit<'_> {
let mut inspection = H264AccessUnit::default();
let mut start_code = next_start_code(data, 0);
while let Some((offset, prefix_length)) = start_code {
let nal_start = offset + prefix_length;
let next = next_start_code(data, nal_start);
let nal_end = next.map_or(data.len(), |(next_offset, _)| next_offset);
if nal_start < nal_end {
match data[nal_start] & 0x1f {
5 => inspection.is_idr = true,
7 => inspection.sps = Some(&data[nal_start..nal_end]),
8 => inspection.pps = Some(&data[nal_start..nal_end]),
_ => {}
}
}
start_code = next;
}
inspection
}
pub fn extract_sps_pps(data: &[u8]) -> (Option<Vec<u8>>, Option<Vec<u8>>) {
let inspection = inspect_annex_b(data);
(
inspection.sps.map(<[u8]>::to_vec),
inspection.pps.map(<[u8]>::to_vec),
)
}
pub fn has_sps_pps(data: &[u8]) -> bool {
@@ -215,8 +206,9 @@ pub fn parse_profile_level_id_from_sps(sps: &[u8]) -> Option<String> {
}
pub fn extract_profile_level_id(data: &[u8]) -> Option<String> {
let (sps, _) = extract_sps_pps(data);
sps.and_then(|sps_data| parse_profile_level_id_from_sps(&sps_data))
inspect_annex_b(data)
.sps
.and_then(parse_profile_level_id_from_sps)
}
pub fn is_annex_b(data: &[u8]) -> bool {
@@ -292,6 +284,58 @@ pub fn normalize_for_webrtc(data: &[u8]) -> Vec<u8> {
mod tests {
use super::*;
#[test]
fn inspects_parameter_sets_and_idr_in_one_pass() {
let data = [
0, 0, 0, 1, 0x67, 0x42, 0xe0, 0x1f, 0, 0, 1, 0x68, 0xce, 0, 0, 0, 1, 0x65, 0x88,
];
let inspection = inspect_annex_b(&data);
assert_eq!(inspection.sps, Some(&data[4..8]));
assert_eq!(inspection.pps, Some(&data[11..13]));
assert!(inspection.is_idr);
assert!(std::ptr::eq(
inspection.sps.unwrap().as_ptr(),
data[4..].as_ptr()
));
assert_eq!(extract_profile_level_id(&data).as_deref(), Some("42e01f"));
}
#[test]
fn predicted_packet_has_no_parameter_sets_or_idr() {
let data = [0, 0, 0, 1, 0x41, 0xc0];
let inspection = inspect_annex_b(&data);
assert!(inspection.sps.is_none());
assert!(inspection.pps.is_none());
assert!(!inspection.is_idr);
}
#[test]
fn empty_and_truncated_nal_units_are_ignored() {
for data in [
&[][..],
&[0][..],
&[0, 0][..],
&[0, 0, 1][..],
&[0, 0, 0, 1][..],
&[0, 0, 1, 0, 0, 0, 1][..],
] {
let inspection = inspect_annex_b(data);
assert!(inspection.sps.is_none());
assert!(inspection.pps.is_none());
assert!(!inspection.is_idr);
}
}
#[test]
fn inspection_keeps_last_parameter_sets() {
let data = [
0, 0, 1, 0x67, 0x42, 0xc0, 0x1e, 0, 0, 0, 1, 0x67, 0x42, 0xe0, 0x1f,
];
let inspection = inspect_annex_b(&data);
assert_eq!(inspection.sps, Some(&data[11..15]));
assert_eq!(extract_profile_level_id(&data).as_deref(), Some("42e01f"));
}
#[test]
fn detects_h264_keyframes() {
let idr_frame = vec![0x00, 0x00, 0x00, 0x01, 0x65];

View File

@@ -0,0 +1,100 @@
use std::collections::VecDeque;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::Arc;
use std::time::Duration;
use parking_lot::Mutex;
use tokio::sync::mpsc;
use super::frame_mailbox::FrameMailbox;
use super::EncodedVideoFrame;
use crate::video::frame::VideoFrame;
pub(super) type EncodedBatch = Arc<[Arc<EncodedVideoFrame>]>;
pub(super) type FrameMailboxHandle = Arc<Mutex<Option<Arc<FrameMailbox<Arc<VideoFrame>>>>>>;
#[derive(Default)]
pub(super) struct DeliveryStats {
pub sent_frames: AtomicU64,
pub send_time_ns: AtomicU64,
pub send_errors: AtomicU64,
}
pub(super) struct VideoSubscriber {
pub sender: mpsc::Sender<EncodedBatch>,
pub needs_keyframe: AtomicBool,
pub recovery_requested: AtomicBool,
}
pub(super) struct VideoFrameReservation {
pub subscriber: Arc<VideoSubscriber>,
pub permit: mpsc::OwnedPermit<EncodedBatch>,
}
pub struct EncodedVideoFrameReceiver {
receiver: mpsc::Receiver<EncodedBatch>,
pending: VecDeque<Arc<EncodedVideoFrame>>,
mailbox: FrameMailboxHandle,
stats: Arc<DeliveryStats>,
}
impl EncodedVideoFrameReceiver {
pub(super) fn new(
receiver: mpsc::Receiver<EncodedBatch>,
mailbox: FrameMailboxHandle,
stats: Arc<DeliveryStats>,
) -> Self {
Self {
receiver,
pending: VecDeque::new(),
mailbox,
stats,
}
}
fn notify_capacity(&self) {
if let Some(mailbox) = self.mailbox.lock().as_ref() {
mailbox.notify();
}
}
pub async fn recv(&mut self) -> Option<Arc<EncodedVideoFrame>> {
loop {
if let Some(frame) = self.pending.pop_front() {
return Some(frame);
}
let batch = self.receiver.recv().await?;
self.pending.extend(batch.iter().cloned());
self.notify_capacity();
}
}
pub fn try_recv(&mut self) -> Result<Arc<EncodedVideoFrame>, mpsc::error::TryRecvError> {
loop {
if let Some(frame) = self.pending.pop_front() {
return Ok(frame);
}
let batch = self.receiver.try_recv()?;
self.pending.extend(batch.iter().cloned());
self.notify_capacity();
}
}
pub(crate) fn record_send(&self, elapsed: Duration, succeeded: bool) {
self.stats
.send_time_ns
.fetch_add(elapsed.as_nanos() as u64, Ordering::Relaxed);
if succeeded {
self.stats.sent_frames.fetch_add(1, Ordering::Relaxed);
} else {
self.stats.send_errors.fetch_add(1, Ordering::Relaxed);
}
}
}
impl Drop for EncodedVideoFrameReceiver {
fn drop(&mut self) {
self.receiver.close();
self.notify_capacity();
}
}

View File

@@ -256,60 +256,80 @@ fn run(
}
let capture = active.as_mut().expect("opened capture");
let result = match capture.encoder.as_mut().expect("active encoder") {
CaptureEncoder::Dma(encoder) => {
let pts = pipeline.pts_ms();
capture
let reservations = pipeline.reserve_outputs();
let result = if reservations.as_ref().is_none_or(Vec::is_empty) {
match capture.encoder.as_mut().expect("active encoder") {
CaptureEncoder::Dma(_) => capture
.stream
.with_next_dmabuf(|index, bytes_used, fresh_fd| {
let keyframe = pipeline.keyframe_requested.swap(false, Ordering::AcqRel);
// The callback holds the dequeue lease until native encode
// completes (or destroys MPP on error), before QBUF.
unsafe { encoder.encode(index, bytes_used, fresh_fd, pts, keyframe) }
})
.map(|(_, packet)| {
packet
.map(|packet| {
let data = Bytes::from(packet);
let is_keyframe = match config.output_codec {
VideoEncoderType::H264 => h264_bitstream::is_keyframe(&data),
VideoEncoderType::H265 => h265_bitstream::is_keyframe(&data),
_ => false,
};
let (data, is_keyframe) = pipeline.inspect_and_parameterize_packet(
config.output_codec,
data,
is_keyframe,
);
if config.output_codec == VideoEncoderType::H264 {
pipeline.update_h264_profile_level_id(&data);
}
vec![EncodedVideoFrame {
data,
pts_ms: pts,
is_keyframe,
sequence: pipeline.sequence.fetch_add(1, Ordering::Relaxed) + 1,
duration: Duration::from_micros(
1_000_000 / config.fps.max(1) as u64,
),
codec: config.output_codec,
}]
})
.map_err(AppError::VideoError)
})
.with_next_dmabuf(|_, _, _| ())
.map(|_| Ok(Vec::new())),
CaptureEncoder::Copy(_) => {
let mut raw = buffer_pool.take(0);
let result = capture.stream.next_into(&mut raw).map(|_| Ok(Vec::new()));
buffer_pool.put(raw);
result
}
}
CaptureEncoder::Copy(encoder) => {
let mut raw = buffer_pool.take(0);
capture.stream.next_into(&mut raw).map(|meta| {
let frame = VideoFrame::from_pooled(
Arc::new(FrameBuffer::new(raw, Some(buffer_pool.clone()))),
config.resolution,
config.input_format,
capture.stream.stride(),
meta.sequence,
);
pipeline.encode_frame_sync(encoder, &frame)
})
} else {
match capture.encoder.as_mut().expect("active encoder") {
CaptureEncoder::Dma(encoder) => {
let pts = pipeline.pts_ms();
capture
.stream
.with_next_dmabuf(|index, bytes_used, fresh_fd| {
let keyframe =
pipeline.keyframe_requested.swap(false, Ordering::AcqRel);
// The callback holds the dequeue lease until native encode
// completes (or destroys MPP on error), before QBUF.
unsafe { encoder.encode(index, bytes_used, fresh_fd, pts, keyframe) }
})
.map(|(_, packet)| {
packet
.map(|packet| {
let data = Bytes::from(packet);
let is_keyframe = match config.output_codec {
VideoEncoderType::H264 => {
h264_bitstream::is_keyframe(&data)
}
VideoEncoderType::H265 => {
h265_bitstream::is_keyframe(&data)
}
_ => false,
};
let (data, is_keyframe) = pipeline
.inspect_and_parameterize_packet(
config.output_codec,
data,
is_keyframe,
);
vec![EncodedVideoFrame {
data,
pts_ms: pts,
is_keyframe,
sequence: pipeline.sequence.fetch_add(1, Ordering::Relaxed)
+ 1,
duration: Duration::from_micros(
1_000_000 / config.fps.max(1) as u64,
),
codec: config.output_codec,
}]
})
.map_err(AppError::VideoError)
})
}
CaptureEncoder::Copy(encoder) => {
let mut raw = buffer_pool.take(0);
capture.stream.next_into(&mut raw).map(|meta| {
let frame = VideoFrame::from_pooled(
Arc::new(FrameBuffer::new(raw, Some(buffer_pool.clone()))),
config.resolution,
config.input_format,
capture.stream.stride(),
meta.sequence,
);
pipeline.encode_frame_sync(encoder, &frame)
})
}
}
};
@@ -321,10 +341,11 @@ fn run(
config.input_format,
config.fps,
));
for frame in frames {
pipeline.broadcast_encoded(Arc::new(frame));
fps_frames += 1;
}
fps_frames += frames.len() as u32;
pipeline.broadcast_encoded_batch(
frames.into_iter().map(Arc::new).collect::<Vec<_>>().into(),
reservations.unwrap_or_default(),
);
}
Ok(Err(error)) => {
if matches!(capture.encoder, Some(CaptureEncoder::Dma(_))) {
@@ -380,7 +401,7 @@ fn run(
}
}
if fps_start.elapsed() >= Duration::from_secs(1) {
pipeline.stats.blocking_lock().current_fps =
pipeline.stats.lock().current_fps =
fps_frames as f32 / fps_start.elapsed().as_secs_f32();
fps_frames = 0;
fps_start = Instant::now();

View File

@@ -41,6 +41,11 @@ pub(super) trait VideoEncoderTrait: Send {
pub(super) struct EncodedFrame {
pub(super) data: Bytes,
pub(super) key: i32,
/// Packet pts in milliseconds, stamped by the encoder when the input
/// frame was accepted. With a pipelined encoder a packet can be handed
/// back one call later than its input frame, so consumers must use this
/// instead of the wall clock time of the current call.
pub(super) pts: i64,
}
struct H264EncoderWrapper(H264Encoder);
@@ -53,6 +58,7 @@ impl VideoEncoderTrait for H264EncoderWrapper {
.map(|f| EncodedFrame {
data: f.data,
key: f.key,
pts: f.pts,
})
.collect())
}
@@ -99,6 +105,7 @@ impl VideoEncoderTrait for H265EncoderWrapper {
.map(|f| EncodedFrame {
data: f.data,
key: f.key,
pts: f.pts,
})
.collect())
}
@@ -126,6 +133,7 @@ impl VideoEncoderTrait for VP8EncoderWrapper {
.map(|f| EncodedFrame {
data: f.data.into(),
key: f.key,
pts: f.pts,
})
.collect())
}
@@ -151,6 +159,7 @@ impl VideoEncoderTrait for VP9EncoderWrapper {
.map(|f| EncodedFrame {
data: f.data.into(),
key: f.key,
pts: f.pts,
})
.collect())
}

View File

@@ -0,0 +1,178 @@
use std::sync::{Condvar, Mutex};
#[derive(Debug, PartialEq, Eq)]
pub(super) enum PublishResult {
Published,
Replaced,
Rejected,
}
struct FrameState<Frame> {
pending: Option<Frame>,
last_sequence: Option<u64>,
closed: bool,
}
pub(super) struct FrameMailbox<Frame> {
state: Mutex<FrameState<Frame>>,
available: Condvar,
}
impl<Frame> FrameMailbox<Frame> {
pub(super) fn new() -> Self {
Self {
state: Mutex::new(FrameState {
pending: None,
last_sequence: None,
closed: false,
}),
available: Condvar::new(),
}
}
pub(super) fn publish(&self, sequence: u64, frame: Frame) -> PublishResult {
let mut state = self.state.lock().expect("frame mailbox lock poisoned");
if state.closed || state.last_sequence.is_some_and(|last| sequence <= last) {
return PublishResult::Rejected;
}
state.last_sequence = Some(sequence);
let outcome = if state.pending.replace(frame).is_some() {
PublishResult::Replaced
} else {
PublishResult::Published
};
self.available.notify_one();
outcome
}
#[cfg(test)]
pub(super) fn receive(&self) -> Option<Frame> {
self.receive_when(|| Some(())).map(|(frame, ())| frame)
}
pub(super) fn receive_when<Ready>(
&self,
mut ready: impl FnMut() -> Option<Ready>,
) -> Option<(Frame, Ready)> {
let mut state = self.state.lock().expect("frame mailbox lock poisoned");
loop {
if state.closed {
return None;
}
if state.pending.is_some() {
if let Some(ready) = ready() {
return state.pending.take().map(|frame| (frame, ready));
}
}
state = self
.available
.wait(state)
.expect("frame mailbox lock poisoned");
}
}
pub(super) fn notify(&self) {
let _state = self.state.lock().expect("frame mailbox lock poisoned");
self.available.notify_one();
}
pub(super) fn has_pending(&self) -> bool {
self.state
.lock()
.expect("frame mailbox lock poisoned")
.pending
.is_some()
}
pub(super) fn close(&self) {
let mut state = self.state.lock().expect("frame mailbox lock poisoned");
state.closed = true;
state.pending.take();
self.available.notify_all();
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::{mpsc, Arc};
use std::time::Duration;
#[test]
fn pending_frame_is_replaced_by_latest() {
let mailbox = FrameMailbox::new();
assert_eq!(mailbox.publish(1, "first"), PublishResult::Published);
assert!(mailbox.has_pending());
assert_eq!(mailbox.publish(2, "latest"), PublishResult::Replaced);
assert_eq!(mailbox.receive(), Some("latest"));
assert!(!mailbox.has_pending());
}
#[test]
fn consumed_sequence_still_rejects_old_and_duplicate_frames() {
let mailbox = FrameMailbox::new();
assert_eq!(mailbox.publish(10, "current"), PublishResult::Published);
assert_eq!(mailbox.receive(), Some("current"));
assert_eq!(mailbox.publish(9, "old"), PublishResult::Rejected);
assert_eq!(mailbox.publish(10, "duplicate"), PublishResult::Rejected);
assert!(!mailbox.has_pending());
assert_eq!(mailbox.publish(11, "next"), PublishResult::Published);
assert_eq!(mailbox.receive(), Some("next"));
}
#[test]
fn close_discards_pending_and_rejects_future_frames() {
let mailbox = FrameMailbox::new();
mailbox.publish(1, "pending");
mailbox.close();
assert!(!mailbox.has_pending());
assert_eq!(mailbox.receive(), None);
assert_eq!(mailbox.publish(2, "late"), PublishResult::Rejected);
}
#[test]
fn receiver_is_woken_by_publish() {
for sequence in 0..64 {
let mailbox = Arc::new(FrameMailbox::new());
let receiver = mailbox.clone();
let (ready_sender, ready_receiver) = mpsc::channel();
let (result_sender, result_receiver) = mpsc::channel();
let worker = std::thread::spawn(move || {
ready_sender.send(()).unwrap();
result_sender.send(receiver.receive()).unwrap();
});
ready_receiver.recv().unwrap();
mailbox.publish(sequence, sequence);
assert_eq!(
result_receiver
.recv_timeout(Duration::from_secs(2))
.unwrap(),
Some(sequence)
);
worker.join().unwrap();
}
}
#[test]
fn close_wakes_idle_receiver() {
for _ in 0..64 {
let mailbox = Arc::new(FrameMailbox::<u64>::new());
let receiver = mailbox.clone();
let (ready_sender, ready_receiver) = mpsc::channel();
let (result_sender, result_receiver) = mpsc::channel();
let worker = std::thread::spawn(move || {
ready_sender.send(()).unwrap();
result_sender.send(receiver.receive()).unwrap();
});
ready_receiver.recv().unwrap();
mailbox.close();
assert_eq!(
result_receiver
.recv_timeout(Duration::from_secs(2))
.unwrap(),
None
);
worker.join().unwrap();
}
}
}

View File

@@ -1,8 +1,11 @@
//! Video processing pipelines.
mod delivery;
mod encoder_state;
mod frame_mailbox;
mod shared;
pub use delivery::EncodedVideoFrameReceiver;
pub use shared::{
EncodedVideoFrame, PipelineAppliedConfig, PipelineLifecycle, PipelineStateNotification,
SharedVideoPipeline, SharedVideoPipelineConfig, SharedVideoPipelineStats,

File diff suppressed because it is too large Load Diff

View File

@@ -697,9 +697,7 @@ impl VideoStreamManager {
/// Returns None if video capture cannot be started or pipeline creation fails.
pub async fn subscribe_encoded_frames(
&self,
) -> Option<
tokio::sync::mpsc::Receiver<std::sync::Arc<crate::video::pipeline::EncodedVideoFrame>>,
> {
) -> Option<crate::video::pipeline::EncodedVideoFrameReceiver> {
// 1. Ensure video capture is initialized (for config discovery)
if self.streamer.state().await == StreamerState::Uninitialized {
tracing::info!("Initializing video capture for encoded frame subscription");

View File

@@ -1328,7 +1328,7 @@ impl WebConfigUpdate {
}
}
// Cert and key must be provided together (cryptographic validity is checked in the
// handler via `RustlsConfig::from_pem`, same stack as the running HTTPS server).
// handler via `server_config_from_pem`, same stack as the running HTTPS server).
match (&self.ssl_cert_pem, &self.ssl_key_pem) {
(Some(_cert), Some(_key)) => {}
(Some(_), None) => {

View File

@@ -1,9 +1,9 @@
use axum::{extract::State, Json};
use axum_server::tls_rustls::RustlsConfig;
use std::sync::Arc;
use crate::error::{AppError, Result};
use crate::state::AppState;
use crate::utils::tls::server_config_from_pem;
use super::types::{WebConfigResponse, WebConfigUpdate};
@@ -27,8 +27,7 @@ pub async fn update_web_config(
) =
(&req.ssl_cert_pem, &req.ssl_key_pem)
{
RustlsConfig::from_pem(cert_pem.as_bytes().to_vec(), key_pem.as_bytes().to_vec())
.await
server_config_from_pem(cert_pem.as_bytes(), key_pem.as_bytes())
.map_err(|e| {
AppError::BadRequest(
format!(
@@ -88,16 +87,12 @@ pub async fn update_web_config(
#[cfg(test)]
mod tests {
use super::*;
use rustls::crypto::{ring, CryptoProvider};
#[tokio::test]
async fn rustls_accepts_rcgen_self_signed_pem() {
let _ = CryptoProvider::install_default(ring::default_provider());
#[test]
fn openssl_accepts_rcgen_self_signed_pem() {
let cert = rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap();
let cert_pem = cert.cert.pem();
let key_pem = cert.signing_key.serialize_pem();
RustlsConfig::from_pem(cert_pem.into_bytes(), key_pem.into_bytes())
.await
.unwrap();
server_config_from_pem(cert_pem.as_bytes(), key_pem.as_bytes()).unwrap();
}
}

View File

@@ -37,9 +37,8 @@ use crate::error::{AppError, Result};
use crate::hid::datachannel::{parse_hid_message, HidChannelEvent};
use crate::hid::HidController;
use crate::video::codec::{h264_bitstream, h265_bitstream};
use crate::video::types::{
BitratePreset, EncodedVideoFrame, PixelFormat, Resolution, VideoEncoderType,
};
use crate::video::pipeline::EncodedVideoFrameReceiver;
use crate::video::types::{BitratePreset, PixelFormat, Resolution, VideoEncoderType};
const MIME_TYPE_H265: &str = "video/H265";
const KEYFRAME_RETRY_LIMIT: u8 = 3;
@@ -527,7 +526,7 @@ impl UniversalSession {
/// `on_connected` runs once ICE is up (e.g. request a keyframe).
pub async fn start_from_video_pipeline(
&self,
mut frame_rx: tokio::sync::mpsc::Receiver<std::sync::Arc<EncodedVideoFrame>>,
mut frame_rx: EncodedVideoFrameReceiver,
request_keyframe: Arc<dyn Fn() + Send + Sync + 'static>,
) {
if let Some(handle) = self.video_receiver_handle.lock().await.take() {
@@ -670,6 +669,7 @@ impl UniversalSession {
}
}
let send_started = Instant::now();
let send_result = video_track
.write_frame_bytes_at(
encoded_frame.data.clone(),
@@ -677,6 +677,7 @@ impl UniversalSession {
Some(encoded_frame.pts_ms),
)
.await;
frame_rx.record_send(send_started.elapsed(), send_result.is_ok());
match send_result {
Ok(()) => {
frames_sent += 1;

View File

@@ -4,9 +4,9 @@ use bytes::Bytes;
use rtp::codecs::h264::H264Payloader;
use rtp::packetizer::Payloader;
use std::sync::Arc;
use std::time::Duration;
use std::time::{Duration, Instant};
use tokio::sync::Mutex;
use tracing::{debug, trace, warn};
use tracing::{debug, info, trace, warn};
use webrtc::media::Sample;
use webrtc::rtp_transceiver::rtp_codec::RTCRtpCodecCapability;
use webrtc::track::track_local::track_local_static_rtp::TrackLocalStaticRTP;
@@ -191,6 +191,63 @@ struct H264TrackState {
clock: H264RtpClock,
sps: Option<Vec<u8>>,
pps: Option<Vec<u8>>,
performance: H264SendPerformance,
}
#[derive(Default)]
struct H264SendTiming {
prepare: Duration,
lock: Duration,
packetize: Duration,
write: Duration,
}
struct H264SendPerformance {
last_report: Instant,
frames: u64,
packets: u64,
bytes: u64,
timing: H264SendTiming,
}
impl H264SendPerformance {
fn new() -> Self {
Self {
last_report: Instant::now(),
frames: 0,
packets: 0,
bytes: 0,
timing: H264SendTiming::default(),
}
}
fn record(&mut self, bytes: usize, packets: usize, timing: H264SendTiming) {
self.frames += 1;
self.packets += packets as u64;
self.bytes += bytes as u64;
self.timing.prepare += timing.prepare;
self.timing.lock += timing.lock;
self.timing.packetize += timing.packetize;
self.timing.write += timing.write;
let elapsed = self.last_report.elapsed();
if elapsed < Duration::from_secs(5) {
return;
}
let average_ms = |duration: Duration| duration.as_secs_f64() * 1000.0 / self.frames as f64;
info!(
"[VideoSendPerf] window_s={:.2} frames={} packets_per_frame={:.2} frame_kib={:.2} prepare_ms={:.3} lock_ms={:.3} packetize_ms={:.3} write_ms={:.3} write_us_per_packet={:.3}",
elapsed.as_secs_f64(),
self.frames,
self.packets as f64 / self.frames as f64,
self.bytes as f64 / self.frames as f64 / 1024.0,
average_ms(self.timing.prepare),
average_ms(self.timing.lock),
average_ms(self.timing.packetize),
average_ms(self.timing.write),
self.timing.write.as_secs_f64() * 1_000_000.0 / self.packets.max(1) as f64,
);
*self = Self::new();
}
}
pub struct UniversalVideoTrack {
@@ -199,6 +256,7 @@ pub struct UniversalVideoTrack {
config: UniversalVideoTrackConfig,
h265_state: Option<Mutex<H265RtpState>>,
h264_state: Mutex<H264TrackState>,
h264_perf_enabled: bool,
}
impl UniversalVideoTrack {
@@ -253,6 +311,7 @@ impl UniversalVideoTrack {
increment: 90000 / config.fps.max(1),
started: false,
},
performance: H264SendPerformance::new(),
};
Self {
track,
@@ -260,6 +319,7 @@ impl UniversalVideoTrack {
config,
h265_state,
h264_state: Mutex::new(h264_state),
h264_perf_enabled: std::env::var("ONE_KVM_VIDEO_STATS").as_deref() == Ok("1"),
}
}
@@ -309,20 +369,24 @@ impl UniversalVideoTrack {
/// Keep the stack's H.264 STAP/FU payloader, with explicit RTP timestamps.
async fn write_h264_frame(&self, data: Bytes, pts_ms: Option<i64>) -> Result<()> {
let normalized = h264_bitstream::normalize_for_webrtc(data.as_ref());
let mut data = Bytes::from(normalized);
let prepare_started = self.h264_perf_enabled.then(Instant::now);
let mut data = h264_bitstream::normalize_annex_b(data);
let idr = h264_bitstream::is_keyframe(data.as_ref());
let has_parameter_sets = h264_bitstream::has_sps_pps(data.as_ref());
let inspection = h264_bitstream::inspect_annex_b(data.as_ref());
let idr = inspection.is_idr;
let has_parameter_sets = inspection.sps.is_some() && inspection.pps.is_some();
let prepare_time = prepare_started.map(|started| started.elapsed());
let lock_started = self.h264_perf_enabled.then(Instant::now);
let mut state = self.h264_state.lock().await;
let lock_time = lock_started.map(|started| started.elapsed());
let packetize_started = self.h264_perf_enabled.then(Instant::now);
{
let (sps, pps) = h264_bitstream::extract_sps_pps(data.as_ref());
if let Some(sps) = sps {
state.sps = Some(sps);
if let Some(sps) = inspection.sps {
state.sps = Some(sps.to_vec());
}
if let Some(pps) = pps {
state.pps = Some(pps);
if let Some(pps) = inspection.pps {
state.pps = Some(pps.to_vec());
}
if idr && !has_parameter_sets {
@@ -351,6 +415,8 @@ impl UniversalVideoTrack {
}
let timestamp = state.clock.next(pts_ms);
let count = payloads.len();
let packetize_time = packetize_started.map(|started| started.elapsed());
let write_started = self.h264_perf_enabled.then(Instant::now);
// Serialize complete access units, including their sequence allocation.
for (index, payload) in payloads.into_iter().enumerate() {
let sequence_number = state.sequence_number;
@@ -370,6 +436,18 @@ impl UniversalVideoTrack {
.await
.map_err(|e| AppError::WebRtcError(format!("H264 RTP write failed: {e}")))?;
}
if let Some(started) = write_started {
state.performance.record(
data.len(),
count,
H264SendTiming {
prepare: prepare_time.unwrap_or_default(),
lock: lock_time.unwrap_or_default(),
packetize: packetize_time.unwrap_or_default(),
write: started.elapsed(),
},
);
}
Ok(())
}
@@ -533,6 +611,108 @@ mod tests {
assert_eq!(clock.next(None), 6092);
}
#[test]
fn h264_zero_copy_input_preserves_rtp_payloads() {
let mut annex_b = vec![
0, 0, 0, 1, 0x09, 0xf0, 0, 0, 1, 0x67, 0x42, 0xe0, 0x1f, 0, 0, 0, 1, 0x68, 0xce, 0, 0,
1, 0x0c, 0xff, 0, 0, 0, 1, 0x65,
];
annex_b.extend_from_slice(&vec![0x55; 2500]);
let mut avcc = vec![0, 0, 0, 2, 0x09, 0xf0, 0, 0, 1, 1, 0x65];
avcc.extend_from_slice(&vec![0x55; 256]);
let inputs = vec![
annex_b,
avcc,
vec![0, 0, 1, 0x41, 0x88, 0x55],
vec![0x41, 0x88, 0x55],
vec![0, 0, 0, 1, 0x09, 0xf0, 0, 0, 1, 0x0c, 0xff],
Vec::new(),
];
for input in inputs {
let previous = Bytes::from(h264_bitstream::normalize_for_webrtc(&input));
let current = h264_bitstream::normalize_annex_b(Bytes::from(input));
let mut previous_payloader = H264Payloader::default();
let mut current_payloader = H264Payloader::default();
assert_eq!(
previous_payloader.payload(RTP_MTU - 12, &previous).unwrap(),
current_payloader.payload(RTP_MTU - 12, &current).unwrap(),
);
}
}
#[test]
fn h264_zero_copy_input_preserves_cached_parameter_sets() {
let mut previous_payloader = H264Payloader::default();
let mut current_payloader = H264Payloader::default();
for input in [
Bytes::from_static(&[0, 0, 0, 1, 0x67, 0x42, 0xe0, 0x1f]),
Bytes::from_static(&[0, 0, 0, 1, 0x09, 0xf0, 0, 0, 1, 0x68, 0xce]),
Bytes::from_static(&[0, 0, 0, 1, 0x65, 0x88, 0x55]),
] {
let previous = Bytes::from(h264_bitstream::normalize_for_webrtc(&input));
let current = h264_bitstream::normalize_annex_b(input.clone());
assert_eq!(current.as_ptr(), input.as_ptr());
assert_eq!(
previous_payloader.payload(RTP_MTU - 12, &previous).unwrap(),
current_payloader.payload(RTP_MTU - 12, &current).unwrap(),
);
}
}
#[test]
fn h264_send_performance_accumulates_stage_timings() {
let mut performance = H264SendPerformance::new();
for _ in 0..2 {
performance.record(
1200,
2,
H264SendTiming {
prepare: Duration::from_micros(100),
lock: Duration::from_micros(20),
packetize: Duration::from_micros(200),
write: Duration::from_micros(300),
},
);
}
assert_eq!(performance.frames, 2);
assert_eq!(performance.bytes, 2400);
assert_eq!(performance.packets, 4);
assert_eq!(performance.timing.prepare, Duration::from_micros(200));
assert_eq!(performance.timing.lock, Duration::from_micros(40));
assert_eq!(performance.timing.packetize, Duration::from_micros(400));
assert_eq!(performance.timing.write, Duration::from_micros(600));
}
#[test]
fn srtp_backend_preserves_authentication_and_sequence_rollover() {
use webrtc::srtp::context::Context;
use webrtc::srtp::protection_profile::ProtectionProfile;
for profile in [
ProtectionProfile::Aes128CmHmacSha1_80,
ProtectionProfile::AeadAes128Gcm,
] {
let key = vec![42; profile.key_len()];
let salt = vec![13; profile.salt_len()];
let mut sender = Context::new(&key, &salt, profile, None, None).unwrap();
let mut receiver = Context::new(&key, &salt, profile, None, None).unwrap();
for sequence in [u16::MAX - 1, u16::MAX, 0, 1] {
for ssrc in [17u32, 18] {
let mut packet = vec![0x55; 1200];
packet[..12].copy_from_slice(&[0x80, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]);
packet[2..4].copy_from_slice(&sequence.to_be_bytes());
packet[8..12].copy_from_slice(&ssrc.to_be_bytes());
let encrypted = sender.encrypt_rtp(&packet).unwrap();
assert_ne!(&encrypted[12..1200], &packet[12..]);
let mut corrupted = encrypted.to_vec();
corrupted[13] ^= 1;
assert!(receiver.decrypt_rtp(&corrupted).is_err());
assert_eq!(receiver.decrypt_rtp(&encrypted).unwrap().as_ref(), packet);
}
}
}
}
#[test]
fn test_video_codec_properties() {
assert_eq!(VideoCodec::H264.mime_type(), "video/H264");