mirror of
https://github.com/mofeng-git/One-KVM.git
synced 2026-10-27 16:00:49 +08:00
perf(video): 优化视频流水线并统一使用 vendored OpenSSL
使用固定采集、解码和编码线程及同步最新帧槽,减少同步异步跨界和无效 MJPEG 解码。 编码前预留消费者容量,以完整批次投递编码结果并隔离慢客户端,避免丢失 H.264 参考帧;保留 PTS 修正,不增加 IDR 节流。 复用 Annex-B 缓冲并增加低频发送分项统计,默认启用 vendored OpenSSL SRTP AES-CM 后端。 HTTP 客户端、HTTPS 服务端和本地 DTLS 证书验证迁移至 OpenSSL,移除 rustls TLS 引擎依赖,保留证书验证、媒体加密和原有协议协商。
This commit is contained in:
@@ -995,7 +995,6 @@ mod tests {
|
||||
use super::*;
|
||||
|
||||
fn test_provider(endpoint: &str) -> OpenAiComputerProvider {
|
||||
let _ = rustls::crypto::ring::default_provider().install_default();
|
||||
OpenAiComputerProvider::new(
|
||||
"secret".to_string(),
|
||||
endpoint.to_string(),
|
||||
|
||||
14
src/main.rs
14
src/main.rs
@@ -4,10 +4,9 @@ use std::io::Write;
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
use std::path::PathBuf;
|
||||
|
||||
use axum_server::tls_rustls::RustlsConfig;
|
||||
use axum_server::tls_openssl::OpenSSLAcceptor;
|
||||
use clap::{Args, Parser, Subcommand, ValueEnum};
|
||||
use futures::{stream::FuturesUnordered, StreamExt};
|
||||
use rustls::crypto::{ring, CryptoProvider};
|
||||
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
|
||||
|
||||
use one_kvm::auth::{SessionStore, TwoFactorService, UserStore};
|
||||
@@ -17,6 +16,7 @@ use one_kvm::platform::PlatformCapabilities;
|
||||
use one_kvm::runtime::{RuntimeBuilder, WebConfigOverrides};
|
||||
use one_kvm::state::ShutdownAction;
|
||||
use one_kvm::utils::bind_tcp_listener;
|
||||
use one_kvm::utils::tls::server_config_from_pem_file;
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, ValueEnum)]
|
||||
enum LogLevel {
|
||||
@@ -100,9 +100,6 @@ async fn main() -> anyhow::Result<()> {
|
||||
|
||||
init_logging(args.log_level);
|
||||
|
||||
CryptoProvider::install_default(ring::default_provider())
|
||||
.expect("Failed to install rustls crypto provider");
|
||||
|
||||
tracing::info!("Starting One-KVM v{}", env!("CARGO_PKG_VERSION"));
|
||||
let platform = PlatformCapabilities::current();
|
||||
tracing::info!(
|
||||
@@ -187,7 +184,7 @@ async fn main() -> anyhow::Result<()> {
|
||||
let tls_config = if let (Some(cert_path), Some(key_path)) =
|
||||
(&config.web.ssl_cert_path, &config.web.ssl_key_path)
|
||||
{
|
||||
RustlsConfig::from_pem_file(cert_path, key_path).await?
|
||||
server_config_from_pem_file(cert_path, key_path).await?
|
||||
} else {
|
||||
let cert_dir = data_dir.join("certs");
|
||||
let cert_path = cert_dir.join("server.crt");
|
||||
@@ -203,7 +200,7 @@ async fn main() -> anyhow::Result<()> {
|
||||
tracing::info!("Using existing TLS certificate from {}", cert_dir.display());
|
||||
}
|
||||
|
||||
RustlsConfig::from_pem_file(&cert_path, &key_path).await?
|
||||
server_config_from_pem_file(&cert_path, &key_path).await?
|
||||
};
|
||||
|
||||
let servers = FuturesUnordered::new();
|
||||
@@ -211,7 +208,8 @@ async fn main() -> anyhow::Result<()> {
|
||||
let local_addr = listener.local_addr()?;
|
||||
tracing::info!("Starting HTTPS server on {}", local_addr);
|
||||
|
||||
let server = axum_server::from_tcp_rustls(listener, tls_config.clone())?
|
||||
let server = axum_server::from_tcp(listener)?
|
||||
.acceptor(OpenSSLAcceptor::new(tls_config.clone()))
|
||||
.serve(app.clone().into_make_service());
|
||||
servers.push(server);
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ pub mod net;
|
||||
pub mod net;
|
||||
pub mod serial;
|
||||
pub mod throttle;
|
||||
pub mod tls;
|
||||
|
||||
pub use fs::{list_dir_names, read_trimmed};
|
||||
pub use host::{hostname_from_etc, hostname_uname};
|
||||
|
||||
119
src/utils/tls.rs
Normal file
119
src/utils/tls.rs
Normal file
@@ -0,0 +1,119 @@
|
||||
use axum_server::tls_openssl::OpenSSLConfig;
|
||||
use openssl::pkey::PKey;
|
||||
use openssl::ssl::{SslAcceptor, SslMethod, SslVersion};
|
||||
use openssl::x509::X509;
|
||||
use std::io;
|
||||
use std::path::Path;
|
||||
|
||||
pub fn server_config_from_pem(cert_pem: &[u8], key_pem: &[u8]) -> io::Result<OpenSSLConfig> {
|
||||
let mut certificates = X509::stack_from_pem(cert_pem)
|
||||
.map_err(io::Error::other)?
|
||||
.into_iter();
|
||||
let certificate = certificates.next().ok_or_else(|| {
|
||||
io::Error::new(io::ErrorKind::InvalidData, "TLS certificate chain is empty")
|
||||
})?;
|
||||
let private_key = PKey::private_key_from_pem(key_pem).map_err(io::Error::other)?;
|
||||
let mut acceptor =
|
||||
SslAcceptor::mozilla_intermediate_v5(SslMethod::tls_server()).map_err(io::Error::other)?;
|
||||
acceptor
|
||||
.set_min_proto_version(Some(SslVersion::TLS1_2))
|
||||
.map_err(io::Error::other)?;
|
||||
acceptor
|
||||
.set_certificate(&certificate)
|
||||
.map_err(io::Error::other)?;
|
||||
acceptor
|
||||
.set_private_key(&private_key)
|
||||
.map_err(io::Error::other)?;
|
||||
for certificate in certificates {
|
||||
acceptor
|
||||
.add_extra_chain_cert(certificate)
|
||||
.map_err(io::Error::other)?;
|
||||
}
|
||||
OpenSSLConfig::try_from(acceptor).map_err(io::Error::other)
|
||||
}
|
||||
|
||||
pub async fn server_config_from_pem_file(
|
||||
cert_path: impl AsRef<Path>,
|
||||
key_path: impl AsRef<Path>,
|
||||
) -> io::Result<OpenSSLConfig> {
|
||||
let cert_pem = tokio::fs::read(cert_path).await?;
|
||||
let key_pem = tokio::fs::read(key_path).await?;
|
||||
server_config_from_pem(&cert_pem, &key_pem)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use axum_server::tls_openssl::OpenSSLAcceptor;
|
||||
|
||||
#[test]
|
||||
fn preserves_certificate_chain() {
|
||||
let leaf = rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap();
|
||||
let intermediate = rcgen::generate_simple_self_signed(vec!["issuer".into()]).unwrap();
|
||||
let chain = format!("{}{}", leaf.cert.pem(), intermediate.cert.pem());
|
||||
let config = server_config_from_pem(
|
||||
chain.as_bytes(),
|
||||
leaf.signing_key.serialize_pem().as_bytes(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(config.get_inner().context().extra_chain_certs().len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_empty_invalid_and_mismatched_credentials() {
|
||||
let first = rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap();
|
||||
let second = rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap();
|
||||
let cert = first.cert.pem();
|
||||
let key = first.signing_key.serialize_pem();
|
||||
assert!(server_config_from_pem(b"", key.as_bytes()).is_err());
|
||||
assert!(server_config_from_pem(b"invalid", key.as_bytes()).is_err());
|
||||
assert!(server_config_from_pem(cert.as_bytes(), b"invalid").is_err());
|
||||
assert!(server_config_from_pem(
|
||||
cert.as_bytes(),
|
||||
second.signing_key.serialize_pem().as_bytes(),
|
||||
)
|
||||
.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn https_client_verifies_certificates_and_supports_tls12() {
|
||||
let cert = rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap();
|
||||
let cert_pem = cert.cert.pem();
|
||||
let config = server_config_from_pem(
|
||||
cert_pem.as_bytes(),
|
||||
cert.signing_key.serialize_pem().as_bytes(),
|
||||
)
|
||||
.unwrap();
|
||||
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
listener.set_nonblocking(true).unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let handle = axum_server::Handle::new();
|
||||
let app = axum::Router::new().route("/", axum::routing::get(|| async { "tls-ok" }));
|
||||
let server = axum_server::from_tcp(listener)
|
||||
.unwrap()
|
||||
.acceptor(OpenSSLAcceptor::new(config))
|
||||
.handle(handle.clone())
|
||||
.serve(app.into_make_service());
|
||||
let task = tokio::spawn(server);
|
||||
handle.listening().await.unwrap();
|
||||
let url = format!("https://localhost:{}/", address.port());
|
||||
let root = reqwest::Certificate::from_pem(cert_pem.as_bytes()).unwrap();
|
||||
for tls12_only in [false, true] {
|
||||
let mut client = reqwest::Client::builder()
|
||||
.add_root_certificate(root.clone())
|
||||
.timeout(std::time::Duration::from_secs(5));
|
||||
if tls12_only {
|
||||
client = client.max_tls_version(reqwest::tls::Version::TLS_1_2);
|
||||
}
|
||||
let response = client.build().unwrap().get(&url).send().await.unwrap();
|
||||
assert_eq!(response.text().await.unwrap(), "tls-ok");
|
||||
}
|
||||
let untrusted = reqwest::Client::builder()
|
||||
.timeout(std::time::Duration::from_secs(5))
|
||||
.build()
|
||||
.unwrap();
|
||||
assert!(untrusted.get(&url).send().await.is_err());
|
||||
handle.shutdown();
|
||||
task.await.unwrap().unwrap();
|
||||
}
|
||||
}
|
||||
@@ -69,63 +69,54 @@ pub fn strip_aud_nal_units(data: &[u8]) -> Vec<u8> {
|
||||
result
|
||||
}
|
||||
|
||||
pub fn extract_sps_pps(data: &[u8]) -> (Option<Vec<u8>>, Option<Vec<u8>>) {
|
||||
let mut sps: Option<Vec<u8>> = None;
|
||||
let mut pps: Option<Vec<u8>> = None;
|
||||
let mut i = 0;
|
||||
#[derive(Debug, Default)]
|
||||
pub struct H264AccessUnit<'data> {
|
||||
pub sps: Option<&'data [u8]>,
|
||||
pub pps: Option<&'data [u8]>,
|
||||
pub is_idr: bool,
|
||||
}
|
||||
|
||||
while i < data.len() {
|
||||
let start_code_len = if i + 4 <= data.len()
|
||||
&& data[i] == 0
|
||||
&& data[i + 1] == 0
|
||||
&& data[i + 2] == 0
|
||||
&& data[i + 3] == 1
|
||||
{
|
||||
4
|
||||
} else if i + 3 <= data.len() && data[i] == 0 && data[i + 1] == 0 && data[i + 2] == 1 {
|
||||
3
|
||||
} else {
|
||||
i += 1;
|
||||
continue;
|
||||
};
|
||||
|
||||
let nal_start = i + start_code_len;
|
||||
if nal_start >= data.len() {
|
||||
break;
|
||||
}
|
||||
|
||||
let nal_type = data[nal_start] & 0x1F;
|
||||
|
||||
let mut nal_end = data.len();
|
||||
let mut j = nal_start + 1;
|
||||
while j + 3 <= data.len() {
|
||||
if (data[j] == 0 && data[j + 1] == 0 && data[j + 2] == 1)
|
||||
|| (j + 4 <= data.len()
|
||||
&& data[j] == 0
|
||||
&& data[j + 1] == 0
|
||||
&& data[j + 2] == 0
|
||||
&& data[j + 3] == 1)
|
||||
{
|
||||
nal_end = j;
|
||||
break;
|
||||
fn next_start_code(data: &[u8], mut offset: usize) -> Option<(usize, usize)> {
|
||||
while offset + 3 <= data.len() {
|
||||
if data[offset] == 0 && data[offset + 1] == 0 {
|
||||
if data[offset + 2] == 1 {
|
||||
return Some((offset, 3));
|
||||
}
|
||||
j += 1;
|
||||
}
|
||||
|
||||
match nal_type {
|
||||
7 => {
|
||||
sps = Some(data[nal_start..nal_end].to_vec());
|
||||
if offset + 4 <= data.len() && data[offset + 2] == 0 && data[offset + 3] == 1 {
|
||||
return Some((offset, 4));
|
||||
}
|
||||
8 => {
|
||||
pps = Some(data[nal_start..nal_end].to_vec());
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
i = nal_end;
|
||||
offset += 1;
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
(sps, pps)
|
||||
pub fn inspect_annex_b(data: &[u8]) -> H264AccessUnit<'_> {
|
||||
let mut inspection = H264AccessUnit::default();
|
||||
let mut start_code = next_start_code(data, 0);
|
||||
while let Some((offset, prefix_length)) = start_code {
|
||||
let nal_start = offset + prefix_length;
|
||||
let next = next_start_code(data, nal_start);
|
||||
let nal_end = next.map_or(data.len(), |(next_offset, _)| next_offset);
|
||||
if nal_start < nal_end {
|
||||
match data[nal_start] & 0x1f {
|
||||
5 => inspection.is_idr = true,
|
||||
7 => inspection.sps = Some(&data[nal_start..nal_end]),
|
||||
8 => inspection.pps = Some(&data[nal_start..nal_end]),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
start_code = next;
|
||||
}
|
||||
inspection
|
||||
}
|
||||
|
||||
pub fn extract_sps_pps(data: &[u8]) -> (Option<Vec<u8>>, Option<Vec<u8>>) {
|
||||
let inspection = inspect_annex_b(data);
|
||||
(
|
||||
inspection.sps.map(<[u8]>::to_vec),
|
||||
inspection.pps.map(<[u8]>::to_vec),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn has_sps_pps(data: &[u8]) -> bool {
|
||||
@@ -215,8 +206,9 @@ pub fn parse_profile_level_id_from_sps(sps: &[u8]) -> Option<String> {
|
||||
}
|
||||
|
||||
pub fn extract_profile_level_id(data: &[u8]) -> Option<String> {
|
||||
let (sps, _) = extract_sps_pps(data);
|
||||
sps.and_then(|sps_data| parse_profile_level_id_from_sps(&sps_data))
|
||||
inspect_annex_b(data)
|
||||
.sps
|
||||
.and_then(parse_profile_level_id_from_sps)
|
||||
}
|
||||
|
||||
pub fn is_annex_b(data: &[u8]) -> bool {
|
||||
@@ -292,6 +284,58 @@ pub fn normalize_for_webrtc(data: &[u8]) -> Vec<u8> {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn inspects_parameter_sets_and_idr_in_one_pass() {
|
||||
let data = [
|
||||
0, 0, 0, 1, 0x67, 0x42, 0xe0, 0x1f, 0, 0, 1, 0x68, 0xce, 0, 0, 0, 1, 0x65, 0x88,
|
||||
];
|
||||
let inspection = inspect_annex_b(&data);
|
||||
assert_eq!(inspection.sps, Some(&data[4..8]));
|
||||
assert_eq!(inspection.pps, Some(&data[11..13]));
|
||||
assert!(inspection.is_idr);
|
||||
assert!(std::ptr::eq(
|
||||
inspection.sps.unwrap().as_ptr(),
|
||||
data[4..].as_ptr()
|
||||
));
|
||||
assert_eq!(extract_profile_level_id(&data).as_deref(), Some("42e01f"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn predicted_packet_has_no_parameter_sets_or_idr() {
|
||||
let data = [0, 0, 0, 1, 0x41, 0xc0];
|
||||
let inspection = inspect_annex_b(&data);
|
||||
assert!(inspection.sps.is_none());
|
||||
assert!(inspection.pps.is_none());
|
||||
assert!(!inspection.is_idr);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_and_truncated_nal_units_are_ignored() {
|
||||
for data in [
|
||||
&[][..],
|
||||
&[0][..],
|
||||
&[0, 0][..],
|
||||
&[0, 0, 1][..],
|
||||
&[0, 0, 0, 1][..],
|
||||
&[0, 0, 1, 0, 0, 0, 1][..],
|
||||
] {
|
||||
let inspection = inspect_annex_b(data);
|
||||
assert!(inspection.sps.is_none());
|
||||
assert!(inspection.pps.is_none());
|
||||
assert!(!inspection.is_idr);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inspection_keeps_last_parameter_sets() {
|
||||
let data = [
|
||||
0, 0, 1, 0x67, 0x42, 0xc0, 0x1e, 0, 0, 0, 1, 0x67, 0x42, 0xe0, 0x1f,
|
||||
];
|
||||
let inspection = inspect_annex_b(&data);
|
||||
assert_eq!(inspection.sps, Some(&data[11..15]));
|
||||
assert_eq!(extract_profile_level_id(&data).as_deref(), Some("42e01f"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_h264_keyframes() {
|
||||
let idr_frame = vec![0x00, 0x00, 0x00, 0x01, 0x65];
|
||||
|
||||
100
src/video/pipeline/delivery.rs
Normal file
100
src/video/pipeline/delivery.rs
Normal file
@@ -0,0 +1,100 @@
|
||||
use std::collections::VecDeque;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use parking_lot::Mutex;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
use super::frame_mailbox::FrameMailbox;
|
||||
use super::EncodedVideoFrame;
|
||||
use crate::video::frame::VideoFrame;
|
||||
|
||||
pub(super) type EncodedBatch = Arc<[Arc<EncodedVideoFrame>]>;
|
||||
pub(super) type FrameMailboxHandle = Arc<Mutex<Option<Arc<FrameMailbox<Arc<VideoFrame>>>>>>;
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct DeliveryStats {
|
||||
pub sent_frames: AtomicU64,
|
||||
pub send_time_ns: AtomicU64,
|
||||
pub send_errors: AtomicU64,
|
||||
}
|
||||
|
||||
pub(super) struct VideoSubscriber {
|
||||
pub sender: mpsc::Sender<EncodedBatch>,
|
||||
pub needs_keyframe: AtomicBool,
|
||||
pub recovery_requested: AtomicBool,
|
||||
}
|
||||
|
||||
pub(super) struct VideoFrameReservation {
|
||||
pub subscriber: Arc<VideoSubscriber>,
|
||||
pub permit: mpsc::OwnedPermit<EncodedBatch>,
|
||||
}
|
||||
|
||||
pub struct EncodedVideoFrameReceiver {
|
||||
receiver: mpsc::Receiver<EncodedBatch>,
|
||||
pending: VecDeque<Arc<EncodedVideoFrame>>,
|
||||
mailbox: FrameMailboxHandle,
|
||||
stats: Arc<DeliveryStats>,
|
||||
}
|
||||
|
||||
impl EncodedVideoFrameReceiver {
|
||||
pub(super) fn new(
|
||||
receiver: mpsc::Receiver<EncodedBatch>,
|
||||
mailbox: FrameMailboxHandle,
|
||||
stats: Arc<DeliveryStats>,
|
||||
) -> Self {
|
||||
Self {
|
||||
receiver,
|
||||
pending: VecDeque::new(),
|
||||
mailbox,
|
||||
stats,
|
||||
}
|
||||
}
|
||||
|
||||
fn notify_capacity(&self) {
|
||||
if let Some(mailbox) = self.mailbox.lock().as_ref() {
|
||||
mailbox.notify();
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn recv(&mut self) -> Option<Arc<EncodedVideoFrame>> {
|
||||
loop {
|
||||
if let Some(frame) = self.pending.pop_front() {
|
||||
return Some(frame);
|
||||
}
|
||||
let batch = self.receiver.recv().await?;
|
||||
self.pending.extend(batch.iter().cloned());
|
||||
self.notify_capacity();
|
||||
}
|
||||
}
|
||||
|
||||
pub fn try_recv(&mut self) -> Result<Arc<EncodedVideoFrame>, mpsc::error::TryRecvError> {
|
||||
loop {
|
||||
if let Some(frame) = self.pending.pop_front() {
|
||||
return Ok(frame);
|
||||
}
|
||||
let batch = self.receiver.try_recv()?;
|
||||
self.pending.extend(batch.iter().cloned());
|
||||
self.notify_capacity();
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn record_send(&self, elapsed: Duration, succeeded: bool) {
|
||||
self.stats
|
||||
.send_time_ns
|
||||
.fetch_add(elapsed.as_nanos() as u64, Ordering::Relaxed);
|
||||
if succeeded {
|
||||
self.stats.sent_frames.fetch_add(1, Ordering::Relaxed);
|
||||
} else {
|
||||
self.stats.send_errors.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for EncodedVideoFrameReceiver {
|
||||
fn drop(&mut self) {
|
||||
self.receiver.close();
|
||||
self.notify_capacity();
|
||||
}
|
||||
}
|
||||
@@ -256,60 +256,80 @@ fn run(
|
||||
}
|
||||
|
||||
let capture = active.as_mut().expect("opened capture");
|
||||
let result = match capture.encoder.as_mut().expect("active encoder") {
|
||||
CaptureEncoder::Dma(encoder) => {
|
||||
let pts = pipeline.pts_ms();
|
||||
capture
|
||||
let reservations = pipeline.reserve_outputs();
|
||||
let result = if reservations.as_ref().is_none_or(Vec::is_empty) {
|
||||
match capture.encoder.as_mut().expect("active encoder") {
|
||||
CaptureEncoder::Dma(_) => capture
|
||||
.stream
|
||||
.with_next_dmabuf(|index, bytes_used, fresh_fd| {
|
||||
let keyframe = pipeline.keyframe_requested.swap(false, Ordering::AcqRel);
|
||||
// The callback holds the dequeue lease until native encode
|
||||
// completes (or destroys MPP on error), before QBUF.
|
||||
unsafe { encoder.encode(index, bytes_used, fresh_fd, pts, keyframe) }
|
||||
})
|
||||
.map(|(_, packet)| {
|
||||
packet
|
||||
.map(|packet| {
|
||||
let data = Bytes::from(packet);
|
||||
let is_keyframe = match config.output_codec {
|
||||
VideoEncoderType::H264 => h264_bitstream::is_keyframe(&data),
|
||||
VideoEncoderType::H265 => h265_bitstream::is_keyframe(&data),
|
||||
_ => false,
|
||||
};
|
||||
let (data, is_keyframe) = pipeline.inspect_and_parameterize_packet(
|
||||
config.output_codec,
|
||||
data,
|
||||
is_keyframe,
|
||||
);
|
||||
if config.output_codec == VideoEncoderType::H264 {
|
||||
pipeline.update_h264_profile_level_id(&data);
|
||||
}
|
||||
vec![EncodedVideoFrame {
|
||||
data,
|
||||
pts_ms: pts,
|
||||
is_keyframe,
|
||||
sequence: pipeline.sequence.fetch_add(1, Ordering::Relaxed) + 1,
|
||||
duration: Duration::from_micros(
|
||||
1_000_000 / config.fps.max(1) as u64,
|
||||
),
|
||||
codec: config.output_codec,
|
||||
}]
|
||||
})
|
||||
.map_err(AppError::VideoError)
|
||||
})
|
||||
.with_next_dmabuf(|_, _, _| ())
|
||||
.map(|_| Ok(Vec::new())),
|
||||
CaptureEncoder::Copy(_) => {
|
||||
let mut raw = buffer_pool.take(0);
|
||||
let result = capture.stream.next_into(&mut raw).map(|_| Ok(Vec::new()));
|
||||
buffer_pool.put(raw);
|
||||
result
|
||||
}
|
||||
}
|
||||
CaptureEncoder::Copy(encoder) => {
|
||||
let mut raw = buffer_pool.take(0);
|
||||
capture.stream.next_into(&mut raw).map(|meta| {
|
||||
let frame = VideoFrame::from_pooled(
|
||||
Arc::new(FrameBuffer::new(raw, Some(buffer_pool.clone()))),
|
||||
config.resolution,
|
||||
config.input_format,
|
||||
capture.stream.stride(),
|
||||
meta.sequence,
|
||||
);
|
||||
pipeline.encode_frame_sync(encoder, &frame)
|
||||
})
|
||||
} else {
|
||||
match capture.encoder.as_mut().expect("active encoder") {
|
||||
CaptureEncoder::Dma(encoder) => {
|
||||
let pts = pipeline.pts_ms();
|
||||
capture
|
||||
.stream
|
||||
.with_next_dmabuf(|index, bytes_used, fresh_fd| {
|
||||
let keyframe =
|
||||
pipeline.keyframe_requested.swap(false, Ordering::AcqRel);
|
||||
// The callback holds the dequeue lease until native encode
|
||||
// completes (or destroys MPP on error), before QBUF.
|
||||
unsafe { encoder.encode(index, bytes_used, fresh_fd, pts, keyframe) }
|
||||
})
|
||||
.map(|(_, packet)| {
|
||||
packet
|
||||
.map(|packet| {
|
||||
let data = Bytes::from(packet);
|
||||
let is_keyframe = match config.output_codec {
|
||||
VideoEncoderType::H264 => {
|
||||
h264_bitstream::is_keyframe(&data)
|
||||
}
|
||||
VideoEncoderType::H265 => {
|
||||
h265_bitstream::is_keyframe(&data)
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
let (data, is_keyframe) = pipeline
|
||||
.inspect_and_parameterize_packet(
|
||||
config.output_codec,
|
||||
data,
|
||||
is_keyframe,
|
||||
);
|
||||
vec![EncodedVideoFrame {
|
||||
data,
|
||||
pts_ms: pts,
|
||||
is_keyframe,
|
||||
sequence: pipeline.sequence.fetch_add(1, Ordering::Relaxed)
|
||||
+ 1,
|
||||
duration: Duration::from_micros(
|
||||
1_000_000 / config.fps.max(1) as u64,
|
||||
),
|
||||
codec: config.output_codec,
|
||||
}]
|
||||
})
|
||||
.map_err(AppError::VideoError)
|
||||
})
|
||||
}
|
||||
CaptureEncoder::Copy(encoder) => {
|
||||
let mut raw = buffer_pool.take(0);
|
||||
capture.stream.next_into(&mut raw).map(|meta| {
|
||||
let frame = VideoFrame::from_pooled(
|
||||
Arc::new(FrameBuffer::new(raw, Some(buffer_pool.clone()))),
|
||||
config.resolution,
|
||||
config.input_format,
|
||||
capture.stream.stride(),
|
||||
meta.sequence,
|
||||
);
|
||||
pipeline.encode_frame_sync(encoder, &frame)
|
||||
})
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@@ -321,10 +341,11 @@ fn run(
|
||||
config.input_format,
|
||||
config.fps,
|
||||
));
|
||||
for frame in frames {
|
||||
pipeline.broadcast_encoded(Arc::new(frame));
|
||||
fps_frames += 1;
|
||||
}
|
||||
fps_frames += frames.len() as u32;
|
||||
pipeline.broadcast_encoded_batch(
|
||||
frames.into_iter().map(Arc::new).collect::<Vec<_>>().into(),
|
||||
reservations.unwrap_or_default(),
|
||||
);
|
||||
}
|
||||
Ok(Err(error)) => {
|
||||
if matches!(capture.encoder, Some(CaptureEncoder::Dma(_))) {
|
||||
@@ -380,7 +401,7 @@ fn run(
|
||||
}
|
||||
}
|
||||
if fps_start.elapsed() >= Duration::from_secs(1) {
|
||||
pipeline.stats.blocking_lock().current_fps =
|
||||
pipeline.stats.lock().current_fps =
|
||||
fps_frames as f32 / fps_start.elapsed().as_secs_f32();
|
||||
fps_frames = 0;
|
||||
fps_start = Instant::now();
|
||||
|
||||
@@ -41,6 +41,11 @@ pub(super) trait VideoEncoderTrait: Send {
|
||||
pub(super) struct EncodedFrame {
|
||||
pub(super) data: Bytes,
|
||||
pub(super) key: i32,
|
||||
/// Packet pts in milliseconds, stamped by the encoder when the input
|
||||
/// frame was accepted. With a pipelined encoder a packet can be handed
|
||||
/// back one call later than its input frame, so consumers must use this
|
||||
/// instead of the wall clock time of the current call.
|
||||
pub(super) pts: i64,
|
||||
}
|
||||
|
||||
struct H264EncoderWrapper(H264Encoder);
|
||||
@@ -53,6 +58,7 @@ impl VideoEncoderTrait for H264EncoderWrapper {
|
||||
.map(|f| EncodedFrame {
|
||||
data: f.data,
|
||||
key: f.key,
|
||||
pts: f.pts,
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
@@ -99,6 +105,7 @@ impl VideoEncoderTrait for H265EncoderWrapper {
|
||||
.map(|f| EncodedFrame {
|
||||
data: f.data,
|
||||
key: f.key,
|
||||
pts: f.pts,
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
@@ -126,6 +133,7 @@ impl VideoEncoderTrait for VP8EncoderWrapper {
|
||||
.map(|f| EncodedFrame {
|
||||
data: f.data.into(),
|
||||
key: f.key,
|
||||
pts: f.pts,
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
@@ -151,6 +159,7 @@ impl VideoEncoderTrait for VP9EncoderWrapper {
|
||||
.map(|f| EncodedFrame {
|
||||
data: f.data.into(),
|
||||
key: f.key,
|
||||
pts: f.pts,
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
178
src/video/pipeline/frame_mailbox.rs
Normal file
178
src/video/pipeline/frame_mailbox.rs
Normal file
@@ -0,0 +1,178 @@
|
||||
use std::sync::{Condvar, Mutex};
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub(super) enum PublishResult {
|
||||
Published,
|
||||
Replaced,
|
||||
Rejected,
|
||||
}
|
||||
|
||||
struct FrameState<Frame> {
|
||||
pending: Option<Frame>,
|
||||
last_sequence: Option<u64>,
|
||||
closed: bool,
|
||||
}
|
||||
|
||||
pub(super) struct FrameMailbox<Frame> {
|
||||
state: Mutex<FrameState<Frame>>,
|
||||
available: Condvar,
|
||||
}
|
||||
|
||||
impl<Frame> FrameMailbox<Frame> {
|
||||
pub(super) fn new() -> Self {
|
||||
Self {
|
||||
state: Mutex::new(FrameState {
|
||||
pending: None,
|
||||
last_sequence: None,
|
||||
closed: false,
|
||||
}),
|
||||
available: Condvar::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn publish(&self, sequence: u64, frame: Frame) -> PublishResult {
|
||||
let mut state = self.state.lock().expect("frame mailbox lock poisoned");
|
||||
if state.closed || state.last_sequence.is_some_and(|last| sequence <= last) {
|
||||
return PublishResult::Rejected;
|
||||
}
|
||||
state.last_sequence = Some(sequence);
|
||||
let outcome = if state.pending.replace(frame).is_some() {
|
||||
PublishResult::Replaced
|
||||
} else {
|
||||
PublishResult::Published
|
||||
};
|
||||
self.available.notify_one();
|
||||
outcome
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn receive(&self) -> Option<Frame> {
|
||||
self.receive_when(|| Some(())).map(|(frame, ())| frame)
|
||||
}
|
||||
|
||||
pub(super) fn receive_when<Ready>(
|
||||
&self,
|
||||
mut ready: impl FnMut() -> Option<Ready>,
|
||||
) -> Option<(Frame, Ready)> {
|
||||
let mut state = self.state.lock().expect("frame mailbox lock poisoned");
|
||||
loop {
|
||||
if state.closed {
|
||||
return None;
|
||||
}
|
||||
if state.pending.is_some() {
|
||||
if let Some(ready) = ready() {
|
||||
return state.pending.take().map(|frame| (frame, ready));
|
||||
}
|
||||
}
|
||||
state = self
|
||||
.available
|
||||
.wait(state)
|
||||
.expect("frame mailbox lock poisoned");
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn notify(&self) {
|
||||
let _state = self.state.lock().expect("frame mailbox lock poisoned");
|
||||
self.available.notify_one();
|
||||
}
|
||||
|
||||
pub(super) fn has_pending(&self) -> bool {
|
||||
self.state
|
||||
.lock()
|
||||
.expect("frame mailbox lock poisoned")
|
||||
.pending
|
||||
.is_some()
|
||||
}
|
||||
|
||||
pub(super) fn close(&self) {
|
||||
let mut state = self.state.lock().expect("frame mailbox lock poisoned");
|
||||
state.closed = true;
|
||||
state.pending.take();
|
||||
self.available.notify_all();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::{mpsc, Arc};
|
||||
use std::time::Duration;
|
||||
|
||||
#[test]
|
||||
fn pending_frame_is_replaced_by_latest() {
|
||||
let mailbox = FrameMailbox::new();
|
||||
assert_eq!(mailbox.publish(1, "first"), PublishResult::Published);
|
||||
assert!(mailbox.has_pending());
|
||||
assert_eq!(mailbox.publish(2, "latest"), PublishResult::Replaced);
|
||||
assert_eq!(mailbox.receive(), Some("latest"));
|
||||
assert!(!mailbox.has_pending());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn consumed_sequence_still_rejects_old_and_duplicate_frames() {
|
||||
let mailbox = FrameMailbox::new();
|
||||
assert_eq!(mailbox.publish(10, "current"), PublishResult::Published);
|
||||
assert_eq!(mailbox.receive(), Some("current"));
|
||||
assert_eq!(mailbox.publish(9, "old"), PublishResult::Rejected);
|
||||
assert_eq!(mailbox.publish(10, "duplicate"), PublishResult::Rejected);
|
||||
assert!(!mailbox.has_pending());
|
||||
assert_eq!(mailbox.publish(11, "next"), PublishResult::Published);
|
||||
assert_eq!(mailbox.receive(), Some("next"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn close_discards_pending_and_rejects_future_frames() {
|
||||
let mailbox = FrameMailbox::new();
|
||||
mailbox.publish(1, "pending");
|
||||
mailbox.close();
|
||||
assert!(!mailbox.has_pending());
|
||||
assert_eq!(mailbox.receive(), None);
|
||||
assert_eq!(mailbox.publish(2, "late"), PublishResult::Rejected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn receiver_is_woken_by_publish() {
|
||||
for sequence in 0..64 {
|
||||
let mailbox = Arc::new(FrameMailbox::new());
|
||||
let receiver = mailbox.clone();
|
||||
let (ready_sender, ready_receiver) = mpsc::channel();
|
||||
let (result_sender, result_receiver) = mpsc::channel();
|
||||
let worker = std::thread::spawn(move || {
|
||||
ready_sender.send(()).unwrap();
|
||||
result_sender.send(receiver.receive()).unwrap();
|
||||
});
|
||||
ready_receiver.recv().unwrap();
|
||||
mailbox.publish(sequence, sequence);
|
||||
assert_eq!(
|
||||
result_receiver
|
||||
.recv_timeout(Duration::from_secs(2))
|
||||
.unwrap(),
|
||||
Some(sequence)
|
||||
);
|
||||
worker.join().unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn close_wakes_idle_receiver() {
|
||||
for _ in 0..64 {
|
||||
let mailbox = Arc::new(FrameMailbox::<u64>::new());
|
||||
let receiver = mailbox.clone();
|
||||
let (ready_sender, ready_receiver) = mpsc::channel();
|
||||
let (result_sender, result_receiver) = mpsc::channel();
|
||||
let worker = std::thread::spawn(move || {
|
||||
ready_sender.send(()).unwrap();
|
||||
result_sender.send(receiver.receive()).unwrap();
|
||||
});
|
||||
ready_receiver.recv().unwrap();
|
||||
mailbox.close();
|
||||
assert_eq!(
|
||||
result_receiver
|
||||
.recv_timeout(Duration::from_secs(2))
|
||||
.unwrap(),
|
||||
None
|
||||
);
|
||||
worker.join().unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,11 @@
|
||||
//! Video processing pipelines.
|
||||
|
||||
mod delivery;
|
||||
mod encoder_state;
|
||||
mod frame_mailbox;
|
||||
mod shared;
|
||||
|
||||
pub use delivery::EncodedVideoFrameReceiver;
|
||||
pub use shared::{
|
||||
EncodedVideoFrame, PipelineAppliedConfig, PipelineLifecycle, PipelineStateNotification,
|
||||
SharedVideoPipeline, SharedVideoPipelineConfig, SharedVideoPipelineStats,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -697,9 +697,7 @@ impl VideoStreamManager {
|
||||
/// Returns None if video capture cannot be started or pipeline creation fails.
|
||||
pub async fn subscribe_encoded_frames(
|
||||
&self,
|
||||
) -> Option<
|
||||
tokio::sync::mpsc::Receiver<std::sync::Arc<crate::video::pipeline::EncodedVideoFrame>>,
|
||||
> {
|
||||
) -> Option<crate::video::pipeline::EncodedVideoFrameReceiver> {
|
||||
// 1. Ensure video capture is initialized (for config discovery)
|
||||
if self.streamer.state().await == StreamerState::Uninitialized {
|
||||
tracing::info!("Initializing video capture for encoded frame subscription");
|
||||
|
||||
@@ -1328,7 +1328,7 @@ impl WebConfigUpdate {
|
||||
}
|
||||
}
|
||||
// Cert and key must be provided together (cryptographic validity is checked in the
|
||||
// handler via `RustlsConfig::from_pem`, same stack as the running HTTPS server).
|
||||
// handler via `server_config_from_pem`, same stack as the running HTTPS server).
|
||||
match (&self.ssl_cert_pem, &self.ssl_key_pem) {
|
||||
(Some(_cert), Some(_key)) => {}
|
||||
(Some(_), None) => {
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
use axum::{extract::State, Json};
|
||||
use axum_server::tls_rustls::RustlsConfig;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::error::{AppError, Result};
|
||||
use crate::state::AppState;
|
||||
use crate::utils::tls::server_config_from_pem;
|
||||
|
||||
use super::types::{WebConfigResponse, WebConfigUpdate};
|
||||
|
||||
@@ -27,8 +27,7 @@ pub async fn update_web_config(
|
||||
) =
|
||||
(&req.ssl_cert_pem, &req.ssl_key_pem)
|
||||
{
|
||||
RustlsConfig::from_pem(cert_pem.as_bytes().to_vec(), key_pem.as_bytes().to_vec())
|
||||
.await
|
||||
server_config_from_pem(cert_pem.as_bytes(), key_pem.as_bytes())
|
||||
.map_err(|e| {
|
||||
AppError::BadRequest(
|
||||
format!(
|
||||
@@ -88,16 +87,12 @@ pub async fn update_web_config(
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rustls::crypto::{ring, CryptoProvider};
|
||||
|
||||
#[tokio::test]
|
||||
async fn rustls_accepts_rcgen_self_signed_pem() {
|
||||
let _ = CryptoProvider::install_default(ring::default_provider());
|
||||
#[test]
|
||||
fn openssl_accepts_rcgen_self_signed_pem() {
|
||||
let cert = rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap();
|
||||
let cert_pem = cert.cert.pem();
|
||||
let key_pem = cert.signing_key.serialize_pem();
|
||||
RustlsConfig::from_pem(cert_pem.into_bytes(), key_pem.into_bytes())
|
||||
.await
|
||||
.unwrap();
|
||||
server_config_from_pem(cert_pem.as_bytes(), key_pem.as_bytes()).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -37,9 +37,8 @@ use crate::error::{AppError, Result};
|
||||
use crate::hid::datachannel::{parse_hid_message, HidChannelEvent};
|
||||
use crate::hid::HidController;
|
||||
use crate::video::codec::{h264_bitstream, h265_bitstream};
|
||||
use crate::video::types::{
|
||||
BitratePreset, EncodedVideoFrame, PixelFormat, Resolution, VideoEncoderType,
|
||||
};
|
||||
use crate::video::pipeline::EncodedVideoFrameReceiver;
|
||||
use crate::video::types::{BitratePreset, PixelFormat, Resolution, VideoEncoderType};
|
||||
|
||||
const MIME_TYPE_H265: &str = "video/H265";
|
||||
const KEYFRAME_RETRY_LIMIT: u8 = 3;
|
||||
@@ -527,7 +526,7 @@ impl UniversalSession {
|
||||
/// `on_connected` runs once ICE is up (e.g. request a keyframe).
|
||||
pub async fn start_from_video_pipeline(
|
||||
&self,
|
||||
mut frame_rx: tokio::sync::mpsc::Receiver<std::sync::Arc<EncodedVideoFrame>>,
|
||||
mut frame_rx: EncodedVideoFrameReceiver,
|
||||
request_keyframe: Arc<dyn Fn() + Send + Sync + 'static>,
|
||||
) {
|
||||
if let Some(handle) = self.video_receiver_handle.lock().await.take() {
|
||||
@@ -670,6 +669,7 @@ impl UniversalSession {
|
||||
}
|
||||
}
|
||||
|
||||
let send_started = Instant::now();
|
||||
let send_result = video_track
|
||||
.write_frame_bytes_at(
|
||||
encoded_frame.data.clone(),
|
||||
@@ -677,6 +677,7 @@ impl UniversalSession {
|
||||
Some(encoded_frame.pts_ms),
|
||||
)
|
||||
.await;
|
||||
frame_rx.record_send(send_started.elapsed(), send_result.is_ok());
|
||||
match send_result {
|
||||
Ok(()) => {
|
||||
frames_sent += 1;
|
||||
|
||||
@@ -4,9 +4,9 @@ use bytes::Bytes;
|
||||
use rtp::codecs::h264::H264Payloader;
|
||||
use rtp::packetizer::Payloader;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::sync::Mutex;
|
||||
use tracing::{debug, trace, warn};
|
||||
use tracing::{debug, info, trace, warn};
|
||||
use webrtc::media::Sample;
|
||||
use webrtc::rtp_transceiver::rtp_codec::RTCRtpCodecCapability;
|
||||
use webrtc::track::track_local::track_local_static_rtp::TrackLocalStaticRTP;
|
||||
@@ -191,6 +191,63 @@ struct H264TrackState {
|
||||
clock: H264RtpClock,
|
||||
sps: Option<Vec<u8>>,
|
||||
pps: Option<Vec<u8>>,
|
||||
performance: H264SendPerformance,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct H264SendTiming {
|
||||
prepare: Duration,
|
||||
lock: Duration,
|
||||
packetize: Duration,
|
||||
write: Duration,
|
||||
}
|
||||
|
||||
struct H264SendPerformance {
|
||||
last_report: Instant,
|
||||
frames: u64,
|
||||
packets: u64,
|
||||
bytes: u64,
|
||||
timing: H264SendTiming,
|
||||
}
|
||||
|
||||
impl H264SendPerformance {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
last_report: Instant::now(),
|
||||
frames: 0,
|
||||
packets: 0,
|
||||
bytes: 0,
|
||||
timing: H264SendTiming::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn record(&mut self, bytes: usize, packets: usize, timing: H264SendTiming) {
|
||||
self.frames += 1;
|
||||
self.packets += packets as u64;
|
||||
self.bytes += bytes as u64;
|
||||
self.timing.prepare += timing.prepare;
|
||||
self.timing.lock += timing.lock;
|
||||
self.timing.packetize += timing.packetize;
|
||||
self.timing.write += timing.write;
|
||||
let elapsed = self.last_report.elapsed();
|
||||
if elapsed < Duration::from_secs(5) {
|
||||
return;
|
||||
}
|
||||
let average_ms = |duration: Duration| duration.as_secs_f64() * 1000.0 / self.frames as f64;
|
||||
info!(
|
||||
"[VideoSendPerf] window_s={:.2} frames={} packets_per_frame={:.2} frame_kib={:.2} prepare_ms={:.3} lock_ms={:.3} packetize_ms={:.3} write_ms={:.3} write_us_per_packet={:.3}",
|
||||
elapsed.as_secs_f64(),
|
||||
self.frames,
|
||||
self.packets as f64 / self.frames as f64,
|
||||
self.bytes as f64 / self.frames as f64 / 1024.0,
|
||||
average_ms(self.timing.prepare),
|
||||
average_ms(self.timing.lock),
|
||||
average_ms(self.timing.packetize),
|
||||
average_ms(self.timing.write),
|
||||
self.timing.write.as_secs_f64() * 1_000_000.0 / self.packets.max(1) as f64,
|
||||
);
|
||||
*self = Self::new();
|
||||
}
|
||||
}
|
||||
|
||||
pub struct UniversalVideoTrack {
|
||||
@@ -199,6 +256,7 @@ pub struct UniversalVideoTrack {
|
||||
config: UniversalVideoTrackConfig,
|
||||
h265_state: Option<Mutex<H265RtpState>>,
|
||||
h264_state: Mutex<H264TrackState>,
|
||||
h264_perf_enabled: bool,
|
||||
}
|
||||
|
||||
impl UniversalVideoTrack {
|
||||
@@ -253,6 +311,7 @@ impl UniversalVideoTrack {
|
||||
increment: 90000 / config.fps.max(1),
|
||||
started: false,
|
||||
},
|
||||
performance: H264SendPerformance::new(),
|
||||
};
|
||||
Self {
|
||||
track,
|
||||
@@ -260,6 +319,7 @@ impl UniversalVideoTrack {
|
||||
config,
|
||||
h265_state,
|
||||
h264_state: Mutex::new(h264_state),
|
||||
h264_perf_enabled: std::env::var("ONE_KVM_VIDEO_STATS").as_deref() == Ok("1"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -309,20 +369,24 @@ impl UniversalVideoTrack {
|
||||
|
||||
/// Keep the stack's H.264 STAP/FU payloader, with explicit RTP timestamps.
|
||||
async fn write_h264_frame(&self, data: Bytes, pts_ms: Option<i64>) -> Result<()> {
|
||||
let normalized = h264_bitstream::normalize_for_webrtc(data.as_ref());
|
||||
let mut data = Bytes::from(normalized);
|
||||
let prepare_started = self.h264_perf_enabled.then(Instant::now);
|
||||
let mut data = h264_bitstream::normalize_annex_b(data);
|
||||
|
||||
let idr = h264_bitstream::is_keyframe(data.as_ref());
|
||||
let has_parameter_sets = h264_bitstream::has_sps_pps(data.as_ref());
|
||||
let inspection = h264_bitstream::inspect_annex_b(data.as_ref());
|
||||
let idr = inspection.is_idr;
|
||||
let has_parameter_sets = inspection.sps.is_some() && inspection.pps.is_some();
|
||||
|
||||
let prepare_time = prepare_started.map(|started| started.elapsed());
|
||||
let lock_started = self.h264_perf_enabled.then(Instant::now);
|
||||
let mut state = self.h264_state.lock().await;
|
||||
let lock_time = lock_started.map(|started| started.elapsed());
|
||||
let packetize_started = self.h264_perf_enabled.then(Instant::now);
|
||||
{
|
||||
let (sps, pps) = h264_bitstream::extract_sps_pps(data.as_ref());
|
||||
if let Some(sps) = sps {
|
||||
state.sps = Some(sps);
|
||||
if let Some(sps) = inspection.sps {
|
||||
state.sps = Some(sps.to_vec());
|
||||
}
|
||||
if let Some(pps) = pps {
|
||||
state.pps = Some(pps);
|
||||
if let Some(pps) = inspection.pps {
|
||||
state.pps = Some(pps.to_vec());
|
||||
}
|
||||
|
||||
if idr && !has_parameter_sets {
|
||||
@@ -351,6 +415,8 @@ impl UniversalVideoTrack {
|
||||
}
|
||||
let timestamp = state.clock.next(pts_ms);
|
||||
let count = payloads.len();
|
||||
let packetize_time = packetize_started.map(|started| started.elapsed());
|
||||
let write_started = self.h264_perf_enabled.then(Instant::now);
|
||||
// Serialize complete access units, including their sequence allocation.
|
||||
for (index, payload) in payloads.into_iter().enumerate() {
|
||||
let sequence_number = state.sequence_number;
|
||||
@@ -370,6 +436,18 @@ impl UniversalVideoTrack {
|
||||
.await
|
||||
.map_err(|e| AppError::WebRtcError(format!("H264 RTP write failed: {e}")))?;
|
||||
}
|
||||
if let Some(started) = write_started {
|
||||
state.performance.record(
|
||||
data.len(),
|
||||
count,
|
||||
H264SendTiming {
|
||||
prepare: prepare_time.unwrap_or_default(),
|
||||
lock: lock_time.unwrap_or_default(),
|
||||
packetize: packetize_time.unwrap_or_default(),
|
||||
write: started.elapsed(),
|
||||
},
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -533,6 +611,108 @@ mod tests {
|
||||
assert_eq!(clock.next(None), 6092);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn h264_zero_copy_input_preserves_rtp_payloads() {
|
||||
let mut annex_b = vec![
|
||||
0, 0, 0, 1, 0x09, 0xf0, 0, 0, 1, 0x67, 0x42, 0xe0, 0x1f, 0, 0, 0, 1, 0x68, 0xce, 0, 0,
|
||||
1, 0x0c, 0xff, 0, 0, 0, 1, 0x65,
|
||||
];
|
||||
annex_b.extend_from_slice(&vec![0x55; 2500]);
|
||||
let mut avcc = vec![0, 0, 0, 2, 0x09, 0xf0, 0, 0, 1, 1, 0x65];
|
||||
avcc.extend_from_slice(&vec![0x55; 256]);
|
||||
let inputs = vec![
|
||||
annex_b,
|
||||
avcc,
|
||||
vec![0, 0, 1, 0x41, 0x88, 0x55],
|
||||
vec![0x41, 0x88, 0x55],
|
||||
vec![0, 0, 0, 1, 0x09, 0xf0, 0, 0, 1, 0x0c, 0xff],
|
||||
Vec::new(),
|
||||
];
|
||||
for input in inputs {
|
||||
let previous = Bytes::from(h264_bitstream::normalize_for_webrtc(&input));
|
||||
let current = h264_bitstream::normalize_annex_b(Bytes::from(input));
|
||||
let mut previous_payloader = H264Payloader::default();
|
||||
let mut current_payloader = H264Payloader::default();
|
||||
assert_eq!(
|
||||
previous_payloader.payload(RTP_MTU - 12, &previous).unwrap(),
|
||||
current_payloader.payload(RTP_MTU - 12, ¤t).unwrap(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn h264_zero_copy_input_preserves_cached_parameter_sets() {
|
||||
let mut previous_payloader = H264Payloader::default();
|
||||
let mut current_payloader = H264Payloader::default();
|
||||
for input in [
|
||||
Bytes::from_static(&[0, 0, 0, 1, 0x67, 0x42, 0xe0, 0x1f]),
|
||||
Bytes::from_static(&[0, 0, 0, 1, 0x09, 0xf0, 0, 0, 1, 0x68, 0xce]),
|
||||
Bytes::from_static(&[0, 0, 0, 1, 0x65, 0x88, 0x55]),
|
||||
] {
|
||||
let previous = Bytes::from(h264_bitstream::normalize_for_webrtc(&input));
|
||||
let current = h264_bitstream::normalize_annex_b(input.clone());
|
||||
assert_eq!(current.as_ptr(), input.as_ptr());
|
||||
assert_eq!(
|
||||
previous_payloader.payload(RTP_MTU - 12, &previous).unwrap(),
|
||||
current_payloader.payload(RTP_MTU - 12, ¤t).unwrap(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn h264_send_performance_accumulates_stage_timings() {
|
||||
let mut performance = H264SendPerformance::new();
|
||||
for _ in 0..2 {
|
||||
performance.record(
|
||||
1200,
|
||||
2,
|
||||
H264SendTiming {
|
||||
prepare: Duration::from_micros(100),
|
||||
lock: Duration::from_micros(20),
|
||||
packetize: Duration::from_micros(200),
|
||||
write: Duration::from_micros(300),
|
||||
},
|
||||
);
|
||||
}
|
||||
assert_eq!(performance.frames, 2);
|
||||
assert_eq!(performance.bytes, 2400);
|
||||
assert_eq!(performance.packets, 4);
|
||||
assert_eq!(performance.timing.prepare, Duration::from_micros(200));
|
||||
assert_eq!(performance.timing.lock, Duration::from_micros(40));
|
||||
assert_eq!(performance.timing.packetize, Duration::from_micros(400));
|
||||
assert_eq!(performance.timing.write, Duration::from_micros(600));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn srtp_backend_preserves_authentication_and_sequence_rollover() {
|
||||
use webrtc::srtp::context::Context;
|
||||
use webrtc::srtp::protection_profile::ProtectionProfile;
|
||||
|
||||
for profile in [
|
||||
ProtectionProfile::Aes128CmHmacSha1_80,
|
||||
ProtectionProfile::AeadAes128Gcm,
|
||||
] {
|
||||
let key = vec![42; profile.key_len()];
|
||||
let salt = vec![13; profile.salt_len()];
|
||||
let mut sender = Context::new(&key, &salt, profile, None, None).unwrap();
|
||||
let mut receiver = Context::new(&key, &salt, profile, None, None).unwrap();
|
||||
for sequence in [u16::MAX - 1, u16::MAX, 0, 1] {
|
||||
for ssrc in [17u32, 18] {
|
||||
let mut packet = vec![0x55; 1200];
|
||||
packet[..12].copy_from_slice(&[0x80, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]);
|
||||
packet[2..4].copy_from_slice(&sequence.to_be_bytes());
|
||||
packet[8..12].copy_from_slice(&ssrc.to_be_bytes());
|
||||
let encrypted = sender.encrypt_rtp(&packet).unwrap();
|
||||
assert_ne!(&encrypted[12..1200], &packet[12..]);
|
||||
let mut corrupted = encrypted.to_vec();
|
||||
corrupted[13] ^= 1;
|
||||
assert!(receiver.decrypt_rtp(&corrupted).is_err());
|
||||
assert_eq!(receiver.decrypt_rtp(&encrypted).unwrap().as_ref(), packet);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_video_codec_properties() {
|
||||
assert_eq!(VideoCodec::H264.mime_type(), "video/H264");
|
||||
|
||||
Reference in New Issue
Block a user