feat: 增加设备丢失自恢复机制

增加音频设备丢失自恢复机制,完善视频设备丢失自恢复机制

降级部分日志级别,GOSTC key打印脱敏

代码格式化
This commit is contained in:
mofeng-git
2026-05-02 10:54:31 +08:00
parent 52754c862b
commit 12a3f1c947
16 changed files with 929 additions and 60 deletions

View File

@@ -28,6 +28,15 @@ pub fn classify_capture_io_error(err: &io::Error) -> CaptureIoErrorKind {
}
}
pub fn is_device_lost_message(message: &str) -> bool {
message.contains("No such file or directory")
|| message.contains("No such device")
|| message.contains("os error 2")
|| message.contains("ENODEV")
|| message.contains("ENXIO")
|| message.contains("ESHUTDOWN")
}
pub fn capture_error_log_key(err: &io::Error) -> String {
let message = err.to_string();
if message.contains("dqbuf failed") && message.contains("EINVAL") {

View File

@@ -61,6 +61,29 @@ pub struct VideoDeviceInfo {
pub bridge_kind: Option<String>,
}
#[derive(Debug, Clone)]
pub struct VideoDeviceRecoveryHint {
pub path: PathBuf,
pub name: String,
pub driver: String,
pub bus_info: String,
pub card: String,
pub is_capture_card: bool,
}
impl From<&VideoDeviceInfo> for VideoDeviceRecoveryHint {
fn from(device: &VideoDeviceInfo) -> Self {
Self {
path: device.path.clone(),
name: device.name.clone(),
driver: device.driver.clone(),
bus_info: device.bus_info.clone(),
card: device.card.clone(),
is_capture_card: device.is_capture_card,
}
}
}
/// Information about a supported format
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FormatInfo {
@@ -850,7 +873,7 @@ impl VideoDevice {
/// Enumerate all video capture devices
pub fn enumerate_devices() -> Result<Vec<VideoDeviceInfo>> {
info!("Enumerating video devices...");
debug!("Enumerating video devices...");
// First pass: collect candidates that pass the sysfs-based pre-filter.
// This avoids opening orphan /dev/videoN nodes (ENODEV) and m2m codec
@@ -934,6 +957,51 @@ pub fn enumerate_devices() -> Result<Vec<VideoDeviceInfo>> {
Ok(devices)
}
pub fn select_recovery_device(
devices: &[VideoDeviceInfo],
hint: &VideoDeviceRecoveryHint,
) -> Option<VideoDeviceInfo> {
devices
.iter()
.find(|device| device.path == hint.path)
.or_else(|| {
if hint.bus_info.trim().is_empty() {
None
} else {
devices
.iter()
.find(|device| device.bus_info == hint.bus_info)
}
})
.or_else(|| {
if hint.driver.trim().is_empty() || hint.card.trim().is_empty() {
None
} else {
devices
.iter()
.find(|device| device.driver == hint.driver && device.card == hint.card)
}
})
.or_else(|| {
if hint.driver.trim().is_empty() || hint.name.trim().is_empty() {
None
} else {
devices
.iter()
.find(|device| device.driver == hint.driver && device.name == hint.name)
}
})
.or_else(|| {
if hint.is_capture_card {
devices.iter().find(|device| device.is_capture_card)
} else {
None
}
})
.or_else(|| devices.first())
.cloned()
}
/// Collapse platform sub-device nodes that share the same driver + bus_info
/// into a single entry (the one with the highest priority / most formats).
/// Currently applies to the `rkcif` driver on Rockchip SoCs where each
@@ -1215,6 +1283,35 @@ pub fn find_best_device() -> Result<VideoDeviceInfo> {
mod tests {
use super::*;
fn test_device(
path: &str,
name: &str,
driver: &str,
bus_info: &str,
card: &str,
is_capture_card: bool,
priority: u32,
) -> VideoDeviceInfo {
VideoDeviceInfo {
path: PathBuf::from(path),
name: name.to_string(),
driver: driver.to_string(),
bus_info: bus_info.to_string(),
card: card.to_string(),
formats: Vec::new(),
capabilities: DeviceCapabilities {
video_capture: true,
streaming: true,
..Default::default()
},
is_capture_card,
priority,
has_signal: true,
subdev_path: None,
bridge_kind: None,
}
}
#[test]
fn test_pixel_format_conversion() {
let format = PixelFormat::Mjpeg;
@@ -1230,4 +1327,70 @@ mod tests {
assert_eq!(res.height, 1080);
assert!(res.is_valid());
}
#[test]
fn recovery_selection_prefers_original_path() {
let original = test_device(
"/dev/video0",
"USB Capture",
"uvcvideo",
"usb-1",
"USB Capture",
true,
100,
);
let other = test_device(
"/dev/video2",
"USB Capture",
"uvcvideo",
"usb-1",
"USB Capture",
true,
200,
);
let hint = VideoDeviceRecoveryHint::from(&original);
let selected = select_recovery_device(&[other, original.clone()], &hint).unwrap();
assert_eq!(selected.path, original.path);
}
#[test]
fn recovery_selection_matches_bus_info_after_path_change() {
let original = test_device(
"/dev/video0",
"USB Capture",
"uvcvideo",
"usb-1",
"USB Capture",
true,
100,
);
let recovered = test_device(
"/dev/video3",
"USB Capture",
"uvcvideo",
"usb-1",
"USB Capture",
true,
100,
);
let hint = VideoDeviceRecoveryHint::from(&original);
let selected = select_recovery_device(&[recovered.clone()], &hint).unwrap();
assert_eq!(selected.path, recovered.path);
}
#[test]
fn recovery_selection_falls_back_to_capture_priority() {
let hint = VideoDeviceRecoveryHint {
path: PathBuf::from("/dev/video9"),
name: "Gone".to_string(),
driver: "gone".to_string(),
bus_info: String::new(),
card: "Gone".to_string(),
is_capture_card: true,
};
let lower = test_device("/dev/video1", "A", "uvcvideo", "usb-a", "A", true, 10);
let higher = test_device("/dev/video2", "B", "uvcvideo", "usb-b", "B", true, 20);
let selected = select_recovery_device(&[higher.clone(), lower], &hint).unwrap();
assert_eq!(selected.path, higher.path);
}
}

View File

@@ -45,8 +45,8 @@ use crate::error::{AppError, Result};
use crate::utils::LogThrottler;
use crate::video::capture_limits::{should_validate_jpeg_frame, MIN_CAPTURE_FRAME_SIZE};
use crate::video::capture_status::{
capture_error_log_key, classify_capture_io_error, signal_status_from_capture_kind,
CaptureIoErrorKind,
capture_error_log_key, classify_capture_io_error, is_device_lost_message,
signal_status_from_capture_kind, CaptureIoErrorKind,
};
use crate::video::csi_bridge::{self, ProbeResult};
use crate::video::device::parse_bridge_kind;
@@ -272,6 +272,7 @@ pub struct SharedVideoPipeline {
/// Uses AtomicI64 instead of Mutex for lock-free access
pipeline_start_time_ms: AtomicI64,
pending_sync_geometry: ParkingMutex<Option<(Resolution, PixelFormat)>>,
device_lost_reason: ParkingMutex<Option<String>>,
state_notifier: ParkingRwLock<Option<Arc<dyn Fn(PipelineStateNotification) + Send + Sync>>>,
last_state_notification: ParkingMutex<Option<PipelineStateNotification>>,
}
@@ -377,6 +378,7 @@ impl SharedVideoPipeline {
keyframe_requested: AtomicBool::new(false),
pipeline_start_time_ms: AtomicI64::new(0),
pending_sync_geometry: ParkingMutex::new(None),
device_lost_reason: ParkingMutex::new(None),
state_notifier: ParkingRwLock::new(None),
last_state_notification: ParkingMutex::new(None),
});
@@ -388,6 +390,14 @@ impl SharedVideoPipeline {
self.pending_sync_geometry.lock().take()
}
pub fn take_device_lost_reason(&self) -> Option<String> {
self.device_lost_reason.lock().take()
}
fn mark_device_lost(&self, reason: String) {
*self.device_lost_reason.lock() = Some(reason);
}
pub fn set_state_notifier(
&self,
notifier: Option<Arc<dyn Fn(PipelineStateNotification) + Send + Sync>>,
@@ -783,6 +793,7 @@ impl SharedVideoPipeline {
enum OpenResult {
Opened(V4l2rCaptureStream),
NoSignal(SignalStatus),
DeviceLost(String),
Fatal,
}
@@ -807,6 +818,11 @@ impl SharedVideoPipeline {
OpenResult::NoSignal(signal_status_from_capture_kind(&kind))
}
Err(e) => {
let reason = e.to_string();
if is_device_lost_message(&reason) {
error!("Capture device lost during soft-restart: {}", e);
return OpenResult::DeviceLost(reason);
}
error!("Capture soft-restart failed: {}", e);
OpenResult::Fatal
}
@@ -950,6 +966,13 @@ impl SharedVideoPipeline {
std::thread::sleep(Duration::from_millis(wait_ms));
continue;
}
OpenResult::DeviceLost(reason) => {
pipeline.mark_device_lost(reason);
let _ = pipeline.running.send(false);
pipeline.running_flag.store(false, Ordering::Release);
let _ = frame_seq_tx.send(sequence.wrapping_add(1));
break;
}
OpenResult::Fatal => {
let _ = pipeline.running.send(false);
pipeline.running_flag.store(false, Ordering::Release);
@@ -1137,6 +1160,7 @@ impl SharedVideoPipeline {
}
CaptureIoErrorKind::DeviceLost => {
error!("Capture device lost: {}", e);
pipeline.mark_device_lost(e.to_string());
let _ = pipeline.running.send(false);
pipeline.running_flag.store(false, Ordering::Release);
let _ = frame_seq_tx.send(sequence.wrapping_add(1));

View File

@@ -13,7 +13,8 @@ use tracing::{debug, error, info, trace, warn};
use super::csi_bridge;
use super::device::{
enumerate_devices, find_best_device, parse_bridge_kind, VideoDevice, VideoDeviceInfo,
enumerate_devices, find_best_device, parse_bridge_kind, select_recovery_device, VideoDevice,
VideoDeviceInfo, VideoDeviceRecoveryHint,
};
use super::format::{PixelFormat, Resolution};
use super::frame::{FrameBuffer, FrameBufferPool, VideoFrame};
@@ -366,7 +367,7 @@ impl Streamer {
// IMPORTANT: Disconnect all MJPEG clients FIRST before stopping capture
// This prevents race conditions where clients try to reconnect and reopen the device
info!("Disconnecting all MJPEG clients before config change...");
debug!("Disconnecting all MJPEG clients before config change...");
self.mjpeg_handler.disconnect_all_clients();
// Give clients time to receive the disconnect signal and close their connections
@@ -392,7 +393,7 @@ impl Streamer {
*self.state.write().await = StreamerState::Ready;
// Publish "config applied" event
info!(
debug!(
"Publishing StreamConfigApplied event: {}x{} {:?} @ {}fps",
resolution.width, resolution.height, format, fps
);
@@ -408,7 +409,7 @@ impl Streamer {
// Note: We don't auto-start here anymore.
// The stream will be started when MJPEG client connects (handlers.rs:790)
// This avoids race conditions between config change and client reconnection.
info!("Config applied, stream will start when client connects");
debug!("Config applied, stream will start when client connects");
Ok(())
}
@@ -1305,6 +1306,7 @@ impl Streamer {
{
let mut cfg = self.config.write().await;
cfg.device_path = Some(device_info.path.clone());
cfg.format = format;
cfg.resolution = resolution;
}
@@ -1392,6 +1394,20 @@ impl Streamer {
.await
.clone()
.unwrap_or_else(|| "Device lost".to_string());
let recovery_hint = self
.current_device
.read()
.await
.as_ref()
.map(VideoDeviceRecoveryHint::from)
.unwrap_or_else(|| VideoDeviceRecoveryHint {
path: PathBuf::from(&device),
name: String::new(),
driver: String::new(),
bus_info: String::new(),
card: String::new(),
is_capture_card: true,
});
// Store error info
*self.last_lost_device.write().await = Some(device.clone());
@@ -1409,7 +1425,7 @@ impl Streamer {
// Start recovery task
let streamer = Arc::clone(self);
tokio::spawn(async move {
let device_path = device.clone();
let original_device_path = device.clone();
loop {
let attempt = streamer
@@ -1433,13 +1449,13 @@ impl Streamer {
if attempt == 1 || attempt.is_multiple_of(5) {
streamer
.publish_event(SystemEvent::StreamReconnecting {
device: device_path.clone(),
device: original_device_path.clone(),
attempt,
})
.await;
info!(
"Attempting to recover video device {} (attempt {})",
device_path, attempt
original_device_path, attempt
);
}
@@ -1450,9 +1466,28 @@ impl Streamer {
};
tokio::time::sleep(wait).await;
// Check if device file exists
let device_exists = std::path::Path::new(&device_path).exists();
if !device_exists {
let devices = match enumerate_devices() {
Ok(devices) => devices,
Err(e) => {
debug!("Failed to enumerate devices during recovery: {}", e);
continue;
}
};
let Some(device) = select_recovery_device(&devices, &recovery_hint) else {
debug!("No matching video device present yet for recovery");
continue;
};
let device_path = device.path.display().to_string();
if device_path != original_device_path {
info!(
"Recovered video device path changed: {} -> {}",
original_device_path, device_path
);
}
if !std::path::Path::new(&device_path).exists() {
debug!("Device {} not present yet", device_path);
continue;
}