From 694afdd07f3702339f40d1bde4a154d223b68e50 Mon Sep 17 00:00:00 2001 From: uranami Date: Sun, 19 Jul 2026 09:41:54 +0800 Subject: [PATCH 1/3] fix: stop OTG LED listener spin after USB disconnect --- src/hid/otg.rs | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/hid/otg.rs b/src/hid/otg.rs index 83b5ef8b..5dfd9c7e 100644 --- a/src/hid/otg.rs +++ b/src/hid/otg.rs @@ -94,6 +94,7 @@ pub struct OtgBackend { } const HID_WRITE_TIMEOUT_MS: i32 = 20; +const OTG_RUNTIME_POLL_INTERVAL: Duration = Duration::from_millis(500); impl OtgBackend { /// Gadget must already exist; paths come from `OtgService`. @@ -587,7 +588,7 @@ impl OtgBackend { path.display(), err ); - thread::sleep(Duration::from_millis(500)); + thread::sleep(OTG_RUNTIME_POLL_INTERVAL); return false; } } @@ -611,6 +612,7 @@ impl OtgBackend { if revents.contains(PollFlags::POLLERR) || revents.contains(PollFlags::POLLHUP) { *file = None; + thread::sleep(OTG_RUNTIME_POLL_INTERVAL); return true; } @@ -633,8 +635,13 @@ impl OtgBackend { Ok(_) => false, Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => false, Err(err) => { - warn!("OTG keyboard LED listener read failed: {}", err); + if err.raw_os_error() == Some(libc::ESHUTDOWN) { + debug!("OTG keyboard LED listener disconnected: {}", err); + } else { + warn!("OTG keyboard LED listener read failed: {}", err); + } *file = None; + thread::sleep(OTG_RUNTIME_POLL_INTERVAL); true } } @@ -642,6 +649,7 @@ impl OtgBackend { Err(err) => { warn!("OTG keyboard LED listener poll failed: {}", err); *file = None; + thread::sleep(OTG_RUNTIME_POLL_INTERVAL); true } } @@ -676,7 +684,7 @@ impl OtgBackend { changed = true; } - if keyboard_leds_enabled { + if keyboard_leds_enabled && current_udc_configured { if let Some(path) = keyboard_path.as_ref() { changed |= Self::poll_keyboard_led_once( &mut keyboard_led_file, @@ -684,10 +692,11 @@ impl OtgBackend { &led_state, ); } else { - thread::sleep(Duration::from_millis(500)); + thread::sleep(OTG_RUNTIME_POLL_INTERVAL); } } else { - thread::sleep(Duration::from_millis(500)); + keyboard_led_file = None; + thread::sleep(OTG_RUNTIME_POLL_INTERVAL); } if changed { From 4e32b051249edea6e674cd47fd32f870452b8b35 Mon Sep 17 00:00:00 2001 From: arounyf Date: Wed, 22 Jul 2026 04:54:18 +0000 Subject: [PATCH 2/3] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8DOTG=20gadget?= =?UTF-8?q?=E9=87=8D=E5=BB=BA=E5=90=8EHID=E5=90=8E=E7=AB=AF=E6=9C=AA?= =?UTF-8?q?=E9=87=8D=E8=BD=BD=E5=AF=BC=E8=87=B4=E9=94=AE=E9=BC=A0=E5=A4=B1?= =?UTF-8?q?=E6=95=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 当MSD或OTG网络配置变更导致gadget被重建时，新的/dev/hidg*设备 被创建但HID后端因'配置未变'跳过了reload，导致仍持有已删除设备 的旧文件描述符，键鼠操作无效。 修复: 在apply_usb_config中检测gadget是否因MSD/网络变更被重建， 若是且HID后端为OTG，强制reload HID以打开新的/dev/hidg*设备。 同时为MsdConfig添加PartialEq derive以支持变更检测。 --- src/config/schema/web.rs | 2 +- src/web/handlers/config/apply.rs | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/config/schema/web.rs b/src/config/schema/web.rs index 6671855d..c3795c5c 100644 --- a/src/config/schema/web.rs +++ b/src/config/schema/web.rs @@ -44,7 +44,7 @@ impl Default for VideoConfig { } #[typeshare] -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(default)] pub struct MsdConfig { pub enabled: bool, diff --git a/src/web/handlers/config/apply.rs b/src/web/handlers/config/apply.rs index 65d59ef0..24e7a4dd 100644 --- a/src/web/handlers/config/apply.rs +++ b/src/web/handlers/config/apply.rs @@ -365,6 +365,10 @@ pub async fn apply_usb_config( let transitioning_away_from_otg = old_config.hid.backend == HidBackend::Otg && new_config.hid.backend != HidBackend::Otg; + let hid_unchanged = old_config.hid == new_config.hid; + let otg_gadget_rebuilt = + old_config.msd != new_config.msd || old_config.otg_network != new_config.otg_network; + if transitioning_away_from_otg { apply_hid_config( state, @@ -394,6 +398,19 @@ pub async fn apply_usb_config( .await?; } + // When the OTG gadget was rebuilt due to MSD or network config changes + // while HID config stayed the same, the /dev/hidg* devices are new and + // the HID backend must be reloaded to reopen them. + if hid_unchanged && otg_gadget_rebuilt && new_config.hid.backend == HidBackend::Otg { + tracing::info!("OTG gadget rebuilt, reloading HID backend for new devices"); + let hid_backend = hid_backend_type(&new_config.hid); + state + .hid + .reload(hid_backend) + .await + .map_err(|e| AppError::Config(format!("HID reload after gadget rebuild failed: {}", e)))?; + } + apply_msd_config( state, &old_config.msd, From e9bed3688f0c75affea0bbf473745b60b70b06c7 Mon Sep 17 00:00:00 2001 From: arounyf Date: Thu, 23 Jul 2026 06:56:53 +0000 Subject: [PATCH 3/3] feat: UAC USB microphone passthrough - Add UAC1 gadget function (ConfigFS) with optimized endpoint config - Browser mic capture with Opus encoding via WebCodecs AudioEncoder - WebSocket audio transport (Opus 64kbps, 100x bandwidth reduction vs raw PCM) - aplay subprocess for reliable PCM playback to USB gadget - Settings toggle + ActionBar mic button (hidden when UAC disabled) - Dynamic PCM device resolution (/proc/asound) - c_chmask=0 + req_number=4 fixes DWC3 composite isochronous endpoint issue --- src/audio/mod.rs | 2 + src/audio/uac_streamer.rs | 210 +++++++++++++++++++++++++++ src/audio/uac_websocket.rs | 133 +++++++++++++++++ src/config/schema/mod.rs | 2 + src/main.rs | 20 ++- src/otg/manager.rs | 10 ++ src/otg/mod.rs | 6 +- src/otg/service.rs | 67 ++++++++- src/otg/uac.rs | 148 +++++++++++++++++++ src/state.rs | 4 + src/web/handlers/config/apply.rs | 42 +++++- src/web/handlers/config/mod.rs | 4 + src/web/handlers/config/uac.rs | 36 +++++ src/web/handlers/setup.rs | 2 +- src/web/mod.rs | 2 + src/web/routes.rs | 10 ++ src/web/uac_ws.rs | 33 +++++ web/package-lock.json | 26 ++++ web/package.json | 1 + web/src/api/config.ts | 10 ++ web/src/api/index.ts | 1 + web/src/components/ActionBar.vue | 24 +++ web/src/composables/useMicrophone.ts | 161 ++++++++++++++++++++ web/src/i18n/en-US.ts | 2 + web/src/i18n/zh-CN.ts | 5 +- web/src/views/ConsoleView.vue | 11 +- web/src/views/SettingsView.vue | 21 ++- 27 files changed, 977 insertions(+), 16 deletions(-) create mode 100644 src/audio/uac_streamer.rs create mode 100644 src/audio/uac_websocket.rs create mode 100644 src/otg/uac.rs create mode 100644 src/web/handlers/config/uac.rs create mode 100644 src/web/uac_ws.rs create mode 100644 web/src/composables/useMicrophone.ts diff --git a/src/audio/mod.rs b/src/audio/mod.rs index b6973d0e..913f16d0 100644 --- a/src/audio/mod.rs +++ b/src/audio/mod.rs @@ -11,6 +11,8 @@ pub mod monitor; pub mod recovery; pub mod streamer; pub mod types; +pub mod uac_streamer; +pub mod uac_websocket; pub use capture::{AudioCapturer, AudioConfig, AudioFrame}; pub use controller::AudioController; diff --git a/src/audio/uac_streamer.rs b/src/audio/uac_streamer.rs new file mode 100644 index 00000000..7a8edc37 --- /dev/null +++ b/src/audio/uac_streamer.rs @@ -0,0 +1,210 @@ +use std::io::Write; +use std::process::{Child, Command, Stdio}; +use std::time::Duration; + +use tokio::sync::{mpsc, watch}; +use tracing::{info, warn}; + +use crate::error::{AppError, Result}; + +/// Kill aplay after this much idle time (no incoming audio frames). +const IDLE_CLOSE_TIMEOUT_MS: u64 = 2000; + +/// Configuration for the UAC playback stream. +#[derive(Debug, Clone)] +pub struct UacPlaybackConfig { + pub device_name: String, + pub sample_rate: u32, + pub channels: u16, +} + +impl Default for UacPlaybackConfig { + fn default() -> Self { + Self { + device_name: crate::otg::uac::find_uac_pcm_device() + .unwrap_or_else(crate::otg::uac::uac_pcm_device), + sample_rate: 48000, + channels: 2, + } + } +} + +#[derive(Debug, Clone)] +pub struct UacPcmFrame { + pub data: Vec, + pub duration_ms: u32, +} + +/// Writes PCM to the UAC gadget via `aplay` subprocess — same +/// mechanism as the successful manual test: `ffmpeg | aplay hw:0,0`. +#[derive(Clone)] +pub struct UacPlaybackWriter { + pcm_sender: mpsc::Sender, + stop_tx: watch::Sender, +} + +impl UacPlaybackWriter { + pub fn start(config: UacPlaybackConfig) -> Result { + let (pcm_sender, pcm_receiver) = mpsc::channel::(64); + let (stop_tx, stop_rx) = watch::channel(false); + + let device = config.device_name; + let rate = config.sample_rate; + let ch = config.channels; + + let thread_device = device.clone(); + std::thread::Builder::new() + .name("uac-aplay".into()) + .spawn(move || { + Self::playback_loop(&thread_device, rate, ch, pcm_receiver, stop_rx); + info!("UAC aplay thread stopped"); + }) + .map_err(|e| AppError::Internal(format!("spawn: {e}")))?; + + info!("UAC aplay writer started on {device}"); + Ok(Self { pcm_sender, stop_tx }) + } + + pub async fn write(&self, frame: UacPcmFrame) -> Result<()> { + self.pcm_sender.send(frame).await + .map_err(|_| AppError::Internal("UAC channel closed".into())) + } + + pub fn stop(&self) { + let _ = self.stop_tx.send(true); + } + + // ── internals ────────────────────────────────────────── + + fn spawn_aplay(device: &str, rate: u32, ch: u16) -> Option<(Child, Box)> { + let mut cmd = Command::new("aplay"); + cmd.arg("-D").arg(device) + .arg("-f").arg("S16_LE") + .arg("-r").arg(rate.to_string()) + .arg("-c").arg(ch.to_string()) + .arg("-") // stdin + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()); // → journalctl + + match cmd.spawn() { + Ok(mut child) => { + let stdin = child.stdin.take()?; + info!("aplay spawned pid={}", child.id()); + Some((child, Box::new(stdin))) + } + Err(e) => { + warn!("aplay spawn failed: {e}"); + None + } + } + } + + fn kill_aplay(mut child: Child, stdin: Box) { + drop(stdin); // close pipe → EOF for aplay + let _ = child.wait(); + } + + fn playback_loop( + device: &str, + rate: u32, + ch: u16, + mut receiver: mpsc::Receiver, + mut stop_rx: watch::Receiver, + ) { + let idle_timeout = Duration::from_millis(IDLE_CLOSE_TIMEOUT_MS); + let mut aplay: Option<(Child, Box)> = None; + let mut last_write = std::time::Instant::now(); + let mut frame_count: u64 = 0; + let mut byte_count: u64 = 0; + + loop { + // ── wait for frame ────────────────────────── + let need_timeout = aplay.is_some() + && last_write.elapsed() >= idle_timeout; + let deadline = if need_timeout || aplay.is_none() { + Some(std::time::Instant::now() + Duration::from_millis(200)) + } else { + None + }; + + let frame = loop { + if *stop_rx.borrow() { break None; } + match receiver.try_recv() { + Ok(f) => break Some(f), + Err(mpsc::error::TryRecvError::Disconnected) => break None, + Err(mpsc::error::TryRecvError::Empty) => {} + } + if let Some(dl) = deadline { + if std::time::Instant::now() >= dl { + break None; + } + } + std::thread::sleep(Duration::from_millis(20)); + }; + + if *stop_rx.borrow() { + break; + } + + match frame { + Some(f) => { + last_write = std::time::Instant::now(); + + // Ensure aplay is alive + if aplay.is_none() { + aplay = Self::spawn_aplay(device, rate, ch); + } + + if let Some((child, stdin)) = aplay.as_mut() { + // Check child health + match child.try_wait() { + Ok(Some(status)) => { + warn!("aplay died: {status}"); + aplay = Self::spawn_aplay(device, rate, ch); + if aplay.is_none() { continue; } + } + Ok(None) => {} // alive + Err(e) => { + warn!("aplay wait error: {e}"); + aplay = None; + continue; + } + } + } + + if let Some((child, stdin)) = aplay.as_mut() { + match stdin.write_all(&f.data) { + Ok(()) => { + let _ = stdin.flush(); + frame_count += 1; + byte_count += f.data.len() as u64; + } + Err(e) => { + warn!("aplay write error: {e}"); + // aplay died — reap and restart + if let Some((c, s)) = aplay.take() { + Self::kill_aplay(c, s); + } + } + } + } + } + None => { + // Timeout — kill aplay + if let Some((c, s)) = aplay.take() { + Self::kill_aplay(c, s); + } + if *stop_rx.borrow() { + break; + } + } + } + } + + // Cleanup + if let Some((c, s)) = aplay.take() { + Self::kill_aplay(c, s); + } + } +} diff --git a/src/audio/uac_websocket.rs b/src/audio/uac_websocket.rs new file mode 100644 index 00000000..6103131e --- /dev/null +++ b/src/audio/uac_websocket.rs @@ -0,0 +1,133 @@ +use audiopus::coder::Decoder as OpusDecoder; +use audiopus::{Channels, SampleRate}; +use axum::extract::ws::{Message, WebSocket}; +use tracing::{debug, error, info, warn}; + +use super::uac_streamer::{UacPcmFrame, UacPlaybackWriter}; +use std::sync::Arc; + +/// Binary protocol header for UAC audio input. +/// +/// 0x03 — message type (reverse audio / microphone passthrough) +/// timestamp — u32 LE (milliseconds, for future sync) +/// duration — u16 LE (frame duration in ms, typically 20) +/// sequence — u32 LE (frame counter, for loss detection) +/// data_len — u32 LE (Opus payload length in bytes) +const UAC_AUDIO_HEADER_SIZE: usize = 15; +const UAC_AUDIO_MSG_TYPE: u8 = 0x03; + +/// Accept incoming Opus audio frames over WebSocket and route them +/// to the UAC playback writer. +pub async fn handle_uac_audio_ws( + mut ws: WebSocket, + playback: Arc, +) { + // Create an Opus decoder: 48kHz stereo → PCM S16LE. + let mut decoder = match OpusDecoder::new(SampleRate::Hz48000, Channels::Stereo) { + Ok(d) => d, + Err(e) => { + error!("Failed to create Opus decoder: {}", e); + let _ = ws.send(Message::Close(None)).await; + return; + } + }; + + info!("UAC audio WebSocket connected (mic passthrough)"); + + while let Some(msg) = ws.recv().await { + let msg = match msg { + Ok(m) => m, + Err(e) => { + warn!("UAC WebSocket error: {}", e); + break; + } + }; + + match msg { + Message::Binary(data) => { + if data.len() < UAC_AUDIO_HEADER_SIZE { + warn!( + "UAC audio frame too short: {} bytes (min {})", + data.len(), + UAC_AUDIO_HEADER_SIZE + ); + continue; + } + + let msg_type = data[0]; + if msg_type == 0x04 { + // Raw PCM passthrough — no Opus decode needed. + // Useful for testing and for clients that encode locally. + let duration = u16::from_le_bytes([data[5], data[6]]); + let data_len = u32::from_le_bytes([data[11], data[12], data[13], data[14]]) as usize; + if data.len() < UAC_AUDIO_HEADER_SIZE + data_len { + warn!("UAC PCM frame truncated"); + continue; + } + let pcm_bytes = &data[UAC_AUDIO_HEADER_SIZE..UAC_AUDIO_HEADER_SIZE + data_len]; + if let Err(e) = playback + .write(super::uac_streamer::UacPcmFrame { + data: pcm_bytes.to_vec(), + duration_ms: duration as u32, + }) + .await + { + error!("Failed to send UAC PCM frame: {}", e); + break; + } + continue; + } + if msg_type != UAC_AUDIO_MSG_TYPE { + warn!("UAC unknown msg type: 0x{msg_type:02x}"); + continue; + } + + let duration = u16::from_le_bytes([data[5], data[6]]); + let data_len = u32::from_le_bytes([data[11], data[12], data[13], data[14]]) as usize; + + if data.len() < UAC_AUDIO_HEADER_SIZE + data_len { + warn!("UAC audio frame truncated"); + continue; + } + + let opus_payload = &data[UAC_AUDIO_HEADER_SIZE..UAC_AUDIO_HEADER_SIZE + data_len]; + + let frame_samples = (48000u32 * duration as u32 / 1000) as usize * 2; // 2 channels + let mut pcm_i16 = vec![0i16; frame_samples]; + + match decoder.decode(Some(opus_payload), &mut pcm_i16, false) { + Ok(decoded) => { + // Convert i16 → bytes (S16LE interleaved) + let pcm_bytes: Vec = pcm_i16[..decoded] + .iter() + .flat_map(|s| s.to_le_bytes()) + .collect(); + if let Err(e) = playback + .write(UacPcmFrame { + data: pcm_bytes, + duration_ms: duration as u32, + }) + .await + { + error!("Failed to send UAC PCM frame: {}", e); + break; + } + } + Err(e) => { + warn!("Opus decode error: {}", e); + } + } + } + Message::Ping(_) | Message::Pong(_) => {} + Message::Close(_) => { + debug!("UAC audio WebSocket closing"); + break; + } + Message::Text(_) => { + // Ignore text messages + } + } + } + + info!("UAC audio WebSocket disconnected"); +} diff --git a/src/config/schema/mod.rs b/src/config/schema/mod.rs index 161319b2..5dd551eb 100644 --- a/src/config/schema/mod.rs +++ b/src/config/schema/mod.rs @@ -44,6 +44,7 @@ pub struct AppConfig { pub rtsp: RtspConfig, pub redfish: RedfishConfig, pub watchdog: WatchdogConfig, + pub uac: crate::otg::service::UacConfig, } impl AppConfig { @@ -51,6 +52,7 @@ impl AppConfig { if self.hid.backend != HidBackend::Otg { self.msd.enabled = false; self.otg_network.enabled = false; + self.uac.enabled = false; } self.atx.normalize(); } diff --git a/src/main.rs b/src/main.rs index aae6d39f..adf8ac97 100644 --- a/src/main.rs +++ b/src/main.rs @@ -308,7 +308,7 @@ async fn main() -> anyhow::Result<()> { #[cfg(unix)] if let Err(e) = otg_service - .apply_config(&config.hid, &config.msd, &config.otg_network) + .apply_config(&config.hid, &config.msd, &config.otg_network, &config.uac) .await { tracing::warn!("Failed to apply OTG config: {}", e); @@ -579,6 +579,24 @@ async fn main() -> anyhow::Result<()> { data_dir.clone(), ); + // Initialize UAC playback writer if UAC is enabled + if config.uac.enabled { + let uac_cfg = one_kvm::audio::uac_streamer::UacPlaybackConfig { + sample_rate: config.uac.sample_rate, + channels: config.uac.channels as u16, + ..Default::default() + }; + match one_kvm::audio::uac_streamer::UacPlaybackWriter::start(uac_cfg) { + Ok(writer) => { + *state.uac_playback.write().await = Some(writer); + tracing::info!("UAC playback writer started"); + } + Err(e) => { + tracing::warn!("Failed to start UAC playback writer: {}", e); + } + } + } + if config.watchdog.enabled { if let Err(error) = state.watchdog.enable().await { tracing::error!( diff --git a/src/otg/manager.rs b/src/otg/manager.rs index f4eab2d1..94fc65e2 100644 --- a/src/otg/manager.rs +++ b/src/otg/manager.rs @@ -47,6 +47,7 @@ pub struct OtgGadgetManager { hid_instance: u8, msd_instance: u8, network_instance: u8, + uac_instance: u8, functions: Vec>, bound_udc: Option, created_by_us: bool, @@ -73,6 +74,7 @@ impl OtgGadgetManager { hid_instance: 0, msd_instance: 0, network_instance: 0, + uac_instance: 0, functions: Vec::with_capacity(4), bound_udc: None, created_by_us: false, @@ -148,6 +150,14 @@ impl OtgGadgetManager { Ok(func_clone) } + pub fn add_uac(&mut self, sample_rate: u32, channels: u8) -> Result { + let func = super::uac::UacFunction::new(self.uac_instance, sample_rate, channels)?; + let func_clone = func.clone(); + self.add_function(Box::new(func))?; + self.uac_instance += 1; + Ok(func_clone) + } + fn add_function(&mut self, func: Box) -> Result<()> { self.functions.push(func); Ok(()) diff --git a/src/otg/mod.rs b/src/otg/mod.rs index a3dd49ea..9af22aaf 100644 --- a/src/otg/mod.rs +++ b/src/otg/mod.rs @@ -18,6 +18,8 @@ pub mod report_desc; pub mod self_check; #[cfg(unix)] pub mod service; +#[cfg(unix)] +pub mod uac; #[cfg(unix)] pub use manager::{wait_for_hid_devices, OtgGadgetManager}; @@ -26,7 +28,9 @@ pub use msd::{MsdFunction, MsdLunConfig}; #[cfg(unix)] pub use network::NetworkFunction; #[cfg(unix)] -pub use service::{HidDevicePaths, OtgNetworkStatus, OtgRuntimeHealth, OtgService}; +pub use service::{HidDevicePaths, OtgNetworkStatus, OtgRuntimeHealth, OtgService, UacConfig}; +#[cfg(unix)] +pub use uac::UacFunction; /// List USB Device Controller names exposed by sysfs. pub fn list_udc_devices() -> Vec { diff --git a/src/otg/service.rs b/src/otg/service.rs index 707295b0..22e3224c 100644 --- a/src/otg/service.rs +++ b/src/otg/service.rs @@ -12,6 +12,36 @@ use crate::config::{ }; use crate::error::{AppError, Result}; +/// Configuration for the USB Audio Class (UAC) gadget function. +#[derive(Debug, Clone, PartialEq, Eq, Default, serde::Serialize, serde::Deserialize)] +#[serde(default)] +pub struct UacConfig { + /// Enable the virtual USB microphone. + pub enabled: bool, + /// Sample rate in Hz (e.g. 48000). + pub sample_rate: u32, + /// Number of channels (1=mono, 2=stereo). + pub channels: u8, +} + +impl UacConfig { + pub fn validate(&self) -> Result<()> { + if self.sample_rate != 0 && (self.sample_rate < 8000 || self.sample_rate > 384000) { + return Err(AppError::BadRequest(format!( + "UAC sample rate {} out of range (8000-384000)", + self.sample_rate + ))); + } + if self.channels != 0 && self.channels > 8 { + return Err(AppError::BadRequest(format!( + "UAC channel count {} out of range (1-8)", + self.channels + ))); + } + Ok(()) + } +} + #[derive(Debug, Clone, Default)] pub struct HidDevicePaths { pub keyboard: Option, @@ -62,6 +92,7 @@ pub(crate) struct OtgDesiredState { pub msd_enabled: bool, pub msd_lun_capacity: u8, pub network: OtgNetworkConfig, + pub uac_enabled: bool, } impl Default for OtgDesiredState { @@ -74,6 +105,7 @@ impl Default for OtgDesiredState { msd_enabled: false, msd_lun_capacity: 1, network: OtgNetworkConfig::default(), + uac_enabled: false, } } } @@ -83,8 +115,10 @@ impl OtgDesiredState { hid: &HidConfig, msd: &MsdConfig, network: &OtgNetworkConfig, + uac: &UacConfig, ) -> Result { network.validate()?; + uac.validate()?; let hid_functions = if hid.backend == HidBackend::Otg { let functions = hid.constrained_otg_functions(); Some(functions) @@ -93,7 +127,8 @@ impl OtgDesiredState { }; hid.validate_otg_functions()?; - let needs_udc = hid_functions.is_some() || msd.enabled || network.enabled; + let needs_udc = + hid_functions.is_some() || msd.enabled || network.enabled || uac.enabled; let udc = if needs_udc { hid.otg_udc .as_ref() @@ -111,6 +146,7 @@ impl OtgDesiredState { msd_enabled: msd.enabled, msd_lun_capacity: 1, network: network.clone(), + uac_enabled: uac.enabled, }) } @@ -133,6 +169,7 @@ struct OtgServiceState { pub msd_enabled: bool, pub msd_lun_capacity: u8, pub network: OtgNetworkConfig, + pub uac_enabled: bool, pub configured_udc: Option, pub hid_paths: Option, pub hid_functions: Option, @@ -150,6 +187,7 @@ impl Default for OtgServiceState { msd_enabled: false, msd_lun_capacity: 1, network: OtgNetworkConfig::default(), + uac_enabled: false, configured_udc: None, hid_paths: None, hid_functions: None, @@ -215,6 +253,7 @@ impl OtgService { hid: &HidConfig, msd: &MsdConfig, network: &OtgNetworkConfig, + uac: &UacConfig, ) -> Result<()> { if !self.recovery_checked.load(Ordering::SeqCst) { if let Err(error) = NetworkBridgeRuntime::recover_stale_transaction() { @@ -226,7 +265,7 @@ impl OtgService { } let previous = self.desired.read().await.clone(); let desired = self - .desired_from_config_preserving_runtime(hid, msd, network) + .desired_from_config_preserving_runtime(hid, msd, network, uac) .await?; { let mut state = self.state.write().await; @@ -269,8 +308,9 @@ impl OtgService { hid: &HidConfig, msd: &MsdConfig, network: &OtgNetworkConfig, + uac: &UacConfig, ) -> Result { - let mut desired = OtgDesiredState::from_config(hid, msd, network)?; + let mut desired = OtgDesiredState::from_config(hid, msd, network, uac)?; desired.msd_lun_capacity = self.desired.read().await.msd_lun_capacity; Ok(desired) } @@ -305,10 +345,11 @@ impl OtgService { let desired = self.desired.read().await.clone(); debug!( - "Reconciling OTG gadget: HID={}, MSD={}, NET={}, UDC={:?}", + "Reconciling OTG gadget: HID={}, MSD={}, NET={}, UAC={}, UDC={:?}", desired.hid_enabled(), desired.msd_enabled, desired.network_enabled(), + desired.uac_enabled, desired.udc ); @@ -320,6 +361,7 @@ impl OtgService { && state.msd_enabled == desired.msd_enabled && state.msd_lun_capacity == desired.msd_lun_capacity && state.network == desired.network + && state.uac_enabled == desired.uac_enabled && state.configured_udc == desired.udc && state.hid_functions == desired.hid_functions && state.keyboard_leds_enabled == desired.keyboard_leds @@ -359,6 +401,7 @@ impl OtgService { state.msd_enabled = false; state.msd_lun_capacity = 1; state.network = OtgNetworkConfig::default(); + state.uac_enabled = false; state.configured_udc = None; state.hid_paths = None; state.hid_functions = None; @@ -393,6 +436,20 @@ impl OtgService { ); let mut hid_paths = None; + // Add UAC BEFORE HID so the isochronous endpoint gets a + // lower hardware endpoint number. DWC3 seems to have + // trouble with isochronous transfers on higher-numbered + // endpoints when they follow interrupt endpoints. + let _uac_func = if desired.uac_enabled { + let sample_rate: u32 = 48000; + let channels: u8 = 2; + Some(manager.add_uac(sample_rate, channels).map_err(|e| { + AppError::Internal(format!("Failed to add UAC function: {e}")) + })?) + } else { + None + }; + if let Some(hid_functions) = desired.hid_functions.clone() { let mut paths = HidDevicePaths { udc: Some(udc.clone()), @@ -537,6 +594,7 @@ impl OtgService { state.msd_enabled = desired.msd_enabled; state.msd_lun_capacity = desired.msd_lun_capacity; state.network = desired.network.clone(); + state.uac_enabled = desired.uac_enabled; state.configured_udc = Some(udc); state.hid_paths = hid_paths; state.hid_functions = desired.hid_functions; @@ -642,6 +700,7 @@ mod tests { &HidConfig::default(), &MsdConfig::default(), &OtgNetworkConfig::default(), + &UacConfig::default(), ) .await .unwrap(); diff --git a/src/otg/uac.rs b/src/otg/uac.rs new file mode 100644 index 00000000..5b60de13 --- /dev/null +++ b/src/otg/uac.rs @@ -0,0 +1,148 @@ +use std::path::{Path, PathBuf}; + +use tracing::{debug, info}; + +use super::configfs::{create_dir, create_symlink, remove_dir, write_file}; +use super::function::GadgetFunction; +use crate::error::{AppError, Result}; + +/// USB Audio Class 2.0 (UAC1) gadget function. +/// +/// Creates a virtual USB microphone that the USB host sees as a standard +/// USB audio input device. Audio written to the PCM playback device on the +/// gadget side appears as microphone input on the host. +#[derive(Debug, Clone)] +pub struct UacFunction { + name: String, + sample_rate: u32, + channels: u8, +} + +impl UacFunction { + /// Create a new UAC1 function instance. + /// + /// `instance` is a zero-based index to avoid name collisions + /// (e.g. `uac2.usb0`). + pub fn new(instance: u8, sample_rate: u32, channels: u8) -> Result { + if sample_rate == 0 || sample_rate > 384_000 { + return Err(AppError::BadRequest(format!( + "invalid UAC sample rate: {sample_rate}" + ))); + } + if channels == 0 || channels > 8 { + return Err(AppError::BadRequest(format!( + "invalid UAC channel count: {channels}" + ))); + } + Ok(Self { + name: format!("uac1.usb{instance}"), + sample_rate, + channels, + }) + } + + fn function_path(&self, gadget_path: &Path) -> PathBuf { + gadget_path.join("functions").join(&self.name) + } +} + +impl GadgetFunction for UacFunction { + fn name(&self) -> &str { + &self.name + } + + fn create(&self, gadget_path: &Path) -> Result<()> { + let func_path = self.function_path(gadget_path); + create_dir(&func_path)?; + + // Playback direction (gadget → host): the controlled machine sees + // this as a microphone/line-in input. + let chmask: u32 = (1u32 << self.channels) - 1; + write_file(&func_path.join("p_chmask"), &chmask.to_string())?; + write_file(&func_path.join("p_srate"), &self.sample_rate.to_string())?; + write_file(&func_path.join("p_ssize"), "2")?; // 16-bit S16LE + // UAC1 does not need p_hs_bint — Windows has native built-in + // UAC1 drivers and handles isochronous streaming automatically. + + // Only enable playback direction (gadget → host = mic). + // Disabling capture saves one isochronous endpoint. + write_file(&func_path.join("c_chmask"), "0")?; + + // req_number=4: explicitly allocate 4 USB requests for the + // isochronous endpoint. Default (0 = auto) may not be enough + // for composite gadgets on DWC3. + let _ = write_file(&func_path.join("req_number"), "4"); + + debug!( + "UAC1 function {} created: {}ch {}Hz", + &self.name, self.channels, self.sample_rate + ); + Ok(()) + } + + fn link(&self, config_path: &Path, gadget_path: &Path) -> Result<()> { + let func_path = self.function_path(gadget_path); + let link_path = config_path.join(&self.name); + create_symlink(&func_path, &link_path)?; + debug!("UAC1 function {} linked into configuration", &self.name); + Ok(()) + } + + fn unlink(&self, config_path: &Path) -> Result<()> { + let link_path = config_path.join(&self.name); + if link_path.exists() { + std::fs::remove_file(&link_path).map_err(|e| { + AppError::Internal(format!( + "Failed to unlink UAC1 function {}: {}", + &self.name, e + )) + })?; + debug!("UAC1 function {} unlinked", &self.name); + } + Ok(()) + } + + fn cleanup(&self, gadget_path: &Path) -> Result<()> { + let func_path = self.function_path(gadget_path); + if func_path.exists() { + remove_dir(&func_path).map_err(|e| { + AppError::Internal(format!( + "Failed to remove UAC1 function {}: {}", + &self.name, e + )) + })?; + info!("UAC1 function {} removed", &self.name); + } + Ok(()) + } +} + +/// Return the ALSA PCM device name that the kernel assigns to a UAC1 +/// gadget after binding. The device appears as a playback-only PCM on +/// the gadget side. +pub fn uac_pcm_device() -> String { + // The kernel assigns the card name based on the gadget name. + // The PCM name is typically "playback" for UAC1. + "hw:UAC1Gadget,0".to_string() +} + +/// Resolve the actual PCM device name for a UAC1 playback device +/// by scanning /proc/asound/ for the gadget audio card. +pub fn find_uac_pcm_device() -> Option { + for entry in std::fs::read_dir("/proc/asound").ok()? { + let entry = entry.ok()?; + let name = entry.file_name(); + let name = name.to_str()?; + if !name.starts_with("card") { + continue; + } + let card_path = entry.path().join("id"); + if let Ok(id) = std::fs::read_to_string(&card_path) { + if id.trim().starts_with("UAC1Gadget") || id.trim().starts_with("gadget") { + let card_num = name.strip_prefix("card")?; + return Some(format!("hw:{card_num},0")); + } + } + } + None +} diff --git a/src/state.rs b/src/state.rs index 67b6b57d..ca93141f 100644 --- a/src/state.rs +++ b/src/state.rs @@ -77,6 +77,8 @@ pub struct AppState { pub msd: Arc>>, pub atx: Arc>>, pub audio: Arc, + pub uac_playback: Arc>>, + pub uac_config: Arc>, pub rustdesk: Arc>>>, pub vnc: Arc>>>, pub rtsp: Arc>>>, @@ -146,6 +148,8 @@ impl AppState { revoked_sessions: Arc::new(RwLock::new(VecDeque::new())), config_apply_locks: ConfigApplyLocks::new(), data_dir, + uac_playback: Arc::new(RwLock::new(None)), + uac_config: Arc::new(RwLock::new(crate::otg::service::UacConfig::default())), }) } diff --git a/src/web/handlers/config/apply.rs b/src/web/handlers/config/apply.rs index 24e7a4dd..4a0f6f2b 100644 --- a/src/web/handlers/config/apply.rs +++ b/src/web/handlers/config/apply.rs @@ -76,17 +76,18 @@ async fn reconcile_otg_config( hid: &HidConfig, msd: &MsdConfig, network: &OtgNetworkConfig, + uac: &crate::otg::service::UacConfig, ) -> Result<()> { #[cfg(not(unix))] { - let _ = (state, hid, msd, network); + let _ = (state, hid, msd, network, uac); Ok(()) } #[cfg(unix)] { state .otg_service - .apply_config(hid, msd, network) + .apply_config(hid, msd, network, uac) .await .map_err(|e| AppError::Config(format!("OTG reconcile failed: {}", e))) } @@ -236,7 +237,7 @@ pub async fn apply_hid_config( } if otg_config_changed { - reconcile_otg_config(state, new_config, msd_config, network_config).await?; + reconcile_otg_config(state, new_config, msd_config, network_config, &state.config.get().uac).await?; } if !transitioning_away_from_otg { @@ -304,7 +305,7 @@ pub async fn apply_msd_config( if new_msd_enabled { tracing::info!("(Re)initializing MSD..."); - reconcile_otg_config(state, hid_config, new_config, network_config).await?; + reconcile_otg_config(state, hid_config, new_config, network_config, &state.config.get().uac).await?; let mut msd_guard = state.msd.write().await; if let Some(msd) = msd_guard.as_mut() { @@ -339,7 +340,7 @@ pub async fn apply_msd_config( *msd_guard = None; tracing::info!("MSD shutdown complete"); - reconcile_otg_config(state, hid_config, new_config, network_config).await?; + reconcile_otg_config(state, hid_config, new_config, network_config, &state.config.get().uac).await?; } if hid_config.backend == HidBackend::Otg @@ -367,7 +368,9 @@ pub async fn apply_usb_config( let hid_unchanged = old_config.hid == new_config.hid; let otg_gadget_rebuilt = - old_config.msd != new_config.msd || old_config.otg_network != new_config.otg_network; + old_config.msd != new_config.msd + || old_config.otg_network != new_config.otg_network + || old_config.uac != new_config.uac; if transitioning_away_from_otg { apply_hid_config( @@ -385,6 +388,7 @@ pub async fn apply_usb_config( &new_config.hid, &new_config.msd, &new_config.otg_network, + &new_config.uac, ) .await?; apply_hid_config( @@ -411,6 +415,32 @@ pub async fn apply_usb_config( .map_err(|e| AppError::Config(format!("HID reload after gadget rebuild failed: {}", e)))?; } + // UAC playback writer lifecycle + if old_config.uac.enabled != new_config.uac.enabled { + let mut guard = state.uac_playback.write().await; + if new_config.uac.enabled { + let config = crate::audio::uac_streamer::UacPlaybackConfig { + sample_rate: new_config.uac.sample_rate, + channels: new_config.uac.channels as u16, + ..Default::default() + }; + match crate::audio::uac_streamer::UacPlaybackWriter::start(config) { + Ok(writer) => { + tracing::info!("UAC playback writer started"); + *guard = Some(writer); + } + Err(e) => { + tracing::warn!("Failed to start UAC playback writer: {}", e); + } + } + } else { + if let Some(writer) = guard.take() { + writer.stop(); + tracing::info!("UAC playback writer stopped"); + } + } + } + apply_msd_config( state, &old_config.msd, diff --git a/src/web/handlers/config/mod.rs b/src/web/handlers/config/mod.rs index 250d0aff..5852224e 100644 --- a/src/web/handlers/config/mod.rs +++ b/src/web/handlers/config/mod.rs @@ -15,6 +15,8 @@ mod redfish; mod rtsp; mod rustdesk; mod stream; +#[cfg(unix)] +mod uac; mod usb_update; pub(crate) mod video; mod vnc; @@ -40,6 +42,8 @@ pub use rustdesk::{ regenerate_device_password, start_rustdesk_service, stop_rustdesk_service, update_rustdesk_config, }; +#[cfg(unix)] +pub use uac::{get_uac_config, update_uac_config}; pub use stream::{get_stream_config, update_stream_config}; pub use video::{get_video_config, update_video_config}; pub use vnc::{ diff --git a/src/web/handlers/config/uac.rs b/src/web/handlers/config/uac.rs new file mode 100644 index 00000000..d7a7b044 --- /dev/null +++ b/src/web/handlers/config/uac.rs @@ -0,0 +1,36 @@ +use std::sync::Arc; + +use axum::{extract::State, Json}; + +use crate::error::Result; +use crate::otg::service::UacConfig; +use crate::state::AppState; + +use super::apply::try_apply_lock; + +pub async fn get_uac_config(State(state): State>) -> Json { + Json(state.config.get().uac.clone()) +} + +pub async fn update_uac_config( + State(state): State>, + Json(req): Json, +) -> Result> { + req.validate()?; + let _guard = try_apply_lock(&state.config_apply_locks.otg, "uac")?; + + let old_config = (*state.config.get()).clone(); + let mut new_config = old_config.clone(); + new_config.uac = req; + + state + .config + .update(|config| { + config.uac = new_config.uac.clone(); + }) + .await?; + + super::apply::apply_usb_config(&state, &old_config, &new_config).await?; + + Ok(Json(state.config.get().uac.clone())) +} diff --git a/src/web/handlers/setup.rs b/src/web/handlers/setup.rs index b8712dc2..b469405b 100644 --- a/src/web/handlers/setup.rs +++ b/src/web/handlers/setup.rs @@ -147,7 +147,7 @@ pub async fn setup_init( { if let Err(e) = state .otg_service - .apply_config(&new_config.hid, &new_config.msd, &new_config.otg_network) + .apply_config(&new_config.hid, &new_config.msd, &new_config.otg_network, &crate::otg::service::UacConfig::default()) .await { tracing::warn!("Failed to apply OTG config during setup: {}", e); diff --git a/src/web/mod.rs b/src/web/mod.rs index d4a90159..79998a9d 100644 --- a/src/web/mod.rs +++ b/src/web/mod.rs @@ -3,6 +3,7 @@ mod error; mod handlers; mod routes; mod static_files; +mod uac_ws; mod ws; pub use audio_ws::audio_ws_handler; @@ -10,4 +11,5 @@ pub use error::ErrorResponse; pub use routes::create_router; #[cfg(not(debug_assertions))] pub use static_files::StaticAssets; +pub use uac_ws::uac_audio_ws_handler; pub use ws::ws_handler; diff --git a/src/web/routes.rs b/src/web/routes.rs index c2527394..1db95172 100644 --- a/src/web/routes.rs +++ b/src/web/routes.rs @@ -13,6 +13,7 @@ use tower_http::{ use super::audio_ws::audio_ws_handler; use super::handlers; +use super::uac_ws::uac_audio_ws_handler; use super::ws::ws_handler; use crate::auth::auth_middleware; use crate::hid::websocket::ws_hid_handler; @@ -100,6 +101,7 @@ pub fn create_router(state: Arc) -> Router { .route("/audio/devices", get(handlers::list_audio_devices)) // Audio WebSocket endpoint .route("/ws/audio", any(audio_ws_handler)) + .route("/ws/uac-audio", any(uac_audio_ws_handler)) // Configuration management (domain-separated endpoints) .route("/config", get(handlers::config::get_all_config)) .route("/config/video", get(handlers::config::get_video_config)) @@ -278,6 +280,14 @@ pub fn create_router(state: Arc) -> Router { "/otg/network/status", get(handlers::config::get_otg_network_status), ) + .route( + "/config/uac", + get(handlers::config::get_uac_config), + ) + .route( + "/config/uac", + patch(handlers::config::update_uac_config), + ) .route("/msd/status", get(handlers::msd_status)) .route("/msd/images", get(handlers::msd_images_list)) .route("/msd/images/download", post(handlers::msd_image_download)) diff --git a/src/web/uac_ws.rs b/src/web/uac_ws.rs new file mode 100644 index 00000000..ea32e950 --- /dev/null +++ b/src/web/uac_ws.rs @@ -0,0 +1,33 @@ +use axum::extract::ws::WebSocketUpgrade; +use axum::extract::State; +use axum::http::StatusCode; +use axum::response::IntoResponse; +use std::sync::Arc; +use tracing::warn; + +use crate::state::AppState; + +/// WebSocket endpoint for UAC microphone passthrough audio input. +/// +/// Accepts Opus-encoded audio frames (same binary protocol as audio +/// output, message type 0x03) and routes decoded PCM to the UAC +/// playback device on the USB gadget side. +pub async fn uac_audio_ws_handler( + ws: WebSocketUpgrade, + State(state): State>, +) -> impl IntoResponse { + let playback = { + let guard = state.uac_playback.read().await; + match guard.as_ref() { + Some(p) => Arc::new(p.clone()), + None => { + warn!("UAC audio WS rejected: playback not initialized"); + return (StatusCode::SERVICE_UNAVAILABLE, "UAC playback not initialized").into_response(); + } + } + }; + + ws.on_upgrade(move |socket| { + crate::audio::uac_websocket::handle_uac_audio_ws(socket, playback) + }) +} diff --git a/web/package-lock.json b/web/package-lock.json index 40102e0a..dcfd4ddb 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -13,6 +13,7 @@ "clsx": "^2.1.1", "lucide-vue-next": "^0.556.0", "opus-decoder": "^0.7.11", + "opus-media-recorder": "^0.8.0", "pinia": "^3.0.4", "qrcode.vue": "^3.10.0", "reka-ui": "^2.10.1", @@ -1940,6 +1941,12 @@ "integrity": "sha512-7z22QmUWiQ/2d0KkdYmANbRUVABpZ9SNYyH5vx6PZ+nE5bcC0l7uFvEfHlyld/HcGBFTL536ClDt3DEcSlEJAQ==", "license": "MIT" }, + "node_modules/detect-browser": { + "version": "4.8.0", + "resolved": "https://registry.npmmirror.com/detect-browser/-/detect-browser-4.8.0.tgz", + "integrity": "sha512-f4h2dFgzHUIpjpBLjhnDIteXv8VQiUm8XzAuzQtYUqECX/eKh67ykuiVoyb7Db7a0PUSmJa3OGXStG0CbQFUVw==", + "license": "MIT" + }, "node_modules/detect-libc": { "version": "2.1.2", "resolved": "https://registry.npmmirror.com/detect-libc/-/detect-libc-2.1.2.tgz", @@ -2041,6 +2048,15 @@ "integrity": "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==", "license": "MIT" }, + "node_modules/event-target-shim": { + "version": "3.0.2", + "resolved": "https://registry.npmmirror.com/event-target-shim/-/event-target-shim-3.0.2.tgz", + "integrity": "sha512-HK5GhnEAkm7fLy249GtF7DIuYmjLm85Ft6ssj7DhVl8Tx/z9+v0W6aiIVUdT4AXWGYy5Fc+s6gqBI49Bf0LejQ==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, "node_modules/fdir": { "version": "6.5.0", "resolved": "https://registry.npmmirror.com/fdir/-/fdir-6.5.0.tgz", @@ -2472,6 +2488,16 @@ "url": "https://github.com/sponsors/eshaz" } }, + "node_modules/opus-media-recorder": { + "version": "0.8.0", + "resolved": "https://registry.npmmirror.com/opus-media-recorder/-/opus-media-recorder-0.8.0.tgz", + "integrity": "sha512-AIvJMpnJqZ18dFAU7Amtt5cZZp8oPzDoAOtobdTcLzwVNm/j815+GJmBupBzBZGBa4L940TEulm7Uu4tGOYDGQ==", + "license": "MIT", + "dependencies": { + "detect-browser": "^4.1.0", + "event-target-shim": "^3.0.2" + } + }, "node_modules/path-browserify": { "version": "1.0.1", "resolved": "https://registry.npmmirror.com/path-browserify/-/path-browserify-1.0.1.tgz", diff --git a/web/package.json b/web/package.json index 3dce78c5..38c39bca 100644 --- a/web/package.json +++ b/web/package.json @@ -14,6 +14,7 @@ "clsx": "^2.1.1", "lucide-vue-next": "^0.556.0", "opus-decoder": "^0.7.11", + "opus-media-recorder": "^0.8.0", "pinia": "^3.0.4", "qrcode.vue": "^3.10.0", "reka-ui": "^2.10.1", diff --git a/web/src/api/config.ts b/web/src/api/config.ts index b10994fa..8e0fca9b 100644 --- a/web/src/api/config.ts +++ b/web/src/api/config.ts @@ -118,6 +118,16 @@ export const otgNetworkApi = { interfaces: () => request('/devices/network'), } +export const uacApi = { + get: () => request<{enabled: boolean; sample_rate: number; channels: number}>('/config/uac'), + + update: (config: {enabled: boolean; sample_rate: number; channels: number}) => + request('/config/uac', { + method: 'PATCH', + body: JSON.stringify(config), + }), +} + export const otgConfigApi = { update: (config: OtgConfigUpdate) => request('/config/otg', { diff --git a/web/src/api/index.ts b/web/src/api/index.ts index 13c78366..f795039d 100644 --- a/web/src/api/index.ts +++ b/web/src/api/index.ts @@ -855,6 +855,7 @@ export { msdConfigApi, otgConfigApi, otgNetworkApi, + uacApi, atxConfigApi, audioConfigApi, extensionsApi, diff --git a/web/src/components/ActionBar.vue b/web/src/components/ActionBar.vue index eee9c40d..04b84371 100644 --- a/web/src/components/ActionBar.vue +++ b/web/src/components/ActionBar.vue @@ -3,6 +3,7 @@ import { ref, computed, onMounted, onUnmounted, watch, nextTick } from 'vue' import { useI18n } from 'vue-i18n' import { useRouter } from 'vue-router' import { useSystemStore } from '@/stores/system' +import { getMicrophone } from '@/composables/useMicrophone' import { Button } from '@/components/ui/button' import { ButtonGroup } from '@/components/ui/button-group' import { @@ -31,6 +32,7 @@ import { } from '@/components/ui/sheet' import { ClipboardPaste, + Mic, HardDrive, Keyboard, Settings, @@ -68,9 +70,13 @@ const props = defineProps<{ showTerminal?: boolean showComputerUse?: boolean showPasteText?: boolean + showMic?: boolean }>() const showStats = computed(() => (props.videoMode ?? 'mjpeg') !== 'mjpeg') const showPasteText = computed(() => props.showPasteText !== false) +const showMic = computed(() => props.showMic === true) +const mic = getMicrophone() + const emit = defineEmits<{ (e: 'toggleFullscreen'): void @@ -350,6 +356,24 @@ const hasRightOverflow = computed(() => { + + +
+ + + + + + {{ mic.error.value ? mic.error.value : (mic.active.value ? '停止传声' : '开始传声') }} + + +
diff --git a/web/src/composables/useMicrophone.ts b/web/src/composables/useMicrophone.ts new file mode 100644 index 00000000..1b099e8f --- /dev/null +++ b/web/src/composables/useMicrophone.ts @@ -0,0 +1,161 @@ +import { ref } from 'vue' + +let instance: ReturnType | null = null +export function getMicrophone() { + if (!instance) instance = useMicrophone() + return instance +} + +const WS_BASE = `${location.protocol === 'https:' ? 'wss:' : 'ws:'}//${location.host}/api/ws/uac-audio` + +// Opus: 48kHz stereo, 64kbps → ~8 KB/s vs raw PCM 192 KB/s (24× reduction) +const OPUS_CONFIG: AudioEncoderConfig = { + codec: 'opus', + sampleRate: 48000, + numberOfChannels: 2, + bitrate: 64000, +} + +// 15-byte binary header matching server-side UAC_AUDIO_HEADER_SIZE +function buildHeader(msgType: number, durationMs: number, dataLen: number): Uint8Array { + const h = new Uint8Array(15) + const v = new DataView(h.buffer) + v.setUint8(0, msgType) // 0x03 = Opus + v.setUint32(1, 0, true) // timestamp (unused) + v.setUint16(5, durationMs, true) + v.setUint32(7, 0, true) // sequence + v.setUint32(11, dataLen, true) + return h +} + +export function useMicrophone() { + const active = ref(false) + const error = ref(null) + let ws: WebSocket | null = null + let stream: MediaStream | null = null + let encoder: AudioEncoder | null = null + let running = false + + let frameCount = 0 + let byteCount = 0 + + // ── AudioEncoder helper ───────────────────────────────── + function createEncoder(onOpusFrame: (data: Uint8Array, durMs: number) => void): AudioEncoder { + const enc = new AudioEncoder({ + output: (chunk: EncodedAudioChunk) => { + const buf = new Uint8Array(chunk.byteLength) + chunk.copyTo(buf) + // Opus frame duration in microseconds → milliseconds + const durMs = Math.round(chunk.duration! / 1000) + onOpusFrame(buf, durMs) + }, + error: (e: Error) => console.error('[mic] encoder error:', e), + }) + enc.configure(OPUS_CONFIG) + return enc + } + + // ── start / stop ──────────────────────────────────────── + async function start() { + error.value = null + frameCount = 0 + byteCount = 0 + running = true + console.log('[mic] starting...') + + try { + // WebSocket + ws = new WebSocket(WS_BASE) + ws.binaryType = 'arraybuffer' + const wsReady = new Promise((resolve, reject) => { + ws!.onopen = () => { console.log('[mic] WS opened'); active.value = true; resolve() } + ws!.onerror = (ev) => { console.error('[mic] WS error:', ev); reject(new Error('WebSocket failed')) } + }) + ws.onclose = (ev) => { + console.log('[mic] WS closed: code=%d frames=%d bytes=%d', ev.code, frameCount, byteCount) + active.value = false + running = false + } + + // Microphone + console.log('[mic] getUserMedia...') + stream = await navigator.mediaDevices.getUserMedia({ + audio: { sampleRate: 48000, channelCount: 2, echoCancellation: false, noiseSuppression: false } + }) + // AudioEncoder (WebCodecs) for Opus compression + encoder = createEncoder((opusData, durMs) => { + if (!ws || ws.readyState !== WebSocket.OPEN) return + const header = buildHeader(0x03, durMs, opusData.length) + const msg = new Uint8Array(15 + opusData.length) + msg.set(header) + msg.set(opusData, 15) + ws.send(msg) + frameCount++ + byteCount += msg.byteLength + if (frameCount % 50 === 0) { + console.debug('[mic] frame #%d: opus=%dB dur=%dms', + frameCount, opusData.length, durMs) + } + }) + + await wsReady + + // ScriptProcessor → S16LE PCM → AudioData → AudioEncoder → Opus + const audioCtx = new AudioContext({ sampleRate: 48000 }) + const source = audioCtx.createMediaStreamSource(stream) + const processor = audioCtx.createScriptProcessor(4096, 2, 2) + source.connect(processor) + processor.connect(audioCtx.destination) + + processor.onaudioprocess = (e: AudioProcessingEvent) => { + if (!running || !encoder || encoder.state !== 'configured') return + if (!e.inputBuffer) return + const buf = e.inputBuffer as any + const left = buf.getChannelData(0) as Float32Array + const right = buf.getChannelData(1) as Float32Array + const samples = left.length + + // Float32 → S16LE interleaved + const pcm = new Int16Array(samples * 2) + for (let i = 0; i < samples; i++) { + pcm[i * 2] = Math.max(-32768, Math.min(32767, Math.round((left[i] ?? 0) * 32767))) + pcm[i * 2 + 1] = Math.max(-32768, Math.min(32767, Math.round((right[i] ?? 0) * 32767))) + } + + try { + const audioData = new AudioData({ + format: 's16', + sampleRate: 48000, + numberOfFrames: samples, + numberOfChannels: 2, + timestamp: 0, + data: pcm.buffer, + }) + encoder.encode(audioData) + audioData.close() + } catch (e) { + console.warn('[mic] AudioData/encode error:', e) + } + } + } catch (e) { + console.error('[mic] start error:', e) + error.value = e instanceof Error ? e.message : 'Failed to start microphone' + stop() + } + } + + function stop() { + console.log('[mic] stop: frames=%d bytes=%d', frameCount, byteCount) + running = false + if (encoder) { encoder.close(); encoder = null } + if (stream) { stream.getTracks().forEach(t => t.stop()); stream = null } + if (ws) { ws.close(); ws = null } + active.value = false + } + + function toggle() { + if (active.value) { stop() } else { start() } + } + + return { active, error, start, stop, toggle } +} diff --git a/web/src/i18n/en-US.ts b/web/src/i18n/en-US.ts index 461d6b01..20ad8c83 100644 --- a/web/src/i18n/en-US.ts +++ b/web/src/i18n/en-US.ts @@ -653,6 +653,8 @@ export default { otgNetworkDriver: 'Host Driver Mode', otgNetworkInterface: 'Bridge Interface', otgNetworkNone: 'None', + uacMic: 'USB Microphone', + uacMicDesc: 'Creates a virtual USB microphone on the target machine. Audio from your browser is streamed to the target.', otgDescriptor: 'USB Device Descriptor', vendorId: 'Vendor ID (VID)', productId: 'Product ID (PID)', diff --git a/web/src/i18n/zh-CN.ts b/web/src/i18n/zh-CN.ts index 7d4da3b5..280c02de 100644 --- a/web/src/i18n/zh-CN.ts +++ b/web/src/i18n/zh-CN.ts @@ -92,7 +92,8 @@ export default { }, actionbar: { paste: '粘贴文本', - virtualMedia: '虚拟媒体', + micStart: '开始传声', + micStop: '停止传声', virtualMediaTip: '管理虚拟媒体设备', power: '电源', keyboard: '虚拟键盘', @@ -652,6 +653,8 @@ export default { otgNetworkDriver: '目标机驱动模式', otgNetworkInterface: '桥接网卡', otgNetworkNone: '无', + uacMic: 'USB 麦克风', + uacMicDesc: '启用后目标机将看到一个 USB 麦克风设备，音频从浏览器传入', otgDescriptor: 'USB 设备描述符', vendorId: '厂商 ID (VID)', productId: '产品 ID (PID)', diff --git a/web/src/views/ConsoleView.vue b/web/src/views/ConsoleView.vue index 56495f1b..875a7451 100644 --- a/web/src/views/ConsoleView.vue +++ b/web/src/views/ConsoleView.vue @@ -14,7 +14,7 @@ import { useComputerUseSocket, type ComputerUseServerMessage } from '@/composabl import { useFeatureVisibility } from '@/composables/useFeatureVisibility' import { useTheme } from '@/composables/useTheme' import { getUnifiedAudio } from '@/composables/useUnifiedAudio' -import { streamApi, hidApi, atxApi, atxConfigApi, authApi, computerUseApi } from '@/api' +import { streamApi, hidApi, atxApi, atxConfigApi, authApi, computerUseApi, uacApi } from '@/api' import type { ComputerUseScreenshot, ComputerUseSession } from '@/api' import { CanonicalKey, HidBackend } from '@/types/generated' import type { HidKeyboardEvent, HidMouseEvent } from '@/types/hid' @@ -2961,7 +2961,15 @@ function handleToggleMouseMode() { } } +const uacEnabled = ref(false) + onMounted(async () => { + // Check if UAC is enabled (show mic button only if USB mic is available) + try { + const uacCfg = await uacApi.get() + uacEnabled.value = uacCfg.enabled + } catch { /* ignore */ } + consoleEvents.subscribe() watch([wsConnected, wsNetworkError], ([connected, netError], [_prevConnected, prevNetError]) => { @@ -3166,6 +3174,7 @@ onUnmounted(() => { :show-terminal="showTerminal" :show-computer-use="showComputerUse" :show-paste-text="showPasteText" + :show-mic="uacEnabled" @toggle-fullscreen="toggleFullscreen" @toggle-stats="openStatsSheet" @toggle-virtual-keyboard="handleToggleVirtualKeyboard" diff --git a/web/src/views/SettingsView.vue b/web/src/views/SettingsView.vue index 45ec49f0..dad7ef62 100644 --- a/web/src/views/SettingsView.vue +++ b/web/src/views/SettingsView.vue @@ -11,6 +11,7 @@ import { authApi, configApi, otgNetworkApi, + uacApi, hidApi, streamApi, atxConfigApi, @@ -685,6 +686,7 @@ const config = ref({ turn_server: '', turn_username: '', turn_password: '', + uac_enabled: false, }) const otgNetworkInterfaces = ref([]) @@ -1479,6 +1481,12 @@ async function saveConfig() { }, }) otgNetworkStatus.value = response.status + + await uacApi.update({ + enabled: otgEnabled && config.value.uac_enabled, + sample_rate: 48000, + channels: 2, + }) } if (activeSection.value !== 'hid') { @@ -1499,7 +1507,7 @@ async function saveConfig() { async function loadConfig() { try { - const [video, stream, hid, msd, otgNetwork] = await Promise.all([ + const [video, stream, hid, msd, otgNetwork, uac] = await Promise.all([ configStore.refreshVideo(), configStore.refreshStream(), configStore.refreshHid(), @@ -1511,6 +1519,7 @@ async function loadConfig() { host_mac: '', device_mac: '', })), + uacApi.get().catch(() => ({ enabled: false, sample_rate: 48000, channels: 2 })), ]) config.value = { @@ -1536,6 +1545,7 @@ async function loadConfig() { msd_dir: msd.msd_dir || '', otg_network_enabled: otgNetwork.enabled, otg_network_driver: otgNetwork.driver_mode, + uac_enabled: uac.enabled, otg_network_interface: otgNetwork.bridge_interface, encoder_backend: stream.encoder || 'auto', stun_server: stream.stun_server || '', @@ -3342,6 +3352,15 @@ watch(isWindows, () => { {{ t('settings.otgRuntimeDegraded') }}: {{ otgNetworkStatus.error || t('common.error') }}

+
+
+
+ +

{{ t('settings.uacMicDesc') }}

+
+ +
+

{{ t('settings.otgProfileWarning') }}