From ce622e44925c1a86f79bf4d9a198bda1e769313d Mon Sep 17 00:00:00 2001 From: mofeng-git Date: Fri, 20 Feb 2026 13:34:49 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E4=BC=98=E5=8C=96=20WebRTC=20=E5=BB=BA?= =?UTF-8?q?=E8=BF=9E=E6=B5=81=E7=A8=8B=E3=80=81=E4=BF=AE=E5=A4=8D=E5=B9=B3?= =?UTF-8?q?=E5=8F=B0=E4=BF=A1=E6=81=AF=E3=80=81=E4=BF=AE=E5=A4=8D=E8=99=9A?= =?UTF-8?q?=E6=8B=9F=E9=94=AE=E7=9B=98=E9=94=AE=E5=80=BC=E6=98=A0=E5=B0=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - WebRTC：默认 mDNS 调整为 QueryOnly，Answer 阶段改为等待 ICE gathering complete（2.5s 超时），提升首次建连成功率与候选完整性 - WebRTC：前端建连流程增加阶段化状态与串行保护（connectInFlight/ready gate），优化配置变更后的重连时机与失败处理，减少竞态和无效重试 - Device：平台信息补充 `/proc/device-tree/model` 回退并统一展示为“处理器/平台” - HID：键盘输入链路统一为 HID usage + modifier bitmask，修复虚拟键盘/宏/粘贴键值映射错误 --- src/hid/datachannel.rs | 7 +- src/web/handlers/mod.rs | 128 ++++++++--- src/webrtc/mdns.rs | 4 +- src/webrtc/peer.rs | 16 +- src/webrtc/universal_session.rs | 15 +- web/src/api/index.ts | 13 +- web/src/components/PasteModal.vue | 12 +- web/src/components/VirtualKeyboard.vue | 46 ++-- web/src/composables/useHidInput.ts | 24 ++- web/src/composables/useWebRTC.ts | 218 +++++++++++-------- web/src/i18n/en-US.ts | 10 +- web/src/i18n/zh-CN.ts | 10 +- web/src/lib/charToHid.ts | 227 ++++++++++--------- web/src/lib/keyboardMappings.ts | 24 +++ web/src/types/hid.ts | 16 +- web/src/views/ConsoleView.vue | 287 +++++++++++++++++-------- 16 files changed, 667 insertions(+), 390 deletions(-) diff --git a/src/hid/datachannel.rs b/src/hid/datachannel.rs index 00ecfb12..04c76d21 100644 --- a/src/hid/datachannel.rs +++ b/src/hid/datachannel.rs @@ -9,7 +9,7 @@ //! //! Keyboard event (type 0x01): //! - Byte 1: Event type (0x00 = down, 0x01 = up) -//! - Byte 2: Key code (USB HID usage code or JS keyCode) +//! - Byte 2: Key code (USB HID usage code) //! - Byte 3: Modifiers bitmask //! - Bit 0: Left Ctrl //! - Bit 1: Left Shift @@ -119,7 +119,7 @@ fn parse_keyboard_message(data: &[u8]) -> Option { event_type, key, modifiers, - is_usb_hid: false, // WebRTC datachannel sends JS keycodes + is_usb_hid: true, // WebRTC/WebSocket HID channel sends USB HID usages })) } @@ -245,6 +245,7 @@ mod tests { assert_eq!(kb.key, 0x04); assert!(kb.modifiers.left_ctrl); assert!(!kb.modifiers.left_shift); + assert!(kb.is_usb_hid); } _ => panic!("Expected keyboard event"), } @@ -280,7 +281,7 @@ mod tests { right_alt: false, right_meta: false, }, - is_usb_hid: false, + is_usb_hid: true, }; let encoded = encode_keyboard_event(&event); diff --git a/src/web/handlers/mod.rs b/src/web/handlers/mod.rs index c88fab92..90f5fda7 100644 --- a/src/web/handlers/mod.rs +++ b/src/web/handlers/mod.rs @@ -183,31 +183,59 @@ fn get_hostname() -> String { .unwrap_or_else(|_| "unknown".to_string()) } -/// Get CPU model name from /proc/cpuinfo +/// Get CPU model name from /proc/cpuinfo, fallback to device-tree model fn get_cpu_model() -> String { - std::fs::read_to_string("/proc/cpuinfo") + let cpuinfo = std::fs::read_to_string("/proc/cpuinfo").ok(); + + if let Some(model) = cpuinfo + .as_deref() + .and_then(parse_cpu_model_from_cpuinfo_content) + { + return model; + } + + if let Some(model) = read_device_tree_model() { + return model; + } + + if let Some(content) = cpuinfo.as_deref() { + let cores = content + .lines() + .filter(|line| line.starts_with("processor")) + .count(); + if cores > 0 { + return format!("{} {}C", std::env::consts::ARCH, cores); + } + } + + std::env::consts::ARCH.to_string() +} + +fn parse_cpu_model_from_cpuinfo_content(content: &str) -> Option { + content + .lines() + .find(|line| line.starts_with("model name") || line.starts_with("Model")) + .and_then(|line| line.split(':').nth(1)) + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) +} + +fn read_device_tree_model() -> Option { + std::fs::read("/proc/device-tree/model") .ok() - .and_then(|content| { - // Try to get model name - let model = content - .lines() - .find(|line| line.starts_with("model name") || line.starts_with("Model")) - .and_then(|line| line.split(':').nth(1)) - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()); + .and_then(|bytes| parse_device_tree_model_bytes(&bytes)) +} - if model.is_some() { - return model; - } +fn parse_device_tree_model_bytes(bytes: &[u8]) -> Option { + let model = String::from_utf8_lossy(bytes) + .trim_matches(|c: char| c == '\0' || c.is_whitespace()) + .to_string(); - // Fallback: show arch and core count - let cores = content - .lines() - .filter(|line| line.starts_with("processor")) - .count(); - Some(format!("{} {}C", std::env::consts::ARCH, cores)) - }) - .unwrap_or_else(|| std::env::consts::ARCH.to_string()) + if model.is_empty() { + None + } else { + Some(model) + } } /// CPU usage state for calculating usage between samples @@ -389,6 +417,38 @@ fn get_network_addresses() -> Vec { addresses } +#[cfg(test)] +mod tests { + use super::{parse_cpu_model_from_cpuinfo_content, parse_device_tree_model_bytes}; + + #[test] + fn parse_cpu_model_from_model_name_field() { + let input = "processor\t: 0\nmodel name\t: Intel(R) Xeon(R)\n"; + assert_eq!( + parse_cpu_model_from_cpuinfo_content(input), + Some("Intel(R) Xeon(R)".to_string()) + ); + } + + #[test] + fn parse_cpu_model_from_model_field() { + let input = "processor\t: 0\nModel\t\t: Raspberry Pi 4 Model B Rev 1.4\n"; + assert_eq!( + parse_cpu_model_from_cpuinfo_content(input), + Some("Raspberry Pi 4 Model B Rev 1.4".to_string()) + ); + } + + #[test] + fn parse_device_tree_model_trimmed() { + let input = b"Onething OEC Box\0\n"; + assert_eq!( + parse_device_tree_model_bytes(input), + Some("Onething OEC Box".to_string()) + ); + } +} + // ============================================================================ // Authentication // ============================================================================ @@ -2053,10 +2113,11 @@ pub async fn webrtc_offer( )); } - // Create session if client_id not provided + // Backward compatibility: `client_id` is treated as an existing session_id hint. + // New clients should not pass it; each offer creates a fresh session. let webrtc = state.stream_manager.webrtc_streamer(); let session_id = if let Some(client_id) = &req.client_id { - // Check if session exists + // Reuse only when it matches an active session ID. if webrtc.get_session(client_id).await.is_some() { client_id.clone() } else { @@ -2411,15 +2472,13 @@ pub async fn hid_otg_self_check(State(state): State>) -> Json, - selected_udc: Option, - bound_udc: Option, - udc_state: Option, - udc_speed: Option, - available_udcs: Vec, - other_gadgets: Vec, - | { + let build_response = |checks: Vec, + selected_udc: Option, + bound_udc: Option, + udc_state: Option, + udc_speed: Option, + available_udcs: Vec, + other_gadgets: Vec| { let error_count = checks .iter() .filter(|item| item.level == OtgSelfCheckLevel::Error) @@ -2528,7 +2587,9 @@ pub async fn hid_otg_self_check(State(state): State>) -> Json>) -> Json Option { } pub fn mdns_mode() -> MulticastDnsMode { - mdns_mode_from_env().unwrap_or(MulticastDnsMode::QueryAndGather) + // Default to QueryOnly to avoid gathering .local host candidates by default. + // This is generally more stable for LAN first-connection while preserving mDNS queries. + mdns_mode_from_env().unwrap_or(MulticastDnsMode::QueryOnly) } pub fn mdns_mode_label(mode: MulticastDnsMode) -> &'static str { diff --git a/src/webrtc/peer.rs b/src/webrtc/peer.rs index 396b3782..5dbbb8e0 100644 --- a/src/webrtc/peer.rs +++ b/src/webrtc/peer.rs @@ -317,14 +317,26 @@ impl PeerConnection { .await .map_err(|e| AppError::VideoError(format!("Failed to create answer: {}", e)))?; + // Wait for ICE gathering complete (or timeout) after setting local description. + // This improves first-connection robustness by returning a fuller initial candidate set. + let mut gather_complete = self.pc.gathering_complete_promise().await; + // Set local description self.pc .set_local_description(answer.clone()) .await .map_err(|e| AppError::VideoError(format!("Failed to set local description: {}", e)))?; - // Wait a bit for ICE candidates to gather - tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + const ICE_GATHER_TIMEOUT: tokio::time::Duration = tokio::time::Duration::from_millis(2500); + if tokio::time::timeout(ICE_GATHER_TIMEOUT, gather_complete.recv()) + .await + .is_err() + { + debug!( + "ICE gathering timeout after {:?} for session {}", + ICE_GATHER_TIMEOUT, self.session_id + ); + } // Get gathered ICE candidates let candidates = self.ice_candidates.lock().await.clone(); diff --git a/src/webrtc/universal_session.rs b/src/webrtc/universal_session.rs index 30129954..eafda89f 100644 --- a/src/webrtc/universal_session.rs +++ b/src/webrtc/universal_session.rs @@ -833,13 +833,24 @@ impl UniversalSession { } } + let mut gather_complete = self.pc.gathering_complete_promise().await; + self.pc .set_local_description(answer.clone()) .await .map_err(|e| AppError::VideoError(format!("Failed to set local description: {}", e)))?; - // Wait for ICE candidates - tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + // Wait for ICE gathering complete (or timeout) to return a fuller initial candidate set. + const ICE_GATHER_TIMEOUT: Duration = Duration::from_millis(2500); + if tokio::time::timeout(ICE_GATHER_TIMEOUT, gather_complete.recv()) + .await + .is_err() + { + debug!( + "ICE gathering timeout after {:?} for session {}", + ICE_GATHER_TIMEOUT, self.session_id + ); + } let candidates = self.ice_candidates.lock().await.clone(); Ok(SdpAnswer::with_candidates(answer.sdp, candidates)) diff --git a/web/src/api/index.ts b/web/src/api/index.ts index 31453e4e..d3522ee5 100644 --- a/web/src/api/index.ts +++ b/web/src/api/index.ts @@ -242,10 +242,10 @@ export const webrtcApi = { createSession: () => request<{ session_id: string }>('/webrtc/session', { method: 'POST' }), - offer: (sdp: string, clientId?: string) => + offer: (sdp: string) => request<{ sdp: string; session_id: string; ice_candidates: IceCandidate[] }>('/webrtc/offer', { method: 'POST', - body: JSON.stringify({ sdp, client_id: clientId }), + body: JSON.stringify({ sdp }), }), addIceCandidate: (sessionId: string, candidate: IceCandidate) => @@ -325,17 +325,12 @@ export const hidApi = { }> }>('/hid/otg/self-check'), - keyboard: async (type: 'down' | 'up', key: number, modifiers?: { - ctrl?: boolean - shift?: boolean - alt?: boolean - meta?: boolean - }) => { + keyboard: async (type: 'down' | 'up', key: number, modifier?: number) => { await ensureHidConnection() const event: HidKeyboardEvent = { type: type === 'down' ? 'keydown' : 'keyup', key, - modifiers, + modifier: (modifier ?? 0) & 0xff, } await hidWs.sendKeyboard(event) return { success: true } diff --git a/web/src/components/PasteModal.vue b/web/src/components/PasteModal.vue index b574f7ae..6b857690 100644 --- a/web/src/components/PasteModal.vue +++ b/web/src/components/PasteModal.vue @@ -69,24 +69,24 @@ async function typeChar(char: string, signal: AbortSignal): Promise { return true } - const { keyCode, shift } = mapping - const modifiers = shift ? { shift: true } : undefined + const { hidCode, shift } = mapping + const modifier = shift ? 0x02 : 0 try { // Send keydown - await hidApi.keyboard('down', keyCode, modifiers) + await hidApi.keyboard('down', hidCode, modifier) // Small delay between down and up to ensure key is registered await sleep(5) if (signal.aborted) { // Even if aborted, still send keyup to release the key - await hidApi.keyboard('up', keyCode, modifiers) + await hidApi.keyboard('up', hidCode, modifier) return false } // Send keyup - await hidApi.keyboard('up', keyCode, modifiers) + await hidApi.keyboard('up', hidCode, modifier) // Additional small delay after keyup to ensure it's processed await sleep(2) @@ -96,7 +96,7 @@ async function typeChar(char: string, signal: AbortSignal): Promise { console.error('[Paste] Failed to type character:', char, error) // Try to release the key even on error try { - await hidApi.keyboard('up', keyCode, modifiers) + await hidApi.keyboard('up', hidCode, modifier) } catch { // Ignore cleanup errors } diff --git a/web/src/components/VirtualKeyboard.vue b/web/src/components/VirtualKeyboard.vue index 4d4feee8..1dfeb91d 100644 --- a/web/src/components/VirtualKeyboard.vue +++ b/web/src/components/VirtualKeyboard.vue @@ -9,6 +9,7 @@ import { consumerKeys, latchingKeys, modifiers, + updateModifierMaskForHidKey, type KeyName, type ConsumerKeyName, } from '@/lib/keyboardMappings' @@ -304,9 +305,10 @@ async function onKeyDown(key: string) { // Handle latching keys (Caps Lock, etc.) if ((latchingKeys as readonly string[]).includes(cleanKey)) { emit('keyDown', cleanKey) - await sendKeyPress(keyCode, true) + const currentMask = pressedModifiers.value & 0xff + await sendKeyPress(keyCode, true, currentMask) setTimeout(() => { - sendKeyPress(keyCode, false) + sendKeyPress(keyCode, false, currentMask) emit('keyUp', cleanKey) }, 100) return @@ -318,12 +320,14 @@ async function onKeyDown(key: string) { const isCurrentlyDown = (pressedModifiers.value & mask) !== 0 if (isCurrentlyDown) { - pressedModifiers.value &= ~mask - await sendKeyPress(keyCode, false) + const nextMask = pressedModifiers.value & ~mask + pressedModifiers.value = nextMask + await sendKeyPress(keyCode, false, nextMask) emit('keyUp', cleanKey) } else { - pressedModifiers.value |= mask - await sendKeyPress(keyCode, true) + const nextMask = pressedModifiers.value | mask + pressedModifiers.value = nextMask + await sendKeyPress(keyCode, true, nextMask) emit('keyDown', cleanKey) } updateKeyboardButtonTheme() @@ -333,11 +337,12 @@ async function onKeyDown(key: string) { // Regular key: press and release keysDown.value.push(cleanKey) emit('keyDown', cleanKey) - await sendKeyPress(keyCode, true) + const currentMask = pressedModifiers.value & 0xff + await sendKeyPress(keyCode, true, currentMask) updateKeyboardButtonTheme() setTimeout(async () => { keysDown.value = keysDown.value.filter(k => k !== cleanKey) - await sendKeyPress(keyCode, false) + await sendKeyPress(keyCode, false, currentMask) emit('keyUp', cleanKey) updateKeyboardButtonTheme() }, 50) @@ -347,16 +352,9 @@ async function onKeyUp() { // Not used for now - we handle up in onKeyDown with setTimeout } -async function sendKeyPress(keyCode: number, press: boolean) { +async function sendKeyPress(keyCode: number, press: boolean, modifierMask: number) { try { - const mods = { - ctrl: (pressedModifiers.value & 0x11) !== 0, - shift: (pressedModifiers.value & 0x22) !== 0, - alt: (pressedModifiers.value & 0x44) !== 0, - meta: (pressedModifiers.value & 0x88) !== 0, - } - - await hidApi.keyboard(press ? 'down' : 'up', keyCode, mods) + await hidApi.keyboard(press ? 'down' : 'up', keyCode, modifierMask & 0xff) } catch (err) { console.error('[VirtualKeyboard] Key send failed:', err) } @@ -368,16 +366,20 @@ interface MacroStep { } async function executeMacro(steps: MacroStep[]) { + let macroModifierMask = pressedModifiers.value & 0xff + for (const step of steps) { for (const mod of step.modifiers) { if (mod in keys) { - await sendKeyPress(keys[mod as KeyName], true) + const modHid = keys[mod as KeyName] + macroModifierMask = updateModifierMaskForHidKey(macroModifierMask, modHid, true) + await sendKeyPress(modHid, true, macroModifierMask) } } for (const key of step.keys) { if (key in keys) { - await sendKeyPress(keys[key as KeyName], true) + await sendKeyPress(keys[key as KeyName], true, macroModifierMask) } } @@ -385,13 +387,15 @@ async function executeMacro(steps: MacroStep[]) { for (const key of step.keys) { if (key in keys) { - await sendKeyPress(keys[key as KeyName], false) + await sendKeyPress(keys[key as KeyName], false, macroModifierMask) } } for (const mod of step.modifiers) { if (mod in keys) { - await sendKeyPress(keys[mod as KeyName], false) + const modHid = keys[mod as KeyName] + macroModifierMask = updateModifierMaskForHidKey(macroModifierMask, modHid, false) + await sendKeyPress(modHid, false, macroModifierMask) } } } diff --git a/web/src/composables/useHidInput.ts b/web/src/composables/useHidInput.ts index 3c42a8ee..912ef556 100644 --- a/web/src/composables/useHidInput.ts +++ b/web/src/composables/useHidInput.ts @@ -5,6 +5,7 @@ import { ref, type Ref } from 'vue' import { useI18n } from 'vue-i18n' import { toast } from 'vue-sonner' import { hidApi } from '@/api' +import { keyboardEventToHidCode, updateModifierMaskForHidKey } from '@/lib/keyboardMappings' export interface HidInputState { mouseMode: Ref<'absolute' | 'relative'> @@ -32,6 +33,7 @@ export function useHidInput(options: UseHidInputOptions) { numLock: false, scrollLock: false, }) + const activeModifierMask = ref(0) const mousePosition = ref({ x: 0, y: 0 }) const lastMousePosition = ref({ x: 0, y: 0 }) const isPointerLocked = ref(false) @@ -83,14 +85,14 @@ export function useHidInput(options: UseHidInputOptions) { keyboardLed.value.numLock = e.getModifierState('NumLock') keyboardLed.value.scrollLock = e.getModifierState('ScrollLock') - const modifiers = { - ctrl: e.ctrlKey, - shift: e.shiftKey, - alt: e.altKey, - meta: e.metaKey, + const hidKey = keyboardEventToHidCode(e.code, e.key) + if (hidKey === undefined) { + return } - hidApi.keyboard('down', e.keyCode, modifiers).catch(err => handleHidError(err, 'keyboard down')) + const modifierMask = updateModifierMaskForHidKey(activeModifierMask.value, hidKey, true) + activeModifierMask.value = modifierMask + hidApi.keyboard('down', hidKey, modifierMask).catch(err => handleHidError(err, 'keyboard down')) } function handleKeyUp(e: KeyboardEvent) { @@ -107,7 +109,14 @@ export function useHidInput(options: UseHidInputOptions) { const keyName = e.key === ' ' ? 'Space' : e.key pressedKeys.value = pressedKeys.value.filter(k => k !== keyName) - hidApi.keyboard('up', e.keyCode).catch(err => handleHidError(err, 'keyboard up')) + const hidKey = keyboardEventToHidCode(e.code, e.key) + if (hidKey === undefined) { + return + } + + const modifierMask = updateModifierMaskForHidKey(activeModifierMask.value, hidKey, false) + activeModifierMask.value = modifierMask + hidApi.keyboard('up', hidKey, modifierMask).catch(err => handleHidError(err, 'keyboard up')) } // Mouse handlers @@ -233,6 +242,7 @@ export function useHidInput(options: UseHidInputOptions) { function handleBlur() { pressedKeys.value = [] + activeModifierMask.value = 0 if (pressedMouseButton.value !== null) { const button = pressedMouseButton.value pressedMouseButton.value = null diff --git a/web/src/composables/useWebRTC.ts b/web/src/composables/useWebRTC.ts index 3c83cb5d..941d7520 100644 --- a/web/src/composables/useWebRTC.ts +++ b/web/src/composables/useWebRTC.ts @@ -3,7 +3,6 @@ import { ref, onUnmounted, computed, type Ref } from 'vue' import { webrtcApi, type IceCandidate } from '@/api' -import { generateUUID } from '@/lib/utils' import { type HidKeyboardEvent, type HidMouseEvent, @@ -15,6 +14,19 @@ import { useWebSocket } from '@/composables/useWebSocket' export type { HidKeyboardEvent, HidMouseEvent } export type WebRTCState = 'disconnected' | 'connecting' | 'connected' | 'failed' +export type WebRTCConnectStage = + | 'idle' + | 'fetching_ice_servers' + | 'creating_peer_connection' + | 'creating_data_channel' + | 'creating_offer' + | 'waiting_server_answer' + | 'setting_remote_description' + | 'applying_ice_candidates' + | 'waiting_connection' + | 'connected' + | 'disconnected' + | 'failed' // ICE candidate type: host=P2P local, srflx=P2P STUN, relay=TURN relay export type IceCandidateType = 'host' | 'srflx' | 'prflx' | 'relay' | 'unknown' @@ -99,6 +111,7 @@ let dataChannel: RTCDataChannel | null = null let sessionId: string | null = null let statsInterval: number | null = null let isConnecting = false // Lock to prevent concurrent connect calls +let connectInFlight: Promise | null = null let pendingIceCandidates: RTCIceCandidate[] = [] // Queue for ICE candidates before sessionId is set let pendingRemoteCandidates: WebRTCIceCandidateEvent[] = [] // Queue for server ICE candidates let pendingRemoteIceComplete = new Set() // Session IDs waiting for end-of-candidates @@ -131,6 +144,7 @@ const stats = ref({ }) const error = ref(null) const dataChannelReady = ref(false) +const connectStage = ref('idle') // Create RTCPeerConnection with configuration function createPeerConnection(iceServers: RTCIceServer[]): RTCPeerConnection { @@ -149,16 +163,19 @@ function createPeerConnection(iceServers: RTCIceServer[]): RTCPeerConnection { break case 'connected': state.value = 'connected' + connectStage.value = 'connected' error.value = null startStatsCollection() break case 'disconnected': case 'closed': state.value = 'disconnected' + connectStage.value = 'disconnected' stopStatsCollection() break case 'failed': state.value = 'failed' + connectStage.value = 'failed' error.value = 'Connection failed' stopStatsCollection() break @@ -450,100 +467,123 @@ async function flushPendingIceCandidates() { // Connect to WebRTC server async function connect(): Promise { - registerWebSocketHandlers() - - // Prevent concurrent connection attempts - if (isConnecting) { - return false + if (connectInFlight) { + return connectInFlight } - if (peerConnection && state.value === 'connected') { - return true - } + connectInFlight = (async () => { + registerWebSocketHandlers() - isConnecting = true + // Prevent concurrent connection attempts + if (isConnecting) { + return state.value === 'connected' + } - // Clean up any existing connection first - if (peerConnection || sessionId) { - await disconnect() - } + if (peerConnection && state.value === 'connected') { + return true + } - // Clear pending ICE candidates from previous attempt - pendingIceCandidates = [] + isConnecting = true + + // Clean up any existing connection first + if (peerConnection || sessionId) { + await disconnect() + } + + // Clear pending ICE candidates from previous attempt + pendingIceCandidates = [] + + try { + state.value = 'connecting' + error.value = null + connectStage.value = 'fetching_ice_servers' + + // Fetch ICE servers from backend API + const iceServers = await fetchIceServers() + connectStage.value = 'creating_peer_connection' + + // Create peer connection with fetched ICE servers + peerConnection = createPeerConnection(iceServers) + connectStage.value = 'creating_data_channel' + + // Create data channel before offer (for HID) + createDataChannel(peerConnection) + + // Add transceiver for receiving video + peerConnection.addTransceiver('video', { direction: 'recvonly' }) + peerConnection.addTransceiver('audio', { direction: 'recvonly' }) + connectStage.value = 'creating_offer' + + // Create offer + const offer = await peerConnection.createOffer() + await peerConnection.setLocalDescription(offer) + connectStage.value = 'waiting_server_answer' + + // Send offer to server and get answer + // Do not pass client_id here: each connect creates a fresh session. + const response = await webrtcApi.offer(offer.sdp!) + sessionId = response.session_id + + // Send any ICE candidates that were queued while waiting for sessionId + await flushPendingIceCandidates() + + // Set remote description (answer) + const answer: RTCSessionDescriptionInit = { + type: 'answer', + sdp: response.sdp, + } + connectStage.value = 'setting_remote_description' + await peerConnection.setRemoteDescription(answer) + + // Flush any pending server ICE candidates once remote description is set + connectStage.value = 'applying_ice_candidates' + await flushPendingRemoteIce() + + // Add any ICE candidates from the response + if (response.ice_candidates && response.ice_candidates.length > 0) { + for (const candidateObj of response.ice_candidates) { + await addRemoteIceCandidate(candidateObj) + } + } + + // 等待连接真正建立（最多等待 15 秒） + // 直接检查 peerConnection.connectionState 而不是 reactive state + // 因为 TypeScript 不知道 state 会被 onconnectionstatechange 回调异步修改 + const connectionTimeout = 15000 + const pollInterval = 100 + let waited = 0 + connectStage.value = 'waiting_connection' + + while (waited < connectionTimeout && peerConnection) { + const pcState = peerConnection.connectionState + if (pcState === 'connected') { + connectStage.value = 'connected' + isConnecting = false + return true + } + if (pcState === 'failed' || pcState === 'closed') { + throw new Error('Connection failed during ICE negotiation') + } + await new Promise(resolve => setTimeout(resolve, pollInterval)) + waited += pollInterval + } + + // 超时 + throw new Error('Connection timeout waiting for ICE negotiation') + } catch (err) { + state.value = 'failed' + connectStage.value = 'failed' + error.value = err instanceof Error ? err.message : 'Connection failed' + isConnecting = false + await disconnect() + return false + } + })() try { - state.value = 'connecting' - error.value = null - - // Fetch ICE servers from backend API - const iceServers = await fetchIceServers() - - // Create peer connection with fetched ICE servers - peerConnection = createPeerConnection(iceServers) - - // Create data channel before offer (for HID) - createDataChannel(peerConnection) - - // Add transceiver for receiving video - peerConnection.addTransceiver('video', { direction: 'recvonly' }) - peerConnection.addTransceiver('audio', { direction: 'recvonly' }) - - // Create offer - const offer = await peerConnection.createOffer() - await peerConnection.setLocalDescription(offer) - - // Send offer to server and get answer - const response = await webrtcApi.offer(offer.sdp!, generateUUID()) - sessionId = response.session_id - - // Send any ICE candidates that were queued while waiting for sessionId - await flushPendingIceCandidates() - - // Set remote description (answer) - const answer: RTCSessionDescriptionInit = { - type: 'answer', - sdp: response.sdp, - } - await peerConnection.setRemoteDescription(answer) - - // Flush any pending server ICE candidates once remote description is set - await flushPendingRemoteIce() - - // Add any ICE candidates from the response - if (response.ice_candidates && response.ice_candidates.length > 0) { - for (const candidateObj of response.ice_candidates) { - await addRemoteIceCandidate(candidateObj) - } - } - - // 等待连接真正建立（最多等待 15 秒） - // 直接检查 peerConnection.connectionState 而不是 reactive state - // 因为 TypeScript 不知道 state 会被 onconnectionstatechange 回调异步修改 - const connectionTimeout = 15000 - const pollInterval = 100 - let waited = 0 - - while (waited < connectionTimeout && peerConnection) { - const pcState = peerConnection.connectionState - if (pcState === 'connected') { - isConnecting = false - return true - } - if (pcState === 'failed' || pcState === 'closed') { - throw new Error('Connection failed during ICE negotiation') - } - await new Promise(resolve => setTimeout(resolve, pollInterval)) - waited += pollInterval - } - - // 超时 - throw new Error('Connection timeout waiting for ICE negotiation') - } catch (err) { - state.value = 'failed' - error.value = err instanceof Error ? err.message : 'Connection failed' - isConnecting = false - disconnect() - return false + return await connectInFlight + } finally { + connectInFlight = null } } @@ -583,6 +623,7 @@ async function disconnect() { audioTrack.value = null cachedMediaStream = null // Clear cached stream on disconnect state.value = 'disconnected' + connectStage.value = 'disconnected' error.value = null // Reset stats @@ -694,6 +735,7 @@ export function useWebRTC() { stats, error, dataChannelReady, + connectStage, sessionId: computed(() => sessionId), // Methods diff --git a/web/src/i18n/en-US.ts b/web/src/i18n/en-US.ts index dde176d2..35469587 100644 --- a/web/src/i18n/en-US.ts +++ b/web/src/i18n/en-US.ts @@ -312,6 +312,14 @@ export default { webrtcConnectedDesc: 'Using low-latency H.264 video stream', webrtcFailed: 'WebRTC Connection Failed', fallingBackToMjpeg: 'Falling back to MJPEG mode', + webrtcPhaseIceServers: 'Loading ICE servers...', + webrtcPhaseCreatePeer: 'Creating peer connection...', + webrtcPhaseCreateChannel: 'Creating data channel...', + webrtcPhaseCreateOffer: 'Creating local offer...', + webrtcPhaseWaitAnswer: 'Waiting for remote answer...', + webrtcPhaseSetRemote: 'Applying remote description...', + webrtcPhaseApplyIce: 'Applying ICE candidates...', + webrtcPhaseNegotiating: 'Negotiating secure connection...', // Pointer Lock pointerLocked: 'Pointer Locked', pointerLockedDesc: 'Press Escape to release the pointer', @@ -455,7 +463,7 @@ export default { deviceInfo: 'Device Info', deviceInfoDesc: 'Host system information', hostname: 'Hostname', - cpuModel: 'CPU Model', + cpuModel: 'Processor / Platform', cpuUsage: 'CPU Usage', memoryUsage: 'Memory Usage', networkAddresses: 'Network Addresses', diff --git a/web/src/i18n/zh-CN.ts b/web/src/i18n/zh-CN.ts index be87bc94..8b4e3c78 100644 --- a/web/src/i18n/zh-CN.ts +++ b/web/src/i18n/zh-CN.ts @@ -312,6 +312,14 @@ export default { webrtcConnectedDesc: '正在使用 H.264 低延迟视频流', webrtcFailed: 'WebRTC 连接失败', fallingBackToMjpeg: '自动切换到 MJPEG 模式', + webrtcPhaseIceServers: '正在加载 ICE 服务器...', + webrtcPhaseCreatePeer: '正在创建点对点连接...', + webrtcPhaseCreateChannel: '正在创建数据通道...', + webrtcPhaseCreateOffer: '正在创建本地会话描述...', + webrtcPhaseWaitAnswer: '正在等待远端应答...', + webrtcPhaseSetRemote: '正在应用远端会话描述...', + webrtcPhaseApplyIce: '正在应用 ICE 候选...', + webrtcPhaseNegotiating: '正在协商安全连接...', // Pointer Lock pointerLocked: '鼠标已锁定', pointerLockedDesc: '按 Escape 键释放鼠标', @@ -455,7 +463,7 @@ export default { deviceInfo: '设备信息', deviceInfoDesc: '主机系统信息', hostname: '主机名', - cpuModel: 'CPU 型号', + cpuModel: '处理器 / 平台', cpuUsage: 'CPU 利用率', memoryUsage: '内存使用', networkAddresses: '网络地址', diff --git a/web/src/lib/charToHid.ts b/web/src/lib/charToHid.ts index 90edff06..af6e9a82 100644 --- a/web/src/lib/charToHid.ts +++ b/web/src/lib/charToHid.ts @@ -1,136 +1,135 @@ -// Character to JavaScript keyCode mapping for text paste functionality -// Maps printable ASCII characters to JavaScript keyCodes that the backend expects -// The backend (keymap.rs) will convert these JS keyCodes to USB HID keycodes +// Character to HID usage mapping for text paste functionality. +// The table follows US QWERTY layout semantics. + +import { keys } from '@/lib/keyboardMappings' export interface CharKeyMapping { - keyCode: number // JavaScript keyCode (same as KeyboardEvent.keyCode) + hidCode: number // USB HID usage code shift: boolean // Whether Shift modifier is needed } -// US QWERTY keyboard layout mapping -// Maps characters to their JavaScript keyCode and whether Shift is required const charToKeyMap: Record = { - // Lowercase letters (no shift) - JS keyCodes 65-90 - a: { keyCode: 65, shift: false }, - b: { keyCode: 66, shift: false }, - c: { keyCode: 67, shift: false }, - d: { keyCode: 68, shift: false }, - e: { keyCode: 69, shift: false }, - f: { keyCode: 70, shift: false }, - g: { keyCode: 71, shift: false }, - h: { keyCode: 72, shift: false }, - i: { keyCode: 73, shift: false }, - j: { keyCode: 74, shift: false }, - k: { keyCode: 75, shift: false }, - l: { keyCode: 76, shift: false }, - m: { keyCode: 77, shift: false }, - n: { keyCode: 78, shift: false }, - o: { keyCode: 79, shift: false }, - p: { keyCode: 80, shift: false }, - q: { keyCode: 81, shift: false }, - r: { keyCode: 82, shift: false }, - s: { keyCode: 83, shift: false }, - t: { keyCode: 84, shift: false }, - u: { keyCode: 85, shift: false }, - v: { keyCode: 86, shift: false }, - w: { keyCode: 87, shift: false }, - x: { keyCode: 88, shift: false }, - y: { keyCode: 89, shift: false }, - z: { keyCode: 90, shift: false }, + // Lowercase letters + a: { hidCode: keys.KeyA, shift: false }, + b: { hidCode: keys.KeyB, shift: false }, + c: { hidCode: keys.KeyC, shift: false }, + d: { hidCode: keys.KeyD, shift: false }, + e: { hidCode: keys.KeyE, shift: false }, + f: { hidCode: keys.KeyF, shift: false }, + g: { hidCode: keys.KeyG, shift: false }, + h: { hidCode: keys.KeyH, shift: false }, + i: { hidCode: keys.KeyI, shift: false }, + j: { hidCode: keys.KeyJ, shift: false }, + k: { hidCode: keys.KeyK, shift: false }, + l: { hidCode: keys.KeyL, shift: false }, + m: { hidCode: keys.KeyM, shift: false }, + n: { hidCode: keys.KeyN, shift: false }, + o: { hidCode: keys.KeyO, shift: false }, + p: { hidCode: keys.KeyP, shift: false }, + q: { hidCode: keys.KeyQ, shift: false }, + r: { hidCode: keys.KeyR, shift: false }, + s: { hidCode: keys.KeyS, shift: false }, + t: { hidCode: keys.KeyT, shift: false }, + u: { hidCode: keys.KeyU, shift: false }, + v: { hidCode: keys.KeyV, shift: false }, + w: { hidCode: keys.KeyW, shift: false }, + x: { hidCode: keys.KeyX, shift: false }, + y: { hidCode: keys.KeyY, shift: false }, + z: { hidCode: keys.KeyZ, shift: false }, - // Uppercase letters (with shift) - same keyCodes, just with Shift - A: { keyCode: 65, shift: true }, - B: { keyCode: 66, shift: true }, - C: { keyCode: 67, shift: true }, - D: { keyCode: 68, shift: true }, - E: { keyCode: 69, shift: true }, - F: { keyCode: 70, shift: true }, - G: { keyCode: 71, shift: true }, - H: { keyCode: 72, shift: true }, - I: { keyCode: 73, shift: true }, - J: { keyCode: 74, shift: true }, - K: { keyCode: 75, shift: true }, - L: { keyCode: 76, shift: true }, - M: { keyCode: 77, shift: true }, - N: { keyCode: 78, shift: true }, - O: { keyCode: 79, shift: true }, - P: { keyCode: 80, shift: true }, - Q: { keyCode: 81, shift: true }, - R: { keyCode: 82, shift: true }, - S: { keyCode: 83, shift: true }, - T: { keyCode: 84, shift: true }, - U: { keyCode: 85, shift: true }, - V: { keyCode: 86, shift: true }, - W: { keyCode: 87, shift: true }, - X: { keyCode: 88, shift: true }, - Y: { keyCode: 89, shift: true }, - Z: { keyCode: 90, shift: true }, + // Uppercase letters + A: { hidCode: keys.KeyA, shift: true }, + B: { hidCode: keys.KeyB, shift: true }, + C: { hidCode: keys.KeyC, shift: true }, + D: { hidCode: keys.KeyD, shift: true }, + E: { hidCode: keys.KeyE, shift: true }, + F: { hidCode: keys.KeyF, shift: true }, + G: { hidCode: keys.KeyG, shift: true }, + H: { hidCode: keys.KeyH, shift: true }, + I: { hidCode: keys.KeyI, shift: true }, + J: { hidCode: keys.KeyJ, shift: true }, + K: { hidCode: keys.KeyK, shift: true }, + L: { hidCode: keys.KeyL, shift: true }, + M: { hidCode: keys.KeyM, shift: true }, + N: { hidCode: keys.KeyN, shift: true }, + O: { hidCode: keys.KeyO, shift: true }, + P: { hidCode: keys.KeyP, shift: true }, + Q: { hidCode: keys.KeyQ, shift: true }, + R: { hidCode: keys.KeyR, shift: true }, + S: { hidCode: keys.KeyS, shift: true }, + T: { hidCode: keys.KeyT, shift: true }, + U: { hidCode: keys.KeyU, shift: true }, + V: { hidCode: keys.KeyV, shift: true }, + W: { hidCode: keys.KeyW, shift: true }, + X: { hidCode: keys.KeyX, shift: true }, + Y: { hidCode: keys.KeyY, shift: true }, + Z: { hidCode: keys.KeyZ, shift: true }, - // Numbers (no shift) - JS keyCodes 48-57 - '0': { keyCode: 48, shift: false }, - '1': { keyCode: 49, shift: false }, - '2': { keyCode: 50, shift: false }, - '3': { keyCode: 51, shift: false }, - '4': { keyCode: 52, shift: false }, - '5': { keyCode: 53, shift: false }, - '6': { keyCode: 54, shift: false }, - '7': { keyCode: 55, shift: false }, - '8': { keyCode: 56, shift: false }, - '9': { keyCode: 57, shift: false }, + // Number row + '0': { hidCode: keys.Digit0, shift: false }, + '1': { hidCode: keys.Digit1, shift: false }, + '2': { hidCode: keys.Digit2, shift: false }, + '3': { hidCode: keys.Digit3, shift: false }, + '4': { hidCode: keys.Digit4, shift: false }, + '5': { hidCode: keys.Digit5, shift: false }, + '6': { hidCode: keys.Digit6, shift: false }, + '7': { hidCode: keys.Digit7, shift: false }, + '8': { hidCode: keys.Digit8, shift: false }, + '9': { hidCode: keys.Digit9, shift: false }, - // Shifted number row symbols (US layout) - ')': { keyCode: 48, shift: true }, // Shift + 0 - '!': { keyCode: 49, shift: true }, // Shift + 1 - '@': { keyCode: 50, shift: true }, // Shift + 2 - '#': { keyCode: 51, shift: true }, // Shift + 3 - $: { keyCode: 52, shift: true }, // Shift + 4 - '%': { keyCode: 53, shift: true }, // Shift + 5 - '^': { keyCode: 54, shift: true }, // Shift + 6 - '&': { keyCode: 55, shift: true }, // Shift + 7 - '*': { keyCode: 56, shift: true }, // Shift + 8 - '(': { keyCode: 57, shift: true }, // Shift + 9 + // Shifted number row symbols + ')': { hidCode: keys.Digit0, shift: true }, + '!': { hidCode: keys.Digit1, shift: true }, + '@': { hidCode: keys.Digit2, shift: true }, + '#': { hidCode: keys.Digit3, shift: true }, + '$': { hidCode: keys.Digit4, shift: true }, + '%': { hidCode: keys.Digit5, shift: true }, + '^': { hidCode: keys.Digit6, shift: true }, + '&': { hidCode: keys.Digit7, shift: true }, + '*': { hidCode: keys.Digit8, shift: true }, + '(': { hidCode: keys.Digit9, shift: true }, - // Punctuation and symbols (no shift) - US layout JS keyCodes - '-': { keyCode: 189, shift: false }, // Minus - '=': { keyCode: 187, shift: false }, // Equal - '[': { keyCode: 219, shift: false }, // Left bracket - ']': { keyCode: 221, shift: false }, // Right bracket - '\\': { keyCode: 220, shift: false }, // Backslash - ';': { keyCode: 186, shift: false }, // Semicolon - "'": { keyCode: 222, shift: false }, // Apostrophe/Quote - '`': { keyCode: 192, shift: false }, // Grave/Backtick - ',': { keyCode: 188, shift: false }, // Comma - '.': { keyCode: 190, shift: false }, // Period - '/': { keyCode: 191, shift: false }, // Slash + // Punctuation and symbols + '-': { hidCode: keys.Minus, shift: false }, + '=': { hidCode: keys.Equal, shift: false }, + '[': { hidCode: keys.BracketLeft, shift: false }, + ']': { hidCode: keys.BracketRight, shift: false }, + '\\': { hidCode: keys.Backslash, shift: false }, + ';': { hidCode: keys.Semicolon, shift: false }, + "'": { hidCode: keys.Quote, shift: false }, + '`': { hidCode: keys.Backquote, shift: false }, + ',': { hidCode: keys.Comma, shift: false }, + '.': { hidCode: keys.Period, shift: false }, + '/': { hidCode: keys.Slash, shift: false }, - // Shifted punctuation and symbols (US layout) - _: { keyCode: 189, shift: true }, // Shift + Minus = Underscore - '+': { keyCode: 187, shift: true }, // Shift + Equal = Plus - '{': { keyCode: 219, shift: true }, // Shift + [ = { - '}': { keyCode: 221, shift: true }, // Shift + ] = } - '|': { keyCode: 220, shift: true }, // Shift + \ = | - ':': { keyCode: 186, shift: true }, // Shift + ; = : - '"': { keyCode: 222, shift: true }, // Shift + ' = " - '~': { keyCode: 192, shift: true }, // Shift + ` = ~ - '<': { keyCode: 188, shift: true }, // Shift + , = < - '>': { keyCode: 190, shift: true }, // Shift + . = > - '?': { keyCode: 191, shift: true }, // Shift + / = ? + // Shifted punctuation and symbols + _: { hidCode: keys.Minus, shift: true }, + '+': { hidCode: keys.Equal, shift: true }, + '{': { hidCode: keys.BracketLeft, shift: true }, + '}': { hidCode: keys.BracketRight, shift: true }, + '|': { hidCode: keys.Backslash, shift: true }, + ':': { hidCode: keys.Semicolon, shift: true }, + '"': { hidCode: keys.Quote, shift: true }, + '~': { hidCode: keys.Backquote, shift: true }, + '<': { hidCode: keys.Comma, shift: true }, + '>': { hidCode: keys.Period, shift: true }, + '?': { hidCode: keys.Slash, shift: true }, - // Whitespace and control characters - ' ': { keyCode: 32, shift: false }, // Space - '\t': { keyCode: 9, shift: false }, // Tab - '\n': { keyCode: 13, shift: false }, // Enter (LF) - '\r': { keyCode: 13, shift: false }, // Enter (CR) + // Whitespace and control + ' ': { hidCode: keys.Space, shift: false }, + '\t': { hidCode: keys.Tab, shift: false }, + '\n': { hidCode: keys.Enter, shift: false }, + '\r': { hidCode: keys.Enter, shift: false }, } /** - * Get the JavaScript keyCode and modifier state for a character + * Get HID usage code and modifier state for a character * @param char - Single character to convert * @returns CharKeyMapping or null if character is not mappable */ export function charToKey(char: string): CharKeyMapping | null { if (char.length !== 1) return null - return charToKeyMap[char] || null + return charToKeyMap[char] ?? null } /** @@ -138,7 +137,7 @@ export function charToKey(char: string): CharKeyMapping | null { * @param char - Single character to check */ export function isTypableChar(char: string): boolean { - return char.length === 1 && char in charToKeyMap + return charToKey(char) !== null } /** diff --git a/web/src/lib/keyboardMappings.ts b/web/src/lib/keyboardMappings.ts index 7e05924a..18605b82 100644 --- a/web/src/lib/keyboardMappings.ts +++ b/web/src/lib/keyboardMappings.ts @@ -191,6 +191,13 @@ export const hidKeyToModifierMask: Record = { 0xe7: 0x80, // MetaRight } +// Update modifier mask when a HID modifier key is pressed/released. +export function updateModifierMaskForHidKey(mask: number, hidKey: number, press: boolean): number { + const bit = hidKeyToModifierMask[hidKey] ?? 0 + if (bit === 0) return mask + return press ? (mask | bit) : (mask & ~bit) +} + // Keys that latch (toggle state) instead of being held export const latchingKeys = ['CapsLock', 'ScrollLock', 'NumLock'] as const @@ -220,6 +227,23 @@ export function getModifierMask(keyName: string): number { return 0 } +// Normalize browser-specific KeyboardEvent.code variants. +export function normalizeKeyboardCode(code: string, key: string): string { + if (code === 'IntlBackslash' && (key === '`' || key === '~')) return 'Backquote' + if (code === 'Backquote' && (key === '§' || key === '±')) return 'IntlBackslash' + if (code === 'IntlYen') return 'IntlBackslash' + if (code === 'OSLeft') return 'MetaLeft' + if (code === 'OSRight') return 'MetaRight' + if (code === '' && key === 'Shift') return 'ShiftRight' + return code +} + +// Convert KeyboardEvent.code/key to USB HID usage code. +export function keyboardEventToHidCode(code: string, key: string): number | undefined { + const normalizedCode = normalizeKeyboardCode(code, key) + return keys[normalizedCode as KeyName] +} + // Decode modifier byte into individual states export function decodeModifiers(modifier: number) { return { diff --git a/web/src/types/hid.ts b/web/src/types/hid.ts index d1bd96be..0e9dc3ad 100644 --- a/web/src/types/hid.ts +++ b/web/src/types/hid.ts @@ -5,12 +5,8 @@ export interface HidKeyboardEvent { type: 'keydown' | 'keyup' key: number - modifiers?: { - ctrl?: boolean - shift?: boolean - alt?: boolean - meta?: boolean - } + /** Raw HID modifier byte (bit0..bit7 = LCtrl..RMeta) */ + modifier?: number } /** Mouse event for HID input */ @@ -57,13 +53,7 @@ export function encodeKeyboardEvent(event: HidKeyboardEvent): ArrayBuffer { view.setUint8(1, event.type === 'keydown' ? KB_EVENT_DOWN : KB_EVENT_UP) view.setUint8(2, event.key & 0xff) - // Build modifiers bitmask - let modifiers = 0 - if (event.modifiers?.ctrl) modifiers |= 0x01 // Left Ctrl - if (event.modifiers?.shift) modifiers |= 0x02 // Left Shift - if (event.modifiers?.alt) modifiers |= 0x04 // Left Alt - if (event.modifiers?.meta) modifiers |= 0x08 // Left Meta - view.setUint8(3, modifiers) + view.setUint8(3, (event.modifier ?? 0) & 0xff) return buffer } diff --git a/web/src/views/ConsoleView.vue b/web/src/views/ConsoleView.vue index 27172d6b..fa055053 100644 --- a/web/src/views/ConsoleView.vue +++ b/web/src/views/ConsoleView.vue @@ -13,6 +13,7 @@ import { useVideoSession } from '@/composables/useVideoSession' import { getUnifiedAudio } from '@/composables/useUnifiedAudio' import { streamApi, hidApi, atxApi, extensionsApi, atxConfigApi, authApi } from '@/api' import type { HidKeyboardEvent, HidMouseEvent } from '@/types/hid' +import { keyboardEventToHidCode, updateModifierMaskForHidKey } from '@/lib/keyboardMappings' import { toast } from 'vue-sonner' import { generateUUID } from '@/lib/utils' import type { VideoMode } from '@/components/VideoConfigPopover.vue' @@ -121,6 +122,7 @@ const pressedKeys = ref([]) const keyboardLed = ref({ capsLock: false, }) +const activeModifierMask = ref(0) const mousePosition = ref({ x: 0, y: 0 }) const lastMousePosition = ref({ x: 0, y: 0 }) // Track last position for relative mode const isPointerLocked = ref(false) // Track pointer lock state @@ -407,6 +409,37 @@ const msdDetails = computed(() => { return details }) +const webrtcLoadingMessage = computed(() => { + if (videoMode.value === 'mjpeg') { + return videoRestarting.value ? t('console.videoRestarting') : t('console.connecting') + } + + switch (webrtc.connectStage.value) { + case 'fetching_ice_servers': + return t('console.webrtcPhaseIceServers') + case 'creating_peer_connection': + return t('console.webrtcPhaseCreatePeer') + case 'creating_data_channel': + return t('console.webrtcPhaseCreateChannel') + case 'creating_offer': + return t('console.webrtcPhaseCreateOffer') + case 'waiting_server_answer': + return t('console.webrtcPhaseWaitAnswer') + case 'setting_remote_description': + return t('console.webrtcPhaseSetRemote') + case 'applying_ice_candidates': + return t('console.webrtcPhaseApplyIce') + case 'waiting_connection': + return t('console.webrtcPhaseNegotiating') + case 'connected': + return t('console.webrtcConnected') + case 'failed': + return t('console.webrtcFailed') + default: + return videoRestarting.value ? t('console.videoRestarting') : t('console.connecting') + } +}) + const showMsdStatusCard = computed(() => { return !!(systemStore.msd?.available && systemStore.hid?.backend !== 'ch9329') }) @@ -423,6 +456,8 @@ let consecutiveErrors = 0 const BASE_RETRY_DELAY = 2000 const GRACE_PERIOD = 2000 // Ignore errors for 2s after config change (reduced from 3s) const MAX_CONSECUTIVE_ERRORS = 2 // If 2+ errors in grace period, it's a real problem +let pendingWebRTCReadyGate = false +let webrtcConnectTask: Promise | null = null // Last-frame overlay (prevents black flash during mode switches) const frameOverlayUrl = ref(null) @@ -492,6 +527,52 @@ function waitForVideoFirstFrame(el: HTMLVideoElement, timeoutMs = 2000): Promise }) } +function shouldSuppressAutoReconnect(): boolean { + return videoMode.value === 'mjpeg' + || videoSession.localSwitching.value + || videoSession.backendSwitching.value + || videoRestarting.value +} + +function markWebRTCFailure(reason: string, description?: string) { + pendingWebRTCReadyGate = false + videoError.value = true + videoErrorMessage.value = reason + videoLoading.value = false + systemStore.setStreamOnline(false) + + toast.error(reason, { + description: description ?? '', + duration: 5000, + }) +} + +async function waitForWebRTCReadyGate(reason: string, timeoutMs = 3000): Promise { + if (!pendingWebRTCReadyGate) return + const ready = await videoSession.waitForWebRTCReadyAny(timeoutMs) + if (!ready) { + console.warn(`[WebRTC] Ready gate timeout (${reason}), attempting connection anyway`) + } + pendingWebRTCReadyGate = false +} + +async function connectWebRTCSerial(reason: string): Promise { + if (webrtcConnectTask) { + return webrtcConnectTask + } + + webrtcConnectTask = (async () => { + await waitForWebRTCReadyGate(reason) + return webrtc.connect() + })() + + try { + return await webrtcConnectTask + } finally { + webrtcConnectTask = null + } +} + function handleVideoLoad() { // MJPEG video frame loaded successfully - update stream online status // This fixes the timing issue where device_info event may arrive before stream is fully active @@ -612,9 +693,9 @@ async function handleAudioStateChanged(data: { streaming: boolean; device: strin if (!webrtc.audioTrack.value) { // No audio track - need to reconnect WebRTC to get one // This happens when audio was enabled after WebRTC session was created - webrtc.disconnect() + await webrtc.disconnect() await new Promise(resolve => setTimeout(resolve, 300)) - await webrtc.connect() + await connectWebRTCSerial('audio track refresh') // After reconnect, the new session will have audio track // and the watch on audioTrack will add it to MediaStream } else { @@ -645,6 +726,7 @@ function handleStreamConfigChanging(data: any) { // Reset all counters and states videoRestarting.value = true + pendingWebRTCReadyGate = true videoLoading.value = true videoError.value = false retryCount = 0 @@ -670,7 +752,7 @@ async function handleStreamConfigApplied(data: any) { }, GRACE_PERIOD) // Refresh video based on current mode - videoRestarting.value = false + videoRestarting.value = true // 如果正在进行模式切换，不需要在这里处理（WebRTCReady 事件会处理） if (isModeSwitching.value) { @@ -680,16 +762,15 @@ async function handleStreamConfigApplied(data: any) { if (videoMode.value !== 'mjpeg') { // In WebRTC mode, reconnect WebRTC (session was closed due to config change) - const ready = await videoSession.waitForWebRTCReadyAny(3000) - if (!ready) { - console.warn('[WebRTC] Backend not ready after timeout (config change), attempting connection anyway') - } - switchToWebRTC(videoMode.value) + // connectWebRTCSerial() will wait on stream.webrtc_ready when gate is enabled. + await switchToWebRTC(videoMode.value) } else { // In MJPEG mode, refresh the MJPEG stream refreshVideo() } + videoRestarting.value = false + toast.success(t('console.videoRestarted'), { description: `${data.device} - ${data.resolution[0]}x${data.resolution[1]} @ ${data.fps}fps`, duration: 3000, @@ -699,11 +780,15 @@ async function handleStreamConfigApplied(data: any) { // 处理 WebRTC 就绪事件 - 这是后端真正准备好接受 WebRTC 连接的信号 function handleWebRTCReady(data: { codec: string; hardware: boolean; transition_id?: string }) { console.log(`[WebRTCReady] Backend ready: codec=${data.codec}, hardware=${data.hardware}, transition_id=${data.transition_id || '-'}`) + pendingWebRTCReadyGate = false videoSession.onWebRTCReady(data) } function handleStreamModeReady(data: { transition_id: string; mode: string }) { videoSession.onModeReady(data) + if (data.mode === 'mjpeg') { + pendingWebRTCReadyGate = false + } videoRestarting.value = false } @@ -714,6 +799,7 @@ function handleStreamModeSwitching(data: { transition_id: string; to_mode: strin videoLoading.value = true captureFrameOverlay().catch(() => {}) } + pendingWebRTCReadyGate = true videoSession.onModeSwitching(data) } @@ -758,6 +844,40 @@ function handleStreamStatsUpdate(data: any) { // Track if we've received the initial device_info let initialDeviceInfoReceived = false +let initialModeRestoreDone = false +let initialModeRestoreInProgress = false + +function normalizeServerMode(mode: string | undefined): VideoMode | null { + if (!mode) return null + if (mode === 'webrtc') return 'h264' + if (mode === 'mjpeg' || mode === 'h264' || mode === 'h265' || mode === 'vp8' || mode === 'vp9') { + return mode + } + return null +} + +async function restoreInitialMode(serverMode: VideoMode) { + if (initialModeRestoreDone || initialModeRestoreInProgress) return + initialModeRestoreInProgress = true + + try { + initialDeviceInfoReceived = true + if (serverMode !== videoMode.value) { + videoMode.value = serverMode + localStorage.setItem('videoMode', serverMode) + } + + if (serverMode !== 'mjpeg') { + await connectWebRTCOnly(serverMode) + } else if (mjpegTimestamp.value === 0) { + refreshVideo() + } + + initialModeRestoreDone = true + } finally { + initialModeRestoreInProgress = false + } +} function handleDeviceInfo(data: any) { systemStore.updateFromDeviceInfo(data) @@ -770,40 +890,28 @@ function handleDeviceInfo(data: any) { // Sync video mode from server's stream_mode if (data.video?.stream_mode) { - // Server returns: 'mjpeg', 'h264', 'h265', 'vp8', 'vp9', or 'webrtc' - const serverStreamMode = data.video.stream_mode - const serverMode = serverStreamMode === 'webrtc' ? 'h264' : serverStreamMode as VideoMode + const serverMode = normalizeServerMode(data.video.stream_mode) + if (!serverMode) return if (!initialDeviceInfoReceived) { - // First device_info - initialize to server mode initialDeviceInfoReceived = true - - if (serverMode !== videoMode.value) { - // Server mode differs from default, sync to server mode without calling setMode - videoMode.value = serverMode - if (serverMode !== 'mjpeg') { - setTimeout(() => connectWebRTCOnly(serverMode), 100) - } else { - setTimeout(() => refreshVideo(), 100) - } - } else if (serverMode !== 'mjpeg') { - // Server is in WebRTC mode and client default matches, connect WebRTC (no setMode) - setTimeout(() => connectWebRTCOnly(serverMode), 100) - } else if (serverMode === 'mjpeg') { - // Server is in MJPEG mode and client default is also mjpeg, start MJPEG stream - setTimeout(() => refreshVideo(), 100) + if (!initialModeRestoreDone && !initialModeRestoreInProgress) { + void restoreInitialMode(serverMode) + return } - } else if (serverMode !== videoMode.value) { - // Subsequent device_info with mode change - sync to server (no setMode) - syncToServerMode(serverMode as VideoMode) + } + + if (initialModeRestoreInProgress) return + if (serverMode !== videoMode.value) { + syncToServerMode(serverMode) } } } // Handle stream mode change event from server (WebSocket broadcast) function handleStreamModeChanged(data: { mode: string; previous_mode: string }) { - // Server returns: 'mjpeg', 'h264', 'h265', 'vp8', 'vp9', or 'webrtc' - const newMode = data.mode === 'webrtc' ? 'h264' : data.mode as VideoMode + const newMode = normalizeServerMode(data.mode) + if (!newMode) return // 如果正在进行模式切换，忽略这个事件（这是我们自己触发的切换产生的） if (isModeSwitching.value) { @@ -819,7 +927,7 @@ function handleStreamModeChanged(data: { mode: string; previous_mode: string }) // Switch to new mode (external sync handled by device_info after mode_ready) if (newMode !== videoMode.value) { - syncToServerMode(newMode as VideoMode) + syncToServerMode(newMode) } } @@ -892,7 +1000,7 @@ async function connectWebRTCOnly(codec: VideoMode = 'h264') { videoErrorMessage.value = '' try { - const success = await webrtc.connect() + const success = await connectWebRTCSerial('connectWebRTCOnly') if (success) { toast.success(t('console.webrtcConnected'), { description: t('console.webrtcConnectedDesc'), @@ -910,7 +1018,7 @@ async function connectWebRTCOnly(codec: VideoMode = 'h264') { throw new Error('WebRTC connection failed') } } catch { - await fallbackToMJPEG(t('console.webrtcFailed'), t('console.fallingBackToMjpeg')) + markWebRTCFailure(t('console.webrtcFailed')) } } @@ -961,6 +1069,7 @@ async function switchToWebRTC(codec: VideoMode = 'h264') { videoLoading.value = true videoError.value = false videoErrorMessage.value = '' + pendingWebRTCReadyGate = true try { // Step 1: Disconnect existing WebRTC connection FIRST @@ -995,7 +1104,7 @@ async function switchToWebRTC(codec: VideoMode = 'h264') { let retries = 3 let success = false while (retries > 0 && !success) { - success = await webrtc.connect() + success = await connectWebRTCSerial('switchToWebRTC') if (!success) { retries-- if (retries > 0) { @@ -1021,30 +1130,7 @@ async function switchToWebRTC(codec: VideoMode = 'h264') { throw new Error('WebRTC connection failed') } } catch { - await fallbackToMJPEG(t('console.webrtcFailed'), t('console.fallingBackToMjpeg'), true) - } -} - -async function fallbackToMJPEG(reason: string, description?: string, force = false) { - if (fallbackInProgress) return - if (videoMode.value === 'mjpeg') return - if (!force && (videoSession.localSwitching.value || videoSession.backendSwitching.value)) return - - fallbackInProgress = true - videoError.value = true - videoErrorMessage.value = reason - videoLoading.value = false - - toast.error(reason, { - description: description ?? '', - duration: 5000, - }) - - videoMode.value = 'mjpeg' - try { - await switchToMJPEG() - } finally { - fallbackInProgress = false + markWebRTCFailure(t('console.webrtcFailed')) } } @@ -1052,6 +1138,7 @@ async function switchToMJPEG() { videoLoading.value = true videoError.value = false videoErrorMessage.value = '' + pendingWebRTCReadyGate = false // Step 1: Call backend API to switch mode FIRST // This ensures the MJPEG endpoint will accept our request @@ -1069,9 +1156,9 @@ async function switchToMJPEG() { // Continue anyway - the mode might already be correct } - // Step 2: Disconnect WebRTC if connected - if (webrtc.isConnected.value) { - webrtc.disconnect() + // Step 2: Disconnect WebRTC if connected or session still exists + if (webrtc.isConnected.value || webrtc.sessionId.value) { + await webrtc.disconnect() } // Clear WebRTC video @@ -1181,10 +1268,19 @@ watch(webrtc.stats, (stats) => { // Watch for WebRTC connection state changes - auto-reconnect on disconnect let webrtcReconnectTimeout: ReturnType | null = null let webrtcReconnectFailures = 0 -let fallbackInProgress = false watch(() => webrtc.state.value, (newState, oldState) => { console.log('[WebRTC] State changed:', oldState, '->', newState) + // Clear any pending reconnect + if (webrtcReconnectTimeout) { + clearTimeout(webrtcReconnectTimeout) + webrtcReconnectTimeout = null + } + + if (shouldSuppressAutoReconnect()) { + return + } + // Update stream online status based on WebRTC connection state if (videoMode.value !== 'mjpeg') { if (newState === 'connected') { @@ -1196,28 +1292,22 @@ watch(() => webrtc.state.value, (newState, oldState) => { } } - // Clear any pending reconnect - if (webrtcReconnectTimeout) { - clearTimeout(webrtcReconnectTimeout) - webrtcReconnectTimeout = null - } - // Auto-reconnect when disconnected (but was previously connected) if (newState === 'disconnected' && oldState === 'connected' && videoMode.value !== 'mjpeg') { webrtcReconnectTimeout = setTimeout(async () => { if (videoMode.value !== 'mjpeg' && webrtc.state.value === 'disconnected') { try { - const success = await webrtc.connect() + const success = await connectWebRTCSerial('auto reconnect') if (!success) { webrtcReconnectFailures += 1 if (webrtcReconnectFailures >= 2) { - await fallbackToMJPEG(t('console.webrtcFailed'), t('console.fallingBackToMjpeg')) + markWebRTCFailure(t('console.webrtcFailed')) } } } catch { webrtcReconnectFailures += 1 if (webrtcReconnectFailures >= 2) { - await fallbackToMJPEG(t('console.webrtcFailed'), t('console.fallingBackToMjpeg')) + markWebRTCFailure(t('console.webrtcFailed')) } } } @@ -1227,7 +1317,7 @@ watch(() => webrtc.state.value, (newState, oldState) => { if (newState === 'failed' && videoMode.value !== 'mjpeg') { webrtcReconnectFailures += 1 if (webrtcReconnectFailures >= 1) { - fallbackToMJPEG(t('console.webrtcFailed'), t('console.fallingBackToMjpeg')).catch(() => {}) + markWebRTCFailure(t('console.webrtcFailed')) } } }) @@ -1358,20 +1448,20 @@ function handleHidError(_error: any, _operation: string) { } // HID channel selection: use WebRTC DataChannel when available, fallback to WebSocket -function sendKeyboardEvent(type: 'down' | 'up', key: number, modifiers?: { ctrl?: boolean; shift?: boolean; alt?: boolean; meta?: boolean }) { +function sendKeyboardEvent(type: 'down' | 'up', key: number, modifier?: number) { // In WebRTC mode with DataChannel ready, use DataChannel for lower latency if (videoMode.value !== 'mjpeg' && webrtc.dataChannelReady.value) { const event: HidKeyboardEvent = { type: type === 'down' ? 'keydown' : 'keyup', key, - modifiers, + modifier, } const sent = webrtc.sendKeyboard(event) if (sent) return // Fallback to WebSocket if DataChannel send failed } // Use WebSocket as fallback or for MJPEG mode - hidApi.keyboard(type, key, modifiers).catch(err => handleHidError(err, `keyboard ${type}`)) + hidApi.keyboard(type, key, modifier).catch(err => handleHidError(err, `keyboard ${type}`)) } function sendMouseEvent(data: { type: 'move' | 'move_abs' | 'down' | 'up' | 'scroll'; x?: number; y?: number; button?: 'left' | 'right' | 'middle'; scroll?: number }) { @@ -1444,14 +1534,15 @@ function handleKeyDown(e: KeyboardEvent) { keyboardLed.value.capsLock = e.getModifierState('CapsLock') - const modifiers = { - ctrl: e.ctrlKey, - shift: e.shiftKey, - alt: e.altKey, - meta: e.metaKey, + const hidKey = keyboardEventToHidCode(e.code, e.key) + if (hidKey === undefined) { + console.warn(`[HID] Unmapped key down: code=${e.code}, key=${e.key}`) + return } - sendKeyboardEvent('down', e.keyCode, modifiers) + const modifierMask = updateModifierMaskForHidKey(activeModifierMask.value, hidKey, true) + activeModifierMask.value = modifierMask + sendKeyboardEvent('down', hidKey, modifierMask) } function handleKeyUp(e: KeyboardEvent) { @@ -1470,7 +1561,15 @@ function handleKeyUp(e: KeyboardEvent) { const keyName = e.key === ' ' ? 'Space' : e.key pressedKeys.value = pressedKeys.value.filter(k => k !== keyName) - sendKeyboardEvent('up', e.keyCode) + const hidKey = keyboardEventToHidCode(e.code, e.key) + if (hidKey === undefined) { + console.warn(`[HID] Unmapped key up: code=${e.code}, key=${e.key}`) + return + } + + const modifierMask = updateModifierMaskForHidKey(activeModifierMask.value, hidKey, false) + activeModifierMask.value = modifierMask + sendKeyboardEvent('up', hidKey, modifierMask) } function handleMouseMove(e: MouseEvent) { @@ -1689,6 +1788,7 @@ function handlePointerLockError() { function handleBlur() { pressedKeys.value = [] + activeModifierMask.value = 0 // Release any pressed mouse button when window loses focus if (pressedMouseButton.value !== null) { const button = pressedMouseButton.value @@ -1846,11 +1946,22 @@ onMounted(async () => { // Note: Video mode is now synced from server via device_info event // The handleDeviceInfo function will automatically switch to the server's mode // localStorage preference is only used when server mode matches + try { + const modeResp = await streamApi.getMode() + const serverMode = normalizeServerMode(modeResp?.mode) + if (serverMode && !initialModeRestoreDone && !initialModeRestoreInProgress) { + await restoreInitialMode(serverMode) + } + } catch (err) { + console.warn('[Console] Failed to fetch stream mode on enter, fallback to WS events:', err) + } }) onUnmounted(() => { // Reset initial device info flag initialDeviceInfoReceived = false + initialModeRestoreDone = false + initialModeRestoreInProgress = false // Clear mouse flush timer if (mouseFlushTimer !== null) { @@ -1881,9 +1992,9 @@ onUnmounted(() => { consoleEvents.unsubscribe() consecutiveErrors = 0 - // Disconnect WebRTC if connected - if (webrtc.isConnected.value) { - webrtc.disconnect() + // Disconnect WebRTC if connected or session still exists + if (webrtc.isConnected.value || webrtc.sessionId.value) { + void webrtc.disconnect() } // Exit pointer lock if active @@ -2161,7 +2272,7 @@ onUnmounted(() => {

- {{ videoRestarting ? t('console.videoRestarting') : t('console.connecting') }} + {{ webrtcLoadingMessage }}

{{ t('console.pleaseWait') }}