fix: 优化 WebRTC 建连流程、修复平台信息、修复虚拟键盘键值映射

- WebRTC：默认 mDNS 调整为 QueryOnly，Answer 阶段改为等待 ICE gathering complete（2.5s 超时），提升首次建连成功率与候选完整性
- WebRTC：前端建连流程增加阶段化状态与串行保护（connectInFlight/ready gate），优化配置变更后的重连时机与失败处理，减少竞态和无效重试
- Device：平台信息补充 `/proc/device-tree/model` 回退并统一展示为“处理器/平台”
- HID：键盘输入链路统一为 HID usage + modifier bitmask，修复虚拟键盘/宏/粘贴键值映射错误

src/hid/datachannel.rs

@@ -9,7 +9,7 @@
 //!
 //! Keyboard event (type 0x01):
 //! - Byte 1: Event type (0x00 = down, 0x01 = up)
-//! - Byte 2: Key code (USB HID usage code or JS keyCode)
+//! - Byte 2: Key code (USB HID usage code)
 //! - Byte 3: Modifiers bitmask
 //!   - Bit 0: Left Ctrl
 //!   - Bit 1: Left Shift
@@ -119,7 +119,7 @@ fn parse_keyboard_message(data: &[u8]) -> Option<HidChannelEvent> {
         event_type,
         key,
         modifiers,
-        is_usb_hid: false, // WebRTC datachannel sends JS keycodes
+        is_usb_hid: true, // WebRTC/WebSocket HID channel sends USB HID usages
     }))
 }
 
@@ -245,6 +245,7 @@ mod tests {
                 assert_eq!(kb.key, 0x04);
                 assert!(kb.modifiers.left_ctrl);
                 assert!(!kb.modifiers.left_shift);
+                assert!(kb.is_usb_hid);
             }
             _ => panic!("Expected keyboard event"),
         }
@@ -280,7 +281,7 @@ mod tests {
                 right_alt: false,
                 right_meta: false,
             },
-            is_usb_hid: false,
+            is_usb_hid: true,
         };
 
         let encoded = encode_keyboard_event(&event);

src/web/handlers/mod.rs

@@ -183,31 +183,59 @@ fn get_hostname() -> String {
         .unwrap_or_else(|_| "unknown".to_string())
 }
 
-/// Get CPU model name from /proc/cpuinfo
+/// Get CPU model name from /proc/cpuinfo, fallback to device-tree model
 fn get_cpu_model() -> String {
-    std::fs::read_to_string("/proc/cpuinfo")
+    let cpuinfo = std::fs::read_to_string("/proc/cpuinfo").ok();
+
+    if let Some(model) = cpuinfo
+        .as_deref()
+        .and_then(parse_cpu_model_from_cpuinfo_content)
+    {
+        return model;
+    }
+
+    if let Some(model) = read_device_tree_model() {
+        return model;
+    }
+
+    if let Some(content) = cpuinfo.as_deref() {
+        let cores = content
+            .lines()
+            .filter(|line| line.starts_with("processor"))
+            .count();
+        if cores > 0 {
+            return format!("{} {}C", std::env::consts::ARCH, cores);
+        }
+    }
+
+    std::env::consts::ARCH.to_string()
+}
+
+fn parse_cpu_model_from_cpuinfo_content(content: &str) -> Option<String> {
+    content
+        .lines()
+        .find(|line| line.starts_with("model name") || line.starts_with("Model"))
+        .and_then(|line| line.split(':').nth(1))
+        .map(|s| s.trim().to_string())
+        .filter(|s| !s.is_empty())
+}
+
+fn read_device_tree_model() -> Option<String> {
+    std::fs::read("/proc/device-tree/model")
         .ok()
-        .and_then(|content| {
+        .and_then(|bytes| parse_device_tree_model_bytes(&bytes))
-            // Try to get model name
+}
-            let model = content
-                .lines()
-                .find(|line| line.starts_with("model name") || line.starts_with("Model"))
-                .and_then(|line| line.split(':').nth(1))
-                .map(|s| s.trim().to_string())
-                .filter(|s| !s.is_empty());
 
-            if model.is_some() {
+fn parse_device_tree_model_bytes(bytes: &[u8]) -> Option<String> {
-                return model;
+    let model = String::from_utf8_lossy(bytes)
-            }
+        .trim_matches(|c: char| c == '\0' || c.is_whitespace())
+        .to_string();
 
-            // Fallback: show arch and core count
+    if model.is_empty() {
-            let cores = content
+        None
-                .lines()
+    } else {
-                .filter(|line| line.starts_with("processor"))
+        Some(model)
-                .count();
+    }
-            Some(format!("{} {}C", std::env::consts::ARCH, cores))
-        })
-        .unwrap_or_else(|| std::env::consts::ARCH.to_string())
 }
 
 /// CPU usage state for calculating usage between samples
@@ -389,6 +417,38 @@ fn get_network_addresses() -> Vec<NetworkAddress> {
     addresses
 }
 
+#[cfg(test)]
+mod tests {
+    use super::{parse_cpu_model_from_cpuinfo_content, parse_device_tree_model_bytes};
+
+    #[test]
+    fn parse_cpu_model_from_model_name_field() {
+        let input = "processor\t: 0\nmodel name\t: Intel(R) Xeon(R)\n";
+        assert_eq!(
+            parse_cpu_model_from_cpuinfo_content(input),
+            Some("Intel(R) Xeon(R)".to_string())
+        );
+    }
+
+    #[test]
+    fn parse_cpu_model_from_model_field() {
+        let input = "processor\t: 0\nModel\t\t: Raspberry Pi 4 Model B Rev 1.4\n";
+        assert_eq!(
+            parse_cpu_model_from_cpuinfo_content(input),
+            Some("Raspberry Pi 4 Model B Rev 1.4".to_string())
+        );
+    }
+
+    #[test]
+    fn parse_device_tree_model_trimmed() {
+        let input = b"Onething OEC Box\0\n";
+        assert_eq!(
+            parse_device_tree_model_bytes(input),
+            Some("Onething OEC Box".to_string())
+        );
+    }
+}
+
 // ============================================================================
 // Authentication
 // ============================================================================
@@ -2053,10 +2113,11 @@ pub async fn webrtc_offer(
         ));
     }
 
-    // Create session if client_id not provided
+    // Backward compatibility: `client_id` is treated as an existing session_id hint.
+    // New clients should not pass it; each offer creates a fresh session.
     let webrtc = state.stream_manager.webrtc_streamer();
     let session_id = if let Some(client_id) = &req.client_id {
-        // Check if session exists
+        // Reuse only when it matches an active session ID.
         if webrtc.get_session(client_id).await.is_some() {
             client_id.clone()
         } else {
@@ -2411,15 +2472,13 @@ pub async fn hid_otg_self_check(State(state): State<Arc<AppState>>) -> Json<OtgS
     let hid_backend_is_otg = matches!(config.hid.backend, crate::config::HidBackend::Otg);
     let mut checks = Vec::new();
 
-    let build_response = |
+    let build_response = |checks: Vec<OtgSelfCheckItem>,
-        checks: Vec<OtgSelfCheckItem>,
+                          selected_udc: Option<String>,
-        selected_udc: Option<String>,
+                          bound_udc: Option<String>,
-        bound_udc: Option<String>,
+                          udc_state: Option<String>,
-        udc_state: Option<String>,
+                          udc_speed: Option<String>,
-        udc_speed: Option<String>,
+                          available_udcs: Vec<String>,
-        available_udcs: Vec<String>,
+                          other_gadgets: Vec<String>| {
-        other_gadgets: Vec<String>,
-    | {
         let error_count = checks
             .iter()
             .filter(|item| item.level == OtgSelfCheckLevel::Error)
@@ -2528,7 +2587,9 @@ pub async fn hid_otg_self_check(State(state): State<Arc<AppState>>) -> Json<OtgS
                 OtgSelfCheckLevel::Info
             },
             "Check configured UDC validity",
-            Some("You can set hid_otg_udc in settings to avoid ambiguity in multi-controller setups"),
+            Some(
+                "You can set hid_otg_udc in settings to avoid ambiguity in multi-controller setups",
+            ),
             Some("/sys/class/udc"),
         );
     }
@@ -2854,7 +2915,6 @@ pub async fn hid_otg_self_check(State(state): State<Arc<AppState>>) -> Json<OtgS
                 );
             }
         }
-
     }
 
     if !other_gadgets.is_empty() {

src/webrtc/mdns.rs

@@ -18,7 +18,9 @@ pub fn mdns_mode_from_env() -> Option<MulticastDnsMode> {
 }
 
 pub fn mdns_mode() -> MulticastDnsMode {
-    mdns_mode_from_env().unwrap_or(MulticastDnsMode::QueryAndGather)
+    // Default to QueryOnly to avoid gathering .local host candidates by default.
+    // This is generally more stable for LAN first-connection while preserving mDNS queries.
+    mdns_mode_from_env().unwrap_or(MulticastDnsMode::QueryOnly)
 }
 
 pub fn mdns_mode_label(mode: MulticastDnsMode) -> &'static str {

src/webrtc/peer.rs

@@ -317,14 +317,26 @@ impl PeerConnection {
             .await
             .map_err(|e| AppError::VideoError(format!("Failed to create answer: {}", e)))?;
 
+        // Wait for ICE gathering complete (or timeout) after setting local description.
+        // This improves first-connection robustness by returning a fuller initial candidate set.
+        let mut gather_complete = self.pc.gathering_complete_promise().await;
+
         // Set local description
         self.pc
             .set_local_description(answer.clone())
             .await
             .map_err(|e| AppError::VideoError(format!("Failed to set local description: {}", e)))?;
 
-        // Wait a bit for ICE candidates to gather
+        const ICE_GATHER_TIMEOUT: tokio::time::Duration = tokio::time::Duration::from_millis(2500);
-        tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
+        if tokio::time::timeout(ICE_GATHER_TIMEOUT, gather_complete.recv())
+            .await
+            .is_err()
+        {
+            debug!(
+                "ICE gathering timeout after {:?} for session {}",
+                ICE_GATHER_TIMEOUT, self.session_id
+            );
+        }
 
         // Get gathered ICE candidates
         let candidates = self.ice_candidates.lock().await.clone();

src/webrtc/universal_session.rs

@@ -833,13 +833,24 @@ impl UniversalSession {
             }
         }
 
+        let mut gather_complete = self.pc.gathering_complete_promise().await;
+
         self.pc
             .set_local_description(answer.clone())
             .await
             .map_err(|e| AppError::VideoError(format!("Failed to set local description: {}", e)))?;
 
-        // Wait for ICE candidates
+        // Wait for ICE gathering complete (or timeout) to return a fuller initial candidate set.
-        tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
+        const ICE_GATHER_TIMEOUT: Duration = Duration::from_millis(2500);
+        if tokio::time::timeout(ICE_GATHER_TIMEOUT, gather_complete.recv())
+            .await
+            .is_err()
+        {
+            debug!(
+                "ICE gathering timeout after {:?} for session {}",
+                ICE_GATHER_TIMEOUT, self.session_id
+            );
+        }
 
         let candidates = self.ice_candidates.lock().await.clone();
         Ok(SdpAnswer::with_candidates(answer.sdp, candidates))

web/src/api/index.ts

@@ -242,10 +242,10 @@ export const webrtcApi = {
   createSession: () =>
     request<{ session_id: string }>('/webrtc/session', { method: 'POST' }),
 
-  offer: (sdp: string, clientId?: string) =>
+  offer: (sdp: string) =>
     request<{ sdp: string; session_id: string; ice_candidates: IceCandidate[] }>('/webrtc/offer', {
       method: 'POST',
-      body: JSON.stringify({ sdp, client_id: clientId }),
+      body: JSON.stringify({ sdp }),
     }),
 
   addIceCandidate: (sessionId: string, candidate: IceCandidate) =>
@@ -325,17 +325,12 @@ export const hidApi = {
       }>
     }>('/hid/otg/self-check'),
 
-  keyboard: async (type: 'down' | 'up', key: number, modifiers?: {
+  keyboard: async (type: 'down' | 'up', key: number, modifier?: number) => {
-    ctrl?: boolean
-    shift?: boolean
-    alt?: boolean
-    meta?: boolean
-  }) => {
     await ensureHidConnection()
     const event: HidKeyboardEvent = {
       type: type === 'down' ? 'keydown' : 'keyup',
       key,
-      modifiers,
+      modifier: (modifier ?? 0) & 0xff,
     }
     await hidWs.sendKeyboard(event)
     return { success: true }

web/src/components/PasteModal.vue

@@ -69,24 +69,24 @@ async function typeChar(char: string, signal: AbortSignal): Promise<boolean> {
     return true
   }
 
-  const { keyCode, shift } = mapping
+  const { hidCode, shift } = mapping
-  const modifiers = shift ? { shift: true } : undefined
+  const modifier = shift ? 0x02 : 0
 
   try {
     // Send keydown
-    await hidApi.keyboard('down', keyCode, modifiers)
+    await hidApi.keyboard('down', hidCode, modifier)
 
     // Small delay between down and up to ensure key is registered
     await sleep(5)
 
     if (signal.aborted) {
       // Even if aborted, still send keyup to release the key
-      await hidApi.keyboard('up', keyCode, modifiers)
+      await hidApi.keyboard('up', hidCode, modifier)
       return false
     }
 
     // Send keyup
-    await hidApi.keyboard('up', keyCode, modifiers)
+    await hidApi.keyboard('up', hidCode, modifier)
 
     // Additional small delay after keyup to ensure it's processed
     await sleep(2)
@@ -96,7 +96,7 @@ async function typeChar(char: string, signal: AbortSignal): Promise<boolean> {
     console.error('[Paste] Failed to type character:', char, error)
     // Try to release the key even on error
     try {
-      await hidApi.keyboard('up', keyCode, modifiers)
+      await hidApi.keyboard('up', hidCode, modifier)
     } catch {
       // Ignore cleanup errors
     }

web/src/components/VirtualKeyboard.vue

@@ -9,6 +9,7 @@ import {
   consumerKeys,
   latchingKeys,
   modifiers,
+  updateModifierMaskForHidKey,
   type KeyName,
   type ConsumerKeyName,
 } from '@/lib/keyboardMappings'
@@ -304,9 +305,10 @@ async function onKeyDown(key: string) {
   // Handle latching keys (Caps Lock, etc.)
   if ((latchingKeys as readonly string[]).includes(cleanKey)) {
     emit('keyDown', cleanKey)
-    await sendKeyPress(keyCode, true)
+    const currentMask = pressedModifiers.value & 0xff
+    await sendKeyPress(keyCode, true, currentMask)
     setTimeout(() => {
-      sendKeyPress(keyCode, false)
+      sendKeyPress(keyCode, false, currentMask)
       emit('keyUp', cleanKey)
     }, 100)
     return
@@ -318,12 +320,14 @@ async function onKeyDown(key: string) {
     const isCurrentlyDown = (pressedModifiers.value & mask) !== 0
 
     if (isCurrentlyDown) {
-      pressedModifiers.value &= ~mask
+      const nextMask = pressedModifiers.value & ~mask
-      await sendKeyPress(keyCode, false)
+      pressedModifiers.value = nextMask
+      await sendKeyPress(keyCode, false, nextMask)
       emit('keyUp', cleanKey)
     } else {
-      pressedModifiers.value |= mask
+      const nextMask = pressedModifiers.value | mask
-      await sendKeyPress(keyCode, true)
+      pressedModifiers.value = nextMask
+      await sendKeyPress(keyCode, true, nextMask)
       emit('keyDown', cleanKey)
     }
     updateKeyboardButtonTheme()
@@ -333,11 +337,12 @@ async function onKeyDown(key: string) {
   // Regular key: press and release
   keysDown.value.push(cleanKey)
   emit('keyDown', cleanKey)
-  await sendKeyPress(keyCode, true)
+  const currentMask = pressedModifiers.value & 0xff
+  await sendKeyPress(keyCode, true, currentMask)
   updateKeyboardButtonTheme()
   setTimeout(async () => {
     keysDown.value = keysDown.value.filter(k => k !== cleanKey)
-    await sendKeyPress(keyCode, false)
+    await sendKeyPress(keyCode, false, currentMask)
     emit('keyUp', cleanKey)
     updateKeyboardButtonTheme()
   }, 50)
@@ -347,16 +352,9 @@ async function onKeyUp() {
   // Not used for now - we handle up in onKeyDown with setTimeout
 }
 
-async function sendKeyPress(keyCode: number, press: boolean) {
+async function sendKeyPress(keyCode: number, press: boolean, modifierMask: number) {
   try {
-    const mods = {
+    await hidApi.keyboard(press ? 'down' : 'up', keyCode, modifierMask & 0xff)
-      ctrl: (pressedModifiers.value & 0x11) !== 0,
-      shift: (pressedModifiers.value & 0x22) !== 0,
-      alt: (pressedModifiers.value & 0x44) !== 0,
-      meta: (pressedModifiers.value & 0x88) !== 0,
-    }
-
-    await hidApi.keyboard(press ? 'down' : 'up', keyCode, mods)
   } catch (err) {
     console.error('[VirtualKeyboard] Key send failed:', err)
   }
@@ -368,16 +366,20 @@ interface MacroStep {
 }
 
 async function executeMacro(steps: MacroStep[]) {
+  let macroModifierMask = pressedModifiers.value & 0xff
+
   for (const step of steps) {
     for (const mod of step.modifiers) {
       if (mod in keys) {
-        await sendKeyPress(keys[mod as KeyName], true)
+        const modHid = keys[mod as KeyName]
+        macroModifierMask = updateModifierMaskForHidKey(macroModifierMask, modHid, true)
+        await sendKeyPress(modHid, true, macroModifierMask)
       }
     }
 
     for (const key of step.keys) {
       if (key in keys) {
-        await sendKeyPress(keys[key as KeyName], true)
+        await sendKeyPress(keys[key as KeyName], true, macroModifierMask)
       }
     }
 
@@ -385,13 +387,15 @@ async function executeMacro(steps: MacroStep[]) {
 
     for (const key of step.keys) {
       if (key in keys) {
-        await sendKeyPress(keys[key as KeyName], false)
+        await sendKeyPress(keys[key as KeyName], false, macroModifierMask)
       }
     }
 
     for (const mod of step.modifiers) {
       if (mod in keys) {
-        await sendKeyPress(keys[mod as KeyName], false)
+        const modHid = keys[mod as KeyName]
+        macroModifierMask = updateModifierMaskForHidKey(macroModifierMask, modHid, false)
+        await sendKeyPress(modHid, false, macroModifierMask)
       }
     }
   }

web/src/composables/useHidInput.ts

@@ -5,6 +5,7 @@ import { ref, type Ref } from 'vue'
 import { useI18n } from 'vue-i18n'
 import { toast } from 'vue-sonner'
 import { hidApi } from '@/api'
+import { keyboardEventToHidCode, updateModifierMaskForHidKey } from '@/lib/keyboardMappings'
 
 export interface HidInputState {
   mouseMode: Ref<'absolute' | 'relative'>
@@ -32,6 +33,7 @@ export function useHidInput(options: UseHidInputOptions) {
     numLock: false,
     scrollLock: false,
   })
+  const activeModifierMask = ref(0)
   const mousePosition = ref({ x: 0, y: 0 })
   const lastMousePosition = ref({ x: 0, y: 0 })
   const isPointerLocked = ref(false)
@@ -83,14 +85,14 @@ export function useHidInput(options: UseHidInputOptions) {
     keyboardLed.value.numLock = e.getModifierState('NumLock')
     keyboardLed.value.scrollLock = e.getModifierState('ScrollLock')
 
-    const modifiers = {
+    const hidKey = keyboardEventToHidCode(e.code, e.key)
-      ctrl: e.ctrlKey,
+    if (hidKey === undefined) {
-      shift: e.shiftKey,
+      return
-      alt: e.altKey,
-      meta: e.metaKey,
     }
 
-    hidApi.keyboard('down', e.keyCode, modifiers).catch(err => handleHidError(err, 'keyboard down'))
+    const modifierMask = updateModifierMaskForHidKey(activeModifierMask.value, hidKey, true)
+    activeModifierMask.value = modifierMask
+    hidApi.keyboard('down', hidKey, modifierMask).catch(err => handleHidError(err, 'keyboard down'))
   }
 
   function handleKeyUp(e: KeyboardEvent) {
@@ -107,7 +109,14 @@ export function useHidInput(options: UseHidInputOptions) {
     const keyName = e.key === ' ' ? 'Space' : e.key
     pressedKeys.value = pressedKeys.value.filter(k => k !== keyName)
 
-    hidApi.keyboard('up', e.keyCode).catch(err => handleHidError(err, 'keyboard up'))
+    const hidKey = keyboardEventToHidCode(e.code, e.key)
+    if (hidKey === undefined) {
+      return
+    }
+
+    const modifierMask = updateModifierMaskForHidKey(activeModifierMask.value, hidKey, false)
+    activeModifierMask.value = modifierMask
+    hidApi.keyboard('up', hidKey, modifierMask).catch(err => handleHidError(err, 'keyboard up'))
   }
 
   // Mouse handlers
@@ -233,6 +242,7 @@ export function useHidInput(options: UseHidInputOptions) {
 
   function handleBlur() {
     pressedKeys.value = []
+    activeModifierMask.value = 0
     if (pressedMouseButton.value !== null) {
       const button = pressedMouseButton.value
       pressedMouseButton.value = null

web/src/composables/useWebRTC.ts

@@ -3,7 +3,6 @@
 
 import { ref, onUnmounted, computed, type Ref } from 'vue'
 import { webrtcApi, type IceCandidate } from '@/api'
-import { generateUUID } from '@/lib/utils'
 import {
   type HidKeyboardEvent,
   type HidMouseEvent,
@@ -15,6 +14,19 @@ import { useWebSocket } from '@/composables/useWebSocket'
 export type { HidKeyboardEvent, HidMouseEvent }
 
 export type WebRTCState = 'disconnected' | 'connecting' | 'connected' | 'failed'
+export type WebRTCConnectStage =
+  | 'idle'
+  | 'fetching_ice_servers'
+  | 'creating_peer_connection'
+  | 'creating_data_channel'
+  | 'creating_offer'
+  | 'waiting_server_answer'
+  | 'setting_remote_description'
+  | 'applying_ice_candidates'
+  | 'waiting_connection'
+  | 'connected'
+  | 'disconnected'
+  | 'failed'
 
 // ICE candidate type: host=P2P local, srflx=P2P STUN, relay=TURN relay
 export type IceCandidateType = 'host' | 'srflx' | 'prflx' | 'relay' | 'unknown'
@@ -99,6 +111,7 @@ let dataChannel: RTCDataChannel | null = null
 let sessionId: string | null = null
 let statsInterval: number | null = null
 let isConnecting = false // Lock to prevent concurrent connect calls
+let connectInFlight: Promise<boolean> | null = null
 let pendingIceCandidates: RTCIceCandidate[] = [] // Queue for ICE candidates before sessionId is set
 let pendingRemoteCandidates: WebRTCIceCandidateEvent[] = [] // Queue for server ICE candidates
 let pendingRemoteIceComplete = new Set<string>() // Session IDs waiting for end-of-candidates
@@ -131,6 +144,7 @@ const stats = ref<WebRTCStats>({
 })
 const error = ref<string | null>(null)
 const dataChannelReady = ref(false)
+const connectStage = ref<WebRTCConnectStage>('idle')
 
 // Create RTCPeerConnection with configuration
 function createPeerConnection(iceServers: RTCIceServer[]): RTCPeerConnection {
@@ -149,16 +163,19 @@ function createPeerConnection(iceServers: RTCIceServer[]): RTCPeerConnection {
         break
       case 'connected':
         state.value = 'connected'
+        connectStage.value = 'connected'
         error.value = null
         startStatsCollection()
         break
       case 'disconnected':
       case 'closed':
         state.value = 'disconnected'
+        connectStage.value = 'disconnected'
         stopStatsCollection()
         break
       case 'failed':
         state.value = 'failed'
+        connectStage.value = 'failed'
         error.value = 'Connection failed'
         stopStatsCollection()
         break
@@ -450,100 +467,123 @@ async function flushPendingIceCandidates() {
 
 // Connect to WebRTC server
 async function connect(): Promise<boolean> {
-  registerWebSocketHandlers()
+  if (connectInFlight) {
-
+    return connectInFlight
-  // Prevent concurrent connection attempts
-  if (isConnecting) {
-    return false
   }
 
-  if (peerConnection && state.value === 'connected') {
+  connectInFlight = (async () => {
-    return true
+    registerWebSocketHandlers()
-  }
 
-  isConnecting = true
+    // Prevent concurrent connection attempts
+    if (isConnecting) {
+      return state.value === 'connected'
+    }
 
-  // Clean up any existing connection first
+    if (peerConnection && state.value === 'connected') {
-  if (peerConnection || sessionId) {
+      return true
-    await disconnect()
+    }
-  }
 
-  // Clear pending ICE candidates from previous attempt
+    isConnecting = true
-  pendingIceCandidates = []
+
+    // Clean up any existing connection first
+    if (peerConnection || sessionId) {
+      await disconnect()
+    }
+
+    // Clear pending ICE candidates from previous attempt
+    pendingIceCandidates = []
+
+    try {
+      state.value = 'connecting'
+      error.value = null
+      connectStage.value = 'fetching_ice_servers'
+
+      // Fetch ICE servers from backend API
+      const iceServers = await fetchIceServers()
+      connectStage.value = 'creating_peer_connection'
+
+      // Create peer connection with fetched ICE servers
+      peerConnection = createPeerConnection(iceServers)
+      connectStage.value = 'creating_data_channel'
+
+      // Create data channel before offer (for HID)
+      createDataChannel(peerConnection)
+
+      // Add transceiver for receiving video
+      peerConnection.addTransceiver('video', { direction: 'recvonly' })
+      peerConnection.addTransceiver('audio', { direction: 'recvonly' })
+      connectStage.value = 'creating_offer'
+
+      // Create offer
+      const offer = await peerConnection.createOffer()
+      await peerConnection.setLocalDescription(offer)
+      connectStage.value = 'waiting_server_answer'
+
+      // Send offer to server and get answer
+      // Do not pass client_id here: each connect creates a fresh session.
+      const response = await webrtcApi.offer(offer.sdp!)
+      sessionId = response.session_id
+
+      // Send any ICE candidates that were queued while waiting for sessionId
+      await flushPendingIceCandidates()
+
+      // Set remote description (answer)
+      const answer: RTCSessionDescriptionInit = {
+        type: 'answer',
+        sdp: response.sdp,
+      }
+      connectStage.value = 'setting_remote_description'
+      await peerConnection.setRemoteDescription(answer)
+
+      // Flush any pending server ICE candidates once remote description is set
+      connectStage.value = 'applying_ice_candidates'
+      await flushPendingRemoteIce()
+
+      // Add any ICE candidates from the response
+      if (response.ice_candidates && response.ice_candidates.length > 0) {
+        for (const candidateObj of response.ice_candidates) {
+          await addRemoteIceCandidate(candidateObj)
+        }
+      }
+
+      // 等待连接真正建立（最多等待 15 秒）
+      // 直接检查 peerConnection.connectionState 而不是 reactive state
+      // 因为 TypeScript 不知道 state 会被 onconnectionstatechange 回调异步修改
+      const connectionTimeout = 15000
+      const pollInterval = 100
+      let waited = 0
+      connectStage.value = 'waiting_connection'
+
+      while (waited < connectionTimeout && peerConnection) {
+        const pcState = peerConnection.connectionState
+        if (pcState === 'connected') {
+          connectStage.value = 'connected'
+          isConnecting = false
+          return true
+        }
+        if (pcState === 'failed' || pcState === 'closed') {
+          throw new Error('Connection failed during ICE negotiation')
+        }
+        await new Promise(resolve => setTimeout(resolve, pollInterval))
+        waited += pollInterval
+      }
+
+      // 超时
+      throw new Error('Connection timeout waiting for ICE negotiation')
+    } catch (err) {
+      state.value = 'failed'
+      connectStage.value = 'failed'
+      error.value = err instanceof Error ? err.message : 'Connection failed'
+      isConnecting = false
+      await disconnect()
+      return false
+    }
+  })()
 
   try {
-    state.value = 'connecting'
+    return await connectInFlight
-    error.value = null
+  } finally {
-
+    connectInFlight = null
-    // Fetch ICE servers from backend API
-    const iceServers = await fetchIceServers()
-
-    // Create peer connection with fetched ICE servers
-    peerConnection = createPeerConnection(iceServers)
-
-    // Create data channel before offer (for HID)
-    createDataChannel(peerConnection)
-
-    // Add transceiver for receiving video
-    peerConnection.addTransceiver('video', { direction: 'recvonly' })
-    peerConnection.addTransceiver('audio', { direction: 'recvonly' })
-
-    // Create offer
-    const offer = await peerConnection.createOffer()
-    await peerConnection.setLocalDescription(offer)
-
-    // Send offer to server and get answer
-    const response = await webrtcApi.offer(offer.sdp!, generateUUID())
-    sessionId = response.session_id
-
-    // Send any ICE candidates that were queued while waiting for sessionId
-    await flushPendingIceCandidates()
-
-    // Set remote description (answer)
-    const answer: RTCSessionDescriptionInit = {
-      type: 'answer',
-      sdp: response.sdp,
-    }
-    await peerConnection.setRemoteDescription(answer)
-
-    // Flush any pending server ICE candidates once remote description is set
-    await flushPendingRemoteIce()
-
-    // Add any ICE candidates from the response
-    if (response.ice_candidates && response.ice_candidates.length > 0) {
-      for (const candidateObj of response.ice_candidates) {
-        await addRemoteIceCandidate(candidateObj)
-      }
-    }
-
-    // 等待连接真正建立（最多等待 15 秒）
-    // 直接检查 peerConnection.connectionState 而不是 reactive state
-    // 因为 TypeScript 不知道 state 会被 onconnectionstatechange 回调异步修改
-    const connectionTimeout = 15000
-    const pollInterval = 100
-    let waited = 0
-
-    while (waited < connectionTimeout && peerConnection) {
-      const pcState = peerConnection.connectionState
-      if (pcState === 'connected') {
-        isConnecting = false
-        return true
-      }
-      if (pcState === 'failed' || pcState === 'closed') {
-        throw new Error('Connection failed during ICE negotiation')
-      }
-      await new Promise(resolve => setTimeout(resolve, pollInterval))
-      waited += pollInterval
-    }
-
-    // 超时
-    throw new Error('Connection timeout waiting for ICE negotiation')
-  } catch (err) {
-    state.value = 'failed'
-    error.value = err instanceof Error ? err.message : 'Connection failed'
-    isConnecting = false
-    disconnect()
-    return false
   }
 }
 
@@ -583,6 +623,7 @@ async function disconnect() {
   audioTrack.value = null
   cachedMediaStream = null // Clear cached stream on disconnect
   state.value = 'disconnected'
+  connectStage.value = 'disconnected'
   error.value = null
 
   // Reset stats
@@ -694,6 +735,7 @@ export function useWebRTC() {
     stats,
     error,
     dataChannelReady,
+    connectStage,
     sessionId: computed(() => sessionId),
 
     // Methods

web/src/i18n/en-US.ts

@@ -312,6 +312,14 @@ export default {
     webrtcConnectedDesc: 'Using low-latency H.264 video stream',
     webrtcFailed: 'WebRTC Connection Failed',
     fallingBackToMjpeg: 'Falling back to MJPEG mode',
+    webrtcPhaseIceServers: 'Loading ICE servers...',
+    webrtcPhaseCreatePeer: 'Creating peer connection...',
+    webrtcPhaseCreateChannel: 'Creating data channel...',
+    webrtcPhaseCreateOffer: 'Creating local offer...',
+    webrtcPhaseWaitAnswer: 'Waiting for remote answer...',
+    webrtcPhaseSetRemote: 'Applying remote description...',
+    webrtcPhaseApplyIce: 'Applying ICE candidates...',
+    webrtcPhaseNegotiating: 'Negotiating secure connection...',
     // Pointer Lock
     pointerLocked: 'Pointer Locked',
     pointerLockedDesc: 'Press Escape to release the pointer',
@@ -455,7 +463,7 @@ export default {
     deviceInfo: 'Device Info',
     deviceInfoDesc: 'Host system information',
     hostname: 'Hostname',
-    cpuModel: 'CPU Model',
+    cpuModel: 'Processor / Platform',
     cpuUsage: 'CPU Usage',
     memoryUsage: 'Memory Usage',
     networkAddresses: 'Network Addresses',

web/src/i18n/zh-CN.ts

@@ -312,6 +312,14 @@ export default {
     webrtcConnectedDesc: '正在使用 H.264 低延迟视频流',
     webrtcFailed: 'WebRTC 连接失败',
     fallingBackToMjpeg: '自动切换到 MJPEG 模式',
+    webrtcPhaseIceServers: '正在加载 ICE 服务器...',
+    webrtcPhaseCreatePeer: '正在创建点对点连接...',
+    webrtcPhaseCreateChannel: '正在创建数据通道...',
+    webrtcPhaseCreateOffer: '正在创建本地会话描述...',
+    webrtcPhaseWaitAnswer: '正在等待远端应答...',
+    webrtcPhaseSetRemote: '正在应用远端会话描述...',
+    webrtcPhaseApplyIce: '正在应用 ICE 候选...',
+    webrtcPhaseNegotiating: '正在协商安全连接...',
     // Pointer Lock
     pointerLocked: '鼠标已锁定',
     pointerLockedDesc: '按 Escape 键释放鼠标',
@@ -455,7 +463,7 @@ export default {
     deviceInfo: '设备信息',
     deviceInfoDesc: '主机系统信息',
     hostname: '主机名',
-    cpuModel: 'CPU 型号',
+    cpuModel: '处理器 / 平台',
     cpuUsage: 'CPU 利用率',
     memoryUsage: '内存使用',
     networkAddresses: '网络地址',

web/src/lib/charToHid.ts

@@ -1,136 +1,135 @@
-// Character to JavaScript keyCode mapping for text paste functionality
+// Character to HID usage mapping for text paste functionality.
-// Maps printable ASCII characters to JavaScript keyCodes that the backend expects
+// The table follows US QWERTY layout semantics.
-// The backend (keymap.rs) will convert these JS keyCodes to USB HID keycodes
+
+import { keys } from '@/lib/keyboardMappings'
 
 export interface CharKeyMapping {
-  keyCode: number // JavaScript keyCode (same as KeyboardEvent.keyCode)
+  hidCode: number // USB HID usage code
   shift: boolean // Whether Shift modifier is needed
 }
 
-// US QWERTY keyboard layout mapping
-// Maps characters to their JavaScript keyCode and whether Shift is required
 const charToKeyMap: Record<string, CharKeyMapping> = {
-  // Lowercase letters (no shift) - JS keyCodes 65-90
+  // Lowercase letters
-  a: { keyCode: 65, shift: false },
+  a: { hidCode: keys.KeyA, shift: false },
-  b: { keyCode: 66, shift: false },
+  b: { hidCode: keys.KeyB, shift: false },
-  c: { keyCode: 67, shift: false },
+  c: { hidCode: keys.KeyC, shift: false },
-  d: { keyCode: 68, shift: false },
+  d: { hidCode: keys.KeyD, shift: false },
-  e: { keyCode: 69, shift: false },
+  e: { hidCode: keys.KeyE, shift: false },
-  f: { keyCode: 70, shift: false },
+  f: { hidCode: keys.KeyF, shift: false },
-  g: { keyCode: 71, shift: false },
+  g: { hidCode: keys.KeyG, shift: false },
-  h: { keyCode: 72, shift: false },
+  h: { hidCode: keys.KeyH, shift: false },
-  i: { keyCode: 73, shift: false },
+  i: { hidCode: keys.KeyI, shift: false },
-  j: { keyCode: 74, shift: false },
+  j: { hidCode: keys.KeyJ, shift: false },
-  k: { keyCode: 75, shift: false },
+  k: { hidCode: keys.KeyK, shift: false },
-  l: { keyCode: 76, shift: false },
+  l: { hidCode: keys.KeyL, shift: false },
-  m: { keyCode: 77, shift: false },
+  m: { hidCode: keys.KeyM, shift: false },
-  n: { keyCode: 78, shift: false },
+  n: { hidCode: keys.KeyN, shift: false },
-  o: { keyCode: 79, shift: false },
+  o: { hidCode: keys.KeyO, shift: false },
-  p: { keyCode: 80, shift: false },
+  p: { hidCode: keys.KeyP, shift: false },
-  q: { keyCode: 81, shift: false },
+  q: { hidCode: keys.KeyQ, shift: false },
-  r: { keyCode: 82, shift: false },
+  r: { hidCode: keys.KeyR, shift: false },
-  s: { keyCode: 83, shift: false },
+  s: { hidCode: keys.KeyS, shift: false },
-  t: { keyCode: 84, shift: false },
+  t: { hidCode: keys.KeyT, shift: false },
-  u: { keyCode: 85, shift: false },
+  u: { hidCode: keys.KeyU, shift: false },
-  v: { keyCode: 86, shift: false },
+  v: { hidCode: keys.KeyV, shift: false },
-  w: { keyCode: 87, shift: false },
+  w: { hidCode: keys.KeyW, shift: false },
-  x: { keyCode: 88, shift: false },
+  x: { hidCode: keys.KeyX, shift: false },
-  y: { keyCode: 89, shift: false },
+  y: { hidCode: keys.KeyY, shift: false },
-  z: { keyCode: 90, shift: false },
+  z: { hidCode: keys.KeyZ, shift: false },
 
-  // Uppercase letters (with shift) - same keyCodes, just with Shift
+  // Uppercase letters
-  A: { keyCode: 65, shift: true },
+  A: { hidCode: keys.KeyA, shift: true },
-  B: { keyCode: 66, shift: true },
+  B: { hidCode: keys.KeyB, shift: true },
-  C: { keyCode: 67, shift: true },
+  C: { hidCode: keys.KeyC, shift: true },
-  D: { keyCode: 68, shift: true },
+  D: { hidCode: keys.KeyD, shift: true },
-  E: { keyCode: 69, shift: true },
+  E: { hidCode: keys.KeyE, shift: true },
-  F: { keyCode: 70, shift: true },
+  F: { hidCode: keys.KeyF, shift: true },
-  G: { keyCode: 71, shift: true },
+  G: { hidCode: keys.KeyG, shift: true },
-  H: { keyCode: 72, shift: true },
+  H: { hidCode: keys.KeyH, shift: true },
-  I: { keyCode: 73, shift: true },
+  I: { hidCode: keys.KeyI, shift: true },
-  J: { keyCode: 74, shift: true },
+  J: { hidCode: keys.KeyJ, shift: true },
-  K: { keyCode: 75, shift: true },
+  K: { hidCode: keys.KeyK, shift: true },
-  L: { keyCode: 76, shift: true },
+  L: { hidCode: keys.KeyL, shift: true },
-  M: { keyCode: 77, shift: true },
+  M: { hidCode: keys.KeyM, shift: true },
-  N: { keyCode: 78, shift: true },
+  N: { hidCode: keys.KeyN, shift: true },
-  O: { keyCode: 79, shift: true },
+  O: { hidCode: keys.KeyO, shift: true },
-  P: { keyCode: 80, shift: true },
+  P: { hidCode: keys.KeyP, shift: true },
-  Q: { keyCode: 81, shift: true },
+  Q: { hidCode: keys.KeyQ, shift: true },
-  R: { keyCode: 82, shift: true },
+  R: { hidCode: keys.KeyR, shift: true },
-  S: { keyCode: 83, shift: true },
+  S: { hidCode: keys.KeyS, shift: true },
-  T: { keyCode: 84, shift: true },
+  T: { hidCode: keys.KeyT, shift: true },
-  U: { keyCode: 85, shift: true },
+  U: { hidCode: keys.KeyU, shift: true },
-  V: { keyCode: 86, shift: true },
+  V: { hidCode: keys.KeyV, shift: true },
-  W: { keyCode: 87, shift: true },
+  W: { hidCode: keys.KeyW, shift: true },
-  X: { keyCode: 88, shift: true },
+  X: { hidCode: keys.KeyX, shift: true },
-  Y: { keyCode: 89, shift: true },
+  Y: { hidCode: keys.KeyY, shift: true },
-  Z: { keyCode: 90, shift: true },
+  Z: { hidCode: keys.KeyZ, shift: true },
 
-  // Numbers (no shift) - JS keyCodes 48-57
+  // Number row
-  '0': { keyCode: 48, shift: false },
+  '0': { hidCode: keys.Digit0, shift: false },
-  '1': { keyCode: 49, shift: false },
+  '1': { hidCode: keys.Digit1, shift: false },
-  '2': { keyCode: 50, shift: false },
+  '2': { hidCode: keys.Digit2, shift: false },
-  '3': { keyCode: 51, shift: false },
+  '3': { hidCode: keys.Digit3, shift: false },
-  '4': { keyCode: 52, shift: false },
+  '4': { hidCode: keys.Digit4, shift: false },
-  '5': { keyCode: 53, shift: false },
+  '5': { hidCode: keys.Digit5, shift: false },
-  '6': { keyCode: 54, shift: false },
+  '6': { hidCode: keys.Digit6, shift: false },
-  '7': { keyCode: 55, shift: false },
+  '7': { hidCode: keys.Digit7, shift: false },
-  '8': { keyCode: 56, shift: false },
+  '8': { hidCode: keys.Digit8, shift: false },
-  '9': { keyCode: 57, shift: false },
+  '9': { hidCode: keys.Digit9, shift: false },
 
-  // Shifted number row symbols (US layout)
+  // Shifted number row symbols
-  ')': { keyCode: 48, shift: true }, // Shift + 0
+  ')': { hidCode: keys.Digit0, shift: true },
-  '!': { keyCode: 49, shift: true }, // Shift + 1
+  '!': { hidCode: keys.Digit1, shift: true },
-  '@': { keyCode: 50, shift: true }, // Shift + 2
+  '@': { hidCode: keys.Digit2, shift: true },
-  '#': { keyCode: 51, shift: true }, // Shift + 3
+  '#': { hidCode: keys.Digit3, shift: true },
-  $: { keyCode: 52, shift: true }, // Shift + 4
+  '$': { hidCode: keys.Digit4, shift: true },
-  '%': { keyCode: 53, shift: true }, // Shift + 5
+  '%': { hidCode: keys.Digit5, shift: true },
-  '^': { keyCode: 54, shift: true }, // Shift + 6
+  '^': { hidCode: keys.Digit6, shift: true },
-  '&': { keyCode: 55, shift: true }, // Shift + 7
+  '&': { hidCode: keys.Digit7, shift: true },
-  '*': { keyCode: 56, shift: true }, // Shift + 8
+  '*': { hidCode: keys.Digit8, shift: true },
-  '(': { keyCode: 57, shift: true }, // Shift + 9
+  '(': { hidCode: keys.Digit9, shift: true },
 
-  // Punctuation and symbols (no shift) - US layout JS keyCodes
+  // Punctuation and symbols
-  '-': { keyCode: 189, shift: false }, // Minus
+  '-': { hidCode: keys.Minus, shift: false },
-  '=': { keyCode: 187, shift: false }, // Equal
+  '=': { hidCode: keys.Equal, shift: false },
-  '[': { keyCode: 219, shift: false }, // Left bracket
+  '[': { hidCode: keys.BracketLeft, shift: false },
-  ']': { keyCode: 221, shift: false }, // Right bracket
+  ']': { hidCode: keys.BracketRight, shift: false },
-  '\\': { keyCode: 220, shift: false }, // Backslash
+  '\\': { hidCode: keys.Backslash, shift: false },
-  ';': { keyCode: 186, shift: false }, // Semicolon
+  ';': { hidCode: keys.Semicolon, shift: false },
-  "'": { keyCode: 222, shift: false }, // Apostrophe/Quote
+  "'": { hidCode: keys.Quote, shift: false },
-  '`': { keyCode: 192, shift: false }, // Grave/Backtick
+  '`': { hidCode: keys.Backquote, shift: false },
-  ',': { keyCode: 188, shift: false }, // Comma
+  ',': { hidCode: keys.Comma, shift: false },
-  '.': { keyCode: 190, shift: false }, // Period
+  '.': { hidCode: keys.Period, shift: false },
-  '/': { keyCode: 191, shift: false }, // Slash
+  '/': { hidCode: keys.Slash, shift: false },
 
-  // Shifted punctuation and symbols (US layout)
+  // Shifted punctuation and symbols
-  _: { keyCode: 189, shift: true }, // Shift + Minus = Underscore
+  _: { hidCode: keys.Minus, shift: true },
-  '+': { keyCode: 187, shift: true }, // Shift + Equal = Plus
+  '+': { hidCode: keys.Equal, shift: true },
-  '{': { keyCode: 219, shift: true }, // Shift + [ = {
+  '{': { hidCode: keys.BracketLeft, shift: true },
-  '}': { keyCode: 221, shift: true }, // Shift + ] = }
+  '}': { hidCode: keys.BracketRight, shift: true },
-  '|': { keyCode: 220, shift: true }, // Shift + \ = |
+  '|': { hidCode: keys.Backslash, shift: true },
-  ':': { keyCode: 186, shift: true }, // Shift + ; = :
+  ':': { hidCode: keys.Semicolon, shift: true },
-  '"': { keyCode: 222, shift: true }, // Shift + ' = "
+  '"': { hidCode: keys.Quote, shift: true },
-  '~': { keyCode: 192, shift: true }, // Shift + ` = ~
+  '~': { hidCode: keys.Backquote, shift: true },
-  '<': { keyCode: 188, shift: true }, // Shift + , = <
+  '<': { hidCode: keys.Comma, shift: true },
-  '>': { keyCode: 190, shift: true }, // Shift + . = >
+  '>': { hidCode: keys.Period, shift: true },
-  '?': { keyCode: 191, shift: true }, // Shift + / = ?
+  '?': { hidCode: keys.Slash, shift: true },
 
-  // Whitespace and control characters
+  // Whitespace and control
-  ' ': { keyCode: 32, shift: false }, // Space
+  ' ': { hidCode: keys.Space, shift: false },
-  '\t': { keyCode: 9, shift: false }, // Tab
+  '\t': { hidCode: keys.Tab, shift: false },
-  '\n': { keyCode: 13, shift: false }, // Enter (LF)
+  '\n': { hidCode: keys.Enter, shift: false },
-  '\r': { keyCode: 13, shift: false }, // Enter (CR)
+  '\r': { hidCode: keys.Enter, shift: false },
 }
 
 /**
- * Get the JavaScript keyCode and modifier state for a character
+ * Get HID usage code and modifier state for a character
  * @param char - Single character to convert
  * @returns CharKeyMapping or null if character is not mappable
  */
 export function charToKey(char: string): CharKeyMapping | null {
   if (char.length !== 1) return null
-  return charToKeyMap[char] || null
+  return charToKeyMap[char] ?? null
 }
 
 /**
@@ -138,7 +137,7 @@ export function charToKey(char: string): CharKeyMapping | null {
  * @param char - Single character to check
  */
 export function isTypableChar(char: string): boolean {
-  return char.length === 1 && char in charToKeyMap
+  return charToKey(char) !== null
 }
 
 /**

web/src/lib/keyboardMappings.ts

@@ -191,6 +191,13 @@ export const hidKeyToModifierMask: Record<number, number> = {
   0xe7: 0x80, // MetaRight
 }
 
+// Update modifier mask when a HID modifier key is pressed/released.
+export function updateModifierMaskForHidKey(mask: number, hidKey: number, press: boolean): number {
+  const bit = hidKeyToModifierMask[hidKey] ?? 0
+  if (bit === 0) return mask
+  return press ? (mask | bit) : (mask & ~bit)
+}
+
 // Keys that latch (toggle state) instead of being held
 export const latchingKeys = ['CapsLock', 'ScrollLock', 'NumLock'] as const
 
@@ -220,6 +227,23 @@ export function getModifierMask(keyName: string): number {
   return 0
 }
 
+// Normalize browser-specific KeyboardEvent.code variants.
+export function normalizeKeyboardCode(code: string, key: string): string {
+  if (code === 'IntlBackslash' && (key === '`' || key === '~')) return 'Backquote'
+  if (code === 'Backquote' && (key === '§' || key === '±')) return 'IntlBackslash'
+  if (code === 'IntlYen') return 'IntlBackslash'
+  if (code === 'OSLeft') return 'MetaLeft'
+  if (code === 'OSRight') return 'MetaRight'
+  if (code === '' && key === 'Shift') return 'ShiftRight'
+  return code
+}
+
+// Convert KeyboardEvent.code/key to USB HID usage code.
+export function keyboardEventToHidCode(code: string, key: string): number | undefined {
+  const normalizedCode = normalizeKeyboardCode(code, key)
+  return keys[normalizedCode as KeyName]
+}
+
 // Decode modifier byte into individual states
 export function decodeModifiers(modifier: number) {
   return {

web/src/types/hid.ts

@@ -5,12 +5,8 @@
 export interface HidKeyboardEvent {
   type: 'keydown' | 'keyup'
   key: number
-  modifiers?: {
+  /** Raw HID modifier byte (bit0..bit7 = LCtrl..RMeta) */
-    ctrl?: boolean
+  modifier?: number
-    shift?: boolean
-    alt?: boolean
-    meta?: boolean
-  }
 }
 
 /** Mouse event for HID input */
@@ -57,13 +53,7 @@ export function encodeKeyboardEvent(event: HidKeyboardEvent): ArrayBuffer {
   view.setUint8(1, event.type === 'keydown' ? KB_EVENT_DOWN : KB_EVENT_UP)
   view.setUint8(2, event.key & 0xff)
 
-  // Build modifiers bitmask
+  view.setUint8(3, (event.modifier ?? 0) & 0xff)
-  let modifiers = 0
-  if (event.modifiers?.ctrl) modifiers |= 0x01 // Left Ctrl
-  if (event.modifiers?.shift) modifiers |= 0x02 // Left Shift
-  if (event.modifiers?.alt) modifiers |= 0x04 // Left Alt
-  if (event.modifiers?.meta) modifiers |= 0x08 // Left Meta
-  view.setUint8(3, modifiers)
 
   return buffer
 }

web/src/views/ConsoleView.vue

@@ -13,6 +13,7 @@ import { useVideoSession } from '@/composables/useVideoSession'
 import { getUnifiedAudio } from '@/composables/useUnifiedAudio'
 import { streamApi, hidApi, atxApi, extensionsApi, atxConfigApi, authApi } from '@/api'
 import type { HidKeyboardEvent, HidMouseEvent } from '@/types/hid'
+import { keyboardEventToHidCode, updateModifierMaskForHidKey } from '@/lib/keyboardMappings'
 import { toast } from 'vue-sonner'
 import { generateUUID } from '@/lib/utils'
 import type { VideoMode } from '@/components/VideoConfigPopover.vue'
@@ -121,6 +122,7 @@ const pressedKeys = ref<string[]>([])
 const keyboardLed = ref({
   capsLock: false,
 })
+const activeModifierMask = ref(0)
 const mousePosition = ref({ x: 0, y: 0 })
 const lastMousePosition = ref({ x: 0, y: 0 }) // Track last position for relative mode
 const isPointerLocked = ref(false) // Track pointer lock state
@@ -407,6 +409,37 @@ const msdDetails = computed<StatusDetail[]>(() => {
   return details
 })
 
+const webrtcLoadingMessage = computed(() => {
+  if (videoMode.value === 'mjpeg') {
+    return videoRestarting.value ? t('console.videoRestarting') : t('console.connecting')
+  }
+
+  switch (webrtc.connectStage.value) {
+    case 'fetching_ice_servers':
+      return t('console.webrtcPhaseIceServers')
+    case 'creating_peer_connection':
+      return t('console.webrtcPhaseCreatePeer')
+    case 'creating_data_channel':
+      return t('console.webrtcPhaseCreateChannel')
+    case 'creating_offer':
+      return t('console.webrtcPhaseCreateOffer')
+    case 'waiting_server_answer':
+      return t('console.webrtcPhaseWaitAnswer')
+    case 'setting_remote_description':
+      return t('console.webrtcPhaseSetRemote')
+    case 'applying_ice_candidates':
+      return t('console.webrtcPhaseApplyIce')
+    case 'waiting_connection':
+      return t('console.webrtcPhaseNegotiating')
+    case 'connected':
+      return t('console.webrtcConnected')
+    case 'failed':
+      return t('console.webrtcFailed')
+    default:
+      return videoRestarting.value ? t('console.videoRestarting') : t('console.connecting')
+  }
+})
+
 const showMsdStatusCard = computed(() => {
   return !!(systemStore.msd?.available && systemStore.hid?.backend !== 'ch9329')
 })
@@ -423,6 +456,8 @@ let consecutiveErrors = 0
 const BASE_RETRY_DELAY = 2000
 const GRACE_PERIOD = 2000 // Ignore errors for 2s after config change (reduced from 3s)
 const MAX_CONSECUTIVE_ERRORS = 2 // If 2+ errors in grace period, it's a real problem
+let pendingWebRTCReadyGate = false
+let webrtcConnectTask: Promise<boolean> | null = null
 
 // Last-frame overlay (prevents black flash during mode switches)
 const frameOverlayUrl = ref<string | null>(null)
@@ -492,6 +527,52 @@ function waitForVideoFirstFrame(el: HTMLVideoElement, timeoutMs = 2000): Promise
   })
 }
 
+function shouldSuppressAutoReconnect(): boolean {
+  return videoMode.value === 'mjpeg'
+    || videoSession.localSwitching.value
+    || videoSession.backendSwitching.value
+    || videoRestarting.value
+}
+
+function markWebRTCFailure(reason: string, description?: string) {
+  pendingWebRTCReadyGate = false
+  videoError.value = true
+  videoErrorMessage.value = reason
+  videoLoading.value = false
+  systemStore.setStreamOnline(false)
+
+  toast.error(reason, {
+    description: description ?? '',
+    duration: 5000,
+  })
+}
+
+async function waitForWebRTCReadyGate(reason: string, timeoutMs = 3000): Promise<void> {
+  if (!pendingWebRTCReadyGate) return
+  const ready = await videoSession.waitForWebRTCReadyAny(timeoutMs)
+  if (!ready) {
+    console.warn(`[WebRTC] Ready gate timeout (${reason}), attempting connection anyway`)
+  }
+  pendingWebRTCReadyGate = false
+}
+
+async function connectWebRTCSerial(reason: string): Promise<boolean> {
+  if (webrtcConnectTask) {
+    return webrtcConnectTask
+  }
+
+  webrtcConnectTask = (async () => {
+    await waitForWebRTCReadyGate(reason)
+    return webrtc.connect()
+  })()
+
+  try {
+    return await webrtcConnectTask
+  } finally {
+    webrtcConnectTask = null
+  }
+}
+
 function handleVideoLoad() {
   // MJPEG video frame loaded successfully - update stream online status
   // This fixes the timing issue where device_info event may arrive before stream is fully active
@@ -612,9 +693,9 @@ async function handleAudioStateChanged(data: { streaming: boolean; device: strin
     if (!webrtc.audioTrack.value) {
       // No audio track - need to reconnect WebRTC to get one
       // This happens when audio was enabled after WebRTC session was created
-      webrtc.disconnect()
+      await webrtc.disconnect()
       await new Promise(resolve => setTimeout(resolve, 300))
-      await webrtc.connect()
+      await connectWebRTCSerial('audio track refresh')
       // After reconnect, the new session will have audio track
       // and the watch on audioTrack will add it to MediaStream
     } else {
@@ -645,6 +726,7 @@ function handleStreamConfigChanging(data: any) {
 
   // Reset all counters and states
   videoRestarting.value = true
+  pendingWebRTCReadyGate = true
   videoLoading.value = true
   videoError.value = false
   retryCount = 0
@@ -670,7 +752,7 @@ async function handleStreamConfigApplied(data: any) {
   }, GRACE_PERIOD)
 
   // Refresh video based on current mode
-  videoRestarting.value = false
+  videoRestarting.value = true
 
   // 如果正在进行模式切换，不需要在这里处理（WebRTCReady 事件会处理）
   if (isModeSwitching.value) {
@@ -680,16 +762,15 @@ async function handleStreamConfigApplied(data: any) {
 
   if (videoMode.value !== 'mjpeg') {
     // In WebRTC mode, reconnect WebRTC (session was closed due to config change)
-    const ready = await videoSession.waitForWebRTCReadyAny(3000)
+    // connectWebRTCSerial() will wait on stream.webrtc_ready when gate is enabled.
-    if (!ready) {
+    await switchToWebRTC(videoMode.value)
-      console.warn('[WebRTC] Backend not ready after timeout (config change), attempting connection anyway')
-    }
-    switchToWebRTC(videoMode.value)
   } else {
     // In MJPEG mode, refresh the MJPEG stream
     refreshVideo()
   }
 
+  videoRestarting.value = false
+
   toast.success(t('console.videoRestarted'), {
     description: `${data.device} - ${data.resolution[0]}x${data.resolution[1]} @ ${data.fps}fps`,
     duration: 3000,
@@ -699,11 +780,15 @@ async function handleStreamConfigApplied(data: any) {
 // 处理 WebRTC 就绪事件 - 这是后端真正准备好接受 WebRTC 连接的信号
 function handleWebRTCReady(data: { codec: string; hardware: boolean; transition_id?: string }) {
   console.log(`[WebRTCReady] Backend ready: codec=${data.codec}, hardware=${data.hardware}, transition_id=${data.transition_id || '-'}`)
+  pendingWebRTCReadyGate = false
   videoSession.onWebRTCReady(data)
 }
 
 function handleStreamModeReady(data: { transition_id: string; mode: string }) {
   videoSession.onModeReady(data)
+  if (data.mode === 'mjpeg') {
+    pendingWebRTCReadyGate = false
+  }
   videoRestarting.value = false
 }
 
@@ -714,6 +799,7 @@ function handleStreamModeSwitching(data: { transition_id: string; to_mode: strin
     videoLoading.value = true
     captureFrameOverlay().catch(() => {})
   }
+  pendingWebRTCReadyGate = true
   videoSession.onModeSwitching(data)
 }
 
@@ -758,6 +844,40 @@ function handleStreamStatsUpdate(data: any) {
 
 // Track if we've received the initial device_info
 let initialDeviceInfoReceived = false
+let initialModeRestoreDone = false
+let initialModeRestoreInProgress = false
+
+function normalizeServerMode(mode: string | undefined): VideoMode | null {
+  if (!mode) return null
+  if (mode === 'webrtc') return 'h264'
+  if (mode === 'mjpeg' || mode === 'h264' || mode === 'h265' || mode === 'vp8' || mode === 'vp9') {
+    return mode
+  }
+  return null
+}
+
+async function restoreInitialMode(serverMode: VideoMode) {
+  if (initialModeRestoreDone || initialModeRestoreInProgress) return
+  initialModeRestoreInProgress = true
+
+  try {
+    initialDeviceInfoReceived = true
+    if (serverMode !== videoMode.value) {
+      videoMode.value = serverMode
+      localStorage.setItem('videoMode', serverMode)
+    }
+
+    if (serverMode !== 'mjpeg') {
+      await connectWebRTCOnly(serverMode)
+    } else if (mjpegTimestamp.value === 0) {
+      refreshVideo()
+    }
+
+    initialModeRestoreDone = true
+  } finally {
+    initialModeRestoreInProgress = false
+  }
+}
 
 function handleDeviceInfo(data: any) {
   systemStore.updateFromDeviceInfo(data)
@@ -770,40 +890,28 @@ function handleDeviceInfo(data: any) {
 
   // Sync video mode from server's stream_mode
   if (data.video?.stream_mode) {
-    // Server returns: 'mjpeg', 'h264', 'h265', 'vp8', 'vp9', or 'webrtc'
+    const serverMode = normalizeServerMode(data.video.stream_mode)
-    const serverStreamMode = data.video.stream_mode
+    if (!serverMode) return
-    const serverMode = serverStreamMode === 'webrtc' ? 'h264' : serverStreamMode as VideoMode
 
     if (!initialDeviceInfoReceived) {
-      // First device_info - initialize to server mode
       initialDeviceInfoReceived = true
-
+      if (!initialModeRestoreDone && !initialModeRestoreInProgress) {
-      if (serverMode !== videoMode.value) {
+        void restoreInitialMode(serverMode)
-        // Server mode differs from default, sync to server mode without calling setMode
+        return
-        videoMode.value = serverMode
-        if (serverMode !== 'mjpeg') {
-          setTimeout(() => connectWebRTCOnly(serverMode), 100)
-        } else {
-          setTimeout(() => refreshVideo(), 100)
-        }
-      } else if (serverMode !== 'mjpeg') {
-        // Server is in WebRTC mode and client default matches, connect WebRTC (no setMode)
-        setTimeout(() => connectWebRTCOnly(serverMode), 100)
-      } else if (serverMode === 'mjpeg') {
-        // Server is in MJPEG mode and client default is also mjpeg, start MJPEG stream
-        setTimeout(() => refreshVideo(), 100)
       }
-    } else if (serverMode !== videoMode.value) {
+    }
-      // Subsequent device_info with mode change - sync to server (no setMode)
+
-      syncToServerMode(serverMode as VideoMode)
+    if (initialModeRestoreInProgress) return
+    if (serverMode !== videoMode.value) {
+      syncToServerMode(serverMode)
     }
   }
 }
 
 // Handle stream mode change event from server (WebSocket broadcast)
 function handleStreamModeChanged(data: { mode: string; previous_mode: string }) {
-  // Server returns: 'mjpeg', 'h264', 'h265', 'vp8', 'vp9', or 'webrtc'
+  const newMode = normalizeServerMode(data.mode)
-  const newMode = data.mode === 'webrtc' ? 'h264' : data.mode as VideoMode
+  if (!newMode) return
 
   // 如果正在进行模式切换，忽略这个事件（这是我们自己触发的切换产生的）
   if (isModeSwitching.value) {
@@ -819,7 +927,7 @@ function handleStreamModeChanged(data: { mode: string; previous_mode: string })
 
   // Switch to new mode (external sync handled by device_info after mode_ready)
   if (newMode !== videoMode.value) {
-    syncToServerMode(newMode as VideoMode)
+    syncToServerMode(newMode)
   }
 }
 
@@ -892,7 +1000,7 @@ async function connectWebRTCOnly(codec: VideoMode = 'h264') {
   videoErrorMessage.value = ''
 
   try {
-    const success = await webrtc.connect()
+    const success = await connectWebRTCSerial('connectWebRTCOnly')
     if (success) {
       toast.success(t('console.webrtcConnected'), {
         description: t('console.webrtcConnectedDesc'),
@@ -910,7 +1018,7 @@ async function connectWebRTCOnly(codec: VideoMode = 'h264') {
       throw new Error('WebRTC connection failed')
     }
   } catch {
-    await fallbackToMJPEG(t('console.webrtcFailed'), t('console.fallingBackToMjpeg'))
+    markWebRTCFailure(t('console.webrtcFailed'))
   }
 }
 
@@ -961,6 +1069,7 @@ async function switchToWebRTC(codec: VideoMode = 'h264') {
   videoLoading.value = true
   videoError.value = false
   videoErrorMessage.value = ''
+  pendingWebRTCReadyGate = true
 
   try {
     // Step 1: Disconnect existing WebRTC connection FIRST
@@ -995,7 +1104,7 @@ async function switchToWebRTC(codec: VideoMode = 'h264') {
     let retries = 3
     let success = false
     while (retries > 0 && !success) {
-      success = await webrtc.connect()
+      success = await connectWebRTCSerial('switchToWebRTC')
       if (!success) {
         retries--
         if (retries > 0) {
@@ -1021,30 +1130,7 @@ async function switchToWebRTC(codec: VideoMode = 'h264') {
       throw new Error('WebRTC connection failed')
     }
   } catch {
-    await fallbackToMJPEG(t('console.webrtcFailed'), t('console.fallingBackToMjpeg'), true)
+    markWebRTCFailure(t('console.webrtcFailed'))
-  }
-}
-
-async function fallbackToMJPEG(reason: string, description?: string, force = false) {
-  if (fallbackInProgress) return
-  if (videoMode.value === 'mjpeg') return
-  if (!force && (videoSession.localSwitching.value || videoSession.backendSwitching.value)) return
-
-  fallbackInProgress = true
-  videoError.value = true
-  videoErrorMessage.value = reason
-  videoLoading.value = false
-
-  toast.error(reason, {
-    description: description ?? '',
-    duration: 5000,
-  })
-
-  videoMode.value = 'mjpeg'
-  try {
-    await switchToMJPEG()
-  } finally {
-    fallbackInProgress = false
   }
 }
 
@@ -1052,6 +1138,7 @@ async function switchToMJPEG() {
   videoLoading.value = true
   videoError.value = false
   videoErrorMessage.value = ''
+  pendingWebRTCReadyGate = false
 
   // Step 1: Call backend API to switch mode FIRST
   // This ensures the MJPEG endpoint will accept our request
@@ -1069,9 +1156,9 @@ async function switchToMJPEG() {
     // Continue anyway - the mode might already be correct
   }
 
-  // Step 2: Disconnect WebRTC if connected
+  // Step 2: Disconnect WebRTC if connected or session still exists
-  if (webrtc.isConnected.value) {
+  if (webrtc.isConnected.value || webrtc.sessionId.value) {
-    webrtc.disconnect()
+    await webrtc.disconnect()
   }
 
   // Clear WebRTC video
@@ -1181,10 +1268,19 @@ watch(webrtc.stats, (stats) => {
 // Watch for WebRTC connection state changes - auto-reconnect on disconnect
 let webrtcReconnectTimeout: ReturnType<typeof setTimeout> | null = null
 let webrtcReconnectFailures = 0
-let fallbackInProgress = false
 watch(() => webrtc.state.value, (newState, oldState) => {
   console.log('[WebRTC] State changed:', oldState, '->', newState)
 
+  // Clear any pending reconnect
+  if (webrtcReconnectTimeout) {
+    clearTimeout(webrtcReconnectTimeout)
+    webrtcReconnectTimeout = null
+  }
+
+  if (shouldSuppressAutoReconnect()) {
+    return
+  }
+
   // Update stream online status based on WebRTC connection state
   if (videoMode.value !== 'mjpeg') {
     if (newState === 'connected') {
@@ -1196,28 +1292,22 @@ watch(() => webrtc.state.value, (newState, oldState) => {
     }
   }
 
-  // Clear any pending reconnect
-  if (webrtcReconnectTimeout) {
-    clearTimeout(webrtcReconnectTimeout)
-    webrtcReconnectTimeout = null
-  }
-
   // Auto-reconnect when disconnected (but was previously connected)
   if (newState === 'disconnected' && oldState === 'connected' && videoMode.value !== 'mjpeg') {
     webrtcReconnectTimeout = setTimeout(async () => {
       if (videoMode.value !== 'mjpeg' && webrtc.state.value === 'disconnected') {
         try {
-          const success = await webrtc.connect()
+          const success = await connectWebRTCSerial('auto reconnect')
           if (!success) {
             webrtcReconnectFailures += 1
             if (webrtcReconnectFailures >= 2) {
-              await fallbackToMJPEG(t('console.webrtcFailed'), t('console.fallingBackToMjpeg'))
+              markWebRTCFailure(t('console.webrtcFailed'))
             }
           }
         } catch {
           webrtcReconnectFailures += 1
           if (webrtcReconnectFailures >= 2) {
-            await fallbackToMJPEG(t('console.webrtcFailed'), t('console.fallingBackToMjpeg'))
+            markWebRTCFailure(t('console.webrtcFailed'))
           }
         }
       }
@@ -1227,7 +1317,7 @@ watch(() => webrtc.state.value, (newState, oldState) => {
   if (newState === 'failed' && videoMode.value !== 'mjpeg') {
     webrtcReconnectFailures += 1
     if (webrtcReconnectFailures >= 1) {
-      fallbackToMJPEG(t('console.webrtcFailed'), t('console.fallingBackToMjpeg')).catch(() => {})
+      markWebRTCFailure(t('console.webrtcFailed'))
     }
   }
 })
@@ -1358,20 +1448,20 @@ function handleHidError(_error: any, _operation: string) {
 }
 
 // HID channel selection: use WebRTC DataChannel when available, fallback to WebSocket
-function sendKeyboardEvent(type: 'down' | 'up', key: number, modifiers?: { ctrl?: boolean; shift?: boolean; alt?: boolean; meta?: boolean }) {
+function sendKeyboardEvent(type: 'down' | 'up', key: number, modifier?: number) {
   // In WebRTC mode with DataChannel ready, use DataChannel for lower latency
   if (videoMode.value !== 'mjpeg' && webrtc.dataChannelReady.value) {
     const event: HidKeyboardEvent = {
       type: type === 'down' ? 'keydown' : 'keyup',
       key,
-      modifiers,
+      modifier,
     }
     const sent = webrtc.sendKeyboard(event)
     if (sent) return
     // Fallback to WebSocket if DataChannel send failed
   }
   // Use WebSocket as fallback or for MJPEG mode
-  hidApi.keyboard(type, key, modifiers).catch(err => handleHidError(err, `keyboard ${type}`))
+  hidApi.keyboard(type, key, modifier).catch(err => handleHidError(err, `keyboard ${type}`))
 }
 
 function sendMouseEvent(data: { type: 'move' | 'move_abs' | 'down' | 'up' | 'scroll'; x?: number; y?: number; button?: 'left' | 'right' | 'middle'; scroll?: number }) {
@@ -1444,14 +1534,15 @@ function handleKeyDown(e: KeyboardEvent) {
 
   keyboardLed.value.capsLock = e.getModifierState('CapsLock')
 
-  const modifiers = {
+  const hidKey = keyboardEventToHidCode(e.code, e.key)
-    ctrl: e.ctrlKey,
+  if (hidKey === undefined) {
-    shift: e.shiftKey,
+    console.warn(`[HID] Unmapped key down: code=${e.code}, key=${e.key}`)
-    alt: e.altKey,
+    return
-    meta: e.metaKey,
   }
 
-  sendKeyboardEvent('down', e.keyCode, modifiers)
+  const modifierMask = updateModifierMaskForHidKey(activeModifierMask.value, hidKey, true)
+  activeModifierMask.value = modifierMask
+  sendKeyboardEvent('down', hidKey, modifierMask)
 }
 
 function handleKeyUp(e: KeyboardEvent) {
@@ -1470,7 +1561,15 @@ function handleKeyUp(e: KeyboardEvent) {
   const keyName = e.key === ' ' ? 'Space' : e.key
   pressedKeys.value = pressedKeys.value.filter(k => k !== keyName)
 
-  sendKeyboardEvent('up', e.keyCode)
+  const hidKey = keyboardEventToHidCode(e.code, e.key)
+  if (hidKey === undefined) {
+    console.warn(`[HID] Unmapped key up: code=${e.code}, key=${e.key}`)
+    return
+  }
+
+  const modifierMask = updateModifierMaskForHidKey(activeModifierMask.value, hidKey, false)
+  activeModifierMask.value = modifierMask
+  sendKeyboardEvent('up', hidKey, modifierMask)
 }
 
 function handleMouseMove(e: MouseEvent) {
@@ -1689,6 +1788,7 @@ function handlePointerLockError() {
 
 function handleBlur() {
   pressedKeys.value = []
+  activeModifierMask.value = 0
   // Release any pressed mouse button when window loses focus
   if (pressedMouseButton.value !== null) {
     const button = pressedMouseButton.value
@@ -1846,11 +1946,22 @@ onMounted(async () => {
   // Note: Video mode is now synced from server via device_info event
   // The handleDeviceInfo function will automatically switch to the server's mode
   // localStorage preference is only used when server mode matches
+  try {
+    const modeResp = await streamApi.getMode()
+    const serverMode = normalizeServerMode(modeResp?.mode)
+    if (serverMode && !initialModeRestoreDone && !initialModeRestoreInProgress) {
+      await restoreInitialMode(serverMode)
+    }
+  } catch (err) {
+    console.warn('[Console] Failed to fetch stream mode on enter, fallback to WS events:', err)
+  }
 })
 
 onUnmounted(() => {
   // Reset initial device info flag
   initialDeviceInfoReceived = false
+  initialModeRestoreDone = false
+  initialModeRestoreInProgress = false
 
   // Clear mouse flush timer
   if (mouseFlushTimer !== null) {
@@ -1881,9 +1992,9 @@ onUnmounted(() => {
   consoleEvents.unsubscribe()
   consecutiveErrors = 0
 
-  // Disconnect WebRTC if connected
+  // Disconnect WebRTC if connected or session still exists
-  if (webrtc.isConnected.value) {
+  if (webrtc.isConnected.value || webrtc.sessionId.value) {
-    webrtc.disconnect()
+    void webrtc.disconnect()
   }
 
   // Exit pointer lock if active
@@ -2161,7 +2272,7 @@ onUnmounted(() => {
 
               <Spinner class="h-16 w-16 text-white mb-4" />
               <p class="text-white/90 text-lg font-medium">
-                {{ videoRestarting ? t('console.videoRestarting') : t('console.connecting') }}
+                {{ webrtcLoadingMessage }}
               </p>
               <p class="text-white/50 text-sm mt-2">
                 {{ t('console.pleaseWait') }}