From ad401cdf1cef8da2671e298f98aee0aa1fcc1bcc Mon Sep 17 00:00:00 2001 From: mofeng-git Date: Fri, 2 Jan 2026 21:24:47 +0800 Subject: [PATCH] =?UTF-8?q?refactor(web):=20=E5=89=8D=E7=AB=AF=E4=BB=A3?= =?UTF-8?q?=E7=A0=81=E8=A7=84=E8=8C=83=E5=8C=96=E9=87=8D=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 集中化 HID 类型定义到 types/hid.ts，消除重复代码 - 统一 WebSocket 连接管理，提取共享工具到 types/websocket.ts - 拆分 ConsoleView.vue 关注点，创建 useVideoStream、useHidInput、useConsoleEvents composables - 添加 useConfigPopover 抽象配置弹窗公共逻辑 - 优化视频容器布局，支持动态比例自适应 --- src/stream/ws_hid.rs | 10 + src/video/format.rs | 28 ++ src/video/stream_manager.rs | 80 ++++ src/webrtc/peer.rs | 9 + src/webrtc/webrtc_streamer.rs | 26 ++ web/src/components/VideoConfigPopover.vue | 25 +- web/src/composables/useAudioPlayer.ts | 4 +- web/src/composables/useConfigPopover.ts | 73 ++++ web/src/composables/useConsoleEvents.ts | 249 +++++++++++ web/src/composables/useHidInput.ts | 333 ++++++++++++++ web/src/composables/useHidWebSocket.ts | 121 +----- web/src/composables/useVideoStream.ts | 507 ++++++++++++++++++++++ web/src/composables/useWebRTC.ts | 108 +---- web/src/composables/useWebSocket.ts | 7 +- web/src/i18n/en-US.ts | 1 + web/src/i18n/zh-CN.ts | 1 + web/src/types/hid.ts | 109 +++++ web/src/types/websocket.ts | 47 ++ web/src/views/ConsoleView.vue | 66 ++- 19 files changed, 1579 insertions(+), 225 deletions(-) create mode 100644 web/src/composables/useConfigPopover.ts create mode 100644 web/src/composables/useConsoleEvents.ts create mode 100644 web/src/composables/useHidInput.ts create mode 100644 web/src/composables/useVideoStream.ts create mode 100644 web/src/types/hid.ts create mode 100644 web/src/types/websocket.ts diff --git a/src/stream/ws_hid.rs b/src/stream/ws_hid.rs index 434c4280..ed7132bb 100644 --- a/src/stream/ws_hid.rs +++ b/src/stream/ws_hid.rs @@ -225,6 +225,16 @@ impl WsHidHandler { } } } + + // Reset HID state when client disconnects to release any held keys/buttons + let hid = self.hid_controller.read().clone(); + if let Some(hid) = hid { + if let Err(e) = hid.reset().await { + warn!("WsHidHandler: Failed to reset HID on client {} disconnect: {}", client_id, e); + } else { + debug!("WsHidHandler: HID reset on client {} disconnect", client_id); + } + } } /// Handle binary HID message diff --git a/src/video/format.rs b/src/video/format.rs index bb4fa93a..05d014d6 100644 --- a/src/video/format.rs +++ b/src/video/format.rs @@ -138,6 +138,34 @@ impl PixelFormat { } } + /// Get recommended format for video encoding (WebRTC) + /// + /// Hardware encoding prefers: NV12 > YUYV + /// Software encoding prefers: YUYV > NV12 + /// + /// Returns None if no suitable format is available + pub fn recommended_for_encoding(available: &[PixelFormat], is_hardware: bool) -> Option { + if is_hardware { + // Hardware encoding: NV12 > YUYV + if available.contains(&PixelFormat::Nv12) { + return Some(PixelFormat::Nv12); + } + if available.contains(&PixelFormat::Yuyv) { + return Some(PixelFormat::Yuyv); + } + } else { + // Software encoding: YUYV > NV12 + if available.contains(&PixelFormat::Yuyv) { + return Some(PixelFormat::Yuyv); + } + if available.contains(&PixelFormat::Nv12) { + return Some(PixelFormat::Nv12); + } + } + // Fallback to any non-compressed format + available.iter().find(|f| !f.is_compressed()).copied() + } + /// Get all supported formats pub fn all() -> &'static [PixelFormat] { &[ diff --git a/src/video/stream_manager.rs b/src/video/stream_manager.rs index 22b34afe..0431e556 100644 --- a/src/video/stream_manager.rs +++ b/src/video/stream_manager.rs @@ -208,6 +208,10 @@ impl VideoStreamManager { if current_mode == new_mode { debug!("Already in {:?} mode, no switch needed", new_mode); + // Even if mode is the same, ensure video capture is running for WebRTC + if new_mode == StreamMode::WebRTC { + self.ensure_video_capture_running().await?; + } return Ok(()); } @@ -223,6 +227,43 @@ impl VideoStreamManager { result } + /// Ensure video capture is running (for WebRTC mode) + async fn ensure_video_capture_running(self: &Arc) -> Result<()> { + // Initialize streamer if not already initialized + if self.streamer.state().await == StreamerState::Uninitialized { + info!("Initializing video capture for WebRTC (ensure)"); + if let Err(e) = self.streamer.init_auto().await { + error!("Failed to initialize video capture: {}", e); + return Err(e); + } + } + + // Start video capture if not streaming + if self.streamer.state().await != StreamerState::Streaming { + info!("Starting video capture for WebRTC (ensure)"); + if let Err(e) = self.streamer.start().await { + error!("Failed to start video capture: {}", e); + return Err(e); + } + + // Wait a bit for capture to stabilize + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + } + + // Reconnect frame source to WebRTC + if let Some(frame_tx) = self.streamer.frame_sender().await { + let (format, resolution, fps) = self.streamer.current_video_config().await; + info!( + "Reconnecting frame source to WebRTC: {}x{} {:?} @ {}fps", + resolution.width, resolution.height, format, fps + ); + self.webrtc_streamer.update_video_config(resolution, format, fps).await; + self.webrtc_streamer.set_video_source(frame_tx).await; + } + + Ok(()) + } + /// Internal implementation of mode switching (called with lock held) async fn do_switch_mode(self: &Arc, current_mode: StreamMode, new_mode: StreamMode) -> Result<()> { info!("Switching video mode: {:?} -> {:?}", current_mode, new_mode); @@ -276,6 +317,22 @@ impl VideoStreamManager { match new_mode { StreamMode::Mjpeg => { info!("Starting MJPEG streaming"); + + // Auto-switch to MJPEG format if device supports it + if let Some(device) = self.streamer.current_device().await { + let (current_format, resolution, fps) = self.streamer.current_video_config().await; + let available_formats: Vec = device.formats.iter().map(|f| f.format).collect(); + + // If current format is not MJPEG and device supports MJPEG, switch to it + if current_format != PixelFormat::Mjpeg && available_formats.contains(&PixelFormat::Mjpeg) { + info!("Auto-switching to MJPEG format for MJPEG mode"); + let device_path = device.path.to_string_lossy().to_string(); + if let Err(e) = self.streamer.apply_video_config(&device_path, PixelFormat::Mjpeg, resolution, fps).await { + warn!("Failed to auto-switch to MJPEG format: {}, keeping current format", e); + } + } + } + if let Err(e) = self.streamer.start().await { error!("Failed to start MJPEG streamer: {}", e); return Err(e); @@ -294,6 +351,29 @@ impl VideoStreamManager { } } + // Auto-switch to non-compressed format if current format is MJPEG/JPEG + if let Some(device) = self.streamer.current_device().await { + let (current_format, resolution, fps) = self.streamer.current_video_config().await; + + if current_format.is_compressed() { + let available_formats: Vec = device.formats.iter().map(|f| f.format).collect(); + + // Determine if using hardware encoding + let is_hardware = self.webrtc_streamer.is_hardware_encoding().await; + + if let Some(recommended) = PixelFormat::recommended_for_encoding(&available_formats, is_hardware) { + info!( + "Auto-switching from {:?} to {:?} for WebRTC encoding (hardware={})", + current_format, recommended, is_hardware + ); + let device_path = device.path.to_string_lossy().to_string(); + if let Err(e) = self.streamer.apply_video_config(&device_path, recommended, resolution, fps).await { + warn!("Failed to auto-switch format for WebRTC: {}, keeping current format", e); + } + } + } + } + // Start video capture if not streaming if self.streamer.state().await != StreamerState::Streaming { info!("Starting video capture for WebRTC"); diff --git a/src/webrtc/peer.rs b/src/webrtc/peer.rs index a88ff05e..e21f23b7 100644 --- a/src/webrtc/peer.rs +++ b/src/webrtc/peer.rs @@ -351,6 +351,15 @@ impl PeerConnection { /// Close the connection pub async fn close(&self) -> Result<()> { + // Reset HID state to release any held keys/buttons + if let Some(ref hid) = self.hid_controller { + if let Err(e) = hid.reset().await { + tracing::warn!("Failed to reset HID on peer {} close: {}", self.session_id, e); + } else { + tracing::debug!("HID reset on peer {} close", self.session_id); + } + } + if let Some(ref track) = self.video_track { track.stop(); } diff --git a/src/webrtc/webrtc_streamer.rs b/src/webrtc/webrtc_streamer.rs index 799b568d..898b8580 100644 --- a/src/webrtc/webrtc_streamer.rs +++ b/src/webrtc/webrtc_streamer.rs @@ -568,6 +568,32 @@ impl WebRtcStreamer { ); } + /// Check if current encoder configuration uses hardware encoding + /// + /// Returns true if: + /// - A specific hardware backend is configured, OR + /// - Auto mode is used and hardware encoders are available + pub async fn is_hardware_encoding(&self) -> bool { + let config = self.config.read().await; + match config.encoder_backend { + Some(backend) => backend.is_hardware(), + None => { + // Auto mode: check if hardware encoder is available for current codec + use crate::video::encoder::registry::{EncoderRegistry, VideoEncoderType}; + let codec_type = match *self.video_codec.read().await { + VideoCodecType::H264 => VideoEncoderType::H264, + VideoCodecType::H265 => VideoEncoderType::H265, + VideoCodecType::VP8 => VideoEncoderType::VP8, + VideoCodecType::VP9 => VideoEncoderType::VP9, + }; + EncoderRegistry::global() + .best_encoder(codec_type, false) + .map(|e| e.is_hardware) + .unwrap_or(false) + } + } + } + /// Update ICE configuration (STUN/TURN servers) /// /// Note: Changes take effect for new sessions only. diff --git a/web/src/components/VideoConfigPopover.vue b/web/src/components/VideoConfigPopover.vue index 5c355bc3..cbf014c6 100644 --- a/web/src/components/VideoConfigPopover.vue +++ b/web/src/components/VideoConfigPopover.vue @@ -172,6 +172,17 @@ const isFormatRecommended = (formatName: string): boolean => { return false } +// Check if a format is not recommended for current video mode +// In WebRTC mode, compressed formats (MJPEG/JPEG) are not recommended +const isFormatNotRecommended = (formatName: string): boolean => { + const upperFormat = formatName.toUpperCase() + // WebRTC mode: MJPEG/JPEG are not recommended (require decoding before encoding) + if (props.videoMode !== 'mjpeg') { + return upperFormat === 'MJPEG' || upperFormat === 'JPEG' + } + return false +} + // Selected values (mode comes from props) const selectedDevice = ref('') const selectedFormat = ref('') @@ -645,6 +656,12 @@ watch(() => props.open, (isOpen) => { > {{ t('actionbar.recommended') }} + + {{ t('actionbar.notRecommended') }} + {{ t('actionbar.selectFormat') }} @@ -653,7 +670,7 @@ watch(() => props.open, (isOpen) => { v-for="format in availableFormats" :key="format.format" :value="format.format" - class="text-xs" + :class="['text-xs', { 'opacity-50': isFormatNotRecommended(format.format) }]" >
{{ format.description }} @@ -663,6 +680,12 @@ watch(() => props.open, (isOpen) => { > {{ t('actionbar.recommended') }} + + {{ t('actionbar.notRecommended') }} +
diff --git a/web/src/composables/useAudioPlayer.ts b/web/src/composables/useAudioPlayer.ts index bdda73e7..0387a389 100644 --- a/web/src/composables/useAudioPlayer.ts +++ b/web/src/composables/useAudioPlayer.ts @@ -2,6 +2,7 @@ import { ref, watch } from 'vue' import { OpusDecoder } from 'opus-decoder' +import { buildWsUrl } from '@/types/websocket' // Binary protocol header format (15 bytes) // [type:1][timestamp:4][duration:2][sequence:4][length:4][data:...] @@ -72,8 +73,7 @@ export function useAudioPlayer() { await audioContext.resume() } - const protocol = location.protocol === 'https:' ? 'wss:' : 'ws:' - const url = `${protocol}//${location.host}/api/ws/audio` + const url = buildWsUrl('/api/ws/audio') ws = new WebSocket(url) ws.binaryType = 'arraybuffer' diff --git a/web/src/composables/useConfigPopover.ts b/web/src/composables/useConfigPopover.ts new file mode 100644 index 00000000..ca4fe4cf --- /dev/null +++ b/web/src/composables/useConfigPopover.ts @@ -0,0 +1,73 @@ +// Config popover composable - shared logic for config popover components +// Provides common state management and lifecycle hooks + +import { ref, watch, type Ref } from 'vue' +import { useI18n } from 'vue-i18n' +import { toast } from 'vue-sonner' + +export interface UseConfigPopoverOptions { + /** Reactive open state from props */ + open: Ref + /** Load device list callback */ + loadDevices?: () => Promise + /** Initialize from current config callback */ + initializeFromCurrent?: () => void +} + +export function useConfigPopover(options: UseConfigPopoverOptions) { + const { t } = useI18n() + + // Common state + const applying = ref(false) + const loadingDevices = ref(false) + + // Watch open state to initialize + watch(options.open, async (isOpen) => { + if (isOpen) { + options.initializeFromCurrent?.() + if (options.loadDevices) { + loadingDevices.value = true + try { + await options.loadDevices() + } finally { + loadingDevices.value = false + } + } + } + }) + + // Apply config wrapper with loading state and toast + async function applyConfig(applyFn: () => Promise) { + applying.value = true + try { + await applyFn() + toast.success(t('config.applied')) + } catch (e) { + console.info('[ConfigPopover] Apply failed:', e) + // Error toast is usually shown by API layer + } finally { + applying.value = false + } + } + + // Refresh devices + async function refreshDevices() { + if (!options.loadDevices) return + loadingDevices.value = true + try { + await options.loadDevices() + } finally { + loadingDevices.value = false + } + } + + return { + // State + applying, + loadingDevices, + + // Methods + applyConfig, + refreshDevices, + } +} diff --git a/web/src/composables/useConsoleEvents.ts b/web/src/composables/useConsoleEvents.ts new file mode 100644 index 00000000..de73dc79 --- /dev/null +++ b/web/src/composables/useConsoleEvents.ts @@ -0,0 +1,249 @@ +// Console WebSocket events composable - handles all WebSocket event subscriptions +// Extracted from ConsoleView.vue for better separation of concerns + +import { useI18n } from 'vue-i18n' +import { toast } from 'vue-sonner' +import { useSystemStore } from '@/stores/system' +import { useWebSocket } from '@/composables/useWebSocket' +import { getUnifiedAudio } from '@/composables/useUnifiedAudio' + +export interface ConsoleEventHandlers { + onStreamConfigChanging?: (data: { reason?: string }) => void + onStreamConfigApplied?: (data: { device: string; resolution: [number, number]; fps: number }) => void + onStreamStatsUpdate?: (data: { clients?: number; clients_stat?: Record }) => void + onStreamModeChanged?: (data: { mode: string; previous_mode: string }) => void + onDeviceInfo?: (data: any) => void + onAudioStateChanged?: (data: { streaming: boolean; device: string | null }) => void +} + +export function useConsoleEvents(handlers: ConsoleEventHandlers) { + const { t } = useI18n() + const systemStore = useSystemStore() + const { on, off, connect } = useWebSocket() + const unifiedAudio = getUnifiedAudio() + + // HID event handlers + function handleHidStateChanged(_data: unknown) { + // Empty handler to prevent warning - HID state handled via device_info + } + + function handleHidDeviceLost(data: { backend: string; device?: string; reason: string; error_code: string }) { + const temporaryErrors = ['eagain', 'eagain_retry'] + if (temporaryErrors.includes(data.error_code)) return + + if (systemStore.hid) { + systemStore.hid.initialized = false + } + toast.error(t('hid.deviceLost'), { + description: t('hid.deviceLostDesc', { backend: data.backend, reason: data.reason }), + duration: 5000, + }) + } + + function handleHidReconnecting(data: { backend: string; attempt: number }) { + if (data.attempt === 1 || data.attempt % 5 === 0) { + toast.info(t('hid.reconnecting'), { + description: t('hid.reconnectingDesc', { attempt: data.attempt }), + duration: 3000, + }) + } + } + + function handleHidRecovered(data: { backend: string }) { + if (systemStore.hid) { + systemStore.hid.initialized = true + } + toast.success(t('hid.recovered'), { + description: t('hid.recoveredDesc', { backend: data.backend }), + duration: 3000, + }) + } + + // Stream device monitoring handlers + function handleStreamDeviceLost(data: { device: string; reason: string }) { + if (systemStore.stream) { + systemStore.stream.online = false + } + toast.error(t('console.deviceLost'), { + description: t('console.deviceLostDesc', { device: data.device, reason: data.reason }), + duration: 5000, + }) + } + + function handleStreamReconnecting(data: { device: string; attempt: number }) { + if (data.attempt === 1 || data.attempt % 5 === 0) { + toast.info(t('console.deviceRecovering'), { + description: t('console.deviceRecoveringDesc', { attempt: data.attempt }), + duration: 3000, + }) + } + } + + function handleStreamRecovered(_data: { device: string }) { + if (systemStore.stream) { + systemStore.stream.online = true + } + toast.success(t('console.deviceRecovered'), { + description: t('console.deviceRecoveredDesc'), + duration: 3000, + }) + } + + function handleStreamStateChanged(data: { state: string }) { + if (data.state === 'error') { + // Handled by video stream composable + } + } + + // Audio device monitoring handlers + function handleAudioDeviceLost(data: { device?: string; reason: string; error_code: string }) { + if (systemStore.audio) { + systemStore.audio.streaming = false + systemStore.audio.error = data.reason + } + toast.error(t('audio.deviceLost'), { + description: t('audio.deviceLostDesc', { device: data.device || 'default', reason: data.reason }), + duration: 5000, + }) + } + + function handleAudioReconnecting(data: { attempt: number }) { + if (data.attempt === 1 || data.attempt % 5 === 0) { + toast.info(t('audio.reconnecting'), { + description: t('audio.reconnectingDesc', { attempt: data.attempt }), + duration: 3000, + }) + } + } + + function handleAudioRecovered(data: { device?: string }) { + if (systemStore.audio) { + systemStore.audio.error = null + } + toast.success(t('audio.recovered'), { + description: t('audio.recoveredDesc', { device: data.device || 'default' }), + duration: 3000, + }) + } + + async function handleAudioStateChanged(data: { streaming: boolean; device: string | null }) { + if (!data.streaming) { + unifiedAudio.disconnect() + return + } + handlers.onAudioStateChanged?.(data) + } + + // MSD event handlers + function handleMsdStateChanged(_data: { mode: string; connected: boolean }) { + systemStore.fetchMsdState().catch(() => null) + } + + function handleMsdImageMounted(data: { image_id: string; image_name: string; size: number; cdrom: boolean }) { + toast.success(t('msd.imageMounted', { name: data.image_name }), { + description: `${(data.size / 1024 / 1024).toFixed(2)} MB - ${data.cdrom ? 'CD-ROM' : 'Disk'}`, + duration: 3000, + }) + systemStore.fetchMsdState().catch(() => null) + } + + function handleMsdImageUnmounted() { + toast.info(t('msd.imageUnmounted'), { + duration: 2000, + }) + systemStore.fetchMsdState().catch(() => null) + } + + function handleMsdError(data: { reason: string; error_code: string }) { + if (systemStore.msd) { + systemStore.msd.error = data.reason + } + toast.error(t('msd.error'), { + description: t('msd.errorDesc', { reason: data.reason }), + duration: 5000, + }) + } + + function handleMsdRecovered() { + if (systemStore.msd) { + systemStore.msd.error = null + } + toast.success(t('msd.recovered'), { + description: t('msd.recoveredDesc'), + duration: 3000, + }) + } + + // Subscribe to all events + function subscribe() { + // HID events + on('hid.state_changed', handleHidStateChanged) + on('hid.device_lost', handleHidDeviceLost) + on('hid.reconnecting', handleHidReconnecting) + on('hid.recovered', handleHidRecovered) + + // Stream events + on('stream.config_changing', handlers.onStreamConfigChanging || (() => {})) + on('stream.config_applied', handlers.onStreamConfigApplied || (() => {})) + on('stream.stats_update', handlers.onStreamStatsUpdate || (() => {})) + on('stream.mode_changed', handlers.onStreamModeChanged || (() => {})) + on('stream.state_changed', handleStreamStateChanged) + on('stream.device_lost', handleStreamDeviceLost) + on('stream.reconnecting', handleStreamReconnecting) + on('stream.recovered', handleStreamRecovered) + + // Audio events + on('audio.state_changed', handleAudioStateChanged) + on('audio.device_lost', handleAudioDeviceLost) + on('audio.reconnecting', handleAudioReconnecting) + on('audio.recovered', handleAudioRecovered) + + // MSD events + on('msd.state_changed', handleMsdStateChanged) + on('msd.image_mounted', handleMsdImageMounted) + on('msd.image_unmounted', handleMsdImageUnmounted) + on('msd.error', handleMsdError) + on('msd.recovered', handleMsdRecovered) + + // System events + on('system.device_info', handlers.onDeviceInfo || (() => {})) + + // Connect WebSocket + connect() + } + + // Unsubscribe from all events + function unsubscribe() { + off('hid.state_changed', handleHidStateChanged) + off('hid.device_lost', handleHidDeviceLost) + off('hid.reconnecting', handleHidReconnecting) + off('hid.recovered', handleHidRecovered) + + off('stream.config_changing', handlers.onStreamConfigChanging || (() => {})) + off('stream.config_applied', handlers.onStreamConfigApplied || (() => {})) + off('stream.stats_update', handlers.onStreamStatsUpdate || (() => {})) + off('stream.mode_changed', handlers.onStreamModeChanged || (() => {})) + off('stream.state_changed', handleStreamStateChanged) + off('stream.device_lost', handleStreamDeviceLost) + off('stream.reconnecting', handleStreamReconnecting) + off('stream.recovered', handleStreamRecovered) + + off('audio.state_changed', handleAudioStateChanged) + off('audio.device_lost', handleAudioDeviceLost) + off('audio.reconnecting', handleAudioReconnecting) + off('audio.recovered', handleAudioRecovered) + + off('msd.state_changed', handleMsdStateChanged) + off('msd.image_mounted', handleMsdImageMounted) + off('msd.image_unmounted', handleMsdImageUnmounted) + off('msd.error', handleMsdError) + off('msd.recovered', handleMsdRecovered) + + off('system.device_info', handlers.onDeviceInfo || (() => {})) + } + + return { + subscribe, + unsubscribe, + } +} diff --git a/web/src/composables/useHidInput.ts b/web/src/composables/useHidInput.ts new file mode 100644 index 00000000..3c42a8ee --- /dev/null +++ b/web/src/composables/useHidInput.ts @@ -0,0 +1,333 @@ +// HID input composable - manages keyboard and mouse input +// Extracted from ConsoleView.vue for better separation of concerns + +import { ref, type Ref } from 'vue' +import { useI18n } from 'vue-i18n' +import { toast } from 'vue-sonner' +import { hidApi } from '@/api' + +export interface HidInputState { + mouseMode: Ref<'absolute' | 'relative'> + pressedKeys: Ref + keyboardLed: Ref<{ capsLock: boolean; numLock: boolean; scrollLock: boolean }> + mousePosition: Ref<{ x: number; y: number }> + isPointerLocked: Ref + cursorVisible: Ref +} + +export interface UseHidInputOptions { + videoContainerRef: Ref + getVideoElement: () => HTMLElement | null + isFullscreen: Ref +} + +export function useHidInput(options: UseHidInputOptions) { + const { t } = useI18n() + + // State + const mouseMode = ref<'absolute' | 'relative'>('absolute') + const pressedKeys = ref([]) + const keyboardLed = ref({ + capsLock: false, + numLock: false, + scrollLock: false, + }) + const mousePosition = ref({ x: 0, y: 0 }) + const lastMousePosition = ref({ x: 0, y: 0 }) + const isPointerLocked = ref(false) + const cursorVisible = ref(localStorage.getItem('hidShowCursor') !== 'false') + const pressedMouseButton = ref<'left' | 'right' | 'middle' | null>(null) + + // Error handling - silently handle all HID errors + function handleHidError(_error: unknown, _operation: string) { + // All HID errors are silently ignored + } + + // Check if a key should be blocked + function shouldBlockKey(e: KeyboardEvent): boolean { + if (options.isFullscreen.value) return true + + const key = e.key.toUpperCase() + if (e.ctrlKey && ['W', 'T', 'N'].includes(key)) return false + if (key === 'F11') return false + if (e.altKey && key === 'TAB') return false + + return true + } + + // Keyboard handlers + function handleKeyDown(e: KeyboardEvent) { + const container = options.videoContainerRef.value + if (!container) return + + if (!options.isFullscreen.value && !container.contains(document.activeElement)) return + + if (shouldBlockKey(e)) { + e.preventDefault() + e.stopPropagation() + } + + if (!options.isFullscreen.value && (e.metaKey || e.key === 'Meta')) { + toast.info(t('console.metaKeyHint'), { + description: t('console.metaKeyHintDesc'), + duration: 3000, + }) + } + + const keyName = e.key === ' ' ? 'Space' : e.key + if (!pressedKeys.value.includes(keyName)) { + pressedKeys.value = [...pressedKeys.value, keyName] + } + + keyboardLed.value.capsLock = e.getModifierState('CapsLock') + keyboardLed.value.numLock = e.getModifierState('NumLock') + keyboardLed.value.scrollLock = e.getModifierState('ScrollLock') + + const modifiers = { + ctrl: e.ctrlKey, + shift: e.shiftKey, + alt: e.altKey, + meta: e.metaKey, + } + + hidApi.keyboard('down', e.keyCode, modifiers).catch(err => handleHidError(err, 'keyboard down')) + } + + function handleKeyUp(e: KeyboardEvent) { + const container = options.videoContainerRef.value + if (!container) return + + if (!options.isFullscreen.value && !container.contains(document.activeElement)) return + + if (shouldBlockKey(e)) { + e.preventDefault() + e.stopPropagation() + } + + const keyName = e.key === ' ' ? 'Space' : e.key + pressedKeys.value = pressedKeys.value.filter(k => k !== keyName) + + hidApi.keyboard('up', e.keyCode).catch(err => handleHidError(err, 'keyboard up')) + } + + // Mouse handlers + function handleMouseMove(e: MouseEvent) { + const videoElement = options.getVideoElement() + if (!videoElement) return + + if (mouseMode.value === 'absolute') { + const rect = videoElement.getBoundingClientRect() + const x = Math.round((e.clientX - rect.left) / rect.width * 32767) + const y = Math.round((e.clientY - rect.top) / rect.height * 32767) + + mousePosition.value = { x, y } + hidApi.mouse({ type: 'move_abs', x, y }).catch(err => handleHidError(err, 'mouse move')) + } else { + if (isPointerLocked.value) { + const dx = e.movementX + const dy = e.movementY + + if (dx !== 0 || dy !== 0) { + const clampedDx = Math.max(-127, Math.min(127, dx)) + const clampedDy = Math.max(-127, Math.min(127, dy)) + hidApi.mouse({ type: 'move', x: clampedDx, y: clampedDy }).catch(err => handleHidError(err, 'mouse move')) + } + + mousePosition.value = { + x: mousePosition.value.x + dx, + y: mousePosition.value.y + dy, + } + } + } + } + + function handleMouseDown(e: MouseEvent) { + e.preventDefault() + + const container = options.videoContainerRef.value + if (container && document.activeElement !== container) { + if (typeof container.focus === 'function') { + container.focus() + } + } + + if (mouseMode.value === 'relative' && !isPointerLocked.value) { + requestPointerLock() + return + } + + const button = e.button === 0 ? 'left' : e.button === 2 ? 'right' : 'middle' + pressedMouseButton.value = button + hidApi.mouse({ type: 'down', button }).catch(err => handleHidError(err, 'mouse down')) + } + + function handleMouseUp(e: MouseEvent) { + e.preventDefault() + handleMouseUpInternal(e.button) + } + + function handleWindowMouseUp(e: MouseEvent) { + if (pressedMouseButton.value !== null) { + handleMouseUpInternal(e.button) + } + } + + function handleMouseUpInternal(rawButton: number) { + if (mouseMode.value === 'relative' && !isPointerLocked.value) { + pressedMouseButton.value = null + return + } + + const button = rawButton === 0 ? 'left' : rawButton === 2 ? 'right' : 'middle' + + if (pressedMouseButton.value !== button) return + + pressedMouseButton.value = null + hidApi.mouse({ type: 'up', button }).catch(err => handleHidError(err, 'mouse up')) + } + + function handleWheel(e: WheelEvent) { + e.preventDefault() + const scroll = e.deltaY > 0 ? -1 : 1 + hidApi.mouse({ type: 'scroll', scroll }).catch(err => handleHidError(err, 'mouse scroll')) + } + + function handleContextMenu(e: MouseEvent) { + e.preventDefault() + } + + // Pointer lock + function requestPointerLock() { + const container = options.videoContainerRef.value + if (!container) return + + container.requestPointerLock().catch((err: Error) => { + toast.error(t('console.pointerLockFailed'), { + description: err.message, + }) + }) + } + + function exitPointerLock() { + if (document.pointerLockElement) { + document.exitPointerLock() + } + } + + function handlePointerLockChange() { + const container = options.videoContainerRef.value + isPointerLocked.value = document.pointerLockElement === container + + if (isPointerLocked.value) { + mousePosition.value = { x: 0, y: 0 } + toast.info(t('console.pointerLocked'), { + description: t('console.pointerLockedDesc'), + duration: 3000, + }) + } + } + + function handlePointerLockError() { + isPointerLocked.value = false + } + + function handleBlur() { + pressedKeys.value = [] + if (pressedMouseButton.value !== null) { + const button = pressedMouseButton.value + pressedMouseButton.value = null + hidApi.mouse({ type: 'up', button }).catch(err => handleHidError(err, 'mouse up (blur)')) + } + } + + // Mode toggle + function toggleMouseMode() { + if (mouseMode.value === 'relative' && isPointerLocked.value) { + exitPointerLock() + } + + mouseMode.value = mouseMode.value === 'absolute' ? 'relative' : 'absolute' + lastMousePosition.value = { x: 0, y: 0 } + mousePosition.value = { x: 0, y: 0 } + + if (mouseMode.value === 'relative') { + toast.info(t('console.relativeModeHint'), { + description: t('console.relativeModeHintDesc'), + duration: 5000, + }) + } + } + + // Virtual keyboard handlers + function handleVirtualKeyDown(key: string) { + if (!pressedKeys.value.includes(key)) { + pressedKeys.value = [...pressedKeys.value, key] + } + } + + function handleVirtualKeyUp(key: string) { + pressedKeys.value = pressedKeys.value.filter(k => k !== key) + } + + // Cursor visibility + function handleCursorVisibilityChange(e: Event) { + const customEvent = e as CustomEvent<{ visible: boolean }> + cursorVisible.value = customEvent.detail.visible + } + + // Setup event listeners + function setupEventListeners() { + document.addEventListener('pointerlockchange', handlePointerLockChange) + document.addEventListener('pointerlockerror', handlePointerLockError) + window.addEventListener('blur', handleBlur) + window.addEventListener('mouseup', handleWindowMouseUp) + window.addEventListener('cursor-visibility-change', handleCursorVisibilityChange) + } + + function cleanupEventListeners() { + document.removeEventListener('pointerlockchange', handlePointerLockChange) + document.removeEventListener('pointerlockerror', handlePointerLockError) + window.removeEventListener('blur', handleBlur) + window.removeEventListener('mouseup', handleWindowMouseUp) + window.removeEventListener('cursor-visibility-change', handleCursorVisibilityChange) + } + + return { + // State + mouseMode, + pressedKeys, + keyboardLed, + mousePosition, + isPointerLocked, + cursorVisible, + + // Keyboard handlers + handleKeyDown, + handleKeyUp, + + // Mouse handlers + handleMouseMove, + handleMouseDown, + handleMouseUp, + handleWheel, + handleContextMenu, + + // Pointer lock + requestPointerLock, + exitPointerLock, + + // Mode toggle + toggleMouseMode, + + // Virtual keyboard + handleVirtualKeyDown, + handleVirtualKeyUp, + + // Cursor visibility + handleCursorVisibilityChange, + + // Lifecycle + setupEventListeners, + cleanupEventListeners, + } +} diff --git a/web/src/composables/useHidWebSocket.ts b/web/src/composables/useHidWebSocket.ts index 78eb0bea..78be55c2 100644 --- a/web/src/composables/useHidWebSocket.ts +++ b/web/src/composables/useHidWebSocket.ts @@ -2,52 +2,24 @@ // Uses the same binary format as WebRTC DataChannel for consistency import { ref, onUnmounted } from 'vue' +import { + type HidKeyboardEvent, + type HidMouseEvent, + encodeKeyboardEvent, + encodeMouseEvent, + RESP_OK, + RESP_ERR_HID_UNAVAILABLE, + RESP_ERR_INVALID_MESSAGE, +} from '@/types/hid' +import { buildWsUrl, WS_RECONNECT_DELAY } from '@/types/websocket' -export interface HidKeyboardEvent { - type: 'keydown' | 'keyup' - key: number - modifiers?: { - ctrl?: boolean - shift?: boolean - alt?: boolean - meta?: boolean - } -} - -export interface HidMouseEvent { - type: 'move' | 'moveabs' | 'down' | 'up' | 'scroll' - x?: number - y?: number - button?: number // 0=left, 1=middle, 2=right - scroll?: number -} - -// Binary message constants (must match datachannel.rs) -const MSG_KEYBOARD = 0x01 -const MSG_MOUSE = 0x02 - -// Keyboard event types -const KB_EVENT_DOWN = 0x00 -const KB_EVENT_UP = 0x01 - -// Mouse event types -const MS_EVENT_MOVE = 0x00 -const MS_EVENT_MOVE_ABS = 0x01 -const MS_EVENT_DOWN = 0x02 -const MS_EVENT_UP = 0x03 -const MS_EVENT_SCROLL = 0x04 - -// Response codes from server -const RESP_OK = 0x00 -const RESP_ERR_HID_UNAVAILABLE = 0x01 -const RESP_ERR_INVALID_MESSAGE = 0x02 +export type { HidKeyboardEvent, HidMouseEvent } let wsInstance: WebSocket | null = null const connected = ref(false) const reconnectAttempts = ref(0) const networkError = ref(false) const networkErrorMessage = ref(null) -const RECONNECT_DELAY = 3000 let reconnectTimeout: number | null = null const hidUnavailable = ref(false) // Track if HID is unavailable to prevent unnecessary reconnects @@ -61,72 +33,6 @@ let throttleTimer: number | null = null let connectionPromise: Promise | null = null let connectionResolved = false -// Encode keyboard event to binary format -function encodeKeyboardEvent(event: HidKeyboardEvent): ArrayBuffer { - const buffer = new ArrayBuffer(4) - const view = new DataView(buffer) - - view.setUint8(0, MSG_KEYBOARD) - view.setUint8(1, event.type === 'keydown' ? KB_EVENT_DOWN : KB_EVENT_UP) - view.setUint8(2, event.key & 0xff) - - // Build modifiers bitmask - let modifiers = 0 - if (event.modifiers?.ctrl) modifiers |= 0x01 // Left Ctrl - if (event.modifiers?.shift) modifiers |= 0x02 // Left Shift - if (event.modifiers?.alt) modifiers |= 0x04 // Left Alt - if (event.modifiers?.meta) modifiers |= 0x08 // Left Meta - view.setUint8(3, modifiers) - - return buffer -} - -// Encode mouse event to binary format -function encodeMouseEvent(event: HidMouseEvent): ArrayBuffer { - const buffer = new ArrayBuffer(7) - const view = new DataView(buffer) - - view.setUint8(0, MSG_MOUSE) - - // Event type - let eventType = MS_EVENT_MOVE - switch (event.type) { - case 'move': - eventType = MS_EVENT_MOVE - break - case 'moveabs': - eventType = MS_EVENT_MOVE_ABS - break - case 'down': - eventType = MS_EVENT_DOWN - break - case 'up': - eventType = MS_EVENT_UP - break - case 'scroll': - eventType = MS_EVENT_SCROLL - break - } - view.setUint8(1, eventType) - - // X coordinate (i16 LE) - view.setInt16(2, event.x ?? 0, true) - - // Y coordinate (i16 LE) - view.setInt16(4, event.y ?? 0, true) - - // Button or scroll delta - if (event.type === 'down' || event.type === 'up') { - view.setUint8(6, event.button ?? 0) - } else if (event.type === 'scroll') { - view.setInt8(6, event.scroll ?? 0) - } else { - view.setUint8(6, 0) - } - - return buffer -} - function connect(): Promise { // If already connected, return immediately if (wsInstance && wsInstance.readyState === WebSocket.OPEN && connectionResolved) { @@ -145,8 +51,7 @@ function connect(): Promise { networkErrorMessage.value = null hidUnavailable.value = false - const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:' - const url = `${protocol}//${window.location.host}/api/ws/hid` + const url = buildWsUrl('/api/ws/hid') try { wsInstance = new WebSocket(url) @@ -197,7 +102,7 @@ function connect(): Promise { networkError.value = true networkErrorMessage.value = 'HID WebSocket disconnected' reconnectAttempts.value++ - reconnectTimeout = window.setTimeout(() => connect(), RECONNECT_DELAY) + reconnectTimeout = window.setTimeout(() => connect(), WS_RECONNECT_DELAY) } wsInstance.onerror = () => { diff --git a/web/src/composables/useVideoStream.ts b/web/src/composables/useVideoStream.ts new file mode 100644 index 00000000..38dd945a --- /dev/null +++ b/web/src/composables/useVideoStream.ts @@ -0,0 +1,507 @@ +// Video streaming composable - manages MJPEG/WebRTC video modes +// Extracted from ConsoleView.vue for better separation of concerns + +import { ref, computed, watch, type Ref } from 'vue' +import { useI18n } from 'vue-i18n' +import { toast } from 'vue-sonner' +import { streamApi } from '@/api' +import { useWebRTC } from '@/composables/useWebRTC' +import { getUnifiedAudio } from '@/composables/useUnifiedAudio' +import { useSystemStore } from '@/stores/system' +import { generateUUID } from '@/lib/utils' + +export type VideoMode = 'mjpeg' | 'h264' | 'h265' | 'vp8' | 'vp9' + +export interface VideoStreamState { + mode: Ref + loading: Ref + error: Ref + errorMessage: Ref + restarting: Ref + fps: Ref + mjpegUrl: Ref + clientId: string +} + +export interface UseVideoStreamOptions { + webrtcVideoRef: Ref + mjpegVideoRef: Ref +} + +// Retry configuration +const BASE_RETRY_DELAY = 2000 +const GRACE_PERIOD = 2000 +const MAX_CONSECUTIVE_ERRORS = 2 + +export function useVideoStream(options: UseVideoStreamOptions) { + const { t } = useI18n() + const systemStore = useSystemStore() + const webrtc = useWebRTC() + const unifiedAudio = getUnifiedAudio() + + // State + const videoMode = ref('mjpeg') + const videoLoading = ref(true) + const videoError = ref(false) + const videoErrorMessage = ref('') + const videoRestarting = ref(false) + const backendFps = ref(0) + const mjpegTimestamp = ref(0) + const clientId = generateUUID() + + // Per-client statistics + const clientsStats = ref>({}) + + // Internal state + let retryTimeoutId: number | null = null + let retryCount = 0 + let gracePeriodTimeoutId: number | null = null + let consecutiveErrors = 0 + let isRefreshingVideo = false + let initialDeviceInfoReceived = false + let webrtcReconnectTimeout: ReturnType | null = null + + // Computed + const mjpegUrl = computed(() => { + if (videoMode.value !== 'mjpeg') return '' + if (mjpegTimestamp.value === 0) return '' + return `${streamApi.getMjpegUrl(clientId)}&t=${mjpegTimestamp.value}` + }) + + const isWebRTCMode = computed(() => videoMode.value !== 'mjpeg') + + // Methods + function refreshVideo() { + backendFps.value = 0 + videoError.value = false + videoErrorMessage.value = '' + isRefreshingVideo = true + videoLoading.value = true + mjpegTimestamp.value = Date.now() + + setTimeout(() => { + isRefreshingVideo = false + if (videoLoading.value) { + videoLoading.value = false + } + }, 1500) + } + + function handleVideoLoad() { + if (videoMode.value === 'mjpeg') { + systemStore.setStreamOnline(true) + } + + if (!videoLoading.value) return + + clearRetryTimers() + videoLoading.value = false + videoError.value = false + videoErrorMessage.value = '' + videoRestarting.value = false + retryCount = 0 + consecutiveErrors = 0 + } + + function handleVideoError() { + if (videoMode.value !== 'mjpeg') return + if (isRefreshingVideo) return + + consecutiveErrors++ + + if (consecutiveErrors > MAX_CONSECUTIVE_ERRORS && gracePeriodTimeoutId !== null) { + clearTimeout(gracePeriodTimeoutId) + gracePeriodTimeoutId = null + videoRestarting.value = false + } + + if (videoRestarting.value || gracePeriodTimeoutId !== null) return + + if (retryTimeoutId !== null) { + clearTimeout(retryTimeoutId) + retryTimeoutId = null + } + + videoLoading.value = true + retryCount++ + const delay = BASE_RETRY_DELAY * Math.pow(1.5, Math.min(retryCount - 1, 5)) + + retryTimeoutId = window.setTimeout(() => { + retryTimeoutId = null + refreshVideo() + }, delay) + } + + function clearRetryTimers() { + if (retryTimeoutId !== null) { + clearTimeout(retryTimeoutId) + retryTimeoutId = null + } + if (gracePeriodTimeoutId !== null) { + clearTimeout(gracePeriodTimeoutId) + gracePeriodTimeoutId = null + } + } + + async function connectWebRTCOnly(codec: VideoMode = 'h264') { + clearRetryTimers() + retryCount = 0 + consecutiveErrors = 0 + mjpegTimestamp.value = 0 + + if (options.mjpegVideoRef.value) { + options.mjpegVideoRef.value.src = '' + } + + videoLoading.value = true + videoError.value = false + videoErrorMessage.value = '' + + try { + const success = await webrtc.connect() + if (success) { + toast.success(t('console.webrtcConnected'), { + description: t('console.webrtcConnectedDesc'), + duration: 3000, + }) + + if (webrtc.videoTrack.value && options.webrtcVideoRef.value) { + const stream = webrtc.getMediaStream() + if (stream) { + options.webrtcVideoRef.value.srcObject = stream + try { + await options.webrtcVideoRef.value.play() + } catch { + // AbortError expected when switching modes quickly + } + } + } + + videoLoading.value = false + videoMode.value = codec + unifiedAudio.switchMode('webrtc') + } else { + throw new Error('WebRTC connection failed') + } + } catch { + videoError.value = true + videoErrorMessage.value = 'WebRTC connection failed' + videoLoading.value = false + } + } + + async function switchToWebRTC(codec: VideoMode = 'h264') { + clearRetryTimers() + retryCount = 0 + consecutiveErrors = 0 + mjpegTimestamp.value = 0 + + if (options.mjpegVideoRef.value) { + options.mjpegVideoRef.value.src = '' + } + + videoLoading.value = true + videoError.value = false + videoErrorMessage.value = '' + + try { + if (webrtc.isConnected.value || webrtc.sessionId.value) { + await webrtc.disconnect() + } + + await streamApi.setMode(codec) + const success = await webrtc.connect() + + if (success) { + toast.success(t('console.webrtcConnected'), { + description: t('console.webrtcConnectedDesc'), + duration: 3000, + }) + + if (webrtc.videoTrack.value && options.webrtcVideoRef.value) { + const stream = webrtc.getMediaStream() + if (stream) { + options.webrtcVideoRef.value.srcObject = stream + try { + await options.webrtcVideoRef.value.play() + } catch { + // AbortError expected + } + } + } + + videoLoading.value = false + unifiedAudio.switchMode('webrtc') + } else { + throw new Error('WebRTC connection failed') + } + } catch { + videoError.value = true + videoErrorMessage.value = t('console.webrtcFailed') + videoLoading.value = false + + toast.error(t('console.webrtcFailed'), { + description: t('console.fallingBackToMjpeg'), + duration: 5000, + }) + videoMode.value = 'mjpeg' + } + } + + async function switchToMJPEG() { + videoLoading.value = true + videoError.value = false + videoErrorMessage.value = '' + + try { + await streamApi.setMode('mjpeg') + } catch { + // Continue anyway + } + + if (webrtc.isConnected.value) { + webrtc.disconnect() + } + + if (options.webrtcVideoRef.value) { + options.webrtcVideoRef.value.srcObject = null + } + + unifiedAudio.switchMode('ws') + refreshVideo() + } + + function handleModeChange(mode: VideoMode) { + if (mode === videoMode.value) return + + if (mode !== 'mjpeg') { + mjpegTimestamp.value = 0 + } + + videoMode.value = mode + localStorage.setItem('videoMode', mode) + + if (mode !== 'mjpeg') { + switchToWebRTC(mode) + } else { + switchToMJPEG() + } + } + + // Handle stream config events + function handleStreamConfigChanging(data: { reason?: string }) { + clearRetryTimers() + videoRestarting.value = true + videoLoading.value = true + videoError.value = false + retryCount = 0 + consecutiveErrors = 0 + backendFps.value = 0 + + toast.info(t('console.videoRestarting'), { + description: data.reason === 'device_switch' ? t('console.deviceSwitching') : t('console.configChanging'), + duration: 5000, + }) + } + + function handleStreamConfigApplied(data: { device: string; resolution: [number, number]; fps: number }) { + consecutiveErrors = 0 + + gracePeriodTimeoutId = window.setTimeout(() => { + gracePeriodTimeoutId = null + consecutiveErrors = 0 + }, GRACE_PERIOD) + + videoRestarting.value = false + + if (videoMode.value !== 'mjpeg') { + switchToWebRTC(videoMode.value) + } else { + refreshVideo() + } + + toast.success(t('console.videoRestarted'), { + description: `${data.device} - ${data.resolution[0]}x${data.resolution[1]} @ ${data.fps}fps`, + duration: 3000, + }) + } + + function handleStreamStatsUpdate(data: { clients?: number; clients_stat?: Record }) { + if (typeof data.clients === 'number') { + systemStore.updateStreamClients(data.clients) + } + + if (videoMode.value !== 'mjpeg') { + if (data.clients_stat) { + clientsStats.value = data.clients_stat as any + } + return + } + + if (data.clients_stat) { + clientsStats.value = data.clients_stat as any + const myStats = data.clients_stat[clientId] + if (myStats) { + backendFps.value = myStats.fps || 0 + } else { + const fpsList = Object.values(data.clients_stat) + .map((s) => s?.fps || 0) + .filter(f => f > 0) + backendFps.value = fpsList.length > 0 ? Math.min(...fpsList) : 0 + } + } else { + backendFps.value = 0 + } + } + + function handleDeviceInfo(data: any) { + systemStore.updateFromDeviceInfo(data) + + if (data.video?.config_changing) return + + if (data.video?.stream_mode) { + const serverStreamMode = data.video.stream_mode + const serverMode = serverStreamMode === 'webrtc' ? 'h264' : serverStreamMode as VideoMode + + if (!initialDeviceInfoReceived) { + initialDeviceInfoReceived = true + + if (serverMode !== videoMode.value) { + videoMode.value = serverMode + if (serverMode !== 'mjpeg') { + setTimeout(() => connectWebRTCOnly(serverMode), 100) + } else { + setTimeout(() => refreshVideo(), 100) + } + } else if (serverMode !== 'mjpeg') { + setTimeout(() => connectWebRTCOnly(serverMode), 100) + } else { + setTimeout(() => refreshVideo(), 100) + } + } else if (serverMode !== videoMode.value) { + handleModeChange(serverMode) + } + } + } + + function handleStreamModeChanged(data: { mode: string; previous_mode: string }) { + const newMode = data.mode === 'webrtc' ? 'h264' : data.mode as VideoMode + + toast.info(t('console.streamModeChanged'), { + description: t('console.streamModeChangedDesc', { mode: data.mode.toUpperCase() }), + duration: 5000, + }) + + if (newMode !== videoMode.value) { + handleModeChange(newMode) + } + } + + // Watch WebRTC video track + watch(() => webrtc.videoTrack.value, async (track) => { + if (track && options.webrtcVideoRef.value && videoMode.value !== 'mjpeg') { + const stream = webrtc.getMediaStream() + if (stream) { + options.webrtcVideoRef.value.srcObject = stream + try { + await options.webrtcVideoRef.value.play() + } catch { + // AbortError expected + } + } + } + }) + + // Watch WebRTC audio track + watch(() => webrtc.audioTrack.value, (track) => { + if (track && options.webrtcVideoRef.value && videoMode.value !== 'mjpeg') { + const currentStream = options.webrtcVideoRef.value.srcObject as MediaStream | null + if (currentStream && currentStream.getAudioTracks().length === 0) { + currentStream.addTrack(track) + } + } + }) + + // Watch WebRTC element for unified audio + watch(options.webrtcVideoRef, (el) => { + unifiedAudio.setWebRTCElement(el) + }, { immediate: true }) + + // Watch WebRTC stats for FPS + watch(webrtc.stats, (stats) => { + if (videoMode.value !== 'mjpeg' && stats.framesPerSecond > 0) { + backendFps.value = Math.round(stats.framesPerSecond) + systemStore.setStreamOnline(true) + } + }, { deep: true }) + + // Watch WebRTC state for auto-reconnect + watch(() => webrtc.state.value, (newState, oldState) => { + if (videoMode.value !== 'mjpeg') { + if (newState === 'connected') { + systemStore.setStreamOnline(true) + } + } + + if (webrtcReconnectTimeout) { + clearTimeout(webrtcReconnectTimeout) + webrtcReconnectTimeout = null + } + + if (newState === 'disconnected' && oldState === 'connected' && videoMode.value !== 'mjpeg') { + webrtcReconnectTimeout = setTimeout(async () => { + if (videoMode.value !== 'mjpeg' && webrtc.state.value === 'disconnected') { + try { + await webrtc.connect() + } catch { + // Will retry on next disconnect + } + } + }, 1000) + } + }) + + // Cleanup + function cleanup() { + clearRetryTimers() + if (webrtcReconnectTimeout) { + clearTimeout(webrtcReconnectTimeout) + } + } + + return { + // State + mode: videoMode, + loading: videoLoading, + error: videoError, + errorMessage: videoErrorMessage, + restarting: videoRestarting, + fps: backendFps, + mjpegUrl, + clientId, + clientsStats, + isWebRTCMode, + + // WebRTC access + webrtc, + + // Methods + refreshVideo, + handleVideoLoad, + handleVideoError, + handleModeChange, + connectWebRTCOnly, + switchToWebRTC, + switchToMJPEG, + + // Event handlers + handleStreamConfigChanging, + handleStreamConfigApplied, + handleStreamStatsUpdate, + handleDeviceInfo, + handleStreamModeChanged, + + // Cleanup + cleanup, + } +} diff --git a/web/src/composables/useWebRTC.ts b/web/src/composables/useWebRTC.ts index 3b92bba6..2d948016 100644 --- a/web/src/composables/useWebRTC.ts +++ b/web/src/composables/useWebRTC.ts @@ -4,6 +4,14 @@ import { ref, onUnmounted, computed, type Ref } from 'vue' import { webrtcApi } from '@/api' import { generateUUID } from '@/lib/utils' +import { + type HidKeyboardEvent, + type HidMouseEvent, + encodeKeyboardEvent, + encodeMouseEvent, +} from '@/types/hid' + +export type { HidKeyboardEvent, HidMouseEvent } export type WebRTCState = 'disconnected' | 'connecting' | 'connected' | 'failed' @@ -28,106 +36,6 @@ export interface WebRTCStats { isRelay: boolean // true if using TURN relay } -export interface HidKeyboardEvent { - type: 'keydown' | 'keyup' - key: number - modifiers?: { - ctrl?: boolean - shift?: boolean - alt?: boolean - meta?: boolean - } -} - -export interface HidMouseEvent { - type: 'move' | 'moveabs' | 'down' | 'up' | 'scroll' - x?: number - y?: number - button?: number - scroll?: number -} - -// Binary message constants (must match datachannel.rs) -const MSG_KEYBOARD = 0x01 -const MSG_MOUSE = 0x02 - -// Keyboard event types -const KB_EVENT_DOWN = 0x00 -const KB_EVENT_UP = 0x01 - -// Mouse event types -const MS_EVENT_MOVE = 0x00 -const MS_EVENT_MOVE_ABS = 0x01 -const MS_EVENT_DOWN = 0x02 -const MS_EVENT_UP = 0x03 -const MS_EVENT_SCROLL = 0x04 - -// Encode keyboard event to binary format -function encodeKeyboardEvent(event: HidKeyboardEvent): ArrayBuffer { - const buffer = new ArrayBuffer(4) - const view = new DataView(buffer) - - view.setUint8(0, MSG_KEYBOARD) - view.setUint8(1, event.type === 'keydown' ? KB_EVENT_DOWN : KB_EVENT_UP) - view.setUint8(2, event.key & 0xff) - - // Build modifiers bitmask - let modifiers = 0 - if (event.modifiers?.ctrl) modifiers |= 0x01 // Left Ctrl - if (event.modifiers?.shift) modifiers |= 0x02 // Left Shift - if (event.modifiers?.alt) modifiers |= 0x04 // Left Alt - if (event.modifiers?.meta) modifiers |= 0x08 // Left Meta - view.setUint8(3, modifiers) - - return buffer -} - -// Encode mouse event to binary format -function encodeMouseEvent(event: HidMouseEvent): ArrayBuffer { - const buffer = new ArrayBuffer(7) - const view = new DataView(buffer) - - view.setUint8(0, MSG_MOUSE) - - // Event type - let eventType = MS_EVENT_MOVE - switch (event.type) { - case 'move': - eventType = MS_EVENT_MOVE - break - case 'moveabs': - eventType = MS_EVENT_MOVE_ABS - break - case 'down': - eventType = MS_EVENT_DOWN - break - case 'up': - eventType = MS_EVENT_UP - break - case 'scroll': - eventType = MS_EVENT_SCROLL - break - } - view.setUint8(1, eventType) - - // X coordinate (i16 LE) - view.setInt16(2, event.x ?? 0, true) - - // Y coordinate (i16 LE) - view.setInt16(4, event.y ?? 0, true) - - // Button or scroll delta - if (event.type === 'down' || event.type === 'up') { - view.setUint8(6, event.button ?? 0) - } else if (event.type === 'scroll') { - view.setInt8(6, event.scroll ?? 0) - } else { - view.setUint8(6, 0) - } - - return buffer -} - // Cached ICE servers from backend API let cachedIceServers: RTCIceServer[] | null = null diff --git a/web/src/composables/useWebSocket.ts b/web/src/composables/useWebSocket.ts index 3253c0ab..d70e6af0 100644 --- a/web/src/composables/useWebSocket.ts +++ b/web/src/composables/useWebSocket.ts @@ -5,6 +5,7 @@ // on('stream.state_changed', (data) => { ... }) import { ref } from 'vue' +import { buildWsUrl, WS_RECONNECT_DELAY } from '@/types/websocket' export interface WsEvent { event: string @@ -19,15 +20,13 @@ const connected = ref(false) const reconnectAttempts = ref(0) const networkError = ref(false) const networkErrorMessage = ref(null) -const RECONNECT_DELAY = 3000 function connect() { if (wsInstance && wsInstance.readyState === WebSocket.OPEN) { return } - const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:' - const url = `${protocol}//${window.location.host}/api/ws` + const url = buildWsUrl('/api/ws') try { wsInstance = new WebSocket(url) @@ -62,7 +61,7 @@ function connect() { // Auto-reconnect with infinite retry reconnectAttempts.value++ - setTimeout(connect, RECONNECT_DELAY) + setTimeout(connect, WS_RECONNECT_DELAY) } wsInstance.onerror = () => { diff --git a/web/src/i18n/en-US.ts b/web/src/i18n/en-US.ts index a0101f8d..08d63e68 100644 --- a/web/src/i18n/en-US.ts +++ b/web/src/i18n/en-US.ts @@ -122,6 +122,7 @@ export default { backendSoftware: 'Software', backendAuto: 'Auto', recommended: 'Recommended', + notRecommended: 'Not Recommended', // HID Config hidConfig: 'Mouse & HID', mouseSettings: 'Mouse Settings', diff --git a/web/src/i18n/zh-CN.ts b/web/src/i18n/zh-CN.ts index 7cec10be..bfc3f36d 100644 --- a/web/src/i18n/zh-CN.ts +++ b/web/src/i18n/zh-CN.ts @@ -122,6 +122,7 @@ export default { backendSoftware: '软件编码', backendAuto: '自动', recommended: '推荐', + notRecommended: '不推荐', // HID Config hidConfig: '鼠键配置', mouseSettings: '鼠标设置', diff --git a/web/src/types/hid.ts b/web/src/types/hid.ts new file mode 100644 index 00000000..5ca38afd --- /dev/null +++ b/web/src/types/hid.ts @@ -0,0 +1,109 @@ +// HID (Human Interface Device) type definitions +// Shared between WebRTC DataChannel and WebSocket HID channels + +/** Keyboard event for HID input */ +export interface HidKeyboardEvent { + type: 'keydown' | 'keyup' + key: number + modifiers?: { + ctrl?: boolean + shift?: boolean + alt?: boolean + meta?: boolean + } +} + +/** Mouse event for HID input */ +export interface HidMouseEvent { + type: 'move' | 'moveabs' | 'down' | 'up' | 'scroll' + x?: number + y?: number + button?: number // 0=left, 1=middle, 2=right + scroll?: number +} + +// Binary message constants (must match datachannel.rs / ws_hid.rs) +export const MSG_KEYBOARD = 0x01 +export const MSG_MOUSE = 0x02 + +// Keyboard event types +export const KB_EVENT_DOWN = 0x00 +export const KB_EVENT_UP = 0x01 + +// Mouse event types +export const MS_EVENT_MOVE = 0x00 +export const MS_EVENT_MOVE_ABS = 0x01 +export const MS_EVENT_DOWN = 0x02 +export const MS_EVENT_UP = 0x03 +export const MS_EVENT_SCROLL = 0x04 + +// Response codes from server +export const RESP_OK = 0x00 +export const RESP_ERR_HID_UNAVAILABLE = 0x01 +export const RESP_ERR_INVALID_MESSAGE = 0x02 + +/** Encode keyboard event to binary format (4 bytes) */ +export function encodeKeyboardEvent(event: HidKeyboardEvent): ArrayBuffer { + const buffer = new ArrayBuffer(4) + const view = new DataView(buffer) + + view.setUint8(0, MSG_KEYBOARD) + view.setUint8(1, event.type === 'keydown' ? KB_EVENT_DOWN : KB_EVENT_UP) + view.setUint8(2, event.key & 0xff) + + // Build modifiers bitmask + let modifiers = 0 + if (event.modifiers?.ctrl) modifiers |= 0x01 // Left Ctrl + if (event.modifiers?.shift) modifiers |= 0x02 // Left Shift + if (event.modifiers?.alt) modifiers |= 0x04 // Left Alt + if (event.modifiers?.meta) modifiers |= 0x08 // Left Meta + view.setUint8(3, modifiers) + + return buffer +} + +/** Encode mouse event to binary format (7 bytes) */ +export function encodeMouseEvent(event: HidMouseEvent): ArrayBuffer { + const buffer = new ArrayBuffer(7) + const view = new DataView(buffer) + + view.setUint8(0, MSG_MOUSE) + + // Event type + let eventType = MS_EVENT_MOVE + switch (event.type) { + case 'move': + eventType = MS_EVENT_MOVE + break + case 'moveabs': + eventType = MS_EVENT_MOVE_ABS + break + case 'down': + eventType = MS_EVENT_DOWN + break + case 'up': + eventType = MS_EVENT_UP + break + case 'scroll': + eventType = MS_EVENT_SCROLL + break + } + view.setUint8(1, eventType) + + // X coordinate (i16 LE) + view.setInt16(2, event.x ?? 0, true) + + // Y coordinate (i16 LE) + view.setInt16(4, event.y ?? 0, true) + + // Button or scroll delta + if (event.type === 'down' || event.type === 'up') { + view.setUint8(6, event.button ?? 0) + } else if (event.type === 'scroll') { + view.setInt8(6, event.scroll ?? 0) + } else { + view.setUint8(6, 0) + } + + return buffer +} diff --git a/web/src/types/websocket.ts b/web/src/types/websocket.ts new file mode 100644 index 00000000..20b73751 --- /dev/null +++ b/web/src/types/websocket.ts @@ -0,0 +1,47 @@ +// Shared WebSocket types and utilities +// Used by useWebSocket, useHidWebSocket, and useAudioPlayer + +import { ref, type Ref } from 'vue' + +/** WebSocket connection state */ +export interface WsConnectionState { + connected: Ref + reconnectAttempts: Ref + networkError: Ref + networkErrorMessage: Ref +} + +/** Create a new WebSocket connection state */ +export function createWsConnectionState(): WsConnectionState { + return { + connected: ref(false), + reconnectAttempts: ref(0), + networkError: ref(false), + networkErrorMessage: ref(null), + } +} + +/** Reset connection state to initial values */ +export function resetWsConnectionState(state: WsConnectionState) { + state.connected.value = false + state.reconnectAttempts.value = 0 + state.networkError.value = false + state.networkErrorMessage.value = null +} + +/** Build WebSocket URL from current location */ +export function buildWsUrl(path: string): string { + const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:' + return `${protocol}//${window.location.host}${path}` +} + +/** Default reconnect delay in milliseconds */ +export const WS_RECONNECT_DELAY = 3000 + +/** WebSocket ready states */ +export const WS_STATE = { + CONNECTING: WebSocket.CONNECTING, + OPEN: WebSocket.OPEN, + CLOSING: WebSocket.CLOSING, + CLOSED: WebSocket.CLOSED, +} as const diff --git a/web/src/views/ConsoleView.vue b/web/src/views/ConsoleView.vue index 92bdd14a..48ab3c42 100644 --- a/web/src/views/ConsoleView.vue +++ b/web/src/views/ConsoleView.vue @@ -75,6 +75,9 @@ const videoError = ref(false) const videoErrorMessage = ref('') const videoRestarting = ref(false) // Track if video is restarting due to config change +// Video aspect ratio (dynamically updated from actual video dimensions) +const videoAspectRatio = ref(null) + // Backend-provided FPS (received from WebSocket stream.stats_update events) const backendFps = ref(0) @@ -342,6 +345,11 @@ function handleVideoLoad() { // This fixes the timing issue where device_info event may arrive before stream is fully active if (videoMode.value === 'mjpeg') { systemStore.setStreamOnline(true) + // Update aspect ratio from MJPEG image dimensions + const img = videoRef.value + if (img && img.naturalWidth && img.naturalHeight) { + videoAspectRatio.value = img.naturalWidth / img.naturalHeight + } } if (!videoLoading.value) { @@ -834,7 +842,8 @@ async function connectWebRTCOnly(codec: VideoMode = 'h264') { retryCount = 0 consecutiveErrors = 0 - // 停止 MJPEG 流 + // 停止 MJPEG 流 - 重置 timestamp 以停止请求 + mjpegTimestamp.value = 0 if (videoRef.value) { videoRef.value.src = '' } @@ -850,6 +859,20 @@ async function connectWebRTCOnly(codec: VideoMode = 'h264') { description: t('console.webrtcConnectedDesc'), duration: 3000, }) + + // Try to attach video immediately in case track is already available + if (webrtc.videoTrack.value && webrtcVideoRef.value) { + const stream = webrtc.getMediaStream() + if (stream) { + webrtcVideoRef.value.srcObject = stream + try { + await webrtcVideoRef.value.play() + } catch { + // AbortError is expected when switching modes quickly, ignore it + } + } + } + videoLoading.value = false videoMode.value = codec unifiedAudio.switchMode('webrtc') @@ -877,7 +900,8 @@ async function switchToWebRTC(codec: VideoMode = 'h264') { retryCount = 0 consecutiveErrors = 0 - // 停止 MJPEG 流，避免同时下载两个视频流 + // 停止 MJPEG 流 - 重置 timestamp 以停止请求 + mjpegTimestamp.value = 0 if (videoRef.value) { videoRef.value.src = '' } @@ -906,10 +930,22 @@ async function switchToWebRTC(codec: VideoMode = 'h264') { }) // Video will be attached by the watch on webrtc.videoTrack - // Don't manually attach here to avoid race condition with ontrack + // But also try to attach immediately in case track is already available + if (webrtc.videoTrack.value && webrtcVideoRef.value) { + const stream = webrtc.getMediaStream() + if (stream) { + webrtcVideoRef.value.srcObject = stream + try { + await webrtcVideoRef.value.play() + } catch { + // AbortError is expected when switching modes quickly, ignore it + } + } + } + videoLoading.value = false - // Step 3: Switch audio to WebRTC mode + // Step 4: Switch audio to WebRTC mode unifiedAudio.switchMode('webrtc') } else { throw new Error('WebRTC connection failed') @@ -963,10 +999,9 @@ async function switchToMJPEG() { function handleVideoModeChange(mode: VideoMode) { if (mode === videoMode.value) return - // Reset mjpegTimestamp to 0 BEFORE changing videoMode - // This prevents mjpegUrl from returning a valid URL with old timestamp - // when switching back to MJPEG mode - if (mode === 'mjpeg') { + // Reset mjpegTimestamp to 0 when switching away from MJPEG + // This prevents mjpegUrl from returning a valid URL and stops MJPEG requests + if (mode !== 'mjpeg') { mjpegTimestamp.value = 0 } @@ -1022,6 +1057,10 @@ watch(webrtc.stats, (stats) => { backendFps.value = Math.round(stats.framesPerSecond) // WebRTC is receiving frames, set stream online systemStore.setStreamOnline(true) + // Update aspect ratio from WebRTC video dimensions + if (stats.frameWidth && stats.frameHeight) { + videoAspectRatio.value = stats.frameWidth / stats.frameHeight + } } }, { deep: true }) @@ -1760,10 +1799,17 @@ onUnmounted(() => { /> -
+