GithubFollow/One-KVM
1
0
Fork 0
mirror of https://github.com/mofeng-git/One-KVM.git synced 2026-01-28 08:31:52 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

refactor(web): 前端代码规范化重构

Browse Source 
- 集中化 HID 类型定义到 types/hid.ts，消除重复代码
- 统一 WebSocket 连接管理，提取共享工具到 types/websocket.ts
- 拆分 ConsoleView.vue 关注点，创建 useVideoStream、useHidInput、useConsoleEvents composables
- 添加 useConfigPopover 抽象配置弹窗公共逻辑
- 优化视频容器布局，支持动态比例自适应
This commit is contained in:
mofeng-git
2026-01-02 21:24:47 +08:00
parent 427751da24
commit ad401cdf1c
19 changed files with 1579 additions and 225 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/stream/ws_hid.rs
View File

@@ -225,6 +225,16 @@ impl WsHidHandler {
}
}
}
// Reset HID state when client disconnects to release any held keys/buttons
let hid = self.hid_controller.read().clone();
if let Some(hid) = hid {
if let Err(e) = hid.reset().await {
warn!("WsHidHandler: Failed to reset HID on client {} disconnect: {}", client_id, e);
} else {
debug!("WsHidHandler: HID reset on client {} disconnect", client_id);
}
}
}
/// Handle binary HID message

28
src/video/format.rs
View File

@@ -138,6 +138,34 @@ impl PixelFormat {
}
}
/// Get recommended format for video encoding (WebRTC)
///
/// Hardware encoding prefers: NV12 > YUYV
/// Software encoding prefers: YUYV > NV12
///
/// Returns None if no suitable format is available
pub fn recommended_for_encoding(available: &[PixelFormat], is_hardware: bool) -> Option<PixelFormat> {
if is_hardware {
// Hardware encoding: NV12 > YUYV
if available.contains(&PixelFormat::Nv12) {
return Some(PixelFormat::Nv12);
}
if available.contains(&PixelFormat::Yuyv) {
return Some(PixelFormat::Yuyv);
}
} else {
// Software encoding: YUYV > NV12
if available.contains(&PixelFormat::Yuyv) {
return Some(PixelFormat::Yuyv);
}
if available.contains(&PixelFormat::Nv12) {
return Some(PixelFormat::Nv12);
}
}
// Fallback to any non-compressed format
available.iter().find(|f| !f.is_compressed()).copied()
}
/// Get all supported formats
pub fn all() -> &'static [PixelFormat] {
&[

80
src/video/stream_manager.rs
View File

@@ -208,6 +208,10 @@ impl VideoStreamManager {
if current_mode == new_mode {
debug!("Already in {:?} mode, no switch needed", new_mode);
// Even if mode is the same, ensure video capture is running for WebRTC
if new_mode == StreamMode::WebRTC {
self.ensure_video_capture_running().await?;
}
return Ok(());
}
@@ -223,6 +227,43 @@ impl VideoStreamManager {
result
}
/// Ensure video capture is running (for WebRTC mode)
async fn ensure_video_capture_running(self: &Arc<Self>) -> Result<()> {
// Initialize streamer if not already initialized
if self.streamer.state().await == StreamerState::Uninitialized {
info!("Initializing video capture for WebRTC (ensure)");
if let Err(e) = self.streamer.init_auto().await {
error!("Failed to initialize video capture: {}", e);
return Err(e);
}
}
// Start video capture if not streaming
if self.streamer.state().await != StreamerState::Streaming {
info!("Starting video capture for WebRTC (ensure)");
if let Err(e) = self.streamer.start().await {
error!("Failed to start video capture: {}", e);
return Err(e);
}
// Wait a bit for capture to stabilize
tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
}
// Reconnect frame source to WebRTC
if let Some(frame_tx) = self.streamer.frame_sender().await {
let (format, resolution, fps) = self.streamer.current_video_config().await;
info!(
"Reconnecting frame source to WebRTC: {}x{} {:?} @ {}fps",
resolution.width, resolution.height, format, fps
);
self.webrtc_streamer.update_video_config(resolution, format, fps).await;
self.webrtc_streamer.set_video_source(frame_tx).await;
}
Ok(())
}
/// Internal implementation of mode switching (called with lock held)
async fn do_switch_mode(self: &Arc<Self>, current_mode: StreamMode, new_mode: StreamMode) -> Result<()> {
info!("Switching video mode: {:?} -> {:?}", current_mode, new_mode);
@@ -276,6 +317,22 @@ impl VideoStreamManager {
match new_mode {
StreamMode::Mjpeg => {
info!("Starting MJPEG streaming");
// Auto-switch to MJPEG format if device supports it
if let Some(device) = self.streamer.current_device().await {
let (current_format, resolution, fps) = self.streamer.current_video_config().await;
let available_formats: Vec<PixelFormat> = device.formats.iter().map(|f| f.format).collect();
// If current format is not MJPEG and device supports MJPEG, switch to it
if current_format != PixelFormat::Mjpeg && available_formats.contains(&PixelFormat::Mjpeg) {
info!("Auto-switching to MJPEG format for MJPEG mode");
let device_path = device.path.to_string_lossy().to_string();
if let Err(e) = self.streamer.apply_video_config(&device_path, PixelFormat::Mjpeg, resolution, fps).await {
warn!("Failed to auto-switch to MJPEG format: {}, keeping current format", e);
}
}
}
if let Err(e) = self.streamer.start().await {
error!("Failed to start MJPEG streamer: {}", e);
return Err(e);
@@ -294,6 +351,29 @@ impl VideoStreamManager {
}
}
// Auto-switch to non-compressed format if current format is MJPEG/JPEG
if let Some(device) = self.streamer.current_device().await {
let (current_format, resolution, fps) = self.streamer.current_video_config().await;
if current_format.is_compressed() {
let available_formats: Vec<PixelFormat> = device.formats.iter().map(|f| f.format).collect();
// Determine if using hardware encoding
let is_hardware = self.webrtc_streamer.is_hardware_encoding().await;
if let Some(recommended) = PixelFormat::recommended_for_encoding(&available_formats, is_hardware) {
info!(
"Auto-switching from {:?} to {:?} for WebRTC encoding (hardware={})",
current_format, recommended, is_hardware
);
let device_path = device.path.to_string_lossy().to_string();
if let Err(e) = self.streamer.apply_video_config(&device_path, recommended, resolution, fps).await {
warn!("Failed to auto-switch format for WebRTC: {}, keeping current format", e);
}
}
}
}
// Start video capture if not streaming
if self.streamer.state().await != StreamerState::Streaming {
info!("Starting video capture for WebRTC");

9
src/webrtc/peer.rs
View File

@@ -351,6 +351,15 @@ impl PeerConnection {
/// Close the connection
pub async fn close(&self) -> Result<()> {
// Reset HID state to release any held keys/buttons
if let Some(ref hid) = self.hid_controller {
if let Err(e) = hid.reset().await {
tracing::warn!("Failed to reset HID on peer {} close: {}", self.session_id, e);
} else {
tracing::debug!("HID reset on peer {} close", self.session_id);
}
}
if let Some(ref track) = self.video_track {
track.stop();
}

26
src/webrtc/webrtc_streamer.rs
View File

@@ -568,6 +568,32 @@ impl WebRtcStreamer {
);
}
/// Check if current encoder configuration uses hardware encoding
///
/// Returns true if:
/// - A specific hardware backend is configured, OR
/// - Auto mode is used and hardware encoders are available
pub async fn is_hardware_encoding(&self) -> bool {
let config = self.config.read().await;
match config.encoder_backend {
Some(backend) => backend.is_hardware(),
None => {
// Auto mode: check if hardware encoder is available for current codec
use crate::video::encoder::registry::{EncoderRegistry, VideoEncoderType};
let codec_type = match *self.video_codec.read().await {
VideoCodecType::H264 => VideoEncoderType::H264,
VideoCodecType::H265 => VideoEncoderType::H265,
VideoCodecType::VP8 => VideoEncoderType::VP8,
VideoCodecType::VP9 => VideoEncoderType::VP9,
};
EncoderRegistry::global()
.best_encoder(codec_type, false)
.map(|e| e.is_hardware)
.unwrap_or(false)
}
}
}
/// Update ICE configuration (STUN/TURN servers)
///
/// Note: Changes take effect for new sessions only.

25
web/src/components/VideoConfigPopover.vue
View File

@@ -172,6 +172,17 @@ const isFormatRecommended = (formatName: string): boolean => {
return false
}
// Check if a format is not recommended for current video mode
// In WebRTC mode, compressed formats (MJPEG/JPEG) are not recommended
const isFormatNotRecommended = (formatName: string): boolean => {
const upperFormat = formatName.toUpperCase()
// WebRTC mode: MJPEG/JPEG are not recommended (require decoding before encoding)
if (props.videoMode !== 'mjpeg') {
return upperFormat === 'MJPEG' || upperFormat === 'JPEG'
}
return false
}
// Selected values (mode comes from props)
const selectedDevice = ref<string>('')
const selectedFormat = ref<string>('')
@@ -645,6 +656,12 @@ watch(() => props.open, (isOpen) => {
>
{{ t('actionbar.recommended') }}
</span>
<span
v-else-if="isFormatNotRecommended(selectedFormatInfo.format)"
class="text-[10px] px-1 py-0.5 rounded bg-orange-100 text-orange-700 dark:bg-orange-900 dark:text-orange-300 shrink-0"
>
{{ t('actionbar.notRecommended') }}
</span>
</div>
<span v-else class="text-muted-foreground">{{ t('actionbar.selectFormat') }}</span>
</SelectTrigger>
@@ -653,7 +670,7 @@ watch(() => props.open, (isOpen) => {
v-for="format in availableFormats"
:key="format.format"
:value="format.format"
class="text-xs"
:class="['text-xs', { 'opacity-50': isFormatNotRecommended(format.format) }]"
>
<div class="flex items-center gap-2">
<span>{{ format.description }}</span>
@@ -663,6 +680,12 @@ watch(() => props.open, (isOpen) => {
>
{{ t('actionbar.recommended') }}
</span>
<span
v-else-if="isFormatNotRecommended(format.format)"
class="text-[10px] px-1.5 py-0.5 rounded bg-orange-100 text-orange-700 dark:bg-orange-900 dark:text-orange-300"
>
{{ t('actionbar.notRecommended') }}
</span>
</div>
</SelectItem>
</SelectContent>

4
web/src/composables/useAudioPlayer.ts
View File

@@ -2,6 +2,7 @@
import { ref, watch } from 'vue'
import { OpusDecoder } from 'opus-decoder'
import { buildWsUrl } from '@/types/websocket'
// Binary protocol header format (15 bytes)
// [type:1][timestamp:4][duration:2][sequence:4][length:4][data:...]
@@ -72,8 +73,7 @@ export function useAudioPlayer() {
await audioContext.resume()
}
const protocol = location.protocol === 'https:' ? 'wss:' : 'ws:'
const url = `${protocol}//${location.host}/api/ws/audio`
const url = buildWsUrl('/api/ws/audio')
ws = new WebSocket(url)
ws.binaryType = 'arraybuffer'

73
web/src/composables/useConfigPopover.ts Normal file
View File

@@ -0,0 +1,73 @@
// Config popover composable - shared logic for config popover components
// Provides common state management and lifecycle hooks
import { ref, watch, type Ref } from 'vue'
import { useI18n } from 'vue-i18n'
import { toast } from 'vue-sonner'
export interface UseConfigPopoverOptions {
/** Reactive open state from props */
open: Ref<boolean>
/** Load device list callback */
loadDevices?: () => Promise<void>
/** Initialize from current config callback */
initializeFromCurrent?: () => void
}
export function useConfigPopover(options: UseConfigPopoverOptions) {
const { t } = useI18n()
// Common state
const applying = ref(false)
const loadingDevices = ref(false)
// Watch open state to initialize
watch(options.open, async (isOpen) => {
if (isOpen) {
options.initializeFromCurrent?.()
if (options.loadDevices) {
loadingDevices.value = true
try {
await options.loadDevices()
} finally {
loadingDevices.value = false
}
}
}
})
// Apply config wrapper with loading state and toast
async function applyConfig(applyFn: () => Promise<void>) {
applying.value = true
try {
await applyFn()
toast.success(t('config.applied'))
} catch (e) {
console.info('[ConfigPopover] Apply failed:', e)
// Error toast is usually shown by API layer
} finally {
applying.value = false
}
}
// Refresh devices
async function refreshDevices() {
if (!options.loadDevices) return
loadingDevices.value = true
try {
await options.loadDevices()
} finally {
loadingDevices.value = false
}
}
return {
// State
applying,
loadingDevices,
// Methods
applyConfig,
refreshDevices,
}
}

249
web/src/composables/useConsoleEvents.ts Normal file
View File

@@ -0,0 +1,249 @@
// Console WebSocket events composable - handles all WebSocket event subscriptions
// Extracted from ConsoleView.vue for better separation of concerns
import { useI18n } from 'vue-i18n'
import { toast } from 'vue-sonner'
import { useSystemStore } from '@/stores/system'
import { useWebSocket } from '@/composables/useWebSocket'
import { getUnifiedAudio } from '@/composables/useUnifiedAudio'
export interface ConsoleEventHandlers {
onStreamConfigChanging?: (data: { reason?: string }) => void
onStreamConfigApplied?: (data: { device: string; resolution: [number, number]; fps: number }) => void
onStreamStatsUpdate?: (data: { clients?: number; clients_stat?: Record<string, { fps: number }> }) => void
onStreamModeChanged?: (data: { mode: string; previous_mode: string }) => void
onDeviceInfo?: (data: any) => void
onAudioStateChanged?: (data: { streaming: boolean; device: string | null }) => void
}
export function useConsoleEvents(handlers: ConsoleEventHandlers) {
const { t } = useI18n()
const systemStore = useSystemStore()
const { on, off, connect } = useWebSocket()
const unifiedAudio = getUnifiedAudio()
// HID event handlers
function handleHidStateChanged(_data: unknown) {
// Empty handler to prevent warning - HID state handled via device_info
}
function handleHidDeviceLost(data: { backend: string; device?: string; reason: string; error_code: string }) {
const temporaryErrors = ['eagain', 'eagain_retry']
if (temporaryErrors.includes(data.error_code)) return
if (systemStore.hid) {
systemStore.hid.initialized = false
}
toast.error(t('hid.deviceLost'), {
description: t('hid.deviceLostDesc', { backend: data.backend, reason: data.reason }),
duration: 5000,
})
}
function handleHidReconnecting(data: { backend: string; attempt: number }) {
if (data.attempt === 1 || data.attempt % 5 === 0) {
toast.info(t('hid.reconnecting'), {
description: t('hid.reconnectingDesc', { attempt: data.attempt }),
duration: 3000,
})
}
}
function handleHidRecovered(data: { backend: string }) {
if (systemStore.hid) {
systemStore.hid.initialized = true
}
toast.success(t('hid.recovered'), {
description: t('hid.recoveredDesc', { backend: data.backend }),
duration: 3000,
})
}
// Stream device monitoring handlers
function handleStreamDeviceLost(data: { device: string; reason: string }) {
if (systemStore.stream) {
systemStore.stream.online = false
}
toast.error(t('console.deviceLost'), {
description: t('console.deviceLostDesc', { device: data.device, reason: data.reason }),
duration: 5000,
})
}
function handleStreamReconnecting(data: { device: string; attempt: number }) {
if (data.attempt === 1 || data.attempt % 5 === 0) {
toast.info(t('console.deviceRecovering'), {
description: t('console.deviceRecoveringDesc', { attempt: data.attempt }),
duration: 3000,
})
}
}
function handleStreamRecovered(_data: { device: string }) {
if (systemStore.stream) {
systemStore.stream.online = true
}
toast.success(t('console.deviceRecovered'), {
description: t('console.deviceRecoveredDesc'),
duration: 3000,
})
}
function handleStreamStateChanged(data: { state: string }) {
if (data.state === 'error') {
// Handled by video stream composable
}
}
// Audio device monitoring handlers
function handleAudioDeviceLost(data: { device?: string; reason: string; error_code: string }) {
if (systemStore.audio) {
systemStore.audio.streaming = false
systemStore.audio.error = data.reason
}
toast.error(t('audio.deviceLost'), {
description: t('audio.deviceLostDesc', { device: data.device || 'default', reason: data.reason }),
duration: 5000,
})
}
function handleAudioReconnecting(data: { attempt: number }) {
if (data.attempt === 1 || data.attempt % 5 === 0) {
toast.info(t('audio.reconnecting'), {
description: t('audio.reconnectingDesc', { attempt: data.attempt }),
duration: 3000,
})
}
}
function handleAudioRecovered(data: { device?: string }) {
if (systemStore.audio) {
systemStore.audio.error = null
}
toast.success(t('audio.recovered'), {
description: t('audio.recoveredDesc', { device: data.device || 'default' }),
duration: 3000,
})
}
async function handleAudioStateChanged(data: { streaming: boolean; device: string | null }) {
if (!data.streaming) {
unifiedAudio.disconnect()
return
}
handlers.onAudioStateChanged?.(data)
}
// MSD event handlers
function handleMsdStateChanged(_data: { mode: string; connected: boolean }) {
systemStore.fetchMsdState().catch(() => null)
}
function handleMsdImageMounted(data: { image_id: string; image_name: string; size: number; cdrom: boolean }) {
toast.success(t('msd.imageMounted', { name: data.image_name }), {
description: `${(data.size / 1024 / 1024).toFixed(2)} MB - ${data.cdrom ? 'CD-ROM' : 'Disk'}`,
duration: 3000,
})
systemStore.fetchMsdState().catch(() => null)
}
function handleMsdImageUnmounted() {
toast.info(t('msd.imageUnmounted'), {
duration: 2000,
})
systemStore.fetchMsdState().catch(() => null)
}
function handleMsdError(data: { reason: string; error_code: string }) {
if (systemStore.msd) {
systemStore.msd.error = data.reason
}
toast.error(t('msd.error'), {
description: t('msd.errorDesc', { reason: data.reason }),
duration: 5000,
})
}
function handleMsdRecovered() {
if (systemStore.msd) {
systemStore.msd.error = null
}
toast.success(t('msd.recovered'), {
description: t('msd.recoveredDesc'),
duration: 3000,
})
}
// Subscribe to all events
function subscribe() {
// HID events
on('hid.state_changed', handleHidStateChanged)
on('hid.device_lost', handleHidDeviceLost)
on('hid.reconnecting', handleHidReconnecting)
on('hid.recovered', handleHidRecovered)
// Stream events
on('stream.config_changing', handlers.onStreamConfigChanging || (() => {}))
on('stream.config_applied', handlers.onStreamConfigApplied || (() => {}))
on('stream.stats_update', handlers.onStreamStatsUpdate || (() => {}))
on('stream.mode_changed', handlers.onStreamModeChanged || (() => {}))
on('stream.state_changed', handleStreamStateChanged)
on('stream.device_lost', handleStreamDeviceLost)
on('stream.reconnecting', handleStreamReconnecting)
on('stream.recovered', handleStreamRecovered)
// Audio events
on('audio.state_changed', handleAudioStateChanged)
on('audio.device_lost', handleAudioDeviceLost)
on('audio.reconnecting', handleAudioReconnecting)
on('audio.recovered', handleAudioRecovered)
// MSD events
on('msd.state_changed', handleMsdStateChanged)
on('msd.image_mounted', handleMsdImageMounted)
on('msd.image_unmounted', handleMsdImageUnmounted)
on('msd.error', handleMsdError)
on('msd.recovered', handleMsdRecovered)
// System events
on('system.device_info', handlers.onDeviceInfo || (() => {}))
// Connect WebSocket
connect()
}
// Unsubscribe from all events
function unsubscribe() {
off('hid.state_changed', handleHidStateChanged)
off('hid.device_lost', handleHidDeviceLost)
off('hid.reconnecting', handleHidReconnecting)
off('hid.recovered', handleHidRecovered)
off('stream.config_changing', handlers.onStreamConfigChanging || (() => {}))
off('stream.config_applied', handlers.onStreamConfigApplied || (() => {}))
off('stream.stats_update', handlers.onStreamStatsUpdate || (() => {}))
off('stream.mode_changed', handlers.onStreamModeChanged || (() => {}))
off('stream.state_changed', handleStreamStateChanged)
off('stream.device_lost', handleStreamDeviceLost)
off('stream.reconnecting', handleStreamReconnecting)
off('stream.recovered', handleStreamRecovered)
off('audio.state_changed', handleAudioStateChanged)
off('audio.device_lost', handleAudioDeviceLost)
off('audio.reconnecting', handleAudioReconnecting)
off('audio.recovered', handleAudioRecovered)
off('msd.state_changed', handleMsdStateChanged)
off('msd.image_mounted', handleMsdImageMounted)
off('msd.image_unmounted', handleMsdImageUnmounted)
off('msd.error', handleMsdError)
off('msd.recovered', handleMsdRecovered)
off('system.device_info', handlers.onDeviceInfo || (() => {}))
}
return {
subscribe,
unsubscribe,
}
}

333
web/src/composables/useHidInput.ts Normal file
View File

@@ -0,0 +1,333 @@
// HID input composable - manages keyboard and mouse input
// Extracted from ConsoleView.vue for better separation of concerns
import { ref, type Ref } from 'vue'
import { useI18n } from 'vue-i18n'
import { toast } from 'vue-sonner'
import { hidApi } from '@/api'
export interface HidInputState {
mouseMode: Ref<'absolute' | 'relative'>
pressedKeys: Ref<string[]>
keyboardLed: Ref<{ capsLock: boolean; numLock: boolean; scrollLock: boolean }>
mousePosition: Ref<{ x: number; y: number }>
isPointerLocked: Ref<boolean>
cursorVisible: Ref<boolean>
}
export interface UseHidInputOptions {
videoContainerRef: Ref<HTMLDivElement | null>
getVideoElement: () => HTMLElement | null
isFullscreen: Ref<boolean>
}
export function useHidInput(options: UseHidInputOptions) {
const { t } = useI18n()
// State
const mouseMode = ref<'absolute' | 'relative'>('absolute')
const pressedKeys = ref<string[]>([])
const keyboardLed = ref({
capsLock: false,
numLock: false,
scrollLock: false,
})
const mousePosition = ref({ x: 0, y: 0 })
const lastMousePosition = ref({ x: 0, y: 0 })
const isPointerLocked = ref(false)
const cursorVisible = ref(localStorage.getItem('hidShowCursor') !== 'false')
const pressedMouseButton = ref<'left' | 'right' | 'middle' | null>(null)
// Error handling - silently handle all HID errors
function handleHidError(_error: unknown, _operation: string) {
// All HID errors are silently ignored
}
// Check if a key should be blocked
function shouldBlockKey(e: KeyboardEvent): boolean {
if (options.isFullscreen.value) return true
const key = e.key.toUpperCase()
if (e.ctrlKey && ['W', 'T', 'N'].includes(key)) return false
if (key === 'F11') return false
if (e.altKey && key === 'TAB') return false
return true
}
// Keyboard handlers
function handleKeyDown(e: KeyboardEvent) {
const container = options.videoContainerRef.value
if (!container) return
if (!options.isFullscreen.value && !container.contains(document.activeElement)) return
if (shouldBlockKey(e)) {
e.preventDefault()
e.stopPropagation()
}
if (!options.isFullscreen.value && (e.metaKey || e.key === 'Meta')) {
toast.info(t('console.metaKeyHint'), {
description: t('console.metaKeyHintDesc'),
duration: 3000,
})
}
const keyName = e.key === ' ' ? 'Space' : e.key
if (!pressedKeys.value.includes(keyName)) {
pressedKeys.value = [...pressedKeys.value, keyName]
}
keyboardLed.value.capsLock = e.getModifierState('CapsLock')
keyboardLed.value.numLock = e.getModifierState('NumLock')
keyboardLed.value.scrollLock = e.getModifierState('ScrollLock')
const modifiers = {
ctrl: e.ctrlKey,
shift: e.shiftKey,
alt: e.altKey,
meta: e.metaKey,
}
hidApi.keyboard('down', e.keyCode, modifiers).catch(err => handleHidError(err, 'keyboard down'))
}
function handleKeyUp(e: KeyboardEvent) {
const container = options.videoContainerRef.value
if (!container) return
if (!options.isFullscreen.value && !container.contains(document.activeElement)) return
if (shouldBlockKey(e)) {
e.preventDefault()
e.stopPropagation()
}
const keyName = e.key === ' ' ? 'Space' : e.key
pressedKeys.value = pressedKeys.value.filter(k => k !== keyName)
hidApi.keyboard('up', e.keyCode).catch(err => handleHidError(err, 'keyboard up'))
}
// Mouse handlers
function handleMouseMove(e: MouseEvent) {
const videoElement = options.getVideoElement()
if (!videoElement) return
if (mouseMode.value === 'absolute') {
const rect = videoElement.getBoundingClientRect()
const x = Math.round((e.clientX - rect.left) / rect.width * 32767)
const y = Math.round((e.clientY - rect.top) / rect.height * 32767)
mousePosition.value = { x, y }
hidApi.mouse({ type: 'move_abs', x, y }).catch(err => handleHidError(err, 'mouse move'))
} else {
if (isPointerLocked.value) {
const dx = e.movementX
const dy = e.movementY
if (dx !== 0 || dy !== 0) {
const clampedDx = Math.max(-127, Math.min(127, dx))
const clampedDy = Math.max(-127, Math.min(127, dy))
hidApi.mouse({ type: 'move', x: clampedDx, y: clampedDy }).catch(err => handleHidError(err, 'mouse move'))
}
mousePosition.value = {
x: mousePosition.value.x + dx,
y: mousePosition.value.y + dy,
}
}
}
}
function handleMouseDown(e: MouseEvent) {
e.preventDefault()
const container = options.videoContainerRef.value
if (container && document.activeElement !== container) {
if (typeof container.focus === 'function') {
container.focus()
}
}
if (mouseMode.value === 'relative' && !isPointerLocked.value) {
requestPointerLock()
return
}
const button = e.button === 0 ? 'left' : e.button === 2 ? 'right' : 'middle'
pressedMouseButton.value = button
hidApi.mouse({ type: 'down', button }).catch(err => handleHidError(err, 'mouse down'))
}
function handleMouseUp(e: MouseEvent) {
e.preventDefault()
handleMouseUpInternal(e.button)
}
function handleWindowMouseUp(e: MouseEvent) {
if (pressedMouseButton.value !== null) {
handleMouseUpInternal(e.button)
}
}
function handleMouseUpInternal(rawButton: number) {
if (mouseMode.value === 'relative' && !isPointerLocked.value) {
pressedMouseButton.value = null
return
}
const button = rawButton === 0 ? 'left' : rawButton === 2 ? 'right' : 'middle'
if (pressedMouseButton.value !== button) return
pressedMouseButton.value = null
hidApi.mouse({ type: 'up', button }).catch(err => handleHidError(err, 'mouse up'))
}
function handleWheel(e: WheelEvent) {
e.preventDefault()
const scroll = e.deltaY > 0 ? -1 : 1
hidApi.mouse({ type: 'scroll', scroll }).catch(err => handleHidError(err, 'mouse scroll'))
}
function handleContextMenu(e: MouseEvent) {
e.preventDefault()
}
// Pointer lock
function requestPointerLock() {
const container = options.videoContainerRef.value
if (!container) return
container.requestPointerLock().catch((err: Error) => {
toast.error(t('console.pointerLockFailed'), {
description: err.message,
})
})
}
function exitPointerLock() {
if (document.pointerLockElement) {
document.exitPointerLock()
}
}
function handlePointerLockChange() {
const container = options.videoContainerRef.value
isPointerLocked.value = document.pointerLockElement === container
if (isPointerLocked.value) {
mousePosition.value = { x: 0, y: 0 }
toast.info(t('console.pointerLocked'), {
description: t('console.pointerLockedDesc'),
duration: 3000,
})
}
}
function handlePointerLockError() {
isPointerLocked.value = false
}
function handleBlur() {
pressedKeys.value = []
if (pressedMouseButton.value !== null) {
const button = pressedMouseButton.value
pressedMouseButton.value = null
hidApi.mouse({ type: 'up', button }).catch(err => handleHidError(err, 'mouse up (blur)'))
}
}
// Mode toggle
function toggleMouseMode() {
if (mouseMode.value === 'relative' && isPointerLocked.value) {
exitPointerLock()
}
mouseMode.value = mouseMode.value === 'absolute' ? 'relative' : 'absolute'
lastMousePosition.value = { x: 0, y: 0 }
mousePosition.value = { x: 0, y: 0 }
if (mouseMode.value === 'relative') {
toast.info(t('console.relativeModeHint'), {
description: t('console.relativeModeHintDesc'),
duration: 5000,
})
}
}
// Virtual keyboard handlers
function handleVirtualKeyDown(key: string) {
if (!pressedKeys.value.includes(key)) {
pressedKeys.value = [...pressedKeys.value, key]
}
}
function handleVirtualKeyUp(key: string) {
pressedKeys.value = pressedKeys.value.filter(k => k !== key)
}
// Cursor visibility
function handleCursorVisibilityChange(e: Event) {
const customEvent = e as CustomEvent<{ visible: boolean }>
cursorVisible.value = customEvent.detail.visible
}
// Setup event listeners
function setupEventListeners() {
document.addEventListener('pointerlockchange', handlePointerLockChange)
document.addEventListener('pointerlockerror', handlePointerLockError)
window.addEventListener('blur', handleBlur)
window.addEventListener('mouseup', handleWindowMouseUp)
window.addEventListener('cursor-visibility-change', handleCursorVisibilityChange)
}
function cleanupEventListeners() {
document.removeEventListener('pointerlockchange', handlePointerLockChange)
document.removeEventListener('pointerlockerror', handlePointerLockError)
window.removeEventListener('blur', handleBlur)
window.removeEventListener('mouseup', handleWindowMouseUp)
window.removeEventListener('cursor-visibility-change', handleCursorVisibilityChange)
}
return {
// State
mouseMode,
pressedKeys,
keyboardLed,
mousePosition,
isPointerLocked,
cursorVisible,
// Keyboard handlers
handleKeyDown,
handleKeyUp,
// Mouse handlers
handleMouseMove,
handleMouseDown,
handleMouseUp,
handleWheel,
handleContextMenu,
// Pointer lock
requestPointerLock,
exitPointerLock,
// Mode toggle
toggleMouseMode,
// Virtual keyboard
handleVirtualKeyDown,
handleVirtualKeyUp,
// Cursor visibility
handleCursorVisibilityChange,
// Lifecycle
setupEventListeners,
cleanupEventListeners,
}
}

121
web/src/composables/useHidWebSocket.ts
View File

@@ -2,52 +2,24 @@
// Uses the same binary format as WebRTC DataChannel for consistency
import { ref, onUnmounted } from 'vue'
import {
type HidKeyboardEvent,
type HidMouseEvent,
encodeKeyboardEvent,
encodeMouseEvent,
RESP_OK,
RESP_ERR_HID_UNAVAILABLE,
RESP_ERR_INVALID_MESSAGE,
} from '@/types/hid'
import { buildWsUrl, WS_RECONNECT_DELAY } from '@/types/websocket'
export interface HidKeyboardEvent {
type: 'keydown' | 'keyup'
key: number
modifiers?: {
ctrl?: boolean
shift?: boolean
alt?: boolean
meta?: boolean
}
}
export interface HidMouseEvent {
type: 'move' | 'moveabs' | 'down' | 'up' | 'scroll'
x?: number
y?: number
button?: number // 0=left, 1=middle, 2=right
scroll?: number
}
// Binary message constants (must match datachannel.rs)
const MSG_KEYBOARD = 0x01
const MSG_MOUSE = 0x02
// Keyboard event types
const KB_EVENT_DOWN = 0x00
const KB_EVENT_UP = 0x01
// Mouse event types
const MS_EVENT_MOVE = 0x00
const MS_EVENT_MOVE_ABS = 0x01
const MS_EVENT_DOWN = 0x02
const MS_EVENT_UP = 0x03
const MS_EVENT_SCROLL = 0x04
// Response codes from server
const RESP_OK = 0x00
const RESP_ERR_HID_UNAVAILABLE = 0x01
const RESP_ERR_INVALID_MESSAGE = 0x02
export type { HidKeyboardEvent, HidMouseEvent }
let wsInstance: WebSocket | null = null
const connected = ref(false)
const reconnectAttempts = ref(0)
const networkError = ref(false)
const networkErrorMessage = ref<string | null>(null)
const RECONNECT_DELAY = 3000
let reconnectTimeout: number | null = null
const hidUnavailable = ref(false) // Track if HID is unavailable to prevent unnecessary reconnects
@@ -61,72 +33,6 @@ let throttleTimer: number | null = null
let connectionPromise: Promise<boolean> | null = null
let connectionResolved = false
// Encode keyboard event to binary format
function encodeKeyboardEvent(event: HidKeyboardEvent): ArrayBuffer {
const buffer = new ArrayBuffer(4)
const view = new DataView(buffer)
view.setUint8(0, MSG_KEYBOARD)
view.setUint8(1, event.type === 'keydown' ? KB_EVENT_DOWN : KB_EVENT_UP)
view.setUint8(2, event.key & 0xff)
// Build modifiers bitmask
let modifiers = 0
if (event.modifiers?.ctrl) modifiers |= 0x01 // Left Ctrl
if (event.modifiers?.shift) modifiers |= 0x02 // Left Shift
if (event.modifiers?.alt) modifiers |= 0x04 // Left Alt
if (event.modifiers?.meta) modifiers |= 0x08 // Left Meta
view.setUint8(3, modifiers)
return buffer
}
// Encode mouse event to binary format
function encodeMouseEvent(event: HidMouseEvent): ArrayBuffer {
const buffer = new ArrayBuffer(7)
const view = new DataView(buffer)
view.setUint8(0, MSG_MOUSE)
// Event type
let eventType = MS_EVENT_MOVE
switch (event.type) {
case 'move':
eventType = MS_EVENT_MOVE
break
case 'moveabs':
eventType = MS_EVENT_MOVE_ABS
break
case 'down':
eventType = MS_EVENT_DOWN
break
case 'up':
eventType = MS_EVENT_UP
break
case 'scroll':
eventType = MS_EVENT_SCROLL
break
}
view.setUint8(1, eventType)
// X coordinate (i16 LE)
view.setInt16(2, event.x ?? 0, true)
// Y coordinate (i16 LE)
view.setInt16(4, event.y ?? 0, true)
// Button or scroll delta
if (event.type === 'down' || event.type === 'up') {
view.setUint8(6, event.button ?? 0)
} else if (event.type === 'scroll') {
view.setInt8(6, event.scroll ?? 0)
} else {
view.setUint8(6, 0)
}
return buffer
}
function connect(): Promise<boolean> {
// If already connected, return immediately
if (wsInstance && wsInstance.readyState === WebSocket.OPEN && connectionResolved) {
@@ -145,8 +51,7 @@ function connect(): Promise<boolean> {
networkErrorMessage.value = null
hidUnavailable.value = false
const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'
const url = `${protocol}//${window.location.host}/api/ws/hid`
const url = buildWsUrl('/api/ws/hid')
try {
wsInstance = new WebSocket(url)
@@ -197,7 +102,7 @@ function connect(): Promise<boolean> {
networkError.value = true
networkErrorMessage.value = 'HID WebSocket disconnected'
reconnectAttempts.value++
reconnectTimeout = window.setTimeout(() => connect(), RECONNECT_DELAY)
reconnectTimeout = window.setTimeout(() => connect(), WS_RECONNECT_DELAY)
}
wsInstance.onerror = () => {

507
web/src/composables/useVideoStream.ts Normal file
View File

@@ -0,0 +1,507 @@
// Video streaming composable - manages MJPEG/WebRTC video modes
// Extracted from ConsoleView.vue for better separation of concerns
import { ref, computed, watch, type Ref } from 'vue'
import { useI18n } from 'vue-i18n'
import { toast } from 'vue-sonner'
import { streamApi } from '@/api'
import { useWebRTC } from '@/composables/useWebRTC'
import { getUnifiedAudio } from '@/composables/useUnifiedAudio'
import { useSystemStore } from '@/stores/system'
import { generateUUID } from '@/lib/utils'
export type VideoMode = 'mjpeg' | 'h264' | 'h265' | 'vp8' | 'vp9'
export interface VideoStreamState {
mode: Ref<VideoMode>
loading: Ref<boolean>
error: Ref<boolean>
errorMessage: Ref<string>
restarting: Ref<boolean>
fps: Ref<number>
mjpegUrl: Ref<string>
clientId: string
}
export interface UseVideoStreamOptions {
webrtcVideoRef: Ref<HTMLVideoElement | null>
mjpegVideoRef: Ref<HTMLImageElement | null>
}
// Retry configuration
const BASE_RETRY_DELAY = 2000
const GRACE_PERIOD = 2000
const MAX_CONSECUTIVE_ERRORS = 2
export function useVideoStream(options: UseVideoStreamOptions) {
const { t } = useI18n()
const systemStore = useSystemStore()
const webrtc = useWebRTC()
const unifiedAudio = getUnifiedAudio()
// State
const videoMode = ref<VideoMode>('mjpeg')
const videoLoading = ref(true)
const videoError = ref(false)
const videoErrorMessage = ref('')
const videoRestarting = ref(false)
const backendFps = ref(0)
const mjpegTimestamp = ref(0)
const clientId = generateUUID()
// Per-client statistics
const clientsStats = ref<Record<string, { id: string; fps: number; connected_secs: number }>>({})
// Internal state
let retryTimeoutId: number | null = null
let retryCount = 0
let gracePeriodTimeoutId: number | null = null
let consecutiveErrors = 0
let isRefreshingVideo = false
let initialDeviceInfoReceived = false
let webrtcReconnectTimeout: ReturnType<typeof setTimeout> | null = null
// Computed
const mjpegUrl = computed(() => {
if (videoMode.value !== 'mjpeg') return ''
if (mjpegTimestamp.value === 0) return ''
return `${streamApi.getMjpegUrl(clientId)}&t=${mjpegTimestamp.value}`
})
const isWebRTCMode = computed(() => videoMode.value !== 'mjpeg')
// Methods
function refreshVideo() {
backendFps.value = 0
videoError.value = false
videoErrorMessage.value = ''
isRefreshingVideo = true
videoLoading.value = true
mjpegTimestamp.value = Date.now()
setTimeout(() => {
isRefreshingVideo = false
if (videoLoading.value) {
videoLoading.value = false
}
}, 1500)
}
function handleVideoLoad() {
if (videoMode.value === 'mjpeg') {
systemStore.setStreamOnline(true)
}
if (!videoLoading.value) return
clearRetryTimers()
videoLoading.value = false
videoError.value = false
videoErrorMessage.value = ''
videoRestarting.value = false
retryCount = 0
consecutiveErrors = 0
}
function handleVideoError() {
if (videoMode.value !== 'mjpeg') return
if (isRefreshingVideo) return
consecutiveErrors++
if (consecutiveErrors > MAX_CONSECUTIVE_ERRORS && gracePeriodTimeoutId !== null) {
clearTimeout(gracePeriodTimeoutId)
gracePeriodTimeoutId = null
videoRestarting.value = false
}
if (videoRestarting.value || gracePeriodTimeoutId !== null) return
if (retryTimeoutId !== null) {
clearTimeout(retryTimeoutId)
retryTimeoutId = null
}
videoLoading.value = true
retryCount++
const delay = BASE_RETRY_DELAY * Math.pow(1.5, Math.min(retryCount - 1, 5))
retryTimeoutId = window.setTimeout(() => {
retryTimeoutId = null
refreshVideo()
}, delay)
}
function clearRetryTimers() {
if (retryTimeoutId !== null) {
clearTimeout(retryTimeoutId)
retryTimeoutId = null
}
if (gracePeriodTimeoutId !== null) {
clearTimeout(gracePeriodTimeoutId)
gracePeriodTimeoutId = null
}
}
async function connectWebRTCOnly(codec: VideoMode = 'h264') {
clearRetryTimers()
retryCount = 0
consecutiveErrors = 0
mjpegTimestamp.value = 0
if (options.mjpegVideoRef.value) {
options.mjpegVideoRef.value.src = ''
}
videoLoading.value = true
videoError.value = false
videoErrorMessage.value = ''
try {
const success = await webrtc.connect()
if (success) {
toast.success(t('console.webrtcConnected'), {
description: t('console.webrtcConnectedDesc'),
duration: 3000,
})
if (webrtc.videoTrack.value && options.webrtcVideoRef.value) {
const stream = webrtc.getMediaStream()
if (stream) {
options.webrtcVideoRef.value.srcObject = stream
try {
await options.webrtcVideoRef.value.play()
} catch {
// AbortError expected when switching modes quickly
}
}
}
videoLoading.value = false
videoMode.value = codec
unifiedAudio.switchMode('webrtc')
} else {
throw new Error('WebRTC connection failed')
}
} catch {
videoError.value = true
videoErrorMessage.value = 'WebRTC connection failed'
videoLoading.value = false
}
}
async function switchToWebRTC(codec: VideoMode = 'h264') {
clearRetryTimers()
retryCount = 0
consecutiveErrors = 0
mjpegTimestamp.value = 0
if (options.mjpegVideoRef.value) {
options.mjpegVideoRef.value.src = ''
}
videoLoading.value = true
videoError.value = false
videoErrorMessage.value = ''
try {
if (webrtc.isConnected.value || webrtc.sessionId.value) {
await webrtc.disconnect()
}
await streamApi.setMode(codec)
const success = await webrtc.connect()
if (success) {
toast.success(t('console.webrtcConnected'), {
description: t('console.webrtcConnectedDesc'),
duration: 3000,
})
if (webrtc.videoTrack.value && options.webrtcVideoRef.value) {
const stream = webrtc.getMediaStream()
if (stream) {
options.webrtcVideoRef.value.srcObject = stream
try {
await options.webrtcVideoRef.value.play()
} catch {
// AbortError expected
}
}
}
videoLoading.value = false
unifiedAudio.switchMode('webrtc')
} else {
throw new Error('WebRTC connection failed')
}
} catch {
videoError.value = true
videoErrorMessage.value = t('console.webrtcFailed')
videoLoading.value = false
toast.error(t('console.webrtcFailed'), {
description: t('console.fallingBackToMjpeg'),
duration: 5000,
})
videoMode.value = 'mjpeg'
}
}
async function switchToMJPEG() {
videoLoading.value = true
videoError.value = false
videoErrorMessage.value = ''
try {
await streamApi.setMode('mjpeg')
} catch {
// Continue anyway
}
if (webrtc.isConnected.value) {
webrtc.disconnect()
}
if (options.webrtcVideoRef.value) {
options.webrtcVideoRef.value.srcObject = null
}
unifiedAudio.switchMode('ws')
refreshVideo()
}
function handleModeChange(mode: VideoMode) {
if (mode === videoMode.value) return
if (mode !== 'mjpeg') {
mjpegTimestamp.value = 0
}
videoMode.value = mode
localStorage.setItem('videoMode', mode)
if (mode !== 'mjpeg') {
switchToWebRTC(mode)
} else {
switchToMJPEG()
}
}
// Handle stream config events
function handleStreamConfigChanging(data: { reason?: string }) {
clearRetryTimers()
videoRestarting.value = true
videoLoading.value = true
videoError.value = false
retryCount = 0
consecutiveErrors = 0
backendFps.value = 0
toast.info(t('console.videoRestarting'), {
description: data.reason === 'device_switch' ? t('console.deviceSwitching') : t('console.configChanging'),
duration: 5000,
})
}
function handleStreamConfigApplied(data: { device: string; resolution: [number, number]; fps: number }) {
consecutiveErrors = 0
gracePeriodTimeoutId = window.setTimeout(() => {
gracePeriodTimeoutId = null
consecutiveErrors = 0
}, GRACE_PERIOD)
videoRestarting.value = false
if (videoMode.value !== 'mjpeg') {
switchToWebRTC(videoMode.value)
} else {
refreshVideo()
}
toast.success(t('console.videoRestarted'), {
description: `${data.device} - ${data.resolution[0]}x${data.resolution[1]} @ ${data.fps}fps`,
duration: 3000,
})
}
function handleStreamStatsUpdate(data: { clients?: number; clients_stat?: Record<string, { fps: number }> }) {
if (typeof data.clients === 'number') {
systemStore.updateStreamClients(data.clients)
}
if (videoMode.value !== 'mjpeg') {
if (data.clients_stat) {
clientsStats.value = data.clients_stat as any
}
return
}
if (data.clients_stat) {
clientsStats.value = data.clients_stat as any
const myStats = data.clients_stat[clientId]
if (myStats) {
backendFps.value = myStats.fps || 0
} else {
const fpsList = Object.values(data.clients_stat)
.map((s) => s?.fps || 0)
.filter(f => f > 0)
backendFps.value = fpsList.length > 0 ? Math.min(...fpsList) : 0
}
} else {
backendFps.value = 0
}
}
function handleDeviceInfo(data: any) {
systemStore.updateFromDeviceInfo(data)
if (data.video?.config_changing) return
if (data.video?.stream_mode) {
const serverStreamMode = data.video.stream_mode
const serverMode = serverStreamMode === 'webrtc' ? 'h264' : serverStreamMode as VideoMode
if (!initialDeviceInfoReceived) {
initialDeviceInfoReceived = true
if (serverMode !== videoMode.value) {
videoMode.value = serverMode
if (serverMode !== 'mjpeg') {
setTimeout(() => connectWebRTCOnly(serverMode), 100)
} else {
setTimeout(() => refreshVideo(), 100)
}
} else if (serverMode !== 'mjpeg') {
setTimeout(() => connectWebRTCOnly(serverMode), 100)
} else {
setTimeout(() => refreshVideo(), 100)
}
} else if (serverMode !== videoMode.value) {
handleModeChange(serverMode)
}
}
}
function handleStreamModeChanged(data: { mode: string; previous_mode: string }) {
const newMode = data.mode === 'webrtc' ? 'h264' : data.mode as VideoMode
toast.info(t('console.streamModeChanged'), {
description: t('console.streamModeChangedDesc', { mode: data.mode.toUpperCase() }),
duration: 5000,
})
if (newMode !== videoMode.value) {
handleModeChange(newMode)
}
}
// Watch WebRTC video track
watch(() => webrtc.videoTrack.value, async (track) => {
if (track && options.webrtcVideoRef.value && videoMode.value !== 'mjpeg') {
const stream = webrtc.getMediaStream()
if (stream) {
options.webrtcVideoRef.value.srcObject = stream
try {
await options.webrtcVideoRef.value.play()
} catch {
// AbortError expected
}
}
}
})
// Watch WebRTC audio track
watch(() => webrtc.audioTrack.value, (track) => {
if (track && options.webrtcVideoRef.value && videoMode.value !== 'mjpeg') {
const currentStream = options.webrtcVideoRef.value.srcObject as MediaStream | null
if (currentStream && currentStream.getAudioTracks().length === 0) {
currentStream.addTrack(track)
}
}
})
// Watch WebRTC element for unified audio
watch(options.webrtcVideoRef, (el) => {
unifiedAudio.setWebRTCElement(el)
}, { immediate: true })
// Watch WebRTC stats for FPS
watch(webrtc.stats, (stats) => {
if (videoMode.value !== 'mjpeg' && stats.framesPerSecond > 0) {
backendFps.value = Math.round(stats.framesPerSecond)
systemStore.setStreamOnline(true)
}
}, { deep: true })
// Watch WebRTC state for auto-reconnect
watch(() => webrtc.state.value, (newState, oldState) => {
if (videoMode.value !== 'mjpeg') {
if (newState === 'connected') {
systemStore.setStreamOnline(true)
}
}
if (webrtcReconnectTimeout) {
clearTimeout(webrtcReconnectTimeout)
webrtcReconnectTimeout = null
}
if (newState === 'disconnected' && oldState === 'connected' && videoMode.value !== 'mjpeg') {
webrtcReconnectTimeout = setTimeout(async () => {
if (videoMode.value !== 'mjpeg' && webrtc.state.value === 'disconnected') {
try {
await webrtc.connect()
} catch {
// Will retry on next disconnect
}
}
}, 1000)
}
})
// Cleanup
function cleanup() {
clearRetryTimers()
if (webrtcReconnectTimeout) {
clearTimeout(webrtcReconnectTimeout)
}
}
return {
// State
mode: videoMode,
loading: videoLoading,
error: videoError,
errorMessage: videoErrorMessage,
restarting: videoRestarting,
fps: backendFps,
mjpegUrl,
clientId,
clientsStats,
isWebRTCMode,
// WebRTC access
webrtc,
// Methods
refreshVideo,
handleVideoLoad,
handleVideoError,
handleModeChange,
connectWebRTCOnly,
switchToWebRTC,
switchToMJPEG,
// Event handlers
handleStreamConfigChanging,
handleStreamConfigApplied,
handleStreamStatsUpdate,
handleDeviceInfo,
handleStreamModeChanged,
// Cleanup
cleanup,
}
}

108
web/src/composables/useWebRTC.ts
View File

@@ -4,6 +4,14 @@
import { ref, onUnmounted, computed, type Ref } from 'vue'
import { webrtcApi } from '@/api'
import { generateUUID } from '@/lib/utils'
import {
type HidKeyboardEvent,
type HidMouseEvent,
encodeKeyboardEvent,
encodeMouseEvent,
} from '@/types/hid'
export type { HidKeyboardEvent, HidMouseEvent }
export type WebRTCState = 'disconnected' | 'connecting' | 'connected' | 'failed'
@@ -28,106 +36,6 @@ export interface WebRTCStats {
isRelay: boolean // true if using TURN relay
}
export interface HidKeyboardEvent {
type: 'keydown' | 'keyup'
key: number
modifiers?: {
ctrl?: boolean
shift?: boolean
alt?: boolean
meta?: boolean
}
}
export interface HidMouseEvent {
type: 'move' | 'moveabs' | 'down' | 'up' | 'scroll'
x?: number
y?: number
button?: number
scroll?: number
}
// Binary message constants (must match datachannel.rs)
const MSG_KEYBOARD = 0x01
const MSG_MOUSE = 0x02
// Keyboard event types
const KB_EVENT_DOWN = 0x00
const KB_EVENT_UP = 0x01
// Mouse event types
const MS_EVENT_MOVE = 0x00
const MS_EVENT_MOVE_ABS = 0x01
const MS_EVENT_DOWN = 0x02
const MS_EVENT_UP = 0x03
const MS_EVENT_SCROLL = 0x04
// Encode keyboard event to binary format
function encodeKeyboardEvent(event: HidKeyboardEvent): ArrayBuffer {
const buffer = new ArrayBuffer(4)
const view = new DataView(buffer)
view.setUint8(0, MSG_KEYBOARD)
view.setUint8(1, event.type === 'keydown' ? KB_EVENT_DOWN : KB_EVENT_UP)
view.setUint8(2, event.key & 0xff)
// Build modifiers bitmask
let modifiers = 0
if (event.modifiers?.ctrl) modifiers |= 0x01 // Left Ctrl
if (event.modifiers?.shift) modifiers |= 0x02 // Left Shift
if (event.modifiers?.alt) modifiers |= 0x04 // Left Alt
if (event.modifiers?.meta) modifiers |= 0x08 // Left Meta
view.setUint8(3, modifiers)
return buffer
}
// Encode mouse event to binary format
function encodeMouseEvent(event: HidMouseEvent): ArrayBuffer {
const buffer = new ArrayBuffer(7)
const view = new DataView(buffer)
view.setUint8(0, MSG_MOUSE)
// Event type
let eventType = MS_EVENT_MOVE
switch (event.type) {
case 'move':
eventType = MS_EVENT_MOVE
break
case 'moveabs':
eventType = MS_EVENT_MOVE_ABS
break
case 'down':
eventType = MS_EVENT_DOWN
break
case 'up':
eventType = MS_EVENT_UP
break
case 'scroll':
eventType = MS_EVENT_SCROLL
break
}
view.setUint8(1, eventType)
// X coordinate (i16 LE)
view.setInt16(2, event.x ?? 0, true)
// Y coordinate (i16 LE)
view.setInt16(4, event.y ?? 0, true)
// Button or scroll delta
if (event.type === 'down' || event.type === 'up') {
view.setUint8(6, event.button ?? 0)
} else if (event.type === 'scroll') {
view.setInt8(6, event.scroll ?? 0)
} else {
view.setUint8(6, 0)
}
return buffer
}
// Cached ICE servers from backend API
let cachedIceServers: RTCIceServer[] | null = null

7
web/src/composables/useWebSocket.ts
View File

@@ -5,6 +5,7 @@
// on('stream.state_changed', (data) => { ... })
import { ref } from 'vue'
import { buildWsUrl, WS_RECONNECT_DELAY } from '@/types/websocket'
export interface WsEvent {
event: string
@@ -19,15 +20,13 @@ const connected = ref(false)
const reconnectAttempts = ref(0)
const networkError = ref(false)
const networkErrorMessage = ref<string | null>(null)
const RECONNECT_DELAY = 3000
function connect() {
if (wsInstance && wsInstance.readyState === WebSocket.OPEN) {
return
}
const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'
const url = `${protocol}//${window.location.host}/api/ws`
const url = buildWsUrl('/api/ws')
try {
wsInstance = new WebSocket(url)
@@ -62,7 +61,7 @@ function connect() {
// Auto-reconnect with infinite retry
reconnectAttempts.value++
setTimeout(connect, RECONNECT_DELAY)
setTimeout(connect, WS_RECONNECT_DELAY)
}
wsInstance.onerror = () => {

1
web/src/i18n/en-US.ts
View File

@@ -122,6 +122,7 @@ export default {
backendSoftware: 'Software',
backendAuto: 'Auto',
recommended: 'Recommended',
notRecommended: 'Not Recommended',
// HID Config
hidConfig: 'Mouse & HID',
mouseSettings: 'Mouse Settings',

1
web/src/i18n/zh-CN.ts
View File

@@ -122,6 +122,7 @@ export default {
backendSoftware: '软件编码',
backendAuto: '自动',
recommended: '推荐',
notRecommended: '不推荐',
// HID Config
hidConfig: '鼠键配置',
mouseSettings: '鼠标设置',

109
web/src/types/hid.ts Normal file
View File

@@ -0,0 +1,109 @@
// HID (Human Interface Device) type definitions
// Shared between WebRTC DataChannel and WebSocket HID channels
/** Keyboard event for HID input */
export interface HidKeyboardEvent {
type: 'keydown' | 'keyup'
key: number
modifiers?: {
ctrl?: boolean
shift?: boolean
alt?: boolean
meta?: boolean
}
}
/** Mouse event for HID input */
export interface HidMouseEvent {
type: 'move' | 'moveabs' | 'down' | 'up' | 'scroll'
x?: number
y?: number
button?: number // 0=left, 1=middle, 2=right
scroll?: number
}
// Binary message constants (must match datachannel.rs / ws_hid.rs)
export const MSG_KEYBOARD = 0x01
export const MSG_MOUSE = 0x02
// Keyboard event types
export const KB_EVENT_DOWN = 0x00
export const KB_EVENT_UP = 0x01
// Mouse event types
export const MS_EVENT_MOVE = 0x00
export const MS_EVENT_MOVE_ABS = 0x01
export const MS_EVENT_DOWN = 0x02
export const MS_EVENT_UP = 0x03
export const MS_EVENT_SCROLL = 0x04
// Response codes from server
export const RESP_OK = 0x00
export const RESP_ERR_HID_UNAVAILABLE = 0x01
export const RESP_ERR_INVALID_MESSAGE = 0x02
/** Encode keyboard event to binary format (4 bytes) */
export function encodeKeyboardEvent(event: HidKeyboardEvent): ArrayBuffer {
const buffer = new ArrayBuffer(4)
const view = new DataView(buffer)
view.setUint8(0, MSG_KEYBOARD)
view.setUint8(1, event.type === 'keydown' ? KB_EVENT_DOWN : KB_EVENT_UP)
view.setUint8(2, event.key & 0xff)
// Build modifiers bitmask
let modifiers = 0
if (event.modifiers?.ctrl) modifiers |= 0x01 // Left Ctrl
if (event.modifiers?.shift) modifiers |= 0x02 // Left Shift
if (event.modifiers?.alt) modifiers |= 0x04 // Left Alt
if (event.modifiers?.meta) modifiers |= 0x08 // Left Meta
view.setUint8(3, modifiers)
return buffer
}
/** Encode mouse event to binary format (7 bytes) */
export function encodeMouseEvent(event: HidMouseEvent): ArrayBuffer {
const buffer = new ArrayBuffer(7)
const view = new DataView(buffer)
view.setUint8(0, MSG_MOUSE)
// Event type
let eventType = MS_EVENT_MOVE
switch (event.type) {
case 'move':
eventType = MS_EVENT_MOVE
break
case 'moveabs':
eventType = MS_EVENT_MOVE_ABS
break
case 'down':
eventType = MS_EVENT_DOWN
break
case 'up':
eventType = MS_EVENT_UP
break
case 'scroll':
eventType = MS_EVENT_SCROLL
break
}
view.setUint8(1, eventType)
// X coordinate (i16 LE)
view.setInt16(2, event.x ?? 0, true)
// Y coordinate (i16 LE)
view.setInt16(4, event.y ?? 0, true)
// Button or scroll delta
if (event.type === 'down' || event.type === 'up') {
view.setUint8(6, event.button ?? 0)
} else if (event.type === 'scroll') {
view.setInt8(6, event.scroll ?? 0)
} else {
view.setUint8(6, 0)
}
return buffer
}

47
web/src/types/websocket.ts Normal file
View File

@@ -0,0 +1,47 @@
// Shared WebSocket types and utilities
// Used by useWebSocket, useHidWebSocket, and useAudioPlayer
import { ref, type Ref } from 'vue'
/** WebSocket connection state */
export interface WsConnectionState {
connected: Ref<boolean>
reconnectAttempts: Ref<number>
networkError: Ref<boolean>
networkErrorMessage: Ref<string | null>
}
/** Create a new WebSocket connection state */
export function createWsConnectionState(): WsConnectionState {
return {
connected: ref(false),
reconnectAttempts: ref(0),
networkError: ref(false),
networkErrorMessage: ref(null),
}
}
/** Reset connection state to initial values */
export function resetWsConnectionState(state: WsConnectionState) {
state.connected.value = false
state.reconnectAttempts.value = 0
state.networkError.value = false
state.networkErrorMessage.value = null
}
/** Build WebSocket URL from current location */
export function buildWsUrl(path: string): string {
const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'
return `${protocol}//${window.location.host}${path}`
}
/** Default reconnect delay in milliseconds */
export const WS_RECONNECT_DELAY = 3000
/** WebSocket ready states */
export const WS_STATE = {
CONNECTING: WebSocket.CONNECTING,
OPEN: WebSocket.OPEN,
CLOSING: WebSocket.CLOSING,
CLOSED: WebSocket.CLOSED,
} as const

66
web/src/views/ConsoleView.vue
View File

@@ -75,6 +75,9 @@ const videoError = ref(false)
const videoErrorMessage = ref('')
const videoRestarting = ref(false) // Track if video is restarting due to config change
// Video aspect ratio (dynamically updated from actual video dimensions)
const videoAspectRatio = ref<number | null>(null)
// Backend-provided FPS (received from WebSocket stream.stats_update events)
const backendFps = ref(0)
@@ -342,6 +345,11 @@ function handleVideoLoad() {
// This fixes the timing issue where device_info event may arrive before stream is fully active
if (videoMode.value === 'mjpeg') {
systemStore.setStreamOnline(true)
// Update aspect ratio from MJPEG image dimensions
const img = videoRef.value
if (img && img.naturalWidth && img.naturalHeight) {
videoAspectRatio.value = img.naturalWidth / img.naturalHeight
}
}
if (!videoLoading.value) {
@@ -834,7 +842,8 @@ async function connectWebRTCOnly(codec: VideoMode = 'h264') {
retryCount = 0
consecutiveErrors = 0
// 停止 MJPEG 流
// 停止 MJPEG 流 - 重置 timestamp 以停止请求
mjpegTimestamp.value = 0
if (videoRef.value) {
videoRef.value.src = ''
}
@@ -850,6 +859,20 @@ async function connectWebRTCOnly(codec: VideoMode = 'h264') {
description: t('console.webrtcConnectedDesc'),
duration: 3000,
})
// Try to attach video immediately in case track is already available
if (webrtc.videoTrack.value && webrtcVideoRef.value) {
const stream = webrtc.getMediaStream()
if (stream) {
webrtcVideoRef.value.srcObject = stream
try {
await webrtcVideoRef.value.play()
} catch {
// AbortError is expected when switching modes quickly, ignore it
}
}
}
videoLoading.value = false
videoMode.value = codec
unifiedAudio.switchMode('webrtc')
@@ -877,7 +900,8 @@ async function switchToWebRTC(codec: VideoMode = 'h264') {
retryCount = 0
consecutiveErrors = 0
// 停止 MJPEG 流，避免同时下载两个视频流
// 停止 MJPEG 流 - 重置 timestamp 以停止请求
mjpegTimestamp.value = 0
if (videoRef.value) {
videoRef.value.src = ''
}
@@ -906,10 +930,22 @@ async function switchToWebRTC(codec: VideoMode = 'h264') {
})
// Video will be attached by the watch on webrtc.videoTrack
// Don't manually attach here to avoid race condition with ontrack
// But also try to attach immediately in case track is already available
if (webrtc.videoTrack.value && webrtcVideoRef.value) {
const stream = webrtc.getMediaStream()
if (stream) {
webrtcVideoRef.value.srcObject = stream
try {
await webrtcVideoRef.value.play()
} catch {
// AbortError is expected when switching modes quickly, ignore it
}
}
}
videoLoading.value = false
// Step 3: Switch audio to WebRTC mode
// Step 4: Switch audio to WebRTC mode
unifiedAudio.switchMode('webrtc')
} else {
throw new Error('WebRTC connection failed')
@@ -963,10 +999,9 @@ async function switchToMJPEG() {
function handleVideoModeChange(mode: VideoMode) {
if (mode === videoMode.value) return
// Reset mjpegTimestamp to 0 BEFORE changing videoMode
// This prevents mjpegUrl from returning a valid URL with old timestamp
// when switching back to MJPEG mode
if (mode === 'mjpeg') {
// Reset mjpegTimestamp to 0 when switching away from MJPEG
// This prevents mjpegUrl from returning a valid URL and stops MJPEG requests
if (mode !== 'mjpeg') {
mjpegTimestamp.value = 0
}
@@ -1022,6 +1057,10 @@ watch(webrtc.stats, (stats) => {
backendFps.value = Math.round(stats.framesPerSecond)
// WebRTC is receiving frames, set stream online
systemStore.setStreamOnline(true)
// Update aspect ratio from WebRTC video dimensions
if (stats.frameWidth && stats.frameHeight) {
videoAspectRatio.value = stats.frameWidth / stats.frameHeight
}
}
}, { deep: true })
@@ -1760,10 +1799,17 @@ onUnmounted(() => {
/>
<!-- Video Container -->
<div class="relative h-full flex items-center justify-center p-4">
<div class="relative h-full w-full flex items-center justify-center p-2 sm:p-4">
<div
ref="videoContainerRef"
class="relative max-w-full max-h-full aspect-video bg-black overflow-hidden"
class="relative bg-black overflow-hidden flex items-center justify-center"
:style="{
aspectRatio: videoAspectRatio ? String(videoAspectRatio) : '16/9',
maxWidth: '100%',
maxHeight: '100%',
minWidth: '320px',
minHeight: '180px',
}"
:class="{
'opacity-60': videoLoading || videoError,
'cursor-crosshair': cursorVisible,