feat(webrtc): 添加公共ICE服务器支持和优化HID延迟

- 重构ICE配置:将TURN配置改为统一的ICE配置,支持STUN和多TURN URL
- 添加公共ICE服务器:类似RustDesk,用户留空时使用编译时配置的公共服务器
- 优化DataChannel HID消息:使用tokio::spawn立即处理,避免依赖webrtc-rs轮询
- 添加WebRTCReady事件:客户端等待此事件后再建立连接
- 初始化时启动音频流,确保WebRTC可订阅
- 移除多余的trace/debug日志减少开销
- 更新前端配置界面支持公共ICE服务器显示
This commit is contained in:
mofeng-git
2026-01-04 15:06:08 +08:00
parent 0c82d1a840
commit 9ab3d052f9
24 changed files with 766 additions and 258 deletions

View File

@@ -1,7 +1,8 @@
<script setup lang="ts">
import { ref } from 'vue'
import { ref, computed } from 'vue'
import { useI18n } from 'vue-i18n'
import { useRouter } from 'vue-router'
import { useSystemStore } from '@/stores/system'
import { Button } from '@/components/ui/button'
import {
Popover,
@@ -42,10 +43,16 @@ import MsdDialog from '@/components/MsdDialog.vue'
const { t } = useI18n()
const router = useRouter()
const systemStore = useSystemStore()
// Overflow menu state
const overflowMenuOpen = ref(false)
// MSD is only available when HID backend is not CH9329 (CH9329 is serial-only, no USB gadget)
const showMsd = computed(() => {
return props.isAdmin && systemStore.hid?.backend !== 'ch9329'
})
const props = defineProps<{
mouseMode?: 'absolute' | 'relative'
videoMode?: VideoMode
@@ -100,7 +107,8 @@ const extensionOpen = ref(false)
/>
<!-- Virtual Media (MSD) - Hidden on small screens, shown in overflow -->
<TooltipProvider v-if="props.isAdmin" class="hidden sm:block">
<!-- Also hidden when HID backend is CH9329 (no USB gadget support) -->
<TooltipProvider v-if="showMsd" class="hidden sm:block">
<Tooltip>
<TooltipTrigger as-child>
<Button variant="ghost" size="sm" class="h-8 gap-1.5 text-xs" @click="msdDialogOpen = true">
@@ -253,8 +261,8 @@ const extensionOpen = ref(false)
</Button>
</DropdownMenuTrigger>
<DropdownMenuContent align="end" class="w-48">
<!-- MSD - Mobile only -->
<DropdownMenuItem v-if="props.isAdmin" class="sm:hidden" @click="msdDialogOpen = true; overflowMenuOpen = false">
<!-- MSD - Mobile only, hidden when CH9329 backend -->
<DropdownMenuItem v-if="showMsd" class="sm:hidden" @click="msdDialogOpen = true; overflowMenuOpen = false">
<HardDrive class="h-4 w-4 mr-2" />
{{ t('actionbar.virtualMedia') }}
</DropdownMenuItem>

View File

@@ -52,9 +52,9 @@ async function handleLogout() {
</script>
<template>
<div class="min-h-screen bg-background">
<div class="h-screen flex flex-col bg-background overflow-hidden">
<!-- Header -->
<header class="sticky top-0 z-50 w-full border-b bg-background/95 backdrop-blur supports-[backdrop-filter]:bg-background/60">
<header class="shrink-0 z-50 w-full border-b bg-background/95 backdrop-blur supports-[backdrop-filter]:bg-background/60">
<div class="flex h-14 items-center px-4 max-w-full">
<!-- Logo -->
<RouterLink to="/" class="flex items-center gap-2 font-semibold">
@@ -128,7 +128,7 @@ async function handleLogout() {
</header>
<!-- Main Content -->
<main class="px-4 py-6 max-w-full">
<main class="flex-1 overflow-hidden">
<slot />
</main>
</div>

View File

@@ -25,7 +25,7 @@ const networkErrorMessage = ref<string | null>(null)
let reconnectTimeout: number | null = null
const hidUnavailable = ref(false) // Track if HID is unavailable to prevent unnecessary reconnects
// Mouse throttle mechanism
// Mouse throttle mechanism (10ms = 100Hz for smoother cursor movement)
let mouseThrottleMs = 10
let lastMouseSendTime = 0
let pendingMouseEvent: HidMouseEvent | null = null
@@ -183,36 +183,40 @@ function _sendMouseInternal(event: HidMouseEvent): Promise<void> {
}
// Throttled mouse event sender
// Note: Returns immediately for throttled events to avoid Promise memory leak.
// When an event is throttled, we store it as pending and resolve immediately.
// A timer will send the pending event later, but that's fire-and-forget.
function sendMouse(event: HidMouseEvent): Promise<void> {
return new Promise((resolve, reject) => {
const now = Date.now()
const elapsed = now - lastMouseSendTime
const now = Date.now()
const elapsed = now - lastMouseSendTime
if (elapsed >= mouseThrottleMs) {
// Send immediately if enough time has passed
lastMouseSendTime = now
_sendMouseInternal(event).then(resolve).catch(reject)
} else {
// Queue the event and send after throttle period
pendingMouseEvent = event
if (elapsed >= mouseThrottleMs) {
// Send immediately if enough time has passed
lastMouseSendTime = now
return _sendMouseInternal(event)
} else {
// Throttle: store event for later, resolve immediately to avoid Promise leak
pendingMouseEvent = event
// Clear existing timer
if (throttleTimer !== null) {
clearTimeout(throttleTimer)
}
// Schedule send after remaining throttle time
throttleTimer = window.setTimeout(() => {
if (pendingMouseEvent) {
lastMouseSendTime = Date.now()
_sendMouseInternal(pendingMouseEvent)
.then(resolve)
.catch(reject)
pendingMouseEvent = null
}
}, mouseThrottleMs - elapsed)
// Clear existing timer and set a new one
if (throttleTimer !== null) {
clearTimeout(throttleTimer)
}
})
// Schedule send after remaining throttle time (fire-and-forget)
throttleTimer = window.setTimeout(() => {
if (pendingMouseEvent) {
lastMouseSendTime = Date.now()
_sendMouseInternal(pendingMouseEvent).catch(() => {
// Silently ignore errors for throttled events
})
pendingMouseEvent = null
}
}, mouseThrottleMs - elapsed)
// Resolve immediately - the event is queued, caller doesn't need to wait
return Promise.resolve()
}
}
// Send consumer control event (multimedia keys)

View File

@@ -83,6 +83,7 @@ let sessionId: string | null = null
let statsInterval: number | null = null
let isConnecting = false // Lock to prevent concurrent connect calls
let pendingIceCandidates: RTCIceCandidate[] = [] // Queue for ICE candidates before sessionId is set
let cachedMediaStream: MediaStream | null = null // Cached MediaStream to avoid recreating
const state = ref<WebRTCState>('disconnected')
const videoTrack = ref<MediaStreamTrack | null>(null)
@@ -399,8 +400,28 @@ async function connect(): Promise<boolean> {
}
}
isConnecting = false
return true
// 等待连接真正建立(最多等待 15 秒)
// 直接检查 peerConnection.connectionState 而不是 reactive state
// 因为 TypeScript 不知道 state 会被 onconnectionstatechange 回调异步修改
const connectionTimeout = 15000
const pollInterval = 100
let waited = 0
while (waited < connectionTimeout && peerConnection) {
const pcState = peerConnection.connectionState
if (pcState === 'connected') {
isConnecting = false
return true
}
if (pcState === 'failed' || pcState === 'closed') {
throw new Error('Connection failed during ICE negotiation')
}
await new Promise(resolve => setTimeout(resolve, pollInterval))
waited += pollInterval
}
// 超时
throw new Error('Connection timeout waiting for ICE negotiation')
} catch (err) {
state.value = 'failed'
error.value = err instanceof Error ? err.message : 'Connection failed'
@@ -441,6 +462,7 @@ async function disconnect() {
videoTrack.value = null
audioTrack.value = null
cachedMediaStream = null // Clear cached stream on disconnect
state.value = 'disconnected'
error.value = null
@@ -493,20 +515,49 @@ function sendMouse(event: HidMouseEvent): boolean {
}
}
// Get MediaStream for video element
// Get MediaStream for video element (cached to avoid recreating)
function getMediaStream(): MediaStream | null {
if (!videoTrack.value && !audioTrack.value) {
return null
}
const stream = new MediaStream()
// Reuse cached stream if tracks match
if (cachedMediaStream) {
const currentVideoTracks = cachedMediaStream.getVideoTracks()
const currentAudioTracks = cachedMediaStream.getAudioTracks()
const videoMatches = videoTrack.value
? currentVideoTracks.includes(videoTrack.value)
: currentVideoTracks.length === 0
const audioMatches = audioTrack.value
? currentAudioTracks.includes(audioTrack.value)
: currentAudioTracks.length === 0
if (videoMatches && audioMatches) {
return cachedMediaStream
}
// Tracks changed, update the cached stream
// Remove old tracks
currentVideoTracks.forEach(t => cachedMediaStream!.removeTrack(t))
currentAudioTracks.forEach(t => cachedMediaStream!.removeTrack(t))
// Add new tracks
if (videoTrack.value) cachedMediaStream.addTrack(videoTrack.value)
if (audioTrack.value) cachedMediaStream.addTrack(audioTrack.value)
return cachedMediaStream
}
// Create new cached stream
cachedMediaStream = new MediaStream()
if (videoTrack.value) {
stream.addTrack(videoTrack.value)
cachedMediaStream.addTrack(videoTrack.value)
}
if (audioTrack.value) {
stream.addTrack(audioTrack.value)
cachedMediaStream.addTrack(audioTrack.value)
}
return stream
return cachedMediaStream
}
// Composable export

View File

@@ -283,12 +283,12 @@ export default {
fullscreen: 'Fullscreen',
exitFullscreen: 'Exit Fullscreen',
screenshot: 'Screenshot',
reconnect: 'Reconnect',
reconnect: 'Refresh Page',
noVideo: 'No video signal',
connecting: 'Connecting...',
streamOffline: 'Stream offline',
connectionFailed: 'Connection Failed',
connectionFailedDesc: 'Unable to connect to video stream, please check device status',
connectionFailedDesc: 'Unable to connect to video stream, please refresh page or check device status',
videoRestarting: 'Video stream is restarting',
deviceSwitching: 'Switching video device...',
configChanging: 'Applying new configuration...',
@@ -570,16 +570,18 @@ export default {
// WebRTC / ICE
webrtcSettings: 'WebRTC Settings',
webrtcSettingsDesc: 'Configure STUN/TURN servers for NAT traversal',
usingPublicIceServers: 'Using public ICE servers',
publicIceServersHint: 'Leave empty to use built-in public STUN/TURN servers for NAT traversal',
stunServer: 'STUN Server',
stunServerPlaceholder: 'stun:stun.l.google.com:19302',
stunServerHint: 'STUN server for NAT traversal (e.g., stun:stun.l.google.com:19302)',
stunServerHint: 'Custom STUN server (leave empty to use public server)',
turnServer: 'TURN Server',
turnServerPlaceholder: 'turn:turn.example.com:3478',
turnServerHint: 'TURN relay server for restrictive networks (optional)',
turnServerHint: 'Custom TURN relay server (leave empty to use public server)',
turnUsername: 'TURN Username',
turnPassword: 'TURN Password',
turnPasswordConfigured: 'Password already configured. Leave empty to keep current password.',
turnCredentialsHint: 'Credentials for TURN server authentication',
turnCredentialsHint: 'Credentials for TURN server authentication (only needed for custom servers)',
iceConfigNote: 'Note: Changes require reconnecting the WebRTC session to take effect.',
},
virtualKeyboard: {
@@ -628,6 +630,7 @@ export default {
absolute: 'Absolute',
relative: 'Relative',
connection: 'Connection',
channel: 'Channel',
networkError: 'Network Error',
disconnected: 'Disconnected',
availability: 'Availability',
@@ -637,6 +640,7 @@ export default {
quality: 'Quality',
streaming: 'Streaming',
off: 'Off',
defaultDevice: 'Default',
notConnected: 'Not Connected',
connected: 'Connected',
image: 'Image',

View File

@@ -283,12 +283,12 @@ export default {
fullscreen: '全屏',
exitFullscreen: '退出全屏',
screenshot: '截图',
reconnect: '重新连接',
reconnect: '刷新网页',
noVideo: '无视频信号',
connecting: '正在连接...',
streamOffline: '视频流离线',
connectionFailed: '连接失败',
connectionFailedDesc: '无法连接到视频流,请检查设备状态',
connectionFailedDesc: '无法连接到视频流,请刷新网页或检查设备状态',
videoRestarting: '视频流正在重启',
deviceSwitching: '正在切换视频设备...',
configChanging: '正在应用新配置...',
@@ -570,16 +570,18 @@ export default {
// WebRTC / ICE
webrtcSettings: 'WebRTC 设置',
webrtcSettingsDesc: '配置 STUN/TURN 服务器以实现 NAT 穿透',
usingPublicIceServers: '正在使用公共 ICE 服务器',
publicIceServersHint: '留空以使用内置的公共 STUN/TURN 服务器进行 NAT 穿透',
stunServer: 'STUN 服务器',
stunServerPlaceholder: 'stun:stun.l.google.com:19302',
stunServerHint: '用于 NAT 穿透的 STUN 服务器例如stun:stun.l.google.com:19302',
stunServerHint: '自定义 STUN 服务器(留空则使用公共服务器',
turnServer: 'TURN 服务器',
turnServerPlaceholder: 'turn:turn.example.com:3478',
turnServerHint: '用于限制性网络的 TURN 中继服务器(可选',
turnServerHint: '自定义 TURN 中继服务器(留空则使用公共服务器',
turnUsername: 'TURN 用户名',
turnPassword: 'TURN 密码',
turnPasswordConfigured: '密码已配置。留空则保持当前密码。',
turnCredentialsHint: '用于 TURN 服务器认证的凭据',
turnCredentialsHint: '用于 TURN 服务器认证的凭据(仅自定义服务器需要)',
iceConfigNote: '注意:更改后需要重新连接 WebRTC 会话才能生效。',
},
virtualKeyboard: {
@@ -628,6 +630,7 @@ export default {
absolute: '绝对定位',
relative: '相对定位',
connection: '连接',
channel: '通道',
networkError: '网络错误',
disconnected: '已断开',
availability: '可用性',
@@ -637,6 +640,7 @@ export default {
quality: '质量',
streaming: '传输中',
off: '关闭',
defaultDevice: '默认',
notConnected: '未连接',
connected: '已连接',
image: '镜像',

View File

@@ -232,9 +232,15 @@ export interface StreamConfig {
encoder: EncoderType;
/** Bitrate preset (Speed/Balanced/Quality) */
bitrate_preset: BitratePreset;
/** Custom STUN server (e.g., "stun:stun.l.google.com:19302") */
/**
* Custom STUN server (e.g., "stun:stun.l.google.com:19302")
* If empty, uses public ICE servers from secrets.toml
*/
stun_server?: string;
/** Custom TURN server (e.g., "turn:turn.example.com:3478") */
/**
* Custom TURN server (e.g., "turn:turn.example.com:3478")
* If empty, uses public ICE servers from secrets.toml
*/
turn_server?: string;
/** TURN username */
turn_username?: string;
@@ -532,6 +538,10 @@ export interface StreamConfigResponse {
mode: StreamMode;
encoder: EncoderType;
bitrate_preset: BitratePreset;
/** 是否有公共 ICE 服务器可用(编译时确定) */
has_public_ice_servers: boolean;
/** 当前是否正在使用公共 ICE 服务器STUN/TURN 都为空时) */
using_public_ice_servers: boolean;
stun_server?: string;
turn_server?: string;
turn_username?: string;
@@ -543,9 +553,15 @@ export interface StreamConfigUpdate {
mode?: StreamMode;
encoder?: EncoderType;
bitrate_preset?: BitratePreset;
/** STUN server URL (e.g., "stun:stun.l.google.com:19302") */
/**
* STUN server URL (e.g., "stun:stun.l.google.com:19302")
* Leave empty to use public ICE servers
*/
stun_server?: string;
/** TURN server URL (e.g., "turn:turn.example.com:3478") */
/**
* TURN server URL (e.g., "turn:turn.example.com:3478")
* Leave empty to use public ICE servers
*/
turn_server?: string;
/** TURN username */
turn_username?: string;

View File

@@ -1,5 +1,5 @@
<script setup lang="ts">
import { ref, onMounted, onUnmounted, computed, watch } from 'vue'
import { ref, onMounted, onUnmounted, computed, watch, nextTick } from 'vue'
import { useI18n } from 'vue-i18n'
import { useRouter } from 'vue-router'
import { useSystemStore } from '@/stores/system'
@@ -9,6 +9,7 @@ import { useHidWebSocket } from '@/composables/useHidWebSocket'
import { useWebRTC } from '@/composables/useWebRTC'
import { getUnifiedAudio } from '@/composables/useUnifiedAudio'
import { streamApi, hidApi, atxApi, extensionsApi, atxConfigApi, userApi } from '@/api'
import type { HidKeyboardEvent, HidMouseEvent } from '@/types/hid'
import { toast } from 'vue-sonner'
import { generateUUID } from '@/lib/utils'
import type { VideoMode } from '@/components/VideoConfigPopover.vue'
@@ -186,6 +187,18 @@ const videoDetails = computed<StatusDetail[]>(() => {
})
const hidStatus = computed<'connected' | 'connecting' | 'disconnected' | 'error'>(() => {
// In WebRTC mode, check DataChannel status first
if (videoMode.value !== 'mjpeg') {
// DataChannel is ready - HID is connected via WebRTC
if (webrtc.dataChannelReady.value) return 'connected'
// WebRTC is connecting - HID is also connecting
if (webrtc.isConnecting.value) return 'connecting'
// WebRTC is connected but DataChannel not ready - still connecting
if (webrtc.isConnected.value) return 'connecting'
// WebRTC not connected - fall through to WebSocket check as fallback
}
// MJPEG mode or WebRTC fallback: check WebSocket HID status
// If HID WebSocket has network error, show connecting (yellow)
if (hidWs.networkError.value) return 'connecting'
@@ -221,13 +234,31 @@ const hidDetails = computed<StatusDetail[]>(() => {
{ label: t('statusCard.currentMode'), value: mouseMode.value === 'absolute' ? t('statusCard.absolute') : t('statusCard.relative'), status: 'ok' },
]
// Add connection status
if (hidWs.networkError.value) {
details.push({ label: t('statusCard.connection'), value: t('statusCard.networkError'), status: 'warning' })
} else if (!hidWs.connected.value) {
details.push({ label: t('statusCard.connection'), value: t('statusCard.disconnected'), status: 'warning' })
} else if (hidWs.hidUnavailable.value) {
details.push({ label: t('statusCard.availability'), value: t('statusCard.hidUnavailable'), status: 'warning' })
// Add HID channel info based on video mode
if (videoMode.value !== 'mjpeg') {
// WebRTC mode - show DataChannel status
if (webrtc.dataChannelReady.value) {
details.push({ label: t('statusCard.channel'), value: 'DataChannel (WebRTC)', status: 'ok' })
} else if (webrtc.isConnecting.value || webrtc.isConnected.value) {
details.push({ label: t('statusCard.channel'), value: 'DataChannel', status: 'warning' })
} else {
// Fallback to WebSocket
details.push({ label: t('statusCard.channel'), value: 'WebSocket (fallback)', status: hidWs.connected.value ? 'ok' : 'warning' })
}
} else {
// MJPEG mode - WebSocket HID
details.push({ label: t('statusCard.channel'), value: 'WebSocket', status: hidWs.connected.value ? 'ok' : 'warning' })
}
// Add connection status for WebSocket (only relevant for MJPEG or fallback)
if (videoMode.value === 'mjpeg' || !webrtc.dataChannelReady.value) {
if (hidWs.networkError.value) {
details.push({ label: t('statusCard.connection'), value: t('statusCard.networkError'), status: 'warning' })
} else if (!hidWs.connected.value) {
details.push({ label: t('statusCard.connection'), value: t('statusCard.disconnected'), status: 'warning' })
} else if (hidWs.hidUnavailable.value) {
details.push({ label: t('statusCard.availability'), value: t('statusCard.hidUnavailable'), status: 'warning' })
}
}
return details
@@ -242,10 +273,20 @@ const audioStatus = computed<'connected' | 'connecting' | 'disconnected' | 'erro
return 'disconnected'
})
// Helper function to translate audio quality
function translateAudioQuality(quality: string | undefined): string {
if (!quality) return t('common.unknown')
const qualityLower = quality.toLowerCase()
if (qualityLower === 'voice') return t('actionbar.qualityVoice')
if (qualityLower === 'balanced') return t('actionbar.qualityBalanced')
if (qualityLower === 'high') return t('actionbar.qualityHigh')
return quality // fallback to original value
}
const audioQuickInfo = computed(() => {
const audio = systemStore.audio
if (!audio?.available) return ''
if (audio.streaming) return audio.quality
if (audio.streaming) return translateAudioQuality(audio.quality)
return t('statusCard.off')
})
@@ -258,8 +299,8 @@ const audioDetails = computed<StatusDetail[]>(() => {
if (!audio) return []
return [
{ label: t('statusCard.device'), value: audio.device || 'default' },
{ label: t('statusCard.quality'), value: audio.quality },
{ label: t('statusCard.device'), value: audio.device || t('statusCard.defaultDevice') },
{ label: t('statusCard.quality'), value: translateAudioQuality(audio.quality) },
{ label: t('statusCard.streaming'), value: audio.streaming ? t('statusCard.yes') : t('statusCard.no'), status: audio.streaming ? 'ok' : undefined },
]
})
@@ -387,6 +428,11 @@ function handleVideoError() {
return
}
// 如果正在切换模式,忽略错误(可能是 503 错误,因为后端已切换模式)
if (isModeSwitching.value) {
return
}
// 如果正在刷新视频,忽略清空 src 时触发的错误
if (isRefreshingVideo) {
return
@@ -676,6 +722,12 @@ function handleStreamConfigApplied(data: any) {
// Refresh video based on current mode
videoRestarting.value = false
// 如果正在进行模式切换不需要在这里处理WebRTCReady 事件会处理)
if (isModeSwitching.value) {
console.log('[StreamConfigApplied] Mode switch in progress, waiting for WebRTCReady')
return
}
if (videoMode.value !== 'mjpeg') {
// In WebRTC mode, reconnect WebRTC (session was closed due to config change)
switchToWebRTC(videoMode.value)
@@ -690,6 +742,17 @@ function handleStreamConfigApplied(data: any) {
})
}
// 处理 WebRTC 就绪事件 - 这是后端真正准备好接受 WebRTC 连接的信号
function handleWebRTCReady(data: { codec: string; hardware: boolean }) {
console.log(`[WebRTCReady] Backend ready: codec=${data.codec}, hardware=${data.hardware}`)
// 如果正在进行模式切换,标记后端已就绪
if (isModeSwitching.value) {
console.log('[WebRTCReady] Signaling backend ready for WebRTC connection')
backendReadyForWebRTC = true
}
}
function handleStreamStateChanged(data: any) {
if (data.state === 'error') {
videoError.value = true
@@ -778,7 +841,13 @@ function handleStreamModeChanged(data: { mode: string; previous_mode: string })
// Server returns: 'mjpeg', 'h264', 'h265', 'vp8', 'vp9', or 'webrtc'
const newMode = data.mode === 'webrtc' ? 'h264' : data.mode as VideoMode
// Show toast notification
// 如果正在进行模式切换,忽略这个事件(这是我们自己触发的切换产生的)
if (isModeSwitching.value) {
console.log('[StreamModeChanged] Mode switch in progress, ignoring event')
return
}
// Show toast notification only if this is an external mode change
toast.info(t('console.streamModeChanged'), {
description: t('console.streamModeChangedDesc', { mode: data.mode.toUpperCase() }),
duration: 5000,
@@ -792,6 +861,14 @@ function handleStreamModeChanged(data: { mode: string; previous_mode: string })
// 标记是否正在刷新视频(用于忽略清空 src 时触发的 error 事件)
let isRefreshingVideo = false
// 标记是否正在切换模式(防止竞态条件和 503 错误)
const isModeSwitching = ref(false)
// 标记后端是否已准备好接受 WebRTC 连接(由 StreamConfigApplied 事件设置)
let backendReadyForWebRTC = false
function reloadPage() {
window.location.reload()
}
function refreshVideo() {
backendFps.value = 0
@@ -845,6 +922,7 @@ async function connectWebRTCOnly(codec: VideoMode = 'h264') {
mjpegTimestamp.value = 0
if (videoRef.value) {
videoRef.value.src = ''
videoRef.value.removeAttribute('src')
}
videoLoading.value = true
@@ -859,18 +937,9 @@ async function connectWebRTCOnly(codec: VideoMode = 'h264') {
duration: 3000,
})
// Try to attach video immediately in case track is already available
if (webrtc.videoTrack.value && webrtcVideoRef.value) {
const stream = webrtc.getMediaStream()
if (stream) {
webrtcVideoRef.value.srcObject = stream
try {
await webrtcVideoRef.value.play()
} catch {
// AbortError is expected when switching modes quickly, ignore it
}
}
}
// 强制重新绑定视频(即使 track 已存在)
// 这解决了页面返回时视频不显示的问题
await rebindWebRTCVideo()
videoLoading.value = false
videoMode.value = codec
@@ -885,6 +954,28 @@ async function connectWebRTCOnly(codec: VideoMode = 'h264') {
}
}
// 强制重新绑定 WebRTC 视频到视频元素
// 解决页面切换后视频不显示的问题
async function rebindWebRTCVideo() {
if (!webrtcVideoRef.value) return
// 先清空再重新绑定,确保浏览器重新渲染
webrtcVideoRef.value.srcObject = null
await nextTick()
if (webrtc.videoTrack.value) {
const stream = webrtc.getMediaStream()
if (stream) {
webrtcVideoRef.value.srcObject = stream
try {
await webrtcVideoRef.value.play()
} catch {
// AbortError is expected when switching modes quickly, ignore it
}
}
}
}
// WebRTC video mode handling (switches server mode)
async function switchToWebRTC(codec: VideoMode = 'h264') {
// 清除 MJPEG 相关的定时器,防止切换后重新加载 MJPEG
@@ -918,29 +1009,48 @@ async function switchToWebRTC(codec: VideoMode = 'h264') {
}
// Step 2: Call backend API to switch mode with specific codec
// 重置就绪标志
backendReadyForWebRTC = false
await streamApi.setMode(codec)
// Step 3: Connect WebRTC with new codec
const success = await webrtc.connect()
// Step 3: 等待后端完成格式切换(由 StreamConfigApplied 事件触发)
// 后端需要时间来:停止捕获 → 切换格式 → 重启捕获 → 连接 frame source
// 使用轮询等待,最多等待 3 秒
const maxWaitTime = 3000
const pollInterval = 100
let waited = 0
while (!backendReadyForWebRTC && waited < maxWaitTime) {
await new Promise(resolve => setTimeout(resolve, pollInterval))
waited += pollInterval
}
if (!backendReadyForWebRTC) {
console.warn('[WebRTC] Backend not ready after timeout, attempting connection anyway')
} else {
console.log('[WebRTC] Backend ready signal received, connecting')
}
// Step 4: Connect WebRTC with retry
let retries = 3
let success = false
while (retries > 0 && !success) {
success = await webrtc.connect()
if (!success) {
retries--
if (retries > 0) {
console.log(`[WebRTC] Connection failed, retrying (${retries} attempts left)`)
await new Promise(resolve => setTimeout(resolve, 500))
}
}
}
if (success) {
toast.success(t('console.webrtcConnected'), {
description: t('console.webrtcConnectedDesc'),
duration: 3000,
})
// Video will be attached by the watch on webrtc.videoTrack
// But also try to attach immediately in case track is already available
if (webrtc.videoTrack.value && webrtcVideoRef.value) {
const stream = webrtc.getMediaStream()
if (stream) {
webrtcVideoRef.value.srcObject = stream
try {
await webrtcVideoRef.value.play()
} catch {
// AbortError is expected when switching modes quickly, ignore it
}
}
}
// 强制重新绑定视频
await rebindWebRTCVideo()
videoLoading.value = false
@@ -995,40 +1105,49 @@ async function switchToMJPEG() {
}
// Handle video mode change
function handleVideoModeChange(mode: VideoMode) {
async function handleVideoModeChange(mode: VideoMode) {
// 防止重复切换和竞态条件
if (mode === videoMode.value) return
// Reset mjpegTimestamp to 0 when switching away from MJPEG
// This prevents mjpegUrl from returning a valid URL and stops MJPEG requests
if (mode !== 'mjpeg') {
mjpegTimestamp.value = 0
if (isModeSwitching.value) {
console.log('[VideoMode] Switch already in progress, ignoring')
return
}
videoMode.value = mode
localStorage.setItem('videoMode', mode)
isModeSwitching.value = true
// All WebRTC modes: h264, h265, vp8, vp9
if (mode !== 'mjpeg') {
switchToWebRTC(mode)
} else {
switchToMJPEG()
try {
// Reset mjpegTimestamp to 0 when switching away from MJPEG
// This prevents mjpegUrl from returning a valid URL and stops MJPEG requests
if (mode !== 'mjpeg') {
mjpegTimestamp.value = 0
// 完全清理 MJPEG 图片元素
if (videoRef.value) {
videoRef.value.src = ''
videoRef.value.removeAttribute('src')
}
// 等待一小段时间确保浏览器取消 pending 请求
await new Promise(resolve => setTimeout(resolve, 50))
}
videoMode.value = mode
localStorage.setItem('videoMode', mode)
// All WebRTC modes: h264, h265, vp8, vp9
if (mode !== 'mjpeg') {
await switchToWebRTC(mode)
} else {
await switchToMJPEG()
}
} finally {
isModeSwitching.value = false
}
}
// Watch for WebRTC video track changes
watch(() => webrtc.videoTrack.value, async (track) => {
if (track && webrtcVideoRef.value && videoMode.value !== 'mjpeg') {
const stream = webrtc.getMediaStream()
if (stream) {
webrtcVideoRef.value.srcObject = stream
try {
await webrtcVideoRef.value.play()
} catch {
// AbortError is expected when switching modes quickly, ignore it
}
}
// 使用统一的重新绑定函数
await rebindWebRTCVideo()
}
})
@@ -1232,6 +1351,41 @@ function handleHidError(_error: any, _operation: string) {
// All HID errors are silently ignored
}
// HID channel selection: use WebRTC DataChannel when available, fallback to WebSocket
function sendKeyboardEvent(type: 'down' | 'up', key: number, modifiers?: { ctrl?: boolean; shift?: boolean; alt?: boolean; meta?: boolean }) {
// In WebRTC mode with DataChannel ready, use DataChannel for lower latency
if (videoMode.value !== 'mjpeg' && webrtc.dataChannelReady.value) {
const event: HidKeyboardEvent = {
type: type === 'down' ? 'keydown' : 'keyup',
key,
modifiers,
}
const sent = webrtc.sendKeyboard(event)
if (sent) return
// Fallback to WebSocket if DataChannel send failed
}
// Use WebSocket as fallback or for MJPEG mode
hidApi.keyboard(type, key, modifiers).catch(err => handleHidError(err, `keyboard ${type}`))
}
function sendMouseEvent(data: { type: 'move' | 'move_abs' | 'down' | 'up' | 'scroll'; x?: number; y?: number; button?: 'left' | 'right' | 'middle'; scroll?: number }) {
// In WebRTC mode with DataChannel ready, use DataChannel for lower latency
if (videoMode.value !== 'mjpeg' && webrtc.dataChannelReady.value) {
const event: HidMouseEvent = {
type: data.type === 'move_abs' ? 'moveabs' : data.type,
x: data.x,
y: data.y,
button: data.button === 'left' ? 0 : data.button === 'middle' ? 1 : data.button === 'right' ? 2 : undefined,
scroll: data.scroll,
}
const sent = webrtc.sendMouse(event)
if (sent) return
// Fallback to WebSocket if DataChannel send failed
}
// Use WebSocket as fallback or for MJPEG mode
hidApi.mouse(data).catch(err => handleHidError(err, `mouse ${data.type}`))
}
// Check if a key should be blocked (prevented from default behavior)
function shouldBlockKey(e: KeyboardEvent): boolean {
// In fullscreen mode, block all keys for maximum capture
@@ -1291,7 +1445,7 @@ function handleKeyDown(e: KeyboardEvent) {
meta: e.metaKey,
}
hidApi.keyboard('down', e.keyCode, modifiers).catch(err => handleHidError(err, 'keyboard down'))
sendKeyboardEvent('down', e.keyCode, modifiers)
}
function handleKeyUp(e: KeyboardEvent) {
@@ -1310,7 +1464,7 @@ function handleKeyUp(e: KeyboardEvent) {
const keyName = e.key === ' ' ? 'Space' : e.key
pressedKeys.value = pressedKeys.value.filter(k => k !== keyName)
hidApi.keyboard('up', e.keyCode).catch(err => handleHidError(err, 'keyboard up'))
sendKeyboardEvent('up', e.keyCode)
}
function handleMouseMove(e: MouseEvent) {
@@ -1325,7 +1479,7 @@ function handleMouseMove(e: MouseEvent) {
const y = Math.round((e.clientY - rect.top) / rect.height * 32767)
mousePosition.value = { x, y }
hidApi.mouse({ type: 'move_abs', x, y }).catch(err => handleHidError(err, 'mouse move'))
sendMouseEvent({ type: 'move_abs', x, y })
} else {
// Relative mode: use movementX/Y when pointer is locked
if (isPointerLocked.value) {
@@ -1338,7 +1492,7 @@ function handleMouseMove(e: MouseEvent) {
const clampedDx = Math.max(-127, Math.min(127, dx))
const clampedDy = Math.max(-127, Math.min(127, dy))
hidApi.mouse({ type: 'move', x: clampedDx, y: clampedDy }).catch(err => handleHidError(err, 'mouse move'))
sendMouseEvent({ type: 'move', x: clampedDx, y: clampedDy })
}
// Update display position (accumulated delta for display only)
@@ -1372,7 +1526,7 @@ function handleMouseDown(e: MouseEvent) {
const button = e.button === 0 ? 'left' : e.button === 2 ? 'right' : 'middle'
pressedMouseButton.value = button
hidApi.mouse({ type: 'down', button }).catch(err => handleHidError(err, 'mouse down'))
sendMouseEvent({ type: 'down', button })
}
function handleMouseUp(e: MouseEvent) {
@@ -1401,13 +1555,13 @@ function handleMouseUpInternal(rawButton: number) {
}
pressedMouseButton.value = null
hidApi.mouse({ type: 'up', button }).catch(err => handleHidError(err, 'mouse up'))
sendMouseEvent({ type: 'up', button })
}
function handleWheel(e: WheelEvent) {
e.preventDefault()
const scroll = e.deltaY > 0 ? -1 : 1
hidApi.mouse({ type: 'scroll', scroll }).catch(err => handleHidError(err, 'mouse scroll'))
sendMouseEvent({ type: 'scroll', scroll })
}
function handleContextMenu(e: MouseEvent) {
@@ -1456,7 +1610,7 @@ function handleBlur() {
if (pressedMouseButton.value !== null) {
const button = pressedMouseButton.value
pressedMouseButton.value = null
hidApi.mouse({ type: 'up', button }).catch(err => handleHidError(err, 'mouse up (blur)'))
sendMouseEvent({ type: 'up', button })
}
}
@@ -1514,6 +1668,7 @@ onMounted(async () => {
// 1. 先注册 WebSocket 事件监听器
on('stream.config_changing', handleStreamConfigChanging)
on('stream.config_applied', handleStreamConfigApplied)
on('stream.webrtc_ready', handleWebRTCReady)
on('stream.state_changed', handleStreamStateChanged)
on('stream.stats_update', handleStreamStatsUpdate)
on('stream.mode_changed', handleStreamModeChanged)
@@ -1613,6 +1768,7 @@ onUnmounted(() => {
// Unregister WebSocket event handlers
off('stream.config_changing', handleStreamConfigChanging)
off('stream.config_applied', handleStreamConfigApplied)
off('stream.webrtc_ready', handleWebRTCReady)
off('stream.state_changed', handleStreamStateChanged)
off('stream.stats_update', handleStreamStatsUpdate)
off('stream.mode_changed', handleStreamModeChanged)
@@ -1646,6 +1802,7 @@ onUnmounted(() => {
// Remove WebSocket event listeners
off('stream.config_changing', handleStreamConfigChanging)
off('stream.config_applied', handleStreamConfigApplied)
off('stream.webrtc_ready', handleWebRTCReady)
off('stream.state_changed', handleStreamStateChanged)
off('stream.stats_update', handleStreamStatsUpdate)
off('stream.mode_changed', handleStreamModeChanged)
@@ -1710,9 +1867,9 @@ onUnmounted(() => {
:details="hidDetails"
/>
<!-- MSD Status - Admin only -->
<!-- MSD Status - Admin only, hidden when CH9329 backend (no USB gadget support) -->
<StatusCard
v-if="authStore.isAdmin && systemStore.msd?.available"
v-if="authStore.isAdmin && systemStore.msd?.available && systemStore.hid?.backend !== 'ch9329'"
:title="t('statusCard.msd')"
type="msd"
:status="msdStatus"
@@ -1882,7 +2039,7 @@ onUnmounted(() => {
</div>
</div>
<div class="flex gap-2">
<Button variant="secondary" size="sm" @click="refreshVideo">
<Button variant="secondary" size="sm" @click="reloadPage">
<RefreshCw class="h-4 w-4 mr-2" />
{{ t('console.reconnect') }}
</Button>

View File

@@ -1,5 +1,5 @@
<script setup lang="ts">
import { ref, onMounted } from 'vue'
import { ref, computed, onMounted } from 'vue'
import { useI18n } from 'vue-i18n'
import { useSystemStore } from '@/stores/system'
import {
@@ -252,6 +252,12 @@ const config = ref({
// 跟踪服务器是否已配置 TURN 密码
const hasTurnPassword = ref(false)
// 跟踪公共 ICE 服务器状态
const hasPublicIceServers = ref(false)
const usingPublicIceServers = computed(() => {
return !config.value.stun_server && !config.value.turn_server && hasPublicIceServers.value
})
// OTG Descriptor settings
const otgVendorIdHex = ref('1d6b')
const otgProductIdHex = ref('0104')
@@ -305,7 +311,7 @@ const selectedBackendFormats = computed(() => {
})
// Video selection computed properties
import { computed, watch } from 'vue'
import { watch } from 'vue'
const selectedDevice = computed(() => {
return devices.value.video.find(d => d.path === config.value.video_device)
@@ -555,6 +561,9 @@ async function loadConfig() {
// 设置是否已配置 TURN 密码
hasTurnPassword.value = stream.has_turn_password || false
// 设置公共 ICE 服务器状态
hasPublicIceServers.value = stream.has_public_ice_servers || false
// 加载 OTG 描述符配置
if (hid.otg_descriptor) {
otgVendorIdHex.value = hid.otg_descriptor.vendor_id?.toString(16).padStart(4, '0') || '1d6b'
@@ -1068,7 +1077,7 @@ onMounted(async () => {
<template>
<AppLayout>
<div class="flex h-[calc(100vh-6rem)]">
<div class="flex h-full overflow-hidden">
<!-- Mobile Header -->
<div class="lg:hidden fixed top-16 left-0 right-0 z-20 flex items-center justify-between px-4 py-3 border-b bg-background">
<h1 class="text-lg font-semibold">{{ t('settings.title') }}</h1>
@@ -1259,6 +1268,9 @@ onMounted(async () => {
:placeholder="t('settings.stunServerPlaceholder')"
/>
<p class="text-xs text-muted-foreground">{{ t('settings.stunServerHint') }}</p>
<p v-if="usingPublicIceServers && hasPublicIceServers" class="text-xs text-blue-500">
{{ t('settings.usingPublicIceServers') }}
</p>
</div>
<Separator />
<div class="space-y-2">