feat(video): 事务化切换与前端统一编排,增强视频输入格式支持

- 后端:切换事务+transition_id,/stream/mode 返回 switching/transition_id 与实际 codec

- 事件:新增 mode_switching/mode_ready,config/webrtc_ready/mode_changed 关联事务

- 编码/格式:扩展 NV21/NV16/NV24/RGB/BGR 输入与转换链路,RKMPP direct input 优化

- 前端:useVideoSession 统一切换,失败回退真实切回 MJPEG,菜单格式同步修复

- 清理:useVideoStream 降级为 MJPEG-only
This commit is contained in:
mofeng-git
2026-01-11 10:41:57 +08:00
parent 9feb74b72c
commit 206594e292
110 changed files with 3955 additions and 2251 deletions

View File

@@ -12,6 +12,13 @@ export interface ConsoleEventHandlers {
onStreamConfigApplied?: (data: { device: string; resolution: [number, number]; fps: number }) => void
onStreamStatsUpdate?: (data: { clients?: number; clients_stat?: Record<string, { fps: number }> }) => void
onStreamModeChanged?: (data: { mode: string; previous_mode: string }) => void
onStreamModeSwitching?: (data: { transition_id: string; to_mode: string; from_mode: string }) => void
onStreamModeReady?: (data: { transition_id: string; mode: string }) => void
onWebRTCReady?: (data: { codec: string; hardware: boolean; transition_id?: string }) => void
onStreamStateChanged?: (data: { state: string; device?: string | null }) => void
onStreamDeviceLost?: (data: { device: string; reason: string }) => void
onStreamReconnecting?: (data: { device: string; attempt: number }) => void
onStreamRecovered?: (data: { device: string }) => void
onDeviceInfo?: (data: any) => void
onAudioStateChanged?: (data: { streaming: boolean; device: string | null }) => void
}
@@ -21,6 +28,7 @@ export function useConsoleEvents(handlers: ConsoleEventHandlers) {
const systemStore = useSystemStore()
const { on, off, connect } = useWebSocket()
const unifiedAudio = getUnifiedAudio()
const noop = () => {}
// HID event handlers
function handleHidStateChanged(_data: unknown) {
@@ -68,6 +76,7 @@ export function useConsoleEvents(handlers: ConsoleEventHandlers) {
description: t('console.deviceLostDesc', { device: data.device, reason: data.reason }),
duration: 5000,
})
handlers.onStreamDeviceLost?.(data)
}
function handleStreamReconnecting(data: { device: string; attempt: number }) {
@@ -77,6 +86,7 @@ export function useConsoleEvents(handlers: ConsoleEventHandlers) {
duration: 3000,
})
}
handlers.onStreamReconnecting?.(data)
}
function handleStreamRecovered(_data: { device: string }) {
@@ -87,6 +97,7 @@ export function useConsoleEvents(handlers: ConsoleEventHandlers) {
description: t('console.deviceRecoveredDesc'),
duration: 3000,
})
handlers.onStreamRecovered?.(_data)
}
function handleStreamStateChanged(data: { state: string }) {
@@ -95,6 +106,11 @@ export function useConsoleEvents(handlers: ConsoleEventHandlers) {
}
}
function handleStreamStateChangedForward(data: { state: string; device?: string | null }) {
handleStreamStateChanged(data)
handlers.onStreamStateChanged?.(data)
}
// Audio device monitoring handlers
function handleAudioDeviceLost(data: { device?: string; reason: string; error_code: string }) {
if (systemStore.audio) {
@@ -183,11 +199,14 @@ export function useConsoleEvents(handlers: ConsoleEventHandlers) {
on('hid.recovered', handleHidRecovered)
// Stream events
on('stream.config_changing', handlers.onStreamConfigChanging || (() => {}))
on('stream.config_applied', handlers.onStreamConfigApplied || (() => {}))
on('stream.stats_update', handlers.onStreamStatsUpdate || (() => {}))
on('stream.mode_changed', handlers.onStreamModeChanged || (() => {}))
on('stream.state_changed', handleStreamStateChanged)
on('stream.config_changing', handlers.onStreamConfigChanging ?? noop)
on('stream.config_applied', handlers.onStreamConfigApplied ?? noop)
on('stream.stats_update', handlers.onStreamStatsUpdate ?? noop)
on('stream.mode_changed', handlers.onStreamModeChanged ?? noop)
on('stream.mode_switching', handlers.onStreamModeSwitching ?? noop)
on('stream.mode_ready', handlers.onStreamModeReady ?? noop)
on('stream.webrtc_ready', handlers.onWebRTCReady ?? noop)
on('stream.state_changed', handleStreamStateChangedForward)
on('stream.device_lost', handleStreamDeviceLost)
on('stream.reconnecting', handleStreamReconnecting)
on('stream.recovered', handleStreamRecovered)
@@ -206,7 +225,7 @@ export function useConsoleEvents(handlers: ConsoleEventHandlers) {
on('msd.recovered', handleMsdRecovered)
// System events
on('system.device_info', handlers.onDeviceInfo || (() => {}))
on('system.device_info', handlers.onDeviceInfo ?? noop)
// Connect WebSocket
connect()
@@ -219,11 +238,14 @@ export function useConsoleEvents(handlers: ConsoleEventHandlers) {
off('hid.reconnecting', handleHidReconnecting)
off('hid.recovered', handleHidRecovered)
off('stream.config_changing', handlers.onStreamConfigChanging || (() => {}))
off('stream.config_applied', handlers.onStreamConfigApplied || (() => {}))
off('stream.stats_update', handlers.onStreamStatsUpdate || (() => {}))
off('stream.mode_changed', handlers.onStreamModeChanged || (() => {}))
off('stream.state_changed', handleStreamStateChanged)
off('stream.config_changing', handlers.onStreamConfigChanging ?? noop)
off('stream.config_applied', handlers.onStreamConfigApplied ?? noop)
off('stream.stats_update', handlers.onStreamStatsUpdate ?? noop)
off('stream.mode_changed', handlers.onStreamModeChanged ?? noop)
off('stream.mode_switching', handlers.onStreamModeSwitching ?? noop)
off('stream.mode_ready', handlers.onStreamModeReady ?? noop)
off('stream.webrtc_ready', handlers.onWebRTCReady ?? noop)
off('stream.state_changed', handleStreamStateChangedForward)
off('stream.device_lost', handleStreamDeviceLost)
off('stream.reconnecting', handleStreamReconnecting)
off('stream.recovered', handleStreamRecovered)
@@ -239,7 +261,7 @@ export function useConsoleEvents(handlers: ConsoleEventHandlers) {
off('msd.error', handleMsdError)
off('msd.recovered', handleMsdRecovered)
off('system.device_info', handlers.onDeviceInfo || (() => {}))
off('system.device_info', handlers.onDeviceInfo ?? noop)
}
return {

View File

@@ -0,0 +1,185 @@
import { ref } from 'vue'
export interface StreamModeSwitchingEvent {
transition_id: string
to_mode: string
from_mode: string
}
export interface StreamModeReadyEvent {
transition_id: string
mode: string
}
export interface WebRTCReadyEvent {
codec: string
hardware: boolean
transition_id?: string
}
let singleton: ReturnType<typeof createVideoSession> | null = null
function createVideoSession() {
const localSwitching = ref(false)
const backendSwitching = ref(false)
const activeTransitionId = ref<string | null>(null)
const expectedTransitionId = ref<string | null>(null)
let lastUserSwitchAt = 0
let webrtcReadyWaiter: {
transitionId: string
resolve: (ready: boolean) => void
timer: ReturnType<typeof setTimeout>
} | null = null
let modeReadyWaiter: {
transitionId: string
resolve: (mode: string | null) => void
timer: ReturnType<typeof setTimeout>
} | null = null
function startLocalSwitch() {
localSwitching.value = true
}
function tryStartLocalSwitch(minIntervalMs = 800): boolean {
const now = Date.now()
if (localSwitching.value) return false
if (now - lastUserSwitchAt < minIntervalMs) return false
lastUserSwitchAt = now
localSwitching.value = true
return true
}
function endLocalSwitch() {
localSwitching.value = false
}
function clearWaiters() {
if (webrtcReadyWaiter) {
clearTimeout(webrtcReadyWaiter.timer)
webrtcReadyWaiter.resolve(false)
webrtcReadyWaiter = null
}
if (modeReadyWaiter) {
clearTimeout(modeReadyWaiter.timer)
modeReadyWaiter.resolve(null)
modeReadyWaiter = null
}
expectedTransitionId.value = null
}
function registerTransition(transitionId: string) {
expectedTransitionId.value = transitionId
activeTransitionId.value = transitionId
backendSwitching.value = true
}
function isStaleTransition(transitionId?: string): boolean {
if (!transitionId) return false
return expectedTransitionId.value !== null && transitionId !== expectedTransitionId.value
}
function waitForWebRTCReady(transitionId: string, timeoutMs = 3000): Promise<boolean> {
if (webrtcReadyWaiter) {
clearTimeout(webrtcReadyWaiter.timer)
webrtcReadyWaiter.resolve(false)
webrtcReadyWaiter = null
}
return new Promise((resolve) => {
const timer = setTimeout(() => {
if (webrtcReadyWaiter?.transitionId === transitionId) {
webrtcReadyWaiter = null
}
resolve(false)
}, timeoutMs)
webrtcReadyWaiter = {
transitionId,
resolve,
timer,
}
})
}
function waitForModeReady(transitionId: string, timeoutMs = 5000): Promise<string | null> {
if (modeReadyWaiter) {
clearTimeout(modeReadyWaiter.timer)
modeReadyWaiter.resolve(null)
modeReadyWaiter = null
}
return new Promise((resolve) => {
const timer = setTimeout(() => {
if (modeReadyWaiter?.transitionId === transitionId) {
modeReadyWaiter = null
}
resolve(null)
}, timeoutMs)
modeReadyWaiter = {
transitionId,
resolve,
timer,
}
})
}
function onModeSwitching(data: StreamModeSwitchingEvent) {
if (localSwitching.value && expectedTransitionId.value && data.transition_id !== expectedTransitionId.value) {
return
}
backendSwitching.value = true
activeTransitionId.value = data.transition_id
expectedTransitionId.value = data.transition_id
}
function onModeReady(data: StreamModeReadyEvent) {
if (isStaleTransition(data.transition_id)) return
backendSwitching.value = false
activeTransitionId.value = null
expectedTransitionId.value = null
if (modeReadyWaiter?.transitionId === data.transition_id) {
clearTimeout(modeReadyWaiter.timer)
modeReadyWaiter.resolve(data.mode)
modeReadyWaiter = null
}
}
function onWebRTCReady(data: WebRTCReadyEvent) {
if (isStaleTransition(data.transition_id)) return
if (data.transition_id && webrtcReadyWaiter?.transitionId === data.transition_id) {
clearTimeout(webrtcReadyWaiter.timer)
webrtcReadyWaiter.resolve(true)
webrtcReadyWaiter = null
}
}
return {
localSwitching,
backendSwitching,
activeTransitionId,
expectedTransitionId,
startLocalSwitch,
tryStartLocalSwitch,
endLocalSwitch,
clearWaiters,
registerTransition,
waitForWebRTCReady,
waitForModeReady,
onModeSwitching,
onModeReady,
onWebRTCReady,
}
}
export function useVideoSession(): ReturnType<typeof createVideoSession> {
if (!singleton) {
singleton = createVideoSession()
}
return singleton
}

View File

@@ -1,16 +1,12 @@
// Video streaming composable - manages MJPEG/WebRTC video modes
// Extracted from ConsoleView.vue for better separation of concerns
// Legacy MJPEG-only streaming composable.
// Deprecated: Console now uses useVideoSession for all switching/connection logic.
import { ref, computed, watch, type Ref } from 'vue'
import { useI18n } from 'vue-i18n'
import { toast } from 'vue-sonner'
import { streamApi } from '@/api'
import { useWebRTC } from '@/composables/useWebRTC'
import { getUnifiedAudio } from '@/composables/useUnifiedAudio'
import { ref, computed, type Ref } from 'vue'
import { useSystemStore } from '@/stores/system'
import { streamApi } from '@/api'
import { generateUUID } from '@/lib/utils'
export type VideoMode = 'mjpeg' | 'h264' | 'h265' | 'vp8' | 'vp9'
export type VideoMode = 'mjpeg'
export interface VideoStreamState {
mode: Ref<VideoMode>
@@ -23,23 +19,14 @@ export interface VideoStreamState {
clientId: string
}
export interface UseVideoStreamOptions {
webrtcVideoRef: Ref<HTMLVideoElement | null>
mjpegVideoRef: Ref<HTMLImageElement | null>
}
// Retry configuration
const BASE_RETRY_DELAY = 2000
const GRACE_PERIOD = 2000
const MAX_CONSECUTIVE_ERRORS = 2
export function useVideoStream(options: UseVideoStreamOptions) {
const { t } = useI18n()
/** @deprecated Use useVideoSession + ConsoleView instead. */
export function useVideoStream() {
const systemStore = useSystemStore()
const webrtc = useWebRTC()
const unifiedAudio = getUnifiedAudio()
// State
const videoMode = ref<VideoMode>('mjpeg')
const videoLoading = ref(true)
const videoError = ref(false)
@@ -48,29 +35,21 @@ export function useVideoStream(options: UseVideoStreamOptions) {
const backendFps = ref(0)
const mjpegTimestamp = ref(0)
const clientId = generateUUID()
// Per-client statistics
const clientsStats = ref<Record<string, { id: string; fps: number; connected_secs: number }>>({})
// Internal state
let retryTimeoutId: number | null = null
let retryCount = 0
let gracePeriodTimeoutId: number | null = null
let consecutiveErrors = 0
let isRefreshingVideo = false
let initialDeviceInfoReceived = false
let webrtcReconnectTimeout: ReturnType<typeof setTimeout> | null = null
// Computed
const mjpegUrl = computed(() => {
if (videoMode.value !== 'mjpeg') return ''
if (mjpegTimestamp.value === 0) return ''
return `${streamApi.getMjpegUrl(clientId)}&t=${mjpegTimestamp.value}`
})
const isWebRTCMode = computed(() => videoMode.value !== 'mjpeg')
// Methods
function refreshVideo() {
backendFps.value = 0
videoError.value = false
@@ -107,7 +86,7 @@ export function useVideoStream(options: UseVideoStreamOptions) {
if (videoMode.value !== 'mjpeg') return
if (isRefreshingVideo) return
consecutiveErrors++
consecutiveErrors += 1
if (consecutiveErrors > MAX_CONSECUTIVE_ERRORS && gracePeriodTimeoutId !== null) {
clearTimeout(gracePeriodTimeoutId)
@@ -123,7 +102,7 @@ export function useVideoStream(options: UseVideoStreamOptions) {
}
videoLoading.value = true
retryCount++
retryCount += 1
const delay = BASE_RETRY_DELAY * Math.pow(1.5, Math.min(retryCount - 1, 5))
retryTimeoutId = window.setTimeout(() => {
@@ -143,153 +122,7 @@ export function useVideoStream(options: UseVideoStreamOptions) {
}
}
async function connectWebRTCOnly(codec: VideoMode = 'h264') {
clearRetryTimers()
retryCount = 0
consecutiveErrors = 0
mjpegTimestamp.value = 0
if (options.mjpegVideoRef.value) {
options.mjpegVideoRef.value.src = ''
}
videoLoading.value = true
videoError.value = false
videoErrorMessage.value = ''
try {
const success = await webrtc.connect()
if (success) {
toast.success(t('console.webrtcConnected'), {
description: t('console.webrtcConnectedDesc'),
duration: 3000,
})
if (webrtc.videoTrack.value && options.webrtcVideoRef.value) {
const stream = webrtc.getMediaStream()
if (stream) {
options.webrtcVideoRef.value.srcObject = stream
try {
await options.webrtcVideoRef.value.play()
} catch {
// AbortError expected when switching modes quickly
}
}
}
videoLoading.value = false
videoMode.value = codec
unifiedAudio.switchMode('webrtc')
} else {
throw new Error('WebRTC connection failed')
}
} catch {
videoError.value = true
videoErrorMessage.value = 'WebRTC connection failed'
videoLoading.value = false
}
}
async function switchToWebRTC(codec: VideoMode = 'h264') {
clearRetryTimers()
retryCount = 0
consecutiveErrors = 0
mjpegTimestamp.value = 0
if (options.mjpegVideoRef.value) {
options.mjpegVideoRef.value.src = ''
}
videoLoading.value = true
videoError.value = false
videoErrorMessage.value = ''
try {
if (webrtc.isConnected.value || webrtc.sessionId.value) {
await webrtc.disconnect()
}
await streamApi.setMode(codec)
const success = await webrtc.connect()
if (success) {
toast.success(t('console.webrtcConnected'), {
description: t('console.webrtcConnectedDesc'),
duration: 3000,
})
if (webrtc.videoTrack.value && options.webrtcVideoRef.value) {
const stream = webrtc.getMediaStream()
if (stream) {
options.webrtcVideoRef.value.srcObject = stream
try {
await options.webrtcVideoRef.value.play()
} catch {
// AbortError expected
}
}
}
videoLoading.value = false
unifiedAudio.switchMode('webrtc')
} else {
throw new Error('WebRTC connection failed')
}
} catch {
videoError.value = true
videoErrorMessage.value = t('console.webrtcFailed')
videoLoading.value = false
toast.error(t('console.webrtcFailed'), {
description: t('console.fallingBackToMjpeg'),
duration: 5000,
})
videoMode.value = 'mjpeg'
}
}
async function switchToMJPEG() {
videoLoading.value = true
videoError.value = false
videoErrorMessage.value = ''
try {
await streamApi.setMode('mjpeg')
} catch {
// Continue anyway
}
if (webrtc.isConnected.value) {
webrtc.disconnect()
}
if (options.webrtcVideoRef.value) {
options.webrtcVideoRef.value.srcObject = null
}
unifiedAudio.switchMode('ws')
refreshVideo()
}
function handleModeChange(mode: VideoMode) {
if (mode === videoMode.value) return
if (mode !== 'mjpeg') {
mjpegTimestamp.value = 0
}
videoMode.value = mode
localStorage.setItem('videoMode', mode)
if (mode !== 'mjpeg') {
switchToWebRTC(mode)
} else {
switchToMJPEG()
}
}
// Handle stream config events
function handleStreamConfigChanging(data: { reason?: string }) {
function handleStreamConfigChanging() {
clearRetryTimers()
videoRestarting.value = true
videoLoading.value = true
@@ -297,14 +130,9 @@ export function useVideoStream(options: UseVideoStreamOptions) {
retryCount = 0
consecutiveErrors = 0
backendFps.value = 0
toast.info(t('console.videoRestarting'), {
description: data.reason === 'device_switch' ? t('console.deviceSwitching') : t('console.configChanging'),
duration: 5000,
})
}
function handleStreamConfigApplied(data: { device: string; resolution: [number, number]; fps: number }) {
function handleStreamConfigApplied() {
consecutiveErrors = 0
gracePeriodTimeoutId = window.setTimeout(() => {
@@ -313,17 +141,7 @@ export function useVideoStream(options: UseVideoStreamOptions) {
}, GRACE_PERIOD)
videoRestarting.value = false
if (videoMode.value !== 'mjpeg') {
switchToWebRTC(videoMode.value)
} else {
refreshVideo()
}
toast.success(t('console.videoRestarted'), {
description: `${data.device} - ${data.resolution[0]}x${data.resolution[1]} @ ${data.fps}fps`,
duration: 3000,
})
refreshVideo()
}
function handleStreamStatsUpdate(data: { clients?: number; clients_stat?: Record<string, { fps: number }> }) {
@@ -331,13 +149,6 @@ export function useVideoStream(options: UseVideoStreamOptions) {
systemStore.updateStreamClients(data.clients)
}
if (videoMode.value !== 'mjpeg') {
if (data.clients_stat) {
clientsStats.value = data.clients_stat as any
}
return
}
if (data.clients_stat) {
clientsStats.value = data.clients_stat as any
const myStats = data.clients_stat[clientId]
@@ -359,118 +170,27 @@ export function useVideoStream(options: UseVideoStreamOptions) {
if (data.video?.config_changing) return
if (data.video?.stream_mode) {
const serverStreamMode = data.video.stream_mode
const serverMode = serverStreamMode === 'webrtc' ? 'h264' : serverStreamMode as VideoMode
if (!initialDeviceInfoReceived) {
initialDeviceInfoReceived = true
if (serverMode !== videoMode.value) {
videoMode.value = serverMode
if (serverMode !== 'mjpeg') {
setTimeout(() => connectWebRTCOnly(serverMode), 100)
} else {
setTimeout(() => refreshVideo(), 100)
}
} else if (serverMode !== 'mjpeg') {
setTimeout(() => connectWebRTCOnly(serverMode), 100)
} else {
setTimeout(() => refreshVideo(), 100)
}
} else if (serverMode !== videoMode.value) {
handleModeChange(serverMode)
if (!initialDeviceInfoReceived) {
initialDeviceInfoReceived = true
if (data.video?.stream_mode === 'mjpeg') {
setTimeout(() => refreshVideo(), 100)
}
}
}
function handleStreamModeChanged(data: { mode: string; previous_mode: string }) {
const newMode = data.mode === 'webrtc' ? 'h264' : data.mode as VideoMode
toast.info(t('console.streamModeChanged'), {
description: t('console.streamModeChangedDesc', { mode: data.mode.toUpperCase() }),
duration: 5000,
})
if (newMode !== videoMode.value) {
handleModeChange(newMode)
}
function handleModeChange(mode: VideoMode) {
if (mode !== 'mjpeg') return
if (mode === videoMode.value) return
videoMode.value = mode
localStorage.setItem('videoMode', mode)
refreshVideo()
}
// Watch WebRTC video track
watch(() => webrtc.videoTrack.value, async (track) => {
if (track && options.webrtcVideoRef.value && videoMode.value !== 'mjpeg') {
const stream = webrtc.getMediaStream()
if (stream) {
options.webrtcVideoRef.value.srcObject = stream
try {
await options.webrtcVideoRef.value.play()
} catch {
// AbortError expected
}
}
}
})
// Watch WebRTC audio track
watch(() => webrtc.audioTrack.value, (track) => {
if (track && options.webrtcVideoRef.value && videoMode.value !== 'mjpeg') {
const currentStream = options.webrtcVideoRef.value.srcObject as MediaStream | null
if (currentStream && currentStream.getAudioTracks().length === 0) {
currentStream.addTrack(track)
}
}
})
// Watch WebRTC element for unified audio
watch(options.webrtcVideoRef, (el) => {
unifiedAudio.setWebRTCElement(el)
}, { immediate: true })
// Watch WebRTC stats for FPS
watch(webrtc.stats, (stats) => {
if (videoMode.value !== 'mjpeg' && stats.framesPerSecond > 0) {
backendFps.value = Math.round(stats.framesPerSecond)
systemStore.setStreamOnline(true)
}
}, { deep: true })
// Watch WebRTC state for auto-reconnect
watch(() => webrtc.state.value, (newState, oldState) => {
if (videoMode.value !== 'mjpeg') {
if (newState === 'connected') {
systemStore.setStreamOnline(true)
}
}
if (webrtcReconnectTimeout) {
clearTimeout(webrtcReconnectTimeout)
webrtcReconnectTimeout = null
}
if (newState === 'disconnected' && oldState === 'connected' && videoMode.value !== 'mjpeg') {
webrtcReconnectTimeout = setTimeout(async () => {
if (videoMode.value !== 'mjpeg' && webrtc.state.value === 'disconnected') {
try {
await webrtc.connect()
} catch {
// Will retry on next disconnect
}
}
}, 1000)
}
})
// Cleanup
function cleanup() {
clearRetryTimers()
if (webrtcReconnectTimeout) {
clearTimeout(webrtcReconnectTimeout)
}
}
return {
// State
const state: VideoStreamState = {
mode: videoMode,
loading: videoLoading,
error: videoError,
@@ -479,29 +199,19 @@ export function useVideoStream(options: UseVideoStreamOptions) {
fps: backendFps,
mjpegUrl,
clientId,
}
return {
state,
clientsStats,
isWebRTCMode,
// WebRTC access
webrtc,
// Methods
refreshVideo,
handleVideoLoad,
handleVideoError,
handleModeChange,
connectWebRTCOnly,
switchToWebRTC,
switchToMJPEG,
// Event handlers
handleStreamConfigChanging,
handleStreamConfigApplied,
handleStreamStatsUpdate,
handleDeviceInfo,
handleStreamModeChanged,
// Cleanup
handleModeChange,
cleanup,
}
}