feat: UAC USB microphone passthrough

- Add UAC1 gadget function (ConfigFS) with optimized endpoint config - Browser mic capture with Opus encoding via WebCodecs AudioEncoder - WebSocket audio transport (Opus 64kbps, 100x bandwidth reduction vs raw PCM) - aplay subprocess for reliable PCM playback to USB gadget - Settings toggle + ActionBar mic button (hidden when UAC disabled) - Dynamic PCM device resolution (/proc/asound) - c_chmask=0 + req_number=4 fixes DWC3 composite isochronous endpoint issue
2026-09-12 18:44:25 +08:00 · 2026-07-23 06:56:53 +00:00
parent 4e32b05124
commit e9bed3688f
27 changed files with 977 additions and 16 deletions
--- a/web/src/api/config.ts
+++ b/web/src/api/config.ts
@@ -118,6 +118,16 @@ export const otgNetworkApi = {
  interfaces: () => request<NetworkInterfaceInfo[]>('/devices/network'),
 }

+export const uacApi = {
+  get: () => request<{enabled: boolean; sample_rate: number; channels: number}>('/config/uac'),
+
+  update: (config: {enabled: boolean; sample_rate: number; channels: number}) =>
+    request('/config/uac', {
+      method: 'PATCH',
+      body: JSON.stringify(config),
+    }),
+}
+
 export const otgConfigApi = {
  update: (config: OtgConfigUpdate) =>
    request<OtgConfigResponse>('/config/otg', {
--- a/web/src/api/index.ts
+++ b/web/src/api/index.ts
@@ -855,6 +855,7 @@ export {
  msdConfigApi,
  otgConfigApi,
  otgNetworkApi,
+  uacApi,
  atxConfigApi,
  audioConfigApi,
  extensionsApi,
--- a/web/src/components/ActionBar.vue
+++ b/web/src/components/ActionBar.vue
@@ -3,6 +3,7 @@ import { ref, computed, onMounted, onUnmounted, watch, nextTick } from 'vue'
 import { useI18n } from 'vue-i18n'
 import { useRouter } from 'vue-router'
 import { useSystemStore } from '@/stores/system'
+import { getMicrophone } from '@/composables/useMicrophone'
 import { Button } from '@/components/ui/button'
 import { ButtonGroup } from '@/components/ui/button-group'
 import {
@@ -31,6 +32,7 @@ import {
 } from '@/components/ui/sheet'
 import {
  ClipboardPaste,
+  Mic,
  HardDrive,
  Keyboard,
  Settings,
@@ -68,9 +70,13 @@ const props = defineProps<{
  showTerminal?: boolean
  showComputerUse?: boolean
  showPasteText?: boolean
+  showMic?: boolean
 }>()
 const showStats = computed(() => (props.videoMode ?? 'mjpeg') !== 'mjpeg')
 const showPasteText = computed(() => props.showPasteText !== false)
+const showMic = computed(() => props.showMic === true)
+const mic = getMicrophone()
+

 const emit = defineEmits<{
  (e: 'toggleFullscreen'): void
@@ -350,6 +356,24 @@ const hasRightOverflow = computed(() => {
            </PopoverContent>
          </Popover>
        </div>
+
+        <!-- Mic button -->
+        <div v-if="showMic">
+          <TooltipProvider>
+            <Tooltip>
+              <TooltipTrigger as-child>
+                <Button variant="ghost" size="sm" class="h-8 gap-1.5 text-xs"
+                  :class="mic.active.value ? 'text-destructive' : mic.error.value ? 'text-yellow-500' : ''"
+                  @click="mic.toggle()">
+                  <Mic class="size-4" :class="mic.active.value ? 'animate-pulse' : ''" />
+                  <span>{{ mic.active.value ? '关闭' : '麦克风' }}</span>
+                  <span v-if="mic.error.value" class="text-[10px]">{{ mic.error.value }}</span>
+                </Button>
+              </TooltipTrigger>
+              <TooltipContent>{{ mic.error.value ? mic.error.value : (mic.active.value ? '停止传声' : '开始传声') }}</TooltipContent>
+            </Tooltip>
+          </TooltipProvider>
+        </div>
      </ButtonGroup>

      <!-- Right side buttons -->
--- a/web/src/composables/useMicrophone.ts
+++ b/web/src/composables/useMicrophone.ts
@@ -0,0 +1,161 @@
+import { ref } from 'vue'
+
+let instance: ReturnType<typeof useMicrophone> | null = null
+export function getMicrophone() {
+  if (!instance) instance = useMicrophone()
+  return instance
+}
+
+const WS_BASE = `${location.protocol === 'https:' ? 'wss:' : 'ws:'}//${location.host}/api/ws/uac-audio`
+
+// Opus: 48kHz stereo, 64kbps → ~8 KB/s vs raw PCM 192 KB/s (24× reduction)
+const OPUS_CONFIG: AudioEncoderConfig = {
+  codec: 'opus',
+  sampleRate: 48000,
+  numberOfChannels: 2,
+  bitrate: 64000,
+}
+
+// 15-byte binary header matching server-side UAC_AUDIO_HEADER_SIZE
+function buildHeader(msgType: number, durationMs: number, dataLen: number): Uint8Array {
+  const h = new Uint8Array(15)
+  const v = new DataView(h.buffer)
+  v.setUint8(0, msgType)       // 0x03 = Opus
+  v.setUint32(1, 0, true)      // timestamp (unused)
+  v.setUint16(5, durationMs, true)
+  v.setUint32(7, 0, true)      // sequence
+  v.setUint32(11, dataLen, true)
+  return h
+}
+
+export function useMicrophone() {
+  const active = ref(false)
+  const error = ref<string | null>(null)
+  let ws: WebSocket | null = null
+  let stream: MediaStream | null = null
+  let encoder: AudioEncoder | null = null
+  let running = false
+
+  let frameCount = 0
+  let byteCount = 0
+
+  // ── AudioEncoder helper ─────────────────────────────────
+  function createEncoder(onOpusFrame: (data: Uint8Array, durMs: number) => void): AudioEncoder {
+    const enc = new AudioEncoder({
+      output: (chunk: EncodedAudioChunk) => {
+        const buf = new Uint8Array(chunk.byteLength)
+        chunk.copyTo(buf)
+        // Opus frame duration in microseconds → milliseconds
+        const durMs = Math.round(chunk.duration! / 1000)
+        onOpusFrame(buf, durMs)
+      },
+      error: (e: Error) => console.error('[mic] encoder error:', e),
+    })
+    enc.configure(OPUS_CONFIG)
+    return enc
+  }
+
+  // ── start / stop ────────────────────────────────────────
+  async function start() {
+    error.value = null
+    frameCount = 0
+    byteCount = 0
+    running = true
+    console.log('[mic] starting...')
+
+    try {
+      // WebSocket
+      ws = new WebSocket(WS_BASE)
+      ws.binaryType = 'arraybuffer'
+      const wsReady = new Promise<void>((resolve, reject) => {
+        ws!.onopen = () => { console.log('[mic] WS opened'); active.value = true; resolve() }
+        ws!.onerror = (ev) => { console.error('[mic] WS error:', ev); reject(new Error('WebSocket failed')) }
+      })
+      ws.onclose = (ev) => {
+        console.log('[mic] WS closed: code=%d frames=%d bytes=%d', ev.code, frameCount, byteCount)
+        active.value = false
+        running = false
+      }
+
+      // Microphone
+      console.log('[mic] getUserMedia...')
+      stream = await navigator.mediaDevices.getUserMedia({
+        audio: { sampleRate: 48000, channelCount: 2, echoCancellation: false, noiseSuppression: false }
+      })
+      // AudioEncoder (WebCodecs) for Opus compression
+      encoder = createEncoder((opusData, durMs) => {
+        if (!ws || ws.readyState !== WebSocket.OPEN) return
+        const header = buildHeader(0x03, durMs, opusData.length)
+        const msg = new Uint8Array(15 + opusData.length)
+        msg.set(header)
+        msg.set(opusData, 15)
+        ws.send(msg)
+        frameCount++
+        byteCount += msg.byteLength
+        if (frameCount % 50 === 0) {
+          console.debug('[mic] frame #%d: opus=%dB dur=%dms',
+            frameCount, opusData.length, durMs)
+        }
+      })
+
+      await wsReady
+
+      // ScriptProcessor → S16LE PCM → AudioData → AudioEncoder → Opus
+      const audioCtx = new AudioContext({ sampleRate: 48000 })
+      const source = audioCtx.createMediaStreamSource(stream)
+      const processor = audioCtx.createScriptProcessor(4096, 2, 2)
+      source.connect(processor)
+      processor.connect(audioCtx.destination)
+
+      processor.onaudioprocess = (e: AudioProcessingEvent) => {
+        if (!running || !encoder || encoder.state !== 'configured') return
+        if (!e.inputBuffer) return
+        const buf = e.inputBuffer as any
+        const left = buf.getChannelData(0) as Float32Array
+        const right = buf.getChannelData(1) as Float32Array
+        const samples = left.length
+
+        // Float32 → S16LE interleaved
+        const pcm = new Int16Array(samples * 2)
+        for (let i = 0; i < samples; i++) {
+          pcm[i * 2]     = Math.max(-32768, Math.min(32767, Math.round((left[i] ?? 0) * 32767)))
+          pcm[i * 2 + 1] = Math.max(-32768, Math.min(32767, Math.round((right[i] ?? 0) * 32767)))
+        }
+
+        try {
+          const audioData = new AudioData({
+            format: 's16',
+            sampleRate: 48000,
+            numberOfFrames: samples,
+            numberOfChannels: 2,
+            timestamp: 0,
+            data: pcm.buffer,
+          })
+          encoder.encode(audioData)
+          audioData.close()
+        } catch (e) {
+          console.warn('[mic] AudioData/encode error:', e)
+        }
+      }
+    } catch (e) {
+      console.error('[mic] start error:', e)
+      error.value = e instanceof Error ? e.message : 'Failed to start microphone'
+      stop()
+    }
+  }
+
+  function stop() {
+    console.log('[mic] stop: frames=%d bytes=%d', frameCount, byteCount)
+    running = false
+    if (encoder) { encoder.close(); encoder = null }
+    if (stream) { stream.getTracks().forEach(t => t.stop()); stream = null }
+    if (ws) { ws.close(); ws = null }
+    active.value = false
+  }
+
+  function toggle() {
+    if (active.value) { stop() } else { start() }
+  }
+
+  return { active, error, start, stop, toggle }
+}
--- a/web/src/i18n/en-US.ts
+++ b/web/src/i18n/en-US.ts
@@ -653,6 +653,8 @@ export default {
    otgNetworkDriver: 'Host Driver Mode',
    otgNetworkInterface: 'Bridge Interface',
    otgNetworkNone: 'None',
+    uacMic: 'USB Microphone',
+    uacMicDesc: 'Creates a virtual USB microphone on the target machine. Audio from your browser is streamed to the target.',
    otgDescriptor: 'USB Device Descriptor',
    vendorId: 'Vendor ID (VID)',
    productId: 'Product ID (PID)',
--- a/web/src/i18n/zh-CN.ts
+++ b/web/src/i18n/zh-CN.ts
@@ -92,7 +92,8 @@ export default {
  },
  actionbar: {
    paste: '粘贴文本',
-    virtualMedia: '虚拟媒体',
+    micStart: '开始传声',
+    micStop: '停止传声',
    virtualMediaTip: '管理虚拟媒体设备',
    power: '电源',
    keyboard: '虚拟键盘',
@@ -652,6 +653,8 @@ export default {
    otgNetworkDriver: '目标机驱动模式',
    otgNetworkInterface: '桥接网卡',
    otgNetworkNone: '无',
+    uacMic: 'USB 麦克风',
+    uacMicDesc: '启用后目标机将看到一个 USB 麦克风设备，音频从浏览器传入',
    otgDescriptor: 'USB 设备描述符',
    vendorId: '厂商 ID (VID)',
    productId: '产品 ID (PID)',
--- a/web/src/views/ConsoleView.vue
+++ b/web/src/views/ConsoleView.vue
@@ -14,7 +14,7 @@ import { useComputerUseSocket, type ComputerUseServerMessage } from '@/composabl
 import { useFeatureVisibility } from '@/composables/useFeatureVisibility'
 import { useTheme } from '@/composables/useTheme'
 import { getUnifiedAudio } from '@/composables/useUnifiedAudio'
-import { streamApi, hidApi, atxApi, atxConfigApi, authApi, computerUseApi } from '@/api'
+import { streamApi, hidApi, atxApi, atxConfigApi, authApi, computerUseApi, uacApi } from '@/api'
 import type { ComputerUseScreenshot, ComputerUseSession } from '@/api'
 import { CanonicalKey, HidBackend } from '@/types/generated'
 import type { HidKeyboardEvent, HidMouseEvent } from '@/types/hid'
@@ -2961,7 +2961,15 @@ function handleToggleMouseMode() {
  }
 }

+const uacEnabled = ref(false)
+
 onMounted(async () => {
+  // Check if UAC is enabled (show mic button only if USB mic is available)
+  try {
+    const uacCfg = await uacApi.get()
+    uacEnabled.value = uacCfg.enabled
+  } catch { /* ignore */ }
+
  consoleEvents.subscribe()

  watch([wsConnected, wsNetworkError], ([connected, netError], [_prevConnected, prevNetError]) => {
@@ -3166,6 +3174,7 @@ onUnmounted(() => {
      :show-terminal="showTerminal"
      :show-computer-use="showComputerUse"
      :show-paste-text="showPasteText"
+      :show-mic="uacEnabled"
      @toggle-fullscreen="toggleFullscreen"
      @toggle-stats="openStatsSheet"
      @toggle-virtual-keyboard="handleToggleVirtualKeyboard"
--- a/web/src/views/SettingsView.vue
+++ b/web/src/views/SettingsView.vue
@@ -11,6 +11,7 @@ import {
  authApi,
  configApi,
  otgNetworkApi,
+  uacApi,
  hidApi,
  streamApi,
  atxConfigApi,
@@ -685,6 +686,7 @@ const config = ref({
  turn_server: '',
  turn_username: '',
  turn_password: '',
+  uac_enabled: false,
 })

 const otgNetworkInterfaces = ref<NetworkInterfaceInfo[]>([])
@@ -1479,6 +1481,12 @@ async function saveConfig() {
        },
      })
      otgNetworkStatus.value = response.status
+
+      await uacApi.update({
+        enabled: otgEnabled && config.value.uac_enabled,
+        sample_rate: 48000,
+        channels: 2,
+      })
    }

    if (activeSection.value !== 'hid') {
@@ -1499,7 +1507,7 @@ async function saveConfig() {

 async function loadConfig() {
  try {
-    const [video, stream, hid, msd, otgNetwork] = await Promise.all([
+    const [video, stream, hid, msd, otgNetwork, uac] = await Promise.all([
      configStore.refreshVideo(),
      configStore.refreshStream(),
      configStore.refreshHid(),
@@ -1511,6 +1519,7 @@ async function loadConfig() {
        host_mac: '',
        device_mac: '',
      })),
+      uacApi.get().catch(() => ({ enabled: false, sample_rate: 48000, channels: 2 })),
    ])

    config.value = {
@@ -1536,6 +1545,7 @@ async function loadConfig() {
      msd_dir: msd.msd_dir || '',
      otg_network_enabled: otgNetwork.enabled,
      otg_network_driver: otgNetwork.driver_mode,
+      uac_enabled: uac.enabled,
      otg_network_interface: otgNetwork.bridge_interface,
      encoder_backend: stream.encoder || 'auto',
      stun_server: stream.stun_server || '',
@@ -3342,6 +3352,15 @@ watch(isWindows, () => {
                          {{ t('settings.otgRuntimeDegraded') }}: {{ otgNetworkStatus.error || t('common.error') }}
                        </p>
                      </div>
+                      <div class="space-y-3 rounded-md border border-border/60 p-3">
+                        <div class="flex items-center justify-between gap-4">
+                          <div>
+                            <Label>{{ t('settings.uacMic') }}</Label>
+                            <p class="text-xs text-muted-foreground">{{ t('settings.uacMicDesc') }}</p>
+                          </div>
+                          <Switch v-model="config.uac_enabled" />
+                        </div>
+                      </div>
                    </div>
                    <p class="text-xs text-warning">
                      {{ t('settings.otgProfileWarning') }}