feat(mcp): 用 MCP 服务替换内置 AI Agent (#319)

在现有 HTTP 服务挂载 /mcp,让外部 AI 经共享 API Key 调用原分辨率截图、批量键鼠、ATX/WOL 和虚拟介质八个工具。所有调用共用有界 FIFO 和全局限流,复用现有采集管线与硬件控制器。

移除内置模型调用、网页 Agent 面板及旧配置,设置页提供可持续查看的 Key、统一的地址/密钥复制样式。补齐 DEB 的 libudev1 运行依赖。

验证:583 项 Rust 库测试、前端生产构建、ARMv7 设备视觉操作与介质/WOL 实测,以及默认 Release 构建和 DEB 包校验通过。

Refs #319
This commit is contained in:
mofeng-git
2026-10-19 12:12:03 +08:00
parent 3f970ee834
commit 7ea2936436
50 changed files with 2910 additions and 3713 deletions

View File

@@ -2,12 +2,6 @@ import { request, uploadRequest, ApiError } from './request'
import type {
CanonicalKey,
Ch9329DescriptorState,
ComputerUseButton,
ComputerUseConfigResponse,
ComputerUseConfigUpdate,
ComputerUseSessionStatus,
ComputerUseSessionSummary,
ComputerUseStartRequest,
} from '@/types/generated'
import { useHidWebSocket, type HidKeyboardEvent, type HidMouseEvent } from '@/composables/useHidWebSocket'
import type { StreamSignalReason, StreamState } from '@/types/websocket'
@@ -508,48 +502,17 @@ export const hidApi = {
isWebSocketConnected: () => hidWs.connected.value,
}
export type ComputerUseStatus = ComputerUseSessionStatus
export type ComputerUseAction =
| { type: 'click'; x: number; y: number; button?: ComputerUseButton }
| { type: 'double_click'; x: number; y: number; button?: ComputerUseButton }
| { type: 'move'; x: number; y: number }
| { type: 'drag'; path: Array<{ x: number; y: number }>; button?: ComputerUseButton }
| { type: 'scroll'; x: number; y: number; dx?: number; dy?: number }
| { type: 'type'; text: string }
| { type: 'keypress'; keys: string[] }
| { type: 'wait'; ms: number }
| { type: 'screenshot' }
export type ComputerUseConversationMessage =
| { role: 'user'; text: string }
| { role: 'assistant'; text: string }
export type ComputerUseConfig = ComputerUseConfigResponse
export type ComputerUseSession = ComputerUseSessionSummary
export const computerUseApi = {
config: () => request<ComputerUseConfig>('/config/computer-use'),
updateConfig: (data: ComputerUseConfigUpdate) =>
request<ComputerUseConfig>('/config/computer-use', {
method: 'PATCH',
body: JSON.stringify(data),
}),
session: () => request<ComputerUseSession>('/computer-use/session'),
start: (data: ComputerUseStartRequest) =>
request<ComputerUseSession>('/computer-use/session', {
method: 'POST',
body: JSON.stringify(data),
}),
stop: () =>
request<ComputerUseSession>('/computer-use/session/stop', {
method: 'POST',
}),
export interface McpConfig {
enabled: boolean
requests_per_second: number
api_key_configured: boolean
endpoint: string
api_key?: string
}
export const mcpApi = {
config: () => request<McpConfig>('/config/mcp'),
updateConfig: (data: { enabled?: boolean; requests_per_second?: number; regenerate_api_key?: boolean }) =>
request<McpConfig>('/config/mcp', { method: 'PATCH', body: JSON.stringify(data) }),
}
export const atxApi = {
@@ -884,13 +847,6 @@ export type {
StreamMode,
EncoderType,
BitratePreset,
ComputerUseButton,
ComputerUseConfigResponse,
ComputerUseConfigUpdate,
ComputerUseScreenshot,
ComputerUseSessionStatus,
ComputerUseSessionSummary,
ComputerUseStartRequest,
} from '@/types/generated'
export const audioApi = {

View File

@@ -38,7 +38,6 @@ import {
BarChart3,
Terminal,
MoreHorizontal,
Bot,
ChevronUp,
ChevronDown,
Keyboard,
@@ -73,7 +72,6 @@ const props = defineProps<{
showPower?: boolean
atxEnabled?: boolean
showTerminal?: boolean
showComputerUse?: boolean
showPasteText?: boolean
showMic?: boolean
scaleMode?: VideoScaleMode
@@ -111,7 +109,6 @@ const emit = defineEmits<{
(e: 'reset'): void
(e: 'wol', macAddress: string): void
(e: 'openTerminal'): void
(e: 'openComputerUse'): void
}>()
const pasteOpen = ref(false)
@@ -166,7 +163,7 @@ let layoutResizeObserver: ResizeObserver | null = null
type CollapsibleItem =
| 'msd' | 'atx' | 'paste'
| 'stats' | 'terminal' | 'settings' | 'ai'
| 'stats' | 'terminal' | 'settings'
interface ItemSpec {
id: CollapsibleItem
@@ -180,7 +177,6 @@ const ITEM_SPECS: ItemSpec[] = [
{ id: 'stats', side: 'right' },
{ id: 'terminal', side: 'right' },
{ id: 'settings', side: 'right' },
{ id: 'ai', side: 'right' },
]
const measuredWidths = ref<Map<CollapsibleItem, { icon: number; label: number }>>(new Map())
@@ -256,7 +252,7 @@ watch(locale, () => {
void measureLayout()
})
watch([() => props.layout, () => props.showComputerUse, floatingCollapsed, minimalDisplayControls], () => {
watch([() => props.layout, floatingCollapsed, minimalDisplayControls], () => {
// Layout changes can move this bar while its portal based controls are open.
// Close every transient surface first so no old portal remains attached to
// the document after the new layout is painted.
@@ -296,7 +292,6 @@ const collapsibleItems = computed(() => {
if (item.id === 'paste' && !showPasteText.value) return false
if (item.id === 'stats' && !showStats.value) return false
if (item.id === 'terminal' && props.showTerminal === false) return false
if (item.id === 'ai' && props.showComputerUse === false) return false
return true
})
return items
@@ -309,7 +304,7 @@ const visibleSet = computed(() => {
if (isSidebarLayout.value) {
// Reserve More and a gap between the two groups before assigning vertical slots.
let available = barHeight.value - coreHeight.value - fixedHeight.value - actionHeight.value - 16
const priority: CollapsibleItem[] = ['paste', 'settings', 'msd', 'atx', 'stats', 'terminal', 'ai']
const priority: CollapsibleItem[] = ['paste', 'settings', 'msd', 'atx', 'stats', 'terminal']
for (const id of priority) {
if (!collapsibleItems.value.some(item => item.id === id)) continue
if (available < actionHeight.value) break
@@ -567,26 +562,7 @@ const hasRightOverflow = computed(() => {
</TooltipProvider>
</div>
<!-- Computer Use - Optional -->
<TooltipProvider v-if="isVisible('ai')">
<Tooltip>
<TooltipTrigger as-child>
<Button
variant="ghost"
size="sm"
class="h-8 gap-1.5 text-xs"
:aria-label="t('computerUse.title')"
@click="emit('openComputerUse')"
>
<Bot class="size-3.5 sm:size-4" />
<span v-if="visibleSet.get('ai') === 'label'">AI</span>
</Button>
</TooltipTrigger>
<TooltipContent>
<p>{{ t('computerUse.title') }}</p>
</TooltipContent>
</Tooltip>
</TooltipProvider>
<!-- Settings - Adaptive -->
<div v-if="isVisible('settings')">
@@ -661,10 +637,6 @@ const hasRightOverflow = computed(() => {
{{ t('actionbar.webTerminal') }}
</DropdownMenuItem>
<DropdownMenuItem v-if="!isFloatingLayout && props.showComputerUse !== false && !isVisible('ai')" @click="openFromOverflow(() => emit('openComputerUse'))">
<Bot class="size-4 mr-2" />{{ t('computerUse.title') }}
</DropdownMenuItem>
<!-- Settings -->
<DropdownMenuItem v-if="!isFloatingLayout && !isVisible('settings')" @click="openFromOverflow(() => router.push('/settings'))">
<Settings class="size-4 mr-2" />
@@ -752,8 +724,6 @@ const hasRightOverflow = computed(() => {
<!-- Settings -->
<Button data-measure="settings-icon" variant="ghost" size="sm" class="h-8 gap-1.5 text-xs"><Settings class="size-4" /></Button>
<Button data-measure="settings-label" variant="ghost" size="sm" class="h-8 gap-1.5 text-xs"><Settings class="size-4" />{{ t('actionbar.settings') }}</Button>
<Button data-measure="ai-icon" variant="ghost" size="sm" class="h-8 gap-1.5 text-xs"><Bot class="size-4" /></Button>
<Button data-measure="ai-label" variant="ghost" size="sm" class="h-8 gap-1.5 text-xs"><Bot class="size-4" />AI</Button>
</div>
</div>
</template>

View File

@@ -1,460 +0,0 @@
<script setup lang="ts">
import { useConsoleAppearance } from "@/composables/useConsoleAppearance"
import { computed, nextTick, onMounted, ref, watch } from 'vue'
import { useI18n } from 'vue-i18n'
import {
Bot,
Brain,
ChevronDown,
ChevronRight,
Image,
KeyRound,
SendHorizontal,
Settings,
Square,
Trash2,
Wifi,
WifiOff,
X,
} from 'lucide-vue-next'
import { toast } from 'vue-sonner'
import { computerUseApi, type ComputerUseAction, type ComputerUseConfig, type ComputerUseSession } from '@/api'
import { ApiError } from '@/api/request'
import type { ComputerUseTimelineItem } from '@/types/computerUseTimeline'
import { Button } from '@/components/ui/button'
import { Input } from '@/components/ui/input'
import { Label } from '@/components/ui/label'
import { Textarea } from '@/components/ui/textarea'
import { Switch } from '@/components/ui/switch'
import {
Dialog,
DialogContent,
DialogFooter,
DialogHeader,
DialogTitle,
} from '@/components/ui/dialog'
const props = defineProps<{
open: boolean
connected: boolean
wsError: string | null
session: ComputerUseSession | null
timeline: ComputerUseTimelineItem[]
}>()
const { t } = useI18n()
const consoleAppearance = useConsoleAppearance()
const emit = defineEmits<{
(e: 'update:open', value: boolean): void
(e: 'start', prompt: string): void
(e: 'stop'): void
(e: 'clear'): void
}>()
const config = ref<ComputerUseConfig | null>(null)
const prompt = ref('')
const apiKey = ref('')
const savingConfig = ref(false)
const starting = ref(false)
const settingsOpen = ref(false)
const messagesRef = ref<HTMLDivElement | null>(null)
const reasoningOverrides = ref<Record<string, boolean>>({})
const defaultModel = computed({
get: () => config.value?.model ?? 'gpt-5.5',
set: (value: string) => {
if (config.value) config.value.model = value
},
})
const defaultBaseUrl = computed({
get: () => config.value?.base_url ?? 'https://api.openai.com/v1/responses',
set: (value: string) => {
if (config.value) config.value.base_url = value
},
})
const status = computed(() => props.session?.status ?? 'idle')
const isRunning = computed(() => ['waiting_screenshot', 'thinking', 'executing'].includes(status.value))
const canStart = computed(() => (
!!config.value?.enabled
&& !!config.value?.api_key_configured
&& prompt.value.trim().length > 0
&& !isRunning.value
))
const statusLabel = computed(() => {
switch (status.value) {
case 'waiting_screenshot': return t('computerUse.status.waitingScreenshot')
case 'thinking': return t('computerUse.status.thinking')
case 'executing': return t('computerUse.status.executing')
case 'completed': return t('computerUse.status.completed')
case 'failed': return t('computerUse.status.failed')
case 'stopped': return t('computerUse.status.stopped')
default: return t('computerUse.status.idle')
}
})
const apiKeyPlaceholder = computed(() => {
if (!config.value?.api_key_configured) return t('computerUse.settings.apiKey')
const source = config.value.api_key_source
const sourceLabel = source === 'env'
? t('computerUse.settings.sourceEnv')
: source === 'config'
? t('computerUse.settings.sourceConfig')
: t('computerUse.settings.sourceNone')
return t('computerUse.settings.configured', { source: sourceLabel })
})
async function loadConfig() {
try {
config.value = await computerUseApi.config()
} catch (err) {
// Auth failures are handled globally (redirect to login); don't toast over it.
if (err instanceof ApiError && err.status === 401) return
toast.error(t('computerUse.errors.configLoadFailed'), {
description: err instanceof Error ? err.message : undefined,
})
}
}
async function saveConfig() {
if (!config.value) return
savingConfig.value = true
try {
config.value = await computerUseApi.updateConfig({
enabled: config.value.enabled,
base_url: config.value.base_url,
model: config.value.model,
api_key: apiKey.value.trim() || undefined,
})
apiKey.value = ''
settingsOpen.value = false
toast.success(t('computerUse.success.configSaved'))
} finally {
savingConfig.value = false
}
}
async function clearApiKey() {
savingConfig.value = true
try {
config.value = await computerUseApi.updateConfig({ clear_api_key: true })
apiKey.value = ''
toast.success(t('computerUse.success.apiKeyCleared'))
} finally {
savingConfig.value = false
}
}
function start() {
if (!canStart.value) return
const text = prompt.value.trim()
starting.value = true
emit('start', text)
prompt.value = ''
starting.value = false
}
function formatAction(action: ComputerUseAction): string {
switch (action.type) {
case 'click':
return t('computerUse.actions.click', {
x: action.x,
y: action.y,
button: t(`computerUse.mouseButtons.${action.button ?? 'left'}`),
})
case 'double_click':
return t('computerUse.actions.doubleClick', {
x: action.x,
y: action.y,
button: t(`computerUse.mouseButtons.${action.button ?? 'left'}`),
})
case 'move':
return t('computerUse.actions.move', { x: action.x, y: action.y })
case 'drag':
return t('computerUse.actions.drag', { count: action.path.length })
case 'scroll':
return t('computerUse.actions.scroll', {
x: action.x,
y: action.y,
dx: action.dx ?? 0,
dy: action.dy ?? 0,
})
case 'type':
return t('computerUse.actions.typeAscii', { count: action.text.length })
case 'keypress':
return t('computerUse.actions.keypress', { keys: action.keys.join('+') })
case 'wait':
return t('computerUse.actions.wait', { ms: action.ms })
case 'screenshot':
return t('computerUse.actions.screenshot')
}
}
function reasoningOpen(item: Extract<ComputerUseTimelineItem, { type: 'reasoning' }>) {
return reasoningOverrides.value[item.id] ?? !item.completed
}
function toggleReasoning(item: Extract<ComputerUseTimelineItem, { type: 'reasoning' }>) {
reasoningOverrides.value[item.id] = !reasoningOpen(item)
}
function scrollToBottom() {
nextTick(() => {
const el = messagesRef.value
if (el) el.scrollTop = el.scrollHeight
})
}
watch(() => props.timeline, scrollToBottom, { deep: true })
watch(() => props.open, (open) => {
if (open) scrollToBottom()
})
onMounted(loadConfig)
</script>
<template>
<aside
data-slot="computer-use-panel"
:data-console-layout="consoleAppearance"
v-show="open"
class="absolute inset-y-0 right-0 z-30 h-full min-h-0 w-full border-l bg-background shadow-xl sm:w-[420px] md:relative md:z-auto xl:w-[460px]"
>
<div class="flex h-full min-h-0 flex-col">
<header class="flex h-12 shrink-0 items-center gap-2 border-b px-3">
<Bot class="size-5 shrink-0" />
<div class="min-w-0 flex-1">
<div class="truncate text-sm font-semibold">{{ t('computerUse.title') }}</div>
<div
class="flex min-w-0 items-center gap-1 text-[11px]"
:class="connected ? 'text-emerald-600 dark:text-emerald-400' : 'text-muted-foreground'"
>
<Wifi v-if="connected" class="size-3 shrink-0" />
<WifiOff v-else class="size-3 shrink-0" />
<span class="truncate">
{{ connected ? t('computerUse.connection.connected') : (wsError || t('computerUse.connection.disconnected')) }}
</span>
<span class="text-muted-foreground">· {{ statusLabel }}</span>
</div>
</div>
<Button
variant="ghost"
size="icon"
class="size-8"
:title="t('computerUse.buttons.settings')"
:aria-label="t('computerUse.buttons.settings')"
@click="settingsOpen = true"
>
<Settings class="size-4" />
</Button>
<Button
variant="ghost"
size="icon"
class="size-8"
:title="t('computerUse.buttons.clear')"
:aria-label="t('computerUse.buttons.clear')"
:disabled="isRunning || timeline.length === 0"
@click="emit('clear')"
>
<Trash2 class="size-4" />
</Button>
<Button
variant="ghost"
size="icon"
class="size-8"
:title="t('computerUse.buttons.close')"
:aria-label="t('computerUse.buttons.close')"
@click="emit('update:open', false)"
>
<X class="size-4" />
</Button>
</header>
<div ref="messagesRef" class="min-h-0 flex-1 overflow-y-auto px-4 py-4">
<div v-if="timeline.length === 0" class="flex h-full items-center justify-center text-muted-foreground/45">
<Bot class="size-10" />
</div>
<div v-else class="space-y-5">
<template v-for="item in timeline" :key="item.id">
<div v-if="item.type === 'user'" class="flex justify-end">
<div class="max-w-[86%] whitespace-pre-wrap break-words rounded-md bg-muted px-3 py-2 text-sm">
{{ item.text }}
</div>
</div>
<div v-else-if="item.type === 'assistant'" class="whitespace-pre-wrap break-words text-sm leading-6">
{{ item.text }}
</div>
<div v-else-if="item.type === 'reasoning'" class="border-l-2 border-muted pl-3 text-xs text-muted-foreground">
<button
type="button"
class="flex w-full items-center gap-1.5 py-1 text-left font-medium hover:text-foreground"
@click="toggleReasoning(item)"
>
<ChevronDown v-if="reasoningOpen(item)" class="size-3.5 shrink-0" />
<ChevronRight v-else class="size-3.5 shrink-0" />
<Brain class="size-3.5 shrink-0" />
<span>
{{ item.failed
? t('computerUse.reasoning.failed')
: (item.completed ? t('computerUse.reasoning.process') : t('computerUse.reasoning.thinking')) }}
</span>
</button>
<div
v-if="reasoningOpen(item)"
class="max-h-72 overflow-y-auto whitespace-pre-wrap break-words pb-2 pt-1 leading-5"
>
{{ item.text || t('computerUse.reasoning.thinking') }}
</div>
</div>
<div v-else-if="item.type === 'screenshot'" class="overflow-hidden rounded-md border bg-card">
<div class="flex items-center justify-between border-b px-2.5 py-1.5 text-[11px] text-muted-foreground">
<span class="inline-flex items-center gap-1.5">
<Image class="size-3.5" />{{ t('computerUse.trace.screenshot') }}
</span>
<span>{{ item.screenshot.width }}x{{ item.screenshot.height }}</span>
</div>
<div
class="w-full bg-black"
:style="{ aspectRatio: `${item.screenshot.width} / ${item.screenshot.height}` }"
>
<img
:src="item.screenshot.data_url"
class="h-full w-full object-contain"
:alt="t('computerUse.trace.screenshotAlt')"
/>
</div>
</div>
<div v-else-if="item.type === 'actions_executed'" class="border-l-2 border-emerald-500/50 pl-3 text-xs">
<div class="mb-1 text-[11px] font-medium text-emerald-600 dark:text-emerald-400">
{{ t('computerUse.trace.executed') }}
</div>
<div class="space-y-1 text-muted-foreground">
<div v-for="(action, index) in item.actions" :key="index" class="break-words">
{{ formatAction(action) }}
</div>
</div>
</div>
<div
v-else-if="item.type === 'error'"
class="whitespace-pre-wrap break-words rounded-md border border-destructive/40 bg-destructive/10 px-3 py-2 text-xs leading-5 text-destructive"
>
{{ item.text }}
</div>
<div v-else class="text-center text-xs text-muted-foreground">
{{ item.text }}
</div>
</template>
</div>
</div>
<footer class="shrink-0 border-t bg-background p-3">
<div class="relative">
<Textarea
v-model="prompt"
rows="3"
class="max-h-40 min-h-20 resize-none pr-12"
:placeholder="t('computerUse.input.placeholder')"
:disabled="isRunning"
@keydown.meta.enter.prevent="start"
@keydown.ctrl.enter.prevent="start"
/>
<Button
v-if="!isRunning"
size="icon"
class="absolute bottom-2 right-2 size-8"
:title="t('computerUse.buttons.send')"
:aria-label="t('computerUse.buttons.send')"
:disabled="!canStart || starting"
@click="start"
>
<SendHorizontal class="size-4" />
</Button>
<Button
v-else
size="icon"
variant="destructive"
class="absolute bottom-2 right-2 size-8"
:title="t('computerUse.buttons.stop')"
:aria-label="t('computerUse.buttons.stop')"
@click="emit('stop')"
>
<Square class="size-3.5 fill-current" />
</Button>
</div>
<p v-if="config && !config.api_key_configured" class="mt-2 text-xs text-muted-foreground">
{{ t('computerUse.input.apiKeyRequired') }}
</p>
</footer>
</div>
<Dialog v-model:open="settingsOpen">
<DialogContent class="max-w-md">
<DialogHeader>
<DialogTitle>{{ t('computerUse.settings.title') }}</DialogTitle>
</DialogHeader>
<div class="space-y-4 py-2">
<div class="flex items-center justify-between gap-4">
<Label for="cua-enabled">{{ t('computerUse.settings.enableAi') }}</Label>
<Switch
id="cua-enabled"
:model-value="config?.enabled ?? false"
@update:model-value="(value) => { if (config) config.enabled = value }"
/>
</div>
<div class="space-y-1.5">
<Label for="cua-model">{{ t('computerUse.settings.model') }}</Label>
<Input id="cua-model" v-model="defaultModel" :disabled="!config" placeholder="gpt-5.5" />
</div>
<div class="space-y-1.5">
<Label for="cua-url">{{ t('computerUse.settings.apiUrl') }}</Label>
<Input
id="cua-url"
v-model="defaultBaseUrl"
:disabled="!config"
placeholder="https://api.example.com/v1/responses"
/>
</div>
<div class="space-y-1.5">
<Label for="cua-key" class="flex items-center gap-1.5">
<KeyRound class="size-3.5" />
{{ t('computerUse.settings.apiKey') }}
</Label>
<Input
id="cua-key"
v-model="apiKey"
type="password"
autocomplete="off"
:placeholder="apiKeyPlaceholder"
/>
</div>
</div>
<DialogFooter class="gap-2 sm:justify-between">
<Button
variant="outline"
:disabled="savingConfig || !config?.api_key_configured"
@click="clearApiKey"
>
{{ t('computerUse.buttons.clearKey') }}
</Button>
<Button :disabled="savingConfig || !config" @click="saveConfig">
{{ t('computerUse.buttons.save') }}
</Button>
</DialogFooter>
</DialogContent>
</Dialog>
</aside>
</template>

View File

@@ -2,7 +2,7 @@
import { ref } from 'vue'
import { useRouter } from 'vue-router'
import { useI18n } from 'vue-i18n'
import { BarChart3, Bot, Settings, Terminal, MoreHorizontal } from 'lucide-vue-next'
import { BarChart3, Settings, Terminal, MoreHorizontal } from 'lucide-vue-next'
import { Button } from '@/components/ui/button'
import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuTrigger } from '@/components/ui/dropdown-menu'
@@ -10,12 +10,10 @@ defineProps<{
showStats: boolean
showTerminal: boolean
terminalRunning: boolean
showComputerUse: boolean
}>()
const emit = defineEmits<{
(e: 'openStats'): void
(e: 'openTerminal'): void
(e: 'openComputerUse'): void
}>()
const router = useRouter()
const { t } = useI18n()
@@ -35,18 +33,14 @@ function openFromMenu(action: () => void) {
<Button v-if="showTerminal" variant="ghost" size="sm" class="gap-1.5 rounded-lg text-xs" :aria-label="t('actionbar.webTerminal')" :title="t('actionbar.webTerminal')" :disabled="!terminalRunning" @click="emit('openTerminal')">
<Terminal class="size-4" /><span class="hidden lg:inline">{{ t('actionbar.webTerminal') }}</span>
</Button>
<Button v-if="showComputerUse" variant="ghost" size="sm" class="gap-1.5 rounded-lg text-xs" :aria-label="t('computerUse.title')" :title="t('computerUse.title')" @click="emit('openComputerUse')">
<Bot class="size-4" /><span class="hidden lg:inline">AI</span>
</Button>
</div>
<DropdownMenu v-if="showStats || showTerminal || showComputerUse" v-model:open="menuOpen">
<DropdownMenu v-if="showStats || showTerminal" v-model:open="menuOpen">
<DropdownMenuTrigger as-child class="md:hidden">
<Button variant="ghost" size="icon-sm" :aria-label="t('actionbar.more')"><MoreHorizontal class="size-4" /></Button>
</DropdownMenuTrigger>
<DropdownMenuContent align="end" class="w-48">
<DropdownMenuItem v-if="showStats" @select="openFromMenu(() => emit('openStats'))"><BarChart3 class="size-4" />{{ t('actionbar.stats') }}</DropdownMenuItem>
<DropdownMenuItem v-if="showTerminal" :disabled="!terminalRunning" @select="openFromMenu(() => emit('openTerminal'))"><Terminal class="size-4" />{{ t('actionbar.webTerminal') }}</DropdownMenuItem>
<DropdownMenuItem v-if="showComputerUse" @select="openFromMenu(() => emit('openComputerUse'))"><Bot class="size-4" />{{ t('computerUse.title') }}</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
<Button variant="ghost" size="sm" class="gap-1.5 rounded-lg px-2 text-xs" :aria-label="t('actionbar.settings')" :title="t('actionbar.settings')" @click="router.push('/settings')">

View File

@@ -0,0 +1,93 @@
<script setup lang="ts">
import { computed, ref, onMounted } from 'vue'
import { useI18n } from 'vue-i18n'
import { toast } from 'vue-sonner'
import { Copy } from 'lucide-vue-next'
import { mcpApi, type McpConfig } from '@/api'
import { Button } from '@/components/ui/button'
import { Input } from '@/components/ui/input'
import { Label } from '@/components/ui/label'
import { Switch } from '@/components/ui/switch'
import { Card, CardContent, CardHeader, CardTitle, CardDescription } from '@/components/ui/card'
import { useClipboard } from '@/composables/useClipboard'
const clipboard = useClipboard()
const { t } = useI18n()
const config = ref<McpConfig | null>(null)
const enabled = ref(false)
const rate = ref(5)
const apiKey = ref('')
const busy = ref(false)
const endpoint = computed(() => new URL(config.value?.endpoint ?? '/mcp', window.location.origin).href)
function apply(value: McpConfig) {
config.value = value
enabled.value = value.enabled
rate.value = value.requests_per_second
apiKey.value = value.api_key ?? ''
}
async function load() {
busy.value = true
config.value = null
try { apply(await mcpApi.config()) }
catch (error) { toast.error(error instanceof Error ? error.message : t('mcp.loadFailed')) }
finally { busy.value = false }
}
async function save(regenerate = false) {
if (!Number.isInteger(rate.value) || rate.value < 1 || rate.value > 100) {
toast.error(t('mcp.invalidRate'))
return
}
busy.value = true
try {
apply(await mcpApi.updateConfig({ enabled: enabled.value, requests_per_second: rate.value, regenerate_api_key: regenerate }))
toast.success(t('mcp.saved'))
} catch (error) { toast.error(error instanceof Error ? error.message : t('mcp.saveFailed')) }
finally { busy.value = false }
}
async function copy(value: string) {
if (await clipboard.copy(value)) toast.success(t('mcp.copied'))
else toast.error(t('mcp.copyFailed'))
}
onMounted(load)
</script>
<template>
<Card>
<CardHeader>
<CardTitle>{{ t('mcp.title') }}</CardTitle>
<CardDescription>{{ t('mcp.description') }}</CardDescription>
</CardHeader>
<CardContent class="space-y-5">
<div class="flex items-center justify-between gap-4">
<Label for="mcp-enabled">{{ t('mcp.enabled') }}</Label>
<Switch id="mcp-enabled" v-model="enabled" :disabled="busy || !config" />
</div>
<div class="space-y-2">
<Label for="mcp-rate">{{ t('mcp.rate') }}</Label>
<Input id="mcp-rate" v-model.number="rate" type="number" min="1" max="100" step="1" :disabled="busy || !config" />
</div>
<div class="space-y-2">
<Label>{{ t('mcp.endpoint') }}</Label>
<div class="flex items-center gap-2">
<code id="mcp-endpoint" class="min-w-0 flex-1 break-all rounded bg-muted px-3 py-2 font-mono text-sm">{{ endpoint }}</code>
<Button variant="ghost" size="icon" :aria-label="t('mcp.copyEndpoint')" @click="copy(endpoint)"><Copy class="size-4" /></Button>
</div>
</div>
<div class="space-y-2">
<Label>API Key</Label>
<div v-if="apiKey" class="flex items-center gap-2">
<code id="mcp-key" class="min-w-0 flex-1 break-all rounded bg-muted px-3 py-2 font-mono text-sm">{{ apiKey }}</code>
<Button variant="ghost" size="icon" :aria-label="t('mcp.copyKey')" @click="copy(apiKey)"><Copy class="size-4" /></Button>
</div>
<p v-if="!apiKey" class="text-sm text-muted-foreground">{{ config?.api_key_configured ? t('mcp.keyConfigured') : t('mcp.keyMissing') }}</p>
<Button variant="outline" :disabled="busy || !config" @click="save(true)">{{ t('mcp.regenerate') }}</Button>
</div>
<p class="text-sm text-muted-foreground">{{ t('mcp.connectHint') }}</p>
<div class="flex justify-end gap-2">
<Button v-if="!config && !busy" variant="outline" @click="load">{{ t('mcp.retry') }}</Button>
<Button :disabled="busy || !config" @click="save()">{{ t('mcp.save') }}</Button>
</div>
</CardContent>
</Card>
</template>

View File

@@ -1,108 +0,0 @@
import { ref, onUnmounted } from 'vue'
import { useI18n } from 'vue-i18n'
import { buildWsUrl } from '@/types/websocket'
import { generateUUID } from '@/lib/utils'
import type { ComputerUseScreenshot, ComputerUseSession, ComputerUseAction } from '@/api'
export type ComputerUseServerMessage =
| { type: 'session_updated'; session: ComputerUseSession }
| { type: 'screenshot_requested'; request_id: string }
| { type: 'screenshot_captured'; screenshot: ComputerUseScreenshot }
| { type: 'step_started'; step: number }
| { type: 'reasoning_delta'; delta: string }
| { type: 'reasoning_completed'; failed: boolean }
| { type: 'actions_executed'; actions: ComputerUseAction[] }
| { type: 'error'; message: string }
export function useComputerUseSocket(options: {
onMessage: (message: ComputerUseServerMessage) => void
onScreenshotRequested: (requestId: string) => Promise<ComputerUseScreenshot | null>
}) {
const { t } = useI18n()
const connected = ref(false)
const error = ref<string | null>(null)
const clientId = generateUUID()
let ws: WebSocket | null = null
let connectPromise: Promise<void> | null = null
function connect(): Promise<void> {
if (ws && ws.readyState === WebSocket.OPEN) return Promise.resolve()
if (connectPromise) return connectPromise
ws = new WebSocket(buildWsUrl(`/api/ws/computer-use?client_id=${encodeURIComponent(clientId)}`))
connectPromise = new Promise((resolve, reject) => {
if (!ws) {
reject(new Error(t('computerUse.connection.socketFailed')))
return
}
ws.onopen = () => {
connected.value = true
error.value = null
connectPromise = null
resolve()
}
ws.onerror = () => {
error.value = t('computerUse.connection.socketFailed')
connectPromise = null
reject(new Error(error.value))
}
})
ws.onclose = () => {
connected.value = false
connectPromise = null
}
ws.onmessage = async (event) => {
try {
const message = JSON.parse(event.data) as ComputerUseServerMessage
options.onMessage(message)
if (message.type === 'screenshot_requested') {
try {
const screenshot = await options.onScreenshotRequested(message.request_id)
if (!screenshot) throw new Error(t('computerUse.errors.captureUnavailable'))
if (ws?.readyState === WebSocket.OPEN) {
ws.send(JSON.stringify({
type: 'screenshot_result',
request_id: message.request_id,
screenshot,
}))
}
} catch (err) {
if (ws?.readyState === WebSocket.OPEN) {
ws.send(JSON.stringify({
type: 'screenshot_error',
request_id: message.request_id,
message: err instanceof Error ? err.message : t('computerUse.errors.screenshotFailed'),
}))
}
}
}
} catch (err) {
console.error(`[ComputerUse] ${t('computerUse.connection.messageFailed')}:`, err)
}
}
return connectPromise
}
function disconnect() {
ws?.close()
ws = null
connected.value = false
connectPromise = null
}
onUnmounted(disconnect)
return {
connected,
error,
clientId,
connect,
disconnect,
}
}

View File

@@ -1,13 +1,12 @@
import { useLocalStorage } from '@vueuse/core'
import type { RemovableRef } from '@vueuse/core'
export type FeatureVisibilityKey = 'power' | 'webTerminal' | 'computerUse' | 'pasteText'
export type FeatureVisibilityKey = 'power' | 'webTerminal' | 'pasteText'
export type FeatureVisibility = Record<FeatureVisibilityKey, boolean>
const DEFAULT_FEATURE_VISIBILITY: FeatureVisibility = {
power: true,
webTerminal: true,
computerUse: true,
pasteText: true,
}

View File

@@ -800,7 +800,6 @@ export default {
languageDesc: 'Choose the interface display language',
featureVisibility: 'Feature Visibility',
featureVisibilityDesc: 'Control which feature entry points are shown on the console page',
computerUseAgent: 'Computer Use Agent',
pasteText: 'Paste Text',
videoSettings: 'Video Capture',
videoSettingsDesc: 'Select a capture device. Source-following inputs report their active mode automatically.',
@@ -983,85 +982,26 @@ export default {
loadDevicesFailed: 'Failed to load device list',
updateFailed: 'Update failed',
},
computerUse: {
title: 'Computer Use',
status: {
waitingScreenshot: 'Capturing screen',
thinking: 'Thinking',
executing: 'Executing',
completed: 'Completed',
failed: 'Failed',
stopped: 'Stopped',
idle: 'Idle',
},
connection: {
connected: 'Connected',
disconnected: 'Disconnected',
socketFailed: 'Computer Use WebSocket connection failed',
messageFailed: 'Failed to process Computer Use WebSocket message',
},
buttons: {
settings: 'Settings',
clear: 'Clear',
close: 'Close',
send: 'Send',
stop: 'Stop',
save: 'Save',
clearKey: 'Clear Key',
},
reasoning: {
failed: 'Thinking failed',
process: 'Reasoning',
thinking: 'Thinking',
},
trace: {
screenshot: 'Screenshot',
screenshotAlt: 'Computer Use screenshot',
executed: 'Executed',
},
input: {
placeholder: 'Enter a task. Press Enter for a new line; press Ctrl+Enter to send.',
apiKeyRequired: 'Configure an API Key first.',
},
settings: {
title: 'Computer Use Settings',
enableAi: 'Enable AI control',
model: 'Model',
apiUrl: 'API URL',
apiKey: 'API Key',
configured: 'Configured: {source}',
sourceEnv: 'environment variable',
sourceConfig: 'configuration file',
sourceNone: 'not configured',
},
actions: {
click: 'Click ({x}, {y}) {button}',
doubleClick: 'Double-click ({x}, {y}) {button}',
move: 'Move to ({x}, {y})',
drag: 'Drag through {count} points',
scroll: 'Scroll at ({x}, {y}) dx={dx} dy={dy}',
typeAscii: 'Type {count} ASCII characters',
keypress: 'Press {keys}',
wait: 'Wait {ms}ms',
screenshot: 'Request screenshot',
},
mouseButtons: {
left: 'left button',
middle: 'middle button',
right: 'right button',
},
errors: {
configLoadFailed: 'Failed to load Computer Use settings',
taskFailed: 'Computer Use task failed',
startFailed: 'Failed to start Computer Use',
stopFailed: 'Failed to stop Computer Use',
captureUnavailable: 'The current video frame is unavailable',
screenshotFailed: 'Screenshot capture failed',
},
success: {
configSaved: 'Computer Use settings saved',
apiKeyCleared: 'API Key cleared',
},
mcp: {
title: "MCP Service",
description: "Let external AI tools view the screen and control input, power, and virtual media.",
enabled: "Enable MCP",
rate: "Global request rate (1–100 per second)",
endpoint: "Service URL",
copyEndpoint: "Copy service URL",
copyKey: "Copy API key",
copied: "Copied",
copyFailed: "Copy failed. Select and copy the text manually.",
keyConfigured: "A key is configured. Generate a new key to view it.",
keyMissing: "A key is generated automatically when enabled.",
regenerate: "Generate new key",
connectHint: "Configure the service URL and Authorization: Bearer <API Key> in your AI tool.",
save: "Save",
saved: "MCP settings saved",
loadFailed: "Failed to load MCP settings",
saveFailed: "Failed to save MCP settings",
invalidRate: "Request rate must be an integer from 1 to 100",
retry: "Retry",
},
statusCard: {
connectionDetails: 'Connection details',

View File

@@ -799,7 +799,6 @@ export default {
languageDesc: '选择界面显示语言',
featureVisibility: '功能展示',
featureVisibilityDesc: '控制控制台页面显示的功能入口',
computerUseAgent: 'Computer Use Agent',
pasteText: '粘贴文本',
videoSettings: '视频采集',
videoSettingsDesc: '选择视频采集设备;输入跟随型设备会自动显示当前输入模式',
@@ -982,85 +981,26 @@ export default {
loadDevicesFailed: '加载设备列表失败',
updateFailed: '更新失败',
},
computerUse: {
title: 'Computer Use',
status: {
waitingScreenshot: '截屏中',
thinking: '思考中',
executing: '执行中',
completed: '已完成',
failed: '失败',
stopped: '已停止',
idle: '空闲',
},
connection: {
connected: '已连接',
disconnected: '未连接',
socketFailed: 'Computer Use WebSocket 连接失败',
messageFailed: '处理 Computer Use WebSocket 消息失败',
},
buttons: {
settings: '设置',
clear: '清空',
close: '关闭',
send: '发送',
stop: '停止',
save: '保存',
clearKey: '清除 Key',
},
reasoning: {
failed: '思考失败',
process: '思考过程',
thinking: '思考中',
},
trace: {
screenshot: '截图',
screenshotAlt: 'Computer Use 截图',
executed: '已执行',
},
input: {
placeholder: '输入任务,按下回车换行,按下 Ctrl+Enter 发送',
apiKeyRequired: '请先配置 API Key。',
},
settings: {
title: 'Computer Use 设置',
enableAi: '启用 AI 操作',
model: '模型',
apiUrl: 'API URL',
apiKey: 'API Key',
configured: '已配置:{source}',
sourceEnv: '环境变量',
sourceConfig: '配置文件',
sourceNone: '未配置',
},
actions: {
click: '点击 ({x}, {y}) {button}',
doubleClick: '双击 ({x}, {y}) {button}',
move: '移动到 ({x}, {y})',
drag: '拖拽 {count} 个点',
scroll: '滚动 ({x}, {y}) dx={dx} dy={dy}',
typeAscii: '输入 {count} 个 ASCII 字符',
keypress: '按键 {keys}',
wait: '等待 {ms}ms',
screenshot: '请求截图',
},
mouseButtons: {
left: '左键',
middle: '中键',
right: '右键',
},
errors: {
configLoadFailed: '无法加载 Computer Use 配置',
taskFailed: 'Computer Use 任务失败',
startFailed: 'Computer Use 启动失败',
stopFailed: 'Computer Use 停止失败',
captureUnavailable: '当前视频画面尚不可用',
screenshotFailed: '截图失败',
},
success: {
configSaved: 'Computer Use 配置已保存',
apiKeyCleared: 'API Key 已清除',
},
mcp: {
title: "MCP 服务",
description: "让外部 AI 工具查看画面,操作键鼠、电源和虚拟介质。",
enabled: "启用 MCP",
rate: "全局请求频率(每秒 1–100 次)",
endpoint: "服务地址",
copyEndpoint: "复制服务地址",
copyKey: "复制 API Key",
copied: "已复制",
copyFailed: "复制失败,请手动选择复制",
keyConfigured: "已配置 Key,可重新生成后查看。",
keyMissing: "启用时会自动生成 Key。",
regenerate: "重新生成 Key",
connectHint: "在 AI 工具中填写服务地址,并设置 Authorization: Bearer <API Key>。",
save: "保存",
saved: "MCP 设置已保存",
loadFailed: "无法加载 MCP 设置",
saveFailed: "无法保存 MCP 设置",
invalidRate: "请求频率必须是 1–100 的整数",
retry: "重试",
},
statusCard: {
connectionDetails: '连接详情',

View File

@@ -254,10 +254,3 @@ body {
min-height: 44px;
}
}
[data-console-layout][data-slot="computer-use-panel"] {
overflow: hidden;
border: 1px solid var(--border);
border-radius: var(--console-surface-radius);
box-shadow: var(--console-surface-shadow);
}

View File

@@ -1,16 +0,0 @@
import type { ComputerUseAction, ComputerUseScreenshot } from '@/api'
export type ComputerUseTimelineItem =
| { id: string; type: 'user'; text: string }
| { id: string; type: 'assistant'; text: string }
| { id: string; type: 'reasoning'; text: string; completed: boolean; failed: boolean }
| { id: string; type: 'screenshot'; screenshot: ComputerUseScreenshot }
| { id: string; type: 'actions_executed'; actions: ComputerUseAction[] }
| { id: string; type: 'error'; text: string }
| { id: string; type: 'status'; text: string }
export type NewComputerUseTimelineItem = ComputerUseTimelineItem extends infer Item
? Item extends { id: string }
? Omit<Item, 'id'>
: never
: never

View File

@@ -185,12 +185,6 @@ export interface WebConfig {
ssl_key_path?: string;
}
export interface ComputerUseConfig {
enabled: boolean;
base_url: string;
model: string;
}
export interface TtydConfig {
enabled: boolean;
shell: string;
@@ -329,7 +323,7 @@ export interface AppConfig {
audio: AudioConfig;
stream: StreamConfig;
web: WebConfig;
computer_use: ComputerUseConfig;
mcp: McpConfig;
extensions: ExtensionsConfig;
rustdesk: RustDeskConfig;
vnc: VncConfig;
@@ -405,58 +399,6 @@ export interface Ch9329DescriptorState {
config_mode_available: boolean;
}
export interface ComputerUseConfigResponse {
enabled: boolean;
base_url: string;
model: string;
api_key_configured: boolean;
api_key_source: string;
}
export interface ComputerUseConfigUpdate {
enabled?: boolean;
base_url?: string;
model?: string;
api_key?: string;
clear_api_key?: boolean;
}
export interface ComputerUsePoint {
x: number;
y: number;
}
export interface ComputerUseScreenshot {
data_url: string;
width: number;
height: number;
}
export enum ComputerUseSessionStatus {
Idle = "idle",
WaitingScreenshot = "waiting_screenshot",
Thinking = "thinking",
Executing = "executing",
Completed = "completed",
Failed = "failed",
Stopped = "stopped",
}
export interface ComputerUseSessionSummary {
id?: string;
status: ComputerUseSessionStatus;
prompt?: string;
step: number;
last_error?: string;
final_message?: string;
}
export interface ComputerUseStartRequest {
prompt: string;
continue_conversation?: boolean;
client_id: string;
}
export interface EasytierConfigUpdate {
enabled?: boolean;
config_mode?: EasytierConfigMode;
@@ -898,8 +840,8 @@ export enum CanonicalKey {
MetaRight = "MetaRight",
}
export enum ComputerUseButton {
Left = "left",
Middle = "middle",
Right = "right",
export interface McpConfig {
enabled: boolean;
requests_per_second: number;
}

View File

@@ -11,14 +11,12 @@ import { useHidWebSocket } from '@/composables/useHidWebSocket'
import { useWebRTC } from '@/composables/useWebRTC'
import { useVideoSession } from '@/composables/useVideoSession'
import { useVideoScaling, type VideoRotation } from '@/composables/useVideoScaling'
import { useComputerUseSocket, type ComputerUseServerMessage } from '@/composables/useComputerUseSocket'
import { useFeatureVisibility } from '@/composables/useFeatureVisibility'
import { useTheme } from '@/composables/useTheme'
import { useConsoleLayout } from '@/composables/useConsoleLayout'
import { getUnifiedAudio } from '@/composables/useUnifiedAudio'
import { getMicrophone } from '@/composables/useMicrophone'
import { streamApi, hidApi, atxApi, atxConfigApi, authApi, computerUseApi, uacApi } from '@/api'
import type { ComputerUseScreenshot, ComputerUseSession } from '@/api'
import { streamApi, hidApi, atxApi, atxConfigApi, authApi, uacApi } from '@/api'
import { CanonicalKey, HidBackend } from '@/types/generated'
import type { HidKeyboardEvent, HidMouseEvent } from '@/types/hid'
import { keyboardEventToCanonicalKey, updateModifierMaskForKey } from '@/lib/keyboardMappings'
@@ -40,9 +38,7 @@ import ConsoleStatusSummary, { type ConsoleStatusItem } from '@/components/Conso
import InfoBar from '@/components/InfoBar.vue'
import VirtualKeyboard from '@/components/VirtualKeyboard.vue'
import StatsSheet from '@/components/StatsSheet.vue'
import ComputerUseSheet from '@/components/ComputerUseSheet.vue'
import TerminalDialog from '@/components/TerminalDialog.vue'
import type { ComputerUseTimelineItem, NewComputerUseTimelineItem } from '@/types/computerUseTimeline'
import LanguageToggleButton from '@/components/LanguageToggleButton.vue'
import BrandMark from '@/components/BrandMark.vue'
import { Button } from '@/components/ui/button'
@@ -119,11 +115,6 @@ function setVideoRotation(rotation: VideoRotation) {
videoRotation.value = rotation
localStorage.setItem('videoRotation', String(rotation))
}
const computerUseOpen = ref(false)
const computerUseSession = ref<ComputerUseSession | null>(null)
const computerUseTimeline = ref<ComputerUseTimelineItem[]>([])
const computerUseConversationStarted = ref(false)
let computerUseTimelineSeq = 0
const videoRef = ref<HTMLImageElement | null>(null)
const webrtcVideoRef = ref<HTMLVideoElement | null>(null)
@@ -166,11 +157,6 @@ const clientsStats = ref<Record<string, ClientStat>>({})
const myClientId = generateUUID()
const computerUseSocket = useComputerUseSocket({
onMessage: handleComputerUseMessage,
onScreenshotRequested: captureComputerUseFrame,
})
const mouseMode = ref<'absolute' | 'relative'>('absolute')
const pressedKeys = ref<CanonicalKey[]>([])
const keyboardLed = computed(() => ({
@@ -262,7 +248,6 @@ provideConsoleAppearance(consoleLayout)
const terminalAvailable = computed(() => ttydStatus.value?.available !== false)
const showPower = computed(() => featureVisibility.value.power)
const showTerminal = computed(() => terminalAvailable.value && featureVisibility.value.webTerminal)
const showComputerUse = computed(() => featureVisibility.value.computerUse)
const showPasteText = computed(() => featureVisibility.value.pasteText)
const videoStatus = computed<ConnectionStatus>(() => {
@@ -749,7 +734,6 @@ watch(videoLoading, (loading) => {
if (loading) clearVideoSourceSize()
})
const computerUsePanelVisible = computed(() => computerUseOpen.value && !isFullscreen.value)
const showMsdStatusCard = computed(() => {
return !!(systemStore.msd?.available && systemStore.hid?.backend !== 'ch9329')
@@ -813,140 +797,6 @@ async function captureFrameOverlay() {
}
}
async function captureComputerUseFrame(): Promise<ComputerUseScreenshot | null> {
try {
const canvas = document.createElement('canvas')
const ctx = canvas.getContext('2d')
if (!ctx) return null
const MAX_WIDTH = 1920
if (videoMode.value === 'mjpeg') {
const img = videoRef.value
if (!img || !img.naturalWidth || !img.naturalHeight) return null
const scale = Math.min(1, MAX_WIDTH / img.naturalWidth)
canvas.width = Math.max(1, Math.round(img.naturalWidth * scale))
canvas.height = Math.max(1, Math.round(img.naturalHeight * scale))
ctx.drawImage(img, 0, 0, canvas.width, canvas.height)
} else {
const video = webrtcVideoRef.value
if (!video || !video.videoWidth || !video.videoHeight) return null
const scale = Math.min(1, MAX_WIDTH / video.videoWidth)
canvas.width = Math.max(1, Math.round(video.videoWidth * scale))
canvas.height = Math.max(1, Math.round(video.videoHeight * scale))
ctx.drawImage(video, 0, 0, canvas.width, canvas.height)
}
return {
data_url: canvas.toDataURL('image/jpeg', 0.82),
width: canvas.width,
height: canvas.height,
}
} catch (err) {
console.error('[ComputerUse] Failed to capture frame:', err)
return null
}
}
function handleComputerUseMessage(message: ComputerUseServerMessage) {
switch (message.type) {
case 'session_updated':
computerUseSession.value = message.session
if (message.session.last_error) {
pushComputerUseTimeline({ type: 'error', text: message.session.last_error })
}
if (message.session.final_message) {
pushComputerUseTimeline({ type: 'assistant', text: message.session.final_message })
}
break
case 'screenshot_captured':
pushComputerUseTimeline({ type: 'screenshot', screenshot: message.screenshot })
break
case 'actions_executed':
pushComputerUseTimeline({ type: 'actions_executed', actions: message.actions })
break
case 'step_started':
pushComputerUseTimeline({ type: 'reasoning', text: '', completed: false, failed: false })
break
case 'reasoning_delta': {
const last = computerUseTimeline.value[computerUseTimeline.value.length - 1]
if (last?.type === 'reasoning' && !last.completed) {
last.text += message.delta
} else {
pushComputerUseTimeline({ type: 'reasoning', text: message.delta, completed: false, failed: false })
}
break
}
case 'reasoning_completed': {
for (let index = computerUseTimeline.value.length - 1; index >= 0; index -= 1) {
const item = computerUseTimeline.value[index]
if (item?.type !== 'reasoning' || item.completed) continue
if (!message.failed && !item.text) {
computerUseTimeline.value.splice(index, 1)
} else {
item.completed = true
item.failed = message.failed
}
break
}
break
}
case 'error':
pushComputerUseTimeline({ type: 'error', text: message.message })
toast.error(t('computerUse.errors.taskFailed'), { description: message.message })
break
}
}
function pushComputerUseTimeline(item: NewComputerUseTimelineItem) {
const last = computerUseTimeline.value[computerUseTimeline.value.length - 1]
if (last?.type === item.type) {
if ('text' in last && 'text' in item && last.text === item.text) return
if (last.type === 'actions_executed' && item.type === 'actions_executed' && JSON.stringify(last.actions) === JSON.stringify(item.actions)) return
}
computerUseTimeline.value.push({
id: `${Date.now()}-${computerUseTimelineSeq++}`,
...item,
} as ComputerUseTimelineItem)
}
function clearComputerUseTimeline() {
computerUseTimeline.value = []
computerUseConversationStarted.value = false
}
async function openComputerUse() {
if (!showComputerUse.value) return
computerUseOpen.value = true
await computerUseSocket.connect().catch(() => {})
computerUseSession.value = await computerUseApi.session().catch(() => computerUseSession.value)
}
async function startComputerUse(prompt: string) {
try {
await computerUseSocket.connect()
pushComputerUseTimeline({ type: 'user', text: prompt })
computerUseSession.value = await computerUseApi.start({
prompt,
continue_conversation: computerUseConversationStarted.value,
client_id: computerUseSocket.clientId,
})
computerUseConversationStarted.value = true
} catch (err: any) {
pushComputerUseTimeline({ type: 'error', text: err?.message ?? t('computerUse.errors.startFailed') })
toast.error(t('computerUse.errors.startFailed'), { description: err?.message })
}
}
async function stopComputerUse() {
try {
computerUseSession.value = await computerUseApi.stop()
} catch (err: any) {
toast.error(t('computerUse.errors.stopFailed'), { description: err?.message })
}
}
function waitForVideoFirstFrame(el: HTMLVideoElement, timeoutMs = 2000): Promise<boolean> {
return new Promise((resolve) => {
@@ -2004,9 +1854,6 @@ watch(showTerminal, (visible) => {
if (!visible) showTerminalDialog.value = false
})
watch(showComputerUse, (visible) => {
if (!visible) computerUseOpen.value = false
})
watch(() => webrtc.audioTrack.value, async (track) => {
videoDebugLog('WebRTC audio track ref changed', {
@@ -3299,10 +3146,10 @@ onUnmounted(() => {
:show-stats="showConnectionStats"
:show-terminal="showTerminal"
:terminal-running="!!ttydStatus?.running"
:show-computer-use="showComputerUse"
@open-stats="openStatsSheet"
@open-terminal="openTerminal"
@open-computer-use="openComputerUse"
/>
<div v-if="consoleLayout === 'current'" class="hidden md:flex items-center gap-2">
<StatusCard
@@ -3397,7 +3244,7 @@ onUnmounted(() => {
:show-power="showPower"
:atx-enabled="systemStore.atx?.available === true"
:show-terminal="showTerminal"
:show-computer-use="showComputerUse"
:show-paste-text="showPasteText"
:show-mic="microphoneTransferEnabled"
:scale-mode="videoScaleMode"
@@ -3414,7 +3261,7 @@ onUnmounted(() => {
@reset="handleReset"
@wol="handleWol"
@open-terminal="openTerminal"
@open-computer-use="openComputerUse"
/>
</div>
<div
@@ -3425,7 +3272,6 @@ onUnmounted(() => {
<div class="relative flex h-full w-full min-w-0 items-stretch gap-3">
<div
class="flex min-w-0 flex-1 items-center justify-center transition-all duration-300"
:class="{ 'md:pr-1': computerUsePanelVisible }"
>
<div
ref="videoWorkspaceRef"
@@ -3635,17 +3481,7 @@ onUnmounted(() => {
</div>
</div>
</div>
<ComputerUseSheet
v-if="showComputerUse"
v-model:open="computerUseOpen"
:connected="computerUseSocket.connected.value"
:ws-error="computerUseSocket.error.value"
:session="computerUseSession"
:timeline="computerUseTimeline"
@start="startComputerUse"
@stop="stopComputerUse"
@clear="clearComputerUseTimeline"
/>
</div>
</div>
<Teleport :to="virtualKeyboardAttached ? '#keyboard-anchor' : 'body'" :disabled="virtualKeyboardAttached">

View File

@@ -1,4 +1,5 @@
<script setup lang="ts">
import McpSettingsCard from '@/components/McpSettingsCard.vue'
import HidDeviceSettings from '@/components/HidDeviceSettings.vue'
import ConsoleLayoutPreview from '@/components/ConsoleLayoutPreview.vue'
import { ref, computed, onMounted, watch, nextTick } from 'vue'
@@ -192,6 +193,7 @@ const SETTINGS_SECTION_IDS = [
'other',
'ext-ttyd',
'third-party-access',
'mcp',
'ext-remote-access',
'about',
] as const
@@ -225,6 +227,7 @@ const navGroups = computed(() => [
title: t('settings.extensions'),
items: [
{ id: 'ext-ttyd', label: t('extensions.ttyd.title'), icon: Terminal },
{ id: 'mcp', label: t('mcp.title'), icon: Bot },
{ id: 'third-party-access', label: t('extensions.thirdPartyAccess.title'), icon: ScreenShare },
{ id: 'ext-remote-access', label: t('extensions.remoteAccess.title'), icon: ExternalLink },
]
@@ -2942,18 +2945,13 @@ watch(isWindows, () => {
</Label>
<Switch id="feature-web-terminal" v-model="featureVisibility.webTerminal" />
</div>
<div class="flex items-center justify-between gap-4 px-3 py-3">
<Label for="feature-computer-use" class="flex min-w-0 items-center gap-2 font-normal">
<Bot class="size-4 shrink-0 text-muted-foreground" />
<span class="truncate">{{ t('settings.computerUseAgent') }}</span>
</Label>
<Switch id="feature-computer-use" v-model="featureVisibility.computerUse" />
</div>
</CardContent>
</Card>
</div>
<!-- Account Section -->
<McpSettingsCard v-if="activeSection === 'mcp'" />
<div v-show="activeSection === 'account'" class="space-y-4">
<Card>
<CardHeader>