GithubFollow/One-KVM
1
0
Fork 0
mirror of https://github.com/mofeng-git/One-KVM.git synced 2026-07-29 12:41:45 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: 优化 CUA 功能体验

Browse Source
This commit is contained in:
mofeng-git
2026-07-17 16:32:17 +08:00
parent 4031abd8d6
commit fdea52d59d
13 changed files with 1744 additions and 717 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
src/computer_use/actions.rs
View File

@@ -104,19 +104,14 @@ pub struct ComputerUseStartRequest {
#[serde(default)]
pub continue_conversation: bool,
pub client_id: String,
pub max_steps: Option<u32>,
pub timeout_seconds: Option<u32>,
}
#[typeshare]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ComputerUseConfigResponse {
pub enabled: bool,
pub provider: String,
pub base_url: String,
pub model: String,
pub max_steps: u32,
pub timeout_seconds: u32,
pub api_key_configured: bool,
pub api_key_source: String,
}
@@ -127,10 +122,10 @@ pub struct ComputerUseConfigUpdate {
pub enabled: Option<bool>,
pub base_url: Option<String>,
pub model: Option<String>,
pub max_steps: Option<u32>,
pub timeout_seconds: Option<u32>,
pub openai_api_key: Option<String>,
pub clear_openai_api_key: Option<bool>,
#[serde(alias = "openai_api_key")]
pub api_key: Option<String>,
#[serde(alias = "clear_openai_api_key")]
pub clear_api_key: Option<bool>,
}
#[typeshare]
@@ -140,7 +135,6 @@ pub struct ComputerUseSessionSummary {
pub status: ComputerUseSessionStatus,
pub prompt: Option<String>,
pub step: u32,
pub max_steps: u32,
pub last_error: Option<String>,
pub final_message: Option<String>,
}
@@ -152,6 +146,10 @@ pub enum ComputerUseWsClientMessage {
request_id: String,
screenshot: ComputerUseScreenshot,
},
ScreenshotError {
request_id: String,
message: String,
},
}
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -161,6 +159,8 @@ pub enum ComputerUseWsServerMessage {
ScreenshotRequested { request_id: String },
ScreenshotCaptured { screenshot: ComputerUseScreenshot },
StepStarted { step: u32 },
ReasoningDelta { delta: String },
ReasoningCompleted { failed: bool },
ActionsExecuted { actions: Vec<ComputerUseAction> },
Error { message: String },
}
@@ -203,4 +203,16 @@ mod tests {
})
);
}
#[test]
fn config_update_accepts_legacy_api_key_names() {
let update: ComputerUseConfigUpdate = serde_json::from_value(json!({
"openai_api_key": "legacy-key",
"clear_openai_api_key": true
}))
.unwrap();
assert_eq!(update.api_key.as_deref(), Some("legacy-key"));
assert_eq!(update.clear_api_key, Some(true));
}
}

250
src/computer_use/manager.rs
View File

@@ -1,9 +1,8 @@
use std::sync::Arc;
use std::time::{Duration, Instant};
use std::time::Duration;
use axum::extract::ws::{Message, WebSocket};
use futures::{SinkExt, StreamExt};
use serde_json::Value;
use tokio::sync::{broadcast, oneshot, watch, Mutex};
use tokio::task::JoinHandle;
use uuid::Uuid;
@@ -43,7 +42,7 @@ struct ManagerState {
struct ScreenshotWaiter {
request_id: String,
client_id: String,
tx: oneshot::Sender<ComputerUseScreenshot>,
tx: oneshot::Sender<Result<ComputerUseScreenshot>>,
}
#[derive(Debug, Clone)]
@@ -74,24 +73,16 @@ impl ComputerUseManager {
pub fn config_response(&self) -> ComputerUseConfigResponse {
let config = self.config.get();
let key_env = std::env::var("OPENAI_API_KEY")
.ok()
.filter(|key| !key.is_empty());
let key_env = cua_api_key_env();
let key_db = config
.computer_use
.openai_api_key
.api_key
.as_ref()
.filter(|key| !key.is_empty());
ComputerUseConfigResponse {
enabled: config.computer_use.enabled,
provider: config.computer_use.provider.clone(),
base_url: std::env::var("ONE_KVM_OPENAI_BASE_URL")
.ok()
.filter(|url| !url.trim().is_empty())
.unwrap_or_else(|| config.computer_use.base_url.clone()),
base_url: cua_base_url_env().unwrap_or_else(|| config.computer_use.base_url.clone()),
model: config.computer_use.model.clone(),
max_steps: config.computer_use.max_steps,
timeout_seconds: config.computer_use.timeout_seconds,
api_key_configured: key_env.is_some() || key_db.is_some(),
api_key_source: if key_env.is_some() {
"env".to_string()
@@ -107,7 +98,6 @@ impl ComputerUseManager {
&self,
req: ComputerUseConfigUpdate,
) -> Result<ComputerUseConfigResponse> {
validate_limits(req.max_steps, req.timeout_seconds)?;
if let Some(base_url) = req
.base_url
.as_ref()
@@ -131,17 +121,11 @@ impl ComputerUseManager {
{
config.computer_use.base_url = base_url.trim().to_string();
}
if let Some(max_steps) = req.max_steps {
config.computer_use.max_steps = max_steps;
if req.clear_api_key.unwrap_or(false) {
config.computer_use.api_key = None;
}
if let Some(timeout_seconds) = req.timeout_seconds {
config.computer_use.timeout_seconds = timeout_seconds;
}
if req.clear_openai_api_key.unwrap_or(false) {
config.computer_use.openai_api_key = None;
}
if let Some(key) = req.openai_api_key.as_ref() {
config.computer_use.openai_api_key = if key.trim().is_empty() {
if let Some(key) = req.api_key.as_ref() {
config.computer_use.api_key = if key.trim().is_empty() {
None
} else {
Some(key.trim().to_string())
@@ -169,7 +153,6 @@ impl ComputerUseManager {
if req.prompt.trim().is_empty() {
return Err(AppError::BadRequest("Task prompt is required".to_string()));
}
validate_limits(req.max_steps, req.timeout_seconds)?;
let client_id = req.client_id.trim();
if client_id.is_empty() {
return Err(AppError::BadRequest(
@@ -184,15 +167,12 @@ impl ComputerUseManager {
));
}
let api_key = std::env::var("OPENAI_API_KEY")
.ok()
.filter(|key| !key.is_empty())
.or(config.openai_api_key.clone())
.ok_or_else(|| AppError::BadRequest("OpenAI API key is not configured".to_string()))?;
let base_url = std::env::var("ONE_KVM_OPENAI_BASE_URL")
.ok()
.filter(|url| !url.trim().is_empty())
.unwrap_or_else(|| config.base_url.clone());
let api_key = cua_api_key_env()
.or(config.api_key.clone())
.ok_or_else(|| {
AppError::BadRequest("Computer Use API key is not configured".to_string())
})?;
let base_url = cua_base_url_env().unwrap_or_else(|| config.base_url.clone());
validate_endpoint_url(&base_url)?;
let mut state = self.state.lock().await;
@@ -225,10 +205,9 @@ impl ComputerUseManager {
let session_id = Uuid::new_v4().to_string();
state.session = ComputerUseSessionSummary {
id: Some(session_id),
status: ComputerUseSessionStatus::WaitingScreenshot,
status: ComputerUseSessionStatus::Thinking,
prompt: Some(req.prompt.trim().to_string()),
step: 0,
max_steps: req.max_steps.unwrap_or(config.max_steps),
last_error: None,
final_message: None,
};
@@ -240,9 +219,6 @@ impl ComputerUseManager {
self.publish_session().await;
let manager = self.clone();
let prompt = req.prompt.trim().to_string();
let max_steps = summary.max_steps;
let timeout =
Duration::from_secs(req.timeout_seconds.unwrap_or(config.timeout_seconds) as u64);
let model = config.model.clone();
let handle = tokio::spawn(async move {
manager
@@ -253,8 +229,6 @@ impl ComputerUseManager {
model,
conversation,
client_id,
max_steps,
timeout,
cancel_rx,
stop_rx,
)
@@ -304,10 +278,30 @@ impl ComputerUseManager {
state.screenshot_waiter = Some(waiter);
return Ok(());
}
let _ = waiter.tx.send(screenshot);
let _ = waiter.tx.send(Ok(screenshot));
Ok(())
}
async fn submit_screenshot_error(&self, client_id: &str, request_id: String, message: String) {
let mut state = self.state.lock().await;
let Some(waiter) = state.screenshot_waiter.take() else {
return;
};
if waiter.request_id != request_id || waiter.client_id != client_id {
state.screenshot_waiter = Some(waiter);
return;
}
let message: String = message.chars().take(300).collect();
let _ = waiter.tx.send(Err(AppError::ServiceUnavailable(format!(
"Screenshot capture failed: {}",
if message.trim().is_empty() {
"client did not provide an error"
} else {
message.trim()
}
))));
}
pub async fn handle_socket(self: Arc<Self>, socket: WebSocket, client_id: Option<String>) {
let (mut sender, mut receiver) = socket.split();
let mut event_rx = self.event_tx.subscribe();
@@ -352,11 +346,15 @@ impl ComputerUseManager {
msg = receiver.next() => {
match msg {
Some(Ok(Message::Text(text))) => {
if let Ok(ComputerUseWsClientMessage::ScreenshotResult { request_id, screenshot }) =
serde_json::from_str::<ComputerUseWsClientMessage>(&text)
{
match serde_json::from_str::<ComputerUseWsClientMessage>(&text) {
Ok(ComputerUseWsClientMessage::ScreenshotResult { request_id, screenshot }) => {
let _ = self.submit_screenshot(&client_id, request_id, screenshot).await;
}
Ok(ComputerUseWsClientMessage::ScreenshotError { request_id, message }) => {
self.submit_screenshot_error(&client_id, request_id, message).await;
}
Err(_) => {}
}
}
Some(Ok(Message::Close(_))) | None => break,
Some(Err(_)) => break,
@@ -375,84 +373,81 @@ impl ComputerUseManager {
model: String,
conversation: Vec<ComputerUseConversationMessage>,
client_id: String,
max_steps: u32,
timeout: Duration,
cancel_rx: watch::Receiver<bool>,
mut stop_rx: oneshot::Receiver<()>,
) {
let provider = OpenAiComputerProvider::new(api_key, base_url, model);
let started_at = Instant::now();
let mut previous_response_id: Option<String> = None;
let mut previous_call_id: Option<String> = None;
let mut safety_checks: Vec<Value> = Vec::new();
for step in 1..=max_steps {
if started_at.elapsed() > timeout {
self.fail("Computer use task timed out").await;
return;
}
self.set_status(ComputerUseSessionStatus::WaitingScreenshot, step, None)
.await;
let screenshot = tokio::select! {
_ = &mut stop_rx => {
self.set_stopped().await;
return;
}
screenshot = self.request_screenshot(&client_id) => screenshot,
};
let screenshot = match screenshot {
Ok(screenshot) => screenshot,
Err(err) => {
self.fail(&err.to_string()).await;
return;
}
};
let _ = self
.event_tx
.send(ComputerUseWsServerMessage::ScreenshotCaptured {
screenshot: screenshot.clone(),
});
let mut latest_screenshot: Option<ComputerUseScreenshot> = None;
let mut action_history: Vec<String> = Vec::new();
let mut step = 0_u32;
loop {
step = step.saturating_add(1);
self.set_status(ComputerUseSessionStatus::Thinking, step, None)
.await;
let response = tokio::select! {
_ = &mut stop_rx => {
let _ = self.event_tx.send(ComputerUseWsServerMessage::ReasoningCompleted {
failed: true,
});
self.set_stopped().await;
return;
}
response = provider.next_actions(
&prompt,
&conversation,
&screenshot,
previous_response_id.as_deref(),
previous_call_id.as_deref(),
safety_checks.clone(),
&action_history,
latest_screenshot.as_ref(),
|delta| {
let _ = self.event_tx.send(ComputerUseWsServerMessage::ReasoningDelta {
delta: delta.to_string(),
});
},
) => response,
};
let response = match response {
Ok(response) => response,
Ok(response) => {
let _ = self
.event_tx
.send(ComputerUseWsServerMessage::ReasoningCompleted { failed: false });
response
}
Err(err) => {
let _ = self
.event_tx
.send(ComputerUseWsServerMessage::ReasoningCompleted { failed: true });
self.fail(&err.to_string()).await;
return;
}
};
previous_response_id = response.response_id;
previous_call_id = response.call_id;
safety_checks = response.safety_checks;
if response.actions.is_empty() {
self.complete(response.final_message).await;
if *cancel_rx.borrow() {
self.set_stopped().await;
return;
}
if response.done {
self.complete(response.message).await;
return;
}
let executable = &response.actions[..response.actions.len().saturating_sub(1)];
action_history.push(format!(
"Step {step}: {}",
serde_json::to_string(&response.actions).unwrap_or_else(|_| "[]".to_string())
));
if !executable.is_empty() {
let Some(screenshot) = latest_screenshot.as_ref() else {
self.fail("Computer Use protocol error: actions require a screenshot")
.await;
return;
};
self.set_status(ComputerUseSessionStatus::Executing, step, None)
.await;
if let Err(err) = self
.execute_actions(
&response.actions,
executable,
screenshot.width,
screenshot.height,
cancel_rx.clone(),
@@ -469,12 +464,33 @@ impl ComputerUseManager {
let _ = self
.event_tx
.send(ComputerUseWsServerMessage::ActionsExecuted {
actions: response.actions,
actions: executable.to_vec(),
});
}
self.complete(Some("Reached the maximum number of steps.".to_string()))
self.set_status(ComputerUseSessionStatus::WaitingScreenshot, step, None)
.await;
let screenshot = tokio::select! {
_ = &mut stop_rx => {
self.set_stopped().await;
return;
}
screenshot = self.request_screenshot(&client_id) => screenshot,
};
let screenshot = match screenshot {
Ok(screenshot) => screenshot,
Err(err) => {
self.fail(&err.to_string()).await;
return;
}
};
let _ = self
.event_tx
.send(ComputerUseWsServerMessage::ScreenshotCaptured {
screenshot: screenshot.clone(),
});
latest_screenshot = Some(screenshot);
}
}
async fn request_screenshot(&self, client_id: &str) -> Result<ComputerUseScreenshot> {
@@ -492,14 +508,15 @@ impl ComputerUseManager {
request_id,
client_id: client_id.to_string(),
});
tokio::time::timeout(SCREENSHOT_TIMEOUT, rx)
let reply = tokio::time::timeout(SCREENSHOT_TIMEOUT, rx)
.await
.map_err(|_| {
AppError::ServiceUnavailable("Timed out waiting for screenshot".to_string())
})?
.map_err(|_| {
AppError::ServiceUnavailable("Screenshot request was cancelled".to_string())
})
})?;
reply
}
async fn execute_actions(
@@ -742,36 +759,39 @@ fn stopped_error() -> AppError {
AppError::BadRequest(STOPPED_MESSAGE.to_string())
}
fn validate_limits(max_steps: Option<u32>, timeout_seconds: Option<u32>) -> Result<()> {
if let Some(max_steps) = max_steps {
if !(1..=100).contains(&max_steps) {
return Err(AppError::BadRequest(
"max_steps must be between 1 and 100".to_string(),
));
}
}
if let Some(timeout_seconds) = timeout_seconds {
if !(30..=3600).contains(&timeout_seconds) {
return Err(AppError::BadRequest(
"timeout_seconds must be between 30 and 3600".to_string(),
));
}
}
Ok(())
}
fn empty_session() -> ComputerUseSessionSummary {
ComputerUseSessionSummary {
id: None,
status: ComputerUseSessionStatus::Idle,
prompt: None,
step: 0,
max_steps: 0,
last_error: None,
final_message: None,
}
}
fn cua_api_key_env() -> Option<String> {
std::env::var("ONE_KVM_CUA_API_KEY")
.ok()
.filter(|key| !key.trim().is_empty())
.or_else(|| {
std::env::var("OPENAI_API_KEY")
.ok()
.filter(|key| !key.trim().is_empty())
})
}
fn cua_base_url_env() -> Option<String> {
std::env::var("ONE_KVM_CUA_BASE_URL")
.ok()
.filter(|url| !url.trim().is_empty())
.or_else(|| {
std::env::var("ONE_KVM_OPENAI_BASE_URL")
.ok()
.filter(|url| !url.trim().is_empty())
})
}
fn validate_endpoint_url(url: &str) -> Result<()> {
let trimmed = url.trim();
if !(trimmed.starts_with("https://") || trimmed.starts_with("http://")) {

1316
src/computer_use/openai.rs
View File

File diff suppressed because it is too large Load Diff

33
src/config/schema/computer_use.rs
View File

@@ -6,25 +6,42 @@ use typeshare::typeshare;
#[serde(default)]
pub struct ComputerUseConfig {
pub enabled: bool,
pub provider: String,
pub base_url: String,
pub model: String,
#[typeshare(skip)]
pub openai_api_key: Option<String>,
pub max_steps: u32,
pub timeout_seconds: u32,
#[serde(alias = "openai_api_key")]
pub api_key: Option<String>,
}
impl Default for ComputerUseConfig {
fn default() -> Self {
Self {
enabled: false,
provider: "openai".to_string(),
base_url: "https://api.openai.com/v1/responses".to_string(),
model: "gpt-5.5".to_string(),
openai_api_key: None,
max_steps: 30,
timeout_seconds: 600,
api_key: None,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn legacy_openai_api_key_migrates_to_generic_key() {
let config: ComputerUseConfig = serde_json::from_value(serde_json::json!({
"enabled": true,
"provider": "openai",
"base_url": "https://example.test/v1/chat/completions",
"model": "vision-model",
"openai_api_key": "legacy-key",
"max_steps": 30,
"timeout_seconds": 600
}))
.unwrap();
assert_eq!(config.api_key.as_deref(), Some("legacy-key"));
assert_eq!(config.model, "vision-model");
}
}

2
src/web/handlers/config/mod.rs
View File

@@ -57,7 +57,7 @@ fn sanitize_config_for_api(config: &mut AppConfig) {
config.auth.totp_secret = None;
config.stream.turn_password = None;
config.computer_use.openai_api_key = None;
config.computer_use.api_key = None;
config.rustdesk.device_password.clear();
config.rustdesk.relay_key = None;

2
web/src/components/ActionBar.vue
View File

@@ -409,7 +409,7 @@ const hasRightOverflow = computed(() => {
</Button>
</TooltipTrigger>
<TooltipContent>
<p>Computer Use</p>
<p>{{ t('computerUse.title') }}</p>
</TooltipContent>
</Tooltip>
</TooltipProvider>

413
web/src/components/ComputerUseSheet.vue
View File

@@ -1,6 +1,21 @@
<script setup lang="ts">
import { computed, nextTick, onMounted, ref, watch } from 'vue'
import { Bot, ChevronDown, Image, KeyRound, Play, Square } from 'lucide-vue-next'
import { useI18n } from 'vue-i18n'
import {
Bot,
BrainCircuit,
ChevronDown,
ChevronRight,
Image,
KeyRound,
SendHorizontal,
Settings,
Square,
Trash2,
Wifi,
WifiOff,
X,
} from 'lucide-vue-next'
import { toast } from 'vue-sonner'
import { computerUseApi, type ComputerUseAction, type ComputerUseConfig, type ComputerUseSession } from '@/api'
import type { ComputerUseTimelineItem } from '@/types/computerUseTimeline'
@@ -8,9 +23,14 @@ import { Button } from '@/components/ui/button'
import { Input } from '@/components/ui/input'
import { Label } from '@/components/ui/label'
import { Textarea } from '@/components/ui/textarea'
import { Badge } from '@/components/ui/badge'
import { Switch } from '@/components/ui/switch'
import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs'
import {
Dialog,
DialogContent,
DialogFooter,
DialogHeader,
DialogTitle,
} from '@/components/ui/dialog'
const props = defineProps<{
open: boolean
@@ -20,6 +40,8 @@ const props = defineProps<{
timeline: ComputerUseTimelineItem[]
}>()
const { t } = useI18n()
const emit = defineEmits<{
(e: 'update:open', value: boolean): void
(e: 'start', prompt: string): void
@@ -32,8 +54,9 @@ const prompt = ref('')
const apiKey = ref('')
const savingConfig = ref(false)
const starting = ref(false)
const activeTab = ref('chat')
const settingsOpen = ref(false)
const messagesRef = ref<HTMLDivElement | null>(null)
const reasoningOverrides = ref<Record<string, boolean>>({})
const defaultModel = computed({
get: () => config.value?.model ?? 'gpt-5.5',
@@ -47,53 +70,62 @@ const defaultBaseUrl = computed({
if (config.value) config.value.base_url = value
},
})
const defaultMaxSteps = computed({
get: () => String(config.value?.max_steps ?? 30),
set: (value: string) => {
if (config.value) config.value.max_steps = Number(value) || 30
},
})
const defaultTimeoutSeconds = computed({
get: () => String(config.value?.timeout_seconds ?? 600),
set: (value: string) => {
if (config.value) config.value.timeout_seconds = Number(value) || 600
},
})
const status = computed(() => props.session?.status ?? 'idle')
const isRunning = computed(() => ['waiting_screenshot', 'thinking', 'executing'].includes(status.value))
const canStart = computed(() => !!config.value?.enabled && !!config.value?.api_key_configured && prompt.value.trim().length > 0 && !isRunning.value)
const showWelcome = computed(() => props.timeline.length === 0 && !props.session?.last_error && !props.session?.final_message)
const canStart = computed(() => (
!!config.value?.enabled
&& !!config.value?.api_key_configured
&& prompt.value.trim().length > 0
&& !isRunning.value
))
const statusLabel = computed(() => {
switch (status.value) {
case 'waiting_screenshot': return '截屏中'
case 'thinking': return '思考中'
case 'executing': return '执行中'
case 'completed': return '已完成'
case 'failed': return '失败'
case 'stopped': return '已停止'
default: return '空闲'
case 'waiting_screenshot': return t('computerUse.status.waitingScreenshot')
case 'thinking': return t('computerUse.status.thinking')
case 'executing': return t('computerUse.status.executing')
case 'completed': return t('computerUse.status.completed')
case 'failed': return t('computerUse.status.failed')
case 'stopped': return t('computerUse.status.stopped')
default: return t('computerUse.status.idle')
}
})
const apiKeyPlaceholder = computed(() => {
if (!config.value?.api_key_configured) return t('computerUse.settings.apiKey')
const source = config.value.api_key_source
const sourceLabel = source === 'env'
? t('computerUse.settings.sourceEnv')
: source === 'config'
? t('computerUse.settings.sourceConfig')
: t('computerUse.settings.sourceNone')
return t('computerUse.settings.configured', { source: sourceLabel })
})
async function loadConfig() {
try {
config.value = await computerUseApi.config()
} catch (err) {
toast.error(t('computerUse.errors.configLoadFailed'), {
description: err instanceof Error ? err.message : undefined,
})
}
}
async function saveConfig() {
if (!config.value) return
savingConfig.value = true
try {
config.value = await computerUseApi.updateConfig({
enabled: config.value?.enabled ?? true,
base_url: config.value?.base_url || 'https://api.openai.com/v1/responses',
model: config.value?.model || 'gpt-5.5',
max_steps: config.value?.max_steps || 30,
timeout_seconds: config.value?.timeout_seconds || 600,
openai_api_key: apiKey.value.trim() || undefined,
enabled: config.value.enabled,
base_url: config.value.base_url,
model: config.value.model,
api_key: apiKey.value.trim() || undefined,
})
apiKey.value = ''
toast.success('Computer Use 配置已保存')
settingsOpen.value = false
toast.success(t('computerUse.success.configSaved'))
} finally {
savingConfig.value = false
}
@@ -102,60 +134,75 @@ async function saveConfig() {
async function clearApiKey() {
savingConfig.value = true
try {
config.value = await computerUseApi.updateConfig({
clear_openai_api_key: true,
})
config.value = await computerUseApi.updateConfig({ clear_api_key: true })
apiKey.value = ''
toast.success('OpenAI API Key 已清除')
toast.success(t('computerUse.success.apiKeyCleared'))
} finally {
savingConfig.value = false
}
}
async function start() {
function start() {
if (!canStart.value) return
const text = prompt.value.trim()
starting.value = true
try {
emit('start', text)
prompt.value = ''
} finally {
starting.value = false
}
}
function formatAction(action: ComputerUseAction): string {
switch (action.type) {
case 'click':
return `点击 (${action.x}, ${action.y}) ${action.button ?? 'left'}`
return t('computerUse.actions.click', {
x: action.x,
y: action.y,
button: t(`computerUse.mouseButtons.${action.button ?? 'left'}`),
})
case 'double_click':
return `双击 (${action.x}, ${action.y}) ${action.button ?? 'left'}`
return t('computerUse.actions.doubleClick', {
x: action.x,
y: action.y,
button: t(`computerUse.mouseButtons.${action.button ?? 'left'}`),
})
case 'move':
return `移动到 (${action.x}, ${action.y})`
return t('computerUse.actions.move', { x: action.x, y: action.y })
case 'drag':
return `拖拽 ${action.path.length} 个点`
return t('computerUse.actions.drag', { count: action.path.length })
case 'scroll':
return `滚动 (${action.x}, ${action.y}) dx=${action.dx ?? 0} dy=${action.dy ?? 0}`
return t('computerUse.actions.scroll', {
x: action.x,
y: action.y,
dx: action.dx ?? 0,
dy: action.dy ?? 0,
})
case 'type':
return `输入 ${action.text.length} 字符`
return t('computerUse.actions.typeAscii', { count: action.text.length })
case 'keypress':
return `按键 ${action.keys.join('+')}`
return t('computerUse.actions.keypress', { keys: action.keys.join('+') })
case 'wait':
return `等待 ${action.ms}ms`
return t('computerUse.actions.wait', { ms: action.ms })
case 'screenshot':
return '请求截图'
return t('computerUse.actions.screenshot')
}
}
function reasoningOpen(item: Extract<ComputerUseTimelineItem, { type: 'reasoning' }>) {
return reasoningOverrides.value[item.id] ?? !item.completed
}
function toggleReasoning(item: Extract<ComputerUseTimelineItem, { type: 'reasoning' }>) {
reasoningOverrides.value[item.id] = !reasoningOpen(item)
}
function scrollToBottom() {
nextTick(() => {
const el = messagesRef.value
if (!el) return
el.scrollTop = el.scrollHeight
if (el) el.scrollTop = el.scrollHeight
})
}
watch(() => props.timeline.length, scrollToBottom)
watch(() => props.timeline, scrollToBottom, { deep: true })
watch(() => props.open, (open) => {
if (open) scrollToBottom()
})
@@ -166,80 +213,132 @@ onMounted(loadConfig)
<template>
<aside
v-show="open"
class="absolute inset-y-0 right-0 z-30 h-full min-h-0 w-[min(100%,420px)] border-l bg-background/98 shadow-xl backdrop-blur md:relative md:z-auto md:w-[420px] xl:w-[460px]"
class="absolute inset-y-0 right-0 z-30 h-full min-h-0 w-full border-l bg-background shadow-xl sm:w-[420px] md:relative md:z-auto xl:w-[460px]"
>
<div class="flex h-full min-h-0 flex-col">
<div class="flex h-12 shrink-0 items-center justify-between border-b px-3">
<div class="flex min-w-0 items-center gap-2">
<header class="flex h-12 shrink-0 items-center gap-2 border-b px-3">
<Bot class="h-5 w-5 shrink-0" />
<div class="min-w-0">
<div class="truncate text-sm font-semibold">Computer Use</div>
<div class="truncate text-[11px] text-muted-foreground">
WebSocket {{ connected ? '已连接' : '未连接' }}
<span v-if="wsError"> · {{ wsError }}</span>
<div class="min-w-0 flex-1">
<div class="truncate text-sm font-semibold">{{ t('computerUse.title') }}</div>
<div
class="flex min-w-0 items-center gap-1 text-[11px]"
:class="connected ? 'text-emerald-600 dark:text-emerald-400' : 'text-muted-foreground'"
>
<Wifi v-if="connected" class="h-3 w-3 shrink-0" />
<WifiOff v-else class="h-3 w-3 shrink-0" />
<span class="truncate">
{{ connected ? t('computerUse.connection.connected') : (wsError || t('computerUse.connection.disconnected')) }}
</span>
<span class="text-muted-foreground">· {{ statusLabel }}</span>
</div>
</div>
</div>
<div class="flex items-center gap-1.5">
<Badge :variant="status === 'failed' ? 'destructive' : 'secondary'">
{{ statusLabel }}
</Badge>
<Button variant="ghost" size="icon" class="h-8 w-8" @click="emit('update:open', false)">
<ChevronDown class="h-4 w-4 rotate-90" />
<Button
variant="ghost"
size="icon"
class="h-8 w-8"
:title="t('computerUse.buttons.settings')"
:aria-label="t('computerUse.buttons.settings')"
@click="settingsOpen = true"
>
<Settings class="h-4 w-4" />
</Button>
</div>
</div>
<Tabs v-model="activeTab" class="flex min-h-0 flex-1 flex-col">
<div class="px-3 py-2">
<TabsList class="grid w-full grid-cols-2">
<TabsTrigger value="chat">对话</TabsTrigger>
<TabsTrigger value="settings">设置</TabsTrigger>
</TabsList>
</div>
<TabsContent value="chat" class="m-0 flex min-h-0 flex-1 flex-col data-[state=inactive]:hidden">
<div ref="messagesRef" class="min-h-0 flex-1 space-y-3 overflow-y-auto p-3">
<div v-if="showWelcome" class="rounded-md border border-dashed p-4 text-center text-xs text-muted-foreground">
发送任务后，这里会显示对话、截图和坐标操作。
<Button
variant="ghost"
size="icon"
class="h-8 w-8"
:title="t('computerUse.buttons.clear')"
:aria-label="t('computerUse.buttons.clear')"
:disabled="isRunning || timeline.length === 0"
@click="emit('clear')"
>
<Trash2 class="h-4 w-4" />
</Button>
<Button
variant="ghost"
size="icon"
class="h-8 w-8"
:title="t('computerUse.buttons.close')"
:aria-label="t('computerUse.buttons.close')"
@click="emit('update:open', false)"
>
<X class="h-4 w-4" />
</Button>
</header>
<div ref="messagesRef" class="min-h-0 flex-1 overflow-y-auto px-4 py-4">
<div v-if="timeline.length === 0" class="flex h-full items-center justify-center text-muted-foreground/45">
<Bot class="h-10 w-10" />
</div>
<div v-else class="space-y-5">
<template v-for="item in timeline" :key="item.id">
<div v-if="item.type === 'user'" class="flex justify-end">
<div class="max-w-[86%] rounded-md bg-primary px-3 py-2 text-sm text-primary-foreground">
<div class="max-w-[86%] whitespace-pre-wrap break-words rounded-md bg-muted px-3 py-2 text-sm">
{{ item.text }}
</div>
</div>
<div v-else-if="item.type === 'assistant'" class="flex justify-start">
<div class="max-w-[86%] rounded-md border bg-muted/50 px-3 py-2 text-sm">
<div v-else-if="item.type === 'assistant'" class="whitespace-pre-wrap break-words text-sm leading-6">
{{ item.text }}
</div>
<div v-else-if="item.type === 'reasoning'" class="border-l-2 border-muted pl-3 text-xs text-muted-foreground">
<button
type="button"
class="flex w-full items-center gap-1.5 py-1 text-left font-medium hover:text-foreground"
@click="toggleReasoning(item)"
>
<ChevronDown v-if="reasoningOpen(item)" class="h-3.5 w-3.5 shrink-0" />
<ChevronRight v-else class="h-3.5 w-3.5 shrink-0" />
<BrainCircuit class="h-3.5 w-3.5 shrink-0" />
<span>
{{ item.failed
? t('computerUse.reasoning.failed')
: (item.completed ? t('computerUse.reasoning.process') : t('computerUse.reasoning.thinking')) }}
</span>
</button>
<div
v-if="reasoningOpen(item)"
class="max-h-72 overflow-y-auto whitespace-pre-wrap break-words pb-2 pt-1 leading-5"
>
{{ item.text || t('computerUse.reasoning.thinking') }}
</div>
</div>
<div v-else-if="item.type === 'screenshot'" class="rounded-md border bg-card p-2">
<div class="mb-2 flex items-center justify-between text-xs text-muted-foreground">
<span class="inline-flex items-center gap-1.5"><Image class="h-3.5 w-3.5" />截图</span>
<div v-else-if="item.type === 'screenshot'" class="overflow-hidden rounded-md border bg-card">
<div class="flex items-center justify-between border-b px-2.5 py-1.5 text-[11px] text-muted-foreground">
<span class="inline-flex items-center gap-1.5">
<Image class="h-3.5 w-3.5" />{{ t('computerUse.trace.screenshot') }}
</span>
<span>{{ item.screenshot.width }}x{{ item.screenshot.height }}</span>
</div>
<div
class="w-full overflow-hidden rounded-sm bg-black"
class="w-full bg-black"
:style="{ aspectRatio: `${item.screenshot.width} / ${item.screenshot.height}` }"
>
<img :src="item.screenshot.data_url" class="h-full w-full object-cover" alt="Computer Use screenshot" />
<img
:src="item.screenshot.data_url"
class="h-full w-full object-contain"
:alt="t('computerUse.trace.screenshotAlt')"
/>
</div>
</div>
<div v-else-if="item.type === 'actions_executed'" class="rounded-md border border-success/35 bg-success/10 p-2 text-success">
<div class="mb-2 text-xs font-medium">已执行</div>
<div class="space-y-1">
<div v-for="(action, index) in item.actions" :key="index" class="rounded-sm bg-background/60 px-2 py-1.5 text-xs">
<div v-else-if="item.type === 'actions_executed'" class="border-l-2 border-emerald-500/50 pl-3 text-xs">
<div class="mb-1 text-[11px] font-medium text-emerald-600 dark:text-emerald-400">
{{ t('computerUse.trace.executed') }}
</div>
<div class="space-y-1 text-muted-foreground">
<div v-for="(action, index) in item.actions" :key="index" class="break-words">
{{ formatAction(action) }}
</div>
</div>
</div>
<div v-else-if="item.type === 'error'" class="rounded-md border border-destructive/40 bg-destructive/10 px-3 py-2 text-xs text-destructive">
<div
v-else-if="item.type === 'error'"
class="whitespace-pre-wrap break-words rounded-md border border-destructive/40 bg-destructive/10 px-3 py-2 text-xs leading-5 text-destructive"
>
{{ item.text }}
</div>
@@ -248,91 +347,107 @@ onMounted(loadConfig)
</div>
</template>
</div>
</div>
<div class="shrink-0 border-t p-3">
<footer class="shrink-0 border-t bg-background p-3">
<div class="relative">
<Textarea
v-model="prompt"
rows="3"
placeholder="继续输入任务或追问"
class="max-h-40 min-h-20 resize-none pr-12"
:placeholder="t('computerUse.input.placeholder')"
:disabled="isRunning"
@keydown.meta.enter.prevent="start"
@keydown.ctrl.enter.prevent="start"
/>
<div class="mt-2 flex gap-2">
<Button class="flex-1 gap-2" :disabled="!canStart || starting" @click="start">
<Play class="h-4 w-4" />
发送
<Button
v-if="!isRunning"
size="icon"
class="absolute bottom-2 right-2 h-8 w-8"
:title="t('computerUse.buttons.send')"
:aria-label="t('computerUse.buttons.send')"
:disabled="!canStart || starting"
@click="start"
>
<SendHorizontal class="h-4 w-4" />
</Button>
<Button variant="outline" class="gap-2" :disabled="!isRunning" @click="emit('stop')">
<Square class="h-4 w-4" />
停止
</Button>
<Button variant="ghost" size="sm" :disabled="isRunning || timeline.length === 0" @click="emit('clear')">
清空
<Button
v-else
size="icon"
variant="destructive"
class="absolute bottom-2 right-2 h-8 w-8"
:title="t('computerUse.buttons.stop')"
:aria-label="t('computerUse.buttons.stop')"
@click="emit('stop')"
>
<Square class="h-3.5 w-3.5 fill-current" />
</Button>
</div>
<p v-if="!config?.api_key_configured" class="mt-2 text-xs text-muted-foreground">
需要先在设置里保存 OpenAI API Key。
<p v-if="config && !config.api_key_configured" class="mt-2 text-xs text-muted-foreground">
{{ t('computerUse.input.apiKeyRequired') }}
</p>
</footer>
</div>
</TabsContent>
<TabsContent value="settings" class="m-0 min-h-0 flex-1 overflow-y-auto p-3 data-[state=inactive]:hidden">
<div class="space-y-4">
<div class="flex items-center justify-between rounded-md border p-3">
<div>
<div class="text-sm font-medium">启用 AI 操作</div>
<div class="text-xs text-muted-foreground">配置保存后立即生效</div>
</div>
<Dialog v-model:open="settingsOpen">
<DialogContent class="max-w-md">
<DialogHeader>
<DialogTitle>{{ t('computerUse.settings.title') }}</DialogTitle>
</DialogHeader>
<div class="space-y-4 py-2">
<div class="flex items-center justify-between gap-4">
<Label for="cua-enabled">{{ t('computerUse.settings.enableAi') }}</Label>
<Switch
id="cua-enabled"
:model-value="config?.enabled ?? false"
@update:model-value="(value) => { if (config) config.enabled = value }"
/>
</div>
<div class="space-y-3 rounded-md border p-3">
<div class="grid grid-cols-2 gap-2">
<div class="space-y-1">
<Label class="text-xs">模型</Label>
<Input v-model="defaultModel" :disabled="!config" placeholder="gpt-5.5" />
<div class="space-y-1.5">
<Label for="cua-model">{{ t('computerUse.settings.model') }}</Label>
<Input id="cua-model" v-model="defaultModel" :disabled="!config" placeholder="gpt-5.5" />
</div>
<div class="space-y-1">
<Label class="text-xs">最大步数</Label>
<Input v-model="defaultMaxSteps" type="number" min="1" max="100" />
<div class="space-y-1.5">
<Label for="cua-url">{{ t('computerUse.settings.apiUrl') }}</Label>
<Input
id="cua-url"
v-model="defaultBaseUrl"
:disabled="!config"
placeholder="https://api.example.com/v1/responses"
/>
</div>
</div>
<div class="space-y-1">
<Label class="text-xs">超时秒数</Label>
<Input v-model="defaultTimeoutSeconds" type="number" min="30" max="3600" />
</div>
<div class="space-y-1">
<Label class="text-xs">API URL</Label>
<Input v-model="defaultBaseUrl" :disabled="!config" placeholder="https://api.openai.com/v1/responses" />
</div>
<div class="space-y-1">
<Label class="text-xs flex items-center gap-1">
<div class="space-y-1.5">
<Label for="cua-key" class="flex items-center gap-1.5">
<KeyRound class="h-3.5 w-3.5" />
OpenAI API Key
{{ t('computerUse.settings.apiKey') }}
</Label>
<Input
id="cua-key"
v-model="apiKey"
type="password"
autocomplete="off"
:placeholder="config?.api_key_configured ? `已配置：${config.api_key_source}` : 'sk-...'"
:placeholder="apiKeyPlaceholder"
/>
</div>
<div class="grid grid-cols-2 gap-2">
<Button size="sm" :disabled="savingConfig || !config" @click="saveConfig">
保存配置
</div>
<DialogFooter class="gap-2 sm:justify-between">
<Button
variant="outline"
:disabled="savingConfig || !config?.api_key_configured"
@click="clearApiKey"
>
{{ t('computerUse.buttons.clearKey') }}
</Button>
<Button size="sm" variant="outline" :disabled="savingConfig || !config?.api_key_configured" @click="clearApiKey">
清除 Key
<Button :disabled="savingConfig || !config" @click="saveConfig">
{{ t('computerUse.buttons.save') }}
</Button>
</div>
</div>
</div>
</TabsContent>
</Tabs>
</div>
</DialogFooter>
</DialogContent>
</Dialog>
</aside>
</template>

23
web/src/composables/useComputerUseSocket.ts
View File

@@ -1,4 +1,5 @@
import { ref, onUnmounted } from 'vue'
import { useI18n } from 'vue-i18n'
import { buildWsUrl } from '@/types/websocket'
import { generateUUID } from '@/lib/utils'
import type { ComputerUseScreenshot, ComputerUseSession, ComputerUseAction } from '@/api'
@@ -8,6 +9,8 @@ export type ComputerUseServerMessage =
| { type: 'screenshot_requested'; request_id: string }
| { type: 'screenshot_captured'; screenshot: ComputerUseScreenshot }
| { type: 'step_started'; step: number }
| { type: 'reasoning_delta'; delta: string }
| { type: 'reasoning_completed'; failed: boolean }
| { type: 'actions_executed'; actions: ComputerUseAction[] }
| { type: 'error'; message: string }
@@ -15,6 +18,7 @@ export function useComputerUseSocket(options: {
onMessage: (message: ComputerUseServerMessage) => void
onScreenshotRequested: (requestId: string) => Promise<ComputerUseScreenshot | null>
}) {
const { t } = useI18n()
const connected = ref(false)
const error = ref<string | null>(null)
const clientId = generateUUID()
@@ -29,7 +33,7 @@ export function useComputerUseSocket(options: {
connectPromise = new Promise((resolve, reject) => {
if (!ws) {
reject(new Error('Computer use WebSocket failed'))
reject(new Error(t('computerUse.connection.socketFailed')))
return
}
@@ -41,7 +45,7 @@ export function useComputerUseSocket(options: {
}
ws.onerror = () => {
error.value = 'Computer use WebSocket failed'
error.value = t('computerUse.connection.socketFailed')
connectPromise = null
reject(new Error(error.value))
}
@@ -57,17 +61,28 @@ export function useComputerUseSocket(options: {
const message = JSON.parse(event.data) as ComputerUseServerMessage
options.onMessage(message)
if (message.type === 'screenshot_requested') {
try {
const screenshot = await options.onScreenshotRequested(message.request_id)
if (screenshot && ws?.readyState === WebSocket.OPEN) {
if (!screenshot) throw new Error(t('computerUse.errors.captureUnavailable'))
if (ws?.readyState === WebSocket.OPEN) {
ws.send(JSON.stringify({
type: 'screenshot_result',
request_id: message.request_id,
screenshot,
}))
}
} catch (err) {
if (ws?.readyState === WebSocket.OPEN) {
ws.send(JSON.stringify({
type: 'screenshot_error',
request_id: message.request_id,
message: err instanceof Error ? err.message : t('computerUse.errors.screenshotFailed'),
}))
}
}
}
} catch (err) {
console.error('[ComputerUse] Failed to handle WS message:', err)
console.error(`[ComputerUse] ${t('computerUse.connection.messageFailed')}:`, err)
}
}

80
web/src/i18n/en-US.ts
View File

@@ -751,6 +751,86 @@ export default {
loadDevicesFailed: 'Failed to load device list',
updateFailed: 'Update failed',
},
computerUse: {
title: 'Computer Use',
status: {
waitingScreenshot: 'Capturing screen',
thinking: 'Thinking',
executing: 'Executing',
completed: 'Completed',
failed: 'Failed',
stopped: 'Stopped',
idle: 'Idle',
},
connection: {
connected: 'Connected',
disconnected: 'Disconnected',
socketFailed: 'Computer Use WebSocket connection failed',
messageFailed: 'Failed to process Computer Use WebSocket message',
},
buttons: {
settings: 'Settings',
clear: 'Clear',
close: 'Close',
send: 'Send',
stop: 'Stop',
save: 'Save',
clearKey: 'Clear Key',
},
reasoning: {
failed: 'Thinking failed',
process: 'Reasoning',
thinking: 'Thinking',
},
trace: {
screenshot: 'Screenshot',
screenshotAlt: 'Computer Use screenshot',
executed: 'Executed',
},
input: {
placeholder: 'Enter a task. Press Enter for a new line; press Ctrl+Enter to send.',
apiKeyRequired: 'Configure an API Key first.',
},
settings: {
title: 'Computer Use Settings',
enableAi: 'Enable AI control',
model: 'Model',
apiUrl: 'API URL',
apiKey: 'API Key',
configured: 'Configured: {source}',
sourceEnv: 'environment variable',
sourceConfig: 'configuration file',
sourceNone: 'not configured',
},
actions: {
click: 'Click ({x}, {y}) {button}',
doubleClick: 'Double-click ({x}, {y}) {button}',
move: 'Move to ({x}, {y})',
drag: 'Drag through {count} points',
scroll: 'Scroll at ({x}, {y}) dx={dx} dy={dy}',
typeAscii: 'Type {count} ASCII characters',
keypress: 'Press {keys}',
wait: 'Wait {ms}ms',
screenshot: 'Request screenshot',
},
mouseButtons: {
left: 'left button',
middle: 'middle button',
right: 'right button',
},
errors: {
configLoadFailed: 'Failed to load Computer Use settings',
taskFailed: 'Computer Use task failed',
startFailed: 'Failed to start Computer Use',
stopFailed: 'Failed to stop Computer Use',
captureUnavailable: 'The current video frame is unavailable',
screenshotFailed: 'Screenshot capture failed',
},
success: {
configSaved: 'Computer Use settings saved',
apiKeyCleared: 'API Key cleared',
},
},
statusCard: {
device: 'Device',
video: 'Video',

80
web/src/i18n/zh-CN.ts
View File

@@ -750,6 +750,86 @@ export default {
loadDevicesFailed: '加载设备列表失败',
updateFailed: '更新失败',
},
computerUse: {
title: 'Computer Use',
status: {
waitingScreenshot: '截屏中',
thinking: '思考中',
executing: '执行中',
completed: '已完成',
failed: '失败',
stopped: '已停止',
idle: '空闲',
},
connection: {
connected: '已连接',
disconnected: '未连接',
socketFailed: 'Computer Use WebSocket 连接失败',
messageFailed: '处理 Computer Use WebSocket 消息失败',
},
buttons: {
settings: '设置',
clear: '清空',
close: '关闭',
send: '发送',
stop: '停止',
save: '保存',
clearKey: '清除 Key',
},
reasoning: {
failed: '思考失败',
process: '思考过程',
thinking: '思考中',
},
trace: {
screenshot: '截图',
screenshotAlt: 'Computer Use 截图',
executed: '已执行',
},
input: {
placeholder: '输入任务，按下回车换行，按下 Ctrl+Enter 发送',
apiKeyRequired: '请先配置 API Key。',
},
settings: {
title: 'Computer Use 设置',
enableAi: '启用 AI 操作',
model: '模型',
apiUrl: 'API URL',
apiKey: 'API Key',
configured: '已配置：{source}',
sourceEnv: '环境变量',
sourceConfig: '配置文件',
sourceNone: '未配置',
},
actions: {
click: '点击 ({x}, {y}) {button}',
doubleClick: '双击 ({x}, {y}) {button}',
move: '移动到 ({x}, {y})',
drag: '拖拽 {count} 个点',
scroll: '滚动 ({x}, {y}) dx={dx} dy={dy}',
typeAscii: '输入 {count} 个 ASCII 字符',
keypress: '按键 {keys}',
wait: '等待 {ms}ms',
screenshot: '请求截图',
},
mouseButtons: {
left: '左键',
middle: '中键',
right: '右键',
},
errors: {
configLoadFailed: '无法加载 Computer Use 配置',
taskFailed: 'Computer Use 任务失败',
startFailed: 'Computer Use 启动失败',
stopFailed: 'Computer Use 停止失败',
captureUnavailable: '当前视频画面尚不可用',
screenshotFailed: '截图失败',
},
success: {
configSaved: 'Computer Use 配置已保存',
apiKeyCleared: 'API Key 已清除',
},
},
statusCard: {
device: '设备',
video: '视频',

1
web/src/types/computerUseTimeline.ts
View File

@@ -3,6 +3,7 @@ import type { ComputerUseAction, ComputerUseScreenshot } from '@/api'
export type ComputerUseTimelineItem =
| { id: string; type: 'user'; text: string }
| { id: string; type: 'assistant'; text: string }
| { id: string; type: 'reasoning'; text: string; completed: boolean; failed: boolean }
| { id: string; type: 'screenshot'; screenshot: ComputerUseScreenshot }
| { id: string; type: 'actions_executed'; actions: ComputerUseAction[] }
| { id: string; type: 'error'; text: string }

15
web/src/types/generated.ts
View File

@@ -178,11 +178,8 @@ export interface WebConfig {
export interface ComputerUseConfig {
enabled: boolean;
provider: string;
base_url: string;
model: string;
max_steps: number;
timeout_seconds: number;
}
export interface TtydConfig {
@@ -381,11 +378,8 @@ export interface Ch9329DescriptorState {
export interface ComputerUseConfigResponse {
enabled: boolean;
provider: string;
base_url: string;
model: string;
max_steps: number;
timeout_seconds: number;
api_key_configured: boolean;
api_key_source: string;
}
@@ -394,10 +388,8 @@ export interface ComputerUseConfigUpdate {
enabled?: boolean;
base_url?: string;
model?: string;
max_steps?: number;
timeout_seconds?: number;
openai_api_key?: string;
clear_openai_api_key?: boolean;
api_key?: string;
clear_api_key?: boolean;
}
export interface ComputerUsePoint {
@@ -426,7 +418,6 @@ export interface ComputerUseSessionSummary {
status: ComputerUseSessionStatus;
prompt?: string;
step: number;
max_steps: number;
last_error?: string;
final_message?: string;
}
@@ -435,8 +426,6 @@ export interface ComputerUseStartRequest {
prompt: string;
continue_conversation?: boolean;
client_id: string;
max_steps?: number;
timeout_seconds?: number;
}
export interface EasytierConfigUpdate {

34
web/src/views/ConsoleView.vue
View File

@@ -791,9 +791,35 @@ function handleComputerUseMessage(message: ComputerUseServerMessage) {
case 'actions_executed':
pushComputerUseTimeline({ type: 'actions_executed', actions: message.actions })
break
case 'step_started':
pushComputerUseTimeline({ type: 'reasoning', text: '', completed: false, failed: false })
break
case 'reasoning_delta': {
const last = computerUseTimeline.value[computerUseTimeline.value.length - 1]
if (last?.type === 'reasoning' && !last.completed) {
last.text += message.delta
} else {
pushComputerUseTimeline({ type: 'reasoning', text: message.delta, completed: false, failed: false })
}
break
}
case 'reasoning_completed': {
for (let index = computerUseTimeline.value.length - 1; index >= 0; index -= 1) {
const item = computerUseTimeline.value[index]
if (item?.type !== 'reasoning' || item.completed) continue
if (!message.failed && !item.text) {
computerUseTimeline.value.splice(index, 1)
} else {
item.completed = true
item.failed = message.failed
}
break
}
break
}
case 'error':
pushComputerUseTimeline({ type: 'error', text: message.message })
toast.error('Computer Use failed', { description: message.message })
toast.error(t('computerUse.errors.taskFailed'), { description: message.message })
break
}
}
@@ -833,8 +859,8 @@ async function startComputerUse(prompt: string) {
})
computerUseConversationStarted.value = true
} catch (err: any) {
pushComputerUseTimeline({ type: 'error', text: err?.message ?? 'Computer Use start failed' })
toast.error('Computer Use start failed', { description: err?.message })
pushComputerUseTimeline({ type: 'error', text: err?.message ?? t('computerUse.errors.startFailed') })
toast.error(t('computerUse.errors.startFailed'), { description: err?.message })
}
}
@@ -842,7 +868,7 @@ async function stopComputerUse() {
try {
computerUseSession.value = await computerUseApi.stop()
} catch (err: any) {
toast.error('Computer Use stop failed', { description: err?.message })
toast.error(t('computerUse.errors.stopFailed'), { description: err?.message })
}
}