feat: 优化 CUA 功能体验

This commit is contained in:
mofeng-git
2026-07-17 16:32:17 +08:00
parent 4031abd8d6
commit fdea52d59d
13 changed files with 1744 additions and 717 deletions

View File

@@ -104,19 +104,14 @@ pub struct ComputerUseStartRequest {
#[serde(default)] #[serde(default)]
pub continue_conversation: bool, pub continue_conversation: bool,
pub client_id: String, pub client_id: String,
pub max_steps: Option<u32>,
pub timeout_seconds: Option<u32>,
} }
#[typeshare] #[typeshare]
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ComputerUseConfigResponse { pub struct ComputerUseConfigResponse {
pub enabled: bool, pub enabled: bool,
pub provider: String,
pub base_url: String, pub base_url: String,
pub model: String, pub model: String,
pub max_steps: u32,
pub timeout_seconds: u32,
pub api_key_configured: bool, pub api_key_configured: bool,
pub api_key_source: String, pub api_key_source: String,
} }
@@ -127,10 +122,10 @@ pub struct ComputerUseConfigUpdate {
pub enabled: Option<bool>, pub enabled: Option<bool>,
pub base_url: Option<String>, pub base_url: Option<String>,
pub model: Option<String>, pub model: Option<String>,
pub max_steps: Option<u32>, #[serde(alias = "openai_api_key")]
pub timeout_seconds: Option<u32>, pub api_key: Option<String>,
pub openai_api_key: Option<String>, #[serde(alias = "clear_openai_api_key")]
pub clear_openai_api_key: Option<bool>, pub clear_api_key: Option<bool>,
} }
#[typeshare] #[typeshare]
@@ -140,7 +135,6 @@ pub struct ComputerUseSessionSummary {
pub status: ComputerUseSessionStatus, pub status: ComputerUseSessionStatus,
pub prompt: Option<String>, pub prompt: Option<String>,
pub step: u32, pub step: u32,
pub max_steps: u32,
pub last_error: Option<String>, pub last_error: Option<String>,
pub final_message: Option<String>, pub final_message: Option<String>,
} }
@@ -152,6 +146,10 @@ pub enum ComputerUseWsClientMessage {
request_id: String, request_id: String,
screenshot: ComputerUseScreenshot, screenshot: ComputerUseScreenshot,
}, },
ScreenshotError {
request_id: String,
message: String,
},
} }
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -161,6 +159,8 @@ pub enum ComputerUseWsServerMessage {
ScreenshotRequested { request_id: String }, ScreenshotRequested { request_id: String },
ScreenshotCaptured { screenshot: ComputerUseScreenshot }, ScreenshotCaptured { screenshot: ComputerUseScreenshot },
StepStarted { step: u32 }, StepStarted { step: u32 },
ReasoningDelta { delta: String },
ReasoningCompleted { failed: bool },
ActionsExecuted { actions: Vec<ComputerUseAction> }, ActionsExecuted { actions: Vec<ComputerUseAction> },
Error { message: String }, Error { message: String },
} }
@@ -203,4 +203,16 @@ mod tests {
}) })
); );
} }
#[test]
fn config_update_accepts_legacy_api_key_names() {
let update: ComputerUseConfigUpdate = serde_json::from_value(json!({
"openai_api_key": "legacy-key",
"clear_openai_api_key": true
}))
.unwrap();
assert_eq!(update.api_key.as_deref(), Some("legacy-key"));
assert_eq!(update.clear_api_key, Some(true));
}
} }

View File

@@ -1,9 +1,8 @@
use std::sync::Arc; use std::sync::Arc;
use std::time::{Duration, Instant}; use std::time::Duration;
use axum::extract::ws::{Message, WebSocket}; use axum::extract::ws::{Message, WebSocket};
use futures::{SinkExt, StreamExt}; use futures::{SinkExt, StreamExt};
use serde_json::Value;
use tokio::sync::{broadcast, oneshot, watch, Mutex}; use tokio::sync::{broadcast, oneshot, watch, Mutex};
use tokio::task::JoinHandle; use tokio::task::JoinHandle;
use uuid::Uuid; use uuid::Uuid;
@@ -43,7 +42,7 @@ struct ManagerState {
struct ScreenshotWaiter { struct ScreenshotWaiter {
request_id: String, request_id: String,
client_id: String, client_id: String,
tx: oneshot::Sender<ComputerUseScreenshot>, tx: oneshot::Sender<Result<ComputerUseScreenshot>>,
} }
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
@@ -74,24 +73,16 @@ impl ComputerUseManager {
pub fn config_response(&self) -> ComputerUseConfigResponse { pub fn config_response(&self) -> ComputerUseConfigResponse {
let config = self.config.get(); let config = self.config.get();
let key_env = std::env::var("OPENAI_API_KEY") let key_env = cua_api_key_env();
.ok()
.filter(|key| !key.is_empty());
let key_db = config let key_db = config
.computer_use .computer_use
.openai_api_key .api_key
.as_ref() .as_ref()
.filter(|key| !key.is_empty()); .filter(|key| !key.is_empty());
ComputerUseConfigResponse { ComputerUseConfigResponse {
enabled: config.computer_use.enabled, enabled: config.computer_use.enabled,
provider: config.computer_use.provider.clone(), base_url: cua_base_url_env().unwrap_or_else(|| config.computer_use.base_url.clone()),
base_url: std::env::var("ONE_KVM_OPENAI_BASE_URL")
.ok()
.filter(|url| !url.trim().is_empty())
.unwrap_or_else(|| config.computer_use.base_url.clone()),
model: config.computer_use.model.clone(), model: config.computer_use.model.clone(),
max_steps: config.computer_use.max_steps,
timeout_seconds: config.computer_use.timeout_seconds,
api_key_configured: key_env.is_some() || key_db.is_some(), api_key_configured: key_env.is_some() || key_db.is_some(),
api_key_source: if key_env.is_some() { api_key_source: if key_env.is_some() {
"env".to_string() "env".to_string()
@@ -107,7 +98,6 @@ impl ComputerUseManager {
&self, &self,
req: ComputerUseConfigUpdate, req: ComputerUseConfigUpdate,
) -> Result<ComputerUseConfigResponse> { ) -> Result<ComputerUseConfigResponse> {
validate_limits(req.max_steps, req.timeout_seconds)?;
if let Some(base_url) = req if let Some(base_url) = req
.base_url .base_url
.as_ref() .as_ref()
@@ -131,17 +121,11 @@ impl ComputerUseManager {
{ {
config.computer_use.base_url = base_url.trim().to_string(); config.computer_use.base_url = base_url.trim().to_string();
} }
if let Some(max_steps) = req.max_steps { if req.clear_api_key.unwrap_or(false) {
config.computer_use.max_steps = max_steps; config.computer_use.api_key = None;
} }
if let Some(timeout_seconds) = req.timeout_seconds { if let Some(key) = req.api_key.as_ref() {
config.computer_use.timeout_seconds = timeout_seconds; config.computer_use.api_key = if key.trim().is_empty() {
}
if req.clear_openai_api_key.unwrap_or(false) {
config.computer_use.openai_api_key = None;
}
if let Some(key) = req.openai_api_key.as_ref() {
config.computer_use.openai_api_key = if key.trim().is_empty() {
None None
} else { } else {
Some(key.trim().to_string()) Some(key.trim().to_string())
@@ -169,7 +153,6 @@ impl ComputerUseManager {
if req.prompt.trim().is_empty() { if req.prompt.trim().is_empty() {
return Err(AppError::BadRequest("Task prompt is required".to_string())); return Err(AppError::BadRequest("Task prompt is required".to_string()));
} }
validate_limits(req.max_steps, req.timeout_seconds)?;
let client_id = req.client_id.trim(); let client_id = req.client_id.trim();
if client_id.is_empty() { if client_id.is_empty() {
return Err(AppError::BadRequest( return Err(AppError::BadRequest(
@@ -184,15 +167,12 @@ impl ComputerUseManager {
)); ));
} }
let api_key = std::env::var("OPENAI_API_KEY") let api_key = cua_api_key_env()
.ok() .or(config.api_key.clone())
.filter(|key| !key.is_empty()) .ok_or_else(|| {
.or(config.openai_api_key.clone()) AppError::BadRequest("Computer Use API key is not configured".to_string())
.ok_or_else(|| AppError::BadRequest("OpenAI API key is not configured".to_string()))?; })?;
let base_url = std::env::var("ONE_KVM_OPENAI_BASE_URL") let base_url = cua_base_url_env().unwrap_or_else(|| config.base_url.clone());
.ok()
.filter(|url| !url.trim().is_empty())
.unwrap_or_else(|| config.base_url.clone());
validate_endpoint_url(&base_url)?; validate_endpoint_url(&base_url)?;
let mut state = self.state.lock().await; let mut state = self.state.lock().await;
@@ -225,10 +205,9 @@ impl ComputerUseManager {
let session_id = Uuid::new_v4().to_string(); let session_id = Uuid::new_v4().to_string();
state.session = ComputerUseSessionSummary { state.session = ComputerUseSessionSummary {
id: Some(session_id), id: Some(session_id),
status: ComputerUseSessionStatus::WaitingScreenshot, status: ComputerUseSessionStatus::Thinking,
prompt: Some(req.prompt.trim().to_string()), prompt: Some(req.prompt.trim().to_string()),
step: 0, step: 0,
max_steps: req.max_steps.unwrap_or(config.max_steps),
last_error: None, last_error: None,
final_message: None, final_message: None,
}; };
@@ -240,9 +219,6 @@ impl ComputerUseManager {
self.publish_session().await; self.publish_session().await;
let manager = self.clone(); let manager = self.clone();
let prompt = req.prompt.trim().to_string(); let prompt = req.prompt.trim().to_string();
let max_steps = summary.max_steps;
let timeout =
Duration::from_secs(req.timeout_seconds.unwrap_or(config.timeout_seconds) as u64);
let model = config.model.clone(); let model = config.model.clone();
let handle = tokio::spawn(async move { let handle = tokio::spawn(async move {
manager manager
@@ -253,8 +229,6 @@ impl ComputerUseManager {
model, model,
conversation, conversation,
client_id, client_id,
max_steps,
timeout,
cancel_rx, cancel_rx,
stop_rx, stop_rx,
) )
@@ -304,10 +278,30 @@ impl ComputerUseManager {
state.screenshot_waiter = Some(waiter); state.screenshot_waiter = Some(waiter);
return Ok(()); return Ok(());
} }
let _ = waiter.tx.send(screenshot); let _ = waiter.tx.send(Ok(screenshot));
Ok(()) Ok(())
} }
async fn submit_screenshot_error(&self, client_id: &str, request_id: String, message: String) {
let mut state = self.state.lock().await;
let Some(waiter) = state.screenshot_waiter.take() else {
return;
};
if waiter.request_id != request_id || waiter.client_id != client_id {
state.screenshot_waiter = Some(waiter);
return;
}
let message: String = message.chars().take(300).collect();
let _ = waiter.tx.send(Err(AppError::ServiceUnavailable(format!(
"Screenshot capture failed: {}",
if message.trim().is_empty() {
"client did not provide an error"
} else {
message.trim()
}
))));
}
pub async fn handle_socket(self: Arc<Self>, socket: WebSocket, client_id: Option<String>) { pub async fn handle_socket(self: Arc<Self>, socket: WebSocket, client_id: Option<String>) {
let (mut sender, mut receiver) = socket.split(); let (mut sender, mut receiver) = socket.split();
let mut event_rx = self.event_tx.subscribe(); let mut event_rx = self.event_tx.subscribe();
@@ -352,11 +346,15 @@ impl ComputerUseManager {
msg = receiver.next() => { msg = receiver.next() => {
match msg { match msg {
Some(Ok(Message::Text(text))) => { Some(Ok(Message::Text(text))) => {
if let Ok(ComputerUseWsClientMessage::ScreenshotResult { request_id, screenshot }) = match serde_json::from_str::<ComputerUseWsClientMessage>(&text) {
serde_json::from_str::<ComputerUseWsClientMessage>(&text) Ok(ComputerUseWsClientMessage::ScreenshotResult { request_id, screenshot }) => {
{
let _ = self.submit_screenshot(&client_id, request_id, screenshot).await; let _ = self.submit_screenshot(&client_id, request_id, screenshot).await;
} }
Ok(ComputerUseWsClientMessage::ScreenshotError { request_id, message }) => {
self.submit_screenshot_error(&client_id, request_id, message).await;
}
Err(_) => {}
}
} }
Some(Ok(Message::Close(_))) | None => break, Some(Ok(Message::Close(_))) | None => break,
Some(Err(_)) => break, Some(Err(_)) => break,
@@ -375,84 +373,81 @@ impl ComputerUseManager {
model: String, model: String,
conversation: Vec<ComputerUseConversationMessage>, conversation: Vec<ComputerUseConversationMessage>,
client_id: String, client_id: String,
max_steps: u32,
timeout: Duration,
cancel_rx: watch::Receiver<bool>, cancel_rx: watch::Receiver<bool>,
mut stop_rx: oneshot::Receiver<()>, mut stop_rx: oneshot::Receiver<()>,
) { ) {
let provider = OpenAiComputerProvider::new(api_key, base_url, model); let provider = OpenAiComputerProvider::new(api_key, base_url, model);
let started_at = Instant::now(); let mut latest_screenshot: Option<ComputerUseScreenshot> = None;
let mut previous_response_id: Option<String> = None; let mut action_history: Vec<String> = Vec::new();
let mut previous_call_id: Option<String> = None; let mut step = 0_u32;
let mut safety_checks: Vec<Value> = Vec::new();
for step in 1..=max_steps {
if started_at.elapsed() > timeout {
self.fail("Computer use task timed out").await;
return;
}
self.set_status(ComputerUseSessionStatus::WaitingScreenshot, step, None)
.await;
let screenshot = tokio::select! {
_ = &mut stop_rx => {
self.set_stopped().await;
return;
}
screenshot = self.request_screenshot(&client_id) => screenshot,
};
let screenshot = match screenshot {
Ok(screenshot) => screenshot,
Err(err) => {
self.fail(&err.to_string()).await;
return;
}
};
let _ = self
.event_tx
.send(ComputerUseWsServerMessage::ScreenshotCaptured {
screenshot: screenshot.clone(),
});
loop {
step = step.saturating_add(1);
self.set_status(ComputerUseSessionStatus::Thinking, step, None) self.set_status(ComputerUseSessionStatus::Thinking, step, None)
.await; .await;
let response = tokio::select! { let response = tokio::select! {
_ = &mut stop_rx => { _ = &mut stop_rx => {
let _ = self.event_tx.send(ComputerUseWsServerMessage::ReasoningCompleted {
failed: true,
});
self.set_stopped().await; self.set_stopped().await;
return; return;
} }
response = provider.next_actions( response = provider.next_actions(
&prompt, &prompt,
&conversation, &conversation,
&screenshot, &action_history,
previous_response_id.as_deref(), latest_screenshot.as_ref(),
previous_call_id.as_deref(), |delta| {
safety_checks.clone(), let _ = self.event_tx.send(ComputerUseWsServerMessage::ReasoningDelta {
delta: delta.to_string(),
});
},
) => response, ) => response,
}; };
let response = match response { let response = match response {
Ok(response) => response, Ok(response) => {
let _ = self
.event_tx
.send(ComputerUseWsServerMessage::ReasoningCompleted { failed: false });
response
}
Err(err) => { Err(err) => {
let _ = self
.event_tx
.send(ComputerUseWsServerMessage::ReasoningCompleted { failed: true });
self.fail(&err.to_string()).await; self.fail(&err.to_string()).await;
return; return;
} }
}; };
previous_response_id = response.response_id;
previous_call_id = response.call_id;
safety_checks = response.safety_checks;
if response.actions.is_empty() { if *cancel_rx.borrow() {
self.complete(response.final_message).await; self.set_stopped().await;
return; return;
} }
if response.done {
self.complete(response.message).await;
return;
}
let executable = &response.actions[..response.actions.len().saturating_sub(1)];
action_history.push(format!(
"Step {step}: {}",
serde_json::to_string(&response.actions).unwrap_or_else(|_| "[]".to_string())
));
if !executable.is_empty() {
let Some(screenshot) = latest_screenshot.as_ref() else {
self.fail("Computer Use protocol error: actions require a screenshot")
.await;
return;
};
self.set_status(ComputerUseSessionStatus::Executing, step, None) self.set_status(ComputerUseSessionStatus::Executing, step, None)
.await; .await;
if let Err(err) = self if let Err(err) = self
.execute_actions( .execute_actions(
&response.actions, executable,
screenshot.width, screenshot.width,
screenshot.height, screenshot.height,
cancel_rx.clone(), cancel_rx.clone(),
@@ -469,12 +464,33 @@ impl ComputerUseManager {
let _ = self let _ = self
.event_tx .event_tx
.send(ComputerUseWsServerMessage::ActionsExecuted { .send(ComputerUseWsServerMessage::ActionsExecuted {
actions: response.actions, actions: executable.to_vec(),
}); });
} }
self.complete(Some("Reached the maximum number of steps.".to_string())) self.set_status(ComputerUseSessionStatus::WaitingScreenshot, step, None)
.await; .await;
let screenshot = tokio::select! {
_ = &mut stop_rx => {
self.set_stopped().await;
return;
}
screenshot = self.request_screenshot(&client_id) => screenshot,
};
let screenshot = match screenshot {
Ok(screenshot) => screenshot,
Err(err) => {
self.fail(&err.to_string()).await;
return;
}
};
let _ = self
.event_tx
.send(ComputerUseWsServerMessage::ScreenshotCaptured {
screenshot: screenshot.clone(),
});
latest_screenshot = Some(screenshot);
}
} }
async fn request_screenshot(&self, client_id: &str) -> Result<ComputerUseScreenshot> { async fn request_screenshot(&self, client_id: &str) -> Result<ComputerUseScreenshot> {
@@ -492,14 +508,15 @@ impl ComputerUseManager {
request_id, request_id,
client_id: client_id.to_string(), client_id: client_id.to_string(),
}); });
tokio::time::timeout(SCREENSHOT_TIMEOUT, rx) let reply = tokio::time::timeout(SCREENSHOT_TIMEOUT, rx)
.await .await
.map_err(|_| { .map_err(|_| {
AppError::ServiceUnavailable("Timed out waiting for screenshot".to_string()) AppError::ServiceUnavailable("Timed out waiting for screenshot".to_string())
})? })?
.map_err(|_| { .map_err(|_| {
AppError::ServiceUnavailable("Screenshot request was cancelled".to_string()) AppError::ServiceUnavailable("Screenshot request was cancelled".to_string())
}) })?;
reply
} }
async fn execute_actions( async fn execute_actions(
@@ -742,36 +759,39 @@ fn stopped_error() -> AppError {
AppError::BadRequest(STOPPED_MESSAGE.to_string()) AppError::BadRequest(STOPPED_MESSAGE.to_string())
} }
fn validate_limits(max_steps: Option<u32>, timeout_seconds: Option<u32>) -> Result<()> {
if let Some(max_steps) = max_steps {
if !(1..=100).contains(&max_steps) {
return Err(AppError::BadRequest(
"max_steps must be between 1 and 100".to_string(),
));
}
}
if let Some(timeout_seconds) = timeout_seconds {
if !(30..=3600).contains(&timeout_seconds) {
return Err(AppError::BadRequest(
"timeout_seconds must be between 30 and 3600".to_string(),
));
}
}
Ok(())
}
fn empty_session() -> ComputerUseSessionSummary { fn empty_session() -> ComputerUseSessionSummary {
ComputerUseSessionSummary { ComputerUseSessionSummary {
id: None, id: None,
status: ComputerUseSessionStatus::Idle, status: ComputerUseSessionStatus::Idle,
prompt: None, prompt: None,
step: 0, step: 0,
max_steps: 0,
last_error: None, last_error: None,
final_message: None, final_message: None,
} }
} }
fn cua_api_key_env() -> Option<String> {
std::env::var("ONE_KVM_CUA_API_KEY")
.ok()
.filter(|key| !key.trim().is_empty())
.or_else(|| {
std::env::var("OPENAI_API_KEY")
.ok()
.filter(|key| !key.trim().is_empty())
})
}
fn cua_base_url_env() -> Option<String> {
std::env::var("ONE_KVM_CUA_BASE_URL")
.ok()
.filter(|url| !url.trim().is_empty())
.or_else(|| {
std::env::var("ONE_KVM_OPENAI_BASE_URL")
.ok()
.filter(|url| !url.trim().is_empty())
})
}
fn validate_endpoint_url(url: &str) -> Result<()> { fn validate_endpoint_url(url: &str) -> Result<()> {
let trimmed = url.trim(); let trimmed = url.trim();
if !(trimmed.starts_with("https://") || trimmed.starts_with("http://")) { if !(trimmed.starts_with("https://") || trimmed.starts_with("http://")) {

File diff suppressed because it is too large Load Diff

View File

@@ -6,25 +6,42 @@ use typeshare::typeshare;
#[serde(default)] #[serde(default)]
pub struct ComputerUseConfig { pub struct ComputerUseConfig {
pub enabled: bool, pub enabled: bool,
pub provider: String,
pub base_url: String, pub base_url: String,
pub model: String, pub model: String,
#[typeshare(skip)] #[typeshare(skip)]
pub openai_api_key: Option<String>, #[serde(alias = "openai_api_key")]
pub max_steps: u32, pub api_key: Option<String>,
pub timeout_seconds: u32,
} }
impl Default for ComputerUseConfig { impl Default for ComputerUseConfig {
fn default() -> Self { fn default() -> Self {
Self { Self {
enabled: false, enabled: false,
provider: "openai".to_string(),
base_url: "https://api.openai.com/v1/responses".to_string(), base_url: "https://api.openai.com/v1/responses".to_string(),
model: "gpt-5.5".to_string(), model: "gpt-5.5".to_string(),
openai_api_key: None, api_key: None,
max_steps: 30,
timeout_seconds: 600,
} }
} }
} }
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn legacy_openai_api_key_migrates_to_generic_key() {
let config: ComputerUseConfig = serde_json::from_value(serde_json::json!({
"enabled": true,
"provider": "openai",
"base_url": "https://example.test/v1/chat/completions",
"model": "vision-model",
"openai_api_key": "legacy-key",
"max_steps": 30,
"timeout_seconds": 600
}))
.unwrap();
assert_eq!(config.api_key.as_deref(), Some("legacy-key"));
assert_eq!(config.model, "vision-model");
}
}

View File

@@ -57,7 +57,7 @@ fn sanitize_config_for_api(config: &mut AppConfig) {
config.auth.totp_secret = None; config.auth.totp_secret = None;
config.stream.turn_password = None; config.stream.turn_password = None;
config.computer_use.openai_api_key = None; config.computer_use.api_key = None;
config.rustdesk.device_password.clear(); config.rustdesk.device_password.clear();
config.rustdesk.relay_key = None; config.rustdesk.relay_key = None;

View File

@@ -409,7 +409,7 @@ const hasRightOverflow = computed(() => {
</Button> </Button>
</TooltipTrigger> </TooltipTrigger>
<TooltipContent> <TooltipContent>
<p>Computer Use</p> <p>{{ t('computerUse.title') }}</p>
</TooltipContent> </TooltipContent>
</Tooltip> </Tooltip>
</TooltipProvider> </TooltipProvider>

View File

@@ -1,6 +1,21 @@
<script setup lang="ts"> <script setup lang="ts">
import { computed, nextTick, onMounted, ref, watch } from 'vue' import { computed, nextTick, onMounted, ref, watch } from 'vue'
import { Bot, ChevronDown, Image, KeyRound, Play, Square } from 'lucide-vue-next' import { useI18n } from 'vue-i18n'
import {
Bot,
BrainCircuit,
ChevronDown,
ChevronRight,
Image,
KeyRound,
SendHorizontal,
Settings,
Square,
Trash2,
Wifi,
WifiOff,
X,
} from 'lucide-vue-next'
import { toast } from 'vue-sonner' import { toast } from 'vue-sonner'
import { computerUseApi, type ComputerUseAction, type ComputerUseConfig, type ComputerUseSession } from '@/api' import { computerUseApi, type ComputerUseAction, type ComputerUseConfig, type ComputerUseSession } from '@/api'
import type { ComputerUseTimelineItem } from '@/types/computerUseTimeline' import type { ComputerUseTimelineItem } from '@/types/computerUseTimeline'
@@ -8,9 +23,14 @@ import { Button } from '@/components/ui/button'
import { Input } from '@/components/ui/input' import { Input } from '@/components/ui/input'
import { Label } from '@/components/ui/label' import { Label } from '@/components/ui/label'
import { Textarea } from '@/components/ui/textarea' import { Textarea } from '@/components/ui/textarea'
import { Badge } from '@/components/ui/badge'
import { Switch } from '@/components/ui/switch' import { Switch } from '@/components/ui/switch'
import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs' import {
Dialog,
DialogContent,
DialogFooter,
DialogHeader,
DialogTitle,
} from '@/components/ui/dialog'
const props = defineProps<{ const props = defineProps<{
open: boolean open: boolean
@@ -20,6 +40,8 @@ const props = defineProps<{
timeline: ComputerUseTimelineItem[] timeline: ComputerUseTimelineItem[]
}>() }>()
const { t } = useI18n()
const emit = defineEmits<{ const emit = defineEmits<{
(e: 'update:open', value: boolean): void (e: 'update:open', value: boolean): void
(e: 'start', prompt: string): void (e: 'start', prompt: string): void
@@ -32,8 +54,9 @@ const prompt = ref('')
const apiKey = ref('') const apiKey = ref('')
const savingConfig = ref(false) const savingConfig = ref(false)
const starting = ref(false) const starting = ref(false)
const activeTab = ref('chat') const settingsOpen = ref(false)
const messagesRef = ref<HTMLDivElement | null>(null) const messagesRef = ref<HTMLDivElement | null>(null)
const reasoningOverrides = ref<Record<string, boolean>>({})
const defaultModel = computed({ const defaultModel = computed({
get: () => config.value?.model ?? 'gpt-5.5', get: () => config.value?.model ?? 'gpt-5.5',
@@ -47,53 +70,62 @@ const defaultBaseUrl = computed({
if (config.value) config.value.base_url = value if (config.value) config.value.base_url = value
}, },
}) })
const defaultMaxSteps = computed({
get: () => String(config.value?.max_steps ?? 30),
set: (value: string) => {
if (config.value) config.value.max_steps = Number(value) || 30
},
})
const defaultTimeoutSeconds = computed({
get: () => String(config.value?.timeout_seconds ?? 600),
set: (value: string) => {
if (config.value) config.value.timeout_seconds = Number(value) || 600
},
})
const status = computed(() => props.session?.status ?? 'idle') const status = computed(() => props.session?.status ?? 'idle')
const isRunning = computed(() => ['waiting_screenshot', 'thinking', 'executing'].includes(status.value)) const isRunning = computed(() => ['waiting_screenshot', 'thinking', 'executing'].includes(status.value))
const canStart = computed(() => !!config.value?.enabled && !!config.value?.api_key_configured && prompt.value.trim().length > 0 && !isRunning.value) const canStart = computed(() => (
const showWelcome = computed(() => props.timeline.length === 0 && !props.session?.last_error && !props.session?.final_message) !!config.value?.enabled
&& !!config.value?.api_key_configured
&& prompt.value.trim().length > 0
&& !isRunning.value
))
const statusLabel = computed(() => { const statusLabel = computed(() => {
switch (status.value) { switch (status.value) {
case 'waiting_screenshot': return '截屏中' case 'waiting_screenshot': return t('computerUse.status.waitingScreenshot')
case 'thinking': return '思考中' case 'thinking': return t('computerUse.status.thinking')
case 'executing': return '执行中' case 'executing': return t('computerUse.status.executing')
case 'completed': return '已完成' case 'completed': return t('computerUse.status.completed')
case 'failed': return '失败' case 'failed': return t('computerUse.status.failed')
case 'stopped': return '已停止' case 'stopped': return t('computerUse.status.stopped')
default: return '空闲' default: return t('computerUse.status.idle')
} }
}) })
const apiKeyPlaceholder = computed(() => {
if (!config.value?.api_key_configured) return t('computerUse.settings.apiKey')
const source = config.value.api_key_source
const sourceLabel = source === 'env'
? t('computerUse.settings.sourceEnv')
: source === 'config'
? t('computerUse.settings.sourceConfig')
: t('computerUse.settings.sourceNone')
return t('computerUse.settings.configured', { source: sourceLabel })
})
async function loadConfig() { async function loadConfig() {
try {
config.value = await computerUseApi.config() config.value = await computerUseApi.config()
} catch (err) {
toast.error(t('computerUse.errors.configLoadFailed'), {
description: err instanceof Error ? err.message : undefined,
})
}
} }
async function saveConfig() { async function saveConfig() {
if (!config.value) return
savingConfig.value = true savingConfig.value = true
try { try {
config.value = await computerUseApi.updateConfig({ config.value = await computerUseApi.updateConfig({
enabled: config.value?.enabled ?? true, enabled: config.value.enabled,
base_url: config.value?.base_url || 'https://api.openai.com/v1/responses', base_url: config.value.base_url,
model: config.value?.model || 'gpt-5.5', model: config.value.model,
max_steps: config.value?.max_steps || 30, api_key: apiKey.value.trim() || undefined,
timeout_seconds: config.value?.timeout_seconds || 600,
openai_api_key: apiKey.value.trim() || undefined,
}) })
apiKey.value = '' apiKey.value = ''
toast.success('Computer Use 配置已保存') settingsOpen.value = false
toast.success(t('computerUse.success.configSaved'))
} finally { } finally {
savingConfig.value = false savingConfig.value = false
} }
@@ -102,60 +134,75 @@ async function saveConfig() {
async function clearApiKey() { async function clearApiKey() {
savingConfig.value = true savingConfig.value = true
try { try {
config.value = await computerUseApi.updateConfig({ config.value = await computerUseApi.updateConfig({ clear_api_key: true })
clear_openai_api_key: true,
})
apiKey.value = '' apiKey.value = ''
toast.success('OpenAI API Key 已清除') toast.success(t('computerUse.success.apiKeyCleared'))
} finally { } finally {
savingConfig.value = false savingConfig.value = false
} }
} }
async function start() { function start() {
if (!canStart.value) return if (!canStart.value) return
const text = prompt.value.trim() const text = prompt.value.trim()
starting.value = true starting.value = true
try {
emit('start', text) emit('start', text)
prompt.value = '' prompt.value = ''
} finally {
starting.value = false starting.value = false
} }
}
function formatAction(action: ComputerUseAction): string { function formatAction(action: ComputerUseAction): string {
switch (action.type) { switch (action.type) {
case 'click': case 'click':
return `点击 (${action.x}, ${action.y}) ${action.button ?? 'left'}` return t('computerUse.actions.click', {
x: action.x,
y: action.y,
button: t(`computerUse.mouseButtons.${action.button ?? 'left'}`),
})
case 'double_click': case 'double_click':
return `双击 (${action.x}, ${action.y}) ${action.button ?? 'left'}` return t('computerUse.actions.doubleClick', {
x: action.x,
y: action.y,
button: t(`computerUse.mouseButtons.${action.button ?? 'left'}`),
})
case 'move': case 'move':
return `移动到 (${action.x}, ${action.y})` return t('computerUse.actions.move', { x: action.x, y: action.y })
case 'drag': case 'drag':
return `拖拽 ${action.path.length} 个点` return t('computerUse.actions.drag', { count: action.path.length })
case 'scroll': case 'scroll':
return `滚动 (${action.x}, ${action.y}) dx=${action.dx ?? 0} dy=${action.dy ?? 0}` return t('computerUse.actions.scroll', {
x: action.x,
y: action.y,
dx: action.dx ?? 0,
dy: action.dy ?? 0,
})
case 'type': case 'type':
return `输入 ${action.text.length} 字符` return t('computerUse.actions.typeAscii', { count: action.text.length })
case 'keypress': case 'keypress':
return `按键 ${action.keys.join('+')}` return t('computerUse.actions.keypress', { keys: action.keys.join('+') })
case 'wait': case 'wait':
return `等待 ${action.ms}ms` return t('computerUse.actions.wait', { ms: action.ms })
case 'screenshot': case 'screenshot':
return '请求截图' return t('computerUse.actions.screenshot')
} }
} }
function reasoningOpen(item: Extract<ComputerUseTimelineItem, { type: 'reasoning' }>) {
return reasoningOverrides.value[item.id] ?? !item.completed
}
function toggleReasoning(item: Extract<ComputerUseTimelineItem, { type: 'reasoning' }>) {
reasoningOverrides.value[item.id] = !reasoningOpen(item)
}
function scrollToBottom() { function scrollToBottom() {
nextTick(() => { nextTick(() => {
const el = messagesRef.value const el = messagesRef.value
if (!el) return if (el) el.scrollTop = el.scrollHeight
el.scrollTop = el.scrollHeight
}) })
} }
watch(() => props.timeline.length, scrollToBottom) watch(() => props.timeline, scrollToBottom, { deep: true })
watch(() => props.open, (open) => { watch(() => props.open, (open) => {
if (open) scrollToBottom() if (open) scrollToBottom()
}) })
@@ -166,80 +213,132 @@ onMounted(loadConfig)
<template> <template>
<aside <aside
v-show="open" v-show="open"
class="absolute inset-y-0 right-0 z-30 h-full min-h-0 w-[min(100%,420px)] border-l bg-background/98 shadow-xl backdrop-blur md:relative md:z-auto md:w-[420px] xl:w-[460px]" class="absolute inset-y-0 right-0 z-30 h-full min-h-0 w-full border-l bg-background shadow-xl sm:w-[420px] md:relative md:z-auto xl:w-[460px]"
> >
<div class="flex h-full min-h-0 flex-col"> <div class="flex h-full min-h-0 flex-col">
<div class="flex h-12 shrink-0 items-center justify-between border-b px-3"> <header class="flex h-12 shrink-0 items-center gap-2 border-b px-3">
<div class="flex min-w-0 items-center gap-2">
<Bot class="h-5 w-5 shrink-0" /> <Bot class="h-5 w-5 shrink-0" />
<div class="min-w-0"> <div class="min-w-0 flex-1">
<div class="truncate text-sm font-semibold">Computer Use</div> <div class="truncate text-sm font-semibold">{{ t('computerUse.title') }}</div>
<div class="truncate text-[11px] text-muted-foreground"> <div
WebSocket {{ connected ? '已连接' : '未连接' }} class="flex min-w-0 items-center gap-1 text-[11px]"
<span v-if="wsError"> · {{ wsError }}</span> :class="connected ? 'text-emerald-600 dark:text-emerald-400' : 'text-muted-foreground'"
>
<Wifi v-if="connected" class="h-3 w-3 shrink-0" />
<WifiOff v-else class="h-3 w-3 shrink-0" />
<span class="truncate">
{{ connected ? t('computerUse.connection.connected') : (wsError || t('computerUse.connection.disconnected')) }}
</span>
<span class="text-muted-foreground">· {{ statusLabel }}</span>
</div> </div>
</div> </div>
</div> <Button
<div class="flex items-center gap-1.5"> variant="ghost"
<Badge :variant="status === 'failed' ? 'destructive' : 'secondary'"> size="icon"
{{ statusLabel }} class="h-8 w-8"
</Badge> :title="t('computerUse.buttons.settings')"
<Button variant="ghost" size="icon" class="h-8 w-8" @click="emit('update:open', false)"> :aria-label="t('computerUse.buttons.settings')"
<ChevronDown class="h-4 w-4 rotate-90" /> @click="settingsOpen = true"
>
<Settings class="h-4 w-4" />
</Button> </Button>
</div> <Button
</div> variant="ghost"
size="icon"
<Tabs v-model="activeTab" class="flex min-h-0 flex-1 flex-col"> class="h-8 w-8"
<div class="px-3 py-2"> :title="t('computerUse.buttons.clear')"
<TabsList class="grid w-full grid-cols-2"> :aria-label="t('computerUse.buttons.clear')"
<TabsTrigger value="chat">对话</TabsTrigger> :disabled="isRunning || timeline.length === 0"
<TabsTrigger value="settings">设置</TabsTrigger> @click="emit('clear')"
</TabsList> >
</div> <Trash2 class="h-4 w-4" />
</Button>
<TabsContent value="chat" class="m-0 flex min-h-0 flex-1 flex-col data-[state=inactive]:hidden"> <Button
<div ref="messagesRef" class="min-h-0 flex-1 space-y-3 overflow-y-auto p-3"> variant="ghost"
<div v-if="showWelcome" class="rounded-md border border-dashed p-4 text-center text-xs text-muted-foreground"> size="icon"
发送任务后这里会显示对话截图和坐标操作 class="h-8 w-8"
:title="t('computerUse.buttons.close')"
:aria-label="t('computerUse.buttons.close')"
@click="emit('update:open', false)"
>
<X class="h-4 w-4" />
</Button>
</header>
<div ref="messagesRef" class="min-h-0 flex-1 overflow-y-auto px-4 py-4">
<div v-if="timeline.length === 0" class="flex h-full items-center justify-center text-muted-foreground/45">
<Bot class="h-10 w-10" />
</div> </div>
<div v-else class="space-y-5">
<template v-for="item in timeline" :key="item.id"> <template v-for="item in timeline" :key="item.id">
<div v-if="item.type === 'user'" class="flex justify-end"> <div v-if="item.type === 'user'" class="flex justify-end">
<div class="max-w-[86%] rounded-md bg-primary px-3 py-2 text-sm text-primary-foreground"> <div class="max-w-[86%] whitespace-pre-wrap break-words rounded-md bg-muted px-3 py-2 text-sm">
{{ item.text }} {{ item.text }}
</div> </div>
</div> </div>
<div v-else-if="item.type === 'assistant'" class="flex justify-start"> <div v-else-if="item.type === 'assistant'" class="whitespace-pre-wrap break-words text-sm leading-6">
<div class="max-w-[86%] rounded-md border bg-muted/50 px-3 py-2 text-sm">
{{ item.text }} {{ item.text }}
</div> </div>
<div v-else-if="item.type === 'reasoning'" class="border-l-2 border-muted pl-3 text-xs text-muted-foreground">
<button
type="button"
class="flex w-full items-center gap-1.5 py-1 text-left font-medium hover:text-foreground"
@click="toggleReasoning(item)"
>
<ChevronDown v-if="reasoningOpen(item)" class="h-3.5 w-3.5 shrink-0" />
<ChevronRight v-else class="h-3.5 w-3.5 shrink-0" />
<BrainCircuit class="h-3.5 w-3.5 shrink-0" />
<span>
{{ item.failed
? t('computerUse.reasoning.failed')
: (item.completed ? t('computerUse.reasoning.process') : t('computerUse.reasoning.thinking')) }}
</span>
</button>
<div
v-if="reasoningOpen(item)"
class="max-h-72 overflow-y-auto whitespace-pre-wrap break-words pb-2 pt-1 leading-5"
>
{{ item.text || t('computerUse.reasoning.thinking') }}
</div>
</div> </div>
<div v-else-if="item.type === 'screenshot'" class="rounded-md border bg-card p-2"> <div v-else-if="item.type === 'screenshot'" class="overflow-hidden rounded-md border bg-card">
<div class="mb-2 flex items-center justify-between text-xs text-muted-foreground"> <div class="flex items-center justify-between border-b px-2.5 py-1.5 text-[11px] text-muted-foreground">
<span class="inline-flex items-center gap-1.5"><Image class="h-3.5 w-3.5" />截图</span> <span class="inline-flex items-center gap-1.5">
<Image class="h-3.5 w-3.5" />{{ t('computerUse.trace.screenshot') }}
</span>
<span>{{ item.screenshot.width }}x{{ item.screenshot.height }}</span> <span>{{ item.screenshot.width }}x{{ item.screenshot.height }}</span>
</div> </div>
<div <div
class="w-full overflow-hidden rounded-sm bg-black" class="w-full bg-black"
:style="{ aspectRatio: `${item.screenshot.width} / ${item.screenshot.height}` }" :style="{ aspectRatio: `${item.screenshot.width} / ${item.screenshot.height}` }"
> >
<img :src="item.screenshot.data_url" class="h-full w-full object-cover" alt="Computer Use screenshot" /> <img
:src="item.screenshot.data_url"
class="h-full w-full object-contain"
:alt="t('computerUse.trace.screenshotAlt')"
/>
</div> </div>
</div> </div>
<div v-else-if="item.type === 'actions_executed'" class="rounded-md border border-success/35 bg-success/10 p-2 text-success"> <div v-else-if="item.type === 'actions_executed'" class="border-l-2 border-emerald-500/50 pl-3 text-xs">
<div class="mb-2 text-xs font-medium">已执行</div> <div class="mb-1 text-[11px] font-medium text-emerald-600 dark:text-emerald-400">
<div class="space-y-1"> {{ t('computerUse.trace.executed') }}
<div v-for="(action, index) in item.actions" :key="index" class="rounded-sm bg-background/60 px-2 py-1.5 text-xs"> </div>
<div class="space-y-1 text-muted-foreground">
<div v-for="(action, index) in item.actions" :key="index" class="break-words">
{{ formatAction(action) }} {{ formatAction(action) }}
</div> </div>
</div> </div>
</div> </div>
<div v-else-if="item.type === 'error'" class="rounded-md border border-destructive/40 bg-destructive/10 px-3 py-2 text-xs text-destructive"> <div
v-else-if="item.type === 'error'"
class="whitespace-pre-wrap break-words rounded-md border border-destructive/40 bg-destructive/10 px-3 py-2 text-xs leading-5 text-destructive"
>
{{ item.text }} {{ item.text }}
</div> </div>
@@ -248,91 +347,107 @@ onMounted(loadConfig)
</div> </div>
</template> </template>
</div> </div>
</div>
<div class="shrink-0 border-t p-3"> <footer class="shrink-0 border-t bg-background p-3">
<div class="relative">
<Textarea <Textarea
v-model="prompt" v-model="prompt"
rows="3" rows="3"
placeholder="继续输入任务或追问" class="max-h-40 min-h-20 resize-none pr-12"
:placeholder="t('computerUse.input.placeholder')"
:disabled="isRunning" :disabled="isRunning"
@keydown.meta.enter.prevent="start" @keydown.meta.enter.prevent="start"
@keydown.ctrl.enter.prevent="start" @keydown.ctrl.enter.prevent="start"
/> />
<div class="mt-2 flex gap-2"> <Button
<Button class="flex-1 gap-2" :disabled="!canStart || starting" @click="start"> v-if="!isRunning"
<Play class="h-4 w-4" /> size="icon"
发送 class="absolute bottom-2 right-2 h-8 w-8"
:title="t('computerUse.buttons.send')"
:aria-label="t('computerUse.buttons.send')"
:disabled="!canStart || starting"
@click="start"
>
<SendHorizontal class="h-4 w-4" />
</Button> </Button>
<Button variant="outline" class="gap-2" :disabled="!isRunning" @click="emit('stop')"> <Button
<Square class="h-4 w-4" /> v-else
停止 size="icon"
</Button> variant="destructive"
<Button variant="ghost" size="sm" :disabled="isRunning || timeline.length === 0" @click="emit('clear')"> class="absolute bottom-2 right-2 h-8 w-8"
清空 :title="t('computerUse.buttons.stop')"
:aria-label="t('computerUse.buttons.stop')"
@click="emit('stop')"
>
<Square class="h-3.5 w-3.5 fill-current" />
</Button> </Button>
</div> </div>
<p v-if="!config?.api_key_configured" class="mt-2 text-xs text-muted-foreground"> <p v-if="config && !config.api_key_configured" class="mt-2 text-xs text-muted-foreground">
需要先在设置里保存 OpenAI API Key {{ t('computerUse.input.apiKeyRequired') }}
</p> </p>
</footer>
</div> </div>
</TabsContent>
<TabsContent value="settings" class="m-0 min-h-0 flex-1 overflow-y-auto p-3 data-[state=inactive]:hidden"> <Dialog v-model:open="settingsOpen">
<div class="space-y-4"> <DialogContent class="max-w-md">
<div class="flex items-center justify-between rounded-md border p-3"> <DialogHeader>
<div> <DialogTitle>{{ t('computerUse.settings.title') }}</DialogTitle>
<div class="text-sm font-medium">启用 AI 操作</div> </DialogHeader>
<div class="text-xs text-muted-foreground">配置保存后立即生效</div>
</div> <div class="space-y-4 py-2">
<div class="flex items-center justify-between gap-4">
<Label for="cua-enabled">{{ t('computerUse.settings.enableAi') }}</Label>
<Switch <Switch
id="cua-enabled"
:model-value="config?.enabled ?? false" :model-value="config?.enabled ?? false"
@update:model-value="(value) => { if (config) config.enabled = value }" @update:model-value="(value) => { if (config) config.enabled = value }"
/> />
</div> </div>
<div class="space-y-3 rounded-md border p-3"> <div class="space-y-1.5">
<div class="grid grid-cols-2 gap-2"> <Label for="cua-model">{{ t('computerUse.settings.model') }}</Label>
<div class="space-y-1"> <Input id="cua-model" v-model="defaultModel" :disabled="!config" placeholder="gpt-5.5" />
<Label class="text-xs">模型</Label>
<Input v-model="defaultModel" :disabled="!config" placeholder="gpt-5.5" />
</div> </div>
<div class="space-y-1">
<Label class="text-xs">最大步数</Label> <div class="space-y-1.5">
<Input v-model="defaultMaxSteps" type="number" min="1" max="100" /> <Label for="cua-url">{{ t('computerUse.settings.apiUrl') }}</Label>
<Input
id="cua-url"
v-model="defaultBaseUrl"
:disabled="!config"
placeholder="https://api.example.com/v1/responses"
/>
</div> </div>
</div>
<div class="space-y-1"> <div class="space-y-1.5">
<Label class="text-xs">超时秒数</Label> <Label for="cua-key" class="flex items-center gap-1.5">
<Input v-model="defaultTimeoutSeconds" type="number" min="30" max="3600" />
</div>
<div class="space-y-1">
<Label class="text-xs">API URL</Label>
<Input v-model="defaultBaseUrl" :disabled="!config" placeholder="https://api.openai.com/v1/responses" />
</div>
<div class="space-y-1">
<Label class="text-xs flex items-center gap-1">
<KeyRound class="h-3.5 w-3.5" /> <KeyRound class="h-3.5 w-3.5" />
OpenAI API Key {{ t('computerUse.settings.apiKey') }}
</Label> </Label>
<Input <Input
id="cua-key"
v-model="apiKey" v-model="apiKey"
type="password" type="password"
autocomplete="off" autocomplete="off"
:placeholder="config?.api_key_configured ? `已配置:${config.api_key_source}` : 'sk-...'" :placeholder="apiKeyPlaceholder"
/> />
</div> </div>
<div class="grid grid-cols-2 gap-2"> </div>
<Button size="sm" :disabled="savingConfig || !config" @click="saveConfig">
保存配置 <DialogFooter class="gap-2 sm:justify-between">
<Button
variant="outline"
:disabled="savingConfig || !config?.api_key_configured"
@click="clearApiKey"
>
{{ t('computerUse.buttons.clearKey') }}
</Button> </Button>
<Button size="sm" variant="outline" :disabled="savingConfig || !config?.api_key_configured" @click="clearApiKey"> <Button :disabled="savingConfig || !config" @click="saveConfig">
清除 Key {{ t('computerUse.buttons.save') }}
</Button> </Button>
</div> </DialogFooter>
</div> </DialogContent>
</div> </Dialog>
</TabsContent>
</Tabs>
</div>
</aside> </aside>
</template> </template>

View File

@@ -1,4 +1,5 @@
import { ref, onUnmounted } from 'vue' import { ref, onUnmounted } from 'vue'
import { useI18n } from 'vue-i18n'
import { buildWsUrl } from '@/types/websocket' import { buildWsUrl } from '@/types/websocket'
import { generateUUID } from '@/lib/utils' import { generateUUID } from '@/lib/utils'
import type { ComputerUseScreenshot, ComputerUseSession, ComputerUseAction } from '@/api' import type { ComputerUseScreenshot, ComputerUseSession, ComputerUseAction } from '@/api'
@@ -8,6 +9,8 @@ export type ComputerUseServerMessage =
| { type: 'screenshot_requested'; request_id: string } | { type: 'screenshot_requested'; request_id: string }
| { type: 'screenshot_captured'; screenshot: ComputerUseScreenshot } | { type: 'screenshot_captured'; screenshot: ComputerUseScreenshot }
| { type: 'step_started'; step: number } | { type: 'step_started'; step: number }
| { type: 'reasoning_delta'; delta: string }
| { type: 'reasoning_completed'; failed: boolean }
| { type: 'actions_executed'; actions: ComputerUseAction[] } | { type: 'actions_executed'; actions: ComputerUseAction[] }
| { type: 'error'; message: string } | { type: 'error'; message: string }
@@ -15,6 +18,7 @@ export function useComputerUseSocket(options: {
onMessage: (message: ComputerUseServerMessage) => void onMessage: (message: ComputerUseServerMessage) => void
onScreenshotRequested: (requestId: string) => Promise<ComputerUseScreenshot | null> onScreenshotRequested: (requestId: string) => Promise<ComputerUseScreenshot | null>
}) { }) {
const { t } = useI18n()
const connected = ref(false) const connected = ref(false)
const error = ref<string | null>(null) const error = ref<string | null>(null)
const clientId = generateUUID() const clientId = generateUUID()
@@ -29,7 +33,7 @@ export function useComputerUseSocket(options: {
connectPromise = new Promise((resolve, reject) => { connectPromise = new Promise((resolve, reject) => {
if (!ws) { if (!ws) {
reject(new Error('Computer use WebSocket failed')) reject(new Error(t('computerUse.connection.socketFailed')))
return return
} }
@@ -41,7 +45,7 @@ export function useComputerUseSocket(options: {
} }
ws.onerror = () => { ws.onerror = () => {
error.value = 'Computer use WebSocket failed' error.value = t('computerUse.connection.socketFailed')
connectPromise = null connectPromise = null
reject(new Error(error.value)) reject(new Error(error.value))
} }
@@ -57,17 +61,28 @@ export function useComputerUseSocket(options: {
const message = JSON.parse(event.data) as ComputerUseServerMessage const message = JSON.parse(event.data) as ComputerUseServerMessage
options.onMessage(message) options.onMessage(message)
if (message.type === 'screenshot_requested') { if (message.type === 'screenshot_requested') {
try {
const screenshot = await options.onScreenshotRequested(message.request_id) const screenshot = await options.onScreenshotRequested(message.request_id)
if (screenshot && ws?.readyState === WebSocket.OPEN) { if (!screenshot) throw new Error(t('computerUse.errors.captureUnavailable'))
if (ws?.readyState === WebSocket.OPEN) {
ws.send(JSON.stringify({ ws.send(JSON.stringify({
type: 'screenshot_result', type: 'screenshot_result',
request_id: message.request_id, request_id: message.request_id,
screenshot, screenshot,
})) }))
} }
} catch (err) {
if (ws?.readyState === WebSocket.OPEN) {
ws.send(JSON.stringify({
type: 'screenshot_error',
request_id: message.request_id,
message: err instanceof Error ? err.message : t('computerUse.errors.screenshotFailed'),
}))
}
}
} }
} catch (err) { } catch (err) {
console.error('[ComputerUse] Failed to handle WS message:', err) console.error(`[ComputerUse] ${t('computerUse.connection.messageFailed')}:`, err)
} }
} }

View File

@@ -751,6 +751,86 @@ export default {
loadDevicesFailed: 'Failed to load device list', loadDevicesFailed: 'Failed to load device list',
updateFailed: 'Update failed', updateFailed: 'Update failed',
}, },
computerUse: {
title: 'Computer Use',
status: {
waitingScreenshot: 'Capturing screen',
thinking: 'Thinking',
executing: 'Executing',
completed: 'Completed',
failed: 'Failed',
stopped: 'Stopped',
idle: 'Idle',
},
connection: {
connected: 'Connected',
disconnected: 'Disconnected',
socketFailed: 'Computer Use WebSocket connection failed',
messageFailed: 'Failed to process Computer Use WebSocket message',
},
buttons: {
settings: 'Settings',
clear: 'Clear',
close: 'Close',
send: 'Send',
stop: 'Stop',
save: 'Save',
clearKey: 'Clear Key',
},
reasoning: {
failed: 'Thinking failed',
process: 'Reasoning',
thinking: 'Thinking',
},
trace: {
screenshot: 'Screenshot',
screenshotAlt: 'Computer Use screenshot',
executed: 'Executed',
},
input: {
placeholder: 'Enter a task. Press Enter for a new line; press Ctrl+Enter to send.',
apiKeyRequired: 'Configure an API Key first.',
},
settings: {
title: 'Computer Use Settings',
enableAi: 'Enable AI control',
model: 'Model',
apiUrl: 'API URL',
apiKey: 'API Key',
configured: 'Configured: {source}',
sourceEnv: 'environment variable',
sourceConfig: 'configuration file',
sourceNone: 'not configured',
},
actions: {
click: 'Click ({x}, {y}) {button}',
doubleClick: 'Double-click ({x}, {y}) {button}',
move: 'Move to ({x}, {y})',
drag: 'Drag through {count} points',
scroll: 'Scroll at ({x}, {y}) dx={dx} dy={dy}',
typeAscii: 'Type {count} ASCII characters',
keypress: 'Press {keys}',
wait: 'Wait {ms}ms',
screenshot: 'Request screenshot',
},
mouseButtons: {
left: 'left button',
middle: 'middle button',
right: 'right button',
},
errors: {
configLoadFailed: 'Failed to load Computer Use settings',
taskFailed: 'Computer Use task failed',
startFailed: 'Failed to start Computer Use',
stopFailed: 'Failed to stop Computer Use',
captureUnavailable: 'The current video frame is unavailable',
screenshotFailed: 'Screenshot capture failed',
},
success: {
configSaved: 'Computer Use settings saved',
apiKeyCleared: 'API Key cleared',
},
},
statusCard: { statusCard: {
device: 'Device', device: 'Device',
video: 'Video', video: 'Video',

View File

@@ -750,6 +750,86 @@ export default {
loadDevicesFailed: '加载设备列表失败', loadDevicesFailed: '加载设备列表失败',
updateFailed: '更新失败', updateFailed: '更新失败',
}, },
computerUse: {
title: 'Computer Use',
status: {
waitingScreenshot: '截屏中',
thinking: '思考中',
executing: '执行中',
completed: '已完成',
failed: '失败',
stopped: '已停止',
idle: '空闲',
},
connection: {
connected: '已连接',
disconnected: '未连接',
socketFailed: 'Computer Use WebSocket 连接失败',
messageFailed: '处理 Computer Use WebSocket 消息失败',
},
buttons: {
settings: '设置',
clear: '清空',
close: '关闭',
send: '发送',
stop: '停止',
save: '保存',
clearKey: '清除 Key',
},
reasoning: {
failed: '思考失败',
process: '思考过程',
thinking: '思考中',
},
trace: {
screenshot: '截图',
screenshotAlt: 'Computer Use 截图',
executed: '已执行',
},
input: {
placeholder: '输入任务,按下回车换行,按下 Ctrl+Enter 发送',
apiKeyRequired: '请先配置 API Key。',
},
settings: {
title: 'Computer Use 设置',
enableAi: '启用 AI 操作',
model: '模型',
apiUrl: 'API URL',
apiKey: 'API Key',
configured: '已配置:{source}',
sourceEnv: '环境变量',
sourceConfig: '配置文件',
sourceNone: '未配置',
},
actions: {
click: '点击 ({x}, {y}) {button}',
doubleClick: '双击 ({x}, {y}) {button}',
move: '移动到 ({x}, {y})',
drag: '拖拽 {count} 个点',
scroll: '滚动 ({x}, {y}) dx={dx} dy={dy}',
typeAscii: '输入 {count} 个 ASCII 字符',
keypress: '按键 {keys}',
wait: '等待 {ms}ms',
screenshot: '请求截图',
},
mouseButtons: {
left: '左键',
middle: '中键',
right: '右键',
},
errors: {
configLoadFailed: '无法加载 Computer Use 配置',
taskFailed: 'Computer Use 任务失败',
startFailed: 'Computer Use 启动失败',
stopFailed: 'Computer Use 停止失败',
captureUnavailable: '当前视频画面尚不可用',
screenshotFailed: '截图失败',
},
success: {
configSaved: 'Computer Use 配置已保存',
apiKeyCleared: 'API Key 已清除',
},
},
statusCard: { statusCard: {
device: '设备', device: '设备',
video: '视频', video: '视频',

View File

@@ -3,6 +3,7 @@ import type { ComputerUseAction, ComputerUseScreenshot } from '@/api'
export type ComputerUseTimelineItem = export type ComputerUseTimelineItem =
| { id: string; type: 'user'; text: string } | { id: string; type: 'user'; text: string }
| { id: string; type: 'assistant'; text: string } | { id: string; type: 'assistant'; text: string }
| { id: string; type: 'reasoning'; text: string; completed: boolean; failed: boolean }
| { id: string; type: 'screenshot'; screenshot: ComputerUseScreenshot } | { id: string; type: 'screenshot'; screenshot: ComputerUseScreenshot }
| { id: string; type: 'actions_executed'; actions: ComputerUseAction[] } | { id: string; type: 'actions_executed'; actions: ComputerUseAction[] }
| { id: string; type: 'error'; text: string } | { id: string; type: 'error'; text: string }

View File

@@ -178,11 +178,8 @@ export interface WebConfig {
export interface ComputerUseConfig { export interface ComputerUseConfig {
enabled: boolean; enabled: boolean;
provider: string;
base_url: string; base_url: string;
model: string; model: string;
max_steps: number;
timeout_seconds: number;
} }
export interface TtydConfig { export interface TtydConfig {
@@ -381,11 +378,8 @@ export interface Ch9329DescriptorState {
export interface ComputerUseConfigResponse { export interface ComputerUseConfigResponse {
enabled: boolean; enabled: boolean;
provider: string;
base_url: string; base_url: string;
model: string; model: string;
max_steps: number;
timeout_seconds: number;
api_key_configured: boolean; api_key_configured: boolean;
api_key_source: string; api_key_source: string;
} }
@@ -394,10 +388,8 @@ export interface ComputerUseConfigUpdate {
enabled?: boolean; enabled?: boolean;
base_url?: string; base_url?: string;
model?: string; model?: string;
max_steps?: number; api_key?: string;
timeout_seconds?: number; clear_api_key?: boolean;
openai_api_key?: string;
clear_openai_api_key?: boolean;
} }
export interface ComputerUsePoint { export interface ComputerUsePoint {
@@ -426,7 +418,6 @@ export interface ComputerUseSessionSummary {
status: ComputerUseSessionStatus; status: ComputerUseSessionStatus;
prompt?: string; prompt?: string;
step: number; step: number;
max_steps: number;
last_error?: string; last_error?: string;
final_message?: string; final_message?: string;
} }
@@ -435,8 +426,6 @@ export interface ComputerUseStartRequest {
prompt: string; prompt: string;
continue_conversation?: boolean; continue_conversation?: boolean;
client_id: string; client_id: string;
max_steps?: number;
timeout_seconds?: number;
} }
export interface EasytierConfigUpdate { export interface EasytierConfigUpdate {

View File

@@ -791,9 +791,35 @@ function handleComputerUseMessage(message: ComputerUseServerMessage) {
case 'actions_executed': case 'actions_executed':
pushComputerUseTimeline({ type: 'actions_executed', actions: message.actions }) pushComputerUseTimeline({ type: 'actions_executed', actions: message.actions })
break break
case 'step_started':
pushComputerUseTimeline({ type: 'reasoning', text: '', completed: false, failed: false })
break
case 'reasoning_delta': {
const last = computerUseTimeline.value[computerUseTimeline.value.length - 1]
if (last?.type === 'reasoning' && !last.completed) {
last.text += message.delta
} else {
pushComputerUseTimeline({ type: 'reasoning', text: message.delta, completed: false, failed: false })
}
break
}
case 'reasoning_completed': {
for (let index = computerUseTimeline.value.length - 1; index >= 0; index -= 1) {
const item = computerUseTimeline.value[index]
if (item?.type !== 'reasoning' || item.completed) continue
if (!message.failed && !item.text) {
computerUseTimeline.value.splice(index, 1)
} else {
item.completed = true
item.failed = message.failed
}
break
}
break
}
case 'error': case 'error':
pushComputerUseTimeline({ type: 'error', text: message.message }) pushComputerUseTimeline({ type: 'error', text: message.message })
toast.error('Computer Use failed', { description: message.message }) toast.error(t('computerUse.errors.taskFailed'), { description: message.message })
break break
} }
} }
@@ -833,8 +859,8 @@ async function startComputerUse(prompt: string) {
}) })
computerUseConversationStarted.value = true computerUseConversationStarted.value = true
} catch (err: any) { } catch (err: any) {
pushComputerUseTimeline({ type: 'error', text: err?.message ?? 'Computer Use start failed' }) pushComputerUseTimeline({ type: 'error', text: err?.message ?? t('computerUse.errors.startFailed') })
toast.error('Computer Use start failed', { description: err?.message }) toast.error(t('computerUse.errors.startFailed'), { description: err?.message })
} }
} }
@@ -842,7 +868,7 @@ async function stopComputerUse() {
try { try {
computerUseSession.value = await computerUseApi.stop() computerUseSession.value = await computerUseApi.stop()
} catch (err: any) { } catch (err: any) {
toast.error('Computer Use stop failed', { description: err?.message }) toast.error(t('computerUse.errors.stopFailed'), { description: err?.message })
} }
} }