mirror of
https://github.com/mofeng-git/One-KVM.git
synced 2026-10-28 08:20:49 +08:00
feat(mcp): 用 MCP 服务替换内置 AI Agent (#319)
在现有 HTTP 服务挂载 /mcp，让外部 AI 经共享 API Key 调用原分辨率截图、批量键鼠、ATX/WOL 和虚拟介质八个工具。所有调用共用有界 FIFO 和全局限流，复用现有采集管线与硬件控制器。 移除内置模型调用、网页 Agent 面板及旧配置，设置页提供可持续查看的 Key、统一的地址/密钥复制样式。补齐 DEB 的 libudev1 运行依赖。 验证：583 项 Rust 库测试、前端生产构建、ARMv7 设备视觉操作与介质/WOL 实测，以及默认 Release 构建和 DEB 包校验通过。 Refs #319
This commit is contained in:
mofeng-git
174
Cargo.lock
generated
174
Cargo.lock
generated
@@ -719,6 +719,7 @@ dependencies = [
|
||||
"iana-time-zone",
|
||||
"js-sys",
|
||||
"num-traits",
|
||||
"serde",
|
||||
"wasm-bindgen",
|
||||
"windows-link",
|
||||
]
|
||||
@@ -1063,7 +1064,7 @@ version = "0.6.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a707ceda8652f6c7624f2be725652e9524c815bf3b9d55a0b2320be2303f9c11"
|
||||
dependencies = [
|
||||
"darling",
|
||||
"darling 0.20.11",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
@@ -1076,8 +1077,18 @@ version = "0.20.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee"
|
||||
dependencies = [
|
||||
"darling_core",
|
||||
"darling_macro",
|
||||
"darling_core 0.20.11",
|
||||
"darling_macro 0.20.11",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling"
|
||||
version = "0.24.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec"
|
||||
dependencies = [
|
||||
"darling_core 0.24.1",
|
||||
"darling_macro 0.24.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1094,17 +1105,41 @@ dependencies = [
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling_core"
|
||||
version = "0.24.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff"
|
||||
dependencies = [
|
||||
"ident_case",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"strsim",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling_macro"
|
||||
version = "0.20.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead"
|
||||
dependencies = [
|
||||
"darling_core",
|
||||
"darling_core 0.20.11",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling_macro"
|
||||
version = "0.24.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785"
|
||||
dependencies = [
|
||||
"darling_core 0.24.1",
|
||||
"quote",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dary_heap"
|
||||
version = "0.3.9"
|
||||
@@ -1298,6 +1333,12 @@ dependencies = [
|
||||
"x509-parser 0.16.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dyn-clone"
|
||||
version = "1.0.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
|
||||
|
||||
[[package]]
|
||||
name = "ecdsa"
|
||||
version = "0.16.9"
|
||||
@@ -2070,6 +2111,8 @@ checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855"
|
||||
dependencies = [
|
||||
"equivalent",
|
||||
"hashbrown 0.17.1",
|
||||
"serde",
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2879,6 +2922,7 @@ dependencies = [
|
||||
"rand 0.10.3",
|
||||
"rcgen 0.14.10",
|
||||
"reqwest",
|
||||
"rmcp",
|
||||
"rtp",
|
||||
"rtsp-types",
|
||||
"rust-embed",
|
||||
@@ -3049,6 +3093,12 @@ dependencies = [
|
||||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pastey"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4"
|
||||
|
||||
[[package]]
|
||||
name = "pem"
|
||||
version = "3.0.6"
|
||||
@@ -3438,6 +3488,26 @@ dependencies = [
|
||||
"bitflags 2.13.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ref-cast"
|
||||
version = "1.0.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3"
|
||||
dependencies = [
|
||||
"ref-cast-impl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ref-cast-impl"
|
||||
version = "1.0.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.13.1"
|
||||
@@ -3574,6 +3644,51 @@ version = "1.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422"
|
||||
|
||||
[[package]]
|
||||
name = "rmcp"
|
||||
version = "3.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ab5023804cbe0304cbe06197ac16e645a0a0d0ffa2953bf735c15c2fa0213350"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.23.1",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"futures",
|
||||
"http",
|
||||
"http-body",
|
||||
"http-body-util",
|
||||
"indexmap",
|
||||
"pastey",
|
||||
"pin-project-lite",
|
||||
"rand 0.10.3",
|
||||
"rmcp-macros",
|
||||
"schemars",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sse-stream",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"tokio-util",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rmcp-macros"
|
||||
version = "3.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a1da498700030856ba63114bceefa30ba21f5e278cb975e5b9de3529fe842b40"
|
||||
dependencies = [
|
||||
"darling 0.24.1",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"serde_json",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rtcp"
|
||||
version = "0.17.2"
|
||||
@@ -3738,6 +3853,32 @@ dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "schemars"
|
||||
version = "1.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ab508826f74a77ca9d5aba6ff19b522583ee3eaf28a19384ff3d0e5835fadf6e"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"dyn-clone",
|
||||
"ref-cast",
|
||||
"schemars_derive",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "schemars_derive"
|
||||
version = "1.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e1c3a92094fa7d61aa124645844facb6b554dfc797136d0f5fd1f890e2bffc69"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"serde_derive_internals",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "scopeguard"
|
||||
version = "1.2.0"
|
||||
@@ -3839,6 +3980,17 @@ dependencies = [
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive_internals"
|
||||
version = "0.30.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.151"
|
||||
@@ -4166,6 +4318,19 @@ dependencies = [
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sse-stream"
|
||||
version = "0.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c25ac7aff0abd1dbc474536e40416e1102c7dd9bfba0b9861c6d357f835dcfb4"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-util",
|
||||
"http-body",
|
||||
"http-body-util",
|
||||
"pin-project-lite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "stable_deref_trait"
|
||||
version = "1.2.1"
|
||||
@@ -4563,6 +4728,7 @@ dependencies = [
|
||||
"futures-util",
|
||||
"http",
|
||||
"http-body",
|
||||
"http-body-util",
|
||||
"pin-project-lite",
|
||||
"tower",
|
||||
"tower-layer",
|
||||
|
||||
@@ -16,6 +16,7 @@ desktop = [
|
||||
"dep:tokio",
|
||||
"dep:tokio-util",
|
||||
"dep:axum",
|
||||
"dep:rmcp",
|
||||
"dep:axum-extra",
|
||||
"dep:tower-http",
|
||||
"dep:sqlx",
|
||||
@@ -82,7 +83,9 @@ tokio-util = { version = "0.7", features = ["rt"], optional = true }
|
||||
# Web framework
|
||||
axum = { version = "0.8", features = ["ws", "multipart", "tokio"], optional = true }
|
||||
axum-extra = { version = "0.12", features = ["cookie"], optional = true }
|
||||
tower-http = { version = "0.6", features = ["cors", "trace", "set-header"], optional = true }
|
||||
tower-http = { version = "0.6", features = ["cors", "trace", "set-header", "limit"], optional = true }
|
||||
|
||||
rmcp = { version = "3.5.1", default-features = false, features = ["server", "macros", "schemars", "transport-streamable-http-server"], optional = true }
|
||||
|
||||
# Database - Use bundled SQLite for static linking
|
||||
sqlx = { version = "0.9", default-features = false, features = ["runtime-tokio", "sqlite-bundled"], optional = true }
|
||||
|
||||
@@ -125,7 +125,7 @@ EOF
|
||||
chmod 755 "$PKG_DIR/DEBIAN/prerm"
|
||||
|
||||
# Create control file
|
||||
BASE_DEPS="libc6 (>= 2.36), libgcc-s1, libstdc++6, libasound2t64 (>= 1.1) | libasound2 (>= 1.1), libdrm2 (>= 2.4)"
|
||||
BASE_DEPS="libc6 (>= 2.36), libgcc-s1, libstdc++6, libasound2t64 (>= 1.1) | libasound2 (>= 1.1), libdrm2 (>= 2.4), libudev1"
|
||||
AMD64_DEPS="libva2 (>= 2.0), libva-drm2 (>= 2.10), libva-x11-2 (>= 2.10), libmfx-gen1.2 (>= 22.0) | libmfx1 (>= 21.1), libx11-6 (>= 1.6), libxcb1 (>= 1.14), i965-va-driver-shaders (>= 2.4), intel-media-va-driver-non-free (>= 21.1)"
|
||||
DEPS="$BASE_DEPS"
|
||||
if [ "$DEB_ARCH" = "amd64" ]; then
|
||||
|
||||
@@ -27,7 +27,7 @@ pub use types::{
|
||||
ActiveLevel, AtxAction, AtxDevices, AtxDriverType, AtxInputBinding, AtxKeyConfig,
|
||||
AtxOutputBinding, AtxPowerRequest, AtxState, HddStatus, PowerStatus, LCUS_RELAY_MAX_CHANNEL,
|
||||
};
|
||||
pub use wol::send_wol;
|
||||
pub use wol::{normalize_wol_mac_address, send_wol};
|
||||
|
||||
#[cfg(any(unix, test))]
|
||||
fn hidraw_uevent_is_usb_relay(uevent: &str) -> bool {
|
||||
|
||||
@@ -9,6 +9,28 @@ use crate::error::{AppError, Result};
|
||||
|
||||
const MAGIC_PACKET_SIZE: usize = 102;
|
||||
|
||||
/// Validate and normalize the MAC formats accepted by the web and MCP APIs.
|
||||
pub fn normalize_wol_mac_address(mac_address: &str) -> Result<String> {
|
||||
let normalized = mac_address.trim().replace('-', ":");
|
||||
let normalized = if normalized.len() == 12 && normalized.bytes().all(|b| b.is_ascii_hexdigit())
|
||||
{
|
||||
normalized
|
||||
.as_bytes()
|
||||
.chunks(2)
|
||||
.map(|b| std::str::from_utf8(b).unwrap())
|
||||
.collect::<Vec<_>>()
|
||||
.join(":")
|
||||
} else {
|
||||
normalized
|
||||
};
|
||||
let mac = parse_mac_address(&normalized)?;
|
||||
Ok(mac
|
||||
.iter()
|
||||
.map(|byte| format!("{byte:02X}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join(":"))
|
||||
}
|
||||
|
||||
fn parse_mac_address(mac: &str) -> Result<[u8; 6]> {
|
||||
let mac = mac.trim().to_uppercase();
|
||||
let parts: Vec<&str> = if mac.contains(':') {
|
||||
@@ -139,6 +161,17 @@ mod tests {
|
||||
assert!(parse_mac_address("AA:BB:CC:DD:EE:GG").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalizes_web_and_mcp_mac_formats() {
|
||||
for value in [" aa-bb-cc-dd-ee-ff ", "aabbccddeeff", "AA:BB:CC:DD:EE:FF"] {
|
||||
assert_eq!(
|
||||
normalize_wol_mac_address(value).unwrap(),
|
||||
"AA:BB:CC:DD:EE:FF"
|
||||
);
|
||||
}
|
||||
assert!(normalize_wol_mac_address("invalid").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_magic_packet() {
|
||||
let mac = [0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF];
|
||||
|
||||
@@ -1,218 +0,0 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use typeshare::typeshare;
|
||||
|
||||
#[typeshare]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ComputerUseSessionStatus {
|
||||
Idle,
|
||||
WaitingScreenshot,
|
||||
Thinking,
|
||||
Executing,
|
||||
Completed,
|
||||
Failed,
|
||||
Stopped,
|
||||
}
|
||||
|
||||
#[typeshare]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ComputerUseButton {
|
||||
Left,
|
||||
Middle,
|
||||
Right,
|
||||
}
|
||||
|
||||
impl Default for ComputerUseButton {
|
||||
fn default() -> Self {
|
||||
Self::Left
|
||||
}
|
||||
}
|
||||
|
||||
// Kept out of typeshare because these internally tagged enums are consumed by
|
||||
// manually maintained frontend types and changing serde shape would break WS/API.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
pub enum ComputerUseAction {
|
||||
Click {
|
||||
x: u32,
|
||||
y: u32,
|
||||
#[serde(default)]
|
||||
button: ComputerUseButton,
|
||||
},
|
||||
DoubleClick {
|
||||
x: u32,
|
||||
y: u32,
|
||||
#[serde(default)]
|
||||
button: ComputerUseButton,
|
||||
},
|
||||
Move {
|
||||
x: u32,
|
||||
y: u32,
|
||||
},
|
||||
Drag {
|
||||
path: Vec<ComputerUsePoint>,
|
||||
#[serde(default)]
|
||||
button: ComputerUseButton,
|
||||
},
|
||||
Scroll {
|
||||
x: u32,
|
||||
y: u32,
|
||||
#[serde(default)]
|
||||
dx: i32,
|
||||
#[serde(default)]
|
||||
dy: i32,
|
||||
},
|
||||
Type {
|
||||
text: String,
|
||||
},
|
||||
Keypress {
|
||||
keys: Vec<String>,
|
||||
},
|
||||
Wait {
|
||||
ms: u64,
|
||||
},
|
||||
Screenshot,
|
||||
}
|
||||
|
||||
#[typeshare]
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
|
||||
pub struct ComputerUsePoint {
|
||||
pub x: u32,
|
||||
pub y: u32,
|
||||
}
|
||||
|
||||
#[typeshare]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ComputerUseScreenshot {
|
||||
pub data_url: String,
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "role", rename_all = "snake_case")]
|
||||
pub enum ComputerUseConversationMessage {
|
||||
User { text: String },
|
||||
Assistant { text: String },
|
||||
}
|
||||
|
||||
#[typeshare]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ComputerUseStartRequest {
|
||||
pub prompt: String,
|
||||
#[serde(default)]
|
||||
pub continue_conversation: bool,
|
||||
pub client_id: String,
|
||||
}
|
||||
|
||||
#[typeshare]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ComputerUseConfigResponse {
|
||||
pub enabled: bool,
|
||||
pub base_url: String,
|
||||
pub model: String,
|
||||
pub api_key_configured: bool,
|
||||
pub api_key_source: String,
|
||||
}
|
||||
|
||||
#[typeshare]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ComputerUseConfigUpdate {
|
||||
pub enabled: Option<bool>,
|
||||
pub base_url: Option<String>,
|
||||
pub model: Option<String>,
|
||||
#[serde(alias = "openai_api_key")]
|
||||
pub api_key: Option<String>,
|
||||
#[serde(alias = "clear_openai_api_key")]
|
||||
pub clear_api_key: Option<bool>,
|
||||
}
|
||||
|
||||
#[typeshare]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ComputerUseSessionSummary {
|
||||
pub id: Option<String>,
|
||||
pub status: ComputerUseSessionStatus,
|
||||
pub prompt: Option<String>,
|
||||
pub step: u32,
|
||||
pub last_error: Option<String>,
|
||||
pub final_message: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
pub enum ComputerUseWsClientMessage {
|
||||
ScreenshotResult {
|
||||
request_id: String,
|
||||
screenshot: ComputerUseScreenshot,
|
||||
},
|
||||
ScreenshotError {
|
||||
request_id: String,
|
||||
message: String,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
pub enum ComputerUseWsServerMessage {
|
||||
SessionUpdated { session: ComputerUseSessionSummary },
|
||||
ScreenshotRequested { request_id: String },
|
||||
ScreenshotCaptured { screenshot: ComputerUseScreenshot },
|
||||
StepStarted { step: u32 },
|
||||
ReasoningDelta { delta: String },
|
||||
ReasoningCompleted { failed: bool },
|
||||
ActionsExecuted { actions: Vec<ComputerUseAction> },
|
||||
Error { message: String },
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn computer_use_action_keeps_flat_tagged_json_shape() {
|
||||
let action = ComputerUseAction::Click {
|
||||
x: 10,
|
||||
y: 20,
|
||||
button: ComputerUseButton::Left,
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
serde_json::to_value(action).unwrap(),
|
||||
json!({
|
||||
"type": "click",
|
||||
"x": 10,
|
||||
"y": 20,
|
||||
"button": "left"
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn computer_use_ws_message_keeps_flat_tagged_json_shape() {
|
||||
let message = ComputerUseWsServerMessage::ScreenshotRequested {
|
||||
request_id: "req-1".to_string(),
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
serde_json::to_value(message).unwrap(),
|
||||
json!({
|
||||
"type": "screenshot_requested",
|
||||
"request_id": "req-1"
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_update_accepts_legacy_api_key_names() {
|
||||
let update: ComputerUseConfigUpdate = serde_json::from_value(json!({
|
||||
"openai_api_key": "legacy-key",
|
||||
"clear_openai_api_key": true
|
||||
}))
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(update.api_key.as_deref(), Some("legacy-key"));
|
||||
assert_eq!(update.clear_api_key, Some(true));
|
||||
}
|
||||
}
|
||||
@@ -1,983 +0,0 @@
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use axum::extract::ws::{Message, WebSocket};
|
||||
use futures::{SinkExt, StreamExt};
|
||||
use tokio::sync::{broadcast, oneshot, watch, Mutex};
|
||||
use tokio::task::JoinHandle;
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::actions::*;
|
||||
use super::openai::{normalize_data_url, OpenAiComputerProvider};
|
||||
use crate::config::ConfigStore;
|
||||
use crate::error::{AppError, Result};
|
||||
use crate::hid::{
|
||||
CanonicalKey, HidController, KeyEventType, KeyboardEvent, KeyboardModifiers, MouseButton,
|
||||
MouseEvent,
|
||||
};
|
||||
|
||||
const SCREENSHOT_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
const KEY_DELAY: Duration = Duration::from_millis(35);
|
||||
const ACTION_DELAY: Duration = Duration::from_millis(120);
|
||||
const STOPPED_MESSAGE: &str = "Computer use task was stopped";
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct ComputerUseManager {
|
||||
config: ConfigStore,
|
||||
hid: Arc<HidController>,
|
||||
state: Arc<Mutex<ManagerState>>,
|
||||
event_tx: broadcast::Sender<ComputerUseWsServerMessage>,
|
||||
screenshot_tx: broadcast::Sender<ScreenshotRequest>,
|
||||
}
|
||||
|
||||
struct ManagerState {
|
||||
session: ComputerUseSessionSummary,
|
||||
conversation: Vec<ComputerUseConversationMessage>,
|
||||
screenshot_waiter: Option<ScreenshotWaiter>,
|
||||
stop_tx: Option<oneshot::Sender<()>>,
|
||||
cancel_tx: Option<watch::Sender<bool>>,
|
||||
task: Option<JoinHandle<()>>,
|
||||
}
|
||||
|
||||
struct ScreenshotWaiter {
|
||||
request_id: String,
|
||||
client_id: String,
|
||||
tx: oneshot::Sender<Result<ComputerUseScreenshot>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct ScreenshotRequest {
|
||||
request_id: String,
|
||||
client_id: String,
|
||||
}
|
||||
|
||||
impl ComputerUseManager {
|
||||
pub fn new(config: ConfigStore, hid: Arc<HidController>) -> Arc<Self> {
|
||||
let (event_tx, _) = broadcast::channel(128);
|
||||
let (screenshot_tx, _) = broadcast::channel(8);
|
||||
Arc::new(Self {
|
||||
config,
|
||||
hid,
|
||||
state: Arc::new(Mutex::new(ManagerState {
|
||||
session: empty_session(),
|
||||
conversation: Vec::new(),
|
||||
screenshot_waiter: None,
|
||||
stop_tx: None,
|
||||
cancel_tx: None,
|
||||
task: None,
|
||||
})),
|
||||
event_tx,
|
||||
screenshot_tx,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn config_response(&self) -> ComputerUseConfigResponse {
|
||||
let config = self.config.get();
|
||||
let key_env = cua_api_key_env();
|
||||
let key_db = config
|
||||
.computer_use
|
||||
.api_key
|
||||
.as_ref()
|
||||
.filter(|key| !key.is_empty());
|
||||
ComputerUseConfigResponse {
|
||||
enabled: config.computer_use.enabled,
|
||||
base_url: cua_base_url_env().unwrap_or_else(|| config.computer_use.base_url.clone()),
|
||||
model: config.computer_use.model.clone(),
|
||||
api_key_configured: key_env.is_some() || key_db.is_some(),
|
||||
api_key_source: if key_env.is_some() {
|
||||
"env".to_string()
|
||||
} else if key_db.is_some() {
|
||||
"config".to_string()
|
||||
} else {
|
||||
"none".to_string()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn update_config(
|
||||
&self,
|
||||
req: ComputerUseConfigUpdate,
|
||||
) -> Result<ComputerUseConfigResponse> {
|
||||
if let Some(base_url) = req
|
||||
.base_url
|
||||
.as_ref()
|
||||
.filter(|base_url| !base_url.trim().is_empty())
|
||||
{
|
||||
validate_endpoint_url(base_url)?;
|
||||
}
|
||||
|
||||
self.config
|
||||
.update(|config| {
|
||||
if let Some(enabled) = req.enabled {
|
||||
config.computer_use.enabled = enabled;
|
||||
}
|
||||
if let Some(model) = req.model.as_ref().filter(|model| !model.trim().is_empty()) {
|
||||
config.computer_use.model = model.trim().to_string();
|
||||
}
|
||||
if let Some(base_url) = req
|
||||
.base_url
|
||||
.as_ref()
|
||||
.filter(|base_url| !base_url.trim().is_empty())
|
||||
{
|
||||
config.computer_use.base_url = base_url.trim().to_string();
|
||||
}
|
||||
if req.clear_api_key.unwrap_or(false) {
|
||||
config.computer_use.api_key = None;
|
||||
}
|
||||
if let Some(key) = req.api_key.as_ref() {
|
||||
config.computer_use.api_key = if key.trim().is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(key.trim().to_string())
|
||||
};
|
||||
}
|
||||
})
|
||||
.await?;
|
||||
|
||||
Ok(self.config_response())
|
||||
}
|
||||
|
||||
pub async fn summary(&self) -> ComputerUseSessionSummary {
|
||||
self.state.lock().await.session.clone()
|
||||
}
|
||||
|
||||
pub async fn start(
|
||||
self: &Arc<Self>,
|
||||
req: ComputerUseStartRequest,
|
||||
) -> Result<ComputerUseSessionSummary> {
|
||||
let app_config = self.config.get();
|
||||
let config = app_config.computer_use.clone();
|
||||
if !config.enabled {
|
||||
return Err(AppError::BadRequest("Computer use is disabled".to_string()));
|
||||
}
|
||||
if req.prompt.trim().is_empty() {
|
||||
return Err(AppError::BadRequest("Task prompt is required".to_string()));
|
||||
}
|
||||
let client_id = req.client_id.trim();
|
||||
if client_id.is_empty() {
|
||||
return Err(AppError::BadRequest(
|
||||
"Computer use client_id is required".to_string(),
|
||||
));
|
||||
}
|
||||
let client_id = client_id.to_string();
|
||||
let hid = self.hid.snapshot().await;
|
||||
if !hid.initialized || !hid.supports_absolute_mouse {
|
||||
return Err(AppError::BadRequest(
|
||||
"Computer use requires an initialized absolute mouse HID backend".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let api_key = cua_api_key_env()
|
||||
.or(config.api_key.clone())
|
||||
.ok_or_else(|| {
|
||||
AppError::BadRequest("Computer Use API key is not configured".to_string())
|
||||
})?;
|
||||
let base_url = cua_base_url_env().unwrap_or_else(|| config.base_url.clone());
|
||||
validate_endpoint_url(&base_url)?;
|
||||
|
||||
let mut state = self.state.lock().await;
|
||||
if matches!(
|
||||
state.session.status,
|
||||
ComputerUseSessionStatus::WaitingScreenshot
|
||||
| ComputerUseSessionStatus::Thinking
|
||||
| ComputerUseSessionStatus::Executing
|
||||
) {
|
||||
return Err(AppError::BadRequest(
|
||||
"A computer use session is already running".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
if let Some(handle) = state.task.take() {
|
||||
handle.abort();
|
||||
}
|
||||
if !req.continue_conversation {
|
||||
state.conversation.clear();
|
||||
}
|
||||
let conversation = state.conversation.clone();
|
||||
state
|
||||
.conversation
|
||||
.push(ComputerUseConversationMessage::User {
|
||||
text: req.prompt.trim().to_string(),
|
||||
});
|
||||
|
||||
let (stop_tx, stop_rx) = oneshot::channel();
|
||||
let (cancel_tx, cancel_rx) = watch::channel(false);
|
||||
let session_id = Uuid::new_v4().to_string();
|
||||
state.session = ComputerUseSessionSummary {
|
||||
id: Some(session_id),
|
||||
status: ComputerUseSessionStatus::Thinking,
|
||||
prompt: Some(req.prompt.trim().to_string()),
|
||||
step: 0,
|
||||
last_error: None,
|
||||
final_message: None,
|
||||
};
|
||||
state.stop_tx = Some(stop_tx);
|
||||
state.cancel_tx = Some(cancel_tx);
|
||||
let summary = state.session.clone();
|
||||
drop(state);
|
||||
|
||||
self.publish_session().await;
|
||||
let manager = self.clone();
|
||||
let prompt = req.prompt.trim().to_string();
|
||||
let model = config.model.clone();
|
||||
let handle = tokio::spawn(async move {
|
||||
manager
|
||||
.run_loop(
|
||||
prompt,
|
||||
api_key,
|
||||
base_url,
|
||||
model,
|
||||
conversation,
|
||||
client_id,
|
||||
cancel_rx,
|
||||
stop_rx,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
|
||||
self.state.lock().await.task = Some(handle);
|
||||
Ok(summary)
|
||||
}
|
||||
|
||||
pub async fn stop(&self) -> Result<ComputerUseSessionSummary> {
|
||||
let mut state = self.state.lock().await;
|
||||
if let Some(tx) = state.stop_tx.take() {
|
||||
let _ = tx.send(());
|
||||
}
|
||||
if let Some(tx) = state.cancel_tx.take() {
|
||||
let _ = tx.send(true);
|
||||
}
|
||||
if let Some(waiter) = state.screenshot_waiter.take() {
|
||||
drop(waiter.tx);
|
||||
}
|
||||
state.session.status = ComputerUseSessionStatus::Stopped;
|
||||
drop(state);
|
||||
let _ = self.hid.reset().await;
|
||||
self.publish_session().await;
|
||||
Ok(self.summary().await)
|
||||
}
|
||||
|
||||
pub async fn submit_screenshot(
|
||||
&self,
|
||||
client_id: &str,
|
||||
request_id: String,
|
||||
mut screenshot: ComputerUseScreenshot,
|
||||
) -> Result<()> {
|
||||
if screenshot.width == 0 || screenshot.height == 0 {
|
||||
return Err(AppError::BadRequest(
|
||||
"Screenshot dimensions are invalid".to_string(),
|
||||
));
|
||||
}
|
||||
screenshot.data_url = normalize_data_url(&screenshot.data_url)?;
|
||||
|
||||
let mut state = self.state.lock().await;
|
||||
let Some(waiter) = state.screenshot_waiter.take() else {
|
||||
return Ok(());
|
||||
};
|
||||
if waiter.request_id != request_id || waiter.client_id != client_id {
|
||||
state.screenshot_waiter = Some(waiter);
|
||||
return Ok(());
|
||||
}
|
||||
let _ = waiter.tx.send(Ok(screenshot));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn submit_screenshot_error(&self, client_id: &str, request_id: String, message: String) {
|
||||
let mut state = self.state.lock().await;
|
||||
let Some(waiter) = state.screenshot_waiter.take() else {
|
||||
return;
|
||||
};
|
||||
if waiter.request_id != request_id || waiter.client_id != client_id {
|
||||
state.screenshot_waiter = Some(waiter);
|
||||
return;
|
||||
}
|
||||
let message: String = message.chars().take(300).collect();
|
||||
let _ = waiter.tx.send(Err(AppError::ServiceUnavailable(format!(
|
||||
"Screenshot capture failed: {}",
|
||||
if message.trim().is_empty() {
|
||||
"client did not provide an error"
|
||||
} else {
|
||||
message.trim()
|
||||
}
|
||||
))));
|
||||
}
|
||||
|
||||
pub async fn handle_socket(self: Arc<Self>, socket: WebSocket, client_id: Option<String>) {
|
||||
let (mut sender, mut receiver) = socket.split();
|
||||
let mut event_rx = self.event_tx.subscribe();
|
||||
let client_id = client_id
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|client_id| !client_id.is_empty())
|
||||
.map(str::to_string)
|
||||
.unwrap_or_else(|| Uuid::new_v4().to_string());
|
||||
let mut screenshot_rx = self.screenshot_tx.subscribe();
|
||||
|
||||
let _ = sender
|
||||
.send(Message::Text(
|
||||
serde_json::to_string(&ComputerUseWsServerMessage::SessionUpdated {
|
||||
session: self.summary().await,
|
||||
})
|
||||
.unwrap_or_default()
|
||||
.into(),
|
||||
))
|
||||
.await;
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
Ok(event) = event_rx.recv() => {
|
||||
if let Ok(text) = serde_json::to_string(&event) {
|
||||
if sender.send(Message::Text(text.into())).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(req) = screenshot_rx.recv() => {
|
||||
if req.client_id != client_id {
|
||||
continue;
|
||||
}
|
||||
let event = ComputerUseWsServerMessage::ScreenshotRequested { request_id: req.request_id };
|
||||
if let Ok(text) = serde_json::to_string(&event) {
|
||||
if sender.send(Message::Text(text.into())).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
msg = receiver.next() => {
|
||||
match msg {
|
||||
Some(Ok(Message::Text(text))) => {
|
||||
match serde_json::from_str::<ComputerUseWsClientMessage>(&text) {
|
||||
Ok(ComputerUseWsClientMessage::ScreenshotResult { request_id, screenshot }) => {
|
||||
let _ = self.submit_screenshot(&client_id, request_id, screenshot).await;
|
||||
}
|
||||
Ok(ComputerUseWsClientMessage::ScreenshotError { request_id, message }) => {
|
||||
self.submit_screenshot_error(&client_id, request_id, message).await;
|
||||
}
|
||||
Err(_) => {}
|
||||
}
|
||||
}
|
||||
Some(Ok(Message::Close(_))) | None => break,
|
||||
Some(Err(_)) => break,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_loop(
|
||||
&self,
|
||||
prompt: String,
|
||||
api_key: String,
|
||||
base_url: String,
|
||||
model: String,
|
||||
conversation: Vec<ComputerUseConversationMessage>,
|
||||
client_id: String,
|
||||
cancel_rx: watch::Receiver<bool>,
|
||||
mut stop_rx: oneshot::Receiver<()>,
|
||||
) {
|
||||
let provider = OpenAiComputerProvider::new(api_key, base_url, model);
|
||||
let mut latest_screenshot: Option<ComputerUseScreenshot> = None;
|
||||
let mut action_history: Vec<String> = Vec::new();
|
||||
let mut step = 0_u32;
|
||||
|
||||
loop {
|
||||
step = step.saturating_add(1);
|
||||
self.set_status(ComputerUseSessionStatus::Thinking, step, None)
|
||||
.await;
|
||||
let response = tokio::select! {
|
||||
_ = &mut stop_rx => {
|
||||
let _ = self.event_tx.send(ComputerUseWsServerMessage::ReasoningCompleted {
|
||||
failed: true,
|
||||
});
|
||||
self.set_stopped().await;
|
||||
return;
|
||||
}
|
||||
response = provider.next_actions(
|
||||
&prompt,
|
||||
&conversation,
|
||||
&action_history,
|
||||
latest_screenshot.as_ref(),
|
||||
|delta| {
|
||||
let _ = self.event_tx.send(ComputerUseWsServerMessage::ReasoningDelta {
|
||||
delta: delta.to_string(),
|
||||
});
|
||||
},
|
||||
) => response,
|
||||
};
|
||||
|
||||
let response = match response {
|
||||
Ok(response) => {
|
||||
let _ = self
|
||||
.event_tx
|
||||
.send(ComputerUseWsServerMessage::ReasoningCompleted { failed: false });
|
||||
response
|
||||
}
|
||||
Err(err) => {
|
||||
let _ = self
|
||||
.event_tx
|
||||
.send(ComputerUseWsServerMessage::ReasoningCompleted { failed: true });
|
||||
self.fail(&err.to_string()).await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
if *cancel_rx.borrow() {
|
||||
self.set_stopped().await;
|
||||
return;
|
||||
}
|
||||
|
||||
if response.done {
|
||||
self.complete(response.message).await;
|
||||
return;
|
||||
}
|
||||
|
||||
let executable = &response.actions[..response.actions.len().saturating_sub(1)];
|
||||
action_history.push(format!(
|
||||
"Step {step}: {}",
|
||||
serde_json::to_string(&response.actions).unwrap_or_else(|_| "[]".to_string())
|
||||
));
|
||||
if !executable.is_empty() {
|
||||
let Some(screenshot) = latest_screenshot.as_ref() else {
|
||||
self.fail("Computer Use protocol error: actions require a screenshot")
|
||||
.await;
|
||||
return;
|
||||
};
|
||||
self.set_status(ComputerUseSessionStatus::Executing, step, None)
|
||||
.await;
|
||||
if let Err(err) = self
|
||||
.execute_actions(
|
||||
executable,
|
||||
screenshot.width,
|
||||
screenshot.height,
|
||||
cancel_rx.clone(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
if *cancel_rx.borrow() {
|
||||
self.set_stopped().await;
|
||||
} else {
|
||||
self.fail(&err.to_string()).await;
|
||||
}
|
||||
return;
|
||||
}
|
||||
let _ = self
|
||||
.event_tx
|
||||
.send(ComputerUseWsServerMessage::ActionsExecuted {
|
||||
actions: executable.to_vec(),
|
||||
});
|
||||
}
|
||||
|
||||
self.set_status(ComputerUseSessionStatus::WaitingScreenshot, step, None)
|
||||
.await;
|
||||
let screenshot = tokio::select! {
|
||||
_ = &mut stop_rx => {
|
||||
self.set_stopped().await;
|
||||
return;
|
||||
}
|
||||
screenshot = self.request_screenshot(&client_id) => screenshot,
|
||||
};
|
||||
let screenshot = match screenshot {
|
||||
Ok(screenshot) => screenshot,
|
||||
Err(err) => {
|
||||
self.fail(&err.to_string()).await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
let _ = self
|
||||
.event_tx
|
||||
.send(ComputerUseWsServerMessage::ScreenshotCaptured {
|
||||
screenshot: screenshot.clone(),
|
||||
});
|
||||
latest_screenshot = Some(screenshot);
|
||||
}
|
||||
}
|
||||
|
||||
async fn request_screenshot(&self, client_id: &str) -> Result<ComputerUseScreenshot> {
|
||||
let request_id = Uuid::new_v4().to_string();
|
||||
let (tx, rx) = oneshot::channel();
|
||||
{
|
||||
let mut state = self.state.lock().await;
|
||||
state.screenshot_waiter = Some(ScreenshotWaiter {
|
||||
request_id: request_id.clone(),
|
||||
client_id: client_id.to_string(),
|
||||
tx,
|
||||
});
|
||||
}
|
||||
let _ = self.screenshot_tx.send(ScreenshotRequest {
|
||||
request_id,
|
||||
client_id: client_id.to_string(),
|
||||
});
|
||||
let reply = tokio::time::timeout(SCREENSHOT_TIMEOUT, rx)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
AppError::ServiceUnavailable("Timed out waiting for screenshot".to_string())
|
||||
})?
|
||||
.map_err(|_| {
|
||||
AppError::ServiceUnavailable("Screenshot request was cancelled".to_string())
|
||||
})?;
|
||||
reply
|
||||
}
|
||||
|
||||
async fn execute_actions(
|
||||
&self,
|
||||
actions: &[ComputerUseAction],
|
||||
width: u32,
|
||||
height: u32,
|
||||
mut cancel_rx: watch::Receiver<bool>,
|
||||
) -> Result<()> {
|
||||
for action in actions {
|
||||
if *cancel_rx.borrow() {
|
||||
return Err(stopped_error());
|
||||
}
|
||||
match action {
|
||||
ComputerUseAction::Click { x, y, button } => {
|
||||
self.move_abs(*x, *y, width, height).await?;
|
||||
self.mouse_button(*button, true).await?;
|
||||
let click_result = sleep_or_cancel(KEY_DELAY, &mut cancel_rx).await;
|
||||
self.mouse_button(*button, false).await?;
|
||||
click_result?;
|
||||
}
|
||||
ComputerUseAction::DoubleClick { x, y, button } => {
|
||||
for _ in 0..2 {
|
||||
self.move_abs(*x, *y, width, height).await?;
|
||||
self.mouse_button(*button, true).await?;
|
||||
let click_result = sleep_or_cancel(KEY_DELAY, &mut cancel_rx).await;
|
||||
self.mouse_button(*button, false).await?;
|
||||
click_result?;
|
||||
sleep_or_cancel(KEY_DELAY, &mut cancel_rx).await?;
|
||||
}
|
||||
}
|
||||
ComputerUseAction::Move { x, y } => self.move_abs(*x, *y, width, height).await?,
|
||||
ComputerUseAction::Drag { path, button } => {
|
||||
if let Some(first) = path.first() {
|
||||
self.move_abs(first.x, first.y, width, height).await?;
|
||||
self.mouse_button(*button, true).await?;
|
||||
let drag_result = async {
|
||||
for point in path.iter().skip(1) {
|
||||
sleep_or_cancel(KEY_DELAY, &mut cancel_rx).await?;
|
||||
self.move_abs(point.x, point.y, width, height).await?;
|
||||
}
|
||||
Result::<()>::Ok(())
|
||||
}
|
||||
.await;
|
||||
self.mouse_button(*button, false).await?;
|
||||
drag_result?;
|
||||
}
|
||||
}
|
||||
ComputerUseAction::Scroll { x, y, dy, .. } => {
|
||||
self.move_abs(*x, *y, width, height).await?;
|
||||
let ticks = ((*dy).clamp(-1200, 1200) / 120).clamp(-10, 10);
|
||||
let ticks = if ticks == 0 { dy.signum() } else { ticks };
|
||||
for _ in 0..ticks.abs() {
|
||||
if *cancel_rx.borrow() {
|
||||
return Err(stopped_error());
|
||||
}
|
||||
self.hid
|
||||
.send_mouse(MouseEvent::scroll(if ticks > 0 { 1 } else { -1 }))
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
ComputerUseAction::Type { text } => self.type_text(text, &mut cancel_rx).await?,
|
||||
ComputerUseAction::Keypress { keys } => self.keypress(keys, &mut cancel_rx).await?,
|
||||
ComputerUseAction::Wait { ms } => {
|
||||
sleep_or_cancel(Duration::from_millis((*ms).min(5000)), &mut cancel_rx).await?
|
||||
}
|
||||
ComputerUseAction::Screenshot => {}
|
||||
}
|
||||
sleep_or_cancel(ACTION_DELAY, &mut cancel_rx).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn move_abs(&self, x: u32, y: u32, width: u32, height: u32) -> Result<()> {
|
||||
let hid_x = ((x.min(width.saturating_sub(1)) as f64 / width.max(1) as f64) * 32767.0)
|
||||
.round() as i32;
|
||||
let hid_y = ((y.min(height.saturating_sub(1)) as f64 / height.max(1) as f64) * 32767.0)
|
||||
.round() as i32;
|
||||
self.hid
|
||||
.send_mouse(MouseEvent::move_abs(hid_x, hid_y))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn mouse_button(&self, button: ComputerUseButton, down: bool) -> Result<()> {
|
||||
let button = match button {
|
||||
ComputerUseButton::Left => MouseButton::Left,
|
||||
ComputerUseButton::Middle => MouseButton::Middle,
|
||||
ComputerUseButton::Right => MouseButton::Right,
|
||||
};
|
||||
let event = if down {
|
||||
MouseEvent::button_down(button)
|
||||
} else {
|
||||
MouseEvent::button_up(button)
|
||||
};
|
||||
self.hid.send_mouse(event).await
|
||||
}
|
||||
|
||||
async fn type_text(&self, text: &str, cancel_rx: &mut watch::Receiver<bool>) -> Result<()> {
|
||||
for ch in text.chars() {
|
||||
if *cancel_rx.borrow() {
|
||||
return Err(stopped_error());
|
||||
}
|
||||
let (key, mods) = char_to_key(ch).ok_or_else(|| {
|
||||
AppError::BadRequest(format!(
|
||||
"Cannot type unsupported character {ch:?} through HID keyboard mapping"
|
||||
))
|
||||
})?;
|
||||
self.key_down_up(key, mods, cancel_rx).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn keypress(&self, keys: &[String], cancel_rx: &mut watch::Receiver<bool>) -> Result<()> {
|
||||
let mut mods = KeyboardModifiers::default();
|
||||
let mut key = None;
|
||||
for item in keys {
|
||||
match item.to_lowercase().as_str() {
|
||||
"ctrl" | "control" | "controlleft" => mods.left_ctrl = true,
|
||||
"shift" | "shiftleft" => mods.left_shift = true,
|
||||
"alt" | "altleft" => mods.left_alt = true,
|
||||
"meta" | "win" | "cmd" | "super" => mods.left_meta = true,
|
||||
other => key = key_name_to_canonical(other),
|
||||
}
|
||||
}
|
||||
if let Some(key) = key {
|
||||
self.key_down_up(key, mods, cancel_rx).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn key_down_up(
|
||||
&self,
|
||||
key: CanonicalKey,
|
||||
mods: KeyboardModifiers,
|
||||
cancel_rx: &mut watch::Receiver<bool>,
|
||||
) -> Result<()> {
|
||||
self.hid
|
||||
.send_keyboard(KeyboardEvent {
|
||||
event_type: KeyEventType::Down,
|
||||
key,
|
||||
modifiers: mods,
|
||||
})
|
||||
.await?;
|
||||
let key_result = sleep_or_cancel(KEY_DELAY, cancel_rx).await;
|
||||
self.hid
|
||||
.send_keyboard(KeyboardEvent {
|
||||
event_type: KeyEventType::Up,
|
||||
key,
|
||||
modifiers: KeyboardModifiers::default(),
|
||||
})
|
||||
.await?;
|
||||
key_result
|
||||
}
|
||||
|
||||
async fn publish_session(&self) {
|
||||
let _ = self
|
||||
.event_tx
|
||||
.send(ComputerUseWsServerMessage::SessionUpdated {
|
||||
session: self.summary().await,
|
||||
});
|
||||
}
|
||||
|
||||
async fn set_status(&self, status: ComputerUseSessionStatus, step: u32, error: Option<String>) {
|
||||
{
|
||||
let mut state = self.state.lock().await;
|
||||
state.session.status = status;
|
||||
state.session.step = step;
|
||||
state.session.last_error = error;
|
||||
}
|
||||
if matches!(status, ComputerUseSessionStatus::Thinking) {
|
||||
let _ = self
|
||||
.event_tx
|
||||
.send(ComputerUseWsServerMessage::StepStarted { step });
|
||||
}
|
||||
self.publish_session().await;
|
||||
}
|
||||
|
||||
async fn complete(&self, message: Option<String>) {
|
||||
{
|
||||
let mut state = self.state.lock().await;
|
||||
if let Some(message) = message.as_ref().filter(|message| !message.is_empty()) {
|
||||
state
|
||||
.conversation
|
||||
.push(ComputerUseConversationMessage::Assistant {
|
||||
text: message.clone(),
|
||||
});
|
||||
}
|
||||
state.session.status = ComputerUseSessionStatus::Completed;
|
||||
state.session.final_message = message;
|
||||
state.stop_tx = None;
|
||||
}
|
||||
self.publish_session().await;
|
||||
let _ = self.hid.reset().await;
|
||||
}
|
||||
|
||||
async fn fail(&self, message: &str) {
|
||||
{
|
||||
let mut state = self.state.lock().await;
|
||||
state.session.status = ComputerUseSessionStatus::Failed;
|
||||
state.session.last_error = Some(message.to_string());
|
||||
state.stop_tx = None;
|
||||
}
|
||||
let _ = self.event_tx.send(ComputerUseWsServerMessage::Error {
|
||||
message: message.to_string(),
|
||||
});
|
||||
self.publish_session().await;
|
||||
let _ = self.hid.reset().await;
|
||||
}
|
||||
|
||||
async fn set_stopped(&self) {
|
||||
{
|
||||
let mut state = self.state.lock().await;
|
||||
state.session.status = ComputerUseSessionStatus::Stopped;
|
||||
state.stop_tx = None;
|
||||
}
|
||||
self.publish_session().await;
|
||||
let _ = self.hid.reset().await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn sleep_or_cancel(duration: Duration, cancel_rx: &mut watch::Receiver<bool>) -> Result<()> {
|
||||
if *cancel_rx.borrow() {
|
||||
return Err(stopped_error());
|
||||
}
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(duration) => Ok(()),
|
||||
changed = cancel_rx.changed() => {
|
||||
match changed {
|
||||
Ok(()) if *cancel_rx.borrow() => {
|
||||
Err(stopped_error())
|
||||
}
|
||||
Ok(()) => Ok(()),
|
||||
Err(_) => Err(stopped_error()),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn stopped_error() -> AppError {
|
||||
AppError::BadRequest(STOPPED_MESSAGE.to_string())
|
||||
}
|
||||
|
||||
fn empty_session() -> ComputerUseSessionSummary {
|
||||
ComputerUseSessionSummary {
|
||||
id: None,
|
||||
status: ComputerUseSessionStatus::Idle,
|
||||
prompt: None,
|
||||
step: 0,
|
||||
last_error: None,
|
||||
final_message: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn cua_api_key_env() -> Option<String> {
|
||||
std::env::var("ONE_KVM_CUA_API_KEY")
|
||||
.ok()
|
||||
.filter(|key| !key.trim().is_empty())
|
||||
.or_else(|| {
|
||||
std::env::var("OPENAI_API_KEY")
|
||||
.ok()
|
||||
.filter(|key| !key.trim().is_empty())
|
||||
})
|
||||
}
|
||||
|
||||
fn cua_base_url_env() -> Option<String> {
|
||||
std::env::var("ONE_KVM_CUA_BASE_URL")
|
||||
.ok()
|
||||
.filter(|url| !url.trim().is_empty())
|
||||
.or_else(|| {
|
||||
std::env::var("ONE_KVM_OPENAI_BASE_URL")
|
||||
.ok()
|
||||
.filter(|url| !url.trim().is_empty())
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_endpoint_url(url: &str) -> Result<()> {
|
||||
let trimmed = url.trim();
|
||||
if !(trimmed.starts_with("https://") || trimmed.starts_with("http://")) {
|
||||
return Err(AppError::BadRequest(
|
||||
"API URL must be a complete http(s) endpoint".to_string(),
|
||||
));
|
||||
}
|
||||
if trimmed.ends_with('/') {
|
||||
return Err(AppError::BadRequest(
|
||||
"API URL must include the full endpoint path without a trailing slash".to_string(),
|
||||
));
|
||||
}
|
||||
if !trimmed.contains("/responses") && !trimmed.contains("/chat/completions") {
|
||||
return Err(AppError::BadRequest(
|
||||
"API URL must include /responses or /chat/completions".to_string(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn char_to_key(ch: char) -> Option<(CanonicalKey, KeyboardModifiers)> {
|
||||
let mut mods = KeyboardModifiers::default();
|
||||
let key = match ch {
|
||||
'a'..='z' => key_name_to_canonical(&ch.to_string())?,
|
||||
'A'..='Z' => {
|
||||
mods.left_shift = true;
|
||||
key_name_to_canonical(&ch.to_ascii_lowercase().to_string())?
|
||||
}
|
||||
'0' => CanonicalKey::Digit0,
|
||||
'1' => CanonicalKey::Digit1,
|
||||
'2' => CanonicalKey::Digit2,
|
||||
'3' => CanonicalKey::Digit3,
|
||||
'4' => CanonicalKey::Digit4,
|
||||
'5' => CanonicalKey::Digit5,
|
||||
'6' => CanonicalKey::Digit6,
|
||||
'7' => CanonicalKey::Digit7,
|
||||
'8' => CanonicalKey::Digit8,
|
||||
'9' => CanonicalKey::Digit9,
|
||||
' ' => CanonicalKey::Space,
|
||||
'\n' => CanonicalKey::Enter,
|
||||
'-' => CanonicalKey::Minus,
|
||||
'_' => {
|
||||
mods.left_shift = true;
|
||||
CanonicalKey::Minus
|
||||
}
|
||||
'=' => CanonicalKey::Equal,
|
||||
'+' => {
|
||||
mods.left_shift = true;
|
||||
CanonicalKey::Equal
|
||||
}
|
||||
'.' => CanonicalKey::Period,
|
||||
',' => CanonicalKey::Comma,
|
||||
'/' => CanonicalKey::Slash,
|
||||
'?' => {
|
||||
mods.left_shift = true;
|
||||
CanonicalKey::Slash
|
||||
}
|
||||
';' => CanonicalKey::Semicolon,
|
||||
':' => {
|
||||
mods.left_shift = true;
|
||||
CanonicalKey::Semicolon
|
||||
}
|
||||
'\'' => CanonicalKey::Quote,
|
||||
'"' => {
|
||||
mods.left_shift = true;
|
||||
CanonicalKey::Quote
|
||||
}
|
||||
'[' => CanonicalKey::BracketLeft,
|
||||
'{' => {
|
||||
mods.left_shift = true;
|
||||
CanonicalKey::BracketLeft
|
||||
}
|
||||
']' => CanonicalKey::BracketRight,
|
||||
'}' => {
|
||||
mods.left_shift = true;
|
||||
CanonicalKey::BracketRight
|
||||
}
|
||||
'\\' => CanonicalKey::Backslash,
|
||||
'|' => {
|
||||
mods.left_shift = true;
|
||||
CanonicalKey::Backslash
|
||||
}
|
||||
'`' => CanonicalKey::Backquote,
|
||||
'~' => {
|
||||
mods.left_shift = true;
|
||||
CanonicalKey::Backquote
|
||||
}
|
||||
'!' => {
|
||||
mods.left_shift = true;
|
||||
CanonicalKey::Digit1
|
||||
}
|
||||
'@' => {
|
||||
mods.left_shift = true;
|
||||
CanonicalKey::Digit2
|
||||
}
|
||||
'#' => {
|
||||
mods.left_shift = true;
|
||||
CanonicalKey::Digit3
|
||||
}
|
||||
'$' => {
|
||||
mods.left_shift = true;
|
||||
CanonicalKey::Digit4
|
||||
}
|
||||
'%' => {
|
||||
mods.left_shift = true;
|
||||
CanonicalKey::Digit5
|
||||
}
|
||||
'^' => {
|
||||
mods.left_shift = true;
|
||||
CanonicalKey::Digit6
|
||||
}
|
||||
'&' => {
|
||||
mods.left_shift = true;
|
||||
CanonicalKey::Digit7
|
||||
}
|
||||
'*' => {
|
||||
mods.left_shift = true;
|
||||
CanonicalKey::Digit8
|
||||
}
|
||||
'(' => {
|
||||
mods.left_shift = true;
|
||||
CanonicalKey::Digit9
|
||||
}
|
||||
')' => {
|
||||
mods.left_shift = true;
|
||||
CanonicalKey::Digit0
|
||||
}
|
||||
_ => return None,
|
||||
};
|
||||
Some((key, mods))
|
||||
}
|
||||
|
||||
fn key_name_to_canonical(name: &str) -> Option<CanonicalKey> {
|
||||
match name.trim().to_lowercase().as_str() {
|
||||
"a" => Some(CanonicalKey::KeyA),
|
||||
"b" => Some(CanonicalKey::KeyB),
|
||||
"c" => Some(CanonicalKey::KeyC),
|
||||
"d" => Some(CanonicalKey::KeyD),
|
||||
"e" => Some(CanonicalKey::KeyE),
|
||||
"f" => Some(CanonicalKey::KeyF),
|
||||
"g" => Some(CanonicalKey::KeyG),
|
||||
"h" => Some(CanonicalKey::KeyH),
|
||||
"i" => Some(CanonicalKey::KeyI),
|
||||
"j" => Some(CanonicalKey::KeyJ),
|
||||
"k" => Some(CanonicalKey::KeyK),
|
||||
"l" => Some(CanonicalKey::KeyL),
|
||||
"m" => Some(CanonicalKey::KeyM),
|
||||
"n" => Some(CanonicalKey::KeyN),
|
||||
"o" => Some(CanonicalKey::KeyO),
|
||||
"p" => Some(CanonicalKey::KeyP),
|
||||
"q" => Some(CanonicalKey::KeyQ),
|
||||
"r" => Some(CanonicalKey::KeyR),
|
||||
"s" => Some(CanonicalKey::KeyS),
|
||||
"t" => Some(CanonicalKey::KeyT),
|
||||
"u" => Some(CanonicalKey::KeyU),
|
||||
"v" => Some(CanonicalKey::KeyV),
|
||||
"w" => Some(CanonicalKey::KeyW),
|
||||
"x" => Some(CanonicalKey::KeyX),
|
||||
"y" => Some(CanonicalKey::KeyY),
|
||||
"z" => Some(CanonicalKey::KeyZ),
|
||||
"enter" | "return" => Some(CanonicalKey::Enter),
|
||||
"escape" | "esc" => Some(CanonicalKey::Escape),
|
||||
"backspace" => Some(CanonicalKey::Backspace),
|
||||
"tab" => Some(CanonicalKey::Tab),
|
||||
"space" => Some(CanonicalKey::Space),
|
||||
"delete" | "del" => Some(CanonicalKey::Delete),
|
||||
"arrowup" | "up" => Some(CanonicalKey::ArrowUp),
|
||||
"arrowdown" | "down" => Some(CanonicalKey::ArrowDown),
|
||||
"arrowleft" | "left" => Some(CanonicalKey::ArrowLeft),
|
||||
"arrowright" | "right" => Some(CanonicalKey::ArrowRight),
|
||||
"home" => Some(CanonicalKey::Home),
|
||||
"end" => Some(CanonicalKey::End),
|
||||
"pageup" => Some(CanonicalKey::PageUp),
|
||||
"pagedown" => Some(CanonicalKey::PageDown),
|
||||
"f1" => Some(CanonicalKey::F1),
|
||||
"f2" => Some(CanonicalKey::F2),
|
||||
"f3" => Some(CanonicalKey::F3),
|
||||
"f4" => Some(CanonicalKey::F4),
|
||||
"f5" => Some(CanonicalKey::F5),
|
||||
"f6" => Some(CanonicalKey::F6),
|
||||
"f7" => Some(CanonicalKey::F7),
|
||||
"f8" => Some(CanonicalKey::F8),
|
||||
"f9" => Some(CanonicalKey::F9),
|
||||
"f10" => Some(CanonicalKey::F10),
|
||||
"f11" => Some(CanonicalKey::F11),
|
||||
"f12" => Some(CanonicalKey::F12),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
mod actions;
|
||||
mod manager;
|
||||
mod openai;
|
||||
|
||||
pub use actions::*;
|
||||
pub use manager::*;
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,47 +0,0 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use typeshare::typeshare;
|
||||
|
||||
#[typeshare]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
#[serde(default)]
|
||||
pub struct ComputerUseConfig {
|
||||
pub enabled: bool,
|
||||
pub base_url: String,
|
||||
pub model: String,
|
||||
#[typeshare(skip)]
|
||||
#[serde(alias = "openai_api_key")]
|
||||
pub api_key: Option<String>,
|
||||
}
|
||||
|
||||
impl Default for ComputerUseConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: false,
|
||||
base_url: "https://api.openai.com/v1/responses".to_string(),
|
||||
model: "gpt-5.5".to_string(),
|
||||
api_key: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn legacy_openai_api_key_migrates_to_generic_key() {
|
||||
let config: ComputerUseConfig = serde_json::from_value(serde_json::json!({
|
||||
"enabled": true,
|
||||
"provider": "openai",
|
||||
"base_url": "https://example.test/v1/chat/completions",
|
||||
"model": "vision-model",
|
||||
"openai_api_key": "legacy-key",
|
||||
"max_steps": 30,
|
||||
"timeout_seconds": 600
|
||||
}))
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(config.api_key.as_deref(), Some("legacy-key"));
|
||||
assert_eq!(config.model, "vision-model");
|
||||
}
|
||||
}
|
||||
43
src/config/schema/mcp.rs
Normal file
43
src/config/schema/mcp.rs
Normal file
@@ -0,0 +1,43 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use typeshare::typeshare;
|
||||
|
||||
#[typeshare]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct McpConfig {
|
||||
pub enabled: bool,
|
||||
pub requests_per_second: u32,
|
||||
#[typeshare(skip)]
|
||||
pub api_key: Option<String>,
|
||||
#[typeshare(skip)]
|
||||
pub api_key_hash: Option<String>,
|
||||
}
|
||||
impl Default for McpConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: false,
|
||||
requests_per_second: 5,
|
||||
api_key: None,
|
||||
api_key_hash: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn mcp_defaults_are_disabled_and_have_no_model_credentials() {
|
||||
let config: super::super::AppConfig = serde_json::from_value(
|
||||
serde_json::json!({"computer_use":{"enabled":true,"api_key":"old-model-key"}}),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(!config.mcp.enabled);
|
||||
assert!(config.mcp.api_key_hash.is_none());
|
||||
assert!(config.mcp.api_key.is_none());
|
||||
assert_eq!(config.mcp.requests_per_second, 5);
|
||||
assert!(serde_json::to_value(&config)
|
||||
.unwrap()
|
||||
.get("computer_use")
|
||||
.is_none());
|
||||
}
|
||||
}
|
||||
@@ -6,8 +6,8 @@ pub use crate::rustdesk::config::RustDeskConfig;
|
||||
|
||||
mod atx;
|
||||
mod common;
|
||||
mod computer_use;
|
||||
mod hid;
|
||||
mod mcp;
|
||||
mod otg_network;
|
||||
mod stream;
|
||||
mod uac;
|
||||
@@ -16,8 +16,8 @@ mod web;
|
||||
|
||||
pub use atx::*;
|
||||
pub use common::*;
|
||||
pub use computer_use::*;
|
||||
pub use hid::*;
|
||||
pub use mcp::*;
|
||||
pub use otg_network::*;
|
||||
pub use stream::*;
|
||||
pub use uac::*;
|
||||
@@ -39,7 +39,7 @@ pub struct AppConfig {
|
||||
pub audio: AudioConfig,
|
||||
pub stream: StreamConfig,
|
||||
pub web: WebConfig,
|
||||
pub computer_use: ComputerUseConfig,
|
||||
pub mcp: McpConfig,
|
||||
pub extensions: ExtensionsConfig,
|
||||
pub rustdesk: RustDeskConfig,
|
||||
pub vnc: VncConfig,
|
||||
@@ -60,6 +60,7 @@ impl AppConfig {
|
||||
self.hid.mouse_absolute = false;
|
||||
}
|
||||
self.atx.normalize();
|
||||
self.mcp.requests_per_second = self.mcp.requests_per_second.clamp(1, 100);
|
||||
}
|
||||
|
||||
pub fn apply_platform_defaults(&mut self) {
|
||||
|
||||
@@ -27,9 +27,9 @@ impl ConfigStore {
|
||||
}
|
||||
|
||||
pub async fn load(&self) -> Result<()> {
|
||||
let (mut config, removed_legacy_totp) = Self::load_config(&self.pool).await?;
|
||||
let (mut config, removed_legacy_fields) = Self::load_config(&self.pool).await?;
|
||||
config.enforce_invariants();
|
||||
if removed_legacy_totp {
|
||||
if removed_legacy_fields {
|
||||
Self::save_config_to_db(&self.pool, &config).await?;
|
||||
}
|
||||
self.cache.store(Arc::new(config));
|
||||
@@ -46,7 +46,9 @@ impl ConfigStore {
|
||||
Some((json,)) => {
|
||||
let mut value: serde_json::Value =
|
||||
serde_json::from_str(&json).map_err(|e| AppError::Config(e.to_string()))?;
|
||||
let mut removed = false;
|
||||
let mut removed = value
|
||||
.as_object_mut()
|
||||
.is_some_and(|object| object.remove("computer_use").is_some());
|
||||
if let Some(auth) = value
|
||||
.get_mut("auth")
|
||||
.and_then(|value| value.as_object_mut())
|
||||
@@ -175,6 +177,60 @@ mod tests {
|
||||
assert_eq!(config.web.http_port, 9000);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mcp_key_remains_readable_after_config_reload() {
|
||||
let dir = tempdir().unwrap();
|
||||
let db = DatabasePool::new(&dir.path().join("mcp-key.db"))
|
||||
.await
|
||||
.unwrap();
|
||||
db.init_schema().await.unwrap();
|
||||
let store = ConfigStore::new(db.clone_pool());
|
||||
store.load().await.unwrap();
|
||||
let key = crate::mcp::generate_key();
|
||||
store
|
||||
.update(|config| {
|
||||
config.mcp.enabled = true;
|
||||
config.mcp.api_key_hash = Some(crate::mcp::hash_key(&key));
|
||||
config.mcp.api_key = Some(key.clone());
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let reloaded = ConfigStore::new(db.clone_pool());
|
||||
reloaded.load().await.unwrap();
|
||||
assert_eq!(reloaded.get().mcp.api_key.as_deref(), Some(key.as_str()));
|
||||
assert_eq!(
|
||||
reloaded.get().mcp.api_key_hash,
|
||||
store.get().mcp.api_key_hash
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn loading_legacy_agent_cleans_persisted_model_credentials() {
|
||||
let dir = tempdir().unwrap();
|
||||
let db = DatabasePool::new(&dir.path().join("legacy-agent.db"))
|
||||
.await
|
||||
.unwrap();
|
||||
db.init_schema().await.unwrap();
|
||||
let value =
|
||||
serde_json::json!({"computer_use":{"enabled":true,"api_key":"legacy-model-secret"}});
|
||||
sqlx::query("INSERT INTO config (key, value) VALUES ('app_config', ?1)")
|
||||
.bind(value.to_string())
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
let store = ConfigStore::new(db.clone_pool());
|
||||
store.load().await.unwrap();
|
||||
assert!(!store.get().mcp.enabled);
|
||||
assert!(store.get().mcp.api_key_hash.is_none());
|
||||
let (persisted,): (String,) =
|
||||
sqlx::query_as("SELECT value FROM config WHERE key = 'app_config'")
|
||||
.fetch_one(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(!persisted.contains("legacy-model-secret"));
|
||||
assert!(!persisted.contains("computer_use"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn failed_watchdog_persistence_does_not_update_cache() {
|
||||
let dir = tempdir().unwrap();
|
||||
|
||||
@@ -120,6 +120,10 @@ enum QueuedHidEvent {
|
||||
Mouse(MouseEvent),
|
||||
Consumer(ConsumerEvent),
|
||||
Reset,
|
||||
Confirmed(
|
||||
Box<QueuedHidEvent>,
|
||||
tokio::sync::oneshot::Sender<Result<()>>,
|
||||
),
|
||||
}
|
||||
|
||||
pub struct HidController {
|
||||
@@ -313,6 +317,36 @@ impl HidController {
|
||||
self.enqueue_event(QueuedHidEvent::Consumer(event)).await
|
||||
}
|
||||
|
||||
/// MCP uses acknowledged events so timing starts after the hardware send.
|
||||
pub async fn send_keyboard_confirmed(&self, event: KeyboardEvent) -> Result<()> {
|
||||
self.send_confirmed(QueuedHidEvent::Keyboard(event)).await
|
||||
}
|
||||
|
||||
pub async fn send_mouse_confirmed(&self, event: MouseEvent) -> Result<()> {
|
||||
self.send_confirmed(QueuedHidEvent::Mouse(event)).await
|
||||
}
|
||||
|
||||
pub async fn reset_confirmed(&self) -> Result<()> {
|
||||
self.send_confirmed(QueuedHidEvent::Reset).await
|
||||
}
|
||||
|
||||
async fn send_confirmed(&self, event: QueuedHidEvent) -> Result<()> {
|
||||
if !self.backend_available.load(Ordering::Acquire) {
|
||||
return Err(AppError::ServiceUnavailable(
|
||||
"HID backend not available".into(),
|
||||
));
|
||||
}
|
||||
let (tx, rx) = tokio::sync::oneshot::channel();
|
||||
self.enqueue_event(QueuedHidEvent::Confirmed(Box::new(event), tx))
|
||||
.await?;
|
||||
tokio::time::timeout(Duration::from_secs(3), rx)
|
||||
.await
|
||||
.map_err(|_| AppError::ServiceUnavailable("HID hardware send timed out".into()))?
|
||||
.map_err(|_| {
|
||||
AppError::ServiceUnavailable("HID event discarded before sending".into())
|
||||
})?
|
||||
}
|
||||
|
||||
pub async fn bluetooth_status(&self) -> Result<serde_json::Value> {
|
||||
let backend = self
|
||||
.backend
|
||||
@@ -625,6 +659,18 @@ impl HidController {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn test_controller(backend: Arc<dyn HidBackend>) -> Arc<HidController> {
|
||||
#[cfg(unix)]
|
||||
let controller = Arc::new(HidController::new(HidBackendType::None, None));
|
||||
#[cfg(not(unix))]
|
||||
let controller = Arc::new(HidController::new(HidBackendType::None));
|
||||
*controller.backend.write().await = Some(backend);
|
||||
controller.sync_runtime_state_from_backend().await;
|
||||
controller.start_event_worker().await;
|
||||
controller
|
||||
}
|
||||
|
||||
async fn apply_backend_runtime_state(
|
||||
backend_type: &Arc<RwLock<HidBackendType>>,
|
||||
runtime_state: &Arc<RwLock<HidRuntimeState>>,
|
||||
@@ -645,35 +691,34 @@ async fn process_hid_event(
|
||||
event: QueuedHidEvent,
|
||||
backend: &Arc<RwLock<Option<Arc<dyn HidBackend>>>>,
|
||||
) {
|
||||
let backend_opt = backend.read().await.clone();
|
||||
let backend = match backend_opt {
|
||||
Some(b) => b,
|
||||
None => return,
|
||||
let (event, reply) = match event {
|
||||
QueuedHidEvent::Confirmed(event, reply) => (*event, Some(reply)),
|
||||
event => (event, None),
|
||||
};
|
||||
|
||||
let backend_for_send = backend.clone();
|
||||
let result = tokio::task::spawn_blocking(move || {
|
||||
let backend = backend.read().await.clone();
|
||||
let result = match backend {
|
||||
Some(backend) => tokio::task::spawn_blocking(move || {
|
||||
futures::executor::block_on(async move {
|
||||
match event {
|
||||
QueuedHidEvent::Keyboard(ev) => backend_for_send.send_keyboard(ev).await,
|
||||
QueuedHidEvent::Mouse(ev) => backend_for_send.send_mouse(ev).await,
|
||||
QueuedHidEvent::Consumer(ev) => backend_for_send.send_consumer(ev).await,
|
||||
QueuedHidEvent::Reset => backend_for_send.reset().await,
|
||||
QueuedHidEvent::Keyboard(ev) => backend.send_keyboard(ev).await,
|
||||
QueuedHidEvent::Mouse(ev) => backend.send_mouse(ev).await,
|
||||
QueuedHidEvent::Consumer(ev) => backend.send_consumer(ev).await,
|
||||
QueuedHidEvent::Reset => backend.reset().await,
|
||||
QueuedHidEvent::Confirmed(..) => unreachable!("nested confirmed HID event"),
|
||||
}
|
||||
})
|
||||
})
|
||||
.await;
|
||||
|
||||
let result = match result {
|
||||
Ok(r) => r,
|
||||
Err(_) => return,
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(AppError::ServiceUnavailable("HID worker failed".into()))),
|
||||
None => Err(AppError::ServiceUnavailable(
|
||||
"HID backend not available".into(),
|
||||
)),
|
||||
};
|
||||
|
||||
match result {
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
warn!("HID event processing failed: {}", e);
|
||||
if let Err(error) = &result {
|
||||
warn!("HID event processing failed: {}", error);
|
||||
}
|
||||
if let Some(reply) = reply {
|
||||
let _ = reply.send(result);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -10,8 +10,6 @@ pub mod audio;
|
||||
#[cfg(feature = "desktop")]
|
||||
pub mod auth;
|
||||
#[cfg(feature = "desktop")]
|
||||
pub mod computer_use;
|
||||
#[cfg(feature = "desktop")]
|
||||
pub mod config;
|
||||
#[cfg(feature = "desktop")]
|
||||
pub mod db;
|
||||
@@ -25,6 +23,8 @@ pub mod events;
|
||||
pub mod extensions;
|
||||
#[cfg(feature = "desktop")]
|
||||
pub mod hid;
|
||||
#[cfg(feature = "desktop")]
|
||||
pub mod mcp;
|
||||
#[cfg(all(unix, feature = "desktop"))]
|
||||
pub mod msd;
|
||||
#[cfg(all(unix, feature = "desktop"))]
|
||||
|
||||
332
src/mcp/actions.rs
Normal file
332
src/mcp/actions.rs
Normal file
@@ -0,0 +1,332 @@
|
||||
use super::keyboard::{char_to_key, key_name_to_canonical};
|
||||
use crate::hid::{
|
||||
CanonicalKey, HidController, KeyEventType, KeyboardEvent, KeyboardModifiers, MouseButton,
|
||||
MouseEvent,
|
||||
};
|
||||
use rmcp::schemars::{self, JsonSchema};
|
||||
use serde::Deserialize;
|
||||
use std::time::Duration;
|
||||
|
||||
#[derive(Debug, Clone, Copy, Deserialize, JsonSchema, Default)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum Button {
|
||||
#[default]
|
||||
Left,
|
||||
Middle,
|
||||
Right,
|
||||
}
|
||||
impl From<Button> for MouseButton {
|
||||
fn from(b: Button) -> Self {
|
||||
match b {
|
||||
Button::Left => Self::Left,
|
||||
Button::Middle => Self::Middle,
|
||||
Button::Right => Self::Right,
|
||||
}
|
||||
}
|
||||
}
|
||||
#[derive(Debug, Deserialize, JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Point {
|
||||
pub x: f64,
|
||||
pub y: f64,
|
||||
}
|
||||
#[derive(Debug, Deserialize, JsonSchema)]
|
||||
#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
|
||||
pub enum Action {
|
||||
Click {
|
||||
x: f64,
|
||||
y: f64,
|
||||
#[serde(default)]
|
||||
button: Button,
|
||||
},
|
||||
DoubleClick {
|
||||
x: f64,
|
||||
y: f64,
|
||||
#[serde(default)]
|
||||
button: Button,
|
||||
},
|
||||
Move {
|
||||
x: f64,
|
||||
y: f64,
|
||||
},
|
||||
Drag {
|
||||
path: Vec<Point>,
|
||||
#[serde(default)]
|
||||
button: Button,
|
||||
},
|
||||
Scroll {
|
||||
x: f64,
|
||||
y: f64,
|
||||
vertical_steps: i32,
|
||||
},
|
||||
Type {
|
||||
text: String,
|
||||
},
|
||||
Keypress {
|
||||
/// Modifiers (ctrl, shift, alt, meta) plus one key: a-z, 0-9, enter, esc, tab, space, backspace, delete, arrows, home/end, pageup/pagedown, f1-f12.
|
||||
keys: Vec<String>,
|
||||
#[serde(default = "default_hold")]
|
||||
hold_ms: u64,
|
||||
},
|
||||
Wait {
|
||||
ms: u64,
|
||||
},
|
||||
}
|
||||
fn default_hold() -> u64 {
|
||||
35
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub(super) enum Step {
|
||||
Mouse(MouseEvent),
|
||||
Keyboard(KeyboardEvent),
|
||||
Pause(u64),
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub(super) struct PreparedActions {
|
||||
pub actions: Vec<Vec<Step>>,
|
||||
pub needs_absolute_mouse: bool,
|
||||
}
|
||||
|
||||
fn position(x: f64, y: f64) -> Result<Step, String> {
|
||||
if !x.is_finite() || !y.is_finite() || !(0.0..=1.0).contains(&x) || !(0.0..=1.0).contains(&y) {
|
||||
return Err("Mouse coordinates must be finite numbers in [0,1]".into());
|
||||
}
|
||||
Ok(Step::Mouse(MouseEvent::move_abs(
|
||||
(x * 32767.0).round() as i32,
|
||||
(y * 32767.0).round() as i32,
|
||||
)))
|
||||
}
|
||||
fn key_steps(out: &mut Vec<Step>, key: CanonicalKey, modifiers: KeyboardModifiers, hold: u64) {
|
||||
out.push(Step::Keyboard(KeyboardEvent {
|
||||
event_type: KeyEventType::Down,
|
||||
key,
|
||||
modifiers,
|
||||
}));
|
||||
out.push(Step::Pause(hold));
|
||||
out.push(Step::Keyboard(KeyboardEvent {
|
||||
event_type: KeyEventType::Up,
|
||||
key,
|
||||
modifiers: KeyboardModifiers::default(),
|
||||
}));
|
||||
}
|
||||
fn chord(keys: &[String]) -> Result<(CanonicalKey, KeyboardModifiers), String> {
|
||||
if keys.is_empty() || keys.len() > 5 {
|
||||
return Err("keypress requires modifiers and one ordinary key".into());
|
||||
}
|
||||
let mut mods = KeyboardModifiers::default();
|
||||
let mut key = None;
|
||||
let mut modifier_key = None;
|
||||
for name in keys {
|
||||
let modifier = match name.trim().to_lowercase().as_str() {
|
||||
"ctrl" | "control" | "controlleft" => {
|
||||
mods.left_ctrl = true;
|
||||
Some(CanonicalKey::ControlLeft)
|
||||
}
|
||||
"shift" | "shiftleft" => {
|
||||
mods.left_shift = true;
|
||||
Some(CanonicalKey::ShiftLeft)
|
||||
}
|
||||
"alt" | "altleft" => {
|
||||
mods.left_alt = true;
|
||||
Some(CanonicalKey::AltLeft)
|
||||
}
|
||||
"meta" | "win" | "cmd" | "super" | "metaleft" => {
|
||||
mods.left_meta = true;
|
||||
Some(CanonicalKey::MetaLeft)
|
||||
}
|
||||
"controlright" => {
|
||||
mods.right_ctrl = true;
|
||||
Some(CanonicalKey::ControlRight)
|
||||
}
|
||||
"shiftright" => {
|
||||
mods.right_shift = true;
|
||||
Some(CanonicalKey::ShiftRight)
|
||||
}
|
||||
"altright" | "altgr" => {
|
||||
mods.right_alt = true;
|
||||
Some(CanonicalKey::AltRight)
|
||||
}
|
||||
"metaright" => {
|
||||
mods.right_meta = true;
|
||||
Some(CanonicalKey::MetaRight)
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
if let Some(modifier) = modifier {
|
||||
modifier_key = Some(modifier);
|
||||
continue;
|
||||
}
|
||||
let parsed = key_name_to_canonical(name)
|
||||
.or_else(|| {
|
||||
let mut chars = name.chars();
|
||||
let ch = chars.next()?;
|
||||
if chars.next().is_some() {
|
||||
return None;
|
||||
}
|
||||
let (key, implied) = char_to_key(ch)?;
|
||||
mods.left_shift |= implied.left_shift;
|
||||
Some(key)
|
||||
})
|
||||
.ok_or("Unsupported key name")?;
|
||||
if key.replace(parsed).is_some() {
|
||||
return Err("Only one ordinary key per keypress is supported".into());
|
||||
}
|
||||
}
|
||||
Ok((key.or(modifier_key).ok_or("No key provided")?, mods))
|
||||
}
|
||||
|
||||
pub(super) fn prepare(actions: Vec<Action>) -> Result<PreparedActions, String> {
|
||||
if actions.is_empty() || actions.len() > 32 {
|
||||
return Err("Provide between 1 and 32 actions".into());
|
||||
}
|
||||
let mut prepared = PreparedActions {
|
||||
actions: Vec::new(),
|
||||
needs_absolute_mouse: false,
|
||||
};
|
||||
let mut characters = 0;
|
||||
let mut duration = 0u64;
|
||||
for action in actions {
|
||||
let mut steps = Vec::new();
|
||||
match action {
|
||||
Action::Click { x, y, button } | Action::DoubleClick { x, y, button } => {
|
||||
let count = if matches!(action, Action::DoubleClick { .. }) {
|
||||
2
|
||||
} else {
|
||||
1
|
||||
};
|
||||
steps.push(position(x, y)?);
|
||||
for _ in 0..count {
|
||||
steps.push(Step::Mouse(MouseEvent::button_down(button.into())));
|
||||
steps.push(Step::Pause(35));
|
||||
steps.push(Step::Mouse(MouseEvent::button_up(button.into())));
|
||||
steps.push(Step::Pause(35));
|
||||
}
|
||||
prepared.needs_absolute_mouse = true;
|
||||
}
|
||||
Action::Move { x, y } => {
|
||||
steps.push(position(x, y)?);
|
||||
prepared.needs_absolute_mouse = true;
|
||||
}
|
||||
Action::Drag { path, button } => {
|
||||
if path.len() < 2 || path.len() > 128 {
|
||||
return Err("Drag path needs 2 to 128 points".into());
|
||||
}
|
||||
for (i, p) in path.into_iter().enumerate() {
|
||||
steps.push(position(p.x, p.y)?);
|
||||
if i == 0 {
|
||||
steps.push(Step::Mouse(MouseEvent::button_down(button.into())));
|
||||
}
|
||||
steps.push(Step::Pause(35));
|
||||
}
|
||||
steps.push(Step::Mouse(MouseEvent::button_up(button.into())));
|
||||
prepared.needs_absolute_mouse = true;
|
||||
}
|
||||
Action::Scroll {
|
||||
x,
|
||||
y,
|
||||
vertical_steps,
|
||||
} => {
|
||||
if !(-100..=100).contains(&vertical_steps) {
|
||||
return Err("vertical_steps must be between -100 and 100".into());
|
||||
}
|
||||
steps.push(position(x, y)?);
|
||||
for _ in 0..vertical_steps.unsigned_abs() {
|
||||
// The device wheel convention is positive up; public MCP is positive down.
|
||||
steps.push(Step::Mouse(MouseEvent::scroll(
|
||||
-vertical_steps.signum() as i8
|
||||
)));
|
||||
steps.push(Step::Pause(35));
|
||||
}
|
||||
prepared.needs_absolute_mouse = true;
|
||||
}
|
||||
Action::Type { text } => {
|
||||
characters += text.chars().count();
|
||||
if characters > 256 {
|
||||
return Err("At most 256 characters per batch".into());
|
||||
}
|
||||
for ch in text.chars() {
|
||||
let (key, mods) = char_to_key(ch).ok_or(
|
||||
"Text must use the US keyboard ASCII mapping (including newline and tab)",
|
||||
)?;
|
||||
key_steps(&mut steps, key, mods, 35);
|
||||
}
|
||||
}
|
||||
Action::Keypress { keys, hold_ms } => {
|
||||
if hold_ms > 5000 {
|
||||
return Err("hold_ms must not exceed 5000".into());
|
||||
}
|
||||
let (key, mods) = chord(&keys)?;
|
||||
key_steps(&mut steps, key, mods, hold_ms);
|
||||
}
|
||||
Action::Wait { ms } => {
|
||||
if ms > 5000 {
|
||||
return Err("wait ms must not exceed 5000".into());
|
||||
}
|
||||
steps.push(Step::Pause(ms));
|
||||
}
|
||||
}
|
||||
steps.push(Step::Pause(120));
|
||||
duration += steps
|
||||
.iter()
|
||||
.map(|s| match s {
|
||||
Step::Pause(ms) => *ms,
|
||||
_ => 1,
|
||||
})
|
||||
.sum::<u64>();
|
||||
if duration > 25_000 {
|
||||
return Err("Batch duration exceeds 25 seconds before hardware delays".into());
|
||||
}
|
||||
prepared.actions.push(steps);
|
||||
}
|
||||
Ok(prepared)
|
||||
}
|
||||
|
||||
pub(super) async fn execute(hid: &HidController, steps: Vec<Step>) -> crate::Result<()> {
|
||||
for step in steps {
|
||||
match step {
|
||||
Step::Mouse(event) => hid.send_mouse_confirmed(event).await?,
|
||||
Step::Keyboard(event) => hid.send_keyboard_confirmed(event).await?,
|
||||
Step::Pause(ms) => tokio::time::sleep(Duration::from_millis(ms)).await,
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn validates_entire_batch_before_any_hid_input() {
|
||||
assert!(prepare(vec![
|
||||
Action::Type {
|
||||
text: "valid".into()
|
||||
},
|
||||
Action::Type {
|
||||
text: "中文".into()
|
||||
}
|
||||
])
|
||||
.is_err());
|
||||
assert!(prepare(vec![Action::Move {
|
||||
x: f64::NAN,
|
||||
y: 0.5
|
||||
}])
|
||||
.is_err());
|
||||
assert!(prepare(vec![Action::Move { x: 1.01, y: 0.5 }]).is_err());
|
||||
assert!(chord(&["ctrl".into(), "unknown".into()]).is_err());
|
||||
assert!(chord(&["a".into(), "b".into()]).is_err());
|
||||
assert!(chord(&["ctrl".into(), "alt".into(), "delete".into()]).is_ok());
|
||||
}
|
||||
#[test]
|
||||
fn uses_full_hid_range_and_bounded_waits() {
|
||||
let actions = prepare(vec![Action::Move { x: 1.0, y: 0.0 }]).unwrap();
|
||||
match &actions.actions[0][0] {
|
||||
Step::Mouse(event) => assert_eq!((event.x, event.y), (32767, 0)),
|
||||
_ => panic!(),
|
||||
}
|
||||
assert!(prepare((0..6).map(|_| Action::Wait { ms: 5000 }).collect()).is_err());
|
||||
assert!(prepare(vec![Action::Type {
|
||||
text: "x".repeat(257)
|
||||
}])
|
||||
.is_err());
|
||||
}
|
||||
}
|
||||
430
src/mcp/executor.rs
Normal file
430
src/mcp/executor.rs
Normal file
@@ -0,0 +1,430 @@
|
||||
use super::actions::{self, PreparedActions};
|
||||
use super::hardware::{HardwareCommand, McpHardware};
|
||||
use crate::{
|
||||
config::ConfigStore,
|
||||
hid::HidController,
|
||||
video::{snapshot::Snapshot, VideoStreamManager},
|
||||
};
|
||||
use base64::{engine::general_purpose::STANDARD, Engine as _};
|
||||
use parking_lot::Mutex;
|
||||
use rmcp::model::{CallToolResult, ContentBlock};
|
||||
use serde_json::{json, Value};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::sync::{mpsc, oneshot};
|
||||
|
||||
pub(super) enum Command {
|
||||
Status,
|
||||
Screenshot(Option<u8>),
|
||||
Actions(PreparedActions, bool),
|
||||
Hardware(HardwareCommand),
|
||||
}
|
||||
struct Job {
|
||||
command: Command,
|
||||
enqueued: Instant,
|
||||
reply: oneshot::Sender<CallToolResult>,
|
||||
}
|
||||
|
||||
struct Bucket {
|
||||
tokens: f64,
|
||||
updated: Instant,
|
||||
rate: u32,
|
||||
}
|
||||
impl Bucket {
|
||||
fn take(&mut self, rate: u32, now: Instant) -> bool {
|
||||
let rate = rate.clamp(1, 100);
|
||||
self.tokens = (self.tokens
|
||||
+ now.duration_since(self.updated).as_secs_f64() * self.rate as f64)
|
||||
.min(rate as f64);
|
||||
self.updated = now;
|
||||
self.rate = rate;
|
||||
if self.tokens < 1.0 {
|
||||
false
|
||||
} else {
|
||||
self.tokens -= 1.0;
|
||||
true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct McpService {
|
||||
config: ConfigStore,
|
||||
tx: mpsc::Sender<Job>,
|
||||
bucket: Mutex<Bucket>,
|
||||
}
|
||||
impl McpService {
|
||||
pub(crate) fn new(
|
||||
config: ConfigStore,
|
||||
hid: Arc<HidController>,
|
||||
video: Arc<VideoStreamManager>,
|
||||
hardware: McpHardware,
|
||||
) -> Arc<Self> {
|
||||
let (tx, mut rx) = mpsc::channel::<Job>(8);
|
||||
let rate = config.get().mcp.requests_per_second.clamp(1, 100);
|
||||
let service = Arc::new(Self {
|
||||
config,
|
||||
tx,
|
||||
bucket: Mutex::new(Bucket {
|
||||
tokens: rate as f64,
|
||||
updated: Instant::now(),
|
||||
rate,
|
||||
}),
|
||||
});
|
||||
tokio::spawn(async move {
|
||||
while let Some(job) = rx.recv().await {
|
||||
// Accepted jobs are independent of the caller's transport lifetime.
|
||||
let result = if job.enqueued.elapsed() > Duration::from_secs(30) {
|
||||
failure(
|
||||
"queue_timeout",
|
||||
"Request expired while waiting; no action executed",
|
||||
0,
|
||||
)
|
||||
} else {
|
||||
match job.command {
|
||||
Command::Status => {
|
||||
let h = hid.snapshot().await;
|
||||
let v = video.stats().await;
|
||||
let devices = hardware.status().await;
|
||||
output(
|
||||
json!({ "version": env!("CARGO_PKG_VERSION"), "video": {"state": v.state, "resolution": v.resolution, "format": v.format}, "hid": {"initialized": h.initialized, "online": h.online, "backend": h.backend, "supports_absolute_mouse": h.supports_absolute_mouse}, "atx": devices["atx"], "media": devices["media"], "queue_length": rx.len() }),
|
||||
false,
|
||||
)
|
||||
}
|
||||
Command::Screenshot(quality) => match video.capture_snapshot(quality).await
|
||||
{
|
||||
Ok(snapshot) => {
|
||||
with_snapshot(output(json!({"success": true}), false), snapshot)
|
||||
}
|
||||
Err(error) => failure("screenshot_failed", &error.to_string(), 0),
|
||||
},
|
||||
Command::Actions(prepared, screenshot) => {
|
||||
run_actions(&hid, &video, prepared, screenshot).await
|
||||
}
|
||||
Command::Hardware(command) => hardware.execute(command).await,
|
||||
}
|
||||
};
|
||||
let _ = job.reply.send(result);
|
||||
}
|
||||
});
|
||||
service
|
||||
}
|
||||
pub(super) async fn call(&self, command: Command) -> CallToolResult {
|
||||
let (reply, rx) = oneshot::channel();
|
||||
{
|
||||
// One lock defines admission order and a single service-wide quota.
|
||||
let mut bucket = self.bucket.lock();
|
||||
if !bucket.take(self.config.get().mcp.requests_per_second, Instant::now()) {
|
||||
return failure(
|
||||
"rate_limited",
|
||||
"Global MCP rate limit exceeded; retry after 1 second",
|
||||
0,
|
||||
);
|
||||
}
|
||||
if self
|
||||
.tx
|
||||
.try_send(Job {
|
||||
command,
|
||||
enqueued: Instant::now(),
|
||||
reply,
|
||||
})
|
||||
.is_err()
|
||||
{
|
||||
return failure("busy", "MCP queue is full; retry later", 0);
|
||||
}
|
||||
}
|
||||
rx.await
|
||||
.unwrap_or_else(|_| failure("unavailable", "MCP worker unavailable", 0))
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_actions(
|
||||
hid: &HidController,
|
||||
video: &VideoStreamManager,
|
||||
prepared: PreparedActions,
|
||||
screenshot: bool,
|
||||
) -> CallToolResult {
|
||||
let state = hid.snapshot().await;
|
||||
if !state.initialized || (prepared.needs_absolute_mouse && !state.supports_absolute_mouse) {
|
||||
return failure("hid_unavailable", "Actions require an initialized HID backend; pointer actions require absolute mouse support", 0);
|
||||
}
|
||||
let mut completed = 0;
|
||||
let result = tokio::time::timeout(Duration::from_secs(30), async {
|
||||
for action in prepared.actions {
|
||||
actions::execute(hid, action).await?;
|
||||
completed += 1;
|
||||
}
|
||||
Ok::<(), crate::AppError>(())
|
||||
})
|
||||
.await;
|
||||
// Successful actions already release their keys/buttons. A backend reset
|
||||
// also moves an absolute pointer to (0,0), so reserve it for failed or
|
||||
// timed-out batches that may have left input held.
|
||||
let cleanup = if matches!(&result, Ok(Ok(()))) {
|
||||
Ok(())
|
||||
} else {
|
||||
hid.reset_confirmed().await
|
||||
};
|
||||
let mut response = match result {
|
||||
Ok(Ok(())) => output(
|
||||
json!({"success": true, "completed_actions": completed}),
|
||||
false,
|
||||
),
|
||||
Ok(Err(error)) => failure("execution_failed", &error.to_string(), completed),
|
||||
Err(_) => failure(
|
||||
"execution_timeout",
|
||||
"Execution time limit exceeded; last action may be partially executed",
|
||||
completed,
|
||||
),
|
||||
};
|
||||
if let Err(error) = cleanup {
|
||||
response.is_error = Some(true);
|
||||
if let Some(Value::Object(value)) = response.structured_content.as_mut() {
|
||||
value.insert("success".into(), json!(false));
|
||||
value.insert("cleanup_error".into(), json!(error.to_string()));
|
||||
}
|
||||
refresh_text(&mut response);
|
||||
return response;
|
||||
}
|
||||
if screenshot {
|
||||
match video.capture_snapshot(None).await {
|
||||
Ok(snapshot) => response = with_snapshot(response, snapshot),
|
||||
Err(error) => {
|
||||
if let Some(Value::Object(value)) = response.structured_content.as_mut() {
|
||||
value.insert("screenshot_error".into(), json!(error.to_string()));
|
||||
}
|
||||
refresh_text(&mut response);
|
||||
}
|
||||
}
|
||||
}
|
||||
response
|
||||
}
|
||||
|
||||
pub(super) fn output(value: Value, failed: bool) -> CallToolResult {
|
||||
let mut result = CallToolResult::success(vec![ContentBlock::text(value.to_string())]);
|
||||
result.structured_content = Some(value);
|
||||
result.is_error = Some(failed);
|
||||
result
|
||||
}
|
||||
pub(super) fn failure(code: &str, message: &str, completed: usize) -> CallToolResult {
|
||||
output(
|
||||
json!({"success": false, "error": code, "message": message, "completed_actions": completed, "failed_action_index": completed}),
|
||||
true,
|
||||
)
|
||||
}
|
||||
fn refresh_text(result: &mut CallToolResult) {
|
||||
if let Some(value) = &result.structured_content {
|
||||
result.content[0] = ContentBlock::text(value.to_string());
|
||||
}
|
||||
}
|
||||
fn with_snapshot(mut result: CallToolResult, snapshot: Snapshot) -> CallToolResult {
|
||||
if let Some(Value::Object(value)) = result.structured_content.as_mut() {
|
||||
value.insert(
|
||||
"screenshot".into(),
|
||||
json!({"width": snapshot.width, "height": snapshot.height, "age_ms": snapshot.age_ms}),
|
||||
);
|
||||
}
|
||||
refresh_text(&mut result);
|
||||
result.content.push(ContentBlock::image(
|
||||
STANDARD.encode(snapshot.jpeg),
|
||||
"image/jpeg",
|
||||
));
|
||||
result
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn global_bucket_has_bounded_burst_and_recovers() {
|
||||
let now = Instant::now();
|
||||
let mut bucket = Bucket {
|
||||
tokens: 2.0,
|
||||
updated: now,
|
||||
rate: 2,
|
||||
};
|
||||
assert!(bucket.take(2, now));
|
||||
assert!(bucket.take(2, now));
|
||||
assert!(!bucket.take(2, now));
|
||||
assert!(bucket.take(2, now + Duration::from_millis(500)));
|
||||
assert!(!bucket.take(2, now + Duration::from_millis(500)));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod integration_tests {
|
||||
use super::*;
|
||||
use crate::hid::{
|
||||
backend::{HidBackend, HidBackendRuntimeSnapshot},
|
||||
CanonicalKey, KeyEventType, KeyboardEvent, MouseEvent,
|
||||
};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use tokio::sync::{watch, Notify};
|
||||
struct Backend {
|
||||
events: Mutex<Vec<String>>,
|
||||
entered: Notify,
|
||||
proceed: Notify,
|
||||
pressed: AtomicBool,
|
||||
runtime: watch::Sender<()>,
|
||||
}
|
||||
#[async_trait::async_trait]
|
||||
impl HidBackend for Backend {
|
||||
async fn init(&self) -> crate::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
async fn send_keyboard(&self, event: KeyboardEvent) -> crate::Result<()> {
|
||||
self.events
|
||||
.lock()
|
||||
.push(format!("{:?}:{:?}", event.key, event.event_type));
|
||||
self.pressed.store(
|
||||
matches!(event.event_type, KeyEventType::Down),
|
||||
Ordering::SeqCst,
|
||||
);
|
||||
if event.key == CanonicalKey::KeyA && matches!(event.event_type, KeyEventType::Down) {
|
||||
self.entered.notify_one();
|
||||
self.proceed.notified().await;
|
||||
}
|
||||
if event.key == CanonicalKey::KeyC && matches!(event.event_type, KeyEventType::Down) {
|
||||
return Err(crate::AppError::ServiceUnavailable(
|
||||
"Simulated hardware failure".into(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
async fn send_mouse(&self, _: MouseEvent) -> crate::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
async fn reset(&self) -> crate::Result<()> {
|
||||
self.events.lock().push("reset".into());
|
||||
self.pressed.store(false, Ordering::SeqCst);
|
||||
Ok(())
|
||||
}
|
||||
async fn shutdown(&self) -> crate::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
fn runtime_snapshot(&self) -> HidBackendRuntimeSnapshot {
|
||||
HidBackendRuntimeSnapshot {
|
||||
initialized: true,
|
||||
online: true,
|
||||
supports_absolute_mouse: true,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
fn subscribe_runtime(&self) -> watch::Receiver<()> {
|
||||
self.runtime.subscribe()
|
||||
}
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn fifo_batches_survive_caller_disconnect_and_release_input() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let db = crate::db::DatabasePool::new(&dir.path().join("mcp.db"))
|
||||
.await
|
||||
.unwrap();
|
||||
db.init_schema().await.unwrap();
|
||||
let config = ConfigStore::new(db.clone_pool());
|
||||
config.load().await.unwrap();
|
||||
config
|
||||
.update(|c| c.mcp.requests_per_second = 100)
|
||||
.await
|
||||
.unwrap();
|
||||
let backend = Arc::new(Backend {
|
||||
events: Mutex::new(Vec::new()),
|
||||
entered: Notify::new(),
|
||||
proceed: Notify::new(),
|
||||
pressed: AtomicBool::new(false),
|
||||
runtime: watch::channel(()).0,
|
||||
});
|
||||
let hid = crate::hid::test_controller(backend.clone()).await;
|
||||
let video = VideoStreamManager::with_webrtc_streamer(
|
||||
crate::video::streamer::Streamer::new(),
|
||||
crate::webrtc::WebRtcStreamer::with_config(Default::default()),
|
||||
);
|
||||
let hardware = McpHardware::for_test(config.clone(), &db);
|
||||
let service = McpService::new(config, hid.clone(), video.clone(), hardware);
|
||||
let call = |text: &str| {
|
||||
Command::Actions(
|
||||
actions::prepare(vec![actions::Action::Type { text: text.into() }]).unwrap(),
|
||||
false,
|
||||
)
|
||||
};
|
||||
let s = service.clone();
|
||||
let first = call("A");
|
||||
let caller = tokio::spawn(async move { s.call(first).await });
|
||||
tokio::time::timeout(Duration::from_secs(2), backend.entered.notified())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(backend.pressed.load(Ordering::SeqCst));
|
||||
caller.abort();
|
||||
let s = service.clone();
|
||||
let second = call("B");
|
||||
let second = tokio::spawn(async move { s.call(second).await });
|
||||
// Wait for admission, then fill the remaining bounded queue with status reads.
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
while service.tx.capacity() != 7 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let mut pending = Vec::new();
|
||||
for _ in 0..7 {
|
||||
let s = service.clone();
|
||||
pending.push(tokio::spawn(async move { s.call(Command::Status).await }));
|
||||
}
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
while service.tx.capacity() != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let busy = service.call(Command::Status).await;
|
||||
assert_eq!(busy.structured_content.unwrap()["error"], "busy");
|
||||
backend.proceed.notify_one();
|
||||
let result = tokio::time::timeout(Duration::from_secs(3), second)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(result.structured_content.unwrap()["completed_actions"], 1);
|
||||
for job in pending {
|
||||
assert_eq!(job.await.unwrap().is_error, Some(false));
|
||||
}
|
||||
assert!(!backend.pressed.load(Ordering::SeqCst));
|
||||
assert_eq!(
|
||||
*backend.events.lock(),
|
||||
["KeyA:Down", "KeyA:Up", "KeyB:Down", "KeyB:Up"]
|
||||
);
|
||||
let partial = service
|
||||
.call(Command::Actions(
|
||||
actions::prepare(vec![
|
||||
actions::Action::Type { text: "B".into() },
|
||||
actions::Action::Type { text: "C".into() },
|
||||
])
|
||||
.unwrap(),
|
||||
false,
|
||||
))
|
||||
.await;
|
||||
assert_eq!(partial.is_error, Some(true));
|
||||
let partial = partial.structured_content.unwrap();
|
||||
assert_eq!(partial["completed_actions"], 1);
|
||||
assert_eq!(partial["failed_action_index"], 1);
|
||||
assert!(!backend.pressed.load(Ordering::SeqCst));
|
||||
assert_eq!(backend.events.lock().last().unwrap(), "reset");
|
||||
// Hold a snapshot request so capture deterministically fails as busy,
|
||||
// independently of video hardware present on the build machine.
|
||||
let pending_snapshot = video.streamer().snapshot_slot.request().unwrap();
|
||||
let screenshot_failure = service
|
||||
.call(Command::Actions(
|
||||
actions::prepare(vec![actions::Action::Type { text: "B".into() }]).unwrap(),
|
||||
true,
|
||||
))
|
||||
.await;
|
||||
assert_eq!(screenshot_failure.is_error, Some(false));
|
||||
let value = screenshot_failure.structured_content.unwrap();
|
||||
assert_eq!(value["success"], true);
|
||||
assert_eq!(value["completed_actions"], 1);
|
||||
assert!(value.get("screenshot_error").is_some());
|
||||
assert_eq!(backend.events.lock().last().unwrap(), "KeyB:Up");
|
||||
drop(pending_snapshot);
|
||||
hid.shutdown().await.unwrap();
|
||||
}
|
||||
}
|
||||
328
src/mcp/hardware.rs
Normal file
328
src/mcp/hardware.rs
Normal file
@@ -0,0 +1,328 @@
|
||||
//! Device operations share the MCP worker and existing controller locks.
|
||||
use super::executor::output;
|
||||
use crate::{
|
||||
atx::{AtxController, AtxState},
|
||||
config::ConfigStore,
|
||||
db::WolHistoryStore,
|
||||
error::{AppError, Result},
|
||||
};
|
||||
use rmcp::{
|
||||
model::CallToolResult,
|
||||
schemars::{self, JsonSchema},
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{json, Value};
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
#[derive(Debug, Clone, Copy, Deserialize, Serialize, JsonSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub(super) enum PowerAction {
|
||||
PressShort,
|
||||
PressLong,
|
||||
Reset,
|
||||
}
|
||||
|
||||
pub(super) enum HardwareCommand {
|
||||
Power(PowerAction),
|
||||
Wake(String),
|
||||
ListMedia,
|
||||
MountMedia {
|
||||
image_id: String,
|
||||
cdrom: bool,
|
||||
read_only: bool,
|
||||
},
|
||||
UnmountMedia(String),
|
||||
}
|
||||
|
||||
pub(crate) struct McpHardware {
|
||||
pub config: ConfigStore,
|
||||
pub atx: Arc<RwLock<Option<AtxController>>>,
|
||||
pub wol_history: WolHistoryStore,
|
||||
#[cfg(unix)]
|
||||
pub msd: Arc<RwLock<Option<crate::msd::MsdController>>>,
|
||||
#[cfg(unix)]
|
||||
pub otg_lock: Arc<tokio::sync::Mutex<()>>,
|
||||
}
|
||||
|
||||
fn atx_json(state: AtxState) -> Value {
|
||||
json!({
|
||||
"available": state.available, "backend": state.driver,
|
||||
"power_configured": state.power_configured, "reset_configured": state.reset_configured,
|
||||
"power_status": state.power_status, "led_supported": state.led_supported,
|
||||
"hdd_status": state.hdd_status, "hdd_supported": state.hdd_supported,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn valid_image_id(id: &str) -> bool {
|
||||
!id.is_empty() && id.len() <= 16 && id.bytes().all(|b| b.is_ascii_hexdigit())
|
||||
}
|
||||
|
||||
impl McpHardware {
|
||||
pub(super) async fn status(&self) -> Value {
|
||||
let atx = self.atx.read().await;
|
||||
let atx = match atx.as_ref() {
|
||||
Some(controller) => controller.state().await,
|
||||
None => AtxState::default(),
|
||||
};
|
||||
json!({"atx": atx_json(atx), "media": self.media_status().await})
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
async fn media_status(&self) -> Value {
|
||||
let guard = self.msd.read().await;
|
||||
let state = match guard.as_ref() {
|
||||
Some(controller) => controller.state().await,
|
||||
None => Default::default(),
|
||||
};
|
||||
json!({"supported": true, "available": state.available,
|
||||
"disk_mode": state.disk_mode, "slot_capacity": state.disk_mode.capacity(),
|
||||
"mounted_count": state.mounted_media.len(), "usb_reenumerating": state.usb_reenumerating})
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
async fn media_status(&self) -> Value {
|
||||
json!({"supported": false, "available": false})
|
||||
}
|
||||
|
||||
pub(super) async fn execute(&self, command: HardwareCommand) -> CallToolResult {
|
||||
match self.run(command).await {
|
||||
Ok(value) => output(value, false),
|
||||
Err(error) => {
|
||||
let code = match &error {
|
||||
AppError::Msd(msd) => msd.code().as_str(),
|
||||
AppError::ServiceUnavailable(_) => "unavailable",
|
||||
AppError::BadRequest(_) => "invalid_arguments",
|
||||
AppError::Conflict(_) => "busy",
|
||||
_ => "hardware_error",
|
||||
};
|
||||
output(
|
||||
json!({"success": false, "error": code, "message": error.to_string()}),
|
||||
true,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn run(&self, command: HardwareCommand) -> Result<Value> {
|
||||
match command {
|
||||
HardwareCommand::Power(action) => {
|
||||
let guard = self.atx.read().await;
|
||||
let controller = guard.as_ref().ok_or_else(|| {
|
||||
AppError::ServiceUnavailable("ATX controller is not configured".into())
|
||||
})?;
|
||||
let state = controller.state().await;
|
||||
let configured = match action {
|
||||
PowerAction::Reset => state.reset_configured,
|
||||
_ => state.power_configured,
|
||||
};
|
||||
if !state.available || !configured {
|
||||
return Err(AppError::ServiceUnavailable(
|
||||
"Requested ATX button is not configured".into(),
|
||||
));
|
||||
}
|
||||
match action {
|
||||
PowerAction::PressShort => controller.power_short().await?,
|
||||
PowerAction::PressLong => controller.power_long().await?,
|
||||
PowerAction::Reset => controller.reset().await?,
|
||||
}
|
||||
Ok(
|
||||
json!({"success": true, "action_sent": true, "action": action,
|
||||
"atx": atx_json(controller.state().await)}),
|
||||
)
|
||||
}
|
||||
HardwareCommand::Wake(mac_address) => {
|
||||
let interface = self.config.get().atx.wol_interface.clone();
|
||||
let target = mac_address.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
crate::atx::send_wol(
|
||||
&target,
|
||||
(!interface.is_empty()).then_some(interface.as_str()),
|
||||
)
|
||||
})
|
||||
.await
|
||||
.map_err(|e| AppError::Internal(format!("WOL worker failed: {e}")))??;
|
||||
let mut result =
|
||||
json!({"success": true, "packet_sent": true, "mac_address": mac_address});
|
||||
// Sending succeeded even if recording history fails. Never invite a write retry.
|
||||
if let Err(error) = self.wol_history.record(&mac_address).await {
|
||||
tracing::warn!(%error, "Failed to persist MCP WOL history");
|
||||
result["history_saved"] = json!(false);
|
||||
} else {
|
||||
result["history_saved"] = json!(true);
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
HardwareCommand::ListMedia => self.list_media().await,
|
||||
command => self.change_media(command).await,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
async fn list_media(&self) -> Result<Value> {
|
||||
let guard = self.msd.read().await;
|
||||
let state = match guard.as_ref() {
|
||||
Some(controller) => controller.state().await,
|
||||
None => Default::default(),
|
||||
};
|
||||
let path = self.config.get().msd.images_dir();
|
||||
let images =
|
||||
tokio::task::spawn_blocking(move || crate::msd::ImageManager::new(path).list())
|
||||
.await
|
||||
.map_err(|e| AppError::Internal(format!("Media listing failed: {e}")))??;
|
||||
Ok(json!({"success": true, "available": state.available,
|
||||
"disk_mode": state.disk_mode, "slot_capacity": state.disk_mode.capacity(),
|
||||
"mounted_count": state.mounted_media.len(), "mounted_media": state.mounted_media,
|
||||
"usb_reenumerating": state.usb_reenumerating, "images": images}))
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
async fn change_media(&self, command: HardwareCommand) -> Result<Value> {
|
||||
use crate::error::MsdErrorCode;
|
||||
// Same lock as USB settings and disk-mode changes, followed by the controller lock.
|
||||
let _otg = self
|
||||
.otg_lock
|
||||
.try_lock()
|
||||
.map_err(|_| AppError::from(MsdErrorCode::MsdOperationInProgress))?;
|
||||
let guard = self.msd.write().await;
|
||||
let controller = guard
|
||||
.as_ref()
|
||||
.ok_or_else(|| AppError::from(MsdErrorCode::MsdUnavailable))?;
|
||||
match command {
|
||||
HardwareCommand::MountMedia {
|
||||
image_id,
|
||||
cdrom,
|
||||
read_only,
|
||||
} => {
|
||||
let path = self.config.get().msd.images_dir();
|
||||
let id = image_id.clone();
|
||||
let image = tokio::task::spawn_blocking(move || {
|
||||
crate::msd::ImageManager::new(path).get(&id)
|
||||
})
|
||||
.await
|
||||
.map_err(|e| AppError::Internal(format!("Media lookup failed: {e}")))??;
|
||||
controller.mount_image(&image, cdrom, read_only).await?;
|
||||
Ok(
|
||||
json!({"success": true, "image_id": image_id, "mounted": true,
|
||||
"cdrom": cdrom, "read_only": cdrom || read_only}),
|
||||
)
|
||||
}
|
||||
HardwareCommand::UnmountMedia(image_id) => {
|
||||
// Idempotent: an absent image is already unmounted. Never eject other media.
|
||||
controller.unmount_image(&image_id).await?;
|
||||
Ok(json!({"success": true, "image_id": image_id, "mounted": false}))
|
||||
}
|
||||
_ => unreachable!("only media mutations reach change_media"),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
async fn list_media(&self) -> Result<Value> {
|
||||
Err(AppError::ServiceUnavailable(
|
||||
"Virtual media is unsupported on this platform".into(),
|
||||
))
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
async fn change_media(&self, _: HardwareCommand) -> Result<Value> {
|
||||
self.list_media().await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn for_test(config: ConfigStore, db: &crate::db::DatabasePool) -> Self {
|
||||
Self {
|
||||
config,
|
||||
atx: Arc::new(RwLock::new(None)),
|
||||
wol_history: db.wol_history(),
|
||||
#[cfg(unix)]
|
||||
msd: Arc::new(RwLock::new(None)),
|
||||
#[cfg(unix)]
|
||||
otg_lock: Arc::new(tokio::sync::Mutex::new(())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn unavailable_power_is_unknown_and_media_is_serialized_with_usb_settings() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let db = crate::db::DatabasePool::new(&dir.path().join("hardware.db"))
|
||||
.await
|
||||
.unwrap();
|
||||
db.init_schema().await.unwrap();
|
||||
let config = ConfigStore::new(db.clone_pool());
|
||||
config.load().await.unwrap();
|
||||
config
|
||||
.update(|c| c.msd.msd_dir = dir.path().join("msd").to_string_lossy().into())
|
||||
.await
|
||||
.unwrap();
|
||||
let hardware = McpHardware::for_test(config, &db);
|
||||
let status = hardware.status().await;
|
||||
assert_eq!(status["atx"]["available"], false);
|
||||
assert_eq!(status["atx"]["power_status"], "unknown");
|
||||
let power = hardware
|
||||
.execute(HardwareCommand::Power(PowerAction::PressShort))
|
||||
.await;
|
||||
assert_eq!(power.is_error, Some(true));
|
||||
assert_eq!(power.structured_content.unwrap()["error"], "unavailable");
|
||||
#[cfg(unix)]
|
||||
{
|
||||
let lock = hardware.otg_lock.lock().await;
|
||||
let mount = hardware
|
||||
.execute(HardwareCommand::MountMedia {
|
||||
image_id: "abc".into(),
|
||||
cdrom: true,
|
||||
read_only: true,
|
||||
})
|
||||
.await;
|
||||
assert_eq!(
|
||||
mount.structured_content.unwrap()["error"],
|
||||
"MSD_OPERATION_IN_PROGRESS"
|
||||
);
|
||||
drop(lock);
|
||||
let missing = hardware
|
||||
.execute(HardwareCommand::UnmountMedia("abc".into()))
|
||||
.await;
|
||||
assert_eq!(
|
||||
missing.structured_content.unwrap()["error"],
|
||||
"MSD_UNAVAILABLE"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn media_listing_returns_ids_and_capacity_without_host_paths() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let db = crate::db::DatabasePool::new(&dir.path().join("hardware.db"))
|
||||
.await
|
||||
.unwrap();
|
||||
db.init_schema().await.unwrap();
|
||||
let config = ConfigStore::new(db.clone_pool());
|
||||
config.load().await.unwrap();
|
||||
config
|
||||
.update(|c| c.msd.msd_dir = dir.path().join("msd").to_string_lossy().into())
|
||||
.await
|
||||
.unwrap();
|
||||
let images = config.get().msd.images_dir();
|
||||
std::fs::create_dir_all(&images).unwrap();
|
||||
std::fs::write(images.join("mcp-test.iso"), [0u8; 2048]).unwrap();
|
||||
let hardware = McpHardware::for_test(config, &db);
|
||||
let result = hardware.execute(HardwareCommand::ListMedia).await;
|
||||
assert_eq!(result.is_error, Some(false));
|
||||
let value = result.structured_content.unwrap();
|
||||
assert_eq!(value["slot_capacity"], 1);
|
||||
assert_eq!(value["mounted_count"], 0);
|
||||
let image = &value["images"][0];
|
||||
assert_eq!(image["name"], "mcp-test.iso");
|
||||
assert_eq!(image["size"], 2048);
|
||||
assert!(valid_image_id(image["id"].as_str().unwrap()));
|
||||
assert!(image.get("path").is_none());
|
||||
assert!(!value
|
||||
.to_string()
|
||||
.contains(&dir.path().to_string_lossy().to_string()));
|
||||
}
|
||||
}
|
||||
185
src/mcp/keyboard.rs
Normal file
185
src/mcp/keyboard.rs
Normal file
@@ -0,0 +1,185 @@
|
||||
use crate::hid::{CanonicalKey, KeyboardModifiers};
|
||||
|
||||
pub(super) fn char_to_key(ch: char) -> Option<(CanonicalKey, KeyboardModifiers)> {
|
||||
let mut mods = KeyboardModifiers::default();
|
||||
let key = match ch {
|
||||
'a'..='z' => key_name_to_canonical(&ch.to_string())?,
|
||||
'A'..='Z' => {
|
||||
mods.left_shift = true;
|
||||
key_name_to_canonical(&ch.to_ascii_lowercase().to_string())?
|
||||
}
|
||||
'0' => CanonicalKey::Digit0,
|
||||
'1' => CanonicalKey::Digit1,
|
||||
'2' => CanonicalKey::Digit2,
|
||||
'3' => CanonicalKey::Digit3,
|
||||
'4' => CanonicalKey::Digit4,
|
||||
'5' => CanonicalKey::Digit5,
|
||||
'6' => CanonicalKey::Digit6,
|
||||
'7' => CanonicalKey::Digit7,
|
||||
'8' => CanonicalKey::Digit8,
|
||||
'9' => CanonicalKey::Digit9,
|
||||
' ' => CanonicalKey::Space,
|
||||
'\n' => CanonicalKey::Enter,
|
||||
'\t' => CanonicalKey::Tab,
|
||||
'-' => CanonicalKey::Minus,
|
||||
'_' => {
|
||||
mods.left_shift = true;
|
||||
CanonicalKey::Minus
|
||||
}
|
||||
'=' => CanonicalKey::Equal,
|
||||
'+' => {
|
||||
mods.left_shift = true;
|
||||
CanonicalKey::Equal
|
||||
}
|
||||
'.' => CanonicalKey::Period,
|
||||
',' => CanonicalKey::Comma,
|
||||
'/' => CanonicalKey::Slash,
|
||||
'?' => {
|
||||
mods.left_shift = true;
|
||||
CanonicalKey::Slash
|
||||
}
|
||||
';' => CanonicalKey::Semicolon,
|
||||
':' => {
|
||||
mods.left_shift = true;
|
||||
CanonicalKey::Semicolon
|
||||
}
|
||||
'\'' => CanonicalKey::Quote,
|
||||
'"' => {
|
||||
mods.left_shift = true;
|
||||
CanonicalKey::Quote
|
||||
}
|
||||
'[' => CanonicalKey::BracketLeft,
|
||||
'{' => {
|
||||
mods.left_shift = true;
|
||||
CanonicalKey::BracketLeft
|
||||
}
|
||||
']' => CanonicalKey::BracketRight,
|
||||
'}' => {
|
||||
mods.left_shift = true;
|
||||
CanonicalKey::BracketRight
|
||||
}
|
||||
'\\' => CanonicalKey::Backslash,
|
||||
'|' => {
|
||||
mods.left_shift = true;
|
||||
CanonicalKey::Backslash
|
||||
}
|
||||
'`' => CanonicalKey::Backquote,
|
||||
'~' => {
|
||||
mods.left_shift = true;
|
||||
CanonicalKey::Backquote
|
||||
}
|
||||
'!' => {
|
||||
mods.left_shift = true;
|
||||
CanonicalKey::Digit1
|
||||
}
|
||||
'@' => {
|
||||
mods.left_shift = true;
|
||||
CanonicalKey::Digit2
|
||||
}
|
||||
'#' => {
|
||||
mods.left_shift = true;
|
||||
CanonicalKey::Digit3
|
||||
}
|
||||
'$' => {
|
||||
mods.left_shift = true;
|
||||
CanonicalKey::Digit4
|
||||
}
|
||||
'%' => {
|
||||
mods.left_shift = true;
|
||||
CanonicalKey::Digit5
|
||||
}
|
||||
'^' => {
|
||||
mods.left_shift = true;
|
||||
CanonicalKey::Digit6
|
||||
}
|
||||
'&' => {
|
||||
mods.left_shift = true;
|
||||
CanonicalKey::Digit7
|
||||
}
|
||||
'*' => {
|
||||
mods.left_shift = true;
|
||||
CanonicalKey::Digit8
|
||||
}
|
||||
'(' => {
|
||||
mods.left_shift = true;
|
||||
CanonicalKey::Digit9
|
||||
}
|
||||
')' => {
|
||||
mods.left_shift = true;
|
||||
CanonicalKey::Digit0
|
||||
}
|
||||
_ => return None,
|
||||
};
|
||||
Some((key, mods))
|
||||
}
|
||||
|
||||
pub(super) fn key_name_to_canonical(name: &str) -> Option<CanonicalKey> {
|
||||
match name.trim().to_lowercase().as_str() {
|
||||
"a" => Some(CanonicalKey::KeyA),
|
||||
"b" => Some(CanonicalKey::KeyB),
|
||||
"c" => Some(CanonicalKey::KeyC),
|
||||
"d" => Some(CanonicalKey::KeyD),
|
||||
"e" => Some(CanonicalKey::KeyE),
|
||||
"f" => Some(CanonicalKey::KeyF),
|
||||
"g" => Some(CanonicalKey::KeyG),
|
||||
"h" => Some(CanonicalKey::KeyH),
|
||||
"i" => Some(CanonicalKey::KeyI),
|
||||
"j" => Some(CanonicalKey::KeyJ),
|
||||
"k" => Some(CanonicalKey::KeyK),
|
||||
"l" => Some(CanonicalKey::KeyL),
|
||||
"m" => Some(CanonicalKey::KeyM),
|
||||
"n" => Some(CanonicalKey::KeyN),
|
||||
"o" => Some(CanonicalKey::KeyO),
|
||||
"p" => Some(CanonicalKey::KeyP),
|
||||
"q" => Some(CanonicalKey::KeyQ),
|
||||
"r" => Some(CanonicalKey::KeyR),
|
||||
"s" => Some(CanonicalKey::KeyS),
|
||||
"t" => Some(CanonicalKey::KeyT),
|
||||
"u" => Some(CanonicalKey::KeyU),
|
||||
"v" => Some(CanonicalKey::KeyV),
|
||||
"w" => Some(CanonicalKey::KeyW),
|
||||
"x" => Some(CanonicalKey::KeyX),
|
||||
"y" => Some(CanonicalKey::KeyY),
|
||||
"z" => Some(CanonicalKey::KeyZ),
|
||||
"0" => Some(CanonicalKey::Digit0),
|
||||
"1" => Some(CanonicalKey::Digit1),
|
||||
"2" => Some(CanonicalKey::Digit2),
|
||||
"3" => Some(CanonicalKey::Digit3),
|
||||
"4" => Some(CanonicalKey::Digit4),
|
||||
"5" => Some(CanonicalKey::Digit5),
|
||||
"6" => Some(CanonicalKey::Digit6),
|
||||
"7" => Some(CanonicalKey::Digit7),
|
||||
"8" => Some(CanonicalKey::Digit8),
|
||||
"9" => Some(CanonicalKey::Digit9),
|
||||
"insert" => Some(CanonicalKey::Insert),
|
||||
"capslock" => Some(CanonicalKey::CapsLock),
|
||||
"printscreen" => Some(CanonicalKey::PrintScreen),
|
||||
"enter" | "return" => Some(CanonicalKey::Enter),
|
||||
"escape" | "esc" => Some(CanonicalKey::Escape),
|
||||
"backspace" => Some(CanonicalKey::Backspace),
|
||||
"tab" => Some(CanonicalKey::Tab),
|
||||
"space" => Some(CanonicalKey::Space),
|
||||
"delete" | "del" => Some(CanonicalKey::Delete),
|
||||
"arrowup" | "up" => Some(CanonicalKey::ArrowUp),
|
||||
"arrowdown" | "down" => Some(CanonicalKey::ArrowDown),
|
||||
"arrowleft" | "left" => Some(CanonicalKey::ArrowLeft),
|
||||
"arrowright" | "right" => Some(CanonicalKey::ArrowRight),
|
||||
"home" => Some(CanonicalKey::Home),
|
||||
"end" => Some(CanonicalKey::End),
|
||||
"pageup" => Some(CanonicalKey::PageUp),
|
||||
"pagedown" => Some(CanonicalKey::PageDown),
|
||||
"f1" => Some(CanonicalKey::F1),
|
||||
"f2" => Some(CanonicalKey::F2),
|
||||
"f3" => Some(CanonicalKey::F3),
|
||||
"f4" => Some(CanonicalKey::F4),
|
||||
"f5" => Some(CanonicalKey::F5),
|
||||
"f6" => Some(CanonicalKey::F6),
|
||||
"f7" => Some(CanonicalKey::F7),
|
||||
"f8" => Some(CanonicalKey::F8),
|
||||
"f9" => Some(CanonicalKey::F9),
|
||||
"f10" => Some(CanonicalKey::F10),
|
||||
"f11" => Some(CanonicalKey::F11),
|
||||
"f12" => Some(CanonicalKey::F12),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
395
src/mcp/mod.rs
Normal file
395
src/mcp/mod.rs
Normal file
@@ -0,0 +1,395 @@
|
||||
//! Native MCP service: one FIFO and one shared API key.
|
||||
mod actions;
|
||||
mod executor;
|
||||
mod hardware;
|
||||
mod keyboard;
|
||||
mod tools;
|
||||
pub use executor::McpService;
|
||||
pub(crate) use hardware::McpHardware;
|
||||
|
||||
use crate::config::ConfigStore;
|
||||
use axum::{
|
||||
extract::{Request, State},
|
||||
http::{header, StatusCode, Uri},
|
||||
middleware::{self, Next},
|
||||
response::{IntoResponse, Response},
|
||||
Router,
|
||||
};
|
||||
use rmcp::transport::streamable_http_server::{
|
||||
session::local::LocalSessionManager, StreamableHttpServerConfig, StreamableHttpService,
|
||||
};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::sync::Arc;
|
||||
|
||||
pub(crate) fn hash_key(key: &str) -> String {
|
||||
Sha256::digest(key.as_bytes())
|
||||
.iter()
|
||||
.map(|byte| format!("{byte:02x}"))
|
||||
.collect()
|
||||
}
|
||||
pub(crate) fn generate_key() -> String {
|
||||
format!(
|
||||
"okvm_{}{}",
|
||||
uuid::Uuid::new_v4().simple(),
|
||||
uuid::Uuid::new_v4().simple()
|
||||
)
|
||||
}
|
||||
fn key_matches(key: &str, expected: Option<&str>) -> bool {
|
||||
expected.is_some_and(|expected| {
|
||||
expected.len() == 64 && openssl::memcmp::eq(hash_key(key).as_bytes(), expected.as_bytes())
|
||||
})
|
||||
}
|
||||
fn valid_origin(request: &Request) -> bool {
|
||||
let Some(origin) = request.headers().get(header::ORIGIN) else {
|
||||
return true;
|
||||
};
|
||||
let Some(origin) = origin.to_str().ok().and_then(|s| s.parse::<Uri>().ok()) else {
|
||||
return false;
|
||||
};
|
||||
matches!(origin.scheme_str(), Some("http" | "https"))
|
||||
&& origin.authority().is_some_and(|authority| {
|
||||
request
|
||||
.headers()
|
||||
.get(header::HOST)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.is_some_and(|host| host.eq_ignore_ascii_case(authority.as_str()))
|
||||
})
|
||||
&& origin.path_and_query().is_none_or(|p| p.as_str() == "/")
|
||||
}
|
||||
async fn authorize(State(config): State<ConfigStore>, request: Request, next: Next) -> Response {
|
||||
if !valid_origin(&request) {
|
||||
return (StatusCode::FORBIDDEN, "Origin not allowed").into_response();
|
||||
}
|
||||
let config = config.get();
|
||||
if !config.mcp.enabled {
|
||||
return (StatusCode::SERVICE_UNAVAILABLE, "MCP is disabled").into_response();
|
||||
}
|
||||
let supplied = request
|
||||
.headers()
|
||||
.get(header::AUTHORIZATION)
|
||||
.and_then(|h| h.to_str().ok())
|
||||
.and_then(|h| h.strip_prefix("Bearer "));
|
||||
if !supplied.is_some_and(|key| key_matches(key, config.mcp.api_key_hash.as_deref())) {
|
||||
return (
|
||||
StatusCode::UNAUTHORIZED,
|
||||
[(header::WWW_AUTHENTICATE, "Bearer")],
|
||||
"Invalid MCP API key",
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
drop(config);
|
||||
next.run(request).await
|
||||
}
|
||||
pub fn router<S: Clone + Send + Sync + 'static>(
|
||||
service: Arc<McpService>,
|
||||
config: ConfigStore,
|
||||
) -> Router<S> {
|
||||
let http = StreamableHttpService::new(
|
||||
move || Ok(tools::McpTools::new(service.clone())),
|
||||
LocalSessionManager::default().into(),
|
||||
StreamableHttpServerConfig::default()
|
||||
// One-KVM is served on device IPs and user-provided hostnames.
|
||||
// Shared-key authentication and same-origin checks run above the SDK.
|
||||
.disable_allowed_hosts()
|
||||
.with_legacy_session_mode(false)
|
||||
.with_json_response(true),
|
||||
);
|
||||
Router::new()
|
||||
.route_service("/mcp", http)
|
||||
.layer(middleware::from_fn_with_state(config, authorize))
|
||||
.layer(tower_http::limit::RequestBodyLimitLayer::new(64 * 1024))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn shared_key_and_origin_validation() {
|
||||
let key = generate_key();
|
||||
let hash = hash_key(&key);
|
||||
assert!(key_matches(&key, Some(&hash)));
|
||||
assert!(!key_matches("wrong", Some(&hash)));
|
||||
assert!(!key_matches(&key, None));
|
||||
assert!(!key_matches(&key, Some("malformed")));
|
||||
let build = |origin: &str| {
|
||||
Request::builder()
|
||||
.header("host", "kvm.local:8080")
|
||||
.header("origin", origin)
|
||||
.body(axum::body::Body::empty())
|
||||
.unwrap()
|
||||
};
|
||||
assert!(valid_origin(&build("http://kvm.local:8080")));
|
||||
assert!(!valid_origin(&build("https://attacker.example")));
|
||||
assert!(!valid_origin(&build("null")));
|
||||
assert!(!valid_origin(&build("http://kvm.local:8080/extra")));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod http_tests {
|
||||
use super::*;
|
||||
use crate::{
|
||||
db::DatabasePool,
|
||||
hid::{HidBackendType, HidController},
|
||||
video::{streamer::Streamer, VideoStreamManager},
|
||||
webrtc::WebRtcStreamer,
|
||||
};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_auth_discovery_native_schema_and_shared_rate_limit() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let db = DatabasePool::new(&dir.path().join("mcp-http.db"))
|
||||
.await
|
||||
.unwrap();
|
||||
db.init_schema().await.unwrap();
|
||||
let config = ConfigStore::new(db.clone_pool());
|
||||
config.load().await.unwrap();
|
||||
#[cfg(unix)]
|
||||
let hid = Arc::new(HidController::new(HidBackendType::None, None));
|
||||
#[cfg(not(unix))]
|
||||
let hid = Arc::new(HidController::new(HidBackendType::None));
|
||||
let video = VideoStreamManager::with_webrtc_streamer(
|
||||
Streamer::new(),
|
||||
WebRtcStreamer::with_config(Default::default()),
|
||||
);
|
||||
config
|
||||
.update(|c| c.msd.msd_dir = dir.path().join("msd").to_string_lossy().into())
|
||||
.await
|
||||
.unwrap();
|
||||
let hardware = McpHardware::for_test(config.clone(), &db);
|
||||
let atx = hardware.atx.clone();
|
||||
let service = McpService::new(config.clone(), hid, video, hardware);
|
||||
let app = router::<()>(service, config.clone());
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
|
||||
let url = format!("http://{address}/mcp");
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(std::time::Duration::from_secs(5))
|
||||
.build()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
client.post(&url).send().await.unwrap().status(),
|
||||
StatusCode::SERVICE_UNAVAILABLE
|
||||
);
|
||||
let key = generate_key();
|
||||
config
|
||||
.update(|c| {
|
||||
c.mcp.enabled = true;
|
||||
c.mcp.api_key_hash = Some(hash_key(&key));
|
||||
c.mcp.requests_per_second = 100;
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
client.post(&url).send().await.unwrap().status(),
|
||||
StatusCode::UNAUTHORIZED
|
||||
);
|
||||
assert_eq!(
|
||||
client
|
||||
.post(&url)
|
||||
.bearer_auth(&key)
|
||||
.header("Origin", "https://other.invalid")
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
StatusCode::FORBIDDEN
|
||||
);
|
||||
let rpc = |method: &str, params: Value| {
|
||||
client
|
||||
.post(&url)
|
||||
.bearer_auth(&key)
|
||||
.header("Host", "192.168.1.15:8080")
|
||||
.header("Origin", "http://192.168.1.15:8080")
|
||||
.header("Accept", "application/json, text/event-stream")
|
||||
.header("MCP-Protocol-Version", "2025-11-25")
|
||||
.json(&json!({"jsonrpc":"2.0", "id":1, "method":method, "params":params}))
|
||||
};
|
||||
let init: Value = rpc("initialize", json!({"protocolVersion":"2025-11-25", "clientInfo":{"name":"mcp-test","version":"1"}, "capabilities":{}})).send().await.unwrap().json().await.unwrap();
|
||||
assert!(init.get("result").is_some(), "{init}");
|
||||
assert_eq!(init["result"]["serverInfo"]["name"], "one-kvm");
|
||||
assert_eq!(
|
||||
init["result"]["serverInfo"]["version"],
|
||||
env!("CARGO_PKG_VERSION")
|
||||
);
|
||||
let list: Value = rpc("tools/list", json!({}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.json()
|
||||
.await
|
||||
.unwrap();
|
||||
let tools = list["result"]["tools"].as_array().unwrap();
|
||||
let names = tools
|
||||
.iter()
|
||||
.map(|t| t["name"].as_str().unwrap())
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(names.len(), 8);
|
||||
assert!(names.contains(&"one_kvm_screenshot"));
|
||||
assert!(names.contains(&"one_kvm_execute_actions"));
|
||||
assert!(names.contains(&"one_kvm_get_status"));
|
||||
for name in [
|
||||
"one_kvm_power",
|
||||
"one_kvm_wake_on_lan",
|
||||
"one_kvm_list_media",
|
||||
"one_kvm_mount_media",
|
||||
"one_kvm_unmount_media",
|
||||
] {
|
||||
assert!(names.contains(&name));
|
||||
}
|
||||
let screenshot = tools
|
||||
.iter()
|
||||
.find(|t| t["name"] == "one_kvm_screenshot")
|
||||
.unwrap();
|
||||
assert!(screenshot["inputSchema"]["properties"]
|
||||
.get("max_width")
|
||||
.is_none());
|
||||
assert!(screenshot["inputSchema"]["properties"]
|
||||
.get("max_height")
|
||||
.is_none());
|
||||
assert_eq!(screenshot["inputSchema"]["additionalProperties"], false);
|
||||
let status: Value = rpc(
|
||||
"tools/call",
|
||||
json!({"name":"one_kvm_get_status", "arguments":{}}),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.json()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(status["result"]["isError"], false, "{status}");
|
||||
assert!(status["result"]["structuredContent"]["hid"]["initialized"] == false);
|
||||
assert_eq!(
|
||||
status["result"]["structuredContent"]["atx"]["power_status"],
|
||||
"unknown"
|
||||
);
|
||||
let power: Value = rpc(
|
||||
"tools/call",
|
||||
json!({"name":"one_kvm_power", "arguments":{"action":"press_short"}}),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.json()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(power["result"]["isError"], true);
|
||||
let media: Value = rpc(
|
||||
"tools/call",
|
||||
json!({"name":"one_kvm_list_media", "arguments":{}}),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.json()
|
||||
.await
|
||||
.unwrap();
|
||||
#[cfg(unix)]
|
||||
assert_eq!(media["result"]["structuredContent"]["images"], json!([]));
|
||||
#[cfg(not(unix))]
|
||||
assert_eq!(media["result"]["isError"], true);
|
||||
let invalid_id: Value = rpc(
|
||||
"tools/call",
|
||||
json!({"name":"one_kvm_mount_media", "arguments":{"image_id":"/etc/passwd"}}),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.json()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
invalid_id["result"]["structuredContent"]["error"],
|
||||
"invalid_arguments"
|
||||
);
|
||||
// A slow controller must retain its HTTP response while waiting in the FIFO.
|
||||
for _ in 0..6 {
|
||||
let guard = atx.write().await;
|
||||
let request = rpc(
|
||||
"tools/call",
|
||||
json!({"name":"one_kvm_get_status", "arguments":{}}),
|
||||
);
|
||||
let response = tokio::spawn(async move {
|
||||
request.send().await.unwrap().json::<Value>().await.unwrap()
|
||||
});
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
assert!(!response.is_finished());
|
||||
drop(guard);
|
||||
assert_eq!(response.await.unwrap()["result"]["isError"], false);
|
||||
}
|
||||
// Current MCP revision: per-request metadata, without a transport session.
|
||||
let modern: Value = client
|
||||
.post(&url)
|
||||
.bearer_auth(&key)
|
||||
.header("Accept", "application/json, text/event-stream")
|
||||
.header("MCP-Protocol-Version", "2026-07-28")
|
||||
.header("Mcp-Method", "tools/call")
|
||||
.header("Mcp-Name", "one_kvm_get_status")
|
||||
.json(
|
||||
&json!({"jsonrpc":"2.0", "id":2, "method":"tools/call", "params":{
|
||||
"name":"one_kvm_get_status", "arguments":{},
|
||||
"_meta":{
|
||||
"io.modelcontextprotocol/protocolVersion":"2026-07-28",
|
||||
"io.modelcontextprotocol/clientInfo":{"name":"mcp-test", "version":"1"},
|
||||
"io.modelcontextprotocol/clientCapabilities":{}
|
||||
}
|
||||
}}),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.json()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(modern["result"]["isError"], false, "{modern}");
|
||||
let invalid: Value = rpc("tools/call", json!({"name":"one_kvm_execute_actions", "arguments":{"actions":[{"type":"type", "text":"中文"}]}})).send().await.unwrap().json().await.unwrap();
|
||||
assert_eq!(invalid["result"]["isError"], true, "{invalid}");
|
||||
let oversize = rpc("tools/list", json!({"padding":"x".repeat(70_000)}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(oversize.status(), StatusCode::PAYLOAD_TOO_LARGE);
|
||||
// All independent HTTP connections use the same service bucket.
|
||||
config
|
||||
.update(|c| c.mcp.requests_per_second = 1)
|
||||
.await
|
||||
.unwrap();
|
||||
let a: Value = rpc(
|
||||
"tools/call",
|
||||
json!({"name":"one_kvm_get_status", "arguments":{}}),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.json()
|
||||
.await
|
||||
.unwrap();
|
||||
let b: Value = rpc(
|
||||
"tools/call",
|
||||
json!({"name":"one_kvm_list_media", "arguments":{}}),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.json()
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
a["result"]["structuredContent"]["error"] == "rate_limited"
|
||||
|| b["result"]["structuredContent"]["error"] == "rate_limited"
|
||||
);
|
||||
let rotated = generate_key();
|
||||
config
|
||||
.update(|c| c.mcp.api_key_hash = Some(hash_key(&rotated)))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
rpc("tools/list", json!({})).send().await.unwrap().status(),
|
||||
StatusCode::UNAUTHORIZED
|
||||
);
|
||||
server.abort();
|
||||
}
|
||||
}
|
||||
246
src/mcp/tools.rs
Normal file
246
src/mcp/tools.rs
Normal file
@@ -0,0 +1,246 @@
|
||||
use super::{
|
||||
actions::{self, Action},
|
||||
executor::{failure, Command},
|
||||
hardware::{valid_image_id, HardwareCommand, PowerAction},
|
||||
McpService,
|
||||
};
|
||||
use rmcp::{
|
||||
handler::server::{router::tool::ToolRouter, wrapper::Parameters},
|
||||
model::{CallToolResult, ServerCapabilities, ServerConfig},
|
||||
schemars::{self, JsonSchema},
|
||||
tool, tool_handler, tool_router, ServerHandler,
|
||||
};
|
||||
use serde::Deserialize;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[derive(Debug, Deserialize, JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ScreenshotParams {
|
||||
/// Optional JPEG quality, 1 to 100. Native width and height are always preserved.
|
||||
pub quality: Option<u8>,
|
||||
}
|
||||
#[derive(Debug, Deserialize, JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ActionsParams {
|
||||
/// One to 32 actions. Pointer x/y values are normalized to [0,1] over the whole screen.
|
||||
pub actions: Vec<Action>,
|
||||
/// Return a fresh native-resolution JPEG after executing the batch.
|
||||
#[serde(default)]
|
||||
pub return_screenshot: bool,
|
||||
}
|
||||
#[derive(Debug, Deserialize, JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct PowerParams {
|
||||
/// Physical ATX button: short power press, long power press (force off), or reset.
|
||||
action: PowerAction,
|
||||
}
|
||||
#[derive(Debug, Deserialize, JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct WakeParams {
|
||||
/// Target MAC address, e.g. AA:BB:CC:DD:EE:FF. Uses the configured WOL interface.
|
||||
mac_address: String,
|
||||
}
|
||||
fn default_true() -> bool {
|
||||
true
|
||||
}
|
||||
#[derive(Debug, Deserialize, JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MountMediaParams {
|
||||
/// Image ID from one_kvm_list_media, never a filesystem path or URL.
|
||||
image_id: String,
|
||||
/// Present as a CD-ROM. Defaults to true.
|
||||
#[serde(default = "default_true")]
|
||||
cdrom: bool,
|
||||
/// Read-only access. Defaults to true; CD-ROM is always read-only.
|
||||
#[serde(default = "default_true")]
|
||||
read_only: bool,
|
||||
}
|
||||
#[derive(Debug, Deserialize, JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct UnmountMediaParams {
|
||||
/// Image ID from one_kvm_list_media. Does not unmount other images or the virtual drive.
|
||||
image_id: String,
|
||||
}
|
||||
#[derive(Clone)]
|
||||
pub struct McpTools {
|
||||
service: Arc<McpService>,
|
||||
tool_router: ToolRouter<Self>,
|
||||
}
|
||||
impl McpTools {
|
||||
pub fn new(service: Arc<McpService>) -> Self {
|
||||
Self {
|
||||
service,
|
||||
tool_router: Self::tool_router(),
|
||||
}
|
||||
}
|
||||
}
|
||||
#[tool_router]
|
||||
impl McpTools {
|
||||
#[tool(
|
||||
name = "one_kvm_get_status",
|
||||
description = "Get One-KVM video signal, current resolution, HID capabilities and global queue length."
|
||||
)]
|
||||
async fn status(&self) -> CallToolResult {
|
||||
self.service.call(Command::Status).await
|
||||
}
|
||||
|
||||
#[tool(
|
||||
name = "one_kvm_screenshot",
|
||||
description = "Capture a fresh JPEG at the current source resolution, without resizing or cropping. No browser is needed. Coordinates for pointer actions are normalized to [0,1] over this whole image."
|
||||
)]
|
||||
async fn screenshot(&self, Parameters(params): Parameters<ScreenshotParams>) -> CallToolResult {
|
||||
if params.quality.is_some_and(|q| !(1..=100).contains(&q)) {
|
||||
return failure("invalid_arguments", "quality must be between 1 and 100", 0);
|
||||
}
|
||||
self.service.call(Command::Screenshot(params.quality)).await
|
||||
}
|
||||
|
||||
#[tool(
|
||||
name = "one_kvm_execute_actions",
|
||||
description = "Execute a bounded batch of physical HID actions in global FIFO order. Pointer coordinates are normalized: (0,0) top-left, (1,1) bottom-right. Positive vertical_steps scrolls down. Text uses the target's US keyboard layout (ASCII, newline, tab); no clipboard or arbitrary Unicode. All parameters are checked before input. No client ownership, tasks or cancellation. On connection failure, inspect the screen before retrying: actions may already have executed."
|
||||
)]
|
||||
async fn actions(&self, Parameters(params): Parameters<ActionsParams>) -> CallToolResult {
|
||||
match actions::prepare(params.actions) {
|
||||
Ok(actions) => {
|
||||
self.service
|
||||
.call(Command::Actions(actions, params.return_screenshot))
|
||||
.await
|
||||
}
|
||||
Err(message) => failure("invalid_arguments", &message, 0),
|
||||
}
|
||||
}
|
||||
|
||||
#[tool(
|
||||
name = "one_kvm_power",
|
||||
description = "Press a configured physical ATX button: press_short, press_long (force off), or reset. Inspect ATX availability with one_kvm_get_status first. Success means the button pulse was sent, not that the target OS completed startup/shutdown. LED state is unknown when not wired. Uses the global FIFO and rate limit; never automatically repeat power operations."
|
||||
)]
|
||||
async fn power(&self, Parameters(params): Parameters<PowerParams>) -> CallToolResult {
|
||||
self.service
|
||||
.call(Command::Hardware(HardwareCommand::Power(params.action)))
|
||||
.await
|
||||
}
|
||||
|
||||
#[tool(
|
||||
name = "one_kvm_wake_on_lan",
|
||||
description = "Send Wake-on-LAN magic packets to a MAC address, using One-KVM's configured network interface. Reports packet transmission, not successful startup. Does not require ATX wiring. Shares the global FIFO and rate limit."
|
||||
)]
|
||||
async fn wake(&self, Parameters(params): Parameters<WakeParams>) -> CallToolResult {
|
||||
match crate::atx::normalize_wol_mac_address(¶ms.mac_address) {
|
||||
Ok(mac) => {
|
||||
self.service
|
||||
.call(Command::Hardware(HardwareCommand::Wake(mac)))
|
||||
.await
|
||||
}
|
||||
Err(error) => failure("invalid_arguments", &error.to_string(), 0),
|
||||
}
|
||||
}
|
||||
|
||||
#[tool(
|
||||
name = "one_kvm_list_media",
|
||||
description = "List uploaded ISO/IMG images, their image IDs and sizes, currently mounted media, and USB media slot capacity. No upload or file paths. Virtual media requires a supported USB gadget backend. Uses the global FIFO and rate limit.",
|
||||
annotations(read_only_hint = true)
|
||||
)]
|
||||
async fn list_media(&self) -> CallToolResult {
|
||||
self.service
|
||||
.call(Command::Hardware(HardwareCommand::ListMedia))
|
||||
.await
|
||||
}
|
||||
|
||||
#[tool(
|
||||
name = "one_kvm_mount_media",
|
||||
description = "Mount an uploaded image by its image_id from one_kvm_list_media into a free USB media slot. Defaults to CD-ROM and read-only; CD-ROM always forces read-only. To mount a disk, explicitly set cdrom=false. Does not upload, download, replace existing media, change USB configuration, or initialize/format a drive. Shares global FIFO/rate limits and existing USB operation locks."
|
||||
)]
|
||||
async fn mount_media(
|
||||
&self,
|
||||
Parameters(params): Parameters<MountMediaParams>,
|
||||
) -> CallToolResult {
|
||||
if !valid_image_id(¶ms.image_id) {
|
||||
return failure(
|
||||
"invalid_arguments",
|
||||
"image_id must be an image ID returned by one_kvm_list_media",
|
||||
0,
|
||||
);
|
||||
}
|
||||
self.service
|
||||
.call(Command::Hardware(HardwareCommand::MountMedia {
|
||||
image_id: params.image_id,
|
||||
cdrom: params.cdrom,
|
||||
read_only: params.read_only,
|
||||
}))
|
||||
.await
|
||||
}
|
||||
|
||||
#[tool(
|
||||
name = "one_kvm_unmount_media",
|
||||
description = "Unmount only the uploaded image identified by image_id, releasing its USB media slot. An already unmounted image succeeds. Does not unmount the virtual drive or other images, delete files, or force removal when the target prevents ejection. Shares the global FIFO/rate limit and existing USB operation locks."
|
||||
)]
|
||||
async fn unmount_media(
|
||||
&self,
|
||||
Parameters(params): Parameters<UnmountMediaParams>,
|
||||
) -> CallToolResult {
|
||||
if !valid_image_id(¶ms.image_id) {
|
||||
return failure(
|
||||
"invalid_arguments",
|
||||
"image_id must be an image ID returned by one_kvm_list_media",
|
||||
0,
|
||||
);
|
||||
}
|
||||
self.service
|
||||
.call(Command::Hardware(HardwareCommand::UnmountMedia(
|
||||
params.image_id,
|
||||
)))
|
||||
.await
|
||||
}
|
||||
}
|
||||
#[tool_handler(router = self.tool_router)]
|
||||
impl ServerHandler for McpTools {
|
||||
fn get_info(&self) -> ServerConfig {
|
||||
ServerConfig::new(ServerCapabilities::builder().enable_tools().build())
|
||||
.with_server_info(rmcp::model::Implementation::new("one-kvm", env!("CARGO_PKG_VERSION")))
|
||||
.with_instructions("One-KVM controls a physical computer using video and HID. First inspect the screen, then execute short batches and observe again. The service has one global FIFO and rate limit; coordinate multiple operators externally.")
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn screenshot_accepts_native_resolution_only() {
|
||||
assert!(serde_json::from_value::<ScreenshotParams>(serde_json::json!({})).is_ok());
|
||||
assert!(
|
||||
serde_json::from_value::<ScreenshotParams>(serde_json::json!({"max_width": 1920}))
|
||||
.is_err()
|
||||
);
|
||||
assert!(serde_json::from_value::<ScreenshotParams>(
|
||||
serde_json::json!({"max_height": 1080})
|
||||
)
|
||||
.is_err());
|
||||
assert!(serde_json::from_value::<ActionsParams>(
|
||||
serde_json::json!({"actions": [{"type": "wait", "ms": 1}], "control_id": "unneeded"})
|
||||
)
|
||||
.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hardware_params_have_safe_defaults_and_require_ids() {
|
||||
let mount: MountMediaParams =
|
||||
serde_json::from_value(serde_json::json!({"image_id":"abcdef"})).unwrap();
|
||||
assert!(mount.cdrom && mount.read_only);
|
||||
assert!(valid_image_id(&mount.image_id));
|
||||
for id in [
|
||||
"",
|
||||
"drive",
|
||||
"/etc/passwd",
|
||||
"../image.iso",
|
||||
"https://host/disk.iso",
|
||||
"0123456789abcdef0",
|
||||
] {
|
||||
assert!(!valid_image_id(id), "{id}");
|
||||
}
|
||||
assert!(serde_json::from_value::<PowerParams>(serde_json::json!({"action":"on"})).is_err());
|
||||
assert!(serde_json::from_value::<MountMediaParams>(
|
||||
serde_json::json!({"image_id":"abc", "path":"/tmp/a"})
|
||||
)
|
||||
.is_err());
|
||||
}
|
||||
}
|
||||
@@ -6,7 +6,6 @@ use tokio::sync::broadcast;
|
||||
use crate::atx::AtxController;
|
||||
use crate::audio::{AudioController, AudioControllerConfig, AudioQuality};
|
||||
use crate::auth::{SessionStore, TwoFactorService, UserStore};
|
||||
use crate::computer_use::ComputerUseManager;
|
||||
use crate::config::{self, AppConfig, ConfigStore};
|
||||
use crate::db::{open_database_pool, DatabasePool};
|
||||
use crate::events::EventBus;
|
||||
@@ -152,7 +151,6 @@ impl RuntimeBuilder {
|
||||
);
|
||||
}
|
||||
|
||||
let computer_use = ComputerUseManager::new(config_store.clone(), hid.clone());
|
||||
let state = AppState::new(
|
||||
db,
|
||||
config_store.clone(),
|
||||
@@ -164,7 +162,6 @@ impl RuntimeBuilder {
|
||||
stream_manager,
|
||||
webrtc,
|
||||
hid,
|
||||
computer_use,
|
||||
#[cfg(unix)]
|
||||
msd,
|
||||
atx,
|
||||
|
||||
28
src/state.rs
28
src/state.rs
@@ -4,7 +4,6 @@ use tokio::sync::{broadcast, watch, Mutex, RwLock};
|
||||
use crate::atx::AtxController;
|
||||
use crate::audio::AudioController;
|
||||
use crate::auth::{SessionStore, TwoFactorService, UserStore};
|
||||
use crate::computer_use::ComputerUseManager;
|
||||
use crate::config::ConfigStore;
|
||||
use crate::db::DatabasePool;
|
||||
#[cfg(unix)]
|
||||
@@ -15,6 +14,7 @@ use crate::events::{
|
||||
};
|
||||
use crate::extensions::{ExtensionId, ExtensionManager};
|
||||
use crate::hid::HidController;
|
||||
use crate::mcp::{McpHardware, McpService};
|
||||
#[cfg(unix)]
|
||||
use crate::msd::MsdController;
|
||||
#[cfg(unix)]
|
||||
@@ -72,7 +72,7 @@ pub struct AppState {
|
||||
pub stream_manager: Arc<VideoStreamManager>,
|
||||
pub webrtc: Arc<WebRtcStreamer>,
|
||||
pub hid: Arc<HidController>,
|
||||
pub computer_use: Arc<ComputerUseManager>,
|
||||
pub mcp: Arc<McpService>,
|
||||
#[cfg(unix)]
|
||||
pub msd: Arc<RwLock<Option<MsdController>>>,
|
||||
pub atx: Arc<RwLock<Option<AtxController>>>,
|
||||
@@ -104,7 +104,6 @@ impl AppState {
|
||||
stream_manager: Arc<VideoStreamManager>,
|
||||
webrtc: Arc<WebRtcStreamer>,
|
||||
hid: Arc<HidController>,
|
||||
computer_use: Arc<ComputerUseManager>,
|
||||
#[cfg(unix)] msd: Option<MsdController>,
|
||||
atx: Option<AtxController>,
|
||||
audio: Arc<AudioController>,
|
||||
@@ -138,6 +137,23 @@ impl AppState {
|
||||
data_dir.clone(),
|
||||
);
|
||||
|
||||
let atx = Arc::new(RwLock::new(atx));
|
||||
let config_apply_locks = ConfigApplyLocks::new();
|
||||
let hardware = McpHardware {
|
||||
config: config.clone(),
|
||||
atx: atx.clone(),
|
||||
wol_history: db.wol_history(),
|
||||
#[cfg(unix)]
|
||||
msd: msd.clone(),
|
||||
#[cfg(unix)]
|
||||
otg_lock: config_apply_locks.otg.clone(),
|
||||
};
|
||||
let mcp = McpService::new(
|
||||
config.clone(),
|
||||
hid.clone(),
|
||||
stream_manager.clone(),
|
||||
hardware,
|
||||
);
|
||||
Arc::new(Self {
|
||||
db,
|
||||
config,
|
||||
@@ -149,10 +165,10 @@ impl AppState {
|
||||
stream_manager,
|
||||
webrtc,
|
||||
hid,
|
||||
computer_use,
|
||||
mcp,
|
||||
#[cfg(unix)]
|
||||
msd,
|
||||
atx: Arc::new(RwLock::new(atx)),
|
||||
atx,
|
||||
audio,
|
||||
usb,
|
||||
remote_access,
|
||||
@@ -163,7 +179,7 @@ impl AppState {
|
||||
watchdog: Arc::new(WatchdogController::new()),
|
||||
shutdown_tx,
|
||||
revoked_sessions: Arc::new(RwLock::new(VecDeque::new())),
|
||||
config_apply_locks: ConfigApplyLocks::new(),
|
||||
config_apply_locks,
|
||||
data_dir,
|
||||
#[cfg(unix)]
|
||||
uac_playback,
|
||||
|
||||
@@ -13,6 +13,8 @@ pub mod pipeline;
|
||||
pub mod recovery;
|
||||
pub mod signal;
|
||||
#[cfg(feature = "desktop")]
|
||||
pub mod snapshot;
|
||||
#[cfg(feature = "desktop")]
|
||||
pub mod stream_manager;
|
||||
#[cfg(feature = "desktop")]
|
||||
pub mod streamer;
|
||||
|
||||
@@ -256,6 +256,22 @@ fn run(
|
||||
}
|
||||
|
||||
let capture = active.as_mut().expect("opened capture");
|
||||
if pipeline.snapshot_slot.pending() {
|
||||
let mut raw = Vec::new();
|
||||
if let Ok(meta) = capture.stream.next_into(&mut raw) {
|
||||
raw.truncate(meta.bytes_used);
|
||||
if meta.bytes_used >= MIN_CAPTURE_FRAME_SIZE {
|
||||
pipeline.snapshot_slot.publish(&VideoFrame::from_vec(
|
||||
raw,
|
||||
capture.stream.resolution(),
|
||||
capture.stream.format(),
|
||||
capture.stream.stride(),
|
||||
meta.sequence,
|
||||
));
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let reservations = pipeline.reserve_outputs();
|
||||
let result = if reservations.as_ref().is_none_or(Vec::is_empty) {
|
||||
match capture.encoder.as_mut().expect("active encoder") {
|
||||
|
||||
@@ -7,6 +7,7 @@ mod mjpeg_admission;
|
||||
mod shared;
|
||||
|
||||
pub use delivery::EncodedVideoFrameReceiver;
|
||||
pub(crate) use shared::compact_strided_frame_for_encoder;
|
||||
pub use shared::{
|
||||
EncodedVideoFrame, PipelineAppliedConfig, PipelineLifecycle, PipelineStateNotification,
|
||||
SharedVideoPipeline, SharedVideoPipelineConfig, SharedVideoPipelineStats,
|
||||
|
||||
@@ -39,10 +39,7 @@ use super::mjpeg_admission::{MjpegAdmissionPolicy, MjpegAdmissionStage, MjpegDro
|
||||
#[path = "dmabuf.rs"]
|
||||
mod dmabuf;
|
||||
|
||||
#[cfg(any(
|
||||
test,
|
||||
all(target_os = "linux", feature = "rkmpp")
|
||||
))]
|
||||
#[cfg(any(test, all(target_os = "linux", feature = "rkmpp")))]
|
||||
fn rkmpp_dma_eligible(
|
||||
backend: Option<EncoderBackend>,
|
||||
codec: VideoEncoderType,
|
||||
@@ -524,6 +521,7 @@ struct CachedH26xParameterSets {
|
||||
|
||||
/// Universal shared video pipeline
|
||||
pub struct SharedVideoPipeline {
|
||||
pub(crate) snapshot_slot: crate::video::snapshot::SnapshotSlot,
|
||||
config: RwLock<SharedVideoPipelineConfig>,
|
||||
subscribers: ParkingRwLock<Vec<Arc<VideoSubscriber>>>,
|
||||
stats: ParkingMutex<SharedVideoPipelineStats>,
|
||||
@@ -588,6 +586,7 @@ impl SharedVideoPipeline {
|
||||
let (h264_profile_tx, h264_profile_rx) = watch::channel(None);
|
||||
|
||||
let pipeline = Arc::new(Self {
|
||||
snapshot_slot: Default::default(),
|
||||
config: RwLock::new(config),
|
||||
subscribers: ParkingRwLock::new(Vec::new()),
|
||||
stats: ParkingMutex::new(SharedVideoPipelineStats::default()),
|
||||
@@ -1698,6 +1697,15 @@ impl SharedVideoPipeline {
|
||||
|
||||
owned.truncate(frame_size);
|
||||
capture_sequence = capture_sequence.wrapping_add(1);
|
||||
if pipeline.snapshot_slot.pending() {
|
||||
pipeline.snapshot_slot.publish(&VideoFrame::from_vec(
|
||||
owned.clone(),
|
||||
resolution,
|
||||
pixel_format,
|
||||
stride,
|
||||
capture_sequence,
|
||||
));
|
||||
}
|
||||
|
||||
if parallel_mjpeg_decode
|
||||
&& pixel_format == PixelFormat::Mjpeg
|
||||
@@ -2079,7 +2087,10 @@ impl SharedVideoPipeline {
|
||||
}
|
||||
}
|
||||
|
||||
fn compact_strided_frame_for_encoder(frame: &VideoFrame, data: &[u8]) -> Result<Option<Vec<u8>>> {
|
||||
pub(crate) fn compact_strided_frame_for_encoder(
|
||||
frame: &VideoFrame,
|
||||
data: &[u8],
|
||||
) -> Result<Option<Vec<u8>>> {
|
||||
let width = frame.resolution.width as usize;
|
||||
let height = frame.resolution.height as usize;
|
||||
let stride = frame.stride as usize;
|
||||
|
||||
271
src/video/snapshot.rs
Normal file
271
src/video/snapshot.rs
Normal file
@@ -0,0 +1,271 @@
|
||||
//! One-shot capture at the current source resolution. No resize or browser dependency.
|
||||
use super::codec::{Encoder, EncoderConfig, JpegEncoder, PixelConverter};
|
||||
use super::{format::PixelFormat, frame::VideoFrame, VideoStreamManager};
|
||||
use crate::config::StreamMode;
|
||||
use crate::error::{AppError, Result};
|
||||
use bytes::Bytes;
|
||||
use parking_lot::Mutex;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::oneshot;
|
||||
|
||||
#[derive(Default)]
|
||||
pub(crate) struct SnapshotSlot(Mutex<Option<oneshot::Sender<VideoFrame>>>);
|
||||
impl SnapshotSlot {
|
||||
pub fn request(&self) -> Result<oneshot::Receiver<VideoFrame>> {
|
||||
let mut slot = self.0.lock();
|
||||
if slot.as_ref().is_some_and(|tx| !tx.is_closed()) {
|
||||
return Err(AppError::ServiceUnavailable(
|
||||
"Snapshot already pending".into(),
|
||||
));
|
||||
}
|
||||
let (tx, rx) = oneshot::channel();
|
||||
*slot = Some(tx);
|
||||
Ok(rx)
|
||||
}
|
||||
pub fn pending(&self) -> bool {
|
||||
self.0.lock().as_ref().is_some_and(|tx| !tx.is_closed())
|
||||
}
|
||||
pub fn publish(&self, frame: &VideoFrame) {
|
||||
let mut slot = self.0.lock();
|
||||
if !slot.as_ref().is_some_and(|tx| !tx.is_closed()) {
|
||||
*slot = None;
|
||||
return;
|
||||
}
|
||||
// UVC devices can start with both partial JPEGs and structurally valid
|
||||
// but noisy JPEGs. Skip the initial capture frames and keep waiting for
|
||||
// a valid frame within the existing timeout, without reopening capture.
|
||||
if frame.online
|
||||
&& frame.format.is_compressed()
|
||||
&& (frame.sequence < 3 || turbojpeg::read_header(frame.data()).is_err())
|
||||
{
|
||||
return;
|
||||
}
|
||||
if let Some(tx) = slot.take() {
|
||||
let _ = tx.send(frame.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct Snapshot {
|
||||
pub jpeg: Bytes,
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
pub age_ms: u64,
|
||||
}
|
||||
|
||||
impl VideoStreamManager {
|
||||
pub async fn capture_snapshot(&self, quality: Option<u8>) -> Result<Snapshot> {
|
||||
if self.is_switching() || self.is_config_changing() {
|
||||
return Err(AppError::ServiceUnavailable(
|
||||
"Video configuration is changing; retry later".into(),
|
||||
));
|
||||
}
|
||||
if quality.is_some_and(|q| !(1..=100).contains(&q)) {
|
||||
return Err(AppError::BadRequest(
|
||||
"quality must be between 1 and 100".into(),
|
||||
));
|
||||
}
|
||||
let capture = async {
|
||||
let frame = match self.current_mode().await {
|
||||
StreamMode::Mjpeg => {
|
||||
let streamer = self.streamer();
|
||||
let handler = self.mjpeg_handler();
|
||||
let _guard = crate::stream::ClientGuard::new(
|
||||
format!("snapshot-{}", uuid::Uuid::new_v4()),
|
||||
handler,
|
||||
);
|
||||
let rx = streamer.snapshot_slot.request()?;
|
||||
streamer.start().await?;
|
||||
rx.await.map_err(|_| {
|
||||
AppError::ServiceUnavailable("Capture stopped before snapshot".into())
|
||||
})?
|
||||
}
|
||||
StreamMode::WebRTC => {
|
||||
let pipeline = self.snapshot_pipeline().await?;
|
||||
// Keeps the shared capture pipeline alive without opening another video device.
|
||||
let _consumer = pipeline.subscribe();
|
||||
let rx = pipeline.snapshot_slot.request()?;
|
||||
rx.await.map_err(|_| {
|
||||
AppError::ServiceUnavailable("Capture stopped before snapshot".into())
|
||||
})?
|
||||
}
|
||||
};
|
||||
tokio::task::spawn_blocking(move || encode_snapshot(frame, quality))
|
||||
.await
|
||||
.map_err(|_| AppError::VideoError("Snapshot encoder failed".into()))?
|
||||
};
|
||||
tokio::time::timeout(Duration::from_secs(10), capture)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
AppError::ServiceUnavailable("Timed out waiting for a fresh video frame".into())
|
||||
})?
|
||||
}
|
||||
}
|
||||
|
||||
fn encode_snapshot(frame: VideoFrame, quality: Option<u8>) -> Result<Snapshot> {
|
||||
if !frame.online {
|
||||
return Err(AppError::ServiceUnavailable("No video signal".into()));
|
||||
}
|
||||
let resolution = frame.resolution;
|
||||
let age = frame.capture_ts;
|
||||
let jpeg = if frame.format.is_compressed() {
|
||||
let header = turbojpeg::read_header(frame.data()).map_err(jpeg_error)?;
|
||||
// Use dimensions from the actual image, including source-following resolution changes.
|
||||
let jpeg = if let Some(quality) = quality {
|
||||
let image = turbojpeg::decompress(frame.data(), turbojpeg::PixelFormat::RGB)
|
||||
.map_err(jpeg_error)?;
|
||||
Bytes::from(
|
||||
turbojpeg::compress(image.as_deref(), quality as i32, turbojpeg::Subsamp::Sub2x2)
|
||||
.map_err(jpeg_error)?
|
||||
.to_vec(),
|
||||
)
|
||||
} else {
|
||||
frame.owned_data_bytes()
|
||||
};
|
||||
return Ok(Snapshot {
|
||||
jpeg,
|
||||
width: header.width as u32,
|
||||
height: header.height as u32,
|
||||
age_ms: age.elapsed().as_millis() as u64,
|
||||
});
|
||||
} else {
|
||||
let compact = super::pipeline::compact_strided_frame_for_encoder(&frame, frame.data())?;
|
||||
let data = compact.as_deref().unwrap_or(frame.data());
|
||||
let quality = quality.unwrap_or(80) as u32;
|
||||
let mut config = EncoderConfig::jpeg(resolution, quality);
|
||||
config.input_format = frame.format;
|
||||
let mut encoder = JpegEncoder::new(config)?;
|
||||
if encoder.supports_format(frame.format) {
|
||||
encoder.encode(data, frame.sequence)?.data
|
||||
} else {
|
||||
let mut converter = match frame.format {
|
||||
PixelFormat::Uyvy => Some(PixelConverter::uyvy_to_yuv420p(resolution)),
|
||||
PixelFormat::Nv21 => Some(PixelConverter::nv21_to_yuv420p(resolution)),
|
||||
PixelFormat::Yvu420 => Some(PixelConverter::yvu420_to_yuv420p(resolution)),
|
||||
PixelFormat::Yuv420 => None,
|
||||
other => {
|
||||
return Err(AppError::VideoError(format!(
|
||||
"Snapshot format unsupported: {other}"
|
||||
)))
|
||||
}
|
||||
};
|
||||
let i420 = match converter.as_mut() {
|
||||
Some(c) => c.convert(data)?,
|
||||
None => data,
|
||||
};
|
||||
Bytes::from(
|
||||
turbojpeg::compress_yuv(
|
||||
turbojpeg::YuvImage {
|
||||
pixels: i420,
|
||||
width: resolution.width as usize,
|
||||
height: resolution.height as usize,
|
||||
align: 1,
|
||||
subsamp: turbojpeg::Subsamp::Sub2x2,
|
||||
},
|
||||
quality as i32,
|
||||
)
|
||||
.map_err(jpeg_error)?
|
||||
.to_vec(),
|
||||
)
|
||||
}
|
||||
};
|
||||
Ok(Snapshot {
|
||||
jpeg,
|
||||
width: resolution.width,
|
||||
height: resolution.height,
|
||||
age_ms: age.elapsed().as_millis() as u64,
|
||||
})
|
||||
}
|
||||
fn jpeg_error(error: turbojpeg::Error) -> AppError {
|
||||
AppError::VideoError(format!("Snapshot JPEG: {error}"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::format::Resolution;
|
||||
use super::*;
|
||||
#[test]
|
||||
fn snapshot_preserves_native_resolution_and_padded_rows() {
|
||||
let width = 2048;
|
||||
let height = 4;
|
||||
let stride = width * 3 + 32;
|
||||
let frame = VideoFrame::from_vec(
|
||||
vec![90; stride * height],
|
||||
Resolution::new(width as u32, height as u32),
|
||||
PixelFormat::Rgb24,
|
||||
stride as u32,
|
||||
1,
|
||||
);
|
||||
let snapshot = encode_snapshot(frame, Some(80)).unwrap();
|
||||
let header = turbojpeg::read_header(&snapshot.jpeg).unwrap();
|
||||
assert_eq!((header.width, header.height), (width, height));
|
||||
assert_eq!(
|
||||
(snapshot.width, snapshot.height),
|
||||
(width as u32, height as u32)
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn jpeg_passthrough_and_quality_change_preserve_dimensions() {
|
||||
let resolution = Resolution::new(2048, 4);
|
||||
let raw = VideoFrame::from_vec(
|
||||
vec![80; 2048 * 4 * 3],
|
||||
resolution,
|
||||
PixelFormat::Rgb24,
|
||||
2048 * 3,
|
||||
1,
|
||||
);
|
||||
let first = encode_snapshot(raw, Some(80)).unwrap();
|
||||
let source = VideoFrame::new(first.jpeg.clone(), resolution, PixelFormat::Mjpeg, 0, 2);
|
||||
let copied = encode_snapshot(source.clone(), None).unwrap();
|
||||
assert_eq!(copied.jpeg, first.jpeg);
|
||||
let changed = encode_snapshot(source, Some(60)).unwrap();
|
||||
let header = turbojpeg::read_header(&changed.jpeg).unwrap();
|
||||
assert_eq!((header.width, header.height), (2048, 4));
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn slot_delivers_only_a_requested_new_frame() {
|
||||
let slot = SnapshotSlot::default();
|
||||
assert!(!slot.pending());
|
||||
let rx = slot.request().unwrap();
|
||||
assert!(slot.request().is_err());
|
||||
let frame = VideoFrame::from_vec(
|
||||
vec![0; 24],
|
||||
Resolution::new(4, 2),
|
||||
PixelFormat::Rgb24,
|
||||
12,
|
||||
7,
|
||||
);
|
||||
slot.publish(&frame);
|
||||
assert_eq!(rx.await.unwrap().sequence, 7);
|
||||
assert!(!slot.pending());
|
||||
let abandoned = slot.request().unwrap();
|
||||
drop(abandoned);
|
||||
assert!(slot.request().is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snapshot_waits_past_invalid_and_unsettled_startup_jpeg() {
|
||||
let slot = SnapshotSlot::default();
|
||||
let mut rx = slot.request().unwrap();
|
||||
let resolution = Resolution::new(4, 2);
|
||||
let partial = VideoFrame::from_vec(vec![0; 128], resolution, PixelFormat::Mjpeg, 0, 3);
|
||||
slot.publish(&partial);
|
||||
assert!(matches!(
|
||||
rx.try_recv(),
|
||||
Err(oneshot::error::TryRecvError::Empty)
|
||||
));
|
||||
assert!(slot.pending());
|
||||
let raw = VideoFrame::from_vec(vec![80; 24], resolution, PixelFormat::Rgb24, 12, 2);
|
||||
let jpeg = encode_snapshot(raw, None).unwrap();
|
||||
let startup = VideoFrame::new(jpeg.jpeg.clone(), resolution, PixelFormat::Mjpeg, 0, 2);
|
||||
slot.publish(&startup);
|
||||
assert!(matches!(
|
||||
rx.try_recv(),
|
||||
Err(oneshot::error::TryRecvError::Empty)
|
||||
));
|
||||
let complete = VideoFrame::new(jpeg.jpeg, resolution, PixelFormat::Mjpeg, 0, 3);
|
||||
slot.publish(&complete);
|
||||
assert_eq!(rx.await.unwrap().sequence, 3);
|
||||
assert!(!slot.pending());
|
||||
}
|
||||
}
|
||||
@@ -732,6 +732,22 @@ impl VideoStreamManager {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn snapshot_pipeline(
|
||||
&self,
|
||||
) -> Result<Arc<crate::video::pipeline::SharedVideoPipeline>> {
|
||||
// Preserve a running pipeline's negotiated/source-following geometry.
|
||||
// Syncing the MJPEG configuration on every snapshot could restart WebRTC viewers.
|
||||
if self.webrtc_streamer.get_pipeline_config().await.is_none() {
|
||||
if self.streamer.state().await == StreamerState::Uninitialized {
|
||||
self.streamer.init_auto().await?;
|
||||
}
|
||||
self.sync_webrtc_capture_source("for snapshot").await;
|
||||
}
|
||||
self.webrtc_streamer
|
||||
.ensure_video_pipeline_for_external()
|
||||
.await
|
||||
}
|
||||
|
||||
/// Get the current video encoding configuration from the shared pipeline
|
||||
pub async fn get_encoding_config(
|
||||
&self,
|
||||
|
||||
@@ -165,6 +165,7 @@ impl StreamerState {
|
||||
pub struct Streamer {
|
||||
config: RwLock<StreamerConfig>,
|
||||
mjpeg_handler: Arc<MjpegStreamHandler>,
|
||||
pub(crate) snapshot_slot: super::snapshot::SnapshotSlot,
|
||||
current_device: RwLock<Option<VideoDeviceInfo>>,
|
||||
state: RwLock<StreamerState>,
|
||||
start_lock: tokio::sync::Mutex<()>,
|
||||
@@ -197,6 +198,7 @@ impl Streamer {
|
||||
Arc::new(Self {
|
||||
config: RwLock::new(StreamerConfig::default()),
|
||||
mjpeg_handler: Arc::new(MjpegStreamHandler::new()),
|
||||
snapshot_slot: Default::default(),
|
||||
current_device: RwLock::new(None),
|
||||
state: RwLock::new(StreamerState::Uninitialized),
|
||||
start_lock: tokio::sync::Mutex::new(()),
|
||||
@@ -221,6 +223,7 @@ impl Streamer {
|
||||
Arc::new(Self {
|
||||
config: RwLock::new(config),
|
||||
mjpeg_handler: Arc::new(MjpegStreamHandler::new()),
|
||||
snapshot_slot: Default::default(),
|
||||
current_device: RwLock::new(None),
|
||||
state: RwLock::new(StreamerState::Uninitialized),
|
||||
start_lock: tokio::sync::Mutex::new(()),
|
||||
@@ -1236,6 +1239,7 @@ impl Streamer {
|
||||
});
|
||||
}
|
||||
|
||||
self.snapshot_slot.publish(&frame);
|
||||
self.mjpeg_handler.update_frame(frame);
|
||||
|
||||
fps_frame_count += 1;
|
||||
@@ -1534,6 +1538,7 @@ impl Default for Streamer {
|
||||
Self {
|
||||
config: RwLock::new(StreamerConfig::default()),
|
||||
mjpeg_handler: Arc::new(MjpegStreamHandler::new()),
|
||||
snapshot_slot: Default::default(),
|
||||
current_device: RwLock::new(None),
|
||||
state: RwLock::new(StreamerState::Uninitialized),
|
||||
start_lock: tokio::sync::Mutex::new(()),
|
||||
|
||||
@@ -135,30 +135,12 @@ pub struct WolHistoryResponse {
|
||||
pub history: Vec<WolHistoryEntry>,
|
||||
}
|
||||
|
||||
fn normalize_wol_mac_address(mac_address: &str) -> String {
|
||||
let normalized = mac_address.trim().to_uppercase().replace('-', ":");
|
||||
|
||||
if normalized.len() == 12 && normalized.chars().all(|c| c.is_ascii_hexdigit()) {
|
||||
let mut mac_with_separator = String::with_capacity(17);
|
||||
for (index, chunk) in normalized.as_bytes().chunks(2).enumerate() {
|
||||
if index > 0 {
|
||||
mac_with_separator.push(':');
|
||||
}
|
||||
mac_with_separator.push(chunk[0] as char);
|
||||
mac_with_separator.push(chunk[1] as char);
|
||||
}
|
||||
mac_with_separator
|
||||
} else {
|
||||
normalized
|
||||
}
|
||||
}
|
||||
|
||||
/// Send Wake-on-LAN magic packet
|
||||
pub async fn atx_wol(
|
||||
State(state): State<Arc<AppState>>,
|
||||
Json(req): Json<WolRequest>,
|
||||
) -> Result<Json<LoginResponse>> {
|
||||
let mac_address = normalize_wol_mac_address(&req.mac_address);
|
||||
let mac_address = crate::atx::normalize_wol_mac_address(&req.mac_address)?;
|
||||
|
||||
// Get WOL interface from config
|
||||
let config = state.config.get();
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
use axum::{
|
||||
extract::{ws::WebSocketUpgrade, Query, State},
|
||||
response::Response,
|
||||
Json,
|
||||
};
|
||||
use serde::Deserialize;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::computer_use::{
|
||||
ComputerUseConfigResponse, ComputerUseConfigUpdate, ComputerUseSessionSummary,
|
||||
ComputerUseStartRequest,
|
||||
};
|
||||
use crate::error::Result;
|
||||
use crate::state::AppState;
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct ComputerUseWsQuery {
|
||||
client_id: Option<String>,
|
||||
}
|
||||
|
||||
pub async fn computer_use_config(
|
||||
State(state): State<Arc<AppState>>,
|
||||
) -> Json<ComputerUseConfigResponse> {
|
||||
Json(state.computer_use.config_response())
|
||||
}
|
||||
|
||||
pub async fn computer_use_update_config(
|
||||
State(state): State<Arc<AppState>>,
|
||||
Json(req): Json<ComputerUseConfigUpdate>,
|
||||
) -> Result<Json<ComputerUseConfigResponse>> {
|
||||
Ok(Json(state.computer_use.update_config(req).await?))
|
||||
}
|
||||
|
||||
pub async fn computer_use_session(
|
||||
State(state): State<Arc<AppState>>,
|
||||
) -> Json<ComputerUseSessionSummary> {
|
||||
Json(state.computer_use.summary().await)
|
||||
}
|
||||
|
||||
pub async fn computer_use_start(
|
||||
State(state): State<Arc<AppState>>,
|
||||
Json(req): Json<ComputerUseStartRequest>,
|
||||
) -> Result<Json<ComputerUseSessionSummary>> {
|
||||
Ok(Json(state.computer_use.start(req).await?))
|
||||
}
|
||||
|
||||
pub async fn computer_use_stop(
|
||||
State(state): State<Arc<AppState>>,
|
||||
) -> Result<Json<ComputerUseSessionSummary>> {
|
||||
Ok(Json(state.computer_use.stop().await?))
|
||||
}
|
||||
|
||||
pub async fn computer_use_ws(
|
||||
ws: WebSocketUpgrade,
|
||||
State(state): State<Arc<AppState>>,
|
||||
Query(query): Query<ComputerUseWsQuery>,
|
||||
) -> Response {
|
||||
ws.on_upgrade(move |socket| {
|
||||
state
|
||||
.computer_use
|
||||
.clone()
|
||||
.handle_socket(socket, query.client_id)
|
||||
})
|
||||
}
|
||||
@@ -60,7 +60,8 @@ use crate::state::AppState;
|
||||
|
||||
fn sanitize_config_for_api(config: &mut AppConfig) {
|
||||
config.stream.turn_password = None;
|
||||
config.computer_use.api_key = None;
|
||||
config.mcp.api_key_hash = None;
|
||||
config.mcp.api_key = None;
|
||||
|
||||
config.rustdesk.device_password.clear();
|
||||
config.rustdesk.relay_key = None;
|
||||
|
||||
82
src/web/handlers/mcp.rs
Normal file
82
src/web/handlers/mcp.rs
Normal file
@@ -0,0 +1,82 @@
|
||||
use crate::{
|
||||
error::{AppError, Result},
|
||||
state::AppState,
|
||||
};
|
||||
use axum::{extract::State, http::header, Json};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::sync::Arc;
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct McpConfigResponse {
|
||||
pub enabled: bool,
|
||||
pub requests_per_second: u32,
|
||||
pub api_key_configured: bool,
|
||||
pub endpoint: &'static str,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub api_key: Option<String>,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct McpConfigUpdate {
|
||||
pub enabled: Option<bool>,
|
||||
pub requests_per_second: Option<u32>,
|
||||
#[serde(default)]
|
||||
pub regenerate_api_key: bool,
|
||||
}
|
||||
fn response(state: &AppState) -> McpConfigResponse {
|
||||
let config = state.config.get();
|
||||
McpConfigResponse {
|
||||
enabled: config.mcp.enabled,
|
||||
requests_per_second: config.mcp.requests_per_second,
|
||||
api_key_configured: config.mcp.api_key_hash.is_some(),
|
||||
endpoint: "/mcp",
|
||||
api_key: config.mcp.api_key.clone(),
|
||||
}
|
||||
}
|
||||
pub async fn mcp_config(
|
||||
State(state): State<Arc<AppState>>,
|
||||
) -> (
|
||||
[(header::HeaderName, &'static str); 1],
|
||||
Json<McpConfigResponse>,
|
||||
) {
|
||||
(
|
||||
[(header::CACHE_CONTROL, "no-store")],
|
||||
Json(response(&state)),
|
||||
)
|
||||
}
|
||||
pub async fn mcp_update_config(
|
||||
State(state): State<Arc<AppState>>,
|
||||
Json(req): Json<McpConfigUpdate>,
|
||||
) -> Result<(
|
||||
[(header::HeaderName, &'static str); 1],
|
||||
Json<McpConfigResponse>,
|
||||
)> {
|
||||
if req
|
||||
.requests_per_second
|
||||
.is_some_and(|rate| !(1..=100).contains(&rate))
|
||||
{
|
||||
return Err(AppError::BadRequest(
|
||||
"requests_per_second must be between 1 and 100".into(),
|
||||
));
|
||||
}
|
||||
state
|
||||
.config
|
||||
.update(|config| {
|
||||
if let Some(enabled) = req.enabled {
|
||||
config.mcp.enabled = enabled;
|
||||
}
|
||||
if let Some(rate) = req.requests_per_second {
|
||||
config.mcp.requests_per_second = rate;
|
||||
}
|
||||
if req.regenerate_api_key || (config.mcp.enabled && config.mcp.api_key_hash.is_none()) {
|
||||
let key = crate::mcp::generate_key();
|
||||
config.mcp.api_key_hash = Some(crate::mcp::hash_key(&key));
|
||||
config.mcp.api_key = Some(key);
|
||||
}
|
||||
})
|
||||
.await?;
|
||||
Ok((
|
||||
[(header::CACHE_CONTROL, "no-store")],
|
||||
Json(response(&state)),
|
||||
))
|
||||
}
|
||||
@@ -7,9 +7,9 @@ mod account;
|
||||
mod atx_api;
|
||||
mod audio_api;
|
||||
mod auth;
|
||||
mod computer_use;
|
||||
mod hid_api;
|
||||
mod inventory;
|
||||
mod mcp;
|
||||
#[cfg(unix)]
|
||||
mod msd_api;
|
||||
mod setup;
|
||||
@@ -22,9 +22,9 @@ pub use account::*;
|
||||
pub use atx_api::*;
|
||||
pub use audio_api::*;
|
||||
pub use auth::*;
|
||||
pub use computer_use::*;
|
||||
pub use hid_api::*;
|
||||
pub use inventory::*;
|
||||
pub use mcp::*;
|
||||
#[cfg(unix)]
|
||||
pub use msd_api::*;
|
||||
pub use setup::*;
|
||||
|
||||
@@ -206,18 +206,10 @@ pub fn create_router(state: Arc<AppState>) -> Router {
|
||||
"/config/watchdog",
|
||||
patch(handlers::config::update_watchdog_config),
|
||||
)
|
||||
.route("/config/computer-use", get(handlers::computer_use_config))
|
||||
.route(
|
||||
"/config/computer-use",
|
||||
patch(handlers::computer_use_update_config),
|
||||
"/config/mcp",
|
||||
get(handlers::mcp_config).patch(handlers::mcp_update_config),
|
||||
)
|
||||
.route("/computer-use/session", get(handlers::computer_use_session))
|
||||
.route("/computer-use/session", post(handlers::computer_use_start))
|
||||
.route(
|
||||
"/computer-use/session/stop",
|
||||
post(handlers::computer_use_stop),
|
||||
)
|
||||
.route("/ws/computer-use", any(handlers::computer_use_ws))
|
||||
// Auth configuration
|
||||
.route("/config/auth", get(handlers::config::get_auth_config))
|
||||
.route("/config/auth", patch(handlers::config::update_auth_config))
|
||||
@@ -382,6 +374,7 @@ pub fn create_router(state: Arc<AppState>) -> Router {
|
||||
// Main router
|
||||
let main_router = Router::new()
|
||||
.nest("/api", api_routes)
|
||||
.merge(crate::mcp::router(state.mcp.clone(), state.config.clone()))
|
||||
.merge(static_routes)
|
||||
.layer(TraceLayer::new_for_http())
|
||||
.layer(cors)
|
||||
|
||||
@@ -2,12 +2,6 @@ import { request, uploadRequest, ApiError } from './request'
|
||||
import type {
|
||||
CanonicalKey,
|
||||
Ch9329DescriptorState,
|
||||
ComputerUseButton,
|
||||
ComputerUseConfigResponse,
|
||||
ComputerUseConfigUpdate,
|
||||
ComputerUseSessionStatus,
|
||||
ComputerUseSessionSummary,
|
||||
ComputerUseStartRequest,
|
||||
} from '@/types/generated'
|
||||
import { useHidWebSocket, type HidKeyboardEvent, type HidMouseEvent } from '@/composables/useHidWebSocket'
|
||||
import type { StreamSignalReason, StreamState } from '@/types/websocket'
|
||||
@@ -508,48 +502,17 @@ export const hidApi = {
|
||||
isWebSocketConnected: () => hidWs.connected.value,
|
||||
}
|
||||
|
||||
export type ComputerUseStatus = ComputerUseSessionStatus
|
||||
|
||||
export type ComputerUseAction =
|
||||
| { type: 'click'; x: number; y: number; button?: ComputerUseButton }
|
||||
| { type: 'double_click'; x: number; y: number; button?: ComputerUseButton }
|
||||
| { type: 'move'; x: number; y: number }
|
||||
| { type: 'drag'; path: Array<{ x: number; y: number }>; button?: ComputerUseButton }
|
||||
| { type: 'scroll'; x: number; y: number; dx?: number; dy?: number }
|
||||
| { type: 'type'; text: string }
|
||||
| { type: 'keypress'; keys: string[] }
|
||||
| { type: 'wait'; ms: number }
|
||||
| { type: 'screenshot' }
|
||||
|
||||
export type ComputerUseConversationMessage =
|
||||
| { role: 'user'; text: string }
|
||||
| { role: 'assistant'; text: string }
|
||||
|
||||
export type ComputerUseConfig = ComputerUseConfigResponse
|
||||
|
||||
export type ComputerUseSession = ComputerUseSessionSummary
|
||||
|
||||
export const computerUseApi = {
|
||||
config: () => request<ComputerUseConfig>('/config/computer-use'),
|
||||
|
||||
updateConfig: (data: ComputerUseConfigUpdate) =>
|
||||
request<ComputerUseConfig>('/config/computer-use', {
|
||||
method: 'PATCH',
|
||||
body: JSON.stringify(data),
|
||||
}),
|
||||
|
||||
session: () => request<ComputerUseSession>('/computer-use/session'),
|
||||
|
||||
start: (data: ComputerUseStartRequest) =>
|
||||
request<ComputerUseSession>('/computer-use/session', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify(data),
|
||||
}),
|
||||
|
||||
stop: () =>
|
||||
request<ComputerUseSession>('/computer-use/session/stop', {
|
||||
method: 'POST',
|
||||
}),
|
||||
export interface McpConfig {
|
||||
enabled: boolean
|
||||
requests_per_second: number
|
||||
api_key_configured: boolean
|
||||
endpoint: string
|
||||
api_key?: string
|
||||
}
|
||||
export const mcpApi = {
|
||||
config: () => request<McpConfig>('/config/mcp'),
|
||||
updateConfig: (data: { enabled?: boolean; requests_per_second?: number; regenerate_api_key?: boolean }) =>
|
||||
request<McpConfig>('/config/mcp', { method: 'PATCH', body: JSON.stringify(data) }),
|
||||
}
|
||||
|
||||
export const atxApi = {
|
||||
@@ -884,13 +847,6 @@ export type {
|
||||
StreamMode,
|
||||
EncoderType,
|
||||
BitratePreset,
|
||||
ComputerUseButton,
|
||||
ComputerUseConfigResponse,
|
||||
ComputerUseConfigUpdate,
|
||||
ComputerUseScreenshot,
|
||||
ComputerUseSessionStatus,
|
||||
ComputerUseSessionSummary,
|
||||
ComputerUseStartRequest,
|
||||
} from '@/types/generated'
|
||||
|
||||
export const audioApi = {
|
||||
|
||||
@@ -38,7 +38,6 @@ import {
|
||||
BarChart3,
|
||||
Terminal,
|
||||
MoreHorizontal,
|
||||
Bot,
|
||||
ChevronUp,
|
||||
ChevronDown,
|
||||
Keyboard,
|
||||
@@ -73,7 +72,6 @@ const props = defineProps<{
|
||||
showPower?: boolean
|
||||
atxEnabled?: boolean
|
||||
showTerminal?: boolean
|
||||
showComputerUse?: boolean
|
||||
showPasteText?: boolean
|
||||
showMic?: boolean
|
||||
scaleMode?: VideoScaleMode
|
||||
@@ -111,7 +109,6 @@ const emit = defineEmits<{
|
||||
(e: 'reset'): void
|
||||
(e: 'wol', macAddress: string): void
|
||||
(e: 'openTerminal'): void
|
||||
(e: 'openComputerUse'): void
|
||||
}>()
|
||||
|
||||
const pasteOpen = ref(false)
|
||||
@@ -166,7 +163,7 @@ let layoutResizeObserver: ResizeObserver | null = null
|
||||
|
||||
type CollapsibleItem =
|
||||
| 'msd' | 'atx' | 'paste'
|
||||
| 'stats' | 'terminal' | 'settings' | 'ai'
|
||||
| 'stats' | 'terminal' | 'settings'
|
||||
|
||||
interface ItemSpec {
|
||||
id: CollapsibleItem
|
||||
@@ -180,7 +177,6 @@ const ITEM_SPECS: ItemSpec[] = [
|
||||
{ id: 'stats', side: 'right' },
|
||||
{ id: 'terminal', side: 'right' },
|
||||
{ id: 'settings', side: 'right' },
|
||||
{ id: 'ai', side: 'right' },
|
||||
]
|
||||
|
||||
const measuredWidths = ref<Map<CollapsibleItem, { icon: number; label: number }>>(new Map())
|
||||
@@ -256,7 +252,7 @@ watch(locale, () => {
|
||||
void measureLayout()
|
||||
})
|
||||
|
||||
watch([() => props.layout, () => props.showComputerUse, floatingCollapsed, minimalDisplayControls], () => {
|
||||
watch([() => props.layout, floatingCollapsed, minimalDisplayControls], () => {
|
||||
// Layout changes can move this bar while its portal based controls are open.
|
||||
// Close every transient surface first so no old portal remains attached to
|
||||
// the document after the new layout is painted.
|
||||
@@ -296,7 +292,6 @@ const collapsibleItems = computed(() => {
|
||||
if (item.id === 'paste' && !showPasteText.value) return false
|
||||
if (item.id === 'stats' && !showStats.value) return false
|
||||
if (item.id === 'terminal' && props.showTerminal === false) return false
|
||||
if (item.id === 'ai' && props.showComputerUse === false) return false
|
||||
return true
|
||||
})
|
||||
return items
|
||||
@@ -309,7 +304,7 @@ const visibleSet = computed(() => {
|
||||
if (isSidebarLayout.value) {
|
||||
// Reserve More and a gap between the two groups before assigning vertical slots.
|
||||
let available = barHeight.value - coreHeight.value - fixedHeight.value - actionHeight.value - 16
|
||||
const priority: CollapsibleItem[] = ['paste', 'settings', 'msd', 'atx', 'stats', 'terminal', 'ai']
|
||||
const priority: CollapsibleItem[] = ['paste', 'settings', 'msd', 'atx', 'stats', 'terminal']
|
||||
for (const id of priority) {
|
||||
if (!collapsibleItems.value.some(item => item.id === id)) continue
|
||||
if (available < actionHeight.value) break
|
||||
@@ -567,26 +562,7 @@ const hasRightOverflow = computed(() => {
|
||||
</TooltipProvider>
|
||||
</div>
|
||||
|
||||
<!-- Computer Use - Optional -->
|
||||
<TooltipProvider v-if="isVisible('ai')">
|
||||
<Tooltip>
|
||||
<TooltipTrigger as-child>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
class="h-8 gap-1.5 text-xs"
|
||||
:aria-label="t('computerUse.title')"
|
||||
@click="emit('openComputerUse')"
|
||||
>
|
||||
<Bot class="size-3.5 sm:size-4" />
|
||||
<span v-if="visibleSet.get('ai') === 'label'">AI</span>
|
||||
</Button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>
|
||||
<p>{{ t('computerUse.title') }}</p>
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
</TooltipProvider>
|
||||
|
||||
|
||||
<!-- Settings - Adaptive -->
|
||||
<div v-if="isVisible('settings')">
|
||||
@@ -661,10 +637,6 @@ const hasRightOverflow = computed(() => {
|
||||
{{ t('actionbar.webTerminal') }}
|
||||
</DropdownMenuItem>
|
||||
|
||||
<DropdownMenuItem v-if="!isFloatingLayout && props.showComputerUse !== false && !isVisible('ai')" @click="openFromOverflow(() => emit('openComputerUse'))">
|
||||
<Bot class="size-4 mr-2" />{{ t('computerUse.title') }}
|
||||
</DropdownMenuItem>
|
||||
|
||||
<!-- Settings -->
|
||||
<DropdownMenuItem v-if="!isFloatingLayout && !isVisible('settings')" @click="openFromOverflow(() => router.push('/settings'))">
|
||||
<Settings class="size-4 mr-2" />
|
||||
@@ -752,8 +724,6 @@ const hasRightOverflow = computed(() => {
|
||||
<!-- Settings -->
|
||||
<Button data-measure="settings-icon" variant="ghost" size="sm" class="h-8 gap-1.5 text-xs"><Settings class="size-4" /></Button>
|
||||
<Button data-measure="settings-label" variant="ghost" size="sm" class="h-8 gap-1.5 text-xs"><Settings class="size-4" />{{ t('actionbar.settings') }}</Button>
|
||||
<Button data-measure="ai-icon" variant="ghost" size="sm" class="h-8 gap-1.5 text-xs"><Bot class="size-4" /></Button>
|
||||
<Button data-measure="ai-label" variant="ghost" size="sm" class="h-8 gap-1.5 text-xs"><Bot class="size-4" />AI</Button>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
@@ -1,460 +0,0 @@
|
||||
<script setup lang="ts">
|
||||
import { useConsoleAppearance } from "@/composables/useConsoleAppearance"
|
||||
import { computed, nextTick, onMounted, ref, watch } from 'vue'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
import {
|
||||
Bot,
|
||||
Brain,
|
||||
ChevronDown,
|
||||
ChevronRight,
|
||||
Image,
|
||||
KeyRound,
|
||||
SendHorizontal,
|
||||
Settings,
|
||||
Square,
|
||||
Trash2,
|
||||
Wifi,
|
||||
WifiOff,
|
||||
X,
|
||||
} from 'lucide-vue-next'
|
||||
import { toast } from 'vue-sonner'
|
||||
import { computerUseApi, type ComputerUseAction, type ComputerUseConfig, type ComputerUseSession } from '@/api'
|
||||
import { ApiError } from '@/api/request'
|
||||
import type { ComputerUseTimelineItem } from '@/types/computerUseTimeline'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import { Input } from '@/components/ui/input'
|
||||
import { Label } from '@/components/ui/label'
|
||||
import { Textarea } from '@/components/ui/textarea'
|
||||
import { Switch } from '@/components/ui/switch'
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogFooter,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
} from '@/components/ui/dialog'
|
||||
|
||||
const props = defineProps<{
|
||||
open: boolean
|
||||
connected: boolean
|
||||
wsError: string | null
|
||||
session: ComputerUseSession | null
|
||||
timeline: ComputerUseTimelineItem[]
|
||||
}>()
|
||||
|
||||
const { t } = useI18n()
|
||||
const consoleAppearance = useConsoleAppearance()
|
||||
|
||||
const emit = defineEmits<{
|
||||
(e: 'update:open', value: boolean): void
|
||||
(e: 'start', prompt: string): void
|
||||
(e: 'stop'): void
|
||||
(e: 'clear'): void
|
||||
}>()
|
||||
|
||||
const config = ref<ComputerUseConfig | null>(null)
|
||||
const prompt = ref('')
|
||||
const apiKey = ref('')
|
||||
const savingConfig = ref(false)
|
||||
const starting = ref(false)
|
||||
const settingsOpen = ref(false)
|
||||
const messagesRef = ref<HTMLDivElement | null>(null)
|
||||
const reasoningOverrides = ref<Record<string, boolean>>({})
|
||||
|
||||
const defaultModel = computed({
|
||||
get: () => config.value?.model ?? 'gpt-5.5',
|
||||
set: (value: string) => {
|
||||
if (config.value) config.value.model = value
|
||||
},
|
||||
})
|
||||
const defaultBaseUrl = computed({
|
||||
get: () => config.value?.base_url ?? 'https://api.openai.com/v1/responses',
|
||||
set: (value: string) => {
|
||||
if (config.value) config.value.base_url = value
|
||||
},
|
||||
})
|
||||
|
||||
const status = computed(() => props.session?.status ?? 'idle')
|
||||
const isRunning = computed(() => ['waiting_screenshot', 'thinking', 'executing'].includes(status.value))
|
||||
const canStart = computed(() => (
|
||||
!!config.value?.enabled
|
||||
&& !!config.value?.api_key_configured
|
||||
&& prompt.value.trim().length > 0
|
||||
&& !isRunning.value
|
||||
))
|
||||
|
||||
const statusLabel = computed(() => {
|
||||
switch (status.value) {
|
||||
case 'waiting_screenshot': return t('computerUse.status.waitingScreenshot')
|
||||
case 'thinking': return t('computerUse.status.thinking')
|
||||
case 'executing': return t('computerUse.status.executing')
|
||||
case 'completed': return t('computerUse.status.completed')
|
||||
case 'failed': return t('computerUse.status.failed')
|
||||
case 'stopped': return t('computerUse.status.stopped')
|
||||
default: return t('computerUse.status.idle')
|
||||
}
|
||||
})
|
||||
|
||||
const apiKeyPlaceholder = computed(() => {
|
||||
if (!config.value?.api_key_configured) return t('computerUse.settings.apiKey')
|
||||
const source = config.value.api_key_source
|
||||
const sourceLabel = source === 'env'
|
||||
? t('computerUse.settings.sourceEnv')
|
||||
: source === 'config'
|
||||
? t('computerUse.settings.sourceConfig')
|
||||
: t('computerUse.settings.sourceNone')
|
||||
return t('computerUse.settings.configured', { source: sourceLabel })
|
||||
})
|
||||
|
||||
async function loadConfig() {
|
||||
try {
|
||||
config.value = await computerUseApi.config()
|
||||
} catch (err) {
|
||||
// Auth failures are handled globally (redirect to login); don't toast over it.
|
||||
if (err instanceof ApiError && err.status === 401) return
|
||||
toast.error(t('computerUse.errors.configLoadFailed'), {
|
||||
description: err instanceof Error ? err.message : undefined,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async function saveConfig() {
|
||||
if (!config.value) return
|
||||
savingConfig.value = true
|
||||
try {
|
||||
config.value = await computerUseApi.updateConfig({
|
||||
enabled: config.value.enabled,
|
||||
base_url: config.value.base_url,
|
||||
model: config.value.model,
|
||||
api_key: apiKey.value.trim() || undefined,
|
||||
})
|
||||
apiKey.value = ''
|
||||
settingsOpen.value = false
|
||||
toast.success(t('computerUse.success.configSaved'))
|
||||
} finally {
|
||||
savingConfig.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function clearApiKey() {
|
||||
savingConfig.value = true
|
||||
try {
|
||||
config.value = await computerUseApi.updateConfig({ clear_api_key: true })
|
||||
apiKey.value = ''
|
||||
toast.success(t('computerUse.success.apiKeyCleared'))
|
||||
} finally {
|
||||
savingConfig.value = false
|
||||
}
|
||||
}
|
||||
|
||||
function start() {
|
||||
if (!canStart.value) return
|
||||
const text = prompt.value.trim()
|
||||
starting.value = true
|
||||
emit('start', text)
|
||||
prompt.value = ''
|
||||
starting.value = false
|
||||
}
|
||||
|
||||
function formatAction(action: ComputerUseAction): string {
|
||||
switch (action.type) {
|
||||
case 'click':
|
||||
return t('computerUse.actions.click', {
|
||||
x: action.x,
|
||||
y: action.y,
|
||||
button: t(`computerUse.mouseButtons.${action.button ?? 'left'}`),
|
||||
})
|
||||
case 'double_click':
|
||||
return t('computerUse.actions.doubleClick', {
|
||||
x: action.x,
|
||||
y: action.y,
|
||||
button: t(`computerUse.mouseButtons.${action.button ?? 'left'}`),
|
||||
})
|
||||
case 'move':
|
||||
return t('computerUse.actions.move', { x: action.x, y: action.y })
|
||||
case 'drag':
|
||||
return t('computerUse.actions.drag', { count: action.path.length })
|
||||
case 'scroll':
|
||||
return t('computerUse.actions.scroll', {
|
||||
x: action.x,
|
||||
y: action.y,
|
||||
dx: action.dx ?? 0,
|
||||
dy: action.dy ?? 0,
|
||||
})
|
||||
case 'type':
|
||||
return t('computerUse.actions.typeAscii', { count: action.text.length })
|
||||
case 'keypress':
|
||||
return t('computerUse.actions.keypress', { keys: action.keys.join('+') })
|
||||
case 'wait':
|
||||
return t('computerUse.actions.wait', { ms: action.ms })
|
||||
case 'screenshot':
|
||||
return t('computerUse.actions.screenshot')
|
||||
}
|
||||
}
|
||||
|
||||
function reasoningOpen(item: Extract<ComputerUseTimelineItem, { type: 'reasoning' }>) {
|
||||
return reasoningOverrides.value[item.id] ?? !item.completed
|
||||
}
|
||||
|
||||
function toggleReasoning(item: Extract<ComputerUseTimelineItem, { type: 'reasoning' }>) {
|
||||
reasoningOverrides.value[item.id] = !reasoningOpen(item)
|
||||
}
|
||||
|
||||
function scrollToBottom() {
|
||||
nextTick(() => {
|
||||
const el = messagesRef.value
|
||||
if (el) el.scrollTop = el.scrollHeight
|
||||
})
|
||||
}
|
||||
|
||||
watch(() => props.timeline, scrollToBottom, { deep: true })
|
||||
watch(() => props.open, (open) => {
|
||||
if (open) scrollToBottom()
|
||||
})
|
||||
|
||||
onMounted(loadConfig)
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<aside
|
||||
data-slot="computer-use-panel"
|
||||
:data-console-layout="consoleAppearance"
|
||||
v-show="open"
|
||||
class="absolute inset-y-0 right-0 z-30 h-full min-h-0 w-full border-l bg-background shadow-xl sm:w-[420px] md:relative md:z-auto xl:w-[460px]"
|
||||
>
|
||||
<div class="flex h-full min-h-0 flex-col">
|
||||
<header class="flex h-12 shrink-0 items-center gap-2 border-b px-3">
|
||||
<Bot class="size-5 shrink-0" />
|
||||
<div class="min-w-0 flex-1">
|
||||
<div class="truncate text-sm font-semibold">{{ t('computerUse.title') }}</div>
|
||||
<div
|
||||
class="flex min-w-0 items-center gap-1 text-[11px]"
|
||||
:class="connected ? 'text-emerald-600 dark:text-emerald-400' : 'text-muted-foreground'"
|
||||
>
|
||||
<Wifi v-if="connected" class="size-3 shrink-0" />
|
||||
<WifiOff v-else class="size-3 shrink-0" />
|
||||
<span class="truncate">
|
||||
{{ connected ? t('computerUse.connection.connected') : (wsError || t('computerUse.connection.disconnected')) }}
|
||||
</span>
|
||||
<span class="text-muted-foreground">· {{ statusLabel }}</span>
|
||||
</div>
|
||||
</div>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
class="size-8"
|
||||
:title="t('computerUse.buttons.settings')"
|
||||
:aria-label="t('computerUse.buttons.settings')"
|
||||
@click="settingsOpen = true"
|
||||
>
|
||||
<Settings class="size-4" />
|
||||
</Button>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
class="size-8"
|
||||
:title="t('computerUse.buttons.clear')"
|
||||
:aria-label="t('computerUse.buttons.clear')"
|
||||
:disabled="isRunning || timeline.length === 0"
|
||||
@click="emit('clear')"
|
||||
>
|
||||
<Trash2 class="size-4" />
|
||||
</Button>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
class="size-8"
|
||||
:title="t('computerUse.buttons.close')"
|
||||
:aria-label="t('computerUse.buttons.close')"
|
||||
@click="emit('update:open', false)"
|
||||
>
|
||||
<X class="size-4" />
|
||||
</Button>
|
||||
</header>
|
||||
|
||||
<div ref="messagesRef" class="min-h-0 flex-1 overflow-y-auto px-4 py-4">
|
||||
<div v-if="timeline.length === 0" class="flex h-full items-center justify-center text-muted-foreground/45">
|
||||
<Bot class="size-10" />
|
||||
</div>
|
||||
|
||||
<div v-else class="space-y-5">
|
||||
<template v-for="item in timeline" :key="item.id">
|
||||
<div v-if="item.type === 'user'" class="flex justify-end">
|
||||
<div class="max-w-[86%] whitespace-pre-wrap break-words rounded-md bg-muted px-3 py-2 text-sm">
|
||||
{{ item.text }}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div v-else-if="item.type === 'assistant'" class="whitespace-pre-wrap break-words text-sm leading-6">
|
||||
{{ item.text }}
|
||||
</div>
|
||||
|
||||
<div v-else-if="item.type === 'reasoning'" class="border-l-2 border-muted pl-3 text-xs text-muted-foreground">
|
||||
<button
|
||||
type="button"
|
||||
class="flex w-full items-center gap-1.5 py-1 text-left font-medium hover:text-foreground"
|
||||
@click="toggleReasoning(item)"
|
||||
>
|
||||
<ChevronDown v-if="reasoningOpen(item)" class="size-3.5 shrink-0" />
|
||||
<ChevronRight v-else class="size-3.5 shrink-0" />
|
||||
<Brain class="size-3.5 shrink-0" />
|
||||
<span>
|
||||
{{ item.failed
|
||||
? t('computerUse.reasoning.failed')
|
||||
: (item.completed ? t('computerUse.reasoning.process') : t('computerUse.reasoning.thinking')) }}
|
||||
</span>
|
||||
</button>
|
||||
<div
|
||||
v-if="reasoningOpen(item)"
|
||||
class="max-h-72 overflow-y-auto whitespace-pre-wrap break-words pb-2 pt-1 leading-5"
|
||||
>
|
||||
{{ item.text || t('computerUse.reasoning.thinking') }}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div v-else-if="item.type === 'screenshot'" class="overflow-hidden rounded-md border bg-card">
|
||||
<div class="flex items-center justify-between border-b px-2.5 py-1.5 text-[11px] text-muted-foreground">
|
||||
<span class="inline-flex items-center gap-1.5">
|
||||
<Image class="size-3.5" />{{ t('computerUse.trace.screenshot') }}
|
||||
</span>
|
||||
<span>{{ item.screenshot.width }}x{{ item.screenshot.height }}</span>
|
||||
</div>
|
||||
<div
|
||||
class="w-full bg-black"
|
||||
:style="{ aspectRatio: `${item.screenshot.width} / ${item.screenshot.height}` }"
|
||||
>
|
||||
<img
|
||||
:src="item.screenshot.data_url"
|
||||
class="h-full w-full object-contain"
|
||||
:alt="t('computerUse.trace.screenshotAlt')"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div v-else-if="item.type === 'actions_executed'" class="border-l-2 border-emerald-500/50 pl-3 text-xs">
|
||||
<div class="mb-1 text-[11px] font-medium text-emerald-600 dark:text-emerald-400">
|
||||
{{ t('computerUse.trace.executed') }}
|
||||
</div>
|
||||
<div class="space-y-1 text-muted-foreground">
|
||||
<div v-for="(action, index) in item.actions" :key="index" class="break-words">
|
||||
{{ formatAction(action) }}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div
|
||||
v-else-if="item.type === 'error'"
|
||||
class="whitespace-pre-wrap break-words rounded-md border border-destructive/40 bg-destructive/10 px-3 py-2 text-xs leading-5 text-destructive"
|
||||
>
|
||||
{{ item.text }}
|
||||
</div>
|
||||
|
||||
<div v-else class="text-center text-xs text-muted-foreground">
|
||||
{{ item.text }}
|
||||
</div>
|
||||
</template>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<footer class="shrink-0 border-t bg-background p-3">
|
||||
<div class="relative">
|
||||
<Textarea
|
||||
v-model="prompt"
|
||||
rows="3"
|
||||
class="max-h-40 min-h-20 resize-none pr-12"
|
||||
:placeholder="t('computerUse.input.placeholder')"
|
||||
:disabled="isRunning"
|
||||
@keydown.meta.enter.prevent="start"
|
||||
@keydown.ctrl.enter.prevent="start"
|
||||
/>
|
||||
<Button
|
||||
v-if="!isRunning"
|
||||
size="icon"
|
||||
class="absolute bottom-2 right-2 size-8"
|
||||
:title="t('computerUse.buttons.send')"
|
||||
:aria-label="t('computerUse.buttons.send')"
|
||||
:disabled="!canStart || starting"
|
||||
@click="start"
|
||||
>
|
||||
<SendHorizontal class="size-4" />
|
||||
</Button>
|
||||
<Button
|
||||
v-else
|
||||
size="icon"
|
||||
variant="destructive"
|
||||
class="absolute bottom-2 right-2 size-8"
|
||||
:title="t('computerUse.buttons.stop')"
|
||||
:aria-label="t('computerUse.buttons.stop')"
|
||||
@click="emit('stop')"
|
||||
>
|
||||
<Square class="size-3.5 fill-current" />
|
||||
</Button>
|
||||
</div>
|
||||
<p v-if="config && !config.api_key_configured" class="mt-2 text-xs text-muted-foreground">
|
||||
{{ t('computerUse.input.apiKeyRequired') }}
|
||||
</p>
|
||||
</footer>
|
||||
</div>
|
||||
|
||||
<Dialog v-model:open="settingsOpen">
|
||||
<DialogContent class="max-w-md">
|
||||
<DialogHeader>
|
||||
<DialogTitle>{{ t('computerUse.settings.title') }}</DialogTitle>
|
||||
</DialogHeader>
|
||||
|
||||
<div class="space-y-4 py-2">
|
||||
<div class="flex items-center justify-between gap-4">
|
||||
<Label for="cua-enabled">{{ t('computerUse.settings.enableAi') }}</Label>
|
||||
<Switch
|
||||
id="cua-enabled"
|
||||
:model-value="config?.enabled ?? false"
|
||||
@update:model-value="(value) => { if (config) config.enabled = value }"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div class="space-y-1.5">
|
||||
<Label for="cua-model">{{ t('computerUse.settings.model') }}</Label>
|
||||
<Input id="cua-model" v-model="defaultModel" :disabled="!config" placeholder="gpt-5.5" />
|
||||
</div>
|
||||
|
||||
<div class="space-y-1.5">
|
||||
<Label for="cua-url">{{ t('computerUse.settings.apiUrl') }}</Label>
|
||||
<Input
|
||||
id="cua-url"
|
||||
v-model="defaultBaseUrl"
|
||||
:disabled="!config"
|
||||
placeholder="https://api.example.com/v1/responses"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div class="space-y-1.5">
|
||||
<Label for="cua-key" class="flex items-center gap-1.5">
|
||||
<KeyRound class="size-3.5" />
|
||||
{{ t('computerUse.settings.apiKey') }}
|
||||
</Label>
|
||||
<Input
|
||||
id="cua-key"
|
||||
v-model="apiKey"
|
||||
type="password"
|
||||
autocomplete="off"
|
||||
:placeholder="apiKeyPlaceholder"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<DialogFooter class="gap-2 sm:justify-between">
|
||||
<Button
|
||||
variant="outline"
|
||||
:disabled="savingConfig || !config?.api_key_configured"
|
||||
@click="clearApiKey"
|
||||
>
|
||||
{{ t('computerUse.buttons.clearKey') }}
|
||||
</Button>
|
||||
<Button :disabled="savingConfig || !config" @click="saveConfig">
|
||||
{{ t('computerUse.buttons.save') }}
|
||||
</Button>
|
||||
</DialogFooter>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
</aside>
|
||||
</template>
|
||||
@@ -2,7 +2,7 @@
|
||||
import { ref } from 'vue'
|
||||
import { useRouter } from 'vue-router'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
import { BarChart3, Bot, Settings, Terminal, MoreHorizontal } from 'lucide-vue-next'
|
||||
import { BarChart3, Settings, Terminal, MoreHorizontal } from 'lucide-vue-next'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuTrigger } from '@/components/ui/dropdown-menu'
|
||||
|
||||
@@ -10,12 +10,10 @@ defineProps<{
|
||||
showStats: boolean
|
||||
showTerminal: boolean
|
||||
terminalRunning: boolean
|
||||
showComputerUse: boolean
|
||||
}>()
|
||||
const emit = defineEmits<{
|
||||
(e: 'openStats'): void
|
||||
(e: 'openTerminal'): void
|
||||
(e: 'openComputerUse'): void
|
||||
}>()
|
||||
const router = useRouter()
|
||||
const { t } = useI18n()
|
||||
@@ -35,18 +33,14 @@ function openFromMenu(action: () => void) {
|
||||
<Button v-if="showTerminal" variant="ghost" size="sm" class="gap-1.5 rounded-lg text-xs" :aria-label="t('actionbar.webTerminal')" :title="t('actionbar.webTerminal')" :disabled="!terminalRunning" @click="emit('openTerminal')">
|
||||
<Terminal class="size-4" /><span class="hidden lg:inline">{{ t('actionbar.webTerminal') }}</span>
|
||||
</Button>
|
||||
<Button v-if="showComputerUse" variant="ghost" size="sm" class="gap-1.5 rounded-lg text-xs" :aria-label="t('computerUse.title')" :title="t('computerUse.title')" @click="emit('openComputerUse')">
|
||||
<Bot class="size-4" /><span class="hidden lg:inline">AI</span>
|
||||
</Button>
|
||||
</div>
|
||||
<DropdownMenu v-if="showStats || showTerminal || showComputerUse" v-model:open="menuOpen">
|
||||
<DropdownMenu v-if="showStats || showTerminal" v-model:open="menuOpen">
|
||||
<DropdownMenuTrigger as-child class="md:hidden">
|
||||
<Button variant="ghost" size="icon-sm" :aria-label="t('actionbar.more')"><MoreHorizontal class="size-4" /></Button>
|
||||
</DropdownMenuTrigger>
|
||||
<DropdownMenuContent align="end" class="w-48">
|
||||
<DropdownMenuItem v-if="showStats" @select="openFromMenu(() => emit('openStats'))"><BarChart3 class="size-4" />{{ t('actionbar.stats') }}</DropdownMenuItem>
|
||||
<DropdownMenuItem v-if="showTerminal" :disabled="!terminalRunning" @select="openFromMenu(() => emit('openTerminal'))"><Terminal class="size-4" />{{ t('actionbar.webTerminal') }}</DropdownMenuItem>
|
||||
<DropdownMenuItem v-if="showComputerUse" @select="openFromMenu(() => emit('openComputerUse'))"><Bot class="size-4" />{{ t('computerUse.title') }}</DropdownMenuItem>
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
<Button variant="ghost" size="sm" class="gap-1.5 rounded-lg px-2 text-xs" :aria-label="t('actionbar.settings')" :title="t('actionbar.settings')" @click="router.push('/settings')">
|
||||
|
||||
93
web/src/components/McpSettingsCard.vue
Normal file
93
web/src/components/McpSettingsCard.vue
Normal file
@@ -0,0 +1,93 @@
|
||||
<script setup lang="ts">
|
||||
import { computed, ref, onMounted } from 'vue'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
import { toast } from 'vue-sonner'
|
||||
import { Copy } from 'lucide-vue-next'
|
||||
import { mcpApi, type McpConfig } from '@/api'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import { Input } from '@/components/ui/input'
|
||||
import { Label } from '@/components/ui/label'
|
||||
import { Switch } from '@/components/ui/switch'
|
||||
import { Card, CardContent, CardHeader, CardTitle, CardDescription } from '@/components/ui/card'
|
||||
import { useClipboard } from '@/composables/useClipboard'
|
||||
|
||||
const clipboard = useClipboard()
|
||||
const { t } = useI18n()
|
||||
const config = ref<McpConfig | null>(null)
|
||||
const enabled = ref(false)
|
||||
const rate = ref(5)
|
||||
const apiKey = ref('')
|
||||
const busy = ref(false)
|
||||
const endpoint = computed(() => new URL(config.value?.endpoint ?? '/mcp', window.location.origin).href)
|
||||
|
||||
function apply(value: McpConfig) {
|
||||
config.value = value
|
||||
enabled.value = value.enabled
|
||||
rate.value = value.requests_per_second
|
||||
apiKey.value = value.api_key ?? ''
|
||||
}
|
||||
async function load() {
|
||||
busy.value = true
|
||||
config.value = null
|
||||
try { apply(await mcpApi.config()) }
|
||||
catch (error) { toast.error(error instanceof Error ? error.message : t('mcp.loadFailed')) }
|
||||
finally { busy.value = false }
|
||||
}
|
||||
async function save(regenerate = false) {
|
||||
if (!Number.isInteger(rate.value) || rate.value < 1 || rate.value > 100) {
|
||||
toast.error(t('mcp.invalidRate'))
|
||||
return
|
||||
}
|
||||
busy.value = true
|
||||
try {
|
||||
apply(await mcpApi.updateConfig({ enabled: enabled.value, requests_per_second: rate.value, regenerate_api_key: regenerate }))
|
||||
toast.success(t('mcp.saved'))
|
||||
} catch (error) { toast.error(error instanceof Error ? error.message : t('mcp.saveFailed')) }
|
||||
finally { busy.value = false }
|
||||
}
|
||||
async function copy(value: string) {
|
||||
if (await clipboard.copy(value)) toast.success(t('mcp.copied'))
|
||||
else toast.error(t('mcp.copyFailed'))
|
||||
}
|
||||
onMounted(load)
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>{{ t('mcp.title') }}</CardTitle>
|
||||
<CardDescription>{{ t('mcp.description') }}</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent class="space-y-5">
|
||||
<div class="flex items-center justify-between gap-4">
|
||||
<Label for="mcp-enabled">{{ t('mcp.enabled') }}</Label>
|
||||
<Switch id="mcp-enabled" v-model="enabled" :disabled="busy || !config" />
|
||||
</div>
|
||||
<div class="space-y-2">
|
||||
<Label for="mcp-rate">{{ t('mcp.rate') }}</Label>
|
||||
<Input id="mcp-rate" v-model.number="rate" type="number" min="1" max="100" step="1" :disabled="busy || !config" />
|
||||
</div>
|
||||
<div class="space-y-2">
|
||||
<Label>{{ t('mcp.endpoint') }}</Label>
|
||||
<div class="flex items-center gap-2">
|
||||
<code id="mcp-endpoint" class="min-w-0 flex-1 break-all rounded bg-muted px-3 py-2 font-mono text-sm">{{ endpoint }}</code>
|
||||
<Button variant="ghost" size="icon" :aria-label="t('mcp.copyEndpoint')" @click="copy(endpoint)"><Copy class="size-4" /></Button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="space-y-2">
|
||||
<Label>API Key</Label>
|
||||
<div v-if="apiKey" class="flex items-center gap-2">
|
||||
<code id="mcp-key" class="min-w-0 flex-1 break-all rounded bg-muted px-3 py-2 font-mono text-sm">{{ apiKey }}</code>
|
||||
<Button variant="ghost" size="icon" :aria-label="t('mcp.copyKey')" @click="copy(apiKey)"><Copy class="size-4" /></Button>
|
||||
</div>
|
||||
<p v-if="!apiKey" class="text-sm text-muted-foreground">{{ config?.api_key_configured ? t('mcp.keyConfigured') : t('mcp.keyMissing') }}</p>
|
||||
<Button variant="outline" :disabled="busy || !config" @click="save(true)">{{ t('mcp.regenerate') }}</Button>
|
||||
</div>
|
||||
<p class="text-sm text-muted-foreground">{{ t('mcp.connectHint') }}</p>
|
||||
<div class="flex justify-end gap-2">
|
||||
<Button v-if="!config && !busy" variant="outline" @click="load">{{ t('mcp.retry') }}</Button>
|
||||
<Button :disabled="busy || !config" @click="save()">{{ t('mcp.save') }}</Button>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
</template>
|
||||
@@ -1,108 +0,0 @@
|
||||
import { ref, onUnmounted } from 'vue'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
import { buildWsUrl } from '@/types/websocket'
|
||||
import { generateUUID } from '@/lib/utils'
|
||||
import type { ComputerUseScreenshot, ComputerUseSession, ComputerUseAction } from '@/api'
|
||||
|
||||
export type ComputerUseServerMessage =
|
||||
| { type: 'session_updated'; session: ComputerUseSession }
|
||||
| { type: 'screenshot_requested'; request_id: string }
|
||||
| { type: 'screenshot_captured'; screenshot: ComputerUseScreenshot }
|
||||
| { type: 'step_started'; step: number }
|
||||
| { type: 'reasoning_delta'; delta: string }
|
||||
| { type: 'reasoning_completed'; failed: boolean }
|
||||
| { type: 'actions_executed'; actions: ComputerUseAction[] }
|
||||
| { type: 'error'; message: string }
|
||||
|
||||
export function useComputerUseSocket(options: {
|
||||
onMessage: (message: ComputerUseServerMessage) => void
|
||||
onScreenshotRequested: (requestId: string) => Promise<ComputerUseScreenshot | null>
|
||||
}) {
|
||||
const { t } = useI18n()
|
||||
const connected = ref(false)
|
||||
const error = ref<string | null>(null)
|
||||
const clientId = generateUUID()
|
||||
let ws: WebSocket | null = null
|
||||
let connectPromise: Promise<void> | null = null
|
||||
|
||||
function connect(): Promise<void> {
|
||||
if (ws && ws.readyState === WebSocket.OPEN) return Promise.resolve()
|
||||
if (connectPromise) return connectPromise
|
||||
|
||||
ws = new WebSocket(buildWsUrl(`/api/ws/computer-use?client_id=${encodeURIComponent(clientId)}`))
|
||||
|
||||
connectPromise = new Promise((resolve, reject) => {
|
||||
if (!ws) {
|
||||
reject(new Error(t('computerUse.connection.socketFailed')))
|
||||
return
|
||||
}
|
||||
|
||||
ws.onopen = () => {
|
||||
connected.value = true
|
||||
error.value = null
|
||||
connectPromise = null
|
||||
resolve()
|
||||
}
|
||||
|
||||
ws.onerror = () => {
|
||||
error.value = t('computerUse.connection.socketFailed')
|
||||
connectPromise = null
|
||||
reject(new Error(error.value))
|
||||
}
|
||||
})
|
||||
|
||||
ws.onclose = () => {
|
||||
connected.value = false
|
||||
connectPromise = null
|
||||
}
|
||||
|
||||
ws.onmessage = async (event) => {
|
||||
try {
|
||||
const message = JSON.parse(event.data) as ComputerUseServerMessage
|
||||
options.onMessage(message)
|
||||
if (message.type === 'screenshot_requested') {
|
||||
try {
|
||||
const screenshot = await options.onScreenshotRequested(message.request_id)
|
||||
if (!screenshot) throw new Error(t('computerUse.errors.captureUnavailable'))
|
||||
if (ws?.readyState === WebSocket.OPEN) {
|
||||
ws.send(JSON.stringify({
|
||||
type: 'screenshot_result',
|
||||
request_id: message.request_id,
|
||||
screenshot,
|
||||
}))
|
||||
}
|
||||
} catch (err) {
|
||||
if (ws?.readyState === WebSocket.OPEN) {
|
||||
ws.send(JSON.stringify({
|
||||
type: 'screenshot_error',
|
||||
request_id: message.request_id,
|
||||
message: err instanceof Error ? err.message : t('computerUse.errors.screenshotFailed'),
|
||||
}))
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`[ComputerUse] ${t('computerUse.connection.messageFailed')}:`, err)
|
||||
}
|
||||
}
|
||||
|
||||
return connectPromise
|
||||
}
|
||||
|
||||
function disconnect() {
|
||||
ws?.close()
|
||||
ws = null
|
||||
connected.value = false
|
||||
connectPromise = null
|
||||
}
|
||||
|
||||
onUnmounted(disconnect)
|
||||
|
||||
return {
|
||||
connected,
|
||||
error,
|
||||
clientId,
|
||||
connect,
|
||||
disconnect,
|
||||
}
|
||||
}
|
||||
@@ -1,13 +1,12 @@
|
||||
import { useLocalStorage } from '@vueuse/core'
|
||||
import type { RemovableRef } from '@vueuse/core'
|
||||
|
||||
export type FeatureVisibilityKey = 'power' | 'webTerminal' | 'computerUse' | 'pasteText'
|
||||
export type FeatureVisibilityKey = 'power' | 'webTerminal' | 'pasteText'
|
||||
export type FeatureVisibility = Record<FeatureVisibilityKey, boolean>
|
||||
|
||||
const DEFAULT_FEATURE_VISIBILITY: FeatureVisibility = {
|
||||
power: true,
|
||||
webTerminal: true,
|
||||
computerUse: true,
|
||||
pasteText: true,
|
||||
}
|
||||
|
||||
|
||||
@@ -800,7 +800,6 @@ export default {
|
||||
languageDesc: 'Choose the interface display language',
|
||||
featureVisibility: 'Feature Visibility',
|
||||
featureVisibilityDesc: 'Control which feature entry points are shown on the console page',
|
||||
computerUseAgent: 'Computer Use Agent',
|
||||
pasteText: 'Paste Text',
|
||||
videoSettings: 'Video Capture',
|
||||
videoSettingsDesc: 'Select a capture device. Source-following inputs report their active mode automatically.',
|
||||
@@ -983,85 +982,26 @@ export default {
|
||||
loadDevicesFailed: 'Failed to load device list',
|
||||
updateFailed: 'Update failed',
|
||||
},
|
||||
computerUse: {
|
||||
title: 'Computer Use',
|
||||
status: {
|
||||
waitingScreenshot: 'Capturing screen',
|
||||
thinking: 'Thinking',
|
||||
executing: 'Executing',
|
||||
completed: 'Completed',
|
||||
failed: 'Failed',
|
||||
stopped: 'Stopped',
|
||||
idle: 'Idle',
|
||||
},
|
||||
connection: {
|
||||
connected: 'Connected',
|
||||
disconnected: 'Disconnected',
|
||||
socketFailed: 'Computer Use WebSocket connection failed',
|
||||
messageFailed: 'Failed to process Computer Use WebSocket message',
|
||||
},
|
||||
buttons: {
|
||||
settings: 'Settings',
|
||||
clear: 'Clear',
|
||||
close: 'Close',
|
||||
send: 'Send',
|
||||
stop: 'Stop',
|
||||
save: 'Save',
|
||||
clearKey: 'Clear Key',
|
||||
},
|
||||
reasoning: {
|
||||
failed: 'Thinking failed',
|
||||
process: 'Reasoning',
|
||||
thinking: 'Thinking',
|
||||
},
|
||||
trace: {
|
||||
screenshot: 'Screenshot',
|
||||
screenshotAlt: 'Computer Use screenshot',
|
||||
executed: 'Executed',
|
||||
},
|
||||
input: {
|
||||
placeholder: 'Enter a task. Press Enter for a new line; press Ctrl+Enter to send.',
|
||||
apiKeyRequired: 'Configure an API Key first.',
|
||||
},
|
||||
settings: {
|
||||
title: 'Computer Use Settings',
|
||||
enableAi: 'Enable AI control',
|
||||
model: 'Model',
|
||||
apiUrl: 'API URL',
|
||||
apiKey: 'API Key',
|
||||
configured: 'Configured: {source}',
|
||||
sourceEnv: 'environment variable',
|
||||
sourceConfig: 'configuration file',
|
||||
sourceNone: 'not configured',
|
||||
},
|
||||
actions: {
|
||||
click: 'Click ({x}, {y}) {button}',
|
||||
doubleClick: 'Double-click ({x}, {y}) {button}',
|
||||
move: 'Move to ({x}, {y})',
|
||||
drag: 'Drag through {count} points',
|
||||
scroll: 'Scroll at ({x}, {y}) dx={dx} dy={dy}',
|
||||
typeAscii: 'Type {count} ASCII characters',
|
||||
keypress: 'Press {keys}',
|
||||
wait: 'Wait {ms}ms',
|
||||
screenshot: 'Request screenshot',
|
||||
},
|
||||
mouseButtons: {
|
||||
left: 'left button',
|
||||
middle: 'middle button',
|
||||
right: 'right button',
|
||||
},
|
||||
errors: {
|
||||
configLoadFailed: 'Failed to load Computer Use settings',
|
||||
taskFailed: 'Computer Use task failed',
|
||||
startFailed: 'Failed to start Computer Use',
|
||||
stopFailed: 'Failed to stop Computer Use',
|
||||
captureUnavailable: 'The current video frame is unavailable',
|
||||
screenshotFailed: 'Screenshot capture failed',
|
||||
},
|
||||
success: {
|
||||
configSaved: 'Computer Use settings saved',
|
||||
apiKeyCleared: 'API Key cleared',
|
||||
},
|
||||
mcp: {
|
||||
title: "MCP Service",
|
||||
description: "Let external AI tools view the screen and control input, power, and virtual media.",
|
||||
enabled: "Enable MCP",
|
||||
rate: "Global request rate (1–100 per second)",
|
||||
endpoint: "Service URL",
|
||||
copyEndpoint: "Copy service URL",
|
||||
copyKey: "Copy API key",
|
||||
copied: "Copied",
|
||||
copyFailed: "Copy failed. Select and copy the text manually.",
|
||||
keyConfigured: "A key is configured. Generate a new key to view it.",
|
||||
keyMissing: "A key is generated automatically when enabled.",
|
||||
regenerate: "Generate new key",
|
||||
connectHint: "Configure the service URL and Authorization: Bearer <API Key> in your AI tool.",
|
||||
save: "Save",
|
||||
saved: "MCP settings saved",
|
||||
loadFailed: "Failed to load MCP settings",
|
||||
saveFailed: "Failed to save MCP settings",
|
||||
invalidRate: "Request rate must be an integer from 1 to 100",
|
||||
retry: "Retry",
|
||||
},
|
||||
statusCard: {
|
||||
connectionDetails: 'Connection details',
|
||||
|
||||
@@ -799,7 +799,6 @@ export default {
|
||||
languageDesc: '选择界面显示语言',
|
||||
featureVisibility: '功能展示',
|
||||
featureVisibilityDesc: '控制控制台页面显示的功能入口',
|
||||
computerUseAgent: 'Computer Use Agent',
|
||||
pasteText: '粘贴文本',
|
||||
videoSettings: '视频采集',
|
||||
videoSettingsDesc: '选择视频采集设备；输入跟随型设备会自动显示当前输入模式',
|
||||
@@ -982,85 +981,26 @@ export default {
|
||||
loadDevicesFailed: '加载设备列表失败',
|
||||
updateFailed: '更新失败',
|
||||
},
|
||||
computerUse: {
|
||||
title: 'Computer Use',
|
||||
status: {
|
||||
waitingScreenshot: '截屏中',
|
||||
thinking: '思考中',
|
||||
executing: '执行中',
|
||||
completed: '已完成',
|
||||
failed: '失败',
|
||||
stopped: '已停止',
|
||||
idle: '空闲',
|
||||
},
|
||||
connection: {
|
||||
connected: '已连接',
|
||||
disconnected: '未连接',
|
||||
socketFailed: 'Computer Use WebSocket 连接失败',
|
||||
messageFailed: '处理 Computer Use WebSocket 消息失败',
|
||||
},
|
||||
buttons: {
|
||||
settings: '设置',
|
||||
clear: '清空',
|
||||
close: '关闭',
|
||||
send: '发送',
|
||||
stop: '停止',
|
||||
save: '保存',
|
||||
clearKey: '清除 Key',
|
||||
},
|
||||
reasoning: {
|
||||
failed: '思考失败',
|
||||
process: '思考过程',
|
||||
thinking: '思考中',
|
||||
},
|
||||
trace: {
|
||||
screenshot: '截图',
|
||||
screenshotAlt: 'Computer Use 截图',
|
||||
executed: '已执行',
|
||||
},
|
||||
input: {
|
||||
placeholder: '输入任务，按下回车换行，按下 Ctrl+Enter 发送',
|
||||
apiKeyRequired: '请先配置 API Key。',
|
||||
},
|
||||
settings: {
|
||||
title: 'Computer Use 设置',
|
||||
enableAi: '启用 AI 操作',
|
||||
model: '模型',
|
||||
apiUrl: 'API URL',
|
||||
apiKey: 'API Key',
|
||||
configured: '已配置：{source}',
|
||||
sourceEnv: '环境变量',
|
||||
sourceConfig: '配置文件',
|
||||
sourceNone: '未配置',
|
||||
},
|
||||
actions: {
|
||||
click: '点击 ({x}, {y}) {button}',
|
||||
doubleClick: '双击 ({x}, {y}) {button}',
|
||||
move: '移动到 ({x}, {y})',
|
||||
drag: '拖拽 {count} 个点',
|
||||
scroll: '滚动 ({x}, {y}) dx={dx} dy={dy}',
|
||||
typeAscii: '输入 {count} 个 ASCII 字符',
|
||||
keypress: '按键 {keys}',
|
||||
wait: '等待 {ms}ms',
|
||||
screenshot: '请求截图',
|
||||
},
|
||||
mouseButtons: {
|
||||
left: '左键',
|
||||
middle: '中键',
|
||||
right: '右键',
|
||||
},
|
||||
errors: {
|
||||
configLoadFailed: '无法加载 Computer Use 配置',
|
||||
taskFailed: 'Computer Use 任务失败',
|
||||
startFailed: 'Computer Use 启动失败',
|
||||
stopFailed: 'Computer Use 停止失败',
|
||||
captureUnavailable: '当前视频画面尚不可用',
|
||||
screenshotFailed: '截图失败',
|
||||
},
|
||||
success: {
|
||||
configSaved: 'Computer Use 配置已保存',
|
||||
apiKeyCleared: 'API Key 已清除',
|
||||
},
|
||||
mcp: {
|
||||
title: "MCP 服务",
|
||||
description: "让外部 AI 工具查看画面，操作键鼠、电源和虚拟介质。",
|
||||
enabled: "启用 MCP",
|
||||
rate: "全局请求频率（每秒 1–100 次）",
|
||||
endpoint: "服务地址",
|
||||
copyEndpoint: "复制服务地址",
|
||||
copyKey: "复制 API Key",
|
||||
copied: "已复制",
|
||||
copyFailed: "复制失败，请手动选择复制",
|
||||
keyConfigured: "已配置 Key，可重新生成后查看。",
|
||||
keyMissing: "启用时会自动生成 Key。",
|
||||
regenerate: "重新生成 Key",
|
||||
connectHint: "在 AI 工具中填写服务地址，并设置 Authorization: Bearer <API Key>。",
|
||||
save: "保存",
|
||||
saved: "MCP 设置已保存",
|
||||
loadFailed: "无法加载 MCP 设置",
|
||||
saveFailed: "无法保存 MCP 设置",
|
||||
invalidRate: "请求频率必须是 1–100 的整数",
|
||||
retry: "重试",
|
||||
},
|
||||
statusCard: {
|
||||
connectionDetails: '连接详情',
|
||||
|
||||
@@ -254,10 +254,3 @@ body {
|
||||
min-height: 44px;
|
||||
}
|
||||
}
|
||||
|
||||
[data-console-layout][data-slot="computer-use-panel"] {
|
||||
overflow: hidden;
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--console-surface-radius);
|
||||
box-shadow: var(--console-surface-shadow);
|
||||
}
|
||||
|
||||
@@ -1,16 +0,0 @@
|
||||
import type { ComputerUseAction, ComputerUseScreenshot } from '@/api'
|
||||
|
||||
export type ComputerUseTimelineItem =
|
||||
| { id: string; type: 'user'; text: string }
|
||||
| { id: string; type: 'assistant'; text: string }
|
||||
| { id: string; type: 'reasoning'; text: string; completed: boolean; failed: boolean }
|
||||
| { id: string; type: 'screenshot'; screenshot: ComputerUseScreenshot }
|
||||
| { id: string; type: 'actions_executed'; actions: ComputerUseAction[] }
|
||||
| { id: string; type: 'error'; text: string }
|
||||
| { id: string; type: 'status'; text: string }
|
||||
|
||||
export type NewComputerUseTimelineItem = ComputerUseTimelineItem extends infer Item
|
||||
? Item extends { id: string }
|
||||
? Omit<Item, 'id'>
|
||||
: never
|
||||
: never
|
||||
@@ -185,12 +185,6 @@ export interface WebConfig {
|
||||
ssl_key_path?: string;
|
||||
}
|
||||
|
||||
export interface ComputerUseConfig {
|
||||
enabled: boolean;
|
||||
base_url: string;
|
||||
model: string;
|
||||
}
|
||||
|
||||
export interface TtydConfig {
|
||||
enabled: boolean;
|
||||
shell: string;
|
||||
@@ -329,7 +323,7 @@ export interface AppConfig {
|
||||
audio: AudioConfig;
|
||||
stream: StreamConfig;
|
||||
web: WebConfig;
|
||||
computer_use: ComputerUseConfig;
|
||||
mcp: McpConfig;
|
||||
extensions: ExtensionsConfig;
|
||||
rustdesk: RustDeskConfig;
|
||||
vnc: VncConfig;
|
||||
@@ -405,58 +399,6 @@ export interface Ch9329DescriptorState {
|
||||
config_mode_available: boolean;
|
||||
}
|
||||
|
||||
export interface ComputerUseConfigResponse {
|
||||
enabled: boolean;
|
||||
base_url: string;
|
||||
model: string;
|
||||
api_key_configured: boolean;
|
||||
api_key_source: string;
|
||||
}
|
||||
|
||||
export interface ComputerUseConfigUpdate {
|
||||
enabled?: boolean;
|
||||
base_url?: string;
|
||||
model?: string;
|
||||
api_key?: string;
|
||||
clear_api_key?: boolean;
|
||||
}
|
||||
|
||||
export interface ComputerUsePoint {
|
||||
x: number;
|
||||
y: number;
|
||||
}
|
||||
|
||||
export interface ComputerUseScreenshot {
|
||||
data_url: string;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export enum ComputerUseSessionStatus {
|
||||
Idle = "idle",
|
||||
WaitingScreenshot = "waiting_screenshot",
|
||||
Thinking = "thinking",
|
||||
Executing = "executing",
|
||||
Completed = "completed",
|
||||
Failed = "failed",
|
||||
Stopped = "stopped",
|
||||
}
|
||||
|
||||
export interface ComputerUseSessionSummary {
|
||||
id?: string;
|
||||
status: ComputerUseSessionStatus;
|
||||
prompt?: string;
|
||||
step: number;
|
||||
last_error?: string;
|
||||
final_message?: string;
|
||||
}
|
||||
|
||||
export interface ComputerUseStartRequest {
|
||||
prompt: string;
|
||||
continue_conversation?: boolean;
|
||||
client_id: string;
|
||||
}
|
||||
|
||||
export interface EasytierConfigUpdate {
|
||||
enabled?: boolean;
|
||||
config_mode?: EasytierConfigMode;
|
||||
@@ -898,8 +840,8 @@ export enum CanonicalKey {
|
||||
MetaRight = "MetaRight",
|
||||
}
|
||||
|
||||
export enum ComputerUseButton {
|
||||
Left = "left",
|
||||
Middle = "middle",
|
||||
Right = "right",
|
||||
|
||||
export interface McpConfig {
|
||||
enabled: boolean;
|
||||
requests_per_second: number;
|
||||
}
|
||||
|
||||
@@ -11,14 +11,12 @@ import { useHidWebSocket } from '@/composables/useHidWebSocket'
|
||||
import { useWebRTC } from '@/composables/useWebRTC'
|
||||
import { useVideoSession } from '@/composables/useVideoSession'
|
||||
import { useVideoScaling, type VideoRotation } from '@/composables/useVideoScaling'
|
||||
import { useComputerUseSocket, type ComputerUseServerMessage } from '@/composables/useComputerUseSocket'
|
||||
import { useFeatureVisibility } from '@/composables/useFeatureVisibility'
|
||||
import { useTheme } from '@/composables/useTheme'
|
||||
import { useConsoleLayout } from '@/composables/useConsoleLayout'
|
||||
import { getUnifiedAudio } from '@/composables/useUnifiedAudio'
|
||||
import { getMicrophone } from '@/composables/useMicrophone'
|
||||
import { streamApi, hidApi, atxApi, atxConfigApi, authApi, computerUseApi, uacApi } from '@/api'
|
||||
import type { ComputerUseScreenshot, ComputerUseSession } from '@/api'
|
||||
import { streamApi, hidApi, atxApi, atxConfigApi, authApi, uacApi } from '@/api'
|
||||
import { CanonicalKey, HidBackend } from '@/types/generated'
|
||||
import type { HidKeyboardEvent, HidMouseEvent } from '@/types/hid'
|
||||
import { keyboardEventToCanonicalKey, updateModifierMaskForKey } from '@/lib/keyboardMappings'
|
||||
@@ -40,9 +38,7 @@ import ConsoleStatusSummary, { type ConsoleStatusItem } from '@/components/Conso
|
||||
import InfoBar from '@/components/InfoBar.vue'
|
||||
import VirtualKeyboard from '@/components/VirtualKeyboard.vue'
|
||||
import StatsSheet from '@/components/StatsSheet.vue'
|
||||
import ComputerUseSheet from '@/components/ComputerUseSheet.vue'
|
||||
import TerminalDialog from '@/components/TerminalDialog.vue'
|
||||
import type { ComputerUseTimelineItem, NewComputerUseTimelineItem } from '@/types/computerUseTimeline'
|
||||
import LanguageToggleButton from '@/components/LanguageToggleButton.vue'
|
||||
import BrandMark from '@/components/BrandMark.vue'
|
||||
import { Button } from '@/components/ui/button'
|
||||
@@ -119,11 +115,6 @@ function setVideoRotation(rotation: VideoRotation) {
|
||||
videoRotation.value = rotation
|
||||
localStorage.setItem('videoRotation', String(rotation))
|
||||
}
|
||||
const computerUseOpen = ref(false)
|
||||
const computerUseSession = ref<ComputerUseSession | null>(null)
|
||||
const computerUseTimeline = ref<ComputerUseTimelineItem[]>([])
|
||||
const computerUseConversationStarted = ref(false)
|
||||
let computerUseTimelineSeq = 0
|
||||
|
||||
const videoRef = ref<HTMLImageElement | null>(null)
|
||||
const webrtcVideoRef = ref<HTMLVideoElement | null>(null)
|
||||
@@ -166,11 +157,6 @@ const clientsStats = ref<Record<string, ClientStat>>({})
|
||||
|
||||
const myClientId = generateUUID()
|
||||
|
||||
const computerUseSocket = useComputerUseSocket({
|
||||
onMessage: handleComputerUseMessage,
|
||||
onScreenshotRequested: captureComputerUseFrame,
|
||||
})
|
||||
|
||||
const mouseMode = ref<'absolute' | 'relative'>('absolute')
|
||||
const pressedKeys = ref<CanonicalKey[]>([])
|
||||
const keyboardLed = computed(() => ({
|
||||
@@ -262,7 +248,6 @@ provideConsoleAppearance(consoleLayout)
|
||||
const terminalAvailable = computed(() => ttydStatus.value?.available !== false)
|
||||
const showPower = computed(() => featureVisibility.value.power)
|
||||
const showTerminal = computed(() => terminalAvailable.value && featureVisibility.value.webTerminal)
|
||||
const showComputerUse = computed(() => featureVisibility.value.computerUse)
|
||||
const showPasteText = computed(() => featureVisibility.value.pasteText)
|
||||
|
||||
const videoStatus = computed<ConnectionStatus>(() => {
|
||||
@@ -749,7 +734,6 @@ watch(videoLoading, (loading) => {
|
||||
if (loading) clearVideoSourceSize()
|
||||
})
|
||||
|
||||
const computerUsePanelVisible = computed(() => computerUseOpen.value && !isFullscreen.value)
|
||||
|
||||
const showMsdStatusCard = computed(() => {
|
||||
return !!(systemStore.msd?.available && systemStore.hid?.backend !== 'ch9329')
|
||||
@@ -813,140 +797,6 @@ async function captureFrameOverlay() {
|
||||
}
|
||||
}
|
||||
|
||||
async function captureComputerUseFrame(): Promise<ComputerUseScreenshot | null> {
|
||||
try {
|
||||
const canvas = document.createElement('canvas')
|
||||
const ctx = canvas.getContext('2d')
|
||||
if (!ctx) return null
|
||||
|
||||
const MAX_WIDTH = 1920
|
||||
|
||||
if (videoMode.value === 'mjpeg') {
|
||||
const img = videoRef.value
|
||||
if (!img || !img.naturalWidth || !img.naturalHeight) return null
|
||||
|
||||
const scale = Math.min(1, MAX_WIDTH / img.naturalWidth)
|
||||
canvas.width = Math.max(1, Math.round(img.naturalWidth * scale))
|
||||
canvas.height = Math.max(1, Math.round(img.naturalHeight * scale))
|
||||
ctx.drawImage(img, 0, 0, canvas.width, canvas.height)
|
||||
} else {
|
||||
const video = webrtcVideoRef.value
|
||||
if (!video || !video.videoWidth || !video.videoHeight) return null
|
||||
|
||||
const scale = Math.min(1, MAX_WIDTH / video.videoWidth)
|
||||
canvas.width = Math.max(1, Math.round(video.videoWidth * scale))
|
||||
canvas.height = Math.max(1, Math.round(video.videoHeight * scale))
|
||||
ctx.drawImage(video, 0, 0, canvas.width, canvas.height)
|
||||
}
|
||||
|
||||
return {
|
||||
data_url: canvas.toDataURL('image/jpeg', 0.82),
|
||||
width: canvas.width,
|
||||
height: canvas.height,
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('[ComputerUse] Failed to capture frame:', err)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function handleComputerUseMessage(message: ComputerUseServerMessage) {
|
||||
switch (message.type) {
|
||||
case 'session_updated':
|
||||
computerUseSession.value = message.session
|
||||
if (message.session.last_error) {
|
||||
pushComputerUseTimeline({ type: 'error', text: message.session.last_error })
|
||||
}
|
||||
if (message.session.final_message) {
|
||||
pushComputerUseTimeline({ type: 'assistant', text: message.session.final_message })
|
||||
}
|
||||
break
|
||||
case 'screenshot_captured':
|
||||
pushComputerUseTimeline({ type: 'screenshot', screenshot: message.screenshot })
|
||||
break
|
||||
case 'actions_executed':
|
||||
pushComputerUseTimeline({ type: 'actions_executed', actions: message.actions })
|
||||
break
|
||||
case 'step_started':
|
||||
pushComputerUseTimeline({ type: 'reasoning', text: '', completed: false, failed: false })
|
||||
break
|
||||
case 'reasoning_delta': {
|
||||
const last = computerUseTimeline.value[computerUseTimeline.value.length - 1]
|
||||
if (last?.type === 'reasoning' && !last.completed) {
|
||||
last.text += message.delta
|
||||
} else {
|
||||
pushComputerUseTimeline({ type: 'reasoning', text: message.delta, completed: false, failed: false })
|
||||
}
|
||||
break
|
||||
}
|
||||
case 'reasoning_completed': {
|
||||
for (let index = computerUseTimeline.value.length - 1; index >= 0; index -= 1) {
|
||||
const item = computerUseTimeline.value[index]
|
||||
if (item?.type !== 'reasoning' || item.completed) continue
|
||||
if (!message.failed && !item.text) {
|
||||
computerUseTimeline.value.splice(index, 1)
|
||||
} else {
|
||||
item.completed = true
|
||||
item.failed = message.failed
|
||||
}
|
||||
break
|
||||
}
|
||||
break
|
||||
}
|
||||
case 'error':
|
||||
pushComputerUseTimeline({ type: 'error', text: message.message })
|
||||
toast.error(t('computerUse.errors.taskFailed'), { description: message.message })
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
function pushComputerUseTimeline(item: NewComputerUseTimelineItem) {
|
||||
const last = computerUseTimeline.value[computerUseTimeline.value.length - 1]
|
||||
if (last?.type === item.type) {
|
||||
if ('text' in last && 'text' in item && last.text === item.text) return
|
||||
if (last.type === 'actions_executed' && item.type === 'actions_executed' && JSON.stringify(last.actions) === JSON.stringify(item.actions)) return
|
||||
}
|
||||
computerUseTimeline.value.push({
|
||||
id: `${Date.now()}-${computerUseTimelineSeq++}`,
|
||||
...item,
|
||||
} as ComputerUseTimelineItem)
|
||||
}
|
||||
|
||||
function clearComputerUseTimeline() {
|
||||
computerUseTimeline.value = []
|
||||
computerUseConversationStarted.value = false
|
||||
}
|
||||
|
||||
async function openComputerUse() {
|
||||
if (!showComputerUse.value) return
|
||||
computerUseOpen.value = true
|
||||
await computerUseSocket.connect().catch(() => {})
|
||||
computerUseSession.value = await computerUseApi.session().catch(() => computerUseSession.value)
|
||||
}
|
||||
|
||||
async function startComputerUse(prompt: string) {
|
||||
try {
|
||||
await computerUseSocket.connect()
|
||||
pushComputerUseTimeline({ type: 'user', text: prompt })
|
||||
computerUseSession.value = await computerUseApi.start({
|
||||
prompt,
|
||||
continue_conversation: computerUseConversationStarted.value,
|
||||
client_id: computerUseSocket.clientId,
|
||||
})
|
||||
computerUseConversationStarted.value = true
|
||||
} catch (err: any) {
|
||||
pushComputerUseTimeline({ type: 'error', text: err?.message ?? t('computerUse.errors.startFailed') })
|
||||
toast.error(t('computerUse.errors.startFailed'), { description: err?.message })
|
||||
}
|
||||
}
|
||||
|
||||
async function stopComputerUse() {
|
||||
try {
|
||||
computerUseSession.value = await computerUseApi.stop()
|
||||
} catch (err: any) {
|
||||
toast.error(t('computerUse.errors.stopFailed'), { description: err?.message })
|
||||
}
|
||||
}
|
||||
|
||||
function waitForVideoFirstFrame(el: HTMLVideoElement, timeoutMs = 2000): Promise<boolean> {
|
||||
return new Promise((resolve) => {
|
||||
@@ -2004,9 +1854,6 @@ watch(showTerminal, (visible) => {
|
||||
if (!visible) showTerminalDialog.value = false
|
||||
})
|
||||
|
||||
watch(showComputerUse, (visible) => {
|
||||
if (!visible) computerUseOpen.value = false
|
||||
})
|
||||
|
||||
watch(() => webrtc.audioTrack.value, async (track) => {
|
||||
videoDebugLog('WebRTC audio track ref changed', {
|
||||
@@ -3299,10 +3146,10 @@ onUnmounted(() => {
|
||||
:show-stats="showConnectionStats"
|
||||
:show-terminal="showTerminal"
|
||||
:terminal-running="!!ttydStatus?.running"
|
||||
:show-computer-use="showComputerUse"
|
||||
|
||||
@open-stats="openStatsSheet"
|
||||
@open-terminal="openTerminal"
|
||||
@open-computer-use="openComputerUse"
|
||||
|
||||
/>
|
||||
<div v-if="consoleLayout === 'current'" class="hidden md:flex items-center gap-2">
|
||||
<StatusCard
|
||||
@@ -3397,7 +3244,7 @@ onUnmounted(() => {
|
||||
:show-power="showPower"
|
||||
:atx-enabled="systemStore.atx?.available === true"
|
||||
:show-terminal="showTerminal"
|
||||
:show-computer-use="showComputerUse"
|
||||
|
||||
:show-paste-text="showPasteText"
|
||||
:show-mic="microphoneTransferEnabled"
|
||||
:scale-mode="videoScaleMode"
|
||||
@@ -3414,7 +3261,7 @@ onUnmounted(() => {
|
||||
@reset="handleReset"
|
||||
@wol="handleWol"
|
||||
@open-terminal="openTerminal"
|
||||
@open-computer-use="openComputerUse"
|
||||
|
||||
/>
|
||||
</div>
|
||||
<div
|
||||
@@ -3425,7 +3272,6 @@ onUnmounted(() => {
|
||||
<div class="relative flex h-full w-full min-w-0 items-stretch gap-3">
|
||||
<div
|
||||
class="flex min-w-0 flex-1 items-center justify-center transition-all duration-300"
|
||||
:class="{ 'md:pr-1': computerUsePanelVisible }"
|
||||
>
|
||||
<div
|
||||
ref="videoWorkspaceRef"
|
||||
@@ -3635,17 +3481,7 @@ onUnmounted(() => {
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<ComputerUseSheet
|
||||
v-if="showComputerUse"
|
||||
v-model:open="computerUseOpen"
|
||||
:connected="computerUseSocket.connected.value"
|
||||
:ws-error="computerUseSocket.error.value"
|
||||
:session="computerUseSession"
|
||||
:timeline="computerUseTimeline"
|
||||
@start="startComputerUse"
|
||||
@stop="stopComputerUse"
|
||||
@clear="clearComputerUseTimeline"
|
||||
/>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<Teleport :to="virtualKeyboardAttached ? '#keyboard-anchor' : 'body'" :disabled="virtualKeyboardAttached">
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
<script setup lang="ts">
|
||||
import McpSettingsCard from '@/components/McpSettingsCard.vue'
|
||||
import HidDeviceSettings from '@/components/HidDeviceSettings.vue'
|
||||
import ConsoleLayoutPreview from '@/components/ConsoleLayoutPreview.vue'
|
||||
import { ref, computed, onMounted, watch, nextTick } from 'vue'
|
||||
@@ -192,6 +193,7 @@ const SETTINGS_SECTION_IDS = [
|
||||
'other',
|
||||
'ext-ttyd',
|
||||
'third-party-access',
|
||||
'mcp',
|
||||
'ext-remote-access',
|
||||
'about',
|
||||
] as const
|
||||
@@ -225,6 +227,7 @@ const navGroups = computed(() => [
|
||||
title: t('settings.extensions'),
|
||||
items: [
|
||||
{ id: 'ext-ttyd', label: t('extensions.ttyd.title'), icon: Terminal },
|
||||
{ id: 'mcp', label: t('mcp.title'), icon: Bot },
|
||||
{ id: 'third-party-access', label: t('extensions.thirdPartyAccess.title'), icon: ScreenShare },
|
||||
{ id: 'ext-remote-access', label: t('extensions.remoteAccess.title'), icon: ExternalLink },
|
||||
]
|
||||
@@ -2942,18 +2945,13 @@ watch(isWindows, () => {
|
||||
</Label>
|
||||
<Switch id="feature-web-terminal" v-model="featureVisibility.webTerminal" />
|
||||
</div>
|
||||
<div class="flex items-center justify-between gap-4 px-3 py-3">
|
||||
<Label for="feature-computer-use" class="flex min-w-0 items-center gap-2 font-normal">
|
||||
<Bot class="size-4 shrink-0 text-muted-foreground" />
|
||||
<span class="truncate">{{ t('settings.computerUseAgent') }}</span>
|
||||
</Label>
|
||||
<Switch id="feature-computer-use" v-model="featureVisibility.computerUse" />
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
</div>
|
||||
|
||||
<!-- Account Section -->
|
||||
<McpSettingsCard v-if="activeSection === 'mcp'" />
|
||||
|
||||
<div v-show="activeSection === 'account'" class="space-y-4">
|
||||
<Card>
|
||||
<CardHeader>
|
||||
|
||||
Reference in New Issue
Block a user