feat(hid): 添加 Consumer Control 多媒体按键和多平台键盘布局

- 新增 Consumer Control HID 支持(播放/暂停、音量控制等)
- 虚拟键盘支持 Windows/Mac/Android 三种布局切换
- 移除键盘 LED 反馈以节省 USB 端点(从 2 减至 1)
- InfoBar 优化:按键名称友好显示,移除未实现的 Num/Scroll 指示器
- 更新 HID 模块文档
This commit is contained in:
mofeng-git
2026-01-02 23:52:12 +08:00
parent ad401cdf1c
commit cb7d9882a2
27 changed files with 888 additions and 262 deletions

View File

@@ -42,7 +42,7 @@ rand = "0.8"
uuid = { version = "1", features = ["v4", "serde"] }
chrono = { version = "0.4", features = ["serde"] }
base64 = "0.22"
nix = { version = "0.29", features = ["fs", "net", "hostname"] }
nix = { version = "0.29", features = ["fs", "net", "hostname", "poll"] }
# HTTP client (for URL downloads)
# Use rustls-tls by default, but allow native-tls for systems with older GLIBC

View File

@@ -8,11 +8,28 @@ HID (Human Interface Device) 模块负责将键盘和鼠标事件转发到目标
- 键盘事件处理 (按键、修饰键)
- 鼠标事件处理 (移动、点击、滚轮)
- 多媒体键支持 (Consumer Control)
- 支持绝对和相对鼠标模式
- 多后端支持 (OTG、CH9329)
- WebSocket 和 DataChannel 输入
### 1.2 文件结构
### 1.2 USB Endpoint 使用
OTG 模式下的 endpoint 分配:
| 功能 | IN 端点 | OUT 端点 | 说明 |
|------|---------|----------|------|
| Keyboard | 1 | 0 | 无 LED 反馈 |
| MouseRelative | 1 | 0 | 相对鼠标 |
| MouseAbsolute | 1 | 0 | 绝对鼠标 |
| ConsumerControl | 1 | 0 | 多媒体键 |
| **HID 总计** | **4** | **0** | |
| MSD | 1 | 1 | 大容量存储 |
| **全部总计** | **5** | **1** | 兼容 6 endpoint 设备 |
> EP0 (控制端点) 独立于数据端点,不计入上述统计。
### 1.3 文件结构
```
src/hid/
@@ -20,6 +37,7 @@ src/hid/
├── backend.rs # 后端抽象
├── otg.rs # OTG 后端 (33KB)
├── ch9329.rs # CH9329 串口后端 (46KB)
├── consumer.rs # Consumer Control 常量定义
├── keymap.rs # 按键映射 (14KB)
├── types.rs # 类型定义
├── monitor.rs # 健康监视 (14KB)
@@ -93,10 +111,11 @@ HidBackendType::Otg
├── 检查 OtgService 是否可用
├── 请求 HID 函数 (3个设备)
│ ├── /dev/hidg0 (键盘)
│ ├── /dev/hidg1 (相对鼠标)
── /dev/hidg2 (绝对鼠标)
├── 请求 HID 函数 (4个设备, 共4个IN端点)
│ ├── /dev/hidg0 (键盘, 1 IN)
│ ├── /dev/hidg1 (相对鼠标, 1 IN)
── /dev/hidg2 (绝对鼠标, 1 IN)
│ └── /dev/hidg3 (Consumer Control, 1 IN)
└── 创建 OtgHidBackend
@@ -159,6 +178,9 @@ impl HidController {
/// 发送鼠标事件
pub async fn send_mouse(&self, event: &MouseEvent) -> Result<()>;
/// 发送多媒体键事件 (Consumer Control)
pub async fn send_consumer(&self, event: &ConsumerEvent) -> Result<()>;
/// 设置鼠标模式
pub fn set_mouse_mode(&self, mode: MouseMode);
@@ -306,6 +328,23 @@ pub struct MouseAbsoluteReport {
pub y: u16, // Y 坐标 (0 ~ 32767)
pub wheel: i8, // 滚轮 (-127 ~ 127)
}
/// Consumer Control 报告 (2 字节)
#[repr(C, packed)]
pub struct ConsumerControlReport {
pub usage: u16, // Consumer Control Usage Code (LE)
}
// 常用 Consumer Control Usage Codes
pub mod consumer_usage {
pub const PLAY_PAUSE: u16 = 0x00CD;
pub const STOP: u16 = 0x00B7;
pub const NEXT_TRACK: u16 = 0x00B5;
pub const PREV_TRACK: u16 = 0x00B6;
pub const MUTE: u16 = 0x00E2;
pub const VOLUME_UP: u16 = 0x00E9;
pub const VOLUME_DOWN: u16 = 0x00EA;
}
```
### 3.4 CH9329 后端 (ch9329.rs)
@@ -455,6 +494,25 @@ pub enum MouseMode {
}
```
### 4.3 Consumer Control 事件 (types.rs)
```rust
/// Consumer Control 事件 (多媒体键)
pub struct ConsumerEvent {
/// USB HID Consumer Control Usage Code
pub usage: u16,
}
// 常用 Usage Codes (参考 USB HID Usage Tables)
// 0x00CD - Play/Pause
// 0x00B7 - Stop
// 0x00B5 - Next Track
// 0x00B6 - Previous Track
// 0x00E2 - Mute
// 0x00E9 - Volume Up
// 0x00EA - Volume Down
```
---
## 5. 按键映射
@@ -573,7 +631,36 @@ pub enum HidMessage {
### 6.2 DataChannel Handler (datachannel.rs)
用于 WebRTC 模式下的 HID 事件处理。
用于 WebRTC 模式下的 HID 事件处理,使用二进制协议
#### 二进制消息格式
```
消息类型常量:
MSG_KEYBOARD = 0x01 // 键盘事件
MSG_MOUSE = 0x02 // 鼠标事件
MSG_CONSUMER = 0x03 // Consumer Control 事件
键盘消息 (4 字节):
┌──────────┬──────────┬──────────┬──────────┐
│ MSG_TYPE │ EVENT │ KEY_CODE │ MODIFIER │
│ 0x01 │ 0/1 │ scancode │ bitmask │
└──────────┴──────────┴──────────┴──────────┘
EVENT: 0=keydown, 1=keyup
鼠标消息 (7 字节):
┌──────────┬──────────┬──────────┬──────────┬──────────┐
│ MSG_TYPE │ EVENT │ X (i16) │ Y (i16) │ BTN/SCRL │
│ 0x02 │ 0-4 │ LE │ LE │ u8/i8 │
└──────────┴──────────┴──────────┴──────────┴──────────┘
EVENT: 0=move, 1=moveabs, 2=down, 3=up, 4=scroll
Consumer Control 消息 (3 字节):
┌──────────┬──────────────────────┐
│ MSG_TYPE │ USAGE CODE (u16 LE) │
│ 0x03 │ e.g. 0x00CD │
└──────────┴──────────────────────┘
```
```rust
pub struct HidDataChannelHandler {

View File

@@ -3,7 +3,7 @@
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use super::types::{KeyboardEvent, MouseEvent};
use super::types::{ConsumerEvent, KeyboardEvent, MouseEvent};
use crate::error::Result;
/// Default CH9329 baud rate
@@ -94,6 +94,14 @@ pub trait HidBackend: Send + Sync {
/// Send a mouse event
async fn send_mouse(&self, event: MouseEvent) -> Result<()>;
/// Send a consumer control event (multimedia keys)
/// Default implementation returns an error (not supported)
async fn send_consumer(&self, _event: ConsumerEvent) -> Result<()> {
Err(crate::error::AppError::BadRequest(
"Consumer control not supported by this backend".to_string(),
))
}
/// Reset all inputs (release all keys/buttons)
async fn reset(&self) -> Result<()>;

45
src/hid/consumer.rs Normal file
View File

@@ -0,0 +1,45 @@
//! USB HID Consumer Control Usage codes
//!
//! Reference: USB HID Usage Tables 1.12, Section 15 (Consumer Page 0x0C)
/// Consumer Control Usage codes for multimedia keys
pub mod usage {
// Transport Controls
pub const PLAY_PAUSE: u16 = 0x00CD;
pub const STOP: u16 = 0x00B7;
pub const NEXT_TRACK: u16 = 0x00B5;
pub const PREV_TRACK: u16 = 0x00B6;
// Volume Controls
pub const MUTE: u16 = 0x00E2;
pub const VOLUME_UP: u16 = 0x00E9;
pub const VOLUME_DOWN: u16 = 0x00EA;
}
/// Check if a usage code is valid
pub fn is_valid_usage(usage: u16) -> bool {
matches!(
usage,
usage::PLAY_PAUSE
| usage::STOP
| usage::NEXT_TRACK
| usage::PREV_TRACK
| usage::MUTE
| usage::VOLUME_UP
| usage::VOLUME_DOWN
)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_valid_usage_codes() {
assert!(is_valid_usage(usage::PLAY_PAUSE));
assert!(is_valid_usage(usage::MUTE));
assert!(is_valid_usage(usage::VOLUME_UP));
assert!(!is_valid_usage(0x0000));
assert!(!is_valid_usage(0xFFFF));
}
}

View File

@@ -4,6 +4,7 @@
//! - Byte 0: Message type
//! - 0x01: Keyboard event
//! - 0x02: Mouse event
//! - 0x03: Consumer control event (multimedia keys)
//! - Remaining bytes: Event data
//!
//! Keyboard event (type 0x01):
@@ -29,9 +30,13 @@
//! - Bytes 2-3: X coordinate (i16 LE for relative, u16 LE for absolute)
//! - Bytes 4-5: Y coordinate (i16 LE for relative, u16 LE for absolute)
//! - Byte 6: Button (0=left, 1=middle, 2=right) or Scroll delta (i8)
//!
//! Consumer control event (type 0x03):
//! - Bytes 1-2: Usage code (u16 LE)
use tracing::{debug, warn};
use super::types::ConsumerEvent;
use super::{
KeyEventType, KeyboardEvent, KeyboardModifiers, MouseButton, MouseEvent, MouseEventType,
};
@@ -39,6 +44,7 @@ use super::{
/// Message types
pub const MSG_KEYBOARD: u8 = 0x01;
pub const MSG_MOUSE: u8 = 0x02;
pub const MSG_CONSUMER: u8 = 0x03;
/// Keyboard event types
pub const KB_EVENT_DOWN: u8 = 0x00;
@@ -56,6 +62,7 @@ pub const MS_EVENT_SCROLL: u8 = 0x04;
pub enum HidChannelEvent {
Keyboard(KeyboardEvent),
Mouse(MouseEvent),
Consumer(ConsumerEvent),
}
/// Parse a binary HID message from DataChannel
@@ -70,6 +77,7 @@ pub fn parse_hid_message(data: &[u8]) -> Option<HidChannelEvent> {
match msg_type {
MSG_KEYBOARD => parse_keyboard_message(&data[1..]),
MSG_MOUSE => parse_mouse_message(&data[1..]),
MSG_CONSUMER => parse_consumer_message(&data[1..]),
_ => {
warn!("Unknown HID message type: 0x{:02X}", msg_type);
None
@@ -173,6 +181,20 @@ fn parse_mouse_message(data: &[u8]) -> Option<HidChannelEvent> {
}))
}
/// Parse consumer control message payload
fn parse_consumer_message(data: &[u8]) -> Option<HidChannelEvent> {
if data.len() < 2 {
warn!("Consumer message too short: {} bytes", data.len());
return None;
}
let usage = u16::from_le_bytes([data[0], data[1]]);
debug!("Parsed consumer: usage=0x{:04X}", usage);
Some(HidChannelEvent::Consumer(ConsumerEvent { usage }))
}
/// Encode a keyboard event to binary format (for sending to client if needed)
pub fn encode_keyboard_event(event: &KeyboardEvent) -> Vec<u8> {
let event_type = match event.event_type {

View File

@@ -13,6 +13,7 @@
pub mod backend;
pub mod ch9329;
pub mod consumer;
pub mod datachannel;
pub mod keymap;
pub mod monitor;
@@ -24,7 +25,8 @@ pub use backend::{HidBackend, HidBackendType};
pub use monitor::{HidHealthMonitor, HidHealthStatus, HidMonitorConfig};
pub use otg::LedState;
pub use types::{
KeyEventType, KeyboardEvent, KeyboardModifiers, MouseButton, MouseEvent, MouseEventType,
ConsumerEvent, KeyEventType, KeyboardEvent, KeyboardModifiers, MouseButton, MouseEvent,
MouseEventType,
};
/// HID backend information
@@ -199,6 +201,33 @@ impl HidController {
}
}
/// Send consumer control event (multimedia keys)
pub async fn send_consumer(&self, event: ConsumerEvent) -> Result<()> {
let backend = self.backend.read().await;
match backend.as_ref() {
Some(b) => {
match b.send_consumer(event).await {
Ok(_) => {
if self.monitor.is_error().await {
let backend_type = self.backend_type.read().await;
self.monitor.report_recovered(backend_type.name_str()).await;
}
Ok(())
}
Err(e) => {
if let AppError::HidError { ref backend, ref reason, ref error_code } = e {
if error_code != "eagain_retry" {
self.monitor.report_error(backend, None, reason, error_code).await;
}
}
Err(e)
}
}
}
None => Err(AppError::BadRequest("HID backend not available".to_string())),
}
}
/// Reset all keys (release all pressed keys)
pub async fn reset(&self) -> Result<()> {
let backend = self.backend.read().await;

View File

@@ -18,17 +18,19 @@
//! See: https://github.com/raspberrypi/linux/issues/4373
use async_trait::async_trait;
use nix::poll::{poll, PollFd, PollFlags, PollTimeout};
use parking_lot::Mutex;
use std::fs::{self, File, OpenOptions};
use std::io::{Read, Write};
use std::os::unix::fs::OpenOptionsExt;
use std::os::unix::io::AsFd;
use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, AtomicU8, Ordering};
use tracing::{debug, info, trace, warn};
use super::backend::HidBackend;
use super::keymap;
use super::types::{KeyEventType, KeyboardEvent, KeyboardReport, MouseEvent, MouseEventType};
use super::types::{ConsumerEvent, KeyEventType, KeyboardEvent, KeyboardReport, MouseEvent, MouseEventType};
use crate::error::{AppError, Result};
use crate::otg::{HidDevicePaths, wait_for_hid_devices};
@@ -38,6 +40,7 @@ enum DeviceType {
Keyboard,
MouseRelative,
MouseAbsolute,
ConsumerControl,
}
/// Keyboard LED state
@@ -79,7 +82,7 @@ impl LedState {
}
}
/// OTG HID backend with 3 devices
/// OTG HID backend with 4 devices
///
/// This backend opens HID device files created by OtgService.
/// It does NOT manage the USB gadget itself - that's handled by OtgService.
@@ -99,12 +102,16 @@ pub struct OtgBackend {
mouse_rel_path: PathBuf,
/// Absolute mouse device path (/dev/hidg2)
mouse_abs_path: PathBuf,
/// Consumer control device path (/dev/hidg3)
consumer_path: PathBuf,
/// Keyboard device file
keyboard_dev: Mutex<Option<File>>,
/// Relative mouse device file
mouse_rel_dev: Mutex<Option<File>>,
/// Absolute mouse device file
mouse_abs_dev: Mutex<Option<File>>,
/// Consumer control device file
consumer_dev: Mutex<Option<File>>,
/// Current keyboard state
keyboard_state: Mutex<KeyboardReport>,
/// Current mouse button state
@@ -125,8 +132,8 @@ pub struct OtgBackend {
eagain_count: AtomicU8,
}
/// Threshold for consecutive EAGAIN errors before reporting offline
const EAGAIN_OFFLINE_THRESHOLD: u8 = 3;
/// Write timeout in milliseconds (same as JetKVM's hidWriteTimeout)
const HID_WRITE_TIMEOUT_MS: i32 = 500;
impl OtgBackend {
/// Create OTG backend from device paths provided by OtgService
@@ -138,9 +145,11 @@ impl OtgBackend {
keyboard_path: paths.keyboard,
mouse_rel_path: paths.mouse_relative,
mouse_abs_path: paths.mouse_absolute,
consumer_path: paths.consumer.unwrap_or_else(|| PathBuf::from("/dev/hidg3")),
keyboard_dev: Mutex::new(None),
mouse_rel_dev: Mutex::new(None),
mouse_abs_dev: Mutex::new(None),
consumer_dev: Mutex::new(None),
keyboard_state: Mutex::new(KeyboardReport::default()),
mouse_buttons: AtomicU8::new(0),
led_state: parking_lot::RwLock::new(LedState::default()),
@@ -177,6 +186,39 @@ impl OtgBackend {
self.eagain_count.store(0, Ordering::Relaxed);
}
/// Write data to HID device with timeout (JetKVM style)
///
/// Uses poll() to wait for device to be ready for writing.
/// If timeout expires, silently drops the data (acceptable for mouse movement).
/// Returns Ok(true) if write succeeded, Ok(false) if timed out (silently dropped).
fn write_with_timeout(&self, file: &mut File, data: &[u8]) -> std::io::Result<bool> {
let mut pollfd = [PollFd::new(file.as_fd(), PollFlags::POLLOUT)];
match poll(&mut pollfd, PollTimeout::from(HID_WRITE_TIMEOUT_MS as u16)) {
Ok(1) => {
// Device ready, check for errors
if let Some(revents) = pollfd[0].revents() {
if revents.contains(PollFlags::POLLERR) || revents.contains(PollFlags::POLLHUP) {
return Err(std::io::Error::new(
std::io::ErrorKind::BrokenPipe,
"Device error or hangup",
));
}
}
// Write the data
file.write_all(data)?;
Ok(true)
}
Ok(0) => {
// Timeout - silently drop (JetKVM behavior)
trace!("HID write timeout, dropping data");
Ok(false)
}
Ok(_) => Ok(false),
Err(e) => Err(std::io::Error::new(std::io::ErrorKind::Other, e)),
}
}
/// Set the UDC name for state checking
pub fn set_udc_name(&self, udc: &str) {
*self.udc_name.write() = Some(udc.to_string());
@@ -247,6 +289,7 @@ impl OtgBackend {
DeviceType::Keyboard => (&self.keyboard_path, &self.keyboard_dev),
DeviceType::MouseRelative => (&self.mouse_rel_path, &self.mouse_rel_dev),
DeviceType::MouseAbsolute => (&self.mouse_abs_path, &self.mouse_abs_dev),
DeviceType::ConsumerControl => (&self.consumer_path, &self.consumer_dev),
};
// Check if device path exists
@@ -342,7 +385,7 @@ impl OtgBackend {
///
/// This method ensures the device is open before writing, and handles
/// ESHUTDOWN errors by closing the device handle for later reconnection.
/// EAGAIN errors are treated as temporary - device stays open.
/// Uses write_with_timeout to avoid blocking on busy devices.
fn send_keyboard_report(&self, report: &KeyboardReport) -> Result<()> {
// Ensure device is ready
self.ensure_device(DeviceType::Keyboard)?;
@@ -350,13 +393,18 @@ impl OtgBackend {
let mut dev = self.keyboard_dev.lock();
if let Some(ref mut file) = *dev {
let data = report.to_bytes();
match file.write_all(&data) {
Ok(_) => {
match self.write_with_timeout(file, &data) {
Ok(true) => {
self.online.store(true, Ordering::Relaxed);
self.reset_error_count();
trace!("Sent keyboard report: {:02X?}", data);
Ok(())
}
Ok(false) => {
// Timeout - silently dropped (JetKVM behavior)
self.log_throttled_error("HID keyboard write timeout, dropped");
Ok(())
}
Err(e) => {
let error_code = e.raw_os_error();
@@ -370,26 +418,9 @@ impl OtgBackend {
Err(Self::io_error_to_hid_error(e, "Failed to write keyboard report"))
}
Some(11) => {
// EAGAIN - temporary busy, track consecutive count
self.log_throttled_error("HID keyboard busy (EAGAIN)");
let count = self.eagain_count.fetch_add(1, Ordering::Relaxed) + 1;
if count >= EAGAIN_OFFLINE_THRESHOLD {
// Exceeded threshold, report as offline
self.online.store(false, Ordering::Relaxed);
Err(AppError::HidError {
backend: "otg".to_string(),
reason: format!("Device busy ({} consecutive EAGAIN)", count),
error_code: "eagain".to_string(),
})
} else {
// Within threshold, return retry error (won't trigger offline event)
Err(AppError::HidError {
backend: "otg".to_string(),
reason: "Device temporarily busy".to_string(),
error_code: "eagain_retry".to_string(),
})
}
// EAGAIN after poll - should be rare, silently drop
trace!("Keyboard EAGAIN after poll, dropping");
Ok(())
}
_ => {
self.online.store(false, Ordering::Relaxed);
@@ -413,7 +444,7 @@ impl OtgBackend {
///
/// This method ensures the device is open before writing, and handles
/// ESHUTDOWN errors by closing the device handle for later reconnection.
/// EAGAIN errors are treated as temporary - device stays open.
/// Uses write_with_timeout to avoid blocking on busy devices.
fn send_mouse_report_relative(&self, buttons: u8, dx: i8, dy: i8, wheel: i8) -> Result<()> {
// Ensure device is ready
self.ensure_device(DeviceType::MouseRelative)?;
@@ -421,13 +452,17 @@ impl OtgBackend {
let mut dev = self.mouse_rel_dev.lock();
if let Some(ref mut file) = *dev {
let data = [buttons, dx as u8, dy as u8, wheel as u8];
match file.write_all(&data) {
Ok(_) => {
match self.write_with_timeout(file, &data) {
Ok(true) => {
self.online.store(true, Ordering::Relaxed);
self.reset_error_count();
trace!("Sent relative mouse report: {:02X?}", data);
Ok(())
}
Ok(false) => {
// Timeout - silently dropped (JetKVM behavior)
Ok(())
}
Err(e) => {
let error_code = e.raw_os_error();
@@ -440,26 +475,8 @@ impl OtgBackend {
Err(Self::io_error_to_hid_error(e, "Failed to write mouse report"))
}
Some(11) => {
// EAGAIN - temporary busy, track consecutive count
self.log_throttled_error("HID relative mouse busy (EAGAIN)");
let count = self.eagain_count.fetch_add(1, Ordering::Relaxed) + 1;
if count >= EAGAIN_OFFLINE_THRESHOLD {
// Exceeded threshold, report as offline
self.online.store(false, Ordering::Relaxed);
Err(AppError::HidError {
backend: "otg".to_string(),
reason: format!("Device busy ({} consecutive EAGAIN)", count),
error_code: "eagain".to_string(),
})
} else {
// Within threshold, return retry error (won't trigger offline event)
Err(AppError::HidError {
backend: "otg".to_string(),
reason: "Device temporarily busy".to_string(),
error_code: "eagain_retry".to_string(),
})
}
// EAGAIN after poll - should be rare, silently drop
Ok(())
}
_ => {
self.online.store(false, Ordering::Relaxed);
@@ -483,7 +500,7 @@ impl OtgBackend {
///
/// This method ensures the device is open before writing, and handles
/// ESHUTDOWN errors by closing the device handle for later reconnection.
/// EAGAIN errors are treated as temporary - device stays open.
/// Uses write_with_timeout to avoid blocking on busy devices.
fn send_mouse_report_absolute(&self, buttons: u8, x: u16, y: u16, wheel: i8) -> Result<()> {
// Ensure device is ready
self.ensure_device(DeviceType::MouseAbsolute)?;
@@ -498,12 +515,16 @@ impl OtgBackend {
(y >> 8) as u8,
wheel as u8,
];
match file.write_all(&data) {
Ok(_) => {
match self.write_with_timeout(file, &data) {
Ok(true) => {
self.online.store(true, Ordering::Relaxed);
self.reset_error_count();
Ok(())
}
Ok(false) => {
// Timeout - silently dropped (JetKVM behavior)
Ok(())
}
Err(e) => {
let error_code = e.raw_os_error();
@@ -516,26 +537,8 @@ impl OtgBackend {
Err(Self::io_error_to_hid_error(e, "Failed to write mouse report"))
}
Some(11) => {
// EAGAIN - temporary busy, track consecutive count
self.log_throttled_error("HID absolute mouse busy (EAGAIN)");
let count = self.eagain_count.fetch_add(1, Ordering::Relaxed) + 1;
if count >= EAGAIN_OFFLINE_THRESHOLD {
// Exceeded threshold, report as offline
self.online.store(false, Ordering::Relaxed);
Err(AppError::HidError {
backend: "otg".to_string(),
reason: format!("Device busy ({} consecutive EAGAIN)", count),
error_code: "eagain".to_string(),
})
} else {
// Within threshold, return retry error (won't trigger offline event)
Err(AppError::HidError {
backend: "otg".to_string(),
reason: "Device temporarily busy".to_string(),
error_code: "eagain_retry".to_string(),
})
}
// EAGAIN after poll - should be rare, silently drop
Ok(())
}
_ => {
self.online.store(false, Ordering::Relaxed);
@@ -555,6 +558,66 @@ impl OtgBackend {
}
}
/// Send consumer control report (2 bytes: usage_lo, usage_hi)
///
/// Sends a consumer control usage code and then releases it (sends 0x0000).
fn send_consumer_report(&self, usage: u16) -> Result<()> {
// Ensure device is ready
self.ensure_device(DeviceType::ConsumerControl)?;
let mut dev = self.consumer_dev.lock();
if let Some(ref mut file) = *dev {
// Send the usage code
let data = [(usage & 0xFF) as u8, (usage >> 8) as u8];
match self.write_with_timeout(file, &data) {
Ok(true) => {
trace!("Sent consumer report: {:02X?}", data);
// Send release (0x0000)
let release = [0u8, 0u8];
let _ = self.write_with_timeout(file, &release);
self.online.store(true, Ordering::Relaxed);
self.reset_error_count();
Ok(())
}
Ok(false) => {
// Timeout - silently dropped
Ok(())
}
Err(e) => {
let error_code = e.raw_os_error();
match error_code {
Some(108) => {
self.online.store(false, Ordering::Relaxed);
debug!("Consumer control ESHUTDOWN, closing for recovery");
*dev = None;
Err(Self::io_error_to_hid_error(e, "Failed to write consumer report"))
}
Some(11) => {
// EAGAIN after poll - silently drop
Ok(())
}
_ => {
self.online.store(false, Ordering::Relaxed);
warn!("Consumer control write error: {}", e);
Err(Self::io_error_to_hid_error(e, "Failed to write consumer report"))
}
}
}
}
} else {
Err(AppError::HidError {
backend: "otg".to_string(),
reason: "Consumer control device not opened".to_string(),
error_code: "not_opened".to_string(),
})
}
}
/// Send consumer control event
pub fn send_consumer(&self, event: ConsumerEvent) -> Result<()> {
self.send_consumer_report(event.usage)
}
/// Read keyboard LED state (non-blocking)
pub fn read_led_state(&self) -> Result<Option<LedState>> {
let mut dev = self.keyboard_dev.lock();
@@ -635,6 +698,15 @@ impl HidBackend for OtgBackend {
warn!("Absolute mouse device not found: {}", self.mouse_abs_path.display());
}
// Open consumer control device (optional, may not exist on older setups)
if self.consumer_path.exists() {
let file = Self::open_device(&self.consumer_path)?;
*self.consumer_dev.lock() = Some(file);
info!("Consumer control device opened: {}", self.consumer_path.display());
} else {
debug!("Consumer control device not found: {}", self.consumer_path.display());
}
// Mark as online if all devices opened successfully
self.online.store(true, Ordering::Relaxed);
@@ -751,6 +823,7 @@ impl HidBackend for OtgBackend {
*self.keyboard_dev.lock() = None;
*self.mouse_rel_dev.lock() = None;
*self.mouse_abs_dev.lock() = None;
*self.consumer_dev.lock() = None;
// Gadget cleanup is handled by OtgService, not here
@@ -762,6 +835,10 @@ impl HidBackend for OtgBackend {
self.mouse_abs_path.exists()
}
async fn send_consumer(&self, event: ConsumerEvent) -> Result<()> {
self.send_consumer_report(event.usage)
}
fn screen_resolution(&self) -> Option<(u32, u32)> {
*self.screen_resolution.read()
}
@@ -789,6 +866,7 @@ impl Drop for OtgBackend {
*self.keyboard_dev.lock() = None;
*self.mouse_rel_dev.lock() = None;
*self.mouse_abs_dev.lock() = None;
*self.consumer_dev.lock() = None;
debug!("OtgBackend dropped, device files closed");
}
}

View File

@@ -255,6 +255,14 @@ impl MouseEvent {
pub enum HidEvent {
Keyboard(KeyboardEvent),
Mouse(MouseEvent),
Consumer(ConsumerEvent),
}
/// Consumer control event (multimedia keys)
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ConsumerEvent {
/// Consumer control usage code (e.g., 0x00CD for Play/Pause)
pub usage: u16,
}
/// USB HID keyboard report (8 bytes)

View File

@@ -122,6 +122,13 @@ async fn handle_binary_message(data: &[u8], state: &AppState) -> Result<(), Stri
.await
.map_err(|e| e.to_string())?;
}
HidChannelEvent::Consumer(consumer_event) => {
state
.hid
.send_consumer(consumer_event)
.await
.map_err(|e| e.to_string())?;
}
}
Ok(())

View File

@@ -5,14 +5,14 @@ use tracing::debug;
use super::configfs::{create_dir, create_symlink, remove_dir, remove_file, write_bytes, write_file};
use super::function::{FunctionMeta, GadgetFunction};
use super::report_desc::{KEYBOARD_WITH_LED, MOUSE_ABSOLUTE, MOUSE_RELATIVE};
use super::report_desc::{CONSUMER_CONTROL, KEYBOARD, MOUSE_ABSOLUTE, MOUSE_RELATIVE};
use crate::error::Result;
/// HID function type
#[derive(Debug, Clone)]
pub enum HidFunctionType {
/// Keyboard with LED feedback support
/// Uses 2 endpoints: IN (reports) + OUT (LED status)
/// Keyboard (no LED feedback)
/// Uses 1 endpoint: IN
Keyboard,
/// Relative mouse (traditional mouse movement)
/// Uses 1 endpoint: IN
@@ -20,33 +20,39 @@ pub enum HidFunctionType {
/// Absolute mouse (touchscreen-like positioning)
/// Uses 1 endpoint: IN
MouseAbsolute,
/// Consumer control (multimedia keys)
/// Uses 1 endpoint: IN
ConsumerControl,
}
impl HidFunctionType {
/// Get endpoints required for this function type
pub fn endpoints(&self) -> u8 {
match self {
HidFunctionType::Keyboard => 2, // IN + OUT for LED
HidFunctionType::Keyboard => 1,
HidFunctionType::MouseRelative => 1,
HidFunctionType::MouseAbsolute => 1,
HidFunctionType::ConsumerControl => 1,
}
}
/// Get HID protocol
pub fn protocol(&self) -> u8 {
match self {
HidFunctionType::Keyboard => 1, // Keyboard
HidFunctionType::MouseRelative => 2, // Mouse
HidFunctionType::MouseAbsolute => 2, // Mouse
HidFunctionType::Keyboard => 1, // Keyboard
HidFunctionType::MouseRelative => 2, // Mouse
HidFunctionType::MouseAbsolute => 2, // Mouse
HidFunctionType::ConsumerControl => 0, // None
}
}
/// Get HID subclass
pub fn subclass(&self) -> u8 {
match self {
HidFunctionType::Keyboard => 1, // Boot interface
HidFunctionType::MouseRelative => 1, // Boot interface
HidFunctionType::MouseAbsolute => 0, // No boot interface (absolute not in boot protocol)
HidFunctionType::Keyboard => 1, // Boot interface
HidFunctionType::MouseRelative => 1, // Boot interface
HidFunctionType::MouseAbsolute => 0, // No boot interface
HidFunctionType::ConsumerControl => 0, // No boot interface
}
}
@@ -56,15 +62,17 @@ impl HidFunctionType {
HidFunctionType::Keyboard => 8,
HidFunctionType::MouseRelative => 4,
HidFunctionType::MouseAbsolute => 6,
HidFunctionType::ConsumerControl => 2,
}
}
/// Get report descriptor
pub fn report_desc(&self) -> &'static [u8] {
match self {
HidFunctionType::Keyboard => KEYBOARD_WITH_LED,
HidFunctionType::Keyboard => KEYBOARD,
HidFunctionType::MouseRelative => MOUSE_RELATIVE,
HidFunctionType::MouseAbsolute => MOUSE_ABSOLUTE,
HidFunctionType::ConsumerControl => CONSUMER_CONTROL,
}
}
@@ -74,6 +82,7 @@ impl HidFunctionType {
HidFunctionType::Keyboard => "Keyboard",
HidFunctionType::MouseRelative => "Relative Mouse",
HidFunctionType::MouseAbsolute => "Absolute Mouse",
HidFunctionType::ConsumerControl => "Consumer Control",
}
}
}
@@ -117,6 +126,15 @@ impl HidFunction {
}
}
/// Create a consumer control function
pub fn consumer_control(instance: u8) -> Self {
Self {
instance,
func_type: HidFunctionType::ConsumerControl,
name: format!("hid.usb{}", instance),
}
}
/// Get function path in gadget
fn function_path(&self, gadget_path: &Path) -> PathBuf {
gadget_path.join("functions").join(self.name())
@@ -155,16 +173,6 @@ impl GadgetFunction for HidFunction {
write_file(&func_path.join("subclass"), &self.func_type.subclass().to_string())?;
write_file(&func_path.join("report_length"), &self.func_type.report_length().to_string())?;
// For keyboard, enable OUT endpoint for LED feedback
// no_out_endpoint: 0 = enable OUT endpoint, 1 = disable
if matches!(self.func_type, HidFunctionType::Keyboard) {
let no_out_path = func_path.join("no_out_endpoint");
if no_out_path.exists() || func_path.exists() {
// Try to write, ignore error if file doesn't exist yet
let _ = write_file(&no_out_path, "0");
}
}
// Write report descriptor
write_bytes(&func_path.join("report_desc"), self.func_type.report_desc())?;
@@ -205,7 +213,7 @@ mod tests {
#[test]
fn test_hid_function_types() {
assert_eq!(HidFunctionType::Keyboard.endpoints(), 2);
assert_eq!(HidFunctionType::Keyboard.endpoints(), 1);
assert_eq!(HidFunctionType::MouseRelative.endpoints(), 1);
assert_eq!(HidFunctionType::MouseAbsolute.endpoints(), 1);

View File

@@ -118,6 +118,15 @@ impl OtgGadgetManager {
Ok(device_path)
}
/// Add consumer control function (multimedia keys)
pub fn add_consumer_control(&mut self) -> Result<PathBuf> {
let func = HidFunction::consumer_control(self.hid_instance);
let device_path = func.device_path();
self.add_function(Box::new(func))?;
self.hid_instance += 1;
Ok(device_path)
}
/// Add MSD function (returns MsdFunction handle for LUN configuration)
pub fn add_msd(&mut self) -> Result<MsdFunction> {
let func = MsdFunction::new(self.msd_instance);

View File

@@ -31,5 +31,5 @@ pub use function::{FunctionMeta, GadgetFunction};
pub use hid::{HidFunction, HidFunctionType};
pub use manager::{wait_for_hid_devices, OtgGadgetManager};
pub use msd::{MsdFunction, MsdLunConfig};
pub use report_desc::{KEYBOARD_WITH_LED, MOUSE_ABSOLUTE, MOUSE_RELATIVE};
pub use report_desc::{KEYBOARD, MOUSE_ABSOLUTE, MOUSE_RELATIVE};
pub use service::{HidDevicePaths, OtgService, OtgServiceState};

View File

@@ -1,17 +1,11 @@
//! HID Report Descriptors
/// Keyboard HID Report Descriptor with LED output support
/// Keyboard HID Report Descriptor (no LED output - saves 1 endpoint)
/// Report format (8 bytes input):
/// [0] Modifier keys (8 bits)
/// [1] Reserved
/// [2-7] Key codes (6 keys)
/// LED output (1 byte):
/// Bit 0: Num Lock
/// Bit 1: Caps Lock
/// Bit 2: Scroll Lock
/// Bit 3: Compose
/// Bit 4: Kana
pub const KEYBOARD_WITH_LED: &[u8] = &[
pub const KEYBOARD: &[u8] = &[
0x05, 0x01, // Usage Page (Generic Desktop)
0x09, 0x06, // Usage (Keyboard)
0xA1, 0x01, // Collection (Application)
@@ -28,17 +22,6 @@ pub const KEYBOARD_WITH_LED: &[u8] = &[
0x95, 0x01, // Report Count (1)
0x75, 0x08, // Report Size (8)
0x81, 0x01, // Input (Constant) - Reserved byte
// LED output (5 bits)
0x95, 0x05, // Report Count (5)
0x75, 0x01, // Report Size (1)
0x05, 0x08, // Usage Page (LEDs)
0x19, 0x01, // Usage Minimum (1) - Num Lock
0x29, 0x05, // Usage Maximum (5) - Kana
0x91, 0x02, // Output (Data, Variable, Absolute) - LED bits
// LED padding (3 bits)
0x95, 0x01, // Report Count (1)
0x75, 0x03, // Report Size (3)
0x91, 0x01, // Output (Constant) - Padding
// Key array (6 bytes)
0x95, 0x06, // Report Count (6)
0x75, 0x08, // Report Size (8)
@@ -147,14 +130,33 @@ pub const MOUSE_ABSOLUTE: &[u8] = &[
0xC0, // End Collection
];
/// Consumer Control HID Report Descriptor (2 bytes report)
/// Report format:
/// [0-1] Consumer Control Usage (16-bit little-endian)
/// Supports: Play/Pause, Stop, Next/Prev Track, Mute, Volume Up/Down, etc.
pub const CONSUMER_CONTROL: &[u8] = &[
0x05, 0x0C, // Usage Page (Consumer)
0x09, 0x01, // Usage (Consumer Control)
0xA1, 0x01, // Collection (Application)
0x15, 0x00, // Logical Minimum (0)
0x26, 0xFF, 0x03, // Logical Maximum (1023)
0x19, 0x00, // Usage Minimum (0)
0x2A, 0xFF, 0x03, // Usage Maximum (1023)
0x75, 0x10, // Report Size (16)
0x95, 0x01, // Report Count (1)
0x81, 0x00, // Input (Data, Array)
0xC0, // End Collection
];
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_report_descriptor_sizes() {
assert!(!KEYBOARD_WITH_LED.is_empty());
assert!(!KEYBOARD.is_empty());
assert!(!MOUSE_RELATIVE.is_empty());
assert!(!MOUSE_ABSOLUTE.is_empty());
assert!(!CONSUMER_CONTROL.is_empty());
}
}

View File

@@ -39,6 +39,7 @@ pub struct HidDevicePaths {
pub keyboard: PathBuf,
pub mouse_relative: PathBuf,
pub mouse_absolute: PathBuf,
pub consumer: Option<PathBuf>,
}
impl Default for HidDevicePaths {
@@ -47,6 +48,7 @@ impl Default for HidDevicePaths {
keyboard: PathBuf::from("/dev/hidg0"),
mouse_relative: PathBuf::from("/dev/hidg1"),
mouse_absolute: PathBuf::from("/dev/hidg2"),
consumer: Some(PathBuf::from("/dev/hidg3")),
}
}
}
@@ -353,16 +355,18 @@ impl OtgService {
manager.add_keyboard(),
manager.add_mouse_relative(),
manager.add_mouse_absolute(),
manager.add_consumer_control(),
) {
(Ok(kb), Ok(rel), Ok(abs)) => {
(Ok(kb), Ok(rel), Ok(abs), Ok(consumer)) => {
hid_paths = Some(HidDevicePaths {
keyboard: kb,
mouse_relative: rel,
mouse_absolute: abs,
consumer: Some(consumer),
});
debug!("HID functions added to gadget");
}
(Err(e), _, _) | (_, Err(e), _) | (_, _, Err(e)) => {
(Err(e), _, _, _) | (_, Err(e), _, _) | (_, _, Err(e), _) | (_, _, _, Err(e)) => {
let error = format!("Failed to add HID functions: {}", e);
let mut state = self.state.write().await;
state.error = Some(error.clone());

View File

@@ -256,6 +256,11 @@ impl WsHidHandler {
HidChannelEvent::Mouse(ms_event) => {
hid.send_mouse(ms_event).await.map_err(|e| e.to_string())?;
}
HidChannelEvent::Consumer(consumer_event) => {
hid.send_consumer(consumer_event)
.await
.map_err(|e| e.to_string())?;
}
}
client.events_processed.fetch_add(1, Ordering::Relaxed);

View File

@@ -226,6 +226,11 @@ impl PeerConnection {
debug!("Failed to send mouse event: {}", e);
}
}
HidChannelEvent::Consumer(consumer_event) => {
if let Err(e) = hid.send_consumer(consumer_event).await {
debug!("Failed to send consumer event: {}", e);
}
}
}
}
})

View File

@@ -437,6 +437,11 @@ impl UniversalSession {
debug!("Failed to send mouse event: {}", e);
}
}
HidChannelEvent::Consumer(consumer_event) => {
if let Err(e) = hid.send_consumer(consumer_event).await {
debug!("Failed to send consumer event: {}", e);
}
}
}
}
})

View File

@@ -355,6 +355,12 @@ export const hidApi = {
reset: () =>
request<{ success: boolean }>('/hid/reset', { method: 'POST' }),
consumer: async (usage: number) => {
await ensureHidConnection()
await hidWs.sendConsumer({ usage })
return { success: true }
},
// WebSocket connection management
connectWebSocket: () => hidWs.connect(),
disconnectWebSocket: () => hidWs.disconnect(),

View File

@@ -6,8 +6,6 @@ import { cn } from '@/lib/utils'
const props = defineProps<{
pressedKeys?: string[]
capsLock?: boolean
numLock?: boolean
scrollLock?: boolean
mousePosition?: { x: number; y: number }
debugMode?: boolean
compact?: boolean
@@ -15,34 +13,39 @@ const props = defineProps<{
const { t } = useI18n()
// Key name mapping for friendly display
const keyNameMap: Record<string, string> = {
MetaLeft: 'Win', MetaRight: 'Win',
ControlLeft: 'Ctrl', ControlRight: 'Ctrl',
ShiftLeft: 'Shift', ShiftRight: 'Shift',
AltLeft: 'Alt', AltRight: 'Alt',
CapsLock: 'Caps', NumLock: 'Num', ScrollLock: 'Scroll',
Backspace: 'Back', Delete: 'Del',
ArrowUp: '↑', ArrowDown: '↓', ArrowLeft: '←', ArrowRight: '→',
Escape: 'Esc', Enter: 'Enter', Tab: 'Tab', Space: 'Space',
PageUp: 'PgUp', PageDown: 'PgDn',
Insert: 'Ins', Home: 'Home', End: 'End',
}
const keysDisplay = computed(() => {
if (!props.pressedKeys || props.pressedKeys.length === 0) return ''
return props.pressedKeys.join(', ')
return props.pressedKeys
.map(key => keyNameMap[key] || key.replace(/^(Key|Digit)/, ''))
.join(', ')
})
// Has any LED active
const hasActiveLed = computed(() => props.capsLock || props.numLock || props.scrollLock)
</script>
<template>
<div class="w-full border-t border-slate-200 bg-white dark:border-slate-800 dark:bg-slate-900">
<!-- Compact mode for small screens -->
<div v-if="compact" class="flex items-center justify-between text-xs px-2 py-0.5">
<!-- LED indicators only in compact mode -->
<!-- LED indicator only in compact mode -->
<div class="flex items-center gap-1">
<span
v-if="capsLock"
class="px-1.5 py-0.5 bg-primary/10 text-primary rounded text-[10px] font-medium"
>C</span>
<span
v-if="numLock"
class="px-1.5 py-0.5 bg-primary/10 text-primary rounded text-[10px] font-medium"
>N</span>
<span
v-if="scrollLock"
class="px-1.5 py-0.5 bg-primary/10 text-primary rounded text-[10px] font-medium"
>S</span>
<span v-if="!hasActiveLed" class="text-muted-foreground/40 text-[10px]">-</span>
<span v-else class="text-muted-foreground/40 text-[10px]">-</span>
</div>
<!-- Keys in compact mode -->
<div v-if="keysDisplay" class="text-[10px] text-muted-foreground truncate max-w-[150px]">
@@ -67,8 +70,8 @@ const hasActiveLed = computed(() => props.capsLock || props.numLock || props.scr
</div>
</div>
<!-- Right side: Keyboard LED states -->
<div class="flex items-center divide-x divide-slate-200 dark:divide-slate-700 shrink-0">
<!-- Right side: Caps Lock LED state -->
<div class="flex items-center shrink-0">
<div
:class="cn(
'px-2 py-1 select-none transition-colors',
@@ -78,24 +81,6 @@ const hasActiveLed = computed(() => props.capsLock || props.numLock || props.scr
<span class="hidden sm:inline">{{ t('infobar.caps') }}</span>
<span class="sm:hidden">C</span>
</div>
<div
:class="cn(
'px-2 py-1 select-none transition-colors',
numLock ? 'text-foreground font-medium bg-primary/5' : 'text-muted-foreground/40'
)"
>
<span class="hidden sm:inline">{{ t('infobar.num') }}</span>
<span class="sm:hidden">N</span>
</div>
<div
:class="cn(
'px-2 py-1 select-none transition-colors',
scrollLock ? 'text-foreground font-medium bg-primary/5' : 'text-muted-foreground/40'
)"
>
<span class="hidden sm:inline">{{ t('infobar.scroll') }}</span>
<span class="sm:hidden">S</span>
</div>
</div>
</div>
</div>

View File

@@ -6,10 +6,18 @@ import 'simple-keyboard/build/css/index.css'
import { hidApi } from '@/api'
import {
keys,
consumerKeys,
latchingKeys,
modifiers,
type KeyName,
type ConsumerKeyName,
} from '@/lib/keyboardMappings'
import {
type KeyboardOsType,
osBottomRows,
mediaKeys,
mediaKeyLabels,
} from '@/lib/keyboardLayouts'
const props = defineProps<{
visible: boolean
@@ -27,6 +35,7 @@ const { t } = useI18n()
// State
const isAttached = ref(props.attached ?? true)
const selectedOs = ref<KeyboardOsType>('windows')
// Keyboard instances
const mainKeyboard = ref<Keyboard | null>(null)
@@ -65,6 +74,9 @@ const position = ref({ x: 100, y: 100 })
// Unique ID for this keyboard instance
const keyboardId = ref(`kb-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`)
// Get bottom row based on selected OS
const getBottomRow = () => osBottomRows[selectedOs.value].join(' ')
// Keyboard layouts - matching JetKVM style
const keyboardLayout = {
main: {
@@ -103,91 +115,139 @@ const keyboardLayout = {
}
// Key display mapping with Unicode symbols (JetKVM style)
const keyDisplayMap: Record<string, string> = {
// Macros - compact format
CtrlAltDelete: 'Ctrl+Alt+Del',
AltMetaEscape: 'Alt+Meta+Esc',
CtrlAltBackspace: 'Ctrl+Alt+Bksp',
const keyDisplayMap = computed<Record<string, string>>(() => {
// OS-specific Meta key labels
const metaLabel = selectedOs.value === 'windows' ? '⊞ Win'
: selectedOs.value === 'mac' ? '⌘ Cmd' : 'Meta'
// Modifiers with symbols
ControlLeft: '^Ctrl',
ControlRight: 'Ctrl^',
ShiftLeft: '⇧Shift',
ShiftRight: 'Shift⇧',
AltLeft: '⌥Alt',
AltGr: 'AltGr',
MetaLeft: '⌘Meta',
MetaRight: 'Meta⌘',
Menu: 'Menu',
return {
// Macros - compact format
CtrlAltDelete: 'Ctrl+Alt+Del',
AltMetaEscape: 'Alt+Meta+Esc',
CtrlAltBackspace: 'Ctrl+Alt+Bksp',
// Special keys with symbols
Escape: 'Esc',
Backspace: '',
Tab: '⇥Tab',
CapsLock: '⇪Caps',
Enter: '',
Space: ' ',
// Modifiers - simplified
ControlLeft: 'Ctrl',
ControlRight: 'Ctrl',
ShiftLeft: 'Shift',
ShiftRight: 'Shift',
AltLeft: 'Alt',
AltRight: 'Alt',
AltGr: 'AltGr',
MetaLeft: metaLabel,
MetaRight: metaLabel,
Menu: 'Menu',
// Navigation with symbols
Insert: 'Ins',
Delete: '⌫Del',
Home: 'Home',
End: 'End',
PageUp: 'PgUp',
PageDown: 'PgDn',
// Special keys
Escape: 'Esc',
Backspace: '⌫',
Tab: 'Tab',
CapsLock: 'Caps',
Enter: 'Enter',
Space: ' ',
// Arrows
ArrowUp: '',
ArrowDown: '',
ArrowLeft: '',
ArrowRight: '',
// Navigation
Insert: 'Ins',
Delete: 'Del',
Home: 'Home',
End: 'End',
PageUp: 'PgUp',
PageDown: 'PgDn',
// Control cluster
PrintScreen: 'PrtSc',
ScrollLock: 'ScrLk',
Pause: 'Pause',
// Arrows
ArrowUp: '',
ArrowDown: '',
ArrowLeft: '',
ArrowRight: '→',
// Function keys
F1: 'F1', F2: 'F2', F3: 'F3', F4: 'F4',
F5: 'F5', F6: 'F6', F7: 'F7', F8: 'F8',
F9: 'F9', F10: 'F10', F11: 'F11', F12: 'F12',
// Control cluster
PrintScreen: 'PrtSc',
ScrollLock: 'ScrLk',
Pause: 'Pause',
// Letters
KeyA: 'a', KeyB: 'b', KeyC: 'c', KeyD: 'd', KeyE: 'e',
KeyF: 'f', KeyG: 'g', KeyH: 'h', KeyI: 'i', KeyJ: 'j',
KeyK: 'k', KeyL: 'l', KeyM: 'm', KeyN: 'n', KeyO: 'o',
KeyP: 'p', KeyQ: 'q', KeyR: 'r', KeyS: 's', KeyT: 't',
KeyU: 'u', KeyV: 'v', KeyW: 'w', KeyX: 'x', KeyY: 'y',
KeyZ: 'z',
// Function keys
F1: 'F1', F2: 'F2', F3: 'F3', F4: 'F4',
F5: 'F5', F6: 'F6', F7: 'F7', F8: 'F8',
F9: 'F9', F10: 'F10', F11: 'F11', F12: 'F12',
// Capital letters
'(KeyA)': 'A', '(KeyB)': 'B', '(KeyC)': 'C', '(KeyD)': 'D', '(KeyE)': 'E',
'(KeyF)': 'F', '(KeyG)': 'G', '(KeyH)': 'H', '(KeyI)': 'I', '(KeyJ)': 'J',
'(KeyK)': 'K', '(KeyL)': 'L', '(KeyM)': 'M', '(KeyN)': 'N', '(KeyO)': 'O',
'(KeyP)': 'P', '(KeyQ)': 'Q', '(KeyR)': 'R', '(KeyS)': 'S', '(KeyT)': 'T',
'(KeyU)': 'U', '(KeyV)': 'V', '(KeyW)': 'W', '(KeyX)': 'X', '(KeyY)': 'Y',
'(KeyZ)': 'Z',
// Letters
KeyA: 'a', KeyB: 'b', KeyC: 'c', KeyD: 'd', KeyE: 'e',
KeyF: 'f', KeyG: 'g', KeyH: 'h', KeyI: 'i', KeyJ: 'j',
KeyK: 'k', KeyL: 'l', KeyM: 'm', KeyN: 'n', KeyO: 'o',
KeyP: 'p', KeyQ: 'q', KeyR: 'r', KeyS: 's', KeyT: 't',
KeyU: 'u', KeyV: 'v', KeyW: 'w', KeyX: 'x', KeyY: 'y',
KeyZ: 'z',
// Numbers
Digit1: '1', Digit2: '2', Digit3: '3', Digit4: '4', Digit5: '5',
Digit6: '6', Digit7: '7', Digit8: '8', Digit9: '9', Digit0: '0',
// Capital letters
'(KeyA)': 'A', '(KeyB)': 'B', '(KeyC)': 'C', '(KeyD)': 'D', '(KeyE)': 'E',
'(KeyF)': 'F', '(KeyG)': 'G', '(KeyH)': 'H', '(KeyI)': 'I', '(KeyJ)': 'J',
'(KeyK)': 'K', '(KeyL)': 'L', '(KeyM)': 'M', '(KeyN)': 'N', '(KeyO)': 'O',
'(KeyP)': 'P', '(KeyQ)': 'Q', '(KeyR)': 'R', '(KeyS)': 'S', '(KeyT)': 'T',
'(KeyU)': 'U', '(KeyV)': 'V', '(KeyW)': 'W', '(KeyX)': 'X', '(KeyY)': 'Y',
'(KeyZ)': 'Z',
// Shifted Numbers
'(Digit1)': '!', '(Digit2)': '@', '(Digit3)': '#', '(Digit4)': '$', '(Digit5)': '%',
'(Digit6)': '^', '(Digit7)': '&', '(Digit8)': '*', '(Digit9)': '(', '(Digit0)': ')',
// Numbers
Digit1: '1', Digit2: '2', Digit3: '3', Digit4: '4', Digit5: '5',
Digit6: '6', Digit7: '7', Digit8: '8', Digit9: '9', Digit0: '0',
// Symbols
Minus: '-', '(Minus)': '_',
Equal: '=', '(Equal)': '+',
BracketLeft: '[', '(BracketLeft)': '{',
BracketRight: ']', '(BracketRight)': '}',
Backslash: '\\', '(Backslash)': '|',
Semicolon: ';', '(Semicolon)': ':',
Quote: "'", '(Quote)': '"',
Comma: ',', '(Comma)': '<',
Period: '.', '(Period)': '>',
Slash: '/', '(Slash)': '?',
Backquote: '`', '(Backquote)': '~',
// Shifted Numbers
'(Digit1)': '!', '(Digit2)': '@', '(Digit3)': '#', '(Digit4)': '$', '(Digit5)': '%',
'(Digit6)': '^', '(Digit7)': '&', '(Digit8)': '*', '(Digit9)': '(', '(Digit0)': ')',
// Symbols
Minus: '-', '(Minus)': '_',
Equal: '=', '(Equal)': '+',
BracketLeft: '[', '(BracketLeft)': '{',
BracketRight: ']', '(BracketRight)': '}',
Backslash: '\\', '(Backslash)': '|',
Semicolon: ';', '(Semicolon)': ':',
Quote: "'", '(Quote)': '"',
Comma: ',', '(Comma)': '<',
Period: '.', '(Period)': '>',
Slash: '/', '(Slash)': '?',
Backquote: '`', '(Backquote)': '~',
}
})
// Handle media key press (Consumer Control)
async function onMediaKeyPress(key: string) {
if (key in consumerKeys) {
const usage = consumerKeys[key as ConsumerKeyName]
try {
await hidApi.consumer(usage)
} catch (err) {
console.error('[VirtualKeyboard] Media key send failed:', err)
}
}
}
// Switch OS layout
function switchOsLayout(os: KeyboardOsType) {
selectedOs.value = os
// Save preference to localStorage
localStorage.setItem('vkb-os-layout', os)
// Update keyboard layout
updateKeyboardLayout()
}
// Update keyboard layout based on selected OS
function updateKeyboardLayout() {
const bottomRow = getBottomRow()
const newLayout = {
...keyboardLayout.main,
default: [
...keyboardLayout.main.default.slice(0, -1),
bottomRow,
],
shift: [
...keyboardLayout.main.shift.slice(0, -1),
bottomRow,
],
}
mainKeyboard.value?.setOptions({ layout: newLayout, display: keyDisplayMap.value })
controlKeyboard.value?.setOptions({ display: keyDisplayMap.value })
arrowsKeyboard.value?.setOptions({ display: keyDisplayMap.value })
updateKeyboardButtonTheme()
}
// Key press handler
@@ -364,7 +424,7 @@ function initKeyboards() {
mainKeyboard.value = new Keyboard(mainEl, {
layout: keyboardLayout.main,
layoutName: layoutName.value,
display: keyDisplayMap,
display: keyDisplayMap.value,
theme: 'hg-theme-default hg-layout-default vkb-keyboard',
onKeyPress: onKeyDown,
onKeyReleased: onKeyUp,
@@ -385,7 +445,7 @@ function initKeyboards() {
controlKeyboard.value = new Keyboard(controlEl, {
layout: keyboardLayout.control,
layoutName: 'default',
display: keyDisplayMap,
display: keyDisplayMap.value,
theme: 'hg-theme-default hg-layout-default vkb-keyboard',
onKeyPress: onKeyDown,
onKeyReleased: onKeyUp,
@@ -400,7 +460,7 @@ function initKeyboards() {
arrowsKeyboard.value = new Keyboard(arrowsEl, {
layout: keyboardLayout.arrows,
layoutName: 'default',
display: keyDisplayMap,
display: keyDisplayMap.value,
theme: 'hg-theme-default hg-layout-default vkb-keyboard',
onKeyPress: onKeyDown,
onKeyReleased: onKeyUp,
@@ -504,6 +564,12 @@ watch(() => props.attached, (value) => {
})
onMounted(() => {
// Load saved OS layout preference
const savedOs = localStorage.getItem('vkb-os-layout') as KeyboardOsType | null
if (savedOs && ['windows', 'mac', 'android'].includes(savedOs)) {
selectedOs.value = savedOs
}
document.addEventListener('mousemove', onDrag)
document.addEventListener('touchmove', onDrag)
document.addEventListener('mouseup', endDrag)
@@ -539,9 +605,22 @@ onUnmounted(() => {
@mousedown="startDrag"
@touchstart="startDrag"
>
<button class="vkb-btn" @click="toggleAttached">
{{ isAttached ? t('virtualKeyboard.detach') : t('virtualKeyboard.attach') }}
</button>
<div class="vkb-header-left">
<button class="vkb-btn" @click="toggleAttached">
{{ isAttached ? t('virtualKeyboard.detach') : t('virtualKeyboard.attach') }}
</button>
<div class="vkb-os-selector">
<button
v-for="os in (['windows', 'mac', 'android'] as KeyboardOsType[])"
:key="os"
class="vkb-os-btn"
:class="{ 'vkb-os-btn--active': selectedOs === os }"
@click.stop="switchOsLayout(os)"
>
{{ os === 'windows' ? 'Win' : os === 'mac' ? 'Mac' : 'Android' }}
</button>
</div>
</div>
<span class="vkb-title">{{ t('virtualKeyboard.title') }}</span>
<button class="vkb-btn" @click="close">
{{ t('virtualKeyboard.hide') }}
@@ -550,10 +629,23 @@ onUnmounted(() => {
<!-- Keyboard body -->
<div class="vkb-body">
<div :id="`${keyboardId}-main`" class="kb-main-container"></div>
<div class="vkb-side">
<div :id="`${keyboardId}-control`" class="kb-control-container"></div>
<div :id="`${keyboardId}-arrows`" class="kb-arrows-container"></div>
<!-- Media keys row -->
<div class="vkb-media-row">
<button
v-for="key in mediaKeys"
:key="key"
class="vkb-media-btn"
@click="onMediaKeyPress(key)"
>
{{ mediaKeyLabels[key] || key }}
</button>
</div>
<div class="vkb-keyboards">
<div :id="`${keyboardId}-main`" class="kb-main-container"></div>
<div class="vkb-side">
<div :id="`${keyboardId}-control`" class="kb-control-container"></div>
<div :id="`${keyboardId}-arrows`" class="kb-arrows-container"></div>
</div>
</div>
</div>
</div>
@@ -820,6 +912,7 @@ html.dark .hg-theme-default .hg-button.down-key,
border-bottom: 1px solid #e5e7eb;
background: #f9fafb;
min-height: 28px;
position: relative;
}
:global(.dark .vkb-header) {
@@ -832,7 +925,16 @@ html.dark .hg-theme-default .hg-button.down-key,
border-radius: 8px 8px 0 0;
}
.vkb-header-left {
display: flex;
align-items: center;
gap: 8px;
}
.vkb-title {
position: absolute;
left: 50%;
transform: translateX(-50%);
font-size: 12px;
font-weight: 500;
color: #374151;
@@ -868,10 +970,57 @@ html.dark .hg-theme-default .hg-button.down-key,
background: #4b5563;
}
/* OS selector */
.vkb-os-selector {
display: flex;
gap: 2px;
background: #e5e7eb;
border-radius: 4px;
padding: 2px;
}
:global(.dark .vkb-os-selector) {
background: #374151;
}
.vkb-os-btn {
padding: 2px 8px;
font-size: 10px;
font-weight: 500;
color: #6b7280;
background: transparent;
border: none;
border-radius: 3px;
cursor: pointer;
}
.vkb-os-btn:hover {
color: #374151;
}
.vkb-os-btn--active {
background: white;
color: #374151;
box-shadow: 0 1px 2px rgb(0 0 0 / 0.1);
}
:global(.dark .vkb-os-btn) {
color: #9ca3af;
}
:global(.dark .vkb-os-btn:hover) {
color: #d1d5db;
}
:global(.dark .vkb-os-btn--active) {
background: #4b5563;
color: #f9fafb;
}
/* Keyboard body */
.vkb-body {
display: flex;
flex-direction: row;
flex-direction: column;
padding: 8px;
gap: 8px;
background: #f3f4f6;
@@ -885,6 +1034,55 @@ html.dark .hg-theme-default .hg-button.down-key,
border-radius: 0 0 8px 8px;
}
/* Media keys row */
.vkb-media-row {
display: flex;
gap: 4px;
justify-content: center;
padding-bottom: 4px;
border-bottom: 1px solid #e5e7eb;
}
:global(.dark .vkb-media-row) {
border-color: #374151;
}
.vkb-media-btn {
padding: 4px 12px;
font-size: 16px;
background: white;
border: 1px solid #d1d5db;
border-radius: 6px;
cursor: pointer;
min-width: 40px;
}
.vkb-media-btn:hover {
background: #f3f4f6;
}
.vkb-media-btn:active {
background: #3b82f6;
color: white;
}
:global(.dark .vkb-media-btn) {
background: #374151;
border-color: #4b5563;
color: #f9fafb;
}
:global(.dark .vkb-media-btn:hover) {
background: #4b5563;
}
/* Keyboards container */
.vkb-keyboards {
display: flex;
flex-direction: row;
gap: 8px;
}
.kb-main-container {
flex: 1;
}
@@ -903,7 +1101,7 @@ html.dark .hg-theme-default .hg-button.down-key,
/* Responsive */
@media (max-width: 900px) {
.vkb-body {
.vkb-keyboards {
flex-direction: column;
}

View File

@@ -5,15 +5,17 @@ import { ref, onUnmounted } from 'vue'
import {
type HidKeyboardEvent,
type HidMouseEvent,
type HidConsumerEvent,
encodeKeyboardEvent,
encodeMouseEvent,
encodeConsumerEvent,
RESP_OK,
RESP_ERR_HID_UNAVAILABLE,
RESP_ERR_INVALID_MESSAGE,
} from '@/types/hid'
import { buildWsUrl, WS_RECONNECT_DELAY } from '@/types/websocket'
export type { HidKeyboardEvent, HidMouseEvent }
export type { HidKeyboardEvent, HidMouseEvent, HidConsumerEvent }
let wsInstance: WebSocket | null = null
const connected = ref(false)
@@ -213,6 +215,23 @@ function sendMouse(event: HidMouseEvent): Promise<void> {
})
}
// Send consumer control event (multimedia keys)
function sendConsumer(event: HidConsumerEvent): Promise<void> {
return new Promise((resolve, reject) => {
if (!wsInstance || wsInstance.readyState !== WebSocket.OPEN) {
reject(new Error('WebSocket not connected'))
return
}
try {
wsInstance.send(encodeConsumerEvent(event))
resolve()
} catch (err) {
reject(err)
}
})
}
export function useHidWebSocket() {
onUnmounted(() => {
// Don't disconnect on component unmount - WebSocket is shared
@@ -229,6 +248,7 @@ export function useHidWebSocket() {
disconnect,
sendKeyboard,
sendMouse,
sendConsumer,
}
}

View File

@@ -550,6 +550,10 @@ export default {
hide: 'Hide',
show: 'Show Virtual Keyboard',
layoutSelect: 'Keyboard Layout',
osWindows: 'Windows',
osMac: 'Mac',
osAndroid: 'Android',
mediaKeys: 'Media Keys',
},
config: {
applied: 'Configuration applied',

View File

@@ -550,6 +550,10 @@ export default {
hide: '隐藏',
show: '显示虚拟键盘',
layoutSelect: '键盘布局',
osWindows: 'Windows',
osMac: 'Mac',
osAndroid: 'Android',
mediaKeys: '多媒体键',
},
config: {
applied: '配置已应用',

View File

@@ -17,9 +17,71 @@ export interface KeyboardLayout {
}
control: string[][]
arrows: string[][]
media: string[] // Media keys row
}
}
// OS-specific keyboard layout type
export type KeyboardOsType = 'windows' | 'mac' | 'android'
// Bottom row layouts for different OS
export const osBottomRows: Record<KeyboardOsType, string[]> = {
// Windows: Ctrl - Win - Alt - Space - Alt - Win - Menu - Ctrl
windows: ['ControlLeft', 'MetaLeft', 'AltLeft', 'Space', 'AltRight', 'MetaRight', 'Menu', 'ControlRight'],
// Mac: Ctrl - Option - Cmd - Space - Cmd - Option - Ctrl
mac: ['ControlLeft', 'AltLeft', 'MetaLeft', 'Space', 'MetaRight', 'AltRight', 'ControlRight'],
// Android: simplified layout
android: ['ControlLeft', 'AltLeft', 'Space', 'AltRight', 'ControlRight'],
}
// OS-specific modifier display names
export const osModifierLabels: Record<KeyboardOsType, Record<string, string>> = {
windows: {
ControlLeft: '^Ctrl',
ControlRight: 'Ctrl^',
MetaLeft: '⊞Win',
MetaRight: 'Win⊞',
AltLeft: 'Alt',
AltRight: 'Alt',
AltGr: 'AltGr',
Menu: 'Menu',
},
mac: {
ControlLeft: '^Ctrl',
ControlRight: 'Ctrl^',
MetaLeft: '⌘Cmd',
MetaRight: 'Cmd⌘',
AltLeft: '⌥Opt',
AltRight: 'Opt⌥',
AltGr: '⌥Opt',
Menu: 'Menu',
},
android: {
ControlLeft: 'Ctrl',
ControlRight: 'Ctrl',
MetaLeft: 'Meta',
MetaRight: 'Meta',
AltLeft: 'Alt',
AltRight: 'Alt',
AltGr: 'Alt',
Menu: 'Menu',
},
}
// Media keys (Consumer Control)
export const mediaKeys = ['PrevTrack', 'PlayPause', 'NextTrack', 'Stop', 'Mute', 'VolumeDown', 'VolumeUp']
// Media key display names
export const mediaKeyLabels: Record<string, string> = {
PlayPause: '⏯',
Stop: '⏹',
NextTrack: '⏭',
PrevTrack: '⏮',
Mute: '🔇',
VolumeUp: '🔊',
VolumeDown: '🔉',
}
// English US Layout
export const enUSLayout: KeyboardLayout = {
id: 'en-US',
@@ -153,6 +215,7 @@ export const enUSLayout: KeyboardLayout = {
['ArrowUp'],
['ArrowLeft', 'ArrowDown', 'ArrowRight'],
],
media: ['PrevTrack', 'PlayPause', 'NextTrack', 'Stop', 'Mute', 'VolumeDown', 'VolumeUp'],
},
}

View File

@@ -136,12 +136,6 @@ export const keys = {
F23: 0x72,
F24: 0x73,
// Media/System keys (Consumer Control)
// Note: These are Consumer Control keys, may need special handling
Mute: 0x7f,
VolumeUp: 0x80,
VolumeDown: 0x81,
// Modifiers (these are special - HID codes 0xE0-0xE7)
ControlLeft: 0xe0,
ShiftLeft: 0xe1,
@@ -156,6 +150,20 @@ export const keys = {
export type KeyName = keyof typeof keys
// Consumer Control Usage codes (for multimedia keys)
// These are sent via a separate Consumer Control HID report
export const consumerKeys = {
PlayPause: 0x00cd,
Stop: 0x00b7,
NextTrack: 0x00b5,
PrevTrack: 0x00b6,
Mute: 0x00e2,
VolumeUp: 0x00e9,
VolumeDown: 0x00ea,
} as const
export type ConsumerKeyName = keyof typeof consumerKeys
// Modifier bitmasks for HID report byte 0
export const modifiers = {
ControlLeft: 0x01,

View File

@@ -22,9 +22,15 @@ export interface HidMouseEvent {
scroll?: number
}
/** Consumer control event for HID input (multimedia keys) */
export interface HidConsumerEvent {
usage: number // Consumer Control Usage code (e.g., 0x00CD for Play/Pause)
}
// Binary message constants (must match datachannel.rs / ws_hid.rs)
export const MSG_KEYBOARD = 0x01
export const MSG_MOUSE = 0x02
export const MSG_CONSUMER = 0x03
// Keyboard event types
export const KB_EVENT_DOWN = 0x00
@@ -107,3 +113,15 @@ export function encodeMouseEvent(event: HidMouseEvent): ArrayBuffer {
return buffer
}
/** Encode consumer control event to binary format (3 bytes) */
export function encodeConsumerEvent(event: HidConsumerEvent): ArrayBuffer {
const buffer = new ArrayBuffer(3)
const view = new DataView(buffer)
view.setUint8(0, MSG_CONSUMER)
// Usage code as u16 LE
view.setUint16(1, event.usage, true)
return buffer
}

View File

@@ -98,8 +98,6 @@ const mouseMode = ref<'absolute' | 'relative'>('absolute')
const pressedKeys = ref<string[]>([])
const keyboardLed = ref({
capsLock: false,
numLock: false,
scrollLock: false,
})
const mousePosition = ref({ x: 0, y: 0 })
const lastMousePosition = ref({ x: 0, y: 0 }) // Track last position for relative mode
@@ -1284,8 +1282,6 @@ function handleKeyDown(e: KeyboardEvent) {
}
keyboardLed.value.capsLock = e.getModifierState('CapsLock')
keyboardLed.value.numLock = e.getModifierState('NumLock')
keyboardLed.value.scrollLock = e.getModifierState('ScrollLock')
const modifiers = {
ctrl: e.ctrlKey,
@@ -1482,6 +1478,10 @@ function handleVirtualKeyDown(key: string) {
if (!pressedKeys.value.includes(key)) {
pressedKeys.value = [...pressedKeys.value, key]
}
// Toggle CapsLock state when virtual keyboard presses CapsLock
if (key === 'CapsLock') {
keyboardLed.value.capsLock = !keyboardLed.value.capsLock
}
}
function handleVirtualKeyUp(key: string) {
@@ -1910,8 +1910,6 @@ onUnmounted(() => {
<InfoBar
:pressed-keys="pressedKeys"
:caps-lock="keyboardLed.capsLock"
:num-lock="keyboardLed.numLock"
:scroll-lock="keyboardLed.scrollLock"
:mouse-position="mousePosition"
:debug-mode="false"
/>