very effective binary mouse protocol

kvmd/apps/kvmd/api/hid.py

@@ -23,6 +23,7 @@
 import os
 import stat
 import functools
+import struct
 
 from typing import Callable
 
@@ -153,6 +154,38 @@ class HidApi:
 
     # =====
 
+    @exposed_ws(3, binary=True)
+    async def __ws_bin_mouse_move_handler(self, _: WsSession, data: bytes) -> None:
+        try:
+            (to_x, to_y) = struct.unpack(">hh", data)
+            to_x = valid_hid_mouse_move(to_x)
+            to_y = valid_hid_mouse_move(to_y)
+        except Exception:
+            return
+        self.__send_mouse_move_event(to_x, to_y)
+
+    @exposed_ws(4, binary=True)
+    async def __ws_bin_mouse_relative_handler(self, _: WsSession, data: bytes) -> None:
+        self.__process_ws_bin_delta_request(data, self.__hid.send_mouse_relative_event)
+
+    @exposed_ws(5, binary=True)
+    async def __ws_bin_mouse_wheel_handler(self, _: WsSession, data: bytes) -> None:
+        self.__process_ws_bin_delta_request(data, self.__hid.send_mouse_wheel_event)
+
+    def __process_ws_bin_delta_request(self, data: bytes, handler: Callable[[int, int], None]) -> None:
+        try:
+            squash = valid_bool(data[0])
+            data = data[1:]
+            deltas: list[tuple[int, int]] = []
+            for index in range(0, len(data), 2):
+                (delta_x, delta_y) = struct.unpack(">bb", data[index:index + 2])
+                deltas.append((valid_hid_mouse_delta(delta_x), valid_hid_mouse_delta(delta_y)))
+        except Exception:
+            return
+        self.__send_mouse_delta_event(deltas, squash, handler)
+
+    # =====
+
     @exposed_ws("key")
     async def __ws_key_handler(self, _: WsSession, event: dict) -> None:
         try:
@@ -179,17 +212,17 @@ class HidApi:
             to_y = valid_hid_mouse_move(event["to"]["y"])
         except Exception:
             return
-        self.__send_mouse_move_event_remapped(to_x, to_y)
+        self.__send_mouse_move_event(to_x, to_y)
 
     @exposed_ws("mouse_relative")
     async def __ws_mouse_relative_handler(self, _: WsSession, event: dict) -> None:
-        self.__process_delta_ws_request(event, self.__hid.send_mouse_relative_event)
+        self.__process_ws_delta_event(event, self.__hid.send_mouse_relative_event)
 
     @exposed_ws("mouse_wheel")
     async def __ws_mouse_wheel_handler(self, _: WsSession, event: dict) -> None:
-        self.__process_delta_ws_request(event, self.__hid.send_mouse_wheel_event)
+        self.__process_ws_delta_event(event, self.__hid.send_mouse_wheel_event)
 
-    def __process_delta_ws_request(self, event: dict, handler: Callable[[int, int], None]) -> None:
+    def __process_ws_delta_event(self, event: dict, handler: Callable[[int, int], None]) -> None:
         try:
             raw_delta = event["delta"]
             deltas = [
@@ -199,19 +232,7 @@ class HidApi:
             squash = valid_bool(event.get("squash", False))
         except Exception:
             return
-        if squash:
-            prev = (0, 0)
-            for cur in deltas:
-                if abs(prev[0] + cur[0]) > 127 or abs(prev[1] + cur[1]) > 127:
-                    handler(*prev)
-                    prev = cur
-                else:
-                    prev = (prev[0] + cur[0], prev[1] + cur[1])
-            if prev[0] or prev[1]:
-                handler(*prev)
-        else:
-            for xy in deltas:
-                handler(*xy)
+        self.__send_mouse_delta_event(deltas, squash, handler)
 
     # =====
 
@@ -241,26 +262,49 @@ class HidApi:
     async def __events_send_mouse_move_handler(self, request: Request) -> Response:
         to_x = valid_hid_mouse_move(request.query.get("to_x"))
         to_y = valid_hid_mouse_move(request.query.get("to_y"))
-        self.__send_mouse_move_event_remapped(to_x, to_y)
+        self.__send_mouse_move_event(to_x, to_y)
         return make_json_response()
 
     @exposed_http("POST", "/hid/events/send_mouse_relative")
     async def __events_send_mouse_relative_handler(self, request: Request) -> Response:
-        return self.__process_delta_request(request, self.__hid.send_mouse_relative_event)
+        return self.__process_http_delta_event(request, self.__hid.send_mouse_relative_event)
 
     @exposed_http("POST", "/hid/events/send_mouse_wheel")
     async def __events_send_mouse_wheel_handler(self, request: Request) -> Response:
-        return self.__process_delta_request(request, self.__hid.send_mouse_wheel_event)
+        return self.__process_http_delta_event(request, self.__hid.send_mouse_wheel_event)
 
-    def __send_mouse_move_event_remapped(self, to_x: int, to_y: int) -> None:
+    def __process_http_delta_event(self, request: Request, handler: Callable[[int, int], None]) -> Response:
+        delta_x = valid_hid_mouse_delta(request.query.get("delta_x"))
+        delta_y = valid_hid_mouse_delta(request.query.get("delta_y"))
+        handler(delta_x, delta_y)
+        return make_json_response()
+
+    # =====
+
+    def __send_mouse_move_event(self, to_x: int, to_y: int) -> None:
         if self.__mouse_x_range != MouseRange.RANGE:
             to_x = MouseRange.remap(to_x, *self.__mouse_x_range)
         if self.__mouse_y_range != MouseRange.RANGE:
             to_y = MouseRange.remap(to_y, *self.__mouse_y_range)
         self.__hid.send_mouse_move_event(to_x, to_y)
 
-    def __process_delta_request(self, request: Request, handler: Callable[[int, int], None]) -> Response:
-        delta_x = valid_hid_mouse_delta(request.query.get("delta_x"))
-        delta_y = valid_hid_mouse_delta(request.query.get("delta_y"))
-        handler(delta_x, delta_y)
-        return make_json_response()
+    def __send_mouse_delta_event(
+        self,
+        deltas: list[tuple[int, int]],
+        squash: bool,
+        handler: Callable[[int, int], None],
+    ) -> None:
+
+        if squash:
+            prev = (0, 0)
+            for cur in deltas:
+                if abs(prev[0] + cur[0]) > 127 or abs(prev[1] + cur[1]) > 127:
+                    handler(*prev)
+                    prev = cur
+                else:
+                    prev = (prev[0] + cur[0], prev[1] + cur[1])
+            if prev[0] or prev[1]:
+                handler(*prev)
+        else:
+            for xy in deltas:
+                handler(*xy)

kvmd/clients/kvmd.py

@@ -22,6 +22,7 @@
 
 import asyncio
 import contextlib
+import struct
 import types
 
 from typing import Callable
@@ -127,7 +128,7 @@ class KvmdClientWs:
     def __init__(self, ws: aiohttp.ClientWebSocketResponse) -> None:
         self.__ws = ws
 
-        self.__writer_queue: "asyncio.Queue[tuple[str, dict]]" = asyncio.Queue()
+        self.__writer_queue: "asyncio.Queue[tuple[str, dict] | bytes]" = asyncio.Queue()
         self.__communicated = False
 
     async def communicate(self) -> AsyncGenerator[tuple[str, dict], None]:  # pylint: disable=too-many-branches
@@ -157,7 +158,11 @@ class KvmdClientWs:
                     receive_task = None
 
                 if writer_task in done:
-                    await htserver.send_ws_event(self.__ws, *writer_task.result())
+                    payload = writer_task.result()
+                    if isinstance(payload, bytes):
+                        await self.__ws.send_bytes(payload)
+                    else:
+                        await htserver.send_ws_event(self.__ws, *payload)
                     writer_task = None
         finally:
             if receive_task:
@@ -178,10 +183,10 @@ class KvmdClientWs:
         await self.__writer_queue.put(("mouse_button", {"button": button, "state": state}))
 
     async def send_mouse_move_event(self, to_x: int, to_y: int) -> None:
-        await self.__writer_queue.put(("mouse_move", {"to": {"x": to_x, "y": to_y}}))
+        await self.__writer_queue.put(struct.pack(">bhh", 3, to_x, to_y))
 
     async def send_mouse_wheel_event(self, delta_x: int, delta_y: int) -> None:
-        await self.__writer_queue.put(("mouse_wheel", {"delta": {"x": delta_x, "y": delta_y}}))
+        await self.__writer_queue.put(struct.pack(">bbbb", 5, 0, delta_x, delta_y))
 
 
 class KvmdClientSession:

kvmd/htserver.py

@@ -123,17 +123,25 @@ def _get_exposed_http(obj: object) -> list[HttpExposed]:
 @dataclasses.dataclass(frozen=True)
 class WsExposed:
     event_type: str
+    binary: bool
     handler: Callable
 
 
 _WS_EXPOSED = "_ws_exposed"
+_WS_BINARY = "_ws_binary"
 _WS_EVENT_TYPE = "_ws_event_type"
 
 
-def exposed_ws(event_type: str) -> Callable:
+def exposed_ws(event_type: (str | int), binary: bool=False) -> Callable:
+    if binary:
+        assert isinstance(event_type, int)
+    else:
+        assert isinstance(event_type, str)
+
     def set_attrs(handler: Callable) -> Callable:
         setattr(handler, _WS_EXPOSED, True)
-        setattr(handler, _WS_EVENT_TYPE, event_type)
+        setattr(handler, _WS_BINARY, binary)
+        setattr(handler, _WS_EVENT_TYPE, str(event_type))
         return handler
     return set_attrs
 
@@ -142,6 +150,7 @@ def _get_exposed_ws(obj: object) -> list[WsExposed]:
     return [
         WsExposed(
             event_type=getattr(handler, _WS_EVENT_TYPE),
+            binary=getattr(handler, _WS_BINARY),
             handler=handler,
         )
         for handler in [getattr(obj, name) for name in dir(obj)]
@@ -277,6 +286,7 @@ class HttpServer:
     def __init__(self) -> None:
         self.__ws_heartbeat: (float | None) = None
         self.__ws_handlers: dict[str, Callable] = {}
+        self.__ws_bin_handlers: dict[int, Callable] = {}
         self.__ws_sessions: list[WsSession] = []
         self.__ws_sessions_lock = asyncio.Lock()
 
@@ -330,7 +340,10 @@ class HttpServer:
         self.__app.router.add_route(exposed.method, exposed.path, wrapper)
 
     def __add_exposed_ws(self, exposed: WsExposed) -> None:
-        self.__ws_handlers[exposed.event_type] = exposed.handler
+        if exposed.binary:
+            self.__ws_bin_handlers[int(exposed.event_type)] = exposed.handler
+        else:
+            self.__ws_handlers[exposed.event_type] = exposed.handler
 
     # =====
 
@@ -354,18 +367,27 @@ class HttpServer:
     async def _ws_loop(self, ws: WsSession) -> WebSocketResponse:
         logger = get_logger()
         async for msg in ws.wsr:
-            if msg.type != WSMsgType.TEXT:
-                break
-            try:
-                (event_type, event) = parse_ws_event(msg.data)
-            except Exception as err:
-                logger.error("Can't parse JSON event from websocket: %r", err)
-            else:
-                handler = self.__ws_handlers.get(event_type)
-                if handler:
-                    await handler(ws, event)
+            if msg.type == WSMsgType.TEXT:
+                try:
+                    (event_type, event) = parse_ws_event(msg.data)
+                except Exception as err:
+                    logger.error("Can't parse JSON event from websocket: %r", err)
                 else:
-                    logger.error("Unknown websocket event: %r", msg.data)
+                    handler = self.__ws_handlers.get(event_type)
+                    if handler:
+                        await handler(ws, event)
+                    else:
+                        logger.error("Unknown websocket event: %r", msg.data)
+
+            elif msg.type == WSMsgType.BINARY and len(msg.data) >= 1:
+                handler = self.__ws_bin_handlers.get(msg.data[0])
+                if handler:
+                    await handler(ws, msg.data[1:])
+                else:
+                    logger.error("Unknown websocket binary event: %r", msg.data)
+
+            else:
+                break
         return ws.wsr
 
     async def _broadcast_ws_event(self, event_type: str, event: (dict | None)) -> None:

web/share/js/kvm/mouse.js

@@ -345,11 +345,38 @@ export function Mouse(__getGeometry, __recordWsEvent) {
 	};
 
 	var __sendEvent = function(event_type, event) {
-		event = {"event_type": event_type, "event": event};
+		let wrapped_event = {"event_type": event_type, "event": event};
 		if (__ws && !$("hid-mute-switch").checked) {
-			__ws.send(JSON.stringify(event));
+			if (event_type == "mouse_move") {
+				let data = new Uint8Array([
+					3,
+					(event.to.x >> 8) & 0xFF, event.to.x & 0xFF,
+					(event.to.y >> 8) & 0xFF, event.to.y & 0xFF,
+				]);
+				__ws.send(data);
+
+			} else if (event_type == "mouse_relative" || event_type == "mouse_wheel") {
+				let data;
+				if (Array.isArray(event.delta)) {
+					data = new Int8Array(2 + event.delta.length * 2);
+					let index = 0;
+					for (let delta of event.delta) {
+						data[index + 2] = delta["x"];
+						data[index + 3] = delta["y"];
+						index += 2;
+					}
+				} else {
+					data = new Int8Array([0, 0, event.delta.x, event.delta.y]);
+				}
+				data[0] = (event_type == "mouse_relative" ? 4 : 5);
+				data[1] = (event.squash ? 1 : 0);
+				__ws.send(data);
+
+			} else {
+				__ws.send(JSON.stringify(wrapped_event));
+			}
 		}
-		__recordWsEvent(event);
+		__recordWsEvent(wrapped_event);
 	};
 
 	__init__();