mouse support

hid/src/main.cpp

@@ -42,7 +42,6 @@ INLINE void cmdMouseMoveEvent() { // 4 bytes
 	x |= (int)CMD_SERIAL.read();
 	int y = (int)CMD_SERIAL.read() << 8;
 	y |= (int)CMD_SERIAL.read();
-	AbsoluteMouse.moveTo(0, 0);
 	AbsoluteMouse.moveTo(x, y);
 }
 

kvmd/kvmd/hid.py

@@ -2,6 +2,7 @@ import asyncio
 import multiprocessing
 import multiprocessing.queues
 import queue
+import struct
 import pkgutil
 
 from typing import Dict
@@ -22,16 +23,25 @@ def _get_keymap() -> Dict[str, int]:
 _KEYMAP = _get_keymap()
 
 
-def _keymap(key: str) -> bytes:
-    code = _KEYMAP.get(key)
-    return (bytes([code]) if code else b"")  # type: ignore
-
-
 class _KeyEvent(NamedTuple):
     key: str
     state: bool
 
 
+class _MouseMoveEvent(NamedTuple):
+    to_x: int
+    to_y: int
+
+
+class _MouseButtonEvent(NamedTuple):
+    button: str
+    state: bool
+
+
+class _MouseWheelEvent(NamedTuple):
+    delta_y: int
+
+
 class Hid(multiprocessing.Process):
     def __init__(
         self,
@@ -45,6 +55,7 @@ class Hid(multiprocessing.Process):
         self.__speed = speed
 
         self.__pressed_keys: Set[str] = set()
+        self.__pressed_mouse_buttons: Set[str] = set()
         self.__lock = asyncio.Lock()
         self.__queue: multiprocessing.queues.Queue = multiprocessing.Queue()
 
@@ -56,14 +67,6 @@ class Hid(multiprocessing.Process):
 
     # TODO: add reset or power switching
 
-    def get_state(self) -> Dict:
-        return {
-            "features": {
-                "keyboard": True,  # Always
-                "mouse": False,  # TODO
-            },
-        }
-
     async def send_key_event(self, key: str, state: bool) -> None:
         if not self.__stop_event.is_set():
             async with self.__lock:
@@ -74,6 +77,26 @@ class Hid(multiprocessing.Process):
                     self.__pressed_keys.remove(key)
                     self.__queue.put(_KeyEvent(key, state))
 
+    async def send_mouse_move_event(self, to_x: int, to_y: int) -> None:
+        if not self.__stop_event.is_set():
+            async with self.__lock:
+                self.__queue.put(_MouseMoveEvent(to_x, to_y))
+
+    async def send_mouse_button_event(self, button: str, state: bool) -> None:
+        if not self.__stop_event.is_set():
+            async with self.__lock:
+                if state and button not in self.__pressed_mouse_buttons:
+                    self.__pressed_mouse_buttons.add(button)
+                    self.__queue.put(_MouseButtonEvent(button, state))
+                elif not state and button in self.__pressed_mouse_buttons:
+                    self.__pressed_mouse_buttons.remove(button)
+                    self.__queue.put(_MouseButtonEvent(button, state))
+
+    async def send_mouse_wheel_event(self, delta_y: int) -> None:
+        if not self.__stop_event.is_set():
+            async with self.__lock:
+                self.__queue.put(_MouseWheelEvent(delta_y))
+
     async def clear_events(self) -> None:
         if not self.__stop_event.is_set():
             async with self.__lock:
@@ -87,10 +110,13 @@ class Hid(multiprocessing.Process):
                 self.__stop_event.set()
                 self.join()
             else:
-                get_logger().warning("Emergency cleaning up keyboard events ...")
+                get_logger().warning("Emergency cleaning up HID events ...")
                 self.__emergency_clear_events()
 
     def __unsafe_clear_events(self) -> None:
+        for button in self.__pressed_mouse_buttons:
+            self.__queue.put(_MouseButtonEvent(button, False))
+        self.__pressed_mouse_buttons.clear()
         for key in self.__pressed_keys:
             self.__queue.put(_KeyEvent(key, False))
         self.__pressed_keys.clear()
@@ -100,7 +126,7 @@ class Hid(multiprocessing.Process):
             with serial.Serial(self.__device_path, self.__speed) as tty:
                 self.__send_clear_hid(tty)
         except Exception:
-            get_logger().exception("Can't execute emergency clear events")
+            get_logger().exception("Can't execute emergency clear HID events")
 
     def run(self) -> None:
         try:
@@ -111,7 +137,14 @@ class Hid(multiprocessing.Process):
                     except queue.Empty:
                         pass
                     else:
-                        self.__send_key_event(tty, event)
+                        if isinstance(event, _KeyEvent):
+                            self.__send_key_event(tty, event)
+                        elif isinstance(event, _MouseMoveEvent):
+                            self.__send_mouse_move_event(tty, event)
+                        elif isinstance(event, _MouseButtonEvent):
+                            self.__send_mouse_button_event(tty, event)
+                        elif isinstance(event, _MouseWheelEvent):
+                            self.__send_mouse_wheel_event(tty, event)
                     if self.__stop_event.is_set() and self.__queue.qsize() == 0:
                         break
         except Exception:
@@ -119,8 +152,9 @@ class Hid(multiprocessing.Process):
             raise
 
     def __send_key_event(self, tty: serial.Serial, event: _KeyEvent) -> None:
-        key_bytes = _keymap(event.key)
+        code = _KEYMAP.get(event.key)
-        if key_bytes:
+        if code:
+            key_bytes = bytes([code])
             assert len(key_bytes) == 1, (event, key_bytes)
             tty.write(
                 b"\01"
@@ -129,5 +163,24 @@ class Hid(multiprocessing.Process):
                 + b"\00\00"
             )
 
+    def __send_mouse_move_event(self, tty: serial.Serial, event: _MouseMoveEvent) -> None:
+        to_x = min(max(-32768, event.to_x), 32767)
+        to_y = min(max(-32768, event.to_y), 32767)
+        tty.write(b"\02" + struct.pack(">hh", to_x, to_y))
+
+    def __send_mouse_button_event(self, tty: serial.Serial, event: _MouseButtonEvent) -> None:
+        if event.button == "left":
+            code = (0b10000000 | (0b00001000 if event.state else 0))
+        elif event.button == "right":
+            code = (0b01000000 | (0b00000100 if event.state else 0))
+        else:
+            code = 0
+        if code:
+            tty.write(b"\03" + bytes([code]) + b"\00\00\00")
+
+    def __send_mouse_wheel_event(self, tty: serial.Serial, event: _MouseWheelEvent) -> None:
+        delta_y = min(max(-128, event.delta_y), 127)
+        tty.write(b"\04\00" + struct.pack(">b", delta_y) + b"\00\00")
+
     def __send_clear_hid(self, tty: serial.Serial) -> None:
         tty.write(b"\00\00\00\00\00")

kvmd/kvmd/server.py

@@ -137,8 +137,6 @@ class Server:  # pylint: disable=too-many-instance-attributes
 
         app.router.add_get("/ws", self.__ws_handler)
 
-        app.router.add_get("/hid", self.__hid_state_handler)
-
         app.router.add_get("/atx", self.__atx_state_handler)
         app.router.add_post("/atx/click", self.__atx_click_handler)
 
@@ -161,9 +159,13 @@ class Server:  # pylint: disable=too-many-instance-attributes
 
         aiohttp.web.run_app(app, host=host, port=port, print=self.__run_app_print)
 
+    # ===== INFO
+
     async def __info_handler(self, _: aiohttp.web.Request) -> aiohttp.web.WebSocketResponse:
         return _json(_get_system_info())
 
+    # ===== WEBSOCKET
+
     async def __ws_handler(self, request: aiohttp.web.Request) -> aiohttp.web.WebSocketResponse:
         logger = get_logger(0)
         ws = aiohttp.web.WebSocketResponse(heartbeat=self.__heartbeat)
@@ -176,23 +178,51 @@ class Server:  # pylint: disable=too-many-instance-attributes
                 except Exception as err:
                     logger.error("Can't parse JSON event from websocket: %s", err)
                 else:
-                    if event.get("event_type") == "ping":
+                    event_type = event.get("event_type")
+                    if event_type == "ping":
                         await ws.send_str(json.dumps({"msg_type": "pong"}))
-                    elif event.get("event_type") == "key":
+                    elif event_type == "key":
-                        key = str(event.get("key", ""))[:64].strip()
+                        await self.__handle_ws_key_event(event)
-                        state = event.get("state")
+                    elif event_type == "mouse_move":
-                        if key and state in [True, False]:
+                        await self.__handle_ws_mouse_move_event(event)
-                            await self.__hid.send_key_event(key, state)
+                    elif event_type == "mouse_button":
-                    elif event.get("event_type") in ["mouse_move", "mouse_button", "mouse_wheel"]:
+                        await self.__handle_ws_mouse_button_event(event)
-                        pass
+                    elif event_type == "mouse_wheel":
+                        await self.__handle_ws_mouse_wheel_event(event)
                     else:
-                        logger.error("Invalid websocket event: %r", event)
+                        logger.error("Unknown websocket event: %r", event)
             else:
                 break
         return ws
 
-    async def __hid_state_handler(self, _: aiohttp.web.Request) -> aiohttp.web.Response:
+    async def __handle_ws_key_event(self, event: Dict) -> None:
-        return _json(self.__hid.get_state())
+        key = str(event.get("key", ""))[:64].strip()
+        state = event.get("state")
+        if key and state in [True, False]:
+            await self.__hid.send_key_event(key, state)
+
+    async def __handle_ws_mouse_move_event(self, event: Dict) -> None:
+        try:
+            to_x = int(event["to"]["x"])
+            to_y = int(event["to"]["y"])
+        except Exception:
+            return
+        await self.__hid.send_mouse_move_event(to_x, to_y)
+
+    async def __handle_ws_mouse_button_event(self, event: Dict) -> None:
+        button = str(event.get("button", ""))[:64].strip()
+        state = event.get("state")
+        if button and state in [True, False]:
+            await self.__hid.send_mouse_button_event(button, state)
+
+    async def __handle_ws_mouse_wheel_event(self, event: Dict) -> None:
+        try:
+            delta_y = int(event["delta"]["y"])
+        except Exception:
+            return
+        await self.__hid.send_mouse_wheel_event(delta_y)
+
+    # ===== ATX
 
     async def __atx_state_handler(self, _: aiohttp.web.Request) -> aiohttp.web.Response:
         return _json(self.__atx.get_state())
@@ -212,6 +242,8 @@ class Server:  # pylint: disable=too-many-instance-attributes
         await self.__broadcast_event("atx_click", button=None)  # type: ignore
         return _json({"clicked": button})
 
+    # ===== MSD
+
     async def __msd_state_handler(self, _: aiohttp.web.Request) -> aiohttp.web.Response:
         return _json(self.__msd.get_state())
 
@@ -261,6 +293,8 @@ class Server:  # pylint: disable=too-many-instance-attributes
                 logger.info("Written %d bytes to mass-storage device", written)
         return _json({"written": written})
 
+    # ===== STREAMER
+
     async def __streamer_state_handler(self, _: aiohttp.web.Request) -> aiohttp.web.Response:
         return _json(self.__streamer.get_state())
 
@@ -268,6 +302,8 @@ class Server:  # pylint: disable=too-many-instance-attributes
         self.__reset_streamer = True
         return _json()
 
+    # =====
+
     def __run_app_print(self, text: str) -> None:
         logger = get_logger()
         for line in text.strip().splitlines():

kvmd/web/css/stream.css

@@ -2,7 +2,6 @@ img#stream-image {
 	width: 640px;
 	height: 480px;
 	display: inline-block;
-	border: var(--dark-border);
 	background-color: var(--bg-color-stream-screen);
 }
 
@@ -19,9 +18,7 @@ img.stream-image-inactive {
 div#stream-box {
 	position: relative;
 	display: inline-block;
-}
+	border: var(--dark-border);
-div.stream-box-active: {
-	cursor: crosshair;
 }
 div.stream-box-inactive::after {
 	cursor: wait;
@@ -34,3 +31,6 @@ div.stream-box-inactive::after {
 	display: inline-block;
 	background: radial-gradient(transparent 20%, black);
 }
+div.stream-box-mouse-enabled {
+	cursor: url("../svg/stream-mouse-cursor.svg"), pointer;
+}

kvmd/web/js/hid.js

@@ -1,10 +1,4 @@
 var hid = new function() {
-	var __install_timer = null;
-	var __installed = false;
-
-	var __hidden_attr = null;
-	var __visibility_change_attr = null;
-
 	this.init = function() {
 		keyboard.init();
 		mouse.init();
@@ -32,32 +26,12 @@ var hid = new function() {
 	};
 
 	this.installCapture = function(ws) {
-		var http = tools.makeRequest("GET", "/kvmd/hid", function() {
+		keyboard.setSocket(ws);
-			if (http.readyState === 4) {
+		mouse.setSocket(ws);
-				if (http.status === 200) {
-					features = JSON.parse(http.responseText).result.features;
-					if (features.mouse) {
-						mouse.setSocket(ws);
-					}
-					keyboard.setSocket(ws);
-					__installed = true;
-				} else {
-					tools.error("Can't resolve HID features:", http.responseText);
-					__install_timer = setTimeout(() => hid.installCapture(ws), 1000);
-				}
-			}
-		});
 	};
 
 	this.clearCapture = function() {
-		if (__install_timer) {
+		mouse.setSocket(null);
-			clearTimeout(__install_timer);
+		keyboard.setSocket(null);
-			__install_timer = null;
-		}
-		if (__installed) {
-			mouse.setSocket(null);
-			keyboard.setSocket(null);
-			__installed = false;
-		}
 	};
 }

kvmd/web/js/keyboard.js

@@ -27,8 +27,10 @@ var keyboard = new function() {
 	};
 
 	this.setSocket = function(ws) {
-		keyboard.releaseAll();
+		if (ws !== __ws) {
-		__ws = ws;
+			keyboard.releaseAll();
+			__ws = ws;
+		}
 		keyboard.updateLeds();
 	};
 

kvmd/web/js/mouse.js

@@ -2,9 +2,12 @@ var mouse = new function() {
 	var __ws = null;
 	var __current_pos = {x: 0, y:0};
 	var __sent_pos = {x: 0, y:0};
+	var __stream_hovered = false;
 
 	this.init = function() {
 		el_stream_box = $("stream-box");
+		el_stream_box.onmouseenter = __hoverStream;
+		el_stream_box.onmouseleave = __leaveStream;
 		el_stream_box.onmousedown = (event) => __buttonHandler(event, true);
 		el_stream_box.onmouseup = (event) => __buttonHandler(event, false);
 		el_stream_box.oncontextmenu = (event) => event.preventDefault();
@@ -15,11 +18,25 @@ var mouse = new function() {
 
 	this.setSocket = function(ws) {
 		__ws = ws;
+		if (ws) {
+			$("stream-box").classList.add("stream-box-mouse-enabled");
+		} else {
+			$("stream-box").classList.remove("stream-box-mouse-enabled");
+		}
+	};
+
+	var __hoverStream = function() {
+		__stream_hovered = true;
+		mouse.updateLeds();
+	};
+
+	var __leaveStream = function() {
+		__stream_hovered = false;
+		mouse.updateLeds();
 	};
 
 	this.updateLeds = function() {
-		var focused = (__ws && document.activeElement === $("stream-window"));
+		$("hid-mouse-led").className = (__ws && __stream_hovered ? "led-on" : "led-off");
-		$("hid-mouse-led").className = (focused ? "led-on" : "led-off");
 	};
 
 	var __buttonHandler = function(event, state) {
@@ -56,15 +73,23 @@ var mouse = new function() {
 		if (pos.x !== __sent_pos.x || pos.y !== __sent_pos.y) {
 			tools.debug("Mouse move:", pos);
 			if (__ws) {
+				el_stream_image = $("stream-image");
 				__ws.send(JSON.stringify({
 					event_type: "mouse_move",
-					to: pos,
+					to: {
+						x: __translate(pos.x, 0, el_stream_image.clientWidth, -32768, 32767),
+						y: __translate(pos.y, 0, el_stream_image.clientHeight, -32768, 32767),
+					},
 				}));
 			}
 			__sent_pos = pos;
 		}
 	};
 
+	var __translate = function(x, a, b, c, d) {
+		return Math.round((x - a) / (b - a) * (d - c) + c);
+	};
+
 	var __wheelHandler = function(event) {
 		// https://learn.javascript.ru/mousewheel
 		if (event.preventDefault) {

kvmd/web/js/stream.js

@@ -11,14 +11,14 @@ var stream = new function() {
 					tools.info("Refreshing stream ...");
 					__prev_state = false;
 					$("stream-image").className = "stream-image-inactive";
-					$("stream-box").className = "stream-box-inactive";
+					$("stream-box").classList.add("stream-box-inactive");
 					$("stream-led").className = "led-off";
 					$("stream-reset-button").disabled = true;
 				} else if (!__prev_state) {
 					__refreshImage();
 					__prev_state = true;
 					$("stream-image").className = "stream-image-active";
-					$("stream-box").className = "stream-box-active";
+					$("stream-box").classList.remove("stream-box-inactive");
 					$("stream-led").className = "led-on";
 					$("stream-reset-button").disabled = false;
 				}

kvmd/web/svg/stream-mouse-cursor.svg

@@ -0,0 +1,68 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="2.6458335mm"
+   height="2.6458335mm"
+   viewBox="0 0 2.6458335 2.6458335"
+   version="1.1"
+   id="svg8"
+   sodipodi:docname="stream-mouse-cursor.svg"
+   inkscape:version="0.92.2 2405546, 2018-03-11">
+  <defs
+     id="defs2" />
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="31.678384"
+     inkscape:cx="13.114187"
+     inkscape:cy="8.129091"
+     inkscape:document-units="mm"
+     inkscape:current-layer="layer1"
+     showgrid="false"
+     units="px"
+     inkscape:window-width="1920"
+     inkscape:window-height="1020"
+     inkscape:window-x="0"
+     inkscape:window-y="30"
+     inkscape:window-maximized="1"
+     fit-margin-top="0"
+     fit-margin-left="0"
+     fit-margin-right="0"
+     fit-margin-bottom="0" />
+  <metadata
+     id="metadata5">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title></dc:title>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(-102.90015,-148.41315)">
+    <circle
+       style="opacity:1;fill:#5b90bb;fill-opacity:1;stroke:#e8e8e8;stroke-width:0.26458332;stroke-linecap:square;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:75.59055328;stroke-opacity:0.80303034;paint-order:normal"
+       id="path4915"
+       cx="104.22307"
+       cy="149.73607"
+       r="1.1906251" />
+  </g>
+</svg>