Relative (#18)

* refactoring

* basic relative mouse mode

kvmd/apps/kvmd/api/hid.py

@@ -40,7 +40,7 @@ from ....validators.os import valid_printable_filename
 from ....validators.kvm import valid_hid_key
 from ....validators.kvm import valid_hid_mouse_move
 from ....validators.kvm import valid_hid_mouse_button
-from ....validators.kvm import valid_hid_mouse_wheel
+from ....validators.kvm import valid_hid_mouse_delta
 
 from ....keyboard.keysym import build_symmap
 from ....keyboard.printer import text_to_web_keys
@@ -142,11 +142,20 @@ class HidApi:
             return
         self.__hid.send_mouse_move_event(to_x, to_y)
 
+    @exposed_ws("mouse_relative")
+    async def __ws_mouse_relative_handler(self, _: WebSocketResponse, event: Dict) -> None:
+        try:
+            delta_x = valid_hid_mouse_delta(event["delta"]["x"])
+            delta_y = valid_hid_mouse_delta(event["delta"]["y"])
+        except Exception:
+            return
+        self.__hid.send_mouse_relative_event(delta_x, delta_y)
+
     @exposed_ws("mouse_wheel")
     async def __ws_mouse_wheel_handler(self, _: WebSocketResponse, event: Dict) -> None:
         try:
-            delta_x = valid_hid_mouse_wheel(event["delta"]["x"])
+            delta_x = valid_hid_mouse_delta(event["delta"]["x"])
-            delta_y = valid_hid_mouse_wheel(event["delta"]["y"])
+            delta_y = valid_hid_mouse_delta(event["delta"]["y"])
         except Exception:
             return
         self.__hid.send_mouse_wheel_event(delta_x, delta_y)
@@ -181,9 +190,16 @@ class HidApi:
         self.__hid.send_mouse_move_event(to_x, to_y)
         return make_json_response()
 
+    @exposed_http("POST", "/hid/events/send_mouse_relative")
+    async def __events_send_mouse_relative_handler(self, request: Request) -> Response:
+        delta_x = valid_hid_mouse_delta(request.query.get("delta_x"))
+        delta_y = valid_hid_mouse_delta(request.query.get("delta_y"))
+        self.__hid.send_mouse_relative_event(delta_x, delta_y)
+        return make_json_response()
+
     @exposed_http("POST", "/hid/events/send_mouse_wheel")
-    async def __events_send_mouse_wheel(self, request: Request) -> Response:
+    async def __events_send_mouse_wheel_handler(self, request: Request) -> Response:
-        delta_x = valid_hid_mouse_wheel(request.query.get("delta_x"))
+        delta_x = valid_hid_mouse_delta(request.query.get("delta_x"))
-        delta_y = valid_hid_mouse_wheel(request.query.get("delta_y"))
+        delta_y = valid_hid_mouse_delta(request.query.get("delta_y"))
         self.__hid.send_mouse_wheel_event(delta_x, delta_y)
         return make_json_response()

kvmd/apps/otg/__init__.py

@@ -43,7 +43,8 @@ from .. import init
 
 from .hid import Hid
 from .hid.keyboard import KEYBOARD_HID
-from .hid.mouse import MOUSE_HID
+from .hid.mouse import MOUSE_ABSOLUTE_HID
+from .hid.mouse import MOUSE_RELATIVE_HID
 
 
 # =====
@@ -203,7 +204,10 @@ def _cmd_start(config: Section) -> None:
     if config.kvmd.hid.type == "otg":
         logger.info("===== Required HID =====")
         _create_hid(gadget_path, config_path, 0, KEYBOARD_HID)
-        _create_hid(gadget_path, config_path, 1, MOUSE_HID)
+        if config.kvmd.hid.mouse.absolute:
+            _create_hid(gadget_path, config_path, 1, MOUSE_ABSOLUTE_HID)
+        else:
+            _create_hid(gadget_path, config_path, 1, MOUSE_RELATIVE_HID)
 
     if config.kvmd.msd.type == "otg":
         logger.info("===== Required MSD =====")

kvmd/apps/otg/hid/mouse.py

@@ -24,7 +24,7 @@ from . import Hid
 
 
 # =====
-MOUSE_HID = Hid(
+MOUSE_ABSOLUTE_HID = Hid(
     protocol=0,  # None protocol
     subclass=0,  # No subclass
 
@@ -84,3 +84,54 @@ MOUSE_HID = Hid(
         0xC0,  # END_COLLECTION
     ]),
 )
+
+MOUSE_RELATIVE_HID = Hid(
+    protocol=2,  # Mouse protocol
+    subclass=1,  # Boot interface subclass
+
+    report_length=5,
+
+    report_descriptor=bytes([
+        # https://github.com/NicoHood/HID/blob/0835e6a/src/SingleReport/BootMouse.cpp
+
+        # Relative mouse
+        0x05, 0x01,  # USAGE_PAGE (Generic Desktop)
+        0x09, 0x02,  # USAGE (Mouse)
+        0xA1, 0x01,  # COLLECTION (Application)
+
+        # 8 Buttons
+        0x05, 0x09,  # USAGE_PAGE (Button)
+        0x19, 0x01,  # USAGE_MINIMUM (Button 1)
+        0x29, 0x08,  # USAGE_MAXIMUM (Button 8)
+        0x15, 0x00,  # LOGICAL_MINIMUM (0)
+        0x25, 0x01,  # LOGICAL_MAXIMUM (1)
+        0x95, 0x08,  # REPORT_COUNT (8)
+        0x75, 0x01,  # REPORT_SIZE (1)
+        0x81, 0x02,  # INPUT (Data,Var,Abs)
+
+        # X, Y
+        0x05, 0x01,  # USAGE_PAGE (Generic Desktop)
+        0x09, 0x30,  # USAGE (X)
+        0x09, 0x31,  # USAGE (Y)
+
+        # Wheel
+        0x09, 0x38,  # USAGE (Wheel)
+        0x15, 0x81,  # LOGICAL_MINIMUM (-127)
+        0x25, 0x7F,  # LOGICAL_MAXIMUM (127)
+        0x75, 0x08,  # REPORT_SIZE (8)
+        0x95, 0x03,  # REPORT_COUNT (3)
+        0x81, 0x06,  # INPUT (Data,Var,Rel)
+
+        # Horizontal wheel
+        0x05, 0x0C,  # USAGE PAGE (Consumer Devices)
+        0x0A, 0x38, 0x02,  # USAGE (AC Pan)
+        0x15, 0x81,  # LOGICAL_MINIMUM (-127)
+        0x25, 0x7F,  # LOGICAL_MAXIMUM (127)
+        0x75, 0x08,  # REPORT_SIZE (8)
+        0x95, 0x01,  # REPORT_COUNT (1)
+        0x81, 0x06,  # INPUT (Data,Var,Rel)
+
+        # End
+        0xC0,  # END_COLLECTION
+    ]),
+)

kvmd/plugins/hid/__init__.py

@@ -59,6 +59,9 @@ class BaseHid(BasePlugin):
     def send_mouse_move_event(self, to_x: int, to_y: int) -> None:
         raise NotImplementedError
 
+    def send_mouse_relative_event(self, delta_x: int, delta_y: int) -> None:
+        pass  # FIXME: SPI
+
     def send_mouse_wheel_event(self, delta_x: int, delta_y: int) -> None:
         raise NotImplementedError
 

kvmd/plugins/hid/otg/__init__.py

@@ -75,6 +75,7 @@ class Plugin(BaseHid):
                 "write_retries":       Option(5,   type=valid_int_f1),
                 "write_retries_delay": Option(0.1, type=valid_float_f01),
                 "reopen_delay":        Option(0.5, type=valid_float_f01),
+                "absolute":            Option(True, type=valid_bool),
             },
             "noop": Option(False, type=valid_bool),
         }
@@ -130,6 +131,9 @@ class Plugin(BaseHid):
     def send_mouse_move_event(self, to_x: int, to_y: int) -> None:
         self.__mouse_proc.send_move_event(to_x, to_y)
 
+    def send_mouse_relative_event(self, delta_x: int, delta_y: int) -> None:
+        self.__mouse_proc.send_relative_event(delta_x, delta_y)
+
     def send_mouse_wheel_event(self, delta_x: int, delta_y: int) -> None:
         self.__mouse_proc.send_wheel_event(delta_x, delta_y)
 

kvmd/plugins/hid/otg/mouse.py

@@ -23,6 +23,7 @@
 import struct
 import dataclasses
 
+from typing import Optional
 from typing import Any
 
 from ....logging import get_logger
@@ -52,6 +53,12 @@ class _MoveEvent(BaseEvent):
     to_y: int
 
 
+@dataclasses.dataclass(frozen=True)
+class _RelativeEvent(BaseEvent):
+    delta_x: int
+    delta_y: int
+
+
 @dataclasses.dataclass(frozen=True)
 class _WheelEvent(BaseEvent):
     delta_x: int
@@ -61,21 +68,26 @@ class _WheelEvent(BaseEvent):
 # =====
 class MouseProcess(BaseDeviceProcess):
     def __init__(self, **kwargs: Any) -> None:
+        self.__absolute: bool = kwargs.pop("absolute")
+
         super().__init__(
             name="mouse",
             read_size=0,
-            initial_state={},
+            initial_state={"absolute": self.__absolute},  # Just for the state
             **kwargs,
         )
 
         self.__pressed_buttons: int = 0
-        self.__x = 0
+        self.__x = 0  # For absolute
         self.__y = 0
 
     def cleanup(self) -> None:
         self._stop()
         get_logger().info("Clearing HID-mouse events ...")
+        if self.__absolute:
             report = self.__make_report(0, self.__x, self.__y, 0, 0)
+        else:
+            report = self.__make_report(0, 0, 0, 0, 0)
         self._ensure_write(report, close=True)  # Release all buttons
 
     def send_clear_event(self) -> None:
@@ -97,12 +109,19 @@ class MouseProcess(BaseDeviceProcess):
         self._queue_event(_ButtonEvent(code, state))
 
     def send_move_event(self, to_x: int, to_y: int) -> None:
+        if self.__absolute:
             assert -32768 <= to_x <= 32767
             assert -32768 <= to_y <= 32767
             to_x = (to_x + 32768) // 2
             to_y = (to_y + 32768) // 2
             self._queue_event(_MoveEvent(to_x, to_y))
 
+    def send_relative_event(self, delta_x: int, delta_y: int) -> None:
+        if not self.__absolute:
+            assert -127 <= delta_x <= 127
+            assert -127 <= delta_y <= 127
+            self._queue_event(_RelativeEvent(delta_x, delta_y))
+
     def send_wheel_event(self, delta_x: int, delta_y: int) -> None:
         assert -127 <= delta_x <= 127
         assert -127 <= delta_y <= 127
@@ -119,6 +138,8 @@ class MouseProcess(BaseDeviceProcess):
             return self.__process_button_event(event)
         elif isinstance(event, _MoveEvent):
             return self.__process_move_event(event)
+        elif isinstance(event, _RelativeEvent):
+            return self.__process_relative_event(event)
         elif isinstance(event, _WheelEvent):
             return self.__process_wheel_event(event)
         raise RuntimeError(f"Not implemented event: {event}")
@@ -144,19 +165,40 @@ class MouseProcess(BaseDeviceProcess):
         self.__y = event.to_y
         return self.__send_current_state()
 
+    def __process_relative_event(self, event: _RelativeEvent) -> bool:
+        return self.__send_current_state(relative_event=event)
+
     def __process_wheel_event(self, event: _WheelEvent) -> bool:
-        return self.__send_current_state(event.delta_x, event.delta_y)
+        return self.__send_current_state(wheel_event=event)
 
     # =====
 
-    def __send_current_state(self, delta_x: int=0, delta_y: int=0, reopen: bool=False) -> bool:
+    def __send_current_state(
-        report = self.__make_report(
+        self,
-            buttons=self.__pressed_buttons,
+        relative_event: Optional[_RelativeEvent]=None,
-            to_x=self.__x,
+        wheel_event: Optional[_WheelEvent]=None,
-            to_y=self.__y,
+        reopen: bool=False,
-            delta_x=delta_x,
+    ) -> bool:
-            delta_y=delta_y,
+
-        )
+        if self.__absolute:
+            assert relative_event is None
+            move_x = self.__x
+            move_y = self.__y
+        else:
+            assert self.__x == self.__y == 0
+            if relative_event is not None:
+                move_x = relative_event.delta_x
+                move_y = relative_event.delta_y
+            else:
+                move_x = move_y = 0
+
+        if wheel_event is not None:
+            wheel_x = wheel_event.delta_x
+            wheel_y = wheel_event.delta_y
+        else:
+            wheel_x = wheel_y = 0
+
+        report = self.__make_report(self.__pressed_buttons, move_x, move_y, wheel_x, wheel_y)
         if not self._ensure_write(report, reopen=reopen):
             self.__clear_state()
             return False
@@ -167,7 +209,7 @@ class MouseProcess(BaseDeviceProcess):
         self.__x = 0
         self.__y = 0
 
-    def __make_report(self, buttons: int, to_x: int, to_y: int, delta_x: int, delta_y: int) -> bytes:
+    def __make_report(self, buttons: int, move_x: int, move_y: int, wheel_x: int, wheel_y: int) -> bytes:
-        # XXX: Delta Y before X: it's ok.
+        # XXX: Wheel Y before X: it's ok.
-        # See /kvmd/apps/otg/hid/keyboard.py for details
+        # See /kvmd/apps/otg/hid/mouse.py for details
-        return struct.pack("<BHHbb", buttons, to_x, to_y, delta_y, delta_x)
+        return struct.pack(("<BHHbb" if self.__absolute else "<Bbbbb"), buttons, move_x, move_y, wheel_y, wheel_x)

kvmd/validators/kvm.py

@@ -96,8 +96,8 @@ def valid_hid_mouse_button(arg: Any) -> str:
     return check_string_in_list(arg, "HID mouse button", ["left", "right", "middle", "up", "down"])
 
 
-def valid_hid_mouse_wheel(arg: Any) -> int:
+def valid_hid_mouse_delta(arg: Any) -> int:
-    arg = valid_number(arg, name="HID mouse wheel")
+    arg = valid_number(arg, name="HID mouse delta")
     return min(max(-127, arg), 127)
 
 

testenv/tests/validators/test_kvm.py

@@ -38,7 +38,7 @@ from kvmd.validators.kvm import valid_stream_resolution
 from kvmd.validators.kvm import valid_hid_key
 from kvmd.validators.kvm import valid_hid_mouse_move
 from kvmd.validators.kvm import valid_hid_mouse_button
-from kvmd.validators.kvm import valid_hid_mouse_wheel
+from kvmd.validators.kvm import valid_hid_mouse_delta
 from kvmd.validators.kvm import valid_ugpio_driver
 from kvmd.validators.kvm import valid_ugpio_channel
 from kvmd.validators.kvm import valid_ugpio_mode
@@ -188,22 +188,22 @@ def test_fail__valid_hid_mouse_button(arg: Any) -> None:
 
 # =====
 @pytest.mark.parametrize("arg", [-100, "1 ", "-1", 1, -1, 0, "100 "])
-def test_ok__valid_hid_mouse_wheel(arg: Any) -> None:
+def test_ok__valid_hid_mouse_delta(arg: Any) -> None:
-    assert valid_hid_mouse_wheel(arg) == int(str(arg).strip())
+    assert valid_hid_mouse_delta(arg) == int(str(arg).strip())
 
 
-def test_ok__valid_hid_mouse_wheel__m200() -> None:
+def test_ok__valid_hid_mouse_delta__m200() -> None:
-    assert valid_hid_mouse_wheel(-200) == -127
+    assert valid_hid_mouse_delta(-200) == -127
 
 
-def test_ok__valid_hid_mouse_wheel__p200() -> None:
+def test_ok__valid_hid_mouse_delta__p200() -> None:
-    assert valid_hid_mouse_wheel(200) == 127
+    assert valid_hid_mouse_delta(200) == 127
 
 
 @pytest.mark.parametrize("arg", ["test", "", None, 1.1])
-def test_fail__valid_hid_mouse_wheel(arg: Any) -> None:
+def test_fail__valid_hid_mouse_delta(arg: Any) -> None:
     with pytest.raises(ValidatorError):
-        print(valid_hid_mouse_wheel(arg))
+        print(valid_hid_mouse_delta(arg))
 
 
 # =====

web/share/js/kvm/mouse.js

@@ -36,6 +36,7 @@ export function Mouse(record_callback) {
 
 	var __ws = null;
 	var __online = true;
+	var __absolute = true;
 
 	var __keypad = null;
 
@@ -50,8 +51,10 @@ export function Mouse(record_callback) {
 
 		$("hid-mouse-led").title = "Mouse free";
 
-		$("stream-box").onmouseenter = __hoverStream;
+		document.onpointerlockchange = __relativeCapturedHandler; // Only for relative
-		$("stream-box").onmouseleave = __leaveStream;
+		document.onpointerlockerror = __relativeCapturedHandler;
+		$("stream-box").onmouseenter = () => __streamHoveredHandler(true);
+		$("stream-box").onmouseleave = () => __streamHoveredHandler(false);
 		$("stream-box").onmousedown = (event) => __streamButtonHandler(event, true);
 		$("stream-box").onmouseup = (event) => __streamButtonHandler(event, false);
 		$("stream-box").oncontextmenu = (event) => event.preventDefault();
@@ -59,7 +62,7 @@ export function Mouse(record_callback) {
 		$("stream-box").onwheel = __streamWheelHandler;
 		$("stream-box").ontouchstart = (event) => __streamTouchMoveHandler(event);
 
-		setInterval(__sendMove, 100);
+		setInterval(__sendMove, 100); // Only for absolute
 	};
 
 	/************************************************************************/
@@ -72,6 +75,13 @@ export function Mouse(record_callback) {
 
 	self.setState = function(state) {
 		__online = state.online;
+		if (!("absolute" in state)) { // FIXME: SPI
+			state.absolute = true;
+		}
+		if (state.absolute && !__absolute && __isRelativeCaptured()) {
+			$("stream-box").exitPointerLock();
+		}
+		__absolute = state.absolute;
 		__updateOnlineLeds();
 	};
 
@@ -79,33 +89,35 @@ export function Mouse(record_callback) {
 		__keypad.releaseAll();
 	};
 
-	var __hoverStream = function() {
+	var __streamHoveredHandler = function(hovered) {
-		__stream_hovered = true;
+		if (__absolute) {
-		__updateOnlineLeds();
+			__stream_hovered = hovered;
-	};
-
-	var __leaveStream = function() {
-		__stream_hovered = false;
 			__updateOnlineLeds();
+		}
 	};
 
 	var __updateOnlineLeds = function() {
-		let is_captured = (__stream_hovered || tools.browser.is_ios);
+		let captured;
+		if (__absolute) {
+			captured = (__stream_hovered || tools.browser.is_ios);
+		} else {
+			captured = __isRelativeCaptured();
+		}
 		let led = "led-gray";
 		let title = "Mouse free";
 
 		if (__ws) {
 			if (__online) {
-				if (is_captured) {
+				if (captured) {
 					led = "led-green";
 					title = "Mouse captured";
 				}
 			} else {
 				led = "led-yellow";
-				title = (is_captured ? "Mouse captured, HID offline" : "Mouse free, HID offline");
+				title = (captured ? "Mouse captured, HID offline" : "Mouse free, HID offline");
 			}
 		} else {
-			if (is_captured) {
+			if (captured) {
 				title = "Mouse captured, Pi-KVM offline";
 			}
 		}
@@ -113,9 +125,19 @@ export function Mouse(record_callback) {
 		$("hid-mouse-led").title = title;
 	};
 
+	var __isRelativeCaptured = function() {
+		return (document.pointerLockElement === $("stream-box"));
+	};
+
+	var __relativeCapturedHandler = function() {
+		tools.info("Relative mouse", (__isRelativeCaptured() ? "captured" : "released"), "by pointer lock");
+		__updateOnlineLeds();
+	};
+
 	var __streamButtonHandler = function(event, state) {
 		// https://www.w3schools.com/jsref/event_button.asp
 		event.preventDefault();
+		if (__absolute || __isRelativeCaptured()) {
 			switch (event.button) {
 				case 0: __keypad.emit("left", state); break;
 				case 2: __keypad.emit("right", state); break;
@@ -123,10 +145,14 @@ export function Mouse(record_callback) {
 				case 3: __keypad.emit("up", state); break;
 				case 4: __keypad.emit("down", state); break;
 			}
+		} else if (!__absolute && !__isRelativeCaptured() && !state) {
+			$("stream-box").requestPointerLock();
+		}
 	};
 
 	var __streamTouchMoveHandler = function(event) {
 		event.preventDefault();
+		if (__absolute) {
 			if (event.touches[0].target && event.touches[0].target.getBoundingClientRect) {
 				let rect = event.touches[0].target.getBoundingClientRect();
 				__current_pos = {
@@ -135,16 +161,25 @@ export function Mouse(record_callback) {
 				};
 				__sendMove();
 			}
+		}
 	};
 
 	var __streamMoveHandler = function(event) {
+		if (__absolute) {
 			let rect = event.target.getBoundingClientRect();
 			__current_pos = {
 				x: Math.max(Math.round(event.clientX - rect.left), 0),
 				y: Math.max(Math.round(event.clientY - rect.top), 0),
 			};
+		} else if (__isRelativeCaptured()) {
+			let delta = {
+				x: Math.min(Math.max(-127, event.movementX), 127),
+				y: Math.min(Math.max(-127, event.movementY), 127),
+			};
+			tools.debug("Mouse: relative:", delta);
+			__sendEvent("mouse_relative", {"delta": delta});
+		}
 	};
-
 
 	var __sendButton = function(button, state) {
 		tools.debug("Mouse: button", (state ? "pressed:" : "released:"), button);
@@ -153,6 +188,7 @@ export function Mouse(record_callback) {
 	};
 
 	var __sendMove = function() {
+		if (__absolute) {
 			let pos = __current_pos;
 			if (pos.x !== __sent_pos.x || pos.y !== __sent_pos.y) {
 				let el_stream_image = $("stream-image");
@@ -165,6 +201,7 @@ export function Mouse(record_callback) {
 				__sendEvent("mouse_move", {"to": to});
 				__sent_pos = pos;
 			}
+		}
 	};
 
 	var __translate = function(x, a, b, c, d) {
@@ -178,6 +215,10 @@ export function Mouse(record_callback) {
 			event.preventDefault();
 		}
 
+		if (!__absolute && !__isRelativeCaptured()) {
+			return;
+		}
+
 		let delta = {x: 0, y: 0};
 
 		if (tools.browser.is_firefox && !tools.browser.is_mac) {