powerful configuration management

PKGBUILD

@@ -21,6 +21,7 @@ depends=(
 	python-setproctitle
 	python-systemd
 	python-dbus
+	python-pygments
 	v4l-utils
 )
 makedepends=(python-setuptools)

kvmd/application.py

@@ -1,21 +1,188 @@
+import sys
+import os
 import argparse
 import logging
 import logging.config
 
+from typing import Tuple
+from typing import List
 from typing import Dict
+from typing import Sequence
+from typing import Union
 
-from .yaml import load_yaml_file
+import pygments
+import pygments.lexers.data
+import pygments.formatters
+
+from .yamlconf import make_config
+from .yamlconf import Section
+from .yamlconf import Option
+from .yamlconf import build_raw_from_options
+from .yamlconf.dumper import make_config_dump
+from .yamlconf.loader import load_yaml_file
 
 
 # =====
-def init() -> Dict:
+def init() -> Tuple[argparse.ArgumentParser, List[str], Section]:
-    parser = argparse.ArgumentParser()
+    args_parser = argparse.ArgumentParser(add_help=False)
-    parser.add_argument("-c", "--config", required=True, metavar="<path>")
+    args_parser.add_argument("-c", "--config", dest="config_path", default="/etc/kvmd/kvmd.yaml", metavar="<file>")
-    options = parser.parse_args()
+    args_parser.add_argument("-o", "--set-options", dest="set_options", default=[], nargs="+")
+    args_parser.add_argument("-m", "--dump-config", dest="dump_config", action="store_true")
+    (options, remaining) = args_parser.parse_known_args(sys.argv)
 
-    config: Dict = load_yaml_file(options.config)
+    options.config_path = os.path.expanduser(options.config_path)
+    if os.path.exists(options.config_path):
+        raw_config = load_yaml_file(options.config_path)
+    else:
+        raw_config = {}
+    _merge_dicts(raw_config, build_raw_from_options(options.set_options))
+    scheme = _get_config_scheme()
+    config = make_config(raw_config, scheme)
+
+    if options.dump_config:
+        dump = make_config_dump(config)
+        if sys.stdout.isatty():
+            dump = pygments.highlight(
+                dump,
+                pygments.lexers.data.YamlLexer(),
+                pygments.formatters.TerminalFormatter(bg="dark"),  # pylint: disable=no-member
+            )
+        print(dump)
+        sys.exit(0)
 
     logging.captureWarnings(True)
-    logging.config.dictConfig(config["logging"])
+    logging.config.dictConfig(config.logging)
+    return (args_parser, remaining, config)
 
-    return config
+
+# =====
+def _merge_dicts(dest: Dict, src: Dict) -> None:
+    for key in src:
+        if key in dest:
+            if isinstance(dest[key], dict) and isinstance(src[key], dict):
+                _merge_dicts(dest[key], src[key])
+                continue
+        dest[key] = src[key]
+
+
+def _as_pin(pin: int) -> int:
+    if not isinstance(pin, int) or pin <= 0:
+        raise ValueError("Invalid pin number")
+    return pin
+
+
+def _as_optional_pin(pin: int) -> int:
+    if not isinstance(pin, int) or pin == 0:
+        raise ValueError("Invalid optional pin number")
+    return pin
+
+
+def _as_path(path: str) -> str:
+    if not isinstance(path, str):
+        raise ValueError("Invalid path")
+    path = str(path).strip()
+    if not path:
+        raise ValueError("Invalid path")
+    return path
+
+
+def _as_optional_path(path: str) -> str:
+    if not isinstance(path, str):
+        raise ValueError("Invalid path")
+    return str(path).strip()
+
+
+def _as_string_list(values: Union[str, Sequence]) -> List[str]:
+    if isinstance(values, str):
+        values = [values]
+    return list(map(str, values))
+
+
+def _get_config_scheme() -> Dict:
+    return {
+        "kvmd": {
+            "server": {
+                "host": Option(default="localhost"),
+                "port": Option(default=0),
+                "unix": Option(default="", type=_as_optional_path),
+                "unix_rm": Option(default=False),
+                "unix_mode": Option(default=0),
+                "heartbeat": Option(default=3.0),
+                "access_log_format": Option(default="[%P / %{X-Real-IP}i] '%r' => %s; size=%b ---"
+                                                    " referer='%{Referer}i'; user_agent='%{User-Agent}i'"),
+            },
+
+            "auth": {
+                "htpasswd": Option(default="/etc/kvmd/htpasswd", type=_as_path),
+            },
+
+            "info": {
+                "meta":   Option(default="/etc/kvmd/meta.yaml", type=_as_path),
+                "extras": Option(default="/usr/share/kvmd/extras", type=_as_path),
+            },
+
+            "hid": {
+                "pinout": {
+                    "reset": Option(default=0, type=_as_pin),
+                },
+                "reset_delay":    Option(default=0.1),
+                "device":         Option(default="", type=_as_path),
+                "speed":          Option(default=115200),
+                "read_timeout":   Option(default=2.0),
+                "read_retries":   Option(default=10),
+                "common_retries": Option(default=100),
+                "retries_delay":  Option(default=0.1),
+                "noop":           Option(default=False),
+                "state_poll":     Option(default=0.1),
+            },
+
+            "atx": {
+                "pinout": {
+                    "power_led":    Option(default=0, type=_as_pin),
+                    "hdd_led":      Option(default=0, type=_as_pin),
+                    "power_switch": Option(default=0, type=_as_pin),
+                    "reset_switch": Option(default=0, type=_as_pin),
+                },
+                "click_delay":      Option(default=0.1),
+                "long_click_delay": Option(default=5.5),
+                "state_poll":       Option(default=0.1),
+            },
+
+            "msd": {
+                "pinout": {
+                    "target": Option(default=0, type=_as_pin),
+                    "reset":  Option(default=0, type=_as_pin),
+                },
+                "device":      Option(default="", type=_as_path),
+                "init_delay":  Option(default=2.0),
+                "reset_delay": Option(default=1.0),
+                "write_meta":  Option(default=True),
+                "chunk_size":  Option(default=65536),
+            },
+
+            "streamer": {
+                "pinout": {
+                    "cap":  Option(default=-1, type=_as_optional_pin),
+                    "conv": Option(default=-1, type=_as_optional_pin),
+                },
+
+                "sync_delay":         Option(default=1.0),
+                "init_delay":         Option(default=1.0),
+                "init_restart_after": Option(default=0.0),
+                "shutdown_delay":     Option(default=10.0),
+                "state_poll":         Option(default=1.0),
+
+                "quality":     Option(default=80),
+                "desired_fps": Option(default=0),
+
+                "host":    Option(default="localhost"),
+                "port":    Option(default=0),
+                "unix":    Option(default="", type=_as_optional_path),
+                "timeout": Option(default=2.0),
+
+                "cmd": Option(default=["/bin/true"], type=_as_string_list),
+            },
+        },
+
+        "logging": Option(default={}),
+    }

kvmd/apps/cleanup/__init__.py

@@ -10,25 +10,25 @@ from ... import gpio
 
 # =====
 def main() -> None:
-    config = init()["kvmd"]
+    config = init()[2].kvmd
     logger = get_logger(0)
 
     logger.info("Cleaning up ...")
     with gpio.bcm():
         for (name, pin) in [
-            ("hid_reset", config["hid"]["pinout"]["reset"]),
+            ("hid_reset", config.hid.pinout.reset),
-            ("msd_target", config["msd"]["pinout"]["target"]),
+            ("msd_target", config.hid.pinout.target),
-            ("msd_reset", config["msd"]["pinout"]["reset"]),
+            ("msd_reset", config.msd.pinout.reset),
-            ("atx_power_switch", config["atx"]["pinout"]["power_switch"]),
+            ("atx_power_switch", config.atx.pinout.power_switch),
-            ("atx_reset_switch", config["atx"]["pinout"]["reset_switch"]),
+            ("atx_reset_switch", config.atx.pinout.reset_switch),
-            ("streamer_cap", config["streamer"]["pinout"].get("cap", -1)),
+            ("streamer_cap", config.streamer.pinout.cap),
-            ("streamer_conv", config["streamer"]["pinout"].get("conv", -1)),
+            ("streamer_conv", config.streamer.pinout.conv),
         ]:
             if pin > 0:
                 logger.info("Writing value=0 to pin=%d (%s)", pin, name)
                 gpio.set_output(pin, initial=False)
 
-    streamer = os.path.basename(config["streamer"]["cmd"][0])
+    streamer = os.path.basename(config.streamer.cmd[0])
     logger.info("Trying to find and kill %r ...", streamer)
     try:
         subprocess.check_output(["killall", streamer], stderr=subprocess.STDOUT)
@@ -37,7 +37,7 @@ def main() -> None:
     except subprocess.CalledProcessError:
         pass
 
-    unix_path = config["server"].get("unix", "")
+    unix_path = config.server.unix
     if unix_path and os.path.exists(unix_path):
         logger.info("Removing socket %r ...", unix_path)
         os.remove(unix_path)

kvmd/apps/kvmd/__init__.py

@@ -17,77 +17,77 @@ from .server import Server
 
 # =====
 def main() -> None:
-    config = init()["kvmd"]
+    config = init()[2].kvmd
     with gpio.bcm():
         loop = asyncio.get_event_loop()
 
         auth_manager = AuthManager(
-            htpasswd_path=str(config.get("auth", {}).get("htpasswd", "/etc/kvmd/htpasswd")),
+            htpasswd_path=config.auth.htpasswd,
         )
 
         info_manager = InfoManager(
-            meta_path=str(config.get("info", {}).get("meta", "/etc/kvmd/meta.yaml")),
+            meta_path=config.info.meta,
-            extras_path=str(config.get("info", {}).get("extras", "/usr/share/kvmd/extras")),
+            extras_path=config.info.extras,
             loop=loop,
         )
 
         log_reader = LogReader(loop)
 
         hid = Hid(
-            reset=int(config["hid"]["pinout"]["reset"]),
+            reset=config.hid.pinout.reset,
-            reset_delay=float(config["hid"].get("reset_delay", 0.1)),
+            reset_delay=config.hid.reset_delay,
 
-            device_path=str(config["hid"]["device"]),
+            device_path=config.hid.device,
-            speed=int(config["hid"].get("speed", 115200)),
+            speed=config.hid.speed,
-            read_timeout=float(config["hid"].get("read_timeout", 2)),
+            read_timeout=config.hid.read_timeout,
-            read_retries=int(config["hid"].get("read_retries", 10)),
+            read_retries=config.hid.read_retries,
-            common_retries=int(config["hid"].get("common_retries", 100)),
+            common_retries=config.hid.common_retries,
-            retries_delay=float(config["hid"].get("retries_delay", 0.1)),
+            retries_delay=config.hid.retries_delay,
-            noop=bool(config["hid"].get("noop", False)),
+            noop=config.hid.noop,
 
-            state_poll=float(config["hid"].get("state_poll", 0.1)),
+            state_poll=config.hid.state_poll,
         )
 
         atx = Atx(
-            power_led=int(config["atx"]["pinout"]["power_led"]),
+            power_led=config.atx.pinout.power_led,
-            hdd_led=int(config["atx"]["pinout"]["hdd_led"]),
+            hdd_led=config.atx.pinout.hdd_led,
+            power_switch=config.atx.pinout.power_switch,
+            reset_switch=config.atx.pinout.reset_switch,
 
-            power_switch=int(config["atx"]["pinout"]["power_switch"]),
+            click_delay=config.atx.click_delay,
-            reset_switch=int(config["atx"]["pinout"]["reset_switch"]),
+            long_click_delay=config.atx.long_click_delay,
-            click_delay=float(config["atx"].get("click_delay", 0.1)),
+            state_poll=config.atx.state_poll,
-            long_click_delay=float(config["atx"].get("long_click_delay", 5.5)),
-            state_poll=float(config["atx"].get("state_poll", 0.1)),
         )
 
         msd = MassStorageDevice(
-            target=int(config["msd"]["pinout"]["target"]),
+            target=config.msd.pinout.target,
-            reset=int(config["msd"]["pinout"]["reset"]),
+            reset=config.msd.pinout.reset,
 
-            device_path=str(config["msd"]["device"]),
+            device_path=config.msd.device,
-            init_delay=float(config["msd"].get("init_delay", 2)),
+            init_delay=config.msd.init_delay,
-            reset_delay=float(config["msd"].get("reset_delay", 1)),
+            reset_delay=config.msd.reset_delay,
-            write_meta=bool(config["msd"].get("write_meta", True)),
+            write_meta=config.msd.write_meta,
 
             loop=loop,
         )
 
         streamer = Streamer(
-            cap_power=int(config["streamer"].get("pinout", {}).get("cap", -1)),
+            cap_power=config.streamer.pinout.cap,
-            conv_power=int(config["streamer"].get("pinout", {}).get("conv", -1)),
+            conv_power=config.streamer.pinout.conv,
-            sync_delay=float(config["streamer"].get("sync_delay", 1)),
+            sync_delay=config.streamer.sync_delay,
-            init_delay=float(config["streamer"].get("init_delay", 1)),
+            init_delay=config.streamer.init_delay,
-            init_restart_after=float(config["streamer"].get("init_restart_after", 0)),
+            init_restart_after=config.streamer.init_restart_after,
-            state_poll=float(config["streamer"].get("state_poll", 1)),
+            state_poll=config.streamer.state_poll,
 
-            quality=int(config["streamer"].get("quality", 80)),
+            quality=config.streamer.quality,
-            desired_fps=int(config["streamer"].get("desired_fps", 0)),
+            desired_fps=config.streamer.desired_fps,
 
-            host=str(config["streamer"].get("host", "localhost")),
+            host=config.streamer.host,
-            port=int(config["streamer"].get("port", 0)),
+            port=config.streamer.port,
-            unix_path=str(config["streamer"].get("unix", "")),
+            unix_path=config.streamer.unix,
-            timeout=float(config["streamer"].get("timeout", 2)),
+            timeout=config.streamer.timeout,
 
-            cmd=list(map(str, config["streamer"]["cmd"])),
+            cmd=config.streamer.cmd,
 
             loop=loop,
         )
@@ -102,21 +102,18 @@ def main() -> None:
             msd=msd,
             streamer=streamer,
 
-            access_log_format=str(config["server"].get(
+            access_log_format=config.server.access_log_format,
-                "access_log_format",
+            heartbeat=config.server.heartbeat,
-                "[%P / %{X-Real-IP}i] '%r' => %s; size=%b --- referer='%{Referer}i'; user_agent='%{User-Agent}i'",
+            streamer_shutdown_delay=config.streamer.shutdown_delay,
-            )),
+            msd_chunk_size=config.msd.chunk_size,
-            heartbeat=float(config["server"].get("heartbeat", 3)),
-            streamer_shutdown_delay=float(config["streamer"].get("shutdown_delay", 10)),
-            msd_chunk_size=int(config["msd"].get("chunk_size", 65536)),
 
             loop=loop,
         ).run(
-            host=str(config["server"].get("host", "localhost")),
+            host=config.server.host,
-            port=int(config["server"].get("port", 0)),
+            port=config.server.port,
-            unix_path=str(config["server"].get("unix", "")),
+            unix_path=config.server.unix,
-            unix_rm=bool(config["server"].get("unix_rm", False)),
+            unix_rm=config.server.unix_rm,
-            unix_mode=int(config["server"].get("unix_mode", 0)),
+            unix_mode=config.server.unix_mode,
         )
 
     get_logger().info("Bye-bye")

kvmd/apps/kvmd/info.py

@@ -6,7 +6,7 @@ from typing import Dict
 import dbus  # pylint: disable=import-error
 import dbus.exceptions  # pylint: disable=import-error
 
-from ...yaml import load_yaml_file
+from ...yamlconf.loader import load_yaml_file
 
 
 # =====

kvmd/yamlconf/__init__.py

@@ -0,0 +1,105 @@
+import json
+
+from typing import Tuple
+from typing import List
+from typing import Dict
+from typing import Callable
+from typing import Optional
+from typing import Any
+
+
+# =====
+def build_raw_from_options(options: List[str]) -> Dict[str, Any]:
+    raw: Dict[str, Any] = {}
+    for option in options:
+        (key, value) = (option.split("=", 1) + [None])[:2]  # type: ignore
+        if len(key.strip()) == 0:
+            raise ValueError("Empty option key (required 'key=value' instead of '{}')".format(option))
+        if value is None:
+            raise ValueError("No value for key '{}'".format(key))
+
+        section = raw
+        subs = list(map(str.strip, key.split("/")))
+        for sub in subs[:-1]:
+            section.setdefault(sub, {})
+            section = section[sub]
+        section[subs[-1]] = _parse_value(value)
+    return raw
+
+
+def _parse_value(value: str) -> Any:
+    value = value.strip()
+    if (
+        not value.isdigit()
+        and value not in ["true", "false", "null"]
+        and not value.startswith(("{", "[", "\""))
+    ):
+        value = "\"{}\"".format(value)
+    return json.loads(value)
+
+
+# =====
+class Section(dict):
+    def __init__(self) -> None:
+        dict.__init__(self)
+        self.__meta: Dict[str, Dict[str, Any]] = {}
+
+    def _set_meta(self, name: str, default: Any, help: str) -> None:  # pylint: disable=redefined-builtin
+        self.__meta[name] = {
+            "default": default,
+            "help":    help,
+        }
+
+    def _get_default(self, name: str) -> Any:
+        return self.__meta[name]["default"]
+
+    def _get_help(self, name: str) -> str:
+        return self.__meta[name]["help"]
+
+    def __getattribute__(self, name: str) -> Any:
+        if name in self:
+            return self[name]
+        else:  # For pickling
+            return dict.__getattribute__(self, name)
+
+
+class Option:
+    __type = type
+
+    def __init__(self, default: Any, help: str="", type: Optional[Callable[[Any], Any]]=None) -> None:  # pylint: disable=redefined-builtin
+        self.default = default
+        self.help = help
+        self.type: Callable[[Any], Any] = (type or (self.__type(default) if default is not None else str))  # type: ignore
+
+    def __repr__(self) -> str:
+        return "<Option(default={self.default}, type={self.type}, help={self.help})>".format(self=self)
+
+
+# =====
+def make_config(raw: Dict[str, Any], scheme: Dict[str, Any], _keys: Tuple[str, ...]=()) -> Section:
+    if not isinstance(raw, dict):
+        raise ValueError("The node '{}' must be a dictionary".format("/".join(_keys) or "/"))
+
+    config = Section()
+    for (key, option) in scheme.items():
+        full_key = _keys + (key,)
+        full_name = "/".join(full_key)
+
+        if isinstance(option, Option):
+            value = raw.get(key, option.default)
+            try:
+                value = option.type(value)
+            except Exception:
+                raise ValueError("Invalid value '{value}' for key '{key}'".format(key=full_name, value=value))
+            config[key] = value
+            config._set_meta(  # pylint: disable=protected-access
+                name=key,
+                default=option.default,
+                help=option.help,
+            )
+        elif isinstance(option, dict):
+            config[key] = make_config(raw.get(key, {}), option, full_key)
+        else:
+            raise RuntimeError("Incorrect scheme definition for key '{}':"
+                               " the value is {}, not dict or Option()".format(full_name, type(option)))
+    return config

kvmd/yamlconf/dumper.py

@@ -0,0 +1,41 @@
+# pylint: skip-file
+# infinite recursion
+
+
+import operator
+
+from typing import Tuple
+from typing import List
+from typing import Any
+
+import yaml
+
+from . import Section
+
+
+# =====
+def make_config_dump(config: Section) -> str:
+    return "\n".join(_inner_make_dump(config))
+
+
+def _inner_make_dump(config: Section, _path: Tuple[str, ...]=()) -> List[str]:
+    lines = []
+    for (key, value) in sorted(config.items(), key=operator.itemgetter(0)):
+        indent = len(_path) * "    "
+        if isinstance(value, Section):
+            lines.append("{}{}:".format(indent, key))
+            lines += _inner_make_dump(value, _path + (key,))
+            lines.append("")
+        else:
+            default = config._get_default(key)  # pylint: disable=protected-access
+            comment = config._get_help(key)  # pylint: disable=protected-access
+            if default == value:
+                lines.append("{}{}: {} # {}".format(indent, key, _make_yaml(value), comment))
+            else:
+                lines.append("{}# {}: {} # {}".format(indent, key, _make_yaml(default), comment))
+                lines.append("{}{}: {}".format(indent, key, _make_yaml(value)))
+    return lines
+
+
+def _make_yaml(value: Any) -> str:
+    return yaml.dump(value, allow_unicode=True).replace("\n...\n", "").strip()

setup.py

@@ -18,6 +18,7 @@ def main() -> None:
 
         packages=[
             "kvmd",
+            "kvmd.yamlconf",
             "kvmd.apps",
             "kvmd.apps.kvmd",
             "kvmd.apps.cleanup",

testenv/requirements.txt

@@ -8,4 +8,5 @@ pyserial
 setproctitle
 systemd-python
 dbus-python
+pygments
 tox