diff --git a/Makefile b/Makefile index fdc4fb09..ab47094f 100644 --- a/Makefile +++ b/Makefile @@ -86,7 +86,7 @@ tox: testenv && cp /usr/share/kvmd/configs.default/kvmd/*.yaml /etc/kvmd \ && cp /usr/share/kvmd/configs.default/kvmd/*passwd /etc/kvmd \ && cp /usr/share/kvmd/configs.default/kvmd/*.secret /etc/kvmd \ - && cp /usr/share/kvmd/configs.default/kvmd/main/$(if $(P),$(P),$(DEFAULT_PLATFORM)).yaml /etc/kvmd/main.yaml \ + && cp /usr/share/kvmd/configs.default/kvmd/main.yaml /etc/kvmd/main.yaml \ && mkdir -p /etc/kvmd/override.d \ && cp /src/testenv/$(if $(P),$(P),$(DEFAULT_PLATFORM)).override.yaml /etc/kvmd/override.yaml \ && cd /src \ @@ -155,7 +155,7 @@ run-cfg: testenv && cp /usr/share/kvmd/configs.default/kvmd/*.yaml /etc/kvmd \ && cp /usr/share/kvmd/configs.default/kvmd/*passwd /etc/kvmd \ && cp /usr/share/kvmd/configs.default/kvmd/*.secret /etc/kvmd \ - && cp /usr/share/kvmd/configs.default/kvmd/main/$(if $(P),$(P),$(DEFAULT_PLATFORM)).yaml /etc/kvmd/main.yaml \ + && cp /usr/share/kvmd/configs.default/kvmd/main.yaml /etc/kvmd/main.yaml \ && mkdir -p /etc/kvmd/override.d \ && cp /testenv/$(if $(P),$(P),$(DEFAULT_PLATFORM)).override.yaml /etc/kvmd/override.yaml \ && $(if $(CMD),$(CMD),python -m kvmd.apps.kvmd -m) \ @@ -178,7 +178,7 @@ run-ipmi: testenv && cp /usr/share/kvmd/configs.default/kvmd/*.yaml /etc/kvmd \ && cp /usr/share/kvmd/configs.default/kvmd/*passwd /etc/kvmd \ && cp /usr/share/kvmd/configs.default/kvmd/*.secret /etc/kvmd \ - && cp /usr/share/kvmd/configs.default/kvmd/main/$(if $(P),$(P),$(DEFAULT_PLATFORM)).yaml /etc/kvmd/main.yaml \ + && cp /usr/share/kvmd/configs.default/kvmd/main.yaml /etc/kvmd/main.yaml \ && mkdir -p /etc/kvmd/override.d \ && cp /testenv/$(if $(P),$(P),$(DEFAULT_PLATFORM)).override.yaml /etc/kvmd/override.yaml \ && $(if $(CMD),$(CMD),python -m kvmd.apps.ipmi --run) \ @@ -201,7 +201,7 @@ run-vnc: testenv && cp /usr/share/kvmd/configs.default/kvmd/*.yaml /etc/kvmd \ && cp /usr/share/kvmd/configs.default/kvmd/*passwd /etc/kvmd \ && cp /usr/share/kvmd/configs.default/kvmd/*.secret /etc/kvmd \ - && cp /usr/share/kvmd/configs.default/kvmd/main/$(if $(P),$(P),$(DEFAULT_PLATFORM)).yaml /etc/kvmd/main.yaml \ + && cp /usr/share/kvmd/configs.default/kvmd/main.yaml /etc/kvmd/main.yaml \ && mkdir -p /etc/kvmd/override.d \ && cp /testenv/$(if $(P),$(P),$(DEFAULT_PLATFORM)).override.yaml /etc/kvmd/override.yaml \ && $(if $(CMD),$(CMD),python -m kvmd.apps.vnc --run) \ diff --git a/build/Dockerfile b/build/Dockerfile index 4b59732c..77b279a6 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -18,6 +18,7 @@ ENV TZ=Asia/Shanghai RUN cp /tmp/lib/* /lib/*-linux-*/ \ && pip install --no-cache-dir --root-user-action=ignore --disable-pip-version-check /tmp/wheel/*.whl \ + && pip install --no-cache-dir --root-user-action=ignore --disable-pip-version-check pyfatfs \ && rm -rf /tmp/lib /tmp/wheel RUN sed -i 's/deb.debian.org/mirrors.tuna.tsinghua.edu.cn/' /etc/apt/sources.list.d/debian.sources \ @@ -31,7 +32,7 @@ RUN if [ ${TARGETARCH} = arm ]; then ARCH=armhf; elif [ ${TARGETARCH} = arm64 ]; && chmod +x /usr/local/bin/ttyd \ && adduser kvmd --gecos "" --disabled-password \ && ln -sf /usr/share/tesseract-ocr/*/tessdata /usr/share/tessdata \ - && mkdir -p /etc/kvmd_backup/override.d /var/lib/kvmd/msd/images /var/lib/kvmd/msd/meta /var/lib/kvmd/pst/data /opt/vc/bin /run/kvmd /tmp/kvmd-nginx \ + && mkdir -p /etc/kvmd_backup/override.d /var/lib/kvmd/msd/images /var/lib/kvmd/msd/meta /var/lib/kvmd/pst/data /var/lib/kvmd/msd/NormalFiles /opt/vc/bin /run/kvmd /tmp/kvmd-nginx \ && touch /run/kvmd/ustreamer.sock diff --git a/configs/kvmd/override.yaml b/configs/kvmd/override.yaml index db5c54d1..a21e9064 100644 --- a/configs/kvmd/override.yaml +++ b/configs/kvmd/override.yaml @@ -21,6 +21,9 @@ kvmd: msd: #type: otg remount_cmd: /bin/true + msd_path: /var/lib/kvmd/msd + normalfiles_path: NormalFiles + normalfiles_size: 256 ocr: langs: diff --git a/kvmd/apps/kvmd/api/msd.py b/kvmd/apps/kvmd/api/msd.py index 98e85412..ca1c6cf1 100644 --- a/kvmd/apps/kvmd/api/msd.py +++ b/kvmd/apps/kvmd/api/msd.py @@ -87,6 +87,11 @@ class MsdApi: async def __set_connected_handler(self, req: Request) -> Response: await self.__msd.set_connected(valid_bool(req.query.get("connected"))) return make_json_response() + + @exposed_http("POST", "/msd/make_image") + async def __set_zipped_handler(self, req: Request) -> Response: + await self.__msd.make_image(valid_bool(req.query.get("zipped"))) + return make_json_response() # ===== diff --git a/kvmd/fstab.py b/kvmd/fstab.py index 3aadaa43..fd4ce050 100644 --- a/kvmd/fstab.py +++ b/kvmd/fstab.py @@ -37,8 +37,8 @@ class Partition: # ===== -def find_msd() -> Partition: - return _find_single("otgmsd") +def find_msd(msd_directory_path) -> Partition: + return _find_single("otgmsd", msd_directory_path) def find_pst() -> Partition: @@ -46,12 +46,12 @@ def find_pst() -> Partition: # ===== -def _find_single(part_type: str) -> Partition: +def _find_single(part_type: str, msd_directory_path: str) -> Partition: parts = _find_partitions(part_type, True) if len(parts) == 0: - if os.path.exists('/var/lib/kvmd/msd'): + if os.path.exists(msd_directory_path): #set default value - parts = [Partition(mount_path='/var/lib/kvmd/msd', root_path='/var/lib/kvmd/msd',group='kvmd', user='kvmd')] + parts = [Partition(mount_path = msd_directory_path, root_path = msd_directory_path, group = 'kvmd', user = 'kvmd')] else: raise RuntimeError(f"Can't find {part_type!r} mountpoint") return parts[0] diff --git a/kvmd/plugins/msd/__init__.py b/kvmd/plugins/msd/__init__.py index b2f9d50e..0b59b552 100644 --- a/kvmd/plugins/msd/__init__.py +++ b/kvmd/plugins/msd/__init__.py @@ -156,6 +156,9 @@ class BaseMsd(BasePlugin): async def set_connected(self, connected: bool) -> None: raise NotImplementedError() + + async def make_image(self, zipped: bool) -> None: + raise NotImplementedError() @contextlib.asynccontextmanager async def read_image(self, name: str) -> AsyncGenerator[BaseMsdReader, None]: diff --git a/kvmd/plugins/msd/otg/__init__.py b/kvmd/plugins/msd/otg/__init__.py index 92eac741..a1d7f8fe 100644 --- a/kvmd/plugins/msd/otg/__init__.py +++ b/kvmd/plugins/msd/otg/__init__.py @@ -27,6 +27,9 @@ import functools import time import os import copy +import pyfatfs +import pyfatfs.PyFat +import pyfatfs.PyFatFS from typing import AsyncGenerator @@ -37,8 +40,8 @@ from ....inotify import Inotify from ....yamlconf import Option from ....validators.basic import valid_bool -from ....validators.basic import valid_number -from ....validators.os import valid_command +from ....validators.basic import valid_number, valid_stripped_string_not_empty +from ....validators.os import valid_command, valid_abs_path from ....validators.kvm import valid_msd_image_name from .... import aiotools @@ -120,23 +123,30 @@ class Plugin(BaseMsd): # pylint: disable=too-many-instance-attributes read_chunk_size: int, write_chunk_size: int, sync_chunk_size: int, + normalfiles_size: int, remount_cmd: list[str], initial: dict, + normalfiles_path: str, + msd_path: str, + gadget: str, # XXX: Not from options, see /kvmd/apps/kvmd/__init__.py for details ) -> None: self.__read_chunk_size = read_chunk_size self.__write_chunk_size = write_chunk_size self.__sync_chunk_size = sync_chunk_size + self.__normalfiles_path = normalfiles_path + self.__msd_path = msd_path + self.__normalfiles_size = normalfiles_size self.__initial_image: str = initial["image"] self.__initial_cdrom: bool = initial["cdrom"] self.__drive = Drive(gadget, instance=0, lun=0) - self.__storage = Storage(fstab.find_msd().root_path, remount_cmd) + self.__storage = Storage(fstab.find_msd(msd_path).root_path, remount_cmd) self.__reader: (MsdFileReader | None) = None self.__writer: (MsdFileWriter | None) = None @@ -165,10 +175,15 @@ class Plugin(BaseMsd): # pylint: disable=too-many-instance-attributes "image": Option("", type=valid_msd_image_name, if_empty=""), "cdrom": Option(False, type=valid_bool), }, + "msd_path": Option("/var/lib/kvmd/msd", type=valid_abs_path), + "normalfiles_path": Option("NormalFiles", type=valid_stripped_string_not_empty), + "normalfiles_size": Option(256, type=functools.partial(valid_number, min=64)), } # ===== + # ===== + async def get_state(self) -> dict: async with self.__state._lock: # pylint: disable=protected-access storage: (dict | None) = None @@ -184,6 +199,7 @@ class Plugin(BaseMsd): # pylint: disable=too-many-instance-attributes storage["downloading"] = (self.__reader.get_state() if self.__reader else None) storage["uploading"] = (self.__writer.get_state() if self.__writer else None) + storage["filespath"] = self.__normalfiles_path vd: (dict | None) = None if self.__state.vd: @@ -286,15 +302,19 @@ class Plugin(BaseMsd): # pylint: disable=too-many-instance-attributes self.__drive.set_rw_flag(self.__state.vd.rw) self.__drive.set_cdrom_flag(self.__state.vd.cdrom) - #reboot UDC to fix otg cd-rom and flash switch - udc_path = self.__drive.get_udc_path() - with open(udc_path) as file: - enabled = bool(file.read().strip()) - if enabled: + #reset UDC to fix otg cd-rom and flash switch + try: + udc_path = self.__drive.get_udc_path() + with open(udc_path) as file: + enabled = bool(file.read().strip()) + if enabled: + with open(udc_path, "w") as file: + file.write("\n") with open(udc_path, "w") as file: - file.write("\n") - with open(udc_path, "w") as file: - file.write(sorted(os.listdir("/sys/class/udc"))[0]) + file.write(sorted(os.listdir("/sys/class/udc"))[0]) + except: + logger = get_logger(0) + logger.error("Can't reset UDC") if self.__state.vd.rw: await self.__state.vd.image.remount_rw(True) self.__drive.set_image_path(self.__state.vd.image.path) @@ -306,6 +326,86 @@ class Plugin(BaseMsd): # pylint: disable=too-many-instance-attributes self.__state.vd.connected = connected + @aiotools.atomic_fg + async def make_image(self, zipped: bool) -> None: + #Note: img size >= 64M + def create_fat_image(img_size: int, file_img_path: str, source_dir: str, fat_type: int = 32, label: str = 'One-KVM'): + def add_directory_to_fat(fat: str, src_path: str, dst_path: str): + for item in os.listdir(src_path): + src_item_path = os.path.join(src_path, item) + dst_item_path = os.path.join(dst_path, item) + + if os.path.isdir(src_item_path): + fat.makedir(dst_item_path) + add_directory_to_fat(fat, src_item_path, dst_item_path) + elif os.path.isfile(src_item_path): + with open(src_item_path, 'rb') as src_file: + fat.create(dst_item_path) + with fat.open(dst_item_path, 'wb') as dst_file: + dst_file.write(src_file.read()) + print(file_img_path) + with open(file_img_path, 'wb') as f: + f.seek(img_size * 1024 *1024 - 1) + f.write(b'\0') + fat_file = pyfatfs.PyFat.PyFat() + try: + fat_file.mkfs(file_img_path, fat_type = fat_type, label = label) + except Exception as e: + get_logger(0).exception(f"Error making FAT Filesystem: {e}") + finally: + fat_file.close() + fat_handle = pyfatfs.PyFatFS.PyFatFS(file_img_path) + try: + add_directory_to_fat(fat_handle, source_dir, '/') + except Exception as e: + get_logger(0).exception(f"Error adding directory to FAT image: {e}") + finally: + fat_handle.close() + + def extract_fat_image(file_img_path: str, output_dir: str): + try: + for root, dirs, files in os.walk(output_dir, topdown=False): + for name in files: + os.remove(os.path.join(root, name)) + for name in dirs: + os.rmdir(os.path.join(root, name)) + except Exception as e: + get_logger(0).exception(f"Error removing normal file or directory: {e}") + fat_handle = pyfatfs.PyFatFS.PyFatFS(file_img_path) + try: + def extract_directory(fat_handle, src_path: str, dst_path: str): + for entry in fat_handle.listdir(src_path): + src_item_path = os.path.join(src_path, entry) + dst_item_path = os.path.join(dst_path, entry) + + if fat_handle.gettype(src_item_path) is pyfatfs.PyFatFS.ResourceType.directory: + os.makedirs(dst_item_path, exist_ok=True) + extract_directory(fat_handle, src_item_path, dst_item_path) + else: + with fat_handle.open(src_item_path, 'rb') as src_file: + with open(dst_item_path, 'wb') as dst_file: + dst_file.write(src_file.read()) + extract_directory(fat_handle, '/', output_dir) + except Exception as e: + get_logger(0).exception(f"Error extracting FAT image: {e}") + finally: + fat_handle.close() + + async with self.__state.busy(): + msd_path = self.__msd_path + file_storage_path = os.path.join(msd_path, self.__normalfiles_path) + file_img_path = os.path.join(msd_path, self.__normalfiles_path + ".img") + img_size = self.__normalfiles_size + if zipped: + if not os.path.exists(file_storage_path): + os.makedirs(file_storage_path) + if os.path.exists(file_img_path): + os.remove(file_img_path) + create_fat_image(img_size, file_img_path, file_storage_path) + else: + if os.path.exists(file_img_path): + extract_fat_image(file_img_path, file_storage_path) + @contextlib.asynccontextmanager async def read_image(self, name: str) -> AsyncGenerator[MsdFileReader, None]: try: diff --git a/testenv/requirements.txt b/testenv/requirements.txt index 36d2407a..ba60982e 100644 --- a/testenv/requirements.txt +++ b/testenv/requirements.txt @@ -6,3 +6,4 @@ pyrad types-PyYAML types-aiofiles luma.oled +pyfatfs diff --git a/testenv/v2-hdmiusb-rpi4.override.yaml b/testenv/v2-hdmiusb-rpi4.override.yaml index febd94fb..d09407b7 100644 --- a/testenv/v2-hdmiusb-rpi4.override.yaml +++ b/testenv/v2-hdmiusb-rpi4.override.yaml @@ -13,7 +13,11 @@ kvmd: noop: true msd: - type: disabled + type: otg + remount_cmd: /bin/true + msd_path: /var/lib/kvmd/msd + normalfiles_path: NormalFiles + normalfiles_size: 64 streamer: cmd: diff --git a/web/kvm/index.html b/web/kvm/index.html index 23b77b61..68643697 100644 --- a/web/kvm/index.html +++ b/web/kvm/index.html @@ -596,8 +596,18 @@ +