/msd/write_remote handle

kvmd/apps/kvmd/api/msd.py

@@ -20,19 +20,34 @@
 # ========================================================================== #
 
 
+import time
+
+from typing import Dict
+from typing import Optional
+
+import aiohttp
+
 from aiohttp.web import Request
 from aiohttp.web import Response
+from aiohttp.web import StreamResponse
 
 from ....logging import get_logger
 
+from .... import htclient
+
 from ....plugins.msd import BaseMsd
 
 from ....validators.basic import valid_bool
 from ....validators.basic import valid_int_f0
+from ....validators.basic import valid_float_f01
+from ....validators.net import valid_url
 from ....validators.kvm import valid_msd_image_name
 
 from ..http import exposed_http
 from ..http import make_json_response
+from ..http import make_json_exception
+from ..http import start_streaming
+from ..http import stream_json
 from ..http import get_multipart_reader
 from ..http import get_multipart_reader_str
 from ..http import get_multipart_reader_field
@@ -67,29 +82,79 @@ class MsdApi:
         await self.__msd.set_connected(valid_bool(request.query.get("connected")))
         return make_json_response()
 
+    # =====
+
     @exposed_http("POST", "/msd/write")
     async def __write_handler(self, request: Request) -> Response:
-        logger = get_logger(0)
         reader = await get_multipart_reader(request)
-        name = ""
+        name = valid_msd_image_name(await get_multipart_reader_str(reader, "image"))
+        size = valid_int_f0(await get_multipart_reader_str(reader, "size"))
+        data_field = await get_multipart_reader_field(reader, "data")
+
         written = 0
+        async with self.__msd.write_image(name, size) as chunk_size:
+            while True:
+                chunk = await data_field.read_chunk(chunk_size)
+                if not chunk:
+                    break
+                written = await self.__msd.write_image_chunk(chunk)
+
+        return make_json_response(self.__make_write_info(name, size, written))
+
+    @exposed_http("POST", "/msd/write_remote")
+    async def __write_remote_handler(self, request: Request) -> StreamResponse:  # pylint: disable=too-many-locals
+        url = valid_url(request.query.get("url"))
+        insecure = valid_bool(request.query.get("insecure", "0"))
+        timeout = valid_float_f01(request.query.get("timeout", 10.0))
+
+        name = ""
+        size = written = 0
+        response: Optional[StreamResponse] = None
+
+        async def stream_write_info() -> None:
+            assert response is not None
+            await stream_json(response, self.__make_write_info(name, size, written))
+
         try:
-            name = valid_msd_image_name(await get_multipart_reader_str(reader, "image"))
-            size = valid_int_f0(await get_multipart_reader_str(reader, "size"))
+            async with htclient.download(
+                url=url,
+                verify=(not insecure),
+                timeout=timeout,
+                read_timeout=(7 * 24 * 3600),
+            ) as remote:
 
-            data_field = await get_multipart_reader_field(reader, "data")
+                name = str(request.query.get("image", "")).strip()
+                if len(name) == 0:
+                    name = htclient.get_filename(remote)
+                name = valid_msd_image_name(name)
 
-            async with self.__msd.write_image(name, size):
-                logger.info("Writing image %r to MSD ...", name)
-                while True:
-                    chunk = await data_field.read_chunk(self.__msd.get_upload_chunk_size())
-                    if not chunk:
-                        break
-                    written = await self.__msd.write_image_chunk(chunk)
-        finally:
-            if written != 0:
-                logger.info("Written image %r with size=%d bytes to MSD", name, written)
-        return make_json_response({"image": {"name": name, "size": written}})
+                size = htclient.get_content_length(remote)
+
+                get_logger(0).info("Downloading image %r as %r to MSD ...", url, name)
+                async with self.__msd.write_image(name, size) as chunk_size:
+                    response = await start_streaming(request, "application/stream+json")
+                    last_report_ts = 0
+                    async for chunk in remote.content.iter_chunked(chunk_size):
+                        written = await self.__msd.write_image_chunk(chunk)
+                        now = int(time.time())
+                        if last_report_ts + 1 < now:
+                            await stream_write_info()
+                            last_report_ts = now
+
+                await stream_write_info()
+                return response
+
+        except Exception as err:
+            if response is not None:
+                await stream_write_info()
+            elif isinstance(err, aiohttp.ClientError):
+                return make_json_exception(err, 400)
+            raise
+
+    def __make_write_info(self, name: str, size: int, written: int) -> Dict:
+        return {"image": {"name": name, "size": size, "written": written}}
+
+    # =====
 
     @exposed_http("POST", "/msd/remove")
     async def __remove_handler(self, request: Request) -> Response:

kvmd/apps/kvmd/http.py

@@ -176,6 +176,10 @@ async def start_streaming(request: aiohttp.web.Request, content_type: str) -> ai
     return response
 
 
+async def stream_json(response: aiohttp.web.StreamResponse, result: Dict) -> None:
+    await response.write(json.dumps(result).encode("utf-8") + b"\r\n")
+
+
 # =====
 async def get_multipart_reader(request: aiohttp.web.Request) -> aiohttp.MultipartReader:
     try: