From 48b3c4538efbb5e5ac4e3326c8b5e1b7e3165de7 Mon Sep 17 00:00:00 2001
From: SukkaW <isukkaw@gmail.com>
Date: Sun, 31 Dec 2023 00:04:17 +0800
Subject: [PATCH] Chore: refactor fetch text by line & load previous speedtest
 set

---
 Build/build-speedtest-domainset.ts | 11 ++++++++
 Build/lib/fetch-text-by-line.ts    | 41 ++++++++++--------------------
 2 files changed, 25 insertions(+), 27 deletions(-)

diff --git a/Build/build-speedtest-domainset.ts b/Build/build-speedtest-domainset.ts
index ab52d57a..2b72a945 100644
--- a/Build/build-speedtest-domainset.ts
+++ b/Build/build-speedtest-domainset.ts
@@ -10,6 +10,8 @@ import { fetchWithRetry } from './lib/fetch-retry';
 import { SHARED_DESCRIPTION } from './lib/constants';
 import { getGorhillPublicSuffixPromise } from './lib/get-gorhill-publicsuffix';
 import picocolors from 'picocolors';
+import { fetchRemoteTextByLine } from './lib/fetch-text-by-line';
+import { processLine } from './lib/process-line';
 
 const s = new Sema(2);
 
@@ -65,6 +67,7 @@ const querySpeedtestApi = async (keyword: string): Promise<Array<string | null>>
 };
 
 export const buildSpeedtestDomainSet = task(import.meta.path, async () => {
+  // Predefined domainset
   /** @type {Set<string>} */
   const domains = new Set<string>([
     '.speedtest.net',
@@ -130,6 +133,14 @@ export const buildSpeedtestDomainSet = task(import.meta.path, async () => {
     '.backend.librespeed.org'
   ]);
 
+  // Download previous speedtest domainset
+  for await (const l of await fetchRemoteTextByLine('https://ruleset.skk.moe/List/domainset/speedtest.conf')) {
+    const line = processLine(l);
+    if (line) {
+      domains.add(line);
+    }
+  }
+
   let timer;
 
   const pMap = ([
diff --git a/Build/lib/fetch-text-by-line.ts b/Build/lib/fetch-text-by-line.ts
index a82e94cc..53a87e71 100644
--- a/Build/lib/fetch-text-by-line.ts
+++ b/Build/lib/fetch-text-by-line.ts
@@ -23,16 +23,10 @@ import { fetchWithRetry, defaultRequestInit } from './fetch-retry';
 
 const decoder = new TextDecoder('utf-8');
 
-export async function *readFileByLine(file: string | URL | BunFile): AsyncGenerator<string> {
-  if (typeof file === 'string') {
-    file = Bun.file(file);
-  } else if (!('writer' in file)) {
-    file = Bun.file(file);
-  }
-
+async function *createTextLineAsyncGeneratorFromStreamSource(stream: ReadableStream<Uint8Array>): AsyncGenerator<string> {
   let buf = '';
 
-  for await (const chunk of file.stream()) {
+  for await (const chunk of stream) {
     const chunkStr = decoder.decode(chunk).replaceAll('\r\n', '\n');
     for (let i = 0, len = chunkStr.length; i < len; i++) {
       const char = chunkStr[i];
@@ -50,7 +44,17 @@ export async function *readFileByLine(file: string | URL | BunFile): AsyncGenera
   }
 }
 
-export async function *createReadlineInterfaceFromResponse(resp: Response): AsyncGenerator<string> {
+export function readFileByLine(file: string | URL | BunFile): AsyncGenerator<string> {
+  if (typeof file === 'string') {
+    file = Bun.file(file);
+  } else if (!('writer' in file)) {
+    file = Bun.file(file);
+  }
+
+  return createTextLineAsyncGeneratorFromStreamSource(file.stream());
+}
+
+export function createReadlineInterfaceFromResponse(resp: Response): AsyncGenerator<string> {
   if (!resp.body) {
     throw new Error('Failed to fetch remote text');
   }
@@ -58,24 +62,7 @@ export async function *createReadlineInterfaceFromResponse(resp: Response): Asyn
     throw new Error('Body has already been consumed.');
   }
 
-  let buf = '';
-
-  for await (const chunk of resp.body) {
-    const chunkStr = decoder.decode(chunk).replaceAll('\r\n', '\n');
-    for (let i = 0, len = chunkStr.length; i < len; i++) {
-      const char = chunkStr[i];
-      if (char === '\n') {
-        yield buf;
-        buf = '';
-      } else {
-        buf += char;
-      }
-    }
-  }
-
-  if (buf) {
-    yield buf;
-  }
+  return createTextLineAsyncGeneratorFromStreamSource(resp.body);
 }
 
 export function fetchRemoteTextByLine(url: string | URL) {