Chore: update build infra & performance improvements

Build/build-chn-cidr.ts

@@ -10,6 +10,9 @@ const EXCLUDE_CIDRS = [
   '223.120.0.0/15'
 ];
 
+// preload the module
+import('cidr-tools-wasm');
+
 export const buildChnCidr = task(import.meta.path, async () => {
   const [{ exclude }, cidr] = await Promise.all([
     import('cidr-tools-wasm'),

Build/build-internal-reverse-chn-cidr.ts

@@ -22,6 +22,9 @@ const RESERVED_IPV4_CIDR = [
   '240.0.0.0/4'
 ];
 
+// preload the module
+import('cidr-tools-wasm');
+
 export const buildInternalReverseChnCIDR = task(import.meta.path, async () => {
   const [{ exclude }, cidr] = await Promise.all([
     import('cidr-tools-wasm'),

Build/download-previous-build.ts

@@ -67,7 +67,6 @@ export const downloadPreviousBuild = task(import.meta.path, async () => {
           async onentry(entry) {
             if (entry.type !== 'File') {
               // not a file, throw away
-              console.log(entry.type, entry.path)
               entry.resume();
               return;
             }

Build/lib/create-file.ts

@@ -6,10 +6,12 @@ export async function compareAndWriteFile(linesA: string[], filePath: string) {
   let isEqual = true;
   const file = Bun.file(filePath);
 
+  const linesALen = linesA.length;
+
   if (!(await file.exists())) {
     console.log(`${filePath} does not exists, writing...`);
     isEqual = false;
-  } else if (linesA.length === 0) {
+  } else if (linesALen === 0) {
     console.log(`Nothing to write to ${filePath}...`);
     isEqual = false;
   } else {
@@ -35,23 +37,24 @@ export async function compareAndWriteFile(linesA: string[], filePath: string) {
       }
     }
 
-    if (index !== linesA.length) {
+    if (index !== linesALen) {
+      // The file becomes larger
       isEqual = false;
     }
   }
 
-  if (!isEqual) {
-    const writer = file.writer();
-
-    for (let i = 0, len = linesA.length; i < len; i++) {
-      writer.write(`${linesA[i]}\n`);
-    }
-
-    await writer.end();
+  if (isEqual) {
+    console.log(`Same Content, bail out writing: ${filePath}`);
     return;
   }
 
-  console.log(`Same Content, bail out writing: ${filePath}`);
+  const writer = file.writer();
+
+  for (let i = 0; i < linesALen; i++) {
+    writer.write(`${linesA[i]}\n`);
+  }
+
+  return writer.end();
 }
 
 export const withBannerArray = (title: string, description: string[], date: Date, content: string[]) => {

Build/lib/fetch-retry.ts

@@ -1,5 +1,112 @@
-// @ts-expect-error -- missing types
-import createFetchRetry from '@vercel/fetch-retry';
+import retry from 'async-retry';
+
+// retry settings
+const MIN_TIMEOUT = 10;
+const MAX_RETRIES = 5;
+const MAX_RETRY_AFTER = 20;
+const FACTOR = 6;
+
+function isClientError(err: any): err is NodeJS.ErrnoException {
+  if (!err) return false;
+  return (
+    err.code === 'ERR_UNESCAPED_CHARACTERS' ||
+    err.message === 'Request path contains unescaped characters'
+  );
+}
+
+interface FetchRetryOpt {
+  minTimeout?: number,
+  retries?: number,
+  factor?: number,
+  maxRetryAfter?: number,
+  retry?: number,
+  onRetry?: (err: Error) => void
+}
+
+function createFetchRetry($fetch: typeof fetch): typeof fetch {
+  const fetchRetry = async (url: string | URL, opts: RequestInit & { retry?: FetchRetryOpt } = {}) => {
+    const retryOpts = Object.assign(
+      {
+        // timeouts will be [10, 60, 360, 2160, 12960]
+        // (before randomization is added)
+        minTimeout: MIN_TIMEOUT,
+        retries: MAX_RETRIES,
+        factor: FACTOR,
+        maxRetryAfter: MAX_RETRY_AFTER,
+      },
+      opts.retry
+    );
+
+    try {
+      return await retry(async (bail) => {
+        try {
+          // this will be retried
+          const res = await $fetch(url, opts);
+
+          if ((res.status >= 500 && res.status < 600) || res.status === 429) {
+            // NOTE: doesn't support http-date format
+            const retryAfterHeader = res.headers.get('retry-after');
+            if (retryAfterHeader) {
+              const retryAfter = parseInt(retryAfterHeader, 10);
+              if (retryAfter) {
+                if (retryAfter > retryOpts.maxRetryAfter) {
+                  return res;
+                } else {
+                  await new Promise((r) => setTimeout(r, retryAfter * 1e3));
+                }
+              }
+            }
+            throw new ResponseError(res);
+          } else {
+            return res;
+          }
+        } catch (err: unknown) {
+          if (err instanceof Error) {
+            if (err.name === 'AbortError') {
+              return bail(err);
+            }
+          }
+          if (isClientError(err)) {
+            return bail(err);
+          }
+          throw err;
+        }
+      }, retryOpts);
+    } catch (err) {
+      if (err instanceof ResponseError) {
+        return err.res;
+      }
+      throw err;
+    }
+  }
+
+  for (const k of Object.keys($fetch)) {
+    const key = k as keyof typeof $fetch;
+    fetchRetry[key] = $fetch[key];
+  }
+
+  return fetchRetry as typeof fetch;
+}
+
+export class ResponseError extends Error {
+  readonly res: Response;
+  readonly code: number;
+  readonly statusCode: number;
+  readonly url: string;
+
+  constructor(res: Response) {
+    super(res.statusText);
+
+    if (Error.captureStackTrace) {
+      Error.captureStackTrace(this, ResponseError);
+    }
+
+    this.name = this.constructor.name;
+    this.res = res;
+    this.code = this.statusCode = res.status;
+    this.url = res.url;
+  }
+}
 
 export const defaultRequestInit: RequestInit = {
   headers: {
@@ -7,4 +114,4 @@ export const defaultRequestInit: RequestInit = {
   }
 }
 
-export const fetchWithRetry: typeof fetch = createFetchRetry(fetch);
+export const fetchWithRetry = createFetchRetry(fetch);

Build/lib/text-line-transform-stream.ts

@@ -19,14 +19,15 @@ interface TextLineStreamOptions {
  * ```
  */
 export class TextLineStream extends TransformStream<string, string> {
-  private __buf = '';
-
+  // private __buf = '';
   constructor(options?: TextLineStreamOptions) {
     const allowCR = options?.allowCR ?? false;
 
+    let __buf = '';
+
     super({
-      transform: (chunk, controller) => {
-        chunk = this.__buf + chunk;
+      transform(chunk, controller) {
+        chunk = __buf + chunk;
 
         for (; ;) {
           const lfIndex = chunk.indexOf('\n');
@@ -57,14 +58,14 @@ export class TextLineStream extends TransformStream<string, string> {
           break;
         }
 
-        this.__buf = chunk;
+        __buf = chunk;
       },
-      flush: (controller) => {
-        if (this.__buf.length > 0) {
-          if (allowCR && this.__buf[this.__buf.length - 1] === '\r') {
-            controller.enqueue(this.__buf.slice(0, -1));
+      flush(controller) {
+        if (__buf.length > 0) {
+          if (allowCR && __buf[__buf.length - 1] === '\r') {
+            controller.enqueue(__buf.slice(0, -1));
           } else {
-            controller.enqueue(this.__buf);
+            controller.enqueue(__buf);
           };
         }
       },

package.json

@@ -16,11 +16,11 @@
   "dependencies": {
     "@cliqz/adblocker": "^1.26.12",
     "@sukka/listdir": "^0.2.0",
-    "@vercel/fetch-retry": "^5.1.3",
+    "async-retry": "^1.3.3",
     "async-sema": "^3.1.1",
     "ci-info": "^4.0.0",
-    "cidr-tools-wasm": "^0.0.13",
-    "eslint": "^8.53.0",
+    "cidr-tools-wasm": "^0.0.14",
+    "eslint": "^8.54.0",
     "gorhill-publicsuffixlist": "github:gorhill/publicsuffixlist.js",
     "mnemonist": "^0.39.5",
     "path-scurry": "^1.10.1",
@@ -28,11 +28,12 @@
     "punycode": "^2.3.1",
     "table": "^6.8.1",
     "tar": "^6.2.0",
-    "tldts": "^6.0.19"
+    "tldts": "^6.0.22"
   },
   "devDependencies": {
     "@eslint-sukka/node": "4.1.9",
     "@eslint-sukka/ts": "4.1.9",
+    "@types/async-retry": "^1.4.8",
     "@types/mocha": "10.0.2",
     "@types/tar": "^6.1.9",
     "bun-types": "^1.0.11",