Chore: minor changes

Build/lib/fetch-assets.ts

@@ -11,23 +11,7 @@ export class CustomAbortError extends Error {
   public readonly digest = 'AbortError';
 }
 
-export class Custom304NotModifiedError extends Error {
-  public readonly name = 'Custom304NotModifiedError';
-  public readonly digest = 'Custom304NotModifiedError';
-
-  constructor(public readonly url: string, public readonly data: string) {
-    super('304 Not Modified');
-  }
-}
-
-export class CustomNoETagFallbackError extends Error {
-  public readonly name = 'CustomNoETagFallbackError';
-  public readonly digest = 'CustomNoETagFallbackError';
-
-  constructor(public readonly data: string) {
-    super('No ETag Fallback');
-  }
-}
+const reusedCustomAbortError = new CustomAbortError();
 
 export async function fetchAssets(url: string, fallbackUrls: null | undefined | string[] | readonly string[], processLine = false) {
   const controller = new AbortController();
@@ -39,12 +23,12 @@ export async function fetchAssets(url: string, fallbackUrls: null | undefined |
         await waitWithAbort(50 + (index + 1) * 100, controller.signal);
       } catch {
         console.log(picocolors.gray('[fetch cancelled early]'), picocolors.gray(url));
-        throw new CustomAbortError();
+        throw reusedCustomAbortError;
       }
     }
     if (controller.signal.aborted) {
       console.log(picocolors.gray('[fetch cancelled]'), picocolors.gray(url));
-      throw new CustomAbortError();
+      throw reusedCustomAbortError;
     }
     const res = await $$fetch(url, { signal: controller.signal, ...defaultRequestInit });
 

Build/lib/fetch-retry.ts

@@ -34,11 +34,7 @@ setGlobalDispatcher(agent.compose(
     // TODO: this part of code is only for allow more errors to be retried by default
     // This should be removed once https://github.com/nodejs/undici/issues/3728 is implemented
     retry(err, { state, opts }, cb) {
-      const statusCode = 'statusCode' in err && typeof err.statusCode === 'number' ? err.statusCode : null;
       const errorCode = 'code' in err ? (err as NodeJS.ErrnoException).code : undefined;
-      const headers = ('headers' in err && typeof err.headers === 'object') ? err.headers : undefined;
-
-      const { counter } = state;
 
       // Any code that is not a Undici's originated and allowed to retry
       if (
@@ -49,29 +45,7 @@ setGlobalDispatcher(agent.compose(
         return cb(err);
       }
 
-      // if (errorCode === 'UND_ERR_REQ_RETRY') {
-      //   return cb(err);
-      // }
-
-      const { method, retryOptions = {} } = opts;
-
-      const {
-        maxRetries = 5,
-        minTimeout = 500,
-        maxTimeout = 10 * 1000,
-        timeoutFactor = 2,
-        methods = ['GET', 'HEAD', 'OPTIONS', 'PUT', 'DELETE', 'TRACE']
-      } = retryOptions;
-
-      // If we reached the max number of retries
-      if (counter > maxRetries) {
-        return cb(err);
-      }
-
-      // If a set of method are provided and the current method is not in the list
-      if (Array.isArray(methods) && !methods.includes(method)) {
-        return cb(err);
-      }
+      const statusCode = 'statusCode' in err && typeof err.statusCode === 'number' ? err.statusCode : null;
 
       // bail out if the status code matches one of the following
       if (
@@ -86,6 +60,30 @@ setGlobalDispatcher(agent.compose(
         return cb(err);
       }
 
+      // if (errorCode === 'UND_ERR_REQ_RETRY') {
+      //   return cb(err);
+      // }
+
+      const {
+        maxRetries = 5,
+        minTimeout = 500,
+        maxTimeout = 10 * 1000,
+        timeoutFactor = 2,
+        methods = ['GET', 'HEAD', 'OPTIONS', 'PUT', 'DELETE', 'TRACE']
+      } = opts.retryOptions || {};
+
+      // If we reached the max number of retries
+      if (state.counter > maxRetries) {
+        return cb(err);
+      }
+
+      // If a set of method are provided and the current method is not in the list
+      if (Array.isArray(methods) && !methods.includes(opts.method)) {
+        return cb(err);
+      }
+
+      const headers = ('headers' in err && typeof err.headers === 'object') ? err.headers : undefined;
+
       const retryAfterHeader = (headers as Record<string, string> | null | undefined)?.['retry-after'];
       let retryAfter = -1;
       if (retryAfterHeader) {
@@ -97,7 +95,7 @@ setGlobalDispatcher(agent.compose(
 
       const retryTimeout = retryAfter > 0
         ? Math.min(retryAfter, maxTimeout)
-        : Math.min(minTimeout * (timeoutFactor ** (counter - 1)), maxTimeout);
+        : Math.min(minTimeout * (timeoutFactor ** (state.counter - 1)), maxTimeout);
 
       console.log('[fetch retry]', 'schedule retry', { statusCode, retryTimeout, errorCode, url: opts.origin });
       // eslint-disable-next-line sukka/prefer-timer-id -- won't leak

Build/lib/fetch-text-by-line.bench.ts

@@ -1,4 +1,4 @@
-import { readFileByLine, readFileByLineLegacy, readFileByLineNew } from './fetch-text-by-line';
+import { readFileByLine, readFileByLineNew } from './fetch-text-by-line';
 import path from 'node:path';
 import fsp from 'node:fs/promises';
 import { OUTPUT_SURGE_DIR } from '../constants/dir';
@@ -10,7 +10,6 @@ const file = path.join(OUTPUT_SURGE_DIR, 'domainset/reject_extra.conf');
 
   group(() => {
     bench('readFileByLine', () => Array.fromAsync(readFileByLine(file)));
-    bench('readFileByLineLegacy', () => Array.fromAsync(readFileByLineLegacy(file)));
     bench('readFileByLineNew', async () => Array.fromAsync(await readFileByLineNew(file)));
     bench('fsp.readFile', () => fsp.readFile(file, 'utf-8').then((content) => content.split('\n')));
   });

Build/lib/fetch-text-by-line.ts

@@ -1,5 +1,4 @@
 import fs from 'node:fs';
-import { Readable } from 'node:stream';
 import fsp from 'node:fs/promises';
 import type { FileHandle } from 'node:fs/promises';
 import readline from 'node:readline';
@@ -11,19 +10,7 @@ import { processLine, ProcessLineStream } from './process-line';
 import { $$fetch } from './fetch-retry';
 import type { UndiciResponseData } from './fetch-retry';
 import type { Response as UnidiciWebResponse } from 'undici';
-
-function getReadableStream(file: string | FileHandle): ReadableStream {
-  if (typeof file === 'string') {
-    // return fs.openAsBlob(file).then(blob => blob.stream())
-    return Readable.toWeb(fs.createReadStream(file/* , { encoding: 'utf-8' } */));
-  }
-  return file.readableWebStream();
-}
-
-// TODO: use FileHandle.readLine()
-export const readFileByLineLegacy: ((file: string /* | FileHandle */) => AsyncIterable<string>) = (file: string | FileHandle) => getReadableStream(file)
-  .pipeThrough(new TextDecoderStream())
-  .pipeThrough(new TextLineStream());
+import { invariant } from 'foxts/guard';
 
 export function readFileByLine(file: string): AsyncIterable<string> {
   return readline.createInterface({
@@ -37,26 +24,17 @@ export async function readFileByLineNew(file: string): Promise<AsyncIterable<str
   return fsp.open(file, 'r').then(fdReadLines);
 }
 
-function ensureResponseBody<T extends UndiciResponseData | UnidiciWebResponse>(resp: T): NonNullable<T['body']> {
-  if (resp.body == null) {
-    throw new Error('Failed to fetch remote text');
-  }
+export const createReadlineInterfaceFromResponse: ((resp: UndiciResponseData | UnidiciWebResponse, processLine?: boolean) => ReadableStream<string>) = (resp, processLine = false) => {
+  invariant(resp.body, 'Failed to fetch remote text');
   if ('bodyUsed' in resp && resp.bodyUsed) {
     throw new Error('Body has already been consumed.');
   }
-  return resp.body;
-}
-
-export const createReadlineInterfaceFromResponse: ((resp: UndiciResponseData | UnidiciWebResponse, processLine?: boolean) => ReadableStream<string>) = (resp, processLine = false) => {
-  const stream = ensureResponseBody(resp);
-
-  const webStream: ReadableStream<Uint8Array> = 'getReader' in stream
-    ? stream
-    : (
-      'text' in stream
-        ? stream.body as any
-        : Readable.toWeb(new Readable().wrap(stream))
-    );
+  let webStream: ReadableStream<Uint8Array>;
+  if ('pipeThrough' in resp.body) {
+    webStream = resp.body;
+  } else {
+    throw new TypeError('Invalid response body!');
+  }
 
   const resultStream = webStream
     .pipeThrough(new TextDecoderStream())

Build/lib/parse-dnsmasq.ts

@@ -1,16 +1,8 @@
 import { createReadlineInterfaceFromResponse } from './fetch-text-by-line';
 
-// https://github.com/remusao/tldts/issues/2121
-// In short, single label domain suffix is ignored due to the size optimization, so no isIcann
-// import tldts from 'tldts-experimental';
-import tldts from 'tldts';
 import type { UndiciResponseData } from './fetch-retry';
 import type { Response } from 'undici';
-
-function isDomainLoose(domain: string): boolean {
-  const r = tldts.parse(domain);
-  return !!(!r.isIp && (r.isIcann || r.isPrivate));
-}
+import { fastNormalizeDomainIgnoreWww } from './normalize-domain';
 
 export function extractDomainsFromFelixDnsmasq(line: string): string | null {
   if (line.startsWith('server=/') && line.endsWith('/114.114.114.114')) {
@@ -24,7 +16,7 @@ export async function parseFelixDnsmasqFromResp(resp: UndiciResponseData | Respo
 
   for await (const line of createReadlineInterfaceFromResponse(resp, true)) {
     const domain = extractDomainsFromFelixDnsmasq(line);
-    if (domain && isDomainLoose(domain)) {
+    if (domain && fastNormalizeDomainIgnoreWww(domain)) {
       results.push(domain);
     }
   }