Chore: minor changes

Build/build-common.ts

@@ -16,8 +16,6 @@ const MAGIC_COMMAND_TITLE = '# $ meta_title ';
 const MAGIC_COMMAND_DESCRIPTION = '# $ meta_description ';
 const MAGIC_COMMAND_SGMODULE_MITM_HOSTNAMES = '# $ sgmodule_mitm_hostnames ';
 
-const domainsetSrcFolder = 'domainset' + path.sep;
-
 const clawSourceDirPromise = new Fdir()
   .withRelativePaths()
   .filter((filepath, isDirectory) => {
@@ -39,15 +37,11 @@ export const buildCommon = task(require.main === module, __filename)(async (span
     const relativePath = paths[i];
     const fullPath = SOURCE_DIR + path.sep + relativePath;
 
-    if (relativePath.startsWith(domainsetSrcFolder)) {
-      promises.push(transformDomainset(span, fullPath));
-      continue;
-    }
     // if (
     //   relativePath.startsWith('ip/')
     //   || relativePath.startsWith('non_ip/')
     // ) {
-    promises.push(transformRuleset(span, fullPath, relativePath));
+    promises.push(transform(span, fullPath, relativePath));
     // continue;
     // }
 
@@ -102,71 +96,44 @@ function processFile(span: Span, sourcePath: string) {
   });
 }
 
-function transformDomainset(parentSpan: Span, sourcePath: string) {
+async function transform(parentSpan: Span, sourcePath: string, relativePath: string) {
   const extname = path.extname(sourcePath);
-  const basename = path.basename(sourcePath, extname);
+  const id = path.basename(sourcePath, extname);
+
   return parentSpan
-    .traceChildAsync(
-      `transform domainset: ${basename}`,
-      async (span) => {
-        const res = await processFile(span, sourcePath);
-        if (res === $skip) return;
+    .traceChild(`transform ruleset: ${id}`)
+    .traceAsyncFn(async (span) => {
+      const type = relativePath.split(path.sep)[0];
 
-        const id = basename;
-        const [title, incomingDescriptions, lines] = res;
+      if (type !== 'ip' && type !== 'non_ip' && type !== 'domainset') {
+        throw new TypeError(`Invalid type: ${type}`);
+      }
 
-        let finalDescriptions: string[];
-        if (incomingDescriptions.length) {
-          finalDescriptions = SHARED_DESCRIPTION.slice();
-          finalDescriptions.push('');
-          appendArrayInPlace(finalDescriptions, incomingDescriptions);
-        } else {
-          finalDescriptions = SHARED_DESCRIPTION;
-        }
+      const res = await processFile(span, sourcePath);
+      if (res === $skip) return;
 
+      const [title, descriptions, lines, sgmodulePathname] = res;
+
+      let finalDescriptions: string[];
+      if (descriptions.length) {
+        finalDescriptions = SHARED_DESCRIPTION.slice();
+        finalDescriptions.push('');
+        appendArrayInPlace(finalDescriptions, descriptions);
+      } else {
+        finalDescriptions = SHARED_DESCRIPTION;
+      }
+
+      if (type === 'domainset') {
         return new DomainsetOutput(span, id)
           .withTitle(title)
           .withDescription(finalDescriptions)
           .addFromDomainset(lines)
           .write();
       }
-    );
-}
-
-/**
- * Output Surge RULE-SET and Clash classical text format
- */
-async function transformRuleset(parentSpan: Span, sourcePath: string, relativePath: string) {
-  const extname = path.extname(sourcePath);
-  const basename = path.basename(sourcePath, extname);
-
-  return parentSpan
-    .traceChild(`transform ruleset: ${basename}`)
-    .traceAsyncFn(async (span) => {
-      const res = await processFile(span, sourcePath);
-      if (res === $skip) return;
-
-      const id = basename;
-      const type = relativePath.split(path.sep)[0];
-
-      if (type !== 'ip' && type !== 'non_ip') {
-        throw new TypeError(`Invalid type: ${type}`);
-      }
-
-      const [title, descriptions, lines, sgmodulePathname] = res;
-
-      let description: string[];
-      if (descriptions.length) {
-        description = SHARED_DESCRIPTION.slice();
-        description.push('');
-        appendArrayInPlace(description, descriptions);
-      } else {
-        description = SHARED_DESCRIPTION;
-      }
 
       return new RulesetOutput(span, id, type)
         .withTitle(title)
-        .withDescription(description)
+        .withDescription(finalDescriptions)
         .withMitmSgmodulePath(sgmodulePathname)
         .addFromRuleset(lines)
         .write();

Build/lib/fetch-assets.ts

@@ -11,23 +11,7 @@ export class CustomAbortError extends Error {
   public readonly digest = 'AbortError';
 }
 
-export class Custom304NotModifiedError extends Error {
-  public readonly name = 'Custom304NotModifiedError';
-  public readonly digest = 'Custom304NotModifiedError';
-
-  constructor(public readonly url: string, public readonly data: string) {
-    super('304 Not Modified');
-  }
-}
-
-export class CustomNoETagFallbackError extends Error {
-  public readonly name = 'CustomNoETagFallbackError';
-  public readonly digest = 'CustomNoETagFallbackError';
-
-  constructor(public readonly data: string) {
-    super('No ETag Fallback');
-  }
-}
+const reusedCustomAbortError = new CustomAbortError();
 
 export async function fetchAssets(url: string, fallbackUrls: null | undefined | string[] | readonly string[], processLine = false) {
   const controller = new AbortController();
@@ -39,12 +23,12 @@ export async function fetchAssets(url: string, fallbackUrls: null | undefined |
         await waitWithAbort(50 + (index + 1) * 100, controller.signal);
       } catch {
         console.log(picocolors.gray('[fetch cancelled early]'), picocolors.gray(url));
-        throw new CustomAbortError();
+        throw reusedCustomAbortError;
       }
     }
     if (controller.signal.aborted) {
       console.log(picocolors.gray('[fetch cancelled]'), picocolors.gray(url));
-      throw new CustomAbortError();
+      throw reusedCustomAbortError;
     }
     const res = await $$fetch(url, { signal: controller.signal, ...defaultRequestInit });
 

Build/lib/fetch-retry.ts

@@ -34,11 +34,7 @@ setGlobalDispatcher(agent.compose(
     // TODO: this part of code is only for allow more errors to be retried by default
     // This should be removed once https://github.com/nodejs/undici/issues/3728 is implemented
     retry(err, { state, opts }, cb) {
-      const statusCode = 'statusCode' in err && typeof err.statusCode === 'number' ? err.statusCode : null;
       const errorCode = 'code' in err ? (err as NodeJS.ErrnoException).code : undefined;
-      const headers = ('headers' in err && typeof err.headers === 'object') ? err.headers : undefined;
-
-      const { counter } = state;
 
       // Any code that is not a Undici's originated and allowed to retry
       if (
@@ -49,29 +45,7 @@ setGlobalDispatcher(agent.compose(
         return cb(err);
       }
 
-      // if (errorCode === 'UND_ERR_REQ_RETRY') {
-      //   return cb(err);
-      // }
-
-      const { method, retryOptions = {} } = opts;
-
-      const {
-        maxRetries = 5,
-        minTimeout = 500,
-        maxTimeout = 10 * 1000,
-        timeoutFactor = 2,
-        methods = ['GET', 'HEAD', 'OPTIONS', 'PUT', 'DELETE', 'TRACE']
-      } = retryOptions;
-
-      // If we reached the max number of retries
-      if (counter > maxRetries) {
-        return cb(err);
-      }
-
-      // If a set of method are provided and the current method is not in the list
-      if (Array.isArray(methods) && !methods.includes(method)) {
-        return cb(err);
-      }
+      const statusCode = 'statusCode' in err && typeof err.statusCode === 'number' ? err.statusCode : null;
 
       // bail out if the status code matches one of the following
       if (
@@ -86,6 +60,30 @@ setGlobalDispatcher(agent.compose(
         return cb(err);
       }
 
+      // if (errorCode === 'UND_ERR_REQ_RETRY') {
+      //   return cb(err);
+      // }
+
+      const {
+        maxRetries = 5,
+        minTimeout = 500,
+        maxTimeout = 10 * 1000,
+        timeoutFactor = 2,
+        methods = ['GET', 'HEAD', 'OPTIONS', 'PUT', 'DELETE', 'TRACE']
+      } = opts.retryOptions || {};
+
+      // If we reached the max number of retries
+      if (state.counter > maxRetries) {
+        return cb(err);
+      }
+
+      // If a set of method are provided and the current method is not in the list
+      if (Array.isArray(methods) && !methods.includes(opts.method)) {
+        return cb(err);
+      }
+
+      const headers = ('headers' in err && typeof err.headers === 'object') ? err.headers : undefined;
+
       const retryAfterHeader = (headers as Record<string, string> | null | undefined)?.['retry-after'];
       let retryAfter = -1;
       if (retryAfterHeader) {
@@ -97,7 +95,7 @@ setGlobalDispatcher(agent.compose(
 
       const retryTimeout = retryAfter > 0
         ? Math.min(retryAfter, maxTimeout)
-        : Math.min(minTimeout * (timeoutFactor ** (counter - 1)), maxTimeout);
+        : Math.min(minTimeout * (timeoutFactor ** (state.counter - 1)), maxTimeout);
 
       console.log('[fetch retry]', 'schedule retry', { statusCode, retryTimeout, errorCode, url: opts.origin });
       // eslint-disable-next-line sukka/prefer-timer-id -- won't leak

Build/lib/fetch-text-by-line.bench.ts

@@ -1,4 +1,4 @@
-import { readFileByLine, readFileByLineLegacy, readFileByLineNew } from './fetch-text-by-line';
+import { readFileByLine, readFileByLineNew } from './fetch-text-by-line';
 import path from 'node:path';
 import fsp from 'node:fs/promises';
 import { OUTPUT_SURGE_DIR } from '../constants/dir';
@@ -10,7 +10,6 @@ const file = path.join(OUTPUT_SURGE_DIR, 'domainset/reject_extra.conf');
 
   group(() => {
     bench('readFileByLine', () => Array.fromAsync(readFileByLine(file)));
-    bench('readFileByLineLegacy', () => Array.fromAsync(readFileByLineLegacy(file)));
     bench('readFileByLineNew', async () => Array.fromAsync(await readFileByLineNew(file)));
     bench('fsp.readFile', () => fsp.readFile(file, 'utf-8').then((content) => content.split('\n')));
   });

Build/lib/fetch-text-by-line.ts

@@ -1,5 +1,4 @@
 import fs from 'node:fs';
-import { Readable } from 'node:stream';
 import fsp from 'node:fs/promises';
 import type { FileHandle } from 'node:fs/promises';
 import readline from 'node:readline';
@@ -11,19 +10,7 @@ import { processLine, ProcessLineStream } from './process-line';
 import { $$fetch } from './fetch-retry';
 import type { UndiciResponseData } from './fetch-retry';
 import type { Response as UnidiciWebResponse } from 'undici';
-
-function getReadableStream(file: string | FileHandle): ReadableStream {
-  if (typeof file === 'string') {
-    // return fs.openAsBlob(file).then(blob => blob.stream())
-    return Readable.toWeb(fs.createReadStream(file/* , { encoding: 'utf-8' } */));
-  }
-  return file.readableWebStream();
-}
-
-// TODO: use FileHandle.readLine()
-export const readFileByLineLegacy: ((file: string /* | FileHandle */) => AsyncIterable<string>) = (file: string | FileHandle) => getReadableStream(file)
-  .pipeThrough(new TextDecoderStream())
-  .pipeThrough(new TextLineStream());
+import { invariant } from 'foxts/guard';
 
 export function readFileByLine(file: string): AsyncIterable<string> {
   return readline.createInterface({
@@ -37,26 +24,17 @@ export async function readFileByLineNew(file: string): Promise<AsyncIterable<str
   return fsp.open(file, 'r').then(fdReadLines);
 }
 
-function ensureResponseBody<T extends UndiciResponseData | UnidiciWebResponse>(resp: T): NonNullable<T['body']> {
-  if (resp.body == null) {
-    throw new Error('Failed to fetch remote text');
-  }
+export const createReadlineInterfaceFromResponse: ((resp: UndiciResponseData | UnidiciWebResponse, processLine?: boolean) => ReadableStream<string>) = (resp, processLine = false) => {
+  invariant(resp.body, 'Failed to fetch remote text');
   if ('bodyUsed' in resp && resp.bodyUsed) {
     throw new Error('Body has already been consumed.');
   }
-  return resp.body;
-}
-
-export const createReadlineInterfaceFromResponse: ((resp: UndiciResponseData | UnidiciWebResponse, processLine?: boolean) => ReadableStream<string>) = (resp, processLine = false) => {
-  const stream = ensureResponseBody(resp);
-
-  const webStream: ReadableStream<Uint8Array> = 'getReader' in stream
-    ? stream
-    : (
-      'text' in stream
-        ? stream.body as any
-        : Readable.toWeb(new Readable().wrap(stream))
-    );
+  let webStream: ReadableStream<Uint8Array>;
+  if ('pipeThrough' in resp.body) {
+    webStream = resp.body;
+  } else {
+    throw new TypeError('Invalid response body!');
+  }
 
   const resultStream = webStream
     .pipeThrough(new TextDecoderStream())

Build/lib/parse-dnsmasq.ts

@@ -1,16 +1,8 @@
 import { createReadlineInterfaceFromResponse } from './fetch-text-by-line';
 
-// https://github.com/remusao/tldts/issues/2121
-// In short, single label domain suffix is ignored due to the size optimization, so no isIcann
-// import tldts from 'tldts-experimental';
-import tldts from 'tldts';
 import type { UndiciResponseData } from './fetch-retry';
 import type { Response } from 'undici';
-
-function isDomainLoose(domain: string): boolean {
-  const r = tldts.parse(domain);
-  return !!(!r.isIp && (r.isIcann || r.isPrivate));
-}
+import { fastNormalizeDomainIgnoreWww } from './normalize-domain';
 
 export function extractDomainsFromFelixDnsmasq(line: string): string | null {
   if (line.startsWith('server=/') && line.endsWith('/114.114.114.114')) {
@@ -24,7 +16,7 @@ export async function parseFelixDnsmasqFromResp(resp: UndiciResponseData | Respo
 
   for await (const line of createReadlineInterfaceFromResponse(resp, true)) {
     const domain = extractDomainsFromFelixDnsmasq(line);
-    if (domain && isDomainLoose(domain)) {
+    if (domain && fastNormalizeDomainIgnoreWww(domain)) {
       results.push(domain);
     }
   }