Chore: minor changes

Build/build-common.ts

@@ -16,8 +16,6 @@ const MAGIC_COMMAND_TITLE = '# $ meta_title ';
 const MAGIC_COMMAND_DESCRIPTION = '# $ meta_description ';
 const MAGIC_COMMAND_SGMODULE_MITM_HOSTNAMES = '# $ sgmodule_mitm_hostnames ';
 
-const domainsetSrcFolder = 'domainset' + path.sep;
-
 const clawSourceDirPromise = new Fdir()
   .withRelativePaths()
   .filter((filepath, isDirectory) => {
@@ -39,15 +37,11 @@ export const buildCommon = task(require.main === module, __filename)(async (span
     const relativePath = paths[i];
     const fullPath = SOURCE_DIR + path.sep + relativePath;
 
-    if (relativePath.startsWith(domainsetSrcFolder)) {
-      promises.push(transformDomainset(span, fullPath));
-      continue;
-    }
     // if (
     //   relativePath.startsWith('ip/')
     //   || relativePath.startsWith('non_ip/')
     // ) {
-    promises.push(transformRuleset(span, fullPath, relativePath));
+    promises.push(transform(span, fullPath, relativePath));
     // continue;
     // }
 
@@ -102,71 +96,44 @@ function processFile(span: Span, sourcePath: string) {
   });
 }
 
-function transformDomainset(parentSpan: Span, sourcePath: string) {
+async function transform(parentSpan: Span, sourcePath: string, relativePath: string) {
   const extname = path.extname(sourcePath);
-  const basename = path.basename(sourcePath, extname);
+  const id = path.basename(sourcePath, extname);
+
   return parentSpan
-    .traceChildAsync(
+    .traceChild(`transform ruleset: ${id}`)
-      `transform domainset: ${basename}`,
+    .traceAsyncFn(async (span) => {
-      async (span) => {
+      const type = relativePath.split(path.sep)[0];
-        const res = await processFile(span, sourcePath);
-        if (res === $skip) return;
 
-        const id = basename;
+      if (type !== 'ip' && type !== 'non_ip' && type !== 'domainset') {
-        const [title, incomingDescriptions, lines] = res;
+        throw new TypeError(`Invalid type: ${type}`);
+      }
 
-        let finalDescriptions: string[];
+      const res = await processFile(span, sourcePath);
-        if (incomingDescriptions.length) {
+      if (res === $skip) return;
-          finalDescriptions = SHARED_DESCRIPTION.slice();
-          finalDescriptions.push('');
-          appendArrayInPlace(finalDescriptions, incomingDescriptions);
-        } else {
-          finalDescriptions = SHARED_DESCRIPTION;
-        }
 
+      const [title, descriptions, lines, sgmodulePathname] = res;
+
+      let finalDescriptions: string[];
+      if (descriptions.length) {
+        finalDescriptions = SHARED_DESCRIPTION.slice();
+        finalDescriptions.push('');
+        appendArrayInPlace(finalDescriptions, descriptions);
+      } else {
+        finalDescriptions = SHARED_DESCRIPTION;
+      }
+
+      if (type === 'domainset') {
         return new DomainsetOutput(span, id)
           .withTitle(title)
           .withDescription(finalDescriptions)
           .addFromDomainset(lines)
           .write();
       }
-    );
-}
-
-/**
- * Output Surge RULE-SET and Clash classical text format
- */
-async function transformRuleset(parentSpan: Span, sourcePath: string, relativePath: string) {
-  const extname = path.extname(sourcePath);
-  const basename = path.basename(sourcePath, extname);
-
-  return parentSpan
-    .traceChild(`transform ruleset: ${basename}`)
-    .traceAsyncFn(async (span) => {
-      const res = await processFile(span, sourcePath);
-      if (res === $skip) return;
-
-      const id = basename;
-      const type = relativePath.split(path.sep)[0];
-
-      if (type !== 'ip' && type !== 'non_ip') {
-        throw new TypeError(`Invalid type: ${type}`);
-      }
-
-      const [title, descriptions, lines, sgmodulePathname] = res;
-
-      let description: string[];
-      if (descriptions.length) {
-        description = SHARED_DESCRIPTION.slice();
-        description.push('');
-        appendArrayInPlace(description, descriptions);
-      } else {
-        description = SHARED_DESCRIPTION;
-      }
 
       return new RulesetOutput(span, id, type)
         .withTitle(title)
-        .withDescription(description)
+        .withDescription(finalDescriptions)
         .withMitmSgmodulePath(sgmodulePathname)
         .addFromRuleset(lines)
         .write();

Build/lib/fetch-assets.ts

@@ -11,23 +11,7 @@ export class CustomAbortError extends Error {
   public readonly digest = 'AbortError';
 }
 
-export class Custom304NotModifiedError extends Error {
+const reusedCustomAbortError = new CustomAbortError();
-  public readonly name = 'Custom304NotModifiedError';
-  public readonly digest = 'Custom304NotModifiedError';
-
-  constructor(public readonly url: string, public readonly data: string) {
-    super('304 Not Modified');
-  }
-}
-
-export class CustomNoETagFallbackError extends Error {
-  public readonly name = 'CustomNoETagFallbackError';
-  public readonly digest = 'CustomNoETagFallbackError';
-
-  constructor(public readonly data: string) {
-    super('No ETag Fallback');
-  }
-}
 
 export async function fetchAssets(url: string, fallbackUrls: null | undefined | string[] | readonly string[], processLine = false) {
   const controller = new AbortController();
@@ -39,12 +23,12 @@ export async function fetchAssets(url: string, fallbackUrls: null | undefined |
         await waitWithAbort(50 + (index + 1) * 100, controller.signal);
       } catch {
         console.log(picocolors.gray('[fetch cancelled early]'), picocolors.gray(url));
-        throw new CustomAbortError();
+        throw reusedCustomAbortError;
       }
     }
     if (controller.signal.aborted) {
       console.log(picocolors.gray('[fetch cancelled]'), picocolors.gray(url));
-      throw new CustomAbortError();
+      throw reusedCustomAbortError;
     }
     const res = await $$fetch(url, { signal: controller.signal, ...defaultRequestInit });
 

Build/lib/fetch-retry.ts

@@ -34,11 +34,7 @@ setGlobalDispatcher(agent.compose(
     // TODO: this part of code is only for allow more errors to be retried by default
     // This should be removed once https://github.com/nodejs/undici/issues/3728 is implemented
     retry(err, { state, opts }, cb) {
-      const statusCode = 'statusCode' in err && typeof err.statusCode === 'number' ? err.statusCode : null;
       const errorCode = 'code' in err ? (err as NodeJS.ErrnoException).code : undefined;
-      const headers = ('headers' in err && typeof err.headers === 'object') ? err.headers : undefined;
-
-      const { counter } = state;
 
       // Any code that is not a Undici's originated and allowed to retry
       if (
@@ -49,29 +45,7 @@ setGlobalDispatcher(agent.compose(
         return cb(err);
       }
 
-      // if (errorCode === 'UND_ERR_REQ_RETRY') {
+      const statusCode = 'statusCode' in err && typeof err.statusCode === 'number' ? err.statusCode : null;
-      //   return cb(err);
-      // }
-
-      const { method, retryOptions = {} } = opts;
-
-      const {
-        maxRetries = 5,
-        minTimeout = 500,
-        maxTimeout = 10 * 1000,
-        timeoutFactor = 2,
-        methods = ['GET', 'HEAD', 'OPTIONS', 'PUT', 'DELETE', 'TRACE']
-      } = retryOptions;
-
-      // If we reached the max number of retries
-      if (counter > maxRetries) {
-        return cb(err);
-      }
-
-      // If a set of method are provided and the current method is not in the list
-      if (Array.isArray(methods) && !methods.includes(method)) {
-        return cb(err);
-      }
 
       // bail out if the status code matches one of the following
       if (
@@ -86,6 +60,30 @@ setGlobalDispatcher(agent.compose(
         return cb(err);
       }
 
+      // if (errorCode === 'UND_ERR_REQ_RETRY') {
+      //   return cb(err);
+      // }
+
+      const {
+        maxRetries = 5,
+        minTimeout = 500,
+        maxTimeout = 10 * 1000,
+        timeoutFactor = 2,
+        methods = ['GET', 'HEAD', 'OPTIONS', 'PUT', 'DELETE', 'TRACE']
+      } = opts.retryOptions || {};
+
+      // If we reached the max number of retries
+      if (state.counter > maxRetries) {
+        return cb(err);
+      }
+
+      // If a set of method are provided and the current method is not in the list
+      if (Array.isArray(methods) && !methods.includes(opts.method)) {
+        return cb(err);
+      }
+
+      const headers = ('headers' in err && typeof err.headers === 'object') ? err.headers : undefined;
+
       const retryAfterHeader = (headers as Record<string, string> | null | undefined)?.['retry-after'];
       let retryAfter = -1;
       if (retryAfterHeader) {
@@ -97,7 +95,7 @@ setGlobalDispatcher(agent.compose(
 
       const retryTimeout = retryAfter > 0
         ? Math.min(retryAfter, maxTimeout)
-        : Math.min(minTimeout * (timeoutFactor ** (counter - 1)), maxTimeout);
+        : Math.min(minTimeout * (timeoutFactor ** (state.counter - 1)), maxTimeout);
 
       console.log('[fetch retry]', 'schedule retry', { statusCode, retryTimeout, errorCode, url: opts.origin });
       // eslint-disable-next-line sukka/prefer-timer-id -- won't leak

Build/lib/fetch-text-by-line.bench.ts

@@ -1,4 +1,4 @@
-import { readFileByLine, readFileByLineLegacy, readFileByLineNew } from './fetch-text-by-line';
+import { readFileByLine, readFileByLineNew } from './fetch-text-by-line';
 import path from 'node:path';
 import fsp from 'node:fs/promises';
 import { OUTPUT_SURGE_DIR } from '../constants/dir';
@@ -10,7 +10,6 @@ const file = path.join(OUTPUT_SURGE_DIR, 'domainset/reject_extra.conf');
 
   group(() => {
     bench('readFileByLine', () => Array.fromAsync(readFileByLine(file)));
-    bench('readFileByLineLegacy', () => Array.fromAsync(readFileByLineLegacy(file)));
     bench('readFileByLineNew', async () => Array.fromAsync(await readFileByLineNew(file)));
     bench('fsp.readFile', () => fsp.readFile(file, 'utf-8').then((content) => content.split('\n')));
   });

Build/lib/fetch-text-by-line.ts

@@ -1,5 +1,4 @@
 import fs from 'node:fs';
-import { Readable } from 'node:stream';
 import fsp from 'node:fs/promises';
 import type { FileHandle } from 'node:fs/promises';
 import readline from 'node:readline';
@@ -11,19 +10,7 @@ import { processLine, ProcessLineStream } from './process-line';
 import { $$fetch } from './fetch-retry';
 import type { UndiciResponseData } from './fetch-retry';
 import type { Response as UnidiciWebResponse } from 'undici';
-
+import { invariant } from 'foxts/guard';
-function getReadableStream(file: string | FileHandle): ReadableStream {
-  if (typeof file === 'string') {
-    // return fs.openAsBlob(file).then(blob => blob.stream())
-    return Readable.toWeb(fs.createReadStream(file/* , { encoding: 'utf-8' } */));
-  }
-  return file.readableWebStream();
-}
-
-// TODO: use FileHandle.readLine()
-export const readFileByLineLegacy: ((file: string /* | FileHandle */) => AsyncIterable<string>) = (file: string | FileHandle) => getReadableStream(file)
-  .pipeThrough(new TextDecoderStream())
-  .pipeThrough(new TextLineStream());
 
 export function readFileByLine(file: string): AsyncIterable<string> {
   return readline.createInterface({
@@ -37,26 +24,17 @@ export async function readFileByLineNew(file: string): Promise<AsyncIterable<str
   return fsp.open(file, 'r').then(fdReadLines);
 }
 
-function ensureResponseBody<T extends UndiciResponseData | UnidiciWebResponse>(resp: T): NonNullable<T['body']> {
+export const createReadlineInterfaceFromResponse: ((resp: UndiciResponseData | UnidiciWebResponse, processLine?: boolean) => ReadableStream<string>) = (resp, processLine = false) => {
-  if (resp.body == null) {
+  invariant(resp.body, 'Failed to fetch remote text');
-    throw new Error('Failed to fetch remote text');
-  }
   if ('bodyUsed' in resp && resp.bodyUsed) {
     throw new Error('Body has already been consumed.');
   }
-  return resp.body;
+  let webStream: ReadableStream<Uint8Array>;
-}
+  if ('pipeThrough' in resp.body) {
-
+    webStream = resp.body;
-export const createReadlineInterfaceFromResponse: ((resp: UndiciResponseData | UnidiciWebResponse, processLine?: boolean) => ReadableStream<string>) = (resp, processLine = false) => {
+  } else {
-  const stream = ensureResponseBody(resp);
+    throw new TypeError('Invalid response body!');
-
+  }
-  const webStream: ReadableStream<Uint8Array> = 'getReader' in stream
-    ? stream
-    : (
-      'text' in stream
-        ? stream.body as any
-        : Readable.toWeb(new Readable().wrap(stream))
-    );
 
   const resultStream = webStream
     .pipeThrough(new TextDecoderStream())

Build/lib/parse-dnsmasq.ts

@@ -1,16 +1,8 @@
 import { createReadlineInterfaceFromResponse } from './fetch-text-by-line';
 
-// https://github.com/remusao/tldts/issues/2121
-// In short, single label domain suffix is ignored due to the size optimization, so no isIcann
-// import tldts from 'tldts-experimental';
-import tldts from 'tldts';
 import type { UndiciResponseData } from './fetch-retry';
 import type { Response } from 'undici';
-
+import { fastNormalizeDomainIgnoreWww } from './normalize-domain';
-function isDomainLoose(domain: string): boolean {
-  const r = tldts.parse(domain);
-  return !!(!r.isIp && (r.isIcann || r.isPrivate));
-}
 
 export function extractDomainsFromFelixDnsmasq(line: string): string | null {
   if (line.startsWith('server=/') && line.endsWith('/114.114.114.114')) {
@@ -24,7 +16,7 @@ export async function parseFelixDnsmasqFromResp(resp: UndiciResponseData | Respo
 
   for await (const line of createReadlineInterfaceFromResponse(resp, true)) {
     const domain = extractDomainsFromFelixDnsmasq(line);
-    if (domain && isDomainLoose(domain)) {
+    if (domain && fastNormalizeDomainIgnoreWww(domain)) {
       results.push(domain);
     }
   }

package.json

@@ -59,7 +59,6 @@
     "@types/fast-fifo": "^1.3.0",
     "@types/mocha": "^10.0.10",
     "@types/node": "^22.10.7",
-    "@types/node-fetch": "^2.6.12",
     "@types/tar-fs": "^2.0.4",
     "@types/tar-stream": "^3.1.3",
     "eslint": "^9.18.0",

pnpm-lock.yaml

@@ -131,9 +131,6 @@ importers:
       '@types/node':
         specifier: ^22.10.7
         version: 22.10.7
-      '@types/node-fetch':
-        specifier: ^2.6.12
-        version: 2.6.12
       '@types/tar-fs':
         specifier: ^2.0.4
         version: 2.0.4
@@ -548,9 +545,6 @@ packages:
   '@types/mocha@10.0.10':
     resolution: {integrity: sha512-xPyYSz1cMPnJQhl0CLMH68j3gprKZaTjG3s5Vi+fDgx+uhG9NOXwbVt52eFS8ECyXhyKcjDLCBEqBExKuiZb7Q==}
 
-  '@types/node-fetch@2.6.12':
-    resolution: {integrity: sha512-8nneRWKCg3rMtF69nLQJnOYUcbafYeFSjqkw3jCRLsqkWFlHaoQrr5mXmofFGOx3DKn7UfmBMyov8ySvLRVldA==}
-
   '@types/node@22.10.7':
     resolution: {integrity: sha512-V09KvXxFiutGp6B7XkpaDXlNadZxrzajcY50EuoLIpQ6WWYCSvf19lVIazzfIzQvhUN2HjX12spLojTnhuKlGg==}
 
@@ -697,9 +691,6 @@ packages:
   async-retry@1.3.3:
     resolution: {integrity: sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==}
 
-  asynckit@0.4.0:
-    resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==}
-
   b4a@1.6.7:
     resolution: {integrity: sha512-OnAYlL5b7LEkALw87fUVafQw5rVR9RjwGd4KUwNQ6DrrNmaVaUCgLipfVlzrPQ4tWOR9P0IXGNOx50jYCCdSJg==}
 
@@ -806,10 +797,6 @@ packages:
   colorette@2.0.20:
     resolution: {integrity: sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==}
 
-  combined-stream@1.0.8:
-    resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==}
-    engines: {node: '>= 0.8'}
-
   comment-parser@1.4.1:
     resolution: {integrity: sha512-buhp5kePrmda3vhc5B9t7pUQXAb2Tnd0qgpkIhPhkHXxJpiPJ11H0ZEU0oBpJ2QztSbzG/ZxMj/CHsYJqRHmyg==}
     engines: {node: '>= 12.0.0'}
@@ -859,10 +846,6 @@ packages:
   defu@6.1.4:
     resolution: {integrity: sha512-mEQCMmwJu317oSz8CwdIOdwf3xMif1ttiM8LTufzc3g6kR+9Pe236twL8j3IYT1F7GfRgGcW6MWxzZjLIkuHIg==}
 
-  delayed-stream@1.0.0:
-    resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==}
-    engines: {node: '>=0.4.0'}
-
   detect-libc@2.0.3:
     resolution: {integrity: sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==}
     engines: {node: '>=8'}
@@ -1138,10 +1121,6 @@ packages:
     resolution: {integrity: sha512-Ld2g8rrAyMYFXBhEqMz8ZAHBi4J4uS1i/CxGMDnjyFWddMXLVcDp051DZfu+t7+ab7Wv6SMqpWmyFIj5UbfFvg==}
     engines: {node: '>=14'}
 
-  form-data@4.0.1:
-    resolution: {integrity: sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==}
-    engines: {node: '>= 6'}
-
   foxts@1.1.7:
     resolution: {integrity: sha512-Pw7S1yI20GY8gfj6RXt9usRE5TdQ/lgAqpy2EaWKUVNARC+jW0hxx/MQH8xkNlT3NSpt0X1P99CJTEvh3kVdUQ==}
 
@@ -1349,14 +1328,6 @@ packages:
     resolution: {integrity: sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==}
     engines: {node: '>=8.6'}
 
-  mime-db@1.52.0:
-    resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==}
-    engines: {node: '>= 0.6'}
-
-  mime-types@2.1.35:
-    resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==}
-    engines: {node: '>= 0.6'}
-
   mimic-response@3.1.0:
     resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==}
     engines: {node: '>=10'}
@@ -2239,11 +2210,6 @@ snapshots:
 
   '@types/mocha@10.0.10': {}
 
-  '@types/node-fetch@2.6.12':
-    dependencies:
-      '@types/node': 22.10.7
-      form-data: 4.0.1
-
   '@types/node@22.10.7':
     dependencies:
       undici-types: 6.20.0
@@ -2421,8 +2387,6 @@ snapshots:
     dependencies:
       retry: 0.13.1
 
-  asynckit@0.4.0: {}
-
   b4a@1.6.7: {}
 
   balanced-match@1.0.2: {}
@@ -2549,10 +2513,6 @@ snapshots:
 
   colorette@2.0.20: {}
 
-  combined-stream@1.0.8:
-    dependencies:
-      delayed-stream: 1.0.0
-
   comment-parser@1.4.1: {}
 
   concat-map@0.0.1: {}
@@ -2587,8 +2547,6 @@ snapshots:
 
   defu@6.1.4: {}
 
-  delayed-stream@1.0.0: {}
-
   detect-libc@2.0.3: {}
 
   diff-sequences@29.6.3: {}
@@ -2936,12 +2894,6 @@ snapshots:
       cross-spawn: 7.0.6
       signal-exit: 4.1.0
 
-  form-data@4.0.1:
-    dependencies:
-      asynckit: 0.4.0
-      combined-stream: 1.0.8
-      mime-types: 2.1.35
-
   foxts@1.1.7: {}
 
   fs-constants@1.0.0: {}
@@ -3136,12 +3088,6 @@ snapshots:
       braces: 3.0.3
       picomatch: 2.3.1
 
-  mime-db@1.52.0: {}
-
-  mime-types@2.1.35:
-    dependencies:
-      mime-db: 1.52.0
-
   mimic-response@3.1.0: {}
 
   minimatch@3.1.2: