From 3c2b49df763962bd70f123664615142482e3874b Mon Sep 17 00:00:00 2001 From: SukkaW Date: Wed, 22 Jan 2025 20:49:56 +0800 Subject: [PATCH] Chore: minor changes --- Build/build-common.ts | 83 ++++++++------------------- Build/lib/fetch-assets.ts | 22 +------ Build/lib/fetch-retry.ts | 54 +++++++++-------- Build/lib/fetch-text-by-line.bench.ts | 3 +- Build/lib/fetch-text-by-line.ts | 40 +++---------- Build/lib/parse-dnsmasq.ts | 12 +--- package.json | 1 - pnpm-lock.yaml | 54 ----------------- 8 files changed, 66 insertions(+), 203 deletions(-) diff --git a/Build/build-common.ts b/Build/build-common.ts index 277f06b4..5cffc3cb 100644 --- a/Build/build-common.ts +++ b/Build/build-common.ts @@ -16,8 +16,6 @@ const MAGIC_COMMAND_TITLE = '# $ meta_title '; const MAGIC_COMMAND_DESCRIPTION = '# $ meta_description '; const MAGIC_COMMAND_SGMODULE_MITM_HOSTNAMES = '# $ sgmodule_mitm_hostnames '; -const domainsetSrcFolder = 'domainset' + path.sep; - const clawSourceDirPromise = new Fdir() .withRelativePaths() .filter((filepath, isDirectory) => { @@ -39,15 +37,11 @@ export const buildCommon = task(require.main === module, __filename)(async (span const relativePath = paths[i]; const fullPath = SOURCE_DIR + path.sep + relativePath; - if (relativePath.startsWith(domainsetSrcFolder)) { - promises.push(transformDomainset(span, fullPath)); - continue; - } // if ( // relativePath.startsWith('ip/') // || relativePath.startsWith('non_ip/') // ) { - promises.push(transformRuleset(span, fullPath, relativePath)); + promises.push(transform(span, fullPath, relativePath)); // continue; // } @@ -102,71 +96,44 @@ function processFile(span: Span, sourcePath: string) { }); } -function transformDomainset(parentSpan: Span, sourcePath: string) { +async function transform(parentSpan: Span, sourcePath: string, relativePath: string) { const extname = path.extname(sourcePath); - const basename = path.basename(sourcePath, extname); + const id = path.basename(sourcePath, extname); + return parentSpan - .traceChildAsync( - `transform domainset: ${basename}`, - async (span) => { - const res = await processFile(span, sourcePath); - if (res === $skip) return; + .traceChild(`transform ruleset: ${id}`) + .traceAsyncFn(async (span) => { + const type = relativePath.split(path.sep)[0]; - const id = basename; - const [title, incomingDescriptions, lines] = res; + if (type !== 'ip' && type !== 'non_ip' && type !== 'domainset') { + throw new TypeError(`Invalid type: ${type}`); + } - let finalDescriptions: string[]; - if (incomingDescriptions.length) { - finalDescriptions = SHARED_DESCRIPTION.slice(); - finalDescriptions.push(''); - appendArrayInPlace(finalDescriptions, incomingDescriptions); - } else { - finalDescriptions = SHARED_DESCRIPTION; - } + const res = await processFile(span, sourcePath); + if (res === $skip) return; + const [title, descriptions, lines, sgmodulePathname] = res; + + let finalDescriptions: string[]; + if (descriptions.length) { + finalDescriptions = SHARED_DESCRIPTION.slice(); + finalDescriptions.push(''); + appendArrayInPlace(finalDescriptions, descriptions); + } else { + finalDescriptions = SHARED_DESCRIPTION; + } + + if (type === 'domainset') { return new DomainsetOutput(span, id) .withTitle(title) .withDescription(finalDescriptions) .addFromDomainset(lines) .write(); } - ); -} - -/** - * Output Surge RULE-SET and Clash classical text format - */ -async function transformRuleset(parentSpan: Span, sourcePath: string, relativePath: string) { - const extname = path.extname(sourcePath); - const basename = path.basename(sourcePath, extname); - - return parentSpan - .traceChild(`transform ruleset: ${basename}`) - .traceAsyncFn(async (span) => { - const res = await processFile(span, sourcePath); - if (res === $skip) return; - - const id = basename; - const type = relativePath.split(path.sep)[0]; - - if (type !== 'ip' && type !== 'non_ip') { - throw new TypeError(`Invalid type: ${type}`); - } - - const [title, descriptions, lines, sgmodulePathname] = res; - - let description: string[]; - if (descriptions.length) { - description = SHARED_DESCRIPTION.slice(); - description.push(''); - appendArrayInPlace(description, descriptions); - } else { - description = SHARED_DESCRIPTION; - } return new RulesetOutput(span, id, type) .withTitle(title) - .withDescription(description) + .withDescription(finalDescriptions) .withMitmSgmodulePath(sgmodulePathname) .addFromRuleset(lines) .write(); diff --git a/Build/lib/fetch-assets.ts b/Build/lib/fetch-assets.ts index 63310480..bd12b5f6 100644 --- a/Build/lib/fetch-assets.ts +++ b/Build/lib/fetch-assets.ts @@ -11,23 +11,7 @@ export class CustomAbortError extends Error { public readonly digest = 'AbortError'; } -export class Custom304NotModifiedError extends Error { - public readonly name = 'Custom304NotModifiedError'; - public readonly digest = 'Custom304NotModifiedError'; - - constructor(public readonly url: string, public readonly data: string) { - super('304 Not Modified'); - } -} - -export class CustomNoETagFallbackError extends Error { - public readonly name = 'CustomNoETagFallbackError'; - public readonly digest = 'CustomNoETagFallbackError'; - - constructor(public readonly data: string) { - super('No ETag Fallback'); - } -} +const reusedCustomAbortError = new CustomAbortError(); export async function fetchAssets(url: string, fallbackUrls: null | undefined | string[] | readonly string[], processLine = false) { const controller = new AbortController(); @@ -39,12 +23,12 @@ export async function fetchAssets(url: string, fallbackUrls: null | undefined | await waitWithAbort(50 + (index + 1) * 100, controller.signal); } catch { console.log(picocolors.gray('[fetch cancelled early]'), picocolors.gray(url)); - throw new CustomAbortError(); + throw reusedCustomAbortError; } } if (controller.signal.aborted) { console.log(picocolors.gray('[fetch cancelled]'), picocolors.gray(url)); - throw new CustomAbortError(); + throw reusedCustomAbortError; } const res = await $$fetch(url, { signal: controller.signal, ...defaultRequestInit }); diff --git a/Build/lib/fetch-retry.ts b/Build/lib/fetch-retry.ts index 64d2c2e3..c70ea10e 100644 --- a/Build/lib/fetch-retry.ts +++ b/Build/lib/fetch-retry.ts @@ -34,11 +34,7 @@ setGlobalDispatcher(agent.compose( // TODO: this part of code is only for allow more errors to be retried by default // This should be removed once https://github.com/nodejs/undici/issues/3728 is implemented retry(err, { state, opts }, cb) { - const statusCode = 'statusCode' in err && typeof err.statusCode === 'number' ? err.statusCode : null; const errorCode = 'code' in err ? (err as NodeJS.ErrnoException).code : undefined; - const headers = ('headers' in err && typeof err.headers === 'object') ? err.headers : undefined; - - const { counter } = state; // Any code that is not a Undici's originated and allowed to retry if ( @@ -49,29 +45,7 @@ setGlobalDispatcher(agent.compose( return cb(err); } - // if (errorCode === 'UND_ERR_REQ_RETRY') { - // return cb(err); - // } - - const { method, retryOptions = {} } = opts; - - const { - maxRetries = 5, - minTimeout = 500, - maxTimeout = 10 * 1000, - timeoutFactor = 2, - methods = ['GET', 'HEAD', 'OPTIONS', 'PUT', 'DELETE', 'TRACE'] - } = retryOptions; - - // If we reached the max number of retries - if (counter > maxRetries) { - return cb(err); - } - - // If a set of method are provided and the current method is not in the list - if (Array.isArray(methods) && !methods.includes(method)) { - return cb(err); - } + const statusCode = 'statusCode' in err && typeof err.statusCode === 'number' ? err.statusCode : null; // bail out if the status code matches one of the following if ( @@ -86,6 +60,30 @@ setGlobalDispatcher(agent.compose( return cb(err); } + // if (errorCode === 'UND_ERR_REQ_RETRY') { + // return cb(err); + // } + + const { + maxRetries = 5, + minTimeout = 500, + maxTimeout = 10 * 1000, + timeoutFactor = 2, + methods = ['GET', 'HEAD', 'OPTIONS', 'PUT', 'DELETE', 'TRACE'] + } = opts.retryOptions || {}; + + // If we reached the max number of retries + if (state.counter > maxRetries) { + return cb(err); + } + + // If a set of method are provided and the current method is not in the list + if (Array.isArray(methods) && !methods.includes(opts.method)) { + return cb(err); + } + + const headers = ('headers' in err && typeof err.headers === 'object') ? err.headers : undefined; + const retryAfterHeader = (headers as Record | null | undefined)?.['retry-after']; let retryAfter = -1; if (retryAfterHeader) { @@ -97,7 +95,7 @@ setGlobalDispatcher(agent.compose( const retryTimeout = retryAfter > 0 ? Math.min(retryAfter, maxTimeout) - : Math.min(minTimeout * (timeoutFactor ** (counter - 1)), maxTimeout); + : Math.min(minTimeout * (timeoutFactor ** (state.counter - 1)), maxTimeout); console.log('[fetch retry]', 'schedule retry', { statusCode, retryTimeout, errorCode, url: opts.origin }); // eslint-disable-next-line sukka/prefer-timer-id -- won't leak diff --git a/Build/lib/fetch-text-by-line.bench.ts b/Build/lib/fetch-text-by-line.bench.ts index 34d8c6cf..02660c11 100644 --- a/Build/lib/fetch-text-by-line.bench.ts +++ b/Build/lib/fetch-text-by-line.bench.ts @@ -1,4 +1,4 @@ -import { readFileByLine, readFileByLineLegacy, readFileByLineNew } from './fetch-text-by-line'; +import { readFileByLine, readFileByLineNew } from './fetch-text-by-line'; import path from 'node:path'; import fsp from 'node:fs/promises'; import { OUTPUT_SURGE_DIR } from '../constants/dir'; @@ -10,7 +10,6 @@ const file = path.join(OUTPUT_SURGE_DIR, 'domainset/reject_extra.conf'); group(() => { bench('readFileByLine', () => Array.fromAsync(readFileByLine(file))); - bench('readFileByLineLegacy', () => Array.fromAsync(readFileByLineLegacy(file))); bench('readFileByLineNew', async () => Array.fromAsync(await readFileByLineNew(file))); bench('fsp.readFile', () => fsp.readFile(file, 'utf-8').then((content) => content.split('\n'))); }); diff --git a/Build/lib/fetch-text-by-line.ts b/Build/lib/fetch-text-by-line.ts index c4044650..025f0fa5 100644 --- a/Build/lib/fetch-text-by-line.ts +++ b/Build/lib/fetch-text-by-line.ts @@ -1,5 +1,4 @@ import fs from 'node:fs'; -import { Readable } from 'node:stream'; import fsp from 'node:fs/promises'; import type { FileHandle } from 'node:fs/promises'; import readline from 'node:readline'; @@ -11,19 +10,7 @@ import { processLine, ProcessLineStream } from './process-line'; import { $$fetch } from './fetch-retry'; import type { UndiciResponseData } from './fetch-retry'; import type { Response as UnidiciWebResponse } from 'undici'; - -function getReadableStream(file: string | FileHandle): ReadableStream { - if (typeof file === 'string') { - // return fs.openAsBlob(file).then(blob => blob.stream()) - return Readable.toWeb(fs.createReadStream(file/* , { encoding: 'utf-8' } */)); - } - return file.readableWebStream(); -} - -// TODO: use FileHandle.readLine() -export const readFileByLineLegacy: ((file: string /* | FileHandle */) => AsyncIterable) = (file: string | FileHandle) => getReadableStream(file) - .pipeThrough(new TextDecoderStream()) - .pipeThrough(new TextLineStream()); +import { invariant } from 'foxts/guard'; export function readFileByLine(file: string): AsyncIterable { return readline.createInterface({ @@ -37,26 +24,17 @@ export async function readFileByLineNew(file: string): Promise(resp: T): NonNullable { - if (resp.body == null) { - throw new Error('Failed to fetch remote text'); - } +export const createReadlineInterfaceFromResponse: ((resp: UndiciResponseData | UnidiciWebResponse, processLine?: boolean) => ReadableStream) = (resp, processLine = false) => { + invariant(resp.body, 'Failed to fetch remote text'); if ('bodyUsed' in resp && resp.bodyUsed) { throw new Error('Body has already been consumed.'); } - return resp.body; -} - -export const createReadlineInterfaceFromResponse: ((resp: UndiciResponseData | UnidiciWebResponse, processLine?: boolean) => ReadableStream) = (resp, processLine = false) => { - const stream = ensureResponseBody(resp); - - const webStream: ReadableStream = 'getReader' in stream - ? stream - : ( - 'text' in stream - ? stream.body as any - : Readable.toWeb(new Readable().wrap(stream)) - ); + let webStream: ReadableStream; + if ('pipeThrough' in resp.body) { + webStream = resp.body; + } else { + throw new TypeError('Invalid response body!'); + } const resultStream = webStream .pipeThrough(new TextDecoderStream()) diff --git a/Build/lib/parse-dnsmasq.ts b/Build/lib/parse-dnsmasq.ts index b1d493f7..959c7497 100644 --- a/Build/lib/parse-dnsmasq.ts +++ b/Build/lib/parse-dnsmasq.ts @@ -1,16 +1,8 @@ import { createReadlineInterfaceFromResponse } from './fetch-text-by-line'; -// https://github.com/remusao/tldts/issues/2121 -// In short, single label domain suffix is ignored due to the size optimization, so no isIcann -// import tldts from 'tldts-experimental'; -import tldts from 'tldts'; import type { UndiciResponseData } from './fetch-retry'; import type { Response } from 'undici'; - -function isDomainLoose(domain: string): boolean { - const r = tldts.parse(domain); - return !!(!r.isIp && (r.isIcann || r.isPrivate)); -} +import { fastNormalizeDomainIgnoreWww } from './normalize-domain'; export function extractDomainsFromFelixDnsmasq(line: string): string | null { if (line.startsWith('server=/') && line.endsWith('/114.114.114.114')) { @@ -24,7 +16,7 @@ export async function parseFelixDnsmasqFromResp(resp: UndiciResponseData | Respo for await (const line of createReadlineInterfaceFromResponse(resp, true)) { const domain = extractDomainsFromFelixDnsmasq(line); - if (domain && isDomainLoose(domain)) { + if (domain && fastNormalizeDomainIgnoreWww(domain)) { results.push(domain); } } diff --git a/package.json b/package.json index a3351a31..6f492c84 100644 --- a/package.json +++ b/package.json @@ -59,7 +59,6 @@ "@types/fast-fifo": "^1.3.0", "@types/mocha": "^10.0.10", "@types/node": "^22.10.7", - "@types/node-fetch": "^2.6.12", "@types/tar-fs": "^2.0.4", "@types/tar-stream": "^3.1.3", "eslint": "^9.18.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ede50c55..843f2eee 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -131,9 +131,6 @@ importers: '@types/node': specifier: ^22.10.7 version: 22.10.7 - '@types/node-fetch': - specifier: ^2.6.12 - version: 2.6.12 '@types/tar-fs': specifier: ^2.0.4 version: 2.0.4 @@ -548,9 +545,6 @@ packages: '@types/mocha@10.0.10': resolution: {integrity: sha512-xPyYSz1cMPnJQhl0CLMH68j3gprKZaTjG3s5Vi+fDgx+uhG9NOXwbVt52eFS8ECyXhyKcjDLCBEqBExKuiZb7Q==} - '@types/node-fetch@2.6.12': - resolution: {integrity: sha512-8nneRWKCg3rMtF69nLQJnOYUcbafYeFSjqkw3jCRLsqkWFlHaoQrr5mXmofFGOx3DKn7UfmBMyov8ySvLRVldA==} - '@types/node@22.10.7': resolution: {integrity: sha512-V09KvXxFiutGp6B7XkpaDXlNadZxrzajcY50EuoLIpQ6WWYCSvf19lVIazzfIzQvhUN2HjX12spLojTnhuKlGg==} @@ -697,9 +691,6 @@ packages: async-retry@1.3.3: resolution: {integrity: sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==} - asynckit@0.4.0: - resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} - b4a@1.6.7: resolution: {integrity: sha512-OnAYlL5b7LEkALw87fUVafQw5rVR9RjwGd4KUwNQ6DrrNmaVaUCgLipfVlzrPQ4tWOR9P0IXGNOx50jYCCdSJg==} @@ -806,10 +797,6 @@ packages: colorette@2.0.20: resolution: {integrity: sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==} - combined-stream@1.0.8: - resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} - engines: {node: '>= 0.8'} - comment-parser@1.4.1: resolution: {integrity: sha512-buhp5kePrmda3vhc5B9t7pUQXAb2Tnd0qgpkIhPhkHXxJpiPJ11H0ZEU0oBpJ2QztSbzG/ZxMj/CHsYJqRHmyg==} engines: {node: '>= 12.0.0'} @@ -859,10 +846,6 @@ packages: defu@6.1.4: resolution: {integrity: sha512-mEQCMmwJu317oSz8CwdIOdwf3xMif1ttiM8LTufzc3g6kR+9Pe236twL8j3IYT1F7GfRgGcW6MWxzZjLIkuHIg==} - delayed-stream@1.0.0: - resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} - engines: {node: '>=0.4.0'} - detect-libc@2.0.3: resolution: {integrity: sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==} engines: {node: '>=8'} @@ -1138,10 +1121,6 @@ packages: resolution: {integrity: sha512-Ld2g8rrAyMYFXBhEqMz8ZAHBi4J4uS1i/CxGMDnjyFWddMXLVcDp051DZfu+t7+ab7Wv6SMqpWmyFIj5UbfFvg==} engines: {node: '>=14'} - form-data@4.0.1: - resolution: {integrity: sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==} - engines: {node: '>= 6'} - foxts@1.1.7: resolution: {integrity: sha512-Pw7S1yI20GY8gfj6RXt9usRE5TdQ/lgAqpy2EaWKUVNARC+jW0hxx/MQH8xkNlT3NSpt0X1P99CJTEvh3kVdUQ==} @@ -1349,14 +1328,6 @@ packages: resolution: {integrity: sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==} engines: {node: '>=8.6'} - mime-db@1.52.0: - resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} - engines: {node: '>= 0.6'} - - mime-types@2.1.35: - resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} - engines: {node: '>= 0.6'} - mimic-response@3.1.0: resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} engines: {node: '>=10'} @@ -2239,11 +2210,6 @@ snapshots: '@types/mocha@10.0.10': {} - '@types/node-fetch@2.6.12': - dependencies: - '@types/node': 22.10.7 - form-data: 4.0.1 - '@types/node@22.10.7': dependencies: undici-types: 6.20.0 @@ -2421,8 +2387,6 @@ snapshots: dependencies: retry: 0.13.1 - asynckit@0.4.0: {} - b4a@1.6.7: {} balanced-match@1.0.2: {} @@ -2549,10 +2513,6 @@ snapshots: colorette@2.0.20: {} - combined-stream@1.0.8: - dependencies: - delayed-stream: 1.0.0 - comment-parser@1.4.1: {} concat-map@0.0.1: {} @@ -2587,8 +2547,6 @@ snapshots: defu@6.1.4: {} - delayed-stream@1.0.0: {} - detect-libc@2.0.3: {} diff-sequences@29.6.3: {} @@ -2936,12 +2894,6 @@ snapshots: cross-spawn: 7.0.6 signal-exit: 4.1.0 - form-data@4.0.1: - dependencies: - asynckit: 0.4.0 - combined-stream: 1.0.8 - mime-types: 2.1.35 - foxts@1.1.7: {} fs-constants@1.0.0: {} @@ -3136,12 +3088,6 @@ snapshots: braces: 3.0.3 picomatch: 2.3.1 - mime-db@1.52.0: {} - - mime-types@2.1.35: - dependencies: - mime-db: 1.52.0 - mimic-response@3.1.0: {} minimatch@3.1.2: