diff --git a/Build/build-anti-bogus-domain.ts b/Build/build-anti-bogus-domain.ts index 69ac7ae6..90967ce3 100644 --- a/Build/build-anti-bogus-domain.ts +++ b/Build/build-anti-bogus-domain.ts @@ -1,7 +1,7 @@ // @ts-check import path from 'path'; import { createRuleset } from './lib/create-file'; -import { fetchRemoteTextByLine, readFileByLine } from './lib/fetch-text-by-line'; +import { fetchRemoteTextByLine, readFileByLine, readFileIntoProcessedArray } from './lib/fetch-text-by-line'; import { processLine } from './lib/process-line'; import { task } from './trace'; import { SHARED_DESCRIPTION } from './lib/constants'; @@ -34,14 +34,7 @@ const getBogusNxDomainIPsPromise = fsCache.apply( ); export const buildAntiBogusDomain = task(import.meta.path, async (span) => { - const result: string[] = []; - for await (const line of readFileByLine(path.resolve(import.meta.dir, '../Source/ip/reject.conf'))) { - const l = processLine(line); - if (l) { - result.push(l); - } - } - + const result: string[] = await readFileIntoProcessedArray(path.resolve(import.meta.dir, '../Source/ip/reject.conf')); result.push(...(await getBogusNxDomainIPsPromise)); const description = [ diff --git a/Build/build-apple-cdn.ts b/Build/build-apple-cdn.ts index aa5e5714..73a7fd2c 100644 --- a/Build/build-apple-cdn.ts +++ b/Build/build-apple-cdn.ts @@ -18,7 +18,7 @@ export const getAppleCdnDomainsPromise = createMemoizedPromise(() => fsCache.app )); export const buildAppleCdn = task(import.meta.path, async (span) => { - const res = await getAppleCdnDomainsPromise(); + const res = await span.traceChild('get apple cdn domains').traceAsyncFn(getAppleCdnDomainsPromise); const description = [ ...SHARED_DESCRIPTION, diff --git a/Build/build-cdn-conf.ts b/Build/build-cdn-conf.ts index 1d34e821..a7aa6aaf 100644 --- a/Build/build-cdn-conf.ts +++ b/Build/build-cdn-conf.ts @@ -1,9 +1,8 @@ import path from 'path'; import { createRuleset } from './lib/create-file'; -import { readFileByLine } from './lib/fetch-text-by-line'; +import { readFileIntoProcessedArray } from './lib/fetch-text-by-line'; import { createTrie } from './lib/trie'; import { task } from './trace'; -import { processLine } from './lib/process-line'; import { SHARED_DESCRIPTION } from './lib/constants'; import { getPublicSuffixListTextPromise } from './download-publicsuffixlist'; @@ -44,15 +43,7 @@ const getS3OSSDomainsPromise = (async (): Promise> => { const buildCdnConf = task(import.meta.path, async (span) => { /** @type {string[]} */ - const cdnDomainsList: string[] = []; - - for await (const l of readFileByLine(path.resolve(import.meta.dir, '../Source/non_ip/cdn.conf'))) { - const line = processLine(l); - if (line) { - cdnDomainsList.push(line); - } - } - + const cdnDomainsList: string[] = await readFileIntoProcessedArray(path.resolve(import.meta.dir, '../Source/non_ip/cdn.conf')); (await getS3OSSDomainsPromise).forEach((domain: string) => { cdnDomainsList.push(`DOMAIN-SUFFIX,${domain}`); }); const description: string[] = [ diff --git a/Build/build-domestic-ruleset.ts b/Build/build-domestic-ruleset.ts index 7a1970bd..db8732fe 100644 --- a/Build/build-domestic-ruleset.ts +++ b/Build/build-domestic-ruleset.ts @@ -1,15 +1,14 @@ // @ts-check import path from 'path'; import { DOMESTICS } from '../Source/non_ip/domestic'; -import { readFileByLine } from './lib/fetch-text-by-line'; -import { processLineFromReadline } from './lib/process-line'; +import { readFileIntoProcessedArray } from './lib/fetch-text-by-line'; import { compareAndWriteFile, createRuleset } from './lib/create-file'; import { task } from './trace'; import { SHARED_DESCRIPTION } from './lib/constants'; import { createMemoizedPromise } from './lib/memo-promise'; export const getDomesticDomainsRulesetPromise = createMemoizedPromise(async () => { - const results = await processLineFromReadline(readFileByLine(path.resolve(import.meta.dir, '../Source/non_ip/domestic.conf'))); + const results = await readFileIntoProcessedArray(path.resolve(import.meta.dir, '../Source/non_ip/domestic.conf')); results.push( ...Object.entries(DOMESTICS).reduce((acc, [key, { domains }]) => { diff --git a/Build/build-sspanel-appprofile.ts b/Build/build-sspanel-appprofile.ts index 6ac9c204..f0050d31 100644 --- a/Build/build-sspanel-appprofile.ts +++ b/Build/build-sspanel-appprofile.ts @@ -1,8 +1,7 @@ import { getAppleCdnDomainsPromise } from './build-apple-cdn'; import { getDomesticDomainsRulesetPromise } from './build-domestic-ruleset'; import { surgeRulesetToClashClassicalTextRuleset } from './lib/clash'; -import { readFileByLine } from './lib/fetch-text-by-line'; -import { processLineFromReadline } from './lib/process-line'; +import { readFileIntoProcessedArray } from './lib/fetch-text-by-line'; import { task } from './trace'; import path from 'path'; @@ -49,19 +48,19 @@ export const buildSSPanelUIMAppProfile = task(import.meta.path, async (span) => getDomesticDomainsRulesetPromise().then(surgeRulesetToClashClassicalTextRuleset), getAppleCdnDomainsPromise().then(domains => domains.map(domain => `DOMAIN-SUFFIX,${domain}`)), getMicrosoftCdnRulesetPromise().then(surgeRulesetToClashClassicalTextRuleset), - processLineFromReadline(readFileByLine(path.resolve(import.meta.dir, '../Source/non_ip/apple_cn.conf'))), - processLineFromReadline(readFileByLine(path.resolve(import.meta.dir, '../Source/non_ip/neteasemusic.conf'))).then(surgeRulesetToClashClassicalTextRuleset), + readFileIntoProcessedArray(path.resolve(import.meta.dir, '../Source/non_ip/apple_cn.conf')), + readFileIntoProcessedArray(path.resolve(import.meta.dir, '../Source/non_ip/neteasemusic.conf')).then(surgeRulesetToClashClassicalTextRuleset), // microsoft & apple - domains - processLineFromReadline(readFileByLine(path.resolve(import.meta.dir, '../Source/non_ip/microsoft.conf'))), - (processLineFromReadline(readFileByLine(path.resolve(import.meta.dir, '../Source/non_ip/apple_services.conf')))).then(surgeRulesetToClashClassicalTextRuleset), + readFileIntoProcessedArray(path.resolve(import.meta.dir, '../Source/non_ip/microsoft.conf')), + readFileIntoProcessedArray(path.resolve(import.meta.dir, '../Source/non_ip/apple_services.conf')).then(surgeRulesetToClashClassicalTextRuleset), // stream - domains surgeRulesetToClashClassicalTextRuleset(AllStreamServices.flatMap((i) => i.rules)), // global - domains - processLineFromReadline(readFileByLine(path.resolve(import.meta.dir, '../Source/non_ip/global.conf'))).then(surgeRulesetToClashClassicalTextRuleset), - processLineFromReadline(readFileByLine(path.resolve(import.meta.dir, '../Source/non_ip/global_plus.conf'))).then(surgeRulesetToClashClassicalTextRuleset), - processLineFromReadline(readFileByLine(path.resolve(import.meta.dir, '../Source/non_ip/telegram.conf'))).then(surgeRulesetToClashClassicalTextRuleset), + readFileIntoProcessedArray(path.resolve(import.meta.dir, '../Source/non_ip/global.conf')).then(surgeRulesetToClashClassicalTextRuleset), + readFileIntoProcessedArray(path.resolve(import.meta.dir, '../Source/non_ip/global_plus.conf')).then(surgeRulesetToClashClassicalTextRuleset), + readFileIntoProcessedArray(path.resolve(import.meta.dir, '../Source/non_ip/telegram.conf')).then(surgeRulesetToClashClassicalTextRuleset), // lan - domains - processLineFromReadline(readFileByLine(path.resolve(import.meta.dir, '../Source/non_ip/lan.conf'))), + readFileIntoProcessedArray(path.resolve(import.meta.dir, '../Source/non_ip/lan.conf')), // domestic - ip cidr getChnCidrPromise().then(cidrs => cidrs.map(cidr => `IP-CIDR,${cidr}`)), AllStreamServices.flatMap((i) => ( @@ -75,7 +74,7 @@ export const buildSSPanelUIMAppProfile = task(import.meta.path, async (span) => // global - ip cidr getTelegramCIDRPromise(), // lan - ip cidr - processLineFromReadline(readFileByLine(path.resolve(import.meta.dir, '../Source/ip/lan.conf'))) + readFileIntoProcessedArray(path.resolve(import.meta.dir, '../Source/ip/lan.conf')) ] as const); const telegramCidrs = rawTelegramCidrs.map(removeNoResolved); diff --git a/Build/lib/fetch-text-by-line.bench.ts b/Build/lib/fetch-text-by-line.bench.ts new file mode 100644 index 00000000..6093d7e3 --- /dev/null +++ b/Build/lib/fetch-text-by-line.bench.ts @@ -0,0 +1,17 @@ +import { bench, group, run } from 'mitata'; +import { processLine, processLineFromReadline } from './process-line'; +import { readFileByLine } from './fetch-text-by-line'; +import path from 'path'; +import fsp from 'fs/promises'; + +const file = path.resolve(import.meta.dir, '../../Source/domainset/cdn.conf'); + +group('read file by line', () => { + bench('readline', () => processLineFromReadline(readFileByLine(file))); + + bench('fsp.readFile', () => fsp.readFile(file, 'utf-8').then((content) => content.split('\n').filter(processLine))); + + bench('Bun.file', () => Bun.file(file).text().then((content) => content.split('\n').filter(processLine))); +}); + +run(); diff --git a/Build/lib/fetch-text-by-line.ts b/Build/lib/fetch-text-by-line.ts index 37036bc0..e012017c 100644 --- a/Build/lib/fetch-text-by-line.ts +++ b/Build/lib/fetch-text-by-line.ts @@ -3,6 +3,7 @@ import { fetchWithRetry, defaultRequestInit } from './fetch-retry'; import { TextLineStream } from './text-line-transform-stream'; import { PolyfillTextDecoderStream } from './text-decoder-stream'; +import { processLine } from './process-line'; // function createTextLineStreamFromStreamSource(stream: ReadableStream) { // return stream // .pipeThrough(new PolyfillTextDecoderStream()) @@ -54,3 +55,14 @@ export function createReadlineInterfaceFromResponse(this: void, resp: Response) export function fetchRemoteTextByLine(url: string | URL) { return fetchWithRetry(url, defaultRequestInit).then(createReadlineInterfaceFromResponse); } + +export async function readFileIntoProcessedArray(file: string | URL | BunFile) { + if (typeof file === 'string') { + file = Bun.file(file); + } else if (!('writer' in file)) { + file = Bun.file(file); + } + + const content = await file.text(); + return content.split('\n').filter(processLine); +}