diff --git a/Build/index.ts b/Build/index.ts index aaa46283..5b5da248 100644 --- a/Build/index.ts +++ b/Build/index.ts @@ -9,7 +9,7 @@ import { buildTelegramCIDR } from './build-telegram-cidr'; import { buildChnCidr } from './build-chn-cidr'; import { buildSpeedtestDomainSet } from './build-speedtest-domainset'; import { buildInternalCDNDomains } from './build-internal-cdn-rules'; -import { buildInternalChnDomains } from './build-internal-chn-domains'; +// import { buildInternalChnDomains } from './build-internal-chn-domains'; import { buildDomesticRuleset } from './build-domestic-ruleset'; import { buildStreamService } from './build-stream-service'; import { buildRedirectModule } from './build-redirect-module'; @@ -64,7 +64,7 @@ import { buildPublicHtml } from './build-public'; // buildInternalReverseChnCIDRWorker.postMessage('build'); // }); - const buildInternalChnDomainsPromise = buildInternalChnDomains(); + // const buildInternalChnDomainsPromise = buildInternalChnDomains(); const buildDomesticRulesetPromise = downloadPreviousBuildPromise.then(() => buildDomesticRuleset()); const buildRedirectModulePromise = downloadPreviousBuildPromise.then(() => buildRedirectModule()); @@ -84,7 +84,7 @@ import { buildPublicHtml } from './build-public'; buildSpeedtestDomainSetPromise, buildInternalCDNDomainsPromise, // buildInternalReverseChnCIDRPromise, - buildInternalChnDomainsPromise, + // buildInternalChnDomainsPromise, buildDomesticRulesetPromise, buildRedirectModulePromise, buildStreamServicePromise diff --git a/Build/lib/create-file.ts b/Build/lib/create-file.ts index fd86c7ee..ebe864b5 100644 --- a/Build/lib/create-file.ts +++ b/Build/lib/create-file.ts @@ -48,13 +48,25 @@ export async function compareAndWriteFile(linesA: string[], filePath: string) { return; } - const writer = file.writer(); + console.log(`Writing ${filePath}...`); - for (let i = 0; i < linesALen; i++) { - writer.write(`${linesA[i]}\n`); + const start = Bun.nanoseconds(); + + if (linesALen < 10000) { + await Bun.write(file, `${linesA.join('\n')}\n`); + } else { + const writer = file.writer(); + + for (let i = 0; i < linesALen; i++) { + writer.write(linesA[i]); + writer.write('\n'); + } + + writer.flush(); + await writer.end(); } - return writer.end(); + console.log(`Done writing ${filePath} in ${(Bun.nanoseconds() - start) / 1e6}ms`); } export const withBannerArray = (title: string, description: string[], date: Date, content: string[]) => { diff --git a/Build/lib/fetch-remote-text-by-line.ts b/Build/lib/fetch-remote-text-by-line.ts index ee546c2f..2a8f7071 100644 --- a/Build/lib/fetch-remote-text-by-line.ts +++ b/Build/lib/fetch-remote-text-by-line.ts @@ -1,16 +1,54 @@ import type { BunFile } from 'bun'; import { fetchWithRetry, defaultRequestInit } from './fetch-retry'; -import { TextLineStream } from './text-line-transform-stream'; -import { PolyfillTextDecoderStream } from './text-decoder-stream'; +// import { TextLineStream } from './text-line-transform-stream'; +// import { PolyfillTextDecoderStream } from './text-decoder-stream'; -export function readFileByLine(file: string | BunFile) { +// export function readFileByLine(file: string | BunFile) { +// if (typeof file === 'string') { +// file = Bun.file(file); +// } +// return file.stream().pipeThrough(new PolyfillTextDecoderStream()).pipeThrough(new TextLineStream()); +// } + +// export function createReadlineInterfaceFromResponse(resp: Response) { +// if (!resp.body) { +// throw new Error('Failed to fetch remote text'); +// } +// if (resp.bodyUsed) { +// throw new Error('Body has already been consumed.'); +// } + +// return (resp.body as ReadableStream).pipeThrough(new PolyfillTextDecoderStream()).pipeThrough(new TextLineStream()); +// } + +const decoder = new TextDecoder('utf-8'); + +export async function *readFileByLine(file: string | BunFile): AsyncGenerator { if (typeof file === 'string') { file = Bun.file(file); } - return file.stream().pipeThrough(new PolyfillTextDecoderStream()).pipeThrough(new TextLineStream()); + + let buf = ''; + + for await (const chunk of file.stream()) { + const chunkStr = decoder.decode(chunk).replaceAll('\r\n', '\n'); + for (let i = 0, len = chunkStr.length; i < len; i++) { + const char = chunkStr[i]; + if (char === '\n') { + yield buf; + buf = ''; + } else { + buf += char; + } + } + } + + if (buf) { + yield buf; + } } -export function createReadlineInterfaceFromResponse(resp: Response) { +export async function *createReadlineInterfaceFromResponse(resp: Response): AsyncGenerator { if (!resp.body) { throw new Error('Failed to fetch remote text'); } @@ -18,9 +56,26 @@ export function createReadlineInterfaceFromResponse(resp: Response) { throw new Error('Body has already been consumed.'); } - return (resp.body as ReadableStream).pipeThrough(new PolyfillTextDecoderStream()).pipeThrough(new TextLineStream()); + let buf = ''; + + for await (const chunk of resp.body) { + const chunkStr = decoder.decode(chunk).replaceAll('\r\n', '\n'); + for (let i = 0, len = chunkStr.length; i < len; i++) { + const char = chunkStr[i]; + if (char === '\n') { + yield buf; + buf = ''; + } else { + buf += char; + } + } + } + + if (buf) { + yield buf; + } } export function fetchRemoteTextAndCreateReadlineInterface(url: string | URL) { - return fetchWithRetry(url, defaultRequestInit).then(res => createReadlineInterfaceFromResponse(res)); + return fetchWithRetry(url, defaultRequestInit).then(res => createReadlineInterfaceFromResponse(res as Response)); } diff --git a/bun.lockb b/bun.lockb index d7b3132c..f68926df 100755 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/package.json b/package.json index 9998233f..116ec9a7 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "license": "ISC", "dependencies": { "@cliqz/adblocker": "^1.26.12", - "@sukka/listdir": "^0.2.0", + "@sukka/listdir": "^0.3.1", "async-retry": "^1.3.3", "async-sema": "^3.1.1", "ci-info": "^4.0.0",