From da7c6764c5ac5f8f796b5262054e0f685ec3ebd9 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Mon, 4 Dec 2023 00:54:52 +0800 Subject: [PATCH] Chore: revert stream back to asyncgenerator --- Build/index.ts | 6 +-- Build/lib/create-file.ts | 20 +++++-- Build/lib/fetch-remote-text-by-line.ts | 69 ++++++++++++++++++++++--- bun.lockb | Bin 106738 -> 106738 bytes package.json | 2 +- 5 files changed, 82 insertions(+), 15 deletions(-) diff --git a/Build/index.ts b/Build/index.ts index aaa46283..5b5da248 100644 --- a/Build/index.ts +++ b/Build/index.ts @@ -9,7 +9,7 @@ import { buildTelegramCIDR } from './build-telegram-cidr'; import { buildChnCidr } from './build-chn-cidr'; import { buildSpeedtestDomainSet } from './build-speedtest-domainset'; import { buildInternalCDNDomains } from './build-internal-cdn-rules'; -import { buildInternalChnDomains } from './build-internal-chn-domains'; +// import { buildInternalChnDomains } from './build-internal-chn-domains'; import { buildDomesticRuleset } from './build-domestic-ruleset'; import { buildStreamService } from './build-stream-service'; import { buildRedirectModule } from './build-redirect-module'; @@ -64,7 +64,7 @@ import { buildPublicHtml } from './build-public'; // buildInternalReverseChnCIDRWorker.postMessage('build'); // }); - const buildInternalChnDomainsPromise = buildInternalChnDomains(); + // const buildInternalChnDomainsPromise = buildInternalChnDomains(); const buildDomesticRulesetPromise = downloadPreviousBuildPromise.then(() => buildDomesticRuleset()); const buildRedirectModulePromise = downloadPreviousBuildPromise.then(() => buildRedirectModule()); @@ -84,7 +84,7 @@ import { buildPublicHtml } from './build-public'; buildSpeedtestDomainSetPromise, buildInternalCDNDomainsPromise, // buildInternalReverseChnCIDRPromise, - buildInternalChnDomainsPromise, + // buildInternalChnDomainsPromise, buildDomesticRulesetPromise, buildRedirectModulePromise, buildStreamServicePromise diff --git a/Build/lib/create-file.ts b/Build/lib/create-file.ts index fd86c7ee..ebe864b5 100644 --- a/Build/lib/create-file.ts +++ b/Build/lib/create-file.ts @@ -48,13 +48,25 @@ export async function compareAndWriteFile(linesA: string[], filePath: string) { return; } - const writer = file.writer(); + console.log(`Writing ${filePath}...`); - for (let i = 0; i < linesALen; i++) { - writer.write(`${linesA[i]}\n`); + const start = Bun.nanoseconds(); + + if (linesALen < 10000) { + await Bun.write(file, `${linesA.join('\n')}\n`); + } else { + const writer = file.writer(); + + for (let i = 0; i < linesALen; i++) { + writer.write(linesA[i]); + writer.write('\n'); + } + + writer.flush(); + await writer.end(); } - return writer.end(); + console.log(`Done writing ${filePath} in ${(Bun.nanoseconds() - start) / 1e6}ms`); } export const withBannerArray = (title: string, description: string[], date: Date, content: string[]) => { diff --git a/Build/lib/fetch-remote-text-by-line.ts b/Build/lib/fetch-remote-text-by-line.ts index ee546c2f..2a8f7071 100644 --- a/Build/lib/fetch-remote-text-by-line.ts +++ b/Build/lib/fetch-remote-text-by-line.ts @@ -1,16 +1,54 @@ import type { BunFile } from 'bun'; import { fetchWithRetry, defaultRequestInit } from './fetch-retry'; -import { TextLineStream } from './text-line-transform-stream'; -import { PolyfillTextDecoderStream } from './text-decoder-stream'; +// import { TextLineStream } from './text-line-transform-stream'; +// import { PolyfillTextDecoderStream } from './text-decoder-stream'; -export function readFileByLine(file: string | BunFile) { +// export function readFileByLine(file: string | BunFile) { +// if (typeof file === 'string') { +// file = Bun.file(file); +// } +// return file.stream().pipeThrough(new PolyfillTextDecoderStream()).pipeThrough(new TextLineStream()); +// } + +// export function createReadlineInterfaceFromResponse(resp: Response) { +// if (!resp.body) { +// throw new Error('Failed to fetch remote text'); +// } +// if (resp.bodyUsed) { +// throw new Error('Body has already been consumed.'); +// } + +// return (resp.body as ReadableStream).pipeThrough(new PolyfillTextDecoderStream()).pipeThrough(new TextLineStream()); +// } + +const decoder = new TextDecoder('utf-8'); + +export async function *readFileByLine(file: string | BunFile): AsyncGenerator { if (typeof file === 'string') { file = Bun.file(file); } - return file.stream().pipeThrough(new PolyfillTextDecoderStream()).pipeThrough(new TextLineStream()); + + let buf = ''; + + for await (const chunk of file.stream()) { + const chunkStr = decoder.decode(chunk).replaceAll('\r\n', '\n'); + for (let i = 0, len = chunkStr.length; i < len; i++) { + const char = chunkStr[i]; + if (char === '\n') { + yield buf; + buf = ''; + } else { + buf += char; + } + } + } + + if (buf) { + yield buf; + } } -export function createReadlineInterfaceFromResponse(resp: Response) { +export async function *createReadlineInterfaceFromResponse(resp: Response): AsyncGenerator { if (!resp.body) { throw new Error('Failed to fetch remote text'); } @@ -18,9 +56,26 @@ export function createReadlineInterfaceFromResponse(resp: Response) { throw new Error('Body has already been consumed.'); } - return (resp.body as ReadableStream).pipeThrough(new PolyfillTextDecoderStream()).pipeThrough(new TextLineStream()); + let buf = ''; + + for await (const chunk of resp.body) { + const chunkStr = decoder.decode(chunk).replaceAll('\r\n', '\n'); + for (let i = 0, len = chunkStr.length; i < len; i++) { + const char = chunkStr[i]; + if (char === '\n') { + yield buf; + buf = ''; + } else { + buf += char; + } + } + } + + if (buf) { + yield buf; + } } export function fetchRemoteTextAndCreateReadlineInterface(url: string | URL) { - return fetchWithRetry(url, defaultRequestInit).then(res => createReadlineInterfaceFromResponse(res)); + return fetchWithRetry(url, defaultRequestInit).then(res => createReadlineInterfaceFromResponse(res as Response)); } diff --git a/bun.lockb b/bun.lockb index d7b3132c1a10d80c88738ec19ca1ace3ac7d32aa..f68926dfdb4148a526222b43a717eea429c378a2 100755 GIT binary patch delta 154 zcmV;L0A>I3!3Ofd29Pcw4uLRyb#?5IrldXYZc0)an^jEeTx+W2zM6fVzw&e$u}*d} zlLQzvv-mO{VnC~t2^7d9#Q46PwwP2$O)W0_yVrj;kkH=ks@NEk?ux$V9c1QUB$Ibp z=yHABS(~P$?{{Vmb(vNCY>6(EbJMd`mCTC=0ssI20002DC?NqvQxc% Io^%0JEaS;XHUIzs delta 156 zcmV;N0Av61!3Ofd29Pcw`z1Pi`n-|Jo=+=ZVsqRAebZ(NJB5WOLFWUa)hPH@u}*d} zlY|m9v-mO{Vn9H!Su|j zN$~)-D2yX`Izcza-^Q{u3M&%q`J=N{mCTC=0bVdJGA=N;C?NqvQvouUF=qiGmoIq% K5VxLm0aPr}*+O#w diff --git a/package.json b/package.json index 9998233f..116ec9a7 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "license": "ISC", "dependencies": { "@cliqz/adblocker": "^1.26.12", - "@sukka/listdir": "^0.2.0", + "@sukka/listdir": "^0.3.1", "async-retry": "^1.3.3", "async-sema": "^3.1.1", "ci-info": "^4.0.0",