diff --git a/Build/lib/fetch-text-by-line.bench.ts b/Build/lib/fetch-text-by-line.bench.ts index a93ff214..c3bcfa5b 100644 --- a/Build/lib/fetch-text-by-line.bench.ts +++ b/Build/lib/fetch-text-by-line.bench.ts @@ -1,15 +1,19 @@ -import { bench, group, run } from 'mitata'; import { processLine, processLineFromReadline } from './process-line'; -import { readFileByLine } from './fetch-text-by-line'; +import { readFileByLine, readFileByLineLegacy } from './fetch-text-by-line'; import path from 'node:path'; import fsp from 'node:fs/promises'; import { SOURCE_DIR } from '../constants/dir'; const file = path.join(SOURCE_DIR, 'domainset/cdn.conf'); -group(() => { - bench('readFileByLine', () => processLineFromReadline(readFileByLine(file))); - bench('fsp.readFile', () => fsp.readFile(file, 'utf-8').then((content) => content.split('\n').filter(processLine))); -}); +(async () => { + const { bench, group, run } = await import('mitata'); -run(); + group(() => { + bench('readFileByLine', () => processLineFromReadline(readFileByLine(file))); + bench('readFileByLineLegacy', () => processLineFromReadline(readFileByLineLegacy(file))); + bench('fsp.readFile', () => fsp.readFile(file, 'utf-8').then((content) => content.split('\n').filter(processLine))); + }); + + run(); +})(); diff --git a/Build/lib/fetch-text-by-line.ts b/Build/lib/fetch-text-by-line.ts index 52db0d2b..cab25060 100644 --- a/Build/lib/fetch-text-by-line.ts +++ b/Build/lib/fetch-text-by-line.ts @@ -1,6 +1,7 @@ import fs from 'node:fs'; import { Readable } from 'node:stream'; import type { FileHandle } from 'node:fs/promises'; +import readline from 'node:readline'; import { TextLineStream } from './text-line-transform-stream'; import type { ReadableStream } from 'node:stream/web'; @@ -18,11 +19,17 @@ function getReadableStream(file: string | FileHandle): ReadableStream { } return file.readableWebStream(); } + // TODO: use FileHandle.readLine() -export const readFileByLine: ((file: string | FileHandle) => AsyncIterable) = (file: string | FileHandle) => getReadableStream(file) +export const readFileByLineLegacy: ((file: string /* | FileHandle */) => AsyncIterable) = (file: string | FileHandle) => getReadableStream(file) .pipeThrough(new TextDecoderStream()) .pipeThrough(new TextLineStream()); +export const readFileByLine: ((file: string /* | FileHandle */) => AsyncIterable) = (file: string) => readline.createInterface({ + input: fs.createReadStream(file/* , { encoding: 'utf-8' } */), + crlfDelay: Infinity +}); + function ensureResponseBody(resp: T): NonNullable { if (resp.body == null) { throw new Error('Failed to fetch remote text'); @@ -53,7 +60,7 @@ export function fetchRemoteTextByLine(url: string) { return $fetch(url).then(createReadlineInterfaceFromResponse); } -export async function readFileIntoProcessedArray(file: string | FileHandle) { +export async function readFileIntoProcessedArray(file: string /* | FileHandle */) { const results = []; for await (const line of readFileByLine(file)) { if (processLine(line)) {