GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2026-01-29 01:51:52 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Perf: use readline to readFileByLine (50% faster)

Browse Source
This commit is contained in:
SukkaW
2024-11-13 11:12:38 +08:00
parent 7499fea63f
commit 3107e72787
2 changed files with 20 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
Build/lib/fetch-text-by-line.ts
View File

@@ -1,6 +1,7 @@
import fs from 'node:fs';
import { Readable } from 'node:stream';
import type { FileHandle } from 'node:fs/promises';
import readline from 'node:readline';
import { TextLineStream } from './text-line-transform-stream';
import type { ReadableStream } from 'node:stream/web';
@@ -18,11 +19,17 @@ function getReadableStream(file: string | FileHandle): ReadableStream {
}
return file.readableWebStream();
}
// TODO: use FileHandle.readLine()
export const readFileByLine: ((file: string | FileHandle) => AsyncIterable<string>) = (file: string | FileHandle) => getReadableStream(file)
export const readFileByLineLegacy: ((file: string /* | FileHandle */) => AsyncIterable<string>) = (file: string | FileHandle) => getReadableStream(file)
.pipeThrough(new TextDecoderStream())
.pipeThrough(new TextLineStream());
export const readFileByLine: ((file: string /* | FileHandle */) => AsyncIterable<string>) = (file: string) => readline.createInterface({
input: fs.createReadStream(file/* , { encoding: 'utf-8' } */),
crlfDelay: Infinity
});
function ensureResponseBody<T extends NodeFetchResponse | UndiciResponseData | UnidiciWebResponse>(resp: T): NonNullable<T['body']> {
if (resp.body == null) {
throw new Error('Failed to fetch remote text');
@@ -53,7 +60,7 @@ export function fetchRemoteTextByLine(url: string) {
return $fetch(url).then(createReadlineInterfaceFromResponse);
}
export async function readFileIntoProcessedArray(file: string | FileHandle) {
export async function readFileIntoProcessedArray(file: string /* | FileHandle */) {
const results = [];
for await (const line of readFileByLine(file)) {
if (processLine(line)) {