GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2026-01-29 10:01:54 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Perf: read most of files in one pass

Browse Source
This commit is contained in:
SukkaW
2024-01-15 13:32:12 +08:00
parent a3e1a85c70
commit a7fc13b355
7 changed files with 46 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
Build/lib/fetch-text-by-line.bench.ts Normal file
View File

@@ -0,0 +1,17 @@
import { bench, group, run } from 'mitata';
import { processLine, processLineFromReadline } from './process-line';
import { readFileByLine } from './fetch-text-by-line';
import path from 'path';
import fsp from 'fs/promises';
const file = path.resolve(import.meta.dir, '../../Source/domainset/cdn.conf');
group('read file by line', () => {
bench('readline', () => processLineFromReadline(readFileByLine(file)));
bench('fsp.readFile', () => fsp.readFile(file, 'utf-8').then((content) => content.split('\n').filter(processLine)));
bench('Bun.file', () => Bun.file(file).text().then((content) => content.split('\n').filter(processLine)));
});
run();

12
Build/lib/fetch-text-by-line.ts
View File

@@ -3,6 +3,7 @@ import { fetchWithRetry, defaultRequestInit } from './fetch-retry';
import { TextLineStream } from './text-line-transform-stream';
import { PolyfillTextDecoderStream } from './text-decoder-stream';
import { processLine } from './process-line';
// function createTextLineStreamFromStreamSource(stream: ReadableStream<Uint8Array>) {
// return stream
// .pipeThrough(new PolyfillTextDecoderStream())
@@ -54,3 +55,14 @@ export function createReadlineInterfaceFromResponse(this: void, resp: Response)
export function fetchRemoteTextByLine(url: string | URL) {
return fetchWithRetry(url, defaultRequestInit).then(createReadlineInterfaceFromResponse);
}
export async function readFileIntoProcessedArray(file: string | URL | BunFile) {
if (typeof file === 'string') {
file = Bun.file(file);
} else if (!('writer' in file)) {
file = Bun.file(file);
}
const content = await file.text();
return content.split('\n').filter(processLine);
}