GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2026-01-29 01:51:52 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Chore: enable TextLineStream with env

Browse Source
This commit is contained in:
SukkaW
2024-03-17 16:21:29 +08:00
parent c276ac3330
commit 0b0dda6cc2
3 changed files with 46 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
Build/lib/fetch-text-by-line.ts
View File

@@ -1,20 +1,21 @@
import type { BunFile } from 'bun';
import { fetchWithRetry, defaultRequestInit } from './fetch-retry';
// import { TextLineStream } from './text-line-transform-stream';
// import { PolyfillTextDecoderStream } from './text-decoder-stream';
import { TextLineStream } from './text-line-transform-stream';
import { PolyfillTextDecoderStream } from './text-decoder-stream';
import { processLine } from './process-line';
// function createTextLineStreamFromStreamSource(stream: ReadableStream<Uint8Array>) {
// return stream
// .pipeThrough(new PolyfillTextDecoderStream())
// .pipeThrough(new TextLineStream());
// }
const enableTextLineStream = !!process.env.ENABLE_TEXT_LINE_STREAM;
interface TextLineStreamLike {
[Symbol.asyncIterator](): AsyncIterableIterator<string>
}
const decoder = new TextDecoder('utf-8');
async function *createTextLineAsyncGeneratorFromStreamSource(stream: ReadableStream<Uint8Array>): AsyncGenerator<string> {
let buf = '';
for await (const chunk of stream) {
for await (const chunk of stream as any) {
const chunkStr = decoder.decode(chunk).replaceAll('\r\n', '\n');
for (let i = 0, len = chunkStr.length; i < len; i++) {
const char = chunkStr[i];
@@ -32,25 +33,45 @@ async function *createTextLineAsyncGeneratorFromStreamSource(stream: ReadableStr
}
}
export function readFileByLine(file: string | URL | BunFile) {
if (typeof file === 'string') {
file = Bun.file(file);
} else if (!('writer' in file)) {
file = Bun.file(file);
}
export const readFileByLine: ((file: string | URL | BunFile) => TextLineStreamLike) = enableTextLineStream
? (file: string | URL | BunFile) => {
if (typeof file === 'string') {
file = Bun.file(file);
} else if (!('writer' in file)) {
file = Bun.file(file);
}
return createTextLineAsyncGeneratorFromStreamSource(file.stream());
}
return file.stream().pipeThrough(new PolyfillTextDecoderStream()).pipeThrough(new TextLineStream());
}
: (file: string | URL | BunFile) => {
if (typeof file === 'string') {
file = Bun.file(file);
} else if (!('writer' in file)) {
file = Bun.file(file);
}
export function createReadlineInterfaceFromResponse(this: void, resp: Response) {
if (!resp.body) {
throw new Error('Failed to fetch remote text');
return createTextLineAsyncGeneratorFromStreamSource(file.stream()) as any;
};
export const createReadlineInterfaceFromResponse: ((resp: Response) => TextLineStreamLike) = enableTextLineStream
? (resp) => {
if (!resp.body) {
throw new Error('Failed to fetch remote text');
}
if (resp.bodyUsed) {
throw new Error('Body has already been consumed.');
}
return resp.body.pipeThrough(new PolyfillTextDecoderStream()).pipeThrough(new TextLineStream());
}
if (resp.bodyUsed) {
throw new Error('Body has already been consumed.');
}
return createTextLineAsyncGeneratorFromStreamSource(resp.body);
}
: (resp) => {
if (!resp.body) {
throw new Error('Failed to fetch remote text');
}
if (resp.bodyUsed) {
throw new Error('Body has already been consumed.');
}
return createTextLineAsyncGeneratorFromStreamSource(resp.body) as any;
};
export function fetchRemoteTextByLine(url: string | URL) {
return fetchWithRetry(url, defaultRequestInit).then(createReadlineInterfaceFromResponse);