import type { BunFile } from 'bun'; import { fetchWithRetry, defaultRequestInit } from './fetch-retry'; import { TextLineStream } from './text-line-transform-stream'; import { PolyfillTextDecoderStream } from './text-decoder-stream'; function createTextLineStreamFromStreamSource(stream: ReadableStream) { return stream .pipeThrough(new PolyfillTextDecoderStream()) .pipeThrough(new TextLineStream()); } // const decoder = new TextDecoder('utf-8'); // async function *createTextLineAsyncGeneratorFromStreamSource(stream: ReadableStream): AsyncGenerator { // let buf = ''; // for await (const chunk of stream) { // const chunkStr = decoder.decode(chunk).replaceAll('\r\n', '\n'); // for (let i = 0, len = chunkStr.length; i < len; i++) { // const char = chunkStr[i]; // if (char === '\n') { // yield buf; // buf = ''; // } else { // buf += char; // } // } // } // if (buf) { // yield buf; // } // } export function readFileByLine(file: string | URL | BunFile) { if (typeof file === 'string') { file = Bun.file(file); } else if (!('writer' in file)) { file = Bun.file(file); } return createTextLineStreamFromStreamSource(file.stream()); } export function createReadlineInterfaceFromResponse(resp: Response) { if (!resp.body) { throw new Error('Failed to fetch remote text'); } if (resp.bodyUsed) { throw new Error('Body has already been consumed.'); } return createTextLineStreamFromStreamSource(resp.body); } export function fetchRemoteTextByLine(url: string | URL) { return fetchWithRetry(url, defaultRequestInit).then(res => createReadlineInterfaceFromResponse(res)); }