GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2026-01-29 01:51:52 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Feat: introduce make-fetch-happen (#44)

Browse Source
This commit is contained in:
Sukka
2024-10-10 16:30:36 +08:00
committed by GitHub
parent bb07225f6c
commit c75f7fcc76
12 changed files with 656 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
Build/lib/fetch-text-by-line.ts
View File

@@ -1,12 +1,13 @@
import fs from 'node:fs';
import { Readable } from 'node:stream';
import { fetchWithRetry, defaultRequestInit } from './fetch-retry';
import type { FileHandle } from 'node:fs/promises';
import { TextLineStream } from './text-line-transform-stream';
import type { ReadableStream } from 'node:stream/web';
import { TextDecoderStream } from 'node:stream/web';
import { processLine } from './process-line';
import { $fetch } from './make-fetch-happen';
import type { NodeFetchResponse } from './make-fetch-happen';
const getReadableStream = (file: string | FileHandle): ReadableStream => {
if (typeof file === 'string') {
@@ -20,7 +21,7 @@ export const readFileByLine: ((file: string | FileHandle) => AsyncIterable<strin
.pipeThrough(new TextDecoderStream())
.pipeThrough(new TextLineStream());
const ensureResponseBody = (resp: Response) => {
const ensureResponseBody = <T extends Response | NodeFetchResponse>(resp: T): NonNullable<T['body']> => {
if (!resp.body) {
throw new Error('Failed to fetch remote text');
}
@@ -30,12 +31,20 @@ const ensureResponseBody = (resp: Response) => {
return resp.body;
};
export const createReadlineInterfaceFromResponse: ((resp: Response) => AsyncIterable<string>) = (resp) => ensureResponseBody(resp)
.pipeThrough(new TextDecoderStream())
.pipeThrough(new TextLineStream());
export const createReadlineInterfaceFromResponse: ((resp: Response | NodeFetchResponse) => AsyncIterable<string>) = (resp) => {
const stream = ensureResponseBody(resp);
export function fetchRemoteTextByLine(url: string | URL) {
return fetchWithRetry(url, defaultRequestInit).then(createReadlineInterfaceFromResponse);
const webStream: ReadableStream<Uint8Array> = 'getReader' in stream
? stream
: Readable.toWeb(new Readable().wrap(stream)) as any;
return webStream
.pipeThrough(new TextDecoderStream())
.pipeThrough(new TextLineStream());
};
export function fetchRemoteTextByLine(url: string) {
return $fetch(url).then(createReadlineInterfaceFromResponse);
}
export async function readFileIntoProcessedArray(file: string | FileHandle) {