Feat: introduce make-fetch-happen (#44)

Build/lib/convert-clash-meta-mrs.ts

@@ -1,14 +1,13 @@
 import path from 'node:path';
 import fs from 'node:fs';
 import fsp from 'node:fs/promises';
-import { Readable } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
 import zlib from 'node:zlib';
 import process from 'node:process';
 
 import { async as ezspawn } from '@jsdevtools/ez-spawn';
 import { mkdirp } from './misc';
-import { fetchWithRetry } from './fetch-retry';
+import { $fetch } from './make-fetch-happen';
 
 const mihomoBinaryDir = path.join(__dirname, '../../node_modules/.cache/mihomo');
 const mihomoBinaryPath = path.join(mihomoBinaryDir, 'mihomo');
@@ -33,7 +32,7 @@ const ensureMihomoBinary = async () => {
       throw new Error(`Unsupported platform: ${process.platform} ${process.arch}`);
     }
 
-    const res = await fetchWithRetry(downloadUrl);
+    const res = await $fetch(downloadUrl);
 
     if (!res.ok || !res.body) {
       throw new Error(`Failed to download mihomo binary: ${res.statusText}`);
@@ -42,7 +41,7 @@ const ensureMihomoBinary = async () => {
     const gunzip = zlib.createGunzip();
 
     await pipeline(
-      Readable.fromWeb(res.body),
+      res.body,
       gunzip,
       writeStream
     );

Build/lib/fetch-text-by-line.ts

@@ -1,12 +1,13 @@
 import fs from 'node:fs';
 import { Readable } from 'node:stream';
-import { fetchWithRetry, defaultRequestInit } from './fetch-retry';
 import type { FileHandle } from 'node:fs/promises';
 
 import { TextLineStream } from './text-line-transform-stream';
 import type { ReadableStream } from 'node:stream/web';
 import { TextDecoderStream } from 'node:stream/web';
 import { processLine } from './process-line';
+import { $fetch } from './make-fetch-happen';
+import type { NodeFetchResponse } from './make-fetch-happen';
 
 const getReadableStream = (file: string | FileHandle): ReadableStream => {
   if (typeof file === 'string') {
@@ -20,7 +21,7 @@ export const readFileByLine: ((file: string | FileHandle) => AsyncIterable<strin
   .pipeThrough(new TextDecoderStream())
   .pipeThrough(new TextLineStream());
 
-const ensureResponseBody = (resp: Response) => {
+const ensureResponseBody = <T extends Response | NodeFetchResponse>(resp: T): NonNullable<T['body']> => {
   if (!resp.body) {
     throw new Error('Failed to fetch remote text');
   }
@@ -30,12 +31,20 @@ const ensureResponseBody = (resp: Response) => {
   return resp.body;
 };
 
-export const createReadlineInterfaceFromResponse: ((resp: Response) => AsyncIterable<string>) = (resp) => ensureResponseBody(resp)
-  .pipeThrough(new TextDecoderStream())
-  .pipeThrough(new TextLineStream());
+export const createReadlineInterfaceFromResponse: ((resp: Response | NodeFetchResponse) => AsyncIterable<string>) = (resp) => {
+  const stream = ensureResponseBody(resp);
 
-export function fetchRemoteTextByLine(url: string | URL) {
-  return fetchWithRetry(url, defaultRequestInit).then(createReadlineInterfaceFromResponse);
+  const webStream: ReadableStream<Uint8Array> = 'getReader' in stream
+    ? stream
+    : Readable.toWeb(new Readable().wrap(stream)) as any;
+
+  return webStream
+    .pipeThrough(new TextDecoderStream())
+    .pipeThrough(new TextLineStream());
+};
+
+export function fetchRemoteTextByLine(url: string) {
+  return $fetch(url).then(createReadlineInterfaceFromResponse);
 }
 
 export async function readFileIntoProcessedArray(file: string | FileHandle) {

Build/lib/make-fetch-happen.ts

@@ -0,0 +1,23 @@
+import path from 'node:path';
+import fs from 'node:fs';
+import makeFetchHappen from 'make-fetch-happen';
+// eslint-disable-next-line @typescript-eslint/no-restricted-imports -- type only
+export type { Response as NodeFetchResponse } from 'node-fetch';
+
+const cachePath = path.resolve(__dirname, '../../.cache/__make_fetch_happen__');
+fs.mkdirSync(cachePath, { recursive: true });
+
+export const $fetch = makeFetchHappen.defaults({
+  cachePath,
+  maxSockets: 32, /**
+   * They said 15 is a good default that prevents knocking out others' routers,
+   * I disagree. 32 is a good number.
+   */
+  headers: {
+    'User-Agent': 'curl/8.9.1 (https://github.com/SukkaW/Surge)'
+  },
+  retry: {
+    retries: 5,
+    randomize: true
+  }
+});

Build/lib/parse-dnsmasq.ts

@@ -1,6 +1,7 @@
 import { createReadlineInterfaceFromResponse } from './fetch-text-by-line';
 import { parse as tldtsParse } from 'tldts';
-import { fetchWithRetry, defaultRequestInit } from './fetch-retry';
+import { $fetch } from './make-fetch-happen';
+import type { NodeFetchResponse } from './make-fetch-happen';
 
 const isDomainLoose = (domain: string): boolean => {
   const { isIcann, isPrivate, isIp } = tldtsParse(domain);
@@ -14,7 +15,7 @@ export const extractDomainsFromFelixDnsmasq = (line: string): string | null => {
   return null;
 };
 
-export const parseFelixDnsmasqFromResp = async (resp: Response): Promise<string[]> => {
+export const parseFelixDnsmasqFromResp = async (resp: Response | NodeFetchResponse): Promise<string[]> => {
   const results: string[] = [];
 
   for await (const line of createReadlineInterfaceFromResponse(resp)) {
@@ -27,7 +28,7 @@ export const parseFelixDnsmasqFromResp = async (resp: Response): Promise<string[
   return results;
 };
 
-export const parseFelixDnsmasq = async (url: string | URL): Promise<string[]> => {
-  const resp = await fetchWithRetry(url, defaultRequestInit);
+export const parseFelixDnsmasq = async (url: string): Promise<string[]> => {
+  const resp = await $fetch(url);
   return parseFelixDnsmasqFromResp(resp);
 };