Enable more feature w/ undici

Build/download-previous-build.ts

@@ -6,7 +6,8 @@ import { task } from './trace';
 import { extract as tarExtract } from 'tar-fs';
 import type { Headers as TarEntryHeaders } from 'tar-fs';
 import zlib from 'node:zlib';
-import { $fetch } from './lib/make-fetch-happen';
+import { fetchWithRetry } from './lib/fetch-retry';
+import { Readable } from 'node:stream';
 
 const GITHUB_CODELOAD_URL = 'https://codeload.github.com/sukkalab/ruleset.skk.moe/tar.gz/master';
 const GITLAB_CODELOAD_URL = 'https://gitlab.com/SukkaW/ruleset.skk.moe/-/archive/master/ruleset.skk.moe-master.tar.gz';
@@ -20,7 +21,7 @@ export const downloadPreviousBuild = task(require.main === module, __filename)(a
   }
 
   const tarGzUrl = await span.traceChildAsync('get tar.gz url', async () => {
-    const resp = await $fetch(GITHUB_CODELOAD_URL, { method: 'HEAD' });
+    const resp = await fetchWithRetry(GITHUB_CODELOAD_URL, { method: 'HEAD' });
     if (resp.status !== 200) {
       console.warn('Download previous build from GitHub failed! Status:', resp.status);
       console.warn('Switch to GitLab');
@@ -30,7 +31,7 @@ export const downloadPreviousBuild = task(require.main === module, __filename)(a
   });
 
   return span.traceChildAsync('download & extract previoud build', async () => {
-    const resp = await $fetch(tarGzUrl, {
+    const resp = await fetchWithRetry(tarGzUrl, {
       headers: {
         'User-Agent': 'curl/8.9.1',
         // https://github.com/unjs/giget/issues/97
@@ -66,7 +67,7 @@ export const downloadPreviousBuild = task(require.main === module, __filename)(a
     );
 
     return pipeline(
-      resp.body,
+      Readable.fromWeb(resp.body),
       gunzip,
       extract
     );

Build/lib/fetch-retry.ts

@@ -1,14 +1,46 @@
 import retry from 'async-retry';
 import picocolors from 'picocolors';
 import { setTimeout } from 'node:timers/promises';
-import { fetch as _fetch } from 'undici';
+import {
+  fetch as _fetch,
+  interceptors,
+  EnvHttpProxyAgent,
+  setGlobalDispatcher
+} from 'undici';
+
+import type { Request, Response, RequestInit } from 'undici';
+
+import CacheableLookup from 'cacheable-lookup';
+import type { LookupOptions as CacheableLookupOptions } from 'cacheable-lookup';
+
+const cacheableLookup = new CacheableLookup();
+
+const agent = new EnvHttpProxyAgent({
+  allowH2: true,
+  connect: {
+    lookup(hostname, opt, cb) {
+      return cacheableLookup.lookup(hostname, opt as CacheableLookupOptions, cb);
+    }
+  }
+});
+
+setGlobalDispatcher(agent.compose(
+  interceptors.retry({
+    maxRetries: 5,
+    minTimeout: 10000,
+    errorCodes: ['UND_ERR_HEADERS_TIMEOUT', 'ECONNRESET', 'ECONNREFUSED', 'ENOTFOUND', 'ENETDOWN', 'ENETUNREACH', 'EHOSTDOWN', 'EHOSTUNREACH', 'EPIPE']
+  }),
+  interceptors.redirect({
+    maxRedirections: 5
+  })
+));
 
 function isClientError(err: unknown): err is NodeJS.ErrnoException {
   if (!err || typeof err !== 'object') return false;
 
   if ('code' in err) return err.code === 'ERR_UNESCAPED_CHARACTERS';
   if ('message' in err) return err.message === 'Request path contains unescaped characters';
-  if ('name' in err) return err.name === 'DOMException' || err.name === 'AbortError';
+  if ('name' in err) return err.name === 'AbortError';
 
   return false;
 }
@@ -41,7 +73,6 @@ interface FetchRetryOpt {
   factor?: number,
   maxRetryAfter?: number,
   // onRetry?: (err: Error) => void,
-  retryOnAborted?: boolean,
   retryOnNon2xx?: boolean,
   retryOn404?: boolean
 }
@@ -57,12 +88,11 @@ const DEFAULT_OPT: Required<FetchRetryOpt> = {
   retries: 5,
   factor: 6,
   maxRetryAfter: 20,
-  retryOnAborted: false,
   retryOnNon2xx: true,
   retryOn404: false
 };
 
-function createFetchRetry($fetch: typeof fetch): FetchWithRetry {
+function createFetchRetry(fetch: typeof _fetch): FetchWithRetry {
   const fetchRetry: FetchWithRetry = async (url, opts = {}) => {
     const retryOpts = Object.assign(
       DEFAULT_OPT,
@@ -70,10 +100,10 @@ function createFetchRetry($fetch: typeof fetch): FetchWithRetry {
     );
 
     try {
-      return await retry<Response>(async (bail) => {
+      return await retry(async (bail) => {
         try {
           // this will be retried
-          const res = (await $fetch(url, opts));
+          const res = (await fetch(url, opts));
 
           if ((res.status >= 500 && res.status < 600) || res.status === 429) {
             // NOTE: doesn't support http-date format
@@ -126,7 +156,7 @@ function createFetchRetry($fetch: typeof fetch): FetchWithRetry {
           if ((
             err.name === 'AbortError'
             || ('digest' in err && err.digest === 'AbortError')
-          ) && !retryOpts.retryOnAborted) {
+          )) {
             console.log(picocolors.gray('[fetch abort]'), url);
             return true;
           }
@@ -148,9 +178,9 @@ function createFetchRetry($fetch: typeof fetch): FetchWithRetry {
     }
   };
 
-  for (const k of Object.keys($fetch)) {
+  for (const k of Object.keys(_fetch)) {
-    const key = k as keyof typeof $fetch;
+    const key = k as keyof typeof _fetch;
-    fetchRetry[key] = $fetch[key];
+    fetchRetry[key] = _fetch[key];
   }
 
   return fetchRetry;

Build/lib/make-fetch-happen.ts

@@ -1,8 +1,11 @@
 import path from 'node:path';
 import fs from 'node:fs';
 import makeFetchHappen from 'make-fetch-happen';
+import picocolors from 'picocolors';
 // eslint-disable-next-line @typescript-eslint/no-restricted-imports -- type only
-export type { Response as NodeFetchResponse } from 'node-fetch';
+import type { Response as NodeFetchResponse } from 'node-fetch';
+
+export type { NodeFetchResponse };
 
 const cachePath = path.resolve(__dirname, '../../.cache/__make_fetch_happen__');
 fs.mkdirSync(cachePath, { recursive: true });
@@ -21,3 +24,10 @@ export const $fetch = makeFetchHappen.defaults({
     randomize: true
   }
 });
+
+export function printResponseStatus(resp: NodeFetchResponse) {
+  const status = resp.headers.get('X-Local-Cache-Status');
+  if (status) {
+    console.log('[$fetch cache]', { status }, picocolors.gray(resp.url));
+  }
+}

package.json

@@ -25,6 +25,7 @@
     "async-retry": "^1.3.3",
     "async-sema": "^3.1.1",
     "better-sqlite3": "^11.3.0",
+    "cacheable-lookup": "^6.1.0",
     "ci-info": "^4.0.0",
     "cli-table3": "^0.6.5",
     "csv-parse": "^5.5.6",

pnpm-lock.yaml

@@ -26,6 +26,9 @@ importers:
       better-sqlite3:
         specifier: ^11.3.0
         version: 11.3.0
+      cacheable-lookup:
+        specifier: ^6.1.0
+        version: 6.1.0
       ci-info:
         specifier: ^4.0.0
         version: 4.0.0
@@ -730,6 +733,10 @@ packages:
     resolution: {integrity: sha512-hdsUxulXCi5STId78vRVYEtDAjq99ICAUktLTeTYsLoTE6Z8dS0c8pWNCxwdrk9YfJeobDZc2Y186hD/5ZQgFQ==}
     engines: {node: ^18.17.0 || >=20.5.0}
 
+  cacheable-lookup@6.1.0:
+    resolution: {integrity: sha512-KJ/Dmo1lDDhmW2XDPMo+9oiy/CeqosPguPCrgcVzKyZrL6pM1gU2GmPY/xo6OQPTUaA/c0kwHuywB4E6nmT9ww==}
+    engines: {node: '>=10.6.0'}
+
   call-me-maybe@1.0.2:
     resolution: {integrity: sha512-HpX65o1Hnr9HH25ojC1YGs7HCQLq0GCOibSaWER0eNpgJ/Z1MZv2mTc7+xh6WOPxbRVcmgbv4hGU+uSQ/2xFZQ==}
 
@@ -2502,6 +2509,8 @@ snapshots:
       tar: 7.4.3
       unique-filename: 4.0.0
 
+  cacheable-lookup@6.1.0: {}
+
   call-me-maybe@1.0.2: {}
 
   callsites@3.1.0: {}