diff --git a/Build/download-previous-build.ts b/Build/download-previous-build.ts index 83a5b3a4..6f372d52 100644 --- a/Build/download-previous-build.ts +++ b/Build/download-previous-build.ts @@ -6,7 +6,8 @@ import { task } from './trace'; import { extract as tarExtract } from 'tar-fs'; import type { Headers as TarEntryHeaders } from 'tar-fs'; import zlib from 'node:zlib'; -import { $fetch } from './lib/make-fetch-happen'; +import { fetchWithRetry } from './lib/fetch-retry'; +import { Readable } from 'node:stream'; const GITHUB_CODELOAD_URL = 'https://codeload.github.com/sukkalab/ruleset.skk.moe/tar.gz/master'; const GITLAB_CODELOAD_URL = 'https://gitlab.com/SukkaW/ruleset.skk.moe/-/archive/master/ruleset.skk.moe-master.tar.gz'; @@ -20,7 +21,7 @@ export const downloadPreviousBuild = task(require.main === module, __filename)(a } const tarGzUrl = await span.traceChildAsync('get tar.gz url', async () => { - const resp = await $fetch(GITHUB_CODELOAD_URL, { method: 'HEAD' }); + const resp = await fetchWithRetry(GITHUB_CODELOAD_URL, { method: 'HEAD' }); if (resp.status !== 200) { console.warn('Download previous build from GitHub failed! Status:', resp.status); console.warn('Switch to GitLab'); @@ -30,7 +31,7 @@ export const downloadPreviousBuild = task(require.main === module, __filename)(a }); return span.traceChildAsync('download & extract previoud build', async () => { - const resp = await $fetch(tarGzUrl, { + const resp = await fetchWithRetry(tarGzUrl, { headers: { 'User-Agent': 'curl/8.9.1', // https://github.com/unjs/giget/issues/97 @@ -66,7 +67,7 @@ export const downloadPreviousBuild = task(require.main === module, __filename)(a ); return pipeline( - resp.body, + Readable.fromWeb(resp.body), gunzip, extract ); diff --git a/Build/lib/fetch-retry.ts b/Build/lib/fetch-retry.ts index 8c6471c7..fa7219ca 100644 --- a/Build/lib/fetch-retry.ts +++ b/Build/lib/fetch-retry.ts @@ -1,14 +1,46 @@ import retry from 'async-retry'; import picocolors from 'picocolors'; import { setTimeout } from 'node:timers/promises'; -import { fetch as _fetch } from 'undici'; +import { + fetch as _fetch, + interceptors, + EnvHttpProxyAgent, + setGlobalDispatcher +} from 'undici'; + +import type { Request, Response, RequestInit } from 'undici'; + +import CacheableLookup from 'cacheable-lookup'; +import type { LookupOptions as CacheableLookupOptions } from 'cacheable-lookup'; + +const cacheableLookup = new CacheableLookup(); + +const agent = new EnvHttpProxyAgent({ + allowH2: true, + connect: { + lookup(hostname, opt, cb) { + return cacheableLookup.lookup(hostname, opt as CacheableLookupOptions, cb); + } + } +}); + +setGlobalDispatcher(agent.compose( + interceptors.retry({ + maxRetries: 5, + minTimeout: 10000, + errorCodes: ['UND_ERR_HEADERS_TIMEOUT', 'ECONNRESET', 'ECONNREFUSED', 'ENOTFOUND', 'ENETDOWN', 'ENETUNREACH', 'EHOSTDOWN', 'EHOSTUNREACH', 'EPIPE'] + }), + interceptors.redirect({ + maxRedirections: 5 + }) +)); function isClientError(err: unknown): err is NodeJS.ErrnoException { if (!err || typeof err !== 'object') return false; if ('code' in err) return err.code === 'ERR_UNESCAPED_CHARACTERS'; if ('message' in err) return err.message === 'Request path contains unescaped characters'; - if ('name' in err) return err.name === 'DOMException' || err.name === 'AbortError'; + if ('name' in err) return err.name === 'AbortError'; return false; } @@ -41,7 +73,6 @@ interface FetchRetryOpt { factor?: number, maxRetryAfter?: number, // onRetry?: (err: Error) => void, - retryOnAborted?: boolean, retryOnNon2xx?: boolean, retryOn404?: boolean } @@ -57,12 +88,11 @@ const DEFAULT_OPT: Required = { retries: 5, factor: 6, maxRetryAfter: 20, - retryOnAborted: false, retryOnNon2xx: true, retryOn404: false }; -function createFetchRetry($fetch: typeof fetch): FetchWithRetry { +function createFetchRetry(fetch: typeof _fetch): FetchWithRetry { const fetchRetry: FetchWithRetry = async (url, opts = {}) => { const retryOpts = Object.assign( DEFAULT_OPT, @@ -70,10 +100,10 @@ function createFetchRetry($fetch: typeof fetch): FetchWithRetry { ); try { - return await retry(async (bail) => { + return await retry(async (bail) => { try { // this will be retried - const res = (await $fetch(url, opts)); + const res = (await fetch(url, opts)); if ((res.status >= 500 && res.status < 600) || res.status === 429) { // NOTE: doesn't support http-date format @@ -126,7 +156,7 @@ function createFetchRetry($fetch: typeof fetch): FetchWithRetry { if (( err.name === 'AbortError' || ('digest' in err && err.digest === 'AbortError') - ) && !retryOpts.retryOnAborted) { + )) { console.log(picocolors.gray('[fetch abort]'), url); return true; } @@ -148,9 +178,9 @@ function createFetchRetry($fetch: typeof fetch): FetchWithRetry { } }; - for (const k of Object.keys($fetch)) { - const key = k as keyof typeof $fetch; - fetchRetry[key] = $fetch[key]; + for (const k of Object.keys(_fetch)) { + const key = k as keyof typeof _fetch; + fetchRetry[key] = _fetch[key]; } return fetchRetry; diff --git a/Build/lib/make-fetch-happen.ts b/Build/lib/make-fetch-happen.ts index 5e63223c..0d11480b 100644 --- a/Build/lib/make-fetch-happen.ts +++ b/Build/lib/make-fetch-happen.ts @@ -1,8 +1,11 @@ import path from 'node:path'; import fs from 'node:fs'; import makeFetchHappen from 'make-fetch-happen'; +import picocolors from 'picocolors'; // eslint-disable-next-line @typescript-eslint/no-restricted-imports -- type only -export type { Response as NodeFetchResponse } from 'node-fetch'; +import type { Response as NodeFetchResponse } from 'node-fetch'; + +export type { NodeFetchResponse }; const cachePath = path.resolve(__dirname, '../../.cache/__make_fetch_happen__'); fs.mkdirSync(cachePath, { recursive: true }); @@ -21,3 +24,10 @@ export const $fetch = makeFetchHappen.defaults({ randomize: true } }); + +export function printResponseStatus(resp: NodeFetchResponse) { + const status = resp.headers.get('X-Local-Cache-Status'); + if (status) { + console.log('[$fetch cache]', { status }, picocolors.gray(resp.url)); + } +} diff --git a/package.json b/package.json index e3a25a4e..60de942b 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "async-retry": "^1.3.3", "async-sema": "^3.1.1", "better-sqlite3": "^11.3.0", + "cacheable-lookup": "^6.1.0", "ci-info": "^4.0.0", "cli-table3": "^0.6.5", "csv-parse": "^5.5.6", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 219b0dbf..fb0175b5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -26,6 +26,9 @@ importers: better-sqlite3: specifier: ^11.3.0 version: 11.3.0 + cacheable-lookup: + specifier: ^6.1.0 + version: 6.1.0 ci-info: specifier: ^4.0.0 version: 4.0.0 @@ -730,6 +733,10 @@ packages: resolution: {integrity: sha512-hdsUxulXCi5STId78vRVYEtDAjq99ICAUktLTeTYsLoTE6Z8dS0c8pWNCxwdrk9YfJeobDZc2Y186hD/5ZQgFQ==} engines: {node: ^18.17.0 || >=20.5.0} + cacheable-lookup@6.1.0: + resolution: {integrity: sha512-KJ/Dmo1lDDhmW2XDPMo+9oiy/CeqosPguPCrgcVzKyZrL6pM1gU2GmPY/xo6OQPTUaA/c0kwHuywB4E6nmT9ww==} + engines: {node: '>=10.6.0'} + call-me-maybe@1.0.2: resolution: {integrity: sha512-HpX65o1Hnr9HH25ojC1YGs7HCQLq0GCOibSaWER0eNpgJ/Z1MZv2mTc7+xh6WOPxbRVcmgbv4hGU+uSQ/2xFZQ==} @@ -2502,6 +2509,8 @@ snapshots: tar: 7.4.3 unique-filename: 4.0.0 + cacheable-lookup@6.1.0: {} + call-me-maybe@1.0.2: {} callsites@3.1.0: {}