GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2026-01-28 17:41:54 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Refactor: use retry from undici

Browse Source
This commit is contained in:
SukkaW
2024-10-13 15:43:33 +08:00
parent 2695de7410
commit 79da4e18fc
7 changed files with 151 additions and 201 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
Build/download-previous-build.ts
View File

@@ -6,7 +6,7 @@ import { task } from './trace';
import { extract as tarExtract } from 'tar-fs';
import type { Headers as TarEntryHeaders } from 'tar-fs';
import zlib from 'node:zlib';
import { fetchWithRetry } from './lib/fetch-retry';
import { fetchWithLog } from './lib/fetch-retry';
import { Readable } from 'node:stream';
const GITHUB_CODELOAD_URL = 'https://codeload.github.com/sukkalab/ruleset.skk.moe/tar.gz/master';
@@ -21,7 +21,7 @@ export const downloadPreviousBuild = task(require.main === module, __filename)(a
}
const tarGzUrl = await span.traceChildAsync('get tar.gz url', async () => {
const resp = await fetchWithRetry(GITHUB_CODELOAD_URL, { method: 'HEAD' });
const resp = await fetchWithLog(GITHUB_CODELOAD_URL, { method: 'HEAD' });
if (resp.status !== 200) {
console.warn('Download previous build from GitHub failed! Status:', resp.status);
console.warn('Switch to GitLab');
@@ -31,7 +31,7 @@ export const downloadPreviousBuild = task(require.main === module, __filename)(a
});
return span.traceChildAsync('download & extract previoud build', async () => {
const resp = await fetchWithRetry(tarGzUrl, {
const resp = await fetchWithLog(tarGzUrl, {
headers: {
'User-Agent': 'curl/8.9.1',
// https://github.com/unjs/giget/issues/97

12
Build/lib/cache-filesystem.ts
View File

@@ -8,9 +8,11 @@ import { fastStringArrayJoin, identity, mergeHeaders } from './misc';
import { performance } from 'node:perf_hooks';
import fs from 'node:fs';
import { stringHash } from './string-hash';
import { defaultRequestInit, fetchWithRetry } from './fetch-retry';
import { defaultRequestInit, fetchWithLog } from './fetch-retry';
import { Custom304NotModifiedError, CustomAbortError, CustomNoETagFallbackError, fetchAssets, sleepWithAbort } from './fetch-assets';
import type { Response, RequestInit } from 'undici';
const enum CacheStatus {
Hit = 'hit',
Stale = 'stale',
@@ -216,7 +218,7 @@ export class Cache<S = string> {
requestInit?: RequestInit
): Promise<T> {
if (opt.temporaryBypass) {
return fn(await fetchWithRetry(url, requestInit ?? defaultRequestInit));
return fn(await fetchWithLog(url, requestInit));
}
const baseKey = url + '$' + extraCacheKey;
@@ -255,10 +257,10 @@ export class Cache<S = string> {
const cached = this.get(cachedKey);
if (cached == null) {
return onMiss(await fetchWithRetry(url, requestInit ?? defaultRequestInit));
return onMiss(await fetchWithLog(url, requestInit));
}
const resp = await fetchWithRetry(
const resp = await fetchWithLog(
url,
{
...(requestInit ?? defaultRequestInit),
@@ -321,7 +323,7 @@ export class Cache<S = string> {
}
const etag = this.get(getETagKey(url));
const res = await fetchWithRetry(
const res = await fetchWithLog(
url,
{
signal: controller.signal,

4
Build/lib/fetch-assets.ts
View File

@@ -1,5 +1,5 @@
import picocolors from 'picocolors';
import { defaultRequestInit, fetchWithRetry } from './fetch-retry';
import { defaultRequestInit, fetchWithLog } from './fetch-retry';
import { setTimeout } from 'node:timers/promises';
// eslint-disable-next-line sukka/unicorn/custom-error-definition -- typescript is better
@@ -59,7 +59,7 @@ export async function fetchAssets(url: string, fallbackUrls: string[] | readonly
console.log(picocolors.gray('[fetch cancelled]'), picocolors.gray(url));
throw new CustomAbortError();
}
const res = await fetchWithRetry(url, { signal: controller.signal, ...defaultRequestInit });
const res = await fetchWithLog(url, { signal: controller.signal, ...defaultRequestInit });
const text = await res.text();
controller.abort();
return text;

243
Build/lib/fetch-retry.ts
View File

@@ -1,14 +1,12 @@
import retry from 'async-retry';
import picocolors from 'picocolors';
import { setTimeout } from 'node:timers/promises';
import {
fetch as _fetch,
fetch,
interceptors,
EnvHttpProxyAgent,
setGlobalDispatcher
} from 'undici';
import type { Request, Response, RequestInit } from 'undici';
import type { Response, RequestInit, RequestInfo } from 'undici';
import CacheableLookup from 'cacheable-lookup';
import type { LookupOptions as CacheableLookupOptions } from 'cacheable-lookup';
@@ -16,7 +14,7 @@ import type { LookupOptions as CacheableLookupOptions } from 'cacheable-lookup';
const cacheableLookup = new CacheableLookup();
const agent = new EnvHttpProxyAgent({
allowH2: true,
// allowH2: true,
connect: {
lookup(hostname, opt, cb) {
return cacheableLookup.lookup(hostname, opt as CacheableLookupOptions, cb);
@@ -28,21 +26,89 @@ setGlobalDispatcher(agent.compose(
interceptors.retry({
maxRetries: 5,
minTimeout: 10000,
errorCodes: ['UND_ERR_HEADERS_TIMEOUT', 'ECONNRESET', 'ECONNREFUSED', 'ENOTFOUND', 'ENETDOWN', 'ENETUNREACH', 'EHOSTDOWN', 'EHOSTUNREACH', 'EPIPE', 'ETIMEDOUT']
// TODO: this part of code is only for allow more errors to be retried by default
// This should be removed once https://github.com/nodejs/undici/issues/3728 is implemented
// @ts-expect-error -- retry return type should be void
retry(err, { state, opts }, cb) {
const statusCode = 'statusCode' in err && typeof err.statusCode === 'number' ? err.statusCode : null;
const errorCode = 'code' in err ? (err as NodeJS.ErrnoException).code : undefined;
const headers = ('headers' in err && typeof err.headers === 'object') ? err.headers : undefined;
const { counter } = state;
// Any code that is not a Undici's originated and allowed to retry
if (
errorCode === 'ERR_UNESCAPED_CHARACTERS'
|| err.message === 'Request path contains unescaped characters'
|| err.name === 'AbortError'
) {
return cb(err);
}
if (errorCode !== 'UND_ERR_REQ_RETRY') {
return cb(err);
}
const { method, retryOptions = {} } = opts;
const {
maxRetries = 5,
minTimeout = 500,
maxTimeout = 30 * 1000,
timeoutFactor = 2,
methods = ['GET', 'HEAD', 'OPTIONS', 'PUT', 'DELETE', 'TRACE']
} = retryOptions;
// If we reached the max number of retries
if (counter > maxRetries) {
return cb(err);
}
// If a set of method are provided and the current method is not in the list
if (Array.isArray(methods) && !methods.includes(method)) {
return cb(err);
}
// bail out if the status code matches one of the following
if (
statusCode != null
&& (
statusCode === 401 // Unauthorized, should check credentials instead of retrying
|| statusCode === 403 // Forbidden, should check permissions instead of retrying
|| statusCode === 404 // Not Found, should check URL instead of retrying
|| statusCode === 405 // Method Not Allowed, should check method instead of retrying
)
) {
return cb(err);
}
const retryAfterHeader = (headers as Record<string, string> | null | undefined)?.['retry-after'];
let retryAfter = -1;
if (retryAfterHeader) {
retryAfter = Number(retryAfterHeader);
retryAfter = Number.isNaN(retryAfter)
? calculateRetryAfterHeader(retryAfterHeader)
: retryAfter * 1e3; // Retry-After is in seconds
}
const retryTimeout
= retryAfter > 0
? Math.min(retryAfter, maxTimeout)
: Math.min(minTimeout * (timeoutFactor ** (counter - 1)), maxTimeout);
// eslint-disable-next-line sukka/prefer-timer-id -- won't leak
setTimeout(() => cb(null), retryTimeout);
}
// errorCodes: ['UND_ERR_HEADERS_TIMEOUT', 'ECONNRESET', 'ECONNREFUSED', 'ENOTFOUND', 'ENETDOWN', 'ENETUNREACH', 'EHOSTDOWN', 'EHOSTUNREACH', 'EPIPE', 'ETIMEDOUT']
}),
interceptors.redirect({
maxRedirections: 5
})
));
function isClientError(err: unknown): err is NodeJS.ErrnoException {
if (!err || typeof err !== 'object') return false;
if ('code' in err) return err.code === 'ERR_UNESCAPED_CHARACTERS';
if ('message' in err) return err.message === 'Request path contains unescaped characters';
if ('name' in err) return err.name === 'AbortError';
return false;
function calculateRetryAfterHeader(retryAfter: string) {
const current = Date.now();
return new Date(retryAfter).getTime() - current;
}
export class ResponseError extends Error {
@@ -67,129 +133,38 @@ export class ResponseError extends Error {
}
}
interface FetchRetryOpt {
minTimeout?: number,
retries?: number,
factor?: number,
maxRetryAfter?: number,
// onRetry?: (err: Error) => void,
retryOnNon2xx?: boolean,
retryOn404?: boolean
}
interface FetchWithRetry {
(url: string | URL | Request, opts?: RequestInit & { retry?: FetchRetryOpt }): Promise<Response>
}
const DEFAULT_OPT: Required<FetchRetryOpt> = {
// timeouts will be [10, 60, 360, 2160, 12960]
// (before randomization is added)
minTimeout: 10,
retries: 5,
factor: 6,
maxRetryAfter: 20,
retryOnNon2xx: true,
retryOn404: false
};
function createFetchRetry(fetch: typeof _fetch): FetchWithRetry {
const fetchRetry: FetchWithRetry = async (url, opts = {}) => {
const retryOpts = Object.assign(
DEFAULT_OPT,
opts.retry
);
try {
return await retry(async (bail) => {
try {
// this will be retried
const res = (await fetch(url, opts));
if ((res.status >= 500 && res.status < 600) || res.status === 429) {
// NOTE: doesn't support http-date format
const retryAfterHeader = res.headers.get('retry-after');
if (retryAfterHeader) {
const retryAfter = Number.parseInt(retryAfterHeader, 10);
if (retryAfter) {
if (retryAfter > retryOpts.maxRetryAfter) {
return res;
}
await setTimeout(retryAfter * 1e3, undefined, { ref: false });
}
}
throw new ResponseError(res);
} else {
if ((!res.ok && res.status !== 304) && retryOpts.retryOnNon2xx) {
throw new ResponseError(res);
}
return res;
}
} catch (err: unknown) {
if (mayBailError(err)) {
return bail(err) as never;
};
if (err instanceof AggregateError) {
for (const e of err.errors) {
if (mayBailError(e)) {
// bail original error
return bail(err) as never;
};
}
}
console.log(picocolors.gray('[fetch fail]'), url, { name: (err as any).name }, err);
// Do not retry on 404
if (err instanceof ResponseError && err.res.status === 404) {
return bail(err) as never;
}
const newErr = new Error('Fetch failed');
newErr.cause = err;
throw newErr;
}
}, retryOpts);
function mayBailError(err: unknown) {
if (typeof err === 'object' && err !== null && 'name' in err) {
if ((
err.name === 'AbortError'
|| ('digest' in err && err.digest === 'AbortError')
)) {
console.log(picocolors.gray('[fetch abort]'), url);
return true;
}
if (err.name === 'Custom304NotModifiedError') {
return true;
}
if (err.name === 'CustomNoETagFallbackError') {
return true;
}
}
return !!(isClientError(err));
};
} catch (err) {
if (err instanceof ResponseError) {
return err.res;
}
throw err;
}
};
for (const k of Object.keys(_fetch)) {
const key = k as keyof typeof _fetch;
fetchRetry[key] = _fetch[key];
}
return fetchRetry;
}
export const defaultRequestInit: RequestInit = {
headers: {
'User-Agent': 'curl/8.9.1 (https://github.com/SukkaW/Surge)'
}
};
export const fetchWithRetry = createFetchRetry(_fetch as any);
export async function fetchWithLog(url: RequestInfo, opts: RequestInit = defaultRequestInit) {
try {
// this will be retried
const res = (await fetch(url, opts));
if (res.status >= 400) {
throw new ResponseError(res);
}
if (!res.ok && res.status !== 304) {
throw new ResponseError(res);
}
return res;
} catch (err: unknown) {
if (typeof err === 'object' && err !== null && 'name' in err) {
if ((
err.name === 'AbortError'
|| ('digest' in err && err.digest === 'AbortError')
)) {
console.log(picocolors.gray('[fetch abort]'), url);
}
} else {
console.log(picocolors.gray('[fetch fail]'), url, { name: (err as any).name }, err);
}
throw err;
}
};

11
Build/lib/misc.ts
View File

@@ -2,6 +2,7 @@ import path, { dirname } from 'node:path';
import fs from 'node:fs';
import fsp from 'node:fs/promises';
import { OUTPUT_CLASH_DIR, OUTPUT_SINGBOX_DIR, OUTPUT_SURGE_DIR } from '../constants/dir';
import type { HeadersInit } from 'undici';
export const isTruthy = <T>(i: T | 0 | '' | false | null | undefined): i is T => !!i;
@@ -102,7 +103,7 @@ export function withBannerArray(title: string, description: string[] | readonly
];
};
export function mergeHeaders(headersA: RequestInit['headers'] | undefined, headersB: RequestInit['headers']) {
export function mergeHeaders<T extends RequestInit['headers'] | HeadersInit>(headersA: T | undefined, headersB: T): T {
if (headersA == null) {
return headersB;
}
@@ -111,20 +112,20 @@ export function mergeHeaders(headersA: RequestInit['headers'] | undefined, heade
throw new TypeError('Array headers is not supported');
}
const result = new Headers(headersA);
const result = new Headers(headersA as any);
if (headersB instanceof Headers) {
headersB.forEach((value, key) => {
result.set(key, value);
});
return result;
return result as T;
}
for (const key in headersB) {
if (Object.hasOwn(headersB, key)) {
result.set(key, (headersB)[key]);
result.set(key, (headersB as Record<string, any>)[key]);
}
}
return result;
return result as T;
}

8
package.json
View File

@@ -22,7 +22,6 @@
"dependencies": {
"@cliqz/adblocker": "^1.33.2",
"@jsdevtools/ez-spawn": "^3.0.4",
"async-retry": "^1.3.3",
"async-sema": "^3.1.1",
"better-sqlite3": "^11.3.0",
"cacache": "^19.0.1",
@@ -32,7 +31,7 @@
"csv-parse": "^5.5.6",
"fast-cidr-tools": "^0.3.1",
"fdir": "^6.4.0",
"foxact": "^0.2.38",
"foxact": "^0.2.39",
"hash-wasm": "^4.11.0",
"json-stringify-pretty-compact": "^3.0.0",
"make-fetch-happen": "^14.0.1",
@@ -40,8 +39,8 @@
"picocolors": "^1.1.0",
"punycode": "^2.3.1",
"tar-fs": "^3.0.6",
"tldts": "^6.1.50",
"tldts-experimental": "^6.1.50",
"tldts": "^6.1.51",
"tldts-experimental": "^6.1.51",
"undici": "^6.20.0",
"wtfnode": "^0.9.3",
"yaml": "^2.5.1"
@@ -50,7 +49,6 @@
"@eslint-sukka/node": "^6.7.0",
"@swc-node/register": "^1.10.9",
"@swc/core": "^1.7.35",
"@types/async-retry": "^1.4.9",
"@types/better-sqlite3": "^7.6.11",
"@types/cacache": "^17.0.2",
"@types/chai": "^4.3.20",

68
pnpm-lock.yaml generated
View File

@@ -17,9 +17,6 @@ importers:
'@jsdevtools/ez-spawn':
specifier: ^3.0.4
version: 3.0.4
async-retry:
specifier: ^1.3.3
version: 1.3.3
async-sema:
specifier: ^3.1.1
version: 3.1.1
@@ -48,8 +45,8 @@ importers:
specifier: ^6.4.0
version: 6.4.0(picomatch@4.0.2)
foxact:
specifier: ^0.2.38
version: 0.2.38
specifier: ^0.2.39
version: 0.2.39
hash-wasm:
specifier: ^4.11.0
version: 4.11.0
@@ -72,11 +69,11 @@ importers:
specifier: ^3.0.6
version: 3.0.6
tldts:
specifier: ^6.1.50
version: 6.1.50
specifier: ^6.1.51
version: 6.1.51
tldts-experimental:
specifier: ^6.1.50
version: 6.1.50
specifier: ^6.1.51
version: 6.1.51
undici:
specifier: ^6.20.0
version: 6.20.0
@@ -96,9 +93,6 @@ importers:
'@swc/core':
specifier: ^1.7.35
version: 1.7.35
'@types/async-retry':
specifier: ^1.4.9
version: 1.4.9
'@types/better-sqlite3':
specifier: ^7.6.11
version: 7.6.11
@@ -472,9 +466,6 @@ packages:
'@tybys/wasm-util@0.9.0':
resolution: {integrity: sha512-6+7nlbMVX/PVDCwaIQ8nTOPveOcFLSt8GcXdx8hD0bt39uWxYT88uXzqTd4fTvqta7oeUJqudepapKNt2DYJFw==}
'@types/async-retry@1.4.9':
resolution: {integrity: sha512-s1ciZQJzRh3708X/m3vPExr5KJlzlZJvXsKpbtE2luqNcbROr64qU+3KpJsYHqWMeaxI839OvXf9PrUSw1Xtyg==}
'@types/better-sqlite3@7.6.11':
resolution: {integrity: sha512-i8KcD3PgGtGBLl3+mMYA8PdKkButvPyARxA7IQAd6qeslht13qxb1zzO8dRCtE7U3IoJS782zDBAeoKiM695kg==}
@@ -673,9 +664,6 @@ packages:
assertion-error@1.1.0:
resolution: {integrity: sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==}
async-retry@1.3.3:
resolution: {integrity: sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==}
async-sema@3.1.1:
resolution: {integrity: sha512-tLRNUXati5MFePdAk8dw7Qt7DpxPB60ofAgn8WRhW6a2rcimZnYBP9oxHiv0OHy+Wz7kPMG+t4LGdt31+4EmGg==}
@@ -1107,8 +1095,8 @@ packages:
resolution: {integrity: sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==}
engines: {node: '>= 6'}
foxact@0.2.38:
resolution: {integrity: sha512-ClxI9lwzhWpE/JIGfPjSpUNqG6MccNq60jrxuPidNl4CAUrATba4ViQTBFn1Zc5+9q9nAFXWaZKendXIbGvrvQ==}
foxact@0.2.39:
resolution: {integrity: sha512-iIe0eakDQuGL5ArCVzijffkSAm6jNGC3apTkUWBarvnIZuX6tmx/nhXYFNirKG4Vxo+fM3sL6GP36BE/3w4xng==}
peerDependencies:
react: '*'
peerDependenciesMeta:
@@ -1572,10 +1560,6 @@ packages:
resolution: {integrity: sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==}
engines: {node: '>= 4'}
retry@0.13.1:
resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==}
engines: {node: '>= 4'}
reusify@1.0.4:
resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==}
engines: {iojs: '>=1.0.0', node: '>=0.10.0'}
@@ -1736,14 +1720,14 @@ packages:
text-table@0.2.0:
resolution: {integrity: sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==}
tldts-core@6.1.50:
resolution: {integrity: sha512-na2EcZqmdA2iV9zHV7OHQDxxdciEpxrjbkp+aHmZgnZKHzoElLajP59np5/4+sare9fQBfixgvXKx8ev1d7ytw==}
tldts-core@6.1.51:
resolution: {integrity: sha512-bu9oCYYWC1iRjx+3UnAjqCsfrWNZV1ghNQf49b3w5xE8J/tNShHTzp5syWJfwGH+pxUgTTLUnzHnfuydW7wmbg==}
tldts-experimental@6.1.50:
resolution: {integrity: sha512-11HJNqCCbZb6g3CuEOGmFxqia8Nx7sT97IOo4nC3VArbjh6pvgE2+onemkxSbeDSZIcpNFobRGOOIo1J8DSHgQ==}
tldts-experimental@6.1.51:
resolution: {integrity: sha512-aDFHR+bRBXiIeDEPG7nV9vxgVu08Y98MUOQe2eyUpbzyapaeKQiulSedN484mKtLIsZdSbSPiboFlWUSM3TWGw==}
tldts@6.1.50:
resolution: {integrity: sha512-q9GOap6q3KCsLMdOjXhWU5jVZ8/1dIib898JBRLsN+tBhENpBDcAVQbE0epADOjw11FhQQy9AcbqKGBQPUfTQA==}
tldts@6.1.51:
resolution: {integrity: sha512-33lfQoL0JsDogIbZ8fgRyvv77GnRtwkNE/MOKocwUgPO1WrSfsq7+vQRKxRQZai5zd+zg97Iv9fpFQSzHyWdLA==}
hasBin: true
to-regex-range@5.0.1:
@@ -1883,7 +1867,7 @@ snapshots:
'@remusao/smaz': 1.10.0
'@types/chrome': 0.0.270
'@types/firefox-webext-browser': 120.0.4
tldts-experimental: 6.1.50
tldts-experimental: 6.1.51
'@colors/colors@1.5.0':
optional: true
@@ -2195,10 +2179,6 @@ snapshots:
tslib: 2.7.0
optional: true
'@types/async-retry@1.4.9':
dependencies:
'@types/retry': 0.12.5
'@types/better-sqlite3@7.6.11':
dependencies:
'@types/node': 22.7.5
@@ -2430,10 +2410,6 @@ snapshots:
assertion-error@1.1.0: {}
async-retry@1.3.3:
dependencies:
retry: 0.13.1
async-sema@3.1.1: {}
asynckit@0.4.0: {}
@@ -2948,7 +2924,7 @@ snapshots:
combined-stream: 1.0.8
mime-types: 2.1.35
foxact@0.2.38:
foxact@0.2.39:
dependencies:
client-only: 0.0.1
server-only: 0.0.1
@@ -3417,8 +3393,6 @@ snapshots:
retry@0.12.0: {}
retry@0.13.1: {}
reusify@1.0.4: {}
rimraf@5.0.10:
@@ -3594,15 +3568,15 @@ snapshots:
text-table@0.2.0: {}
tldts-core@6.1.50: {}
tldts-core@6.1.51: {}
tldts-experimental@6.1.50:
tldts-experimental@6.1.51:
dependencies:
tldts-core: 6.1.50
tldts-core: 6.1.51
tldts@6.1.50:
tldts@6.1.51:
dependencies:
tldts-core: 6.1.50
tldts-core: 6.1.51
to-regex-range@5.0.1:
dependencies: