From 87abb12dc743f3ae929e010deebc16d752ee32a8 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Thu, 17 Oct 2024 01:54:53 +0800 Subject: [PATCH] Patch undici and use w/ request --- Build/lib/cache-filesystem.ts | 30 ++++++++++++++------------ Build/lib/download-publicsuffixlist.ts | 2 +- Build/lib/fetch-assets.ts | 6 +++--- package.json | 5 +++++ patches/undici.patch | 13 +++++++++++ pnpm-lock.yaml | 9 ++++++-- 6 files changed, 45 insertions(+), 20 deletions(-) create mode 100644 patches/undici.patch diff --git a/Build/lib/cache-filesystem.ts b/Build/lib/cache-filesystem.ts index 48200864..d90143dc 100644 --- a/Build/lib/cache-filesystem.ts +++ b/Build/lib/cache-filesystem.ts @@ -8,11 +8,11 @@ import { fastStringArrayJoin, identity, mergeHeaders } from './misc'; import { performance } from 'node:perf_hooks'; import fs from 'node:fs'; import { stringHash } from './string-hash'; -import { defaultRequestInit, fetchWithLog, ResponseError } from './fetch-retry'; +import { defaultRequestInit, requestWithLog, ResponseError } from './fetch-retry'; +import type { UndiciResponseData } from './fetch-retry'; // import type { UndiciResponseData } from './fetch-retry'; import { Custom304NotModifiedError, CustomAbortError, CustomNoETagFallbackError, fetchAssetsWithout304, sleepWithAbort } from './fetch-assets'; -import type { Response } from 'undici'; import type { IncomingHttpHeaders } from 'undici/types/header'; import { Headers } from 'undici'; @@ -230,12 +230,12 @@ export class Cache { async applyWithHttp304( url: string, extraCacheKey: string, - fn: (resp: Response) => Promise, + fn: (resp: UndiciResponseData) => Promise, opt: Omit, 'incrementTtlWhenHit'> // requestInit?: RequestInit ): Promise { if (opt.temporaryBypass) { - return fn(await fetchWithLog(url)); + return fn(await requestWithLog(url)); } const baseKey = url + '$' + extraCacheKey; @@ -244,7 +244,7 @@ export class Cache { const etag = this.get(etagKey); - const onMiss = async (resp: Response) => { + const onMiss = async (resp: UndiciResponseData) => { const serializer = 'serializer' in opt ? opt.serializer : identity as any; const value = await fn(resp); @@ -256,7 +256,7 @@ export class Cache { serverETag = serverETag.replace('-gzip', ''); } - console.log(picocolors.yellow('[cache] miss'), url, { status: resp.status, cachedETag: etag, serverETag }); + console.log(picocolors.yellow('[cache] miss'), url, { status: resp.statusCode, cachedETag: etag, serverETag }); this.set(etagKey, serverETag, TTL.ONE_WEEK_STATIC); this.set(cachedKey, serializer(value), TTL.ONE_WEEK_STATIC); @@ -274,10 +274,10 @@ export class Cache { const cached = this.get(cachedKey); if (cached == null) { - return onMiss(await fetchWithLog(url)); + return onMiss(await requestWithLog(url)); } - const resp = await fetchWithLog( + const resp = await requestWithLog( url, { ...defaultRequestInit, @@ -288,11 +288,11 @@ export class Cache { ); // Only miss if previously a ETag was present and the server responded with a 304 - if (!ensureETag(resp.headers) && resp.status !== 304) { + if (!ensureETag(resp.headers) && resp.statusCode !== 304) { return onMiss(resp); } - console.log(picocolors.green(`[cache] ${resp.status === 304 ? 'http 304' : 'cache hit'}`), picocolors.gray(url)); + console.log(picocolors.green(`[cache] ${resp.statusCode === 304 ? 'http 304' : 'cache hit'}`), picocolors.gray(url)); this.updateTtl(cachedKey, TTL.ONE_WEEK_STATIC); const deserializer = 'deserializer' in opt ? opt.deserializer : identity as any; @@ -311,7 +311,7 @@ export class Cache { } if (mirrorUrls.length === 0) { - return this.applyWithHttp304(primaryUrl, extraCacheKey, async (resp) => fn(await resp.text()), opt); + return this.applyWithHttp304(primaryUrl, extraCacheKey, async (resp) => fn(await resp.body.text()), opt); } const baseKey = primaryUrl + '$' + extraCacheKey; @@ -338,7 +338,7 @@ export class Cache { } const etag = this.get(getETagKey(url)); - const res = await fetchWithLog( + const res = await requestWithLog( url, { signal: controller.signal, @@ -354,7 +354,7 @@ export class Cache { this.set(getETagKey(url), serverETag, TTL.ONE_WEEK_STATIC); } // If we do not have a cached value, we ignore 304 - if (res.status === 304 && typeof previouslyCached === 'string' && previouslyCached.length > 1) { + if (res.statusCode === 304 && typeof previouslyCached === 'string' && previouslyCached.length > 1) { const err = new Custom304NotModifiedError(url, previouslyCached); controller.abort(err); throw err; @@ -367,7 +367,7 @@ export class Cache { // either no etag and not cached // or has etag but not 304 - const text = await res.text(); + const text = await res.body.text(); if (text.length < 2) { throw new ResponseError(res, url, 'empty response'); @@ -416,6 +416,8 @@ export class Cache { } } + console.log({ e }); + console.log(`Download Rule for [${primaryUrl}] failed`); throw e; } diff --git a/Build/lib/download-publicsuffixlist.ts b/Build/lib/download-publicsuffixlist.ts index 08cfdeba..8a39fec8 100644 --- a/Build/lib/download-publicsuffixlist.ts +++ b/Build/lib/download-publicsuffixlist.ts @@ -4,7 +4,7 @@ import { createMemoizedPromise } from './memo-promise'; export const getPublicSuffixListTextPromise = createMemoizedPromise(() => fsFetchCache.applyWithHttp304( 'https://publicsuffix.org/list/public_suffix_list.dat', getFileContentHash(__filename), - (r) => r.text().then(text => text.split('\n')), + (r) => r.body.text().then(text => text.split('\n')), { // https://github.com/publicsuffix/list/blob/master/.github/workflows/tld-update.yml // Though the action runs every 24 hours, the IANA list is updated every 7 days. diff --git a/Build/lib/fetch-assets.ts b/Build/lib/fetch-assets.ts index 78bdc700..f7e418f0 100644 --- a/Build/lib/fetch-assets.ts +++ b/Build/lib/fetch-assets.ts @@ -1,5 +1,5 @@ import picocolors from 'picocolors'; -import { defaultRequestInit, fetchWithLog, ResponseError } from './fetch-retry'; +import { defaultRequestInit, requestWithLog, ResponseError } from './fetch-retry'; import { setTimeout } from 'node:timers/promises'; // eslint-disable-next-line sukka/unicorn/custom-error-definition -- typescript is better @@ -59,8 +59,8 @@ export async function fetchAssetsWithout304(url: string, fallbackUrls: string[] console.log(picocolors.gray('[fetch cancelled]'), picocolors.gray(url)); throw new CustomAbortError(); } - const res = await fetchWithLog(url, { signal: controller.signal, ...defaultRequestInit }); - const text = await res.text(); + const res = await requestWithLog(url, { signal: controller.signal, ...defaultRequestInit }); + const text = await res.body.text(); if (text.length < 2) { throw new ResponseError(res, url, 'empty response w/o 304'); diff --git a/package.json b/package.json index a0605c7c..e4bc6ac8 100644 --- a/package.json +++ b/package.json @@ -70,5 +70,10 @@ "packageManager": "pnpm@9.12.1", "resolutions": { "has": "npm:@nolyfill/has@latest" + }, + "pnpm": { + "patchedDependencies": { + "undici": "patches/undici.patch" + } } } diff --git a/patches/undici.patch b/patches/undici.patch new file mode 100644 index 00000000..577b9284 --- /dev/null +++ b/patches/undici.patch @@ -0,0 +1,13 @@ +diff --git a/lib/api/api-request.js b/lib/api/api-request.js +index ced5590d21053ddd4c6e81e25fb1a3baea08e2c5..be17d62877403cfc8afe10a75ceb67bf1de5e56d 100644 +--- a/lib/api/api-request.js ++++ b/lib/api/api-request.js +@@ -73,7 +73,7 @@ class RequestHandler extends AsyncResource { + this.removeAbortListener = util.addAbortListener(this.signal, () => { + this.reason = this.signal.reason ?? new RequestAbortedError() + if (this.res) { +- util.destroy(this.res, this.reason) ++ util.destroy(this.res.on('error', util.nop), this.reason) + } else if (this.abort) { + this.abort(this.reason) + } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 893bde99..cdf3a23d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -7,6 +7,11 @@ settings: overrides: has: npm:@nolyfill/has@latest +patchedDependencies: + undici: + hash: yuj5uy4vvwj67xoliq5togiyme + path: patches/undici.patch + importers: .: @@ -76,7 +81,7 @@ importers: version: 6.1.51 undici: specifier: 6.20.1 - version: 6.20.1 + version: 6.20.1(patch_hash=yuj5uy4vvwj67xoliq5togiyme) why-is-node-running: specifier: ^3.2.0 version: 3.2.0 @@ -3606,7 +3611,7 @@ snapshots: undici-types@6.19.8: {} - undici@6.20.1: {} + undici@6.20.1(patch_hash=yuj5uy4vvwj67xoliq5togiyme): {} unique-filename@4.0.0: dependencies: