From c5ee4bad535f5353d4cfe6b4d8e0789e8e25c3a4 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Tue, 15 Oct 2024 22:34:53 +0800 Subject: [PATCH] Revert `undici.request` and use `fetch` again --- Build/lib/cache-filesystem.ts | 62 ++++++++++++++------------ Build/lib/download-publicsuffixlist.ts | 2 +- Build/lib/fetch-assets.ts | 10 ++--- Build/lib/fetch-retry.ts | 61 ++++++++++++++++++------- Build/lib/fetch-text-by-line.ts | 6 +-- Build/lib/parse-dnsmasq.ts | 3 +- 6 files changed, 90 insertions(+), 54 deletions(-) diff --git a/Build/lib/cache-filesystem.ts b/Build/lib/cache-filesystem.ts index db93c5af..48200864 100644 --- a/Build/lib/cache-filesystem.ts +++ b/Build/lib/cache-filesystem.ts @@ -8,12 +8,13 @@ import { fastStringArrayJoin, identity, mergeHeaders } from './misc'; import { performance } from 'node:perf_hooks'; import fs from 'node:fs'; import { stringHash } from './string-hash'; -import { defaultRequestInit, requestWithLog, UndiciResponseError } from './fetch-retry'; -import type { UndiciResponseData } from './fetch-retry'; -import { Custom304NotModifiedError, CustomAbortError, CustomNoETagFallbackError, fetchAssetsWith304, sleepWithAbort } from './fetch-assets'; +import { defaultRequestInit, fetchWithLog, ResponseError } from './fetch-retry'; +// import type { UndiciResponseData } from './fetch-retry'; +import { Custom304NotModifiedError, CustomAbortError, CustomNoETagFallbackError, fetchAssetsWithout304, sleepWithAbort } from './fetch-assets'; -import type { HeadersInit } from 'undici'; +import type { Response } from 'undici'; import type { IncomingHttpHeaders } from 'undici/types/header'; +import { Headers } from 'undici'; const enum CacheStatus { Hit = 'hit', @@ -70,14 +71,18 @@ export const TTL = { TWO_WEEKS: () => randomInt(10, 14) * ONE_DAY }; -function ensureETag(headers: IncomingHttpHeaders) { +function ensureETag(headers: IncomingHttpHeaders | Headers) { + if (headers instanceof Headers && headers.has('etag')) { + return headers.get('etag'); + } + if ('etag' in headers && typeof headers.etag === 'string' && headers.etag.length > 0) { return headers.etag; } if ('ETag' in headers && typeof headers.ETag === 'string' && headers.ETag.length > 0) { return headers.ETag; } - return ''; + return null; } export class Cache { @@ -225,12 +230,12 @@ export class Cache { async applyWithHttp304( url: string, extraCacheKey: string, - fn: (resp: UndiciResponseData) => Promise, + fn: (resp: Response) => Promise, opt: Omit, 'incrementTtlWhenHit'> // requestInit?: RequestInit ): Promise { if (opt.temporaryBypass) { - return fn(await requestWithLog(url)); + return fn(await fetchWithLog(url)); } const baseKey = url + '$' + extraCacheKey; @@ -239,7 +244,7 @@ export class Cache { const etag = this.get(etagKey); - const onMiss = async (resp: UndiciResponseData) => { + const onMiss = async (resp: Response) => { const serializer = 'serializer' in opt ? opt.serializer : identity as any; const value = await fn(resp); @@ -251,7 +256,7 @@ export class Cache { serverETag = serverETag.replace('-gzip', ''); } - console.log(picocolors.yellow('[cache] miss'), url, { status: resp.statusCode, cachedETag: etag, serverETag }); + console.log(picocolors.yellow('[cache] miss'), url, { status: resp.status, cachedETag: etag, serverETag }); this.set(etagKey, serverETag, TTL.ONE_WEEK_STATIC); this.set(cachedKey, serializer(value), TTL.ONE_WEEK_STATIC); @@ -269,24 +274,25 @@ export class Cache { const cached = this.get(cachedKey); if (cached == null) { - return onMiss(await requestWithLog(url)); + return onMiss(await fetchWithLog(url)); } - const resp = await requestWithLog( + const resp = await fetchWithLog( url, { + ...defaultRequestInit, headers: (typeof etag === 'string' && etag.length > 0) - ? { 'If-None-Match': etag } - : {} + ? mergeHeaders>(defaultRequestInit.headers, { 'If-None-Match': etag }) + : defaultRequestInit.headers } ); // Only miss if previously a ETag was present and the server responded with a 304 - if (!ensureETag(resp.headers) && resp.statusCode !== 304) { + if (!ensureETag(resp.headers) && resp.status !== 304) { return onMiss(resp); } - console.log(picocolors.green(`[cache] ${resp.statusCode === 304 ? 'http 304' : 'cache hit'}`), picocolors.gray(url)); + console.log(picocolors.green(`[cache] ${resp.status === 304 ? 'http 304' : 'cache hit'}`), picocolors.gray(url)); this.updateTtl(cachedKey, TTL.ONE_WEEK_STATIC); const deserializer = 'deserializer' in opt ? opt.deserializer : identity as any; @@ -301,16 +307,17 @@ export class Cache { opt: Omit, 'incrementTtlWhenHit'> ): Promise { if (opt.temporaryBypass) { - return fn(await fetchAssetsWith304(primaryUrl, mirrorUrls)); + return fn(await fetchAssetsWithout304(primaryUrl, mirrorUrls)); } if (mirrorUrls.length === 0) { - return this.applyWithHttp304(primaryUrl, extraCacheKey, async (resp) => fn(await resp.body.text()), opt); + return this.applyWithHttp304(primaryUrl, extraCacheKey, async (resp) => fn(await resp.text()), opt); } const baseKey = primaryUrl + '$' + extraCacheKey; const getETagKey = (url: string) => baseKey + '$' + url + '$etag'; const cachedKey = baseKey + '$cached'; + const controller = new AbortController(); const previouslyCached = this.get(cachedKey); @@ -331,16 +338,13 @@ export class Cache { } const etag = this.get(getETagKey(url)); - const res = await requestWithLog( + const res = await fetchWithLog( url, { signal: controller.signal, ...defaultRequestInit, - headers: (typeof etag === 'string' && etag.length > 0) - ? mergeHeaders( - { 'If-None-Match': etag }, - defaultRequestInit.headers - ) + headers: (typeof etag === 'string' && etag.length > 0 && typeof previouslyCached === 'string' && previouslyCached.length > 1) + ? mergeHeaders>(defaultRequestInit.headers, { 'If-None-Match': etag }) : defaultRequestInit.headers } ); @@ -350,7 +354,7 @@ export class Cache { this.set(getETagKey(url), serverETag, TTL.ONE_WEEK_STATIC); } // If we do not have a cached value, we ignore 304 - if (res.statusCode === 304 && typeof previouslyCached === 'string' && previouslyCached.length > 1) { + if (res.status === 304 && typeof previouslyCached === 'string' && previouslyCached.length > 1) { const err = new Custom304NotModifiedError(url, previouslyCached); controller.abort(err); throw err; @@ -363,10 +367,10 @@ export class Cache { // either no etag and not cached // or has etag but not 304 - const text = await res.body.text(); + const text = await res.text(); if (text.length < 2) { - throw new UndiciResponseError(res, url); + throw new ResponseError(res, url, 'empty response'); } controller.abort(); @@ -391,8 +395,6 @@ export class Cache { if (e && typeof e === 'object' && 'errors' in e && Array.isArray(e.errors)) { const deserializer = 'deserializer' in opt ? opt.deserializer : identity as any; - console.log(e.errors); - for (let i = 0, len = e.errors.length; i < len; i++) { const error = e.errors[i]; if ('name' in error && (error.name === 'CustomAbortError' || error.name === 'AbortError')) { @@ -409,6 +411,8 @@ export class Cache { return deserializer(error.data); } } + + console.log(picocolors.red('[fetch error]'), picocolors.gray(error.url), error); } } diff --git a/Build/lib/download-publicsuffixlist.ts b/Build/lib/download-publicsuffixlist.ts index 8a39fec8..08cfdeba 100644 --- a/Build/lib/download-publicsuffixlist.ts +++ b/Build/lib/download-publicsuffixlist.ts @@ -4,7 +4,7 @@ import { createMemoizedPromise } from './memo-promise'; export const getPublicSuffixListTextPromise = createMemoizedPromise(() => fsFetchCache.applyWithHttp304( 'https://publicsuffix.org/list/public_suffix_list.dat', getFileContentHash(__filename), - (r) => r.body.text().then(text => text.split('\n')), + (r) => r.text().then(text => text.split('\n')), { // https://github.com/publicsuffix/list/blob/master/.github/workflows/tld-update.yml // Though the action runs every 24 hours, the IANA list is updated every 7 days. diff --git a/Build/lib/fetch-assets.ts b/Build/lib/fetch-assets.ts index 0e9394e4..78bdc700 100644 --- a/Build/lib/fetch-assets.ts +++ b/Build/lib/fetch-assets.ts @@ -1,5 +1,5 @@ import picocolors from 'picocolors'; -import { defaultRequestInit, requestWithLog, UndiciResponseError } from './fetch-retry'; +import { defaultRequestInit, fetchWithLog, ResponseError } from './fetch-retry'; import { setTimeout } from 'node:timers/promises'; // eslint-disable-next-line sukka/unicorn/custom-error-definition -- typescript is better @@ -42,7 +42,7 @@ export function sleepWithAbort(ms: number, signal: AbortSignal) { }); } -export async function fetchAssetsWith304(url: string, fallbackUrls: string[] | readonly string[]) { +export async function fetchAssetsWithout304(url: string, fallbackUrls: string[] | readonly string[]) { const controller = new AbortController(); const createFetchFallbackPromise = async (url: string, index: number) => { @@ -59,11 +59,11 @@ export async function fetchAssetsWith304(url: string, fallbackUrls: string[] | r console.log(picocolors.gray('[fetch cancelled]'), picocolors.gray(url)); throw new CustomAbortError(); } - const res = await requestWithLog(url, { signal: controller.signal, ...defaultRequestInit }); - const text = await res.body.text(); + const res = await fetchWithLog(url, { signal: controller.signal, ...defaultRequestInit }); + const text = await res.text(); if (text.length < 2) { - throw new UndiciResponseError(res, url); + throw new ResponseError(res, url, 'empty response w/o 304'); } controller.abort(); diff --git a/Build/lib/fetch-retry.ts b/Build/lib/fetch-retry.ts index 25bf557e..e2353545 100644 --- a/Build/lib/fetch-retry.ts +++ b/Build/lib/fetch-retry.ts @@ -6,18 +6,20 @@ import undici, { } from 'undici'; import type { - Dispatcher + Dispatcher, + RequestInit, + Response } from 'undici'; -export type UndiciResponseData = Dispatcher.ResponseData; +export type UndiciResponseData = Dispatcher.ResponseData; import CacheableLookup from 'cacheable-lookup'; import type { LookupOptions as CacheableLookupOptions } from 'cacheable-lookup'; +import { inspect } from 'node:util'; const cacheableLookup = new CacheableLookup(); const agent = new EnvHttpProxyAgent({ - // allowH2: true, connect: { lookup(hostname, opt, cb) { return cacheableLookup.lookup(hostname, opt as CacheableLookupOptions, cb); @@ -114,22 +116,23 @@ function calculateRetryAfterHeader(retryAfter: string) { return new Date(retryAfter).getTime() - current; } -export class UndiciResponseError extends Error { +export class ResponseError extends Error { readonly code: number; readonly statusCode: number; - constructor(public readonly res: UndiciResponseData, public readonly url: string) { - super('HTTP ' + res.statusCode); + constructor(public readonly res: T, public readonly url: string, ...args: any[]) { + const statusCode = 'statusCode' in res ? res.statusCode : res.status; + super('HTTP ' + statusCode + ' ' + args.map(_ => inspect(_)).join(' ')); if ('captureStackTrace' in Error) { - Error.captureStackTrace(this, UndiciResponseError); + Error.captureStackTrace(this, ResponseError); } // eslint-disable-next-line sukka/unicorn/custom-error-definition -- deliberatly use previous name this.name = this.constructor.name; this.res = res; - this.code = res.statusCode; - this.statusCode = res.statusCode; + this.code = statusCode; + this.statusCode = statusCode; } } @@ -139,15 +142,43 @@ export const defaultRequestInit = { } }; -export async function requestWithLog(url: string, opt?: Parameters[1]) { +export async function fetchWithLog(url: string, init?: RequestInit) { try { - const res = await undici.request(url, opt); - if (res.statusCode >= 400) { - throw new UndiciResponseError(res, url); + const res = await undici.fetch(url, init); + if (res.status >= 400) { + throw new ResponseError(res, url); } - if (!(res.statusCode >= 200 && res.statusCode <= 299) && res.statusCode !== 304) { - throw new UndiciResponseError(res, url); + if (!(res.status >= 200 && res.status <= 299) && res.status !== 304) { + throw new ResponseError(res, url); + } + + return res; + } catch (err: unknown) { + if (typeof err === 'object' && err !== null && 'name' in err) { + if (( + err.name === 'AbortError' + || ('digest' in err && err.digest === 'AbortError') + )) { + console.log(picocolors.gray('[fetch abort]'), url); + } + } else { + console.log(picocolors.gray('[fetch fail]'), url, { name: (err as any).name }, err); + } + + throw err; + } +} + +export async function requestWithLog(url: string, opt?: Parameters[1]) { + try { + const res = await undici.request(url, opt); + if (res.statusCode >= 400) { + throw new ResponseError(res, url); + } + + if (!(res.statusCode >= 200 && res.statusCode <= 299) && res.statusCode !== 304) { + throw new ResponseError(res, url); } return res; diff --git a/Build/lib/fetch-text-by-line.ts b/Build/lib/fetch-text-by-line.ts index aca73cfa..a32c3779 100644 --- a/Build/lib/fetch-text-by-line.ts +++ b/Build/lib/fetch-text-by-line.ts @@ -9,6 +9,7 @@ import { processLine } from './process-line'; import { $fetch } from './make-fetch-happen'; import type { NodeFetchResponse } from './make-fetch-happen'; import type { UndiciResponseData } from './fetch-retry'; +import type { Response } from 'undici'; function getReadableStream(file: string | FileHandle): ReadableStream { if (typeof file === 'string') { @@ -22,8 +23,7 @@ export const readFileByLine: ((file: string | FileHandle) => AsyncIterable(resp: T): NonNullable { - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- NodeFetchResponse['body'] is nullable +function ensureResponseBody(resp: T): NonNullable { if (resp.body == null) { throw new Error('Failed to fetch remote text'); } @@ -33,7 +33,7 @@ function ensureResponseBody(re return resp.body; } -export const createReadlineInterfaceFromResponse: ((resp: NodeFetchResponse | UndiciResponseData) => AsyncIterable) = (resp) => { +export const createReadlineInterfaceFromResponse: ((resp: NodeFetchResponse | UndiciResponseData | Response) => AsyncIterable) = (resp) => { const stream = ensureResponseBody(resp); const webStream: ReadableStream = 'getReader' in stream diff --git a/Build/lib/parse-dnsmasq.ts b/Build/lib/parse-dnsmasq.ts index c52fc78e..2f1111ff 100644 --- a/Build/lib/parse-dnsmasq.ts +++ b/Build/lib/parse-dnsmasq.ts @@ -2,6 +2,7 @@ import { createReadlineInterfaceFromResponse } from './fetch-text-by-line'; import { parse as tldtsParse } from 'tldts'; import type { NodeFetchResponse } from './make-fetch-happen'; import type { UndiciResponseData } from './fetch-retry'; +import type { Response } from 'undici'; function isDomainLoose(domain: string): boolean { const { isIcann, isPrivate, isIp } = tldtsParse(domain); @@ -15,7 +16,7 @@ export function extractDomainsFromFelixDnsmasq(line: string): string | null { return null; } -export async function parseFelixDnsmasqFromResp(resp: NodeFetchResponse | UndiciResponseData): Promise { +export async function parseFelixDnsmasqFromResp(resp: NodeFetchResponse | UndiciResponseData | Response): Promise { const results: string[] = []; for await (const line of createReadlineInterfaceFromResponse(resp)) {