diff --git a/Build/lib/cache-filesystem.ts b/Build/lib/cache-filesystem.ts index 99feb760..1b00c554 100644 --- a/Build/lib/cache-filesystem.ts +++ b/Build/lib/cache-filesystem.ts @@ -8,10 +8,12 @@ import { fastStringArrayJoin, identity, mergeHeaders } from './misc'; import { performance } from 'node:perf_hooks'; import fs from 'node:fs'; import { stringHash } from './string-hash'; -import { defaultRequestInit, fetchWithLog, ResponseError } from './fetch-retry'; +import { defaultRequestInit, requestWithLog, UndiciResponseError } from './fetch-retry'; +import type { UndiciResponseData } from './fetch-retry'; import { Custom304NotModifiedError, CustomAbortError, CustomNoETagFallbackError, fetchAssetsWith304, sleepWithAbort } from './fetch-assets'; -import type { Response, RequestInit, HeadersInit } from 'undici'; +import type { HeadersInit } from 'undici'; +import type { IncomingHttpHeaders } from 'undici/types/header'; const enum CacheStatus { Hit = 'hit', @@ -68,6 +70,16 @@ export const TTL = { TWO_WEEKS: () => randomInt(10, 14) * ONE_DAY }; +function ensureETag(headers: IncomingHttpHeaders) { + if ('etag' in headers && typeof headers.etag === 'string' && headers.etag.length > 0) { + return headers.etag; + } + if ('ETag' in headers && typeof headers.ETag === 'string' && headers.ETag.length > 0) { + return headers.ETag; + } + return ''; +} + export class Cache { db: Database; /** Time before deletion */ @@ -213,12 +225,12 @@ export class Cache { async applyWithHttp304( url: string, extraCacheKey: string, - fn: (resp: Response) => Promise, - opt: Omit, 'incrementTtlWhenHit'>, - requestInit?: RequestInit + fn: (resp: UndiciResponseData) => Promise, + opt: Omit, 'incrementTtlWhenHit'> + // requestInit?: RequestInit ): Promise { if (opt.temporaryBypass) { - return fn(await fetchWithLog(url, requestInit)); + return fn(await requestWithLog(url)); } const baseKey = url + '$' + extraCacheKey; @@ -227,19 +239,19 @@ export class Cache { const etag = this.get(etagKey); - const onMiss = async (resp: Response) => { + const onMiss = async (resp: UndiciResponseData) => { const serializer = 'serializer' in opt ? opt.serializer : identity as any; const value = await fn(resp); - if (resp.headers.has('ETag')) { - let serverETag = resp.headers.get('ETag')!; + let serverETag = ensureETag(resp.headers); + if (serverETag) { // FUCK someonewhocares.org if (url.includes('someonewhocares.org')) { serverETag = serverETag.replace('-gzip', ''); } - console.log(picocolors.yellow('[cache] miss'), url, { status: resp.status, cachedETag: etag, serverETag }); + console.log(picocolors.yellow('[cache] miss'), url, { status: resp.statusCode, cachedETag: etag, serverETag }); this.set(etagKey, serverETag, TTL.ONE_WEEK_STATIC); this.set(cachedKey, serializer(value), TTL.ONE_WEEK_STATIC); @@ -257,28 +269,24 @@ export class Cache { const cached = this.get(cachedKey); if (cached == null) { - return onMiss(await fetchWithLog(url, requestInit)); + return onMiss(await requestWithLog(url)); } - const resp = await fetchWithLog( + const resp = await requestWithLog( url, { - ...(requestInit ?? defaultRequestInit), headers: (typeof etag === 'string' && etag.length > 0) - ? mergeHeaders( - (requestInit ?? defaultRequestInit).headers, - { 'If-None-Match': etag } - ) - : (requestInit ?? defaultRequestInit).headers + ? { 'If-None-Match': etag } + : {} } ); // Only miss if previously a ETag was present and the server responded with a 304 - if (resp.headers.has('ETag') && resp.status !== 304) { + if (!ensureETag(resp.headers) && resp.statusCode !== 304) { return onMiss(resp); } - console.log(picocolors.green(`[cache] ${resp.status === 304 ? 'http 304' : 'cache hit'}`), picocolors.gray(url)); + console.log(picocolors.green(`[cache] ${resp.statusCode === 304 ? 'http 304' : 'cache hit'}`), picocolors.gray(url)); this.updateTtl(cachedKey, TTL.ONE_WEEK_STATIC); const deserializer = 'deserializer' in opt ? opt.deserializer : identity as any; @@ -297,7 +305,7 @@ export class Cache { } if (mirrorUrls.length === 0) { - return this.applyWithHttp304(primaryUrl, extraCacheKey, async (resp) => fn(await resp.text()), opt); + return this.applyWithHttp304(primaryUrl, extraCacheKey, async (resp) => fn(await resp.body.text()), opt); } const baseKey = primaryUrl + '$' + extraCacheKey; @@ -323,7 +331,7 @@ export class Cache { } const etag = this.get(getETagKey(url)); - const res = await fetchWithLog( + const res = await requestWithLog( url, { signal: controller.signal, @@ -337,26 +345,26 @@ export class Cache { } ); - const responseHasETag = res.headers.has('etag'); - if (responseHasETag) { - this.set(getETagKey(url), res.headers.get('etag')!, TTL.ONE_WEEK_STATIC); + const serverETag = ensureETag(res.headers); + if (serverETag) { + this.set(getETagKey(url), serverETag, TTL.ONE_WEEK_STATIC); } // If we do not have a cached value, we ignore 304 - if (res.status === 304 && typeof previouslyCached === 'string') { + if (res.statusCode === 304 && typeof previouslyCached === 'string') { controller.abort(); throw new Custom304NotModifiedError(url, previouslyCached); } - if (!responseHasETag && !this.get(getETagKey(primaryUrl)) && typeof previouslyCached === 'string') { + if (!serverETag && !this.get(getETagKey(primaryUrl)) && typeof previouslyCached === 'string') { controller.abort(); throw new CustomNoETagFallbackError(previouslyCached); } // either no etag and not cached // or has etag but not 304 - const text = await res.text(); + const text = await res.body.text(); if (text.length < 2) { - throw new ResponseError(res); + throw new UndiciResponseError(res, url); } controller.abort(); diff --git a/Build/lib/download-publicsuffixlist.ts b/Build/lib/download-publicsuffixlist.ts index 08cfdeba..8a39fec8 100644 --- a/Build/lib/download-publicsuffixlist.ts +++ b/Build/lib/download-publicsuffixlist.ts @@ -4,7 +4,7 @@ import { createMemoizedPromise } from './memo-promise'; export const getPublicSuffixListTextPromise = createMemoizedPromise(() => fsFetchCache.applyWithHttp304( 'https://publicsuffix.org/list/public_suffix_list.dat', getFileContentHash(__filename), - (r) => r.text().then(text => text.split('\n')), + (r) => r.body.text().then(text => text.split('\n')), { // https://github.com/publicsuffix/list/blob/master/.github/workflows/tld-update.yml // Though the action runs every 24 hours, the IANA list is updated every 7 days. diff --git a/Build/lib/fetch-assets.ts b/Build/lib/fetch-assets.ts index 1162e24f..0e9394e4 100644 --- a/Build/lib/fetch-assets.ts +++ b/Build/lib/fetch-assets.ts @@ -1,5 +1,5 @@ import picocolors from 'picocolors'; -import { defaultRequestInit, fetchWithLog, ResponseError } from './fetch-retry'; +import { defaultRequestInit, requestWithLog, UndiciResponseError } from './fetch-retry'; import { setTimeout } from 'node:timers/promises'; // eslint-disable-next-line sukka/unicorn/custom-error-definition -- typescript is better @@ -59,11 +59,11 @@ export async function fetchAssetsWith304(url: string, fallbackUrls: string[] | r console.log(picocolors.gray('[fetch cancelled]'), picocolors.gray(url)); throw new CustomAbortError(); } - const res = await fetchWithLog(url, { signal: controller.signal, ...defaultRequestInit }); - const text = await res.text(); + const res = await requestWithLog(url, { signal: controller.signal, ...defaultRequestInit }); + const text = await res.body.text(); if (text.length < 2) { - throw new ResponseError(res); + throw new UndiciResponseError(res, url); } controller.abort(); diff --git a/Build/lib/fetch-retry.ts b/Build/lib/fetch-retry.ts index e99a2433..25bf557e 100644 --- a/Build/lib/fetch-retry.ts +++ b/Build/lib/fetch-retry.ts @@ -1,12 +1,15 @@ import picocolors from 'picocolors'; -import { - fetch, +import undici, { interceptors, EnvHttpProxyAgent, setGlobalDispatcher } from 'undici'; -import type { Response, RequestInit, RequestInfo } from 'undici'; +import type { + Dispatcher +} from 'undici'; + +export type UndiciResponseData = Dispatcher.ResponseData; import CacheableLookup from 'cacheable-lookup'; import type { LookupOptions as CacheableLookupOptions } from 'cacheable-lookup'; @@ -111,25 +114,22 @@ function calculateRetryAfterHeader(retryAfter: string) { return new Date(retryAfter).getTime() - current; } -export class ResponseError extends Error { - readonly res: Response; +export class UndiciResponseError extends Error { readonly code: number; readonly statusCode: number; - readonly url: string; - constructor(res: Response) { - super(res.statusText); + constructor(public readonly res: UndiciResponseData, public readonly url: string) { + super('HTTP ' + res.statusCode); if ('captureStackTrace' in Error) { - Error.captureStackTrace(this, ResponseError); + Error.captureStackTrace(this, UndiciResponseError); } // eslint-disable-next-line sukka/unicorn/custom-error-definition -- deliberatly use previous name this.name = this.constructor.name; this.res = res; - this.code = res.status; - this.statusCode = res.status; - this.url = res.url; + this.code = res.statusCode; + this.statusCode = res.statusCode; } } @@ -139,17 +139,15 @@ export const defaultRequestInit = { } }; -export async function fetchWithLog(url: RequestInfo, opts: RequestInit = defaultRequestInit) { +export async function requestWithLog(url: string, opt?: Parameters[1]) { try { - // this will be retried - const res = (await fetch(url, opts)); - - if (res.status >= 400) { - throw new ResponseError(res); + const res = await undici.request(url, opt); + if (res.statusCode >= 400) { + throw new UndiciResponseError(res, url); } - if (!res.ok && res.status !== 304) { - throw new ResponseError(res); + if (!(res.statusCode >= 200 && res.statusCode <= 299) && res.statusCode !== 304) { + throw new UndiciResponseError(res, url); } return res; @@ -167,4 +165,4 @@ export async function fetchWithLog(url: RequestInfo, opts: RequestInit = default throw err; } -}; +} diff --git a/Build/lib/fetch-text-by-line.ts b/Build/lib/fetch-text-by-line.ts index 604b701d..3524e9d1 100644 --- a/Build/lib/fetch-text-by-line.ts +++ b/Build/lib/fetch-text-by-line.ts @@ -8,6 +8,7 @@ import { TextDecoderStream } from 'node:stream/web'; import { processLine } from './process-line'; import { $fetch } from './make-fetch-happen'; import type { NodeFetchResponse } from './make-fetch-happen'; +import type { UndiciResponseData } from './fetch-retry'; function getReadableStream(file: string | FileHandle): ReadableStream { if (typeof file === 'string') { @@ -21,22 +22,26 @@ export const readFileByLine: ((file: string | FileHandle) => AsyncIterable(resp: T): NonNullable { +function ensureResponseBody(resp: T): NonNullable { if (!resp.body) { throw new Error('Failed to fetch remote text'); } - if (resp.bodyUsed) { + if ('bodyUsed' in resp && resp.bodyUsed) { throw new Error('Body has already been consumed.'); } return resp.body; } -export const createReadlineInterfaceFromResponse: ((resp: Response | NodeFetchResponse) => AsyncIterable) = (resp) => { +export const createReadlineInterfaceFromResponse: ((resp: Response | NodeFetchResponse | UndiciResponseData) => AsyncIterable) = (resp) => { const stream = ensureResponseBody(resp); const webStream: ReadableStream = 'getReader' in stream ? stream - : Readable.toWeb(new Readable().wrap(stream)) as any; + : ( + 'body' in stream + ? stream.body + : Readable.toWeb(new Readable().wrap(stream)) as any + ); return webStream .pipeThrough(new TextDecoderStream()) diff --git a/Build/lib/parse-dnsmasq.ts b/Build/lib/parse-dnsmasq.ts index 967c0b11..cbdcd748 100644 --- a/Build/lib/parse-dnsmasq.ts +++ b/Build/lib/parse-dnsmasq.ts @@ -2,6 +2,7 @@ import { createReadlineInterfaceFromResponse } from './fetch-text-by-line'; import { parse as tldtsParse } from 'tldts'; import { $fetch } from './make-fetch-happen'; import type { NodeFetchResponse } from './make-fetch-happen'; +import type { UndiciResponseData } from './fetch-retry'; function isDomainLoose(domain: string): boolean { const { isIcann, isPrivate, isIp } = tldtsParse(domain); @@ -15,7 +16,7 @@ export function extractDomainsFromFelixDnsmasq(line: string): string | null { return null; } -export async function parseFelixDnsmasqFromResp(resp: Response | NodeFetchResponse): Promise { +export async function parseFelixDnsmasqFromResp(resp: NodeFetchResponse | UndiciResponseData): Promise { const results: string[] = []; for await (const line of createReadlineInterfaceFromResponse(resp)) {