From 6235d9fee0641076a42d9a6ed19a9c877e5b47c8 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Fri, 17 Jan 2025 23:06:51 +0800 Subject: [PATCH] Chore: minor changes --- Build/build-reject-ip-list.ts | 4 ++-- Build/lib/fetch-assets.ts | 21 +++------------------ Build/lib/fetch-retry.ts | 7 +++---- Build/lib/parse-filter/domainlists.ts | 4 ++-- Build/lib/parse-filter/filters.ts | 4 ++-- Build/lib/parse-filter/hosts.ts | 4 ++-- package.json | 2 +- pnpm-lock.yaml | 10 +++++----- 8 files changed, 20 insertions(+), 36 deletions(-) diff --git a/Build/build-reject-ip-list.ts b/Build/build-reject-ip-list.ts index eb059d0b..14b8ea17 100644 --- a/Build/build-reject-ip-list.ts +++ b/Build/build-reject-ip-list.ts @@ -8,7 +8,7 @@ import { processLine } from './lib/process-line'; import { RulesetOutput } from './lib/create-file'; import { SOURCE_DIR } from './constants/dir'; import { $$fetch } from './lib/fetch-retry'; -import { fetchAssetsWithout304 } from './lib/fetch-assets'; +import { fetchAssets } from './lib/fetch-assets'; const BOGUS_NXDOMAIN_URL = 'https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/bogus-nxdomain.china.conf'; const getBogusNxDomainIPsPromise: Promise<[ipv4: string[], ipv6: string[]]> = $$fetch(BOGUS_NXDOMAIN_URL).then(async (resp) => { @@ -37,7 +37,7 @@ const BOTNET_FILTER_MIRROR_URL = [ // https://curbengh.github.io/malware-filter/botnet-filter-dnscrypt-blocked-ips.txt ]; -const getBotNetFilterIPsPromise: Promise<[ipv4: string[], ipv6: string[]]> = fetchAssetsWithout304(BOTNET_FILTER_URL, BOTNET_FILTER_MIRROR_URL).then(text => text.split('\n').reduce<[ipv4: string[], ipv6: string[]]>((acc, cur) => { +const getBotNetFilterIPsPromise: Promise<[ipv4: string[], ipv6: string[]]> = fetchAssets(BOTNET_FILTER_URL, BOTNET_FILTER_MIRROR_URL).then(text => text.split('\n').reduce<[ipv4: string[], ipv6: string[]]>((acc, cur) => { const ip = processLine(cur); if (ip) { if (isProbablyIpv4(ip)) { diff --git a/Build/lib/fetch-assets.ts b/Build/lib/fetch-assets.ts index 558012af..295e293d 100644 --- a/Build/lib/fetch-assets.ts +++ b/Build/lib/fetch-assets.ts @@ -1,6 +1,6 @@ import picocolors from 'picocolors'; import { $$fetch, defaultRequestInit, ResponseError } from './fetch-retry'; -import { wait } from 'foxts/wait'; +import { waitWithAbort } from 'foxts/wait'; // eslint-disable-next-line sukka/unicorn/custom-error-definition -- typescript is better export class CustomAbortError extends Error { @@ -26,29 +26,14 @@ export class CustomNoETagFallbackError extends Error { } } -export function sleepWithAbort(ms: number, signal: AbortSignal) { - return new Promise((resolve, reject) => { - if (signal.aborted) { - reject(signal.reason as Error); - return; - } - - signal.addEventListener('abort', stop, { once: true }); - - wait(ms).then(resolve).catch(reject).finally(() => signal.removeEventListener('abort', stop)); - - function stop(this: AbortSignal) { reject(this.reason as Error); } - }); -} - -export async function fetchAssetsWithout304(url: string, fallbackUrls: null | undefined | string[] | readonly string[]) { +export async function fetchAssets(url: string, fallbackUrls: null | undefined | string[] | readonly string[]) { const controller = new AbortController(); const createFetchFallbackPromise = async (url: string, index: number) => { if (index >= 0) { // Most assets can be downloaded within 250ms. To avoid wasting bandwidth, we will wait for 500ms before downloading from the fallback URL. try { - await sleepWithAbort(50 + (index + 1) * 100, controller.signal); + await waitWithAbort(50 + (index + 1) * 100, controller.signal); } catch { console.log(picocolors.gray('[fetch cancelled early]'), picocolors.gray(url)); throw new CustomAbortError(); diff --git a/Build/lib/fetch-retry.ts b/Build/lib/fetch-retry.ts index 7842f821..64d2c2e3 100644 --- a/Build/lib/fetch-retry.ts +++ b/Build/lib/fetch-retry.ts @@ -95,10 +95,9 @@ setGlobalDispatcher(agent.compose( : retryAfter * 1e3; // Retry-After is in seconds } - const retryTimeout - = retryAfter > 0 - ? Math.min(retryAfter, maxTimeout) - : Math.min(minTimeout * (timeoutFactor ** (counter - 1)), maxTimeout); + const retryTimeout = retryAfter > 0 + ? Math.min(retryAfter, maxTimeout) + : Math.min(minTimeout * (timeoutFactor ** (counter - 1)), maxTimeout); console.log('[fetch retry]', 'schedule retry', { statusCode, retryTimeout, errorCode, url: opts.origin }); // eslint-disable-next-line sukka/prefer-timer-id -- won't leak diff --git a/Build/lib/parse-filter/domainlists.ts b/Build/lib/parse-filter/domainlists.ts index 52b776d8..a0284079 100644 --- a/Build/lib/parse-filter/domainlists.ts +++ b/Build/lib/parse-filter/domainlists.ts @@ -2,7 +2,7 @@ import picocolors from 'picocolors'; import { fastNormalizeDomain } from '../normalize-domain'; import { processLine } from '../process-line'; import { onBlackFound } from './shared'; -import { fetchAssetsWithout304 } from '../fetch-assets'; +import { fetchAssets } from '../fetch-assets'; import type { Span } from '../../trace'; function domainListLineCb(l: string, set: string[], includeAllSubDomain: boolean, meta: string) { @@ -32,7 +32,7 @@ export function processDomainLists( domainListsUrl: string, mirrors: string[] | null, includeAllSubDomain = false ) { return span.traceChildAsync(`process domainlist: ${domainListsUrl}`, async (span) => { - const text = await span.traceChildAsync('download', () => fetchAssetsWithout304( + const text = await span.traceChildAsync('download', () => fetchAssets( domainListsUrl, mirrors )); diff --git a/Build/lib/parse-filter/filters.ts b/Build/lib/parse-filter/filters.ts index 18b14fe9..132ab06d 100644 --- a/Build/lib/parse-filter/filters.ts +++ b/Build/lib/parse-filter/filters.ts @@ -1,6 +1,6 @@ import picocolors from 'picocolors'; import type { Span } from '../../trace'; -import { fetchAssetsWithout304 } from '../fetch-assets'; +import { fetchAssets } from '../fetch-assets'; import { onBlackFound, onWhiteFound } from './shared'; import { createRetrieKeywordFilter as createKeywordFilter } from 'foxts/retrie'; import { fastNormalizeDomain } from '../normalize-domain'; @@ -27,7 +27,7 @@ export async function processFilterRules( allowThirdParty = false ): Promise<{ white: string[], black: string[] }> { const [white, black, warningMessages] = await parentSpan.traceChild(`process filter rules: ${filterRulesUrl}`).traceAsyncFn(async (span) => { - const text = await span.traceChildAsync('download', () => fetchAssetsWithout304(filterRulesUrl, fallbackUrls)); + const text = await span.traceChildAsync('download', () => fetchAssets(filterRulesUrl, fallbackUrls)); const whitelistDomainSets = new Set(); const blacklistDomainSets = new Set(); diff --git a/Build/lib/parse-filter/hosts.ts b/Build/lib/parse-filter/hosts.ts index 4d8fcec6..6df00c40 100644 --- a/Build/lib/parse-filter/hosts.ts +++ b/Build/lib/parse-filter/hosts.ts @@ -1,5 +1,5 @@ import type { Span } from '../../trace'; -import { fetchAssetsWithout304 } from '../fetch-assets'; +import { fetchAssets } from '../fetch-assets'; import { fastNormalizeDomain } from '../normalize-domain'; import { processLine } from '../process-line'; import { onBlackFound } from './shared'; @@ -29,7 +29,7 @@ export function processHosts( hostsUrl: string, mirrors: string[] | null, includeAllSubDomain = false ) { return span.traceChildAsync(`process hosts: ${hostsUrl}`, async (span) => { - const text = await span.traceChild('download').traceAsyncFn(() => fetchAssetsWithout304(hostsUrl, mirrors)); + const text = await span.traceChild('download').traceAsyncFn(() => fetchAssets(hostsUrl, mirrors)); const domainSets: string[] = []; diff --git a/package.json b/package.json index 0d128db1..438165a3 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "fast-cidr-tools": "^0.3.1", "fast-fifo": "^1.3.2", "fdir": "^6.4.2", - "foxts": "^1.1.5", + "foxts": "^1.1.6", "hash-wasm": "^4.12.0", "json-stringify-pretty-compact": "^3.0.0", "picocolors": "^1.1.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 40b00875..0788a03b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -53,8 +53,8 @@ importers: specifier: ^6.4.2 version: 6.4.2(picomatch@4.0.2) foxts: - specifier: ^1.1.5 - version: 1.1.5 + specifier: ^1.1.6 + version: 1.1.6 hash-wasm: specifier: ^4.12.0 version: 4.12.0 @@ -1134,8 +1134,8 @@ packages: resolution: {integrity: sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==} engines: {node: '>= 6'} - foxts@1.1.5: - resolution: {integrity: sha512-s5SRvB7PTTOl5ZWYE4PrBhHXi6sw/0fLrYDDZThFHcHJCmEMaWmZwM0A5Op9APs6y0nQGrUgroV6WFKNdTbgSQ==} + foxts@1.1.6: + resolution: {integrity: sha512-O2UR/MDLo0w4igcFHwLn2KyXUD84P6bE3U4OpVsxvcYrWLFvvDO8zKLBS/o++tFJTCq7p/3USR48E8/dF2vAAQ==} fs-constants@1.0.0: resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==} @@ -2932,7 +2932,7 @@ snapshots: combined-stream: 1.0.8 mime-types: 2.1.35 - foxts@1.1.5: {} + foxts@1.1.6: {} fs-constants@1.0.0: {}