From db7a4bc97a93fae1c15569676903b9074d03ed1e Mon Sep 17 00:00:00 2001 From: SukkaW Date: Mon, 21 Oct 2024 02:09:25 +0800 Subject: [PATCH] Chore: use more make-fetch-happen --- Build/build-apple-cdn.ts | 13 ++----- Build/build-cdn-download-conf.ts | 26 +++++++------- Build/build-reject-ip-list.ts | 34 +++++++----------- Build/lib/cache-filesystem.ts | 14 ++++---- Build/lib/download-publicsuffixlist.ts | 15 -------- Build/lib/fetch-retry.ts | 49 +++++++++++++------------- 6 files changed, 57 insertions(+), 94 deletions(-) delete mode 100644 Build/lib/download-publicsuffixlist.ts diff --git a/Build/build-apple-cdn.ts b/Build/build-apple-cdn.ts index 130def55..893aca75 100644 --- a/Build/build-apple-cdn.ts +++ b/Build/build-apple-cdn.ts @@ -2,19 +2,10 @@ import { parseFelixDnsmasqFromResp } from './lib/parse-dnsmasq'; import { task } from './trace'; import { SHARED_DESCRIPTION } from './lib/constants'; import { createMemoizedPromise } from './lib/memo-promise'; -import { deserializeArray, fsFetchCache, serializeArray, getFileContentHash } from './lib/cache-filesystem'; import { DomainsetOutput } from './lib/create-file'; +import { $fetch } from './lib/make-fetch-happen'; -const url = 'https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf'; -export const getAppleCdnDomainsPromise = createMemoizedPromise(() => fsFetchCache.applyWithHttp304( - url, - getFileContentHash(__filename), - parseFelixDnsmasqFromResp, - { - serializer: serializeArray, - deserializer: deserializeArray - } -)); +export const getAppleCdnDomainsPromise = createMemoizedPromise(() => $fetch('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf').then(parseFelixDnsmasqFromResp)); export const buildAppleCdn = task(require.main === module, __filename)(async (span) => { const res: string[] = await span.traceChildPromise('get apple cdn domains', getAppleCdnDomainsPromise()); diff --git a/Build/build-cdn-download-conf.ts b/Build/build-cdn-download-conf.ts index c3d988d7..68718b0f 100644 --- a/Build/build-cdn-download-conf.ts +++ b/Build/build-cdn-download-conf.ts @@ -1,9 +1,8 @@ import path from 'node:path'; -import { readFileIntoProcessedArray } from './lib/fetch-text-by-line'; -import { createTrie } from './lib/trie'; +import { fetchRemoteTextByLine, readFileIntoProcessedArray } from './lib/fetch-text-by-line'; +import { HostnameTrie } from './lib/trie'; import { task } from './trace'; import { SHARED_DESCRIPTION } from './lib/constants'; -import { getPublicSuffixListTextPromise } from './lib/download-publicsuffixlist'; import { appendArrayInPlace } from './lib/append-array-in-place'; import { SOURCE_DIR } from './constants/dir'; import { processLine } from './lib/process-line'; @@ -11,16 +10,14 @@ import { DomainsetOutput } from './lib/create-file'; import { CRASHLYTICS_WHITELIST } from './constants/reject-data-source'; const getS3OSSDomainsPromise = (async (): Promise => { - const trie = createTrie((await getPublicSuffixListTextPromise()).reduce( - (acc, cur) => { - const tmp = processLine(cur); - if (tmp) { - acc.push(tmp); - } - return acc; - }, - [] - )); + const trie = new HostnameTrie(); + + for await (const line of await fetchRemoteTextByLine('https://publicsuffix.org/list/public_suffix_list.dat')) { + const tmp = processLine(line); + if (tmp) { + trie.add(tmp); + } + } /** * Extract OSS domain from publicsuffix list @@ -68,10 +65,11 @@ export const buildCdnDownloadConf = task(require.main === module, __filename)(as readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/steam.conf')) ]); + // Move S3 domains to download domain set, since S3 files may be large appendArrayInPlace(downloadDomainSet, S3OSSDomains.map(domain => `.${domain}`)); appendArrayInPlace(downloadDomainSet, steamDomainSet); - // we have whitelisted the crashlytics domain, but it doesn't mean we can't put it in CDN policy + // we have whitelisted the crashlytics domain, and we also want to put it in CDN policy appendArrayInPlace(cdnDomainsList, CRASHLYTICS_WHITELIST); return Promise.all([ diff --git a/Build/build-reject-ip-list.ts b/Build/build-reject-ip-list.ts index 180013de..02d07f06 100644 --- a/Build/build-reject-ip-list.ts +++ b/Build/build-reject-ip-list.ts @@ -8,33 +8,25 @@ import { fsFetchCache, getFileContentHash } from './lib/cache-filesystem'; import { processLine } from './lib/process-line'; import { RulesetOutput } from './lib/create-file'; import { SOURCE_DIR } from './constants/dir'; +import { $fetch } from './lib/make-fetch-happen'; const BOGUS_NXDOMAIN_URL = 'https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/bogus-nxdomain.china.conf'; +const getBogusNxDomainIPsPromise: Promise<[ipv4: string[], ipv6: string[]]> = $fetch(BOGUS_NXDOMAIN_URL).then(async (resp) => { + const ipv4: string[] = []; + const ipv6: string[] = []; -const getBogusNxDomainIPsPromise = fsFetchCache.applyWithHttp304( - BOGUS_NXDOMAIN_URL, - getFileContentHash(__filename), - async (resp) => { - const ipv4: string[] = []; - const ipv6: string[] = []; - - for await (const line of createReadlineInterfaceFromResponse(resp)) { - if (line.startsWith('bogus-nxdomain=')) { - const ip = line.slice(15).trim(); - if (isProbablyIpv4(ip)) { - ipv4.push(ip); - } else if (isProbablyIpv6(ip)) { - ipv6.push(ip); - } + for await (const line of createReadlineInterfaceFromResponse(resp)) { + if (line.startsWith('bogus-nxdomain=')) { + const ip = line.slice(15).trim(); + if (isProbablyIpv4(ip)) { + ipv4.push(ip); + } else if (isProbablyIpv6(ip)) { + ipv6.push(ip); } } - return [ipv4, ipv6] as const; - }, - { - serializer: JSON.stringify, - deserializer: JSON.parse } -); + return [ipv4, ipv6] as const; +}); const BOTNET_FILTER_URL = 'https://malware-filter.pages.dev/botnet-filter-dnscrypt-blocked-ips.txt'; const BOTNET_FILTER_MIRROR_URL = [ diff --git a/Build/lib/cache-filesystem.ts b/Build/lib/cache-filesystem.ts index 4750b204..a5c2d164 100644 --- a/Build/lib/cache-filesystem.ts +++ b/Build/lib/cache-filesystem.ts @@ -278,10 +278,6 @@ export class Cache { return fn(await fetchAssetsWithout304(primaryUrl, mirrorUrls)); } - if (mirrorUrls.length === 0) { - return this.applyWithHttp304(primaryUrl, extraCacheKey, async (resp) => fn(await resp.body.text()), opt); - } - const baseKey = primaryUrl + '$' + extraCacheKey; const getETagKey = (url: string) => baseKey + '$' + url + '$etag'; const cachedKey = baseKey + '$cached'; @@ -346,10 +342,12 @@ export class Cache { }; try { - const text = await Promise.any([ - createFetchFallbackPromise(primaryUrl, -1), - ...mirrorUrls.map(createFetchFallbackPromise) - ]); + const text = mirrorUrls.length === 0 + ? await createFetchFallbackPromise(primaryUrl, -1) + : await Promise.any([ + createFetchFallbackPromise(primaryUrl, -1), + ...mirrorUrls.map(createFetchFallbackPromise) + ]); console.log(picocolors.yellow('[cache] miss'), primaryUrl); const serializer = 'serializer' in opt ? opt.serializer : identity as any; diff --git a/Build/lib/download-publicsuffixlist.ts b/Build/lib/download-publicsuffixlist.ts deleted file mode 100644 index 8a39fec8..00000000 --- a/Build/lib/download-publicsuffixlist.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { deserializeArray, fsFetchCache, getFileContentHash, serializeArray } from './cache-filesystem'; -import { createMemoizedPromise } from './memo-promise'; - -export const getPublicSuffixListTextPromise = createMemoizedPromise(() => fsFetchCache.applyWithHttp304( - 'https://publicsuffix.org/list/public_suffix_list.dat', - getFileContentHash(__filename), - (r) => r.body.text().then(text => text.split('\n')), - { - // https://github.com/publicsuffix/list/blob/master/.github/workflows/tld-update.yml - // Though the action runs every 24 hours, the IANA list is updated every 7 days. - // So a 3 day TTL should be enough. - serializer: serializeArray, - deserializer: deserializeArray - } -)); diff --git a/Build/lib/fetch-retry.ts b/Build/lib/fetch-retry.ts index 58fbb028..c37f8e1e 100644 --- a/Build/lib/fetch-retry.ts +++ b/Build/lib/fetch-retry.ts @@ -7,7 +7,6 @@ import undici, { import type { Dispatcher, - RequestInit, Response } from 'undici'; @@ -143,33 +142,33 @@ export const defaultRequestInit = { } }; -export async function fetchWithLog(url: string, init?: RequestInit) { - try { - const res = await undici.fetch(url, init); - if (res.status >= 400) { - throw new ResponseError(res, url); - } +// export async function fetchWithLog(url: string, init?: RequestInit) { +// try { +// const res = await undici.fetch(url, init); +// if (res.status >= 400) { +// throw new ResponseError(res, url); +// } - if (!(res.status >= 200 && res.status <= 299) && res.status !== 304) { - throw new ResponseError(res, url); - } +// if (!(res.status >= 200 && res.status <= 299) && res.status !== 304) { +// throw new ResponseError(res, url); +// } - return res; - } catch (err: unknown) { - if (typeof err === 'object' && err !== null && 'name' in err) { - if (( - err.name === 'AbortError' - || ('digest' in err && err.digest === 'AbortError') - )) { - console.log(picocolors.gray('[fetch abort]'), url); - } - } else { - console.log(picocolors.gray('[fetch fail]'), url, { name: (err as any).name }, err); - } +// return res; +// } catch (err: unknown) { +// if (typeof err === 'object' && err !== null && 'name' in err) { +// if (( +// err.name === 'AbortError' +// || ('digest' in err && err.digest === 'AbortError') +// )) { +// console.log(picocolors.gray('[fetch abort]'), url); +// } +// } else { +// console.log(picocolors.gray('[fetch fail]'), url, { name: (err as any).name }, err); +// } - throw err; - } -} +// throw err; +// } +// } export async function requestWithLog(url: string, opt?: Parameters[1]) { try {