From 5e40f2cbe083a45da5ac83a226efe4b03753d34d Mon Sep 17 00:00:00 2001 From: SukkaW Date: Fri, 10 Jan 2025 00:50:54 +0800 Subject: [PATCH] Use undici fetch --- Build/build-apple-cdn.ts | 4 ++-- Build/build-reject-ip-list.ts | 4 ++-- Build/build-speedtest-domainset.ts | 15 +++++++-------- Build/build-telegram-cidr.ts | 4 ++-- Build/download-mock-assets.ts | 4 ++-- Build/lib/convert-clash-meta-mrs.ts | 4 ++-- Build/lib/fetch-text-by-line.ts | 4 ++-- Build/validate-domestic.ts | 6 +++--- 8 files changed, 22 insertions(+), 23 deletions(-) diff --git a/Build/build-apple-cdn.ts b/Build/build-apple-cdn.ts index 419afa46..e9d30a96 100644 --- a/Build/build-apple-cdn.ts +++ b/Build/build-apple-cdn.ts @@ -3,9 +3,9 @@ import { task } from './trace'; import { SHARED_DESCRIPTION } from './constants/description'; import { createMemoizedPromise } from './lib/memo-promise'; import { DomainsetOutput } from './lib/create-file'; -import { $fetch } from './lib/make-fetch-happen'; +import { $$fetch } from './lib/fetch-retry'; -export const getAppleCdnDomainsPromise = createMemoizedPromise(() => $fetch('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf').then(parseFelixDnsmasqFromResp)); +export const getAppleCdnDomainsPromise = createMemoizedPromise(() => $$fetch('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf').then(parseFelixDnsmasqFromResp)); export const buildAppleCdn = task(require.main === module, __filename)(async (span) => { const res: string[] = await span.traceChildPromise('get apple cdn domains', getAppleCdnDomainsPromise()); diff --git a/Build/build-reject-ip-list.ts b/Build/build-reject-ip-list.ts index 3a42d86b..00c7a3de 100644 --- a/Build/build-reject-ip-list.ts +++ b/Build/build-reject-ip-list.ts @@ -8,10 +8,10 @@ import { fsFetchCache, getFileContentHash } from './lib/cache-filesystem'; import { processLine } from './lib/process-line'; import { RulesetOutput } from './lib/create-file'; import { SOURCE_DIR } from './constants/dir'; -import { $fetch } from './lib/make-fetch-happen'; +import { $$fetch } from './lib/fetch-retry'; const BOGUS_NXDOMAIN_URL = 'https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/bogus-nxdomain.china.conf'; -const getBogusNxDomainIPsPromise: Promise<[ipv4: string[], ipv6: string[]]> = $fetch(BOGUS_NXDOMAIN_URL).then(async (resp) => { +const getBogusNxDomainIPsPromise: Promise<[ipv4: string[], ipv6: string[]]> = $$fetch(BOGUS_NXDOMAIN_URL).then(async (resp) => { const ipv4: string[] = []; const ipv6: string[] = []; diff --git a/Build/build-speedtest-domainset.ts b/Build/build-speedtest-domainset.ts index ef34c4fa..475866be 100644 --- a/Build/build-speedtest-domainset.ts +++ b/Build/build-speedtest-domainset.ts @@ -2,13 +2,13 @@ import path from 'node:path'; import tldts from 'tldts-experimental'; import { task } from './trace'; -import { $fetch } from './lib/make-fetch-happen'; import { SHARED_DESCRIPTION } from './constants/description'; import { readFileIntoProcessedArray } from './lib/fetch-text-by-line'; import { DomainsetOutput } from './lib/create-file'; import { OUTPUT_SURGE_DIR, SOURCE_DIR } from './constants/dir'; import { newQueue } from '@henrygd/queue'; +import { $$fetch } from './lib/fetch-retry'; const KEYWORDS = [ 'Hong Kong', @@ -44,6 +44,9 @@ const KEYWORDS = [ const s = newQueue(2); +const latestTopUserAgentsPromise = $$fetch('https://raw.githubusercontent.com/microlinkhq/top-user-agents/master/src/desktop.json') + .then(res => res.json() as Promise) + .then((userAgents: string[]) => userAgents.filter(ua => ua.startsWith('Mozilla/5.0 '))); const getSpeedtestHostsGroupsPromise = Promise.all(KEYWORDS.flatMap(querySpeedtestApi)); export const buildSpeedtestDomainSet = task(require.main === module, __filename)(async (span) => { @@ -74,10 +77,6 @@ export const buildSpeedtestDomainSet = task(require.main === module, __filename) return output.write(); }); -const latestTopUserAgentsPromise = $fetch('https://raw.githubusercontent.com/microlinkhq/top-user-agents/master/src/desktop.json') - .then(res => res.json()) - .then((userAgents: string[]) => userAgents.filter(ua => ua.startsWith('Mozilla/5.0 '))); - async function querySpeedtestApi(keyword: string) { const topUserAgents = await latestTopUserAgentsPromise; @@ -86,7 +85,7 @@ async function querySpeedtestApi(keyword: string) { try { const randomUserAgent = topUserAgents[Math.floor(Math.random() * topUserAgents.length)]; - const data = await s.add>(() => $fetch(url, { + const data = await s.add>(() => $$fetch(url, { headers: { dnt: '1', Referer: 'https://www.speedtest.net/', @@ -103,8 +102,8 @@ async function querySpeedtestApi(keyword: string) { } : {}) }, - timeout: 1000 * 60 - }).then(res => res.json())); + signal: AbortSignal.timeout(1000 * 60) + }).then(res => res.json() as any)); return data.reduce( (prev, cur) => { diff --git a/Build/build-telegram-cidr.ts b/Build/build-telegram-cidr.ts index ce5251bd..4f10f9e6 100644 --- a/Build/build-telegram-cidr.ts +++ b/Build/build-telegram-cidr.ts @@ -5,10 +5,10 @@ import { task } from './trace'; import { SHARED_DESCRIPTION } from './constants/description'; import { createMemoizedPromise } from './lib/memo-promise'; import { RulesetOutput } from './lib/create-file'; -import { $fetch } from './lib/make-fetch-happen'; +import { $$fetch } from './lib/fetch-retry'; export const getTelegramCIDRPromise = createMemoizedPromise(async () => { - const resp = await $fetch('https://core.telegram.org/resources/cidr.txt'); + const resp = await $$fetch('https://core.telegram.org/resources/cidr.txt'); const lastModified = resp.headers.get('last-modified'); const date = lastModified ? new Date(lastModified) : new Date(); diff --git a/Build/download-mock-assets.ts b/Build/download-mock-assets.ts index edcecca1..c28e6c48 100644 --- a/Build/download-mock-assets.ts +++ b/Build/download-mock-assets.ts @@ -4,7 +4,7 @@ import fs from 'node:fs'; import { pipeline } from 'node:stream/promises'; import { OUTPUT_MOCK_DIR } from './constants/dir'; import { mkdirp } from './lib/misc'; -import { $fetch } from './lib/make-fetch-happen'; +import { $$fetch } from './lib/fetch-retry'; const ASSETS_LIST = { 'www-google-analytics-com_ga.js': 'https://unpkg.com/@adguard/scriptlets@latest/dist/redirect-files/google-analytics-ga.js', @@ -23,7 +23,7 @@ export const downloadMockAssets = task(require.main === module, __filename)(asyn return Promise.all(Object.entries(ASSETS_LIST).map( ([filename, url]) => span .traceChildAsync(url, async () => { - const res = await $fetch(url); + const res = await $$fetch(url); if (!res.ok) { console.error(`Failed to download ${url}`); diff --git a/Build/lib/convert-clash-meta-mrs.ts b/Build/lib/convert-clash-meta-mrs.ts index 7860d485..60dfa130 100644 --- a/Build/lib/convert-clash-meta-mrs.ts +++ b/Build/lib/convert-clash-meta-mrs.ts @@ -7,7 +7,7 @@ import process from 'node:process'; import { exec } from 'tinyexec'; import { mkdirp } from './misc'; -import { $fetch } from './make-fetch-happen'; +import { $$fetch } from './fetch-retry'; const mihomoBinaryDir = path.join(__dirname, '../../node_modules/.cache/mihomo'); const mihomoBinaryPath = path.join(mihomoBinaryDir, 'mihomo'); @@ -32,7 +32,7 @@ async function ensureMihomoBinary() { throw new Error(`Unsupported platform: ${process.platform} ${process.arch}`); } - const res = await $fetch(downloadUrl); + const res = await $$fetch(downloadUrl); if (!res.ok || !res.body) { throw new Error(`Failed to download mihomo binary: ${res.statusText}`); diff --git a/Build/lib/fetch-text-by-line.ts b/Build/lib/fetch-text-by-line.ts index cec59c87..bbef23c1 100644 --- a/Build/lib/fetch-text-by-line.ts +++ b/Build/lib/fetch-text-by-line.ts @@ -8,8 +8,8 @@ import { TextLineStream } from './text-line-transform-stream'; import type { ReadableStream } from 'node:stream/web'; import { TextDecoderStream } from 'node:stream/web'; import { processLine, ProcessLineStream } from './process-line'; -import { $fetch } from './make-fetch-happen'; import type { NodeFetchResponse } from './make-fetch-happen'; +import { $$fetch } from './fetch-retry'; import type { UndiciResponseData } from './fetch-retry'; import type { Response as UnidiciWebResponse } from 'undici'; @@ -70,7 +70,7 @@ export const createReadlineInterfaceFromResponse: ((resp: NodeFetchResponse | Un }; export function fetchRemoteTextByLine(url: string, processLine = false): Promise> { - return $fetch(url).then(resp => createReadlineInterfaceFromResponse(resp, processLine)); + return $$fetch(url).then(resp => createReadlineInterfaceFromResponse(resp, processLine)); } export async function readFileIntoProcessedArray(file: string /* | FileHandle */) { diff --git a/Build/validate-domestic.ts b/Build/validate-domestic.ts index 60f0ac5e..2e5c0a95 100644 --- a/Build/validate-domestic.ts +++ b/Build/validate-domestic.ts @@ -4,15 +4,15 @@ import { HostnameSmolTrie } from './lib/trie'; import path from 'node:path'; import { processLine } from './lib/process-line'; import { SOURCE_DIR } from './constants/dir'; -import { $fetch } from './lib/make-fetch-happen'; import { parseFelixDnsmasqFromResp } from './lib/parse-dnsmasq'; +import { $$fetch } from './lib/fetch-retry'; export async function parseDomesticList() { - const trie = new HostnameSmolTrie(await parseFelixDnsmasqFromResp(await $fetch('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf'))); + const trie = new HostnameSmolTrie(await parseFelixDnsmasqFromResp(await $$fetch('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf'))); const top5000 = new Set(); - const res = await (await $fetch('https://radar.cloudflare.com/charts/LargerTopDomainsTable/attachment?id=1077&top=10000', { + const res = await (await $$fetch('https://radar.cloudflare.com/charts/LargerTopDomainsTable/attachment?id=1077&top=10000', { headers: { accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', 'accept-language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7,zh-TW;q=0.6,es;q=0.5',