GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2025-12-12 01:00:34 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Chore: use more make-fetch-happen

Browse Source
This commit is contained in:
SukkaW 2024-10-21 02:09:25 +08:00
parent ed85377503
commit db7a4bc97a
6 changed files with 57 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
Build/build-apple-cdn.ts
View File

@ -2,19 +2,10 @@ import { parseFelixDnsmasqFromResp } from './lib/parse-dnsmasq';
import { task } from './trace';
import { SHARED_DESCRIPTION } from './lib/constants';
import { createMemoizedPromise } from './lib/memo-promise';
import { deserializeArray, fsFetchCache, serializeArray, getFileContentHash } from './lib/cache-filesystem';
import { DomainsetOutput } from './lib/create-file';
import { $fetch } from './lib/make-fetch-happen';
const url = 'https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf';
export const getAppleCdnDomainsPromise = createMemoizedPromise(() => fsFetchCache.applyWithHttp304(
url,
getFileContentHash(__filename),
parseFelixDnsmasqFromResp,
{
serializer: serializeArray,
deserializer: deserializeArray
}
));
export const getAppleCdnDomainsPromise = createMemoizedPromise(() => $fetch('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf').then(parseFelixDnsmasqFromResp));
export const buildAppleCdn = task(require.main === module, __filename)(async (span) => {
const res: string[] = await span.traceChildPromise('get apple cdn domains', getAppleCdnDomainsPromise());

26
Build/build-cdn-download-conf.ts
View File

@ -1,9 +1,8 @@
import path from 'node:path';
import { readFileIntoProcessedArray } from './lib/fetch-text-by-line';
import { createTrie } from './lib/trie';
import { fetchRemoteTextByLine, readFileIntoProcessedArray } from './lib/fetch-text-by-line';
import { HostnameTrie } from './lib/trie';
import { task } from './trace';
import { SHARED_DESCRIPTION } from './lib/constants';
import { getPublicSuffixListTextPromise } from './lib/download-publicsuffixlist';
import { appendArrayInPlace } from './lib/append-array-in-place';
import { SOURCE_DIR } from './constants/dir';
import { processLine } from './lib/process-line';
@ -11,16 +10,14 @@ import { DomainsetOutput } from './lib/create-file';
import { CRASHLYTICS_WHITELIST } from './constants/reject-data-source';
const getS3OSSDomainsPromise = (async (): Promise<string[]> => {
const trie = createTrie((await getPublicSuffixListTextPromise()).reduce<string[]>(
(acc, cur) => {
const tmp = processLine(cur);
if (tmp) {
acc.push(tmp);
}
return acc;
},
[]
));
const trie = new HostnameTrie();
for await (const line of await fetchRemoteTextByLine('https://publicsuffix.org/list/public_suffix_list.dat')) {
const tmp = processLine(line);
if (tmp) {
trie.add(tmp);
}
}
/**
* Extract OSS domain from publicsuffix list
@ -68,10 +65,11 @@ export const buildCdnDownloadConf = task(require.main === module, __filename)(as
readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/steam.conf'))
]);
// Move S3 domains to download domain set, since S3 files may be large
appendArrayInPlace(downloadDomainSet, S3OSSDomains.map(domain => `.${domain}`));
appendArrayInPlace(downloadDomainSet, steamDomainSet);
// we have whitelisted the crashlytics domain, but it doesn't mean we can't put it in CDN policy
// we have whitelisted the crashlytics domain, and we also want to put it in CDN policy
appendArrayInPlace(cdnDomainsList, CRASHLYTICS_WHITELIST);
return Promise.all([

34
Build/build-reject-ip-list.ts
View File

@ -8,33 +8,25 @@ import { fsFetchCache, getFileContentHash } from './lib/cache-filesystem';
import { processLine } from './lib/process-line';
import { RulesetOutput } from './lib/create-file';
import { SOURCE_DIR } from './constants/dir';
import { $fetch } from './lib/make-fetch-happen';
const BOGUS_NXDOMAIN_URL = 'https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/bogus-nxdomain.china.conf';
const getBogusNxDomainIPsPromise: Promise<[ipv4: string[], ipv6: string[]]> = $fetch(BOGUS_NXDOMAIN_URL).then(async (resp) => {
const ipv4: string[] = [];
const ipv6: string[] = [];
const getBogusNxDomainIPsPromise = fsFetchCache.applyWithHttp304(
BOGUS_NXDOMAIN_URL,
getFileContentHash(__filename),
async (resp) => {
const ipv4: string[] = [];
const ipv6: string[] = [];
for await (const line of createReadlineInterfaceFromResponse(resp)) {
if (line.startsWith('bogus-nxdomain=')) {
const ip = line.slice(15).trim();
if (isProbablyIpv4(ip)) {
ipv4.push(ip);
} else if (isProbablyIpv6(ip)) {
ipv6.push(ip);
}
for await (const line of createReadlineInterfaceFromResponse(resp)) {
if (line.startsWith('bogus-nxdomain=')) {
const ip = line.slice(15).trim();
if (isProbablyIpv4(ip)) {
ipv4.push(ip);
} else if (isProbablyIpv6(ip)) {
ipv6.push(ip);
}
}
return [ipv4, ipv6] as const;
},
{
serializer: JSON.stringify,
deserializer: JSON.parse
}
);
return [ipv4, ipv6] as const;
});
const BOTNET_FILTER_URL = 'https://malware-filter.pages.dev/botnet-filter-dnscrypt-blocked-ips.txt';
const BOTNET_FILTER_MIRROR_URL = [

14
Build/lib/cache-filesystem.ts
View File

@ -278,10 +278,6 @@ export class Cache<S = string> {
return fn(await fetchAssetsWithout304(primaryUrl, mirrorUrls));
}
if (mirrorUrls.length === 0) {
return this.applyWithHttp304(primaryUrl, extraCacheKey, async (resp) => fn(await resp.body.text()), opt);
}
const baseKey = primaryUrl + '$' + extraCacheKey;
const getETagKey = (url: string) => baseKey + '$' + url + '$etag';
const cachedKey = baseKey + '$cached';
@ -346,10 +342,12 @@ export class Cache<S = string> {
};
try {
const text = await Promise.any([
createFetchFallbackPromise(primaryUrl, -1),
...mirrorUrls.map(createFetchFallbackPromise)
]);
const text = mirrorUrls.length === 0
? await createFetchFallbackPromise(primaryUrl, -1)
: await Promise.any([
createFetchFallbackPromise(primaryUrl, -1),
...mirrorUrls.map(createFetchFallbackPromise)
]);
console.log(picocolors.yellow('[cache] miss'), primaryUrl);
const serializer = 'serializer' in opt ? opt.serializer : identity as any;

15
Build/lib/download-publicsuffixlist.ts
View File

@ -1,15 +0,0 @@
import { deserializeArray, fsFetchCache, getFileContentHash, serializeArray } from './cache-filesystem';
import { createMemoizedPromise } from './memo-promise';
export const getPublicSuffixListTextPromise = createMemoizedPromise(() => fsFetchCache.applyWithHttp304<string[]>(
'https://publicsuffix.org/list/public_suffix_list.dat',
getFileContentHash(__filename),
(r) => r.body.text().then(text => text.split('\n')),
{
// https://github.com/publicsuffix/list/blob/master/.github/workflows/tld-update.yml
// Though the action runs every 24 hours, the IANA list is updated every 7 days.
// So a 3 day TTL should be enough.
serializer: serializeArray,
deserializer: deserializeArray
}
));

49
Build/lib/fetch-retry.ts
View File

@ -7,7 +7,6 @@ import undici, {
import type {
Dispatcher,
RequestInit,
Response
} from 'undici';
@ -143,33 +142,33 @@ export const defaultRequestInit = {
}
};
export async function fetchWithLog(url: string, init?: RequestInit) {
try {
const res = await undici.fetch(url, init);
if (res.status >= 400) {
throw new ResponseError(res, url);
}
// export async function fetchWithLog(url: string, init?: RequestInit) {
// try {
// const res = await undici.fetch(url, init);
// if (res.status >= 400) {
// throw new ResponseError(res, url);
// }
if (!(res.status >= 200 && res.status <= 299) && res.status !== 304) {
throw new ResponseError(res, url);
}
// if (!(res.status >= 200 && res.status <= 299) && res.status !== 304) {
// throw new ResponseError(res, url);
// }
return res;
} catch (err: unknown) {
if (typeof err === 'object' && err !== null && 'name' in err) {
if ((
err.name === 'AbortError'
|| ('digest' in err && err.digest === 'AbortError')
)) {
console.log(picocolors.gray('[fetch abort]'), url);
}
} else {
console.log(picocolors.gray('[fetch fail]'), url, { name: (err as any).name }, err);
}
// return res;
// } catch (err: unknown) {
// if (typeof err === 'object' && err !== null && 'name' in err) {
// if ((
// err.name === 'AbortError'
// || ('digest' in err && err.digest === 'AbortError')
// )) {
// console.log(picocolors.gray('[fetch abort]'), url);
// }
// } else {
// console.log(picocolors.gray('[fetch fail]'), url, { name: (err as any).name }, err);
// }
throw err;
}
}
// throw err;
// }
// }
export async function requestWithLog(url: string, opt?: Parameters<typeof undici.request>[1]) {
try {