diff --git a/Build/build-internal-cdn-rules.ts b/Build/build-internal-cdn-rules.ts index 2b226985..8e783aae 100644 --- a/Build/build-internal-cdn-rules.ts +++ b/Build/build-internal-cdn-rules.ts @@ -10,69 +10,69 @@ import { getGorhillPublicSuffixPromise } from './lib/get-gorhill-publicsuffix'; const escapeRegExp = (string = '') => string.replaceAll(/[$()*+.?[\\\]^{|}]/g, '\\$&'); +const addApexDomain = (input: string, set: Set) => { + // We are including the private domains themselves + const d = tldts.getDomain(input, { allowPrivateDomains: false }); + if (d) { + set.add(d); + } +}; + +const processLocalDomainSet = async (domainSetPath: string, set: Set) => { + for await (const line of readFileByLine(domainSetPath)) { + // console.log({ line }); + + const parsed = tldts.parse(line, { allowPrivateDomains: true, detectIp: false }); + if (parsed.isIp) continue; + if (parsed.isIcann || parsed.isPrivate) { + if (parsed.domain) { + set.add(parsed.domain); + } + continue; + } + + if (processLine(line)) { + console.warn('[drop line from domainset]', line); + } + } +}; + +const processLocalRuleSet = async (ruleSetPath: string, set: Set, keywords: Set) => { + for await (const line of readFileByLine(ruleSetPath)) { + if (line.startsWith('DOMAIN-SUFFIX,')) { + addApexDomain(line.replace('DOMAIN-SUFFIX,', ''), set); + } else if (line.startsWith('DOMAIN,')) { + addApexDomain(line.replace('DOMAIN,', ''), set); + } else if (line.startsWith('DOMAIN-KEYWORD')) { + keywords.add(escapeRegExp(line.replace('DOMAIN-KEYWORD,', ''))); + } else if (line.startsWith('USER-AGENT,') || line.startsWith('PROCESS-NAME,') || line.startsWith('URL-REGEX,')) { + // do nothing + } else if (processLine(line)) { + console.warn('[drop line from ruleset]', line); + } + } +}; + export const buildInternalCDNDomains = task(import.meta.path, async () => { - const set = new Set(); - const keywords = new Set(); - - const addApexDomain = (input: string) => { - // We are including the private domains themselves - const d = tldts.getDomain(input, { allowPrivateDomains: false }); - if (d) { - set.add(d); - } - }; - - const processLocalDomainSet = async (domainSetPath: string) => { - for await (const line of readFileByLine(domainSetPath)) { - // console.log({ line }); - - const parsed = tldts.parse(line, { allowPrivateDomains: true, detectIp: false }); - if (parsed.isIp) continue; - if (parsed.isIcann || parsed.isPrivate) { - if (parsed.domain) { - set.add(parsed.domain); - } - continue; - } - - if (processLine(line)) { - console.warn('[drop line from domainset]', line); - } - } - }; - - const processLocalRuleSet = async (ruleSetPath: string) => { - for await (const line of readFileByLine(ruleSetPath)) { - if (line.startsWith('DOMAIN-SUFFIX,')) { - addApexDomain(line.replace('DOMAIN-SUFFIX,', '')); - } else if (line.startsWith('DOMAIN,')) { - addApexDomain(line.replace('DOMAIN,', '')); - } else if (line.startsWith('DOMAIN-KEYWORD')) { - keywords.add(escapeRegExp(line.replace('DOMAIN-KEYWORD,', ''))); - } else if (line.startsWith('USER-AGENT,') || line.startsWith('PROCESS-NAME,') || line.startsWith('URL-REGEX,')) { - // do nothing - } else if (processLine(line)) { - console.warn('[drop line from ruleset]', line); - } - } - }; + const proxySet = new Set(); + const proxyKeywords = new Set(); const gorhill = (await Promise.all([ getGorhillPublicSuffixPromise(), - processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/cdn.conf')), - processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/global.conf')), - processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/global_plus.conf')), - processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/my_proxy.conf')), - processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/stream.conf')), - processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/telegram.conf')), - processLocalDomainSet(path.resolve(import.meta.dir, '../List/domainset/cdn.conf')), - processLocalDomainSet(path.resolve(import.meta.dir, '../List/domainset/download.conf')) + processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/cdn.conf'), proxySet, proxyKeywords), + processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/global.conf'), proxySet, proxyKeywords), + processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/global_plus.conf'), proxySet, proxyKeywords), + processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/my_proxy.conf'), proxySet, proxyKeywords), + processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/stream.conf'), proxySet, proxyKeywords), + processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/telegram.conf'), proxySet, proxyKeywords), + processLocalDomainSet(path.resolve(import.meta.dir, '../List/domainset/cdn.conf'), proxySet), + processLocalDomainSet(path.resolve(import.meta.dir, '../List/domainset/download.conf'), proxySet) ]))[0]; return compareAndWriteFile( [ - ...sortDomains(Array.from(set), gorhill).map(i => `SUFFIX,${i}`), - ...Array.from(keywords).sort().map(i => `REGEX,${i}`) + ...sortDomains(Array.from(proxySet), gorhill).map(i => `SUFFIX,${i}`), + ...Array.from(proxyKeywords).sort().map(i => `REGEX,${i}`) ], path.resolve(import.meta.dir, '../List/internal/cdn.txt') ); diff --git a/Build/build-speedtest-domainset.ts b/Build/build-speedtest-domainset.ts index 6e5a9595..a19fbc70 100644 --- a/Build/build-speedtest-domainset.ts +++ b/Build/build-speedtest-domainset.ts @@ -24,6 +24,7 @@ const querySpeedtestApi = async (keyword: string): Promise> try { const randomUserAgent = topUserAgents[Math.floor(Math.random() * topUserAgents.length)]; const key = `fetch speedtest endpoints: ${keyword}`; + console.log(key); console.time(key); const res = await fetchWithRetry(`https://www.speedtest.net/api/js/servers?engine=js&search=${keyword}&limit=100`, { diff --git a/Build/build-sspanel-appprofile.ts b/Build/build-sspanel-appprofile.ts index 110b7790..3ed8eebc 100644 --- a/Build/build-sspanel-appprofile.ts +++ b/Build/build-sspanel-appprofile.ts @@ -1,5 +1,3 @@ -import fsp from 'fs/promises'; - import { getAppleCdnDomainsPromise } from './build-apple-cdn'; import { getDomesticDomainsRulesetPromise } from './build-domestic-ruleset'; import { surgeRulesetToClashClassicalTextRuleset } from './lib/clash'; diff --git a/Build/build-telegram-cidr.ts b/Build/build-telegram-cidr.ts index d7a1514e..e42798f4 100644 --- a/Build/build-telegram-cidr.ts +++ b/Build/build-telegram-cidr.ts @@ -10,7 +10,7 @@ import { SHARED_DESCRIPTION } from './lib/constants'; import { createMemoizedPromise } from './lib/memo-promise'; export const getTelegramCIDRPromise = createMemoizedPromise(async () => { - const resp = await fetchWithRetry('https://core.telegram.org/resources/cidr.txt', defaultRequestInit) as Response; + const resp = await fetchWithRetry('https://core.telegram.org/resources/cidr.txt', defaultRequestInit); const lastModified = resp.headers.get('last-modified'); const date = lastModified ? new Date(lastModified) : new Date(); diff --git a/Build/download-mock-assets.ts b/Build/download-mock-assets.ts index 435a9327..9cee2264 100644 --- a/Build/download-mock-assets.ts +++ b/Build/download-mock-assets.ts @@ -18,7 +18,7 @@ export const downloadMockAssets = task(import.meta.path, () => Promise.all(Objec const key = picocolors.gray(`Download ${filename}`); console.time(key); const res = await fetchWithRetry(url); - await Bun.write(targetPath, res as Response); + await Bun.write(targetPath, res); console.timeEnd(key); }))); diff --git a/Build/download-previous-build.ts b/Build/download-previous-build.ts index 814e69cc..a39d7269 100644 --- a/Build/download-previous-build.ts +++ b/Build/download-previous-build.ts @@ -48,40 +48,37 @@ export const downloadPreviousBuild = task(import.meta.path, async () => { const filesList = buildOutputList.map(f => path.join('ruleset.skk.moe-master', f)); - await traceAsync( - 'Download and extract previous build', - async () => { - const resp = await fetchWithRetry('https://codeload.github.com/sukkalab/ruleset.skk.moe/tar.gz/master', defaultRequestInit); + const resp = await fetchWithRetry('https://codeload.github.com/sukkalab/ruleset.skk.moe/tar.gz/master', defaultRequestInit); - if (!resp.body) { - throw new Error('Download previous build failed! No body found'); - } + if (!resp.body) { + throw new Error('Download previous build failed! No body found'); + } - const extract = tarStream.extract(); - Readable.fromWeb(resp.body).pipe(zlib.createGunzip()).pipe(extract); + const extract = tarStream.extract(); + Readable.fromWeb(resp.body as any).pipe(zlib.createGunzip()).pipe(extract); - for await (const entry of extract) { - if (entry.header.type !== 'file') { - entry.resume(); // Drain the entry - continue; - } - // filter entry - if (!filesList.some(f => entry.header.name.startsWith(f))) { - entry.resume(); // Drain the entry - continue; - } + const pathPrefix = `ruleset.skk.moe-master${path.sep}`; - const relativeEntryPath = entry.header.name.replace(`ruleset.skk.moe-master${path.sep}`, ''); - const targetPath = path.join(import.meta.dir, '..', relativeEntryPath); - - await fsp.mkdir(path.dirname(targetPath), { recursive: true }); - await pipeline( - entry, - fs.createWriteStream(targetPath) - ); - } + for await (const entry of extract) { + if (entry.header.type !== 'file') { + entry.resume(); // Drain the entry + continue; } - ); + // filter entry + if (!filesList.some(f => entry.header.name.startsWith(f))) { + entry.resume(); // Drain the entry + continue; + } + + const relativeEntryPath = entry.header.name.replace(pathPrefix, ''); + const targetPath = path.join(import.meta.dir, '..', relativeEntryPath); + + await fsp.mkdir(path.dirname(targetPath), { recursive: true }); + await pipeline( + entry, + fs.createWriteStream(targetPath) + ); + } }); if (import.meta.main) { diff --git a/Build/lib/fetch-retry.ts b/Build/lib/fetch-retry.ts index 2a822f6b..ace61eec 100644 --- a/Build/lib/fetch-retry.ts +++ b/Build/lib/fetch-retry.ts @@ -60,7 +60,8 @@ function createFetchRetry($fetch: typeof fetch): FetchWithRetry { retries: MAX_RETRIES, factor: FACTOR, maxRetryAfter: MAX_RETRY_AFTER, - retryOnAborted: false + retryOnAborted: false, + retryOnNon2xx: true }, opts.retry ); @@ -85,6 +86,9 @@ function createFetchRetry($fetch: typeof fetch): FetchWithRetry { } throw new ResponseError(res); } else { + if (!res.ok && retryOpts.retryOnNon2xx) { + throw new ResponseError(res); + } return res; } } catch (err: unknown) { diff --git a/Build/lib/fetch-text-by-line.ts b/Build/lib/fetch-text-by-line.ts index 4f372481..a82e94cc 100644 --- a/Build/lib/fetch-text-by-line.ts +++ b/Build/lib/fetch-text-by-line.ts @@ -79,5 +79,5 @@ export async function *createReadlineInterfaceFromResponse(resp: Response): Asyn } export function fetchRemoteTextByLine(url: string | URL) { - return fetchWithRetry(url, defaultRequestInit).then(res => createReadlineInterfaceFromResponse(res as Response)); + return fetchWithRetry(url, defaultRequestInit).then(res => createReadlineInterfaceFromResponse(res)); } diff --git a/Build/lib/get-phishing-domains.ts b/Build/lib/get-phishing-domains.ts index e8c978c7..3d4aa720 100644 --- a/Build/lib/get-phishing-domains.ts +++ b/Build/lib/get-phishing-domains.ts @@ -1,5 +1,3 @@ -import fsp from 'fs/promises'; -import path from 'path'; import { getGorhillPublicSuffixPromise } from './get-gorhill-publicsuffix'; import { processDomainLists, processHosts } from './parse-filter'; import { traceAsync, traceSync } from './trace-runner';