From 9bb0c14d5fa132902861365933ec0316359396e5 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Sun, 14 Jan 2024 22:10:16 +0800 Subject: [PATCH] Refactor: full span tracer --- Build/build-anti-bogus-domain.ts | 7 +- Build/build-apple-cdn.ts | 8 +- Build/build-cdn-conf.ts | 7 +- Build/build-chn-cidr.ts | 4 +- Build/build-cloudmounter-rules.ts | 7 +- Build/build-common.ts | 92 +++++++++++++---------- Build/build-domestic-ruleset.ts | 6 +- Build/build-internal-cdn-rules.ts | 3 +- Build/build-microsoft-cdn.ts | 7 +- Build/build-reject-domainset.ts | 22 +++--- Build/build-sgmodule-always-realip.ts | 3 +- Build/build-sgmodule-redirect.ts | 3 +- Build/build-speedtest-domainset.ts | 18 +++-- Build/build-sspanel-appprofile.ts | 3 +- Build/build-stream-service.ts | 26 ++++--- Build/build-telegram-cidr.ts | 7 +- Build/download-mock-assets.ts | 14 ++-- Build/lib/create-file.ts | 103 +++++++++++++------------- Build/lib/fetch-text-by-line.ts | 5 +- Build/lib/parse-filter.ts | 23 +++--- Build/lib/reject-data-source.ts | 6 +- Build/trace/index.ts | 46 +++++++++--- Source/non_ip/ai.conf | 1 + 23 files changed, 238 insertions(+), 183 deletions(-) diff --git a/Build/build-anti-bogus-domain.ts b/Build/build-anti-bogus-domain.ts index 023091b5..07cd0a84 100644 --- a/Build/build-anti-bogus-domain.ts +++ b/Build/build-anti-bogus-domain.ts @@ -35,7 +35,7 @@ const getBogusNxDomainIPs = async () => { ); }; -export const buildAntiBogusDomain = task(import.meta.path, async () => { +export const buildAntiBogusDomain = task(import.meta.path, async (span) => { const bogusIpPromise = getBogusNxDomainIPs(); const result: string[] = []; @@ -57,7 +57,8 @@ export const buildAntiBogusDomain = task(import.meta.path, async () => { ' - https://github.com/felixonmars/dnsmasq-china-list' ]; - return Promise.all(createRuleset( + return createRuleset( + span, 'Sukka\'s Ruleset - Anti Bogus Domain', description, new Date(), @@ -65,7 +66,7 @@ export const buildAntiBogusDomain = task(import.meta.path, async () => { 'ruleset', path.resolve(import.meta.dir, '../List/ip/reject.conf'), path.resolve(import.meta.dir, '../Clash/ip/reject.txt') - )); + ); }); if (import.meta.main) { diff --git a/Build/build-apple-cdn.ts b/Build/build-apple-cdn.ts index 0a739c90..68bcb559 100644 --- a/Build/build-apple-cdn.ts +++ b/Build/build-apple-cdn.ts @@ -23,7 +23,7 @@ export const getAppleCdnDomainsPromise = createMemoizedPromise(() => traceAsync( picocolors.gray )); -export const buildAppleCdn = task(import.meta.path, async () => { +export const buildAppleCdn = task(import.meta.path, async (span) => { const res = await getAppleCdnDomainsPromise(); const description = [ @@ -39,7 +39,8 @@ export const buildAppleCdn = task(import.meta.path, async () => { const domainset = res.map(i => `.${i}`); return Promise.all([ - ...createRuleset( + createRuleset( + span, 'Sukka\'s Ruleset - Apple CDN', description, new Date(), @@ -48,7 +49,8 @@ export const buildAppleCdn = task(import.meta.path, async () => { path.resolve(import.meta.dir, '../List/non_ip/apple_cdn.conf'), path.resolve(import.meta.dir, '../Clash/non_ip/apple_cdn.txt') ), - ...createRuleset( + createRuleset( + span, 'Sukka\'s Ruleset - Apple CDN', description, new Date(), diff --git a/Build/build-cdn-conf.ts b/Build/build-cdn-conf.ts index 0c9c24ca..563cbadd 100644 --- a/Build/build-cdn-conf.ts +++ b/Build/build-cdn-conf.ts @@ -41,7 +41,7 @@ const getS3OSSDomains = async (): Promise> => { return S3OSSDomains; }; -const buildCdnConf = task(import.meta.path, async () => { +const buildCdnConf = task(import.meta.path, async (span) => { /** @type {string[]} */ const cdnDomainsList: string[] = []; @@ -62,7 +62,8 @@ const buildCdnConf = task(import.meta.path, async () => { 'This file contains object storage and static assets CDN domains.' ]; - return Promise.all(createRuleset( + return createRuleset( + span, 'Sukka\'s Ruleset - CDN Domains', description, new Date(), @@ -70,7 +71,7 @@ const buildCdnConf = task(import.meta.path, async () => { 'ruleset', path.resolve(import.meta.dir, '../List/non_ip/cdn.conf'), path.resolve(import.meta.dir, '../Clash/non_ip/cdn.txt') - )); + ); }); export { buildCdnConf }; diff --git a/Build/build-chn-cidr.ts b/Build/build-chn-cidr.ts index 89a8ec3a..3d480903 100644 --- a/Build/build-chn-cidr.ts +++ b/Build/build-chn-cidr.ts @@ -32,7 +32,7 @@ export const getChnCidrPromise = createMemoizedPromise(async () => { ); }); -export const buildChnCidr = task(import.meta.path, async () => { +export const buildChnCidr = task(import.meta.path, async (span) => { const filteredCidr = await getChnCidrPromise(); // Can not use SHARED_DESCRIPTION here as different license @@ -47,6 +47,7 @@ export const buildChnCidr = task(import.meta.path, async () => { // Can not use createRuleset here, as Clash support advanced ipset syntax return Promise.all([ compareAndWriteFile( + span, withBannerArray( 'Sukka\'s Ruleset - Mainland China IPv4 CIDR', description, @@ -56,6 +57,7 @@ export const buildChnCidr = task(import.meta.path, async () => { pathResolve(import.meta.dir, '../List/ip/china_ip.conf') ), compareAndWriteFile( + span, withBannerArray( 'Sukka\'s Ruleset - Mainland China IPv4 CIDR', description, diff --git a/Build/build-cloudmounter-rules.ts b/Build/build-cloudmounter-rules.ts index 77b251f5..cd0481d1 100644 --- a/Build/build-cloudmounter-rules.ts +++ b/Build/build-cloudmounter-rules.ts @@ -7,7 +7,7 @@ import { task } from './trace'; const outputSurgeDir = path.resolve(import.meta.dir, '../List'); const outputClashDir = path.resolve(import.meta.dir, '../Clash'); -export const buildCloudMounterRules = task(import.meta.path, async () => { +export const buildCloudMounterRules = task(import.meta.path, async (span) => { // AND,((SRC-IP,192.168.1.110), (DOMAIN, example.com)) const results = DOMAINS.flatMap(domain => { @@ -18,7 +18,8 @@ export const buildCloudMounterRules = task(import.meta.path, async () => { const description = SHARED_DESCRIPTION; - return Promise.all(createRuleset( + return createRuleset( + span, 'Sukka\'s Ruleset - CloudMounter / RaiDrive', description, new Date(), @@ -26,7 +27,7 @@ export const buildCloudMounterRules = task(import.meta.path, async () => { 'domainset', path.resolve(outputSurgeDir, 'non_ip', 'cloudmounter.conf'), path.resolve(outputClashDir, 'non_ip', 'cloudmounter.txt') - )); + ); }); if (import.meta.main) { diff --git a/Build/build-common.ts b/Build/build-common.ts index c7b3b525..249b2d62 100644 --- a/Build/build-common.ts +++ b/Build/build-common.ts @@ -6,6 +6,7 @@ import { readFileByLine } from './lib/fetch-text-by-line'; import { processLine } from './lib/process-line'; import { createRuleset } from './lib/create-file'; import { domainDeduper } from './lib/domain-deduper'; +import type { Span } from './trace'; import { task } from './trace'; import { SHARED_DESCRIPTION } from './lib/constants'; @@ -17,7 +18,7 @@ const sourceDir = path.resolve(import.meta.dir, '../Source'); const outputSurgeDir = path.resolve(import.meta.dir, '../List'); const outputClashDir = path.resolve(import.meta.dir, '../Clash'); -export const buildCommon = task(import.meta.path, async () => { +export const buildCommon = task(import.meta.path, async (span) => { const promises: Array> = []; const pw = new PathScurry(sourceDir); @@ -33,14 +34,14 @@ export const buildCommon = task(import.meta.path, async () => { const relativePath = entry.relative(); if (relativePath.startsWith('domainset/')) { - promises.push(transformDomainset(entry.fullpath(), relativePath)); + promises.push(transformDomainset(span, entry.fullpath(), relativePath)); continue; } if ( relativePath.startsWith('ip/') || relativePath.startsWith('non_ip/') ) { - promises.push(transformRuleset(entry.fullpath(), relativePath)); + promises.push(transformRuleset(span, entry.fullpath(), relativePath)); continue; } } @@ -52,46 +53,50 @@ if (import.meta.main) { buildCommon(); } -const processFile = async (sourcePath: string) => { - console.log('Processing', sourcePath); +const processFile = (span: Span, sourcePath: string) => { + // console.log('Processing', sourcePath); + return span.traceChild(`process file: ${sourcePath}`).traceAsyncFn(async () => { + const lines: string[] = []; - const lines: string[] = []; + let title = ''; + const descriptions: string[] = []; - let title = ''; - const descriptions: string[] = []; + try { + for await (const line of readFileByLine(sourcePath)) { + if (line === MAGIC_COMMAND_SKIP) { + return null; + } - try { - for await (const line of readFileByLine(sourcePath)) { - if (line === MAGIC_COMMAND_SKIP) { - return; - } - - if (line.startsWith(MAGIC_COMMAND_TITLE)) { - title = line.slice(MAGIC_COMMAND_TITLE.length).trim(); - continue; - } - - if (line.startsWith(MAGIC_COMMAND_DESCRIPTION)) { - descriptions.push(line.slice(MAGIC_COMMAND_DESCRIPTION.length).trim()); - continue; - } - - const l = processLine(line); - if (l) { - lines.push(l); + if (line.startsWith(MAGIC_COMMAND_TITLE)) { + title = line.slice(MAGIC_COMMAND_TITLE.length).trim(); + continue; + } + + if (line.startsWith(MAGIC_COMMAND_DESCRIPTION)) { + descriptions.push(line.slice(MAGIC_COMMAND_DESCRIPTION.length).trim()); + continue; + } + + const l = processLine(line); + if (l) { + lines.push(l); + } } + } catch (e) { + console.error('Error processing', sourcePath); + console.trace(e); } - } catch (e) { - console.error('Error processing', sourcePath); - console.trace(e); - } - return [title, descriptions, lines] as const; + return [title, descriptions, lines] as const; + }); }; -async function transformDomainset(sourcePath: string, relativePath: string) { - const res = await processFile(sourcePath); +async function transformDomainset(parentSpan: Span, sourcePath: string, relativePath: string) { + const span = parentSpan.traceChild(`transform domainset: ${path.basename(sourcePath, path.extname(sourcePath))}`); + + const res = await processFile(span, sourcePath); if (!res) return; + const [title, descriptions, lines] = res; const deduped = domainDeduper(lines); @@ -104,7 +109,8 @@ async function transformDomainset(sourcePath: string, relativePath: string) { ) ]; - return Promise.all(createRuleset( + return createRuleset( + span, title, description, new Date(), @@ -112,15 +118,18 @@ async function transformDomainset(sourcePath: string, relativePath: string) { 'domainset', path.resolve(outputSurgeDir, relativePath), path.resolve(outputClashDir, `${relativePath.slice(0, -path.extname(relativePath).length)}.txt`) - )); + ); } /** * Output Surge RULE-SET and Clash classical text format */ -async function transformRuleset(sourcePath: string, relativePath: string) { - const res = await processFile(sourcePath); - if (!res) return; +async function transformRuleset(parentSpan: Span, sourcePath: string, relativePath: string) { + const span = parentSpan.traceChild(`transform ruleset: ${path.basename(sourcePath, path.extname(sourcePath))}`); + + const res = await processFile(span, sourcePath); + if (!res) return null; + const [title, descriptions, lines] = res; const description = [ @@ -132,7 +141,8 @@ async function transformRuleset(sourcePath: string, relativePath: string) { ) ]; - return Promise.all(createRuleset( + return createRuleset( + span, title, description, new Date(), @@ -140,5 +150,5 @@ async function transformRuleset(sourcePath: string, relativePath: string) { 'ruleset', path.resolve(outputSurgeDir, relativePath), path.resolve(outputClashDir, `${relativePath.slice(0, -path.extname(relativePath).length)}.txt`) - )); + ); } diff --git a/Build/build-domestic-ruleset.ts b/Build/build-domestic-ruleset.ts index fa929516..7a1970bd 100644 --- a/Build/build-domestic-ruleset.ts +++ b/Build/build-domestic-ruleset.ts @@ -21,7 +21,7 @@ export const getDomesticDomainsRulesetPromise = createMemoizedPromise(async () = return results; }); -export const buildDomesticRuleset = task(import.meta.path, async () => { +export const buildDomesticRuleset = task(import.meta.path, async (span) => { const rulesetDescription = [ ...SHARED_DESCRIPTION, '', @@ -29,7 +29,8 @@ export const buildDomesticRuleset = task(import.meta.path, async () => { ]; return Promise.all([ - ...createRuleset( + createRuleset( + span, 'Sukka\'s Ruleset - Domestic Domains', rulesetDescription, new Date(), @@ -39,6 +40,7 @@ export const buildDomesticRuleset = task(import.meta.path, async () => { path.resolve(import.meta.dir, '../Clash/non_ip/domestic.txt') ), compareAndWriteFile( + span, [ '#!name=[Sukka] Local DNS Mapping', `#!desc=Last Updated: ${new Date().toISOString()}`, diff --git a/Build/build-internal-cdn-rules.ts b/Build/build-internal-cdn-rules.ts index c4160da7..91621101 100644 --- a/Build/build-internal-cdn-rules.ts +++ b/Build/build-internal-cdn-rules.ts @@ -33,7 +33,7 @@ const processLocalRuleSet = async (ruleSetPath: string, set: Set, keywor } }; -export const buildInternalCDNDomains = task(import.meta.path, async () => { +export const buildInternalCDNDomains = task(import.meta.path, async (span) => { const proxySet = new Set(); const proxyKeywords = new Set(); @@ -50,6 +50,7 @@ export const buildInternalCDNDomains = task(import.meta.path, async () => { ]))[0]; return compareAndWriteFile( + span, [ ...sortDomains(Array.from(proxySet), gorhill).map(i => `SUFFIX,${i}`), ...Array.from(proxyKeywords).sort().map(i => `REGEX,${i}`) diff --git a/Build/build-microsoft-cdn.ts b/Build/build-microsoft-cdn.ts index b8e881e3..70a7e3f0 100644 --- a/Build/build-microsoft-cdn.ts +++ b/Build/build-microsoft-cdn.ts @@ -41,7 +41,7 @@ export const getMicrosoftCdnRulesetPromise = createMemoizedPromise(async () => { return Array.from(set).map(d => `DOMAIN-SUFFIX,${d}`).concat(WHITELIST); }); -export const buildMicrosoftCdn = task(import.meta.path, async () => { +export const buildMicrosoftCdn = task(import.meta.path, async (span) => { const description = [ ...SHARED_DESCRIPTION, '', @@ -51,7 +51,8 @@ export const buildMicrosoftCdn = task(import.meta.path, async () => { ' - https://github.com/felixonmars/dnsmasq-china-list' ]; - return Promise.all(createRuleset( + return createRuleset( + span, 'Sukka\'s Ruleset - Microsoft CDN', description, new Date(), @@ -59,7 +60,7 @@ export const buildMicrosoftCdn = task(import.meta.path, async () => { 'ruleset', path.resolve(import.meta.dir, '../List/non_ip/microsoft_cdn.conf'), path.resolve(import.meta.dir, '../Clash/non_ip/microsoft_cdn.txt') - )); + ); }); if (import.meta.main) { diff --git a/Build/build-reject-domainset.ts b/Build/build-reject-domainset.ts index 892e7a31..802fcddc 100644 --- a/Build/build-reject-domainset.ts +++ b/Build/build-reject-domainset.ts @@ -29,20 +29,20 @@ export const buildRejectDomainSet = task(import.meta.path, async (span) => { // Parse from AdGuard Filters const [gorhill, shouldStop] = await span .traceChild('download and process hosts / adblock filter rules') - .traceAsyncFn(async () => { + .traceAsyncFn(async (childSpan) => { let shouldStop = false; const [gorhill] = await Promise.all([ getGorhillPublicSuffixPromise(), // Parse from remote hosts & domain lists - ...HOSTS.map(entry => processHosts(span, entry[0], entry[1], entry[2]).then(hosts => { + ...HOSTS.map(entry => processHosts(childSpan, entry[0], entry[1], entry[2]).then(hosts => { SetHelpers.add(domainSets, hosts); })), - ...DOMAIN_LISTS.map(entry => processDomainLists(span, entry[0], entry[1], entry[2])), + ...DOMAIN_LISTS.map(entry => processDomainLists(childSpan, entry[0], entry[1], entry[2])), ...ADGUARD_FILTERS.map(input => { const promise = typeof input === 'string' - ? processFilterRules(span, input) - : processFilterRules(span, input[0], input[1], input[2]); + ? processFilterRules(childSpan, input) + : processFilterRules(childSpan, input[0], input[1], input[2]); return promise.then(({ white, black, foundDebugDomain }) => { if (foundDebugDomain) { @@ -56,22 +56,22 @@ export const buildRejectDomainSet = task(import.meta.path, async (span) => { ...([ 'https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt', 'https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt' - ].map(input => processFilterRules(span, input).then(({ white, black }) => { + ].map(input => processFilterRules(childSpan, input).then(({ white, black }) => { setAddFromArray(filterRuleWhitelistDomainSets, white); setAddFromArray(filterRuleWhitelistDomainSets, black); }))), - getPhishingDomains(span).then(([purePhishingDomains, fullPhishingDomainSet]) => { + getPhishingDomains(childSpan).then(([purePhishingDomains, fullPhishingDomainSet]) => { SetHelpers.add(domainSets, fullPhishingDomainSet); setAddFromArray(domainSets, purePhishingDomains); }), - (async () => { + childSpan.traceChild('process reject_sukka.conf').traceAsyncFn(async () => { for await (const l of readFileByLine(path.resolve(import.meta.dir, '../Source/domainset/reject_sukka.conf'))) { const line = processLine(l); if (line) { domainSets.add(line); } } - })() + }) ]); // remove pre-defined enforced blacklist from whitelist @@ -187,7 +187,8 @@ export const buildRejectDomainSet = task(import.meta.path, async (span) => { ]; return Promise.all([ - ...createRuleset( + createRuleset( + span, 'Sukka\'s Ruleset - Reject Base', description, new Date(), @@ -197,6 +198,7 @@ export const buildRejectDomainSet = task(import.meta.path, async (span) => { path.resolve(import.meta.dir, '../Clash/domainset/reject.txt') ), compareAndWriteFile( + span, rejectDomainsStats.map(([domain, count]) => `${domain}${' '.repeat(100 - domain.length)}${count}`), path.resolve(import.meta.dir, '../List/internal/reject-stats.txt') ), diff --git a/Build/build-sgmodule-always-realip.ts b/Build/build-sgmodule-always-realip.ts index 110be56d..ac5602c9 100644 --- a/Build/build-sgmodule-always-realip.ts +++ b/Build/build-sgmodule-always-realip.ts @@ -38,8 +38,9 @@ const HOSTNAMES = [ 'GC._msDCS.*.*' ] as const; -export const buildAlwaysRealIPModule = task(import.meta.path, async () => { +export const buildAlwaysRealIPModule = task(import.meta.path, async (span) => { return compareAndWriteFile( + span, [ '#!name=[Sukka] Always Real IP Plus', `#!desc=Last Updated: ${new Date().toISOString()}`, diff --git a/Build/build-sgmodule-redirect.ts b/Build/build-sgmodule-redirect.ts index 2aa1c046..f84627a8 100644 --- a/Build/build-sgmodule-redirect.ts +++ b/Build/build-sgmodule-redirect.ts @@ -71,10 +71,11 @@ const REDIRECT = [ ['googleajax.wp-china-yes.net/', 'https://ajax.googleapis.com/'] ] as const; -export const buildRedirectModule = task(import.meta.path, async () => { +export const buildRedirectModule = task(import.meta.path, async (span) => { const domains = Array.from(new Set(REDIRECT.map(([from]) => tldts.getHostname(from, { detectIp: false })))).filter(Boolean); return compareAndWriteFile( + span, [ '#!name=[Sukka] URL Redirect', `#!desc=Last Updated: ${new Date().toISOString()}`, diff --git a/Build/build-speedtest-domainset.ts b/Build/build-speedtest-domainset.ts index e4acd3e4..69d8ae8d 100644 --- a/Build/build-speedtest-domainset.ts +++ b/Build/build-speedtest-domainset.ts @@ -151,13 +151,14 @@ export const buildSpeedtestDomainSet = task(import.meta.path, async (span) => { '.backend.librespeed.org' ]); - // Download previous speedtest domainset - for await (const l of await fetchRemoteTextByLine('https://ruleset.skk.moe/List/domainset/speedtest.conf')) { - const line = processLine(l); - if (line) { - domains.add(line); + await span.traceChild('fetch previous speedtest domainset').traceAsyncFn(async () => { + for await (const l of await fetchRemoteTextByLine('https://ruleset.skk.moe/List/domainset/speedtest.conf')) { + const line = processLine(l); + if (line) { + domains.add(line); + } } - } + }); await new Promise((resolve) => { const pMap = ([ @@ -225,7 +226,8 @@ export const buildSpeedtestDomainSet = task(import.meta.path, async (span) => { 'This file contains common speedtest endpoints.' ]; - return Promise.all(createRuleset( + return createRuleset( + span, 'Sukka\'s Ruleset - Speedtest Domains', description, new Date(), @@ -233,7 +235,7 @@ export const buildSpeedtestDomainSet = task(import.meta.path, async (span) => { 'domainset', path.resolve(import.meta.dir, '../List/domainset/speedtest.conf'), path.resolve(import.meta.dir, '../Clash/domainset/speedtest.txt') - )); + ); }); if (import.meta.main) { diff --git a/Build/build-sspanel-appprofile.ts b/Build/build-sspanel-appprofile.ts index 3882364c..6ac9c204 100644 --- a/Build/build-sspanel-appprofile.ts +++ b/Build/build-sspanel-appprofile.ts @@ -26,7 +26,7 @@ const removeNoResolved = (line: string) => line.replace(',no-resolve', ''); /** * This only generates a simplified version, for under-used users only. */ -export const buildSSPanelUIMAppProfile = task(import.meta.path, async () => { +export const buildSSPanelUIMAppProfile = task(import.meta.path, async (span) => { const [ domesticDomains, appleCdnDomains, @@ -108,6 +108,7 @@ export const buildSSPanelUIMAppProfile = task(import.meta.path, async () => { ); await compareAndWriteFile( + span, output, path.resolve(import.meta.dir, '../List/internal/appprofile.php') ); diff --git a/Build/build-stream-service.ts b/Build/build-stream-service.ts index da628c4b..d321073d 100644 --- a/Build/build-stream-service.ts +++ b/Build/build-stream-service.ts @@ -1,4 +1,5 @@ // @ts-check +import type { Span } from './trace'; import { task } from './trace'; import path from 'path'; @@ -7,10 +8,12 @@ import { createRuleset } from './lib/create-file'; import { ALL, NORTH_AMERICA, EU, HK, TW, JP, KR } from '../Source/stream'; import { SHARED_DESCRIPTION } from './lib/constants'; -export const createRulesetForStreamService = (fileId: string, title: string, streamServices: Array) => { +export const createRulesetForStreamService = (span: Span, fileId: string, title: string, streamServices: Array) => { + const childSpan = span.traceChild(fileId); return [ // Domains - ...createRuleset( + createRuleset( + childSpan, `Sukka's Ruleset - Stream Services: ${title}`, [ ...SHARED_DESCRIPTION, @@ -24,7 +27,8 @@ export const createRulesetForStreamService = (fileId: string, title: string, str path.resolve(import.meta.dir, `../Clash/non_ip/${fileId}.txt`) ), // IP - ...createRuleset( + createRuleset( + childSpan, `Sukka's Ruleset - Stream Services' IPs: ${title}`, [ ...SHARED_DESCRIPTION, @@ -47,16 +51,16 @@ export const createRulesetForStreamService = (fileId: string, title: string, str ]; }; -export const buildStreamService = task(import.meta.path, async () => { +export const buildStreamService = task(import.meta.path, async (span) => { return Promise.all([ - ...createRulesetForStreamService('stream', 'All', ALL), - ...createRulesetForStreamService('stream_us', 'North America', NORTH_AMERICA), - ...createRulesetForStreamService('stream_eu', 'Europe', EU), - ...createRulesetForStreamService('stream_hk', 'Hong Kong', HK), - ...createRulesetForStreamService('stream_tw', 'Taiwan', TW), - ...createRulesetForStreamService('stream_jp', 'Japan', JP), + ...createRulesetForStreamService(span, 'stream', 'All', ALL), + ...createRulesetForStreamService(span, 'stream_us', 'North America', NORTH_AMERICA), + ...createRulesetForStreamService(span, 'stream_eu', 'Europe', EU), + ...createRulesetForStreamService(span, 'stream_hk', 'Hong Kong', HK), + ...createRulesetForStreamService(span, 'stream_tw', 'Taiwan', TW), + ...createRulesetForStreamService(span, 'stream_jp', 'Japan', JP), // ...createRulesetForStreamService('stream_au', 'Oceania', AU), - ...createRulesetForStreamService('stream_kr', 'Korean', KR) + ...createRulesetForStreamService(span, 'stream_kr', 'Korean', KR) // ...createRulesetForStreamService('stream_south_east_asia', 'South East Asia', SOUTH_EAST_ASIA) ]); }); diff --git a/Build/build-telegram-cidr.ts b/Build/build-telegram-cidr.ts index 058816e2..db6217ce 100644 --- a/Build/build-telegram-cidr.ts +++ b/Build/build-telegram-cidr.ts @@ -32,7 +32,7 @@ export const getTelegramCIDRPromise = createMemoizedPromise(async () => { return { date, results }; }); -export const buildTelegramCIDR = task(import.meta.path, async () => { +export const buildTelegramCIDR = task(import.meta.path, async (span) => { const { date, results } = await getTelegramCIDRPromise(); if (results.length === 0) { @@ -45,7 +45,8 @@ export const buildTelegramCIDR = task(import.meta.path, async () => { ' - https://core.telegram.org/resources/cidr.txt' ]; - return Promise.all(createRuleset( + return createRuleset( + span, 'Sukka\'s Ruleset - Telegram IP CIDR', description, date, @@ -53,7 +54,7 @@ export const buildTelegramCIDR = task(import.meta.path, async () => { 'ruleset', path.resolve(import.meta.dir, '../List/ip/telegram.conf'), path.resolve(import.meta.dir, '../Clash/ip/telegram.txt') - )); + ); }); if (import.meta.main) { diff --git a/Build/download-mock-assets.ts b/Build/download-mock-assets.ts index 3ba9ddcd..bd0a8d09 100644 --- a/Build/download-mock-assets.ts +++ b/Build/download-mock-assets.ts @@ -12,15 +12,11 @@ const ASSETS_LIST = { const mockDir = path.resolve(import.meta.dir, '../Mock'); -export const downloadMockAssets = task(import.meta.path, () => Promise.all(Object.entries(ASSETS_LIST).map(async ([filename, url]) => { - const targetPath = path.join(mockDir, filename); - - const key = picocolors.gray(`Download ${filename}`); - console.time(key); - const res = await fetchWithRetry(url); - await Bun.write(targetPath, res); - console.timeEnd(key); -}))); +export const downloadMockAssets = task(import.meta.path, (span) => Promise.all(Object.entries(ASSETS_LIST).map( + ([filename, url]) => span + .traceChild(url) + .traceAsyncFn(() => fetchWithRetry(url).then(res => Bun.write(path.join(mockDir, filename), res))) +))); if (import.meta.main) { downloadMockAssets(); diff --git a/Build/lib/create-file.ts b/Build/lib/create-file.ts index 6fd14e14..91d25291 100644 --- a/Build/lib/create-file.ts +++ b/Build/lib/create-file.ts @@ -3,8 +3,10 @@ import { readFileByLine } from './fetch-text-by-line'; import { surgeDomainsetToClashDomainset, surgeRulesetToClashClassicalTextRuleset } from './clash'; import { traceAsync } from './trace-runner'; import picocolors from 'picocolors'; +import type { Span } from '../trace'; +import path from 'path'; -export async function compareAndWriteFile(linesA: string[], filePath: string) { +export async function compareAndWriteFile(span: Span, linesA: string[], filePath: string) { let isEqual = true; const file = Bun.file(filePath); @@ -17,48 +19,44 @@ export async function compareAndWriteFile(linesA: string[], filePath: string) { console.log(`Nothing to write to ${filePath}...`); isEqual = false; } else { - isEqual = await traceAsync( - picocolors.gray(`comparing ${filePath}`), - async () => { - let index = 0; + isEqual = await span.traceChild(`comparing ${filePath}`).traceAsyncFn(async () => { + let index = 0; - for await (const lineB of readFileByLine(file)) { - const lineA = linesA[index]; - index++; + for await (const lineB of readFileByLine(file)) { + const lineA = linesA[index]; + index++; - if (lineA == null) { - // The file becomes smaller - return false; - } + if (lineA == null) { + // The file becomes smaller + return false; + } - if (lineA[0] === '#' && lineB[0] === '#') { - continue; - } - if ( - lineA[0] === '/' + if (lineA[0] === '#' && lineB[0] === '#') { + continue; + } + if ( + lineA[0] === '/' && lineA[1] === '/' && lineA[3] === '#' && lineB[0] === '/' && lineB[1] === '/' && lineB[3] === '#' - ) { - continue; - } - - if (lineA !== lineB) { - return false; - } + ) { + continue; } - if (index !== linesALen) { - // The file becomes larger + if (lineA !== lineB) { return false; } + } - return true; - }, - picocolors.gray - ); + if (index !== linesALen) { + // The file becomes larger + return false; + } + + return true; + }); } if (isEqual) { @@ -66,7 +64,7 @@ export async function compareAndWriteFile(linesA: string[], filePath: string) { return; } - await traceAsync(picocolors.gray(`writing ${filePath}`), async () => { + await span.traceChild(`writing ${filePath}`).traceAsyncFn(async () => { if (linesALen < 10000) { return Bun.write(file, `${linesA.join('\n')}\n`); } @@ -79,7 +77,7 @@ export async function compareAndWriteFile(linesA: string[], filePath: string) { } return writer.end(); - }, picocolors.gray); + }); } export const withBannerArray = (title: string, description: string[] | readonly string[], date: Date, content: string[]) => { @@ -96,27 +94,28 @@ export const withBannerArray = (title: string, description: string[] | readonly }; export const createRuleset = ( + parentSpan: Span, title: string, description: string[] | readonly string[], date: Date, content: string[], type: 'ruleset' | 'domainset', surgePath: string, clashPath: string -) => { +) => parentSpan.traceChild(`create ruleset: ${path.basename(surgePath, path.extname(surgePath))}`).traceAsyncFn((childSpan) => { const surgeContent = withBannerArray(title, description, date, content); + const clashContent = childSpan.traceChild('convert incoming ruleset to clash').traceSyncFn(() => { + let _clashContent; + switch (type) { + case 'domainset': + _clashContent = surgeDomainsetToClashDomainset(content); + break; + case 'ruleset': + _clashContent = surgeRulesetToClashClassicalTextRuleset(content); + break; + default: + throw new TypeError(`Unknown type: ${type as any}`); + } + return withBannerArray(title, description, date, _clashContent); + }); - let _clashContent; - switch (type) { - case 'domainset': - _clashContent = surgeDomainsetToClashDomainset(content); - break; - case 'ruleset': - _clashContent = surgeRulesetToClashClassicalTextRuleset(content); - break; - default: - throw new TypeError(`Unknown type: ${type as any}`); - } - - const clashContent = withBannerArray(title, description, date, _clashContent); - - return [ - compareAndWriteFile(surgeContent, surgePath), - compareAndWriteFile(clashContent, clashPath) - ]; -}; + return Promise.all([ + compareAndWriteFile(childSpan, surgeContent, surgePath), + compareAndWriteFile(childSpan, clashContent, clashPath) + ]); +}); diff --git a/Build/lib/fetch-text-by-line.ts b/Build/lib/fetch-text-by-line.ts index 093eb3e6..37036bc0 100644 --- a/Build/lib/fetch-text-by-line.ts +++ b/Build/lib/fetch-text-by-line.ts @@ -41,17 +41,16 @@ export function readFileByLine(file: string | URL | BunFile) { return createTextLineAsyncGeneratorFromStreamSource(file.stream()); } -export function createReadlineInterfaceFromResponse(resp: Response) { +export function createReadlineInterfaceFromResponse(this: void, resp: Response) { if (!resp.body) { throw new Error('Failed to fetch remote text'); } if (resp.bodyUsed) { throw new Error('Body has already been consumed.'); } - return createTextLineAsyncGeneratorFromStreamSource(resp.body); } export function fetchRemoteTextByLine(url: string | URL) { - return fetchWithRetry(url, defaultRequestInit).then(res => createReadlineInterfaceFromResponse(res)); + return fetchWithRetry(url, defaultRequestInit).then(createReadlineInterfaceFromResponse); } diff --git a/Build/lib/parse-filter.ts b/Build/lib/parse-filter.ts index fa1d6876..106d3d7a 100644 --- a/Build/lib/parse-filter.ts +++ b/Build/lib/parse-filter.ts @@ -96,12 +96,12 @@ const enum ParseType { } export async function processFilterRules( - span: Span, + parentSpan: Span, filterRulesUrl: string, fallbackUrls?: readonly string[] | undefined | null, ttl: number | null = null ): Promise<{ white: string[], black: string[], foundDebugDomain: boolean }> { - const [white, black, warningMessages] = await span.traceChild('process filter rules: domainListsUrl').traceAsyncFn(() => fsCache.apply fsCache.apply fetchAssets(filterRulesUrl, fallbackUrls), - picocolors.gray - )).split('\n'); + const filterRules = await span.traceChild('download adguard filter').traceAsyncFn(() => { + return fetchAssets(filterRulesUrl, fallbackUrls).then(text => text.split('\n')); + }); - const key = picocolors.gray(`- parse adguard filter ${filterRulesUrl}`); - console.time(key); - for (let i = 0, len = filterRules.length; i < len; i++) { - lineCb(filterRules[i]); - } - console.timeEnd(key); + span.traceChild('parse adguard filter').traceSyncFn(() => { + for (let i = 0, len = filterRules.length; i < len; i++) { + lineCb(filterRules[i]); + } + }); } return [ diff --git a/Build/lib/reject-data-source.ts b/Build/lib/reject-data-source.ts index ec4767f2..363340b0 100644 --- a/Build/lib/reject-data-source.ts +++ b/Build/lib/reject-data-source.ts @@ -47,7 +47,9 @@ export const DOMAIN_LISTS = [ ['https://raw.githubusercontent.com/AdguardTeam/cname-trackers/master/data/combined_disguised_mail_trackers_justdomains.txt', true, TTL.THREE_DAYS()] ] as const; -export const ADGUARD_FILTERS = [ +type AdGuardFilterSource = string | [main: string, mirrors: string[] | null, ttl: number]; + +export const ADGUARD_FILTERS: AdGuardFilterSource[] = [ // EasyList [ 'https://easylist.to/easylist/easylist.txt', @@ -156,7 +158,7 @@ export const ADGUARD_FILTERS = [ // Not actively maintained, let's use a 10 days cache ttl ['https://raw.githubusercontent.com/Spam404/lists/master/adblock-list.txt', null, TTL.TEN_DAYS()], // Brave First Party & First Party CNAME - 'https://raw.githubusercontent.com/brave/adblock-lists/master/brave-lists/brave-firstparty.txt' + ['https://raw.githubusercontent.com/brave/adblock-lists/master/brave-lists/brave-firstparty.txt', null, TTL.ONE_DAY()] ] as const; export const PREDEFINED_WHITELIST = [ diff --git a/Build/trace/index.ts b/Build/trace/index.ts index 52fc806d..89cee4ed 100644 --- a/Build/trace/index.ts +++ b/Build/trace/index.ts @@ -114,17 +114,45 @@ export const universalify = (taskname: string, fn: (this: vo }; }; -export const printTraceResult = (traceResult: TraceResult = rootTraceResult, level = 0, isLast = false) => { - if (level === 0) { - printStats(traceResult.children); +export const printTraceResult = (traceResult: TraceResult = rootTraceResult) => { + printStats(traceResult.children); + printTree(traceResult, node => `${node.name} ${picocolors.bold(`${(node.end - node.start).toFixed(3)}ms`)}`); +}; + +function printTree(initialTree: TraceResult, printNode: (node: TraceResult, branch: string) => string) { + function printBranch(tree: TraceResult, branch: string) { + const isGraphHead = branch.length === 0; + const children = tree.children; + + let branchHead = ''; + + if (!isGraphHead) { + branchHead = children.length > 0 ? '┬ ' : '─ '; + } + + const toPrint = printNode(tree, `${branch}${branchHead}`); + + if (typeof toPrint === 'string') { + console.log(`${branch}${branchHead}${toPrint}`); + } + + let baseBranch = branch; + + if (!isGraphHead) { + const isChildOfLastBranch = branch.endsWith('└─'); + baseBranch = branch.slice(0, -2) + (isChildOfLastBranch ? ' ' : '│ '); + } + + const nextBranch = `${baseBranch}├─`; + const lastBranch = `${baseBranch}└─`; + + children.forEach((child, index) => { + printBranch(child, children.length - 1 === index ? lastBranch : nextBranch); + }); } - const prefix = (level > 0 ? ` ${'│ '.repeat(level - 1)}` : '') + (level > 0 ? (isLast ? '└─' : '├─') : ''); - - console.log(`${prefix} ${traceResult.name} ${picocolors.bold(`${(traceResult.end - traceResult.start).toFixed(2)}ms`)}`); - - traceResult.children.forEach((child, index, arr) => printTraceResult(child, level + 1, index === arr.length - 1)); -}; + printBranch(initialTree, ''); +} function printStats(stats: TraceResult[]): void { stats.sort((a, b) => a.start - b.start); diff --git a/Source/non_ip/ai.conf b/Source/non_ip/ai.conf index f7a243a5..b49730f5 100644 --- a/Source/non_ip/ai.conf +++ b/Source/non_ip/ai.conf @@ -10,3 +10,4 @@ DOMAIN-SUFFIX,openaiapi-site.azureedge.net DOMAIN-SUFFIX,perplexity.ai DOMAIN-SUFFIX,anthropic.com DOMAIN-SUFFIX,claude.ai +DOMAIN-SUFFIX,generativelanguage.googleapis.com