From 0f257e992ad8480ba6c13b35c5378d4ff86926c1 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Sun, 14 Jan 2024 04:16:28 +0800 Subject: [PATCH] Refactor/Perf: initial span tracer --- Build/build-anti-bogus-domain.ts | 2 +- Build/build-apple-cdn.ts | 3 +- Build/build-cdn-conf.ts | 2 +- Build/build-chn-cidr.ts | 3 +- Build/build-cloudmounter-rules.ts | 2 +- Build/build-common.ts | 2 +- Build/build-domestic-ruleset.ts | 2 +- Build/build-internal-cdn-rules.ts | 2 +- Build/build-internal-reverse-chn-cidr.ts | 2 +- Build/build-microsoft-cdn.ts | 3 +- Build/build-public.ts | 2 +- Build/build-reject-domainset.ts | 116 +++++++++--------- Build/build-sgmodule-always-realip.ts | 2 +- Build/build-sgmodule-redirect.ts | 2 +- Build/build-speedtest-domainset.ts | 19 +-- Build/build-sspanel-appprofile.ts | 2 +- Build/build-stream-service.ts | 2 +- Build/build-telegram-cidr.ts | 2 +- Build/download-mock-assets.ts | 2 +- Build/download-previous-build.ts | 102 ++++++++-------- Build/index.ts | 70 +++++------ Build/lib/get-phishing-domains.ts | 13 +- Build/lib/parse-filter.ts | 12 +- Build/lib/trace-runner.ts | 13 -- Build/trace/index.ts | 145 +++++++++++++++++++++++ Build/validate-domainset.ts | 2 +- 26 files changed, 329 insertions(+), 200 deletions(-) create mode 100644 Build/trace/index.ts diff --git a/Build/build-anti-bogus-domain.ts b/Build/build-anti-bogus-domain.ts index 1e64d9e1..023091b5 100644 --- a/Build/build-anti-bogus-domain.ts +++ b/Build/build-anti-bogus-domain.ts @@ -3,7 +3,7 @@ import path from 'path'; import { createRuleset } from './lib/create-file'; import { fetchRemoteTextByLine, readFileByLine } from './lib/fetch-text-by-line'; import { processLine } from './lib/process-line'; -import { task } from './lib/trace-runner'; +import { task } from './trace'; import { SHARED_DESCRIPTION } from './lib/constants'; import { isProbablyIpv4, isProbablyIpv6 } from './lib/is-fast-ip'; import { TTL, deserializeArray, fsCache, serializeArray } from './lib/cache-filesystem'; diff --git a/Build/build-apple-cdn.ts b/Build/build-apple-cdn.ts index 76d05755..0a739c90 100644 --- a/Build/build-apple-cdn.ts +++ b/Build/build-apple-cdn.ts @@ -2,7 +2,8 @@ import path from 'path'; import { createRuleset } from './lib/create-file'; import { parseFelixDnsmasq } from './lib/parse-dnsmasq'; -import { task, traceAsync } from './lib/trace-runner'; +import { traceAsync } from './lib/trace-runner'; +import { task } from './trace'; import { SHARED_DESCRIPTION } from './lib/constants'; import picocolors from 'picocolors'; import { createMemoizedPromise } from './lib/memo-promise'; diff --git a/Build/build-cdn-conf.ts b/Build/build-cdn-conf.ts index a4174a48..0c9c24ca 100644 --- a/Build/build-cdn-conf.ts +++ b/Build/build-cdn-conf.ts @@ -2,7 +2,7 @@ import path from 'path'; import { createRuleset } from './lib/create-file'; import { readFileByLine } from './lib/fetch-text-by-line'; import { createTrie } from './lib/trie'; -import { task } from './lib/trace-runner'; +import { task } from './trace'; import { processLine } from './lib/process-line'; import { SHARED_DESCRIPTION } from './lib/constants'; import { getPublicSuffixListTextPromise } from './download-publicsuffixlist'; diff --git a/Build/build-chn-cidr.ts b/Build/build-chn-cidr.ts index b32fb838..89a8ec3a 100644 --- a/Build/build-chn-cidr.ts +++ b/Build/build-chn-cidr.ts @@ -2,7 +2,8 @@ import { fetchRemoteTextByLine } from './lib/fetch-text-by-line'; import { resolve as pathResolve } from 'path'; import { compareAndWriteFile, withBannerArray } from './lib/create-file'; import { processLineFromReadline } from './lib/process-line'; -import { task, traceAsync, traceSync } from './lib/trace-runner'; +import { traceAsync, traceSync } from './lib/trace-runner'; +import { task } from './trace'; import { exclude } from 'fast-cidr-tools'; import picocolors from 'picocolors'; diff --git a/Build/build-cloudmounter-rules.ts b/Build/build-cloudmounter-rules.ts index b39b7e1c..77b251f5 100644 --- a/Build/build-cloudmounter-rules.ts +++ b/Build/build-cloudmounter-rules.ts @@ -2,7 +2,7 @@ import path from 'path'; import { DOMAINS, PROCESS_NAMES } from '../Source/non_ip/cloudmounter'; import { SHARED_DESCRIPTION } from './lib/constants'; import { createRuleset } from './lib/create-file'; -import { task } from './lib/trace-runner'; +import { task } from './trace'; const outputSurgeDir = path.resolve(import.meta.dir, '../List'); const outputClashDir = path.resolve(import.meta.dir, '../Clash'); diff --git a/Build/build-common.ts b/Build/build-common.ts index 7bd0e2be..c7b3b525 100644 --- a/Build/build-common.ts +++ b/Build/build-common.ts @@ -6,7 +6,7 @@ import { readFileByLine } from './lib/fetch-text-by-line'; import { processLine } from './lib/process-line'; import { createRuleset } from './lib/create-file'; import { domainDeduper } from './lib/domain-deduper'; -import { task } from './lib/trace-runner'; +import { task } from './trace'; import { SHARED_DESCRIPTION } from './lib/constants'; const MAGIC_COMMAND_SKIP = '# $ custom_build_script'; diff --git a/Build/build-domestic-ruleset.ts b/Build/build-domestic-ruleset.ts index 7b71b12e..fa929516 100644 --- a/Build/build-domestic-ruleset.ts +++ b/Build/build-domestic-ruleset.ts @@ -4,7 +4,7 @@ import { DOMESTICS } from '../Source/non_ip/domestic'; import { readFileByLine } from './lib/fetch-text-by-line'; import { processLineFromReadline } from './lib/process-line'; import { compareAndWriteFile, createRuleset } from './lib/create-file'; -import { task } from './lib/trace-runner'; +import { task } from './trace'; import { SHARED_DESCRIPTION } from './lib/constants'; import { createMemoizedPromise } from './lib/memo-promise'; diff --git a/Build/build-internal-cdn-rules.ts b/Build/build-internal-cdn-rules.ts index 8ac268f5..c4160da7 100644 --- a/Build/build-internal-cdn-rules.ts +++ b/Build/build-internal-cdn-rules.ts @@ -2,7 +2,7 @@ import path from 'path'; import { processLine } from './lib/process-line'; import { readFileByLine } from './lib/fetch-text-by-line'; import { sortDomains } from './lib/stable-sort-domain'; -import { task } from './lib/trace-runner'; +import { task } from './trace'; import { compareAndWriteFile } from './lib/create-file'; import { getGorhillPublicSuffixPromise } from './lib/get-gorhill-publicsuffix'; diff --git a/Build/build-internal-reverse-chn-cidr.ts b/Build/build-internal-reverse-chn-cidr.ts index c3a5bcb5..7a6edf35 100644 --- a/Build/build-internal-reverse-chn-cidr.ts +++ b/Build/build-internal-reverse-chn-cidr.ts @@ -1,7 +1,7 @@ import { fetchRemoteTextByLine } from './lib/fetch-text-by-line'; import { processLineFromReadline } from './lib/process-line'; import path from 'path'; -import { task } from './lib/trace-runner'; +import { task } from './trace'; import { exclude, merge } from 'fast-cidr-tools'; diff --git a/Build/build-microsoft-cdn.ts b/Build/build-microsoft-cdn.ts index 6eda4c54..b8e881e3 100644 --- a/Build/build-microsoft-cdn.ts +++ b/Build/build-microsoft-cdn.ts @@ -1,5 +1,6 @@ import path from 'path'; -import { task, traceAsync } from './lib/trace-runner'; +import { traceAsync } from './lib/trace-runner'; +import { task } from './trace'; import { createRuleset } from './lib/create-file'; import { fetchRemoteTextByLine } from './lib/fetch-text-by-line'; import { createTrie } from './lib/trie'; diff --git a/Build/build-public.ts b/Build/build-public.ts index c207fef6..8b5c85a1 100644 --- a/Build/build-public.ts +++ b/Build/build-public.ts @@ -1,5 +1,5 @@ import path from 'path'; -import { task } from './lib/trace-runner'; +import { task } from './trace'; import { treeDir } from './lib/tree-dir'; import type { TreeType, TreeTypeArray } from './lib/tree-dir'; import listDir from '@sukka/listdir'; diff --git a/Build/build-reject-domainset.ts b/Build/build-reject-domainset.ts index 428f9b56..892e7a31 100644 --- a/Build/build-reject-domainset.ts +++ b/Build/build-reject-domainset.ts @@ -11,7 +11,7 @@ import { domainDeduper } from './lib/domain-deduper'; import createKeywordFilter from './lib/aho-corasick'; import { readFileByLine } from './lib/fetch-text-by-line'; import { sortDomains } from './lib/stable-sort-domain'; -import { traceSync, task, traceAsync } from './lib/trace-runner'; +import { task } from './trace'; import { getGorhillPublicSuffixPromise } from './lib/get-gorhill-publicsuffix'; import * as tldts from 'tldts'; import { SHARED_DESCRIPTION } from './lib/constants'; @@ -20,71 +20,73 @@ import { getPhishingDomains } from './lib/get-phishing-domains'; import * as SetHelpers from 'mnemonist/set'; import { setAddFromArray } from './lib/set-add-from-array'; -export const buildRejectDomainSet = task(import.meta.path, async () => { +export const buildRejectDomainSet = task(import.meta.path, async (span) => { /** Whitelists */ const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST); const domainSets = new Set(); // Parse from AdGuard Filters - const [gorhill, shouldStop] = await traceAsync('* Download and process Hosts / AdBlock Filter Rules', async () => { - let shouldStop = false; + const [gorhill, shouldStop] = await span + .traceChild('download and process hosts / adblock filter rules') + .traceAsyncFn(async () => { + let shouldStop = false; - const [gorhill] = await Promise.all([ - getGorhillPublicSuffixPromise(), - // Parse from remote hosts & domain lists - ...HOSTS.map(entry => processHosts(entry[0], entry[1], entry[2]).then(hosts => { - SetHelpers.add(domainSets, hosts); - })), - ...DOMAIN_LISTS.map(entry => processDomainLists(entry[0], entry[1], entry[2])), - ...ADGUARD_FILTERS.map(input => { - const promise = typeof input === 'string' - ? processFilterRules(input) - : processFilterRules(input[0], input[1], input[2]); + const [gorhill] = await Promise.all([ + getGorhillPublicSuffixPromise(), + // Parse from remote hosts & domain lists + ...HOSTS.map(entry => processHosts(span, entry[0], entry[1], entry[2]).then(hosts => { + SetHelpers.add(domainSets, hosts); + })), + ...DOMAIN_LISTS.map(entry => processDomainLists(span, entry[0], entry[1], entry[2])), + ...ADGUARD_FILTERS.map(input => { + const promise = typeof input === 'string' + ? processFilterRules(span, input) + : processFilterRules(span, input[0], input[1], input[2]); - return promise.then(({ white, black, foundDebugDomain }) => { - if (foundDebugDomain) { - shouldStop = true; + return promise.then(({ white, black, foundDebugDomain }) => { + if (foundDebugDomain) { + shouldStop = true; // we should not break here, as we want to see full matches from all data source - } + } + setAddFromArray(filterRuleWhitelistDomainSets, white); + setAddFromArray(domainSets, black); + }); + }), + ...([ + 'https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt', + 'https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt' + ].map(input => processFilterRules(span, input).then(({ white, black }) => { setAddFromArray(filterRuleWhitelistDomainSets, white); - setAddFromArray(domainSets, black); - }); - }), - ...([ - 'https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt', - 'https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt' - ].map(input => processFilterRules(input).then(({ white, black }) => { - setAddFromArray(filterRuleWhitelistDomainSets, white); - setAddFromArray(filterRuleWhitelistDomainSets, black); - }))), - getPhishingDomains().then(([purePhishingDomains, fullPhishingDomainSet]) => { - SetHelpers.add(domainSets, fullPhishingDomainSet); - setAddFromArray(domainSets, purePhishingDomains); - }), - (async () => { - for await (const l of readFileByLine(path.resolve(import.meta.dir, '../Source/domainset/reject_sukka.conf'))) { - const line = processLine(l); - if (line) { - domainSets.add(line); + setAddFromArray(filterRuleWhitelistDomainSets, black); + }))), + getPhishingDomains(span).then(([purePhishingDomains, fullPhishingDomainSet]) => { + SetHelpers.add(domainSets, fullPhishingDomainSet); + setAddFromArray(domainSets, purePhishingDomains); + }), + (async () => { + for await (const l of readFileByLine(path.resolve(import.meta.dir, '../Source/domainset/reject_sukka.conf'))) { + const line = processLine(l); + if (line) { + domainSets.add(line); + } } + })() + ]); + + // remove pre-defined enforced blacklist from whitelist + const trie0 = createTrie(filterRuleWhitelistDomainSets); + + for (let i = 0, len1 = PREDEFINED_ENFORCED_BACKLIST.length; i < len1; i++) { + const enforcedBlack = PREDEFINED_ENFORCED_BACKLIST[i]; + const found = trie0.find(enforcedBlack); + for (let j = 0, len2 = found.length; j < len2; j++) { + filterRuleWhitelistDomainSets.delete(found[j]); } - })() - ]); - - // remove pre-defined enforced blacklist from whitelist - const trie0 = createTrie(filterRuleWhitelistDomainSets); - - for (let i = 0, len1 = PREDEFINED_ENFORCED_BACKLIST.length; i < len1; i++) { - const enforcedBlack = PREDEFINED_ENFORCED_BACKLIST[i]; - const found = trie0.find(enforcedBlack); - for (let j = 0, len2 = found.length; j < len2; j++) { - filterRuleWhitelistDomainSets.delete(found[j]); } - } - return [gorhill, shouldStop] as const; - }); + return [gorhill, shouldStop] as const; + }); if (shouldStop) { process.exit(1); @@ -94,7 +96,7 @@ export const buildRejectDomainSet = task(import.meta.path, async () => { console.log(`Import ${previousSize} rules from Hosts / AdBlock Filter Rules & reject_sukka.conf!`); // Dedupe domainSets - await traceAsync('* Dedupe from black keywords/suffixes', async () => { + await span.traceChild('dedupe from black keywords/suffixes').traceAsyncFn(async () => { /** Collect DOMAIN-SUFFIX from non_ip/reject.conf for deduplication */ const domainSuffixSet = new Set(); /** Collect DOMAIN-KEYWORD from non_ip/reject.conf for deduplication */ @@ -146,13 +148,13 @@ export const buildRejectDomainSet = task(import.meta.path, async () => { previousSize = domainSets.size; // Dedupe domainSets - const dudupedDominArray = traceSync('* Dedupe from covered subdomain', () => domainDeduper(Array.from(domainSets))); + const dudupedDominArray = span.traceChild('dedupe from covered subdomain').traceSyncFn(() => domainDeduper(Array.from(domainSets))); + console.log(`Deduped ${previousSize - dudupedDominArray.length} rules from covered subdomain!`); console.log(`Final size ${dudupedDominArray.length}`); // Create reject stats - const rejectDomainsStats: Array<[string, number]> = traceSync( - '* Collect reject domain stats', + const rejectDomainsStats: Array<[string, number]> = span.traceChild('create reject stats').traceSyncFn( () => Object.entries( dudupedDominArray.reduce>((acc, cur) => { const suffix = tldts.getDomain(cur, { allowPrivateDomains: false, detectIp: false, validateHostname: false }); @@ -189,7 +191,7 @@ export const buildRejectDomainSet = task(import.meta.path, async () => { 'Sukka\'s Ruleset - Reject Base', description, new Date(), - traceSync('* Sort reject domainset', () => sortDomains(dudupedDominArray, gorhill)), + span.traceChild('sort reject domainset').traceSyncFn(() => sortDomains(dudupedDominArray, gorhill)), 'domainset', path.resolve(import.meta.dir, '../List/domainset/reject.conf'), path.resolve(import.meta.dir, '../Clash/domainset/reject.txt') diff --git a/Build/build-sgmodule-always-realip.ts b/Build/build-sgmodule-always-realip.ts index aa95c827..110be56d 100644 --- a/Build/build-sgmodule-always-realip.ts +++ b/Build/build-sgmodule-always-realip.ts @@ -1,5 +1,5 @@ import path from 'path'; -import { task } from './lib/trace-runner'; +import { task } from './trace'; import { compareAndWriteFile } from './lib/create-file'; const HOSTNAMES = [ diff --git a/Build/build-sgmodule-redirect.ts b/Build/build-sgmodule-redirect.ts index 9c858df3..2aa1c046 100644 --- a/Build/build-sgmodule-redirect.ts +++ b/Build/build-sgmodule-redirect.ts @@ -1,5 +1,5 @@ import path from 'path'; -import { task } from './lib/trace-runner'; +import { task } from './trace'; import { compareAndWriteFile } from './lib/create-file'; import * as tldts from 'tldts'; diff --git a/Build/build-speedtest-domainset.ts b/Build/build-speedtest-domainset.ts index fdb2aecf..e4acd3e4 100644 --- a/Build/build-speedtest-domainset.ts +++ b/Build/build-speedtest-domainset.ts @@ -5,7 +5,7 @@ import { sortDomains } from './lib/stable-sort-domain'; import { Sema } from 'async-sema'; import * as tldts from 'tldts'; -import { task } from './lib/trace-runner'; +import { task } from './trace'; import { fetchWithRetry } from './lib/fetch-retry'; import { SHARED_DESCRIPTION } from './lib/constants'; import { getGorhillPublicSuffixPromise } from './lib/get-gorhill-publicsuffix'; @@ -35,11 +35,8 @@ const querySpeedtestApi = async (keyword: string): Promise> try { const randomUserAgent = topUserAgents[Math.floor(Math.random() * topUserAgents.length)]; - const key = `fetch speedtest endpoints: ${keyword}`; - console.log(key); - console.time(key); - const json = await fsCache.apply( + return await fsCache.apply( url, () => s.acquire().then(() => fetchWithRetry(url, { headers: { @@ -77,17 +74,13 @@ const querySpeedtestApi = async (keyword: string): Promise> deserializer: deserializeArray } ); - - console.timeEnd(key); - - return json; } catch (e) { - console.log(e); + console.error(e); return []; } }; -export const buildSpeedtestDomainSet = task(import.meta.path, async () => { +export const buildSpeedtestDomainSet = task(import.meta.path, async (span) => { // Predefined domainset /** @type {Set} */ const domains = new Set([ @@ -197,7 +190,7 @@ export const buildSpeedtestDomainSet = task(import.meta.path, async () => { 'Brazil', 'Turkey' ]).reduce>>((pMap, keyword) => { - pMap[keyword] = querySpeedtestApi(keyword).then(hostnameGroup => { + pMap[keyword] = span.traceChild(`fetch speedtest endpoints: ${keyword}`).traceAsyncFn(() => querySpeedtestApi(keyword)).then(hostnameGroup => { hostnameGroup.forEach(hostname => { if (hostname) { domains.add(hostname); @@ -224,7 +217,7 @@ export const buildSpeedtestDomainSet = task(import.meta.path, async () => { }); const gorhill = await getGorhillPublicSuffixPromise(); - const deduped = sortDomains(domainDeduper(Array.from(domains)), gorhill); + const deduped = span.traceChild('sort result').traceSyncFn(() => sortDomains(domainDeduper(Array.from(domains)), gorhill)); const description = [ ...SHARED_DESCRIPTION, diff --git a/Build/build-sspanel-appprofile.ts b/Build/build-sspanel-appprofile.ts index be63f8e0..3882364c 100644 --- a/Build/build-sspanel-appprofile.ts +++ b/Build/build-sspanel-appprofile.ts @@ -3,7 +3,7 @@ import { getDomesticDomainsRulesetPromise } from './build-domestic-ruleset'; import { surgeRulesetToClashClassicalTextRuleset } from './lib/clash'; import { readFileByLine } from './lib/fetch-text-by-line'; import { processLineFromReadline } from './lib/process-line'; -import { task } from './lib/trace-runner'; +import { task } from './trace'; import path from 'path'; import { ALL as AllStreamServices } from '../Source/stream'; diff --git a/Build/build-stream-service.ts b/Build/build-stream-service.ts index 2cf3e68f..da628c4b 100644 --- a/Build/build-stream-service.ts +++ b/Build/build-stream-service.ts @@ -1,5 +1,5 @@ // @ts-check -import { task } from './lib/trace-runner'; +import { task } from './trace'; import path from 'path'; import { createRuleset } from './lib/create-file'; diff --git a/Build/build-telegram-cidr.ts b/Build/build-telegram-cidr.ts index e42798f4..058816e2 100644 --- a/Build/build-telegram-cidr.ts +++ b/Build/build-telegram-cidr.ts @@ -5,7 +5,7 @@ import path from 'path'; import { isProbablyIpv4, isProbablyIpv6 } from './lib/is-fast-ip'; import { processLine } from './lib/process-line'; import { createRuleset } from './lib/create-file'; -import { task } from './lib/trace-runner'; +import { task } from './trace'; import { SHARED_DESCRIPTION } from './lib/constants'; import { createMemoizedPromise } from './lib/memo-promise'; diff --git a/Build/download-mock-assets.ts b/Build/download-mock-assets.ts index 19952ee7..3ba9ddcd 100644 --- a/Build/download-mock-assets.ts +++ b/Build/download-mock-assets.ts @@ -1,5 +1,5 @@ import picocolors from 'picocolors'; -import { task } from './lib/trace-runner'; +import { task } from './trace'; import path from 'path'; import { fetchWithRetry } from './lib/fetch-retry'; diff --git a/Build/download-previous-build.ts b/Build/download-previous-build.ts index 4bf39473..96965fb1 100644 --- a/Build/download-previous-build.ts +++ b/Build/download-previous-build.ts @@ -4,7 +4,7 @@ import path from 'path'; import { pipeline } from 'stream/promises'; import { readFileByLine } from './lib/fetch-text-by-line'; import { isCI } from 'ci-info'; -import { task } from './lib/trace-runner'; +import { task } from './trace'; import { defaultRequestInit, fetchWithRetry } from './lib/fetch-retry'; import tarStream from 'tar-stream'; import zlib from 'zlib'; @@ -13,29 +13,33 @@ import { Readable } from 'stream'; const IS_READING_BUILD_OUTPUT = 1 << 2; const ALL_FILES_EXISTS = 1 << 3; -export const downloadPreviousBuild = task(import.meta.path, async () => { +export const downloadPreviousBuild = task(import.meta.path, async (span) => { const buildOutputList: string[] = []; let flag = 1 | ALL_FILES_EXISTS; - for await (const line of readFileByLine(path.resolve(import.meta.dir, '../.gitignore'))) { - if (line === '# $ build output') { - flag = flag | IS_READING_BUILD_OUTPUT; - continue; - } - if (!(flag & IS_READING_BUILD_OUTPUT)) { - continue; - } + await span + .traceChild('read .gitignore') + .traceAsyncFn(async () => { + for await (const line of readFileByLine(path.resolve(import.meta.dir, '../.gitignore'))) { + if (line === '# $ build output') { + flag = flag | IS_READING_BUILD_OUTPUT; + continue; + } + if (!(flag & IS_READING_BUILD_OUTPUT)) { + continue; + } - buildOutputList.push(line); + buildOutputList.push(line); - if (!isCI) { - // Bun.file().exists() doesn't check directory - if (!existsSync(path.join(import.meta.dir, '..', line))) { - flag = flag & ~ALL_FILES_EXISTS; + if (!isCI) { + // Bun.file().exists() doesn't check directory + if (!existsSync(path.join(import.meta.dir, '..', line))) { + flag = flag & ~ALL_FILES_EXISTS; + } + } } - } - } + }); if (isCI) { flag = flag & ~ALL_FILES_EXISTS; @@ -48,42 +52,46 @@ export const downloadPreviousBuild = task(import.meta.path, async () => { const filesList = buildOutputList.map(f => path.join('ruleset.skk.moe-master', f)); - const resp = await fetchWithRetry('https://codeload.github.com/sukkalab/ruleset.skk.moe/tar.gz/master', defaultRequestInit); + return span + .traceChild('download & extract previoud build') + .traceAsyncFn(async () => { + const resp = await fetchWithRetry('https://codeload.github.com/sukkalab/ruleset.skk.moe/tar.gz/master', defaultRequestInit); - if (!resp.body) { - throw new Error('Download previous build failed! No body found'); - } + if (!resp.body) { + throw new Error('Download previous build failed! No body found'); + } - const extract = tarStream.extract(); - const gunzip = zlib.createGunzip(); - pipeline( - Readable.fromWeb(resp.body) as any, - gunzip, - extract - ); + const extract = tarStream.extract(); + const gunzip = zlib.createGunzip(); + pipeline( + Readable.fromWeb(resp.body) as any, + gunzip, + extract + ); - const pathPrefix = `ruleset.skk.moe-master${path.sep}`; + const pathPrefix = `ruleset.skk.moe-master${path.sep}`; - for await (const entry of extract) { - if (entry.header.type !== 'file') { - entry.resume(); // Drain the entry - continue; - } - // filter entry - if (!filesList.some(f => entry.header.name.startsWith(f))) { - entry.resume(); // Drain the entry - continue; - } + for await (const entry of extract) { + if (entry.header.type !== 'file') { + entry.resume(); // Drain the entry + continue; + } + // filter entry + if (!filesList.some(f => entry.header.name.startsWith(f))) { + entry.resume(); // Drain the entry + continue; + } - const relativeEntryPath = entry.header.name.replace(pathPrefix, ''); - const targetPath = path.join(import.meta.dir, '..', relativeEntryPath); + const relativeEntryPath = entry.header.name.replace(pathPrefix, ''); + const targetPath = path.join(import.meta.dir, '..', relativeEntryPath); - await mkdir(path.dirname(targetPath), { recursive: true }); - await pipeline( - entry as any, - createWriteStream(targetPath) - ); - } + await mkdir(path.dirname(targetPath), { recursive: true }); + await pipeline( + entry as any, + createWriteStream(targetPath) + ); + } + }); }); if (import.meta.main) { diff --git a/Build/index.ts b/Build/index.ts index 0cca7854..94ba8cbf 100644 --- a/Build/index.ts +++ b/Build/index.ts @@ -23,29 +23,33 @@ import { buildSSPanelUIMAppProfile } from './build-sspanel-appprofile'; import { buildPublic } from './build-public'; import { downloadMockAssets } from './download-mock-assets'; -import type { TaskResult } from './lib/trace-runner'; import { buildCloudMounterRules } from './build-cloudmounter-rules'; +import { createSpan, printTraceResult } from './trace'; + (async () => { console.log('Bun version:', Bun.version, Bun.revision); + const rootSpan = createSpan('root'); + try { // TODO: restore this once Bun has fixed their worker // const buildInternalReverseChnCIDRWorker = new Worker(new URL('./workers/build-internal-reverse-chn-cidr-worker.ts', import.meta.url)); - const downloadPreviousBuildPromise = downloadPreviousBuild(); - const buildCommonPromise = downloadPreviousBuildPromise.then(() => buildCommon()); - const buildAntiBogusDomainPromise = downloadPreviousBuildPromise.then(() => buildAntiBogusDomain()); - const buildAppleCdnPromise = downloadPreviousBuildPromise.then(() => buildAppleCdn()); - const buildCdnConfPromise = downloadPreviousBuildPromise.then(() => buildCdnConf()); - const buildRejectDomainSetPromise = downloadPreviousBuildPromise.then(() => buildRejectDomainSet()); - const buildTelegramCIDRPromise = downloadPreviousBuildPromise.then(() => buildTelegramCIDR()); - const buildChnCidrPromise = downloadPreviousBuildPromise.then(() => buildChnCidr()); - const buildSpeedtestDomainSetPromise = downloadPreviousBuildPromise.then(() => buildSpeedtestDomainSet()); + const downloadPreviousBuildPromise = downloadPreviousBuild(rootSpan); + + const buildCommonPromise = downloadPreviousBuildPromise.then(() => buildCommon(rootSpan)); + const buildAntiBogusDomainPromise = downloadPreviousBuildPromise.then(() => buildAntiBogusDomain(rootSpan)); + const buildAppleCdnPromise = downloadPreviousBuildPromise.then(() => buildAppleCdn(rootSpan)); + const buildCdnConfPromise = downloadPreviousBuildPromise.then(() => buildCdnConf(rootSpan)); + const buildRejectDomainSetPromise = downloadPreviousBuildPromise.then(() => buildRejectDomainSet(rootSpan)); + const buildTelegramCIDRPromise = downloadPreviousBuildPromise.then(() => buildTelegramCIDR(rootSpan)); + const buildChnCidrPromise = downloadPreviousBuildPromise.then(() => buildChnCidr(rootSpan)); + const buildSpeedtestDomainSetPromise = downloadPreviousBuildPromise.then(() => buildSpeedtestDomainSet(rootSpan)); const buildInternalCDNDomainsPromise = Promise.all([ buildCommonPromise, buildCdnConfPromise - ]).then(() => buildInternalCDNDomains()); + ]).then(() => buildInternalCDNDomains(rootSpan)); // const buildInternalReverseChnCIDRPromise = new Promise(resolve => { // const handleMessage = (e: MessageEvent) => { @@ -60,24 +64,24 @@ import { buildCloudMounterRules } from './build-cloudmounter-rules'; // }); // const buildInternalChnDomainsPromise = buildInternalChnDomains(); - const buildDomesticRulesetPromise = downloadPreviousBuildPromise.then(() => buildDomesticRuleset()); + const buildDomesticRulesetPromise = downloadPreviousBuildPromise.then(() => buildDomesticRuleset(rootSpan)); - const buildRedirectModulePromise = downloadPreviousBuildPromise.then(() => buildRedirectModule()); - const buildAlwaysRealIPModulePromise = downloadPreviousBuildPromise.then(() => buildAlwaysRealIPModule()); + const buildRedirectModulePromise = downloadPreviousBuildPromise.then(() => buildRedirectModule(rootSpan)); + const buildAlwaysRealIPModulePromise = downloadPreviousBuildPromise.then(() => buildAlwaysRealIPModule(rootSpan)); - const buildStreamServicePromise = downloadPreviousBuildPromise.then(() => buildStreamService()); + const buildStreamServicePromise = downloadPreviousBuildPromise.then(() => buildStreamService(rootSpan)); - const buildMicrosoftCdnPromise = downloadPreviousBuildPromise.then(() => buildMicrosoftCdn()); + const buildMicrosoftCdnPromise = downloadPreviousBuildPromise.then(() => buildMicrosoftCdn(rootSpan)); const buildSSPanelUIMAppProfilePromise = Promise.all([ downloadPreviousBuildPromise - ]).then(() => buildSSPanelUIMAppProfile()); + ]).then(() => buildSSPanelUIMAppProfile(rootSpan)); - const downloadMockAssetsPromise = downloadMockAssets(); + const downloadMockAssetsPromise = downloadMockAssets(rootSpan); - const buildCloudMounterRulesPromise = downloadPreviousBuildPromise.then(() => buildCloudMounterRules()); + const buildCloudMounterRulesPromise = downloadPreviousBuildPromise.then(() => buildCloudMounterRules(rootSpan)); - const stats = await Promise.all([ + await Promise.all([ downloadPreviousBuildPromise, buildCommonPromise, buildAntiBogusDomainPromise, @@ -101,11 +105,13 @@ import { buildCloudMounterRules } from './build-cloudmounter-rules'; ]); await Promise.all([ - buildPublic(), - validate() + buildPublic(rootSpan), + validate(rootSpan) ]); - printStats(stats); + rootSpan.stop(); + + printTraceResult(rootSpan.traceResult); // Finish the build to avoid leaking timer/fetch ref process.exit(0); @@ -115,21 +121,3 @@ import { buildCloudMounterRules } from './build-cloudmounter-rules'; process.exit(1); } })(); - -function printStats(stats: TaskResult[]): void { - stats.sort((a, b) => a.start - b.start); - - const longestTaskName = Math.max(...stats.map(i => i.taskName.length)); - const realStart = Math.min(...stats.map(i => i.start)); - const realEnd = Math.max(...stats.map(i => i.end)); - - const statsStep = ((realEnd - realStart) / 160) | 0; - - stats.forEach(stat => { - console.log( - `[${stat.taskName}]${' '.repeat(longestTaskName - stat.taskName.length)}`, - ' '.repeat(((stat.start - realStart) / statsStep) | 0), - '='.repeat(Math.max(((stat.end - stat.start) / statsStep) | 0, 1)) - ); - }); -} diff --git a/Build/lib/get-phishing-domains.ts b/Build/lib/get-phishing-domains.ts index 742ece1f..5d0f7b16 100644 --- a/Build/lib/get-phishing-domains.ts +++ b/Build/lib/get-phishing-domains.ts @@ -9,6 +9,7 @@ import { TTL } from './cache-filesystem'; import { isCI } from 'ci-info'; import { add as SetAdd } from 'mnemonist/set'; +import type { Span } from '../trace'; const WHITELIST_DOMAIN = new Set([ 'w3s.link', @@ -86,11 +87,11 @@ const BLACK_TLD = new Set([ 'za.com' ]); -export const getPhishingDomains = () => traceAsync('get phishing domains', async () => { +export const getPhishingDomains = (parentSpan: Span) => parentSpan.traceChild('get phishing domains').traceAsyncFn(async (span) => { const [domainSet, domainSet2, gorhill] = await Promise.all([ - processDomainLists('https://curbengh.github.io/phishing-filter/phishing-filter-domains.txt', true, TTL.THREE_HOURS()), + processDomainLists(span, 'https://curbengh.github.io/phishing-filter/phishing-filter-domains.txt', true, TTL.THREE_HOURS()), isCI - ? processDomainLists('https://phishing.army/download/phishing_army_blocklist.txt', true, TTL.THREE_HOURS()) + ? processDomainLists(span, 'https://phishing.army/download/phishing_army_blocklist.txt', true, TTL.THREE_HOURS()) : null, getGorhillPublicSuffixPromise() ]); @@ -98,7 +99,7 @@ export const getPhishingDomains = () => traceAsync('get phishing domains', async SetAdd(domainSet, domainSet2); } - traceSync.skip('* whitelisting phishing domains', () => { + span.traceChild('whitelisting phishing domains').traceSyncFn(() => { const trieForRemovingWhiteListed = createTrie(domainSet); for (const white of WHITELIST_DOMAIN) { const found = trieForRemovingWhiteListed.find(`.${white}`, false); @@ -112,7 +113,7 @@ export const getPhishingDomains = () => traceAsync('get phishing domains', async const domainCountMap: Record = {}; const getDomain = createCachedGorhillGetDomain(gorhill); - traceSync.skip('* process phishing domain set', () => { + span.traceChild('process phishing domain set').traceSyncFn(() => { const domainArr = Array.from(domainSet); for (let i = 0, len = domainArr.length; i < len; i++) { @@ -173,7 +174,7 @@ export const getPhishingDomains = () => traceAsync('get phishing domains', async } }); - const results = traceSync.skip('* get final phishing results', () => Object.entries(domainCountMap) + const results = span.traceChild('get final phishing results').traceSyncFn(() => Object.entries(domainCountMap) .filter(([, count]) => count >= 5) .map(([apexDomain]) => apexDomain)); diff --git a/Build/lib/parse-filter.ts b/Build/lib/parse-filter.ts index 1c915334..fa1d6876 100644 --- a/Build/lib/parse-filter.ts +++ b/Build/lib/parse-filter.ts @@ -10,12 +10,13 @@ import picocolors from 'picocolors'; import { normalizeDomain } from './normalize-domain'; import { fetchAssets } from './fetch-assets'; import { deserializeSet, fsCache, serializeSet } from './cache-filesystem'; +import type { Span } from '../trace'; const DEBUG_DOMAIN_TO_FIND: string | null = null; // example.com | null let foundDebugDomain = false; -export function processDomainLists(domainListsUrl: string, includeAllSubDomain = false, ttl: number | null = null) { - return traceAsync(`- processDomainLists: ${domainListsUrl}`, () => fsCache.apply( +export function processDomainLists(span: Span, domainListsUrl: string, includeAllSubDomain = false, ttl: number | null = null) { + return span.traceChild(`process domainlist: ${domainListsUrl}`).traceAsyncFn(() => fsCache.apply( domainListsUrl, async () => { const domainSets = new Set(); @@ -44,8 +45,8 @@ export function processDomainLists(domainListsUrl: string, includeAllSubDomain = } )); } -export function processHosts(hostsUrl: string, includeAllSubDomain = false, ttl: number | null = null) { - return traceAsync(`- processHosts: ${hostsUrl}`, () => fsCache.apply( +export function processHosts(span: Span, hostsUrl: string, includeAllSubDomain = false, ttl: number | null = null) { + return span.traceChild(`processhosts: ${hostsUrl}`).traceAsyncFn(() => fsCache.apply( hostsUrl, async () => { const domainSets = new Set(); @@ -95,11 +96,12 @@ const enum ParseType { } export async function processFilterRules( + span: Span, filterRulesUrl: string, fallbackUrls?: readonly string[] | undefined | null, ttl: number | null = null ): Promise<{ white: string[], black: string[], foundDebugDomain: boolean }> { - const [white, black, warningMessages] = await traceAsync(`- processFilterRules: ${filterRulesUrl}`, () => fsCache.apply fsCache.apply(importMetaPath: string, fn: () => Promise, customname: string | null = null) => { - const taskName = customname ?? path.basename(importMetaPath, path.extname(importMetaPath)); - return async () => { - console.log(`🏃 [${taskName}] Start executing`); - const start = Bun.nanoseconds(); - await fn(); - const end = Bun.nanoseconds(); - console.log(`✅ [${taskName}] ${picocolors.blue(`[${((end - start) / 1e6).toFixed(3)}ms]`)} Executed successfully`); - - return { start, end, taskName } as TaskResult; - }; -}; diff --git a/Build/trace/index.ts b/Build/trace/index.ts new file mode 100644 index 00000000..52fc806d --- /dev/null +++ b/Build/trace/index.ts @@ -0,0 +1,145 @@ +import path from 'path'; +import picocolors from 'picocolors'; + +const SPAN_STATUS_START = 0; +const SPAN_STATUS_END = 1; + +const NUM_OF_MS_IN_NANOSEC = 1_000_000; + +const spanTag = Symbol('span'); + +export interface TraceResult { + name: string, + start: number, + end: number, + children: TraceResult[] +} + +const rootTraceResult: TraceResult = { + name: 'root', + start: 0, + end: 0, + children: [] +}; + +export interface Span { + [spanTag]: true, + readonly stop: (time?: number) => void, + readonly traceChild: (name: string) => Span, + readonly traceSyncFn: (fn: (span: Span) => T) => T, + readonly traceAsyncFn: (fn: (span: Span) => T | Promise) => Promise, + readonly traceResult: TraceResult +} + +export const createSpan = (name: string, parentTraceResult?: TraceResult): Span => { + const start = Bun.nanoseconds(); + + let curTraceResult: TraceResult; + + if (parentTraceResult == null) { + curTraceResult = rootTraceResult; + } else { + curTraceResult = { + name, + start: start / NUM_OF_MS_IN_NANOSEC, + end: 0, + children: [] + }; + parentTraceResult.children.push(curTraceResult); + } + + let status: typeof SPAN_STATUS_START | typeof SPAN_STATUS_END = SPAN_STATUS_START; + + const stop = (time?: number) => { + if (status === SPAN_STATUS_END) { + throw new Error('span already stopped'); + } + const end = time ?? Bun.nanoseconds(); + + curTraceResult.end = end / NUM_OF_MS_IN_NANOSEC; + + status = SPAN_STATUS_END; + }; + + const traceChild = (name: string) => createSpan(name, curTraceResult); + + const span: Span = { + [spanTag]: true, + stop, + traceChild, + traceSyncFn(fn: (span: Span) => T) { + try { + return fn(span); + } finally { + span.stop(); + } + }, + async traceAsyncFn(fn: (span: Span) => T | Promise): Promise { + try { + return await fn(span); + } finally { + span.stop(); + } + }, + get traceResult() { + return curTraceResult; + } + }; + + // eslint-disable-next-line sukka/no-redundant-variable -- self reference + return span; +}; + +export const task = (importMetaPath: string, fn: (span: Span) => T, customname?: string) => { + const taskName = customname ?? path.basename(importMetaPath, path.extname(importMetaPath)); + return async (span?: Span) => { + if (span) { + return span.traceChild(taskName).traceAsyncFn(fn); + } + return fn(createSpan(taskName)); + }; +}; + +const isSpan = (obj: any): obj is Span => { + return typeof obj === 'object' && obj && spanTag in obj; +}; + +export const universalify = (taskname: string, fn: (this: void, ...args: A) => R) => { + return (...args: A) => { + const lastArg = args[args.length - 1]; + if (isSpan(lastArg)) { + return lastArg.traceChild(taskname).traceSyncFn(() => fn(...args)); + } + return fn(...args); + }; +}; + +export const printTraceResult = (traceResult: TraceResult = rootTraceResult, level = 0, isLast = false) => { + if (level === 0) { + printStats(traceResult.children); + } + + const prefix = (level > 0 ? ` ${'│ '.repeat(level - 1)}` : '') + (level > 0 ? (isLast ? '└─' : '├─') : ''); + + console.log(`${prefix} ${traceResult.name} ${picocolors.bold(`${(traceResult.end - traceResult.start).toFixed(2)}ms`)}`); + + traceResult.children.forEach((child, index, arr) => printTraceResult(child, level + 1, index === arr.length - 1)); +}; + +function printStats(stats: TraceResult[]): void { + stats.sort((a, b) => a.start - b.start); + + const longestTaskName = Math.max(...stats.map(i => i.name.length)); + const realStart = Math.min(...stats.map(i => i.start)); + const realEnd = Math.max(...stats.map(i => i.end)); + + const statsStep = ((realEnd - realStart) / 160) | 0; + + stats.forEach(stat => { + console.log( + `[${stat.name}]${' '.repeat(longestTaskName - stat.name.length)}`, + ' '.repeat(((stat.start - realStart) / statsStep) | 0), + '='.repeat(Math.max(((stat.end - stat.start) / statsStep) | 0, 1)) + ); + }); +} diff --git a/Build/validate-domainset.ts b/Build/validate-domainset.ts index df81cd7c..70026698 100644 --- a/Build/validate-domainset.ts +++ b/Build/validate-domainset.ts @@ -6,7 +6,7 @@ import path from 'path'; import listDir from '@sukka/listdir'; import { readFileByLine } from './lib/fetch-text-by-line'; import { processLine } from './lib/process-line'; -import { task } from './lib/trace-runner'; +import { task } from './trace'; const SPECIAL_SUFFIXES = new Set([ 'linodeobjects.com', // only *.linodeobjects.com are public suffix