Refactor/Perf: initial span tracer

Build/build-anti-bogus-domain.ts

@@ -3,7 +3,7 @@ import path from 'path';
 import { createRuleset } from './lib/create-file';
 import { fetchRemoteTextByLine, readFileByLine } from './lib/fetch-text-by-line';
 import { processLine } from './lib/process-line';
-import { task } from './lib/trace-runner';
+import { task } from './trace';
 import { SHARED_DESCRIPTION } from './lib/constants';
 import { isProbablyIpv4, isProbablyIpv6 } from './lib/is-fast-ip';
 import { TTL, deserializeArray, fsCache, serializeArray } from './lib/cache-filesystem';

Build/build-apple-cdn.ts

@@ -2,7 +2,8 @@
 import path from 'path';
 import { createRuleset } from './lib/create-file';
 import { parseFelixDnsmasq } from './lib/parse-dnsmasq';
-import { task, traceAsync } from './lib/trace-runner';
+import { traceAsync } from './lib/trace-runner';
+import { task } from './trace';
 import { SHARED_DESCRIPTION } from './lib/constants';
 import picocolors from 'picocolors';
 import { createMemoizedPromise } from './lib/memo-promise';

Build/build-cdn-conf.ts

@@ -2,7 +2,7 @@ import path from 'path';
 import { createRuleset } from './lib/create-file';
 import { readFileByLine } from './lib/fetch-text-by-line';
 import { createTrie } from './lib/trie';
-import { task } from './lib/trace-runner';
+import { task } from './trace';
 import { processLine } from './lib/process-line';
 import { SHARED_DESCRIPTION } from './lib/constants';
 import { getPublicSuffixListTextPromise } from './download-publicsuffixlist';

Build/build-chn-cidr.ts

@@ -2,7 +2,8 @@ import { fetchRemoteTextByLine } from './lib/fetch-text-by-line';
 import { resolve as pathResolve } from 'path';
 import { compareAndWriteFile, withBannerArray } from './lib/create-file';
 import { processLineFromReadline } from './lib/process-line';
-import { task, traceAsync, traceSync } from './lib/trace-runner';
+import { traceAsync, traceSync } from './lib/trace-runner';
+import { task } from './trace';
 
 import { exclude } from 'fast-cidr-tools';
 import picocolors from 'picocolors';

Build/build-cloudmounter-rules.ts

@@ -2,7 +2,7 @@ import path from 'path';
 import { DOMAINS, PROCESS_NAMES } from '../Source/non_ip/cloudmounter';
 import { SHARED_DESCRIPTION } from './lib/constants';
 import { createRuleset } from './lib/create-file';
-import { task } from './lib/trace-runner';
+import { task } from './trace';
 
 const outputSurgeDir = path.resolve(import.meta.dir, '../List');
 const outputClashDir = path.resolve(import.meta.dir, '../Clash');

Build/build-common.ts

@@ -6,7 +6,7 @@ import { readFileByLine } from './lib/fetch-text-by-line';
 import { processLine } from './lib/process-line';
 import { createRuleset } from './lib/create-file';
 import { domainDeduper } from './lib/domain-deduper';
-import { task } from './lib/trace-runner';
+import { task } from './trace';
 import { SHARED_DESCRIPTION } from './lib/constants';
 
 const MAGIC_COMMAND_SKIP = '# $ custom_build_script';

Build/build-domestic-ruleset.ts

@@ -4,7 +4,7 @@ import { DOMESTICS } from '../Source/non_ip/domestic';
 import { readFileByLine } from './lib/fetch-text-by-line';
 import { processLineFromReadline } from './lib/process-line';
 import { compareAndWriteFile, createRuleset } from './lib/create-file';
-import { task } from './lib/trace-runner';
+import { task } from './trace';
 import { SHARED_DESCRIPTION } from './lib/constants';
 import { createMemoizedPromise } from './lib/memo-promise';
 

Build/build-internal-cdn-rules.ts

@@ -2,7 +2,7 @@ import path from 'path';
 import { processLine } from './lib/process-line';
 import { readFileByLine } from './lib/fetch-text-by-line';
 import { sortDomains } from './lib/stable-sort-domain';
-import { task } from './lib/trace-runner';
+import { task } from './trace';
 import { compareAndWriteFile } from './lib/create-file';
 import { getGorhillPublicSuffixPromise } from './lib/get-gorhill-publicsuffix';
 

Build/build-internal-reverse-chn-cidr.ts

@@ -1,7 +1,7 @@
 import { fetchRemoteTextByLine } from './lib/fetch-text-by-line';
 import { processLineFromReadline } from './lib/process-line';
 import path from 'path';
-import { task } from './lib/trace-runner';
+import { task } from './trace';
 
 import { exclude, merge } from 'fast-cidr-tools';
 

Build/build-microsoft-cdn.ts

@@ -1,5 +1,6 @@
 import path from 'path';
-import { task, traceAsync } from './lib/trace-runner';
+import { traceAsync } from './lib/trace-runner';
+import { task } from './trace';
 import { createRuleset } from './lib/create-file';
 import { fetchRemoteTextByLine } from './lib/fetch-text-by-line';
 import { createTrie } from './lib/trie';

Build/build-public.ts

@@ -1,5 +1,5 @@
 import path from 'path';
-import { task } from './lib/trace-runner';
+import { task } from './trace';
 import { treeDir } from './lib/tree-dir';
 import type { TreeType, TreeTypeArray } from './lib/tree-dir';
 import listDir from '@sukka/listdir';

Build/build-reject-domainset.ts

@@ -11,7 +11,7 @@ import { domainDeduper } from './lib/domain-deduper';
 import createKeywordFilter from './lib/aho-corasick';
 import { readFileByLine } from './lib/fetch-text-by-line';
 import { sortDomains } from './lib/stable-sort-domain';
-import { traceSync, task, traceAsync } from './lib/trace-runner';
+import { task } from './trace';
 import { getGorhillPublicSuffixPromise } from './lib/get-gorhill-publicsuffix';
 import * as tldts from 'tldts';
 import { SHARED_DESCRIPTION } from './lib/constants';
@@ -20,27 +20,29 @@ import { getPhishingDomains } from './lib/get-phishing-domains';
 import * as SetHelpers from 'mnemonist/set';
 import { setAddFromArray } from './lib/set-add-from-array';
 
-export const buildRejectDomainSet = task(import.meta.path, async () => {
+export const buildRejectDomainSet = task(import.meta.path, async (span) => {
   /** Whitelists */
   const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST);
 
   const domainSets = new Set<string>();
 
   // Parse from AdGuard Filters
-  const [gorhill, shouldStop] = await traceAsync('* Download and process Hosts / AdBlock Filter Rules', async () => {
+  const [gorhill, shouldStop] = await span
+    .traceChild('download and process hosts / adblock filter rules')
+    .traceAsyncFn(async () => {
       let shouldStop = false;
 
       const [gorhill] = await Promise.all([
         getGorhillPublicSuffixPromise(),
         // Parse from remote hosts & domain lists
-      ...HOSTS.map(entry => processHosts(entry[0], entry[1], entry[2]).then(hosts => {
+        ...HOSTS.map(entry => processHosts(span, entry[0], entry[1], entry[2]).then(hosts => {
           SetHelpers.add(domainSets, hosts);
         })),
-      ...DOMAIN_LISTS.map(entry => processDomainLists(entry[0], entry[1], entry[2])),
+        ...DOMAIN_LISTS.map(entry => processDomainLists(span, entry[0], entry[1], entry[2])),
         ...ADGUARD_FILTERS.map(input => {
           const promise = typeof input === 'string'
-          ? processFilterRules(input)
+            ? processFilterRules(span, input)
-          : processFilterRules(input[0], input[1], input[2]);
+            : processFilterRules(span, input[0], input[1], input[2]);
 
           return promise.then(({ white, black, foundDebugDomain }) => {
             if (foundDebugDomain) {
@@ -54,11 +56,11 @@ export const buildRejectDomainSet = task(import.meta.path, async () => {
         ...([
           'https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt',
           'https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt'
-      ].map(input => processFilterRules(input).then(({ white, black }) => {
+        ].map(input => processFilterRules(span, input).then(({ white, black }) => {
           setAddFromArray(filterRuleWhitelistDomainSets, white);
           setAddFromArray(filterRuleWhitelistDomainSets, black);
         }))),
-      getPhishingDomains().then(([purePhishingDomains, fullPhishingDomainSet]) => {
+        getPhishingDomains(span).then(([purePhishingDomains, fullPhishingDomainSet]) => {
           SetHelpers.add(domainSets, fullPhishingDomainSet);
           setAddFromArray(domainSets, purePhishingDomains);
         }),
@@ -94,7 +96,7 @@ export const buildRejectDomainSet = task(import.meta.path, async () => {
   console.log(`Import ${previousSize} rules from Hosts / AdBlock Filter Rules & reject_sukka.conf!`);
 
   // Dedupe domainSets
-  await traceAsync('* Dedupe from black keywords/suffixes', async () => {
+  await span.traceChild('dedupe from black keywords/suffixes').traceAsyncFn(async () => {
   /** Collect DOMAIN-SUFFIX from non_ip/reject.conf for deduplication */
     const domainSuffixSet = new Set<string>();
     /** Collect DOMAIN-KEYWORD from non_ip/reject.conf for deduplication */
@@ -146,13 +148,13 @@ export const buildRejectDomainSet = task(import.meta.path, async () => {
   previousSize = domainSets.size;
 
   // Dedupe domainSets
-  const dudupedDominArray = traceSync('* Dedupe from covered subdomain', () => domainDeduper(Array.from(domainSets)));
+  const dudupedDominArray = span.traceChild('dedupe from covered subdomain').traceSyncFn(() => domainDeduper(Array.from(domainSets)));
+
   console.log(`Deduped ${previousSize - dudupedDominArray.length} rules from covered subdomain!`);
   console.log(`Final size ${dudupedDominArray.length}`);
 
   // Create reject stats
-  const rejectDomainsStats: Array<[string, number]> = traceSync(
+  const rejectDomainsStats: Array<[string, number]> = span.traceChild('create reject stats').traceSyncFn(
-    '* Collect reject domain stats',
     () => Object.entries(
       dudupedDominArray.reduce<Record<string, number>>((acc, cur) => {
         const suffix = tldts.getDomain(cur, { allowPrivateDomains: false, detectIp: false, validateHostname: false });
@@ -189,7 +191,7 @@ export const buildRejectDomainSet = task(import.meta.path, async () => {
       'Sukka\'s Ruleset - Reject Base',
       description,
       new Date(),
-      traceSync('* Sort reject domainset', () => sortDomains(dudupedDominArray, gorhill)),
+      span.traceChild('sort reject domainset').traceSyncFn(() => sortDomains(dudupedDominArray, gorhill)),
       'domainset',
       path.resolve(import.meta.dir, '../List/domainset/reject.conf'),
       path.resolve(import.meta.dir, '../Clash/domainset/reject.txt')

Build/build-sgmodule-always-realip.ts

@@ -1,5 +1,5 @@
 import path from 'path';
-import { task } from './lib/trace-runner';
+import { task } from './trace';
 import { compareAndWriteFile } from './lib/create-file';
 
 const HOSTNAMES = [

Build/build-sgmodule-redirect.ts

@@ -1,5 +1,5 @@
 import path from 'path';
-import { task } from './lib/trace-runner';
+import { task } from './trace';
 import { compareAndWriteFile } from './lib/create-file';
 import * as tldts from 'tldts';
 

Build/build-speedtest-domainset.ts

@@ -5,7 +5,7 @@ import { sortDomains } from './lib/stable-sort-domain';
 
 import { Sema } from 'async-sema';
 import * as tldts from 'tldts';
-import { task } from './lib/trace-runner';
+import { task } from './trace';
 import { fetchWithRetry } from './lib/fetch-retry';
 import { SHARED_DESCRIPTION } from './lib/constants';
 import { getGorhillPublicSuffixPromise } from './lib/get-gorhill-publicsuffix';
@@ -35,11 +35,8 @@ const querySpeedtestApi = async (keyword: string): Promise<Array<string | null>>
 
   try {
     const randomUserAgent = topUserAgents[Math.floor(Math.random() * topUserAgents.length)];
-    const key = `fetch speedtest endpoints: ${keyword}`;
-    console.log(key);
-    console.time(key);
 
-    const json = await fsCache.apply(
+    return await fsCache.apply(
       url,
       () => s.acquire().then(() => fetchWithRetry(url, {
         headers: {
@@ -77,17 +74,13 @@ const querySpeedtestApi = async (keyword: string): Promise<Array<string | null>>
         deserializer: deserializeArray
       }
     );
-
-    console.timeEnd(key);
-
-    return json;
   } catch (e) {
-    console.log(e);
+    console.error(e);
     return [];
   }
 };
 
-export const buildSpeedtestDomainSet = task(import.meta.path, async () => {
+export const buildSpeedtestDomainSet = task(import.meta.path, async (span) => {
   // Predefined domainset
   /** @type {Set<string>} */
   const domains = new Set<string>([
@@ -197,7 +190,7 @@ export const buildSpeedtestDomainSet = task(import.meta.path, async () => {
       'Brazil',
       'Turkey'
     ]).reduce<Record<string, Promise<void>>>((pMap, keyword) => {
-      pMap[keyword] = querySpeedtestApi(keyword).then(hostnameGroup => {
+      pMap[keyword] = span.traceChild(`fetch speedtest endpoints: ${keyword}`).traceAsyncFn(() => querySpeedtestApi(keyword)).then(hostnameGroup => {
         hostnameGroup.forEach(hostname => {
           if (hostname) {
             domains.add(hostname);
@@ -224,7 +217,7 @@ export const buildSpeedtestDomainSet = task(import.meta.path, async () => {
   });
 
   const gorhill = await getGorhillPublicSuffixPromise();
-  const deduped = sortDomains(domainDeduper(Array.from(domains)), gorhill);
+  const deduped = span.traceChild('sort result').traceSyncFn(() => sortDomains(domainDeduper(Array.from(domains)), gorhill));
 
   const description = [
     ...SHARED_DESCRIPTION,

Build/build-sspanel-appprofile.ts

@@ -3,7 +3,7 @@ import { getDomesticDomainsRulesetPromise } from './build-domestic-ruleset';
 import { surgeRulesetToClashClassicalTextRuleset } from './lib/clash';
 import { readFileByLine } from './lib/fetch-text-by-line';
 import { processLineFromReadline } from './lib/process-line';
-import { task } from './lib/trace-runner';
+import { task } from './trace';
 import path from 'path';
 
 import { ALL as AllStreamServices } from '../Source/stream';

Build/build-stream-service.ts

@@ -1,5 +1,5 @@
 // @ts-check
-import { task } from './lib/trace-runner';
+import { task } from './trace';
 
 import path from 'path';
 import { createRuleset } from './lib/create-file';

Build/build-telegram-cidr.ts

@@ -5,7 +5,7 @@ import path from 'path';
 import { isProbablyIpv4, isProbablyIpv6 } from './lib/is-fast-ip';
 import { processLine } from './lib/process-line';
 import { createRuleset } from './lib/create-file';
-import { task } from './lib/trace-runner';
+import { task } from './trace';
 import { SHARED_DESCRIPTION } from './lib/constants';
 import { createMemoizedPromise } from './lib/memo-promise';
 

Build/download-mock-assets.ts

@@ -1,5 +1,5 @@
 import picocolors from 'picocolors';
-import { task } from './lib/trace-runner';
+import { task } from './trace';
 import path from 'path';
 import { fetchWithRetry } from './lib/fetch-retry';
 

Build/download-previous-build.ts

@@ -4,7 +4,7 @@ import path from 'path';
 import { pipeline } from 'stream/promises';
 import { readFileByLine } from './lib/fetch-text-by-line';
 import { isCI } from 'ci-info';
-import { task } from './lib/trace-runner';
+import { task } from './trace';
 import { defaultRequestInit, fetchWithRetry } from './lib/fetch-retry';
 import tarStream from 'tar-stream';
 import zlib from 'zlib';
@@ -13,11 +13,14 @@ import { Readable } from 'stream';
 const IS_READING_BUILD_OUTPUT = 1 << 2;
 const ALL_FILES_EXISTS = 1 << 3;
 
-export const downloadPreviousBuild = task(import.meta.path, async () => {
+export const downloadPreviousBuild = task(import.meta.path, async (span) => {
   const buildOutputList: string[] = [];
 
   let flag = 1 | ALL_FILES_EXISTS;
 
+  await span
+    .traceChild('read .gitignore')
+    .traceAsyncFn(async () => {
       for await (const line of readFileByLine(path.resolve(import.meta.dir, '../.gitignore'))) {
         if (line === '# $ build output') {
           flag = flag | IS_READING_BUILD_OUTPUT;
@@ -36,6 +39,7 @@ export const downloadPreviousBuild = task(import.meta.path, async () => {
           }
         }
       }
+    });
 
   if (isCI) {
     flag = flag & ~ALL_FILES_EXISTS;
@@ -48,6 +52,9 @@ export const downloadPreviousBuild = task(import.meta.path, async () => {
 
   const filesList = buildOutputList.map(f => path.join('ruleset.skk.moe-master', f));
 
+  return span
+    .traceChild('download & extract previoud build')
+    .traceAsyncFn(async () => {
       const resp = await fetchWithRetry('https://codeload.github.com/sukkalab/ruleset.skk.moe/tar.gz/master', defaultRequestInit);
 
       if (!resp.body) {
@@ -84,6 +91,7 @@ export const downloadPreviousBuild = task(import.meta.path, async () => {
           createWriteStream(targetPath)
         );
       }
+    });
 });
 
 if (import.meta.main) {

Build/index.ts

@@ -23,29 +23,33 @@ import { buildSSPanelUIMAppProfile } from './build-sspanel-appprofile';
 import { buildPublic } from './build-public';
 import { downloadMockAssets } from './download-mock-assets';
 
-import type { TaskResult } from './lib/trace-runner';
 import { buildCloudMounterRules } from './build-cloudmounter-rules';
 
+import { createSpan, printTraceResult } from './trace';
+
 (async () => {
   console.log('Bun version:', Bun.version, Bun.revision);
 
+  const rootSpan = createSpan('root');
+
   try {
     // TODO: restore this once Bun has fixed their worker
     // const buildInternalReverseChnCIDRWorker = new Worker(new URL('./workers/build-internal-reverse-chn-cidr-worker.ts', import.meta.url));
 
-    const downloadPreviousBuildPromise = downloadPreviousBuild();
+    const downloadPreviousBuildPromise = downloadPreviousBuild(rootSpan);
-    const buildCommonPromise = downloadPreviousBuildPromise.then(() => buildCommon());
+
-    const buildAntiBogusDomainPromise = downloadPreviousBuildPromise.then(() => buildAntiBogusDomain());
+    const buildCommonPromise = downloadPreviousBuildPromise.then(() => buildCommon(rootSpan));
-    const buildAppleCdnPromise = downloadPreviousBuildPromise.then(() => buildAppleCdn());
+    const buildAntiBogusDomainPromise = downloadPreviousBuildPromise.then(() => buildAntiBogusDomain(rootSpan));
-    const buildCdnConfPromise = downloadPreviousBuildPromise.then(() => buildCdnConf());
+    const buildAppleCdnPromise = downloadPreviousBuildPromise.then(() => buildAppleCdn(rootSpan));
-    const buildRejectDomainSetPromise = downloadPreviousBuildPromise.then(() => buildRejectDomainSet());
+    const buildCdnConfPromise = downloadPreviousBuildPromise.then(() => buildCdnConf(rootSpan));
-    const buildTelegramCIDRPromise = downloadPreviousBuildPromise.then(() => buildTelegramCIDR());
+    const buildRejectDomainSetPromise = downloadPreviousBuildPromise.then(() => buildRejectDomainSet(rootSpan));
-    const buildChnCidrPromise = downloadPreviousBuildPromise.then(() => buildChnCidr());
+    const buildTelegramCIDRPromise = downloadPreviousBuildPromise.then(() => buildTelegramCIDR(rootSpan));
-    const buildSpeedtestDomainSetPromise = downloadPreviousBuildPromise.then(() => buildSpeedtestDomainSet());
+    const buildChnCidrPromise = downloadPreviousBuildPromise.then(() => buildChnCidr(rootSpan));
+    const buildSpeedtestDomainSetPromise = downloadPreviousBuildPromise.then(() => buildSpeedtestDomainSet(rootSpan));
     const buildInternalCDNDomainsPromise = Promise.all([
       buildCommonPromise,
       buildCdnConfPromise
-    ]).then(() => buildInternalCDNDomains());
+    ]).then(() => buildInternalCDNDomains(rootSpan));
 
     // const buildInternalReverseChnCIDRPromise = new Promise<TaskResult>(resolve => {
     //   const handleMessage = (e: MessageEvent<TaskResult>) => {
@@ -60,24 +64,24 @@ import { buildCloudMounterRules } from './build-cloudmounter-rules';
     // });
 
     // const buildInternalChnDomainsPromise = buildInternalChnDomains();
-    const buildDomesticRulesetPromise = downloadPreviousBuildPromise.then(() => buildDomesticRuleset());
+    const buildDomesticRulesetPromise = downloadPreviousBuildPromise.then(() => buildDomesticRuleset(rootSpan));
 
-    const buildRedirectModulePromise = downloadPreviousBuildPromise.then(() => buildRedirectModule());
+    const buildRedirectModulePromise = downloadPreviousBuildPromise.then(() => buildRedirectModule(rootSpan));
-    const buildAlwaysRealIPModulePromise = downloadPreviousBuildPromise.then(() => buildAlwaysRealIPModule());
+    const buildAlwaysRealIPModulePromise = downloadPreviousBuildPromise.then(() => buildAlwaysRealIPModule(rootSpan));
 
-    const buildStreamServicePromise = downloadPreviousBuildPromise.then(() => buildStreamService());
+    const buildStreamServicePromise = downloadPreviousBuildPromise.then(() => buildStreamService(rootSpan));
 
-    const buildMicrosoftCdnPromise = downloadPreviousBuildPromise.then(() => buildMicrosoftCdn());
+    const buildMicrosoftCdnPromise = downloadPreviousBuildPromise.then(() => buildMicrosoftCdn(rootSpan));
 
     const buildSSPanelUIMAppProfilePromise = Promise.all([
       downloadPreviousBuildPromise
-    ]).then(() => buildSSPanelUIMAppProfile());
+    ]).then(() => buildSSPanelUIMAppProfile(rootSpan));
 
-    const downloadMockAssetsPromise = downloadMockAssets();
+    const downloadMockAssetsPromise = downloadMockAssets(rootSpan);
 
-    const buildCloudMounterRulesPromise = downloadPreviousBuildPromise.then(() => buildCloudMounterRules());
+    const buildCloudMounterRulesPromise = downloadPreviousBuildPromise.then(() => buildCloudMounterRules(rootSpan));
 
-    const stats = await Promise.all([
+    await Promise.all([
       downloadPreviousBuildPromise,
       buildCommonPromise,
       buildAntiBogusDomainPromise,
@@ -101,11 +105,13 @@ import { buildCloudMounterRules } from './build-cloudmounter-rules';
     ]);
 
     await Promise.all([
-      buildPublic(),
+      buildPublic(rootSpan),
-      validate()
+      validate(rootSpan)
     ]);
 
-    printStats(stats);
+    rootSpan.stop();
+
+    printTraceResult(rootSpan.traceResult);
 
     // Finish the build to avoid leaking timer/fetch ref
     process.exit(0);
@@ -115,21 +121,3 @@ import { buildCloudMounterRules } from './build-cloudmounter-rules';
     process.exit(1);
   }
 })();
-
-function printStats(stats: TaskResult[]): void {
-  stats.sort((a, b) => a.start - b.start);
-
-  const longestTaskName = Math.max(...stats.map(i => i.taskName.length));
-  const realStart = Math.min(...stats.map(i => i.start));
-  const realEnd = Math.max(...stats.map(i => i.end));
-
-  const statsStep = ((realEnd - realStart) / 160) | 0;
-
-  stats.forEach(stat => {
-    console.log(
-      `[${stat.taskName}]${' '.repeat(longestTaskName - stat.taskName.length)}`,
-      ' '.repeat(((stat.start - realStart) / statsStep) | 0),
-      '='.repeat(Math.max(((stat.end - stat.start) / statsStep) | 0, 1))
-    );
-  });
-}

Build/lib/get-phishing-domains.ts

@@ -9,6 +9,7 @@ import { TTL } from './cache-filesystem';
 import { isCI } from 'ci-info';
 
 import { add as SetAdd } from 'mnemonist/set';
+import type { Span } from '../trace';
 
 const WHITELIST_DOMAIN = new Set([
   'w3s.link',
@@ -86,11 +87,11 @@ const BLACK_TLD = new Set([
   'za.com'
 ]);
 
-export const getPhishingDomains = () => traceAsync('get phishing domains', async () => {
+export const getPhishingDomains = (parentSpan: Span) => parentSpan.traceChild('get phishing domains').traceAsyncFn(async (span) => {
   const [domainSet, domainSet2, gorhill] = await Promise.all([
-    processDomainLists('https://curbengh.github.io/phishing-filter/phishing-filter-domains.txt', true, TTL.THREE_HOURS()),
+    processDomainLists(span, 'https://curbengh.github.io/phishing-filter/phishing-filter-domains.txt', true, TTL.THREE_HOURS()),
     isCI
-      ? processDomainLists('https://phishing.army/download/phishing_army_blocklist.txt', true, TTL.THREE_HOURS())
+      ? processDomainLists(span, 'https://phishing.army/download/phishing_army_blocklist.txt', true, TTL.THREE_HOURS())
       : null,
     getGorhillPublicSuffixPromise()
   ]);
@@ -98,7 +99,7 @@ export const getPhishingDomains = () => traceAsync('get phishing domains', async
     SetAdd(domainSet, domainSet2);
   }
 
-  traceSync.skip('* whitelisting phishing domains', () => {
+  span.traceChild('whitelisting phishing domains').traceSyncFn(() => {
     const trieForRemovingWhiteListed = createTrie(domainSet);
     for (const white of WHITELIST_DOMAIN) {
       const found = trieForRemovingWhiteListed.find(`.${white}`, false);
@@ -112,7 +113,7 @@ export const getPhishingDomains = () => traceAsync('get phishing domains', async
   const domainCountMap: Record<string, number> = {};
   const getDomain = createCachedGorhillGetDomain(gorhill);
 
-  traceSync.skip('* process phishing domain set', () => {
+  span.traceChild('process phishing domain set').traceSyncFn(() => {
     const domainArr = Array.from(domainSet);
 
     for (let i = 0, len = domainArr.length; i < len; i++) {
@@ -173,7 +174,7 @@ export const getPhishingDomains = () => traceAsync('get phishing domains', async
     }
   });
 
-  const results = traceSync.skip('* get final phishing results', () => Object.entries(domainCountMap)
+  const results = span.traceChild('get final phishing results').traceSyncFn(() => Object.entries(domainCountMap)
     .filter(([, count]) => count >= 5)
     .map(([apexDomain]) => apexDomain));
 

Build/lib/parse-filter.ts

@@ -10,12 +10,13 @@ import picocolors from 'picocolors';
 import { normalizeDomain } from './normalize-domain';
 import { fetchAssets } from './fetch-assets';
 import { deserializeSet, fsCache, serializeSet } from './cache-filesystem';
+import type { Span } from '../trace';
 
 const DEBUG_DOMAIN_TO_FIND: string | null = null; // example.com | null
 let foundDebugDomain = false;
 
-export function processDomainLists(domainListsUrl: string, includeAllSubDomain = false, ttl: number | null = null) {
+export function processDomainLists(span: Span, domainListsUrl: string, includeAllSubDomain = false, ttl: number | null = null) {
-  return traceAsync(`- processDomainLists: ${domainListsUrl}`, () => fsCache.apply(
+  return span.traceChild(`process domainlist: ${domainListsUrl}`).traceAsyncFn(() => fsCache.apply(
     domainListsUrl,
     async () => {
       const domainSets = new Set<string>();
@@ -44,8 +45,8 @@ export function processDomainLists(domainListsUrl: string, includeAllSubDomain =
     }
   ));
 }
-export function processHosts(hostsUrl: string, includeAllSubDomain = false, ttl: number | null = null) {
+export function processHosts(span: Span, hostsUrl: string, includeAllSubDomain = false, ttl: number | null = null) {
-  return traceAsync(`- processHosts: ${hostsUrl}`, () => fsCache.apply(
+  return span.traceChild(`processhosts: ${hostsUrl}`).traceAsyncFn(() => fsCache.apply(
     hostsUrl,
     async () => {
       const domainSets = new Set<string>();
@@ -95,11 +96,12 @@ const enum ParseType {
 }
 
 export async function processFilterRules(
+  span: Span,
   filterRulesUrl: string,
   fallbackUrls?: readonly string[] | undefined | null,
   ttl: number | null = null
 ): Promise<{ white: string[], black: string[], foundDebugDomain: boolean }> {
-  const [white, black, warningMessages] = await traceAsync(`- processFilterRules: ${filterRulesUrl}`, () => fsCache.apply<Readonly<[
+  const [white, black, warningMessages] = await span.traceChild('process filter rules: domainListsUrl').traceAsyncFn(() => fsCache.apply<Readonly<[
     white: string[],
     black: string[],
     warningMessages: string[]

Build/lib/trace-runner.ts

@@ -25,16 +25,3 @@ export interface TaskResult {
   readonly end: number,
   readonly taskName: string
 }
-
-export const task = <T>(importMetaPath: string, fn: () => Promise<T>, customname: string | null = null) => {
-  const taskName = customname ?? path.basename(importMetaPath, path.extname(importMetaPath));
-  return async () => {
-    console.log(`🏃 [${taskName}] Start executing`);
-    const start = Bun.nanoseconds();
-    await fn();
-    const end = Bun.nanoseconds();
-    console.log(`✅ [${taskName}] ${picocolors.blue(`[${((end - start) / 1e6).toFixed(3)}ms]`)} Executed successfully`);
-
-    return { start, end, taskName } as TaskResult;
-  };
-};

Build/trace/index.ts

@@ -0,0 +1,145 @@
+import path from 'path';
+import picocolors from 'picocolors';
+
+const SPAN_STATUS_START = 0;
+const SPAN_STATUS_END = 1;
+
+const NUM_OF_MS_IN_NANOSEC = 1_000_000;
+
+const spanTag = Symbol('span');
+
+export interface TraceResult {
+  name: string,
+  start: number,
+  end: number,
+  children: TraceResult[]
+}
+
+const rootTraceResult: TraceResult = {
+  name: 'root',
+  start: 0,
+  end: 0,
+  children: []
+};
+
+export interface Span {
+  [spanTag]: true,
+  readonly stop: (time?: number) => void,
+  readonly traceChild: (name: string) => Span,
+  readonly traceSyncFn: <T>(fn: (span: Span) => T) => T,
+  readonly traceAsyncFn: <T>(fn: (span: Span) => T | Promise<T>) => Promise<T>,
+  readonly traceResult: TraceResult
+}
+
+export const createSpan = (name: string, parentTraceResult?: TraceResult): Span => {
+  const start = Bun.nanoseconds();
+
+  let curTraceResult: TraceResult;
+
+  if (parentTraceResult == null) {
+    curTraceResult = rootTraceResult;
+  } else {
+    curTraceResult = {
+      name,
+      start: start / NUM_OF_MS_IN_NANOSEC,
+      end: 0,
+      children: []
+    };
+    parentTraceResult.children.push(curTraceResult);
+  }
+
+  let status: typeof SPAN_STATUS_START | typeof SPAN_STATUS_END = SPAN_STATUS_START;
+
+  const stop = (time?: number) => {
+    if (status === SPAN_STATUS_END) {
+      throw new Error('span already stopped');
+    }
+    const end = time ?? Bun.nanoseconds();
+
+    curTraceResult.end = end / NUM_OF_MS_IN_NANOSEC;
+
+    status = SPAN_STATUS_END;
+  };
+
+  const traceChild = (name: string) => createSpan(name, curTraceResult);
+
+  const span: Span = {
+    [spanTag]: true,
+    stop,
+    traceChild,
+    traceSyncFn<T>(fn: (span: Span) => T) {
+      try {
+        return fn(span);
+      } finally {
+        span.stop();
+      }
+    },
+    async traceAsyncFn<T>(fn: (span: Span) => T | Promise<T>): Promise<T> {
+      try {
+        return await fn(span);
+      } finally {
+        span.stop();
+      }
+    },
+    get traceResult() {
+      return curTraceResult;
+    }
+  };
+
+  // eslint-disable-next-line sukka/no-redundant-variable -- self reference
+  return span;
+};
+
+export const task = <T>(importMetaPath: string, fn: (span: Span) => T, customname?: string) => {
+  const taskName = customname ?? path.basename(importMetaPath, path.extname(importMetaPath));
+  return async (span?: Span) => {
+    if (span) {
+      return span.traceChild(taskName).traceAsyncFn(fn);
+    }
+    return fn(createSpan(taskName));
+  };
+};
+
+const isSpan = (obj: any): obj is Span => {
+  return typeof obj === 'object' && obj && spanTag in obj;
+};
+
+export const universalify = <A extends any[], R>(taskname: string, fn: (this: void, ...args: A) => R) => {
+  return (...args: A) => {
+    const lastArg = args[args.length - 1];
+    if (isSpan(lastArg)) {
+      return lastArg.traceChild(taskname).traceSyncFn(() => fn(...args));
+    }
+    return fn(...args);
+  };
+};
+
+export const printTraceResult = (traceResult: TraceResult = rootTraceResult, level = 0, isLast = false) => {
+  if (level === 0) {
+    printStats(traceResult.children);
+  }
+
+  const prefix = (level > 0 ? `  ${'│  '.repeat(level - 1)}` : '') + (level > 0 ? (isLast ? '└─' : '├─') : '');
+
+  console.log(`${prefix} ${traceResult.name} ${picocolors.bold(`${(traceResult.end - traceResult.start).toFixed(2)}ms`)}`);
+
+  traceResult.children.forEach((child, index, arr) => printTraceResult(child, level + 1, index === arr.length - 1));
+};
+
+function printStats(stats: TraceResult[]): void {
+  stats.sort((a, b) => a.start - b.start);
+
+  const longestTaskName = Math.max(...stats.map(i => i.name.length));
+  const realStart = Math.min(...stats.map(i => i.start));
+  const realEnd = Math.max(...stats.map(i => i.end));
+
+  const statsStep = ((realEnd - realStart) / 160) | 0;
+
+  stats.forEach(stat => {
+    console.log(
+      `[${stat.name}]${' '.repeat(longestTaskName - stat.name.length)}`,
+      ' '.repeat(((stat.start - realStart) / statsStep) | 0),
+      '='.repeat(Math.max(((stat.end - stat.start) / statsStep) | 0, 1))
+    );
+  });
+}

Build/validate-domainset.ts

@@ -6,7 +6,7 @@ import path from 'path';
 import listDir from '@sukka/listdir';
 import { readFileByLine } from './lib/fetch-text-by-line';
 import { processLine } from './lib/process-line';
-import { task } from './lib/trace-runner';
+import { task } from './trace';
 
 const SPECIAL_SUFFIXES = new Set([
   'linodeobjects.com', // only *.linodeobjects.com are public suffix