Perf: offload reject fetch & parse into a worker thread

Build/build-reject-domainset.ts

@@ -2,11 +2,9 @@
 import path from 'node:path';
 import process from 'node:process';
 
-import { processHostsWithPreload } from './lib/parse-filter/hosts';
+import type { ProcessFilterRulesResult } from './lib/parse-filter/filters';
-import { processDomainListsWithPreload } from './lib/parse-filter/domainlists';
-import { processFilterRulesWithPreload } from './lib/parse-filter/filters';
 
-import { HOSTS, ADGUARD_FILTERS, PREDEFINED_WHITELIST, DOMAIN_LISTS, HOSTS_EXTRA, DOMAIN_LISTS_EXTRA, ADGUARD_FILTERS_EXTRA, ADGUARD_FILTERS_WHITELIST, PHISHING_HOSTS_EXTRA, PHISHING_DOMAIN_LISTS_EXTRA, BOGUS_NXDOMAIN_DNSMASQ, ENFORCED_BLACKLIST_FROM_WHITELIST } from './constants/reject-data-source';
+import { HOSTS, PREDEFINED_WHITELIST, DOMAIN_LISTS, HOSTS_EXTRA, DOMAIN_LISTS_EXTRA, PHISHING_HOSTS_EXTRA, PHISHING_DOMAIN_LISTS_EXTRA, BOGUS_NXDOMAIN_DNSMASQ, ENFORCED_BLACKLIST_FROM_WHITELIST } from './constants/reject-data-source';
 import { readFileIntoProcessedArray } from './lib/fetch-text-by-line';
 import { SpanCategory, task } from './trace';
 // tldts-experimental is way faster than tldts, but very little bit inaccurate
@@ -17,31 +15,27 @@ import { SHARED_DESCRIPTION } from './constants/description';
 import { addArrayElementsToSet } from 'foxts/add-array-elements-to-set';
 import { OUTPUT_INTERNAL_DIR, SOURCE_DIR } from './constants/dir';
 import { DomainsetOutput, AdGuardHomeOutput } from './lib/rules/domainset';
-import { foundDebugDomain } from './lib/parse-filter/shared';
 import { createWorker } from './lib/worker';
-import type { MaybePromise } from './lib/misc';
 import { RulesetOutput } from './lib/rules/ruleset';
 import { fetchAssets } from './lib/fetch-assets';
 import { AUGUST_ASN, HUIZE_ASN } from '../Source/ip/badboy_asn';
-import { arrayPushNonNullish } from 'foxts/array-push-non-nullish';
 
 const readLocalRejectDomainsetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/reject.conf'));
 const readLocalRejectExtraDomainsetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/reject_extra.conf'));
 const readLocalRejectRulesetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/reject.conf'));
 const readLocalRejectIpListPromise = readFileIntoProcessedArray(path.resolve(SOURCE_DIR, 'ip/reject.conf'));
 
-const hostsDownloads = HOSTS.map(entry => processHostsWithPreload(...entry));
+// Downloading + parsing the remote sources is ~1s of CPU that used to sit on the
-const hostsExtraDownloads = HOSTS_EXTRA.map(entry => processHostsWithPreload(...entry));
+// main thread's critical path; the parsed arrays cross back in ~25ms (see
-const domainListsDownloads = DOMAIN_LISTS.map(entry => processDomainListsWithPreload(...entry));
+// lib/worker-transfer.bench.ts). Two threads because jest-worker runs one call at
-const domainListsExtraDownloads = DOMAIN_LISTS_EXTRA.map(entry => processDomainListsWithPreload(...entry));
+// a time per thread and the two jobs must overlap. Booted at import time so the
-const adguardFiltersDownloads = ADGUARD_FILTERS.map(entry => processFilterRulesWithPreload(...entry));
+// thread spin-up (loading the module graph) overlaps with the rest of startup.
-const adguardFiltersExtraDownloads = ADGUARD_FILTERS_EXTRA.map(entry => processFilterRulesWithPreload(...entry));
+const rejectWorker = createWorker<typeof import('./lib/reject.worker')>(
-const adguardFiltersWhitelistsDownloads = ADGUARD_FILTERS_WHITELIST.map(entry => processFilterRulesWithPreload(...entry));
+  require.resolve('./lib/reject.worker'),
+  2
+)(['getPhishingDomains', 'getRejectSources']);
 
 export const buildRejectDomainSet = task(require.main === module, __filename)(async (span) => {
-  const phishingWorker = createWorker<typeof import('./lib/get-phishing-domains')>(
-    require.resolve('./lib/get-phishing-domains')
-  )(['getPhishingDomains']);
   const rejectDomainsetOutput = new DomainsetOutput(span, 'reject')
     .withTitle('Sukka\'s Ruleset - Reject Base')
     .appendDescription(
@@ -111,123 +105,107 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as
 
   rejectIPOutput.addFromRuleset(readLocalRejectIpListPromise);
 
-  const appendArrayToRejectOutput = (source: MaybePromise<AsyncIterable<string> | Iterable<string> | string[]>) => rejectDomainsetOutput.addFromDomainset(source);
-  const appendArrayToRejectExtraOutput = (source: MaybePromise<AsyncIterable<string> | Iterable<string> | string[]>) => rejectExtraDomainsetOutput.addFromDomainset(source);
-
   /** Whitelists */
   const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST);
   const filterRuleWhiteKeywords = new Set<string>();
 
-  // Parse from AdGuard Filters
+  const mergeAdGuardFilter = (
-  await span
+    domainsetOutput: DomainsetOutput,
+    {
+      filterRulesUrl,
+      whiteDomains, whiteDomainSuffixes,
+      blackDomains, blackDomainSuffixes,
+      blackIPs, blackWildcard,
+      whiteKeyword, blackKeyword
+    }: ProcessFilterRulesResult
+  ) => {
+    addArrayElementsToSet(filterRuleWhitelistDomainSets, whiteDomains);
+    addArrayElementsToSet(filterRuleWhitelistDomainSets, whiteDomainSuffixes, suffix => '.' + suffix);
+
+    addArrayElementsToSet(filterRuleWhiteKeywords, whiteKeyword);
+
+    domainsetOutput.bulkAddDomain(blackDomains);
+    domainsetOutput.bulkAddDomainSuffix(blackDomainSuffixes);
+
+    domainsetOutput.bulkAddDomainKeyword(blackKeyword);
+
+    domainsetOutput.appendDataSource(filterRulesUrl);
+
+    rejectNonIpRulesetOutput.bulkAddDomainWildcard(blackWildcard);
+    rejectNonIpRulesetOutput.appendDataSource(filterRulesUrl);
+
+    rejectIPOutput.bulkAddAnyCIDR(blackIPs, false);
+    rejectIPOutput.appendDataSource(filterRulesUrl);
+  };
+
+  const mergeAdGuardWhitelistFilter = ({ whiteDomains, whiteDomainSuffixes, blackDomains, blackDomainSuffixes, whiteKeyword, blackKeyword }: ProcessFilterRulesResult) => {
+    addArrayElementsToSet(filterRuleWhitelistDomainSets, whiteDomains);
+    addArrayElementsToSet(filterRuleWhitelistDomainSets, whiteDomainSuffixes, suffix => '.' + suffix);
+    addArrayElementsToSet(filterRuleWhitelistDomainSets, blackDomains);
+    addArrayElementsToSet(filterRuleWhitelistDomainSets, blackDomainSuffixes, suffix => '.' + suffix);
+    addArrayElementsToSet(filterRuleWhiteKeywords, whiteKeyword);
+    addArrayElementsToSet(filterRuleWhiteKeywords, blackKeyword);
+  };
+
+  const foundDebugDomain = await span
     .traceChild('download and process hosts / adblock filter rules')
-    .traceAsyncFn((childSpan) => {
+    .traceAsyncFn(async (childSpan) => {
-      const promises: Array<Promise<void>> = [];
-      // Parse from remote hosts & domain lists
-
-      arrayPushNonNullish(promises, hostsDownloads.map(task => task(childSpan).then(appendArrayToRejectOutput)));
-      arrayPushNonNullish(promises, hostsExtraDownloads.map(task => task(childSpan).then(appendArrayToRejectExtraOutput)));
-      arrayPushNonNullish(promises, domainListsDownloads.map(task => task(childSpan).then(appendArrayToRejectOutput)));
-      arrayPushNonNullish(promises, domainListsExtraDownloads.map(task => task(childSpan).then(appendArrayToRejectExtraOutput)));
-
       rejectPhisingDomainsetOutput.addFromDomainset(
-        span.traceWorkerChild('get phishing domains', rawSpan => phishingWorker.getPhishingDomains(rawSpan))
+        childSpan.traceWorkerChild('get phishing domains', rawSpan => rejectWorker.getPhishingDomains(rawSpan))
       );
 
-      arrayPushNonNullish(
+      const [sources] = await Promise.all([
-        promises,
+        childSpan.traceWorkerChild('get reject sources', rawSpan => rejectWorker.getRejectSources(rawSpan)),
-        adguardFiltersDownloads.map(
-          task => task(childSpan).then(({
-            filterRulesUrl,
-            whiteDomains, whiteDomainSuffixes,
-            blackDomains, blackDomainSuffixes,
-            blackIPs, blackWildcard,
-            whiteKeyword, blackKeyword
-          }) => {
-            addArrayElementsToSet(filterRuleWhitelistDomainSets, whiteDomains);
-            addArrayElementsToSet(filterRuleWhitelistDomainSets, whiteDomainSuffixes, suffix => '.' + suffix);
 
-            addArrayElementsToSet(filterRuleWhiteKeywords, whiteKeyword);
+        childSpan.traceChildAsync(
-
+          'get bogus nxdomain ips',
-            rejectDomainsetOutput.bulkAddDomain(blackDomains);
+          () => fetchAssets(...BOGUS_NXDOMAIN_DNSMASQ, true, false).then(arr => {
-            rejectDomainsetOutput.bulkAddDomainSuffix(blackDomainSuffixes);
+            for (let i = 0, len = arr.length; i < len; i++) {
-
+              const line = arr[i];
-            rejectDomainsetOutput.bulkAddDomainKeyword(blackKeyword);
+              if (line.startsWith('bogus-nxdomain=')) {
-
+                rejectIPOutput.addAnyCIDR(
-            rejectDomainsetOutput.appendDataSource(filterRulesUrl);
+                  line.slice(15).trim(),
-
+                  false // bogus nxdomain needs to be blocked even after resolved
-            rejectNonIpRulesetOutput.bulkAddDomainWildcard(blackWildcard);
+                );
-            rejectNonIpRulesetOutput.appendDataSource(filterRulesUrl);
+              }
-
-            rejectIPOutput.bulkAddAnyCIDR(blackIPs, false);
-            rejectIPOutput.appendDataSource(filterRulesUrl);
-          })
-        )
-      );
-
-      arrayPushNonNullish(
-        promises,
-        adguardFiltersExtraDownloads.map(
-          task => task(childSpan).then(({
-            filterRulesUrl,
-            whiteDomains, whiteDomainSuffixes,
-            blackDomains, blackDomainSuffixes,
-            blackIPs, blackWildcard, whiteKeyword, blackKeyword
-          }) => {
-            addArrayElementsToSet(filterRuleWhitelistDomainSets, whiteDomains);
-            addArrayElementsToSet(filterRuleWhitelistDomainSets, whiteDomainSuffixes, suffix => '.' + suffix);
-            addArrayElementsToSet(filterRuleWhiteKeywords, whiteKeyword);
-
-            rejectExtraDomainsetOutput.bulkAddDomain(blackDomains);
-            rejectExtraDomainsetOutput.bulkAddDomainSuffix(blackDomainSuffixes);
-
-            rejectExtraDomainsetOutput.bulkAddDomainKeyword(blackKeyword);
-
-            rejectExtraDomainsetOutput.appendDataSource(filterRulesUrl);
-
-            rejectIPOutput.bulkAddAnyCIDR(blackIPs, false);
-            rejectIPOutput.appendDataSource(filterRulesUrl);
-
-            rejectNonIpRulesetOutput.bulkAddDomainWildcard(blackWildcard);
-            rejectNonIpRulesetOutput.appendDataSource(filterRulesUrl);
-          })
-        )
-      );
-      arrayPushNonNullish(
-        promises,
-        adguardFiltersWhitelistsDownloads.map(
-          task => task(childSpan).then(({ whiteDomains, whiteDomainSuffixes, blackDomains, blackDomainSuffixes, whiteKeyword, blackKeyword }) => {
-            addArrayElementsToSet(filterRuleWhitelistDomainSets, whiteDomains);
-            addArrayElementsToSet(filterRuleWhitelistDomainSets, whiteDomainSuffixes, suffix => '.' + suffix);
-            addArrayElementsToSet(filterRuleWhitelistDomainSets, blackDomains);
-            addArrayElementsToSet(filterRuleWhitelistDomainSets, blackDomainSuffixes, suffix => '.' + suffix);
-            addArrayElementsToSet(filterRuleWhiteKeywords, whiteKeyword);
-            addArrayElementsToSet(filterRuleWhiteKeywords, blackKeyword);
-          })
-        )
-      );
-
-      promises.push(span.traceChildAsync(
-        'get bogus nxdomain ips',
-        () => fetchAssets(...BOGUS_NXDOMAIN_DNSMASQ, true, false).then(arr => {
-          for (let i = 0, len = arr.length; i < len; i++) {
-            const line = arr[i];
-            if (line.startsWith('bogus-nxdomain=')) {
-              rejectIPOutput.addAnyCIDR(
-                line.slice(15).trim(),
-                false // bogus nxdomain needs to be blocked even after resolved
-              );
             }
-          }
+            // return arr;
-          // return arr;
+          }),
-        }),
+          SpanCategory.Network
-        SpanCategory.Network
+        )
-      ));
+      ]);
 
-      return Promise.all(promises);
+      // Everything below is trie / set insertion on the main thread -- the part
+      // that can not move, since the tries live here.
+      childSpan.traceChildSync('merge reject sources', () => {
+        for (let i = 0, len = sources.hosts.length; i < len; i++) {
+          rejectDomainsetOutput.addFromDomainset(sources.hosts[i]);
+        }
+        for (let i = 0, len = sources.domainLists.length; i < len; i++) {
+          rejectDomainsetOutput.addFromDomainset(sources.domainLists[i]);
+        }
+        for (let i = 0, len = sources.hostsExtra.length; i < len; i++) {
+          rejectExtraDomainsetOutput.addFromDomainset(sources.hostsExtra[i]);
+        }
+        for (let i = 0, len = sources.domainListsExtra.length; i < len; i++) {
+          rejectExtraDomainsetOutput.addFromDomainset(sources.domainListsExtra[i]);
+        }
+
+        for (let i = 0, len = sources.adguardFilters.length; i < len; i++) {
+          mergeAdGuardFilter(rejectDomainsetOutput, sources.adguardFilters[i]);
+        }
+        for (let i = 0, len = sources.adguardFiltersExtra.length; i < len; i++) {
+          mergeAdGuardFilter(rejectExtraDomainsetOutput, sources.adguardFiltersExtra[i]);
+        }
+        for (let i = 0, len = sources.adguardFiltersWhitelist.length; i < len; i++) {
+          mergeAdGuardWhitelistFilter(sources.adguardFiltersWhitelist[i]);
+        }
+      }, SpanCategory.Compute);
+
+      return sources.foundDebugDomain;
     });
 
-  if (foundDebugDomain.value) {
+  if (foundDebugDomain) {
     // eslint-disable-next-line sukka/unicorn/no-process-exit -- cli App
     process.exit(1);
   }
@@ -309,5 +287,5 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as
     .addFromRuleset(readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/my_reject.conf')))
     .write();
 
-  await phishingWorker.end();
+  await rejectWorker.end();
 });

Build/lib/get-reject-sources.ts

@@ -0,0 +1,65 @@
+import { workerJob } from '../trace';
+import type { RawSpan, Span, WorkerJobResult } from '../trace';
+
+import {
+  HOSTS, HOSTS_EXTRA,
+  DOMAIN_LISTS, DOMAIN_LISTS_EXTRA,
+  ADGUARD_FILTERS, ADGUARD_FILTERS_EXTRA, ADGUARD_FILTERS_WHITELIST
+} from '../constants/reject-data-source';
+import { processHostsWithPreload } from './parse-filter/hosts';
+import { processDomainListsWithPreload } from './parse-filter/domainlists';
+import { processFilterRulesWithPreload } from './parse-filter/filters';
+import type { ProcessFilterRulesResult } from './parse-filter/filters';
+import { foundDebugDomain } from './parse-filter/shared';
+
+/**
+ * Everything build-reject-domainset pulls from remote hosts / domain lists /
+ * AdGuard filters, already parsed. Plain arrays only
+ */
+export interface RejectSources {
+  hosts: string[][],
+  hostsExtra: string[][],
+  domainLists: string[][],
+  domainListsExtra: string[][],
+  adguardFilters: ProcessFilterRulesResult[],
+  adguardFiltersExtra: ProcessFilterRulesResult[],
+  adguardFiltersWhitelist: ProcessFilterRulesResult[],
+  /** DEBUG_DOMAIN_TO_FIND was seen while parsing -- lives in this thread's module state, so it has to be reported back */
+  foundDebugDomain: boolean
+}
+
+export function getRejectSources(rawSpan?: RawSpan): Promise<WorkerJobResult<RejectSources>> {
+  return workerJob(rawSpan, async (span) => {
+    // Kick every download off before awaiting any of them.
+    const hosts = HOSTS.map(entry => processHostsWithPreload(...entry));
+    const hostsExtra = HOSTS_EXTRA.map(entry => processHostsWithPreload(...entry));
+    const domainLists = DOMAIN_LISTS.map(entry => processDomainListsWithPreload(...entry));
+    const domainListsExtra = DOMAIN_LISTS_EXTRA.map(entry => processDomainListsWithPreload(...entry));
+    const adguardFilters = ADGUARD_FILTERS.map(entry => processFilterRulesWithPreload(...entry));
+    const adguardFiltersExtra = ADGUARD_FILTERS_EXTRA.map(entry => processFilterRulesWithPreload(...entry));
+    const adguardFiltersWhitelist = ADGUARD_FILTERS_WHITELIST.map(entry => processFilterRulesWithPreload(...entry));
+
+    const run = <T>(tasks: Array<(span: Span) => Promise<T>>) => Promise.all(tasks.map(task => task(span)));
+
+    const [
+      hostsResults, hostsExtraResults,
+      domainListsResults, domainListsExtraResults,
+      adguardFiltersResults, adguardFiltersExtraResults, adguardFiltersWhitelistResults
+    ] = await Promise.all([
+      run(hosts), run(hostsExtra),
+      run(domainLists), run(domainListsExtra),
+      run(adguardFilters), run(adguardFiltersExtra), run(adguardFiltersWhitelist)
+    ]);
+
+    return {
+      hosts: hostsResults,
+      hostsExtra: hostsExtraResults,
+      domainLists: domainListsResults,
+      domainListsExtra: domainListsExtraResults,
+      adguardFilters: adguardFiltersResults,
+      adguardFiltersExtra: adguardFiltersExtraResults,
+      adguardFiltersWhitelist: adguardFiltersWhitelistResults,
+      foundDebugDomain: foundDebugDomain.value
+    };
+  });
+}

Build/lib/parse-filter/filters.ts

@@ -25,6 +25,21 @@ const enum ParseType {
 
 export { type ParseType };
 
+/** Plain data: safe to hand across a worker thread boundary */
+export type ProcessFilterRulesResult = Record<
+  'whiteDomains'
+  | 'whiteDomainSuffixes'
+  | 'blackDomains'
+  | 'blackDomainSuffixes'
+  | 'blackIPs'
+  | 'blackWildcard'
+  | 'whiteKeyword'
+  | 'blackKeyword',
+  string[]
+> & {
+  filterRulesUrl: string
+};
+
 export function processFilterRulesWithPreload(
   filterRulesUrl: string,
   fallbackUrls?: string[] | null,
@@ -35,21 +50,7 @@ export function processFilterRulesWithPreload(
     true, false, true
   );
 
-  return (span: Span) => span.traceChildAsync<
+  return (span: Span) => span.traceChildAsync<ProcessFilterRulesResult>(`process filter rules: ${filterRulesUrl}`, async (span) => {
-    Record<
-      'whiteDomains'
-      | 'whiteDomainSuffixes'
-      | 'blackDomains'
-      | 'blackDomainSuffixes'
-      | 'blackIPs'
-      | 'blackWildcard'
-      | 'whiteKeyword'
-      | 'blackKeyword',
-      string[]
-    > & {
-      filterRulesUrl: string
-    }
-  >(`process filter rules: ${filterRulesUrl}`, async (span) => {
     const filterRules = await span.traceChildPromise('download', downloadPromise, SpanCategory.Network);
 
     const whiteDomains = new Set<string>();

Build/lib/reject.worker.ts

@@ -0,0 +1,2 @@
+export { getPhishingDomains } from './get-phishing-domains';
+export { getRejectSources } from './get-reject-sources';

Build/lib/rules/base.ts

@@ -189,19 +189,32 @@ export class FileOutput {
     return this;
   }
 
+  private addDomainsetLine(line: string) {
+    const otherPoundSign = line.lastIndexOf('#');
+
+    if (otherPoundSign > 0) {
+      line = line.slice(0, otherPoundSign).trimEnd();
+    }
+
+    if (line[0] === '.') {
+      this.addDomainSuffix(line);
+    } else {
+      this.domainTrie.add(line);
+    }
+  }
+
   private async addFromDomainsetPromise(source: MaybePromise<AsyncIterable<string> | Iterable<string> | string[]>) {
-    for await (let line of await source) {
+    const resolved = await source;
-      const otherPoundSign = line.lastIndexOf('#');
+    // `for await` over a plain array still yields to the microtask queue once
-
+    // per element; the parsed remote lists are hundreds of thousands of lines
-      if (otherPoundSign > 0) {
+    if (Array.isArray(resolved)) {
-        line = line.slice(0, otherPoundSign).trimEnd();
+      for (let i = 0, len = resolved.length; i < len; i++) {
-      }
+        this.addDomainsetLine(resolved[i]);
-
-      if (line[0] === '.') {
-        this.addDomainSuffix(line);
-      } else {
-        this.domainTrie.add(line);
       }
+      return;
+    }
+    for await (const line of resolved) {
+      this.addDomainsetLine(line);
     }
   }