Perf: offload reject fetch & parse into a worker thread

Build/lib/get-reject-sources.ts

@@ -0,0 +1,65 @@
+import { workerJob } from '../trace';
+import type { RawSpan, Span, WorkerJobResult } from '../trace';
+
+import {
+  HOSTS, HOSTS_EXTRA,
+  DOMAIN_LISTS, DOMAIN_LISTS_EXTRA,
+  ADGUARD_FILTERS, ADGUARD_FILTERS_EXTRA, ADGUARD_FILTERS_WHITELIST
+} from '../constants/reject-data-source';
+import { processHostsWithPreload } from './parse-filter/hosts';
+import { processDomainListsWithPreload } from './parse-filter/domainlists';
+import { processFilterRulesWithPreload } from './parse-filter/filters';
+import type { ProcessFilterRulesResult } from './parse-filter/filters';
+import { foundDebugDomain } from './parse-filter/shared';
+
+/**
+ * Everything build-reject-domainset pulls from remote hosts / domain lists /
+ * AdGuard filters, already parsed. Plain arrays only
+ */
+export interface RejectSources {
+  hosts: string[][],
+  hostsExtra: string[][],
+  domainLists: string[][],
+  domainListsExtra: string[][],
+  adguardFilters: ProcessFilterRulesResult[],
+  adguardFiltersExtra: ProcessFilterRulesResult[],
+  adguardFiltersWhitelist: ProcessFilterRulesResult[],
+  /** DEBUG_DOMAIN_TO_FIND was seen while parsing -- lives in this thread's module state, so it has to be reported back */
+  foundDebugDomain: boolean
+}
+
+export function getRejectSources(rawSpan?: RawSpan): Promise<WorkerJobResult<RejectSources>> {
+  return workerJob(rawSpan, async (span) => {
+    // Kick every download off before awaiting any of them.
+    const hosts = HOSTS.map(entry => processHostsWithPreload(...entry));
+    const hostsExtra = HOSTS_EXTRA.map(entry => processHostsWithPreload(...entry));
+    const domainLists = DOMAIN_LISTS.map(entry => processDomainListsWithPreload(...entry));
+    const domainListsExtra = DOMAIN_LISTS_EXTRA.map(entry => processDomainListsWithPreload(...entry));
+    const adguardFilters = ADGUARD_FILTERS.map(entry => processFilterRulesWithPreload(...entry));
+    const adguardFiltersExtra = ADGUARD_FILTERS_EXTRA.map(entry => processFilterRulesWithPreload(...entry));
+    const adguardFiltersWhitelist = ADGUARD_FILTERS_WHITELIST.map(entry => processFilterRulesWithPreload(...entry));
+
+    const run = <T>(tasks: Array<(span: Span) => Promise<T>>) => Promise.all(tasks.map(task => task(span)));
+
+    const [
+      hostsResults, hostsExtraResults,
+      domainListsResults, domainListsExtraResults,
+      adguardFiltersResults, adguardFiltersExtraResults, adguardFiltersWhitelistResults
+    ] = await Promise.all([
+      run(hosts), run(hostsExtra),
+      run(domainLists), run(domainListsExtra),
+      run(adguardFilters), run(adguardFiltersExtra), run(adguardFiltersWhitelist)
+    ]);
+
+    return {
+      hosts: hostsResults,
+      hostsExtra: hostsExtraResults,
+      domainLists: domainListsResults,
+      domainListsExtra: domainListsExtraResults,
+      adguardFilters: adguardFiltersResults,
+      adguardFiltersExtra: adguardFiltersExtraResults,
+      adguardFiltersWhitelist: adguardFiltersWhitelistResults,
+      foundDebugDomain: foundDebugDomain.value
+    };
+  });
+}

Build/lib/parse-filter/filters.ts

@@ -25,6 +25,21 @@ const enum ParseType {
 
 export { type ParseType };
 
+/** Plain data: safe to hand across a worker thread boundary */
+export type ProcessFilterRulesResult = Record<
+  'whiteDomains'
+  | 'whiteDomainSuffixes'
+  | 'blackDomains'
+  | 'blackDomainSuffixes'
+  | 'blackIPs'
+  | 'blackWildcard'
+  | 'whiteKeyword'
+  | 'blackKeyword',
+  string[]
+> & {
+  filterRulesUrl: string
+};
+
 export function processFilterRulesWithPreload(
   filterRulesUrl: string,
   fallbackUrls?: string[] | null,
@@ -35,21 +50,7 @@ export function processFilterRulesWithPreload(
     true, false, true
   );
 
-  return (span: Span) => span.traceChildAsync<
-    Record<
-      'whiteDomains'
-      | 'whiteDomainSuffixes'
-      | 'blackDomains'
-      | 'blackDomainSuffixes'
-      | 'blackIPs'
-      | 'blackWildcard'
-      | 'whiteKeyword'
-      | 'blackKeyword',
-      string[]
-    > & {
-      filterRulesUrl: string
-    }
-  >(`process filter rules: ${filterRulesUrl}`, async (span) => {
+  return (span: Span) => span.traceChildAsync<ProcessFilterRulesResult>(`process filter rules: ${filterRulesUrl}`, async (span) => {
     const filterRules = await span.traceChildPromise('download', downloadPromise, SpanCategory.Network);
 
     const whiteDomains = new Set<string>();

Build/lib/reject.worker.ts

@@ -0,0 +1,2 @@
+export { getPhishingDomains } from './get-phishing-domains';
+export { getRejectSources } from './get-reject-sources';

Build/lib/rules/base.ts

@@ -189,19 +189,32 @@ export class FileOutput {
     return this;
   }
 
+  private addDomainsetLine(line: string) {
+    const otherPoundSign = line.lastIndexOf('#');
+
+    if (otherPoundSign > 0) {
+      line = line.slice(0, otherPoundSign).trimEnd();
+    }
+
+    if (line[0] === '.') {
+      this.addDomainSuffix(line);
+    } else {
+      this.domainTrie.add(line);
+    }
+  }
+
   private async addFromDomainsetPromise(source: MaybePromise<AsyncIterable<string> | Iterable<string> | string[]>) {
-    for await (let line of await source) {
-      const otherPoundSign = line.lastIndexOf('#');
-
-      if (otherPoundSign > 0) {
-        line = line.slice(0, otherPoundSign).trimEnd();
-      }
-
-      if (line[0] === '.') {
-        this.addDomainSuffix(line);
-      } else {
-        this.domainTrie.add(line);
+    const resolved = await source;
+    // `for await` over a plain array still yields to the microtask queue once
+    // per element; the parsed remote lists are hundreds of thousands of lines
+    if (Array.isArray(resolved)) {
+      for (let i = 0, len = resolved.length; i < len; i++) {
+        this.addDomainsetLine(resolved[i]);
       }
+      return;
+    }
+    for await (const line of resolved) {
+      this.addDomainsetLine(line);
     }
   }