Chore: dispatch write to workers

Build/build-reject-domainset.ts

@@ -19,6 +19,7 @@ import { OUTPUT_INTERNAL_DIR, SOURCE_DIR } from './constants/dir';
 import { DomainsetOutput, AdGuardHomeOutput } from './lib/rules/domainset';
 import { foundDebugDomain } from './lib/parse-filter/shared';
 import { createWorker } from './lib/worker';
+import { endOutputWorkerFarm } from './lib/rules/output-worker-farm';
 import type { MaybePromise } from './lib/misc';
 import { RulesetOutput } from './lib/rules/ruleset';
 import { fetchAssets } from './lib/fetch-assets';
@@ -307,5 +308,8 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as
     .addFromRuleset(readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/my_reject.conf')))
     .write();
 
-  await phishingWorker.end();
+  await Promise.all([
+    phishingWorker.end(),
+    endOutputWorkerFarm()
+  ]);
 });

Build/index.ts

@@ -28,6 +28,7 @@ import path from 'node:path';
 import { ROOT_DIR } from './constants/dir';
 import { isCI } from 'ci-info';
 import { printExternalDownloadStats } from './lib/download-stats';
+import { endOutputWorkerFarm } from './lib/rules/output-worker-farm';
 
 process.on('uncaughtException', (error) => {
   console.error('Uncaught exception:', error);
@@ -130,7 +131,9 @@ const buildFinishedLock = path.join(ROOT_DIR, '.BUILD_FINISHED');
       microsoftCdnWorker.end(),
       cdnDownloadWorker.end(),
       telegramCidrWorker.end(),
-      mockAssetsWorker.end()
+      mockAssetsWorker.end(),
+      // defensive: no-op unless some FileOutput crossed the offload threshold
+      endOutputWorkerFarm()
     ]);
 
     // Finish the build to avoid leaking timer/fetch ref

Build/lib/content-hash.ts

@@ -1,7 +1,6 @@
 import { fastStringArrayJoin } from 'foxts/fast-string-array-join';
 import { Buffer } from 'node:buffer';
 import { createHash } from 'node:crypto';
-import fs from 'node:fs';
 import fsp from 'node:fs/promises';
 
 /**
@@ -65,21 +64,6 @@ export async function extractContentHashFromFile(filePath: string): Promise<stri
   return extractContentHash(await readFileHead(filePath));
 }
 
-/** Synchronous variant of {@link extractContentHashFromFile}, for worker threads where blocking is fine */
-export function extractContentHashFromFileSync(filePath: string): string | null {
-  let fd: number | null = null;
-  try {
-    fd = fs.openSync(filePath, 'r');
-    const buf = Buffer.allocUnsafe(FILE_HEAD_CHUNK_SIZE);
-    const bytesRead = fs.readSync(fd, buf, 0, FILE_HEAD_CHUNK_SIZE, 0);
-    return extractContentHash(buf.toString('utf8', 0, bytesRead));
-  } finally {
-    if (fd !== null) {
-      fs.closeSync(fd);
-    }
-  }
-}
-
 function extractContentHash(fileHead: string): string | null {
   const markerIndex = fileHead.indexOf(CONTENT_HASH_MARKER_START);
   if (markerIndex === -1) {

Build/lib/create-file.ts

@@ -1,5 +1,6 @@
 import { fastStringArrayJoin } from 'foxts/fast-string-array-join';
 import fs from 'node:fs';
+import path from 'node:path';
 import picocolors from 'picocolors';
 import type { Span } from '../trace';
 import { readFileByLine } from './fetch-text-by-line';
@@ -10,16 +11,12 @@ import { extractContentHashFromFile } from './content-hash';
 const fileEqual = createCompareSource(fileEqualWithCommentComparator);
 
 /**
- * To keep metadata comment `last updated` not change if real content is the same,
- * we only write when the actual content differs, and the new `last updated` will
- * be written along with new content.
- *
- * When `contentHash` is provided (and the previous output already embeds a
- * content hash marker), the comparison only reads the first chunk of the
- * previous file. Otherwise it falls back to a full comment-insensitive
- * line-by-line comparison.
+ * The comparison half shared by {@link compareAndWriteFile} and
+ * {@link compareAndWriteFileInWorker} -- there is exactly one implementation of
+ * "is the previous output already up to date", regardless of how the new content
+ * ends up being written.
  */
-export async function compareAndWriteFile(span: Span, linesA: string[], filePath: string, contentHash: string | null = null) {
+async function isPreviousOutputEqual(span: Span, linesA: string[], filePath: string, contentHash: string | null) {
   // readFileByLine will not include last empty line. So we always pop the linesA for comparison purpose
   if (linesA.length > 0 && linesA[linesA.length - 1] === '') {
     linesA.pop();
@@ -44,15 +41,53 @@ export async function compareAndWriteFile(span: Span, linesA: string[], filePath
 
   if (isEqual) {
     console.log(picocolors.gray(picocolors.dim(`same content, bail out writing: ${filePath}`)));
+  }
+
+  return isEqual;
+}
+
+/**
+ * To keep metadata comment `last updated` not change if real content is the same,
+ * we only write when the actual content differs, and the new `last updated` will
+ * be written along with new content.
+ *
+ * When `contentHash` is provided (and the previous output already embeds a
+ * content hash marker), the comparison only reads the first chunk of the
+ * previous file. Otherwise it falls back to a full comment-insensitive
+ * line-by-line comparison.
+ */
+export async function compareAndWriteFile(span: Span, linesA: string[], filePath: string, contentHash: string | null = null) {
+  if (await isPreviousOutputEqual(span, linesA, filePath, contentHash)) {
     return;
   }
 
   return writeFileLines(span, linesA, filePath);
 }
 
+/**
+ * Same comparison as {@link compareAndWriteFile}, but the write itself is
+ * synchronous. For worker threads only: blocking a dedicated worker is free,
+ * whereas an async write hands the completion back through libuv to an event
+ * loop we would rather not depend on being idle.
+ */
+export async function compareAndWriteFileInWorker(span: Span, linesA: string[], filePath: string, contentHash: string | null = null) {
+  if (await isPreviousOutputEqual(span, linesA, filePath, contentHash)) {
+    return;
+  }
+
+  writeFileLinesSync(span, linesA, filePath);
+}
+
 export function writeFileLines(span: Span, linesA: string[], filePath: string): Promise<void> {
   return span.traceChildAsync<void>(
     `writing ${filePath}`,
     () => writeFile(filePath, fastStringArrayJoin(linesA, '\n') + '\n')
   );
 }
+
+export function writeFileLinesSync(span: Span, linesA: string[], filePath: string): void {
+  span.traceChildSync(`writing ${filePath}`, () => {
+    fs.mkdirSync(path.dirname(filePath), { recursive: true });
+    fs.writeFileSync(filePath, fastStringArrayJoin(linesA, '\n') + '\n');
+  });
+}

Build/lib/rules/base.ts

@@ -1,16 +1,24 @@
 import type { Span } from '../../trace';
 import { HostnameSmolTrie } from 'hntrie/smol';
-import { domainToASCII } from 'node:url';
 import { not, nullthrow } from 'foxts/guard';
 import { fastIpVersion } from 'foxts/fast-ip-version';
 import { addArrayElementsToSet } from 'foxts/add-array-elements-to-set';
 import type { MaybePromise } from '../misc';
 import type { BaseWriteStrategy } from '../writing-strategy/base';
-import { merge as mergeCidr } from 'fast-cidr-tools';
-import { createRetrieKeywordFilter as createKeywordFilter } from 'foxts/retrie';
-import path from 'node:path';
 import { SurgeMitmSgmodule } from '../writing-strategy/surge';
 import { appendArrayInPlace } from 'foxts/append-array-in-place';
+import { isMainThread } from 'node:worker_threads';
+import { resolveStrategyOutputPath, serializeStrategy, writeDataToStrategies } from './strategy-write-data';
+import type { OutputWorkerPayload, StrategyWriteData } from './strategy-write-data';
+import { getOutputWorkerFarm } from './output-worker-farm';
+
+/**
+ * Below this many dumped domain entries, formatting + hashing + writing inline is
+ * cheaper than a worker round trip (and avoids tiny outputs queueing behind big
+ * jobs in the farm). Today only the reject domainsets / adguardhome outputs cross
+ * this threshold.
+ */
+const OUTPUT_WORKER_THRESHOLD = 20000;
 
 /**
  * Holds the universal rule data (domain, ip, url-regex, etc. etc.)
@@ -399,7 +407,42 @@ export class FileOutput {
   // }
   private strategiesWritten = false;
 
-  private writeToStrategies() {
+  /** Collect the trie content, '.'-prefix encoded, WITHOUT punycoding (that happens in the fanout) */
+  private dumpDomains(): string[] {
+    const domains: string[] = [];
+    this.domainTrie.dump((domain, includeSubdomain) => {
+      domains.push(includeSubdomain ? '.' + domain : domain);
+    });
+    return domains;
+  }
+
+  private buildStrategyWriteData(domains: string[]): StrategyWriteData {
+    return {
+      domains,
+      domainKeywords: Array.from(this.domainKeywords),
+      whitelistKeywords: Array.from(this.whitelistKeywords),
+      wildcards: Array.from(this.wildcardSet),
+      userAgents: Array.from(this.userAgent),
+      processNames: Array.from(this.processName),
+      processPaths: Array.from(this.processPath),
+      urlRegexes: Array.from(this.urlRegex),
+      ipcidr: Array.from(this.ipcidr),
+      ipcidrNoResolve: Array.from(this.ipcidrNoResolve),
+      ipcidr6: Array.from(this.ipcidr6),
+      ipcidr6NoResolve: Array.from(this.ipcidr6NoResolve),
+      ipasn: Array.from(this.ipasn),
+      ipasnNoResolve: Array.from(this.ipasnNoResolve),
+      geoip: Array.from(this.geoip),
+      geoipNoResolve: Array.from(this.groipNoResolve),
+      sourceIpOrCidr: Array.from(this.sourceIpOrCidr),
+      sourcePort: Array.from(this.sourcePort),
+      destPort: Array.from(this.destPort),
+      protocol: Array.from(this.protocol),
+      otherRules: this.otherRules
+    };
+  }
+
+  private guardBeforeWritingToStrategies() {
     if (this.pendingPromise) {
       throw new Error('You should call done() before calling writeToStrategies()');
     }
@@ -412,158 +455,20 @@ export class FileOutput {
     if (this.strategies.filter(not(false)).length === 0) {
       throw new Error('No strategies to write ' + this.id);
     }
-
-    // We use both DOMAIN-KEYWORD and whitelisted keyword to whitelist DOMAIN and DOMAIN-SUFFIX
-    const kwfilter = createKeywordFilter(
-      Array.from(this.domainKeywords)
-        .concat(Array.from(this.whitelistKeywords))
-    );
-
-    const strategiesLen = this.strategies.length;
-
-    const domainEntries: Array<[domain: string, subdomain: boolean]> = [];
-    this.domainTrie.dump((domain, includeSubdomain) => {
-      const d = domainToASCII(domain);
-      if (d) domainEntries.push([d, includeSubdomain]);
-    });
-    domainEntries.sort((a, b) => (a[0].length - b[0].length) || (a[0] < b[0] ? -1 : (a[0] > b[0] ? 1 : 0)));
-    for (let j = 0, entriesLen = domainEntries.length; j < entriesLen; j++) {
-      const [domain, includeAllSubdomain] = domainEntries[j];
-      if (kwfilter(domain)) {
-        continue;
   }
 
-      for (let i = 0; i < strategiesLen; i++) {
-        const strategy = this.strategies[i];
-        if (includeAllSubdomain) {
-          strategy.writeDomainSuffix(domain);
-        } else {
-          strategy.writeDomain(domain);
-        }
-      }
-    }
-
-    // Now, we whitelisted out DOMAIN-KEYWORD
-    const whiteKwfilter = createKeywordFilter(Array.from(this.whitelistKeywords));
-    const whitelistedKeywords = Array.from(this.domainKeywords).filter(kw => !whiteKwfilter(kw));
-
-    for (let i = 0; i < strategiesLen; i++) {
-      const strategy = this.strategies[i];
-      if (whitelistedKeywords.length) {
-        strategy.writeDomainKeywords(this.domainKeywords);
-      }
-
-      if (this.protocol.size) {
-        strategy.writeProtocols(this.protocol);
-      }
-    }
-
-    if (this.wildcardSet.size) {
-      this.wildcardSet.forEach((wildcard) => {
-        // Overlapped w/ DOMAIN-kEYWORD
-        if (kwfilter(wildcard)) {
-          return;
-        }
-
-        for (let i = 0; i < strategiesLen; i++) {
-          const strategy = this.strategies[i];
-          strategy.writeDomainWildcard(wildcard);
-        }
-      });
-    }
-
-    const sourceIpOrCidr = Array.from(this.sourceIpOrCidr);
-
-    for (let i = 0; i < strategiesLen; i++) {
-      const strategy = this.strategies[i];
-
-      if (this.userAgent.size) {
-        strategy.writeUserAgents(this.userAgent);
-      }
-      if (this.processName.size) {
-        strategy.writeProcessNames(this.processName);
-      }
-      if (this.processPath.size) {
-        strategy.writeProcessPaths(this.processPath);
-      }
-
-      if (this.sourceIpOrCidr.size) {
-        strategy.writeSourceIpCidrs(sourceIpOrCidr);
-      }
-
-      if (this.sourcePort.size) {
-        strategy.writeSourcePorts(this.sourcePort);
-      }
-      if (this.destPort.size) {
-        strategy.writeDestinationPorts(this.destPort);
-      }
-      if (this.otherRules.length) {
-        strategy.writeOtherRules(this.otherRules);
-      }
-      if (this.urlRegex.size) {
-        strategy.writeUrlRegexes(this.urlRegex);
-      }
-    }
-
-    let ipcidr: string[] | null = null;
-    let ipcidrNoResolve: string[] | null = null;
-    let ipcidr6: string[] | null = null;
-    let ipcidr6NoResolve: string[] | null = null;
-
-    if (this.ipcidr.size) {
-      ipcidr = mergeCidr(Array.from(this.ipcidr), true);
-    }
-    if (this.ipcidrNoResolve.size) {
-      ipcidrNoResolve = mergeCidr(Array.from(this.ipcidrNoResolve), true);
-    }
-    if (this.ipcidr6.size) {
-      ipcidr6 = Array.from(this.ipcidr6);
-    }
-    if (this.ipcidr6NoResolve.size) {
-      ipcidr6NoResolve = Array.from(this.ipcidr6NoResolve);
-    }
-
-    for (let i = 0; i < strategiesLen; i++) {
-      const strategy = this.strategies[i];
-      // no-resolve
-      if (ipcidrNoResolve) {
-        strategy.writeIpCidrs(ipcidrNoResolve, true);
-      }
-      if (ipcidr6NoResolve) {
-        strategy.writeIpCidr6s(ipcidr6NoResolve, true);
-      }
-      if (this.ipasnNoResolve.size) {
-        strategy.writeIpAsns(this.ipasnNoResolve, true);
-      }
-      if (this.groipNoResolve.size) {
-        strategy.writeGeoip(this.groipNoResolve, true);
-      }
-
-      // triggers DNS resolution
-      if (ipcidr?.length) {
-        strategy.writeIpCidrs(ipcidr, false);
-      }
-      if (ipcidr6?.length) {
-        strategy.writeIpCidr6s(ipcidr6, false);
-      }
-      if (this.ipasn.size) {
-        strategy.writeIpAsns(this.ipasn, false);
-      }
-      if (this.geoip.size) {
-        strategy.writeGeoip(this.geoip, false);
-      }
-    }
+  private writeToStrategies(domains: string[]) {
+    this.guardBeforeWritingToStrategies();
+    writeDataToStrategies(this.buildStrategyWriteData(domains), this.strategies);
   }
 
   write(): Promise<unknown> {
     return this.span.traceChildAsync('write all', async (childSpan) => {
       await childSpan.traceChildAsync('done', () => this.done());
 
-      childSpan.traceChildSync('write to strategies', () => this.writeToStrategies());
-
-      return childSpan.traceChildAsync('output to disk', (childSpan) => {
-        const promises: Array<Promise<void>> = [];
+      const domains = childSpan.traceChildSync('dump domain trie', () => this.dumpDomains());
 
+      const title = nullthrow(this.title, 'Missing title');
       const descriptions = nullthrow(this.description, 'Missing description');
 
       if (this.dataSource.size) {
@@ -574,24 +479,49 @@ export class FileOutput {
         appendArrayInPlace(descriptions, Array.from(this.dataSource).sort().map((source) => `  - ${source}`));
       }
 
+      // Big outputs offload everything from punycode to the write onto a worker
+      // thread: the payload crosses in a few ms (flat string arrays structured-clone
+      // cheaply), the main-thread event loop stays free, and the worker writes
+      // synchronously so no completion ever waits on a busy main thread.
+      //
+      // Only worth doing from the main thread -- tasks that already run entirely on
+      // a worker (build-microsoft-cdn, build-telegram-cidr, build-cdn-download-conf)
+      // are not contending with anything, and must not spawn a nested worker farm.
+      if (isMainThread && domains.length >= OUTPUT_WORKER_THRESHOLD) {
+        this.guardBeforeWritingToStrategies();
+
+        const payload: OutputWorkerPayload = {
+          id: this.id,
+          title,
+          description: descriptions,
+          dateMs: this.date.getTime(),
+          strategies: this.strategies.map(serializeStrategy),
+          data: this.buildStrategyWriteData(domains)
+        };
+
+        return childSpan.traceWorkerChild(
+          'write via output worker',
+          rawSpan => getOutputWorkerFarm().writeOutput(rawSpan, payload)
+        );
+      }
+
+      childSpan.traceChildSync('write to strategies', () => this.writeToStrategies(domains));
+
+      return childSpan.traceChildAsync('output to disk', (childSpan) => {
+        const promises: Array<Promise<void>> = [];
+
         for (let i = 0, len = this.strategies.length; i < len; i++) {
           const strategy = this.strategies[i];
-
-          const basename = (strategy.overwriteFilename || this.id) + '.' + strategy.fileExtension;
+          const filePath = resolveStrategyOutputPath(strategy, this.id);
 
           promises.push(
-            childSpan.traceChildAsync('write ' + strategy.name, (childSpan) => Promise.resolve(strategy.output(
-              childSpan,
-              nullthrow(this.title, 'Missing title'),
-              descriptions,
-              this.date,
-              path.join(
-                strategy.outputDir,
-                strategy.type
-                  ? path.join(strategy.type, basename)
-                  : basename
-              )
-            )))
+            childSpan.traceChildAsync('write ' + strategy.name, (childSpan) => Promise.resolve(
+              // Already off the main thread: block on the write instead of handing
+              // the completion back through libuv.
+              isMainThread
+                ? strategy.output(childSpan, title, descriptions, this.date, filePath)
+                : strategy.outputInWorker(childSpan, title, descriptions, this.date, filePath)
+            ))
           );
         }
 
@@ -602,7 +532,7 @@ export class FileOutput {
 
   async compile(): Promise<Array<string[] | null>> {
     await this.done();
-    this.writeToStrategies();
+    this.writeToStrategies(this.dumpDomains());
 
     return this.strategies.reduce<Array<string[] | null>>((acc, strategy) => {
       acc.push(strategy.content);

Build/lib/rules/output-worker-farm.ts

@@ -0,0 +1,30 @@
+import type { JestWorkerFarm } from 'jest-worker';
+import { createWorker } from '../worker';
+
+type OutputWorkerModule = typeof import('./output.worker');
+type OutputWorkerFarm = JestWorkerFarm<Pick<OutputWorkerModule, 'writeOutput'>>;
+
+let farm: OutputWorkerFarm | null = null;
+
+/**
+ * Lazily boot the output worker farm. Only FileOutput#write dispatches here, and
+ * only when an output crosses the offload threshold -- today that is exclusively
+ * the reject domainsets / adguardhome outputs of build-reject-domainset.
+ *
+ * IMPORTANT: whoever triggers the lazy boot is responsible for a matching
+ * endOutputWorkerFarm() (idempotent, safe to call unconditionally), otherwise the
+ * worker threads keep a standalone task run alive.
+ */
+export function getOutputWorkerFarm(): OutputWorkerFarm {
+  // 3 workers: reject, reject_extra and reject_phishing format & write in parallel
+  farm ??= createWorker<OutputWorkerModule>(require.resolve('./output.worker'), 3)(['writeOutput']);
+  return farm;
+}
+
+export async function endOutputWorkerFarm(): Promise<void> {
+  if (farm) {
+    const f = farm;
+    farm = null;
+    await f.end();
+  }
+}

Build/lib/rules/output.worker.ts

@@ -0,0 +1,37 @@
+import { workerJob } from '../../trace';
+import type { RawSpan, WorkerJobResult } from '../../trace';
+import { resolveStrategyOutputPath, reviveStrategy, writeDataToStrategies } from './strategy-write-data';
+import type { OutputWorkerPayload } from './strategy-write-data';
+
+/**
+ * The off-main-thread half of FileOutput#write. Everything from "raw collections"
+ * to "bytes on disk" happens here: punycode, keyword filtering, sorting, the
+ * per-strategy format fanout (incl. the sing-box JSON stringify), banner and
+ * content hash, compare and write. All fs calls are synchronous on purpose --
+ * blocking this worker thread is free, and it avoids bouncing libuv completions
+ * off a busy main-thread event loop.
+ */
+export function writeOutput(rawSpan: RawSpan | undefined, payload: OutputWorkerPayload): Promise<WorkerJobResult<void>> {
+  return workerJob(rawSpan, (span) => {
+    const strategies = payload.strategies.map(reviveStrategy);
+
+    span.traceChildSync('write to strategies', () => writeDataToStrategies(payload.data, strategies));
+
+    const date = new Date(payload.dateMs);
+
+    return span.traceChildAsync('output to disk', async (childSpan) => {
+      // Sequential on purpose: the writes are synchronous, so there is nothing to
+      // overlap, and this keeps the emitted trace in strategy order.
+      for (let i = 0, len = strategies.length; i < len; i++) {
+        const strategy = strategies[i];
+        const filePath = resolveStrategyOutputPath(strategy, payload.id);
+
+        // eslint-disable-next-line no-await-in-loop -- see above
+        await childSpan.traceChildAsync(
+          'write ' + strategy.name,
+          (strategySpan) => strategy.outputInWorker(strategySpan, payload.title, payload.description, date, filePath)
+        );
+      }
+    });
+  });
+}

Build/lib/rules/strategy-write-data.ts

@@ -0,0 +1,265 @@
+import path from 'node:path';
+import { domainToASCII } from 'node:url';
+import { merge as mergeCidr } from 'fast-cidr-tools';
+import { createRetrieKeywordFilter as createKeywordFilter } from 'foxts/retrie';
+
+import type { BaseWriteStrategy } from '../writing-strategy/base';
+import { AdGuardHome } from '../writing-strategy/adguardhome';
+import { ClashClassicRuleSet, ClashDomainSet, ClashIPSet } from '../writing-strategy/clash';
+import { LegacyClashPremiumClassicRuleSet } from '../writing-strategy/legacy-clash-premium';
+import { SingboxSource } from '../writing-strategy/singbox';
+import { SurfboardRuleSet } from '../writing-strategy/surfboard';
+import { SurgeDomainSet, SurgeMitmSgmodule, SurgeRuleSet } from '../writing-strategy/surge';
+
+/**
+ * A structured-clone friendly snapshot of everything FileOutput feeds into its
+ * strategies. Only `domains` is big (the '.'-prefixed trie dump, still unicode);
+ * everything else is at most a few thousand short strings. Plain arrays are used
+ * (not Set) so the contract survives any worker serialization mode.
+ */
+export interface StrategyWriteData {
+  /** Raw trie dump, '.'-prefix means include-all-subdomain, NOT yet punycoded */
+  domains: string[],
+  domainKeywords: string[],
+  whitelistKeywords: string[],
+  wildcards: string[],
+  userAgents: string[],
+  processNames: string[],
+  processPaths: string[],
+  urlRegexes: string[],
+  ipcidr: string[],
+  ipcidrNoResolve: string[],
+  ipcidr6: string[],
+  ipcidr6NoResolve: string[],
+  ipasn: string[],
+  ipasnNoResolve: string[],
+  geoip: string[],
+  geoipNoResolve: string[],
+  sourceIpOrCidr: string[],
+  sourcePort: string[],
+  destPort: string[],
+  protocol: string[],
+  otherRules: string[]
+}
+
+/**
+ * Write a StrategyWriteData snapshot into strategies. This is the extracted body
+ * of FileOutput#writeToStrategies and MUST keep the exact same write order, since
+ * the order of writeXxx calls determines the line order of the final output.
+ */
+export function writeDataToStrategies(data: StrategyWriteData, strategies: BaseWriteStrategy[]): void {
+  // We use both DOMAIN-KEYWORD and whitelisted keyword to whitelist DOMAIN and DOMAIN-SUFFIX
+  const kwfilter = createKeywordFilter(data.domainKeywords.concat(data.whitelistKeywords));
+
+  const strategiesLen = strategies.length;
+
+  const domainEntries: Array<[domain: string, subdomain: boolean]> = [];
+  for (let j = 0, len = data.domains.length; j < len; j++) {
+    const line = data.domains[j];
+    const includeSubdomain = line.codePointAt(0) === 46; /* '.' */
+    const d = domainToASCII(includeSubdomain ? line.slice(1) : line);
+    if (d) domainEntries.push([d, includeSubdomain]);
+  }
+  domainEntries.sort((a, b) => (a[0].length - b[0].length) || (a[0] < b[0] ? -1 : (a[0] > b[0] ? 1 : 0)));
+  for (let j = 0, entriesLen = domainEntries.length; j < entriesLen; j++) {
+    const [domain, includeAllSubdomain] = domainEntries[j];
+    if (kwfilter(domain)) {
+      continue;
+    }
+
+    for (let i = 0; i < strategiesLen; i++) {
+      const strategy = strategies[i];
+      if (includeAllSubdomain) {
+        strategy.writeDomainSuffix(domain);
+      } else {
+        strategy.writeDomain(domain);
+      }
+    }
+  }
+
+  // Now, we whitelisted out DOMAIN-KEYWORD
+  const whiteKwfilter = createKeywordFilter(data.whitelistKeywords);
+  const whitelistedKeywords = data.domainKeywords.filter(kw => !whiteKwfilter(kw));
+
+  const domainKeywords = new Set(data.domainKeywords);
+  const protocol = new Set(data.protocol);
+
+  for (let i = 0; i < strategiesLen; i++) {
+    const strategy = strategies[i];
+    if (whitelistedKeywords.length) {
+      strategy.writeDomainKeywords(domainKeywords);
+    }
+
+    if (protocol.size) {
+      strategy.writeProtocols(protocol);
+    }
+  }
+
+  if (data.wildcards.length) {
+    data.wildcards.forEach((wildcard) => {
+      // Overlapped w/ DOMAIN-kEYWORD
+      if (kwfilter(wildcard)) {
+        return;
+      }
+
+      for (let i = 0; i < strategiesLen; i++) {
+        const strategy = strategies[i];
+        strategy.writeDomainWildcard(wildcard);
+      }
+    });
+  }
+
+  const userAgent = new Set(data.userAgents);
+  const processName = new Set(data.processNames);
+  const processPath = new Set(data.processPaths);
+  const sourcePort = new Set(data.sourcePort);
+  const destPort = new Set(data.destPort);
+  const urlRegex = new Set(data.urlRegexes);
+
+  for (let i = 0; i < strategiesLen; i++) {
+    const strategy = strategies[i];
+
+    if (userAgent.size) {
+      strategy.writeUserAgents(userAgent);
+    }
+    if (processName.size) {
+      strategy.writeProcessNames(processName);
+    }
+    if (processPath.size) {
+      strategy.writeProcessPaths(processPath);
+    }
+
+    if (data.sourceIpOrCidr.length) {
+      strategy.writeSourceIpCidrs(data.sourceIpOrCidr);
+    }
+
+    if (sourcePort.size) {
+      strategy.writeSourcePorts(sourcePort);
+    }
+    if (destPort.size) {
+      strategy.writeDestinationPorts(destPort);
+    }
+    if (data.otherRules.length) {
+      strategy.writeOtherRules(data.otherRules);
+    }
+    if (urlRegex.size) {
+      strategy.writeUrlRegexes(urlRegex);
+    }
+  }
+
+  let ipcidr: string[] | null = null;
+  let ipcidrNoResolve: string[] | null = null;
+  let ipcidr6: string[] | null = null;
+  let ipcidr6NoResolve: string[] | null = null;
+
+  if (data.ipcidr.length) {
+    ipcidr = mergeCidr(data.ipcidr, true);
+  }
+  if (data.ipcidrNoResolve.length) {
+    ipcidrNoResolve = mergeCidr(data.ipcidrNoResolve, true);
+  }
+  if (data.ipcidr6.length) {
+    ipcidr6 = data.ipcidr6;
+  }
+  if (data.ipcidr6NoResolve.length) {
+    ipcidr6NoResolve = data.ipcidr6NoResolve;
+  }
+
+  const ipasn = new Set(data.ipasn);
+  const ipasnNoResolve = new Set(data.ipasnNoResolve);
+  const geoip = new Set(data.geoip);
+  const geoipNoResolve = new Set(data.geoipNoResolve);
+
+  for (let i = 0; i < strategiesLen; i++) {
+    const strategy = strategies[i];
+    // no-resolve
+    if (ipcidrNoResolve) {
+      strategy.writeIpCidrs(ipcidrNoResolve, true);
+    }
+    if (ipcidr6NoResolve) {
+      strategy.writeIpCidr6s(ipcidr6NoResolve, true);
+    }
+    if (ipasnNoResolve.size) {
+      strategy.writeIpAsns(ipasnNoResolve, true);
+    }
+    if (geoipNoResolve.size) {
+      strategy.writeGeoip(geoipNoResolve, true);
+    }
+
+    // triggers DNS resolution
+    if (ipcidr?.length) {
+      strategy.writeIpCidrs(ipcidr, false);
+    }
+    if (ipcidr6?.length) {
+      strategy.writeIpCidr6s(ipcidr6, false);
+    }
+    if (ipasn.size) {
+      strategy.writeIpAsns(ipasn, false);
+    }
+    if (geoip.size) {
+      strategy.writeGeoip(geoip, false);
+    }
+  }
+}
+
+/** Where a strategy's output for a given output id lands on disk */
+export function resolveStrategyOutputPath(strategy: BaseWriteStrategy, id: string): string {
+  const basename = (strategy.overwriteFilename || id) + '.' + strategy.fileExtension;
+  return path.join(
+    strategy.outputDir,
+    strategy.type
+      ? path.join(strategy.type, basename)
+      : basename
+  );
+}
+
+/** Serializable stand-in for a strategy class instance crossing the worker boundary */
+export interface StrategyDescriptor {
+  name: string,
+  type: string,
+  outputDir: string,
+  overwriteFilename: string | null
+}
+
+export function serializeStrategy(strategy: BaseWriteStrategy): StrategyDescriptor {
+  return {
+    name: strategy.name,
+    type: strategy.type,
+    outputDir: strategy.outputDir,
+    overwriteFilename: strategy.overwriteFilename
+  };
+}
+
+const strategyRegistry: Record<string, (d: StrategyDescriptor) => BaseWriteStrategy> = {
+  'surge domainset': (d) => new SurgeDomainSet(d.outputDir),
+  'surge ruleset': (d) => new SurgeRuleSet(d.type, d.outputDir),
+  'surge sgmodule': (d) => new SurgeMitmSgmodule(d.overwriteFilename ?? '', d.outputDir),
+  'clash domainset': (d) => new ClashDomainSet(d.outputDir),
+  'clash ipcidr': (d) => new ClashIPSet(d.outputDir),
+  'clash classic ruleset': (d) => new ClashClassicRuleSet(d.type, d.outputDir),
+  'legacy clash premium classic ruleset': (d) => new LegacyClashPremiumClassicRuleSet(d.type as 'ip' | 'non_ip', d.outputDir),
+  'surfboard for android ruleset': (d) => new SurfboardRuleSet(d.type as 'ip' | 'non_ip', d.outputDir),
+  singbox: (d) => new SingboxSource(d.type as 'domainset' | 'non_ip' | 'ip', d.outputDir),
+  adguardhome: (d) => new AdGuardHome(d.outputDir)
+};
+
+export function reviveStrategy(descriptor: StrategyDescriptor): BaseWriteStrategy {
+  if (!(descriptor.name in strategyRegistry)) {
+    throw new TypeError(`Unknown strategy "${descriptor.name}", add it to strategyRegistry before offloading it to the output worker`);
+  }
+  const strategy = strategyRegistry[descriptor.name](descriptor);
+  if (descriptor.overwriteFilename) {
+    strategy.withFilename(descriptor.overwriteFilename);
+  }
+  return strategy;
+}
+
+/** The payload of a single FileOutput#write offloaded to the output worker */
+export interface OutputWorkerPayload {
+  id: string,
+  title: string,
+  description: string[],
+  dateMs: number,
+  strategies: StrategyDescriptor[],
+  data: StrategyWriteData
+}

Build/lib/writing-strategy/base.ts

@@ -1,7 +1,7 @@
 import { isCI } from 'ci-info';
 import type { Span } from '../../trace';
 import { calculateContentHash } from '../content-hash';
-import { compareAndWriteFile, writeFileLines } from '../create-file';
+import { compareAndWriteFile, compareAndWriteFileInWorker, writeFileLines, writeFileLinesSync } from '../create-file';
 
 /**
  * The class is not about holding rule data, instead it determines how the
@@ -98,6 +98,47 @@ export abstract class BaseWriteStrategy {
     );
   };
 
+  /**
+   * Worker-thread twin of {@link output}: identical comparison, but the write is
+   * synchronous since blocking a dedicated worker costs nothing.
+   */
+  public async outputInWorker(
+    span: Span,
+    title: string,
+    description: string[] | readonly string[],
+    date: Date,
+    filePath: string
+  ): Promise<void> {
+    const result = this.result;
+    if (!result) {
+      return;
+    }
+
+    if (isCI && this.skipCompareOnCI) {
+      writeFileLinesSync(
+        span,
+        this.withPadding(title, description, date, result, null),
+        filePath
+      );
+      return;
+    }
+
+    const contentHash = calculateContentHash(title, description, result);
+
+    await compareAndWriteFileInWorker(
+      span,
+      this.withPadding(
+        title,
+        description,
+        date,
+        result,
+        contentHash
+      ),
+      filePath,
+      contentHash
+    );
+  }
+
   public get content() {
     return this.result;
   }