Perf: worker farm

Build/lib/build-worker-farm.ts

@@ -0,0 +1,49 @@
+import type { JestWorkerFarm } from 'jest-worker';
+import { createWorker } from './worker';
+import { once } from 'foxts/once';
+
+type BuildWorkerModule = typeof import('./build.worker');
+type BuildWorkerFarm = JestWorkerFarm<BuildWorkerModule>;
+
+/**
+ * Every thread costs ~250ms of CPU to boot (Node + @swc-node/register + the module
+ * graph), and CI has 4 cores. One farm of 3 threads plus the main thread fills
+ * them exactly, and 3 is enough because the two phases that use the farm do not
+ * overlap and each needs at most 3 threads at once:
+ *
+ *   early:  getRejectSources | getPhishingDomains | buildMicrosoftCdn -> buildCdnDownloadConf
+ *   late:   writeOutput reject | reject_extra | reject_phishing -> reject-adguardhome
+ *
+ * jest-worker holds a lock per in-flight call, so a 4th concurrent call queues
+ * behind the first thread to free up; in the early phase that is the ~0.3s
+ * cdn-download-conf job waiting for the ~0.75s microsoft-cdn job, both well
+ * inside the ~2.4s the reject sources take on the critical path.
+ */
+const NUM_WORKERS = 3;
+
+/**
+ * IMPORTANT: whoever triggers the lazy boot is responsible for a matching
+ * endBuildWorkerFarm() (idempotent, safe to call unconditionally), otherwise the
+ * worker threads keep a standalone task run alive.
+ */
+export const getBuildWorkerFarm = once((): BuildWorkerFarm => createWorker<BuildWorkerModule>(require.resolve('./build.worker'), NUM_WORKERS)([
+  'writeOutput',
+  'getPhishingDomains',
+  'getRejectSources',
+  'buildMicrosoftCdn',
+  'buildCdnDownloadConf'
+]), false /* warm manually */);
+
+/**
+ * Boot the farm ahead of time so the thread spin-up overlaps with whatever the
+ * main thread is doing first, instead of landing on the critical path of the
+ * first job dispatched to it.
+ */
+export function warmBuildWorkerFarm(): void {
+  getBuildWorkerFarm();
+}
+
+export async function endBuildWorkerFarm(): Promise<void> {
+  const farm = getBuildWorkerFarm();
+  await farm.end();
+}

Build/lib/build.worker.ts

@@ -0,0 +1,13 @@
+/**
+ * The single worker module behind the build's shared thread farm. Every job that
+ * runs off the main thread is exported from here; each is implemented in its own
+ * module, this file is only the surface jest-worker loads.
+ *
+ * jest-worker runs one call at a time per thread, so the farm size is chosen for
+ * the phase with the most concurrent calls (see build-worker-farm.ts).
+ */
+export { writeOutput } from './rules/output.worker';
+export { getPhishingDomains } from './get-phishing-domains';
+export { getRejectSources } from './get-reject-sources';
+export { buildMicrosoftCdn } from '../build-microsoft-cdn';
+export { buildCdnDownloadConf } from '../build-cdn-download-conf';

Build/lib/reject.worker.ts

@@ -1,2 +0,0 @@
-export { getPhishingDomains } from './get-phishing-domains';
-export { getRejectSources } from './get-reject-sources';

Build/lib/rules/base.ts

@@ -11,7 +11,7 @@ import { appendArrayInPlace } from 'foxts/append-array-in-place';
 import { isMainThread } from 'node:worker_threads';
 import { resolveStrategyOutputPath, serializeStrategy, writeDataToStrategies } from './strategy-write-data';
 import type { OutputWorkerPayload, StrategyWriteData } from './strategy-write-data';
-import { getOutputWorkerFarm } from './output-worker-farm';
+import { getBuildWorkerFarm } from '../build-worker-farm';
 
 /**
  * Below this many dumped domain entries, formatting + hashing + writing inline is
@@ -506,8 +506,8 @@ export class FileOutput {
       // synchronously so no completion ever waits on a busy main thread.
       //
       // Only worth doing from the main thread -- tasks that already run entirely on
-      // a worker (build-microsoft-cdn, build-telegram-cidr, build-cdn-download-conf)
-      // are not contending with anything, and must not spawn a nested worker farm.
+      // a worker (build-microsoft-cdn, build-cdn-download-conf) are not contending
+      // with anything, and must not spawn a nested worker farm.
       if (isMainThread && domains.length >= OUTPUT_WORKER_THRESHOLD) {
         this.guardBeforeWritingToStrategies();
 
@@ -522,7 +522,7 @@ export class FileOutput {
 
         return childSpan.traceWorkerChild(
           'write via output worker',
-          rawSpan => getOutputWorkerFarm().writeOutput(rawSpan, payload)
+          rawSpan => getBuildWorkerFarm().writeOutput(rawSpan, payload)
         );
       }
 

Build/lib/rules/output-worker-farm.ts

@@ -1,40 +0,0 @@
-import type { JestWorkerFarm } from 'jest-worker';
-import { createWorker } from '../worker';
-
-type OutputWorkerModule = typeof import('./output.worker');
-type OutputWorkerFarm = JestWorkerFarm<Pick<OutputWorkerModule, 'writeOutput'>>;
-
-let farm: OutputWorkerFarm | null = null;
-
-/**
- * Lazily boot the output worker farm. Only FileOutput#write dispatches here, and
- * only when an output crosses the offload threshold -- today that is exclusively
- * the reject domainsets / adguardhome outputs of build-reject-domainset.
- *
- * IMPORTANT: whoever triggers the lazy boot is responsible for a matching
- * endOutputWorkerFarm() (idempotent, safe to call unconditionally), otherwise the
- * worker threads keep a standalone task run alive.
- */
-export function getOutputWorkerFarm(): OutputWorkerFarm {
-  // 3 workers: reject, reject_extra and reject_phishing format & write in parallel
-  farm ??= createWorker<OutputWorkerModule>(require.resolve('./output.worker'), 3)(['writeOutput']);
-  return farm;
-}
-
-/**
- * Boot the farm ahead of time. Spawning the threads costs ~200-300ms (each loads
- * @swc-node/register and compiles the module graph), and since every big output
- * dispatches at the very end of a task, that cost otherwise lands entirely on the
- * critical path. Call this as early as the task starts so it overlaps the downloads.
- */
-export function warmOutputWorkerFarm(): void {
-  getOutputWorkerFarm();
-}
-
-export async function endOutputWorkerFarm(): Promise<void> {
-  if (farm) {
-    const f = farm;
-    farm = null;
-    await f.end();
-  }
-}