Perf: worker farm

Build/build-reject-domainset.ts

@@ -15,7 +15,7 @@ import { SHARED_DESCRIPTION } from './constants/description';
 import { addArrayElementsToSet } from 'foxts/add-array-elements-to-set';
 import { OUTPUT_INTERNAL_DIR, SOURCE_DIR } from './constants/dir';
 import { DomainsetOutput, AdGuardHomeOutput } from './lib/rules/domainset';
-import { createWorker } from './lib/worker';
+import { getBuildWorkerFarm } from './lib/build-worker-farm';
 import { RulesetOutput } from './lib/rules/ruleset';
 import { fetchAssets } from './lib/fetch-assets';
 import { AUGUST_ASN, HUIZE_ASN } from '../Source/ip/badboy_asn';
@@ -25,17 +25,12 @@ const readLocalRejectExtraDomainsetPromise = readFileIntoProcessedArray(path.joi
 const readLocalRejectRulesetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/reject.conf'));
 const readLocalRejectIpListPromise = readFileIntoProcessedArray(path.resolve(SOURCE_DIR, 'ip/reject.conf'));
 
-// Downloading + parsing the remote sources is ~1s of CPU that used to sit on the
-// main thread's critical path; the parsed arrays cross back in ~25ms (see
-// lib/worker-transfer.bench.ts). Two threads because jest-worker runs one call at
-// a time per thread and the two jobs must overlap. Booted at import time so the
-// thread spin-up (loading the module graph) overlaps with the rest of startup.
-const rejectWorker = createWorker<typeof import('./lib/reject.worker')>(
-  require.resolve('./lib/reject.worker'),
-  2
-)(['getPhishingDomains', 'getRejectSources']);
-
 export const buildRejectDomainSet = task(require.main === module, __filename)(async (span) => {
+  // Downloading + parsing the remote sources is ~1s of CPU that used to sit on the
+  // main thread's critical path; the parsed arrays cross back in ~25ms (see
+  // lib/worker-transfer.bench.ts). The farm is shared with the rest of the build
+  // and ended by whoever booted it (index.ts, or process exit in standalone runs).
+  const rejectWorker = getBuildWorkerFarm();
   const rejectDomainsetOutput = new DomainsetOutput(span, 'reject')
     .withTitle('Sukka\'s Ruleset - Reject Base')
     .appendDescription(
@@ -286,6 +281,4 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as
   await myRejectOutputAdGuardHome
     .addFromRuleset(readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/my_reject.conf')))
     .write();
-
-  await rejectWorker.end();
 });

Build/index.ts

@@ -17,7 +17,8 @@ import { buildStreamService } from './build-stream-service';
 import { buildRedirectModule } from './build-sgmodule-redirect';
 import { buildAlwaysRealIPModule } from './build-sgmodule-always-realip';
 
-import { createWorker } from './lib/worker';
+import { buildTelegram } from './build-telegram';
+import { downloadMockAssets } from './download-mock-assets';
 
 import { buildPublic } from './build-public';
 import { buildCloudMounterRules } from './build-cloudmounter-rules';
@@ -30,7 +31,7 @@ import path from 'node:path';
 import { ROOT_DIR } from './constants/dir';
 import { isCI } from 'ci-info';
 import { printExternalDownloadStats } from './lib/download-stats';
-import { endOutputWorkerFarm, warmOutputWorkerFarm } from './lib/rules/output-worker-farm';
+import { endBuildWorkerFarm, getBuildWorkerFarm, warmBuildWorkerFarm } from './lib/build-worker-farm';
 import { appendArrayInPlace } from 'foxts/append-array-in-place';
 
 process.on('uncaughtException', (error) => {
@@ -78,26 +79,13 @@ const buildFinishedLock = path.join(ROOT_DIR, '.BUILD_FINISHED');
   const eluAtStart = performance.eventLoopUtilization();
   const cpuAtStart = process.cpuUsage();
 
-  const microsoftCdnWorker = createWorker<typeof import('./build-microsoft-cdn.worker')>(
+  // The one thread farm every off-main-thread job goes through: the reject
-    require.resolve('./build-microsoft-cdn.worker')
+  // sources / phishing parse, the two trie-heavy CDN tasks, and any FileOutput
-  )(['buildMicrosoftCdn']);
+  // that crosses the offload threshold. Booted here rather than on first use so
-
+  // the ~250ms-per-thread spin-up overlaps the first downloads instead of landing
-  const cdnDownloadWorker = createWorker<typeof import('./build-cdn-download-conf.worker')>(
+  // on the critical path. Everything else is I/O-bound and stays on this thread.
-    require.resolve('./build-cdn-download-conf.worker')
+  warmBuildWorkerFarm();
-  )(['buildCdnDownloadConf']);
+  const farm = getBuildWorkerFarm();
-
-  const telegramWorker = createWorker<typeof import('./build-telegram.worker')>(
-    require.resolve('./build-telegram.worker')
-  )(['buildTelegram']);
-
-  const mockAssetsWorker = createWorker<typeof import('./download-mock-assets.worker')>(
-    require.resolve('./download-mock-assets.worker')
-  )(['downloadMockAssets']);
-
-  // Shared by any task whose FileOutput crosses the offload threshold. Booted here
-  // rather than inside a task so the ~250ms thread spin-up overlaps the downloads
-  // instead of landing on the critical path when the writes finally dispatch.
-  warmOutputWorkerFarm();
 
   try {
     // only enable why-is-node-running in GitHub Actions debug mode
@@ -114,7 +102,7 @@ const buildFinishedLock = path.join(ROOT_DIR, '.BUILD_FINISHED');
         downloadPreviousBuildPromise.then(() => buildRejectIPList()),
         downloadPreviousBuildPromise.then(() => buildAppleCdn()),
         downloadPreviousBuildPromise.then(() => buildAICIDR()),
-        downloadPreviousBuildPromise.then(() => cdnDownloadWorker.buildCdnDownloadConf()),
+        downloadPreviousBuildPromise.then(() => farm.buildCdnDownloadConf()),
         downloadPreviousBuildPromise.then(() => buildRejectDomainSet()),
         downloadPreviousBuildPromise.then(() => buildChnCidr()),
         downloadPreviousBuildPromise.then(() => buildSpeedtestDomainSet()),
@@ -123,11 +111,11 @@ const buildFinishedLock = path.join(ROOT_DIR, '.BUILD_FINISHED');
         downloadPreviousBuildPromise.then(() => buildRedirectModule()),
         downloadPreviousBuildPromise.then(() => buildAlwaysRealIPModule()),
         downloadPreviousBuildPromise.then(() => buildStreamService()),
-        downloadPreviousBuildPromise.then(() => microsoftCdnWorker.buildMicrosoftCdn()),
+        downloadPreviousBuildPromise.then(() => farm.buildMicrosoftCdn()),
         downloadPreviousBuildPromise.then(() => buildCloudMounterRules()),
-        mockAssetsWorker.downloadMockAssets()
+        downloadMockAssets()
       ]),
-      downloadPreviousBuildPromise.then(() => telegramWorker.buildTelegram())
+      downloadPreviousBuildPromise.then(() => buildTelegram())
     ]);
 
     appendArrayInPlace(traces, telegramTraces);
@@ -142,13 +130,7 @@ const buildFinishedLock = path.join(ROOT_DIR, '.BUILD_FINISHED');
       cpu: process.cpuUsage(cpuAtStart)
     });
 
-    await Promise.all([
+    await endBuildWorkerFarm();
-      microsoftCdnWorker.end(),
-      cdnDownloadWorker.end(),
-      telegramWorker.end(),
-      mockAssetsWorker.end(),
-      endOutputWorkerFarm()
-    ]);
 
     // Finish the build to avoid leaking timer/fetch ref
     await whyIsNodeRunning();

Build/lib/build-worker-farm.ts

@@ -0,0 +1,49 @@
+import type { JestWorkerFarm } from 'jest-worker';
+import { createWorker } from './worker';
+import { once } from 'foxts/once';
+
+type BuildWorkerModule = typeof import('./build.worker');
+type BuildWorkerFarm = JestWorkerFarm<BuildWorkerModule>;
+
+/**
+ * Every thread costs ~250ms of CPU to boot (Node + @swc-node/register + the module
+ * graph), and CI has 4 cores. One farm of 3 threads plus the main thread fills
+ * them exactly, and 3 is enough because the two phases that use the farm do not
+ * overlap and each needs at most 3 threads at once:
+ *
+ *   early:  getRejectSources | getPhishingDomains | buildMicrosoftCdn -> buildCdnDownloadConf
+ *   late:   writeOutput reject | reject_extra | reject_phishing -> reject-adguardhome
+ *
+ * jest-worker holds a lock per in-flight call, so a 4th concurrent call queues
+ * behind the first thread to free up; in the early phase that is the ~0.3s
+ * cdn-download-conf job waiting for the ~0.75s microsoft-cdn job, both well
+ * inside the ~2.4s the reject sources take on the critical path.
+ */
+const NUM_WORKERS = 3;
+
+/**
+ * IMPORTANT: whoever triggers the lazy boot is responsible for a matching
+ * endBuildWorkerFarm() (idempotent, safe to call unconditionally), otherwise the
+ * worker threads keep a standalone task run alive.
+ */
+export const getBuildWorkerFarm = once((): BuildWorkerFarm => createWorker<BuildWorkerModule>(require.resolve('./build.worker'), NUM_WORKERS)([
+  'writeOutput',
+  'getPhishingDomains',
+  'getRejectSources',
+  'buildMicrosoftCdn',
+  'buildCdnDownloadConf'
+]), false /* warm manually */);
+
+/**
+ * Boot the farm ahead of time so the thread spin-up overlaps with whatever the
+ * main thread is doing first, instead of landing on the critical path of the
+ * first job dispatched to it.
+ */
+export function warmBuildWorkerFarm(): void {
+  getBuildWorkerFarm();
+}
+
+export async function endBuildWorkerFarm(): Promise<void> {
+  const farm = getBuildWorkerFarm();
+  await farm.end();
+}

Build/lib/build.worker.ts

@@ -0,0 +1,13 @@
+/**
+ * The single worker module behind the build's shared thread farm. Every job that
+ * runs off the main thread is exported from here; each is implemented in its own
+ * module, this file is only the surface jest-worker loads.
+ *
+ * jest-worker runs one call at a time per thread, so the farm size is chosen for
+ * the phase with the most concurrent calls (see build-worker-farm.ts).
+ */
+export { writeOutput } from './rules/output.worker';
+export { getPhishingDomains } from './get-phishing-domains';
+export { getRejectSources } from './get-reject-sources';
+export { buildMicrosoftCdn } from '../build-microsoft-cdn';
+export { buildCdnDownloadConf } from '../build-cdn-download-conf';

Build/lib/reject.worker.ts

@@ -1,2 +0,0 @@
-export { getPhishingDomains } from './get-phishing-domains';
-export { getRejectSources } from './get-reject-sources';

Build/lib/rules/base.ts

@@ -11,7 +11,7 @@ import { appendArrayInPlace } from 'foxts/append-array-in-place';
 import { isMainThread } from 'node:worker_threads';
 import { resolveStrategyOutputPath, serializeStrategy, writeDataToStrategies } from './strategy-write-data';
 import type { OutputWorkerPayload, StrategyWriteData } from './strategy-write-data';
-import { getOutputWorkerFarm } from './output-worker-farm';
+import { getBuildWorkerFarm } from '../build-worker-farm';
 
 /**
  * Below this many dumped domain entries, formatting + hashing + writing inline is
@@ -506,8 +506,8 @@ export class FileOutput {
       // synchronously so no completion ever waits on a busy main thread.
       //
       // Only worth doing from the main thread -- tasks that already run entirely on
-      // a worker (build-microsoft-cdn, build-telegram-cidr, build-cdn-download-conf)
+      // a worker (build-microsoft-cdn, build-cdn-download-conf) are not contending
-      // are not contending with anything, and must not spawn a nested worker farm.
+      // with anything, and must not spawn a nested worker farm.
       if (isMainThread && domains.length >= OUTPUT_WORKER_THRESHOLD) {
         this.guardBeforeWritingToStrategies();
 
@@ -522,7 +522,7 @@ export class FileOutput {
 
         return childSpan.traceWorkerChild(
           'write via output worker',
-          rawSpan => getOutputWorkerFarm().writeOutput(rawSpan, payload)
+          rawSpan => getBuildWorkerFarm().writeOutput(rawSpan, payload)
         );
       }
 

Build/lib/rules/output-worker-farm.ts

@@ -1,40 +0,0 @@
-import type { JestWorkerFarm } from 'jest-worker';
-import { createWorker } from '../worker';
-
-type OutputWorkerModule = typeof import('./output.worker');
-type OutputWorkerFarm = JestWorkerFarm<Pick<OutputWorkerModule, 'writeOutput'>>;
-
-let farm: OutputWorkerFarm | null = null;
-
-/**
- * Lazily boot the output worker farm. Only FileOutput#write dispatches here, and
- * only when an output crosses the offload threshold -- today that is exclusively
- * the reject domainsets / adguardhome outputs of build-reject-domainset.
- *
- * IMPORTANT: whoever triggers the lazy boot is responsible for a matching
- * endOutputWorkerFarm() (idempotent, safe to call unconditionally), otherwise the
- * worker threads keep a standalone task run alive.
- */
-export function getOutputWorkerFarm(): OutputWorkerFarm {
-  // 3 workers: reject, reject_extra and reject_phishing format & write in parallel
-  farm ??= createWorker<OutputWorkerModule>(require.resolve('./output.worker'), 3)(['writeOutput']);
-  return farm;
-}
-
-/**
- * Boot the farm ahead of time. Spawning the threads costs ~200-300ms (each loads
- * @swc-node/register and compiles the module graph), and since every big output
- * dispatches at the very end of a task, that cost otherwise lands entirely on the
- * critical path. Call this as early as the task starts so it overlaps the downloads.
- */
-export function warmOutputWorkerFarm(): void {
-  getOutputWorkerFarm();
-}
-
-export async function endOutputWorkerFarm(): Promise<void> {
-  if (farm) {
-    const f = farm;
-    farm = null;
-    await f.end();
-  }
-}