Perf: worker farm

Build/index.ts

@@ -17,7 +17,8 @@ import { buildStreamService } from './build-stream-service';
 import { buildRedirectModule } from './build-sgmodule-redirect';
 import { buildAlwaysRealIPModule } from './build-sgmodule-always-realip';
 
-import { createWorker } from './lib/worker';
+import { buildTelegram } from './build-telegram';
+import { downloadMockAssets } from './download-mock-assets';
 
 import { buildPublic } from './build-public';
 import { buildCloudMounterRules } from './build-cloudmounter-rules';
@@ -30,7 +31,7 @@ import path from 'node:path';
 import { ROOT_DIR } from './constants/dir';
 import { isCI } from 'ci-info';
 import { printExternalDownloadStats } from './lib/download-stats';
-import { endOutputWorkerFarm, warmOutputWorkerFarm } from './lib/rules/output-worker-farm';
+import { endBuildWorkerFarm, getBuildWorkerFarm, warmBuildWorkerFarm } from './lib/build-worker-farm';
 import { appendArrayInPlace } from 'foxts/append-array-in-place';
 
 process.on('uncaughtException', (error) => {
@@ -78,26 +79,13 @@ const buildFinishedLock = path.join(ROOT_DIR, '.BUILD_FINISHED');
   const eluAtStart = performance.eventLoopUtilization();
   const cpuAtStart = process.cpuUsage();
 
-  const microsoftCdnWorker = createWorker<typeof import('./build-microsoft-cdn.worker')>(
-    require.resolve('./build-microsoft-cdn.worker')
-  )(['buildMicrosoftCdn']);
-
-  const cdnDownloadWorker = createWorker<typeof import('./build-cdn-download-conf.worker')>(
-    require.resolve('./build-cdn-download-conf.worker')
-  )(['buildCdnDownloadConf']);
-
-  const telegramWorker = createWorker<typeof import('./build-telegram.worker')>(
-    require.resolve('./build-telegram.worker')
-  )(['buildTelegram']);
-
-  const mockAssetsWorker = createWorker<typeof import('./download-mock-assets.worker')>(
-    require.resolve('./download-mock-assets.worker')
-  )(['downloadMockAssets']);
-
-  // Shared by any task whose FileOutput crosses the offload threshold. Booted here
-  // rather than inside a task so the ~250ms thread spin-up overlaps the downloads
-  // instead of landing on the critical path when the writes finally dispatch.
-  warmOutputWorkerFarm();
+  // The one thread farm every off-main-thread job goes through: the reject
+  // sources / phishing parse, the two trie-heavy CDN tasks, and any FileOutput
+  // that crosses the offload threshold. Booted here rather than on first use so
+  // the ~250ms-per-thread spin-up overlaps the first downloads instead of landing
+  // on the critical path. Everything else is I/O-bound and stays on this thread.
+  warmBuildWorkerFarm();
+  const farm = getBuildWorkerFarm();
 
   try {
     // only enable why-is-node-running in GitHub Actions debug mode
@@ -114,7 +102,7 @@ const buildFinishedLock = path.join(ROOT_DIR, '.BUILD_FINISHED');
         downloadPreviousBuildPromise.then(() => buildRejectIPList()),
         downloadPreviousBuildPromise.then(() => buildAppleCdn()),
         downloadPreviousBuildPromise.then(() => buildAICIDR()),
-        downloadPreviousBuildPromise.then(() => cdnDownloadWorker.buildCdnDownloadConf()),
+        downloadPreviousBuildPromise.then(() => farm.buildCdnDownloadConf()),
         downloadPreviousBuildPromise.then(() => buildRejectDomainSet()),
         downloadPreviousBuildPromise.then(() => buildChnCidr()),
         downloadPreviousBuildPromise.then(() => buildSpeedtestDomainSet()),
@@ -123,11 +111,11 @@ const buildFinishedLock = path.join(ROOT_DIR, '.BUILD_FINISHED');
         downloadPreviousBuildPromise.then(() => buildRedirectModule()),
         downloadPreviousBuildPromise.then(() => buildAlwaysRealIPModule()),
         downloadPreviousBuildPromise.then(() => buildStreamService()),
-        downloadPreviousBuildPromise.then(() => microsoftCdnWorker.buildMicrosoftCdn()),
+        downloadPreviousBuildPromise.then(() => farm.buildMicrosoftCdn()),
         downloadPreviousBuildPromise.then(() => buildCloudMounterRules()),
-        mockAssetsWorker.downloadMockAssets()
+        downloadMockAssets()
       ]),
-      downloadPreviousBuildPromise.then(() => telegramWorker.buildTelegram())
+      downloadPreviousBuildPromise.then(() => buildTelegram())
     ]);
 
     appendArrayInPlace(traces, telegramTraces);
@@ -142,13 +130,7 @@ const buildFinishedLock = path.join(ROOT_DIR, '.BUILD_FINISHED');
       cpu: process.cpuUsage(cpuAtStart)
     });
 
-    await Promise.all([
-      microsoftCdnWorker.end(),
-      cdnDownloadWorker.end(),
-      telegramWorker.end(),
-      mockAssetsWorker.end(),
-      endOutputWorkerFarm()
-    ]);
+    await endBuildWorkerFarm();
 
     // Finish the build to avoid leaking timer/fetch ref
     await whyIsNodeRunning();