From 9b155695e77a253bd751ba1447cbccc4ed52096c Mon Sep 17 00:00:00 2001 From: SukkaW Date: Wed, 2 Sep 2026 20:01:48 +0800 Subject: [PATCH] Perf: worker farm --- ...f.worker.ts => build-cdn-download-conf.ts} | 0 ...t-cdn.worker.ts => build-microsoft-cdn.ts} | 0 Build/build-reject-domainset.ts | 19 +++---- ...d-telegram.worker.ts => build-telegram.ts} | 0 ...sets.worker.ts => download-mock-assets.ts} | 0 Build/index.ts | 48 ++++++------------ Build/lib/build-worker-farm.ts | 49 +++++++++++++++++++ Build/lib/build.worker.ts | 13 +++++ Build/lib/reject.worker.ts | 2 - Build/lib/rules/base.ts | 8 +-- Build/lib/rules/output-worker-farm.ts | 40 --------------- 11 files changed, 87 insertions(+), 92 deletions(-) rename Build/{build-cdn-download-conf.worker.ts => build-cdn-download-conf.ts} (100%) rename Build/{build-microsoft-cdn.worker.ts => build-microsoft-cdn.ts} (100%) rename Build/{build-telegram.worker.ts => build-telegram.ts} (100%) rename Build/{download-mock-assets.worker.ts => download-mock-assets.ts} (100%) create mode 100644 Build/lib/build-worker-farm.ts create mode 100644 Build/lib/build.worker.ts delete mode 100644 Build/lib/reject.worker.ts delete mode 100644 Build/lib/rules/output-worker-farm.ts diff --git a/Build/build-cdn-download-conf.worker.ts b/Build/build-cdn-download-conf.ts similarity index 100% rename from Build/build-cdn-download-conf.worker.ts rename to Build/build-cdn-download-conf.ts diff --git a/Build/build-microsoft-cdn.worker.ts b/Build/build-microsoft-cdn.ts similarity index 100% rename from Build/build-microsoft-cdn.worker.ts rename to Build/build-microsoft-cdn.ts diff --git a/Build/build-reject-domainset.ts b/Build/build-reject-domainset.ts index 81673989..236ef7be 100644 --- a/Build/build-reject-domainset.ts +++ b/Build/build-reject-domainset.ts @@ -15,7 +15,7 @@ import { SHARED_DESCRIPTION } from './constants/description'; import { addArrayElementsToSet } from 'foxts/add-array-elements-to-set'; import { OUTPUT_INTERNAL_DIR, SOURCE_DIR } from './constants/dir'; import { DomainsetOutput, AdGuardHomeOutput } from './lib/rules/domainset'; -import { createWorker } from './lib/worker'; +import { getBuildWorkerFarm } from './lib/build-worker-farm'; import { RulesetOutput } from './lib/rules/ruleset'; import { fetchAssets } from './lib/fetch-assets'; import { AUGUST_ASN, HUIZE_ASN } from '../Source/ip/badboy_asn'; @@ -25,17 +25,12 @@ const readLocalRejectExtraDomainsetPromise = readFileIntoProcessedArray(path.joi const readLocalRejectRulesetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/reject.conf')); const readLocalRejectIpListPromise = readFileIntoProcessedArray(path.resolve(SOURCE_DIR, 'ip/reject.conf')); -// Downloading + parsing the remote sources is ~1s of CPU that used to sit on the -// main thread's critical path; the parsed arrays cross back in ~25ms (see -// lib/worker-transfer.bench.ts). Two threads because jest-worker runs one call at -// a time per thread and the two jobs must overlap. Booted at import time so the -// thread spin-up (loading the module graph) overlaps with the rest of startup. -const rejectWorker = createWorker( - require.resolve('./lib/reject.worker'), - 2 -)(['getPhishingDomains', 'getRejectSources']); - export const buildRejectDomainSet = task(require.main === module, __filename)(async (span) => { + // Downloading + parsing the remote sources is ~1s of CPU that used to sit on the + // main thread's critical path; the parsed arrays cross back in ~25ms (see + // lib/worker-transfer.bench.ts). The farm is shared with the rest of the build + // and ended by whoever booted it (index.ts, or process exit in standalone runs). + const rejectWorker = getBuildWorkerFarm(); const rejectDomainsetOutput = new DomainsetOutput(span, 'reject') .withTitle('Sukka\'s Ruleset - Reject Base') .appendDescription( @@ -286,6 +281,4 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as await myRejectOutputAdGuardHome .addFromRuleset(readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/my_reject.conf'))) .write(); - - await rejectWorker.end(); }); diff --git a/Build/build-telegram.worker.ts b/Build/build-telegram.ts similarity index 100% rename from Build/build-telegram.worker.ts rename to Build/build-telegram.ts diff --git a/Build/download-mock-assets.worker.ts b/Build/download-mock-assets.ts similarity index 100% rename from Build/download-mock-assets.worker.ts rename to Build/download-mock-assets.ts diff --git a/Build/index.ts b/Build/index.ts index 105b53c1..e6462cb4 100644 --- a/Build/index.ts +++ b/Build/index.ts @@ -17,7 +17,8 @@ import { buildStreamService } from './build-stream-service'; import { buildRedirectModule } from './build-sgmodule-redirect'; import { buildAlwaysRealIPModule } from './build-sgmodule-always-realip'; -import { createWorker } from './lib/worker'; +import { buildTelegram } from './build-telegram'; +import { downloadMockAssets } from './download-mock-assets'; import { buildPublic } from './build-public'; import { buildCloudMounterRules } from './build-cloudmounter-rules'; @@ -30,7 +31,7 @@ import path from 'node:path'; import { ROOT_DIR } from './constants/dir'; import { isCI } from 'ci-info'; import { printExternalDownloadStats } from './lib/download-stats'; -import { endOutputWorkerFarm, warmOutputWorkerFarm } from './lib/rules/output-worker-farm'; +import { endBuildWorkerFarm, getBuildWorkerFarm, warmBuildWorkerFarm } from './lib/build-worker-farm'; import { appendArrayInPlace } from 'foxts/append-array-in-place'; process.on('uncaughtException', (error) => { @@ -78,26 +79,13 @@ const buildFinishedLock = path.join(ROOT_DIR, '.BUILD_FINISHED'); const eluAtStart = performance.eventLoopUtilization(); const cpuAtStart = process.cpuUsage(); - const microsoftCdnWorker = createWorker( - require.resolve('./build-microsoft-cdn.worker') - )(['buildMicrosoftCdn']); - - const cdnDownloadWorker = createWorker( - require.resolve('./build-cdn-download-conf.worker') - )(['buildCdnDownloadConf']); - - const telegramWorker = createWorker( - require.resolve('./build-telegram.worker') - )(['buildTelegram']); - - const mockAssetsWorker = createWorker( - require.resolve('./download-mock-assets.worker') - )(['downloadMockAssets']); - - // Shared by any task whose FileOutput crosses the offload threshold. Booted here - // rather than inside a task so the ~250ms thread spin-up overlaps the downloads - // instead of landing on the critical path when the writes finally dispatch. - warmOutputWorkerFarm(); + // The one thread farm every off-main-thread job goes through: the reject + // sources / phishing parse, the two trie-heavy CDN tasks, and any FileOutput + // that crosses the offload threshold. Booted here rather than on first use so + // the ~250ms-per-thread spin-up overlaps the first downloads instead of landing + // on the critical path. Everything else is I/O-bound and stays on this thread. + warmBuildWorkerFarm(); + const farm = getBuildWorkerFarm(); try { // only enable why-is-node-running in GitHub Actions debug mode @@ -114,7 +102,7 @@ const buildFinishedLock = path.join(ROOT_DIR, '.BUILD_FINISHED'); downloadPreviousBuildPromise.then(() => buildRejectIPList()), downloadPreviousBuildPromise.then(() => buildAppleCdn()), downloadPreviousBuildPromise.then(() => buildAICIDR()), - downloadPreviousBuildPromise.then(() => cdnDownloadWorker.buildCdnDownloadConf()), + downloadPreviousBuildPromise.then(() => farm.buildCdnDownloadConf()), downloadPreviousBuildPromise.then(() => buildRejectDomainSet()), downloadPreviousBuildPromise.then(() => buildChnCidr()), downloadPreviousBuildPromise.then(() => buildSpeedtestDomainSet()), @@ -123,11 +111,11 @@ const buildFinishedLock = path.join(ROOT_DIR, '.BUILD_FINISHED'); downloadPreviousBuildPromise.then(() => buildRedirectModule()), downloadPreviousBuildPromise.then(() => buildAlwaysRealIPModule()), downloadPreviousBuildPromise.then(() => buildStreamService()), - downloadPreviousBuildPromise.then(() => microsoftCdnWorker.buildMicrosoftCdn()), + downloadPreviousBuildPromise.then(() => farm.buildMicrosoftCdn()), downloadPreviousBuildPromise.then(() => buildCloudMounterRules()), - mockAssetsWorker.downloadMockAssets() + downloadMockAssets() ]), - downloadPreviousBuildPromise.then(() => telegramWorker.buildTelegram()) + downloadPreviousBuildPromise.then(() => buildTelegram()) ]); appendArrayInPlace(traces, telegramTraces); @@ -142,13 +130,7 @@ const buildFinishedLock = path.join(ROOT_DIR, '.BUILD_FINISHED'); cpu: process.cpuUsage(cpuAtStart) }); - await Promise.all([ - microsoftCdnWorker.end(), - cdnDownloadWorker.end(), - telegramWorker.end(), - mockAssetsWorker.end(), - endOutputWorkerFarm() - ]); + await endBuildWorkerFarm(); // Finish the build to avoid leaking timer/fetch ref await whyIsNodeRunning(); diff --git a/Build/lib/build-worker-farm.ts b/Build/lib/build-worker-farm.ts new file mode 100644 index 00000000..82e42ae0 --- /dev/null +++ b/Build/lib/build-worker-farm.ts @@ -0,0 +1,49 @@ +import type { JestWorkerFarm } from 'jest-worker'; +import { createWorker } from './worker'; +import { once } from 'foxts/once'; + +type BuildWorkerModule = typeof import('./build.worker'); +type BuildWorkerFarm = JestWorkerFarm; + +/** + * Every thread costs ~250ms of CPU to boot (Node + @swc-node/register + the module + * graph), and CI has 4 cores. One farm of 3 threads plus the main thread fills + * them exactly, and 3 is enough because the two phases that use the farm do not + * overlap and each needs at most 3 threads at once: + * + * early: getRejectSources | getPhishingDomains | buildMicrosoftCdn -> buildCdnDownloadConf + * late: writeOutput reject | reject_extra | reject_phishing -> reject-adguardhome + * + * jest-worker holds a lock per in-flight call, so a 4th concurrent call queues + * behind the first thread to free up; in the early phase that is the ~0.3s + * cdn-download-conf job waiting for the ~0.75s microsoft-cdn job, both well + * inside the ~2.4s the reject sources take on the critical path. + */ +const NUM_WORKERS = 3; + +/** + * IMPORTANT: whoever triggers the lazy boot is responsible for a matching + * endBuildWorkerFarm() (idempotent, safe to call unconditionally), otherwise the + * worker threads keep a standalone task run alive. + */ +export const getBuildWorkerFarm = once((): BuildWorkerFarm => createWorker(require.resolve('./build.worker'), NUM_WORKERS)([ + 'writeOutput', + 'getPhishingDomains', + 'getRejectSources', + 'buildMicrosoftCdn', + 'buildCdnDownloadConf' +]), false /* warm manually */); + +/** + * Boot the farm ahead of time so the thread spin-up overlaps with whatever the + * main thread is doing first, instead of landing on the critical path of the + * first job dispatched to it. + */ +export function warmBuildWorkerFarm(): void { + getBuildWorkerFarm(); +} + +export async function endBuildWorkerFarm(): Promise { + const farm = getBuildWorkerFarm(); + await farm.end(); +} diff --git a/Build/lib/build.worker.ts b/Build/lib/build.worker.ts new file mode 100644 index 00000000..1c954788 --- /dev/null +++ b/Build/lib/build.worker.ts @@ -0,0 +1,13 @@ +/** + * The single worker module behind the build's shared thread farm. Every job that + * runs off the main thread is exported from here; each is implemented in its own + * module, this file is only the surface jest-worker loads. + * + * jest-worker runs one call at a time per thread, so the farm size is chosen for + * the phase with the most concurrent calls (see build-worker-farm.ts). + */ +export { writeOutput } from './rules/output.worker'; +export { getPhishingDomains } from './get-phishing-domains'; +export { getRejectSources } from './get-reject-sources'; +export { buildMicrosoftCdn } from '../build-microsoft-cdn'; +export { buildCdnDownloadConf } from '../build-cdn-download-conf'; diff --git a/Build/lib/reject.worker.ts b/Build/lib/reject.worker.ts deleted file mode 100644 index 6c05c009..00000000 --- a/Build/lib/reject.worker.ts +++ /dev/null @@ -1,2 +0,0 @@ -export { getPhishingDomains } from './get-phishing-domains'; -export { getRejectSources } from './get-reject-sources'; diff --git a/Build/lib/rules/base.ts b/Build/lib/rules/base.ts index b8d1c0d1..4e227476 100644 --- a/Build/lib/rules/base.ts +++ b/Build/lib/rules/base.ts @@ -11,7 +11,7 @@ import { appendArrayInPlace } from 'foxts/append-array-in-place'; import { isMainThread } from 'node:worker_threads'; import { resolveStrategyOutputPath, serializeStrategy, writeDataToStrategies } from './strategy-write-data'; import type { OutputWorkerPayload, StrategyWriteData } from './strategy-write-data'; -import { getOutputWorkerFarm } from './output-worker-farm'; +import { getBuildWorkerFarm } from '../build-worker-farm'; /** * Below this many dumped domain entries, formatting + hashing + writing inline is @@ -506,8 +506,8 @@ export class FileOutput { // synchronously so no completion ever waits on a busy main thread. // // Only worth doing from the main thread -- tasks that already run entirely on - // a worker (build-microsoft-cdn, build-telegram-cidr, build-cdn-download-conf) - // are not contending with anything, and must not spawn a nested worker farm. + // a worker (build-microsoft-cdn, build-cdn-download-conf) are not contending + // with anything, and must not spawn a nested worker farm. if (isMainThread && domains.length >= OUTPUT_WORKER_THRESHOLD) { this.guardBeforeWritingToStrategies(); @@ -522,7 +522,7 @@ export class FileOutput { return childSpan.traceWorkerChild( 'write via output worker', - rawSpan => getOutputWorkerFarm().writeOutput(rawSpan, payload) + rawSpan => getBuildWorkerFarm().writeOutput(rawSpan, payload) ); } diff --git a/Build/lib/rules/output-worker-farm.ts b/Build/lib/rules/output-worker-farm.ts deleted file mode 100644 index 8fd28594..00000000 --- a/Build/lib/rules/output-worker-farm.ts +++ /dev/null @@ -1,40 +0,0 @@ -import type { JestWorkerFarm } from 'jest-worker'; -import { createWorker } from '../worker'; - -type OutputWorkerModule = typeof import('./output.worker'); -type OutputWorkerFarm = JestWorkerFarm>; - -let farm: OutputWorkerFarm | null = null; - -/** - * Lazily boot the output worker farm. Only FileOutput#write dispatches here, and - * only when an output crosses the offload threshold -- today that is exclusively - * the reject domainsets / adguardhome outputs of build-reject-domainset. - * - * IMPORTANT: whoever triggers the lazy boot is responsible for a matching - * endOutputWorkerFarm() (idempotent, safe to call unconditionally), otherwise the - * worker threads keep a standalone task run alive. - */ -export function getOutputWorkerFarm(): OutputWorkerFarm { - // 3 workers: reject, reject_extra and reject_phishing format & write in parallel - farm ??= createWorker(require.resolve('./output.worker'), 3)(['writeOutput']); - return farm; -} - -/** - * Boot the farm ahead of time. Spawning the threads costs ~200-300ms (each loads - * @swc-node/register and compiles the module graph), and since every big output - * dispatches at the very end of a task, that cost otherwise lands entirely on the - * critical path. Call this as early as the task starts so it overlaps the downloads. - */ -export function warmOutputWorkerFarm(): void { - getOutputWorkerFarm(); -} - -export async function endOutputWorkerFarm(): Promise { - if (farm) { - const f = farm; - farm = null; - await f.end(); - } -}