diff --git a/Build/build-reject-domainset.ts b/Build/build-reject-domainset.ts index 75ebb2fc..cc493631 100644 --- a/Build/build-reject-domainset.ts +++ b/Build/build-reject-domainset.ts @@ -19,6 +19,7 @@ import { OUTPUT_INTERNAL_DIR, SOURCE_DIR } from './constants/dir'; import { DomainsetOutput, AdGuardHomeOutput } from './lib/rules/domainset'; import { foundDebugDomain } from './lib/parse-filter/shared'; import { createWorker } from './lib/worker'; +import { endOutputWorkerFarm } from './lib/rules/output-worker-farm'; import type { MaybePromise } from './lib/misc'; import { RulesetOutput } from './lib/rules/ruleset'; import { fetchAssets } from './lib/fetch-assets'; @@ -307,5 +308,8 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as .addFromRuleset(readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/my_reject.conf'))) .write(); - await phishingWorker.end(); + await Promise.all([ + phishingWorker.end(), + endOutputWorkerFarm() + ]); }); diff --git a/Build/index.ts b/Build/index.ts index b1ee784c..f53f4200 100644 --- a/Build/index.ts +++ b/Build/index.ts @@ -28,6 +28,7 @@ import path from 'node:path'; import { ROOT_DIR } from './constants/dir'; import { isCI } from 'ci-info'; import { printExternalDownloadStats } from './lib/download-stats'; +import { endOutputWorkerFarm } from './lib/rules/output-worker-farm'; process.on('uncaughtException', (error) => { console.error('Uncaught exception:', error); @@ -130,7 +131,9 @@ const buildFinishedLock = path.join(ROOT_DIR, '.BUILD_FINISHED'); microsoftCdnWorker.end(), cdnDownloadWorker.end(), telegramCidrWorker.end(), - mockAssetsWorker.end() + mockAssetsWorker.end(), + // defensive: no-op unless some FileOutput crossed the offload threshold + endOutputWorkerFarm() ]); // Finish the build to avoid leaking timer/fetch ref diff --git a/Build/lib/content-hash.ts b/Build/lib/content-hash.ts index 4431312b..dc500faf 100644 --- a/Build/lib/content-hash.ts +++ b/Build/lib/content-hash.ts @@ -1,7 +1,6 @@ import { fastStringArrayJoin } from 'foxts/fast-string-array-join'; import { Buffer } from 'node:buffer'; import { createHash } from 'node:crypto'; -import fs from 'node:fs'; import fsp from 'node:fs/promises'; /** @@ -65,21 +64,6 @@ export async function extractContentHashFromFile(filePath: string): Promise 0 && linesA[linesA.length - 1] === '') { linesA.pop(); @@ -44,15 +41,53 @@ export async function compareAndWriteFile(span: Span, linesA: string[], filePath if (isEqual) { console.log(picocolors.gray(picocolors.dim(`same content, bail out writing: ${filePath}`))); + } + + return isEqual; +} + +/** + * To keep metadata comment `last updated` not change if real content is the same, + * we only write when the actual content differs, and the new `last updated` will + * be written along with new content. + * + * When `contentHash` is provided (and the previous output already embeds a + * content hash marker), the comparison only reads the first chunk of the + * previous file. Otherwise it falls back to a full comment-insensitive + * line-by-line comparison. + */ +export async function compareAndWriteFile(span: Span, linesA: string[], filePath: string, contentHash: string | null = null) { + if (await isPreviousOutputEqual(span, linesA, filePath, contentHash)) { return; } return writeFileLines(span, linesA, filePath); } +/** + * Same comparison as {@link compareAndWriteFile}, but the write itself is + * synchronous. For worker threads only: blocking a dedicated worker is free, + * whereas an async write hands the completion back through libuv to an event + * loop we would rather not depend on being idle. + */ +export async function compareAndWriteFileInWorker(span: Span, linesA: string[], filePath: string, contentHash: string | null = null) { + if (await isPreviousOutputEqual(span, linesA, filePath, contentHash)) { + return; + } + + writeFileLinesSync(span, linesA, filePath); +} + export function writeFileLines(span: Span, linesA: string[], filePath: string): Promise { return span.traceChildAsync( `writing ${filePath}`, () => writeFile(filePath, fastStringArrayJoin(linesA, '\n') + '\n') ); } + +export function writeFileLinesSync(span: Span, linesA: string[], filePath: string): void { + span.traceChildSync(`writing ${filePath}`, () => { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, fastStringArrayJoin(linesA, '\n') + '\n'); + }); +} diff --git a/Build/lib/rules/base.ts b/Build/lib/rules/base.ts index 531c8d71..1c8b2222 100644 --- a/Build/lib/rules/base.ts +++ b/Build/lib/rules/base.ts @@ -1,16 +1,24 @@ import type { Span } from '../../trace'; import { HostnameSmolTrie } from 'hntrie/smol'; -import { domainToASCII } from 'node:url'; import { not, nullthrow } from 'foxts/guard'; import { fastIpVersion } from 'foxts/fast-ip-version'; import { addArrayElementsToSet } from 'foxts/add-array-elements-to-set'; import type { MaybePromise } from '../misc'; import type { BaseWriteStrategy } from '../writing-strategy/base'; -import { merge as mergeCidr } from 'fast-cidr-tools'; -import { createRetrieKeywordFilter as createKeywordFilter } from 'foxts/retrie'; -import path from 'node:path'; import { SurgeMitmSgmodule } from '../writing-strategy/surge'; import { appendArrayInPlace } from 'foxts/append-array-in-place'; +import { isMainThread } from 'node:worker_threads'; +import { resolveStrategyOutputPath, serializeStrategy, writeDataToStrategies } from './strategy-write-data'; +import type { OutputWorkerPayload, StrategyWriteData } from './strategy-write-data'; +import { getOutputWorkerFarm } from './output-worker-farm'; + +/** + * Below this many dumped domain entries, formatting + hashing + writing inline is + * cheaper than a worker round trip (and avoids tiny outputs queueing behind big + * jobs in the farm). Today only the reject domainsets / adguardhome outputs cross + * this threshold. + */ +const OUTPUT_WORKER_THRESHOLD = 20000; /** * Holds the universal rule data (domain, ip, url-regex, etc. etc.) @@ -399,7 +407,42 @@ export class FileOutput { // } private strategiesWritten = false; - private writeToStrategies() { + /** Collect the trie content, '.'-prefix encoded, WITHOUT punycoding (that happens in the fanout) */ + private dumpDomains(): string[] { + const domains: string[] = []; + this.domainTrie.dump((domain, includeSubdomain) => { + domains.push(includeSubdomain ? '.' + domain : domain); + }); + return domains; + } + + private buildStrategyWriteData(domains: string[]): StrategyWriteData { + return { + domains, + domainKeywords: Array.from(this.domainKeywords), + whitelistKeywords: Array.from(this.whitelistKeywords), + wildcards: Array.from(this.wildcardSet), + userAgents: Array.from(this.userAgent), + processNames: Array.from(this.processName), + processPaths: Array.from(this.processPath), + urlRegexes: Array.from(this.urlRegex), + ipcidr: Array.from(this.ipcidr), + ipcidrNoResolve: Array.from(this.ipcidrNoResolve), + ipcidr6: Array.from(this.ipcidr6), + ipcidr6NoResolve: Array.from(this.ipcidr6NoResolve), + ipasn: Array.from(this.ipasn), + ipasnNoResolve: Array.from(this.ipasnNoResolve), + geoip: Array.from(this.geoip), + geoipNoResolve: Array.from(this.groipNoResolve), + sourceIpOrCidr: Array.from(this.sourceIpOrCidr), + sourcePort: Array.from(this.sourcePort), + destPort: Array.from(this.destPort), + protocol: Array.from(this.protocol), + otherRules: this.otherRules + }; + } + + private guardBeforeWritingToStrategies() { if (this.pendingPromise) { throw new Error('You should call done() before calling writeToStrategies()'); } @@ -412,186 +455,73 @@ export class FileOutput { if (this.strategies.filter(not(false)).length === 0) { throw new Error('No strategies to write ' + this.id); } + } - // We use both DOMAIN-KEYWORD and whitelisted keyword to whitelist DOMAIN and DOMAIN-SUFFIX - const kwfilter = createKeywordFilter( - Array.from(this.domainKeywords) - .concat(Array.from(this.whitelistKeywords)) - ); - - const strategiesLen = this.strategies.length; - - const domainEntries: Array<[domain: string, subdomain: boolean]> = []; - this.domainTrie.dump((domain, includeSubdomain) => { - const d = domainToASCII(domain); - if (d) domainEntries.push([d, includeSubdomain]); - }); - domainEntries.sort((a, b) => (a[0].length - b[0].length) || (a[0] < b[0] ? -1 : (a[0] > b[0] ? 1 : 0))); - for (let j = 0, entriesLen = domainEntries.length; j < entriesLen; j++) { - const [domain, includeAllSubdomain] = domainEntries[j]; - if (kwfilter(domain)) { - continue; - } - - for (let i = 0; i < strategiesLen; i++) { - const strategy = this.strategies[i]; - if (includeAllSubdomain) { - strategy.writeDomainSuffix(domain); - } else { - strategy.writeDomain(domain); - } - } - } - - // Now, we whitelisted out DOMAIN-KEYWORD - const whiteKwfilter = createKeywordFilter(Array.from(this.whitelistKeywords)); - const whitelistedKeywords = Array.from(this.domainKeywords).filter(kw => !whiteKwfilter(kw)); - - for (let i = 0; i < strategiesLen; i++) { - const strategy = this.strategies[i]; - if (whitelistedKeywords.length) { - strategy.writeDomainKeywords(this.domainKeywords); - } - - if (this.protocol.size) { - strategy.writeProtocols(this.protocol); - } - } - - if (this.wildcardSet.size) { - this.wildcardSet.forEach((wildcard) => { - // Overlapped w/ DOMAIN-kEYWORD - if (kwfilter(wildcard)) { - return; - } - - for (let i = 0; i < strategiesLen; i++) { - const strategy = this.strategies[i]; - strategy.writeDomainWildcard(wildcard); - } - }); - } - - const sourceIpOrCidr = Array.from(this.sourceIpOrCidr); - - for (let i = 0; i < strategiesLen; i++) { - const strategy = this.strategies[i]; - - if (this.userAgent.size) { - strategy.writeUserAgents(this.userAgent); - } - if (this.processName.size) { - strategy.writeProcessNames(this.processName); - } - if (this.processPath.size) { - strategy.writeProcessPaths(this.processPath); - } - - if (this.sourceIpOrCidr.size) { - strategy.writeSourceIpCidrs(sourceIpOrCidr); - } - - if (this.sourcePort.size) { - strategy.writeSourcePorts(this.sourcePort); - } - if (this.destPort.size) { - strategy.writeDestinationPorts(this.destPort); - } - if (this.otherRules.length) { - strategy.writeOtherRules(this.otherRules); - } - if (this.urlRegex.size) { - strategy.writeUrlRegexes(this.urlRegex); - } - } - - let ipcidr: string[] | null = null; - let ipcidrNoResolve: string[] | null = null; - let ipcidr6: string[] | null = null; - let ipcidr6NoResolve: string[] | null = null; - - if (this.ipcidr.size) { - ipcidr = mergeCidr(Array.from(this.ipcidr), true); - } - if (this.ipcidrNoResolve.size) { - ipcidrNoResolve = mergeCidr(Array.from(this.ipcidrNoResolve), true); - } - if (this.ipcidr6.size) { - ipcidr6 = Array.from(this.ipcidr6); - } - if (this.ipcidr6NoResolve.size) { - ipcidr6NoResolve = Array.from(this.ipcidr6NoResolve); - } - - for (let i = 0; i < strategiesLen; i++) { - const strategy = this.strategies[i]; - // no-resolve - if (ipcidrNoResolve) { - strategy.writeIpCidrs(ipcidrNoResolve, true); - } - if (ipcidr6NoResolve) { - strategy.writeIpCidr6s(ipcidr6NoResolve, true); - } - if (this.ipasnNoResolve.size) { - strategy.writeIpAsns(this.ipasnNoResolve, true); - } - if (this.groipNoResolve.size) { - strategy.writeGeoip(this.groipNoResolve, true); - } - - // triggers DNS resolution - if (ipcidr?.length) { - strategy.writeIpCidrs(ipcidr, false); - } - if (ipcidr6?.length) { - strategy.writeIpCidr6s(ipcidr6, false); - } - if (this.ipasn.size) { - strategy.writeIpAsns(this.ipasn, false); - } - if (this.geoip.size) { - strategy.writeGeoip(this.geoip, false); - } - } + private writeToStrategies(domains: string[]) { + this.guardBeforeWritingToStrategies(); + writeDataToStrategies(this.buildStrategyWriteData(domains), this.strategies); } write(): Promise { return this.span.traceChildAsync('write all', async (childSpan) => { await childSpan.traceChildAsync('done', () => this.done()); - childSpan.traceChildSync('write to strategies', () => this.writeToStrategies()); + const domains = childSpan.traceChildSync('dump domain trie', () => this.dumpDomains()); + + const title = nullthrow(this.title, 'Missing title'); + const descriptions = nullthrow(this.description, 'Missing description'); + + if (this.dataSource.size) { + descriptions.push( + '', + 'This file contains data from:' + ); + appendArrayInPlace(descriptions, Array.from(this.dataSource).sort().map((source) => ` - ${source}`)); + } + + // Big outputs offload everything from punycode to the write onto a worker + // thread: the payload crosses in a few ms (flat string arrays structured-clone + // cheaply), the main-thread event loop stays free, and the worker writes + // synchronously so no completion ever waits on a busy main thread. + // + // Only worth doing from the main thread -- tasks that already run entirely on + // a worker (build-microsoft-cdn, build-telegram-cidr, build-cdn-download-conf) + // are not contending with anything, and must not spawn a nested worker farm. + if (isMainThread && domains.length >= OUTPUT_WORKER_THRESHOLD) { + this.guardBeforeWritingToStrategies(); + + const payload: OutputWorkerPayload = { + id: this.id, + title, + description: descriptions, + dateMs: this.date.getTime(), + strategies: this.strategies.map(serializeStrategy), + data: this.buildStrategyWriteData(domains) + }; + + return childSpan.traceWorkerChild( + 'write via output worker', + rawSpan => getOutputWorkerFarm().writeOutput(rawSpan, payload) + ); + } + + childSpan.traceChildSync('write to strategies', () => this.writeToStrategies(domains)); return childSpan.traceChildAsync('output to disk', (childSpan) => { const promises: Array> = []; - const descriptions = nullthrow(this.description, 'Missing description'); - - if (this.dataSource.size) { - descriptions.push( - '', - 'This file contains data from:' - ); - appendArrayInPlace(descriptions, Array.from(this.dataSource).sort().map((source) => ` - ${source}`)); - } - for (let i = 0, len = this.strategies.length; i < len; i++) { const strategy = this.strategies[i]; - - const basename = (strategy.overwriteFilename || this.id) + '.' + strategy.fileExtension; + const filePath = resolveStrategyOutputPath(strategy, this.id); promises.push( - childSpan.traceChildAsync('write ' + strategy.name, (childSpan) => Promise.resolve(strategy.output( - childSpan, - nullthrow(this.title, 'Missing title'), - descriptions, - this.date, - path.join( - strategy.outputDir, - strategy.type - ? path.join(strategy.type, basename) - : basename - ) - ))) + childSpan.traceChildAsync('write ' + strategy.name, (childSpan) => Promise.resolve( + // Already off the main thread: block on the write instead of handing + // the completion back through libuv. + isMainThread + ? strategy.output(childSpan, title, descriptions, this.date, filePath) + : strategy.outputInWorker(childSpan, title, descriptions, this.date, filePath) + )) ); } @@ -602,7 +532,7 @@ export class FileOutput { async compile(): Promise> { await this.done(); - this.writeToStrategies(); + this.writeToStrategies(this.dumpDomains()); return this.strategies.reduce>((acc, strategy) => { acc.push(strategy.content); diff --git a/Build/lib/rules/output-worker-farm.ts b/Build/lib/rules/output-worker-farm.ts new file mode 100644 index 00000000..de9ba421 --- /dev/null +++ b/Build/lib/rules/output-worker-farm.ts @@ -0,0 +1,30 @@ +import type { JestWorkerFarm } from 'jest-worker'; +import { createWorker } from '../worker'; + +type OutputWorkerModule = typeof import('./output.worker'); +type OutputWorkerFarm = JestWorkerFarm>; + +let farm: OutputWorkerFarm | null = null; + +/** + * Lazily boot the output worker farm. Only FileOutput#write dispatches here, and + * only when an output crosses the offload threshold -- today that is exclusively + * the reject domainsets / adguardhome outputs of build-reject-domainset. + * + * IMPORTANT: whoever triggers the lazy boot is responsible for a matching + * endOutputWorkerFarm() (idempotent, safe to call unconditionally), otherwise the + * worker threads keep a standalone task run alive. + */ +export function getOutputWorkerFarm(): OutputWorkerFarm { + // 3 workers: reject, reject_extra and reject_phishing format & write in parallel + farm ??= createWorker(require.resolve('./output.worker'), 3)(['writeOutput']); + return farm; +} + +export async function endOutputWorkerFarm(): Promise { + if (farm) { + const f = farm; + farm = null; + await f.end(); + } +} diff --git a/Build/lib/rules/output.worker.ts b/Build/lib/rules/output.worker.ts new file mode 100644 index 00000000..bc0aa147 --- /dev/null +++ b/Build/lib/rules/output.worker.ts @@ -0,0 +1,37 @@ +import { workerJob } from '../../trace'; +import type { RawSpan, WorkerJobResult } from '../../trace'; +import { resolveStrategyOutputPath, reviveStrategy, writeDataToStrategies } from './strategy-write-data'; +import type { OutputWorkerPayload } from './strategy-write-data'; + +/** + * The off-main-thread half of FileOutput#write. Everything from "raw collections" + * to "bytes on disk" happens here: punycode, keyword filtering, sorting, the + * per-strategy format fanout (incl. the sing-box JSON stringify), banner and + * content hash, compare and write. All fs calls are synchronous on purpose -- + * blocking this worker thread is free, and it avoids bouncing libuv completions + * off a busy main-thread event loop. + */ +export function writeOutput(rawSpan: RawSpan | undefined, payload: OutputWorkerPayload): Promise> { + return workerJob(rawSpan, (span) => { + const strategies = payload.strategies.map(reviveStrategy); + + span.traceChildSync('write to strategies', () => writeDataToStrategies(payload.data, strategies)); + + const date = new Date(payload.dateMs); + + return span.traceChildAsync('output to disk', async (childSpan) => { + // Sequential on purpose: the writes are synchronous, so there is nothing to + // overlap, and this keeps the emitted trace in strategy order. + for (let i = 0, len = strategies.length; i < len; i++) { + const strategy = strategies[i]; + const filePath = resolveStrategyOutputPath(strategy, payload.id); + + // eslint-disable-next-line no-await-in-loop -- see above + await childSpan.traceChildAsync( + 'write ' + strategy.name, + (strategySpan) => strategy.outputInWorker(strategySpan, payload.title, payload.description, date, filePath) + ); + } + }); + }); +} diff --git a/Build/lib/rules/strategy-write-data.ts b/Build/lib/rules/strategy-write-data.ts new file mode 100644 index 00000000..2aea984c --- /dev/null +++ b/Build/lib/rules/strategy-write-data.ts @@ -0,0 +1,265 @@ +import path from 'node:path'; +import { domainToASCII } from 'node:url'; +import { merge as mergeCidr } from 'fast-cidr-tools'; +import { createRetrieKeywordFilter as createKeywordFilter } from 'foxts/retrie'; + +import type { BaseWriteStrategy } from '../writing-strategy/base'; +import { AdGuardHome } from '../writing-strategy/adguardhome'; +import { ClashClassicRuleSet, ClashDomainSet, ClashIPSet } from '../writing-strategy/clash'; +import { LegacyClashPremiumClassicRuleSet } from '../writing-strategy/legacy-clash-premium'; +import { SingboxSource } from '../writing-strategy/singbox'; +import { SurfboardRuleSet } from '../writing-strategy/surfboard'; +import { SurgeDomainSet, SurgeMitmSgmodule, SurgeRuleSet } from '../writing-strategy/surge'; + +/** + * A structured-clone friendly snapshot of everything FileOutput feeds into its + * strategies. Only `domains` is big (the '.'-prefixed trie dump, still unicode); + * everything else is at most a few thousand short strings. Plain arrays are used + * (not Set) so the contract survives any worker serialization mode. + */ +export interface StrategyWriteData { + /** Raw trie dump, '.'-prefix means include-all-subdomain, NOT yet punycoded */ + domains: string[], + domainKeywords: string[], + whitelistKeywords: string[], + wildcards: string[], + userAgents: string[], + processNames: string[], + processPaths: string[], + urlRegexes: string[], + ipcidr: string[], + ipcidrNoResolve: string[], + ipcidr6: string[], + ipcidr6NoResolve: string[], + ipasn: string[], + ipasnNoResolve: string[], + geoip: string[], + geoipNoResolve: string[], + sourceIpOrCidr: string[], + sourcePort: string[], + destPort: string[], + protocol: string[], + otherRules: string[] +} + +/** + * Write a StrategyWriteData snapshot into strategies. This is the extracted body + * of FileOutput#writeToStrategies and MUST keep the exact same write order, since + * the order of writeXxx calls determines the line order of the final output. + */ +export function writeDataToStrategies(data: StrategyWriteData, strategies: BaseWriteStrategy[]): void { + // We use both DOMAIN-KEYWORD and whitelisted keyword to whitelist DOMAIN and DOMAIN-SUFFIX + const kwfilter = createKeywordFilter(data.domainKeywords.concat(data.whitelistKeywords)); + + const strategiesLen = strategies.length; + + const domainEntries: Array<[domain: string, subdomain: boolean]> = []; + for (let j = 0, len = data.domains.length; j < len; j++) { + const line = data.domains[j]; + const includeSubdomain = line.codePointAt(0) === 46; /* '.' */ + const d = domainToASCII(includeSubdomain ? line.slice(1) : line); + if (d) domainEntries.push([d, includeSubdomain]); + } + domainEntries.sort((a, b) => (a[0].length - b[0].length) || (a[0] < b[0] ? -1 : (a[0] > b[0] ? 1 : 0))); + for (let j = 0, entriesLen = domainEntries.length; j < entriesLen; j++) { + const [domain, includeAllSubdomain] = domainEntries[j]; + if (kwfilter(domain)) { + continue; + } + + for (let i = 0; i < strategiesLen; i++) { + const strategy = strategies[i]; + if (includeAllSubdomain) { + strategy.writeDomainSuffix(domain); + } else { + strategy.writeDomain(domain); + } + } + } + + // Now, we whitelisted out DOMAIN-KEYWORD + const whiteKwfilter = createKeywordFilter(data.whitelistKeywords); + const whitelistedKeywords = data.domainKeywords.filter(kw => !whiteKwfilter(kw)); + + const domainKeywords = new Set(data.domainKeywords); + const protocol = new Set(data.protocol); + + for (let i = 0; i < strategiesLen; i++) { + const strategy = strategies[i]; + if (whitelistedKeywords.length) { + strategy.writeDomainKeywords(domainKeywords); + } + + if (protocol.size) { + strategy.writeProtocols(protocol); + } + } + + if (data.wildcards.length) { + data.wildcards.forEach((wildcard) => { + // Overlapped w/ DOMAIN-kEYWORD + if (kwfilter(wildcard)) { + return; + } + + for (let i = 0; i < strategiesLen; i++) { + const strategy = strategies[i]; + strategy.writeDomainWildcard(wildcard); + } + }); + } + + const userAgent = new Set(data.userAgents); + const processName = new Set(data.processNames); + const processPath = new Set(data.processPaths); + const sourcePort = new Set(data.sourcePort); + const destPort = new Set(data.destPort); + const urlRegex = new Set(data.urlRegexes); + + for (let i = 0; i < strategiesLen; i++) { + const strategy = strategies[i]; + + if (userAgent.size) { + strategy.writeUserAgents(userAgent); + } + if (processName.size) { + strategy.writeProcessNames(processName); + } + if (processPath.size) { + strategy.writeProcessPaths(processPath); + } + + if (data.sourceIpOrCidr.length) { + strategy.writeSourceIpCidrs(data.sourceIpOrCidr); + } + + if (sourcePort.size) { + strategy.writeSourcePorts(sourcePort); + } + if (destPort.size) { + strategy.writeDestinationPorts(destPort); + } + if (data.otherRules.length) { + strategy.writeOtherRules(data.otherRules); + } + if (urlRegex.size) { + strategy.writeUrlRegexes(urlRegex); + } + } + + let ipcidr: string[] | null = null; + let ipcidrNoResolve: string[] | null = null; + let ipcidr6: string[] | null = null; + let ipcidr6NoResolve: string[] | null = null; + + if (data.ipcidr.length) { + ipcidr = mergeCidr(data.ipcidr, true); + } + if (data.ipcidrNoResolve.length) { + ipcidrNoResolve = mergeCidr(data.ipcidrNoResolve, true); + } + if (data.ipcidr6.length) { + ipcidr6 = data.ipcidr6; + } + if (data.ipcidr6NoResolve.length) { + ipcidr6NoResolve = data.ipcidr6NoResolve; + } + + const ipasn = new Set(data.ipasn); + const ipasnNoResolve = new Set(data.ipasnNoResolve); + const geoip = new Set(data.geoip); + const geoipNoResolve = new Set(data.geoipNoResolve); + + for (let i = 0; i < strategiesLen; i++) { + const strategy = strategies[i]; + // no-resolve + if (ipcidrNoResolve) { + strategy.writeIpCidrs(ipcidrNoResolve, true); + } + if (ipcidr6NoResolve) { + strategy.writeIpCidr6s(ipcidr6NoResolve, true); + } + if (ipasnNoResolve.size) { + strategy.writeIpAsns(ipasnNoResolve, true); + } + if (geoipNoResolve.size) { + strategy.writeGeoip(geoipNoResolve, true); + } + + // triggers DNS resolution + if (ipcidr?.length) { + strategy.writeIpCidrs(ipcidr, false); + } + if (ipcidr6?.length) { + strategy.writeIpCidr6s(ipcidr6, false); + } + if (ipasn.size) { + strategy.writeIpAsns(ipasn, false); + } + if (geoip.size) { + strategy.writeGeoip(geoip, false); + } + } +} + +/** Where a strategy's output for a given output id lands on disk */ +export function resolveStrategyOutputPath(strategy: BaseWriteStrategy, id: string): string { + const basename = (strategy.overwriteFilename || id) + '.' + strategy.fileExtension; + return path.join( + strategy.outputDir, + strategy.type + ? path.join(strategy.type, basename) + : basename + ); +} + +/** Serializable stand-in for a strategy class instance crossing the worker boundary */ +export interface StrategyDescriptor { + name: string, + type: string, + outputDir: string, + overwriteFilename: string | null +} + +export function serializeStrategy(strategy: BaseWriteStrategy): StrategyDescriptor { + return { + name: strategy.name, + type: strategy.type, + outputDir: strategy.outputDir, + overwriteFilename: strategy.overwriteFilename + }; +} + +const strategyRegistry: Record BaseWriteStrategy> = { + 'surge domainset': (d) => new SurgeDomainSet(d.outputDir), + 'surge ruleset': (d) => new SurgeRuleSet(d.type, d.outputDir), + 'surge sgmodule': (d) => new SurgeMitmSgmodule(d.overwriteFilename ?? '', d.outputDir), + 'clash domainset': (d) => new ClashDomainSet(d.outputDir), + 'clash ipcidr': (d) => new ClashIPSet(d.outputDir), + 'clash classic ruleset': (d) => new ClashClassicRuleSet(d.type, d.outputDir), + 'legacy clash premium classic ruleset': (d) => new LegacyClashPremiumClassicRuleSet(d.type as 'ip' | 'non_ip', d.outputDir), + 'surfboard for android ruleset': (d) => new SurfboardRuleSet(d.type as 'ip' | 'non_ip', d.outputDir), + singbox: (d) => new SingboxSource(d.type as 'domainset' | 'non_ip' | 'ip', d.outputDir), + adguardhome: (d) => new AdGuardHome(d.outputDir) +}; + +export function reviveStrategy(descriptor: StrategyDescriptor): BaseWriteStrategy { + if (!(descriptor.name in strategyRegistry)) { + throw new TypeError(`Unknown strategy "${descriptor.name}", add it to strategyRegistry before offloading it to the output worker`); + } + const strategy = strategyRegistry[descriptor.name](descriptor); + if (descriptor.overwriteFilename) { + strategy.withFilename(descriptor.overwriteFilename); + } + return strategy; +} + +/** The payload of a single FileOutput#write offloaded to the output worker */ +export interface OutputWorkerPayload { + id: string, + title: string, + description: string[], + dateMs: number, + strategies: StrategyDescriptor[], + data: StrategyWriteData +} diff --git a/Build/lib/writing-strategy/base.ts b/Build/lib/writing-strategy/base.ts index d6e4503a..905c56af 100644 --- a/Build/lib/writing-strategy/base.ts +++ b/Build/lib/writing-strategy/base.ts @@ -1,7 +1,7 @@ import { isCI } from 'ci-info'; import type { Span } from '../../trace'; import { calculateContentHash } from '../content-hash'; -import { compareAndWriteFile, writeFileLines } from '../create-file'; +import { compareAndWriteFile, compareAndWriteFileInWorker, writeFileLines, writeFileLinesSync } from '../create-file'; /** * The class is not about holding rule data, instead it determines how the @@ -98,6 +98,47 @@ export abstract class BaseWriteStrategy { ); }; + /** + * Worker-thread twin of {@link output}: identical comparison, but the write is + * synchronous since blocking a dedicated worker costs nothing. + */ + public async outputInWorker( + span: Span, + title: string, + description: string[] | readonly string[], + date: Date, + filePath: string + ): Promise { + const result = this.result; + if (!result) { + return; + } + + if (isCI && this.skipCompareOnCI) { + writeFileLinesSync( + span, + this.withPadding(title, description, date, result, null), + filePath + ); + return; + } + + const contentHash = calculateContentHash(title, description, result); + + await compareAndWriteFileInWorker( + span, + this.withPadding( + title, + description, + date, + result, + contentHash + ), + filePath, + contentHash + ); + } + public get content() { return this.result; }