diff --git a/Build/build-reject-domainset.ts b/Build/build-reject-domainset.ts index 9f5d98e3..2c82ca8b 100644 --- a/Build/build-reject-domainset.ts +++ b/Build/build-reject-domainset.ts @@ -76,6 +76,16 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as await span .traceChild('download and process hosts / adblock filter rules') .traceAsyncFn((childSpan) => Promise.all([ + // Dedupe domainSets + // Collect DOMAIN, DOMAIN-SUFFIX, and DOMAIN-KEYWORD from non_ip/reject.conf for deduplication + // DOMAIN-WILDCARD is not really useful for deduplication, it is only included in AdGuardHome output + // It is faster to add base than add others first then whitelist + rejectOutput.addFromRuleset(readLocalRejectRulesetPromise), + rejectExtraOutput.addFromRuleset(readLocalRejectRulesetPromise), + readLocalRejectDomainsetPromise.then(appendArrayToRejectOutput), + readLocalRejectDomainsetPromise.then(appendArrayToRejectExtraOutput), + readLocalRejectExtraDomainsetPromise.then(appendArrayToRejectExtraOutput), + // Parse from remote hosts & domain lists hostsDownloads.map(task => task(childSpan).then(appendArrayToRejectOutput)), hostsExtraDownloads.map(task => task(childSpan).then(appendArrayToRejectExtraOutput)), @@ -109,18 +119,7 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as addArrayElementsToSet(filterRuleWhitelistDomainSets, blackDomainSuffixes, suffix => '.' + suffix); }) ), - getPhishingDomains(childSpan).then(appendArrayToRejectExtraOutput), - readLocalRejectDomainsetPromise.then(appendArrayToRejectOutput), - readLocalRejectDomainsetPromise.then(appendArrayToRejectExtraOutput), - readLocalRejectExtraDomainsetPromise.then(appendArrayToRejectExtraOutput), - // Dedupe domainSets - // span.traceChildAsync('collect black keywords/suffixes', async () => - /** - * Collect DOMAIN, DOMAIN-SUFFIX, and DOMAIN-KEYWORD from non_ip/reject.conf for deduplication - * DOMAIN-WILDCARD is not really useful for deduplication, it is only included in AdGuardHome output - */ - rejectOutput.addFromRuleset(readLocalRejectRulesetPromise), - rejectExtraOutput.addFromRuleset(readLocalRejectRulesetPromise) + getPhishingDomains(childSpan).then(appendArrayToRejectExtraOutput) ].flat())); if (foundDebugDomain.value) { diff --git a/Build/lib/rules/base.ts b/Build/lib/rules/base.ts index c22aa566..ac29964f 100644 --- a/Build/lib/rules/base.ts +++ b/Build/lib/rules/base.ts @@ -124,8 +124,8 @@ export class FileOutput { return this; } - private async addFromDomainsetPromise(source: AsyncIterable | Iterable | string[]) { - for await (const line of source) { + private async addFromDomainsetPromise(source: MaybePromise | Iterable | string[]>) { + for await (const line of await source) { if (line[0] === '.') { this.addDomainSuffix(line, true); } else { @@ -136,23 +136,15 @@ export class FileOutput { addFromDomainset(source: MaybePromise | Iterable | string[]>) { if (this.pendingPromise) { - if ('then' in source) { - this.pendingPromise = this.pendingPromise.then(() => source).then(src => this.addFromDomainsetPromise(src)); - return this; - } this.pendingPromise = this.pendingPromise.then(() => this.addFromDomainsetPromise(source)); return this; } - if ('then' in source) { - this.pendingPromise = source.then(src => this.addFromDomainsetPromise(src)); - return this; - } this.pendingPromise = this.addFromDomainsetPromise(source); return this; } - private async addFromRulesetPromise(source: AsyncIterable | Iterable | string[]) { - for await (const line of source) { + private async addFromRulesetPromise(source: MaybePromise | Iterable | string[]>) { + for await (const line of await source) { const splitted = line.split(','); const type = splitted[0]; const value = splitted[1]; @@ -216,17 +208,9 @@ export class FileOutput { addFromRuleset(source: MaybePromise | Iterable>) { if (this.pendingPromise) { - if ('then' in source) { - this.pendingPromise = this.pendingPromise.then(() => source).then(src => this.addFromRulesetPromise(src)); - return this; - } this.pendingPromise = this.pendingPromise.then(() => this.addFromRulesetPromise(source)); return this; } - if ('then' in source) { - this.pendingPromise = source.then(src => this.addFromRulesetPromise(src)); - return this; - } this.pendingPromise = this.addFromRulesetPromise(source); return this; }