GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2025-12-12 01:00:34 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Perf: add reject base first before adding third-party data

Browse Source
This commit is contained in:
SukkaW 2025-03-07 10:00:25 +08:00
parent 22b1f6f4e1
commit 4b26958578
2 changed files with 15 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
Build/build-reject-domainset.ts
View File

@ -76,6 +76,16 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as
await span
.traceChild('download and process hosts / adblock filter rules')
.traceAsyncFn((childSpan) => Promise.all([
// Dedupe domainSets
// Collect DOMAIN, DOMAIN-SUFFIX, and DOMAIN-KEYWORD from non_ip/reject.conf for deduplication
// DOMAIN-WILDCARD is not really useful for deduplication, it is only included in AdGuardHome output
// It is faster to add base than add others first then whitelist
rejectOutput.addFromRuleset(readLocalRejectRulesetPromise),
rejectExtraOutput.addFromRuleset(readLocalRejectRulesetPromise),
readLocalRejectDomainsetPromise.then(appendArrayToRejectOutput),
readLocalRejectDomainsetPromise.then(appendArrayToRejectExtraOutput),
readLocalRejectExtraDomainsetPromise.then(appendArrayToRejectExtraOutput),
// Parse from remote hosts & domain lists
hostsDownloads.map(task => task(childSpan).then(appendArrayToRejectOutput)),
hostsExtraDownloads.map(task => task(childSpan).then(appendArrayToRejectExtraOutput)),
@ -109,18 +119,7 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as
addArrayElementsToSet(filterRuleWhitelistDomainSets, blackDomainSuffixes, suffix => '.' + suffix);
})
),
getPhishingDomains(childSpan).then(appendArrayToRejectExtraOutput),
readLocalRejectDomainsetPromise.then(appendArrayToRejectOutput),
readLocalRejectDomainsetPromise.then(appendArrayToRejectExtraOutput),
readLocalRejectExtraDomainsetPromise.then(appendArrayToRejectExtraOutput),
// Dedupe domainSets
// span.traceChildAsync('collect black keywords/suffixes', async () =>
/**
* Collect DOMAIN, DOMAIN-SUFFIX, and DOMAIN-KEYWORD from non_ip/reject.conf for deduplication
* DOMAIN-WILDCARD is not really useful for deduplication, it is only included in AdGuardHome output
*/
rejectOutput.addFromRuleset(readLocalRejectRulesetPromise),
rejectExtraOutput.addFromRuleset(readLocalRejectRulesetPromise)
getPhishingDomains(childSpan).then(appendArrayToRejectExtraOutput)
].flat()));
if (foundDebugDomain.value) {

24
Build/lib/rules/base.ts
View File

@ -124,8 +124,8 @@ export class FileOutput {
return this;
}
private async addFromDomainsetPromise(source: AsyncIterable<string> | Iterable<string> | string[]) {
for await (const line of source) {
private async addFromDomainsetPromise(source: MaybePromise<AsyncIterable<string> | Iterable<string> | string[]>) {
for await (const line of await source) {
if (line[0] === '.') {
this.addDomainSuffix(line, true);
} else {
@ -136,23 +136,15 @@ export class FileOutput {
addFromDomainset(source: MaybePromise<AsyncIterable<string> | Iterable<string> | string[]>) {
if (this.pendingPromise) {
if ('then' in source) {
this.pendingPromise = this.pendingPromise.then(() => source).then(src => this.addFromDomainsetPromise(src));
return this;
}
this.pendingPromise = this.pendingPromise.then(() => this.addFromDomainsetPromise(source));
return this;
}
if ('then' in source) {
this.pendingPromise = source.then(src => this.addFromDomainsetPromise(src));
return this;
}
this.pendingPromise = this.addFromDomainsetPromise(source);
return this;
}
private async addFromRulesetPromise(source: AsyncIterable<string> | Iterable<string> | string[]) {
for await (const line of source) {
private async addFromRulesetPromise(source: MaybePromise<AsyncIterable<string> | Iterable<string> | string[]>) {
for await (const line of await source) {
const splitted = line.split(',');
const type = splitted[0];
const value = splitted[1];
@ -216,17 +208,9 @@ export class FileOutput {
addFromRuleset(source: MaybePromise<AsyncIterable<string> | Iterable<string>>) {
if (this.pendingPromise) {
if ('then' in source) {
this.pendingPromise = this.pendingPromise.then(() => source).then(src => this.addFromRulesetPromise(src));
return this;
}
this.pendingPromise = this.pendingPromise.then(() => this.addFromRulesetPromise(source));
return this;
}
if ('then' in source) {
this.pendingPromise = source.then(src => this.addFromRulesetPromise(src));
return this;
}
this.pendingPromise = this.addFromRulesetPromise(source);
return this;
}