GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2025-12-12 17:20:35 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Perf: avoid set.delete by merge trie creation + kwfilter

Browse Source
This commit is contained in:
SukkaW 2024-06-09 01:05:17 +08:00
parent 479032d2ba
commit 6e9652e32a
1 changed files with 23 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
Build/build-reject-domainset.ts
View File

@ -75,11 +75,10 @@ export const buildRejectDomainSet = task(import.meta.main, import.meta.path)(asy
console.log(`Import ${domainSets.size} rules from Hosts / AdBlock Filter Rules & reject_sukka.conf!`); console.log(`Import ${domainSets.size} rules from Hosts / AdBlock Filter Rules & reject_sukka.conf!`);
// Dedupe domainSets // Dedupe domainSets
await span.traceChildAsync('dedupe from black keywords/suffixes', async (childSpan) => { const domainKeywordsSet = await span.traceChildAsync('collect black keywords/suffixes', async () => {
/** Collect DOMAIN-KEYWORD from non_ip/reject.conf for deduplication */ /** Collect DOMAIN-KEYWORD from non_ip/reject.conf for deduplication */
const domainKeywordsSet = new Set<string>(); const domainKeywordsSet = new Set<string>();
await childSpan.traceChildAsync('collect keywords/suffixes', async () => {
for await (const line of readFileByLine(path.resolve(import.meta.dir, '../Source/non_ip/reject.conf'))) { for await (const line of readFileByLine(path.resolve(import.meta.dir, '../Source/non_ip/reject.conf'))) {
const [type, value] = line.split(','); const [type, value] = line.split(',');
@ -89,22 +88,24 @@ export const buildRejectDomainSet = task(import.meta.main, import.meta.path)(asy
domainSets.add(`.${value.trim()}`); // Add to domainSets for later deduplication domainSets.add(`.${value.trim()}`); // Add to domainSets for later deduplication
} }
} }
return domainKeywordsSet;
}); });
// Perform kwfilter to remove as many domains as possible from domainSets before creating trie const trie = span.traceChildSync('create smol trie while deduping black keywords', () => {
childSpan.traceChildSync('dedupe from black keywords', () => { const trie = createTrie(null, true, true);
const kwfilter = createKeywordFilter(domainKeywordsSet); const kwfilter = createKeywordFilter(domainKeywordsSet);
for (const domain of domainSets) { for (const domain of domainSets) {
// Remove keyword // exclude keyword when creating trie
if (kwfilter(domain)) { if (!kwfilter(domain)) {
domainSets.delete(domain); trie.add(domain);
} }
} }
});
});
const trie = span.traceChildSync('create smol trie', () => createTrie(domainSets, true, true)); return trie;
});
span.traceChildSync('dedupe from white suffixes', () => filterRuleWhitelistDomainSets.forEach(trie.whitelist)); span.traceChildSync('dedupe from white suffixes', () => filterRuleWhitelistDomainSets.forEach(trie.whitelist));