GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2025-12-12 09:10:35 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Perf: improve reject set dedupe performance

Browse Source
This commit is contained in:
SukkaW 2022-08-31 02:25:21 +08:00
parent 69c196c8a5
commit 39f3dacf6e
2 changed files with 45 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
Build/build-reject-domainset.js
View File

@ -148,24 +148,56 @@ const threads = require('os').cpus().length - 1;
// Dedupe domainSets // Dedupe domainSets
console.log(`Start deduping! (${previousSize})`); console.log(`Start deduping! (${previousSize})`);
const toBeRemoved = new Set();
for (const domain of domainSets) {
let isTobeRemoved = false;
for (const keyword of domainKeywordsSet) {
if (domain.includes(keyword) || keyword.includes(domain)) {
isTobeRemoved = true;
break;
}
}
if (!isTobeRemoved) {
for (const suffix of domainSuffixSet) {
if (domain.endsWith(suffix)) {
isTobeRemoved = true;
break;
}
}
}
if (!isTobeRemoved) {
for (const white of filterRuleWhitelistDomainSets) {
if (domain.includes(white) || white.includes(domain)) {
isTobeRemoved = true;
break;
}
}
}
if (isTobeRemoved) {
toBeRemoved.add(domain);
}
}
toBeRemoved.forEach((removed) => {
domainSets.delete(removed)
});
// Dedupe domainSets
console.log(`Deduped ${previousSize - domainSets.size} from black keywords and suffixes!`);
previousSize = domainSets.size;
// Dedupe domainSets
console.log(`Start deduping! (${previousSize})`);
const piscina = new Piscina({ const piscina = new Piscina({
filename: pathResolve(__dirname, 'worker/build-reject-domainset-worker.js'), filename: pathResolve(__dirname, 'worker/build-reject-domainset-worker.js'),
workerData: [...domainSets] workerData: [...domainSets]
}); });
(await Promise.all([
piscina.run(
{ keywords: domainKeywordsSet, suffixes: domainSuffixSet },
{ name: 'dedupeKeywords' }
),
piscina.run(
{ whiteList: filterRuleWhitelistDomainSets },
{ name: 'whitelisted' }
)
])).forEach(set => {
set.forEach(i => domainSets.delete(i));
});
(await Promise.all( (await Promise.all(
Array.from(domainSets) Array.from(domainSets)
.reduce((result, element, index) => { .reduce((result, element, index) => {

36
Build/worker/build-reject-domainset-worker.js
View File

@ -43,39 +43,3 @@ exports.dedupe = ({ chunk }) => {
return outputToBeRemoved; return outputToBeRemoved;
}; };
exports.whitelisted = ({ whiteList }) => {
const outputToBeRemoved = new Set();
for (const domain of workerData) {
for (const white of whiteList) {
if (domain.includes(white) || white.includes(domain)) {
outputToBeRemoved.add(domain);
break;
}
}
}
return outputToBeRemoved;
};
exports.dedupeKeywords = ({ keywords, suffixes }) => {
const outputToBeRemoved = new Set();
for (const domain of workerData) {
for (const keyword of keywords) {
if (domain.includes(keyword) || keyword.includes(domain)) {
outputToBeRemoved.add(domain);
break;
}
}
for (const suffix of suffixes) {
if (domain.endsWith(suffix)) {
outputToBeRemoved.add(domain);
break;
}
}
}
return outputToBeRemoved;
}