GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2025-12-12 01:00:34 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Perf: faster reject domainset build

Browse Source
This commit is contained in:
SukkaW 2022-11-05 17:16:10 +08:00
parent 94db04effc
commit e1b57dd48e
3 changed files with 5 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Build/build-reject-domainset.js
View File

@ -352,11 +352,11 @@ const threads = isCI ? cpuCount : cpuCount / 2;
) )
).forEach((result, taskIndex) => { ).forEach((result, taskIndex) => {
const chunk = tasksArray[taskIndex]; const chunk = tasksArray[taskIndex];
result.forEach((value, index) => { for (let i = 0, len = result.length; i < len; i++) {
if (value === 1) { if (result[i]) {
domainSets.delete(chunk[index]) domainSets.delete(chunk[i]);
} }
}) }
}); });
console.log(`* Dedupe from covered subdomain - ${(Date.now() - START_TIME) / 1000}s`); console.log(`* Dedupe from covered subdomain - ${(Date.now() - START_TIME) / 1000}s`);

14
Build/lib/parse-filter.js
View File

@ -227,25 +227,11 @@ async function processFilterRules (filterRulesUrl, fallbackUrls) {
function preprocessFullDomainSetBeforeUsedAsWorkerData (data) { function preprocessFullDomainSetBeforeUsedAsWorkerData (data) {
return data.filter(domain => ( return data.filter(domain => (
domain.charCodeAt(0) === 46 domain.charCodeAt(0) === 46
&& !canExcludeFromDedupe(domain)
)); ));
} }
// duckdns.org domain will not overlap and doesn't need dedupe
function canExcludeFromDedupe (domain) {
if (
// starts with a dot
domain.charCodeAt(0) === 46
&& domain.length === 23
&& domain.endsWith('.duckdns.org')
) {
return true;
}
return false;
}
module.exports.processDomainLists = processDomainLists; module.exports.processDomainLists = processDomainLists;
module.exports.processHosts = processHosts; module.exports.processHosts = processHosts;
module.exports.processFilterRules = processFilterRules; module.exports.processFilterRules = processFilterRules;
module.exports.preprocessFullDomainSetBeforeUsedAsWorkerData = preprocessFullDomainSetBeforeUsedAsWorkerData; module.exports.preprocessFullDomainSetBeforeUsedAsWorkerData = preprocessFullDomainSetBeforeUsedAsWorkerData;
module.exports.canExcludeFromDedupe = canExcludeFromDedupe;

7
Build/worker/build-reject-domainset-worker.js
View File

@ -1,5 +1,4 @@
const Piscina = require('piscina'); const Piscina = require('piscina');
const { canExcludeFromDedupe } = require('../lib/parse-filter')
const fullsetDomainStartsWithADot = Piscina.workerData const fullsetDomainStartsWithADot = Piscina.workerData
const totalLen = fullsetDomainStartsWithADot.length; const totalLen = fullsetDomainStartsWithADot.length;
@ -11,13 +10,9 @@ module.exports.dedupe = ({ chunk }) => {
for (let i = 0; i < chunkLength; i++) { for (let i = 0; i < chunkLength; i++) {
const domainFromInput = chunk[i]; const domainFromInput = chunk[i];
if (canExcludeFromDedupe(domainFromInput)) {
continue;
}
for (let j = 0; j < totalLen; j++) { for (let j = 0; j < totalLen; j++) {
const domainFromFullSet = fullsetDomainStartsWithADot[j]; const domainFromFullSet = fullsetDomainStartsWithADot[j];
// domainFromFullSet is now startsWith a "." // domainFromFullSet is always startsWith "."
if (domainFromFullSet === domainFromInput) continue; if (domainFromFullSet === domainFromInput) continue;