diff --git a/Build/build-reject-domainset.js b/Build/build-reject-domainset.js index 84111f56..f986fa8b 100644 --- a/Build/build-reject-domainset.js +++ b/Build/build-reject-domainset.js @@ -352,11 +352,11 @@ const threads = isCI ? cpuCount : cpuCount / 2; ) ).forEach((result, taskIndex) => { const chunk = tasksArray[taskIndex]; - result.forEach((value, index) => { - if (value === 1) { - domainSets.delete(chunk[index]) + for (let i = 0, len = result.length; i < len; i++) { + if (result[i]) { + domainSets.delete(chunk[i]); } - }) + } }); console.log(`* Dedupe from covered subdomain - ${(Date.now() - START_TIME) / 1000}s`); diff --git a/Build/lib/parse-filter.js b/Build/lib/parse-filter.js index 1ef5ac33..98d7e3da 100644 --- a/Build/lib/parse-filter.js +++ b/Build/lib/parse-filter.js @@ -227,25 +227,11 @@ async function processFilterRules (filterRulesUrl, fallbackUrls) { function preprocessFullDomainSetBeforeUsedAsWorkerData (data) { return data.filter(domain => ( domain.charCodeAt(0) === 46 - && !canExcludeFromDedupe(domain) )); } -// duckdns.org domain will not overlap and doesn't need dedupe -function canExcludeFromDedupe (domain) { - if ( - // starts with a dot - domain.charCodeAt(0) === 46 - && domain.length === 23 - && domain.endsWith('.duckdns.org') - ) { - return true; - } - return false; -} module.exports.processDomainLists = processDomainLists; module.exports.processHosts = processHosts; module.exports.processFilterRules = processFilterRules; module.exports.preprocessFullDomainSetBeforeUsedAsWorkerData = preprocessFullDomainSetBeforeUsedAsWorkerData; -module.exports.canExcludeFromDedupe = canExcludeFromDedupe; diff --git a/Build/worker/build-reject-domainset-worker.js b/Build/worker/build-reject-domainset-worker.js index 9c2849f5..8377a8fa 100644 --- a/Build/worker/build-reject-domainset-worker.js +++ b/Build/worker/build-reject-domainset-worker.js @@ -1,5 +1,4 @@ const Piscina = require('piscina'); -const { canExcludeFromDedupe } = require('../lib/parse-filter') const fullsetDomainStartsWithADot = Piscina.workerData const totalLen = fullsetDomainStartsWithADot.length; @@ -11,13 +10,9 @@ module.exports.dedupe = ({ chunk }) => { for (let i = 0; i < chunkLength; i++) { const domainFromInput = chunk[i]; - if (canExcludeFromDedupe(domainFromInput)) { - continue; - } - for (let j = 0; j < totalLen; j++) { const domainFromFullSet = fullsetDomainStartsWithADot[j]; - // domainFromFullSet is now startsWith a "." + // domainFromFullSet is always startsWith "." if (domainFromFullSet === domainFromInput) continue;