diff --git a/Build/build-reject-domainset.js b/Build/build-reject-domainset.js index e62a2792..70dcd90b 100644 --- a/Build/build-reject-domainset.js +++ b/Build/build-reject-domainset.js @@ -2,7 +2,7 @@ const { promises: fsPromises } = require('fs'); const { resolve: pathResolve } = require('path'); const Piscina = require('piscina'); const { processHosts, processFilterRules } = require('./lib/parse-filter'); -const threads = Math.max(require('os').cpus().length, 12); +const threads = require('os').cpus().length - 1; (async () => { /** @type Set */ @@ -144,24 +144,23 @@ const threads = Math.max(require('os').cpus().length, 12); console.log(`Start deduping! (${beforeDeduping})`); const piscina = new Piscina({ - filename: pathResolve(__dirname, 'worker/build-reject-domainset-worker.js') + filename: pathResolve(__dirname, 'worker/build-reject-domainset-worker.js'), + workerData: domainSets }); (await Promise.all([ piscina.run( - { keywords: domainKeywordsSet, suffixes: domainSuffixSet, input: domainSets }, + { keywords: domainKeywordsSet, suffixes: domainSuffixSet }, { name: 'dedupeKeywords' } ), piscina.run( - { whiteList: filterRuleWhitelistDomainSets, input: domainSets }, + { whiteList: filterRuleWhitelistDomainSets }, { name: 'whitelisted' } ) ])).forEach(set => { set.forEach(i => domainSets.delete(i)); }); - const originalFullSet = new Set([...domainSets]); - (await Promise.all( Array.from(domainSets) .reduce((result, element, index) => { @@ -172,7 +171,7 @@ const threads = Math.max(require('os').cpus().length, 12); return result; }, []) .map(chunk => piscina.run( - { input: chunk, fullSet: originalFullSet }, + { chunk }, { name: 'dedupe' } )) )).forEach(set => { diff --git a/Build/build-telegram-cidr.js b/Build/build-telegram-cidr.js index 436a249c..cf54df94 100644 --- a/Build/build-telegram-cidr.js +++ b/Build/build-telegram-cidr.js @@ -16,7 +16,7 @@ const { isIPv4, isIPv6 } = require('net'); '# Telegram CIDR (https://core.telegram.org/resources/cidr.txt)' + '\n' + '# Last Updated: ' + lastModified.toISOString() + '\n' + res.map(ip => { - const [subnet, range] = ip.split('/'); + const [subnet] = ip.split('/'); if (isIPv4(subnet)) { return `IP-CIDR,${ip},no-resolve`; } diff --git a/Build/worker/build-reject-domainset-worker.js b/Build/worker/build-reject-domainset-worker.js index 132cc4b3..a1f47b9c 100644 --- a/Build/worker/build-reject-domainset-worker.js +++ b/Build/worker/build-reject-domainset-worker.js @@ -1,15 +1,16 @@ -exports.dedupe = ({ fullSet, input }) => { +const { workerData } = require('piscina'); + +exports.dedupe = ({ chunk }) => { const outputToBeRemoved = new Set(); - for (const domainFromInput of input) { - for (const domainFromFullSet of fullSet) { + for (const domainFromInput of chunk) { + for (const domainFromFullSet of workerData) { + if (domainFromFullSet === domainFromInput) continue; + if (domainFromFullSet.charAt(0) !== '.') continue; + if ( - domainFromFullSet.startsWith('.') - && domainFromFullSet !== domainFromInput - && ( - domainFromInput.endsWith(domainFromFullSet) - || `.${domainFromInput}` === domainFromFullSet - ) + `.${domainFromInput}` === domainFromFullSet + || domainFromInput.endsWith(domainFromFullSet) ) { outputToBeRemoved.add(domainFromInput); break; @@ -20,10 +21,10 @@ exports.dedupe = ({ fullSet, input }) => { return outputToBeRemoved; }; -exports.whitelisted = ({ whiteList, input }) => { +exports.whitelisted = ({ whiteList }) => { const outputToBeRemoved = new Set(); - for (const domain of input) { + for (const domain of workerData) { for (const white of whiteList) { if (domain.includes(white) || white.includes(domain)) { outputToBeRemoved.add(domain); @@ -35,10 +36,10 @@ exports.whitelisted = ({ whiteList, input }) => { return outputToBeRemoved; }; -exports.dedupeKeywords = ({ keywords, suffixes, input }) => { +exports.dedupeKeywords = ({ keywords, suffixes }) => { const outputToBeRemoved = new Set(); - for (const domain of input) { + for (const domain of workerData) { for (const keyword of keywords) { if (domain.includes(keyword) || keyword.includes(domain)) { outputToBeRemoved.add(domain);