Chore: update reject toolchain

Build/build-reject-domainset.js

@@ -2,7 +2,7 @@ const { promises: fsPromises } = require('fs');
 const { resolve: pathResolve } = require('path');
 const Piscina = require('piscina');
 const { processHosts, processFilterRules } = require('./lib/parse-filter');
-const threads = Math.max(require('os').cpus().length, 12);
+const threads = require('os').cpus().length - 1;
 
 (async () => {
   /** @type Set<string> */
@@ -144,24 +144,23 @@ const threads = Math.max(require('os').cpus().length, 12);
   console.log(`Start deduping! (${beforeDeduping})`);
 
   const piscina = new Piscina({
-    filename: pathResolve(__dirname, 'worker/build-reject-domainset-worker.js')
+    filename: pathResolve(__dirname, 'worker/build-reject-domainset-worker.js'),
+    workerData: domainSets
   });
 
   (await Promise.all([
     piscina.run(
-      { keywords: domainKeywordsSet, suffixes: domainSuffixSet, input: domainSets },
+      { keywords: domainKeywordsSet, suffixes: domainSuffixSet },
       { name: 'dedupeKeywords' }
     ),
     piscina.run(
-      { whiteList: filterRuleWhitelistDomainSets, input: domainSets },
+      { whiteList: filterRuleWhitelistDomainSets },
       { name: 'whitelisted' }
     )
   ])).forEach(set => {
     set.forEach(i => domainSets.delete(i));
   });
 
-  const originalFullSet = new Set([...domainSets]);
-
   (await Promise.all(
     Array.from(domainSets)
       .reduce((result, element, index) => {
@@ -172,7 +171,7 @@ const threads = Math.max(require('os').cpus().length, 12);
         return result;
       }, [])
       .map(chunk => piscina.run(
-        { input: chunk, fullSet: originalFullSet },
+        { chunk },
         { name: 'dedupe' }
       ))
   )).forEach(set => {

Build/build-telegram-cidr.js

@@ -16,7 +16,7 @@ const { isIPv4, isIPv6 } = require('net');
     '# Telegram CIDR (https://core.telegram.org/resources/cidr.txt)' + '\n' +
     '# Last Updated: ' + lastModified.toISOString() + '\n' +
     res.map(ip => {
-      const [subnet, range] = ip.split('/');
+      const [subnet] = ip.split('/');
       if (isIPv4(subnet)) {
         return `IP-CIDR,${ip},no-resolve`;
       }

Build/worker/build-reject-domainset-worker.js

@@ -1,15 +1,16 @@
-exports.dedupe = ({ fullSet, input }) => {
+const { workerData } = require('piscina');
+
+exports.dedupe = ({ chunk }) => {
   const outputToBeRemoved = new Set();
 
-  for (const domainFromInput of input) {
+  for (const domainFromInput of chunk) {
-    for (const domainFromFullSet of fullSet) {
+    for (const domainFromFullSet of workerData) {
+      if (domainFromFullSet === domainFromInput) continue;
+      if (domainFromFullSet.charAt(0) !== '.') continue;
+
       if (
-        domainFromFullSet.startsWith('.')
+        `.${domainFromInput}` === domainFromFullSet
-        && domainFromFullSet !== domainFromInput
+        || domainFromInput.endsWith(domainFromFullSet)
-        && (
-          domainFromInput.endsWith(domainFromFullSet)
-          || `.${domainFromInput}` === domainFromFullSet
-        )
       ) {
         outputToBeRemoved.add(domainFromInput);
         break;
@@ -20,10 +21,10 @@ exports.dedupe = ({ fullSet, input }) => {
   return outputToBeRemoved;
 };
 
-exports.whitelisted = ({ whiteList, input }) => {
+exports.whitelisted = ({ whiteList }) => {
   const outputToBeRemoved = new Set();
 
-  for (const domain of input) {
+  for (const domain of workerData) {
     for (const white of whiteList) {
       if (domain.includes(white) || white.includes(domain)) {
         outputToBeRemoved.add(domain);
@@ -35,10 +36,10 @@ exports.whitelisted = ({ whiteList, input }) => {
   return outputToBeRemoved;
 };
 
-exports.dedupeKeywords = ({ keywords, suffixes, input }) => {
+exports.dedupeKeywords = ({ keywords, suffixes }) => {
   const outputToBeRemoved = new Set();
 
-  for (const domain of input) {
+  for (const domain of workerData) {
     for (const keyword of keywords) {
       if (domain.includes(keyword) || keyword.includes(domain)) {
         outputToBeRemoved.add(domain);