Chore: better logging

Build/build-anti-bogus-domain.js

@@ -4,6 +4,8 @@ const path = require('path');
 const { isIP } = require('net');
 
 (async () => {
+  console.time('Total Time - build-anti-bogus-domain');
+  console.time('* Download bogus-nxdomain-list')
   const res = (await (await fetchWithRetry('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/bogus-nxdomain.china.conf')).text())
     .split('\n')
     .map(line => {
@@ -14,6 +16,7 @@ const { isIP } = require('net');
       return null
     })
     .filter(ip => typeof ip === 'string' && isIP(ip) !== 0);
+  console.timeEnd('* Download bogus-nxdomain-list')
 
   const filePath = path.resolve(__dirname, '../List/ip/reject.conf');
   const content = (await fs.promises.readFile(filePath, 'utf-8'))
@@ -23,4 +26,5 @@ const { isIP } = require('net');
     );
 
   await fs.promises.writeFile(filePath, content, 'utf-8');
+  console.timeEnd('Total Time - build-anti-bogus-domain');
 })();

Build/build-apple-cdn.js

@@ -5,6 +5,8 @@ const path = require('path');
 const rDomain = /^(((?!\-))(xn\-\-)?[a-z0-9\-_]{0,61}[a-z0-9]{1,1}\.)*(xn\-\-)?([a-z0-9\-]{1,61}|[a-z0-9\-]{1,30})\.[a-z]{2,}$/m;
 
 (async () => {
+  console.time('Total Time - build-apple-cdn-conf');
+
   const res = (await (await fetchWithRetry('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf')).text())
     .split('\n')
     .map(line => {
@@ -28,4 +30,6 @@ const rDomain = /^(((?!\-))(xn\-\-)?[a-z0-9\-_]{0,61}[a-z0-9]{1,1}\.)*(xn\-\-)?(
       'utf-8'
     )
   ])
+
+  console.timeEnd('Total Time - build-apple-cdn-conf');
 })();

Build/build-cdn-conf.js

@@ -3,6 +3,8 @@ const fs = require('fs');
 const path = require('path');
 
 (async () => {
+  console.time('Total Time - build-cdn-conf');
+
   const domains = (await (await fetchWithRetry('https://publicsuffix.org/list/public_suffix_list.dat')).text()).split('\n');
 
   const S3OSSDomains = domains.filter(line => {
@@ -29,4 +31,6 @@ const path = require('path');
     );
 
   await fs.promises.writeFile(filePath, content, 'utf-8');
+
+  console.timeEnd('Total Time - build-cdn-conf');
 })();

Build/build-cidr.js

@@ -3,6 +3,8 @@ const { promises: fsPromises } = require('fs');
 const { resolve: pathResolve } = require('path');
 
 (async () => {
+  console.time('Total Time - build-chnroutes-cidr');
+
   const cidr = (await (await fetchWithRetry('https://raw.githubusercontent.com/misakaio/chnroutes2/master/chnroutes.txt')).text()).split('\n');
 
   const filteredCidr = cidr.filter(line => {
@@ -13,7 +15,9 @@ const { resolve: pathResolve } = require('path');
     return false;
   })
 
-  return fsPromises.writeFile(pathResolve(__dirname, '../List/ip/china_ip.conf'), makeCidrList(filteredCidr), { encoding: 'utf-8' });
+  await fsPromises.writeFile(pathResolve(__dirname, '../List/ip/china_ip.conf'), makeCidrList(filteredCidr), { encoding: 'utf-8' });
+
+  console.timeEnd('Total Time - build-chnroutes-cidr');
 })();
 
 function makeCidrList(cidr) {

Build/build-reject-domainset.js

@@ -7,9 +7,13 @@ const { isCI } = require('ci-info');
 const threads = isCI ? cpuCount : cpuCount / 2;
 
 (async () => {
+  console.time('Total Time - build-reject-domain-set');
+
   /** @type Set<string> */
   const domainSets = new Set();
 
+  console.time('* Download and process Hosts');
+
   // Parse from remote hosts & domain lists
   (await Promise.all([
     processHosts('https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=0&mimetype=plaintext', true),
@@ -23,6 +27,8 @@ const threads = isCI ? cpuCount : cpuCount / 2;
     });
   });
 
+  console.timeEnd('* Download and process Hosts');
+
   let previousSize = domainSets.size;
   console.log(`Import ${previousSize} rules from hosts files!`);
 
@@ -83,6 +89,7 @@ const threads = isCI ? cpuCount : cpuCount / 2;
     'storage.yandexcloud.net' // phishing list
   ]);
 
+  console.time('* Download and process AdBlock Filter Rules');
   (await Promise.all([
     // Easy List
     [
@@ -147,6 +154,8 @@ const threads = isCI ? cpuCount : cpuCount / 2;
     black.forEach(i => domainSets.add(i));
   });
 
+  console.timeEnd('* Download and process AdBlock Filter Rules');
+
   previousSize = domainSets.size - previousSize;
   console.log(`Import ${previousSize} rules from adguard filters!`);
 
@@ -169,7 +178,8 @@ const threads = isCI ? cpuCount : cpuCount / 2;
 
   previousSize = domainSets.size;
   // Dedupe domainSets
-  console.log(`Start deduping! (${previousSize})`);
+  console.log(`Start deduping from black keywords/suffixes! (${previousSize})`);
+  console.time(`* Dedupe from black keywords/suffixes`);
 
   const toBeRemoved = new Set();
   for (const domain of domainSets) {
@@ -209,13 +219,15 @@ const threads = isCI ? cpuCount : cpuCount / 2;
     domainSets.delete(removed)
   });
 
-  // Dedupe domainSets
+  console.timeEnd(`* Dedupe from black keywords/suffixes`);
-  console.log(`Deduped ${previousSize - domainSets.size} from black keywords and suffixes!`);
+  console.log(`Deduped ${previousSize} - ${domainSets.size} = ${previousSize - domainSets.size} from black keywords and suffixes!`);
 
   previousSize = domainSets.size;
   // Dedupe domainSets
   console.log(`Start deduping! (${previousSize})`);
 
+  const START_TIME = Date.now();
+
   const piscina = new Piscina({
     filename: pathResolve(__dirname, 'worker/build-reject-domainset-worker.js'),
     workerData: [...domainSets]
@@ -253,10 +265,13 @@ const threads = isCI ? cpuCount : cpuCount / 2;
     })
   });
 
+  console.log(`* Dedupe from covered subdomain - ${(Date.now() - START_TIME) / 1000}s`);
   console.log(`Deduped ${previousSize - domainSets.size} rules!`);
 
-  return fsPromises.writeFile(
+  await fsPromises.writeFile(
     pathResolve(__dirname, '../List/domainset/reject.conf'),
     `${[...domainSets].join('\n')}\n`,
     { encoding: 'utf-8' });
+
+  console.timeEnd('Total Time - build-reject-domain-set');
 })();

Build/build-telegram-cidr.js

@@ -4,6 +4,8 @@ const path = require('path');
 const { isIPv4, isIPv6 } = require('net');
 
 (async () => {
+  console.time('Total Time - build-telegram-cidr');
+
   const resp = await fetchWithRetry('https://core.telegram.org/resources/cidr.txt');
   const lastModified = new Date(resp.headers.get('last-modified'));
 
@@ -27,4 +29,6 @@ const { isIPv4, isIPv6 } = require('net');
     }).join('\n') + '\n',
     'utf-8'
   );
+
+  console.timeEnd('Total Time - build-telegram-cidr');
 })();

Build/lib/parse-filter.js

@@ -96,7 +96,7 @@ async function processHosts (hostsUrl, includeAllSubDomain = false) {
  * @returns {Promise<{ white: Set<string>, black: Set<string> }>}
  */
 async function processFilterRules (filterRulesUrl, fallbackUrls) {
-  console.time(`processFilterRules: ${filterRulesUrl}`);
+  console.time(`   - processFilterRules: ${filterRulesUrl}`);
 
   if (typeof filterRulesUrl === 'string') {
     filterRulesUrl = new URL(filterRulesUrl);
@@ -206,7 +206,7 @@ async function processFilterRules (filterRulesUrl, fallbackUrls) {
     }
   });
 
-  console.timeEnd(`processFilterRules: ${filterRulesUrl}`);
+  console.timeEnd(`   - processFilterRules: ${filterRulesUrl}`);
 
   return {
     white: whitelistDomainSets,

package.json

@@ -58,7 +58,7 @@
   "dependencies": {
     "@sukka/listdir": "^0.2.0",
     "@vercel/fetch-retry": "^5.1.3",
-    "ci-info": "^3.3.2",
+    "ci-info": "^3.4.0",
     "picocolors": "^1.0.0",
     "piscina": "^3.2.0",
     "table": "^6.8.0",

pnpm-lock.yaml

@@ -3,7 +3,7 @@ lockfileVersion: 5.4
 specifiers:
   '@sukka/listdir': ^0.2.0
   '@vercel/fetch-retry': ^5.1.3
-  ci-info: ^3.3.2
+  ci-info: ^3.4.0
   picocolors: ^1.0.0
   piscina: ^3.2.0
   table: ^6.8.0
@@ -13,7 +13,7 @@ specifiers:
 dependencies:
   '@sukka/listdir': 0.2.0
   '@vercel/fetch-retry': 5.1.3
-  ci-info: 3.3.2
+  ci-info: 3.4.0
   picocolors: 1.0.0
   piscina: 3.2.0
   table: 6.8.0
@@ -135,8 +135,8 @@ packages:
       fsevents: 2.3.2
     dev: true
 
-  /ci-info/3.3.2:
+  /ci-info/3.4.0:
-    resolution: {integrity: sha512-xmDt/QIAdeZ9+nfdPsaBCpMvHNLFiLdjj59qjqn+6iPe6YmHGQ35sBnQ8uslRBXFmXkiZQOJRjvQeoGppoTjjg==}
+    resolution: {integrity: sha512-t5QdPT5jq3o262DOQ8zA6E1tlH2upmUc4Hlvrbx1pGYJuiiHl7O7rvVNI+l8HTVhd/q3Qc9vqimkNk5yiXsAug==}
     dev: false
 
   /color-convert/2.0.1: