From 8a7f32e2cddb9cf47332b31f43af8be4616ebc5c Mon Sep 17 00:00:00 2001 From: SukkaW Date: Thu, 22 Sep 2022 22:57:00 +0800 Subject: [PATCH] Chore: better logging --- Build/build-anti-bogus-domain.js | 4 ++++ Build/build-apple-cdn.js | 4 ++++ Build/build-cdn-conf.js | 4 ++++ Build/build-cidr.js | 6 +++++- Build/build-reject-domainset.js | 23 +++++++++++++++++++---- Build/build-telegram-cidr.js | 4 ++++ Build/lib/parse-filter.js | 4 ++-- package.json | 2 +- pnpm-lock.yaml | 8 ++++---- 9 files changed, 47 insertions(+), 12 deletions(-) diff --git a/Build/build-anti-bogus-domain.js b/Build/build-anti-bogus-domain.js index 642d4a75..3f8759d6 100644 --- a/Build/build-anti-bogus-domain.js +++ b/Build/build-anti-bogus-domain.js @@ -4,6 +4,8 @@ const path = require('path'); const { isIP } = require('net'); (async () => { + console.time('Total Time - build-anti-bogus-domain'); + console.time('* Download bogus-nxdomain-list') const res = (await (await fetchWithRetry('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/bogus-nxdomain.china.conf')).text()) .split('\n') .map(line => { @@ -14,6 +16,7 @@ const { isIP } = require('net'); return null }) .filter(ip => typeof ip === 'string' && isIP(ip) !== 0); + console.timeEnd('* Download bogus-nxdomain-list') const filePath = path.resolve(__dirname, '../List/ip/reject.conf'); const content = (await fs.promises.readFile(filePath, 'utf-8')) @@ -23,4 +26,5 @@ const { isIP } = require('net'); ); await fs.promises.writeFile(filePath, content, 'utf-8'); + console.timeEnd('Total Time - build-anti-bogus-domain'); })(); diff --git a/Build/build-apple-cdn.js b/Build/build-apple-cdn.js index e558d9c5..0e472a23 100644 --- a/Build/build-apple-cdn.js +++ b/Build/build-apple-cdn.js @@ -5,6 +5,8 @@ const path = require('path'); const rDomain = /^(((?!\-))(xn\-\-)?[a-z0-9\-_]{0,61}[a-z0-9]{1,1}\.)*(xn\-\-)?([a-z0-9\-]{1,61}|[a-z0-9\-]{1,30})\.[a-z]{2,}$/m; (async () => { + console.time('Total Time - build-apple-cdn-conf'); + const res = (await (await fetchWithRetry('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf')).text()) .split('\n') .map(line => { @@ -28,4 +30,6 @@ const rDomain = /^(((?!\-))(xn\-\-)?[a-z0-9\-_]{0,61}[a-z0-9]{1,1}\.)*(xn\-\-)?( 'utf-8' ) ]) + + console.timeEnd('Total Time - build-apple-cdn-conf'); })(); diff --git a/Build/build-cdn-conf.js b/Build/build-cdn-conf.js index 50e9d495..94268718 100644 --- a/Build/build-cdn-conf.js +++ b/Build/build-cdn-conf.js @@ -3,6 +3,8 @@ const fs = require('fs'); const path = require('path'); (async () => { + console.time('Total Time - build-cdn-conf'); + const domains = (await (await fetchWithRetry('https://publicsuffix.org/list/public_suffix_list.dat')).text()).split('\n'); const S3OSSDomains = domains.filter(line => { @@ -29,4 +31,6 @@ const path = require('path'); ); await fs.promises.writeFile(filePath, content, 'utf-8'); + + console.timeEnd('Total Time - build-cdn-conf'); })(); diff --git a/Build/build-cidr.js b/Build/build-cidr.js index 78947ad2..777e6d80 100644 --- a/Build/build-cidr.js +++ b/Build/build-cidr.js @@ -3,6 +3,8 @@ const { promises: fsPromises } = require('fs'); const { resolve: pathResolve } = require('path'); (async () => { + console.time('Total Time - build-chnroutes-cidr'); + const cidr = (await (await fetchWithRetry('https://raw.githubusercontent.com/misakaio/chnroutes2/master/chnroutes.txt')).text()).split('\n'); const filteredCidr = cidr.filter(line => { @@ -13,7 +15,9 @@ const { resolve: pathResolve } = require('path'); return false; }) - return fsPromises.writeFile(pathResolve(__dirname, '../List/ip/china_ip.conf'), makeCidrList(filteredCidr), { encoding: 'utf-8' }); + await fsPromises.writeFile(pathResolve(__dirname, '../List/ip/china_ip.conf'), makeCidrList(filteredCidr), { encoding: 'utf-8' }); + + console.timeEnd('Total Time - build-chnroutes-cidr'); })(); function makeCidrList(cidr) { diff --git a/Build/build-reject-domainset.js b/Build/build-reject-domainset.js index 8f885e80..88036fc3 100644 --- a/Build/build-reject-domainset.js +++ b/Build/build-reject-domainset.js @@ -7,9 +7,13 @@ const { isCI } = require('ci-info'); const threads = isCI ? cpuCount : cpuCount / 2; (async () => { + console.time('Total Time - build-reject-domain-set'); + /** @type Set */ const domainSets = new Set(); + console.time('* Download and process Hosts'); + // Parse from remote hosts & domain lists (await Promise.all([ processHosts('https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=0&mimetype=plaintext', true), @@ -23,6 +27,8 @@ const threads = isCI ? cpuCount : cpuCount / 2; }); }); + console.timeEnd('* Download and process Hosts'); + let previousSize = domainSets.size; console.log(`Import ${previousSize} rules from hosts files!`); @@ -83,6 +89,7 @@ const threads = isCI ? cpuCount : cpuCount / 2; 'storage.yandexcloud.net' // phishing list ]); + console.time('* Download and process AdBlock Filter Rules'); (await Promise.all([ // Easy List [ @@ -147,6 +154,8 @@ const threads = isCI ? cpuCount : cpuCount / 2; black.forEach(i => domainSets.add(i)); }); + console.timeEnd('* Download and process AdBlock Filter Rules'); + previousSize = domainSets.size - previousSize; console.log(`Import ${previousSize} rules from adguard filters!`); @@ -169,7 +178,8 @@ const threads = isCI ? cpuCount : cpuCount / 2; previousSize = domainSets.size; // Dedupe domainSets - console.log(`Start deduping! (${previousSize})`); + console.log(`Start deduping from black keywords/suffixes! (${previousSize})`); + console.time(`* Dedupe from black keywords/suffixes`); const toBeRemoved = new Set(); for (const domain of domainSets) { @@ -209,13 +219,15 @@ const threads = isCI ? cpuCount : cpuCount / 2; domainSets.delete(removed) }); - // Dedupe domainSets - console.log(`Deduped ${previousSize - domainSets.size} from black keywords and suffixes!`); + console.timeEnd(`* Dedupe from black keywords/suffixes`); + console.log(`Deduped ${previousSize} - ${domainSets.size} = ${previousSize - domainSets.size} from black keywords and suffixes!`); previousSize = domainSets.size; // Dedupe domainSets console.log(`Start deduping! (${previousSize})`); + const START_TIME = Date.now(); + const piscina = new Piscina({ filename: pathResolve(__dirname, 'worker/build-reject-domainset-worker.js'), workerData: [...domainSets] @@ -253,10 +265,13 @@ const threads = isCI ? cpuCount : cpuCount / 2; }) }); + console.log(`* Dedupe from covered subdomain - ${(Date.now() - START_TIME) / 1000}s`); console.log(`Deduped ${previousSize - domainSets.size} rules!`); - return fsPromises.writeFile( + await fsPromises.writeFile( pathResolve(__dirname, '../List/domainset/reject.conf'), `${[...domainSets].join('\n')}\n`, { encoding: 'utf-8' }); + + console.timeEnd('Total Time - build-reject-domain-set'); })(); diff --git a/Build/build-telegram-cidr.js b/Build/build-telegram-cidr.js index c6900e2f..12b4422f 100644 --- a/Build/build-telegram-cidr.js +++ b/Build/build-telegram-cidr.js @@ -4,6 +4,8 @@ const path = require('path'); const { isIPv4, isIPv6 } = require('net'); (async () => { + console.time('Total Time - build-telegram-cidr'); + const resp = await fetchWithRetry('https://core.telegram.org/resources/cidr.txt'); const lastModified = new Date(resp.headers.get('last-modified')); @@ -27,4 +29,6 @@ const { isIPv4, isIPv6 } = require('net'); }).join('\n') + '\n', 'utf-8' ); + + console.timeEnd('Total Time - build-telegram-cidr'); })(); diff --git a/Build/lib/parse-filter.js b/Build/lib/parse-filter.js index e26176c1..46abf9bc 100644 --- a/Build/lib/parse-filter.js +++ b/Build/lib/parse-filter.js @@ -96,7 +96,7 @@ async function processHosts (hostsUrl, includeAllSubDomain = false) { * @returns {Promise<{ white: Set, black: Set }>} */ async function processFilterRules (filterRulesUrl, fallbackUrls) { - console.time(`processFilterRules: ${filterRulesUrl}`); + console.time(` - processFilterRules: ${filterRulesUrl}`); if (typeof filterRulesUrl === 'string') { filterRulesUrl = new URL(filterRulesUrl); @@ -206,7 +206,7 @@ async function processFilterRules (filterRulesUrl, fallbackUrls) { } }); - console.timeEnd(`processFilterRules: ${filterRulesUrl}`); + console.timeEnd(` - processFilterRules: ${filterRulesUrl}`); return { white: whitelistDomainSets, diff --git a/package.json b/package.json index 9471293c..c779648b 100644 --- a/package.json +++ b/package.json @@ -58,7 +58,7 @@ "dependencies": { "@sukka/listdir": "^0.2.0", "@vercel/fetch-retry": "^5.1.3", - "ci-info": "^3.3.2", + "ci-info": "^3.4.0", "picocolors": "^1.0.0", "piscina": "^3.2.0", "table": "^6.8.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index be4d416b..3b387af4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -3,7 +3,7 @@ lockfileVersion: 5.4 specifiers: '@sukka/listdir': ^0.2.0 '@vercel/fetch-retry': ^5.1.3 - ci-info: ^3.3.2 + ci-info: ^3.4.0 picocolors: ^1.0.0 piscina: ^3.2.0 table: ^6.8.0 @@ -13,7 +13,7 @@ specifiers: dependencies: '@sukka/listdir': 0.2.0 '@vercel/fetch-retry': 5.1.3 - ci-info: 3.3.2 + ci-info: 3.4.0 picocolors: 1.0.0 piscina: 3.2.0 table: 6.8.0 @@ -135,8 +135,8 @@ packages: fsevents: 2.3.2 dev: true - /ci-info/3.3.2: - resolution: {integrity: sha512-xmDt/QIAdeZ9+nfdPsaBCpMvHNLFiLdjj59qjqn+6iPe6YmHGQ35sBnQ8uslRBXFmXkiZQOJRjvQeoGppoTjjg==} + /ci-info/3.4.0: + resolution: {integrity: sha512-t5QdPT5jq3o262DOQ8zA6E1tlH2upmUc4Hlvrbx1pGYJuiiHl7O7rvVNI+l8HTVhd/q3Qc9vqimkNk5yiXsAug==} dev: false /color-convert/2.0.1: