From 9f05105b29a84603bb4e19515c9aafc75bde26b0 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Wed, 13 Sep 2023 14:07:38 +0800 Subject: [PATCH] Print task run duration time --- Build/build-anti-bogus-domain.js | 12 +--- Build/build-apple-cdn.js | 9 +-- Build/build-cdn-conf.js | 9 +-- Build/build-chn-cidr.js | 8 +-- Build/build-domestic-ruleset.js | 5 +- Build/build-internal-cdn-rules.js | 5 +- Build/build-internal-chn-domains.js | 5 +- Build/build-internal-reverse-chn-cidr.js | 5 +- Build/build-phishing-domainset.js | 5 +- Build/build-reject-domainset.js | 74 +++++++++++------------- Build/build-speedtest-domainset.js | 5 +- Build/build-telegram-cidr.js | 9 +-- Build/build.js | 5 +- Build/download-previous-build.js | 5 +- Build/lib/reject-data-source.js | 7 ++- Build/lib/trace-runner.js | 15 +++++ Build/validate-domainset.js | 5 +- 17 files changed, 96 insertions(+), 92 deletions(-) create mode 100644 Build/lib/trace-runner.js diff --git a/Build/build-anti-bogus-domain.js b/Build/build-anti-bogus-domain.js index 7e0e4c0e..13eee4de 100644 --- a/Build/build-anti-bogus-domain.js +++ b/Build/build-anti-bogus-domain.js @@ -4,11 +4,9 @@ const { isIPv4, isIPv6 } = require('net'); const { createRuleset } = require('./lib/create-file'); const { fetchRemoteTextAndCreateReadlineInterface, readFileByLine } = require('./lib/fetch-remote-text-by-line'); const { processLine } = require('./lib/process-line'); +const { runner } = require('./lib/trace-runner'); -(async () => { - console.time('Total Time - build-anti-bogus-domain'); - console.time('* Download bogus-nxdomain-list'); - +runner(__filename, async () => { /** @type {string[]} */ const res = []; for await (const line of await fetchRemoteTextAndCreateReadlineInterface('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/bogus-nxdomain.china.conf')) { @@ -17,8 +15,6 @@ const { processLine } = require('./lib/process-line'); } } - console.timeEnd('* Download bogus-nxdomain-list'); - const filePath = path.resolve(__dirname, '../Source/ip/reject.conf'); /** @type {string[]} */ @@ -60,6 +56,4 @@ const { processLine } = require('./lib/process-line'); path.resolve(__dirname, '../List/ip/reject.conf'), path.resolve(__dirname, '../Clash/ip/reject.txt') )); - - console.timeEnd('Total Time - build-anti-bogus-domain'); -})(); +}); diff --git a/Build/build-apple-cdn.js b/Build/build-apple-cdn.js index 11dfde13..69579386 100644 --- a/Build/build-apple-cdn.js +++ b/Build/build-apple-cdn.js @@ -1,10 +1,9 @@ const path = require('path'); const { createRuleset } = require('./lib/create-file'); const { parseFelixDnsmasq } = require('./lib/parse-dnsmasq'); +const { runner } = require('./lib/trace-runner'); -(async () => { - console.time('Total Time - build-apple-cdn-conf'); - +runner(__filename, async () => { const res = await parseFelixDnsmasq('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf'); const description = [ @@ -41,6 +40,4 @@ const { parseFelixDnsmasq } = require('./lib/parse-dnsmasq'); path.resolve(__dirname, '../Clash/domainset/apple_cdn.txt') ) ]); - - console.timeEnd('Total Time - build-apple-cdn-conf'); -})(); +}); diff --git a/Build/build-cdn-conf.js b/Build/build-cdn-conf.js index 077f5b7c..2f02df6b 100644 --- a/Build/build-cdn-conf.js +++ b/Build/build-cdn-conf.js @@ -4,10 +4,9 @@ const { createRuleset } = require('./lib/create-file'); const { minifyRules } = require('./lib/minify-rules'); const { fetchRemoteTextAndCreateReadlineInterface, readFileByLine } = require('./lib/fetch-remote-text-by-line'); const Trie = require('./lib/trie'); +const { runner } = require('./lib/trace-runner'); -(async () => { - console.time('Total Time - build-cdn-conf'); - +runner(__filename, async () => { const trie = new Trie(); for await (const line of await fetchRemoteTextAndCreateReadlineInterface('https://publicsuffix.org/list/public_suffix_list.dat')) { trie.add(line); @@ -55,6 +54,4 @@ const Trie = require('./lib/trie'); path.resolve(__dirname, '../List/non_ip/cdn.conf'), path.resolve(__dirname, '../Clash/non_ip/cdn.txt') )); - - console.timeEnd('Total Time - build-cdn-conf'); -})(); +}); diff --git a/Build/build-chn-cidr.js b/Build/build-chn-cidr.js index 5433b5c6..5a5864ea 100644 --- a/Build/build-chn-cidr.js +++ b/Build/build-chn-cidr.js @@ -4,6 +4,7 @@ const { resolve: pathResolve } = require('path'); // This should not use `createRuleset` API since we are going to generate ipcidr for Clash const { compareAndWriteFile, withBannerArray } = require('./lib/create-file'); const { processLine } = require('./lib/process-line'); +const { runner } = require('./lib/trace-runner'); // https://github.com/misakaio/chnroutes2/issues/25 const EXCLUDE_CIDRS = [ @@ -11,8 +12,7 @@ const EXCLUDE_CIDRS = [ '223.120.0.0/15' ]; -(async () => { - console.time('Total Time - build-chnroutes-cidr'); +runner(__filename, async () => { const { exclude: excludeCidrs } = await import('cidr-tools-wasm'); /** @type {Set} */ @@ -56,6 +56,4 @@ const EXCLUDE_CIDRS = [ pathResolve(__dirname, '../Clash/ip/china_ip.txt') ) ]); - - console.timeEnd('Total Time - build-chnroutes-cidr'); -})(); +}); diff --git a/Build/build-domestic-ruleset.js b/Build/build-domestic-ruleset.js index 50a090c5..57c99d0c 100644 --- a/Build/build-domestic-ruleset.js +++ b/Build/build-domestic-ruleset.js @@ -5,8 +5,9 @@ const { readFileByLine } = require('./lib/fetch-remote-text-by-line'); const { processLine } = require('./lib/process-line'); const { compareAndWriteFile, createRuleset } = require('./lib/create-file'); const domainSorter = require('./lib/stable-sort-domain'); +const { runner } = require('./lib/trace-runner'); -(async () => { +runner(__filename, async () => { const rl = readFileByLine(path.resolve(__dirname, '../Source/non_ip/domestic.conf')); const results = []; for await (const l of rl) { @@ -60,4 +61,4 @@ const domainSorter = require('./lib/stable-sort-domain'); path.resolve(__dirname, '../Modules/sukka_local_dns_mapping.sgmodule') ) ]); -})(); +}); diff --git a/Build/build-internal-cdn-rules.js b/Build/build-internal-cdn-rules.js index 7b2f93ff..4a00bd4d 100644 --- a/Build/build-internal-cdn-rules.js +++ b/Build/build-internal-cdn-rules.js @@ -7,6 +7,7 @@ const tldts = require('tldts'); const { processLine } = require('./lib/process-line'); const { readFileByLine } = require('./lib/fetch-remote-text-by-line'); const domainSorter = require('./lib/stable-sort-domain'); +const { runner } = require('./lib/trace-runner'); /** * @param {string} string @@ -15,7 +16,7 @@ const escapeRegExp = (string) => { return string.replace(/[\\^$.*+?()[\]{}|]/g, '\\$&'); }; -(async () => { +runner(__filename, async () => { const set = new Set(); const keywords = new Set(); @@ -84,4 +85,4 @@ const escapeRegExp = (string) => { '' ].join('\n') ); -})(); +}); diff --git a/Build/build-internal-chn-domains.js b/Build/build-internal-chn-domains.js index e3622ad5..59c3418e 100644 --- a/Build/build-internal-chn-domains.js +++ b/Build/build-internal-chn-domains.js @@ -3,8 +3,9 @@ const path = require('path'); const fse = require('fs-extra'); const fs = require('fs'); const { parseFelixDnsmasq } = require('./lib/parse-dnsmasq'); +const { runner } = require('./lib/trace-runner'); -(async () => { +runner(__filename, async () => { const [result] = await Promise.all([ parseFelixDnsmasq('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf'), fse.ensureDir(path.resolve(__dirname, '../List/internal')) @@ -14,4 +15,4 @@ const { parseFelixDnsmasq } = require('./lib/parse-dnsmasq'); path.resolve(__dirname, '../List/internal/accelerated-china-domains.txt'), `${result.map(line => `SUFFIX,${line}`).join('\n')}\n` ); -})(); +}); diff --git a/Build/build-internal-reverse-chn-cidr.js b/Build/build-internal-reverse-chn-cidr.js index ec1721a6..057be3ec 100644 --- a/Build/build-internal-reverse-chn-cidr.js +++ b/Build/build-internal-reverse-chn-cidr.js @@ -4,6 +4,7 @@ const { processLine } = require('./lib/process-line'); const path = require('path'); const fse = require('fs-extra'); const fs = require('fs'); +const { runner } = require('./lib/trace-runner'); const RESERVED_IPV4_CIDR = [ '0.0.0.0/8', @@ -23,7 +24,7 @@ const RESERVED_IPV4_CIDR = [ '240.0.0.0/4' ]; -(async () => { +runner(__filename, async () => { const { exclude } = await import('cidr-tools-wasm'); /** @type {Set} */ @@ -46,4 +47,4 @@ const RESERVED_IPV4_CIDR = [ path.resolve(__dirname, '../List/internal/reversed-chn-cidr.txt'), `${reversedCidr.join('\n')}\n` ); -})(); +}); diff --git a/Build/build-phishing-domainset.js b/Build/build-phishing-domainset.js index c2956321..a588efcf 100644 --- a/Build/build-phishing-domainset.js +++ b/Build/build-phishing-domainset.js @@ -4,6 +4,7 @@ const path = require('path'); const { createRuleset } = require('./lib/create-file'); const { processLine } = require('./lib/process-line.js'); const domainSorter = require('./lib/stable-sort-domain'); +const { runner } = require('./lib/trace-runner.js'); const WHITELIST_DOMAIN = new Set([ 'w3s.link', @@ -59,7 +60,7 @@ const BLACK_TLD = new Set([ 'com.cn' ]); -(async () => { +runner(__filename, async () => { const domainSet = Array.from( ( await processFilterRules('https://curbengh.github.io/phishing-filter/phishing-filter-agh.txt') @@ -159,4 +160,4 @@ const BLACK_TLD = new Set([ path.resolve(__dirname, '../List/domainset/reject_phishing.conf'), path.resolve(__dirname, '../Clash/domainset/reject_phishing.txt') )); -})(); +}); diff --git a/Build/build-reject-domainset.js b/Build/build-reject-domainset.js index 1e92e1b6..6134a9b5 100644 --- a/Build/build-reject-domainset.js +++ b/Build/build-reject-domainset.js @@ -23,8 +23,6 @@ const domainKeywordsSet = new Set(); /** @type {Set} Dedupe domains included by DOMAIN-SUFFIX */ const domainSuffixSet = new Set(); (async () => { - console.time('Total Time - build-reject-domain-set'); - /** @type Set */ const domainSets = new Set(); @@ -35,15 +33,14 @@ const domainSuffixSet = new Set(); await Promise.all([ // Parse from remote hosts & domain lists - Promise.all(HOSTS.map(entry => processHosts(entry[0], entry[1]))) - .then(r => r.forEach(hosts => { - hosts.forEach(host => { - if (host) { - domainSets.add(host); - } - }); - })), - Promise.all(ADGUARD_FILTERS.map(input => { + ...HOSTS.map(entry => processHosts(entry[0], entry[1]).then(hosts => { + hosts.forEach(host => { + if (host) { + domainSets.add(host); + } + }); + })), + ...ADGUARD_FILTERS.map(input => { const promise = typeof input === 'string' ? processFilterRules(input, undefined, false) : processFilterRules(input[0], input[1] || undefined, input[2] ?? false); @@ -62,34 +59,34 @@ const domainSuffixSet = new Set(); }); black.forEach(i => domainSets.add(i)); } else { - process.exit(1); + process.exitCode = 1; + throw new Error('Failed to process AdGuard Filter Rules!'); } }); - })), - Promise.all([ + }), + ...([ 'https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt', 'https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt' - ].map( - input => processFilterRules(input).then((i) => { - if (i) { - const { white, black } = i; - white.forEach(i => { - // if (PREDEFINED_ENFORCED_BACKLIST.some(j => i.endsWith(j))) { - // return; - // } - filterRuleWhitelistDomainSets.add(i); - }); - black.forEach(i => { - // if (PREDEFINED_ENFORCED_BACKLIST.some(j => i.endsWith(j))) { - // return; - // } - filterRuleWhitelistDomainSets.add(i); - }); - } else { - process.exit(1); - } - }) - )) + ].map(input => processFilterRules(input).then((i) => { + if (i) { + const { white, black } = i; + white.forEach(i => { + // if (PREDEFINED_ENFORCED_BACKLIST.some(j => i.endsWith(j))) { + // return; + // } + filterRuleWhitelistDomainSets.add(i); + }); + black.forEach(i => { + // if (PREDEFINED_ENFORCED_BACKLIST.some(j => i.endsWith(j))) { + // return; + // } + filterRuleWhitelistDomainSets.add(i); + }); + } else { + process.exitCode = 1; + throw new Error('Failed to process AdGuard Filter Rules!'); + } + }))) ]); const trie0 = Trie.from(Array.from(filterRuleWhitelistDomainSets)); @@ -100,6 +97,7 @@ const domainSuffixSet = new Set(); console.timeEnd('* Download and process Hosts / AdBlock Filter Rules'); if (shouldStop) { + // eslint-disable-next-line n/no-process-exit -- force stop process.exit(1); } @@ -183,8 +181,6 @@ const domainSuffixSet = new Set(); console.log(`* Dedupe from covered subdomain - ${(Date.now() - START_TIME) / 1000}s`); console.log(`Deduped ${previousSize - dudupedDominArray.length} rules!`); - console.time('* Write reject.conf'); - /** @type {Record} */ const rejectDomainsStats = dudupedDominArray.reduce((acc, cur) => { const suffix = tldts.getDomain(cur, { allowPrivateDomains: false }); @@ -234,8 +230,4 @@ const domainSuffixSet = new Set(); // Copy reject_sukka.conf for backward compatibility fse.copy(pathResolve(__dirname, '../Source/domainset/reject_sukka.conf'), pathResolve(__dirname, '../List/domainset/reject_sukka.conf')) ]); - - console.timeEnd('* Write reject.conf'); - - console.timeEnd('Total Time - build-reject-domain-set'); })(); diff --git a/Build/build-speedtest-domainset.js b/Build/build-speedtest-domainset.js index ff5ea28c..10b02e94 100644 --- a/Build/build-speedtest-domainset.js +++ b/Build/build-speedtest-domainset.js @@ -5,6 +5,7 @@ const { createRuleset } = require('./lib/create-file'); const domainSorter = require('./lib/stable-sort-domain'); const { Sema } = require('async-sema'); +const { runner } = require('./lib/trace-runner'); const s = new Sema(2); /** @@ -45,7 +46,7 @@ const querySpeedtestApi = async (keyword) => { } }; -(async () => { +runner(__filename, async () => { /** @type {Set} */ const domains = new Set([ '.speedtest.net', @@ -122,4 +123,4 @@ const querySpeedtestApi = async (keyword) => { path.resolve(__dirname, '../List/domainset/speedtest.conf'), path.resolve(__dirname, '../Clash/domainset/speedtest.txt') )); -})(); +}); diff --git a/Build/build-telegram-cidr.js b/Build/build-telegram-cidr.js index 59152e55..8d63600c 100644 --- a/Build/build-telegram-cidr.js +++ b/Build/build-telegram-cidr.js @@ -4,10 +4,9 @@ const path = require('path'); const { isIPv4, isIPv6 } = require('net'); const { processLine } = require('./lib/process-line'); const { createRuleset } = require('./lib/create-file'); +const { runner } = require('./lib/trace-runner'); -(async () => { - console.time('Total Time - build-telegram-cidr'); - +runner(__filename, async () => { /** @type {Response} */ const resp = await fetchWithRetry('https://core.telegram.org/resources/cidr.txt'); const lastModified = resp.headers.get('last-modified'); @@ -50,6 +49,4 @@ const { createRuleset } = require('./lib/create-file'); path.resolve(__dirname, '../List/ip/telegram.conf'), path.resolve(__dirname, '../Clash/ip/telegram.txt') )); - - console.timeEnd('Total Time - build-telegram-cidr'); -})(); +}); diff --git a/Build/build.js b/Build/build.js index 04722995..2b0056cc 100644 --- a/Build/build.js +++ b/Build/build.js @@ -6,6 +6,7 @@ const { readFileByLine } = require('./lib/fetch-remote-text-by-line'); const { processLine } = require('./lib/process-line'); const { createRuleset } = require('./lib/create-file'); const { domainDeduper } = require('./lib/domain-deduper'); +const { runner } = require('./lib/trace-runner'); const MAGIC_COMMAND_SKIP = '# $ custom_build_script'; const MAGIC_COMMAND_TITLE = '# $ meta_title '; @@ -15,7 +16,7 @@ const sourceDir = path.resolve(__dirname, '../Source'); const outputSurgeDir = path.resolve(__dirname, '../List'); const outputClashDir = path.resolve(__dirname, '../Clash'); -(async () => { +runner(__filename, async () => { /** @type {Promise[]} */ const promises = []; @@ -42,7 +43,7 @@ const outputClashDir = path.resolve(__dirname, '../Clash'); } await Promise.all(promises); -})(); +}); /** * @param {string} sourcePath diff --git a/Build/download-previous-build.js b/Build/download-previous-build.js index 66036ae7..5f431d6c 100644 --- a/Build/download-previous-build.js +++ b/Build/download-previous-build.js @@ -8,6 +8,7 @@ const { Readable } = require('stream'); const { pipeline } = require('stream/promises'); const { readFileByLine } = require('./lib/fetch-remote-text-by-line'); const { isCI } = require('ci-info'); +const { runner } = require('./lib/trace-runner'); const fileExists = (path) => { return fs.promises.access(path, fs.constants.F_OK) @@ -15,7 +16,7 @@ const fileExists = (path) => { .catch(() => false); }; -(async () => { +runner(__filename, async () => { const filesList = ['Clash', 'List']; let allFileExists = true; @@ -77,4 +78,4 @@ const fileExists = (path) => { })); await fs.promises.unlink(extractedPath).catch(() => { }); -})(); +}); diff --git a/Build/lib/reject-data-source.js b/Build/lib/reject-data-source.js index da87a13b..d0d79920 100644 --- a/Build/lib/reject-data-source.js +++ b/Build/lib/reject-data-source.js @@ -172,7 +172,12 @@ const ADGUARD_FILTERS = /** @type {const} */([ // GameConsoleAdblockList 'https://raw.githubusercontent.com/DandelionSprout/adfilt/master/GameConsoleAdblockList.txt', // PiHoleBlocklist - 'https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/SmartTV-AGH.txt', + [ + 'https://perflyst.github.io/PiHoleBlocklist/SmartTV-AGH.txt', + [ + 'https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/SmartTV-AGH.txt' + ] + ], // Spam404 'https://raw.githubusercontent.com/Spam404/lists/master/adblock-list.txt', // BarbBlock diff --git a/Build/lib/trace-runner.js b/Build/lib/trace-runner.js new file mode 100644 index 00000000..649d2295 --- /dev/null +++ b/Build/lib/trace-runner.js @@ -0,0 +1,15 @@ +const path = require('path'); + +/** + * @param {Function} fn + * @param {string} __filename + */ +module.exports.runner = async (__filename, fn) => { + const runnerName = path.basename(__filename, path.extname(__filename)); + + const start = Date.now(); + const result = await fn(); + const end = Date.now(); + console.log(`⌛ [${runnerName}]: ${end - start}ms`); + return result; +}; diff --git a/Build/validate-domainset.js b/Build/validate-domainset.js index 45f8188f..332e5cdd 100644 --- a/Build/validate-domainset.js +++ b/Build/validate-domainset.js @@ -6,6 +6,7 @@ const path = require('path'); const listDir = require('@sukka/listdir'); const { readFileByLine } = require('./lib/fetch-remote-text-by-line'); const { processLine } = require('./lib/process-line'); +const { runner } = require('./lib/trace-runner'); const SPECIAL_SUFFIXES = new Set([ 'linodeobjects.com', // only *.linodeobjects.com are public suffix @@ -57,7 +58,7 @@ const _validateRuleset = async (filePath) => { } }; -(async () => { +runner(__filename, async () => { const [domainsetFiles, _rulesetFiles] = await Promise.all([ listDir(path.resolve(__dirname, '../List/domainset')), listDir(path.resolve(__dirname, '../List/non_ip')) @@ -66,4 +67,4 @@ const _validateRuleset = async (filePath) => { domainsetFiles.map(file => validateDomainSet(file)) // rulesetFiles.map(file => validateRuleset(file)) ); -})(); +});