From b42bd05ccfe5be3a812fdfe2ee347af014783423 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Thu, 14 Sep 2023 21:11:43 +0800 Subject: [PATCH] Perf: reduce infra runtime costs --- Build/build-anti-bogus-domain.js | 6 +- Build/build-apple-cdn.js | 6 +- Build/build-cdn-conf.js | 28 +++-- Build/build-chn-cidr.js | 6 +- Build/build-common.js | 6 +- Build/build-domestic-ruleset.js | 6 +- Build/build-internal-cdn-rules.js | 6 +- Build/build-internal-chn-domains.js | 6 +- Build/build-internal-reverse-chn-cidr.js | 6 +- Build/build-phishing-domainset.js | 6 +- Build/build-public.js | 6 +- Build/build-reject-domainset.js | 6 +- Build/build-speedtest-domainset.js | 6 +- Build/build-telegram-cidr.js | 6 +- Build/build.js | 0 Build/download-previous-build.js | 18 ++- Build/index.js | 82 +++++++++++++ Build/lib/cached-tld-parse.js | 3 +- Build/lib/get-gorhill-publicsuffix.js | 2 +- Build/lib/trace-runner.js | 16 ++- Build/validate-domainset.js | 6 +- package.json | 143 +---------------------- pnpm-lock.yaml | 61 +--------- 23 files changed, 172 insertions(+), 265 deletions(-) delete mode 100644 Build/build.js create mode 100644 Build/index.js diff --git a/Build/build-anti-bogus-domain.js b/Build/build-anti-bogus-domain.js index cc010a75..4fa4e22c 100644 --- a/Build/build-anti-bogus-domain.js +++ b/Build/build-anti-bogus-domain.js @@ -4,9 +4,9 @@ const { isIPv4, isIPv6 } = require('net'); const { createRuleset } = require('./lib/create-file'); const { fetchRemoteTextAndCreateReadlineInterface, readFileByLine } = require('./lib/fetch-remote-text-by-line'); const { processLine } = require('./lib/process-line'); -const { runner } = require('./lib/trace-runner'); +const { runner, task } = require('./lib/trace-runner'); -const buildAntiBogusDomain = async () => { +const buildAntiBogusDomain = task(__filename, async () => { /** @type {string[]} */ const res = []; for await (const line of await fetchRemoteTextAndCreateReadlineInterface('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/bogus-nxdomain.china.conf')) { @@ -56,7 +56,7 @@ const buildAntiBogusDomain = async () => { path.resolve(__dirname, '../List/ip/reject.conf'), path.resolve(__dirname, '../Clash/ip/reject.txt') )); -}; +}); module.exports.buildAntiBogusDomain = buildAntiBogusDomain; diff --git a/Build/build-apple-cdn.js b/Build/build-apple-cdn.js index 86cf149c..7459b97a 100644 --- a/Build/build-apple-cdn.js +++ b/Build/build-apple-cdn.js @@ -1,9 +1,9 @@ const path = require('path'); const { createRuleset } = require('./lib/create-file'); const { parseFelixDnsmasq } = require('./lib/parse-dnsmasq'); -const { runner } = require('./lib/trace-runner'); +const { runner, task } = require('./lib/trace-runner'); -const buildAppleCdn = async () => { +const buildAppleCdn = task(__filename, async () => { const res = await parseFelixDnsmasq('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf'); const description = [ @@ -40,7 +40,7 @@ const buildAppleCdn = async () => { path.resolve(__dirname, '../Clash/domainset/apple_cdn.txt') ) ]); -}; +}); module.exports.buildAppleCdn = buildAppleCdn; diff --git a/Build/build-cdn-conf.js b/Build/build-cdn-conf.js index 39a4e68a..e2795a11 100644 --- a/Build/build-cdn-conf.js +++ b/Build/build-cdn-conf.js @@ -3,13 +3,13 @@ const path = require('path'); const { createRuleset } = require('./lib/create-file'); const { fetchRemoteTextAndCreateReadlineInterface, readFileByLine } = require('./lib/fetch-remote-text-by-line'); const Trie = require('./lib/trie'); -const { runner } = require('./lib/trace-runner'); +const { runner, task } = require('./lib/trace-runner'); const fs = require('fs'); const { processLine } = require('./lib/process-line'); -const publicSuffixPath = path.resolve(__dirname, '../node_modules/.cache/public_suffix-list_dat.txt'); +const publicSuffixPath = path.resolve(__dirname, '../node_modules/.cache/public_suffix_list_dat.txt'); -const buildCdnConf = async () => { +const buildCdnConf = task(__filename, async () => { const trie = new Trie(); if (fs.existsSync(publicSuffixPath)) { @@ -30,12 +30,18 @@ const buildCdnConf = async () => { const S3OSSDomains = new Set(); trie.find('.amazonaws.com').forEach(line => { - if ((line.startsWith('s3-') || line.startsWith('s3.')) && !line.includes('cn-')) { + if ( + (line.startsWith('s3-') || line.startsWith('s3.')) + && !line.includes('cn-') + ) { S3OSSDomains.add(line); } }); trie.find('.scw.cloud').forEach(line => { - if ((line.startsWith('s3-') || line.startsWith('s3.')) && !line.includes('cn-')) { + if ( + (line.startsWith('s3-') || line.startsWith('s3.')) + && !line.includes('cn-') + ) { S3OSSDomains.add(line); } }); @@ -43,10 +49,14 @@ const buildCdnConf = async () => { /** @type {string[]} */ const cdnDomainsList = []; for await (const l of readFileByLine(path.resolve(__dirname, '../Source/non_ip/cdn.conf'))) { + if (l === '# --- [AWS S3 Replace Me] ---') { + console.log(S3OSSDomains); + S3OSSDomains.forEach(domain => { + cdnDomainsList.push(`DOMAIN-SUFFIX,${domain}`); + }); + } const line = processLine(l); - if (line === '# --- [AWS S3 Replace Me] ---') { - S3OSSDomains.forEach(domain => cdnDomainsList.push(`DOMAIN-SUFFIX,${domain}`)); - } else if (line) { + if (line) { cdnDomainsList.push(line); } } @@ -68,7 +78,7 @@ const buildCdnConf = async () => { path.resolve(__dirname, '../List/non_ip/cdn.conf'), path.resolve(__dirname, '../Clash/non_ip/cdn.txt') )); -}; +}); module.exports.buildCdnConf = buildCdnConf; diff --git a/Build/build-chn-cidr.js b/Build/build-chn-cidr.js index 73a4155c..318e25db 100644 --- a/Build/build-chn-cidr.js +++ b/Build/build-chn-cidr.js @@ -4,7 +4,7 @@ const { resolve: pathResolve } = require('path'); // This should not use `createRuleset` API since we are going to generate ipcidr for Clash const { compareAndWriteFile, withBannerArray } = require('./lib/create-file'); const { processLine } = require('./lib/process-line'); -const { runner } = require('./lib/trace-runner'); +const { runner, task } = require('./lib/trace-runner'); // https://github.com/misakaio/chnroutes2/issues/25 const EXCLUDE_CIDRS = [ @@ -12,7 +12,7 @@ const EXCLUDE_CIDRS = [ '223.120.0.0/15' ]; -const buildChnCidr = async () => { +const buildChnCidr = task(__filename, async () => { const { exclude: excludeCidrs } = await import('cidr-tools-wasm'); /** @type {string[]} */ @@ -56,7 +56,7 @@ const buildChnCidr = async () => { pathResolve(__dirname, '../Clash/ip/china_ip.txt') ) ]); -}; +}); module.exports.buildChnCidr = buildChnCidr; diff --git a/Build/build-common.js b/Build/build-common.js index 2902b264..7c6c5eec 100644 --- a/Build/build-common.js +++ b/Build/build-common.js @@ -6,7 +6,7 @@ const { readFileByLine } = require('./lib/fetch-remote-text-by-line'); const { processLine } = require('./lib/process-line'); const { createRuleset } = require('./lib/create-file'); const { domainDeduper } = require('./lib/domain-deduper'); -const { runner } = require('./lib/trace-runner'); +const { runner, task } = require('./lib/trace-runner'); const MAGIC_COMMAND_SKIP = '# $ custom_build_script'; const MAGIC_COMMAND_TITLE = '# $ meta_title '; @@ -16,7 +16,7 @@ const sourceDir = path.resolve(__dirname, '../Source'); const outputSurgeDir = path.resolve(__dirname, '../List'); const outputClashDir = path.resolve(__dirname, '../Clash'); -const buildCommon = async () => { +const buildCommon = task(__filename, async () => { /** @type {Promise[]} */ const promises = []; @@ -43,7 +43,7 @@ const buildCommon = async () => { } return Promise.all(promises); -}; +}); module.exports.buildCommon = buildCommon; diff --git a/Build/build-domestic-ruleset.js b/Build/build-domestic-ruleset.js index fe309257..74bd9b32 100644 --- a/Build/build-domestic-ruleset.js +++ b/Build/build-domestic-ruleset.js @@ -5,9 +5,9 @@ const { readFileByLine } = require('./lib/fetch-remote-text-by-line'); const { processLine } = require('./lib/process-line'); const { compareAndWriteFile, createRuleset } = require('./lib/create-file'); const domainSorter = require('./lib/stable-sort-domain'); -const { runner } = require('./lib/trace-runner'); +const { runner, task } = require('./lib/trace-runner'); -const buildDomesticRuleset = async () => { +const buildDomesticRuleset = task(__filename, async () => { const rl = readFileByLine(path.resolve(__dirname, '../Source/non_ip/domestic.conf')); const results = []; for await (const l of rl) { @@ -67,7 +67,7 @@ const buildDomesticRuleset = async () => { path.resolve(__dirname, '../Modules/sukka_local_dns_mapping.sgmodule') ) ]); -}; +}); module.exports.buildDomesticRuleset = buildDomesticRuleset; diff --git a/Build/build-internal-cdn-rules.js b/Build/build-internal-cdn-rules.js index 6b1e959b..b967e286 100644 --- a/Build/build-internal-cdn-rules.js +++ b/Build/build-internal-cdn-rules.js @@ -5,7 +5,7 @@ const tldts = require('tldts'); const { processLine } = require('./lib/process-line'); const { readFileByLine } = require('./lib/fetch-remote-text-by-line'); const domainSorter = require('./lib/stable-sort-domain'); -const { runner } = require('./lib/trace-runner'); +const { runner, task } = require('./lib/trace-runner'); const { compareAndWriteFile } = require('./lib/create-file'); /** @@ -15,7 +15,7 @@ const escapeRegExp = (string) => { return string.replace(/[\\^$.*+?()[\]{}|]/g, '\\$&'); }; -const buildInternalCDNDomains = async () => { +const buildInternalCDNDomains = task(__filename, async () => { const set = new Set(); const keywords = new Set(); @@ -87,7 +87,7 @@ const buildInternalCDNDomains = async () => { ], path.resolve(__dirname, '../List/internal/cdn.txt') ); -}; +}); module.exports.buildInternalCDNDomains = buildInternalCDNDomains; diff --git a/Build/build-internal-chn-domains.js b/Build/build-internal-chn-domains.js index 537c8cec..70869ec7 100644 --- a/Build/build-internal-chn-domains.js +++ b/Build/build-internal-chn-domains.js @@ -2,10 +2,10 @@ const path = require('path'); const fse = require('fs-extra'); const { parseFelixDnsmasq } = require('./lib/parse-dnsmasq'); -const { runner } = require('./lib/trace-runner'); +const { runner, task } = require('./lib/trace-runner'); const { compareAndWriteFile } = require('./lib/create-file'); -const buildInternalChnDomains = async () => { +const buildInternalChnDomains = task(__filename, async () => { const [result] = await Promise.all([ parseFelixDnsmasq('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf'), fse.ensureDir(path.resolve(__dirname, '../List/internal')) @@ -15,7 +15,7 @@ const buildInternalChnDomains = async () => { result.map(line => `SUFFIX,${line}`), path.resolve(__dirname, '../List/internal/accelerated-china-domains.txt') ); -}; +}); module.exports.buildInternalChnDomains = buildInternalChnDomains; diff --git a/Build/build-internal-reverse-chn-cidr.js b/Build/build-internal-reverse-chn-cidr.js index 05aa17b6..28c82f3e 100644 --- a/Build/build-internal-reverse-chn-cidr.js +++ b/Build/build-internal-reverse-chn-cidr.js @@ -4,7 +4,7 @@ const { processLine } = require('./lib/process-line'); const path = require('path'); const fse = require('fs-extra'); const fs = require('fs'); -const { runner } = require('./lib/trace-runner'); +const { runner, task } = require('./lib/trace-runner'); const RESERVED_IPV4_CIDR = [ '0.0.0.0/8', @@ -24,7 +24,7 @@ const RESERVED_IPV4_CIDR = [ '240.0.0.0/4' ]; -const buildInternalReverseChnCIDR = async () => { +const buildInternalReverseChnCIDR = task(__filename, async () => { const { exclude } = await import('cidr-tools-wasm'); /** @type {string[]} */ @@ -47,7 +47,7 @@ const buildInternalReverseChnCIDR = async () => { path.resolve(__dirname, '../List/internal/reversed-chn-cidr.txt'), `${reversedCidr.join('\n')}\n` ); -}; +}); module.exports.buildInternalReverseChnCIDR = buildInternalReverseChnCIDR; diff --git a/Build/build-phishing-domainset.js b/Build/build-phishing-domainset.js index bcb38673..d9d9bf2d 100644 --- a/Build/build-phishing-domainset.js +++ b/Build/build-phishing-domainset.js @@ -4,7 +4,7 @@ const path = require('path'); const { createRuleset } = require('./lib/create-file'); const { processLine } = require('./lib/process-line.js'); const domainSorter = require('./lib/stable-sort-domain'); -const { runner, traceSync } = require('./lib/trace-runner.js'); +const { runner, traceSync, task } = require('./lib/trace-runner.js'); const WHITELIST_DOMAIN = new Set([ 'w3s.link', @@ -60,7 +60,7 @@ const BLACK_TLD = new Set([ 'com.cn' ]); -const buildPhishingDomainSet = async () => { +const buildPhishingDomainSet = task(__filename, async () => { const domainSet = Array.from((await processFilterRules('https://curbengh.github.io/phishing-filter/phishing-filter-agh.txt')).black); const domainCountMap = {}; @@ -151,7 +151,7 @@ const buildPhishingDomainSet = async () => { path.resolve(__dirname, '../List/domainset/reject_phishing.conf'), path.resolve(__dirname, '../Clash/domainset/reject_phishing.txt') )); -}; +}); module.exports.buildPhishingDomainSet = buildPhishingDomainSet; diff --git a/Build/build-public.js b/Build/build-public.js index 8ab09d4c..7ce50025 100644 --- a/Build/build-public.js +++ b/Build/build-public.js @@ -2,7 +2,7 @@ const listDir = require('@sukka/listdir'); const path = require('path'); const fs = require('fs'); const fse = require('fs-extra'); -const { runner } = require('./lib/trace-runner'); +const { runner, task } = require('./lib/trace-runner'); const rootPath = path.resolve(__dirname, '../'); const publicPath = path.resolve(__dirname, '../public'); @@ -17,7 +17,7 @@ const folderAndFilesToBeDeployed = [ 'README.md' ]; -const buildPublicHtml = async () => { +const buildPublicHtml = task(__filename, async () => { await fse.ensureDir(publicPath); await Promise.all(folderAndFilesToBeDeployed.map(dir => fse.copy(path.resolve(rootPath, dir), path.resolve(publicPath, dir)))); @@ -29,7 +29,7 @@ const buildPublicHtml = async () => { const html = template(list); await fs.promises.writeFile(path.join(publicPath, 'index.html'), html, 'utf-8'); -}; +}); module.exports.buildPublicHtml = buildPublicHtml; diff --git a/Build/build-reject-domainset.js b/Build/build-reject-domainset.js index 5dcc235b..ce13e3da 100644 --- a/Build/build-reject-domainset.js +++ b/Build/build-reject-domainset.js @@ -12,7 +12,7 @@ const { domainDeduper } = require('./lib/domain-deduper'); const createKeywordFilter = require('./lib/aho-corasick'); const { readFileByLine } = require('./lib/fetch-remote-text-by-line'); const { createDomainSorter } = require('./lib/stable-sort-domain'); -const { traceSync, runner } = require('./lib/trace-runner'); +const { traceSync, runner, task } = require('./lib/trace-runner'); const { getGorhillPublicSuffixPromise } = require('./lib/get-gorhill-publicsuffix'); const { createCachedGorhillGetDomain } = require('./lib/cached-tld-parse'); @@ -23,7 +23,7 @@ const domainKeywordsSet = new Set(); /** @type {Set} Dedupe domains included by DOMAIN-SUFFIX */ const domainSuffixSet = new Set(); -const buildRejectDomainSet = async () => { +const buildRejectDomainSet = task(__dirname, async () => { /** @type Set */ const domainSets = new Set(); @@ -228,7 +228,7 @@ const buildRejectDomainSet = async () => { // Copy reject_sukka.conf for backward compatibility fse.copy(pathResolve(__dirname, '../Source/domainset/reject_sukka.conf'), pathResolve(__dirname, '../List/domainset/reject_sukka.conf')) ]); -}; +}); module.exports.buildRejectDomainSet = buildRejectDomainSet; diff --git a/Build/build-speedtest-domainset.js b/Build/build-speedtest-domainset.js index 84bc8d89..0c34dc04 100644 --- a/Build/build-speedtest-domainset.js +++ b/Build/build-speedtest-domainset.js @@ -5,7 +5,7 @@ const { createRuleset } = require('./lib/create-file'); const domainSorter = require('./lib/stable-sort-domain'); const { Sema } = require('async-sema'); -const { runner } = require('./lib/trace-runner'); +const { runner, task } = require('./lib/trace-runner'); const s = new Sema(2); /** @@ -46,7 +46,7 @@ const querySpeedtestApi = async (keyword) => { } }; -const buildSpeedtestDomainSet = async () => { +const buildSpeedtestDomainSet = task(__filename, async () => { /** @type {Set} */ const domains = new Set([ '.speedtest.net', @@ -123,7 +123,7 @@ const buildSpeedtestDomainSet = async () => { path.resolve(__dirname, '../List/domainset/speedtest.conf'), path.resolve(__dirname, '../Clash/domainset/speedtest.txt') )); -}; +}); module.exports.buildSpeedtestDomainSet = buildSpeedtestDomainSet; diff --git a/Build/build-telegram-cidr.js b/Build/build-telegram-cidr.js index 8cc8ee6e..64212bdb 100644 --- a/Build/build-telegram-cidr.js +++ b/Build/build-telegram-cidr.js @@ -4,9 +4,9 @@ const path = require('path'); const { isIPv4, isIPv6 } = require('net'); const { processLine } = require('./lib/process-line'); const { createRuleset } = require('./lib/create-file'); -const { runner } = require('./lib/trace-runner'); +const { runner, task } = require('./lib/trace-runner'); -const buildTelegramCIDR = async () => { +const buildTelegramCIDR = task(__filename, async () => { /** @type {Response} */ const resp = await fetchWithRetry('https://core.telegram.org/resources/cidr.txt'); const lastModified = resp.headers.get('last-modified'); @@ -49,7 +49,7 @@ const buildTelegramCIDR = async () => { path.resolve(__dirname, '../List/ip/telegram.conf'), path.resolve(__dirname, '../Clash/ip/telegram.txt') )); -}; +}); module.exports.buildTelegramCIDR = buildTelegramCIDR; diff --git a/Build/build.js b/Build/build.js deleted file mode 100644 index e69de29b..00000000 diff --git a/Build/download-previous-build.js b/Build/download-previous-build.js index 10e30522..2bba1bce 100644 --- a/Build/download-previous-build.js +++ b/Build/download-previous-build.js @@ -8,7 +8,7 @@ const { Readable } = require('stream'); const { pipeline } = require('stream/promises'); const { readFileByLine } = require('./lib/fetch-remote-text-by-line'); const { isCI } = require('ci-info'); -const { runner } = require('./lib/trace-runner'); +const { runner, task } = require('./lib/trace-runner'); const fileExists = (path) => { return fs.promises.access(path, fs.constants.F_OK) @@ -16,7 +16,7 @@ const fileExists = (path) => { .catch(() => false); }; -const downloadPreviousBuild = async () => { +const downloadPreviousBuild = task(__filename, async () => { const filesList = ['Clash', 'List']; let allFileExists = true; @@ -79,11 +79,11 @@ const downloadPreviousBuild = async () => { })); await fs.promises.unlink(extractedPath).catch(() => { }); -}; +}); const downloadPublicSuffixList = async () => { const publicSuffixDir = resolve(__dirname, '../node_modules/.cache'); - const publicSuffixPath = join(publicSuffixDir, 'public_suffix-list_dat.txt'); + const publicSuffixPath = join(publicSuffixDir, 'public_suffix_list_dat.txt'); console.log('Download public suffix list.'); @@ -102,10 +102,8 @@ module.exports.downloadPreviousBuild = downloadPreviousBuild; module.exports.downloadPublicSuffixList = downloadPublicSuffixList; if (require.main === module) { - runner(__filename, () => { - return Promise.all([ - downloadPreviousBuild(), - downloadPublicSuffixList() - ]); - }); + runner(__filename, () => Promise.all([ + downloadPreviousBuild(), + downloadPublicSuffixList() + ])); } diff --git a/Build/index.js b/Build/index.js new file mode 100644 index 00000000..37597e75 --- /dev/null +++ b/Build/index.js @@ -0,0 +1,82 @@ +const { downloadPreviousBuild, downloadPublicSuffixList } = require('./download-previous-build'); +const { buildCommon } = require('./build-common'); +const { buildAntiBogusDomain } = require('./build-anti-bogus-domain'); +const { buildAppleCdn } = require('./build-apple-cdn'); +const { buildCdnConf } = require('./build-cdn-conf'); +const { buildPhishingDomainSet } = require('./build-phishing-domainset'); +const { buildRejectDomainSet } = require('./build-reject-domainset'); +const { buildTelegramCIDR } = require('./build-telegram-cidr'); +const { buildChnCidr } = require('./build-chn-cidr'); +const { buildSpeedtestDomainSet } = require('./build-speedtest-domainset'); +const { buildInternalCDNDomains } = require('./build-internal-cdn-rules'); +const { buildInternalReverseChnCIDR } = require('./build-internal-reverse-chn-cidr'); +const { buildInternalChnDomains } = require('./build-internal-chn-domains'); +const { buildDomesticRuleset } = require('./build-domestic-ruleset'); +const { validate } = require('./validate-domainset'); + +const { buildPublicHtml } = require('./build-public'); + +(async () => { + // download-previous-build + const downloadPreviousBuildPromise = downloadPreviousBuild(); + const downloadPublicSuffixListPromise = downloadPublicSuffixList(); + // build:common + const buildCommonPromise = downloadPreviousBuildPromise.then(() => buildCommon()); + // build:anti-bogus-domain + const buildAntiBogusDomainPromise = downloadPreviousBuildPromise.then(() => buildAntiBogusDomain()); + // build:apple-cdn + const buildAppleCdnPromise = downloadPreviousBuildPromise.then(() => buildAppleCdn()); + // build:cdn-conf + const buildCdnConfPromise = Promise.all([ + downloadPreviousBuildPromise, + downloadPublicSuffixListPromise + ]).then(() => buildCdnConf()); + // build:phishing-domainset + const buildPhilishingDomainsetPromise = downloadPreviousBuildPromise.then(() => buildPhishingDomainSet()); + // build:reject-domainset + const buildRejectDomainSetPromise = Promise.all([ + downloadPreviousBuildPromise, + downloadPublicSuffixListPromise, + buildPhilishingDomainsetPromise + ]).then(() => buildRejectDomainSet()); + // build:telegram-cidr + const buildTelegramCIDRPromise = downloadPreviousBuildPromise.then(() => buildTelegramCIDR()); + // build:chn-cidr + const buildChnCidrPromise = downloadPreviousBuildPromise.then(() => buildChnCidr()); + // build:speedtest-domainset + const buildSpeedtestDomainSetPromise = downloadPreviousBuildPromise.then(() => buildSpeedtestDomainSet()); + // build:internal-cdn-rules + const buildInternalCDNDomainsPromise = Promise.all([ + buildCommonPromise, + buildCdnConfPromise + ]).then(() => buildInternalCDNDomains()); + // build:internal-reverse-chn-cidr + const buildInternalReverseChnCIDRPromise = buildInternalReverseChnCIDR(); + // build:internal-chn-domains + const buildInternalChnDomainsPromise = buildInternalChnDomains(); + // build:domestic-ruleset + const buildDomesticRulesetPromise = downloadPreviousBuildPromise.then(() => buildDomesticRuleset()); + + await Promise.all([ + downloadPreviousBuildPromise, + downloadPublicSuffixListPromise, + buildCommonPromise, + buildAntiBogusDomainPromise, + buildAppleCdnPromise, + buildCdnConfPromise, + buildPhilishingDomainsetPromise, + buildRejectDomainSetPromise, + buildTelegramCIDRPromise, + buildChnCidrPromise, + buildSpeedtestDomainSetPromise, + buildInternalCDNDomainsPromise, + buildInternalReverseChnCIDRPromise, + buildInternalChnDomainsPromise, + buildDomesticRulesetPromise + ]); + + await Promise.all([ + buildPublicHtml(), + validate() + ]); +})(); diff --git a/Build/lib/cached-tld-parse.js b/Build/lib/cached-tld-parse.js index 89d86860..fbe7fe50 100644 --- a/Build/lib/cached-tld-parse.js +++ b/Build/lib/cached-tld-parse.js @@ -13,11 +13,12 @@ module.exports.parse = (domain) => { return cache.sync(domain, () => tldts.parse(domain, sharedConfig)); }; -const gothillGetDomainCache = createCache('cached-gorhill-get-domain', true); +let gothillGetDomainCache = null; /** * @param {import('gorhill-publicsuffixlist').default | null} gorhill */ module.exports.createCachedGorhillGetDomain = (gorhill) => { + gothillGetDomainCache ||= createCache('cached-gorhill-get-domain', true); /** * @param {string} domain */ diff --git a/Build/lib/get-gorhill-publicsuffix.js b/Build/lib/get-gorhill-publicsuffix.js index 1787a32c..ffebe56c 100644 --- a/Build/lib/get-gorhill-publicsuffix.js +++ b/Build/lib/get-gorhill-publicsuffix.js @@ -2,7 +2,7 @@ const { toASCII } = require('punycode/'); const fs = require('fs'); const path = require('path'); -const publicSuffixPath = path.resolve(__dirname, '../../node_modules/.cache/public_suffix-list_dat.txt'); +const publicSuffixPath = path.resolve(__dirname, '../../node_modules/.cache/public_suffix_list_dat.txt'); const getPublicSuffixListDat = () => { if (fs.existsSync(publicSuffixPath)) { return fs.promises.readFile(publicSuffixPath, 'utf-8'); diff --git a/Build/lib/trace-runner.js b/Build/lib/trace-runner.js index 5bd75200..b299dfa6 100644 --- a/Build/lib/trace-runner.js +++ b/Build/lib/trace-runner.js @@ -1,3 +1,4 @@ +// @ts-check const path = require('path'); const { performance } = require('perf_hooks'); @@ -35,8 +36,21 @@ module.exports.traceAsync = traceAsync; * @template T * @param {string} __filename * @param {() => Promise} fn - * @returns {T} + * @returns {Promise} */ module.exports.runner = async (__filename, fn) => { return traceAsync(`⌛ [${path.basename(__filename, path.extname(__filename))}]`, fn); }; + +/** + * @template T + * @param {string} __filename + * @param {() => Promise} fn + */ +module.exports.task = (__filename, fn) => { + const taskName = path.basename(__filename, path.extname(__filename)); + return () => { + console.log(`🏃 [${taskName}] Start executing`); + return traceAsync(`✅ [${taskName}] Executed successfully`, fn); + }; +}; diff --git a/Build/validate-domainset.js b/Build/validate-domainset.js index 6dc6f519..131b0c36 100644 --- a/Build/validate-domainset.js +++ b/Build/validate-domainset.js @@ -6,7 +6,7 @@ const path = require('path'); const listDir = require('@sukka/listdir'); const { readFileByLine } = require('./lib/fetch-remote-text-by-line'); const { processLine } = require('./lib/process-line'); -const { runner } = require('./lib/trace-runner'); +const { runner, task } = require('./lib/trace-runner'); const SPECIAL_SUFFIXES = new Set([ 'linodeobjects.com', // only *.linodeobjects.com are public suffix @@ -58,7 +58,7 @@ const _validateRuleset = async (filePath) => { } }; -const validate = async () => { +const validate = task(__filename, async () => { const [domainsetFiles, _rulesetFiles] = await Promise.all([ listDir(path.resolve(__dirname, '../List/domainset')), listDir(path.resolve(__dirname, '../List/non_ip')) @@ -67,7 +67,7 @@ const validate = async () => { domainsetFiles.map(file => validateDomainSet(file)) // rulesetFiles.map(file => validateRuleset(file)) ); -}; +}); module.exports.validate = validate; if (require.main === module) { diff --git a/package.json b/package.json index 3dda6cb8..dbddac0b 100644 --- a/package.json +++ b/package.json @@ -4,145 +4,7 @@ "private": true, "description": "", "scripts": { - "build": "wireit", - "build:common": "wireit", - "download-previous-build": "wireit", - "build:anti-bogus-domain": "wireit", - "build:apple-cdn": "wireit", - "build:cdn-conf": "wireit", - "build:public": "wireit", - "build:reject-domainset": "wireit", - "build:phishing-domainset": "wireit", - "build:telegram-cidr": "wireit", - "build:chn-cidr": "wireit", - "build:speedtest-domainset": "wireit", - "build:internal-cdn-rules": "wireit", - "build:internal-reverse-chn-cidr": "wireit", - "build:internal-chn-domains": "wireit", - "build:domestic-ruleset": "wireit", - "validate:domainset": "wireit" - }, - "wireit": { - "download-previous-build": { - "command": "node ./Build/download-previous-build.js" - }, - "build:common": { - "command": "node ./Build/build-common.js", - "dependencies": [ - "download-previous-build" - ] - }, - "build:anti-bogus-domain": { - "command": "node ./Build/build-anti-bogus-domain.js", - "dependencies": [ - "download-previous-build" - ] - }, - "build:apple-cdn": { - "command": "node ./Build/build-apple-cdn.js", - "dependencies": [ - "download-previous-build" - ] - }, - "build:cdn-conf": { - "command": "node ./Build/build-cdn-conf.js", - "dependencies": [ - "download-previous-build" - ] - }, - "build:phishing-domainset": { - "command": "node ./Build/build-phishing-domainset.js", - "dependencies": [ - "download-previous-build" - ] - }, - "build:reject-domainset": { - "command": "node ./Build/build-reject-domainset.js", - "dependencies": [ - "download-previous-build", - "build:phishing-domainset" - ] - }, - "build:telegram-cidr": { - "command": "node ./Build/build-telegram-cidr.js", - "dependencies": [ - "download-previous-build" - ] - }, - "build:chn-cidr": { - "command": "node ./Build/build-chn-cidr.js", - "dependencies": [ - "download-previous-build" - ] - }, - "build:speedtest-domainset": { - "command": "node ./Build/build-speedtest-domainset.js", - "dependencies": [ - "download-previous-build" - ] - }, - "build:internal-cdn-rules": { - "command": "node ./Build/build-internal-cdn-rules.js", - "dependencies": [ - "build:cdn-conf" - ] - }, - "build:internal-reverse-chn-cidr": { - "command": "node ./Build/build-internal-reverse-chn-cidr.js" - }, - "build:internal-chn-domains": { - "command": "node ./Build/build-internal-chn-domains.js" - }, - "build:domestic-ruleset": { - "command": "node ./Build/build-domestic-ruleset.js", - "dependencies": [ - "download-previous-build" - ] - }, - "build:public": { - "command": "node ./Build/build-public.js", - "dependencies": [ - "build:common", - "build:anti-bogus-domain", - "build:apple-cdn", - "build:cdn-conf", - "build:internal-cdn-rules", - "build:internal-reverse-chn-cidr", - "build:internal-chn-domains", - "build:reject-domainset", - "build:telegram-cidr", - "build:speedtest-domainset", - "build:domestic-ruleset" - ] - }, - "validate:domainset": { - "command": "node ./Build/validate-domainset.js", - "dependencies": [ - "build:apple-cdn", - "build:phishing-domainset", - "build:reject-domainset", - "build:cdn-conf", - "build:common" - ] - }, - "build": { - "dependencies": [ - "build:common", - "build:anti-bogus-domain", - "build:apple-cdn", - "build:cdn-conf", - "build:reject-domainset", - "build:telegram-cidr", - "build:chn-cidr", - "build:internal-cdn-rules", - "build:internal-reverse-chn-cidr", - "build:internal-chn-domains", - "build:speedtest-domainset", - "build:domestic-ruleset", - "build:public", - "validate:domainset" - ] - } + "build": "node ./Build/index.js" }, "repository": { "type": "git", @@ -175,8 +37,7 @@ "eslint-config-sukka": "^3.0.6", "eslint-plugin-import": "npm:eslint-plugin-i@2.28.1", "eslint-plugin-n": "^16.1.0", - "mocha": "^10.2.0", - "wireit": "^0.13.0" + "mocha": "^10.2.0" }, "engines": { "node": ">=18.0.0" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3700625e..ec8b55fd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -84,9 +84,6 @@ devDependencies: mocha: specifier: ^10.2.0 version: 10.2.0 - wireit: - specifier: ^0.13.0 - version: 0.13.0 packages: @@ -780,17 +777,6 @@ packages: /fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} - /fast-glob@3.2.11: - resolution: {integrity: sha512-xrO3+1bxSo3ZVHAnqzyuewYT6aMFHRAd4Kcs92MAonjwQZLsK9d0SF1IyQ3k5PoirxTW0Oe/RqFgMQ6TcNE5Ew==} - engines: {node: '>=8.6.0'} - dependencies: - '@nodelib/fs.stat': 2.0.5 - '@nodelib/fs.walk': 1.2.8 - glob-parent: 5.1.2 - merge2: 1.4.1 - micromatch: 4.0.5 - dev: true - /fast-json-stable-stringify@2.1.0: resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==} dev: true @@ -932,6 +918,7 @@ packages: /graceful-fs@4.2.10: resolution: {integrity: sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==} + dev: false /graphemer@1.4.0: resolution: {integrity: sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==} @@ -1047,10 +1034,6 @@ packages: resolution: {integrity: sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==} dev: true - /jsonc-parser@3.0.0: - resolution: {integrity: sha512-fQzRfAbIBnR0IQvftw9FJveWiHp72Fg20giDrHz6TdfB12UH/uue0D3hm57UB5KgAVuniLMCaS8P1IMj9NR7cA==} - dev: true - /jsonfile@6.1.0: resolution: {integrity: sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==} dependencies: @@ -1108,19 +1091,6 @@ packages: yallist: 4.0.0 dev: true - /merge2@1.4.1: - resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==} - engines: {node: '>= 8'} - dev: true - - /micromatch@4.0.5: - resolution: {integrity: sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==} - engines: {node: '>=8.6'} - dependencies: - braces: 3.0.2 - picomatch: 2.3.1 - dev: true - /minimatch@3.1.2: resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==} dependencies: @@ -1316,14 +1286,6 @@ packages: engines: {node: '>= 0.8.0'} dev: true - /proper-lockfile@4.1.2: - resolution: {integrity: sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==} - dependencies: - graceful-fs: 4.2.10 - retry: 0.12.0 - signal-exit: 3.0.7 - dev: true - /punycode@2.3.0: resolution: {integrity: sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==} engines: {node: '>=6'} @@ -1373,11 +1335,6 @@ packages: supports-preserve-symlinks-flag: 1.0.0 dev: true - /retry@0.12.0: - resolution: {integrity: sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==} - engines: {node: '>= 4'} - dev: true - /retry@0.13.1: resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} engines: {node: '>= 4'} @@ -1431,10 +1388,6 @@ packages: engines: {node: '>=8'} dev: true - /signal-exit@3.0.7: - resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==} - dev: true - /slice-ansi@4.0.0: resolution: {integrity: sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==} engines: {node: '>=10'} @@ -1598,18 +1551,6 @@ packages: isexe: 2.0.0 dev: true - /wireit@0.13.0: - resolution: {integrity: sha512-J8+/kWDBLyDTM2isjEFf9nQZxhuXkgICVt6Nntl9ZyZSAjo/P+nxocwKX6D1jUFfT3TDPStUftsV6OYpx149Ow==} - engines: {node: '>=14.14.0'} - hasBin: true - dependencies: - braces: 3.0.2 - chokidar: 3.5.3 - fast-glob: 3.2.11 - jsonc-parser: 3.0.0 - proper-lockfile: 4.1.2 - dev: true - /workerpool@6.2.1: resolution: {integrity: sha512-ILEIE97kDZvF9Wb9f6h5aXK4swSlKGUcOEGiIYb2OOu/IrDU9iwj0fD//SsA6E5ibwJxpEvhullJY4Sl4GcpAw==} dev: true