From 573c0f52741faf1a7c787dbb94645658eb497abd Mon Sep 17 00:00:00 2001 From: SukkaW Date: Thu, 14 Sep 2023 22:34:25 +0800 Subject: [PATCH] Refactor: simplify build infra --- Build/build-anti-bogus-domain.js | 4 +- Build/build-apple-cdn.js | 4 +- Build/build-cdn-conf.js | 5 +- Build/build-chn-cidr.js | 4 +- Build/build-common.js | 4 +- Build/build-domestic-ruleset.js | 4 +- Build/build-internal-cdn-rules.js | 4 +- Build/build-internal-chn-domains.js | 4 +- Build/build-internal-reverse-chn-cidr.js | 4 +- Build/build-phishing-domainset.js | 103 ++++++++++++----------- Build/build-public.js | 4 +- Build/build-reject-domainset.js | 17 ++-- Build/build-speedtest-domainset.js | 4 +- Build/build-telegram-cidr.js | 4 +- Build/download-previous-build.js | 10 +-- Build/lib/create-file.js | 30 ++++--- Build/lib/is-domain-loose.js | 9 -- Build/lib/parse-dnsmasq.js | 7 +- Build/lib/parse-filter.js | 9 +- Build/lib/stable-sort-domain.js | 6 +- Build/lib/trace-runner.js | 15 +--- Build/validate-domainset.js | 4 +- 22 files changed, 127 insertions(+), 132 deletions(-) diff --git a/Build/build-anti-bogus-domain.js b/Build/build-anti-bogus-domain.js index 4fa4e22c..0375e587 100644 --- a/Build/build-anti-bogus-domain.js +++ b/Build/build-anti-bogus-domain.js @@ -4,7 +4,7 @@ const { isIPv4, isIPv6 } = require('net'); const { createRuleset } = require('./lib/create-file'); const { fetchRemoteTextAndCreateReadlineInterface, readFileByLine } = require('./lib/fetch-remote-text-by-line'); const { processLine } = require('./lib/process-line'); -const { runner, task } = require('./lib/trace-runner'); +const { task } = require('./lib/trace-runner'); const buildAntiBogusDomain = task(__filename, async () => { /** @type {string[]} */ @@ -61,5 +61,5 @@ const buildAntiBogusDomain = task(__filename, async () => { module.exports.buildAntiBogusDomain = buildAntiBogusDomain; if (require.main === module) { - runner(__filename, buildAntiBogusDomain); + buildAntiBogusDomain(); } diff --git a/Build/build-apple-cdn.js b/Build/build-apple-cdn.js index 7459b97a..0a1d3866 100644 --- a/Build/build-apple-cdn.js +++ b/Build/build-apple-cdn.js @@ -1,7 +1,7 @@ const path = require('path'); const { createRuleset } = require('./lib/create-file'); const { parseFelixDnsmasq } = require('./lib/parse-dnsmasq'); -const { runner, task } = require('./lib/trace-runner'); +const { task } = require('./lib/trace-runner'); const buildAppleCdn = task(__filename, async () => { const res = await parseFelixDnsmasq('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf'); @@ -45,5 +45,5 @@ const buildAppleCdn = task(__filename, async () => { module.exports.buildAppleCdn = buildAppleCdn; if (require.main === module) { - runner(__filename, buildAppleCdn); + buildAppleCdn(); } diff --git a/Build/build-cdn-conf.js b/Build/build-cdn-conf.js index e2795a11..9a3b14f3 100644 --- a/Build/build-cdn-conf.js +++ b/Build/build-cdn-conf.js @@ -3,7 +3,7 @@ const path = require('path'); const { createRuleset } = require('./lib/create-file'); const { fetchRemoteTextAndCreateReadlineInterface, readFileByLine } = require('./lib/fetch-remote-text-by-line'); const Trie = require('./lib/trie'); -const { runner, task } = require('./lib/trace-runner'); +const { task } = require('./lib/trace-runner'); const fs = require('fs'); const { processLine } = require('./lib/process-line'); @@ -50,7 +50,6 @@ const buildCdnConf = task(__filename, async () => { const cdnDomainsList = []; for await (const l of readFileByLine(path.resolve(__dirname, '../Source/non_ip/cdn.conf'))) { if (l === '# --- [AWS S3 Replace Me] ---') { - console.log(S3OSSDomains); S3OSSDomains.forEach(domain => { cdnDomainsList.push(`DOMAIN-SUFFIX,${domain}`); }); @@ -83,5 +82,5 @@ const buildCdnConf = task(__filename, async () => { module.exports.buildCdnConf = buildCdnConf; if (require.main === module) { - runner(__filename, buildCdnConf); + buildCdnConf(); } diff --git a/Build/build-chn-cidr.js b/Build/build-chn-cidr.js index 318e25db..3b71b42d 100644 --- a/Build/build-chn-cidr.js +++ b/Build/build-chn-cidr.js @@ -4,7 +4,7 @@ const { resolve: pathResolve } = require('path'); // This should not use `createRuleset` API since we are going to generate ipcidr for Clash const { compareAndWriteFile, withBannerArray } = require('./lib/create-file'); const { processLine } = require('./lib/process-line'); -const { runner, task } = require('./lib/trace-runner'); +const { task } = require('./lib/trace-runner'); // https://github.com/misakaio/chnroutes2/issues/25 const EXCLUDE_CIDRS = [ @@ -61,5 +61,5 @@ const buildChnCidr = task(__filename, async () => { module.exports.buildChnCidr = buildChnCidr; if (require.main === module) { - runner(__filename, buildChnCidr); + buildChnCidr(); } diff --git a/Build/build-common.js b/Build/build-common.js index 7c6c5eec..8a6ce775 100644 --- a/Build/build-common.js +++ b/Build/build-common.js @@ -6,7 +6,7 @@ const { readFileByLine } = require('./lib/fetch-remote-text-by-line'); const { processLine } = require('./lib/process-line'); const { createRuleset } = require('./lib/create-file'); const { domainDeduper } = require('./lib/domain-deduper'); -const { runner, task } = require('./lib/trace-runner'); +const { task } = require('./lib/trace-runner'); const MAGIC_COMMAND_SKIP = '# $ custom_build_script'; const MAGIC_COMMAND_TITLE = '# $ meta_title '; @@ -48,7 +48,7 @@ const buildCommon = task(__filename, async () => { module.exports.buildCommon = buildCommon; if (require.main === module) { - runner(__filename, buildCommon); + buildCommon(); } /** diff --git a/Build/build-domestic-ruleset.js b/Build/build-domestic-ruleset.js index 74bd9b32..105c1214 100644 --- a/Build/build-domestic-ruleset.js +++ b/Build/build-domestic-ruleset.js @@ -5,7 +5,7 @@ const { readFileByLine } = require('./lib/fetch-remote-text-by-line'); const { processLine } = require('./lib/process-line'); const { compareAndWriteFile, createRuleset } = require('./lib/create-file'); const domainSorter = require('./lib/stable-sort-domain'); -const { runner, task } = require('./lib/trace-runner'); +const { task } = require('./lib/trace-runner'); const buildDomesticRuleset = task(__filename, async () => { const rl = readFileByLine(path.resolve(__dirname, '../Source/non_ip/domestic.conf')); @@ -72,5 +72,5 @@ const buildDomesticRuleset = task(__filename, async () => { module.exports.buildDomesticRuleset = buildDomesticRuleset; if (require.main === module) { - runner(__filename, buildDomesticRuleset); + buildDomesticRuleset(); } diff --git a/Build/build-internal-cdn-rules.js b/Build/build-internal-cdn-rules.js index b967e286..eb5daf96 100644 --- a/Build/build-internal-cdn-rules.js +++ b/Build/build-internal-cdn-rules.js @@ -5,7 +5,7 @@ const tldts = require('tldts'); const { processLine } = require('./lib/process-line'); const { readFileByLine } = require('./lib/fetch-remote-text-by-line'); const domainSorter = require('./lib/stable-sort-domain'); -const { runner, task } = require('./lib/trace-runner'); +const { task } = require('./lib/trace-runner'); const { compareAndWriteFile } = require('./lib/create-file'); /** @@ -92,5 +92,5 @@ const buildInternalCDNDomains = task(__filename, async () => { module.exports.buildInternalCDNDomains = buildInternalCDNDomains; if (require.main === module) { - runner(__filename, buildInternalCDNDomains); + buildInternalCDNDomains(); } diff --git a/Build/build-internal-chn-domains.js b/Build/build-internal-chn-domains.js index 70869ec7..cc3a9ec4 100644 --- a/Build/build-internal-chn-domains.js +++ b/Build/build-internal-chn-domains.js @@ -2,7 +2,7 @@ const path = require('path'); const fse = require('fs-extra'); const { parseFelixDnsmasq } = require('./lib/parse-dnsmasq'); -const { runner, task } = require('./lib/trace-runner'); +const { task } = require('./lib/trace-runner'); const { compareAndWriteFile } = require('./lib/create-file'); const buildInternalChnDomains = task(__filename, async () => { @@ -20,5 +20,5 @@ const buildInternalChnDomains = task(__filename, async () => { module.exports.buildInternalChnDomains = buildInternalChnDomains; if (require.main === module) { - runner(__filename, buildInternalChnDomains); + buildInternalChnDomains(); } diff --git a/Build/build-internal-reverse-chn-cidr.js b/Build/build-internal-reverse-chn-cidr.js index 28c82f3e..427dd02d 100644 --- a/Build/build-internal-reverse-chn-cidr.js +++ b/Build/build-internal-reverse-chn-cidr.js @@ -4,7 +4,7 @@ const { processLine } = require('./lib/process-line'); const path = require('path'); const fse = require('fs-extra'); const fs = require('fs'); -const { runner, task } = require('./lib/trace-runner'); +const { task } = require('./lib/trace-runner'); const RESERVED_IPV4_CIDR = [ '0.0.0.0/8', @@ -52,5 +52,5 @@ const buildInternalReverseChnCIDR = task(__filename, async () => { module.exports.buildInternalReverseChnCIDR = buildInternalReverseChnCIDR; if (require.main === module) { - runner(__filename, buildInternalReverseChnCIDR); + buildInternalReverseChnCIDR(); } diff --git a/Build/build-phishing-domainset.js b/Build/build-phishing-domainset.js index d9d9bf2d..2acfc004 100644 --- a/Build/build-phishing-domainset.js +++ b/Build/build-phishing-domainset.js @@ -4,7 +4,7 @@ const path = require('path'); const { createRuleset } = require('./lib/create-file'); const { processLine } = require('./lib/process-line.js'); const domainSorter = require('./lib/stable-sort-domain'); -const { runner, traceSync, task } = require('./lib/trace-runner.js'); +const { traceSync, task } = require('./lib/trace-runner.js'); const WHITELIST_DOMAIN = new Set([ 'w3s.link', @@ -61,67 +61,76 @@ const BLACK_TLD = new Set([ ]); const buildPhishingDomainSet = task(__filename, async () => { - const domainSet = Array.from((await processFilterRules('https://curbengh.github.io/phishing-filter/phishing-filter-agh.txt')).black); + const domainSet = Array.from((await processFilterRules( + 'https://phishing-filter.pages.dev/phishing-filter-agh.txt' + // [ + // 'https://malware-filter.gitlab.io/phishing-filter/phishing-filter-agh.txt', + // 'https://malware-filter.pages.dev/phishing-filter-agh.txt', + // 'https://phishing-filter.pages.dev/phishing-filter-agh.txt' + // ] + )).black); const domainCountMap = {}; - for (let i = 0, len = domainSet.length; i < len; i++) { - const line = processLine(domainSet[i]); - if (!line) continue; + traceSync('* process domain set', () => { + for (let i = 0, len = domainSet.length; i < len; i++) { + const line = processLine(domainSet[i]); + if (!line) continue; - const parsed = tldts.parse(line, { allowPrivateDomains: true }); - const apexDomain = parsed.domain; + const parsed = tldts.parse(line, { allowPrivateDomains: true }); + const apexDomain = parsed.domain; - if (apexDomain) { - if (WHITELIST_DOMAIN.has(apexDomain)) { - continue; - } - - domainCountMap[apexDomain] ||= 0; - - let isPhishingDomainMockingAmazon = false; - if (line.startsWith('.amaz')) { - domainCountMap[apexDomain] += 0.5; - - isPhishingDomainMockingAmazon = true; - - if (line.startsWith('.amazon-')) { - domainCountMap[apexDomain] += 4.5; + if (apexDomain) { + if (WHITELIST_DOMAIN.has(apexDomain)) { + continue; } - } else if (line.startsWith('.customer')) { - domainCountMap[apexDomain] += 0.25; - } - if (line.includes('-co-jp')) { - domainCountMap[apexDomain] += (isPhishingDomainMockingAmazon ? 4.5 : 0.5); - } - const tld = parsed.publicSuffix; - if (!tld || !BLACK_TLD.has(tld)) continue; + domainCountMap[apexDomain] ||= 0; - domainCountMap[apexDomain] += 1; + let isPhishingDomainMockingAmazon = false; + if (line.startsWith('.amaz')) { + domainCountMap[apexDomain] += 0.5; - if (line.length > 19) { - // Add more weight if the domain is long enough - if (line.length > 44) { - domainCountMap[apexDomain] += 3.5; - } else if (line.length > 34) { - domainCountMap[apexDomain] += 2.5; - } else if (line.length > 29) { - domainCountMap[apexDomain] += 1.5; - } else if (line.length > 24) { - domainCountMap[apexDomain] += 0.75; - } else if (line.length > 19) { + isPhishingDomainMockingAmazon = true; + + if (line.startsWith('.amazon-')) { + domainCountMap[apexDomain] += 4.5; + } + } else if (line.startsWith('.customer')) { domainCountMap[apexDomain] += 0.25; } + if (line.includes('-co-jp')) { + domainCountMap[apexDomain] += (isPhishingDomainMockingAmazon ? 4.5 : 0.5); + } - if (domainCountMap[apexDomain] < 5) { - const subdomain = parsed.subdomain; - if (subdomain && subdomain.includes('.')) { + const tld = parsed.publicSuffix; + if (!tld || !BLACK_TLD.has(tld)) continue; + + domainCountMap[apexDomain] += 1; + + if (line.length > 19) { + // Add more weight if the domain is long enough + if (line.length > 44) { + domainCountMap[apexDomain] += 3.5; + } else if (line.length > 34) { + domainCountMap[apexDomain] += 2.5; + } else if (line.length > 29) { domainCountMap[apexDomain] += 1.5; + } else if (line.length > 24) { + domainCountMap[apexDomain] += 0.75; + } else if (line.length > 19) { + domainCountMap[apexDomain] += 0.25; + } + + if (domainCountMap[apexDomain] < 5) { + const subdomain = parsed.subdomain; + if (subdomain?.includes('.')) { + domainCountMap[apexDomain] += 1.5; + } } } } } - } + }); const results = traceSync('* get final results', () => Object.entries(domainCountMap) .reduce((acc, [apexDomain, count]) => { @@ -156,5 +165,5 @@ const buildPhishingDomainSet = task(__filename, async () => { module.exports.buildPhishingDomainSet = buildPhishingDomainSet; if (require.main === module) { - runner(__filename, buildPhishingDomainSet); + buildPhishingDomainSet(); } diff --git a/Build/build-public.js b/Build/build-public.js index 7ce50025..569bebd3 100644 --- a/Build/build-public.js +++ b/Build/build-public.js @@ -2,7 +2,7 @@ const listDir = require('@sukka/listdir'); const path = require('path'); const fs = require('fs'); const fse = require('fs-extra'); -const { runner, task } = require('./lib/trace-runner'); +const { task } = require('./lib/trace-runner'); const rootPath = path.resolve(__dirname, '../'); const publicPath = path.resolve(__dirname, '../public'); @@ -34,7 +34,7 @@ const buildPublicHtml = task(__filename, async () => { module.exports.buildPublicHtml = buildPublicHtml; if (require.main === module) { - runner(__filename, buildPublicHtml); + buildPublicHtml(); } /** diff --git a/Build/build-reject-domainset.js b/Build/build-reject-domainset.js index 00974cda..bd1ecd6b 100644 --- a/Build/build-reject-domainset.js +++ b/Build/build-reject-domainset.js @@ -12,7 +12,7 @@ const { domainDeduper } = require('./lib/domain-deduper'); const createKeywordFilter = require('./lib/aho-corasick'); const { readFileByLine } = require('./lib/fetch-remote-text-by-line'); const { createDomainSorter } = require('./lib/stable-sort-domain'); -const { traceSync, runner, task } = require('./lib/trace-runner'); +const { traceSync, task } = require('./lib/trace-runner'); const { getGorhillPublicSuffixPromise } = require('./lib/get-gorhill-publicsuffix'); const { createCachedGorhillGetDomain } = require('./lib/cached-tld-parse'); @@ -23,7 +23,7 @@ const domainKeywordsSet = new Set(); /** @type {Set} Dedupe domains included by DOMAIN-SUFFIX */ const domainSuffixSet = new Set(); -const buildRejectDomainSet = task(__dirname, async () => { +const buildRejectDomainSet = task(__filename, async () => { /** @type Set */ const domainSets = new Set(); @@ -167,11 +167,7 @@ const buildRejectDomainSet = task(__dirname, async () => { // Dedupe domainSets console.log(`Start deduping! (${previousSize})`); - const START_TIME = Date.now(); - const dudupedDominArray = traceSync('* Dedupe from covered subdomain', () => domainDeduper(Array.from(domainSets))); - - console.log(`* Dedupe from covered subdomain - ${(Date.now() - START_TIME) / 1000}s`); console.log(`Deduped ${previousSize - dudupedDominArray.length} rules!`); // Create reject stats @@ -189,10 +185,11 @@ const buildRejectDomainSet = task(__dirname, async () => { }, {}) ).filter(a => a[1] > 2).sort((a, b) => { const t = b[1] - a[1]; - if (t === 0) { - return a[0].localeCompare(b[0]); + if (t !== 0) { + return t; } - return t; + + return a[0].localeCompare(b[0]); }) ); @@ -233,5 +230,5 @@ const buildRejectDomainSet = task(__dirname, async () => { module.exports.buildRejectDomainSet = buildRejectDomainSet; if (require.main === module) { - runner(__filename, buildRejectDomainSet); + buildRejectDomainSet(); } diff --git a/Build/build-speedtest-domainset.js b/Build/build-speedtest-domainset.js index 0c34dc04..9559309a 100644 --- a/Build/build-speedtest-domainset.js +++ b/Build/build-speedtest-domainset.js @@ -5,7 +5,7 @@ const { createRuleset } = require('./lib/create-file'); const domainSorter = require('./lib/stable-sort-domain'); const { Sema } = require('async-sema'); -const { runner, task } = require('./lib/trace-runner'); +const { task } = require('./lib/trace-runner'); const s = new Sema(2); /** @@ -128,5 +128,5 @@ const buildSpeedtestDomainSet = task(__filename, async () => { module.exports.buildSpeedtestDomainSet = buildSpeedtestDomainSet; if (require.main === module) { - runner(__filename, buildSpeedtestDomainSet); + buildSpeedtestDomainSet(); } diff --git a/Build/build-telegram-cidr.js b/Build/build-telegram-cidr.js index 64212bdb..f5a0f12a 100644 --- a/Build/build-telegram-cidr.js +++ b/Build/build-telegram-cidr.js @@ -4,7 +4,7 @@ const path = require('path'); const { isIPv4, isIPv6 } = require('net'); const { processLine } = require('./lib/process-line'); const { createRuleset } = require('./lib/create-file'); -const { runner, task } = require('./lib/trace-runner'); +const { task } = require('./lib/trace-runner'); const buildTelegramCIDR = task(__filename, async () => { /** @type {Response} */ @@ -54,5 +54,5 @@ const buildTelegramCIDR = task(__filename, async () => { module.exports.buildTelegramCIDR = buildTelegramCIDR; if (require.main === module) { - runner(__filename, buildTelegramCIDR); + buildTelegramCIDR(); } diff --git a/Build/download-previous-build.js b/Build/download-previous-build.js index 2bba1bce..3eb8aed3 100644 --- a/Build/download-previous-build.js +++ b/Build/download-previous-build.js @@ -8,7 +8,7 @@ const { Readable } = require('stream'); const { pipeline } = require('stream/promises'); const { readFileByLine } = require('./lib/fetch-remote-text-by-line'); const { isCI } = require('ci-info'); -const { runner, task } = require('./lib/trace-runner'); +const { task } = require('./lib/trace-runner'); const fileExists = (path) => { return fs.promises.access(path, fs.constants.F_OK) @@ -81,7 +81,7 @@ const downloadPreviousBuild = task(__filename, async () => { await fs.promises.unlink(extractedPath).catch(() => { }); }); -const downloadPublicSuffixList = async () => { +const downloadPublicSuffixList = task(__filename, async () => { const publicSuffixDir = resolve(__dirname, '../node_modules/.cache'); const publicSuffixPath = join(publicSuffixDir, 'public_suffix_list_dat.txt'); @@ -96,14 +96,14 @@ const downloadPublicSuffixList = async () => { Readable.fromWeb(resp.body), fs.createWriteStream(publicSuffixPath) ); -}; +}, 'download-publicsuffixlist'); module.exports.downloadPreviousBuild = downloadPreviousBuild; module.exports.downloadPublicSuffixList = downloadPublicSuffixList; if (require.main === module) { - runner(__filename, () => Promise.all([ + Promise.all([ downloadPreviousBuild(), downloadPublicSuffixList() - ])); + ]); } diff --git a/Build/lib/create-file.js b/Build/lib/create-file.js index 6be013e4..a4345afd 100644 --- a/Build/lib/create-file.js +++ b/Build/lib/create-file.js @@ -1,6 +1,5 @@ // @ts-check const fs = require('fs'); -const fse = require('fs-extra'); const { readFileByLine } = require('./fetch-remote-text-by-line'); const { surgeDomainsetToClashDomainset, surgeRulesetToClashClassicalTextRuleset } = require('./clash'); @@ -9,26 +8,33 @@ const { surgeDomainsetToClashDomainset, surgeRulesetToClashClassicalTextRuleset * @param {string} filePath */ async function compareAndWriteFile(linesA, filePath) { - await fse.ensureFile(filePath); - let isEqual = true; - let index = 0; + if (!fs.existsSync(filePath)) { + console.log(`${filePath} does not exists, writing...`); + isEqual = false; + } else { + let index = 0; - for await (const lineB of readFileByLine(filePath)) { - const lineA = linesA[index]; - index++; + for await (const lineB of readFileByLine(filePath)) { + const lineA = linesA[index]; + index++; - if (lineA[0] === '#' && lineB[0] === '#') { - continue; + if (lineA[0] === '#' && lineB[0] === '#') { + continue; + } + + if (lineA !== lineB) { + isEqual = false; + break; + } } - if (lineA !== lineB) { + if (index !== linesA.length) { isEqual = false; - break; } } - if (!isEqual || index !== linesA.length) { + if (!isEqual) { const stream = fs.createWriteStream(filePath, { encoding: 'utf-8' }); for (let i = 0, len = linesA.length; i < len; i++) { diff --git a/Build/lib/is-domain-loose.js b/Build/lib/is-domain-loose.js index 7b170469..4197c564 100644 --- a/Build/lib/is-domain-loose.js +++ b/Build/lib/is-domain-loose.js @@ -1,14 +1,5 @@ // @ts-check const tldts = require('./cached-tld-parse'); - -/** - * @param {string} domain - */ -module.exports.isDomainLoose = (domain) => { - const { isIcann, isPrivate, isIp } = tldts.parse(domain); - return !!(!isIp && (isIcann || isPrivate)); -}; - /** * @param {string | null | undefined} domain */ diff --git a/Build/lib/parse-dnsmasq.js b/Build/lib/parse-dnsmasq.js index 30561699..0051802b 100644 --- a/Build/lib/parse-dnsmasq.js +++ b/Build/lib/parse-dnsmasq.js @@ -1,5 +1,10 @@ const { fetchRemoteTextAndCreateReadlineInterface } = require('./fetch-remote-text-by-line'); -const { isDomainLoose } = require('./is-domain-loose'); +const tldts = require('tldts'); + +const isDomainLoose = (domain) => { + const { isIcann, isPrivate, isIp } = tldts.parse(domain); + return !!(!isIp && (isIcann || isPrivate)); +}; /** * @param {string | URL} url diff --git a/Build/lib/parse-filter.js b/Build/lib/parse-filter.js index 70ca4af9..7489c968 100644 --- a/Build/lib/parse-filter.js +++ b/Build/lib/parse-filter.js @@ -93,9 +93,6 @@ async function processHosts(hostsUrl, includeAllSubDomain = false) { return domainSets; } -const R_KNOWN_NOT_NETWORK_FILTER_PATTERN = /[#&%~=]/; -const R_KNOWN_NOT_NETWORK_FILTER_PATTERN_2 = /(\$popup|\$removeparam|\$popunder)/; - /** * @param {string | URL} filterRulesUrl * @param {readonly (string | URL)[] | undefined} [fallbackUrls] @@ -197,8 +194,7 @@ async function processFilterRules(filterRulesUrl, fallbackUrls, includeThirdPart downloadTime = performance.now() - downloadStart; for (let i = 0, len = filterRules.length; i < len; i++) { - const line = filterRules[i].trim(); - lineCb(line); + lineCb(filterRules[i].trim()); } } @@ -212,6 +208,9 @@ async function processFilterRules(filterRulesUrl, fallbackUrls, includeThirdPart }; } +const R_KNOWN_NOT_NETWORK_FILTER_PATTERN = /[#&%~=]/; +const R_KNOWN_NOT_NETWORK_FILTER_PATTERN_2 = /(\$popup|\$removeparam|\$popunder)/; + /** * @param {string} $line * @param {boolean} includeThirdParties diff --git a/Build/lib/stable-sort-domain.js b/Build/lib/stable-sort-domain.js index 4c3138b7..cb3e4746 100644 --- a/Build/lib/stable-sort-domain.js +++ b/Build/lib/stable-sort-domain.js @@ -42,13 +42,11 @@ const compare = (a, b) => { * @param {import('gorhill-publicsuffixlist').default | null} [gorhill] */ const createDomainSorter = (gorhill = null) => { - const cached = require('./cached-tld-parse'); - if (gorhill) { /** * @param {string} input */ - const getDomain = cached.createCachedGorhillGetDomain(gorhill); + const getDomain = require('./cached-tld-parse').createCachedGorhillGetDomain(gorhill); /** * @param {string} a @@ -66,7 +64,7 @@ const createDomainSorter = (gorhill = null) => { }; } - const tldts = cached; + const tldts = require('./cached-tld-parse'); /** * @param {string} a * @param {string} b diff --git a/Build/lib/trace-runner.js b/Build/lib/trace-runner.js index b299dfa6..b6a80d17 100644 --- a/Build/lib/trace-runner.js +++ b/Build/lib/trace-runner.js @@ -36,19 +36,10 @@ module.exports.traceAsync = traceAsync; * @template T * @param {string} __filename * @param {() => Promise} fn - * @returns {Promise} + * @param {string | null} [customname] */ -module.exports.runner = async (__filename, fn) => { - return traceAsync(`⌛ [${path.basename(__filename, path.extname(__filename))}]`, fn); -}; - -/** - * @template T - * @param {string} __filename - * @param {() => Promise} fn - */ -module.exports.task = (__filename, fn) => { - const taskName = path.basename(__filename, path.extname(__filename)); +module.exports.task = (__filename, fn, customname = null) => { + const taskName = customname ?? path.basename(__filename, path.extname(__filename)); return () => { console.log(`🏃 [${taskName}] Start executing`); return traceAsync(`✅ [${taskName}] Executed successfully`, fn); diff --git a/Build/validate-domainset.js b/Build/validate-domainset.js index 131b0c36..011f65ea 100644 --- a/Build/validate-domainset.js +++ b/Build/validate-domainset.js @@ -6,7 +6,7 @@ const path = require('path'); const listDir = require('@sukka/listdir'); const { readFileByLine } = require('./lib/fetch-remote-text-by-line'); const { processLine } = require('./lib/process-line'); -const { runner, task } = require('./lib/trace-runner'); +const { task } = require('./lib/trace-runner'); const SPECIAL_SUFFIXES = new Set([ 'linodeobjects.com', // only *.linodeobjects.com are public suffix @@ -71,5 +71,5 @@ const validate = task(__filename, async () => { module.exports.validate = validate; if (require.main === module) { - runner(__filename, validate); + validate(); }