diff --git a/Build/build-anti-bogus-domain.js b/Build/build-anti-bogus-domain.js index 5a669bc9..90577070 100644 --- a/Build/build-anti-bogus-domain.js +++ b/Build/build-anti-bogus-domain.js @@ -1,10 +1,9 @@ // @ts-check -const fs = require('fs'); const path = require('path'); const { isIPv4, isIPv6 } = require('net'); const { compareAndWriteFile } = require('./lib/string-array-compare'); const { withBannerArray } = require('./lib/with-banner'); -const { fetchRemoteTextAndCreateReadlineInterface } = require('./lib/fetch-remote-text-by-line'); +const { fetchRemoteTextAndCreateReadlineInterface, readFileByLine } = require('./lib/fetch-remote-text-by-line'); const { minifyRules } = require('./lib/minify-rules'); (async () => { @@ -25,19 +24,22 @@ const { minifyRules } = require('./lib/minify-rules'); const filePath = path.resolve(__dirname, '../Source/ip/reject.conf'); const resultPath = path.resolve(__dirname, '../List/ip/reject.conf'); - const content = (await fs.promises.readFile(filePath, 'utf-8')) - .replace( - '# --- [Anti Bogus Domain Replace Me] ---', - res.map(ip => { + + /** @type {string[]} */ + const result = []; + for await (const line of readFileByLine(filePath)) { + if (line === '# --- [Anti Bogus Domain Replace Me] ---') { + res.forEach(ip => { if (isIPv4(ip)) { - return `IP-CIDR,${ip}/32,no-resolve`; + result.push(`IP-CIDR,${ip}/32,no-resolve`); + } else if (isIPv6(ip)) { + result.push(`IP-CIDR6,${ip}/128,no-resolve`); } - if (isIPv6(ip)) { - return `IP-CIDR6,${ip}/128,no-resolve`; - } - return ''; - }).join('\n') - ); + }); + } else { + result.push(line); + } + } await compareAndWriteFile( withBannerArray( @@ -53,7 +55,7 @@ const { minifyRules } = require('./lib/minify-rules'); ' - https://github.com/felixonmars/dnsmasq-china-list' ], new Date(), - minifyRules(content.split('\n')) + minifyRules(result) ), resultPath ); diff --git a/Build/build-cdn-conf.js b/Build/build-cdn-conf.js index 86be6b9c..9f77e0f4 100644 --- a/Build/build-cdn-conf.js +++ b/Build/build-cdn-conf.js @@ -1,5 +1,4 @@ // @ts-check -const fs = require('fs'); const path = require('path'); const { compareAndWriteFile } = require('./lib/string-array-compare'); const { withBannerArray } = require('./lib/with-banner'); @@ -34,11 +33,15 @@ const { fetchRemoteTextAndCreateReadlineInterface, readFileByLine } = require('. } } - const content = (await fs.promises.readFile(path.resolve(__dirname, '../Source/non_ip/cdn.conf'), 'utf-8')) - .replace( - '# --- [AWS S3 Replace Me] ---', - Array.from(S3OSSDomains).map(domain => `DOMAIN-SUFFIX,${domain}`).join('\n') - ); + /** @type {string[]} */ + const cdnDomainsList = []; + for await (const line of readFileByLine(path.resolve(__dirname, '../Source/non_ip/cdn.conf'))) { + if (line === '# --- [AWS S3 Replace Me] ---') { + S3OSSDomains.forEach(domain => cdnDomainsList.push(`DOMAIN-SUFFIX,${domain}`)); + } else { + cdnDomainsList.push(line); + } + } await compareAndWriteFile( withBannerArray( @@ -51,7 +54,7 @@ const { fetchRemoteTextAndCreateReadlineInterface, readFileByLine } = require('. 'This file contains object storage and static assets CDN domains.' ], new Date(), - minifyRules(content.split('\n')) + minifyRules(cdnDomainsList) ), path.resolve(__dirname, '../List/non_ip/cdn.conf') ); diff --git a/Build/build-reject-domainset.js b/Build/build-reject-domainset.js index 0dfbc20e..65742b0f 100644 --- a/Build/build-reject-domainset.js +++ b/Build/build-reject-domainset.js @@ -209,7 +209,7 @@ const domainSuffixSet = new Set(); }; const sortedDomainSets = dudupedDominArray .map((v) => { - const domain = getDomain(v.charCodeAt(0) === 46 ? v.slice(1) : v) || v; + const domain = getDomain(v[0] === '.' ? v.slice(1) : v) || v; rejectDomainsStats[domain] = (rejectDomainsStats[domain] || 0) + 1; return { v, domain }; }) diff --git a/Build/lib/string-array-compare.js b/Build/lib/string-array-compare.js index ff2f9423..4907d1ad 100644 --- a/Build/lib/string-array-compare.js +++ b/Build/lib/string-array-compare.js @@ -1,6 +1,7 @@ // @ts-check const { promises: fsPromises } = require('fs'); const fse = require('fs-extra'); +const { readFileByLine } = require('./fetch-remote-text-by-line'); /** * @param {string[]} linesA @@ -8,9 +9,26 @@ const fse = require('fs-extra'); */ async function compareAndWriteFile(linesA, filePath) { await fse.ensureFile(filePath); - const linesB = (await fsPromises.readFile(filePath, { encoding: 'utf-8' })).split('\n'); - if (!stringArrayCompare(linesA, linesB)) { + const rl = readFileByLine(filePath); + + let isEqual = true; + let index = 0; + + for await (const lineB of rl) { + const lineA = linesA[index]; + index++; + + if (lineA[0] === '#' && lineB[0] === '#') { + continue; + } + if (lineA !== lineB) { + isEqual = false; + break; + } + } + + if (!isEqual) { await fsPromises.writeFile( filePath, linesA.join('\n'), @@ -21,26 +39,4 @@ async function compareAndWriteFile(linesA, filePath) { } } -/** - * @param {string[]} linesA - * @param {string[]} linesB - */ -function stringArrayCompare(linesA, linesB) { - if (linesA.length !== linesB.length) return false; - - for (let i = 0; i < linesA.length; i++) { - const lineA = linesA[i]; - const lineB = linesB[i]; - if (lineA.startsWith('#') && lineB.startsWith('#')) { - continue; - } - if (lineA !== lineB) { - return false; - } - } - - return true; -} - -module.exports.stringArrayCompare = stringArrayCompare; module.exports.compareAndWriteFile = compareAndWriteFile; diff --git a/Build/lib/with-banner.js b/Build/lib/with-banner.js index f6401485..8ad1f86f 100644 --- a/Build/lib/with-banner.js +++ b/Build/lib/with-banner.js @@ -7,14 +7,15 @@ * @param {string[]} content * @returns {string} */ -const withBanner = (title, description, date, content) => { - return `######################################## -# ${title} -# Last Updated: ${date.toISOString()} -# Size: ${content.length} -${description.map(line => (line ? `# ${line}` : '#')).join('\n')} -########################################\n${content.join('\n')}\n################# END ###################\n`; -}; +// const withBanner = (title, description, date, content) => { +// return `######################################## +// # ${title} +// # Last Updated: ${date.toISOString()} +// # Size: ${content.length} +// ${description.map(line => (line ? `# ${line}` : '#')).join('\n')} +// ########################################\n${content.join('\n')}\n################# END ###################\n`; +// }; + /** * @param {string} title * @param {string[]} description @@ -36,5 +37,5 @@ const withBannerArray = (title, description, date, content) => { ]; }; -module.exports.withBanner = withBanner; +// module.exports.withBanner = withBanner; module.exports.withBannerArray = withBannerArray; diff --git a/Build/validate-domainset.js b/Build/validate-domainset.js index 19b03c83..0180093d 100644 --- a/Build/validate-domainset.js +++ b/Build/validate-domainset.js @@ -2,9 +2,10 @@ const tldts = require('tldts'); const picocolors = require('picocolors'); -const fs = require('fs'); const path = require('path'); const listDir = require('@sukka/listdir'); +const { readFileByLine } = require('./lib/fetch-remote-text-by-line'); +const { processLine } = require('./lib/process-line'); const SPECIAL_SUFFIXES = new Set([ 'linodeobjects.com', // only *.linodeobjects.com are public suffix @@ -13,21 +14,16 @@ const SPECIAL_SUFFIXES = new Set([ ]); const validateDomainSet = async (filePath) => { - const domainSetContent = await fs.promises.readFile( - path.resolve(__dirname, '../List/domainset', filePath), - { encoding: 'utf-8' } + const rl = readFileByLine( + path.resolve(__dirname, '../List/domainset', filePath) ); - const domainSetLines = domainSetContent.split('\n'); - for (let i = 0, len = domainSetLines.length; i < len; i++) { - const line = domainSetLines[i]; - // starts with # - if (line.charCodeAt(0) === 35) { - continue; - } - if (line.trim().length === 0) { - continue; - } + for await (const l of rl) { + // starts with # + const line = processLine(l); + if (!line) { + continue; + } const domain = line.charCodeAt(0) === 46 ? line.slice(1) : line; const parsed = tldts.parse(domain, { allowPrivateDomains: true, detectIp: false }); @@ -43,21 +39,16 @@ const validateDomainSet = async (filePath) => { }; const validateRuleset = async (filePath) => { - const rulesetContent = await fs.promises.readFile( - path.resolve(__dirname, '../List/non_ip', filePath), - { encoding: 'utf-8' } + const rl = readFileByLine( + path.resolve(__dirname, '../List/non_ip', filePath) ); - const rulesetLines = rulesetContent.split('\n'); console.log(`[${filePath}]`); - for (let i = 0, len = rulesetLines.length; i < len; i++) { - const line = rulesetLines[i]; + for await (const l of rl) { // starts with # - if (line.charCodeAt(0) === 35) { - continue; - } - if (line.trim().length === 0) { + const line = processLine(l); + if (!line) { continue; } if (!line.startsWith('DOMAIN-SUFFIX,')) {