From 6da06d8506ad901e9d06f7776988d356350579f7 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Wed, 26 Jul 2023 23:14:41 +0800 Subject: [PATCH] Make CI Faster / Readline util --- Build/build-cdn-conf.js | 11 +++---- Build/build-chn-cidr.js | 1 - Build/build-internal-cdn-rules.js | 16 ++-------- Build/build-reject-domainset.js | 17 +++-------- Build/download-previous-build.js | 42 ++++++++++++-------------- Build/lib/fetch-remote-text-by-line.js | 11 +++++++ 6 files changed, 42 insertions(+), 56 deletions(-) diff --git a/Build/build-cdn-conf.js b/Build/build-cdn-conf.js index e0306839..86be6b9c 100644 --- a/Build/build-cdn-conf.js +++ b/Build/build-cdn-conf.js @@ -6,9 +6,7 @@ const { withBannerArray } = require('./lib/with-banner'); const { minifyRules } = require('./lib/minify-rules'); const { domainDeduper } = require('./lib/domain-deduper'); const { processLine } = require('./lib/process-line'); -const { fetchRemoteTextAndCreateReadlineInterface } = require('./lib/fetch-remote-text-by-line'); - -const readline = require('readline'); +const { fetchRemoteTextAndCreateReadlineInterface, readFileByLine } = require('./lib/fetch-remote-text-by-line'); (async () => { console.time('Total Time - build-cdn-conf'); @@ -64,10 +62,9 @@ const readline = require('readline'); /** @type {Set} */ const cdnDomains = new Set(); - for await (const line of readline.createInterface({ - input: fs.createReadStream(path.resolve(__dirname, '../Source/domainset/cdn.conf'), 'utf-8'), - crlfDelay: Infinity - })) { + for await (const line of readFileByLine( + path.resolve(__dirname, '../Source/domainset/cdn.conf') + )) { const l = processLine(line); if (l) { cdnDomains.add(l); diff --git a/Build/build-chn-cidr.js b/Build/build-chn-cidr.js index dfa0d542..8d32a93a 100644 --- a/Build/build-chn-cidr.js +++ b/Build/build-chn-cidr.js @@ -7,7 +7,6 @@ const { processLine } = require('./lib/process-line'); (async () => { console.time('Total Time - build-chnroutes-cidr'); - // eslint-disable-next-line node/no-unsupported-features/es-syntax -- fuck const { merge: mergeCidrs } = await import('cidr-tools-wasm'); /** @type {Set} */ diff --git a/Build/build-internal-cdn-rules.js b/Build/build-internal-cdn-rules.js index 84ec1010..a1a93d98 100644 --- a/Build/build-internal-cdn-rules.js +++ b/Build/build-internal-cdn-rules.js @@ -2,10 +2,10 @@ const fs = require('fs'); const fse = require('fs-extra'); const path = require('path'); -const readline = require('readline'); const { isDomainLoose } = require('./lib/is-domain-loose'); const tldts = require('tldts'); const { processLine } = require('./lib/process-line'); +const { readFileByLine } = require('./lib/fetch-remote-text-by-line'); /** * @param {string} string @@ -32,12 +32,7 @@ const escapeRegExp = (string) => { * @param {string} domainSetPath */ const processLocalDomainSet = async (domainSetPath) => { - for await ( - const line of readline.createInterface({ - input: fs.createReadStream(domainSetPath), - crlfDelay: Infinity - }) - ) { + for await (const line of readFileByLine(domainSetPath)) { if (line[0] === '.') { addApexDomain(line.slice(1)); } else if (isDomainLoose(line)) { @@ -52,12 +47,7 @@ const escapeRegExp = (string) => { * @param {string} ruleSetPath */ const processLocalRuleSet = async (ruleSetPath) => { - for await ( - const line of readline.createInterface({ - input: fs.createReadStream(ruleSetPath), - crlfDelay: Infinity - }) - ) { + for await (const line of readFileByLine(ruleSetPath)) { if (line.startsWith('DOMAIN-SUFFIX,')) { addApexDomain(line.replace('DOMAIN-SUFFIX,', '')); } else if (line.startsWith('DOMAIN,')) { diff --git a/Build/build-reject-domainset.js b/Build/build-reject-domainset.js index 69ef987b..0dfbc20e 100644 --- a/Build/build-reject-domainset.js +++ b/Build/build-reject-domainset.js @@ -1,7 +1,6 @@ // @ts-check const fs = require('fs'); const fse = require('fs-extra'); -const readline = require('readline'); const { resolve: pathResolve } = require('path'); const { processHosts, processFilterRules } = require('./lib/parse-filter'); @@ -14,6 +13,7 @@ const { compareAndWriteFile } = require('./lib/string-array-compare'); const { processLine } = require('./lib/process-line'); const { domainDeduper } = require('./lib/domain-deduper'); const createKeywordFilter = require('./lib/aho-corasick'); +const { readFileByLine } = require('./lib/fetch-remote-text-by-line'); /** Whitelists */ const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST); @@ -106,10 +106,7 @@ const domainSuffixSet = new Set(); let previousSize = domainSets.size; console.log(`Import ${previousSize} rules from Hosts / AdBlock Filter Rules!`); - const rl1 = readline.createInterface({ - input: fs.createReadStream(pathResolve(__dirname, '../Source/domainset/reject_sukka.conf'), { encoding: 'utf-8' }), - crlfDelay: Infinity - }); + const rl1 = readFileByLine(pathResolve(__dirname, '../Source/domainset/reject_sukka.conf')); for await (const line of rl1) { const l = processLine(line); @@ -121,10 +118,7 @@ const domainSuffixSet = new Set(); previousSize = domainSets.size - previousSize; console.log(`Import ${previousSize} rules from reject_sukka.conf!`); - const rl2 = readline.createInterface({ - input: fs.createReadStream(pathResolve(__dirname, '../List/non_ip/reject.conf'), { encoding: 'utf-8' }), - crlfDelay: Infinity - }); + const rl2 = readFileByLine(pathResolve(__dirname, '../List/non_ip/reject.conf')); for await (const line of rl2) { if (line.startsWith('DOMAIN-KEYWORD')) { const [, ...keywords] = line.split(','); @@ -135,10 +129,7 @@ const domainSuffixSet = new Set(); } } - const rl3 = readline.createInterface({ - input: fs.createReadStream(pathResolve(__dirname, '../List/domainset/reject_phishing.conf'), { encoding: 'utf-8' }), - crlfDelay: Infinity - }); + const rl3 = readFileByLine(pathResolve(__dirname, '../List/domainset/reject_phishing.conf')); for await (const line of rl3) { const l = processLine(line); if (l && l[0] === '.') { diff --git a/Build/download-previous-build.js b/Build/download-previous-build.js index 1e605b18..15f71981 100644 --- a/Build/download-previous-build.js +++ b/Build/download-previous-build.js @@ -6,6 +6,7 @@ const { join, resolve } = require('path'); const { tmpdir } = require('os'); const { Readable } = require('stream'); const { pipeline } = require('stream/promises'); +const { readFileByLine } = require('./lib/fetch-remote-text-by-line'); const fileExists = (path) => { return fs.promises.access(path, fs.constants.F_OK) @@ -14,9 +15,13 @@ const fileExists = (path) => { }; (async () => { - const filesList = (await fs.promises.readFile(resolve(__dirname, '../.gitignore'), { encoding: 'utf-8' })) - .split('\n') - .filter(p => p.startsWith('List/') && !p.endsWith('/')); + const filesList = []; + + for await (const line of readFileByLine(resolve(__dirname, '../.gitignore'))) { + if (line.startsWith('List/') && !line.endsWith('/')) { + filesList.push(line); + } + } if ( !((await Promise.all( @@ -27,32 +32,25 @@ const fileExists = (path) => { return; } - const tempFile = join(tmpdir(), `sukka-surge-last-build-tar-${Date.now()}`); - const resp = await fetch('https://codeload.github.com/sukkaw/surge/tar.gz/gh-pages'); - const readableNodeStream = Readable.fromWeb(resp.body); - await pipeline( - readableNodeStream, - fs.createWriteStream(tempFile) - ); - const extractedPath = join(tmpdir(), `sukka-surge-last-build-extracted-${Date.now()}`); await fse.ensureDir(extractedPath); - await tar.x({ - file: tempFile, - cwd: extractedPath, - filter: (p) => { - return p.split('/')[1] === 'List'; - } - }); + + const resp = await fetch('https://codeload.github.com/sukkaw/surge/tar.gz/gh-pages'); + await pipeline( + Readable.fromWeb(resp.body), + tar.x({ + cwd: extractedPath, + filter(p) { + return p.split('/')[1] === 'List'; + } + }) + ); await Promise.all(filesList.map(p => fse.copy( join(extractedPath, 'Surge-gh-pages', p), join(__dirname, '..', p), - { - overwrite: true - } + { overwrite: true } ))); - await fs.promises.unlink(tempFile).catch(() => { }); await fs.promises.unlink(extractedPath).catch(() => { }); })(); diff --git a/Build/lib/fetch-remote-text-by-line.js b/Build/lib/fetch-remote-text-by-line.js index 7504c378..f4fe3185 100644 --- a/Build/lib/fetch-remote-text-by-line.js +++ b/Build/lib/fetch-remote-text-by-line.js @@ -1,8 +1,19 @@ // @ts-check +const fs = require('fs'); const { fetchWithRetry } = require('./fetch-retry'); const readline = require('readline'); const { Readable } = require('stream'); +/** + * @param {string} path + */ +module.exports.readFileByLine = (path) => { + return readline.createInterface({ + input: fs.createReadStream(path, { encoding: 'utf-8' }), + crlfDelay: Infinity + }); +}; + /** * @param {import('undici').Response} resp */