diff --git a/Build/build-apple-cdn.js b/Build/build-apple-cdn.js index 89527e05..550ee3da 100644 --- a/Build/build-apple-cdn.js +++ b/Build/build-apple-cdn.js @@ -1,26 +1,14 @@ const path = require('path'); -const { isDomainLoose } = require('./lib/is-domain-loose'); const { compareAndWriteFile } = require('./lib/string-array-compare'); const { withBannerArray } = require('./lib/with-banner'); -const { fetchRemoteTextAndCreateReadlineInterface } = require('./lib/fetch-remote-text-by-line'); +const { parseFelixDnsmasq } = require('./lib/parse-dnsmasq'); (async () => { console.time('Total Time - build-apple-cdn-conf'); - const rl = await fetchRemoteTextAndCreateReadlineInterface('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf'); - - /** @type {string[]} */ - const res = []; - for await (const line of rl) { - if (line.startsWith('server=/') && line.endsWith('/114.114.114.114')) { - const domain = line.replace('server=/', '').replace('/114.114.114.114', ''); - if (isDomainLoose(domain)) { - res.push(domain); - } - } - } + const res = await parseFelixDnsmasq('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf'); await Promise.all([ compareAndWriteFile( diff --git a/Build/build-cdn-conf.js b/Build/build-cdn-conf.js index 9f77e0f4..88153427 100644 --- a/Build/build-cdn-conf.js +++ b/Build/build-cdn-conf.js @@ -6,32 +6,29 @@ const { minifyRules } = require('./lib/minify-rules'); const { domainDeduper } = require('./lib/domain-deduper'); const { processLine } = require('./lib/process-line'); const { fetchRemoteTextAndCreateReadlineInterface, readFileByLine } = require('./lib/fetch-remote-text-by-line'); +const Trie = require('./lib/trie'); (async () => { console.time('Total Time - build-cdn-conf'); + const trie = new Trie(); + for await (const line of await fetchRemoteTextAndCreateReadlineInterface('https://publicsuffix.org/list/public_suffix_list.dat')) { + trie.add(line); + } + /** * Extract OSS domain from publicsuffix list * @type {Set} */ const S3OSSDomains = new Set(); - for await (const line of await fetchRemoteTextAndCreateReadlineInterface('https://publicsuffix.org/list/public_suffix_list.dat')) { - if ( - line - && ( - line.startsWith('s3-') - || line.startsWith('s3.') - ) - && ( - line.endsWith('.amazonaws.com') - || line.endsWith('.scw.cloud') - ) - && !line.includes('cn-') - ) { - S3OSSDomains.add(line); - } - } + trie.find('.amazonaws.com') + .filter(line => (line.startsWith('s3-') || line.startsWith('s3.')) && !line.includes('cn-')) + .forEach(line => S3OSSDomains.add(line)); + + trie.find('.scw.cloud') + .filter(line => (line.startsWith('s3-') || line.startsWith('s3.')) && !line.includes('cn-')) + .forEach(line => S3OSSDomains.add(line)); /** @type {string[]} */ const cdnDomainsList = []; diff --git a/Build/build-internal-chn-domains.js b/Build/build-internal-chn-domains.js index 5798362c..88195025 100644 --- a/Build/build-internal-chn-domains.js +++ b/Build/build-internal-chn-domains.js @@ -1,25 +1,15 @@ // @ts-check -const { fetchRemoteTextAndCreateReadlineInterface } = require('./lib/fetch-remote-text-by-line'); -const { processLine } = require('./lib/process-line'); const path = require('path'); const fse = require('fs-extra'); const fs = require('fs'); +const { parseFelixDnsmasq } = require('./lib/parse-dnsmasq'); (async () => { - /** @type {Set} */ - const result = new Set(); - for await (const line of await fetchRemoteTextAndCreateReadlineInterface('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf')) { - const l = processLine(line); - if (l) { - result.add( - l.replace('server=/', '').replace('/114.114.114.114', '') - ); - } - } + const result = await parseFelixDnsmasq('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf'); await fse.ensureDir(path.resolve(__dirname, '../List/internal')); await fs.promises.writeFile( path.resolve(__dirname, '../List/internal/accelerated-china-domains.txt'), - `${Array.from(result).map(line => `SUFFIX,${line}`).join('\n')}\n` + `${result.map(line => `SUFFIX,${line}`).join('\n')}\n` ); })(); diff --git a/Build/build-speedtest-domainset.js b/Build/build-speedtest-domainset.js index 9e15e46d..6215d031 100644 --- a/Build/build-speedtest-domainset.js +++ b/Build/build-speedtest-domainset.js @@ -57,7 +57,8 @@ const querySpeedtestApi = async (keyword) => { '.speedtest.gslnetworks.com', '.speedtest.jsinfo.net', '.speedtest.i3d.net', - '.speedtestkorea.com' + '.speedtestkorea.com', + '.speedtest.telus.com' ]); const hostnameGroups = await Promise.all([ diff --git a/Build/lib/parse-dnsmasq.js b/Build/lib/parse-dnsmasq.js new file mode 100644 index 00000000..30561699 --- /dev/null +++ b/Build/lib/parse-dnsmasq.js @@ -0,0 +1,22 @@ +const { fetchRemoteTextAndCreateReadlineInterface } = require('./fetch-remote-text-by-line'); +const { isDomainLoose } = require('./is-domain-loose'); + +/** + * @param {string | URL} url + */ +const parseFelixDnsmasq = async (url) => { + /** @type {string[]} */ + const res = []; + for await (const line of await fetchRemoteTextAndCreateReadlineInterface(url)) { + if (line.startsWith('server=/') && line.endsWith('/114.114.114.114')) { + const domain = line.replace('server=/', '').replace('/114.114.114.114', ''); + if (isDomainLoose(domain)) { + res.push(domain); + } + } + } + + return res; +}; + +module.exports.parseFelixDnsmasq = parseFelixDnsmasq;