diff --git a/Build/build-internal-cdn-rules.js b/Build/build-internal-cdn-rules.js index 404a6a93..7b2f93ff 100644 --- a/Build/build-internal-cdn-rules.js +++ b/Build/build-internal-cdn-rules.js @@ -6,6 +6,7 @@ const { isDomainLoose } = require('./lib/is-domain-loose'); const tldts = require('tldts'); const { processLine } = require('./lib/process-line'); const { readFileByLine } = require('./lib/fetch-remote-text-by-line'); +const domainSorter = require('./lib/stable-sort-domain'); /** * @param {string} string @@ -78,8 +79,8 @@ const escapeRegExp = (string) => { await fs.promises.writeFile( path.resolve(__dirname, '../List/internal/cdn.txt'), [ - ...Array.from(set).map(i => `SUFFIX,${i}`), - ...Array.from(keywords).map(i => `REGEX,${i}`), + ...Array.from(set).sort(domainSorter).map(i => `SUFFIX,${i}`), + ...Array.from(keywords).sort().map(i => `REGEX,${i}`), '' ].join('\n') ); diff --git a/Build/build-reject-domainset.js b/Build/build-reject-domainset.js index 264e4200..59f50566 100644 --- a/Build/build-reject-domainset.js +++ b/Build/build-reject-domainset.js @@ -1,8 +1,10 @@ // @ts-check const fs = require('fs'); const fse = require('fs-extra'); - const { resolve: pathResolve } = require('path'); + +const tldts = require('tldts'); + const { processHosts, processFilterRules } = require('./lib/parse-filter'); const Trie = require('./lib/trie'); @@ -186,10 +188,13 @@ const domainSuffixSet = new Set(); console.time('* Write reject.conf'); /** @type {Record} */ - const rejectDomainsStats = {}; - - const sortedDomainSets = dudupedDominArray - .sort(domainSorter); + const rejectDomainsStats = dudupedDominArray.reduce((acc, cur) => { + const suffix = tldts.getDomain(cur, { allowPrivateDomains: false }); + if (suffix) { + acc[suffix] = (acc[suffix] ?? 0) + 1; + } + return acc; + }, {}); await Promise.all([ compareAndWriteFile( @@ -207,7 +212,7 @@ const domainSuffixSet = new Set(); ...ADGUARD_FILTERS.map(filter => ` - ${Array.isArray(filter) ? filter[0] : filter}`) ], new Date(), - sortedDomainSets + dudupedDominArray.sort(domainSorter) ), pathResolve(__dirname, '../List/domainset/reject.conf') ), diff --git a/Build/lib/stable-sort-domain.js b/Build/lib/stable-sort-domain.js index 09fdb4c5..27c1299d 100644 --- a/Build/lib/stable-sort-domain.js +++ b/Build/lib/stable-sort-domain.js @@ -9,6 +9,45 @@ const cache1 = Object.create(null); // eslint-disable-next-line no-return-assign -- cache const parse = (url) => (cache1[url] ||= tldts.parse(url, { allowPrivateDomains: true })); +/** + * @param {string | null} a + * @param {string | null} b + * @returns {0 | 1 | -1} + */ +const compare = (a, b) => { + if (a === b) return 0; + if (b == null) { + return 1; + } + if (a == null) { + return -1; + } + + if (a.length !== b.length) { + const r = a.length - b.length; + if (r > 0) { + return 1; + } + if (r < 0) { + return -1; + } + return 0; + } + + for (let i = 0; i < a.length; i++) { + if (b[i] == null) { + return 1; + } + if (a[i] < b[i]) { + return -1; + } + if (a[i] > b[i]) { + return 1; + } + } + return 0; +}; + /** * @param {string} a * @param {string} b @@ -17,85 +56,22 @@ const parse = (url) => (cache1[url] ||= tldts.parse(url, { allowPrivateDomains: const domainSorter = (a, b) => { if (a === b) return 0; - const aParsed = parse(a); - const bParsed = parse(b); + const aParsed = parse(a[0] === '.' ? a.slice(1) : a); + const bParsed = parse(b[0] === '.' ? b.slice(1) : b); - const aSuffix = aParsed.publicSuffix; - const bSuffix = bParsed.publicSuffix; - - if (bSuffix !== aSuffix) { - if (bSuffix == null) { - return 1; - } - if (aSuffix == null) { - return -1; - } - - for (let i = 0, l = aSuffix.length; i < l; i++) { - if (bSuffix[i] == null) { - return 1; - } - - if (aSuffix[i] < bSuffix[i]) { - return -1; - } - - if (aSuffix[i] > bSuffix[i]) { - return 1; - } - } + const resultDomainWithoutSuffix = compare(aParsed.domainWithoutSuffix, bParsed.domainWithoutSuffix); + if (resultDomainWithoutSuffix !== 0) { + return resultDomainWithoutSuffix; } - const aDomainWithoutSuffix = aParsed.domainWithoutSuffix; - const bDomainWithoutSuffix = bParsed.domainWithoutSuffix; - - if (aDomainWithoutSuffix !== bDomainWithoutSuffix) { - if (bDomainWithoutSuffix == null) { - return 1; - } - if (aDomainWithoutSuffix == null) { - return -1; - } - - for (let i = 0, l = aDomainWithoutSuffix.length; i < l; i++) { - if (bDomainWithoutSuffix[i] == null) { - return 1; - } - - if (aDomainWithoutSuffix[i] < bDomainWithoutSuffix[i]) { - return -1; - } - - if (aDomainWithoutSuffix[i] > bDomainWithoutSuffix[i]) { - return 1; - } - } + const resultSuffix = compare(aParsed.publicSuffix, bParsed.publicSuffix); + if (resultSuffix !== 0) { + return resultSuffix; } - const aSubdomain = aParsed.subdomain; - const bSubdomain = bParsed.subdomain; - - if (aSubdomain !== bSubdomain) { - if (bSubdomain == null) { - return 1; - } - if (aSubdomain == null) { - return -1; - } - - for (let i = 0, l = aSubdomain.length; i < l; i++) { - if (bSubdomain[i] == null) { - return 1; - } - - if (aSubdomain[i] < bSubdomain[i]) { - return -1; - } - - if (aSubdomain[i] > bSubdomain[i]) { - return 1; - } - } + const resultSubdomain = compare(aParsed.subdomain, bParsed.subdomain); + if (resultSubdomain !== 0) { + return resultSubdomain; } return 0;