Add stable domain sort

Build/build-domestic-ruleset.js

@@ -5,6 +5,7 @@ const { readFileByLine } = require('./lib/fetch-remote-text-by-line');
 const { processLine } = require('./lib/process-line');
 const { withBannerArray } = require('./lib/with-banner');
 const { compareAndWriteFile } = require('./lib/string-array-compare');
+const domainSorter = require('./lib/stable-sort-domain');
 
 (async () => {
   const rl = readFileByLine(path.resolve(__dirname, '../Source/non_ip/domestic.conf'));
@@ -20,7 +21,7 @@ const { compareAndWriteFile } = require('./lib/string-array-compare');
     ...Object.entries(DOMESTICS)
       .filter(([key]) => key !== 'SYSTEM')
       .flatMap(([, { domains }]) => domains)
-      .sort()
+      .sort(domainSorter)
       .map((domain) => `DOMAIN-SUFFIX,${domain}`)
   );
 

Build/build-phishing-domainset.js

@@ -1,9 +1,10 @@
-const tldts = require('tldts');
+const { parse } = require('tldts');
 const { processFilterRules } = require('./lib/parse-filter.js');
 const path = require('path');
 const { withBannerArray } = require('./lib/with-banner.js');
 const { compareAndWriteFile } = require('./lib/string-array-compare');
 const { processLine } = require('./lib/process-line.js');
+const domainSorter = require('./lib/stable-sort-domain');
 
 const WHITELIST_DOMAIN = new Set([
   'w3s.link',
@@ -13,7 +14,7 @@ const WHITELIST_DOMAIN = new Set([
   'business.site',
   'page.link', // Firebase URL Shortener
   'fleek.cool',
-  'notion.site' 
+  'notion.site'
 ]);
 const BLACK_TLD = new Set([
   'xyz',
@@ -68,7 +69,9 @@ const BLACK_TLD = new Set([
 
     const domain = line.charCodeAt(0) === 46 ? line.slice(1) : line;
 
-    const apexDomain = tldts.getDomain(domain, { allowPrivateDomains: true });
+    const parsed = parse(domain, { allowPrivateDomains: true });
+
+    const apexDomain = parsed.domain;
 
     if (apexDomain) {
       if (WHITELIST_DOMAIN.has(apexDomain)) {
@@ -94,7 +97,7 @@ const BLACK_TLD = new Set([
         domainCountMap[apexDomain] += (isPhishingDomainMockingAmazon ? 4.5 : 0.5);
       }
 
-      const tld = tldts.getPublicSuffix(domain, { allowPrivateDomains: true });
+      const tld = parsed.publicSuffix;
       if (!tld || !BLACK_TLD.has(tld)) continue;
 
       domainCountMap[apexDomain] += 1;
@@ -114,7 +117,7 @@ const BLACK_TLD = new Set([
         }
 
         if (domainCountMap[apexDomain] < 5) {
-          const subdomain = tldts.getSubdomain(domain, { allowPrivateDomains: true });
+          const subdomain = parsed.subdomain;
           if (subdomain && subdomain.includes('.')) {
             domainCountMap[apexDomain] += 1.5;
           }
@@ -134,7 +137,7 @@ const BLACK_TLD = new Set([
     }
   });
 
-  results.sort();
+  results.sort(domainSorter);
 
   await compareAndWriteFile(
     withBannerArray(

Build/build-reject-domainset.js

@@ -4,7 +4,6 @@ const fse = require('fs-extra');
 
 const { resolve: pathResolve } = require('path');
 const { processHosts, processFilterRules } = require('./lib/parse-filter');
-const { getDomain } = require('tldts');
 const Trie = require('./lib/trie');
 
 const { HOSTS, ADGUARD_FILTERS, PREDEFINED_WHITELIST, PREDEFINED_ENFORCED_BACKLIST } = require('./lib/reject-data-source');
@@ -14,6 +13,7 @@ const { processLine } = require('./lib/process-line');
 const { domainDeduper } = require('./lib/domain-deduper');
 const createKeywordFilter = require('./lib/aho-corasick');
 const { readFileByLine } = require('./lib/fetch-remote-text-by-line');
+const domainSorter = require('./lib/stable-sort-domain');
 
 /** Whitelists */
 const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST);
@@ -188,29 +188,8 @@ const domainSuffixSet = new Set();
   /** @type {Record<string, number>} */
   const rejectDomainsStats = {};
 
-  const sorter = (a, b) => {
-    if (a.domain > b.domain) {
-      return 1;
-    }
-    if (a.domain < b.domain) {
-      return -1;
-    }
-    if (a.v > b.v) {
-      return 1;
-    }
-    if (a.v < b.v) {
-      return -1;
-    }
-    return 0;
-  };
   const sortedDomainSets = dudupedDominArray
-    .map((v) => {
-      const domain = getDomain(v[0] === '.' ? v.slice(1) : v) || v;
-      rejectDomainsStats[domain] = (rejectDomainsStats[domain] || 0) + 1;
-      return { v, domain };
-    })
-    .sort(sorter)
-    .map((i) => i.v);
+    .sort(domainSorter);
 
   await Promise.all([
     compareAndWriteFile(

Build/build-speedtest-domainset.js

@@ -3,6 +3,7 @@ const { domainDeduper } = require('./lib/domain-deduper');
 const path = require('path');
 const { withBannerArray } = require('./lib/with-banner.js');
 const { compareAndWriteFile } = require('./lib/string-array-compare');
+const domainSorter = require('./lib/stable-sort-domain');
 
 const { Sema } = require('async-sema');
 const s = new Sema(2);
@@ -100,7 +101,7 @@ const querySpeedtestApi = async (keyword) => {
     }
   }
 
-  const reduped = domainDeduper(Array.from(domains)).sort();
+  const reduped = domainDeduper(Array.from(domains)).sort(domainSorter);
 
   await compareAndWriteFile(
     withBannerArray(

Build/lib/parse-filter.js

@@ -399,16 +399,6 @@ async function processFilterRules(filterRulesUrl, fallbackUrls, includeThirdPart
   };
 }
 
-/**
- * @param {string[]} data
- */
-function preprocessFullDomainSetBeforeUsedAsWorkerData(data) {
-  return data
-    .filter(domain => domain[0] === '.')
-    .sort((a, b) => a.length - b.length);
-}
-
 module.exports.processDomainLists = processDomainLists;
 module.exports.processHosts = processHosts;
 module.exports.processFilterRules = processFilterRules;
-module.exports.preprocessFullDomainSetBeforeUsedAsWorkerData = preprocessFullDomainSetBeforeUsedAsWorkerData;

Build/lib/stable-sort-domain.js

@@ -0,0 +1,104 @@
+// @ts-check
+const tldts = require('tldts');
+
+const cache1 = Object.create(null);
+/**
+ * @param {string} url
+ * @returns {ReturnType<typeof tldts.parse>}
+ */
+// eslint-disable-next-line no-return-assign -- cache
+const parse = (url) => (cache1[url] ||= tldts.parse(url, { allowPrivateDomains: true }));
+
+/**
+ * @param {string} a
+ * @param {string} b
+ * @returns {0 | 1 | -1}
+ */
+const domainSorter = (a, b) => {
+  if (a === b) return 0;
+
+  const aParsed = parse(a);
+  const bParsed = parse(b);
+
+  const aSuffix = aParsed.publicSuffix;
+  const bSuffix = bParsed.publicSuffix;
+
+  if (bSuffix !== aSuffix) {
+    if (bSuffix == null) {
+      return 1;
+    }
+    if (aSuffix == null) {
+      return -1;
+    }
+
+    for (let i = 0, l = aSuffix.length; i < l; i++) {
+      if (bSuffix[i] == null) {
+        return 1;
+      }
+
+      if (aSuffix[i] < bSuffix[i]) {
+        return -1;
+      }
+
+      if (aSuffix[i] > bSuffix[i]) {
+        return 1;
+      }
+    }
+  }
+
+  const aDomainWithoutSuffix = aParsed.domainWithoutSuffix;
+  const bDomainWithoutSuffix = bParsed.domainWithoutSuffix;
+
+  if (aDomainWithoutSuffix !== bDomainWithoutSuffix) {
+    if (bDomainWithoutSuffix == null) {
+      return 1;
+    }
+    if (aDomainWithoutSuffix == null) {
+      return -1;
+    }
+
+    for (let i = 0, l = aDomainWithoutSuffix.length; i < l; i++) {
+      if (bDomainWithoutSuffix[i] == null) {
+        return 1;
+      }
+
+      if (aDomainWithoutSuffix[i] < bDomainWithoutSuffix[i]) {
+        return -1;
+      }
+
+      if (aDomainWithoutSuffix[i] > bDomainWithoutSuffix[i]) {
+        return 1;
+      }
+    }
+  }
+
+  const aSubdomain = aParsed.subdomain;
+  const bSubdomain = bParsed.subdomain;
+
+  if (aSubdomain !== bSubdomain) {
+    if (bSubdomain == null) {
+      return 1;
+    }
+    if (aSubdomain == null) {
+      return -1;
+    }
+
+    for (let i = 0, l = aSubdomain.length; i < l; i++) {
+      if (bSubdomain[i] == null) {
+        return 1;
+      }
+
+      if (aSubdomain[i] < bSubdomain[i]) {
+        return -1;
+      }
+
+      if (aSubdomain[i] > bSubdomain[i]) {
+        return 1;
+      }
+    }
+  }
+
+  return 0;
+};
+
+module.exports = domainSorter;