Chore: extract constants from build script

2026-02-03 12:31:54 +08:00 · 2022-11-07 17:54:26 +08:00
parent 43b1315872
commit 6832dafecc
6 changed files with 231 additions and 210 deletions
--- a/Build/build-reject-domainset.js
+++ b/Build/build-reject-domainset.js
@@ -7,6 +7,10 @@ const cpuCount = require('os').cpus().length;
 const { isCI } = require('ci-info');
 const threads = isCI ? cpuCount : cpuCount / 2;

+const { HOSTS, ADGUARD_FILTERS, PREDEFINED_WHITELIST } = require('./lib/reject-data-source');
+
+const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST);
+
 (async () => {
  console.time('Total Time - build-reject-domain-set');

@@ -17,11 +21,9 @@ const threads = isCI ? cpuCount : cpuCount / 2;
  console.time('* Download and process Hosts');

  // Parse from remote hosts & domain lists
-  (await Promise.all([
-    processHosts('https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=0&mimetype=plaintext', true),
-    processHosts('https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt'),
-    processHosts('https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/spy.txt')
-  ])).forEach(hosts => {
+  (await Promise.all(
+    HOSTS.map(entry => processHosts(entry[0], entry[1]))
+  )).forEach(hosts => {
    hosts.forEach(host => {
      if (host) {
        domainSets.add(host);
@@ -34,195 +36,9 @@ const threads = isCI ? cpuCount : cpuCount / 2;
  let previousSize = domainSets.size;
  console.log(`Import ${previousSize} rules from hosts files!`);

-  await fsPromises.readFile(pathResolve(__dirname, '../Source/domainset/reject_sukka.conf'), { encoding: 'utf-8' }).then(data => {
-    data.split('\n').forEach(line => {
-      const trimmed = line.trim();
-      if (
-        line.startsWith('#')
-        || line.startsWith(' ')
-        || line.startsWith('\r')
-        || line.startsWith('\n')
-        || trimmed === ''
-      ) {
-        return;
-      }
-
-      /* if (domainSets.has(line) || domainSets.has(`.${line}`)) {
-        console.warn(`|${line}| is already in the list!`);
-      } */
-      domainSets.add(trimmed);
-    });
-  });
-
-  // Copy reject_sukka.conf for backward compatibility
-  await fse.copy(pathResolve(__dirname, '../Source/domainset/reject_sukka.conf'), pathResolve(__dirname, '../List/domainset/reject_sukka.conf'))
-
-  previousSize = domainSets.size - previousSize;
-  console.log(`Import ${previousSize} rules from reject_sukka.conf!`);
-
  // Parse from AdGuard Filters
-  /** @type Set<string> */
-  const filterRuleWhitelistDomainSets = new Set([
-    'localhost',
-    'broadcasthost',
-    'ip6-loopback',
-    'ip6-localnet',
-    'ip6-mcastprefix',
-    'ip6-allnodes',
-    'ip6-allrouters',
-    'ip6-allhosts',
-    'mcastprefix',
-    'skk.moe',
-    'analytics.google.com',
-    'msa.cdn.mediaset.net', // Added manually using DOMAIN-KEYWORDS
-    'cloud.answerhub.com',
-    'ae01.alicdn.com',
-    'whoami.akamai.net',
-    'whoami.ds.akahelp.net',
-    'pxlk9.net.', // This one is malformed from EasyList, which I will manually add instead
-    'instant.page', // No, it doesn't violate anyone's privacy. I will whitelist it
-    'piwik.pro',
-    'mixpanel.com',
-    'cdn.mxpnl.com',
-    'heapanalytics.com',
-    'segment.com',
-    'segmentify.com',
-    't.co', // pgl yoyo add t.co to the blacklist
-    'survicate.com', // AdGuardDNSFilter
-    'perfops.io', // AdGuardDNSFilter
-    'd2axgrpnciinw7.cloudfront.net', // ADGuardDNSFilter
-    'tb-lb.sb-cd.com', // AdGuard
-    'storage.yandexcloud.net', // phishing list
-    'login.microsoftonline.com' // phishing list
-  ]);
-
  console.time('* Download and process AdBlock Filter Rules');
-  (await Promise.all([
-    // Easy List
-    [
-      'https://easylist.to/easylist/easylist.txt',
-      [
-        'https://easylist-downloads.adblockplus.org/easylist.txt',
-        'https://raw.githubusercontent.com/easylist/easylist/gh-pages/easylist.txt',
-        'https://secure.fanboy.co.nz/easylist.txt'
-      ]
-    ],
-    // AdGuard DNS Filter
-    'https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt',
-    // uBlock Origin Filter List
-    [
-      'https://ublockorigin.github.io/uAssets/filters/filters.txt',
-      [
-        'https://ublockorigin.github.io/uAssetsCDN/filters/filters.txt',
-        'https://ublockorigin.pages.dev/filters/filters.txt'
-      ]
-    ],
-    [
-      'https://ublockorigin.github.io/uAssets/filters/filters-2020.txt',
-      [
-        'https://ublockorigin.github.io/uAssetsCDN/filters/filters-2020.txt',
-        'https://ublockorigin.pages.dev/filters/filters-2020.txt'
-      ]
-    ],
-    [
-      'https://ublockorigin.github.io/uAssets/filters/filters-2021.txt',
-      [
-        'https://ublockorigin.github.io/uAssetsCDN/filters/filters-2021.txt',
-        'https://ublockorigin.pages.dev/filters/filters-2021.txt'
-      ]
-    ],
-    [
-      'https://ublockorigin.github.io/uAssets/filters/filters-2022.txt',
-      [
-        'https://ublockorigin.github.io/uAssetsCDN/filters/filters-2022.txt',
-        'https://ublockorigin.pages.dev/filters/filters-2022.txt'
-      ]
-    ],
-    // uBlock Origin Badware Risk List
-    [
-      'https://ublockorigin.github.io/uAssets/filters/badware.txt',
-      [
-        'https://ublockorigin.github.io/uAssetsCDN/filters/badware.txt',
-        'https://ublockorigin.pages.dev/filters/badware.txt'
-      ]
-    ],
-    // uBlock Origin Privacy List
-    [
-      'https://ublockorigin.github.io/uAssets/filters/privacy.txt',
-      [
-        'https://ublockorigin.github.io/uAssetsCDN/filters/privacy.txt',
-        'https://ublockorigin.pages.dev/filters/privacy.txt'
-      ]
-    ],
-    // uBlock Origin Resource Abuse
-    [
-      'https://ublockorigin.github.io/uAssets/filters/resource-abuse.txt',
-      [
-        'https://ublockorigin.github.io/uAssetsCDN/filters/resource-abuse.txt',
-        'https://ublockorigin.pages.dev/filters/resource-abuse.txt'
-      ]
-    ],
-    // uBlock Origin Unbreak
-    [
-      'https://ublockorigin.github.io/uAssets/filters/unbreak.txt',
-      [
-        'https://ublockorigin.github.io/uAssetsCDN/filters/unbreak.txt',
-        'https://ublockorigin.pages.dev/filters/unbreak.txt'
-      ]
-    ],
-    // AdGuard Base Filter
-    'https://filters.adtidy.org/extension/ublock/filters/2_without_easylist.txt',
-    // AdGuard Mobile AD
-    'https://filters.adtidy.org/extension/ublock/filters/11.txt',
-    // AdGuard Tracking Protection
-    'https://filters.adtidy.org/extension/ublock/filters/3.txt',
-    // AdGuard Japanese filter
-    'https://filters.adtidy.org/extension/ublock/filters/7.txt',
-    // AdGuard Chinese filter (EasyList China + AdGuard Chinese filter)
-    'https://filters.adtidy.org/extension/ublock/filters/224.txt',
-    // Easy Privacy
-    [
-      'https://easylist.to/easylist/easyprivacy.txt',
-      [
-        'https://secure.fanboy.co.nz/easyprivacy.txt',
-        'https://raw.githubusercontent.com/easylist/easylist/gh-pages/easyprivacy.txt',
-        'https://easylist-downloads.adblockplus.org/easyprivacy.txt'
-      ]
-    ],
-    // Curben's UrlHaus Malicious URL Blocklist
-    [
-      'https://curbengh.github.io/urlhaus-filter/urlhaus-filter-agh-online.txt',
-      [
-        'https://urlhaus-filter.pages.dev/urlhaus-filter-agh-online.txt',
-        // Prefer mirror, since malware-filter.gitlab.io has not been updated for a while
-        // 'https://malware-filter.gitlab.io/urlhaus-filter/urlhaus-filter-agh-online.txt'
-      ]
-    ],
-    // Curben's Phishing URL Blocklist
-    [
-      'https://curbengh.github.io/phishing-filter/phishing-filter-agh.txt',
-      [
-        'https://phishing-filter.pages.dev/phishing-filter-agh.txt',
-        // Prefer mirror, since malware-filter.gitlab.io has not been updated for a while
-        // 'https://malware-filter.gitlab.io/malware-filter/phishing-filter-agh.txt'
-      ]
-    ],
-    // Curben's PUP Domains Blocklist
-    [
-      'https://curbengh.github.io/pup-filter/pup-filter-agh.txt',
-      [
-        'https://pup-filter.pages.dev/pup-filter-agh.txt',
-        // Prefer mirror, since malware-filter.gitlab.io has not been updated for a while
-        // 'https://malware-filter.gitlab.io/malware-filter/pup-filter-agh.txt'
-      ]
-    ],
-    // GameConsoleAdblockList
-    'https://raw.githubusercontent.com/DandelionSprout/adfilt/master/GameConsoleAdblockList.txt',
-    // PiHoleBlocklist
-    'https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/SmartTV-AGH.txt',
-    // Spam404
-    'https://raw.githubusercontent.com/Spam404/lists/master/adblock-list.txt'
-  ].map(input => {
+  (await Promise.all(ADGUARD_FILTERS.map(input => {
    if (typeof input === 'string') {
      return processFilterRules(input);
    }
@@ -239,9 +55,33 @@ const threads = isCI ? cpuCount : cpuCount / 2;
  previousSize = domainSets.size - previousSize;
  console.log(`Import ${previousSize} rules from adguard filters!`);

+  await fsPromises.readFile(pathResolve(__dirname, '../Source/domainset/reject_sukka.conf'), { encoding: 'utf-8' }).then(data => {
+    data.split('\n').forEach(line => {
+      const trimmed = line.trim();
+      if (
+        line.startsWith('#')
+        || line.startsWith(' ')
+        || line.startsWith('\r')
+        || line.startsWith('\n')
+        || trimmed === ''
+      ) {
+        return;
+      }
+
+      domainSets.add(trimmed);
+    });
+  });
+
+  // Copy reject_sukka.conf for backward compatibility
+  await fse.copy(pathResolve(__dirname, '../Source/domainset/reject_sukka.conf'), pathResolve(__dirname, '../List/domainset/reject_sukka.conf'))
+
+  previousSize = domainSets.size - previousSize;
+  console.log(`Import ${previousSize} rules from reject_sukka.conf!`);
+
  // Read DOMAIN Keyword
  const domainKeywordsSet = new Set();
  const domainSuffixSet = new Set();
+
  await fsPromises.readFile(pathResolve(__dirname, '../List/non_ip/reject.conf'), { encoding: 'utf-8' }).then(data => {
    data.split('\n').forEach(line => {
      if (line.startsWith('DOMAIN-KEYWORD')) {
@@ -268,9 +108,6 @@ const threads = isCI ? cpuCount : cpuCount / 2;
        return;
      }

-      /* if (domainSets.has(line) || domainSets.has(`.${line}`)) {
-        console.warn(`|${line}| is already in the list!`);
-      } */
      domainSuffixSet.add(trimmed);
    });
  });
@@ -302,11 +139,8 @@ const threads = isCI ? cpuCount : cpuCount / 2;
    }

    if (!isTobeRemoved) {
-      for (const white of filterRuleWhitelistDomainSets) {
-        if (domain.includes(white) || white.includes(domain)) {
-          isTobeRemoved = true;
-          break;
-        }
+      if (isInWhiteList(domain)) {
+        isTobeRemoved = true;
      }
    }

@@ -380,3 +214,13 @@ const threads = isCI ? cpuCount : cpuCount / 2;
    process.exit(0);
  }
 })();
+
+function isInWhiteList (domain) {
+  for (const white of filterRuleWhitelistDomainSets) {
+    if (domain === white || domain.endsWith(white)) {
+      return true;
+    }
+  }
+
+  return false;
+}