Update Rules

Build/build-cidr.js

@@ -1,4 +1,4 @@
-const https = require('https');
+const { simpleGet } = require('./util-http-get');
 const { promises: fsPromises } = require('fs');
 const { resolve: pathResolve } = require('path');
 
@@ -13,7 +13,7 @@ try {
 }
 
 (async () => {
-  const cidr = (await get('raw.githubusercontent.com', 'misakaio/chnroutes2/master/chnroutes.txt')).split('\n');
+  const cidr = (await simpleGet.https('raw.githubusercontent.com', 'misakaio/chnroutes2/master/chnroutes.txt')).split('\n');
 
   const filteredCidr = cidr.filter(line => {
     if (line) {
@@ -36,33 +36,3 @@ function makeCidrList(cidr) {
 # Routes: ${cidr.length}
 ############################\n` + cidr.map(i => `IP-CIDR,${i}`).join('\n') + '\n########### END ############\n';
 };
-
-function get(hostname, path) {
-  return new Promise((resolve, reject) => {
-    const req = https.request(
-      {
-        hostname,
-        path,
-        method: 'GET',
-      },
-      (res) => {
-        const body = [];
-        res.on('data', (chunk) => {
-          body.push(chunk);
-        });
-        res.on('end', () => {
-          try {
-            resolve(String(Buffer.concat(body)));
-          } catch (e) {
-            reject(e);
-          }
-        });
-        req.on('error', (err) => {
-          reject(err);
-        });
-      }
-    );
-
-    req.end();
-  });
-}

Build/build-reject-domainset.js

@@ -0,0 +1,198 @@
+const { simpleGet } = require('./util-http-get');
+const { promises: fsPromises } = require('fs');
+const { resolve: pathResolve } = require('path');
+
+let cliProgress;
+try {
+  cliProgress = require('cli-progress');
+} catch (e) {
+  console.log('Dependencies not found');
+  console.log('"npm i cli-progress" then try again!');
+
+  console.error(e);
+  process.exit(1);
+}
+
+/**
+ * @param {string | URL} hostsUrl
+ */
+async function processHosts(hostsUrl, includeAllSubDomain = false) {
+  if (typeof hostsUrl === 'string') {
+    hostsUrl = new URL(hostsUrl);
+  }
+
+  /** @type Set<string> */
+  const domainSets = new Set();
+
+  /** @type string[] */
+  const hosts = (await simpleGet.https(hostsUrl)).split('\n');
+  hosts.forEach(line => {
+    if (line.startsWith('#')) {
+      return;
+    }
+    if (line.startsWith(' ') || line === '' || line.startsWith('\r') || line.startsWith('\n')) {
+      return;
+    }
+    const [, ...domains] = line.split(' ');
+    domainSets.add(`${includeAllSubDomain ? '.' : ''}${domains.join(' ')}`);
+  });
+
+  return [...domainSets];
+}
+
+/**
+ * @param {string | URL} filterRulesUrl
+ * @returns {Promise<{ white: string[], black: string[] }>}
+ */
+async function processFilterRules(filterRulesUrl) {
+  if (typeof filterRulesUrl === 'string') {
+    filterRulesUrl = new URL(filterRulesUrl);
+  }
+
+  /** @type Set<string> */
+  const whitelistDomainSets = new Set();
+  /** @type Set<string> */
+  const blacklistDomainSets = new Set();
+
+  /** @type string[] */
+  const filterRules = (await simpleGet.https(filterRulesUrl.hostname, filterRulesUrl.pathname)).split('\n');
+  filterRules.forEach(line => {
+    if (line.startsWith('#') || line.startsWith('!')) {
+      return;
+    }
+    if (line.startsWith(' ') || line === '' || line.startsWith('\r') || line.startsWith('\n')) {
+      return;
+    }
+    if (!line.includes('*') && !line.includes('//')) {
+      if (line.startsWith('@@||') && line.endsWith('^')) {
+        whitelistDomainSets.add(`${line.replaceAll('@@||', '').replaceAll('^', '')}`);
+      } else if (line.startsWith('||') && line.endsWith('^')) {
+        blacklistDomainSets.add(`${line.replaceAll('||', '').replaceAll('^', '')}`);
+      }
+    }
+  });
+
+  return {
+    white: [...whitelistDomainSets],
+    black: [...blacklistDomainSets]
+  };
+}
+
+(async () => {
+  /** @type Set<string> */
+  const domainSets = new Set();
+
+  // Parse from remote hosts
+  (await Promise.all([
+    processHosts('https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=1&mimetype=plaintext', true),
+    processHosts('https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt'),
+    processHosts('https://cdn.jsdelivr.net/gh/neoFelhz/neohosts@gh-pages/full/hosts'),
+    processHosts('https://adaway.org/hosts.txt')
+  ])).forEach(hosts => {
+    hosts.forEach(host => {
+      if (host) {
+        domainSets.add(host);
+      }
+    });
+  });
+
+  console.log(`Import ${domainSets.size} rules from hosts files!`);
+
+  console.log(`Start importing rules from reject_sukka.conf!`);
+
+  await fsPromises.readFile(pathResolve(__dirname, '../List/domainset/reject_sukka.conf'), { encoding: 'utf-8' }).then(data => {
+    data.split('\n').forEach(line => {
+      if (
+        line.startsWith('#')
+        || line.startsWith(' ')
+        || line === ''
+        || line.startsWith('\r')
+        || line.startsWith('\n')
+      ) {
+        return;
+      }
+
+      /* if (domainSets.has(line) || domainSets.has(`.${line}`)) {
+        console.warn(`|${line}| is already in the list!`);
+      } */
+      domainSets.add(line);
+    });
+  });
+
+  // Parse from AdGuard Filters
+  /** @type Set<string> */
+  const filterRuleWhitelistDomainSets = new Set();
+  /** @type Set<string> */
+  const filterRuleBlacklistDomainSets = new Set();
+  (await Promise.all([
+    processFilterRules('https://easylist.to/easylist/easylist.txt'),
+    processFilterRules('https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt'),
+    processFilterRules('https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_11_Mobile/filter.txt'),
+    processFilterRules('https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_3_Spyware/filter.txt'),
+    processFilterRules('https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_2_English/filter.txt'),
+    processFilterRules('https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_224_Chinese/filter.txt')
+  ])).forEach(({ white, black }) => {
+    white.forEach(i => filterRuleWhitelistDomainSets.add(i));
+    black.forEach(i => filterRuleBlacklistDomainSets.add(i));
+  });
+
+  for (const black of filterRuleBlacklistDomainSets) {
+    domainSets.add(`.${black}`);
+  }
+
+  console.log(`Import ${filterRuleBlacklistDomainSets.size} rules from adguard filters!`);
+
+  // Remove whitelist from the domain sets
+  console.log(`Remove whitelist from the domain sets!`);
+  for (const domain of domainSets) {
+    for (const white of filterRuleWhitelistDomainSets) {
+      if (domain.includes(white) || white.includes(domain)) {
+        domainSets.delete(domain);
+      }
+    }
+  }
+
+  // Read DOMAIN Keyword
+  const domainKeywordsSet = new Set();
+  await fsPromises.readFile(pathResolve(__dirname, '../List/non_ip/reject.conf'), { encoding: 'utf-8' }).then(data => {
+    data.split('\n').forEach(line => {
+      if (line.startsWith('DOMAIN-KEYWORD')) {
+        const [, ...keywords] = line.split(',');
+        domainKeywordsSet.add(keywords.join(','));
+      }
+    });
+  });
+
+  // Dedupe domainSets
+  console.log(`Start deduping!`);
+  const bar2 = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);
+
+  const domainSetsClone = [...domainSets];
+  const len = domainSetsClone.length;
+
+  bar2.start(len, 0);
+  for (const domain of domainSets) {
+    for (const keyword of domainKeywordsSet) {
+      if (domain.includes(keyword)) {
+        domainSets.delete(domain);
+        continue;
+      }
+    }
+
+    if (domain.startsWith('.')) {
+      for (const domain2 of domainSets) {
+        if (domain2 !== domain) {
+          if (domain2.endsWith(domain) || `.${domain2}` === domain) {
+            domainSets.delete(domain2);
+          }
+        }
+      }
+    }
+
+    bar2.increment();
+  }
+
+  bar2.stop();
+
+  return fsPromises.writeFile(pathResolve(__dirname, '../List/domainset/reject.conf'), [...domainSets].join('\n'));
+})();

Build/util-http-get.js

@@ -0,0 +1,35 @@
+const https = require('https');
+
+exports.simpleGet = {
+  https(hostname, path) {
+    const requestOpt = hostname instanceof URL ? hostname : {
+      hostname,
+      path,
+      method: 'GET',
+    };
+
+    return new Promise((resolve, reject) => {
+      const req = https.request(
+        requestOpt,
+        (res) => {
+          const body = [];
+          res.on('data', (chunk) => {
+            body.push(chunk);
+          });
+          res.on('end', () => {
+            try {
+              resolve(String(Buffer.concat(body)));
+            } catch (e) {
+              reject(e);
+            }
+          });
+          req.on('error', (err) => {
+            reject(err);
+          });
+        }
+      );
+  
+      req.end();
+    });
+  }
+}

List/non_ip/global_plus.conf

@@ -9,11 +9,12 @@ DOMAIN-SUFFIX,blogger.com
 DOMAIN-SUFFIX,getoutline.org
 DOMAIN-SUFFIX,gvt0.com
 DOMAIN-SUFFIX,gvt1.com
+DOMAIN-SUFFIX,gvt2.com
 DOMAIN-SUFFIX,gvt3.com
+DOMAIN-SUFFIX,googleapis.cn
 DOMAIN-KEYWORD,google
 DOMAIN-SUFFIX,gmail.com
 DOMAIN-KEYWORD,blogspot
-DOMAIN-SUFFIX,googleapis.cn
 
 # >> Facebook
 DOMAIN-SUFFIX,cdninstagram.com

List/non_ip/reject.conf

@@ -20,6 +20,16 @@ DOMAIN-KEYWORD,.freecontent.
 DOMAIN-KEYWORD,track.tiara
 DOMAIN-KEYWORD,adservice
 DOMAIN-KEYWORD,umeng
+DOMAIN-KEYWORD,adsby
+DOMAIN-KEYWORD,adsdk
+DOMAIN-KEYWORD,adserver
+DOMAIN-KEYWORD,admaster
+DOMAIN-KEYWORD,adserve2
+DOMAIN-KEYWORD,admob
+DOMAIN-KEYWORD,adserver
+DOMAIN-KEYWORD,adspace
+DOMAIN-KEYWORD,advertmarket
+DOMAIN-KEYWORD,adsyndication
 
 # >> Google
 DOMAIN-KEYWORD,adsense