Update Rules

Build/build-reject-domainset.js

@@ -2,10 +2,10 @@ const { default: got } = require('got-cjs');
 const { promises: fsPromises } = require('fs');
 const { resolve: pathResolve } = require('path');
 const { cpus } = require('os');
+const { isIP } = require('net');
 
 const threads = Math.max(cpus().length, 12);
 
-const rIPv4 = /((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/;
 const rDomain = /^(((?!\-))(xn\-\-)?[a-z0-9\-_]{0,61}[a-z0-9]{1,1}\.)*(xn\-\-)?([a-z0-9\-]{1,61}|[a-z0-9\-]{1,30})\.[a-z]{2,}$/m
 
 const Piscina = require('piscina');
@@ -83,53 +83,23 @@ async function processFilterRules(filterRulesUrl) {
   }
 
   /** @type Set<string> */
-  const whitelistDomainSets = new Set([
+  const whitelistDomainSets = new Set();
-    'localhost',
-    'broadcasthost',
-    'ip6-loopback',
-    'ip6-localnet',
-    'ip6-mcastprefix',
-    'ip6-allnodes',
-    'ip6-allrouters',
-    'ip6-allhosts',
-    'mcastprefix',
-    'analytics.google.com',
-    'msa.cdn.mediaset.net', // Added manually using DOMAIN-KEYWORDS
-    'cloud.answerhub.com',
-    'ae01.alicdn.com',
-    'whoami.akamai.net',
-    'whoami.ds.akahelp.net',
-    'pxlk9.net.', // This one is malformed from EasyList, which I will manually add instead
-    'instant.page', // No, it doesn't violate anyone's privacy. I will whitelist it
-    'piwik.pro',
-    'mixpanel.com',
-    'heapanalytics.com',
-    'dataunlocker.com',
-    'segment.com',
-    'segment.io',
-    'segmentify.com'
-  ]);
   /** @type Set<string> */
   const blacklistDomainSets = new Set();
 
-  /** @type Set<string> */
-  const blackIPSets = new Set();
-
   /** @type string[] */
-  const filterRules = (await got(filterRulesUrl).text()).split('\n');
+  const filterRules = (await got(filterRulesUrl).text()).split('\n').map(line => line.trim());
 
   filterRules.forEach(line => {
     if (
-      line.includes('#')
+      line === ''
+      || line.includes('#')
       || line.includes('!')
-      || line.startsWith(' ')
-      || line.startsWith('\r')
-      || line.startsWith('\n')
       || line.includes('*')
       || line.includes('/')
-      || line.includes('$')
+      || line.includes('$') && !line.startsWith('@@')
       || line.trim() === ''
-      || rIPv4.test(line)
+      || isIP(line) !== 0
     ) {
       return;
     }
@@ -138,9 +108,17 @@ async function processFilterRules(filterRulesUrl) {
       && (
         line.endsWith('^')
         || line.endsWith('^|')
+        || line.endsWith('^$badfilter')
+        || line.endsWith('^$1p')
       )
     ) {
-      const domain = `${line.replaceAll('@@||', '').replaceAll('^|', '').replaceAll('^', '')}`.trim();
+      const domain = line
+        .replaceAll('@@||', '')
+        .replaceAll('^$badfilter', '')
+        .replaceAll('^$1p', '')
+        .replaceAll('^|', '')
+        .replaceAll('^', '')
+        .trim();
       if (rDomain.test(domain)) {
         whitelistDomainSets.add(domain);
       }
@@ -198,9 +176,9 @@ async function processFilterRules(filterRulesUrl) {
       if (
         line.startsWith('#')
         || line.startsWith(' ')
-        || line === '' || line === ' '
         || line.startsWith('\r')
         || line.startsWith('\n')
+        || line.trim() === ''
       ) {
         return;
       }
@@ -217,23 +195,74 @@ async function processFilterRules(filterRulesUrl) {
 
   // Parse from AdGuard Filters
   /** @type Set<string> */
-  const filterRuleWhitelistDomainSets = new Set();
+  const filterRuleWhitelistDomainSets = new Set([
+    'localhost',
+    'broadcasthost',
+    'ip6-loopback',
+    'ip6-localnet',
+    'ip6-mcastprefix',
+    'ip6-allnodes',
+    'ip6-allrouters',
+    'ip6-allhosts',
+    'mcastprefix',
+    'analytics.google.com',
+    'msa.cdn.mediaset.net', // Added manually using DOMAIN-KEYWORDS
+    'cloud.answerhub.com',
+    'ae01.alicdn.com',
+    'whoami.akamai.net',
+    'whoami.ds.akahelp.net',
+    'pxlk9.net.', // This one is malformed from EasyList, which I will manually add instead
+    'instant.page', // No, it doesn't violate anyone's privacy. I will whitelist it
+    'piwik.pro',
+    'mixpanel.com',
+    'heapanalytics.com',
+    'dataunlocker.com',
+    'segment.com',
+    'segment.io',
+    'segmentify.com'
+  ]);
+
   (await Promise.all([
-    processFilterRules('https://easylist.to/easylist/easylist.txt'),
+    // Easy List
-    processFilterRules('https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt'),
+    'https://easylist.to/easylist/easylist.txt',
-    processFilterRules('https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_11_Mobile/filter.txt'),
+    // AdGuard DNS Filter
-    processFilterRules('https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_3_Spyware/filter.txt'),
+    'https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt',
-    processFilterRules('https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_2_English/filter.txt'),
+    // uBlock Origin Filter List
-    processFilterRules('https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_224_Chinese/filter.txt'),
+    'https://ublockorigin.github.io/uAssetsCDN/filters/filters.txt',
-    processFilterRules('https://filters.adtidy.org/extension/ublock/filters/224.txt'),
+    'https://ublockorigin.github.io/uAssetsCDN/filters/filters-2020.txt',
-    processFilterRules('https://easylist.to/easylist/easyprivacy.txt'),
+    'https://ublockorigin.github.io/uAssetsCDN/filters/filters-2021.txt',
-    processFilterRules('https://raw.githubusercontent.com/DandelionSprout/adfilt/master/GameConsoleAdblockList.txt'),
+    'https://ublockorigin.github.io/uAssetsCDN/filters/filters-2022.txt',
-    processFilterRules('https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/SmartTV-AGH.txt'),
+    // uBlock Origin Badware Risk List
-    processFilterRules('https://curben.gitlab.io/malware-filter/urlhaus-filter-agh-online.txt'),
+    'https://ublockorigin.github.io/uAssets/filters/badware.txt',
-    processFilterRules('https://curben.gitlab.io/malware-filter/pup-filter-agh.txt'),
+    // uBlock Origin Privacy List
-    processFilterRules('https://curben.gitlab.io/malware-filter/phishing-filter-agh.txt'),
+    'https://ublockorigin.github.io/uAssets/filters/privacy.txt',
-    processFilterRules('https://curben.gitlab.io/malware-filter/pup-filter-agh.txt')
+    // uBlock Origin Resource Abuse
-  ])).forEach(({ white, black }) => {
+    'https://ublockorigin.github.io/uAssets/filters/resource-abuse.txt',
+    // uBlock Origin Unbreak
+    'https://ublockorigin.github.io/uAssets/filters/unbreak.txt',
+    // AdGuard Base Filter
+    'https://filters.adtidy.org/extension/ublock/filters/2_without_easylist.txt',
+    // AdGuard Mobile AD
+    'https://filters.adtidy.org/extension/ublock/filters/11.txt',
+    // AdGuard Tracking Protection
+    'https://filters.adtidy.org/extension/ublock/filters/3.txt',
+    // AdGuard Japanese filter
+    'https://filters.adtidy.org/extension/ublock/filters/7.txt',
+    // AdGuard Chinese filter (EasyList China + AdGuard Chinese filter)
+    'https://filters.adtidy.org/extension/ublock/filters/224.txt',
+    // Easy Privacy
+    'https://easylist.to/easylist/easyprivacy.txt',
+    // Curben's Malware Online UrlHaus
+    'https://curben.gitlab.io/malware-filter/urlhaus-filter-agh-online.txt',
+    // Curben's Phishing Online Filter
+    'https://curben.gitlab.io/malware-filter/phishing-filter-agh.txt',
+    // Curben's PUP List
+    'https://curben.gitlab.io/malware-filter/pup-filter-agh.txt',
+    // GameConsoleAdblockList
+    'https://raw.githubusercontent.com/DandelionSprout/adfilt/master/GameConsoleAdblockList.txt',
+    // PiHoleBlocklist
+    'https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/SmartTV-AGH.txt',
+  ].map(processFilterRules))).forEach(({ white, black }) => {
     white.forEach(i => filterRuleWhitelistDomainSets.add(i));
     black.forEach(i => domainSets.add(i));
   });
@@ -256,7 +285,7 @@ async function processFilterRules(filterRulesUrl) {
     });
   });
 
-  console.log(`Import ${domainKeywordsSet.size} black keywords!`);
+  console.log(`Import ${domainKeywordsSet.size} black keywords and ${domainSuffixSet.size} black suffixes!`);
 
   const beforeDeduping = domainSets.size;
   // Dedupe domainSets
@@ -267,8 +296,14 @@ async function processFilterRules(filterRulesUrl) {
   });
 
   (await Promise.all([
-    piscina.run({ keywords: domainKeywordsSet, suffixes: domainSuffixSet, input: domainSets }, { name: 'dedupeKeywords' }),
+    piscina.run(
-    piscina.run({ whiteList: filterRuleWhitelistDomainSets, input: domainSets }, { name: 'whitelisted' })
+      { keywords: domainKeywordsSet, suffixes: domainSuffixSet, input: domainSets },
+      { name: 'dedupeKeywords' }
+    ),
+    piscina.run(
+      { whiteList: filterRuleWhitelistDomainSets, input: domainSets },
+      { name: 'whitelisted' }
+    )
   ])).forEach(set => {
     set.forEach(i => domainSets.delete(i));
   });
@@ -276,13 +311,18 @@ async function processFilterRules(filterRulesUrl) {
   const originalFullSet = new Set([...domainSets]);
 
   (await Promise.all(
-    Array.from(domainSets).reduce((result, element, index) => {
+    Array.from(domainSets)
+      .reduce((result, element, index) => {
         const chunk = index % threads;
         result[chunk] ??= [];
 
         result[chunk].push(element);
         return result;
-    }, []).map(chunk => piscina.run({ input: chunk, fullSet: originalFullSet }, { name: 'dedupe' }))
+      }, [])
+      .map(chunk => piscina.run(
+        { input: chunk, fullSet: originalFullSet },
+        { name: 'dedupe' }
+      ))
   )).forEach(set => {
     set.forEach(i => domainSets.delete(i));
   });

List/domainset/reject_sukka.conf

@@ -629,7 +629,6 @@ analytics.slashdotmedia.com
 .eviltracker.net
 .getexceptional.com
 .jumptap.com
-.localytics.com
 .mobile-collector.newrelic.com
 .playtomic.com
 .stathat.com
@@ -1527,7 +1526,6 @@ gemini.yahoo.com
 ysm.yahoo.com
 yads.c.yimg.jp
 yads.yahoo.co.jp
-.flurry.com
 .flurry.cachefly.net
 
 # >> Disqus
@@ -1628,9 +1626,9 @@ stnetsdk.vivo.com.cn
 .stsdk.vivo.com.cn
 
 # >> Gionee
-0.0.0.0 ads.gionee.com
+ads.gionee.com
-0.0.0.0 pdl.gionee.com
+pdl.gionee.com
-0.0.0.0 adres.myaora.net
+adres.myaora.net
 
 # >> 勾正数据
 .gz-data.com