New phishing blocking domainset

Build/build-phishing-domainset.js

@@ -0,0 +1,72 @@
+const psl = require('psl');
+const { processFilterRules } = require('./lib/parse-filter.js');
+const fs = require('fs');
+const path = require('path');
+
+const WHITELIST_DOMAIN = new Set([]);
+const BLACK_TLD = [
+  '.xyz',
+  '.top',
+  '.win',
+  '.vip',
+  '.site',
+  '.space',
+  '.online',
+  '.icu',
+  '.fun',
+  '.shop',
+  '.cool',
+  '.cyou',
+  '.id'
+];
+
+(async () => {
+  const domainSet = Array.from(
+    (
+      await processFilterRules('https://curbengh.github.io/phishing-filter/phishing-filter-agh.txt')
+    ).black
+  );
+  const domainCountMap = {};
+
+  for (let i = 0, len = domainSet.length; i < len; i++) {
+    const line = domainSet[i];
+    // starts with #
+    if (line.charCodeAt(0) === 35) {
+      continue;
+    }
+    if (line.trim().length === 0) {
+      continue;
+    }
+
+    const domain = line.charCodeAt(0) === 46 ? line.slice(1) : line;
+
+    if (line.length > 25) {
+      const parsed = psl.parse(domain);
+
+      if (parsed.input === parsed.tld) {
+        continue;
+      }
+      const apexDomain = parsed.domain
+
+      if (WHITELIST_DOMAIN.has(apexDomain)) {
+        continue;
+      }
+
+      domainCountMap[apexDomain] ||= 0;
+      domainCountMap[apexDomain] += 1;
+    }
+  }
+
+  const results = [];
+  Object.entries(domainCountMap).forEach(([domain, count]) => {
+    if (
+      count >= 8
+      && BLACK_TLD.some(tld => domain.endsWith(tld))
+    ) {
+      results.push('.' + domain);
+    }
+  });
+
+  const filePath = path.resolve(__dirname, '../List/domainset/reject_phishing.conf');
+  await fs.promises.writeFile(filePath, results.join('\n'), 'utf-8');
+})();

Build/build-reject-domainset.js

@@ -202,6 +202,27 @@ const threads = isCI ? cpuCount : cpuCount / 2;
     });
   });
 
+  // Read Special Phishing Suffix list
+  await fsPromises.readFile(pathResolve(__dirname, '../List/domainset/reject_phishing.conf'), { encoding: 'utf-8' }).then(data => {
+    data.split('\n').forEach(line => {
+      const trimmed = line.trim();
+      if (
+        line.startsWith('#')
+        || line.startsWith(' ')
+        || line.startsWith('\r')
+        || line.startsWith('\n')
+        || trimmed === ''
+      ) {
+        return;
+      }
+
+      /* if (domainSets.has(line) || domainSets.has(`.${line}`)) {
+        console.warn(`|${line}| is already in the list!`);
+      } */
+      domainSuffixSet.add(trimmed);
+    });
+  });
+
   console.log(`Import ${domainKeywordsSet.size} black keywords and ${domainSuffixSet.size} black suffixes!`);
 
   previousSize = domainSets.size;

Build/lib/parse-filter.js

@@ -107,14 +107,20 @@ async function processFilterRules (filterRulesUrl, fallbackUrls) {
   /** @type Set<string> */
   const blacklistDomainSets = new Set();
 
-  /** @type string[] */
-  const filterRules = (
-    await Promise.any(
-      [filterRulesUrl, ...(fallbackUrls || [])].map(
-        async url => (await fetchWithRetry(url)).text()
+  let filterRules;
+  try {
+    /** @type string[] */
+    filterRules = (
+      await Promise.any(
+        [filterRulesUrl, ...(fallbackUrls || [])].map(
+          async url => (await fetchWithRetry(url)).text()
+        )
       )
-    )
-  ).split('\n').map(line => line.trim());
+    ).split('\n').map(line => line.trim());
+  } catch (e) {
+    console.log('Download Rule for [' + filterRulesUrl + '] failed');
+    throw e;
+  }
 
   filterRules.forEach(line => {
     const lineStartsWithDoubleVerticalBar = line.startsWith('||');