Feat: use Cisco Top 1M Domain for validation

Build/validate-gfwlist.ts

@@ -1,18 +1,21 @@
 import { processLine } from './lib/process-line';
 import { fastNormalizeDomain } from './lib/normalize-domain';
 import { HostnameSmolTrie } from './lib/trie';
-// import { Readable } from 'stream';
-import { parse } from 'csv-parse/sync';
+import yauzl from 'yauzl-promise';
 import { fetchRemoteTextByLine } from './lib/fetch-text-by-line';
 import path from 'node:path';
 import { OUTPUT_SURGE_DIR } from './constants/dir';
 import { createRetrieKeywordFilter as createKeywordFilter } from 'foxts/retrie';
 import { $$fetch } from './lib/fetch-retry';
 import runAgainstSourceFile from './lib/run-against-source-file';
+import { nullthrow } from 'foxts/guard';
+import { Buffer } from 'node:buffer';
 
 export async function parseGfwList() {
+  const { parse: csvParser } = await import('csv-parse');
+
   const whiteSet = new Set<string>();
-  const trie = new HostnameSmolTrie();
+  const gfwListTrie = new HostnameSmolTrie();
 
   const excludeGfwList = createKeywordFilter([
     '.*',
@@ -44,42 +47,71 @@ export async function parseGfwList() {
       continue;
     }
     if (line.startsWith('||')) {
-      trie.add('.' + line.slice(2));
+      gfwListTrie.add('.' + line.slice(2));
       continue;
     }
     if (line.startsWith('|')) {
-      trie.add(line.slice(1));
+      gfwListTrie.add(line.slice(1));
       continue;
     }
     if (line.startsWith('.')) {
-      trie.add(line);
+      gfwListTrie.add(line);
       continue;
     }
     const d = fastNormalizeDomain(line);
     if (d) {
-      trie.add(d);
+      gfwListTrie.add(d);
       continue;
     }
   }
   for await (const l of await fetchRemoteTextByLine('https://raw.githubusercontent.com/Loyalsoldier/cn-blocked-domain/release/domains.txt', true)) {
-    trie.add(l);
+    gfwListTrie.add(l);
   }
   for await (const l of await fetchRemoteTextByLine('https://raw.githubusercontent.com/Loyalsoldier/v2ray-rules-dat/release/gfw.txt', true)) {
-    trie.add(l);
+    gfwListTrie.add(l);
   }
 
-  const topDomainsRes = await (await $$fetch('https://downloads.majestic.com/majestic_million.csv', {
+  const topDomainTrie = new HostnameSmolTrie();
+
+  const csvParse = csvParser({ columns: false, skip_empty_lines: true });
+  const topDomainsZipBody = await (await $$fetch('https://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip', {
     headers: {
       accept: '*/*',
       'user-agent': 'curl/8.12.1'
     }
-  })).text();
-  const topDomains = parse(topDomainsRes);
+  })).arrayBuffer();
+  let entry: yauzl.Entry | null = null;
+  for await (const e of await yauzl.fromBuffer(Buffer.from(topDomainsZipBody))) {
+    if (e.filename === 'top-1m.csv') {
+      entry = e;
+      break;
+    }
+  }
+
+  const { promise, resolve, reject } = Promise.withResolvers<HostnameSmolTrie>();
+
+  const readable = await nullthrow(entry, 'top-1m.csv entry not found').openReadStream();
+  const parser = readable.pipe(csvParse);
+  parser.on('readable', () => {
+    let record;
+    while ((record = parser.read()) !== null) {
+      topDomainTrie.add(record[1]);
+    }
+  });
+
+  parser.on('end', () => {
+    resolve(topDomainTrie);
+  });
+  parser.on('error', (err) => {
+    reject(err);
+  });
+
+  await promise;
 
   const keywordSet = new Set<string>();
 
   const callback = (domain: string, includeAllSubdomain: boolean) => {
-    trie.whitelist(domain, includeAllSubdomain);
+    gfwListTrie.whitelist(domain, includeAllSubdomain);
   };
 
   await Promise.all([
@@ -94,24 +126,24 @@ export async function parseGfwList() {
     runAgainstSourceFile(path.resolve(OUTPUT_SURGE_DIR, 'domainset/cdn.conf'), callback, 'domainset')
   ]);
 
-  whiteSet.forEach(domain => trie.whitelist(domain));
+  whiteSet.forEach(domain => gfwListTrie.whitelist(domain));
 
   const kwfilter = createKeywordFilter([...keywordSet]);
 
   const missingTop10000Gfwed = new Set<string>();
 
-  for await (const [domain] of topDomains) {
-    if (trie.has(domain) && !kwfilter(domain)) {
+  topDomainTrie.dump((domain) => {
+    if (gfwListTrie.has(domain) && !kwfilter(domain)) {
       missingTop10000Gfwed.add(domain);
     }
-  }
+  });
 
   console.log(missingTop10000Gfwed.size, '');
   console.log(Array.from(missingTop10000Gfwed).join('\n'));
 
   return [
     whiteSet,
-    trie,
+    gfwListTrie,
     missingTop10000Gfwed
   ] as const;
 }