Chore: update luminati / global domain tools

Build/tools-lum-apex-domains.ts

@@ -0,0 +1,53 @@
+import { fetchRemoteTextByLine, readFileByLine } from './lib/fetch-text-by-line';
+import tldts from 'tldts';
+import { HostnameSmolTrie } from './lib/trie';
+import path from 'node:path';
+import { SOURCE_DIR } from './constants/dir';
+import { processLine } from './lib/process-line';
+
+(async () => {
+  const lines1 = await Array.fromAsync(await fetchRemoteTextByLine('https://raw.githubusercontent.com/durablenapkin/block/master/luminati.txt', true));
+  const lines2 = await Array.fromAsync(await fetchRemoteTextByLine('https://raw.githubusercontent.com/durablenapkin/block/master/tvstream.txt', true));
+
+  const trie = new HostnameSmolTrie();
+
+  lines1.forEach((line) => {
+    const apexDomain = tldts.getDomain(line.slice(8));
+    if (apexDomain) {
+      trie.add(apexDomain);
+    }
+  });
+  lines2.forEach((line) => {
+    const apexDomain = tldts.getDomain(line.slice(8));
+    if (apexDomain) {
+      trie.add(apexDomain);
+    }
+  });
+
+  const dataFromDuckDuckGo = await fetch('https://raw.githubusercontent.com/duckduckgo/tracker-radar/92e086ce38a8a88c964ed0184e5277ec1d5c8038/entities/Bright%20Data%20Ltd..json').then((res) => res.json());
+  if (typeof dataFromDuckDuckGo === 'object' && dataFromDuckDuckGo !== null && 'properties' in dataFromDuckDuckGo && Array.isArray(dataFromDuckDuckGo.properties)) {
+    dataFromDuckDuckGo.properties.forEach((prop) => {
+      trie.add(prop);
+    });
+  }
+
+  for await (const line of readFileByLine(path.join(SOURCE_DIR, 'domainset', 'reject.conf'))) {
+    const l = processLine(line);
+    if (l) {
+      trie.whitelist(l);
+    }
+  }
+  for await (const line of readFileByLine(path.join(SOURCE_DIR, 'non_ip', 'reject.conf'))) {
+    const l = processLine(line);
+    if (l) {
+      const [type, domain] = l.split(',', 3);
+      if (type === 'DOMAIN') {
+        trie.whitelist(domain, false);
+      } else if (type === 'DOMAIN-SUFFIX') {
+        trie.whitelist(domain, true);
+      }
+    }
+  }
+
+  console.log(trie.dump().map(i => '.' + i).join('\n'));
+})();