Update Reject Hosts & Improve Luminati Hosts collection

Build/_get-lum-apex-domains.ts

@@ -1,17 +1,35 @@
-import { fetchRemoteTextByLine } from './lib/fetch-text-by-line';
+import { fetchRemoteTextByLine, readFileByLine } from './lib/fetch-text-by-line';
 import tldts from 'tldts';
+import { HostnameSmolTrie } from './lib/trie';
+import path from 'node:path';
+import { SOURCE_DIR } from './constants/dir';
+import { processLine } from './lib/process-line';
 
 (async () => {
-  const lines = await Array.fromAsync(await fetchRemoteTextByLine('https://raw.githubusercontent.com/durablenapkin/block/master/luminati.txt', true));
+  const lines1 = await Array.fromAsync(await fetchRemoteTextByLine('https://raw.githubusercontent.com/durablenapkin/block/master/luminati.txt', true));
+  const lines2 = await Array.fromAsync(await fetchRemoteTextByLine('https://raw.githubusercontent.com/durablenapkin/block/master/tvstream.txt', true));
 
-  const set = new Set<string>();
+  const trie = new HostnameSmolTrie();
 
-  lines.forEach((line) => {
+  lines1.forEach((line) => {
     const apexDomain = tldts.getDomain(line.slice(8));
     if (apexDomain) {
-      set.add(apexDomain);
+      trie.add(apexDomain);
+    }
+  });
+  lines2.forEach((line) => {
+    const apexDomain = tldts.getDomain(line.slice(8));
+    if (apexDomain) {
+      trie.add(apexDomain);
     }
   });
 
-  console.log(Array.from(set).map(line => '.' + line).join('\n'));
+  for await (const line of readFileByLine(path.join(SOURCE_DIR, 'domainset', 'reject_sukka.conf'))) {
+    const l = processLine(line);
+    if (l) {
+      trie.whitelist(l);
+    }
+  }
+
+  console.log(trie.dump().join('\n'));
 })();

Source/domainset/reject_sukka.conf

@@ -2345,6 +2345,8 @@ tongji.tom.com
 .hola.com.sg
 .hola.sk
 .proxyway.com
+.optmd.com
+.adincube.com
 # .hola.ph # expired
 .hola.ph