Chore: minor changes

Build/lib/aho-corasick.ts

@@ -1,8 +1,8 @@
 interface Node {
   /** @default false */
-  wordEnd?: boolean,
+  wordEnd: boolean,
   children: Map<string, Node | undefined>,
-  fail?: Node
+  fail: Node | undefined
 }
 
 const createNode = (): Node => ({

Build/lib/get-phishing-domains.ts

@@ -5,6 +5,7 @@ import { getSubdomain, getPublicSuffix } from 'tldts-experimental';
 import type { Span } from '../trace';
 import { appendArrayInPlaceCurried } from './append-array-in-place';
 import { PHISHING_DOMAIN_LISTS } from './reject-data-source';
+import { looseTldtsOpt } from '../constants/loose-tldts-opt';
 
 const BLACK_TLD = new Set([
   'accountant',
@@ -99,14 +100,6 @@ export const WHITELIST_MAIN_DOMAINS = new Set([
   'notion.site'
 ]);
 
-const tldtsOpt: Parameters<typeof getSubdomain>[1] = {
-  allowPrivateDomains: false,
-  extractHostname: false,
-  validateHostname: false,
-  detectIp: false,
-  mixedInputs: false
-};
-
 export const getPhishingDomains = (parentSpan: Span) => parentSpan.traceChild('get phishing domains').traceAsyncFn(async (span) => {
   const gorhill = await getGorhillPublicSuffixPromise();
 
@@ -132,7 +125,7 @@ export const getPhishingDomains = (parentSpan: Span) => parentSpan.traceChild('g
         continue;
       }
 
-      const tld = getPublicSuffix(safeGorhillLine, tldtsOpt);
+      const tld = getPublicSuffix(safeGorhillLine, looseTldtsOpt);
       if (!tld || !BLACK_TLD.has(tld)) continue;
 
       domainCountMap[apexDomain] ||= 0;
@@ -187,7 +180,7 @@ export function calcDomainAbuseScore(line: string) {
     }
   }
 
-  const subdomain = getSubdomain(line, tldtsOpt);
+  const subdomain = getSubdomain(line, looseTldtsOpt);
 
   if (subdomain) {
     if (subdomain.slice(1).includes('.')) {

Build/lib/normalize-domain.ts

@@ -7,6 +7,7 @@ export const normalizeDomain = (domain: string) => {
   const parsed = tldtsParse(domain, { allowPrivateDomains: true, detectIp: false });
   // if (parsed.isIp) return null;
   if (!parsed.hostname) return null;
+  // Private invalid domain (things like .tor, .dn42, etc)
   if (!parsed.isIcann && !parsed.isPrivate) return null;
 
   let h = parsed.hostname;

Build/lib/stable-sort-domain.ts

@@ -3,32 +3,51 @@
 // enough when sorting.
 import { getDomain, getSubdomain } from 'tldts-experimental';
 import { sort } from './timsort';
+import { looseTldtsOpt } from '../constants/loose-tldts-opt';
 
 export const compare = (a: string, b: string) => {
   if (a === b) return 0;
   return (a.length - b.length) || a.localeCompare(b);
 };
 
-const tldtsOpt: Parameters<typeof getDomain>[1] = {
-  allowPrivateDomains: false,
-  extractHostname: false,
-  validateHostname: false,
-  detectIp: false,
-  mixedInputs: false
-};
-
-export const sortDomains = (inputs: string[]) => {
+export const buildParseDomainMap = (inputs: string[]) => {
   const domainMap = new Map<string, string>();
   const subdomainMap = new Map<string, string>();
 
   for (let i = 0, len = inputs.length; i < len; i++) {
     const cur = inputs[i];
     if (!domainMap.has(cur)) {
-      const topD = getDomain(cur, tldtsOpt);
+      const topD = getDomain(cur, looseTldtsOpt);
       domainMap.set(cur, topD ?? cur);
     }
     if (!subdomainMap.has(cur)) {
-      const subD = getSubdomain(cur, tldtsOpt);
+      const subD = getSubdomain(cur, looseTldtsOpt);
+      subdomainMap.set(cur, subD ?? cur);
+    }
+  }
+
+  return { domainMap, subdomainMap };
+};
+
+export const sortDomains = (
+  inputs: string[],
+  domainMap?: Map<string, string>,
+  subdomainMap?: Map<string, string>
+) => {
+  if (!domainMap || !subdomainMap) {
+    const { domainMap: dm, subdomainMap: sm } = buildParseDomainMap(inputs);
+    domainMap = dm;
+    subdomainMap = sm;
+  }
+
+  for (let i = 0, len = inputs.length; i < len; i++) {
+    const cur = inputs[i];
+    if (!domainMap.has(cur)) {
+      const topD = getDomain(cur, looseTldtsOpt);
+      domainMap.set(cur, topD ?? cur);
+    }
+    if (!subdomainMap.has(cur)) {
+      const subD = getSubdomain(cur, looseTldtsOpt);
       subdomainMap.set(cur, subD ?? cur);
     }
   }

Build/lib/trie.ts

@@ -36,32 +36,34 @@ const createNode = (parent: TrieNode | null = null): TrieNode => {
   return node;
 };
 
+const hostnameToTokens = (hostname: string): string[] => {
+  let buf = '';
+  const tokens: string[] = [];
+  for (let i = 0, l = hostname.length; i < l; i++) {
+    const c = hostname[i];
+    if (c === '.') {
+      if (buf) {
+        tokens.push(buf, /* . */ c);
+        buf = '';
+      } else {
+        tokens.push(/* . */ c);
+      }
+    } else {
+      buf += c;
+    }
+  }
+  if (buf) {
+    tokens.push(buf);
+  }
+  return tokens;
+};
+
 export const createTrie = (from?: string[] | Set<string> | null, hostnameMode = false, smolTree = false) => {
   let size = 0;
   const root: TrieNode = createNode();
 
   const suffixToTokens = hostnameMode
-    ? (suffix: string) => {
-      let buf = '';
-      const tokens: string[] = [];
-      for (let i = 0, l = suffix.length; i < l; i++) {
-        const c = suffix[i];
-        if (c === '.') {
-          if (buf) {
-            tokens.push(buf, /* . */ c);
-            buf = '';
-          } else {
-            tokens.push(/* . */ c);
-          }
-        } else {
-          buf += c;
-        }
-      }
-      if (buf) {
-        tokens.push(buf);
-      }
-      return tokens;
-    }
+    ? hostnameToTokens
     : (suffix: string) => suffix;
 
   /**