Refactor: minor changes here and there

Build/constants/reject-data-source.ts

@@ -1,5 +1,7 @@
 import { TTL } from '../lib/cache-filesystem';
 
+export const DEBUG_DOMAIN_TO_FIND: string | null = null; // example.com | null
+
 type HostsSource = [main: string, mirrors: string[] | null, includeAllSubDomain: boolean, ttl: number];
 
 export const HOSTS: HostsSource[] = [

Build/lib/aho-corasick.ts

@@ -12,11 +12,9 @@ function createKeywordFilter(keys: string[] | Set<string>) {
 
   // Create a trie with extra fields and information
   const put = (key: string) => {
-    const len = key.length;
-
     let node = root;
 
-    for (let idx = 0; idx < len; idx++) {
+    for (let idx = 0, len = key.length; idx < len; idx++) {
       const char = key[idx];
 
       if (node.has(char)) {

Build/lib/cache-filesystem.ts

@@ -153,7 +153,7 @@ export class Cache<S = string> {
     }
 
     const end = performance.now();
-    console.log(`${picocolors.gray(`[${((end - start) / 1e6).toFixed(3)}ms]`)} cache initialized from ${this.cachePath}`);
+    console.log(`${picocolors.gray(`[${((end - start)).toFixed(3)}ns]`)} cache initialized from ${this.tableName} @ ${this.cachePath}`);
   }
 
   set(key: string, value: string, ttl = 60 * 1000): void {

Build/lib/get-phishing-domains.ts

@@ -4,12 +4,13 @@ import * as tldts from 'tldts-experimental';
 import { dummySpan, printTraceResult } from '../trace';
 import type { Span } from '../trace';
 import { appendArrayInPlaceCurried } from './append-array-in-place';
-import { PHISHING_DOMAIN_LISTS_EXTRA, PHISHING_HOSTS_EXTRA } from '../constants/reject-data-source';
+import { DEBUG_DOMAIN_TO_FIND, PHISHING_DOMAIN_LISTS_EXTRA, PHISHING_HOSTS_EXTRA } from '../constants/reject-data-source';
 import { loosTldOptWithPrivateDomains } from '../constants/loose-tldts-opt';
 import picocolors from 'picocolors';
 import createKeywordFilter from './aho-corasick';
 import { createCacheKey, deserializeArray, serializeArray } from './cache-filesystem';
 import { cache } from './fs-memo';
+import { isCI } from 'ci-info';
 
 const BLACK_TLD = new Set([
   'accountant', 'art', 'autos',
@@ -112,12 +113,22 @@ const processPhihsingDomains = cache(function processPhihsingDomains(domainArr:
   const domainCountMap = new Map<string, number>();
   const domainScoreMap: Record<string, number> = {};
 
+  let line = '';
   let tld: string | null = '';
   let apexDomain: string | null = '';
   let subdomain: string | null = '';
 
+  // const set = new Set<string>();
+  // let duplicateCount = 0;
+
   for (let i = 0, len = domainArr.length; i < len; i++) {
-    const line = domainArr[i];
+    line = domainArr[i];
+
+    // if (set.has(line)) {
+    //   duplicateCount++;
+    // } else {
+    //   set.add(line);
+    // }
 
     const parsed = tldts.parse(line, loosTldOptWithPrivateDomains);
     if (parsed.isPrivate) {
@@ -183,11 +194,13 @@ const processPhihsingDomains = cache(function processPhihsingDomains(domainArr:
   //   count: domainCountMap.get('flk-ipfs.xyz')
   // });
 
+  // console.log({ duplicateCount, domainArrLen: domainArr.length });
+
   return Promise.resolve(domainArr);
 }, {
   serializer: serializeArray,
   deserializer: deserializeArray,
-  temporaryBypass: true
+  temporaryBypass: !isCI || DEBUG_DOMAIN_TO_FIND !== null
 });
 
 const cacheKey = createCacheKey(__filename);
@@ -205,6 +218,8 @@ export function getPhishingDomains(parentSpan: Span) {
       return domainArr;
     });
 
+    console.log({ len: domainArr.length });
+
     return span.traceChildAsync(
       'process phishing domain set',
       () => processPhihsingDomains(domainArr)

Build/lib/parse-filter.ts

@@ -9,8 +9,8 @@ import type { Span } from '../trace';
 import createKeywordFilter from './aho-corasick';
 import { looseTldtsOpt } from '../constants/loose-tldts-opt';
 import { identity } from './misc';
+import { DEBUG_DOMAIN_TO_FIND } from '../constants/reject-data-source';
 
-const DEBUG_DOMAIN_TO_FIND: string | null = null; // example.com | null
 let foundDebugDomain = false;
 const temporaryBypass = typeof DEBUG_DOMAIN_TO_FIND === 'string';
 

Build/lib/rules/base.ts

@@ -65,7 +65,11 @@ export abstract class RuleOutput<TPreprocessed = unknown> {
     return result;
   };
 
-  constructor(protected readonly span: Span, protected readonly id: string) { }
+  protected readonly span: Span;
+
+  constructor($span: Span, protected readonly id: string) {
+    this.span = $span.traceChild('RuleOutput#' + id);
+  }
 
   protected title: string | null = null;
   withTitle(title: string) {
@@ -201,7 +205,7 @@ export abstract class RuleOutput<TPreprocessed = unknown> {
     return this;
   }
 
-  static readonly ipToCidr = (ip: string, version: 4 | 6 = 4) => {
+  static readonly ipToCidr = (ip: string, version: 4 | 6) => {
     if (ip.includes('/')) return ip;
     if (version === 4) {
       return ip + '/32';
@@ -257,7 +261,7 @@ export abstract class RuleOutput<TPreprocessed = unknown> {
     if (this.$$preprocessed === null) {
       this.guardPendingPromise();
 
-      this.$$preprocessed = this.span.traceChildSync('RuleOutput#preprocess: ' + this.id, () => this.preprocess());
+      this.$$preprocessed = this.span.traceChildSync('preprocess', () => this.preprocess());
     }
     return this.$$preprocessed;
   }
@@ -280,9 +284,8 @@ export abstract class RuleOutput<TPreprocessed = unknown> {
     );
   }
 
-  async write(): Promise<void> {
-    await this.done();
-
+  write(): Promise<void> {
+    return this.done().then(() => this.span.traceChildAsync('write all', async () => {
       invariant(this.title, 'Missing title');
       invariant(this.description, 'Missing description');
 
@@ -330,6 +333,7 @@ export abstract class RuleOutput<TPreprocessed = unknown> {
       }
 
       await Promise.all(promises);
+    }));
   }
 
   abstract surge(): string[];

Build/trace/index.ts

@@ -148,7 +148,15 @@ export async function whyIsNodeRunning() {
 
 export function printTraceResult(traceResult: TraceResult = rootTraceResult) {
   printStats(traceResult.children);
-  printTree(traceResult, node => `${node.name} ${picocolors.bold(`${(node.end - node.start).toFixed(3)}ms`)}`);
+  printTree(
+    traceResult,
+    node => {
+      if (node.end - node.start < 0) {
+        return node.name;
+      }
+      return `${node.name} ${picocolors.bold(`${(node.end - node.start).toFixed(3)}ms`)}`;
+    }
+  );
 }
 
 function printTree(initialTree: TraceResult, printNode: (node: TraceResult, branch: string) => string) {