Minor changes

Build/lib/cache-filesystem.ts

@@ -2,8 +2,9 @@
 import { Database } from 'bun:sqlite';
 import os from 'os';
 import path from 'path';
-import fs from 'fs';
+import { mkdirSync } from 'fs';
 import picocolors from 'picocolors';
+import { traceSync } from './trace-runner';
 
 const identity = (x: any) => x;
 
@@ -64,7 +65,7 @@ export class Cache {
 
   constructor({ cachePath = path.join(os.tmpdir() || '/tmp', 'hdc'), tbd }: CacheOptions = {}) {
     this.cachePath = cachePath;
-    fs.mkdirSync(this.cachePath, { recursive: true });
+    mkdirSync(this.cachePath, { recursive: true });
     if (tbd != null) this.tbd = tbd;
 
     const db = new Database(path.join(this.cachePath, 'cache.db'));
@@ -151,7 +152,7 @@ export class Cache {
   }
 }
 
-export const fsCache = new Cache({ cachePath: path.resolve(import.meta.dir, '../../.cache') });
+export const fsCache = traceSync('initializing filesystem cache', () => new Cache({ cachePath: path.resolve(import.meta.dir, '../../.cache') }));
 // process.on('exit', () => {
 //   fsCache.destroy();
 // });

Build/lib/cached-tld-parse.ts

@@ -12,6 +12,7 @@ const sharedConfig2 = { allowPrivateDomains: true, detectIp: false };
 export const parse = (domain: string) => cache.sync(domain, () => tldts.parse(domain, sharedConfig));
 /** { allowPrivateDomains: true, detectIp: false } */
 export const parse2 = (domain: string) => cache2.sync(domain, () => tldts.parse(domain, sharedConfig2));
+export const parseWithoutDetectIp = parse2;
 
 let gothillGetDomainCache: ReturnType<typeof createCache> | null = null;
 export const createCachedGorhillGetDomain = (gorhill: PublicSuffixList) => {

Build/lib/create-file.ts

@@ -18,7 +18,7 @@ export async function compareAndWriteFile(linesA: string[], filePath: string) {
     isEqual = false;
   } else {
     isEqual = await traceAsync(
-      picocolors.gray(`Comparing ${filePath}`),
+      picocolors.gray(`comparing ${filePath}`),
       async () => {
         let index = 0;
 
@@ -62,11 +62,11 @@ export async function compareAndWriteFile(linesA: string[], filePath: string) {
   }
 
   if (isEqual) {
-    console.log(picocolors.gray(`Same Content, bail out writing: ${filePath}`));
+    console.log(picocolors.dim(`same content, bail out writing: ${filePath}`));
     return;
   }
 
-  await traceAsync(picocolors.gray(`Writing ${filePath}`), async () => {
+  await traceAsync(picocolors.gray(`writing ${filePath}`), async () => {
     if (linesALen < 10000) {
       return Bun.write(file, `${linesA.join('\n')}\n`);
     }

Build/lib/fetch-text-by-line.ts

@@ -3,34 +3,34 @@ import { fetchWithRetry, defaultRequestInit } from './fetch-retry';
 
 import { TextLineStream } from './text-line-transform-stream';
 import { PolyfillTextDecoderStream } from './text-decoder-stream';
-function createTextLineStreamFromStreamSource(stream: ReadableStream<Uint8Array>) {
-  return stream
-    .pipeThrough(new PolyfillTextDecoderStream())
-    .pipeThrough(new TextLineStream());
-}
-
-// const decoder = new TextDecoder('utf-8');
-// async function *createTextLineAsyncGeneratorFromStreamSource(stream: ReadableStream<Uint8Array>): AsyncGenerator<string> {
-//   let buf = '';
-
-//   for await (const chunk of stream) {
-//     const chunkStr = decoder.decode(chunk).replaceAll('\r\n', '\n');
-//     for (let i = 0, len = chunkStr.length; i < len; i++) {
-//       const char = chunkStr[i];
-//       if (char === '\n') {
-//         yield buf;
-//         buf = '';
-//       } else {
-//         buf += char;
-//       }
-//     }
-//   }
-
-//   if (buf) {
-//     yield buf;
-//   }
+// function createTextLineStreamFromStreamSource(stream: ReadableStream<Uint8Array>) {
+//   return stream
+//     .pipeThrough(new PolyfillTextDecoderStream())
+//     .pipeThrough(new TextLineStream());
 // }
 
+const decoder = new TextDecoder('utf-8');
+async function *createTextLineAsyncGeneratorFromStreamSource(stream: ReadableStream<Uint8Array>): AsyncGenerator<string> {
+  let buf = '';
+
+  for await (const chunk of stream) {
+    const chunkStr = decoder.decode(chunk).replaceAll('\r\n', '\n');
+    for (let i = 0, len = chunkStr.length; i < len; i++) {
+      const char = chunkStr[i];
+      if (char === '\n') {
+        yield buf;
+        buf = '';
+      } else {
+        buf += char;
+      }
+    }
+  }
+
+  if (buf) {
+    yield buf;
+  }
+}
+
 export function readFileByLine(file: string | URL | BunFile) {
   if (typeof file === 'string') {
     file = Bun.file(file);
@@ -38,7 +38,7 @@ export function readFileByLine(file: string | URL | BunFile) {
     file = Bun.file(file);
   }
 
-  return createTextLineStreamFromStreamSource(file.stream());
+  return createTextLineAsyncGeneratorFromStreamSource(file.stream());
 }
 
 export function createReadlineInterfaceFromResponse(resp: Response) {
@@ -49,7 +49,7 @@ export function createReadlineInterfaceFromResponse(resp: Response) {
     throw new Error('Body has already been consumed.');
   }
 
-  return createTextLineStreamFromStreamSource(resp.body);
+  return createTextLineAsyncGeneratorFromStreamSource(resp.body);
 }
 
 export function fetchRemoteTextByLine(url: string | URL) {

Build/lib/parse-filter.ts

@@ -56,23 +56,20 @@ export function processHosts(hostsUrl: string, includeAllSubDomain = false, ttl:
           continue;
         }
 
-        const domain = line.split(/\s/)[1];
+        const _domain = line.split(/\s/)[1]?.trim();
+        if (!_domain) {
+          continue;
+        }
+        const domain = normalizeDomain(_domain);
         if (!domain) {
           continue;
         }
-        const _domain = domain.trim();
-
-        if (DEBUG_DOMAIN_TO_FIND && _domain.includes(DEBUG_DOMAIN_TO_FIND)) {
-          console.warn(picocolors.red(hostsUrl), '(black)', _domain.replaceAll(DEBUG_DOMAIN_TO_FIND, picocolors.bold(DEBUG_DOMAIN_TO_FIND)));
+        if (DEBUG_DOMAIN_TO_FIND && domain.includes(DEBUG_DOMAIN_TO_FIND)) {
+          console.warn(picocolors.red(hostsUrl), '(black)', domain.replaceAll(DEBUG_DOMAIN_TO_FIND, picocolors.bold(DEBUG_DOMAIN_TO_FIND)));
           foundDebugDomain = true;
         }
 
-        const domainToAdd = normalizeDomain(_domain);
-        if (!domainToAdd) {
-          continue;
-        }
-
-        domainSets.add(includeAllSubDomain ? `.${domainToAdd}` : domainToAdd);
+        domainSets.add(includeAllSubDomain ? `.${domain}` : domain);
       }
 
       console.log(picocolors.gray('[process hosts]'), picocolors.gray(hostsUrl), picocolors.gray(domainSets.size));
@@ -102,11 +99,11 @@ export async function processFilterRules(
   fallbackUrls?: readonly string[] | undefined | null,
   ttl: number | null = null
 ): Promise<{ white: string[], black: string[], foundDebugDomain: boolean }> {
-  const [white, black, warningMessages] = await traceAsync(`- processFilterRules: ${filterRulesUrl}`, () => fsCache.apply<[
+  const [white, black, warningMessages] = await traceAsync(`- processFilterRules: ${filterRulesUrl}`, () => fsCache.apply<Readonly<[
     white: string[],
     black: string[],
     warningMessages: string[]
-  ]>(
+  ]>>(
     filterRulesUrl,
     async () => {
       const whitelistDomainSets = new Set<string>();

Build/lib/reject-data-source.ts

@@ -4,7 +4,7 @@ export const HOSTS = [
   ['https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=0&mimetype=plaintext', true, TTL.THREE_HOURS()],
   ['https://someonewhocares.org/hosts/hosts', true, TTL.THREE_HOURS()],
   // no coin list is not actively maintained, but it updates daily when being maintained, so we set a 3 days cache ttl
-  ['https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt', false, TTL.THREE_DAYS()],
+  ['https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt', true, TTL.THREE_DAYS()],
   // have not been updated for more than a year, so we set a 14 days cache ttl
   ['https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/spy.txt', true, TTL.TWO_WEEKS()],
   ['https://raw.githubusercontent.com/jerryn70/GoodbyeAds/master/Extension/GoodbyeAds-Xiaomi-Extension.txt', false, TTL.THREE_DAYS()],