Refactor: make Node.js run compatible

Build/lib/cache-filesystem.ts

@@ -1,11 +1,11 @@
-// eslint-disable-next-line import-x/no-unresolved -- bun built-in module
-import { Database } from 'bun:sqlite';
+import createDb from 'better-sqlite3';
+import type { Database } from 'better-sqlite3';
 import os from 'os';
 import path from 'path';
 import { mkdirSync } from 'fs';
 import picocolors from 'picocolors';
 import { fastStringArrayJoin } from './misc';
-import { peek } from 'bun';
+import { peek } from './bun';
 import { performance } from 'perf_hooks';
 
 const identity = (x: any) => x;
@@ -92,12 +92,12 @@ export class Cache<S = string> {
       this.type = 'string';
     }
 
-    const db = new Database(path.join(this.cachePath, 'cache.db'));
+    const db = createDb(path.join(this.cachePath, 'cache.db'));
 
-    db.exec('PRAGMA journal_mode = WAL;');
-    db.exec('PRAGMA synchronous = normal;');
-    db.exec('PRAGMA temp_store = memory;');
-    db.exec('PRAGMA optimize;');
+    db.pragma('journal_mode = WAL');
+    db.pragma('synchronous = normal');
+    db.pragma('temp_store = memory');
+    db.pragma('optimize');
 
     db.prepare(`CREATE TABLE IF NOT EXISTS ${this.tableName} (key TEXT PRIMARY KEY, value ${this.type === 'string' ? 'TEXT' : 'BLOB'}, ttl REAL NOT NULL);`).run();
     db.prepare(`CREATE INDEX IF NOT EXISTS cache_ttl ON ${this.tableName} (ttl);`).run();
@@ -130,15 +130,20 @@ export class Cache<S = string> {
       `INSERT INTO ${this.tableName} (key, value, ttl) VALUES ($key, $value, $valid) ON CONFLICT(key) DO UPDATE SET value = $value, ttl = $valid`
     );
 
+    const valid = Date.now() + ttl;
+
     insert.run({
       $key: key,
+      key,
       $value: value,
-      $valid: Date.now() + ttl
+      value,
+      $valid: valid,
+      valid
     });
   }
 
   get(key: string, defaultValue?: S): S | undefined {
-    const rv = this.db.prepare<{ value: S }, string>(
+    const rv = this.db.prepare<string, { value: S }>(
       `SELECT value FROM ${this.tableName} WHERE key = ? LIMIT 1`
     ).get(key);
 
@@ -148,7 +153,7 @@ export class Cache<S = string> {
 
   has(key: string): CacheStatus {
     const now = Date.now();
-    const rv = this.db.prepare<{ ttl: number }, string>(`SELECT ttl FROM ${this.tableName} WHERE key = ?`).get(key);
+    const rv = this.db.prepare<string, { ttl: number }>(`SELECT ttl FROM ${this.tableName} WHERE key = ?`).get(key);
 
     return !rv ? CacheStatus.Miss : (rv.ttl > now ? CacheStatus.Hit : CacheStatus.Stale);
   }
@@ -206,18 +211,14 @@ export class Cache<S = string> {
   }
 }
 
-export const fsFetchCache = new Cache({ cachePath: path.resolve(import.meta.dir, '../../.cache') });
+export const fsFetchCache = new Cache({ cachePath: path.resolve(__dirname, '../../.cache') });
 // process.on('exit', () => {
 //   fsFetchCache.destroy();
 // });
 
-// export const fsCache = traceSync('initializing filesystem cache', () => new Cache<Uint8Array>({ cachePath: path.resolve(import.meta.dir, '../../.cache'), type: 'buffer' }));
+// export const fsCache = traceSync('initializing filesystem cache', () => new Cache<Uint8Array>({ cachePath: path.resolve(__dirname, '../../.cache'), type: 'buffer' }));
 
 const separator = '\u0000';
-// const textEncoder = new TextEncoder();
-// const textDecoder = new TextDecoder();
-// export const serializeString = (str: string) => textEncoder.encode(str);
-// export const deserializeString = (str: string) => textDecoder.decode(new Uint8Array(str.split(separator).map(Number)));
 export const serializeSet = (set: Set<string>) => fastStringArrayJoin(Array.from(set), separator);
 export const deserializeSet = (str: string) => new Set(str.split(separator));
 export const serializeArray = (arr: string[]) => fastStringArrayJoin(arr, separator);

Build/lib/create-file.ts

@@ -11,8 +11,6 @@ import { readFileByLine } from './fetch-text-by-line';
 
 export async function compareAndWriteFile(span: Span, linesA: string[], filePath: string) {
   let isEqual = true;
-  const fd = await fsp.open(filePath);
-
   const linesALen = linesA.length;
 
   if (!fs.existsSync(filePath)) {
@@ -22,12 +20,10 @@ export async function compareAndWriteFile(span: Span, linesA: string[], filePath
     console.log(`Nothing to write to ${filePath}...`);
     isEqual = false;
   } else {
-    /* The `isEqual` variable is used to determine whether the content of a file is equal to the
-    provided lines or not. It is initially set to `true`, and then it is updated based on different
-    conditions: */
+    const fd = await fsp.open(filePath);
+
     isEqual = await span.traceChildAsync(`comparing ${filePath}`, async () => {
       let index = 0;
-
       for await (const lineB of readFileByLine(fd)) {
         const lineA = linesA[index] as string | undefined;
         index++;
@@ -63,6 +59,8 @@ export async function compareAndWriteFile(span: Span, linesA: string[], filePath
 
       return true;
     });
+
+    await fd.close();
   }
 
   if (isEqual) {
@@ -83,8 +81,6 @@ export async function compareAndWriteFile(span: Span, linesA: string[], filePath
 
     // return writer.end();
   });
-
-  await fd.close();
 }
 
 export const withBannerArray = (title: string, description: string[] | readonly string[], date: Date, content: string[]) => {

Build/lib/fetch-text-by-line.bench.ts

@@ -4,7 +4,7 @@ import { readFileByLine } from './fetch-text-by-line';
 import path from 'path';
 import fsp from 'fs/promises';
 
-const file = path.resolve(import.meta.dir, '../../Source/domainset/cdn.conf');
+const file = path.resolve(__dirname, '../../Source/domainset/cdn.conf');
 
 group('read file by line', () => {
   bench('readline', () => processLineFromReadline(readFileByLine(file)));

Build/lib/fetch-text-by-line.ts

@@ -1,9 +1,12 @@
+import fs from 'fs';
+import { Readable } from 'stream';
 import type { BunFile } from 'bun';
 import { fetchWithRetry, defaultRequestInit } from './fetch-retry';
 import type { FileHandle } from 'fs/promises';
 
 import { TextLineStream } from './text-line-transform-stream';
 import { PolyfillTextDecoderStream } from './text-decoder-stream';
+import { TextDecoderStream as NodeTextDecoderStream } from 'stream/web';
 import { processLine } from './process-line';
 
 const enableTextLineStream = !!process.env.ENABLE_TEXT_LINE_STREAM;
@@ -36,19 +39,32 @@ async function *createTextLineAsyncIterableFromStreamSource(stream: ReadableStre
   }
 }
 
-const getReadableStream = (file: string | BunFile | FileHandle): ReadableStream => {
-  if (typeof file === 'string') {
-    return Bun.file(file).stream();
+const getReadableStream = typeof Bun !== 'undefined'
+  ? (file: string | BunFile | FileHandle): ReadableStream => {
+    if (typeof file === 'string') {
+      return Bun.file(file).stream();
+    }
+    if ('writer' in file) {
+      return file.stream();
+    }
+    return file.readableWebStream();
   }
-  if ('writer' in file) {
-    return file.stream();
-  }
-  return file.readableWebStream();
-};
+  : (file: string | BunFile | FileHandle): ReadableStream => {
+    if (typeof file === 'string') {
+      return Readable.toWeb(fs.createReadStream(file /* { encoding: 'utf-8' } */));
+    }
+    if ('writer' in file) {
+      return file.stream();
+    }
+    return file.readableWebStream();
+  };
+
+// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- On Bun, NodeTextDecoderStream is undefined
+const TextDecoderStream = NodeTextDecoderStream ?? PolyfillTextDecoderStream;
 
 // TODO: use FileHandle.readLine()
 export const readFileByLine: ((file: string | BunFile | FileHandle) => AsyncIterable<string>) = enableTextLineStream
-  ? (file: string | BunFile | FileHandle) => getReadableStream(file).pipeThrough(new PolyfillTextDecoderStream()).pipeThrough(new TextLineStream())
+  ? (file: string | BunFile | FileHandle) => getReadableStream(file).pipeThrough(new TextDecoderStream()).pipeThrough(new TextLineStream())
   : (file: string | BunFile | FileHandle) => createTextLineAsyncIterableFromStreamSource(getReadableStream(file));
 
 const ensureResponseBody = (resp: Response) => {
@@ -62,7 +78,7 @@ const ensureResponseBody = (resp: Response) => {
 };
 
 export const createReadlineInterfaceFromResponse: ((resp: Response) => AsyncIterable<string>) = enableTextLineStream
-  ? (resp) => ensureResponseBody(resp).pipeThrough(new PolyfillTextDecoderStream()).pipeThrough(new TextLineStream())
+  ? (resp) => ensureResponseBody(resp).pipeThrough(new TextDecoderStream()).pipeThrough(new TextLineStream())
   : (resp) => createTextLineAsyncIterableFromStreamSource(ensureResponseBody(resp));
 
 export function fetchRemoteTextByLine(url: string | URL) {

Build/lib/get-gorhill-publicsuffix.ts

@@ -1,10 +1,16 @@
-import { toASCII } from 'punycode';
+import fsp from 'fs/promises';
+import { toASCII } from 'punycode/punycode';
 import { createMemoizedPromise } from './memo-promise';
 import { getPublicSuffixListTextPromise } from './download-publicsuffixlist';
+import { fileURLToPath } from 'url';
 
 const customFetch = typeof Bun !== 'undefined'
   ? (url: string | URL) => Promise.resolve(Bun.file(url))
-  : (url: string | URL) => fetch(url).then(resp => resp.blob() as Promise<Blob>);
+  : async (url: string | URL) => {
+    const filePath = fileURLToPath(url);
+    const file = await fsp.readFile(filePath);
+    return new Blob([file]) as any;
+  };
 
 export const getGorhillPublicSuffixPromise = createMemoizedPromise(async () => {
   const [publicSuffixListDat, { default: gorhill }] = await Promise.all([