Chore: a hash collision test

Build/lib/string-hash.ts

@@ -43,4 +43,15 @@ export function fnv1a52(str: string) {
   );
 }
 
+export function fnv1a(s: string) {
+  let h = 0x81_1C_9D_C5;
+
+  for (let i = 0, l = s.length; i < l; i++) {
+    h ^= s.charCodeAt(i);
+    h += (h << 1) + (h << 4) + (h << 7) + (h << 8) + (h << 24);
+  }
+
+  return (h >>> 0);
+}
+
 export const stringHash = (payload: string) => fnv1a52(payload).toString(36) + payload.length.toString(36);

Build/validate-hash-collision-test.ts

@@ -0,0 +1,46 @@
+/* eslint-disable no-await-in-loop -- no concurrent */
+import { fdir as Fdir } from 'fdir';
+import { OUTPUT_SURGE_DIR } from './constants/dir';
+import path from 'node:path';
+import { readFileIntoProcessedArray } from './lib/fetch-text-by-line';
+import { xxhash3 } from 'hash-wasm';
+
+(async () => {
+  const hashMap = new Map<string, Set<string>>();
+
+  const runHash = async (inputs: string[]) => {
+    for (const input of inputs) {
+      const hash = await xxhash3(input);
+      if (!hashMap.has(hash)) {
+        hashMap.set(hash, new Set());
+      }
+      hashMap.get(hash)!.add(input);
+    }
+  };
+
+  const files = await new Fdir()
+    .withRelativePaths()
+    .crawl(OUTPUT_SURGE_DIR)
+    .withPromise();
+
+  for (const file of files) {
+    const fullpath = path.join(OUTPUT_SURGE_DIR, file);
+    if (file.startsWith('domainset' + path.sep)) {
+      await runHash((await readFileIntoProcessedArray(fullpath)).map(i => (i[0] === '.' ? i.slice(1) : i)));
+    } else if (file.startsWith('non_ip' + path.sep)) {
+      await runHash((await readFileIntoProcessedArray(fullpath)).map(i => i.split(',')[1]));
+    }
+  }
+
+  console.log(hashMap.size);
+  let collision = 0;
+  hashMap.forEach((v, k) => {
+    if (v.size > 1) {
+      collision++;
+      console.log(k, '=>', v);
+    }
+  });
+  if (collision === 0) {
+    console.log(hashMap);
+  }
+})();