Chore: embed content hash

Build/lib/content-hash.ts

@@ -0,0 +1,86 @@
+import { fastStringArrayJoin } from 'foxts/fast-string-array-join';
+import { Buffer } from 'node:buffer';
+import { createHash } from 'node:crypto';
+import fsp from 'node:fs/promises';
+
+/**
+ * A special marker embedded in the metadata comment of output files, carrying the hash
+ * of the "real content" (title, description and rules -- but not the volatile
+ * "Last Updated" date). This way, whether a file needs to be re-written can be
+ * determined by reading only the first few hundred bytes of the previous output
+ * when possible.
+ *
+ * If the hash algorithm or what's included in the hash ever changes, bump the
+ * version suffix to force a one-time re-write of every file.
+ */
+const CONTENT_HASH_MARKER_START = '$content-hash-v1$:';
+const CONTENT_HASH_MARKER_END = '$';
+
+/**
+ * Hash the real content and return the ready-to-embed, self-delimited token,
+ * e.g. `$content-hash-v1$:AbCd...$`. Writers only need to prepend their own
+ * comment prefix (`# ` / `! `).
+ */
+export function calculateContentHash(title: string, description: string[] | readonly string[], content: string[]): string {
+  const hasher = createHash('sha256');
+  hasher.update(title);
+  hasher.update('\0');
+  hasher.update(fastStringArrayJoin(description, '\n'));
+  hasher.update('\0');
+  hasher.update(fastStringArrayJoin(content, '\n'));
+  return CONTENT_HASH_MARKER_START + hasher.digest('base64url') + CONTENT_HASH_MARKER_END;
+}
+
+/**
+ * The content hash token sits at the bottom of the leading metadata comment,
+ * so reading the first chunk of the file is enough to locate it. The chunk must
+ * be larger than the biggest banner (title + description + data sources, ~2.2 KiB
+ * as of writing) -- if the token falls outside the chunk, extraction returns null
+ * and the caller silently degrades to the full file comparison.
+ */
+const FILE_HEAD_CHUNK_SIZE = 8192;
+
+async function readFileHead(filePath: string): Promise<string> {
+  let fd: fsp.FileHandle | null = null;
+  try {
+    fd = await fsp.open(filePath, 'r');
+    const buf = Buffer.allocUnsafe(FILE_HEAD_CHUNK_SIZE);
+    const { bytesRead } = await fd.read(buf, 0, FILE_HEAD_CHUNK_SIZE, 0);
+    return buf.toString('utf8', 0, bytesRead);
+  } finally {
+    await fd?.close();
+  }
+}
+
+/**
+ * Extract the content hash token from a previously written file, reading only
+ * the first chunk of it. Returns the same self-delimited token shape that
+ * calculateContentHash produces, so the two can be compared directly. Returns
+ * null if the file predates the content hash marker, or if the token is
+ * malformed / truncated by the fixed-size head read (in which case the caller
+ * falls back to the full comparison, which is always safe).
+ */
+export async function extractContentHashFromFile(filePath: string): Promise<string | null> {
+  return extractContentHash(await readFileHead(filePath));
+}
+
+function extractContentHash(fileHead: string): string | null {
+  const markerIndex = fileHead.indexOf(CONTENT_HASH_MARKER_START);
+  if (markerIndex === -1) {
+    return null;
+  }
+
+  const start = markerIndex + CONTENT_HASH_MARKER_START.length;
+  const end = fileHead.indexOf(CONTENT_HASH_MARKER_END, start);
+  // also covers end === -1 (unterminated token) and end === start (empty hash)
+  if (end <= start) {
+    return null;
+  }
+
+  // the end marker must sit on the same line as the start marker
+  if (fileHead.slice(start, end).includes('\n')) {
+    return null;
+  }
+
+  return fileHead.slice(markerIndex, end + CONTENT_HASH_MARKER_END.length);
+}

Build/lib/create-file.ts

@@ -7,27 +7,41 @@ import { readFileByLine } from './fetch-text-by-line';
 import { writeFile } from './misc';
 import { createCompareSource, fileEqualWithCommentComparator } from 'foxts/compare-source';
 import { promisify } from 'node:util';
+import { extractContentHashFromFile } from './content-hash';
 
 const fileEqual = createCompareSource(fileEqualWithCommentComparator);
 
 /**
  * To keep metadata comment `last updated` not change if real content is the same,
- * we compare real content lines and only write if actual content is different,
+ * we only write when the actual content differs, and the new `last updated` will
- * and the new `last updated` will be written along with new content.
+ * be written along with new content.
+ *
+ * When `contentHash` is provided (and the previous output already embeds a
+ * content hash marker), the comparison only reads the first chunk of the
+ * previous file. Otherwise it falls back to a full comment-insensitive
+ * line-by-line comparison.
  */
-export async function compareAndWriteFile(span: Span, linesA: string[], filePath: string) {
+export async function compareAndWriteFile(span: Span, linesA: string[], filePath: string, contentHash: string | null = null) {
   // readFileByLine will not include last empty line. So we always pop the linesA for comparison purpose
   if (linesA.length > 0 && linesA[linesA.length - 1] === '') {
     linesA.pop();
   }
 
   const isEqual = await span.traceChildAsync(`compare ${filePath}`, async () => {
-    if (fs.existsSync(filePath)) {
+    if (!fs.existsSync(filePath)) {
-      return fileEqual(linesA, readFileByLine(filePath));
-    }
-
       console.log(`${filePath} does not exists, writing...`);
       return false;
+    }
+
+    if (contentHash) {
+      const previousHash = await extractContentHashFromFile(filePath);
+      if (previousHash) {
+        return previousHash === contentHash;
+      }
+      // previous output predates the content hash marker, fall through to full comparison
+    }
+
+    return fileEqual(linesA, readFileByLine(filePath));
   });
 
   if (isEqual) {

Build/lib/misc.ts

@@ -30,7 +30,7 @@ export const writeFile: Write = async (destination: string, input, dir = dirname
   return fsp.writeFile(destination, input, { encoding: 'utf-8' });
 };
 
-export function withBannerArray(title: string, description: string[] | readonly string[], date: Date, content: string[]) {
+export function withBannerArray(title: string, description: string[] | readonly string[], date: Date, content: string[], contentHash: string | null = null) {
   const result: string[] = [
     '#########################################',
     `# ${title}`,
@@ -40,6 +40,10 @@ export function withBannerArray(title: string, description: string[] | readonly
 
   appendArrayInPlace(result, description.map(line => (line ? `# ${line}` : '#')));
 
+  if (contentHash) {
+    result.push('#', `# ${contentHash}`);
+  }
+
   result.push('#########################################');
 
   appendArrayInPlace(result, content);

Build/lib/writing-strategy/adguardhome.ts

@@ -13,7 +13,7 @@ export class AdGuardHome extends BaseWriteStrategy {
   protected result: string[] = [];
 
   // eslint-disable-next-line @typescript-eslint/class-methods-use-this -- abstract method
-  withPadding(title: string, description: string[] | readonly string[], date: Date, content: string[]): string[] {
+  withPadding(title: string, description: string[] | readonly string[], date: Date, content: string[], contentHash: string | null): string[] {
     return [
       `! Title: ${title}`,
       '! Last modified: ' + date.toUTCString(),
@@ -21,6 +21,7 @@ export class AdGuardHome extends BaseWriteStrategy {
       '! License: https://github.com/SukkaW/Surge/blob/master/LICENSE',
       '! Homepage: https://github.com/SukkaW/Surge',
       `! Description: ${description.join(' ')}`,
+      ...(contentHash ? [`! ${contentHash}`] : []),
       '!',
       ...content,
       '! EOF'

Build/lib/writing-strategy/base.ts

@@ -1,4 +1,5 @@
 import type { Span } from '../../trace';
+import { calculateContentHash } from '../content-hash';
 import { compareAndWriteFile } from '../create-file';
 
 /**
@@ -44,7 +45,7 @@ export abstract class BaseWriteStrategy {
   abstract writeProtocols(protocol: Set<string>): void;
   abstract writeOtherRules(rule: string[]): void;
 
-  protected abstract withPadding(title: string, description: string[] | readonly string[], date: Date, content: string[]): string[];
+  protected abstract withPadding(title: string, description: string[] | readonly string[], date: Date, content: string[], contentHash: string | null): string[];
 
   public output(
     span: Span,
@@ -53,19 +54,28 @@ export abstract class BaseWriteStrategy {
     date: Date,
     filePath: string
   ): void | Promise<void> {
-    if (!this.result) {
+    const result = this.result;
+    if (!result) {
       return;
     }
 
+    // The hash covers the real content (title, description and rules) but not the
+    // volatile date, so compareAndWriteFile can bail out by only reading the head
+    // of the previous output. Strategies whose withPadding doesn't embed the marker
+    // (e.g. JSON output) simply fall back to the full comparison.
+    const contentHash = calculateContentHash(title, description, result);
+
     return compareAndWriteFile(
       span,
       this.withPadding(
         title,
         description,
         date,
-        this.result
+        result,
+        contentHash
       ),
-      filePath
+      filePath,
+      contentHash
     );
   };
 

pnpm-lock.yaml

@@ -1037,10 +1037,6 @@ packages:
   brace-expansion@2.0.2:
     resolution: {integrity: sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==}
 
-  brace-expansion@5.0.3:
-    resolution: {integrity: sha512-fy6KJm2RawA5RcHkLa1z/ScpBeA762UF9KmZQxwIbDtRJrgLzM10depAiEQ+CXYcoiqW1/m96OAAoke2nE9EeA==}
-    engines: {node: 18 || 20 || >=22}
-
   brace-expansion@5.0.8:
     resolution: {integrity: sha512-JZyDyq3D4AUifKTPOB7DELf6XsB3WdPuNxCtob1vFXPsSXhdAiHBWJ/tJ8HAc9aH84BK+5JFZLNkJKx3G9kzQg==}
     engines: {node: 20 || >=22}
@@ -1200,10 +1196,6 @@ packages:
   end-of-stream@1.4.5:
     resolution: {integrity: sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==}
 
-  enhanced-resolve@5.24.2:
-    resolution: {integrity: sha512-rpsZEGT1jFuve6QlpyRp9ckQ+kN61hvF9BzCPyMdaKTm8UJce96KBn3sorXOFXlzjPrs3Vc4T1NsSroZ3PxlFw==}
-    engines: {node: '>=10.13.0'}
-
   enhanced-resolve@5.24.3:
     resolution: {integrity: sha512-PwKooW9JUzh5chmYfHM3IQl5OkK2u2Nm011MgeZrss3JmFraUx/fqrf78kk8GUMYoibx/14MdwTl/1WKkG7TpQ==}
     engines: {node: '>=10.13.0'}
@@ -1654,10 +1646,6 @@ packages:
     resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==}
     engines: {node: '>=10'}
 
-  minimatch@10.2.4:
-    resolution: {integrity: sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==}
-    engines: {node: 18 || 20 || >=22}
-
   minimatch@10.2.5:
     resolution: {integrity: sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==}
     engines: {node: 18 || 20 || >=22}
@@ -2178,7 +2166,7 @@ snapshots:
     dependencies:
       '@eslint/object-schema': 3.0.5
       debug: 4.4.3(supports-color@8.1.1)
-      minimatch: 10.2.4
+      minimatch: 10.2.5
     transitivePeerDependencies:
       - supports-color
 
@@ -2867,10 +2855,6 @@ snapshots:
     dependencies:
       balanced-match: 1.0.2
 
-  brace-expansion@5.0.3:
-    dependencies:
-      balanced-match: 4.0.4
-
   brace-expansion@5.0.8:
     dependencies:
       balanced-match: 4.0.4
@@ -3017,11 +3001,6 @@ snapshots:
     dependencies:
       once: 1.4.0
 
-  enhanced-resolve@5.24.2:
-    dependencies:
-      graceful-fs: 4.2.11
-      tapable: 2.3.3
-
   enhanced-resolve@5.24.3:
     dependencies:
       graceful-fs: 4.2.11
@@ -3141,7 +3120,7 @@ snapshots:
       eslint: 10.8.0(supports-color@8.1.1)
       eslint-import-context: 0.1.9(unrs-resolver@1.11.1)
       is-glob: 4.0.3
-      minimatch: 10.2.4
+      minimatch: 10.2.5
       semver: 7.7.3
       stable-hash-x: 0.2.0
       unrs-resolver: 1.11.1
@@ -3168,7 +3147,7 @@ snapshots:
   eslint-plugin-n@17.24.0(eslint@10.8.0(supports-color@8.1.1))(typescript@6.0.3):
     dependencies:
       '@eslint-community/eslint-utils': 4.9.1(eslint@10.8.0(supports-color@8.1.1))
-      enhanced-resolve: 5.24.2
+      enhanced-resolve: 5.24.3
       eslint: 10.8.0(supports-color@8.1.1)
       eslint-plugin-es-x: 7.8.0(eslint@10.8.0(supports-color@8.1.1))
       get-tsconfig: 4.12.0
@@ -3523,10 +3502,6 @@ snapshots:
 
   mimic-response@3.1.0: {}
 
-  minimatch@10.2.4:
-    dependencies:
-      brace-expansion: 5.0.3
-
   minimatch@10.2.5:
     dependencies:
       brace-expansion: 5.0.8