Chore: embed content hash

Build/lib/content-hash.ts

@@ -0,0 +1,86 @@
+import { fastStringArrayJoin } from 'foxts/fast-string-array-join';
+import { Buffer } from 'node:buffer';
+import { createHash } from 'node:crypto';
+import fsp from 'node:fs/promises';
+
+/**
+ * A special marker embedded in the metadata comment of output files, carrying the hash
+ * of the "real content" (title, description and rules -- but not the volatile
+ * "Last Updated" date). This way, whether a file needs to be re-written can be
+ * determined by reading only the first few hundred bytes of the previous output
+ * when possible.
+ *
+ * If the hash algorithm or what's included in the hash ever changes, bump the
+ * version suffix to force a one-time re-write of every file.
+ */
+const CONTENT_HASH_MARKER_START = '$content-hash-v1$:';
+const CONTENT_HASH_MARKER_END = '$';
+
+/**
+ * Hash the real content and return the ready-to-embed, self-delimited token,
+ * e.g. `$content-hash-v1$:AbCd...$`. Writers only need to prepend their own
+ * comment prefix (`# ` / `! `).
+ */
+export function calculateContentHash(title: string, description: string[] | readonly string[], content: string[]): string {
+  const hasher = createHash('sha256');
+  hasher.update(title);
+  hasher.update('\0');
+  hasher.update(fastStringArrayJoin(description, '\n'));
+  hasher.update('\0');
+  hasher.update(fastStringArrayJoin(content, '\n'));
+  return CONTENT_HASH_MARKER_START + hasher.digest('base64url') + CONTENT_HASH_MARKER_END;
+}
+
+/**
+ * The content hash token sits at the bottom of the leading metadata comment,
+ * so reading the first chunk of the file is enough to locate it. The chunk must
+ * be larger than the biggest banner (title + description + data sources, ~2.2 KiB
+ * as of writing) -- if the token falls outside the chunk, extraction returns null
+ * and the caller silently degrades to the full file comparison.
+ */
+const FILE_HEAD_CHUNK_SIZE = 8192;
+
+async function readFileHead(filePath: string): Promise<string> {
+  let fd: fsp.FileHandle | null = null;
+  try {
+    fd = await fsp.open(filePath, 'r');
+    const buf = Buffer.allocUnsafe(FILE_HEAD_CHUNK_SIZE);
+    const { bytesRead } = await fd.read(buf, 0, FILE_HEAD_CHUNK_SIZE, 0);
+    return buf.toString('utf8', 0, bytesRead);
+  } finally {
+    await fd?.close();
+  }
+}
+
+/**
+ * Extract the content hash token from a previously written file, reading only
+ * the first chunk of it. Returns the same self-delimited token shape that
+ * calculateContentHash produces, so the two can be compared directly. Returns
+ * null if the file predates the content hash marker, or if the token is
+ * malformed / truncated by the fixed-size head read (in which case the caller
+ * falls back to the full comparison, which is always safe).
+ */
+export async function extractContentHashFromFile(filePath: string): Promise<string | null> {
+  return extractContentHash(await readFileHead(filePath));
+}
+
+function extractContentHash(fileHead: string): string | null {
+  const markerIndex = fileHead.indexOf(CONTENT_HASH_MARKER_START);
+  if (markerIndex === -1) {
+    return null;
+  }
+
+  const start = markerIndex + CONTENT_HASH_MARKER_START.length;
+  const end = fileHead.indexOf(CONTENT_HASH_MARKER_END, start);
+  // also covers end === -1 (unterminated token) and end === start (empty hash)
+  if (end <= start) {
+    return null;
+  }
+
+  // the end marker must sit on the same line as the start marker
+  if (fileHead.slice(start, end).includes('\n')) {
+    return null;
+  }
+
+  return fileHead.slice(markerIndex, end + CONTENT_HASH_MARKER_END.length);
+}

Build/lib/create-file.ts

@@ -7,27 +7,41 @@ import { readFileByLine } from './fetch-text-by-line';
 import { writeFile } from './misc';
 import { createCompareSource, fileEqualWithCommentComparator } from 'foxts/compare-source';
 import { promisify } from 'node:util';
+import { extractContentHashFromFile } from './content-hash';
 
 const fileEqual = createCompareSource(fileEqualWithCommentComparator);
 
 /**
  * To keep metadata comment `last updated` not change if real content is the same,
- * we compare real content lines and only write if actual content is different,
- * and the new `last updated` will be written along with new content.
+ * we only write when the actual content differs, and the new `last updated` will
+ * be written along with new content.
+ *
+ * When `contentHash` is provided (and the previous output already embeds a
+ * content hash marker), the comparison only reads the first chunk of the
+ * previous file. Otherwise it falls back to a full comment-insensitive
+ * line-by-line comparison.
  */
-export async function compareAndWriteFile(span: Span, linesA: string[], filePath: string) {
+export async function compareAndWriteFile(span: Span, linesA: string[], filePath: string, contentHash: string | null = null) {
   // readFileByLine will not include last empty line. So we always pop the linesA for comparison purpose
   if (linesA.length > 0 && linesA[linesA.length - 1] === '') {
     linesA.pop();
   }
 
   const isEqual = await span.traceChildAsync(`compare ${filePath}`, async () => {
-    if (fs.existsSync(filePath)) {
-      return fileEqual(linesA, readFileByLine(filePath));
+    if (!fs.existsSync(filePath)) {
+      console.log(`${filePath} does not exists, writing...`);
+      return false;
     }
 
-    console.log(`${filePath} does not exists, writing...`);
-    return false;
+    if (contentHash) {
+      const previousHash = await extractContentHashFromFile(filePath);
+      if (previousHash) {
+        return previousHash === contentHash;
+      }
+      // previous output predates the content hash marker, fall through to full comparison
+    }
+
+    return fileEqual(linesA, readFileByLine(filePath));
   });
 
   if (isEqual) {

Build/lib/misc.ts

@@ -30,7 +30,7 @@ export const writeFile: Write = async (destination: string, input, dir = dirname
   return fsp.writeFile(destination, input, { encoding: 'utf-8' });
 };
 
-export function withBannerArray(title: string, description: string[] | readonly string[], date: Date, content: string[]) {
+export function withBannerArray(title: string, description: string[] | readonly string[], date: Date, content: string[], contentHash: string | null = null) {
   const result: string[] = [
     '#########################################',
     `# ${title}`,
@@ -40,6 +40,10 @@ export function withBannerArray(title: string, description: string[] | readonly
 
   appendArrayInPlace(result, description.map(line => (line ? `# ${line}` : '#')));
 
+  if (contentHash) {
+    result.push('#', `# ${contentHash}`);
+  }
+
   result.push('#########################################');
 
   appendArrayInPlace(result, content);

Build/lib/writing-strategy/adguardhome.ts

@@ -13,7 +13,7 @@ export class AdGuardHome extends BaseWriteStrategy {
   protected result: string[] = [];
 
   // eslint-disable-next-line @typescript-eslint/class-methods-use-this -- abstract method
-  withPadding(title: string, description: string[] | readonly string[], date: Date, content: string[]): string[] {
+  withPadding(title: string, description: string[] | readonly string[], date: Date, content: string[], contentHash: string | null): string[] {
     return [
       `! Title: ${title}`,
       '! Last modified: ' + date.toUTCString(),
@@ -21,6 +21,7 @@ export class AdGuardHome extends BaseWriteStrategy {
       '! License: https://github.com/SukkaW/Surge/blob/master/LICENSE',
       '! Homepage: https://github.com/SukkaW/Surge',
       `! Description: ${description.join(' ')}`,
+      ...(contentHash ? [`! ${contentHash}`] : []),
       '!',
       ...content,
       '! EOF'

Build/lib/writing-strategy/base.ts

@@ -1,4 +1,5 @@
 import type { Span } from '../../trace';
+import { calculateContentHash } from '../content-hash';
 import { compareAndWriteFile } from '../create-file';
 
 /**
@@ -44,7 +45,7 @@ export abstract class BaseWriteStrategy {
   abstract writeProtocols(protocol: Set<string>): void;
   abstract writeOtherRules(rule: string[]): void;
 
-  protected abstract withPadding(title: string, description: string[] | readonly string[], date: Date, content: string[]): string[];
+  protected abstract withPadding(title: string, description: string[] | readonly string[], date: Date, content: string[], contentHash: string | null): string[];
 
   public output(
     span: Span,
@@ -53,19 +54,28 @@ export abstract class BaseWriteStrategy {
     date: Date,
     filePath: string
   ): void | Promise<void> {
-    if (!this.result) {
+    const result = this.result;
+    if (!result) {
       return;
     }
 
+    // The hash covers the real content (title, description and rules) but not the
+    // volatile date, so compareAndWriteFile can bail out by only reading the head
+    // of the previous output. Strategies whose withPadding doesn't embed the marker
+    // (e.g. JSON output) simply fall back to the full comparison.
+    const contentHash = calculateContentHash(title, description, result);
+
     return compareAndWriteFile(
       span,
       this.withPadding(
         title,
         description,
         date,
-        this.result
+        result,
+        contentHash
       ),
-      filePath
+      filePath,
+      contentHash
     );
   };