From b3bcec6de6d04f3f604ae13c4fd70e930e00d3cd Mon Sep 17 00:00:00 2001 From: SukkaW Date: Sat, 1 Aug 2026 01:30:47 +0800 Subject: [PATCH] Chore: embed content hash --- Build/lib/content-hash.ts | 86 +++++++++++++++++++++++ Build/lib/create-file.ts | 28 ++++++-- Build/lib/misc.ts | 6 +- Build/lib/writing-strategy/adguardhome.ts | 3 +- Build/lib/writing-strategy/base.ts | 18 +++-- pnpm-lock.yaml | 31 +------- 6 files changed, 131 insertions(+), 41 deletions(-) create mode 100644 Build/lib/content-hash.ts diff --git a/Build/lib/content-hash.ts b/Build/lib/content-hash.ts new file mode 100644 index 00000000..dc500faf --- /dev/null +++ b/Build/lib/content-hash.ts @@ -0,0 +1,86 @@ +import { fastStringArrayJoin } from 'foxts/fast-string-array-join'; +import { Buffer } from 'node:buffer'; +import { createHash } from 'node:crypto'; +import fsp from 'node:fs/promises'; + +/** + * A special marker embedded in the metadata comment of output files, carrying the hash + * of the "real content" (title, description and rules -- but not the volatile + * "Last Updated" date). This way, whether a file needs to be re-written can be + * determined by reading only the first few hundred bytes of the previous output + * when possible. + * + * If the hash algorithm or what's included in the hash ever changes, bump the + * version suffix to force a one-time re-write of every file. + */ +const CONTENT_HASH_MARKER_START = '$content-hash-v1$:'; +const CONTENT_HASH_MARKER_END = '$'; + +/** + * Hash the real content and return the ready-to-embed, self-delimited token, + * e.g. `$content-hash-v1$:AbCd...$`. Writers only need to prepend their own + * comment prefix (`# ` / `! `). + */ +export function calculateContentHash(title: string, description: string[] | readonly string[], content: string[]): string { + const hasher = createHash('sha256'); + hasher.update(title); + hasher.update('\0'); + hasher.update(fastStringArrayJoin(description, '\n')); + hasher.update('\0'); + hasher.update(fastStringArrayJoin(content, '\n')); + return CONTENT_HASH_MARKER_START + hasher.digest('base64url') + CONTENT_HASH_MARKER_END; +} + +/** + * The content hash token sits at the bottom of the leading metadata comment, + * so reading the first chunk of the file is enough to locate it. The chunk must + * be larger than the biggest banner (title + description + data sources, ~2.2 KiB + * as of writing) -- if the token falls outside the chunk, extraction returns null + * and the caller silently degrades to the full file comparison. + */ +const FILE_HEAD_CHUNK_SIZE = 8192; + +async function readFileHead(filePath: string): Promise { + let fd: fsp.FileHandle | null = null; + try { + fd = await fsp.open(filePath, 'r'); + const buf = Buffer.allocUnsafe(FILE_HEAD_CHUNK_SIZE); + const { bytesRead } = await fd.read(buf, 0, FILE_HEAD_CHUNK_SIZE, 0); + return buf.toString('utf8', 0, bytesRead); + } finally { + await fd?.close(); + } +} + +/** + * Extract the content hash token from a previously written file, reading only + * the first chunk of it. Returns the same self-delimited token shape that + * calculateContentHash produces, so the two can be compared directly. Returns + * null if the file predates the content hash marker, or if the token is + * malformed / truncated by the fixed-size head read (in which case the caller + * falls back to the full comparison, which is always safe). + */ +export async function extractContentHashFromFile(filePath: string): Promise { + return extractContentHash(await readFileHead(filePath)); +} + +function extractContentHash(fileHead: string): string | null { + const markerIndex = fileHead.indexOf(CONTENT_HASH_MARKER_START); + if (markerIndex === -1) { + return null; + } + + const start = markerIndex + CONTENT_HASH_MARKER_START.length; + const end = fileHead.indexOf(CONTENT_HASH_MARKER_END, start); + // also covers end === -1 (unterminated token) and end === start (empty hash) + if (end <= start) { + return null; + } + + // the end marker must sit on the same line as the start marker + if (fileHead.slice(start, end).includes('\n')) { + return null; + } + + return fileHead.slice(markerIndex, end + CONTENT_HASH_MARKER_END.length); +} diff --git a/Build/lib/create-file.ts b/Build/lib/create-file.ts index 839c1f68..52c9a22b 100644 --- a/Build/lib/create-file.ts +++ b/Build/lib/create-file.ts @@ -7,27 +7,41 @@ import { readFileByLine } from './fetch-text-by-line'; import { writeFile } from './misc'; import { createCompareSource, fileEqualWithCommentComparator } from 'foxts/compare-source'; import { promisify } from 'node:util'; +import { extractContentHashFromFile } from './content-hash'; const fileEqual = createCompareSource(fileEqualWithCommentComparator); /** * To keep metadata comment `last updated` not change if real content is the same, - * we compare real content lines and only write if actual content is different, - * and the new `last updated` will be written along with new content. + * we only write when the actual content differs, and the new `last updated` will + * be written along with new content. + * + * When `contentHash` is provided (and the previous output already embeds a + * content hash marker), the comparison only reads the first chunk of the + * previous file. Otherwise it falls back to a full comment-insensitive + * line-by-line comparison. */ -export async function compareAndWriteFile(span: Span, linesA: string[], filePath: string) { +export async function compareAndWriteFile(span: Span, linesA: string[], filePath: string, contentHash: string | null = null) { // readFileByLine will not include last empty line. So we always pop the linesA for comparison purpose if (linesA.length > 0 && linesA[linesA.length - 1] === '') { linesA.pop(); } const isEqual = await span.traceChildAsync(`compare ${filePath}`, async () => { - if (fs.existsSync(filePath)) { - return fileEqual(linesA, readFileByLine(filePath)); + if (!fs.existsSync(filePath)) { + console.log(`${filePath} does not exists, writing...`); + return false; } - console.log(`${filePath} does not exists, writing...`); - return false; + if (contentHash) { + const previousHash = await extractContentHashFromFile(filePath); + if (previousHash) { + return previousHash === contentHash; + } + // previous output predates the content hash marker, fall through to full comparison + } + + return fileEqual(linesA, readFileByLine(filePath)); }); if (isEqual) { diff --git a/Build/lib/misc.ts b/Build/lib/misc.ts index 1d3e8aea..c6c81821 100644 --- a/Build/lib/misc.ts +++ b/Build/lib/misc.ts @@ -30,7 +30,7 @@ export const writeFile: Write = async (destination: string, input, dir = dirname return fsp.writeFile(destination, input, { encoding: 'utf-8' }); }; -export function withBannerArray(title: string, description: string[] | readonly string[], date: Date, content: string[]) { +export function withBannerArray(title: string, description: string[] | readonly string[], date: Date, content: string[], contentHash: string | null = null) { const result: string[] = [ '#########################################', `# ${title}`, @@ -40,6 +40,10 @@ export function withBannerArray(title: string, description: string[] | readonly appendArrayInPlace(result, description.map(line => (line ? `# ${line}` : '#'))); + if (contentHash) { + result.push('#', `# ${contentHash}`); + } + result.push('#########################################'); appendArrayInPlace(result, content); diff --git a/Build/lib/writing-strategy/adguardhome.ts b/Build/lib/writing-strategy/adguardhome.ts index 0906b5aa..152a378e 100644 --- a/Build/lib/writing-strategy/adguardhome.ts +++ b/Build/lib/writing-strategy/adguardhome.ts @@ -13,7 +13,7 @@ export class AdGuardHome extends BaseWriteStrategy { protected result: string[] = []; // eslint-disable-next-line @typescript-eslint/class-methods-use-this -- abstract method - withPadding(title: string, description: string[] | readonly string[], date: Date, content: string[]): string[] { + withPadding(title: string, description: string[] | readonly string[], date: Date, content: string[], contentHash: string | null): string[] { return [ `! Title: ${title}`, '! Last modified: ' + date.toUTCString(), @@ -21,6 +21,7 @@ export class AdGuardHome extends BaseWriteStrategy { '! License: https://github.com/SukkaW/Surge/blob/master/LICENSE', '! Homepage: https://github.com/SukkaW/Surge', `! Description: ${description.join(' ')}`, + ...(contentHash ? [`! ${contentHash}`] : []), '!', ...content, '! EOF' diff --git a/Build/lib/writing-strategy/base.ts b/Build/lib/writing-strategy/base.ts index 549a40b9..856a98e0 100644 --- a/Build/lib/writing-strategy/base.ts +++ b/Build/lib/writing-strategy/base.ts @@ -1,4 +1,5 @@ import type { Span } from '../../trace'; +import { calculateContentHash } from '../content-hash'; import { compareAndWriteFile } from '../create-file'; /** @@ -44,7 +45,7 @@ export abstract class BaseWriteStrategy { abstract writeProtocols(protocol: Set): void; abstract writeOtherRules(rule: string[]): void; - protected abstract withPadding(title: string, description: string[] | readonly string[], date: Date, content: string[]): string[]; + protected abstract withPadding(title: string, description: string[] | readonly string[], date: Date, content: string[], contentHash: string | null): string[]; public output( span: Span, @@ -53,19 +54,28 @@ export abstract class BaseWriteStrategy { date: Date, filePath: string ): void | Promise { - if (!this.result) { + const result = this.result; + if (!result) { return; } + // The hash covers the real content (title, description and rules) but not the + // volatile date, so compareAndWriteFile can bail out by only reading the head + // of the previous output. Strategies whose withPadding doesn't embed the marker + // (e.g. JSON output) simply fall back to the full comparison. + const contentHash = calculateContentHash(title, description, result); + return compareAndWriteFile( span, this.withPadding( title, description, date, - this.result + result, + contentHash ), - filePath + filePath, + contentHash ); }; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d6a7f71a..a47cb7ef 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1037,10 +1037,6 @@ packages: brace-expansion@2.0.2: resolution: {integrity: sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==} - brace-expansion@5.0.3: - resolution: {integrity: sha512-fy6KJm2RawA5RcHkLa1z/ScpBeA762UF9KmZQxwIbDtRJrgLzM10depAiEQ+CXYcoiqW1/m96OAAoke2nE9EeA==} - engines: {node: 18 || 20 || >=22} - brace-expansion@5.0.8: resolution: {integrity: sha512-JZyDyq3D4AUifKTPOB7DELf6XsB3WdPuNxCtob1vFXPsSXhdAiHBWJ/tJ8HAc9aH84BK+5JFZLNkJKx3G9kzQg==} engines: {node: 20 || >=22} @@ -1200,10 +1196,6 @@ packages: end-of-stream@1.4.5: resolution: {integrity: sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==} - enhanced-resolve@5.24.2: - resolution: {integrity: sha512-rpsZEGT1jFuve6QlpyRp9ckQ+kN61hvF9BzCPyMdaKTm8UJce96KBn3sorXOFXlzjPrs3Vc4T1NsSroZ3PxlFw==} - engines: {node: '>=10.13.0'} - enhanced-resolve@5.24.3: resolution: {integrity: sha512-PwKooW9JUzh5chmYfHM3IQl5OkK2u2Nm011MgeZrss3JmFraUx/fqrf78kk8GUMYoibx/14MdwTl/1WKkG7TpQ==} engines: {node: '>=10.13.0'} @@ -1654,10 +1646,6 @@ packages: resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} engines: {node: '>=10'} - minimatch@10.2.4: - resolution: {integrity: sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==} - engines: {node: 18 || 20 || >=22} - minimatch@10.2.5: resolution: {integrity: sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==} engines: {node: 18 || 20 || >=22} @@ -2178,7 +2166,7 @@ snapshots: dependencies: '@eslint/object-schema': 3.0.5 debug: 4.4.3(supports-color@8.1.1) - minimatch: 10.2.4 + minimatch: 10.2.5 transitivePeerDependencies: - supports-color @@ -2867,10 +2855,6 @@ snapshots: dependencies: balanced-match: 1.0.2 - brace-expansion@5.0.3: - dependencies: - balanced-match: 4.0.4 - brace-expansion@5.0.8: dependencies: balanced-match: 4.0.4 @@ -3017,11 +3001,6 @@ snapshots: dependencies: once: 1.4.0 - enhanced-resolve@5.24.2: - dependencies: - graceful-fs: 4.2.11 - tapable: 2.3.3 - enhanced-resolve@5.24.3: dependencies: graceful-fs: 4.2.11 @@ -3141,7 +3120,7 @@ snapshots: eslint: 10.8.0(supports-color@8.1.1) eslint-import-context: 0.1.9(unrs-resolver@1.11.1) is-glob: 4.0.3 - minimatch: 10.2.4 + minimatch: 10.2.5 semver: 7.7.3 stable-hash-x: 0.2.0 unrs-resolver: 1.11.1 @@ -3168,7 +3147,7 @@ snapshots: eslint-plugin-n@17.24.0(eslint@10.8.0(supports-color@8.1.1))(typescript@6.0.3): dependencies: '@eslint-community/eslint-utils': 4.9.1(eslint@10.8.0(supports-color@8.1.1)) - enhanced-resolve: 5.24.2 + enhanced-resolve: 5.24.3 eslint: 10.8.0(supports-color@8.1.1) eslint-plugin-es-x: 7.8.0(eslint@10.8.0(supports-color@8.1.1)) get-tsconfig: 4.12.0 @@ -3523,10 +3502,6 @@ snapshots: mimic-response@3.1.0: {} - minimatch@10.2.4: - dependencies: - brace-expansion: 5.0.3 - minimatch@10.2.5: dependencies: brace-expansion: 5.0.8