From 94d343573ba1af4efe541a1a1391559e67ef26e0 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Thu, 16 Jul 2026 02:46:56 +0800 Subject: [PATCH] Chore: parallel checking dead domains --- .github/workflows/check-source-domain.yml | 33 ++++-- Build/lib/enumerate-source-domains.ts | 66 +++++++++++ Build/lib/get-runner-geoip.ts | 30 +++++ Build/lib/shard.ts | 55 +++++++++ Build/validate-domain-alive.ts | 130 +++++++++++++--------- 5 files changed, 251 insertions(+), 63 deletions(-) create mode 100644 Build/lib/enumerate-source-domains.ts create mode 100644 Build/lib/get-runner-geoip.ts create mode 100644 Build/lib/shard.ts diff --git a/.github/workflows/check-source-domain.yml b/.github/workflows/check-source-domain.yml index 9aac6022..e48454ff 100644 --- a/.github/workflows/check-source-domain.yml +++ b/.github/workflows/check-source-domain.yml @@ -5,8 +5,17 @@ on: jobs: check: - name: Check - runs-on: ubuntu-24.04-arm + name: Check (shard ${{ matrix.shard }}/4) + runs-on: ubuntu-slim + + strategy: + # Keep every shard running even if one hits a fatal error, so we still + # collect dead domains from the rest. + fail-fast: false + matrix: + # 4 shards -> 4 runners, each lands on a different Azure region / egress + # IP, spreading DoH load and reducing per-server rate limiting. + shard: [0, 1, 2, 3] steps: # - name: Tune GitHub-hosted runner network @@ -38,24 +47,28 @@ jobs: with: path: | .cache - key: ${{ runner.os }}-v3-${{ steps.date.outputs.year }}-${{ steps.date.outputs.month }}-${{ steps.date.outputs.day }} ${{ steps.date.outputs.hour }}:${{ steps.date.outputs.minute }}:${{ steps.date.outputs.second }} + # Cache is keyed per shard: each shard checks a disjoint subset of + # domains, so their DoH/whois caches don't overlap. + key: ${{ runner.os }}-shard${{ matrix.shard }}-v3-${{ steps.date.outputs.year }}-${{ steps.date.outputs.month }}-${{ steps.date.outputs.day }} ${{ steps.date.outputs.hour }}:${{ steps.date.outputs.minute }}:${{ steps.date.outputs.second }} # If source files changed but packages didn't, rebuild from a prior cache. restore-keys: | - ${{ runner.os }}-v3-${{ steps.date.outputs.year }}-${{ steps.date.outputs.month }}-${{ steps.date.outputs.day }} ${{ steps.date.outputs.hour }}:${{ steps.date.outputs.minute }}: - ${{ runner.os }}-v3-${{ steps.date.outputs.year }}-${{ steps.date.outputs.month }}-${{ steps.date.outputs.day }} ${{ steps.date.outputs.hour }}: - ${{ runner.os }}-v3-${{ steps.date.outputs.year }}-${{ steps.date.outputs.month }}-${{ steps.date.outputs.day }} - ${{ runner.os }}-v3-${{ steps.date.outputs.year }}-${{ steps.date.outputs.month }}- - ${{ runner.os }}-v3-${{ steps.date.outputs.year }}- - ${{ runner.os }}-v3- + ${{ runner.os }}-shard${{ matrix.shard }}-v3-${{ steps.date.outputs.year }}-${{ steps.date.outputs.month }}-${{ steps.date.outputs.day }} ${{ steps.date.outputs.hour }}:${{ steps.date.outputs.minute }}: + ${{ runner.os }}-shard${{ matrix.shard }}-v3-${{ steps.date.outputs.year }}-${{ steps.date.outputs.month }}-${{ steps.date.outputs.day }} ${{ steps.date.outputs.hour }}: + ${{ runner.os }}-shard${{ matrix.shard }}-v3-${{ steps.date.outputs.year }}-${{ steps.date.outputs.month }}-${{ steps.date.outputs.day }} + ${{ runner.os }}-shard${{ matrix.shard }}-v3-${{ steps.date.outputs.year }}-${{ steps.date.outputs.month }}- + ${{ runner.os }}-shard${{ matrix.shard }}-v3-${{ steps.date.outputs.year }}- + ${{ runner.os }}-shard${{ matrix.shard }}-v3- - run: pnpm config set --location=global minimumReleaseAge 0 - run: pnpm install - run: pnpm run node Build/validate-domain-alive.ts env: DEBUG: domain-alive:dead-domain,domain-alive:error:* + SHARD_INDEX: ${{ matrix.shard }} + SHARD_TOTAL: 4 - name: Cache cache.db if: always() uses: actions/cache/save@v5 with: path: | .cache - key: ${{ runner.os }}-v3-${{ steps.date.outputs.year }}-${{ steps.date.outputs.month }}-${{ steps.date.outputs.day }} ${{ steps.date.outputs.hour }}:${{ steps.date.outputs.minute }}:${{ steps.date.outputs.second }} + key: ${{ runner.os }}-shard${{ matrix.shard }}-v3-${{ steps.date.outputs.year }}-${{ steps.date.outputs.month }}-${{ steps.date.outputs.day }} ${{ steps.date.outputs.hour }}:${{ steps.date.outputs.minute }}:${{ steps.date.outputs.second }} diff --git a/Build/lib/enumerate-source-domains.ts b/Build/lib/enumerate-source-domains.ts new file mode 100644 index 00000000..6e450b89 --- /dev/null +++ b/Build/lib/enumerate-source-domains.ts @@ -0,0 +1,66 @@ +import { SOURCE_DIR } from '../constants/dir'; +import path from 'node:path'; +import { fdir as Fdir } from 'fdir'; +import runAgainstSourceFile from './run-against-source-file'; + +export interface SourceDomain { + domain: string, + /** + * `true` for `DOMAIN-SUFFIX` / leading-dot domainset entries (apex domain + * that includes all subdomains), `false` for exact `DOMAIN` entries. + */ + includeAllSubdomain: boolean +} + +function sourceFileFilter(filePath: string, isDirectory: boolean) { + if (isDirectory) return false; + const extname = path.extname(filePath); + return extname === '.txt' || extname === '.conf'; +} + +/** + * Crawl the `domainset` and `non_ip` source directories and return every + * domain entry they declare. + * + * A domain that appears with `includeAllSubdomain: true` in any source wins + * over an exact-only occurrence: checking the apex (with subdomains) also + * covers the exact host, so we collapse duplicates to the broader check. This + * is the natural dedupe that makes sharding by domain hash correct. + */ +export async function enumerateSourceDomains(): Promise { + const [domainSets, domainRules] = await Promise.all([ + new Fdir() + .withFullPaths() + .filter(sourceFileFilter) + .crawl(SOURCE_DIR + path.sep + 'domainset') + .withPromise(), + new Fdir() + .withFullPaths() + .filter(sourceFileFilter) + .crawl(SOURCE_DIR + path.sep + 'non_ip') + .withPromise() + ]); + + // domain -> includeAllSubdomain (true wins) + const seen = new Map(); + + await Promise.all( + [...domainRules, ...domainSets].map(filepath => runAgainstSourceFile( + filepath, + (domain: string, includeAllSubdomain: boolean) => { + const prev = seen.get(domain); + if (prev === undefined) { + seen.set(domain, includeAllSubdomain); + } else if (includeAllSubdomain && !prev) { + seen.set(domain, true); + } + } + )) + ); + + const result: SourceDomain[] = []; + for (const [domain, includeAllSubdomain] of seen) { + result.push({ domain, includeAllSubdomain }); + } + return result; +} diff --git a/Build/lib/get-runner-geoip.ts b/Build/lib/get-runner-geoip.ts new file mode 100644 index 00000000..e03be3d4 --- /dev/null +++ b/Build/lib/get-runner-geoip.ts @@ -0,0 +1,30 @@ +import { $$fetch } from './fetch-retry'; + +export interface RunnerGeoIP { + ip: string, + country: string, + region: string, + city: string, + asn: number, + asOrg: string, + longitude: number, + latitude: number, + timezone: string, + cfEdgeIP: string +} + +/** + * Fetch the current machine's egress IP and geo info. Used to confirm that + * each matrix shard landed on a different GitHub-hosted runner region / egress + * IP — the premise that lets sharding spread DoH load and dodge rate limits. + * + * Returns `null` on any failure so it never aborts the actual domain check. + */ +export async function getRunnerGeoIP(): Promise { + try { + const res = await $$fetch('https://ip.api.skk.moe/cf-geoip'); + return await res.json() as RunnerGeoIP; + } catch { + return null; + } +} diff --git a/Build/lib/shard.ts b/Build/lib/shard.ts new file mode 100644 index 00000000..c91683ed --- /dev/null +++ b/Build/lib/shard.ts @@ -0,0 +1,55 @@ +/** + * Deterministic domain sharding for spreading the domain-alive check across + * multiple GitHub Actions runners. + * + * GitHub-hosted runners are provisioned across many Azure regions, so each + * matrix job gets a distinct egress IP. Splitting the workload by a stable + * hash of the domain lets every runner enumerate the *same* source files but + * only check the subset that belongs to its shard, which: + * + * - spreads DoH load across N distinct egress IPs (fewer per-server rate + * limits), and + * - makes assignment deterministic — the same domain always lands on the + * same shard, so duplicates across source files collapse to one check and + * re-runs are reproducible. + * + * FNV-1a (52-bit, via foxts/fnv1a52) is used because it is fast, has good + * distribution for load balancing, and stays within JavaScript's safe integer + * range. It is NOT cryptographic — never use it where collision resistance + * matters. + */ + +import { fnv1a52 } from 'foxts/fnv1a52'; +import process from 'node:process'; + +export interface ShardConfig { + /** 0-based index of this shard. */ + index: number, + /** Total number of shards. */ + total: number +} + +/** + * Read shard configuration from the environment. Defaults to a single shard + * (index 0 of 1), i.e. a full run — so local invocations and the non-matrix + * path behave exactly like before. + */ +export function getShardConfigFromEnv(env: NodeJS.ProcessEnv = process.env): ShardConfig { + const total = Number.parseInt(env.SHARD_TOTAL ?? '1', 10); + const index = Number.parseInt(env.SHARD_INDEX ?? '0', 10); + + if (!Number.isSafeInteger(total) || total < 1) { + throw new RangeError(`Invalid SHARD_TOTAL: ${JSON.stringify(env.SHARD_TOTAL)} (must be an integer >= 1)`); + } + if (!Number.isSafeInteger(index) || index < 0 || index >= total) { + throw new RangeError(`Invalid SHARD_INDEX: ${JSON.stringify(env.SHARD_INDEX)} (must be an integer in [0, ${total - 1}])`); + } + + return { index, total }; +} + +/** Whether the given domain belongs to this shard. */ +export function isInShard(domain: string, { index, total }: ShardConfig): boolean { + if (total === 1) return true; + return fnv1a52(domain) % total === index; +} diff --git a/Build/validate-domain-alive.ts b/Build/validate-domain-alive.ts index 7025f524..1c7528f4 100644 --- a/Build/validate-domain-alive.ts +++ b/Build/validate-domain-alive.ts @@ -1,8 +1,10 @@ -import { SOURCE_DIR } from './constants/dir'; -import path from 'node:path'; +import fs from 'node:fs'; +import process from 'node:process'; import { getMethods } from './lib/is-domain-alive'; -import { fdir as Fdir } from 'fdir'; -import runAgainstSourceFile from './lib/run-against-source-file'; +import { enumerateSourceDomains } from './lib/enumerate-source-domains'; +import { getShardConfigFromEnv, isInShard } from './lib/shard'; +import { getRunnerGeoIP } from './lib/get-runner-geoip'; +import type { RunnerGeoIP } from './lib/get-runner-geoip'; import cliProgress from 'cli-progress'; import { newQueue } from '@henrygd/queue'; @@ -11,66 +13,86 @@ const queue = newQueue(32); const deadDomains: string[] = []; +/** + * Append this shard's result to the GitHub Actions job summary, if running in + * CI. Each shard writes its own summary (no dedicated merge job) — the union + * of all shards' summaries is the full dead-domain list. + */ +function formatGeo(geo: RunnerGeoIP | null): string { + if (!geo) return 'unknown (geoip lookup failed)'; + return `${geo.ip} — ${geo.city}, ${geo.region}, ${geo.country} (AS${geo.asn} ${geo.asOrg})`; +} + +function writeJobSummary(shardLabel: string, dead: string[], geo: RunnerGeoIP | null) { + const summaryPath = process.env.GITHUB_STEP_SUMMARY; + if (!summaryPath) return; + + let summary = `## Dead domains — shard ${shardLabel}\n\n` + + `Runner egress: \`${formatGeo(geo)}\`\n\n` + + `Found **${dead.length}** dead domain(s) in this shard.\n\n`; + + if (dead.length > 0) { + summary += '
Show list\n\n' + + '```\n' + + dead.join('\n') + '\n' + + '```\n\n' + + '
\n\n' + // Machine-recoverable copy so the union can be scraped from summaries. + + '```json\n' + + JSON.stringify(dead) + '\n' + + '```\n\n'; + } + + fs.appendFileSync(summaryPath, summary); +} + (async () => { + const shard = getShardConfigFromEnv(); + const shardLabel = `${shard.index + 1}/${shard.total}`; + const [ { isDomainAlive, isRegisterableDomainAlive }, - domainSets, - domainRules + allDomains, + geo ] = await Promise.all([ getMethods(), - new Fdir() - .withFullPaths() - .filter((filePath, isDirectory) => { - if (isDirectory) return false; - const extname = path.extname(filePath); - return extname === '.txt' || extname === '.conf'; - }) - .crawl(SOURCE_DIR + path.sep + 'domainset') - .withPromise(), - new Fdir() - .withFullPaths() - .filter((filePath, isDirectory) => { - if (isDirectory) return false; - const extname = path.extname(filePath); - return extname === '.txt' || extname === '.conf'; - }) - .crawl(SOURCE_DIR + path.sep + 'non_ip') - .withPromise() + enumerateSourceDomains(), + getRunnerGeoIP() ]); + console.log(`[shard ${shardLabel}] runner egress: ${formatGeo(geo)}`); + + const shardDomains = allDomains.filter(({ domain }) => isInShard(domain, shard)); + + console.log( + `[shard ${shardLabel}] checking ${shardDomains.length} of ${allDomains.length} domain(s)` + ); + const bar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic); - bar.start(0, 0); + bar.start(shardDomains.length, 0); - await Promise.all([ - ...domainRules, - ...domainSets - ].map(filepath => runAgainstSourceFile( - filepath, - (domain: string, includeAllSubdomain: boolean) => { - bar.setTotal(bar.getTotal() + 1); + for (const { domain, includeAllSubdomain } of shardDomains) { + queue.add(async () => { + let registerableDomainAlive, registerableDomain, alive: boolean | undefined; - return queue.add(async () => { - let registerableDomainAlive, registerableDomain, alive: boolean | undefined; + if (includeAllSubdomain) { + // we only need to check apex domain, because we don't know if there is any stripped subdomain + ({ alive: registerableDomainAlive, registerableDomain } = await isRegisterableDomainAlive(domain)); + } else { + ({ alive, registerableDomainAlive, registerableDomain } = await isDomainAlive(domain)); + } - if (includeAllSubdomain) { - // we only need to check apex domain, because we don't know if there is any stripped subdomain - ({ alive: registerableDomainAlive, registerableDomain } = await isRegisterableDomainAlive(domain)); - } else { - ({ alive, registerableDomainAlive, registerableDomain } = await isDomainAlive(domain)); + bar.increment(); + + if (!registerableDomainAlive) { + if (registerableDomain) { + deadDomains.push('.' + registerableDomain); } - - bar.increment(); - - if (!registerableDomainAlive) { - if (registerableDomain) { - deadDomains.push('.' + registerableDomain); - } - } else if (!includeAllSubdomain && alive != null && !alive) { - deadDomains.push(domain); - } - }); - } - ).then(() => console.log('[crawl]', filepath)))); + } else if (!includeAllSubdomain && alive != null && !alive) { + deadDomains.push(domain); + } + }); + } await queue.done(); @@ -78,5 +100,7 @@ const deadDomains: string[] = []; console.log(); console.log(); - console.log(JSON.stringify(deadDomains)); + console.log(`[shard ${shardLabel}]`, JSON.stringify(deadDomains)); + + writeJobSummary(shardLabel, deadDomains, geo); })();