Chore: parallel checking dead domains

.github/workflows/check-source-domain.yml

@@ -5,8 +5,17 @@ on:
 
 jobs:
   check:
-    name: Check
-    runs-on: ubuntu-24.04-arm
+    name: Check (shard ${{ matrix.shard }}/4)
+    runs-on: ubuntu-slim
+
+    strategy:
+      # Keep every shard running even if one hits a fatal error, so we still
+      # collect dead domains from the rest.
+      fail-fast: false
+      matrix:
+        # 4 shards -> 4 runners, each lands on a different Azure region / egress
+        # IP, spreading DoH load and reducing per-server rate limiting.
+        shard: [0, 1, 2, 3]
 
     steps:
       # - name: Tune GitHub-hosted runner network
@@ -38,24 +47,28 @@ jobs:
         with:
           path: |
             .cache
-          key: ${{ runner.os }}-v3-${{ steps.date.outputs.year }}-${{ steps.date.outputs.month }}-${{ steps.date.outputs.day }} ${{ steps.date.outputs.hour }}:${{ steps.date.outputs.minute }}:${{ steps.date.outputs.second }}
+          # Cache is keyed per shard: each shard checks a disjoint subset of
+          # domains, so their DoH/whois caches don't overlap.
+          key: ${{ runner.os }}-shard${{ matrix.shard }}-v3-${{ steps.date.outputs.year }}-${{ steps.date.outputs.month }}-${{ steps.date.outputs.day }} ${{ steps.date.outputs.hour }}:${{ steps.date.outputs.minute }}:${{ steps.date.outputs.second }}
           # If source files changed but packages didn't, rebuild from a prior cache.
           restore-keys: |
-            ${{ runner.os }}-v3-${{ steps.date.outputs.year }}-${{ steps.date.outputs.month }}-${{ steps.date.outputs.day }} ${{ steps.date.outputs.hour }}:${{ steps.date.outputs.minute }}:
-            ${{ runner.os }}-v3-${{ steps.date.outputs.year }}-${{ steps.date.outputs.month }}-${{ steps.date.outputs.day }} ${{ steps.date.outputs.hour }}:
-            ${{ runner.os }}-v3-${{ steps.date.outputs.year }}-${{ steps.date.outputs.month }}-${{ steps.date.outputs.day }}
-            ${{ runner.os }}-v3-${{ steps.date.outputs.year }}-${{ steps.date.outputs.month }}-
-            ${{ runner.os }}-v3-${{ steps.date.outputs.year }}-
-            ${{ runner.os }}-v3-
+            ${{ runner.os }}-shard${{ matrix.shard }}-v3-${{ steps.date.outputs.year }}-${{ steps.date.outputs.month }}-${{ steps.date.outputs.day }} ${{ steps.date.outputs.hour }}:${{ steps.date.outputs.minute }}:
+            ${{ runner.os }}-shard${{ matrix.shard }}-v3-${{ steps.date.outputs.year }}-${{ steps.date.outputs.month }}-${{ steps.date.outputs.day }} ${{ steps.date.outputs.hour }}:
+            ${{ runner.os }}-shard${{ matrix.shard }}-v3-${{ steps.date.outputs.year }}-${{ steps.date.outputs.month }}-${{ steps.date.outputs.day }}
+            ${{ runner.os }}-shard${{ matrix.shard }}-v3-${{ steps.date.outputs.year }}-${{ steps.date.outputs.month }}-
+            ${{ runner.os }}-shard${{ matrix.shard }}-v3-${{ steps.date.outputs.year }}-
+            ${{ runner.os }}-shard${{ matrix.shard }}-v3-
       - run: pnpm config set --location=global minimumReleaseAge 0
       - run: pnpm install
       - run: pnpm run node Build/validate-domain-alive.ts
         env:
           DEBUG: domain-alive:dead-domain,domain-alive:error:*
+          SHARD_INDEX: ${{ matrix.shard }}
+          SHARD_TOTAL: 4
       - name: Cache cache.db
         if: always()
         uses: actions/cache/save@v5
         with:
           path: |
             .cache
-          key: ${{ runner.os }}-v3-${{ steps.date.outputs.year }}-${{ steps.date.outputs.month }}-${{ steps.date.outputs.day }} ${{ steps.date.outputs.hour }}:${{ steps.date.outputs.minute }}:${{ steps.date.outputs.second }}
+          key: ${{ runner.os }}-shard${{ matrix.shard }}-v3-${{ steps.date.outputs.year }}-${{ steps.date.outputs.month }}-${{ steps.date.outputs.day }} ${{ steps.date.outputs.hour }}:${{ steps.date.outputs.minute }}:${{ steps.date.outputs.second }}

Build/lib/enumerate-source-domains.ts

@@ -0,0 +1,66 @@
+import { SOURCE_DIR } from '../constants/dir';
+import path from 'node:path';
+import { fdir as Fdir } from 'fdir';
+import runAgainstSourceFile from './run-against-source-file';
+
+export interface SourceDomain {
+  domain: string,
+  /**
+   * `true` for `DOMAIN-SUFFIX` / leading-dot domainset entries (apex domain
+   * that includes all subdomains), `false` for exact `DOMAIN` entries.
+   */
+  includeAllSubdomain: boolean
+}
+
+function sourceFileFilter(filePath: string, isDirectory: boolean) {
+  if (isDirectory) return false;
+  const extname = path.extname(filePath);
+  return extname === '.txt' || extname === '.conf';
+}
+
+/**
+ * Crawl the `domainset` and `non_ip` source directories and return every
+ * domain entry they declare.
+ *
+ * A domain that appears with `includeAllSubdomain: true` in any source wins
+ * over an exact-only occurrence: checking the apex (with subdomains) also
+ * covers the exact host, so we collapse duplicates to the broader check. This
+ * is the natural dedupe that makes sharding by domain hash correct.
+ */
+export async function enumerateSourceDomains(): Promise<SourceDomain[]> {
+  const [domainSets, domainRules] = await Promise.all([
+    new Fdir()
+      .withFullPaths()
+      .filter(sourceFileFilter)
+      .crawl(SOURCE_DIR + path.sep + 'domainset')
+      .withPromise(),
+    new Fdir()
+      .withFullPaths()
+      .filter(sourceFileFilter)
+      .crawl(SOURCE_DIR + path.sep + 'non_ip')
+      .withPromise()
+  ]);
+
+  // domain -> includeAllSubdomain (true wins)
+  const seen = new Map<string, boolean>();
+
+  await Promise.all(
+    [...domainRules, ...domainSets].map(filepath => runAgainstSourceFile(
+      filepath,
+      (domain: string, includeAllSubdomain: boolean) => {
+        const prev = seen.get(domain);
+        if (prev === undefined) {
+          seen.set(domain, includeAllSubdomain);
+        } else if (includeAllSubdomain && !prev) {
+          seen.set(domain, true);
+        }
+      }
+    ))
+  );
+
+  const result: SourceDomain[] = [];
+  for (const [domain, includeAllSubdomain] of seen) {
+    result.push({ domain, includeAllSubdomain });
+  }
+  return result;
+}

Build/lib/get-runner-geoip.ts

@@ -0,0 +1,30 @@
+import { $$fetch } from './fetch-retry';
+
+export interface RunnerGeoIP {
+  ip: string,
+  country: string,
+  region: string,
+  city: string,
+  asn: number,
+  asOrg: string,
+  longitude: number,
+  latitude: number,
+  timezone: string,
+  cfEdgeIP: string
+}
+
+/**
+ * Fetch the current machine's egress IP and geo info. Used to confirm that
+ * each matrix shard landed on a different GitHub-hosted runner region / egress
+ * IP — the premise that lets sharding spread DoH load and dodge rate limits.
+ *
+ * Returns `null` on any failure so it never aborts the actual domain check.
+ */
+export async function getRunnerGeoIP(): Promise<RunnerGeoIP | null> {
+  try {
+    const res = await $$fetch('https://ip.api.skk.moe/cf-geoip');
+    return await res.json() as RunnerGeoIP;
+  } catch {
+    return null;
+  }
+}

Build/lib/shard.ts

@@ -0,0 +1,55 @@
+/**
+ * Deterministic domain sharding for spreading the domain-alive check across
+ * multiple GitHub Actions runners.
+ *
+ * GitHub-hosted runners are provisioned across many Azure regions, so each
+ * matrix job gets a distinct egress IP. Splitting the workload by a stable
+ * hash of the domain lets every runner enumerate the *same* source files but
+ * only check the subset that belongs to its shard, which:
+ *
+ *   - spreads DoH load across N distinct egress IPs (fewer per-server rate
+ *     limits), and
+ *   - makes assignment deterministic — the same domain always lands on the
+ *     same shard, so duplicates across source files collapse to one check and
+ *     re-runs are reproducible.
+ *
+ * FNV-1a (52-bit, via foxts/fnv1a52) is used because it is fast, has good
+ * distribution for load balancing, and stays within JavaScript's safe integer
+ * range. It is NOT cryptographic — never use it where collision resistance
+ * matters.
+ */
+
+import { fnv1a52 } from 'foxts/fnv1a52';
+import process from 'node:process';
+
+export interface ShardConfig {
+  /** 0-based index of this shard. */
+  index: number,
+  /** Total number of shards. */
+  total: number
+}
+
+/**
+ * Read shard configuration from the environment. Defaults to a single shard
+ * (index 0 of 1), i.e. a full run — so local invocations and the non-matrix
+ * path behave exactly like before.
+ */
+export function getShardConfigFromEnv(env: NodeJS.ProcessEnv = process.env): ShardConfig {
+  const total = Number.parseInt(env.SHARD_TOTAL ?? '1', 10);
+  const index = Number.parseInt(env.SHARD_INDEX ?? '0', 10);
+
+  if (!Number.isSafeInteger(total) || total < 1) {
+    throw new RangeError(`Invalid SHARD_TOTAL: ${JSON.stringify(env.SHARD_TOTAL)} (must be an integer >= 1)`);
+  }
+  if (!Number.isSafeInteger(index) || index < 0 || index >= total) {
+    throw new RangeError(`Invalid SHARD_INDEX: ${JSON.stringify(env.SHARD_INDEX)} (must be an integer in [0, ${total - 1}])`);
+  }
+
+  return { index, total };
+}
+
+/** Whether the given domain belongs to this shard. */
+export function isInShard(domain: string, { index, total }: ShardConfig): boolean {
+  if (total === 1) return true;
+  return fnv1a52(domain) % total === index;
+}

Build/validate-domain-alive.ts

@@ -1,8 +1,10 @@
-import { SOURCE_DIR } from './constants/dir';
-import path from 'node:path';
+import fs from 'node:fs';
+import process from 'node:process';
 import { getMethods } from './lib/is-domain-alive';
-import { fdir as Fdir } from 'fdir';
-import runAgainstSourceFile from './lib/run-against-source-file';
+import { enumerateSourceDomains } from './lib/enumerate-source-domains';
+import { getShardConfigFromEnv, isInShard } from './lib/shard';
+import { getRunnerGeoIP } from './lib/get-runner-geoip';
+import type { RunnerGeoIP } from './lib/get-runner-geoip';
 
 import cliProgress from 'cli-progress';
 import { newQueue } from '@henrygd/queue';
@@ -11,66 +13,86 @@ const queue = newQueue(32);
 
 const deadDomains: string[] = [];
 
+/**
+ * Append this shard's result to the GitHub Actions job summary, if running in
+ * CI. Each shard writes its own summary (no dedicated merge job) — the union
+ * of all shards' summaries is the full dead-domain list.
+ */
+function formatGeo(geo: RunnerGeoIP | null): string {
+  if (!geo) return 'unknown (geoip lookup failed)';
+  return `${geo.ip} — ${geo.city}, ${geo.region}, ${geo.country} (AS${geo.asn} ${geo.asOrg})`;
+}
+
+function writeJobSummary(shardLabel: string, dead: string[], geo: RunnerGeoIP | null) {
+  const summaryPath = process.env.GITHUB_STEP_SUMMARY;
+  if (!summaryPath) return;
+
+  let summary = `## Dead domains — shard ${shardLabel}\n\n`
+    + `Runner egress: \`${formatGeo(geo)}\`\n\n`
+    + `Found **${dead.length}** dead domain(s) in this shard.\n\n`;
+
+  if (dead.length > 0) {
+    summary += '<details><summary>Show list</summary>\n\n'
+      + '```\n'
+      + dead.join('\n') + '\n'
+      + '```\n\n'
+      + '</details>\n\n'
+      // Machine-recoverable copy so the union can be scraped from summaries.
+      + '```json\n'
+      + JSON.stringify(dead) + '\n'
+      + '```\n\n';
+  }
+
+  fs.appendFileSync(summaryPath, summary);
+}
+
 (async () => {
+  const shard = getShardConfigFromEnv();
+  const shardLabel = `${shard.index + 1}/${shard.total}`;
+
   const [
     { isDomainAlive, isRegisterableDomainAlive },
-    domainSets,
-    domainRules
+    allDomains,
+    geo
   ] = await Promise.all([
     getMethods(),
-    new Fdir()
-      .withFullPaths()
-      .filter((filePath, isDirectory) => {
-        if (isDirectory) return false;
-        const extname = path.extname(filePath);
-        return extname === '.txt' || extname === '.conf';
-      })
-      .crawl(SOURCE_DIR + path.sep + 'domainset')
-      .withPromise(),
-    new Fdir()
-      .withFullPaths()
-      .filter((filePath, isDirectory) => {
-        if (isDirectory) return false;
-        const extname = path.extname(filePath);
-        return extname === '.txt' || extname === '.conf';
-      })
-      .crawl(SOURCE_DIR + path.sep + 'non_ip')
-      .withPromise()
+    enumerateSourceDomains(),
+    getRunnerGeoIP()
   ]);
 
+  console.log(`[shard ${shardLabel}] runner egress: ${formatGeo(geo)}`);
+
+  const shardDomains = allDomains.filter(({ domain }) => isInShard(domain, shard));
+
+  console.log(
+    `[shard ${shardLabel}] checking ${shardDomains.length} of ${allDomains.length} domain(s)`
+  );
+
   const bar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);
-  bar.start(0, 0);
+  bar.start(shardDomains.length, 0);
 
-  await Promise.all([
-    ...domainRules,
-    ...domainSets
-  ].map(filepath => runAgainstSourceFile(
-    filepath,
-    (domain: string, includeAllSubdomain: boolean) => {
-      bar.setTotal(bar.getTotal() + 1);
+  for (const { domain, includeAllSubdomain } of shardDomains) {
+    queue.add(async () => {
+      let registerableDomainAlive, registerableDomain, alive: boolean | undefined;
 
-      return queue.add(async () => {
-        let registerableDomainAlive, registerableDomain, alive: boolean | undefined;
+      if (includeAllSubdomain) {
+        // we only need to check apex domain, because we don't know if there is any stripped subdomain
+        ({ alive: registerableDomainAlive, registerableDomain } = await isRegisterableDomainAlive(domain));
+      } else {
+        ({ alive, registerableDomainAlive, registerableDomain } = await isDomainAlive(domain));
+      }
 
-        if (includeAllSubdomain) {
-          // we only need to check apex domain, because we don't know if there is any stripped subdomain
-          ({ alive: registerableDomainAlive, registerableDomain } = await isRegisterableDomainAlive(domain));
-        } else {
-          ({ alive, registerableDomainAlive, registerableDomain } = await isDomainAlive(domain));
+      bar.increment();
+
+      if (!registerableDomainAlive) {
+        if (registerableDomain) {
+          deadDomains.push('.' + registerableDomain);
         }
-
-        bar.increment();
-
-        if (!registerableDomainAlive) {
-          if (registerableDomain) {
-            deadDomains.push('.' + registerableDomain);
-          }
-        } else if (!includeAllSubdomain && alive != null && !alive) {
-          deadDomains.push(domain);
-        }
-      });
-    }
-  ).then(() => console.log('[crawl]', filepath))));
+      } else if (!includeAllSubdomain && alive != null && !alive) {
+        deadDomains.push(domain);
+      }
+    });
+  }
 
   await queue.done();
 
@@ -78,5 +100,7 @@ const deadDomains: string[] = [];
 
   console.log();
   console.log();
-  console.log(JSON.stringify(deadDomains));
+  console.log(`[shard ${shardLabel}]`, JSON.stringify(deadDomains));
+
+  writeJobSummary(shardLabel, deadDomains, geo);
 })();