Chore: parallel checking dead domains

Build/lib/enumerate-source-domains.ts

@@ -0,0 +1,66 @@
+import { SOURCE_DIR } from '../constants/dir';
+import path from 'node:path';
+import { fdir as Fdir } from 'fdir';
+import runAgainstSourceFile from './run-against-source-file';
+
+export interface SourceDomain {
+  domain: string,
+  /**
+   * `true` for `DOMAIN-SUFFIX` / leading-dot domainset entries (apex domain
+   * that includes all subdomains), `false` for exact `DOMAIN` entries.
+   */
+  includeAllSubdomain: boolean
+}
+
+function sourceFileFilter(filePath: string, isDirectory: boolean) {
+  if (isDirectory) return false;
+  const extname = path.extname(filePath);
+  return extname === '.txt' || extname === '.conf';
+}
+
+/**
+ * Crawl the `domainset` and `non_ip` source directories and return every
+ * domain entry they declare.
+ *
+ * A domain that appears with `includeAllSubdomain: true` in any source wins
+ * over an exact-only occurrence: checking the apex (with subdomains) also
+ * covers the exact host, so we collapse duplicates to the broader check. This
+ * is the natural dedupe that makes sharding by domain hash correct.
+ */
+export async function enumerateSourceDomains(): Promise<SourceDomain[]> {
+  const [domainSets, domainRules] = await Promise.all([
+    new Fdir()
+      .withFullPaths()
+      .filter(sourceFileFilter)
+      .crawl(SOURCE_DIR + path.sep + 'domainset')
+      .withPromise(),
+    new Fdir()
+      .withFullPaths()
+      .filter(sourceFileFilter)
+      .crawl(SOURCE_DIR + path.sep + 'non_ip')
+      .withPromise()
+  ]);
+
+  // domain -> includeAllSubdomain (true wins)
+  const seen = new Map<string, boolean>();
+
+  await Promise.all(
+    [...domainRules, ...domainSets].map(filepath => runAgainstSourceFile(
+      filepath,
+      (domain: string, includeAllSubdomain: boolean) => {
+        const prev = seen.get(domain);
+        if (prev === undefined) {
+          seen.set(domain, includeAllSubdomain);
+        } else if (includeAllSubdomain && !prev) {
+          seen.set(domain, true);
+        }
+      }
+    ))
+  );
+
+  const result: SourceDomain[] = [];
+  for (const [domain, includeAllSubdomain] of seen) {
+    result.push({ domain, includeAllSubdomain });
+  }
+  return result;
+}

Build/lib/get-runner-geoip.ts

@@ -0,0 +1,30 @@
+import { $$fetch } from './fetch-retry';
+
+export interface RunnerGeoIP {
+  ip: string,
+  country: string,
+  region: string,
+  city: string,
+  asn: number,
+  asOrg: string,
+  longitude: number,
+  latitude: number,
+  timezone: string,
+  cfEdgeIP: string
+}
+
+/**
+ * Fetch the current machine's egress IP and geo info. Used to confirm that
+ * each matrix shard landed on a different GitHub-hosted runner region / egress
+ * IP — the premise that lets sharding spread DoH load and dodge rate limits.
+ *
+ * Returns `null` on any failure so it never aborts the actual domain check.
+ */
+export async function getRunnerGeoIP(): Promise<RunnerGeoIP | null> {
+  try {
+    const res = await $$fetch('https://ip.api.skk.moe/cf-geoip');
+    return await res.json() as RunnerGeoIP;
+  } catch {
+    return null;
+  }
+}

Build/lib/shard.ts

@@ -0,0 +1,55 @@
+/**
+ * Deterministic domain sharding for spreading the domain-alive check across
+ * multiple GitHub Actions runners.
+ *
+ * GitHub-hosted runners are provisioned across many Azure regions, so each
+ * matrix job gets a distinct egress IP. Splitting the workload by a stable
+ * hash of the domain lets every runner enumerate the *same* source files but
+ * only check the subset that belongs to its shard, which:
+ *
+ *   - spreads DoH load across N distinct egress IPs (fewer per-server rate
+ *     limits), and
+ *   - makes assignment deterministic — the same domain always lands on the
+ *     same shard, so duplicates across source files collapse to one check and
+ *     re-runs are reproducible.
+ *
+ * FNV-1a (52-bit, via foxts/fnv1a52) is used because it is fast, has good
+ * distribution for load balancing, and stays within JavaScript's safe integer
+ * range. It is NOT cryptographic — never use it where collision resistance
+ * matters.
+ */
+
+import { fnv1a52 } from 'foxts/fnv1a52';
+import process from 'node:process';
+
+export interface ShardConfig {
+  /** 0-based index of this shard. */
+  index: number,
+  /** Total number of shards. */
+  total: number
+}
+
+/**
+ * Read shard configuration from the environment. Defaults to a single shard
+ * (index 0 of 1), i.e. a full run — so local invocations and the non-matrix
+ * path behave exactly like before.
+ */
+export function getShardConfigFromEnv(env: NodeJS.ProcessEnv = process.env): ShardConfig {
+  const total = Number.parseInt(env.SHARD_TOTAL ?? '1', 10);
+  const index = Number.parseInt(env.SHARD_INDEX ?? '0', 10);
+
+  if (!Number.isSafeInteger(total) || total < 1) {
+    throw new RangeError(`Invalid SHARD_TOTAL: ${JSON.stringify(env.SHARD_TOTAL)} (must be an integer >= 1)`);
+  }
+  if (!Number.isSafeInteger(index) || index < 0 || index >= total) {
+    throw new RangeError(`Invalid SHARD_INDEX: ${JSON.stringify(env.SHARD_INDEX)} (must be an integer in [0, ${total - 1}])`);
+  }
+
+  return { index, total };
+}
+
+/** Whether the given domain belongs to this shard. */
+export function isInShard(domain: string, { index, total }: ShardConfig): boolean {
+  if (total === 1) return true;
+  return fnv1a52(domain) % total === index;
+}