Feat: validate domains are alive (and w/ CI)

Build/validate-domain-alive.ts

@@ -0,0 +1,230 @@
+import DNS2 from 'dns2';
+import { readFileByLine } from './lib/fetch-text-by-line';
+import { processLine } from './lib/process-line';
+import tldts from 'tldts';
+import { looseTldtsOpt } from './constants/loose-tldts-opt';
+import { fdir as Fdir } from 'fdir';
+import { SOURCE_DIR } from './constants/dir';
+import path from 'node:path';
+import { newQueue } from '@henrygd/queue';
+import asyncRetry from 'async-retry';
+import * as whoiser from 'whoiser';
+import picocolors from 'picocolors';
+
+const dohServers: Array<[string, DNS2.DnsResolver]> = ([
+  '8.8.8.8',
+  '8.8.4.4',
+  '1.0.0.1',
+  '1.1.1.1',
+  '162.159.36.1',
+  '162.159.46.1',
+  '101.101.101.101', // TWNIC
+  '185.222.222.222', // DNS.SB
+  '45.11.45.11', // DNS.SB
+  '9.9.9.10', // Quad9 unfiltered
+  '149.112.112.10', // Quad9 unfiltered
+  '208.67.222.2', // OpenDNS sandbox (unfiltered)
+  '208.67.220.2', // OpenDNS sandbox (unfiltered)
+  '94.140.14.140', // AdGuard unfiltered
+  '94.140.14.141', // AdGuard unfiltered
+  // '76.76.2.0', // ControlD unfiltered, path not /dns-query
+  // '76.76.10.0', // ControlD unfiltered, path not /dns-query
+  '193.110.81.0', // dns0.eu
+  '185.253.5.0', // dns0.eu
+  'dns.nextdns.io',
+  'wikimedia-dns.org',
+  // 'ordns.he.net',
+  'dns.mullvad.net'
+  // 'ada.openbld.net',
+  // 'dns.rabbitdns.org'
+] as const).map(server => [
+  server,
+  DNS2.DOHClient({
+    dns: server,
+    http: false
+  })
+] as const);
+
+const queue = newQueue(8);
+
+class DnsError extends Error {
+  name = 'DnsError';
+  constructor(readonly message: string, public readonly server: string) {
+    super(message);
+  }
+}
+
+interface DnsResponse extends DNS2.$DnsResponse {
+  dns: string
+}
+
+const resolve: DNS2.DnsResolver<DnsResponse> = async (...args) => {
+  try {
+    return await asyncRetry(async () => {
+      const [dohServer, dohClient] = dohServers[Math.floor(Math.random() * dohServers.length)];
+
+      try {
+        const resp = await dohClient(...args);
+        return {
+          ...resp,
+          dns: dohServer
+        } satisfies DnsResponse;
+      } catch (e) {
+        throw new DnsError((e as Error).message, dohServer);
+      }
+    }, { retries: 5 });
+  } catch (e) {
+    console.log('[doh error]', ...args, e);
+    throw e;
+  }
+};
+
+(async () => {
+  const domainSets = await new Fdir()
+    .withFullPaths()
+    .crawl(SOURCE_DIR + path.sep + 'domainset')
+    .withPromise();
+  const domainRules = await new Fdir()
+    .withFullPaths()
+    .crawl(SOURCE_DIR + path.sep + 'non_ip')
+    .withPromise();
+
+  await Promise.all([
+    ...domainSets.map(runAgainstDomainset),
+    ...domainRules.map(runAgainstRuleset)
+  ]);
+
+  console.log('done');
+})();
+
+const domainAliveMap = new Map<string, boolean>();
+async function isApexDomainAlive(apexDomain: string): Promise<[string, boolean]> {
+  if (domainAliveMap.has(apexDomain)) {
+    return [apexDomain, domainAliveMap.get(apexDomain)!];
+  }
+
+  const resp = await resolve(apexDomain, 'NS');
+
+  if (resp.answers.length > 0) {
+    console.log(picocolors.green('[domain alive]'), 'NS record', apexDomain);
+    return [apexDomain, true];
+  }
+
+  let whois;
+
+  try {
+    whois = await whoiser.domain(apexDomain);
+  } catch (e) {
+    console.log('[whois fail]', 'whois error', { domain: apexDomain }, e);
+    return [apexDomain, true];
+  }
+
+  if (Object.keys(whois).length > 0) {
+    // TODO: this is a workaround for https://github.com/LayeredStudio/whoiser/issues/117
+    if ('text' in whois && (whois.text as string[]).some(value => value.includes('No match for'))) {
+      console.log(picocolors.red('[domain dead]'), 'whois no match', { domain: apexDomain });
+      domainAliveMap.set(apexDomain, false);
+      return [apexDomain, false];
+    }
+
+    console.log(picocolors.green('[domain alive]'), 'recorded in whois', apexDomain);
+    return [apexDomain, true];
+  }
+
+  if (!('dns' in whois)) {
+    console.log({ whois });
+  }
+
+  console.log(picocolors.red('[domain dead]'), 'whois no match', { domain: apexDomain });
+  domainAliveMap.set(apexDomain, false);
+  return [apexDomain, false];
+}
+
+const domainMutex = new Map<string, Promise<[string, boolean]>>();
+
+export async function isDomainAlive(domain: string, isSuffix: boolean): Promise<[string, boolean]> {
+  if (domain[0] === '.') {
+    domain = domain.slice(1);
+  }
+
+  const apexDomain = tldts.getDomain(domain, looseTldtsOpt);
+  if (!apexDomain) {
+    console.log('[domain invalid]', 'no apex domain', { domain });
+    return [domain, true] as const;
+  }
+
+  let apexDomainAlivePromise;
+  if (domainMutex.has(domain)) {
+    apexDomainAlivePromise = domainMutex.get(domain)!;
+  } else {
+    apexDomainAlivePromise = queue.add(() => isApexDomainAlive(apexDomain).then(res => {
+      domainMutex.delete(domain);
+      return res;
+    }));
+    domainMutex.set(domain, apexDomainAlivePromise);
+  }
+  const apexDomainAlive = await apexDomainAlivePromise;
+
+  if (!apexDomainAlive[1]) {
+    domainAliveMap.set(domain, false);
+    return [domain, false] as const;
+  }
+
+  if (!isSuffix) {
+    const aRecords = (await resolve(domain, 'A'));
+    if (aRecords.answers.length === 0) {
+      const aaaaRecords = (await resolve(domain, 'AAAA'));
+      if (aaaaRecords.answers.length === 0) {
+        console.log(picocolors.red('[domain dead]'), 'no A/AAAA records', { domain, a: aRecords.dns, aaaa: aaaaRecords.dns });
+        domainAliveMap.set(domain, false);
+        return [domain, false] as const;
+      }
+    }
+  }
+
+  domainAliveMap.set(domain, true);
+  return [domain, true] as const;
+}
+
+export async function runAgainstRuleset(filepath: string) {
+  const promises: Array<Promise<[string, boolean]>> = [];
+
+  for await (const l of readFileByLine(filepath)) {
+    const line = processLine(l);
+    if (!line) continue;
+    const [type, domain] = line.split(',');
+    switch (type) {
+      case 'DOMAIN-SUFFIX':
+      case 'DOMAIN': {
+        if (!domainMutex.has(domain)) {
+          const promise = queue.add(() => isDomainAlive(domain, type === 'DOMAIN-SUFFIX')).then(res => {
+            domainMutex.delete(domain);
+            return res;
+          });
+          domainMutex.set(domain, promise);
+          promises.push(promise);
+        }
+        break;
+      }
+      // no default
+      // case 'DOMAIN-KEYWORD': {
+      //   break;
+      // }
+      // no default
+    }
+  }
+
+  return Promise.all(promises);
+}
+
+export async function runAgainstDomainset(filepath: string) {
+  const promises: Array<Promise<[string, boolean]>> = [];
+
+  for await (const l of readFileByLine(filepath)) {
+    const line = processLine(l);
+    if (!line) continue;
+    promises.push(isDomainAlive(line, line[0] === '.'));
+  }
+
+  return Promise.all(promises);
+}