Chore: update domain alive check [skip ci]

.github/workflows/check-source-domain.yml

@@ -50,6 +50,8 @@ jobs:
             ${{ runner.os }}-v3-
       - run: pnpm install
       - run: pnpm run node Build/validate-domain-alive.ts
+        env:
+          DEBUG: domain-alive:*
       - name: Cache cache.db
         if: always()
         uses: actions/cache/save@v4

Build/build-telegram-cidr.ts

@@ -36,8 +36,8 @@ export const getTelegramCIDRPromise = once(async () => {
 
   // Backup IP Source 1 (DoH)
   await Promise.all([
-    DNS2.DOHClient({ dns: '8.8.8.8' }),
+    DNS2.DOHClient({ dns: 'https://8.8.4.4/dns-query?dns={query}' }),
-    DNS2.DOHClient({ dns: '1.0.0.1' })
+    DNS2.DOHClient({ dns: 'https://1.0.0.1/dns-query?dns={query}' })
   ].flatMap(
     (client) => [
       'apv3.stel.com', // prod

Build/lib/is-domain-alive.test.ts

@@ -1,16 +0,0 @@
-import { describe, it } from 'mocha';
-
-import { isDomainAlive } from './is-domain-alive';
-import { expect } from 'expect';
-
-describe('isDomainAlive', function () {
-  this.timeout(10000);
-
-  it('samsungcloudsolution.net', async () => {
-    expect((await isDomainAlive('samsungcloudsolution.net', true))).toEqual(false);
-  });
-
-  it('ecdasoin.it', async () => {
-    expect((await isDomainAlive('.ecdasoin.it', true))).toEqual(false);
-  });
-});

Build/lib/is-domain-alive.ts

@@ -1,24 +1,6 @@
-import DNS2 from 'dns2';
+import { createDomainAliveChecker } from 'domain-alive';
-import asyncRetry from 'async-retry';
-import picocolors from 'picocolors';
-import { looseTldtsOpt } from '../constants/loose-tldts-opt';
-import { createKeyedAsyncMutex } from './keyed-async-mutex';
-import tldts from 'tldts-experimental';
-import * as whoiser from 'whoiser';
-import process from 'node:process';
-import { createRetrieKeywordFilter as createKeywordFilter } from 'foxts/retrie';
-import { shuffleArray } from 'foxts/shuffle-array';
 
-const domainAliveMap = new Map<string, boolean>();
+const dnsServers = [
-
-class DnsError extends Error {
-  name = 'DnsError';
-  constructor(readonly message: string, public readonly server: string) {
-    super(message);
-  }
-}
-
-const dohServers: Array<[string, DNS2.DnsResolver]> = ([
   '8.8.8.8',
   '8.8.4.4',
   '1.0.0.1',
@@ -53,316 +35,12 @@ const dohServers: Array<[string, DNS2.DnsResolver]> = ([
   // '198.54.117.10' // NameCheap DNS, supports DoT, DoH, UDP53
   // 'ada.openbld.net',
   // 'dns.rabbitdns.org'
-] as const).map(dns => [
+].map(dns => 'https://' + dns);
-  dns,
-  DNS2.DOHClient({ dns })
-] as const);
 
-const domesticDohServers: Array<[string, DNS2.DnsResolver]> = ([
+console.log({ dnsServers });
-  '223.5.5.5',
-  '223.6.6.6',
-  '120.53.53.53',
-  '1.12.12.12'
-] as const).map(dns => [
-  dns,
-  DNS2.DOHClient({ dns })
-] as const);
 
-const domainAliveMutex = createKeyedAsyncMutex('isDomainAlive');
+export const isDomainAlive = createDomainAliveChecker({
-
+  dns: {
-export async function isDomainAlive(
+    dnsServers
-  domain: string,
-  // we dont need to check domain[0] here, this is only from runAgainstSourceFile
-  isIncludeAllSubdomain: boolean
-): Promise<boolean> {
-  if (domainAliveMap.has(domain)) {
-    return domainAliveMap.get(domain)!;
   }
-  const apexDomain = tldts.getDomain(domain, looseTldtsOpt);
+});
-  if (!apexDomain) {
-    // console.log(picocolors.gray('[domain invalid]'), picocolors.gray('no apex domain'), { domain });
-    domainAliveMap.set('.' + domain, true);
-    return true;
-  }
-
-  const apexDomainAlive = await isApexDomainAlive(apexDomain);
-  if (isIncludeAllSubdomain || domain.length > apexDomain.length) {
-    return apexDomainAlive;
-  }
-  if (!apexDomainAlive) {
-    return false;
-  }
-
-  return domainAliveMutex.acquire(domain, async () => {
-    domain = domain[0] === '.' ? domain.slice(1) : domain;
-
-    const aDns: string[] = [];
-    const aaaaDns: string[] = [];
-
-    // test 2 times before make sure record is empty
-    const servers = shuffleArray(dohServers, { copy: true });
-
-    for (let i = 0, len = servers.length; i < len; i++) {
-      try {
-        // eslint-disable-next-line no-await-in-loop -- sequential
-        const aRecords = (await $resolve(domain, 'A', servers[i]));
-        if (aRecords.answers.length > 0) {
-          domainAliveMap.set(domain, true);
-          return true;
-        }
-
-        aDns.push(servers[i][0]);
-      } catch {}
-
-      if (aDns.length >= 2) {
-        break; // we only need to test 2 times
-      }
-    }
-
-    for (let i = 0, len = servers.length; i < len; i++) {
-      try {
-        // eslint-disable-next-line no-await-in-loop -- sequential
-        const aaaaRecords = await $resolve(domain, 'AAAA', servers[i]);
-        if (aaaaRecords.answers.length > 0) {
-          domainAliveMap.set(domain, true);
-          return true;
-        }
-
-        aaaaDns.push(servers[i][0]);
-      } catch {}
-
-      if (aaaaDns.length >= 2) {
-        break; // we only need to test 2 times
-      }
-    }
-
-    // only then, let's test twice with domesticDohServers
-    const domesticServers = shuffleArray(domesticDohServers, { copy: true });
-    for (let i = 0, len = domesticServers.length; i < len; i++) {
-      try {
-        // eslint-disable-next-line no-await-in-loop -- sequential
-        const aRecords = await $resolve(domain, 'A', domesticServers[i]);
-        if (aRecords.answers.length > 0) {
-          domainAliveMap.set(domain, true);
-          return true;
-        }
-        aDns.push(domesticServers[i][0]);
-      } catch {}
-      if (aDns.length >= 2) {
-        break; // we only need to test 2 times
-      }
-    }
-
-    for (let i = 0, len = domesticServers.length; i < len; i++) {
-      try {
-        // eslint-disable-next-line no-await-in-loop -- sequential
-        const aaaaRecords = await $resolve(domain, 'AAAA', domesticServers[i]);
-        if (aaaaRecords.answers.length > 0) {
-          domainAliveMap.set(domain, true);
-          return true;
-        }
-        aaaaDns.push(domesticServers[i][0]);
-      } catch {}
-
-      if (aaaaDns.length >= 2) {
-        break; // we only need to test 2 times
-      }
-    }
-
-    console.log(picocolors.red('[domain dead]'), 'no A/AAAA records', { domain, a: aDns, aaaa: aaaaDns });
-
-    domainAliveMap.set(domain, false);
-    return false;
-  });
-}
-
-const apexDomainMap = createKeyedAsyncMutex('isApexDomainAlive');
-
-function isApexDomainAlive(apexDomain: string) {
-  if (domainAliveMap.has(apexDomain)) {
-    return domainAliveMap.get(apexDomain)!;
-  }
-
-  return apexDomainMap.acquire(apexDomain, async () => {
-    const servers = shuffleArray(dohServers, { copy: true });
-
-    let nsSuccess = 0;
-
-    for (let i = 0, len = servers.length; i < len; i++) {
-      const server = servers[i];
-      try {
-        // eslint-disable-next-line no-await-in-loop -- one by one
-        const resp = await $resolve(apexDomain, 'NS', server);
-        if (resp.answers.length > 0) {
-          domainAliveMap.set(apexDomain, true);
-          return true;
-        }
-
-        nsSuccess++;
-
-        if (nsSuccess >= 2) {
-          // we only need to test 2 times
-          break;
-        }
-      } catch {}
-    }
-
-    let whois;
-    try {
-      whois = await getWhois(apexDomain);
-    } catch (e) {
-      console.log(picocolors.red('[whois error]'), { domain: apexDomain }, e);
-      domainAliveMap.set(apexDomain, true);
-      return true;
-    }
-
-    const whoisError = noWhois(whois);
-    if (!whoisError) {
-      console.log(picocolors.gray('[domain alive]'), picocolors.gray('whois found'), { domain: apexDomain });
-      domainAliveMap.set(apexDomain, true);
-      return true;
-    }
-
-    console.log(picocolors.red('[domain dead]'), 'whois not found', { domain: apexDomain, err: whoisError });
-
-    domainAliveMap.set(apexDomain, false);
-    return false;
-  });
-}
-
-async function $resolve(name: string, type: DNS2.PacketQuestion, server: [string, DNS2.DnsResolver]) {
-  try {
-    return await asyncRetry(async () => {
-      const [dohServer, dohClient] = server;
-
-      try {
-        return await dohClient(name, type);
-      } catch (e) {
-        // console.error(e);
-        throw new DnsError((e as Error).message, dohServer);
-      }
-    }, { retries: 5 });
-  } catch (e) {
-    console.log('[doh error]', name, type, e);
-    throw e;
-  }
-}
-
-async function getWhois(domain: string) {
-  return asyncRetry(() => whoiser.domain(domain, { raw: true }), { retries: 5 });
-}
-
-// TODO: this is a workaround for https://github.com/LayeredStudio/whoiser/issues/117
-const whoisNotFoundKeywordTest = createKeywordFilter([
-  'no match for',
-  'does not exist',
-  'not found',
-  'no found',
-  'no entries',
-  'no data found',
-  'is available for registration',
-  'currently available for application',
-  'no matching record',
-  'no information available about domain name',
-  'not been registered',
-  'no match!!',
-  'status: available',
-  ' is free',
-  'no object found',
-  'nothing found',
-  'status: free',
-  // 'pendingdelete',
-  ' has been blocked by '
-]);
-// whois server can redirect, so whoiser might/will get info from multiple whois servers
-// some servers (like TLD whois servers) might have cached/outdated results
-// we can only make sure a domain is alive once all response from all whois servers demonstrate so
-function noWhois(whois: whoiser.WhoisSearchResult): null | string {
-  let empty = true;
-
-  for (const key in whois) {
-    if (Object.hasOwn(whois, key)) {
-      empty = false;
-
-      // if (key === 'error') {
-      //   // if (
-      //   //   (typeof whois.error === 'string' && whois.error)
-      //   //   || (Array.isArray(whois.error) && whois.error.length > 0)
-      //   // ) {
-      //   //   console.error(whois);
-      //   //   return true;
-      //   // }
-      //   continue;
-      // }
-
-      // if (key === 'text') {
-      //   if (Array.isArray(whois.text)) {
-      //     for (const value of whois.text) {
-      //       if (whoisNotFoundKeywordTest(value.toLowerCase())) {
-      //         return value;
-      //       }
-      //     }
-      //   }
-      //   continue;
-      // }
-      // if (key === 'Name Server') {
-      //   // if (Array.isArray(whois[key]) && whois[key].length === 0) {
-      //   //   return false;
-      //   // }
-      //   continue;
-      // }
-
-      // if (key === 'Domain Status') {
-      //   if (Array.isArray(whois[key])) {
-      //     for (const status of whois[key]) {
-      //       if (status === 'free' || status === 'AVAILABLE') {
-      //         return key + ': ' + status;
-      //       }
-      //       if (whoisNotFoundKeywordTest(status.toLowerCase())) {
-      //         return key + ': ' + status;
-      //       }
-      //     }
-      //   }
-
-      //   continue;
-      // }
-
-      // if (typeof whois[key] === 'string' && whois[key]) {
-      //   if (whoisNotFoundKeywordTest(whois[key].toLowerCase())) {
-      //     return key + ': ' + whois[key];
-      //   }
-
-      //   continue;
-      // }
-
-      if (key === '__raw' && typeof whois.__raw === 'string') {
-        const lines = whois.__raw.trim().toLowerCase().replaceAll(/[\t ]+/g, ' ').split(/\r?\n/);
-
-        if (process.env.DEBUG) {
-          console.log({ lines });
-        }
-
-        for (const line of lines) {
-          if (whoisNotFoundKeywordTest(line)) {
-            return line;
-          }
-        }
-        continue;
-      }
-
-      if (typeof whois[key] === 'object' && !Array.isArray(whois[key])) {
-        const tmp = noWhois(whois[key]);
-        if (tmp) {
-          return tmp;
-        }
-        continue;
-      }
-    }
-  }
-
-  if (empty) {
-    return 'whois is empty';
-  }
-
-  return null;
-}

Build/lib/keyed-async-mutex.ts

@@ -1,23 +0,0 @@
-const globalMap = new Map<string, Map<string, Promise<unknown>>>();
-
-export function createKeyedAsyncMutex(globalNamespaceKey: string) {
-  let map;
-  if (globalMap.has(globalNamespaceKey)) {
-    map = globalMap.get(globalNamespaceKey)!;
-  } else {
-    map = new Map();
-    globalMap.set(globalNamespaceKey, map);
-  }
-
-  return {
-    async acquire<T = unknown>(key: string, fn: () => Promise<T>) {
-      if (map.has(key)) {
-        return map.get(key);
-      }
-
-      const promise = fn();
-      map.set(key, promise);
-      return promise;
-    }
-  };
-}

Build/validate-domain-alive.ts

@@ -44,13 +44,14 @@ const deadDomains: string[] = [];
         bar.setTotal(bar.getTotal() + 1);
 
         return queue.add(
-          () => isDomainAlive(domain, includeAllSubdomain).then((alive) => {
+          () => isDomainAlive(domain).then(({ alive, registerableDomainAlive, registerableDomain }) => {
             bar.increment();
 
-            if (alive) {
+            if (!registerableDomainAlive) {
-              return;
+              deadDomains.push('.' + registerableDomain);
+            } else if (!alive) {
+              deadDomains.push(includeAllSubdomain ? '.' + domain : domain);
             }
-            deadDomains.push(includeAllSubdomain ? '.' + domain : domain);
           })
         );
       }

package.json

@@ -22,17 +22,17 @@
     "@ghostery/adblocker": "^2.11.5",
     "@henrygd/queue": "^1.0.7",
     "@mitata/counters": "^0.0.8",
-    "async-retry": "^1.3.3",
     "better-sqlite3": "^12.2.0",
     "ci-info": "^4.3.0",
     "cli-progress": "^3.12.0",
     "csv-parse": "^6.1.0",
-    "dns2": "^2.1.0",
+    "dns2": "github:lsongdev/node-dns#e4fa035aca0b8eb730bde3431fbf0c60a31a09c9",
+    "domain-alive": "^0.1.5",
     "fast-cidr-tools": "^0.3.2",
     "fast-fifo": "^1.3.2",
     "fast-uri": "^3.0.6",
     "fdir": "^6.5.0",
-    "foxts": "^3.11.1",
+    "foxts": "^3.12.0",
     "hash-wasm": "^4.12.0",
     "json-stringify-pretty-compact": "3.0.0",
     "null-prototype-object": "^1.2.2",
@@ -44,7 +44,6 @@
     "tldts-experimental": "^6.1.86",
     "undici": "^7.14.0",
     "undici-cache-store-better-sqlite3": "^1.0.0",
-    "whoiser": "^1.18.0",
     "why-is-node-running": "^3.2.2",
     "worktank": "^3.0.2",
     "xbits": "^0.2.0",
@@ -55,7 +54,6 @@
     "@eslint-sukka/node": "^6.23.1",
     "@swc-node/register": "^1.11.1",
     "@swc/core": "^1.13.4",
-    "@types/async-retry": "^1.4.9",
     "@types/better-sqlite3": "^7.6.13",
     "@types/cli-progress": "^3.11.6",
     "@types/dns2": "^2.0.10",
@@ -76,9 +74,6 @@
   },
   "packageManager": "pnpm@10.15.0",
   "pnpm": {
-    "patchedDependencies": {
-      "whoiser": "patches/whoiser.patch"
-    },
     "onlyBuiltDependencies": [
       "@swc/core",
       "better-sqlite3",
@@ -89,6 +84,11 @@
       "globalthis": "npm:@nolyfill/globalthis@^1.0.44",
       "has": "npm:@nolyfill/has@^1.0.44",
       "safe-buffer": "npm:@nolyfill/safe-buffer@^1.0.44"
-    }
+    },
+    "ignoredBuiltDependencies": [
+      "bufferutil",
+      "es5-ext",
+      "utf-8-validate"
+    ]
   }
 }

patches/whoiser.patch

@@ -1,13 +0,0 @@
-diff --git a/src/whoiser.js b/src/whoiser.js
-index ff42b8c7fe1749d389df2d420f68f1ec6590fe69..dea40e123c8bab3c38c1e5d41b6da3bff43acbfe 100644
---- a/src/whoiser.js
-+++ b/src/whoiser.js
-@@ -50,6 +50,8 @@ let cacheTldWhoisServer = {
- 	shop: 'whois.nic.shop',
- 	site: 'whois.nic.site',
- 	xyz: 'whois.nic.xyz',
-+
-+	ga: 'whois.nic.ga'
- }
- 
- // misspelled whois servers..