From b57f8699e9e67febc3290e23f3a6fcd47e52c19e Mon Sep 17 00:00:00 2001 From: SukkaW Date: Sat, 23 Aug 2025 21:14:27 +0800 Subject: [PATCH] Chore: pre-populate WHOIS mapping --- Build/lib/is-domain-alive.ts | 39 ++++++++++++++++++--------- Build/validate-domain-alive.ts | 49 ++++++++++++++++++++-------------- 2 files changed, 55 insertions(+), 33 deletions(-) diff --git a/Build/lib/is-domain-alive.ts b/Build/lib/is-domain-alive.ts index 21be0f92..81743304 100644 --- a/Build/lib/is-domain-alive.ts +++ b/Build/lib/is-domain-alive.ts @@ -1,4 +1,5 @@ import { createDomainAliveChecker, createRegisterableDomainAliveChecker } from 'domain-alive'; +import { $$fetch } from './fetch-retry'; const dnsServers = [ '8.8.8.8', @@ -46,17 +47,29 @@ const dnsServers = [ const resultCache = new Map(); const registerableDomainResultCache = new Map(); -export const isRegisterableDomainAlive = createRegisterableDomainAliveChecker({ - dns: { - dnsServers - }, - registerableDomainResultCache -}); +export async function getMethods() { + const customWhoisServersMapping = await (await ($$fetch('https://cdn.jsdelivr.net/npm/whois-servers-list@latest/list.json'))).json() as any; -export const isDomainAlive = createDomainAliveChecker({ - dns: { - dnsServers - }, - registerableDomainResultCache, - resultCache -}); + const isRegisterableDomainAlive = createRegisterableDomainAliveChecker({ + dns: { + dnsServers + }, + registerableDomainResultCache, + whois: { + customWhoisServersMapping + } + }); + + const isDomainAlive = createDomainAliveChecker({ + dns: { + dnsServers + }, + registerableDomainResultCache, + resultCache, + whois: { + customWhoisServersMapping + } + }); + + return { isRegisterableDomainAlive, isDomainAlive }; +}; diff --git a/Build/validate-domain-alive.ts b/Build/validate-domain-alive.ts index 7f79a00e..dc564b2f 100644 --- a/Build/validate-domain-alive.ts +++ b/Build/validate-domain-alive.ts @@ -1,6 +1,6 @@ import { SOURCE_DIR } from './constants/dir'; import path from 'node:path'; -import { isDomainAlive, isRegisterableDomainAlive } from './lib/is-domain-alive'; +import { getMethods } from './lib/is-domain-alive'; import { fdir as Fdir } from 'fdir'; import runAgainstSourceFile from './lib/run-against-source-file'; @@ -12,24 +12,31 @@ const queue = newQueue(32); const deadDomains: string[] = []; (async () => { - const domainSets = await new Fdir() - .withFullPaths() - .filter((filePath, isDirectory) => { - if (isDirectory) return false; - const extname = path.extname(filePath); - return extname === '.txt' || extname === '.conf'; - }) - .crawl(SOURCE_DIR + path.sep + 'domainset') - .withPromise(); - const domainRules = await new Fdir() - .withFullPaths() - .filter((filePath, isDirectory) => { - if (isDirectory) return false; - const extname = path.extname(filePath); - return extname === '.txt' || extname === '.conf'; - }) - .crawl(SOURCE_DIR + path.sep + 'non_ip') - .withPromise(); + const [ + { isDomainAlive, isRegisterableDomainAlive }, + domainSets, + domainRules + ] = await Promise.all([ + getMethods(), + new Fdir() + .withFullPaths() + .filter((filePath, isDirectory) => { + if (isDirectory) return false; + const extname = path.extname(filePath); + return extname === '.txt' || extname === '.conf'; + }) + .crawl(SOURCE_DIR + path.sep + 'domainset') + .withPromise(), + new Fdir() + .withFullPaths() + .filter((filePath, isDirectory) => { + if (isDirectory) return false; + const extname = path.extname(filePath); + return extname === '.txt' || extname === '.conf'; + }) + .crawl(SOURCE_DIR + path.sep + 'non_ip') + .withPromise() + ]); const bar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic); bar.start(0, 0); @@ -56,7 +63,9 @@ const deadDomains: string[] = []; bar.increment(); if (!registerableDomainAlive) { - deadDomains.push('.' + registerableDomain); + if (registerableDomain) { + deadDomains.push('.' + registerableDomain); + } } else if (!includeAllSubdomain && alive != null && !alive) { deadDomains.push(domain); }