import DNS2 from 'dns2'; import { readFileByLine } from './lib/fetch-text-by-line'; import { processLine } from './lib/process-line'; import tldts from 'tldts'; import { looseTldtsOpt } from './constants/loose-tldts-opt'; import { fdir as Fdir } from 'fdir'; import { SOURCE_DIR } from './constants/dir'; import path from 'node:path'; import { newQueue } from '@henrygd/queue'; import asyncRetry from 'async-retry'; import * as whoiser from 'whoiser'; import picocolors from 'picocolors'; import createKeywordFilter from './lib/aho-corasick'; const dohServers: Array<[string, DNS2.DnsResolver]> = ([ '8.8.8.8', '8.8.4.4', '1.0.0.1', '1.1.1.1', '162.159.36.1', '162.159.46.1', '101.101.101.101', // TWNIC '185.222.222.222', // DNS.SB '45.11.45.11', // DNS.SB '9.9.9.10', // Quad9 unfiltered '149.112.112.10', // Quad9 unfiltered '208.67.222.2', // OpenDNS sandbox (unfiltered) '208.67.220.2', // OpenDNS sandbox (unfiltered) '94.140.14.140', // AdGuard unfiltered '94.140.14.141', // AdGuard unfiltered // '76.76.2.0', // ControlD unfiltered, path not /dns-query // '76.76.10.0', // ControlD unfiltered, path not /dns-query '193.110.81.0', // dns0.eu '185.253.5.0', // dns0.eu 'dns.nextdns.io', 'wikimedia-dns.org', // 'ordns.he.net', 'dns.mullvad.net' // 'ada.openbld.net', // 'dns.rabbitdns.org' ] as const).map(server => [ server, DNS2.DOHClient({ dns: server, http: false }) ] as const); const queue = newQueue(8); class DnsError extends Error { name = 'DnsError'; constructor(readonly message: string, public readonly server: string) { super(message); } } interface DnsResponse extends DNS2.$DnsResponse { dns: string } const resolve: DNS2.DnsResolver = async (...args) => { try { return await asyncRetry(async () => { const [dohServer, dohClient] = dohServers[Math.floor(Math.random() * dohServers.length)]; try { const resp = await dohClient(...args); return { ...resp, dns: dohServer } satisfies DnsResponse; } catch (e) { throw new DnsError((e as Error).message, dohServer); } }, { retries: 5 }); } catch (e) { console.log('[doh error]', ...args, e); throw e; } }; (async () => { const domainSets = await new Fdir() .withFullPaths() .crawl(SOURCE_DIR + path.sep + 'domainset') .withPromise(); const domainRules = await new Fdir() .withFullPaths() .crawl(SOURCE_DIR + path.sep + 'non_ip') .withPromise(); await Promise.all([ ...domainSets.map(runAgainstDomainset), ...domainRules.map(runAgainstRuleset) ]); console.log('done'); })(); const whoisNotFoundKeywordTest = createKeywordFilter([ 'no match for', 'does not exist', 'not found' ]); const domainAliveMap = new Map(); async function isApexDomainAlive(apexDomain: string): Promise<[string, boolean]> { if (domainAliveMap.has(apexDomain)) { return [apexDomain, domainAliveMap.get(apexDomain)!]; } const resp = await resolve(apexDomain, 'NS'); if (resp.answers.length > 0) { console.log(picocolors.green('[domain alive]'), 'NS record', apexDomain); return [apexDomain, true]; } let whois; try { whois = await whoiser.domain(apexDomain); } catch (e) { console.log('[whois fail]', 'whois error', { domain: apexDomain }, e); return [apexDomain, true]; } if (Object.keys(whois).length > 0) { // TODO: this is a workaround for https://github.com/LayeredStudio/whoiser/issues/117 if ('text' in whois && Array.isArray(whois.text) && whois.text.some(value => whoisNotFoundKeywordTest(value.toLowerCase()))) { console.log(picocolors.red('[domain dead]'), 'whois no match', { domain: apexDomain }); domainAliveMap.set(apexDomain, false); return [apexDomain, false]; } console.log(picocolors.green('[domain alive]'), 'recorded in whois', apexDomain); return [apexDomain, true]; } if (!('dns' in whois)) { console.log({ whois }); } console.log(picocolors.red('[domain dead]'), 'whois no match', { domain: apexDomain }); domainAliveMap.set(apexDomain, false); return [apexDomain, false]; } const domainMutex = new Map>(); export async function isDomainAlive(domain: string, isSuffix: boolean): Promise<[string, boolean]> { if (domain[0] === '.') { domain = domain.slice(1); } const apexDomain = tldts.getDomain(domain, looseTldtsOpt); if (!apexDomain) { console.log('[domain invalid]', 'no apex domain', { domain }); return [domain, true] as const; } let apexDomainAlivePromise; if (domainMutex.has(domain)) { apexDomainAlivePromise = domainMutex.get(domain)!; } else { apexDomainAlivePromise = queue.add(() => isApexDomainAlive(apexDomain).then(res => { domainMutex.delete(domain); return res; })); domainMutex.set(domain, apexDomainAlivePromise); } const apexDomainAlive = await apexDomainAlivePromise; if (!apexDomainAlive[1]) { domainAliveMap.set(domain, false); return [domain, false] as const; } if (!isSuffix) { const aRecords = (await resolve(domain, 'A')); if (aRecords.answers.length === 0) { const aaaaRecords = (await resolve(domain, 'AAAA')); if (aaaaRecords.answers.length === 0) { console.log(picocolors.red('[domain dead]'), 'no A/AAAA records', { domain, a: aRecords.dns, aaaa: aaaaRecords.dns }); domainAliveMap.set(domain, false); return [domain, false] as const; } } } domainAliveMap.set(domain, true); return [domain, true] as const; } export async function runAgainstRuleset(filepath: string) { const promises: Array> = []; for await (const l of readFileByLine(filepath)) { const line = processLine(l); if (!line) continue; const [type, domain] = line.split(','); switch (type) { case 'DOMAIN-SUFFIX': case 'DOMAIN': { if (!domainMutex.has(domain)) { const promise = queue.add(() => isDomainAlive(domain, type === 'DOMAIN-SUFFIX')).then(res => { domainMutex.delete(domain); return res; }); domainMutex.set(domain, promise); promises.push(promise); } break; } // no default // case 'DOMAIN-KEYWORD': { // break; // } // no default } } return Promise.all(promises); } export async function runAgainstDomainset(filepath: string) { const promises: Array> = []; for await (const l of readFileByLine(filepath)) { const line = processLine(l); if (!line) continue; promises.push(isDomainAlive(line, line[0] === '.')); } return Promise.all(promises); }