From fd80f45d223f59a40939adbf7c398d45c5f22e37 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Fri, 10 Jan 2025 21:16:46 +0800 Subject: [PATCH] Feat: improve whois detect [skip ci] --- Build/lib/is-domain-alive.test.ts | 71 +------------------ Build/lib/is-domain-alive.ts | 109 ++++++++++++++++++------------ 2 files changed, 67 insertions(+), 113 deletions(-) diff --git a/Build/lib/is-domain-alive.test.ts b/Build/lib/is-domain-alive.test.ts index 4fff9f96..1078e85f 100644 --- a/Build/lib/is-domain-alive.test.ts +++ b/Build/lib/is-domain-alive.test.ts @@ -1,75 +1,10 @@ import { describe, it } from 'mocha'; -import { isDomainAlive, noWhois } from './is-domain-alive'; +import { isDomainAlive } from './is-domain-alive'; import { expect } from 'expect'; import process from 'node:process'; -describe('whoisExists', () => { - it('.cryptocrawler.io', () => { - expect(noWhois({ - 'whois.nic.io': { - 'Domain Status': [], - 'Name Server': [], - '>>> Last update of WHOIS database': '2025-01-05T11:06:38Z <<<', - text: [ - 'Domain not found.', - '', - 'Terms of Use: Access to WHOIS' - ] - } - })).toBe('Domain not found.'); - }); - - it('.tunevideo.ru', () => { - expect(noWhois({ - 'whois.tcinet.ru': { - 'Domain Status': [], - 'Name Server': [], - text: [ - '% TCI Whois Service. Terms of use:', - '% https://tcinet.ru/documents/whois_ru_rf.pdf (in Russian)', - '% https://tcinet.ru/documents/whois_su.pdf (in Russian)', - '', - 'No entries found for the selected source(s).', - '', - 'Last updated on 2025-01-05T11:03:01Z' - ] - } - })).toBe('No entries found for the selected source(s).'); - }); - - it('nosuchpool.cl', () => { - expect(noWhois({ - 'whois.nic.cl': { - 'Domain Status': [], - 'Name Server': [], - 'nosuchpool.cl': 'no entries found.', - text: [ - '%%', - '%% This is the NIC Chile Whois server (whois.nic.cl).', - '%%', - '%% Rights restricted by copyright.', - '%% See https://www.nic.cl/normativa/politica-publicacion-de-datos-cl.pdf', - '%%' - ] - } - })).toBe('nosuchpool.cl: no entries found.'); - }); - - it('whois.domain-registry.nl', () => { - expect(noWhois({ - 'whois.domain-registry.nl': { - 'Domain Status': [], - 'Name Server': [], - text: [ - 'spookyplanet.nl is free' - ] - } - })).toBe('spookyplanet.nl is free'); - }); -}); - describe('isDomainAlive', function () { this.timeout(10000); @@ -105,8 +40,8 @@ describe('isDomainAlive', function () { // expect((await isDomainAlive('.tayfundogdas.me', true))[1]).toEqual(true); // }); - it('spookyplanet.nl', async () => { + it('ecdasoin.it', async () => { process.env.DEBUG = 'true'; - expect((await isDomainAlive('.spookyplanet.nl', true))[1]).toEqual(false); + expect((await isDomainAlive('.ecdasoin.it', true))[1]).toEqual(false); }); }); diff --git a/Build/lib/is-domain-alive.ts b/Build/lib/is-domain-alive.ts index 58c0f0b0..5058c161 100644 --- a/Build/lib/is-domain-alive.ts +++ b/Build/lib/is-domain-alive.ts @@ -107,7 +107,7 @@ const resolve = createResolve(dohServers); const domesticResolve = createResolve(domesticDohServers); async function getWhois(domain: string) { - return asyncRetry(() => whoiser.domain(domain), { retries: 5 }); + return asyncRetry(() => whoiser.domain(domain, { raw: true }), { retries: 5 }); } const domainAliveMap = new Map(); @@ -206,7 +206,7 @@ async function isApexDomainAlive(apexDomain: string): Promise<[string, boolean]> try { whois = await getWhois(apexDomain); } catch (e) { - console.log(picocolors.red('[domain dead]'), 'whois error', { domain: apexDomain }, e); + console.log(picocolors.red('[whois error]'), { domain: apexDomain }, e); return onDomainAlive(apexDomain); } @@ -241,7 +241,8 @@ const whoisNotFoundKeywordTest = createKeywordFilter([ 'status: available', ' is free', 'no object found', - 'nothing found' + 'nothing found', + 'status: free' ]); // whois server can redirect, so whoiser might/will get info from multiple whois servers @@ -254,53 +255,71 @@ export function noWhois(whois: whoiser.WhoisSearchResult): null | string { if (Object.hasOwn(whois, key)) { empty = false; - if (key === 'error') { - // if ( - // (typeof whois.error === 'string' && whois.error) - // || (Array.isArray(whois.error) && whois.error.length > 0) - // ) { - // console.error(whois); - // return true; - // } - continue; - } + // if (key === 'error') { + // // if ( + // // (typeof whois.error === 'string' && whois.error) + // // || (Array.isArray(whois.error) && whois.error.length > 0) + // // ) { + // // console.error(whois); + // // return true; + // // } + // continue; + // } - if (key === 'text') { - if (Array.isArray(whois.text)) { - for (const value of whois.text) { - if (whoisNotFoundKeywordTest(value.toLowerCase())) { - return value; - } + // if (key === 'text') { + // if (Array.isArray(whois.text)) { + // for (const value of whois.text) { + // if (whoisNotFoundKeywordTest(value.toLowerCase())) { + // return value; + // } + // } + // } + // continue; + // } + // if (key === 'Name Server') { + // // if (Array.isArray(whois[key]) && whois[key].length === 0) { + // // return false; + // // } + // continue; + // } + + // if (key === 'Domain Status') { + // if (Array.isArray(whois[key])) { + // for (const status of whois[key]) { + // if (status === 'free' || status === 'AVAILABLE') { + // return key + ': ' + status; + // } + // if (whoisNotFoundKeywordTest(status.toLowerCase())) { + // return key + ': ' + status; + // } + // } + // } + + // continue; + // } + + // if (typeof whois[key] === 'string' && whois[key]) { + // if (whoisNotFoundKeywordTest(whois[key].toLowerCase())) { + // return key + ': ' + whois[key]; + // } + + // continue; + // } + + if (key === '__raw' && typeof whois.__raw === 'string') { + const lines = whois.__raw.trim().toLowerCase().replaceAll(/[\t ]+/g, ' ').split(/\r?\n/); + + if (process.env.DEBUG) { + console.log({ lines }); + } + + for (const line of lines) { + if (whoisNotFoundKeywordTest(line)) { + return line; } } continue; } - if (key === 'Name Server') { - // if (Array.isArray(whois[key]) && whois[key].length === 0) { - // return false; - // } - continue; - } - - if (key === 'Domain Status') { - if (Array.isArray(whois[key])) { - for (const status of whois[key]) { - if (whoisNotFoundKeywordTest(status.toLowerCase())) { - return key + ': ' + status; - } - } - } - - continue; - } - - if (typeof whois[key] === 'string' && whois[key]) { - if (whoisNotFoundKeywordTest(whois[key].toLowerCase())) { - return key + ': ' + whois[key]; - } - - continue; - } if (typeof whois[key] === 'object' && !Array.isArray(whois[key])) { const tmp = noWhois(whois[key]);