GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2025-12-12 17:20:35 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Chore: update domain alive check

Browse Source
This commit is contained in:
SukkaW 2025-05-16 23:25:09 +08:00
parent bcf6244e1e
commit 803e503a1e
4 changed files with 165 additions and 203 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
Build/lib/is-domain-alive.test.ts
View File

@ -3,45 +3,14 @@ import { describe, it } from 'mocha';
import { isDomainAlive } from './is-domain-alive'; import { isDomainAlive } from './is-domain-alive';
import { expect } from 'expect'; import { expect } from 'expect';
import process from 'node:process';
describe('isDomainAlive', function () { describe('isDomainAlive', function () {
this.timeout(10000); this.timeout(10000);
// it('.cryptocrawler.io', async () => { it('samsungcloudsolution.net', async () => {
// expect((await isDomainAlive('.cryptocrawler.io', true))[1]).toEqual(false); expect((await isDomainAlive('samsungcloudsolution.net', true))).toEqual(false);
// }); });
// it('.tunevideo.ru', async () => {
// expect((await isDomainAlive('.tunevideo.ru', true))[1]).toEqual(false);
// });
// it('.myqloud.com', async () => {
// expect((await isDomainAlive('.myqloud.com', true))[1]).toEqual(true);
// });
// it('discount-deal.org', async () => {
// expect((await isDomainAlive('discount-deal.org', false))[1]).toEqual(false);
// });
// it('ithome.com.tw', async () => {
// expect((await isDomainAlive('ithome.com.tw', false))[1]).toEqual(true);
// });
// it('flipkart.com', async () => {
// expect((await isDomainAlive('flipkart.com', false))[1]).toEqual(true);
// });
// it('lzzyimg.com', async () => {
// expect((await isDomainAlive('.lzzyimg.com', true))[1]).toEqual(true);
// });
// it('tayfundogdas.me', async () => {
// expect((await isDomainAlive('.tayfundogdas.me', true))[1]).toEqual(true);
// });
it('ecdasoin.it', async () => { it('ecdasoin.it', async () => {
process.env.DEBUG = 'true'; expect((await isDomainAlive('.ecdasoin.it', true))).toEqual(false);
expect((await isDomainAlive('.ecdasoin.it', true))[1]).toEqual(false);
}); });
}); });

282
Build/lib/is-domain-alive.ts
View File

@ -1,25 +1,15 @@
import tldts from 'tldts-experimental';
import { looseTldtsOpt } from '../constants/loose-tldts-opt';
import picocolors from 'picocolors';
import { pickRandom, pickOne } from 'foxts/pick-random';
import DNS2 from 'dns2'; import DNS2 from 'dns2';
import asyncRetry from 'async-retry'; import asyncRetry from 'async-retry';
import picocolors from 'picocolors';
import { looseTldtsOpt } from '../constants/loose-tldts-opt';
import { createKeyedAsyncMutex } from './keyed-async-mutex';
import { pickRandom, pickOne } from 'foxts/pick-random';
import tldts from 'tldts-experimental';
import * as whoiser from 'whoiser'; import * as whoiser from 'whoiser';
import process from 'node:process';
import { createRetrieKeywordFilter as createKeywordFilter } from 'foxts/retrie'; import { createRetrieKeywordFilter as createKeywordFilter } from 'foxts/retrie';
import process from 'node:process'; const domainAliveMap = new Map<string, boolean>();
const mutex = new Map<string, Promise<unknown>>();
export function keyedAsyncMutexWithQueue<T>(key: string, fn: () => Promise<T>) {
if (mutex.has(key)) {
return mutex.get(key) as Promise<T>;
}
const promise = fn();
mutex.set(key, promise);
return promise;
}
class DnsError extends Error { class DnsError extends Error {
name = 'DnsError'; name = 'DnsError';
@ -88,6 +78,127 @@ const domesticDohServers: Array<[string, DNS2.DnsResolver]> = ([
}) })
] as const); ] as const);
const domainAliveMutex = createKeyedAsyncMutex('isDomainAlive');
export async function isDomainAlive(domain: string, isIncludeAllSubdomain: boolean = domain[0] === '.'): Promise<boolean> {
if (domainAliveMap.has(domain)) {
return domainAliveMap.get(domain)!;
}
const apexDomain = tldts.getDomain(domain, looseTldtsOpt);
if (!apexDomain) {
console.log(picocolors.gray('[domain invalid]'), picocolors.gray('no apex domain'), { domain });
domainAliveMap.set('.' + domain, true);
return true;
}
const apexDomainAlive = await isApexDomainAlive(apexDomain);
if (isIncludeAllSubdomain || domain.length > apexDomain.length) {
return apexDomainAlive;
}
if (!apexDomainAlive) {
return false;
}
return domainAliveMutex.acquire(domain, async () => {
domain = domain[0] === '.' ? domain.slice(1) : domain;
const $domain = isIncludeAllSubdomain ? '.' + domain : domain;
const aDns: string[] = [];
const aaaaDns: string[] = [];
// test 2 times before make sure record is empty
const servers = pickRandom(dohServers, 2);
for (let i = 0; i < 2; i++) {
// eslint-disable-next-line no-await-in-loop -- sequential
const aRecords = (await $resolve(domain, 'A', servers[i]));
if (aRecords.answers.length > 0) {
domainAliveMap.set($domain, true);
return true;
}
aDns.push(aRecords.dns);
}
for (let i = 0; i < 2; i++) {
// eslint-disable-next-line no-await-in-loop -- sequential
const aaaaRecords = (await $resolve(domain, 'AAAA', servers[i]));
if (aaaaRecords.answers.length > 0) {
domainAliveMap.set($domain, true);
return true;
}
aaaaDns.push(aaaaRecords.dns);
}
// only then, let's test twice with domesticDohServers
for (let i = 0; i < 2; i++) {
// eslint-disable-next-line no-await-in-loop -- sequential
const aRecords = (await $resolve(domain, 'A', pickOne(domesticDohServers)));
if (aRecords.answers.length > 0) {
domainAliveMap.set($domain, true);
return true;
}
aDns.push(aRecords.dns);
}
for (let i = 0; i < 2; i++) {
// eslint-disable-next-line no-await-in-loop -- sequential
const aaaaRecords = (await $resolve(domain, 'AAAA', pickOne(domesticDohServers)));
if (aaaaRecords.answers.length > 0) {
domainAliveMap.set($domain, true);
return true;
}
aaaaDns.push(aaaaRecords.dns);
}
console.log(picocolors.red('[domain dead]'), 'no A/AAAA records', { domain, a: aDns, aaaa: aaaaDns });
domainAliveMap.set($domain, false);
return false;
});
}
const apexDomainMap = createKeyedAsyncMutex('isApexDomainAlive');
function isApexDomainAlive(apexDomain: string) {
if (domainAliveMap.has(apexDomain)) {
return domainAliveMap.get(apexDomain)!;
}
return apexDomainMap.acquire(apexDomain, async () => {
const servers = pickRandom(dohServers, 2);
for (let i = 0, len = servers.length; i < len; i++) {
const server = servers[i];
// eslint-disable-next-line no-await-in-loop -- one by one
const resp = await $resolve(apexDomain, 'NS', server);
if (resp.answers.length > 0) {
domainAliveMap.set(apexDomain, true);
return true;
}
}
let whois;
try {
whois = await getWhois(apexDomain);
} catch (e) {
console.log(picocolors.red('[whois error]'), { domain: apexDomain }, e);
domainAliveMap.set(apexDomain, true);
return true;
}
const whoisError = noWhois(whois);
if (!whoisError) {
console.log(picocolors.gray('[domain alive]'), picocolors.gray('whois found'), { domain: apexDomain });
domainAliveMap.set(apexDomain, true);
return true;
}
console.log(picocolors.red('[domain dead]'), 'whois not found', { domain: apexDomain, err: whoisError });
domainAliveMap.set(apexDomain, false);
return false;
});
}
async function $resolve(name: string, type: DNS2.PacketQuestion, server: [string, DNS2.DnsResolver]) { async function $resolve(name: string, type: DNS2.PacketQuestion, server: [string, DNS2.DnsResolver]) {
try { try {
return await asyncRetry(async () => { return await asyncRetry(async () => {
@ -113,140 +224,6 @@ async function getWhois(domain: string) {
return asyncRetry(() => whoiser.domain(domain, { raw: true }), { retries: 5 }); return asyncRetry(() => whoiser.domain(domain, { raw: true }), { retries: 5 });
} }
const domainAliveMap = new Map<string, boolean>();
function onDomainAlive(domain: string): [string, boolean] {
domainAliveMap.set(domain, true);
return [domain, true];
}
function onDomainDead(domain: string): [string, boolean] {
domainAliveMap.set(domain, false);
return [domain, false];
}
export async function isDomainAlive(domain: string, isSuffix: boolean): Promise<[string, boolean]> {
if (domainAliveMap.has(domain)) {
return [domain, domainAliveMap.get(domain)!];
}
const apexDomain = tldts.getDomain(domain, looseTldtsOpt);
if (!apexDomain) {
console.log(picocolors.gray('[domain invalid]'), picocolors.gray('no apex domain'), { domain });
return onDomainAlive(domain);
}
const apexDomainAlive = await keyedAsyncMutexWithQueue(apexDomain, () => isApexDomainAlive(apexDomain));
if (isSuffix) {
return apexDomainAlive;
}
if (!apexDomainAlive[1]) {
return apexDomainAlive;
}
const $domain = domain[0] === '.' ? domain.slice(1) : domain;
const aDns: string[] = [];
const aaaaDns: string[] = [];
// test 2 times before make sure record is empty
const servers = pickRandom(dohServers, 2);
for (let i = 0; i < 2; i++) {
// eslint-disable-next-line no-await-in-loop -- sequential
const aRecords = (await $resolve($domain, 'A', servers[i]));
if (aRecords.answers.length > 0) {
return onDomainAlive(domain);
}
aDns.push(aRecords.dns);
}
for (let i = 0; i < 2; i++) {
// eslint-disable-next-line no-await-in-loop -- sequential
const aaaaRecords = (await $resolve($domain, 'AAAA', servers[i]));
if (aaaaRecords.answers.length > 0) {
return onDomainAlive(domain);
}
aaaaDns.push(aaaaRecords.dns);
}
// only then, let's test twice with domesticDohServers
for (let i = 0; i < 2; i++) {
// eslint-disable-next-line no-await-in-loop -- sequential
const aRecords = (await $resolve($domain, 'A', pickOne(domesticDohServers)));
if (aRecords.answers.length > 0) {
return onDomainAlive(domain);
}
aDns.push(aRecords.dns);
}
for (let i = 0; i < 2; i++) {
// eslint-disable-next-line no-await-in-loop -- sequential
const aaaaRecords = (await $resolve($domain, 'AAAA', pickOne(domesticDohServers)));
if (aaaaRecords.answers.length > 0) {
return onDomainAlive(domain);
}
aaaaDns.push(aaaaRecords.dns);
}
console.log(picocolors.red('[domain dead]'), 'no A/AAAA records', { domain, a: aDns, aaaa: aaaaDns });
return onDomainDead($domain);
}
const apexDomainNsResolvePromiseMap = new Map<string, Promise<boolean>>();
async function getNS(domain: string) {
const servers = pickRandom(dohServers, 2);
for (let i = 0, len = servers.length; i < len; i++) {
const server = servers[i];
// eslint-disable-next-line no-await-in-loop -- one by one
const resp = await $resolve(domain, 'NS', server);
if (resp.answers.length > 0) {
return true;
}
}
return false;
}
async function isApexDomainAlive(apexDomain: string): Promise<[string, boolean]> {
if (domainAliveMap.has(apexDomain)) {
return [apexDomain, domainAliveMap.get(apexDomain)!];
}
let hasNS: boolean;
if (apexDomainNsResolvePromiseMap.has(apexDomain)) {
hasNS = await apexDomainNsResolvePromiseMap.get(apexDomain)!;
} else {
const promise = getNS(apexDomain);
apexDomainNsResolvePromiseMap.set(apexDomain, promise);
hasNS = await promise;
}
if (hasNS) {
return onDomainAlive(apexDomain);
}
let whois;
try {
whois = await getWhois(apexDomain);
} catch (e) {
console.log(picocolors.red('[whois error]'), { domain: apexDomain }, e);
return onDomainAlive(apexDomain);
}
if (process.env.DEBUG) {
console.log(JSON.stringify(whois, null, 2));
}
const whoisError = noWhois(whois);
if (!whoisError) {
console.log(picocolors.gray('[domain alive]'), picocolors.gray('whois found'), { domain: apexDomain });
return onDomainAlive(apexDomain);
}
console.log(picocolors.red('[domain dead]'), 'whois not found', { domain: apexDomain, err: whoisError });
return onDomainDead('.' + apexDomain);
}
// TODO: this is a workaround for https://github.com/LayeredStudio/whoiser/issues/117 // TODO: this is a workaround for https://github.com/LayeredStudio/whoiser/issues/117
const whoisNotFoundKeywordTest = createKeywordFilter([ const whoisNotFoundKeywordTest = createKeywordFilter([
'no match for', 'no match for',
@ -269,11 +246,10 @@ const whoisNotFoundKeywordTest = createKeywordFilter([
// 'pendingdelete', // 'pendingdelete',
' has been blocked by ' ' has been blocked by '
]); ]);
// whois server can redirect, so whoiser might/will get info from multiple whois servers // whois server can redirect, so whoiser might/will get info from multiple whois servers
// some servers (like TLD whois servers) might have cached/outdated results // some servers (like TLD whois servers) might have cached/outdated results
// we can only make sure a domain is alive once all response from all whois servers demonstrate so // we can only make sure a domain is alive once all response from all whois servers demonstrate so
export function noWhois(whois: whoiser.WhoisSearchResult): null | string { function noWhois(whois: whoiser.WhoisSearchResult): null | string {
let empty = true; let empty = true;
for (const key in whois) { for (const key in whois) {

23
Build/lib/keyed-async-mutex.ts Normal file
View File

@ -0,0 +1,23 @@
const globalMap = new Map<string, Map<string, Promise<unknown>>>();
export function createKeyedAsyncMutex(globalNamespaceKey: string) {
let map;
if (globalMap.has(globalNamespaceKey)) {
map = globalMap.get(globalNamespaceKey)!;
} else {
map = new Map();
globalMap.set(globalNamespaceKey, map);
}
return {
async acquire<T = unknown>(key: string, fn: () => Promise<T>) {
if (map.has(key)) {
return map.get(key);
}
const promise = fn();
map.set(key, promise);
return promise;
}
};
}

24
Build/validate-domain-alive.ts
View File

@ -1,18 +1,10 @@
import { SOURCE_DIR } from './constants/dir'; import { SOURCE_DIR } from './constants/dir';
import path from 'node:path'; import path from 'node:path';
import { newQueue } from '@henrygd/queue'; import { isDomainAlive } from './lib/is-domain-alive';
import { isDomainAlive, keyedAsyncMutexWithQueue } from './lib/is-domain-alive';
import { fdir as Fdir } from 'fdir'; import { fdir as Fdir } from 'fdir';
import runAgainstSourceFile from './lib/run-against-source-file'; import runAgainstSourceFile from './lib/run-against-source-file';
const queue = newQueue(24);
const deadDomains: string[] = []; const deadDomains: string[] = [];
function onDomain(args: [string, boolean]) {
if (!args[1]) {
deadDomains.push(args[0]);
}
}
(async () => { (async () => {
const domainSets = await new Fdir() const domainSets = await new Fdir()
@ -42,12 +34,14 @@ function onDomain(args: [string, boolean]) {
].map( ].map(
filepath => runAgainstSourceFile( filepath => runAgainstSourceFile(
filepath, filepath,
(domain: string, includeAllSubdomain: boolean) => promises.push(queue.add( (domain: string, includeAllSubdomain: boolean) => promises.push(
() => keyedAsyncMutexWithQueue( isDomainAlive(domain, includeAllSubdomain).then((alive) => {
domain, if (alive) {
() => isDomainAlive(domain, includeAllSubdomain) return;
).then(onDomain) }
)) deadDomains.push(includeAllSubdomain ? '.' + domain : domain);
})
)
).then(() => console.log('[crawl]', filepath)) ).then(() => console.log('[crawl]', filepath))
)); ));