mirror of
https://github.com/SukkaW/Surge.git
synced 2025-12-12 01:00:34 +08:00
Chore: update domain alive check [skip ci]
This commit is contained in:
SukkaW
parent
49b50a33f5
commit
cf1f657dae
2
.github/workflows/check-source-domain.yml
vendored
2
.github/workflows/check-source-domain.yml
vendored
@ -50,6 +50,8 @@ jobs:
|
||||
${{ runner.os }}-v3-
|
||||
- run: pnpm install
|
||||
- run: pnpm run node Build/validate-domain-alive.ts
|
||||
env:
|
||||
DEBUG: domain-alive:*
|
||||
- name: Cache cache.db
|
||||
if: always()
|
||||
uses: actions/cache/save@v4
|
||||
|
||||
@ -36,8 +36,8 @@ export const getTelegramCIDRPromise = once(async () => {
|
||||
|
||||
// Backup IP Source 1 (DoH)
|
||||
await Promise.all([
|
||||
DNS2.DOHClient({ dns: '8.8.8.8' }),
|
||||
DNS2.DOHClient({ dns: '1.0.0.1' })
|
||||
DNS2.DOHClient({ dns: 'https://8.8.4.4/dns-query?dns={query}' }),
|
||||
DNS2.DOHClient({ dns: 'https://1.0.0.1/dns-query?dns={query}' })
|
||||
].flatMap(
|
||||
(client) => [
|
||||
'apv3.stel.com', // prod
|
||||
|
||||
@ -1,16 +0,0 @@
|
||||
import { describe, it } from 'mocha';
|
||||
|
||||
import { isDomainAlive } from './is-domain-alive';
|
||||
import { expect } from 'expect';
|
||||
|
||||
describe('isDomainAlive', function () {
|
||||
this.timeout(10000);
|
||||
|
||||
it('samsungcloudsolution.net', async () => {
|
||||
expect((await isDomainAlive('samsungcloudsolution.net', true))).toEqual(false);
|
||||
});
|
||||
|
||||
it('ecdasoin.it', async () => {
|
||||
expect((await isDomainAlive('.ecdasoin.it', true))).toEqual(false);
|
||||
});
|
||||
});
|
||||
@ -1,24 +1,6 @@
|
||||
import DNS2 from 'dns2';
|
||||
import asyncRetry from 'async-retry';
|
||||
import picocolors from 'picocolors';
|
||||
import { looseTldtsOpt } from '../constants/loose-tldts-opt';
|
||||
import { createKeyedAsyncMutex } from './keyed-async-mutex';
|
||||
import tldts from 'tldts-experimental';
|
||||
import * as whoiser from 'whoiser';
|
||||
import process from 'node:process';
|
||||
import { createRetrieKeywordFilter as createKeywordFilter } from 'foxts/retrie';
|
||||
import { shuffleArray } from 'foxts/shuffle-array';
|
||||
import { createDomainAliveChecker } from 'domain-alive';
|
||||
|
||||
const domainAliveMap = new Map<string, boolean>();
|
||||
|
||||
class DnsError extends Error {
|
||||
name = 'DnsError';
|
||||
constructor(readonly message: string, public readonly server: string) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
||||
const dohServers: Array<[string, DNS2.DnsResolver]> = ([
|
||||
const dnsServers = [
|
||||
'8.8.8.8',
|
||||
'8.8.4.4',
|
||||
'1.0.0.1',
|
||||
@ -53,316 +35,12 @@ const dohServers: Array<[string, DNS2.DnsResolver]> = ([
|
||||
// '198.54.117.10' // NameCheap DNS, supports DoT, DoH, UDP53
|
||||
// 'ada.openbld.net',
|
||||
// 'dns.rabbitdns.org'
|
||||
] as const).map(dns => [
|
||||
dns,
|
||||
DNS2.DOHClient({ dns })
|
||||
] as const);
|
||||
].map(dns => 'https://' + dns);
|
||||
|
||||
const domesticDohServers: Array<[string, DNS2.DnsResolver]> = ([
|
||||
'223.5.5.5',
|
||||
'223.6.6.6',
|
||||
'120.53.53.53',
|
||||
'1.12.12.12'
|
||||
] as const).map(dns => [
|
||||
dns,
|
||||
DNS2.DOHClient({ dns })
|
||||
] as const);
|
||||
console.log({ dnsServers });
|
||||
|
||||
const domainAliveMutex = createKeyedAsyncMutex('isDomainAlive');
|
||||
|
||||
export async function isDomainAlive(
|
||||
domain: string,
|
||||
// we dont need to check domain[0] here, this is only from runAgainstSourceFile
|
||||
isIncludeAllSubdomain: boolean
|
||||
): Promise<boolean> {
|
||||
if (domainAliveMap.has(domain)) {
|
||||
return domainAliveMap.get(domain)!;
|
||||
export const isDomainAlive = createDomainAliveChecker({
|
||||
dns: {
|
||||
dnsServers
|
||||
}
|
||||
const apexDomain = tldts.getDomain(domain, looseTldtsOpt);
|
||||
if (!apexDomain) {
|
||||
// console.log(picocolors.gray('[domain invalid]'), picocolors.gray('no apex domain'), { domain });
|
||||
domainAliveMap.set('.' + domain, true);
|
||||
return true;
|
||||
}
|
||||
|
||||
const apexDomainAlive = await isApexDomainAlive(apexDomain);
|
||||
if (isIncludeAllSubdomain || domain.length > apexDomain.length) {
|
||||
return apexDomainAlive;
|
||||
}
|
||||
if (!apexDomainAlive) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return domainAliveMutex.acquire(domain, async () => {
|
||||
domain = domain[0] === '.' ? domain.slice(1) : domain;
|
||||
|
||||
const aDns: string[] = [];
|
||||
const aaaaDns: string[] = [];
|
||||
|
||||
// test 2 times before make sure record is empty
|
||||
const servers = shuffleArray(dohServers, { copy: true });
|
||||
|
||||
for (let i = 0, len = servers.length; i < len; i++) {
|
||||
try {
|
||||
// eslint-disable-next-line no-await-in-loop -- sequential
|
||||
const aRecords = (await $resolve(domain, 'A', servers[i]));
|
||||
if (aRecords.answers.length > 0) {
|
||||
domainAliveMap.set(domain, true);
|
||||
return true;
|
||||
}
|
||||
|
||||
aDns.push(servers[i][0]);
|
||||
} catch {}
|
||||
|
||||
if (aDns.length >= 2) {
|
||||
break; // we only need to test 2 times
|
||||
}
|
||||
}
|
||||
|
||||
for (let i = 0, len = servers.length; i < len; i++) {
|
||||
try {
|
||||
// eslint-disable-next-line no-await-in-loop -- sequential
|
||||
const aaaaRecords = await $resolve(domain, 'AAAA', servers[i]);
|
||||
if (aaaaRecords.answers.length > 0) {
|
||||
domainAliveMap.set(domain, true);
|
||||
return true;
|
||||
}
|
||||
|
||||
aaaaDns.push(servers[i][0]);
|
||||
} catch {}
|
||||
|
||||
if (aaaaDns.length >= 2) {
|
||||
break; // we only need to test 2 times
|
||||
}
|
||||
}
|
||||
|
||||
// only then, let's test twice with domesticDohServers
|
||||
const domesticServers = shuffleArray(domesticDohServers, { copy: true });
|
||||
for (let i = 0, len = domesticServers.length; i < len; i++) {
|
||||
try {
|
||||
// eslint-disable-next-line no-await-in-loop -- sequential
|
||||
const aRecords = await $resolve(domain, 'A', domesticServers[i]);
|
||||
if (aRecords.answers.length > 0) {
|
||||
domainAliveMap.set(domain, true);
|
||||
return true;
|
||||
}
|
||||
aDns.push(domesticServers[i][0]);
|
||||
} catch {}
|
||||
if (aDns.length >= 2) {
|
||||
break; // we only need to test 2 times
|
||||
}
|
||||
}
|
||||
|
||||
for (let i = 0, len = domesticServers.length; i < len; i++) {
|
||||
try {
|
||||
// eslint-disable-next-line no-await-in-loop -- sequential
|
||||
const aaaaRecords = await $resolve(domain, 'AAAA', domesticServers[i]);
|
||||
if (aaaaRecords.answers.length > 0) {
|
||||
domainAliveMap.set(domain, true);
|
||||
return true;
|
||||
}
|
||||
aaaaDns.push(domesticServers[i][0]);
|
||||
} catch {}
|
||||
|
||||
if (aaaaDns.length >= 2) {
|
||||
break; // we only need to test 2 times
|
||||
}
|
||||
}
|
||||
|
||||
console.log(picocolors.red('[domain dead]'), 'no A/AAAA records', { domain, a: aDns, aaaa: aaaaDns });
|
||||
|
||||
domainAliveMap.set(domain, false);
|
||||
return false;
|
||||
});
|
||||
}
|
||||
|
||||
const apexDomainMap = createKeyedAsyncMutex('isApexDomainAlive');
|
||||
|
||||
function isApexDomainAlive(apexDomain: string) {
|
||||
if (domainAliveMap.has(apexDomain)) {
|
||||
return domainAliveMap.get(apexDomain)!;
|
||||
}
|
||||
|
||||
return apexDomainMap.acquire(apexDomain, async () => {
|
||||
const servers = shuffleArray(dohServers, { copy: true });
|
||||
|
||||
let nsSuccess = 0;
|
||||
|
||||
for (let i = 0, len = servers.length; i < len; i++) {
|
||||
const server = servers[i];
|
||||
try {
|
||||
// eslint-disable-next-line no-await-in-loop -- one by one
|
||||
const resp = await $resolve(apexDomain, 'NS', server);
|
||||
if (resp.answers.length > 0) {
|
||||
domainAliveMap.set(apexDomain, true);
|
||||
return true;
|
||||
}
|
||||
|
||||
nsSuccess++;
|
||||
|
||||
if (nsSuccess >= 2) {
|
||||
// we only need to test 2 times
|
||||
break;
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
|
||||
let whois;
|
||||
try {
|
||||
whois = await getWhois(apexDomain);
|
||||
} catch (e) {
|
||||
console.log(picocolors.red('[whois error]'), { domain: apexDomain }, e);
|
||||
domainAliveMap.set(apexDomain, true);
|
||||
return true;
|
||||
}
|
||||
|
||||
const whoisError = noWhois(whois);
|
||||
if (!whoisError) {
|
||||
console.log(picocolors.gray('[domain alive]'), picocolors.gray('whois found'), { domain: apexDomain });
|
||||
domainAliveMap.set(apexDomain, true);
|
||||
return true;
|
||||
}
|
||||
|
||||
console.log(picocolors.red('[domain dead]'), 'whois not found', { domain: apexDomain, err: whoisError });
|
||||
|
||||
domainAliveMap.set(apexDomain, false);
|
||||
return false;
|
||||
});
|
||||
}
|
||||
|
||||
async function $resolve(name: string, type: DNS2.PacketQuestion, server: [string, DNS2.DnsResolver]) {
|
||||
try {
|
||||
return await asyncRetry(async () => {
|
||||
const [dohServer, dohClient] = server;
|
||||
|
||||
try {
|
||||
return await dohClient(name, type);
|
||||
} catch (e) {
|
||||
// console.error(e);
|
||||
throw new DnsError((e as Error).message, dohServer);
|
||||
}
|
||||
}, { retries: 5 });
|
||||
} catch (e) {
|
||||
console.log('[doh error]', name, type, e);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
async function getWhois(domain: string) {
|
||||
return asyncRetry(() => whoiser.domain(domain, { raw: true }), { retries: 5 });
|
||||
}
|
||||
|
||||
// TODO: this is a workaround for https://github.com/LayeredStudio/whoiser/issues/117
|
||||
const whoisNotFoundKeywordTest = createKeywordFilter([
|
||||
'no match for',
|
||||
'does not exist',
|
||||
'not found',
|
||||
'no found',
|
||||
'no entries',
|
||||
'no data found',
|
||||
'is available for registration',
|
||||
'currently available for application',
|
||||
'no matching record',
|
||||
'no information available about domain name',
|
||||
'not been registered',
|
||||
'no match!!',
|
||||
'status: available',
|
||||
' is free',
|
||||
'no object found',
|
||||
'nothing found',
|
||||
'status: free',
|
||||
// 'pendingdelete',
|
||||
' has been blocked by '
|
||||
]);
|
||||
// whois server can redirect, so whoiser might/will get info from multiple whois servers
|
||||
// some servers (like TLD whois servers) might have cached/outdated results
|
||||
// we can only make sure a domain is alive once all response from all whois servers demonstrate so
|
||||
function noWhois(whois: whoiser.WhoisSearchResult): null | string {
|
||||
let empty = true;
|
||||
|
||||
for (const key in whois) {
|
||||
if (Object.hasOwn(whois, key)) {
|
||||
empty = false;
|
||||
|
||||
// if (key === 'error') {
|
||||
// // if (
|
||||
// // (typeof whois.error === 'string' && whois.error)
|
||||
// // || (Array.isArray(whois.error) && whois.error.length > 0)
|
||||
// // ) {
|
||||
// // console.error(whois);
|
||||
// // return true;
|
||||
// // }
|
||||
// continue;
|
||||
// }
|
||||
|
||||
// if (key === 'text') {
|
||||
// if (Array.isArray(whois.text)) {
|
||||
// for (const value of whois.text) {
|
||||
// if (whoisNotFoundKeywordTest(value.toLowerCase())) {
|
||||
// return value;
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// continue;
|
||||
// }
|
||||
// if (key === 'Name Server') {
|
||||
// // if (Array.isArray(whois[key]) && whois[key].length === 0) {
|
||||
// // return false;
|
||||
// // }
|
||||
// continue;
|
||||
// }
|
||||
|
||||
// if (key === 'Domain Status') {
|
||||
// if (Array.isArray(whois[key])) {
|
||||
// for (const status of whois[key]) {
|
||||
// if (status === 'free' || status === 'AVAILABLE') {
|
||||
// return key + ': ' + status;
|
||||
// }
|
||||
// if (whoisNotFoundKeywordTest(status.toLowerCase())) {
|
||||
// return key + ': ' + status;
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
// continue;
|
||||
// }
|
||||
|
||||
// if (typeof whois[key] === 'string' && whois[key]) {
|
||||
// if (whoisNotFoundKeywordTest(whois[key].toLowerCase())) {
|
||||
// return key + ': ' + whois[key];
|
||||
// }
|
||||
|
||||
// continue;
|
||||
// }
|
||||
|
||||
if (key === '__raw' && typeof whois.__raw === 'string') {
|
||||
const lines = whois.__raw.trim().toLowerCase().replaceAll(/[\t ]+/g, ' ').split(/\r?\n/);
|
||||
|
||||
if (process.env.DEBUG) {
|
||||
console.log({ lines });
|
||||
}
|
||||
|
||||
for (const line of lines) {
|
||||
if (whoisNotFoundKeywordTest(line)) {
|
||||
return line;
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (typeof whois[key] === 'object' && !Array.isArray(whois[key])) {
|
||||
const tmp = noWhois(whois[key]);
|
||||
if (tmp) {
|
||||
return tmp;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (empty) {
|
||||
return 'whois is empty';
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
});
|
||||
|
||||
@ -1,23 +0,0 @@
|
||||
const globalMap = new Map<string, Map<string, Promise<unknown>>>();
|
||||
|
||||
export function createKeyedAsyncMutex(globalNamespaceKey: string) {
|
||||
let map;
|
||||
if (globalMap.has(globalNamespaceKey)) {
|
||||
map = globalMap.get(globalNamespaceKey)!;
|
||||
} else {
|
||||
map = new Map();
|
||||
globalMap.set(globalNamespaceKey, map);
|
||||
}
|
||||
|
||||
return {
|
||||
async acquire<T = unknown>(key: string, fn: () => Promise<T>) {
|
||||
if (map.has(key)) {
|
||||
return map.get(key);
|
||||
}
|
||||
|
||||
const promise = fn();
|
||||
map.set(key, promise);
|
||||
return promise;
|
||||
}
|
||||
};
|
||||
}
|
||||
@ -44,13 +44,14 @@ const deadDomains: string[] = [];
|
||||
bar.setTotal(bar.getTotal() + 1);
|
||||
|
||||
return queue.add(
|
||||
() => isDomainAlive(domain, includeAllSubdomain).then((alive) => {
|
||||
() => isDomainAlive(domain).then(({ alive, registerableDomainAlive, registerableDomain }) => {
|
||||
bar.increment();
|
||||
|
||||
if (alive) {
|
||||
return;
|
||||
if (!registerableDomainAlive) {
|
||||
deadDomains.push('.' + registerableDomain);
|
||||
} else if (!alive) {
|
||||
deadDomains.push(includeAllSubdomain ? '.' + domain : domain);
|
||||
}
|
||||
deadDomains.push(includeAllSubdomain ? '.' + domain : domain);
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
18
package.json
18
package.json
@ -22,17 +22,17 @@
|
||||
"@ghostery/adblocker": "^2.11.5",
|
||||
"@henrygd/queue": "^1.0.7",
|
||||
"@mitata/counters": "^0.0.8",
|
||||
"async-retry": "^1.3.3",
|
||||
"better-sqlite3": "^12.2.0",
|
||||
"ci-info": "^4.3.0",
|
||||
"cli-progress": "^3.12.0",
|
||||
"csv-parse": "^6.1.0",
|
||||
"dns2": "^2.1.0",
|
||||
"dns2": "github:lsongdev/node-dns#e4fa035aca0b8eb730bde3431fbf0c60a31a09c9",
|
||||
"domain-alive": "^0.1.5",
|
||||
"fast-cidr-tools": "^0.3.2",
|
||||
"fast-fifo": "^1.3.2",
|
||||
"fast-uri": "^3.0.6",
|
||||
"fdir": "^6.5.0",
|
||||
"foxts": "^3.11.1",
|
||||
"foxts": "^3.12.0",
|
||||
"hash-wasm": "^4.12.0",
|
||||
"json-stringify-pretty-compact": "3.0.0",
|
||||
"null-prototype-object": "^1.2.2",
|
||||
@ -44,7 +44,6 @@
|
||||
"tldts-experimental": "^6.1.86",
|
||||
"undici": "^7.14.0",
|
||||
"undici-cache-store-better-sqlite3": "^1.0.0",
|
||||
"whoiser": "^1.18.0",
|
||||
"why-is-node-running": "^3.2.2",
|
||||
"worktank": "^3.0.2",
|
||||
"xbits": "^0.2.0",
|
||||
@ -55,7 +54,6 @@
|
||||
"@eslint-sukka/node": "^6.23.1",
|
||||
"@swc-node/register": "^1.11.1",
|
||||
"@swc/core": "^1.13.4",
|
||||
"@types/async-retry": "^1.4.9",
|
||||
"@types/better-sqlite3": "^7.6.13",
|
||||
"@types/cli-progress": "^3.11.6",
|
||||
"@types/dns2": "^2.0.10",
|
||||
@ -76,9 +74,6 @@
|
||||
},
|
||||
"packageManager": "pnpm@10.15.0",
|
||||
"pnpm": {
|
||||
"patchedDependencies": {
|
||||
"whoiser": "patches/whoiser.patch"
|
||||
},
|
||||
"onlyBuiltDependencies": [
|
||||
"@swc/core",
|
||||
"better-sqlite3",
|
||||
@ -89,6 +84,11 @@
|
||||
"globalthis": "npm:@nolyfill/globalthis@^1.0.44",
|
||||
"has": "npm:@nolyfill/has@^1.0.44",
|
||||
"safe-buffer": "npm:@nolyfill/safe-buffer@^1.0.44"
|
||||
}
|
||||
},
|
||||
"ignoredBuiltDependencies": [
|
||||
"bufferutil",
|
||||
"es5-ext",
|
||||
"utf-8-validate"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,13 +0,0 @@
|
||||
diff --git a/src/whoiser.js b/src/whoiser.js
|
||||
index ff42b8c7fe1749d389df2d420f68f1ec6590fe69..dea40e123c8bab3c38c1e5d41b6da3bff43acbfe 100644
|
||||
--- a/src/whoiser.js
|
||||
+++ b/src/whoiser.js
|
||||
@@ -50,6 +50,8 @@ let cacheTldWhoisServer = {
|
||||
shop: 'whois.nic.shop',
|
||||
site: 'whois.nic.site',
|
||||
xyz: 'whois.nic.xyz',
|
||||
+
|
||||
+ ga: 'whois.nic.ga'
|
||||
}
|
||||
|
||||
// misspelled whois servers..
|
||||
485
pnpm-lock.yaml
generated
485
pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
Loading…
x
Reference in New Issue
Block a user