diff --git a/.github/workflows/check-source-domain.yml b/.github/workflows/check-source-domain.yml new file mode 100644 index 00000000..5de48c7c --- /dev/null +++ b/.github/workflows/check-source-domain.yml @@ -0,0 +1,24 @@ +name: Check Domain Availability +on: + # manual trigger only + workflow_dispatch: + +jobs: + build: + name: Build + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: pnpm/action-setup@v4 + with: + run_install: false + - uses: actions/setup-node@v4 + with: + node-version-file: ".node-version" + cache: "pnpm" + - run: pnpm install + - run: pnpm run node Build/validate-domain-alive.ts diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7116010d..22b3ad1a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,16 +16,13 @@ jobs: runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v4 + - uses: actions/checkout@v4 with: persist-credentials: false - uses: pnpm/action-setup@v4 - name: Install pnpm with: run_install: false - - name: Use Node.js - uses: actions/setup-node@v4 + - uses: actions/setup-node@v4 with: node-version-file: ".node-version" cache: "pnpm" @@ -54,8 +51,7 @@ jobs: ${{ runner.os }}-v3-${{ steps.date.outputs.year }}-${{ steps.date.outputs.month }}- ${{ runner.os }}-v3-${{ steps.date.outputs.year }}- ${{ runner.os }}-v3- - - name: Install dependencies - run: pnpm install + - run: pnpm install - run: pnpm run build - name: Pre-deploy check # If the public directory doesn't exist, the build should fail. diff --git a/.gitignore b/.gitignore index 021ebd1e..096e1f75 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,4 @@ node_modules .wireit .cache public -tmp* +tmp.* diff --git a/Build/mod.d.ts b/Build/mod.d.ts new file mode 100644 index 00000000..be218af2 --- /dev/null +++ b/Build/mod.d.ts @@ -0,0 +1,22 @@ +import 'dns2'; + +declare module 'dns2' { + import DNS from 'dns2'; + + declare namespace DNS { + interface DoHClientOption { + /** @example dns.google.com */ + dns: string, + /** @description whether to use HTTP or HTTPS */ + http: boolean + } + + export type DnsResolver = (name: string, type: PacketQuestion) => Promise; + + declare function DOHClient(opt: DoHClientOption): DnsResolver; + + export type $DnsResponse = DnsResponse; + } + + export = DNS; +} diff --git a/Build/validate-domain-alive.ts b/Build/validate-domain-alive.ts new file mode 100644 index 00000000..7d7a3dad --- /dev/null +++ b/Build/validate-domain-alive.ts @@ -0,0 +1,230 @@ +import DNS2 from 'dns2'; +import { readFileByLine } from './lib/fetch-text-by-line'; +import { processLine } from './lib/process-line'; +import tldts from 'tldts'; +import { looseTldtsOpt } from './constants/loose-tldts-opt'; +import { fdir as Fdir } from 'fdir'; +import { SOURCE_DIR } from './constants/dir'; +import path from 'node:path'; +import { newQueue } from '@henrygd/queue'; +import asyncRetry from 'async-retry'; +import * as whoiser from 'whoiser'; +import picocolors from 'picocolors'; + +const dohServers: Array<[string, DNS2.DnsResolver]> = ([ + '8.8.8.8', + '8.8.4.4', + '1.0.0.1', + '1.1.1.1', + '162.159.36.1', + '162.159.46.1', + '101.101.101.101', // TWNIC + '185.222.222.222', // DNS.SB + '45.11.45.11', // DNS.SB + '9.9.9.10', // Quad9 unfiltered + '149.112.112.10', // Quad9 unfiltered + '208.67.222.2', // OpenDNS sandbox (unfiltered) + '208.67.220.2', // OpenDNS sandbox (unfiltered) + '94.140.14.140', // AdGuard unfiltered + '94.140.14.141', // AdGuard unfiltered + // '76.76.2.0', // ControlD unfiltered, path not /dns-query + // '76.76.10.0', // ControlD unfiltered, path not /dns-query + '193.110.81.0', // dns0.eu + '185.253.5.0', // dns0.eu + 'dns.nextdns.io', + 'wikimedia-dns.org', + // 'ordns.he.net', + 'dns.mullvad.net' + // 'ada.openbld.net', + // 'dns.rabbitdns.org' +] as const).map(server => [ + server, + DNS2.DOHClient({ + dns: server, + http: false + }) +] as const); + +const queue = newQueue(8); + +class DnsError extends Error { + name = 'DnsError'; + constructor(readonly message: string, public readonly server: string) { + super(message); + } +} + +interface DnsResponse extends DNS2.$DnsResponse { + dns: string +} + +const resolve: DNS2.DnsResolver = async (...args) => { + try { + return await asyncRetry(async () => { + const [dohServer, dohClient] = dohServers[Math.floor(Math.random() * dohServers.length)]; + + try { + const resp = await dohClient(...args); + return { + ...resp, + dns: dohServer + } satisfies DnsResponse; + } catch (e) { + throw new DnsError((e as Error).message, dohServer); + } + }, { retries: 5 }); + } catch (e) { + console.log('[doh error]', ...args, e); + throw e; + } +}; + +(async () => { + const domainSets = await new Fdir() + .withFullPaths() + .crawl(SOURCE_DIR + path.sep + 'domainset') + .withPromise(); + const domainRules = await new Fdir() + .withFullPaths() + .crawl(SOURCE_DIR + path.sep + 'non_ip') + .withPromise(); + + await Promise.all([ + ...domainSets.map(runAgainstDomainset), + ...domainRules.map(runAgainstRuleset) + ]); + + console.log('done'); +})(); + +const domainAliveMap = new Map(); +async function isApexDomainAlive(apexDomain: string): Promise<[string, boolean]> { + if (domainAliveMap.has(apexDomain)) { + return [apexDomain, domainAliveMap.get(apexDomain)!]; + } + + const resp = await resolve(apexDomain, 'NS'); + + if (resp.answers.length > 0) { + console.log(picocolors.green('[domain alive]'), 'NS record', apexDomain); + return [apexDomain, true]; + } + + let whois; + + try { + whois = await whoiser.domain(apexDomain); + } catch (e) { + console.log('[whois fail]', 'whois error', { domain: apexDomain }, e); + return [apexDomain, true]; + } + + if (Object.keys(whois).length > 0) { + // TODO: this is a workaround for https://github.com/LayeredStudio/whoiser/issues/117 + if ('text' in whois && (whois.text as string[]).some(value => value.includes('No match for'))) { + console.log(picocolors.red('[domain dead]'), 'whois no match', { domain: apexDomain }); + domainAliveMap.set(apexDomain, false); + return [apexDomain, false]; + } + + console.log(picocolors.green('[domain alive]'), 'recorded in whois', apexDomain); + return [apexDomain, true]; + } + + if (!('dns' in whois)) { + console.log({ whois }); + } + + console.log(picocolors.red('[domain dead]'), 'whois no match', { domain: apexDomain }); + domainAliveMap.set(apexDomain, false); + return [apexDomain, false]; +} + +const domainMutex = new Map>(); + +export async function isDomainAlive(domain: string, isSuffix: boolean): Promise<[string, boolean]> { + if (domain[0] === '.') { + domain = domain.slice(1); + } + + const apexDomain = tldts.getDomain(domain, looseTldtsOpt); + if (!apexDomain) { + console.log('[domain invalid]', 'no apex domain', { domain }); + return [domain, true] as const; + } + + let apexDomainAlivePromise; + if (domainMutex.has(domain)) { + apexDomainAlivePromise = domainMutex.get(domain)!; + } else { + apexDomainAlivePromise = queue.add(() => isApexDomainAlive(apexDomain).then(res => { + domainMutex.delete(domain); + return res; + })); + domainMutex.set(domain, apexDomainAlivePromise); + } + const apexDomainAlive = await apexDomainAlivePromise; + + if (!apexDomainAlive[1]) { + domainAliveMap.set(domain, false); + return [domain, false] as const; + } + + if (!isSuffix) { + const aRecords = (await resolve(domain, 'A')); + if (aRecords.answers.length === 0) { + const aaaaRecords = (await resolve(domain, 'AAAA')); + if (aaaaRecords.answers.length === 0) { + console.log(picocolors.red('[domain dead]'), 'no A/AAAA records', { domain, a: aRecords.dns, aaaa: aaaaRecords.dns }); + domainAliveMap.set(domain, false); + return [domain, false] as const; + } + } + } + + domainAliveMap.set(domain, true); + return [domain, true] as const; +} + +export async function runAgainstRuleset(filepath: string) { + const promises: Array> = []; + + for await (const l of readFileByLine(filepath)) { + const line = processLine(l); + if (!line) continue; + const [type, domain] = line.split(','); + switch (type) { + case 'DOMAIN-SUFFIX': + case 'DOMAIN': { + if (!domainMutex.has(domain)) { + const promise = queue.add(() => isDomainAlive(domain, type === 'DOMAIN-SUFFIX')).then(res => { + domainMutex.delete(domain); + return res; + }); + domainMutex.set(domain, promise); + promises.push(promise); + } + break; + } + // no default + // case 'DOMAIN-KEYWORD': { + // break; + // } + // no default + } + } + + return Promise.all(promises); +} + +export async function runAgainstDomainset(filepath: string) { + const promises: Array> = []; + + for await (const l of readFileByLine(filepath)) { + const line = processLine(l); + if (!line) continue; + promises.push(isDomainAlive(line, line[0] === '.')); + } + + return Promise.all(promises); +} diff --git a/Source/domainset/cdn.conf b/Source/domainset/cdn.conf index 9136a8d8..caeb3fdf 100644 --- a/Source/domainset/cdn.conf +++ b/Source/domainset/cdn.conf @@ -683,7 +683,7 @@ image.ibb.co .ax1x.com # PostImage .postimg.cc -.postimg.org +# .postimg.org - domain locked by registry since no later than Apr. 2018 (https://web.archive.org/web/20190208000038/https://www.phpbb.com/customise/db/extension/postimage/support/topic/191346) # Image Proxy images.weserv.nl # Imageshack diff --git a/package.json b/package.json index 9764ea8e..53966b0b 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,9 @@ "license": "ISC", "dependencies": { "@ghostery/adblocker": "^2.0.3", + "@henrygd/queue": "^1.0.7", "@jsdevtools/ez-spawn": "^3.0.4", + "async-retry": "^1.3.3", "async-sema": "^3.1.1", "better-sqlite3": "^11.5.0", "cacache": "^19.0.1", @@ -30,6 +32,7 @@ "cli-table3": "^0.6.5", "csv-parse": "^5.5.6", "devalue": "^5.1.1", + "dns2": "^2.1.0", "fast-cidr-tools": "^0.3.1", "fdir": "^6.4.2", "foxact": "^0.2.41", @@ -43,6 +46,7 @@ "tldts": "^6.1.58", "tldts-experimental": "^6.1.58", "undici": "6.20.1", + "whoiser": "^1.18.0", "why-is-node-running": "^3.2.1", "yaml": "^2.6.0" }, @@ -50,8 +54,10 @@ "@eslint-sukka/node": "^6.9.0", "@swc-node/register": "^1.10.9", "@swc/core": "^1.7.42", + "@types/async-retry": "^1.4.9", "@types/better-sqlite3": "^7.6.11", "@types/cacache": "^17.0.2", + "@types/dns2": "^2.0.9", "@types/make-fetch-happen": "^10.0.4", "@types/mocha": "^10.0.9", "@types/node": "^22.8.7", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 47bcd829..e3f14815 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -19,9 +19,15 @@ importers: '@ghostery/adblocker': specifier: ^2.0.3 version: 2.0.3 + '@henrygd/queue': + specifier: ^1.0.7 + version: 1.0.7 '@jsdevtools/ez-spawn': specifier: ^3.0.4 version: 3.0.4 + async-retry: + specifier: ^1.3.3 + version: 1.3.3 async-sema: specifier: ^3.1.1 version: 3.1.1 @@ -46,6 +52,9 @@ importers: devalue: specifier: ^5.1.1 version: 5.1.1 + dns2: + specifier: ^2.1.0 + version: 2.1.0 fast-cidr-tools: specifier: ^0.3.1 version: 0.3.1 @@ -85,6 +94,9 @@ importers: undici: specifier: 6.20.1 version: 6.20.1(patch_hash=yuj5uy4vvwj67xoliq5togiyme) + whoiser: + specifier: ^1.18.0 + version: 1.18.0 why-is-node-running: specifier: ^3.2.1 version: 3.2.1 @@ -101,12 +113,18 @@ importers: '@swc/core': specifier: ^1.7.42 version: 1.7.42 + '@types/async-retry': + specifier: ^1.4.9 + version: 1.4.9 '@types/better-sqlite3': specifier: ^7.6.11 version: 7.6.11 '@types/cacache': specifier: ^17.0.2 version: 17.0.2 + '@types/dns2': + specifier: ^2.0.9 + version: 2.0.9 '@types/make-fetch-happen': specifier: ^10.0.4 version: 10.0.4 @@ -246,6 +264,9 @@ packages: '@ghostery/adblocker@2.0.3': resolution: {integrity: sha512-b6sbsYzfwWeFpvOSs8VMiBc+d39xvErpLz8pxCJIyOAiDD41NgT72sDHlTNZkeYYHq2fKe4sArsjUMjCyjtI6A==} + '@henrygd/queue@1.0.7': + resolution: {integrity: sha512-Jmt/iO6yDlz9UYGILkm/Qzi/ckkEiTNZcqDvt3QFLE4OThPeiCj6tKsynHFm/ppl8RumWXAx1dZPBPiRPaaGig==} + '@humanfs/core@0.19.1': resolution: {integrity: sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==} engines: {node: '>=18.18.0'} @@ -507,6 +528,9 @@ packages: '@tybys/wasm-util@0.9.0': resolution: {integrity: sha512-6+7nlbMVX/PVDCwaIQ8nTOPveOcFLSt8GcXdx8hD0bt39uWxYT88uXzqTd4fTvqta7oeUJqudepapKNt2DYJFw==} + '@types/async-retry@1.4.9': + resolution: {integrity: sha512-s1ciZQJzRh3708X/m3vPExr5KJlzlZJvXsKpbtE2luqNcbROr64qU+3KpJsYHqWMeaxI839OvXf9PrUSw1Xtyg==} + '@types/better-sqlite3@7.6.11': resolution: {integrity: sha512-i8KcD3PgGtGBLl3+mMYA8PdKkButvPyARxA7IQAd6qeslht13qxb1zzO8dRCtE7U3IoJS782zDBAeoKiM695kg==} @@ -516,6 +540,9 @@ packages: '@types/chrome@0.0.279': resolution: {integrity: sha512-wl0IxQ2OQiMazPZM5LimHQ7Jwd72/O8UvvzyptplXT2S4eUqXH5C0n8S+v8PtKhyX89p0igCPpNy3Bwksyk57g==} + '@types/dns2@2.0.9': + resolution: {integrity: sha512-+eiPSuo/KfvaaW7DiMf/vrt9mhxbZCCgxoCa/c2qMVSuZQus4BzBuKRh8XBwcUXWAtvI8QuZOp13UODJNq9DXg==} + '@types/eslint@9.6.1': resolution: {integrity: sha512-FXx2pKgId/WyYo2jXw63kk7/+TY7u7AziEJxJAnSFzHlqTAS3Ync6SvgYAN/k4/PQpnnVuzoMuVnByKK2qp0ag==} @@ -723,6 +750,9 @@ packages: argparse@2.0.1: resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} + async-retry@1.3.3: + resolution: {integrity: sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==} + async-sema@3.1.1: resolution: {integrity: sha512-tLRNUXati5MFePdAk8dw7Qt7DpxPB60ofAgn8WRhW6a2rcimZnYBP9oxHiv0OHy+Wz7kPMG+t4LGdt31+4EmGg==} @@ -929,6 +959,9 @@ packages: resolution: {integrity: sha512-uIFDxqpRZGZ6ThOk84hEfqWoHx2devRFvpTZcTHur85vImfaxUbTW9Ryh4CpCuDnToOP1CEtXKIgytHBPVff5A==} engines: {node: '>=0.3.1'} + dns2@2.1.0: + resolution: {integrity: sha512-m27K11aQalRbmUs7RLaz6aPyceLjAoqjPRNTdE7qUouQpl+PC8Bi67O+i9SuJUPbQC8dxFrczAxfmTPuTKHNkw==} + doctrine@3.0.0: resolution: {integrity: sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==} engines: {node: '>=6.0.0'} @@ -1677,6 +1710,10 @@ packages: resolution: {integrity: sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==} engines: {node: '>= 4'} + retry@0.13.1: + resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} + engines: {node: '>= 4'} + reusify@1.0.4: resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==} engines: {iojs: '>=1.0.0', node: '>=0.10.0'} @@ -1926,6 +1963,10 @@ packages: engines: {node: '>= 8'} hasBin: true + whoiser@1.18.0: + resolution: {integrity: sha512-QRIGreBuouc8d9i+UVMFqYJSiG7gaoaGX8nKugYDGqnuNLLgjDBwmlKODOIGHveBawza3Kfkk/OuM9VsTUYwaA==} + engines: {node: '>=15.0.0'} + why-is-node-running@3.2.1: resolution: {integrity: sha512-Tb2FUhB4vUsGQlfSquQLYkApkuPAFQXGFzxWKHHumVz2dK+X1RUm/HnID4+TfIGYJ1kTcwOaCk/buYCEJr6YjQ==} engines: {node: '>=20.11'} @@ -2107,6 +2148,8 @@ snapshots: '@types/firefox-webext-browser': 120.0.4 tldts-experimental: 6.1.58 + '@henrygd/queue@1.0.7': {} + '@humanfs/core@0.19.1': {} '@humanfs/node@0.16.6': @@ -2351,6 +2394,10 @@ snapshots: tslib: 2.8.0 optional: true + '@types/async-retry@1.4.9': + dependencies: + '@types/retry': 0.12.5 + '@types/better-sqlite3@7.6.11': dependencies: '@types/node': 22.8.7 @@ -2364,6 +2411,10 @@ snapshots: '@types/filesystem': 0.0.36 '@types/har-format': 1.2.16 + '@types/dns2@2.0.9': + dependencies: + '@types/node': 22.8.7 + '@types/eslint@9.6.1': dependencies: '@types/estree': 1.0.6 @@ -2604,6 +2655,10 @@ snapshots: argparse@2.0.1: {} + async-retry@1.3.3: + dependencies: + retry: 0.13.1 + async-sema@3.1.1: {} asynckit@0.4.0: {} @@ -2807,6 +2862,8 @@ snapshots: diff@5.2.0: {} + dns2@2.1.0: {} + doctrine@3.0.0: dependencies: esutils: 2.0.3 @@ -3653,6 +3710,8 @@ snapshots: retry@0.12.0: {} + retry@0.13.1: {} + reusify@1.0.4: {} rimraf@5.0.10: @@ -3905,6 +3964,10 @@ snapshots: dependencies: isexe: 2.0.0 + whoiser@1.18.0: + dependencies: + punycode: 2.3.1 + why-is-node-running@3.2.1: {} word-wrap@1.2.5: {} diff --git a/tsconfig.json b/tsconfig.json index 3af67940..3973e4e0 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -17,6 +17,6 @@ "include": [ "./Source/**/*.js", "./Build/**/*.ts", - "Source/**/*.ts" + "./Source/**/*.ts" ] }