diff --git a/Build/build-cdn-download-conf.ts b/Build/build-cdn-download-conf.ts index b65bd4bd..9c582ad2 100644 --- a/Build/build-cdn-download-conf.ts +++ b/Build/build-cdn-download-conf.ts @@ -79,10 +79,13 @@ export const buildCdnDownloadConf = task(require.main === module, __filename)(as downloadDomainSet, steamDomainSet ] = await Promise.all([ - span.traceChildAsync('download public suffix list for s3', () => pool.exec( - 'getS3OSSDomains', - [__filename] - ).finally(() => pool.terminate())), + span.traceChildAsync( + 'download public suffix list for s3', + () => pool.exec( + 'getS3OSSDomains', + [__filename] + ).finally(() => pool.terminate()) + ), cdnDomainsListPromise, downloadDomainSetPromise, steamDomainSetPromise diff --git a/Build/build-microsoft-cdn.ts b/Build/build-microsoft-cdn.ts index f9faa097..d8a6d8f6 100644 --- a/Build/build-microsoft-cdn.ts +++ b/Build/build-microsoft-cdn.ts @@ -3,6 +3,7 @@ import { SHARED_DESCRIPTION } from './constants/description'; import { RulesetOutput } from './lib/rules/ruleset'; import Worktank from 'worktank'; import { RULES } from './constants/microsoft-cdn'; +import { wait } from 'foxts/wait'; const pool = new Worktank({ pool: { @@ -47,10 +48,10 @@ const pool = new Worktank({ } }); -export const getMicrosoftCdnRulesetPromise = pool.exec( +export const getMicrosoftCdnRulesetPromise = wait(0).then(() => pool.exec( 'getMicrosoftCdnRuleset', [__filename] -).finally(() => pool.terminate()); +)).finally(() => pool.terminate()); export const buildMicrosoftCdn = task(require.main === module, __filename)(async (span) => { const [domains, domainSuffixes] = await span.traceChildPromise('get microsoft cdn domains', getMicrosoftCdnRulesetPromise); diff --git a/Build/build-sspanel-appprofile.ts b/Build/build-sspanel-appprofile.ts index afebac76..87a9677b 100644 --- a/Build/build-sspanel-appprofile.ts +++ b/Build/build-sspanel-appprofile.ts @@ -6,7 +6,7 @@ import path from 'node:path'; import { ALL as AllStreamServices } from '../Source/stream'; import { getChnCidrPromise } from './build-chn-cidr'; -import { getTelegramCIDRPromise } from './build-telegram-cidr'; +import { getTelegramCIDRPromise } from './lib/get-telegram-backup-ip'; import { compareAndWriteFile } from './lib/create-file'; import { getMicrosoftCdnRulesetPromise } from './build-microsoft-cdn'; import { isTruthy, nullthrow } from 'foxts/guard'; @@ -75,7 +75,7 @@ export const buildSSPanelUIMAppProfile = task(require.main === module, __filenam // domestic - ip cidr getChnCidrPromise(), // global - ip cidr - getTelegramCIDRPromise(), + getTelegramCIDRPromise, // lan - ip cidr readFileIntoProcessedArray(path.join(OUTPUT_SURGE_DIR, 'ip/lan.conf')) ] as const); diff --git a/Build/build-telegram-cidr.ts b/Build/build-telegram-cidr.ts index 4bdef426..aa2ec4a2 100644 --- a/Build/build-telegram-cidr.ts +++ b/Build/build-telegram-cidr.ts @@ -1,137 +1,11 @@ // @ts-check -import { createReadlineInterfaceFromResponse } from './lib/fetch-text-by-line'; import { task } from './trace'; import { SHARED_DESCRIPTION } from './constants/description'; -import { once } from 'foxts/once'; import { RulesetOutput } from './lib/rules/ruleset'; -import { $$fetch } from './lib/fetch-retry'; -import { fastIpVersion } from 'foxts/fast-ip-version'; -import DNS2 from 'dns2'; -import { getTelegramBackupIPFromBase64 } from './lib/get-telegram-backup-ip'; -import picocolors from 'picocolors'; - -export const getTelegramCIDRPromise = once(async () => { - const resp = await $$fetch('https://core.telegram.org/resources/cidr.txt'); - const lastModified = resp.headers.get('last-modified'); - const date = lastModified ? new Date(lastModified) : new Date(); - - const ipcidr: string[] = [ - // Unused secret Telegram backup CIDR, announced by AS62041 - '95.161.64.0/20' - ]; - const ipcidr6: string[] = []; - - for await (const cidr of createReadlineInterfaceFromResponse(resp, true)) { - const v = fastIpVersion(cidr); - if (v === 4) { - ipcidr.push(cidr); - } else if (v === 6) { - ipcidr6.push(cidr); - } - } - - const backupIPs = new Set(); - - // https://github.com/tdlib/td/blob/master/td/telegram/ConfigManager.cpp - - // Backup IP Source 1 (DoH) - await Promise.all([ - DNS2.DOHClient({ dns: 'https://8.8.4.4/dns-query?dns={query}' }), - DNS2.DOHClient({ dns: 'https://1.0.0.1/dns-query?dns={query}' }) - ].flatMap( - (client) => [ - 'apv3.stel.com', // prod - 'tapv3.stel.com' // test - ].map(async (domain) => { - try { - // tapv3.stel.com was for testing server - const resp = await client(domain, 'TXT'); - const strings = resp.answers.map(i => i.data); - - const str = strings[0]!.length > strings[1]!.length - ? strings[0]! + strings[1]! - : strings[1]! + strings[0]!; - - const ips = getTelegramBackupIPFromBase64(str); - ips.forEach(i => backupIPs.add(i.ip)); - - console.log('[telegram backup ip]', picocolors.green('DoH TXT'), { domain, ips }); - } catch (e) { - console.error('[telegram backup ip]', picocolors.red('DoH TXT error'), { domain }, e); - } - }) - )); - - // Backup IP Source 2: Firebase Realtime Database (test server not supported) - try { - const text = await (await $$fetch('https://reserve-5a846.firebaseio.com/ipconfigv3.json')).json(); - if (typeof text === 'string' && text.length === 344) { - const ips = getTelegramBackupIPFromBase64(text); - ips.forEach(i => backupIPs.add(i.ip)); - - console.log('[telegram backup ip]', picocolors.green('Firebase Realtime DB'), { ips }); - } - } catch (e) { - console.error('[telegram backup ip]', picocolors.red('Firebase Realtime DB error'), e); - // ignore all errors - } - - // Backup IP Source 3: Firebase Value Store (test server not supported) - try { - const json = await (await fetch('https://firestore.googleapis.com/v1/projects/reserve-5a846/databases/(default)/documents/ipconfig/v3', { - headers: { - Accept: '*/*', - Origin: undefined // Without this line, Google API will return "Bad request: Origin doesn't match Host for XD3.". Probably have something to do with sqlite cache store - } - })).json(); - - // const json = await resp.json(); - if ( - json && typeof json === 'object' - && 'fields' in json && typeof json.fields === 'object' && json.fields - && 'data' in json.fields && typeof json.fields.data === 'object' && json.fields.data - && 'stringValue' in json.fields.data && typeof json.fields.data.stringValue === 'string' && json.fields.data.stringValue.length === 344 - ) { - const ips = getTelegramBackupIPFromBase64(json.fields.data.stringValue); - ips.forEach(i => backupIPs.add(i.ip)); - - console.log('[telegram backup ip]', picocolors.green('Firebase Value Store'), { ips }); - } else { - console.error('[telegram backup ip]', picocolors.red('Firebase Value Store data format invalid'), { json }); - } - } catch (e) { - console.error('[telegram backup ip]', picocolors.red('Firebase Value Store error'), e); - } - - // Backup IP Source 4: Google App Engine - await Promise.all([ - 'https://dns-telegram.appspot.com', - 'https://dns-telegram.appspot.com/test' - ].map(async (url) => { - try { - const text = await (await $$fetch(url)).text(); - if (text.length === 344) { - const ips = getTelegramBackupIPFromBase64(text); - ips.forEach(i => backupIPs.add(i.ip)); - - console.log('[telegram backup ip]', picocolors.green('Google App Engine'), { url, ips }); - } - } catch (e) { - console.error('[telegram backup ip]', picocolors.red('Google App Engine error'), { url }, e); - } - })); - - // tcdnb.azureedge.net no longer works - - console.log('[telegram backup ip]', `Found ${backupIPs.size} backup IPs:`, backupIPs); - - ipcidr.push(...Array.from(backupIPs).map(i => i + '/32')); - - return { date, ipcidr, ipcidr6 }; -}); +import { getTelegramCIDRPromise } from './lib/get-telegram-backup-ip'; export const buildTelegramCIDR = task(require.main === module, __filename)(async (span) => { - const { date, ipcidr, ipcidr6 } = await span.traceChildAsync('get telegram cidr', getTelegramCIDRPromise); + const { timestamp, ipcidr, ipcidr6 } = await span.traceChildPromise('get telegram cidr', getTelegramCIDRPromise); if (ipcidr.length + ipcidr6.length === 0) { throw new Error('Failed to fetch data!'); @@ -148,7 +22,7 @@ export const buildTelegramCIDR = task(require.main === module, __filename)(async .withDescription(description) // .withDate(date) // With extra data source, we no longer use last-modified for file date .appendDataSource( - 'https://core.telegram.org/resources/cidr.txt (last updated: ' + date.toISOString() + ')' + 'https://core.telegram.org/resources/cidr.txt (last updated: ' + new Date(timestamp).toISOString() + ')' ) .bulkAddCIDR4NoResolve(ipcidr) .bulkAddCIDR6NoResolve(ipcidr6) diff --git a/Build/lib/fetch-retry.ts b/Build/lib/fetch-retry.ts index 98cf7e0f..04fa869f 100644 --- a/Build/lib/fetch-retry.ts +++ b/Build/lib/fetch-retry.ts @@ -177,6 +177,8 @@ export async function $$fetch(url: string, init: RequestInit = defaultRequestIni } } +export const fetch = $$fetch; + /** @deprecated -- undici.requests doesn't support gzip/br/deflate, and has difficulty w/ undidi cache */ export async function requestWithLog(url: string, opt?: Parameters[1]) { try { diff --git a/Build/lib/get-phishing-domains.ts b/Build/lib/get-phishing-domains.ts index 51fee16d..1580e582 100644 --- a/Build/lib/get-phishing-domains.ts +++ b/Build/lib/get-phishing-domains.ts @@ -202,17 +202,10 @@ const pool = new Worktank({ export function getPhishingDomains(parentSpan: Span) { return parentSpan.traceChild('get phishing domains').traceAsyncFn(async (span) => span.traceChildAsync( 'process phishing domain set', - async () => { - const phishingDomains = await pool.exec( - 'getPhishingDomains', - [ - __filename, - require.main === module - ] - ); - pool.terminate(); - return phishingDomains; - } + () => pool.exec( + 'getPhishingDomains', + [__filename, require.main === module] + ).finally(() => pool.terminate()) )); } diff --git a/Build/lib/get-telegram-backup-ip.ts b/Build/lib/get-telegram-backup-ip.ts index c8366e9c..5a61f388 100644 --- a/Build/lib/get-telegram-backup-ip.ts +++ b/Build/lib/get-telegram-backup-ip.ts @@ -7,6 +7,9 @@ import { bigint2ip } from 'fast-cidr-tools'; import { base64ToUint8Array, concatUint8Arrays } from 'foxts/uint8array-utils'; +import Worktank from 'worktank'; +import { wait } from 'foxts/wait'; + const mtptoto_public_rsa = `-----BEGIN RSA PUBLIC KEY----- MIIBCgKCAQEAyr+18Rex2ohtVy8sroGP BwXD3DOoKCSpjDqYoXgCqB7ioln4eDCFfOBUlfXUEvM/fnKCpF46VkAftlb4VuPD @@ -108,3 +111,153 @@ export function getTelegramBackupIPFromBase64(base64: string) { } })); } + +const pool = new Worktank({ + pool: { + name: 'get-telegram-backup-ips', + size: 1 // The number of workers to keep in the pool, if more workers are needed they will be spawned up to this limit + }, + worker: { + autoAbort: 10000, + autoTerminate: 30000, // The interval of milliseconds at which to check if the pool can be automatically terminated, to free up resources, workers will be spawned up again if needed + autoInstantiate: true, + methods: { + // eslint-disable-next-line object-shorthand -- workertank + getTelegramBackupIPs: async function (__filename: string): Promise<{ timestamp: number, ipcidr: string[], ipcidr6: string[] }> { + // TODO: createRequire is a temporary workaround for https://github.com/nodejs/node/issues/51956 + const { default: module } = await import('node:module'); + const __require = module.createRequire(__filename); + + const picocolors = __require('picocolors') as typeof import('picocolors'); + const { fetch } = __require('./fetch-retry') as typeof import('./fetch-retry'); + const DNS2 = __require('dns2') as typeof import('dns2'); + const { createReadlineInterfaceFromResponse } = __require('./fetch-text-by-line') as typeof import('./fetch-text-by-line'); + const { getTelegramBackupIPFromBase64 } = __require('./get-telegram-backup-ip') as typeof import('./get-telegram-backup-ip'); + const { fastIpVersion } = __require('foxts/fast-ip-version') as typeof import('foxts/fast-ip-version'); + + const resp = await fetch('https://core.telegram.org/resources/cidr.txt'); + const lastModified = resp.headers.get('last-modified'); + const date = lastModified ? new Date(lastModified) : new Date(); + + const ipcidr: string[] = [ + // Unused secret Telegram backup CIDR, announced by AS62041 + '95.161.64.0/20' + ]; + const ipcidr6: string[] = []; + + for await (const cidr of createReadlineInterfaceFromResponse(resp, true)) { + const v = fastIpVersion(cidr); + if (v === 4) { + ipcidr.push(cidr); + } else if (v === 6) { + ipcidr6.push(cidr); + } + } + + const backupIPs = new Set(); + + // https://github.com/tdlib/td/blob/master/td/telegram/ConfigManager.cpp + + // Backup IP Source 1 (DoH) + await Promise.all([ + DNS2.DOHClient({ dns: 'https://8.8.4.4/dns-query?dns={query}' }), + DNS2.DOHClient({ dns: 'https://1.0.0.1/dns-query?dns={query}' }) + ].flatMap( + (client) => [ + 'apv3.stel.com', // prod + 'tapv3.stel.com' // test + ].map(async (domain) => { + try { + // tapv3.stel.com was for testing server + const resp = await client(domain, 'TXT'); + const strings = resp.answers.map(i => i.data); + + const str = strings[0]!.length > strings[1]!.length + ? strings[0]! + strings[1]! + : strings[1]! + strings[0]!; + + const ips = getTelegramBackupIPFromBase64(str); + ips.forEach(i => backupIPs.add(i.ip)); + + console.log('[telegram backup ip]', picocolors.green('DoH TXT'), { domain, ips }); + } catch (e) { + console.error('[telegram backup ip]', picocolors.red('DoH TXT error'), { domain }, e); + } + }) + )); + + // Backup IP Source 2: Firebase Realtime Database (test server not supported) + try { + const text = await (await fetch('https://reserve-5a846.firebaseio.com/ipconfigv3.json')).json(); + if (typeof text === 'string' && text.length === 344) { + const ips = getTelegramBackupIPFromBase64(text); + ips.forEach(i => backupIPs.add(i.ip)); + + console.log('[telegram backup ip]', picocolors.green('Firebase Realtime DB'), { ips }); + } + } catch (e) { + console.error('[telegram backup ip]', picocolors.red('Firebase Realtime DB error'), e); + // ignore all errors + } + + // Backup IP Source 3: Firebase Value Store (test server not supported) + try { + const json = await (await fetch('https://firestore.googleapis.com/v1/projects/reserve-5a846/databases/(default)/documents/ipconfig/v3', { + headers: { + Accept: '*/*', + Origin: undefined // Without this line, Google API will return "Bad request: Origin doesn't match Host for XD3.". Probably have something to do with sqlite cache store + } + })).json(); + + // const json = await resp.json(); + if ( + json && typeof json === 'object' + && 'fields' in json && typeof json.fields === 'object' && json.fields + && 'data' in json.fields && typeof json.fields.data === 'object' && json.fields.data + && 'stringValue' in json.fields.data && typeof json.fields.data.stringValue === 'string' && json.fields.data.stringValue.length === 344 + ) { + const ips = getTelegramBackupIPFromBase64(json.fields.data.stringValue); + ips.forEach(i => backupIPs.add(i.ip)); + + console.log('[telegram backup ip]', picocolors.green('Firebase Value Store'), { ips }); + } else { + console.error('[telegram backup ip]', picocolors.red('Firebase Value Store data format invalid'), { json }); + } + } catch (e) { + console.error('[telegram backup ip]', picocolors.red('Firebase Value Store error'), e); + } + + // Backup IP Source 4: Google App Engine + await Promise.all([ + 'https://dns-telegram.appspot.com', + 'https://dns-telegram.appspot.com/test' + ].map(async (url) => { + try { + const text = await (await fetch(url)).text(); + if (text.length === 344) { + const ips = getTelegramBackupIPFromBase64(text); + ips.forEach(i => backupIPs.add(i.ip)); + + console.log('[telegram backup ip]', picocolors.green('Google App Engine'), { url, ips }); + } + } catch (e) { + console.error('[telegram backup ip]', picocolors.red('Google App Engine error'), { url }, e); + } + })); + + // tcdnb.azureedge.net no longer works + + console.log('[telegram backup ip]', `Found ${backupIPs.size} backup IPs:`, backupIPs); + + ipcidr.push(...Array.from(backupIPs).map(i => i + '/32')); + + return { timestamp: date.getTime(), ipcidr, ipcidr6 }; + } + } + } +}); + +export const getTelegramCIDRPromise = wait(0).then(() => pool.exec( + 'getTelegramBackupIPs', + [__filename] +)).finally(() => pool.terminate());