diff --git a/Build/build-telegram-cidr.worker.ts b/Build/build-telegram-cidr.worker.ts deleted file mode 100644 index 86e6df2c..00000000 --- a/Build/build-telegram-cidr.worker.ts +++ /dev/null @@ -1,168 +0,0 @@ -// @ts-check -import { task } from './trace'; -import { SHARED_DESCRIPTION } from './constants/description'; -import { RulesetOutput } from './lib/rules/ruleset'; -import { getTelegramBackupIPFromBase64 } from './lib/get-telegram-backup-ip'; -import picocolors from 'picocolors'; -import { $$fetch } from './lib/fetch-retry'; -import dns from 'node:dns/promises'; -import { createReadlineInterfaceFromResponse } from './lib/fetch-text-by-line'; -import { fastIpVersion } from 'foxts/fast-ip-version'; -import { fastStringArrayJoin } from 'foxts/fast-string-array-join'; -import { appendArrayInPlace } from 'foxts/append-array-in-place'; - -export const buildTelegramCIDR = task(require.main === module, __filename)(async (span) => { - const { timestamp, ipcidr, ipcidr6 } = await span.traceChildAsync('get telegram cidr', async (childSpan) => { - const ipcidr: string[] = [ - // Unused secret Telegram backup CIDR, announced by AS62041 - '95.161.64.0/20' - ]; - const ipcidr6: string[] = []; - - const date = await childSpan.traceChildAsync('fetch from official cidr list', async () => { - const resp = await $$fetch('https://core.telegram.org/resources/cidr.txt'); - const lastModified = resp.headers.get('last-modified'); - - for await (const cidr of createReadlineInterfaceFromResponse(resp, true)) { - const v = fastIpVersion(cidr); - if (v === 4) { - ipcidr.push(cidr); - } else if (v === 6) { - ipcidr6.push(cidr); - } - } - - return lastModified ? new Date(lastModified) : new Date(); - }); - - // https://github.com/tdlib/td/blob/master/td/telegram/ConfigManager.cpp - const backupIPs = await childSpan.traceChildAsync('fetch backup ip', async (innerSpan) => { - const backupIPs = new Set(); - const resolvers = ['8.8.8.8', '1.0.0.1'].map((ip) => { - const resolver = new dns.Resolver(); - resolver.setServers([ip]); - return Object.assign(resolver, { server: ip }); - }); - - await innerSpan.traceChildAsync('backup source 1: DNS TXT', () => Promise.all(resolvers.flatMap((resolver) => [ - 'apv3.stel.com', // prod - 'tapv3.stel.com' // test - ].map(async (domain) => { - try { - // tapv3.stel.com was for testing server - const resp = await resolver.resolveTxt(domain); - const strings = resp.map(r => fastStringArrayJoin(r, '')); // flatten - if (strings.length !== 2) { - throw new TypeError(`Unexpected TXT record count: ${strings.length}`); - } - - const str = strings[0].length > strings[1].length - ? strings[0] + strings[1] - : strings[1] + strings[0]; - - const ips = getTelegramBackupIPFromBase64(str); - ips.forEach(i => backupIPs.add(i.ip)); - - console.log('[telegram backup ip]', picocolors.green('DNS TXT'), { domain, ips, server: resolver.server }); - } catch (e) { - console.error('[telegram backup ip]', picocolors.red('DNS TXT error'), { domain }, e); - } - })))); - - // Backup IP Source 2: Firebase Realtime Database (test server not supported) - await innerSpan.traceChildAsync('backup source 2: Firebase Realtime DB', async () => { - try { - const text = await (await $$fetch('https://reserve-5a846.firebaseio.com/ipconfigv3.json')).json(); - if (typeof text === 'string' && text.length === 344) { - const ips = getTelegramBackupIPFromBase64(text); - ips.forEach(i => backupIPs.add(i.ip)); - - console.log('[telegram backup ip]', picocolors.green('Firebase Realtime DB'), { ips }); - } - } catch (e) { - console.error('[telegram backup ip]', picocolors.red('Firebase Realtime DB error'), e); - // ignore all errors - } - }); - - // Backup IP Source 3: Firebase Value Store (test server not supported) - await innerSpan.traceChildAsync('backup source 3: Firebase Value Store', async () => { - try { - const json = await (await $$fetch('https://firestore.googleapis.com/v1/projects/reserve-5a846/databases/(default)/documents/ipconfig/v3', { - headers: { - Accept: '*/*', - Origin: undefined // Without this line, Google API will return "Bad request: Origin doesn't match Host for XD3.". Probably have something to do with sqlite cache store - } - })).json(); - - if ( - json && typeof json === 'object' - && 'fields' in json && typeof json.fields === 'object' && json.fields - && 'data' in json.fields && typeof json.fields.data === 'object' && json.fields.data - && 'stringValue' in json.fields.data && typeof json.fields.data.stringValue === 'string' && json.fields.data.stringValue.length === 344 - ) { - const ips = getTelegramBackupIPFromBase64(json.fields.data.stringValue); - ips.forEach(i => backupIPs.add(i.ip)); - - console.log('[telegram backup ip]', picocolors.green('Firebase Value Store'), { ips }); - } else { - console.error('[telegram backup ip]', picocolors.red('Firebase Value Store data format invalid'), { json }); - } - } catch (e) { - console.error('[telegram backup ip]', picocolors.red('Firebase Value Store error'), e); - } - }); - - // Backup IP Source 4: Google App Engine - await innerSpan.traceChildAsync('backup source 4: Google App Engine', () => Promise.all([ - 'https://dns-telegram.appspot.com', - 'https://dns-telegram.appspot.com/test' - ].map(async (url) => { - try { - const text = await (await $$fetch(url)).text(); - if (text.length === 344) { - const ips = getTelegramBackupIPFromBase64(text); - ips.forEach(i => backupIPs.add(i.ip)); - - console.log('[telegram backup ip]', picocolors.green('Google App Engine'), { url, ips }); - } - } catch (e) { - console.error('[telegram backup ip]', picocolors.red('Google App Engine error'), { url }, e); - } - }))); - - // tcdnb.azureedge.net no longer works - - return backupIPs; - }); - - console.log('[telegram backup ip]', `Found ${backupIPs.size} backup IPs:`, backupIPs); - - appendArrayInPlace(ipcidr, Array.from(backupIPs).map(i => i + '/32')); - - return { timestamp: date.getTime(), ipcidr, ipcidr6 }; - }); - - if (ipcidr.length + ipcidr6.length === 0) { - throw new Error('Failed to fetch data!'); - } - - const description = [ - ...SHARED_DESCRIPTION, - 'Data from:', - ' - https://core.telegram.org/resources/cidr.txt' - ]; - - return new RulesetOutput(span, 'telegram', 'ip') - .withTitle('Sukka\'s Ruleset - Telegram IP CIDR') - .withDescription(description) - // .withDate(date) // With extra data source, we no longer use last-modified for file date - .appendDataSource( - 'https://core.telegram.org/resources/cidr.txt (last updated: ' + new Date(timestamp).toISOString() + ')' - ) - .bulkAddCIDR4NoResolve(ipcidr) - .bulkAddCIDR6NoResolve(ipcidr6) - .write(); -}); - -export const ___ = ''; diff --git a/Build/build-telegram.worker.ts b/Build/build-telegram.worker.ts new file mode 100644 index 00000000..e7d8100d --- /dev/null +++ b/Build/build-telegram.worker.ts @@ -0,0 +1,179 @@ +// @ts-check +import { task } from './trace'; +import { SHARED_DESCRIPTION } from './constants/description'; +import { RulesetOutput } from './lib/rules/ruleset'; +import { $$fetch } from './lib/fetch-retry'; +import { createReadlineInterfaceFromResponse } from './lib/fetch-text-by-line'; +import { fastIpVersion } from 'foxts/fast-ip-version'; +import { appendArrayInPlace } from 'foxts/append-array-in-place'; +import { fetchTelegramBackupEndpoints } from './lib/fetch-telegram-backup-endpoints'; + +const buildTelegramCIDR = task(require.main === module, __filename)(async (span) => { + const { timestamp, ipcidr, ipcidr6 } = await span.traceChildAsync('get telegram cidr', async (childSpan) => { + const ipcidr: string[] = [ + // Unused secret Telegram backup CIDR, announced by AS62041 + '95.161.64.0/20' + ]; + const ipcidr6: string[] = []; + + const date = await childSpan.traceChildAsync('fetch from official cidr list', async () => { + const resp = await $$fetch('https://core.telegram.org/resources/cidr.txt'); + const lastModified = resp.headers.get('last-modified'); + + for await (const cidr of createReadlineInterfaceFromResponse(resp, true)) { + const v = fastIpVersion(cidr); + if (v === 4) { + ipcidr.push(cidr); + } else if (v === 6) { + ipcidr6.push(cidr); + } + } + + return lastModified ? new Date(lastModified) : new Date(); + }); + + // https://github.com/tdlib/td/blob/master/td/telegram/ConfigManager.cpp + const backupEndpoints = await childSpan.traceChildAsync( + 'fetch backup ip', + innerSpan => fetchTelegramBackupEndpoints(innerSpan, { includeTestServers: true }) + ); + const backupIPs = new Set(backupEndpoints.map(endpoint => endpoint.ip)); + + console.log('[telegram backup ip]', `Found ${backupIPs.size} backup IPs:`, backupIPs); + + appendArrayInPlace(ipcidr, Array.from(backupIPs, i => i + '/32')); + + return { timestamp: date.getTime(), ipcidr, ipcidr6 }; + }); + + if (ipcidr.length + ipcidr6.length === 0) { + throw new Error('Failed to fetch data!'); + } + + const description = [ + ...SHARED_DESCRIPTION, + 'Data from:', + ' - https://core.telegram.org/resources/cidr.txt' + ]; + + return new RulesetOutput(span, 'telegram', 'ip') + .withTitle('Sukka\'s Ruleset - Telegram IP CIDR') + .withDescription(description) + // .withDate(date) // With extra data source, we no longer use last-modified for file date + .appendDataSource( + 'https://core.telegram.org/resources/cidr.txt (last updated: ' + new Date(timestamp).toISOString() + ')' + ) + .bulkAddCIDR4NoResolve(ipcidr) + .bulkAddCIDR6NoResolve(ipcidr6) + .write(); +}); + +// @ts-check +import path from 'node:path'; +import process from 'node:process'; + +import { Api as TgApi, TelegramClient as TgClient } from 'telegram'; +import { Logger as TgLogger, LogLevel as TgLogLevel } from 'telegram/extensions/Logger'; +import { ConnectionTCPAbridged as TgConnectionTCPAbridged } from 'telegram/network/connection'; +import { MemorySession as TgMemorySession } from 'telegram/sessions'; + +import { OUTPUT_INTERNAL_DIR } from './constants/dir'; +import { compareAndWriteFile } from './lib/create-file'; +import { + mergeFallbackEndpoints, + normalizeTelegramConfig, + TELEGRAM_BOOTSTRAP_ENDPOINTS +} from './lib/mtproto-dc-config'; +import type { MTProtoDCConfig } from './lib/mtproto-dc-config'; + +const TELEGRAM_API_ID = 2040; +const OUTPUT_PATH = path.join(OUTPUT_INTERNAL_DIR, 'mtproto-dc-config.json'); + +async function fetchConfig(host: string, port: number, dcId: number) { + const session = new TgMemorySession(); + session.setDC(dcId, host, port); + + const client = new TgClient(session, TELEGRAM_API_ID, 'not-used-for-unauthenticated-rpc', { + appVersion: '1.0', + autoReconnect: false, + baseLogger: new TgLogger(TgLogLevel.NONE), + connection: TgConnectionTCPAbridged, + connectionRetries: 1, + deviceModel: 'Surge', + langCode: 'en', + reconnectRetries: 0, + requestRetries: 1, + securityChecks: true, + systemLangCode: 'en', + systemVersion: process.platform, + timeout: 10, + // GramJS uses this option to select port 443 even for a raw TCP connection. + useWSS: true + }); + + try { + const connected = await client.connect(); + if (!connected && !client.connected) { + throw new Error('MTProto client did not connect'); + } + + return normalizeTelegramConfig(await client.invoke(new TgApi.help.GetConfig())); + } finally { + await client.disconnect(); + } +} + +async function fetchConfigFromBootstrapEndpoints() { + let lastError: unknown; + + for (let i = 0, len = TELEGRAM_BOOTSTRAP_ENDPOINTS.length; i < len; i++) { + const endpoint = TELEGRAM_BOOTSTRAP_ENDPOINTS[i]; + console.log(`[telegram mtproto config] Fetching help.getConfig from ${endpoint.ip}:${endpoint.port}`); + try { + // Bootstrap order is significant, and one successful response ends the loop. + // eslint-disable-next-line no-await-in-loop -- Bootstrap endpoints must be attempted in order. + return await fetchConfig(endpoint.ip, endpoint.port, endpoint.dcId); + } catch (error) { + lastError = error; + console.error(`[telegram mtproto config] ${endpoint.ip}:${endpoint.port} failed`, error); + } + } + + throw new AggregateError( + lastError === undefined ? [] : [lastError], + 'All Telegram MTProto bootstrap endpoints failed' + ); +} + +export const buildMTProtoDCConfig = task(require.main === module, __filename)(async (span) => { + const config = await span.traceChildAsync( + 'fetch help.getConfig', + fetchConfigFromBootstrapEndpoints + ); + + const backupEndpoints = await span.traceChildAsync( + 'fetch telegram backup endpoints', + childSpan => fetchTelegramBackupEndpoints(childSpan, { includeTestServers: false }) + ); + + const liveEndpoints = config.options.length; + const mergeResult = mergeFallbackEndpoints(config, backupEndpoints); + console.log('[telegram mtproto config]', { + liveEndpoints, + backupEndpoints: backupEndpoints.length, + ...mergeResult, + outputEndpoints: config.options.length + }); + + const output = JSON.stringify(config satisfies MTProtoDCConfig, null, 2).split('\n'); + await compareAndWriteFile(span, output, OUTPUT_PATH); +}); + +// Start both tasks in this worker concurrently. They retain independent trace +// results while sharing the module-scoped production backup endpoint promise. +export function buildTelegram() { + return Promise.all([ + buildTelegramCIDR(), + buildMTProtoDCConfig() + ]); +} diff --git a/Build/index.ts b/Build/index.ts index e03d7be1..62e59a4b 100644 --- a/Build/index.ts +++ b/Build/index.ts @@ -30,6 +30,7 @@ import { ROOT_DIR } from './constants/dir'; import { isCI } from 'ci-info'; import { printExternalDownloadStats } from './lib/download-stats'; import { endOutputWorkerFarm, warmOutputWorkerFarm } from './lib/rules/output-worker-farm'; +import { appendArrayInPlace } from 'foxts/append-array-in-place'; process.on('uncaughtException', (error) => { console.error('Uncaught exception:', error); @@ -78,9 +79,9 @@ const buildFinishedLock = path.join(ROOT_DIR, '.BUILD_FINISHED'); require.resolve('./build-cdn-download-conf.worker') )(['buildCdnDownloadConf']); - const telegramCidrWorker = createWorker( - require.resolve('./build-telegram-cidr.worker') - )(['buildTelegramCIDR']); + const telegramWorker = createWorker( + require.resolve('./build-telegram.worker') + )(['buildTelegram']); const mockAssetsWorker = createWorker( require.resolve('./download-mock-assets.worker') @@ -99,31 +100,31 @@ const buildFinishedLock = path.join(ROOT_DIR, '.BUILD_FINISHED'); const downloadPreviousBuildPromise = downloadPreviousBuild(); - const traces: TraceResult[] = await Promise.all([ - downloadPreviousBuildPromise, - downloadPreviousBuildPromise.then(() => buildCommon()), - downloadPreviousBuildPromise.then(() => buildRejectIPList()), - downloadPreviousBuildPromise.then(() => buildAppleCdn()), - downloadPreviousBuildPromise.then(() => buildAICIDR()), - downloadPreviousBuildPromise.then(() => cdnDownloadWorker.buildCdnDownloadConf()), - downloadPreviousBuildPromise.then(() => buildRejectDomainSet()), - downloadPreviousBuildPromise.then(() => telegramCidrWorker.buildTelegramCIDR()), - downloadPreviousBuildPromise.then(() => buildChnCidr()), - downloadPreviousBuildPromise.then(() => buildSpeedtestDomainSet()), - downloadPreviousBuildPromise.then(() => buildDomesticRuleset()), - downloadPreviousBuildPromise.then(() => buildGlobalRuleset()), - downloadPreviousBuildPromise.then(() => buildRedirectModule()), - downloadPreviousBuildPromise.then(() => buildAlwaysRealIPModule()), - downloadPreviousBuildPromise.then(() => buildStreamService()), - downloadPreviousBuildPromise.then(() => microsoftCdnWorker.buildMicrosoftCdn()), - downloadPreviousBuildPromise.then(() => buildCloudMounterRules()), - mockAssetsWorker.downloadMockAssets() + const [traces, telegramTraces]: [TraceResult[], TraceResult[]] = await Promise.all([ + Promise.all([ + downloadPreviousBuildPromise, + downloadPreviousBuildPromise.then(() => buildCommon()), + downloadPreviousBuildPromise.then(() => buildRejectIPList()), + downloadPreviousBuildPromise.then(() => buildAppleCdn()), + downloadPreviousBuildPromise.then(() => buildAICIDR()), + downloadPreviousBuildPromise.then(() => cdnDownloadWorker.buildCdnDownloadConf()), + downloadPreviousBuildPromise.then(() => buildRejectDomainSet()), + downloadPreviousBuildPromise.then(() => buildChnCidr()), + downloadPreviousBuildPromise.then(() => buildSpeedtestDomainSet()), + downloadPreviousBuildPromise.then(() => buildDomesticRuleset()), + downloadPreviousBuildPromise.then(() => buildGlobalRuleset()), + downloadPreviousBuildPromise.then(() => buildRedirectModule()), + downloadPreviousBuildPromise.then(() => buildAlwaysRealIPModule()), + downloadPreviousBuildPromise.then(() => buildStreamService()), + downloadPreviousBuildPromise.then(() => microsoftCdnWorker.buildMicrosoftCdn()), + downloadPreviousBuildPromise.then(() => buildCloudMounterRules()), + mockAssetsWorker.downloadMockAssets() + ]), + downloadPreviousBuildPromise.then(() => telegramWorker.buildTelegram()) ]); - traces.push( - await buildDeprecateFiles(), - await buildPublic() - ); + appendArrayInPlace(traces, telegramTraces); + traces.push(await buildDeprecateFiles(), await buildPublic()); // write a file to demonstrate that the build is finished fs.writeFileSync(buildFinishedLock, 'BUILD_FINISHED\n'); @@ -137,7 +138,7 @@ const buildFinishedLock = path.join(ROOT_DIR, '.BUILD_FINISHED'); await Promise.all([ microsoftCdnWorker.end(), cdnDownloadWorker.end(), - telegramCidrWorker.end(), + telegramWorker.end(), mockAssetsWorker.end(), endOutputWorkerFarm() ]); diff --git a/Build/lib/fetch-telegram-backup-endpoints.ts b/Build/lib/fetch-telegram-backup-endpoints.ts new file mode 100644 index 00000000..771db609 --- /dev/null +++ b/Build/lib/fetch-telegram-backup-endpoints.ts @@ -0,0 +1,180 @@ +import dns from 'node:dns/promises'; +import { Buffer } from 'node:buffer'; + +import picocolors from 'picocolors'; +import { fastStringArrayJoin } from 'foxts/fast-string-array-join'; +import { stableHash } from 'stable-hash'; + +import type { Span } from '../trace'; +import { $$fetch } from './fetch-retry'; +import { getTelegramBackupIPFromBase64 } from './get-telegram-backup-ip'; +import type { TelegramBackupEndpoint } from './get-telegram-backup-ip'; +import { appendArrayInPlace } from 'foxts/append-array-in-place'; + +interface FetchTelegramBackupEndpointsOptions { + includeTestServers?: boolean +} + +let productionEndpointsPromise: Promise | undefined; +let testEndpointsPromise: Promise | undefined; + +function deduplicateEndpoints(endpoints: readonly TelegramBackupEndpoint[]) { + const deduplicated = new Map(); + endpoints.forEach(endpoint => deduplicated.set(stableHash([ + endpoint.dcId, + endpoint.ip, + endpoint.port, + endpoint.secret ? Buffer.from(endpoint.secret).toString('base64') : null + ]), endpoint)); + return Array.from(deduplicated.values()); +} + +async function fetchDnsEndpoints(span: Span, domain: string, traceName: string) { + const endpoints: TelegramBackupEndpoint[] = []; + const resolvers = ['8.8.8.8', '1.0.0.1'].map((ip) => { + const resolver = new dns.Resolver(); + resolver.setServers([ip]); + return Object.assign(resolver, { server: ip }); + }); + + await span.traceChildAsync(traceName, () => Promise.all(resolvers.map(async (resolver) => { + try { + const response = await resolver.resolveTxt(domain); + const strings = response.map(result => fastStringArrayJoin(result, '')); + if (strings.length !== 2) { + throw new TypeError(`Unexpected TXT record count: ${strings.length}`); + } + + const base64 = strings[0].length > strings[1].length + ? strings[0] + strings[1] + : strings[1] + strings[0]; + const decoded = getTelegramBackupIPFromBase64(base64); + appendArrayInPlace(endpoints, decoded); + + console.log('[telegram backup ip]', picocolors.green('DNS TXT'), { domain, endpoints: decoded, server: resolver.server }); + } catch (error) { + console.error('[telegram backup ip]', picocolors.red('DNS TXT error'), { domain }, error); + } + }))); + + return deduplicateEndpoints(endpoints); +} + +async function fetchRealtimeDatabaseEndpoints(span: Span) { + return span.traceChildAsync('backup source 2: Firebase Realtime DB', async () => { + try { + const data = await (await $$fetch('https://reserve-5a846.firebaseio.com/ipconfigv3.json')).json(); + if (typeof data !== 'string' || data.length !== 344) { + throw new TypeError('Firebase Realtime DB data format is invalid'); + } + + const endpoints = getTelegramBackupIPFromBase64(data); + console.log('[telegram backup ip]', picocolors.green('Firebase Realtime DB'), { endpoints }); + return endpoints; + } catch (error) { + console.error('[telegram backup ip]', picocolors.red('Firebase Realtime DB error'), error); + return []; + } + }); +} + +async function fetchValueStoreEndpoints(span: Span) { + return span.traceChildAsync('backup source 3: Firebase Value Store', async () => { + try { + const json = await (await $$fetch('https://firestore.googleapis.com/v1/projects/reserve-5a846/databases/(default)/documents/ipconfig/v3', { + headers: { + Accept: '*/*', + // Google rejects this request when the shared HTTP cache adds an Origin. + Origin: undefined + } + })).json(); + + if ( + !json || typeof json !== 'object' + || !('fields' in json) || typeof json.fields !== 'object' || !json.fields + || !('data' in json.fields) || typeof json.fields.data !== 'object' || !json.fields.data + || !('stringValue' in json.fields.data) || typeof json.fields.data.stringValue !== 'string' + || json.fields.data.stringValue.length !== 344 + ) { + throw new TypeError('Firebase Value Store data format is invalid'); + } + + const endpoints = getTelegramBackupIPFromBase64(json.fields.data.stringValue); + console.log('[telegram backup ip]', picocolors.green('Firebase Value Store'), { endpoints }); + return endpoints; + } catch (error) { + console.error('[telegram backup ip]', picocolors.red('Firebase Value Store error'), error); + return []; + } + }); +} + +async function fetchAppEngineEndpoints(span: Span, url: string, traceName: string) { + return span.traceChildAsync(traceName, async () => { + try { + const data = (await (await $$fetch(url)).text()).trim(); + if (data.length !== 344) { + throw new TypeError(`Google App Engine data has an unexpected length: ${data.length}`); + } + + const endpoints = getTelegramBackupIPFromBase64(data); + console.log('[telegram backup ip]', picocolors.green('Google App Engine'), { url, endpoints }); + return endpoints; + } catch (error) { + console.error('[telegram backup ip]', picocolors.red('Google App Engine error'), { url }, error); + return []; + } + }); +} + +async function fetchProductionEndpoints(span: Span) { + const endpointGroups = await Promise.all([ + fetchDnsEndpoints(span, 'apv3.stel.com', 'backup source 1: DNS TXT'), + fetchRealtimeDatabaseEndpoints(span), + fetchValueStoreEndpoints(span), + fetchAppEngineEndpoints(span, 'https://dns-telegram.appspot.com', 'backup source 4: Google App Engine') + ]); + return deduplicateEndpoints(endpointGroups.flat()); +} + +async function fetchTestEndpoints(span: Span) { + const endpointGroups = await Promise.all([ + fetchDnsEndpoints(span, 'tapv3.stel.com', 'test backup source 1: DNS TXT'), + fetchAppEngineEndpoints(span, 'https://dns-telegram.appspot.com/test', 'test backup source 4: Google App Engine') + ]); + return deduplicateEndpoints(endpointGroups.flat()); +} + +function getProductionEndpoints(span: Span) { + if (productionEndpointsPromise) { + return span.traceChildAsync('reuse production backup endpoints', () => productionEndpointsPromise!); + } + productionEndpointsPromise = fetchProductionEndpoints(span); + return productionEndpointsPromise; +} + +function getTestEndpoints(span: Span) { + if (testEndpointsPromise) { + return span.traceChildAsync('reuse test backup endpoints', () => testEndpointsPromise!); + } + testEndpointsPromise = fetchTestEndpoints(span); + return testEndpointsPromise; +} + +/** + * Fetches Telegram's signed backup endpoints once per worker process. The CIDR + * and MTProto config tasks share the production promise; only the CIDR task + * requests the additional test-server promise. + */ +export async function fetchTelegramBackupEndpoints( + span: Span, + { includeTestServers = false }: FetchTelegramBackupEndpointsOptions = {} +) { + if (!includeTestServers) return getProductionEndpoints(span); + + const [productionEndpoints, testEndpoints] = await Promise.all([ + getProductionEndpoints(span), + getTestEndpoints(span) + ]); + return deduplicateEndpoints([...productionEndpoints, ...testEndpoints]); +} diff --git a/Build/lib/get-telegram-backup-ip.ts b/Build/lib/get-telegram-backup-ip.ts index da077486..f392fac3 100644 --- a/Build/lib/get-telegram-backup-ip.ts +++ b/Build/lib/get-telegram-backup-ip.ts @@ -19,6 +19,42 @@ xwIDAQAB -----END RSA PUBLIC KEY----- `; +export interface TelegramBackupEndpoint { + dcId: number, + ip: string, + port: number, + secret?: Uint8Array +} + +// Telegram's original, pre-AccessPointRule backup schema: +// help.configSimple#d997c3c5 date:int expires:int dc_id:int +// ip_port_list:Vector = help.ConfigSimple; +// It applies one DC ID to a bare vector of (ipv4, port) pairs. GramJS only +// generates TgApi.help.ConfigSimple for the current #5a592a6c rule-based schema, +// so the removed legacy constructor has no TgApi symbol to reference. +const LEGACY_CONFIG_SIMPLE_CONSTRUCTOR_ID = 0xD9_97_C3_C5; + +// The legacy vector contains bare ipPort values, so TgBinaryReader#tgReadVector +// cannot decode it as a vector of boxed TL objects. GramJS does not export the +// core vector constructor ID, either, so validate it explicitly before reading. +const TL_VECTOR_CONSTRUCTOR_ID = 0x1C_B5_C4_15; + +function ipv4ToString(ipv4: number) { + return bigint2ip( + ipv4 > 0 + ? BigInt(ipv4) + : (2n ** 32n) + BigInt(ipv4), + 4 + ); +} + +function validateBackupEndpoint(endpoint: TelegramBackupEndpoint) { + if (endpoint.dcId < 1 || endpoint.dcId > 5 || endpoint.port < 1 || endpoint.port > 65535) { + throw new TypeError(`Invalid Telegram backup endpoint: DC ${endpoint.dcId}, port ${endpoint.port}`); + } + return endpoint; +} + export function getTelegramBackupIPFromBase64(base64: string) { // 1. Check base64 size if (base64.length !== 344) { @@ -31,7 +67,7 @@ export function getTelegramBackupIPFromBase64(base64: string) { // 3. Decode base64 to Buffer const decoded = base64ToUint8Array(base64); if (decoded.length !== 256) { - throw new TypeError('Decoded buffer length is not 344 bytes, received ' + decoded.length); + throw new TypeError('Decoded buffer length is not 256 bytes, received ' + decoded.length); } // 4. RSA decrypt (public key, "decrypt signature" - usually means "verify and extract") @@ -79,33 +115,58 @@ export function getTelegramBackupIPFromBase64(base64: string) { const parser = new TgBinaryReader(Buffer.from(decryptedCbc.buffer, decryptedCbc.byteOffset, decryptedCbc.byteLength)); const len = parser.readInt(); - if (len < 8 || len > 208) throw new Error(`Invalid TL data length: ${len}`); + if (len < 4 || len > 204 || len % 4 !== 0) throw new Error(`Invalid TL data length: ${len}`); - const constructorId = parser.readInt(); + const constructorId = parser.readInt() >>> 0; - if (constructorId !== TgApi.help.ConfigSimple.CONSTRUCTOR_ID) { - throw new Error(`Invalid constructor ID: ${constructorId.toString(16)}`); + const endpoints: TelegramBackupEndpoint[] = []; + let date: number; + let expires: number; + + if (constructorId === TgApi.help.ConfigSimple.CONSTRUCTOR_ID) { + const payload = decryptedCbc.subarray(8, 4 + len); + const configSimple = TgApi.help.ConfigSimple.fromReader(new TgBinaryReader(Buffer.from(payload.buffer, payload.byteOffset, payload.byteLength))); + + date = configSimple.date; + expires = configSimple.expires; + for (let ruleIndex = 0, ruleCount = configSimple.rules.length; ruleIndex < ruleCount; ruleIndex++) { + const rule = configSimple.rules[ruleIndex]; + for (let ipIndex = 0, ipCount = rule.ips.length; ipIndex < ipCount; ipIndex++) { + const ip = rule.ips[ipIndex]; + endpoints.push(validateBackupEndpoint({ + dcId: rule.dcId, + ip: ipv4ToString(ip.ipv4), + port: ip.port, + ...((ip instanceof TgApi.IpPortSecret) && { secret: ip.secret }) + })); + } + } + } else if (constructorId === LEGACY_CONFIG_SIMPLE_CONSTRUCTOR_ID) { + date = parser.readInt(); + expires = parser.readInt(); + const dcId = parser.readInt(); + const vectorConstructorId = parser.readInt() >>> 0; + const count = parser.readInt(); + + if (vectorConstructorId !== TL_VECTOR_CONSTRUCTOR_ID || count < 1 || count > 1024) { + throw new Error('Invalid legacy Telegram backup endpoint vector'); + } + + for (let i = 0; i < count; i++) { + endpoints.push(validateBackupEndpoint({ + dcId, + ip: ipv4ToString(parser.readInt()), + port: parser.readInt() + })); + } + } else { + throw new Error(`Invalid constructor ID: 0x${constructorId.toString(16)}`); } - const payload = decryptedCbc.subarray(8, len); + const now = Math.floor(Date.now() / 1000); + if (date >= now + 20 * 60 || expires <= now - 20 * 60) { + throw new Error(`Telegram backup configuration is outside its validity interval (${date}...${expires}, now ${now})`); + } - const configSimple = TgApi.help.ConfigSimple.fromReader(new TgBinaryReader(Buffer.from(payload.buffer, payload.byteOffset, payload.byteLength))); - - return configSimple.rules.flatMap(rule => rule.ips.map(ip => { - switch (ip.CONSTRUCTOR_ID) { - case TgApi.IpPort.CONSTRUCTOR_ID: - case TgApi.IpPortSecret.CONSTRUCTOR_ID: - return { - ip: bigint2ip( - ip.ipv4 > 0 - ? BigInt(ip.ipv4) - : (2n ** 32n) + BigInt(ip.ipv4), - 4 - ), - port: ip.port - }; - default: - throw new TypeError(`Unknown IP type: 0x${ip.CONSTRUCTOR_ID.toString(16)}`); - } - })); + return endpoints; } diff --git a/Build/lib/mtproto-dc-config.test.ts b/Build/lib/mtproto-dc-config.test.ts new file mode 100644 index 00000000..a7edbcec --- /dev/null +++ b/Build/lib/mtproto-dc-config.test.ts @@ -0,0 +1,96 @@ +import { describe, it } from 'mocha'; +import { expect } from 'earl'; + +import { + DC_OPTION_FLAG_IPV6, + DC_OPTION_FLAG_STATIC, + mergeFallbackEndpoints, + TELEGRAM_BOOTSTRAP_ENDPOINTS +} from './mtproto-dc-config'; +import type { MTProtoDCConfig } from './mtproto-dc-config'; +import { setBit } from 'foxts/bitwise'; + +describe('MTProto DC config', () => { + it('merges static fallbacks and removes exact duplicates', () => { + const config: MTProtoDCConfig = { + version: 1, + date: 1, + expires: 2, + this_dc: 5, + options: [ + { id: 5, ip: '91.108.56.191', port: 443, flags: 0 }, + { id: 5, ip: '91.108.56.191', port: 443, flags: DC_OPTION_FLAG_STATIC } + ] + }; + + const result = mergeFallbackEndpoints(config, [ + { dcId: 5, ip: '91.108.56.201', port: 443 }, + { dcId: 5, ip: '91.108.56.191', port: 443 } + ]); + + expect(result.backupAdded).toEqual(1); + expect(result.duplicatesRemoved).toEqual(1); + expect(config.options.filter(option => option.ip === '91.108.56.191').length).toEqual(1); + expect(config.options.some(option => ( + option.id === 5 + && option.ip === '91.108.56.201' + && option.flags === DC_OPTION_FLAG_STATIC + ))).toEqual(true); + expect(config.options.some(option => ( + option.id === 5 + && option.ip === '2001:b28:f23f:f005::a' + && option.flags === (setBit(DC_OPTION_FLAG_STATIC, DC_OPTION_FLAG_IPV6)) + ))).toEqual(true); + }); + + it('matches expanded and compressed spellings of the same IPv6 endpoint', () => { + const config: MTProtoDCConfig = { + version: 1, + date: 1, + expires: 2, + this_dc: 1, + options: [ + // help.getConfig returns IPv6 fully expanded, the bootstrap list does not. + { id: 1, ip: '2001:0b28:f23d:f001:0000:0000:0000:000a', port: 443, flags: DC_OPTION_FLAG_IPV6 } + ] + }; + + const result = mergeFallbackEndpoints(config, []); + const ipv6Options = config.options.filter(option => option.ip.includes(':')); + + // The DC1 bootstrap merges into the existing option instead of being appended. + expect(ipv6Options.filter(option => option.id === 1).length).toEqual(1); + expect(ipv6Options.some(option => ( + option.id === 1 + && option.ip === '2001:b28:f23d:f001::a' + && option.flags === setBit(DC_OPTION_FLAG_IPV6, DC_OPTION_FLAG_STATIC) + ))).toEqual(true); + // DC2-5 IPv6 bootstraps are still absent from the config, so they get added. + expect(result.bootstrapAdded).toEqual(TELEGRAM_BOOTSTRAP_ENDPOINTS.length - 1); + }); + + it('keeps functional variants separate and preserves backup secrets', () => { + const config: MTProtoDCConfig = { + version: 1, + date: 1, + expires: 2, + this_dc: 2, + options: [ + { id: 2, ip: '149.154.167.50', port: 443, flags: 1 << 3 } + ] + }; + + mergeFallbackEndpoints(config, [{ + dcId: 2, + ip: '149.154.167.50', + port: 443, + secret: Uint8Array.from([1, 2, 3]) + }]); + + expect(config.options.some(option => ( + option.ip === '149.154.167.50' + && option.secret === 'AQID' + && option.flags === (setBit(1 << 10, DC_OPTION_FLAG_STATIC)) + ))).toEqual(true); + }); +}); diff --git a/Build/lib/mtproto-dc-config.ts b/Build/lib/mtproto-dc-config.ts new file mode 100644 index 00000000..7ecfa786 --- /dev/null +++ b/Build/lib/mtproto-dc-config.ts @@ -0,0 +1,194 @@ +import { Buffer } from 'node:buffer'; +import { stableHash } from 'stable-hash'; + +import type { Api as TgApi } from 'telegram'; + +import type { TelegramBackupEndpoint } from './get-telegram-backup-ip'; +import { setBit, getBit } from 'foxts/bitwise'; +import { bigint2ip, ip2bigint } from 'fast-cidr-tools'; +import { isProbablyIpv6 } from 'foxts/is-probably-ip'; + +export const DC_OPTION_FLAG_IPV6 = 1 << 0; +export const DC_OPTION_FLAG_MEDIA_ONLY = 1 << 1; +export const DC_OPTION_FLAG_TCPO_ONLY = 1 << 2; +export const DC_OPTION_FLAG_CDN = 1 << 3; +export const DC_OPTION_FLAG_STATIC = 1 << 4; +export const DC_OPTION_FLAG_THIS_PORT_ONLY = 1 << 5; +export const DC_OPTION_FLAG_SECRET = 1 << 10; + +export interface MTProtoDCConfigOption { + id: number, + ip: string, + port: number, + flags: number, + secret?: string +} + +export interface MTProtoDCConfig { + version: 1, + date: number, + expires: number, + this_dc: number, + options: MTProtoDCConfigOption[] +} + +export interface MTProtoEndpoint { + dcId: number, + ip: string, + port: number, + secret?: Uint8Array +} + +export const TELEGRAM_BOOTSTRAP_ENDPOINTS: readonly MTProtoEndpoint[] = [ + { dcId: 1, ip: '149.154.175.50', port: 443 }, + { dcId: 1, ip: '2001:b28:f23d:f001::a', port: 443 }, + { dcId: 2, ip: '149.154.167.50', port: 443 }, + { dcId: 2, ip: '149.154.167.51', port: 443 }, + { dcId: 2, ip: '95.161.76.100', port: 443 }, + { dcId: 2, ip: '2001:67c:4e8:f002::a', port: 443 }, + { dcId: 3, ip: '149.154.175.100', port: 443 }, + { dcId: 3, ip: '2001:b28:f23d:f003::a', port: 443 }, + { dcId: 4, ip: '149.154.167.91', port: 443 }, + { dcId: 4, ip: '2001:67c:4e8:f004::a', port: 443 }, + { dcId: 5, ip: '149.154.171.5', port: 443 }, + { dcId: 5, ip: '2001:b28:f23f:f005::a', port: 443 } +]; + +type DcOptionWithFlags = TgApi.DcOption & { flags?: number }; + +/** + * help.getConfig returns IPv6 addresses fully expanded (leading zeros in every + * hextet, no "::"), while TELEGRAM_BOOTSTRAP_ENDPOINTS is written in the + * canonical RFC 5952 form. Without canonicalizing, mergeEndpoint's string + * compare never matches the two spellings of the same address, so every IPv6 + * bootstrap gets appended as a duplicate option and the live option never + * receives DC_OPTION_FLAG_STATIC. IPv4 has a single spelling and is passed + * through untouched. + */ +function canonicalizeIp(ip: string) { + if (!isProbablyIpv6(ip)) return ip; + return bigint2ip(ip2bigint(ip, 6), 6, true); +} + +function deriveDcOptionFlags(option: TgApi.DcOption) { + let flags = 0; + if (option.ipv6) flags = setBit(flags, DC_OPTION_FLAG_IPV6); + if (option.mediaOnly) flags = setBit(flags, DC_OPTION_FLAG_MEDIA_ONLY); + if (option.tcpoOnly) flags = setBit(flags, DC_OPTION_FLAG_TCPO_ONLY); + if (option.cdn) flags = setBit(flags, DC_OPTION_FLAG_CDN); + if (option.static) flags = setBit(flags, DC_OPTION_FLAG_STATIC); + if (option.thisPortOnly) flags = setBit(flags, DC_OPTION_FLAG_THIS_PORT_ONLY); + if (option.secret) flags = setBit(flags, DC_OPTION_FLAG_SECRET); + return flags; +} + +export function normalizeTelegramConfig(config: TgApi.Config): MTProtoDCConfig { + return { + version: 1, + date: config.date, + expires: config.expires, + this_dc: config.thisDc, + options: config.dcOptions.map((rawOption) => { + const option = rawOption as DcOptionWithFlags; + const normalized: MTProtoDCConfigOption = { + id: option.id, + ip: canonicalizeIp(option.ipAddress), + port: option.port, + flags: Number.isSafeInteger(option.flags) ? option.flags! : deriveDcOptionFlags(option) + }; + + if (option.secret) { + normalized.secret = Buffer.from(option.secret).toString('base64'); + } + + return normalized; + }) + }; +} + +function fallbackFlags(ip: string, hasSecret: boolean) { + let flags = DC_OPTION_FLAG_STATIC; + if (ip.includes(':')) flags = setBit(flags, DC_OPTION_FLAG_IPV6); + if (hasSecret) flags = setBit(flags, DC_OPTION_FLAG_SECRET); + return flags; +} + +function mergeEndpoint(config: MTProtoDCConfig, endpoint: MTProtoEndpoint) { + const secret = endpoint.secret ? Buffer.from(endpoint.secret).toString('base64') : undefined; + const ip = canonicalizeIp(endpoint.ip); + const functionalFlags = setBit(setBit(DC_OPTION_FLAG_MEDIA_ONLY, DC_OPTION_FLAG_TCPO_ONLY), DC_OPTION_FLAG_CDN); + let matched = false; + + for (let i = 0, len = config.options.length; i < len; i++) { + const option = config.options[i]; + if ( + option.id !== endpoint.dcId + || option.ip !== ip + || option.port !== endpoint.port + || getBit(option.flags, functionalFlags) + || option.secret !== secret + ) { + continue; + } + + option.flags = setBit(option.flags, fallbackFlags(ip, secret !== undefined)); + matched = true; + } + + if (!matched) { + config.options.push({ + id: endpoint.dcId, + ip, + port: endpoint.port, + flags: fallbackFlags(ip, secret !== undefined), + ...(!(secret === undefined) && { secret }) + }); + } + + return !matched; +} + +function deduplicateOptions(config: MTProtoDCConfig) { + const previousCount = config.options.length; + const seen = new Set(); + + config.options = config.options.filter((option) => { + const key = stableHash(option); + if (seen.has(key)) return false; + seen.add(key); + return true; + }); + + return previousCount - config.options.length; +} + +export function mergeFallbackEndpoints( + config: MTProtoDCConfig, + backupEndpoints: readonly TelegramBackupEndpoint[] +) { + // mergeEndpoint compares IP strings, so both sides must already be canonical. + // normalizeTelegramConfig canonicalizes the help.getConfig response, but the + // merge cannot assume its input came from there. + for (let i = 0, len = config.options.length; i < len; i++) { + const option = config.options[i]; + option.ip = canonicalizeIp(option.ip); + } + + let backupAdded = 0; + for (let i = 0, len = backupEndpoints.length; i < len; i++) { + const endpoint = backupEndpoints[i]; + if (mergeEndpoint(config, endpoint)) backupAdded++; + } + + let bootstrapAdded = 0; + for (let i = 0, len = TELEGRAM_BOOTSTRAP_ENDPOINTS.length; i < len; i++) { + const endpoint = TELEGRAM_BOOTSTRAP_ENDPOINTS[i]; + if (mergeEndpoint(config, endpoint)) bootstrapAdded++; + } + + return { + backupAdded, + bootstrapAdded, + duplicatesRemoved: deduplicateOptions(config) + }; +} diff --git a/package.json b/package.json index 416c21e6..3b9d3161 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "hntrie": "^1.1.0", "null-prototype-object": "^1.2.7", "picocolors": "^1.1.1", + "stable-hash": "^0.0.6", "tar-fs": "^3.1.3", "telegram": "^2.26.22", "tinyglobby": "^0.2.17", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 95bf8448..f6f90035 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -62,6 +62,9 @@ importers: picocolors: specifier: ^1.1.1 version: 1.1.1 + stable-hash: + specifier: ^0.0.6 + version: 0.0.6 tar-fs: specifier: ^3.1.3 version: 3.1.3 @@ -1856,6 +1859,9 @@ packages: resolution: {integrity: sha512-o3yWv49B/o4QZk5ZcsALc6t0+eCelPc44zZsLtCQnZPDwFpDYSWcDnrv2TtMmMbQ7uKo3J0HTURCqckw23czNQ==} engines: {node: '>=12.0.0'} + stable-hash@0.0.6: + resolution: {integrity: sha512-0afH4mobqTybYZsXImQRLOjHV4gvOW+92HdUIax9t7a8d9v54KWykEuMVIcXhD9BCi+w3kS4x7O6fmZQ3JlG/g==} + stackframe@1.3.4: resolution: {integrity: sha512-oeVtt7eWQS+Na6F//S4kJ2K2VbRlS9D43mAlMyVpVWovy9o+jfgH8O9agzANzaiLjclA0oYzUXEM4PurhSUChw==} @@ -3724,6 +3730,8 @@ snapshots: stable-hash-x@0.2.0: {} + stable-hash@0.0.6: {} + stackframe@1.3.4: {} store2@2.14.4: {}