diff --git a/Build/build-telegram.ts b/Build/build-telegram.ts index 05ee139c..e8ca9ddb 100644 --- a/Build/build-telegram.ts +++ b/Build/build-telegram.ts @@ -8,7 +8,7 @@ import { fastIpVersion } from 'foxts/fast-ip-version'; import { appendArrayInPlace } from 'foxts/append-array-in-place'; import { fetchTelegramBackupEndpoints } from './lib/fetch-telegram-backup-endpoints'; -const buildTelegramCIDR = task(require.main === module, __filename)(async (span) => { +const buildTelegramCIDR = task(require.main === module, 'build-telegram-cidr')(async (span) => { const { timestamp, ipcidr, ipcidr6 } = await span.traceChildAsync('get telegram cidr', async (childSpan) => { const ipcidr: string[] = [ // Unused secret Telegram backup CIDR, announced by AS62041 @@ -73,9 +73,13 @@ import path from 'node:path'; import process from 'node:process'; import { Api as TgApi, TelegramClient as TgClient } from 'telegram'; +import { AuthKey as TgAuthKey } from 'telegram/crypto/AuthKey'; import { Logger as TgLogger, LogLevel as TgLogLevel } from 'telegram/extensions/Logger'; import { ConnectionTCPAbridged as TgConnectionTCPAbridged } from 'telegram/network/connection'; import { MemorySession as TgMemorySession } from 'telegram/sessions'; +import type { Buffer } from 'node:buffer'; +import type { Span } from './trace'; +import { mtprotoAuthKeyStore } from './lib/mtproto-auth-key-store'; import { OUTPUT_INTERNAL_DIR } from './constants/dir'; import { compareAndWriteFile } from './lib/create-file'; @@ -84,14 +88,40 @@ import { normalizeTelegramConfig, TELEGRAM_BOOTSTRAP_ENDPOINTS } from './lib/mtproto-dc-config'; -import type { MTProtoDCConfig } from './lib/mtproto-dc-config'; +import type { MTProtoDCConfig, MTProtoEndpoint } from './lib/mtproto-dc-config'; const TELEGRAM_API_ID = 2040; const OUTPUT_PATH = path.join(OUTPUT_INTERNAL_DIR, 'mtproto-dc-config.json'); -async function fetchConfig(host: string, port: number, dcId: number) { +/** + * How long a connect + help.getConfig with a persisted auth key may take before + * we give the key up. Normally 0.5-1.5s. It has to be a hard deadline: when a DC + * has forgotten the key it answers with transport error -404, and GramJS merely + * logs "Broken authorization key" and fires a connection-state event -- the + * pending request promise never settles. + */ +const PERSISTED_AUTH_KEY_TIMEOUT = 5000; + +class PersistedAuthKeyTimeoutError extends Error { + constructor(dcId: number, options?: ErrorOptions) { + super(`No help.getConfig response within ${PERSISTED_AUTH_KEY_TIMEOUT}ms using the persisted auth key for dc${dcId}`, options); + this.name = 'PersistedAuthKeyTimeoutError'; + } +} + +/** + * One connect + help.getConfig against a DC. With a persisted auth key the + * connect skips the DH handshake (three round trips); without one, GramJS + * negotiates a key and we persist it for the next build. + */ +async function fetchConfig(host: string, port: number, dcId: number, persistedAuthKey: Buffer | null) { const session = new TgMemorySession(); session.setDC(dcId, host, port); + if (persistedAuthKey) { + const authKey = new TgAuthKey(); + await authKey.setKey(persistedAuthKey); + session.setAuthKey(authKey, dcId); + } const client = new TgClient(session, TELEGRAM_API_ID, 'not-used-for-unauthenticated-rpc', { appVersion: '1.0', @@ -111,19 +141,73 @@ async function fetchConfig(host: string, port: number, dcId: number) { useWSS: true }); - try { + const work = async () => { const connected = await client.connect(); if (!connected && !client.connected) { throw new Error('MTProto client did not connect'); } - return normalizeTelegramConfig(await client.invoke(new TgApi.help.GetConfig())); + const config = normalizeTelegramConfig(await client.invoke(new TgApi.help.GetConfig())); + + if (!persistedAuthKey) { + const negotiated = client.session.authKey?.getKey(); + if (negotiated) { + await mtprotoAuthKeyStore.save(dcId, negotiated); + } + } + + return config; + }; + + let timer: NodeJS.Timeout | null = null; + try { + if (!persistedAuthKey) { + return await work(); + } + return await Promise.race([ + work(), + new Promise((_resolve, reject) => { + timer = setTimeout(reject, PERSISTED_AUTH_KEY_TIMEOUT, new PersistedAuthKeyTimeoutError(dcId)); + }) + ]); } finally { - await client.disconnect(); + if (timer) { + clearTimeout(timer); + } + // destroy, not disconnect: a sender stuck on a rejected key must not keep the + // process alive after we have moved on to a fresh handshake + await client.destroy(); } } -async function fetchConfigFromBootstrapEndpoints() { +async function fetchConfigFromEndpoint(span: Span, endpoint: MTProtoEndpoint) { + const label = `${endpoint.ip}:${endpoint.port}`; + + const persistedAuthKey = await mtprotoAuthKeyStore.load(endpoint.dcId); + if (persistedAuthKey) { + try { + return await span.traceChildAsync( + `help.getConfig via ${label} (persisted auth key)`, + () => fetchConfig(endpoint.ip, endpoint.port, endpoint.dcId, persistedAuthKey), + SpanCategory.Network + ); + } catch (error) { + // Most likely the DC no longer knows this key, which surfaces as the + // timeout above (see PERSISTED_AUTH_KEY_TIMEOUT). Whatever it was, a key + // that fails once is not worth a second try: fall through to a fresh handshake. + console.warn(`[telegram mtproto config] ${label} failed with the persisted auth key for dc${endpoint.dcId}, renegotiating`, error); + await mtprotoAuthKeyStore.drop(endpoint.dcId); + } + } + + return span.traceChildAsync( + `help.getConfig via ${label} (fresh handshake)`, + () => fetchConfig(endpoint.ip, endpoint.port, endpoint.dcId, null), + SpanCategory.Network + ); +} + +async function fetchConfigFromBootstrapEndpoints(span: Span) { let lastError: unknown; for (let i = 0, len = TELEGRAM_BOOTSTRAP_ENDPOINTS.length; i < len; i++) { @@ -132,7 +216,7 @@ async function fetchConfigFromBootstrapEndpoints() { try { // Bootstrap order is significant, and one successful response ends the loop. // eslint-disable-next-line no-await-in-loop -- Bootstrap endpoints must be attempted in order. - return await fetchConfig(endpoint.ip, endpoint.port, endpoint.dcId); + return await fetchConfigFromEndpoint(span, endpoint); } catch (error) { lastError = error; console.error(`[telegram mtproto config] ${endpoint.ip}:${endpoint.port} failed`, error); @@ -145,7 +229,7 @@ async function fetchConfigFromBootstrapEndpoints() { ); } -export const buildMTProtoDCConfig = task(require.main === module, __filename)(async (span) => { +export const buildMTProtoDCConfig = task(require.main === module, 'build-mtproto-dc-config')(async (span) => { const config = await span.traceChildAsync( 'fetch help.getConfig', fetchConfigFromBootstrapEndpoints, diff --git a/Build/lib/mtproto-auth-key-store.test.ts b/Build/lib/mtproto-auth-key-store.test.ts new file mode 100644 index 00000000..f8004da6 --- /dev/null +++ b/Build/lib/mtproto-auth-key-store.test.ts @@ -0,0 +1,52 @@ +import { describe, it } from 'mocha'; +import { expect } from 'earl'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { randomBytes } from 'node:crypto'; + +import { createMTProtoAuthKeyStore } from './mtproto-auth-key-store'; + +function tmpStorePath() { + return path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'mtproto-auth-key-')), 'keys'); +} + +describe('mtproto auth key store', () => { + it('round-trips a key per DC and survives reopening the directory', async () => { + const dir = tmpStorePath(); + const dc1 = randomBytes(256); + const dc2 = randomBytes(256); + + const store = createMTProtoAuthKeyStore(dir); + expect(await store.load(1)).toEqual(null); + await store.save(1, dc1); + await store.save(2, dc2); + expect((await store.load(1))?.equals(dc1)).toEqual(true); + expect((await store.load(2))?.equals(dc2)).toEqual(true); + + // a fresh handle on the same directory is what the next build sees + const reopened = createMTProtoAuthKeyStore(dir); + expect((await reopened.load(1))?.equals(dc1)).toEqual(true); + }); + + it('replaces on save and forgets on drop', async () => { + const store = createMTProtoAuthKeyStore(tmpStorePath()); + const first = randomBytes(256); + const second = randomBytes(256); + + await store.save(4, first); + await store.save(4, second); + expect((await store.load(4))?.equals(second)).toEqual(true); + + await store.drop(4); + expect(await store.load(4)).toEqual(null); + // dropping an absent key is a no-op + await store.drop(4); + }); + + it('treats a key of the wrong size as absent', async () => { + const store = createMTProtoAuthKeyStore(tmpStorePath()); + await store.save(5, randomBytes(16)); + expect(await store.load(5)).toEqual(null); + }); +}); diff --git a/Build/lib/mtproto-auth-key-store.ts b/Build/lib/mtproto-auth-key-store.ts new file mode 100644 index 00000000..96f4f591 --- /dev/null +++ b/Build/lib/mtproto-auth-key-store.ts @@ -0,0 +1,48 @@ +import path from 'node:path'; +import cacache from 'cacache'; +import type { Buffer } from 'node:buffer'; +import { CACHE_DIR } from '../constants/dir'; + +export interface MTProtoAuthKeyStore { + load(dcId: number): Promise, + save(dcId: number, key: Buffer): Promise, + drop(dcId: number): Promise +} + +/** a permanent auth key is exactly 2048 bits */ +const AUTH_KEY_BYTES = 256; + +const VERIFY_INTERVAL = 30 * 24 * 60 * 60 * 1000; +async function verifyIfStale(cachePath: string) { + const lastRun = await cacache.verify.lastRun(cachePath).catch(() => null); + if (lastRun === null || Date.now() - lastRun.getTime() > VERIFY_INTERVAL) { + await cacache.verify(cachePath); + } +} + +const entryKey = (dcId: number) => `dc${dcId}`; + +export function createMTProtoAuthKeyStore(cachePath: string): MTProtoAuthKeyStore { + return { + async load(dcId) { + let data: Buffer; + try { + ({ data } = await cacache.get(cachePath, entryKey(dcId))); + } catch { + // ENOENT (never negotiated) or EINTEGRITY (corrupt), both mean "no key" + return null; + } + return data.length === AUTH_KEY_BYTES ? data : null; + }, + async save(dcId, key) { + await cacache.put(cachePath, entryKey(dcId), key); + await verifyIfStale(cachePath); + }, + async drop(dcId) { + await cacache.rm.entry(cachePath, entryKey(dcId)); + await verifyIfStale(cachePath); + } + }; +} + +export const mtprotoAuthKeyStore = createMTProtoAuthKeyStore(path.join(CACHE_DIR, 'telegram-mtproto-auth-keys')); diff --git a/Build/trace/index.ts b/Build/trace/index.ts index 270334c3..131701e9 100644 --- a/Build/trace/index.ts +++ b/Build/trace/index.ts @@ -65,22 +65,30 @@ export function makeSpan(rawSpan: RawSpan): Span { return span; }, traceChild, + // Spans are stopped in `finally` so a throwing function is still measured + // (and the tree does not end up with an "(unfinished)" hole where it failed). traceSyncFn(fn: (span: Span) => T) { traceResult.sync = true; - const res = fn(span); - span.stop(); - return res; + try { + return fn(span); + } finally { + span.stop(); + } }, async traceAsyncFn(fn: (span: Span) => T | Promise): Promise { - const res = await fn(span); - span.stop(); - return res; + try { + return await fn(span); + } finally { + span.stop(); + } }, traceResult, async tracePromise(promise: Promise): Promise { - const res = await promise; - span.stop(); - return res; + try { + return await promise; + } finally { + span.stop(); + } }, traceChildSync: (name: string, fn: (span: Span) => T, category?: SpanCategory): T => traceChild(name, category).traceSyncFn(fn), traceChildAsync: (name: string, fn: (span: Span) => T | Promise, category?: SpanCategory): Promise => traceChild(name, category).traceAsyncFn(fn), diff --git a/package.json b/package.json index c86e84e4..f0119dbc 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "@ghostery/adblocker": "^2.18.2", "@henrygd/queue": "^1.2.0", "@mitata/counters": "^0.0.8", + "cacache": "^21.0.1", "ci-info": "^4.4.0", "cli-progress": "^3.12.0", "csv-parse": "^7.0.2", @@ -53,6 +54,7 @@ "@eslint-sukka/node": "^9.0.1", "@swc-node/register": "^1.12.1", "@swc/core": "1.16.1", + "@types/cacache": "^20.0.1", "@types/cli-progress": "^3.11.6", "@types/mocha": "^10.0.10", "@types/node": "^26.4.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index da62fe20..f72c4892 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -124,6 +124,9 @@ importers: '@mitata/counters': specifier: ^0.0.8 version: 0.0.8 + cacache: + specifier: ^21.0.1 + version: 21.0.1 ci-info: specifier: ^4.4.0 version: 4.4.0 @@ -209,6 +212,9 @@ importers: '@swc/core': specifier: 1.16.1 version: 1.16.1 + '@types/cacache': + specifier: ^20.0.1 + version: 20.0.1 '@types/cli-progress': specifier: ^3.11.6 version: 3.11.6 @@ -581,6 +587,10 @@ packages: '@nolyfill/shared@1.0.44': resolution: {integrity: sha512-NI1zxDh4LYL7PYlKKCwojjuc5CEZslywrOTKBNyodjmWjRiZ4AlCMs3Gp+zDoPQPNkYCSQp/luNojHmJWWfCbw==} + '@npmcli/fs@6.0.0': + resolution: {integrity: sha512-AheOs4swKka/XLtht6xxJDPezlQ7K2IYQ9Y8lST4JLDjnralnWuMM9AE2CdVcgQJ5omrXhsRzM7F7aYmeZBvKQ==} + engines: {node: ^22.22.2 || ^24.15.0 || >=26.0.0} + '@ota-meshi/ast-token-store@0.3.0': resolution: {integrity: sha512-XRO0zi2NIUKq2lUk3T1ecFSld1fMWRKE6naRFGkgkdeosx7IslyUKNv5Dcb5PJTja9tHJoFu0v/7yEpAkrkrTg==} engines: {node: ^20.19.0 || ^22.13.0 || >=24} @@ -828,6 +838,9 @@ packages: '@tybys/wasm-util@0.10.2': resolution: {integrity: sha512-RoBvJ2X0wuKlWFIjrwffGw1IqZHKQqzIchKaadZZfnNpsAYp2mM0h36JtPCjNDAHGgYez/15uMBpfGwchhiMgg==} + '@types/cacache@20.0.1': + resolution: {integrity: sha512-QlKW3AFoFr/hvPHwFHMIVUH/ZCYeetBNou3PCmxu5LaNDvrtBlPJtIA6uhmU9JRt9oxj7IYoqoLcpxtzpPiTcw==} + '@types/cli-progress@3.11.6': resolution: {integrity: sha512-cE3+jb9WRlu+uOSAugewNpITJDt1VF8dHOopPO4IABFc3SXYL5WE/+PTz/FCdZRRfIujiWW3n3aMbv1eIGVRWA==} @@ -1128,6 +1141,10 @@ packages: resolution: {integrity: sha512-WDtdLmJvAuNNPzByAYpRo2rF1Mmradw6gvWsQKf63476DDXmomT9zUiGypLcG4ibIM67vhAj8jJRdbmEws2Aqw==} engines: {node: '>=6.14.2'} + cacache@21.0.1: + resolution: {integrity: sha512-pTwz/uj3Jyp6WXdJ6fWhR+7LVxVs6RyroQSn7KJwHsSxXuyGSp0pcMVcwSwTpCFq1X2YG8QBe0W+vN+cr0SwzA==} + engines: {node: ^22.22.2 || ^24.15.0 || >=26.0.0} + chalk@4.1.2: resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} engines: {node: '>=10'} @@ -1478,6 +1495,10 @@ packages: foxts@5.9.1: resolution: {integrity: sha512-NrIHGlf4LZGTjjGlVi3gmMbtvC4871SFUflu9yb8X91FYppAk+WMiYS6ZlaEgkPqAkgIGNeq+Y5YCL80GL6v8g==} + fs-minipass@3.0.3: + resolution: {integrity: sha512-XUBA9XClHbnJWSfBzjkm6RvPsyg3sryZt06BEQoXcF7EK/xpGaQYJgQKDJSUH5SGZ76Y7pFx1QBnXz09rU5Fbw==} + engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} + get-tsconfig@4.12.0: resolution: {integrity: sha512-LScr2aNr2FbjAjZh2C6X6BxRx1/x+aTDExct/xyq2XKbYOiG5c0aK7pMsSuyc0brz3ibr/lbQiHD9jzt4lccJw==} @@ -1624,6 +1645,22 @@ packages: resolution: {integrity: sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==} engines: {node: 18 || 20 || >=22} + minipass-collect@2.0.1: + resolution: {integrity: sha512-D7V8PO9oaz7PWGLbCACuI1qEOsq7UKfLotx/C0Aet43fCUB/wfQ7DYeq2oR/svFJGYDHPr38SHATeaj/ZoKHKw==} + engines: {node: '>=16 || 14 >=14.17'} + + minipass-flush@1.0.7: + resolution: {integrity: sha512-TbqTz9cUwWyHS2Dy89P3ocAGUGxKjjLuR9z8w4WUTGAVgEj17/4nhgo2Du56i0Fm3Pm30g4iA8Lcqctc76jCzA==} + engines: {node: '>= 8'} + + minipass-pipeline@1.2.4: + resolution: {integrity: sha512-xuIq7cIOt09RPRJ19gdi4b+RiNvDFYe5JH+ggNvBqGqpQXcru3PcRmOZuHBKWK1Txf9+cQ+HMVN4d6z46LZP7A==} + engines: {node: '>=8'} + + minipass@3.3.6: + resolution: {integrity: sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==} + engines: {node: '>=8'} + minipass@7.1.3: resolution: {integrity: sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==} engines: {node: '>=16 || 14 >=14.17'} @@ -1687,6 +1724,10 @@ packages: resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==} engines: {node: '>=10'} + p-map@7.0.7: + resolution: {integrity: sha512-VaWRu2i4FJNRtiRWCuuQRgfQ1B7a6+gMSrO+3j0EQi/k0ULfS9kosRxGoiqwzIjZTDI02tGfk5mXXltLg6QtfQ==} + engines: {node: '>=18'} + pako@2.1.0: resolution: {integrity: sha512-w+eufiZ1WuJYgPXbV/PO3NCMEc3xqylkKHzp8bxp1uW4qaSNQUkwmLLEc3kKsfz8lpV1F8Ht3U1Cm+9Srog2ug==} @@ -1788,6 +1829,10 @@ packages: resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==} engines: {node: '>=0.10.0'} + ssri@14.0.0: + resolution: {integrity: sha512-jQxKI0yx0ZnTKrqjKkLDV2DXkBQn3k49JVmVqDGcDwKDtGDbImD/GXsq04KD0VVzCQQ9wZJYal3RwR1GzWTSow==} + engines: {node: ^22.22.2 || ^24.15.0 || >=26.0.0} + stable-hash-x@0.2.0: resolution: {integrity: sha512-o3yWv49B/o4QZk5ZcsALc6t0+eCelPc44zZsLtCQnZPDwFpDYSWcDnrv2TtMmMbQ7uKo3J0HTURCqckw23czNQ==} engines: {node: '>=12.0.0'} @@ -1960,6 +2005,9 @@ packages: engines: {node: '>=0.10.32'} deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. + yallist@4.0.0: + resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} + yaml@2.9.0: resolution: {integrity: sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==} engines: {node: '>= 14.6'} @@ -2266,6 +2314,10 @@ snapshots: '@nolyfill/shared@1.0.44': {} + '@npmcli/fs@6.0.0': + dependencies: + semver: 7.7.3 + '@ota-meshi/ast-token-store@0.3.0': {} '@oxc-resolver/binding-android-arm-eabi@11.20.0': @@ -2444,6 +2496,11 @@ snapshots: tslib: 2.8.1 optional: true + '@types/cacache@20.0.1': + dependencies: + '@types/node': 26.4.1 + minipass: 7.1.3 + '@types/cli-progress@3.11.6': dependencies: '@types/node': 26.4.1 @@ -2720,6 +2777,19 @@ snapshots: dependencies: node-gyp-build: 4.8.4 + cacache@21.0.1: + dependencies: + '@npmcli/fs': 6.0.0 + fs-minipass: 3.0.3 + glob: 13.0.6 + lru-cache: 11.5.2 + minipass: 7.1.3 + minipass-collect: 2.0.1 + minipass-flush: 1.0.7 + minipass-pipeline: 1.2.4 + p-map: 7.0.7 + ssri: 14.0.0 + chalk@4.1.2: dependencies: ansi-styles: 4.3.0 @@ -3145,6 +3215,10 @@ snapshots: fast-fnv1a: 1.0.0 fnv1a52: 1.0.0 + fs-minipass@3.0.3: + dependencies: + minipass: 7.1.3 + get-tsconfig@4.12.0: dependencies: resolve-pkg-maps: 1.0.0 @@ -3274,6 +3348,22 @@ snapshots: dependencies: brace-expansion: 5.0.8 + minipass-collect@2.0.1: + dependencies: + minipass: 7.1.3 + + minipass-flush@1.0.7: + dependencies: + minipass: 3.3.6 + + minipass-pipeline@1.2.4: + dependencies: + minipass: 3.3.6 + + minipass@3.3.6: + dependencies: + yallist: 4.0.0 + minipass@7.1.3: {} mitata@1.0.34: {} @@ -3360,6 +3450,8 @@ snapshots: dependencies: p-limit: 3.1.0 + p-map@7.0.7: {} + pako@2.1.0: {} path-browserify@1.0.1: {} @@ -3437,6 +3529,10 @@ snapshots: source-map@0.6.1: {} + ssri@14.0.0: + dependencies: + minipass: 7.1.3 + stable-hash-x@0.2.0: {} stable-hash@0.0.6: {} @@ -3648,6 +3744,8 @@ snapshots: yaeti@0.0.6: {} + yallist@4.0.0: {} + yaml@2.9.0: {} yauzl-promise@4.0.0: