Feat: Surge Compatible MTProto DC Config

Build/build-telegram-cidr.worker.ts

@@ -1,168 +0,0 @@
-// @ts-check
-import { task } from './trace';
-import { SHARED_DESCRIPTION } from './constants/description';
-import { RulesetOutput } from './lib/rules/ruleset';
-import { getTelegramBackupIPFromBase64 } from './lib/get-telegram-backup-ip';
-import picocolors from 'picocolors';
-import { $$fetch } from './lib/fetch-retry';
-import dns from 'node:dns/promises';
-import { createReadlineInterfaceFromResponse } from './lib/fetch-text-by-line';
-import { fastIpVersion } from 'foxts/fast-ip-version';
-import { fastStringArrayJoin } from 'foxts/fast-string-array-join';
-import { appendArrayInPlace } from 'foxts/append-array-in-place';
-
-export const buildTelegramCIDR = task(require.main === module, __filename)(async (span) => {
-  const { timestamp, ipcidr, ipcidr6 } = await span.traceChildAsync('get telegram cidr', async (childSpan) => {
-    const ipcidr: string[] = [
-      // Unused secret Telegram backup CIDR, announced by AS62041
-      '95.161.64.0/20'
-    ];
-    const ipcidr6: string[] = [];
-
-    const date = await childSpan.traceChildAsync('fetch from official cidr list', async () => {
-      const resp = await $$fetch('https://core.telegram.org/resources/cidr.txt');
-      const lastModified = resp.headers.get('last-modified');
-
-      for await (const cidr of createReadlineInterfaceFromResponse(resp, true)) {
-        const v = fastIpVersion(cidr);
-        if (v === 4) {
-          ipcidr.push(cidr);
-        } else if (v === 6) {
-          ipcidr6.push(cidr);
-        }
-      }
-
-      return lastModified ? new Date(lastModified) : new Date();
-    });
-
-    // https://github.com/tdlib/td/blob/master/td/telegram/ConfigManager.cpp
-    const backupIPs = await childSpan.traceChildAsync('fetch backup ip', async (innerSpan) => {
-      const backupIPs = new Set<string>();
-      const resolvers = ['8.8.8.8', '1.0.0.1'].map((ip) => {
-        const resolver = new dns.Resolver();
-        resolver.setServers([ip]);
-        return Object.assign(resolver, { server: ip });
-      });
-
-      await innerSpan.traceChildAsync('backup source 1: DNS TXT', () => Promise.all(resolvers.flatMap((resolver) => [
-        'apv3.stel.com', // prod
-        'tapv3.stel.com' // test
-      ].map(async (domain) => {
-        try {
-        // tapv3.stel.com was for testing server
-          const resp = await resolver.resolveTxt(domain);
-          const strings = resp.map(r => fastStringArrayJoin(r, '')); // flatten
-          if (strings.length !== 2) {
-            throw new TypeError(`Unexpected TXT record count: ${strings.length}`);
-          }
-
-          const str = strings[0].length > strings[1].length
-            ? strings[0] + strings[1]
-            : strings[1] + strings[0];
-
-          const ips = getTelegramBackupIPFromBase64(str);
-          ips.forEach(i => backupIPs.add(i.ip));
-
-          console.log('[telegram backup ip]', picocolors.green('DNS TXT'), { domain, ips, server: resolver.server });
-        } catch (e) {
-          console.error('[telegram backup ip]', picocolors.red('DNS TXT error'), { domain }, e);
-        }
-      }))));
-
-      // Backup IP Source 2: Firebase Realtime Database (test server not supported)
-      await innerSpan.traceChildAsync('backup source 2: Firebase Realtime DB', async () => {
-        try {
-          const text = await (await $$fetch('https://reserve-5a846.firebaseio.com/ipconfigv3.json')).json();
-          if (typeof text === 'string' && text.length === 344) {
-            const ips = getTelegramBackupIPFromBase64(text);
-            ips.forEach(i => backupIPs.add(i.ip));
-
-            console.log('[telegram backup ip]', picocolors.green('Firebase Realtime DB'), { ips });
-          }
-        } catch (e) {
-          console.error('[telegram backup ip]', picocolors.red('Firebase Realtime DB error'), e);
-          // ignore all errors
-        }
-      });
-
-      // Backup IP Source 3: Firebase Value Store (test server not supported)
-      await innerSpan.traceChildAsync('backup source 3: Firebase Value Store', async () => {
-        try {
-          const json = await (await $$fetch('https://firestore.googleapis.com/v1/projects/reserve-5a846/databases/(default)/documents/ipconfig/v3', {
-            headers: {
-              Accept: '*/*',
-              Origin: undefined // Without this line, Google API will return "Bad request: Origin doesn't match Host for XD3.". Probably have something to do with sqlite cache store
-            }
-          })).json();
-
-          if (
-            json && typeof json === 'object'
-            && 'fields' in json && typeof json.fields === 'object' && json.fields
-            && 'data' in json.fields && typeof json.fields.data === 'object' && json.fields.data
-            && 'stringValue' in json.fields.data && typeof json.fields.data.stringValue === 'string' && json.fields.data.stringValue.length === 344
-          ) {
-            const ips = getTelegramBackupIPFromBase64(json.fields.data.stringValue);
-            ips.forEach(i => backupIPs.add(i.ip));
-
-            console.log('[telegram backup ip]', picocolors.green('Firebase Value Store'), { ips });
-          } else {
-            console.error('[telegram backup ip]', picocolors.red('Firebase Value Store data format invalid'), { json });
-          }
-        } catch (e) {
-          console.error('[telegram backup ip]', picocolors.red('Firebase Value Store error'), e);
-        }
-      });
-
-      // Backup IP Source 4: Google App Engine
-      await innerSpan.traceChildAsync('backup source 4: Google App Engine', () => Promise.all([
-        'https://dns-telegram.appspot.com',
-        'https://dns-telegram.appspot.com/test'
-      ].map(async (url) => {
-        try {
-          const text = await (await $$fetch(url)).text();
-          if (text.length === 344) {
-            const ips = getTelegramBackupIPFromBase64(text);
-            ips.forEach(i => backupIPs.add(i.ip));
-
-            console.log('[telegram backup ip]', picocolors.green('Google App Engine'), { url, ips });
-          }
-        } catch (e) {
-          console.error('[telegram backup ip]', picocolors.red('Google App Engine error'), { url }, e);
-        }
-      })));
-
-      // tcdnb.azureedge.net no longer works
-
-      return backupIPs;
-    });
-
-    console.log('[telegram backup ip]', `Found ${backupIPs.size} backup IPs:`, backupIPs);
-
-    appendArrayInPlace(ipcidr, Array.from(backupIPs).map(i => i + '/32'));
-
-    return { timestamp: date.getTime(), ipcidr, ipcidr6 };
-  });
-
-  if (ipcidr.length + ipcidr6.length === 0) {
-    throw new Error('Failed to fetch data!');
-  }
-
-  const description = [
-    ...SHARED_DESCRIPTION,
-    'Data from:',
-    ' - https://core.telegram.org/resources/cidr.txt'
-  ];
-
-  return new RulesetOutput(span, 'telegram', 'ip')
-    .withTitle('Sukka\'s Ruleset - Telegram IP CIDR')
-    .withDescription(description)
-    // .withDate(date) // With extra data source, we no longer use last-modified for file date
-    .appendDataSource(
-      'https://core.telegram.org/resources/cidr.txt (last updated: ' + new Date(timestamp).toISOString() + ')'
-    )
-    .bulkAddCIDR4NoResolve(ipcidr)
-    .bulkAddCIDR6NoResolve(ipcidr6)
-    .write();
-});
-
-export const ___ = '';

Build/build-telegram.worker.ts

@@ -0,0 +1,179 @@
+// @ts-check
+import { task } from './trace';
+import { SHARED_DESCRIPTION } from './constants/description';
+import { RulesetOutput } from './lib/rules/ruleset';
+import { $$fetch } from './lib/fetch-retry';
+import { createReadlineInterfaceFromResponse } from './lib/fetch-text-by-line';
+import { fastIpVersion } from 'foxts/fast-ip-version';
+import { appendArrayInPlace } from 'foxts/append-array-in-place';
+import { fetchTelegramBackupEndpoints } from './lib/fetch-telegram-backup-endpoints';
+
+const buildTelegramCIDR = task(require.main === module, __filename)(async (span) => {
+  const { timestamp, ipcidr, ipcidr6 } = await span.traceChildAsync('get telegram cidr', async (childSpan) => {
+    const ipcidr: string[] = [
+      // Unused secret Telegram backup CIDR, announced by AS62041
+      '95.161.64.0/20'
+    ];
+    const ipcidr6: string[] = [];
+
+    const date = await childSpan.traceChildAsync('fetch from official cidr list', async () => {
+      const resp = await $$fetch('https://core.telegram.org/resources/cidr.txt');
+      const lastModified = resp.headers.get('last-modified');
+
+      for await (const cidr of createReadlineInterfaceFromResponse(resp, true)) {
+        const v = fastIpVersion(cidr);
+        if (v === 4) {
+          ipcidr.push(cidr);
+        } else if (v === 6) {
+          ipcidr6.push(cidr);
+        }
+      }
+
+      return lastModified ? new Date(lastModified) : new Date();
+    });
+
+    // https://github.com/tdlib/td/blob/master/td/telegram/ConfigManager.cpp
+    const backupEndpoints = await childSpan.traceChildAsync(
+      'fetch backup ip',
+      innerSpan => fetchTelegramBackupEndpoints(innerSpan, { includeTestServers: true })
+    );
+    const backupIPs = new Set(backupEndpoints.map(endpoint => endpoint.ip));
+
+    console.log('[telegram backup ip]', `Found ${backupIPs.size} backup IPs:`, backupIPs);
+
+    appendArrayInPlace(ipcidr, Array.from(backupIPs, i => i + '/32'));
+
+    return { timestamp: date.getTime(), ipcidr, ipcidr6 };
+  });
+
+  if (ipcidr.length + ipcidr6.length === 0) {
+    throw new Error('Failed to fetch data!');
+  }
+
+  const description = [
+    ...SHARED_DESCRIPTION,
+    'Data from:',
+    ' - https://core.telegram.org/resources/cidr.txt'
+  ];
+
+  return new RulesetOutput(span, 'telegram', 'ip')
+    .withTitle('Sukka\'s Ruleset - Telegram IP CIDR')
+    .withDescription(description)
+    // .withDate(date) // With extra data source, we no longer use last-modified for file date
+    .appendDataSource(
+      'https://core.telegram.org/resources/cidr.txt (last updated: ' + new Date(timestamp).toISOString() + ')'
+    )
+    .bulkAddCIDR4NoResolve(ipcidr)
+    .bulkAddCIDR6NoResolve(ipcidr6)
+    .write();
+});
+
+// @ts-check
+import path from 'node:path';
+import process from 'node:process';
+
+import { Api as TgApi, TelegramClient as TgClient } from 'telegram';
+import { Logger as TgLogger, LogLevel as TgLogLevel } from 'telegram/extensions/Logger';
+import { ConnectionTCPAbridged as TgConnectionTCPAbridged } from 'telegram/network/connection';
+import { MemorySession as TgMemorySession } from 'telegram/sessions';
+
+import { OUTPUT_INTERNAL_DIR } from './constants/dir';
+import { compareAndWriteFile } from './lib/create-file';
+import {
+  mergeFallbackEndpoints,
+  normalizeTelegramConfig,
+  TELEGRAM_BOOTSTRAP_ENDPOINTS
+} from './lib/mtproto-dc-config';
+import type { MTProtoDCConfig } from './lib/mtproto-dc-config';
+
+const TELEGRAM_API_ID = 2040;
+const OUTPUT_PATH = path.join(OUTPUT_INTERNAL_DIR, 'mtproto-dc-config.json');
+
+async function fetchConfig(host: string, port: number, dcId: number) {
+  const session = new TgMemorySession();
+  session.setDC(dcId, host, port);
+
+  const client = new TgClient(session, TELEGRAM_API_ID, 'not-used-for-unauthenticated-rpc', {
+    appVersion: '1.0',
+    autoReconnect: false,
+    baseLogger: new TgLogger(TgLogLevel.NONE),
+    connection: TgConnectionTCPAbridged,
+    connectionRetries: 1,
+    deviceModel: 'Surge',
+    langCode: 'en',
+    reconnectRetries: 0,
+    requestRetries: 1,
+    securityChecks: true,
+    systemLangCode: 'en',
+    systemVersion: process.platform,
+    timeout: 10,
+    // GramJS uses this option to select port 443 even for a raw TCP connection.
+    useWSS: true
+  });
+
+  try {
+    const connected = await client.connect();
+    if (!connected && !client.connected) {
+      throw new Error('MTProto client did not connect');
+    }
+
+    return normalizeTelegramConfig(await client.invoke(new TgApi.help.GetConfig()));
+  } finally {
+    await client.disconnect();
+  }
+}
+
+async function fetchConfigFromBootstrapEndpoints() {
+  let lastError: unknown;
+
+  for (let i = 0, len = TELEGRAM_BOOTSTRAP_ENDPOINTS.length; i < len; i++) {
+    const endpoint = TELEGRAM_BOOTSTRAP_ENDPOINTS[i];
+    console.log(`[telegram mtproto config] Fetching help.getConfig from ${endpoint.ip}:${endpoint.port}`);
+    try {
+      // Bootstrap order is significant, and one successful response ends the loop.
+      // eslint-disable-next-line no-await-in-loop -- Bootstrap endpoints must be attempted in order.
+      return await fetchConfig(endpoint.ip, endpoint.port, endpoint.dcId);
+    } catch (error) {
+      lastError = error;
+      console.error(`[telegram mtproto config] ${endpoint.ip}:${endpoint.port} failed`, error);
+    }
+  }
+
+  throw new AggregateError(
+    lastError === undefined ? [] : [lastError],
+    'All Telegram MTProto bootstrap endpoints failed'
+  );
+}
+
+export const buildMTProtoDCConfig = task(require.main === module, __filename)(async (span) => {
+  const config = await span.traceChildAsync(
+    'fetch help.getConfig',
+    fetchConfigFromBootstrapEndpoints
+  );
+
+  const backupEndpoints = await span.traceChildAsync(
+    'fetch telegram backup endpoints',
+    childSpan => fetchTelegramBackupEndpoints(childSpan, { includeTestServers: false })
+  );
+
+  const liveEndpoints = config.options.length;
+  const mergeResult = mergeFallbackEndpoints(config, backupEndpoints);
+  console.log('[telegram mtproto config]', {
+    liveEndpoints,
+    backupEndpoints: backupEndpoints.length,
+    ...mergeResult,
+    outputEndpoints: config.options.length
+  });
+
+  const output = JSON.stringify(config satisfies MTProtoDCConfig, null, 2).split('\n');
+  await compareAndWriteFile(span, output, OUTPUT_PATH);
+});
+
+// Start both tasks in this worker concurrently. They retain independent trace
+// results while sharing the module-scoped production backup endpoint promise.
+export function buildTelegram() {
+  return Promise.all([
+    buildTelegramCIDR(),
+    buildMTProtoDCConfig()
+  ]);
+}

Build/index.ts

@@ -30,6 +30,7 @@ import { ROOT_DIR } from './constants/dir';
 import { isCI } from 'ci-info';
 import { printExternalDownloadStats } from './lib/download-stats';
 import { endOutputWorkerFarm, warmOutputWorkerFarm } from './lib/rules/output-worker-farm';
+import { appendArrayInPlace } from 'foxts/append-array-in-place';
 
 process.on('uncaughtException', (error) => {
   console.error('Uncaught exception:', error);
@@ -78,9 +79,9 @@ const buildFinishedLock = path.join(ROOT_DIR, '.BUILD_FINISHED');
     require.resolve('./build-cdn-download-conf.worker')
   )(['buildCdnDownloadConf']);
 
-  const telegramCidrWorker = createWorker<typeof import('./build-telegram-cidr.worker')>(
+  const telegramWorker = createWorker<typeof import('./build-telegram.worker')>(
-    require.resolve('./build-telegram-cidr.worker')
+    require.resolve('./build-telegram.worker')
-  )(['buildTelegramCIDR']);
+  )(['buildTelegram']);
 
   const mockAssetsWorker = createWorker<typeof import('./download-mock-assets.worker')>(
     require.resolve('./download-mock-assets.worker')
@@ -99,31 +100,31 @@ const buildFinishedLock = path.join(ROOT_DIR, '.BUILD_FINISHED');
 
     const downloadPreviousBuildPromise = downloadPreviousBuild();
 
-    const traces: TraceResult[] = await Promise.all([
+    const [traces, telegramTraces]: [TraceResult[], TraceResult[]] = await Promise.all([
-      downloadPreviousBuildPromise,
+      Promise.all([
-      downloadPreviousBuildPromise.then(() => buildCommon()),
+        downloadPreviousBuildPromise,
-      downloadPreviousBuildPromise.then(() => buildRejectIPList()),
+        downloadPreviousBuildPromise.then(() => buildCommon()),
-      downloadPreviousBuildPromise.then(() => buildAppleCdn()),
+        downloadPreviousBuildPromise.then(() => buildRejectIPList()),
-      downloadPreviousBuildPromise.then(() => buildAICIDR()),
+        downloadPreviousBuildPromise.then(() => buildAppleCdn()),
-      downloadPreviousBuildPromise.then(() => cdnDownloadWorker.buildCdnDownloadConf()),
+        downloadPreviousBuildPromise.then(() => buildAICIDR()),
-      downloadPreviousBuildPromise.then(() => buildRejectDomainSet()),
+        downloadPreviousBuildPromise.then(() => cdnDownloadWorker.buildCdnDownloadConf()),
-      downloadPreviousBuildPromise.then(() => telegramCidrWorker.buildTelegramCIDR()),
+        downloadPreviousBuildPromise.then(() => buildRejectDomainSet()),
-      downloadPreviousBuildPromise.then(() => buildChnCidr()),
+        downloadPreviousBuildPromise.then(() => buildChnCidr()),
-      downloadPreviousBuildPromise.then(() => buildSpeedtestDomainSet()),
+        downloadPreviousBuildPromise.then(() => buildSpeedtestDomainSet()),
-      downloadPreviousBuildPromise.then(() => buildDomesticRuleset()),
+        downloadPreviousBuildPromise.then(() => buildDomesticRuleset()),
-      downloadPreviousBuildPromise.then(() => buildGlobalRuleset()),
+        downloadPreviousBuildPromise.then(() => buildGlobalRuleset()),
-      downloadPreviousBuildPromise.then(() => buildRedirectModule()),
+        downloadPreviousBuildPromise.then(() => buildRedirectModule()),
-      downloadPreviousBuildPromise.then(() => buildAlwaysRealIPModule()),
+        downloadPreviousBuildPromise.then(() => buildAlwaysRealIPModule()),
-      downloadPreviousBuildPromise.then(() => buildStreamService()),
+        downloadPreviousBuildPromise.then(() => buildStreamService()),
-      downloadPreviousBuildPromise.then(() => microsoftCdnWorker.buildMicrosoftCdn()),
+        downloadPreviousBuildPromise.then(() => microsoftCdnWorker.buildMicrosoftCdn()),
-      downloadPreviousBuildPromise.then(() => buildCloudMounterRules()),
+        downloadPreviousBuildPromise.then(() => buildCloudMounterRules()),
-      mockAssetsWorker.downloadMockAssets()
+        mockAssetsWorker.downloadMockAssets()
+      ]),
+      downloadPreviousBuildPromise.then(() => telegramWorker.buildTelegram())
     ]);
 
-    traces.push(
+    appendArrayInPlace(traces, telegramTraces);
-      await buildDeprecateFiles(),
+    traces.push(await buildDeprecateFiles(), await buildPublic());
-      await buildPublic()
-    );
 
     // write a file to demonstrate that the build is finished
     fs.writeFileSync(buildFinishedLock, 'BUILD_FINISHED\n');
@@ -137,7 +138,7 @@ const buildFinishedLock = path.join(ROOT_DIR, '.BUILD_FINISHED');
     await Promise.all([
       microsoftCdnWorker.end(),
       cdnDownloadWorker.end(),
-      telegramCidrWorker.end(),
+      telegramWorker.end(),
       mockAssetsWorker.end(),
       endOutputWorkerFarm()
     ]);

Build/lib/fetch-telegram-backup-endpoints.ts

@@ -0,0 +1,180 @@
+import dns from 'node:dns/promises';
+import { Buffer } from 'node:buffer';
+
+import picocolors from 'picocolors';
+import { fastStringArrayJoin } from 'foxts/fast-string-array-join';
+import { stableHash } from 'stable-hash';
+
+import type { Span } from '../trace';
+import { $$fetch } from './fetch-retry';
+import { getTelegramBackupIPFromBase64 } from './get-telegram-backup-ip';
+import type { TelegramBackupEndpoint } from './get-telegram-backup-ip';
+import { appendArrayInPlace } from 'foxts/append-array-in-place';
+
+interface FetchTelegramBackupEndpointsOptions {
+  includeTestServers?: boolean
+}
+
+let productionEndpointsPromise: Promise<TelegramBackupEndpoint[]> | undefined;
+let testEndpointsPromise: Promise<TelegramBackupEndpoint[]> | undefined;
+
+function deduplicateEndpoints(endpoints: readonly TelegramBackupEndpoint[]) {
+  const deduplicated = new Map<string, TelegramBackupEndpoint>();
+  endpoints.forEach(endpoint => deduplicated.set(stableHash([
+    endpoint.dcId,
+    endpoint.ip,
+    endpoint.port,
+    endpoint.secret ? Buffer.from(endpoint.secret).toString('base64') : null
+  ]), endpoint));
+  return Array.from(deduplicated.values());
+}
+
+async function fetchDnsEndpoints(span: Span, domain: string, traceName: string) {
+  const endpoints: TelegramBackupEndpoint[] = [];
+  const resolvers = ['8.8.8.8', '1.0.0.1'].map((ip) => {
+    const resolver = new dns.Resolver();
+    resolver.setServers([ip]);
+    return Object.assign(resolver, { server: ip });
+  });
+
+  await span.traceChildAsync(traceName, () => Promise.all(resolvers.map(async (resolver) => {
+    try {
+      const response = await resolver.resolveTxt(domain);
+      const strings = response.map(result => fastStringArrayJoin(result, ''));
+      if (strings.length !== 2) {
+        throw new TypeError(`Unexpected TXT record count: ${strings.length}`);
+      }
+
+      const base64 = strings[0].length > strings[1].length
+        ? strings[0] + strings[1]
+        : strings[1] + strings[0];
+      const decoded = getTelegramBackupIPFromBase64(base64);
+      appendArrayInPlace(endpoints, decoded);
+
+      console.log('[telegram backup ip]', picocolors.green('DNS TXT'), { domain, endpoints: decoded, server: resolver.server });
+    } catch (error) {
+      console.error('[telegram backup ip]', picocolors.red('DNS TXT error'), { domain }, error);
+    }
+  })));
+
+  return deduplicateEndpoints(endpoints);
+}
+
+async function fetchRealtimeDatabaseEndpoints(span: Span) {
+  return span.traceChildAsync('backup source 2: Firebase Realtime DB', async () => {
+    try {
+      const data = await (await $$fetch('https://reserve-5a846.firebaseio.com/ipconfigv3.json')).json();
+      if (typeof data !== 'string' || data.length !== 344) {
+        throw new TypeError('Firebase Realtime DB data format is invalid');
+      }
+
+      const endpoints = getTelegramBackupIPFromBase64(data);
+      console.log('[telegram backup ip]', picocolors.green('Firebase Realtime DB'), { endpoints });
+      return endpoints;
+    } catch (error) {
+      console.error('[telegram backup ip]', picocolors.red('Firebase Realtime DB error'), error);
+      return [];
+    }
+  });
+}
+
+async function fetchValueStoreEndpoints(span: Span) {
+  return span.traceChildAsync('backup source 3: Firebase Value Store', async () => {
+    try {
+      const json = await (await $$fetch('https://firestore.googleapis.com/v1/projects/reserve-5a846/databases/(default)/documents/ipconfig/v3', {
+        headers: {
+          Accept: '*/*',
+          // Google rejects this request when the shared HTTP cache adds an Origin.
+          Origin: undefined
+        }
+      })).json();
+
+      if (
+        !json || typeof json !== 'object'
+        || !('fields' in json) || typeof json.fields !== 'object' || !json.fields
+        || !('data' in json.fields) || typeof json.fields.data !== 'object' || !json.fields.data
+        || !('stringValue' in json.fields.data) || typeof json.fields.data.stringValue !== 'string'
+        || json.fields.data.stringValue.length !== 344
+      ) {
+        throw new TypeError('Firebase Value Store data format is invalid');
+      }
+
+      const endpoints = getTelegramBackupIPFromBase64(json.fields.data.stringValue);
+      console.log('[telegram backup ip]', picocolors.green('Firebase Value Store'), { endpoints });
+      return endpoints;
+    } catch (error) {
+      console.error('[telegram backup ip]', picocolors.red('Firebase Value Store error'), error);
+      return [];
+    }
+  });
+}
+
+async function fetchAppEngineEndpoints(span: Span, url: string, traceName: string) {
+  return span.traceChildAsync(traceName, async () => {
+    try {
+      const data = (await (await $$fetch(url)).text()).trim();
+      if (data.length !== 344) {
+        throw new TypeError(`Google App Engine data has an unexpected length: ${data.length}`);
+      }
+
+      const endpoints = getTelegramBackupIPFromBase64(data);
+      console.log('[telegram backup ip]', picocolors.green('Google App Engine'), { url, endpoints });
+      return endpoints;
+    } catch (error) {
+      console.error('[telegram backup ip]', picocolors.red('Google App Engine error'), { url }, error);
+      return [];
+    }
+  });
+}
+
+async function fetchProductionEndpoints(span: Span) {
+  const endpointGroups = await Promise.all([
+    fetchDnsEndpoints(span, 'apv3.stel.com', 'backup source 1: DNS TXT'),
+    fetchRealtimeDatabaseEndpoints(span),
+    fetchValueStoreEndpoints(span),
+    fetchAppEngineEndpoints(span, 'https://dns-telegram.appspot.com', 'backup source 4: Google App Engine')
+  ]);
+  return deduplicateEndpoints(endpointGroups.flat());
+}
+
+async function fetchTestEndpoints(span: Span) {
+  const endpointGroups = await Promise.all([
+    fetchDnsEndpoints(span, 'tapv3.stel.com', 'test backup source 1: DNS TXT'),
+    fetchAppEngineEndpoints(span, 'https://dns-telegram.appspot.com/test', 'test backup source 4: Google App Engine')
+  ]);
+  return deduplicateEndpoints(endpointGroups.flat());
+}
+
+function getProductionEndpoints(span: Span) {
+  if (productionEndpointsPromise) {
+    return span.traceChildAsync('reuse production backup endpoints', () => productionEndpointsPromise!);
+  }
+  productionEndpointsPromise = fetchProductionEndpoints(span);
+  return productionEndpointsPromise;
+}
+
+function getTestEndpoints(span: Span) {
+  if (testEndpointsPromise) {
+    return span.traceChildAsync('reuse test backup endpoints', () => testEndpointsPromise!);
+  }
+  testEndpointsPromise = fetchTestEndpoints(span);
+  return testEndpointsPromise;
+}
+
+/**
+ * Fetches Telegram's signed backup endpoints once per worker process. The CIDR
+ * and MTProto config tasks share the production promise; only the CIDR task
+ * requests the additional test-server promise.
+ */
+export async function fetchTelegramBackupEndpoints(
+  span: Span,
+  { includeTestServers = false }: FetchTelegramBackupEndpointsOptions = {}
+) {
+  if (!includeTestServers) return getProductionEndpoints(span);
+
+  const [productionEndpoints, testEndpoints] = await Promise.all([
+    getProductionEndpoints(span),
+    getTestEndpoints(span)
+  ]);
+  return deduplicateEndpoints([...productionEndpoints, ...testEndpoints]);
+}

Build/lib/get-telegram-backup-ip.ts

@@ -19,6 +19,42 @@ xwIDAQAB
 -----END RSA PUBLIC KEY-----
 `;
 
+export interface TelegramBackupEndpoint {
+  dcId: number,
+  ip: string,
+  port: number,
+  secret?: Uint8Array
+}
+
+// Telegram's original, pre-AccessPointRule backup schema:
+// help.configSimple#d997c3c5 date:int expires:int dc_id:int
+//   ip_port_list:Vector<ipPort> = help.ConfigSimple;
+// It applies one DC ID to a bare vector of (ipv4, port) pairs. GramJS only
+// generates TgApi.help.ConfigSimple for the current #5a592a6c rule-based schema,
+// so the removed legacy constructor has no TgApi symbol to reference.
+const LEGACY_CONFIG_SIMPLE_CONSTRUCTOR_ID = 0xD9_97_C3_C5;
+
+// The legacy vector contains bare ipPort values, so TgBinaryReader#tgReadVector
+// cannot decode it as a vector of boxed TL objects. GramJS does not export the
+// core vector constructor ID, either, so validate it explicitly before reading.
+const TL_VECTOR_CONSTRUCTOR_ID = 0x1C_B5_C4_15;
+
+function ipv4ToString(ipv4: number) {
+  return bigint2ip(
+    ipv4 > 0
+      ? BigInt(ipv4)
+      : (2n ** 32n) + BigInt(ipv4),
+    4
+  );
+}
+
+function validateBackupEndpoint(endpoint: TelegramBackupEndpoint) {
+  if (endpoint.dcId < 1 || endpoint.dcId > 5 || endpoint.port < 1 || endpoint.port > 65535) {
+    throw new TypeError(`Invalid Telegram backup endpoint: DC ${endpoint.dcId}, port ${endpoint.port}`);
+  }
+  return endpoint;
+}
+
 export function getTelegramBackupIPFromBase64(base64: string) {
   // 1. Check base64 size
   if (base64.length !== 344) {
@@ -31,7 +67,7 @@ export function getTelegramBackupIPFromBase64(base64: string) {
   // 3. Decode base64 to Buffer
   const decoded = base64ToUint8Array(base64);
   if (decoded.length !== 256) {
-    throw new TypeError('Decoded buffer length is not 344 bytes, received ' + decoded.length);
+    throw new TypeError('Decoded buffer length is not 256 bytes, received ' + decoded.length);
   }
 
   // 4. RSA decrypt (public key, "decrypt signature" - usually means "verify and extract")
@@ -79,33 +115,58 @@ export function getTelegramBackupIPFromBase64(base64: string) {
 
   const parser = new TgBinaryReader(Buffer.from(decryptedCbc.buffer, decryptedCbc.byteOffset, decryptedCbc.byteLength));
   const len = parser.readInt();
-  if (len < 8 || len > 208) throw new Error(`Invalid TL data length: ${len}`);
+  if (len < 4 || len > 204 || len % 4 !== 0) throw new Error(`Invalid TL data length: ${len}`);
 
-  const constructorId = parser.readInt();
+  const constructorId = parser.readInt() >>> 0;
 
-  if (constructorId !== TgApi.help.ConfigSimple.CONSTRUCTOR_ID) {
+  const endpoints: TelegramBackupEndpoint[] = [];
-    throw new Error(`Invalid constructor ID: ${constructorId.toString(16)}`);
+  let date: number;
+  let expires: number;
+
+  if (constructorId === TgApi.help.ConfigSimple.CONSTRUCTOR_ID) {
+    const payload = decryptedCbc.subarray(8, 4 + len);
+    const configSimple = TgApi.help.ConfigSimple.fromReader(new TgBinaryReader(Buffer.from(payload.buffer, payload.byteOffset, payload.byteLength)));
+
+    date = configSimple.date;
+    expires = configSimple.expires;
+    for (let ruleIndex = 0, ruleCount = configSimple.rules.length; ruleIndex < ruleCount; ruleIndex++) {
+      const rule = configSimple.rules[ruleIndex];
+      for (let ipIndex = 0, ipCount = rule.ips.length; ipIndex < ipCount; ipIndex++) {
+        const ip = rule.ips[ipIndex];
+        endpoints.push(validateBackupEndpoint({
+          dcId: rule.dcId,
+          ip: ipv4ToString(ip.ipv4),
+          port: ip.port,
+          ...((ip instanceof TgApi.IpPortSecret) && { secret: ip.secret })
+        }));
+      }
+    }
+  } else if (constructorId === LEGACY_CONFIG_SIMPLE_CONSTRUCTOR_ID) {
+    date = parser.readInt();
+    expires = parser.readInt();
+    const dcId = parser.readInt();
+    const vectorConstructorId = parser.readInt() >>> 0;
+    const count = parser.readInt();
+
+    if (vectorConstructorId !== TL_VECTOR_CONSTRUCTOR_ID || count < 1 || count > 1024) {
+      throw new Error('Invalid legacy Telegram backup endpoint vector');
+    }
+
+    for (let i = 0; i < count; i++) {
+      endpoints.push(validateBackupEndpoint({
+        dcId,
+        ip: ipv4ToString(parser.readInt()),
+        port: parser.readInt()
+      }));
+    }
+  } else {
+    throw new Error(`Invalid constructor ID: 0x${constructorId.toString(16)}`);
   }
 
-  const payload = decryptedCbc.subarray(8, len);
+  const now = Math.floor(Date.now() / 1000);
+  if (date >= now + 20 * 60 || expires <= now - 20 * 60) {
+    throw new Error(`Telegram backup configuration is outside its validity interval (${date}...${expires}, now ${now})`);
+  }
 
-  const configSimple = TgApi.help.ConfigSimple.fromReader(new TgBinaryReader(Buffer.from(payload.buffer, payload.byteOffset, payload.byteLength)));
+  return endpoints;
-
-  return configSimple.rules.flatMap(rule => rule.ips.map(ip => {
-    switch (ip.CONSTRUCTOR_ID) {
-      case TgApi.IpPort.CONSTRUCTOR_ID:
-      case TgApi.IpPortSecret.CONSTRUCTOR_ID:
-        return {
-          ip: bigint2ip(
-            ip.ipv4 > 0
-              ? BigInt(ip.ipv4)
-              : (2n ** 32n) + BigInt(ip.ipv4),
-            4
-          ),
-          port: ip.port
-        };
-      default:
-        throw new TypeError(`Unknown IP type: 0x${ip.CONSTRUCTOR_ID.toString(16)}`);
-    }
-  }));
 }

Build/lib/mtproto-dc-config.test.ts

@@ -0,0 +1,96 @@
+import { describe, it } from 'mocha';
+import { expect } from 'earl';
+
+import {
+  DC_OPTION_FLAG_IPV6,
+  DC_OPTION_FLAG_STATIC,
+  mergeFallbackEndpoints,
+  TELEGRAM_BOOTSTRAP_ENDPOINTS
+} from './mtproto-dc-config';
+import type { MTProtoDCConfig } from './mtproto-dc-config';
+import { setBit } from 'foxts/bitwise';
+
+describe('MTProto DC config', () => {
+  it('merges static fallbacks and removes exact duplicates', () => {
+    const config: MTProtoDCConfig = {
+      version: 1,
+      date: 1,
+      expires: 2,
+      this_dc: 5,
+      options: [
+        { id: 5, ip: '91.108.56.191', port: 443, flags: 0 },
+        { id: 5, ip: '91.108.56.191', port: 443, flags: DC_OPTION_FLAG_STATIC }
+      ]
+    };
+
+    const result = mergeFallbackEndpoints(config, [
+      { dcId: 5, ip: '91.108.56.201', port: 443 },
+      { dcId: 5, ip: '91.108.56.191', port: 443 }
+    ]);
+
+    expect(result.backupAdded).toEqual(1);
+    expect(result.duplicatesRemoved).toEqual(1);
+    expect(config.options.filter(option => option.ip === '91.108.56.191').length).toEqual(1);
+    expect(config.options.some(option => (
+      option.id === 5
+      && option.ip === '91.108.56.201'
+      && option.flags === DC_OPTION_FLAG_STATIC
+    ))).toEqual(true);
+    expect(config.options.some(option => (
+      option.id === 5
+      && option.ip === '2001:b28:f23f:f005::a'
+      && option.flags === (setBit(DC_OPTION_FLAG_STATIC, DC_OPTION_FLAG_IPV6))
+    ))).toEqual(true);
+  });
+
+  it('matches expanded and compressed spellings of the same IPv6 endpoint', () => {
+    const config: MTProtoDCConfig = {
+      version: 1,
+      date: 1,
+      expires: 2,
+      this_dc: 1,
+      options: [
+        // help.getConfig returns IPv6 fully expanded, the bootstrap list does not.
+        { id: 1, ip: '2001:0b28:f23d:f001:0000:0000:0000:000a', port: 443, flags: DC_OPTION_FLAG_IPV6 }
+      ]
+    };
+
+    const result = mergeFallbackEndpoints(config, []);
+    const ipv6Options = config.options.filter(option => option.ip.includes(':'));
+
+    // The DC1 bootstrap merges into the existing option instead of being appended.
+    expect(ipv6Options.filter(option => option.id === 1).length).toEqual(1);
+    expect(ipv6Options.some(option => (
+      option.id === 1
+      && option.ip === '2001:b28:f23d:f001::a'
+      && option.flags === setBit(DC_OPTION_FLAG_IPV6, DC_OPTION_FLAG_STATIC)
+    ))).toEqual(true);
+    // DC2-5 IPv6 bootstraps are still absent from the config, so they get added.
+    expect(result.bootstrapAdded).toEqual(TELEGRAM_BOOTSTRAP_ENDPOINTS.length - 1);
+  });
+
+  it('keeps functional variants separate and preserves backup secrets', () => {
+    const config: MTProtoDCConfig = {
+      version: 1,
+      date: 1,
+      expires: 2,
+      this_dc: 2,
+      options: [
+        { id: 2, ip: '149.154.167.50', port: 443, flags: 1 << 3 }
+      ]
+    };
+
+    mergeFallbackEndpoints(config, [{
+      dcId: 2,
+      ip: '149.154.167.50',
+      port: 443,
+      secret: Uint8Array.from([1, 2, 3])
+    }]);
+
+    expect(config.options.some(option => (
+      option.ip === '149.154.167.50'
+      && option.secret === 'AQID'
+      && option.flags === (setBit(1 << 10, DC_OPTION_FLAG_STATIC))
+    ))).toEqual(true);
+  });
+});

Build/lib/mtproto-dc-config.ts

@@ -0,0 +1,194 @@
+import { Buffer } from 'node:buffer';
+import { stableHash } from 'stable-hash';
+
+import type { Api as TgApi } from 'telegram';
+
+import type { TelegramBackupEndpoint } from './get-telegram-backup-ip';
+import { setBit, getBit } from 'foxts/bitwise';
+import { bigint2ip, ip2bigint } from 'fast-cidr-tools';
+import { isProbablyIpv6 } from 'foxts/is-probably-ip';
+
+export const DC_OPTION_FLAG_IPV6 = 1 << 0;
+export const DC_OPTION_FLAG_MEDIA_ONLY = 1 << 1;
+export const DC_OPTION_FLAG_TCPO_ONLY = 1 << 2;
+export const DC_OPTION_FLAG_CDN = 1 << 3;
+export const DC_OPTION_FLAG_STATIC = 1 << 4;
+export const DC_OPTION_FLAG_THIS_PORT_ONLY = 1 << 5;
+export const DC_OPTION_FLAG_SECRET = 1 << 10;
+
+export interface MTProtoDCConfigOption {
+  id: number,
+  ip: string,
+  port: number,
+  flags: number,
+  secret?: string
+}
+
+export interface MTProtoDCConfig {
+  version: 1,
+  date: number,
+  expires: number,
+  this_dc: number,
+  options: MTProtoDCConfigOption[]
+}
+
+export interface MTProtoEndpoint {
+  dcId: number,
+  ip: string,
+  port: number,
+  secret?: Uint8Array
+}
+
+export const TELEGRAM_BOOTSTRAP_ENDPOINTS: readonly MTProtoEndpoint[] = [
+  { dcId: 1, ip: '149.154.175.50', port: 443 },
+  { dcId: 1, ip: '2001:b28:f23d:f001::a', port: 443 },
+  { dcId: 2, ip: '149.154.167.50', port: 443 },
+  { dcId: 2, ip: '149.154.167.51', port: 443 },
+  { dcId: 2, ip: '95.161.76.100', port: 443 },
+  { dcId: 2, ip: '2001:67c:4e8:f002::a', port: 443 },
+  { dcId: 3, ip: '149.154.175.100', port: 443 },
+  { dcId: 3, ip: '2001:b28:f23d:f003::a', port: 443 },
+  { dcId: 4, ip: '149.154.167.91', port: 443 },
+  { dcId: 4, ip: '2001:67c:4e8:f004::a', port: 443 },
+  { dcId: 5, ip: '149.154.171.5', port: 443 },
+  { dcId: 5, ip: '2001:b28:f23f:f005::a', port: 443 }
+];
+
+type DcOptionWithFlags = TgApi.DcOption & { flags?: number };
+
+/**
+ * help.getConfig returns IPv6 addresses fully expanded (leading zeros in every
+ * hextet, no "::"), while TELEGRAM_BOOTSTRAP_ENDPOINTS is written in the
+ * canonical RFC 5952 form. Without canonicalizing, mergeEndpoint's string
+ * compare never matches the two spellings of the same address, so every IPv6
+ * bootstrap gets appended as a duplicate option and the live option never
+ * receives DC_OPTION_FLAG_STATIC. IPv4 has a single spelling and is passed
+ * through untouched.
+ */
+function canonicalizeIp(ip: string) {
+  if (!isProbablyIpv6(ip)) return ip;
+  return bigint2ip(ip2bigint(ip, 6), 6, true);
+}
+
+function deriveDcOptionFlags(option: TgApi.DcOption) {
+  let flags = 0;
+  if (option.ipv6) flags = setBit(flags, DC_OPTION_FLAG_IPV6);
+  if (option.mediaOnly) flags = setBit(flags, DC_OPTION_FLAG_MEDIA_ONLY);
+  if (option.tcpoOnly) flags = setBit(flags, DC_OPTION_FLAG_TCPO_ONLY);
+  if (option.cdn) flags = setBit(flags, DC_OPTION_FLAG_CDN);
+  if (option.static) flags = setBit(flags, DC_OPTION_FLAG_STATIC);
+  if (option.thisPortOnly) flags = setBit(flags, DC_OPTION_FLAG_THIS_PORT_ONLY);
+  if (option.secret) flags = setBit(flags, DC_OPTION_FLAG_SECRET);
+  return flags;
+}
+
+export function normalizeTelegramConfig(config: TgApi.Config): MTProtoDCConfig {
+  return {
+    version: 1,
+    date: config.date,
+    expires: config.expires,
+    this_dc: config.thisDc,
+    options: config.dcOptions.map((rawOption) => {
+      const option = rawOption as DcOptionWithFlags;
+      const normalized: MTProtoDCConfigOption = {
+        id: option.id,
+        ip: canonicalizeIp(option.ipAddress),
+        port: option.port,
+        flags: Number.isSafeInteger(option.flags) ? option.flags! : deriveDcOptionFlags(option)
+      };
+
+      if (option.secret) {
+        normalized.secret = Buffer.from(option.secret).toString('base64');
+      }
+
+      return normalized;
+    })
+  };
+}
+
+function fallbackFlags(ip: string, hasSecret: boolean) {
+  let flags = DC_OPTION_FLAG_STATIC;
+  if (ip.includes(':')) flags = setBit(flags, DC_OPTION_FLAG_IPV6);
+  if (hasSecret) flags = setBit(flags, DC_OPTION_FLAG_SECRET);
+  return flags;
+}
+
+function mergeEndpoint(config: MTProtoDCConfig, endpoint: MTProtoEndpoint) {
+  const secret = endpoint.secret ? Buffer.from(endpoint.secret).toString('base64') : undefined;
+  const ip = canonicalizeIp(endpoint.ip);
+  const functionalFlags = setBit(setBit(DC_OPTION_FLAG_MEDIA_ONLY, DC_OPTION_FLAG_TCPO_ONLY), DC_OPTION_FLAG_CDN);
+  let matched = false;
+
+  for (let i = 0, len = config.options.length; i < len; i++) {
+    const option = config.options[i];
+    if (
+      option.id !== endpoint.dcId
+      || option.ip !== ip
+      || option.port !== endpoint.port
+      || getBit(option.flags, functionalFlags)
+      || option.secret !== secret
+    ) {
+      continue;
+    }
+
+    option.flags = setBit(option.flags, fallbackFlags(ip, secret !== undefined));
+    matched = true;
+  }
+
+  if (!matched) {
+    config.options.push({
+      id: endpoint.dcId,
+      ip,
+      port: endpoint.port,
+      flags: fallbackFlags(ip, secret !== undefined),
+      ...(!(secret === undefined) && { secret })
+    });
+  }
+
+  return !matched;
+}
+
+function deduplicateOptions(config: MTProtoDCConfig) {
+  const previousCount = config.options.length;
+  const seen = new Set<string>();
+
+  config.options = config.options.filter((option) => {
+    const key = stableHash(option);
+    if (seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  });
+
+  return previousCount - config.options.length;
+}
+
+export function mergeFallbackEndpoints(
+  config: MTProtoDCConfig,
+  backupEndpoints: readonly TelegramBackupEndpoint[]
+) {
+  // mergeEndpoint compares IP strings, so both sides must already be canonical.
+  // normalizeTelegramConfig canonicalizes the help.getConfig response, but the
+  // merge cannot assume its input came from there.
+  for (let i = 0, len = config.options.length; i < len; i++) {
+    const option = config.options[i];
+    option.ip = canonicalizeIp(option.ip);
+  }
+
+  let backupAdded = 0;
+  for (let i = 0, len = backupEndpoints.length; i < len; i++) {
+    const endpoint = backupEndpoints[i];
+    if (mergeEndpoint(config, endpoint)) backupAdded++;
+  }
+
+  let bootstrapAdded = 0;
+  for (let i = 0, len = TELEGRAM_BOOTSTRAP_ENDPOINTS.length; i < len; i++) {
+    const endpoint = TELEGRAM_BOOTSTRAP_ENDPOINTS[i];
+    if (mergeEndpoint(config, endpoint)) bootstrapAdded++;
+  }
+
+  return {
+    backupAdded,
+    bootstrapAdded,
+    duplicatesRemoved: deduplicateOptions(config)
+  };
+}

package.json

@@ -36,6 +36,7 @@
     "hntrie": "^1.1.0",
     "null-prototype-object": "^1.2.7",
     "picocolors": "^1.1.1",
+    "stable-hash": "^0.0.6",
     "tar-fs": "^3.1.3",
     "telegram": "^2.26.22",
     "tinyglobby": "^0.2.17",

pnpm-lock.yaml

@@ -62,6 +62,9 @@ importers:
       picocolors:
         specifier: ^1.1.1
         version: 1.1.1
+      stable-hash:
+        specifier: ^0.0.6
+        version: 0.0.6
       tar-fs:
         specifier: ^3.1.3
         version: 3.1.3
@@ -1856,6 +1859,9 @@ packages:
     resolution: {integrity: sha512-o3yWv49B/o4QZk5ZcsALc6t0+eCelPc44zZsLtCQnZPDwFpDYSWcDnrv2TtMmMbQ7uKo3J0HTURCqckw23czNQ==}
     engines: {node: '>=12.0.0'}
 
+  stable-hash@0.0.6:
+    resolution: {integrity: sha512-0afH4mobqTybYZsXImQRLOjHV4gvOW+92HdUIax9t7a8d9v54KWykEuMVIcXhD9BCi+w3kS4x7O6fmZQ3JlG/g==}
+
   stackframe@1.3.4:
     resolution: {integrity: sha512-oeVtt7eWQS+Na6F//S4kJ2K2VbRlS9D43mAlMyVpVWovy9o+jfgH8O9agzANzaiLjclA0oYzUXEM4PurhSUChw==}
 
@@ -3724,6 +3730,8 @@ snapshots:
 
   stable-hash-x@0.2.0: {}
 
+  stable-hash@0.0.6: {}
+
   stackframe@1.3.4: {}
 
   store2@2.14.4: {}