mirror of
https://github.com/SukkaW/Surge.git
synced 2025-12-11 16:50:33 +08:00
Perf: offload Telegram CIDR fetching into separate thread
This commit is contained in:
SukkaW
parent
f41bb4a3b8
commit
6a35d3e4a6
@ -79,10 +79,13 @@ export const buildCdnDownloadConf = task(require.main === module, __filename)(as
|
||||
downloadDomainSet,
|
||||
steamDomainSet
|
||||
] = await Promise.all([
|
||||
span.traceChildAsync('download public suffix list for s3', () => pool.exec(
|
||||
'getS3OSSDomains',
|
||||
[__filename]
|
||||
).finally(() => pool.terminate())),
|
||||
span.traceChildAsync(
|
||||
'download public suffix list for s3',
|
||||
() => pool.exec(
|
||||
'getS3OSSDomains',
|
||||
[__filename]
|
||||
).finally(() => pool.terminate())
|
||||
),
|
||||
cdnDomainsListPromise,
|
||||
downloadDomainSetPromise,
|
||||
steamDomainSetPromise
|
||||
|
||||
@ -3,6 +3,7 @@ import { SHARED_DESCRIPTION } from './constants/description';
|
||||
import { RulesetOutput } from './lib/rules/ruleset';
|
||||
import Worktank from 'worktank';
|
||||
import { RULES } from './constants/microsoft-cdn';
|
||||
import { wait } from 'foxts/wait';
|
||||
|
||||
const pool = new Worktank({
|
||||
pool: {
|
||||
@ -47,10 +48,10 @@ const pool = new Worktank({
|
||||
}
|
||||
});
|
||||
|
||||
export const getMicrosoftCdnRulesetPromise = pool.exec(
|
||||
export const getMicrosoftCdnRulesetPromise = wait(0).then(() => pool.exec(
|
||||
'getMicrosoftCdnRuleset',
|
||||
[__filename]
|
||||
).finally(() => pool.terminate());
|
||||
)).finally(() => pool.terminate());
|
||||
|
||||
export const buildMicrosoftCdn = task(require.main === module, __filename)(async (span) => {
|
||||
const [domains, domainSuffixes] = await span.traceChildPromise('get microsoft cdn domains', getMicrosoftCdnRulesetPromise);
|
||||
|
||||
@ -6,7 +6,7 @@ import path from 'node:path';
|
||||
|
||||
import { ALL as AllStreamServices } from '../Source/stream';
|
||||
import { getChnCidrPromise } from './build-chn-cidr';
|
||||
import { getTelegramCIDRPromise } from './build-telegram-cidr';
|
||||
import { getTelegramCIDRPromise } from './lib/get-telegram-backup-ip';
|
||||
import { compareAndWriteFile } from './lib/create-file';
|
||||
import { getMicrosoftCdnRulesetPromise } from './build-microsoft-cdn';
|
||||
import { isTruthy, nullthrow } from 'foxts/guard';
|
||||
@ -75,7 +75,7 @@ export const buildSSPanelUIMAppProfile = task(require.main === module, __filenam
|
||||
// domestic - ip cidr
|
||||
getChnCidrPromise(),
|
||||
// global - ip cidr
|
||||
getTelegramCIDRPromise(),
|
||||
getTelegramCIDRPromise,
|
||||
// lan - ip cidr
|
||||
readFileIntoProcessedArray(path.join(OUTPUT_SURGE_DIR, 'ip/lan.conf'))
|
||||
] as const);
|
||||
|
||||
@ -1,137 +1,11 @@
|
||||
// @ts-check
|
||||
import { createReadlineInterfaceFromResponse } from './lib/fetch-text-by-line';
|
||||
import { task } from './trace';
|
||||
import { SHARED_DESCRIPTION } from './constants/description';
|
||||
import { once } from 'foxts/once';
|
||||
import { RulesetOutput } from './lib/rules/ruleset';
|
||||
import { $$fetch } from './lib/fetch-retry';
|
||||
import { fastIpVersion } from 'foxts/fast-ip-version';
|
||||
import DNS2 from 'dns2';
|
||||
import { getTelegramBackupIPFromBase64 } from './lib/get-telegram-backup-ip';
|
||||
import picocolors from 'picocolors';
|
||||
|
||||
export const getTelegramCIDRPromise = once(async () => {
|
||||
const resp = await $$fetch('https://core.telegram.org/resources/cidr.txt');
|
||||
const lastModified = resp.headers.get('last-modified');
|
||||
const date = lastModified ? new Date(lastModified) : new Date();
|
||||
|
||||
const ipcidr: string[] = [
|
||||
// Unused secret Telegram backup CIDR, announced by AS62041
|
||||
'95.161.64.0/20'
|
||||
];
|
||||
const ipcidr6: string[] = [];
|
||||
|
||||
for await (const cidr of createReadlineInterfaceFromResponse(resp, true)) {
|
||||
const v = fastIpVersion(cidr);
|
||||
if (v === 4) {
|
||||
ipcidr.push(cidr);
|
||||
} else if (v === 6) {
|
||||
ipcidr6.push(cidr);
|
||||
}
|
||||
}
|
||||
|
||||
const backupIPs = new Set<string>();
|
||||
|
||||
// https://github.com/tdlib/td/blob/master/td/telegram/ConfigManager.cpp
|
||||
|
||||
// Backup IP Source 1 (DoH)
|
||||
await Promise.all([
|
||||
DNS2.DOHClient({ dns: 'https://8.8.4.4/dns-query?dns={query}' }),
|
||||
DNS2.DOHClient({ dns: 'https://1.0.0.1/dns-query?dns={query}' })
|
||||
].flatMap(
|
||||
(client) => [
|
||||
'apv3.stel.com', // prod
|
||||
'tapv3.stel.com' // test
|
||||
].map(async (domain) => {
|
||||
try {
|
||||
// tapv3.stel.com was for testing server
|
||||
const resp = await client(domain, 'TXT');
|
||||
const strings = resp.answers.map(i => i.data);
|
||||
|
||||
const str = strings[0]!.length > strings[1]!.length
|
||||
? strings[0]! + strings[1]!
|
||||
: strings[1]! + strings[0]!;
|
||||
|
||||
const ips = getTelegramBackupIPFromBase64(str);
|
||||
ips.forEach(i => backupIPs.add(i.ip));
|
||||
|
||||
console.log('[telegram backup ip]', picocolors.green('DoH TXT'), { domain, ips });
|
||||
} catch (e) {
|
||||
console.error('[telegram backup ip]', picocolors.red('DoH TXT error'), { domain }, e);
|
||||
}
|
||||
})
|
||||
));
|
||||
|
||||
// Backup IP Source 2: Firebase Realtime Database (test server not supported)
|
||||
try {
|
||||
const text = await (await $$fetch('https://reserve-5a846.firebaseio.com/ipconfigv3.json')).json();
|
||||
if (typeof text === 'string' && text.length === 344) {
|
||||
const ips = getTelegramBackupIPFromBase64(text);
|
||||
ips.forEach(i => backupIPs.add(i.ip));
|
||||
|
||||
console.log('[telegram backup ip]', picocolors.green('Firebase Realtime DB'), { ips });
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('[telegram backup ip]', picocolors.red('Firebase Realtime DB error'), e);
|
||||
// ignore all errors
|
||||
}
|
||||
|
||||
// Backup IP Source 3: Firebase Value Store (test server not supported)
|
||||
try {
|
||||
const json = await (await fetch('https://firestore.googleapis.com/v1/projects/reserve-5a846/databases/(default)/documents/ipconfig/v3', {
|
||||
headers: {
|
||||
Accept: '*/*',
|
||||
Origin: undefined // Without this line, Google API will return "Bad request: Origin doesn't match Host for XD3.". Probably have something to do with sqlite cache store
|
||||
}
|
||||
})).json();
|
||||
|
||||
// const json = await resp.json();
|
||||
if (
|
||||
json && typeof json === 'object'
|
||||
&& 'fields' in json && typeof json.fields === 'object' && json.fields
|
||||
&& 'data' in json.fields && typeof json.fields.data === 'object' && json.fields.data
|
||||
&& 'stringValue' in json.fields.data && typeof json.fields.data.stringValue === 'string' && json.fields.data.stringValue.length === 344
|
||||
) {
|
||||
const ips = getTelegramBackupIPFromBase64(json.fields.data.stringValue);
|
||||
ips.forEach(i => backupIPs.add(i.ip));
|
||||
|
||||
console.log('[telegram backup ip]', picocolors.green('Firebase Value Store'), { ips });
|
||||
} else {
|
||||
console.error('[telegram backup ip]', picocolors.red('Firebase Value Store data format invalid'), { json });
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('[telegram backup ip]', picocolors.red('Firebase Value Store error'), e);
|
||||
}
|
||||
|
||||
// Backup IP Source 4: Google App Engine
|
||||
await Promise.all([
|
||||
'https://dns-telegram.appspot.com',
|
||||
'https://dns-telegram.appspot.com/test'
|
||||
].map(async (url) => {
|
||||
try {
|
||||
const text = await (await $$fetch(url)).text();
|
||||
if (text.length === 344) {
|
||||
const ips = getTelegramBackupIPFromBase64(text);
|
||||
ips.forEach(i => backupIPs.add(i.ip));
|
||||
|
||||
console.log('[telegram backup ip]', picocolors.green('Google App Engine'), { url, ips });
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('[telegram backup ip]', picocolors.red('Google App Engine error'), { url }, e);
|
||||
}
|
||||
}));
|
||||
|
||||
// tcdnb.azureedge.net no longer works
|
||||
|
||||
console.log('[telegram backup ip]', `Found ${backupIPs.size} backup IPs:`, backupIPs);
|
||||
|
||||
ipcidr.push(...Array.from(backupIPs).map(i => i + '/32'));
|
||||
|
||||
return { date, ipcidr, ipcidr6 };
|
||||
});
|
||||
import { getTelegramCIDRPromise } from './lib/get-telegram-backup-ip';
|
||||
|
||||
export const buildTelegramCIDR = task(require.main === module, __filename)(async (span) => {
|
||||
const { date, ipcidr, ipcidr6 } = await span.traceChildAsync('get telegram cidr', getTelegramCIDRPromise);
|
||||
const { timestamp, ipcidr, ipcidr6 } = await span.traceChildPromise('get telegram cidr', getTelegramCIDRPromise);
|
||||
|
||||
if (ipcidr.length + ipcidr6.length === 0) {
|
||||
throw new Error('Failed to fetch data!');
|
||||
@ -148,7 +22,7 @@ export const buildTelegramCIDR = task(require.main === module, __filename)(async
|
||||
.withDescription(description)
|
||||
// .withDate(date) // With extra data source, we no longer use last-modified for file date
|
||||
.appendDataSource(
|
||||
'https://core.telegram.org/resources/cidr.txt (last updated: ' + date.toISOString() + ')'
|
||||
'https://core.telegram.org/resources/cidr.txt (last updated: ' + new Date(timestamp).toISOString() + ')'
|
||||
)
|
||||
.bulkAddCIDR4NoResolve(ipcidr)
|
||||
.bulkAddCIDR6NoResolve(ipcidr6)
|
||||
|
||||
@ -177,6 +177,8 @@ export async function $$fetch(url: string, init: RequestInit = defaultRequestIni
|
||||
}
|
||||
}
|
||||
|
||||
export const fetch = $$fetch;
|
||||
|
||||
/** @deprecated -- undici.requests doesn't support gzip/br/deflate, and has difficulty w/ undidi cache */
|
||||
export async function requestWithLog(url: string, opt?: Parameters<typeof undici.request>[1]) {
|
||||
try {
|
||||
|
||||
@ -202,17 +202,10 @@ const pool = new Worktank({
|
||||
export function getPhishingDomains(parentSpan: Span) {
|
||||
return parentSpan.traceChild('get phishing domains').traceAsyncFn(async (span) => span.traceChildAsync(
|
||||
'process phishing domain set',
|
||||
async () => {
|
||||
const phishingDomains = await pool.exec(
|
||||
'getPhishingDomains',
|
||||
[
|
||||
__filename,
|
||||
require.main === module
|
||||
]
|
||||
);
|
||||
pool.terminate();
|
||||
return phishingDomains;
|
||||
}
|
||||
() => pool.exec(
|
||||
'getPhishingDomains',
|
||||
[__filename, require.main === module]
|
||||
).finally(() => pool.terminate())
|
||||
));
|
||||
}
|
||||
|
||||
|
||||
@ -7,6 +7,9 @@ import { bigint2ip } from 'fast-cidr-tools';
|
||||
|
||||
import { base64ToUint8Array, concatUint8Arrays } from 'foxts/uint8array-utils';
|
||||
|
||||
import Worktank from 'worktank';
|
||||
import { wait } from 'foxts/wait';
|
||||
|
||||
const mtptoto_public_rsa = `-----BEGIN RSA PUBLIC KEY-----
|
||||
MIIBCgKCAQEAyr+18Rex2ohtVy8sroGP
|
||||
BwXD3DOoKCSpjDqYoXgCqB7ioln4eDCFfOBUlfXUEvM/fnKCpF46VkAftlb4VuPD
|
||||
@ -108,3 +111,153 @@ export function getTelegramBackupIPFromBase64(base64: string) {
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
const pool = new Worktank({
|
||||
pool: {
|
||||
name: 'get-telegram-backup-ips',
|
||||
size: 1 // The number of workers to keep in the pool, if more workers are needed they will be spawned up to this limit
|
||||
},
|
||||
worker: {
|
||||
autoAbort: 10000,
|
||||
autoTerminate: 30000, // The interval of milliseconds at which to check if the pool can be automatically terminated, to free up resources, workers will be spawned up again if needed
|
||||
autoInstantiate: true,
|
||||
methods: {
|
||||
// eslint-disable-next-line object-shorthand -- workertank
|
||||
getTelegramBackupIPs: async function (__filename: string): Promise<{ timestamp: number, ipcidr: string[], ipcidr6: string[] }> {
|
||||
// TODO: createRequire is a temporary workaround for https://github.com/nodejs/node/issues/51956
|
||||
const { default: module } = await import('node:module');
|
||||
const __require = module.createRequire(__filename);
|
||||
|
||||
const picocolors = __require('picocolors') as typeof import('picocolors');
|
||||
const { fetch } = __require('./fetch-retry') as typeof import('./fetch-retry');
|
||||
const DNS2 = __require('dns2') as typeof import('dns2');
|
||||
const { createReadlineInterfaceFromResponse } = __require('./fetch-text-by-line') as typeof import('./fetch-text-by-line');
|
||||
const { getTelegramBackupIPFromBase64 } = __require('./get-telegram-backup-ip') as typeof import('./get-telegram-backup-ip');
|
||||
const { fastIpVersion } = __require('foxts/fast-ip-version') as typeof import('foxts/fast-ip-version');
|
||||
|
||||
const resp = await fetch('https://core.telegram.org/resources/cidr.txt');
|
||||
const lastModified = resp.headers.get('last-modified');
|
||||
const date = lastModified ? new Date(lastModified) : new Date();
|
||||
|
||||
const ipcidr: string[] = [
|
||||
// Unused secret Telegram backup CIDR, announced by AS62041
|
||||
'95.161.64.0/20'
|
||||
];
|
||||
const ipcidr6: string[] = [];
|
||||
|
||||
for await (const cidr of createReadlineInterfaceFromResponse(resp, true)) {
|
||||
const v = fastIpVersion(cidr);
|
||||
if (v === 4) {
|
||||
ipcidr.push(cidr);
|
||||
} else if (v === 6) {
|
||||
ipcidr6.push(cidr);
|
||||
}
|
||||
}
|
||||
|
||||
const backupIPs = new Set<string>();
|
||||
|
||||
// https://github.com/tdlib/td/blob/master/td/telegram/ConfigManager.cpp
|
||||
|
||||
// Backup IP Source 1 (DoH)
|
||||
await Promise.all([
|
||||
DNS2.DOHClient({ dns: 'https://8.8.4.4/dns-query?dns={query}' }),
|
||||
DNS2.DOHClient({ dns: 'https://1.0.0.1/dns-query?dns={query}' })
|
||||
].flatMap(
|
||||
(client) => [
|
||||
'apv3.stel.com', // prod
|
||||
'tapv3.stel.com' // test
|
||||
].map(async (domain) => {
|
||||
try {
|
||||
// tapv3.stel.com was for testing server
|
||||
const resp = await client(domain, 'TXT');
|
||||
const strings = resp.answers.map(i => i.data);
|
||||
|
||||
const str = strings[0]!.length > strings[1]!.length
|
||||
? strings[0]! + strings[1]!
|
||||
: strings[1]! + strings[0]!;
|
||||
|
||||
const ips = getTelegramBackupIPFromBase64(str);
|
||||
ips.forEach(i => backupIPs.add(i.ip));
|
||||
|
||||
console.log('[telegram backup ip]', picocolors.green('DoH TXT'), { domain, ips });
|
||||
} catch (e) {
|
||||
console.error('[telegram backup ip]', picocolors.red('DoH TXT error'), { domain }, e);
|
||||
}
|
||||
})
|
||||
));
|
||||
|
||||
// Backup IP Source 2: Firebase Realtime Database (test server not supported)
|
||||
try {
|
||||
const text = await (await fetch('https://reserve-5a846.firebaseio.com/ipconfigv3.json')).json();
|
||||
if (typeof text === 'string' && text.length === 344) {
|
||||
const ips = getTelegramBackupIPFromBase64(text);
|
||||
ips.forEach(i => backupIPs.add(i.ip));
|
||||
|
||||
console.log('[telegram backup ip]', picocolors.green('Firebase Realtime DB'), { ips });
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('[telegram backup ip]', picocolors.red('Firebase Realtime DB error'), e);
|
||||
// ignore all errors
|
||||
}
|
||||
|
||||
// Backup IP Source 3: Firebase Value Store (test server not supported)
|
||||
try {
|
||||
const json = await (await fetch('https://firestore.googleapis.com/v1/projects/reserve-5a846/databases/(default)/documents/ipconfig/v3', {
|
||||
headers: {
|
||||
Accept: '*/*',
|
||||
Origin: undefined // Without this line, Google API will return "Bad request: Origin doesn't match Host for XD3.". Probably have something to do with sqlite cache store
|
||||
}
|
||||
})).json();
|
||||
|
||||
// const json = await resp.json();
|
||||
if (
|
||||
json && typeof json === 'object'
|
||||
&& 'fields' in json && typeof json.fields === 'object' && json.fields
|
||||
&& 'data' in json.fields && typeof json.fields.data === 'object' && json.fields.data
|
||||
&& 'stringValue' in json.fields.data && typeof json.fields.data.stringValue === 'string' && json.fields.data.stringValue.length === 344
|
||||
) {
|
||||
const ips = getTelegramBackupIPFromBase64(json.fields.data.stringValue);
|
||||
ips.forEach(i => backupIPs.add(i.ip));
|
||||
|
||||
console.log('[telegram backup ip]', picocolors.green('Firebase Value Store'), { ips });
|
||||
} else {
|
||||
console.error('[telegram backup ip]', picocolors.red('Firebase Value Store data format invalid'), { json });
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('[telegram backup ip]', picocolors.red('Firebase Value Store error'), e);
|
||||
}
|
||||
|
||||
// Backup IP Source 4: Google App Engine
|
||||
await Promise.all([
|
||||
'https://dns-telegram.appspot.com',
|
||||
'https://dns-telegram.appspot.com/test'
|
||||
].map(async (url) => {
|
||||
try {
|
||||
const text = await (await fetch(url)).text();
|
||||
if (text.length === 344) {
|
||||
const ips = getTelegramBackupIPFromBase64(text);
|
||||
ips.forEach(i => backupIPs.add(i.ip));
|
||||
|
||||
console.log('[telegram backup ip]', picocolors.green('Google App Engine'), { url, ips });
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('[telegram backup ip]', picocolors.red('Google App Engine error'), { url }, e);
|
||||
}
|
||||
}));
|
||||
|
||||
// tcdnb.azureedge.net no longer works
|
||||
|
||||
console.log('[telegram backup ip]', `Found ${backupIPs.size} backup IPs:`, backupIPs);
|
||||
|
||||
ipcidr.push(...Array.from(backupIPs).map(i => i + '/32'));
|
||||
|
||||
return { timestamp: date.getTime(), ipcidr, ipcidr6 };
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
export const getTelegramCIDRPromise = wait(0).then(() => pool.exec(
|
||||
'getTelegramBackupIPs',
|
||||
[__filename]
|
||||
)).finally(() => pool.terminate());
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user