From 939fa0d2a0c82bed1e887f428a87e85244f9ff7d Mon Sep 17 00:00:00 2001 From: SukkaW Date: Tue, 31 Mar 2026 22:23:43 +0800 Subject: [PATCH] Refactor: use `jest-worker` --- ...f.ts => build-cdn-download-conf.worker.ts} | 52 +-- Build/build-microsoft-cdn.ts | 70 ---- Build/build-microsoft-cdn.worker.ts | 42 ++ Build/build-reject-domainset.ts | 12 +- Build/build-telegram-cidr.ts | 131 ++++++- Build/build-telegram-cidr.worker.ts | 159 ++++++++ ...sets.ts => download-mock-assets.worker.ts} | 0 Build/index.ts | 70 ++-- Build/lib/get-phishing-domains.ts | 368 ++++++++---------- Build/lib/get-telegram-backup-ip.ts | 161 -------- Build/lib/worker.ts | 28 ++ package.json | 2 +- pnpm-lock.yaml | 151 +++++-- 13 files changed, 686 insertions(+), 560 deletions(-) rename Build/{build-cdn-download-conf.ts => build-cdn-download-conf.worker.ts} (75%) delete mode 100644 Build/build-microsoft-cdn.ts create mode 100644 Build/build-microsoft-cdn.worker.ts create mode 100644 Build/build-telegram-cidr.worker.ts rename Build/{download-mock-assets.ts => download-mock-assets.worker.ts} (100%) create mode 100644 Build/lib/worker.ts diff --git a/Build/build-cdn-download-conf.ts b/Build/build-cdn-download-conf.worker.ts similarity index 75% rename from Build/build-cdn-download-conf.ts rename to Build/build-cdn-download-conf.worker.ts index 14f2d6cd..065c123c 100644 --- a/Build/build-cdn-download-conf.ts +++ b/Build/build-cdn-download-conf.worker.ts @@ -1,12 +1,12 @@ import path from 'node:path'; -import { readFileIntoProcessedArray } from './lib/fetch-text-by-line'; +import { readFileIntoProcessedArray, fetchRemoteTextByLine } from './lib/fetch-text-by-line'; import { task } from './trace'; import { SHARED_DESCRIPTION } from './constants/description'; import { appendArrayInPlace } from 'foxts/append-array-in-place'; import { SOURCE_DIR } from './constants/dir'; import { DomainsetOutput } from './lib/rules/domainset'; import { CRASHLYTICS_WHITELIST } from './constants/reject-data-source'; -import Worktank from 'worktank'; +import { HostnameTrie } from './lib/trie'; import { $$fetch } from './lib/fetch-retry'; import { fastUri } from 'fast-uri'; @@ -14,25 +14,17 @@ const cdnDomainSetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'do const downloadDomainSetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/download.conf')); const steamDomainSetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/game-download.conf')); -const pool = new Worktank({ - pool: { - name: 'extract-s3-from-publicssuffix', - size: 1 // The number of workers to keep in the pool, if more workers are needed they will be spawned up to this limit - }, - worker: { - autoAbort: 10000, - autoTerminate: 20000, // The interval of milliseconds at which to check if the pool can be automatically terminated, to free up resources, workers will be spawned up again if needed - autoInstantiate: true, - methods: { - // eslint-disable-next-line object-shorthand -- workertank - getS3OSSDomains: async function (__filename: string): Promise { - // TODO: createRequire is a temporary workaround for https://github.com/nodejs/node/issues/51956 - const { default: module } = await import('node:module'); - const __require = module.createRequire(__filename); - - const { HostnameTrie } = __require('./lib/trie') as typeof import('./lib/trie'); - const { fetchRemoteTextByLine } = __require('./lib/fetch-text-by-line') as typeof import('./lib/fetch-text-by-line'); - +export const buildCdnDownloadConf = task(require.main === module, __filename)(async (span) => { + const [ + S3OSSDomains, + IPFSDomains, + cdnDomainsList, + downloadDomainSet, + steamDomainSet + ] = await Promise.all([ + span.traceChildAsync( + 'download public suffix list for s3', + async () => { const trie = new HostnameTrie(); for await (const line of await fetchRemoteTextByLine('https://publicsuffix.org/list/public_suffix_list.dat', true)) { @@ -70,24 +62,6 @@ const pool = new Worktank({ return S3OSSDomains; } - } - } -}); - -export const buildCdnDownloadConf = task(require.main === module, __filename)(async (span) => { - const [ - S3OSSDomains, - IPFSDomains, - cdnDomainsList, - downloadDomainSet, - steamDomainSet - ] = await Promise.all([ - span.traceChildAsync( - 'download public suffix list for s3', - () => pool.exec( - 'getS3OSSDomains', - [__filename] - ).finally(() => pool.terminate()) ), span.traceChildAsync( 'load public ipfs gateway list', diff --git a/Build/build-microsoft-cdn.ts b/Build/build-microsoft-cdn.ts deleted file mode 100644 index af761d8d..00000000 --- a/Build/build-microsoft-cdn.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { task } from './trace'; -import { SHARED_DESCRIPTION } from './constants/description'; -import { RulesetOutput } from './lib/rules/ruleset'; -import Worktank from 'worktank'; -import { RULES } from './constants/microsoft-cdn'; - -const pool = new Worktank({ - pool: { - name: 'get-microsoft-cdn', - size: 1 // The number of workers to keep in the pool, if more workers are needed they will be spawned up to this limit - }, - worker: { - autoAbort: 10000, - autoTerminate: 30000, // The interval of milliseconds at which to check if the pool can be automatically terminated, to free up resources, workers will be spawned up again if needed - autoInstantiate: true, - methods: { - // eslint-disable-next-line object-shorthand -- workertank - getMicrosoftCdnRuleset: async function (__filename: string): Promise<[domains: string[], domainSuffixes: string[]]> { - // TODO: createRequire is a temporary workaround for https://github.com/nodejs/node/issues/51956 - const { default: module } = await import('node:module'); - const __require = module.createRequire(__filename); - - const { HostnameSmolTrie } = __require('./lib/trie'); - const { PROBE_DOMAINS, DOMAINS, DOMAIN_SUFFIXES, BLACKLIST } = __require('./constants/microsoft-cdn') as typeof import('./constants/microsoft-cdn'); - const { fetchRemoteTextByLine } = __require('./lib/fetch-text-by-line') as typeof import('./lib/fetch-text-by-line'); - const { appendArrayInPlace } = __require('foxts/append-array-in-place') as typeof import('foxts/append-array-in-place'); - const { extractDomainsFromFelixDnsmasq } = __require('./lib/parse-dnsmasq') as typeof import('./lib/parse-dnsmasq'); - - const trie = new HostnameSmolTrie(); - - for await (const line of await fetchRemoteTextByLine('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf')) { - const domain = extractDomainsFromFelixDnsmasq(line); - if (domain) { - trie.add(domain); - } - } - - // remove blacklist domain from trie, to prevent them from being included in the later dump - BLACKLIST.forEach(black => trie.whitelist(black)); - - const domains: string[] = DOMAINS; - const domainSuffixes = appendArrayInPlace(PROBE_DOMAINS.flatMap(domain => trie.find(domain)), DOMAIN_SUFFIXES); - - return [domains, domainSuffixes] as const; - } - } - } -}); - -const getMicrosoftCdnRulesetPromise = pool.exec( - 'getMicrosoftCdnRuleset', - [__filename] -).finally(() => pool.terminate()); - -export const buildMicrosoftCdn = task(require.main === module, __filename)(async (span) => { - const [domains, domainSuffixes] = await span.traceChildPromise('get microsoft cdn domains', getMicrosoftCdnRulesetPromise); - - return new RulesetOutput(span, 'microsoft_cdn', 'non_ip') - .withTitle('Sukka\'s Ruleset - Microsoft CDN') - .appendDescription(SHARED_DESCRIPTION) - .appendDescription( - '', - 'This file contains Microsoft\'s domains using their China mainland CDN servers.' - ) - .addFromRuleset(RULES) - .appendDataSource('https://github.com/felixonmars/dnsmasq-china-list') - .bulkAddDomain(domains) - .bulkAddDomainSuffix(domainSuffixes) - .write(); -}); diff --git a/Build/build-microsoft-cdn.worker.ts b/Build/build-microsoft-cdn.worker.ts new file mode 100644 index 00000000..45ea588e --- /dev/null +++ b/Build/build-microsoft-cdn.worker.ts @@ -0,0 +1,42 @@ +import { task } from './trace'; +import { SHARED_DESCRIPTION } from './constants/description'; +import { RulesetOutput } from './lib/rules/ruleset'; +import { RULES, PROBE_DOMAINS, DOMAINS, DOMAIN_SUFFIXES, BLACKLIST } from './constants/microsoft-cdn'; +import { HostnameSmolTrie } from './lib/trie'; +import { fetchRemoteTextByLine } from './lib/fetch-text-by-line'; +import { appendArrayInPlace } from 'foxts/append-array-in-place'; +import { extractDomainsFromFelixDnsmasq } from './lib/parse-dnsmasq'; + +export const buildMicrosoftCdn = task(require.main === module, __filename)(async (span) => { + const [domains, domainSuffixes] = await span.traceChildAsync('get microsoft cdn domains', async () => { + const trie = new HostnameSmolTrie(); + + for await (const line of await fetchRemoteTextByLine('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf')) { + const domain = extractDomainsFromFelixDnsmasq(line); + if (domain) { + trie.add(domain); + } + } + + // remove blacklist domain from trie, to prevent them from being included in the later dump + BLACKLIST.forEach(black => trie.whitelist(black)); + + const domains: string[] = DOMAINS; + const domainSuffixes = appendArrayInPlace(PROBE_DOMAINS.flatMap(domain => trie.find(domain)), DOMAIN_SUFFIXES); + + return [domains, domainSuffixes] as [string[], string[]]; + }); + + return new RulesetOutput(span, 'microsoft_cdn', 'non_ip') + .withTitle('Sukka\'s Ruleset - Microsoft CDN') + .appendDescription(SHARED_DESCRIPTION) + .appendDescription( + '', + 'This file contains Microsoft\'s domains using their China mainland CDN servers.' + ) + .addFromRuleset(RULES) + .appendDataSource('https://github.com/felixonmars/dnsmasq-china-list') + .bulkAddDomain(domains) + .bulkAddDomainSuffix(domainSuffixes) + .write(); +}); diff --git a/Build/build-reject-domainset.ts b/Build/build-reject-domainset.ts index 111bd0b6..f9cad52a 100644 --- a/Build/build-reject-domainset.ts +++ b/Build/build-reject-domainset.ts @@ -18,7 +18,7 @@ import { addArrayElementsToSet } from 'foxts/add-array-elements-to-set'; import { OUTPUT_INTERNAL_DIR, SOURCE_DIR } from './constants/dir'; import { DomainsetOutput, AdGuardHomeOutput } from './lib/rules/domainset'; import { foundDebugDomain } from './lib/parse-filter/shared'; -import { getPhishingDomains } from './lib/get-phishing-domains'; +import { createWorker } from './lib/worker'; import type { MaybePromise } from './lib/misc'; import { RulesetOutput } from './lib/rules/ruleset'; import { fetchAssets } from './lib/fetch-assets'; @@ -39,6 +39,9 @@ const adguardFiltersExtraDownloads = ADGUARD_FILTERS_EXTRA.map(entry => processF const adguardFiltersWhitelistsDownloads = ADGUARD_FILTERS_WHITELIST.map(entry => processFilterRulesWithPreload(...entry)); export const buildRejectDomainSet = task(require.main === module, __filename)(async (span) => { + const phishingWorker = createWorker( + require.resolve('./lib/get-phishing-domains') + )(['getPhishingDomains']); const rejectDomainsetOutput = new DomainsetOutput(span, 'reject') .withTitle('Sukka\'s Ruleset - Reject Base') .appendDescription( @@ -126,7 +129,9 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as arrayPushNonNullish(promises, domainListsDownloads.map(task => task(childSpan).then(appendArrayToRejectOutput))); arrayPushNonNullish(promises, domainListsExtraDownloads.map(task => task(childSpan).then(appendArrayToRejectExtraOutput))); - rejectPhisingDomainsetOutput.addFromDomainset(getPhishingDomains(childSpan)); + rejectPhisingDomainsetOutput.addFromDomainset( + span.traceChildPromise('get phishing domains', phishingWorker.getPhishingDomains()) + ); arrayPushNonNullish( promises, @@ -253,6 +258,7 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as rejectNonIpRulesetOutput.whitelistKeyword(keyword); } + // Deduplicate reject_extra and reject_phishing from the base reject domainset rejectDomainsetOutput.domainTrie.dump(arg => { rejectExtraDomainsetOutput.whitelistDomain(arg); rejectPhisingDomainsetOutput.whitelistDomain(arg); @@ -295,4 +301,6 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as await myRejectOutputAdGuardHome .addFromRuleset(readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/my_reject.conf'))) .write(); + + await phishingWorker.end(); }); diff --git a/Build/build-telegram-cidr.ts b/Build/build-telegram-cidr.ts index 5d0a4c89..c8642041 100644 --- a/Build/build-telegram-cidr.ts +++ b/Build/build-telegram-cidr.ts @@ -2,10 +2,137 @@ import { task } from './trace'; import { SHARED_DESCRIPTION } from './constants/description'; import { RulesetOutput } from './lib/rules/ruleset'; -import { getTelegramCIDRPromise } from './lib/get-telegram-backup-ip'; +import { getTelegramBackupIPFromBase64 } from './lib/get-telegram-backup-ip'; +import picocolors from 'picocolors'; +import { $$fetch } from './lib/fetch-retry'; +import dns from 'node:dns/promises'; +import { createReadlineInterfaceFromResponse } from './lib/fetch-text-by-line'; +import { fastIpVersion } from 'foxts/fast-ip-version'; +import { fastStringArrayJoin } from 'foxts/fast-string-array-join'; export const buildTelegramCIDR = task(require.main === module, __filename)(async (span) => { - const { timestamp, ipcidr, ipcidr6 } = await span.traceChildAsync('get telegram cidr', getTelegramCIDRPromise); + const { timestamp, ipcidr, ipcidr6 } = await span.traceChildAsync('get telegram cidr', async () => { + const resp = await $$fetch('https://core.telegram.org/resources/cidr.txt'); + const lastModified = resp.headers.get('last-modified'); + const date = lastModified ? new Date(lastModified) : new Date(); + + const ipcidr: string[] = [ + // Unused secret Telegram backup CIDR, announced by AS62041 + '95.161.64.0/20' + ]; + const ipcidr6: string[] = []; + + for await (const cidr of createReadlineInterfaceFromResponse(resp, true)) { + const v = fastIpVersion(cidr); + if (v === 4) { + ipcidr.push(cidr); + } else if (v === 6) { + ipcidr6.push(cidr); + } + } + + const backupIPs = new Set(); + + // https://github.com/tdlib/td/blob/master/td/telegram/ConfigManager.cpp + + const resolvers = ['8.8.8.8', '1.0.0.1'].map((ip) => { + const resolver = new dns.Resolver(); + resolver.setServers([ip]); + return Object.assign(resolver, { server: ip }); + }); + + // Backup IP Source 1 (DNS) + await Promise.all(resolvers.flatMap((resolver) => [ + 'apv3.stel.com', // prod + 'tapv3.stel.com' // test + ].map(async (domain) => { + try { + // tapv3.stel.com was for testing server + const resp = await resolver.resolveTxt(domain); + const strings = resp.map(r => fastStringArrayJoin(r, '')); // flatten + if (strings.length !== 2) { + throw new TypeError(`Unexpected TXT record count: ${strings.length}`); + } + + const str = strings[0].length > strings[1].length + ? strings[0] + strings[1] + : strings[1] + strings[0]; + + const ips = getTelegramBackupIPFromBase64(str); + ips.forEach(i => backupIPs.add(i.ip)); + + console.log('[telegram backup ip]', picocolors.green('DNS TXT'), { domain, ips, server: resolver.server }); + } catch (e) { + console.error('[telegram backup ip]', picocolors.red('DNS TXT error'), { domain }, e); + } + }))); + + // Backup IP Source 2: Firebase Realtime Database (test server not supported) + try { + const text = await (await $$fetch('https://reserve-5a846.firebaseio.com/ipconfigv3.json')).json(); + if (typeof text === 'string' && text.length === 344) { + const ips = getTelegramBackupIPFromBase64(text); + ips.forEach(i => backupIPs.add(i.ip)); + + console.log('[telegram backup ip]', picocolors.green('Firebase Realtime DB'), { ips }); + } + } catch (e) { + console.error('[telegram backup ip]', picocolors.red('Firebase Realtime DB error'), e); + // ignore all errors + } + + // Backup IP Source 3: Firebase Value Store (test server not supported) + try { + const json = await (await $$fetch('https://firestore.googleapis.com/v1/projects/reserve-5a846/databases/(default)/documents/ipconfig/v3', { + headers: { + Accept: '*/*', + Origin: undefined // Without this line, Google API will return "Bad request: Origin doesn't match Host for XD3.". Probably have something to do with sqlite cache store + } + })).json(); + + if ( + json && typeof json === 'object' + && 'fields' in json && typeof json.fields === 'object' && json.fields + && 'data' in json.fields && typeof json.fields.data === 'object' && json.fields.data + && 'stringValue' in json.fields.data && typeof json.fields.data.stringValue === 'string' && json.fields.data.stringValue.length === 344 + ) { + const ips = getTelegramBackupIPFromBase64(json.fields.data.stringValue); + ips.forEach(i => backupIPs.add(i.ip)); + + console.log('[telegram backup ip]', picocolors.green('Firebase Value Store'), { ips }); + } else { + console.error('[telegram backup ip]', picocolors.red('Firebase Value Store data format invalid'), { json }); + } + } catch (e) { + console.error('[telegram backup ip]', picocolors.red('Firebase Value Store error'), e); + } + + // Backup IP Source 4: Google App Engine + await Promise.all([ + 'https://dns-telegram.appspot.com', + 'https://dns-telegram.appspot.com/test' + ].map(async (url) => { + try { + const text = await (await $$fetch(url)).text(); + if (text.length === 344) { + const ips = getTelegramBackupIPFromBase64(text); + ips.forEach(i => backupIPs.add(i.ip)); + + console.log('[telegram backup ip]', picocolors.green('Google App Engine'), { url, ips }); + } + } catch (e) { + console.error('[telegram backup ip]', picocolors.red('Google App Engine error'), { url }, e); + } + })); + + // tcdnb.azureedge.net no longer works + + console.log('[telegram backup ip]', `Found ${backupIPs.size} backup IPs:`, backupIPs); + + ipcidr.push(...Array.from(backupIPs).map(i => i + '/32')); + + return { timestamp: date.getTime(), ipcidr, ipcidr6 }; + }); if (ipcidr.length + ipcidr6.length === 0) { throw new Error('Failed to fetch data!'); diff --git a/Build/build-telegram-cidr.worker.ts b/Build/build-telegram-cidr.worker.ts new file mode 100644 index 00000000..f17e824c --- /dev/null +++ b/Build/build-telegram-cidr.worker.ts @@ -0,0 +1,159 @@ +// @ts-check +import { task } from './trace'; +import { SHARED_DESCRIPTION } from './constants/description'; +import { RulesetOutput } from './lib/rules/ruleset'; +import { getTelegramBackupIPFromBase64 } from './lib/get-telegram-backup-ip'; +import picocolors from 'picocolors'; +import { $$fetch } from './lib/fetch-retry'; +import dns from 'node:dns/promises'; +import { createReadlineInterfaceFromResponse } from './lib/fetch-text-by-line'; +import { fastIpVersion } from 'foxts/fast-ip-version'; +import { fastStringArrayJoin } from 'foxts/fast-string-array-join'; + +export const buildTelegramCIDR = task(require.main === module, __filename)(async (span) => { + const { timestamp, ipcidr, ipcidr6 } = await span.traceChildAsync('get telegram cidr', async () => { + const resp = await $$fetch('https://core.telegram.org/resources/cidr.txt'); + const lastModified = resp.headers.get('last-modified'); + const date = lastModified ? new Date(lastModified) : new Date(); + + const ipcidr: string[] = [ + // Unused secret Telegram backup CIDR, announced by AS62041 + '95.161.64.0/20' + ]; + const ipcidr6: string[] = []; + + for await (const cidr of createReadlineInterfaceFromResponse(resp, true)) { + const v = fastIpVersion(cidr); + if (v === 4) { + ipcidr.push(cidr); + } else if (v === 6) { + ipcidr6.push(cidr); + } + } + + const backupIPs = new Set(); + + // https://github.com/tdlib/td/blob/master/td/telegram/ConfigManager.cpp + + const resolvers = ['8.8.8.8', '1.0.0.1'].map((ip) => { + const resolver = new dns.Resolver(); + resolver.setServers([ip]); + return Object.assign(resolver, { server: ip }); + }); + + // Backup IP Source 1 (DNS) + await Promise.all(resolvers.flatMap((resolver) => [ + 'apv3.stel.com', // prod + 'tapv3.stel.com' // test + ].map(async (domain) => { + try { + // tapv3.stel.com was for testing server + const resp = await resolver.resolveTxt(domain); + const strings = resp.map(r => fastStringArrayJoin(r, '')); // flatten + if (strings.length !== 2) { + throw new TypeError(`Unexpected TXT record count: ${strings.length}`); + } + + const str = strings[0].length > strings[1].length + ? strings[0] + strings[1] + : strings[1] + strings[0]; + + const ips = getTelegramBackupIPFromBase64(str); + ips.forEach(i => backupIPs.add(i.ip)); + + console.log('[telegram backup ip]', picocolors.green('DNS TXT'), { domain, ips, server: resolver.server }); + } catch (e) { + console.error('[telegram backup ip]', picocolors.red('DNS TXT error'), { domain }, e); + } + }))); + + // Backup IP Source 2: Firebase Realtime Database (test server not supported) + try { + const text = await (await $$fetch('https://reserve-5a846.firebaseio.com/ipconfigv3.json')).json(); + if (typeof text === 'string' && text.length === 344) { + const ips = getTelegramBackupIPFromBase64(text); + ips.forEach(i => backupIPs.add(i.ip)); + + console.log('[telegram backup ip]', picocolors.green('Firebase Realtime DB'), { ips }); + } + } catch (e) { + console.error('[telegram backup ip]', picocolors.red('Firebase Realtime DB error'), e); + // ignore all errors + } + + // Backup IP Source 3: Firebase Value Store (test server not supported) + try { + const json = await (await $$fetch('https://firestore.googleapis.com/v1/projects/reserve-5a846/databases/(default)/documents/ipconfig/v3', { + headers: { + Accept: '*/*', + Origin: undefined // Without this line, Google API will return "Bad request: Origin doesn't match Host for XD3.". Probably have something to do with sqlite cache store + } + })).json(); + + if ( + json && typeof json === 'object' + && 'fields' in json && typeof json.fields === 'object' && json.fields + && 'data' in json.fields && typeof json.fields.data === 'object' && json.fields.data + && 'stringValue' in json.fields.data && typeof json.fields.data.stringValue === 'string' && json.fields.data.stringValue.length === 344 + ) { + const ips = getTelegramBackupIPFromBase64(json.fields.data.stringValue); + ips.forEach(i => backupIPs.add(i.ip)); + + console.log('[telegram backup ip]', picocolors.green('Firebase Value Store'), { ips }); + } else { + console.error('[telegram backup ip]', picocolors.red('Firebase Value Store data format invalid'), { json }); + } + } catch (e) { + console.error('[telegram backup ip]', picocolors.red('Firebase Value Store error'), e); + } + + // Backup IP Source 4: Google App Engine + await Promise.all([ + 'https://dns-telegram.appspot.com', + 'https://dns-telegram.appspot.com/test' + ].map(async (url) => { + try { + const text = await (await $$fetch(url)).text(); + if (text.length === 344) { + const ips = getTelegramBackupIPFromBase64(text); + ips.forEach(i => backupIPs.add(i.ip)); + + console.log('[telegram backup ip]', picocolors.green('Google App Engine'), { url, ips }); + } + } catch (e) { + console.error('[telegram backup ip]', picocolors.red('Google App Engine error'), { url }, e); + } + })); + + // tcdnb.azureedge.net no longer works + + console.log('[telegram backup ip]', `Found ${backupIPs.size} backup IPs:`, backupIPs); + + ipcidr.push(...Array.from(backupIPs).map(i => i + '/32')); + + return { timestamp: date.getTime(), ipcidr, ipcidr6 }; + }); + + if (ipcidr.length + ipcidr6.length === 0) { + throw new Error('Failed to fetch data!'); + } + + const description = [ + ...SHARED_DESCRIPTION, + 'Data from:', + ' - https://core.telegram.org/resources/cidr.txt' + ]; + + return new RulesetOutput(span, 'telegram', 'ip') + .withTitle('Sukka\'s Ruleset - Telegram IP CIDR') + .withDescription(description) + // .withDate(date) // With extra data source, we no longer use last-modified for file date + .appendDataSource( + 'https://core.telegram.org/resources/cidr.txt (last updated: ' + new Date(timestamp).toISOString() + ')' + ) + .bulkAddCIDR4NoResolve(ipcidr) + .bulkAddCIDR6NoResolve(ipcidr6) + .write(); +}); + +export const ___ = ''; diff --git a/Build/download-mock-assets.ts b/Build/download-mock-assets.worker.ts similarity index 100% rename from Build/download-mock-assets.ts rename to Build/download-mock-assets.worker.ts diff --git a/Build/index.ts b/Build/index.ts index 8037d8bd..8ebf34f1 100644 --- a/Build/index.ts +++ b/Build/index.ts @@ -6,9 +6,7 @@ import { downloadPreviousBuild } from './download-previous-build'; import { buildCommon } from './build-common'; import { buildRejectIPList } from './build-reject-ip-list'; import { buildAppleCdn } from './build-apple-cdn'; -import { buildCdnDownloadConf } from './build-cdn-download-conf'; import { buildRejectDomainSet } from './build-reject-domainset'; -import { buildTelegramCIDR } from './build-telegram-cidr'; import { buildChnCidr } from './build-chn-cidr'; import { buildSpeedtestDomainSet } from './build-speedtest-domainset'; import { buildDomesticRuleset } from './build-domestic-direct-lan-ruleset-dns-mapping-module'; @@ -18,11 +16,9 @@ import { buildStreamService } from './build-stream-service'; import { buildRedirectModule } from './build-sgmodule-redirect'; import { buildAlwaysRealIPModule } from './build-sgmodule-always-realip'; -import { buildMicrosoftCdn } from './build-microsoft-cdn'; +import { createWorker } from './lib/worker'; import { buildPublic } from './build-public'; -import { downloadMockAssets } from './download-mock-assets'; - import { buildCloudMounterRules } from './build-cloudmounter-rules'; import { printStats, printTraceResult, whyIsNodeRunning } from './trace'; @@ -71,6 +67,22 @@ const buildFinishedLock = path.join(ROOT_DIR, '.BUILD_FINISHED'); fs.unlinkSync(buildFinishedLock); } + const microsoftCdnWorker = createWorker( + require.resolve('./build-microsoft-cdn.worker') + )(['buildMicrosoftCdn']); + + const cdnDownloadWorker = createWorker( + require.resolve('./build-cdn-download-conf.worker') + )(['buildCdnDownloadConf']); + + const telegramCidrWorker = createWorker( + require.resolve('./build-telegram-cidr.worker') + )(['buildTelegramCIDR']); + + const mockAssetsWorker = createWorker( + require.resolve('./download-mock-assets.worker') + )(['downloadMockAssets']); + try { // only enable why-is-node-running in GitHub Actions debug mode if (isCI && process.env.RUNNER_DEBUG === '1') { @@ -79,14 +91,14 @@ const buildFinishedLock = path.join(ROOT_DIR, '.BUILD_FINISHED'); const downloadPreviousBuildPromise = downloadPreviousBuild(); - await Promise.all([ + const traces: TraceResult[] = await Promise.all([ downloadPreviousBuildPromise, downloadPreviousBuildPromise.then(() => buildCommon()), downloadPreviousBuildPromise.then(() => buildRejectIPList()), downloadPreviousBuildPromise.then(() => buildAppleCdn()), - downloadPreviousBuildPromise.then(() => buildCdnDownloadConf()), + downloadPreviousBuildPromise.then(() => cdnDownloadWorker.buildCdnDownloadConf()), downloadPreviousBuildPromise.then(() => buildRejectDomainSet()), - downloadPreviousBuildPromise.then(() => buildTelegramCIDR()), + downloadPreviousBuildPromise.then(() => telegramCidrWorker.buildTelegramCIDR()), downloadPreviousBuildPromise.then(() => buildChnCidr()), downloadPreviousBuildPromise.then(() => buildSpeedtestDomainSet()), downloadPreviousBuildPromise.then(() => buildDomesticRuleset()), @@ -94,45 +106,29 @@ const buildFinishedLock = path.join(ROOT_DIR, '.BUILD_FINISHED'); downloadPreviousBuildPromise.then(() => buildRedirectModule()), downloadPreviousBuildPromise.then(() => buildAlwaysRealIPModule()), downloadPreviousBuildPromise.then(() => buildStreamService()), - downloadPreviousBuildPromise.then(() => buildMicrosoftCdn()), + downloadPreviousBuildPromise.then(() => microsoftCdnWorker.buildMicrosoftCdn()), downloadPreviousBuildPromise.then(() => buildCloudMounterRules()), - downloadMockAssets() + mockAssetsWorker.downloadMockAssets() ]); - await buildDeprecateFiles(); - await buildPublic(); + traces.push( + await buildDeprecateFiles(), + await buildPublic() + ); // write a file to demonstrate that the build is finished fs.writeFileSync(buildFinishedLock, 'BUILD_FINISHED\n'); - const traces: TraceResult[] = []; - [ - downloadPreviousBuild, - downloadMockAssets, - buildCommon, - buildRejectIPList, - buildAppleCdn, - buildCdnDownloadConf, - buildRejectDomainSet, - buildTelegramCIDR, - buildChnCidr, - buildSpeedtestDomainSet, - buildDomesticRuleset, - buildGlobalRuleset, - buildRedirectModule, - buildAlwaysRealIPModule, - buildStreamService, - buildMicrosoftCdn, - buildCloudMounterRules, - buildPublic, - buildDeprecateFiles - ].forEach((fn) => { - const trace = fn.getInternalTraceResult(); - printTraceResult(trace); - traces.push(trace); + traces.forEach((t) => { + printTraceResult(t); }); printStats(traces); + await microsoftCdnWorker.end(); + await cdnDownloadWorker.end(); + await telegramCidrWorker.end(); + await mockAssetsWorker.end(); + // Finish the build to avoid leaking timer/fetch ref await whyIsNodeRunning(); process.exit(0); diff --git a/Build/lib/get-phishing-domains.ts b/Build/lib/get-phishing-domains.ts index 1580e582..0c5b7ee9 100644 --- a/Build/lib/get-phishing-domains.ts +++ b/Build/lib/get-phishing-domains.ts @@ -1,219 +1,171 @@ -import Worktank from 'worktank'; +import picocolors from 'picocolors'; +import { parse } from 'tldts-experimental'; +import { appendArrayInPlaceCurried } from 'foxts/append-array-in-place'; -import { dummySpan, printTraceResult } from '../trace'; -import type { Span } from '../trace'; +import { dummySpan } from '../trace'; import type { TldTsParsed } from './normalize-domain'; -const pool = new Worktank({ - pool: { - name: 'process-phishing-domains', - size: 1 - }, - worker: { - autoAbort: 20000, // The maximum number of milliseconds to wait for the result from the worker, if exceeded the worker is terminated and the execution promise rejects - autoInstantiate: true, - autoTerminate: 30000, // The interval of milliseconds at which to check if the pool can be automatically terminated, to free up resources, workers will be spawned up again if needed - env: {}, - methods: { - // eslint-disable-next-line object-shorthand -- workertank - getPhishingDomains: async function ( - importMetaUrl: string, - /** require.main === module */ isDebug = false - ): Promise { - // TODO: createRequire is a temporary workaround for https://github.com/nodejs/node/issues/51956 - const { default: module } = await import('node:module'); - const __require = module.createRequire(importMetaUrl); +import { loosTldOptWithPrivateDomains } from '../constants/loose-tldts-opt'; +import { BLACK_TLD, WHITELIST_MAIN_DOMAINS, leathalKeywords, lowKeywords, sensitiveKeywords } from '../constants/phishing-score-source'; +import { PHISHING_DOMAIN_LISTS_EXTRA, PHISHING_HOSTS_EXTRA } from '../constants/reject-data-source'; - const picocolors = __require('picocolors') as typeof import('picocolors'); - const tldts = __require('tldts-experimental') as typeof import('tldts-experimental'); +import { processHostsWithPreload } from './parse-filter/hosts'; +import { processDomainListsWithPreload } from './parse-filter/domainlists'; - const { appendArrayInPlaceCurried } = __require('foxts/append-array-in-place') as typeof import('foxts/append-array-in-place'); +import process from 'node:process'; - const { loosTldOptWithPrivateDomains } = __require('../constants/loose-tldts-opt') as typeof import('../constants/loose-tldts-opt'); - const { BLACK_TLD, WHITELIST_MAIN_DOMAINS, leathalKeywords, lowKeywords, sensitiveKeywords } = __require('../constants/phishing-score-source') as typeof import('../constants/phishing-score-source'); - const { PHISHING_DOMAIN_LISTS_EXTRA, PHISHING_HOSTS_EXTRA } = __require('../constants/reject-data-source') as typeof import('../constants/reject-data-source'); - const { dummySpan } = __require('../trace') as typeof import('../trace'); - const NullPrototypeObject = __require('null-prototype-object') as typeof import('null-prototype-object'); - - const { processHostsWithPreload } = __require('./parse-filter/hosts') as typeof import('./parse-filter/hosts'); - const { processDomainListsWithPreload } = __require('./parse-filter/domainlists') as typeof import('./parse-filter/domainlists'); - - const downloads = [ - ...PHISHING_DOMAIN_LISTS_EXTRA.map(entry => processDomainListsWithPreload(...entry)), - ...PHISHING_HOSTS_EXTRA.map(entry => processHostsWithPreload(...entry)) - ]; - - const domainArr: string[] = []; - - const domainGroups = await Promise.all(downloads.map(task => task(dummySpan))); - domainGroups.forEach(appendArrayInPlaceCurried(domainArr)); - - // return domainArr; - - const domainCountMap = new Map(); - const domainScoreMap: Record = new NullPrototypeObject(); - - let line = ''; - let tld: string | null = ''; - let apexDomain: string | null = ''; - let subdomain: string | null = ''; - let parsed: TldTsParsed; - - // const set = new Set(); - // let duplicateCount = 0; - - for (let i = 0, len = domainArr.length; i < len; i++) { - line = domainArr[i]; - - // if (set.has(line)) { - // duplicateCount++; - // } else { - // set.add(line); - // } - - parsed = tldts.parse(line, loosTldOptWithPrivateDomains); - if (parsed.isPrivate) { - continue; - } - - tld = parsed.publicSuffix; - apexDomain = parsed.domain; - - if (!tld) { - console.log(picocolors.yellow('[phishing domains] E0001'), 'missing tld', { line, tld }); - continue; - } - if (!apexDomain) { - console.log(picocolors.yellow('[phishing domains] E0002'), 'missing domain', { line, apexDomain }); - continue; - } - if (WHITELIST_MAIN_DOMAINS.has(apexDomain)) { - continue; - } - - domainCountMap.set( - apexDomain, - domainCountMap.has(apexDomain) - ? domainCountMap.get(apexDomain)! + 1 - : 1 - ); - - let score = 0; - - if (apexDomain in domainScoreMap) { - score = domainScoreMap[apexDomain]; - } else { - if (BLACK_TLD.has(tld)) { - score += 3; - } else if (tld.length > 4) { - score += 2; - } else if (tld.length > 5) { - score += 4; - } - if (apexDomain.length >= 18) { - score += 0.5; - } - } - - subdomain = parsed.subdomain; - - if (subdomain) { - score += calcDomainAbuseScore(subdomain, line); - } - - domainScoreMap[apexDomain] = score; - } - - domainCountMap.forEach((count, apexDomain) => { - const score = domainScoreMap[apexDomain]; - if ( - // !WHITELIST_MAIN_DOMAINS.has(apexDomain) - (score >= 24) - || (score >= 16 && count >= 7) - || (score >= 13 && count >= 11) - || (score >= 5 && count >= 14) - || (score >= 3 && count >= 21) - || (score >= 1 && count >= 60) - ) { - domainArr.push('.' + apexDomain); - } - }); - - if (isDebug) { - console.log({ - v: 1, - score: domainScoreMap['com-ticketry.world'], - count: domainCountMap.get('com-ticketry.world'), - domainArrLen: domainArr.length - }); - } - - return domainArr; - - function calcDomainAbuseScore(subdomain: string, fullDomain: string = subdomain) { - if (leathalKeywords(fullDomain)) { - return 100; - } - - let weight = 0; - - const hitLowKeywords = lowKeywords(fullDomain); - const sensitiveKeywordsHit = sensitiveKeywords(fullDomain); - - if (sensitiveKeywordsHit) { - weight += 15; - if (hitLowKeywords) { - weight += 10; - } - } else if (hitLowKeywords) { - weight += 2; - } - - const subdomainLength = subdomain.length; - - if (subdomainLength > 6) { - weight += 0.015; - - if (subdomainLength > 13) { - weight += 0.2; - if (subdomainLength > 20) { - weight += 1; - if (subdomainLength > 30) { - weight += 5; - if (subdomainLength > 40) { - weight += 10; - } - } - } - - if (subdomain.indexOf('.', 1) > 1) { - weight += 1; - } - } - } - - return weight; - } - } - } - } -}); - -export function getPhishingDomains(parentSpan: Span) { - return parentSpan.traceChild('get phishing domains').traceAsyncFn(async (span) => span.traceChildAsync( +export function getPhishingDomains(isDebug = false): Promise { + return dummySpan.traceChild('get phishing domains').traceAsyncFn(async (span) => span.traceChildAsync( 'process phishing domain set', - () => pool.exec( - 'getPhishingDomains', - [__filename, require.main === module] - ).finally(() => pool.terminate()) + async () => { + const downloads = [ + ...PHISHING_DOMAIN_LISTS_EXTRA.map(entry => processDomainListsWithPreload(...entry)), + ...PHISHING_HOSTS_EXTRA.map(entry => processHostsWithPreload(...entry)) + ]; + + const domainArr: string[] = []; + + const domainGroups = await Promise.all(downloads.map(task => task(dummySpan))); + domainGroups.forEach(appendArrayInPlaceCurried(domainArr)); + + const domainCountMap = new Map(); + const domainScoreMap: Record = Object.create(null) as Record; + + let line: string; + let tld: string | null; + let apexDomain: string | null; + let subdomain: string | null; + let parsed: TldTsParsed; + + for (let i = 0, len = domainArr.length; i < len; i++) { + line = domainArr[i]; + + parsed = parse(line, loosTldOptWithPrivateDomains); + if (parsed.isPrivate) { + continue; + } + + tld = parsed.publicSuffix; + apexDomain = parsed.domain; + + if (!tld) { + console.log(picocolors.yellow('[phishing domains] E0001'), 'missing tld', { line, tld }); + continue; + } + if (!apexDomain) { + console.log(picocolors.yellow('[phishing domains] E0002'), 'missing domain', { line, apexDomain }); + continue; + } + if (WHITELIST_MAIN_DOMAINS.has(apexDomain)) { + continue; + } + + domainCountMap.set( + apexDomain, + domainCountMap.has(apexDomain) + ? domainCountMap.get(apexDomain)! + 1 + : 1 + ); + + let score = 0; + + if (apexDomain in domainScoreMap) { + score = domainScoreMap[apexDomain]; + } else { + if (BLACK_TLD.has(tld)) { + score += 3; + } else if (tld.length > 4) { + score += 2; + } else if (tld.length > 5) { + score += 4; + } + if (apexDomain.length >= 18) { + score += 0.5; + } + } + + subdomain = parsed.subdomain; + + if (subdomain) { + score += calcDomainAbuseScore(subdomain, line); + } + + domainScoreMap[apexDomain] = score; + } + + domainCountMap.forEach((count, apexDomain) => { + const score = domainScoreMap[apexDomain]; + if ( + (score >= 24) + || (score >= 16 && count >= 7) + || (score >= 13 && count >= 11) + || (score >= 5 && count >= 14) + || (score >= 3 && count >= 21) + || (score >= 1 && count >= 60) + ) { + domainArr.push('.' + apexDomain); + } + }); + + if (isDebug) { + console.log({ + v: 1, + score: domainScoreMap['com-ticketry.world'], + count: domainCountMap.get('com-ticketry.world'), + domainArrLen: domainArr.length + }); + } + + return domainArr; + } )); } -if (require.main === module) { - getPhishingDomains(dummySpan) - .catch(console.error) - .finally(() => { - dummySpan.stop(); - printTraceResult(dummySpan.traceResult); - }); +function calcDomainAbuseScore(subdomain: string, fullDomain: string = subdomain) { + if (leathalKeywords(fullDomain)) { + return 100; + } + + let weight = 0; + + const hitLowKeywords = lowKeywords(fullDomain); + const sensitiveKeywordsHit = sensitiveKeywords(fullDomain); + + if (sensitiveKeywordsHit) { + weight += 15; + if (hitLowKeywords) { + weight += 10; + } + } else if (hitLowKeywords) { + weight += 2; + } + + const subdomainLength = subdomain.length; + + if (subdomainLength > 6) { + weight += 0.015; + + if (subdomainLength > 13) { + weight += 0.2; + if (subdomainLength > 20) { + weight += 1; + if (subdomainLength > 30) { + weight += 5; + if (subdomainLength > 40) { + weight += 10; + } + } + } + + if (subdomain.indexOf('.', 1) > 1) { + weight += 1; + } + } + } + + return weight; +} + +if (!process.env.JEST_WORKER_ID && require.main === module) { + getPhishingDomains(true).catch(console.error); } diff --git a/Build/lib/get-telegram-backup-ip.ts b/Build/lib/get-telegram-backup-ip.ts index 286268a6..c8366e9c 100644 --- a/Build/lib/get-telegram-backup-ip.ts +++ b/Build/lib/get-telegram-backup-ip.ts @@ -7,10 +7,6 @@ import { bigint2ip } from 'fast-cidr-tools'; import { base64ToUint8Array, concatUint8Arrays } from 'foxts/uint8array-utils'; -import Worktank from 'worktank'; -import { wait } from 'foxts/wait'; -import { once } from 'foxts/once'; - const mtptoto_public_rsa = `-----BEGIN RSA PUBLIC KEY----- MIIBCgKCAQEAyr+18Rex2ohtVy8sroGP BwXD3DOoKCSpjDqYoXgCqB7ioln4eDCFfOBUlfXUEvM/fnKCpF46VkAftlb4VuPD @@ -112,160 +108,3 @@ export function getTelegramBackupIPFromBase64(base64: string) { } })); } - -const pool = new Worktank({ - pool: { - name: 'get-telegram-backup-ips', - size: 1 // The number of workers to keep in the pool, if more workers are needed they will be spawned up to this limit - }, - worker: { - autoAbort: 10000, - autoTerminate: 30000, // The interval of milliseconds at which to check if the pool can be automatically terminated, to free up resources, workers will be spawned up again if needed - autoInstantiate: true, - methods: { - // eslint-disable-next-line object-shorthand -- workertank - getTelegramBackupIPs: async function (__filename: string): Promise<{ timestamp: number, ipcidr: string[], ipcidr6: string[] }> { - // TODO: createRequire is a temporary workaround for https://github.com/nodejs/node/issues/51956 - const { default: module } = await import('node:module'); - const __require = module.createRequire(__filename); - - const picocolors = __require('picocolors') as typeof import('picocolors'); - const { '~fetch': fetch } = __require('./fetch-retry') as typeof import('./fetch-retry'); - - const dns = __require('node:dns/promises') as typeof import('node:dns/promises'); - - const { createReadlineInterfaceFromResponse } = __require('./fetch-text-by-line') as typeof import('./fetch-text-by-line'); - const { getTelegramBackupIPFromBase64 } = __require('./get-telegram-backup-ip') as typeof import('./get-telegram-backup-ip'); - const { fastIpVersion } = __require('foxts/fast-ip-version') as typeof import('foxts/fast-ip-version'); - const { fastStringArrayJoin } = __require('foxts/fast-string-array-join') as typeof import('foxts/fast-string-array-join'); - - const resp = await fetch('https://core.telegram.org/resources/cidr.txt'); - const lastModified = resp.headers.get('last-modified'); - const date = lastModified ? new Date(lastModified) : new Date(); - - const ipcidr: string[] = [ - // Unused secret Telegram backup CIDR, announced by AS62041 - '95.161.64.0/20' - ]; - const ipcidr6: string[] = []; - - for await (const cidr of createReadlineInterfaceFromResponse(resp, true)) { - const v = fastIpVersion(cidr); - if (v === 4) { - ipcidr.push(cidr); - } else if (v === 6) { - ipcidr6.push(cidr); - } - } - - const backupIPs = new Set(); - - // https://github.com/tdlib/td/blob/master/td/telegram/ConfigManager.cpp - - const resolvers = ['8.8.8.8', '1.0.0.1'].map((ip) => { - const resolver = new dns.Resolver(); - resolver.setServers([ip]); - return Object.assign(resolver, { server: ip }); - }); - - // Backup IP Source 1 (DNS) - await Promise.all(resolvers.flatMap((resolver) => [ - 'apv3.stel.com', // prod - 'tapv3.stel.com' // test - ].map(async (domain) => { - try { - // tapv3.stel.com was for testing server - const resp = await resolver.resolveTxt(domain); - const strings = resp.map(r => fastStringArrayJoin(r, '')); // flatten - if (strings.length !== 2) { - throw new TypeError(`Unexpected TXT record count: ${strings.length}`); - } - - const str = strings[0].length > strings[1].length - ? strings[0] + strings[1] - : strings[1] + strings[0]; - - const ips = getTelegramBackupIPFromBase64(str); - ips.forEach(i => backupIPs.add(i.ip)); - - console.log('[telegram backup ip]', picocolors.green('DNS TXT'), { domain, ips, server: resolver.server }); - } catch (e) { - console.error('[telegram backup ip]', picocolors.red('DNS TXT error'), { domain }, e); - } - }))); - - // Backup IP Source 2: Firebase Realtime Database (test server not supported) - try { - const text = await (await fetch('https://reserve-5a846.firebaseio.com/ipconfigv3.json')).json(); - if (typeof text === 'string' && text.length === 344) { - const ips = getTelegramBackupIPFromBase64(text); - ips.forEach(i => backupIPs.add(i.ip)); - - console.log('[telegram backup ip]', picocolors.green('Firebase Realtime DB'), { ips }); - } - } catch (e) { - console.error('[telegram backup ip]', picocolors.red('Firebase Realtime DB error'), e); - // ignore all errors - } - - // Backup IP Source 3: Firebase Value Store (test server not supported) - try { - const json = await (await fetch('https://firestore.googleapis.com/v1/projects/reserve-5a846/databases/(default)/documents/ipconfig/v3', { - headers: { - Accept: '*/*', - Origin: undefined // Without this line, Google API will return "Bad request: Origin doesn't match Host for XD3.". Probably have something to do with sqlite cache store - } - })).json(); - - // const json = await resp.json(); - if ( - json && typeof json === 'object' - && 'fields' in json && typeof json.fields === 'object' && json.fields - && 'data' in json.fields && typeof json.fields.data === 'object' && json.fields.data - && 'stringValue' in json.fields.data && typeof json.fields.data.stringValue === 'string' && json.fields.data.stringValue.length === 344 - ) { - const ips = getTelegramBackupIPFromBase64(json.fields.data.stringValue); - ips.forEach(i => backupIPs.add(i.ip)); - - console.log('[telegram backup ip]', picocolors.green('Firebase Value Store'), { ips }); - } else { - console.error('[telegram backup ip]', picocolors.red('Firebase Value Store data format invalid'), { json }); - } - } catch (e) { - console.error('[telegram backup ip]', picocolors.red('Firebase Value Store error'), e); - } - - // Backup IP Source 4: Google App Engine - await Promise.all([ - 'https://dns-telegram.appspot.com', - 'https://dns-telegram.appspot.com/test' - ].map(async (url) => { - try { - const text = await (await fetch(url)).text(); - if (text.length === 344) { - const ips = getTelegramBackupIPFromBase64(text); - ips.forEach(i => backupIPs.add(i.ip)); - - console.log('[telegram backup ip]', picocolors.green('Google App Engine'), { url, ips }); - } - } catch (e) { - console.error('[telegram backup ip]', picocolors.red('Google App Engine error'), { url }, e); - } - })); - - // tcdnb.azureedge.net no longer works - - console.log('[telegram backup ip]', `Found ${backupIPs.size} backup IPs:`, backupIPs); - - ipcidr.push(...Array.from(backupIPs).map(i => i + '/32')); - - return { timestamp: date.getTime(), ipcidr, ipcidr6 }; - } - } - } -}); - -export const getTelegramCIDRPromise = once(() => wait(0).then(() => pool.exec( - 'getTelegramBackupIPs', - [__filename] -)).finally(() => pool.terminate()), false); diff --git a/Build/lib/worker.ts b/Build/lib/worker.ts new file mode 100644 index 00000000..2a173829 --- /dev/null +++ b/Build/lib/worker.ts @@ -0,0 +1,28 @@ +import process from 'node:process'; +import type { JestWorkerFarm } from 'jest-worker'; +import { Worker as JestWorker } from 'jest-worker'; + +const sharedWorkerOptions = { + numWorkers: 1, + enableWorkerThreads: true, + forkOptions: { + env: { + ...process.env, + NODE_OPTIONS: process.env.NODE_OPTIONS + } + } +} satisfies ConstructorParameters[1]; + +export function createWorker>(workerPath: string) { + return >(exposedMethods: K): JestWorkerFarm> => { + const worker = new JestWorker(workerPath, { + ...sharedWorkerOptions, + exposedMethods + }) as JestWorkerFarm>; + + worker.getStdout().pipe(process.stdout); + worker.getStderr().pipe(process.stderr); + + return worker; + }; +} diff --git a/package.json b/package.json index 639ff50c..26fb61d8 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,6 @@ "undici": "^7.24.5", "undici-cache-store-better-sqlite3": "^1.0.1", "why-is-node-running": "^3.2.2", - "worktank": "^3.0.2", "xbits": "^0.2.0", "yaml": "^2.8.3", "yauzl-promise": "^4.0.0" @@ -62,6 +61,7 @@ "eslint": "^10.1.0", "eslint-config-sukka": "^8.9.0", "eslint-formatter-sukka": "^8.9.0", + "jest-worker": "^30.3.0", "mitata": "^1.0.34", "mocha": "^11.7.5", "tinyexec": "^1.0.4", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index acd56a65..ecc82cb9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -86,9 +86,6 @@ importers: why-is-node-running: specifier: ^3.2.2 version: 3.2.2 - worktank: - specifier: ^3.0.2 - version: 3.0.2 xbits: specifier: ^0.2.0 version: 0.2.0 @@ -138,6 +135,9 @@ importers: eslint-formatter-sukka: specifier: ^8.9.0 version: 8.9.0(eslint@10.1.0) + jest-worker: + specifier: ^30.3.0 + version: 30.3.0 mitata: specifier: ^1.0.34 version: 1.0.34 @@ -274,6 +274,18 @@ packages: resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} engines: {node: '>=12'} + '@jest/pattern@30.0.1': + resolution: {integrity: sha512-gWp7NfQW27LaBQz3TITS8L7ZCQ0TLvtmI//4OwlQRx4rnWxcPNIYjxZpDcN4+UlGxgm3jS5QPz8IPTCkb59wZA==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + '@jest/schemas@30.0.5': + resolution: {integrity: sha512-DmdYgtezMkh3cpU8/1uyXakv3tJRcmcXxBOcO0tbaozPwpmh4YMsnWrQm9ZmZMfa5ocbxzbFk6O4bDPEc/iAnA==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + '@jest/types@30.3.0': + resolution: {integrity: sha512-JHm87k7bA33hpBngtU8h6UBub/fqqA9uXfw+21j5Hmk7ooPHlboRNxHq0JcMtC+n8VJGP1mcfnD3Mk+XKe1oSw==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + '@mitata/counters@0.0.8': resolution: {integrity: sha512-f11w0Y1ETFlarDP7CePj8Z+y8Gv5Ax4gMxWsEwrqh0kH/YIY030Ezx5SUJeQg0YPTZ2OHKGcLG1oGJbIqHzaJA==} @@ -529,6 +541,9 @@ packages: '@remusao/trie@2.1.0': resolution: {integrity: sha512-Er3Q8q0/2OcCJPQYJOPLmCuqO0wu7cav3SPtpjlxSbjFi1x+A1pZkkLD6c9q2rGEkGW/tkrRzfrhNMt8VQjzXg==} + '@sinclair/typebox@0.34.49': + resolution: {integrity: sha512-brySQQs7Jtn0joV8Xh9ZV/hZb9Ozb0pmazDIASBkYKCjXrXU3mpcFahmK/z4YDhGkQvP9mWJbVyahdtU5wQA+A==} + '@swc-node/core@1.14.1': resolution: {integrity: sha512-jrt5GUaZUU6cmMS+WTJEvGvaB6j1YNKPHPzC2PUi2BjaFbtxURHj6641Az6xN7b665hNniAIdvjxWcRml5yCnw==} engines: {node: '>= 10'} @@ -639,6 +654,15 @@ packages: '@types/estree@1.0.8': resolution: {integrity: sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==} + '@types/istanbul-lib-coverage@2.0.6': + resolution: {integrity: sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==} + + '@types/istanbul-lib-report@3.0.3': + resolution: {integrity: sha512-NQn7AHQnk/RSLOxrBbGyJM/aVQ+pjj5HCgasFxc0K/KhoATfQ/47AyUl15I2yBUpihjmas+a+VJBOqecrFH+uA==} + + '@types/istanbul-reports@3.0.4': + resolution: {integrity: sha512-pk2B1NWalF9toCRu6gjBzR69syFjP4Od8WRAX+0mmf9lAjCRicLOWc+ZrxZHx/0XRjotgkF9t6iaMJ+aXcOdZQ==} + '@types/json-schema@7.0.15': resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==} @@ -654,6 +678,12 @@ packages: '@types/tar-stream@3.1.4': resolution: {integrity: sha512-921gW0+g29mCJX0fRvqeHzBlE/XclDaAG0Ousy1LCghsOhvaKacDeRGEVzQP9IPfKn8Vysy7FEXAIxycpc/CMg==} + '@types/yargs-parser@21.0.3': + resolution: {integrity: sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==} + + '@types/yargs@17.0.35': + resolution: {integrity: sha512-qUHkeCyQFxMXg79wQfTtfndEC+N9ZZg76HJftDJp+qH2tV7Gj4OJi7l+PiWwJ+pWtW8GwSmqsDj/oymhrTWXjg==} + '@types/yauzl-promise@4.0.1': resolution: {integrity: sha512-qYEC3rJwqiJpdQ9b+bPNeuSY0c3JUM8vIuDy08qfuVN7xHm3ZDsHn2kGphUIB0ruEXrPGNXZ64nMUcu4fDjViQ==} @@ -716,6 +746,9 @@ packages: resolution: {integrity: sha512-zhahknjobV2FiD6Ee9iLbS7OV9zi10rG26odsQdfBO/hjSzUQbkIYgda+iNKK1zNiW2ey+Lf8MU5btN17V3dUw==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + '@ungap/structured-clone@1.3.0': + resolution: {integrity: sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==} + '@unrs/resolver-binding-android-arm-eabi@1.11.1': resolution: {integrity: sha512-ppLRUgHVaGRWUx0R0Ut06Mjo9gBaBkg3v/8AxusGLhsIotbBLuRk51rAzqLC8gq6NyyAojEXglNjzf6R948DNw==} cpu: [arm] @@ -1404,9 +1437,6 @@ packages: resolution: {integrity: sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==} engines: {node: '>= 4'} - immediato@1.1.0: - resolution: {integrity: sha512-6DTWQWiM3SyxAbNRDmMvFgZVwVP6wT8ciQv7GivxXejtXZFIcemC0Wlzfd/jEouJ2JroCIp4qZVloKW4BviUpQ==} - imurmurhash@0.1.4: resolution: {integrity: sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==} engines: {node: '>=0.8.19'} @@ -1462,15 +1492,21 @@ packages: isexe@2.0.0: resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} - isoconcurrency@1.0.0: - resolution: {integrity: sha512-YhuPf5V6uOtQQHt9gIkOTbq75ceXqraDvxtZZeS/XbNsre6fmM+WpJgNTSkGX5jB3+gnbwoTVqW1c3qdfyVpOA==} - - isotimer@1.0.0: - resolution: {integrity: sha512-1p1wborMl9fFbulXx9YBpIqFnfUn/2tN8Ne9g3GLMaiQAPmN/wLlpNOKCNT822div3Sq7LKkApZJ+6JipDUusQ==} - jackspeak@3.4.3: resolution: {integrity: sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==} + jest-regex-util@30.0.1: + resolution: {integrity: sha512-jHEQgBXAgc+Gh4g0p3bCevgRCVRkB4VB70zhoAE48gxeSr1hfUOsM/C2WoJgVL7Eyg//hudYENbm3Ne+/dRVVA==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + jest-util@30.3.0: + resolution: {integrity: sha512-/jZDa00a3Sz7rdyu55NLrQCIrbyIkbBxareejQI315f/i8HjYN+ZWsDLLpoQSiUIEIyZF/R8fDg3BmB8AtHttg==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + jest-worker@30.3.0: + resolution: {integrity: sha512-DrCKkaQwHexjRUFTmPzs7sHQe0TSj9nvDALKGdwmK5mW9v7j90BudWirKAJHt3QQ9Dhrg1F7DogPzhChppkJpQ==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + js-yaml@4.1.0: resolution: {integrity: sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==} hasBin: true @@ -1513,6 +1549,9 @@ packages: lru-cache@10.4.3: resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} + merge-stream@2.0.0: + resolution: {integrity: sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==} + mime@3.0.0: resolution: {integrity: sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==} engines: {node: '>=10.0.0'} @@ -1650,9 +1689,6 @@ packages: resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==} engines: {node: '>= 0.8.0'} - promise-make-naked@3.0.2: - resolution: {integrity: sha512-B+b+kQ1YrYS7zO7P7bQcoqqMUizP06BOyNSBEnB5VJKDSWo8fsVuDkfSmwdjF0JsRtaNh83so5MMFJ95soH5jg==} - pump@3.0.3: resolution: {integrity: sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA==} @@ -1919,9 +1955,6 @@ packages: resolution: {integrity: sha512-/REy6amwPZl44DDzvRCkaI1q1bIiQB0mEFQLUrhz3z2EK91cp3n72rAjUlrTP0zV22HJIUOVHQGPxhFRjxjt+Q==} engines: {node: '>=4.0.0'} - webworker-shim@1.1.4: - resolution: {integrity: sha512-W/40L5W6ZQyGhYr3hJ7N/2SjdK5OdFtnYm94j6xlRyjckegXnIGwz0EdxdkQx6VGTglJjK8mqBhMz3fd3AY4bg==} - which@2.0.2: resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} engines: {node: '>= 8'} @@ -1939,9 +1972,6 @@ packages: workerpool@9.3.3: resolution: {integrity: sha512-slxCaKbYjEdFT/o2rH9xS1hf4uRDch1w7Uo+apxhZ+sf/1d9e0ZVkn42kPNGP2dgjIx6YFvSevj0zHvbWe2jdw==} - worktank@3.0.2: - resolution: {integrity: sha512-ry5gPtWnakOnUBAAa2aiyWZwAFJuBtd/MwZH6o9DXnQHD4AZvidtl2uTLrb2d3Zjy9D04n84lHJNnIETQl7tuA==} - wrap-ansi@7.0.0: resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==} engines: {node: '>=10'} @@ -2136,6 +2166,25 @@ snapshots: wrap-ansi: 8.1.0 wrap-ansi-cjs: wrap-ansi@7.0.0 + '@jest/pattern@30.0.1': + dependencies: + '@types/node': 24.12.0 + jest-regex-util: 30.0.1 + + '@jest/schemas@30.0.5': + dependencies: + '@sinclair/typebox': 0.34.49 + + '@jest/types@30.3.0': + dependencies: + '@jest/pattern': 30.0.1 + '@jest/schemas': 30.0.5 + '@types/istanbul-lib-coverage': 2.0.6 + '@types/istanbul-reports': 3.0.4 + '@types/node': 24.12.0 + '@types/yargs': 17.0.35 + chalk: 4.1.2 + '@mitata/counters@0.0.8': {} '@napi-rs/wasm-runtime@0.2.12': @@ -2311,6 +2360,8 @@ snapshots: '@remusao/trie@2.1.0': {} + '@sinclair/typebox@0.34.49': {} + '@swc-node/core@1.14.1(@swc/core@1.13.5)(@swc/types@0.1.25)': dependencies: '@swc/core': 1.13.5 @@ -2405,6 +2456,16 @@ snapshots: '@types/estree@1.0.8': {} + '@types/istanbul-lib-coverage@2.0.6': {} + + '@types/istanbul-lib-report@3.0.3': + dependencies: + '@types/istanbul-lib-coverage': 2.0.6 + + '@types/istanbul-reports@3.0.4': + dependencies: + '@types/istanbul-lib-report': 3.0.3 + '@types/json-schema@7.0.15': {} '@types/mocha@10.0.10': {} @@ -2422,6 +2483,12 @@ snapshots: dependencies: '@types/node': 24.12.0 + '@types/yargs-parser@21.0.3': {} + + '@types/yargs@17.0.35': + dependencies: + '@types/yargs-parser': 21.0.3 + '@types/yauzl-promise@4.0.1': dependencies: '@types/node': 24.12.0 @@ -2517,6 +2584,8 @@ snapshots: '@typescript-eslint/types': 8.57.2 eslint-visitor-keys: 5.0.1 + '@ungap/structured-clone@1.3.0': {} + '@unrs/resolver-binding-android-arm-eabi@1.11.1': optional: true @@ -3201,8 +3270,6 @@ snapshots: ignore@7.0.5: {} - immediato@1.1.0: {} - imurmurhash@0.1.4: {} inherits@2.0.4: {} @@ -3242,18 +3309,31 @@ snapshots: isexe@2.0.0: {} - isoconcurrency@1.0.0: {} - - isotimer@1.0.0: - dependencies: - immediato: 1.1.0 - jackspeak@3.4.3: dependencies: '@isaacs/cliui': 8.0.2 optionalDependencies: '@pkgjs/parseargs': 0.11.0 + jest-regex-util@30.0.1: {} + + jest-util@30.3.0: + dependencies: + '@jest/types': 30.3.0 + '@types/node': 24.12.0 + chalk: 4.1.2 + ci-info: 4.4.0 + graceful-fs: 4.2.11 + picomatch: 4.0.3 + + jest-worker@30.3.0: + dependencies: + '@types/node': 24.12.0 + '@ungap/structured-clone': 1.3.0 + jest-util: 30.3.0 + merge-stream: 2.0.0 + supports-color: 8.1.1 + js-yaml@4.1.0: dependencies: argparse: 2.0.1 @@ -3294,6 +3374,8 @@ snapshots: lru-cache@10.4.3: {} + merge-stream@2.0.0: {} + mime@3.0.0: {} mimic-response@3.1.0: {} @@ -3449,8 +3531,6 @@ snapshots: prelude-ls@1.2.1: {} - promise-make-naked@3.0.2: {} - pump@3.0.3: dependencies: end-of-stream: 1.4.5 @@ -3772,8 +3852,6 @@ snapshots: transitivePeerDependencies: - supports-color - webworker-shim@1.1.4: {} - which@2.0.2: dependencies: isexe: 2.0.0 @@ -3784,13 +3862,6 @@ snapshots: workerpool@9.3.3: {} - worktank@3.0.2: - dependencies: - isoconcurrency: 1.0.0 - isotimer: 1.0.0 - promise-make-naked: 3.0.2 - webworker-shim: 1.1.4 - wrap-ansi@7.0.0: dependencies: ansi-styles: 4.3.0