From eeeadbc86bdef9521ee48797b5fa0cbf94c92133 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Sat, 21 Sep 2024 19:04:11 +0800 Subject: [PATCH] Refactor: adjust more output --- ...c-direct-lan-ruleset-dns-mapping-module.ts | 8 +- Build/build-mitm-hostname.ts | 3 +- Build/build-sspanel-appprofile.ts | 175 ++++++++++-------- Build/build-telegram-cidr.ts | 16 +- Build/lib/clash.ts | 35 +--- Build/lib/rules/base.ts | 6 +- Build/lib/rules/domainset.ts | 6 +- Build/lib/rules/ruleset.ts | 10 +- Build/validate-domestic.ts | 3 +- Build/validate-gfwlist.ts | 2 +- 10 files changed, 131 insertions(+), 133 deletions(-) diff --git a/Build/build-domestic-direct-lan-ruleset-dns-mapping-module.ts b/Build/build-domestic-direct-lan-ruleset-dns-mapping-module.ts index b5276541..e6440d1b 100644 --- a/Build/build-domestic-direct-lan-ruleset-dns-mapping-module.ts +++ b/Build/build-domestic-direct-lan-ruleset-dns-mapping-module.ts @@ -32,7 +32,7 @@ export const getDomesticAndDirectDomainsRulesetPromise = createMemoizedPromise(a }); export const buildDomesticRuleset = task(require.main === module, __filename)(async (span) => { - const res = await getDomesticAndDirectDomainsRulesetPromise(); + const [domestics, directs, lans] = await getDomesticAndDirectDomainsRulesetPromise(); const dataset = Object.entries(DOMESTICS); appendArrayInPlace(dataset, Object.entries(DIRECTS)); @@ -46,7 +46,7 @@ export const buildDomesticRuleset = task(require.main === module, __filename)(as '', 'This file contains known addresses that are avaliable in the Mainland China.' ]) - .addFromRuleset(res[0]) + .addFromRuleset(domestics) .write(), new RulesetOutput(span, 'direct', 'non_ip') .withTitle('Sukka\'s Ruleset - Direct Rules') @@ -55,7 +55,7 @@ export const buildDomesticRuleset = task(require.main === module, __filename)(as '', 'This file contains domains and process that should not be proxied.' ]) - .addFromRuleset(res[1]) + .addFromRuleset(directs) .write(), new RulesetOutput(span, 'lan', 'non_ip') .withTitle('Sukka\'s Ruleset - LAN') @@ -64,7 +64,7 @@ export const buildDomesticRuleset = task(require.main === module, __filename)(as '', 'This file includes rules for LAN DOMAIN and reserved TLDs.' ]) - .addFromRuleset(res[2]) + .addFromRuleset(lans) .write(), compareAndWriteFile( span, diff --git a/Build/build-mitm-hostname.ts b/Build/build-mitm-hostname.ts index af7d9430..9d95dd5f 100644 --- a/Build/build-mitm-hostname.ts +++ b/Build/build-mitm-hostname.ts @@ -5,6 +5,7 @@ import { fdir as Fdir } from 'fdir'; import { green, yellow } from 'picocolors'; import { processLineFromReadline } from './lib/process-line'; import { getHostname } from 'tldts'; +import { OUTPUT_SURGE_DIR } from './constants/dir'; const PRESET_MITM_HOSTNAMES = [ // '*baidu.com', @@ -36,7 +37,7 @@ const PRESET_MITM_HOSTNAMES = [ ]; (async () => { - const folderListPath = pathFn.resolve(__dirname, '../List/'); + const folderListPath = pathFn.resolve(OUTPUT_SURGE_DIR, 'List/'); const rulesets = await new Fdir() .withFullPaths() diff --git a/Build/build-sspanel-appprofile.ts b/Build/build-sspanel-appprofile.ts index 09b63198..0415de72 100644 --- a/Build/build-sspanel-appprofile.ts +++ b/Build/build-sspanel-appprofile.ts @@ -1,6 +1,5 @@ import { getAppleCdnDomainsPromise } from './build-apple-cdn'; import { getDomesticAndDirectDomainsRulesetPromise } from './build-domestic-direct-lan-ruleset-dns-mapping-module'; -import { surgeRulesetToClashClassicalTextRuleset, surgeDomainsetToClashRuleset } from './lib/clash'; import { readFileIntoProcessedArray } from './lib/fetch-text-by-line'; import { task } from './trace'; import path from 'node:path'; @@ -8,11 +7,11 @@ import path from 'node:path'; import { ALL as AllStreamServices } from '../Source/stream'; import { getChnCidrPromise } from './build-chn-cidr'; import { getTelegramCIDRPromise } from './build-telegram-cidr'; -import { compareAndWriteFile } from './lib/create-file'; +import { compareAndWriteFile, RulesetOutput } from './lib/create-file'; import { getMicrosoftCdnRulesetPromise } from './build-microsoft-cdn'; import { isTruthy } from './lib/misc'; import { appendArrayInPlace } from './lib/append-array-in-place'; -import { OUTPUT_INTERNAL_DIR, SOURCE_DIR } from './constants/dir'; +import { OUTPUT_INTERNAL_DIR, OUTPUT_SURGE_DIR, SOURCE_DIR } from './constants/dir'; const POLICY_GROUPS: Array<[name: string, insertProxy: boolean, insertDirect: boolean]> = [ ['Default Proxy', true, false], @@ -24,99 +23,123 @@ const POLICY_GROUPS: Array<[name: string, insertProxy: boolean, insertDirect: bo ['Final Match', true, true] ]; -const removeNoResolved = (line: string) => line.replace(',no-resolve', ''); - /** * This only generates a simplified version, for under-used users only. */ export const buildSSPanelUIMAppProfile = task(require.main === module, __filename)(async (span) => { const [ - [domesticDomains, directDomains, lanDomains], + [domesticRules, directRules, lanRules], appleCdnDomains, - microsoftCdnDomains, - appleCnDomains, - neteaseMusicDomains, - microsoftDomains, - appleDomains, - streamDomains, - steamDomains, - globalDomains, - telegramDomains, - domesticCidrs, - streamCidrs, - { results: rawTelegramCidrs }, - lanCidrs + microsoftCdnRules, + appleCnRules, + neteaseMusicRules, + microsoftRules, + appleRules, + streamRules, + steamDomainset, + globalRules, + telegramRules, + [domesticCidrs4, domesticCidrs6], + [streamCidrs4, streamCidrs6], + { ipcidr: telegramCidrs4, ipcidr6: telegramCidrs6 }, + rawLanCidrs ] = await Promise.all([ // domestic - domains - getDomesticAndDirectDomainsRulesetPromise() - .then( - data => ( - data.map(surgeRulesetToClashClassicalTextRuleset) - ) as [string[], string[], string[]] - ), - getAppleCdnDomainsPromise().then(domains => domains.map(domain => `DOMAIN-SUFFIX,${domain}`)), - getMicrosoftCdnRulesetPromise().then(surgeRulesetToClashClassicalTextRuleset), - readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/apple_cn.conf')), - readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/neteasemusic.conf')).then(surgeRulesetToClashClassicalTextRuleset), + getDomesticAndDirectDomainsRulesetPromise(), + getAppleCdnDomainsPromise(), + getMicrosoftCdnRulesetPromise(), + readFileIntoProcessedArray(path.join(OUTPUT_SURGE_DIR, 'non_ip/apple_cn.conf')), + readFileIntoProcessedArray(path.join(OUTPUT_SURGE_DIR, 'non_ip/neteasemusic.conf')), // microsoft & apple - domains - readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/microsoft.conf')), - readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/apple_services.conf')).then(surgeRulesetToClashClassicalTextRuleset), + readFileIntoProcessedArray(path.join(OUTPUT_SURGE_DIR, 'non_ip/microsoft.conf')), + readFileIntoProcessedArray(path.join(OUTPUT_SURGE_DIR, 'non_ip/apple_services.conf')), // stream - domains - surgeRulesetToClashClassicalTextRuleset(AllStreamServices.flatMap((i) => i.rules)), + AllStreamServices.flatMap((i) => i.rules), // steam - domains - readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/steam.conf')).then(surgeDomainsetToClashRuleset), + readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/steam.conf')), // global - domains - readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/global.conf')).then(surgeRulesetToClashClassicalTextRuleset), - readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/telegram.conf')).then(surgeRulesetToClashClassicalTextRuleset), + readFileIntoProcessedArray(path.join(OUTPUT_SURGE_DIR, 'non_ip/global.conf')), + readFileIntoProcessedArray(path.join(OUTPUT_SURGE_DIR, 'non_ip/telegram.conf')), // domestic - ip cidr - getChnCidrPromise().then(([cidrs4, cidrs6]) => [ - ...cidrs4.map(cidr => `IP-CIDR,${cidr}`), - ...cidrs6.map(cidr => `IP-CIDR,${cidr}`) - ]), - AllStreamServices.flatMap((i) => ( - i.ip - ? [ - ...i.ip.v4.map((ip) => `IP-CIDR,${ip}`), - ...i.ip.v6.map((ip) => `IP-CIDR6,${ip}`) - ] - : [] - )), + getChnCidrPromise(), + AllStreamServices.reduce<[cidr4: string[], cidr6: string[]]>((acc, i) => { + if (i.ip) { + appendArrayInPlace(acc[0], i.ip.v4); + appendArrayInPlace(acc[1], i.ip.v6); + } + + return acc; + }, [[], []]), // global - ip cidr getTelegramCIDRPromise(), // lan - ip cidr - readFileIntoProcessedArray(path.join(SOURCE_DIR, 'ip/lan.conf')) + readFileIntoProcessedArray(path.join(OUTPUT_SURGE_DIR, 'ip/lan.conf')) ] as const); - const telegramCidrs = rawTelegramCidrs.map(removeNoResolved); + const domestic = new RulesetOutput(span, '_', 'non_ip') + .addFromRuleset(domesticRules) + .bulkAddDomainSuffix(appleCdnDomains) + .addFromRuleset(microsoftCdnRules) + .addFromRuleset(appleCnRules) + .addFromRuleset(neteaseMusicRules); + + const microsoftApple = new RulesetOutput(span, '_', 'non_ip') + .addFromRuleset(microsoftRules) + .addFromRuleset(appleRules); + + const stream = new RulesetOutput(span, '_', 'non_ip') + .addFromRuleset(streamRules); + + const steam = new RulesetOutput(span, '_', 'non_ip') + .addFromDomainset(steamDomainset); + + const global = new RulesetOutput(span, '_', 'non_ip') + .addFromRuleset(globalRules) + .addFromRuleset(telegramRules); + + const direct = new RulesetOutput(span, '_', 'non_ip') + .addFromRuleset(directRules) + .addFromRuleset(lanRules); + + const domesticCidr = new RulesetOutput(span, '_', 'ip') + .bulkAddCIDR4(domesticCidrs4) + .bulkAddCIDR6(domesticCidrs6); + + const streamCidr = new RulesetOutput(span, '_', 'ip') + .bulkAddCIDR4(streamCidrs4) + .bulkAddCIDR6(streamCidrs6); + + const telegramCidr = new RulesetOutput(span, '_', 'ip') + .bulkAddCIDR4(telegramCidrs4) + .bulkAddCIDR6(telegramCidrs6); + + const lanCidrs = new RulesetOutput(span, '_', 'ip') + .addFromRuleset(rawLanCidrs); + + await Promise.all([ + domestic.done(), + microsoftApple.done(), + stream.done(), + steam.done(), + global.done(), + direct.done(), + domesticCidr.done(), + streamCidr.done(), + telegramCidr.done(), + lanCidrs.done() + ]); const output = generateAppProfile( - [ - ...domesticDomains, - ...appleCdnDomains, - ...microsoftCdnDomains, - ...appleCnDomains, - ...neteaseMusicDomains - ], - [ - ...microsoftDomains, - ...appleDomains - ], - streamDomains, - steamDomains, - [ - ...globalDomains, - ...telegramDomains - ], - [ - ...directDomains, - ...lanDomains - ], - domesticCidrs, - streamCidrs, - [ - ...telegramCidrs - ], - lanCidrs + domestic.clash(), + microsoftApple.clash(), + stream.clash(), + steam.clash(), + global.clash(), + direct.clash(), + domesticCidr.clash(), + streamCidr.clash(), + telegramCidr.clash(), + lanCidrs.clash() ); await compareAndWriteFile( diff --git a/Build/build-telegram-cidr.ts b/Build/build-telegram-cidr.ts index 62a51608..727ed6fd 100644 --- a/Build/build-telegram-cidr.ts +++ b/Build/build-telegram-cidr.ts @@ -13,7 +13,8 @@ export const getTelegramCIDRPromise = createMemoizedPromise(async () => { const lastModified = resp.headers.get('last-modified'); const date = lastModified ? new Date(lastModified) : new Date(); - const results: string[] = []; + const ipcidr: string[] = []; + const ipcidr6: string[] = []; for await (const line of createReadlineInterfaceFromResponse(resp)) { const cidr = processLine(line); @@ -21,20 +22,20 @@ export const getTelegramCIDRPromise = createMemoizedPromise(async () => { const [subnet] = cidr.split('/'); if (isProbablyIpv4(subnet)) { - results.push(`IP-CIDR,${cidr},no-resolve`); + ipcidr.push(cidr); } if (isProbablyIpv6(subnet)) { - results.push(`IP-CIDR6,${cidr},no-resolve`); + ipcidr6.push(cidr); } } - return { date, results }; + return { date, ipcidr, ipcidr6 }; }); export const buildTelegramCIDR = task(require.main === module, __filename)(async (span) => { - const { date, results } = await span.traceChildAsync('get telegram cidr', getTelegramCIDRPromise); + const { date, ipcidr, ipcidr6 } = await span.traceChildAsync('get telegram cidr', getTelegramCIDRPromise); - if (results.length === 0) { + if (ipcidr.length + ipcidr6.length === 0) { throw new Error('Failed to fetch data!'); } @@ -48,6 +49,7 @@ export const buildTelegramCIDR = task(require.main === module, __filename)(async .withTitle('Sukka\'s Ruleset - Telegram IP CIDR') .withDescription(description) .withDate(date) - .addFromRuleset(results) + .bulkAddCIDR4NoResolve(ipcidr) + .bulkAddCIDR6NoResolve(ipcidr6) .write(); }); diff --git a/Build/lib/clash.ts b/Build/lib/clash.ts index 0951375a..924a8f5d 100644 --- a/Build/lib/clash.ts +++ b/Build/lib/clash.ts @@ -1,11 +1,10 @@ -import picocolors from 'picocolors'; import { domainWildCardToRegex, identity } from './misc'; import { isProbablyIpv4, isProbablyIpv6 } from './is-fast-ip'; const unsupported = Symbol('unsupported'); // https://dreamacro.github.io/clash/configuration/rules.html -const PROCESSOR: Record string) | typeof unsupported> = { +export const PROCESSOR: Record string) | typeof unsupported> = { DOMAIN: identity, 'DOMAIN-SUFFIX': identity, 'DOMAIN-KEYWORD': identity, @@ -35,35 +34,3 @@ const PROCESSOR: Record s 'URL-REGEX': unsupported, 'USER-AGENT': unsupported }; - -export const surgeRulesetToClashClassicalTextRuleset = (rules: string[] | Set) => { - return Array.from(rules).reduce((acc, cur) => { - let buf = ''; - let type = ''; - let i = 0; - for (const len = cur.length; i < len; i++) { - if (cur[i] === ',') { - type = buf; - break; - } - buf += cur[i]; - } - if (type === '') { - return acc; - } - const value = cur.slice(i + 1); - if (type in PROCESSOR) { - const proc = PROCESSOR[type]; - if (proc !== unsupported) { - acc.push(proc(cur, type, value)); - } - } else { - console.log(picocolors.yellow(`[clash] unknown rule type: ${type}`), cur); - } - return acc; - }, []); -}; - -export const surgeDomainsetToClashRuleset = (domainset: string[]) => { - return domainset.map(i => (i[0] === '.' ? `DOMAIN-SUFFIX,${i.slice(1)}` : `DOMAIN,${i}`)); -}; diff --git a/Build/lib/rules/base.ts b/Build/lib/rules/base.ts index f8a98c98..af2f7457 100644 --- a/Build/lib/rules/base.ts +++ b/Build/lib/rules/base.ts @@ -218,8 +218,12 @@ export abstract class RuleOutput { abstract clash(): string[]; abstract singbox(): string[]; + done() { + return this.pendingPromise; + } + async write(): Promise { - await this.pendingPromise; + await this.done(); invariant(this.title, 'Missing title'); invariant(this.description, 'Missing description'); diff --git a/Build/lib/rules/domainset.ts b/Build/lib/rules/domainset.ts index 4be35f22..4cf1a215 100644 --- a/Build/lib/rules/domainset.ts +++ b/Build/lib/rules/domainset.ts @@ -76,15 +76,15 @@ export class DomainsetOutput extends RuleOutput { invariant(this.apexDomainMap, 'Missing apex domain map'); return Array.from( - ( - nullthrow(this.sorted, 'Non dumped yet').reduce>((acc, cur) => { + nullthrow(this.sorted, 'Non dumped yet') + .reduce>((acc, cur) => { const suffix = this.apexDomainMap!.get(cur); if (suffix) { acc.set(suffix, (acc.get(suffix) ?? 0) + 1); } return acc; }, new Map()) - ).entries() + .entries() ) .filter(a => a[1] > 9) .sort( diff --git a/Build/lib/rules/ruleset.ts b/Build/lib/rules/ruleset.ts index 7c4e6e75..064c5d34 100644 --- a/Build/lib/rules/ruleset.ts +++ b/Build/lib/rules/ruleset.ts @@ -12,7 +12,7 @@ export class RulesetOutput extends RuleOutput { } private $computed: [domain: string[], domainSuffix: string[], sortedDomainRules: string[]] | null = null; - private get computed() { + private computed() { if (!this.$computed) { const kwfilter = createKeywordFilter(this.domainKeywords); @@ -40,7 +40,7 @@ export class RulesetOutput extends RuleOutput { surge(): string[] { const results: string[] = ['DOMAIN,this_ruleset_is_made_by_sukkaw.ruleset.skk.moe']; - appendArrayInPlace(results, this.computed[2]); + appendArrayInPlace(results, this.computed()[2]); appendArrayFromSet(results, this.domainKeywords, i => `DOMAIN-KEYWORD,${i}`); appendArrayFromSet(results, this.domainWildcard, i => `DOMAIN-WILDCARD,${i}`); @@ -70,7 +70,7 @@ export class RulesetOutput extends RuleOutput { clash(): string[] { const results: string[] = ['DOMAIN,this_ruleset_is_made_by_sukkaw.ruleset.skk.moe']; - appendArrayInPlace(results, this.computed[2]); + appendArrayInPlace(results, this.computed()[2]); appendArrayFromSet(results, this.domainKeywords, i => `DOMAIN-KEYWORD,${i}`); appendArrayFromSet(results, this.domainWildcard, i => `DOMAIN-REGEX,${RuleOutput.domainWildCardToRegex(i)}`); @@ -97,8 +97,8 @@ export class RulesetOutput extends RuleOutput { const singbox: SingboxSourceFormat = { version: 2, rules: [{ - domain: ['this_ruleset_is_made_by_sukkaw.ruleset.skk.moe'].concat(this.computed[0]), - domain_suffix: this.computed[1], + domain: ['this_ruleset_is_made_by_sukkaw.ruleset.skk.moe'].concat(this.computed()[0]), + domain_suffix: this.computed()[1], domain_keyword: Array.from(this.domainKeywords), domain_regex: Array.from(this.domainWildcard).map(RuleOutput.domainWildCardToRegex), ip_cidr: appendArrayFromSet([], [this.ipcidr, this.ipcidrNoResolve, this.ipcidr6, this.ipcidr6NoResolve]), diff --git a/Build/validate-domestic.ts b/Build/validate-domestic.ts index fdfdadc6..a09cc34d 100644 --- a/Build/validate-domestic.ts +++ b/Build/validate-domestic.ts @@ -4,6 +4,7 @@ import { createTrie } from './lib/trie'; import path from 'node:path'; import { processLine } from './lib/process-line'; import { extractDomainsFromFelixDnsmasq } from './lib/parse-dnsmasq'; +import { SOURCE_DIR } from './constants/dir'; export const parseDomesticList = async () => { const set = new Set(); @@ -62,7 +63,7 @@ export const parseDomesticList = async () => { }; // await Promise.all([ - await runAgainstRuleset(path.resolve(__dirname, '../List/non_ip/domestic.conf')); + await runAgainstRuleset(path.resolve(SOURCE_DIR, 'non_ip/domestic.conf')); // ]); console.log(notIncludedDomestic.size, notIncludedDomestic); diff --git a/Build/validate-gfwlist.ts b/Build/validate-gfwlist.ts index 30069180..72f97657 100644 --- a/Build/validate-gfwlist.ts +++ b/Build/validate-gfwlist.ts @@ -108,7 +108,7 @@ export const parseGfwList = async () => { await Promise.all([ runAgainstRuleset(path.join(SOURCE_DIR, 'non_ip/global.conf')), runAgainstRuleset(path.join(SOURCE_DIR, 'non_ip/telegram.conf')), - runAgainstRuleset(path.resolve(__dirname, '../List/non_ip/stream.conf')) + runAgainstRuleset(path.resolve(SOURCE_DIR, 'non_ip/stream.conf')) ]); console.log(notIncludedTop500Gfwed);