From b22079f5eb4860fc24443c7307bf3ea0887b20c0 Mon Sep 17 00:00:00 2001 From: Sukka Date: Wed, 29 Jan 2025 03:58:49 +0800 Subject: [PATCH] Refactor: new write strategy (#58) --- Build/build-common.ts | 4 +- ...c-direct-lan-ruleset-dns-mapping-module.ts | 25 +- Build/build-reject-domainset.ts | 57 ++- Build/build-sgmodule-always-realip.ts | 5 +- Build/build-sspanel-appprofile.ts | 53 +-- Build/lib/misc.ts | 11 + Build/lib/rules/base.ts | 346 +++++++++++------- Build/lib/rules/domainset.ts | 114 +----- Build/lib/rules/ip.ts | 78 +--- Build/lib/rules/ruleset.ts | 284 +------------- Build/lib/singbox.ts | 19 - Build/lib/writing-strategy/adguardhome.ts | 107 ++++++ Build/lib/writing-strategy/base.ts | 81 ++++ Build/lib/writing-strategy/clash.ts | 169 +++++++++ Build/lib/writing-strategy/singbox.ts | 152 ++++++++ Build/lib/writing-strategy/surge.ts | 262 +++++++++++++ Source/non_ip/reject-url-regex.conf | 2 +- 17 files changed, 1096 insertions(+), 673 deletions(-) delete mode 100644 Build/lib/singbox.ts create mode 100644 Build/lib/writing-strategy/adguardhome.ts create mode 100644 Build/lib/writing-strategy/base.ts create mode 100644 Build/lib/writing-strategy/clash.ts create mode 100644 Build/lib/writing-strategy/singbox.ts create mode 100644 Build/lib/writing-strategy/surge.ts diff --git a/Build/build-common.ts b/Build/build-common.ts index 5cffc3cb..96abfdf4 100644 --- a/Build/build-common.ts +++ b/Build/build-common.ts @@ -112,7 +112,7 @@ async function transform(parentSpan: Span, sourcePath: string, relativePath: str const res = await processFile(span, sourcePath); if (res === $skip) return; - const [title, descriptions, lines, sgmodulePathname] = res; + const [title, descriptions, lines, sgmoduleName] = res; let finalDescriptions: string[]; if (descriptions.length) { @@ -134,7 +134,7 @@ async function transform(parentSpan: Span, sourcePath: string, relativePath: str return new RulesetOutput(span, id, type) .withTitle(title) .withDescription(finalDescriptions) - .withMitmSgmodulePath(sgmodulePathname) + .withMitmSgmodulePath(sgmoduleName) .addFromRuleset(lines) .write(); }); diff --git a/Build/build-domestic-direct-lan-ruleset-dns-mapping-module.ts b/Build/build-domestic-direct-lan-ruleset-dns-mapping-module.ts index cc16a071..bef1bfeb 100644 --- a/Build/build-domestic-direct-lan-ruleset-dns-mapping-module.ts +++ b/Build/build-domestic-direct-lan-ruleset-dns-mapping-module.ts @@ -12,6 +12,7 @@ import * as yaml from 'yaml'; import { appendArrayInPlace } from './lib/append-array-in-place'; import { OUTPUT_INTERNAL_DIR, OUTPUT_MODULES_DIR, OUTPUT_MODULES_RULES_DIR, SOURCE_DIR } from './constants/dir'; import { RulesetOutput } from './lib/create-file'; +import { SurgeRuleSet } from './lib/writing-strategy/surge'; export function createGetDnsMappingRule(allowWildcard: boolean) { const hasWildcard = (domain: string) => { @@ -114,12 +115,17 @@ export const buildDomesticRuleset = task(require.main === module, __filename)(as return; } - const output = new RulesetOutput(span, name.toLowerCase(), 'sukka_local_dns_mapping').withTitle(`Sukka's Ruleset - Local DNS Mapping (${name})`).withDescription([ - ...SHARED_DESCRIPTION, - '', - 'This is an internal rule that is only referenced by sukka_local_dns_mapping.sgmodule', - 'Do not use this file in your Rule section, all rules are included in non_ip/domestic.conf already.' - ]); + const output = new RulesetOutput(span, name.toLowerCase(), 'sukka_local_dns_mapping') + .withTitle(`Sukka's Ruleset - Local DNS Mapping (${name})`) + .withDescription([ + ...SHARED_DESCRIPTION, + '', + 'This is an internal rule that is only referenced by sukka_local_dns_mapping.sgmodule', + 'Do not use this file in your Rule section, all rules are included in non_ip/domestic.conf already.' + ]) + .replaceStrategies([ + new SurgeRuleSet('sukka_local_dns_mapping', OUTPUT_MODULES_RULES_DIR) + ]); domains.forEach((domain) => { switch (domain[0]) { @@ -135,12 +141,7 @@ export const buildDomesticRuleset = task(require.main === module, __filename)(as } }); - return output.write({ - surge: true, - clash: false, - singbox: false, - surgeDir: OUTPUT_MODULES_RULES_DIR - }); + return output.write(); }), compareAndWriteFile( diff --git a/Build/build-reject-domainset.ts b/Build/build-reject-domainset.ts index ec7c7705..84155019 100644 --- a/Build/build-reject-domainset.ts +++ b/Build/build-reject-domainset.ts @@ -7,7 +7,6 @@ import { processDomainListsWithPreload } from './lib/parse-filter/domainlists'; import { processFilterRulesWithPreload } from './lib/parse-filter/filters'; import { HOSTS, ADGUARD_FILTERS, PREDEFINED_WHITELIST, DOMAIN_LISTS, HOSTS_EXTRA, DOMAIN_LISTS_EXTRA, ADGUARD_FILTERS_EXTRA, PHISHING_DOMAIN_LISTS_EXTRA, ADGUARD_FILTERS_WHITELIST } from './constants/reject-data-source'; -import { compareAndWriteFile } from './lib/create-file'; import { readFileIntoProcessedArray } from './lib/fetch-text-by-line'; import { task } from './trace'; // tldts-experimental is way faster than tldts, but very little bit inaccurate @@ -17,10 +16,11 @@ import { SHARED_DESCRIPTION } from './constants/description'; import { getPhishingDomains } from './lib/get-phishing-domains'; import { addArrayElementsToSet } from 'foxts/add-array-elements-to-set'; -import { appendArrayInPlace } from './lib/append-array-in-place'; import { OUTPUT_INTERNAL_DIR, SOURCE_DIR } from './constants/dir'; import { DomainsetOutput } from './lib/create-file'; import { foundDebugDomain } from './lib/parse-filter/shared'; +import { AdGuardHome } from './lib/writing-strategy/adguardhome'; +import { FileOutput } from './lib/rules/base'; const readLocalRejectDomainsetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/reject_sukka.conf')); const readLocalRejectExtraDomainsetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/reject_sukka_extra.conf')); @@ -125,6 +125,7 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as ].flat())); if (foundDebugDomain.value) { + // eslint-disable-next-line sukka/unicorn/no-process-exit -- cli App process.exit(1); } @@ -140,39 +141,29 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as rejectExtraOutput.whitelistDomain(domain); } - for (let i = 0, len = rejectOutput.$preprocessed.length; i < len; i++) { - rejectExtraOutput.whitelistDomain(rejectOutput.$preprocessed[i]); - } + rejectOutput.domainTrie.dump(rejectExtraOutput.whitelistDomain.bind(rejectExtraOutput)); }); - return Promise.all([ + await Promise.all([ rejectOutput.write(), - rejectExtraOutput.write(), - compareAndWriteFile( - span, - appendArrayInPlace( - [ - '! Title: Sukka\'s Ruleset - Blocklist for AdGuardHome', - '! Last modified: ' + new Date().toUTCString(), - '! Expires: 6 hours', - '! License: https://github.com/SukkaW/Surge/blob/master/LICENSE', - '! Homepage: https://github.com/SukkaW/Surge', - '! Description: The domainset supports AD blocking, tracking protection, privacy protection, anti-phishing, anti-mining', - '!' - ], - appendArrayInPlace( - rejectOutput.adguardhome(), - ( - await new DomainsetOutput(span, 'my_reject') - .addFromRuleset(readLocalMyRejectRulesetPromise) - .addFromRuleset(readLocalRejectRulesetPromise) - .addFromRuleset(readLocalRejectDropRulesetPromise) - .addFromRuleset(readLocalRejectNoDropRulesetPromise) - .done() - ).adguardhome() - ) - ), - path.join(OUTPUT_INTERNAL_DIR, 'reject-adguardhome.txt') - ) + rejectExtraOutput.write() ]); + + // we are going to re-use rejectOutput's domainTrie and mutate it + // so we must wait until we write rejectOutput to disk after we can mutate its trie + const rejectOutputAdGuardHome = new FileOutput(span, 'reject-adguardhome') + .withTitle('Sukka\'s Ruleset - Blocklist for AdGuardHome') + .withDescription([ + 'The domainset supports AD blocking, tracking protection, privacy protection, anti-phishing, anti-mining' + ]) + .replaceStrategies([new AdGuardHome(OUTPUT_INTERNAL_DIR)]); + + rejectOutputAdGuardHome.domainTrie = rejectOutput.domainTrie; + + await rejectOutputAdGuardHome + .addFromRuleset(readLocalMyRejectRulesetPromise) + .addFromRuleset(readLocalRejectRulesetPromise) + .addFromRuleset(readLocalRejectDropRulesetPromise) + .addFromRuleset(readLocalRejectNoDropRulesetPromise) + .write(); }); diff --git a/Build/build-sgmodule-always-realip.ts b/Build/build-sgmodule-always-realip.ts index c9ee3a81..3bda5d65 100644 --- a/Build/build-sgmodule-always-realip.ts +++ b/Build/build-sgmodule-always-realip.ts @@ -9,6 +9,7 @@ import { OUTPUT_INTERNAL_DIR, OUTPUT_MODULES_DIR } from './constants/dir'; import { appendArrayInPlace } from './lib/append-array-in-place'; import { SHARED_DESCRIPTION } from './constants/description'; import { createGetDnsMappingRule } from './build-domestic-direct-lan-ruleset-dns-mapping-module'; +import { ClashDomainSet } from './lib/writing-strategy/clash'; const HOSTNAMES = [ // Network Detection, Captive Portal @@ -44,6 +45,9 @@ export const buildAlwaysRealIPModule = task(require.main === module, __filename) ...SHARED_DESCRIPTION, '', 'Clash.Meta fake-ip-filter as ruleset' + ]) + .replaceStrategies([ + new ClashDomainSet('domainset') ]); // Intranet, Router Setup, and mant more @@ -75,7 +79,6 @@ export const buildAlwaysRealIPModule = task(require.main === module, __filename) ], path.resolve(OUTPUT_MODULES_DIR, 'sukka_common_always_realip.sgmodule') ), - clashFakeIpFilter.writeClash(), compareAndWriteFile( span, yaml.stringify( diff --git a/Build/build-sspanel-appprofile.ts b/Build/build-sspanel-appprofile.ts index 57ae2206..411a53be 100644 --- a/Build/build-sspanel-appprofile.ts +++ b/Build/build-sspanel-appprofile.ts @@ -9,9 +9,10 @@ import { getChnCidrPromise } from './build-chn-cidr'; import { getTelegramCIDRPromise } from './build-telegram-cidr'; import { compareAndWriteFile, RulesetOutput } from './lib/create-file'; import { getMicrosoftCdnRulesetPromise } from './build-microsoft-cdn'; -import { isTruthy } from 'foxts/guard'; +import { isTruthy, nullthrow } from 'foxts/guard'; import { appendArrayInPlace } from './lib/append-array-in-place'; import { OUTPUT_INTERNAL_DIR, OUTPUT_SURGE_DIR, SOURCE_DIR } from './constants/dir'; +import { ClashClassicRuleSet } from './lib/writing-strategy/clash'; const POLICY_GROUPS: Array<[name: string, insertProxy: boolean, insertDirect: boolean]> = [ ['Default Proxy', true, false], @@ -79,6 +80,7 @@ export const buildSSPanelUIMAppProfile = task(require.main === module, __filenam ] as const); const domestic = new RulesetOutput(span, '_', 'non_ip') + .replaceStrategies([new ClashClassicRuleSet('non_ip')]) .addFromRuleset(domesticRules) .bulkAddDomainSuffix(appleCdnDomains) .bulkAddDomain(microsoftCdnDomains) @@ -87,62 +89,65 @@ export const buildSSPanelUIMAppProfile = task(require.main === module, __filenam .addFromRuleset(neteaseMusicRules); const microsoftApple = new RulesetOutput(span, '_', 'non_ip') + .replaceStrategies([new ClashClassicRuleSet('non_ip')]) .addFromRuleset(microsoftRules) .addFromRuleset(appleRules); const stream = new RulesetOutput(span, '_', 'non_ip') + .replaceStrategies([new ClashClassicRuleSet('non_ip')]) .addFromRuleset(streamRules); const steam = new RulesetOutput(span, '_', 'non_ip') + .replaceStrategies([new ClashClassicRuleSet('non_ip')]) .addFromDomainset(steamDomainset); const global = new RulesetOutput(span, '_', 'non_ip') + .replaceStrategies([new ClashClassicRuleSet('non_ip')]) .addFromRuleset(globalRules) .addFromRuleset(telegramRules); const direct = new RulesetOutput(span, '_', 'non_ip') + .replaceStrategies([new ClashClassicRuleSet('non_ip')]) .addFromRuleset(directRules) .addFromRuleset(lanRules); const domesticCidr = new RulesetOutput(span, '_', 'ip') + .replaceStrategies([new ClashClassicRuleSet('ip')]) .bulkAddCIDR4(domesticCidrs4) .bulkAddCIDR6(domesticCidrs6); const streamCidr = new RulesetOutput(span, '_', 'ip') + .replaceStrategies([new ClashClassicRuleSet('ip')]) .bulkAddCIDR4(streamCidrs4) .bulkAddCIDR6(streamCidrs6); const telegramCidr = new RulesetOutput(span, '_', 'ip') + .replaceStrategies([new ClashClassicRuleSet('ip')]) .bulkAddCIDR4(telegramCidrs4) .bulkAddCIDR6(telegramCidrs6); const lanCidrs = new RulesetOutput(span, '_', 'ip') + .replaceStrategies([new ClashClassicRuleSet('ip')]) .addFromRuleset(rawLanCidrs); - await Promise.all([ - domestic.done(), - microsoftApple.done(), - stream.done(), - steam.done(), - global.done(), - direct.done(), - domesticCidr.done(), - streamCidr.done(), - telegramCidr.done(), - lanCidrs.done() - ]); - const output = generateAppProfile( - domestic.clash(), - microsoftApple.clash(), - stream.clash(), - steam.clash(), - global.clash(), - direct.clash(), - domesticCidr.clash(), - streamCidr.clash(), - telegramCidr.clash(), - lanCidrs.clash() + ...( + (await Promise.all([ + domestic.output(), + microsoftApple.output(), + stream.output(), + steam.output(), + global.output(), + direct.output(), + domesticCidr.output(), + streamCidr.output(), + telegramCidr.output(), + lanCidrs.output() + ])).map(output => nullthrow(output[0])) + ) as [ + string[], string[], string[], string[], string[], + string[], string[], string[], string[], string[] + ] ); await compareAndWriteFile( diff --git a/Build/lib/misc.ts b/Build/lib/misc.ts index e2b13ee3..2267281f 100644 --- a/Build/lib/misc.ts +++ b/Build/lib/misc.ts @@ -60,6 +60,17 @@ export function withBannerArray(title: string, description: string[] | readonly ]; }; +export function notSupported(name: string) { + return (...args: unknown[]) => { + console.error(`${name}: not supported.`, args); + throw new Error(`${name}: not implemented.`); + }; +} + +export function withIdentityContent(title: string, description: string[] | readonly string[], date: Date, content: string[]) { + return content; +}; + export function isDirectoryEmptySync(path: PathLike) { const directoryHandle = fs.opendirSync(path); diff --git a/Build/lib/rules/base.ts b/Build/lib/rules/base.ts index cdc97471..21ba3f1a 100644 --- a/Build/lib/rules/base.ts +++ b/Build/lib/rules/base.ts @@ -1,19 +1,22 @@ -import { OUTPUT_CLASH_DIR, OUTPUT_MODULES_DIR, OUTPUT_SINGBOX_DIR, OUTPUT_SURGE_DIR } from '../../constants/dir'; import type { Span } from '../../trace'; import { HostnameSmolTrie } from '../trie'; -import stringify from 'json-stringify-pretty-compact'; -import path from 'node:path'; -import { withBannerArray } from '../misc'; -import { invariant } from 'foxts/guard'; +import { invariant, not } from 'foxts/guard'; import picocolors from 'picocolors'; import fs from 'node:fs'; import { writeFile } from '../misc'; import { fastStringArrayJoin } from 'foxts/fast-string-array-join'; import { readFileByLine } from '../fetch-text-by-line'; import { asyncWriteToStream } from 'foxts/async-write-to-stream'; +import type { BaseWriteStrategy } from '../writing-strategy/base'; +import { merge } from 'fast-cidr-tools'; +import { createRetrieKeywordFilter as createKeywordFilter } from 'foxts/retrie'; +import path from 'node:path'; +import { SurgeMitmSgmodule } from '../writing-strategy/surge'; -export abstract class RuleOutput { - protected domainTrie = new HostnameSmolTrie(null); +export class FileOutput { + protected strategies: Array = []; + + public domainTrie = new HostnameSmolTrie(null); protected domainKeywords = new Set(); protected domainWildcard = new Set(); protected userAgent = new Set(); @@ -34,38 +37,14 @@ export abstract class RuleOutput { protected destPort = new Set(); protected otherRules: string[] = []; - protected abstract type: 'domainset' | 'non_ip' | 'ip' | (string & {}); private pendingPromise: Promise | null = null; - static readonly jsonToLines = (json: unknown): string[] => stringify(json).split('\n'); - whitelistDomain = (domain: string) => { this.domainTrie.whitelist(domain); return this; }; - static readonly domainWildCardToRegex = (domain: string) => { - let result = '^'; - for (let i = 0, len = domain.length; i < len; i++) { - switch (domain[i]) { - case '.': - result += String.raw`\.`; - break; - case '*': - result += String.raw`[\w.-]*?`; - break; - case '?': - result += String.raw`[\w.-]`; - break; - default: - result += domain[i]; - } - } - result += '$'; - return result; - }; - protected readonly span: Span; constructor($span: Span, protected readonly id: string) { @@ -78,6 +57,17 @@ export abstract class RuleOutput { return this; } + replaceStrategies(strategies: Array) { + this.strategies = strategies; + return this; + } + + withExtraStrategies(strategy: BaseWriteStrategy | false) { + if (strategy) { + this.strategies.push(strategy); + } + } + protected description: string[] | readonly string[] | null = null; withDescription(description: string[] | readonly string[]) { this.description = description; @@ -233,164 +223,246 @@ export abstract class RuleOutput { bulkAddCIDR4(cidrs: string[]) { for (let i = 0, len = cidrs.length; i < len; i++) { - this.ipcidr.add(RuleOutput.ipToCidr(cidrs[i], 4)); + this.ipcidr.add(FileOutput.ipToCidr(cidrs[i], 4)); } return this; } bulkAddCIDR4NoResolve(cidrs: string[]) { for (let i = 0, len = cidrs.length; i < len; i++) { - this.ipcidrNoResolve.add(RuleOutput.ipToCidr(cidrs[i], 4)); + this.ipcidrNoResolve.add(FileOutput.ipToCidr(cidrs[i], 4)); } return this; } bulkAddCIDR6(cidrs: string[]) { for (let i = 0, len = cidrs.length; i < len; i++) { - this.ipcidr6.add(RuleOutput.ipToCidr(cidrs[i], 6)); + this.ipcidr6.add(FileOutput.ipToCidr(cidrs[i], 6)); } return this; } bulkAddCIDR6NoResolve(cidrs: string[]) { for (let i = 0, len = cidrs.length; i < len; i++) { - this.ipcidr6NoResolve.add(RuleOutput.ipToCidr(cidrs[i], 6)); + this.ipcidr6NoResolve.add(FileOutput.ipToCidr(cidrs[i], 6)); } return this; } - protected abstract preprocess(): TPreprocessed extends null ? null : NonNullable; - async done() { await this.pendingPromise; this.pendingPromise = null; return this; } - private guardPendingPromise() { - // reverse invariant - if (this.pendingPromise !== null) { - console.trace('Pending promise:', this.pendingPromise); - throw new Error('You should call done() before calling this method'); + // private guardPendingPromise() { + // // reverse invariant + // if (this.pendingPromise !== null) { + // console.trace('Pending promise:', this.pendingPromise); + // throw new Error('You should call done() before calling this method'); + // } + // } + + // async writeClash(outputDir?: null | string) { + // await this.done(); + + // invariant(this.title, 'Missing title'); + // invariant(this.description, 'Missing description'); + + // return compareAndWriteFile( + // this.span, + // withBannerArray( + // this.title, + // this.description, + // this.date, + // this.clash() + // ), + // path.join(outputDir ?? OUTPUT_CLASH_DIR, this.type, this.id + '.txt') + // ); + // } + private strategiesWritten = false; + + private async writeToStrategies() { + if (this.strategiesWritten) { + throw new Error('Strategies already written'); } - } - private $$preprocessed: TPreprocessed | null = null; - protected runPreprocess() { - if (this.$$preprocessed === null) { - this.guardPendingPromise(); + this.strategiesWritten = true; - this.$$preprocessed = this.span.traceChildSync('preprocess', () => this.preprocess()); - } - } - - get $preprocessed(): TPreprocessed extends null ? null : NonNullable { - this.runPreprocess(); - return this.$$preprocessed as any; - } - - async writeClash(outputDir?: null | string) { await this.done(); - invariant(this.title, 'Missing title'); - invariant(this.description, 'Missing description'); + const kwfilter = createKeywordFilter(Array.from(this.domainKeywords)); - return compareAndWriteFile( - this.span, - withBannerArray( - this.title, - this.description, - this.date, - this.clash() - ), - path.join(outputDir ?? OUTPUT_CLASH_DIR, this.type, this.id + '.txt') - ); + if (this.strategies.filter(not(false)).length === 0) { + throw new Error('No strategies to write ' + this.id); + } + + this.domainTrie.dumpWithoutDot((domain, includeAllSubdomain) => { + if (kwfilter(domain)) { + return; + } + + for (let i = 0, len = this.strategies.length; i < len; i++) { + const strategy = this.strategies[i]; + if (strategy) { + if (includeAllSubdomain) { + strategy.writeDomainSuffix(domain); + } else { + strategy.writeDomain(domain); + } + } + } + }, true); + + for (let i = 0, len = this.strategies.length; i < len; i++) { + const strategy = this.strategies[i]; + if (!strategy) continue; + + if (this.domainKeywords.size) { + strategy.writeDomainKeywords(this.domainKeywords); + } + if (this.domainWildcard.size) { + strategy.writeDomainWildcards(this.domainWildcard); + } + if (this.userAgent.size) { + strategy.writeUserAgents(this.userAgent); + } + if (this.processName.size) { + strategy.writeProcessNames(this.processName); + } + if (this.processPath.size) { + strategy.writeProcessPaths(this.processPath); + } + } + + if (this.sourceIpOrCidr.size) { + const sourceIpOrCidr = Array.from(this.sourceIpOrCidr); + for (let i = 0, len = this.strategies.length; i < len; i++) { + const strategy = this.strategies[i]; + if (strategy) { + strategy.writeSourceIpCidrs(sourceIpOrCidr); + } + } + } + + for (let i = 0, len = this.strategies.length; i < len; i++) { + const strategy = this.strategies[i]; + if (strategy) { + if (this.sourcePort.size) { + strategy.writeSourcePorts(this.sourcePort); + } + if (this.destPort.size) { + strategy.writeDestinationPorts(this.destPort); + } + if (this.otherRules.length) { + strategy.writeOtherRules(this.otherRules); + } + if (this.urlRegex.size) { + strategy.writeUrlRegexes(this.urlRegex); + } + } + } + + let ipcidr: string[] | null = null; + let ipcidrNoResolve: string[] | null = null; + let ipcidr6: string[] | null = null; + let ipcidr6NoResolve: string[] | null = null; + + if (this.ipcidr.size) { + ipcidr = merge(Array.from(this.ipcidr)); + } + if (this.ipcidrNoResolve.size) { + ipcidrNoResolve = merge(Array.from(this.ipcidrNoResolve)); + } + if (this.ipcidr6.size) { + ipcidr6 = Array.from(this.ipcidr6); + } + if (this.ipcidr6NoResolve.size) { + ipcidr6NoResolve = Array.from(this.ipcidr6NoResolve); + } + + for (let i = 0, len = this.strategies.length; i < len; i++) { + const strategy = this.strategies[i]; + if (strategy) { + // no-resolve + if (ipcidrNoResolve?.length) { + strategy.writeIpCidrs(ipcidrNoResolve, true); + } + if (ipcidr6NoResolve?.length) { + strategy.writeIpCidr6s(ipcidr6NoResolve, true); + } + if (this.ipasnNoResolve.size) { + strategy.writeIpAsns(this.ipasnNoResolve, true); + } + if (this.groipNoResolve.size) { + strategy.writeGeoip(this.groipNoResolve, true); + } + + // triggers DNS resolution + if (ipcidr?.length) { + strategy.writeIpCidrs(ipcidr, false); + } + if (ipcidr6?.length) { + strategy.writeIpCidr6s(ipcidr6, false); + } + if (this.ipasn.size) { + strategy.writeIpAsns(this.ipasn, false); + } + if (this.geoip.size) { + strategy.writeGeoip(this.geoip, false); + } + } + } } - write({ - surge = true, - clash = true, - singbox = true, - surgeDir = OUTPUT_SURGE_DIR, - clashDir = OUTPUT_CLASH_DIR, - singboxDir = OUTPUT_SINGBOX_DIR - }: { - surge?: boolean, - clash?: boolean, - singbox?: boolean, - surgeDir?: string, - clashDir?: string, - singboxDir?: string - } = {}): Promise { - return this.done().then(() => this.span.traceChildAsync('write all', async () => { + write(): Promise { + return this.span.traceChildAsync('write all', async () => { + const promises: Array | void> = []; + + await this.writeToStrategies(); + invariant(this.title, 'Missing title'); invariant(this.description, 'Missing description'); - const promises: Array> = []; - - if (surge) { - promises.push(compareAndWriteFile( - this.span, - withBannerArray( + for (let i = 0, len = this.strategies.length; i < len; i++) { + const strategy = this.strategies[i]; + if (strategy) { + const basename = (strategy.overwriteFilename || this.id) + '.' + strategy.fileExtension; + promises.push(strategy.output( + this.span, this.title, this.description, this.date, - this.surge() - ), - path.join(surgeDir, this.type, this.id + '.conf') - )); - } - if (clash) { - promises.push(compareAndWriteFile( - this.span, - withBannerArray( - this.title, - this.description, - this.date, - this.clash() - ), - path.join(clashDir, this.type, this.id + '.txt') - )); - } - if (singbox) { - promises.push(compareAndWriteFile( - this.span, - this.singbox(), - path.join(singboxDir, this.type, this.id + '.json') - )); - } - - if (this.mitmSgmodule) { - const sgmodule = this.mitmSgmodule(); - const sgModulePath = this.mitmSgmodulePath ?? path.join(this.type, this.id + '.sgmodule'); - - if (sgmodule) { - promises.push( - compareAndWriteFile( - this.span, - sgmodule, - path.join(OUTPUT_MODULES_DIR, sgModulePath) - ) - ); + strategy.type + ? path.join(strategy.type, basename) + : basename + )); } } await Promise.all(promises); - })); + }); } - abstract surge(): string[]; - abstract clash(): string[]; - abstract singbox(): string[]; + async output(): Promise> { + await this.writeToStrategies(); - protected mitmSgmodulePath: string | null = null; - withMitmSgmodulePath(path: string | null) { - if (path) { - this.mitmSgmodulePath = path; + return this.strategies.reduce>((acc, strategy) => { + if (strategy) { + acc.push(strategy.content); + } else { + acc.push(null); + } + return acc; + }, []); + } + + withMitmSgmodulePath(moduleName: string | null) { + if (moduleName) { + this.withExtraStrategies(new SurgeMitmSgmodule(moduleName)); } return this; } - abstract mitmSgmodule?(): string[] | null; } export async function fileEqual(linesA: string[], source: AsyncIterable | Iterable): Promise { diff --git a/Build/lib/rules/domainset.ts b/Build/lib/rules/domainset.ts index 5643774f..396248b6 100644 --- a/Build/lib/rules/domainset.ts +++ b/Build/lib/rules/domainset.ts @@ -1,107 +1,15 @@ -import { createRetrieKeywordFilter as createKeywordFilter } from 'foxts/retrie'; -import { RuleOutput } from './base'; -import type { SingboxSourceFormat } from '../singbox'; +import type { BaseWriteStrategy } from '../writing-strategy/base'; +import { ClashDomainSet } from '../writing-strategy/clash'; +import { SingboxSource } from '../writing-strategy/singbox'; +import { SurgeDomainSet } from '../writing-strategy/surge'; +import { FileOutput } from './base'; -import { escapeStringRegexp } from 'foxts/escape-string-regexp'; - -export class DomainsetOutput extends RuleOutput { +export class DomainsetOutput extends FileOutput { protected type = 'domainset' as const; - private $surge: string[] = ['this_ruleset_is_made_by_sukkaw.ruleset.skk.moe']; - private $clash: string[] = ['this_ruleset_is_made_by_sukkaw.ruleset.skk.moe']; - private $singbox_domains: string[] = ['this_ruleset_is_made_by_sukkaw.ruleset.skk.moe']; - private $singbox_domains_suffixes: string[] = ['this_ruleset_is_made_by_sukkaw.ruleset.skk.moe']; - private $adguardhome: string[] = []; - preprocess() { - const kwfilter = createKeywordFilter(Array.from(this.domainKeywords)); - - this.domainTrie.dumpWithoutDot((domain, subdomain) => { - if (kwfilter(domain)) { - return; - } - - this.$surge.push(subdomain ? '.' + domain : domain); - this.$clash.push(subdomain ? `+.${domain}` : domain); - (subdomain ? this.$singbox_domains_suffixes : this.$singbox_domains).push(domain); - this.$adguardhome.push(subdomain ? `||${domain}^` : `|${domain}^`); - }, true); - - return this.$surge; - } - - surge(): string[] { - this.runPreprocess(); - return this.$surge; - } - - clash(): string[] { - this.runPreprocess(); - return this.$clash; - } - - singbox(): string[] { - this.runPreprocess(); - - return RuleOutput.jsonToLines({ - version: 2, - rules: [{ - domain: this.$singbox_domains, - domain_suffix: this.$singbox_domains_suffixes - }] - } satisfies SingboxSourceFormat); - } - - mitmSgmodule = undefined; - - adguardhome(): string[] { - this.runPreprocess(); - - // const whitelistArray = sortDomains(Array.from(whitelist)); - // for (let i = 0, len = whitelistArray.length; i < len; i++) { - // const domain = whitelistArray[i]; - // if (domain[0] === '.') { - // results.push(`@@||${domain.slice(1)}^`); - // } else { - // results.push(`@@|${domain}^`); - // } - // } - - for (const wildcard of this.domainWildcard) { - const processed = wildcard.replaceAll('?', '*'); - if (processed.startsWith('*.')) { - this.$adguardhome.push(`||${processed.slice(2)}^`); - } else { - this.$adguardhome.push(`|${processed}^`); - } - } - - for (const keyword of this.domainKeywords) { - // Use regex to match keyword - this.$adguardhome.push(`/${escapeStringRegexp(keyword)}/`); - } - - for (const ipGroup of [this.ipcidr, this.ipcidrNoResolve]) { - for (const ipcidr of ipGroup) { - if (ipcidr.endsWith('/32')) { - this.$adguardhome.push(`||${ipcidr.slice(0, -3)}`); - /* else if (ipcidr.endsWith('.0/24')) { - results.push(`||${ipcidr.slice(0, -6)}.*`); - } */ - } else { - this.$adguardhome.push(`||${ipcidr}^`); - } - } - } - for (const ipGroup of [this.ipcidr6, this.ipcidr6NoResolve]) { - for (const ipcidr of ipGroup) { - if (ipcidr.endsWith('/128')) { - this.$adguardhome.push(`||${ipcidr.slice(0, -4)}`); - } else { - this.$adguardhome.push(`||${ipcidr}`); - } - } - } - - return this.$adguardhome; - } + strategies: Array = [ + new SurgeDomainSet(), + new ClashDomainSet(), + new SingboxSource(this.type) + ]; } diff --git a/Build/lib/rules/ip.ts b/Build/lib/rules/ip.ts index 7577ec80..09274bb6 100644 --- a/Build/lib/rules/ip.ts +++ b/Build/lib/rules/ip.ts @@ -1,75 +1,21 @@ import type { Span } from '../../trace'; -import { appendArrayInPlace } from '../append-array-in-place'; -import { appendSetElementsToArray } from 'foxts/append-set-elements-to-array'; -import type { SingboxSourceFormat } from '../singbox'; -import { RuleOutput } from './base'; +import type { BaseWriteStrategy } from '../writing-strategy/base'; +import { ClashClassicRuleSet, ClashIPSet } from '../writing-strategy/clash'; +import { SingboxSource } from '../writing-strategy/singbox'; +import { SurgeRuleSet } from '../writing-strategy/surge'; +import { FileOutput } from './base'; -import { merge } from 'fast-cidr-tools'; - -type Preprocessed = string[]; - -export class IPListOutput extends RuleOutput { +export class IPListOutput extends FileOutput { protected type = 'ip' as const; + strategies: Array; constructor(span: Span, id: string, private readonly clashUseRule = true) { super(span, id); - } - mitmSgmodule = undefined; - - protected preprocess() { - const results: string[] = []; - appendArrayInPlace( - results, - merge( - appendSetElementsToArray(Array.from(this.ipcidrNoResolve), this.ipcidr), - true - ) - ); - appendSetElementsToArray(results, this.ipcidr6NoResolve); - appendSetElementsToArray(results, this.ipcidr6); - - return results; - } - - private $surge: string[] | null = null; - - surge(): string[] { - if (!this.$surge) { - const results: string[] = ['DOMAIN,this_ruleset_is_made_by_sukkaw.ruleset.skk.moe']; - - appendArrayInPlace( - results, - merge(Array.from(this.ipcidrNoResolve)).map(i => `IP-CIDR,${i},no-resolve`, true) - ); - appendSetElementsToArray(results, this.ipcidr6NoResolve, i => `IP-CIDR6,${i},no-resolve`); - appendArrayInPlace( - results, - merge(Array.from(this.ipcidr)).map(i => `IP-CIDR,${i}`, true) - ); - appendSetElementsToArray(results, this.ipcidr6, i => `IP-CIDR6,${i}`); - - this.$surge = results; - } - return this.$surge; - } - - clash(): string[] { - if (this.clashUseRule) { - return this.surge(); - } - - return this.$preprocessed; - } - - singbox(): string[] { - const singbox: SingboxSourceFormat = { - version: 2, - rules: [{ - domain: ['this_ruleset_is_made_by_sukkaw.ruleset.skk.moe'], - ip_cidr: this.$preprocessed - }] - }; - return RuleOutput.jsonToLines(singbox); + this.strategies = [ + new SurgeRuleSet(this.type), + this.clashUseRule ? new ClashClassicRuleSet(this.type) : new ClashIPSet(), + new SingboxSource(this.type) + ]; } } diff --git a/Build/lib/rules/ruleset.ts b/Build/lib/rules/ruleset.ts index 9e8a5776..d00e4f90 100644 --- a/Build/lib/rules/ruleset.ts +++ b/Build/lib/rules/ruleset.ts @@ -1,283 +1,17 @@ -import { merge } from 'fast-cidr-tools'; import type { Span } from '../../trace'; -import { createRetrieKeywordFilter as createKeywordFilter } from 'foxts/retrie'; -import { appendArrayInPlace } from '../append-array-in-place'; -import { appendSetElementsToArray } from 'foxts/append-set-elements-to-array'; -import type { SingboxSourceFormat } from '../singbox'; -import { RuleOutput } from './base'; -import picocolors from 'picocolors'; -import { normalizeDomain } from '../normalize-domain'; -import { isProbablyIpv4 } from 'foxts/is-probably-ip'; -import { fastIpVersion } from '../misc'; +import { ClashClassicRuleSet } from '../writing-strategy/clash'; +import { SingboxSource } from '../writing-strategy/singbox'; +import { SurgeRuleSet } from '../writing-strategy/surge'; +import { FileOutput } from './base'; -type Preprocessed = [domain: string[], domainSuffix: string[], sortedDomainRules: string[]]; - -export class RulesetOutput extends RuleOutput { +export class RulesetOutput extends FileOutput { constructor(span: Span, id: string, protected type: 'non_ip' | 'ip' | (string & {})) { super(span, id); - } - protected preprocess() { - const kwfilter = createKeywordFilter(Array.from(this.domainKeywords)); - - const domains: string[] = []; - const domainSuffixes: string[] = []; - const sortedDomainRules: string[] = []; - - this.domainTrie.dumpWithoutDot((domain, includeAllSubdomain) => { - if (kwfilter(domain)) { - return; - } - if (includeAllSubdomain) { - domainSuffixes.push(domain); - sortedDomainRules.push(`DOMAIN-SUFFIX,${domain}`); - } else { - domains.push(domain); - sortedDomainRules.push(`DOMAIN,${domain}`); - } - }, true); - - return [domains, domainSuffixes, sortedDomainRules] satisfies Preprocessed; - } - - surge(): string[] { - const results: string[] = ['DOMAIN,this_ruleset_is_made_by_sukkaw.ruleset.skk.moe']; - appendArrayInPlace(results, this.$preprocessed[2]); - - appendSetElementsToArray(results, this.domainKeywords, i => `DOMAIN-KEYWORD,${i}`); - appendSetElementsToArray(results, this.domainWildcard, i => `DOMAIN-WILDCARD,${i}`); - - appendSetElementsToArray(results, this.userAgent, i => `USER-AGENT,${i}`); - - appendSetElementsToArray(results, this.processName, i => `PROCESS-NAME,${i}`); - appendSetElementsToArray(results, this.processPath, i => `PROCESS-NAME,${i}`); - - appendSetElementsToArray(results, this.sourceIpOrCidr, i => `SRC-IP,${i}`); - appendSetElementsToArray(results, this.sourcePort, i => `SRC-PORT,${i}`); - appendSetElementsToArray(results, this.destPort, i => `DEST-PORT,${i}`); - - appendArrayInPlace(results, this.otherRules); - - appendSetElementsToArray(results, this.urlRegex, i => `URL-REGEX,${i}`); - - appendArrayInPlace( - results, - merge(Array.from(this.ipcidrNoResolve), true).map(i => `IP-CIDR,${i},no-resolve`) - ); - appendSetElementsToArray(results, this.ipcidr6NoResolve, i => `IP-CIDR6,${i},no-resolve`); - appendSetElementsToArray(results, this.ipasnNoResolve, i => `IP-ASN,${i},no-resolve`); - appendSetElementsToArray(results, this.groipNoResolve, i => `GEOIP,${i},no-resolve`); - - appendArrayInPlace( - results, - merge(Array.from(this.ipcidr), true).map(i => `IP-CIDR,${i}`) - ); - appendSetElementsToArray(results, this.ipcidr6, i => `IP-CIDR6,${i}`); - appendSetElementsToArray(results, this.ipasn, i => `IP-ASN,${i}`); - appendSetElementsToArray(results, this.geoip, i => `GEOIP,${i}`); - - return results; - } - - clash(): string[] { - const results: string[] = ['DOMAIN,this_ruleset_is_made_by_sukkaw.ruleset.skk.moe']; - - appendArrayInPlace(results, this.$preprocessed[2]); - - appendSetElementsToArray(results, this.domainKeywords, i => `DOMAIN-KEYWORD,${i}`); - appendSetElementsToArray(results, this.domainWildcard, i => `DOMAIN-REGEX,${RuleOutput.domainWildCardToRegex(i)}`); - - appendSetElementsToArray(results, this.processName, i => `PROCESS-NAME,${i}`); - appendSetElementsToArray(results, this.processPath, i => `PROCESS-PATH,${i}`); - - appendSetElementsToArray(results, this.sourceIpOrCidr, value => { - if (value.includes('/')) { - return `SRC-IP-CIDR,${value}`; - } - const v = fastIpVersion(value); - if (v === 4) { - return `SRC-IP-CIDR,${value}/32`; - } - if (v === 6) { - return `SRC-IP-CIDR6,${value}/128`; - } - return ''; - }); - appendSetElementsToArray(results, this.sourcePort, i => `SRC-PORT,${i}`); - appendSetElementsToArray(results, this.destPort, i => `DST-PORT,${i}`); - - // appendArrayInPlace(results, this.otherRules); - - appendArrayInPlace( - results, - merge(Array.from(this.ipcidrNoResolve), true).map(i => `IP-CIDR,${i},no-resolve`) - ); - appendSetElementsToArray(results, this.ipcidr6NoResolve, i => `IP-CIDR6,${i},no-resolve`); - appendSetElementsToArray(results, this.ipasnNoResolve, i => `IP-ASN,${i},no-resolve`); - appendSetElementsToArray(results, this.groipNoResolve, i => `GEOIP,${i},no-resolve`); - - appendArrayInPlace( - results, - merge(Array.from(this.ipcidr), true).map(i => `IP-CIDR,${i}`) - ); - appendSetElementsToArray(results, this.ipcidr6, i => `IP-CIDR6,${i}`); - appendSetElementsToArray(results, this.ipasn, i => `IP-ASN,${i}`); - appendSetElementsToArray(results, this.geoip, i => `GEOIP,${i}`); - - return results; - } - - singbox(): string[] { - const ip_cidr: string[] = []; - appendArrayInPlace( - ip_cidr, - merge( - appendSetElementsToArray(Array.from(this.ipcidrNoResolve), this.ipcidr), - true - ) - ); - appendSetElementsToArray(ip_cidr, this.ipcidr6NoResolve); - appendSetElementsToArray(ip_cidr, this.ipcidr6); - - const singbox: SingboxSourceFormat = { - version: 2, - rules: [{ - domain: appendArrayInPlace(['this_ruleset_is_made_by_sukkaw.ruleset.skk.moe'], this.$preprocessed[0]), - domain_suffix: this.$preprocessed[1], - domain_keyword: Array.from(this.domainKeywords), - domain_regex: Array.from(this.domainWildcard, RuleOutput.domainWildCardToRegex), - ip_cidr, - source_ip_cidr: [...this.sourceIpOrCidr].reduce((acc, cur) => { - if (cur.includes('/')) { - acc.push(cur); - } else { - const v = fastIpVersion(cur); - - if (v === 4) { - acc.push(cur + '/32'); - } else if (v === 6) { - acc.push(cur + '/128'); - } - } - - return acc; - }, []), - source_port: [...this.sourcePort].reduce((acc, cur) => { - const tmp = Number(cur); - if (!Number.isNaN(tmp)) { - acc.push(tmp); - } - return acc; - }, []), - port: [...this.destPort].reduce((acc, cur) => { - const tmp = Number(cur); - if (!Number.isNaN(tmp)) { - acc.push(tmp); - } - return acc; - }, []), - process_name: Array.from(this.processName), - process_path: Array.from(this.processPath) - }] - }; - - return RuleOutput.jsonToLines(singbox); - } - - mitmSgmodule(): string[] | null { - if (this.urlRegex.size === 0 || this.mitmSgmodulePath === null) { - return null; - } - - const urlRegexResults: Array<{ origin: string, processed: string[] }> = []; - - const parsedFailures: Array<[original: string, processed: string]> = []; - const parsed: Array<[original: string, domain: string]> = []; - - for (let urlRegex of this.urlRegex) { - if ( - urlRegex.startsWith('http://') - || urlRegex.startsWith('^http://') - ) { - continue; - } - if (urlRegex.startsWith('^https?://')) { - urlRegex = urlRegex.slice(10); - } - if (urlRegex.startsWith('^https://')) { - urlRegex = urlRegex.slice(9); - } - - const potentialHostname = urlRegex.split('/')[0] - // pre process regex - .replaceAll(String.raw`\.`, '.') - .replaceAll('.+', '*') - .replaceAll(/([a-z])\?/g, '($1|)') - // convert regex to surge hostlist syntax - .replaceAll('([a-z])', '?') - .replaceAll(String.raw`\d`, '?') - .replaceAll(/\*+/g, '*'); - - let processed: string[] = [potentialHostname]; - - const matches = [...potentialHostname.matchAll(/\((?:([^()|]+)\|)+([^()|]*)\)/g)]; - - if (matches.length > 0) { - const replaceVariant = (combinations: string[], fullMatch: string, options: string[]): string[] => { - const newCombinations: string[] = []; - - combinations.forEach(combination => { - options.forEach(option => { - newCombinations.push(combination.replace(fullMatch, option)); - }); - }); - - return newCombinations; - }; - - for (let i = 0; i < matches.length; i++) { - const match = matches[i]; - const [_, ...options] = match; - - processed = replaceVariant(processed, _, options); - } - } - - urlRegexResults.push({ - origin: potentialHostname, - processed - }); - } - - for (const i of urlRegexResults) { - for (const processed of i.processed) { - if ( - normalizeDomain( - processed - .replaceAll('*', 'a') - .replaceAll('?', 'b') - ) - ) { - parsed.push([i.origin, processed]); - } else if (!isProbablyIpv4(processed)) { - parsedFailures.push([i.origin, processed]); - } - } - } - - if (parsedFailures.length > 0) { - console.error(picocolors.bold('Parsed Failed')); - console.table(parsedFailures); - } - - const hostnames = Array.from(new Set(parsed.map(i => i[1]))); - - return [ - '#!name=[Sukka] Surge Reject MITM', - `#!desc=为 URL Regex 规则组启用 MITM (size: ${hostnames.length})`, - '', - '[MITM]', - 'hostname = %APPEND% ' + hostnames.join(', ') + this.strategies = [ + new SurgeRuleSet(this.type), + new ClashClassicRuleSet(this.type), + new SingboxSource(this.type) ]; } } diff --git a/Build/lib/singbox.ts b/Build/lib/singbox.ts deleted file mode 100644 index 4507173e..00000000 --- a/Build/lib/singbox.ts +++ /dev/null @@ -1,19 +0,0 @@ -interface SingboxHeadlessRule { - domain?: string[], - domain_suffix?: string[], - domain_keyword?: string[], - domain_regex?: string[], - source_ip_cidr?: string[], - ip_cidr?: string[], - source_port?: number[], - source_port_range?: string[], - port?: number[], - port_range?: string[], - process_name?: string[], - process_path?: string[] -} - -export interface SingboxSourceFormat { - version: 2 | number & {}, - rules: SingboxHeadlessRule[] -} diff --git a/Build/lib/writing-strategy/adguardhome.ts b/Build/lib/writing-strategy/adguardhome.ts new file mode 100644 index 00000000..a35a7a28 --- /dev/null +++ b/Build/lib/writing-strategy/adguardhome.ts @@ -0,0 +1,107 @@ +import { escapeStringRegexp } from 'foxts/escape-string-regexp'; +import { BaseWriteStrategy } from './base'; +import { noop } from 'foxts/noop'; +import { notSupported } from '../misc'; + +export class AdGuardHome extends BaseWriteStrategy { + // readonly type = 'domainset'; + readonly fileExtension = 'txt'; + readonly type = ''; + + protected result: string[] = []; + + // eslint-disable-next-line @typescript-eslint/class-methods-use-this -- abstract method + withPadding(title: string, description: string[] | readonly string[], date: Date, content: string[]): string[] { + return [ + `! Title: ${title}`, + '! Last modified: ' + date.toUTCString(), + '! Expires: 6 hours', + '! License: https://github.com/SukkaW/Surge/blob/master/LICENSE', + '! Homepage: https://github.com/SukkaW/Surge', + `! Description: ${description.join(' ')}`, + '!', + ...content, + '! EOF' + ]; + } + + writeDomain(domain: string): void { + this.result.push(`|${domain}^`); + } + + // const whitelistArray = sortDomains(Array.from(whitelist)); + // for (let i = 0, len = whitelistArray.length; i < len; i++) { + // const domain = whitelistArray[i]; + // if (domain[0] === '.') { + // results.push(`@@||${domain.slice(1)}^`); + // } else { + // results.push(`@@|${domain}^`); + // } + // } + + writeDomainSuffix(domain: string): void { + this.result.push(`||${domain}^`); + } + + writeDomainKeywords(keywords: Set): void { + for (const keyword of keywords) { + // Use regex to match keyword + this.result.push(`/${escapeStringRegexp(keyword)}/`); + } + } + + writeDomainWildcards(wildcards: Set): void { + for (const wildcard of wildcards) { + const processed = wildcard.replaceAll('?', '*'); + if (processed.startsWith('*.')) { + this.result.push(`||${processed.slice(2)}^`); + } else { + this.result.push(`|${processed}^`); + } + } + } + + writeUserAgents = noop; + writeProcessNames = noop; + writeProcessPaths = noop; + writeUrlRegexes = noop; + writeIpCidrs(ipGroup: string[], noResolve: boolean): void { + if (noResolve) { + // When IP is provided to AdGuardHome, any domain resolve to those IP will be blocked + // So we can't do noResolve + return; + } + for (const ipcidr of ipGroup) { + if (ipcidr.endsWith('/32')) { + this.result.push(`||${ipcidr.slice(0, -3)}`); + /* else if (ipcidr.endsWith('.0/24')) { + results.push(`||${ipcidr.slice(0, -6)}.*`); + } */ + } else { + this.result.push(`||${ipcidr}^`); + } + } + } + + writeIpCidr6s(ipGroup: string[], noResolve: boolean): void { + if (noResolve) { + // When IP is provided to AdGuardHome, any domain resolve to those IP will be blocked + // So we can't do noResolve + return; + } + for (const ipcidr of ipGroup) { + if (ipcidr.endsWith('/128')) { + this.result.push(`||${ipcidr.slice(0, -4)}`); + } else { + this.result.push(`||${ipcidr}`); + } + } + }; + + writeGeoip = notSupported('writeGeoip'); + writeIpAsns = notSupported('writeIpAsns'); + writeSourceIpCidrs = notSupported('writeSourceIpCidrs'); + writeSourcePorts = notSupported('writeSourcePorts'); + writeDestinationPorts = noop; + writeOtherRules = noop; +} diff --git a/Build/lib/writing-strategy/base.ts b/Build/lib/writing-strategy/base.ts new file mode 100644 index 00000000..a644cdb1 --- /dev/null +++ b/Build/lib/writing-strategy/base.ts @@ -0,0 +1,81 @@ +import path from 'node:path'; +import type { Span } from '../../trace'; +import { compareAndWriteFile } from '../create-file'; + +export abstract class BaseWriteStrategy { + // abstract readonly type: 'domainset' | 'non_ip' | 'ip' | (string & {}); + public overwriteFilename: string | null = null; + public abstract readonly type: 'domainset' | 'non_ip' | 'ip' | (string & {}); + + abstract readonly fileExtension: 'conf' | 'txt' | 'json' | (string & {}); + + constructor(protected outputDir: string) {} + + protected abstract result: string[] | null; + + abstract writeDomain(domain: string): void; + abstract writeDomainSuffix(domain: string): void; + abstract writeDomainKeywords(keyword: Set): void; + abstract writeDomainWildcards(wildcard: Set): void; + abstract writeUserAgents(userAgent: Set): void; + abstract writeProcessNames(processName: Set): void; + abstract writeProcessPaths(processPath: Set): void; + abstract writeUrlRegexes(urlRegex: Set): void; + abstract writeIpCidrs(ipCidr: string[], noResolve: boolean): void; + abstract writeIpCidr6s(ipCidr6: string[], noResolve: boolean): void; + abstract writeGeoip(geoip: Set, noResolve: boolean): void; + abstract writeIpAsns(asns: Set, noResolve: boolean): void; + abstract writeSourceIpCidrs(sourceIpCidr: string[]): void; + abstract writeSourcePorts(port: Set): void; + abstract writeDestinationPorts(port: Set): void; + abstract writeOtherRules(rule: string[]): void; + + static readonly domainWildCardToRegex = (domain: string) => { + let result = '^'; + for (let i = 0, len = domain.length; i < len; i++) { + switch (domain[i]) { + case '.': + result += String.raw`\.`; + break; + case '*': + result += String.raw`[\w.-]*?`; + break; + case '?': + result += String.raw`[\w.-]`; + break; + default: + result += domain[i]; + } + } + result += '$'; + return result; + }; + + abstract withPadding(title: string, description: string[] | readonly string[], date: Date, content: string[]): string[]; + + output( + span: Span, + title: string, + description: string[] | readonly string[], + date: Date, + relativePath: string + ): void | Promise { + if (!this.result) { + return; + } + return compareAndWriteFile( + span, + this.withPadding( + title, + description, + date, + this.result + ), + path.join(this.outputDir, relativePath) + ); + }; + + get content() { + return this.result; + } +} diff --git a/Build/lib/writing-strategy/clash.ts b/Build/lib/writing-strategy/clash.ts new file mode 100644 index 00000000..30428929 --- /dev/null +++ b/Build/lib/writing-strategy/clash.ts @@ -0,0 +1,169 @@ +import { appendSetElementsToArray } from 'foxts/append-set-elements-to-array'; +import { BaseWriteStrategy } from './base'; +import { noop } from 'foxts/noop'; +import { fastIpVersion, notSupported, withBannerArray } from '../misc'; +import { OUTPUT_CLASH_DIR } from '../../constants/dir'; +import { appendArrayInPlace } from '../append-array-in-place'; + +export class ClashDomainSet extends BaseWriteStrategy { + // readonly type = 'domainset'; + readonly fileExtension = 'txt'; + readonly type = 'domainset'; + + protected result: string[] = ['this_ruleset_is_made_by_sukkaw.ruleset.skk.moe']; + + constructor(protected outputDir = OUTPUT_CLASH_DIR) { + super(outputDir); + } + + withPadding = withBannerArray; + + writeDomain(domain: string): void { + this.result.push(domain); + } + + writeDomainSuffix(domain: string): void { + this.result.push('+.' + domain); + } + + writeDomainKeywords = noop; + writeDomainWildcards = noop; + writeUserAgents = noop; + writeProcessNames = noop; + writeProcessPaths = noop; + writeUrlRegexes = noop; + writeIpCidrs = noop; + writeIpCidr6s = noop; + writeGeoip = noop; + writeIpAsns = noop; + writeSourceIpCidrs = noop; + writeSourcePorts = noop; + writeDestinationPorts = noop; + writeOtherRules = noop; +} + +export class ClashIPSet extends BaseWriteStrategy { + // readonly type = 'domainset'; + readonly fileExtension = 'txt'; + readonly type = 'ip'; + + protected result: string[] = []; + + constructor(protected outputDir = OUTPUT_CLASH_DIR) { + super(outputDir); + } + + withPadding = withBannerArray; + + writeDomain = notSupported('writeDomain'); + writeDomainSuffix = notSupported('writeDomainSuffix'); + writeDomainKeywords = notSupported('writeDomainKeywords'); + writeDomainWildcards = notSupported('writeDomainWildcards'); + writeUserAgents = notSupported('writeUserAgents'); + writeProcessNames = notSupported('writeProcessNames'); + writeProcessPaths = notSupported('writeProcessPaths'); + writeUrlRegexes = notSupported('writeUrlRegexes'); + writeIpCidrs(ipCidr: string[]): void { + appendArrayInPlace(this.result, ipCidr); + } + + writeIpCidr6s(ipCidr6: string[]): void { + appendArrayInPlace(this.result, ipCidr6); + } + + writeGeoip = notSupported('writeGeoip'); + writeIpAsns = notSupported('writeIpAsns'); + writeSourceIpCidrs = notSupported('writeSourceIpCidrs'); + writeSourcePorts = notSupported('writeSourcePorts'); + writeDestinationPorts = noop; + writeOtherRules = noop; +} + +export class ClashClassicRuleSet extends BaseWriteStrategy { + readonly fileExtension = 'txt'; + + protected result: string[] = ['DOMAIN,this_ruleset_is_made_by_sukkaw.ruleset.skk.moe']; + + constructor(public readonly type: string, protected outputDir = OUTPUT_CLASH_DIR) { + super(outputDir); + } + + withPadding = withBannerArray; + + writeDomain(domain: string): void { + this.result.push('DOMAIN,' + domain); + } + + writeDomainSuffix(domain: string): void { + this.result.push('DOMAIN-SUFFIX,' + domain); + } + + writeDomainKeywords(keyword: Set): void { + appendSetElementsToArray(this.result, keyword, i => `DOMAIN-KEYWORD,${i}`); + } + + writeDomainWildcards(wildcard: Set): void { + appendSetElementsToArray(this.result, wildcard, i => `DOMAIN-REGEX,${ClashClassicRuleSet.domainWildCardToRegex(i)}`); + } + + writeUserAgents = noop; + + writeProcessNames(processName: Set): void { + appendSetElementsToArray(this.result, processName, i => `PROCESS-NAME,${i}`); + } + + writeProcessPaths(processPath: Set): void { + appendSetElementsToArray(this.result, processPath, i => `PROCESS-PATH,${i}`); + } + + writeUrlRegexes = noop; + + writeIpCidrs(ipCidr: string[], noResolve: boolean): void { + for (let i = 0, len = ipCidr.length; i < len; i++) { + this.result.push(`IP-CIDR,${ipCidr[i]}${noResolve ? ',no-resolve' : ''}`); + } + } + + writeIpCidr6s(ipCidr6: string[], noResolve: boolean): void { + for (let i = 0, len = ipCidr6.length; i < len; i++) { + this.result.push(`IP-CIDR6,${ipCidr6[i]}${noResolve ? ',no-resolve' : ''}`); + } + } + + writeGeoip(geoip: Set, noResolve: boolean): void { + appendSetElementsToArray(this.result, geoip, i => `GEOIP,${i}${noResolve ? ',no-resolve' : ''}`); + } + + writeIpAsns(asns: Set, noResolve: boolean): void { + appendSetElementsToArray(this.result, asns, i => `IP-ASN,${i}${noResolve ? ',no-resolve' : ''}`); + } + + writeSourceIpCidrs(sourceIpCidr: string[]): void { + for (let i = 0, len = sourceIpCidr.length; i < len; i++) { + const value = sourceIpCidr[i]; + if (value.includes('/')) { + this.result.push(`SRC-IP-CIDR,${value}`); + continue; + } + const v = fastIpVersion(value); + if (v === 4) { + this.result.push(`SRC-IP-CIDR,${value}/32`); + continue; + } + if (v === 6) { + this.result.push(`SRC-IP-CIDR6,${value}/128`); + continue; + } + } + } + + writeSourcePorts(port: Set): void { + appendSetElementsToArray(this.result, port, i => `SRC-PORT,${i}`); + } + + writeDestinationPorts(port: Set): void { + appendSetElementsToArray(this.result, port, i => `DST-PORT,${i}`); + } + + writeOtherRules = noop; +} diff --git a/Build/lib/writing-strategy/singbox.ts b/Build/lib/writing-strategy/singbox.ts new file mode 100644 index 00000000..73874b2a --- /dev/null +++ b/Build/lib/writing-strategy/singbox.ts @@ -0,0 +1,152 @@ +import { BaseWriteStrategy } from './base'; +import { appendArrayInPlace } from '../append-array-in-place'; +import { noop } from 'foxts/noop'; +import { fastIpVersion, withIdentityContent } from '../misc'; +import stringify from 'json-stringify-pretty-compact'; +import { OUTPUT_SINGBOX_DIR } from '../../constants/dir'; + +interface SingboxHeadlessRule { + domain: string[], // this_ruleset_is_made_by_sukkaw.ruleset.skk.moe + domain_suffix: string[], // this_ruleset_is_made_by_sukkaw.ruleset.skk.moe + domain_keyword?: string[], + domain_regex?: string[], + source_ip_cidr?: string[], + ip_cidr?: string[], + source_port?: number[], + source_port_range?: string[], + port?: number[], + port_range?: string[], + process_name?: string[], + process_path?: string[] +} + +export interface SingboxSourceFormat { + version: 2 | number & {}, + rules: SingboxHeadlessRule[] +} + +export class SingboxSource extends BaseWriteStrategy { + readonly fileExtension = 'json'; + + static readonly jsonToLines = (json: unknown): string[] => stringify(json).split('\n'); + + private singbox: SingboxHeadlessRule = { + domain: ['this_ruleset_is_made_by_sukkaw.ruleset.skk.moe'], + domain_suffix: ['this_ruleset_is_made_by_sukkaw.ruleset.skk.moe'] + }; + + protected get result() { + return SingboxSource.jsonToLines({ + version: 2, + rules: [this.singbox] + }); + } + + constructor(public type: string, protected outputDir = OUTPUT_SINGBOX_DIR) { + super(outputDir); + } + + withPadding = withIdentityContent; + + writeDomain(domain: string): void { + this.singbox.domain.push(domain); + } + + writeDomainSuffix(domain: string): void { + (this.singbox.domain_suffix ??= []).push(domain); + } + + writeDomainKeywords(keyword: Set): void { + appendArrayInPlace( + this.singbox.domain_keyword ??= [], + Array.from(keyword) + ); + } + + writeDomainWildcards(wildcard: Set): void { + appendArrayInPlace( + this.singbox.domain_regex ??= [], + Array.from(wildcard, SingboxSource.domainWildCardToRegex) + ); + } + + writeUserAgents = noop; + + writeProcessNames(processName: Set): void { + appendArrayInPlace( + this.singbox.process_name ??= [], + Array.from(processName) + ); + } + + writeProcessPaths(processPath: Set): void { + appendArrayInPlace( + this.singbox.process_path ??= [], + Array.from(processPath) + ); + } + + writeUrlRegexes = noop; + + writeIpCidrs(ipCidr: string[]): void { + appendArrayInPlace( + this.singbox.ip_cidr ??= [], + ipCidr + ); + } + + writeIpCidr6s(ipCidr6: string[]): void { + appendArrayInPlace( + this.singbox.ip_cidr ??= [], + ipCidr6 + ); + } + + writeGeoip = noop; + + writeIpAsns = noop; + + writeSourceIpCidrs(sourceIpCidr: string[]): void { + this.singbox.source_ip_cidr ??= []; + for (let i = 0, len = sourceIpCidr.length; i < len; i++) { + const value = sourceIpCidr[i]; + if (value.includes('/')) { + this.singbox.source_ip_cidr.push(value); + continue; + } + const v = fastIpVersion(value); + if (v === 4) { + this.singbox.source_ip_cidr.push(`${value}/32`); + continue; + } + if (v === 6) { + this.singbox.source_ip_cidr.push(`${value}/128`); + continue; + } + } + } + + writeSourcePorts(port: Set): void { + this.singbox.source_port ??= []; + + for (const i of port) { + const tmp = Number(i); + if (!Number.isNaN(tmp)) { + this.singbox.source_port.push(tmp); + } + } + } + + writeDestinationPorts(port: Set): void { + this.singbox.port ??= []; + + for (const i of port) { + const tmp = Number(i); + if (!Number.isNaN(tmp)) { + this.singbox.port.push(tmp); + } + } + } + + writeOtherRules = noop; +} diff --git a/Build/lib/writing-strategy/surge.ts b/Build/lib/writing-strategy/surge.ts new file mode 100644 index 00000000..f31b8aa9 --- /dev/null +++ b/Build/lib/writing-strategy/surge.ts @@ -0,0 +1,262 @@ +import { appendSetElementsToArray } from 'foxts/append-set-elements-to-array'; +import { BaseWriteStrategy } from './base'; +import { appendArrayInPlace } from '../append-array-in-place'; +import { noop } from 'foxts/noop'; +import { isProbablyIpv4 } from 'foxts/is-probably-ip'; +import picocolors from 'picocolors'; +import { normalizeDomain } from '../normalize-domain'; +import { OUTPUT_MODULES_DIR, OUTPUT_SURGE_DIR } from '../../constants/dir'; +import { withBannerArray, withIdentityContent } from '../misc'; + +export class SurgeDomainSet extends BaseWriteStrategy { + // readonly type = 'domainset'; + readonly fileExtension = 'conf'; + type = 'domainset'; + + protected result: string[] = ['this_ruleset_is_made_by_sukkaw.ruleset.skk.moe']; + + constructor(outputDir = OUTPUT_SURGE_DIR) { + super(outputDir); + } + + withPadding = withBannerArray; + + writeDomain(domain: string): void { + this.result.push(domain); + } + + writeDomainSuffix(domain: string): void { + this.result.push('.' + domain); + } + + writeDomainKeywords = noop; + writeDomainWildcards = noop; + writeUserAgents = noop; + writeProcessNames = noop; + writeProcessPaths = noop; + writeUrlRegexes = noop; + writeIpCidrs = noop; + writeIpCidr6s = noop; + writeGeoip = noop; + writeIpAsns = noop; + writeSourceIpCidrs = noop; + writeSourcePorts = noop; + writeDestinationPorts = noop; + writeOtherRules = noop; +} + +export class SurgeRuleSet extends BaseWriteStrategy { + readonly fileExtension = 'conf'; + + protected result: string[] = ['DOMAIN,this_ruleset_is_made_by_sukkaw.ruleset.skk.moe']; + + constructor(public readonly type: string, outputDir = OUTPUT_SURGE_DIR) { + super(outputDir); + } + + withPadding = withBannerArray; + + writeDomain(domain: string): void { + this.result.push('DOMAIN,' + domain); + } + + writeDomainSuffix(domain: string): void { + this.result.push('DOMAIN-SUFFIX,' + domain); + } + + writeDomainKeywords(keyword: Set): void { + appendSetElementsToArray(this.result, keyword, i => `DOMAIN-KEYWORD,${i}`); + } + + writeDomainWildcards(wildcard: Set): void { + appendSetElementsToArray(this.result, wildcard, i => `DOMAIN-WILDCARD,${i}`); + } + + writeUserAgents(userAgent: Set): void { + appendSetElementsToArray(this.result, userAgent, i => `USER-AGENT,${i}`); + } + + writeProcessNames(processName: Set): void { + appendSetElementsToArray(this.result, processName, i => `PROCESS-NAME,${i}`); + } + + writeProcessPaths(processPath: Set): void { + appendSetElementsToArray(this.result, processPath, i => `PROCESS-NAME,${i}`); + } + + writeUrlRegexes(urlRegex: Set): void { + appendSetElementsToArray(this.result, urlRegex, i => `URL-REGEX,${i}`); + } + + writeIpCidrs(ipCidr: string[], noResolve: boolean): void { + for (let i = 0, len = ipCidr.length; i < len; i++) { + this.result.push(`IP-CIDR,${ipCidr[i]}${noResolve ? ',no-resolve' : ''}`); + } + } + + writeIpCidr6s(ipCidr6: string[], noResolve: boolean): void { + for (let i = 0, len = ipCidr6.length; i < len; i++) { + this.result.push(`IP-CIDR6,${ipCidr6[i]}${noResolve ? ',no-resolve' : ''}`); + } + } + + writeGeoip(geoip: Set, noResolve: boolean): void { + appendSetElementsToArray(this.result, geoip, i => `GEOIP,${i}${noResolve ? ',no-resolve' : ''}`); + } + + writeIpAsns(asns: Set, noResolve: boolean): void { + appendSetElementsToArray(this.result, asns, i => `IP-ASN,${i}${noResolve ? ',no-resolve' : ''}`); + } + + writeSourceIpCidrs(sourceIpCidr: string[]): void { + for (let i = 0, len = sourceIpCidr.length; i < len; i++) { + this.result.push(`SRC-IP,${sourceIpCidr[i]}`); + } + } + + writeSourcePorts(port: Set): void { + appendSetElementsToArray(this.result, port, i => `SRC-PORT,${i}`); + } + + writeDestinationPorts(port: Set): void { + appendSetElementsToArray(this.result, port, i => `DEST-PORT,${i}`); + } + + writeOtherRules(rule: string[]): void { + appendArrayInPlace(this.result, rule); + } +} + +export class SurgeMitmSgmodule extends BaseWriteStrategy { + // readonly type = 'domainset'; + readonly fileExtension = 'sgmodule'; + type = ''; + + private rules = new Set(); + + protected get result() { + if (this.rules.size === 0) { + return null; + } + + return [ + '#!name=[Sukka] Surge Reject MITM', + `#!desc=为 URL Regex 规则组启用 MITM (size: ${this.rules.size})`, + '', + '[MITM]', + 'hostname = %APPEND% ' + Array.from(this.rules).join(', ') + ]; + } + + withPadding = withIdentityContent; + + constructor(moduleName: string, outputDir = OUTPUT_MODULES_DIR) { + super(outputDir); + this.overwriteFilename = moduleName; + } + + writeDomain = noop; + + writeDomainSuffix = noop; + + writeDomainKeywords = noop; + writeDomainWildcards = noop; + writeUserAgents = noop; + writeProcessNames = noop; + writeProcessPaths = noop; + writeUrlRegexes(urlRegexes: Set): void { + const urlRegexResults: Array<{ origin: string, processed: string[] }> = []; + + const parsedFailures: Array<[original: string, processed: string]> = []; + const parsed: Array<[original: string, domain: string]> = []; + + for (let urlRegex of urlRegexes) { + if ( + urlRegex.startsWith('http://') + || urlRegex.startsWith('^http://') + ) { + continue; + } + if (urlRegex.startsWith('^https?://')) { + urlRegex = urlRegex.slice(10); + } + if (urlRegex.startsWith('^https://')) { + urlRegex = urlRegex.slice(9); + } + + const potentialHostname = urlRegex.split('/')[0] + // pre process regex + .replaceAll(String.raw`\.`, '.') + .replaceAll('.+', '*') + .replaceAll(/([a-z])\?/g, '($1|)') + // convert regex to surge hostlist syntax + .replaceAll('([a-z])', '?') + .replaceAll(String.raw`\d`, '?') + .replaceAll(/\*+/g, '*'); + + let processed: string[] = [potentialHostname]; + + const matches = [...potentialHostname.matchAll(/\((?:([^()|]+)\|)+([^()|]*)\)/g)]; + + if (matches.length > 0) { + const replaceVariant = (combinations: string[], fullMatch: string, options: string[]): string[] => { + const newCombinations: string[] = []; + + combinations.forEach(combination => { + options.forEach(option => { + newCombinations.push(combination.replace(fullMatch, option)); + }); + }); + + return newCombinations; + }; + + for (let i = 0; i < matches.length; i++) { + const match = matches[i]; + const [_, ...options] = match; + + processed = replaceVariant(processed, _, options); + } + } + + urlRegexResults.push({ + origin: potentialHostname, + processed + }); + } + + for (const i of urlRegexResults) { + for (const processed of i.processed) { + if ( + normalizeDomain( + processed + .replaceAll('*', 'a') + .replaceAll('?', 'b') + ) + ) { + parsed.push([i.origin, processed]); + } else if (!isProbablyIpv4(processed)) { + parsedFailures.push([i.origin, processed]); + } + } + } + + if (parsedFailures.length > 0) { + console.error(picocolors.bold('Parsed Failed')); + console.table(parsedFailures); + } + + for (let i = 0, len = parsed.length; i < len; i++) { + this.rules.add(parsed[i][1]); + } + } + + writeIpCidrs = noop; + writeIpCidr6s = noop; + writeGeoip = noop; + writeIpAsns = noop; + writeSourceIpCidrs = noop; + writeSourcePorts = noop; + writeDestinationPorts = noop; + writeOtherRules = noop; +} diff --git a/Source/non_ip/reject-url-regex.conf b/Source/non_ip/reject-url-regex.conf index faeba973..1644906a 100644 --- a/Source/non_ip/reject-url-regex.conf +++ b/Source/non_ip/reject-url-regex.conf @@ -1,7 +1,7 @@ # $ meta_title Sukka's Ruleset - Reject URL # $ meta_description The ruleset supports AD blocking, tracking protection, privacy protection, anti-phishing, anti-mining # $ meta_description Need Surge Module: https://ruleset.skk.moe/Modules/sukka_mitm_hostnames.sgmodule -# $ sgmodule_mitm_hostnames sukka_mitm_hostnames.sgmodule +# $ sgmodule_mitm_hostnames sukka_mitm_hostnames # URL-REGEX,^https?://.+\.youtube\.com/api/stats/.+adformat # URL-REGEX,^https?://.+\.youtube\.com/api/stats/ads