From 6f7c743d89bd085aa6b401c93976a5b26418216d Mon Sep 17 00:00:00 2001 From: SukkaW Date: Sat, 1 Feb 2025 15:04:28 +0800 Subject: [PATCH] Minor changes --- ...c-direct-lan-ruleset-dns-mapping-module.ts | 19 ++-- Build/build-speedtest-domainset.ts | 16 +-- Build/build-stream-service.ts | 16 +-- Build/lib/misc.ts | 2 + Build/lib/rules/base.ts | 99 ++++++++++++------- Build/lib/writing-strategy/base.ts | 19 +++- Build/lib/writing-strategy/surge.ts | 2 +- 7 files changed, 106 insertions(+), 67 deletions(-) diff --git a/Build/build-domestic-direct-lan-ruleset-dns-mapping-module.ts b/Build/build-domestic-direct-lan-ruleset-dns-mapping-module.ts index 6035c104..8119e43f 100644 --- a/Build/build-domestic-direct-lan-ruleset-dns-mapping-module.ts +++ b/Build/build-domestic-direct-lan-ruleset-dns-mapping-module.ts @@ -193,16 +193,19 @@ export const buildDomesticRuleset = task(require.main === module, __filename)(as }>((acc, cur) => { const { domains, dns, ...rest } = cur[1]; domains.forEach((domain) => { - let domainWildcard = domain; - if (domain[0] === '$') { - domainWildcard = domain.slice(1); - } else if (domain[0] === '+') { - domainWildcard = `*.${domain.slice(1)}`; - } else { - domainWildcard = `+.${domain}`; + switch (domain[0]) { + case '$': + domain = domain.slice(1); + break; + case '+': + domain = `*.${domain.slice(1)}`; + break; + default: + domain = `+.${domain}`; + break; } - acc.dns['nameserver-policy'][domainWildcard] = dns === 'system' + acc.dns['nameserver-policy'][domain] = dns === 'system' ? ['system://', 'system', 'dhcp://system'] : dns; }); diff --git a/Build/build-speedtest-domainset.ts b/Build/build-speedtest-domainset.ts index 475866be..1dd0b532 100644 --- a/Build/build-speedtest-domainset.ts +++ b/Build/build-speedtest-domainset.ts @@ -57,22 +57,12 @@ export const buildSpeedtestDomainSet = task(require.main === module, __filename) '', 'This file contains common speedtest endpoints.' ]) - .addFromDomainset(await readFileIntoProcessedArray(path.resolve(SOURCE_DIR, 'domainset/speedtest.conf'))) - .addFromDomainset( - (await readFileIntoProcessedArray(path.resolve(OUTPUT_SURGE_DIR, 'domainset/speedtest.conf'))) - .reduce((acc, cur) => { - const hn = tldts.getHostname(cur, { detectIp: false, validateHostname: true }); - if (hn) { - acc.push(hn); - } - return acc; - }, []) - ); + .addFromDomainset(readFileIntoProcessedArray(path.resolve(SOURCE_DIR, 'domainset/speedtest.conf'))) + .addFromDomainset(readFileIntoProcessedArray(path.resolve(OUTPUT_SURGE_DIR, 'domainset/speedtest.conf'))); const hostnameGroup = await span.traceChildPromise('get speedtest hosts groups', getSpeedtestHostsGroupsPromise); - hostnameGroup.forEach(hostname => output.bulkAddDomain(hostname)); - await output.done(); + hostnameGroup.forEach(output.bulkAddDomain.bind(output)); return output.write(); }); diff --git a/Build/build-stream-service.ts b/Build/build-stream-service.ts index f88256be..a008ca1d 100644 --- a/Build/build-stream-service.ts +++ b/Build/build-stream-service.ts @@ -6,12 +6,14 @@ import { ALL, NORTH_AMERICA, EU, HK, TW, JP, KR } from '../Source/stream'; import { SHARED_DESCRIPTION } from './constants/description'; import { RulesetOutput } from './lib/create-file'; -export function createRulesetForStreamService(span: Span, +function createRulesetForStreamService( + span: Span, fileId: string, title: string, - streamServices: Array) { - return span.traceChildAsync(fileId, async (childSpan) => Promise.all([ + streamServices: Array +) { + return [ // Domains - new RulesetOutput(childSpan, fileId, 'non_ip') + new RulesetOutput(span, fileId, 'non_ip') .withTitle(`Sukka's Ruleset - Stream Services: ${title}`) .withDescription([ ...SHARED_DESCRIPTION, @@ -21,7 +23,7 @@ export function createRulesetForStreamService(span: Span, .addFromRuleset(streamServices.flatMap((i) => i.rules)) .write(), // IP - new RulesetOutput(childSpan, fileId, 'ip') + new RulesetOutput(span, fileId, 'ip') .withTitle(`Sukka's Ruleset - Stream Services IPs: ${title}`) .withDescription([ ...SHARED_DESCRIPTION, @@ -31,7 +33,7 @@ export function createRulesetForStreamService(span: Span, .bulkAddCIDR4NoResolve(streamServices.flatMap(i => i.ip?.v4 ?? [])) .bulkAddCIDR6NoResolve(streamServices.flatMap(i => i.ip?.v6 ?? [])) .write() - ])); + ]; } export const buildStreamService = task(require.main === module, __filename)(async (span) => Promise.all([ @@ -44,4 +46,4 @@ export const buildStreamService = task(require.main === module, __filename)(asyn // createRulesetForStreamService('stream_au', 'Oceania', AU), createRulesetForStreamService(span, 'stream_kr', 'Korean', KR) // createRulesetForStreamService('stream_south_east_asia', 'South East Asia', SOUTH_EAST_ASIA) -])); +].flat())); diff --git a/Build/lib/misc.ts b/Build/lib/misc.ts index 2267281f..83e7150f 100644 --- a/Build/lib/misc.ts +++ b/Build/lib/misc.ts @@ -3,6 +3,8 @@ import fs from 'node:fs'; import type { PathLike } from 'node:fs'; import fsp from 'node:fs/promises'; +export type MaybePromise = T | Promise; + export function fastStringCompare(a: string, b: string) { const lenA = a.length; const lenB = b.length; diff --git a/Build/lib/rules/base.ts b/Build/lib/rules/base.ts index 6b371609..07e0b0fe 100644 --- a/Build/lib/rules/base.ts +++ b/Build/lib/rules/base.ts @@ -4,6 +4,7 @@ import { invariant, not } from 'foxts/guard'; import picocolors from 'picocolors'; import fs from 'node:fs'; import { writeFile } from '../misc'; +import type { MaybePromise } from '../misc'; import { fastStringArrayJoin } from 'foxts/fast-string-array-join'; import { readFileByLine } from '../fetch-text-by-line'; import { asyncWriteToStream } from 'foxts/async-write-to-stream'; @@ -13,6 +14,10 @@ import { createRetrieKeywordFilter as createKeywordFilter } from 'foxts/retrie'; import path from 'node:path'; import { SurgeMitmSgmodule } from '../writing-strategy/surge'; +/** + * Holds the universal rule data (domain, ip, url-regex, etc. etc.) + * This class is not about format, instead it will call the class that does + */ export class FileOutput { protected strategies: Array = []; @@ -135,12 +140,24 @@ export class FileOutput { } } - addFromDomainset(source: AsyncIterable | Iterable | string[]) { - this.pendingPromise = (this.pendingPromise ||= Promise.resolve()).then(() => this.addFromDomainsetPromise(source)); + addFromDomainset(source: MaybePromise | Iterable | string[]>) { + if (this.pendingPromise) { + if ('then' in source) { + this.pendingPromise = this.pendingPromise.then(() => source).then(src => this.addFromDomainsetPromise(src)); + return this; + } + this.pendingPromise = this.pendingPromise.then(() => this.addFromDomainsetPromise(source)); + return this; + } + if ('then' in source) { + this.pendingPromise = source.then(src => this.addFromDomainsetPromise(src)); + return this; + } + this.pendingPromise = this.addFromDomainsetPromise(source); return this; } - private async addFromRulesetPromise(source: AsyncIterable | Iterable) { + private async addFromRulesetPromise(source: AsyncIterable | Iterable | string[]) { for await (const line of source) { const splitted = line.split(','); const type = splitted[0]; @@ -203,13 +220,20 @@ export class FileOutput { } } - addFromRuleset(source: AsyncIterable | Iterable | Promise>) { + addFromRuleset(source: MaybePromise | Iterable>) { if (this.pendingPromise) { - this.pendingPromise = this.pendingPromise.then(() => source); - } else { - this.pendingPromise = Promise.resolve(source); + if ('then' in source) { + this.pendingPromise = this.pendingPromise.then(() => source).then(src => this.addFromRulesetPromise(src)); + return this; + } + this.pendingPromise = this.pendingPromise.then(() => this.addFromRulesetPromise(source)); + return this; } - this.pendingPromise = this.pendingPromise.then((source) => this.addFromRulesetPromise(source)); + if ('then' in source) { + this.pendingPromise = source.then(src => this.addFromRulesetPromise(src)); + return this; + } + this.pendingPromise = this.addFromRulesetPromise(source); return this; } @@ -282,15 +306,16 @@ export class FileOutput { // } private strategiesWritten = false; - private async writeToStrategies() { + private writeToStrategies() { + if (this.pendingPromise) { + throw new Error('You should call done() before calling writeToStrategies()'); + } if (this.strategiesWritten) { throw new Error('Strategies already written'); } this.strategiesWritten = true; - await this.done(); - const kwfilter = createKeywordFilter(Array.from(this.domainKeywords)); if (this.strategies.filter(not(false)).length === 0) { @@ -415,40 +440,44 @@ export class FileOutput { } } - write(): Promise { + write(): Promise { return this.span.traceChildAsync('write all', async (childSpan) => { - const promises: Array | void> = []; + await this.done(); + childSpan.traceChildSync('write to strategies', this.writeToStrategies.bind(this)); - await childSpan.traceChildAsync('write to strategies', this.writeToStrategies.bind(this)); + return childSpan.traceChildAsync('output to disk', (childSpan) => { + const promises: Array | void> = []; - invariant(this.title, 'Missing title'); - invariant(this.description, 'Missing description'); + invariant(this.title, 'Missing title'); + invariant(this.description, 'Missing description'); - for (let i = 0, len = this.strategies.length; i < len; i++) { - const strategy = this.strategies[i]; - if (strategy) { - const basename = (strategy.overwriteFilename || this.id) + '.' + strategy.fileExtension; - promises.push(strategy.output( - childSpan, - this.title, - this.description, - this.date, - path.join( - strategy.outputDir, - strategy.type - ? path.join(strategy.type, basename) - : basename - ) - )); + for (let i = 0, len = this.strategies.length; i < len; i++) { + const strategy = this.strategies[i]; + if (strategy) { + const basename = (strategy.overwriteFilename || this.id) + '.' + strategy.fileExtension; + promises.push(strategy.output( + childSpan, + this.title, + this.description, + this.date, + path.join( + strategy.outputDir, + strategy.type + ? path.join(strategy.type, basename) + : basename + ) + )); + } } - } - await Promise.all(promises); + return Promise.all(promises); + }); }); } async compile(): Promise> { - await this.writeToStrategies(); + await this.done(); + this.writeToStrategies(); return this.strategies.reduce>((acc, strategy) => { if (strategy) { diff --git a/Build/lib/writing-strategy/base.ts b/Build/lib/writing-strategy/base.ts index 49776dae..781bd172 100644 --- a/Build/lib/writing-strategy/base.ts +++ b/Build/lib/writing-strategy/base.ts @@ -1,11 +1,24 @@ import type { Span } from '../../trace'; import { compareAndWriteFile } from '../create-file'; +/** + * The class is not about holding rule data, instead it determines how the + * date is written to a file. + */ export abstract class BaseWriteStrategy { + /** + * Sometimes a ruleset will create extra files (e.g. reject-url-regex w/ mitm.sgmodule), + * and doesn't share the same filename and id. This property is used to overwrite the filename. + */ public overwriteFilename: string | null = null; + public withFilename(filename: string) { + this.overwriteFilename = filename; + return this; + } + public abstract readonly type: 'domainset' | 'non_ip' | 'ip' | (string & {}); - abstract readonly fileExtension: 'conf' | 'txt' | 'json' | (string & {}); + abstract readonly fileExtension: 'conf' | 'txt' | 'json' | 'sgmodule' /* | (string & {}) */; constructor(public readonly outputDir: string) {} @@ -28,6 +41,8 @@ export abstract class BaseWriteStrategy { abstract writeDestinationPorts(port: Set): void; abstract writeOtherRules(rule: string[]): void; + protected abstract withPadding(title: string, description: string[] | readonly string[], date: Date, content: string[]): string[]; + static readonly domainWildCardToRegex = (domain: string) => { let result = '^'; for (let i = 0, len = domain.length; i < len; i++) { @@ -49,8 +64,6 @@ export abstract class BaseWriteStrategy { return result; }; - protected abstract withPadding(title: string, description: string[] | readonly string[], date: Date, content: string[]): string[]; - public output( span: Span, title: string, diff --git a/Build/lib/writing-strategy/surge.ts b/Build/lib/writing-strategy/surge.ts index 3854cd62..4a1f1624 100644 --- a/Build/lib/writing-strategy/surge.ts +++ b/Build/lib/writing-strategy/surge.ts @@ -156,7 +156,7 @@ export class SurgeMitmSgmodule extends BaseWriteStrategy { constructor(moduleName: string, outputDir = OUTPUT_MODULES_DIR) { super(outputDir); - this.overwriteFilename = moduleName; + this.withFilename(moduleName); } writeDomain = noop;