Refactor: adjust more output

Build/build-domestic-direct-lan-ruleset-dns-mapping-module.ts

@@ -32,7 +32,7 @@ export const getDomesticAndDirectDomainsRulesetPromise = createMemoizedPromise(a
 });
 
 export const buildDomesticRuleset = task(require.main === module, __filename)(async (span) => {
-  const res = await getDomesticAndDirectDomainsRulesetPromise();
+  const [domestics, directs, lans] = await getDomesticAndDirectDomainsRulesetPromise();
 
   const dataset = Object.entries(DOMESTICS);
   appendArrayInPlace(dataset, Object.entries(DIRECTS));
@@ -46,7 +46,7 @@ export const buildDomesticRuleset = task(require.main === module, __filename)(as
         '',
         'This file contains known addresses that are avaliable in the Mainland China.'
       ])
-      .addFromRuleset(res[0])
+      .addFromRuleset(domestics)
       .write(),
     new RulesetOutput(span, 'direct', 'non_ip')
       .withTitle('Sukka\'s Ruleset - Direct Rules')
@@ -55,7 +55,7 @@ export const buildDomesticRuleset = task(require.main === module, __filename)(as
         '',
         'This file contains domains and process that should not be proxied.'
       ])
-      .addFromRuleset(res[1])
+      .addFromRuleset(directs)
       .write(),
     new RulesetOutput(span, 'lan', 'non_ip')
       .withTitle('Sukka\'s Ruleset - LAN')
@@ -64,7 +64,7 @@ export const buildDomesticRuleset = task(require.main === module, __filename)(as
         '',
         'This file includes rules for LAN DOMAIN and reserved TLDs.'
       ])
-      .addFromRuleset(res[2])
+      .addFromRuleset(lans)
       .write(),
     compareAndWriteFile(
       span,

Build/build-mitm-hostname.ts

@@ -5,6 +5,7 @@ import { fdir as Fdir } from 'fdir';
 import { green, yellow } from 'picocolors';
 import { processLineFromReadline } from './lib/process-line';
 import { getHostname } from 'tldts';
+import { OUTPUT_SURGE_DIR } from './constants/dir';
 
 const PRESET_MITM_HOSTNAMES = [
   // '*baidu.com',
@@ -36,7 +37,7 @@ const PRESET_MITM_HOSTNAMES = [
 ];
 
 (async () => {
-  const folderListPath = pathFn.resolve(__dirname, '../List/');
+  const folderListPath = pathFn.resolve(OUTPUT_SURGE_DIR, 'List/');
 
   const rulesets = await new Fdir()
     .withFullPaths()

Build/build-sspanel-appprofile.ts

@@ -1,6 +1,5 @@
 import { getAppleCdnDomainsPromise } from './build-apple-cdn';
 import { getDomesticAndDirectDomainsRulesetPromise } from './build-domestic-direct-lan-ruleset-dns-mapping-module';
-import { surgeRulesetToClashClassicalTextRuleset, surgeDomainsetToClashRuleset } from './lib/clash';
 import { readFileIntoProcessedArray } from './lib/fetch-text-by-line';
 import { task } from './trace';
 import path from 'node:path';
@@ -8,11 +7,11 @@ import path from 'node:path';
 import { ALL as AllStreamServices } from '../Source/stream';
 import { getChnCidrPromise } from './build-chn-cidr';
 import { getTelegramCIDRPromise } from './build-telegram-cidr';
-import { compareAndWriteFile } from './lib/create-file';
+import { compareAndWriteFile, RulesetOutput } from './lib/create-file';
 import { getMicrosoftCdnRulesetPromise } from './build-microsoft-cdn';
 import { isTruthy } from './lib/misc';
 import { appendArrayInPlace } from './lib/append-array-in-place';
-import { OUTPUT_INTERNAL_DIR, SOURCE_DIR } from './constants/dir';
+import { OUTPUT_INTERNAL_DIR, OUTPUT_SURGE_DIR, SOURCE_DIR } from './constants/dir';
 
 const POLICY_GROUPS: Array<[name: string, insertProxy: boolean, insertDirect: boolean]> = [
   ['Default Proxy', true, false],
@@ -24,99 +23,123 @@ const POLICY_GROUPS: Array<[name: string, insertProxy: boolean, insertDirect: bo
   ['Final Match', true, true]
 ];
 
-const removeNoResolved = (line: string) => line.replace(',no-resolve', '');
-
 /**
  * This only generates a simplified version, for under-used users only.
  */
 export const buildSSPanelUIMAppProfile = task(require.main === module, __filename)(async (span) => {
   const [
-    [domesticDomains, directDomains, lanDomains],
+    [domesticRules, directRules, lanRules],
     appleCdnDomains,
-    microsoftCdnDomains,
+    microsoftCdnRules,
-    appleCnDomains,
+    appleCnRules,
-    neteaseMusicDomains,
+    neteaseMusicRules,
-    microsoftDomains,
+    microsoftRules,
-    appleDomains,
+    appleRules,
-    streamDomains,
+    streamRules,
-    steamDomains,
+    steamDomainset,
-    globalDomains,
+    globalRules,
-    telegramDomains,
+    telegramRules,
-    domesticCidrs,
+    [domesticCidrs4, domesticCidrs6],
-    streamCidrs,
+    [streamCidrs4, streamCidrs6],
-    { results: rawTelegramCidrs },
+    { ipcidr: telegramCidrs4, ipcidr6: telegramCidrs6 },
-    lanCidrs
+    rawLanCidrs
   ] = await Promise.all([
     // domestic - domains
-    getDomesticAndDirectDomainsRulesetPromise()
+    getDomesticAndDirectDomainsRulesetPromise(),
-      .then(
+    getAppleCdnDomainsPromise(),
-        data => (
+    getMicrosoftCdnRulesetPromise(),
-          data.map(surgeRulesetToClashClassicalTextRuleset)
+    readFileIntoProcessedArray(path.join(OUTPUT_SURGE_DIR, 'non_ip/apple_cn.conf')),
-        ) as [string[], string[], string[]]
+    readFileIntoProcessedArray(path.join(OUTPUT_SURGE_DIR, 'non_ip/neteasemusic.conf')),
-      ),
-    getAppleCdnDomainsPromise().then(domains => domains.map(domain => `DOMAIN-SUFFIX,${domain}`)),
-    getMicrosoftCdnRulesetPromise().then(surgeRulesetToClashClassicalTextRuleset),
-    readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/apple_cn.conf')),
-    readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/neteasemusic.conf')).then(surgeRulesetToClashClassicalTextRuleset),
     // microsoft & apple - domains
-    readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/microsoft.conf')),
+    readFileIntoProcessedArray(path.join(OUTPUT_SURGE_DIR, 'non_ip/microsoft.conf')),
-    readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/apple_services.conf')).then(surgeRulesetToClashClassicalTextRuleset),
+    readFileIntoProcessedArray(path.join(OUTPUT_SURGE_DIR, 'non_ip/apple_services.conf')),
     // stream - domains
-    surgeRulesetToClashClassicalTextRuleset(AllStreamServices.flatMap((i) => i.rules)),
+    AllStreamServices.flatMap((i) => i.rules),
     // steam - domains
-    readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/steam.conf')).then(surgeDomainsetToClashRuleset),
+    readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/steam.conf')),
     // global - domains
-    readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/global.conf')).then(surgeRulesetToClashClassicalTextRuleset),
+    readFileIntoProcessedArray(path.join(OUTPUT_SURGE_DIR, 'non_ip/global.conf')),
-    readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/telegram.conf')).then(surgeRulesetToClashClassicalTextRuleset),
+    readFileIntoProcessedArray(path.join(OUTPUT_SURGE_DIR, 'non_ip/telegram.conf')),
     // domestic - ip cidr
-    getChnCidrPromise().then(([cidrs4, cidrs6]) => [
+    getChnCidrPromise(),
-      ...cidrs4.map(cidr => `IP-CIDR,${cidr}`),
+    AllStreamServices.reduce<[cidr4: string[], cidr6: string[]]>((acc, i) => {
-      ...cidrs6.map(cidr => `IP-CIDR,${cidr}`)
+      if (i.ip) {
-    ]),
+        appendArrayInPlace(acc[0], i.ip.v4);
-    AllStreamServices.flatMap((i) => (
+        appendArrayInPlace(acc[1], i.ip.v6);
-      i.ip
+      }
-        ? [
+
-          ...i.ip.v4.map((ip) => `IP-CIDR,${ip}`),
+      return acc;
-          ...i.ip.v6.map((ip) => `IP-CIDR6,${ip}`)
+    }, [[], []]),
-        ]
-        : []
-    )),
     // global - ip cidr
     getTelegramCIDRPromise(),
     // lan - ip cidr
-    readFileIntoProcessedArray(path.join(SOURCE_DIR, 'ip/lan.conf'))
+    readFileIntoProcessedArray(path.join(OUTPUT_SURGE_DIR, 'ip/lan.conf'))
   ] as const);
 
-  const telegramCidrs = rawTelegramCidrs.map(removeNoResolved);
+  const domestic = new RulesetOutput(span, '_', 'non_ip')
+    .addFromRuleset(domesticRules)
+    .bulkAddDomainSuffix(appleCdnDomains)
+    .addFromRuleset(microsoftCdnRules)
+    .addFromRuleset(appleCnRules)
+    .addFromRuleset(neteaseMusicRules);
+
+  const microsoftApple = new RulesetOutput(span, '_', 'non_ip')
+    .addFromRuleset(microsoftRules)
+    .addFromRuleset(appleRules);
+
+  const stream = new RulesetOutput(span, '_', 'non_ip')
+    .addFromRuleset(streamRules);
+
+  const steam = new RulesetOutput(span, '_', 'non_ip')
+    .addFromDomainset(steamDomainset);
+
+  const global = new RulesetOutput(span, '_', 'non_ip')
+    .addFromRuleset(globalRules)
+    .addFromRuleset(telegramRules);
+
+  const direct = new RulesetOutput(span, '_', 'non_ip')
+    .addFromRuleset(directRules)
+    .addFromRuleset(lanRules);
+
+  const domesticCidr = new RulesetOutput(span, '_', 'ip')
+    .bulkAddCIDR4(domesticCidrs4)
+    .bulkAddCIDR6(domesticCidrs6);
+
+  const streamCidr = new RulesetOutput(span, '_', 'ip')
+    .bulkAddCIDR4(streamCidrs4)
+    .bulkAddCIDR6(streamCidrs6);
+
+  const telegramCidr = new RulesetOutput(span, '_', 'ip')
+    .bulkAddCIDR4(telegramCidrs4)
+    .bulkAddCIDR6(telegramCidrs6);
+
+  const lanCidrs = new RulesetOutput(span, '_', 'ip')
+    .addFromRuleset(rawLanCidrs);
+
+  await Promise.all([
+    domestic.done(),
+    microsoftApple.done(),
+    stream.done(),
+    steam.done(),
+    global.done(),
+    direct.done(),
+    domesticCidr.done(),
+    streamCidr.done(),
+    telegramCidr.done(),
+    lanCidrs.done()
+  ]);
 
   const output = generateAppProfile(
-    [
+    domestic.clash(),
-      ...domesticDomains,
+    microsoftApple.clash(),
-      ...appleCdnDomains,
+    stream.clash(),
-      ...microsoftCdnDomains,
+    steam.clash(),
-      ...appleCnDomains,
+    global.clash(),
-      ...neteaseMusicDomains
+    direct.clash(),
-    ],
+    domesticCidr.clash(),
-    [
+    streamCidr.clash(),
-      ...microsoftDomains,
+    telegramCidr.clash(),
-      ...appleDomains
+    lanCidrs.clash()
-    ],
-    streamDomains,
-    steamDomains,
-    [
-      ...globalDomains,
-      ...telegramDomains
-    ],
-    [
-      ...directDomains,
-      ...lanDomains
-    ],
-    domesticCidrs,
-    streamCidrs,
-    [
-      ...telegramCidrs
-    ],
-    lanCidrs
   );
 
   await compareAndWriteFile(

Build/build-telegram-cidr.ts

@@ -13,7 +13,8 @@ export const getTelegramCIDRPromise = createMemoizedPromise(async () => {
   const lastModified = resp.headers.get('last-modified');
   const date = lastModified ? new Date(lastModified) : new Date();
 
-  const results: string[] = [];
+  const ipcidr: string[] = [];
+  const ipcidr6: string[] = [];
 
   for await (const line of createReadlineInterfaceFromResponse(resp)) {
     const cidr = processLine(line);
@@ -21,20 +22,20 @@ export const getTelegramCIDRPromise = createMemoizedPromise(async () => {
 
     const [subnet] = cidr.split('/');
     if (isProbablyIpv4(subnet)) {
-      results.push(`IP-CIDR,${cidr},no-resolve`);
+      ipcidr.push(cidr);
     }
     if (isProbablyIpv6(subnet)) {
-      results.push(`IP-CIDR6,${cidr},no-resolve`);
+      ipcidr6.push(cidr);
     }
   }
 
-  return { date, results };
+  return { date, ipcidr, ipcidr6 };
 });
 
 export const buildTelegramCIDR = task(require.main === module, __filename)(async (span) => {
-  const { date, results } = await span.traceChildAsync('get telegram cidr', getTelegramCIDRPromise);
+  const { date, ipcidr, ipcidr6 } = await span.traceChildAsync('get telegram cidr', getTelegramCIDRPromise);
 
-  if (results.length === 0) {
+  if (ipcidr.length + ipcidr6.length === 0) {
     throw new Error('Failed to fetch data!');
   }
 
@@ -48,6 +49,7 @@ export const buildTelegramCIDR = task(require.main === module, __filename)(async
     .withTitle('Sukka\'s Ruleset - Telegram IP CIDR')
     .withDescription(description)
     .withDate(date)
-    .addFromRuleset(results)
+    .bulkAddCIDR4NoResolve(ipcidr)
+    .bulkAddCIDR6NoResolve(ipcidr6)
     .write();
 });

Build/lib/clash.ts

@@ -1,11 +1,10 @@
-import picocolors from 'picocolors';
 import { domainWildCardToRegex, identity } from './misc';
 import { isProbablyIpv4, isProbablyIpv6 } from './is-fast-ip';
 
 const unsupported = Symbol('unsupported');
 
 // https://dreamacro.github.io/clash/configuration/rules.html
-const PROCESSOR: Record<string, ((raw: string, type: string, value: string) => string) | typeof unsupported> = {
+export const PROCESSOR: Record<string, ((raw: string, type: string, value: string) => string) | typeof unsupported> = {
   DOMAIN: identity,
   'DOMAIN-SUFFIX': identity,
   'DOMAIN-KEYWORD': identity,
@@ -35,35 +34,3 @@ const PROCESSOR: Record<string, ((raw: string, type: string, value: string) => s
   'URL-REGEX': unsupported,
   'USER-AGENT': unsupported
 };
-
-export const surgeRulesetToClashClassicalTextRuleset = (rules: string[] | Set<string>) => {
-  return Array.from(rules).reduce<string[]>((acc, cur) => {
-    let buf = '';
-    let type = '';
-    let i = 0;
-    for (const len = cur.length; i < len; i++) {
-      if (cur[i] === ',') {
-        type = buf;
-        break;
-      }
-      buf += cur[i];
-    }
-    if (type === '') {
-      return acc;
-    }
-    const value = cur.slice(i + 1);
-    if (type in PROCESSOR) {
-      const proc = PROCESSOR[type];
-      if (proc !== unsupported) {
-        acc.push(proc(cur, type, value));
-      }
-    } else {
-      console.log(picocolors.yellow(`[clash] unknown rule type: ${type}`), cur);
-    }
-    return acc;
-  }, []);
-};
-
-export const surgeDomainsetToClashRuleset = (domainset: string[]) => {
-  return domainset.map(i => (i[0] === '.' ? `DOMAIN-SUFFIX,${i.slice(1)}` : `DOMAIN,${i}`));
-};

Build/lib/rules/base.ts

@@ -218,8 +218,12 @@ export abstract class RuleOutput {
   abstract clash(): string[];
   abstract singbox(): string[];
 
+  done() {
+    return this.pendingPromise;
+  }
+
   async write(): Promise<void> {
-    await this.pendingPromise;
+    await this.done();
 
     invariant(this.title, 'Missing title');
     invariant(this.description, 'Missing description');

Build/lib/rules/domainset.ts

@@ -76,15 +76,15 @@ export class DomainsetOutput extends RuleOutput {
     invariant(this.apexDomainMap, 'Missing apex domain map');
 
     return Array.from(
-      (
+      nullthrow(this.sorted, 'Non dumped yet')
-        nullthrow(this.sorted, 'Non dumped yet').reduce<Map<string, number>>((acc, cur) => {
+        .reduce<Map<string, number>>((acc, cur) => {
           const suffix = this.apexDomainMap!.get(cur);
           if (suffix) {
             acc.set(suffix, (acc.get(suffix) ?? 0) + 1);
           }
           return acc;
         }, new Map())
-      ).entries()
+        .entries()
     )
       .filter(a => a[1] > 9)
       .sort(

Build/lib/rules/ruleset.ts

@@ -12,7 +12,7 @@ export class RulesetOutput extends RuleOutput {
   }
 
   private $computed: [domain: string[], domainSuffix: string[], sortedDomainRules: string[]] | null = null;
-  private get computed() {
+  private computed() {
     if (!this.$computed) {
       const kwfilter = createKeywordFilter(this.domainKeywords);
 
@@ -40,7 +40,7 @@ export class RulesetOutput extends RuleOutput {
 
   surge(): string[] {
     const results: string[] = ['DOMAIN,this_ruleset_is_made_by_sukkaw.ruleset.skk.moe'];
-    appendArrayInPlace(results, this.computed[2]);
+    appendArrayInPlace(results, this.computed()[2]);
 
     appendArrayFromSet(results, this.domainKeywords, i => `DOMAIN-KEYWORD,${i}`);
     appendArrayFromSet(results, this.domainWildcard, i => `DOMAIN-WILDCARD,${i}`);
@@ -70,7 +70,7 @@ export class RulesetOutput extends RuleOutput {
   clash(): string[] {
     const results: string[] = ['DOMAIN,this_ruleset_is_made_by_sukkaw.ruleset.skk.moe'];
 
-    appendArrayInPlace(results, this.computed[2]);
+    appendArrayInPlace(results, this.computed()[2]);
 
     appendArrayFromSet(results, this.domainKeywords, i => `DOMAIN-KEYWORD,${i}`);
     appendArrayFromSet(results, this.domainWildcard, i => `DOMAIN-REGEX,${RuleOutput.domainWildCardToRegex(i)}`);
@@ -97,8 +97,8 @@ export class RulesetOutput extends RuleOutput {
     const singbox: SingboxSourceFormat = {
       version: 2,
       rules: [{
-        domain: ['this_ruleset_is_made_by_sukkaw.ruleset.skk.moe'].concat(this.computed[0]),
+        domain: ['this_ruleset_is_made_by_sukkaw.ruleset.skk.moe'].concat(this.computed()[0]),
-        domain_suffix: this.computed[1],
+        domain_suffix: this.computed()[1],
         domain_keyword: Array.from(this.domainKeywords),
         domain_regex: Array.from(this.domainWildcard).map(RuleOutput.domainWildCardToRegex),
         ip_cidr: appendArrayFromSet([], [this.ipcidr, this.ipcidrNoResolve, this.ipcidr6, this.ipcidr6NoResolve]),

Build/validate-domestic.ts

@@ -4,6 +4,7 @@ import { createTrie } from './lib/trie';
 import path from 'node:path';
 import { processLine } from './lib/process-line';
 import { extractDomainsFromFelixDnsmasq } from './lib/parse-dnsmasq';
+import { SOURCE_DIR } from './constants/dir';
 
 export const parseDomesticList = async () => {
   const set = new Set<string>();
@@ -62,7 +63,7 @@ export const parseDomesticList = async () => {
   };
 
   // await Promise.all([
-  await runAgainstRuleset(path.resolve(__dirname, '../List/non_ip/domestic.conf'));
+  await runAgainstRuleset(path.resolve(SOURCE_DIR, 'non_ip/domestic.conf'));
   // ]);
 
   console.log(notIncludedDomestic.size, notIncludedDomestic);

Build/validate-gfwlist.ts

@@ -108,7 +108,7 @@ export const parseGfwList = async () => {
   await Promise.all([
     runAgainstRuleset(path.join(SOURCE_DIR, 'non_ip/global.conf')),
     runAgainstRuleset(path.join(SOURCE_DIR, 'non_ip/telegram.conf')),
-    runAgainstRuleset(path.resolve(__dirname, '../List/non_ip/stream.conf'))
+    runAgainstRuleset(path.resolve(SOURCE_DIR, 'non_ip/stream.conf'))
   ]);
 
   console.log(notIncludedTop500Gfwed);