Refactor: new MITM hostname sgmodule codegen

Build/build-common.ts

@@ -14,6 +14,7 @@ import { DomainsetOutput, RulesetOutput } from './lib/create-file';
 const MAGIC_COMMAND_SKIP = '# $ custom_build_script';
 const MAGIC_COMMAND_TITLE = '# $ meta_title ';
 const MAGIC_COMMAND_DESCRIPTION = '# $ meta_description ';
+const MAGIC_COMMAND_SGMODULE_MITM_HOSTNAMES = '# $ sgmodule_mitm_hostnames ';
 
 const domainsetSrcFolder = 'domainset' + path.sep;
 
@@ -73,6 +74,7 @@ const processFile = (span: Span, sourcePath: string) => {
 
     let title = '';
     const descriptions: string[] = [];
+    let sgmodulePathname: string | null = null;
 
     try {
       for await (const line of readFileByLine(sourcePath)) {
@@ -90,6 +92,11 @@ const processFile = (span: Span, sourcePath: string) => {
           continue;
         }
 
+        if (line.startsWith(MAGIC_COMMAND_SGMODULE_MITM_HOSTNAMES)) {
+          sgmodulePathname = line.slice(MAGIC_COMMAND_SGMODULE_MITM_HOSTNAMES.length).trim();
+          continue;
+        }
+
         const l = processLine(line);
         if (l) {
           lines.push(l);
@@ -100,7 +107,7 @@ const processFile = (span: Span, sourcePath: string) => {
       console.trace(e);
     }
 
-    return [title, descriptions, lines] as const;
+    return [title, descriptions, lines, sgmodulePathname] as const;
   });
 };
 
@@ -148,7 +155,7 @@ async function transformRuleset(parentSpan: Span, sourcePath: string, relativePa
         throw new TypeError(`Invalid type: ${type}`);
       }
 
-      const [title, descriptions, lines] = res;
+      const [title, descriptions, lines, sgmodulePathname] = res;
 
       let description: string[];
       if (descriptions.length) {
@@ -162,6 +169,7 @@ async function transformRuleset(parentSpan: Span, sourcePath: string, relativePa
       return new RulesetOutput(span, id, type)
         .withTitle(title)
         .withDescription(description)
+        .withMitmSgmodulePath(sgmodulePathname)
         .addFromRuleset(lines)
         .write();
     });

Build/build-mitm-hostname.ts

@@ -7,30 +7,6 @@ import { getHostname } from 'tldts';
 import { OUTPUT_SURGE_DIR } from './constants/dir';
 
 const PRESET_MITM_HOSTNAMES = [
-  // '*baidu.com',
-  '*.ydstatic.com',
-  // '*snssdk.com',
-  // '*musical.com',
-  // '*musical.ly',
-  // '*snssdk.ly',
-  'api.zhihu.com',
-  'www.zhihu.com',
-  'api.chelaile.net.cn',
-  'atrace.chelaile.net.cn',
-  '*.meituan.net',
-  'ctrl.playcvn.com',
-  'ctrl.playcvn.net',
-  'ctrl.zmzapi.com',
-  'ctrl.zmzapi.net',
-  'api.zhuishushenqi.com',
-  'b.zhuishushenqi.com',
-  'ggic.cmvideo.cn',
-  'ggic2.cmvideo.cn',
-  'mrobot.pcauto.com.cn',
-  'mrobot.pconline.com.cn',
-  'home.umetrip.com',
-  'discardrp.umetrip.com',
-  'startup.umetrip.com',
   'dsp-x.jd.com',
   'bdsp-x.jd.com'
 ];

Build/lib/rules/base.ts

@@ -1,4 +1,4 @@
-import { OUTPUT_CLASH_DIR, OUTPUT_SINGBOX_DIR, OUTPUT_SURGE_DIR } from '../../constants/dir';
+import { OUTPUT_CLASH_DIR, OUTPUT_MODULES_DIR, OUTPUT_SINGBOX_DIR, OUTPUT_SURGE_DIR } from '../../constants/dir';
 import type { Span } from '../../trace';
 import { createTrie } from '../trie';
 import stringify from 'json-stringify-pretty-compact';
@@ -256,7 +256,7 @@ export abstract class RuleOutput<TPreprocessed = unknown> {
     invariant(this.title, 'Missing title');
     invariant(this.description, 'Missing description');
 
-    await Promise.all([
+    const promises = [
       compareAndWriteFile(
         this.span,
         withBannerArray(
@@ -282,12 +282,37 @@ export abstract class RuleOutput<TPreprocessed = unknown> {
         this.singbox(),
         path.join(OUTPUT_SINGBOX_DIR, this.type, this.id + '.json')
       )
-    ]);
+    ];
+
+    if (this.mitmSgmodule) {
+      const sgmodule = this.mitmSgmodule();
+      const sgMOdulePath = this.mitmSgmodulePath ?? path.join(this.type, this.id + '.sgmodule');
+      if (sgmodule) {
+        promises.push(
+          compareAndWriteFile(
+            this.span,
+            sgmodule,
+            path.join(OUTPUT_MODULES_DIR, sgMOdulePath)
+          )
+        );
+      }
+    }
+
+    await Promise.all(promises);
   }
 
   abstract surge(): string[];
   abstract clash(): string[];
   abstract singbox(): string[];
+
+  protected mitmSgmodulePath: string | null = null;
+  withMitmSgmodulePath(path: string | null) {
+    if (path) {
+      this.mitmSgmodulePath = path;
+    }
+    return this;
+  }
+  abstract mitmSgmodule?(): string[] | null;
 }
 
 export const fileEqual = async (linesA: string[], source: AsyncIterable<string>): Promise<boolean> => {

Build/lib/rules/ruleset.ts

@@ -6,6 +6,9 @@ import { appendArrayFromSet } from '../misc';
 import type { SingboxSourceFormat } from '../singbox';
 import { sortDomains } from '../stable-sort-domain';
 import { RuleOutput } from './base';
+import picocolors from 'picocolors';
+import { normalizeDomain } from '../normalize-domain';
+import { isProbablyIpv4 } from '../is-fast-ip';
 
 type Preprocessed = [domain: string[], domainSuffix: string[], sortedDomainRules: string[]];
 
@@ -131,4 +134,97 @@ export class RulesetOutput extends RuleOutput<Preprocessed> {
 
     return RuleOutput.jsonToLines(singbox);
   }
+
+  mitmSgmodule(): string[] | null {
+    if (this.urlRegex.size === 0 || this.mitmSgmodulePath === null) {
+      return null;
+    }
+
+    const urlRegexResults: Array<{ origin: string, processed: string[] }> = [];
+
+    const parsedFailures: Array<[original: string, processed: string]> = [];
+    const parsed: Array<[original: string, domain: string]> = [];
+
+    for (let urlRegex of this.urlRegex) {
+      if (
+        urlRegex.startsWith('http://')
+        || urlRegex.startsWith('^http://')
+      ) {
+        continue;
+      }
+      if (urlRegex.startsWith('^https?://')) {
+        urlRegex = urlRegex.slice(10);
+      }
+      if (urlRegex.startsWith('^https://')) {
+        urlRegex = urlRegex.slice(9);
+      }
+
+      const potentialHostname = urlRegex.split('/')[0]
+        // pre process regex
+        .replaceAll(String.raw`\.`, '.')
+        .replaceAll('.+', '*')
+        .replaceAll(/([a-z])\?/g, '($1|)')
+        // convert regex to surge hostlist syntax
+        .replaceAll('([a-z])', '?')
+        .replaceAll(String.raw`\d`, '?')
+        .replaceAll(/\*+/g, '*');
+
+      let processed: string[] = [potentialHostname];
+
+      const matches = [...potentialHostname.matchAll(/\((?:([^()|]+)\|)+([^()|]*)\)/g)];
+
+      if (matches.length > 0) {
+        const replaceVariant = (combinations: string[], fullMatch: string, options: string[]): string[] => {
+          const newCombinations: string[] = [];
+
+          combinations.forEach(combination => {
+            options.forEach(option => {
+              newCombinations.push(combination.replace(fullMatch, option));
+            });
+          });
+
+          return newCombinations;
+        };
+
+        for (let i = 0; i < matches.length; i++) {
+          const match = matches[i];
+          const [_, ...options] = match;
+
+          processed = replaceVariant(processed, _, options);
+        }
+      }
+
+      urlRegexResults.push({
+        origin: potentialHostname,
+        processed
+      });
+    }
+
+    for (const i of urlRegexResults) {
+      for (const processed of i.processed) {
+        if (normalizeDomain(
+          processed
+            .replaceAll('*', 'a')
+            .replaceAll('?', 'b')
+        )) {
+          parsed.push([i.origin, processed]);
+        } else if (!isProbablyIpv4(processed)) {
+          parsedFailures.push([i.origin, processed]);
+        }
+      }
+    }
+
+    console.error(picocolors.bold('Parsed Failed'));
+    if (parsedFailures.length > 0) {
+      console.table(parsedFailures);
+    }
+
+    return [
+      '#!name=[Sukka] Surge Reject MITM',
+      '#!desc=为 URL Regex 规则组启用 MITM',
+      '',
+      '[MITM]',
+      'hostname = %APPEND% ' + Array.from(new Set(parsed.map(i => i[1]))).join(', ')
+    ];
+  }
 }

Modules/sukka_mitm_hostnames.sgmodule

@@ -1,7 +0,0 @@
-#!name=[Sukka] Surge Reject MITM
-#!desc=为 URL Regex 规则组启用 MITM
-# Use Build/build-mitm-hostname.ts to generate the list
-
-[MITM]
-
-hostname = %APPEND% *.ydstatic.com, api.zhihu.com, www.zhihu.com, api.chelaile.net.cn, atrace.chelaile.net.cn, *.meituan.net, ctrl.playcvn.com, ctrl.playcvn.net, ctrl.zmzapi.com, ctrl.zmzapi.net, api.zhuishushenqi.com, b.zhuishushenqi.com, ggic.cmvideo.cn, ggic2.cmvideo.cn, mrobot.pcauto.com.cn, mrobot.pconline.com.cn, home.umetrip.com, discardrp.umetrip.com, startup.umetrip.com, dsp-x.jd.com, bdsp-x.jd.com, premiumyva.appspot.com, cover.baidu.com, c.tieba.baidu.com, issuecdn.baidupcs.com, app.bilibili.com, www.cntv.cn, img-ys011.didistatic.com, act.vip.iqiyi.com, iface.iqiyi.com, counter.ksosoft.com, ios.wps.cn, mobile-pic.cache.iciba.com, service.iciba.com, client.mail.163.com, c.m.163.com, dsp-impr2.youdao.com, sp.kaola.com, support.you.163.com, agent-count.pconline.com.cn, tqt.weibo.cn, edit.sinaapp.com, wbapp.uve.weibo.com, api.k.sohu.com, api.tv.sohu.com, hui.sohu.com, s1.api.tv.itc.cn, b, api.m.jd.com, ms.jr.jd.com, huichuan.sm.cn, iflow.uczzd.cn, adse.ximalaya.com, app.58.com, aes.acfun.cn, acs.m.taobao.com, dsp.toutiao.com, nnapp.cloudbae.cn, gw.aihuishou.com, m*.amap.com, 7n.bczcdn.com, www.myhug.cn, app.api.ke.com, channel.beitaichufang.com, iapi.bishijie.com, api.intsig.net, cap.caocaokeji.cn, pic1.chelaile.net.cn, m.client.10010.com, www.dandanzan.com, mapi.dangdang.com, api.daydaycook.com.cn, cms.daydaycook.com.cn, www.flyertea.com, cdn.api.fotoable.com, gateway.shouqiev.com, m.ibuscloud.com, smkmp.96225.com, imeclient.openspeech.cn, img.jiemian.com, api.jxedt.com, richmanapi.jxedt.com, api.gotokeep.com, res.kfc.com.cn, api.smzdm.com, api5.futunn.com, qt.qq.com, 3gimg.qq.com, newsso.map.qq.com, vv.video.qq.com, szextshort.weixin.qq.com, y.gtimg.cn, api.gaoqingdianshi.com, pss.txffp.com, app.variflight.com, static.vuevideo.net, api.wallstreetcn.com, app.wy.guahao.com, thor.weidian.com, nochange.ggsafe.com, api-release.wuta-cam.com, res-release.wuta-cam.com, api.xiachufang.com, mapi.mafengwo.cn, mangaapi.manhuaren.com, img.meituan.net, capi.mwee.cn, api.m.mi.com, b-api.ins.miaopai.com, app.mixcapp.com, api.mgzf.com, dili.bdatu.com, wap.ngchina.cn, app3.qdaily.com, notch.qdaily.com, media.qyer.com, api.qiuduoduo.cn, api.rr.tv, api.videozhishi.com, msspjh.emarbox.com, www.shihuo.cn, api.psy-1.com, m.yap.yahoo.com, i.ys7.com, api.catch.gift, a.qiumibao.com, api01pbmp.zhuishushenqi.com, dspsdk.abreader.com, mi.gdt.qq.com, service.4gtv.tv, api.bilibili.com, manga.bilibili.com, sdkapp.uve.weibo.com, app.10086.cn, mobile-api2011.elong.com, foodie-api.yiruikecorp.com, gw-passenger.01zhuanche.com, snailsleep.net, a.sfansclub.com, r.inews.qq.com, mp.weixin.qq.com, cmsapi.wifi8.com, mob.mddcloud.com.cn, p*.meituan.net, api.jr.mi.com, home.mi.com, api-mifit.huami.com, cdn.moji.com, open.qyer.com, portal-xunyou.qingcdn.com, asp.cntv.myalicdn.com, cntv.hls.cdn.myqcloud.com, v.cctv.com, *.kingsoft-office-service.com, oimage*.ydstatic.com, *.58cdn.com.cn, static1.keepcdn.com, adpai.thepaper.cn, images.client.vip.xunlei.com, s3plus.meituan.net, *.iydsj.com, dict-mobile.iciba.com, games.mobileapi.hupu.com, api.kkmh.com

Modules/sukka_url_rewrite.sgmodule

@@ -1,21 +0,0 @@
-#!name=[Sukka] URL Rewrite
-#!desc=Enable this module to use Sukka URL Rewrite rules
-
-[MITM]
-
-hostname = %APPEND% api.abema.io, union.click.jd.com, nomo.dafork.com
-
-[URL Rewrite]
-# AbeamTV Unlock
-^https?://api\.abema\.io/v\d/ip/check - reject
-
-# JD Protection
-^https?://union\.click\.jd\.com/jda? http://union.click.jd.com/jda?adblock= header
-^https?://union\.click\.jd\.com/sem.php? http://union.click.jd.com/sem.php?adblock= header
-
-# Special AD Block Section
-
-# >> eLong
-^https?://\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/(adgateway|adv)/ - reject
-# >> NOMO
-^https?://nomo.dafork.com/api/v3/iap/ios_product_list https://ruleset.skk.moe/Mock/nomo.json 302

Source/non_ip/reject-url-regex.conf

@@ -1,6 +1,7 @@
 # $ meta_title Sukka's Ruleset - Reject URL
 # $ meta_description The ruleset supports AD blocking, tracking protection, privacy protection, anti-phishing, anti-mining
 # $ meta_description Need Surge Module: https://ruleset.skk.moe/Modules/sukka_mitm_hostnames.sgmodule
+# $ sgmodule_mitm_hostnames sukka_mitm_hostnames.sgmodule
 
 # URL-REGEX,^https?://.+\.youtube\.com/api/stats/.+adformat
 # URL-REGEX,^https?://.+\.youtube\.com/api/stats/ads
@@ -45,7 +46,7 @@ URL-REGEX,^https://img-ys011\.didistatic\.com/static/ad_oss/
 
 # >> Hanglvzongheng
 URL-REGEX,^https?://(discardrp|startup)\.umetrip\.com/gateway/api/umetrip/native
-URL-REGEX,^https?://(114\.115\.217\.129)|(home\.umetrip\.com)/gateway/api/umetrip/native$
+URL-REGEX,^https?://(114\.115\.217\.129|home\.umetrip\.com)/gateway/api/umetrip/native$
 
 # >> iQiyi
 URL-REGEX,^https?://act\.vip\.iqiyi\.com/interact/api/show\.do
@@ -176,6 +177,7 @@ URL-REGEX,^https?://api\.daydaycook\.com\.cn/daydaycook/server/ad/
 URL-REGEX,^https?://cms\.daydaycook\.com\.cn/api/cms/advertisement/
 # >> eLong
 URL-REGEX,^https?://mobile-api2011\.elong\.com/(adgateway|adv)/
+URL-REGEX,^http://\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/(adgateway|adv)/
 # >> Flyer Tea
 URL-REGEX,^https?://www\.flyertea\.com/source/plugin/mobile/mobile\.php\?module=advis
 # >> Foodie
@@ -340,4 +342,4 @@ URL-REGEX,^https?://dspsdk\.abreader\.com/v\d/api/ad\?
 # URL-REGEX,^https?://itunes\.apple\.com/lookup\?id=575826903
 URL-REGEX,^https?://mi\.gdt\.qq\.com/gdt_mview\.fcg
 # >> Kugou Music
-URL-REGEX,^https?://\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/(EcomResourceServer/AdPlayPage/adinfo|MobileAdServer/)
+URL-REGEX,^http://\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/(EcomResourceServer/AdPlayPage/adinfo|MobileAdServer/)