diff --git a/Build/build-common.ts b/Build/build-common.ts index e267a5f7..9e7ba426 100644 --- a/Build/build-common.ts +++ b/Build/build-common.ts @@ -14,6 +14,7 @@ import { DomainsetOutput, RulesetOutput } from './lib/create-file'; const MAGIC_COMMAND_SKIP = '# $ custom_build_script'; const MAGIC_COMMAND_TITLE = '# $ meta_title '; const MAGIC_COMMAND_DESCRIPTION = '# $ meta_description '; +const MAGIC_COMMAND_SGMODULE_MITM_HOSTNAMES = '# $ sgmodule_mitm_hostnames '; const domainsetSrcFolder = 'domainset' + path.sep; @@ -73,6 +74,7 @@ const processFile = (span: Span, sourcePath: string) => { let title = ''; const descriptions: string[] = []; + let sgmodulePathname: string | null = null; try { for await (const line of readFileByLine(sourcePath)) { @@ -90,6 +92,11 @@ const processFile = (span: Span, sourcePath: string) => { continue; } + if (line.startsWith(MAGIC_COMMAND_SGMODULE_MITM_HOSTNAMES)) { + sgmodulePathname = line.slice(MAGIC_COMMAND_SGMODULE_MITM_HOSTNAMES.length).trim(); + continue; + } + const l = processLine(line); if (l) { lines.push(l); @@ -100,7 +107,7 @@ const processFile = (span: Span, sourcePath: string) => { console.trace(e); } - return [title, descriptions, lines] as const; + return [title, descriptions, lines, sgmodulePathname] as const; }); }; @@ -148,7 +155,7 @@ async function transformRuleset(parentSpan: Span, sourcePath: string, relativePa throw new TypeError(`Invalid type: ${type}`); } - const [title, descriptions, lines] = res; + const [title, descriptions, lines, sgmodulePathname] = res; let description: string[]; if (descriptions.length) { @@ -162,6 +169,7 @@ async function transformRuleset(parentSpan: Span, sourcePath: string, relativePa return new RulesetOutput(span, id, type) .withTitle(title) .withDescription(description) + .withMitmSgmodulePath(sgmodulePathname) .addFromRuleset(lines) .write(); }); diff --git a/Build/build-mitm-hostname.ts b/Build/build-mitm-hostname.ts index 52364cd8..537ae293 100644 --- a/Build/build-mitm-hostname.ts +++ b/Build/build-mitm-hostname.ts @@ -7,30 +7,6 @@ import { getHostname } from 'tldts'; import { OUTPUT_SURGE_DIR } from './constants/dir'; const PRESET_MITM_HOSTNAMES = [ - // '*baidu.com', - '*.ydstatic.com', - // '*snssdk.com', - // '*musical.com', - // '*musical.ly', - // '*snssdk.ly', - 'api.zhihu.com', - 'www.zhihu.com', - 'api.chelaile.net.cn', - 'atrace.chelaile.net.cn', - '*.meituan.net', - 'ctrl.playcvn.com', - 'ctrl.playcvn.net', - 'ctrl.zmzapi.com', - 'ctrl.zmzapi.net', - 'api.zhuishushenqi.com', - 'b.zhuishushenqi.com', - 'ggic.cmvideo.cn', - 'ggic2.cmvideo.cn', - 'mrobot.pcauto.com.cn', - 'mrobot.pconline.com.cn', - 'home.umetrip.com', - 'discardrp.umetrip.com', - 'startup.umetrip.com', 'dsp-x.jd.com', 'bdsp-x.jd.com' ]; diff --git a/Build/lib/rules/base.ts b/Build/lib/rules/base.ts index 65282356..9db86bb1 100644 --- a/Build/lib/rules/base.ts +++ b/Build/lib/rules/base.ts @@ -1,4 +1,4 @@ -import { OUTPUT_CLASH_DIR, OUTPUT_SINGBOX_DIR, OUTPUT_SURGE_DIR } from '../../constants/dir'; +import { OUTPUT_CLASH_DIR, OUTPUT_MODULES_DIR, OUTPUT_SINGBOX_DIR, OUTPUT_SURGE_DIR } from '../../constants/dir'; import type { Span } from '../../trace'; import { createTrie } from '../trie'; import stringify from 'json-stringify-pretty-compact'; @@ -256,7 +256,7 @@ export abstract class RuleOutput { invariant(this.title, 'Missing title'); invariant(this.description, 'Missing description'); - await Promise.all([ + const promises = [ compareAndWriteFile( this.span, withBannerArray( @@ -282,12 +282,37 @@ export abstract class RuleOutput { this.singbox(), path.join(OUTPUT_SINGBOX_DIR, this.type, this.id + '.json') ) - ]); + ]; + + if (this.mitmSgmodule) { + const sgmodule = this.mitmSgmodule(); + const sgMOdulePath = this.mitmSgmodulePath ?? path.join(this.type, this.id + '.sgmodule'); + if (sgmodule) { + promises.push( + compareAndWriteFile( + this.span, + sgmodule, + path.join(OUTPUT_MODULES_DIR, sgMOdulePath) + ) + ); + } + } + + await Promise.all(promises); } abstract surge(): string[]; abstract clash(): string[]; abstract singbox(): string[]; + + protected mitmSgmodulePath: string | null = null; + withMitmSgmodulePath(path: string | null) { + if (path) { + this.mitmSgmodulePath = path; + } + return this; + } + abstract mitmSgmodule?(): string[] | null; } export const fileEqual = async (linesA: string[], source: AsyncIterable): Promise => { diff --git a/Build/lib/rules/ruleset.ts b/Build/lib/rules/ruleset.ts index fb1fc1d8..74e7ba1b 100644 --- a/Build/lib/rules/ruleset.ts +++ b/Build/lib/rules/ruleset.ts @@ -6,6 +6,9 @@ import { appendArrayFromSet } from '../misc'; import type { SingboxSourceFormat } from '../singbox'; import { sortDomains } from '../stable-sort-domain'; import { RuleOutput } from './base'; +import picocolors from 'picocolors'; +import { normalizeDomain } from '../normalize-domain'; +import { isProbablyIpv4 } from '../is-fast-ip'; type Preprocessed = [domain: string[], domainSuffix: string[], sortedDomainRules: string[]]; @@ -131,4 +134,97 @@ export class RulesetOutput extends RuleOutput { return RuleOutput.jsonToLines(singbox); } + + mitmSgmodule(): string[] | null { + if (this.urlRegex.size === 0 || this.mitmSgmodulePath === null) { + return null; + } + + const urlRegexResults: Array<{ origin: string, processed: string[] }> = []; + + const parsedFailures: Array<[original: string, processed: string]> = []; + const parsed: Array<[original: string, domain: string]> = []; + + for (let urlRegex of this.urlRegex) { + if ( + urlRegex.startsWith('http://') + || urlRegex.startsWith('^http://') + ) { + continue; + } + if (urlRegex.startsWith('^https?://')) { + urlRegex = urlRegex.slice(10); + } + if (urlRegex.startsWith('^https://')) { + urlRegex = urlRegex.slice(9); + } + + const potentialHostname = urlRegex.split('/')[0] + // pre process regex + .replaceAll(String.raw`\.`, '.') + .replaceAll('.+', '*') + .replaceAll(/([a-z])\?/g, '($1|)') + // convert regex to surge hostlist syntax + .replaceAll('([a-z])', '?') + .replaceAll(String.raw`\d`, '?') + .replaceAll(/\*+/g, '*'); + + let processed: string[] = [potentialHostname]; + + const matches = [...potentialHostname.matchAll(/\((?:([^()|]+)\|)+([^()|]*)\)/g)]; + + if (matches.length > 0) { + const replaceVariant = (combinations: string[], fullMatch: string, options: string[]): string[] => { + const newCombinations: string[] = []; + + combinations.forEach(combination => { + options.forEach(option => { + newCombinations.push(combination.replace(fullMatch, option)); + }); + }); + + return newCombinations; + }; + + for (let i = 0; i < matches.length; i++) { + const match = matches[i]; + const [_, ...options] = match; + + processed = replaceVariant(processed, _, options); + } + } + + urlRegexResults.push({ + origin: potentialHostname, + processed + }); + } + + for (const i of urlRegexResults) { + for (const processed of i.processed) { + if (normalizeDomain( + processed + .replaceAll('*', 'a') + .replaceAll('?', 'b') + )) { + parsed.push([i.origin, processed]); + } else if (!isProbablyIpv4(processed)) { + parsedFailures.push([i.origin, processed]); + } + } + } + + console.error(picocolors.bold('Parsed Failed')); + if (parsedFailures.length > 0) { + console.table(parsedFailures); + } + + return [ + '#!name=[Sukka] Surge Reject MITM', + '#!desc=为 URL Regex 规则组启用 MITM', + '', + '[MITM]', + 'hostname = %APPEND% ' + Array.from(new Set(parsed.map(i => i[1]))).join(', ') + ]; + } } diff --git a/Modules/sukka_mitm_hostnames.sgmodule b/Modules/sukka_mitm_hostnames.sgmodule deleted file mode 100644 index ea98a1b9..00000000 --- a/Modules/sukka_mitm_hostnames.sgmodule +++ /dev/null @@ -1,7 +0,0 @@ -#!name=[Sukka] Surge Reject MITM -#!desc=为 URL Regex 规则组启用 MITM -# Use Build/build-mitm-hostname.ts to generate the list - -[MITM] - -hostname = %APPEND% *.ydstatic.com, api.zhihu.com, www.zhihu.com, api.chelaile.net.cn, atrace.chelaile.net.cn, *.meituan.net, ctrl.playcvn.com, ctrl.playcvn.net, ctrl.zmzapi.com, ctrl.zmzapi.net, api.zhuishushenqi.com, b.zhuishushenqi.com, ggic.cmvideo.cn, ggic2.cmvideo.cn, mrobot.pcauto.com.cn, mrobot.pconline.com.cn, home.umetrip.com, discardrp.umetrip.com, startup.umetrip.com, dsp-x.jd.com, bdsp-x.jd.com, premiumyva.appspot.com, cover.baidu.com, c.tieba.baidu.com, issuecdn.baidupcs.com, app.bilibili.com, www.cntv.cn, img-ys011.didistatic.com, act.vip.iqiyi.com, iface.iqiyi.com, counter.ksosoft.com, ios.wps.cn, mobile-pic.cache.iciba.com, service.iciba.com, client.mail.163.com, c.m.163.com, dsp-impr2.youdao.com, sp.kaola.com, support.you.163.com, agent-count.pconline.com.cn, tqt.weibo.cn, edit.sinaapp.com, wbapp.uve.weibo.com, api.k.sohu.com, api.tv.sohu.com, hui.sohu.com, s1.api.tv.itc.cn, b, api.m.jd.com, ms.jr.jd.com, huichuan.sm.cn, iflow.uczzd.cn, adse.ximalaya.com, app.58.com, aes.acfun.cn, acs.m.taobao.com, dsp.toutiao.com, nnapp.cloudbae.cn, gw.aihuishou.com, m*.amap.com, 7n.bczcdn.com, www.myhug.cn, app.api.ke.com, channel.beitaichufang.com, iapi.bishijie.com, api.intsig.net, cap.caocaokeji.cn, pic1.chelaile.net.cn, m.client.10010.com, www.dandanzan.com, mapi.dangdang.com, api.daydaycook.com.cn, cms.daydaycook.com.cn, www.flyertea.com, cdn.api.fotoable.com, gateway.shouqiev.com, m.ibuscloud.com, smkmp.96225.com, imeclient.openspeech.cn, img.jiemian.com, api.jxedt.com, richmanapi.jxedt.com, api.gotokeep.com, res.kfc.com.cn, api.smzdm.com, api5.futunn.com, qt.qq.com, 3gimg.qq.com, newsso.map.qq.com, vv.video.qq.com, szextshort.weixin.qq.com, y.gtimg.cn, api.gaoqingdianshi.com, pss.txffp.com, app.variflight.com, static.vuevideo.net, api.wallstreetcn.com, app.wy.guahao.com, thor.weidian.com, nochange.ggsafe.com, api-release.wuta-cam.com, res-release.wuta-cam.com, api.xiachufang.com, mapi.mafengwo.cn, mangaapi.manhuaren.com, img.meituan.net, capi.mwee.cn, api.m.mi.com, b-api.ins.miaopai.com, app.mixcapp.com, api.mgzf.com, dili.bdatu.com, wap.ngchina.cn, app3.qdaily.com, notch.qdaily.com, media.qyer.com, api.qiuduoduo.cn, api.rr.tv, api.videozhishi.com, msspjh.emarbox.com, www.shihuo.cn, api.psy-1.com, m.yap.yahoo.com, i.ys7.com, api.catch.gift, a.qiumibao.com, api01pbmp.zhuishushenqi.com, dspsdk.abreader.com, mi.gdt.qq.com, service.4gtv.tv, api.bilibili.com, manga.bilibili.com, sdkapp.uve.weibo.com, app.10086.cn, mobile-api2011.elong.com, foodie-api.yiruikecorp.com, gw-passenger.01zhuanche.com, snailsleep.net, a.sfansclub.com, r.inews.qq.com, mp.weixin.qq.com, cmsapi.wifi8.com, mob.mddcloud.com.cn, p*.meituan.net, api.jr.mi.com, home.mi.com, api-mifit.huami.com, cdn.moji.com, open.qyer.com, portal-xunyou.qingcdn.com, asp.cntv.myalicdn.com, cntv.hls.cdn.myqcloud.com, v.cctv.com, *.kingsoft-office-service.com, oimage*.ydstatic.com, *.58cdn.com.cn, static1.keepcdn.com, adpai.thepaper.cn, images.client.vip.xunlei.com, s3plus.meituan.net, *.iydsj.com, dict-mobile.iciba.com, games.mobileapi.hupu.com, api.kkmh.com diff --git a/Modules/sukka_url_rewrite.sgmodule b/Modules/sukka_url_rewrite.sgmodule deleted file mode 100644 index c575bca3..00000000 --- a/Modules/sukka_url_rewrite.sgmodule +++ /dev/null @@ -1,21 +0,0 @@ -#!name=[Sukka] URL Rewrite -#!desc=Enable this module to use Sukka URL Rewrite rules - -[MITM] - -hostname = %APPEND% api.abema.io, union.click.jd.com, nomo.dafork.com - -[URL Rewrite] -# AbeamTV Unlock -^https?://api\.abema\.io/v\d/ip/check - reject - -# JD Protection -^https?://union\.click\.jd\.com/jda? http://union.click.jd.com/jda?adblock= header -^https?://union\.click\.jd\.com/sem.php? http://union.click.jd.com/sem.php?adblock= header - -# Special AD Block Section - -# >> eLong -^https?://\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/(adgateway|adv)/ - reject -# >> NOMO -^https?://nomo.dafork.com/api/v3/iap/ios_product_list https://ruleset.skk.moe/Mock/nomo.json 302 diff --git a/Source/non_ip/reject-url-regex.conf b/Source/non_ip/reject-url-regex.conf index 69595153..faeba973 100644 --- a/Source/non_ip/reject-url-regex.conf +++ b/Source/non_ip/reject-url-regex.conf @@ -1,6 +1,7 @@ # $ meta_title Sukka's Ruleset - Reject URL # $ meta_description The ruleset supports AD blocking, tracking protection, privacy protection, anti-phishing, anti-mining # $ meta_description Need Surge Module: https://ruleset.skk.moe/Modules/sukka_mitm_hostnames.sgmodule +# $ sgmodule_mitm_hostnames sukka_mitm_hostnames.sgmodule # URL-REGEX,^https?://.+\.youtube\.com/api/stats/.+adformat # URL-REGEX,^https?://.+\.youtube\.com/api/stats/ads @@ -45,7 +46,7 @@ URL-REGEX,^https://img-ys011\.didistatic\.com/static/ad_oss/ # >> Hanglvzongheng URL-REGEX,^https?://(discardrp|startup)\.umetrip\.com/gateway/api/umetrip/native -URL-REGEX,^https?://(114\.115\.217\.129)|(home\.umetrip\.com)/gateway/api/umetrip/native$ +URL-REGEX,^https?://(114\.115\.217\.129|home\.umetrip\.com)/gateway/api/umetrip/native$ # >> iQiyi URL-REGEX,^https?://act\.vip\.iqiyi\.com/interact/api/show\.do @@ -176,6 +177,7 @@ URL-REGEX,^https?://api\.daydaycook\.com\.cn/daydaycook/server/ad/ URL-REGEX,^https?://cms\.daydaycook\.com\.cn/api/cms/advertisement/ # >> eLong URL-REGEX,^https?://mobile-api2011\.elong\.com/(adgateway|adv)/ +URL-REGEX,^http://\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/(adgateway|adv)/ # >> Flyer Tea URL-REGEX,^https?://www\.flyertea\.com/source/plugin/mobile/mobile\.php\?module=advis # >> Foodie @@ -340,4 +342,4 @@ URL-REGEX,^https?://dspsdk\.abreader\.com/v\d/api/ad\? # URL-REGEX,^https?://itunes\.apple\.com/lookup\?id=575826903 URL-REGEX,^https?://mi\.gdt\.qq\.com/gdt_mview\.fcg # >> Kugou Music -URL-REGEX,^https?://\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/(EcomResourceServer/AdPlayPage/adinfo|MobileAdServer/) +URL-REGEX,^http://\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/(EcomResourceServer/AdPlayPage/adinfo|MobileAdServer/)