mirror of
https://github.com/SukkaW/Surge.git
synced 2025-12-12 09:10:35 +08:00
Refactor: new MITM hostname sgmodule codegen
This commit is contained in:
SukkaW
parent
d3a015ff1b
commit
5577f3b14b
@ -14,6 +14,7 @@ import { DomainsetOutput, RulesetOutput } from './lib/create-file';
|
||||
const MAGIC_COMMAND_SKIP = '# $ custom_build_script';
|
||||
const MAGIC_COMMAND_TITLE = '# $ meta_title ';
|
||||
const MAGIC_COMMAND_DESCRIPTION = '# $ meta_description ';
|
||||
const MAGIC_COMMAND_SGMODULE_MITM_HOSTNAMES = '# $ sgmodule_mitm_hostnames ';
|
||||
|
||||
const domainsetSrcFolder = 'domainset' + path.sep;
|
||||
|
||||
@ -73,6 +74,7 @@ const processFile = (span: Span, sourcePath: string) => {
|
||||
|
||||
let title = '';
|
||||
const descriptions: string[] = [];
|
||||
let sgmodulePathname: string | null = null;
|
||||
|
||||
try {
|
||||
for await (const line of readFileByLine(sourcePath)) {
|
||||
@ -90,6 +92,11 @@ const processFile = (span: Span, sourcePath: string) => {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (line.startsWith(MAGIC_COMMAND_SGMODULE_MITM_HOSTNAMES)) {
|
||||
sgmodulePathname = line.slice(MAGIC_COMMAND_SGMODULE_MITM_HOSTNAMES.length).trim();
|
||||
continue;
|
||||
}
|
||||
|
||||
const l = processLine(line);
|
||||
if (l) {
|
||||
lines.push(l);
|
||||
@ -100,7 +107,7 @@ const processFile = (span: Span, sourcePath: string) => {
|
||||
console.trace(e);
|
||||
}
|
||||
|
||||
return [title, descriptions, lines] as const;
|
||||
return [title, descriptions, lines, sgmodulePathname] as const;
|
||||
});
|
||||
};
|
||||
|
||||
@ -148,7 +155,7 @@ async function transformRuleset(parentSpan: Span, sourcePath: string, relativePa
|
||||
throw new TypeError(`Invalid type: ${type}`);
|
||||
}
|
||||
|
||||
const [title, descriptions, lines] = res;
|
||||
const [title, descriptions, lines, sgmodulePathname] = res;
|
||||
|
||||
let description: string[];
|
||||
if (descriptions.length) {
|
||||
@ -162,6 +169,7 @@ async function transformRuleset(parentSpan: Span, sourcePath: string, relativePa
|
||||
return new RulesetOutput(span, id, type)
|
||||
.withTitle(title)
|
||||
.withDescription(description)
|
||||
.withMitmSgmodulePath(sgmodulePathname)
|
||||
.addFromRuleset(lines)
|
||||
.write();
|
||||
});
|
||||
|
||||
@ -7,30 +7,6 @@ import { getHostname } from 'tldts';
|
||||
import { OUTPUT_SURGE_DIR } from './constants/dir';
|
||||
|
||||
const PRESET_MITM_HOSTNAMES = [
|
||||
// '*baidu.com',
|
||||
'*.ydstatic.com',
|
||||
// '*snssdk.com',
|
||||
// '*musical.com',
|
||||
// '*musical.ly',
|
||||
// '*snssdk.ly',
|
||||
'api.zhihu.com',
|
||||
'www.zhihu.com',
|
||||
'api.chelaile.net.cn',
|
||||
'atrace.chelaile.net.cn',
|
||||
'*.meituan.net',
|
||||
'ctrl.playcvn.com',
|
||||
'ctrl.playcvn.net',
|
||||
'ctrl.zmzapi.com',
|
||||
'ctrl.zmzapi.net',
|
||||
'api.zhuishushenqi.com',
|
||||
'b.zhuishushenqi.com',
|
||||
'ggic.cmvideo.cn',
|
||||
'ggic2.cmvideo.cn',
|
||||
'mrobot.pcauto.com.cn',
|
||||
'mrobot.pconline.com.cn',
|
||||
'home.umetrip.com',
|
||||
'discardrp.umetrip.com',
|
||||
'startup.umetrip.com',
|
||||
'dsp-x.jd.com',
|
||||
'bdsp-x.jd.com'
|
||||
];
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import { OUTPUT_CLASH_DIR, OUTPUT_SINGBOX_DIR, OUTPUT_SURGE_DIR } from '../../constants/dir';
|
||||
import { OUTPUT_CLASH_DIR, OUTPUT_MODULES_DIR, OUTPUT_SINGBOX_DIR, OUTPUT_SURGE_DIR } from '../../constants/dir';
|
||||
import type { Span } from '../../trace';
|
||||
import { createTrie } from '../trie';
|
||||
import stringify from 'json-stringify-pretty-compact';
|
||||
@ -256,7 +256,7 @@ export abstract class RuleOutput<TPreprocessed = unknown> {
|
||||
invariant(this.title, 'Missing title');
|
||||
invariant(this.description, 'Missing description');
|
||||
|
||||
await Promise.all([
|
||||
const promises = [
|
||||
compareAndWriteFile(
|
||||
this.span,
|
||||
withBannerArray(
|
||||
@ -282,12 +282,37 @@ export abstract class RuleOutput<TPreprocessed = unknown> {
|
||||
this.singbox(),
|
||||
path.join(OUTPUT_SINGBOX_DIR, this.type, this.id + '.json')
|
||||
)
|
||||
]);
|
||||
];
|
||||
|
||||
if (this.mitmSgmodule) {
|
||||
const sgmodule = this.mitmSgmodule();
|
||||
const sgMOdulePath = this.mitmSgmodulePath ?? path.join(this.type, this.id + '.sgmodule');
|
||||
if (sgmodule) {
|
||||
promises.push(
|
||||
compareAndWriteFile(
|
||||
this.span,
|
||||
sgmodule,
|
||||
path.join(OUTPUT_MODULES_DIR, sgMOdulePath)
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
await Promise.all(promises);
|
||||
}
|
||||
|
||||
abstract surge(): string[];
|
||||
abstract clash(): string[];
|
||||
abstract singbox(): string[];
|
||||
|
||||
protected mitmSgmodulePath: string | null = null;
|
||||
withMitmSgmodulePath(path: string | null) {
|
||||
if (path) {
|
||||
this.mitmSgmodulePath = path;
|
||||
}
|
||||
return this;
|
||||
}
|
||||
abstract mitmSgmodule?(): string[] | null;
|
||||
}
|
||||
|
||||
export const fileEqual = async (linesA: string[], source: AsyncIterable<string>): Promise<boolean> => {
|
||||
|
||||
@ -6,6 +6,9 @@ import { appendArrayFromSet } from '../misc';
|
||||
import type { SingboxSourceFormat } from '../singbox';
|
||||
import { sortDomains } from '../stable-sort-domain';
|
||||
import { RuleOutput } from './base';
|
||||
import picocolors from 'picocolors';
|
||||
import { normalizeDomain } from '../normalize-domain';
|
||||
import { isProbablyIpv4 } from '../is-fast-ip';
|
||||
|
||||
type Preprocessed = [domain: string[], domainSuffix: string[], sortedDomainRules: string[]];
|
||||
|
||||
@ -131,4 +134,97 @@ export class RulesetOutput extends RuleOutput<Preprocessed> {
|
||||
|
||||
return RuleOutput.jsonToLines(singbox);
|
||||
}
|
||||
|
||||
mitmSgmodule(): string[] | null {
|
||||
if (this.urlRegex.size === 0 || this.mitmSgmodulePath === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const urlRegexResults: Array<{ origin: string, processed: string[] }> = [];
|
||||
|
||||
const parsedFailures: Array<[original: string, processed: string]> = [];
|
||||
const parsed: Array<[original: string, domain: string]> = [];
|
||||
|
||||
for (let urlRegex of this.urlRegex) {
|
||||
if (
|
||||
urlRegex.startsWith('http://')
|
||||
|| urlRegex.startsWith('^http://')
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
if (urlRegex.startsWith('^https?://')) {
|
||||
urlRegex = urlRegex.slice(10);
|
||||
}
|
||||
if (urlRegex.startsWith('^https://')) {
|
||||
urlRegex = urlRegex.slice(9);
|
||||
}
|
||||
|
||||
const potentialHostname = urlRegex.split('/')[0]
|
||||
// pre process regex
|
||||
.replaceAll(String.raw`\.`, '.')
|
||||
.replaceAll('.+', '*')
|
||||
.replaceAll(/([a-z])\?/g, '($1|)')
|
||||
// convert regex to surge hostlist syntax
|
||||
.replaceAll('([a-z])', '?')
|
||||
.replaceAll(String.raw`\d`, '?')
|
||||
.replaceAll(/\*+/g, '*');
|
||||
|
||||
let processed: string[] = [potentialHostname];
|
||||
|
||||
const matches = [...potentialHostname.matchAll(/\((?:([^()|]+)\|)+([^()|]*)\)/g)];
|
||||
|
||||
if (matches.length > 0) {
|
||||
const replaceVariant = (combinations: string[], fullMatch: string, options: string[]): string[] => {
|
||||
const newCombinations: string[] = [];
|
||||
|
||||
combinations.forEach(combination => {
|
||||
options.forEach(option => {
|
||||
newCombinations.push(combination.replace(fullMatch, option));
|
||||
});
|
||||
});
|
||||
|
||||
return newCombinations;
|
||||
};
|
||||
|
||||
for (let i = 0; i < matches.length; i++) {
|
||||
const match = matches[i];
|
||||
const [_, ...options] = match;
|
||||
|
||||
processed = replaceVariant(processed, _, options);
|
||||
}
|
||||
}
|
||||
|
||||
urlRegexResults.push({
|
||||
origin: potentialHostname,
|
||||
processed
|
||||
});
|
||||
}
|
||||
|
||||
for (const i of urlRegexResults) {
|
||||
for (const processed of i.processed) {
|
||||
if (normalizeDomain(
|
||||
processed
|
||||
.replaceAll('*', 'a')
|
||||
.replaceAll('?', 'b')
|
||||
)) {
|
||||
parsed.push([i.origin, processed]);
|
||||
} else if (!isProbablyIpv4(processed)) {
|
||||
parsedFailures.push([i.origin, processed]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.error(picocolors.bold('Parsed Failed'));
|
||||
if (parsedFailures.length > 0) {
|
||||
console.table(parsedFailures);
|
||||
}
|
||||
|
||||
return [
|
||||
'#!name=[Sukka] Surge Reject MITM',
|
||||
'#!desc=为 URL Regex 规则组启用 MITM',
|
||||
'',
|
||||
'[MITM]',
|
||||
'hostname = %APPEND% ' + Array.from(new Set(parsed.map(i => i[1]))).join(', ')
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,7 +0,0 @@
|
||||
#!name=[Sukka] Surge Reject MITM
|
||||
#!desc=为 URL Regex 规则组启用 MITM
|
||||
# Use Build/build-mitm-hostname.ts to generate the list
|
||||
|
||||
[MITM]
|
||||
|
||||
hostname = %APPEND% *.ydstatic.com, api.zhihu.com, www.zhihu.com, api.chelaile.net.cn, atrace.chelaile.net.cn, *.meituan.net, ctrl.playcvn.com, ctrl.playcvn.net, ctrl.zmzapi.com, ctrl.zmzapi.net, api.zhuishushenqi.com, b.zhuishushenqi.com, ggic.cmvideo.cn, ggic2.cmvideo.cn, mrobot.pcauto.com.cn, mrobot.pconline.com.cn, home.umetrip.com, discardrp.umetrip.com, startup.umetrip.com, dsp-x.jd.com, bdsp-x.jd.com, premiumyva.appspot.com, cover.baidu.com, c.tieba.baidu.com, issuecdn.baidupcs.com, app.bilibili.com, www.cntv.cn, img-ys011.didistatic.com, act.vip.iqiyi.com, iface.iqiyi.com, counter.ksosoft.com, ios.wps.cn, mobile-pic.cache.iciba.com, service.iciba.com, client.mail.163.com, c.m.163.com, dsp-impr2.youdao.com, sp.kaola.com, support.you.163.com, agent-count.pconline.com.cn, tqt.weibo.cn, edit.sinaapp.com, wbapp.uve.weibo.com, api.k.sohu.com, api.tv.sohu.com, hui.sohu.com, s1.api.tv.itc.cn, b, api.m.jd.com, ms.jr.jd.com, huichuan.sm.cn, iflow.uczzd.cn, adse.ximalaya.com, app.58.com, aes.acfun.cn, acs.m.taobao.com, dsp.toutiao.com, nnapp.cloudbae.cn, gw.aihuishou.com, m*.amap.com, 7n.bczcdn.com, www.myhug.cn, app.api.ke.com, channel.beitaichufang.com, iapi.bishijie.com, api.intsig.net, cap.caocaokeji.cn, pic1.chelaile.net.cn, m.client.10010.com, www.dandanzan.com, mapi.dangdang.com, api.daydaycook.com.cn, cms.daydaycook.com.cn, www.flyertea.com, cdn.api.fotoable.com, gateway.shouqiev.com, m.ibuscloud.com, smkmp.96225.com, imeclient.openspeech.cn, img.jiemian.com, api.jxedt.com, richmanapi.jxedt.com, api.gotokeep.com, res.kfc.com.cn, api.smzdm.com, api5.futunn.com, qt.qq.com, 3gimg.qq.com, newsso.map.qq.com, vv.video.qq.com, szextshort.weixin.qq.com, y.gtimg.cn, api.gaoqingdianshi.com, pss.txffp.com, app.variflight.com, static.vuevideo.net, api.wallstreetcn.com, app.wy.guahao.com, thor.weidian.com, nochange.ggsafe.com, api-release.wuta-cam.com, res-release.wuta-cam.com, api.xiachufang.com, mapi.mafengwo.cn, mangaapi.manhuaren.com, img.meituan.net, capi.mwee.cn, api.m.mi.com, b-api.ins.miaopai.com, app.mixcapp.com, api.mgzf.com, dili.bdatu.com, wap.ngchina.cn, app3.qdaily.com, notch.qdaily.com, media.qyer.com, api.qiuduoduo.cn, api.rr.tv, api.videozhishi.com, msspjh.emarbox.com, www.shihuo.cn, api.psy-1.com, m.yap.yahoo.com, i.ys7.com, api.catch.gift, a.qiumibao.com, api01pbmp.zhuishushenqi.com, dspsdk.abreader.com, mi.gdt.qq.com, service.4gtv.tv, api.bilibili.com, manga.bilibili.com, sdkapp.uve.weibo.com, app.10086.cn, mobile-api2011.elong.com, foodie-api.yiruikecorp.com, gw-passenger.01zhuanche.com, snailsleep.net, a.sfansclub.com, r.inews.qq.com, mp.weixin.qq.com, cmsapi.wifi8.com, mob.mddcloud.com.cn, p*.meituan.net, api.jr.mi.com, home.mi.com, api-mifit.huami.com, cdn.moji.com, open.qyer.com, portal-xunyou.qingcdn.com, asp.cntv.myalicdn.com, cntv.hls.cdn.myqcloud.com, v.cctv.com, *.kingsoft-office-service.com, oimage*.ydstatic.com, *.58cdn.com.cn, static1.keepcdn.com, adpai.thepaper.cn, images.client.vip.xunlei.com, s3plus.meituan.net, *.iydsj.com, dict-mobile.iciba.com, games.mobileapi.hupu.com, api.kkmh.com
|
||||
@ -1,21 +0,0 @@
|
||||
#!name=[Sukka] URL Rewrite
|
||||
#!desc=Enable this module to use Sukka URL Rewrite rules
|
||||
|
||||
[MITM]
|
||||
|
||||
hostname = %APPEND% api.abema.io, union.click.jd.com, nomo.dafork.com
|
||||
|
||||
[URL Rewrite]
|
||||
# AbeamTV Unlock
|
||||
^https?://api\.abema\.io/v\d/ip/check - reject
|
||||
|
||||
# JD Protection
|
||||
^https?://union\.click\.jd\.com/jda? http://union.click.jd.com/jda?adblock= header
|
||||
^https?://union\.click\.jd\.com/sem.php? http://union.click.jd.com/sem.php?adblock= header
|
||||
|
||||
# Special AD Block Section
|
||||
|
||||
# >> eLong
|
||||
^https?://\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/(adgateway|adv)/ - reject
|
||||
# >> NOMO
|
||||
^https?://nomo.dafork.com/api/v3/iap/ios_product_list https://ruleset.skk.moe/Mock/nomo.json 302
|
||||
@ -1,6 +1,7 @@
|
||||
# $ meta_title Sukka's Ruleset - Reject URL
|
||||
# $ meta_description The ruleset supports AD blocking, tracking protection, privacy protection, anti-phishing, anti-mining
|
||||
# $ meta_description Need Surge Module: https://ruleset.skk.moe/Modules/sukka_mitm_hostnames.sgmodule
|
||||
# $ sgmodule_mitm_hostnames sukka_mitm_hostnames.sgmodule
|
||||
|
||||
# URL-REGEX,^https?://.+\.youtube\.com/api/stats/.+adformat
|
||||
# URL-REGEX,^https?://.+\.youtube\.com/api/stats/ads
|
||||
@ -45,7 +46,7 @@ URL-REGEX,^https://img-ys011\.didistatic\.com/static/ad_oss/
|
||||
|
||||
# >> Hanglvzongheng
|
||||
URL-REGEX,^https?://(discardrp|startup)\.umetrip\.com/gateway/api/umetrip/native
|
||||
URL-REGEX,^https?://(114\.115\.217\.129)|(home\.umetrip\.com)/gateway/api/umetrip/native$
|
||||
URL-REGEX,^https?://(114\.115\.217\.129|home\.umetrip\.com)/gateway/api/umetrip/native$
|
||||
|
||||
# >> iQiyi
|
||||
URL-REGEX,^https?://act\.vip\.iqiyi\.com/interact/api/show\.do
|
||||
@ -176,6 +177,7 @@ URL-REGEX,^https?://api\.daydaycook\.com\.cn/daydaycook/server/ad/
|
||||
URL-REGEX,^https?://cms\.daydaycook\.com\.cn/api/cms/advertisement/
|
||||
# >> eLong
|
||||
URL-REGEX,^https?://mobile-api2011\.elong\.com/(adgateway|adv)/
|
||||
URL-REGEX,^http://\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/(adgateway|adv)/
|
||||
# >> Flyer Tea
|
||||
URL-REGEX,^https?://www\.flyertea\.com/source/plugin/mobile/mobile\.php\?module=advis
|
||||
# >> Foodie
|
||||
@ -340,4 +342,4 @@ URL-REGEX,^https?://dspsdk\.abreader\.com/v\d/api/ad\?
|
||||
# URL-REGEX,^https?://itunes\.apple\.com/lookup\?id=575826903
|
||||
URL-REGEX,^https?://mi\.gdt\.qq\.com/gdt_mview\.fcg
|
||||
# >> Kugou Music
|
||||
URL-REGEX,^https?://\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/(EcomResourceServer/AdPlayPage/adinfo|MobileAdServer/)
|
||||
URL-REGEX,^http://\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/(EcomResourceServer/AdPlayPage/adinfo|MobileAdServer/)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user