Update Rules

This commit is contained in:
SukkaW 2021-11-22 23:40:06 +08:00
parent 25b04ce4b5
commit 369dc87e92
4 changed files with 66 additions and 71 deletions

View File

@ -34,7 +34,11 @@ async function processHosts(hostsUrl, includeAllSubDomain = false) {
return; return;
} }
const [, ...domains] = line.split(' '); const [, ...domains] = line.split(' ');
domainSets.add(`${includeAllSubDomain ? '.' : ''}${domains.join(' ')}`); if (includeAllSubDomain) {
domainSets.add(`.${domains.join(' ')}`.trim());
} else {
domainSets.add(domains.join(' ').trim());
}
}); });
return [...domainSets]; return [...domainSets];
@ -42,7 +46,7 @@ async function processHosts(hostsUrl, includeAllSubDomain = false) {
/** /**
* @param {string | URL} filterRulesUrl * @param {string | URL} filterRulesUrl
* @returns {Promise<{ white: string[], black: string[] }>} * @returns {Promise<{ white: Set<string>, black: Set<string> }>}
*/ */
async function processFilterRules(filterRulesUrl) { async function processFilterRules(filterRulesUrl) {
if (typeof filterRulesUrl === 'string') { if (typeof filterRulesUrl === 'string') {
@ -50,31 +54,37 @@ async function processFilterRules(filterRulesUrl) {
} }
/** @type Set<string> */ /** @type Set<string> */
const whitelistDomainSets = new Set(); const whitelistDomainSets = new Set(['localhost', 'analytics.google.com']);
/** @type Set<string> */ /** @type Set<string> */
const blacklistDomainSets = new Set(); const blacklistDomainSets = new Set();
/** @type string[] */ /** @type string[] */
const filterRules = (await simpleGet.https(filterRulesUrl.hostname, filterRulesUrl.pathname)).split('\n'); const filterRules = (await simpleGet.https(filterRulesUrl.hostname, filterRulesUrl.pathname)).split('\n');
filterRules.forEach(line => { filterRules.forEach(line => {
if (line.startsWith('#') || line.startsWith('!')) { if (
line.startsWith('#')
|| line.startsWith('!')
|| line.startsWith(' ')
|| line === ''
|| line.startsWith('\r')
|| line.startsWith('\n')
|| line.includes('*')
|| line.includes('/')
|| line.includes('$')
) {
return; return;
} }
if (line.startsWith(' ') || line === '' || line.startsWith('\r') || line.startsWith('\n')) {
return; if (line.startsWith('@@||') && line.endsWith('^')) {
} whitelistDomainSets.add(`${line.replaceAll('@@||', '').replaceAll('^', '')}`.trim());
if (!line.includes('*') && !line.includes('//')) { } else if (line.startsWith('||') && line.endsWith('^')) {
if (line.startsWith('@@||') && line.endsWith('^')) { blacklistDomainSets.add(`${line.replaceAll('||', '').replaceAll('^', '')}`.trim());
whitelistDomainSets.add(`${line.replaceAll('@@||', '').replaceAll('^', '')}`);
} else if (line.startsWith('||') && line.endsWith('^')) {
blacklistDomainSets.add(`${line.replaceAll('||', '').replaceAll('^', '')}`);
}
} }
}); });
return { return {
white: [...whitelistDomainSets], white: whitelistDomainSets,
black: [...blacklistDomainSets] black: blacklistDomainSets
}; };
} }
@ -91,21 +101,19 @@ async function processFilterRules(filterRulesUrl) {
])).forEach(hosts => { ])).forEach(hosts => {
hosts.forEach(host => { hosts.forEach(host => {
if (host) { if (host) {
domainSets.add(host); domainSets.add(host.trim());
} }
}); });
}); });
console.log(`Import ${domainSets.size} rules from hosts files!`); console.log(`Import ${domainSets.size} rules from hosts files!`);
console.log(`Start importing rules from reject_sukka.conf!`);
await fsPromises.readFile(pathResolve(__dirname, '../List/domainset/reject_sukka.conf'), { encoding: 'utf-8' }).then(data => { await fsPromises.readFile(pathResolve(__dirname, '../List/domainset/reject_sukka.conf'), { encoding: 'utf-8' }).then(data => {
data.split('\n').forEach(line => { data.split('\n').forEach(line => {
if ( if (
line.startsWith('#') line.startsWith('#')
|| line.startsWith(' ') || line.startsWith(' ')
|| line === '' || line === '' || line === ' '
|| line.startsWith('\r') || line.startsWith('\r')
|| line.startsWith('\n') || line.startsWith('\n')
) { ) {
@ -115,10 +123,12 @@ async function processFilterRules(filterRulesUrl) {
/* if (domainSets.has(line) || domainSets.has(`.${line}`)) { /* if (domainSets.has(line) || domainSets.has(`.${line}`)) {
console.warn(`|${line}| is already in the list!`); console.warn(`|${line}| is already in the list!`);
} */ } */
domainSets.add(line); domainSets.add(line.trim());
}); });
}); });
console.log(`Import rules from reject_sukka.conf!`);
// Parse from AdGuard Filters // Parse from AdGuard Filters
/** @type Set<string> */ /** @type Set<string> */
const filterRuleWhitelistDomainSets = new Set(); const filterRuleWhitelistDomainSets = new Set();
@ -140,7 +150,8 @@ async function processFilterRules(filterRulesUrl) {
domainSets.add(`.${black}`); domainSets.add(`.${black}`);
} }
console.log(`Import ${filterRuleBlacklistDomainSets.size} rules from adguard filters!`); console.log(`Import ${filterRuleBlacklistDomainSets.size} black rules from adguard filters!`);
console.log(`Import ${filterRuleWhitelistDomainSets.size} white rules from adguard filters!`);
// Read DOMAIN Keyword // Read DOMAIN Keyword
const domainKeywordsSet = new Set(); const domainKeywordsSet = new Set();
@ -148,20 +159,22 @@ async function processFilterRules(filterRulesUrl) {
data.split('\n').forEach(line => { data.split('\n').forEach(line => {
if (line.startsWith('DOMAIN-KEYWORD')) { if (line.startsWith('DOMAIN-KEYWORD')) {
const [, ...keywords] = line.split(','); const [, ...keywords] = line.split(',');
domainKeywordsSet.add(keywords.join(',')); domainKeywordsSet.add(keywords.join(',').trim());
} }
}); });
}); });
console.log(`Import ${domainKeywordsSet.size} black keywords!`);
// Dedupe domainSets // Dedupe domainSets
console.log(`Start deduping!`); console.log(`Start deduping!`);
const bar2 = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic); const bar2 = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);
const domainSetsClone = [...domainSets]; bar2.start(domainSets.size, 0);
const len = domainSetsClone.length;
bar2.start(len, 0);
for (const domain of domainSets) { for (const domain of domainSets) {
bar2.increment();
let shouldContinue = false; let shouldContinue = false;
for (const white of filterRuleWhitelistDomainSets) { for (const white of filterRuleWhitelistDomainSets) {
@ -189,13 +202,18 @@ async function processFilterRules(filterRulesUrl) {
} }
for (const domain2 of domainSets) { for (const domain2 of domainSets) {
if (domain2.startsWith('.') && domain2 !== domain && (domain.endsWith(domain2) || `.${domain}` === domain2)) { if (
domain2.startsWith('.')
&& domain2 !== domain
&& (
domain.endsWith(domain2)
|| `.${domain}` === domain2
)
) {
domainSets.delete(domain); domainSets.delete(domain);
break; break;
} }
} }
bar2.increment();
} }
bar2.stop(); bar2.stop();

View File

@ -900,7 +900,6 @@
.analytics.emarketer.com .analytics.emarketer.com
.analytics.ext.go-tellm.com .analytics.ext.go-tellm.com
.analytics.fkz.re .analytics.fkz.re
.analytics.google.com
.analytics.htmedia.in .analytics.htmedia.in
.analytics.icons8.com .analytics.icons8.com
.analytics.inlinemanual.com .analytics.inlinemanual.com
@ -2491,6 +2490,7 @@
.plista.com .plista.com
.plotrabbit.com .plotrabbit.com
.ployea.com .ployea.com
.pltraffic8.com
.pluckypocket.com .pluckypocket.com
.plugrush.com .plugrush.com
.pocketmath.com .pocketmath.com
@ -3670,22 +3670,12 @@ sunnimiq6.cf
www.besocial.online www.besocial.online
srcips.com srcips.com
gxbrowser.net gxbrowser.net
localhost
ip6-localhost
ip6-loopback ip6-loopback
0aqpqdju.me 0aqpqdju.me
1100.adsina.allyes.com
1148.adsina.allyes.com
114.allyes.com
114so.cn 114so.cn
1.21shebao.com 1.21shebao.com
123.sogou.com 123.sogou.com
1251.adsina.allyes.com
1276.adsina.allyes.com
144.dragonparking.com 144.dragonparking.com
154.adsina.allyes.com
161.adsina.allyes.com
17173im.allyes.com
18av.mm-cg.co 18av.mm-cg.co
1.allyes.com.cn 1.allyes.com.cn
1.nanrenwo.net 1.nanrenwo.net
@ -3699,14 +3689,11 @@ ip6-loopback
3dns-2.adobe.com 3dns-2.adobe.com
3dns-3.adobe.com 3dns-3.adobe.com
3rd.t.sohu.com 3rd.t.sohu.com
44.adsina.allyes.com
45.adsina.allyes.com
46sg.com 46sg.com
526dimg.uunice.com 526dimg.uunice.com
526d.uunice.com 526d.uunice.com
58lm.vip 58lm.vip
5vz3cfs0yd.me 5vz3cfs0yd.me
60.adsina.allyes.com
6tsbe1zs.me 6tsbe1zs.me
801.tianya.cn 801.tianya.cn
803.tianya.cn 803.tianya.cn
@ -3718,7 +3705,6 @@ ip6-loopback
888.tv.sohu.com 888.tv.sohu.com
930.dragonparking.com 930.dragonparking.com
a.19869.com a.19869.com
a3.allyes.com
a.ads2.msn.com a.ads2.msn.com
a.alimama.cn a.alimama.cn
aa.zldh123.com aa.zldh123.com
@ -3813,7 +3799,6 @@ ads.google.com
adsgroup.qq.com adsgroup.qq.com
adshmct.qq.com adshmct.qq.com
adshmmsg.qq.com adshmmsg.qq.com
adsina.allyes.com
adslvfile.qq.com adslvfile.qq.com
adslvseed.qq.com adslvseed.qq.com
ad.sohu.com ad.sohu.com
@ -3846,7 +3831,6 @@ ad.ximalaya.com
adx.pro.cn adx.pro.cn
adx-static.ksosoft.com adx-static.ksosoft.com
ad.xxguan.cn ad.xxguan.cn
adzjvnet.allyes.com
ad.zuimeitianqi.com ad.zuimeitianqi.com
ae.bdstatic.com ae.bdstatic.com
aecpm.alicdn.com aecpm.alicdn.com
@ -3866,9 +3850,6 @@ alipay.dajiadou6.com
alissl.ucdl.pp.uc.cn alissl.ucdl.pp.uc.cn
aliunion.cn.yahoo.com aliunion.cn.yahoo.com
allnews.uodoo.com allnews.uodoo.com
allyesbjafa.allyes.com
allyes.com
allyesshafa.allyes.com
a.mct01.com a.mct01.com
am.g.ireader.com am.g.ireader.com
amoeba.fudata.cn amoeba.fudata.cn
@ -4000,7 +3981,6 @@ blogad02.myweb.hinet.net
bm.alimama.cn bm.alimama.cn
bmvip.alimama.cn bmvip.alimama.cn
bobo.163.com bobo.163.com
bokee.allyes.com
breeze.olclient.baofeng.com breeze.olclient.baofeng.com
bro.flyme.cn bro.flyme.cn
btlaunch.baidu.com btlaunch.baidu.com
@ -4260,9 +4240,7 @@ dxp.baidu.com
d.yoyi.tv d.yoyi.tv
dzl.baidu.com dzl.baidu.com
e.28487.net e.28487.net
e7free.allyes.com
e.admin60.com e.admin60.com
eastmoney.allyes.com
eavs02.eqifa.com eavs02.eqifa.com
ebook.res.meizu.com ebook.res.meizu.com
e.breezily168.com e.breezily168.com
@ -4321,7 +4299,6 @@ fge9vbrzwt.bid
fgmtv.org fgmtv.org
files2.sogou.com files2.sogou.com
fm.qzone.qq.com fm.qzone.qq.com
focusbaiduafp.allyes.com
fodder.qq.com fodder.qq.com
fodder.tc.qq.com fodder.tc.qq.com
fota4.adups.cn fota4.adups.cn
@ -4422,7 +4399,6 @@ iad.g.163.com
ias816.ujikdd041o.cn ias816.ujikdd041o.cn
i.bestcontentfare.top i.bestcontentfare.top
i.clkservice.youdao.com i.clkservice.youdao.com
idigger.allyes.com
idm.bce.baidu.com idm.bce.baidu.com
idm-su.baidu.com idm-su.baidu.com
idx.m.hub.sandai.net idx.m.hub.sandai.net
@ -4586,7 +4562,6 @@ ldpgl.code.mytanwan.com
ldy.350.com ldy.350.com
leak.360.cn leak.360.cn
ledou.dl.uu.cc ledou.dl.uu.cc
letv.allyes.com
links.services.disqus.com links.services.disqus.com
listlog.baofeng.net listlog.baofeng.net
livep.l.ott.video.qq.com livep.l.ott.video.qq.com
@ -4611,7 +4586,6 @@ log.houyi.baofeng.net
log.kuwo.cn log.kuwo.cn
log.ltype.baofeng.com log.ltype.baofeng.com
log.music.baidu.com log.music.baidu.com
log.music.baidu.com
log.nvwa.baofeng.com log.nvwa.baofeng.com
log.p2p.hunantv.com log.p2p.hunantv.com
log.rc.hunantv.com log.rc.hunantv.com
@ -4726,7 +4700,6 @@ msg.shouji.360.cn
m.simba.taobao.com m.simba.taobao.com
msite.baidu.com msite.baidu.com
m.sjzhushou.com m.sjzhushou.com
mso.allyes.com
msoftdl.360.cn msoftdl.360.cn
m.symab.cn m.symab.cn
m.tixing51.net m.tixing51.net
@ -4916,7 +4889,6 @@ py2.qlogo.cn
py.qlogo.cn py.qlogo.cn
qhl.bealge.sogou.com qhl.bealge.sogou.com
qianclick.baidu.com qianclick.baidu.com
qianclick.baidu.com
qqshow2-item.qq.com qqshow2-item.qq.com
qss-client.qq.com qss-client.qq.com
qt002x.corp.youdao.com qt002x.corp.youdao.com
@ -5070,8 +5042,6 @@ shrek.6.cn
simaba.taobao.com simaba.taobao.com
simba.m.taobao.com simba.m.taobao.com
s.img.mix.sina.com.cn s.img.mix.sina.com.cn
sina.allyes.com
sinas.allyes.com
sina.yinstar.org sina.yinstar.org
s.jandan.com s.jandan.com
s.jlminte.com s.jlminte.com
@ -5079,9 +5049,6 @@ s.l8l9.com
slides.discovery.tom.com slides.discovery.tom.com
slog.sina.cn slog.sina.cn
slog.sina.com.cn slog.sina.com.cn
sm51shai.allyes.com
smcreative.allyes.com
smjs.allyes.com
sobar.baidu.com sobar.baidu.com
sobartop.baidu.com sobartop.baidu.com
soft.data.weather.360.cn soft.data.weather.360.cn
@ -5098,7 +5065,6 @@ sqm.telemetry.microsoft.com.nsatc.net
s.qqhpg.com s.qqhpg.com
s.qtad.qingting.fm s.qtad.qingting.fm
srd.simba.taobao.com srd.simba.taobao.com
sroomafp.allyes.com
ssac.suning.com ssac.suning.com
ss.cnczjy.com ss.cnczjy.com
ss.cnnic.cn ss.cnnic.cn
@ -5237,7 +5203,6 @@ tj.vidown.cn
tk.baidu.com tk.baidu.com
tkweb.baidu.com tkweb.baidu.com
tob-cms.bj.bcebos.com tob-cms.bj.bcebos.com
tom.allyes.com
tongji.meizu.com tongji.meizu.com
tongji-res1.meizu.com tongji-res1.meizu.com
tongji.tom.com tongji.tom.com
@ -5265,7 +5230,6 @@ tvupgrade.yunos.com
tw13b093.sandai.net tw13b093.sandai.net
tw.alimama.cn tw.alimama.cn
tw.fgmtv.org tw.fgmtv.org
twsina.allyes.com
txtad.jijiplayer.com txtad.jijiplayer.com
txt.go.sohu.com txt.go.sohu.com
tz.1688988.com tz.1688988.com
@ -5401,7 +5365,6 @@ www.706529.com
www.716703.com www.716703.com
www.aboluowang.com www.aboluowang.com
www.admarket.mobi www.admarket.mobi
www.allyes.com
www.appuu.cn www.appuu.cn
www.baidushifen.com www.baidushifen.com
www.baiduzhidahao.cc www.baiduzhidahao.cc
@ -5509,7 +5472,6 @@ xz-development.oss-cn-beijing.aliyuncs.com
yads.c.yimg.jp yads.c.yimg.jp
yam.adsbro.com yam.adsbro.com
yellowto.com yellowto.com
yeskyafp.allyes.com
yiliao.hupan.com yiliao.hupan.com
ylog.hiido.com ylog.hiido.com
youjia2016.com youjia2016.com
@ -5531,11 +5493,9 @@ zhengwunet.org
zhihu-web-analytics.zhihu.com zhihu-web-analytics.zhihu.com
zhuichaguoji.org zhuichaguoji.org
zhwnlapi.etouch.cn zhwnlapi.etouch.cn
zjvnet.allyes.com
znsv.baidu.com znsv.baidu.com
zymo.mps.weibo.com zymo.mps.weibo.com
zzy1.quyaoya.com zzy1.quyaoya.com
localhost
crash.163.com crash.163.com
ads.1mobile.com ads.1mobile.com
api4.1mobile.com api4.1mobile.com
@ -11647,6 +11607,7 @@ file.apicvn.com
.coostuni.com .coostuni.com
.cootewie.com .cootewie.com
.coothupu.net .coothupu.net
.cootro.com
.coovouch.com .coovouch.com
.copacet.com .copacet.com
.coperledsinhe.info .coperledsinhe.info
@ -11701,6 +11662,7 @@ file.apicvn.com
.cpalabtracking.com .cpalabtracking.com
.cpamatik.com .cpamatik.com
.cpanuk.com .cpanuk.com
.cpaoffers.network
.cpaprrrrrofit.com .cpaprrrrrofit.com
.cpaway.com .cpaway.com
.cpcmart.com .cpcmart.com
@ -12785,6 +12747,7 @@ file.apicvn.com
.ejoonsoo.net .ejoonsoo.net
.ejurzsax.com .ejurzsax.com
.ekansovi.com .ekansovi.com
.ekfottltamc.com
.ekmas.com .ekmas.com
.eknowin.club .eknowin.club
.ekoatchooze.com .ekoatchooze.com
@ -20114,6 +20077,7 @@ file.apicvn.com
.snarecathedrallipstick.com .snarecathedrallipstick.com
.snarlleadenpsychology.com .snarlleadenpsychology.com
.sneakystamp.com .sneakystamp.com
.snegol.com
.snezynrzsz.com .snezynrzsz.com
.sngakyf.com .sngakyf.com
.snickgainfulmuch.com .snickgainfulmuch.com
@ -20936,6 +20900,7 @@ file.apicvn.com
.tintregionalretire.com .tintregionalretire.com
.tintslingpromote.com .tintslingpromote.com
.tionedwhen.biz .tionedwhen.biz
.tioniaukmlas.one
.tionmodat.club .tionmodat.club
.tionmustai.fun .tionmustai.fun
.tirusoge.com .tirusoge.com
@ -21282,6 +21247,7 @@ file.apicvn.com
.tyrvictions.xyz .tyrvictions.xyz
.tyshochime.com .tyshochime.com
.tysicyse.com .tysicyse.com
.tysolomona.co
.tytheglupy.com .tytheglupy.com
.tywzyhfliwdbu.com .tywzyhfliwdbu.com
.tzlchxdwreirbv.com .tzlchxdwreirbv.com
@ -22354,6 +22320,7 @@ file.apicvn.com
.wuckaity.com .wuckaity.com
.wudr.net .wudr.net
.wufel.ml .wufel.ml
.wunteetoco.com
.wurtoosy.com .wurtoosy.com
.wussucko.com .wussucko.com
.wusths.com .wusths.com
@ -23277,6 +23244,7 @@ file.apicvn.com
.ddvfoj5yrl2oi.cloudfront.net .ddvfoj5yrl2oi.cloudfront.net
.dew9ckzjyt2gn.cloudfront.net .dew9ckzjyt2gn.cloudfront.net
.dgw7ae5vrovs7.cloudfront.net .dgw7ae5vrovs7.cloudfront.net
.dhrhzii89gpwo.cloudfront.net
.dihutyaiafuhr.cloudfront.net .dihutyaiafuhr.cloudfront.net
.dita6jhhqwoiz.cloudfront.net .dita6jhhqwoiz.cloudfront.net
.djm080u34wfc5.cloudfront.net .djm080u34wfc5.cloudfront.net
@ -23437,7 +23405,6 @@ file.apicvn.com
.x0r.urlgalleries.net .x0r.urlgalleries.net
.yotta.scrolller.com .yotta.scrolller.com
.ytre9jk.txxx.com .ytre9jk.txxx.com
.tysolomona.co
.aggingleag.one .aggingleag.one
.oweqas.xyz .oweqas.xyz
.gyrivehmic.com .gyrivehmic.com

View File

@ -548,7 +548,6 @@ beacon.wikia-services.com
.analytics.live.com .analytics.live.com
.gridsumdissector.com .gridsumdissector.com
.gridsum.com .gridsum.com
.gridsum.com
.growingio.com .growingio.com
.gtags.net .gtags.net
.statcounter.com .statcounter.com

View File

@ -31,6 +31,17 @@ DOMAIN-KEYWORD,adspace
DOMAIN-KEYWORD,advertmarket DOMAIN-KEYWORD,advertmarket
DOMAIN-KEYWORD,adsyndication DOMAIN-KEYWORD,adsyndication
DOMAIN-KEYWORD,bahoom,REJECT
DOMAIN,daisydiskapp.com,REJECT
# >> Tencent Lemon
PROCESS-NAME,Tencent Lemon,REJECT
PROCESS-NAME,LemonMonitor,REJECT
PROCESS-NAME,LemonDaemon,REJECT
PROCESS-NAME,LemonAgent,REJECT
PROCESS-NAME,LemonService,REJECT
# >> Google # >> Google
DOMAIN-KEYWORD,adsense DOMAIN-KEYWORD,adsense
DOMAIN-KEYWORD,adwords DOMAIN-KEYWORD,adwords