Update Rules

This commit is contained in:
SukkaW 2021-11-29 00:46:48 +08:00
parent 6fade2cba1
commit 58ce1dafd0
6 changed files with 3411 additions and 127 deletions

View File

@ -2,19 +2,42 @@ const { simpleGet } = require('./util-http-get');
const { promises: fsPromises } = require('fs');
const { resolve: pathResolve } = require('path');
let cliProgress;
let Piscina;
try {
Piscina = require('piscina');
cliProgress = require('cli-progress');
} catch (e) {
console.log('Dependencies not found');
console.log('"npm i cli-progress piscina" then try again!');
console.log('"npm i piscina" then try again!');
console.error(e);
process.exit(1);
}
/**
* @param {string | URL} domainListsUrl
*/
async function processDomainLists(domainListsUrl) {
if (typeof domainListsUrl === 'string') {
domainListsUrl = new URL(domainListsUrl);
}
/** @type Set<string> */
const domainSets = new Set();
/** @type string[] */
const domains = (await simpleGet.https(domainListsUrl)).split('\n');
domains.forEach(line => {
if (line.startsWith('#')) {
return;
}
if (line.startsWith(' ') || line === '' || line.startsWith('\r') || line.startsWith('\n')) {
return;
}
domainSets.add(line.trim());
});
return [...domainSets];
}
/**
* @param {string | URL} hostsUrl
*/
@ -29,7 +52,7 @@ async function processHosts(hostsUrl, includeAllSubDomain = false) {
/** @type string[] */
const hosts = (await simpleGet.https(hostsUrl)).split('\n');
hosts.forEach(line => {
if (line.startsWith('#')) {
if (line.includes('#')) {
return;
}
if (line.startsWith(' ') || line === '' || line.startsWith('\r') || line.startsWith('\n')) {
@ -58,8 +81,20 @@ async function processFilterRules(filterRulesUrl) {
/** @type Set<string> */
const whitelistDomainSets = new Set([
'localhost',
'broadcasthost',
'ip6-loopback',
'ip6-localnet',
'ip6-mcastprefix',
'ip6-allnodes',
'ip6-allrouters',
'ip6-allhosts',
'mcastprefix',
'analytics.google.com',
'msa.cdn.mediaset.net' // Added manually using DOMAIN-KEYWORDS
'msa.cdn.mediaset.net', // Added manually using DOMAIN-KEYWORDS
'cloud.answerhub.com',
'ae01.alicdn.com',
'whoami.akamai.net',
'whoami.ds.akahelp.net'
]);
/** @type Set<string> */
const blacklistDomainSets = new Set();
@ -116,7 +151,7 @@ async function processFilterRules(filterRulesUrl) {
/** @type Set<string> */
const domainSets = new Set();
// Parse from remote hosts
// Parse from remote hosts & domain lists
(await Promise.all([
processHosts('https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=1&mimetype=plaintext', true),
processHosts('https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt'),
@ -129,7 +164,8 @@ async function processFilterRules(filterRulesUrl) {
});
});
console.log(`Import ${domainSets.size} rules from hosts files!`);
const hostsSize = domainSets.size;
console.log(`Import ${hostsSize} rules from hosts files!`);
await fsPromises.readFile(pathResolve(__dirname, '../List/domainset/reject_sukka.conf'), { encoding: 'utf-8' }).then(data => {
data.split('\n').forEach(line => {
@ -150,7 +186,8 @@ async function processFilterRules(filterRulesUrl) {
});
});
console.log(`Import rules from reject_sukka.conf!`);
const sukkaSize = domainSets.size - hostsSize;
console.log(`Import ${sukkaSize} rules from reject_sukka.conf!`);
// Parse from AdGuard Filters
/** @type Set<string> */
@ -167,38 +204,54 @@ async function processFilterRules(filterRulesUrl) {
black.forEach(i => domainSets.add(i));
});
console.log(`Import rules from adguard filters!`);
const adguardSize = domainSets.size - hostsSize - sukkaSize;
console.log(`Import ${adguardSize} rules from adguard filters!`);
// Read DOMAIN Keyword
const domainKeywordsSet = new Set();
const domainSuffixSet = new Set();
await fsPromises.readFile(pathResolve(__dirname, '../List/non_ip/reject.conf'), { encoding: 'utf-8' }).then(data => {
data.split('\n').forEach(line => {
if (line.startsWith('DOMAIN-KEYWORD')) {
const [, ...keywords] = line.split(',');
domainKeywordsSet.add(keywords.join(',').trim());
} else if (line.startsWith('DOMAIN-SUFFIX')) {
const [, ...keywords] = line.split(',');
domainSuffixSet.add(keywords.join(',').trim());
}
});
});
console.log(`Import ${domainKeywordsSet.size} black keywords!`);
const beforeDeduping = domainSets.size;
// Dedupe domainSets
console.log(`Start deduping!`);
console.log(`Start deduping! (${beforeDeduping})`);
const piscina = new Piscina({
filename: pathResolve(__dirname, 'worker/build-reject-domainset-worker.js')
});
const res = await Promise.all([
piscina.run({ keywords: domainKeywordsSet, input: domainSets }, { name: 'dedupeKeywords' }),
const res2 = await Promise.all([
piscina.run({ keywords: domainKeywordsSet, suffixes: domainSuffixSet, input: domainSets }, { name: 'dedupeKeywords' }),
piscina.run({ whiteList: filterRuleWhitelistDomainSets, input: domainSets }, { name: 'whitelisted' }),
...sliceIntoChunks(Array.from(domainSets), 5000).map(chunk => piscina.run({ input: chunk, fullSet: domainSets }, { name: 'dedupe' }))
]);
res.forEach(set => {
Array.from(domainSets).reduce((result, element, index) => {
const chunk = index % 12;
result[chunk] = result[chunk] ?? [];
result[chunk].push(element);
return result;
}, []).map(chunk => piscina.run({ input: chunk, fullSet: domainSets }, { name: 'dedupe' }))
]);
res2.forEach(set => {
set.forEach(i => domainSets.delete(i));
});
const diffDeduping = beforeDeduping - domainSets.size;
console.log(`Deduped ${diffDeduping} rules!`);
return fsPromises.writeFile(pathResolve(__dirname, '../List/domainset/reject.conf'), `${[...domainSets].join('\n')}\n`);
})();

View File

@ -1,17 +1,17 @@
exports.dedupe = ({ fullSet, input }) => {
const output = new Set();
for (const domain of input) {
for (const domain2 of fullSet) {
for (const domainFromInput of input) {
for (const domainFromFullSet of fullSet) {
if (
domain2.startsWith('.')
&& domain2 !== domain
domainFromFullSet.startsWith('.')
&& domainFromFullSet !== domainFromInput
&& (
domain.endsWith(domain2)
|| `.${domain}` === domain2
domainFromInput.endsWith(domainFromFullSet)
|| `.${domainFromInput}` === domainFromFullSet
)
) {
output.add(domain);
output.add(domainFromInput);
break;
}
}
@ -35,7 +35,7 @@ exports.whitelisted = ({ whiteList, input }) => {
return output;
};
exports.dedupeKeywords = ({ keywords, input }) => {
exports.dedupeKeywords = ({ keywords, suffixes, input }) => {
const output = new Set();
for (const domain of input) {
@ -45,6 +45,12 @@ exports.dedupeKeywords = ({ keywords, input }) => {
break;
}
}
for (const suffix of suffixes) {
if (domain.endsWith(suffix)) {
output.add(domain);
break;
}
}
}
return output;

View File

@ -56,6 +56,7 @@ platform.twitter.com
.ampproject.org
# >> Google CDN
dl.google.com
fonts.googleapis.com
fonts.gstatic.com
ajax.googleapis.com
@ -129,7 +130,7 @@ img.vim-cn.com
# >> General CDN
# AWS S3
s3.amazonaws.com
.s3.amazonaws.com
# WordPress
.files.wordpress.com
# Instagram
@ -142,6 +143,8 @@ cdn.sstatic.net
.fonts.net
# Naver
.pstatic.net
# Steam
.steamstatic.com
# Others
.v2ex.co
cdn.v2ex.com

File diff suppressed because it is too large Load Diff

View File

@ -365,7 +365,6 @@ xy-log.tagtic.cn
.vrtcal.com
.webspectator.com
.xadchina.cn
.xdrig.com
.yldbt.com
.ymtracking.com
.adwo.com
@ -388,6 +387,7 @@ ad.huajiao.com
ad.hzyoka.com
ad.jiemian.com
ad.qingting.fm
.qtad.qingting.fm
adunion.wifi8.com
adfile.wifi8.com
ad.yun.wifi8.com
@ -396,7 +396,6 @@ adsapi.manhuaren.com
adsdk.dmzj.com
adshows.21cn.com
adui.tg.meitu.com
adx.hupu.com
.ad.cmvideo.cn
app-ad.variflight.com
applog.mobike.com
@ -412,7 +411,6 @@ mopnativeadv.037201.com
.jiguang.cn
.jpush.io
static.vuevideo.net
stats.jpush.cn
stats.umsns.com
trackapp.guahao.cn
traffic.mogujie.com
@ -424,6 +422,10 @@ ssp.api.tappx.com
.zynga.com
.zyngaplayersupport.com
.adx.io
advertpay-vip-ssl.xunlei.com
.advertising.speed.cdn.vip.xunlei.com
.advertpay.vip.xunlei.com
.youmi.net
.adcolony.com
.adpro.cn
@ -514,12 +516,18 @@ adcloud.jp
.districtm.io
.buysellads.net
.ad.wx.com
.ad.weixin.qq.com
.guomob.com
.ad1.udn.com
.ad2.udn.com
.ad3.udn.com
.ad4.udn.com
.ad5.udn.com
# >> Tracking
tracking.yorg.app
stats.wp.com
data.alexa.com
device-metrics-us.amazon.com
device-metrics-us-2.amazon.com
js.monitor.azure.com
.cloudflareinsights.com
.histats.com
@ -559,6 +567,7 @@ analytics.slashdotmedia.com
.bkrtx.com
.ads.pro-market.net
.stat.xtom.com
.analytics.archive.org
.adjust.io
.airbrake.io
@ -575,7 +584,6 @@ analytics.slashdotmedia.com
.do-not-tracker.org
.eviltracker.net
.getexceptional.com
.inmobi.com
.jumptap.com
.localytics.com
.mixpanel.com
@ -588,6 +596,15 @@ analytics.slashdotmedia.com
.usergrid.com
.vungle.com
.telemetry.mozilla.org
.union.6.cn
api-analytics-cn.huami.com
# >> Alexa
data.alexa.com
device-metrics-us.amazon.com
device-metrics-us-2.amazon.com
# >> Apple
metrics.apple.com
securemetrics.apple.com
@ -633,7 +650,6 @@ corp.sts.microsoft.com
diagnostics.support.microsoft.com
i1.services.social.microsoft.com
cache.datamart.windows.com
diagnostics.support.microsoft.com
spynet2.microsoft.com
spynetalt.microsoft.com
.msads.net
@ -669,6 +685,7 @@ logservice.hicloud.com
logservice1.hicloud.com
# >> JD
jrad.jd.com
.union.jd.com
c-nfa.jd.com
cps.360buy.com
@ -912,9 +929,18 @@ news.qhstatic.com
show.look.360.cn
.dftoutiao.com
new.taobc.com
s.360.cn
.s.360.cn
mclean.lato.cloud.360safe.com
mvconf.lato.cloud.360safe.com
mclean.cloud.360safe.com
mvconf.cloud.360safe.com
mclean.uk.cloud.360safe.com
mvconf.uk.cloud.360safe.com
aicleaner.shouji.360.cn
ssl.360antivirus.org
ad.360in.com
# >> Youku
# >> Youku + Tudou
.atm.youku.com
actives.youku.com
ad.api.3g.youku.com
@ -961,6 +987,12 @@ wan.youku.com
ykatr.youku.com
ykrec.youku.com
iwstat.tudou.com
nstat.tudou.com
adplay.tudou.com
stats.tudou.com
.ad.cmvideo.cn
# >> Netease
ad.bn.netease.com
ad.yixin.im
@ -1008,8 +1040,11 @@ log.tbs.qq.com
.mpush.qq.com
.push.qq.com
pgdt.gtimg.cn
pgdt.ugdtimg.com
union.gdtimg.com
.bugly.qq.com
.gdt.qq.com
.adnet.qq.com
.rqd.qq.com
oth.eve.mdt.qq.com
oth.str.mdt.qq.com
@ -1030,6 +1065,11 @@ splashqqlive.gtimg.com
access-sngapm.qcloud.com
beacon.gtimg.com
.l.qq.com
ad.tencentmusic.com
adstats.tencentmusic.com
adcdn.tencentmusic.com
adexpo.tencentmusic.com
adclick.tencentmusic.com
# >> Sina
ad4.sina.com.cn
@ -1130,6 +1170,7 @@ cdn.media.innity.net
ssl-cdn.media.innity.net
# >> iQiyi
.cupid.ptqy.gitv.tv
a.ckm.iqiyi.com
ad.m.iqiyi.com
afp.iqiyi.com
@ -1143,13 +1184,14 @@ msg.video.qiyi.com
msg2.video.qiyi.com
policy.video.iqiyi.com
yuedu.iqiyi.com
cupid.iqiyi.com
.cupid.iqiyi.com
ifacelog.iqiyi.com
mbdlog.iqiyi.com
msg.71.am
msg.qy.net
# >> Kugou and Kuwo
ads.bssdl.kugou.com
adserviceretry.kugou.com
ads.service.kugou.com
adsfile.bssdlbig.kugou.com
@ -1234,10 +1276,9 @@ pro.letv.com
stat.letv.com
static.app.m.letv.com
# >> Xiaomi
# >> Xiaomi (MIUI)
ad.intl.xiaomi.com
.ad.mi.com
.ad.xiaomi.com
.admob.xiaomi.com
ad1.xiaomi.com
mistat.xiaomi.com
@ -1249,11 +1290,23 @@ adeng.hpplay.cn
.kuyun.com
.ad.xiaomi.com
.ad.intl.xiaomi.com
.data.mistat.xiaomi.com
adv.sec.miui.com
o2o.api.xiaomi.com
api.miui.security.xiaomi.com
.auth.be.sec.miui.com
.adfuture.cn
sa.api.intl.miui.com
.data.mistat.xiaomi.com
.data.mistat.intl.xiaomi.com
.data.mistat.india.xiaomi.com
.data.mistat.rus.xiaomi.com
.tracking.miui.com
.tracking.intl.miui.com
.tracking.india.miui.com
.tracking.rus.miui.com
.xlmc.sec.miui.com
wtradv.market.xiaomi.com
video.market.xiaomi.com
.pandora.xiaomi.com
# >> PCOnline
ad-analysis.pconline.com.cn
@ -1271,12 +1324,18 @@ cdn.adapi.fotoable.com
ads.aplus.pptv.com
asimgs.pplive.cn
de.as.pptv.com
ads.data.pplive.com
gas.data.pplive.com
plt.data.pplive.com
web.data.pplive.com
.game.pps.tv
# >> ele
log.star.ele.me
# >> HunanTV
.da.mgtv.com
.log.mgtv.com
.da.hunantv.com
.log.hunantv.com
log.v2.hunantv.com
@ -1366,12 +1425,26 @@ galaxy.bjcathay.com
mdrecv.app.cntvwb.cn
sdapprecv.app.cntvwb.cn
vdapprecv.app.cntvwb.cn
.ad.cctv.com
# >> Kingsoft
ad-stat.ksosoft.com
img.auction-ads.wpscdn.cn
minfo.wps.cn
# >> FunShion
adm.funshion.com
conf.funshion.com
vs.funshion.com
rt.funshion.net
stat.funshion.net
# >> Hupu
adx.hupu.com
shence.hupu.com
ccdace.hupu.com
hd.hupu.com
# >> Tonghuahshun
adm.10jqka.com.cn
stat.10jqka.com.cn
@ -1412,6 +1485,16 @@ disqusads.com
# >> XiMaLaYa
ad.ximalaya.com
adse.ximalaya.com
adse.wsa.ximalaya.com
adsebs.ximalaya.com
adsehera.ximalaya.com
adbehavior.ximalaya.com
adwbs.ximalaya.com
adbs.ximalaya.com
dns.ximalaya.com
mobwsa.ximalaya.com
location.ximalaya.com
xdcs-collector.ximalaya.com
# Roku
logs.roku.com
@ -1455,28 +1538,28 @@ ntp.msn.cn
assets.msn.cn
api.msn.com
browser.events.data.msn.com
#edge.activity.windows.com
img-s-msn-com.akamaized.net
# >> MIUI
.adfuture.cn
sa.api.intl.miui.com
.data.mistat.xiaomi.com
.data.mistat.intl.xiaomi.com
.data.mistat.india.xiaomi.com
.data.mistat.rus.xiaomi.com
.tracking.miui.com
.tracking.intl.miui.com
.tracking.india.miui.com
.tracking.rus.miui.com
.xlmc.sec.miui.com
wtradv.market.xiaomi.com
.pandora.xiaomi.com
# >> OPPO
adsfs.oppomobile.com
.ads.oppomobile.com
.adsfs-sdkconfig.heytapimage.com
httpdns.push.oppomobile.com
# >> VIVO
adlog.vivo.com.cn
adreq.vivo.com.cn
adsdk.vivo.com.cn
adsstatic.vivo.com.cn
adxlog.vivo.com.cn
stnetsdk.appstore.vivo.com.cn
monitor-stsdk.vivo.com.cn
onrt-stsdk.vivo.com.cn
ort-stsdk.vivo.com.cn
pnrt-stsdk.vivo.com.cn
prt-stsdk.vivo.com.cn
stnetsdk.vivo.com.cn
.stsdk.vivo.com.cn
# >> 勾正数据
.gz-data.com

View File

@ -30,7 +30,18 @@ DOMAIN-KEYWORD,adserver
DOMAIN-KEYWORD,adspace
DOMAIN-KEYWORD,advertmarket
DOMAIN-KEYWORD,adsyndication
DOMAIN-KEYWORD,doubleclick
DOMAIN-KEYWORD,doubleclick.
DOMAIN-KEYWORD,adjust.
DOMAIN-KEYWORD,appsflyer
# Important: Force add the following domains without whitelisting
DOMAIN-SUFFIX,openx.net
DOMAIN-SUFFIX,mmstat.com
DOMAIN-SUFFIX,amplitude.com
DOMAIN-KEYWORD,advertising.com
# Tencent AD KEYWORD
DOMAIN-KEYWORD,wxsnsdy
DOMAIN-KEYWORD,bahoom,REJECT
DOMAIN,daisydiskapp.com,REJECT