GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2026-01-29 01:51:52 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Update Rules

Browse Source
This commit is contained in:
SukkaW
2022-04-17 23:16:33 +08:00
parent 42302b0310
commit 00d9bf8023
3 changed files with 11277 additions and 5254 deletions
Download Patch File Download Diff File Expand all files Collapse all files

168
Build/build-reject-domainset.js
View File

@@ -2,10 +2,10 @@ const { default: got } = require('got-cjs');
const { promises: fsPromises } = require('fs'); const { promises: fsPromises } = require('fs');
const { resolve: pathResolve } = require('path'); const { resolve: pathResolve } = require('path');
const { cpus } = require('os'); const { cpus } = require('os');
const { isIP } = require('net');
const threads = Math.max(cpus().length, 12); const threads = Math.max(cpus().length, 12);
const rIPv4 = /((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/;
const rDomain = /^(((?!\-))(xn\-\-)?[a-z0-9\-_]{0,61}[a-z0-9]{1,1}\.)*(xn\-\-)?([a-z0-9\-]{1,61}|[a-z0-9\-]{1,30})\.[a-z]{2,}$/m const rDomain = /^(((?!\-))(xn\-\-)?[a-z0-9\-_]{0,61}[a-z0-9]{1,1}\.)*(xn\-\-)?([a-z0-9\-]{1,61}|[a-z0-9\-]{1,30})\.[a-z]{2,}$/m
const Piscina = require('piscina'); const Piscina = require('piscina');
@@ -83,53 +83,23 @@ async function processFilterRules(filterRulesUrl) {
} }
/** @type Set<string> */ /** @type Set<string> */
const whitelistDomainSets = new Set([ const whitelistDomainSets = new Set();
'localhost',
'broadcasthost',
'ip6-loopback',
'ip6-localnet',
'ip6-mcastprefix',
'ip6-allnodes',
'ip6-allrouters',
'ip6-allhosts',
'mcastprefix',
'analytics.google.com',
'msa.cdn.mediaset.net', // Added manually using DOMAIN-KEYWORDS
'cloud.answerhub.com',
'ae01.alicdn.com',
'whoami.akamai.net',
'whoami.ds.akahelp.net',
'pxlk9.net.', // This one is malformed from EasyList, which I will manually add instead
'instant.page', // No, it doesn't violate anyone's privacy. I will whitelist it
'piwik.pro',
'mixpanel.com',
'heapanalytics.com',
'dataunlocker.com',
'segment.com',
'segment.io',
'segmentify.com'
]);
/** @type Set<string> */ /** @type Set<string> */
const blacklistDomainSets = new Set(); const blacklistDomainSets = new Set();
/** @type Set<string> */
const blackIPSets = new Set();
/** @type string[] */ /** @type string[] */
const filterRules = (await got(filterRulesUrl).text()).split('\n'); const filterRules = (await got(filterRulesUrl).text()).split('\n').map(line => line.trim());
filterRules.forEach(line => { filterRules.forEach(line => {
if ( if (
line.includes('#') line === ''
|| line.includes('#')
|| line.includes('!') || line.includes('!')
|| line.startsWith(' ')
|| line.startsWith('\r')
|| line.startsWith('\n')
|| line.includes('*') || line.includes('*')
|| line.includes('/') || line.includes('/')
|| line.includes('$') || line.includes('$') && !line.startsWith('@@')
|| line.trim() === '' || line.trim() === ''
|| rIPv4.test(line) || isIP(line) !== 0
) { ) {
return; return;
} }
@@ -138,9 +108,17 @@ async function processFilterRules(filterRulesUrl) {
&& ( && (
line.endsWith('^') line.endsWith('^')
|| line.endsWith('^|') || line.endsWith('^|')
|| line.endsWith('^$badfilter')
|| line.endsWith('^$1p')
) )
) { ) {
const domain = `${line.replaceAll('@@||', '').replaceAll('^|', '').replaceAll('^', '')}`.trim(); const domain = line
.replaceAll('@@||', '')
.replaceAll('^$badfilter', '')
.replaceAll('^$1p', '')
.replaceAll('^|', '')
.replaceAll('^', '')
.trim();
if (rDomain.test(domain)) { if (rDomain.test(domain)) {
whitelistDomainSets.add(domain); whitelistDomainSets.add(domain);
} }
@@ -198,9 +176,9 @@ async function processFilterRules(filterRulesUrl) {
if ( if (
line.startsWith('#') line.startsWith('#')
|| line.startsWith(' ') || line.startsWith(' ')
|| line === '' || line === ' '
|| line.startsWith('\r') || line.startsWith('\r')
|| line.startsWith('\n') || line.startsWith('\n')
|| line.trim() === ''
) { ) {
return; return;
} }
@@ -217,23 +195,74 @@ async function processFilterRules(filterRulesUrl) {
// Parse from AdGuard Filters // Parse from AdGuard Filters
/** @type Set<string> */ /** @type Set<string> */
const filterRuleWhitelistDomainSets = new Set(); const filterRuleWhitelistDomainSets = new Set([
'localhost',
'broadcasthost',
'ip6-loopback',
'ip6-localnet',
'ip6-mcastprefix',
'ip6-allnodes',
'ip6-allrouters',
'ip6-allhosts',
'mcastprefix',
'analytics.google.com',
'msa.cdn.mediaset.net', // Added manually using DOMAIN-KEYWORDS
'cloud.answerhub.com',
'ae01.alicdn.com',
'whoami.akamai.net',
'whoami.ds.akahelp.net',
'pxlk9.net.', // This one is malformed from EasyList, which I will manually add instead
'instant.page', // No, it doesn't violate anyone's privacy. I will whitelist it
'piwik.pro',
'mixpanel.com',
'heapanalytics.com',
'dataunlocker.com',
'segment.com',
'segment.io',
'segmentify.com'
]);
(await Promise.all([ (await Promise.all([
processFilterRules('https://easylist.to/easylist/easylist.txt'), // Easy List
processFilterRules('https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt'), 'https://easylist.to/easylist/easylist.txt',
processFilterRules('https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_11_Mobile/filter.txt'), // AdGuard DNS Filter
processFilterRules('https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_3_Spyware/filter.txt'), 'https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt',
processFilterRules('https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_2_English/filter.txt'), // uBlock Origin Filter List
processFilterRules('https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_224_Chinese/filter.txt'), 'https://ublockorigin.github.io/uAssetsCDN/filters/filters.txt',
processFilterRules('https://filters.adtidy.org/extension/ublock/filters/224.txt'), 'https://ublockorigin.github.io/uAssetsCDN/filters/filters-2020.txt',
processFilterRules('https://easylist.to/easylist/easyprivacy.txt'), 'https://ublockorigin.github.io/uAssetsCDN/filters/filters-2021.txt',
processFilterRules('https://raw.githubusercontent.com/DandelionSprout/adfilt/master/GameConsoleAdblockList.txt'), 'https://ublockorigin.github.io/uAssetsCDN/filters/filters-2022.txt',
processFilterRules('https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/SmartTV-AGH.txt'), // uBlock Origin Badware Risk List
processFilterRules('https://curben.gitlab.io/malware-filter/urlhaus-filter-agh-online.txt'), 'https://ublockorigin.github.io/uAssets/filters/badware.txt',
processFilterRules('https://curben.gitlab.io/malware-filter/pup-filter-agh.txt'), // uBlock Origin Privacy List
processFilterRules('https://curben.gitlab.io/malware-filter/phishing-filter-agh.txt'), 'https://ublockorigin.github.io/uAssets/filters/privacy.txt',
processFilterRules('https://curben.gitlab.io/malware-filter/pup-filter-agh.txt') // uBlock Origin Resource Abuse
])).forEach(({ white, black }) => { 'https://ublockorigin.github.io/uAssets/filters/resource-abuse.txt',
// uBlock Origin Unbreak
'https://ublockorigin.github.io/uAssets/filters/unbreak.txt',
// AdGuard Base Filter
'https://filters.adtidy.org/extension/ublock/filters/2_without_easylist.txt',
// AdGuard Mobile AD
'https://filters.adtidy.org/extension/ublock/filters/11.txt',
// AdGuard Tracking Protection
'https://filters.adtidy.org/extension/ublock/filters/3.txt',
// AdGuard Japanese filter
'https://filters.adtidy.org/extension/ublock/filters/7.txt',
// AdGuard Chinese filter (EasyList China + AdGuard Chinese filter)
'https://filters.adtidy.org/extension/ublock/filters/224.txt',
// Easy Privacy
'https://easylist.to/easylist/easyprivacy.txt',
// Curben's Malware Online UrlHaus
'https://curben.gitlab.io/malware-filter/urlhaus-filter-agh-online.txt',
// Curben's Phishing Online Filter
'https://curben.gitlab.io/malware-filter/phishing-filter-agh.txt',
// Curben's PUP List
'https://curben.gitlab.io/malware-filter/pup-filter-agh.txt',
// GameConsoleAdblockList
'https://raw.githubusercontent.com/DandelionSprout/adfilt/master/GameConsoleAdblockList.txt',
// PiHoleBlocklist
'https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/SmartTV-AGH.txt',
].map(processFilterRules))).forEach(({ white, black }) => {
white.forEach(i => filterRuleWhitelistDomainSets.add(i)); white.forEach(i => filterRuleWhitelistDomainSets.add(i));
black.forEach(i => domainSets.add(i)); black.forEach(i => domainSets.add(i));
}); });
@@ -256,7 +285,7 @@ async function processFilterRules(filterRulesUrl) {
}); });
}); });
console.log(`Import ${domainKeywordsSet.size} black keywords!`); console.log(`Import ${domainKeywordsSet.size} black keywords and ${domainSuffixSet.size} black suffixes!`);
const beforeDeduping = domainSets.size; const beforeDeduping = domainSets.size;
// Dedupe domainSets // Dedupe domainSets
@@ -267,8 +296,14 @@ async function processFilterRules(filterRulesUrl) {
}); });
(await Promise.all([ (await Promise.all([
piscina.run({ keywords: domainKeywordsSet, suffixes: domainSuffixSet, input: domainSets }, { name: 'dedupeKeywords' }), piscina.run(
piscina.run({ whiteList: filterRuleWhitelistDomainSets, input: domainSets }, { name: 'whitelisted' }) { keywords: domainKeywordsSet, suffixes: domainSuffixSet, input: domainSets },
{ name: 'dedupeKeywords' }
),
piscina.run(
{ whiteList: filterRuleWhitelistDomainSets, input: domainSets },
{ name: 'whitelisted' }
)
])).forEach(set => { ])).forEach(set => {
set.forEach(i => domainSets.delete(i)); set.forEach(i => domainSets.delete(i));
}); });
@@ -276,13 +311,18 @@ async function processFilterRules(filterRulesUrl) {
const originalFullSet = new Set([...domainSets]); const originalFullSet = new Set([...domainSets]);
(await Promise.all( (await Promise.all(
Array.from(domainSets).reduce((result, element, index) => { Array.from(domainSets)
const chunk = index % threads; .reduce((result, element, index) => {
result[chunk] ??= []; const chunk = index % threads;
result[chunk] ??= [];
result[chunk].push(element); result[chunk].push(element);
return result; return result;
}, []).map(chunk => piscina.run({ input: chunk, fullSet: originalFullSet }, { name: 'dedupe' })) }, [])
.map(chunk => piscina.run(
{ input: chunk, fullSet: originalFullSet },
{ name: 'dedupe' }
))
)).forEach(set => { )).forEach(set => {
set.forEach(i => domainSets.delete(i)); set.forEach(i => domainSets.delete(i));
}); });

16355
List/domainset/reject.conf
View File

File diff suppressed because it is too large Load Diff

8
List/domainset/reject_sukka.conf
View File

@@ -629,7 +629,6 @@ analytics.slashdotmedia.com
.eviltracker.net .eviltracker.net
.getexceptional.com .getexceptional.com
.jumptap.com .jumptap.com
.localytics.com
.mobile-collector.newrelic.com .mobile-collector.newrelic.com
.playtomic.com .playtomic.com
.stathat.com .stathat.com
@@ -1527,7 +1526,6 @@ gemini.yahoo.com
ysm.yahoo.com ysm.yahoo.com
yads.c.yimg.jp yads.c.yimg.jp
yads.yahoo.co.jp yads.yahoo.co.jp
.flurry.com
.flurry.cachefly.net .flurry.cachefly.net
# >> Disqus # >> Disqus
@@ -1628,9 +1626,9 @@ stnetsdk.vivo.com.cn
.stsdk.vivo.com.cn .stsdk.vivo.com.cn
# >> Gionee # >> Gionee
0.0.0.0 ads.gionee.com ads.gionee.com
0.0.0.0 pdl.gionee.com pdl.gionee.com
0.0.0.0 adres.myaora.net adres.myaora.net
# >> 勾正数据 # >> 勾正数据
.gz-data.com .gz-data.com