GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2025-12-12 01:00:34 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Feat: support reject ip from adguard filter

Browse Source
This commit is contained in:
SukkaW 2025-06-20 16:30:07 +08:00
parent 172e4efd8a
commit 515f262042
5 changed files with 116 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
Build/build-reject-domainset.ts
View File

@ -6,7 +6,7 @@ import { processHostsWithPreload } from './lib/parse-filter/hosts';
import { processDomainListsWithPreload } from './lib/parse-filter/domainlists';
import { processFilterRulesWithPreload } from './lib/parse-filter/filters';
import { HOSTS, ADGUARD_FILTERS, PREDEFINED_WHITELIST, DOMAIN_LISTS, HOSTS_EXTRA, DOMAIN_LISTS_EXTRA, ADGUARD_FILTERS_EXTRA, ADGUARD_FILTERS_WHITELIST, PHISHING_HOSTS_EXTRA, PHISHING_DOMAIN_LISTS_EXTRA } from './constants/reject-data-source';
import { HOSTS, ADGUARD_FILTERS, PREDEFINED_WHITELIST, DOMAIN_LISTS, HOSTS_EXTRA, DOMAIN_LISTS_EXTRA, ADGUARD_FILTERS_EXTRA, ADGUARD_FILTERS_WHITELIST, PHISHING_HOSTS_EXTRA, PHISHING_DOMAIN_LISTS_EXTRA, BOTNET_FILTER, BOGUS_NXDOMAIN_DNSMASQ } from './constants/reject-data-source';
import { readFileIntoProcessedArray } from './lib/fetch-text-by-line';
import { task } from './trace';
// tldts-experimental is way faster than tldts, but very little bit inaccurate
@ -21,10 +21,14 @@ import { foundDebugDomain } from './lib/parse-filter/shared';
import { AdGuardHomeOutput } from './lib/rules/domainset';
import { getPhishingDomains } from './lib/get-phishing-domains';
import type { MaybePromise } from './lib/misc';
import { RulesetOutput } from './lib/rules/ruleset';
import { fetchAssets } from './lib/fetch-assets';
import { AUGUST_ASN, HUIZE_ASN } from '../Source/ip/badboy_asn';
const readLocalRejectDomainsetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/reject.conf'));
const readLocalRejectExtraDomainsetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/reject_extra.conf'));
const readLocalRejectRulesetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/reject.conf'));
const readLocalRejectIpListPromise = readFileIntoProcessedArray(path.resolve(SOURCE_DIR, 'ip/reject.conf'));
const hostsDownloads = HOSTS.map(entry => processHostsWithPreload(...entry));
const hostsExtraDownloads = HOSTS_EXTRA.map(entry => processHostsWithPreload(...entry));
@ -75,6 +79,36 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as
...PHISHING_DOMAIN_LISTS_EXTRA.map(domainList => ` - ${domainList[0]}`)
]);
const rejectIPOutput = new RulesetOutput(span, 'reject', 'ip')
.withTitle('Sukka\'s Ruleset - Anti Bogus Domain')
.withDescription([
...SHARED_DESCRIPTION,
'',
'This file contains known addresses that are hijacking NXDOMAIN results returned by DNS servers, and botnet controller IPs.',
'',
'Data from:',
' - https://github.com/felixonmars/dnsmasq-china-list',
' - https://github.com/curbengh/botnet-filter',
' - And other sources mentioned in /domainset/reject file'
])
.bulkAddIPASN(AUGUST_ASN)
.bulkAddIPASN(HUIZE_ASN);
// Dedupe domainSets (no need to await this)
// Collect DOMAIN, DOMAIN-SUFFIX, and DOMAIN-KEYWORD from non_ip/reject.conf for deduplication
// DOMAIN-WILDCARD is not really useful for deduplication, it is only included in AdGuardHome output
// It is faster to add base than add others first then whitelist
rejectOutput.addFromRuleset(readLocalRejectRulesetPromise);
rejectExtraOutput.addFromRuleset(readLocalRejectRulesetPromise);
rejectOutput.addFromDomainset(readLocalRejectDomainsetPromise);
rejectExtraOutput.addFromDomainset(readLocalRejectDomainsetPromise);
rejectPhisingOutput.addFromDomainset(readLocalRejectDomainsetPromise);
rejectExtraOutput.addFromDomainset(readLocalRejectExtraDomainsetPromise);
rejectIPOutput.addFromRuleset(readLocalRejectIpListPromise);
const appendArrayToRejectOutput = (source: MaybePromise<AsyncIterable<string> | Iterable<string> | string[]>) => rejectOutput.addFromDomainset(source);
const appendArrayToRejectExtraOutput = (source: MaybePromise<AsyncIterable<string> | Iterable<string> | string[]>) => rejectExtraOutput.addFromDomainset(source);
@ -85,18 +119,6 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as
await span
.traceChild('download and process hosts / adblock filter rules')
.traceAsyncFn((childSpan) => Promise.all([
// Dedupe domainSets
// Collect DOMAIN, DOMAIN-SUFFIX, and DOMAIN-KEYWORD from non_ip/reject.conf for deduplication
// DOMAIN-WILDCARD is not really useful for deduplication, it is only included in AdGuardHome output
// It is faster to add base than add others first then whitelist
rejectOutput.addFromRuleset(readLocalRejectRulesetPromise),
rejectExtraOutput.addFromRuleset(readLocalRejectRulesetPromise),
rejectOutput.addFromDomainset(readLocalRejectDomainsetPromise),
rejectExtraOutput.addFromDomainset(readLocalRejectDomainsetPromise),
rejectPhisingOutput.addFromDomainset(readLocalRejectDomainsetPromise),
rejectExtraOutput.addFromDomainset(readLocalRejectExtraDomainsetPromise),
// Parse from remote hosts & domain lists
hostsDownloads.map(task => task(childSpan).then(appendArrayToRejectOutput)),
@ -108,21 +130,23 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as
rejectPhisingOutput.addFromDomainset(getPhishingDomains(childSpan)),
adguardFiltersDownloads.map(
task => task(childSpan).then(({ whiteDomains, whiteDomainSuffixes, blackDomains, blackDomainSuffixes }) => {
task => task(childSpan).then(({ whiteDomains, whiteDomainSuffixes, blackDomains, blackDomainSuffixes, blackIPs }) => {
addArrayElementsToSet(filterRuleWhitelistDomainSets, whiteDomains);
addArrayElementsToSet(filterRuleWhitelistDomainSets, whiteDomainSuffixes, suffix => '.' + suffix);
rejectOutput.bulkAddDomain(blackDomains);
rejectOutput.bulkAddDomainSuffix(blackDomainSuffixes);
rejectIPOutput.bulkAddAnyCIDR(blackIPs, false);
})
),
adguardFiltersExtraDownloads.map(
task => task(childSpan).then(({ whiteDomains, whiteDomainSuffixes, blackDomains, blackDomainSuffixes }) => {
task => task(childSpan).then(({ whiteDomains, whiteDomainSuffixes, blackDomains, blackDomainSuffixes, blackIPs }) => {
addArrayElementsToSet(filterRuleWhitelistDomainSets, whiteDomains);
addArrayElementsToSet(filterRuleWhitelistDomainSets, whiteDomainSuffixes, suffix => '.' + suffix);
rejectExtraOutput.bulkAddDomain(blackDomains);
rejectExtraOutput.bulkAddDomainSuffix(blackDomainSuffixes);
rejectIPOutput.bulkAddAnyCIDR(blackIPs, false);
})
),
adguardFiltersWhitelistsDownloads.map(
@ -132,7 +156,27 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as
addArrayElementsToSet(filterRuleWhitelistDomainSets, blackDomains);
addArrayElementsToSet(filterRuleWhitelistDomainSets, blackDomainSuffixes, suffix => '.' + suffix);
})
)
),
span.traceChildAsync(
'get botnet ips',
() => fetchAssets(...BOTNET_FILTER, true, true)
).then(arr => rejectIPOutput.bulkAddAnyCIDR(arr, false)),
span.traceChildAsync(
'get bogus nxdomain ips',
() => fetchAssets(...BOGUS_NXDOMAIN_DNSMASQ, true, false)
.then(arr => {
for (let i = 0, len = arr.length; i < len; i++) {
const line = arr[i];
if (line.startsWith('bogus-nxdomain=')) {
arr[i] = line.slice(15).trim();
}
}
return arr;
})
// bogus nxdomain needs to be blocked even after resolved
).then(arr => rejectIPOutput.bulkAddAnyCIDR(arr, false))
].flat()));
if (foundDebugDomain.value) {
@ -143,7 +187,8 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as
await Promise.all([
rejectOutput.done(),
rejectExtraOutput.done(),
rejectPhisingOutput.done()
rejectPhisingOutput.done(),
rejectIPOutput.done()
]);
// whitelist
@ -161,7 +206,8 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as
await Promise.all([
rejectOutput.write(),
rejectExtraOutput.write(),
rejectPhisingOutput.write()
rejectPhisingOutput.write(),
rejectIPOutput.write()
]);
// we are going to re-use rejectOutput's domainTrie and mutate it

87
Build/build-reject-ip-list.ts
View File

@ -1,87 +1,12 @@
// @ts-check
import path from 'node:path';
import { readFileIntoProcessedArray } from './lib/fetch-text-by-line';
import { task } from './trace';
import { SHARED_DESCRIPTION } from './constants/description';
import { compareAndWriteFile } from './lib/create-file';
import { OUTPUT_INTERNAL_DIR, SOURCE_DIR } from './constants/dir';
import { fetchAssets } from './lib/fetch-assets';
import { fastIpVersion } from './lib/misc';
import { OUTPUT_INTERNAL_DIR } from './constants/dir';
import { AUGUST_ASN, HUIZE_ASN } from '../Source/ip/badboy_asn';
import { RulesetOutput } from './lib/rules/ruleset';
const getBogusNxDomainIPsPromise: Promise<[ipv4: string[], ipv6: string[]]> = fetchAssets(
'https://cdn.jsdelivr.net/gh/felixonmars/dnsmasq-china-list@master/bogus-nxdomain.china.conf',
['https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/bogus-nxdomain.china.conf'],
true
).then((arr) => {
const ipv4: string[] = [];
const ipv6: string[] = [];
for (let i = 0, len = arr.length; i < len; i++) {
const line = arr[i];
if (line.startsWith('bogus-nxdomain=')) {
const ip = line.slice(15).trim();
const v = fastIpVersion(ip);
if (v === 4) {
ipv4.push(ip);
} else if (v === 6) {
ipv6.push(ip);
}
}
}
return [ipv4, ipv6] as const;
});
const BOTNET_FILTER_URL = 'https://malware-filter.pages.dev/botnet-filter-dnscrypt-blocked-ips.txt';
const BOTNET_FILTER_MIRROR_URL = [
'https://botnet-filter.pages.dev/botnet-filter-dnscrypt-blocked-ips.txt',
'https://malware-filter.gitlab.io/malware-filter/botnet-filter-dnscrypt-blocked-ips.txt',
'https://malware-filter.gitlab.io/botnet-filter/botnet-filter-dnscrypt-blocked-ips.txt'
// 'https://curbengh.github.io/botnet-filter/botnet-filter-dnscrypt-blocked-ips.txt',
// https://curbengh.github.io/malware-filter/botnet-filter-dnscrypt-blocked-ips.txt
];
const getBotNetFilterIPsPromise: Promise<[ipv4: string[], ipv6: string[]]> = fetchAssets(BOTNET_FILTER_URL, BOTNET_FILTER_MIRROR_URL, true, true).then(arr => arr.reduce<[ipv4: string[], ipv6: string[]]>((acc, ip) => {
const v = fastIpVersion(ip);
if (v === 4) {
acc[0].push(ip);
} else if (v === 6) {
acc[1].push(ip);
}
return acc;
}, [[], []]));
const readLocalRejectIpListPromise = readFileIntoProcessedArray(path.resolve(SOURCE_DIR, 'ip/reject.conf'));
export const buildRejectIPList = task(require.main === module, __filename)(async (span) => {
const [bogusNxDomainIPs, botNetIPs] = await Promise.all([
span.traceChildPromise('get bogus nxdomain ips', getBogusNxDomainIPsPromise),
span.traceChildPromise('get botnet ips', getBotNetFilterIPsPromise)
]);
return Promise.all([
new RulesetOutput(span, 'reject', 'ip')
.withTitle('Sukka\'s Ruleset - Anti Bogus Domain')
.withDescription([
...SHARED_DESCRIPTION,
'',
'This file contains known addresses that are hijacking NXDOMAIN results returned by DNS servers, and botnet controller IPs.',
'',
'Data from:',
' - https://github.com/felixonmars/dnsmasq-china-list',
' - https://github.com/curbengh/botnet-filter'
])
.addFromRuleset(readLocalRejectIpListPromise)
.bulkAddCIDR4NoResolve(bogusNxDomainIPs[0])
.bulkAddCIDR6NoResolve(bogusNxDomainIPs[1])
.bulkAddCIDR4NoResolve(botNetIPs[0])
.bulkAddCIDR6NoResolve(botNetIPs[1])
.bulkAddIPASN(AUGUST_ASN)
.bulkAddIPASN(HUIZE_ASN)
.write(),
compareAndWriteFile(span, [AUGUST_ASN.join(' ')], path.join(OUTPUT_INTERNAL_DIR, 'august_asn.txt')),
compareAndWriteFile(span, [HUIZE_ASN.join(' ')], path.join(OUTPUT_INTERNAL_DIR, 'huize_asn.txt'))
]);
});
// Notice: botnet and bogus_nxdomain has been moved to build-reject-domainset
export const buildRejectIPList = task(require.main === module, __filename)(async (span) => Promise.all([
compareAndWriteFile(span, [AUGUST_ASN.join(' ')], path.join(OUTPUT_INTERNAL_DIR, 'august_asn.txt')),
compareAndWriteFile(span, [HUIZE_ASN.join(' ')], path.join(OUTPUT_INTERNAL_DIR, 'huize_asn.txt'))
]));

16
Build/constants/reject-data-source.ts
View File

@ -562,3 +562,19 @@ export const PREDEFINED_WHITELIST = [
'hbbtv.redbutton.de',
'hbbtv.kika.de'
];
export const BOTNET_FILTER = [
'https://malware-filter.pages.dev/botnet-filter-dnscrypt-blocked-ips.txt',
[
'https://botnet-filter.pages.dev/botnet-filter-dnscrypt-blocked-ips.txt',
'https://malware-filter.gitlab.io/malware-filter/botnet-filter-dnscrypt-blocked-ips.txt',
'https://malware-filter.gitlab.io/botnet-filter/botnet-filter-dnscrypt-blocked-ips.txt'
// 'https://curbengh.github.io/botnet-filter/botnet-filter-dnscrypt-blocked-ips.txt',
// https://curbengh.github.io/malware-filter/botnet-filter-dnscrypt-blocked-ips.txt
]
] as const;
export const BOGUS_NXDOMAIN_DNSMASQ = [
'https://cdn.jsdelivr.net/gh/felixonmars/dnsmasq-china-list@master/bogus-nxdomain.china.conf',
['https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/bogus-nxdomain.china.conf']
] as const;

10
Build/lib/parse-filter/filters.ts
View File

@ -28,7 +28,7 @@ export function processFilterRulesWithPreload(
) {
const downloadPromise = fetchAssets(filterRulesUrl, fallbackUrls);
return (span: Span) => span.traceChildAsync<Record<'whiteDomains' | 'whiteDomainSuffixes' | 'blackDomains' | 'blackDomainSuffixes', string[]>>(`process filter rules: ${filterRulesUrl}`, async (span) => {
return (span: Span) => span.traceChildAsync<Record<'whiteDomains' | 'whiteDomainSuffixes' | 'blackDomains' | 'blackDomainSuffixes' | 'blackIPs', string[]>>(`process filter rules: ${filterRulesUrl}`, async (span) => {
const filterRules = await span.traceChildPromise('download', downloadPromise);
const whiteDomains = new Set<string>();
@ -39,6 +39,8 @@ export function processFilterRulesWithPreload(
const warningMessages: string[] = [];
const blackIPs: string[] = [];
const MUTABLE_PARSE_LINE_RESULT: [string, ParseType] = ['', ParseType.NotParsed];
/**
* @param {string} line
@ -78,6 +80,9 @@ export function processFilterRulesWithPreload(
case ParseType.ErrorMessage:
warningMessages.push(hostname);
break;
case ParseType.BlackIP:
blackIPs.push(hostname);
break;
default:
break;
}
@ -107,7 +112,8 @@ export function processFilterRulesWithPreload(
whiteDomains: Array.from(whiteDomains),
whiteDomainSuffixes: Array.from(whiteDomainSuffixes),
blackDomains: Array.from(blackDomains),
blackDomainSuffixes: Array.from(blackDomainSuffixes)
blackDomainSuffixes: Array.from(blackDomainSuffixes),
blackIPs
};
});
}

22
Build/lib/rules/base.ts
View File

@ -1,6 +1,7 @@
import type { Span } from '../../trace';
import { HostnameSmolTrie } from '../trie';
import { not, nullthrow } from 'foxts/guard';
import { fastIpVersion } from '../misc';
import type { MaybePromise } from '../misc';
import type { BaseWriteStrategy } from '../writing-strategy/base';
import { merge as mergeCidr } from 'fast-cidr-tools';
@ -225,6 +226,27 @@ export class FileOutput {
return ip + '/128';
};
bulkAddAnyCIDR(cidrs: string[], noResolve = false) {
const list4 = noResolve ? this.ipcidrNoResolve : this.ipcidr;
const list6 = noResolve ? this.ipcidr6NoResolve : this.ipcidr6;
for (let i = 0, len = cidrs.length; i < len; i++) {
let cidr = cidrs[i];
const version = fastIpVersion(cidr);
if (version === 0) {
continue; // skip invalid IPs
}
cidr = FileOutput.ipToCidr(cidr, version);
if (version === 4) {
list4.add(cidr);
} else /* if (version === 6) */ {
list6.add(cidr);
}
}
return this;
}
bulkAddCIDR4(cidrs: string[]) {
for (let i = 0, len = cidrs.length; i < len; i++) {
this.ipcidr.add(FileOutput.ipToCidr(cidrs[i], 4));