From 515f26204261a27361d1aec5bca2beae3608af48 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Fri, 20 Jun 2025 16:30:07 +0800 Subject: [PATCH] Feat: support reject ip from adguard filter --- Build/build-reject-domainset.ts | 82 +++++++++++++++++++------ Build/build-reject-ip-list.ts | 87 ++------------------------- Build/constants/reject-data-source.ts | 16 +++++ Build/lib/parse-filter/filters.ts | 10 ++- Build/lib/rules/base.ts | 22 +++++++ 5 files changed, 116 insertions(+), 101 deletions(-) diff --git a/Build/build-reject-domainset.ts b/Build/build-reject-domainset.ts index 37b94bb3..f61e4b23 100644 --- a/Build/build-reject-domainset.ts +++ b/Build/build-reject-domainset.ts @@ -6,7 +6,7 @@ import { processHostsWithPreload } from './lib/parse-filter/hosts'; import { processDomainListsWithPreload } from './lib/parse-filter/domainlists'; import { processFilterRulesWithPreload } from './lib/parse-filter/filters'; -import { HOSTS, ADGUARD_FILTERS, PREDEFINED_WHITELIST, DOMAIN_LISTS, HOSTS_EXTRA, DOMAIN_LISTS_EXTRA, ADGUARD_FILTERS_EXTRA, ADGUARD_FILTERS_WHITELIST, PHISHING_HOSTS_EXTRA, PHISHING_DOMAIN_LISTS_EXTRA } from './constants/reject-data-source'; +import { HOSTS, ADGUARD_FILTERS, PREDEFINED_WHITELIST, DOMAIN_LISTS, HOSTS_EXTRA, DOMAIN_LISTS_EXTRA, ADGUARD_FILTERS_EXTRA, ADGUARD_FILTERS_WHITELIST, PHISHING_HOSTS_EXTRA, PHISHING_DOMAIN_LISTS_EXTRA, BOTNET_FILTER, BOGUS_NXDOMAIN_DNSMASQ } from './constants/reject-data-source'; import { readFileIntoProcessedArray } from './lib/fetch-text-by-line'; import { task } from './trace'; // tldts-experimental is way faster than tldts, but very little bit inaccurate @@ -21,10 +21,14 @@ import { foundDebugDomain } from './lib/parse-filter/shared'; import { AdGuardHomeOutput } from './lib/rules/domainset'; import { getPhishingDomains } from './lib/get-phishing-domains'; import type { MaybePromise } from './lib/misc'; +import { RulesetOutput } from './lib/rules/ruleset'; +import { fetchAssets } from './lib/fetch-assets'; +import { AUGUST_ASN, HUIZE_ASN } from '../Source/ip/badboy_asn'; const readLocalRejectDomainsetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/reject.conf')); const readLocalRejectExtraDomainsetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/reject_extra.conf')); const readLocalRejectRulesetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/reject.conf')); +const readLocalRejectIpListPromise = readFileIntoProcessedArray(path.resolve(SOURCE_DIR, 'ip/reject.conf')); const hostsDownloads = HOSTS.map(entry => processHostsWithPreload(...entry)); const hostsExtraDownloads = HOSTS_EXTRA.map(entry => processHostsWithPreload(...entry)); @@ -75,6 +79,36 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as ...PHISHING_DOMAIN_LISTS_EXTRA.map(domainList => ` - ${domainList[0]}`) ]); + const rejectIPOutput = new RulesetOutput(span, 'reject', 'ip') + .withTitle('Sukka\'s Ruleset - Anti Bogus Domain') + .withDescription([ + ...SHARED_DESCRIPTION, + '', + 'This file contains known addresses that are hijacking NXDOMAIN results returned by DNS servers, and botnet controller IPs.', + '', + 'Data from:', + ' - https://github.com/felixonmars/dnsmasq-china-list', + ' - https://github.com/curbengh/botnet-filter', + ' - And other sources mentioned in /domainset/reject file' + ]) + .bulkAddIPASN(AUGUST_ASN) + .bulkAddIPASN(HUIZE_ASN); + + // Dedupe domainSets (no need to await this) + // Collect DOMAIN, DOMAIN-SUFFIX, and DOMAIN-KEYWORD from non_ip/reject.conf for deduplication + // DOMAIN-WILDCARD is not really useful for deduplication, it is only included in AdGuardHome output + // It is faster to add base than add others first then whitelist + rejectOutput.addFromRuleset(readLocalRejectRulesetPromise); + rejectExtraOutput.addFromRuleset(readLocalRejectRulesetPromise); + + rejectOutput.addFromDomainset(readLocalRejectDomainsetPromise); + rejectExtraOutput.addFromDomainset(readLocalRejectDomainsetPromise); + rejectPhisingOutput.addFromDomainset(readLocalRejectDomainsetPromise); + + rejectExtraOutput.addFromDomainset(readLocalRejectExtraDomainsetPromise); + + rejectIPOutput.addFromRuleset(readLocalRejectIpListPromise); + const appendArrayToRejectOutput = (source: MaybePromise | Iterable | string[]>) => rejectOutput.addFromDomainset(source); const appendArrayToRejectExtraOutput = (source: MaybePromise | Iterable | string[]>) => rejectExtraOutput.addFromDomainset(source); @@ -85,18 +119,6 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as await span .traceChild('download and process hosts / adblock filter rules') .traceAsyncFn((childSpan) => Promise.all([ - // Dedupe domainSets - // Collect DOMAIN, DOMAIN-SUFFIX, and DOMAIN-KEYWORD from non_ip/reject.conf for deduplication - // DOMAIN-WILDCARD is not really useful for deduplication, it is only included in AdGuardHome output - // It is faster to add base than add others first then whitelist - rejectOutput.addFromRuleset(readLocalRejectRulesetPromise), - rejectExtraOutput.addFromRuleset(readLocalRejectRulesetPromise), - - rejectOutput.addFromDomainset(readLocalRejectDomainsetPromise), - rejectExtraOutput.addFromDomainset(readLocalRejectDomainsetPromise), - rejectPhisingOutput.addFromDomainset(readLocalRejectDomainsetPromise), - - rejectExtraOutput.addFromDomainset(readLocalRejectExtraDomainsetPromise), // Parse from remote hosts & domain lists hostsDownloads.map(task => task(childSpan).then(appendArrayToRejectOutput)), @@ -108,21 +130,23 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as rejectPhisingOutput.addFromDomainset(getPhishingDomains(childSpan)), adguardFiltersDownloads.map( - task => task(childSpan).then(({ whiteDomains, whiteDomainSuffixes, blackDomains, blackDomainSuffixes }) => { + task => task(childSpan).then(({ whiteDomains, whiteDomainSuffixes, blackDomains, blackDomainSuffixes, blackIPs }) => { addArrayElementsToSet(filterRuleWhitelistDomainSets, whiteDomains); addArrayElementsToSet(filterRuleWhitelistDomainSets, whiteDomainSuffixes, suffix => '.' + suffix); rejectOutput.bulkAddDomain(blackDomains); rejectOutput.bulkAddDomainSuffix(blackDomainSuffixes); + rejectIPOutput.bulkAddAnyCIDR(blackIPs, false); }) ), adguardFiltersExtraDownloads.map( - task => task(childSpan).then(({ whiteDomains, whiteDomainSuffixes, blackDomains, blackDomainSuffixes }) => { + task => task(childSpan).then(({ whiteDomains, whiteDomainSuffixes, blackDomains, blackDomainSuffixes, blackIPs }) => { addArrayElementsToSet(filterRuleWhitelistDomainSets, whiteDomains); addArrayElementsToSet(filterRuleWhitelistDomainSets, whiteDomainSuffixes, suffix => '.' + suffix); rejectExtraOutput.bulkAddDomain(blackDomains); rejectExtraOutput.bulkAddDomainSuffix(blackDomainSuffixes); + rejectIPOutput.bulkAddAnyCIDR(blackIPs, false); }) ), adguardFiltersWhitelistsDownloads.map( @@ -132,7 +156,27 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as addArrayElementsToSet(filterRuleWhitelistDomainSets, blackDomains); addArrayElementsToSet(filterRuleWhitelistDomainSets, blackDomainSuffixes, suffix => '.' + suffix); }) - ) + ), + + span.traceChildAsync( + 'get botnet ips', + () => fetchAssets(...BOTNET_FILTER, true, true) + ).then(arr => rejectIPOutput.bulkAddAnyCIDR(arr, false)), + span.traceChildAsync( + 'get bogus nxdomain ips', + () => fetchAssets(...BOGUS_NXDOMAIN_DNSMASQ, true, false) + .then(arr => { + for (let i = 0, len = arr.length; i < len; i++) { + const line = arr[i]; + if (line.startsWith('bogus-nxdomain=')) { + arr[i] = line.slice(15).trim(); + } + } + + return arr; + }) + // bogus nxdomain needs to be blocked even after resolved + ).then(arr => rejectIPOutput.bulkAddAnyCIDR(arr, false)) ].flat())); if (foundDebugDomain.value) { @@ -143,7 +187,8 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as await Promise.all([ rejectOutput.done(), rejectExtraOutput.done(), - rejectPhisingOutput.done() + rejectPhisingOutput.done(), + rejectIPOutput.done() ]); // whitelist @@ -161,7 +206,8 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as await Promise.all([ rejectOutput.write(), rejectExtraOutput.write(), - rejectPhisingOutput.write() + rejectPhisingOutput.write(), + rejectIPOutput.write() ]); // we are going to re-use rejectOutput's domainTrie and mutate it diff --git a/Build/build-reject-ip-list.ts b/Build/build-reject-ip-list.ts index 2708cee2..536872cd 100644 --- a/Build/build-reject-ip-list.ts +++ b/Build/build-reject-ip-list.ts @@ -1,87 +1,12 @@ // @ts-check import path from 'node:path'; -import { readFileIntoProcessedArray } from './lib/fetch-text-by-line'; import { task } from './trace'; -import { SHARED_DESCRIPTION } from './constants/description'; import { compareAndWriteFile } from './lib/create-file'; -import { OUTPUT_INTERNAL_DIR, SOURCE_DIR } from './constants/dir'; -import { fetchAssets } from './lib/fetch-assets'; -import { fastIpVersion } from './lib/misc'; +import { OUTPUT_INTERNAL_DIR } from './constants/dir'; import { AUGUST_ASN, HUIZE_ASN } from '../Source/ip/badboy_asn'; -import { RulesetOutput } from './lib/rules/ruleset'; -const getBogusNxDomainIPsPromise: Promise<[ipv4: string[], ipv6: string[]]> = fetchAssets( - 'https://cdn.jsdelivr.net/gh/felixonmars/dnsmasq-china-list@master/bogus-nxdomain.china.conf', - ['https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/bogus-nxdomain.china.conf'], - true -).then((arr) => { - const ipv4: string[] = []; - const ipv6: string[] = []; - - for (let i = 0, len = arr.length; i < len; i++) { - const line = arr[i]; - if (line.startsWith('bogus-nxdomain=')) { - const ip = line.slice(15).trim(); - const v = fastIpVersion(ip); - if (v === 4) { - ipv4.push(ip); - } else if (v === 6) { - ipv6.push(ip); - } - } - } - - return [ipv4, ipv6] as const; -}); - -const BOTNET_FILTER_URL = 'https://malware-filter.pages.dev/botnet-filter-dnscrypt-blocked-ips.txt'; -const BOTNET_FILTER_MIRROR_URL = [ - 'https://botnet-filter.pages.dev/botnet-filter-dnscrypt-blocked-ips.txt', - 'https://malware-filter.gitlab.io/malware-filter/botnet-filter-dnscrypt-blocked-ips.txt', - 'https://malware-filter.gitlab.io/botnet-filter/botnet-filter-dnscrypt-blocked-ips.txt' - // 'https://curbengh.github.io/botnet-filter/botnet-filter-dnscrypt-blocked-ips.txt', - // https://curbengh.github.io/malware-filter/botnet-filter-dnscrypt-blocked-ips.txt -]; - -const getBotNetFilterIPsPromise: Promise<[ipv4: string[], ipv6: string[]]> = fetchAssets(BOTNET_FILTER_URL, BOTNET_FILTER_MIRROR_URL, true, true).then(arr => arr.reduce<[ipv4: string[], ipv6: string[]]>((acc, ip) => { - const v = fastIpVersion(ip); - if (v === 4) { - acc[0].push(ip); - } else if (v === 6) { - acc[1].push(ip); - } - return acc; -}, [[], []])); - -const readLocalRejectIpListPromise = readFileIntoProcessedArray(path.resolve(SOURCE_DIR, 'ip/reject.conf')); - -export const buildRejectIPList = task(require.main === module, __filename)(async (span) => { - const [bogusNxDomainIPs, botNetIPs] = await Promise.all([ - span.traceChildPromise('get bogus nxdomain ips', getBogusNxDomainIPsPromise), - span.traceChildPromise('get botnet ips', getBotNetFilterIPsPromise) - ]); - - return Promise.all([ - new RulesetOutput(span, 'reject', 'ip') - .withTitle('Sukka\'s Ruleset - Anti Bogus Domain') - .withDescription([ - ...SHARED_DESCRIPTION, - '', - 'This file contains known addresses that are hijacking NXDOMAIN results returned by DNS servers, and botnet controller IPs.', - '', - 'Data from:', - ' - https://github.com/felixonmars/dnsmasq-china-list', - ' - https://github.com/curbengh/botnet-filter' - ]) - .addFromRuleset(readLocalRejectIpListPromise) - .bulkAddCIDR4NoResolve(bogusNxDomainIPs[0]) - .bulkAddCIDR6NoResolve(bogusNxDomainIPs[1]) - .bulkAddCIDR4NoResolve(botNetIPs[0]) - .bulkAddCIDR6NoResolve(botNetIPs[1]) - .bulkAddIPASN(AUGUST_ASN) - .bulkAddIPASN(HUIZE_ASN) - .write(), - compareAndWriteFile(span, [AUGUST_ASN.join(' ')], path.join(OUTPUT_INTERNAL_DIR, 'august_asn.txt')), - compareAndWriteFile(span, [HUIZE_ASN.join(' ')], path.join(OUTPUT_INTERNAL_DIR, 'huize_asn.txt')) - ]); -}); +// Notice: botnet and bogus_nxdomain has been moved to build-reject-domainset +export const buildRejectIPList = task(require.main === module, __filename)(async (span) => Promise.all([ + compareAndWriteFile(span, [AUGUST_ASN.join(' ')], path.join(OUTPUT_INTERNAL_DIR, 'august_asn.txt')), + compareAndWriteFile(span, [HUIZE_ASN.join(' ')], path.join(OUTPUT_INTERNAL_DIR, 'huize_asn.txt')) +])); diff --git a/Build/constants/reject-data-source.ts b/Build/constants/reject-data-source.ts index f05918a2..0b9f801e 100644 --- a/Build/constants/reject-data-source.ts +++ b/Build/constants/reject-data-source.ts @@ -562,3 +562,19 @@ export const PREDEFINED_WHITELIST = [ 'hbbtv.redbutton.de', 'hbbtv.kika.de' ]; + +export const BOTNET_FILTER = [ + 'https://malware-filter.pages.dev/botnet-filter-dnscrypt-blocked-ips.txt', + [ + 'https://botnet-filter.pages.dev/botnet-filter-dnscrypt-blocked-ips.txt', + 'https://malware-filter.gitlab.io/malware-filter/botnet-filter-dnscrypt-blocked-ips.txt', + 'https://malware-filter.gitlab.io/botnet-filter/botnet-filter-dnscrypt-blocked-ips.txt' + // 'https://curbengh.github.io/botnet-filter/botnet-filter-dnscrypt-blocked-ips.txt', + // https://curbengh.github.io/malware-filter/botnet-filter-dnscrypt-blocked-ips.txt + ] +] as const; + +export const BOGUS_NXDOMAIN_DNSMASQ = [ + 'https://cdn.jsdelivr.net/gh/felixonmars/dnsmasq-china-list@master/bogus-nxdomain.china.conf', + ['https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/bogus-nxdomain.china.conf'] +] as const; diff --git a/Build/lib/parse-filter/filters.ts b/Build/lib/parse-filter/filters.ts index 7d7071f3..decfd3c3 100644 --- a/Build/lib/parse-filter/filters.ts +++ b/Build/lib/parse-filter/filters.ts @@ -28,7 +28,7 @@ export function processFilterRulesWithPreload( ) { const downloadPromise = fetchAssets(filterRulesUrl, fallbackUrls); - return (span: Span) => span.traceChildAsync>(`process filter rules: ${filterRulesUrl}`, async (span) => { + return (span: Span) => span.traceChildAsync>(`process filter rules: ${filterRulesUrl}`, async (span) => { const filterRules = await span.traceChildPromise('download', downloadPromise); const whiteDomains = new Set(); @@ -39,6 +39,8 @@ export function processFilterRulesWithPreload( const warningMessages: string[] = []; + const blackIPs: string[] = []; + const MUTABLE_PARSE_LINE_RESULT: [string, ParseType] = ['', ParseType.NotParsed]; /** * @param {string} line @@ -78,6 +80,9 @@ export function processFilterRulesWithPreload( case ParseType.ErrorMessage: warningMessages.push(hostname); break; + case ParseType.BlackIP: + blackIPs.push(hostname); + break; default: break; } @@ -107,7 +112,8 @@ export function processFilterRulesWithPreload( whiteDomains: Array.from(whiteDomains), whiteDomainSuffixes: Array.from(whiteDomainSuffixes), blackDomains: Array.from(blackDomains), - blackDomainSuffixes: Array.from(blackDomainSuffixes) + blackDomainSuffixes: Array.from(blackDomainSuffixes), + blackIPs }; }); } diff --git a/Build/lib/rules/base.ts b/Build/lib/rules/base.ts index 08bd0add..5d68dc50 100644 --- a/Build/lib/rules/base.ts +++ b/Build/lib/rules/base.ts @@ -1,6 +1,7 @@ import type { Span } from '../../trace'; import { HostnameSmolTrie } from '../trie'; import { not, nullthrow } from 'foxts/guard'; +import { fastIpVersion } from '../misc'; import type { MaybePromise } from '../misc'; import type { BaseWriteStrategy } from '../writing-strategy/base'; import { merge as mergeCidr } from 'fast-cidr-tools'; @@ -225,6 +226,27 @@ export class FileOutput { return ip + '/128'; }; + bulkAddAnyCIDR(cidrs: string[], noResolve = false) { + const list4 = noResolve ? this.ipcidrNoResolve : this.ipcidr; + const list6 = noResolve ? this.ipcidr6NoResolve : this.ipcidr6; + + for (let i = 0, len = cidrs.length; i < len; i++) { + let cidr = cidrs[i]; + const version = fastIpVersion(cidr); + if (version === 0) { + continue; // skip invalid IPs + } + cidr = FileOutput.ipToCidr(cidr, version); + + if (version === 4) { + list4.add(cidr); + } else /* if (version === 6) */ { + list6.add(cidr); + } + } + return this; + } + bulkAddCIDR4(cidrs: string[]) { for (let i = 0, len = cidrs.length; i < len; i++) { this.ipcidr.add(FileOutput.ipToCidr(cidrs[i], 4));