diff --git a/Build/build-reject-domainset.ts b/Build/build-reject-domainset.ts index 4217b1b4..a6dc4edb 100644 --- a/Build/build-reject-domainset.ts +++ b/Build/build-reject-domainset.ts @@ -46,7 +46,7 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as DOMAIN_LISTS_EXTRA.map(entry => processDomainLists(childSpan, ...entry).then(appendArrayToDomainSetsExtra)), ADGUARD_FILTERS.map( - input => processFilterRules(childSpan, ...input) + entry => processFilterRules(childSpan, ...entry) .then(({ white, black, foundDebugDomain }) => { if (foundDebugDomain) { // eslint-disable-next-line sukka/no-single-return -- not single return @@ -58,7 +58,7 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as }) ), ADGUARD_FILTERS_EXTRA.map( - input => processFilterRules(childSpan, ...input) + entry => processFilterRules(childSpan, ...entry) .then(({ white, black, foundDebugDomain }) => { if (foundDebugDomain) { // eslint-disable-next-line sukka/no-single-return -- not single return diff --git a/Build/constants/reject-data-source.ts b/Build/constants/reject-data-source.ts index b8b89c9b..8361fe2d 100644 --- a/Build/constants/reject-data-source.ts +++ b/Build/constants/reject-data-source.ts @@ -56,8 +56,6 @@ export const DOMAIN_LISTS_EXTRA: HostsSource[] = [ ['https://raw.githubusercontent.com/AdguardTeam/cname-trackers/master/data/combined_disguised_clickthroughs_justdomains.txt', [], true, TTL.THREE_DAYS()], ['https://raw.githubusercontent.com/AdguardTeam/cname-trackers/master/data/combined_disguised_microsites_justdomains.txt', [], true, TTL.THREE_DAYS()], ['https://raw.githubusercontent.com/AdguardTeam/cname-trackers/master/data/combined_disguised_mail_trackers_justdomains.txt', [], true, TTL.THREE_DAYS()], - // AdGuard Cookie Notices - ['https://filters.adtidy.org/extension/ublock/filters/18_optimized.txt', [], true, TTL.THREE_HOURS()], // Curben's PUP Domains Blocklist // 'https://curbengh.github.io/pup-filter/pup-filter-agh.txt' // 'https://pup-filter.pages.dev/pup-filter-agh.txt' @@ -154,6 +152,8 @@ export const ADGUARD_FILTERS: AdGuardFilterSource[] = [ export const ADGUARD_FILTERS_EXTRA: AdGuardFilterSource[] = [ // AdGuard Annoyances filter ['https://filters.adtidy.org/android/filters/14_optimized.txt', null, TTL.THREE_HOURS()], + // AdGuard Cookie Notices + ['https://filters.adtidy.org/extension/ublock/filters/18_optimized.txt', null, TTL.THREE_HOURS()], // EasyList Germany filter [ 'https://easylist.to/easylistgermany/easylistgermany.txt', diff --git a/Build/lib/parse-filter.test.ts b/Build/lib/parse-filter.test.ts new file mode 100644 index 00000000..337b615e --- /dev/null +++ b/Build/lib/parse-filter.test.ts @@ -0,0 +1,18 @@ +import { describe, it } from 'mocha'; + +import { processFilterRules } from './parse-filter'; +import { createCacheKey } from './cache-filesystem'; +import { createSpan } from '../trace'; + +const cacheKey = createCacheKey(__filename); + +describe('processFilterRules', () => { + it('https://filters.adtidy.org/extension/ublock/filters/18_optimized.txt', () => { + console.log(processFilterRules( + createSpan('noop'), + cacheKey('https://filters.adtidy.org/extension/ublock/filters/18_optimized.txt'), + [], + 7_200_000 + )); + }); +}); diff --git a/Build/lib/parse-filter.ts b/Build/lib/parse-filter.ts index c095758f..65135d03 100644 --- a/Build/lib/parse-filter.ts +++ b/Build/lib/parse-filter.ts @@ -14,14 +14,25 @@ import { looseTldtsOpt } from '../constants/loose-tldts-opt'; const DEBUG_DOMAIN_TO_FIND: string | null = null; // example.com | null let foundDebugDomain = false; -const temporaryBypass = true; +const temporaryBypass = typeof DEBUG_DOMAIN_TO_FIND === 'string'; const domainListLineCb = (l: string, set: string[], includeAllSubDomain: boolean, meta: string) => { let line = processLine(l); if (!line) return; + line = line.toLowerCase(); - line = normalizeDomain(line); - if (!line) return; + const domain = normalizeDomain(line); + if (!domain) return; + if (domain !== line) { + console.log( + picocolors.red('[process domain list]'), + picocolors.gray(`line: ${line}`), + picocolors.gray(`domain: ${domain}`), + picocolors.gray(meta) + ); + + return; + } if (DEBUG_DOMAIN_TO_FIND && line.includes(DEBUG_DOMAIN_TO_FIND)) { console.warn(picocolors.red(meta), '(black)', line.replaceAll(DEBUG_DOMAIN_TO_FIND, picocolors.bold(DEBUG_DOMAIN_TO_FIND)));