From 4f56b84adb30f08e874cb5135421f254f3c4d500 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Fri, 25 Apr 2025 21:31:54 +0800 Subject: [PATCH] Chore: minor changes --- Build/lib/create-file.worker.ts | 2 +- Build/lib/parse-filter/hosts.ts | 30 ++++++++++++++++++++++++------ Build/tools-migrate-domains.ts | 25 +++++++++++++++++-------- 3 files changed, 42 insertions(+), 15 deletions(-) diff --git a/Build/lib/create-file.worker.ts b/Build/lib/create-file.worker.ts index 0cd1c69e..6dc21476 100644 --- a/Build/lib/create-file.worker.ts +++ b/Build/lib/create-file.worker.ts @@ -31,7 +31,7 @@ const pool = new Worktank({ isEqual = await fileEqual(linesA, readFileByLine(filePath)); } else { console.log(`${filePath} does not exists, writing...`); - isEqual = false; + // isEqual = false; // isEqual is false by default anyway } if (isEqual) { diff --git a/Build/lib/parse-filter/hosts.ts b/Build/lib/parse-filter/hosts.ts index 37ef044e..7f95499f 100644 --- a/Build/lib/parse-filter/hosts.ts +++ b/Build/lib/parse-filter/hosts.ts @@ -5,25 +5,42 @@ import { onBlackFound } from './shared'; const rSpace = /\s+/; -function hostsLineCb(line: string, set: string[], includeAllSubDomain: boolean, meta: string) { - const _domain = line.split(rSpace, 3)[1]?.trim(); +function hostsLineCb(line: string, set: string[], meta: string) { + const _domain = line.split(rSpace, 3)[1]; if (!_domain) { return; } - const domain = fastNormalizeDomainWithoutWww(_domain); + const domain = fastNormalizeDomainWithoutWww(_domain.trim()); if (!domain) { return; } onBlackFound(domain, meta); - set.push(includeAllSubDomain ? `.${domain}` : domain); + set.push(domain); +} + +function hostsLineCbIncludeAllSubdomain(line: string, set: string[], meta: string) { + const _domain = line.split(rSpace, 3)[1]; + if (!_domain) { + return; + } + const domain = fastNormalizeDomainWithoutWww(_domain.trim()); + if (!domain) { + return; + } + + onBlackFound(domain, meta); + + set.push('.' + domain); } export function processHosts( span: Span, hostsUrl: string, mirrors: string[] | null, includeAllSubDomain = false ) { + const cb = includeAllSubDomain ? hostsLineCbIncludeAllSubdomain : hostsLineCb; + return span.traceChildAsync(`process hosts: ${hostsUrl}`, async (span) => { const filterRules = await span.traceChild('download').traceAsyncFn(() => fetchAssets(hostsUrl, mirrors, true)); @@ -31,7 +48,7 @@ export function processHosts( span.traceChild('parse hosts').traceSyncFn(() => { for (let i = 0, len = filterRules.length; i < len; i++) { - hostsLineCb(filterRules[i], domainSets, includeAllSubDomain, hostsUrl); + cb(filterRules[i], domainSets, hostsUrl); } }); @@ -41,6 +58,7 @@ export function processHosts( export function processHostsWithPreload(hostsUrl: string, mirrors: string[] | null, includeAllSubDomain = false) { const downloadPromise = fetchAssets(hostsUrl, mirrors, true); + const cb = includeAllSubDomain ? hostsLineCbIncludeAllSubdomain : hostsLineCb; return (span: Span) => span.traceChildAsync(`process hosts: ${hostsUrl}`, async (span) => { const filterRules = await span.traceChild('download').tracePromise(downloadPromise); @@ -49,7 +67,7 @@ export function processHostsWithPreload(hostsUrl: string, mirrors: string[] | nu span.traceChild('parse hosts').traceSyncFn(() => { for (let i = 0, len = filterRules.length; i < len; i++) { - hostsLineCb(filterRules[i], domainSets, includeAllSubDomain, hostsUrl); + cb(filterRules[i], domainSets, hostsUrl); } }); diff --git a/Build/tools-migrate-domains.ts b/Build/tools-migrate-domains.ts index 540a71b6..7063dd62 100644 --- a/Build/tools-migrate-domains.ts +++ b/Build/tools-migrate-domains.ts @@ -6,17 +6,17 @@ import { processLine } from './lib/process-line'; import { HostnameSmolTrie } from './lib/trie'; import { dummySpan } from './trace'; import { SOURCE_DIR } from './constants/dir'; +import { PREDEFINED_WHITELIST } from './constants/reject-data-source'; (async () => { const trie = new HostnameSmolTrie(); - writeFiltersToTrie(trie, 'https://cdn.jsdelivr.net/gh/DandelionSprout/adfilt@master/GameConsoleAdblockList.txt', true); + await writeHostsToTrie(trie, 'https://cdn.jsdelivr.net/gh/crazy-max/WindowsSpyBlocker@master/data/hosts/spy.txt', true); - for await (const line of readFileByLine(path.join(SOURCE_DIR, 'domainset', 'reject.conf'))) { - const l = processLine(line); - if (l) { - trie.whitelist(l); - } + await runWhiteOnSource(path.join(SOURCE_DIR, 'domainset', 'reject.conf'), trie); + + for (let i = 0, len = PREDEFINED_WHITELIST.length; i < len; i++) { + trie.whitelist(PREDEFINED_WHITELIST[i]); } console.log('---------------------------'); @@ -24,15 +24,24 @@ import { SOURCE_DIR } from './constants/dir'; console.log('---------------------------'); })(); -// eslint-disable-next-line unused-imports/no-unused-vars -- ready to use function +async function runWhiteOnSource(sourceFile: string, trie: HostnameSmolTrie) { + for await (const line of readFileByLine(sourceFile)) { + const l = processLine(line); + if (l) { + trie.whitelist(l); + } + } +} + async function writeHostsToTrie(trie: HostnameSmolTrie, hostsUrl: string, includeAllSubDomain = false) { - const hosts = await processHosts(dummySpan, 'https://cdn.jsdelivr.net/gh/DandelionSprout/adfilt@master/GameConsoleAdblockList.txt', [], includeAllSubDomain); + const hosts = await processHosts(dummySpan, hostsUrl, [], includeAllSubDomain); for (let i = 0, len = hosts.length; i < len; i++) { trie.add(hosts[i]); } } +// eslint-disable-next-line unused-imports/no-unused-vars -- ready to use function async function writeFiltersToTrie(trie: HostnameSmolTrie, filterUrl: string, includeThirdParty = false) { const { whiteDomainSuffixes, whiteDomains, blackDomainSuffixes, blackDomains } = await processFilterRulesWithPreload(filterUrl, [], includeThirdParty)(dummySpan); for (let i = 0, len = blackDomainSuffixes.length; i < len; i++) {