From a664d2d7793049dc85023b17f871aeb3030bcc79 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Wed, 1 Jul 2026 04:56:18 +0800 Subject: [PATCH] Refactor: migrate to `hntrie` --- Build/build-cdn-download-conf.worker.ts | 2 +- Build/build-microsoft-cdn.worker.ts | 2 +- Build/build-reject-domainset.ts | 3 +- Build/lib/fetch-text-by-line.ts | 1 + Build/lib/rules/base.ts | 36 +- Build/lib/trie.test.ts | 411 --------------- Build/lib/trie.ts | 665 ------------------------ Build/tools-dedupe-src.ts | 21 +- Build/tools-lum-apex-domains.ts | 14 +- Build/tools-migrate-domains.ts | 20 +- Build/validate-domestic.ts | 20 +- Build/validate-gfwlist.ts | 17 +- Build/validate-global-tld.ts | 20 +- package.json | 1 + pnpm-lock.yaml | 10 + 15 files changed, 115 insertions(+), 1128 deletions(-) delete mode 100644 Build/lib/trie.test.ts delete mode 100644 Build/lib/trie.ts diff --git a/Build/build-cdn-download-conf.worker.ts b/Build/build-cdn-download-conf.worker.ts index 065c123c..55823a2d 100644 --- a/Build/build-cdn-download-conf.worker.ts +++ b/Build/build-cdn-download-conf.worker.ts @@ -6,7 +6,7 @@ import { appendArrayInPlace } from 'foxts/append-array-in-place'; import { SOURCE_DIR } from './constants/dir'; import { DomainsetOutput } from './lib/rules/domainset'; import { CRASHLYTICS_WHITELIST } from './constants/reject-data-source'; -import { HostnameTrie } from './lib/trie'; +import { HostnameTrie } from 'hntrie'; import { $$fetch } from './lib/fetch-retry'; import { fastUri } from 'fast-uri'; diff --git a/Build/build-microsoft-cdn.worker.ts b/Build/build-microsoft-cdn.worker.ts index 45ea588e..d0c5031a 100644 --- a/Build/build-microsoft-cdn.worker.ts +++ b/Build/build-microsoft-cdn.worker.ts @@ -2,7 +2,7 @@ import { task } from './trace'; import { SHARED_DESCRIPTION } from './constants/description'; import { RulesetOutput } from './lib/rules/ruleset'; import { RULES, PROBE_DOMAINS, DOMAINS, DOMAIN_SUFFIXES, BLACKLIST } from './constants/microsoft-cdn'; -import { HostnameSmolTrie } from './lib/trie'; +import { HostnameSmolTrie } from 'hntrie/smol'; import { fetchRemoteTextByLine } from './lib/fetch-text-by-line'; import { appendArrayInPlace } from 'foxts/append-array-in-place'; import { extractDomainsFromFelixDnsmasq } from './lib/parse-dnsmasq'; diff --git a/Build/build-reject-domainset.ts b/Build/build-reject-domainset.ts index 94e6023f..75ebb2fc 100644 --- a/Build/build-reject-domainset.ts +++ b/Build/build-reject-domainset.ts @@ -263,7 +263,8 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as } // Deduplicate reject_extra and reject_phishing from the base reject domainset - rejectDomainsetOutput.domainTrie.dump(arg => { + rejectDomainsetOutput.domainTrie.dump((domain, includeSubdomain) => { + const arg = includeSubdomain ? '.' + domain : domain; rejectExtraDomainsetOutput.whitelistDomain(arg); rejectPhisingDomainsetOutput.whitelistDomain(arg); diff --git a/Build/lib/fetch-text-by-line.ts b/Build/lib/fetch-text-by-line.ts index cd70b571..22241a41 100644 --- a/Build/lib/fetch-text-by-line.ts +++ b/Build/lib/fetch-text-by-line.ts @@ -30,6 +30,7 @@ export const createReadlineInterfaceFromResponse: ((resp: UndiciResponseData | U } const resultStream = webStream + // @ts-expect-error -- mismatched Node.js and web types .pipeThrough(new TextDecoderStream()) .pipeThrough(new TextLineStream({ skipEmptyLines: processLine })); diff --git a/Build/lib/rules/base.ts b/Build/lib/rules/base.ts index a66c7dae..dfc8d453 100644 --- a/Build/lib/rules/base.ts +++ b/Build/lib/rules/base.ts @@ -1,5 +1,6 @@ import type { Span } from '../../trace'; -import { HostnameSmolTrie } from '../trie'; +import { HostnameSmolTrie } from 'hntrie/smol'; +import { domainToASCII } from 'node:url'; import { not, nullthrow } from 'foxts/guard'; import { fastIpVersion } from 'foxts/fast-ip-version'; import { addArrayElementsToSet } from 'foxts/add-array-elements-to-set'; @@ -20,7 +21,7 @@ export class FileOutput { protected dataSource = new Set(); - public domainTrie = new HostnameSmolTrie(null); + public domainTrie = new HostnameSmolTrie(); public wildcardSet = new Set(); protected domainKeywords = new Set(); @@ -116,14 +117,18 @@ export class FileOutput { for (let i = 0, len = domains.length; i < len; i++) { const d = domains[i]; if (d !== null) { - this.domainTrie.add(d, false, null, 0); + this.domainTrie.add(d); } } return this; } - addDomainSuffix(domain: string, lineFromDot = domain[0] === '.') { - this.domainTrie.add(domain, true, null, lineFromDot ? 1 : 0); + addDomainSuffix(domain: string) { + if (domain[0] === '.') { + this.domainTrie.add(domain); + } else { + this.domainTrie.addSubdomain(domain); + } return this; } @@ -179,9 +184,9 @@ export class FileOutput { } if (line[0] === '.') { - this.addDomainSuffix(line, true); + this.addDomainSuffix(line); } else { - this.domainTrie.add(line, false, null, 0); + this.domainTrie.add(line); } } } @@ -210,10 +215,10 @@ export class FileOutput { switch (type) { case 'DOMAIN': - this.domainTrie.add(value, false, null, 0); + this.domainTrie.add(value); break; case 'DOMAIN-SUFFIX': - this.addDomainSuffix(value, false); + this.domainTrie.addSubdomain(value); break; case 'DOMAIN-KEYWORD': this.addDomainKeyword(value); @@ -416,9 +421,16 @@ export class FileOutput { const strategiesLen = this.strategies.length; - this.domainTrie.dumpWithoutDot((domain, includeAllSubdomain) => { + const domainEntries: Array<[domain: string, subdomain: boolean]> = []; + this.domainTrie.dump((domain, includeSubdomain) => { + const d = domainToASCII(domain); + if (d) domainEntries.push([d, includeSubdomain]); + }); + domainEntries.sort((a, b) => (a[0].length - b[0].length) || (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0)); + for (let j = 0, entriesLen = domainEntries.length; j < entriesLen; j++) { + const [domain, includeAllSubdomain] = domainEntries[j]; if (kwfilter(domain)) { - return; + continue; } for (let i = 0; i < strategiesLen; i++) { @@ -429,7 +441,7 @@ export class FileOutput { strategy.writeDomain(domain); } } - }, true); + } // Now, we whitelisted out DOMAIN-KEYWORD const whiteKwfilter = createKeywordFilter(Array.from(this.whitelistKeywords)); diff --git a/Build/lib/trie.test.ts b/Build/lib/trie.test.ts deleted file mode 100644 index 9fe67636..00000000 --- a/Build/lib/trie.test.ts +++ /dev/null @@ -1,411 +0,0 @@ -import { describe, it } from 'mocha'; -import { expect } from 'earl'; -import { HostnameSmolTrie, HostnameTrie } from './trie'; - -function createTrie(from: string[] | Set | null, smolTree: true): HostnameSmolTrie; -function createTrie(from?: string[] | Set | null, smolTree?: false): HostnameTrie; -function createTrie<_Meta = any>(from?: string[] | Set | null, smolTree = true) { - if (smolTree) { - return new HostnameSmolTrie(from); - } - return new HostnameTrie(from); -}; - -// describe('hostname to tokens', () => { -// it('should split hostname into tokens.', () => { -// expect(hostnameToTokens('.blog.skk.moe')).toEqual([ -// '.', -// 'blog', -// '.', -// 'skk', -// '.', -// 'moe' -// ]); - -// expect(hostnameToTokens('blog.skk.moe')).toEqual([ -// 'blog', -// '.', -// 'skk', -// '.', -// 'moe' -// ]); - -// expect(hostnameToTokens('skk.moe')).toEqual([ -// 'skk', -// '.', -// 'moe' -// ]); - -// expect(hostnameToTokens('moe')).toEqual([ -// 'moe' -// ]); -// }); -// }); - -describe('Trie', () => { - it('should be possible to add domains to a Trie.', () => { - const trie = createTrie(null, false); - - trie.add('a.skk.moe'); - trie.add('skk.moe'); - trie.add('anotherskk.moe'); - - expect(trie.size).toEqual(3); - - expect(trie.has('a.skk.moe')).toEqual(true); - expect(trie.has('skk.moe')).toEqual(true); - expect(trie.has('anotherskk.moe')).toEqual(true); - expect(trie.has('example.com')).toEqual(false); - expect(trie.has('skk.mo')).toEqual(false); - expect(trie.has('another.skk.moe')).toEqual(false); - }); - - it('adding the same item several times should not increase size.', () => { - const trie = createTrie(null, false); - - trie.add('skk.moe'); - trie.add('blog.skk.moe'); - // eslint-disable-next-line sukka/no-element-overwrite -- deliberately do testing - trie.add('skk.moe'); - - expect(trie.size).toEqual(2); - expect(trie.has('skk.moe')).toEqual(true); - }); - - it('should be possible to set the null sequence.', () => { - const trie = createTrie(null, false); - - trie.add(''); - expect(trie.has('')).toEqual(true); - - const trie2 = createTrie(null, true); - trie2.add(''); - expect(trie2.has('')).toEqual(true); - }); - - it('should be possible to delete items.', () => { - const trie = createTrie(null, false); - - trie.add('skk.moe'); - trie.add('blog.skk.moe'); - trie.add('example.com'); - trie.add('moe.sb'); - - expect(trie.delete('no-match.com')).toEqual(false); - expect(trie.delete('example.org')).toEqual(false); - - expect(trie.delete('skk.moe')).toEqual(true); - expect(trie.has('skk.moe')).toEqual(false); - expect(trie.has('moe.sb')).toEqual(true); - - expect(trie.size).toEqual(3); - - expect(trie.delete('example.com')).toEqual(true); - expect(trie.size).toEqual(2); - expect(trie.delete('moe.sb')).toEqual(true); - expect(trie.size).toEqual(1); - }); - - it('should be possible to check the existence of a sequence in the Trie.', () => { - const trie = createTrie(null, true); - - trie.add('example.org.skk.moe'); - - expect(trie.has('example.org.skk.moe')).toEqual(true); - expect(trie.has('skk.moe')).toEqual(false); - expect(trie.has('example.org')).toEqual(false); - expect(trie.has('')).toEqual(false); - }); - - it('should be possible to retrieve items matching the given prefix.', () => { - const trie = createTrie(null, false); - - trie.add('example.com'); - trie.add('blog.example.com'); - trie.add('cdn.example.com'); - trie.add('example.org'); - - expect(trie.find('example.com')).toEqual(['example.com', 'cdn.example.com', 'blog.example.com']); - expect(trie.find('com')).toEqual(['example.com', 'cdn.example.com', 'blog.example.com']); - expect(trie.find('.example.com')).toEqual(['cdn.example.com', 'blog.example.com']); - expect(trie.find('org')).toEqual(['example.org']); - expect(trie.find('example.net')).toEqual([]); - expect(trie.dump()).toEqual(['example.org', 'example.com', 'cdn.example.com', 'blog.example.com']); - }); - - it('should be possible to retrieve items matching the given prefix even with a smol trie', () => { - const trie = createTrie(null, true); - - trie.add('.example.com'); - trie.add('example.com'); - trie.add('blog.example.com'); - trie.add('cdn.example.com'); - trie.add('example.org'); - - expect(trie.find('example.com')).toEqual(['.example.com']); - expect(trie.find('com')).toEqual(['.example.com']); - expect(trie.find('.example.com')).toEqual(['.example.com']); - expect(trie.find('org')).toEqual(['example.org']); - expect(trie.find('example.net')).toEqual([]); - expect(trie.dump()).toEqual(['example.org', '.example.com']); - }); - - it('should be possible to create a trie from an arbitrary iterable.', () => { - let trie = createTrie(['skk.moe', 'blog.skk.moe'], false); - - expect(trie.size).toEqual(2); - expect(trie.has('skk.moe')).toEqual(true); - - trie = createTrie(new Set(['skk.moe', 'example.com']), false); - expect(trie.size).toEqual(2); - expect(trie.has('skk.moe')).toEqual(true); - }); -}); - -describe('surge domainset dedupe', () => { - it('should not remove same entry', () => { - const trie = createTrie(['.skk.moe', 'noc.one'], false); - - expect(trie.find('.skk.moe')).toEqual(['.skk.moe']); - expect(trie.find('noc.one')).toEqual(['noc.one']); - }); - - it('should match subdomain - 1', () => { - const trie = createTrie(['www.noc.one', 'www.sukkaw.com', 'blog.skk.moe', 'image.cdn.skk.moe', 'cdn.sukkaw.net'], false); - - expect(trie.find('.skk.moe')).toEqual(['image.cdn.skk.moe', 'blog.skk.moe']); - expect(trie.find('.sukkaw.com')).toEqual(['www.sukkaw.com']); - }); - - it('should match subdomain - 2', () => { - const trie = createTrie(['www.noc.one', 'www.sukkaw.com', '.skk.moe', 'blog.skk.moe', 'image.cdn.skk.moe', 'cdn.sukkaw.net'], false); - - expect(trie.find('.skk.moe')).toEqual(['.skk.moe', 'image.cdn.skk.moe', 'blog.skk.moe']); - expect(trie.find('.sukkaw.com')).toEqual(['www.sukkaw.com']); - }); - - it('should not remove non-subdomain', () => { - const trie = createTrie(['skk.moe', 'sukkaskk.moe'], false); - expect(trie.find('.skk.moe')).toEqual([]); - }); -}); - -describe('smol tree', () => { - it('should init tree', () => { - const trie = createTrie([ - 'skk.moe', - 'anotherskk.moe', - 'blog.anotherskk.moe', - 'blog.skk.moe', - '.cdn.local', - 'blog.img.skk.local', - 'img.skk.local' - ], true); - - expect(trie.dump()).toEqual([ - 'img.skk.local', - 'blog.img.skk.local', - '.cdn.local', - 'anotherskk.moe', - 'blog.anotherskk.moe', - 'skk.moe', - 'blog.skk.moe' - ]); - }); - - it('should create simple tree - 1', () => { - const trie = createTrie([ - '.skk.moe', 'blog.skk.moe', '.cdn.skk.moe', 'skk.moe', - 'www.noc.one', 'cdn.noc.one', - '.blog.sub.example.com', 'sub.example.com', 'cdn.sub.example.com', '.sub.example.com' - ], true); - - expect(trie.dump()).toEqual([ - '.sub.example.com', - 'cdn.noc.one', - 'www.noc.one', - '.skk.moe' - ]); - }); - - it('should create simple tree - 2', () => { - const trie = createTrie([ - '.skk.moe', 'blog.skk.moe', '.cdn.skk.moe', 'skk.moe' - ], true); - - expect(trie.dump()).toEqual([ - '.skk.moe' - ]); - }); - - it('should create simple tree - 3', () => { - const trie = createTrie([ - '.blog.sub.example.com', 'cdn.sub.example.com', '.sub.example.com' - ], true); - - expect(trie.dump()).toEqual([ - '.sub.example.com' - ]); - - trie.add('.sub.example.com'); - expect(trie.dump()).toEqual([ - '.sub.example.com' - ]); - }); - - it('should create simple tree - 3', () => { - const trie = createTrie([ - 'commercial.shouji.360.cn', - 'act.commercial.shouji.360.cn', - 'cdn.creative.medialytics.com', - 'px.cdn.creative.medialytics.com' - ], true); - - expect(trie.dump()).toEqual([ - 'cdn.creative.medialytics.com', - 'px.cdn.creative.medialytics.com', - 'commercial.shouji.360.cn', - 'act.commercial.shouji.360.cn' - ]); - }); - - it('should dedupe subdomain properly', () => { - const trie = createTrie([ - 'skk.moe', - 'anotherskk.moe', - 'blog.anotherskk.moe', - 'blog.skk.moe' - ], true); - - expect(trie.dump()).toEqual([ - 'anotherskk.moe', - 'blog.anotherskk.moe', - 'skk.moe', - 'blog.skk.moe' - ]); - }); - - it('should effctly whitelist domains', () => { - const trie = createTrie([ - 'skk.moe', - 'anotherskk.moe', - 'blog.anotherskk.moe', - 'blog.skk.moe', - '.cdn.local', - 'blog.img.skk.local', - 'img.skk.local' - ], true); - - trie.whitelist('.skk.moe'); - - expect(trie.dump()).toEqual([ - 'img.skk.local', - 'blog.img.skk.local', - '.cdn.local', - 'anotherskk.moe', - 'blog.anotherskk.moe' - ]); - - trie.whitelist('anotherskk.moe'); - expect(trie.dump()).toEqual([ - 'img.skk.local', - 'blog.img.skk.local', - '.cdn.local', - 'blog.anotherskk.moe' - ]); - - trie.add('anotherskk.moe'); - trie.whitelist('.anotherskk.moe'); - - expect(trie.dump()).toEqual([ - 'img.skk.local', - 'blog.img.skk.local', - '.cdn.local' - ]); - - trie.whitelist('img.skk.local'); - expect(trie.dump()).toEqual([ - 'blog.img.skk.local', - '.cdn.local' - ]); - - trie.whitelist('cdn.local'); - expect(trie.dump()).toEqual([ - 'blog.img.skk.local' - ]); - - trie.whitelist('.skk.local'); - expect(trie.dump()).toEqual([]); - }); - - it('should whitelist trie correctly', () => { - const trie = createTrie([ - '.t.co', - 't.co', - 'example.t.co', - '.skk.moe', - 'blog.cdn.example.com', - 'cdn.example.com' - ], true); - - expect(trie.dump()).toEqual([ - 'cdn.example.com', 'blog.cdn.example.com', - '.skk.moe', - '.t.co' - ]); - - trie.whitelist('.t.co'); - expect(trie.dump()).toEqual([ - 'cdn.example.com', 'blog.cdn.example.com', '.skk.moe' - ]); - - trie.whitelist('skk.moe'); - expect(trie.dump()).toEqual(['cdn.example.com', 'blog.cdn.example.com']); - - trie.whitelist('cdn.example.com'); - expect(trie.dump()).toEqual(['blog.cdn.example.com']); - }); - - it('contains - normal', () => { - const trie = createTrie([ - 'skk.moe', - 'anotherskk.moe', - 'blog.anotherskk.moe', - 'blog.skk.moe' - ], true); - - expect(trie.contains('skk.moe')).toEqual(true); - expect(trie.contains('blog.skk.moe')).toEqual(true); - expect(trie.contains('anotherskk.moe')).toEqual(true); - expect(trie.contains('blog.anotherskk.moe')).toEqual(true); - - expect(trie.contains('example.com')).toEqual(false); - expect(trie.contains('blog.example.com')).toEqual(false); - expect(trie.contains('skk.mo')).toEqual(false); - expect(trie.contains('cdn.skk.moe')).toEqual(false); - }); - - it('contains - subdomain', () => { - const trie = createTrie([ - 'index.rubygems.org' - ], true); - - expect(trie.contains('rubygems.org')).toEqual(false); - expect(trie.contains('index.rubygems.org')).toEqual(true); - expect(trie.contains('sub.index.rubygems.org')).toEqual(false); - }); - - it('contains - include subdomains', () => { - const trie = createTrie([ - '.skk.moe' - ], true); - - expect(trie.contains('skk.moe')).toEqual(true); - expect(trie.contains('blog.skk.moe')).toEqual(true); - expect(trie.contains('image.cdn.skk.moe')).toEqual(true); - - expect(trie.contains('example.com')).toEqual(false); - expect(trie.contains('blog.example.com')).toEqual(false); - expect(trie.contains('skk.mo')).toEqual(false); - }); -}); diff --git a/Build/lib/trie.ts b/Build/lib/trie.ts deleted file mode 100644 index 6fc281f1..00000000 --- a/Build/lib/trie.ts +++ /dev/null @@ -1,665 +0,0 @@ -/** - * Hostbane-Optimized Trie based on Mnemonist Trie - */ - -import { fastStringCompare } from 'foxts/fast-string-compare'; -import util from 'node:util'; -import { noop } from 'foxts/noop'; -import { fastStringArrayJoin } from 'foxts/fast-string-array-join'; - -import { deleteBit, getBit, missingBit, setBit } from 'foxts/bitwise'; -import { domainToASCII } from 'node:url'; - -const START = 1 << 1; -const INCLUDE_ALL_SUBDOMAIN = 1 << 2; - -type TrieNode = [ - /** end, includeAllSubdomain (.example.org, ||example.com) */ flag: number, - /** parent */ TrieNode | null, - /** children */ Map, - /** token */ token: string, - /** meta */ Meta -]; - -function deepTrieNodeToJSON(node: TrieNode, - unpackMeta: ((meta?: Meta) => string) | undefined) { - const obj: Record = { - ['[start]']: getBit(node[0], START), - ['[subdomain]']: getBit(node[0], INCLUDE_ALL_SUBDOMAIN) - }; - - if (node[4] != null) { - if (unpackMeta) { - obj['[meta]'] = unpackMeta(node[4]); - } else { - obj['[meta]'] = node[4]; - } - } - node[2].forEach((value, key) => { - obj[key] = deepTrieNodeToJSON(value, unpackMeta); - }); - return obj; -} - -const createNode = (token: string, parent: TrieNode | null = null): TrieNode => [1, parent, new Map(), token, null] as TrieNode; - -function hostnameToTokens(hostname: string, hostnameFromIndex: number): string[] { - const tokens = hostname.split('.'); - const results: string[] = []; - let token = ''; - - for (let i = hostnameFromIndex, l = tokens.length; i < l; i++) { - token = tokens[i]; - if (token.length > 0) { - results.push(token); - } else { - throw new TypeError(JSON.stringify({ hostname, hostnameFromIndex }, null, 2)); - } - } - - return results; -} - -function walkHostnameTokens(hostname: string, onToken: (token: string) => boolean | null, hostnameFromIndex: number): boolean | null { - const tokens = hostname.split('.'); - - const l = tokens.length - 1; - - // we are at the first of hostname, no splitor there - let token = ''; - - for (let i = l; i >= hostnameFromIndex; i--) { - token = tokens[i]; - if (token.length > 0) { - const t = onToken(token); - if (t === null) { - return null; - } - // if the callback returns true, we should skip the rest - if (t) { - return true; - } - } - } - - return false; -} - -interface FindSingleChildLeafResult { - node: TrieNode, - toPrune: TrieNode | null, - tokenToPrune: string | null, - parent: TrieNode -} - -abstract class Triebase { - protected readonly $root: TrieNode = createNode('$root'); - protected $size = 0; - - get root() { - return this.$root; - } - - constructor(from?: string[] | Set | null) { - // Actually build trie - if (Array.isArray(from)) { - for (let i = 0, l = from.length; i < l; i++) { - this.add(from[i]); - } - } else if (from) { - from.forEach((value) => this.add(value)); - } - } - - public abstract add(suffix: string, includeAllSubdomain?: boolean, meta?: Meta, hostnameFromIndex?: number): void; - - protected walkIntoLeafWithTokens( - tokens: string[], - onLoop: (node: TrieNode, parent: TrieNode, token: string) => void = noop - ) { - let node: TrieNode = this.$root; - let parent: TrieNode = node; - - let token: string; - let child: Map> = node[2]; - - // reverse lookup from end to start - for (let i = tokens.length - 1; i >= 0; i--) { - token = tokens[i]; - - // if (token === '') { - // break; - // } - - parent = node; - - child = node[2]; - // cache node index access is 20% faster than direct access when doing twice - if (child.has(token)) { - node = child.get(token)!; - } else { - return null; - } - - onLoop(node, parent, token); - } - - return { node, parent }; - }; - - protected walkIntoLeafWithSuffix( - suffix: string, - hostnameFromIndex: number, - onLoop: (node: TrieNode, parent: TrieNode, token: string) => void = noop - ) { - let node: TrieNode = this.$root; - let parent: TrieNode = node; - - let child: Map> = node[2]; - - const onToken = (token: string) => { - // if (token === '') { - // return true; - // } - - parent = node; - - child = node[2]; - - if (child.has(token)) { - node = child.get(token)!; - } else { - return null; - } - - onLoop(node, parent, token); - - return false; - }; - - if (walkHostnameTokens(suffix, onToken, hostnameFromIndex) === null) { - return null; - } - - return { node, parent }; - }; - - public contains(suffix: string, includeAllSubdomain = suffix[0] === '.'): boolean { - const hostnameFromIndex = suffix[0] === '.' ? 1 : 0; - - let node: TrieNode = this.$root; - // let parent: TrieNode = node; - - let child: Map> = node[2]; - - let result = false; - - const onToken = (token: string) => { - // if (token === '') { - // return true; - // } - - // parent = node; - - child = node[2]; - - if (child.has(token)) { - node = child.get(token)!; - } else { - if (getBit(node[0], INCLUDE_ALL_SUBDOMAIN)) { - result = true; - } - return null; - } - - return false; - }; - - if (walkHostnameTokens(suffix, onToken, hostnameFromIndex) === null) { - return result; - } - - if (includeAllSubdomain) return getBit(node[0], INCLUDE_ALL_SUBDOMAIN); - return getBit(node[0], START); - - // if (res === null) return false; - // if (includeAllSubdomain) return getBit(res.node[0], INCLUDE_ALL_SUBDOMAIN); - // return true; - }; - - private static bfsResults: [node: TrieNode | null, suffix: string[]] = [null, []]; - - private static dfs(this: void, nodeStack: Array>, suffixStack: string[][]) { - const node = nodeStack.pop()!; - const suffix = suffixStack.pop()!; - - node[2].forEach((childNode, k) => { - // Pushing the child node to the stack for next iteration of DFS - nodeStack.push(childNode); - - suffixStack.push([k, ...suffix]); - }); - - Triebase.bfsResults[0] = node; - Triebase.bfsResults[1] = suffix; - - return Triebase.bfsResults; - } - - private static dfsWithSort(this: void, nodeStack: Array>, suffixStack: string[][]) { - const node = nodeStack.pop()!; - const suffix = suffixStack.pop()!; - - const child = node[2]; - - if (child.size) { - const keys = Array.from(child.keys()).sort(Triebase.compare); - - for (let i = 0, l = keys.length; i < l; i++) { - const key = keys[i]; - const childNode = child.get(key)!; - - // Pushing the child node to the stack for next iteration of DFS - nodeStack.push(childNode); - suffixStack.push([key, ...suffix]); - } - } - - Triebase.bfsResults[0] = node; - Triebase.bfsResults[1] = suffix; - - return Triebase.bfsResults; - } - - private walk( - onMatches: (suffix: string[], subdomain: boolean, meta: Meta) => void, - withSort = false, - initialNode = this.$root, - initialSuffix: string[] = [] - ) { - const dfsImpl = withSort ? Triebase.dfsWithSort : Triebase.dfs; - - const nodeStack: Array> = [initialNode]; - - // Resolving initial string (begin the start of the stack) - const suffixStack: string[][] = [initialSuffix]; - - let node: TrieNode = initialNode; - - do { - const r = dfsImpl(nodeStack, suffixStack); - node = r[0]!; - const suffix = r[1]; - - // If the node is a sentinel, we push the suffix to the results - if (getBit(node[0], START)) { - onMatches(suffix, getBit(node[0], INCLUDE_ALL_SUBDOMAIN), node[4]); - } - } while (nodeStack.length); - }; - - static compare(this: void, a: string, b: string) { - if (a === b) return 0; - return (a.length - b.length) || fastStringCompare(a, b); - } - - protected getSingleChildLeaf(tokens: string[]): FindSingleChildLeafResult | null { - let toPrune: TrieNode | null = null; - let tokenToPrune: string | null = null; - - const onLoop = (node: TrieNode, parent: TrieNode, token: string) => { - // Keeping track of a potential branch to prune - - const child = node[2]; - - const childSize = child.size + (getBit(node[0], INCLUDE_ALL_SUBDOMAIN) ? 1 : 0); - - if (toPrune !== null) { // the most near branch that could potentially being pruned - if (childSize >= 1) { - // The branch has some children, the branch need retain. - // And we need to abort prune that parent branch, so we set it to null - toPrune = null; - tokenToPrune = null; - } - } else if (childSize < 1) { - // There is only one token child, or no child at all, we can prune it safely - // It is now the top-est branch that could potentially being pruned - toPrune = parent; - tokenToPrune = token; - } - }; - - const res = this.walkIntoLeafWithTokens(tokens, onLoop); - - if (res === null) return null; - return { node: res.node, toPrune, tokenToPrune, parent: res.parent }; - }; - - /** - * Method used to retrieve every item in the trie with the given prefix. - */ - public find( - inputSuffix: string, - subdomainOnly = inputSuffix[0] === '.', - hostnameFromIndex = inputSuffix[0] === '.' ? 1 : 0 - // /** @default true */ includeEqualWithSuffix = true - ): string[] { - const inputTokens = hostnameToTokens(inputSuffix, hostnameFromIndex); - const res = this.walkIntoLeafWithTokens(inputTokens); - if (res === null) return []; - - const results: string[] = []; - - const onMatches = subdomainOnly - ? (suffix: string[], subdomain: boolean) => { // fast path (default option) - const d = domainToASCII(fastStringArrayJoin(suffix, '.')); - if (!subdomain && subStringEqual(inputSuffix, d, 1)) return; - - results.push(subdomain ? '.' + d : d); - } - : (suffix: string[], subdomain: boolean) => { // fast path (default option) - const d = domainToASCII(fastStringArrayJoin(suffix, '.')); - results.push(subdomain ? '.' + d : d); - }; - - this.walk( - onMatches, - false, - res.node, // Performing DFS from prefix - inputTokens - ); - - return results; - }; - - /** - * Method used to delete a prefix from the trie. - */ - public remove(suffix: string): boolean { - const res = this.getSingleChildLeaf(hostnameToTokens(suffix, 0)); - if (res === null) return false; - - if (missingBit(res.node[0], START)) return false; - - this.$size--; - const { node, toPrune, tokenToPrune } = res; - - if (tokenToPrune && toPrune) { - toPrune[2].delete(tokenToPrune); - } else { - node[0] = deleteBit(node[0], START); - } - - return true; - }; - - // eslint-disable-next-line @typescript-eslint/unbound-method -- safe - public delete = this.remove; - - /** - * Method used to assert whether the given prefix exists in the Trie. - */ - public has(suffix: string, includeAllSubdomain = suffix[0] === '.'): boolean { - const hostnameFromIndex = suffix[0] === '.' ? 1 : 0; - - const res = this.walkIntoLeafWithSuffix(suffix, hostnameFromIndex); - - if (res === null) return false; - if (missingBit(res.node[0], START)) return false; - if (includeAllSubdomain) return getBit(res.node[0], INCLUDE_ALL_SUBDOMAIN); - return true; - }; - - public dumpWithoutDot(onSuffix: (suffix: string, subdomain: boolean) => void, withSort = false) { - const handleSuffix = (suffix: string[], subdomain: boolean) => { - onSuffix(domainToASCII(fastStringArrayJoin(suffix, '.')), subdomain); - }; - - this.walk(handleSuffix, withSort); - } - - public dump(onSuffix: (suffix: string) => void, withSort?: boolean): void; - public dump(onSuffix?: null, withSort?: boolean): string[]; - public dump(onSuffix?: ((suffix: string) => void) | null, withSort = false): string[] | void { - const results: string[] = []; - - const handleSuffix = onSuffix - ? (suffix: string[], subdomain: boolean) => { - const d = domainToASCII(fastStringArrayJoin(suffix, '.')); - onSuffix(subdomain ? '.' + d : d); - } - : (suffix: string[], subdomain: boolean) => { - const d = domainToASCII(fastStringArrayJoin(suffix, '.')); - results.push(subdomain ? '.' + d : d); - }; - - this.walk(handleSuffix, withSort); - - return results; - }; - - public dumpMeta(onMeta: (meta: Meta) => void, withSort?: boolean): void; - public dumpMeta(onMeta?: null, withSort?: boolean): Meta[]; - public dumpMeta(onMeta?: ((meta: Meta) => void) | null, withSort = false): Meta[] | void { - const results: Meta[] = []; - - const handleMeta = onMeta - ? (_suffix: string[], _subdomain: boolean, meta: Meta) => onMeta(meta) - : (_suffix: string[], _subdomain: boolean, meta: Meta) => results.push(meta); - - this.walk(handleMeta, withSort); - - return results; - }; - - public dumpWithMeta(onSuffix: (suffix: string, meta: Meta | undefined) => void, withSort?: boolean): void; - public dumpWithMeta(onMeta?: null, withSort?: boolean): Array<[string, Meta | undefined]>; - public dumpWithMeta(onSuffix?: ((suffix: string, meta: Meta | undefined) => void) | null, withSort = false): Array<[string, Meta | undefined]> | void { - const results: Array<[string, Meta | undefined]> = []; - - const handleSuffix = onSuffix - ? (suffix: string[], subdomain: boolean, meta: Meta | undefined) => { - const d = domainToASCII(fastStringArrayJoin(suffix, '.')); - return onSuffix(subdomain ? '.' + d : d, meta); - } - : (suffix: string[], subdomain: boolean, meta: Meta | undefined) => { - const d = domainToASCII(fastStringArrayJoin(suffix, '.')); - results.push([subdomain ? '.' + d : d, meta]); - }; - - this.walk(handleSuffix, withSort); - - return results; - }; - - public inspect(depth: number, unpackMeta?: (meta?: Meta) => any) { - return fastStringArrayJoin( - JSON.stringify(deepTrieNodeToJSON(this.$root, unpackMeta), null, 2).split('\n').map((line) => ' '.repeat(depth) + line), - '\n' - ); - } - - public [util.inspect.custom](depth: number) { - return this.inspect(depth); - }; - - public merge(trie: Triebase) { - const handleSuffix = (suffix: string[], subdomain: boolean, meta: Meta) => { - this.add(fastStringArrayJoin(suffix, '.'), subdomain, meta); - }; - - trie.walk(handleSuffix); - - return this; - } -} - -export class HostnameSmolTrie extends Triebase { - public smolTree = true; - - add(suffix: string, includeAllSubdomain = suffix[0] === '.', meta?: Meta, hostnameFromIndex = suffix[0] === '.' ? 1 : 0): void { - let node: TrieNode = this.$root; - let curNodeChildren: Map> = node[2]; - - const onToken = (token: string) => { - curNodeChildren = node[2]; - if (curNodeChildren.has(token)) { - node = curNodeChildren.get(token)!; - - // During the adding of `[start]blog|.skk.moe` and find out that there is a `[start].skk.moe` in the trie, skip adding the rest of the node - if (getBit(node[0], INCLUDE_ALL_SUBDOMAIN)) { - return true; - } - } else { - const newNode = createNode(token, node); - curNodeChildren.set(token, newNode); - node = newNode; - } - - return false; - }; - - // When walkHostnameTokens returns true, we should skip the rest - if (walkHostnameTokens(suffix, onToken, hostnameFromIndex)) { - return; - } - - // If we are in smolTree mode, we need to do something at the end of the loop - if (includeAllSubdomain) { - // Trying to add `[.]sub.example.com` where there is already a `blog.sub.example.com` in the trie - - // Make sure parent `[start]sub.example.com` (without dot) is removed (SETINEL to false) - // (/** parent */ node[2]!)[0] = false; - - // Removing the rest of the parent's child nodes - node[2].clear(); - // The SENTINEL of this node will be set to true at the end of the function, so we don't need to set it here - - // we can use else-if here, because the children is now empty, we don't need to check the leading "." - } else if (getBit(node[0], INCLUDE_ALL_SUBDOMAIN)) { - // Trying to add `example.com` when there is already a `.example.com` in the trie - // No need to increment size and set SENTINEL to true (skip this "new" item) - return; - } - - node[0] = setBit(node[0], START); - if (includeAllSubdomain) { - node[0] = setBit(node[0], INCLUDE_ALL_SUBDOMAIN); - } else { - node[0] = deleteBit(node[0], INCLUDE_ALL_SUBDOMAIN); - } - node[4] = meta!; - } - - public whitelist(suffix: string, includeAllSubdomain = suffix[0] === '.', hostnameFromIndex = suffix[0] === '.' ? 1 : 0) { - const tokens = hostnameToTokens(suffix, hostnameFromIndex); - const res = this.getSingleChildLeaf(tokens); - if (res === null) return; - - const { node, toPrune, tokenToPrune } = res; - - // Trying to whitelist `[start].sub.example.com` where there might already be a `[start]blog.sub.example.com` in the trie - if (includeAllSubdomain) { - // If there is a `[start]sub.example.com` here, remove it - node[0] = deleteBit(node[0], INCLUDE_ALL_SUBDOMAIN); - // Removing all the child nodes by empty the children - node[2].clear(); - // we do not remove sub.example.com for now, we will do that later - } else { - // Trying to whitelist `example.com` when there is already a `.example.com` in the trie - node[0] = deleteBit(node[0], INCLUDE_ALL_SUBDOMAIN); - } - - if (includeAllSubdomain) { - node[1]?.[2].delete(node[3]); - } else if (missingBit(node[0], START) && node[1]) { - return; - } - - if (toPrune && tokenToPrune) { - toPrune[2].delete(tokenToPrune); - } else { - node[0] = deleteBit(node[0], START); - } - - cleanUpEmptyTrailNode(node); - }; -} - -function cleanUpEmptyTrailNode(node: TrieNode) { - let current = node; - while ( - // the current node is not an "end node", a.k.a. not the start of a domain - missingBit(current[0], START) - // also no leading "." (no subdomain) - && missingBit(current[0], INCLUDE_ALL_SUBDOMAIN) - // child is empty - && current[2].size === 0 - // has parent: we need to delete the current node from the parent - // we also need to clean up the parent node - && current[1] - ) { - current[1][2].delete(current[3]); - current = current[1]; - } -} - -export class HostnameTrie extends Triebase { - get size() { - return this.$size; - } - - add(suffix: string, includeAllSubdomain = suffix[0] === '.', meta?: Meta, hostnameFromIndex = suffix[0] === '.' ? 1 : 0): void { - let node: TrieNode = this.$root; - let child: Map> = node[2]; - - const onToken = (token: string) => { - child = node[2]; - if (child.has(token)) { - node = child.get(token)!; - } else { - const newNode = createNode(token, node); - child.set(token, newNode); - node = newNode; - } - - return false; - }; - - // When walkHostnameTokens returns true, we should skip the rest - if (walkHostnameTokens(suffix, onToken, hostnameFromIndex)) { - return; - } - - // if same entry has been added before, skip - if (getBit(node[0], START)) { - return; - } - - this.$size++; - - node[0] = setBit(node[0], START); - if (includeAllSubdomain) { - node[0] = setBit(node[0], INCLUDE_ALL_SUBDOMAIN); - } else { - node[0] = deleteBit(node[0], INCLUDE_ALL_SUBDOMAIN); - } - node[4] = meta!; - } -} - -// function deepEqualArray(a: string[], b: string[]) { -// let len = a.length; -// if (len !== b.length) return false; -// while (len--) { -// if (a[len] !== b[len]) return false; -// } -// return true; -// }; - -function subStringEqual(needle: string, haystack: string, needleIndex = 0) { - for (let i = 0, l = haystack.length; i < l; i++) { - if (needle[i + needleIndex] !== haystack[i]) return false; - } - return true; -} diff --git a/Build/tools-dedupe-src.ts b/Build/tools-dedupe-src.ts index d527499b..f0845f33 100644 --- a/Build/tools-dedupe-src.ts +++ b/Build/tools-dedupe-src.ts @@ -4,7 +4,7 @@ import fsp from 'node:fs/promises'; import { SOURCE_DIR } from './constants/dir'; import { readFileByLine } from './lib/fetch-text-by-line'; import { processLine } from './lib/process-line'; -import { HostnameSmolTrie } from './lib/trie'; +import { HostnameSmolTrie } from 'hntrie/smol'; import { task } from './trace'; const ENFORCED_WHITELIST = [ @@ -48,6 +48,15 @@ task(require.main === module, __filename)(async (span) => { await Promise.all(files.map(file => span.traceChildAsync('dedupe ' + file, () => dedupeFile(file, whiteTrie)))); }); +function trieHasEntry(trie: HostnameSmolTrie, line: string): boolean { + if (line[0] === '.') return trie.hasSubdomain(line.slice(1)); + return trie.has(line) || trie.hasSubdomain(line); +} + +function trieContains(trie: HostnameSmolTrie, line: string): boolean { + return trie.match(line); +} + async function dedupeFile(file: string, whitelist: HostnameSmolTrie) { const result: string[] = []; @@ -55,8 +64,7 @@ async function dedupeFile(file: string, whitelist: HostnameSmolTrie) { let line: string | null = ''; - // eslint-disable-next-line @typescript-eslint/unbound-method -- .call - let trieHasOrContains = HostnameSmolTrie.prototype.has; + let trieCheck = trieHasEntry; for await (const l of readFileByLine(file)) { line = processLine(l); @@ -66,19 +74,18 @@ async function dedupeFile(file: string, whitelist: HostnameSmolTrie) { return; } if (l.startsWith('# $ dedupe_use_trie_contains')) { - // eslint-disable-next-line @typescript-eslint/unbound-method -- .call - trieHasOrContains = HostnameSmolTrie.prototype.contains; + trieCheck = trieContains; } result.push(l); // keep all comments and blank lines continue; } - if (trieHasOrContains.call(trie, line)) { + if (trieCheck(trie, line)) { continue; // drop duplicate } - if (whitelist.has(line)) { + if (trieHasEntry(whitelist, line)) { continue; // drop whitelisted items } diff --git a/Build/tools-lum-apex-domains.ts b/Build/tools-lum-apex-domains.ts index 5e42ba88..4c4d1b5d 100644 --- a/Build/tools-lum-apex-domains.ts +++ b/Build/tools-lum-apex-domains.ts @@ -1,6 +1,7 @@ import { fetchRemoteTextByLine } from './lib/fetch-text-by-line'; import tldts from 'tldts-experimental'; -import { HostnameSmolTrie } from './lib/trie'; +import { HostnameSmolTrie } from 'hntrie/smol'; +import { domainToASCII } from 'node:url'; import path from 'node:path'; import { SOURCE_DIR } from './constants/dir'; import runAgainstSourceFile from './lib/run-against-source-file'; @@ -32,11 +33,16 @@ import runAgainstSourceFile from './lib/run-against-source-file'; } await runAgainstSourceFile(path.join(SOURCE_DIR, 'domainset', 'reject.conf'), (domain, includeAllSubDomain) => { - trie.whitelist(domain, includeAllSubDomain); + trie.whitelist(includeAllSubDomain ? '.' + domain : domain); }, 'domainset'); await runAgainstSourceFile(path.join(SOURCE_DIR, 'non_ip', 'reject.conf'), (domain, includeAllSubDomain) => { - trie.whitelist(domain, includeAllSubDomain); + trie.whitelist(includeAllSubDomain ? '.' + domain : domain); }, 'ruleset'); - console.log(trie.dump().map(i => '.' + i).join('\n')); + const dump: string[] = []; + trie.dump((rawDomain) => { + const domain = domainToASCII(rawDomain); + if (domain) dump.push(domain); + }); + console.log(dump.map(i => '.' + i).join('\n')); })(); diff --git a/Build/tools-migrate-domains.ts b/Build/tools-migrate-domains.ts index df9ef463..7770713c 100644 --- a/Build/tools-migrate-domains.ts +++ b/Build/tools-migrate-domains.ts @@ -2,7 +2,8 @@ import path from 'node:path'; import { processFilterRulesWithPreload } from './lib/parse-filter/filters'; import { processHosts } from './lib/parse-filter/hosts'; -import { HostnameSmolTrie } from './lib/trie'; +import { HostnameSmolTrie } from 'hntrie/smol'; +import { domainToASCII } from 'node:url'; import { dummySpan } from './trace'; import { SOURCE_DIR } from './constants/dir'; import { PREDEFINED_WHITELIST } from './constants/reject-data-source'; @@ -15,7 +16,7 @@ import runAgainstSourceFile from './lib/run-against-source-file'; // const { whiteDomainSuffixes, whiteDomains } = await writeFiltersToTrie(trie, 'https://cdn.jsdelivr.net/gh/Perflyst/PiHoleBlocklist@master/SmartTV-AGH.txt', true); const callback = (domain: string, includeAllSubDomain: boolean) => { - trie.whitelist(domain, includeAllSubDomain); + trie.whitelist(includeAllSubDomain ? '.' + domain : domain); }; await runAgainstSourceFile(path.join(SOURCE_DIR, 'domainset', 'reject.conf'), callback, 'domainset'); @@ -26,7 +27,12 @@ import runAgainstSourceFile from './lib/run-against-source-file'; } console.log('---------------------------'); - console.log(trie.dump().join('\n')); + const dump: string[] = []; + trie.dump((rawDomain, includeSubdomain) => { + const domain = domainToASCII(rawDomain); + if (domain) dump.push(includeSubdomain ? '.' + domain : domain); + }); + console.log(dump.join('\n')); console.log('---------------------------'); // console.log('whitelist domain suffixes:'); // console.log(whiteDomainSuffixes.join('\n')); @@ -46,16 +52,16 @@ async function writeHostsToTrie(trie: HostnameSmolTrie, hostsUrl: string, includ async function writeFiltersToTrie(trie: HostnameSmolTrie, filterUrl: string, includeThirdParty = false) { const { whiteDomainSuffixes, whiteDomains, blackDomainSuffixes, blackDomains } = await processFilterRulesWithPreload(filterUrl, [], includeThirdParty)(dummySpan); for (let i = 0, len = blackDomainSuffixes.length; i < len; i++) { - trie.add(blackDomainSuffixes[i], true); + trie.addSubdomain(blackDomainSuffixes[i]); } for (let i = 0, len = blackDomains.length; i < len; i++) { - trie.add(blackDomains[i], false); + trie.add(blackDomains[i]); } for (let i = 0, len = whiteDomainSuffixes.length; i < len; i++) { - trie.whitelist(whiteDomainSuffixes[i], true); + trie.whitelist('.' + whiteDomainSuffixes[i]); } for (let i = 0, len = whiteDomains.length; i < len; i++) { - trie.whitelist(whiteDomains[i], false); + trie.whitelist(whiteDomains[i]); } return { whiteDomainSuffixes, whiteDomains }; diff --git a/Build/validate-domestic.ts b/Build/validate-domestic.ts index a77d5e78..b267180a 100644 --- a/Build/validate-domestic.ts +++ b/Build/validate-domestic.ts @@ -4,7 +4,8 @@ import { parseFelixDnsmasqFromResp } from './lib/parse-dnsmasq'; import { $$fetch } from './lib/fetch-retry'; import runAgainstSourceFile from './lib/run-against-source-file'; import { getTopOneMillionDomains } from './validate-gfwlist'; -import { HostnameSmolTrie } from './lib/trie'; +import { HostnameSmolTrie } from 'hntrie/smol'; +import { domainToASCII } from 'node:url'; import tldts from 'tldts-experimental'; import { DOMESTICS } from '../Source/non_ip/domestic'; @@ -15,15 +16,15 @@ export async function parseDomesticList() { const resultTrie = new HostnameSmolTrie(); - topDomainTrie.dumpWithoutDot((domain) => { + topDomainTrie.dump((domain) => { const apexDomain = tldts.getDomain(domain); if (apexDomain && allChinaDomains.has(apexDomain)) { - resultTrie.add(apexDomain, false); + resultTrie.add(apexDomain); } }); - const callback = (domain: string, includeAllSubdomain: boolean) => resultTrie.whitelist(domain, includeAllSubdomain); + const callback = (domain: string, includeAllSubdomain: boolean) => resultTrie.whitelist(includeAllSubdomain ? '.' + domain : domain); // await Promise.all([ await runAgainstSourceFile( @@ -40,11 +41,11 @@ export async function parseDomesticList() { switch (domain[0]) { case '+': case '$': { - resultTrie.whitelist(domain.slice(1), true); + resultTrie.whitelist('.' + domain.slice(1)); break; } default: { - resultTrie.whitelist(domain, true); + resultTrie.whitelist('.' + domain); break; } } @@ -58,7 +59,12 @@ export async function parseDomesticList() { // } // } // ]); - const dump = resultTrie.dump(null, true); + const dump: string[] = []; + resultTrie.dump((rawDomain, includeSubdomain) => { + const domain = domainToASCII(rawDomain); + if (domain) dump.push(includeSubdomain ? '.' + domain : domain); + }); + dump.sort((a, b) => (a.length - b.length) || (a < b ? -1 : a > b ? 1 : 0)); console.log(dump.join('\n') + '\n'); diff --git a/Build/validate-gfwlist.ts b/Build/validate-gfwlist.ts index 11825357..90bea794 100644 --- a/Build/validate-gfwlist.ts +++ b/Build/validate-gfwlist.ts @@ -1,6 +1,7 @@ import { processLine } from './lib/process-line'; import { fastNormalizeDomain } from './lib/normalize-domain'; -import { HostnameSmolTrie } from './lib/trie'; +import { HostnameSmolTrie } from 'hntrie/smol'; +import { domainToASCII } from 'node:url'; import yauzl from 'yauzl-promise'; import { fetchRemoteTextByLine } from './lib/fetch-text-by-line'; import path from 'node:path'; @@ -120,8 +121,9 @@ export async function parseGfwList() { const keywordSet = new Set(); const callback = (domain: string, includeAllSubdomain: boolean) => { - gfwListTrie.whitelist(domain, includeAllSubdomain); - topDomainTrie.whitelist(domain, includeAllSubdomain); + const d = includeAllSubdomain ? '.' + domain : domain; + gfwListTrie.whitelist(d); + topDomainTrie.whitelist(d); }; await Promise.all([ runAgainstSourceFile(path.join(SOURCE_DIR, 'non_ip/global.conf'), callback, 'ruleset', keywordSet), @@ -149,17 +151,18 @@ export async function parseGfwList() { }); }); - whiteSet.forEach(domain => gfwListTrie.whitelist(domain, true)); + whiteSet.forEach(domain => gfwListTrie.whitelist(domain[0] === '.' ? domain : '.' + domain)); let dedupedGfwListSize = 0; - gfwListTrie.dump(() => dedupedGfwListSize++); + gfwListTrie.dump(() => { dedupedGfwListSize++; }); const kwfilter = createKeywordFilter([...keywordSet]); const missingTop10000Gfwed = new Set(); - topDomainTrie.dump((domain) => { - if (gfwListTrie.has(domain) && !kwfilter(domain)) { + topDomainTrie.dump((rawDomain) => { + const domain = domainToASCII(rawDomain); + if (domain && gfwListTrie.match(domain) && !kwfilter(domain)) { missingTop10000Gfwed.add(domain); } }); diff --git a/Build/validate-global-tld.ts b/Build/validate-global-tld.ts index 3eb0f649..4d438532 100644 --- a/Build/validate-global-tld.ts +++ b/Build/validate-global-tld.ts @@ -1,5 +1,6 @@ import path from 'node:path'; -import { HostnameSmolTrie } from './lib/trie'; +import { HostnameSmolTrie } from 'hntrie/smol'; +import { domainToASCII } from 'node:url'; import { OUTPUT_SURGE_DIR } from './constants/dir'; import { ICP_TLD } from './constants/domains'; import tldts from 'tldts-experimental'; @@ -22,11 +23,20 @@ import { MARKER_DOMAIN } from './constants/description'; }, 'ruleset'); await runAgainstSourceFile(path.join(OUTPUT_SURGE_DIR, 'non_ip', 'global.conf'), (domain, includeAllSubdomain) => { - trie.add(domain, includeAllSubdomain); + if (includeAllSubdomain) { + trie.addSubdomain(domain); + } else { + trie.add(domain); + } }, 'ruleset'); - ICP_TLD.forEach(tld => trie.whitelist(tld, true)); - extraWhiteTLDs.forEach(tld => trie.whitelist(tld, true)); + ICP_TLD.forEach(tld => trie.whitelist('.' + tld)); + extraWhiteTLDs.forEach(tld => trie.whitelist('.' + tld)); - console.log(trie.dump().join('\n')); + const dump: string[] = []; + trie.dump((rawDomain, includeSubdomain) => { + const domain = domainToASCII(rawDomain); + if (domain) dump.push(includeSubdomain ? '.' + domain : domain); + }); + console.log(dump.join('\n')); })(); diff --git a/package.json b/package.json index d4095153..d1458a72 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "fast-uri": "^4.1.0", "fdir": "^6.5.0", "hash-wasm": "^4.12.0", + "hntrie": "^1.0.1", "json-stringify-pretty-compact": "4.0.0", "null-prototype-object": "^1.2.7", "picocolors": "^1.1.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e9e2fc24..1251d4af 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -50,6 +50,9 @@ importers: hash-wasm: specifier: ^4.12.0 version: 4.12.0 + hntrie: + specifier: ^1.0.1 + version: 1.0.1 json-stringify-pretty-compact: specifier: 4.0.0 version: 4.0.0 @@ -1409,6 +1412,9 @@ packages: resolution: {integrity: sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==} hasBin: true + hntrie@1.0.1: + resolution: {integrity: sha512-k3OIFupPUGL1WrAiZ0Xa5on/vE/MKlub7ZQtEH0AfzfskMqq4/H6xgSqMeCXzVgNDWruP/Ff+9O5mHRzL4XZyg==} + htmlparser2@6.1.0: resolution: {integrity: sha512-gyyPk6rgonLFEDGoeRgQNaEUvdJ4ktTmmUh/h2t7s+M8oPpIPxgNACWa+6ESR57kXstwqPiCut0V8NRpcwgU7A==} @@ -3210,6 +3216,10 @@ snapshots: he@1.2.0: {} + hntrie@1.0.1: + dependencies: + foxts: 5.5.1 + htmlparser2@6.1.0: dependencies: domelementtype: 2.3.0