GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2026-07-29 08:31:46 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Refactor: migrate to hntrie

Browse Source
This commit is contained in:
SukkaW
2026-07-01 04:56:18 +08:00
parent 51f1c898a3
commit a664d2d779
15 changed files with 115 additions and 1128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Build/build-cdn-download-conf.worker.ts
View File

@@ -6,7 +6,7 @@ import { appendArrayInPlace } from 'foxts/append-array-in-place';
import { SOURCE_DIR } from './constants/dir';
import { DomainsetOutput } from './lib/rules/domainset';
import { CRASHLYTICS_WHITELIST } from './constants/reject-data-source';
import { HostnameTrie } from './lib/trie';
import { HostnameTrie } from 'hntrie';
import { $$fetch } from './lib/fetch-retry';
import { fastUri } from 'fast-uri';

2
Build/build-microsoft-cdn.worker.ts
View File

@@ -2,7 +2,7 @@ import { task } from './trace';
import { SHARED_DESCRIPTION } from './constants/description';
import { RulesetOutput } from './lib/rules/ruleset';
import { RULES, PROBE_DOMAINS, DOMAINS, DOMAIN_SUFFIXES, BLACKLIST } from './constants/microsoft-cdn';
import { HostnameSmolTrie } from './lib/trie';
import { HostnameSmolTrie } from 'hntrie/smol';
import { fetchRemoteTextByLine } from './lib/fetch-text-by-line';
import { appendArrayInPlace } from 'foxts/append-array-in-place';
import { extractDomainsFromFelixDnsmasq } from './lib/parse-dnsmasq';

3
Build/build-reject-domainset.ts
View File

@@ -263,7 +263,8 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as
}
// Deduplicate reject_extra and reject_phishing from the base reject domainset
rejectDomainsetOutput.domainTrie.dump(arg => {
rejectDomainsetOutput.domainTrie.dump((domain, includeSubdomain) => {
const arg = includeSubdomain ? '.' + domain : domain;
rejectExtraDomainsetOutput.whitelistDomain(arg);
rejectPhisingDomainsetOutput.whitelistDomain(arg);

1
Build/lib/fetch-text-by-line.ts
View File

@@ -30,6 +30,7 @@ export const createReadlineInterfaceFromResponse: ((resp: UndiciResponseData | U
}
const resultStream = webStream
// @ts-expect-error -- mismatched Node.js and web types
.pipeThrough(new TextDecoderStream())
.pipeThrough(new TextLineStream({ skipEmptyLines: processLine }));

36
Build/lib/rules/base.ts
View File

@@ -1,5 +1,6 @@
import type { Span } from '../../trace';
import { HostnameSmolTrie } from '../trie';
import { HostnameSmolTrie } from 'hntrie/smol';
import { domainToASCII } from 'node:url';
import { not, nullthrow } from 'foxts/guard';
import { fastIpVersion } from 'foxts/fast-ip-version';
import { addArrayElementsToSet } from 'foxts/add-array-elements-to-set';
@@ -20,7 +21,7 @@ export class FileOutput {
protected dataSource = new Set<string>();
public domainTrie = new HostnameSmolTrie(null);
public domainTrie = new HostnameSmolTrie();
public wildcardSet = new Set<string>();
protected domainKeywords = new Set<string>();
@@ -116,14 +117,18 @@ export class FileOutput {
for (let i = 0, len = domains.length; i < len; i++) {
const d = domains[i];
if (d !== null) {
this.domainTrie.add(d, false, null, 0);
this.domainTrie.add(d);
}
}
return this;
}
addDomainSuffix(domain: string, lineFromDot = domain[0] === '.') {
this.domainTrie.add(domain, true, null, lineFromDot ? 1 : 0);
addDomainSuffix(domain: string) {
if (domain[0] === '.') {
this.domainTrie.add(domain);
} else {
this.domainTrie.addSubdomain(domain);
}
return this;
}
@@ -179,9 +184,9 @@ export class FileOutput {
}
if (line[0] === '.') {
this.addDomainSuffix(line, true);
this.addDomainSuffix(line);
} else {
this.domainTrie.add(line, false, null, 0);
this.domainTrie.add(line);
}
}
}
@@ -210,10 +215,10 @@ export class FileOutput {
switch (type) {
case 'DOMAIN':
this.domainTrie.add(value, false, null, 0);
this.domainTrie.add(value);
break;
case 'DOMAIN-SUFFIX':
this.addDomainSuffix(value, false);
this.domainTrie.addSubdomain(value);
break;
case 'DOMAIN-KEYWORD':
this.addDomainKeyword(value);
@@ -416,9 +421,16 @@ export class FileOutput {
const strategiesLen = this.strategies.length;
this.domainTrie.dumpWithoutDot((domain, includeAllSubdomain) => {
const domainEntries: Array<[domain: string, subdomain: boolean]> = [];
this.domainTrie.dump((domain, includeSubdomain) => {
const d = domainToASCII(domain);
if (d) domainEntries.push([d, includeSubdomain]);
});
domainEntries.sort((a, b) => (a[0].length - b[0].length) || (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0));
for (let j = 0, entriesLen = domainEntries.length; j < entriesLen; j++) {
const [domain, includeAllSubdomain] = domainEntries[j];
if (kwfilter(domain)) {
return;
continue;
}
for (let i = 0; i < strategiesLen; i++) {
@@ -429,7 +441,7 @@ export class FileOutput {
strategy.writeDomain(domain);
}
}
}, true);
}
// Now, we whitelisted out DOMAIN-KEYWORD
const whiteKwfilter = createKeywordFilter(Array.from(this.whitelistKeywords));

411
Build/lib/trie.test.ts
View File

@@ -1,411 +0,0 @@
import { describe, it } from 'mocha';
import { expect } from 'earl';
import { HostnameSmolTrie, HostnameTrie } from './trie';
function createTrie<Meta = any>(from: string[] | Set<string> | null, smolTree: true): HostnameSmolTrie<Meta>;
function createTrie<Meta = any>(from?: string[] | Set<string> | null, smolTree?: false): HostnameTrie<Meta>;
function createTrie<_Meta = any>(from?: string[] | Set<string> | null, smolTree = true) {
if (smolTree) {
return new HostnameSmolTrie(from);
}
return new HostnameTrie(from);
};
// describe('hostname to tokens', () => {
// it('should split hostname into tokens.', () => {
// expect(hostnameToTokens('.blog.skk.moe')).toEqual([
// '.',
// 'blog',
// '.',
// 'skk',
// '.',
// 'moe'
// ]);
// expect(hostnameToTokens('blog.skk.moe')).toEqual([
// 'blog',
// '.',
// 'skk',
// '.',
// 'moe'
// ]);
// expect(hostnameToTokens('skk.moe')).toEqual([
// 'skk',
// '.',
// 'moe'
// ]);
// expect(hostnameToTokens('moe')).toEqual([
// 'moe'
// ]);
// });
// });
describe('Trie', () => {
it('should be possible to add domains to a Trie.', () => {
const trie = createTrie(null, false);
trie.add('a.skk.moe');
trie.add('skk.moe');
trie.add('anotherskk.moe');
expect(trie.size).toEqual(3);
expect(trie.has('a.skk.moe')).toEqual(true);
expect(trie.has('skk.moe')).toEqual(true);
expect(trie.has('anotherskk.moe')).toEqual(true);
expect(trie.has('example.com')).toEqual(false);
expect(trie.has('skk.mo')).toEqual(false);
expect(trie.has('another.skk.moe')).toEqual(false);
});
it('adding the same item several times should not increase size.', () => {
const trie = createTrie(null, false);
trie.add('skk.moe');
trie.add('blog.skk.moe');
// eslint-disable-next-line sukka/no-element-overwrite -- deliberately do testing
trie.add('skk.moe');
expect(trie.size).toEqual(2);
expect(trie.has('skk.moe')).toEqual(true);
});
it('should be possible to set the null sequence.', () => {
const trie = createTrie(null, false);
trie.add('');
expect(trie.has('')).toEqual(true);
const trie2 = createTrie(null, true);
trie2.add('');
expect(trie2.has('')).toEqual(true);
});
it('should be possible to delete items.', () => {
const trie = createTrie(null, false);
trie.add('skk.moe');
trie.add('blog.skk.moe');
trie.add('example.com');
trie.add('moe.sb');
expect(trie.delete('no-match.com')).toEqual(false);
expect(trie.delete('example.org')).toEqual(false);
expect(trie.delete('skk.moe')).toEqual(true);
expect(trie.has('skk.moe')).toEqual(false);
expect(trie.has('moe.sb')).toEqual(true);
expect(trie.size).toEqual(3);
expect(trie.delete('example.com')).toEqual(true);
expect(trie.size).toEqual(2);
expect(trie.delete('moe.sb')).toEqual(true);
expect(trie.size).toEqual(1);
});
it('should be possible to check the existence of a sequence in the Trie.', () => {
const trie = createTrie(null, true);
trie.add('example.org.skk.moe');
expect(trie.has('example.org.skk.moe')).toEqual(true);
expect(trie.has('skk.moe')).toEqual(false);
expect(trie.has('example.org')).toEqual(false);
expect(trie.has('')).toEqual(false);
});
it('should be possible to retrieve items matching the given prefix.', () => {
const trie = createTrie(null, false);
trie.add('example.com');
trie.add('blog.example.com');
trie.add('cdn.example.com');
trie.add('example.org');
expect(trie.find('example.com')).toEqual(['example.com', 'cdn.example.com', 'blog.example.com']);
expect(trie.find('com')).toEqual(['example.com', 'cdn.example.com', 'blog.example.com']);
expect(trie.find('.example.com')).toEqual(['cdn.example.com', 'blog.example.com']);
expect(trie.find('org')).toEqual(['example.org']);
expect(trie.find('example.net')).toEqual([]);
expect(trie.dump()).toEqual(['example.org', 'example.com', 'cdn.example.com', 'blog.example.com']);
});
it('should be possible to retrieve items matching the given prefix even with a smol trie', () => {
const trie = createTrie(null, true);
trie.add('.example.com');
trie.add('example.com');
trie.add('blog.example.com');
trie.add('cdn.example.com');
trie.add('example.org');
expect(trie.find('example.com')).toEqual(['.example.com']);
expect(trie.find('com')).toEqual(['.example.com']);
expect(trie.find('.example.com')).toEqual(['.example.com']);
expect(trie.find('org')).toEqual(['example.org']);
expect(trie.find('example.net')).toEqual([]);
expect(trie.dump()).toEqual(['example.org', '.example.com']);
});
it('should be possible to create a trie from an arbitrary iterable.', () => {
let trie = createTrie(['skk.moe', 'blog.skk.moe'], false);
expect(trie.size).toEqual(2);
expect(trie.has('skk.moe')).toEqual(true);
trie = createTrie(new Set(['skk.moe', 'example.com']), false);
expect(trie.size).toEqual(2);
expect(trie.has('skk.moe')).toEqual(true);
});
});
describe('surge domainset dedupe', () => {
it('should not remove same entry', () => {
const trie = createTrie(['.skk.moe', 'noc.one'], false);
expect(trie.find('.skk.moe')).toEqual(['.skk.moe']);
expect(trie.find('noc.one')).toEqual(['noc.one']);
});
it('should match subdomain - 1', () => {
const trie = createTrie(['www.noc.one', 'www.sukkaw.com', 'blog.skk.moe', 'image.cdn.skk.moe', 'cdn.sukkaw.net'], false);
expect(trie.find('.skk.moe')).toEqual(['image.cdn.skk.moe', 'blog.skk.moe']);
expect(trie.find('.sukkaw.com')).toEqual(['www.sukkaw.com']);
});
it('should match subdomain - 2', () => {
const trie = createTrie(['www.noc.one', 'www.sukkaw.com', '.skk.moe', 'blog.skk.moe', 'image.cdn.skk.moe', 'cdn.sukkaw.net'], false);
expect(trie.find('.skk.moe')).toEqual(['.skk.moe', 'image.cdn.skk.moe', 'blog.skk.moe']);
expect(trie.find('.sukkaw.com')).toEqual(['www.sukkaw.com']);
});
it('should not remove non-subdomain', () => {
const trie = createTrie(['skk.moe', 'sukkaskk.moe'], false);
expect(trie.find('.skk.moe')).toEqual([]);
});
});
describe('smol tree', () => {
it('should init tree', () => {
const trie = createTrie([
'skk.moe',
'anotherskk.moe',
'blog.anotherskk.moe',
'blog.skk.moe',
'.cdn.local',
'blog.img.skk.local',
'img.skk.local'
], true);
expect(trie.dump()).toEqual([
'img.skk.local',
'blog.img.skk.local',
'.cdn.local',
'anotherskk.moe',
'blog.anotherskk.moe',
'skk.moe',
'blog.skk.moe'
]);
});
it('should create simple tree - 1', () => {
const trie = createTrie([
'.skk.moe', 'blog.skk.moe', '.cdn.skk.moe', 'skk.moe',
'www.noc.one', 'cdn.noc.one',
'.blog.sub.example.com', 'sub.example.com', 'cdn.sub.example.com', '.sub.example.com'
], true);
expect(trie.dump()).toEqual([
'.sub.example.com',
'cdn.noc.one',
'www.noc.one',
'.skk.moe'
]);
});
it('should create simple tree - 2', () => {
const trie = createTrie([
'.skk.moe', 'blog.skk.moe', '.cdn.skk.moe', 'skk.moe'
], true);
expect(trie.dump()).toEqual([
'.skk.moe'
]);
});
it('should create simple tree - 3', () => {
const trie = createTrie([
'.blog.sub.example.com', 'cdn.sub.example.com', '.sub.example.com'
], true);
expect(trie.dump()).toEqual([
'.sub.example.com'
]);
trie.add('.sub.example.com');
expect(trie.dump()).toEqual([
'.sub.example.com'
]);
});
it('should create simple tree - 3', () => {
const trie = createTrie([
'commercial.shouji.360.cn',
'act.commercial.shouji.360.cn',
'cdn.creative.medialytics.com',
'px.cdn.creative.medialytics.com'
], true);
expect(trie.dump()).toEqual([
'cdn.creative.medialytics.com',
'px.cdn.creative.medialytics.com',
'commercial.shouji.360.cn',
'act.commercial.shouji.360.cn'
]);
});
it('should dedupe subdomain properly', () => {
const trie = createTrie([
'skk.moe',
'anotherskk.moe',
'blog.anotherskk.moe',
'blog.skk.moe'
], true);
expect(trie.dump()).toEqual([
'anotherskk.moe',
'blog.anotherskk.moe',
'skk.moe',
'blog.skk.moe'
]);
});
it('should effctly whitelist domains', () => {
const trie = createTrie([
'skk.moe',
'anotherskk.moe',
'blog.anotherskk.moe',
'blog.skk.moe',
'.cdn.local',
'blog.img.skk.local',
'img.skk.local'
], true);
trie.whitelist('.skk.moe');
expect(trie.dump()).toEqual([
'img.skk.local',
'blog.img.skk.local',
'.cdn.local',
'anotherskk.moe',
'blog.anotherskk.moe'
]);
trie.whitelist('anotherskk.moe');
expect(trie.dump()).toEqual([
'img.skk.local',
'blog.img.skk.local',
'.cdn.local',
'blog.anotherskk.moe'
]);
trie.add('anotherskk.moe');
trie.whitelist('.anotherskk.moe');
expect(trie.dump()).toEqual([
'img.skk.local',
'blog.img.skk.local',
'.cdn.local'
]);
trie.whitelist('img.skk.local');
expect(trie.dump()).toEqual([
'blog.img.skk.local',
'.cdn.local'
]);
trie.whitelist('cdn.local');
expect(trie.dump()).toEqual([
'blog.img.skk.local'
]);
trie.whitelist('.skk.local');
expect(trie.dump()).toEqual([]);
});
it('should whitelist trie correctly', () => {
const trie = createTrie([
'.t.co',
't.co',
'example.t.co',
'.skk.moe',
'blog.cdn.example.com',
'cdn.example.com'
], true);
expect(trie.dump()).toEqual([
'cdn.example.com', 'blog.cdn.example.com',
'.skk.moe',
'.t.co'
]);
trie.whitelist('.t.co');
expect(trie.dump()).toEqual([
'cdn.example.com', 'blog.cdn.example.com', '.skk.moe'
]);
trie.whitelist('skk.moe');
expect(trie.dump()).toEqual(['cdn.example.com', 'blog.cdn.example.com']);
trie.whitelist('cdn.example.com');
expect(trie.dump()).toEqual(['blog.cdn.example.com']);
});
it('contains - normal', () => {
const trie = createTrie([
'skk.moe',
'anotherskk.moe',
'blog.anotherskk.moe',
'blog.skk.moe'
], true);
expect(trie.contains('skk.moe')).toEqual(true);
expect(trie.contains('blog.skk.moe')).toEqual(true);
expect(trie.contains('anotherskk.moe')).toEqual(true);
expect(trie.contains('blog.anotherskk.moe')).toEqual(true);
expect(trie.contains('example.com')).toEqual(false);
expect(trie.contains('blog.example.com')).toEqual(false);
expect(trie.contains('skk.mo')).toEqual(false);
expect(trie.contains('cdn.skk.moe')).toEqual(false);
});
it('contains - subdomain', () => {
const trie = createTrie([
'index.rubygems.org'
], true);
expect(trie.contains('rubygems.org')).toEqual(false);
expect(trie.contains('index.rubygems.org')).toEqual(true);
expect(trie.contains('sub.index.rubygems.org')).toEqual(false);
});
it('contains - include subdomains', () => {
const trie = createTrie([
'.skk.moe'
], true);
expect(trie.contains('skk.moe')).toEqual(true);
expect(trie.contains('blog.skk.moe')).toEqual(true);
expect(trie.contains('image.cdn.skk.moe')).toEqual(true);
expect(trie.contains('example.com')).toEqual(false);
expect(trie.contains('blog.example.com')).toEqual(false);
expect(trie.contains('skk.mo')).toEqual(false);
});
});

665
Build/lib/trie.ts
View File

@@ -1,665 +0,0 @@
/**
* Hostbane-Optimized Trie based on Mnemonist Trie
*/
import { fastStringCompare } from 'foxts/fast-string-compare';
import util from 'node:util';
import { noop } from 'foxts/noop';
import { fastStringArrayJoin } from 'foxts/fast-string-array-join';
import { deleteBit, getBit, missingBit, setBit } from 'foxts/bitwise';
import { domainToASCII } from 'node:url';
const START = 1 << 1;
const INCLUDE_ALL_SUBDOMAIN = 1 << 2;
type TrieNode<Meta = any> = [
/** end, includeAllSubdomain (.example.org, ||example.com) */ flag: number,
/** parent */ TrieNode | null,
/** children */ Map<string, TrieNode>,
/** token */ token: string,
/** meta */ Meta
];
function deepTrieNodeToJSON<Meta = unknown>(node: TrieNode,
unpackMeta: ((meta?: Meta) => string) | undefined) {
const obj: Record<string, unknown> = {
['[start]']: getBit(node[0], START),
['[subdomain]']: getBit(node[0], INCLUDE_ALL_SUBDOMAIN)
};
if (node[4] != null) {
if (unpackMeta) {
obj['[meta]'] = unpackMeta(node[4]);
} else {
obj['[meta]'] = node[4];
}
}
node[2].forEach((value, key) => {
obj[key] = deepTrieNodeToJSON(value, unpackMeta);
});
return obj;
}
const createNode = <Meta = unknown>(token: string, parent: TrieNode | null = null): TrieNode => [1, parent, new Map<string, TrieNode>(), token, null] as TrieNode<Meta>;
function hostnameToTokens(hostname: string, hostnameFromIndex: number): string[] {
const tokens = hostname.split('.');
const results: string[] = [];
let token = '';
for (let i = hostnameFromIndex, l = tokens.length; i < l; i++) {
token = tokens[i];
if (token.length > 0) {
results.push(token);
} else {
throw new TypeError(JSON.stringify({ hostname, hostnameFromIndex }, null, 2));
}
}
return results;
}
function walkHostnameTokens(hostname: string, onToken: (token: string) => boolean | null, hostnameFromIndex: number): boolean | null {
const tokens = hostname.split('.');
const l = tokens.length - 1;
// we are at the first of hostname, no splitor there
let token = '';
for (let i = l; i >= hostnameFromIndex; i--) {
token = tokens[i];
if (token.length > 0) {
const t = onToken(token);
if (t === null) {
return null;
}
// if the callback returns true, we should skip the rest
if (t) {
return true;
}
}
}
return false;
}
interface FindSingleChildLeafResult<Meta> {
node: TrieNode<Meta>,
toPrune: TrieNode<Meta> | null,
tokenToPrune: string | null,
parent: TrieNode<Meta>
}
abstract class Triebase<Meta = unknown> {
protected readonly $root: TrieNode<Meta> = createNode('$root');
protected $size = 0;
get root() {
return this.$root;
}
constructor(from?: string[] | Set<string> | null) {
// Actually build trie
if (Array.isArray(from)) {
for (let i = 0, l = from.length; i < l; i++) {
this.add(from[i]);
}
} else if (from) {
from.forEach((value) => this.add(value));
}
}
public abstract add(suffix: string, includeAllSubdomain?: boolean, meta?: Meta, hostnameFromIndex?: number): void;
protected walkIntoLeafWithTokens(
tokens: string[],
onLoop: (node: TrieNode, parent: TrieNode, token: string) => void = noop
) {
let node: TrieNode = this.$root;
let parent: TrieNode = node;
let token: string;
let child: Map<string, TrieNode<Meta>> = node[2];
// reverse lookup from end to start
for (let i = tokens.length - 1; i >= 0; i--) {
token = tokens[i];
// if (token === '') {
// break;
// }
parent = node;
child = node[2];
// cache node index access is 20% faster than direct access when doing twice
if (child.has(token)) {
node = child.get(token)!;
} else {
return null;
}
onLoop(node, parent, token);
}
return { node, parent };
};
protected walkIntoLeafWithSuffix(
suffix: string,
hostnameFromIndex: number,
onLoop: (node: TrieNode, parent: TrieNode, token: string) => void = noop
) {
let node: TrieNode = this.$root;
let parent: TrieNode = node;
let child: Map<string, TrieNode<Meta>> = node[2];
const onToken = (token: string) => {
// if (token === '') {
// return true;
// }
parent = node;
child = node[2];
if (child.has(token)) {
node = child.get(token)!;
} else {
return null;
}
onLoop(node, parent, token);
return false;
};
if (walkHostnameTokens(suffix, onToken, hostnameFromIndex) === null) {
return null;
}
return { node, parent };
};
public contains(suffix: string, includeAllSubdomain = suffix[0] === '.'): boolean {
const hostnameFromIndex = suffix[0] === '.' ? 1 : 0;
let node: TrieNode = this.$root;
// let parent: TrieNode = node;
let child: Map<string, TrieNode<Meta>> = node[2];
let result = false;
const onToken = (token: string) => {
// if (token === '') {
// return true;
// }
// parent = node;
child = node[2];
if (child.has(token)) {
node = child.get(token)!;
} else {
if (getBit(node[0], INCLUDE_ALL_SUBDOMAIN)) {
result = true;
}
return null;
}
return false;
};
if (walkHostnameTokens(suffix, onToken, hostnameFromIndex) === null) {
return result;
}
if (includeAllSubdomain) return getBit(node[0], INCLUDE_ALL_SUBDOMAIN);
return getBit(node[0], START);
// if (res === null) return false;
// if (includeAllSubdomain) return getBit(res.node[0], INCLUDE_ALL_SUBDOMAIN);
// return true;
};
private static bfsResults: [node: TrieNode | null, suffix: string[]] = [null, []];
private static dfs<Meta>(this: void, nodeStack: Array<TrieNode<Meta>>, suffixStack: string[][]) {
const node = nodeStack.pop()!;
const suffix = suffixStack.pop()!;
node[2].forEach((childNode, k) => {
// Pushing the child node to the stack for next iteration of DFS
nodeStack.push(childNode);
suffixStack.push([k, ...suffix]);
});
Triebase.bfsResults[0] = node;
Triebase.bfsResults[1] = suffix;
return Triebase.bfsResults;
}
private static dfsWithSort<Meta>(this: void, nodeStack: Array<TrieNode<Meta>>, suffixStack: string[][]) {
const node = nodeStack.pop()!;
const suffix = suffixStack.pop()!;
const child = node[2];
if (child.size) {
const keys = Array.from(child.keys()).sort(Triebase.compare);
for (let i = 0, l = keys.length; i < l; i++) {
const key = keys[i];
const childNode = child.get(key)!;
// Pushing the child node to the stack for next iteration of DFS
nodeStack.push(childNode);
suffixStack.push([key, ...suffix]);
}
}
Triebase.bfsResults[0] = node;
Triebase.bfsResults[1] = suffix;
return Triebase.bfsResults;
}
private walk(
onMatches: (suffix: string[], subdomain: boolean, meta: Meta) => void,
withSort = false,
initialNode = this.$root,
initialSuffix: string[] = []
) {
const dfsImpl = withSort ? Triebase.dfsWithSort : Triebase.dfs;
const nodeStack: Array<TrieNode<Meta>> = [initialNode];
// Resolving initial string (begin the start of the stack)
const suffixStack: string[][] = [initialSuffix];
let node: TrieNode<Meta> = initialNode;
do {
const r = dfsImpl(nodeStack, suffixStack);
node = r[0]!;
const suffix = r[1];
// If the node is a sentinel, we push the suffix to the results
if (getBit(node[0], START)) {
onMatches(suffix, getBit(node[0], INCLUDE_ALL_SUBDOMAIN), node[4]);
}
} while (nodeStack.length);
};
static compare(this: void, a: string, b: string) {
if (a === b) return 0;
return (a.length - b.length) || fastStringCompare(a, b);
}
protected getSingleChildLeaf(tokens: string[]): FindSingleChildLeafResult<Meta> | null {
let toPrune: TrieNode | null = null;
let tokenToPrune: string | null = null;
const onLoop = (node: TrieNode, parent: TrieNode, token: string) => {
// Keeping track of a potential branch to prune
const child = node[2];
const childSize = child.size + (getBit(node[0], INCLUDE_ALL_SUBDOMAIN) ? 1 : 0);
if (toPrune !== null) { // the most near branch that could potentially being pruned
if (childSize >= 1) {
// The branch has some children, the branch need retain.
// And we need to abort prune that parent branch, so we set it to null
toPrune = null;
tokenToPrune = null;
}
} else if (childSize < 1) {
// There is only one token child, or no child at all, we can prune it safely
// It is now the top-est branch that could potentially being pruned
toPrune = parent;
tokenToPrune = token;
}
};
const res = this.walkIntoLeafWithTokens(tokens, onLoop);
if (res === null) return null;
return { node: res.node, toPrune, tokenToPrune, parent: res.parent };
};
/**
* Method used to retrieve every item in the trie with the given prefix.
*/
public find(
inputSuffix: string,
subdomainOnly = inputSuffix[0] === '.',
hostnameFromIndex = inputSuffix[0] === '.' ? 1 : 0
// /** @default true */ includeEqualWithSuffix = true
): string[] {
const inputTokens = hostnameToTokens(inputSuffix, hostnameFromIndex);
const res = this.walkIntoLeafWithTokens(inputTokens);
if (res === null) return [];
const results: string[] = [];
const onMatches = subdomainOnly
? (suffix: string[], subdomain: boolean) => { // fast path (default option)
const d = domainToASCII(fastStringArrayJoin(suffix, '.'));
if (!subdomain && subStringEqual(inputSuffix, d, 1)) return;
results.push(subdomain ? '.' + d : d);
}
: (suffix: string[], subdomain: boolean) => { // fast path (default option)
const d = domainToASCII(fastStringArrayJoin(suffix, '.'));
results.push(subdomain ? '.' + d : d);
};
this.walk(
onMatches,
false,
res.node, // Performing DFS from prefix
inputTokens
);
return results;
};
/**
* Method used to delete a prefix from the trie.
*/
public remove(suffix: string): boolean {
const res = this.getSingleChildLeaf(hostnameToTokens(suffix, 0));
if (res === null) return false;
if (missingBit(res.node[0], START)) return false;
this.$size--;
const { node, toPrune, tokenToPrune } = res;
if (tokenToPrune && toPrune) {
toPrune[2].delete(tokenToPrune);
} else {
node[0] = deleteBit(node[0], START);
}
return true;
};
// eslint-disable-next-line @typescript-eslint/unbound-method -- safe
public delete = this.remove;
/**
* Method used to assert whether the given prefix exists in the Trie.
*/
public has(suffix: string, includeAllSubdomain = suffix[0] === '.'): boolean {
const hostnameFromIndex = suffix[0] === '.' ? 1 : 0;
const res = this.walkIntoLeafWithSuffix(suffix, hostnameFromIndex);
if (res === null) return false;
if (missingBit(res.node[0], START)) return false;
if (includeAllSubdomain) return getBit(res.node[0], INCLUDE_ALL_SUBDOMAIN);
return true;
};
public dumpWithoutDot(onSuffix: (suffix: string, subdomain: boolean) => void, withSort = false) {
const handleSuffix = (suffix: string[], subdomain: boolean) => {
onSuffix(domainToASCII(fastStringArrayJoin(suffix, '.')), subdomain);
};
this.walk(handleSuffix, withSort);
}
public dump(onSuffix: (suffix: string) => void, withSort?: boolean): void;
public dump(onSuffix?: null, withSort?: boolean): string[];
public dump(onSuffix?: ((suffix: string) => void) | null, withSort = false): string[] | void {
const results: string[] = [];
const handleSuffix = onSuffix
? (suffix: string[], subdomain: boolean) => {
const d = domainToASCII(fastStringArrayJoin(suffix, '.'));
onSuffix(subdomain ? '.' + d : d);
}
: (suffix: string[], subdomain: boolean) => {
const d = domainToASCII(fastStringArrayJoin(suffix, '.'));
results.push(subdomain ? '.' + d : d);
};
this.walk(handleSuffix, withSort);
return results;
};
public dumpMeta(onMeta: (meta: Meta) => void, withSort?: boolean): void;
public dumpMeta(onMeta?: null, withSort?: boolean): Meta[];
public dumpMeta(onMeta?: ((meta: Meta) => void) | null, withSort = false): Meta[] | void {
const results: Meta[] = [];
const handleMeta = onMeta
? (_suffix: string[], _subdomain: boolean, meta: Meta) => onMeta(meta)
: (_suffix: string[], _subdomain: boolean, meta: Meta) => results.push(meta);
this.walk(handleMeta, withSort);
return results;
};
public dumpWithMeta(onSuffix: (suffix: string, meta: Meta | undefined) => void, withSort?: boolean): void;
public dumpWithMeta(onMeta?: null, withSort?: boolean): Array<[string, Meta | undefined]>;
public dumpWithMeta(onSuffix?: ((suffix: string, meta: Meta | undefined) => void) | null, withSort = false): Array<[string, Meta | undefined]> | void {
const results: Array<[string, Meta | undefined]> = [];
const handleSuffix = onSuffix
? (suffix: string[], subdomain: boolean, meta: Meta | undefined) => {
const d = domainToASCII(fastStringArrayJoin(suffix, '.'));
return onSuffix(subdomain ? '.' + d : d, meta);
}
: (suffix: string[], subdomain: boolean, meta: Meta | undefined) => {
const d = domainToASCII(fastStringArrayJoin(suffix, '.'));
results.push([subdomain ? '.' + d : d, meta]);
};
this.walk(handleSuffix, withSort);
return results;
};
public inspect(depth: number, unpackMeta?: (meta?: Meta) => any) {
return fastStringArrayJoin(
JSON.stringify(deepTrieNodeToJSON(this.$root, unpackMeta), null, 2).split('\n').map((line) => ' '.repeat(depth) + line),
'\n'
);
}
public [util.inspect.custom](depth: number) {
return this.inspect(depth);
};
public merge(trie: Triebase<Meta>) {
const handleSuffix = (suffix: string[], subdomain: boolean, meta: Meta) => {
this.add(fastStringArrayJoin(suffix, '.'), subdomain, meta);
};
trie.walk(handleSuffix);
return this;
}
}
export class HostnameSmolTrie<Meta = unknown> extends Triebase<Meta> {
public smolTree = true;
add(suffix: string, includeAllSubdomain = suffix[0] === '.', meta?: Meta, hostnameFromIndex = suffix[0] === '.' ? 1 : 0): void {
let node: TrieNode<Meta> = this.$root;
let curNodeChildren: Map<string, TrieNode<Meta>> = node[2];
const onToken = (token: string) => {
curNodeChildren = node[2];
if (curNodeChildren.has(token)) {
node = curNodeChildren.get(token)!;
// During the adding of `[start]blog|.skk.moe` and find out that there is a `[start].skk.moe` in the trie, skip adding the rest of the node
if (getBit(node[0], INCLUDE_ALL_SUBDOMAIN)) {
return true;
}
} else {
const newNode = createNode(token, node);
curNodeChildren.set(token, newNode);
node = newNode;
}
return false;
};
// When walkHostnameTokens returns true, we should skip the rest
if (walkHostnameTokens(suffix, onToken, hostnameFromIndex)) {
return;
}
// If we are in smolTree mode, we need to do something at the end of the loop
if (includeAllSubdomain) {
// Trying to add `[.]sub.example.com` where there is already a `blog.sub.example.com` in the trie
// Make sure parent `[start]sub.example.com` (without dot) is removed (SETINEL to false)
// (/** parent */ node[2]!)[0] = false;
// Removing the rest of the parent's child nodes
node[2].clear();
// The SENTINEL of this node will be set to true at the end of the function, so we don't need to set it here
// we can use else-if here, because the children is now empty, we don't need to check the leading "."
} else if (getBit(node[0], INCLUDE_ALL_SUBDOMAIN)) {
// Trying to add `example.com` when there is already a `.example.com` in the trie
// No need to increment size and set SENTINEL to true (skip this "new" item)
return;
}
node[0] = setBit(node[0], START);
if (includeAllSubdomain) {
node[0] = setBit(node[0], INCLUDE_ALL_SUBDOMAIN);
} else {
node[0] = deleteBit(node[0], INCLUDE_ALL_SUBDOMAIN);
}
node[4] = meta!;
}
public whitelist(suffix: string, includeAllSubdomain = suffix[0] === '.', hostnameFromIndex = suffix[0] === '.' ? 1 : 0) {
const tokens = hostnameToTokens(suffix, hostnameFromIndex);
const res = this.getSingleChildLeaf(tokens);
if (res === null) return;
const { node, toPrune, tokenToPrune } = res;
// Trying to whitelist `[start].sub.example.com` where there might already be a `[start]blog.sub.example.com` in the trie
if (includeAllSubdomain) {
// If there is a `[start]sub.example.com` here, remove it
node[0] = deleteBit(node[0], INCLUDE_ALL_SUBDOMAIN);
// Removing all the child nodes by empty the children
node[2].clear();
// we do not remove sub.example.com for now, we will do that later
} else {
// Trying to whitelist `example.com` when there is already a `.example.com` in the trie
node[0] = deleteBit(node[0], INCLUDE_ALL_SUBDOMAIN);
}
if (includeAllSubdomain) {
node[1]?.[2].delete(node[3]);
} else if (missingBit(node[0], START) && node[1]) {
return;
}
if (toPrune && tokenToPrune) {
toPrune[2].delete(tokenToPrune);
} else {
node[0] = deleteBit(node[0], START);
}
cleanUpEmptyTrailNode(node);
};
}
function cleanUpEmptyTrailNode(node: TrieNode<unknown>) {
let current = node;
while (
// the current node is not an "end node", a.k.a. not the start of a domain
missingBit(current[0], START)
// also no leading "." (no subdomain)
&& missingBit(current[0], INCLUDE_ALL_SUBDOMAIN)
// child is empty
&& current[2].size === 0
// has parent: we need to delete the current node from the parent
// we also need to clean up the parent node
&& current[1]
) {
current[1][2].delete(current[3]);
current = current[1];
}
}
export class HostnameTrie<Meta = unknown> extends Triebase<Meta> {
get size() {
return this.$size;
}
add(suffix: string, includeAllSubdomain = suffix[0] === '.', meta?: Meta, hostnameFromIndex = suffix[0] === '.' ? 1 : 0): void {
let node: TrieNode<Meta> = this.$root;
let child: Map<string, TrieNode<Meta>> = node[2];
const onToken = (token: string) => {
child = node[2];
if (child.has(token)) {
node = child.get(token)!;
} else {
const newNode = createNode(token, node);
child.set(token, newNode);
node = newNode;
}
return false;
};
// When walkHostnameTokens returns true, we should skip the rest
if (walkHostnameTokens(suffix, onToken, hostnameFromIndex)) {
return;
}
// if same entry has been added before, skip
if (getBit(node[0], START)) {
return;
}
this.$size++;
node[0] = setBit(node[0], START);
if (includeAllSubdomain) {
node[0] = setBit(node[0], INCLUDE_ALL_SUBDOMAIN);
} else {
node[0] = deleteBit(node[0], INCLUDE_ALL_SUBDOMAIN);
}
node[4] = meta!;
}
}
// function deepEqualArray(a: string[], b: string[]) {
// let len = a.length;
// if (len !== b.length) return false;
// while (len--) {
// if (a[len] !== b[len]) return false;
// }
// return true;
// };
function subStringEqual(needle: string, haystack: string, needleIndex = 0) {
for (let i = 0, l = haystack.length; i < l; i++) {
if (needle[i + needleIndex] !== haystack[i]) return false;
}
return true;
}

21
Build/tools-dedupe-src.ts
View File

@@ -4,7 +4,7 @@ import fsp from 'node:fs/promises';
import { SOURCE_DIR } from './constants/dir';
import { readFileByLine } from './lib/fetch-text-by-line';
import { processLine } from './lib/process-line';
import { HostnameSmolTrie } from './lib/trie';
import { HostnameSmolTrie } from 'hntrie/smol';
import { task } from './trace';
const ENFORCED_WHITELIST = [
@@ -48,6 +48,15 @@ task(require.main === module, __filename)(async (span) => {
await Promise.all(files.map(file => span.traceChildAsync('dedupe ' + file, () => dedupeFile(file, whiteTrie))));
});
function trieHasEntry(trie: HostnameSmolTrie, line: string): boolean {
if (line[0] === '.') return trie.hasSubdomain(line.slice(1));
return trie.has(line) || trie.hasSubdomain(line);
}
function trieContains(trie: HostnameSmolTrie, line: string): boolean {
return trie.match(line);
}
async function dedupeFile(file: string, whitelist: HostnameSmolTrie) {
const result: string[] = [];
@@ -55,8 +64,7 @@ async function dedupeFile(file: string, whitelist: HostnameSmolTrie) {
let line: string | null = '';
// eslint-disable-next-line @typescript-eslint/unbound-method -- .call
let trieHasOrContains = HostnameSmolTrie.prototype.has;
let trieCheck = trieHasEntry;
for await (const l of readFileByLine(file)) {
line = processLine(l);
@@ -66,19 +74,18 @@ async function dedupeFile(file: string, whitelist: HostnameSmolTrie) {
return;
}
if (l.startsWith('# $ dedupe_use_trie_contains')) {
// eslint-disable-next-line @typescript-eslint/unbound-method -- .call
trieHasOrContains = HostnameSmolTrie.prototype.contains;
trieCheck = trieContains;
}
result.push(l); // keep all comments and blank lines
continue;
}
if (trieHasOrContains.call(trie, line)) {
if (trieCheck(trie, line)) {
continue; // drop duplicate
}
if (whitelist.has(line)) {
if (trieHasEntry(whitelist, line)) {
continue; // drop whitelisted items
}

14
Build/tools-lum-apex-domains.ts
View File

@@ -1,6 +1,7 @@
import { fetchRemoteTextByLine } from './lib/fetch-text-by-line';
import tldts from 'tldts-experimental';
import { HostnameSmolTrie } from './lib/trie';
import { HostnameSmolTrie } from 'hntrie/smol';
import { domainToASCII } from 'node:url';
import path from 'node:path';
import { SOURCE_DIR } from './constants/dir';
import runAgainstSourceFile from './lib/run-against-source-file';
@@ -32,11 +33,16 @@ import runAgainstSourceFile from './lib/run-against-source-file';
}
await runAgainstSourceFile(path.join(SOURCE_DIR, 'domainset', 'reject.conf'), (domain, includeAllSubDomain) => {
trie.whitelist(domain, includeAllSubDomain);
trie.whitelist(includeAllSubDomain ? '.' + domain : domain);
}, 'domainset');
await runAgainstSourceFile(path.join(SOURCE_DIR, 'non_ip', 'reject.conf'), (domain, includeAllSubDomain) => {
trie.whitelist(domain, includeAllSubDomain);
trie.whitelist(includeAllSubDomain ? '.' + domain : domain);
}, 'ruleset');
console.log(trie.dump().map(i => '.' + i).join('\n'));
const dump: string[] = [];
trie.dump((rawDomain) => {
const domain = domainToASCII(rawDomain);
if (domain) dump.push(domain);
});
console.log(dump.map(i => '.' + i).join('\n'));
})();

20
Build/tools-migrate-domains.ts
View File

@@ -2,7 +2,8 @@
import path from 'node:path';
import { processFilterRulesWithPreload } from './lib/parse-filter/filters';
import { processHosts } from './lib/parse-filter/hosts';
import { HostnameSmolTrie } from './lib/trie';
import { HostnameSmolTrie } from 'hntrie/smol';
import { domainToASCII } from 'node:url';
import { dummySpan } from './trace';
import { SOURCE_DIR } from './constants/dir';
import { PREDEFINED_WHITELIST } from './constants/reject-data-source';
@@ -15,7 +16,7 @@ import runAgainstSourceFile from './lib/run-against-source-file';
// const { whiteDomainSuffixes, whiteDomains } = await writeFiltersToTrie(trie, 'https://cdn.jsdelivr.net/gh/Perflyst/PiHoleBlocklist@master/SmartTV-AGH.txt', true);
const callback = (domain: string, includeAllSubDomain: boolean) => {
trie.whitelist(domain, includeAllSubDomain);
trie.whitelist(includeAllSubDomain ? '.' + domain : domain);
};
await runAgainstSourceFile(path.join(SOURCE_DIR, 'domainset', 'reject.conf'), callback, 'domainset');
@@ -26,7 +27,12 @@ import runAgainstSourceFile from './lib/run-against-source-file';
}
console.log('---------------------------');
console.log(trie.dump().join('\n'));
const dump: string[] = [];
trie.dump((rawDomain, includeSubdomain) => {
const domain = domainToASCII(rawDomain);
if (domain) dump.push(includeSubdomain ? '.' + domain : domain);
});
console.log(dump.join('\n'));
console.log('---------------------------');
// console.log('whitelist domain suffixes:');
// console.log(whiteDomainSuffixes.join('\n'));
@@ -46,16 +52,16 @@ async function writeHostsToTrie(trie: HostnameSmolTrie, hostsUrl: string, includ
async function writeFiltersToTrie(trie: HostnameSmolTrie, filterUrl: string, includeThirdParty = false) {
const { whiteDomainSuffixes, whiteDomains, blackDomainSuffixes, blackDomains } = await processFilterRulesWithPreload(filterUrl, [], includeThirdParty)(dummySpan);
for (let i = 0, len = blackDomainSuffixes.length; i < len; i++) {
trie.add(blackDomainSuffixes[i], true);
trie.addSubdomain(blackDomainSuffixes[i]);
}
for (let i = 0, len = blackDomains.length; i < len; i++) {
trie.add(blackDomains[i], false);
trie.add(blackDomains[i]);
}
for (let i = 0, len = whiteDomainSuffixes.length; i < len; i++) {
trie.whitelist(whiteDomainSuffixes[i], true);
trie.whitelist('.' + whiteDomainSuffixes[i]);
}
for (let i = 0, len = whiteDomains.length; i < len; i++) {
trie.whitelist(whiteDomains[i], false);
trie.whitelist(whiteDomains[i]);
}
return { whiteDomainSuffixes, whiteDomains };

20
Build/validate-domestic.ts
View File

@@ -4,7 +4,8 @@ import { parseFelixDnsmasqFromResp } from './lib/parse-dnsmasq';
import { $$fetch } from './lib/fetch-retry';
import runAgainstSourceFile from './lib/run-against-source-file';
import { getTopOneMillionDomains } from './validate-gfwlist';
import { HostnameSmolTrie } from './lib/trie';
import { HostnameSmolTrie } from 'hntrie/smol';
import { domainToASCII } from 'node:url';
import tldts from 'tldts-experimental';
import { DOMESTICS } from '../Source/non_ip/domestic';
@@ -15,15 +16,15 @@ export async function parseDomesticList() {
const resultTrie = new HostnameSmolTrie();
topDomainTrie.dumpWithoutDot((domain) => {
topDomainTrie.dump((domain) => {
const apexDomain = tldts.getDomain(domain);
if (apexDomain && allChinaDomains.has(apexDomain)) {
resultTrie.add(apexDomain, false);
resultTrie.add(apexDomain);
}
});
const callback = (domain: string, includeAllSubdomain: boolean) => resultTrie.whitelist(domain, includeAllSubdomain);
const callback = (domain: string, includeAllSubdomain: boolean) => resultTrie.whitelist(includeAllSubdomain ? '.' + domain : domain);
// await Promise.all([
await runAgainstSourceFile(
@@ -40,11 +41,11 @@ export async function parseDomesticList() {
switch (domain[0]) {
case '+':
case '$': {
resultTrie.whitelist(domain.slice(1), true);
resultTrie.whitelist('.' + domain.slice(1));
break;
}
default: {
resultTrie.whitelist(domain, true);
resultTrie.whitelist('.' + domain);
break;
}
}
@@ -58,7 +59,12 @@ export async function parseDomesticList() {
// }
// }
// ]);
const dump = resultTrie.dump(null, true);
const dump: string[] = [];
resultTrie.dump((rawDomain, includeSubdomain) => {
const domain = domainToASCII(rawDomain);
if (domain) dump.push(includeSubdomain ? '.' + domain : domain);
});
dump.sort((a, b) => (a.length - b.length) || (a < b ? -1 : a > b ? 1 : 0));
console.log(dump.join('\n') + '\n');

17
Build/validate-gfwlist.ts
View File

@@ -1,6 +1,7 @@
import { processLine } from './lib/process-line';
import { fastNormalizeDomain } from './lib/normalize-domain';
import { HostnameSmolTrie } from './lib/trie';
import { HostnameSmolTrie } from 'hntrie/smol';
import { domainToASCII } from 'node:url';
import yauzl from 'yauzl-promise';
import { fetchRemoteTextByLine } from './lib/fetch-text-by-line';
import path from 'node:path';
@@ -120,8 +121,9 @@ export async function parseGfwList() {
const keywordSet = new Set<string>();
const callback = (domain: string, includeAllSubdomain: boolean) => {
gfwListTrie.whitelist(domain, includeAllSubdomain);
topDomainTrie.whitelist(domain, includeAllSubdomain);
const d = includeAllSubdomain ? '.' + domain : domain;
gfwListTrie.whitelist(d);
topDomainTrie.whitelist(d);
};
await Promise.all([
runAgainstSourceFile(path.join(SOURCE_DIR, 'non_ip/global.conf'), callback, 'ruleset', keywordSet),
@@ -149,17 +151,18 @@ export async function parseGfwList() {
});
});
whiteSet.forEach(domain => gfwListTrie.whitelist(domain, true));
whiteSet.forEach(domain => gfwListTrie.whitelist(domain[0] === '.' ? domain : '.' + domain));
let dedupedGfwListSize = 0;
gfwListTrie.dump(() => dedupedGfwListSize++);
gfwListTrie.dump(() => { dedupedGfwListSize++; });
const kwfilter = createKeywordFilter([...keywordSet]);
const missingTop10000Gfwed = new Set<string>();
topDomainTrie.dump((domain) => {
if (gfwListTrie.has(domain) && !kwfilter(domain)) {
topDomainTrie.dump((rawDomain) => {
const domain = domainToASCII(rawDomain);
if (domain && gfwListTrie.match(domain) && !kwfilter(domain)) {
missingTop10000Gfwed.add(domain);
}
});

20
Build/validate-global-tld.ts
View File

@@ -1,5 +1,6 @@
import path from 'node:path';
import { HostnameSmolTrie } from './lib/trie';
import { HostnameSmolTrie } from 'hntrie/smol';
import { domainToASCII } from 'node:url';
import { OUTPUT_SURGE_DIR } from './constants/dir';
import { ICP_TLD } from './constants/domains';
import tldts from 'tldts-experimental';
@@ -22,11 +23,20 @@ import { MARKER_DOMAIN } from './constants/description';
}, 'ruleset');
await runAgainstSourceFile(path.join(OUTPUT_SURGE_DIR, 'non_ip', 'global.conf'), (domain, includeAllSubdomain) => {
trie.add(domain, includeAllSubdomain);
if (includeAllSubdomain) {
trie.addSubdomain(domain);
} else {
trie.add(domain);
}
}, 'ruleset');
ICP_TLD.forEach(tld => trie.whitelist(tld, true));
extraWhiteTLDs.forEach(tld => trie.whitelist(tld, true));
ICP_TLD.forEach(tld => trie.whitelist('.' + tld));
extraWhiteTLDs.forEach(tld => trie.whitelist('.' + tld));
console.log(trie.dump().join('\n'));
const dump: string[] = [];
trie.dump((rawDomain, includeSubdomain) => {
const domain = domainToASCII(rawDomain);
if (domain) dump.push(includeSubdomain ? '.' + domain : domain);
});
console.log(dump.join('\n'));
})();

1
package.json
View File

@@ -31,6 +31,7 @@
"fast-uri": "^4.1.0",
"fdir": "^6.5.0",
"hash-wasm": "^4.12.0",
"hntrie": "^1.0.1",
"json-stringify-pretty-compact": "4.0.0",
"null-prototype-object": "^1.2.7",
"picocolors": "^1.1.1",

10
pnpm-lock.yaml generated
View File

@@ -50,6 +50,9 @@ importers:
hash-wasm:
specifier: ^4.12.0
version: 4.12.0
hntrie:
specifier: ^1.0.1
version: 1.0.1
json-stringify-pretty-compact:
specifier: 4.0.0
version: 4.0.0
@@ -1409,6 +1412,9 @@ packages:
resolution: {integrity: sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==}
hasBin: true
hntrie@1.0.1:
resolution: {integrity: sha512-k3OIFupPUGL1WrAiZ0Xa5on/vE/MKlub7ZQtEH0AfzfskMqq4/H6xgSqMeCXzVgNDWruP/Ff+9O5mHRzL4XZyg==}
htmlparser2@6.1.0:
resolution: {integrity: sha512-gyyPk6rgonLFEDGoeRgQNaEUvdJ4ktTmmUh/h2t7s+M8oPpIPxgNACWa+6ESR57kXstwqPiCut0V8NRpcwgU7A==}
@@ -3210,6 +3216,10 @@ snapshots:
he@1.2.0: {}
hntrie@1.0.1:
dependencies:
foxts: 5.5.1
htmlparser2@6.1.0:
dependencies:
domelementtype: 2.3.0