GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2026-07-29 16:51:45 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Refactor: migrate to hntrie

Browse Source
This commit is contained in:
SukkaW
2026-07-01 04:56:18 +08:00
parent 51f1c898a3
commit a664d2d779
15 changed files with 115 additions and 1128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
Build/validate-gfwlist.ts
View File

@@ -1,6 +1,7 @@
import { processLine } from './lib/process-line';
import { fastNormalizeDomain } from './lib/normalize-domain';
import { HostnameSmolTrie } from './lib/trie';
import { HostnameSmolTrie } from 'hntrie/smol';
import { domainToASCII } from 'node:url';
import yauzl from 'yauzl-promise';
import { fetchRemoteTextByLine } from './lib/fetch-text-by-line';
import path from 'node:path';
@@ -120,8 +121,9 @@ export async function parseGfwList() {
const keywordSet = new Set<string>();
const callback = (domain: string, includeAllSubdomain: boolean) => {
gfwListTrie.whitelist(domain, includeAllSubdomain);
topDomainTrie.whitelist(domain, includeAllSubdomain);
const d = includeAllSubdomain ? '.' + domain : domain;
gfwListTrie.whitelist(d);
topDomainTrie.whitelist(d);
};
await Promise.all([
runAgainstSourceFile(path.join(SOURCE_DIR, 'non_ip/global.conf'), callback, 'ruleset', keywordSet),
@@ -149,17 +151,18 @@ export async function parseGfwList() {
});
});
whiteSet.forEach(domain => gfwListTrie.whitelist(domain, true));
whiteSet.forEach(domain => gfwListTrie.whitelist(domain[0] === '.' ? domain : '.' + domain));
let dedupedGfwListSize = 0;
gfwListTrie.dump(() => dedupedGfwListSize++);
gfwListTrie.dump(() => { dedupedGfwListSize++; });
const kwfilter = createKeywordFilter([...keywordSet]);
const missingTop10000Gfwed = new Set<string>();
topDomainTrie.dump((domain) => {
if (gfwListTrie.has(domain) && !kwfilter(domain)) {
topDomainTrie.dump((rawDomain) => {
const domain = domainToASCII(rawDomain);
if (domain && gfwListTrie.match(domain) && !kwfilter(domain)) {
missingTop10000Gfwed.add(domain);
}
});