GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2026-01-28 17:41:54 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Chore: validate domain tools

Browse Source
This commit is contained in:
SukkaW
2025-07-11 19:03:30 +08:00
parent f05916177b
commit ede1b7e25b
2 changed files with 94 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

88
Build/validate-domestic.ts
View File

@@ -1,59 +1,67 @@
import { parse } from 'csv-parse/sync';
import { HostnameSmolTrie } from './lib/trie';
import path from 'node:path';
import { SOURCE_DIR } from './constants/dir';
import { parseFelixDnsmasqFromResp } from './lib/parse-dnsmasq';
import { $$fetch } from './lib/fetch-retry';
import runAgainstSourceFile from './lib/run-against-source-file';
import { getTopOneMillionDomains } from './validate-gfwlist';
import { HostnameSmolTrie } from './lib/trie';
import tldts from 'tldts-experimental';
import { DOMESTICS } from '../Source/non_ip/domestic';
export async function parseDomesticList() {
const trie = new HostnameSmolTrie(await parseFelixDnsmasqFromResp(await $$fetch('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf')));
const allChinaDomains = new Set<string>(await parseFelixDnsmasqFromResp(await $$fetch('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf')));
const top5000 = new Set<string>();
const topDomainTrie = await getTopOneMillionDomains();
const res = await (await $$fetch('https://radar.cloudflare.com/charts/LargerTopDomainsTable/attachment?id=1077&top=10000', {
headers: {
accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'accept-language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7,zh-TW;q=0.6,es;q=0.5',
'sec-ch-ua': '"Google Chrome";v="123", "Not:A-Brand";v="8", "Chromium";v="123"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"macOS"',
'sec-fetch-dest': 'document',
'sec-fetch-mode': 'navigate',
'sec-fetch-site': 'none',
'sec-fetch-user': '?1',
'upgrade-insecure-requests': '1'
const resultTrie = new HostnameSmolTrie();
topDomainTrie.dumpWithoutDot((domain) => {
const apexDomain = tldts.getDomain(domain);
if (apexDomain && allChinaDomains.has(apexDomain)) {
resultTrie.add(apexDomain, false);
}
})).text();
const stream = parse(res);
for await (const [domain] of stream) {
if (trie.has(domain)) {
top5000.add(domain);
}
console.log({ domain });
}
});
const notIncludedDomestic = new Set<string>(top5000);
const callback = (domain: string, includeAllSubdomain: boolean) => resultTrie.whitelist(domain, includeAllSubdomain);
// await Promise.all([
await runAgainstSourceFile(
path.resolve(SOURCE_DIR, 'non_ip/domestic.conf'),
(domain, includeAllSubdomain) => {
if (includeAllSubdomain) {
if (top5000.has(domain)) {
notIncludedDomestic.delete(domain);
}
} else {
// noop, DOMAIN-KEYWORD handing
// for (const d of top5000) {
// if (d.includes(domain)) {
// notIncludedDomestic.delete(d);
// }
// }
}
}
callback
);
await runAgainstSourceFile(
path.resolve(SOURCE_DIR, 'domainset/reject.conf'),
callback
);
Object.values(DOMESTICS).forEach(domestic => {
domestic.domains.forEach(domain => {
switch (domain[0]) {
case '+':
case '$': {
resultTrie.whitelist(domain.slice(1), true);
break;
}
default: {
resultTrie.whitelist(domain, true);
break;
}
}
});
});
// noop, DOMAIN-KEYWORD handing
// for (const d of top5000) {
// if (d.includes(domain)) {
// notIncludedDomestic.delete(d);
// }
// }
// ]);
console.log(notIncludedDomestic.size, notIncludedDomestic);
console.log(resultTrie.dump().join('\n') + '\n');
}
if (require.main === module) {
parseDomesticList().catch(console.error);
}