GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2025-12-12 17:20:35 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Chore: validate domain tools

Browse Source
This commit is contained in:
SukkaW 2025-07-11 19:03:30 +08:00
parent f05916177b
commit ede1b7e25b
2 changed files with 94 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

78
Build/validate-domestic.ts
View File

@ -1,59 +1,67 @@
import { parse } from 'csv-parse/sync';
import { HostnameSmolTrie } from './lib/trie';
import path from 'node:path';
import { SOURCE_DIR } from './constants/dir';
import { parseFelixDnsmasqFromResp } from './lib/parse-dnsmasq';
import { $$fetch } from './lib/fetch-retry';
import runAgainstSourceFile from './lib/run-against-source-file';
import { getTopOneMillionDomains } from './validate-gfwlist';
import { HostnameSmolTrie } from './lib/trie';
import tldts from 'tldts-experimental';
import { DOMESTICS } from '../Source/non_ip/domestic';
export async function parseDomesticList() {
const trie = new HostnameSmolTrie(await parseFelixDnsmasqFromResp(await $$fetch('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf')));
const allChinaDomains = new Set<string>(await parseFelixDnsmasqFromResp(await $$fetch('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf')));
const top5000 = new Set<string>();
const topDomainTrie = await getTopOneMillionDomains();
const res = await (await $$fetch('https://radar.cloudflare.com/charts/LargerTopDomainsTable/attachment?id=1077&top=10000', {
headers: {
accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'accept-language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7,zh-TW;q=0.6,es;q=0.5',
'sec-ch-ua': '"Google Chrome";v="123", "Not:A-Brand";v="8", "Chromium";v="123"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"macOS"',
'sec-fetch-dest': 'document',
'sec-fetch-mode': 'navigate',
'sec-fetch-site': 'none',
'sec-fetch-user': '?1',
'upgrade-insecure-requests': '1'
}
})).text();
const stream = parse(res);
for await (const [domain] of stream) {
if (trie.has(domain)) {
top5000.add(domain);
}
console.log({ domain });
}
const resultTrie = new HostnameSmolTrie();
const notIncludedDomestic = new Set<string>(top5000);
topDomainTrie.dumpWithoutDot((domain) => {
const apexDomain = tldts.getDomain(domain);
if (apexDomain && allChinaDomains.has(apexDomain)) {
resultTrie.add(apexDomain, false);
}
});
const callback = (domain: string, includeAllSubdomain: boolean) => resultTrie.whitelist(domain, includeAllSubdomain);
// await Promise.all([
await runAgainstSourceFile(
path.resolve(SOURCE_DIR, 'non_ip/domestic.conf'),
(domain, includeAllSubdomain) => {
if (includeAllSubdomain) {
if (top5000.has(domain)) {
notIncludedDomestic.delete(domain);
callback
);
await runAgainstSourceFile(
path.resolve(SOURCE_DIR, 'domainset/reject.conf'),
callback
);
Object.values(DOMESTICS).forEach(domestic => {
domestic.domains.forEach(domain => {
switch (domain[0]) {
case '+':
case '$': {
resultTrie.whitelist(domain.slice(1), true);
break;
}
} else {
default: {
resultTrie.whitelist(domain, true);
break;
}
}
});
});
// noop, DOMAIN-KEYWORD handing
// for (const d of top5000) {
// if (d.includes(domain)) {
// notIncludedDomestic.delete(d);
// }
// }
}
}
);
// ]);
console.log(notIncludedDomestic.size, notIncludedDomestic);
console.log(resultTrie.dump().join('\n') + '\n');
}
if (require.main === module) {
parseDomesticList().catch(console.error);
}

87
Build/validate-gfwlist.ts
View File

@ -11,13 +11,52 @@ import runAgainstSourceFile from './lib/run-against-source-file';
import { nullthrow } from 'foxts/guard';
import { Buffer } from 'node:buffer';
export async function parseGfwList() {
export async function getTopOneMillionDomains() {
const { parse: csvParser } = await import('csv-parse');
const topDomainTrie = new HostnameSmolTrie();
const csvParse = csvParser({ columns: false, skip_empty_lines: true });
const topDomainsZipBody = await (await $$fetch('https://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip', {
headers: {
accept: '*/*',
'user-agent': 'curl/8.12.1'
}
})).arrayBuffer();
let entry: yauzl.Entry | null = null;
for await (const e of await yauzl.fromBuffer(Buffer.from(topDomainsZipBody))) {
if (e.filename === 'top-1m.csv') {
entry = e;
break;
}
}
const { promise, resolve, reject } = Promise.withResolvers<HostnameSmolTrie>();
const readable = await nullthrow(entry, 'top-1m.csv entry not found').openReadStream();
const parser = readable.pipe(csvParse);
parser.on('readable', () => {
let record;
while ((record = parser.read()) !== null) {
topDomainTrie.add(record[1]);
}
});
parser.on('end', () => {
resolve(topDomainTrie);
});
parser.on('error', (err) => {
reject(err);
});
return promise;
}
export async function parseGfwList() {
const whiteSet = new Set<string>();
const gfwListTrie = new HostnameSmolTrie();
const excludeGfwList = createKeywordFilter([
const gfwlistIgnoreLineKwfilter = createKeywordFilter([
'.*',
'*',
'=',
@ -31,7 +70,7 @@ export async function parseGfwList() {
const line = processLine(l);
if (!line) continue;
if (excludeGfwList(line)) {
if (gfwlistIgnoreLineKwfilter(line)) {
continue;
}
if (line.startsWith('@@||')) {
@ -71,42 +110,7 @@ export async function parseGfwList() {
gfwListTrie.add(l);
}
const topDomainTrie = new HostnameSmolTrie();
const csvParse = csvParser({ columns: false, skip_empty_lines: true });
const topDomainsZipBody = await (await $$fetch('https://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip', {
headers: {
accept: '*/*',
'user-agent': 'curl/8.12.1'
}
})).arrayBuffer();
let entry: yauzl.Entry | null = null;
for await (const e of await yauzl.fromBuffer(Buffer.from(topDomainsZipBody))) {
if (e.filename === 'top-1m.csv') {
entry = e;
break;
}
}
const { promise, resolve, reject } = Promise.withResolvers<HostnameSmolTrie>();
const readable = await nullthrow(entry, 'top-1m.csv entry not found').openReadStream();
const parser = readable.pipe(csvParse);
parser.on('readable', () => {
let record;
while ((record = parser.read()) !== null) {
topDomainTrie.add(record[1]);
}
});
parser.on('end', () => {
resolve(topDomainTrie);
});
parser.on('error', (err) => {
reject(err);
});
await promise;
const topDomainTrie = await getTopOneMillionDomains();
const keywordSet = new Set<string>();
@ -116,18 +120,19 @@ export async function parseGfwList() {
};
await Promise.all([
runAgainstSourceFile(path.join(SOURCE_DIR, 'non_ip/global.conf'), callback, 'ruleset', keywordSet),
runAgainstSourceFile(path.join(OUTPUT_SURGE_DIR, 'non_ip/domestic.conf'), callback, 'ruleset', keywordSet),
// runAgainstSourceFile(path.join(OUTPUT_SURGE_DIR, 'non_ip/domestic.conf'), callback, 'ruleset', keywordSet),
runAgainstSourceFile(path.join(SOURCE_DIR, 'non_ip/reject.conf'), callback, 'ruleset', keywordSet),
runAgainstSourceFile(path.join(SOURCE_DIR, 'non_ip/telegram.conf'), callback, 'ruleset', keywordSet),
runAgainstSourceFile(path.resolve(OUTPUT_SURGE_DIR, 'non_ip/stream.conf'), callback, 'ruleset', keywordSet),
runAgainstSourceFile(path.resolve(SOURCE_DIR, 'non_ip/ai.conf'), callback, 'ruleset', keywordSet),
runAgainstSourceFile(path.resolve(SOURCE_DIR, 'non_ip/microsoft.conf'), callback, 'ruleset', keywordSet),
runAgainstSourceFile(path.resolve(SOURCE_DIR, 'non_ip/apple_service.conf'), callback, 'ruleset', keywordSet),
runAgainstSourceFile(path.resolve(OUTPUT_SURGE_DIR, 'domainset/reject.conf'), callback, 'domainset'),
runAgainstSourceFile(path.resolve(OUTPUT_SURGE_DIR, 'domainset/reject_extra.conf'), callback, 'domainset'),
runAgainstSourceFile(path.resolve(OUTPUT_SURGE_DIR, 'domainset/cdn.conf'), callback, 'domainset')
]);
whiteSet.forEach(domain => gfwListTrie.whitelist(domain));
whiteSet.forEach(domain => gfwListTrie.whitelist(domain, true));
const kwfilter = createKeywordFilter([...keywordSet]);