GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2025-12-12 01:00:34 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Chore: validate gfwlist

Browse Source
This commit is contained in:
SukkaW 2024-11-02 03:53:26 +08:00
parent 7a921f791f
commit 90b6580377
1 changed files with 74 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

128
Build/validate-gfwlist.ts
View File

@ -1,123 +1,143 @@
import { processLine } from './lib/process-line';
import { normalizeDomain } from './lib/normalize-domain';
import { createTrie } from './lib/trie';
import { HostnameSmolTrie } from './lib/trie';
// import { Readable } from 'stream';
import { parse } from 'csv-parse/sync';
import { readFileByLine } from './lib/fetch-text-by-line';
import path from 'node:path';
import { SOURCE_DIR } from './constants/dir';
import { OUTPUT_SURGE_DIR } from './constants/dir';
import { $fetch } from './lib/make-fetch-happen';
import createKeywordFilter from './lib/aho-corasick';
export async function parseGfwList() {
const whiteSet = new Set<string>();
const blackSet = new Set<string>();
const trie = new HostnameSmolTrie();
const excludeGfwList = createKeywordFilter([
'.*',
'*',
'=',
'[',
'/',
'?'
]);
const text = await (await $fetch('https://raw.githubusercontent.com/gfwlist/gfwlist/master/gfwlist.txt')).text();
for (const l of atob(text).split('\n')) {
const line = processLine(l);
if (!line) continue;
if (line[0] === '[') {
continue;
}
if (line.includes('.*')) {
continue;
}
if (line.includes('*')) {
if (excludeGfwList(line)) {
continue;
}
if (line.startsWith('@@||')) {
whiteSet.add(line.slice(4));
whiteSet.add('.' + line.slice(4));
continue;
}
if (line.startsWith('@@|http://')) {
whiteSet.add(line.slice(8));
whiteSet.add(line.slice(10));
continue;
}
if (line.startsWith('@@|https://')) {
whiteSet.add(line.slice(9));
whiteSet.add(line.slice(11));
continue;
}
if (line.startsWith('||')) {
blackSet.add(line.slice(2));
trie.add('.' + line.slice(2));
continue;
}
if (line.startsWith('|')) {
blackSet.add(line.slice(1));
trie.add(line.slice(1));
continue;
}
if (line.startsWith('.')) {
blackSet.add(line.slice(1));
trie.add(line);
continue;
}
const d = normalizeDomain(line);
if (d) {
blackSet.add(d);
trie.add(d);
continue;
}
}
for (const l of (await (await $fetch('https://raw.githubusercontent.com/Loyalsoldier/cn-blocked-domain/release/domains.txt')).text()).split('\n')) {
blackSet.add(l);
trie.add(l);
}
const top500Gfwed = new Set<string>();
const res = await (await $fetch('https://radar.cloudflare.com/charts/LargerTopDomainsTable/attachment?id=1077&top=10000', {
const res = await (await $fetch('https://litter.catbox.moe/wed48e.csv', {
headers: {
accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'accept-language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7,zh-TW;q=0.6,es;q=0.5',
'sec-ch-ua': '"Google Chrome";v="123", "Not:A-Brand";v="8", "Chromium";v="123"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"macOS"',
'sec-fetch-dest': 'document',
'sec-fetch-mode': 'navigate',
'sec-fetch-site': 'none',
'sec-fetch-user': '?1',
'upgrade-insecure-requests': '1'
accept: '*/*',
'user-agent': 'curl/8.9.1'
}
})).text();
const topDomains = parse(res);
const trie = createTrie(blackSet);
for await (const [domain] of topDomains) {
if (trie.has(domain)) {
top500Gfwed.add(domain);
}
}
const notIncludedTop500Gfwed = new Set<string>(top500Gfwed);
const keywordSet = new Set<string>();
const runAgainstRuleset = async (ruleset: string) => {
for await (const l of readFileByLine(ruleset)) {
const line = processLine(l);
if (!line) continue;
const [type, domain] = line.split(',');
if (type === 'DOMAIN-SUFFIX') {
if (top500Gfwed.has(domain)) {
notIncludedTop500Gfwed.delete(domain);
switch (type) {
case 'DOMAIN-SUFFIX': {
trie.whitelist('.' + domain);
break;
}
} else if (type === 'DOMAIN-KEYWORD') {
for (const d of top500Gfwed) {
if (d.includes(domain)) {
notIncludedTop500Gfwed.delete(d);
}
case 'DOMAIN': {
trie.whitelist(domain);
break;
}
case 'DOMAIN-KEYWORD': {
keywordSet.add(domain);
break;
}
// no default
}
}
};
const runAgainstDomainset = async (ruleset: string) => {
for await (const l of readFileByLine(ruleset)) {
const line = processLine(l);
if (!line) continue;
trie.whitelist(line);
}
};
await Promise.all([
runAgainstRuleset(path.join(SOURCE_DIR, 'non_ip/global.conf')),
runAgainstRuleset(path.join(SOURCE_DIR, 'non_ip/telegram.conf')),
runAgainstRuleset(path.resolve(SOURCE_DIR, 'non_ip/stream.conf'))
runAgainstRuleset(path.join(OUTPUT_SURGE_DIR, 'non_ip/global.conf')),
runAgainstRuleset(path.join(OUTPUT_SURGE_DIR, 'non_ip/reject.conf')),
runAgainstRuleset(path.join(OUTPUT_SURGE_DIR, 'non_ip/telegram.conf')),
runAgainstRuleset(path.resolve(OUTPUT_SURGE_DIR, 'non_ip/stream.conf')),
runAgainstRuleset(path.resolve(OUTPUT_SURGE_DIR, 'non_ip/ai.conf')),
runAgainstRuleset(path.resolve(OUTPUT_SURGE_DIR, 'non_ip/microsoft.conf')),
runAgainstDomainset(path.resolve(OUTPUT_SURGE_DIR, 'domainset/reject.conf')),
runAgainstDomainset(path.resolve(OUTPUT_SURGE_DIR, 'domainset/cdn.conf'))
]);
console.log(notIncludedTop500Gfwed);
whiteSet.forEach(domain => trie.whitelist(domain));
const kwfilter = createKeywordFilter(keywordSet);
const missingTop10000Gfwed = new Set<string>();
console.log(trie.has('.mojim.com'));
for await (const [domain] of topDomains) {
if (trie.has(domain) && !kwfilter(domain)) {
missingTop10000Gfwed.add(domain);
}
}
console.log(JSON.stringify(Array.from(missingTop10000Gfwed)), null, 2);
return [
whiteSet,
blackSet,
trie,
top500Gfwed
missingTop10000Gfwed
] as const;
}
if (require.main === module) {
parseGfwList().catch(console.error);
}