diff --git a/Build/_get-lum-apex-domains.ts b/Build/tools-lum-apex-domains.ts similarity index 82% rename from Build/_get-lum-apex-domains.ts rename to Build/tools-lum-apex-domains.ts index f1f7c55e..1894b1c5 100644 --- a/Build/_get-lum-apex-domains.ts +++ b/Build/tools-lum-apex-domains.ts @@ -37,6 +37,17 @@ import { processLine } from './lib/process-line'; trie.whitelist(l); } } + for await (const line of readFileByLine(path.join(SOURCE_DIR, 'non_ip', 'reject.conf'))) { + const l = processLine(line); + if (l) { + const [type, domain] = l.split(',', 3); + if (type === 'DOMAIN') { + trie.whitelist(domain, false); + } else if (type === 'DOMAIN-SUFFIX') { + trie.whitelist(domain, true); + } + } + } console.log(trie.dump().map(i => '.' + i).join('\n')); })(); diff --git a/Build/validate-gfwlist.ts b/Build/validate-gfwlist.ts index 02e46d65..1bf0c1a3 100644 --- a/Build/validate-gfwlist.ts +++ b/Build/validate-gfwlist.ts @@ -67,13 +67,13 @@ export async function parseGfwList() { trie.add(l); } - const res = await (await $$fetch('https://litter.catbox.moe/gv0bw6.csv', { + const topDomainsRes = await (await $$fetch('https://downloads.majestic.com/majestic_million.csv', { headers: { accept: '*/*', 'user-agent': 'curl/8.12.1' } })).text(); - const topDomains = parse(res); + const topDomains = parse(topDomainsRes); const keywordSet = new Set(); @@ -115,6 +115,7 @@ export async function parseGfwList() { runAgainstRuleset(path.resolve(OUTPUT_SURGE_DIR, 'non_ip/ai.conf')), runAgainstRuleset(path.resolve(OUTPUT_SURGE_DIR, 'non_ip/microsoft.conf')), runAgainstDomainset(path.resolve(OUTPUT_SURGE_DIR, 'domainset/reject.conf')), + runAgainstDomainset(path.resolve(OUTPUT_SURGE_DIR, 'domainset/reject_extra.conf')), runAgainstDomainset(path.resolve(OUTPUT_SURGE_DIR, 'domainset/cdn.conf')) ]); @@ -124,15 +125,14 @@ export async function parseGfwList() { const missingTop10000Gfwed = new Set(); - console.log(trie.has('.mojim.com')); - for await (const [domain] of topDomains) { if (trie.has(domain) && !kwfilter(domain)) { missingTop10000Gfwed.add(domain); } } - console.log(JSON.stringify(Array.from(missingTop10000Gfwed), null, 2)); + console.log(missingTop10000Gfwed.size, ''); + console.log(Array.from(missingTop10000Gfwed).join('\n')); return [ whiteSet,