Surge_by_SukkaW/Dist/Build/validate-gfwlist2.cjs

'use strict';Object.defineProperty(exports,Symbol.toStringTag,{value:'Module'});const validateGfwlist=require('../_virtual/validate-gfwlist.cjs'),processLine=require('./lib/process-line.cjs'),normalizeDomain=require('./lib/normalize-domain.cjs'),trie=require('./lib/trie.cjs'),require$$3$1=require('csv-parse/sync'),fetchTextByLine=require('./lib/fetch-text-by-line.cjs'),require$$0=require('node:path'),dir=require('./constants/dir.cjs'),require$$3=require('foxts/retrie'),fetchRetry=require('./lib/fetch-retry.cjs');var hasRequiredValidateGfwlist;

function requireValidateGfwlist () {
	if (hasRequiredValidateGfwlist) return validateGfwlist.__module.exports;
	hasRequiredValidateGfwlist = 1;
	(function (module, exports) {
		Object.defineProperty(exports, "__esModule", {
		    value: true
		});
		Object.defineProperty(exports, "parseGfwList", {
		    enumerable: true,
		    get: function() {
		        return parseGfwList;
		    }
		});
		const _processline = /*@__PURE__*/ processLine.__require();
		const _normalizedomain = /*@__PURE__*/ normalizeDomain.__require();
		const _trie = /*@__PURE__*/ trie.__require();
		const _sync = require$$3$1;
		const _fetchtextbyline = /*@__PURE__*/ fetchTextByLine.__require();
		const _nodepath = /*#__PURE__*/ _interop_require_default(require$$0);
		const _dir = /*@__PURE__*/ dir.__require();
		const _retrie = require$$3;
		const _fetchretry = /*@__PURE__*/ fetchRetry.__require();
		function _interop_require_default(obj) {
		    return obj && obj.__esModule ? obj : {
		        default: obj
		    };
		}
		async function parseGfwList() {
		    const whiteSet = new Set();
		    const trie = new _trie.HostnameSmolTrie();
		    const excludeGfwList = (0, _retrie.createRetrieKeywordFilter)([
		        '.*',
		        '*',
		        '=',
		        '[',
		        '/',
		        '?'
		    ]);
		    const text = await (await (0, _fetchretry.$$fetch)('https://raw.githubusercontent.com/gfwlist/gfwlist/master/gfwlist.txt')).text();
		    for (const l of atob(text).split('\n')){
		        const line = (0, _processline.processLine)(l);
		        if (!line) continue;
		        if (excludeGfwList(line)) {
		            continue;
		        }
		        if (line.startsWith('@@||')) {
		            whiteSet.add('.' + line.slice(4));
		            continue;
		        }
		        if (line.startsWith('@@|http://')) {
		            whiteSet.add(line.slice(10));
		            continue;
		        }
		        if (line.startsWith('@@|https://')) {
		            whiteSet.add(line.slice(11));
		            continue;
		        }
		        if (line.startsWith('||')) {
		            trie.add('.' + line.slice(2));
		            continue;
		        }
		        if (line.startsWith('|')) {
		            trie.add(line.slice(1));
		            continue;
		        }
		        if (line.startsWith('.')) {
		            trie.add(line);
		            continue;
		        }
		        const d = (0, _normalizedomain.fastNormalizeDomain)(line);
		        if (d) {
		            trie.add(d);
		            continue;
		        }
		    }
		    for (const l of (await (await (0, _fetchretry.$$fetch)('https://raw.githubusercontent.com/Loyalsoldier/cn-blocked-domain/release/domains.txt')).text()).split('\n')){
		        trie.add(l);
		    }
		    const res = await (await (0, _fetchretry.$$fetch)('https://litter.catbox.moe/sqmgyn.csv', {
		        headers: {
		            accept: '*/*',
		            'user-agent': 'curl/8.9.1'
		        }
		    })).text();
		    const topDomains = (0, _sync.parse)(res);
		    const keywordSet = new Set();
		    const runAgainstRuleset = async (ruleset)=>{
		        for await (const l of (0, _fetchtextbyline.readFileByLine)(ruleset)){
		            const line = (0, _processline.processLine)(l);
		            if (!line) continue;
		            const [type, domain] = line.split(',');
		            switch(type){
		                case 'DOMAIN-SUFFIX':
		                    {
		                        trie.whitelist('.' + domain);
		                        break;
		                    }
		                case 'DOMAIN':
		                    {
		                        trie.whitelist(domain);
		                        break;
		                    }
		                case 'DOMAIN-KEYWORD':
		                    {
		                        keywordSet.add(domain);
		                        break;
		                    }
		            }
		        }
		    };
		    const runAgainstDomainset = async (ruleset)=>{
		        for await (const l of (0, _fetchtextbyline.readFileByLine)(ruleset)){
		            const line = (0, _processline.processLine)(l);
		            if (!line) continue;
		            trie.whitelist(line);
		        }
		    };
		    await Promise.all([
		        runAgainstRuleset(_nodepath.default.join(_dir.OUTPUT_SURGE_DIR, 'non_ip/global.conf')),
		        runAgainstRuleset(_nodepath.default.join(_dir.OUTPUT_SURGE_DIR, 'non_ip/reject.conf')),
		        runAgainstRuleset(_nodepath.default.join(_dir.OUTPUT_SURGE_DIR, 'non_ip/telegram.conf')),
		        runAgainstRuleset(_nodepath.default.resolve(_dir.OUTPUT_SURGE_DIR, 'non_ip/stream.conf')),
		        runAgainstRuleset(_nodepath.default.resolve(_dir.OUTPUT_SURGE_DIR, 'non_ip/ai.conf')),
		        runAgainstRuleset(_nodepath.default.resolve(_dir.OUTPUT_SURGE_DIR, 'non_ip/microsoft.conf')),
		        runAgainstDomainset(_nodepath.default.resolve(_dir.OUTPUT_SURGE_DIR, 'domainset/reject.conf')),
		        runAgainstDomainset(_nodepath.default.resolve(_dir.OUTPUT_SURGE_DIR, 'domainset/cdn.conf'))
		    ]);
		    whiteSet.forEach((domain)=>trie.whitelist(domain));
		    const kwfilter = (0, _retrie.createRetrieKeywordFilter)([
		        ...keywordSet
		    ]);
		    const missingTop10000Gfwed = new Set();
		    console.log(trie.has('.mojim.com'));
		    for await (const [domain] of topDomains){
		        if (trie.has(domain) && !kwfilter(domain)) {
		            missingTop10000Gfwed.add(domain);
		        }
		    }
		    console.log(JSON.stringify(Array.from(missingTop10000Gfwed), null, 2));
		    return [
		        whiteSet,
		        trie,
		        missingTop10000Gfwed
		    ];
		}
		if (require.main === module) {
		    parseGfwList().catch(console.error);
		} 
	} (validateGfwlist.__module, validateGfwlist.__module.exports));
	return validateGfwlist.__module.exports;
}exports.__require=requireValidateGfwlist;