GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2025-12-12 09:10:35 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Chore: simplify build infra

Browse Source
This commit is contained in:
SukkaW 2023-12-09 21:27:03 +08:00
parent b22393a62a
commit a558b82c08
4 changed files with 65 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
Build/build-mitm-hostname.js
View File

@ -94,7 +94,7 @@ const PRESET_MITM_HOSTNAMES = [
); );
})); }));
let mitmDomains = new Set(PRESET_MITM_HOSTNAMES); // Special case for parsed failed const mitmDomains = new Set(PRESET_MITM_HOSTNAMES); // Special case for parsed failed
const parsedFailures = []; const parsedFailures = [];
const dedupedUrlRegexPaths = [...new Set(urlRegexPaths)]; const dedupedUrlRegexPaths = [...new Set(urlRegexPaths)];
@ -109,23 +109,24 @@ const PRESET_MITM_HOSTNAMES = [
} }
}); });
mitmDomains = [...mitmDomains].filter(i => { const mitmDomainsRegExpArray = mitmDomains
return i.length > 3 .slice()
&& !i.includes('.mp4') // Special Case .filter(i => {
&& i !== '(www.)' // Special Case return i.length > 3
&& !(i !== '*.meituan.net' && i.endsWith('.meituan.net')) && !i.includes('.mp4') // Special Case
&& !i.startsWith('.') && i !== '(www.)' // Special Case
&& !i.endsWith('.') && !(i !== '*.meituan.net' && i.endsWith('.meituan.net'))
&& !i.endsWith('*'); && !i.startsWith('.')
}); && !i.endsWith('.')
&& !i.endsWith('*');
const mitmDomainsRegExpArray = mitmDomains.map(i => { })
return new RegExp( .map(i => {
escapeRegExp(i) return new RegExp(
.replaceAll('{www or not}', '(www.)?') escapeRegExp(i)
.replaceAll('\\*', '(.*)') .replaceAll('{www or not}', '(www.)?')
); .replaceAll('\\*', '(.*)')
}); );
});
const parsedDomainsData = []; const parsedDomainsData = [];
dedupedUrlRegexPaths.forEach(i => { dedupedUrlRegexPaths.forEach(i => {

14
Build/build-reject-domainset.ts
View File

@ -108,18 +108,18 @@ export const buildRejectDomainSet = task(import.meta.path, async () => {
console.log(`Import ${previousSize} rules from reject_sukka.conf!`); console.log(`Import ${previousSize} rules from reject_sukka.conf!`);
for await (const line of readFileByLine(path.resolve(import.meta.dir, '../Source/non_ip/reject.conf'))) { for await (const line of readFileByLine(path.resolve(import.meta.dir, '../Source/non_ip/reject.conf'))) {
if (line.startsWith('DOMAIN-KEYWORD')) { const [type, keyword] = line.split(',');
const [, ...keywords] = line.split(',');
domainKeywordsSet.add(keywords.join(',').trim()); if (type === 'DOMAIN-KEYWORD') {
} else if (line.startsWith('DOMAIN-SUFFIX')) { domainKeywordsSet.add(keyword.trim());
const [, ...keywords] = line.split(','); } else if (type === 'DOMAIN-SUFFIX') {
domainSuffixSet.add(keywords.join(',').trim()); domainSuffixSet.add(keyword.trim());
} }
} }
for await (const line of readFileByLine(path.resolve(import.meta.dir, '../List/domainset/reject_phishing.conf'))) { for await (const line of readFileByLine(path.resolve(import.meta.dir, '../List/domainset/reject_phishing.conf'))) {
const l = processLine(line); const l = processLine(line);
if (l && l[0] === '.') { if (l?.[0] === '.') {
domainSuffixSet.add(l.slice(1)); domainSuffixSet.add(l.slice(1));
} }
} }

4
Build/lib/fetch-remote-text-by-line.ts
View File

@ -23,9 +23,11 @@ import { fetchWithRetry, defaultRequestInit } from './fetch-retry';
const decoder = new TextDecoder('utf-8'); const decoder = new TextDecoder('utf-8');
export async function *readFileByLine(file: string | BunFile): AsyncGenerator<string> { export async function *readFileByLine(file: string | URL | BunFile): AsyncGenerator<string> {
if (typeof file === 'string') { if (typeof file === 'string') {
file = Bun.file(file); file = Bun.file(file);
} else if (!('writer' in file)) {
file = Bun.file(file);
} }
let buf = ''; let buf = '';

86
Build/lib/parse-filter.ts
View File

@ -111,29 +111,6 @@ export async function processFilterRules(
const whitelistDomainSets = new Set<string>(); const whitelistDomainSets = new Set<string>();
const blacklistDomainSets = new Set<string>(); const blacklistDomainSets = new Set<string>();
/**
* @param {string} domainToBeAddedToBlack
* @param {boolean} isSubDomain
*/
const addToBlackList = (domainToBeAddedToBlack: string, isSubDomain: boolean) => {
if (isSubDomain && domainToBeAddedToBlack[0] !== '.') {
blacklistDomainSets.add(`.${domainToBeAddedToBlack}`);
} else {
blacklistDomainSets.add(domainToBeAddedToBlack);
}
};
/**
* @param {string} domainToBeAddedToWhite
* @param {boolean} [isSubDomain]
*/
const addToWhiteList = (domainToBeAddedToWhite: string, isSubDomain = true) => {
if (isSubDomain && domainToBeAddedToWhite[0] !== '.') {
whitelistDomainSets.add(`.${domainToBeAddedToWhite}`);
} else {
whitelistDomainSets.add(domainToBeAddedToWhite);
}
};
let downloadTime = 0; let downloadTime = 0;
const gorhill = await getGorhillPublicSuffixPromise(); const gorhill = await getGorhillPublicSuffixPromise();
@ -142,35 +119,45 @@ export async function processFilterRules(
*/ */
const lineCb = (line: string) => { const lineCb = (line: string) => {
const result = parse(line, gorhill); const result = parse(line, gorhill);
if (result) { if (!result) {
const flag = result[1]; return;
const hostname = result[0]; }
if (DEBUG_DOMAIN_TO_FIND) { const flag = result[1];
if (hostname.includes(DEBUG_DOMAIN_TO_FIND)) { const hostname = result[0];
warnOnce(filterRulesUrl.toString(), flag === 0 || flag === -1, DEBUG_DOMAIN_TO_FIND);
foundDebugDomain = true;
console.log({ result, flag }); if (DEBUG_DOMAIN_TO_FIND) {
if (hostname.includes(DEBUG_DOMAIN_TO_FIND)) {
warnOnce(filterRulesUrl.toString(), flag === 0 || flag === -1, DEBUG_DOMAIN_TO_FIND);
foundDebugDomain = true;
console.log({ result, flag });
}
}
switch (flag) {
case 0:
if (hostname[0] !== '.') {
whitelistDomainSets.add(`.${hostname}`);
} else {
whitelistDomainSets.add(hostname);
} }
} break;
case -1:
switch (flag) { whitelistDomainSets.add(hostname);
case 0: break;
addToWhiteList(hostname, true); case 1:
break; blacklistDomainSets.add(hostname);
case -1: break;
addToWhiteList(hostname, false); case 2:
break; if (hostname[0] !== '.') {
case 1: blacklistDomainSets.add(`.${hostname}`);
addToBlackList(hostname, false); } else {
break; blacklistDomainSets.add(hostname);
case 2: }
addToBlackList(hostname, true); break;
break; default:
default: throw new Error(`Unknown flag: ${flag as any}`);
throw new Error(`Unknown flag: ${flag as any}`);
}
} }
}; };
@ -302,7 +289,6 @@ function parse($line: string, gorhill: PublicSuffixList): null | [hostname: stri
) { ) {
const hostname = normalizeDomain(filter.hostname); const hostname = normalizeDomain(filter.hostname);
if (!hostname) { if (!hostname) {
console.log(' * [parse-filter E0000] invalid domain:', filter.hostname);
return null; return null;
} }