GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2025-12-12 01:00:34 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Chore: simplify build infra

Browse Source
This commit is contained in:
SukkaW 2023-12-09 21:27:03 +08:00
parent b22393a62a
commit a558b82c08
4 changed files with 65 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
Build/build-mitm-hostname.js
View File

@ -94,7 +94,7 @@ const PRESET_MITM_HOSTNAMES = [
);
}));
let mitmDomains = new Set(PRESET_MITM_HOSTNAMES); // Special case for parsed failed
const mitmDomains = new Set(PRESET_MITM_HOSTNAMES); // Special case for parsed failed
const parsedFailures = [];
const dedupedUrlRegexPaths = [...new Set(urlRegexPaths)];
@ -109,23 +109,24 @@ const PRESET_MITM_HOSTNAMES = [
}
});
mitmDomains = [...mitmDomains].filter(i => {
return i.length > 3
&& !i.includes('.mp4') // Special Case
&& i !== '(www.)' // Special Case
&& !(i !== '*.meituan.net' && i.endsWith('.meituan.net'))
&& !i.startsWith('.')
&& !i.endsWith('.')
&& !i.endsWith('*');
});
const mitmDomainsRegExpArray = mitmDomains.map(i => {
return new RegExp(
escapeRegExp(i)
.replaceAll('{www or not}', '(www.)?')
.replaceAll('\\*', '(.*)')
);
});
const mitmDomainsRegExpArray = mitmDomains
.slice()
.filter(i => {
return i.length > 3
&& !i.includes('.mp4') // Special Case
&& i !== '(www.)' // Special Case
&& !(i !== '*.meituan.net' && i.endsWith('.meituan.net'))
&& !i.startsWith('.')
&& !i.endsWith('.')
&& !i.endsWith('*');
})
.map(i => {
return new RegExp(
escapeRegExp(i)
.replaceAll('{www or not}', '(www.)?')
.replaceAll('\\*', '(.*)')
);
});
const parsedDomainsData = [];
dedupedUrlRegexPaths.forEach(i => {

14
Build/build-reject-domainset.ts
View File

@ -108,18 +108,18 @@ export const buildRejectDomainSet = task(import.meta.path, async () => {
console.log(`Import ${previousSize} rules from reject_sukka.conf!`);
for await (const line of readFileByLine(path.resolve(import.meta.dir, '../Source/non_ip/reject.conf'))) {
if (line.startsWith('DOMAIN-KEYWORD')) {
const [, ...keywords] = line.split(',');
domainKeywordsSet.add(keywords.join(',').trim());
} else if (line.startsWith('DOMAIN-SUFFIX')) {
const [, ...keywords] = line.split(',');
domainSuffixSet.add(keywords.join(',').trim());
const [type, keyword] = line.split(',');
if (type === 'DOMAIN-KEYWORD') {
domainKeywordsSet.add(keyword.trim());
} else if (type === 'DOMAIN-SUFFIX') {
domainSuffixSet.add(keyword.trim());
}
}
for await (const line of readFileByLine(path.resolve(import.meta.dir, '../List/domainset/reject_phishing.conf'))) {
const l = processLine(line);
if (l && l[0] === '.') {
if (l?.[0] === '.') {
domainSuffixSet.add(l.slice(1));
}
}

4
Build/lib/fetch-remote-text-by-line.ts
View File

@ -23,9 +23,11 @@ import { fetchWithRetry, defaultRequestInit } from './fetch-retry';
const decoder = new TextDecoder('utf-8');
export async function *readFileByLine(file: string | BunFile): AsyncGenerator<string> {
export async function *readFileByLine(file: string | URL | BunFile): AsyncGenerator<string> {
if (typeof file === 'string') {
file = Bun.file(file);
} else if (!('writer' in file)) {
file = Bun.file(file);
}
let buf = '';

86
Build/lib/parse-filter.ts
View File

@ -111,29 +111,6 @@ export async function processFilterRules(
const whitelistDomainSets = new Set<string>();
const blacklistDomainSets = new Set<string>();
/**
* @param {string} domainToBeAddedToBlack
* @param {boolean} isSubDomain
*/
const addToBlackList = (domainToBeAddedToBlack: string, isSubDomain: boolean) => {
if (isSubDomain && domainToBeAddedToBlack[0] !== '.') {
blacklistDomainSets.add(`.${domainToBeAddedToBlack}`);
} else {
blacklistDomainSets.add(domainToBeAddedToBlack);
}
};
/**
* @param {string} domainToBeAddedToWhite
* @param {boolean} [isSubDomain]
*/
const addToWhiteList = (domainToBeAddedToWhite: string, isSubDomain = true) => {
if (isSubDomain && domainToBeAddedToWhite[0] !== '.') {
whitelistDomainSets.add(`.${domainToBeAddedToWhite}`);
} else {
whitelistDomainSets.add(domainToBeAddedToWhite);
}
};
let downloadTime = 0;
const gorhill = await getGorhillPublicSuffixPromise();
@ -142,35 +119,45 @@ export async function processFilterRules(
*/
const lineCb = (line: string) => {
const result = parse(line, gorhill);
if (result) {
const flag = result[1];
const hostname = result[0];
if (!result) {
return;
}
if (DEBUG_DOMAIN_TO_FIND) {
if (hostname.includes(DEBUG_DOMAIN_TO_FIND)) {
warnOnce(filterRulesUrl.toString(), flag === 0 || flag === -1, DEBUG_DOMAIN_TO_FIND);
foundDebugDomain = true;
const flag = result[1];
const hostname = result[0];
console.log({ result, flag });
if (DEBUG_DOMAIN_TO_FIND) {
if (hostname.includes(DEBUG_DOMAIN_TO_FIND)) {
warnOnce(filterRulesUrl.toString(), flag === 0 || flag === -1, DEBUG_DOMAIN_TO_FIND);
foundDebugDomain = true;
console.log({ result, flag });
}
}
switch (flag) {
case 0:
if (hostname[0] !== '.') {
whitelistDomainSets.add(`.${hostname}`);
} else {
whitelistDomainSets.add(hostname);
}
}
switch (flag) {
case 0:
addToWhiteList(hostname, true);
break;
case -1:
addToWhiteList(hostname, false);
break;
case 1:
addToBlackList(hostname, false);
break;
case 2:
addToBlackList(hostname, true);
break;
default:
throw new Error(`Unknown flag: ${flag as any}`);
}
break;
case -1:
whitelistDomainSets.add(hostname);
break;
case 1:
blacklistDomainSets.add(hostname);
break;
case 2:
if (hostname[0] !== '.') {
blacklistDomainSets.add(`.${hostname}`);
} else {
blacklistDomainSets.add(hostname);
}
break;
default:
throw new Error(`Unknown flag: ${flag as any}`);
}
};
@ -302,7 +289,6 @@ function parse($line: string, gorhill: PublicSuffixList): null | [hostname: stri
) {
const hostname = normalizeDomain(filter.hostname);
if (!hostname) {
console.log(' * [parse-filter E0000] invalid domain:', filter.hostname);
return null;
}