From a558b82c08b9ff855a7d2b0ed0fae7fd764ffabf Mon Sep 17 00:00:00 2001 From: SukkaW Date: Sat, 9 Dec 2023 21:27:03 +0800 Subject: [PATCH] Chore: simplify build infra --- Build/build-mitm-hostname.js | 37 +++++------ Build/build-reject-domainset.ts | 14 ++--- Build/lib/fetch-remote-text-by-line.ts | 4 +- Build/lib/parse-filter.ts | 86 +++++++++++--------------- 4 files changed, 65 insertions(+), 76 deletions(-) diff --git a/Build/build-mitm-hostname.js b/Build/build-mitm-hostname.js index e1563736..fcba416c 100644 --- a/Build/build-mitm-hostname.js +++ b/Build/build-mitm-hostname.js @@ -94,7 +94,7 @@ const PRESET_MITM_HOSTNAMES = [ ); })); - let mitmDomains = new Set(PRESET_MITM_HOSTNAMES); // Special case for parsed failed + const mitmDomains = new Set(PRESET_MITM_HOSTNAMES); // Special case for parsed failed const parsedFailures = []; const dedupedUrlRegexPaths = [...new Set(urlRegexPaths)]; @@ -109,23 +109,24 @@ const PRESET_MITM_HOSTNAMES = [ } }); - mitmDomains = [...mitmDomains].filter(i => { - return i.length > 3 - && !i.includes('.mp4') // Special Case - && i !== '(www.)' // Special Case - && !(i !== '*.meituan.net' && i.endsWith('.meituan.net')) - && !i.startsWith('.') - && !i.endsWith('.') - && !i.endsWith('*'); - }); - - const mitmDomainsRegExpArray = mitmDomains.map(i => { - return new RegExp( - escapeRegExp(i) - .replaceAll('{www or not}', '(www.)?') - .replaceAll('\\*', '(.*)') - ); - }); + const mitmDomainsRegExpArray = mitmDomains + .slice() + .filter(i => { + return i.length > 3 + && !i.includes('.mp4') // Special Case + && i !== '(www.)' // Special Case + && !(i !== '*.meituan.net' && i.endsWith('.meituan.net')) + && !i.startsWith('.') + && !i.endsWith('.') + && !i.endsWith('*'); + }) + .map(i => { + return new RegExp( + escapeRegExp(i) + .replaceAll('{www or not}', '(www.)?') + .replaceAll('\\*', '(.*)') + ); + }); const parsedDomainsData = []; dedupedUrlRegexPaths.forEach(i => { diff --git a/Build/build-reject-domainset.ts b/Build/build-reject-domainset.ts index 983aec99..0ddf98d1 100644 --- a/Build/build-reject-domainset.ts +++ b/Build/build-reject-domainset.ts @@ -108,18 +108,18 @@ export const buildRejectDomainSet = task(import.meta.path, async () => { console.log(`Import ${previousSize} rules from reject_sukka.conf!`); for await (const line of readFileByLine(path.resolve(import.meta.dir, '../Source/non_ip/reject.conf'))) { - if (line.startsWith('DOMAIN-KEYWORD')) { - const [, ...keywords] = line.split(','); - domainKeywordsSet.add(keywords.join(',').trim()); - } else if (line.startsWith('DOMAIN-SUFFIX')) { - const [, ...keywords] = line.split(','); - domainSuffixSet.add(keywords.join(',').trim()); + const [type, keyword] = line.split(','); + + if (type === 'DOMAIN-KEYWORD') { + domainKeywordsSet.add(keyword.trim()); + } else if (type === 'DOMAIN-SUFFIX') { + domainSuffixSet.add(keyword.trim()); } } for await (const line of readFileByLine(path.resolve(import.meta.dir, '../List/domainset/reject_phishing.conf'))) { const l = processLine(line); - if (l && l[0] === '.') { + if (l?.[0] === '.') { domainSuffixSet.add(l.slice(1)); } } diff --git a/Build/lib/fetch-remote-text-by-line.ts b/Build/lib/fetch-remote-text-by-line.ts index 2a8f7071..079fd794 100644 --- a/Build/lib/fetch-remote-text-by-line.ts +++ b/Build/lib/fetch-remote-text-by-line.ts @@ -23,9 +23,11 @@ import { fetchWithRetry, defaultRequestInit } from './fetch-retry'; const decoder = new TextDecoder('utf-8'); -export async function *readFileByLine(file: string | BunFile): AsyncGenerator { +export async function *readFileByLine(file: string | URL | BunFile): AsyncGenerator { if (typeof file === 'string') { file = Bun.file(file); + } else if (!('writer' in file)) { + file = Bun.file(file); } let buf = ''; diff --git a/Build/lib/parse-filter.ts b/Build/lib/parse-filter.ts index 20da8cef..d5b3afec 100644 --- a/Build/lib/parse-filter.ts +++ b/Build/lib/parse-filter.ts @@ -111,29 +111,6 @@ export async function processFilterRules( const whitelistDomainSets = new Set(); const blacklistDomainSets = new Set(); - /** - * @param {string} domainToBeAddedToBlack - * @param {boolean} isSubDomain - */ - const addToBlackList = (domainToBeAddedToBlack: string, isSubDomain: boolean) => { - if (isSubDomain && domainToBeAddedToBlack[0] !== '.') { - blacklistDomainSets.add(`.${domainToBeAddedToBlack}`); - } else { - blacklistDomainSets.add(domainToBeAddedToBlack); - } - }; - /** - * @param {string} domainToBeAddedToWhite - * @param {boolean} [isSubDomain] - */ - const addToWhiteList = (domainToBeAddedToWhite: string, isSubDomain = true) => { - if (isSubDomain && domainToBeAddedToWhite[0] !== '.') { - whitelistDomainSets.add(`.${domainToBeAddedToWhite}`); - } else { - whitelistDomainSets.add(domainToBeAddedToWhite); - } - }; - let downloadTime = 0; const gorhill = await getGorhillPublicSuffixPromise(); @@ -142,35 +119,45 @@ export async function processFilterRules( */ const lineCb = (line: string) => { const result = parse(line, gorhill); - if (result) { - const flag = result[1]; - const hostname = result[0]; + if (!result) { + return; + } - if (DEBUG_DOMAIN_TO_FIND) { - if (hostname.includes(DEBUG_DOMAIN_TO_FIND)) { - warnOnce(filterRulesUrl.toString(), flag === 0 || flag === -1, DEBUG_DOMAIN_TO_FIND); - foundDebugDomain = true; + const flag = result[1]; + const hostname = result[0]; - console.log({ result, flag }); + if (DEBUG_DOMAIN_TO_FIND) { + if (hostname.includes(DEBUG_DOMAIN_TO_FIND)) { + warnOnce(filterRulesUrl.toString(), flag === 0 || flag === -1, DEBUG_DOMAIN_TO_FIND); + foundDebugDomain = true; + + console.log({ result, flag }); + } + } + + switch (flag) { + case 0: + if (hostname[0] !== '.') { + whitelistDomainSets.add(`.${hostname}`); + } else { + whitelistDomainSets.add(hostname); } - } - - switch (flag) { - case 0: - addToWhiteList(hostname, true); - break; - case -1: - addToWhiteList(hostname, false); - break; - case 1: - addToBlackList(hostname, false); - break; - case 2: - addToBlackList(hostname, true); - break; - default: - throw new Error(`Unknown flag: ${flag as any}`); - } + break; + case -1: + whitelistDomainSets.add(hostname); + break; + case 1: + blacklistDomainSets.add(hostname); + break; + case 2: + if (hostname[0] !== '.') { + blacklistDomainSets.add(`.${hostname}`); + } else { + blacklistDomainSets.add(hostname); + } + break; + default: + throw new Error(`Unknown flag: ${flag as any}`); } }; @@ -302,7 +289,6 @@ function parse($line: string, gorhill: PublicSuffixList): null | [hostname: stri ) { const hostname = normalizeDomain(filter.hostname); if (!hostname) { - console.log(' * [parse-filter E0000] invalid domain:', filter.hostname); return null; }