GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2025-12-12 01:00:34 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Make build faster

Browse Source
This commit is contained in:
SukkaW 2023-05-26 23:43:57 +08:00
parent 72f8d06bd2
commit c12a75b00a
3 changed files with 119 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

178
Build/build-reject-domainset.js
View File

@ -1,3 +1,4 @@
// @ts-check
const { promises: fsPromises } = require('fs'); const { promises: fsPromises } = require('fs');
const fse = require('fs-extra'); const fse = require('fs-extra');
const { resolve: pathResolve } = require('path'); const { resolve: pathResolve } = require('path');
@ -12,7 +13,12 @@ const { HOSTS, ADGUARD_FILTERS, PREDEFINED_WHITELIST, PREDEFINED_ENFORCED_BACKLI
const { withBannerArray } = require('./lib/with-banner'); const { withBannerArray } = require('./lib/with-banner');
const { compareAndWriteFile } = require('./lib/string-array-compare'); const { compareAndWriteFile } = require('./lib/string-array-compare');
/** Whitelists */
const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST); const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST);
/** @type {Set<string>} Dedupe domains inclued by DOMAIN-KEYWORD */
const domainKeywordsSet = new Set();
/** @type {Set<string>} Dedupe domains included by DOMAIN-SUFFIX */
const domainSuffixSet = new Set();
(async () => { (async () => {
console.time('Total Time - build-reject-domain-set'); console.time('Total Time - build-reject-domain-set');
@ -46,7 +52,7 @@ const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST);
await Promise.all(ADGUARD_FILTERS.map(input => { await Promise.all(ADGUARD_FILTERS.map(input => {
const promise = typeof input === 'string' const promise = typeof input === 'string'
? processFilterRules(input, undefined, false) ? processFilterRules(input, undefined, false)
: processFilterRules(input[0], input[1] ?? undefined, input[2] ?? false) : processFilterRules(input[0], input[1] || undefined, input[2] ?? false)
return promise.then((i) => { return promise.then((i) => {
if (i) { if (i) {
@ -118,45 +124,41 @@ const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST);
}); });
}); });
// Copy reject_sukka.conf for backward compatibility
await fse.copy(pathResolve(__dirname, '../Source/domainset/reject_sukka.conf'), pathResolve(__dirname, '../List/domainset/reject_sukka.conf'))
previousSize = domainSets.size - previousSize; previousSize = domainSets.size - previousSize;
console.log(`Import ${previousSize} rules from reject_sukka.conf!`); console.log(`Import ${previousSize} rules from reject_sukka.conf!`);
// Read DOMAIN Keyword await Promise.all([
const domainKeywordsSet = new Set(); // Copy reject_sukka.conf for backward compatibility
const domainSuffixSet = new Set(); fse.copy(pathResolve(__dirname, '../Source/domainset/reject_sukka.conf'), pathResolve(__dirname, '../List/domainset/reject_sukka.conf')),
fsPromises.readFile(pathResolve(__dirname, '../List/non_ip/reject.conf'), { encoding: 'utf-8' }).then(data => {
data.split('\n').forEach(line => {
if (line.startsWith('DOMAIN-KEYWORD')) {
const [, ...keywords] = line.split(',');
domainKeywordsSet.add(keywords.join(',').trim());
} else if (line.startsWith('DOMAIN-SUFFIX')) {
const [, ...keywords] = line.split(',');
domainSuffixSet.add(keywords.join(',').trim());
}
});
}),
// Read Special Phishing Suffix list
fsPromises.readFile(pathResolve(__dirname, '../List/domainset/reject_phishing.conf'), { encoding: 'utf-8' }).then(data => {
data.split('\n').forEach(line => {
const trimmed = line.trim();
if (
line.startsWith('#')
|| line.startsWith(' ')
|| line.startsWith('\r')
|| line.startsWith('\n')
|| trimmed === ''
) {
return;
}
await fsPromises.readFile(pathResolve(__dirname, '../List/non_ip/reject.conf'), { encoding: 'utf-8' }).then(data => { domainSuffixSet.add(trimmed);
data.split('\n').forEach(line => { });
if (line.startsWith('DOMAIN-KEYWORD')) { })
const [, ...keywords] = line.split(','); ]);
domainKeywordsSet.add(keywords.join(',').trim());
} else if (line.startsWith('DOMAIN-SUFFIX')) {
const [, ...keywords] = line.split(',');
domainSuffixSet.add(keywords.join(',').trim());
}
});
});
// Read Special Phishing Suffix list
await fsPromises.readFile(pathResolve(__dirname, '../List/domainset/reject_phishing.conf'), { encoding: 'utf-8' }).then(data => {
data.split('\n').forEach(line => {
const trimmed = line.trim();
if (
line.startsWith('#')
|| line.startsWith(' ')
|| line.startsWith('\r')
|| line.startsWith('\n')
|| trimmed === ''
) {
return;
}
domainSuffixSet.add(trimmed);
});
});
console.log(`Import ${domainKeywordsSet.size} black keywords and ${domainSuffixSet.size} black suffixes!`); console.log(`Import ${domainKeywordsSet.size} black keywords and ${domainSuffixSet.size} black suffixes!`);
@ -166,31 +168,7 @@ const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST);
console.time(`* Dedupe from black keywords/suffixes`); console.time(`* Dedupe from black keywords/suffixes`);
for (const domain of domainSets) { for (const domain of domainSets) {
let isTobeRemoved = false; if (isMatchKeyword(domain) || isMatchSuffix(domain) || isInWhiteList(domain)) {
for (const suffix of domainSuffixSet) {
if (domain.endsWith(suffix)) {
isTobeRemoved = true;
break;
}
}
if (!isTobeRemoved) {
for (const keyword of domainKeywordsSet) {
if (domain.includes(keyword)) {
isTobeRemoved = true;
break;
}
}
}
if (!isTobeRemoved) {
if (isInWhiteList(domain)) {
isTobeRemoved = true;
}
}
if (isTobeRemoved) {
domainSets.delete(domain); domainSets.delete(domain);
} }
} }
@ -204,44 +182,37 @@ const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST);
const START_TIME = Date.now(); const START_TIME = Date.now();
const domainSetsArray = Array.from(domainSets);
const piscina = new Piscina({ const piscina = new Piscina({
filename: pathResolve(__dirname, 'worker/build-reject-domainset-worker.js'), filename: pathResolve(__dirname, 'worker/build-reject-domainset-worker.js'),
workerData: preprocessFullDomainSetBeforeUsedAsWorkerData([...domainSets]), workerData: preprocessFullDomainSetBeforeUsedAsWorkerData(Array.from(domainSetsArray)),
idleTimeout: 50, idleTimeout: 50,
minThreads: threads, minThreads: threads,
maxThreads: threads maxThreads: threads
}); });
console.log(`Launching ${threads} threads...`) console.log(preprocessFullDomainSetBeforeUsedAsWorkerData(Array.from(domainSetsArray)).length);
const tasksArray = Array.from(domainSets) console.log(`Launching ${threads} threads...`);
.reduce((result, element, index) => {
const chunk = index % threads;
result[chunk] ??= [];
result[chunk].push(element); const tasksArray = domainSetsArray.reduce((result, element, index) => {
return result; const chunk = index % threads;
}, []); result[chunk] ??= [];
( result[chunk].push(element);
await Promise.all( return result;
Array.from(domainSets) }, /** @type {string[][]} */([]));
.reduce((result, element, index) => {
const chunk = index % threads; (await Promise.all(
result[chunk] ??= []; tasksArray.map(chunk => piscina.run({ chunk }))
result[chunk].push(element); )).forEach((result, taskIndex) => {
return result; const chunk = tasksArray[taskIndex];
}, []) for (let i = 0, len = result.length; i < len; i++) {
.map(chunk => piscina.run({ chunk }, { name: 'dedupe' })) if (result[i]) {
) domainSets.delete(chunk[i]);
).forEach((result, taskIndex) => { }
const chunk = tasksArray[taskIndex];
for (let i = 0, len = result.length; i < len; i++) {
if (result[i]) {
domainSets.delete(chunk[i]);
} }
} });
});
console.log(`* Dedupe from covered subdomain - ${(Date.now() - START_TIME) / 1000}s`); console.log(`* Dedupe from covered subdomain - ${(Date.now() - START_TIME) / 1000}s`);
console.log(`Deduped ${previousSize - domainSets.size} rules!`); console.log(`Deduped ${previousSize - domainSets.size} rules!`);
@ -259,7 +230,7 @@ const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST);
} }
return 0; return 0;
}; };
const sortedDomainSets = [...domainSets] const sortedDomainSets = Array.from(domainSets)
.map((v) => { .map((v) => {
return { v, domain: getDomain(v.charCodeAt(0) === 46 ? v.slice(1) : v)?.toLowerCase() || v }; return { v, domain: getDomain(v.charCodeAt(0) === 46 ? v.slice(1) : v)?.toLowerCase() || v };
}) })
@ -296,6 +267,35 @@ const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST);
} }
})(); })();
/**
* @param {string} domain
*/
function isMatchKeyword(domain) {
for (const keyword of domainKeywordsSet) {
if (domain.includes(keyword)) {
return true;
}
}
return false;
}
/**
* @param {string} domain
*/
function isMatchSuffix(domain) {
for (const suffix of domainSuffixSet) {
if (domain.endsWith(suffix)) {
return true;
}
}
return false;
}
/**
* @param {string} domain
*/
function isInWhiteList(domain) { function isInWhiteList(domain) {
for (const white of filterRuleWhitelistDomainSets) { for (const white of filterRuleWhitelistDomainSets) {
if (domain === white || domain.endsWith(white)) { if (domain === white || domain.endsWith(white)) {

19
Build/lib/parse-filter.js
View File

@ -18,7 +18,7 @@ const warnOnce = (url, isWhite, ...message) => {
/** /**
* @param {string | URL} domainListsUrl * @param {string | URL} domainListsUrl
*/ */
async function processDomainLists (domainListsUrl) { async function processDomainLists(domainListsUrl) {
if (typeof domainListsUrl === 'string') { if (typeof domainListsUrl === 'string') {
domainListsUrl = new URL(domainListsUrl); domainListsUrl = new URL(domainListsUrl);
} }
@ -55,7 +55,7 @@ async function processDomainLists (domainListsUrl) {
/** /**
* @param {string | URL} hostsUrl * @param {string | URL} hostsUrl
*/ */
async function processHosts (hostsUrl, includeAllSubDomain = false) { async function processHosts(hostsUrl, includeAllSubDomain = false) {
console.time(` - processHosts: ${hostsUrl}`); console.time(` - processHosts: ${hostsUrl}`);
if (typeof hostsUrl === 'string') { if (typeof hostsUrl === 'string') {
@ -99,10 +99,10 @@ async function processHosts (hostsUrl, includeAllSubDomain = false) {
/** /**
* @param {string | URL} filterRulesUrl * @param {string | URL} filterRulesUrl
* @param {(string | URL)[] | undefined} fallbackUrls * @param {readonly (string | URL)[] | undefined} [fallbackUrls]
* @returns {Promise<{ white: Set<string>, black: Set<string>, foundDebugDomain: boolean, parseFailed: boolean }>} * @returns {Promise<{ white: Set<string>, black: Set<string>, foundDebugDomain: boolean, parseFailed: boolean }>}
*/ */
async function processFilterRules (filterRulesUrl, fallbackUrls, includeThirdParties = false) { async function processFilterRules(filterRulesUrl, fallbackUrls, includeThirdParties = false) {
console.time(` - processFilterRules: ${filterRulesUrl}`); console.time(` - processFilterRules: ${filterRulesUrl}`);
/** @type Set<string> */ /** @type Set<string> */
@ -390,10 +390,13 @@ async function processFilterRules (filterRulesUrl, fallbackUrls, includeThirdPar
}; };
} }
function preprocessFullDomainSetBeforeUsedAsWorkerData (data) { /**
return data.filter(domain => ( * @param {string[]} data
domain.charCodeAt(0) === 46 */
)); function preprocessFullDomainSetBeforeUsedAsWorkerData(data) {
return data
.filter(domain => domain.charCodeAt(0) === 46)
.sort((a, b) => a.length - b.length);
} }

48
Build/worker/build-reject-domainset-worker.js
View File

@ -1,51 +1,41 @@
const Piscina = require('piscina'); const Piscina = require('piscina');
const { isCI } = require('ci-info'); // const { isCI } = require('ci-info');
const fullsetDomainStartsWithADot = Piscina.workerData const fullsetDomainStartsWithADot = Piscina.workerData
const totalLen = fullsetDomainStartsWithADot.length; const totalLen = fullsetDomainStartsWithADot.length;
const log = isCI ? () => { } : console.log.bind(console); // const log = isCI ? () => { } : console.log.bind(console);
module.exports.dedupe = ({ chunk }) => { module.exports = ({ chunk }) => {
const chunkLength = chunk.length; const chunkLength = chunk.length;
const outputToBeRemoved = new Int8Array(chunkLength); const outputToBeRemoved = new Int8Array(chunkLength);
for (let i = 0; i < chunkLength; i++) { for (let i = 0; i < chunkLength; i++) {
const domainFromInput = chunk[i]; const domainFromInputChunk = chunk[i];
for (let j = 0; j < totalLen; j++) { for (let j = 0; j < totalLen; j++) {
const domainFromFullSet = fullsetDomainStartsWithADot[j]; const domainStartsWithADotAndFromFullSet = fullsetDomainStartsWithADot[j];
// domainFromFullSet is always startsWith "." // domainFromFullSet is always startsWith "."
if (domainStartsWithADotAndFromFullSet === domainFromInputChunk) continue;
if (domainFromFullSet === domainFromInput) continue; const domainFromInputLen = domainFromInputChunk.length;
const domainFromFullSetLen = domainStartsWithADotAndFromFullSet.length;
const domainFromInputLen = domainFromInput.length; if (domainFromInputLen < domainFromFullSetLen) {
const domainFromFullSetLen = domainFromFullSet.length; if (domainFromInputLen + 1 === domainFromFullSetLen) {
// !domainFromInput.starsWith('.') && `.${domainFromInput}` === domainFromFullSet
// !domainFromInput.starsWith('.') && `.${domainFromInput}` === domainFromFullSet if (domainFromInputChunk.charCodeAt(0) !== 46 && domainFromInputChunk.endsWith(domainStartsWithADotAndFromFullSet)) {
if (domainFromInput[0] !== '.' && domainFromInputLen + 1 === domainFromFullSetLen) { outputToBeRemoved[i] = 1;
let shouldBeRemoved = true; // log(domainFromInputChunk, domainStartsWithADotAndFromFullSet)
for (let k = 0; k < domainFromInputLen; k++) {
if (domainFromFullSet[k + 1] !== domainFromInput[k]) {
shouldBeRemoved = false;
break; break;
} }
} } else {
if (shouldBeRemoved) {
outputToBeRemoved[i] = 1;
log(domainFromInput, domainFromFullSet)
break;
}
}
if (domainFromInputLen > domainFromFullSetLen) {
// domainFromInput is now startsWith a "."
if (domainFromInput.endsWith(domainFromFullSet)) {
outputToBeRemoved[i] = 1;
log(domainFromInput, domainFromFullSet)
break; break;
} }
} else if (domainFromInputChunk.endsWith(domainStartsWithADotAndFromFullSet)) {
outputToBeRemoved[i] = 1;
// log(domainFromInputChunk, domainStartsWithADotAndFromFullSet)
break;
} }
} }
} }