GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2025-12-14 02:00:37 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Make build faster

Browse Source
This commit is contained in:
SukkaW 2023-05-26 23:43:57 +08:00
parent 72f8d06bd2
commit c12a75b00a
3 changed files with 119 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

112
Build/build-reject-domainset.js
View File

@ -1,3 +1,4 @@
// @ts-check
const { promises: fsPromises } = require('fs'); const { promises: fsPromises } = require('fs');
const fse = require('fs-extra'); const fse = require('fs-extra');
const { resolve: pathResolve } = require('path'); const { resolve: pathResolve } = require('path');
@ -12,7 +13,12 @@ const { HOSTS, ADGUARD_FILTERS, PREDEFINED_WHITELIST, PREDEFINED_ENFORCED_BACKLI
const { withBannerArray } = require('./lib/with-banner'); const { withBannerArray } = require('./lib/with-banner');
const { compareAndWriteFile } = require('./lib/string-array-compare'); const { compareAndWriteFile } = require('./lib/string-array-compare');
/** Whitelists */
const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST); const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST);
/** @type {Set<string>} Dedupe domains inclued by DOMAIN-KEYWORD */
const domainKeywordsSet = new Set();
/** @type {Set<string>} Dedupe domains included by DOMAIN-SUFFIX */
const domainSuffixSet = new Set();
(async () => { (async () => {
console.time('Total Time - build-reject-domain-set'); console.time('Total Time - build-reject-domain-set');
@ -46,7 +52,7 @@ const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST);
await Promise.all(ADGUARD_FILTERS.map(input => { await Promise.all(ADGUARD_FILTERS.map(input => {
const promise = typeof input === 'string' const promise = typeof input === 'string'
? processFilterRules(input, undefined, false) ? processFilterRules(input, undefined, false)
: processFilterRules(input[0], input[1] ?? undefined, input[2] ?? false) : processFilterRules(input[0], input[1] || undefined, input[2] ?? false)
return promise.then((i) => { return promise.then((i) => {
if (i) { if (i) {
@ -118,17 +124,13 @@ const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST);
}); });
}); });
// Copy reject_sukka.conf for backward compatibility
await fse.copy(pathResolve(__dirname, '../Source/domainset/reject_sukka.conf'), pathResolve(__dirname, '../List/domainset/reject_sukka.conf'))
previousSize = domainSets.size - previousSize; previousSize = domainSets.size - previousSize;
console.log(`Import ${previousSize} rules from reject_sukka.conf!`); console.log(`Import ${previousSize} rules from reject_sukka.conf!`);
// Read DOMAIN Keyword await Promise.all([
const domainKeywordsSet = new Set(); // Copy reject_sukka.conf for backward compatibility
const domainSuffixSet = new Set(); fse.copy(pathResolve(__dirname, '../Source/domainset/reject_sukka.conf'), pathResolve(__dirname, '../List/domainset/reject_sukka.conf')),
fsPromises.readFile(pathResolve(__dirname, '../List/non_ip/reject.conf'), { encoding: 'utf-8' }).then(data => {
await fsPromises.readFile(pathResolve(__dirname, '../List/non_ip/reject.conf'), { encoding: 'utf-8' }).then(data => {
data.split('\n').forEach(line => { data.split('\n').forEach(line => {
if (line.startsWith('DOMAIN-KEYWORD')) { if (line.startsWith('DOMAIN-KEYWORD')) {
const [, ...keywords] = line.split(','); const [, ...keywords] = line.split(',');
@ -138,10 +140,9 @@ const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST);
domainSuffixSet.add(keywords.join(',').trim()); domainSuffixSet.add(keywords.join(',').trim());
} }
}); });
}); }),
// Read Special Phishing Suffix list // Read Special Phishing Suffix list
await fsPromises.readFile(pathResolve(__dirname, '../List/domainset/reject_phishing.conf'), { encoding: 'utf-8' }).then(data => { fsPromises.readFile(pathResolve(__dirname, '../List/domainset/reject_phishing.conf'), { encoding: 'utf-8' }).then(data => {
data.split('\n').forEach(line => { data.split('\n').forEach(line => {
const trimmed = line.trim(); const trimmed = line.trim();
if ( if (
@ -156,7 +157,8 @@ const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST);
domainSuffixSet.add(trimmed); domainSuffixSet.add(trimmed);
}); });
}); })
]);
console.log(`Import ${domainKeywordsSet.size} black keywords and ${domainSuffixSet.size} black suffixes!`); console.log(`Import ${domainKeywordsSet.size} black keywords and ${domainSuffixSet.size} black suffixes!`);
@ -166,31 +168,7 @@ const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST);
console.time(`* Dedupe from black keywords/suffixes`); console.time(`* Dedupe from black keywords/suffixes`);
for (const domain of domainSets) { for (const domain of domainSets) {
let isTobeRemoved = false; if (isMatchKeyword(domain) || isMatchSuffix(domain) || isInWhiteList(domain)) {
for (const suffix of domainSuffixSet) {
if (domain.endsWith(suffix)) {
isTobeRemoved = true;
break;
}
}
if (!isTobeRemoved) {
for (const keyword of domainKeywordsSet) {
if (domain.includes(keyword)) {
isTobeRemoved = true;
break;
}
}
}
if (!isTobeRemoved) {
if (isInWhiteList(domain)) {
isTobeRemoved = true;
}
}
if (isTobeRemoved) {
domainSets.delete(domain); domainSets.delete(domain);
} }
} }
@ -204,37 +182,30 @@ const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST);
const START_TIME = Date.now(); const START_TIME = Date.now();
const domainSetsArray = Array.from(domainSets);
const piscina = new Piscina({ const piscina = new Piscina({
filename: pathResolve(__dirname, 'worker/build-reject-domainset-worker.js'), filename: pathResolve(__dirname, 'worker/build-reject-domainset-worker.js'),
workerData: preprocessFullDomainSetBeforeUsedAsWorkerData([...domainSets]), workerData: preprocessFullDomainSetBeforeUsedAsWorkerData(Array.from(domainSetsArray)),
idleTimeout: 50, idleTimeout: 50,
minThreads: threads, minThreads: threads,
maxThreads: threads maxThreads: threads
}); });
console.log(`Launching ${threads} threads...`) console.log(preprocessFullDomainSetBeforeUsedAsWorkerData(Array.from(domainSetsArray)).length);
const tasksArray = Array.from(domainSets) console.log(`Launching ${threads} threads...`);
.reduce((result, element, index) => {
const tasksArray = domainSetsArray.reduce((result, element, index) => {
const chunk = index % threads; const chunk = index % threads;
result[chunk] ??= []; result[chunk] ??= [];
result[chunk].push(element); result[chunk].push(element);
return result; return result;
}, []); }, /** @type {string[][]} */([]));
( (await Promise.all(
await Promise.all( tasksArray.map(chunk => piscina.run({ chunk }))
Array.from(domainSets) )).forEach((result, taskIndex) => {
.reduce((result, element, index) => {
const chunk = index % threads;
result[chunk] ??= [];
result[chunk].push(element);
return result;
}, [])
.map(chunk => piscina.run({ chunk }, { name: 'dedupe' }))
)
).forEach((result, taskIndex) => {
const chunk = tasksArray[taskIndex]; const chunk = tasksArray[taskIndex];
for (let i = 0, len = result.length; i < len; i++) { for (let i = 0, len = result.length; i < len; i++) {
if (result[i]) { if (result[i]) {
@ -259,7 +230,7 @@ const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST);
} }
return 0; return 0;
}; };
const sortedDomainSets = [...domainSets] const sortedDomainSets = Array.from(domainSets)
.map((v) => { .map((v) => {
return { v, domain: getDomain(v.charCodeAt(0) === 46 ? v.slice(1) : v)?.toLowerCase() || v }; return { v, domain: getDomain(v.charCodeAt(0) === 46 ? v.slice(1) : v)?.toLowerCase() || v };
}) })
@ -296,6 +267,35 @@ const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST);
} }
})(); })();
/**
* @param {string} domain
*/
function isMatchKeyword(domain) {
for (const keyword of domainKeywordsSet) {
if (domain.includes(keyword)) {
return true;
}
}
return false;
}
/**
* @param {string} domain
*/
function isMatchSuffix(domain) {
for (const suffix of domainSuffixSet) {
if (domain.endsWith(suffix)) {
return true;
}
}
return false;
}
/**
* @param {string} domain
*/
function isInWhiteList(domain) { function isInWhiteList(domain) {
for (const white of filterRuleWhitelistDomainSets) { for (const white of filterRuleWhitelistDomainSets) {
if (domain === white || domain.endsWith(white)) { if (domain === white || domain.endsWith(white)) {

11
Build/lib/parse-filter.js
View File

@ -99,7 +99,7 @@ async function processHosts (hostsUrl, includeAllSubDomain = false) {
/** /**
* @param {string | URL} filterRulesUrl * @param {string | URL} filterRulesUrl
* @param {(string | URL)[] | undefined} fallbackUrls * @param {readonly (string | URL)[] | undefined} [fallbackUrls]
* @returns {Promise<{ white: Set<string>, black: Set<string>, foundDebugDomain: boolean, parseFailed: boolean }>} * @returns {Promise<{ white: Set<string>, black: Set<string>, foundDebugDomain: boolean, parseFailed: boolean }>}
*/ */
async function processFilterRules(filterRulesUrl, fallbackUrls, includeThirdParties = false) { async function processFilterRules(filterRulesUrl, fallbackUrls, includeThirdParties = false) {
@ -390,10 +390,13 @@ async function processFilterRules (filterRulesUrl, fallbackUrls, includeThirdPar
}; };
} }
/**
* @param {string[]} data
*/
function preprocessFullDomainSetBeforeUsedAsWorkerData(data) { function preprocessFullDomainSetBeforeUsedAsWorkerData(data) {
return data.filter(domain => ( return data
domain.charCodeAt(0) === 46 .filter(domain => domain.charCodeAt(0) === 46)
)); .sort((a, b) => a.length - b.length);
} }

42
Build/worker/build-reject-domainset-worker.js
View File

@ -1,54 +1,44 @@
const Piscina = require('piscina'); const Piscina = require('piscina');
const { isCI } = require('ci-info'); // const { isCI } = require('ci-info');
const fullsetDomainStartsWithADot = Piscina.workerData const fullsetDomainStartsWithADot = Piscina.workerData
const totalLen = fullsetDomainStartsWithADot.length; const totalLen = fullsetDomainStartsWithADot.length;
const log = isCI ? () => { } : console.log.bind(console); // const log = isCI ? () => { } : console.log.bind(console);
module.exports.dedupe = ({ chunk }) => { module.exports = ({ chunk }) => {
const chunkLength = chunk.length; const chunkLength = chunk.length;
const outputToBeRemoved = new Int8Array(chunkLength); const outputToBeRemoved = new Int8Array(chunkLength);
for (let i = 0; i < chunkLength; i++) { for (let i = 0; i < chunkLength; i++) {
const domainFromInput = chunk[i]; const domainFromInputChunk = chunk[i];
for (let j = 0; j < totalLen; j++) { for (let j = 0; j < totalLen; j++) {
const domainFromFullSet = fullsetDomainStartsWithADot[j]; const domainStartsWithADotAndFromFullSet = fullsetDomainStartsWithADot[j];
// domainFromFullSet is always startsWith "." // domainFromFullSet is always startsWith "."
if (domainStartsWithADotAndFromFullSet === domainFromInputChunk) continue;
if (domainFromFullSet === domainFromInput) continue; const domainFromInputLen = domainFromInputChunk.length;
const domainFromFullSetLen = domainStartsWithADotAndFromFullSet.length;
const domainFromInputLen = domainFromInput.length;
const domainFromFullSetLen = domainFromFullSet.length;
if (domainFromInputLen < domainFromFullSetLen) {
if (domainFromInputLen + 1 === domainFromFullSetLen) {
// !domainFromInput.starsWith('.') && `.${domainFromInput}` === domainFromFullSet // !domainFromInput.starsWith('.') && `.${domainFromInput}` === domainFromFullSet
if (domainFromInput[0] !== '.' && domainFromInputLen + 1 === domainFromFullSetLen) { if (domainFromInputChunk.charCodeAt(0) !== 46 && domainFromInputChunk.endsWith(domainStartsWithADotAndFromFullSet)) {
let shouldBeRemoved = true;
for (let k = 0; k < domainFromInputLen; k++) {
if (domainFromFullSet[k + 1] !== domainFromInput[k]) {
shouldBeRemoved = false;
break;
}
}
if (shouldBeRemoved) {
outputToBeRemoved[i] = 1; outputToBeRemoved[i] = 1;
log(domainFromInput, domainFromFullSet) // log(domainFromInputChunk, domainStartsWithADotAndFromFullSet)
break; break;
} }
} else {
break;
} }
if (domainFromInputLen > domainFromFullSetLen) { } else if (domainFromInputChunk.endsWith(domainStartsWithADotAndFromFullSet)) {
// domainFromInput is now startsWith a "."
if (domainFromInput.endsWith(domainFromFullSet)) {
outputToBeRemoved[i] = 1; outputToBeRemoved[i] = 1;
log(domainFromInput, domainFromFullSet) // log(domainFromInputChunk, domainStartsWithADotAndFromFullSet)
break; break;
} }
} }
} }
}
return Piscina.move(outputToBeRemoved); return Piscina.move(outputToBeRemoved);
}; };