GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2026-01-29 01:51:52 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Chore: better logging

Browse Source
This commit is contained in:
SukkaW
2022-09-22 22:57:00 +08:00
parent d5f0f2428f
commit 8a7f32e2cd
9 changed files with 47 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
Build/build-reject-domainset.js
View File

@@ -7,9 +7,13 @@ const { isCI } = require('ci-info');
const threads = isCI ? cpuCount : cpuCount / 2;
(async () => {
console.time('Total Time - build-reject-domain-set');
/** @type Set<string> */
const domainSets = new Set();
console.time('* Download and process Hosts');
// Parse from remote hosts & domain lists
(await Promise.all([
processHosts('https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=0&mimetype=plaintext', true),
@@ -23,6 +27,8 @@ const threads = isCI ? cpuCount : cpuCount / 2;
});
});
console.timeEnd('* Download and process Hosts');
let previousSize = domainSets.size;
console.log(`Import ${previousSize} rules from hosts files!`);
@@ -83,6 +89,7 @@ const threads = isCI ? cpuCount : cpuCount / 2;
'storage.yandexcloud.net' // phishing list
]);
console.time('* Download and process AdBlock Filter Rules');
(await Promise.all([
// Easy List
[
@@ -147,6 +154,8 @@ const threads = isCI ? cpuCount : cpuCount / 2;
black.forEach(i => domainSets.add(i));
});
console.timeEnd('* Download and process AdBlock Filter Rules');
previousSize = domainSets.size - previousSize;
console.log(`Import ${previousSize} rules from adguard filters!`);
@@ -169,7 +178,8 @@ const threads = isCI ? cpuCount : cpuCount / 2;
previousSize = domainSets.size;
// Dedupe domainSets
console.log(`Start deduping! (${previousSize})`);
console.log(`Start deduping from black keywords/suffixes! (${previousSize})`);
console.time(`* Dedupe from black keywords/suffixes`);
const toBeRemoved = new Set();
for (const domain of domainSets) {
@@ -209,13 +219,15 @@ const threads = isCI ? cpuCount : cpuCount / 2;
domainSets.delete(removed)
});
// Dedupe domainSets
console.log(`Deduped ${previousSize - domainSets.size} from black keywords and suffixes!`);
console.timeEnd(`* Dedupe from black keywords/suffixes`);
console.log(`Deduped ${previousSize} - ${domainSets.size} = ${previousSize - domainSets.size} from black keywords and suffixes!`);
previousSize = domainSets.size;
// Dedupe domainSets
console.log(`Start deduping! (${previousSize})`);
const START_TIME = Date.now();
const piscina = new Piscina({
filename: pathResolve(__dirname, 'worker/build-reject-domainset-worker.js'),
workerData: [...domainSets]
@@ -253,10 +265,13 @@ const threads = isCI ? cpuCount : cpuCount / 2;
})
});
console.log(`* Dedupe from covered subdomain - ${(Date.now() - START_TIME) / 1000}s`);
console.log(`Deduped ${previousSize - domainSets.size} rules!`);
return fsPromises.writeFile(
await fsPromises.writeFile(
pathResolve(__dirname, '../List/domainset/reject.conf'),
`${[...domainSets].join('\n')}\n`,
{ encoding: 'utf-8' });
console.timeEnd('Total Time - build-reject-domain-set');
})();