GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2026-01-29 01:51:52 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Chore: better logging

Browse Source
This commit is contained in:
SukkaW
2022-09-22 22:57:00 +08:00
parent d5f0f2428f
commit 8a7f32e2cd
9 changed files with 47 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
Build/build-anti-bogus-domain.js
View File

@@ -4,6 +4,8 @@ const path = require('path');
const { isIP } = require('net');
(async () => {
console.time('Total Time - build-anti-bogus-domain');
console.time('* Download bogus-nxdomain-list')
const res = (await (await fetchWithRetry('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/bogus-nxdomain.china.conf')).text())
.split('\n')
.map(line => {
@@ -14,6 +16,7 @@ const { isIP } = require('net');
return null
})
.filter(ip => typeof ip === 'string' && isIP(ip) !== 0);
console.timeEnd('* Download bogus-nxdomain-list')
const filePath = path.resolve(__dirname, '../List/ip/reject.conf');
const content = (await fs.promises.readFile(filePath, 'utf-8'))
@@ -23,4 +26,5 @@ const { isIP } = require('net');
);
await fs.promises.writeFile(filePath, content, 'utf-8');
console.timeEnd('Total Time - build-anti-bogus-domain');
})();

4
Build/build-apple-cdn.js
View File

@@ -5,6 +5,8 @@ const path = require('path');
const rDomain = /^(((?!\-))(xn\-\-)?[a-z0-9\-_]{0,61}[a-z0-9]{1,1}\.)*(xn\-\-)?([a-z0-9\-]{1,61}|[a-z0-9\-]{1,30})\.[a-z]{2,}$/m;
(async () => {
console.time('Total Time - build-apple-cdn-conf');
const res = (await (await fetchWithRetry('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf')).text())
.split('\n')
.map(line => {
@@ -28,4 +30,6 @@ const rDomain = /^(((?!\-))(xn\-\-)?[a-z0-9\-_]{0,61}[a-z0-9]{1,1}\.)*(xn\-\-)?(
'utf-8'
)
])
console.timeEnd('Total Time - build-apple-cdn-conf');
})();

4
Build/build-cdn-conf.js
View File

@@ -3,6 +3,8 @@ const fs = require('fs');
const path = require('path');
(async () => {
console.time('Total Time - build-cdn-conf');
const domains = (await (await fetchWithRetry('https://publicsuffix.org/list/public_suffix_list.dat')).text()).split('\n');
const S3OSSDomains = domains.filter(line => {
@@ -29,4 +31,6 @@ const path = require('path');
);
await fs.promises.writeFile(filePath, content, 'utf-8');
console.timeEnd('Total Time - build-cdn-conf');
})();

6
Build/build-cidr.js
View File

@@ -3,6 +3,8 @@ const { promises: fsPromises } = require('fs');
const { resolve: pathResolve } = require('path');
(async () => {
console.time('Total Time - build-chnroutes-cidr');
const cidr = (await (await fetchWithRetry('https://raw.githubusercontent.com/misakaio/chnroutes2/master/chnroutes.txt')).text()).split('\n');
const filteredCidr = cidr.filter(line => {
@@ -13,7 +15,9 @@ const { resolve: pathResolve } = require('path');
return false;
})
return fsPromises.writeFile(pathResolve(__dirname, '../List/ip/china_ip.conf'), makeCidrList(filteredCidr), { encoding: 'utf-8' });
await fsPromises.writeFile(pathResolve(__dirname, '../List/ip/china_ip.conf'), makeCidrList(filteredCidr), { encoding: 'utf-8' });
console.timeEnd('Total Time - build-chnroutes-cidr');
})();
function makeCidrList(cidr) {

23
Build/build-reject-domainset.js
View File

@@ -7,9 +7,13 @@ const { isCI } = require('ci-info');
const threads = isCI ? cpuCount : cpuCount / 2;
(async () => {
console.time('Total Time - build-reject-domain-set');
/** @type Set<string> */
const domainSets = new Set();
console.time('* Download and process Hosts');
// Parse from remote hosts & domain lists
(await Promise.all([
processHosts('https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=0&mimetype=plaintext', true),
@@ -23,6 +27,8 @@ const threads = isCI ? cpuCount : cpuCount / 2;
});
});
console.timeEnd('* Download and process Hosts');
let previousSize = domainSets.size;
console.log(`Import ${previousSize} rules from hosts files!`);
@@ -83,6 +89,7 @@ const threads = isCI ? cpuCount : cpuCount / 2;
'storage.yandexcloud.net' // phishing list
]);
console.time('* Download and process AdBlock Filter Rules');
(await Promise.all([
// Easy List
[
@@ -147,6 +154,8 @@ const threads = isCI ? cpuCount : cpuCount / 2;
black.forEach(i => domainSets.add(i));
});
console.timeEnd('* Download and process AdBlock Filter Rules');
previousSize = domainSets.size - previousSize;
console.log(`Import ${previousSize} rules from adguard filters!`);
@@ -169,7 +178,8 @@ const threads = isCI ? cpuCount : cpuCount / 2;
previousSize = domainSets.size;
// Dedupe domainSets
console.log(`Start deduping! (${previousSize})`);
console.log(`Start deduping from black keywords/suffixes! (${previousSize})`);
console.time(`* Dedupe from black keywords/suffixes`);
const toBeRemoved = new Set();
for (const domain of domainSets) {
@@ -209,13 +219,15 @@ const threads = isCI ? cpuCount : cpuCount / 2;
domainSets.delete(removed)
});
// Dedupe domainSets
console.log(`Deduped ${previousSize - domainSets.size} from black keywords and suffixes!`);
console.timeEnd(`* Dedupe from black keywords/suffixes`);
console.log(`Deduped ${previousSize} - ${domainSets.size} = ${previousSize - domainSets.size} from black keywords and suffixes!`);
previousSize = domainSets.size;
// Dedupe domainSets
console.log(`Start deduping! (${previousSize})`);
const START_TIME = Date.now();
const piscina = new Piscina({
filename: pathResolve(__dirname, 'worker/build-reject-domainset-worker.js'),
workerData: [...domainSets]
@@ -253,10 +265,13 @@ const threads = isCI ? cpuCount : cpuCount / 2;
})
});
console.log(`* Dedupe from covered subdomain - ${(Date.now() - START_TIME) / 1000}s`);
console.log(`Deduped ${previousSize - domainSets.size} rules!`);
return fsPromises.writeFile(
await fsPromises.writeFile(
pathResolve(__dirname, '../List/domainset/reject.conf'),
`${[...domainSets].join('\n')}\n`,
{ encoding: 'utf-8' });
console.timeEnd('Total Time - build-reject-domain-set');
})();

4
Build/build-telegram-cidr.js
View File

@@ -4,6 +4,8 @@ const path = require('path');
const { isIPv4, isIPv6 } = require('net');
(async () => {
console.time('Total Time - build-telegram-cidr');
const resp = await fetchWithRetry('https://core.telegram.org/resources/cidr.txt');
const lastModified = new Date(resp.headers.get('last-modified'));
@@ -27,4 +29,6 @@ const { isIPv4, isIPv6 } = require('net');
}).join('\n') + '\n',
'utf-8'
);
console.timeEnd('Total Time - build-telegram-cidr');
})();

4
Build/lib/parse-filter.js
View File

@@ -96,7 +96,7 @@ async function processHosts (hostsUrl, includeAllSubDomain = false) {
* @returns {Promise<{ white: Set<string>, black: Set<string> }>}
*/
async function processFilterRules (filterRulesUrl, fallbackUrls) {
console.time(`processFilterRules: ${filterRulesUrl}`);
console.time(` - processFilterRules: ${filterRulesUrl}`);
if (typeof filterRulesUrl === 'string') {
filterRulesUrl = new URL(filterRulesUrl);
@@ -206,7 +206,7 @@ async function processFilterRules (filterRulesUrl, fallbackUrls) {
}
});
console.timeEnd(`processFilterRules: ${filterRulesUrl}`);
console.timeEnd(` - processFilterRules: ${filterRulesUrl}`);
return {
white: whitelistDomainSets,