GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2026-02-03 12:31:54 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Perf: further speed up infra

Browse Source
This commit is contained in:
SukkaW
2023-09-14 20:15:49 +08:00
parent adb8b43357
commit 78afa595a9
25 changed files with 431 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
Build/build-phishing-domainset.js
View File

@@ -1,10 +1,10 @@
const { parse } = require('tldts');
const tldts = require('tldts');
const { processFilterRules } = require('./lib/parse-filter.js');
const path = require('path');
const { createRuleset } = require('./lib/create-file');
const { processLine } = require('./lib/process-line.js');
const domainSorter = require('./lib/stable-sort-domain');
const { runner } = require('./lib/trace-runner.js');
const { runner, traceSync } = require('./lib/trace-runner.js');
const WHITELIST_DOMAIN = new Set([
'w3s.link',
@@ -61,19 +61,14 @@ const BLACK_TLD = new Set([
]);
runner(__filename, async () => {
const domainSet = Array.from(
(await processFilterRules('https://curbengh.github.io/phishing-filter/phishing-filter-agh.txt')).black
);
const domainSet = Array.from((await processFilterRules('https://curbengh.github.io/phishing-filter/phishing-filter-agh.txt')).black);
const domainCountMap = {};
for (let i = 0, len = domainSet.length; i < len; i++) {
const line = processLine(domainSet[i]);
if (!line) continue;
const domain = line.charCodeAt(0) === 46 ? line.slice(1) : line;
const parsed = parse(domain, { allowPrivateDomains: true });
const parsed = tldts.parse(line, { allowPrivateDomains: true });
const apexDomain = parsed.domain;
if (apexDomain) {
@@ -84,19 +79,18 @@ runner(__filename, async () => {
domainCountMap[apexDomain] ||= 0;
let isPhishingDomainMockingAmazon = false;
if (domain.startsWith('amaz')) {
if (line.startsWith('.amaz')) {
domainCountMap[apexDomain] += 0.5;
isPhishingDomainMockingAmazon = true;
if (domain.startsWith('amazon-')) {
if (line.startsWith('.amazon-')) {
domainCountMap[apexDomain] += 4.5;
}
} else if (domain.startsWith('customer')) {
} else if (line.startsWith('.customer')) {
domainCountMap[apexDomain] += 0.25;
}
if (domain.includes('-co-jp')) {
if (line.includes('-co-jp')) {
domainCountMap[apexDomain] += (isPhishingDomainMockingAmazon ? 4.5 : 0.5);
}
@@ -105,17 +99,17 @@ runner(__filename, async () => {
domainCountMap[apexDomain] += 1;
if (domain.length > 19) {
if (line.length > 19) {
// Add more weight if the domain is long enough
if (domain.length > 44) {
if (line.length > 44) {
domainCountMap[apexDomain] += 3.5;
} else if (domain.length > 34) {
} else if (line.length > 34) {
domainCountMap[apexDomain] += 2.5;
} else if (domain.length > 29) {
} else if (line.length > 29) {
domainCountMap[apexDomain] += 1.5;
} else if (domain.length > 24) {
} else if (line.length > 24) {
domainCountMap[apexDomain] += 0.75;
} else if (domain.length > 19) {
} else if (line.length > 19) {
domainCountMap[apexDomain] += 0.25;
}
@@ -129,15 +123,14 @@ runner(__filename, async () => {
}
}
const results = [];
Object.entries(domainCountMap).forEach(([domain, count]) => {
if (count >= 5) {
results.push(`.${domain}`);
}
});
results.sort(domainSorter);
const results = traceSync('* get final results', () => Object.entries(domainCountMap)
.reduce((acc, [apexDomain, count]) => {
if (count >= 5) {
acc.push(`.${apexDomain}`);
}
return acc;
}, /** @type {string[]} */([]))
.sort(domainSorter));
const description = [
'License: AGPL 3.0',