GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2026-02-03 20:41:52 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Refactor: simplify build infra

Browse Source
This commit is contained in:
SukkaW
2023-09-14 22:34:25 +08:00
parent 2448cbe39a
commit 573c0f5274
22 changed files with 127 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files

103
Build/build-phishing-domainset.js
View File

@@ -4,7 +4,7 @@ const path = require('path');
const { createRuleset } = require('./lib/create-file');
const { processLine } = require('./lib/process-line.js');
const domainSorter = require('./lib/stable-sort-domain');
const { runner, traceSync, task } = require('./lib/trace-runner.js');
const { traceSync, task } = require('./lib/trace-runner.js');
const WHITELIST_DOMAIN = new Set([
'w3s.link',
@@ -61,67 +61,76 @@ const BLACK_TLD = new Set([
]);
const buildPhishingDomainSet = task(__filename, async () => {
const domainSet = Array.from((await processFilterRules('https://curbengh.github.io/phishing-filter/phishing-filter-agh.txt')).black);
const domainSet = Array.from((await processFilterRules(
'https://phishing-filter.pages.dev/phishing-filter-agh.txt'
// [
// 'https://malware-filter.gitlab.io/phishing-filter/phishing-filter-agh.txt',
// 'https://malware-filter.pages.dev/phishing-filter-agh.txt',
// 'https://phishing-filter.pages.dev/phishing-filter-agh.txt'
// ]
)).black);
const domainCountMap = {};
for (let i = 0, len = domainSet.length; i < len; i++) {
const line = processLine(domainSet[i]);
if (!line) continue;
traceSync('* process domain set', () => {
for (let i = 0, len = domainSet.length; i < len; i++) {
const line = processLine(domainSet[i]);
if (!line) continue;
const parsed = tldts.parse(line, { allowPrivateDomains: true });
const apexDomain = parsed.domain;
const parsed = tldts.parse(line, { allowPrivateDomains: true });
const apexDomain = parsed.domain;
if (apexDomain) {
if (WHITELIST_DOMAIN.has(apexDomain)) {
continue;
}
domainCountMap[apexDomain] ||= 0;
let isPhishingDomainMockingAmazon = false;
if (line.startsWith('.amaz')) {
domainCountMap[apexDomain] += 0.5;
isPhishingDomainMockingAmazon = true;
if (line.startsWith('.amazon-')) {
domainCountMap[apexDomain] += 4.5;
if (apexDomain) {
if (WHITELIST_DOMAIN.has(apexDomain)) {
continue;
}
} else if (line.startsWith('.customer')) {
domainCountMap[apexDomain] += 0.25;
}
if (line.includes('-co-jp')) {
domainCountMap[apexDomain] += (isPhishingDomainMockingAmazon ? 4.5 : 0.5);
}
const tld = parsed.publicSuffix;
if (!tld || !BLACK_TLD.has(tld)) continue;
domainCountMap[apexDomain] ||= 0;
domainCountMap[apexDomain] += 1;
let isPhishingDomainMockingAmazon = false;
if (line.startsWith('.amaz')) {
domainCountMap[apexDomain] += 0.5;
if (line.length > 19) {
// Add more weight if the domain is long enough
if (line.length > 44) {
domainCountMap[apexDomain] += 3.5;
} else if (line.length > 34) {
domainCountMap[apexDomain] += 2.5;
} else if (line.length > 29) {
domainCountMap[apexDomain] += 1.5;
} else if (line.length > 24) {
domainCountMap[apexDomain] += 0.75;
} else if (line.length > 19) {
isPhishingDomainMockingAmazon = true;
if (line.startsWith('.amazon-')) {
domainCountMap[apexDomain] += 4.5;
}
} else if (line.startsWith('.customer')) {
domainCountMap[apexDomain] += 0.25;
}
if (line.includes('-co-jp')) {
domainCountMap[apexDomain] += (isPhishingDomainMockingAmazon ? 4.5 : 0.5);
}
if (domainCountMap[apexDomain] < 5) {
const subdomain = parsed.subdomain;
if (subdomain && subdomain.includes('.')) {
const tld = parsed.publicSuffix;
if (!tld || !BLACK_TLD.has(tld)) continue;
domainCountMap[apexDomain] += 1;
if (line.length > 19) {
// Add more weight if the domain is long enough
if (line.length > 44) {
domainCountMap[apexDomain] += 3.5;
} else if (line.length > 34) {
domainCountMap[apexDomain] += 2.5;
} else if (line.length > 29) {
domainCountMap[apexDomain] += 1.5;
} else if (line.length > 24) {
domainCountMap[apexDomain] += 0.75;
} else if (line.length > 19) {
domainCountMap[apexDomain] += 0.25;
}
if (domainCountMap[apexDomain] < 5) {
const subdomain = parsed.subdomain;
if (subdomain?.includes('.')) {
domainCountMap[apexDomain] += 1.5;
}
}
}
}
}
}
});
const results = traceSync('* get final results', () => Object.entries(domainCountMap)
.reduce((acc, [apexDomain, count]) => {
@@ -156,5 +165,5 @@ const buildPhishingDomainSet = task(__filename, async () => {
module.exports.buildPhishingDomainSet = buildPhishingDomainSet;
if (require.main === module) {
runner(__filename, buildPhishingDomainSet);
buildPhishingDomainSet();
}