GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2026-01-29 01:51:52 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Add new phishing feed / speed up domains sort

Browse Source
This commit is contained in:
SukkaW
2023-12-12 17:10:55 +08:00
parent e56f601fbc
commit e970006445
8 changed files with 72 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
Build/lib/get-phishing-domains.ts
View File

@@ -1,7 +1,7 @@
import fsp from 'fs/promises';
import path from 'path';
import { getGorhillPublicSuffixPromise } from './get-gorhill-publicsuffix';
import { processHosts } from './parse-filter';
import { processDomainLists, processHosts } from './parse-filter';
import { traceAsync, traceSync } from './trace-runner';
import * as tldts from 'tldts';
import { createTrie } from './trie';
@@ -33,7 +33,12 @@ const BLACK_TLD = new Set([
'club',
'cn',
'codes',
'co.uk',
'co.in',
'com.br',
'com.cn',
'com.pl',
'com.vn',
'cool',
'cyou',
'fit',
@@ -53,6 +58,7 @@ const BLACK_TLD = new Set([
'ltd',
'ml',
'mobi',
'net.pl',
'one',
'online',
'pro',
@@ -79,19 +85,12 @@ const BLACK_TLD = new Set([
]);
export const getPhishingDomains = () => traceAsync('get phishing domains', async () => {
const [domainSet, gorhill] = await Promise.all([
const [domainSet, domainSet2, gorhill] = await Promise.all([
processHosts('https://curbengh.github.io/phishing-filter/phishing-filter-hosts.txt', true, true),
// processDomainLists('https://phishing.army/download/phishing_army_blocklist.txt', true),
// processFilterRules(
// 'https://curbengh.github.io/phishing-filter/phishing-filter-agh.txt',
// [
// 'https://phishing-filter.pages.dev/phishing-filter-agh.txt'
// // Prefer mirror, since malware-filter.gitlab.io has not been updated for a while
// // 'https://malware-filter.gitlab.io/malware-filter/phishing-filter-agh.txt'
// ]
// ),
processDomainLists('https://phishing.army/download/phishing_army_blocklist.txt', true),
getGorhillPublicSuffixPromise()
]);
domainSet2.forEach((domain) => domainSet.add(domain));
traceSync.skip('* whitelisting phishing domains', () => {
const trieForRemovingWhiteListed = createTrie(domainSet);

32
Build/lib/parse-filter.ts
View File

@@ -37,27 +37,27 @@ const normalizeDomain = (domain: string) => {
return h[0] === '.' ? h.slice(1) : h;
};
export async function processDomainLists(domainListsUrl: string, includeAllSubDomain = false) {
const domainSets = new Set<string>();
export function processDomainLists(domainListsUrl: string, includeAllSubDomain = false) {
return traceAsync(`- processDomainLists: ${domainListsUrl}`, async () => {
const domainSets = new Set<string>();
for await (const line of await fetchRemoteTextAndReadByLine(domainListsUrl)) {
const domainToAdd = processLine(line);
if (!domainToAdd) {
continue;
for await (const line of await fetchRemoteTextAndReadByLine(domainListsUrl)) {
const domainToAdd = processLine(line);
if (!domainToAdd) continue;
if (DEBUG_DOMAIN_TO_FIND && domainToAdd.includes(DEBUG_DOMAIN_TO_FIND)) {
warnOnce(domainListsUrl, false, DEBUG_DOMAIN_TO_FIND);
foundDebugDomain = true;
}
domainSets.add(includeAllSubDomain ? `.${domainToAdd}` : domainToAdd);
}
if (DEBUG_DOMAIN_TO_FIND && domainToAdd.includes(DEBUG_DOMAIN_TO_FIND)) {
warnOnce(domainListsUrl, false, DEBUG_DOMAIN_TO_FIND);
foundDebugDomain = true;
}
domainSets.add(includeAllSubDomain ? `.${domainToAdd}` : domainToAdd);
}
return domainSets;
return domainSets;
});
}
export async function processHosts(hostsUrl: string, includeAllSubDomain = false, skipDomainCheck = false) {
export function processHosts(hostsUrl: string, includeAllSubDomain = false, skipDomainCheck = false) {
return traceAsync(`- processHosts: ${hostsUrl}`, async () => {
const domainSets = new Set<string>();

5
Build/lib/reject-data-source.ts
View File

@@ -26,6 +26,11 @@ export const HOSTS = [
['https://paulgb.github.io/BarbBlock/blacklists/hosts-file.txt', true, true]
] as const;
export const DOMAIN_LISTS = [
// DigitalSide Threat-Intel - OSINT Hub
['https://osint.digitalside.it/Threat-Intel/lists/latestdomains.txt', true]
] as const;
export const ADGUARD_FILTERS = [
// EasyList
[

13
Build/lib/stable-sort-domain.test.ts
View File

@@ -1,13 +0,0 @@
import domainSorter from './stable-sort-domain';
// eslint-disable-next-line import/no-unresolved -- fuck eslint-import
import { describe, it, expect } from 'bun:test';
describe('stable-sort-domain', () => {
it('.ks.cn, .tag.unclaimedproperty.ks.gov', () => {
expect(domainSorter('.ks.cn', '.tag.unclaimedproperty.ks.gov')).toBe(-1);
});
it('.fgnzdb.xyz, .hub.fghtem.com', () => {
expect(domainSorter('.fgnzdb.xyz', '.hub.fghtem.com')).toBe(1);
});
});

55
Build/lib/stable-sort-domain.ts
View File

@@ -10,18 +10,16 @@ const compare = (a: string | null, b: string | null) => {
return -1;
}
if (a.length !== b.length) {
const r = a.length - b.length;
if (r > 0) {
return 1;
}
if (r < 0) {
return -1;
}
return 0;
const aLen = a.length;
const r = aLen - b.length;
if (r > 0) {
return 1;
}
if (r < 0) {
return -1;
}
for (let i = 0; i < a.length; i++) {
for (let i = 0; i < aLen; i++) {
if (b[i] == null) {
return 1;
}
@@ -35,34 +33,21 @@ const compare = (a: string | null, b: string | null) => {
return 0;
};
const createDomainSorter = (gorhill: PublicSuffixList | null = null) => {
if (gorhill) {
const getDomain = createCachedGorhillGetDomain(gorhill);
export const sortDomains = (inputs: string[], gorhill: PublicSuffixList) => {
const getDomain = createCachedGorhillGetDomain(gorhill);
const domains = inputs.reduce<Record<string, string>>((acc, cur) => {
acc[cur] ||= getDomain(cur);
return acc;
}, {});
return (a: string, b: string) => {
if (a === b) return 0;
const aDomain = getDomain(a);
const bDomain = getDomain(b);
const resultDomain = compare(aDomain, bDomain);
return resultDomain !== 0 ? resultDomain : compare(a, b);
};
}
// eslint-disable-next-line @typescript-eslint/no-var-requires -- fuck
const tldts = require('./cached-tld-parse');
return (a: string, b: string) => {
const sorter = (a: string, b: string) => {
if (a === b) return 0;
const aDomain = tldts.parse(a).domain;
const bDomain = tldts.parse(b).domain;
const aDomain = domains[a];
const bDomain = domains[b];
const resultDomain = compare(aDomain, bDomain);
return resultDomain !== 0 ? resultDomain : compare(a, b);
return compare(aDomain, bDomain) || compare(a, b);
};
};
export default createDomainSorter();
export { createDomainSorter };
return inputs.sort(sorter);
};