GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2025-12-12 01:00:34 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Fix: enable domain check for some reject data source

Browse Source
This commit is contained in:
SukkaW 2024-01-07 01:05:20 +08:00
parent 61b88c5807
commit ca169b9db5
6 changed files with 32 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Build/build-reject-domainset.ts
View File

@ -37,7 +37,7 @@ export const buildRejectDomainSet = task(import.meta.path, async () => {
domainSets.add(host);
});
})),
...DOMAIN_LISTS.map(entry => processDomainLists(entry[0], entry[1], entry[2])),
...DOMAIN_LISTS.map(entry => processDomainLists(entry[0], entry[1], entry[2], entry[3])),
...ADGUARD_FILTERS.map(input => {
const promise = typeof input === 'string'
? processFilterRules(input)

3
Build/lib/cached-tld-parse.ts
View File

@ -3,6 +3,7 @@ import { createCache } from './cache-apply';
import type { PublicSuffixList } from '@gorhill/publicsuffixlist';
const cache = createCache('cached-tld-parse', true);
const cache2 = createCache('cached-tld-parse2', true);
const sharedConfig = { allowPrivateDomains: true };
const sharedConfig2 = { allowPrivateDomains: true, detectIp: false };
@ -10,7 +11,7 @@ const sharedConfig2 = { allowPrivateDomains: true, detectIp: false };
/** { allowPrivateDomains: true } */
export const parse = (domain: string) => cache.sync(domain, () => tldts.parse(domain, sharedConfig));
/** { allowPrivateDomains: true, detectIp: false } */
export const parse2 = (domain: string) => cache.sync(domain, () => tldts.parse(domain, sharedConfig2));
export const parse2 = (domain: string) => cache2.sync(domain, () => tldts.parse(domain, sharedConfig2));
let gothillGetDomainCache: ReturnType<typeof createCache> | null = null;
export const createCachedGorhillGetDomain = (gorhill: PublicSuffixList) => {

4
Build/lib/get-phishing-domains.ts
View File

@ -85,8 +85,8 @@ const BLACK_TLD = new Set([
export const getPhishingDomains = () => traceAsync('get phishing domains', async () => {
const [domainSet, domainSet2, gorhill] = await Promise.all([
processHosts('https://curbengh.github.io/phishing-filter/phishing-filter-hosts.txt', true, true, TTL.THREE_HOURS()),
processDomainLists('https://phishing.army/download/phishing_army_blocklist.txt', true, TTL.THREE_HOURS()),
processDomainLists('https://curbengh.github.io/phishing-filter/phishing-filter-domains.txt', true, false, TTL.THREE_HOURS()),
processDomainLists('https://phishing.army/download/phishing_army_blocklist.txt', true, true, TTL.THREE_HOURS()),
getGorhillPublicSuffixPromise()
]);
domainSet2.forEach((domain) => domainSet.add(domain));

11
Build/lib/normalize-domain.ts
View File

@ -5,11 +5,14 @@ export const normalizeDomain = (domain: string) => {
if (isProbablyIpv4(domain)) return null;
const parsed = tldts.parse2(domain);
if (parsed.isIp) return null;
// if (parsed.isIp) return null;
if (!parsed.hostname) return null;
if (!parsed.isIcann && !parsed.isPrivate) return null;
const h = parsed.hostname;
if (!h) return null;
let h = parsed.hostname;
if (h[0] === '.') h = h.slice(1);
if (h.endsWith('.')) h = h.slice(0, -1);
return h[0] === '.' ? h.slice(1) : h;
if (h) return h;
return null;
};

15
Build/lib/parse-filter.ts
View File

@ -11,17 +11,23 @@ import { normalizeDomain } from './normalize-domain';
import { fetchAssets } from './fetch-assets';
import { deserializeSet, fsCache, serializeSet } from './cache-filesystem';
const DEBUG_DOMAIN_TO_FIND: string | null = null; // example.com | null
const DEBUG_DOMAIN_TO_FIND: string | null = '.j3.4z0vc.chileinsumos.cl'; // example.com | null
let foundDebugDomain = false;
export function processDomainLists(domainListsUrl: string, includeAllSubDomain = false, ttl: number | null = null) {
export function processDomainLists(domainListsUrl: string, includeAllSubDomain = false, skipDomainCheck = false, ttl: number | null = null) {
return traceAsync(`- processDomainLists: ${domainListsUrl}`, () => fsCache.apply(
domainListsUrl,
async () => {
const domainSets = new Set<string>();
for await (const line of await fetchRemoteTextByLine(domainListsUrl)) {
const domainToAdd = processLine(line);
let domainToAdd = processLine(line);
if (!domainToAdd) continue;
if (!skipDomainCheck) {
domainToAdd = normalizeDomain(domainToAdd);
}
if (!domainToAdd) continue;
if (DEBUG_DOMAIN_TO_FIND && domainToAdd.includes(DEBUG_DOMAIN_TO_FIND)) {
@ -123,6 +129,9 @@ export async function processFilterRules(
const flag = result[1];
const hostname = result[0];
// if (hostname.endsWith('.')) {
// hostname = hostname.slice(0, -1);
// }
if (DEBUG_DOMAIN_TO_FIND) {
if (hostname.includes(DEBUG_DOMAIN_TO_FIND)) {

16
Build/lib/reject-data-source.ts
View File

@ -31,20 +31,20 @@ export const HOSTS = [
export const DOMAIN_LISTS = [
// CoinBlockerList
// Although the hosts file is still actively maintained, the hosts_browser file is not updated since 2021-07, so we set a 14 days cache ttl
['https://zerodot1.gitlab.io/CoinBlockerLists/list_browser.txt', true, TTL.TWO_WEEKS()],
['https://zerodot1.gitlab.io/CoinBlockerLists/list_browser.txt', true, true, TTL.TWO_WEEKS()],
// BarbBlock
// The barbblock list has never been updated since 2019-05, so we set a 14 days cache ttl
['https://paulgb.github.io/BarbBlock/blacklists/domain-list.txt', true, TTL.TWO_WEEKS()],
['https://paulgb.github.io/BarbBlock/blacklists/domain-list.txt', true, true, TTL.TWO_WEEKS()],
// DigitalSide Threat-Intel - OSINT Hub
// Update once per day
['https://osint.digitalside.it/Threat-Intel/lists/latestdomains.txt', true, TTL.ONE_DAY()],
['https://osint.digitalside.it/Threat-Intel/lists/latestdomains.txt', true, true, TTL.ONE_DAY()],
// AdGuard CNAME Filter Combined
// Update on a 7 days basis, so we add a 3 hours cache ttl
['https://raw.githubusercontent.com/AdguardTeam/cname-trackers/master/data/combined_disguised_ads_justdomains.txt', true, TTL.THREE_DAYS()],
['https://raw.githubusercontent.com/AdguardTeam/cname-trackers/master/data/combined_disguised_trackers_justdomains.txt', true, TTL.THREE_DAYS()],
['https://raw.githubusercontent.com/AdguardTeam/cname-trackers/master/data/combined_disguised_clickthroughs_justdomains.txt', true, TTL.THREE_DAYS()],
['https://raw.githubusercontent.com/AdguardTeam/cname-trackers/master/data/combined_disguised_microsites_justdomains.txt', true, TTL.THREE_DAYS()],
['https://raw.githubusercontent.com/AdguardTeam/cname-trackers/master/data/combined_disguised_mail_trackers_justdomains.txt', true, TTL.THREE_DAYS()]
['https://raw.githubusercontent.com/AdguardTeam/cname-trackers/master/data/combined_disguised_ads_justdomains.txt', true, true, TTL.THREE_DAYS()],
['https://raw.githubusercontent.com/AdguardTeam/cname-trackers/master/data/combined_disguised_trackers_justdomains.txt', true, true, TTL.THREE_DAYS()],
['https://raw.githubusercontent.com/AdguardTeam/cname-trackers/master/data/combined_disguised_clickthroughs_justdomains.txt', true, true, TTL.THREE_DAYS()],
['https://raw.githubusercontent.com/AdguardTeam/cname-trackers/master/data/combined_disguised_microsites_justdomains.txt', true, true, TTL.THREE_DAYS()],
['https://raw.githubusercontent.com/AdguardTeam/cname-trackers/master/data/combined_disguised_mail_trackers_justdomains.txt', true, true, TTL.THREE_DAYS()]
] as const;
export const ADGUARD_FILTERS = [