GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2026-01-29 01:51:52 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Update Reject Infra & Data Source

Browse Source
This commit is contained in:
SukkaW
2024-09-30 08:05:47 +08:00
parent c158217dda
commit 0d1fddcb81
5 changed files with 98 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
Build/lib/get-phishing-domains.ts
View File

@@ -1,10 +1,10 @@
import { processDomainLists } from './parse-filter';
import { processDomainLists, processHosts } from './parse-filter';
import * as tldts from 'tldts-experimental';
import { dummySpan } from '../trace';
import type { Span } from '../trace';
import { appendArrayInPlaceCurried } from './append-array-in-place';
import { PHISHING_DOMAIN_LISTS_EXTRA } from '../constants/reject-data-source';
import { PHISHING_DOMAIN_LISTS_EXTRA, PHISHING_HOSTS_EXTRA } from '../constants/reject-data-source';
import { loosTldOptWithPrivateDomains } from '../constants/loose-tldts-opt';
import picocolors from 'picocolors';
import createKeywordFilter from './aho-corasick';
@@ -162,6 +162,8 @@ export const getPhishingDomains = (parentSpan: Span) => parentSpan.traceChild('g
(await Promise.all(PHISHING_DOMAIN_LISTS_EXTRA.map(entry => processDomainLists(curSpan, ...entry, cacheKey))))
.forEach(appendArrayInPlaceCurried(domainArr));
(await Promise.all(PHISHING_HOSTS_EXTRA.map(entry => processHosts(curSpan, ...entry, cacheKey))))
.forEach(appendArrayInPlaceCurried(domainArr));
return domainArr;
});

14
Build/lib/parse-filter.ts
View File

@@ -45,7 +45,11 @@ const domainListLineCb = (l: string, set: string[], includeAllSubDomain: boolean
const cacheKey = createCacheKey(__filename);
export function processDomainLists(span: Span, domainListsUrl: string, mirrors: string[] | null, includeAllSubDomain = false, ttl: number | null = null, extraCacheKey: (input: string) => string = identity) {
export function processDomainLists(
span: Span,
domainListsUrl: string, mirrors: string[] | null, includeAllSubDomain = false,
ttl: number | null = null, extraCacheKey: (input: string) => string = identity
) {
return span.traceChild(`process domainlist: ${domainListsUrl}`).traceAsyncFn((childSpan) => fsFetchCache.apply(
extraCacheKey(cacheKey(domainListsUrl)),
async () => {
@@ -100,9 +104,13 @@ const hostsLineCb = (l: string, set: string[], includeAllSubDomain: boolean, met
set.push(includeAllSubDomain ? `.${domain}` : domain);
};
export function processHosts(span: Span, hostsUrl: string, mirrors: string[] | null, includeAllSubDomain = false, ttl: number | null = null) {
export function processHosts(
span: Span,
hostsUrl: string, mirrors: string[] | null, includeAllSubDomain = false,
ttl: number | null = null, extraCacheKey: (input: string) => string = identity
) {
return span.traceChild(`processhosts: ${hostsUrl}`).traceAsyncFn((childSpan) => fsFetchCache.apply(
cacheKey(hostsUrl),
extraCacheKey(cacheKey(hostsUrl)),
async () => {
const domainSets: string[] = [];

2
Build/lib/rules/ruleset.ts
View File

@@ -214,8 +214,8 @@ export class RulesetOutput extends RuleOutput<Preprocessed> {
}
}
console.error(picocolors.bold('Parsed Failed'));
if (parsedFailures.length > 0) {
console.error(picocolors.bold('Parsed Failed'));
console.table(parsedFailures);
}