GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2025-12-13 01:30:37 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Support extra cacheKey

Browse Source
This commit is contained in:
SukkaW 2024-09-02 20:46:10 +08:00
parent 4245fac2e1
commit 27c4004c0d
3 changed files with 9 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Build/lib/cache-filesystem.ts
View File

@ -217,5 +217,5 @@ export const deserializeArray = (str: string) => str.split(separator);
export const createCacheKey = (filename: string) => { export const createCacheKey = (filename: string) => {
const fileHash = stringHash(fs.readFileSync(filename, 'utf-8')); const fileHash = stringHash(fs.readFileSync(filename, 'utf-8'));
return (key: string) => key + '$' + fileHash; return (key: string) => key + '$' + fileHash + '$';
}; };

5
Build/lib/get-phishing-domains.ts
View File

@ -7,6 +7,7 @@ import { PHISHING_DOMAIN_LISTS_EXTRA } from '../constants/reject-data-source';
import { looseTldtsOpt } from '../constants/loose-tldts-opt'; import { looseTldtsOpt } from '../constants/loose-tldts-opt';
import picocolors from 'picocolors'; import picocolors from 'picocolors';
import createKeywordFilter from './aho-corasick'; import createKeywordFilter from './aho-corasick';
import { createCacheKey } from './cache-filesystem';
const BLACK_TLD = new Set([ const BLACK_TLD = new Set([
'accountant', 'accountant',
@ -130,11 +131,13 @@ const lowKeywords = createKeywordFilter([
'.www-' '.www-'
]); ]);
const cacheKey = createCacheKey(__filename);
export const getPhishingDomains = (parentSpan: Span) => parentSpan.traceChild('get phishing domains').traceAsyncFn(async (span) => { export const getPhishingDomains = (parentSpan: Span) => parentSpan.traceChild('get phishing domains').traceAsyncFn(async (span) => {
const domainArr = await span.traceChildAsync('download/parse/merge phishing domains', async (curSpan) => { const domainArr = await span.traceChildAsync('download/parse/merge phishing domains', async (curSpan) => {
const domainArr: string[] = []; const domainArr: string[] = [];
(await Promise.all(PHISHING_DOMAIN_LISTS_EXTRA.map(entry => processDomainLists(curSpan, ...entry)))) (await Promise.all(PHISHING_DOMAIN_LISTS_EXTRA.map(entry => processDomainLists(curSpan, ...entry, cacheKey))))
.forEach(appendArrayInPlaceCurried(domainArr)); .forEach(appendArrayInPlaceCurried(domainArr));
return domainArr; return domainArr;

6
Build/lib/parse-filter.ts
View File

@ -16,6 +16,8 @@ const DEBUG_DOMAIN_TO_FIND: string | null = null; // example.com | null
let foundDebugDomain = false; let foundDebugDomain = false;
const temporaryBypass = typeof DEBUG_DOMAIN_TO_FIND === 'string'; const temporaryBypass = typeof DEBUG_DOMAIN_TO_FIND === 'string';
const identity = <T>(x: T) => x;
const domainListLineCb = (l: string, set: string[], includeAllSubDomain: boolean, meta: string) => { const domainListLineCb = (l: string, set: string[], includeAllSubDomain: boolean, meta: string) => {
let line = processLine(l); let line = processLine(l);
if (!line) return; if (!line) return;
@ -44,9 +46,9 @@ const domainListLineCb = (l: string, set: string[], includeAllSubDomain: boolean
const cacheKey = createCacheKey(__filename); const cacheKey = createCacheKey(__filename);
export function processDomainLists(span: Span, domainListsUrl: string, mirrors: string[] | null, includeAllSubDomain = false, ttl: number | null = null) { export function processDomainLists(span: Span, domainListsUrl: string, mirrors: string[] | null, includeAllSubDomain = false, ttl: number | null = null, extraCacheKey: (input: string) => string = identity) {
return span.traceChild(`process domainlist: ${domainListsUrl}`).traceAsyncFn((childSpan) => fsFetchCache.apply( return span.traceChild(`process domainlist: ${domainListsUrl}`).traceAsyncFn((childSpan) => fsFetchCache.apply(
cacheKey(domainListsUrl), extraCacheKey(cacheKey(domainListsUrl)),
async () => { async () => {
const domainSets: string[] = []; const domainSets: string[] = [];