GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2025-12-12 17:20:35 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Refactor: minor simplifications

Browse Source
This commit is contained in:
SukkaW 2024-05-27 00:20:49 +08:00
parent e4429a62ee
commit 8b1eeb1c14
4 changed files with 45 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
Build/build-reject-domainset.ts
View File

@ -18,7 +18,7 @@ import * as tldts from 'tldts-experimental';
import { SHARED_DESCRIPTION } from './lib/constants'; import { SHARED_DESCRIPTION } from './lib/constants';
import { getPhishingDomains } from './lib/get-phishing-domains'; import { getPhishingDomains } from './lib/get-phishing-domains';
import { add as SetAdd, subtract as SetSubstract } from 'mnemonist/set'; import { subtract as SetSubstract } from 'mnemonist/set';
import { setAddFromArray, setAddFromArrayCurried } from './lib/set-add-from-array'; import { setAddFromArray, setAddFromArrayCurried } from './lib/set-add-from-array';
import { sort } from './lib/timsort'; import { sort } from './lib/timsort';
@ -27,6 +27,7 @@ export const buildRejectDomainSet = task(import.meta.path, async (span) => {
const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST); const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST);
const domainSets = new Set<string>(); const domainSets = new Set<string>();
const appendArrayToDomainSets = setAddFromArrayCurried(domainSets);
// Parse from AdGuard Filters // Parse from AdGuard Filters
const shouldStop = await span const shouldStop = await span
@ -36,33 +37,35 @@ export const buildRejectDomainSet = task(import.meta.path, async (span) => {
let shouldStop = false; let shouldStop = false;
await Promise.all([ await Promise.all([
// Parse from remote hosts & domain lists // Parse from remote hosts & domain lists
...HOSTS.map(entry => processHosts(childSpan, ...entry).then(setAddFromArrayCurried(domainSets))), ...HOSTS.map(entry => processHosts(childSpan, ...entry).then(appendArrayToDomainSets)),
...DOMAIN_LISTS.map(entry => processDomainLists(childSpan, ...entry).then(setAddFromArrayCurried(domainSets))), ...DOMAIN_LISTS.map(entry => processDomainLists(childSpan, ...entry).then(appendArrayToDomainSets)),
...ADGUARD_FILTERS.map(input => ( ...ADGUARD_FILTERS.map(
typeof input === 'string' input => processFilterRules(childSpan, ...input)
? processFilterRules(childSpan, input) .then(({ white, black, foundDebugDomain }) => {
: processFilterRules(childSpan, ...input) if (foundDebugDomain) {
).then(({ white, black, foundDebugDomain }) => { // eslint-disable-next-line sukka/no-single-return -- not single return
if (foundDebugDomain) { shouldStop = true;
// eslint-disable-next-line sukka/no-single-return -- not single return // we should not break here, as we want to see full matches from all data source
shouldStop = true; }
// we should not break here, as we want to see full matches from all data source setAddFromArray(filterRuleWhitelistDomainSets, white);
} setAddFromArray(domainSets, black);
setAddFromArray(filterRuleWhitelistDomainSets, white); })
setAddFromArray(domainSets, black); ),
})),
...([ ...([
'https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt', 'https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt',
'https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt' 'https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt'
].map(input => processFilterRules(childSpan, input).then(({ white, black }) => { ].map(
setAddFromArray(filterRuleWhitelistDomainSets, white); input => processFilterRules(childSpan, input)
setAddFromArray(filterRuleWhitelistDomainSets, black); .then(({ white, black }) => {
}))), setAddFromArray(filterRuleWhitelistDomainSets, white);
getPhishingDomains(childSpan).then(setAddFromArrayCurried(domainSets)), setAddFromArray(filterRuleWhitelistDomainSets, black);
})
)),
getPhishingDomains(childSpan).then(appendArrayToDomainSets),
childSpan.traceChildAsync('process reject_sukka.conf', () => readFileIntoProcessedArray(path.resolve(import.meta.dir, '../Source/domainset/reject_sukka.conf')) childSpan.traceChildAsync('process reject_sukka.conf', () => readFileIntoProcessedArray(path.resolve(import.meta.dir, '../Source/domainset/reject_sukka.conf'))
.then(setAddFromArrayCurried(domainSets))) .then(appendArrayToDomainSets))
]); ]);
// eslint-disable-next-line sukka/no-single-return -- not single return // eslint-disable-next-line sukka/no-single-return -- not single return
return shouldStop; return shouldStop;

28
Build/lib/fetch-retry.ts
View File

@ -42,28 +42,30 @@ interface FetchRetryOpt {
retries?: number, retries?: number,
factor?: number, factor?: number,
maxRetryAfter?: number, maxRetryAfter?: number,
retry?: number, // onRetry?: (err: Error) => void,
onRetry?: (err: Error) => void, retryOnAborted?: boolean,
retryOnAborted?: boolean retryOnNon2xx?: boolean
} }
interface FetchWithRetry { interface FetchWithRetry {
(url: string | URL | Request, opts?: RequestInit & { retry?: FetchRetryOpt }): Promise<Response> (url: string | URL | Request, opts?: RequestInit & { retry?: FetchRetryOpt }): Promise<Response>
} }
const DEFAULT_OPT: Required<FetchRetryOpt> = {
// timeouts will be [10, 60, 360, 2160, 12960]
// (before randomization is added)
minTimeout: MIN_TIMEOUT,
retries: MAX_RETRIES,
factor: FACTOR,
maxRetryAfter: MAX_RETRY_AFTER,
retryOnAborted: false,
retryOnNon2xx: true
};
function createFetchRetry($fetch: typeof fetch): FetchWithRetry { function createFetchRetry($fetch: typeof fetch): FetchWithRetry {
const fetchRetry: FetchWithRetry = async (url, opts = {}) => { const fetchRetry: FetchWithRetry = async (url, opts = {}) => {
const retryOpts = Object.assign( const retryOpts = Object.assign(
{ DEFAULT_OPT,
// timeouts will be [10, 60, 360, 2160, 12960]
// (before randomization is added)
minTimeout: MIN_TIMEOUT,
retries: MAX_RETRIES,
factor: FACTOR,
maxRetryAfter: MAX_RETRY_AFTER,
retryOnAborted: false,
retryOnNon2xx: true
},
opts.retry opts.retry
); );

10
Build/lib/get-phishing-domains.ts
View File

@ -131,13 +131,11 @@ export const getPhishingDomains = (parentSpan: Span) => parentSpan.traceChild('g
} }
}); });
span.traceChildSync('get final phishing results', () => { for (const domain in domainCountMap) {
for (const domain in domainCountMap) { if (domainCountMap[domain] >= 8) {
if (domainCountMap[domain] >= 8) { domainArr.push(`.${domain}`);
domainArr.push(`.${domain}`);
}
} }
}); }
return domainArr; return domainArr;
}); });

2
Build/lib/reject-data-source.ts
View File

@ -78,7 +78,7 @@ export const PHISHING_DOMAIN_LISTS: [HostsSource, HostsSource] = [
] ]
] as const; ] as const;
type AdGuardFilterSource = string | [main: string, mirrors: string[] | null, ttl: number]; type AdGuardFilterSource = [main: string, mirrors: string[] | null, ttl: number];
export const ADGUARD_FILTERS: AdGuardFilterSource[] = [ export const ADGUARD_FILTERS: AdGuardFilterSource[] = [
// EasyList // EasyList