GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2025-12-12 01:00:34 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Refactor: minor simplifications

Browse Source
This commit is contained in:
SukkaW 2024-05-27 00:20:49 +08:00
parent e4429a62ee
commit 8b1eeb1c14
4 changed files with 45 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
Build/build-reject-domainset.ts
View File

@ -18,7 +18,7 @@ import * as tldts from 'tldts-experimental';
import { SHARED_DESCRIPTION } from './lib/constants';
import { getPhishingDomains } from './lib/get-phishing-domains';
import { add as SetAdd, subtract as SetSubstract } from 'mnemonist/set';
import { subtract as SetSubstract } from 'mnemonist/set';
import { setAddFromArray, setAddFromArrayCurried } from './lib/set-add-from-array';
import { sort } from './lib/timsort';
@ -27,6 +27,7 @@ export const buildRejectDomainSet = task(import.meta.path, async (span) => {
const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST);
const domainSets = new Set<string>();
const appendArrayToDomainSets = setAddFromArrayCurried(domainSets);
// Parse from AdGuard Filters
const shouldStop = await span
@ -36,33 +37,35 @@ export const buildRejectDomainSet = task(import.meta.path, async (span) => {
let shouldStop = false;
await Promise.all([
// Parse from remote hosts & domain lists
...HOSTS.map(entry => processHosts(childSpan, ...entry).then(setAddFromArrayCurried(domainSets))),
...HOSTS.map(entry => processHosts(childSpan, ...entry).then(appendArrayToDomainSets)),
...DOMAIN_LISTS.map(entry => processDomainLists(childSpan, ...entry).then(setAddFromArrayCurried(domainSets))),
...DOMAIN_LISTS.map(entry => processDomainLists(childSpan, ...entry).then(appendArrayToDomainSets)),
...ADGUARD_FILTERS.map(input => (
typeof input === 'string'
? processFilterRules(childSpan, input)
: processFilterRules(childSpan, ...input)
).then(({ white, black, foundDebugDomain }) => {
if (foundDebugDomain) {
// eslint-disable-next-line sukka/no-single-return -- not single return
shouldStop = true;
// we should not break here, as we want to see full matches from all data source
}
setAddFromArray(filterRuleWhitelistDomainSets, white);
setAddFromArray(domainSets, black);
})),
...ADGUARD_FILTERS.map(
input => processFilterRules(childSpan, ...input)
.then(({ white, black, foundDebugDomain }) => {
if (foundDebugDomain) {
// eslint-disable-next-line sukka/no-single-return -- not single return
shouldStop = true;
// we should not break here, as we want to see full matches from all data source
}
setAddFromArray(filterRuleWhitelistDomainSets, white);
setAddFromArray(domainSets, black);
})
),
...([
'https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt',
'https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt'
].map(input => processFilterRules(childSpan, input).then(({ white, black }) => {
setAddFromArray(filterRuleWhitelistDomainSets, white);
setAddFromArray(filterRuleWhitelistDomainSets, black);
}))),
getPhishingDomains(childSpan).then(setAddFromArrayCurried(domainSets)),
].map(
input => processFilterRules(childSpan, input)
.then(({ white, black }) => {
setAddFromArray(filterRuleWhitelistDomainSets, white);
setAddFromArray(filterRuleWhitelistDomainSets, black);
})
)),
getPhishingDomains(childSpan).then(appendArrayToDomainSets),
childSpan.traceChildAsync('process reject_sukka.conf', () => readFileIntoProcessedArray(path.resolve(import.meta.dir, '../Source/domainset/reject_sukka.conf'))
.then(setAddFromArrayCurried(domainSets)))
.then(appendArrayToDomainSets))
]);
// eslint-disable-next-line sukka/no-single-return -- not single return
return shouldStop;

28
Build/lib/fetch-retry.ts
View File

@ -42,28 +42,30 @@ interface FetchRetryOpt {
retries?: number,
factor?: number,
maxRetryAfter?: number,
retry?: number,
onRetry?: (err: Error) => void,
retryOnAborted?: boolean
// onRetry?: (err: Error) => void,
retryOnAborted?: boolean,
retryOnNon2xx?: boolean
}
interface FetchWithRetry {
(url: string | URL | Request, opts?: RequestInit & { retry?: FetchRetryOpt }): Promise<Response>
}
const DEFAULT_OPT: Required<FetchRetryOpt> = {
// timeouts will be [10, 60, 360, 2160, 12960]
// (before randomization is added)
minTimeout: MIN_TIMEOUT,
retries: MAX_RETRIES,
factor: FACTOR,
maxRetryAfter: MAX_RETRY_AFTER,
retryOnAborted: false,
retryOnNon2xx: true
};
function createFetchRetry($fetch: typeof fetch): FetchWithRetry {
const fetchRetry: FetchWithRetry = async (url, opts = {}) => {
const retryOpts = Object.assign(
{
// timeouts will be [10, 60, 360, 2160, 12960]
// (before randomization is added)
minTimeout: MIN_TIMEOUT,
retries: MAX_RETRIES,
factor: FACTOR,
maxRetryAfter: MAX_RETRY_AFTER,
retryOnAborted: false,
retryOnNon2xx: true
},
DEFAULT_OPT,
opts.retry
);

10
Build/lib/get-phishing-domains.ts
View File

@ -131,13 +131,11 @@ export const getPhishingDomains = (parentSpan: Span) => parentSpan.traceChild('g
}
});
span.traceChildSync('get final phishing results', () => {
for (const domain in domainCountMap) {
if (domainCountMap[domain] >= 8) {
domainArr.push(`.${domain}`);
}
for (const domain in domainCountMap) {
if (domainCountMap[domain] >= 8) {
domainArr.push(`.${domain}`);
}
});
}
return domainArr;
});

2
Build/lib/reject-data-source.ts
View File

@ -78,7 +78,7 @@ export const PHISHING_DOMAIN_LISTS: [HostsSource, HostsSource] = [
]
] as const;
type AdGuardFilterSource = string | [main: string, mirrors: string[] | null, ttl: number];
type AdGuardFilterSource = [main: string, mirrors: string[] | null, ttl: number];
export const ADGUARD_FILTERS: AdGuardFilterSource[] = [
// EasyList