diff --git a/Build/build-reject-domainset.ts b/Build/build-reject-domainset.ts index 1a3492ce..7172858f 100644 --- a/Build/build-reject-domainset.ts +++ b/Build/build-reject-domainset.ts @@ -18,7 +18,7 @@ import * as tldts from 'tldts-experimental'; import { SHARED_DESCRIPTION } from './lib/constants'; import { getPhishingDomains } from './lib/get-phishing-domains'; -import { add as SetAdd, subtract as SetSubstract } from 'mnemonist/set'; +import { subtract as SetSubstract } from 'mnemonist/set'; import { setAddFromArray, setAddFromArrayCurried } from './lib/set-add-from-array'; import { sort } from './lib/timsort'; @@ -27,6 +27,7 @@ export const buildRejectDomainSet = task(import.meta.path, async (span) => { const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST); const domainSets = new Set(); + const appendArrayToDomainSets = setAddFromArrayCurried(domainSets); // Parse from AdGuard Filters const shouldStop = await span @@ -36,33 +37,35 @@ export const buildRejectDomainSet = task(import.meta.path, async (span) => { let shouldStop = false; await Promise.all([ // Parse from remote hosts & domain lists - ...HOSTS.map(entry => processHosts(childSpan, ...entry).then(setAddFromArrayCurried(domainSets))), + ...HOSTS.map(entry => processHosts(childSpan, ...entry).then(appendArrayToDomainSets)), - ...DOMAIN_LISTS.map(entry => processDomainLists(childSpan, ...entry).then(setAddFromArrayCurried(domainSets))), + ...DOMAIN_LISTS.map(entry => processDomainLists(childSpan, ...entry).then(appendArrayToDomainSets)), - ...ADGUARD_FILTERS.map(input => ( - typeof input === 'string' - ? processFilterRules(childSpan, input) - : processFilterRules(childSpan, ...input) - ).then(({ white, black, foundDebugDomain }) => { - if (foundDebugDomain) { - // eslint-disable-next-line sukka/no-single-return -- not single return - shouldStop = true; - // we should not break here, as we want to see full matches from all data source - } - setAddFromArray(filterRuleWhitelistDomainSets, white); - setAddFromArray(domainSets, black); - })), + ...ADGUARD_FILTERS.map( + input => processFilterRules(childSpan, ...input) + .then(({ white, black, foundDebugDomain }) => { + if (foundDebugDomain) { + // eslint-disable-next-line sukka/no-single-return -- not single return + shouldStop = true; + // we should not break here, as we want to see full matches from all data source + } + setAddFromArray(filterRuleWhitelistDomainSets, white); + setAddFromArray(domainSets, black); + }) + ), ...([ 'https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt', 'https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt' - ].map(input => processFilterRules(childSpan, input).then(({ white, black }) => { - setAddFromArray(filterRuleWhitelistDomainSets, white); - setAddFromArray(filterRuleWhitelistDomainSets, black); - }))), - getPhishingDomains(childSpan).then(setAddFromArrayCurried(domainSets)), + ].map( + input => processFilterRules(childSpan, input) + .then(({ white, black }) => { + setAddFromArray(filterRuleWhitelistDomainSets, white); + setAddFromArray(filterRuleWhitelistDomainSets, black); + }) + )), + getPhishingDomains(childSpan).then(appendArrayToDomainSets), childSpan.traceChildAsync('process reject_sukka.conf', () => readFileIntoProcessedArray(path.resolve(import.meta.dir, '../Source/domainset/reject_sukka.conf')) - .then(setAddFromArrayCurried(domainSets))) + .then(appendArrayToDomainSets)) ]); // eslint-disable-next-line sukka/no-single-return -- not single return return shouldStop; diff --git a/Build/lib/fetch-retry.ts b/Build/lib/fetch-retry.ts index 09539272..1c475152 100644 --- a/Build/lib/fetch-retry.ts +++ b/Build/lib/fetch-retry.ts @@ -42,28 +42,30 @@ interface FetchRetryOpt { retries?: number, factor?: number, maxRetryAfter?: number, - retry?: number, - onRetry?: (err: Error) => void, - retryOnAborted?: boolean + // onRetry?: (err: Error) => void, + retryOnAborted?: boolean, + retryOnNon2xx?: boolean } interface FetchWithRetry { (url: string | URL | Request, opts?: RequestInit & { retry?: FetchRetryOpt }): Promise } +const DEFAULT_OPT: Required = { + // timeouts will be [10, 60, 360, 2160, 12960] + // (before randomization is added) + minTimeout: MIN_TIMEOUT, + retries: MAX_RETRIES, + factor: FACTOR, + maxRetryAfter: MAX_RETRY_AFTER, + retryOnAborted: false, + retryOnNon2xx: true +}; + function createFetchRetry($fetch: typeof fetch): FetchWithRetry { const fetchRetry: FetchWithRetry = async (url, opts = {}) => { const retryOpts = Object.assign( - { - // timeouts will be [10, 60, 360, 2160, 12960] - // (before randomization is added) - minTimeout: MIN_TIMEOUT, - retries: MAX_RETRIES, - factor: FACTOR, - maxRetryAfter: MAX_RETRY_AFTER, - retryOnAborted: false, - retryOnNon2xx: true - }, + DEFAULT_OPT, opts.retry ); diff --git a/Build/lib/get-phishing-domains.ts b/Build/lib/get-phishing-domains.ts index 8faf5425..8296fadf 100644 --- a/Build/lib/get-phishing-domains.ts +++ b/Build/lib/get-phishing-domains.ts @@ -131,13 +131,11 @@ export const getPhishingDomains = (parentSpan: Span) => parentSpan.traceChild('g } }); - span.traceChildSync('get final phishing results', () => { - for (const domain in domainCountMap) { - if (domainCountMap[domain] >= 8) { - domainArr.push(`.${domain}`); - } + for (const domain in domainCountMap) { + if (domainCountMap[domain] >= 8) { + domainArr.push(`.${domain}`); } - }); + } return domainArr; }); diff --git a/Build/lib/reject-data-source.ts b/Build/lib/reject-data-source.ts index baf53748..92747bf3 100644 --- a/Build/lib/reject-data-source.ts +++ b/Build/lib/reject-data-source.ts @@ -78,7 +78,7 @@ export const PHISHING_DOMAIN_LISTS: [HostsSource, HostsSource] = [ ] ] as const; -type AdGuardFilterSource = string | [main: string, mirrors: string[] | null, ttl: number]; +type AdGuardFilterSource = [main: string, mirrors: string[] | null, ttl: number]; export const ADGUARD_FILTERS: AdGuardFilterSource[] = [ // EasyList