diff --git a/Build/validate-domain-alive.ts b/Build/validate-domain-alive.ts index 5d98f2aa..57266bce 100644 --- a/Build/validate-domain-alive.ts +++ b/Build/validate-domain-alive.ts @@ -35,39 +35,19 @@ function onDomain(args: [string, boolean]) { .withPromise(); await Promise.all([ - ...domainSets.map(runAgainstDomainset), - ...domainRules.map(runAgainstRuleset) - ]); + ...domainRules, + ...domainSets + ].map(filepath => runAgainstSourceFile( + filepath, + (domain: string, includeAllSubdomain: boolean) => queue.add( + () => keyedAsyncMutexWithQueue( + domain, + () => isDomainAlive(domain, includeAllSubdomain) + ).then(onDomain) + ).then(() => console.log('[done]', filepath)) + ))); console.log(); console.log(); console.log(JSON.stringify(deadDomains)); })(); - -export async function runAgainstRuleset(filepath: string) { - const promises: Array> = []; - await runAgainstSourceFile( - filepath, - (domain: string, includeAllSubdomain: boolean) => queue.add(() => keyedAsyncMutexWithQueue( - domain, - () => isDomainAlive(domain, includeAllSubdomain) - ).then(onDomain)) - ); - - await Promise.all(promises); - console.log('[done]', filepath); -} - -export async function runAgainstDomainset(filepath: string) { - const promises: Array> = []; - - await runAgainstSourceFile( - filepath, - (domain: string, includeAllSubdomain: boolean) => queue.add(() => keyedAsyncMutexWithQueue( - domain, - () => isDomainAlive(domain, includeAllSubdomain) - ).then(onDomain)) - ); - await Promise.all(promises); - console.log('[done]', filepath); -} diff --git a/Build/validate-reject-stats.ts b/Build/validate-reject-stats.ts index 8dd5951b..3301eefd 100644 --- a/Build/validate-reject-stats.ts +++ b/Build/validate-reject-stats.ts @@ -5,8 +5,31 @@ import { loosTldOptWithPrivateDomains } from './constants/loose-tldts-opt'; import runAgainstSourceFile from './lib/run-against-source-file'; (async () => { - const rejectDomainCountMap = await runAgainstDomainset(new Map(), path.join(OUTPUT_SURGE_DIR, 'domainset', 'reject.conf')); - const rejectExtraDomainCountMap = await runAgainstDomainset(new Map(), path.join(OUTPUT_SURGE_DIR, 'domainset', 'reject_extra.conf')); + const rejectDomainCountMap = new Map(); + const rejectExtraDomainCountMap = new Map(); + + const callback = (map: Map) => (domain: string) => { + const apexDomain = tldts.getDomain(domain, loosTldOptWithPrivateDomains); + if (!apexDomain) { + return; + } + + map.set( + apexDomain, + rejectDomainCountMap.has(apexDomain) + ? rejectDomainCountMap.get(apexDomain)! + 1 + : 1 + ); + }; + + await runAgainstSourceFile( + path.join(OUTPUT_SURGE_DIR, 'domainset', 'reject.conf'), + callback(rejectDomainCountMap) + ); + await runAgainstSourceFile( + path.join(OUTPUT_SURGE_DIR, 'domainset', 'reject_extra.conf'), + callback(rejectExtraDomainCountMap) + ); const rejectDomainCountArr = Array.from(rejectDomainCountMap).sort((a, b) => b[1] - a[1]).filter(([, count]) => count > 20); const rejectExtraDomainCountArr = Array.from(rejectExtraDomainCountMap).sort((a, b) => b[1] - a[1]).filter(([, count]) => count > 20); @@ -14,24 +37,3 @@ import runAgainstSourceFile from './lib/run-against-source-file'; console.table(rejectDomainCountArr); console.table(rejectExtraDomainCountArr); })(); - -async function runAgainstDomainset(rejectDomainCountMap: Map, file: string) { - await runAgainstSourceFile( - file, - (domain: string) => { - const apexDomain = tldts.getDomain(domain, loosTldOptWithPrivateDomains); - if (!apexDomain) { - return; - } - - rejectDomainCountMap.set( - apexDomain, - rejectDomainCountMap.has(apexDomain) - ? rejectDomainCountMap.get(apexDomain)! + 1 - : 1 - ); - } - ); - - return rejectDomainCountMap; -}