Add new phishing feed / speed up domains sort

This commit is contained in:
SukkaW
2023-12-12 17:10:55 +08:00
parent e56f601fbc
commit e970006445
8 changed files with 72 additions and 89 deletions

View File

@@ -1,13 +1,14 @@
import { domainDeduper } from './lib/domain-deduper';
import path from 'path';
import { createRuleset } from './lib/create-file';
import domainSorter from './lib/stable-sort-domain';
import { sortDomains } from './lib/stable-sort-domain';
import { Sema } from 'async-sema';
import * as tldts from 'tldts';
import { task } from './lib/trace-runner';
import { fetchWithRetry } from './lib/fetch-retry';
import { SHARED_DESCRIPTION } from './lib/constants';
import { getGorhillPublicSuffixPromise } from './lib/get-gorhill-publicsuffix';
const s = new Sema(3);
@@ -140,7 +141,9 @@ export const buildSpeedtestDomainSet = task(import.meta.path, async () => {
}
}
const deduped = domainDeduper(Array.from(domains)).sort(domainSorter);
const gorhill = await getGorhillPublicSuffixPromise();
const deduped = sortDomains(domainDeduper(Array.from(domains)), gorhill);
const description = [
...SHARED_DESCRIPTION,
'',