Fix: correctly process phishing domains

This commit is contained in:
SukkaW 2024-01-22 00:06:07 +08:00
parent 725f26b428
commit 41b2f543f8
8 changed files with 6 additions and 13 deletions

View File

@ -1,8 +1,7 @@
// @ts-check
import path from 'path';
import { createRuleset } from './lib/create-file';
import { fetchRemoteTextByLine, readFileByLine, readFileIntoProcessedArray } from './lib/fetch-text-by-line';
import { processLine } from './lib/process-line';
import { fetchRemoteTextByLine, readFileIntoProcessedArray } from './lib/fetch-text-by-line';
import { task } from './trace';
import { SHARED_DESCRIPTION } from './lib/constants';
import { isProbablyIpv4, isProbablyIpv6 } from './lib/is-fast-ip';

View File

@ -6,7 +6,6 @@ import { createTrie } from './lib/trie';
import { HOSTS, ADGUARD_FILTERS, PREDEFINED_WHITELIST, DOMAIN_LISTS } from './lib/reject-data-source';
import { createRuleset, compareAndWriteFile } from './lib/create-file';
import { processLine } from './lib/process-line';
import { domainDeduper } from './lib/domain-deduper';
import createKeywordFilter from './lib/aho-corasick';
import { readFileByLine, readFileIntoProcessedArray } from './lib/fetch-text-by-line';

View File

@ -1,4 +1,3 @@
import picocolors from 'picocolors';
import { task } from './trace';
import path from 'path';
import { fetchWithRetry } from './lib/fetch-retry';

View File

@ -1,7 +1,6 @@
// @ts-check
import { readFileByLine } from './fetch-text-by-line';
import { surgeDomainsetToClashDomainset, surgeRulesetToClashClassicalTextRuleset } from './clash';
import { traceAsync } from './trace-runner';
import picocolors from 'picocolors';
import type { Span } from '../trace';
import path from 'path';

View File

@ -1,8 +1,8 @@
import type { BunFile } from 'bun';
import { fetchWithRetry, defaultRequestInit } from './fetch-retry';
import { TextLineStream } from './text-line-transform-stream';
import { PolyfillTextDecoderStream } from './text-decoder-stream';
// import { TextLineStream } from './text-line-transform-stream';
// import { PolyfillTextDecoderStream } from './text-decoder-stream';
import { processLine } from './process-line';
// function createTextLineStreamFromStreamSource(stream: ReadableStream<Uint8Array>) {
// return stream

View File

@ -1,6 +1,5 @@
import { getGorhillPublicSuffixPromise } from './get-gorhill-publicsuffix';
import { processDomainLists, processHosts } from './parse-filter';
import { traceAsync, traceSync } from './trace-runner';
import { processDomainLists } from './parse-filter';
import * as tldts from 'tldts';
import { createTrie } from './trie';
import { createCachedGorhillGetDomain } from './cached-tld-parse';
@ -182,8 +181,8 @@ export const getPhishingDomains = (parentSpan: Span) => parentSpan.traceChild('g
const results = span.traceChild('get final phishing results').traceSyncFn(() => {
const results: string[] = [];
for (const domain in domainCountMap) {
if (domainCountMap[domain] > 5) {
results.push(domain);
if (domainCountMap[domain] >= 5) {
results.push(`.${domain}`);
}
}
return results;

View File

@ -5,7 +5,6 @@ import { processLine } from './process-line';
import { getGorhillPublicSuffixPromise } from './get-gorhill-publicsuffix';
import type { PublicSuffixList } from '@gorhill/publicsuffixlist';
import { traceAsync } from './trace-runner';
import picocolors from 'picocolors';
import { normalizeDomain } from './normalize-domain';
import { fetchAssets } from './fetch-assets';

View File

@ -1,4 +1,3 @@
import path from 'path';
import picocolors from 'picocolors';
type Formatter = (result: string) => string;