diff --git a/Build/constants/reject-data-source.ts b/Build/constants/reject-data-source.ts index 9aacbe81..1aa79f52 100644 --- a/Build/constants/reject-data-source.ts +++ b/Build/constants/reject-data-source.ts @@ -1,5 +1,7 @@ import { TTL } from '../lib/cache-filesystem'; +export const DEBUG_DOMAIN_TO_FIND: string | null = null; // example.com | null + type HostsSource = [main: string, mirrors: string[] | null, includeAllSubDomain: boolean, ttl: number]; export const HOSTS: HostsSource[] = [ diff --git a/Build/lib/aho-corasick.ts b/Build/lib/aho-corasick.ts index d6d9f281..dc095b66 100644 --- a/Build/lib/aho-corasick.ts +++ b/Build/lib/aho-corasick.ts @@ -12,11 +12,9 @@ function createKeywordFilter(keys: string[] | Set) { // Create a trie with extra fields and information const put = (key: string) => { - const len = key.length; - let node = root; - for (let idx = 0; idx < len; idx++) { + for (let idx = 0, len = key.length; idx < len; idx++) { const char = key[idx]; if (node.has(char)) { diff --git a/Build/lib/cache-filesystem.ts b/Build/lib/cache-filesystem.ts index 31af2a34..79a7b95d 100644 --- a/Build/lib/cache-filesystem.ts +++ b/Build/lib/cache-filesystem.ts @@ -153,7 +153,7 @@ export class Cache { } const end = performance.now(); - console.log(`${picocolors.gray(`[${((end - start) / 1e6).toFixed(3)}ms]`)} cache initialized from ${this.cachePath}`); + console.log(`${picocolors.gray(`[${((end - start)).toFixed(3)}ns]`)} cache initialized from ${this.tableName} @ ${this.cachePath}`); } set(key: string, value: string, ttl = 60 * 1000): void { diff --git a/Build/lib/get-phishing-domains.ts b/Build/lib/get-phishing-domains.ts index 0e18baa2..b8741be5 100644 --- a/Build/lib/get-phishing-domains.ts +++ b/Build/lib/get-phishing-domains.ts @@ -4,12 +4,13 @@ import * as tldts from 'tldts-experimental'; import { dummySpan, printTraceResult } from '../trace'; import type { Span } from '../trace'; import { appendArrayInPlaceCurried } from './append-array-in-place'; -import { PHISHING_DOMAIN_LISTS_EXTRA, PHISHING_HOSTS_EXTRA } from '../constants/reject-data-source'; +import { DEBUG_DOMAIN_TO_FIND, PHISHING_DOMAIN_LISTS_EXTRA, PHISHING_HOSTS_EXTRA } from '../constants/reject-data-source'; import { loosTldOptWithPrivateDomains } from '../constants/loose-tldts-opt'; import picocolors from 'picocolors'; import createKeywordFilter from './aho-corasick'; import { createCacheKey, deserializeArray, serializeArray } from './cache-filesystem'; import { cache } from './fs-memo'; +import { isCI } from 'ci-info'; const BLACK_TLD = new Set([ 'accountant', 'art', 'autos', @@ -112,12 +113,22 @@ const processPhihsingDomains = cache(function processPhihsingDomains(domainArr: const domainCountMap = new Map(); const domainScoreMap: Record = {}; + let line = ''; let tld: string | null = ''; let apexDomain: string | null = ''; let subdomain: string | null = ''; + // const set = new Set(); + // let duplicateCount = 0; + for (let i = 0, len = domainArr.length; i < len; i++) { - const line = domainArr[i]; + line = domainArr[i]; + + // if (set.has(line)) { + // duplicateCount++; + // } else { + // set.add(line); + // } const parsed = tldts.parse(line, loosTldOptWithPrivateDomains); if (parsed.isPrivate) { @@ -183,11 +194,13 @@ const processPhihsingDomains = cache(function processPhihsingDomains(domainArr: // count: domainCountMap.get('flk-ipfs.xyz') // }); + // console.log({ duplicateCount, domainArrLen: domainArr.length }); + return Promise.resolve(domainArr); }, { serializer: serializeArray, deserializer: deserializeArray, - temporaryBypass: true + temporaryBypass: !isCI || DEBUG_DOMAIN_TO_FIND !== null }); const cacheKey = createCacheKey(__filename); @@ -205,6 +218,8 @@ export function getPhishingDomains(parentSpan: Span) { return domainArr; }); + console.log({ len: domainArr.length }); + return span.traceChildAsync( 'process phishing domain set', () => processPhihsingDomains(domainArr) diff --git a/Build/lib/parse-filter.ts b/Build/lib/parse-filter.ts index f7351115..f2b893ba 100644 --- a/Build/lib/parse-filter.ts +++ b/Build/lib/parse-filter.ts @@ -9,8 +9,8 @@ import type { Span } from '../trace'; import createKeywordFilter from './aho-corasick'; import { looseTldtsOpt } from '../constants/loose-tldts-opt'; import { identity } from './misc'; +import { DEBUG_DOMAIN_TO_FIND } from '../constants/reject-data-source'; -const DEBUG_DOMAIN_TO_FIND: string | null = null; // example.com | null let foundDebugDomain = false; const temporaryBypass = typeof DEBUG_DOMAIN_TO_FIND === 'string'; diff --git a/Build/lib/rules/base.ts b/Build/lib/rules/base.ts index 6f53df10..0d3450a1 100644 --- a/Build/lib/rules/base.ts +++ b/Build/lib/rules/base.ts @@ -65,7 +65,11 @@ export abstract class RuleOutput { return result; }; - constructor(protected readonly span: Span, protected readonly id: string) { } + protected readonly span: Span; + + constructor($span: Span, protected readonly id: string) { + this.span = $span.traceChild('RuleOutput#' + id); + } protected title: string | null = null; withTitle(title: string) { @@ -201,7 +205,7 @@ export abstract class RuleOutput { return this; } - static readonly ipToCidr = (ip: string, version: 4 | 6 = 4) => { + static readonly ipToCidr = (ip: string, version: 4 | 6) => { if (ip.includes('/')) return ip; if (version === 4) { return ip + '/32'; @@ -257,7 +261,7 @@ export abstract class RuleOutput { if (this.$$preprocessed === null) { this.guardPendingPromise(); - this.$$preprocessed = this.span.traceChildSync('RuleOutput#preprocess: ' + this.id, () => this.preprocess()); + this.$$preprocessed = this.span.traceChildSync('preprocess', () => this.preprocess()); } return this.$$preprocessed; } @@ -280,56 +284,56 @@ export abstract class RuleOutput { ); } - async write(): Promise { - await this.done(); + write(): Promise { + return this.done().then(() => this.span.traceChildAsync('write all', async () => { + invariant(this.title, 'Missing title'); + invariant(this.description, 'Missing description'); - invariant(this.title, 'Missing title'); - invariant(this.description, 'Missing description'); - - const promises = [ - compareAndWriteFile( - this.span, - withBannerArray( - this.title, - this.description, - this.date, - this.surge() + const promises = [ + compareAndWriteFile( + this.span, + withBannerArray( + this.title, + this.description, + this.date, + this.surge() + ), + path.join(OUTPUT_SURGE_DIR, this.type, this.id + '.conf') ), - path.join(OUTPUT_SURGE_DIR, this.type, this.id + '.conf') - ), - compareAndWriteFile( - this.span, - withBannerArray( - this.title, - this.description, - this.date, - this.clash() + compareAndWriteFile( + this.span, + withBannerArray( + this.title, + this.description, + this.date, + this.clash() + ), + path.join(OUTPUT_CLASH_DIR, this.type, this.id + '.txt') ), - path.join(OUTPUT_CLASH_DIR, this.type, this.id + '.txt') - ), - compareAndWriteFile( - this.span, - this.singbox(), - path.join(OUTPUT_SINGBOX_DIR, this.type, this.id + '.json') - ) - ]; + compareAndWriteFile( + this.span, + this.singbox(), + path.join(OUTPUT_SINGBOX_DIR, this.type, this.id + '.json') + ) + ]; - if (this.mitmSgmodule) { - const sgmodule = this.mitmSgmodule(); - const sgModulePath = this.mitmSgmodulePath ?? path.join(this.type, this.id + '.sgmodule'); + if (this.mitmSgmodule) { + const sgmodule = this.mitmSgmodule(); + const sgModulePath = this.mitmSgmodulePath ?? path.join(this.type, this.id + '.sgmodule'); - if (sgmodule) { - promises.push( - compareAndWriteFile( - this.span, - sgmodule, - path.join(OUTPUT_MODULES_DIR, sgModulePath) - ) - ); + if (sgmodule) { + promises.push( + compareAndWriteFile( + this.span, + sgmodule, + path.join(OUTPUT_MODULES_DIR, sgModulePath) + ) + ); + } } - } - await Promise.all(promises); + await Promise.all(promises); + })); } abstract surge(): string[]; diff --git a/Build/trace/index.ts b/Build/trace/index.ts index 0116930a..6b9db967 100644 --- a/Build/trace/index.ts +++ b/Build/trace/index.ts @@ -148,7 +148,15 @@ export async function whyIsNodeRunning() { export function printTraceResult(traceResult: TraceResult = rootTraceResult) { printStats(traceResult.children); - printTree(traceResult, node => `${node.name} ${picocolors.bold(`${(node.end - node.start).toFixed(3)}ms`)}`); + printTree( + traceResult, + node => { + if (node.end - node.start < 0) { + return node.name; + } + return `${node.name} ${picocolors.bold(`${(node.end - node.start).toFixed(3)}ms`)}`; + } + ); } function printTree(initialTree: TraceResult, printNode: (node: TraceResult, branch: string) => string) {