From 4f76dd089a800df515a2df9eaaa5a9f94704f887 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Tue, 8 Oct 2024 21:32:56 +0800 Subject: [PATCH] Perf: replace sha256 w/ xxhash --- Build/lib/get-phishing-domains.ts | 12 ++--- Build/lib/text-line-transform-stream.ts | 20 +++++---- package.json | 1 - pnpm-lock.yaml | 58 ++++++------------------- 4 files changed, 30 insertions(+), 61 deletions(-) diff --git a/Build/lib/get-phishing-domains.ts b/Build/lib/get-phishing-domains.ts index 33ec9bfd..5f2a4e8b 100644 --- a/Build/lib/get-phishing-domains.ts +++ b/Build/lib/get-phishing-domains.ts @@ -10,8 +10,7 @@ import picocolors from 'picocolors'; import createKeywordFilter from './aho-corasick'; import { createCacheKey, deserializeArray, fsFetchCache, serializeArray } from './cache-filesystem'; import { fastStringArrayJoin } from './misc'; - -import { sha256 } from 'hash-wasm'; +import { stringHash } from './string-hash'; const BLACK_TLD = new Set([ 'accountant', 'autos', @@ -114,16 +113,17 @@ export const getPhishingDomains = (parentSpan: Span) => parentSpan.traceChild('g return domainArr; }); + const cacheHash = span.traceChildSync('get hash', () => stringHash(fastStringArrayJoin(domainArr, '|'))); + return span.traceChildAsync( 'process phishing domain set', - () => processPhihsingDomains(domainArr) + () => processPhihsingDomains(domainArr, cacheHash) ); }); -async function processPhihsingDomains(domainArr: string[]) { - const hash = await sha256(fastStringArrayJoin(domainArr, '|')); +async function processPhihsingDomains(domainArr: string[], cacheHash = '') { return fsFetchCache.apply( - cacheKey('processPhihsingDomains|' + hash), + cacheKey('processPhihsingDomains|' + cacheHash), () => { const domainCountMap: Record = {}; const domainScoreMap: Record = {}; diff --git a/Build/lib/text-line-transform-stream.ts b/Build/lib/text-line-transform-stream.ts index 618aabe1..ac841afc 100644 --- a/Build/lib/text-line-transform-stream.ts +++ b/Build/lib/text-line-transform-stream.ts @@ -48,17 +48,19 @@ export class TextLineStream extends TransformStream { } } - if (lfIndex !== -1) { - let crOrLfIndex = lfIndex; - if (chunk[lfIndex - 1] === '\r') { - crOrLfIndex--; - } - controller.enqueue(chunk.slice(chunkIndex, crOrLfIndex)); - chunkIndex = lfIndex + 1; - continue; + if (lfIndex === -1) { + // we can no longer find a line break in the chunk, break the current loop + break; } - break; + // enqueue current line, and loop again to find next line + let crOrLfIndex = lfIndex; + if (chunk[lfIndex - 1] === '\r') { + crOrLfIndex--; + } + controller.enqueue(chunk.slice(chunkIndex, crOrLfIndex)); + chunkIndex = lfIndex + 1; + continue; } __buf = chunk.slice(chunkIndex); diff --git a/package.json b/package.json index e2d70a62..24ca0fbd 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,6 @@ "fast-cidr-tools": "^0.3.1", "fdir": "^6.4.0", "foxact": "^0.2.38", - "hash-wasm": "^4.11.0", "json-stringify-pretty-compact": "^3.0.0", "mnemonist": "^0.39.8", "picocolors": "^1.1.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3e38177a..9e2a2560 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -44,9 +44,6 @@ importers: foxact: specifier: ^0.2.38 version: 0.2.38 - hash-wasm: - specifier: ^4.11.0 - version: 4.11.0 json-stringify-pretty-compact: specifier: ^3.0.0 version: 3.0.0 @@ -188,10 +185,6 @@ packages: resolution: {integrity: sha512-4Bfj15dVJdoy3RfZmmo86RK1Fwzn6SstsvK9JS+BaVKqC6QQQQyXekNaC+g+LKNgkQ+2VhGAzm6hO40AhMR3zQ==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} - '@eslint/js@9.11.1': - resolution: {integrity: sha512-/qu+TWz8WwPWc7/HcIJKi+c+MOm46GdVaSlTTQcaqaL53+GsoA6MxWp5PtTx48qbSP7ylM1Kn7nhvkugfJvRSA==} - engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} - '@eslint/js@9.12.0': resolution: {integrity: sha512-eohesHH8WFRUprDNyEREgqP6beG6htMeUYeCpkEgBCieCMme5r9zFWjzAJp//9S+Kub4rqE+jXe9Cp1a7IYIIA==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} @@ -706,15 +699,6 @@ packages: supports-color: optional: true - debug@4.3.6: - resolution: {integrity: sha512-O/09Bd4Z1fBrU4VzkhFqVgpPzaGbw6Sm9FEkBT1A/YBXQFGuuSxa1dN2nxgxS34JmKXqYx8CZAwEVoJFImUXIg==} - engines: {node: '>=6.0'} - peerDependencies: - supports-color: '*' - peerDependenciesMeta: - supports-color: - optional: true - debug@4.3.7: resolution: {integrity: sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==} engines: {node: '>=6.0'} @@ -1030,9 +1014,6 @@ packages: resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} engines: {node: '>=8'} - hash-wasm@4.11.0: - resolution: {integrity: sha512-HVusNXlVqHe0fzIzdQOGolnFN6mX/fqcrSAOcTBXdvzrXVHwTz11vXeKRmkR5gTuwVpvHZEIyKoePDvuAR+XwQ==} - hasown@2.0.2: resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} engines: {node: '>= 0.4'} @@ -1186,9 +1167,6 @@ packages: engines: {node: '>= 14.0.0'} hasBin: true - ms@2.1.2: - resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==} - ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} @@ -1624,7 +1602,7 @@ snapshots: '@eslint/config-array@0.18.0': dependencies: '@eslint/object-schema': 2.1.4 - debug: 4.3.7 + debug: 4.3.7(supports-color@8.1.1) minimatch: 3.1.2 transitivePeerDependencies: - supports-color @@ -1634,7 +1612,7 @@ snapshots: '@eslint/eslintrc@3.1.0': dependencies: ajv: 6.12.6 - debug: 4.3.7 + debug: 4.3.7(supports-color@8.1.1) espree: 10.2.0 globals: 14.0.0 ignore: 5.3.2 @@ -1645,8 +1623,6 @@ snapshots: transitivePeerDependencies: - supports-color - '@eslint/js@9.11.1': {} - '@eslint/js@9.12.0': {} '@eslint/object-schema@2.1.4': {} @@ -1777,7 +1753,7 @@ snapshots: '@swc-node/sourcemap-support': 0.5.1 '@swc/core': 1.7.28 colorette: 2.0.20 - debug: 4.3.6(supports-color@8.1.1) + debug: 4.3.7(supports-color@8.1.1) oxc-resolver: 1.10.2 pirates: 4.0.6 tslib: 2.7.0 @@ -1925,7 +1901,7 @@ snapshots: '@typescript-eslint/types': 8.7.0 '@typescript-eslint/typescript-estree': 8.7.0(typescript@5.6.2) '@typescript-eslint/visitor-keys': 8.7.0 - debug: 4.3.7 + debug: 4.3.7(supports-color@8.1.1) eslint: 9.12.0 optionalDependencies: typescript: 5.6.2 @@ -1941,7 +1917,7 @@ snapshots: dependencies: '@typescript-eslint/typescript-estree': 8.7.0(typescript@5.6.2) '@typescript-eslint/utils': 8.7.0(eslint@9.12.0)(typescript@5.6.2) - debug: 4.3.7 + debug: 4.3.7(supports-color@8.1.1) ts-api-utils: 1.3.0(typescript@5.6.2) optionalDependencies: typescript: 5.6.2 @@ -1955,7 +1931,7 @@ snapshots: dependencies: '@typescript-eslint/types': 8.7.0 '@typescript-eslint/visitor-keys': 8.7.0 - debug: 4.3.7 + debug: 4.3.7(supports-color@8.1.1) fast-glob: 3.3.2 is-glob: 4.0.3 minimatch: 9.0.5 @@ -2166,15 +2142,11 @@ snapshots: dependencies: ms: 2.1.3 - debug@4.3.6(supports-color@8.1.1): - dependencies: - ms: 2.1.2 - optionalDependencies: - supports-color: 8.1.1 - - debug@4.3.7: + debug@4.3.7(supports-color@8.1.1): dependencies: ms: 2.1.3 + optionalDependencies: + supports-color: 8.1.1 decamelize@4.0.0: {} @@ -2224,7 +2196,7 @@ snapshots: dependencies: '@eslint-community/eslint-plugin-eslint-comments': 4.4.0(eslint@9.12.0) '@eslint-sukka/shared': 6.6.1(eslint@9.12.0)(typescript@5.6.2) - '@eslint/js': 9.11.1 + '@eslint/js': 9.12.0 '@stylistic/eslint-plugin-js': 2.8.0(eslint@9.12.0) '@stylistic/eslint-plugin-plus': 2.8.0(eslint@9.12.0) '@stylistic/eslint-plugin-ts': 2.8.0(eslint@9.12.0)(typescript@5.6.2) @@ -2283,7 +2255,7 @@ snapshots: eslint-plugin-import-x@4.3.0(eslint@9.12.0)(typescript@5.6.2): dependencies: '@typescript-eslint/utils': 8.7.0(eslint@9.12.0)(typescript@5.6.2) - debug: 4.3.7 + debug: 4.3.7(supports-color@8.1.1) doctrine: 3.0.0 eslint: 9.12.0 eslint-import-resolver-node: 0.3.9 @@ -2380,7 +2352,7 @@ snapshots: ajv: 6.12.6 chalk: 4.1.2 cross-spawn: 7.0.3 - debug: 4.3.7 + debug: 4.3.7(supports-color@8.1.1) escape-string-regexp: 4.0.0 eslint-scope: 8.1.0 eslint-visitor-keys: 4.1.0 @@ -2529,8 +2501,6 @@ snapshots: has-flag@4.0.0: {} - hash-wasm@4.11.0: {} - hasown@2.0.2: dependencies: function-bind: 1.1.2 @@ -2662,7 +2632,7 @@ snapshots: ansi-colors: 4.1.3 browser-stdout: 1.3.1 chokidar: 3.6.0 - debug: 4.3.6(supports-color@8.1.1) + debug: 4.3.7(supports-color@8.1.1) diff: 5.2.0 escape-string-regexp: 4.0.0 find-up: 5.0.0 @@ -2680,8 +2650,6 @@ snapshots: yargs-parser: 20.2.9 yargs-unparser: 2.0.0 - ms@2.1.2: {} - ms@2.1.3: {} napi-build-utils@1.0.2: {}