From 1d8b452f8763c25539c01dd843cd34e58b8cdcbd Mon Sep 17 00:00:00 2001 From: SukkaW Date: Sat, 16 Aug 2025 22:34:01 +0800 Subject: [PATCH] Chore: drop file writing worker [skip ci] --- Build/lib/create-file.worker.ts | 59 ------------------------------ Build/lib/writing-strategy/base.ts | 13 ------- 2 files changed, 72 deletions(-) delete mode 100644 Build/lib/create-file.worker.ts diff --git a/Build/lib/create-file.worker.ts b/Build/lib/create-file.worker.ts deleted file mode 100644 index 6de496fa..00000000 --- a/Build/lib/create-file.worker.ts +++ /dev/null @@ -1,59 +0,0 @@ -import Worktank from 'worktank'; -import os from 'node:os'; -import process from 'node:process'; -import type { Span } from '../trace'; -import { availableParallelism } from 'foxts/available-parallelism'; - -const pool = new Worktank({ - pool: { - name: 'process-phishing-domains', - size: (availableParallelism(os) - 1) || 1 - }, - worker: { - autoAbort: 10000, // The maximum number of milliseconds to wait for the result from the worker, if exceeded the worker is terminated and the execution promise rejects - autoTerminate: 30000, // The interval of milliseconds at which to check if the pool can be automatically terminated, to free up resources, workers will be spawned up again if needed - env: {}, - methods: { - // eslint-disable-next-line object-shorthand -- workertank - compareAndWriteFile: async function ( - linesA: string[], filePath: string, - importMetaUrl: string - ): Promise { - const { default: module } = await import('node:module'); - const __require = module.createRequire(importMetaUrl); - - const fs = __require('fs') as typeof import('fs'); - const { readFileByLine } = __require('./fetch-text-by-line') as typeof import('./fetch-text-by-line'); - const { fileEqual } = __require('./create-file') as typeof import('./create-file'); - const path = __require('node:path') as typeof import('node:path'); - const { fastStringArrayJoin } = __require('foxts/fast-string-array-join') as typeof import('foxts/fast-string-array-join'); - const picocolors = __require('picocolors') as typeof import('picocolors'); - - let isEqual = false; - if (fs.existsSync(filePath)) { - isEqual = await fileEqual(linesA, readFileByLine(filePath)); - } else { - console.log(`${filePath} does not exists, writing...`); - // isEqual = false; // isEqual is false by default anyway - } - - if (isEqual) { - console.log(picocolors.gray(picocolors.dim(`same content, bail out writing: ${filePath}`))); - return; - } - - const dir = path.dirname(filePath); - if (!fs.existsSync(dir)) { - fs.mkdirSync(dir, { recursive: true }); - } - fs.writeFileSync(filePath, fastStringArrayJoin(linesA, '\n') + '\n', { encoding: 'utf-8' }); - } - } - } -}); - -export function compareAndWriteFileInWorker(span: Span, linesA: string[], filePath: string) { - return span.traceChildAsync(`compare and write (worker) ${filePath}`, () => pool.exec('compareAndWriteFile', [linesA, filePath, import.meta.url])); -} - -process.on('beforeExit', () => pool.terminate()); diff --git a/Build/lib/writing-strategy/base.ts b/Build/lib/writing-strategy/base.ts index a2c61f5c..c358c360 100644 --- a/Build/lib/writing-strategy/base.ts +++ b/Build/lib/writing-strategy/base.ts @@ -1,6 +1,5 @@ import type { Span } from '../../trace'; import { compareAndWriteFile } from '../create-file'; -import { compareAndWriteFileInWorker } from '../create-file.worker'; /** * The class is not about holding rule data, instead it determines how the @@ -79,18 +78,6 @@ export abstract class BaseWriteStrategy { return; } - if (this.result.length > 3000) { - return compareAndWriteFileInWorker( - span, - this.withPadding( - title, - description, - date, - this.result - ), - filePath - ); - } return compareAndWriteFile( span, this.withPadding(