mirror of
https://github.com/SukkaW/Surge.git
synced 2025-12-13 17:50:34 +08:00
Chore: drop file writing worker [skip ci]
This commit is contained in:
parent
4e44565515
commit
1d8b452f87
@ -1,59 +0,0 @@
|
|||||||
import Worktank from 'worktank';
|
|
||||||
import os from 'node:os';
|
|
||||||
import process from 'node:process';
|
|
||||||
import type { Span } from '../trace';
|
|
||||||
import { availableParallelism } from 'foxts/available-parallelism';
|
|
||||||
|
|
||||||
const pool = new Worktank({
|
|
||||||
pool: {
|
|
||||||
name: 'process-phishing-domains',
|
|
||||||
size: (availableParallelism(os) - 1) || 1
|
|
||||||
},
|
|
||||||
worker: {
|
|
||||||
autoAbort: 10000, // The maximum number of milliseconds to wait for the result from the worker, if exceeded the worker is terminated and the execution promise rejects
|
|
||||||
autoTerminate: 30000, // The interval of milliseconds at which to check if the pool can be automatically terminated, to free up resources, workers will be spawned up again if needed
|
|
||||||
env: {},
|
|
||||||
methods: {
|
|
||||||
// eslint-disable-next-line object-shorthand -- workertank
|
|
||||||
compareAndWriteFile: async function (
|
|
||||||
linesA: string[], filePath: string,
|
|
||||||
importMetaUrl: string
|
|
||||||
): Promise<void> {
|
|
||||||
const { default: module } = await import('node:module');
|
|
||||||
const __require = module.createRequire(importMetaUrl);
|
|
||||||
|
|
||||||
const fs = __require('fs') as typeof import('fs');
|
|
||||||
const { readFileByLine } = __require('./fetch-text-by-line') as typeof import('./fetch-text-by-line');
|
|
||||||
const { fileEqual } = __require('./create-file') as typeof import('./create-file');
|
|
||||||
const path = __require('node:path') as typeof import('node:path');
|
|
||||||
const { fastStringArrayJoin } = __require('foxts/fast-string-array-join') as typeof import('foxts/fast-string-array-join');
|
|
||||||
const picocolors = __require('picocolors') as typeof import('picocolors');
|
|
||||||
|
|
||||||
let isEqual = false;
|
|
||||||
if (fs.existsSync(filePath)) {
|
|
||||||
isEqual = await fileEqual(linesA, readFileByLine(filePath));
|
|
||||||
} else {
|
|
||||||
console.log(`${filePath} does not exists, writing...`);
|
|
||||||
// isEqual = false; // isEqual is false by default anyway
|
|
||||||
}
|
|
||||||
|
|
||||||
if (isEqual) {
|
|
||||||
console.log(picocolors.gray(picocolors.dim(`same content, bail out writing: ${filePath}`)));
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const dir = path.dirname(filePath);
|
|
||||||
if (!fs.existsSync(dir)) {
|
|
||||||
fs.mkdirSync(dir, { recursive: true });
|
|
||||||
}
|
|
||||||
fs.writeFileSync(filePath, fastStringArrayJoin(linesA, '\n') + '\n', { encoding: 'utf-8' });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
export function compareAndWriteFileInWorker(span: Span, linesA: string[], filePath: string) {
|
|
||||||
return span.traceChildAsync(`compare and write (worker) ${filePath}`, () => pool.exec('compareAndWriteFile', [linesA, filePath, import.meta.url]));
|
|
||||||
}
|
|
||||||
|
|
||||||
process.on('beforeExit', () => pool.terminate());
|
|
||||||
@ -1,6 +1,5 @@
|
|||||||
import type { Span } from '../../trace';
|
import type { Span } from '../../trace';
|
||||||
import { compareAndWriteFile } from '../create-file';
|
import { compareAndWriteFile } from '../create-file';
|
||||||
import { compareAndWriteFileInWorker } from '../create-file.worker';
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The class is not about holding rule data, instead it determines how the
|
* The class is not about holding rule data, instead it determines how the
|
||||||
@ -79,18 +78,6 @@ export abstract class BaseWriteStrategy {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (this.result.length > 3000) {
|
|
||||||
return compareAndWriteFileInWorker(
|
|
||||||
span,
|
|
||||||
this.withPadding(
|
|
||||||
title,
|
|
||||||
description,
|
|
||||||
date,
|
|
||||||
this.result
|
|
||||||
),
|
|
||||||
filePath
|
|
||||||
);
|
|
||||||
}
|
|
||||||
return compareAndWriteFile(
|
return compareAndWriteFile(
|
||||||
span,
|
span,
|
||||||
this.withPadding(
|
this.withPadding(
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user