Housekeeping [skip ci]

This commit is contained in:
SukkaW 2025-07-27 21:02:35 +08:00
parent df7af76375
commit 3d1514f0d1
5 changed files with 501 additions and 472 deletions

View File

@ -5,12 +5,14 @@ import { RulesetOutput } from './lib/rules/ruleset';
import Worktank from 'worktank'; import Worktank from 'worktank';
const pool = new Worktank({ const pool = new Worktank({
pool: {
name: 'build-internal-reverse-chn-cidr', name: 'build-internal-reverse-chn-cidr',
size: 1, size: 1 // The number of workers to keep in the pool, if more workers are needed they will be spawned up to this limit
timeout: 10000, // The maximum number of milliseconds to wait for the result from the worker, if exceeded the worker is terminated and the execution promise rejects },
warmup: true, worker: {
autoterminate: 30000, // The interval of milliseconds at which to check if the pool can be automatically terminated, to free up resources, workers will be spawned up again if needed autoAbort: 10000,
env: {}, autoTerminate: 30000, // The interval of milliseconds at which to check if the pool can be automatically terminated, to free up resources, workers will be spawned up again if needed
autoInstantiate: true,
methods: { methods: {
// eslint-disable-next-line object-shorthand -- workertank // eslint-disable-next-line object-shorthand -- workertank
getMicrosoftCdnRuleset: async function (importMetaUrl: string): Promise<[domains: string[], domainSuffixes: string[]]> { getMicrosoftCdnRuleset: async function (importMetaUrl: string): Promise<[domains: string[], domainSuffixes: string[]]> {
@ -42,6 +44,7 @@ const pool = new Worktank({
return [domains, domainSuffixes] as const; return [domains, domainSuffixes] as const;
} }
} }
}
}); });
export const getMicrosoftCdnRulesetPromise = once<Promise<[domains: string[], domainSuffixes: string[]]>>(async () => { export const getMicrosoftCdnRulesetPromise = once<Promise<[domains: string[], domainSuffixes: string[]]>>(async () => {

View File

@ -2,20 +2,16 @@ import Worktank from 'worktank';
import os from 'node:os'; import os from 'node:os';
import process from 'node:process'; import process from 'node:process';
import type { Span } from '../trace'; import type { Span } from '../trace';
import { availableParallelism } from 'foxts/available-parallelism';
const pool = new Worktank({ const pool = new Worktank({
pool: {
name: 'process-phishing-domains', name: 'process-phishing-domains',
size: Math.max( size: (availableParallelism(os) - 1) || 1
1, },
( worker: {
'availableParallelism' in os autoAbort: 10000, // The maximum number of milliseconds to wait for the result from the worker, if exceeded the worker is terminated and the execution promise rejects
? os.availableParallelism() autoTerminate: 30000, // The interval of milliseconds at which to check if the pool can be automatically terminated, to free up resources, workers will be spawned up again if needed
: (os as typeof import('node:os')).cpus().length
) - 1
),
timeout: 10000, // The maximum number of milliseconds to wait for the result from the worker, if exceeded the worker is terminated and the execution promise rejects
warmup: true,
autoterminate: 30000, // The interval of milliseconds at which to check if the pool can be automatically terminated, to free up resources, workers will be spawned up again if needed
env: {}, env: {},
methods: { methods: {
// eslint-disable-next-line object-shorthand -- workertank // eslint-disable-next-line object-shorthand -- workertank
@ -53,6 +49,7 @@ const pool = new Worktank({
fs.writeFileSync(filePath, fastStringArrayJoin(linesA, '\n') + '\n', { encoding: 'utf-8' }); fs.writeFileSync(filePath, fastStringArrayJoin(linesA, '\n') + '\n', { encoding: 'utf-8' });
} }
} }
}
}); });
export function compareAndWriteFileInWorker(span: Span, linesA: string[], filePath: string) { export function compareAndWriteFileInWorker(span: Span, linesA: string[], filePath: string) {

View File

@ -5,11 +5,14 @@ import type { Span } from '../trace';
import type { TldTsParsed } from './normalize-domain'; import type { TldTsParsed } from './normalize-domain';
const pool = new Worktank({ const pool = new Worktank({
pool: {
name: 'process-phishing-domains', name: 'process-phishing-domains',
size: 1, size: 1
timeout: 20000, // The maximum number of milliseconds to wait for the result from the worker, if exceeded the worker is terminated and the execution promise rejects },
warmup: true, worker: {
autoterminate: 30000, // The interval of milliseconds at which to check if the pool can be automatically terminated, to free up resources, workers will be spawned up again if needed autoAbort: 20000, // The maximum number of milliseconds to wait for the result from the worker, if exceeded the worker is terminated and the execution promise rejects
autoInstantiate: true,
autoTerminate: 30000, // The interval of milliseconds at which to check if the pool can be automatically terminated, to free up resources, workers will be spawned up again if needed
env: {}, env: {},
methods: { methods: {
// eslint-disable-next-line object-shorthand -- workertank // eslint-disable-next-line object-shorthand -- workertank
@ -191,6 +194,7 @@ const pool = new Worktank({
} }
} }
} }
}
}); });
export function getPhishingDomains(parentSpan: Span) { export function getPhishingDomains(parentSpan: Span) {

View File

@ -20,20 +20,20 @@
"author": "", "author": "",
"license": "ISC", "license": "ISC",
"dependencies": { "dependencies": {
"@ghostery/adblocker": "^2.11.2", "@ghostery/adblocker": "^2.11.3",
"@henrygd/queue": "^1.0.7", "@henrygd/queue": "^1.0.7",
"@mitata/counters": "^0.0.8", "@mitata/counters": "^0.0.8",
"async-retry": "^1.3.3", "async-retry": "^1.3.3",
"better-sqlite3": "^12.2.0", "better-sqlite3": "^12.2.0",
"ci-info": "^4.3.0", "ci-info": "^4.3.0",
"cli-progress": "^3.12.0", "cli-progress": "^3.12.0",
"csv-parse": "^5.6.0", "csv-parse": "^6.1.0",
"dns2": "^2.1.0", "dns2": "^2.1.0",
"fast-cidr-tools": "^0.3.2", "fast-cidr-tools": "^0.3.2",
"fast-fifo": "^1.3.2", "fast-fifo": "^1.3.2",
"fast-uri": "^3.0.6", "fast-uri": "^3.0.6",
"fdir": "^6.4.6", "fdir": "^6.4.6",
"foxts": "^3.9.0", "foxts": "^3.10.0",
"hash-wasm": "^4.12.0", "hash-wasm": "^4.12.0",
"json-stringify-pretty-compact": "3.0.0", "json-stringify-pretty-compact": "3.0.0",
"null-prototype-object": "^1.2.0", "null-prototype-object": "^1.2.0",
@ -46,7 +46,7 @@
"undici-cache-store-better-sqlite3": "^1.0.0", "undici-cache-store-better-sqlite3": "^1.0.0",
"whoiser": "^1.18.0", "whoiser": "^1.18.0",
"why-is-node-running": "^3.2.2", "why-is-node-running": "^3.2.2",
"worktank": "^2.7.3", "worktank": "^3.0.2",
"xbits": "^0.2.0", "xbits": "^0.2.0",
"yaml": "^2.8.0", "yaml": "^2.8.0",
"yauzl-promise": "^4.0.0" "yauzl-promise": "^4.0.0"
@ -54,21 +54,21 @@
"devDependencies": { "devDependencies": {
"@eslint-sukka/node": "^6.22.1", "@eslint-sukka/node": "^6.22.1",
"@swc-node/register": "^1.10.10", "@swc-node/register": "^1.10.10",
"@swc/core": "^1.13.1", "@swc/core": "^1.13.2",
"@types/async-retry": "^1.4.9", "@types/async-retry": "^1.4.9",
"@types/better-sqlite3": "^7.6.13", "@types/better-sqlite3": "^7.6.13",
"@types/cli-progress": "^3.11.6", "@types/cli-progress": "^3.11.6",
"@types/dns2": "^2.0.9", "@types/dns2": "^2.0.9",
"@types/fast-fifo": "^1.3.0", "@types/fast-fifo": "^1.3.0",
"@types/mocha": "^10.0.10", "@types/mocha": "^10.0.10",
"@types/node": "^24.0.15", "@types/node": "^24.1.0",
"@types/punycode": "^2.1.4", "@types/punycode": "^2.1.4",
"@types/tar-fs": "^2.0.4", "@types/tar-fs": "^2.0.4",
"@types/yauzl-promise": "^4.0.1", "@types/yauzl-promise": "^4.0.1",
"eslint": "^9.31.0", "eslint": "^9.32.0",
"eslint-config-sukka": "^6.22.1", "eslint-config-sukka": "^6.22.1",
"eslint-formatter-sukka": "^6.22.1", "eslint-formatter-sukka": "^6.22.1",
"expect": "^30.0.4", "expect": "^30.0.5",
"mitata": "^1.0.34", "mitata": "^1.0.34",
"mocha": "^11.7.1", "mocha": "^11.7.1",
"tinyexec": "^1.0.1", "tinyexec": "^1.0.1",

519
pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff