Perf: run entire reverse cn cidr in worker threads

Build/build-internal-reverse-chn-cidr.ts

@@ -1,14 +1,6 @@
-import path from 'node:path';
 import { task } from './trace';
 
 import { getChnCidrPromise } from './build-chn-cidr';
-// import { RESERVED_IPV4_CIDR, NON_CN_CIDR_INCLUDED_IN_CHNROUTE } from './constants/cidr';
-
-import fs from 'node:fs';
-import { OUTPUT_INTERNAL_DIR } from './constants/dir';
-import { asyncWriteToStream } from 'foxts/async-write-to-stream';
-import { mkdirp } from './lib/misc';
-// import { appendArrayInPlace } from './lib/append-array-in-place';
 import Worktank from 'worktank';
 
 const pool = new Worktank({
@@ -20,15 +12,24 @@ const pool = new Worktank({
   env: {},
   methods: {
     // eslint-disable-next-line object-shorthand -- workertank
-    getreversedCidr: async function (cidr: string[], importMetaUrl: string): Promise<string[]> {
+    getreversedCidr: async function (cidr: string[], importMetaUrl: string): Promise<void> {
       // TODO: createRequire is a temporary workaround for https://github.com/nodejs/node/issues/51956
       const { default: module } = await import('node:module');
       const __require = module.createRequire(importMetaUrl);
-      const { exclude, merge } = __require('fast-cidr-tools');
+      const path = __require('node:path') as typeof import('node:path');
-      const { RESERVED_IPV4_CIDR, NON_CN_CIDR_INCLUDED_IN_CHNROUTE } = __require('./constants/cidr');
+      const fs = __require('fs') as typeof import('fs');
-      const { appendArrayInPlace } = __require('./lib/append-array-in-place');
 
-      return merge(
+      const { OUTPUT_INTERNAL_DIR } = __require('./constants/dir') as typeof import('./constants/dir');
+      const { exclude, merge } = __require('fast-cidr-tools') as typeof import('fast-cidr-tools');
+      const { RESERVED_IPV4_CIDR, NON_CN_CIDR_INCLUDED_IN_CHNROUTE } = __require('./constants/cidr') as typeof import('./constants/cidr');
+      const { appendArrayInPlace } = __require('./lib/append-array-in-place') as typeof import('./lib/append-array-in-place');
+      const { fastStringArrayJoin } = __require('foxts/fast-string-array-join') as typeof import('foxts/fast-string-array-join');
+
+      const outputFile = path.join(OUTPUT_INTERNAL_DIR, 'reversed-chn-cidr.txt');
+
+      fs.mkdirSync(OUTPUT_INTERNAL_DIR, { recursive: true });
+
+      const result = merge(
         appendArrayInPlace(
           exclude(
             ['0.0.0.0/0'],
@@ -40,6 +41,8 @@ const pool = new Worktank({
         ),
         true
       );
+
+      fs.writeFileSync(outputFile, fastStringArrayJoin(result, '\n') + '\n', { encoding: 'utf-8' });
     }
   }
 });
@@ -47,28 +50,11 @@ const pool = new Worktank({
 export const buildInternalReverseChnCIDR = task(require.main === module, __filename)(async (span) => {
   const [cidr] = await span.traceChildPromise('download chnroutes2', getChnCidrPromise());
 
-  const reversedCidr = await span.traceChildAsync('build reversed chn cidr', async () => {
+  return span.traceChildAsync(
-    const reversedCidr = await pool.exec(
+    'build reversed chn cidr',
+    async () => pool.exec(
       'getreversedCidr',
       [cidr, import.meta.url]
-    );
+    ).finally(() => pool.terminate())
-    pool.terminate();
+  );
-
-    return reversedCidr;
-  });
-
-  const outputFile = path.join(OUTPUT_INTERNAL_DIR, 'reversed-chn-cidr.txt');
-  await mkdirp(OUTPUT_INTERNAL_DIR);
-
-  const writeStream = fs.createWriteStream(outputFile);
-  for (const line of reversedCidr) {
-    const p = asyncWriteToStream(writeStream, line + '\n');
-    if (p) {
-      // eslint-disable-next-line no-await-in-loop -- stream high water mark
-      await p;
-    }
-  }
-  await asyncWriteToStream(writeStream, '\n');
-
-  writeStream.end();
 });

Build/index.ts

@@ -61,6 +61,10 @@ const buildFinishedLock = path.join(ROOT_DIR, '.BUILD_FINISHED');
       .map((key) => `${key} x ${cpus[key]}`)
       .join('\n')
   }`);
+  if ('availableParallelism' in os) {
+    console.log(`Available parallelism: ${os.availableParallelism()}`);
+  }
+
   console.log(`Memory: ${os.totalmem() / (1024 * 1024)} MiB`);
 
   const rootSpan = createSpan('root');