Chore: async write stream & mkdirp

Build/build-internal-reverse-chn-cidr.ts

@@ -5,8 +5,10 @@ import { exclude, merge } from 'fast-cidr-tools';
 import { getChnCidrPromise } from './build-chn-cidr';
 import { NON_CN_CIDR_INCLUDED_IN_CHNROUTE, RESERVED_IPV4_CIDR } from './constants/cidr';
 
-import { writeFile } from './lib/misc';
+import fs from 'node:fs';
 import { OUTPUT_INTERNAL_DIR } from './constants/dir';
+import { asyncWriteToStream } from './lib/async-write-to-stream';
+import { mkdirp } from './lib/misc';
 
 export const buildInternalReverseChnCIDR = task(require.main === module, __filename)(async () => {
   const [cidr] = await getChnCidrPromise();
@@ -23,9 +25,17 @@ export const buildInternalReverseChnCIDR = task(require.main === module, __filen
   );
 
   const outputFile = path.join(OUTPUT_INTERNAL_DIR, 'reversed-chn-cidr.txt');
+  await mkdirp(OUTPUT_INTERNAL_DIR);
 
-  return writeFile(
-    outputFile,
-    reversedCidr.join('\n') + '\n'
-  );
+  const writeStream = fs.createWriteStream(outputFile);
+  for (const line of reversedCidr) {
+    const p = asyncWriteToStream(writeStream, line + '\n');
+    if (p) {
+      // eslint-disable-next-line no-await-in-loop -- stream high water mark
+      await p;
+    }
+  }
+  await asyncWriteToStream(writeStream, '\n');
+
+  writeStream.end();
 });

Build/build-public.ts

@@ -7,7 +7,7 @@ import { treeDir } from './lib/tree-dir';
 import type { TreeType, TreeTypeArray } from './lib/tree-dir';
 
 import { OUTPUT_MOCK_DIR, OUTPUT_MODULES_DIR, PUBLIC_DIR, ROOT_DIR } from './constants/dir';
-import { writeFile } from './lib/misc';
+import { mkdirp, writeFile } from './lib/misc';
 import picocolors from 'picocolors';
 
 const mockDir = path.join(ROOT_DIR, 'Mock');
@@ -32,8 +32,8 @@ const copyDirContents = async (srcDir: string, destDir: string) => {
 export const buildPublic = task(require.main === module, __filename)(async (span) => {
   await span.traceChildAsync('copy rest of the files', async () => {
     await Promise.all([
-      fsp.mkdir(OUTPUT_MODULES_DIR, { recursive: true }),
-      fsp.mkdir(OUTPUT_MOCK_DIR, { recursive: true })
+      mkdirp(OUTPUT_MODULES_DIR),
+      mkdirp(OUTPUT_MOCK_DIR)
     ]);
 
     await Promise.all([

Build/download-mock-assets.ts

@@ -1,11 +1,11 @@
 import { task } from './trace';
 import path from 'node:path';
 import fs from 'node:fs';
-import fsp from 'node:fs/promises';
 import { Readable } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
 import { fetchWithRetry } from './lib/fetch-retry';
 import { OUTPUT_MOCK_DIR } from './constants/dir';
+import { mkdirp } from './lib/misc';
 
 const ASSETS_LIST = {
   'www-google-analytics-com_ga.js': 'https://raw.githubusercontent.com/AdguardTeam/Scriptlets/master/dist/redirect-files/google-analytics-ga.js',
@@ -25,7 +25,8 @@ export const downloadMockAssets = task(require.main === module, __filename)((spa
         throw new Error(`Empty body from ${url}`);
       }
 
-      await fsp.mkdir(OUTPUT_MOCK_DIR, { recursive: true });
+      await mkdirp(OUTPUT_MOCK_DIR);
+
       return pipeline(
         Readable.fromWeb(res.body),
         fs.createWriteStream(src, 'utf-8')

Build/lib/async-write-to-stream.ts

@@ -0,0 +1,9 @@
+import type { Writable } from 'node:stream';
+import { once } from 'node:events';
+
+export const asyncWriteToStream = <T>(stream: Writable, chunk: T) => {
+  const res = stream.write(chunk);
+  if (!res) {
+    return once(stream, 'drain'); // returns a promise only if needed
+  }
+};

Build/lib/convert-clash-meta-mrs.ts

@@ -7,6 +7,7 @@ import zlib from 'node:zlib';
 import process from 'node:process';
 
 import { async as ezspawn } from '@jsdevtools/ez-spawn';
+import { mkdirp } from './misc';
 
 const mihomoBinaryDir = path.join(__dirname, '../../node_modules/.cache/mihomo');
 const mihomoBinaryPath = path.join(mihomoBinaryDir, 'mihomo');
@@ -22,7 +23,7 @@ const mihomoBinaryUrl: Partial<Record<NodeJS.Platform, Partial<Record<NodeJS.Arc
 };
 
 const ensureMihomoBinary = async () => {
-  await fsp.mkdir(mihomoBinaryDir, { recursive: true });
+  await mkdirp(mihomoBinaryDir);
   if (!fs.existsSync(mihomoBinaryPath)) {
     const writeStream = fs.createWriteStream(mihomoBinaryPath);
 

Build/lib/create-file.ts

@@ -10,6 +10,7 @@ import stringify from 'json-stringify-pretty-compact';
 import { ipCidrListToSingbox, surgeDomainsetToSingbox, surgeRulesetToSingbox } from './singbox';
 import { createTrie } from './trie';
 import { pack, unpackFirst, unpackSecond } from './bitwise';
+import { asyncWriteToStream } from './async-write-to-stream';
 
 export async function compareAndWriteFile(span: Span, linesA: string[], filePath: string) {
   let isEqual = true;
@@ -67,17 +68,24 @@ export async function compareAndWriteFile(span: Span, linesA: string[], filePath
   }
 
   await span.traceChildAsync(`writing ${filePath}`, async () => {
-    // if (linesALen < 10000) {
-    return writeFile(filePath, fastStringArrayJoin(linesA, '\n') + '\n');
-    // }
-    // const writer = file.writer();
+    // The default highwater mark is normally 16384,
+    // So we make sure direct write to file if the content is
+    // most likely less than 500 lines
+    if (linesALen < 500) {
+      return writeFile(filePath, fastStringArrayJoin(linesA, '\n') + '\n');
+    }
 
-    // for (let i = 0; i < linesALen; i++) {
-    //   writer.write(linesA[i]);
-    //   writer.write('\n');
-    // }
-
-    // return writer.end();
+    const writeStream = fs.createWriteStream(filePath);
+    for (let i = 0; i < linesALen; i++) {
+      let p = asyncWriteToStream(writeStream, linesA[i]);
+      // eslint-disable-next-line no-await-in-loop -- stream high water mark
+      if (p) await p;
+      p = asyncWriteToStream(writeStream, '\n');
+      // eslint-disable-next-line no-await-in-loop -- stream high water mark
+      if (p) await p;
+    }
+    await asyncWriteToStream(writeStream, '\n');
+    writeStream.end();
   });
 }
 

Build/lib/misc.ts

@@ -23,9 +23,17 @@ interface Write {
   ): Promise<unknown>
 }
 
+export const mkdirp = (dir: string) => {
+  if (fs.existsSync(dir)) {
+    return;
+  }
+  return fsp.mkdir(dir, { recursive: true });
+};
+
 export const writeFile: Write = async (destination: string, input, dir = dirname(destination)) => {
-  if (!fs.existsSync(dir)) {
-    await fsp.mkdir(dir, { recursive: true });
+  const p = mkdirp(dir);
+  if (p) {
+    await p;
   }
   return fsp.writeFile(destination, input, { encoding: 'utf-8' });
 };