Use stream in build tools

Build/build-anti-bogus-domain.js

@@ -1,10 +1,9 @@
 // @ts-check
-const fs = require('fs');
 const path = require('path');
 const { isIPv4, isIPv6 } = require('net');
 const { compareAndWriteFile } = require('./lib/string-array-compare');
 const { withBannerArray } = require('./lib/with-banner');
-const { fetchRemoteTextAndCreateReadlineInterface } = require('./lib/fetch-remote-text-by-line');
+const { fetchRemoteTextAndCreateReadlineInterface, readFileByLine } = require('./lib/fetch-remote-text-by-line');
 const { minifyRules } = require('./lib/minify-rules');
 
 (async () => {
@@ -25,19 +24,22 @@ const { minifyRules } = require('./lib/minify-rules');
 
   const filePath = path.resolve(__dirname, '../Source/ip/reject.conf');
   const resultPath = path.resolve(__dirname, '../List/ip/reject.conf');
-  const content = (await fs.promises.readFile(filePath, 'utf-8'))
+
-    .replace(
+  /** @type {string[]} */
-      '# --- [Anti Bogus Domain Replace Me] ---',
+  const result = [];
-      res.map(ip => {
+  for await (const line of readFileByLine(filePath)) {
+    if (line === '# --- [Anti Bogus Domain Replace Me] ---') {
+      res.forEach(ip => {
         if (isIPv4(ip)) {
-          return `IP-CIDR,${ip}/32,no-resolve`;
+          result.push(`IP-CIDR,${ip}/32,no-resolve`);
+        } else if (isIPv6(ip)) {
+          result.push(`IP-CIDR6,${ip}/128,no-resolve`);
         }
-        if (isIPv6(ip)) {
+      });
-          return `IP-CIDR6,${ip}/128,no-resolve`;
+    } else {
-        }
+      result.push(line);
-        return '';
+    }
-      }).join('\n')
+  }
-    );
 
   await compareAndWriteFile(
     withBannerArray(
@@ -53,7 +55,7 @@ const { minifyRules } = require('./lib/minify-rules');
         ' - https://github.com/felixonmars/dnsmasq-china-list'
       ],
       new Date(),
-      minifyRules(content.split('\n'))
+      minifyRules(result)
     ),
     resultPath
   );

Build/build-cdn-conf.js

@@ -1,5 +1,4 @@
 // @ts-check
-const fs = require('fs');
 const path = require('path');
 const { compareAndWriteFile } = require('./lib/string-array-compare');
 const { withBannerArray } = require('./lib/with-banner');
@@ -34,11 +33,15 @@ const { fetchRemoteTextAndCreateReadlineInterface, readFileByLine } = require('.
     }
   }
 
-  const content = (await fs.promises.readFile(path.resolve(__dirname, '../Source/non_ip/cdn.conf'), 'utf-8'))
+  /** @type {string[]} */
-    .replace(
+  const cdnDomainsList = [];
-      '# --- [AWS S3 Replace Me] ---',
+  for await (const line of readFileByLine(path.resolve(__dirname, '../Source/non_ip/cdn.conf'))) {
-      Array.from(S3OSSDomains).map(domain => `DOMAIN-SUFFIX,${domain}`).join('\n')
+    if (line === '# --- [AWS S3 Replace Me] ---') {
-    );
+      S3OSSDomains.forEach(domain => cdnDomainsList.push(`DOMAIN-SUFFIX,${domain}`));
+    } else {
+      cdnDomainsList.push(line);
+    }
+  }
 
   await compareAndWriteFile(
     withBannerArray(
@@ -51,7 +54,7 @@ const { fetchRemoteTextAndCreateReadlineInterface, readFileByLine } = require('.
         'This file contains object storage and static assets CDN domains.'
       ],
       new Date(),
-      minifyRules(content.split('\n'))
+      minifyRules(cdnDomainsList)
     ),
     path.resolve(__dirname, '../List/non_ip/cdn.conf')
   );

Build/build-reject-domainset.js

@@ -209,7 +209,7 @@ const domainSuffixSet = new Set();
   };
   const sortedDomainSets = dudupedDominArray
     .map((v) => {
-      const domain = getDomain(v.charCodeAt(0) === 46 ? v.slice(1) : v) || v;
+      const domain = getDomain(v[0] === '.' ? v.slice(1) : v) || v;
       rejectDomainsStats[domain] = (rejectDomainsStats[domain] || 0) + 1;
       return { v, domain };
     })

Build/lib/string-array-compare.js

@@ -1,6 +1,7 @@
 // @ts-check
 const { promises: fsPromises } = require('fs');
 const fse = require('fs-extra');
+const { readFileByLine } = require('./fetch-remote-text-by-line');
 
 /**
  * @param {string[]} linesA
@@ -8,9 +9,26 @@ const fse = require('fs-extra');
  */
 async function compareAndWriteFile(linesA, filePath) {
   await fse.ensureFile(filePath);
-  const linesB = (await fsPromises.readFile(filePath, { encoding: 'utf-8' })).split('\n');
 
-  if (!stringArrayCompare(linesA, linesB)) {
+  const rl = readFileByLine(filePath);
+
+  let isEqual = true;
+  let index = 0;
+
+  for await (const lineB of rl) {
+    const lineA = linesA[index];
+    index++;
+
+    if (lineA[0] === '#' && lineB[0] === '#') {
+      continue;
+    }
+    if (lineA !== lineB) {
+      isEqual = false;
+      break;
+    }
+  }
+
+  if (!isEqual) {
     await fsPromises.writeFile(
       filePath,
       linesA.join('\n'),
@@ -21,26 +39,4 @@ async function compareAndWriteFile(linesA, filePath) {
   }
 }
 
-/**
- * @param {string[]} linesA
- * @param {string[]} linesB
- */
-function stringArrayCompare(linesA, linesB) {
-  if (linesA.length !== linesB.length) return false;
-
-  for (let i = 0; i < linesA.length; i++) {
-    const lineA = linesA[i];
-    const lineB = linesB[i];
-    if (lineA.startsWith('#') && lineB.startsWith('#')) {
-      continue;
-    }
-    if (lineA !== lineB) {
-      return false;
-    }
-  }
-
-  return true;
-}
-
-module.exports.stringArrayCompare = stringArrayCompare;
 module.exports.compareAndWriteFile = compareAndWriteFile;

Build/lib/with-banner.js

@@ -7,14 +7,15 @@
  * @param {string[]} content
  * @returns {string}
  */
-const withBanner = (title, description, date, content) => {
+// const withBanner = (title, description, date, content) => {
-  return `########################################
+//   return `########################################
-# ${title}
+// # ${title}
-# Last Updated: ${date.toISOString()}
+// # Last Updated: ${date.toISOString()}
-# Size: ${content.length}
+// # Size: ${content.length}
-${description.map(line => (line ? `# ${line}` : '#')).join('\n')}
+// ${description.map(line => (line ? `# ${line}` : '#')).join('\n')}
-########################################\n${content.join('\n')}\n################# END ###################\n`;
+// ########################################\n${content.join('\n')}\n################# END ###################\n`;
-};
+// };
+
 /**
  * @param {string} title
  * @param {string[]} description
@@ -36,5 +37,5 @@ const withBannerArray = (title, description, date, content) => {
   ];
 };
 
-module.exports.withBanner = withBanner;
+// module.exports.withBanner = withBanner;
 module.exports.withBannerArray = withBannerArray;

Build/validate-domainset.js

@@ -2,9 +2,10 @@
 
 const tldts = require('tldts');
 const picocolors = require('picocolors');
-const fs = require('fs');
 const path = require('path');
 const listDir = require('@sukka/listdir');
+const { readFileByLine } = require('./lib/fetch-remote-text-by-line');
+const { processLine } = require('./lib/process-line');
 
 const SPECIAL_SUFFIXES = new Set([
   'linodeobjects.com', // only *.linodeobjects.com are public suffix
@@ -13,21 +14,16 @@ const SPECIAL_SUFFIXES = new Set([
 ]);
 
 const validateDomainSet = async (filePath) => {
-  const domainSetContent = await fs.promises.readFile(
+  const rl = readFileByLine(
-    path.resolve(__dirname, '../List/domainset', filePath),
+    path.resolve(__dirname, '../List/domainset', filePath)
-    { encoding: 'utf-8' }
   );
-  const domainSetLines = domainSetContent.split('\n');
-  for (let i = 0, len = domainSetLines.length; i < len; i++) {
-    const line = domainSetLines[i];
-    // starts with #
-    if (line.charCodeAt(0) === 35) {
-      continue;
-    }
-    if (line.trim().length === 0) {
-      continue;
-    }
 
+  for await (const l of rl) {
+    // starts with #
+    const line = processLine(l);
+    if (!line) {
+      continue;
+    }
     const domain = line.charCodeAt(0) === 46 ? line.slice(1) : line;
     const parsed = tldts.parse(domain, { allowPrivateDomains: true, detectIp: false });
 
@@ -43,21 +39,16 @@ const validateDomainSet = async (filePath) => {
 };
 
 const validateRuleset = async (filePath) => {
-  const rulesetContent = await fs.promises.readFile(
+  const rl = readFileByLine(
-    path.resolve(__dirname, '../List/non_ip', filePath),
+    path.resolve(__dirname, '../List/non_ip', filePath)
-    { encoding: 'utf-8' }
   );
-  const rulesetLines = rulesetContent.split('\n');
 
   console.log(`[${filePath}]`);
 
-  for (let i = 0, len = rulesetLines.length; i < len; i++) {
+  for await (const l of rl) {
-    const line = rulesetLines[i];
     // starts with #
-    if (line.charCodeAt(0) === 35) {
+    const line = processLine(l);
-      continue;
+    if (!line) {
-    }
-    if (line.trim().length === 0) {
       continue;
     }
     if (!line.startsWith('DOMAIN-SUFFIX,')) {