From 53dcc8df25e6b9bbc5d2f7cb64be85676d16ede1 Mon Sep 17 00:00:00 2001
From: SukkaW <isukkaw@gmail.com>
Date: Wed, 26 Jul 2023 20:57:42 +0800
Subject: [PATCH] Faster Telegram CIDR Builder

---
 Build/build-telegram-cidr.js           | 34 ++++++++++++++------------
 Build/lib/fetch-remote-text-by-line.js | 27 ++++++++++++++------
 2 files changed, 38 insertions(+), 23 deletions(-)

diff --git a/Build/build-telegram-cidr.js b/Build/build-telegram-cidr.js
index 734925e5..d66a5fb9 100644
--- a/Build/build-telegram-cidr.js
+++ b/Build/build-telegram-cidr.js
@@ -1,8 +1,10 @@
 const { fetchWithRetry } = require('./lib/fetch-retry');
+const { createReadlineInterfaceFromResponse } = require('./lib/fetch-remote-text-by-line');
 const fs = require('fs');
 const path = require('path');
 const { isIPv4, isIPv6 } = require('net');
 const { withBanner } = require('./lib/with-banner');
+const { processLine } = require('./lib/process-line');
 
 (async () => {
   console.time('Total Time - build-telegram-cidr');
@@ -12,11 +14,23 @@ const { withBanner } = require('./lib/with-banner');
   const lastModified = resp.headers.get('last-modified');
   const date = lastModified ? new Date(lastModified) : new Date();
 
-  const res = (await resp.text())
-    .split('\n')
-    .filter(line => line.trim() !== '');
+  /** @type {string[]} */
+  const results = [];
 
-  if (res.length === 0) {
+  for await (const line of createReadlineInterfaceFromResponse(resp)) {
+    const cidr = processLine(line);
+    if (cidr) {
+      const [subnet] = cidr.split('/');
+      if (isIPv4(subnet)) {
+        results.push(`IP-CIDR,${cidr},no-resolve`);
+      }
+      if (isIPv6(subnet)) {
+        results.push(`IP-CIDR6,${cidr},no-resolve`);
+      }
+    }
+  }
+
+  if (results.length === 0) {
     throw new Error('Failed to fetch data!');
   }
 
@@ -32,17 +46,7 @@ const { withBanner } = require('./lib/with-banner');
         ' - https://core.telegram.org/resources/cidr.txt'
       ],
       date,
-      res.map(ip => {
-        const [subnet] = ip.split('/');
-        console.log(`  - ${ip}: ${subnet}`);
-        if (isIPv4(subnet)) {
-          return `IP-CIDR,${ip},no-resolve`;
-        }
-        if (isIPv6(subnet)) {
-          return `IP-CIDR6,${ip},no-resolve`;
-        }
-        return '';
-      })
+      results
     )
   );
 
diff --git a/Build/lib/fetch-remote-text-by-line.js b/Build/lib/fetch-remote-text-by-line.js
index 2886d298..7504c378 100644
--- a/Build/lib/fetch-remote-text-by-line.js
+++ b/Build/lib/fetch-remote-text-by-line.js
@@ -3,18 +3,29 @@ const { fetchWithRetry } = require('./fetch-retry');
 const readline = require('readline');
 const { Readable } = require('stream');
 
+/**
+ * @param {import('undici').Response} resp
+ */
+const createReadlineInterfaceFromResponse = (resp) => {
+  if (!resp.body) {
+    throw new Error('Failed to fetch remote text');
+  }
+  if (resp.bodyUsed) {
+    throw new Error('Body has already been consumed.');
+  }
+  return readline.createInterface({
+    input: Readable.fromWeb(resp.body),
+    crlfDelay: Infinity
+  });
+};
+
+module.exports.createReadlineInterfaceFromResponse = createReadlineInterfaceFromResponse;
+
 /**
  * @param {import('undici').RequestInfo} url
  * @param {import('undici').RequestInit | undefined} [opt]
  */
 module.exports.fetchRemoteTextAndCreateReadlineInterface = async (url, opt) => {
   const resp = await fetchWithRetry(url, opt);
-  if (!resp.body) {
-    throw new Error('Failed to fetch remote text');
-  }
-
-  return readline.createInterface({
-    input: Readable.fromWeb(resp.body),
-    crlfDelay: Infinity
-  });
+  return createReadlineInterfaceFromResponse(resp);
 };