Faster Telegram CIDR Builder

Build/build-telegram-cidr.js

@@ -1,8 +1,10 @@
 const { fetchWithRetry } = require('./lib/fetch-retry');
+const { createReadlineInterfaceFromResponse } = require('./lib/fetch-remote-text-by-line');
 const fs = require('fs');
 const path = require('path');
 const { isIPv4, isIPv6 } = require('net');
 const { withBanner } = require('./lib/with-banner');
+const { processLine } = require('./lib/process-line');
 
 (async () => {
   console.time('Total Time - build-telegram-cidr');
@@ -12,11 +14,23 @@ const { withBanner } = require('./lib/with-banner');
   const lastModified = resp.headers.get('last-modified');
   const date = lastModified ? new Date(lastModified) : new Date();
 
-  const res = (await resp.text())
+  /** @type {string[]} */
-    .split('\n')
+  const results = [];
-    .filter(line => line.trim() !== '');
 
-  if (res.length === 0) {
+  for await (const line of createReadlineInterfaceFromResponse(resp)) {
+    const cidr = processLine(line);
+    if (cidr) {
+      const [subnet] = cidr.split('/');
+      if (isIPv4(subnet)) {
+        results.push(`IP-CIDR,${cidr},no-resolve`);
+      }
+      if (isIPv6(subnet)) {
+        results.push(`IP-CIDR6,${cidr},no-resolve`);
+      }
+    }
+  }
+
+  if (results.length === 0) {
     throw new Error('Failed to fetch data!');
   }
 
@@ -32,17 +46,7 @@ const { withBanner } = require('./lib/with-banner');
         ' - https://core.telegram.org/resources/cidr.txt'
       ],
       date,
-      res.map(ip => {
+      results
-        const [subnet] = ip.split('/');
-        console.log(`  - ${ip}: ${subnet}`);
-        if (isIPv4(subnet)) {
-          return `IP-CIDR,${ip},no-resolve`;
-        }
-        if (isIPv6(subnet)) {
-          return `IP-CIDR6,${ip},no-resolve`;
-        }
-        return '';
-      })
     )
   );
 

Build/lib/fetch-remote-text-by-line.js

@@ -3,18 +3,29 @@ const { fetchWithRetry } = require('./fetch-retry');
 const readline = require('readline');
 const { Readable } = require('stream');
 
+/**
+ * @param {import('undici').Response} resp
+ */
+const createReadlineInterfaceFromResponse = (resp) => {
+  if (!resp.body) {
+    throw new Error('Failed to fetch remote text');
+  }
+  if (resp.bodyUsed) {
+    throw new Error('Body has already been consumed.');
+  }
+  return readline.createInterface({
+    input: Readable.fromWeb(resp.body),
+    crlfDelay: Infinity
+  });
+};
+
+module.exports.createReadlineInterfaceFromResponse = createReadlineInterfaceFromResponse;
+
 /**
  * @param {import('undici').RequestInfo} url
  * @param {import('undici').RequestInit | undefined} [opt]
  */
 module.exports.fetchRemoteTextAndCreateReadlineInterface = async (url, opt) => {
   const resp = await fetchWithRetry(url, opt);
-  if (!resp.body) {
+  return createReadlineInterfaceFromResponse(resp);
-    throw new Error('Failed to fetch remote text');
-  }
-
-  return readline.createInterface({
-    input: Readable.fromWeb(resp.body),
-    crlfDelay: Infinity
-  });
 };