Chore: read request stream line by line

Build/build-anti-bogus-domain.js

@@ -1,23 +1,24 @@
-const { fetchWithRetry } = require('./lib/fetch-retry');
+// @ts-check
 const fs = require('fs');
 const path = require('path');
 const { isIPv4, isIPv6 } = require('net');
 const { compareAndWriteFile } = require('./lib/string-array-compare');
 const { withBannerArray } = require('./lib/with-banner');
+const { fetchRemoteTextAndCreateReadlineInterface } = require('./lib/fetch-remote-text-by-line');
 
 (async () => {
   console.time('Total Time - build-anti-bogus-domain');
-  console.time('* Download bogus-nxdomain-list')
+  console.time('* Download bogus-nxdomain-list');
-  const res = (await (await fetchWithRetry('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/bogus-nxdomain.china.conf')).text())
-    .split('\n')
-    .map(line => {
-      if (line.startsWith('bogus-nxdomain=')) {
-        return line.replace('bogus-nxdomain=', '');
-      }
 
-      return null
+  const rl = await fetchRemoteTextAndCreateReadlineInterface('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/bogus-nxdomain.china.conf');
-    })
+
-    .filter(ip => typeof ip === 'string');
+  /** @type {string[]} */
+  const res = [];
+  for await (const line of rl) {
+    if (line.startsWith('bogus-nxdomain=')) {
+      res.push(line.replace('bogus-nxdomain=', ''));
+    }
+  }
 
   console.timeEnd('* Download bogus-nxdomain-list')
 

Build/build-apple-cdn.js

@@ -1,24 +1,26 @@
-const { fetchWithRetry } = require('./lib/fetch-retry');
-const fs = require('fs');
 const path = require('path');
 
 const { isDomainLoose } = require('./lib/is-domain-loose');
 const { compareAndWriteFile } = require('./lib/string-array-compare');
 const { withBannerArray } = require('./lib/with-banner');
 
+const { fetchRemoteTextAndCreateReadlineInterface } = require('./lib/fetch-remote-text-by-line');
+
 (async () => {
   console.time('Total Time - build-apple-cdn-conf');
 
-  const res = (await (await fetchWithRetry('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf')).text())
+  const rl = await fetchRemoteTextAndCreateReadlineInterface('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf');
-    .split('\n')
-    .map(line => {
-      if (line.startsWith('server=/') && line.endsWith('/114.114.114.114')) {
-        return line.replace('server=/', '').replace('/114.114.114.114', '');
-      }
 
-      return null
+  /** @type {string[]} */
-    })
+  const res = [];
-    .filter(domain => typeof domain === 'string' && isDomainLoose(domain));
+  for await (const line of rl) {
+    if (line.startsWith('server=/') && line.endsWith('/114.114.114.114')) {
+      const domain = line.replace('server=/', '').replace('/114.114.114.114', '');
+      if (isDomainLoose(domain)) {
+        res.push(domain);
+      }
+    }
+  }
 
   await Promise.all([
     compareAndWriteFile(

Build/lib/fetch-remote-text-by-line.js

@@ -0,0 +1,20 @@
+// @ts-check
+const { fetchWithRetry } = require('./fetch-retry');
+const readline = require('readline');
+const { Readable } = require('stream');
+
+/**
+ * @param {import('undici').RequestInfo} url
+ * @param {import('undici').RequestInit | undefined} [opt]
+ */
+module.exports.fetchRemoteTextAndCreateReadlineInterface = async (url, opt) => {
+  const resp = await fetchWithRetry(url, opt);
+  if (!resp.body) {
+    throw new Error('Failed to fetch remote text');
+  }
+
+  return readline.createInterface({
+    input: Readable.fromWeb(resp.body),
+    crlfDelay: Infinity
+  });
+}

Build/lib/fetch-retry.js

@@ -1,4 +1,4 @@
 // @ts-check
 const { fetch } = require('undici');
-const fetchWithRetry = require('@vercel/fetch-retry')(fetch);
+const fetchWithRetry = /** @type {fetch} */(require('@vercel/fetch-retry')(fetch));
 module.exports.fetchWithRetry = fetchWithRetry;

Build/lib/parse-filter.js

@@ -1,5 +1,6 @@
 // @ts-check
 const { fetchWithRetry } = require('./fetch-retry');
+const { fetchRemoteTextAndCreateReadlineInterface } = require('./fetch-remote-text-by-line');
 const { NetworkFilter } = require('@cliqz/adblocker');
 const { normalizeDomain } = require('./is-domain-loose');
 
@@ -26,9 +27,10 @@ async function processDomainLists(domainListsUrl) {
 
   /** @type Set<string> */
   const domainSets = new Set();
-  /** @type string[] */
+
-  const domains = (await (await fetchWithRetry(domainListsUrl)).text()).split('\n');
+  const rl = await fetchRemoteTextAndCreateReadlineInterface(domainListsUrl);
-  domains.forEach(line => {
+
+  for await (const line of rl) {
     if (
       line.startsWith('#')
       || line.startsWith('!')
@@ -48,7 +50,7 @@ async function processDomainLists(domainListsUrl) {
     }
 
     domainSets.add(domainToAdd);
-  });
+  }
 
   return [...domainSets];
 }
@@ -66,9 +68,8 @@ async function processHosts(hostsUrl, includeAllSubDomain = false) {
   /** @type Set<string> */
   const domainSets = new Set();
 
-  /** @type string[] */
+  const rl = await fetchRemoteTextAndCreateReadlineInterface(hostsUrl);
-  const hosts = (await (await fetchWithRetry(hostsUrl)).text()).split('\n');
+  for await (const line of rl) {
-  hosts.forEach(line => {
     if (line.includes('#')) {
       return;
     }
@@ -91,7 +92,7 @@ async function processHosts(hostsUrl, includeAllSubDomain = false) {
         domainSets.add(domain);
       }
     }
-  });
+  }
 
   console.timeEnd(`   - processHosts: ${hostsUrl}`);
 

Build/lib/reject-data-source.js

@@ -8,7 +8,7 @@ const HOSTS = [
   ['https://raw.githubusercontent.com/jdlingyu/ad-wars/master/hosts', false]
 ]
 
-const ADGUARD_FILTERS = [
+const ADGUARD_FILTERS = /** @type {const} */([
   // Easy List
   [
     'https://easylist.to/easylist/easylist.txt',
@@ -177,7 +177,7 @@ const ADGUARD_FILTERS = [
   'https://raw.githubusercontent.com/brave/adblock-lists/master/brave-lists/brave-firstparty.txt',
   'https://raw.githubusercontent.com/brave/adblock-lists/master/brave-lists/brave-firstparty-cname.txt',
   'https://raw.githubusercontent.com/brave/adblock-lists/master/coin-miners.txt'
-];
+]);
 
 const PREDEFINED_WHITELIST = [
   'localhost',