Perf: speed up build

Build/lib/cached-tld-parse.js

@@ -9,9 +9,7 @@ const sharedConfig = { allowPrivateDomains: true };
  * @param {string} domain
  * @returns {ReturnType<import('tldts').parse>}
  */
-module.exports.parse = (domain) => {
-  return cache.sync(domain, () => tldts.parse(domain, sharedConfig));
-};
+module.exports.parse = (domain) => cache.sync(domain, () => tldts.parse(domain, sharedConfig));
 
 let gothillGetDomainCache = null;
 /**
@@ -22,5 +20,5 @@ module.exports.createCachedGorhillGetDomain = (gorhill) => {
   /**
    * @param {string} domain
    */
-  return (domain) => gothillGetDomainCache.sync(domain, () => gorhill.getDomain(domain[0] === '.' ? domain.slice(1) : domain));
+  return (domain) => (/** @type {ReturnType<typeof createCache>} */ (gothillGetDomainCache)).sync(domain, () => gorhill.getDomain(domain[0] === '.' ? domain.slice(1) : domain));
 };

Build/lib/get-gorhill-publicsuffix.js

@@ -3,13 +3,6 @@ const fs = require('fs');
 const path = require('path');
 
 const publicSuffixPath = path.resolve(__dirname, '../../node_modules/.cache/public_suffix_list_dat.txt');
-const getPublicSuffixListDat = () => {
-  if (fs.existsSync(publicSuffixPath)) {
-    return fs.promises.readFile(publicSuffixPath, 'utf-8');
-  }
-  console.log('public_suffix_list.dat not found, fetch directly from remote.');
-  return fetch('https://publicsuffix.org/list/public_suffix_list.dat').then(r => r.text());
-};
 
 const getGorhillPublicSuffix = async () => {
   const customFetch = async (url) => {
@@ -20,7 +13,12 @@ const getGorhillPublicSuffix = async () => {
   };
 
   const [publicSuffixListDat, { default: gorhill }] = await Promise.all([
-    getPublicSuffixListDat(),
+    fs.existsSync(publicSuffixPath)
+      ? fs.promises.readFile(publicSuffixPath, 'utf-8')
+      : fetch('https://publicsuffix.org/list/public_suffix_list.dat').then(r => {
+        console.log('public_suffix_list.dat not found, fetch directly from remote.');
+        return r.text();
+      }),
     import('gorhill-publicsuffixlist')
   ]);
 
@@ -30,7 +28,7 @@ const getGorhillPublicSuffix = async () => {
   return gorhill;
 };
 
-/** @type {Promise<import('gorhill-publicsuffixlist').default | null>} */
+/** @type {Promise<import('gorhill-publicsuffixlist').default> | null} */
 let gorhillPublicSuffixPromise = null;
 module.exports.getGorhillPublicSuffixPromise = () => {
   gorhillPublicSuffixPromise ||= getGorhillPublicSuffix();

Build/lib/is-domain-loose.js

@@ -1,24 +0,0 @@
-// @ts-check
-const tldts = require('./cached-tld-parse');
-/**
- * @param {string | null | undefined} domain
- */
-module.exports.normalizeDomain = (domain) => {
-  if (!domain) {
-    return null;
-  }
-
-  const { isIcann, isPrivate, hostname, isIp } = tldts.parse(domain);
-  if (isIp) {
-    return null;
-  }
-
-  if (isIcann || isPrivate) {
-    if (hostname?.[0] === '.') {
-      return hostname.slice(1);
-    }
-    return hostname;
-  }
-
-  return null;
-};

Build/lib/parse-filter.js

@@ -1,8 +1,8 @@
 // @ts-check
 const { fetchWithRetry } = require('./fetch-retry');
+const tldts = require('tldts');
 const { fetchRemoteTextAndCreateReadlineInterface } = require('./fetch-remote-text-by-line');
 const { NetworkFilter } = require('@cliqz/adblocker');
-const { normalizeDomain } = require('./is-domain-loose');
 const { processLine } = require('./process-line');
 const { performance } = require('perf_hooks');
 
@@ -19,6 +19,22 @@ const warnOnce = (url, isWhite, ...message) => {
   console.warn(url, isWhite ? '(white)' : '(black)', ...message);
 };
 
+const normalizeDomain = (domain) => {
+  if (!domain) return null;
+
+  const { isIcann, isPrivate, hostname, isIp } = tldts.parse(domain);
+  if (isIp) return null;
+
+  if (isIcann || isPrivate) {
+    if (hostname?.[0] === '.') {
+      return hostname.slice(1);
+    }
+    return hostname;
+  }
+
+  return null;
+};
+
 /**
  * @param {string | URL} domainListsUrl
  */

Build/lib/process-line.js

@@ -6,7 +6,7 @@
  *
  * @param {string} line
  */
-module.exports.processLine = (line) => {
+const processLine = (line) => {
   if (!line) {
     return null;
   }
@@ -30,3 +30,19 @@ module.exports.processLine = (line) => {
 
   return trimmed;
 };
+module.exports.processLine = processLine;
+
+/**
+ * @param {import('readline').ReadLine} rl
+ */
+module.exports.processLineFromReadline = async (rl) => {
+  /** @type {string[]} */
+  const res = [];
+  for await (const line of rl) {
+    const l = processLine(line);
+    if (l) {
+      res.push(l);
+    }
+  }
+  return res;
+};

Build/lib/trace-runner.js

@@ -40,8 +40,13 @@ module.exports.traceAsync = traceAsync;
  */
 module.exports.task = (__filename, fn, customname = null) => {
   const taskName = customname ?? path.basename(__filename, path.extname(__filename));
-  return () => {
+  return async () => {
     console.log(`🏃 [${taskName}] Start executing`);
-    return traceAsync(`✅ [${taskName}] Executed successfully`, fn);
+    const start = performance.now();
+    await fn();
+    const end = performance.now();
+    console.log(`✅ [${taskName}] Executed successfully: ${(end - start).toFixed(3)}ms`);
+
+    return { start, end, taskName };
   };
 };