Perf: run cpu intensive task with worker

Build/build-reject-domainset.js

@@ -33,7 +33,7 @@ const buildRejectDomainSet = task(__dirname, async () => {
   let shouldStop = false;
 
   const [gorhill] = await Promise.all([
-    getGorhillPublicSuffixPromise,
+    getGorhillPublicSuffixPromise(),
     // Parse from remote hosts & domain lists
     ...HOSTS.map(entry => processHosts(entry[0], entry[1]).then(hosts => {
       hosts.forEach(host => {

Build/index.js

@@ -9,14 +9,54 @@ const { buildTelegramCIDR } = require('./build-telegram-cidr');
 const { buildChnCidr } = require('./build-chn-cidr');
 const { buildSpeedtestDomainSet } = require('./build-speedtest-domainset');
 const { buildInternalCDNDomains } = require('./build-internal-cdn-rules');
-const { buildInternalReverseChnCIDR } = require('./build-internal-reverse-chn-cidr');
 const { buildInternalChnDomains } = require('./build-internal-chn-domains');
 const { buildDomesticRuleset } = require('./build-domestic-ruleset');
 const { validate } = require('./validate-domainset');
 
 const { buildPublicHtml } = require('./build-public');
 
+const { Worker } = require('jest-worker');
+
+/**
+ * @template T
+ * @typedef {import('jest-worker').Worker & { __sukka_worker_name: string } & T} WithWorker
+ */
+
+/**
+ * @template T
+ * @param {string} path
+ * @returns {WithWorker<T>}
+ */
+const requireWorker = (path) => {
+  const _worker = /** @type {WithWorker<T>} */ (new Worker(
+    require.resolve(path),
+    {
+      numWorkers: 1,
+      maxRetries: 0,
+      enableWorkerThreads: true
+    }
+  ));
+  _worker.getStderr().pipe(process.stderr);
+  _worker.getStdout().pipe(process.stdout);
+  _worker.__sukka_worker_name = path;
+  return _worker;
+};
+
+/**
+ * @template T
+ * @param {WithWorker<T>} worker
+ */
+const endWorker = async (worker) => {
+  const { forceExited } = worker.end();
+  if (forceExited && worker.__sukka_worker_name) {
+    console.log(worker.__sukka_worker_name, 'forceExited');
+  }
+};
+
 (async () => {
+  const buildInternalReverseChnCIDRWorker = /** @type {WithWorker<import('./build-internal-reverse-chn-cidr')>} */ (requireWorker('./build-internal-reverse-chn-cidr'));
+  const { buildInternalReverseChnCIDR } = buildInternalReverseChnCIDRWorker;
+
   // download-previous-build
   const downloadPreviousBuildPromise = downloadPreviousBuild();
   const downloadPublicSuffixListPromise = downloadPublicSuffixList();
@@ -77,6 +117,7 @@ const { buildPublicHtml } = require('./build-public');
 
   await Promise.all([
     buildPublicHtml(),
-    validate()
+    validate(),
+    endWorker(buildInternalReverseChnCIDRWorker)
   ]);
 })();

Build/lib/get-gorhill-publicsuffix.js

@@ -30,5 +30,9 @@ const getGorhillPublicSuffix = async () => {
   return gorhill;
 };
 
-const getGorhillPublicSuffixPromise = getGorhillPublicSuffix();
+/** @type {Promise<import('gorhill-publicsuffixlist').default | null>} */
-module.exports.getGorhillPublicSuffixPromise = getGorhillPublicSuffixPromise;
+let gorhillPublicSuffixPromise = null;
+module.exports.getGorhillPublicSuffixPromise = () => {
+  gorhillPublicSuffixPromise ||= getGorhillPublicSuffix();
+  return gorhillPublicSuffixPromise;
+};

Build/lib/is-domain-loose.js

@@ -10,10 +10,10 @@ module.exports.isDomainLoose = (domain) => {
 };
 
 /**
- * @param {string} domain
+ * @param {string | null | undefined} domain
  */
 module.exports.normalizeDomain = (domain) => {
-  if (domain == null) {
+  if (!domain) {
     return null;
   }
 

package.json

@@ -21,6 +21,7 @@
     "cidr-tools-wasm": "^0.0.11",
     "fs-extra": "^11.1.1",
     "gorhill-publicsuffixlist": "github:gorhill/publicsuffixlist.js",
+    "jest-worker": "^29.7.0",
     "mnemonist": "^0.39.5",
     "path-scurry": "^1.10.1",
     "picocolors": "^1.0.0",

pnpm-lock.yaml

@@ -37,6 +37,9 @@ dependencies:
   gorhill-publicsuffixlist:
     specifier: github:gorhill/publicsuffixlist.js
     version: github.com/gorhill/publicsuffixlist.js/3a1bc623073079184ff76933b88b7bf4f5d48978
+  jest-worker:
+    specifier: ^29.7.0
+    version: 29.7.0
   mnemonist:
     specifier: ^0.39.5
     version: 0.39.5
@@ -181,6 +184,25 @@ packages:
     resolution: {integrity: sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==}
     dev: true
 
+  /@jest/schemas@29.6.3:
+    resolution: {integrity: sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==}
+    engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
+    dependencies:
+      '@sinclair/typebox': 0.27.8
+    dev: false
+
+  /@jest/types@29.6.3:
+    resolution: {integrity: sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw==}
+    engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
+    dependencies:
+      '@jest/schemas': 29.6.3
+      '@types/istanbul-lib-coverage': 2.0.4
+      '@types/istanbul-reports': 3.0.1
+      '@types/node': 20.6.0
+      '@types/yargs': 17.0.24
+      chalk: 4.1.2
+    dev: false
+
   /@nodelib/fs.scandir@2.1.5:
     resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==}
     engines: {node: '>= 8'}
@@ -242,6 +264,10 @@ packages:
     resolution: {integrity: sha512-yvwa+aCyYI/UjeD39BnpMypG8N06l86wIDW1/PAc6ihBRnodIfZDwccxQN3n1t74wduzaz74m4ZMHZnB06567Q==}
     dev: false
 
+  /@sinclair/typebox@0.27.8:
+    resolution: {integrity: sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==}
+    dev: false
+
   /@sukka/listdir@0.2.0:
     resolution: {integrity: sha512-UyVirNhAOXKwjiDehjUaGtpfk0QwNHyiXrlLb/FmWMtI+BGhaEvB9MypSfEAtiiMI3g6QTfG38ayNAorEuz5ow==}
     dev: false
@@ -271,13 +297,38 @@ packages:
     resolution: {integrity: sha512-o0J30wqycjF5miWDKYKKzzOU1ZTLuA42HZ4HE7/zqTOc/jTLdQ5NhYWvsRQo45Nfi1KHoRdNhteSI4BAxTF1Pg==}
     dev: false
 
+  /@types/istanbul-lib-coverage@2.0.4:
+    resolution: {integrity: sha512-z/QT1XN4K4KYuslS23k62yDIDLwLFkzxOuMplDtObz0+y7VqJCaO2o+SPwHCvLFZh7xazvvoor2tA/hPz9ee7g==}
+    dev: false
+
+  /@types/istanbul-lib-report@3.0.0:
+    resolution: {integrity: sha512-plGgXAPfVKFoYfa9NpYDAkseG+g6Jr294RqeqcqDixSbU34MZVJRi/P+7Y8GDpzkEwLaGZZOpKIEmeVZNtKsrg==}
+    dependencies:
+      '@types/istanbul-lib-coverage': 2.0.4
+    dev: false
+
+  /@types/istanbul-reports@3.0.1:
+    resolution: {integrity: sha512-c3mAZEuK0lvBp8tmuL74XRKn1+y2dcwOUpH7x4WrF6gk1GIgiluDRgMYQtw2OFcBvAJWlt6ASU3tSqxp0Uu0Aw==}
+    dependencies:
+      '@types/istanbul-lib-report': 3.0.0
+    dev: false
+
   /@types/mocha@10.0.1:
     resolution: {integrity: sha512-/fvYntiO1GeICvqbQ3doGDIP97vWmvFt83GKguJ6prmQM2iXZfFcq6YE8KteFyRtX2/h5Hf91BYvPodJKFYv5Q==}
     dev: true
 
   /@types/node@20.6.0:
     resolution: {integrity: sha512-najjVq5KN2vsH2U/xyh2opaSEz6cZMR2SetLIlxlj08nOcmPOemJmUK2o4kUzfLqfrWE0PIrNeE16XhYDd3nqg==}
-    dev: true
+
+  /@types/yargs-parser@21.0.0:
+    resolution: {integrity: sha512-iO9ZQHkZxHn4mSakYV0vFHAVDyEOIJQrV2uZ06HxEPcx+mt8swXoZHIbaaJ2crJYFfErySgktuTZ3BeLz+XmFA==}
+    dev: false
+
+  /@types/yargs@17.0.24:
+    resolution: {integrity: sha512-6i0aC7jV6QzQB8ne1joVZ0eSFIstHsCrobmOtghM11yGlH0j43FKL2UhWdELkyps0zuf7qVTUVCCR+tgSlyLLw==}
+    dependencies:
+      '@types/yargs-parser': 21.0.0
+    dev: false
 
   /@vercel/fetch-retry@5.1.3(patch_hash=jczsp2gadg65vd4ekschpibzda)(node-fetch@2.6.11):
     resolution: {integrity: sha512-UIbFc4VsEZHOr6dWuE+kxY4NxnOLXFMCWm0fSKRRHUEtrIzaJLzHpWk2QskCXTSzFgFvhkLAvSrBK2XZg7NSzg==}
@@ -445,7 +496,6 @@ packages:
     dependencies:
       ansi-styles: 4.3.0
       supports-color: 7.2.0
-    dev: true
 
   /check-error@1.0.2:
     resolution: {integrity: sha512-BrgHpW9NURQgzoNyjfq0Wu6VFO6D7IZEmJNdtgNqpzGG8RuNFHt2jQxWlAs4HMe119chBnv+34syEZtc6IhLtA==}
@@ -1015,6 +1065,28 @@ packages:
     resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==}
     dev: true
 
+  /jest-util@29.7.0:
+    resolution: {integrity: sha512-z6EbKajIpqGKU56y5KBUgy1dt1ihhQJgWzUlZHArA/+X2ad7Cb5iF+AK1EWVL/Bo7Rz9uurpqw6SiBCefUbCGA==}
+    engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
+    dependencies:
+      '@jest/types': 29.6.3
+      '@types/node': 20.6.0
+      chalk: 4.1.2
+      ci-info: 3.8.0
+      graceful-fs: 4.2.10
+      picomatch: 2.3.1
+    dev: false
+
+  /jest-worker@29.7.0:
+    resolution: {integrity: sha512-eIz2msL/EzL9UFTFFx7jBTkeZfku0yUAyZZZmJ93H2TYEiroIx2PQjEXcwYtYl8zXCxb+PAmA2hLIt/6ZEkPHw==}
+    engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
+    dependencies:
+      '@types/node': 20.6.0
+      jest-util: 29.7.0
+      merge-stream: 2.0.0
+      supports-color: 8.1.1
+    dev: false
+
   /js-yaml@4.1.0:
     resolution: {integrity: sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==}
     hasBin: true
@@ -1091,6 +1163,10 @@ packages:
       yallist: 4.0.0
     dev: true
 
+  /merge-stream@2.0.0:
+    resolution: {integrity: sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==}
+    dev: false
+
   /minimatch@3.1.2:
     resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==}
     dependencies:
@@ -1279,7 +1355,6 @@ packages:
   /picomatch@2.3.1:
     resolution: {integrity: sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==}
     engines: {node: '>=8.6'}
-    dev: true
 
   /prelude-ls@1.2.1:
     resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==}
@@ -1426,7 +1501,6 @@ packages:
     engines: {node: '>=8'}
     dependencies:
       has-flag: 4.0.0
-    dev: true
 
   /supports-color@8.1.1:
     resolution: {integrity: sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==}