Perf: run a few task during downloading previous dist

Build/build-cdn-download-conf.ts

@@ -48,18 +48,21 @@ const getS3OSSDomainsPromise = (async (): Promise<Set<string>> => {
   return S3OSSDomains;
 })();
 
+const cdnDomainsListPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/cdn.conf'));
+const downloadDomainSetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/download.conf'));
+const steamDomainSetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/game-download.conf'));
+
 export const buildCdnDownloadConf = task(require.main === module, __filename)(async (span) => {
   const [
     S3OSSDomains,
-
     cdnDomainsList,
     downloadDomainSet,
     steamDomainSet
   ] = await Promise.all([
     getS3OSSDomainsPromise,
-    readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/cdn.conf')),
+    cdnDomainsListPromise,
-    readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/download.conf')),
+    downloadDomainSetPromise,
-    readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/game-download.conf'))
+    steamDomainSetPromise
   ]);
 
   // Move S3 domains to download domain set, since S3 files may be large

Build/build-common.ts

@@ -18,20 +18,22 @@ const MAGIC_COMMAND_SGMODULE_MITM_HOSTNAMES = '# $ sgmodule_mitm_hostnames ';
 
 const domainsetSrcFolder = 'domainset' + path.sep;
 
+const clawSourceDirPromise = new Fdir()
+  .withRelativePaths()
+  .filter((filepath, isDirectory) => {
+    if (isDirectory) return true;
+
+    const extname = path.extname(filepath);
+
+    return !(extname === '.js' || extname === '.ts');
+  })
+  .crawl(SOURCE_DIR)
+  .withPromise();
+
 export const buildCommon = task(require.main === module, __filename)(async (span) => {
   const promises: Array<Promise<unknown>> = [];
 
-  const paths = await new Fdir()
+  const paths = await clawSourceDirPromise;
-    .withRelativePaths()
-    .filter((filepath, isDirectory) => {
-      if (isDirectory) return true;
-
-      const extname = path.extname(filepath);
-
-      return !(extname === '.js' || extname === '.ts');
-    })
-    .crawl(SOURCE_DIR)
-    .withPromise();
 
   for (let i = 0, len = paths.length; i < len; i++) {
     const relativePath = paths[i];

Build/build-reject-domainset.ts

@@ -19,6 +19,8 @@ import { appendArrayInPlace } from './lib/append-array-in-place';
 import { OUTPUT_INTERNAL_DIR, SOURCE_DIR } from './constants/dir';
 import { DomainsetOutput } from './lib/create-file';
 
+const readLocalNonIpRejectRulesetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/reject_sukka.conf'));
+
 export const buildRejectDomainSet = task(require.main === module, __filename)(async (span) => {
   const rejectBaseDescription = [
     ...SHARED_DESCRIPTION,
@@ -98,7 +100,7 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as
           addArrayElementsToSet(filterRuleWhitelistDomainSets, black);
         })),
         getPhishingDomains(childSpan).then(appendArrayToRejectExtraOutput),
-        readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/reject_sukka.conf')).then(appendArrayToRejectOutput),
+        readLocalNonIpRejectRulesetPromise.then(appendArrayToRejectOutput),
         // Dedupe domainSets
         // span.traceChildAsync('collect black keywords/suffixes', async () =>
         /**

Build/build-sspanel-appprofile.ts

@@ -23,6 +23,8 @@ const POLICY_GROUPS: Array<[name: string, insertProxy: boolean, insertDirect: bo
   ['Final Match', true, true]
 ];
 
+const steamDomainsPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/game-download.conf'));
+
 /**
  * This only generates a simplified version, for under-used users only.
  */
@@ -56,7 +58,7 @@ export const buildSSPanelUIMAppProfile = task(require.main === module, __filenam
     // stream - domains
     AllStreamServices.flatMap((i) => i.rules),
     // steam - domains
-    readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/game-download.conf')),
+    steamDomainsPromise,
     // global - domains
     readFileIntoProcessedArray(path.join(OUTPUT_SURGE_DIR, 'non_ip/global.conf')),
     readFileIntoProcessedArray(path.join(OUTPUT_SURGE_DIR, 'non_ip/telegram.conf')),

Build/download-mock-assets.ts

@@ -14,20 +14,36 @@ const ASSETS_LIST = {
   'amazon-adsystem-com_amazon-apstag.js': 'https://unpkg.com/@adguard/scriptlets@latest/dist/redirect-files/amazon-apstag.js'
 } as const;
 
-export const downloadMockAssets = task(require.main === module, __filename)((span) => Promise.all(Object.entries(ASSETS_LIST).map(
+export const downloadMockAssets = task(require.main === module, __filename)(async (span) => {
-  ([filename, url]) => span
+  const p = mkdirp(OUTPUT_MOCK_DIR);
-    .traceChildAsync(url, async () => {
+  if (p) {
-      const res = await $fetch(url);
+    await p;
-      if (!res.body) {
+  }
-        throw new Error(`Empty body from ${url}`);
-      }
 
-      await mkdirp(OUTPUT_MOCK_DIR);
+  return Promise.all(Object.entries(ASSETS_LIST).map(
-      const src = path.join(OUTPUT_MOCK_DIR, filename);
+    ([filename, url]) => span
+      .traceChildAsync(url, async () => {
+        const res = await $fetch(url);
+        if (!res.ok) {
+          console.error(`Failed to download ${url}`);
 
-      return pipeline(
+          // we can safely skip this since we can always use previous version
-        res.body,
+          return;
-        fs.createWriteStream(src, 'utf-8')
+        }
-      );
+
-    })
+        if (!res.body) {
-)));
+          console.error(`Empty body from ${url}`);
+
+          // we can safely skip this since we can always use previous version
+          return;
+        }
+
+        const src = path.join(OUTPUT_MOCK_DIR, filename);
+
+        return pipeline(
+          res.body,
+          fs.createWriteStream(src, 'utf-8')
+        );
+      })
+  ));
+});

Build/download-previous-build.ts

@@ -7,14 +7,13 @@ import type { Headers as TarEntryHeaders } from 'tar-fs';
 import zlib from 'node:zlib';
 import undici from 'undici';
 import picocolors from 'picocolors';
+import { PUBLIC_DIR } from './constants/dir';
 
 const GITHUB_CODELOAD_URL = 'https://codeload.github.com/sukkalab/ruleset.skk.moe/tar.gz/master';
 const GITLAB_CODELOAD_URL = 'https://gitlab.com/SukkaW/ruleset.skk.moe/-/archive/master/ruleset.skk.moe-master.tar.gz';
 
 export const downloadPreviousBuild = task(require.main === module, __filename)(async (span) => {
-  const publicDir = path.resolve(__dirname, '..', 'public');
+  if (fs.existsSync(PUBLIC_DIR)) {
-
-  if (fs.existsSync(publicDir)) {
     console.log(picocolors.blue('Public directory exists, skip downloading previous build'));
     return;
   }
@@ -61,7 +60,7 @@ export const downloadPreviousBuild = task(require.main === module, __filename)(a
 
     const gunzip = zlib.createGunzip();
     const extract = tarExtract(
-      publicDir,
+      PUBLIC_DIR,
       {
         ignore(_: string, header?: TarEntryHeaders) {
           if (header) {

Build/index.ts

@@ -62,55 +62,30 @@ process.on('unhandledRejection', (reason) => {
 
   try {
     const downloadPreviousBuildPromise = downloadPreviousBuild(rootSpan);
-
     const buildCommonPromise = downloadPreviousBuildPromise.then(() => buildCommon(rootSpan));
-    const buildRejectIPListPromise = downloadPreviousBuildPromise.then(() => buildRejectIPList(rootSpan));
-    const buildAppleCdnPromise = downloadPreviousBuildPromise.then(() => buildAppleCdn(rootSpan));
-    const buildCdnConfPromise = downloadPreviousBuildPromise.then(() => buildCdnDownloadConf(rootSpan));
-    const buildRejectDomainSetPromise = downloadPreviousBuildPromise.then(() => buildRejectDomainSet(rootSpan));
-    const buildTelegramCIDRPromise = downloadPreviousBuildPromise.then(() => buildTelegramCIDR(rootSpan));
-    const buildChnCidrPromise = downloadPreviousBuildPromise.then(() => buildChnCidr(rootSpan));
-    const buildSpeedtestDomainSetPromise = downloadPreviousBuildPromise.then(() => buildSpeedtestDomainSet(rootSpan));
-
-    const buildInternalReverseChnCIDRPromise = buildInternalReverseChnCIDR(rootSpan);
-
-    // const buildInternalChnDomainsPromise = buildInternalChnDomains();
-    const buildDomesticRulesetPromise = downloadPreviousBuildPromise.then(() => buildDomesticRuleset(rootSpan));
-
-    const buildRedirectModulePromise = downloadPreviousBuildPromise.then(() => buildRedirectModule(rootSpan));
-    const buildAlwaysRealIPModulePromise = downloadPreviousBuildPromise.then(() => buildAlwaysRealIPModule(rootSpan));
-
-    const buildStreamServicePromise = downloadPreviousBuildPromise.then(() => buildStreamService(rootSpan));
-
-    const buildMicrosoftCdnPromise = downloadPreviousBuildPromise.then(() => buildMicrosoftCdn(rootSpan));
-
-    const buildSSPanelUIMAppProfilePromise = downloadPreviousBuildPromise.then(() => buildSSPanelUIMAppProfile(rootSpan));
-
-    const downloadMockAssetsPromise = downloadMockAssets(rootSpan);
-
-    const buildCloudMounterRulesPromise = downloadPreviousBuildPromise.then(() => buildCloudMounterRules(rootSpan));
 
     await Promise.all([
       downloadPreviousBuildPromise,
       buildCommonPromise,
-      buildRejectIPListPromise,
+      downloadPreviousBuildPromise.then(() => buildRejectIPList(rootSpan)),
-      buildAppleCdnPromise,
+      downloadPreviousBuildPromise.then(() => buildAppleCdn(rootSpan)),
-      buildCdnConfPromise,
+      downloadPreviousBuildPromise.then(() => buildCdnDownloadConf(rootSpan)),
-      buildRejectDomainSetPromise,
+      downloadPreviousBuildPromise.then(() => buildRejectDomainSet(rootSpan)),
-      buildTelegramCIDRPromise,
+      downloadPreviousBuildPromise.then(() => buildTelegramCIDR(rootSpan)),
-      buildChnCidrPromise,
+      downloadPreviousBuildPromise.then(() => buildChnCidr(rootSpan)),
-      buildSpeedtestDomainSetPromise,
+      downloadPreviousBuildPromise.then(() => buildSpeedtestDomainSet(rootSpan)),
-      buildInternalReverseChnCIDRPromise,
+      buildInternalReverseChnCIDR(rootSpan),
-      buildInternalReverseChnCIDRPromise,
+      downloadPreviousBuildPromise.then(() => buildDomesticRuleset(rootSpan)),
-      // buildInternalChnDomainsPromise,
+      downloadPreviousBuildPromise.then(() => buildRedirectModule(rootSpan)),
-      buildDomesticRulesetPromise,
+      downloadPreviousBuildPromise.then(() => buildAlwaysRealIPModule(rootSpan)),
-      buildRedirectModulePromise,
+      downloadPreviousBuildPromise.then(() => buildStreamService(rootSpan)),
-      buildAlwaysRealIPModulePromise,
+      downloadPreviousBuildPromise.then(() => buildMicrosoftCdn(rootSpan)),
-      buildStreamServicePromise,
+      Promise.all([
-      buildMicrosoftCdnPromise,
+        downloadPreviousBuildPromise,
-      buildSSPanelUIMAppProfilePromise,
+        buildCommonPromise
-      buildCloudMounterRulesPromise,
+      ]).then(() => buildSSPanelUIMAppProfile(rootSpan)),
-      downloadMockAssetsPromise
+      downloadPreviousBuildPromise.then(() => buildCloudMounterRules(rootSpan)),
+      downloadMockAssets(rootSpan)
     ]);
 
     await Promise.all([

Build/lib/get-phishing-domains.ts

@@ -219,8 +219,6 @@ export function getPhishingDomains(parentSpan: Span) {
       return domainArr;
     });
 
-    console.log({ len: domainArr.length });
-
     return span.traceChildAsync(
       'process phishing domain set',
       () => processPhihsingDomains(domainArr)