Refactor: use jest-worker

Build/build-cdn-download-conf.ts

@@ -1,146 +0,0 @@
-import path from 'node:path';
-import { readFileIntoProcessedArray } from './lib/fetch-text-by-line';
-import { task } from './trace';
-import { SHARED_DESCRIPTION } from './constants/description';
-import { appendArrayInPlace } from 'foxts/append-array-in-place';
-import { SOURCE_DIR } from './constants/dir';
-import { DomainsetOutput } from './lib/rules/domainset';
-import { CRASHLYTICS_WHITELIST } from './constants/reject-data-source';
-import Worktank from 'worktank';
-import { $$fetch } from './lib/fetch-retry';
-import { fastUri } from 'fast-uri';
-
-const cdnDomainSetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/cdn.conf'));
-const downloadDomainSetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/download.conf'));
-const steamDomainSetPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/game-download.conf'));
-
-const pool = new Worktank({
-  pool: {
-    name: 'extract-s3-from-publicssuffix',
-    size: 1 // The number of workers to keep in the pool, if more workers are needed they will be spawned up to this limit
-  },
-  worker: {
-    autoAbort: 10000,
-    autoTerminate: 20000, // The interval of milliseconds at which to check if the pool can be automatically terminated, to free up resources, workers will be spawned up again if needed
-    autoInstantiate: true,
-    methods: {
-      // eslint-disable-next-line object-shorthand -- workertank
-      getS3OSSDomains: async function (__filename: string): Promise<string[]> {
-        // TODO: createRequire is a temporary workaround for https://github.com/nodejs/node/issues/51956
-        const { default: module } = await import('node:module');
-        const __require = module.createRequire(__filename);
-
-        const { HostnameTrie } = __require('./lib/trie') as typeof import('./lib/trie');
-        const { fetchRemoteTextByLine } = __require('./lib/fetch-text-by-line') as typeof import('./lib/fetch-text-by-line');
-
-        const trie = new HostnameTrie();
-
-        for await (const line of await fetchRemoteTextByLine('https://publicsuffix.org/list/public_suffix_list.dat', true)) {
-          trie.add(line);
-        }
-
-        /**
-         * Extract OSS domain from publicsuffix list
-         */
-        const S3OSSDomains: string[] = [];
-
-        trie.find('.amazonaws.com').forEach((line: string) => {
-          if (
-            (line.startsWith('s3-') || line.startsWith('s3.'))
-            && !line.includes('cn-')
-          ) {
-            S3OSSDomains.push('.' + line);
-          }
-        });
-        trie.find('.scw.cloud').forEach((line: string) => {
-          if (
-            (line.startsWith('s3-') || line.startsWith('s3.'))
-          // && !line.includes('cn-')
-          ) {
-            S3OSSDomains.push('.' + line);
-          }
-        });
-        trie.find('sakurastorage.jp').forEach((line: string) => {
-          if (
-            (line.startsWith('s3-') || line.startsWith('s3.'))
-          ) {
-            S3OSSDomains.push('.' + line);
-          }
-        });
-
-        return S3OSSDomains;
-      }
-    }
-  }
-});
-
-export const buildCdnDownloadConf = task(require.main === module, __filename)(async (span) => {
-  const [
-    S3OSSDomains,
-    IPFSDomains,
-    cdnDomainsList,
-    downloadDomainSet,
-    steamDomainSet
-  ] = await Promise.all([
-    span.traceChildAsync(
-      'download public suffix list for s3',
-      () => pool.exec(
-        'getS3OSSDomains',
-        [__filename]
-      ).finally(() => pool.terminate())
-    ),
-    span.traceChildAsync(
-      'load public ipfs gateway list',
-      async () => {
-        const data = await (await $$fetch('https://cdn.jsdelivr.net/gh/ipfs/public-gateway-checker@main/gateways.json')).json();
-        if (!Array.isArray(data)) {
-          console.error('Invalid IPFS gateway list format');
-          return [];
-        }
-        return data.reduce<string[]>((acc, gateway) => {
-          if (typeof gateway !== 'string') {
-            return acc;
-          }
-          const hn = fastUri.parse(gateway).host;
-          if (hn) {
-            acc.push(hn.trim());
-          }
-          return acc;
-        }, []);
-      }
-    ),
-    cdnDomainSetPromise,
-    downloadDomainSetPromise,
-    steamDomainSetPromise
-  ]);
-
-  // Move S3 domains to download domain set, since S3 files may be large
-  appendArrayInPlace(downloadDomainSet, S3OSSDomains);
-  appendArrayInPlace(downloadDomainSet, steamDomainSet);
-
-  // we have whitelisted the crashlytics domain, and we also want to put it in CDN policy
-  appendArrayInPlace(cdnDomainsList, CRASHLYTICS_WHITELIST);
-
-  return Promise.all([
-    new DomainsetOutput(span, 'cdn')
-      .withTitle('Sukka\'s Ruleset - CDN Domains')
-      .appendDescription(SHARED_DESCRIPTION)
-      .appendDescription(
-        '',
-        'This file contains object storage and static assets CDN domains.'
-      )
-      .addFromDomainset(cdnDomainsList)
-      .bulkAddDomainSuffix(IPFSDomains)
-      .write(),
-
-    new DomainsetOutput(span, 'download')
-      .withTitle('Sukka\'s Ruleset - Large Files Hosting Domains')
-      .appendDescription(SHARED_DESCRIPTION)
-      .appendDescription(
-        '',
-        'This file contains domains for software updating & large file hosting.'
-      )
-      .addFromDomainset(downloadDomainSet)
-      .write()
-  ]);
-});