Refactor: CDN build prefer domainset

Build/build-cdn-download-conf.ts

@@ -5,6 +5,7 @@ import { createTrie } from './lib/trie';
 import { task } from './trace';
 import { SHARED_DESCRIPTION } from './lib/constants';
 import { getPublicSuffixListTextPromise } from './lib/download-publicsuffixlist';
+import { domainDeduper } from './lib/domain-deduper';
 
 const getS3OSSDomainsPromise = (async (): Promise<Set<string>> => {
   const trie = createTrie((await getPublicSuffixListTextPromise()).split('\n'));
@@ -43,18 +44,19 @@ const getS3OSSDomainsPromise = (async (): Promise<Set<string>> => {
 
 export const buildCdnDownloadConf = task(import.meta.path, async (span) => {
   const [
-    cdnDomainsList,
     S3OSSDomains,
+
+    cdnDomainsList,
     downloadDomainSet,
     steamDomainSet
   ] = await Promise.all([
-    readFileIntoProcessedArray(path.resolve(import.meta.dir, '../Source/non_ip/cdn.conf')),
     getS3OSSDomainsPromise,
+    readFileIntoProcessedArray(path.resolve(import.meta.dir, '../Source/domainset/cdn.conf')),
     readFileIntoProcessedArray(path.resolve(import.meta.dir, '../Source/domainset/download.conf')),
     readFileIntoProcessedArray(path.resolve(import.meta.dir, '../Source/domainset/steam.conf'))
   ]);
 
-  cdnDomainsList.push(...Array.from(S3OSSDomains).map((domain) => `DOMAIN-SUFFIX,${domain}`));
+  cdnDomainsList.push(...Array.from(S3OSSDomains).map((domain) => `.${domain}`));
 
   return Promise.all([
     createRuleset(
@@ -66,10 +68,10 @@ export const buildCdnDownloadConf = task(import.meta.path, async (span) => {
         'This file contains object storage and static assets CDN domains.'
       ],
       new Date(),
-      cdnDomainsList,
+      domainDeduper(cdnDomainsList),
-      'ruleset',
+      'domainset',
-      path.resolve(import.meta.dir, '../List/non_ip/cdn.conf'),
+      path.resolve(import.meta.dir, '../List/domainset/cdn.conf'),
-      path.resolve(import.meta.dir, '../Clash/non_ip/cdn.txt')
+      path.resolve(import.meta.dir, '../Clash/domainset/cdn.txt')
     ),
     createRuleset(
       span,
@@ -80,10 +82,10 @@ export const buildCdnDownloadConf = task(import.meta.path, async (span) => {
         'This file contains domains for software updating & large file hosting.'
       ],
       new Date(),
-      [
+      domainDeduper([
         ...downloadDomainSet,
         ...steamDomainSet
-      ],
+      ]),
       'domainset',
       path.resolve(import.meta.dir, '../List/domainset/download.conf'),
       path.resolve(import.meta.dir, '../Clash/domainset/download.txt')

Build/build-common.ts

@@ -66,7 +66,7 @@ const processFile = (span: Span, sourcePath: string) => {
 
     try {
       for await (const line of readFileByLine(sourcePath)) {
-        if (line === MAGIC_COMMAND_SKIP) {
+        if (line.startsWith(MAGIC_COMMAND_SKIP)) {
           return null;
         }
 

Build/index.ts

@@ -1,3 +1,5 @@
+console.log('Bun version:', Bun.version, Bun.revision);
+
 import { downloadPreviousBuild } from './download-previous-build';
 import { buildCommon } from './build-common';
 import { buildAntiBogusDomain } from './build-anti-bogus-domain';
@@ -8,7 +10,6 @@ import { buildTelegramCIDR } from './build-telegram-cidr';
 import { buildChnCidr } from './build-chn-cidr';
 import { buildSpeedtestDomainSet } from './build-speedtest-domainset';
 import { buildInternalCDNDomains } from './build-internal-cdn-rules';
-// import { buildInternalChnDomains } from './build-internal-chn-domains';
 import { buildInternalReverseChnCIDR } from './build-internal-reverse-chn-cidr';
 import { buildDomesticRuleset } from './build-domestic-ruleset';
 import { buildStreamService } from './build-stream-service';
@@ -16,8 +17,6 @@ import { buildStreamService } from './build-stream-service';
 import { buildRedirectModule } from './build-sgmodule-redirect';
 import { buildAlwaysRealIPModule } from './build-sgmodule-always-realip';
 
-import { validate } from './validate-domainset';
-
 import { buildMicrosoftCdn } from './build-microsoft-cdn';
 import { buildSSPanelUIMAppProfile } from './build-sspanel-appprofile';
 
@@ -29,14 +28,10 @@ import { buildCloudMounterRules } from './build-cloudmounter-rules';
 import { createSpan, printTraceResult } from './trace';
 import { buildDeprecateFiles } from './build-deprecate-files';
 
-console.log('Bun version:', Bun.version, Bun.revision);
-
 (async () => {
   const rootSpan = createSpan('root');
 
   try {
-    // const buildInternalReverseChnCIDRWorker = new Worker(new URL('./workers/build-internal-reverse-chn-cidr-worker.ts', import.meta.url));
-
     const downloadPreviousBuildPromise = downloadPreviousBuild(rootSpan);
 
     const buildCommonPromise = downloadPreviousBuildPromise.then(() => buildCommon(rootSpan));
@@ -98,8 +93,7 @@ console.log('Bun version:', Bun.version, Bun.revision);
     ]);
 
     await Promise.all([
-      buildPublic(rootSpan),
+      buildPublic(rootSpan)
-      validate(rootSpan)
     ]);
 
     rootSpan.stop();

Build/validate-domainset.ts

@@ -1,75 +0,0 @@
-// Surge Domain Set can not include root domain from public suffix list.
-
-import * as tldts from 'tldts'; // hit ratio way too low, dont cache
-import picocolors from 'picocolors';
-import path from 'path';
-import listDir from '@sukka/listdir';
-import { readFileByLine } from './lib/fetch-text-by-line';
-import { processLine } from './lib/process-line';
-import { task } from './trace';
-
-const SPECIAL_SUFFIXES = new Set([
-  'linodeobjects.com', // only *.linodeobjects.com are public suffix
-  'vultrobjects.com', // only *.vultrobjects.com are public suffix
-  'dweb.link' // only *.dweb.link are public suffix
-]);
-
-const validateDomainSet = async (filePath: string) => {
-  for await (const l of readFileByLine(path.resolve(import.meta.dir, '../List/domainset', filePath))) {
-    // starts with #
-    const line = processLine(l);
-    if (!line) {
-      continue;
-    }
-    const domain = line[0] === '.' ? line.slice(1) : line;
-    const parsed = tldts.parse(domain, { allowPrivateDomains: true, detectIp: false });
-
-    if (
-      (
-        parsed.isPrivate
-        || parsed.isIcann
-      ) && domain === parsed.publicSuffix
-    ) {
-      console.error(`[${filePath}]`, picocolors.yellow(domain), picocolors.red('is in public suffix list!'));
-    }
-  }
-};
-
-const _validateRuleset = async (filePath: string) => {
-  console.log(`[${filePath}]`);
-
-  for await (const l of readFileByLine(path.resolve(import.meta.dir, '../List/non_ip', filePath))) {
-    // starts with #
-    const line = processLine(l);
-    if (!line) {
-      continue;
-    }
-    if (!line.startsWith('DOMAIN-SUFFIX,')) {
-      continue;
-    }
-    const domain = line.slice(14);
-    const parsed = tldts.parse(domain, { allowPrivateDomains: true, detectIp: false });
-
-    if (domain !== parsed.publicSuffix) {
-      if (!SPECIAL_SUFFIXES.has(domain)) {
-        console.warn(picocolors.yellow(domain), picocolors.green('is not in public suffix list!'));
-      }
-    }
-  }
-};
-
-export const validate = task(import.meta.path, async () => {
-  // const [domainsetFiles, _rulesetFiles] = await Promise.all([
-  //   listDir(path.resolve(import.meta.dir, '../List/domainset')),
-  //   listDir(path.resolve(import.meta.dir, '../List/non_ip'))
-  // ]);
-  return Promise.all([
-    listDir(path.resolve(import.meta.dir, '../List/domainset'))
-      .then(domainsetFiles => Promise.all(domainsetFiles.map(file => validateDomainSet(file))))
-    // rulesetFiles.map(file => validateRuleset(file))
-  ]);
-});
-
-if (import.meta.main) {
-  validate();
-}

Source/domainset/cdn.conf

@@ -1,5 +1,4 @@
-# $ meta_title Sukka's Ruleset - CDN Domains
+# $ custom_build_script
-# $ meta_description This file contains object storage and static assets CDN domains.
 
 # >> Sukka
 .ruleset.skk.moe
@@ -2234,7 +2233,6 @@ images.dmca.com
 static.olelive.com
 media.owcnow.com
 photos5.appleinsider.com
-vanillicon.com
 loutre.blockchair.io
 js-cdn.blockchair.io
 static.tokenview.io

Source/non_ip/cdn.conf

@@ -1,4 +1,5 @@
-# $ custom_build_script
+# $ meta_title Sukka's Ruleset - CDN Domains
+# $ meta_description This file contains object storage and static assets CDN domains.
 
 # >> DigitalOcean OSS
 DOMAIN-SUFFIX,digitaloceanspaces.com