diff --git a/Build/build-cdn-download-conf.ts b/Build/build-cdn-download-conf.ts index 468b8bd6..67b186d7 100644 --- a/Build/build-cdn-download-conf.ts +++ b/Build/build-cdn-download-conf.ts @@ -5,6 +5,7 @@ import { createTrie } from './lib/trie'; import { task } from './trace'; import { SHARED_DESCRIPTION } from './lib/constants'; import { getPublicSuffixListTextPromise } from './lib/download-publicsuffixlist'; +import { domainDeduper } from './lib/domain-deduper'; const getS3OSSDomainsPromise = (async (): Promise> => { const trie = createTrie((await getPublicSuffixListTextPromise()).split('\n')); @@ -43,18 +44,19 @@ const getS3OSSDomainsPromise = (async (): Promise> => { export const buildCdnDownloadConf = task(import.meta.path, async (span) => { const [ - cdnDomainsList, S3OSSDomains, + + cdnDomainsList, downloadDomainSet, steamDomainSet ] = await Promise.all([ - readFileIntoProcessedArray(path.resolve(import.meta.dir, '../Source/non_ip/cdn.conf')), getS3OSSDomainsPromise, + readFileIntoProcessedArray(path.resolve(import.meta.dir, '../Source/domainset/cdn.conf')), readFileIntoProcessedArray(path.resolve(import.meta.dir, '../Source/domainset/download.conf')), readFileIntoProcessedArray(path.resolve(import.meta.dir, '../Source/domainset/steam.conf')) ]); - cdnDomainsList.push(...Array.from(S3OSSDomains).map((domain) => `DOMAIN-SUFFIX,${domain}`)); + cdnDomainsList.push(...Array.from(S3OSSDomains).map((domain) => `.${domain}`)); return Promise.all([ createRuleset( @@ -66,10 +68,10 @@ export const buildCdnDownloadConf = task(import.meta.path, async (span) => { 'This file contains object storage and static assets CDN domains.' ], new Date(), - cdnDomainsList, - 'ruleset', - path.resolve(import.meta.dir, '../List/non_ip/cdn.conf'), - path.resolve(import.meta.dir, '../Clash/non_ip/cdn.txt') + domainDeduper(cdnDomainsList), + 'domainset', + path.resolve(import.meta.dir, '../List/domainset/cdn.conf'), + path.resolve(import.meta.dir, '../Clash/domainset/cdn.txt') ), createRuleset( span, @@ -80,10 +82,10 @@ export const buildCdnDownloadConf = task(import.meta.path, async (span) => { 'This file contains domains for software updating & large file hosting.' ], new Date(), - [ + domainDeduper([ ...downloadDomainSet, ...steamDomainSet - ], + ]), 'domainset', path.resolve(import.meta.dir, '../List/domainset/download.conf'), path.resolve(import.meta.dir, '../Clash/domainset/download.txt') diff --git a/Build/build-common.ts b/Build/build-common.ts index 4e4b2d49..950fa979 100644 --- a/Build/build-common.ts +++ b/Build/build-common.ts @@ -66,7 +66,7 @@ const processFile = (span: Span, sourcePath: string) => { try { for await (const line of readFileByLine(sourcePath)) { - if (line === MAGIC_COMMAND_SKIP) { + if (line.startsWith(MAGIC_COMMAND_SKIP)) { return null; } diff --git a/Build/index.ts b/Build/index.ts index c3fb0e66..c0f7b664 100644 --- a/Build/index.ts +++ b/Build/index.ts @@ -1,3 +1,5 @@ +console.log('Bun version:', Bun.version, Bun.revision); + import { downloadPreviousBuild } from './download-previous-build'; import { buildCommon } from './build-common'; import { buildAntiBogusDomain } from './build-anti-bogus-domain'; @@ -8,7 +10,6 @@ import { buildTelegramCIDR } from './build-telegram-cidr'; import { buildChnCidr } from './build-chn-cidr'; import { buildSpeedtestDomainSet } from './build-speedtest-domainset'; import { buildInternalCDNDomains } from './build-internal-cdn-rules'; -// import { buildInternalChnDomains } from './build-internal-chn-domains'; import { buildInternalReverseChnCIDR } from './build-internal-reverse-chn-cidr'; import { buildDomesticRuleset } from './build-domestic-ruleset'; import { buildStreamService } from './build-stream-service'; @@ -16,8 +17,6 @@ import { buildStreamService } from './build-stream-service'; import { buildRedirectModule } from './build-sgmodule-redirect'; import { buildAlwaysRealIPModule } from './build-sgmodule-always-realip'; -import { validate } from './validate-domainset'; - import { buildMicrosoftCdn } from './build-microsoft-cdn'; import { buildSSPanelUIMAppProfile } from './build-sspanel-appprofile'; @@ -29,14 +28,10 @@ import { buildCloudMounterRules } from './build-cloudmounter-rules'; import { createSpan, printTraceResult } from './trace'; import { buildDeprecateFiles } from './build-deprecate-files'; -console.log('Bun version:', Bun.version, Bun.revision); - (async () => { const rootSpan = createSpan('root'); try { - // const buildInternalReverseChnCIDRWorker = new Worker(new URL('./workers/build-internal-reverse-chn-cidr-worker.ts', import.meta.url)); - const downloadPreviousBuildPromise = downloadPreviousBuild(rootSpan); const buildCommonPromise = downloadPreviousBuildPromise.then(() => buildCommon(rootSpan)); @@ -98,8 +93,7 @@ console.log('Bun version:', Bun.version, Bun.revision); ]); await Promise.all([ - buildPublic(rootSpan), - validate(rootSpan) + buildPublic(rootSpan) ]); rootSpan.stop(); diff --git a/Build/validate-domainset.ts b/Build/validate-domainset.ts deleted file mode 100644 index 70026698..00000000 --- a/Build/validate-domainset.ts +++ /dev/null @@ -1,75 +0,0 @@ -// Surge Domain Set can not include root domain from public suffix list. - -import * as tldts from 'tldts'; // hit ratio way too low, dont cache -import picocolors from 'picocolors'; -import path from 'path'; -import listDir from '@sukka/listdir'; -import { readFileByLine } from './lib/fetch-text-by-line'; -import { processLine } from './lib/process-line'; -import { task } from './trace'; - -const SPECIAL_SUFFIXES = new Set([ - 'linodeobjects.com', // only *.linodeobjects.com are public suffix - 'vultrobjects.com', // only *.vultrobjects.com are public suffix - 'dweb.link' // only *.dweb.link are public suffix -]); - -const validateDomainSet = async (filePath: string) => { - for await (const l of readFileByLine(path.resolve(import.meta.dir, '../List/domainset', filePath))) { - // starts with # - const line = processLine(l); - if (!line) { - continue; - } - const domain = line[0] === '.' ? line.slice(1) : line; - const parsed = tldts.parse(domain, { allowPrivateDomains: true, detectIp: false }); - - if ( - ( - parsed.isPrivate - || parsed.isIcann - ) && domain === parsed.publicSuffix - ) { - console.error(`[${filePath}]`, picocolors.yellow(domain), picocolors.red('is in public suffix list!')); - } - } -}; - -const _validateRuleset = async (filePath: string) => { - console.log(`[${filePath}]`); - - for await (const l of readFileByLine(path.resolve(import.meta.dir, '../List/non_ip', filePath))) { - // starts with # - const line = processLine(l); - if (!line) { - continue; - } - if (!line.startsWith('DOMAIN-SUFFIX,')) { - continue; - } - const domain = line.slice(14); - const parsed = tldts.parse(domain, { allowPrivateDomains: true, detectIp: false }); - - if (domain !== parsed.publicSuffix) { - if (!SPECIAL_SUFFIXES.has(domain)) { - console.warn(picocolors.yellow(domain), picocolors.green('is not in public suffix list!')); - } - } - } -}; - -export const validate = task(import.meta.path, async () => { - // const [domainsetFiles, _rulesetFiles] = await Promise.all([ - // listDir(path.resolve(import.meta.dir, '../List/domainset')), - // listDir(path.resolve(import.meta.dir, '../List/non_ip')) - // ]); - return Promise.all([ - listDir(path.resolve(import.meta.dir, '../List/domainset')) - .then(domainsetFiles => Promise.all(domainsetFiles.map(file => validateDomainSet(file)))) - // rulesetFiles.map(file => validateRuleset(file)) - ]); -}); - -if (import.meta.main) { - validate(); -} diff --git a/Source/domainset/cdn.conf b/Source/domainset/cdn.conf index 238f7bf1..2f79371b 100644 --- a/Source/domainset/cdn.conf +++ b/Source/domainset/cdn.conf @@ -1,5 +1,4 @@ -# $ meta_title Sukka's Ruleset - CDN Domains -# $ meta_description This file contains object storage and static assets CDN domains. +# $ custom_build_script # >> Sukka .ruleset.skk.moe @@ -2234,7 +2233,6 @@ images.dmca.com static.olelive.com media.owcnow.com photos5.appleinsider.com -vanillicon.com loutre.blockchair.io js-cdn.blockchair.io static.tokenview.io diff --git a/Source/non_ip/cdn.conf b/Source/non_ip/cdn.conf index f764c576..a406c13a 100644 --- a/Source/non_ip/cdn.conf +++ b/Source/non_ip/cdn.conf @@ -1,4 +1,5 @@ -# $ custom_build_script +# $ meta_title Sukka's Ruleset - CDN Domains +# $ meta_description This file contains object storage and static assets CDN domains. # >> DigitalOcean OSS DOMAIN-SUFFIX,digitaloceanspaces.com