From 0bdf8d30a0dc10978da308e7c02996839af9a074 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Tue, 11 Oct 2022 17:10:11 +0800 Subject: [PATCH] Validate CDN Domains against Public Suffix LIst --- Build/validate-cdn-conf.js | 57 ++++++++++++++++++++++++++++++++++++++ List/domainset/cdn.conf | 11 +++----- List/non_ip/cdn.conf | 15 ++++++---- package.json | 10 +++++-- pnpm-lock.yaml | 6 ++++ 5 files changed, 85 insertions(+), 14 deletions(-) create mode 100644 Build/validate-cdn-conf.js diff --git a/Build/validate-cdn-conf.js b/Build/validate-cdn-conf.js new file mode 100644 index 00000000..c1673bfd --- /dev/null +++ b/Build/validate-cdn-conf.js @@ -0,0 +1,57 @@ +const psl = require('psl'); +const picocolors = require('picocolors'); + +const fs = require('fs'); +const path = require('path'); + +(async () => { + const domainSetContent = await fs.promises.readFile( + path.resolve(__dirname, '../List/domainset/cdn.conf'), + { encoding: 'utf-8' } + ); + + const domainSetLines = domainSetContent.split('\n'); + for (let i = 0, len = domainSetLines.length; i < len; i++) { + const line = domainSetLines[i]; + // starts with # + if (line.charCodeAt(0) === 35) { + continue; + } + if (line.trim().length === 0) { + continue; + } + + const domain = line.charCodeAt(0) === 46 ? line.slice(1) : line; + const parsed = psl.parse(domain); + + if (parsed.listed && parsed.input === parsed.tld) { + console.error('Domain', picocolors.yellow(domain), picocolors.red('is in public suffix list!')); + } + } + + const rulesetContent = await fs.promises.readFile( + path.resolve(__dirname, '../List/non_ip/cdn.conf'), + { encoding: 'utf-8' } + ); + const rulesetLines = rulesetContent.split('\n'); + + for (let i = 0, len = rulesetLines.length; i < len; i++) { + const line = rulesetLines[i]; + // starts with # + if (line.charCodeAt(0) === 35) { + continue; + } + if (line.trim().length === 0) { + continue; + } + + if (line.startsWith('DOMAIN-SUFFIX')) { + const domain = line.slice(14); + const parsed = psl.parse(domain); + + if (parsed.input !== parsed.tld) { + console.error('Domain', picocolors.yellow(domain), picocolors.green('is not in public suffix list!')); + } + } + } +})(); diff --git a/List/domainset/cdn.conf b/List/domainset/cdn.conf index 6451949c..0efd006d 100644 --- a/List/domainset/cdn.conf +++ b/List/domainset/cdn.conf @@ -123,7 +123,6 @@ static.crates.io # >> GitHub raw.github.com -.githubusercontent.com .ghcr.io .githubassets.com .pkg.github.com @@ -155,9 +154,6 @@ scontent.whatsapp.net # >> Disqus .disquscdn.com -# >> DigitalOcean OSS -.cdn.digitaloceanspaces.com - # >> Google CDN dl.google.com fonts.googleapis.com @@ -268,7 +264,6 @@ avatars.hubspot.net .yastatic.net avatars.mds.yandex.net resize.yandex.net -.storage.yandexcloud.net # >> Repl.it .cms.replit.com @@ -505,7 +500,6 @@ updates2.signal.org webfonts.zoho.com # >> IPFS Gateway -.dweb.link .gateway.ipfs.io .ipfs.eternum.io .cloudflare-ipfs.com @@ -640,6 +634,7 @@ cdn.glitch.com .quoracdn.net # GitBook .app.gitbook.com +.files.gitbook.io # Notion .notion-static.com # Roboflow @@ -758,6 +753,9 @@ s.rfi.fr # termius autoupdate.termius.com cdn.termius.com +# Fleek +storageapi.fleek.co +storageapi2.fleek.co # Others .v2ex.co cdn.v2ex.com @@ -808,7 +806,6 @@ static.cracked.to .cdn.ipinfo.io .assets.digitalocean.com ui-cdn.digitalocean.com -.vultrobjects.com ws-assets.zoominfo.com res2.weblium.site code.benco.io diff --git a/List/non_ip/cdn.conf b/List/non_ip/cdn.conf index 7584f89b..50abe596 100644 --- a/List/non_ip/cdn.conf +++ b/List/non_ip/cdn.conf @@ -1,5 +1,15 @@ # >> GitHub Pages DOMAIN-SUFFIX,github.io +# >> GitHub +DOMAIN-SUFFIX,githubusercontent.com +# >> DigitalOcean OSS +DOMAIN-SUFFIX,cdn.digitaloceanspaces.com +# >> Vultr OSS +DOMAIN-SUFFIX,vultrobjects.com +# >> Yandex +DOMAIN-SUFFIX,storage.yandexcloud.net +# >> dweb.link IPFS Gateway +DOMAIN-SUFFIX,dweb.link # >> GitLab Pages DOMAIN-SUFFIX,gitlab.io # >> Cloudflare CDN @@ -23,11 +33,6 @@ DOMAIN-SUFFIX,firebaseapp.com DOMAIN-SUFFIX,csb.app # >> Microsoft Azure DOMAIN-SUFFIX,azurestaticapps.net -# >> Fleek -DOMAIN,storageapi.fleek.co -DOMAIN,storageapi2.fleek.co -# >> GitBook -DOMAIN-KEYWORD,files.gitbook.io # >> AWS S3 DOMAIN-SUFFIX,s3.amazonaws.com # --- [AWS S3 Replace Me] --- diff --git a/package.json b/package.json index 7d3a755d..f0c9f9c1 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,8 @@ "build:index-html": "wireit", "build:reject-domainset": "wireit", "build:telegram-cidr": "wireit", - "build:chn-cidr": "wireit" + "build:chn-cidr": "wireit", + "validate:cdn-conf": "wireit" }, "wireit": { "build:anti-bogus-domain": { @@ -42,6 +43,9 @@ "build:telegram-cidr" ] }, + "validate:cdn-conf": { + "command": "node ./Build/validate-cdn-conf.js" + }, "build": { "dependencies": [ "build:anti-bogus-domain", @@ -50,7 +54,8 @@ "build:reject-domainset", "build:telegram-cidr", "build:chn-cidr", - "build:index-html" + "build:index-html", + "validate:cdn-conf" ] } }, @@ -66,6 +71,7 @@ "ci-info": "^3.5.0", "picocolors": "^1.0.0", "piscina": "^3.2.0", + "psl": "^1.9.0", "table": "^6.8.0", "undici": "5.11.0" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ef85f90c..35c509d3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6,6 +6,7 @@ specifiers: ci-info: ^3.5.0 picocolors: ^1.0.0 piscina: ^3.2.0 + psl: ^1.9.0 table: ^6.8.0 undici: 5.11.0 wireit: ^0.7.2 @@ -16,6 +17,7 @@ dependencies: ci-info: 3.5.0 picocolors: 1.0.0 piscina: 3.2.0 + psl: 1.9.0 table: 6.8.0 undici: 5.11.0 @@ -351,6 +353,10 @@ packages: signal-exit: 3.0.7 dev: true + /psl/1.9.0: + resolution: {integrity: sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==} + dev: false + /punycode/2.1.1: resolution: {integrity: sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==} engines: {node: '>=6'}