From 1a7638b1ec76236890eff039cfeb9a542b45883a Mon Sep 17 00:00:00 2001 From: SukkaW Date: Thu, 29 Jun 2023 14:36:15 +0800 Subject: [PATCH] Clean up --- .gitignore | 1 + Build/build-internal-rules.js | 57 ++++++++++++++++++++++++++ Build/download-previous-build.js | 69 ++++++++++++++++---------------- Build/validate-domainset.js | 4 +- List/domainset/my_proxy.conf | 50 +---------------------- List/non_ip/my_proxy.conf | 52 +++++++++++++++++++++++- List/non_ip/my_proxy_plus.conf | 5 --- package.json | 7 ++++ 8 files changed, 152 insertions(+), 93 deletions(-) create mode 100644 Build/build-internal-rules.js delete mode 100644 List/non_ip/my_proxy_plus.conf diff --git a/.gitignore b/.gitignore index edc9638a..643dbab5 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ List/non_ip/apple_cdn.conf List/ip/telegram.conf List/ip/reject.conf List/ip/china_ip.conf +List/internal/ diff --git a/Build/build-internal-rules.js b/Build/build-internal-rules.js new file mode 100644 index 00000000..381ba8cb --- /dev/null +++ b/Build/build-internal-rules.js @@ -0,0 +1,57 @@ +// @ts-check +const fs = require('fs'); +const fse = require('fs-extra'); +const path = require('path'); +const readline = require('readline'); +const { isDomainLoose } = require('./lib/is-domain-loose'); + +(async () => { + const results = []; + + for await ( + const line of readline.createInterface({ + input: fs.createReadStream(path.resolve(__dirname, '../List/non_ip/cdn.conf')), + crlfDelay: Infinity + }) + ) { + if (line.startsWith('DOMAIN-SUFFIX,')) { + results.push(line.replace('DOMAIN-SUFFIX,', 'SUFFIX,')); + } else if (line.startsWith('DOMAIN,')) { + results.push(line.replace('DOMAIN,', 'SUFFIX,')); + } + } + + for await ( + const line of readline.createInterface({ + input: fs.createReadStream(path.resolve(__dirname, '../List/domainset/cdn.conf')), + crlfDelay: Infinity + }) + ) { + if (line[0] === '.') { + results.push(`SUFFIX,${line.slice(1)}`); + } else if (isDomainLoose(line)) { + results.push(`SUFFIX,${line}`); + } + } + + for await ( + const line of readline.createInterface({ + input: fs.createReadStream(path.resolve(__dirname, '../List/domainset/download.conf')), + crlfDelay: Infinity + }) + ) { + if (line[0] === '.') { + results.push(`SUFFIX,${line.slice(1)}`); + } else if (isDomainLoose(line)) { + results.push(`SUFFIX,${line}`); + } + } + + results.push(''); + + await fse.ensureDir(path.resolve(__dirname, '../List/internal')); + await fs.promises.writeFile( + path.resolve(__dirname, '../List/internal/cdn.csv'), + results.join('\n') + ); +})(); diff --git a/Build/download-previous-build.js b/Build/download-previous-build.js index 7793b48a..f7041f9c 100644 --- a/Build/download-previous-build.js +++ b/Build/download-previous-build.js @@ -14,46 +14,45 @@ const fileExists = (path) => { }; (async () => { - const filesList = ( - await fs.promises.readFile(resolve(__dirname, '../.gitignore'), { encoding: 'utf-8' }) - ) + const filesList = (await fs.promises.readFile(resolve(__dirname, '../.gitignore'), { encoding: 'utf-8' })) .split('\n') - .filter(p => p.startsWith('List/')); + .filter(p => p.startsWith('List/') && !p.endsWith('/')); if ( - (await Promise.all( + !((await Promise.all( filesList.map(p => fileExists(join(__dirname, '..', p))) - )).some(exist => !exist) + )).some(exist => !exist)) ) { - const tempFile = join(tmpdir(), `sukka-surge-last-build-tar-${Date.now()}`); - const resp = await fetch('https://codeload.github.com/sukkaw/surge/tar.gz/gh-pages'); - const readableNodeStream = Readable.fromWeb(resp.body); - await pipeline( - readableNodeStream, - fs.createWriteStream(tempFile) - ); - - const extractedPath = join(tmpdir(), `sukka-surge-last-build-extracted-${Date.now()}`); - await fse.ensureDir(extractedPath); - await tar.x({ - file: tempFile, - cwd: extractedPath, - filter: (p) => { - return p.split('/')[1] === 'List' - } - }); - - await Promise.all(filesList.map(p => fse.copy( - join(extractedPath, 'Surge-gh-pages', p), - join(__dirname, '..', p), - { - overwrite: true - } - ))) - - await fs.promises.unlink(tempFile).catch(() => { }); - await fs.promises.unlink(extractedPath).catch(() => { }); - } else { console.log('All files exists, skip download.'); + return; } + + const tempFile = join(tmpdir(), `sukka-surge-last-build-tar-${Date.now()}`); + const resp = await fetch('https://codeload.github.com/sukkaw/surge/tar.gz/gh-pages'); + const readableNodeStream = Readable.fromWeb(resp.body); + await pipeline( + readableNodeStream, + fs.createWriteStream(tempFile) + ); + + const extractedPath = join(tmpdir(), `sukka-surge-last-build-extracted-${Date.now()}`); + await fse.ensureDir(extractedPath); + await tar.x({ + file: tempFile, + cwd: extractedPath, + filter: (p) => { + return p.split('/')[1] === 'List' + } + }); + + await Promise.all(filesList.map(p => fse.copy( + join(extractedPath, 'Surge-gh-pages', p), + join(__dirname, '..', p), + { + overwrite: true + } + ))) + + await fs.promises.unlink(tempFile).catch(() => { }); + await fs.promises.unlink(extractedPath).catch(() => { }); })(); diff --git a/Build/validate-domainset.js b/Build/validate-domainset.js index 7844f303..9adad984 100644 --- a/Build/validate-domainset.js +++ b/Build/validate-domainset.js @@ -81,9 +81,7 @@ const validateRuleset = async (filePath) => { ]); await Promise.all( domainsetFiles.map(file => validateDomainSet(file)) + // rulesetFiles.map(file => validateRuleset(file)) ); - // await Promise.all( - // rulesetFiles.map(file => validateRuleset(file)) - // ); })(); diff --git a/List/domainset/my_proxy.conf b/List/domainset/my_proxy.conf index bcf365a0..b6141db0 100644 --- a/List/domainset/my_proxy.conf +++ b/List/domainset/my_proxy.conf @@ -1,49 +1 @@ -.mikuclub.xyz -.mikuclub.cn -.saucenao.com -.fork.dev -.nextdns.io -.ibb.co -.sku.moe -.sukeycz.edu.kg -.services.googleapis.cn -.hostloc.com -.t66y.com -.zendesk.com -.influxdata.com - -.services.mozilla.com -vscode-sync.trafficmanager.net -.cloudflare.com -.wakatime.com - -.grammarly.com -.1password.com -.1password.eu -.1password.ca - -.surge-activation.com -.yandex.com - -ip.istatmenus.app -.exp-tas.com -.firefox.com -.visualstudio.com - -.azurecontainer.io -.azurewebsites.net -.azure-mobile.net -.cloudapp.net -.azurestaticapps.net - -.elasticbeanstalk.com -.compute.amazonaws.com -.compute-1.amazonaws.com - -dash.cloudflare.com -.he.net - -.webpkgcache.com -vscodeexperiments.azureedge.net - -.gitlab.com +# deprecated diff --git a/List/non_ip/my_proxy.conf b/List/non_ip/my_proxy.conf index 6088f87e..95bc8426 100644 --- a/List/non_ip/my_proxy.conf +++ b/List/non_ip/my_proxy.conf @@ -1 +1,51 @@ -# Deprecated +DOMAIN_SUFFIX,mikuclub.xyz +DOMAIN_SUFFIX,mikuclub.cn +DOMAIN_SUFFIX,saucenao.com +DOMAIN_SUFFIX,fork.dev +DOMAIN_SUFFIX,nextdns.io +DOMAIN_SUFFIX,ibb.co +DOMAIN_SUFFIX,sku.moe +DOMAIN_SUFFIX,sukeycz.edu.kg +DOMAIN_SUFFIX,services.googleapis.cn +DOMAIN_SUFFIX,hostloc.com +DOMAIN_SUFFIX,t66y.com +DOMAIN_SUFFIX,zendesk.com +DOMAIN_SUFFIX,influxdata.com + +DOMAIN_SUFFIX,services.mozilla.com +DOMAIN_SUFFIX,cloudflare.com +DOMAIN_SUFFIX,wakatime.com + +DOMAIN_SUFFIX,grammarly.com +DOMAIN_SUFFIX,1password.com +DOMAIN_SUFFIX,1password.eu +DOMAIN_SUFFIX,1password.ca + +DOMAIN_SUFFIX,surge-activation.com +DOMAIN_SUFFIX,yandex.com + +DOMAIN,ip.istatmenus.app +DOMAIN_SUFFIX,exp-tas.com +DOMAIN_SUFFIX,firefox.com +DOMAIN_SUFFIX,visualstudio.com + +DOMAIN_SUFFIX,azurecontainer.io +DOMAIN_SUFFIX,azurewebsites.net +DOMAIN_SUFFIX,azure-mobile.net +DOMAIN_SUFFIX,cloudapp.net +DOMAIN_SUFFIX,azurestaticapps.net + +DOMAIN_SUFFIX,elasticbeanstalk.com +DOMAIN_SUFFIX,compute.amazonaws.com +DOMAIN_SUFFIX,compute-1.amazonaws.com + +DOMAIN_SUFFIX,cloudflare.com +DOMAIN_SUFFIX,he.net + +DOMAIN_SUFFIX,webpkgcache.com +DOMAIN,vscodeexperiments.azureedge.net + +DOMAIN_SUFFIX,gitlab.com + +DOMAIN_SUFFIX,bgp.tools +DOMAIN_SUFFIX,setapp.com diff --git a/List/non_ip/my_proxy_plus.conf b/List/non_ip/my_proxy_plus.conf deleted file mode 100644 index 241925e2..00000000 --- a/List/non_ip/my_proxy_plus.conf +++ /dev/null @@ -1,5 +0,0 @@ -DOMAIN-SUFFIX,tailwindcss.com -DOMAIN-SUFFIX,ytoo.li -DOMAIN-SUFFIX,flowercloud.net -DOMAIN-KEYWORD,blogspot -DOMAIN-SUFFIX,codeberg.org diff --git a/package.json b/package.json index ced083c3..f883e75e 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "build:phishing-domainset": "wireit", "build:telegram-cidr": "wireit", "build:chn-cidr": "wireit", + "build:internal-rules": "wireit", "validate:domainset": "wireit" }, "wireit": { @@ -82,6 +83,12 @@ "build:cdn-conf" ] }, + "build:internal-rules": { + "command": "node ./Build/build-internal-rules.js", + "dependencies": [ + "build:cdn-conf" + ] + }, "build": { "dependencies": [ "build:anti-bogus-domain",