GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2026-01-29 01:51:52 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Add build step to CDN domainset

Browse Source
This commit is contained in:
SukkaW
2023-07-13 22:18:53 +08:00
parent 9fde4a3866
commit b43c1628d6
7 changed files with 128 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
Build/build-cdn-conf.js
View File

@@ -1,18 +1,28 @@
const { fetchWithRetry } = require('./lib/fetch-retry');
// @ts-check
const fs = require('fs');
const path = require('path');
const { compareAndWriteFile } = require('./lib/string-array-compare');
const { withBannerArray } = require('./lib/with-banner');
const { minifyRules } = require('./lib/minify-rules');
const { domainDeduper } = require('./lib/domain-deduper');
const { shouldIgnoreLine } = require('./lib/should-ignore-line');
const { fetchRemoteTextAndCreateReadlineInterface } = require('./lib/fetch-remote-text-by-line');
const readline = require('readline');
(async () => {
console.time('Total Time - build-cdn-conf');
const domains = (await (await fetchWithRetry('https://publicsuffix.org/list/public_suffix_list.dat')).text()).split('\n');
/**
* Extract OSS domain from publicsuffix list
* @type {Set<string>}
*/
const S3OSSDomains = new Set();
const S3OSSDomains = domains.filter(line => {
if (line) {
return (
for await (const line of await fetchRemoteTextAndCreateReadlineInterface('https://publicsuffix.org/list/public_suffix_list.dat')) {
if (
line
&& (
line.startsWith('s3-')
|| line.startsWith('s3.')
)
@@ -20,18 +30,16 @@ const { minifyRules } = require('./lib/minify-rules');
line.endsWith('.amazonaws.com')
|| line.endsWith('.scw.cloud')
)
&& !line.includes('cn-');
&& !line.includes('cn-')
) {
S3OSSDomains.add(line);
}
}
return false;
});
const filePath = path.resolve(__dirname, '../Source/non_ip/cdn.conf');
const resultPath = path.resolve(__dirname, '../List/non_ip/cdn.conf');
const content = (await fs.promises.readFile(filePath, 'utf-8'))
const content = (await fs.promises.readFile(path.resolve(__dirname, '../Source/non_ip/cdn.conf'), 'utf-8'))
.replace(
'# --- [AWS S3 Replace Me] ---',
S3OSSDomains.map(domain => `DOMAIN-SUFFIX,${domain}`).join('\n')
Array.from(S3OSSDomains).map(domain => `DOMAIN-SUFFIX,${domain}`).join('\n')
);
await compareAndWriteFile(
@@ -47,7 +55,39 @@ const { minifyRules } = require('./lib/minify-rules');
new Date(),
minifyRules(content.split('\n'))
),
resultPath
path.resolve(__dirname, '../List/non_ip/cdn.conf')
);
/**
* Dedupe cdn.conf
*/
/** @type {Set<string>} */
const cdnDomains = new Set();
for await (const line of readline.createInterface({
input: fs.createReadStream(path.resolve(__dirname, '../Source/domainset/cdn.conf'), 'utf-8'),
crlfDelay: Infinity
})) {
const l = shouldIgnoreLine(line);
if (l) {
cdnDomains.add(l);
}
}
await compareAndWriteFile(
withBannerArray(
'Sukka\'s Surge Rules - CDN Domains',
[
'License: AGPL 3.0',
'Homepage: https://ruleset.skk.moe',
'GitHub: https://github.com/SukkaW/Surge',
'',
'This file contains object storage and static assets CDN domains.'
],
new Date(),
minifyRules(domainDeduper(Array.from(cdnDomains)))
),
path.resolve(__dirname, '../List/domainset/cdn.conf')
);
console.timeEnd('Total Time - build-cdn-conf');