New build infra

This commit is contained in:
SukkaW 2022-12-08 18:10:59 +08:00
parent 6646a1ec98
commit ffb4e24e8a
14 changed files with 353 additions and 78 deletions

View File

@ -13,7 +13,7 @@ jobs:
strategy:
matrix:
node-version:
- 16.x
- 18.x
steps:
- name: Checkout

1
.node-version Normal file
View File

@ -0,0 +1 @@
18

View File

@ -2,6 +2,8 @@ const { fetchWithRetry } = require('./lib/fetch-retry');
const fs = require('fs');
const path = require('path');
const { isIPv4, isIPv6 } = require('net');
const { compareAndWriteFile } = require('./lib/string-array-compare');
const { withBannerArray } = require('./lib/with-banner');
(async () => {
console.time('Total Time - build-anti-bogus-domain');
@ -35,6 +37,24 @@ const { isIPv4, isIPv6 } = require('net');
}).join('\n')
);
await fs.promises.writeFile(resultPath, content, 'utf-8');
await compareAndWriteFile(
withBannerArray(
'Sukka\'s Surge Rules - Telegram IP CIDR',
[
'License: AGPL 3.0',
'Homepage: https://ruleset.skk.moe',
'GitHub: https://github.com/SukkaW/Surge',
'',
'This file contains known addresses that are hijacking NXDOMAIN results returned by DNS servers.',
'',
'Data from:',
' - https://github.com/felixonmars/dnsmasq-china-list'
],
new Date(),
content.split('\n')
),
resultPath
)
console.timeEnd('Total Time - build-anti-bogus-domain');
})();

View File

@ -3,6 +3,8 @@ const fs = require('fs');
const path = require('path');
const { isDomainLoose } = require('./lib/is-domain-loose');
const { compareAndWriteFile } = require('./lib/string-array-compare');
const { withBannerArray } = require('./lib/with-banner');
(async () => {
console.time('Total Time - build-apple-cdn-conf');
@ -19,15 +21,41 @@ const { isDomainLoose } = require('./lib/is-domain-loose');
.filter(domain => typeof domain === 'string' && isDomainLoose(domain));
await Promise.all([
fs.promises.writeFile(
path.resolve(__dirname, '../List/non_ip/apple_cdn.conf'),
res.map(domain => `DOMAIN-SUFFIX,${domain}`).join('\n') + '\n',
'utf-8'
compareAndWriteFile(
withBannerArray(
'Sukka\'s Surge Rules - Apple CDN',
[
'License: AGPL 3.0',
'Homepage: https://ruleset.skk.moe',
'GitHub: https://github.com/SukkaW/Surge',
'',
'This file contains Apple\'s domains using their China mainland CDN servers.',
'',
'Data from:',
' - https://github.com/felixonmars/dnsmasq-china-list',
],
new Date(),
res.map(domain => `DOMAIN-SUFFIX,${domain}`)
),
path.resolve(__dirname, '../List/non_ip/apple_cdn.conf')
),
fs.promises.writeFile(
path.resolve(__dirname, '../List/domainset/apple_cdn.conf'),
res.map(i => `.${i}`).join('\n') + '\n',
'utf-8'
compareAndWriteFile(
withBannerArray(
'Sukka\'s Surge Rules - Apple CDN',
[
'License: AGPL 3.0',
'Homepage: https://ruleset.skk.moe',
'GitHub: https://github.com/SukkaW/Surge',
'',
'This file contains Apple\'s domains using their China mainland CDN servers.',
'',
'Data from:',
' - https://github.com/felixonmars/dnsmasq-china-list',
],
new Date(),
res.map(i => `.${i}`)
),
path.resolve(__dirname, '../List/domainset/apple_cdn.conf')
)
])

View File

@ -1,6 +1,8 @@
const { fetchWithRetry } = require('./lib/fetch-retry');
const fs = require('fs');
const path = require('path');
const { compareAndWriteFile } = require('./lib/string-array-compare');
const { withBannerArray } = require('./lib/with-banner');
(async () => {
console.time('Total Time - build-cdn-conf');
@ -31,7 +33,21 @@ const path = require('path');
S3OSSDomains.map(domain => `DOMAIN-SUFFIX,${domain}`).join('\n')
);
await fs.promises.writeFile(resultPath, content, 'utf-8');
await compareAndWriteFile(
withBannerArray(
'Sukka\'s Surge Rules - CDN Domains',
[
'License: AGPL 3.0',
'Homepage: https://ruleset.skk.moe',
'GitHub: https://github.com/SukkaW/Surge',
'',
'This file contains object storage and static assets CDN domains.'
],
new Date(),
content.split('\n')
),
resultPath
)
console.timeEnd('Total Time - build-cdn-conf');
})();

View File

@ -1,7 +1,7 @@
const { fetchWithRetry } = require('./lib/fetch-retry');
const { withBanner } = require('./lib/with-banner');
const { promises: fsPromises } = require('fs');
const { withBannerArray } = require('./lib/with-banner');
const { resolve: pathResolve } = require('path');
const { compareAndWriteFile } = require('./lib/string-array-compare');
(async () => {
console.time('Total Time - build-chnroutes-cidr');
@ -22,18 +22,21 @@ const { resolve: pathResolve } = require('path');
}));
console.log('After Merge:', filteredCidr.length);
await fsPromises.writeFile(pathResolve(__dirname, '../List/ip/china_ip.conf'), makeCidrList(filteredCidr), { encoding: 'utf-8' });
await compareAndWriteFile(
withBannerArray(
'Sukka\'s Surge Rules - Mainland China IPv4 CIDR',
[
'License: CC BY-SA 2.0',
'Homepage: https://ruleset.skk.moe',
'GitHub: https://github.com/SukkaW/Surge',
'',
'Data from https://misaka.io (misakaio @ GitHub)',
],
new Date(),
cidr.map(i => `IP-CIDR,${i}`)
),
pathResolve(__dirname, '../List/ip/china_ip.conf')
)
console.timeEnd('Total Time - build-chnroutes-cidr');
})();
function makeCidrList(cidr) {
const date = new Date();
return withBanner(
'Mainland China IPv4 CIDR',
['Data from misaka.io (misakaio @ GitHub)', 'License: CC BY-SA 2.0'],
date,
cidr.map(i => `IP-CIDR,${i}`)
);
};

View File

@ -2,7 +2,8 @@ const tldts = require('tldts');
const { processFilterRules } = require('./lib/parse-filter.js');
const fs = require('fs');
const path = require('path');
const { withBanner } = require('./lib/with-banner.js');
const { withBannerArray } = require('./lib/with-banner.js');
const { stringArrayCompare, compareAndWriteFile } = require('./lib/string-array-compare');
const WHITELIST_DOMAIN = new Set([
'w3s.link',
@ -97,19 +98,23 @@ const BLACK_TLD = Array.from(new Set([
}
});
const filePath = path.resolve(__dirname, '../List/domainset/reject_phishing.conf');
await fs.promises.writeFile(
filePath,
withBanner(
'Reject Domain Set for Surge',
results.sort();
await compareAndWriteFile(
withBannerArray(
'Sukka\'s Surge Rules - Reject Phishing',
[
'(Enhanced Phishing Protection)',
'License: AGPL 3.0',
'Homepage: https://ruleset.skk.moe',
'GitHub: https://github.com/SukkaW/Surge',
'',
'The domainset supports enhanced phishing protection',
'Build from:',
'- https://gitlab.com/malware-filter/phishing-filter'
' - https://gitlab.com/malware-filter/phishing-filter'
],
new Date(),
results
),
'utf-8'
);
path.resolve(__dirname, '../List/domainset/reject_phishing.conf')
)
})();

View File

@ -8,7 +8,8 @@ const { isCI } = require('ci-info');
const threads = isCI ? cpuCount : cpuCount / 2;
const { HOSTS, ADGUARD_FILTERS, PREDEFINED_WHITELIST } = require('./lib/reject-data-source');
const { withBanner } = require('./lib/with-banner');
const { withBannerArray } = require('./lib/with-banner');
const { compareAndWriteFile } = require('./lib/string-array-compare');
const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST);
@ -208,25 +209,30 @@ const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST);
console.log(`* Dedupe from covered subdomain - ${(Date.now() - START_TIME) / 1000}s`);
console.log(`Deduped ${previousSize - domainSets.size} rules!`);
await piscina.destroy();
console.time('* Write reject.conf');
await Promise.all([
fsPromises.writeFile(
pathResolve(__dirname, '../List/domainset/reject.conf'),
withBanner(
'Reject Domain Set for Surge',
[
'(AdBlock, Tracking Protection, Privacy Protection, Anti-Phishing, Anti-Mining)',
'Build from:',
...HOSTS.map(host => `- ${host[0]}`),
...ADGUARD_FILTERS.map(filter => `- ${Array.isArray(filter) ? filter[0] : filter}`),
],
new Date(),
[...domainSets].sort()
),
{ encoding: 'utf-8' }
await compareAndWriteFile(
withBannerArray(
'Sukka\'s Surge Rules - Reject Base',
[
'License: AGPL 3.0',
'Homepage: https://ruleset.skk.moe',
'GitHub: https://github.com/SukkaW/Surge',
'',
'The domainset supports AD blocking, tracking protection, privacy protection, anti-phishing, anti-mining',
'',
'Build from:',
...HOSTS.map(host => ` - ${host[0]}`),
...ADGUARD_FILTERS.map(filter => ` - ${Array.isArray(filter) ? filter[0] : filter}`),
],
new Date(),
[...domainSets].sort()
),
piscina.destroy()
]);
pathResolve(__dirname, '../List/domainset/reject.conf')
);
console.timeEnd('* Write reject.conf');
console.timeEnd('Total Time - build-reject-domain-set');

View File

@ -2,12 +2,16 @@ const { fetchWithRetry } = require('./lib/fetch-retry');
const fs = require('fs');
const path = require('path');
const { isIPv4, isIPv6 } = require('net');
const { compareAndWriteFile } = require('./lib/string-array-compare');
const { withBannerArray } = require('./lib/with-banner');
(async () => {
console.time('Total Time - build-telegram-cidr');
/** @type {Response} */
const resp = await fetchWithRetry('https://core.telegram.org/resources/cidr.txt');
const lastModified = new Date(resp.headers.get('last-modified'));
const lastModified = resp.headers.get('last-modified');
const date = lastModified ? new Date(lastModified) : new Date();
const res = (await resp.text())
.split('\n')
@ -17,23 +21,31 @@ const { isIPv4, isIPv6 } = require('net');
throw new Error('Failed to fetch data!');
}
await fs.promises.writeFile(
path.resolve(__dirname, '../List/ip/telegram.conf'),
'# Telegram CIDR (https://core.telegram.org/resources/cidr.txt)' + '\n' +
'# Last Updated: ' + lastModified.toISOString() + '\n' +
res.map(ip => {
const [subnet] = ip.split('/');
console.log(' - ' + ip + ': ' + subnet);
if (isIPv4(subnet)) {
return `IP-CIDR,${ip},no-resolve`;
}
if (isIPv6(subnet)) {
return `IP-CIDR6,${ip},no-resolve`;
}
return '';
}).join('\n') + '\n',
'utf-8'
);
await compareAndWriteFile(
withBannerArray(
'Sukka\'s Surge Rules - Telegram IP CIDR',
[
'License: AGPL 3.0',
'Homepage: https://ruleset.skk.moe',
'GitHub: https://github.com/SukkaW/Surge',
'Data from:',
' - https://core.telegram.org/resources/cidr.txt'
],
date,
res.map(ip => {
const [subnet] = ip.split('/');
console.log(' - ' + ip + ': ' + subnet);
if (isIPv4(subnet)) {
return `IP-CIDR,${ip},no-resolve`;
}
if (isIPv6(subnet)) {
return `IP-CIDR6,${ip},no-resolve`;
}
return '';
})
),
path.resolve(__dirname, '../List/ip/telegram.conf')
)
console.timeEnd('Total Time - build-telegram-cidr');
})();

View File

@ -0,0 +1,46 @@
const { fetch } = require('undici');
const tar = require('tar');
const fs = require('fs');
const fse = require('fs-extra');
const { join, resolve } = require('path');
const { tmpdir } = require('os');
const { Stream, Readable } = require('stream');
const { promisify } = require('util');
const pipeline = promisify(Stream.pipeline);
(async () => {
const filesList = (
await fs.promises.readFile(resolve(__dirname, '../.gitignore'), { encoding: 'utf-8' })
)
.split('\n')
.filter(p => p.startsWith('List/'));
const tempFile = join(tmpdir(), `sukka-surge-last-build-tar-${Date.now()}`);
const resp = await fetch('https://codeload.github.com/sukkaw/surge/tar.gz/gh-pages');
const readableNodeStream = Readable.fromWeb(resp.body);
await pipeline(
readableNodeStream,
fs.createWriteStream(tempFile)
);
const extractedPath = join(tmpdir(), `sukka-surge-last-build-extracted-${Date.now()}`);
await fse.ensureDir(extractedPath);
await tar.x({
file: tempFile,
cwd: extractedPath,
filter: (p) => {
return p.split('/')[1] === 'List'
}
});
await Promise.all(filesList.map(p => fse.copy(
join(extractedPath, 'Surge-gh-pages', p),
join(__dirname, '..', p),
{
overwrite: true
}
)))
await fs.promises.unlink(tempFile).catch(() => {});
await fs.promises.unlink(extractedPath).catch(() => {});
})();

View File

@ -0,0 +1,35 @@
const { promises: fsPromises } = require('fs');
async function compareAndWriteFile(linesA, filePath) {
const linesB = (await fsPromises.readFile(filePath, { encoding: 'utf-8' })).split('\n');
if (!stringArrayCompare(linesA, linesB)) {
await fsPromises.writeFile(
filePath,
linesA.join('\n'),
{ encoding: 'utf-8' }
)
} else {
console.log(`Same Content, bail out writing: ${filePath}`);
}
}
function stringArrayCompare (linesA, linesB) {
if (linesA.length !== linesB.length) return false;
for (let i = 0; i < linesA.length; i++) {
const lineA = linesA[i];
const lineB = linesB[i];
if (lineA.startsWith('#') && lineB.startsWith('#')) {
continue;
}
if (lineA !== lineB) {
return false;
}
}
return true;
}
module.exports.stringArrayCompare = stringArrayCompare;
module.exports.compareAndWriteFile = compareAndWriteFile;

View File

@ -6,12 +6,33 @@
* @returns {string}
*/
const withBanner = (title, description, date, content) => {
return `########################################
return `########################################
# ${title}
${description.map(line => `# ${line}`).join('\n')}
# Last Updated: ${date.toISOString()}
# Size: ${content.length}
${description.map(line => line ? `# ${line}` : '#').join('\n')}
########################################\n` + content.join('\n') + '\n################# END ###################\n';
};
/**
* @param {string} title
* @param {string[]} description
* @param {Date} date
* @param {string[]} content
* @returns {string[]}
*/
const withBannerArray = (title, description, date, content) => {
return [
`########################################`,
`# ${title}`,
`# Last Updated: ${date.toISOString()}`,
`# Size: ${content.length}`,
...description.map(line => line ? `# ${line}` : '#'),
`########################################`,
...content,
`################# END ###################`,
''
];
};
module.exports.withBanner = withBanner;
module.exports.withBannerArray = withBannerArray;

View File

@ -5,6 +5,7 @@
"description": "",
"scripts": {
"build": "wireit",
"download-previous-build": "wireit",
"build:anti-bogus-domain": "wireit",
"build:apple-cdn": "wireit",
"build:cdn-conf": "wireit",
@ -16,21 +17,37 @@
"validate:domainset": "wireit"
},
"wireit": {
"download-previous-build": {
"command": "node ./Build/download-previous-build.js"
},
"build:anti-bogus-domain": {
"command": "node ./Build/build-anti-bogus-domain.js"
"command": "node ./Build/build-anti-bogus-domain.js",
"dependencies": [
"download-previous-build"
]
},
"build:apple-cdn": {
"command": "node ./Build/build-apple-cdn.js"
"command": "node ./Build/build-apple-cdn.js",
"dependencies": [
"download-previous-build"
]
},
"build:cdn-conf": {
"command": "node ./Build/build-cdn-conf.js"
"command": "node ./Build/build-cdn-conf.js",
"dependencies": [
"download-previous-build"
]
},
"build:phishing-domainset": {
"command": "node ./Build/build-phishing-domainset.js"
"command": "node ./Build/build-phishing-domainset.js",
"dependencies": [
"download-previous-build"
]
},
"build:reject-domainset": {
"command": "node ./Build/build-reject-domainset.js",
"dependencies": [
"download-previous-build",
"build:phishing-domainset"
]
},
@ -38,7 +55,10 @@
"command": "node ./Build/build-telegram-cidr.js"
},
"build:chn-cidr": {
"command": "node ./Build/build-chn-cidr.js"
"command": "node ./Build/build-chn-cidr.js",
"dependencies": [
"download-previous-build"
]
},
"build:public": {
"command": "node ./Build/build-public.js",
@ -87,10 +107,14 @@
"picocolors": "^1.0.0",
"piscina": "^3.2.0",
"table": "^6.8.1",
"tar": "^6.1.13",
"tldts": "^5.7.102",
"undici": "5.13.0"
},
"devDependencies": {
"wireit": "^0.9.1"
},
"engines": {
"node": "> 18.0.0"
}
}

58
pnpm-lock.yaml generated
View File

@ -9,6 +9,7 @@ specifiers:
picocolors: ^1.0.0
piscina: ^3.2.0
table: ^6.8.1
tar: ^6.1.13
tldts: ^5.7.102
undici: 5.13.0
wireit: ^0.9.1
@ -22,6 +23,7 @@ dependencies:
picocolors: 1.0.0
piscina: 3.2.0
table: 6.8.1
tar: 6.1.13
tldts: 5.7.102
undici: 5.13.0
@ -148,6 +150,11 @@ packages:
fsevents: 2.3.2
dev: true
/chownr/2.0.0:
resolution: {integrity: sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==}
engines: {node: '>=10'}
dev: false
/ci-info/3.7.0:
resolution: {integrity: sha512-2CpRNYmImPx+RXKLq6jko/L07phmS9I02TyqkcNU20GCF/GgaWvc58hPtjxDX8lPpkdwc9sNh72V9k00S7ezog==}
engines: {node: '>=8'}
@ -238,6 +245,13 @@ packages:
universalify: 2.0.0
dev: false
/fs-minipass/2.1.0:
resolution: {integrity: sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==}
engines: {node: '>= 8'}
dependencies:
minipass: 3.3.6
dev: false
/fsevents/2.3.2:
resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==}
engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0}
@ -342,6 +356,34 @@ packages:
picomatch: 2.3.1
dev: true
/minipass/3.3.6:
resolution: {integrity: sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==}
engines: {node: '>=8'}
dependencies:
yallist: 4.0.0
dev: false
/minipass/4.0.0:
resolution: {integrity: sha512-g2Uuh2jEKoht+zvO6vJqXmYpflPqzRBT+Th2h01DKh5z7wbY/AZ2gCQ78cP70YoHPyFdY30YBV5WxgLOEwOykw==}
engines: {node: '>=8'}
dependencies:
yallist: 4.0.0
dev: false
/minizlib/2.1.2:
resolution: {integrity: sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==}
engines: {node: '>= 8'}
dependencies:
minipass: 3.3.6
yallist: 4.0.0
dev: false
/mkdirp/1.0.4:
resolution: {integrity: sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==}
engines: {node: '>=10'}
hasBin: true
dev: false
/ms/2.1.2:
resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==}
dev: false
@ -494,6 +536,18 @@ packages:
strip-ansi: 6.0.1
dev: false
/tar/6.1.13:
resolution: {integrity: sha512-jdIBIN6LTIe2jqzay/2vtYLlBHa3JF42ot3h1dW8Q0PaAG4v8rm0cvpVePtau5C6OKXGGcgO9q2AMNSWxiLqKw==}
engines: {node: '>=10'}
dependencies:
chownr: 2.0.0
fs-minipass: 2.1.0
minipass: 4.0.0
minizlib: 2.1.2
mkdirp: 1.0.4
yallist: 4.0.0
dev: false
/tldts-core/5.7.102:
resolution: {integrity: sha512-EFhrqqb0abo/9KZ4ZxHSF4e8CYKk/SouOLbNUMZUw+5OjC0hUJbFmHXtHNWYGaqnmwjnvdwFQ1a29Tflpz5Ufg==}
dev: false
@ -541,3 +595,7 @@ packages:
jsonc-parser: 3.0.0
proper-lockfile: 4.1.2
dev: true
/yallist/4.0.0:
resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==}
dev: false