Simplify build infra

This commit is contained in:
SukkaW 2023-09-11 12:02:17 +08:00
parent b2745c7245
commit 96af39c40d
13 changed files with 210 additions and 350 deletions

View File

@ -1,10 +1,8 @@
// @ts-check
const path = require('path');
const { isIPv4, isIPv6 } = require('net');
const { compareAndWriteFile } = require('./lib/string-array-compare');
const { withBannerArray } = require('./lib/with-banner');
const { createRuleset } = require('./lib/create-file');
const { fetchRemoteTextAndCreateReadlineInterface, readFileByLine } = require('./lib/fetch-remote-text-by-line');
const { surgeRulesetToClashClassicalTextRuleset } = require('./lib/clash');
const { processLine } = require('./lib/process-line');
(async () => {
@ -53,36 +51,15 @@ const { processLine } = require('./lib/process-line');
' - https://github.com/felixonmars/dnsmasq-china-list'
];
await compareAndWriteFile(
withBannerArray(
await Promise.all(createRuleset(
'Sukka\'s Ruleset - Anti Bogus Domain',
description,
new Date(),
surgeRulesetToClashClassicalTextRuleset(result)
),
result,
'ruleset',
path.resolve(__dirname, '../List/ip/reject.conf'),
path.resolve(__dirname, '../Clash/ip/reject.txt')
);
await Promise.all([
compareAndWriteFile(
withBannerArray(
'Sukka\'s Ruleset - Anti Bogus Domain',
description,
new Date(),
result
),
path.resolve(__dirname, '../List/ip/reject.conf')
),
compareAndWriteFile(
withBannerArray(
'Sukka\'s Ruleset - Anti Bogus Domain',
description,
new Date(),
surgeRulesetToClashClassicalTextRuleset(result)
),
path.resolve(__dirname, '../Clash/ip/reject.txt')
)
]);
));
console.timeEnd('Total Time - build-anti-bogus-domain');
})();

View File

@ -1,10 +1,6 @@
const path = require('path');
const { compareAndWriteFile } = require('./lib/string-array-compare');
const { withBannerArray } = require('./lib/with-banner');
const { createRuleset } = require('./lib/create-file');
const { parseFelixDnsmasq } = require('./lib/parse-dnsmasq');
const { surgeRulesetToClashClassicalTextRuleset, surgeDomainsetToClashDomainset } = require('./lib/clash');
(async () => {
console.time('Total Time - build-apple-cdn-conf');
@ -26,40 +22,22 @@ const { surgeRulesetToClashClassicalTextRuleset, surgeDomainsetToClashDomainset
const domainset = res.map(i => `.${i}`);
await Promise.all([
compareAndWriteFile(
withBannerArray(
...createRuleset(
'Sukka\'s Ruleset - Apple CDN',
description,
new Date(),
ruleset
),
path.resolve(__dirname, '../List/non_ip/apple_cdn.conf')
),
compareAndWriteFile(
withBannerArray(
'Sukka\'s Ruleset - Apple CDN',
description,
new Date(),
surgeRulesetToClashClassicalTextRuleset(ruleset)
),
ruleset,
'ruleset',
path.resolve(__dirname, '../List/non_ip/apple_cdn.conf'),
path.resolve(__dirname, '../Clash/non_ip/apple_cdn.txt')
),
compareAndWriteFile(
withBannerArray(
...createRuleset(
'Sukka\'s Ruleset - Apple CDN',
description,
new Date(),
domainset
),
path.resolve(__dirname, '../List/domainset/apple_cdn.conf')
),
compareAndWriteFile(
withBannerArray(
'Sukka\'s Ruleset - Apple CDN',
description,
new Date(),
surgeDomainsetToClashDomainset(domainset)
),
domainset,
'domainset',
path.resolve(__dirname, '../List/domainset/apple_cdn.conf'),
path.resolve(__dirname, '../Clash/domainset/apple_cdn.txt')
)
]);

View File

@ -1,11 +1,9 @@
// @ts-check
const path = require('path');
const { compareAndWriteFile } = require('./lib/string-array-compare');
const { withBannerArray } = require('./lib/with-banner');
const { createRuleset } = require('./lib/create-file');
const { minifyRules } = require('./lib/minify-rules');
const { fetchRemoteTextAndCreateReadlineInterface, readFileByLine } = require('./lib/fetch-remote-text-by-line');
const Trie = require('./lib/trie');
const { surgeRulesetToClashClassicalTextRuleset } = require('./lib/clash');
(async () => {
console.time('Total Time - build-cdn-conf');
@ -48,26 +46,15 @@ const { surgeRulesetToClashClassicalTextRuleset } = require('./lib/clash');
];
const ruleset = minifyRules(cdnDomainsList);
await Promise.all([
compareAndWriteFile(
withBannerArray(
await Promise.all(createRuleset(
'Sukka\'s Ruleset - CDN Domains',
description,
new Date(),
ruleset
),
path.resolve(__dirname, '../List/non_ip/cdn.conf')
),
compareAndWriteFile(
withBannerArray(
'Sukka\'s Ruleset - CDN Domains',
description,
new Date(),
surgeRulesetToClashClassicalTextRuleset(ruleset)
),
ruleset,
'ruleset',
path.resolve(__dirname, '../List/non_ip/cdn.conf'),
path.resolve(__dirname, '../Clash/non_ip/cdn.txt')
)
]);
));
console.timeEnd('Total Time - build-cdn-conf');
})();

View File

@ -1,8 +1,8 @@
// @ts-check
const { fetchRemoteTextAndCreateReadlineInterface } = require('./lib/fetch-remote-text-by-line');
const { withBannerArray } = require('./lib/with-banner');
const { resolve: pathResolve } = require('path');
const { compareAndWriteFile } = require('./lib/string-array-compare');
// This should not use `createRuleset` API since we are going to generate ipcidr for Clash
const { compareAndWriteFile, withBannerArray } = require('./lib/create-file');
const { processLine } = require('./lib/process-line');
// https://github.com/misakaio/chnroutes2/issues/25
@ -28,17 +28,19 @@ const EXCLUDE_CIDRS = [
const filteredCidr = excludeCidrs(Array.from(cidr), EXCLUDE_CIDRS, true);
console.log('After Merge:', filteredCidr.length);
await Promise.all([
compareAndWriteFile(
withBannerArray(
'Sukka\'s Ruleset - Mainland China IPv4 CIDR',
[
const description = [
'License: CC BY-SA 2.0',
'Homepage: https://ruleset.skk.moe',
'GitHub: https://github.com/SukkaW/Surge',
'',
'Data from https://misaka.io (misakaio @ GitHub)'
],
];
await Promise.all([
compareAndWriteFile(
withBannerArray(
'Sukka\'s Ruleset - Mainland China IPv4 CIDR',
description,
new Date(),
filteredCidr.map(i => `IP-CIDR,${i}`)
),
@ -47,13 +49,7 @@ const EXCLUDE_CIDRS = [
compareAndWriteFile(
withBannerArray(
'Sukka\'s Ruleset - Mainland China IPv4 CIDR',
[
'License: CC BY-SA 2.0',
'Homepage: https://ruleset.skk.moe',
'GitHub: https://github.com/SukkaW/Surge',
'',
'Data from https://misaka.io (misakaio @ GitHub)'
],
description,
new Date(),
filteredCidr
),

View File

@ -3,10 +3,8 @@ const path = require('path');
const { DOMESTICS } = require('../Source/non_ip/domestic');
const { readFileByLine } = require('./lib/fetch-remote-text-by-line');
const { processLine } = require('./lib/process-line');
const { withBannerArray } = require('./lib/with-banner');
const { compareAndWriteFile } = require('./lib/string-array-compare');
const { compareAndWriteFile, createRuleset } = require('./lib/create-file');
const domainSorter = require('./lib/stable-sort-domain');
const { surgeRulesetToClashClassicalTextRuleset } = require('./lib/clash');
(async () => {
const rl = readFileByLine(path.resolve(__dirname, '../Source/non_ip/domestic.conf'));
@ -35,22 +33,13 @@ const { surgeRulesetToClashClassicalTextRuleset } = require('./lib/clash');
];
await Promise.all([
compareAndWriteFile(
withBannerArray(
...createRuleset(
'Sukka\'s Ruleset - Domestic Domains',
rulesetDescription,
new Date(),
results
),
path.resolve(__dirname, '../List/non_ip/domestic.conf')
),
compareAndWriteFile(
withBannerArray(
'Sukka\'s Ruleset - Domestic Domains',
rulesetDescription,
new Date(),
surgeRulesetToClashClassicalTextRuleset(results)
),
results,
'ruleset',
path.resolve(__dirname, '../List/non_ip/domestic.conf'),
path.resolve(__dirname, '../Clash/non_ip/domestic.txt')
),
compareAndWriteFile(

View File

@ -1,11 +1,9 @@
const { parse } = require('tldts');
const { processFilterRules } = require('./lib/parse-filter.js');
const path = require('path');
const { withBannerArray } = require('./lib/with-banner.js');
const { compareAndWriteFile } = require('./lib/string-array-compare');
const { createRuleset } = require('./lib/create-file');
const { processLine } = require('./lib/process-line.js');
const domainSorter = require('./lib/stable-sort-domain');
const { surgeDomainsetToClashDomainset } = require('./lib/clash.js');
const WHITELIST_DOMAIN = new Set([
'w3s.link',
@ -152,24 +150,13 @@ const BLACK_TLD = new Set([
' - https://gitlab.com/malware-filter/phishing-filter'
];
await Promise.all([
compareAndWriteFile(
withBannerArray(
await Promise.all(createRuleset(
'Sukka\'s Ruleset - Reject Phishing',
description,
new Date(),
results
),
path.resolve(__dirname, '../List/domainset/reject_phishing.conf')
),
compareAndWriteFile(
withBannerArray(
'Sukka\'s Ruleset - Reject Phishing',
description,
new Date(),
surgeDomainsetToClashDomainset(results)
),
results,
'domainset',
path.resolve(__dirname, '../List/domainset/reject_phishing.conf'),
path.resolve(__dirname, '../Clash/domainset/reject_phishing.txt')
)
]);
));
})();

View File

@ -9,14 +9,12 @@ const { processHosts, processFilterRules } = require('./lib/parse-filter');
const Trie = require('./lib/trie');
const { HOSTS, ADGUARD_FILTERS, PREDEFINED_WHITELIST, PREDEFINED_ENFORCED_BACKLIST } = require('./lib/reject-data-source');
const { withBannerArray } = require('./lib/with-banner');
const { compareAndWriteFile } = require('./lib/string-array-compare');
const { createRuleset } = require('./lib/create-file');
const { processLine } = require('./lib/process-line');
const { domainDeduper } = require('./lib/domain-deduper');
const createKeywordFilter = require('./lib/aho-corasick');
const { readFileByLine } = require('./lib/fetch-remote-text-by-line');
const domainSorter = require('./lib/stable-sort-domain');
const { surgeDomainsetToClashDomainset } = require('./lib/clash');
/** Whitelists */
const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST);
@ -210,22 +208,13 @@ const domainSuffixSet = new Set();
const domainset = dudupedDominArray.sort(domainSorter);
await Promise.all([
compareAndWriteFile(
withBannerArray(
...createRuleset(
'Sukka\'s Ruleset - Reject Base',
description,
new Date(),
domainset
),
pathResolve(__dirname, '../List/domainset/reject.conf')
),
compareAndWriteFile(
withBannerArray(
'Sukka\'s Ruleset - Reject Base',
description,
new Date(),
surgeDomainsetToClashDomainset(domainset)
),
domainset,
'domainset',
pathResolve(__dirname, '../List/domainset/reject.conf'),
pathResolve(__dirname, '../Clash/domainset/reject.txt')
),
fs.promises.writeFile(

View File

@ -1,12 +1,10 @@
const { fetch } = require('undici');
const { domainDeduper } = require('./lib/domain-deduper');
const path = require('path');
const { withBannerArray } = require('./lib/with-banner.js');
const { compareAndWriteFile } = require('./lib/string-array-compare');
const { createRuleset } = require('./lib/create-file');
const domainSorter = require('./lib/stable-sort-domain');
const { Sema } = require('async-sema');
const { surgeDomainsetToClashDomainset } = require('./lib/clash');
const s = new Sema(2);
/**
@ -115,24 +113,13 @@ const querySpeedtestApi = async (keyword) => {
'GitHub: https://github.com/SukkaW/Surge'
];
await Promise.all([
compareAndWriteFile(
withBannerArray(
await Promise.all(createRuleset(
'Sukka\'s Ruleset - Speedtest Domains',
description,
new Date(),
deduped
),
path.resolve(__dirname, '../List/domainset/speedtest.conf')
),
compareAndWriteFile(
withBannerArray(
'Sukka\'s Ruleset - Speedtest Domains',
description,
new Date(),
surgeDomainsetToClashDomainset(deduped)
),
deduped,
'domainset',
path.resolve(__dirname, '../List/domainset/speedtest.conf'),
path.resolve(__dirname, '../Clash/domainset/speedtest.txt')
)
]);
));
})();

View File

@ -2,10 +2,8 @@ const { fetchWithRetry } = require('./lib/fetch-retry');
const { createReadlineInterfaceFromResponse } = require('./lib/fetch-remote-text-by-line');
const path = require('path');
const { isIPv4, isIPv6 } = require('net');
const { withBannerArray } = require('./lib/with-banner');
const { processLine } = require('./lib/process-line');
const { compareAndWriteFile } = require('./lib/string-array-compare');
const { surgeRulesetToClashClassicalTextRuleset } = require('./lib/clash');
const { createRuleset } = require('./lib/create-file');
(async () => {
console.time('Total Time - build-telegram-cidr');
@ -43,26 +41,15 @@ const { surgeRulesetToClashClassicalTextRuleset } = require('./lib/clash');
' - https://core.telegram.org/resources/cidr.txt'
];
await Promise.all([
compareAndWriteFile(
withBannerArray(
await Promise.all(createRuleset(
'Sukka\'s Ruleset - Telegram IP CIDR',
description,
date,
results
),
path.resolve(__dirname, '../List/ip/telegram.conf')
),
compareAndWriteFile(
withBannerArray(
'Sukka\'s Ruleset - Telegram IP CIDR',
description,
date,
surgeRulesetToClashClassicalTextRuleset(results)
),
results,
'ruleset',
path.resolve(__dirname, '../List/ip/telegram.conf'),
path.resolve(__dirname, '../Clash/ip/telegram.txt')
)
]);
));
console.timeEnd('Total Time - build-telegram-cidr');
})();

View File

@ -4,10 +4,8 @@ const path = require('path');
const { PathScurry } = require('path-scurry');
const { readFileByLine } = require('./lib/fetch-remote-text-by-line');
const { processLine } = require('./lib/process-line');
const { compareAndWriteFile } = require('./lib/string-array-compare');
const { withBannerArray } = require('./lib/with-banner');
const { createRuleset } = require('./lib/create-file');
const { domainDeduper } = require('./lib/domain-deduper');
const { surgeRulesetToClashClassicalTextRuleset, surgeDomainsetToClashDomainset } = require('./lib/clash');
const MAGIC_COMMAND_SKIP = '# $ custom_build_script';
const MAGIC_COMMAND_TITLE = '# $ meta_title ';
@ -102,29 +100,15 @@ async function transformDomainset(sourcePath, relativePath) {
)
];
await Promise.all([
// Surge DOMAIN-SET
compareAndWriteFile(
withBannerArray(
await Promise.all(createRuleset(
title,
description,
new Date(),
deduped
),
path.resolve(outputSurgeDir, relativePath)
),
// Clash domain text
compareAndWriteFile(
withBannerArray(
title,
description,
new Date(),
surgeDomainsetToClashDomainset(deduped)
),
// change path extname to .txt
deduped,
'domainset',
path.resolve(outputSurgeDir, relativePath),
path.resolve(outputClashDir, `${relativePath.slice(0, -path.extname(relativePath).length)}.txt`)
)
]);
));
}
/**
@ -149,31 +133,13 @@ async function transformRuleset(sourcePath, relativePath) {
)
];
const lines = Array.from(set);
const clashSupported = surgeRulesetToClashClassicalTextRuleset(set);
await Promise.all([
// Surge RULE-SET
compareAndWriteFile(
withBannerArray(
await Promise.all(createRuleset(
title,
description,
new Date(),
lines
),
path.resolve(outputSurgeDir, relativePath)
),
// Clash domainset
compareAndWriteFile(
withBannerArray(
title,
description,
new Date(),
clashSupported
),
// change path extname to .txt
Array.from(set),
'ruleset',
path.resolve(outputSurgeDir, relativePath),
path.resolve(outputClashDir, `${relativePath.slice(0, -path.extname(relativePath).length)}.txt`)
)
]);
));
}

99
Build/lib/create-file.js Normal file
View File

@ -0,0 +1,99 @@
// @ts-check
const { promises: fsPromises } = require('fs');
const fse = require('fs-extra');
const { readFileByLine } = require('./fetch-remote-text-by-line');
const { surgeDomainsetToClashDomainset, surgeRulesetToClashClassicalTextRuleset } = require('./clash');
/**
* @param {string[]} linesA
* @param {string} filePath
*/
async function compareAndWriteFile(linesA, filePath) {
await fse.ensureFile(filePath);
let isEqual = true;
let index = 0;
for await (const lineB of readFileByLine(filePath)) {
const lineA = linesA[index];
index++;
if (lineA[0] === '#' && lineB[0] === '#') {
continue;
}
if (lineA !== lineB) {
isEqual = false;
break;
}
}
if (!isEqual || index !== linesA.length - 1) {
await fsPromises.writeFile(
filePath,
linesA.join('\n'),
{ encoding: 'utf-8' }
);
} else {
console.log(`Same Content, bail out writing: ${filePath}`);
}
}
module.exports.compareAndWriteFile = compareAndWriteFile;
/**
* @param {string} title
* @param {string[]} description
* @param {Date} date
* @param {string[]} content
* @returns {string[]}
*/
const withBannerArray = (title, description, date, content) => {
return [
'########################################',
`# ${title}`,
`# Last Updated: ${date.toISOString()}`,
`# Size: ${content.length}`,
...description.map(line => (line ? `# ${line}` : '#')),
'########################################',
...content,
'################# END ###################',
''
];
};
module.exports.withBannerArray = withBannerArray;
/**
* @param {string} title
* @param {string[]} description
* @param {Date} date
* @param {string[]} content
* @param {'ruleset' | 'domainset'} type
* @param {string} surgePath
* @param {string} clashPath
*/
const createRuleset = (
title, description, date, content,
type, surgePath, clashPath
) => {
const surgeContent = withBannerArray(title, description, date, content);
let _clashContent;
switch (type) {
case 'domainset':
_clashContent = surgeDomainsetToClashDomainset(content);
break;
case 'ruleset':
_clashContent = surgeRulesetToClashClassicalTextRuleset(content);
break;
default:
throw new TypeError(`Unknown type: ${type}`);
}
const clashContent = withBannerArray(title, description, date, _clashContent);
return [
compareAndWriteFile(surgeContent, surgePath),
compareAndWriteFile(clashContent, clashPath)
];
};
module.exports.createRuleset = createRuleset;

View File

@ -1,41 +0,0 @@
// @ts-check
const { promises: fsPromises } = require('fs');
const fse = require('fs-extra');
const { readFileByLine } = require('./fetch-remote-text-by-line');
/**
* @param {string[]} linesA
* @param {string} filePath
*/
async function compareAndWriteFile(linesA, filePath) {
await fse.ensureFile(filePath);
let isEqual = true;
let index = 0;
for await (const lineB of readFileByLine(filePath)) {
const lineA = linesA[index];
index++;
if (lineA[0] === '#' && lineB[0] === '#') {
continue;
}
if (lineA !== lineB) {
isEqual = false;
break;
}
}
if (!isEqual || index !== linesA.length - 1) {
await fsPromises.writeFile(
filePath,
linesA.join('\n'),
{ encoding: 'utf-8' }
);
} else {
console.log(`Same Content, bail out writing: ${filePath}`);
}
}
module.exports.compareAndWriteFile = compareAndWriteFile;

View File

@ -1,41 +0,0 @@
// @ts-check
/**
* @param {string} title
* @param {string[]} description
* @param {Date} date
* @param {string[]} content
* @returns {string}
*/
// const withBanner = (title, description, date, content) => {
// return `########################################
// # ${title}
// # Last Updated: ${date.toISOString()}
// # Size: ${content.length}
// ${description.map(line => (line ? `# ${line}` : '#')).join('\n')}
// ########################################\n${content.join('\n')}\n################# END ###################\n`;
// };
// module.exports.withBanner = withBanner;
/**
* @param {string} title
* @param {string[]} description
* @param {Date} date
* @param {string[]} content
* @returns {string[]}
*/
const withBannerArray = (title, description, date, content) => {
return [
'########################################',
`# ${title}`,
`# Last Updated: ${date.toISOString()}`,
`# Size: ${content.length}`,
...description.map(line => (line ? `# ${line}` : '#')),
'########################################',
...content,
'################# END ###################',
''
];
};
module.exports.withBannerArray = withBannerArray;