Perf: reduce infra runtime costs

This commit is contained in:
SukkaW
2023-09-14 21:11:43 +08:00
parent 0a4c99ab0e
commit b42bd05ccf
23 changed files with 172 additions and 265 deletions

View File

@@ -4,9 +4,9 @@ const { isIPv4, isIPv6 } = require('net');
const { createRuleset } = require('./lib/create-file');
const { fetchRemoteTextAndCreateReadlineInterface, readFileByLine } = require('./lib/fetch-remote-text-by-line');
const { processLine } = require('./lib/process-line');
const { runner } = require('./lib/trace-runner');
const { runner, task } = require('./lib/trace-runner');
const buildAntiBogusDomain = async () => {
const buildAntiBogusDomain = task(__filename, async () => {
/** @type {string[]} */
const res = [];
for await (const line of await fetchRemoteTextAndCreateReadlineInterface('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/bogus-nxdomain.china.conf')) {
@@ -56,7 +56,7 @@ const buildAntiBogusDomain = async () => {
path.resolve(__dirname, '../List/ip/reject.conf'),
path.resolve(__dirname, '../Clash/ip/reject.txt')
));
};
});
module.exports.buildAntiBogusDomain = buildAntiBogusDomain;

View File

@@ -1,9 +1,9 @@
const path = require('path');
const { createRuleset } = require('./lib/create-file');
const { parseFelixDnsmasq } = require('./lib/parse-dnsmasq');
const { runner } = require('./lib/trace-runner');
const { runner, task } = require('./lib/trace-runner');
const buildAppleCdn = async () => {
const buildAppleCdn = task(__filename, async () => {
const res = await parseFelixDnsmasq('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf');
const description = [
@@ -40,7 +40,7 @@ const buildAppleCdn = async () => {
path.resolve(__dirname, '../Clash/domainset/apple_cdn.txt')
)
]);
};
});
module.exports.buildAppleCdn = buildAppleCdn;

View File

@@ -3,13 +3,13 @@ const path = require('path');
const { createRuleset } = require('./lib/create-file');
const { fetchRemoteTextAndCreateReadlineInterface, readFileByLine } = require('./lib/fetch-remote-text-by-line');
const Trie = require('./lib/trie');
const { runner } = require('./lib/trace-runner');
const { runner, task } = require('./lib/trace-runner');
const fs = require('fs');
const { processLine } = require('./lib/process-line');
const publicSuffixPath = path.resolve(__dirname, '../node_modules/.cache/public_suffix-list_dat.txt');
const publicSuffixPath = path.resolve(__dirname, '../node_modules/.cache/public_suffix_list_dat.txt');
const buildCdnConf = async () => {
const buildCdnConf = task(__filename, async () => {
const trie = new Trie();
if (fs.existsSync(publicSuffixPath)) {
@@ -30,12 +30,18 @@ const buildCdnConf = async () => {
const S3OSSDomains = new Set();
trie.find('.amazonaws.com').forEach(line => {
if ((line.startsWith('s3-') || line.startsWith('s3.')) && !line.includes('cn-')) {
if (
(line.startsWith('s3-') || line.startsWith('s3.'))
&& !line.includes('cn-')
) {
S3OSSDomains.add(line);
}
});
trie.find('.scw.cloud').forEach(line => {
if ((line.startsWith('s3-') || line.startsWith('s3.')) && !line.includes('cn-')) {
if (
(line.startsWith('s3-') || line.startsWith('s3.'))
&& !line.includes('cn-')
) {
S3OSSDomains.add(line);
}
});
@@ -43,10 +49,14 @@ const buildCdnConf = async () => {
/** @type {string[]} */
const cdnDomainsList = [];
for await (const l of readFileByLine(path.resolve(__dirname, '../Source/non_ip/cdn.conf'))) {
if (l === '# --- [AWS S3 Replace Me] ---') {
console.log(S3OSSDomains);
S3OSSDomains.forEach(domain => {
cdnDomainsList.push(`DOMAIN-SUFFIX,${domain}`);
});
}
const line = processLine(l);
if (line === '# --- [AWS S3 Replace Me] ---') {
S3OSSDomains.forEach(domain => cdnDomainsList.push(`DOMAIN-SUFFIX,${domain}`));
} else if (line) {
if (line) {
cdnDomainsList.push(line);
}
}
@@ -68,7 +78,7 @@ const buildCdnConf = async () => {
path.resolve(__dirname, '../List/non_ip/cdn.conf'),
path.resolve(__dirname, '../Clash/non_ip/cdn.txt')
));
};
});
module.exports.buildCdnConf = buildCdnConf;

View File

@@ -4,7 +4,7 @@ const { resolve: pathResolve } = require('path');
// This should not use `createRuleset` API since we are going to generate ipcidr for Clash
const { compareAndWriteFile, withBannerArray } = require('./lib/create-file');
const { processLine } = require('./lib/process-line');
const { runner } = require('./lib/trace-runner');
const { runner, task } = require('./lib/trace-runner');
// https://github.com/misakaio/chnroutes2/issues/25
const EXCLUDE_CIDRS = [
@@ -12,7 +12,7 @@ const EXCLUDE_CIDRS = [
'223.120.0.0/15'
];
const buildChnCidr = async () => {
const buildChnCidr = task(__filename, async () => {
const { exclude: excludeCidrs } = await import('cidr-tools-wasm');
/** @type {string[]} */
@@ -56,7 +56,7 @@ const buildChnCidr = async () => {
pathResolve(__dirname, '../Clash/ip/china_ip.txt')
)
]);
};
});
module.exports.buildChnCidr = buildChnCidr;

View File

@@ -6,7 +6,7 @@ const { readFileByLine } = require('./lib/fetch-remote-text-by-line');
const { processLine } = require('./lib/process-line');
const { createRuleset } = require('./lib/create-file');
const { domainDeduper } = require('./lib/domain-deduper');
const { runner } = require('./lib/trace-runner');
const { runner, task } = require('./lib/trace-runner');
const MAGIC_COMMAND_SKIP = '# $ custom_build_script';
const MAGIC_COMMAND_TITLE = '# $ meta_title ';
@@ -16,7 +16,7 @@ const sourceDir = path.resolve(__dirname, '../Source');
const outputSurgeDir = path.resolve(__dirname, '../List');
const outputClashDir = path.resolve(__dirname, '../Clash');
const buildCommon = async () => {
const buildCommon = task(__filename, async () => {
/** @type {Promise<void>[]} */
const promises = [];
@@ -43,7 +43,7 @@ const buildCommon = async () => {
}
return Promise.all(promises);
};
});
module.exports.buildCommon = buildCommon;

View File

@@ -5,9 +5,9 @@ const { readFileByLine } = require('./lib/fetch-remote-text-by-line');
const { processLine } = require('./lib/process-line');
const { compareAndWriteFile, createRuleset } = require('./lib/create-file');
const domainSorter = require('./lib/stable-sort-domain');
const { runner } = require('./lib/trace-runner');
const { runner, task } = require('./lib/trace-runner');
const buildDomesticRuleset = async () => {
const buildDomesticRuleset = task(__filename, async () => {
const rl = readFileByLine(path.resolve(__dirname, '../Source/non_ip/domestic.conf'));
const results = [];
for await (const l of rl) {
@@ -67,7 +67,7 @@ const buildDomesticRuleset = async () => {
path.resolve(__dirname, '../Modules/sukka_local_dns_mapping.sgmodule')
)
]);
};
});
module.exports.buildDomesticRuleset = buildDomesticRuleset;

View File

@@ -5,7 +5,7 @@ const tldts = require('tldts');
const { processLine } = require('./lib/process-line');
const { readFileByLine } = require('./lib/fetch-remote-text-by-line');
const domainSorter = require('./lib/stable-sort-domain');
const { runner } = require('./lib/trace-runner');
const { runner, task } = require('./lib/trace-runner');
const { compareAndWriteFile } = require('./lib/create-file');
/**
@@ -15,7 +15,7 @@ const escapeRegExp = (string) => {
return string.replace(/[\\^$.*+?()[\]{}|]/g, '\\$&');
};
const buildInternalCDNDomains = async () => {
const buildInternalCDNDomains = task(__filename, async () => {
const set = new Set();
const keywords = new Set();
@@ -87,7 +87,7 @@ const buildInternalCDNDomains = async () => {
],
path.resolve(__dirname, '../List/internal/cdn.txt')
);
};
});
module.exports.buildInternalCDNDomains = buildInternalCDNDomains;

View File

@@ -2,10 +2,10 @@
const path = require('path');
const fse = require('fs-extra');
const { parseFelixDnsmasq } = require('./lib/parse-dnsmasq');
const { runner } = require('./lib/trace-runner');
const { runner, task } = require('./lib/trace-runner');
const { compareAndWriteFile } = require('./lib/create-file');
const buildInternalChnDomains = async () => {
const buildInternalChnDomains = task(__filename, async () => {
const [result] = await Promise.all([
parseFelixDnsmasq('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf'),
fse.ensureDir(path.resolve(__dirname, '../List/internal'))
@@ -15,7 +15,7 @@ const buildInternalChnDomains = async () => {
result.map(line => `SUFFIX,${line}`),
path.resolve(__dirname, '../List/internal/accelerated-china-domains.txt')
);
};
});
module.exports.buildInternalChnDomains = buildInternalChnDomains;

View File

@@ -4,7 +4,7 @@ const { processLine } = require('./lib/process-line');
const path = require('path');
const fse = require('fs-extra');
const fs = require('fs');
const { runner } = require('./lib/trace-runner');
const { runner, task } = require('./lib/trace-runner');
const RESERVED_IPV4_CIDR = [
'0.0.0.0/8',
@@ -24,7 +24,7 @@ const RESERVED_IPV4_CIDR = [
'240.0.0.0/4'
];
const buildInternalReverseChnCIDR = async () => {
const buildInternalReverseChnCIDR = task(__filename, async () => {
const { exclude } = await import('cidr-tools-wasm');
/** @type {string[]} */
@@ -47,7 +47,7 @@ const buildInternalReverseChnCIDR = async () => {
path.resolve(__dirname, '../List/internal/reversed-chn-cidr.txt'),
`${reversedCidr.join('\n')}\n`
);
};
});
module.exports.buildInternalReverseChnCIDR = buildInternalReverseChnCIDR;

View File

@@ -4,7 +4,7 @@ const path = require('path');
const { createRuleset } = require('./lib/create-file');
const { processLine } = require('./lib/process-line.js');
const domainSorter = require('./lib/stable-sort-domain');
const { runner, traceSync } = require('./lib/trace-runner.js');
const { runner, traceSync, task } = require('./lib/trace-runner.js');
const WHITELIST_DOMAIN = new Set([
'w3s.link',
@@ -60,7 +60,7 @@ const BLACK_TLD = new Set([
'com.cn'
]);
const buildPhishingDomainSet = async () => {
const buildPhishingDomainSet = task(__filename, async () => {
const domainSet = Array.from((await processFilterRules('https://curbengh.github.io/phishing-filter/phishing-filter-agh.txt')).black);
const domainCountMap = {};
@@ -151,7 +151,7 @@ const buildPhishingDomainSet = async () => {
path.resolve(__dirname, '../List/domainset/reject_phishing.conf'),
path.resolve(__dirname, '../Clash/domainset/reject_phishing.txt')
));
};
});
module.exports.buildPhishingDomainSet = buildPhishingDomainSet;

View File

@@ -2,7 +2,7 @@ const listDir = require('@sukka/listdir');
const path = require('path');
const fs = require('fs');
const fse = require('fs-extra');
const { runner } = require('./lib/trace-runner');
const { runner, task } = require('./lib/trace-runner');
const rootPath = path.resolve(__dirname, '../');
const publicPath = path.resolve(__dirname, '../public');
@@ -17,7 +17,7 @@ const folderAndFilesToBeDeployed = [
'README.md'
];
const buildPublicHtml = async () => {
const buildPublicHtml = task(__filename, async () => {
await fse.ensureDir(publicPath);
await Promise.all(folderAndFilesToBeDeployed.map(dir => fse.copy(path.resolve(rootPath, dir), path.resolve(publicPath, dir))));
@@ -29,7 +29,7 @@ const buildPublicHtml = async () => {
const html = template(list);
await fs.promises.writeFile(path.join(publicPath, 'index.html'), html, 'utf-8');
};
});
module.exports.buildPublicHtml = buildPublicHtml;

View File

@@ -12,7 +12,7 @@ const { domainDeduper } = require('./lib/domain-deduper');
const createKeywordFilter = require('./lib/aho-corasick');
const { readFileByLine } = require('./lib/fetch-remote-text-by-line');
const { createDomainSorter } = require('./lib/stable-sort-domain');
const { traceSync, runner } = require('./lib/trace-runner');
const { traceSync, runner, task } = require('./lib/trace-runner');
const { getGorhillPublicSuffixPromise } = require('./lib/get-gorhill-publicsuffix');
const { createCachedGorhillGetDomain } = require('./lib/cached-tld-parse');
@@ -23,7 +23,7 @@ const domainKeywordsSet = new Set();
/** @type {Set<string>} Dedupe domains included by DOMAIN-SUFFIX */
const domainSuffixSet = new Set();
const buildRejectDomainSet = async () => {
const buildRejectDomainSet = task(__dirname, async () => {
/** @type Set<string> */
const domainSets = new Set();
@@ -228,7 +228,7 @@ const buildRejectDomainSet = async () => {
// Copy reject_sukka.conf for backward compatibility
fse.copy(pathResolve(__dirname, '../Source/domainset/reject_sukka.conf'), pathResolve(__dirname, '../List/domainset/reject_sukka.conf'))
]);
};
});
module.exports.buildRejectDomainSet = buildRejectDomainSet;

View File

@@ -5,7 +5,7 @@ const { createRuleset } = require('./lib/create-file');
const domainSorter = require('./lib/stable-sort-domain');
const { Sema } = require('async-sema');
const { runner } = require('./lib/trace-runner');
const { runner, task } = require('./lib/trace-runner');
const s = new Sema(2);
/**
@@ -46,7 +46,7 @@ const querySpeedtestApi = async (keyword) => {
}
};
const buildSpeedtestDomainSet = async () => {
const buildSpeedtestDomainSet = task(__filename, async () => {
/** @type {Set<string>} */
const domains = new Set([
'.speedtest.net',
@@ -123,7 +123,7 @@ const buildSpeedtestDomainSet = async () => {
path.resolve(__dirname, '../List/domainset/speedtest.conf'),
path.resolve(__dirname, '../Clash/domainset/speedtest.txt')
));
};
});
module.exports.buildSpeedtestDomainSet = buildSpeedtestDomainSet;

View File

@@ -4,9 +4,9 @@ const path = require('path');
const { isIPv4, isIPv6 } = require('net');
const { processLine } = require('./lib/process-line');
const { createRuleset } = require('./lib/create-file');
const { runner } = require('./lib/trace-runner');
const { runner, task } = require('./lib/trace-runner');
const buildTelegramCIDR = async () => {
const buildTelegramCIDR = task(__filename, async () => {
/** @type {Response} */
const resp = await fetchWithRetry('https://core.telegram.org/resources/cidr.txt');
const lastModified = resp.headers.get('last-modified');
@@ -49,7 +49,7 @@ const buildTelegramCIDR = async () => {
path.resolve(__dirname, '../List/ip/telegram.conf'),
path.resolve(__dirname, '../Clash/ip/telegram.txt')
));
};
});
module.exports.buildTelegramCIDR = buildTelegramCIDR;

View File

View File

@@ -8,7 +8,7 @@ const { Readable } = require('stream');
const { pipeline } = require('stream/promises');
const { readFileByLine } = require('./lib/fetch-remote-text-by-line');
const { isCI } = require('ci-info');
const { runner } = require('./lib/trace-runner');
const { runner, task } = require('./lib/trace-runner');
const fileExists = (path) => {
return fs.promises.access(path, fs.constants.F_OK)
@@ -16,7 +16,7 @@ const fileExists = (path) => {
.catch(() => false);
};
const downloadPreviousBuild = async () => {
const downloadPreviousBuild = task(__filename, async () => {
const filesList = ['Clash', 'List'];
let allFileExists = true;
@@ -79,11 +79,11 @@ const downloadPreviousBuild = async () => {
}));
await fs.promises.unlink(extractedPath).catch(() => { });
};
});
const downloadPublicSuffixList = async () => {
const publicSuffixDir = resolve(__dirname, '../node_modules/.cache');
const publicSuffixPath = join(publicSuffixDir, 'public_suffix-list_dat.txt');
const publicSuffixPath = join(publicSuffixDir, 'public_suffix_list_dat.txt');
console.log('Download public suffix list.');
@@ -102,10 +102,8 @@ module.exports.downloadPreviousBuild = downloadPreviousBuild;
module.exports.downloadPublicSuffixList = downloadPublicSuffixList;
if (require.main === module) {
runner(__filename, () => {
return Promise.all([
downloadPreviousBuild(),
downloadPublicSuffixList()
]);
});
runner(__filename, () => Promise.all([
downloadPreviousBuild(),
downloadPublicSuffixList()
]));
}

82
Build/index.js Normal file
View File

@@ -0,0 +1,82 @@
const { downloadPreviousBuild, downloadPublicSuffixList } = require('./download-previous-build');
const { buildCommon } = require('./build-common');
const { buildAntiBogusDomain } = require('./build-anti-bogus-domain');
const { buildAppleCdn } = require('./build-apple-cdn');
const { buildCdnConf } = require('./build-cdn-conf');
const { buildPhishingDomainSet } = require('./build-phishing-domainset');
const { buildRejectDomainSet } = require('./build-reject-domainset');
const { buildTelegramCIDR } = require('./build-telegram-cidr');
const { buildChnCidr } = require('./build-chn-cidr');
const { buildSpeedtestDomainSet } = require('./build-speedtest-domainset');
const { buildInternalCDNDomains } = require('./build-internal-cdn-rules');
const { buildInternalReverseChnCIDR } = require('./build-internal-reverse-chn-cidr');
const { buildInternalChnDomains } = require('./build-internal-chn-domains');
const { buildDomesticRuleset } = require('./build-domestic-ruleset');
const { validate } = require('./validate-domainset');
const { buildPublicHtml } = require('./build-public');
(async () => {
// download-previous-build
const downloadPreviousBuildPromise = downloadPreviousBuild();
const downloadPublicSuffixListPromise = downloadPublicSuffixList();
// build:common
const buildCommonPromise = downloadPreviousBuildPromise.then(() => buildCommon());
// build:anti-bogus-domain
const buildAntiBogusDomainPromise = downloadPreviousBuildPromise.then(() => buildAntiBogusDomain());
// build:apple-cdn
const buildAppleCdnPromise = downloadPreviousBuildPromise.then(() => buildAppleCdn());
// build:cdn-conf
const buildCdnConfPromise = Promise.all([
downloadPreviousBuildPromise,
downloadPublicSuffixListPromise
]).then(() => buildCdnConf());
// build:phishing-domainset
const buildPhilishingDomainsetPromise = downloadPreviousBuildPromise.then(() => buildPhishingDomainSet());
// build:reject-domainset
const buildRejectDomainSetPromise = Promise.all([
downloadPreviousBuildPromise,
downloadPublicSuffixListPromise,
buildPhilishingDomainsetPromise
]).then(() => buildRejectDomainSet());
// build:telegram-cidr
const buildTelegramCIDRPromise = downloadPreviousBuildPromise.then(() => buildTelegramCIDR());
// build:chn-cidr
const buildChnCidrPromise = downloadPreviousBuildPromise.then(() => buildChnCidr());
// build:speedtest-domainset
const buildSpeedtestDomainSetPromise = downloadPreviousBuildPromise.then(() => buildSpeedtestDomainSet());
// build:internal-cdn-rules
const buildInternalCDNDomainsPromise = Promise.all([
buildCommonPromise,
buildCdnConfPromise
]).then(() => buildInternalCDNDomains());
// build:internal-reverse-chn-cidr
const buildInternalReverseChnCIDRPromise = buildInternalReverseChnCIDR();
// build:internal-chn-domains
const buildInternalChnDomainsPromise = buildInternalChnDomains();
// build:domestic-ruleset
const buildDomesticRulesetPromise = downloadPreviousBuildPromise.then(() => buildDomesticRuleset());
await Promise.all([
downloadPreviousBuildPromise,
downloadPublicSuffixListPromise,
buildCommonPromise,
buildAntiBogusDomainPromise,
buildAppleCdnPromise,
buildCdnConfPromise,
buildPhilishingDomainsetPromise,
buildRejectDomainSetPromise,
buildTelegramCIDRPromise,
buildChnCidrPromise,
buildSpeedtestDomainSetPromise,
buildInternalCDNDomainsPromise,
buildInternalReverseChnCIDRPromise,
buildInternalChnDomainsPromise,
buildDomesticRulesetPromise
]);
await Promise.all([
buildPublicHtml(),
validate()
]);
})();

View File

@@ -13,11 +13,12 @@ module.exports.parse = (domain) => {
return cache.sync(domain, () => tldts.parse(domain, sharedConfig));
};
const gothillGetDomainCache = createCache('cached-gorhill-get-domain', true);
let gothillGetDomainCache = null;
/**
* @param {import('gorhill-publicsuffixlist').default | null} gorhill
*/
module.exports.createCachedGorhillGetDomain = (gorhill) => {
gothillGetDomainCache ||= createCache('cached-gorhill-get-domain', true);
/**
* @param {string} domain
*/

View File

@@ -2,7 +2,7 @@ const { toASCII } = require('punycode/');
const fs = require('fs');
const path = require('path');
const publicSuffixPath = path.resolve(__dirname, '../../node_modules/.cache/public_suffix-list_dat.txt');
const publicSuffixPath = path.resolve(__dirname, '../../node_modules/.cache/public_suffix_list_dat.txt');
const getPublicSuffixListDat = () => {
if (fs.existsSync(publicSuffixPath)) {
return fs.promises.readFile(publicSuffixPath, 'utf-8');

View File

@@ -1,3 +1,4 @@
// @ts-check
const path = require('path');
const { performance } = require('perf_hooks');
@@ -35,8 +36,21 @@ module.exports.traceAsync = traceAsync;
* @template T
* @param {string} __filename
* @param {() => Promise<T>} fn
* @returns {T}
* @returns {Promise<T>}
*/
module.exports.runner = async (__filename, fn) => {
return traceAsync(`⌛ [${path.basename(__filename, path.extname(__filename))}]`, fn);
};
/**
* @template T
* @param {string} __filename
* @param {() => Promise<T>} fn
*/
module.exports.task = (__filename, fn) => {
const taskName = path.basename(__filename, path.extname(__filename));
return () => {
console.log(`🏃 [${taskName}] Start executing`);
return traceAsync(`✅ [${taskName}] Executed successfully`, fn);
};
};

View File

@@ -6,7 +6,7 @@ const path = require('path');
const listDir = require('@sukka/listdir');
const { readFileByLine } = require('./lib/fetch-remote-text-by-line');
const { processLine } = require('./lib/process-line');
const { runner } = require('./lib/trace-runner');
const { runner, task } = require('./lib/trace-runner');
const SPECIAL_SUFFIXES = new Set([
'linodeobjects.com', // only *.linodeobjects.com are public suffix
@@ -58,7 +58,7 @@ const _validateRuleset = async (filePath) => {
}
};
const validate = async () => {
const validate = task(__filename, async () => {
const [domainsetFiles, _rulesetFiles] = await Promise.all([
listDir(path.resolve(__dirname, '../List/domainset')),
listDir(path.resolve(__dirname, '../List/non_ip'))
@@ -67,7 +67,7 @@ const validate = async () => {
domainsetFiles.map(file => validateDomainSet(file))
// rulesetFiles.map(file => validateRuleset(file))
);
};
});
module.exports.validate = validate;
if (require.main === module) {