mirror of
https://github.com/SukkaW/Surge.git
synced 2026-01-28 17:41:54 +08:00
Perf: reduce infra runtime costs
This commit is contained in:
@@ -4,9 +4,9 @@ const { isIPv4, isIPv6 } = require('net');
|
||||
const { createRuleset } = require('./lib/create-file');
|
||||
const { fetchRemoteTextAndCreateReadlineInterface, readFileByLine } = require('./lib/fetch-remote-text-by-line');
|
||||
const { processLine } = require('./lib/process-line');
|
||||
const { runner } = require('./lib/trace-runner');
|
||||
const { runner, task } = require('./lib/trace-runner');
|
||||
|
||||
const buildAntiBogusDomain = async () => {
|
||||
const buildAntiBogusDomain = task(__filename, async () => {
|
||||
/** @type {string[]} */
|
||||
const res = [];
|
||||
for await (const line of await fetchRemoteTextAndCreateReadlineInterface('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/bogus-nxdomain.china.conf')) {
|
||||
@@ -56,7 +56,7 @@ const buildAntiBogusDomain = async () => {
|
||||
path.resolve(__dirname, '../List/ip/reject.conf'),
|
||||
path.resolve(__dirname, '../Clash/ip/reject.txt')
|
||||
));
|
||||
};
|
||||
});
|
||||
|
||||
module.exports.buildAntiBogusDomain = buildAntiBogusDomain;
|
||||
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
const path = require('path');
|
||||
const { createRuleset } = require('./lib/create-file');
|
||||
const { parseFelixDnsmasq } = require('./lib/parse-dnsmasq');
|
||||
const { runner } = require('./lib/trace-runner');
|
||||
const { runner, task } = require('./lib/trace-runner');
|
||||
|
||||
const buildAppleCdn = async () => {
|
||||
const buildAppleCdn = task(__filename, async () => {
|
||||
const res = await parseFelixDnsmasq('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf');
|
||||
|
||||
const description = [
|
||||
@@ -40,7 +40,7 @@ const buildAppleCdn = async () => {
|
||||
path.resolve(__dirname, '../Clash/domainset/apple_cdn.txt')
|
||||
)
|
||||
]);
|
||||
};
|
||||
});
|
||||
|
||||
module.exports.buildAppleCdn = buildAppleCdn;
|
||||
|
||||
|
||||
@@ -3,13 +3,13 @@ const path = require('path');
|
||||
const { createRuleset } = require('./lib/create-file');
|
||||
const { fetchRemoteTextAndCreateReadlineInterface, readFileByLine } = require('./lib/fetch-remote-text-by-line');
|
||||
const Trie = require('./lib/trie');
|
||||
const { runner } = require('./lib/trace-runner');
|
||||
const { runner, task } = require('./lib/trace-runner');
|
||||
const fs = require('fs');
|
||||
const { processLine } = require('./lib/process-line');
|
||||
|
||||
const publicSuffixPath = path.resolve(__dirname, '../node_modules/.cache/public_suffix-list_dat.txt');
|
||||
const publicSuffixPath = path.resolve(__dirname, '../node_modules/.cache/public_suffix_list_dat.txt');
|
||||
|
||||
const buildCdnConf = async () => {
|
||||
const buildCdnConf = task(__filename, async () => {
|
||||
const trie = new Trie();
|
||||
|
||||
if (fs.existsSync(publicSuffixPath)) {
|
||||
@@ -30,12 +30,18 @@ const buildCdnConf = async () => {
|
||||
const S3OSSDomains = new Set();
|
||||
|
||||
trie.find('.amazonaws.com').forEach(line => {
|
||||
if ((line.startsWith('s3-') || line.startsWith('s3.')) && !line.includes('cn-')) {
|
||||
if (
|
||||
(line.startsWith('s3-') || line.startsWith('s3.'))
|
||||
&& !line.includes('cn-')
|
||||
) {
|
||||
S3OSSDomains.add(line);
|
||||
}
|
||||
});
|
||||
trie.find('.scw.cloud').forEach(line => {
|
||||
if ((line.startsWith('s3-') || line.startsWith('s3.')) && !line.includes('cn-')) {
|
||||
if (
|
||||
(line.startsWith('s3-') || line.startsWith('s3.'))
|
||||
&& !line.includes('cn-')
|
||||
) {
|
||||
S3OSSDomains.add(line);
|
||||
}
|
||||
});
|
||||
@@ -43,10 +49,14 @@ const buildCdnConf = async () => {
|
||||
/** @type {string[]} */
|
||||
const cdnDomainsList = [];
|
||||
for await (const l of readFileByLine(path.resolve(__dirname, '../Source/non_ip/cdn.conf'))) {
|
||||
if (l === '# --- [AWS S3 Replace Me] ---') {
|
||||
console.log(S3OSSDomains);
|
||||
S3OSSDomains.forEach(domain => {
|
||||
cdnDomainsList.push(`DOMAIN-SUFFIX,${domain}`);
|
||||
});
|
||||
}
|
||||
const line = processLine(l);
|
||||
if (line === '# --- [AWS S3 Replace Me] ---') {
|
||||
S3OSSDomains.forEach(domain => cdnDomainsList.push(`DOMAIN-SUFFIX,${domain}`));
|
||||
} else if (line) {
|
||||
if (line) {
|
||||
cdnDomainsList.push(line);
|
||||
}
|
||||
}
|
||||
@@ -68,7 +78,7 @@ const buildCdnConf = async () => {
|
||||
path.resolve(__dirname, '../List/non_ip/cdn.conf'),
|
||||
path.resolve(__dirname, '../Clash/non_ip/cdn.txt')
|
||||
));
|
||||
};
|
||||
});
|
||||
|
||||
module.exports.buildCdnConf = buildCdnConf;
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ const { resolve: pathResolve } = require('path');
|
||||
// This should not use `createRuleset` API since we are going to generate ipcidr for Clash
|
||||
const { compareAndWriteFile, withBannerArray } = require('./lib/create-file');
|
||||
const { processLine } = require('./lib/process-line');
|
||||
const { runner } = require('./lib/trace-runner');
|
||||
const { runner, task } = require('./lib/trace-runner');
|
||||
|
||||
// https://github.com/misakaio/chnroutes2/issues/25
|
||||
const EXCLUDE_CIDRS = [
|
||||
@@ -12,7 +12,7 @@ const EXCLUDE_CIDRS = [
|
||||
'223.120.0.0/15'
|
||||
];
|
||||
|
||||
const buildChnCidr = async () => {
|
||||
const buildChnCidr = task(__filename, async () => {
|
||||
const { exclude: excludeCidrs } = await import('cidr-tools-wasm');
|
||||
|
||||
/** @type {string[]} */
|
||||
@@ -56,7 +56,7 @@ const buildChnCidr = async () => {
|
||||
pathResolve(__dirname, '../Clash/ip/china_ip.txt')
|
||||
)
|
||||
]);
|
||||
};
|
||||
});
|
||||
|
||||
module.exports.buildChnCidr = buildChnCidr;
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ const { readFileByLine } = require('./lib/fetch-remote-text-by-line');
|
||||
const { processLine } = require('./lib/process-line');
|
||||
const { createRuleset } = require('./lib/create-file');
|
||||
const { domainDeduper } = require('./lib/domain-deduper');
|
||||
const { runner } = require('./lib/trace-runner');
|
||||
const { runner, task } = require('./lib/trace-runner');
|
||||
|
||||
const MAGIC_COMMAND_SKIP = '# $ custom_build_script';
|
||||
const MAGIC_COMMAND_TITLE = '# $ meta_title ';
|
||||
@@ -16,7 +16,7 @@ const sourceDir = path.resolve(__dirname, '../Source');
|
||||
const outputSurgeDir = path.resolve(__dirname, '../List');
|
||||
const outputClashDir = path.resolve(__dirname, '../Clash');
|
||||
|
||||
const buildCommon = async () => {
|
||||
const buildCommon = task(__filename, async () => {
|
||||
/** @type {Promise<void>[]} */
|
||||
const promises = [];
|
||||
|
||||
@@ -43,7 +43,7 @@ const buildCommon = async () => {
|
||||
}
|
||||
|
||||
return Promise.all(promises);
|
||||
};
|
||||
});
|
||||
|
||||
module.exports.buildCommon = buildCommon;
|
||||
|
||||
|
||||
@@ -5,9 +5,9 @@ const { readFileByLine } = require('./lib/fetch-remote-text-by-line');
|
||||
const { processLine } = require('./lib/process-line');
|
||||
const { compareAndWriteFile, createRuleset } = require('./lib/create-file');
|
||||
const domainSorter = require('./lib/stable-sort-domain');
|
||||
const { runner } = require('./lib/trace-runner');
|
||||
const { runner, task } = require('./lib/trace-runner');
|
||||
|
||||
const buildDomesticRuleset = async () => {
|
||||
const buildDomesticRuleset = task(__filename, async () => {
|
||||
const rl = readFileByLine(path.resolve(__dirname, '../Source/non_ip/domestic.conf'));
|
||||
const results = [];
|
||||
for await (const l of rl) {
|
||||
@@ -67,7 +67,7 @@ const buildDomesticRuleset = async () => {
|
||||
path.resolve(__dirname, '../Modules/sukka_local_dns_mapping.sgmodule')
|
||||
)
|
||||
]);
|
||||
};
|
||||
});
|
||||
|
||||
module.exports.buildDomesticRuleset = buildDomesticRuleset;
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ const tldts = require('tldts');
|
||||
const { processLine } = require('./lib/process-line');
|
||||
const { readFileByLine } = require('./lib/fetch-remote-text-by-line');
|
||||
const domainSorter = require('./lib/stable-sort-domain');
|
||||
const { runner } = require('./lib/trace-runner');
|
||||
const { runner, task } = require('./lib/trace-runner');
|
||||
const { compareAndWriteFile } = require('./lib/create-file');
|
||||
|
||||
/**
|
||||
@@ -15,7 +15,7 @@ const escapeRegExp = (string) => {
|
||||
return string.replace(/[\\^$.*+?()[\]{}|]/g, '\\$&');
|
||||
};
|
||||
|
||||
const buildInternalCDNDomains = async () => {
|
||||
const buildInternalCDNDomains = task(__filename, async () => {
|
||||
const set = new Set();
|
||||
const keywords = new Set();
|
||||
|
||||
@@ -87,7 +87,7 @@ const buildInternalCDNDomains = async () => {
|
||||
],
|
||||
path.resolve(__dirname, '../List/internal/cdn.txt')
|
||||
);
|
||||
};
|
||||
});
|
||||
|
||||
module.exports.buildInternalCDNDomains = buildInternalCDNDomains;
|
||||
|
||||
|
||||
@@ -2,10 +2,10 @@
|
||||
const path = require('path');
|
||||
const fse = require('fs-extra');
|
||||
const { parseFelixDnsmasq } = require('./lib/parse-dnsmasq');
|
||||
const { runner } = require('./lib/trace-runner');
|
||||
const { runner, task } = require('./lib/trace-runner');
|
||||
const { compareAndWriteFile } = require('./lib/create-file');
|
||||
|
||||
const buildInternalChnDomains = async () => {
|
||||
const buildInternalChnDomains = task(__filename, async () => {
|
||||
const [result] = await Promise.all([
|
||||
parseFelixDnsmasq('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf'),
|
||||
fse.ensureDir(path.resolve(__dirname, '../List/internal'))
|
||||
@@ -15,7 +15,7 @@ const buildInternalChnDomains = async () => {
|
||||
result.map(line => `SUFFIX,${line}`),
|
||||
path.resolve(__dirname, '../List/internal/accelerated-china-domains.txt')
|
||||
);
|
||||
};
|
||||
});
|
||||
|
||||
module.exports.buildInternalChnDomains = buildInternalChnDomains;
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ const { processLine } = require('./lib/process-line');
|
||||
const path = require('path');
|
||||
const fse = require('fs-extra');
|
||||
const fs = require('fs');
|
||||
const { runner } = require('./lib/trace-runner');
|
||||
const { runner, task } = require('./lib/trace-runner');
|
||||
|
||||
const RESERVED_IPV4_CIDR = [
|
||||
'0.0.0.0/8',
|
||||
@@ -24,7 +24,7 @@ const RESERVED_IPV4_CIDR = [
|
||||
'240.0.0.0/4'
|
||||
];
|
||||
|
||||
const buildInternalReverseChnCIDR = async () => {
|
||||
const buildInternalReverseChnCIDR = task(__filename, async () => {
|
||||
const { exclude } = await import('cidr-tools-wasm');
|
||||
|
||||
/** @type {string[]} */
|
||||
@@ -47,7 +47,7 @@ const buildInternalReverseChnCIDR = async () => {
|
||||
path.resolve(__dirname, '../List/internal/reversed-chn-cidr.txt'),
|
||||
`${reversedCidr.join('\n')}\n`
|
||||
);
|
||||
};
|
||||
});
|
||||
|
||||
module.exports.buildInternalReverseChnCIDR = buildInternalReverseChnCIDR;
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ const path = require('path');
|
||||
const { createRuleset } = require('./lib/create-file');
|
||||
const { processLine } = require('./lib/process-line.js');
|
||||
const domainSorter = require('./lib/stable-sort-domain');
|
||||
const { runner, traceSync } = require('./lib/trace-runner.js');
|
||||
const { runner, traceSync, task } = require('./lib/trace-runner.js');
|
||||
|
||||
const WHITELIST_DOMAIN = new Set([
|
||||
'w3s.link',
|
||||
@@ -60,7 +60,7 @@ const BLACK_TLD = new Set([
|
||||
'com.cn'
|
||||
]);
|
||||
|
||||
const buildPhishingDomainSet = async () => {
|
||||
const buildPhishingDomainSet = task(__filename, async () => {
|
||||
const domainSet = Array.from((await processFilterRules('https://curbengh.github.io/phishing-filter/phishing-filter-agh.txt')).black);
|
||||
const domainCountMap = {};
|
||||
|
||||
@@ -151,7 +151,7 @@ const buildPhishingDomainSet = async () => {
|
||||
path.resolve(__dirname, '../List/domainset/reject_phishing.conf'),
|
||||
path.resolve(__dirname, '../Clash/domainset/reject_phishing.txt')
|
||||
));
|
||||
};
|
||||
});
|
||||
|
||||
module.exports.buildPhishingDomainSet = buildPhishingDomainSet;
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ const listDir = require('@sukka/listdir');
|
||||
const path = require('path');
|
||||
const fs = require('fs');
|
||||
const fse = require('fs-extra');
|
||||
const { runner } = require('./lib/trace-runner');
|
||||
const { runner, task } = require('./lib/trace-runner');
|
||||
|
||||
const rootPath = path.resolve(__dirname, '../');
|
||||
const publicPath = path.resolve(__dirname, '../public');
|
||||
@@ -17,7 +17,7 @@ const folderAndFilesToBeDeployed = [
|
||||
'README.md'
|
||||
];
|
||||
|
||||
const buildPublicHtml = async () => {
|
||||
const buildPublicHtml = task(__filename, async () => {
|
||||
await fse.ensureDir(publicPath);
|
||||
await Promise.all(folderAndFilesToBeDeployed.map(dir => fse.copy(path.resolve(rootPath, dir), path.resolve(publicPath, dir))));
|
||||
|
||||
@@ -29,7 +29,7 @@ const buildPublicHtml = async () => {
|
||||
const html = template(list);
|
||||
|
||||
await fs.promises.writeFile(path.join(publicPath, 'index.html'), html, 'utf-8');
|
||||
};
|
||||
});
|
||||
|
||||
module.exports.buildPublicHtml = buildPublicHtml;
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@ const { domainDeduper } = require('./lib/domain-deduper');
|
||||
const createKeywordFilter = require('./lib/aho-corasick');
|
||||
const { readFileByLine } = require('./lib/fetch-remote-text-by-line');
|
||||
const { createDomainSorter } = require('./lib/stable-sort-domain');
|
||||
const { traceSync, runner } = require('./lib/trace-runner');
|
||||
const { traceSync, runner, task } = require('./lib/trace-runner');
|
||||
const { getGorhillPublicSuffixPromise } = require('./lib/get-gorhill-publicsuffix');
|
||||
const { createCachedGorhillGetDomain } = require('./lib/cached-tld-parse');
|
||||
|
||||
@@ -23,7 +23,7 @@ const domainKeywordsSet = new Set();
|
||||
/** @type {Set<string>} Dedupe domains included by DOMAIN-SUFFIX */
|
||||
const domainSuffixSet = new Set();
|
||||
|
||||
const buildRejectDomainSet = async () => {
|
||||
const buildRejectDomainSet = task(__dirname, async () => {
|
||||
/** @type Set<string> */
|
||||
const domainSets = new Set();
|
||||
|
||||
@@ -228,7 +228,7 @@ const buildRejectDomainSet = async () => {
|
||||
// Copy reject_sukka.conf for backward compatibility
|
||||
fse.copy(pathResolve(__dirname, '../Source/domainset/reject_sukka.conf'), pathResolve(__dirname, '../List/domainset/reject_sukka.conf'))
|
||||
]);
|
||||
};
|
||||
});
|
||||
|
||||
module.exports.buildRejectDomainSet = buildRejectDomainSet;
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ const { createRuleset } = require('./lib/create-file');
|
||||
const domainSorter = require('./lib/stable-sort-domain');
|
||||
|
||||
const { Sema } = require('async-sema');
|
||||
const { runner } = require('./lib/trace-runner');
|
||||
const { runner, task } = require('./lib/trace-runner');
|
||||
const s = new Sema(2);
|
||||
|
||||
/**
|
||||
@@ -46,7 +46,7 @@ const querySpeedtestApi = async (keyword) => {
|
||||
}
|
||||
};
|
||||
|
||||
const buildSpeedtestDomainSet = async () => {
|
||||
const buildSpeedtestDomainSet = task(__filename, async () => {
|
||||
/** @type {Set<string>} */
|
||||
const domains = new Set([
|
||||
'.speedtest.net',
|
||||
@@ -123,7 +123,7 @@ const buildSpeedtestDomainSet = async () => {
|
||||
path.resolve(__dirname, '../List/domainset/speedtest.conf'),
|
||||
path.resolve(__dirname, '../Clash/domainset/speedtest.txt')
|
||||
));
|
||||
};
|
||||
});
|
||||
|
||||
module.exports.buildSpeedtestDomainSet = buildSpeedtestDomainSet;
|
||||
|
||||
|
||||
@@ -4,9 +4,9 @@ const path = require('path');
|
||||
const { isIPv4, isIPv6 } = require('net');
|
||||
const { processLine } = require('./lib/process-line');
|
||||
const { createRuleset } = require('./lib/create-file');
|
||||
const { runner } = require('./lib/trace-runner');
|
||||
const { runner, task } = require('./lib/trace-runner');
|
||||
|
||||
const buildTelegramCIDR = async () => {
|
||||
const buildTelegramCIDR = task(__filename, async () => {
|
||||
/** @type {Response} */
|
||||
const resp = await fetchWithRetry('https://core.telegram.org/resources/cidr.txt');
|
||||
const lastModified = resp.headers.get('last-modified');
|
||||
@@ -49,7 +49,7 @@ const buildTelegramCIDR = async () => {
|
||||
path.resolve(__dirname, '../List/ip/telegram.conf'),
|
||||
path.resolve(__dirname, '../Clash/ip/telegram.txt')
|
||||
));
|
||||
};
|
||||
});
|
||||
|
||||
module.exports.buildTelegramCIDR = buildTelegramCIDR;
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ const { Readable } = require('stream');
|
||||
const { pipeline } = require('stream/promises');
|
||||
const { readFileByLine } = require('./lib/fetch-remote-text-by-line');
|
||||
const { isCI } = require('ci-info');
|
||||
const { runner } = require('./lib/trace-runner');
|
||||
const { runner, task } = require('./lib/trace-runner');
|
||||
|
||||
const fileExists = (path) => {
|
||||
return fs.promises.access(path, fs.constants.F_OK)
|
||||
@@ -16,7 +16,7 @@ const fileExists = (path) => {
|
||||
.catch(() => false);
|
||||
};
|
||||
|
||||
const downloadPreviousBuild = async () => {
|
||||
const downloadPreviousBuild = task(__filename, async () => {
|
||||
const filesList = ['Clash', 'List'];
|
||||
|
||||
let allFileExists = true;
|
||||
@@ -79,11 +79,11 @@ const downloadPreviousBuild = async () => {
|
||||
}));
|
||||
|
||||
await fs.promises.unlink(extractedPath).catch(() => { });
|
||||
};
|
||||
});
|
||||
|
||||
const downloadPublicSuffixList = async () => {
|
||||
const publicSuffixDir = resolve(__dirname, '../node_modules/.cache');
|
||||
const publicSuffixPath = join(publicSuffixDir, 'public_suffix-list_dat.txt');
|
||||
const publicSuffixPath = join(publicSuffixDir, 'public_suffix_list_dat.txt');
|
||||
|
||||
console.log('Download public suffix list.');
|
||||
|
||||
@@ -102,10 +102,8 @@ module.exports.downloadPreviousBuild = downloadPreviousBuild;
|
||||
module.exports.downloadPublicSuffixList = downloadPublicSuffixList;
|
||||
|
||||
if (require.main === module) {
|
||||
runner(__filename, () => {
|
||||
return Promise.all([
|
||||
downloadPreviousBuild(),
|
||||
downloadPublicSuffixList()
|
||||
]);
|
||||
});
|
||||
runner(__filename, () => Promise.all([
|
||||
downloadPreviousBuild(),
|
||||
downloadPublicSuffixList()
|
||||
]));
|
||||
}
|
||||
|
||||
82
Build/index.js
Normal file
82
Build/index.js
Normal file
@@ -0,0 +1,82 @@
|
||||
const { downloadPreviousBuild, downloadPublicSuffixList } = require('./download-previous-build');
|
||||
const { buildCommon } = require('./build-common');
|
||||
const { buildAntiBogusDomain } = require('./build-anti-bogus-domain');
|
||||
const { buildAppleCdn } = require('./build-apple-cdn');
|
||||
const { buildCdnConf } = require('./build-cdn-conf');
|
||||
const { buildPhishingDomainSet } = require('./build-phishing-domainset');
|
||||
const { buildRejectDomainSet } = require('./build-reject-domainset');
|
||||
const { buildTelegramCIDR } = require('./build-telegram-cidr');
|
||||
const { buildChnCidr } = require('./build-chn-cidr');
|
||||
const { buildSpeedtestDomainSet } = require('./build-speedtest-domainset');
|
||||
const { buildInternalCDNDomains } = require('./build-internal-cdn-rules');
|
||||
const { buildInternalReverseChnCIDR } = require('./build-internal-reverse-chn-cidr');
|
||||
const { buildInternalChnDomains } = require('./build-internal-chn-domains');
|
||||
const { buildDomesticRuleset } = require('./build-domestic-ruleset');
|
||||
const { validate } = require('./validate-domainset');
|
||||
|
||||
const { buildPublicHtml } = require('./build-public');
|
||||
|
||||
(async () => {
|
||||
// download-previous-build
|
||||
const downloadPreviousBuildPromise = downloadPreviousBuild();
|
||||
const downloadPublicSuffixListPromise = downloadPublicSuffixList();
|
||||
// build:common
|
||||
const buildCommonPromise = downloadPreviousBuildPromise.then(() => buildCommon());
|
||||
// build:anti-bogus-domain
|
||||
const buildAntiBogusDomainPromise = downloadPreviousBuildPromise.then(() => buildAntiBogusDomain());
|
||||
// build:apple-cdn
|
||||
const buildAppleCdnPromise = downloadPreviousBuildPromise.then(() => buildAppleCdn());
|
||||
// build:cdn-conf
|
||||
const buildCdnConfPromise = Promise.all([
|
||||
downloadPreviousBuildPromise,
|
||||
downloadPublicSuffixListPromise
|
||||
]).then(() => buildCdnConf());
|
||||
// build:phishing-domainset
|
||||
const buildPhilishingDomainsetPromise = downloadPreviousBuildPromise.then(() => buildPhishingDomainSet());
|
||||
// build:reject-domainset
|
||||
const buildRejectDomainSetPromise = Promise.all([
|
||||
downloadPreviousBuildPromise,
|
||||
downloadPublicSuffixListPromise,
|
||||
buildPhilishingDomainsetPromise
|
||||
]).then(() => buildRejectDomainSet());
|
||||
// build:telegram-cidr
|
||||
const buildTelegramCIDRPromise = downloadPreviousBuildPromise.then(() => buildTelegramCIDR());
|
||||
// build:chn-cidr
|
||||
const buildChnCidrPromise = downloadPreviousBuildPromise.then(() => buildChnCidr());
|
||||
// build:speedtest-domainset
|
||||
const buildSpeedtestDomainSetPromise = downloadPreviousBuildPromise.then(() => buildSpeedtestDomainSet());
|
||||
// build:internal-cdn-rules
|
||||
const buildInternalCDNDomainsPromise = Promise.all([
|
||||
buildCommonPromise,
|
||||
buildCdnConfPromise
|
||||
]).then(() => buildInternalCDNDomains());
|
||||
// build:internal-reverse-chn-cidr
|
||||
const buildInternalReverseChnCIDRPromise = buildInternalReverseChnCIDR();
|
||||
// build:internal-chn-domains
|
||||
const buildInternalChnDomainsPromise = buildInternalChnDomains();
|
||||
// build:domestic-ruleset
|
||||
const buildDomesticRulesetPromise = downloadPreviousBuildPromise.then(() => buildDomesticRuleset());
|
||||
|
||||
await Promise.all([
|
||||
downloadPreviousBuildPromise,
|
||||
downloadPublicSuffixListPromise,
|
||||
buildCommonPromise,
|
||||
buildAntiBogusDomainPromise,
|
||||
buildAppleCdnPromise,
|
||||
buildCdnConfPromise,
|
||||
buildPhilishingDomainsetPromise,
|
||||
buildRejectDomainSetPromise,
|
||||
buildTelegramCIDRPromise,
|
||||
buildChnCidrPromise,
|
||||
buildSpeedtestDomainSetPromise,
|
||||
buildInternalCDNDomainsPromise,
|
||||
buildInternalReverseChnCIDRPromise,
|
||||
buildInternalChnDomainsPromise,
|
||||
buildDomesticRulesetPromise
|
||||
]);
|
||||
|
||||
await Promise.all([
|
||||
buildPublicHtml(),
|
||||
validate()
|
||||
]);
|
||||
})();
|
||||
@@ -13,11 +13,12 @@ module.exports.parse = (domain) => {
|
||||
return cache.sync(domain, () => tldts.parse(domain, sharedConfig));
|
||||
};
|
||||
|
||||
const gothillGetDomainCache = createCache('cached-gorhill-get-domain', true);
|
||||
let gothillGetDomainCache = null;
|
||||
/**
|
||||
* @param {import('gorhill-publicsuffixlist').default | null} gorhill
|
||||
*/
|
||||
module.exports.createCachedGorhillGetDomain = (gorhill) => {
|
||||
gothillGetDomainCache ||= createCache('cached-gorhill-get-domain', true);
|
||||
/**
|
||||
* @param {string} domain
|
||||
*/
|
||||
|
||||
@@ -2,7 +2,7 @@ const { toASCII } = require('punycode/');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const publicSuffixPath = path.resolve(__dirname, '../../node_modules/.cache/public_suffix-list_dat.txt');
|
||||
const publicSuffixPath = path.resolve(__dirname, '../../node_modules/.cache/public_suffix_list_dat.txt');
|
||||
const getPublicSuffixListDat = () => {
|
||||
if (fs.existsSync(publicSuffixPath)) {
|
||||
return fs.promises.readFile(publicSuffixPath, 'utf-8');
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
// @ts-check
|
||||
const path = require('path');
|
||||
const { performance } = require('perf_hooks');
|
||||
|
||||
@@ -35,8 +36,21 @@ module.exports.traceAsync = traceAsync;
|
||||
* @template T
|
||||
* @param {string} __filename
|
||||
* @param {() => Promise<T>} fn
|
||||
* @returns {T}
|
||||
* @returns {Promise<T>}
|
||||
*/
|
||||
module.exports.runner = async (__filename, fn) => {
|
||||
return traceAsync(`⌛ [${path.basename(__filename, path.extname(__filename))}]`, fn);
|
||||
};
|
||||
|
||||
/**
|
||||
* @template T
|
||||
* @param {string} __filename
|
||||
* @param {() => Promise<T>} fn
|
||||
*/
|
||||
module.exports.task = (__filename, fn) => {
|
||||
const taskName = path.basename(__filename, path.extname(__filename));
|
||||
return () => {
|
||||
console.log(`🏃 [${taskName}] Start executing`);
|
||||
return traceAsync(`✅ [${taskName}] Executed successfully`, fn);
|
||||
};
|
||||
};
|
||||
|
||||
@@ -6,7 +6,7 @@ const path = require('path');
|
||||
const listDir = require('@sukka/listdir');
|
||||
const { readFileByLine } = require('./lib/fetch-remote-text-by-line');
|
||||
const { processLine } = require('./lib/process-line');
|
||||
const { runner } = require('./lib/trace-runner');
|
||||
const { runner, task } = require('./lib/trace-runner');
|
||||
|
||||
const SPECIAL_SUFFIXES = new Set([
|
||||
'linodeobjects.com', // only *.linodeobjects.com are public suffix
|
||||
@@ -58,7 +58,7 @@ const _validateRuleset = async (filePath) => {
|
||||
}
|
||||
};
|
||||
|
||||
const validate = async () => {
|
||||
const validate = task(__filename, async () => {
|
||||
const [domainsetFiles, _rulesetFiles] = await Promise.all([
|
||||
listDir(path.resolve(__dirname, '../List/domainset')),
|
||||
listDir(path.resolve(__dirname, '../List/non_ip'))
|
||||
@@ -67,7 +67,7 @@ const validate = async () => {
|
||||
domainsetFiles.map(file => validateDomainSet(file))
|
||||
// rulesetFiles.map(file => validateRuleset(file))
|
||||
);
|
||||
};
|
||||
});
|
||||
module.exports.validate = validate;
|
||||
|
||||
if (require.main === module) {
|
||||
|
||||
Reference in New Issue
Block a user