Chore: housekeeping

This commit is contained in:
SukkaW
2023-10-16 22:52:08 +08:00
parent 36ef50cdbf
commit 697cea84ed
12 changed files with 227 additions and 291 deletions

View File

@@ -1,5 +1,5 @@
// @ts-check
const fse = require('fs-extra');
const fsp = require('fs/promises');
const path = require('path');
const tldts = require('tldts');
const { processLine } = require('./lib/process-line');
@@ -79,7 +79,7 @@ const buildInternalCDNDomains = task(__filename, async () => {
processLocalDomainSet(path.resolve(__dirname, '../List/domainset/cdn.conf')),
processLocalDomainSet(path.resolve(__dirname, '../List/domainset/download.conf')),
fse.ensureDir(path.resolve(__dirname, '../List/internal'))
fsp.mkdir(path.resolve(__dirname, '../List/internal'), { recursive: true })
]);
return compareAndWriteFile(

View File

@@ -1,6 +1,6 @@
// @ts-check
const path = require('path');
const fse = require('fs-extra');
const fsp = require('fs/promises');
const { parseFelixDnsmasq } = require('./lib/parse-dnsmasq');
const { task } = require('./lib/trace-runner');
const { compareAndWriteFile } = require('./lib/create-file');
@@ -8,7 +8,7 @@ const { compareAndWriteFile } = require('./lib/create-file');
const buildInternalChnDomains = task(__filename, async () => {
const [result] = await Promise.all([
parseFelixDnsmasq('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf'),
fse.ensureDir(path.resolve(__dirname, '../List/internal'))
fsp.mkdir(path.resolve(__dirname, '../List/internal'), { recursive: true })
]);
return compareAndWriteFile(

View File

@@ -2,8 +2,8 @@
const { fetchRemoteTextAndCreateReadlineInterface } = require('./lib/fetch-remote-text-by-line');
const { processLineFromReadline } = require('./lib/process-line');
const path = require('path');
const fse = require('fs-extra');
const fs = require('fs');
const fsp = require('fs/promises');
const { task } = require('./lib/trace-runner');
const RESERVED_IPV4_CIDR = [
@@ -28,7 +28,7 @@ const buildInternalReverseChnCIDR = task(__filename, async () => {
const [{ exclude }, cidr] = await Promise.all([
import('cidr-tools-wasm'),
processLineFromReadline(await fetchRemoteTextAndCreateReadlineInterface('https://raw.githubusercontent.com/misakaio/chnroutes2/master/chnroutes.txt')),
fse.ensureDir(path.resolve(__dirname, '../List/internal'))
fsp.mkdir(path.resolve(__dirname, '../List/internal'), { recursive: true })
]);
const reversedCidr = exclude(

View File

@@ -72,8 +72,7 @@ const buildPhishingDomainSet = task(__filename, async () => {
'https://phishing-filter.pages.dev/phishing-filter-agh.txt'
// Prefer mirror, since malware-filter.gitlab.io has not been updated for a while
// 'https://malware-filter.gitlab.io/malware-filter/phishing-filter-agh.txt'
],
false
]
),
getGorhillPublicSuffixPromise()
]);

View File

@@ -1,7 +1,8 @@
const listDir = require('@sukka/listdir');
const path = require('path');
const fs = require('fs');
const fse = require('fs-extra');
const fsp = require('fs/promises');
const { copy } = require('fs-extra');
const { task } = require('./lib/trace-runner');
const rootPath = path.resolve(__dirname, '../');
@@ -17,8 +18,8 @@ const folderAndFilesToBeDeployed = [
];
const buildPublicHtml = task(__filename, async () => {
await fse.ensureDir(publicPath);
await Promise.all(folderAndFilesToBeDeployed.map(dir => fse.copy(path.resolve(rootPath, dir), path.resolve(publicPath, dir))));
await fsp.mkdir(publicPath, { recursive: true });
await Promise.all(folderAndFilesToBeDeployed.map(dir => copy(path.resolve(rootPath, dir), path.resolve(publicPath, dir))));
const list = await listDir(publicPath, {
ignoreHidden: true,

View File

@@ -1,5 +1,5 @@
// @ts-check
const fse = require('fs-extra');
const { copy } = require('fs-extra');
const { resolve: pathResolve } = require('path');
const { processHosts, processFilterRules } = require('./lib/parse-filter');
@@ -225,7 +225,7 @@ const buildRejectDomainSet = task(__filename, async () => {
pathResolve(__dirname, '../List/internal/reject-stats.txt')
),
// Copy reject_sukka.conf for backward compatibility
fse.copy(pathResolve(__dirname, '../Source/domainset/reject_sukka.conf'), pathResolve(__dirname, '../List/domainset/reject_sukka.conf'))
copy(pathResolve(__dirname, '../Source/domainset/reject_sukka.conf'), pathResolve(__dirname, '../List/domainset/reject_sukka.conf'))
]);
});

View File

@@ -2,7 +2,7 @@ const { fetch } = require('undici');
const tar = require('tar');
const fs = require('fs');
const fsp = fs.promises;
const fse = require('fs-extra');
const { copy } = require('fs-extra');
const path = require('path');
const { tmpdir } = require('os');
const { Readable } = require('stream');
@@ -72,7 +72,7 @@ const downloadPreviousBuild = task(__filename, async () => {
await Promise.all(filesList.map(async p => {
const src = path.join(extractedPath, 'ruleset.skk.moe-master', p);
if (await fileExists(src)) {
return fse.copy(
return copy(
src,
path.join(__dirname, '..', p),
{ overwrite: true }

View File

@@ -2,10 +2,10 @@
const undici = require('undici');
// Enable HTTP/2 supports
undici.setGlobalDispatcher(new undici.Agent({
allowH2: true,
pipelining: 10
}));
// undici.setGlobalDispatcher(new undici.Agent({
// allowH2: true,
// pipelining: 10
// }));
const fetchWithRetry = /** @type {import('undici').fetch} */(require('@vercel/fetch-retry')(undici.fetch));
module.exports.fetchWithRetry = fetchWithRetry;

View File

@@ -202,7 +202,9 @@ async function processFilterRules(filterRulesUrl, fallbackUrls) {
filterRules = (
await Promise.any(
[filterRulesUrl, ...(fallbackUrls || [])].map(async url => {
const text = await fetchWithRetry(url, { signal: controller.signal }).then(r => r.text());
const r = await fetchWithRetry(url, { signal: controller.signal });
const text = await r.text();
controller.abort();
return text;
})