From 315b38b999c1a813e29a7a9a73b2f7bbc4ffe073 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Fri, 6 Sep 2024 19:49:17 +0800 Subject: [PATCH] Chore: refactor output dir --- .gitignore | 14 ---- Build/build-cdn-download-conf.ts | 7 +- Build/build-common.ts | 34 ++++---- Build/build-deprecate-files.ts | 8 +- ...c-direct-lan-ruleset-dns-mapping-module.ts | 9 ++- Build/build-internal-reverse-chn-cidr.ts | 4 +- Build/build-public.ts | 81 ++++++++----------- Build/build-reject-domainset.ts | 5 +- Build/build-sgmodule-always-realip.ts | 5 +- Build/build-sgmodule-redirect.ts | 3 +- Build/build-sspanel-appprofile.ts | 19 ++--- Build/constants/dir.ts | 13 +++ Build/download-mock-assets.ts | 13 +-- Build/download-previous-build.ts | 51 +----------- Build/lib/fetch-text-by-line.bench.ts | 3 +- Build/lib/misc.ts | 5 +- Build/lib/tree-dir.ts | 5 ++ Build/trim-source.ts | 5 +- Build/validate-gfwlist.ts | 5 +- 19 files changed, 117 insertions(+), 172 deletions(-) create mode 100644 Build/constants/dir.ts diff --git a/.gitignore b/.gitignore index ed64fb00..021ebd1e 100644 --- a/.gitignore +++ b/.gitignore @@ -5,17 +5,3 @@ node_modules .cache public tmp* - -# $ build output -List/ -Clash/ -Internal/ -sing-box/ -Modules/sukka_local_dns_mapping.sgmodule -Modules/sukka_url_redirect.sgmodule -Modules/sukka_common_always_realip.sgmodule -Mock/www-google-analytics-com_ga.js -Mock/www-googletagservices-com_gpt.js -Mock/www-google-analytics-com_analytics.js -Mock/www-googlesyndication-com_adsbygoogle.js -Mock/amazon-adsystem-com_amazon-apstag.js diff --git a/Build/build-cdn-download-conf.ts b/Build/build-cdn-download-conf.ts index dfcf1f40..c4181df9 100644 --- a/Build/build-cdn-download-conf.ts +++ b/Build/build-cdn-download-conf.ts @@ -9,6 +9,7 @@ import { domainDeduper } from './lib/domain-deduper'; import { appendArrayInPlace } from './lib/append-array-in-place'; import { sortDomains } from './lib/stable-sort-domain'; import { output } from './lib/misc'; +import { SOURCE_DIR } from './constants/dir'; const getS3OSSDomainsPromise = (async (): Promise => { const trie = createTrie( @@ -58,9 +59,9 @@ export const buildCdnDownloadConf = task(require.main === module, __filename)(as steamDomainSet ] = await Promise.all([ getS3OSSDomainsPromise, - readFileIntoProcessedArray(path.resolve(__dirname, '../Source/domainset/cdn.conf')), - readFileIntoProcessedArray(path.resolve(__dirname, '../Source/domainset/download.conf')), - readFileIntoProcessedArray(path.resolve(__dirname, '../Source/domainset/steam.conf')) + readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/cdn.conf')), + readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/download.conf')), + readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/steam.conf')) ]); appendArrayInPlace(downloadDomainSet, S3OSSDomains.map(domain => `.${domain}`)); diff --git a/Build/build-common.ts b/Build/build-common.ts index 4116724f..a5e730a2 100644 --- a/Build/build-common.ts +++ b/Build/build-common.ts @@ -11,17 +11,13 @@ import { SHARED_DESCRIPTION } from './lib/constants'; import { fdir as Fdir } from 'fdir'; import { appendArrayInPlace } from './lib/append-array-in-place'; import { removeFiles } from './lib/misc'; +import { OUTPUT_CLASH_DIR, OUTPUT_SINGBOX_DIR, OUTPUT_SURGE_DIR, SOURCE_DIR } from './constants/dir'; const MAGIC_COMMAND_SKIP = '# $ custom_build_script'; const MAGIC_COMMAND_RM = '# $ custom_no_output'; const MAGIC_COMMAND_TITLE = '# $ meta_title '; const MAGIC_COMMAND_DESCRIPTION = '# $ meta_description '; -const sourceDir = path.resolve(__dirname, '../Source'); -const outputSurgeDir = path.resolve(__dirname, '../List'); -const outputClashDir = path.resolve(__dirname, '../Clash'); -const outputSingboxDir = path.resolve(__dirname, '../sing-box'); - const domainsetSrcFolder = 'domainset' + path.sep; export const buildCommon = task(require.main === module, __filename)(async (span) => { @@ -46,12 +42,12 @@ export const buildCommon = task(require.main === module, __filename)(async (span return true; }) - .crawl(sourceDir) + .crawl(SOURCE_DIR) .withPromise(); for (let i = 0, len = paths.length; i < len; i++) { const relativePath = paths[i]; - const fullPath = sourceDir + path.sep + relativePath; + const fullPath = SOURCE_DIR + path.sep + relativePath; if (relativePath.startsWith(domainsetSrcFolder)) { promises.push(transformDomainset(span, fullPath, relativePath)); @@ -127,9 +123,9 @@ function transformDomainset(parentSpan: Span, sourcePath: string, relativePath: if (res === $rm) { return removeFiles([ - path.resolve(outputSurgeDir, relativePath), - path.resolve(outputClashDir, `${clashFileBasename}.txt`), - path.resolve(outputSingboxDir, `${clashFileBasename}.json`) + path.resolve(OUTPUT_SURGE_DIR, relativePath), + path.resolve(OUTPUT_CLASH_DIR, `${clashFileBasename}.txt`), + path.resolve(OUTPUT_SINGBOX_DIR, `${clashFileBasename}.json`) ]); } @@ -153,9 +149,9 @@ function transformDomainset(parentSpan: Span, sourcePath: string, relativePath: deduped, 'domainset', [ - path.resolve(outputSurgeDir, relativePath), - path.resolve(outputClashDir, `${clashFileBasename}.txt`), - path.resolve(outputSingboxDir, `${clashFileBasename}.json`) + path.resolve(OUTPUT_SURGE_DIR, relativePath), + path.resolve(OUTPUT_CLASH_DIR, `${clashFileBasename}.txt`), + path.resolve(OUTPUT_SINGBOX_DIR, `${clashFileBasename}.json`) ] ); } @@ -176,9 +172,9 @@ async function transformRuleset(parentSpan: Span, sourcePath: string, relativePa if (res === $rm) { return removeFiles([ - path.resolve(outputSurgeDir, relativePath), - path.resolve(outputClashDir, `${clashFileBasename}.txt`), - path.resolve(outputSingboxDir, `${clashFileBasename}.json`) + path.resolve(OUTPUT_SURGE_DIR, relativePath), + path.resolve(OUTPUT_CLASH_DIR, `${clashFileBasename}.txt`), + path.resolve(OUTPUT_SINGBOX_DIR, `${clashFileBasename}.json`) ]); } @@ -201,9 +197,9 @@ async function transformRuleset(parentSpan: Span, sourcePath: string, relativePa lines, 'ruleset', [ - path.resolve(outputSurgeDir, relativePath), - path.resolve(outputClashDir, `${clashFileBasename}.txt`), - path.resolve(outputSingboxDir, `${clashFileBasename}.json`) + path.resolve(OUTPUT_SURGE_DIR, relativePath), + path.resolve(OUTPUT_CLASH_DIR, `${clashFileBasename}.txt`), + path.resolve(OUTPUT_SINGBOX_DIR, `${clashFileBasename}.json`) ] ); }); diff --git a/Build/build-deprecate-files.ts b/Build/build-deprecate-files.ts index 5c3885f7..7b0faa18 100644 --- a/Build/build-deprecate-files.ts +++ b/Build/build-deprecate-files.ts @@ -1,3 +1,4 @@ +import { OUTPUT_CLASH_DIR, OUTPUT_SURGE_DIR } from './constants/dir'; import { compareAndWriteFile } from './lib/create-file'; import { task } from './trace'; import path from 'node:path'; @@ -8,15 +9,12 @@ const DEPRECATED_FILES = [ ['domainset/reject_phishing', 'This file has been merged with domainset/reject'] ]; -const outputSurgeDir = path.resolve(__dirname, '../List'); -const outputClashDir = path.resolve(__dirname, '../Clash'); - export const buildDeprecateFiles = task(require.main === module, __filename)((span) => span.traceChildAsync('create deprecated files', async (childSpan) => { const promises: Array> = []; for (const [filePath, description] of DEPRECATED_FILES) { - const surgeFile = path.resolve(outputSurgeDir, `${filePath}.conf`); - const clashFile = path.resolve(outputClashDir, `${filePath}.txt`); + const surgeFile = path.resolve(OUTPUT_SURGE_DIR, `${filePath}.conf`); + const clashFile = path.resolve(OUTPUT_CLASH_DIR, `${filePath}.txt`); const content = [ '#########################################', diff --git a/Build/build-domestic-direct-lan-ruleset-dns-mapping-module.ts b/Build/build-domestic-direct-lan-ruleset-dns-mapping-module.ts index 74663f6f..192c45e2 100644 --- a/Build/build-domestic-direct-lan-ruleset-dns-mapping-module.ts +++ b/Build/build-domestic-direct-lan-ruleset-dns-mapping-module.ts @@ -10,10 +10,11 @@ import { createMemoizedPromise } from './lib/memo-promise'; import * as yaml from 'yaml'; import { appendArrayInPlace } from './lib/append-array-in-place'; import { output, writeFile } from './lib/misc'; +import { OUTPUT_INTERNAL_DIR, OUTPUT_MODULES_DIR, SOURCE_DIR } from './constants/dir'; export const getDomesticAndDirectDomainsRulesetPromise = createMemoizedPromise(async () => { - const domestics = await readFileIntoProcessedArray(path.resolve(__dirname, '../Source/non_ip/domestic.conf')); - const directs = await readFileIntoProcessedArray(path.resolve(__dirname, '../Source/non_ip/direct.conf')); + const domestics = await readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/domestic.conf')); + const directs = await readFileIntoProcessedArray(path.resolve(SOURCE_DIR, 'non_ip/direct.conf')); const lans: string[] = []; Object.entries(DOMESTICS).forEach(([, { domains }]) => { @@ -91,10 +92,10 @@ export const buildDomesticRuleset = task(require.main === module, __filename)(as ]) ]) ], - path.resolve(__dirname, '../Modules/sukka_local_dns_mapping.sgmodule') + path.resolve(OUTPUT_MODULES_DIR, 'sukka_local_dns_mapping.sgmodule') ), writeFile( - path.resolve(__dirname, '../Internal/clash_nameserver_policy.yaml'), + path.join(OUTPUT_INTERNAL_DIR, 'clash_nameserver_policy.yaml'), yaml.stringify( { dns: { diff --git a/Build/build-internal-reverse-chn-cidr.ts b/Build/build-internal-reverse-chn-cidr.ts index d0e57db0..f2b2245b 100644 --- a/Build/build-internal-reverse-chn-cidr.ts +++ b/Build/build-internal-reverse-chn-cidr.ts @@ -6,6 +6,7 @@ import { getChnCidrPromise } from './build-chn-cidr'; import { NON_CN_CIDR_INCLUDED_IN_CHNROUTE, RESERVED_IPV4_CIDR } from './constants/cidr'; import { writeFile } from './lib/misc'; +import { OUTPUT_INTERNAL_DIR } from './constants/dir'; export const buildInternalReverseChnCIDR = task(require.main === module, __filename)(async () => { const [cidr] = await getChnCidrPromise(); @@ -21,8 +22,7 @@ export const buildInternalReverseChnCIDR = task(require.main === module, __filen ) ); - const outputDir = path.resolve(__dirname, '../Internal'); - const outputFile = path.join(outputDir, 'reversed-chn-cidr.txt'); + const outputFile = path.join(OUTPUT_INTERNAL_DIR, 'reversed-chn-cidr.txt'); return writeFile( outputFile, diff --git a/Build/build-public.ts b/Build/build-public.ts index 2103b8c0..30805152 100644 --- a/Build/build-public.ts +++ b/Build/build-public.ts @@ -1,69 +1,52 @@ import path from 'node:path'; import fs from 'node:fs'; import fsp from 'node:fs/promises'; + import { task } from './trace'; import { treeDir } from './lib/tree-dir'; import type { TreeType, TreeTypeArray } from './lib/tree-dir'; -import { fdir as Fdir } from 'fdir'; -import Trie from 'mnemonist/trie'; +import { OUTPUT_MOCK_DIR, OUTPUT_MODULES_DIR, PUBLIC_DIR, ROOT_DIR } from './constants/dir'; import { writeFile } from './lib/misc'; +import picocolors from 'picocolors'; -const rootPath = path.resolve(__dirname, '../'); -const publicPath = path.resolve(__dirname, '../public'); +const mockDir = path.join(ROOT_DIR, 'Mock'); +const modulesDir = path.join(ROOT_DIR, 'Modules'); -const folderAndFilesToBeDeployed = [ - `Mock${path.sep}`, - `List${path.sep}`, - `Clash${path.sep}`, - `sing-box${path.sep}`, - `Modules${path.sep}`, - `Script${path.sep}`, - `Internal${path.sep}`, - 'LICENSE' -]; +const copyDirContents = async (srcDir: string, destDir: string) => { + const promises: Array> = []; + + for await (const entry of await fsp.opendir(srcDir)) { + const src = path.join(srcDir, entry.name); + const dest = path.join(destDir, entry.name); + if (entry.isDirectory()) { + console.warn(picocolors.red('[build public] cant copy directory'), src); + } else { + promises.push(fsp.copyFile(src, dest, fs.constants.COPYFILE_FICLONE)); + } + } + + return Promise.all(promises); +}; export const buildPublic = task(require.main === module, __filename)(async (span) => { - fs.mkdirSync(publicPath, { recursive: true }); + await span.traceChildAsync('copy rest of the files', async () => { + await Promise.all([ + fsp.mkdir(OUTPUT_MODULES_DIR, { recursive: true }), + fsp.mkdir(OUTPUT_MOCK_DIR, { recursive: true }) + ]); - await span - .traceChild('copy public files') - .traceAsyncFn(async () => { - const trie = Trie.from(await new Fdir() - .withRelativePaths() - .exclude((dirName) => ( - dirName === 'node_modules' - || dirName === 'Build' - || dirName === 'public' - || dirName[0] === '.' - )) - .crawl(rootPath) - .withPromise()); - - const filesToBeCopied = folderAndFilesToBeDeployed.flatMap(folderOrFile => trie.find(folderOrFile)); - - return Promise.all(filesToBeCopied.map(file => { - const src = path.join(rootPath, file); - const dest = path.join(publicPath, file); - - const destParen = path.dirname(dest); - if (!fs.existsSync(destParen)) { - fs.mkdirSync(destParen, { recursive: true }); - } - - return fsp.copyFile( - src, - dest, - fs.constants.COPYFILE_FICLONE - ); - })); - }); + await Promise.all([ + copyDirContents(modulesDir, OUTPUT_MODULES_DIR), + copyDirContents(mockDir, OUTPUT_MOCK_DIR) + ]); + }); const html = await span .traceChild('generate index.html') - .traceAsyncFn(() => treeDir(publicPath).then(generateHtml)); + .traceAsyncFn(() => treeDir(PUBLIC_DIR).then(generateHtml)); - return writeFile(path.join(publicPath, 'index.html'), html); + return writeFile(path.join(PUBLIC_DIR, 'index.html'), html); }); const priorityOrder: Record<'default' | string & {}, number> = { diff --git a/Build/build-reject-domainset.ts b/Build/build-reject-domainset.ts index d7bdfad1..e57ccae8 100644 --- a/Build/build-reject-domainset.ts +++ b/Build/build-reject-domainset.ts @@ -21,8 +21,9 @@ import { getPhishingDomains } from './lib/get-phishing-domains'; import { setAddFromArray, setAddFromArrayCurried } from './lib/set-add-from-array'; import { output } from './lib/misc'; import { appendArrayInPlace } from './lib/append-array-in-place'; +import { OUTPUT_INTERNAL_DIR, SOURCE_DIR } from './constants/dir'; -const getRejectSukkaConfPromise = readFileIntoProcessedArray(path.resolve(__dirname, '../Source/domainset/reject_sukka.conf')); +const getRejectSukkaConfPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/reject_sukka.conf')); export const buildRejectDomainSet = task(require.main === module, __filename)(async (span) => { /** Whitelists */ @@ -214,7 +215,7 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as compareAndWriteFile( span, rejectDomainsStats, - path.resolve(__dirname, '../Internal/reject-stats.txt') + path.join(OUTPUT_INTERNAL_DIR, 'reject-stats.txt') ) ]); }); diff --git a/Build/build-sgmodule-always-realip.ts b/Build/build-sgmodule-always-realip.ts index 818369c5..54bdd640 100644 --- a/Build/build-sgmodule-always-realip.ts +++ b/Build/build-sgmodule-always-realip.ts @@ -4,6 +4,7 @@ import { compareAndWriteFile } from './lib/create-file'; import { DIRECTS, LANS } from '../Source/non_ip/direct'; import * as yaml from 'yaml'; import { writeFile } from './lib/misc'; +import { OUTPUT_INTERNAL_DIR, OUTPUT_MODULES_DIR } from './constants/dir'; const HOSTNAMES = [ // Network Detection, Captive Portal @@ -59,10 +60,10 @@ export const buildAlwaysRealIPModule = task(require.main === module, __filename) '[General]', `always-real-ip = %APPEND% ${HOSTNAMES.concat(surge).join(', ')}` ], - path.resolve(__dirname, '../Modules/sukka_common_always_realip.sgmodule') + path.resolve(OUTPUT_MODULES_DIR, 'sukka_common_always_realip.sgmodule') ), writeFile( - path.resolve(__dirname, '../Internal/clash_fake_ip_filter.yaml'), + path.join(OUTPUT_INTERNAL_DIR, 'clash_fake_ip_filter.yaml'), yaml.stringify( { dns: { diff --git a/Build/build-sgmodule-redirect.ts b/Build/build-sgmodule-redirect.ts index 98c45c26..9fa7c044 100644 --- a/Build/build-sgmodule-redirect.ts +++ b/Build/build-sgmodule-redirect.ts @@ -3,6 +3,7 @@ import { task } from './trace'; import { compareAndWriteFile } from './lib/create-file'; import { getHostname } from 'tldts'; import { isTruthy } from './lib/misc'; +import { OUTPUT_MODULES_DIR } from './constants/dir'; function escapeRegExp(string = '') { const reRegExpChar = /[$()*+.?[\\\]^{|}]/g; @@ -148,6 +149,6 @@ export const buildRedirectModule = task(require.main === module, __filename)(asy ...REDIRECT_MIRROR.map(([from, to]) => `^https?://${escapeRegExp(from)}(.*) ${to}$1 header`), ...REDIRECT_FAKEWEBSITES.map(([from, to]) => `^https?://(www.)?${escapeRegExp(from)} ${to} 307`) ], - path.resolve(__dirname, '../Modules/sukka_url_redirect.sgmodule') + path.join(OUTPUT_MODULES_DIR, 'sukka_url_redirect.sgmodule') ); }); diff --git a/Build/build-sspanel-appprofile.ts b/Build/build-sspanel-appprofile.ts index 8aee4960..09b63198 100644 --- a/Build/build-sspanel-appprofile.ts +++ b/Build/build-sspanel-appprofile.ts @@ -12,6 +12,7 @@ import { compareAndWriteFile } from './lib/create-file'; import { getMicrosoftCdnRulesetPromise } from './build-microsoft-cdn'; import { isTruthy } from './lib/misc'; import { appendArrayInPlace } from './lib/append-array-in-place'; +import { OUTPUT_INTERNAL_DIR, SOURCE_DIR } from './constants/dir'; const POLICY_GROUPS: Array<[name: string, insertProxy: boolean, insertDirect: boolean]> = [ ['Default Proxy', true, false], @@ -55,18 +56,18 @@ export const buildSSPanelUIMAppProfile = task(require.main === module, __filenam ), getAppleCdnDomainsPromise().then(domains => domains.map(domain => `DOMAIN-SUFFIX,${domain}`)), getMicrosoftCdnRulesetPromise().then(surgeRulesetToClashClassicalTextRuleset), - readFileIntoProcessedArray(path.resolve(__dirname, '../Source/non_ip/apple_cn.conf')), - readFileIntoProcessedArray(path.resolve(__dirname, '../Source/non_ip/neteasemusic.conf')).then(surgeRulesetToClashClassicalTextRuleset), + readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/apple_cn.conf')), + readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/neteasemusic.conf')).then(surgeRulesetToClashClassicalTextRuleset), // microsoft & apple - domains - readFileIntoProcessedArray(path.resolve(__dirname, '../Source/non_ip/microsoft.conf')), - readFileIntoProcessedArray(path.resolve(__dirname, '../Source/non_ip/apple_services.conf')).then(surgeRulesetToClashClassicalTextRuleset), + readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/microsoft.conf')), + readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/apple_services.conf')).then(surgeRulesetToClashClassicalTextRuleset), // stream - domains surgeRulesetToClashClassicalTextRuleset(AllStreamServices.flatMap((i) => i.rules)), // steam - domains - readFileIntoProcessedArray(path.resolve(__dirname, '../Source/domainset/steam.conf')).then(surgeDomainsetToClashRuleset), + readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/steam.conf')).then(surgeDomainsetToClashRuleset), // global - domains - readFileIntoProcessedArray(path.resolve(__dirname, '../Source/non_ip/global.conf')).then(surgeRulesetToClashClassicalTextRuleset), - readFileIntoProcessedArray(path.resolve(__dirname, '../Source/non_ip/telegram.conf')).then(surgeRulesetToClashClassicalTextRuleset), + readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/global.conf')).then(surgeRulesetToClashClassicalTextRuleset), + readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/telegram.conf')).then(surgeRulesetToClashClassicalTextRuleset), // domestic - ip cidr getChnCidrPromise().then(([cidrs4, cidrs6]) => [ ...cidrs4.map(cidr => `IP-CIDR,${cidr}`), @@ -83,7 +84,7 @@ export const buildSSPanelUIMAppProfile = task(require.main === module, __filenam // global - ip cidr getTelegramCIDRPromise(), // lan - ip cidr - readFileIntoProcessedArray(path.resolve(__dirname, '../Source/ip/lan.conf')) + readFileIntoProcessedArray(path.join(SOURCE_DIR, 'ip/lan.conf')) ] as const); const telegramCidrs = rawTelegramCidrs.map(removeNoResolved); @@ -121,7 +122,7 @@ export const buildSSPanelUIMAppProfile = task(require.main === module, __filenam await compareAndWriteFile( span, output, - path.resolve(__dirname, '../Internal/appprofile.php') + path.resolve(OUTPUT_INTERNAL_DIR, 'appprofile.php') ); }); diff --git a/Build/constants/dir.ts b/Build/constants/dir.ts new file mode 100644 index 00000000..8ca55486 --- /dev/null +++ b/Build/constants/dir.ts @@ -0,0 +1,13 @@ +import path from 'node:path'; + +export const ROOT_DIR = path.resolve(__dirname, '../..'); + +export const SOURCE_DIR = path.join(ROOT_DIR, 'Source'); + +export const PUBLIC_DIR = path.resolve(ROOT_DIR, 'public'); +export const OUTPUT_SURGE_DIR = path.join(PUBLIC_DIR, 'List'); +export const OUTPUT_CLASH_DIR = path.resolve(PUBLIC_DIR, 'Clash'); +export const OUTPUT_SINGBOX_DIR = path.resolve(PUBLIC_DIR, 'sing-box'); +export const OUTPUT_MODULES_DIR = path.resolve(PUBLIC_DIR, 'Modules'); +export const OUTPUT_INTERNAL_DIR = path.resolve(PUBLIC_DIR, 'Internal'); +export const OUTPUT_MOCK_DIR = path.resolve(PUBLIC_DIR, 'Mock'); diff --git a/Build/download-mock-assets.ts b/Build/download-mock-assets.ts index 6fcb4458..2fd074a7 100644 --- a/Build/download-mock-assets.ts +++ b/Build/download-mock-assets.ts @@ -1,9 +1,11 @@ import { task } from './trace'; import path from 'node:path'; import fs from 'node:fs'; +import fsp from 'node:fs/promises'; import { Readable } from 'node:stream'; import { pipeline } from 'node:stream/promises'; import { fetchWithRetry } from './lib/fetch-retry'; +import { OUTPUT_MOCK_DIR } from './constants/dir'; const ASSETS_LIST = { 'www-google-analytics-com_ga.js': 'https://raw.githubusercontent.com/AdguardTeam/Scriptlets/master/dist/redirect-files/google-analytics-ga.js', @@ -13,19 +15,20 @@ const ASSETS_LIST = { 'amazon-adsystem-com_amazon-apstag.js': 'https://raw.githubusercontent.com/AdguardTeam/Scriptlets/master/dist/redirect-files/amazon-apstag.js' } as const; -const mockDir = path.resolve(__dirname, '../Mock'); - export const downloadMockAssets = task(require.main === module, __filename)((span) => Promise.all(Object.entries(ASSETS_LIST).map( ([filename, url]) => span - .traceChildAsync(url, () => fetchWithRetry(url).then(res => { - const src = path.join(mockDir, filename); + .traceChildAsync(url, async () => { + const res = await fetchWithRetry(url); + + const src = path.join(OUTPUT_MOCK_DIR, filename); if (!res.body) { throw new Error(`Empty body from ${url}`); } + await fsp.mkdir(OUTPUT_MOCK_DIR, { recursive: true }); return pipeline( Readable.fromWeb(res.body), fs.createWriteStream(src, 'utf-8') ); - })) + }) ))); diff --git a/Build/download-previous-build.ts b/Build/download-previous-build.ts index f414dabd..c1843fe1 100644 --- a/Build/download-previous-build.ts +++ b/Build/download-previous-build.ts @@ -1,57 +1,17 @@ -import { existsSync, createWriteStream } from 'node:fs'; +import { createWriteStream } from 'node:fs'; import { mkdir } from 'node:fs/promises'; import path from 'node:path'; import { pipeline } from 'node:stream/promises'; -import { readFileByLine } from './lib/fetch-text-by-line'; -import { isCI } from 'ci-info'; import { task } from './trace'; import { defaultRequestInit, fetchWithRetry } from './lib/fetch-retry'; import tarStream from 'tar-stream'; import zlib from 'node:zlib'; import { Readable } from 'node:stream'; -const IS_READING_BUILD_OUTPUT = 1 << 2; -const ALL_FILES_EXISTS = 1 << 3; - const GITHUB_CODELOAD_URL = 'https://codeload.github.com/sukkalab/ruleset.skk.moe/tar.gz/master'; const GITLAB_CODELOAD_URL = 'https://gitlab.com/SukkaW/ruleset.skk.moe/-/archive/master/ruleset.skk.moe-master.tar.gz'; export const downloadPreviousBuild = task(require.main === module, __filename)(async (span) => { - const buildOutputList: string[] = []; - - let flag = 1 | ALL_FILES_EXISTS; - - await span - .traceChild('read .gitignore') - .traceAsyncFn(async () => { - for await (const line of readFileByLine(path.resolve(__dirname, '../.gitignore'))) { - if (line === '# $ build output') { - flag = flag | IS_READING_BUILD_OUTPUT; - continue; - } - if (!(flag & IS_READING_BUILD_OUTPUT)) { - continue; - } - - buildOutputList.push(line); - - if (!isCI && !existsSync(path.join(__dirname, '..', line))) { - flag = flag & ~ALL_FILES_EXISTS; - } - } - }); - - if (isCI) { - flag = flag & ~ALL_FILES_EXISTS; - } - - if (flag & ALL_FILES_EXISTS) { - console.log('All files exists, skip download.'); - return; - } - - const filesList = buildOutputList.map(f => path.join('ruleset.skk.moe-master', f)); - const tarGzUrl = await span.traceChildAsync('get tar.gz url', async () => { const resp = await fetchWithRetry(GITHUB_CODELOAD_URL, { ...defaultRequestInit, @@ -68,6 +28,8 @@ export const downloadPreviousBuild = task(require.main === module, __filename)(a return GITHUB_CODELOAD_URL; }); + const publicDir = path.resolve(__dirname, '..', 'public'); + return span.traceChildAsync('download & extract previoud build', async () => { const resp = await fetchWithRetry(tarGzUrl, { headers: { @@ -112,14 +74,9 @@ export const downloadPreviousBuild = task(require.main === module, __filename)(a entry.resume(); // Drain the entry continue; } - // filter entry - if (!filesList.some(f => entry.header.name.startsWith(f))) { - entry.resume(); // Drain the entry - continue; - } const relativeEntryPath = entry.header.name.replace(pathPrefix, ''); - const targetPath = path.join(__dirname, '..', relativeEntryPath); + const targetPath = path.join(publicDir, relativeEntryPath); await mkdir(path.dirname(targetPath), { recursive: true }); await pipeline(entry, createWriteStream(targetPath)); diff --git a/Build/lib/fetch-text-by-line.bench.ts b/Build/lib/fetch-text-by-line.bench.ts index b26567c3..e102354f 100644 --- a/Build/lib/fetch-text-by-line.bench.ts +++ b/Build/lib/fetch-text-by-line.bench.ts @@ -3,8 +3,9 @@ import { processLine, processLineFromReadline } from './process-line'; import { readFileByLine } from './fetch-text-by-line'; import path from 'node:path'; import fsp from 'node:fs/promises'; +import { SOURCE_DIR } from '../constants/dir'; -const file = path.resolve(__dirname, '../../Source/domainset/cdn.conf'); +const file = path.join(SOURCE_DIR, 'domainset/cdn.conf'); group('read file by line', () => { bench('readFileByLine', () => processLineFromReadline(readFileByLine(file))); diff --git a/Build/lib/misc.ts b/Build/lib/misc.ts index 714a1f72..182e6bbf 100644 --- a/Build/lib/misc.ts +++ b/Build/lib/misc.ts @@ -1,6 +1,7 @@ import path, { dirname } from 'node:path'; import fs from 'node:fs'; import fsp from 'node:fs/promises'; +import { OUTPUT_CLASH_DIR, OUTPUT_SINGBOX_DIR, OUTPUT_SURGE_DIR } from '../constants/dir'; export const isTruthy = (i: T | 0 | '' | false | null | undefined): i is T => !!i; @@ -52,10 +53,6 @@ export const domainWildCardToRegex = (domain: string) => { return result; }; -const OUTPUT_SURGE_DIR = path.resolve(__dirname, '../../List'); -const OUTPUT_CLASH_DIR = path.resolve(__dirname, '../../Clash'); -const OUTPUT_SINGBOX_DIR = path.resolve(__dirname, '../../sing-box'); - export const output = (id: string, type: 'non_ip' | 'ip' | 'domainset') => { return [ path.join(OUTPUT_SURGE_DIR, type, id + '.conf'), diff --git a/Build/lib/tree-dir.ts b/Build/lib/tree-dir.ts index 15f951be..c02c69ec 100644 --- a/Build/lib/tree-dir.ts +++ b/Build/lib/tree-dir.ts @@ -25,6 +25,11 @@ export const treeDir = async (rootPath: string): Promise => { const walk = async (dir: string, node: TreeTypeArray, dirRelativeToRoot = ''): Promise => { const promises: Array> = []; for await (const child of await fsp.opendir(dir)) { + // Ignore hidden files + if (child.name[0] === '.' || child.name === 'CNAME') { + continue; + } + const childFullPath = child.parentPath + sep + child.name; const childRelativeToRoot = dirRelativeToRoot + sep + child.name; diff --git a/Build/trim-source.ts b/Build/trim-source.ts index e9201a1b..da6720d7 100644 --- a/Build/trim-source.ts +++ b/Build/trim-source.ts @@ -2,8 +2,7 @@ import path from 'node:path'; import fsp from 'node:fs/promises'; import { fdir as Fdir } from 'fdir'; import { readFileByLine } from './lib/fetch-text-by-line'; - -const sourceDir = path.resolve(__dirname, '../Source'); +import { SOURCE_DIR } from './constants/dir'; (async () => { const promises: Array> = []; @@ -27,7 +26,7 @@ const sourceDir = path.resolve(__dirname, '../Source'); return true; }) - .crawl(sourceDir) + .crawl(SOURCE_DIR) .withPromise(); for (let i = 0, len = paths.length; i < len; i++) { diff --git a/Build/validate-gfwlist.ts b/Build/validate-gfwlist.ts index 91213bed..30069180 100644 --- a/Build/validate-gfwlist.ts +++ b/Build/validate-gfwlist.ts @@ -5,6 +5,7 @@ import { createTrie } from './lib/trie'; import { parse } from 'csv-parse/sync'; import { readFileByLine } from './lib/fetch-text-by-line'; import path from 'node:path'; +import { SOURCE_DIR } from './constants/dir'; export const parseGfwList = async () => { const whiteSet = new Set(); @@ -105,8 +106,8 @@ export const parseGfwList = async () => { }; await Promise.all([ - runAgainstRuleset(path.resolve(__dirname, '../Source/non_ip/global.conf')), - runAgainstRuleset(path.resolve(__dirname, '../Source/non_ip/telegram.conf')), + runAgainstRuleset(path.join(SOURCE_DIR, 'non_ip/global.conf')), + runAgainstRuleset(path.join(SOURCE_DIR, 'non_ip/telegram.conf')), runAgainstRuleset(path.resolve(__dirname, '../List/non_ip/stream.conf')) ]);