diff --git a/Build/build-apple-cdn.ts b/Build/build-apple-cdn.ts index 4ca3b189..9faf26a2 100644 --- a/Build/build-apple-cdn.ts +++ b/Build/build-apple-cdn.ts @@ -1,11 +1,11 @@ // @ts-check -import path from 'path'; import { createRuleset } from './lib/create-file'; import { parseFelixDnsmasq } from './lib/parse-dnsmasq'; import { task } from './trace'; import { SHARED_DESCRIPTION } from './lib/constants'; import { createMemoizedPromise } from './lib/memo-promise'; import { TTL, deserializeArray, fsFetchCache, serializeArray, createCacheKey } from './lib/cache-filesystem'; +import { output } from './lib/misc'; const cacheKey = createCacheKey(__filename); @@ -42,9 +42,7 @@ export const buildAppleCdn = task(require.main === module, __filename)(async (sp new Date(), ruleset, 'ruleset', - path.resolve(__dirname, '../List/non_ip/apple_cdn.conf'), - path.resolve(__dirname, '../Clash/non_ip/apple_cdn.txt'), - path.resolve(__dirname, '../sing-box/non_ip/apple_cdn.json') + ...output('apple_cdn', 'non_ip') ), createRuleset( span, @@ -53,10 +51,7 @@ export const buildAppleCdn = task(require.main === module, __filename)(async (sp new Date(), domainset, 'domainset', - path.resolve(__dirname, '../List/domainset/apple_cdn.conf'), - path.resolve(__dirname, '../Clash/domainset/apple_cdn.txt'), - path.resolve(__dirname, '../sing-box/domainset/apple_cdn.json'), - path.resolve(__dirname, '../Clash/clash_mrs_domain/apple_cdn.mrs') + ...output('apple_cdn', 'domainset') ) ]); }); diff --git a/Build/build-cdn-download-conf.ts b/Build/build-cdn-download-conf.ts index ee4c2ff1..83152453 100644 --- a/Build/build-cdn-download-conf.ts +++ b/Build/build-cdn-download-conf.ts @@ -8,6 +8,7 @@ import { getPublicSuffixListTextPromise } from './lib/download-publicsuffixlist' import { domainDeduper } from './lib/domain-deduper'; import { appendArrayInPlace } from './lib/append-array-in-place'; import { sortDomains } from './lib/stable-sort-domain'; +import { output } from './lib/misc'; const getS3OSSDomainsPromise = (async (): Promise => { const trie = createTrie( @@ -77,10 +78,7 @@ export const buildCdnDownloadConf = task(require.main === module, __filename)(as new Date(), sortDomains(domainDeduper(cdnDomainsList)), 'domainset', - path.resolve(__dirname, '../List/domainset/cdn.conf'), - path.resolve(__dirname, '../Clash/domainset/cdn.txt'), - path.resolve(__dirname, '../sing-box/domainset/cdn.json'), - path.resolve(__dirname, '../Clash/clash_mrs_domain/cdn.mrs') + ...output('cdn', 'domainset') ), createRuleset( span, @@ -93,10 +91,7 @@ export const buildCdnDownloadConf = task(require.main === module, __filename)(as new Date(), sortDomains(domainDeduper(downloadDomainSet)), 'domainset', - path.resolve(__dirname, '../List/domainset/download.conf'), - path.resolve(__dirname, '../Clash/domainset/download.txt'), - path.resolve(__dirname, '../sing-box/domainset/download.json'), - path.resolve(__dirname, '../Clash/clash_mrs_domain/download.mrs') + ...output('download', 'domainset') ) ]); }); diff --git a/Build/build-chn-cidr.ts b/Build/build-chn-cidr.ts index 0025c102..c371a9bc 100644 --- a/Build/build-chn-cidr.ts +++ b/Build/build-chn-cidr.ts @@ -1,6 +1,5 @@ import { fetchRemoteTextByLine } from './lib/fetch-text-by-line'; -import { resolve as pathResolve } from 'path'; -import { compareAndWriteFile, withBannerArray } from './lib/create-file'; +import { createRuleset } from './lib/create-file'; import { processLineFromReadline } from './lib/process-line'; import { task } from './trace'; @@ -8,6 +7,7 @@ import { exclude } from 'fast-cidr-tools'; import { createMemoizedPromise } from './lib/memo-promise'; import { CN_CIDR_NOT_INCLUDED_IN_CHNROUTE, NON_CN_CIDR_INCLUDED_IN_CHNROUTE } from './constants/cidr'; import { appendArrayInPlace } from './lib/append-array-in-place'; +import { output } from './lib/misc'; export const getChnCidrPromise = createMemoizedPromise(async () => { const cidr4 = await processLineFromReadline(await fetchRemoteTextByLine('https://raw.githubusercontent.com/misakaio/chnroutes2/master/chnroutes.txt')); @@ -30,57 +30,29 @@ export const buildChnCidr = task(require.main === module, __filename)(async (spa // Can not use createRuleset here, as Clash support advanced ipset syntax return Promise.all([ - compareAndWriteFile( + createRuleset( span, - withBannerArray( - 'Sukka\'s Ruleset - Mainland China IPv4 CIDR', - [ - ...description, - 'Data from https://misaka.io (misakaio @ GitHub)' - ], - new Date(), - filteredCidr4.map(i => `IP-CIDR,${i}`) - ), - pathResolve(__dirname, '../List/ip/china_ip.conf') + 'Sukka\'s Ruleset - Mainland China IPv4 CIDR', + [ + ...description, + 'Data from https://misaka.io (misakaio @ GitHub)' + ], + new Date(), + filteredCidr4, + 'ipcidr', + ...output('china_ip', 'ip') ), - compareAndWriteFile( + createRuleset( span, - withBannerArray( - 'Sukka\'s Ruleset - Mainland China IPv6 CIDR', - [ - ...description, - 'Data from https://github.com/gaoyifan/china-operator-ip' - ], - new Date(), - cidr6.map(i => `IP-CIDR6,${i}`) - ), - pathResolve(__dirname, '../List/ip/china_ip_ipv6.conf') - ), - compareAndWriteFile( - span, - withBannerArray( - 'Sukka\'s Ruleset - Mainland China IPv4 CIDR', - [ - ...description, - 'Data from https://misaka.io (misakaio @ GitHub)' - ], - new Date(), - filteredCidr4 - ), - pathResolve(__dirname, '../Clash/ip/china_ip.txt') - ), - compareAndWriteFile( - span, - withBannerArray( - 'Sukka\'s Ruleset - Mainland China IPv6 CIDR', - [ - ...description, - 'Data from https://github.com/gaoyifan/china-operator-ip' - ], - new Date(), - cidr6 - ), - pathResolve(__dirname, '../Clash/ip/china_ip_ipv6.txt') + 'Sukka\'s Ruleset - Mainland China IPv6 CIDR', + [ + ...description, + 'Data from https://github.com/gaoyifan/china-operator-ip' + ], + new Date(), + cidr6, + 'ipcidr6', + ...output('china_ip_ipv6', 'ip') ) ]); }); diff --git a/Build/build-domestic-direct-lan-ruleset-dns-mapping-module.ts b/Build/build-domestic-direct-lan-ruleset-dns-mapping-module.ts index bb6332ef..3c7c8a6e 100644 --- a/Build/build-domestic-direct-lan-ruleset-dns-mapping-module.ts +++ b/Build/build-domestic-direct-lan-ruleset-dns-mapping-module.ts @@ -9,7 +9,7 @@ import { SHARED_DESCRIPTION } from './lib/constants'; import { createMemoizedPromise } from './lib/memo-promise'; import * as yaml from 'yaml'; import { appendArrayInPlace } from './lib/append-array-in-place'; -import { writeFile } from './lib/misc'; +import { output, writeFile } from './lib/misc'; export const getDomesticAndDirectDomainsRulesetPromise = createMemoizedPromise(async () => { const domestics = await readFileIntoProcessedArray(path.resolve(__dirname, '../Source/non_ip/domestic.conf')); @@ -48,9 +48,7 @@ export const buildDomesticRuleset = task(require.main === module, __filename)(as new Date(), res[0], 'ruleset', - path.resolve(__dirname, '../List/non_ip/domestic.conf'), - path.resolve(__dirname, '../Clash/non_ip/domestic.txt'), - path.resolve(__dirname, '../sing-box/non_ip/domestic.json') + ...output('domestic', 'non_ip') ), createRuleset( span, @@ -63,9 +61,7 @@ export const buildDomesticRuleset = task(require.main === module, __filename)(as new Date(), res[1], 'ruleset', - path.resolve(__dirname, '../List/non_ip/direct.conf'), - path.resolve(__dirname, '../Clash/non_ip/direct.txt'), - path.resolve(__dirname, '../sing-box/non_ip/direct.json') + ...output('direct', 'non_ip') ), createRuleset( span, @@ -78,9 +74,7 @@ export const buildDomesticRuleset = task(require.main === module, __filename)(as new Date(), res[2], 'ruleset', - path.resolve(__dirname, '../List/non_ip/lan.conf'), - path.resolve(__dirname, '../Clash/non_ip/lan.txt'), - path.resolve(__dirname, '../sing-box/non_ip/lan.json') + ...output('lan', 'non_ip') ), compareAndWriteFile( span, diff --git a/Build/build-microsoft-cdn.ts b/Build/build-microsoft-cdn.ts index 827154fe..13563447 100644 --- a/Build/build-microsoft-cdn.ts +++ b/Build/build-microsoft-cdn.ts @@ -1,4 +1,3 @@ -import path from 'path'; import { task } from './trace'; import { createRuleset } from './lib/create-file'; import { fetchRemoteTextByLine } from './lib/fetch-text-by-line'; @@ -7,6 +6,7 @@ import { SHARED_DESCRIPTION } from './lib/constants'; import { createMemoizedPromise } from './lib/memo-promise'; import { extractDomainsFromFelixDnsmasq } from './lib/parse-dnsmasq'; import { sortDomains } from './lib/stable-sort-domain'; +import { output } from './lib/misc'; const PROBE_DOMAINS = ['.microsoft.com', '.windows.net', '.windows.com', '.windowsupdate.com', '.windowssearch.com', '.office.net']; @@ -63,8 +63,6 @@ export const buildMicrosoftCdn = task(require.main === module, __filename)(async new Date(), res, 'ruleset', - path.resolve(__dirname, '../List/non_ip/microsoft_cdn.conf'), - path.resolve(__dirname, '../Clash/non_ip/microsoft_cdn.txt'), - path.resolve(__dirname, '../sing-box/non_ip/microsoft_cdn.json') + ...output('microsoft_cdn', 'non_ip') ); }); diff --git a/Build/build-reject-domainset.ts b/Build/build-reject-domainset.ts index 02234dea..1b19771f 100644 --- a/Build/build-reject-domainset.ts +++ b/Build/build-reject-domainset.ts @@ -18,6 +18,7 @@ import { SHARED_DESCRIPTION } from './lib/constants'; import { getPhishingDomains } from './lib/get-phishing-domains'; import { setAddFromArray, setAddFromArrayCurried } from './lib/set-add-from-array'; +import { output } from './lib/misc'; const getRejectSukkaConfPromise = readFileIntoProcessedArray(path.resolve(__dirname, '../Source/domainset/reject_sukka.conf')); @@ -190,10 +191,7 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as new Date(), span.traceChildSync('sort reject domainset (base)', () => sortDomains(dudupedDominArray, domainArrayMainDomainMap, domainArraySubdomainMap)), 'domainset', - path.resolve(__dirname, '../List/domainset/reject.conf'), - path.resolve(__dirname, '../Clash/domainset/reject.txt'), - path.resolve(__dirname, '../sing-box/domainset/reject.json'), - path.resolve(__dirname, '../Clash/clash_mrs_domain/reject.mrs') + ...output('reject', 'domainset') ), createRuleset( span, @@ -212,10 +210,7 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as new Date(), span.traceChildSync('sort reject domainset (extra)', () => sortDomains(dudupedDominArrayExtra, domainArrayMainDomainMap, domainArraySubdomainMap)), 'domainset', - path.resolve(__dirname, '../List/domainset/reject_extra.conf'), - path.resolve(__dirname, '../Clash/domainset/reject_extra.txt'), - path.resolve(__dirname, '../sing-box/domainset/reject_extra.json'), - path.resolve(__dirname, '../Clash/clash_mrs_domain/reject_extra.mrs') + ...output('reject_extra', 'domainset') ), compareAndWriteFile( span, diff --git a/Build/build-reject-ip-list.ts b/Build/build-reject-ip-list.ts index 67fbe61a..c305e9d4 100644 --- a/Build/build-reject-ip-list.ts +++ b/Build/build-reject-ip-list.ts @@ -9,6 +9,7 @@ import { TTL, deserializeArray, fsFetchCache, serializeArray, createCacheKey } f import { fetchAssets } from './lib/fetch-assets'; import { processLine } from './lib/process-line'; import { appendArrayInPlace } from './lib/append-array-in-place'; +import { output } from './lib/misc'; const cacheKey = createCacheKey(__filename); @@ -100,8 +101,6 @@ export const buildRejectIPList = task(require.main === module, __filename)(async new Date(), result, 'ruleset', - path.resolve(__dirname, '../List/ip/reject.conf'), - path.resolve(__dirname, '../Clash/ip/reject.txt'), - path.resolve(__dirname, '../sing-box/ip/reject.json') + ...output('reject', 'ip') ); }); diff --git a/Build/build-speedtest-domainset.ts b/Build/build-speedtest-domainset.ts index 4c54c474..00c2b182 100644 --- a/Build/build-speedtest-domainset.ts +++ b/Build/build-speedtest-domainset.ts @@ -12,6 +12,7 @@ import { readFileIntoProcessedArray } from './lib/fetch-text-by-line'; import { TTL, deserializeArray, fsFetchCache, serializeArray, createCacheKey } from './lib/cache-filesystem'; import { createTrie } from './lib/trie'; +import { output } from './lib/misc'; const s = new Sema(2); const cacheKey = createCacheKey(__filename); @@ -250,9 +251,6 @@ export const buildSpeedtestDomainSet = task(require.main === module, __filename) new Date(), deduped, 'domainset', - path.resolve(__dirname, '../List/domainset/speedtest.conf'), - path.resolve(__dirname, '../Clash/domainset/speedtest.txt'), - path.resolve(__dirname, '../sing-box/domainset/speedtest.json'), - path.resolve(__dirname, '../Clash/clash_mrs_domain/speedtest.mrs') + ...output('speedtest', 'domainset') ); }); diff --git a/Build/build-stream-service.ts b/Build/build-stream-service.ts index 28e6b1dd..575694d0 100644 --- a/Build/build-stream-service.ts +++ b/Build/build-stream-service.ts @@ -2,11 +2,11 @@ import type { Span } from './trace'; import { task } from './trace'; -import path from 'path'; import { createRuleset } from './lib/create-file'; import { ALL, NORTH_AMERICA, EU, HK, TW, JP, KR } from '../Source/stream'; import { SHARED_DESCRIPTION } from './lib/constants'; +import { output } from './lib/misc'; export const createRulesetForStreamService = (span: Span, fileId: string, title: string, streamServices: Array) => { return span.traceChildAsync(fileId, async (childSpan) => Promise.all([ @@ -22,9 +22,7 @@ export const createRulesetForStreamService = (span: Span, fileId: string, title: new Date(), streamServices.flatMap((i) => i.rules), 'ruleset', - path.resolve(__dirname, `../List/non_ip/${fileId}.conf`), - path.resolve(__dirname, `../Clash/non_ip/${fileId}.txt`), - path.resolve(__dirname, `../sing-box/non_ip/${fileId}.json`) + ...output(fileId, 'non_ip') ), // IP createRuleset( @@ -45,9 +43,7 @@ export const createRulesetForStreamService = (span: Span, fileId: string, title: : [] )), 'ruleset', - path.resolve(__dirname, `../List/ip/${fileId}.conf`), - path.resolve(__dirname, `../Clash/ip/${fileId}.txt`), - path.resolve(__dirname, `../sing-box/ip/${fileId}.json`) + ...output(fileId, 'ip') ) ])); }; diff --git a/Build/build-telegram-cidr.ts b/Build/build-telegram-cidr.ts index 437f852c..cc347011 100644 --- a/Build/build-telegram-cidr.ts +++ b/Build/build-telegram-cidr.ts @@ -1,13 +1,13 @@ // @ts-check import { defaultRequestInit, fetchWithRetry } from './lib/fetch-retry'; import { createReadlineInterfaceFromResponse } from './lib/fetch-text-by-line'; -import path from 'path'; import { isProbablyIpv4, isProbablyIpv6 } from './lib/is-fast-ip'; import { processLine } from './lib/process-line'; import { createRuleset } from './lib/create-file'; import { task } from './trace'; import { SHARED_DESCRIPTION } from './lib/constants'; import { createMemoizedPromise } from './lib/memo-promise'; +import { output } from './lib/misc'; export const getTelegramCIDRPromise = createMemoizedPromise(async () => { const resp = await fetchWithRetry('https://core.telegram.org/resources/cidr.txt', defaultRequestInit); @@ -52,8 +52,6 @@ export const buildTelegramCIDR = task(require.main === module, __filename)(async date, results, 'ruleset', - path.resolve(__dirname, '../List/ip/telegram.conf'), - path.resolve(__dirname, '../Clash/ip/telegram.txt'), - path.resolve(__dirname, '../sing-box/ip/telegram.json') + ...output('telegram', 'ip') ); }); diff --git a/Build/lib/create-file.ts b/Build/lib/create-file.ts index e0ba9558..f833b921 100644 --- a/Build/lib/create-file.ts +++ b/Build/lib/create-file.ts @@ -7,7 +7,7 @@ import fs from 'fs'; import { fastStringArrayJoin, writeFile } from './misc'; import { readFileByLine } from './fetch-text-by-line'; import stringify from 'json-stringify-pretty-compact'; -import { surgeDomainsetToSingbox, surgeRulesetToSingbox } from './singbox'; +import { ipCidrListToSingbox, surgeDomainsetToSingbox, surgeRulesetToSingbox } from './singbox'; export async function compareAndWriteFile(span: Span, linesA: string[], filePath: string) { let isEqual = true; @@ -79,7 +79,7 @@ export async function compareAndWriteFile(span: Span, linesA: string[], filePath }); } -export const withBannerArray = (title: string, description: string[] | readonly string[], date: Date, content: string[]) => { +const withBannerArray = (title: string, description: string[] | readonly string[], date: Date, content: string[]) => { return [ '#########################################', `# ${title}`, @@ -154,15 +154,32 @@ const MARK = 'this_ruleset_is_made_by_sukkaw.ruleset.skk.moe'; export const createRuleset = ( parentSpan: Span, title: string, description: string[] | readonly string[], date: Date, content: string[], - type: ('ruleset' | 'domainset' | string & {}), + type: 'ruleset' | 'domainset' | 'ipcidr' | 'ipcidr6', surgePath: string, clashPath: string, singBoxPath: string, _clashMrsPath?: string ) => parentSpan.traceChild(`create ruleset: ${path.basename(surgePath, path.extname(surgePath))}`).traceAsyncFn(async (childSpan) => { - const surgeContent = withBannerArray( - title, description, date, - type === 'domainset' - ? [MARK, ...content] - : sortRuleSet([`DOMAIN,${MARK}`, ...content]) - ); + content = sortRuleSet(content); + const surgeContent = childSpan.traceChildSync('process surge ruleset', () => { + let _surgeContent; + switch (type) { + case 'domainset': + _surgeContent = [MARK, ...content]; + break; + case 'ruleset': + _surgeContent = [`DOMAIN,${MARK}`, ...content]; + break; + case 'ipcidr': + _surgeContent = [`DOMAIN,${MARK}`, ...content.map(i => `IP-CIDR,${i}`)]; + break; + case 'ipcidr6': + _surgeContent = [`DOMAIN,${MARK}`, ...content.map(i => `IP-CIDR6,${i}`)]; + break; + default: + throw new TypeError(`Unknown type: ${type}`); + } + + return withBannerArray(title, description, date, _surgeContent); + }); + const clashContent = childSpan.traceChildSync('convert incoming ruleset to clash', () => { let _clashContent; switch (type) { @@ -172,6 +189,10 @@ export const createRuleset = ( case 'ruleset': _clashContent = [`DOMAIN,${MARK}`, ...surgeRulesetToClashClassicalTextRuleset(content)]; break; + case 'ipcidr': + case 'ipcidr6': + _clashContent = content; + break; default: throw new TypeError(`Unknown type: ${type}`); } @@ -186,6 +207,10 @@ export const createRuleset = ( case 'ruleset': _singBoxContent = surgeRulesetToSingbox([`DOMAIN,${MARK}`, ...content]); break; + case 'ipcidr': + case 'ipcidr6': + _singBoxContent = ipCidrListToSingbox(content); + break; default: throw new TypeError(`Unknown type: ${type}`); } diff --git a/Build/lib/misc.ts b/Build/lib/misc.ts index 0c2ee940..08736f82 100644 --- a/Build/lib/misc.ts +++ b/Build/lib/misc.ts @@ -1,4 +1,4 @@ -import { dirname } from 'path'; +import path, { dirname } from 'path'; import fs from 'fs'; import fsp from 'fs/promises'; import { makeRe } from 'picomatch'; @@ -33,3 +33,15 @@ export const writeFile: Write = async (destination: string, input, dir = dirname export const domainWildCardToRegex = (domain: string) => { return makeRe(domain, { contains: false, strictSlashes: true }).source; }; + +const OUTPUT_SURGE_DIR = path.resolve(__dirname, '../../List'); +const OUTPUT_CLASH_DIR = path.resolve(__dirname, '../../Clash'); +const OUTPUT_SINGBOX_DIR = path.resolve(__dirname, '../../sing-box'); + +export const output = (id: string, type: 'non_ip' | 'ip' | 'domainset') => { + return [ + path.join(OUTPUT_SURGE_DIR, type, id + '.conf'), + path.join(OUTPUT_CLASH_DIR, type, id + '.txt'), + path.join(OUTPUT_SINGBOX_DIR, type, id + '.json') + ] as const; +}; diff --git a/Build/lib/singbox.ts b/Build/lib/singbox.ts index 52551421..f199dc75 100644 --- a/Build/lib/singbox.ts +++ b/Build/lib/singbox.ts @@ -110,3 +110,12 @@ export const surgeDomainsetToSingbox = (domainset: string[]) => { rules: [rule] }; }; + +export const ipCidrListToSingbox = (ipCidrList: string[]): SingboxSourceFormat => { + return { + version: 2, + rules: [{ + ip_cidr: ipCidrList + }] + }; +}; diff --git a/Build/lib/text-decoder-stream.ts b/Build/lib/text-decoder-stream.ts deleted file mode 100644 index c06f589c..00000000 --- a/Build/lib/text-decoder-stream.ts +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2016 Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Polyfill for TextEncoderStream and TextDecoderStream -// Modified by Sukka (https://skk.moe) to increase compatibility and performance with Bun. - -export class PolyfillTextDecoderStream extends TransformStream { - readonly fatal: boolean; - readonly ignoreBOM: boolean; - - constructor( - public readonly encoding: BufferEncoding = 'utf-8', - { - fatal = false, - ignoreBOM = false - }: ConstructorParameters[1] = {} - ) { - const decoder = new TextDecoder(encoding, { fatal, ignoreBOM }); - - const nonLastChunkDecoderOpt = { stream: true }; - - super({ - transform(chunk: Uint8Array, controller: TransformStreamDefaultController) { - const decoded = decoder.decode(chunk, nonLastChunkDecoderOpt); - controller.enqueue(decoded); - }, - flush(controller: TransformStreamDefaultController) { - // If {fatal: false} is in options (the default), then the final call to - // decode() can produce extra output (usually the unicode replacement - // character 0xFFFD). When fatal is true, this call is just used for its - // side-effect of throwing a TypeError exception if the input is - // incomplete. - const output = decoder.decode(); - if (output.length > 0) { - controller.enqueue(output); - } - } - }); - - this.fatal = fatal; - this.ignoreBOM = ignoreBOM; - } -}