mirror of
https://github.com/SukkaW/Surge.git
synced 2026-01-29 01:51:52 +08:00
Refactor: new output
This commit is contained in:
@@ -1,11 +1,9 @@
|
|||||||
// @ts-check
|
|
||||||
import { createRuleset } from './lib/create-file';
|
|
||||||
import { parseFelixDnsmasq } from './lib/parse-dnsmasq';
|
import { parseFelixDnsmasq } from './lib/parse-dnsmasq';
|
||||||
import { task } from './trace';
|
import { task } from './trace';
|
||||||
import { SHARED_DESCRIPTION } from './lib/constants';
|
import { SHARED_DESCRIPTION } from './lib/constants';
|
||||||
import { createMemoizedPromise } from './lib/memo-promise';
|
import { createMemoizedPromise } from './lib/memo-promise';
|
||||||
import { TTL, deserializeArray, fsFetchCache, serializeArray, createCacheKey } from './lib/cache-filesystem';
|
import { TTL, deserializeArray, fsFetchCache, serializeArray, createCacheKey } from './lib/cache-filesystem';
|
||||||
import { output } from './lib/misc';
|
import { DomainsetOutput } from './lib/create-file-new';
|
||||||
|
|
||||||
const cacheKey = createCacheKey(__filename);
|
const cacheKey = createCacheKey(__filename);
|
||||||
|
|
||||||
@@ -23,24 +21,16 @@ export const getAppleCdnDomainsPromise = createMemoizedPromise(() => fsFetchCach
|
|||||||
export const buildAppleCdn = task(require.main === module, __filename)(async (span) => {
|
export const buildAppleCdn = task(require.main === module, __filename)(async (span) => {
|
||||||
const res: string[] = await span.traceChildPromise('get apple cdn domains', getAppleCdnDomainsPromise());
|
const res: string[] = await span.traceChildPromise('get apple cdn domains', getAppleCdnDomainsPromise());
|
||||||
|
|
||||||
const description = [
|
return new DomainsetOutput(span, 'apple_cdn')
|
||||||
|
.withTitle('Sukka\'s Ruleset - Apple CDN')
|
||||||
|
.withDescription([
|
||||||
...SHARED_DESCRIPTION,
|
...SHARED_DESCRIPTION,
|
||||||
'',
|
'',
|
||||||
'This file contains Apple\'s domains using their China mainland CDN servers.',
|
'This file contains Apple\'s domains using their China mainland CDN servers.',
|
||||||
'',
|
'',
|
||||||
'Data from:',
|
'Data from:',
|
||||||
' - https://github.com/felixonmars/dnsmasq-china-list'
|
' - https://github.com/felixonmars/dnsmasq-china-list'
|
||||||
];
|
])
|
||||||
|
.bulkAddDomainSuffix(res)
|
||||||
const domainset = res.map(i => `.${i}`);
|
.write();
|
||||||
|
|
||||||
return createRuleset(
|
|
||||||
span,
|
|
||||||
'Sukka\'s Ruleset - Apple CDN',
|
|
||||||
description,
|
|
||||||
new Date(),
|
|
||||||
domainset,
|
|
||||||
'domainset',
|
|
||||||
output('apple_cdn', 'domainset')
|
|
||||||
);
|
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,16 +1,13 @@
|
|||||||
import path from 'node:path';
|
import path from 'node:path';
|
||||||
import { createRuleset } from './lib/create-file';
|
|
||||||
import { readFileIntoProcessedArray } from './lib/fetch-text-by-line';
|
import { readFileIntoProcessedArray } from './lib/fetch-text-by-line';
|
||||||
import { createTrie } from './lib/trie';
|
import { createTrie } from './lib/trie';
|
||||||
import { task } from './trace';
|
import { task } from './trace';
|
||||||
import { SHARED_DESCRIPTION } from './lib/constants';
|
import { SHARED_DESCRIPTION } from './lib/constants';
|
||||||
import { getPublicSuffixListTextPromise } from './lib/download-publicsuffixlist';
|
import { getPublicSuffixListTextPromise } from './lib/download-publicsuffixlist';
|
||||||
import { domainsetDeduper } from './lib/domain-deduper';
|
|
||||||
import { appendArrayInPlace } from './lib/append-array-in-place';
|
import { appendArrayInPlace } from './lib/append-array-in-place';
|
||||||
import { sortDomains } from './lib/stable-sort-domain';
|
|
||||||
import { output } from './lib/misc';
|
|
||||||
import { SOURCE_DIR } from './constants/dir';
|
import { SOURCE_DIR } from './constants/dir';
|
||||||
import { processLine } from './lib/process-line';
|
import { processLine } from './lib/process-line';
|
||||||
|
import { DomainsetOutput } from './lib/create-file-new';
|
||||||
|
|
||||||
const getS3OSSDomainsPromise = (async (): Promise<string[]> => {
|
const getS3OSSDomainsPromise = (async (): Promise<string[]> => {
|
||||||
const trie = createTrie(
|
const trie = createTrie(
|
||||||
@@ -77,31 +74,24 @@ export const buildCdnDownloadConf = task(require.main === module, __filename)(as
|
|||||||
appendArrayInPlace(downloadDomainSet, steamDomainSet);
|
appendArrayInPlace(downloadDomainSet, steamDomainSet);
|
||||||
|
|
||||||
return Promise.all([
|
return Promise.all([
|
||||||
createRuleset(
|
new DomainsetOutput(span, 'cdn')
|
||||||
span,
|
.withTitle('Sukka\'s Ruleset - CDN Domains')
|
||||||
'Sukka\'s Ruleset - CDN Domains',
|
.withDescription([
|
||||||
[
|
|
||||||
...SHARED_DESCRIPTION,
|
...SHARED_DESCRIPTION,
|
||||||
'',
|
'',
|
||||||
'This file contains object storage and static assets CDN domains.'
|
'This file contains object storage and static assets CDN domains.'
|
||||||
],
|
])
|
||||||
new Date(),
|
.addFromDomainset(cdnDomainsList)
|
||||||
sortDomains(domainsetDeduper(cdnDomainsList)),
|
.write(),
|
||||||
'domainset',
|
|
||||||
output('cdn', 'domainset')
|
new DomainsetOutput(span, 'download')
|
||||||
),
|
.withTitle('Sukka\'s Ruleset - Large Files Hosting Domains')
|
||||||
createRuleset(
|
.withDescription([
|
||||||
span,
|
|
||||||
'Sukka\'s Ruleset - Large Files Hosting Domains',
|
|
||||||
[
|
|
||||||
...SHARED_DESCRIPTION,
|
...SHARED_DESCRIPTION,
|
||||||
'',
|
'',
|
||||||
'This file contains domains for software updating & large file hosting.'
|
'This file contains domains for software updating & large file hosting.'
|
||||||
],
|
])
|
||||||
new Date(),
|
.addFromDomainset(downloadDomainSet)
|
||||||
sortDomains(domainsetDeduper(downloadDomainSet)),
|
.write()
|
||||||
'domainset',
|
|
||||||
output('download', 'domainset')
|
|
||||||
)
|
|
||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,5 +1,4 @@
|
|||||||
import { fetchRemoteTextByLine } from './lib/fetch-text-by-line';
|
import { fetchRemoteTextByLine } from './lib/fetch-text-by-line';
|
||||||
import { createRuleset } from './lib/create-file';
|
|
||||||
import { processLineFromReadline } from './lib/process-line';
|
import { processLineFromReadline } from './lib/process-line';
|
||||||
import { task } from './trace';
|
import { task } from './trace';
|
||||||
|
|
||||||
@@ -7,7 +6,7 @@ import { exclude } from 'fast-cidr-tools';
|
|||||||
import { createMemoizedPromise } from './lib/memo-promise';
|
import { createMemoizedPromise } from './lib/memo-promise';
|
||||||
import { CN_CIDR_NOT_INCLUDED_IN_CHNROUTE, NON_CN_CIDR_INCLUDED_IN_CHNROUTE } from './constants/cidr';
|
import { CN_CIDR_NOT_INCLUDED_IN_CHNROUTE, NON_CN_CIDR_INCLUDED_IN_CHNROUTE } from './constants/cidr';
|
||||||
import { appendArrayInPlace } from './lib/append-array-in-place';
|
import { appendArrayInPlace } from './lib/append-array-in-place';
|
||||||
import { output } from './lib/misc';
|
import { IPListOutput } from './lib/create-file-new';
|
||||||
|
|
||||||
export const getChnCidrPromise = createMemoizedPromise(async () => {
|
export const getChnCidrPromise = createMemoizedPromise(async () => {
|
||||||
const cidr4 = await processLineFromReadline(await fetchRemoteTextByLine('https://raw.githubusercontent.com/misakaio/chnroutes2/master/chnroutes.txt'));
|
const cidr4 = await processLineFromReadline(await fetchRemoteTextByLine('https://raw.githubusercontent.com/misakaio/chnroutes2/master/chnroutes.txt'));
|
||||||
@@ -28,31 +27,22 @@ export const buildChnCidr = task(require.main === module, __filename)(async (spa
|
|||||||
''
|
''
|
||||||
];
|
];
|
||||||
|
|
||||||
// Can not use createRuleset here, as Clash support advanced ipset syntax
|
|
||||||
return Promise.all([
|
return Promise.all([
|
||||||
createRuleset(
|
new IPListOutput(span, 'china_ip', false)
|
||||||
span,
|
.withTitle('Sukka\'s Ruleset - Mainland China IPv4 CIDR')
|
||||||
'Sukka\'s Ruleset - Mainland China IPv4 CIDR',
|
.withDescription([
|
||||||
[
|
|
||||||
...description,
|
...description,
|
||||||
'Data from https://misaka.io (misakaio @ GitHub)'
|
'Data from https://misaka.io (misakaio @ GitHub)'
|
||||||
],
|
])
|
||||||
new Date(),
|
.bulkAddCIDR4(filteredCidr4)
|
||||||
filteredCidr4,
|
.write(),
|
||||||
'ipcidr',
|
new IPListOutput(span, 'china_ip_ipv6', false)
|
||||||
output('china_ip', 'ip')
|
.withTitle('Sukka\'s Ruleset - Mainland China IPv6 CIDR')
|
||||||
),
|
.withDescription([
|
||||||
createRuleset(
|
|
||||||
span,
|
|
||||||
'Sukka\'s Ruleset - Mainland China IPv6 CIDR',
|
|
||||||
[
|
|
||||||
...description,
|
...description,
|
||||||
'Data from https://github.com/gaoyifan/china-operator-ip'
|
'Data from https://github.com/gaoyifan/china-operator-ip'
|
||||||
],
|
])
|
||||||
new Date(),
|
.bulkAddCIDR6(cidr6)
|
||||||
cidr6,
|
.write()
|
||||||
'ipcidr6',
|
|
||||||
output('china_ip_ipv6', 'ip')
|
|
||||||
)
|
|
||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -4,13 +4,13 @@ import * as path from 'node:path';
|
|||||||
import { readFileByLine } from './lib/fetch-text-by-line';
|
import { readFileByLine } from './lib/fetch-text-by-line';
|
||||||
import { processLine } from './lib/process-line';
|
import { processLine } from './lib/process-line';
|
||||||
import { createRuleset } from './lib/create-file';
|
import { createRuleset } from './lib/create-file';
|
||||||
import { domainsetDeduper } from './lib/domain-deduper';
|
|
||||||
import type { Span } from './trace';
|
import type { Span } from './trace';
|
||||||
import { task } from './trace';
|
import { task } from './trace';
|
||||||
import { SHARED_DESCRIPTION } from './lib/constants';
|
import { SHARED_DESCRIPTION } from './lib/constants';
|
||||||
import { fdir as Fdir } from 'fdir';
|
import { fdir as Fdir } from 'fdir';
|
||||||
import { appendArrayInPlace } from './lib/append-array-in-place';
|
import { appendArrayInPlace } from './lib/append-array-in-place';
|
||||||
import { OUTPUT_CLASH_DIR, OUTPUT_SINGBOX_DIR, OUTPUT_SURGE_DIR, SOURCE_DIR } from './constants/dir';
|
import { OUTPUT_CLASH_DIR, OUTPUT_SINGBOX_DIR, OUTPUT_SURGE_DIR, SOURCE_DIR } from './constants/dir';
|
||||||
|
import { DomainsetOutput } from './lib/create-file-new';
|
||||||
|
|
||||||
const MAGIC_COMMAND_SKIP = '# $ custom_build_script';
|
const MAGIC_COMMAND_SKIP = '# $ custom_build_script';
|
||||||
const MAGIC_COMMAND_TITLE = '# $ meta_title ';
|
const MAGIC_COMMAND_TITLE = '# $ meta_title ';
|
||||||
@@ -113,10 +113,8 @@ function transformDomainset(parentSpan: Span, sourcePath: string, relativePath:
|
|||||||
const res = await processFile(span, sourcePath);
|
const res = await processFile(span, sourcePath);
|
||||||
if (res === $skip) return;
|
if (res === $skip) return;
|
||||||
|
|
||||||
const clashFileBasename = relativePath.slice(0, -path.extname(relativePath).length);
|
const id = path.basename(relativePath).slice(0, -path.extname(relativePath).length);
|
||||||
|
|
||||||
const [title, descriptions, lines] = res;
|
const [title, descriptions, lines] = res;
|
||||||
const deduped = domainsetDeduper(lines);
|
|
||||||
|
|
||||||
let description: string[];
|
let description: string[];
|
||||||
if (descriptions.length) {
|
if (descriptions.length) {
|
||||||
@@ -127,19 +125,11 @@ function transformDomainset(parentSpan: Span, sourcePath: string, relativePath:
|
|||||||
description = SHARED_DESCRIPTION;
|
description = SHARED_DESCRIPTION;
|
||||||
}
|
}
|
||||||
|
|
||||||
return createRuleset(
|
return new DomainsetOutput(span, id)
|
||||||
span,
|
.withTitle(title)
|
||||||
title,
|
.withDescription(description)
|
||||||
description,
|
.addFromDomainset(lines)
|
||||||
new Date(),
|
.write();
|
||||||
deduped,
|
|
||||||
'domainset',
|
|
||||||
[
|
|
||||||
path.resolve(OUTPUT_SURGE_DIR, relativePath),
|
|
||||||
path.resolve(OUTPUT_CLASH_DIR, `${clashFileBasename}.txt`),
|
|
||||||
path.resolve(OUTPUT_SINGBOX_DIR, `${clashFileBasename}.json`)
|
|
||||||
]
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,4 @@
|
|||||||
import path from 'node:path';
|
import path from 'node:path';
|
||||||
import { createRuleset } from './lib/create-file';
|
|
||||||
import { sortDomains } from './lib/stable-sort-domain';
|
|
||||||
|
|
||||||
import { Sema } from 'async-sema';
|
import { Sema } from 'async-sema';
|
||||||
import { getHostname } from 'tldts';
|
import { getHostname } from 'tldts';
|
||||||
@@ -10,83 +8,42 @@ import { SHARED_DESCRIPTION } from './lib/constants';
|
|||||||
import { readFileIntoProcessedArray } from './lib/fetch-text-by-line';
|
import { readFileIntoProcessedArray } from './lib/fetch-text-by-line';
|
||||||
import { TTL, deserializeArray, fsFetchCache, serializeArray, createCacheKey } from './lib/cache-filesystem';
|
import { TTL, deserializeArray, fsFetchCache, serializeArray, createCacheKey } from './lib/cache-filesystem';
|
||||||
|
|
||||||
import { createTrie } from './lib/trie';
|
import { DomainsetOutput } from './lib/create-file-new';
|
||||||
import { output } from './lib/misc';
|
import { OUTPUT_SURGE_DIR } from './constants/dir';
|
||||||
|
|
||||||
const s = new Sema(2);
|
const KEYWORDS = [
|
||||||
const cacheKey = createCacheKey(__filename);
|
'Hong Kong',
|
||||||
|
'Taiwan',
|
||||||
|
'China Telecom',
|
||||||
|
'China Mobile',
|
||||||
|
'China Unicom',
|
||||||
|
'Japan',
|
||||||
|
'Tokyo',
|
||||||
|
'Singapore',
|
||||||
|
'Korea',
|
||||||
|
'Seoul',
|
||||||
|
'Canada',
|
||||||
|
'Toronto',
|
||||||
|
'Montreal',
|
||||||
|
'Los Ang',
|
||||||
|
'San Jos',
|
||||||
|
'Seattle',
|
||||||
|
'New York',
|
||||||
|
'Dallas',
|
||||||
|
'Miami',
|
||||||
|
'Berlin',
|
||||||
|
'Frankfurt',
|
||||||
|
'London',
|
||||||
|
'Paris',
|
||||||
|
'Amsterdam',
|
||||||
|
'Moscow',
|
||||||
|
'Australia',
|
||||||
|
'Sydney',
|
||||||
|
'Brazil',
|
||||||
|
'Turkey'
|
||||||
|
];
|
||||||
|
|
||||||
const latestTopUserAgentsPromise = fsFetchCache.apply(
|
const PREDEFINE_DOMAINS = [
|
||||||
cacheKey('https://cdn.jsdelivr.net/npm/top-user-agents@latest/src/desktop.json'),
|
|
||||||
() => fetchWithRetry(
|
|
||||||
'https://cdn.jsdelivr.net/npm/top-user-agents@latest/src/desktop.json',
|
|
||||||
{ signal: AbortSignal.timeout(1000 * 60) }
|
|
||||||
)
|
|
||||||
.then(res => res.json() as Promise<string[]>)
|
|
||||||
.then((userAgents) => userAgents.filter(ua => ua.startsWith('Mozilla/5.0 '))),
|
|
||||||
{
|
|
||||||
serializer: serializeArray,
|
|
||||||
deserializer: deserializeArray,
|
|
||||||
ttl: TTL.THREE_DAYS()
|
|
||||||
}
|
|
||||||
);
|
|
||||||
|
|
||||||
const querySpeedtestApi = async (keyword: string): Promise<Array<string | null>> => {
|
|
||||||
const topUserAgents = await latestTopUserAgentsPromise;
|
|
||||||
|
|
||||||
const url = `https://www.speedtest.net/api/js/servers?engine=js&search=${keyword}&limit=100`;
|
|
||||||
|
|
||||||
try {
|
|
||||||
const randomUserAgent = topUserAgents[Math.floor(Math.random() * topUserAgents.length)];
|
|
||||||
|
|
||||||
return await fsFetchCache.apply(
|
|
||||||
cacheKey(url),
|
|
||||||
() => s.acquire().then(() => fetchWithRetry(url, {
|
|
||||||
headers: {
|
|
||||||
dnt: '1',
|
|
||||||
Referer: 'https://www.speedtest.net/',
|
|
||||||
accept: 'application/json, text/plain, */*',
|
|
||||||
'User-Agent': randomUserAgent,
|
|
||||||
'Accept-Language': 'en-US,en;q=0.9',
|
|
||||||
...(randomUserAgent.includes('Chrome')
|
|
||||||
? {
|
|
||||||
'Sec-Ch-Ua-Mobile': '?0',
|
|
||||||
'Sec-Fetch-Dest': 'empty',
|
|
||||||
'Sec-Fetch-Mode': 'cors',
|
|
||||||
'Sec-Fetch-Site': 'same-origin',
|
|
||||||
'Sec-Gpc': '1'
|
|
||||||
}
|
|
||||||
: {})
|
|
||||||
},
|
|
||||||
signal: AbortSignal.timeout(1000 * 60),
|
|
||||||
retry: {
|
|
||||||
retries: 2
|
|
||||||
}
|
|
||||||
})).then(r => r.json() as any).then((data: Array<{ url: string, host: string }>) => data.reduce<string[]>(
|
|
||||||
(prev, cur) => {
|
|
||||||
const line = cur.host || cur.url;
|
|
||||||
const hn = getHostname(line, { detectIp: false, validateHostname: true });
|
|
||||||
if (hn) {
|
|
||||||
prev.push(hn);
|
|
||||||
}
|
|
||||||
return prev;
|
|
||||||
}, []
|
|
||||||
)).finally(() => s.release()),
|
|
||||||
{
|
|
||||||
ttl: TTL.ONE_WEEK(),
|
|
||||||
serializer: serializeArray,
|
|
||||||
deserializer: deserializeArray
|
|
||||||
}
|
|
||||||
);
|
|
||||||
} catch (e) {
|
|
||||||
console.error(e);
|
|
||||||
return [];
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
export const buildSpeedtestDomainSet = task(require.main === module, __filename)(async (span) => {
|
|
||||||
const domainTrie = createTrie(
|
|
||||||
[
|
|
||||||
// speedtest.net
|
// speedtest.net
|
||||||
'.speedtest.net',
|
'.speedtest.net',
|
||||||
'.speedtestcustom.com',
|
'.speedtestcustom.com',
|
||||||
@@ -175,80 +132,99 @@ export const buildSpeedtestDomainSet = task(require.main === module, __filename)
|
|||||||
'mensura.cdn-apple.com', // From netQuality command
|
'mensura.cdn-apple.com', // From netQuality command
|
||||||
// OpenSpeedtest
|
// OpenSpeedtest
|
||||||
'open.cachefly.net' // This is also used for openspeedtest server download
|
'open.cachefly.net' // This is also used for openspeedtest server download
|
||||||
],
|
|
||||||
true
|
|
||||||
);
|
|
||||||
|
|
||||||
await span.traceChildAsync(
|
];
|
||||||
'fetch previous speedtest domainset',
|
|
||||||
async () => {
|
const s = new Sema(2);
|
||||||
|
const cacheKey = createCacheKey(__filename);
|
||||||
|
|
||||||
|
const latestTopUserAgentsPromise = fsFetchCache.apply(
|
||||||
|
cacheKey('https://cdn.jsdelivr.net/npm/top-user-agents@latest/src/desktop.json'),
|
||||||
|
() => fetchWithRetry(
|
||||||
|
'https://cdn.jsdelivr.net/npm/top-user-agents@latest/src/desktop.json',
|
||||||
|
{ signal: AbortSignal.timeout(1000 * 60) }
|
||||||
|
)
|
||||||
|
.then(res => res.json() as Promise<string[]>)
|
||||||
|
.then((userAgents) => userAgents.filter(ua => ua.startsWith('Mozilla/5.0 '))),
|
||||||
|
{
|
||||||
|
serializer: serializeArray,
|
||||||
|
deserializer: deserializeArray,
|
||||||
|
ttl: TTL.THREE_DAYS()
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
const querySpeedtestApi = async (keyword: string): Promise<Array<string | null>> => {
|
||||||
|
const topUserAgents = await latestTopUserAgentsPromise;
|
||||||
|
|
||||||
|
const url = `https://www.speedtest.net/api/js/servers?engine=js&search=${keyword}&limit=100`;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
(
|
const randomUserAgent = topUserAgents[Math.floor(Math.random() * topUserAgents.length)];
|
||||||
await readFileIntoProcessedArray(path.resolve(__dirname, '../List/domainset/speedtest.conf'))
|
|
||||||
) .forEach(line => {
|
return await fsFetchCache.apply(
|
||||||
|
cacheKey(url),
|
||||||
|
() => s.acquire().then(() => fetchWithRetry(url, {
|
||||||
|
headers: {
|
||||||
|
dnt: '1',
|
||||||
|
Referer: 'https://www.speedtest.net/',
|
||||||
|
accept: 'application/json, text/plain, */*',
|
||||||
|
'User-Agent': randomUserAgent,
|
||||||
|
'Accept-Language': 'en-US,en;q=0.9',
|
||||||
|
...(randomUserAgent.includes('Chrome')
|
||||||
|
? {
|
||||||
|
'Sec-Ch-Ua-Mobile': '?0',
|
||||||
|
'Sec-Fetch-Dest': 'empty',
|
||||||
|
'Sec-Fetch-Mode': 'cors',
|
||||||
|
'Sec-Fetch-Site': 'same-origin',
|
||||||
|
'Sec-Gpc': '1'
|
||||||
|
}
|
||||||
|
: {})
|
||||||
|
},
|
||||||
|
signal: AbortSignal.timeout(1000 * 60),
|
||||||
|
retry: {
|
||||||
|
retries: 2
|
||||||
|
}
|
||||||
|
})).then(r => r.json() as any).then((data: Array<{ url: string, host: string }>) => data.reduce<string[]>(
|
||||||
|
(prev, cur) => {
|
||||||
|
const line = cur.host || cur.url;
|
||||||
const hn = getHostname(line, { detectIp: false, validateHostname: true });
|
const hn = getHostname(line, { detectIp: false, validateHostname: true });
|
||||||
if (hn) {
|
if (hn) {
|
||||||
domainTrie.add(hn);
|
prev.push(hn);
|
||||||
}
|
}
|
||||||
});
|
return prev;
|
||||||
} catch { }
|
}, []
|
||||||
|
)).finally(() => s.release()),
|
||||||
|
{
|
||||||
|
ttl: TTL.ONE_WEEK(),
|
||||||
|
serializer: serializeArray,
|
||||||
|
deserializer: deserializeArray
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
} catch (e) {
|
||||||
|
console.error(e);
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
await Promise.all([
|
export const buildSpeedtestDomainSet = task(require.main === module, __filename)(async (span) => {
|
||||||
'Hong Kong',
|
const output = new DomainsetOutput(span, 'speedtest')
|
||||||
'Taiwan',
|
.withTitle('Sukka\'s Ruleset - Speedtest Domains')
|
||||||
'China Telecom',
|
.withDescription([
|
||||||
'China Mobile',
|
...SHARED_DESCRIPTION,
|
||||||
'China Unicom',
|
'',
|
||||||
'Japan',
|
'This file contains common speedtest endpoints.'
|
||||||
'Tokyo',
|
])
|
||||||
'Singapore',
|
.addFromDomainset(PREDEFINE_DOMAINS)
|
||||||
'Korea',
|
.addFromDomainset(await readFileIntoProcessedArray(path.resolve(OUTPUT_SURGE_DIR, 'domainset/speedtest.conf')));
|
||||||
'Seoul',
|
|
||||||
'Canada',
|
await Promise.all(KEYWORDS.map((keyword) => span.traceChildAsync(
|
||||||
'Toronto',
|
|
||||||
'Montreal',
|
|
||||||
'Los Ang',
|
|
||||||
'San Jos',
|
|
||||||
'Seattle',
|
|
||||||
'New York',
|
|
||||||
'Dallas',
|
|
||||||
'Miami',
|
|
||||||
'Berlin',
|
|
||||||
'Frankfurt',
|
|
||||||
'London',
|
|
||||||
'Paris',
|
|
||||||
'Amsterdam',
|
|
||||||
'Moscow',
|
|
||||||
'Australia',
|
|
||||||
'Sydney',
|
|
||||||
'Brazil',
|
|
||||||
'Turkey'
|
|
||||||
].map((keyword) => span.traceChildAsync(
|
|
||||||
`fetch speedtest endpoints: ${keyword}`,
|
`fetch speedtest endpoints: ${keyword}`,
|
||||||
() => querySpeedtestApi(keyword)
|
() => querySpeedtestApi(keyword)
|
||||||
).then(hostnameGroup => hostnameGroup.forEach(hostname => {
|
).then(hostnameGroup => hostnameGroup.forEach(hostname => {
|
||||||
if (hostname) {
|
if (hostname) {
|
||||||
domainTrie.add(hostname);
|
output.addDomain(hostname);
|
||||||
}
|
}
|
||||||
}))));
|
}))));
|
||||||
|
|
||||||
const deduped = span.traceChildSync('sort result', () => sortDomains(domainTrie.dump()));
|
return output.write();
|
||||||
|
|
||||||
const description = [
|
|
||||||
...SHARED_DESCRIPTION,
|
|
||||||
'',
|
|
||||||
'This file contains common speedtest endpoints.'
|
|
||||||
];
|
|
||||||
|
|
||||||
return createRuleset(
|
|
||||||
span,
|
|
||||||
'Sukka\'s Ruleset - Speedtest Domains',
|
|
||||||
description,
|
|
||||||
new Date(),
|
|
||||||
deduped,
|
|
||||||
'domainset',
|
|
||||||
output('speedtest', 'domainset')
|
|
||||||
);
|
|
||||||
});
|
});
|
||||||
|
|||||||
249
Build/lib/create-file-new.ts
Normal file
249
Build/lib/create-file-new.ts
Normal file
@@ -0,0 +1,249 @@
|
|||||||
|
import path from 'node:path';
|
||||||
|
|
||||||
|
import type { Span } from '../trace';
|
||||||
|
import { surgeDomainsetToClashDomainset } from './clash';
|
||||||
|
import { compareAndWriteFile, withBannerArray } from './create-file';
|
||||||
|
import { ipCidrListToSingbox, surgeDomainsetToSingbox } from './singbox';
|
||||||
|
import { sortDomains } from './stable-sort-domain';
|
||||||
|
import { createTrie } from './trie';
|
||||||
|
import { invariant } from 'foxact/invariant';
|
||||||
|
import { OUTPUT_CLASH_DIR, OUTPUT_SINGBOX_DIR, OUTPUT_SURGE_DIR } from '../constants/dir';
|
||||||
|
import stringify from 'json-stringify-pretty-compact';
|
||||||
|
import { appendArrayInPlace } from './append-array-in-place';
|
||||||
|
|
||||||
|
abstract class RuleOutput {
|
||||||
|
protected domainTrie = createTrie<unknown>(null, true);
|
||||||
|
protected domainKeywords = new Set<string>();
|
||||||
|
protected domainWildcard = new Set<string>();
|
||||||
|
protected ipcidr = new Set<string>();
|
||||||
|
protected ipcidrNoResolve = new Set<string>();
|
||||||
|
protected ipcidr6 = new Set<string>();
|
||||||
|
protected ipcidr6NoResolve = new Set<string>();
|
||||||
|
protected otherRules = new Set<string>();
|
||||||
|
protected abstract type: 'domainset' | 'non_ip' | 'ip';
|
||||||
|
|
||||||
|
protected pendingPromise = Promise.resolve();
|
||||||
|
|
||||||
|
static jsonToLines(this: void, json: unknown): string[] {
|
||||||
|
return stringify(json).split('\n');
|
||||||
|
}
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
protected readonly span: Span,
|
||||||
|
protected readonly id: string
|
||||||
|
) {}
|
||||||
|
|
||||||
|
protected title: string | null = null;
|
||||||
|
withTitle(title: string) {
|
||||||
|
this.title = title;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected description: string[] | readonly string[] | null = null;
|
||||||
|
withDescription(description: string[] | readonly string[]) {
|
||||||
|
this.description = description;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected date = new Date();
|
||||||
|
withDate(date: Date) {
|
||||||
|
this.date = date;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected apexDomainMap: Map<string, string> | null = null;
|
||||||
|
protected subDomainMap: Map<string, string> | null = null;
|
||||||
|
withDomainMap(apexDomainMap: Map<string, string>, subDomainMap: Map<string, string>) {
|
||||||
|
this.apexDomainMap = apexDomainMap;
|
||||||
|
this.subDomainMap = subDomainMap;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
addDomain(domain: string) {
|
||||||
|
this.domainTrie.add(domain);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
addDomainSuffix(domain: string) {
|
||||||
|
this.domainTrie.add(domain[0] === '.' ? domain : '.' + domain);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
bulkAddDomainSuffix(domains: string[]) {
|
||||||
|
for (let i = 0, len = domains.length; i < len; i++) {
|
||||||
|
this.addDomainSuffix(domains[i]);
|
||||||
|
}
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
addDomainKeyword(keyword: string) {
|
||||||
|
this.domainKeywords.add(keyword);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
addDomainWildcard(wildcard: string) {
|
||||||
|
this.domainWildcard.add(wildcard);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async addFromDomainsetPromise(source: AsyncIterable<string> | Iterable<string> | string[]) {
|
||||||
|
for await (const line of source) {
|
||||||
|
if (line[0] === '.') {
|
||||||
|
this.addDomainSuffix(line);
|
||||||
|
} else {
|
||||||
|
this.addDomain(line);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
addFromDomainset(source: AsyncIterable<string> | Iterable<string> | string[]) {
|
||||||
|
this.pendingPromise = this.pendingPromise.then(() => this.addFromDomainsetPromise(source));
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
async addFromRuleset(source: AsyncIterable<string> | Iterable<string>) {
|
||||||
|
for await (const line of source) {
|
||||||
|
const [type, value, arg] = line.split(',');
|
||||||
|
switch (type) {
|
||||||
|
case 'DOMAIN':
|
||||||
|
this.addDomain(value);
|
||||||
|
break;
|
||||||
|
case 'DOMAIN-SUFFIX':
|
||||||
|
this.addDomainSuffix(value);
|
||||||
|
break;
|
||||||
|
case 'DOMAIN-KEYWORD':
|
||||||
|
this.addDomainKeyword(value);
|
||||||
|
break;
|
||||||
|
case 'DOMAIN-WILDCARD':
|
||||||
|
this.addDomainWildcard(value);
|
||||||
|
break;
|
||||||
|
case 'IP-CIDR':
|
||||||
|
(arg === 'no-resolve' ? this.ipcidrNoResolve : this.ipcidr).add(value);
|
||||||
|
break;
|
||||||
|
case 'IP-CIDR6':
|
||||||
|
(arg === 'no-resolve' ? this.ipcidr6NoResolve : this.ipcidr6).add(value);
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
this.otherRules.add(line);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
bulkAddCIDR4(cidr: string[]) {
|
||||||
|
for (let i = 0, len = cidr.length; i < len; i++) {
|
||||||
|
this.ipcidr.add(cidr[i]);
|
||||||
|
}
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
bulkAddCIDR6(cidr: string[]) {
|
||||||
|
for (let i = 0, len = cidr.length; i < len; i++) {
|
||||||
|
this.ipcidr6.add(cidr[i]);
|
||||||
|
}
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
abstract write(): Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class DomainsetOutput extends RuleOutput {
|
||||||
|
protected type = 'domainset' as const;
|
||||||
|
|
||||||
|
async write() {
|
||||||
|
await this.pendingPromise;
|
||||||
|
|
||||||
|
invariant(this.title, 'Missing title');
|
||||||
|
invariant(this.description, 'Missing description');
|
||||||
|
|
||||||
|
const sorted = sortDomains(this.domainTrie.dump(), this.apexDomainMap, this.subDomainMap);
|
||||||
|
sorted.push('this_ruleset_is_made_by_sukkaw.ruleset.skk.moe');
|
||||||
|
|
||||||
|
const surge = sorted;
|
||||||
|
const clash = surgeDomainsetToClashDomainset(sorted);
|
||||||
|
const singbox = RuleOutput.jsonToLines(surgeDomainsetToSingbox(sorted));
|
||||||
|
|
||||||
|
await Promise.all([
|
||||||
|
compareAndWriteFile(
|
||||||
|
this.span,
|
||||||
|
withBannerArray(
|
||||||
|
this.title,
|
||||||
|
this.description,
|
||||||
|
this.date,
|
||||||
|
surge
|
||||||
|
),
|
||||||
|
path.join(OUTPUT_SURGE_DIR, this.type, this.id + '.conf')
|
||||||
|
),
|
||||||
|
compareAndWriteFile(
|
||||||
|
this.span,
|
||||||
|
withBannerArray(
|
||||||
|
this.title,
|
||||||
|
this.description,
|
||||||
|
this.date,
|
||||||
|
clash
|
||||||
|
),
|
||||||
|
path.join(OUTPUT_CLASH_DIR, this.type, this.id + '.txt')
|
||||||
|
),
|
||||||
|
compareAndWriteFile(
|
||||||
|
this.span,
|
||||||
|
singbox,
|
||||||
|
path.join(OUTPUT_SINGBOX_DIR, this.type, this.id + '.json')
|
||||||
|
)
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class IPListOutput extends RuleOutput {
|
||||||
|
protected type = 'ip' as const;
|
||||||
|
|
||||||
|
constructor(span: Span, id: string, private readonly clashUseRule = true) {
|
||||||
|
super(span, id);
|
||||||
|
}
|
||||||
|
|
||||||
|
async write() {
|
||||||
|
await this.pendingPromise;
|
||||||
|
|
||||||
|
invariant(this.title, 'Missing title');
|
||||||
|
invariant(this.description, 'Missing description');
|
||||||
|
|
||||||
|
const sorted4 = Array.from(this.ipcidr);
|
||||||
|
const sorted6 = Array.from(this.ipcidr6);
|
||||||
|
const merged = appendArrayInPlace(appendArrayInPlace([], sorted4), sorted6);
|
||||||
|
|
||||||
|
const surge = sorted4.map(i => `IP-CIDR,${i}`);
|
||||||
|
appendArrayInPlace(surge, sorted6.map(i => `IP-CIDR6,${i}`));
|
||||||
|
surge.push('DOMAIN,this_ruleset_is_made_by_sukkaw.ruleset.skk.moe');
|
||||||
|
|
||||||
|
const clash = this.clashUseRule ? surge : merged;
|
||||||
|
const singbox = RuleOutput.jsonToLines(ipCidrListToSingbox(merged));
|
||||||
|
|
||||||
|
await Promise.all([
|
||||||
|
compareAndWriteFile(
|
||||||
|
this.span,
|
||||||
|
withBannerArray(
|
||||||
|
this.title,
|
||||||
|
this.description,
|
||||||
|
this.date,
|
||||||
|
surge
|
||||||
|
),
|
||||||
|
path.join(OUTPUT_SURGE_DIR, this.type, this.id + '.conf')
|
||||||
|
),
|
||||||
|
compareAndWriteFile(
|
||||||
|
this.span,
|
||||||
|
withBannerArray(
|
||||||
|
this.title,
|
||||||
|
this.description,
|
||||||
|
this.date,
|
||||||
|
clash
|
||||||
|
),
|
||||||
|
path.join(OUTPUT_CLASH_DIR, this.type, this.id + '.txt')
|
||||||
|
),
|
||||||
|
compareAndWriteFile(
|
||||||
|
this.span,
|
||||||
|
singbox,
|
||||||
|
path.join(OUTPUT_SINGBOX_DIR, this.type, this.id + '.json')
|
||||||
|
)
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -7,7 +7,7 @@ import fs from 'node:fs';
|
|||||||
import { fastStringArrayJoin, writeFile } from './misc';
|
import { fastStringArrayJoin, writeFile } from './misc';
|
||||||
import { readFileByLine } from './fetch-text-by-line';
|
import { readFileByLine } from './fetch-text-by-line';
|
||||||
import stringify from 'json-stringify-pretty-compact';
|
import stringify from 'json-stringify-pretty-compact';
|
||||||
import { ipCidrListToSingbox, surgeDomainsetToSingbox, surgeRulesetToSingbox } from './singbox';
|
import { surgeDomainsetToSingbox, surgeRulesetToSingbox } from './singbox';
|
||||||
import { createTrie } from './trie';
|
import { createTrie } from './trie';
|
||||||
import { pack, unpackFirst, unpackSecond } from './bitwise';
|
import { pack, unpackFirst, unpackSecond } from './bitwise';
|
||||||
import { asyncWriteToStream } from './async-write-to-stream';
|
import { asyncWriteToStream } from './async-write-to-stream';
|
||||||
@@ -23,8 +23,7 @@ export const fileEqual = async (linesA: string[], source: AsyncIterable<string>)
|
|||||||
|
|
||||||
if (index > linesA.length - 1) {
|
if (index > linesA.length - 1) {
|
||||||
if (index === linesA.length && lineB === '') {
|
if (index === linesA.length && lineB === '') {
|
||||||
index--;
|
return true;
|
||||||
continue;
|
|
||||||
}
|
}
|
||||||
// The file becomes smaller
|
// The file becomes smaller
|
||||||
return false;
|
return false;
|
||||||
@@ -51,7 +50,7 @@ export const fileEqual = async (linesA: string[], source: AsyncIterable<string>)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (index !== linesA.length - 1) {
|
if (index < linesA.length - 1) {
|
||||||
// The file becomes larger
|
// The file becomes larger
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@@ -96,7 +95,7 @@ export async function compareAndWriteFile(span: Span, linesA: string[], filePath
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
const withBannerArray = (title: string, description: string[] | readonly string[], date: Date, content: string[]) => {
|
export const withBannerArray = (title: string, description: string[] | readonly string[], date: Date, content: string[]) => {
|
||||||
return [
|
return [
|
||||||
'#########################################',
|
'#########################################',
|
||||||
`# ${title}`,
|
`# ${title}`,
|
||||||
@@ -191,7 +190,7 @@ const MARK = 'this_ruleset_is_made_by_sukkaw.ruleset.skk.moe';
|
|||||||
export const createRuleset = (
|
export const createRuleset = (
|
||||||
parentSpan: Span,
|
parentSpan: Span,
|
||||||
title: string, description: string[] | readonly string[], date: Date, content: string[],
|
title: string, description: string[] | readonly string[], date: Date, content: string[],
|
||||||
type: 'ruleset' | 'domainset' | 'ipcidr' | 'ipcidr6',
|
type: 'ruleset' | 'domainset',
|
||||||
[surgePath, clashPath, singBoxPath, _clashMrsPath]: readonly [
|
[surgePath, clashPath, singBoxPath, _clashMrsPath]: readonly [
|
||||||
surgePath: string,
|
surgePath: string,
|
||||||
clashPath: string,
|
clashPath: string,
|
||||||
@@ -210,12 +209,6 @@ export const createRuleset = (
|
|||||||
case 'ruleset':
|
case 'ruleset':
|
||||||
_surgeContent = [`DOMAIN,${MARK}`, ...processRuleSet(content)];
|
_surgeContent = [`DOMAIN,${MARK}`, ...processRuleSet(content)];
|
||||||
break;
|
break;
|
||||||
case 'ipcidr':
|
|
||||||
_surgeContent = [`DOMAIN,${MARK}`, ...processRuleSet(content.map(i => `IP-CIDR,${i}`))];
|
|
||||||
break;
|
|
||||||
case 'ipcidr6':
|
|
||||||
_surgeContent = [`DOMAIN,${MARK}`, ...processRuleSet(content.map(i => `IP-CIDR6,${i}`))];
|
|
||||||
break;
|
|
||||||
default:
|
default:
|
||||||
throw new TypeError(`Unknown type: ${type}`);
|
throw new TypeError(`Unknown type: ${type}`);
|
||||||
}
|
}
|
||||||
@@ -232,10 +225,6 @@ export const createRuleset = (
|
|||||||
case 'ruleset':
|
case 'ruleset':
|
||||||
_clashContent = [`DOMAIN,${MARK}`, ...surgeRulesetToClashClassicalTextRuleset(processRuleSet(content))];
|
_clashContent = [`DOMAIN,${MARK}`, ...surgeRulesetToClashClassicalTextRuleset(processRuleSet(content))];
|
||||||
break;
|
break;
|
||||||
case 'ipcidr':
|
|
||||||
case 'ipcidr6':
|
|
||||||
_clashContent = content;
|
|
||||||
break;
|
|
||||||
default:
|
default:
|
||||||
throw new TypeError(`Unknown type: ${type}`);
|
throw new TypeError(`Unknown type: ${type}`);
|
||||||
}
|
}
|
||||||
@@ -250,10 +239,6 @@ export const createRuleset = (
|
|||||||
case 'ruleset':
|
case 'ruleset':
|
||||||
_singBoxContent = surgeRulesetToSingbox([`DOMAIN,${MARK}`, ...processRuleSet(content)]);
|
_singBoxContent = surgeRulesetToSingbox([`DOMAIN,${MARK}`, ...processRuleSet(content)]);
|
||||||
break;
|
break;
|
||||||
case 'ipcidr':
|
|
||||||
case 'ipcidr6':
|
|
||||||
_singBoxContent = ipCidrListToSingbox(content);
|
|
||||||
break;
|
|
||||||
default:
|
default:
|
||||||
throw new TypeError(`Unknown type: ${type}`);
|
throw new TypeError(`Unknown type: ${type}`);
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user