GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2025-12-12 09:10:35 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Refactor: full span tracer

Browse Source
This commit is contained in:
SukkaW 2024-01-14 22:10:16 +08:00
parent 0f257e992a
commit 9bb0c14d5f
23 changed files with 238 additions and 183 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
Build/build-anti-bogus-domain.ts
View File

@ -35,7 +35,7 @@ const getBogusNxDomainIPs = async () => {
); );
}; };
export const buildAntiBogusDomain = task(import.meta.path, async () => { export const buildAntiBogusDomain = task(import.meta.path, async (span) => {
const bogusIpPromise = getBogusNxDomainIPs(); const bogusIpPromise = getBogusNxDomainIPs();
const result: string[] = []; const result: string[] = [];
@ -57,7 +57,8 @@ export const buildAntiBogusDomain = task(import.meta.path, async () => {
' - https://github.com/felixonmars/dnsmasq-china-list' ' - https://github.com/felixonmars/dnsmasq-china-list'
]; ];
return Promise.all(createRuleset( return createRuleset(
span,
'Sukka\'s Ruleset - Anti Bogus Domain', 'Sukka\'s Ruleset - Anti Bogus Domain',
description, description,
new Date(), new Date(),
@ -65,7 +66,7 @@ export const buildAntiBogusDomain = task(import.meta.path, async () => {
'ruleset', 'ruleset',
path.resolve(import.meta.dir, '../List/ip/reject.conf'), path.resolve(import.meta.dir, '../List/ip/reject.conf'),
path.resolve(import.meta.dir, '../Clash/ip/reject.txt') path.resolve(import.meta.dir, '../Clash/ip/reject.txt')
)); );
}); });
if (import.meta.main) { if (import.meta.main) {

8
Build/build-apple-cdn.ts
View File

@ -23,7 +23,7 @@ export const getAppleCdnDomainsPromise = createMemoizedPromise(() => traceAsync(
picocolors.gray picocolors.gray
)); ));
export const buildAppleCdn = task(import.meta.path, async () => { export const buildAppleCdn = task(import.meta.path, async (span) => {
const res = await getAppleCdnDomainsPromise(); const res = await getAppleCdnDomainsPromise();
const description = [ const description = [
@ -39,7 +39,8 @@ export const buildAppleCdn = task(import.meta.path, async () => {
const domainset = res.map(i => `.${i}`); const domainset = res.map(i => `.${i}`);
return Promise.all([ return Promise.all([
...createRuleset( createRuleset(
span,
'Sukka\'s Ruleset - Apple CDN', 'Sukka\'s Ruleset - Apple CDN',
description, description,
new Date(), new Date(),
@ -48,7 +49,8 @@ export const buildAppleCdn = task(import.meta.path, async () => {
path.resolve(import.meta.dir, '../List/non_ip/apple_cdn.conf'), path.resolve(import.meta.dir, '../List/non_ip/apple_cdn.conf'),
path.resolve(import.meta.dir, '../Clash/non_ip/apple_cdn.txt') path.resolve(import.meta.dir, '../Clash/non_ip/apple_cdn.txt')
), ),
...createRuleset( createRuleset(
span,
'Sukka\'s Ruleset - Apple CDN', 'Sukka\'s Ruleset - Apple CDN',
description, description,
new Date(), new Date(),

7
Build/build-cdn-conf.ts
View File

@ -41,7 +41,7 @@ const getS3OSSDomains = async (): Promise<Set<string>> => {
return S3OSSDomains; return S3OSSDomains;
}; };
const buildCdnConf = task(import.meta.path, async () => { const buildCdnConf = task(import.meta.path, async (span) => {
/** @type {string[]} */ /** @type {string[]} */
const cdnDomainsList: string[] = []; const cdnDomainsList: string[] = [];
@ -62,7 +62,8 @@ const buildCdnConf = task(import.meta.path, async () => {
'This file contains object storage and static assets CDN domains.' 'This file contains object storage and static assets CDN domains.'
]; ];
return Promise.all(createRuleset( return createRuleset(
span,
'Sukka\'s Ruleset - CDN Domains', 'Sukka\'s Ruleset - CDN Domains',
description, description,
new Date(), new Date(),
@ -70,7 +71,7 @@ const buildCdnConf = task(import.meta.path, async () => {
'ruleset', 'ruleset',
path.resolve(import.meta.dir, '../List/non_ip/cdn.conf'), path.resolve(import.meta.dir, '../List/non_ip/cdn.conf'),
path.resolve(import.meta.dir, '../Clash/non_ip/cdn.txt') path.resolve(import.meta.dir, '../Clash/non_ip/cdn.txt')
)); );
}); });
export { buildCdnConf }; export { buildCdnConf };

4
Build/build-chn-cidr.ts
View File

@ -32,7 +32,7 @@ export const getChnCidrPromise = createMemoizedPromise(async () => {
); );
}); });
export const buildChnCidr = task(import.meta.path, async () => { export const buildChnCidr = task(import.meta.path, async (span) => {
const filteredCidr = await getChnCidrPromise(); const filteredCidr = await getChnCidrPromise();
// Can not use SHARED_DESCRIPTION here as different license // Can not use SHARED_DESCRIPTION here as different license
@ -47,6 +47,7 @@ export const buildChnCidr = task(import.meta.path, async () => {
// Can not use createRuleset here, as Clash support advanced ipset syntax // Can not use createRuleset here, as Clash support advanced ipset syntax
return Promise.all([ return Promise.all([
compareAndWriteFile( compareAndWriteFile(
span,
withBannerArray( withBannerArray(
'Sukka\'s Ruleset - Mainland China IPv4 CIDR', 'Sukka\'s Ruleset - Mainland China IPv4 CIDR',
description, description,
@ -56,6 +57,7 @@ export const buildChnCidr = task(import.meta.path, async () => {
pathResolve(import.meta.dir, '../List/ip/china_ip.conf') pathResolve(import.meta.dir, '../List/ip/china_ip.conf')
), ),
compareAndWriteFile( compareAndWriteFile(
span,
withBannerArray( withBannerArray(
'Sukka\'s Ruleset - Mainland China IPv4 CIDR', 'Sukka\'s Ruleset - Mainland China IPv4 CIDR',
description, description,

7
Build/build-cloudmounter-rules.ts
View File

@ -7,7 +7,7 @@ import { task } from './trace';
const outputSurgeDir = path.resolve(import.meta.dir, '../List'); const outputSurgeDir = path.resolve(import.meta.dir, '../List');
const outputClashDir = path.resolve(import.meta.dir, '../Clash'); const outputClashDir = path.resolve(import.meta.dir, '../Clash');
export const buildCloudMounterRules = task(import.meta.path, async () => { export const buildCloudMounterRules = task(import.meta.path, async (span) => {
// AND,((SRC-IP,192.168.1.110), (DOMAIN, example.com)) // AND,((SRC-IP,192.168.1.110), (DOMAIN, example.com))
const results = DOMAINS.flatMap(domain => { const results = DOMAINS.flatMap(domain => {
@ -18,7 +18,8 @@ export const buildCloudMounterRules = task(import.meta.path, async () => {
const description = SHARED_DESCRIPTION; const description = SHARED_DESCRIPTION;
return Promise.all(createRuleset( return createRuleset(
span,
'Sukka\'s Ruleset - CloudMounter / RaiDrive', 'Sukka\'s Ruleset - CloudMounter / RaiDrive',
description, description,
new Date(), new Date(),
@ -26,7 +27,7 @@ export const buildCloudMounterRules = task(import.meta.path, async () => {
'domainset', 'domainset',
path.resolve(outputSurgeDir, 'non_ip', 'cloudmounter.conf'), path.resolve(outputSurgeDir, 'non_ip', 'cloudmounter.conf'),
path.resolve(outputClashDir, 'non_ip', 'cloudmounter.txt') path.resolve(outputClashDir, 'non_ip', 'cloudmounter.txt')
)); );
}); });
if (import.meta.main) { if (import.meta.main) {

42
Build/build-common.ts
View File

@ -6,6 +6,7 @@ import { readFileByLine } from './lib/fetch-text-by-line';
import { processLine } from './lib/process-line'; import { processLine } from './lib/process-line';
import { createRuleset } from './lib/create-file'; import { createRuleset } from './lib/create-file';
import { domainDeduper } from './lib/domain-deduper'; import { domainDeduper } from './lib/domain-deduper';
import type { Span } from './trace';
import { task } from './trace'; import { task } from './trace';
import { SHARED_DESCRIPTION } from './lib/constants'; import { SHARED_DESCRIPTION } from './lib/constants';
@ -17,7 +18,7 @@ const sourceDir = path.resolve(import.meta.dir, '../Source');
const outputSurgeDir = path.resolve(import.meta.dir, '../List'); const outputSurgeDir = path.resolve(import.meta.dir, '../List');
const outputClashDir = path.resolve(import.meta.dir, '../Clash'); const outputClashDir = path.resolve(import.meta.dir, '../Clash');
export const buildCommon = task(import.meta.path, async () => { export const buildCommon = task(import.meta.path, async (span) => {
const promises: Array<Promise<unknown>> = []; const promises: Array<Promise<unknown>> = [];
const pw = new PathScurry(sourceDir); const pw = new PathScurry(sourceDir);
@ -33,14 +34,14 @@ export const buildCommon = task(import.meta.path, async () => {
const relativePath = entry.relative(); const relativePath = entry.relative();
if (relativePath.startsWith('domainset/')) { if (relativePath.startsWith('domainset/')) {
promises.push(transformDomainset(entry.fullpath(), relativePath)); promises.push(transformDomainset(span, entry.fullpath(), relativePath));
continue; continue;
} }
if ( if (
relativePath.startsWith('ip/') relativePath.startsWith('ip/')
|| relativePath.startsWith('non_ip/') || relativePath.startsWith('non_ip/')
) { ) {
promises.push(transformRuleset(entry.fullpath(), relativePath)); promises.push(transformRuleset(span, entry.fullpath(), relativePath));
continue; continue;
} }
} }
@ -52,9 +53,9 @@ if (import.meta.main) {
buildCommon(); buildCommon();
} }
const processFile = async (sourcePath: string) => { const processFile = (span: Span, sourcePath: string) => {
console.log('Processing', sourcePath); // console.log('Processing', sourcePath);
return span.traceChild(`process file: ${sourcePath}`).traceAsyncFn(async () => {
const lines: string[] = []; const lines: string[] = [];
let title = ''; let title = '';
@ -63,7 +64,7 @@ const processFile = async (sourcePath: string) => {
try { try {
for await (const line of readFileByLine(sourcePath)) { for await (const line of readFileByLine(sourcePath)) {
if (line === MAGIC_COMMAND_SKIP) { if (line === MAGIC_COMMAND_SKIP) {
return; return null;
} }
if (line.startsWith(MAGIC_COMMAND_TITLE)) { if (line.startsWith(MAGIC_COMMAND_TITLE)) {
@ -87,11 +88,15 @@ const processFile = async (sourcePath: string) => {
} }
return [title, descriptions, lines] as const; return [title, descriptions, lines] as const;
});
}; };
async function transformDomainset(sourcePath: string, relativePath: string) { async function transformDomainset(parentSpan: Span, sourcePath: string, relativePath: string) {
const res = await processFile(sourcePath); const span = parentSpan.traceChild(`transform domainset: ${path.basename(sourcePath, path.extname(sourcePath))}`);
const res = await processFile(span, sourcePath);
if (!res) return; if (!res) return;
const [title, descriptions, lines] = res; const [title, descriptions, lines] = res;
const deduped = domainDeduper(lines); const deduped = domainDeduper(lines);
@ -104,7 +109,8 @@ async function transformDomainset(sourcePath: string, relativePath: string) {
) )
]; ];
return Promise.all(createRuleset( return createRuleset(
span,
title, title,
description, description,
new Date(), new Date(),
@ -112,15 +118,18 @@ async function transformDomainset(sourcePath: string, relativePath: string) {
'domainset', 'domainset',
path.resolve(outputSurgeDir, relativePath), path.resolve(outputSurgeDir, relativePath),
path.resolve(outputClashDir, `${relativePath.slice(0, -path.extname(relativePath).length)}.txt`) path.resolve(outputClashDir, `${relativePath.slice(0, -path.extname(relativePath).length)}.txt`)
)); );
} }
/** /**
* Output Surge RULE-SET and Clash classical text format * Output Surge RULE-SET and Clash classical text format
*/ */
async function transformRuleset(sourcePath: string, relativePath: string) { async function transformRuleset(parentSpan: Span, sourcePath: string, relativePath: string) {
const res = await processFile(sourcePath); const span = parentSpan.traceChild(`transform ruleset: ${path.basename(sourcePath, path.extname(sourcePath))}`);
if (!res) return;
const res = await processFile(span, sourcePath);
if (!res) return null;
const [title, descriptions, lines] = res; const [title, descriptions, lines] = res;
const description = [ const description = [
@ -132,7 +141,8 @@ async function transformRuleset(sourcePath: string, relativePath: string) {
) )
]; ];
return Promise.all(createRuleset( return createRuleset(
span,
title, title,
description, description,
new Date(), new Date(),
@ -140,5 +150,5 @@ async function transformRuleset(sourcePath: string, relativePath: string) {
'ruleset', 'ruleset',
path.resolve(outputSurgeDir, relativePath), path.resolve(outputSurgeDir, relativePath),
path.resolve(outputClashDir, `${relativePath.slice(0, -path.extname(relativePath).length)}.txt`) path.resolve(outputClashDir, `${relativePath.slice(0, -path.extname(relativePath).length)}.txt`)
)); );
} }

6
Build/build-domestic-ruleset.ts
View File

@ -21,7 +21,7 @@ export const getDomesticDomainsRulesetPromise = createMemoizedPromise(async () =
return results; return results;
}); });
export const buildDomesticRuleset = task(import.meta.path, async () => { export const buildDomesticRuleset = task(import.meta.path, async (span) => {
const rulesetDescription = [ const rulesetDescription = [
...SHARED_DESCRIPTION, ...SHARED_DESCRIPTION,
'', '',
@ -29,7 +29,8 @@ export const buildDomesticRuleset = task(import.meta.path, async () => {
]; ];
return Promise.all([ return Promise.all([
...createRuleset( createRuleset(
span,
'Sukka\'s Ruleset - Domestic Domains', 'Sukka\'s Ruleset - Domestic Domains',
rulesetDescription, rulesetDescription,
new Date(), new Date(),
@ -39,6 +40,7 @@ export const buildDomesticRuleset = task(import.meta.path, async () => {
path.resolve(import.meta.dir, '../Clash/non_ip/domestic.txt') path.resolve(import.meta.dir, '../Clash/non_ip/domestic.txt')
), ),
compareAndWriteFile( compareAndWriteFile(
span,
[ [
'#!name=[Sukka] Local DNS Mapping', '#!name=[Sukka] Local DNS Mapping',
`#!desc=Last Updated: ${new Date().toISOString()}`, `#!desc=Last Updated: ${new Date().toISOString()}`,

3
Build/build-internal-cdn-rules.ts
View File

@ -33,7 +33,7 @@ const processLocalRuleSet = async (ruleSetPath: string, set: Set<string>, keywor
} }
}; };
export const buildInternalCDNDomains = task(import.meta.path, async () => { export const buildInternalCDNDomains = task(import.meta.path, async (span) => {
const proxySet = new Set<string>(); const proxySet = new Set<string>();
const proxyKeywords = new Set<string>(); const proxyKeywords = new Set<string>();
@ -50,6 +50,7 @@ export const buildInternalCDNDomains = task(import.meta.path, async () => {
]))[0]; ]))[0];
return compareAndWriteFile( return compareAndWriteFile(
span,
[ [
...sortDomains(Array.from(proxySet), gorhill).map(i => `SUFFIX,${i}`), ...sortDomains(Array.from(proxySet), gorhill).map(i => `SUFFIX,${i}`),
...Array.from(proxyKeywords).sort().map(i => `REGEX,${i}`) ...Array.from(proxyKeywords).sort().map(i => `REGEX,${i}`)

7
Build/build-microsoft-cdn.ts
View File

@ -41,7 +41,7 @@ export const getMicrosoftCdnRulesetPromise = createMemoizedPromise(async () => {
return Array.from(set).map(d => `DOMAIN-SUFFIX,${d}`).concat(WHITELIST); return Array.from(set).map(d => `DOMAIN-SUFFIX,${d}`).concat(WHITELIST);
}); });
export const buildMicrosoftCdn = task(import.meta.path, async () => { export const buildMicrosoftCdn = task(import.meta.path, async (span) => {
const description = [ const description = [
...SHARED_DESCRIPTION, ...SHARED_DESCRIPTION,
'', '',
@ -51,7 +51,8 @@ export const buildMicrosoftCdn = task(import.meta.path, async () => {
' - https://github.com/felixonmars/dnsmasq-china-list' ' - https://github.com/felixonmars/dnsmasq-china-list'
]; ];
return Promise.all(createRuleset( return createRuleset(
span,
'Sukka\'s Ruleset - Microsoft CDN', 'Sukka\'s Ruleset - Microsoft CDN',
description, description,
new Date(), new Date(),
@ -59,7 +60,7 @@ export const buildMicrosoftCdn = task(import.meta.path, async () => {
'ruleset', 'ruleset',
path.resolve(import.meta.dir, '../List/non_ip/microsoft_cdn.conf'), path.resolve(import.meta.dir, '../List/non_ip/microsoft_cdn.conf'),
path.resolve(import.meta.dir, '../Clash/non_ip/microsoft_cdn.txt') path.resolve(import.meta.dir, '../Clash/non_ip/microsoft_cdn.txt')
)); );
}); });
if (import.meta.main) { if (import.meta.main) {

22
Build/build-reject-domainset.ts
View File

@ -29,20 +29,20 @@ export const buildRejectDomainSet = task(import.meta.path, async (span) => {
// Parse from AdGuard Filters // Parse from AdGuard Filters
const [gorhill, shouldStop] = await span const [gorhill, shouldStop] = await span
.traceChild('download and process hosts / adblock filter rules') .traceChild('download and process hosts / adblock filter rules')
.traceAsyncFn(async () => { .traceAsyncFn(async (childSpan) => {
let shouldStop = false; let shouldStop = false;
const [gorhill] = await Promise.all([ const [gorhill] = await Promise.all([
getGorhillPublicSuffixPromise(), getGorhillPublicSuffixPromise(),
// Parse from remote hosts & domain lists // Parse from remote hosts & domain lists
...HOSTS.map(entry => processHosts(span, entry[0], entry[1], entry[2]).then(hosts => { ...HOSTS.map(entry => processHosts(childSpan, entry[0], entry[1], entry[2]).then(hosts => {
SetHelpers.add(domainSets, hosts); SetHelpers.add(domainSets, hosts);
})), })),
...DOMAIN_LISTS.map(entry => processDomainLists(span, entry[0], entry[1], entry[2])), ...DOMAIN_LISTS.map(entry => processDomainLists(childSpan, entry[0], entry[1], entry[2])),
...ADGUARD_FILTERS.map(input => { ...ADGUARD_FILTERS.map(input => {
const promise = typeof input === 'string' const promise = typeof input === 'string'
? processFilterRules(span, input) ? processFilterRules(childSpan, input)
: processFilterRules(span, input[0], input[1], input[2]); : processFilterRules(childSpan, input[0], input[1], input[2]);
return promise.then(({ white, black, foundDebugDomain }) => { return promise.then(({ white, black, foundDebugDomain }) => {
if (foundDebugDomain) { if (foundDebugDomain) {
@ -56,22 +56,22 @@ export const buildRejectDomainSet = task(import.meta.path, async (span) => {
...([ ...([
'https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt', 'https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt',
'https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt' 'https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt'
].map(input => processFilterRules(span, input).then(({ white, black }) => { ].map(input => processFilterRules(childSpan, input).then(({ white, black }) => {
setAddFromArray(filterRuleWhitelistDomainSets, white); setAddFromArray(filterRuleWhitelistDomainSets, white);
setAddFromArray(filterRuleWhitelistDomainSets, black); setAddFromArray(filterRuleWhitelistDomainSets, black);
}))), }))),
getPhishingDomains(span).then(([purePhishingDomains, fullPhishingDomainSet]) => { getPhishingDomains(childSpan).then(([purePhishingDomains, fullPhishingDomainSet]) => {
SetHelpers.add(domainSets, fullPhishingDomainSet); SetHelpers.add(domainSets, fullPhishingDomainSet);
setAddFromArray(domainSets, purePhishingDomains); setAddFromArray(domainSets, purePhishingDomains);
}), }),
(async () => { childSpan.traceChild('process reject_sukka.conf').traceAsyncFn(async () => {
for await (const l of readFileByLine(path.resolve(import.meta.dir, '../Source/domainset/reject_sukka.conf'))) { for await (const l of readFileByLine(path.resolve(import.meta.dir, '../Source/domainset/reject_sukka.conf'))) {
const line = processLine(l); const line = processLine(l);
if (line) { if (line) {
domainSets.add(line); domainSets.add(line);
} }
} }
})() })
]); ]);
// remove pre-defined enforced blacklist from whitelist // remove pre-defined enforced blacklist from whitelist
@ -187,7 +187,8 @@ export const buildRejectDomainSet = task(import.meta.path, async (span) => {
]; ];
return Promise.all([ return Promise.all([
...createRuleset( createRuleset(
span,
'Sukka\'s Ruleset - Reject Base', 'Sukka\'s Ruleset - Reject Base',
description, description,
new Date(), new Date(),
@ -197,6 +198,7 @@ export const buildRejectDomainSet = task(import.meta.path, async (span) => {
path.resolve(import.meta.dir, '../Clash/domainset/reject.txt') path.resolve(import.meta.dir, '../Clash/domainset/reject.txt')
), ),
compareAndWriteFile( compareAndWriteFile(
span,
rejectDomainsStats.map(([domain, count]) => `${domain}${' '.repeat(100 - domain.length)}${count}`), rejectDomainsStats.map(([domain, count]) => `${domain}${' '.repeat(100 - domain.length)}${count}`),
path.resolve(import.meta.dir, '../List/internal/reject-stats.txt') path.resolve(import.meta.dir, '../List/internal/reject-stats.txt')
), ),

3
Build/build-sgmodule-always-realip.ts
View File

@ -38,8 +38,9 @@ const HOSTNAMES = [
'GC._msDCS.*.*' 'GC._msDCS.*.*'
] as const; ] as const;
export const buildAlwaysRealIPModule = task(import.meta.path, async () => { export const buildAlwaysRealIPModule = task(import.meta.path, async (span) => {
return compareAndWriteFile( return compareAndWriteFile(
span,
[ [
'#!name=[Sukka] Always Real IP Plus', '#!name=[Sukka] Always Real IP Plus',
`#!desc=Last Updated: ${new Date().toISOString()}`, `#!desc=Last Updated: ${new Date().toISOString()}`,

3
Build/build-sgmodule-redirect.ts
View File

@ -71,10 +71,11 @@ const REDIRECT = [
['googleajax.wp-china-yes.net/', 'https://ajax.googleapis.com/'] ['googleajax.wp-china-yes.net/', 'https://ajax.googleapis.com/']
] as const; ] as const;
export const buildRedirectModule = task(import.meta.path, async () => { export const buildRedirectModule = task(import.meta.path, async (span) => {
const domains = Array.from(new Set(REDIRECT.map(([from]) => tldts.getHostname(from, { detectIp: false })))).filter(Boolean); const domains = Array.from(new Set(REDIRECT.map(([from]) => tldts.getHostname(from, { detectIp: false })))).filter(Boolean);
return compareAndWriteFile( return compareAndWriteFile(
span,
[ [
'#!name=[Sukka] URL Redirect', '#!name=[Sukka] URL Redirect',
`#!desc=Last Updated: ${new Date().toISOString()}`, `#!desc=Last Updated: ${new Date().toISOString()}`,

8
Build/build-speedtest-domainset.ts
View File

@ -151,13 +151,14 @@ export const buildSpeedtestDomainSet = task(import.meta.path, async (span) => {
'.backend.librespeed.org' '.backend.librespeed.org'
]); ]);
// Download previous speedtest domainset await span.traceChild('fetch previous speedtest domainset').traceAsyncFn(async () => {
for await (const l of await fetchRemoteTextByLine('https://ruleset.skk.moe/List/domainset/speedtest.conf')) { for await (const l of await fetchRemoteTextByLine('https://ruleset.skk.moe/List/domainset/speedtest.conf')) {
const line = processLine(l); const line = processLine(l);
if (line) { if (line) {
domains.add(line); domains.add(line);
} }
} }
});
await new Promise<void>((resolve) => { await new Promise<void>((resolve) => {
const pMap = ([ const pMap = ([
@ -225,7 +226,8 @@ export const buildSpeedtestDomainSet = task(import.meta.path, async (span) => {
'This file contains common speedtest endpoints.' 'This file contains common speedtest endpoints.'
]; ];
return Promise.all(createRuleset( return createRuleset(
span,
'Sukka\'s Ruleset - Speedtest Domains', 'Sukka\'s Ruleset - Speedtest Domains',
description, description,
new Date(), new Date(),
@ -233,7 +235,7 @@ export const buildSpeedtestDomainSet = task(import.meta.path, async (span) => {
'domainset', 'domainset',
path.resolve(import.meta.dir, '../List/domainset/speedtest.conf'), path.resolve(import.meta.dir, '../List/domainset/speedtest.conf'),
path.resolve(import.meta.dir, '../Clash/domainset/speedtest.txt') path.resolve(import.meta.dir, '../Clash/domainset/speedtest.txt')
)); );
}); });
if (import.meta.main) { if (import.meta.main) {

3
Build/build-sspanel-appprofile.ts
View File

@ -26,7 +26,7 @@ const removeNoResolved = (line: string) => line.replace(',no-resolve', '');
/** /**
* This only generates a simplified version, for under-used users only. * This only generates a simplified version, for under-used users only.
*/ */
export const buildSSPanelUIMAppProfile = task(import.meta.path, async () => { export const buildSSPanelUIMAppProfile = task(import.meta.path, async (span) => {
const [ const [
domesticDomains, domesticDomains,
appleCdnDomains, appleCdnDomains,
@ -108,6 +108,7 @@ export const buildSSPanelUIMAppProfile = task(import.meta.path, async () => {
); );
await compareAndWriteFile( await compareAndWriteFile(
span,
output, output,
path.resolve(import.meta.dir, '../List/internal/appprofile.php') path.resolve(import.meta.dir, '../List/internal/appprofile.php')
); );

26
Build/build-stream-service.ts
View File

@ -1,4 +1,5 @@
// @ts-check // @ts-check
import type { Span } from './trace';
import { task } from './trace'; import { task } from './trace';
import path from 'path'; import path from 'path';
@ -7,10 +8,12 @@ import { createRuleset } from './lib/create-file';
import { ALL, NORTH_AMERICA, EU, HK, TW, JP, KR } from '../Source/stream'; import { ALL, NORTH_AMERICA, EU, HK, TW, JP, KR } from '../Source/stream';
import { SHARED_DESCRIPTION } from './lib/constants'; import { SHARED_DESCRIPTION } from './lib/constants';
export const createRulesetForStreamService = (fileId: string, title: string, streamServices: Array<import('../Source/stream').StreamService>) => { export const createRulesetForStreamService = (span: Span, fileId: string, title: string, streamServices: Array<import('../Source/stream').StreamService>) => {
const childSpan = span.traceChild(fileId);
return [ return [
// Domains // Domains
...createRuleset( createRuleset(
childSpan,
`Sukka's Ruleset - Stream Services: ${title}`, `Sukka's Ruleset - Stream Services: ${title}`,
[ [
...SHARED_DESCRIPTION, ...SHARED_DESCRIPTION,
@ -24,7 +27,8 @@ export const createRulesetForStreamService = (fileId: string, title: string, str
path.resolve(import.meta.dir, `../Clash/non_ip/${fileId}.txt`) path.resolve(import.meta.dir, `../Clash/non_ip/${fileId}.txt`)
), ),
// IP // IP
...createRuleset( createRuleset(
childSpan,
`Sukka's Ruleset - Stream Services' IPs: ${title}`, `Sukka's Ruleset - Stream Services' IPs: ${title}`,
[ [
...SHARED_DESCRIPTION, ...SHARED_DESCRIPTION,
@ -47,16 +51,16 @@ export const createRulesetForStreamService = (fileId: string, title: string, str
]; ];
}; };
export const buildStreamService = task(import.meta.path, async () => { export const buildStreamService = task(import.meta.path, async (span) => {
return Promise.all([ return Promise.all([
...createRulesetForStreamService('stream', 'All', ALL), ...createRulesetForStreamService(span, 'stream', 'All', ALL),
...createRulesetForStreamService('stream_us', 'North America', NORTH_AMERICA), ...createRulesetForStreamService(span, 'stream_us', 'North America', NORTH_AMERICA),
...createRulesetForStreamService('stream_eu', 'Europe', EU), ...createRulesetForStreamService(span, 'stream_eu', 'Europe', EU),
...createRulesetForStreamService('stream_hk', 'Hong Kong', HK), ...createRulesetForStreamService(span, 'stream_hk', 'Hong Kong', HK),
...createRulesetForStreamService('stream_tw', 'Taiwan', TW), ...createRulesetForStreamService(span, 'stream_tw', 'Taiwan', TW),
...createRulesetForStreamService('stream_jp', 'Japan', JP), ...createRulesetForStreamService(span, 'stream_jp', 'Japan', JP),
// ...createRulesetForStreamService('stream_au', 'Oceania', AU), // ...createRulesetForStreamService('stream_au', 'Oceania', AU),
...createRulesetForStreamService('stream_kr', 'Korean', KR) ...createRulesetForStreamService(span, 'stream_kr', 'Korean', KR)
// ...createRulesetForStreamService('stream_south_east_asia', 'South East Asia', SOUTH_EAST_ASIA) // ...createRulesetForStreamService('stream_south_east_asia', 'South East Asia', SOUTH_EAST_ASIA)
]); ]);
}); });

7
Build/build-telegram-cidr.ts
View File

@ -32,7 +32,7 @@ export const getTelegramCIDRPromise = createMemoizedPromise(async () => {
return { date, results }; return { date, results };
}); });
export const buildTelegramCIDR = task(import.meta.path, async () => { export const buildTelegramCIDR = task(import.meta.path, async (span) => {
const { date, results } = await getTelegramCIDRPromise(); const { date, results } = await getTelegramCIDRPromise();
if (results.length === 0) { if (results.length === 0) {
@ -45,7 +45,8 @@ export const buildTelegramCIDR = task(import.meta.path, async () => {
' - https://core.telegram.org/resources/cidr.txt' ' - https://core.telegram.org/resources/cidr.txt'
]; ];
return Promise.all(createRuleset( return createRuleset(
span,
'Sukka\'s Ruleset - Telegram IP CIDR', 'Sukka\'s Ruleset - Telegram IP CIDR',
description, description,
date, date,
@ -53,7 +54,7 @@ export const buildTelegramCIDR = task(import.meta.path, async () => {
'ruleset', 'ruleset',
path.resolve(import.meta.dir, '../List/ip/telegram.conf'), path.resolve(import.meta.dir, '../List/ip/telegram.conf'),
path.resolve(import.meta.dir, '../Clash/ip/telegram.txt') path.resolve(import.meta.dir, '../Clash/ip/telegram.txt')
)); );
}); });
if (import.meta.main) { if (import.meta.main) {

14
Build/download-mock-assets.ts
View File

@ -12,15 +12,11 @@ const ASSETS_LIST = {
const mockDir = path.resolve(import.meta.dir, '../Mock'); const mockDir = path.resolve(import.meta.dir, '../Mock');
export const downloadMockAssets = task(import.meta.path, () => Promise.all(Object.entries(ASSETS_LIST).map(async ([filename, url]) => { export const downloadMockAssets = task(import.meta.path, (span) => Promise.all(Object.entries(ASSETS_LIST).map(
const targetPath = path.join(mockDir, filename); ([filename, url]) => span
.traceChild(url)
const key = picocolors.gray(`Download ${filename}`); .traceAsyncFn(() => fetchWithRetry(url).then(res => Bun.write(path.join(mockDir, filename), res)))
console.time(key); )));
const res = await fetchWithRetry(url);
await Bun.write(targetPath, res);
console.timeEnd(key);
})));
if (import.meta.main) { if (import.meta.main) {
downloadMockAssets(); downloadMockAssets();

35
Build/lib/create-file.ts
View File

@ -3,8 +3,10 @@ import { readFileByLine } from './fetch-text-by-line';
import { surgeDomainsetToClashDomainset, surgeRulesetToClashClassicalTextRuleset } from './clash'; import { surgeDomainsetToClashDomainset, surgeRulesetToClashClassicalTextRuleset } from './clash';
import { traceAsync } from './trace-runner'; import { traceAsync } from './trace-runner';
import picocolors from 'picocolors'; import picocolors from 'picocolors';
import type { Span } from '../trace';
import path from 'path';
export async function compareAndWriteFile(linesA: string[], filePath: string) { export async function compareAndWriteFile(span: Span, linesA: string[], filePath: string) {
let isEqual = true; let isEqual = true;
const file = Bun.file(filePath); const file = Bun.file(filePath);
@ -17,9 +19,7 @@ export async function compareAndWriteFile(linesA: string[], filePath: string) {
console.log(`Nothing to write to ${filePath}...`); console.log(`Nothing to write to ${filePath}...`);
isEqual = false; isEqual = false;
} else { } else {
isEqual = await traceAsync( isEqual = await span.traceChild(`comparing ${filePath}`).traceAsyncFn(async () => {
picocolors.gray(`comparing ${filePath}`),
async () => {
let index = 0; let index = 0;
for await (const lineB of readFileByLine(file)) { for await (const lineB of readFileByLine(file)) {
@ -56,9 +56,7 @@ export async function compareAndWriteFile(linesA: string[], filePath: string) {
} }
return true; return true;
}, });
picocolors.gray
);
} }
if (isEqual) { if (isEqual) {
@ -66,7 +64,7 @@ export async function compareAndWriteFile(linesA: string[], filePath: string) {
return; return;
} }
await traceAsync(picocolors.gray(`writing ${filePath}`), async () => { await span.traceChild(`writing ${filePath}`).traceAsyncFn(async () => {
if (linesALen < 10000) { if (linesALen < 10000) {
return Bun.write(file, `${linesA.join('\n')}\n`); return Bun.write(file, `${linesA.join('\n')}\n`);
} }
@ -79,7 +77,7 @@ export async function compareAndWriteFile(linesA: string[], filePath: string) {
} }
return writer.end(); return writer.end();
}, picocolors.gray); });
} }
export const withBannerArray = (title: string, description: string[] | readonly string[], date: Date, content: string[]) => { export const withBannerArray = (title: string, description: string[] | readonly string[], date: Date, content: string[]) => {
@ -96,11 +94,12 @@ export const withBannerArray = (title: string, description: string[] | readonly
}; };
export const createRuleset = ( export const createRuleset = (
parentSpan: Span,
title: string, description: string[] | readonly string[], date: Date, content: string[], title: string, description: string[] | readonly string[], date: Date, content: string[],
type: 'ruleset' | 'domainset', surgePath: string, clashPath: string type: 'ruleset' | 'domainset', surgePath: string, clashPath: string
) => { ) => parentSpan.traceChild(`create ruleset: ${path.basename(surgePath, path.extname(surgePath))}`).traceAsyncFn((childSpan) => {
const surgeContent = withBannerArray(title, description, date, content); const surgeContent = withBannerArray(title, description, date, content);
const clashContent = childSpan.traceChild('convert incoming ruleset to clash').traceSyncFn(() => {
let _clashContent; let _clashContent;
switch (type) { switch (type) {
case 'domainset': case 'domainset':
@ -112,11 +111,11 @@ export const createRuleset = (
default: default:
throw new TypeError(`Unknown type: ${type as any}`); throw new TypeError(`Unknown type: ${type as any}`);
} }
return withBannerArray(title, description, date, _clashContent);
});
const clashContent = withBannerArray(title, description, date, _clashContent); return Promise.all([
compareAndWriteFile(childSpan, surgeContent, surgePath),
return [ compareAndWriteFile(childSpan, clashContent, clashPath)
compareAndWriteFile(surgeContent, surgePath), ]);
compareAndWriteFile(clashContent, clashPath) });
];
};

5
Build/lib/fetch-text-by-line.ts
View File

@ -41,17 +41,16 @@ export function readFileByLine(file: string | URL | BunFile) {
return createTextLineAsyncGeneratorFromStreamSource(file.stream()); return createTextLineAsyncGeneratorFromStreamSource(file.stream());
} }
export function createReadlineInterfaceFromResponse(resp: Response) { export function createReadlineInterfaceFromResponse(this: void, resp: Response) {
if (!resp.body) { if (!resp.body) {
throw new Error('Failed to fetch remote text'); throw new Error('Failed to fetch remote text');
} }
if (resp.bodyUsed) { if (resp.bodyUsed) {
throw new Error('Body has already been consumed.'); throw new Error('Body has already been consumed.');
} }
return createTextLineAsyncGeneratorFromStreamSource(resp.body); return createTextLineAsyncGeneratorFromStreamSource(resp.body);
} }
export function fetchRemoteTextByLine(url: string | URL) { export function fetchRemoteTextByLine(url: string | URL) {
return fetchWithRetry(url, defaultRequestInit).then(res => createReadlineInterfaceFromResponse(res)); return fetchWithRetry(url, defaultRequestInit).then(createReadlineInterfaceFromResponse);
} }

17
Build/lib/parse-filter.ts
View File

@ -96,12 +96,12 @@ const enum ParseType {
} }
export async function processFilterRules( export async function processFilterRules(
span: Span, parentSpan: Span,
filterRulesUrl: string, filterRulesUrl: string,
fallbackUrls?: readonly string[] | undefined | null, fallbackUrls?: readonly string[] | undefined | null,
ttl: number | null = null ttl: number | null = null
): Promise<{ white: string[], black: string[], foundDebugDomain: boolean }> { ): Promise<{ white: string[], black: string[], foundDebugDomain: boolean }> {
const [white, black, warningMessages] = await span.traceChild('process filter rules: domainListsUrl').traceAsyncFn(() => fsCache.apply<Readonly<[ const [white, black, warningMessages] = await parentSpan.traceChild(`process filter rules: ${filterRulesUrl}`).traceAsyncFn((span) => fsCache.apply<Readonly<[
white: string[], white: string[],
black: string[], black: string[],
warningMessages: string[] warningMessages: string[]
@ -179,18 +179,15 @@ export async function processFilterRules(
// Avoid event loop starvation, so we wait for a macrotask before we start fetching. // Avoid event loop starvation, so we wait for a macrotask before we start fetching.
await Promise.resolve(); await Promise.resolve();
const filterRules = (await traceAsync( const filterRules = await span.traceChild('download adguard filter').traceAsyncFn(() => {
picocolors.gray(`- download ${filterRulesUrl}`), return fetchAssets(filterRulesUrl, fallbackUrls).then(text => text.split('\n'));
() => fetchAssets(filterRulesUrl, fallbackUrls), });
picocolors.gray
)).split('\n');
const key = picocolors.gray(`- parse adguard filter ${filterRulesUrl}`); span.traceChild('parse adguard filter').traceSyncFn(() => {
console.time(key);
for (let i = 0, len = filterRules.length; i < len; i++) { for (let i = 0, len = filterRules.length; i < len; i++) {
lineCb(filterRules[i]); lineCb(filterRules[i]);
} }
console.timeEnd(key); });
} }
return [ return [

6
Build/lib/reject-data-source.ts
View File

@ -47,7 +47,9 @@ export const DOMAIN_LISTS = [
['https://raw.githubusercontent.com/AdguardTeam/cname-trackers/master/data/combined_disguised_mail_trackers_justdomains.txt', true, TTL.THREE_DAYS()] ['https://raw.githubusercontent.com/AdguardTeam/cname-trackers/master/data/combined_disguised_mail_trackers_justdomains.txt', true, TTL.THREE_DAYS()]
] as const; ] as const;
export const ADGUARD_FILTERS = [ type AdGuardFilterSource = string | [main: string, mirrors: string[] | null, ttl: number];
export const ADGUARD_FILTERS: AdGuardFilterSource[] = [
// EasyList // EasyList
[ [
'https://easylist.to/easylist/easylist.txt', 'https://easylist.to/easylist/easylist.txt',
@ -156,7 +158,7 @@ export const ADGUARD_FILTERS = [
// Not actively maintained, let's use a 10 days cache ttl // Not actively maintained, let's use a 10 days cache ttl
['https://raw.githubusercontent.com/Spam404/lists/master/adblock-list.txt', null, TTL.TEN_DAYS()], ['https://raw.githubusercontent.com/Spam404/lists/master/adblock-list.txt', null, TTL.TEN_DAYS()],
// Brave First Party & First Party CNAME // Brave First Party & First Party CNAME
'https://raw.githubusercontent.com/brave/adblock-lists/master/brave-lists/brave-firstparty.txt' ['https://raw.githubusercontent.com/brave/adblock-lists/master/brave-lists/brave-firstparty.txt', null, TTL.ONE_DAY()]
] as const; ] as const;
export const PREDEFINED_WHITELIST = [ export const PREDEFINED_WHITELIST = [

40
Build/trace/index.ts
View File

@ -114,17 +114,45 @@ export const universalify = <A extends any[], R>(taskname: string, fn: (this: vo
}; };
}; };
export const printTraceResult = (traceResult: TraceResult = rootTraceResult, level = 0, isLast = false) => { export const printTraceResult = (traceResult: TraceResult = rootTraceResult) => {
if (level === 0) {
printStats(traceResult.children); printStats(traceResult.children);
printTree(traceResult, node => `${node.name} ${picocolors.bold(`${(node.end - node.start).toFixed(3)}ms`)}`);
};
function printTree(initialTree: TraceResult, printNode: (node: TraceResult, branch: string) => string) {
function printBranch(tree: TraceResult, branch: string) {
const isGraphHead = branch.length === 0;
const children = tree.children;
let branchHead = '';
if (!isGraphHead) {
branchHead = children.length > 0 ? '┬ ' : '─ ';
} }
const prefix = (level > 0 ? ` ${'│ '.repeat(level - 1)}` : '') + (level > 0 ? (isLast ? '└─' : '├─') : ''); const toPrint = printNode(tree, `${branch}${branchHead}`);
console.log(`${prefix} ${traceResult.name} ${picocolors.bold(`${(traceResult.end - traceResult.start).toFixed(2)}ms`)}`); if (typeof toPrint === 'string') {
console.log(`${branch}${branchHead}${toPrint}`);
}
traceResult.children.forEach((child, index, arr) => printTraceResult(child, level + 1, index === arr.length - 1)); let baseBranch = branch;
};
if (!isGraphHead) {
const isChildOfLastBranch = branch.endsWith('└─');
baseBranch = branch.slice(0, -2) + (isChildOfLastBranch ? ' ' : '│ ');
}
const nextBranch = `${baseBranch}├─`;
const lastBranch = `${baseBranch}└─`;
children.forEach((child, index) => {
printBranch(child, children.length - 1 === index ? lastBranch : nextBranch);
});
}
printBranch(initialTree, '');
}
function printStats(stats: TraceResult[]): void { function printStats(stats: TraceResult[]): void {
stats.sort((a, b) => a.start - b.start); stats.sort((a, b) => a.start - b.start);

1
Source/non_ip/ai.conf
View File

@ -10,3 +10,4 @@ DOMAIN-SUFFIX,openaiapi-site.azureedge.net
DOMAIN-SUFFIX,perplexity.ai DOMAIN-SUFFIX,perplexity.ai
DOMAIN-SUFFIX,anthropic.com DOMAIN-SUFFIX,anthropic.com
DOMAIN-SUFFIX,claude.ai DOMAIN-SUFFIX,claude.ai
DOMAIN-SUFFIX,generativelanguage.googleapis.com