GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2025-12-12 01:00:34 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Refactor: full span tracer

Browse Source
This commit is contained in:
SukkaW 2024-01-14 22:10:16 +08:00
parent 0f257e992a
commit 9bb0c14d5f
23 changed files with 238 additions and 183 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
Build/build-anti-bogus-domain.ts
View File

@ -35,7 +35,7 @@ const getBogusNxDomainIPs = async () => {
);
};
export const buildAntiBogusDomain = task(import.meta.path, async () => {
export const buildAntiBogusDomain = task(import.meta.path, async (span) => {
const bogusIpPromise = getBogusNxDomainIPs();
const result: string[] = [];
@ -57,7 +57,8 @@ export const buildAntiBogusDomain = task(import.meta.path, async () => {
' - https://github.com/felixonmars/dnsmasq-china-list'
];
return Promise.all(createRuleset(
return createRuleset(
span,
'Sukka\'s Ruleset - Anti Bogus Domain',
description,
new Date(),
@ -65,7 +66,7 @@ export const buildAntiBogusDomain = task(import.meta.path, async () => {
'ruleset',
path.resolve(import.meta.dir, '../List/ip/reject.conf'),
path.resolve(import.meta.dir, '../Clash/ip/reject.txt')
));
);
});
if (import.meta.main) {

8
Build/build-apple-cdn.ts
View File

@ -23,7 +23,7 @@ export const getAppleCdnDomainsPromise = createMemoizedPromise(() => traceAsync(
picocolors.gray
));
export const buildAppleCdn = task(import.meta.path, async () => {
export const buildAppleCdn = task(import.meta.path, async (span) => {
const res = await getAppleCdnDomainsPromise();
const description = [
@ -39,7 +39,8 @@ export const buildAppleCdn = task(import.meta.path, async () => {
const domainset = res.map(i => `.${i}`);
return Promise.all([
...createRuleset(
createRuleset(
span,
'Sukka\'s Ruleset - Apple CDN',
description,
new Date(),
@ -48,7 +49,8 @@ export const buildAppleCdn = task(import.meta.path, async () => {
path.resolve(import.meta.dir, '../List/non_ip/apple_cdn.conf'),
path.resolve(import.meta.dir, '../Clash/non_ip/apple_cdn.txt')
),
...createRuleset(
createRuleset(
span,
'Sukka\'s Ruleset - Apple CDN',
description,
new Date(),

7
Build/build-cdn-conf.ts
View File

@ -41,7 +41,7 @@ const getS3OSSDomains = async (): Promise<Set<string>> => {
return S3OSSDomains;
};
const buildCdnConf = task(import.meta.path, async () => {
const buildCdnConf = task(import.meta.path, async (span) => {
/** @type {string[]} */
const cdnDomainsList: string[] = [];
@ -62,7 +62,8 @@ const buildCdnConf = task(import.meta.path, async () => {
'This file contains object storage and static assets CDN domains.'
];
return Promise.all(createRuleset(
return createRuleset(
span,
'Sukka\'s Ruleset - CDN Domains',
description,
new Date(),
@ -70,7 +71,7 @@ const buildCdnConf = task(import.meta.path, async () => {
'ruleset',
path.resolve(import.meta.dir, '../List/non_ip/cdn.conf'),
path.resolve(import.meta.dir, '../Clash/non_ip/cdn.txt')
));
);
});
export { buildCdnConf };

4
Build/build-chn-cidr.ts
View File

@ -32,7 +32,7 @@ export const getChnCidrPromise = createMemoizedPromise(async () => {
);
});
export const buildChnCidr = task(import.meta.path, async () => {
export const buildChnCidr = task(import.meta.path, async (span) => {
const filteredCidr = await getChnCidrPromise();
// Can not use SHARED_DESCRIPTION here as different license
@ -47,6 +47,7 @@ export const buildChnCidr = task(import.meta.path, async () => {
// Can not use createRuleset here, as Clash support advanced ipset syntax
return Promise.all([
compareAndWriteFile(
span,
withBannerArray(
'Sukka\'s Ruleset - Mainland China IPv4 CIDR',
description,
@ -56,6 +57,7 @@ export const buildChnCidr = task(import.meta.path, async () => {
pathResolve(import.meta.dir, '../List/ip/china_ip.conf')
),
compareAndWriteFile(
span,
withBannerArray(
'Sukka\'s Ruleset - Mainland China IPv4 CIDR',
description,

7
Build/build-cloudmounter-rules.ts
View File

@ -7,7 +7,7 @@ import { task } from './trace';
const outputSurgeDir = path.resolve(import.meta.dir, '../List');
const outputClashDir = path.resolve(import.meta.dir, '../Clash');
export const buildCloudMounterRules = task(import.meta.path, async () => {
export const buildCloudMounterRules = task(import.meta.path, async (span) => {
// AND,((SRC-IP,192.168.1.110), (DOMAIN, example.com))
const results = DOMAINS.flatMap(domain => {
@ -18,7 +18,8 @@ export const buildCloudMounterRules = task(import.meta.path, async () => {
const description = SHARED_DESCRIPTION;
return Promise.all(createRuleset(
return createRuleset(
span,
'Sukka\'s Ruleset - CloudMounter / RaiDrive',
description,
new Date(),
@ -26,7 +27,7 @@ export const buildCloudMounterRules = task(import.meta.path, async () => {
'domainset',
path.resolve(outputSurgeDir, 'non_ip', 'cloudmounter.conf'),
path.resolve(outputClashDir, 'non_ip', 'cloudmounter.txt')
));
);
});
if (import.meta.main) {

92
Build/build-common.ts
View File

@ -6,6 +6,7 @@ import { readFileByLine } from './lib/fetch-text-by-line';
import { processLine } from './lib/process-line';
import { createRuleset } from './lib/create-file';
import { domainDeduper } from './lib/domain-deduper';
import type { Span } from './trace';
import { task } from './trace';
import { SHARED_DESCRIPTION } from './lib/constants';
@ -17,7 +18,7 @@ const sourceDir = path.resolve(import.meta.dir, '../Source');
const outputSurgeDir = path.resolve(import.meta.dir, '../List');
const outputClashDir = path.resolve(import.meta.dir, '../Clash');
export const buildCommon = task(import.meta.path, async () => {
export const buildCommon = task(import.meta.path, async (span) => {
const promises: Array<Promise<unknown>> = [];
const pw = new PathScurry(sourceDir);
@ -33,14 +34,14 @@ export const buildCommon = task(import.meta.path, async () => {
const relativePath = entry.relative();
if (relativePath.startsWith('domainset/')) {
promises.push(transformDomainset(entry.fullpath(), relativePath));
promises.push(transformDomainset(span, entry.fullpath(), relativePath));
continue;
}
if (
relativePath.startsWith('ip/')
|| relativePath.startsWith('non_ip/')
) {
promises.push(transformRuleset(entry.fullpath(), relativePath));
promises.push(transformRuleset(span, entry.fullpath(), relativePath));
continue;
}
}
@ -52,46 +53,50 @@ if (import.meta.main) {
buildCommon();
}
const processFile = async (sourcePath: string) => {
console.log('Processing', sourcePath);
const processFile = (span: Span, sourcePath: string) => {
// console.log('Processing', sourcePath);
return span.traceChild(`process file: ${sourcePath}`).traceAsyncFn(async () => {
const lines: string[] = [];
const lines: string[] = [];
let title = '';
const descriptions: string[] = [];
let title = '';
const descriptions: string[] = [];
try {
for await (const line of readFileByLine(sourcePath)) {
if (line === MAGIC_COMMAND_SKIP) {
return null;
}
try {
for await (const line of readFileByLine(sourcePath)) {
if (line === MAGIC_COMMAND_SKIP) {
return;
}
if (line.startsWith(MAGIC_COMMAND_TITLE)) {
title = line.slice(MAGIC_COMMAND_TITLE.length).trim();
continue;
}
if (line.startsWith(MAGIC_COMMAND_DESCRIPTION)) {
descriptions.push(line.slice(MAGIC_COMMAND_DESCRIPTION.length).trim());
continue;
}
const l = processLine(line);
if (l) {
lines.push(l);
if (line.startsWith(MAGIC_COMMAND_TITLE)) {
title = line.slice(MAGIC_COMMAND_TITLE.length).trim();
continue;
}
if (line.startsWith(MAGIC_COMMAND_DESCRIPTION)) {
descriptions.push(line.slice(MAGIC_COMMAND_DESCRIPTION.length).trim());
continue;
}
const l = processLine(line);
if (l) {
lines.push(l);
}
}
} catch (e) {
console.error('Error processing', sourcePath);
console.trace(e);
}
} catch (e) {
console.error('Error processing', sourcePath);
console.trace(e);
}
return [title, descriptions, lines] as const;
return [title, descriptions, lines] as const;
});
};
async function transformDomainset(sourcePath: string, relativePath: string) {
const res = await processFile(sourcePath);
async function transformDomainset(parentSpan: Span, sourcePath: string, relativePath: string) {
const span = parentSpan.traceChild(`transform domainset: ${path.basename(sourcePath, path.extname(sourcePath))}`);
const res = await processFile(span, sourcePath);
if (!res) return;
const [title, descriptions, lines] = res;
const deduped = domainDeduper(lines);
@ -104,7 +109,8 @@ async function transformDomainset(sourcePath: string, relativePath: string) {
)
];
return Promise.all(createRuleset(
return createRuleset(
span,
title,
description,
new Date(),
@ -112,15 +118,18 @@ async function transformDomainset(sourcePath: string, relativePath: string) {
'domainset',
path.resolve(outputSurgeDir, relativePath),
path.resolve(outputClashDir, `${relativePath.slice(0, -path.extname(relativePath).length)}.txt`)
));
);
}
/**
* Output Surge RULE-SET and Clash classical text format
*/
async function transformRuleset(sourcePath: string, relativePath: string) {
const res = await processFile(sourcePath);
if (!res) return;
async function transformRuleset(parentSpan: Span, sourcePath: string, relativePath: string) {
const span = parentSpan.traceChild(`transform ruleset: ${path.basename(sourcePath, path.extname(sourcePath))}`);
const res = await processFile(span, sourcePath);
if (!res) return null;
const [title, descriptions, lines] = res;
const description = [
@ -132,7 +141,8 @@ async function transformRuleset(sourcePath: string, relativePath: string) {
)
];
return Promise.all(createRuleset(
return createRuleset(
span,
title,
description,
new Date(),
@ -140,5 +150,5 @@ async function transformRuleset(sourcePath: string, relativePath: string) {
'ruleset',
path.resolve(outputSurgeDir, relativePath),
path.resolve(outputClashDir, `${relativePath.slice(0, -path.extname(relativePath).length)}.txt`)
));
);
}

6
Build/build-domestic-ruleset.ts
View File

@ -21,7 +21,7 @@ export const getDomesticDomainsRulesetPromise = createMemoizedPromise(async () =
return results;
});
export const buildDomesticRuleset = task(import.meta.path, async () => {
export const buildDomesticRuleset = task(import.meta.path, async (span) => {
const rulesetDescription = [
...SHARED_DESCRIPTION,
'',
@ -29,7 +29,8 @@ export const buildDomesticRuleset = task(import.meta.path, async () => {
];
return Promise.all([
...createRuleset(
createRuleset(
span,
'Sukka\'s Ruleset - Domestic Domains',
rulesetDescription,
new Date(),
@ -39,6 +40,7 @@ export const buildDomesticRuleset = task(import.meta.path, async () => {
path.resolve(import.meta.dir, '../Clash/non_ip/domestic.txt')
),
compareAndWriteFile(
span,
[
'#!name=[Sukka] Local DNS Mapping',
`#!desc=Last Updated: ${new Date().toISOString()}`,

3
Build/build-internal-cdn-rules.ts
View File

@ -33,7 +33,7 @@ const processLocalRuleSet = async (ruleSetPath: string, set: Set<string>, keywor
}
};
export const buildInternalCDNDomains = task(import.meta.path, async () => {
export const buildInternalCDNDomains = task(import.meta.path, async (span) => {
const proxySet = new Set<string>();
const proxyKeywords = new Set<string>();
@ -50,6 +50,7 @@ export const buildInternalCDNDomains = task(import.meta.path, async () => {
]))[0];
return compareAndWriteFile(
span,
[
...sortDomains(Array.from(proxySet), gorhill).map(i => `SUFFIX,${i}`),
...Array.from(proxyKeywords).sort().map(i => `REGEX,${i}`)

7
Build/build-microsoft-cdn.ts
View File

@ -41,7 +41,7 @@ export const getMicrosoftCdnRulesetPromise = createMemoizedPromise(async () => {
return Array.from(set).map(d => `DOMAIN-SUFFIX,${d}`).concat(WHITELIST);
});
export const buildMicrosoftCdn = task(import.meta.path, async () => {
export const buildMicrosoftCdn = task(import.meta.path, async (span) => {
const description = [
...SHARED_DESCRIPTION,
'',
@ -51,7 +51,8 @@ export const buildMicrosoftCdn = task(import.meta.path, async () => {
' - https://github.com/felixonmars/dnsmasq-china-list'
];
return Promise.all(createRuleset(
return createRuleset(
span,
'Sukka\'s Ruleset - Microsoft CDN',
description,
new Date(),
@ -59,7 +60,7 @@ export const buildMicrosoftCdn = task(import.meta.path, async () => {
'ruleset',
path.resolve(import.meta.dir, '../List/non_ip/microsoft_cdn.conf'),
path.resolve(import.meta.dir, '../Clash/non_ip/microsoft_cdn.txt')
));
);
});
if (import.meta.main) {

22
Build/build-reject-domainset.ts
View File

@ -29,20 +29,20 @@ export const buildRejectDomainSet = task(import.meta.path, async (span) => {
// Parse from AdGuard Filters
const [gorhill, shouldStop] = await span
.traceChild('download and process hosts / adblock filter rules')
.traceAsyncFn(async () => {
.traceAsyncFn(async (childSpan) => {
let shouldStop = false;
const [gorhill] = await Promise.all([
getGorhillPublicSuffixPromise(),
// Parse from remote hosts & domain lists
...HOSTS.map(entry => processHosts(span, entry[0], entry[1], entry[2]).then(hosts => {
...HOSTS.map(entry => processHosts(childSpan, entry[0], entry[1], entry[2]).then(hosts => {
SetHelpers.add(domainSets, hosts);
})),
...DOMAIN_LISTS.map(entry => processDomainLists(span, entry[0], entry[1], entry[2])),
...DOMAIN_LISTS.map(entry => processDomainLists(childSpan, entry[0], entry[1], entry[2])),
...ADGUARD_FILTERS.map(input => {
const promise = typeof input === 'string'
? processFilterRules(span, input)
: processFilterRules(span, input[0], input[1], input[2]);
? processFilterRules(childSpan, input)
: processFilterRules(childSpan, input[0], input[1], input[2]);
return promise.then(({ white, black, foundDebugDomain }) => {
if (foundDebugDomain) {
@ -56,22 +56,22 @@ export const buildRejectDomainSet = task(import.meta.path, async (span) => {
...([
'https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt',
'https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt'
].map(input => processFilterRules(span, input).then(({ white, black }) => {
].map(input => processFilterRules(childSpan, input).then(({ white, black }) => {
setAddFromArray(filterRuleWhitelistDomainSets, white);
setAddFromArray(filterRuleWhitelistDomainSets, black);
}))),
getPhishingDomains(span).then(([purePhishingDomains, fullPhishingDomainSet]) => {
getPhishingDomains(childSpan).then(([purePhishingDomains, fullPhishingDomainSet]) => {
SetHelpers.add(domainSets, fullPhishingDomainSet);
setAddFromArray(domainSets, purePhishingDomains);
}),
(async () => {
childSpan.traceChild('process reject_sukka.conf').traceAsyncFn(async () => {
for await (const l of readFileByLine(path.resolve(import.meta.dir, '../Source/domainset/reject_sukka.conf'))) {
const line = processLine(l);
if (line) {
domainSets.add(line);
}
}
})()
})
]);
// remove pre-defined enforced blacklist from whitelist
@ -187,7 +187,8 @@ export const buildRejectDomainSet = task(import.meta.path, async (span) => {
];
return Promise.all([
...createRuleset(
createRuleset(
span,
'Sukka\'s Ruleset - Reject Base',
description,
new Date(),
@ -197,6 +198,7 @@ export const buildRejectDomainSet = task(import.meta.path, async (span) => {
path.resolve(import.meta.dir, '../Clash/domainset/reject.txt')
),
compareAndWriteFile(
span,
rejectDomainsStats.map(([domain, count]) => `${domain}${' '.repeat(100 - domain.length)}${count}`),
path.resolve(import.meta.dir, '../List/internal/reject-stats.txt')
),

3
Build/build-sgmodule-always-realip.ts
View File

@ -38,8 +38,9 @@ const HOSTNAMES = [
'GC._msDCS.*.*'
] as const;
export const buildAlwaysRealIPModule = task(import.meta.path, async () => {
export const buildAlwaysRealIPModule = task(import.meta.path, async (span) => {
return compareAndWriteFile(
span,
[
'#!name=[Sukka] Always Real IP Plus',
`#!desc=Last Updated: ${new Date().toISOString()}`,

3
Build/build-sgmodule-redirect.ts
View File

@ -71,10 +71,11 @@ const REDIRECT = [
['googleajax.wp-china-yes.net/', 'https://ajax.googleapis.com/']
] as const;
export const buildRedirectModule = task(import.meta.path, async () => {
export const buildRedirectModule = task(import.meta.path, async (span) => {
const domains = Array.from(new Set(REDIRECT.map(([from]) => tldts.getHostname(from, { detectIp: false })))).filter(Boolean);
return compareAndWriteFile(
span,
[
'#!name=[Sukka] URL Redirect',
`#!desc=Last Updated: ${new Date().toISOString()}`,

18
Build/build-speedtest-domainset.ts
View File

@ -151,13 +151,14 @@ export const buildSpeedtestDomainSet = task(import.meta.path, async (span) => {
'.backend.librespeed.org'
]);
// Download previous speedtest domainset
for await (const l of await fetchRemoteTextByLine('https://ruleset.skk.moe/List/domainset/speedtest.conf')) {
const line = processLine(l);
if (line) {
domains.add(line);
await span.traceChild('fetch previous speedtest domainset').traceAsyncFn(async () => {
for await (const l of await fetchRemoteTextByLine('https://ruleset.skk.moe/List/domainset/speedtest.conf')) {
const line = processLine(l);
if (line) {
domains.add(line);
}
}
}
});
await new Promise<void>((resolve) => {
const pMap = ([
@ -225,7 +226,8 @@ export const buildSpeedtestDomainSet = task(import.meta.path, async (span) => {
'This file contains common speedtest endpoints.'
];
return Promise.all(createRuleset(
return createRuleset(
span,
'Sukka\'s Ruleset - Speedtest Domains',
description,
new Date(),
@ -233,7 +235,7 @@ export const buildSpeedtestDomainSet = task(import.meta.path, async (span) => {
'domainset',
path.resolve(import.meta.dir, '../List/domainset/speedtest.conf'),
path.resolve(import.meta.dir, '../Clash/domainset/speedtest.txt')
));
);
});
if (import.meta.main) {

3
Build/build-sspanel-appprofile.ts
View File

@ -26,7 +26,7 @@ const removeNoResolved = (line: string) => line.replace(',no-resolve', '');
/**
* This only generates a simplified version, for under-used users only.
*/
export const buildSSPanelUIMAppProfile = task(import.meta.path, async () => {
export const buildSSPanelUIMAppProfile = task(import.meta.path, async (span) => {
const [
domesticDomains,
appleCdnDomains,
@ -108,6 +108,7 @@ export const buildSSPanelUIMAppProfile = task(import.meta.path, async () => {
);
await compareAndWriteFile(
span,
output,
path.resolve(import.meta.dir, '../List/internal/appprofile.php')
);

26
Build/build-stream-service.ts
View File

@ -1,4 +1,5 @@
// @ts-check
import type { Span } from './trace';
import { task } from './trace';
import path from 'path';
@ -7,10 +8,12 @@ import { createRuleset } from './lib/create-file';
import { ALL, NORTH_AMERICA, EU, HK, TW, JP, KR } from '../Source/stream';
import { SHARED_DESCRIPTION } from './lib/constants';
export const createRulesetForStreamService = (fileId: string, title: string, streamServices: Array<import('../Source/stream').StreamService>) => {
export const createRulesetForStreamService = (span: Span, fileId: string, title: string, streamServices: Array<import('../Source/stream').StreamService>) => {
const childSpan = span.traceChild(fileId);
return [
// Domains
...createRuleset(
createRuleset(
childSpan,
`Sukka's Ruleset - Stream Services: ${title}`,
[
...SHARED_DESCRIPTION,
@ -24,7 +27,8 @@ export const createRulesetForStreamService = (fileId: string, title: string, str
path.resolve(import.meta.dir, `../Clash/non_ip/${fileId}.txt`)
),
// IP
...createRuleset(
createRuleset(
childSpan,
`Sukka's Ruleset - Stream Services' IPs: ${title}`,
[
...SHARED_DESCRIPTION,
@ -47,16 +51,16 @@ export const createRulesetForStreamService = (fileId: string, title: string, str
];
};
export const buildStreamService = task(import.meta.path, async () => {
export const buildStreamService = task(import.meta.path, async (span) => {
return Promise.all([
...createRulesetForStreamService('stream', 'All', ALL),
...createRulesetForStreamService('stream_us', 'North America', NORTH_AMERICA),
...createRulesetForStreamService('stream_eu', 'Europe', EU),
...createRulesetForStreamService('stream_hk', 'Hong Kong', HK),
...createRulesetForStreamService('stream_tw', 'Taiwan', TW),
...createRulesetForStreamService('stream_jp', 'Japan', JP),
...createRulesetForStreamService(span, 'stream', 'All', ALL),
...createRulesetForStreamService(span, 'stream_us', 'North America', NORTH_AMERICA),
...createRulesetForStreamService(span, 'stream_eu', 'Europe', EU),
...createRulesetForStreamService(span, 'stream_hk', 'Hong Kong', HK),
...createRulesetForStreamService(span, 'stream_tw', 'Taiwan', TW),
...createRulesetForStreamService(span, 'stream_jp', 'Japan', JP),
// ...createRulesetForStreamService('stream_au', 'Oceania', AU),
...createRulesetForStreamService('stream_kr', 'Korean', KR)
...createRulesetForStreamService(span, 'stream_kr', 'Korean', KR)
// ...createRulesetForStreamService('stream_south_east_asia', 'South East Asia', SOUTH_EAST_ASIA)
]);
});

7
Build/build-telegram-cidr.ts
View File

@ -32,7 +32,7 @@ export const getTelegramCIDRPromise = createMemoizedPromise(async () => {
return { date, results };
});
export const buildTelegramCIDR = task(import.meta.path, async () => {
export const buildTelegramCIDR = task(import.meta.path, async (span) => {
const { date, results } = await getTelegramCIDRPromise();
if (results.length === 0) {
@ -45,7 +45,8 @@ export const buildTelegramCIDR = task(import.meta.path, async () => {
' - https://core.telegram.org/resources/cidr.txt'
];
return Promise.all(createRuleset(
return createRuleset(
span,
'Sukka\'s Ruleset - Telegram IP CIDR',
description,
date,
@ -53,7 +54,7 @@ export const buildTelegramCIDR = task(import.meta.path, async () => {
'ruleset',
path.resolve(import.meta.dir, '../List/ip/telegram.conf'),
path.resolve(import.meta.dir, '../Clash/ip/telegram.txt')
));
);
});
if (import.meta.main) {

14
Build/download-mock-assets.ts
View File

@ -12,15 +12,11 @@ const ASSETS_LIST = {
const mockDir = path.resolve(import.meta.dir, '../Mock');
export const downloadMockAssets = task(import.meta.path, () => Promise.all(Object.entries(ASSETS_LIST).map(async ([filename, url]) => {
const targetPath = path.join(mockDir, filename);
const key = picocolors.gray(`Download ${filename}`);
console.time(key);
const res = await fetchWithRetry(url);
await Bun.write(targetPath, res);
console.timeEnd(key);
})));
export const downloadMockAssets = task(import.meta.path, (span) => Promise.all(Object.entries(ASSETS_LIST).map(
([filename, url]) => span
.traceChild(url)
.traceAsyncFn(() => fetchWithRetry(url).then(res => Bun.write(path.join(mockDir, filename), res)))
)));
if (import.meta.main) {
downloadMockAssets();

103
Build/lib/create-file.ts
View File

@ -3,8 +3,10 @@ import { readFileByLine } from './fetch-text-by-line';
import { surgeDomainsetToClashDomainset, surgeRulesetToClashClassicalTextRuleset } from './clash';
import { traceAsync } from './trace-runner';
import picocolors from 'picocolors';
import type { Span } from '../trace';
import path from 'path';
export async function compareAndWriteFile(linesA: string[], filePath: string) {
export async function compareAndWriteFile(span: Span, linesA: string[], filePath: string) {
let isEqual = true;
const file = Bun.file(filePath);
@ -17,48 +19,44 @@ export async function compareAndWriteFile(linesA: string[], filePath: string) {
console.log(`Nothing to write to ${filePath}...`);
isEqual = false;
} else {
isEqual = await traceAsync(
picocolors.gray(`comparing ${filePath}`),
async () => {
let index = 0;
isEqual = await span.traceChild(`comparing ${filePath}`).traceAsyncFn(async () => {
let index = 0;
for await (const lineB of readFileByLine(file)) {
const lineA = linesA[index];
index++;
for await (const lineB of readFileByLine(file)) {
const lineA = linesA[index];
index++;
if (lineA == null) {
// The file becomes smaller
return false;
}
if (lineA == null) {
// The file becomes smaller
return false;
}
if (lineA[0] === '#' && lineB[0] === '#') {
continue;
}
if (
lineA[0] === '/'
if (lineA[0] === '#' && lineB[0] === '#') {
continue;
}
if (
lineA[0] === '/'
&& lineA[1] === '/'
&& lineA[3] === '#'
&& lineB[0] === '/'
&& lineB[1] === '/'
&& lineB[3] === '#'
) {
continue;
}
if (lineA !== lineB) {
return false;
}
) {
continue;
}
if (index !== linesALen) {
// The file becomes larger
if (lineA !== lineB) {
return false;
}
}
return true;
},
picocolors.gray
);
if (index !== linesALen) {
// The file becomes larger
return false;
}
return true;
});
}
if (isEqual) {
@ -66,7 +64,7 @@ export async function compareAndWriteFile(linesA: string[], filePath: string) {
return;
}
await traceAsync(picocolors.gray(`writing ${filePath}`), async () => {
await span.traceChild(`writing ${filePath}`).traceAsyncFn(async () => {
if (linesALen < 10000) {
return Bun.write(file, `${linesA.join('\n')}\n`);
}
@ -79,7 +77,7 @@ export async function compareAndWriteFile(linesA: string[], filePath: string) {
}
return writer.end();
}, picocolors.gray);
});
}
export const withBannerArray = (title: string, description: string[] | readonly string[], date: Date, content: string[]) => {
@ -96,27 +94,28 @@ export const withBannerArray = (title: string, description: string[] | readonly
};
export const createRuleset = (
parentSpan: Span,
title: string, description: string[] | readonly string[], date: Date, content: string[],
type: 'ruleset' | 'domainset', surgePath: string, clashPath: string
) => {
) => parentSpan.traceChild(`create ruleset: ${path.basename(surgePath, path.extname(surgePath))}`).traceAsyncFn((childSpan) => {
const surgeContent = withBannerArray(title, description, date, content);
const clashContent = childSpan.traceChild('convert incoming ruleset to clash').traceSyncFn(() => {
let _clashContent;
switch (type) {
case 'domainset':
_clashContent = surgeDomainsetToClashDomainset(content);
break;
case 'ruleset':
_clashContent = surgeRulesetToClashClassicalTextRuleset(content);
break;
default:
throw new TypeError(`Unknown type: ${type as any}`);
}
return withBannerArray(title, description, date, _clashContent);
});
let _clashContent;
switch (type) {
case 'domainset':
_clashContent = surgeDomainsetToClashDomainset(content);
break;
case 'ruleset':
_clashContent = surgeRulesetToClashClassicalTextRuleset(content);
break;
default:
throw new TypeError(`Unknown type: ${type as any}`);
}
const clashContent = withBannerArray(title, description, date, _clashContent);
return [
compareAndWriteFile(surgeContent, surgePath),
compareAndWriteFile(clashContent, clashPath)
];
};
return Promise.all([
compareAndWriteFile(childSpan, surgeContent, surgePath),
compareAndWriteFile(childSpan, clashContent, clashPath)
]);
});

5
Build/lib/fetch-text-by-line.ts
View File

@ -41,17 +41,16 @@ export function readFileByLine(file: string | URL | BunFile) {
return createTextLineAsyncGeneratorFromStreamSource(file.stream());
}
export function createReadlineInterfaceFromResponse(resp: Response) {
export function createReadlineInterfaceFromResponse(this: void, resp: Response) {
if (!resp.body) {
throw new Error('Failed to fetch remote text');
}
if (resp.bodyUsed) {
throw new Error('Body has already been consumed.');
}
return createTextLineAsyncGeneratorFromStreamSource(resp.body);
}
export function fetchRemoteTextByLine(url: string | URL) {
return fetchWithRetry(url, defaultRequestInit).then(res => createReadlineInterfaceFromResponse(res));
return fetchWithRetry(url, defaultRequestInit).then(createReadlineInterfaceFromResponse);
}

23
Build/lib/parse-filter.ts
View File

@ -96,12 +96,12 @@ const enum ParseType {
}
export async function processFilterRules(
span: Span,
parentSpan: Span,
filterRulesUrl: string,
fallbackUrls?: readonly string[] | undefined | null,
ttl: number | null = null
): Promise<{ white: string[], black: string[], foundDebugDomain: boolean }> {
const [white, black, warningMessages] = await span.traceChild('process filter rules: domainListsUrl').traceAsyncFn(() => fsCache.apply<Readonly<[
const [white, black, warningMessages] = await parentSpan.traceChild(`process filter rules: ${filterRulesUrl}`).traceAsyncFn((span) => fsCache.apply<Readonly<[
white: string[],
black: string[],
warningMessages: string[]
@ -179,18 +179,15 @@ export async function processFilterRules(
// Avoid event loop starvation, so we wait for a macrotask before we start fetching.
await Promise.resolve();
const filterRules = (await traceAsync(
picocolors.gray(`- download ${filterRulesUrl}`),
() => fetchAssets(filterRulesUrl, fallbackUrls),
picocolors.gray
)).split('\n');
const filterRules = await span.traceChild('download adguard filter').traceAsyncFn(() => {
return fetchAssets(filterRulesUrl, fallbackUrls).then(text => text.split('\n'));
});
const key = picocolors.gray(`- parse adguard filter ${filterRulesUrl}`);
console.time(key);
for (let i = 0, len = filterRules.length; i < len; i++) {
lineCb(filterRules[i]);
}
console.timeEnd(key);
span.traceChild('parse adguard filter').traceSyncFn(() => {
for (let i = 0, len = filterRules.length; i < len; i++) {
lineCb(filterRules[i]);
}
});
}
return [

6
Build/lib/reject-data-source.ts
View File

@ -47,7 +47,9 @@ export const DOMAIN_LISTS = [
['https://raw.githubusercontent.com/AdguardTeam/cname-trackers/master/data/combined_disguised_mail_trackers_justdomains.txt', true, TTL.THREE_DAYS()]
] as const;
export const ADGUARD_FILTERS = [
type AdGuardFilterSource = string | [main: string, mirrors: string[] | null, ttl: number];
export const ADGUARD_FILTERS: AdGuardFilterSource[] = [
// EasyList
[
'https://easylist.to/easylist/easylist.txt',
@ -156,7 +158,7 @@ export const ADGUARD_FILTERS = [
// Not actively maintained, let's use a 10 days cache ttl
['https://raw.githubusercontent.com/Spam404/lists/master/adblock-list.txt', null, TTL.TEN_DAYS()],
// Brave First Party & First Party CNAME
'https://raw.githubusercontent.com/brave/adblock-lists/master/brave-lists/brave-firstparty.txt'
['https://raw.githubusercontent.com/brave/adblock-lists/master/brave-lists/brave-firstparty.txt', null, TTL.ONE_DAY()]
] as const;
export const PREDEFINED_WHITELIST = [

46
Build/trace/index.ts
View File

@ -114,17 +114,45 @@ export const universalify = <A extends any[], R>(taskname: string, fn: (this: vo
};
};
export const printTraceResult = (traceResult: TraceResult = rootTraceResult, level = 0, isLast = false) => {
if (level === 0) {
printStats(traceResult.children);
export const printTraceResult = (traceResult: TraceResult = rootTraceResult) => {
printStats(traceResult.children);
printTree(traceResult, node => `${node.name} ${picocolors.bold(`${(node.end - node.start).toFixed(3)}ms`)}`);
};
function printTree(initialTree: TraceResult, printNode: (node: TraceResult, branch: string) => string) {
function printBranch(tree: TraceResult, branch: string) {
const isGraphHead = branch.length === 0;
const children = tree.children;
let branchHead = '';
if (!isGraphHead) {
branchHead = children.length > 0 ? '┬ ' : '─ ';
}
const toPrint = printNode(tree, `${branch}${branchHead}`);
if (typeof toPrint === 'string') {
console.log(`${branch}${branchHead}${toPrint}`);
}
let baseBranch = branch;
if (!isGraphHead) {
const isChildOfLastBranch = branch.endsWith('└─');
baseBranch = branch.slice(0, -2) + (isChildOfLastBranch ? ' ' : '│ ');
}
const nextBranch = `${baseBranch}├─`;
const lastBranch = `${baseBranch}└─`;
children.forEach((child, index) => {
printBranch(child, children.length - 1 === index ? lastBranch : nextBranch);
});
}
const prefix = (level > 0 ? ` ${'│ '.repeat(level - 1)}` : '') + (level > 0 ? (isLast ? '└─' : '├─') : '');
console.log(`${prefix} ${traceResult.name} ${picocolors.bold(`${(traceResult.end - traceResult.start).toFixed(2)}ms`)}`);
traceResult.children.forEach((child, index, arr) => printTraceResult(child, level + 1, index === arr.length - 1));
};
printBranch(initialTree, '');
}
function printStats(stats: TraceResult[]): void {
stats.sort((a, b) => a.start - b.start);

1
Source/non_ip/ai.conf
View File

@ -10,3 +10,4 @@ DOMAIN-SUFFIX,openaiapi-site.azureedge.net
DOMAIN-SUFFIX,perplexity.ai
DOMAIN-SUFFIX,anthropic.com
DOMAIN-SUFFIX,claude.ai
DOMAIN-SUFFIX,generativelanguage.googleapis.com