GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2026-09-12 10:34:35 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Chore: better stat tracking

Browse Source
This commit is contained in:
SukkaW
2026-09-02 15:23:12 +08:00
parent 3c5c5ac4bd
commit 3e03018bbe
28 changed files with 933 additions and 187 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
Build/lib/create-file.ts
View File

@@ -2,6 +2,7 @@ import { fastStringArrayJoin } from 'foxts/fast-string-array-join';
import fs from 'node:fs';
import path from 'node:path';
import picocolors from 'picocolors';
import { SpanCategory } from '../trace';
import type { Span } from '../trace';
import { readFileByLine } from './fetch-text-by-line';
import { writeFile } from './misc';
@@ -37,7 +38,7 @@ async function isPreviousOutputEqual(span: Span, linesA: string[], filePath: str
}
return fileEqual(linesA, readFileByLine(filePath));
});
}, SpanCategory.FsRead);
if (isEqual) {
console.log(picocolors.gray(picocolors.dim(`same content, bail out writing: ${filePath}`)));
@@ -81,7 +82,8 @@ export async function compareAndWriteFileInWorker(span: Span, linesA: string[],
export function writeFileLines(span: Span, linesA: string[], filePath: string): Promise<void> {
return span.traceChildAsync<void>(
`writing ${filePath}`,
() => writeFile(filePath, fastStringArrayJoin(linesA, '\n') + '\n')
() => writeFile(filePath, fastStringArrayJoin(linesA, '\n') + '\n'),
SpanCategory.FsWrite
);
}
@@ -89,5 +91,5 @@ export function writeFileLinesSync(span: Span, linesA: string[], filePath: strin
span.traceChildSync(`writing ${filePath}`, () => {
fs.mkdirSync(path.dirname(filePath), { recursive: true });
fs.writeFileSync(filePath, fastStringArrayJoin(linesA, '\n') + '\n');
});
}, SpanCategory.FsWrite);
}

13
Build/lib/fetch-telegram-backup-endpoints.ts
View File

@@ -5,6 +5,7 @@ import picocolors from 'picocolors';
import { fastStringArrayJoin } from 'foxts/fast-string-array-join';
import { stableHash } from 'stable-hash';
import { SpanCategory } from '../trace';
import type { Span } from '../trace';
import { $$fetch } from './fetch-retry';
import { getTelegramBackupIPFromBase64 } from './get-telegram-backup-ip';
@@ -37,7 +38,7 @@ async function fetchDnsEndpoints(span: Span, domain: string, traceName: string)
return Object.assign(resolver, { server: ip });
});
await span.traceChildAsync(traceName, () => Promise.all(resolvers.map(async (resolver) => {
await span.traceChild(traceName, SpanCategory.Network).traceAsyncFn(() => Promise.all(resolvers.map(async (resolver) => {
try {
const response = await resolver.resolveTxt(domain);
const strings = response.map(result => fastStringArrayJoin(result, ''));
@@ -61,7 +62,7 @@ async function fetchDnsEndpoints(span: Span, domain: string, traceName: string)
}
async function fetchRealtimeDatabaseEndpoints(span: Span) {
return span.traceChildAsync('backup source 2: Firebase Realtime DB', async () => {
return span.traceChild('backup source 2: Firebase Realtime DB', SpanCategory.Network).traceAsyncFn(async () => {
try {
const data = await (await $$fetch('https://reserve-5a846.firebaseio.com/ipconfigv3.json')).json();
if (typeof data !== 'string' || data.length !== 344) {
@@ -79,7 +80,7 @@ async function fetchRealtimeDatabaseEndpoints(span: Span) {
}
async function fetchValueStoreEndpoints(span: Span) {
return span.traceChildAsync('backup source 3: Firebase Value Store', async () => {
return span.traceChild('backup source 3: Firebase Value Store', SpanCategory.Network).traceAsyncFn(async () => {
try {
const json = await (await $$fetch('https://firestore.googleapis.com/v1/projects/reserve-5a846/databases/(default)/documents/ipconfig/v3', {
headers: {
@@ -110,7 +111,7 @@ async function fetchValueStoreEndpoints(span: Span) {
}
async function fetchAppEngineEndpoints(span: Span, url: string, traceName: string) {
return span.traceChildAsync(traceName, async () => {
return span.traceChild(traceName, SpanCategory.Network).traceAsyncFn(async () => {
try {
const data = (await (await $$fetch(url)).text()).trim();
if (data.length !== 344) {
@@ -147,7 +148,7 @@ async function fetchTestEndpoints(span: Span) {
function getProductionEndpoints(span: Span) {
if (productionEndpointsPromise) {
return span.traceChildAsync('reuse production backup endpoints', () => productionEndpointsPromise!);
return span.traceChildAsync('reuse production backup endpoints', () => productionEndpointsPromise!, SpanCategory.Wait);
}
productionEndpointsPromise = fetchProductionEndpoints(span);
return productionEndpointsPromise;
@@ -155,7 +156,7 @@ function getProductionEndpoints(span: Span) {
function getTestEndpoints(span: Span) {
if (testEndpointsPromise) {
return span.traceChildAsync('reuse test backup endpoints', () => testEndpointsPromise!);
return span.traceChildAsync('reuse test backup endpoints', () => testEndpointsPromise!, SpanCategory.Wait);
}
testEndpointsPromise = fetchTestEndpoints(span);
return testEndpointsPromise;

4
Build/lib/get-phishing-domains.ts
View File

@@ -2,7 +2,7 @@ import picocolors from 'picocolors';
import { parse } from 'tldts-experimental';
import { appendArrayInPlaceCurried } from 'foxts/append-array-in-place';
import { workerJob } from '../trace';
import { SpanCategory, workerJob } from '../trace';
import type { RawSpan, WorkerJobResult } from '../trace';
import type { TldTsParsed } from './normalize-domain';
@@ -25,7 +25,7 @@ export function getPhishingDomains(rawSpan?: RawSpan, isDebug = false): Promise<
const domainGroups = await Promise.all(downloads.map(task => task(childSpan)));
return childSpan.traceChildSync<string[]>('calculate and handling mass phishing domains', () => {
return childSpan.traceChild('calculate and handling mass phishing domains', SpanCategory.Compute).traceSyncFn<string[]>(() => {
const domainArr: string[] = [];
domainGroups.forEach(appendArrayInPlaceCurried(domainArr));

5
Build/lib/parse-filter/domainlists.ts
View File

@@ -1,6 +1,7 @@
import { fastNormalizeDomain, fastNormalizeDomainWithoutWww } from '../normalize-domain';
import { onBlackFound } from './shared';
import { fetchAssets } from '../fetch-assets';
import { SpanCategory } from '../../trace';
import type { Span } from '../../trace';
function domainListLineCb(line: string, set: string[], meta: string, normalizeDomain = fastNormalizeDomain) {
@@ -29,14 +30,14 @@ export function processDomainListsWithPreload(
const lineCb = includeAllSubDomain ? domainListLineCbIncludeAllSubdomain : domainListLineCb;
return (span: Span) => span.traceChildAsync(`process domainlist: ${domainListsUrl}`, async (childSpan) => {
const filterRules = await childSpan.traceChildPromise('download', downloadPromise);
const filterRules = await childSpan.traceChildPromise('download', downloadPromise, SpanCategory.Network);
const domainSets: string[] = [];
childSpan.traceChildSync('parse domain list', () => {
for (let i = 0, len = filterRules.length; i < len; i++) {
lineCb(filterRules[i], domainSets, domainListsUrl, fastNormalizeDomainWithoutWww);
}
});
}, SpanCategory.Compute);
return domainSets;
});

5
Build/lib/parse-filter/filters.ts
View File

@@ -1,4 +1,5 @@
import picocolors from 'picocolors';
import { SpanCategory } from '../../trace';
import type { Span } from '../../trace';
import { fetchAssets } from '../fetch-assets';
import { onBlackFound, onWhiteFound } from './shared';
@@ -49,7 +50,7 @@ export function processFilterRulesWithPreload(
filterRulesUrl: string
}
>(`process filter rules: ${filterRulesUrl}`, async (span) => {
const filterRules = await span.traceChildPromise('download', downloadPromise);
const filterRules = await span.traceChildPromise('download', downloadPromise, SpanCategory.Network);
const whiteDomains = new Set<string>();
const whiteDomainSuffixes = new Set<string>();
@@ -121,7 +122,7 @@ export function processFilterRulesWithPreload(
}
};
span.traceChild('parse adguard filter').traceSyncFn(() => {
span.traceChild('parse adguard filter', SpanCategory.Compute).traceSyncFn(() => {
for (let i = 0, len = filterRules.length; i < len; i++) {
lineCb(filterRules[i]);
}

9
Build/lib/parse-filter/hosts.ts
View File

@@ -1,3 +1,4 @@
import { SpanCategory } from '../../trace';
import type { Span } from '../../trace';
import { fetchAssets } from '../fetch-assets';
import { fastNormalizeDomainWithoutWww } from '../normalize-domain';
@@ -42,11 +43,11 @@ export function processHosts(
const cb = includeAllSubDomain ? hostsLineCbIncludeAllSubdomain : hostsLineCb;
return span.traceChildAsync(`process hosts: ${hostsUrl}`, async (span) => {
const filterRules = await span.traceChild('download').traceAsyncFn(() => fetchAssets(hostsUrl, mirrors, true));
const filterRules = await span.traceChild('download', SpanCategory.Network).traceAsyncFn(() => fetchAssets(hostsUrl, mirrors, true));
const domainSets: string[] = [];
span.traceChild('parse hosts').traceSyncFn(() => {
span.traceChild('parse hosts', SpanCategory.Compute).traceSyncFn(() => {
for (let i = 0, len = filterRules.length; i < len; i++) {
cb(filterRules[i], domainSets, hostsUrl);
}
@@ -61,11 +62,11 @@ export function processHostsWithPreload(hostsUrl: string, mirrors: string[] | nu
const cb = includeAllSubDomain ? hostsLineCbIncludeAllSubdomain : hostsLineCb;
return (span: Span) => span.traceChildAsync(`process hosts: ${hostsUrl}`, async (span) => {
const filterRules = await span.traceChild('download').tracePromise(downloadPromise);
const filterRules = await span.traceChild('download', SpanCategory.Network).tracePromise(downloadPromise);
const domainSets: string[] = [];
span.traceChild('parse hosts').traceSyncFn(() => {
span.traceChild('parse hosts', SpanCategory.Compute).traceSyncFn(() => {
for (let i = 0, len = filterRules.length; i < len; i++) {
cb(filterRules[i], domainSets, hostsUrl);
}

23
Build/lib/rules/base.ts
View File

@@ -1,3 +1,4 @@
import { SpanCategory } from '../../trace';
import type { Span } from '../../trace';
import { HostnameSmolTrie } from 'hntrie/smol';
import { not, nullthrow } from 'foxts/guard';
@@ -68,10 +69,15 @@ export class FileOutput {
return this;
};
protected readonly span: Span;
/**
* The `RuleOutput#id` span is only opened by write(): between construction and
* write() this object merely accumulates sources, and that time already belongs
* to the sibling spans doing the downloading / reading.
*/
protected readonly parentSpan: Span;
constructor($span: Span, protected readonly id: string) {
this.span = $span.traceChild('RuleOutput#' + id);
this.parentSpan = $span;
}
protected title: string | null = null;
@@ -463,10 +469,12 @@ export class FileOutput {
}
write(): Promise<unknown> {
return this.span.traceChildAsync('write all', async (childSpan) => {
await childSpan.traceChildAsync('done', () => this.done());
return this.parentSpan.traceChildAsync('RuleOutput#' + this.id, async (childSpan) => {
// pendingPromise is the (untraced) reading + parsing of every source added
// via addFromRuleset / addFromDomainset, so this is waiting on fs + compute
await childSpan.traceChildAsync('done', () => this.done(), SpanCategory.Wait);
const domains = childSpan.traceChildSync('dump domain trie', () => this.dumpDomains());
const domains = childSpan.traceChildSync('dump domain trie', () => this.dumpDomains(), SpanCategory.Compute);
const title = nullthrow(this.title, 'Missing title');
const descriptions = nullthrow(this.description, 'Missing description');
@@ -505,7 +513,7 @@ export class FileOutput {
);
}
childSpan.traceChildSync('write to strategies', () => this.writeToStrategies(domains));
childSpan.traceChildSync('write to strategies', () => this.writeToStrategies(domains), SpanCategory.Compute);
return childSpan.traceChildAsync('output to disk', (childSpan) => {
const promises: Array<Promise<void>> = [];
@@ -521,7 +529,8 @@ export class FileOutput {
isMainThread
? strategy.output(childSpan, title, descriptions, this.date, filePath)
: strategy.outputInWorker(childSpan, title, descriptions, this.date, filePath)
))
// self time here is banner + content hash; compare / writing are traced as children
), SpanCategory.Compute)
);
}

8
Build/lib/rules/output.worker.ts
View File

@@ -1,4 +1,4 @@
import { workerJob } from '../../trace';
import { SpanCategory, workerJob } from '../../trace';
import type { RawSpan, WorkerJobResult } from '../../trace';
import { resolveStrategyOutputPath, reviveStrategy, writeDataToStrategies } from './strategy-write-data';
import type { OutputWorkerPayload } from './strategy-write-data';
@@ -15,7 +15,7 @@ export function writeOutput(rawSpan: RawSpan | undefined, payload: OutputWorkerP
return workerJob(rawSpan, (span) => {
const strategies = payload.strategies.map(reviveStrategy);
span.traceChildSync('write to strategies', () => writeDataToStrategies(payload.data, strategies));
span.traceChildSync('write to strategies', () => writeDataToStrategies(payload.data, strategies), SpanCategory.Compute);
const date = new Date(payload.dateMs);
@@ -29,7 +29,9 @@ export function writeOutput(rawSpan: RawSpan | undefined, payload: OutputWorkerP
// eslint-disable-next-line no-await-in-loop -- see above
await childSpan.traceChildAsync(
'write ' + strategy.name,
(strategySpan) => strategy.outputInWorker(strategySpan, payload.title, payload.description, date, filePath)
(strategySpan) => strategy.outputInWorker(strategySpan, payload.title, payload.description, date, filePath),
// self time here is banner + content hash; compare / writing are traced as children
SpanCategory.Compute
);
}
});