GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2025-12-12 01:00:34 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Finish fs cache changes / tracer optimization

Browse Source
This commit is contained in:
SukkaW 2024-02-02 11:21:16 +08:00
parent 8428b3da42
commit efa1ab254e
14 changed files with 77 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Build/build-anti-bogus-domain.ts
View File

@ -37,7 +37,7 @@ export const buildAntiBogusDomain = task(import.meta.path, async (span) => {
const peeked = Bun.peek(getBogusNxDomainIPsPromise);
const bogusNxDomainIPs = peeked === getBogusNxDomainIPsPromise
? await span.traceChild('get bogus nxdomain ips').traceAsyncFn(() => getBogusNxDomainIPsPromise)
? await span.traceChildPromise('get bogus nxdomain ips', getBogusNxDomainIPsPromise)
: (peeked as string[]);
result.push(...bogusNxDomainIPs);

2
Build/build-apple-cdn.ts
View File

@ -21,7 +21,7 @@ export const buildAppleCdn = task(import.meta.path, async (span) => {
const promise = getAppleCdnDomainsPromise();
const peeked = Bun.peek(promise);
const res: string[] = peeked === promise
? await span.traceChild('get apple cdn domains').traceAsyncFn(() => promise)
? await span.traceChildPromise('get apple cdn domains', promise)
: (peeked as string[]);
const description = [

2
Build/build-chn-cidr.ts
View File

@ -32,7 +32,7 @@ export const buildChnCidr = task(import.meta.path, async (span) => {
const cidrPromise = getChnCidrPromise();
const peeked = Bun.peek(cidrPromise);
const filteredCidr: string[] = peeked === cidrPromise
? await span.traceChild('download chnroutes2').tracePromise(cidrPromise)
? await span.traceChildPromise('download chnroutes2', cidrPromise)
: (peeked as string[]);
// Can not use SHARED_DESCRIPTION here as different license

54
Build/build-common.ts
View File

@ -55,7 +55,7 @@ if (import.meta.main) {
const processFile = (span: Span, sourcePath: string) => {
// console.log('Processing', sourcePath);
return span.traceChild(`process file: ${sourcePath}`).traceAsyncFn(async () => {
return span.traceChildAsync(`process file: ${sourcePath}`, async () => {
const lines: string[] = [];
let title = '';
@ -93,34 +93,36 @@ const processFile = (span: Span, sourcePath: string) => {
function transformDomainset(parentSpan: Span, sourcePath: string, relativePath: string) {
return parentSpan
.traceChild(`transform domainset: ${path.basename(sourcePath, path.extname(sourcePath))}`)
.traceAsyncFn(async (span) => {
const res = await processFile(span, sourcePath);
if (!res) return;
.traceChildAsync(
`transform domainset: ${path.basename(sourcePath, path.extname(sourcePath))}`,
async (span) => {
const res = await processFile(span, sourcePath);
if (!res) return;
const [title, descriptions, lines] = res;
const [title, descriptions, lines] = res;
const deduped = domainDeduper(lines);
const description = [
...SHARED_DESCRIPTION,
...(
descriptions.length
? ['', ...descriptions]
: []
)
];
const deduped = domainDeduper(lines);
const description = [
...SHARED_DESCRIPTION,
...(
descriptions.length
? ['', ...descriptions]
: []
)
];
return createRuleset(
span,
title,
description,
new Date(),
deduped,
'domainset',
path.resolve(outputSurgeDir, relativePath),
path.resolve(outputClashDir, `${relativePath.slice(0, -path.extname(relativePath).length)}.txt`)
);
});
return createRuleset(
span,
title,
description,
new Date(),
deduped,
'domainset',
path.resolve(outputSurgeDir, relativePath),
path.resolve(outputClashDir, `${relativePath.slice(0, -path.extname(relativePath).length)}.txt`)
);
}
);
}
/**

2
Build/build-microsoft-cdn.ts
View File

@ -56,7 +56,7 @@ export const buildMicrosoftCdn = task(import.meta.path, async (span) => {
const promise = getMicrosoftCdnRulesetPromise();
const peeked = Bun.peek(promise);
const res: string[] = peeked === promise
? await span.traceChild('get microsoft cdn domains').tracePromise(promise)
? await span.traceChildPromise('get microsoft cdn domains', promise)
: (peeked as string[]);
return createRuleset(

14
Build/build-reject-domainset.ts
View File

@ -68,7 +68,7 @@ export const buildRejectDomainSet = task(import.meta.path, async (span) => {
SetHelpers.add(domainSets, fullPhishingDomainSet);
setAddFromArray(domainSets, purePhishingDomains);
}),
childSpan.traceChild('process reject_sukka.conf').traceAsyncFn(async () => {
childSpan.traceChildAsync('process reject_sukka.conf', async () => {
setAddFromArray(domainSets, await readFileIntoProcessedArray(path.resolve(import.meta.dir, '../Source/domainset/reject_sukka.conf')));
})
]);
@ -84,13 +84,13 @@ export const buildRejectDomainSet = task(import.meta.path, async (span) => {
console.log(`Import ${previousSize} rules from Hosts / AdBlock Filter Rules & reject_sukka.conf!`);
// Dedupe domainSets
await span.traceChild('dedupe from black keywords/suffixes').traceAsyncFn(async (childSpan) => {
await span.traceChildAsync('dedupe from black keywords/suffixes', async (childSpan) => {
/** Collect DOMAIN-SUFFIX from non_ip/reject.conf for deduplication */
const domainSuffixSet = new Set<string>();
/** Collect DOMAIN-KEYWORD from non_ip/reject.conf for deduplication */
const domainKeywordsSet = new Set<string>();
await childSpan.traceChild('collect keywords/suffixes').traceAsyncFn(async () => {
await childSpan.traceChildAsync('collect keywords/suffixes', async () => {
for await (const line of readFileByLine(path.resolve(import.meta.dir, '../Source/non_ip/reject.conf'))) {
const [type, value] = line.split(',');
@ -106,7 +106,7 @@ export const buildRejectDomainSet = task(import.meta.path, async (span) => {
SetHelpers.subtract(domainSets, domainSuffixSet);
SetHelpers.subtract(domainSets, filterRuleWhitelistDomainSets);
childSpan.traceChild('dedupe from white/suffixes').traceSyncFn(() => {
childSpan.traceChildSync('dedupe from white/suffixes', () => {
const trie = createTrie(domainSets);
domainSuffixSet.forEach(suffix => {
@ -126,7 +126,7 @@ export const buildRejectDomainSet = task(import.meta.path, async (span) => {
});
});
childSpan.traceChild('dedupe from black keywords').traceSyncFn(() => {
childSpan.traceChildSync('dedupe from black keywords', () => {
const kwfilter = createKeywordFilter(domainKeywordsSet);
for (const domain of domainSets) {
@ -142,7 +142,7 @@ export const buildRejectDomainSet = task(import.meta.path, async (span) => {
previousSize = domainSets.size;
// Dedupe domainSets
const dudupedDominArray = span.traceChild('dedupe from covered subdomain').traceSyncFn(() => domainDeduper(Array.from(domainSets)));
const dudupedDominArray = span.traceChildSync('dedupe from covered subdomain', () => domainDeduper(Array.from(domainSets)));
console.log(`Deduped ${previousSize - dudupedDominArray.length} rules from covered subdomain!`);
console.log(`Final size ${dudupedDominArray.length}`);
@ -186,7 +186,7 @@ export const buildRejectDomainSet = task(import.meta.path, async (span) => {
'Sukka\'s Ruleset - Reject Base',
description,
new Date(),
span.traceChild('sort reject domainset').traceSyncFn(() => sortDomains(dudupedDominArray, gorhill)),
span.traceChildSync('sort reject domainset', () => sortDomains(dudupedDominArray, gorhill)),
'domainset',
path.resolve(import.meta.dir, '../List/domainset/reject.conf'),
path.resolve(import.meta.dir, '../Clash/domainset/reject.txt')

6
Build/build-speedtest-domainset.ts
View File

@ -175,7 +175,7 @@ export const buildSpeedtestDomainSet = task(import.meta.path, async (span) => {
'.backend.librespeed.org'
]);
await span.traceChild('fetch previous speedtest domainset').traceAsyncFn(async () => {
await span.traceChildAsync('fetch previous speedtest domainset', async () => {
SetHelpers.add(domains, await getPreviousSpeedtestDomainsPromise());
});
@ -211,7 +211,7 @@ export const buildSpeedtestDomainSet = task(import.meta.path, async (span) => {
'Brazil',
'Turkey'
]).reduce<Record<string, Promise<void>>>((pMap, keyword) => {
pMap[keyword] = span.traceChild(`fetch speedtest endpoints: ${keyword}`).traceAsyncFn(() => querySpeedtestApi(keyword)).then(hostnameGroup => {
pMap[keyword] = span.traceChildAsync(`fetch speedtest endpoints: ${keyword}`, () => querySpeedtestApi(keyword)).then(hostnameGroup => {
hostnameGroup.forEach(hostname => {
if (hostname) {
domains.add(hostname);
@ -238,7 +238,7 @@ export const buildSpeedtestDomainSet = task(import.meta.path, async (span) => {
});
const gorhill = await getGorhillPublicSuffixPromise();
const deduped = span.traceChild('sort result').traceSyncFn(() => sortDomains(domainDeduper(Array.from(domains)), gorhill));
const deduped = span.traceChildSync('sort result', () => sortDomains(domainDeduper(Array.from(domains)), gorhill));
const description = [
...SHARED_DESCRIPTION,

2
Build/build-stream-service.ts
View File

@ -9,7 +9,7 @@ import { ALL, NORTH_AMERICA, EU, HK, TW, JP, KR } from '../Source/stream';
import { SHARED_DESCRIPTION } from './lib/constants';
export const createRulesetForStreamService = (span: Span, fileId: string, title: string, streamServices: Array<import('../Source/stream').StreamService>) => {
return span.traceChild(fileId).traceAsyncFn(async (childSpan) => Promise.all([
return span.traceChildAsync(fileId, async (childSpan) => Promise.all([
// Domains
createRuleset(
childSpan,

2
Build/build-telegram-cidr.ts
View File

@ -36,7 +36,7 @@ export const buildTelegramCIDR = task(import.meta.path, async (span) => {
const promise = getTelegramCIDRPromise();
const peeked = Bun.peek(promise);
const { date, results } = peeked === promise
? await span.traceChild('get telegram cidr').tracePromise(promise)
? await span.traceChildPromise('get telegram cidr', promise)
: (peeked as { date: Date, results: string[] });
if (results.length === 0) {

22
Build/lib/cache-filesystem.ts
View File

@ -25,19 +25,17 @@ export interface CacheOptions<S = string> {
type?: S extends string ? 'string' : 'buffer'
}
interface CacheApplyNonStringOption<T, S = string> {
interface CacheApplyRawOption {
ttl?: number | null,
temporaryBypass?: boolean
}
interface CacheApplyNonRawOption<T, S> extends CacheApplyRawOption {
serializer: (value: T) => S,
deserializer: (cached: S) => T,
temporaryBypass?: boolean
deserializer: (cached: S) => T
}
interface CacheApplyStringOption {
ttl?: number | null,
temporaryBypass?: boolean
}
type CacheApplyOption<T, S = string> = T extends string ? CacheApplyStringOption : CacheApplyNonStringOption<T, S>;
type CacheApplyOption<T, S> = T extends S ? CacheApplyRawOption : CacheApplyNonRawOption<T, S>;
const randomInt = (min: number, max: number) => Math.floor(Math.random() * (max - min + 1)) + min;
@ -127,8 +125,8 @@ export class Cache<S = string> {
});
}
get(key: string, defaultValue?: string): string | undefined {
const rv = this.db.prepare<{ value: string }, string>(
get(key: string, defaultValue?: S): S | undefined {
const rv = this.db.prepare<{ value: S }, string>(
`SELECT value FROM ${this.tableName} WHERE key = ? LIMIT 1`
).get(key);
@ -150,7 +148,7 @@ export class Cache<S = string> {
async apply<T>(
key: string,
fn: () => Promise<T>,
opt: CacheApplyOption<T>
opt: CacheApplyOption<T, S>
): Promise<T> {
const { ttl, temporaryBypass } = opt;

6
Build/lib/create-file.ts
View File

@ -18,7 +18,7 @@ export async function compareAndWriteFile(span: Span, linesA: string[], filePath
console.log(`Nothing to write to ${filePath}...`);
isEqual = false;
} else {
isEqual = await span.traceChild(`comparing ${filePath}`).traceAsyncFn(async () => {
isEqual = await span.traceChildAsync(`comparing ${filePath}`, async () => {
let index = 0;
for await (const lineB of readFileByLine(file)) {
@ -63,7 +63,7 @@ export async function compareAndWriteFile(span: Span, linesA: string[], filePath
return;
}
await span.traceChild(`writing ${filePath}`).traceAsyncFn(async () => {
await span.traceChildAsync(`writing ${filePath}`, async () => {
if (linesALen < 10000) {
return Bun.write(file, `${linesA.join('\n')}\n`);
}
@ -98,7 +98,7 @@ export const createRuleset = (
type: 'ruleset' | 'domainset', surgePath: string, clashPath: string
) => parentSpan.traceChild(`create ruleset: ${path.basename(surgePath, path.extname(surgePath))}`).traceAsyncFn((childSpan) => {
const surgeContent = withBannerArray(title, description, date, content);
const clashContent = childSpan.traceChild('convert incoming ruleset to clash').traceSyncFn(() => {
const clashContent = childSpan.traceChildSync('convert incoming ruleset to clash', () => {
let _clashContent;
switch (type) {
case 'domainset':

14
Build/lib/get-phishing-domains.ts
View File

@ -98,8 +98,8 @@ export const getPhishingDomains = (parentSpan: Span) => parentSpan.traceChild('g
SetAdd(domainSet, domainSet2);
}
span.traceChild('whitelisting phishing domains').traceSyncFn((parentSpan) => {
const trieForRemovingWhiteListed = parentSpan.traceChild('create trie for whitelisting').traceSyncFn(() => createTrie(domainSet));
span.traceChildSync('whitelisting phishing domains', (parentSpan) => {
const trieForRemovingWhiteListed = parentSpan.traceChildSync('create trie for whitelisting', () => createTrie(domainSet));
return parentSpan.traceChild('delete whitelisted from domainset').traceSyncFn(() => {
for (let i = 0, len = WHITELIST_DOMAIN.length; i < len; i++) {
@ -115,7 +115,7 @@ export const getPhishingDomains = (parentSpan: Span) => parentSpan.traceChild('g
const domainCountMap: Record<string, number> = {};
const getDomain = createCachedGorhillGetDomain(gorhill);
span.traceChild('process phishing domain set').traceSyncFn(() => {
span.traceChildSync('process phishing domain set', () => {
const domainArr = Array.from(domainSet);
for (let i = 0, len = domainArr.length; i < len; i++) {
@ -177,14 +177,14 @@ export const getPhishingDomains = (parentSpan: Span) => parentSpan.traceChild('g
}
});
const results = span.traceChild('get final phishing results').traceSyncFn(() => {
const results: string[] = [];
const results = span.traceChildSync('get final phishing results', () => {
const res: string[] = [];
for (const domain in domainCountMap) {
if (domainCountMap[domain] >= 5) {
results.push(`.${domain}`);
res.push(`.${domain}`);
}
}
return results;
return res;
});
return [results, domainSet] as const;

12
Build/trace/index.ts
View File

@ -29,6 +29,9 @@ export interface Span {
readonly traceSyncFn: <T>(fn: (span: Span) => T) => T,
readonly traceAsyncFn: <T>(fn: (span: Span) => T | Promise<T>) => Promise<T>,
readonly tracePromise: <T>(promise: Promise<T>) => Promise<T>,
readonly traceChildSync: <T>(name: string, fn: (span: Span) => T) => T,
readonly traceChildAsync: <T>(name: string, fn: (span: Span) => T | Promise<T>) => Promise<T>,
readonly traceChildPromise: <T>(name: string, promise: Promise<T>) => Promise<T>,
readonly traceResult: TraceResult
}
@ -91,6 +94,15 @@ export const createSpan = (name: string, parentTraceResult?: TraceResult): Span
} finally {
span.stop();
}
},
traceChildSync<T>(name: string, fn: (span: Span) => T): T {
return traceChild(name).traceSyncFn(fn);
},
traceChildAsync<T>(name: string, fn: (span: Span) => T | Promise<T>): Promise<T> {
return traceChild(name).traceAsyncFn(fn);
},
traceChildPromise<T>(name: string, promise: Promise<T>): Promise<T> {
return traceChild(name).tracePromise(promise);
}
};

2
Source/stream.ts
View File

@ -1,4 +1,4 @@
interface StreamService {
export interface StreamService {
name: string,
rules: string[],
ip?: {