GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2025-12-12 01:00:34 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Perf: many changes

Browse Source 
- Hoist process hosts line callback
- Reduce dp hosts file size
- Reduce domain sort
This commit is contained in:
SukkaW 2024-05-02 08:13:15 +08:00
parent 10bde9f1e8
commit e5d511d105
6 changed files with 42 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
Build/build-reject-domainset.ts
View File

@ -18,14 +18,9 @@ import { getPhishingDomains } from './lib/get-phishing-domains';
import * as SetHelpers from 'mnemonist/set';
import { setAddFromArray } from './lib/set-add-from-array';
import type { PublicSuffixList } from '@gorhill/publicsuffixlist';
export const buildRejectDomainSet = task(import.meta.path, async (span) => {
const gorhillPromise = getGorhillPublicSuffixPromise();
const gorhillPeeked = Bun.peek(gorhillPromise);
const gorhill: PublicSuffixList = gorhillPeeked === gorhillPromise
? await gorhillPromise
: (gorhillPeeked as PublicSuffixList);
const gorhill = await getGorhillPublicSuffixPromise();
/** Whitelists */
const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST);

4
Build/build-reject-ip-list.ts
View File

@ -65,8 +65,10 @@ const getBotNetFilterIPsPromise = fsFetchCache.apply(
}
);
const localRejectIPSourcesPromise = readFileIntoProcessedArray(path.resolve(import.meta.dir, '../Source/ip/reject.conf'));
export const buildRejectIPList = task(import.meta.path, async (span) => {
const result: string[] = await readFileIntoProcessedArray(path.resolve(import.meta.dir, '../Source/ip/reject.conf'));
const result = await localRejectIPSourcesPromise;
const bogusNxDomainIPs = await span.traceChildPromise('get bogus nxdomain ips', getBogusNxDomainIPsPromise);
const botNetIPs = await span.traceChildPromise('get botnet ips', getBotNetFilterIPsPromise);

2
Build/build-speedtest-domainset.ts
View File

@ -232,7 +232,7 @@ export const buildSpeedtestDomainSet = task(import.meta.path, async (span) => {
});
resolve();
}, 1000 * 60 * 2);
}, 1000 * 60 * 1.5);
Promise.all(Object.values(pMap)).then(() => {
clearTimeout(timer);

15
Build/lib/parse-filter.ts
View File

@ -46,10 +46,8 @@ export function processDomainLists(span: Span, domainListsUrl: string, includeAl
}
));
}
export function processHosts(span: Span, hostsUrl: string, mirrors: string[] | null, includeAllSubDomain = false, ttl: number | null = null) {
const domainSets = new Set<string>();
const lineCb = (l: string) => {
const hostsLineCb = (l: string, set: Set<string>, includeAllSubDomain: boolean, meta: string) => {
const line = processLine(l);
if (!line) {
return;
@ -64,19 +62,22 @@ export function processHosts(span: Span, hostsUrl: string, mirrors: string[] | n
return;
}
if (DEBUG_DOMAIN_TO_FIND && domain.includes(DEBUG_DOMAIN_TO_FIND)) {
console.warn(picocolors.red(hostsUrl), '(black)', domain.replaceAll(DEBUG_DOMAIN_TO_FIND, picocolors.bold(DEBUG_DOMAIN_TO_FIND)));
console.warn(picocolors.red(meta), '(black)', domain.replaceAll(DEBUG_DOMAIN_TO_FIND, picocolors.bold(DEBUG_DOMAIN_TO_FIND)));
foundDebugDomain = true;
}
domainSets.add(includeAllSubDomain ? `.${domain}` : domain);
set.add(includeAllSubDomain ? `.${domain}` : domain);
};
export function processHosts(span: Span, hostsUrl: string, mirrors: string[] | null, includeAllSubDomain = false, ttl: number | null = null) {
const domainSets = new Set<string>();
return span.traceChild(`processhosts: ${hostsUrl}`).traceAsyncFn((childSpan) => fsFetchCache.apply(
hostsUrl,
async () => {
if (mirrors == null || mirrors.length === 0) {
for await (const l of await fetchRemoteTextByLine(hostsUrl)) {
lineCb(l);
hostsLineCb(l, domainSets, includeAllSubDomain, hostsUrl);
}
} else {
const filterRules = await childSpan
@ -85,7 +86,7 @@ export function processHosts(span: Span, hostsUrl: string, mirrors: string[] | n
childSpan.traceChild('parse hosts').traceSyncFn(() => {
for (let i = 0, len = filterRules.length; i < len; i++) {
lineCb(filterRules[i]);
hostsLineCb(filterRules[i], domainSets, includeAllSubDomain, hostsUrl);
}
});
}

3
Build/lib/reject-data-source.ts
View File

@ -9,7 +9,8 @@ export const HOSTS: HostsSource[] = [
true,
TTL.THREE_HOURS()
],
['https://someonewhocares.org/hosts/hosts', null, true, TTL.THREE_HOURS()],
// Dan Pollock's hosts file, 0.0.0.0 version is 30 KiB smaller
['https://someonewhocares.org/hosts/zero/hosts', null, true, TTL.THREE_HOURS()],
// no coin list is not actively maintained, but it updates daily when being maintained, so we set a 3 days cache ttl
['https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt', null, true, TTL.THREE_DAYS()],
// have not been updated for more than a year, so we set a 14 days cache ttl

10
Build/lib/stable-sort-domain.ts
View File

@ -42,7 +42,15 @@ export const sortDomains = (inputs: string[], gorhill: PublicSuffixList) => {
const sorter = (a: string, b: string) => {
if (a === b) return 0;
return compare(domains.get(a)!, domains.get(b)!) || compare(a, b);
const $a = domains.get(a)!;
const $b = domains.get(b)!;
// avoid compare same thing twice
if (a === $a && b === $b) {
return compare(a, b);
}
return compare($a, $b) || compare(a, b);
};
return inputs.sort(sorter);