GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2025-12-13 17:50:34 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Refactor: remove Bun fallback

Browse Source
This commit is contained in:
SukkaW 2024-07-24 11:36:14 +08:00
parent bb65a4180c
commit 46ae8e8cd8
5 changed files with 59 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

104
Build/build-speedtest-domainset.ts
View File

@ -8,18 +8,19 @@ import { getHostname } from 'tldts';
import { task } from './trace'; import { task } from './trace';
import { fetchWithRetry } from './lib/fetch-retry'; import { fetchWithRetry } from './lib/fetch-retry';
import { SHARED_DESCRIPTION } from './lib/constants'; import { SHARED_DESCRIPTION } from './lib/constants';
import picocolors from 'picocolors';
import { readFileIntoProcessedArray } from './lib/fetch-text-by-line'; import { readFileIntoProcessedArray } from './lib/fetch-text-by-line';
import { TTL, deserializeArray, fsFetchCache, serializeArray } from './lib/cache-filesystem'; import { TTL, deserializeArray, fsFetchCache, serializeArray } from './lib/cache-filesystem';
import { createTrie } from './lib/trie'; import { createTrie } from './lib/trie';
import { peek, track } from './lib/bun';
const s = new Sema(2); const s = new Sema(2);
const latestTopUserAgentsPromise = fsFetchCache.apply( const latestTopUserAgentsPromise = fsFetchCache.apply(
'https://cdn.jsdelivr.net/npm/top-user-agents@latest/src/desktop.json', 'https://cdn.jsdelivr.net/npm/top-user-agents@latest/src/desktop.json',
() => fetchWithRetry('https://cdn.jsdelivr.net/npm/top-user-agents@latest/src/desktop.json') () => fetchWithRetry(
'https://cdn.jsdelivr.net/npm/top-user-agents@latest/src/desktop.json',
{ signal: AbortSignal.timeout(1000 * 60) }
)
.then(res => res.json() as Promise<string[]>) .then(res => res.json() as Promise<string[]>)
.then((userAgents) => userAgents.filter(ua => ua.startsWith('Mozilla/5.0 '))), .then((userAgents) => userAgents.filter(ua => ua.startsWith('Mozilla/5.0 '))),
{ {
@ -56,7 +57,7 @@ const querySpeedtestApi = async (keyword: string): Promise<Array<string | null>>
} }
: {}) : {})
}, },
signal: AbortSignal.timeout(1000 * 4), signal: AbortSignal.timeout(1000 * 60),
retry: { retry: {
retries: 2 retries: 2
} }
@ -194,63 +195,44 @@ export const buildSpeedtestDomainSet = task(require.main === module, __filename)
} }
); );
await new Promise<void>((resolve, reject) => { await Promise.all([
const pMap = ([ 'Hong Kong',
'Hong Kong', 'Taiwan',
'Taiwan', 'China Telecom',
'China Telecom', 'China Mobile',
'China Mobile', 'China Unicom',
'China Unicom', 'Japan',
'Japan', 'Tokyo',
'Tokyo', 'Singapore',
'Singapore', 'Korea',
'Korea', 'Seoul',
'Seoul', 'Canada',
'Canada', 'Toronto',
'Toronto', 'Montreal',
'Montreal', 'Los Ang',
'Los Ang', 'San Jos',
'San Jos', 'Seattle',
'Seattle', 'New York',
'New York', 'Dallas',
'Dallas', 'Miami',
'Miami', 'Berlin',
'Berlin', 'Frankfurt',
'Frankfurt', 'London',
'London', 'Paris',
'Paris', 'Amsterdam',
'Amsterdam', 'Moscow',
'Moscow', 'Australia',
'Australia', 'Sydney',
'Sydney', 'Brazil',
'Brazil', 'Turkey'
'Turkey' ].map((keyword) => span.traceChildAsync(
]).reduce<Record<string, Promise<void>>>((pMap, keyword) => { `fetch speedtest endpoints: ${keyword}`,
pMap[keyword] = track(span.traceChildAsync(`fetch speedtest endpoints: ${keyword}`, () => querySpeedtestApi(keyword)).then(hostnameGroup => { () => querySpeedtestApi(keyword)
return hostnameGroup.forEach(hostname => { ).then(hostnameGroup => hostnameGroup.forEach(hostname => {
if (hostname) { if (hostname) {
domainTrie.add(hostname); domainTrie.add(hostname);
} }
}); }))));
}));
return pMap;
}, {});
const timer = setTimeout(() => {
console.error(picocolors.red('Task timeout!'));
Object.entries(pMap).forEach(([name, p]) => {
console.log(`[${name}]`, peek(p));
});
resolve();
}, 1000 * 60 * 1.5);
Promise.all(Object.values(pMap)).then(() => {
clearTimeout(timer);
return resolve();
}).catch(() => reject);
});
const deduped = span.traceChildSync('sort result', () => sortDomains(domainDeduper(domainTrie))); const deduped = span.traceChildSync('sort result', () => sortDomains(domainDeduper(domainTrie)));

14
Build/lib/fetch-retry.ts
View File

@ -2,12 +2,6 @@ import retry from 'async-retry';
import picocolors from 'picocolors'; import picocolors from 'picocolors';
import { setTimeout } from 'timers/promises'; import { setTimeout } from 'timers/promises';
// retry settings
const MIN_TIMEOUT = 10;
const MAX_RETRIES = 5;
const MAX_RETRY_AFTER = 20;
const FACTOR = 6;
function isClientError(err: unknown): err is NodeJS.ErrnoException { function isClientError(err: unknown): err is NodeJS.ErrnoException {
if (!err || typeof err !== 'object') return false; if (!err || typeof err !== 'object') return false;
@ -55,10 +49,10 @@ interface FetchWithRetry {
const DEFAULT_OPT: Required<FetchRetryOpt> = { const DEFAULT_OPT: Required<FetchRetryOpt> = {
// timeouts will be [10, 60, 360, 2160, 12960] // timeouts will be [10, 60, 360, 2160, 12960]
// (before randomization is added) // (before randomization is added)
minTimeout: MIN_TIMEOUT, minTimeout: 10,
retries: MAX_RETRIES, retries: 5,
factor: FACTOR, factor: 6,
maxRetryAfter: MAX_RETRY_AFTER, maxRetryAfter: 20,
retryOnAborted: false, retryOnAborted: false,
retryOnNon2xx: true retryOnNon2xx: true
}; };

45
Build/lib/fetch-text-by-line.ts
View File

@ -8,47 +8,16 @@ import type { ReadableStream } from 'stream/web';
import { TextDecoderStream } from 'stream/web'; import { TextDecoderStream } from 'stream/web';
import { processLine } from './process-line'; import { processLine } from './process-line';
const enableTextLineStream = !!process.env.ENABLE_TEXT_LINE_STREAM;
const decoder = new TextDecoder('utf-8');
async function *createTextLineAsyncIterableFromStreamSource(stream: ReadableStream<Uint8Array>): AsyncIterable<string> {
let buf = '';
const reader = stream.getReader();
while (true) {
const res = await reader.read();
if (res.done) {
break;
}
const chunkStr = decoder.decode(res.value).replaceAll('\r\n', '\n');
for (let i = 0, len = chunkStr.length; i < len; i++) {
const char = chunkStr[i];
if (char === '\n') {
yield buf;
buf = '';
} else {
buf += char;
}
}
}
if (buf) {
yield buf;
}
}
const getReadableStream = (file: string | FileHandle): ReadableStream => { const getReadableStream = (file: string | FileHandle): ReadableStream => {
if (typeof file === 'string') { if (typeof file === 'string') {
return Readable.toWeb(fs.createReadStream(file /* { encoding: 'utf-8' } */)); return Readable.toWeb(fs.createReadStream(file/* , { encoding: 'utf-8' } */));
} }
return file.readableWebStream(); return file.readableWebStream();
}; };
// TODO: use FileHandle.readLine() // TODO: use FileHandle.readLine()
export const readFileByLine: ((file: string | FileHandle) => AsyncIterable<string>) = enableTextLineStream export const readFileByLine: ((file: string | FileHandle) => AsyncIterable<string>) = (file: string | FileHandle) => getReadableStream(file)
? (file: string | FileHandle) => getReadableStream(file).pipeThrough(new TextDecoderStream()).pipeThrough(new TextLineStream()) .pipeThrough(new TextDecoderStream())
: (file: string | FileHandle) => createTextLineAsyncIterableFromStreamSource(getReadableStream(file)); .pipeThrough(new TextLineStream());
const ensureResponseBody = (resp: Response) => { const ensureResponseBody = (resp: Response) => {
if (!resp.body) { if (!resp.body) {
@ -60,9 +29,9 @@ const ensureResponseBody = (resp: Response) => {
return resp.body; return resp.body;
}; };
export const createReadlineInterfaceFromResponse: ((resp: Response) => AsyncIterable<string>) = enableTextLineStream export const createReadlineInterfaceFromResponse: ((resp: Response) => AsyncIterable<string>) = (resp) => ensureResponseBody(resp)
? (resp) => ensureResponseBody(resp).pipeThrough(new TextDecoderStream()).pipeThrough(new TextLineStream()) .pipeThrough(new TextDecoderStream())
: (resp) => createTextLineAsyncIterableFromStreamSource(ensureResponseBody(resp)); .pipeThrough(new TextLineStream());
export function fetchRemoteTextByLine(url: string | URL) { export function fetchRemoteTextByLine(url: string | URL) {
return fetchWithRetry(url, defaultRequestInit).then(createReadlineInterfaceFromResponse); return fetchWithRetry(url, defaultRequestInit).then(createReadlineInterfaceFromResponse);

3
Build/lib/get-gorhill-publicsuffix.ts
View File

@ -4,7 +4,8 @@ import { createMemoizedPromise } from './memo-promise';
import { getPublicSuffixListTextPromise } from './download-publicsuffixlist'; import { getPublicSuffixListTextPromise } from './download-publicsuffixlist';
import { fileURLToPath } from 'url'; import { fileURLToPath } from 'url';
const customFetch = async (url: string | URL) => { // TODO: node undfici fetch doesn't support file URL reading yet
const customFetch = async (url: URL) => {
const filePath = fileURLToPath(url); const filePath = fileURLToPath(url);
const file = await fsp.readFile(filePath); const file = await fsp.readFile(filePath);
return new Blob([file]) as any; return new Blob([file]) as any;

5
package.json
View File

@ -9,10 +9,11 @@
}, },
"type": "commonjs", "type": "commonjs",
"scripts": { "scripts": {
"node": "SWCRC=true ENABLE_TEXT_LINE_STREAM=true node -r @swc-node/register", "node": "SWCRC=true node -r @swc-node/register",
"dexnode": "SWCRC=true ENABLE_TEXT_LINE_STREAM=true dexnode -r @swc-node/register", "dexnode": "SWCRC=true dexnode -r @swc-node/register",
"build": "pnpm run node ./Build/index.ts", "build": "pnpm run node ./Build/index.ts",
"build-profile": "pnpm run dexnode -r @swc-node/register ./Build/index.ts", "build-profile": "pnpm run dexnode -r @swc-node/register ./Build/index.ts",
"build-webstream": "ENABLE_EXPERIMENTAL_WEBSTREAMS=true pnpm run node ./Build/index.ts",
"lint": "eslint --format=sukka ." "lint": "eslint --format=sukka ."
}, },
"author": "", "author": "",