Chore: minor infra changes

1
This commit is contained in:
SukkaW 2024-08-02 17:20:34 +08:00
parent 3d3abb8b50
commit 7f6e631847
5 changed files with 14 additions and 9 deletions

View File

@ -11,7 +11,7 @@ import { sortDomains } from './lib/stable-sort-domain';
const getS3OSSDomainsPromise = (async (): Promise<string[]> => {
const trie = createTrie(
(await getPublicSuffixListTextPromise()).split('\n'),
await getPublicSuffixListTextPromise(),
true,
false
);

View File

@ -1,14 +1,17 @@
import { TTL, fsFetchCache } from './cache-filesystem';
import { TTL, deserializeArray, fsFetchCache, serializeArray } from './cache-filesystem';
import { defaultRequestInit, fetchWithRetry } from './fetch-retry';
import { createMemoizedPromise } from './memo-promise';
export const getPublicSuffixListTextPromise = createMemoizedPromise(() => fsFetchCache.apply(
'https://publicsuffix.org/list/public_suffix_list.dat',
() => fetchWithRetry('https://publicsuffix.org/list/public_suffix_list.dat', defaultRequestInit).then(r => r.text()),
'https://publicsuffix.org/list/public_suffix_list.dat array',
() => fetchWithRetry('https://publicsuffix.org/list/public_suffix_list.dat', defaultRequestInit)
.then(r => r.text()).then(text => text.split('\n')),
{
// https://github.com/publicsuffix/list/blob/master/.github/workflows/tld-update.yml
// Though the action runs every 24 hours, the IANA list is updated every 7 days.
// So a 3 day TTL should be enough.
ttl: TTL.THREE_DAYS()
ttl: TTL.THREE_DAYS(),
serializer: serializeArray,
deserializer: deserializeArray
}
));

View File

@ -127,7 +127,7 @@ function createFetchRetry($fetch: typeof fetch): FetchWithRetry {
export const defaultRequestInit: RequestInit = {
headers: {
'User-Agent': 'curl/8.1.2 (https://github.com/SukkaW/Surge)'
'User-Agent': 'curl/8.9.0 (https://github.com/SukkaW/Surge)'
}
};

View File

@ -1,4 +1,4 @@
import { parse as tldtsParse } from 'tldts';
import { parse as tldtsParse } from 'tldts-experimental';
import { isProbablyIpv4 } from './is-fast-ip';
export const normalizeDomain = (domain: string) => {
if (!domain) return null;

View File

@ -402,8 +402,10 @@ export const createTrie = (from?: string[] | Set<string> | null, hostnameMode =
},
whitelist,
[inspect.custom]: (depth: number) => JSON.stringify(deepTrieNodeToJSON(root), null, 2).split('\n').map((line) => ' '.repeat(depth) + line).join('\n'),
[inspect.custom]: (depth: number) => fastStringArrayJoin(
JSON.stringify(deepTrieNodeToJSON(root), null, 2).split('\n').map((line) => ' '.repeat(depth) + line),
'\n'
),
hostnameMode,
smolTree
};