Fix corrupt local cache

This commit is contained in:
SukkaW 2024-10-15 20:27:49 +08:00
parent e0f7e7e48a
commit 8a0a50d049
7 changed files with 41 additions and 31 deletions

View File

@ -125,8 +125,8 @@ process.on('unhandledRejection', (reason) => {
wtf.dump(); wtf.dump();
process.exit(0); process.exit(0);
} catch (e) { } catch (e) {
console.trace(e);
console.error('Something went wrong!'); console.error('Something went wrong!');
console.trace(e);
process.exit(1); process.exit(1);
} }
})(); })();

View File

@ -350,13 +350,15 @@ export class Cache<S = string> {
this.set(getETagKey(url), serverETag, TTL.ONE_WEEK_STATIC); this.set(getETagKey(url), serverETag, TTL.ONE_WEEK_STATIC);
} }
// If we do not have a cached value, we ignore 304 // If we do not have a cached value, we ignore 304
if (res.statusCode === 304 && typeof previouslyCached === 'string') { if (res.statusCode === 304 && typeof previouslyCached === 'string' && previouslyCached.length > 1) {
controller.abort(); const err = new Custom304NotModifiedError(url, previouslyCached);
throw new Custom304NotModifiedError(url, previouslyCached); controller.abort(err);
throw err;
} }
if (!serverETag && !this.get(getETagKey(primaryUrl)) && typeof previouslyCached === 'string') { if (!serverETag && !this.get(getETagKey(primaryUrl)) && typeof previouslyCached === 'string') {
controller.abort(); const err = new CustomNoETagFallbackError(previouslyCached);
throw new CustomNoETagFallbackError(previouslyCached); controller.abort(err);
throw err;
} }
// either no etag and not cached // either no etag and not cached
@ -386,18 +388,26 @@ export class Cache<S = string> {
return value; return value;
} catch (e) { } catch (e) {
if (e instanceof AggregateError) { if (e && typeof e === 'object' && 'errors' in e && Array.isArray(e.errors)) {
const deserializer = 'deserializer' in opt ? opt.deserializer : identity as any; const deserializer = 'deserializer' in opt ? opt.deserializer : identity as any;
for (const error of e.errors) { console.log(e.errors);
if (error instanceof Custom304NotModifiedError) {
console.log(picocolors.green('[cache] http 304'), picocolors.gray(primaryUrl)); for (let i = 0, len = e.errors.length; i < len; i++) {
this.updateTtl(cachedKey, TTL.ONE_WEEK_STATIC); const error = e.errors[i];
return deserializer(error.data); if ('name' in error && (error.name === 'CustomAbortError' || error.name === 'AbortError')) {
continue;
} }
if (error instanceof CustomNoETagFallbackError) { if ('digest' in error) {
console.log(picocolors.green('[cache] hit'), picocolors.gray(primaryUrl)); if (error.digest === 'Custom304NotModifiedError') {
return deserializer(error.data); console.log(picocolors.green('[cache] http 304'), picocolors.gray(primaryUrl));
this.updateTtl(cachedKey, TTL.ONE_WEEK_STATIC);
return deserializer(error.data);
}
if (error.digest === 'CustomNoETagFallbackError') {
console.log(picocolors.green('[cache] hit'), picocolors.gray(primaryUrl));
return deserializer(error.data);
}
} }
} }
} }

View File

@ -22,8 +22,9 @@ export const readFileByLine: ((file: string | FileHandle) => AsyncIterable<strin
.pipeThrough(new TextDecoderStream()) .pipeThrough(new TextDecoderStream())
.pipeThrough(new TextLineStream()); .pipeThrough(new TextLineStream());
function ensureResponseBody<T extends Response | NodeFetchResponse | UndiciResponseData>(resp: T): NonNullable<T['body']> { function ensureResponseBody<T extends NodeFetchResponse | UndiciResponseData>(resp: T): NonNullable<T['body']> {
if (!resp.body) { // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- NodeFetchResponse['body'] is nullable
if (resp.body == null) {
throw new Error('Failed to fetch remote text'); throw new Error('Failed to fetch remote text');
} }
if ('bodyUsed' in resp && resp.bodyUsed) { if ('bodyUsed' in resp && resp.bodyUsed) {
@ -32,7 +33,7 @@ function ensureResponseBody<T extends Response | NodeFetchResponse | UndiciRespo
return resp.body; return resp.body;
} }
export const createReadlineInterfaceFromResponse: ((resp: Response | NodeFetchResponse | UndiciResponseData) => AsyncIterable<string>) = (resp) => { export const createReadlineInterfaceFromResponse: ((resp: NodeFetchResponse | UndiciResponseData) => AsyncIterable<string>) = (resp) => {
const stream = ensureResponseBody(resp); const stream = ensureResponseBody(resp);
const webStream: ReadableStream<Uint8Array> = 'getReader' in stream const webStream: ReadableStream<Uint8Array> = 'getReader' in stream

View File

@ -1,6 +1,5 @@
import { createReadlineInterfaceFromResponse } from './fetch-text-by-line'; import { createReadlineInterfaceFromResponse } from './fetch-text-by-line';
import { parse as tldtsParse } from 'tldts'; import { parse as tldtsParse } from 'tldts';
import { $fetch } from './make-fetch-happen';
import type { NodeFetchResponse } from './make-fetch-happen'; import type { NodeFetchResponse } from './make-fetch-happen';
import type { UndiciResponseData } from './fetch-retry'; import type { UndiciResponseData } from './fetch-retry';
@ -28,8 +27,3 @@ export async function parseFelixDnsmasqFromResp(resp: NodeFetchResponse | Undici
return results; return results;
} }
export async function parseFelixDnsmasq(url: string): Promise<string[]> {
const resp = await $fetch(url);
return parseFelixDnsmasqFromResp(resp);
}

View File

@ -3,9 +3,14 @@ import { parse } from 'csv-parse/sync';
import { createTrie } from './lib/trie'; import { createTrie } from './lib/trie';
import path from 'node:path'; import path from 'node:path';
import { processLine } from './lib/process-line'; import { processLine } from './lib/process-line';
import { parseFelixDnsmasq } from './lib/parse-dnsmasq';
import { SOURCE_DIR } from './constants/dir'; import { SOURCE_DIR } from './constants/dir';
import { $fetch } from './lib/make-fetch-happen'; import { $fetch } from './lib/make-fetch-happen';
import { parseFelixDnsmasqFromResp } from './lib/parse-dnsmasq';
async function parseFelixDnsmasq(url: string): Promise<string[]> {
const resp = await $fetch(url);
return parseFelixDnsmasqFromResp(resp);
}
export async function parseDomesticList() { export async function parseDomesticList() {
const trie = createTrie(await parseFelixDnsmasq('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf')); const trie = createTrie(await parseFelixDnsmasq('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf'));

View File

@ -41,7 +41,7 @@
"tar-fs": "^3.0.6", "tar-fs": "^3.0.6",
"tldts": "^6.1.51", "tldts": "^6.1.51",
"tldts-experimental": "^6.1.51", "tldts-experimental": "^6.1.51",
"undici": "7.0.0-alpha.2", "undici": "6.20.1",
"wtfnode": "^0.9.3", "wtfnode": "^0.9.3",
"yaml": "^2.5.1" "yaml": "^2.5.1"
}, },

10
pnpm-lock.yaml generated
View File

@ -75,8 +75,8 @@ importers:
specifier: ^6.1.51 specifier: ^6.1.51
version: 6.1.51 version: 6.1.51
undici: undici:
specifier: 7.0.0-alpha.2 specifier: 6.20.1
version: 7.0.0-alpha.2 version: 6.20.1
wtfnode: wtfnode:
specifier: ^0.9.3 specifier: ^0.9.3
version: 0.9.3 version: 0.9.3
@ -1771,8 +1771,8 @@ packages:
undici-types@6.19.8: undici-types@6.19.8:
resolution: {integrity: sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==} resolution: {integrity: sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==}
undici@7.0.0-alpha.2: undici@6.20.1:
resolution: {integrity: sha512-4Xn6pwsp6U8F9U7qdDhDwcHPvjcfwt3gbswRg30xgmkQ4Bpfbi6J1OrjTe8WIUVecbcGWOZMPAt6crf0PLliUw==} resolution: {integrity: sha512-AjQF1QsmqfJys+LXfGTNum+qw4S88CojRInG/6t31W/1fk6G59s92bnAvGz5Cmur+kQv2SURXEvvudLmbrE8QA==}
engines: {node: '>=18.17'} engines: {node: '>=18.17'}
unique-filename@4.0.0: unique-filename@4.0.0:
@ -3613,7 +3613,7 @@ snapshots:
undici-types@6.19.8: {} undici-types@6.19.8: {}
undici@7.0.0-alpha.2: {} undici@6.20.1: {}
unique-filename@4.0.0: unique-filename@4.0.0:
dependencies: dependencies: