Improve fs memo (hash xxhash64)

This commit is contained in:
SukkaW 2024-10-18 12:05:41 +08:00
parent 4c1c237fc3
commit 113eae281f
2 changed files with 18 additions and 6 deletions

View File

@ -3,14 +3,18 @@ import { Cache } from './cache-filesystem';
import type { CacheApplyOption } from './cache-filesystem'; import type { CacheApplyOption } from './cache-filesystem';
import { isCI } from 'ci-info'; import { isCI } from 'ci-info';
import { Typeson, set, map, typedArrays } from 'typeson-registry'; import { xxhash64 } from 'hash-wasm';
import { Typeson, set, map, typedArrays, undef, infinity } from 'typeson-registry';
import picocolors from 'picocolors'; import picocolors from 'picocolors';
import { identity } from './misc'; import { identity } from './misc';
const typeson = new Typeson().register([ const typeson = new Typeson().register([
typedArrays, typedArrays,
set, set,
map map,
undef,
infinity
]); ]);
const fsMemoCache = new Cache({ cachePath: path.resolve(__dirname, '../../.cache'), tableName: 'fs_memo_cache' }); const fsMemoCache = new Cache({ cachePath: path.resolve(__dirname, '../../.cache'), tableName: 'fs_memo_cache' });
@ -26,6 +30,7 @@ type TypesonValue =
| number | number
| boolean | boolean
| null | null
| undefined
| Set<any> | Set<any>
| Map<any, any> | Map<any, any>
| TypesonObject | TypesonObject
@ -45,14 +50,17 @@ export function cache<Args extends TypesonValue[], T>(
fn: (...args: Args) => Promise<T>, fn: (...args: Args) => Promise<T>,
opt: FsMemoCacheOptions<T> opt: FsMemoCacheOptions<T>
): (...args: Args) => Promise<T> { ): (...args: Args) => Promise<T> {
// TODO if cb.toString() is long we should hash it
const fixedKey = fn.toString(); const fixedKey = fn.toString();
return async function cachedCb(...args: Args) { return async function cachedCb(...args: Args) {
// Construct the complete cache key for this function invocation // Construct the complete cache key for this function invocation
// typeson.stringify is still limited. For now we uses typescript to guard the args. // typeson.stringify is still limited. For now we uses typescript to guard the args.
const cacheKey = `${fixedKey}|${typeson.stringifySync(args)}`; const cacheKey = (await Promise.all([
const cacheName = fn.name || cacheKey; xxhash64(fixedKey),
xxhash64(typeson.stringifySync(args))
])).join('|');
const cacheName = fn.name || fixedKey;
const { temporaryBypass, incrementTtlWhenHit } = opt; const { temporaryBypass, incrementTtlWhenHit } = opt;

View File

@ -1,6 +1,10 @@
const notError = Symbol('notError'); const notError = Symbol('notError');
export function createMemoizedPromise<T>(fn: () => Promise<T>, preload = true): () => Promise<T> { export function createMemoizedPromise<T>(
fn: () => Promise<T>,
/** whether to create promise immediately or only create after first access */
preload = true
): () => Promise<T> {
let error: Error | typeof notError = notError; let error: Error | typeof notError = notError;
let promise: Promise<T> | null = preload let promise: Promise<T> | null = preload