diff --git a/Build/build-chn-cidr.ts b/Build/build-chn-cidr.ts index a618f19b..1e2eec2f 100644 --- a/Build/build-chn-cidr.ts +++ b/Build/build-chn-cidr.ts @@ -10,6 +10,9 @@ const EXCLUDE_CIDRS = [ '223.120.0.0/15' ]; +// preload the module +import('cidr-tools-wasm'); + export const buildChnCidr = task(import.meta.path, async () => { const [{ exclude }, cidr] = await Promise.all([ import('cidr-tools-wasm'), diff --git a/Build/build-internal-reverse-chn-cidr.ts b/Build/build-internal-reverse-chn-cidr.ts index 5f923dd0..19177c78 100644 --- a/Build/build-internal-reverse-chn-cidr.ts +++ b/Build/build-internal-reverse-chn-cidr.ts @@ -22,6 +22,9 @@ const RESERVED_IPV4_CIDR = [ '240.0.0.0/4' ]; +// preload the module +import('cidr-tools-wasm'); + export const buildInternalReverseChnCIDR = task(import.meta.path, async () => { const [{ exclude }, cidr] = await Promise.all([ import('cidr-tools-wasm'), diff --git a/Build/download-previous-build.ts b/Build/download-previous-build.ts index b8bfccae..d8371c01 100644 --- a/Build/download-previous-build.ts +++ b/Build/download-previous-build.ts @@ -67,7 +67,6 @@ export const downloadPreviousBuild = task(import.meta.path, async () => { async onentry(entry) { if (entry.type !== 'File') { // not a file, throw away - console.log(entry.type, entry.path) entry.resume(); return; } diff --git a/Build/lib/create-file.ts b/Build/lib/create-file.ts index 7be6f7b7..d0c09cd6 100644 --- a/Build/lib/create-file.ts +++ b/Build/lib/create-file.ts @@ -6,10 +6,12 @@ export async function compareAndWriteFile(linesA: string[], filePath: string) { let isEqual = true; const file = Bun.file(filePath); + const linesALen = linesA.length; + if (!(await file.exists())) { console.log(`${filePath} does not exists, writing...`); isEqual = false; - } else if (linesA.length === 0) { + } else if (linesALen === 0) { console.log(`Nothing to write to ${filePath}...`); isEqual = false; } else { @@ -35,23 +37,24 @@ export async function compareAndWriteFile(linesA: string[], filePath: string) { } } - if (index !== linesA.length) { + if (index !== linesALen) { + // The file becomes larger isEqual = false; } } - if (!isEqual) { - const writer = file.writer(); - - for (let i = 0, len = linesA.length; i < len; i++) { - writer.write(`${linesA[i]}\n`); - } - - await writer.end(); + if (isEqual) { + console.log(`Same Content, bail out writing: ${filePath}`); return; } - console.log(`Same Content, bail out writing: ${filePath}`); + const writer = file.writer(); + + for (let i = 0; i < linesALen; i++) { + writer.write(`${linesA[i]}\n`); + } + + return writer.end(); } export const withBannerArray = (title: string, description: string[], date: Date, content: string[]) => { diff --git a/Build/lib/fetch-retry.ts b/Build/lib/fetch-retry.ts index 0f8cb2a2..7a6dc0d4 100644 --- a/Build/lib/fetch-retry.ts +++ b/Build/lib/fetch-retry.ts @@ -1,5 +1,112 @@ -// @ts-expect-error -- missing types -import createFetchRetry from '@vercel/fetch-retry'; +import retry from 'async-retry'; + +// retry settings +const MIN_TIMEOUT = 10; +const MAX_RETRIES = 5; +const MAX_RETRY_AFTER = 20; +const FACTOR = 6; + +function isClientError(err: any): err is NodeJS.ErrnoException { + if (!err) return false; + return ( + err.code === 'ERR_UNESCAPED_CHARACTERS' || + err.message === 'Request path contains unescaped characters' + ); +} + +interface FetchRetryOpt { + minTimeout?: number, + retries?: number, + factor?: number, + maxRetryAfter?: number, + retry?: number, + onRetry?: (err: Error) => void +} + +function createFetchRetry($fetch: typeof fetch): typeof fetch { + const fetchRetry = async (url: string | URL, opts: RequestInit & { retry?: FetchRetryOpt } = {}) => { + const retryOpts = Object.assign( + { + // timeouts will be [10, 60, 360, 2160, 12960] + // (before randomization is added) + minTimeout: MIN_TIMEOUT, + retries: MAX_RETRIES, + factor: FACTOR, + maxRetryAfter: MAX_RETRY_AFTER, + }, + opts.retry + ); + + try { + return await retry(async (bail) => { + try { + // this will be retried + const res = await $fetch(url, opts); + + if ((res.status >= 500 && res.status < 600) || res.status === 429) { + // NOTE: doesn't support http-date format + const retryAfterHeader = res.headers.get('retry-after'); + if (retryAfterHeader) { + const retryAfter = parseInt(retryAfterHeader, 10); + if (retryAfter) { + if (retryAfter > retryOpts.maxRetryAfter) { + return res; + } else { + await new Promise((r) => setTimeout(r, retryAfter * 1e3)); + } + } + } + throw new ResponseError(res); + } else { + return res; + } + } catch (err: unknown) { + if (err instanceof Error) { + if (err.name === 'AbortError') { + return bail(err); + } + } + if (isClientError(err)) { + return bail(err); + } + throw err; + } + }, retryOpts); + } catch (err) { + if (err instanceof ResponseError) { + return err.res; + } + throw err; + } + } + + for (const k of Object.keys($fetch)) { + const key = k as keyof typeof $fetch; + fetchRetry[key] = $fetch[key]; + } + + return fetchRetry as typeof fetch; +} + +export class ResponseError extends Error { + readonly res: Response; + readonly code: number; + readonly statusCode: number; + readonly url: string; + + constructor(res: Response) { + super(res.statusText); + + if (Error.captureStackTrace) { + Error.captureStackTrace(this, ResponseError); + } + + this.name = this.constructor.name; + this.res = res; + this.code = this.statusCode = res.status; + this.url = res.url; + } +} export const defaultRequestInit: RequestInit = { headers: { @@ -7,4 +114,4 @@ export const defaultRequestInit: RequestInit = { } } -export const fetchWithRetry: typeof fetch = createFetchRetry(fetch); +export const fetchWithRetry = createFetchRetry(fetch); diff --git a/Build/lib/text-line-transform-stream.ts b/Build/lib/text-line-transform-stream.ts index bd4f44ea..26c4a159 100644 --- a/Build/lib/text-line-transform-stream.ts +++ b/Build/lib/text-line-transform-stream.ts @@ -19,14 +19,15 @@ interface TextLineStreamOptions { * ``` */ export class TextLineStream extends TransformStream { - private __buf = ''; - + // private __buf = ''; constructor(options?: TextLineStreamOptions) { const allowCR = options?.allowCR ?? false; + let __buf = ''; + super({ - transform: (chunk, controller) => { - chunk = this.__buf + chunk; + transform(chunk, controller) { + chunk = __buf + chunk; for (; ;) { const lfIndex = chunk.indexOf('\n'); @@ -57,14 +58,14 @@ export class TextLineStream extends TransformStream { break; } - this.__buf = chunk; + __buf = chunk; }, - flush: (controller) => { - if (this.__buf.length > 0) { - if (allowCR && this.__buf[this.__buf.length - 1] === '\r') { - controller.enqueue(this.__buf.slice(0, -1)); + flush(controller) { + if (__buf.length > 0) { + if (allowCR && __buf[__buf.length - 1] === '\r') { + controller.enqueue(__buf.slice(0, -1)); } else { - controller.enqueue(this.__buf); + controller.enqueue(__buf); }; } }, diff --git a/bun.lockb b/bun.lockb index d69e15cd..dc6d9eba 100755 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/package.json b/package.json index 4b659458..a2e9e1e4 100644 --- a/package.json +++ b/package.json @@ -16,11 +16,11 @@ "dependencies": { "@cliqz/adblocker": "^1.26.12", "@sukka/listdir": "^0.2.0", - "@vercel/fetch-retry": "^5.1.3", + "async-retry": "^1.3.3", "async-sema": "^3.1.1", "ci-info": "^4.0.0", - "cidr-tools-wasm": "^0.0.13", - "eslint": "^8.53.0", + "cidr-tools-wasm": "^0.0.14", + "eslint": "^8.54.0", "gorhill-publicsuffixlist": "github:gorhill/publicsuffixlist.js", "mnemonist": "^0.39.5", "path-scurry": "^1.10.1", @@ -28,11 +28,12 @@ "punycode": "^2.3.1", "table": "^6.8.1", "tar": "^6.2.0", - "tldts": "^6.0.19" + "tldts": "^6.0.22" }, "devDependencies": { "@eslint-sukka/node": "4.1.9", "@eslint-sukka/ts": "4.1.9", + "@types/async-retry": "^1.4.8", "@types/mocha": "10.0.2", "@types/tar": "^6.1.9", "bun-types": "^1.0.11",