From a1098fbfcb31ece3131c7c94f7867d3cdb675446 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Thu, 22 Sep 2022 21:19:45 +0800 Subject: [PATCH] Chore: add fetch retry support --- Build/build-anti-bogus-domain.js | 4 ++-- Build/build-apple-cdn.js | 4 ++-- Build/build-cdn-conf.js | 4 ++-- Build/build-cidr.js | 4 ++-- Build/build-telegram-cidr.js | 4 ++-- Build/lib/fetch-retry.js | 3 +++ Build/lib/parse-filter.js | 8 +++---- List/domainset/cdn.conf | 1 + package.json | 1 + pnpm-lock.yaml | 40 ++++++++++++++++++++++++++++++++ 10 files changed, 59 insertions(+), 14 deletions(-) create mode 100644 Build/lib/fetch-retry.js diff --git a/Build/build-anti-bogus-domain.js b/Build/build-anti-bogus-domain.js index 390f307d..642d4a75 100644 --- a/Build/build-anti-bogus-domain.js +++ b/Build/build-anti-bogus-domain.js @@ -1,10 +1,10 @@ -const { fetch } = require('undici'); +const { fetchWithRetry } = require('./lib/fetch-retry'); const fs = require('fs'); const path = require('path'); const { isIP } = require('net'); (async () => { - const res = (await (await fetch('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/bogus-nxdomain.china.conf')).text()) + const res = (await (await fetchWithRetry('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/bogus-nxdomain.china.conf')).text()) .split('\n') .map(line => { if (line.startsWith('bogus-nxdomain=')) { diff --git a/Build/build-apple-cdn.js b/Build/build-apple-cdn.js index 77649fbf..e558d9c5 100644 --- a/Build/build-apple-cdn.js +++ b/Build/build-apple-cdn.js @@ -1,11 +1,11 @@ -const { fetch } = require('undici'); +const { fetchWithRetry } = require('./lib/fetch-retry'); const fs = require('fs'); const path = require('path'); const rDomain = /^(((?!\-))(xn\-\-)?[a-z0-9\-_]{0,61}[a-z0-9]{1,1}\.)*(xn\-\-)?([a-z0-9\-]{1,61}|[a-z0-9\-]{1,30})\.[a-z]{2,}$/m; (async () => { - const res = (await (await fetch('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf')).text()) + const res = (await (await fetchWithRetry('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf')).text()) .split('\n') .map(line => { if (line.startsWith('server=/') && line.endsWith('/114.114.114.114')) { diff --git a/Build/build-cdn-conf.js b/Build/build-cdn-conf.js index ddb208a9..50e9d495 100644 --- a/Build/build-cdn-conf.js +++ b/Build/build-cdn-conf.js @@ -1,9 +1,9 @@ -const { fetch } = require('undici'); +const { fetchWithRetry } = require('./lib/fetch-retry'); const fs = require('fs'); const path = require('path'); (async () => { - const domains = (await (await fetch('https://publicsuffix.org/list/public_suffix_list.dat')).text()).split('\n'); + const domains = (await (await fetchWithRetry('https://publicsuffix.org/list/public_suffix_list.dat')).text()).split('\n'); const S3OSSDomains = domains.filter(line => { if (line) { diff --git a/Build/build-cidr.js b/Build/build-cidr.js index 1a3d44a9..78947ad2 100644 --- a/Build/build-cidr.js +++ b/Build/build-cidr.js @@ -1,9 +1,9 @@ -const { fetch } = require('undici'); +const { fetchWithRetry } = require('./lib/fetch-retry'); const { promises: fsPromises } = require('fs'); const { resolve: pathResolve } = require('path'); (async () => { - const cidr = (await (await fetch('https://raw.githubusercontent.com/misakaio/chnroutes2/master/chnroutes.txt')).text()).split('\n'); + const cidr = (await (await fetchWithRetry('https://raw.githubusercontent.com/misakaio/chnroutes2/master/chnroutes.txt')).text()).split('\n'); const filteredCidr = cidr.filter(line => { if (line) { diff --git a/Build/build-telegram-cidr.js b/Build/build-telegram-cidr.js index cf54df94..c6900e2f 100644 --- a/Build/build-telegram-cidr.js +++ b/Build/build-telegram-cidr.js @@ -1,10 +1,10 @@ -const { fetch } = require('undici'); +const { fetchWithRetry } = require('./lib/fetch-retry'); const fs = require('fs'); const path = require('path'); const { isIPv4, isIPv6 } = require('net'); (async () => { - const resp = await fetch('https://core.telegram.org/resources/cidr.txt'); + const resp = await fetchWithRetry('https://core.telegram.org/resources/cidr.txt'); const lastModified = new Date(resp.headers.get('last-modified')); const res = (await resp.text()) diff --git a/Build/lib/fetch-retry.js b/Build/lib/fetch-retry.js new file mode 100644 index 00000000..13c8a8ee --- /dev/null +++ b/Build/lib/fetch-retry.js @@ -0,0 +1,3 @@ +const { fetch } = require('undici'); +const fetchWithRetry = require('@vercel/fetch-retry')(fetch); +module.exports.fetchWithRetry = fetchWithRetry; diff --git a/Build/lib/parse-filter.js b/Build/lib/parse-filter.js index e3d42144..6c6525e9 100644 --- a/Build/lib/parse-filter.js +++ b/Build/lib/parse-filter.js @@ -1,5 +1,5 @@ const { isIP } = require('net'); -const { fetch } = require('undici'); +const { fetchWithRetry } = require('./fetch-retry'); const rDomain = /^(((?!\-))(xn\-\-)?[a-z0-9\-_]{0,61}[a-z0-9]{1,1}\.)*(xn\-\-)?([a-z0-9\-]{1,61}|[a-z0-9\-]{1,30})\.[a-z]{2,}$/m @@ -26,7 +26,7 @@ async function processDomainLists (domainListsUrl) { /** @type Set */ const domainSets = new Set(); /** @type string[] */ - const domains = (await (await fetch(domainListsUrl)).text()).split('\n'); + const domains = (await (await fetchWithRetry(domainListsUrl)).text()).split('\n'); domains.forEach(line => { if ( line.startsWith('#') @@ -63,7 +63,7 @@ async function processHosts (hostsUrl, includeAllSubDomain = false) { const domainSets = new Set(); /** @type string[] */ - const hosts = (await (await fetch(hostsUrl)).text()).split('\n'); + const hosts = (await (await fetchWithRetry(hostsUrl)).text()).split('\n'); hosts.forEach(line => { if (line.includes('#')) { return; @@ -105,7 +105,7 @@ async function processFilterRules (filterRulesUrl) { const blacklistDomainSets = new Set(); /** @type string[] */ - const filterRules = (await (await fetch(filterRulesUrl)).text()).split('\n').map(line => line.trim()); + const filterRules = (await (await fetchWithRetry(filterRulesUrl)).text()).split('\n').map(line => line.trim()); filterRules.forEach(line => { const lineStartsWithDoubleVerticalBar = line.startsWith('||'); diff --git a/List/domainset/cdn.conf b/List/domainset/cdn.conf index b1316ec0..d7f64f10 100644 --- a/List/domainset/cdn.conf +++ b/List/domainset/cdn.conf @@ -769,3 +769,4 @@ img.perfops.net assets.ipstack.com vice-web-statics-cdn.vice.com video-images.vice.com +.operacdn.com diff --git a/package.json b/package.json index 6c5c4558..9471293c 100644 --- a/package.json +++ b/package.json @@ -57,6 +57,7 @@ "license": "ISC", "dependencies": { "@sukka/listdir": "^0.2.0", + "@vercel/fetch-retry": "^5.1.3", "ci-info": "^3.3.2", "picocolors": "^1.0.0", "piscina": "^3.2.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 359d48a0..be4d416b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2,6 +2,7 @@ lockfileVersion: 5.4 specifiers: '@sukka/listdir': ^0.2.0 + '@vercel/fetch-retry': ^5.1.3 ci-info: ^3.3.2 picocolors: ^1.0.0 piscina: ^3.2.0 @@ -11,6 +12,7 @@ specifiers: dependencies: '@sukka/listdir': 0.2.0 + '@vercel/fetch-retry': 5.1.3 ci-info: 3.3.2 picocolors: 1.0.0 piscina: 3.2.0 @@ -51,6 +53,17 @@ packages: resolution: {integrity: sha512-UyVirNhAOXKwjiDehjUaGtpfk0QwNHyiXrlLb/FmWMtI+BGhaEvB9MypSfEAtiiMI3g6QTfG38ayNAorEuz5ow==} dev: false + /@vercel/fetch-retry/5.1.3: + resolution: {integrity: sha512-UIbFc4VsEZHOr6dWuE+kxY4NxnOLXFMCWm0fSKRRHUEtrIzaJLzHpWk2QskCXTSzFgFvhkLAvSrBK2XZg7NSzg==} + peerDependencies: + node-fetch: ^2.6.7 + dependencies: + async-retry: 1.3.3 + debug: 4.3.4 + transitivePeerDependencies: + - supports-color + dev: false + /ajv/8.8.2: resolution: {integrity: sha512-x9VuX+R/jcFj1DHo/fCp99esgGDWiHENrKxaCENuCxpoMCmAt/COCGVDwA7kleEpEzJjDnvh3yGoOuLu0Dtllw==} dependencies: @@ -85,6 +98,12 @@ packages: engines: {node: '>=8'} dev: false + /async-retry/1.3.3: + resolution: {integrity: sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==} + dependencies: + retry: 0.13.1 + dev: false + /base64-js/1.5.1: resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} dev: false @@ -131,6 +150,18 @@ packages: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} dev: false + /debug/4.3.4: + resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + dependencies: + ms: 2.1.2 + dev: false + /emoji-regex/8.0.0: resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} dev: false @@ -252,6 +283,10 @@ packages: picomatch: 2.3.1 dev: true + /ms/2.1.2: + resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==} + dev: false + /nice-napi/1.0.2: resolution: {integrity: sha512-px/KnJAJZf5RuBGcfD+Sp2pAKq0ytz8j+1NehvgIGFkvtvFrDM3T8E4x/JJODXK9WZow8RRGrbA9QQ3hs+pDhA==} os: ['!win32'] @@ -335,6 +370,11 @@ packages: engines: {node: '>= 4'} dev: true + /retry/0.13.1: + resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} + engines: {node: '>= 4'} + dev: false + /reusify/1.0.4: resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==} engines: {iojs: '>=1.0.0', node: '>=0.10.0'}