GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2026-01-29 01:51:52 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Chore: add fetch UA

Browse Source
This commit is contained in:
SukkaW
2023-11-20 15:45:44 +08:00
parent 5ea91fa9a4
commit 0bdc69523a
8 changed files with 25 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
Build/lib/fetch-remote-text-by-line.ts
View File

@@ -1,5 +1,5 @@
import type { BunFile } from 'bun';
import { fetchWithRetry } from './fetch-retry';
import { fetchWithRetry, defaultRequestInit } from './fetch-retry';
const decoder = new TextDecoder('utf-8');
@@ -56,6 +56,6 @@ export async function* createReadlineInterfaceFromResponse(resp: Response): Asyn
}
}
export function fetchRemoteTextAndCreateReadlineInterface(url: string | URL, opt?: RequestInit): Promise<AsyncGenerator<string>> {
return fetchWithRetry(url, opt).then(res => createReadlineInterfaceFromResponse(res));
export function fetchRemoteTextAndCreateReadlineInterface(url: string | URL): Promise<AsyncGenerator<string>> {
return fetchWithRetry(url, defaultRequestInit).then(res => createReadlineInterfaceFromResponse(res));
}

6
Build/lib/fetch-retry.ts
View File

@@ -1,4 +1,10 @@
// @ts-expect-error -- missing types
import createFetchRetry from '@vercel/fetch-retry';
export const defaultRequestInit: RequestInit = {
headers: {
'User-Agent': 'curl/8.1.2 (https://github.com/SukkaW/Surge)'
}
}
export const fetchWithRetry: typeof fetch = createFetchRetry(fetch);

3
Build/lib/get-gorhill-publicsuffix.ts
View File

@@ -1,6 +1,7 @@
import { toASCII } from 'punycode';
import path from 'path';
import { traceAsync } from './trace-runner';
import { defaultRequestInit, fetchWithRetry } from './fetch-retry';
import type { PublicSuffixList } from 'gorhill-publicsuffixlist';
const publicSuffixPath = path.resolve(__dirname, '../../node_modules/.cache/public_suffix_list_dat.txt');
@@ -13,7 +14,7 @@ const getGorhillPublicSuffix = () => traceAsync('create gorhill public suffix in
const [publicSuffixListDat, { default: gorhill }] = await Promise.all([
await publicSuffixFile.exists()
? publicSuffixFile.text()
: fetch('https://publicsuffix.org/list/public_suffix_list.dat').then(r => {
: fetchWithRetry('https://publicsuffix.org/list/public_suffix_list.dat', defaultRequestInit).then(r => {
console.log('public_suffix_list.dat not found, fetch directly from remote.');
return r.text();
}),

4
Build/lib/parse-filter.ts
View File

@@ -1,5 +1,5 @@
// @ts-check
import { fetchWithRetry } from './fetch-retry';
import { defaultRequestInit, fetchWithRetry } from './fetch-retry';
import * as tldts from './cached-tld-parse';
import { fetchRemoteTextAndCreateReadlineInterface } from './fetch-remote-text-by-line';
import { NetworkFilter } from '@cliqz/adblocker';
@@ -198,7 +198,7 @@ export async function processFilterRules(
filterRules = (
await Promise.any(
[filterRulesUrl, ...(fallbackUrls || [])].map(async url => {
const r = await fetchWithRetry(url, { signal: controller.signal });
const r = await fetchWithRetry(url, { signal: controller.signal, ...defaultRequestInit });
const text = await r.text();
controller.abort();

5
Build/lib/reject-data-source.ts
View File

@@ -106,9 +106,10 @@ export const ADGUARD_FILTERS = [
[
'https://curbengh.github.io/urlhaus-filter/urlhaus-filter-agh-online.txt',
[
'https://urlhaus-filter.pages.dev/urlhaus-filter-agh-online.txt'
'https://urlhaus-filter.pages.dev/urlhaus-filter-agh-online.txt',
// Prefer mirror, since malware-filter.gitlab.io has not been updated for a while
// 'https://malware-filter.gitlab.io/urlhaus-filter/urlhaus-filter-agh-online.txt'
// 'https://malware-filter.gitlab.io/urlhaus-filter/urlhaus-filter-agh-online.txt',
'https://ublockorigin.github.io/uAssets/thirdparties/urlhaus-filter/urlhaus-filter-online.txt',
]
],
// Curben's Phishing URL Blocklist