GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2025-12-12 01:00:34 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Chore: housekeeping

Browse Source
This commit is contained in:
SukkaW 2023-12-24 21:51:18 +08:00
parent 5df053d93f
commit 9c6c0e1b19
9 changed files with 90 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

110
Build/build-internal-cdn-rules.ts
View File

@ -10,69 +10,69 @@ import { getGorhillPublicSuffixPromise } from './lib/get-gorhill-publicsuffix';
const escapeRegExp = (string = '') => string.replaceAll(/[$()*+.?[\\\]^{|}]/g, '\\$&');
const addApexDomain = (input: string, set: Set<string>) => {
// We are including the private domains themselves
const d = tldts.getDomain(input, { allowPrivateDomains: false });
if (d) {
set.add(d);
}
};
const processLocalDomainSet = async (domainSetPath: string, set: Set<string>) => {
for await (const line of readFileByLine(domainSetPath)) {
// console.log({ line });
const parsed = tldts.parse(line, { allowPrivateDomains: true, detectIp: false });
if (parsed.isIp) continue;
if (parsed.isIcann || parsed.isPrivate) {
if (parsed.domain) {
set.add(parsed.domain);
}
continue;
}
if (processLine(line)) {
console.warn('[drop line from domainset]', line);
}
}
};
const processLocalRuleSet = async (ruleSetPath: string, set: Set<string>, keywords: Set<string>) => {
for await (const line of readFileByLine(ruleSetPath)) {
if (line.startsWith('DOMAIN-SUFFIX,')) {
addApexDomain(line.replace('DOMAIN-SUFFIX,', ''), set);
} else if (line.startsWith('DOMAIN,')) {
addApexDomain(line.replace('DOMAIN,', ''), set);
} else if (line.startsWith('DOMAIN-KEYWORD')) {
keywords.add(escapeRegExp(line.replace('DOMAIN-KEYWORD,', '')));
} else if (line.startsWith('USER-AGENT,') || line.startsWith('PROCESS-NAME,') || line.startsWith('URL-REGEX,')) {
// do nothing
} else if (processLine(line)) {
console.warn('[drop line from ruleset]', line);
}
}
};
export const buildInternalCDNDomains = task(import.meta.path, async () => {
const set = new Set<string>();
const keywords = new Set<string>();
const addApexDomain = (input: string) => {
// We are including the private domains themselves
const d = tldts.getDomain(input, { allowPrivateDomains: false });
if (d) {
set.add(d);
}
};
const processLocalDomainSet = async (domainSetPath: string) => {
for await (const line of readFileByLine(domainSetPath)) {
// console.log({ line });
const parsed = tldts.parse(line, { allowPrivateDomains: true, detectIp: false });
if (parsed.isIp) continue;
if (parsed.isIcann || parsed.isPrivate) {
if (parsed.domain) {
set.add(parsed.domain);
}
continue;
}
if (processLine(line)) {
console.warn('[drop line from domainset]', line);
}
}
};
const processLocalRuleSet = async (ruleSetPath: string) => {
for await (const line of readFileByLine(ruleSetPath)) {
if (line.startsWith('DOMAIN-SUFFIX,')) {
addApexDomain(line.replace('DOMAIN-SUFFIX,', ''));
} else if (line.startsWith('DOMAIN,')) {
addApexDomain(line.replace('DOMAIN,', ''));
} else if (line.startsWith('DOMAIN-KEYWORD')) {
keywords.add(escapeRegExp(line.replace('DOMAIN-KEYWORD,', '')));
} else if (line.startsWith('USER-AGENT,') || line.startsWith('PROCESS-NAME,') || line.startsWith('URL-REGEX,')) {
// do nothing
} else if (processLine(line)) {
console.warn('[drop line from ruleset]', line);
}
}
};
const proxySet = new Set<string>();
const proxyKeywords = new Set<string>();
const gorhill = (await Promise.all([
getGorhillPublicSuffixPromise(),
processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/cdn.conf')),
processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/global.conf')),
processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/global_plus.conf')),
processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/my_proxy.conf')),
processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/stream.conf')),
processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/telegram.conf')),
processLocalDomainSet(path.resolve(import.meta.dir, '../List/domainset/cdn.conf')),
processLocalDomainSet(path.resolve(import.meta.dir, '../List/domainset/download.conf'))
processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/cdn.conf'), proxySet, proxyKeywords),
processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/global.conf'), proxySet, proxyKeywords),
processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/global_plus.conf'), proxySet, proxyKeywords),
processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/my_proxy.conf'), proxySet, proxyKeywords),
processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/stream.conf'), proxySet, proxyKeywords),
processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/telegram.conf'), proxySet, proxyKeywords),
processLocalDomainSet(path.resolve(import.meta.dir, '../List/domainset/cdn.conf'), proxySet),
processLocalDomainSet(path.resolve(import.meta.dir, '../List/domainset/download.conf'), proxySet)
]))[0];
return compareAndWriteFile(
[
...sortDomains(Array.from(set), gorhill).map(i => `SUFFIX,${i}`),
...Array.from(keywords).sort().map(i => `REGEX,${i}`)
...sortDomains(Array.from(proxySet), gorhill).map(i => `SUFFIX,${i}`),
...Array.from(proxyKeywords).sort().map(i => `REGEX,${i}`)
],
path.resolve(import.meta.dir, '../List/internal/cdn.txt')
);

1
Build/build-speedtest-domainset.ts
View File

@ -24,6 +24,7 @@ const querySpeedtestApi = async (keyword: string): Promise<Array<string | null>>
try {
const randomUserAgent = topUserAgents[Math.floor(Math.random() * topUserAgents.length)];
const key = `fetch speedtest endpoints: ${keyword}`;
console.log(key);
console.time(key);
const res = await fetchWithRetry(`https://www.speedtest.net/api/js/servers?engine=js&search=${keyword}&limit=100`, {

2
Build/build-sspanel-appprofile.ts
View File

@ -1,5 +1,3 @@
import fsp from 'fs/promises';
import { getAppleCdnDomainsPromise } from './build-apple-cdn';
import { getDomesticDomainsRulesetPromise } from './build-domestic-ruleset';
import { surgeRulesetToClashClassicalTextRuleset } from './lib/clash';

2
Build/build-telegram-cidr.ts
View File

@ -10,7 +10,7 @@ import { SHARED_DESCRIPTION } from './lib/constants';
import { createMemoizedPromise } from './lib/memo-promise';
export const getTelegramCIDRPromise = createMemoizedPromise(async () => {
const resp = await fetchWithRetry('https://core.telegram.org/resources/cidr.txt', defaultRequestInit) as Response;
const resp = await fetchWithRetry('https://core.telegram.org/resources/cidr.txt', defaultRequestInit);
const lastModified = resp.headers.get('last-modified');
const date = lastModified ? new Date(lastModified) : new Date();

2
Build/download-mock-assets.ts
View File

@ -18,7 +18,7 @@ export const downloadMockAssets = task(import.meta.path, () => Promise.all(Objec
const key = picocolors.gray(`Download ${filename}`);
console.time(key);
const res = await fetchWithRetry(url);
await Bun.write(targetPath, res as Response);
await Bun.write(targetPath, res);
console.timeEnd(key);
})));

55
Build/download-previous-build.ts
View File

@ -48,40 +48,37 @@ export const downloadPreviousBuild = task(import.meta.path, async () => {
const filesList = buildOutputList.map(f => path.join('ruleset.skk.moe-master', f));
await traceAsync(
'Download and extract previous build',
async () => {
const resp = await fetchWithRetry('https://codeload.github.com/sukkalab/ruleset.skk.moe/tar.gz/master', defaultRequestInit);
const resp = await fetchWithRetry('https://codeload.github.com/sukkalab/ruleset.skk.moe/tar.gz/master', defaultRequestInit);
if (!resp.body) {
throw new Error('Download previous build failed! No body found');
}
if (!resp.body) {
throw new Error('Download previous build failed! No body found');
}
const extract = tarStream.extract();
Readable.fromWeb(resp.body).pipe(zlib.createGunzip()).pipe(extract);
const extract = tarStream.extract();
Readable.fromWeb(resp.body as any).pipe(zlib.createGunzip()).pipe(extract);
for await (const entry of extract) {
if (entry.header.type !== 'file') {
entry.resume(); // Drain the entry
continue;
}
// filter entry
if (!filesList.some(f => entry.header.name.startsWith(f))) {
entry.resume(); // Drain the entry
continue;
}
const pathPrefix = `ruleset.skk.moe-master${path.sep}`;
const relativeEntryPath = entry.header.name.replace(`ruleset.skk.moe-master${path.sep}`, '');
const targetPath = path.join(import.meta.dir, '..', relativeEntryPath);
await fsp.mkdir(path.dirname(targetPath), { recursive: true });
await pipeline(
entry,
fs.createWriteStream(targetPath)
);
}
for await (const entry of extract) {
if (entry.header.type !== 'file') {
entry.resume(); // Drain the entry
continue;
}
);
// filter entry
if (!filesList.some(f => entry.header.name.startsWith(f))) {
entry.resume(); // Drain the entry
continue;
}
const relativeEntryPath = entry.header.name.replace(pathPrefix, '');
const targetPath = path.join(import.meta.dir, '..', relativeEntryPath);
await fsp.mkdir(path.dirname(targetPath), { recursive: true });
await pipeline(
entry,
fs.createWriteStream(targetPath)
);
}
});
if (import.meta.main) {

6
Build/lib/fetch-retry.ts
View File

@ -60,7 +60,8 @@ function createFetchRetry($fetch: typeof fetch): FetchWithRetry {
retries: MAX_RETRIES,
factor: FACTOR,
maxRetryAfter: MAX_RETRY_AFTER,
retryOnAborted: false
retryOnAborted: false,
retryOnNon2xx: true
},
opts.retry
);
@ -85,6 +86,9 @@ function createFetchRetry($fetch: typeof fetch): FetchWithRetry {
}
throw new ResponseError(res);
} else {
if (!res.ok && retryOpts.retryOnNon2xx) {
throw new ResponseError(res);
}
return res;
}
} catch (err: unknown) {

2
Build/lib/fetch-text-by-line.ts
View File

@ -79,5 +79,5 @@ export async function *createReadlineInterfaceFromResponse(resp: Response): Asyn
}
export function fetchRemoteTextByLine(url: string | URL) {
return fetchWithRetry(url, defaultRequestInit).then(res => createReadlineInterfaceFromResponse(res as Response));
return fetchWithRetry(url, defaultRequestInit).then(res => createReadlineInterfaceFromResponse(res));
}

2
Build/lib/get-phishing-domains.ts
View File

@ -1,5 +1,3 @@
import fsp from 'fs/promises';
import path from 'path';
import { getGorhillPublicSuffixPromise } from './get-gorhill-publicsuffix';
import { processDomainLists, processHosts } from './parse-filter';
import { traceAsync, traceSync } from './trace-runner';