GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2025-12-14 02:00:37 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Refactor: merge new output

Browse Source
This commit is contained in:
SukkaW 2024-09-21 04:22:08 +08:00
parent 76aa3a78b4
commit 9eec31be10
15 changed files with 461 additions and 598 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Build/build-apple-cdn.ts
View File

@ -3,7 +3,7 @@ import { task } from './trace';
import { SHARED_DESCRIPTION } from './lib/constants'; import { SHARED_DESCRIPTION } from './lib/constants';
import { createMemoizedPromise } from './lib/memo-promise'; import { createMemoizedPromise } from './lib/memo-promise';
import { TTL, deserializeArray, fsFetchCache, serializeArray, createCacheKey } from './lib/cache-filesystem'; import { TTL, deserializeArray, fsFetchCache, serializeArray, createCacheKey } from './lib/cache-filesystem';
import { DomainsetOutput } from './lib/create-file-new'; import { DomainsetOutput } from './lib/create-file';
const cacheKey = createCacheKey(__filename); const cacheKey = createCacheKey(__filename);

2
Build/build-cdn-download-conf.ts
View File

@ -7,7 +7,7 @@ import { getPublicSuffixListTextPromise } from './lib/download-publicsuffixlist'
import { appendArrayInPlace } from './lib/append-array-in-place'; import { appendArrayInPlace } from './lib/append-array-in-place';
import { SOURCE_DIR } from './constants/dir'; import { SOURCE_DIR } from './constants/dir';
import { processLine } from './lib/process-line'; import { processLine } from './lib/process-line';
import { DomainsetOutput } from './lib/create-file-new'; import { DomainsetOutput } from './lib/create-file';
const getS3OSSDomainsPromise = (async (): Promise<string[]> => { const getS3OSSDomainsPromise = (async (): Promise<string[]> => {
const trie = createTrie( const trie = createTrie(

2
Build/build-chn-cidr.ts
View File

@ -6,7 +6,7 @@ import { exclude } from 'fast-cidr-tools';
import { createMemoizedPromise } from './lib/memo-promise'; import { createMemoizedPromise } from './lib/memo-promise';
import { CN_CIDR_NOT_INCLUDED_IN_CHNROUTE, NON_CN_CIDR_INCLUDED_IN_CHNROUTE } from './constants/cidr'; import { CN_CIDR_NOT_INCLUDED_IN_CHNROUTE, NON_CN_CIDR_INCLUDED_IN_CHNROUTE } from './constants/cidr';
import { appendArrayInPlace } from './lib/append-array-in-place'; import { appendArrayInPlace } from './lib/append-array-in-place';
import { IPListOutput } from './lib/create-file-new'; import { IPListOutput } from './lib/create-file';
export const getChnCidrPromise = createMemoizedPromise(async () => { export const getChnCidrPromise = createMemoizedPromise(async () => {
const cidr4 = await processLineFromReadline(await fetchRemoteTextByLine('https://raw.githubusercontent.com/misakaio/chnroutes2/master/chnroutes.txt')); const cidr4 = await processLineFromReadline(await fetchRemoteTextByLine('https://raw.githubusercontent.com/misakaio/chnroutes2/master/chnroutes.txt'));

2
Build/build-cloudmounter-rules.ts
View File

@ -1,7 +1,7 @@
import { DOMAINS, PROCESS_NAMES } from '../Source/non_ip/cloudmounter'; import { DOMAINS, PROCESS_NAMES } from '../Source/non_ip/cloudmounter';
import { SHARED_DESCRIPTION } from './lib/constants'; import { SHARED_DESCRIPTION } from './lib/constants';
import { task } from './trace'; import { task } from './trace';
import { RulesetOutput } from './lib/create-file-new'; import { RulesetOutput } from './lib/create-file';
export const buildCloudMounterRules = task(require.main === module, __filename)(async (span) => { export const buildCloudMounterRules = task(require.main === module, __filename)(async (span) => {
// AND,((SRC-IP,192.168.1.110), (DOMAIN, example.com)) // AND,((SRC-IP,192.168.1.110), (DOMAIN, example.com))

2
Build/build-common.ts
View File

@ -9,7 +9,7 @@ import { SHARED_DESCRIPTION } from './lib/constants';
import { fdir as Fdir } from 'fdir'; import { fdir as Fdir } from 'fdir';
import { appendArrayInPlace } from './lib/append-array-in-place'; import { appendArrayInPlace } from './lib/append-array-in-place';
import { SOURCE_DIR } from './constants/dir'; import { SOURCE_DIR } from './constants/dir';
import { DomainsetOutput, RulesetOutput } from './lib/create-file-new'; import { DomainsetOutput, RulesetOutput } from './lib/create-file';
const MAGIC_COMMAND_SKIP = '# $ custom_build_script'; const MAGIC_COMMAND_SKIP = '# $ custom_build_script';
const MAGIC_COMMAND_TITLE = '# $ meta_title '; const MAGIC_COMMAND_TITLE = '# $ meta_title ';

2
Build/build-domestic-direct-lan-ruleset-dns-mapping-module.ts
View File

@ -11,7 +11,7 @@ import * as yaml from 'yaml';
import { appendArrayInPlace } from './lib/append-array-in-place'; import { appendArrayInPlace } from './lib/append-array-in-place';
import { writeFile } from './lib/misc'; import { writeFile } from './lib/misc';
import { OUTPUT_INTERNAL_DIR, OUTPUT_MODULES_DIR, SOURCE_DIR } from './constants/dir'; import { OUTPUT_INTERNAL_DIR, OUTPUT_MODULES_DIR, SOURCE_DIR } from './constants/dir';
import { RulesetOutput } from './lib/create-file-new'; import { RulesetOutput } from './lib/create-file';
export const getDomesticAndDirectDomainsRulesetPromise = createMemoizedPromise(async () => { export const getDomesticAndDirectDomainsRulesetPromise = createMemoizedPromise(async () => {
const domestics = await readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/domestic.conf')); const domestics = await readFileIntoProcessedArray(path.join(SOURCE_DIR, 'non_ip/domestic.conf'));

2
Build/build-microsoft-cdn.ts
View File

@ -4,7 +4,7 @@ import { createTrie } from './lib/trie';
import { SHARED_DESCRIPTION } from './lib/constants'; import { SHARED_DESCRIPTION } from './lib/constants';
import { createMemoizedPromise } from './lib/memo-promise'; import { createMemoizedPromise } from './lib/memo-promise';
import { extractDomainsFromFelixDnsmasq } from './lib/parse-dnsmasq'; import { extractDomainsFromFelixDnsmasq } from './lib/parse-dnsmasq';
import { RulesetOutput } from './lib/create-file-new'; import { RulesetOutput } from './lib/create-file';
const PROBE_DOMAINS = ['.microsoft.com', '.windows.net', '.windows.com', '.windowsupdate.com', '.windowssearch.com', '.office.net']; const PROBE_DOMAINS = ['.microsoft.com', '.windows.net', '.windows.com', '.windowsupdate.com', '.windowssearch.com', '.office.net'];

2
Build/build-reject-domainset.ts
View File

@ -17,7 +17,7 @@ import { getPhishingDomains } from './lib/get-phishing-domains';
import { setAddFromArray } from './lib/set-add-from-array'; import { setAddFromArray } from './lib/set-add-from-array';
import { appendArrayInPlace } from './lib/append-array-in-place'; import { appendArrayInPlace } from './lib/append-array-in-place';
import { OUTPUT_INTERNAL_DIR, SOURCE_DIR } from './constants/dir'; import { OUTPUT_INTERNAL_DIR, SOURCE_DIR } from './constants/dir';
import { DomainsetOutput } from './lib/create-file-new'; import { DomainsetOutput } from './lib/create-file';
const getRejectSukkaConfPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/reject_sukka.conf')); const getRejectSukkaConfPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/reject_sukka.conf'));

2
Build/build-reject-ip-list.ts
View File

@ -7,7 +7,7 @@ import { isProbablyIpv4, isProbablyIpv6 } from './lib/is-fast-ip';
import { TTL, fsFetchCache, createCacheKey } from './lib/cache-filesystem'; import { TTL, fsFetchCache, createCacheKey } from './lib/cache-filesystem';
import { fetchAssets } from './lib/fetch-assets'; import { fetchAssets } from './lib/fetch-assets';
import { processLine } from './lib/process-line'; import { processLine } from './lib/process-line';
import { RulesetOutput } from './lib/create-file-new'; import { RulesetOutput } from './lib/create-file';
import { SOURCE_DIR } from './constants/dir'; import { SOURCE_DIR } from './constants/dir';
const cacheKey = createCacheKey(__filename); const cacheKey = createCacheKey(__filename);

2
Build/build-speedtest-domainset.ts
View File

@ -8,7 +8,7 @@ import { SHARED_DESCRIPTION } from './lib/constants';
import { readFileIntoProcessedArray } from './lib/fetch-text-by-line'; import { readFileIntoProcessedArray } from './lib/fetch-text-by-line';
import { TTL, deserializeArray, fsFetchCache, serializeArray, createCacheKey } from './lib/cache-filesystem'; import { TTL, deserializeArray, fsFetchCache, serializeArray, createCacheKey } from './lib/cache-filesystem';
import { DomainsetOutput } from './lib/create-file-new'; import { DomainsetOutput } from './lib/create-file';
import { OUTPUT_SURGE_DIR } from './constants/dir'; import { OUTPUT_SURGE_DIR } from './constants/dir';
const KEYWORDS = [ const KEYWORDS = [

2
Build/build-stream-service.ts
View File

@ -4,7 +4,7 @@ import { task } from './trace';
import { ALL, NORTH_AMERICA, EU, HK, TW, JP, KR } from '../Source/stream'; import { ALL, NORTH_AMERICA, EU, HK, TW, JP, KR } from '../Source/stream';
import { SHARED_DESCRIPTION } from './lib/constants'; import { SHARED_DESCRIPTION } from './lib/constants';
import { RulesetOutput } from './lib/create-file-new'; import { RulesetOutput } from './lib/create-file';
export const createRulesetForStreamService = (span: Span, fileId: string, title: string, streamServices: Array<import('../Source/stream').StreamService>) => { export const createRulesetForStreamService = (span: Span, fileId: string, title: string, streamServices: Array<import('../Source/stream').StreamService>) => {
return span.traceChildAsync(fileId, async (childSpan) => Promise.all([ return span.traceChildAsync(fileId, async (childSpan) => Promise.all([

2
Build/build-telegram-cidr.ts
View File

@ -6,7 +6,7 @@ import { processLine } from './lib/process-line';
import { task } from './trace'; import { task } from './trace';
import { SHARED_DESCRIPTION } from './lib/constants'; import { SHARED_DESCRIPTION } from './lib/constants';
import { createMemoizedPromise } from './lib/memo-promise'; import { createMemoizedPromise } from './lib/memo-promise';
import { RulesetOutput } from './lib/create-file-new'; import { RulesetOutput } from './lib/create-file';
export const getTelegramCIDRPromise = createMemoizedPromise(async () => { export const getTelegramCIDRPromise = createMemoizedPromise(async () => {
const resp = await fetchWithRetry('https://core.telegram.org/resources/cidr.txt', defaultRequestInit); const resp = await fetchWithRetry('https://core.telegram.org/resources/cidr.txt', defaultRequestInit);

409
Build/lib/create-file-new.ts
View File

@ -1,409 +0,0 @@
import path from 'node:path';
import type { Span } from '../trace';
import { surgeDomainsetToClashDomainset, surgeRulesetToClashClassicalTextRuleset } from './clash';
import { compareAndWriteFile, defaultSortTypeOrder, sortTypeOrder, withBannerArray } from './create-file';
import { ipCidrListToSingbox, surgeDomainsetToSingbox, surgeRulesetToSingbox } from './singbox';
import { buildParseDomainMap, sortDomains } from './stable-sort-domain';
import { createTrie } from './trie';
import { invariant } from 'foxact/invariant';
import { OUTPUT_CLASH_DIR, OUTPUT_SINGBOX_DIR, OUTPUT_SURGE_DIR } from '../constants/dir';
import stringify from 'json-stringify-pretty-compact';
import { appendArrayInPlace } from './append-array-in-place';
import { nullthrow } from 'foxact/nullthrow';
import createKeywordFilter from './aho-corasick';
abstract class RuleOutput {
protected domainTrie = createTrie<unknown>(null, true);
protected domainKeywords = new Set<string>();
protected domainWildcard = new Set<string>();
protected ipcidr = new Set<string>();
protected ipcidrNoResolve = new Set<string>();
protected ipcidr6 = new Set<string>();
protected ipcidr6NoResolve = new Set<string>();
protected otherRules: Array<[raw: string, orderWeight: number]> = [];
protected abstract type: 'domainset' | 'non_ip' | 'ip';
protected pendingPromise = Promise.resolve();
static jsonToLines(this: void, json: unknown): string[] {
return stringify(json).split('\n');
}
constructor(
protected readonly span: Span,
protected readonly id: string
) {}
protected title: string | null = null;
withTitle(title: string) {
this.title = title;
return this;
}
protected description: string[] | readonly string[] | null = null;
withDescription(description: string[] | readonly string[]) {
this.description = description;
return this;
}
protected date = new Date();
withDate(date: Date) {
this.date = date;
return this;
}
protected apexDomainMap: Map<string, string> | null = null;
protected subDomainMap: Map<string, string> | null = null;
withDomainMap(apexDomainMap: Map<string, string>, subDomainMap: Map<string, string>) {
this.apexDomainMap = apexDomainMap;
this.subDomainMap = subDomainMap;
return this;
}
addDomain(domain: string) {
this.domainTrie.add(domain);
return this;
}
addDomainSuffix(domain: string) {
this.domainTrie.add(domain[0] === '.' ? domain : '.' + domain);
return this;
}
bulkAddDomainSuffix(domains: string[]) {
for (let i = 0, len = domains.length; i < len; i++) {
this.addDomainSuffix(domains[i]);
}
return this;
}
addDomainKeyword(keyword: string) {
this.domainKeywords.add(keyword);
return this;
}
addDomainWildcard(wildcard: string) {
this.domainWildcard.add(wildcard);
return this;
}
private async addFromDomainsetPromise(source: AsyncIterable<string> | Iterable<string> | string[]) {
for await (const line of source) {
if (line[0] === '.') {
this.addDomainSuffix(line);
} else {
this.addDomain(line);
}
}
}
addFromDomainset(source: AsyncIterable<string> | Iterable<string> | string[]) {
this.pendingPromise = this.pendingPromise.then(() => this.addFromDomainsetPromise(source));
return this;
}
private async addFromRulesetPromise(source: AsyncIterable<string> | Iterable<string>) {
for await (const line of source) {
const splitted = line.split(',');
const type = splitted[0];
const value = splitted[1];
const arg = splitted[2];
switch (type) {
case 'DOMAIN':
this.addDomain(value);
break;
case 'DOMAIN-SUFFIX':
this.addDomainSuffix(value);
break;
case 'DOMAIN-KEYWORD':
this.addDomainKeyword(value);
break;
case 'DOMAIN-WILDCARD':
this.addDomainWildcard(value);
break;
case 'IP-CIDR':
(arg === 'no-resolve' ? this.ipcidrNoResolve : this.ipcidr).add(value);
break;
case 'IP-CIDR6':
(arg === 'no-resolve' ? this.ipcidr6NoResolve : this.ipcidr6).add(value);
break;
default:
this.otherRules.push([line, type in sortTypeOrder ? sortTypeOrder[type] : sortTypeOrder[defaultSortTypeOrder]]);
break;
}
}
}
addFromRuleset(source: AsyncIterable<string> | Iterable<string>) {
this.pendingPromise = this.pendingPromise.then(() => this.addFromRulesetPromise(source));
return this;
}
bulkAddCIDR4(cidr: string[]) {
for (let i = 0, len = cidr.length; i < len; i++) {
this.ipcidr.add(cidr[i]);
}
return this;
}
bulkAddCIDR4NoResolve(cidr: string[]) {
for (let i = 0, len = cidr.length; i < len; i++) {
this.ipcidrNoResolve.add(cidr[i]);
}
return this;
}
bulkAddCIDR6(cidr: string[]) {
for (let i = 0, len = cidr.length; i < len; i++) {
this.ipcidr6.add(cidr[i]);
}
return this;
}
bulkAddCIDR6NoResolve(cidr: string[]) {
for (let i = 0, len = cidr.length; i < len; i++) {
this.ipcidr6NoResolve.add(cidr[i]);
}
return this;
}
abstract write(): Promise<void>;
}
export class DomainsetOutput extends RuleOutput {
protected type = 'domainset' as const;
private $dumped: string[] | null = null;
get dumped() {
if (!this.$dumped) {
const kwfilter = createKeywordFilter(this.domainKeywords);
const results: string[] = [];
const dumped = this.domainTrie.dump();
for (let i = 0, len = dumped.length; i < len; i++) {
const domain = dumped[i];
if (!kwfilter(domain)) {
results.push(domain);
}
}
this.$dumped = results;
}
return this.$dumped;
}
calcDomainMap() {
if (!this.apexDomainMap || !this.subDomainMap) {
const { domainMap, subdomainMap } = buildParseDomainMap(this.dumped);
this.apexDomainMap = domainMap;
this.subDomainMap = subdomainMap;
}
}
async write() {
await this.pendingPromise;
invariant(this.title, 'Missing title');
invariant(this.description, 'Missing description');
const sorted = sortDomains(this.dumped, this.apexDomainMap, this.subDomainMap);
sorted.push('this_ruleset_is_made_by_sukkaw.ruleset.skk.moe');
const surge = sorted;
const clash = surgeDomainsetToClashDomainset(sorted);
// TODO: Implement singbox directly using data
const singbox = RuleOutput.jsonToLines(surgeDomainsetToSingbox(sorted));
await Promise.all([
compareAndWriteFile(
this.span,
withBannerArray(
this.title,
this.description,
this.date,
surge
),
path.join(OUTPUT_SURGE_DIR, this.type, this.id + '.conf')
),
compareAndWriteFile(
this.span,
withBannerArray(
this.title,
this.description,
this.date,
clash
),
path.join(OUTPUT_CLASH_DIR, this.type, this.id + '.txt')
),
compareAndWriteFile(
this.span,
singbox,
path.join(OUTPUT_SINGBOX_DIR, this.type, this.id + '.json')
)
]);
}
getStatMap() {
invariant(this.dumped, 'Non dumped yet');
invariant(this.apexDomainMap, 'Missing apex domain map');
return Array.from(
(
nullthrow(this.dumped, 'Non dumped yet').reduce<Map<string, number>>((acc, cur) => {
const suffix = this.apexDomainMap!.get(cur);
if (suffix) {
acc.set(suffix, (acc.get(suffix) ?? 0) + 1);
}
return acc;
}, new Map())
).entries()
)
.filter(a => a[1] > 9)
.sort(
(a, b) => (b[1] - a[1]) || a[0].localeCompare(b[0])
)
.map(([domain, count]) => `${domain}${' '.repeat(100 - domain.length)}${count}`);
}
}
export class IPListOutput extends RuleOutput {
protected type = 'ip' as const;
constructor(span: Span, id: string, private readonly clashUseRule = true) {
super(span, id);
}
async write() {
await this.pendingPromise;
invariant(this.title, 'Missing title');
invariant(this.description, 'Missing description');
const sorted4 = Array.from(this.ipcidr);
const sorted6 = Array.from(this.ipcidr6);
const merged = appendArrayInPlace(appendArrayInPlace([], sorted4), sorted6);
const surge = sorted4.map(i => `IP-CIDR,${i}`);
appendArrayInPlace(surge, sorted6.map(i => `IP-CIDR6,${i}`));
surge.push('DOMAIN,this_ruleset_is_made_by_sukkaw.ruleset.skk.moe');
const clash = this.clashUseRule ? surge : merged;
// TODO: Implement singbox directly using data
const singbox = RuleOutput.jsonToLines(ipCidrListToSingbox(merged));
await Promise.all([
compareAndWriteFile(
this.span,
withBannerArray(
this.title,
this.description,
this.date,
surge
),
path.join(OUTPUT_SURGE_DIR, this.type, this.id + '.conf')
),
compareAndWriteFile(
this.span,
withBannerArray(
this.title,
this.description,
this.date,
clash
),
path.join(OUTPUT_CLASH_DIR, this.type, this.id + '.txt')
),
compareAndWriteFile(
this.span,
singbox,
path.join(OUTPUT_SINGBOX_DIR, this.type, this.id + '.json')
)
]);
}
}
export class RulesetOutput extends RuleOutput {
constructor(span: Span, id: string, protected type: 'non_ip' | 'ip') {
super(span, id);
}
async write() {
await this.pendingPromise;
invariant(this.title, 'Missing title');
invariant(this.description, 'Missing description');
const results: string[] = [
'DOMAIN,this_ruleset_is_made_by_sukkaw.ruleset.skk.moe'
];
const kwfilter = createKeywordFilter(this.domainKeywords);
const sortedDomains = sortDomains(this.domainTrie.dump(), this.apexDomainMap, this.subDomainMap);
for (let i = 0, len = sortedDomains.length; i < len; i++) {
const domain = sortedDomains[i];
if (kwfilter(domain)) {
continue;
}
if (domain[0] === '.') {
results.push(`DOMAIN-SUFFIX,${domain.slice(1)}`);
} else {
results.push(`DOMAIN,${domain}`);
}
}
for (const keyword of this.domainKeywords) {
results.push(`DOMAIN-KEYWORD,${keyword}`);
}
for (const wildcard of this.domainWildcard) {
results.push(`DOMAIN-WILDCARD,${wildcard}`);
}
const sortedRules = this.otherRules.sort((a, b) => a[1] - b[1]);
for (let i = 0, len = sortedRules.length; i < len; i++) {
results.push(sortedRules[i][0]);
}
this.ipcidr.forEach(cidr => results.push(`IP-CIDR,${cidr}`));
this.ipcidrNoResolve.forEach(cidr => results.push(`IP-CIDR,${cidr},no-resolve`));
this.ipcidr6.forEach(cidr => results.push(`IP-CIDR6,${cidr}`));
this.ipcidr6NoResolve.forEach(cidr => results.push(`IP-CIDR6,${cidr},no-resolve`));
const surge = results;
const clash = surgeRulesetToClashClassicalTextRuleset(results);
// TODO: Implement singbox directly using data
const singbox = RuleOutput.jsonToLines(surgeRulesetToSingbox(results));
await Promise.all([
compareAndWriteFile(
this.span,
withBannerArray(
this.title,
this.description,
this.date,
surge
),
path.join(OUTPUT_SURGE_DIR, this.type, this.id + '.conf')
),
compareAndWriteFile(
this.span,
withBannerArray(
this.title,
this.description,
this.date,
clash
),
path.join(OUTPUT_CLASH_DIR, this.type, this.id + '.txt')
),
compareAndWriteFile(
this.span,
singbox,
path.join(OUTPUT_SINGBOX_DIR, this.type, this.id + '.json')
)
]);
}
}

616
Build/lib/create-file.ts
View File

@ -1,17 +1,448 @@
// @ts-check
import { surgeDomainsetToClashDomainset, surgeRulesetToClashClassicalTextRuleset } from './clash';
import picocolors from 'picocolors';
import type { Span } from '../trace';
import path from 'node:path'; import path from 'node:path';
import type { Span } from '../trace';
import { surgeDomainsetToClashDomainset, surgeRulesetToClashClassicalTextRuleset } from './clash';
import { ipCidrListToSingbox, surgeDomainsetToSingbox, surgeRulesetToSingbox } from './singbox';
import { buildParseDomainMap, sortDomains } from './stable-sort-domain';
import { createTrie } from './trie';
import { invariant } from 'foxact/invariant';
import { OUTPUT_CLASH_DIR, OUTPUT_SINGBOX_DIR, OUTPUT_SURGE_DIR } from '../constants/dir';
import stringify from 'json-stringify-pretty-compact';
import { appendArrayInPlace } from './append-array-in-place';
import { nullthrow } from 'foxact/nullthrow';
import createKeywordFilter from './aho-corasick';
import picocolors from 'picocolors';
import fs from 'node:fs'; import fs from 'node:fs';
import { fastStringArrayJoin, writeFile } from './misc'; import { fastStringArrayJoin, writeFile } from './misc';
import { readFileByLine } from './fetch-text-by-line'; import { readFileByLine } from './fetch-text-by-line';
import stringify from 'json-stringify-pretty-compact';
import { surgeDomainsetToSingbox, surgeRulesetToSingbox } from './singbox';
import { createTrie } from './trie';
import { pack, unpackFirst, unpackSecond } from './bitwise';
import { asyncWriteToStream } from './async-write-to-stream'; import { asyncWriteToStream } from './async-write-to-stream';
const defaultSortTypeOrder = Symbol('defaultSortTypeOrder');
const sortTypeOrder: Record<string | typeof defaultSortTypeOrder, number> = {
DOMAIN: 1,
'DOMAIN-SUFFIX': 2,
'DOMAIN-KEYWORD': 10,
// experimental domain wildcard support
'DOMAIN-WILDCARD': 20,
'DOMAIN-REGEX': 21,
'USER-AGENT': 30,
'PROCESS-NAME': 40,
[defaultSortTypeOrder]: 50, // default sort order for unknown type
'URL-REGEX': 100,
AND: 300,
OR: 300,
'IP-CIDR': 400,
'IP-CIDR6': 400
};
abstract class RuleOutput {
protected domainTrie = createTrie<unknown>(null, true);
protected domainKeywords = new Set<string>();
protected domainWildcard = new Set<string>();
protected ipcidr = new Set<string>();
protected ipcidrNoResolve = new Set<string>();
protected ipcidr6 = new Set<string>();
protected ipcidr6NoResolve = new Set<string>();
protected otherRules: Array<[raw: string, orderWeight: number]> = [];
protected abstract type: 'domainset' | 'non_ip' | 'ip';
protected pendingPromise = Promise.resolve();
static jsonToLines(this: void, json: unknown): string[] {
return stringify(json).split('\n');
}
constructor(
protected readonly span: Span,
protected readonly id: string
) {}
protected title: string | null = null;
withTitle(title: string) {
this.title = title;
return this;
}
protected description: string[] | readonly string[] | null = null;
withDescription(description: string[] | readonly string[]) {
this.description = description;
return this;
}
protected date = new Date();
withDate(date: Date) {
this.date = date;
return this;
}
protected apexDomainMap: Map<string, string> | null = null;
protected subDomainMap: Map<string, string> | null = null;
withDomainMap(apexDomainMap: Map<string, string>, subDomainMap: Map<string, string>) {
this.apexDomainMap = apexDomainMap;
this.subDomainMap = subDomainMap;
return this;
}
addDomain(domain: string) {
this.domainTrie.add(domain);
return this;
}
addDomainSuffix(domain: string) {
this.domainTrie.add(domain[0] === '.' ? domain : '.' + domain);
return this;
}
bulkAddDomainSuffix(domains: string[]) {
for (let i = 0, len = domains.length; i < len; i++) {
this.addDomainSuffix(domains[i]);
}
return this;
}
addDomainKeyword(keyword: string) {
this.domainKeywords.add(keyword);
return this;
}
addDomainWildcard(wildcard: string) {
this.domainWildcard.add(wildcard);
return this;
}
private async addFromDomainsetPromise(source: AsyncIterable<string> | Iterable<string> | string[]) {
for await (const line of source) {
if (line[0] === '.') {
this.addDomainSuffix(line);
} else {
this.addDomain(line);
}
}
}
addFromDomainset(source: AsyncIterable<string> | Iterable<string> | string[]) {
this.pendingPromise = this.pendingPromise.then(() => this.addFromDomainsetPromise(source));
return this;
}
private async addFromRulesetPromise(source: AsyncIterable<string> | Iterable<string>) {
for await (const line of source) {
const splitted = line.split(',');
const type = splitted[0];
const value = splitted[1];
const arg = splitted[2];
switch (type) {
case 'DOMAIN':
this.addDomain(value);
break;
case 'DOMAIN-SUFFIX':
this.addDomainSuffix(value);
break;
case 'DOMAIN-KEYWORD':
this.addDomainKeyword(value);
break;
case 'DOMAIN-WILDCARD':
this.addDomainWildcard(value);
break;
case 'IP-CIDR':
(arg === 'no-resolve' ? this.ipcidrNoResolve : this.ipcidr).add(value);
break;
case 'IP-CIDR6':
(arg === 'no-resolve' ? this.ipcidr6NoResolve : this.ipcidr6).add(value);
break;
default:
this.otherRules.push([line, type in sortTypeOrder ? sortTypeOrder[type] : sortTypeOrder[defaultSortTypeOrder]]);
break;
}
}
}
addFromRuleset(source: AsyncIterable<string> | Iterable<string>) {
this.pendingPromise = this.pendingPromise.then(() => this.addFromRulesetPromise(source));
return this;
}
bulkAddCIDR4(cidr: string[]) {
for (let i = 0, len = cidr.length; i < len; i++) {
this.ipcidr.add(cidr[i]);
}
return this;
}
bulkAddCIDR4NoResolve(cidr: string[]) {
for (let i = 0, len = cidr.length; i < len; i++) {
this.ipcidrNoResolve.add(cidr[i]);
}
return this;
}
bulkAddCIDR6(cidr: string[]) {
for (let i = 0, len = cidr.length; i < len; i++) {
this.ipcidr6.add(cidr[i]);
}
return this;
}
bulkAddCIDR6NoResolve(cidr: string[]) {
for (let i = 0, len = cidr.length; i < len; i++) {
this.ipcidr6NoResolve.add(cidr[i]);
}
return this;
}
abstract write(): Promise<void>;
}
export class DomainsetOutput extends RuleOutput {
protected type = 'domainset' as const;
private $dumped: string[] | null = null;
get dumped() {
if (!this.$dumped) {
const kwfilter = createKeywordFilter(this.domainKeywords);
const results: string[] = [];
const dumped = this.domainTrie.dump();
for (let i = 0, len = dumped.length; i < len; i++) {
const domain = dumped[i];
if (!kwfilter(domain)) {
results.push(domain);
}
}
this.$dumped = results;
}
return this.$dumped;
}
calcDomainMap() {
if (!this.apexDomainMap || !this.subDomainMap) {
const { domainMap, subdomainMap } = buildParseDomainMap(this.dumped);
this.apexDomainMap = domainMap;
this.subDomainMap = subdomainMap;
}
}
async write() {
await this.pendingPromise;
invariant(this.title, 'Missing title');
invariant(this.description, 'Missing description');
const sorted = sortDomains(this.dumped, this.apexDomainMap, this.subDomainMap);
sorted.push('this_ruleset_is_made_by_sukkaw.ruleset.skk.moe');
const surge = sorted;
const clash = surgeDomainsetToClashDomainset(sorted);
// TODO: Implement singbox directly using data
const singbox = RuleOutput.jsonToLines(surgeDomainsetToSingbox(sorted));
await Promise.all([
compareAndWriteFile(
this.span,
withBannerArray(
this.title,
this.description,
this.date,
surge
),
path.join(OUTPUT_SURGE_DIR, this.type, this.id + '.conf')
),
compareAndWriteFile(
this.span,
withBannerArray(
this.title,
this.description,
this.date,
clash
),
path.join(OUTPUT_CLASH_DIR, this.type, this.id + '.txt')
),
compareAndWriteFile(
this.span,
singbox,
path.join(OUTPUT_SINGBOX_DIR, this.type, this.id + '.json')
)
]);
}
getStatMap() {
invariant(this.dumped, 'Non dumped yet');
invariant(this.apexDomainMap, 'Missing apex domain map');
return Array.from(
(
nullthrow(this.dumped, 'Non dumped yet').reduce<Map<string, number>>((acc, cur) => {
const suffix = this.apexDomainMap!.get(cur);
if (suffix) {
acc.set(suffix, (acc.get(suffix) ?? 0) + 1);
}
return acc;
}, new Map())
).entries()
)
.filter(a => a[1] > 9)
.sort(
(a, b) => (b[1] - a[1]) || a[0].localeCompare(b[0])
)
.map(([domain, count]) => `${domain}${' '.repeat(100 - domain.length)}${count}`);
}
}
export class IPListOutput extends RuleOutput {
protected type = 'ip' as const;
constructor(span: Span, id: string, private readonly clashUseRule = true) {
super(span, id);
}
async write() {
await this.pendingPromise;
invariant(this.title, 'Missing title');
invariant(this.description, 'Missing description');
const sorted4 = Array.from(this.ipcidr);
const sorted6 = Array.from(this.ipcidr6);
const merged = appendArrayInPlace(appendArrayInPlace([], sorted4), sorted6);
const surge = sorted4.map(i => `IP-CIDR,${i}`);
appendArrayInPlace(surge, sorted6.map(i => `IP-CIDR6,${i}`));
surge.push('DOMAIN,this_ruleset_is_made_by_sukkaw.ruleset.skk.moe');
const clash = this.clashUseRule ? surge : merged;
// TODO: Implement singbox directly using data
const singbox = RuleOutput.jsonToLines(ipCidrListToSingbox(merged));
await Promise.all([
compareAndWriteFile(
this.span,
withBannerArray(
this.title,
this.description,
this.date,
surge
),
path.join(OUTPUT_SURGE_DIR, this.type, this.id + '.conf')
),
compareAndWriteFile(
this.span,
withBannerArray(
this.title,
this.description,
this.date,
clash
),
path.join(OUTPUT_CLASH_DIR, this.type, this.id + '.txt')
),
compareAndWriteFile(
this.span,
singbox,
path.join(OUTPUT_SINGBOX_DIR, this.type, this.id + '.json')
)
]);
}
}
export class RulesetOutput extends RuleOutput {
constructor(span: Span, id: string, protected type: 'non_ip' | 'ip') {
super(span, id);
}
async write() {
await this.pendingPromise;
invariant(this.title, 'Missing title');
invariant(this.description, 'Missing description');
const results: string[] = [
'DOMAIN,this_ruleset_is_made_by_sukkaw.ruleset.skk.moe'
];
const kwfilter = createKeywordFilter(this.domainKeywords);
const sortedDomains = sortDomains(this.domainTrie.dump(), this.apexDomainMap, this.subDomainMap);
for (let i = 0, len = sortedDomains.length; i < len; i++) {
const domain = sortedDomains[i];
if (kwfilter(domain)) {
continue;
}
if (domain[0] === '.') {
results.push(`DOMAIN-SUFFIX,${domain.slice(1)}`);
} else {
results.push(`DOMAIN,${domain}`);
}
}
for (const keyword of this.domainKeywords) {
results.push(`DOMAIN-KEYWORD,${keyword}`);
}
for (const wildcard of this.domainWildcard) {
results.push(`DOMAIN-WILDCARD,${wildcard}`);
}
const sortedRules = this.otherRules.sort((a, b) => a[1] - b[1]);
for (let i = 0, len = sortedRules.length; i < len; i++) {
results.push(sortedRules[i][0]);
}
this.ipcidr.forEach(cidr => results.push(`IP-CIDR,${cidr}`));
this.ipcidrNoResolve.forEach(cidr => results.push(`IP-CIDR,${cidr},no-resolve`));
this.ipcidr6.forEach(cidr => results.push(`IP-CIDR6,${cidr}`));
this.ipcidr6NoResolve.forEach(cidr => results.push(`IP-CIDR6,${cidr},no-resolve`));
const surge = results;
const clash = surgeRulesetToClashClassicalTextRuleset(results);
// TODO: Implement singbox directly using data
const singbox = RuleOutput.jsonToLines(surgeRulesetToSingbox(results));
await Promise.all([
compareAndWriteFile(
this.span,
withBannerArray(
this.title,
this.description,
this.date,
surge
),
path.join(OUTPUT_SURGE_DIR, this.type, this.id + '.conf')
),
compareAndWriteFile(
this.span,
withBannerArray(
this.title,
this.description,
this.date,
clash
),
path.join(OUTPUT_CLASH_DIR, this.type, this.id + '.txt')
),
compareAndWriteFile(
this.span,
singbox,
path.join(OUTPUT_SINGBOX_DIR, this.type, this.id + '.json')
)
]);
}
}
function withBannerArray(title: string, description: string[] | readonly string[], date: Date, content: string[]) {
return [
'#########################################',
`# ${title}`,
`# Last Updated: ${date.toISOString()}`,
`# Size: ${content.length}`,
...description.map(line => (line ? `# ${line}` : '#')),
'#########################################',
...content,
'################## EOF ##################'
];
};
export const fileEqual = async (linesA: string[], source: AsyncIterable<string>): Promise<boolean> => { export const fileEqual = async (linesA: string[], source: AsyncIterable<string>): Promise<boolean> => {
if (linesA.length === 0) { if (linesA.length === 0) {
return false; return false;
@ -94,172 +525,3 @@ export async function compareAndWriteFile(span: Span, linesA: string[], filePath
writeStream.end(); writeStream.end();
}); });
} }
export const withBannerArray = (title: string, description: string[] | readonly string[], date: Date, content: string[]) => {
return [
'#########################################',
`# ${title}`,
`# Last Updated: ${date.toISOString()}`,
`# Size: ${content.length}`,
...description.map(line => (line ? `# ${line}` : '#')),
'#########################################',
...content,
'################## EOF ##################'
];
};
export const defaultSortTypeOrder = Symbol('defaultSortTypeOrder');
export const sortTypeOrder: Record<string | typeof defaultSortTypeOrder, number> = {
DOMAIN: 1,
'DOMAIN-SUFFIX': 2,
'DOMAIN-KEYWORD': 10,
// experimental domain wildcard support
'DOMAIN-WILDCARD': 20,
'DOMAIN-REGEX': 21,
'USER-AGENT': 30,
'PROCESS-NAME': 40,
[defaultSortTypeOrder]: 50, // default sort order for unknown type
'URL-REGEX': 100,
AND: 300,
OR: 300,
'IP-CIDR': 400,
'IP-CIDR6': 400
};
const flagDomain = 1 << 2;
const flagDomainSuffix = 1 << 3;
// dedupe and sort based on rule type
const processRuleSet = (ruleSet: string[]) => {
const trie = createTrie<number>(null, true);
/** Packed Array<[valueIndex: number, weight: number]> */
const sortMap: number[] = [];
for (let i = 0, len = ruleSet.length; i < len; i++) {
const line = ruleSet[i];
const [type, value] = line.split(',');
let extraWeight = 0;
switch (type) {
case 'DOMAIN':
trie.add(value, pack(i, flagDomain));
break;
case 'DOMAIN-SUFFIX':
trie.add('.' + value, pack(i, flagDomainSuffix));
break;
case 'URL-REGEX':
if (value.includes('.+') || value.includes('.*')) {
extraWeight += 10;
}
if (value.includes('|')) {
extraWeight += 1;
}
sortMap.push(pack(i, sortTypeOrder[type] + extraWeight));
break;
case null:
sortMap.push(pack(i, 10));
break;
default:
if (type in sortTypeOrder) {
sortMap.push(pack(i, sortTypeOrder[type]));
} else {
sortMap.push(pack(i, sortTypeOrder[defaultSortTypeOrder]));
}
}
}
const dumped = trie.dumpMeta();
for (let i = 0, len = dumped.length; i < len; i++) {
const originalIndex = unpackFirst(dumped[i]);
const flag = unpackSecond(dumped[i]);
const type = flag === flagDomain ? 'DOMAIN' : 'DOMAIN-SUFFIX';
sortMap.push(pack(originalIndex, sortTypeOrder[type]));
}
return sortMap
.sort((a, b) => unpackSecond(a) - unpackSecond(b))
.map(c => ruleSet[unpackFirst(c)]);
};
const MARK = 'this_ruleset_is_made_by_sukkaw.ruleset.skk.moe';
export const createRuleset = (
parentSpan: Span,
title: string, description: string[] | readonly string[], date: Date, content: string[],
type: 'ruleset' | 'domainset',
[surgePath, clashPath, singBoxPath, _clashMrsPath]: readonly [
surgePath: string,
clashPath: string,
singBoxPath: string,
_clashMrsPath?: string
]
) => parentSpan.traceChildAsync(
`create ruleset: ${path.basename(surgePath, path.extname(surgePath))}`,
async (childSpan) => {
const surgeContent = childSpan.traceChildSync('process surge ruleset', () => {
let _surgeContent;
switch (type) {
case 'domainset':
_surgeContent = [MARK, ...content];
break;
case 'ruleset':
_surgeContent = [`DOMAIN,${MARK}`, ...processRuleSet(content)];
break;
default:
throw new TypeError(`Unknown type: ${type}`);
}
return withBannerArray(title, description, date, _surgeContent);
});
const clashContent = childSpan.traceChildSync('convert incoming ruleset to clash', () => {
let _clashContent;
switch (type) {
case 'domainset':
_clashContent = [MARK, ...surgeDomainsetToClashDomainset(content)];
break;
case 'ruleset':
_clashContent = [`DOMAIN,${MARK}`, ...surgeRulesetToClashClassicalTextRuleset(processRuleSet(content))];
break;
default:
throw new TypeError(`Unknown type: ${type}`);
}
return withBannerArray(title, description, date, _clashContent);
});
const singboxContent = childSpan.traceChildSync('convert incoming ruleset to singbox', () => {
let _singBoxContent;
switch (type) {
case 'domainset':
_singBoxContent = surgeDomainsetToSingbox([MARK, ...processRuleSet(content)]);
break;
case 'ruleset':
_singBoxContent = surgeRulesetToSingbox([`DOMAIN,${MARK}`, ...processRuleSet(content)]);
break;
default:
throw new TypeError(`Unknown type: ${type}`);
}
return stringify(_singBoxContent).split('\n');
});
await Promise.all([
compareAndWriteFile(childSpan, surgeContent, surgePath),
compareAndWriteFile(childSpan, clashContent, clashPath),
compareAndWriteFile(childSpan, singboxContent, singBoxPath)
]);
// if (clashMrsPath) {
// if (type === 'domainset') {
// await childSpan.traceChildAsync('clash meta mrs domain ' + clashMrsPath, async () => {
// await fs.promises.mkdir(path.dirname(clashMrsPath), { recursive: true });
// await convertClashMetaMrs(
// 'domain', 'text', clashPath, clashMrsPath
// );
// });
// }
// }
}
);

10
Source/domainset/cdn.conf
View File

@ -832,6 +832,7 @@ telemetry.nextjs.org
telemetry.vercel.com telemetry.vercel.com
stats.setapp.com stats.setapp.com
.app-analytics-services.com .app-analytics-services.com
.telemetry.services.yofi.ai
# influxdata # influxdata
.cloud.influxdata.com .cloud.influxdata.com
.cloud1.influxdata.com .cloud1.influxdata.com
@ -1001,6 +1002,7 @@ ak.sail-horizon.com
micro.rubiconproject.com micro.rubiconproject.com
static.iris.informa.com static.iris.informa.com
.localizecdn.com .localizecdn.com
js.verygoodvault.com
loader.mantis-intelligence.com loader.mantis-intelligence.com
mantisloader.mantis-awx.com mantisloader.mantis-awx.com
@ -1081,6 +1083,8 @@ images.prismic.io
.website-files.com .website-files.com
.a.storyblok.com .a.storyblok.com
images.contentful.com images.contentful.com
.payloadcms.app
cms.payloadcms.com
# >> Weebly # >> Weebly
cdn2.editmysite.com cdn2.editmysite.com
@ -2881,6 +2885,10 @@ rawmarkdown.cloudflareaccess.org
static.global.visioncritical.com static.global.visioncritical.com
assets.visioncritical.net assets.visioncritical.net
s1.divhunt.com s1.divhunt.com
dh-site.b-cdn.net
divhunt-site.b-cdn.net
divhunt.b-cdn.net
global.divhunt.com
.smushcdn.com .smushcdn.com
.wpmucdn.com .wpmucdn.com
cdn.permutive.com cdn.permutive.com
@ -2921,3 +2929,5 @@ eu-images.contentstack.com
images.contentstack.io images.contentstack.io
ui.contentstack.com ui.contentstack.com
.static.chaosgroup.com .static.chaosgroup.com
static.moonpay.com
static.launchdarkly.com