GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2026-01-28 17:41:54 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Refactor: adapt new output

Browse Source
This commit is contained in:
SukkaW
2024-09-21 03:36:44 +08:00
parent eb2023b9aa
commit 4808ed8d27
7 changed files with 204 additions and 269 deletions
Download Patch File Download Diff File Expand all files Collapse all files

184
Build/build-reject-domainset.ts
View File

@@ -3,13 +3,10 @@ import path from 'node:path';
import process from 'node:process';
import { processHosts, processFilterRules, processDomainLists } from './lib/parse-filter';
import { createTrie } from './lib/trie';
import { HOSTS, ADGUARD_FILTERS, PREDEFINED_WHITELIST, DOMAIN_LISTS, HOSTS_EXTRA, DOMAIN_LISTS_EXTRA, ADGUARD_FILTERS_EXTRA, PHISHING_DOMAIN_LISTS_EXTRA } from './constants/reject-data-source';
import { createRuleset, compareAndWriteFile } from './lib/create-file';
import createKeywordFilter from './lib/aho-corasick';
import { compareAndWriteFile } from './lib/create-file';
import { readFileByLine, readFileIntoProcessedArray } from './lib/fetch-text-by-line';
import { buildParseDomainMap, sortDomains } from './lib/stable-sort-domain';
import { task } from './trace';
// tldts-experimental is way faster than tldts, but very little bit inaccurate
// (since it is hashes based). But the result is still deterministic, which is
@@ -17,23 +14,47 @@ import { task } from './trace';
import { SHARED_DESCRIPTION } from './lib/constants';
import { getPhishingDomains } from './lib/get-phishing-domains';
import { setAddFromArray, setAddFromArrayCurried } from './lib/set-add-from-array';
import { output } from './lib/misc';
import { setAddFromArray } from './lib/set-add-from-array';
import { appendArrayInPlace } from './lib/append-array-in-place';
import { OUTPUT_INTERNAL_DIR, SOURCE_DIR } from './constants/dir';
import { DomainsetOutput } from './lib/create-file-new';
const getRejectSukkaConfPromise = readFileIntoProcessedArray(path.join(SOURCE_DIR, 'domainset/reject_sukka.conf'));
export const buildRejectDomainSet = task(require.main === module, __filename)(async (span) => {
const rejectOutput = new DomainsetOutput(span, 'reject')
.withTitle('Sukka\'s Ruleset - Reject Base')
.withDescription([
...SHARED_DESCRIPTION,
'',
'The domainset supports AD blocking, tracking protection, privacy protection, anti-phishing, anti-mining',
'',
'Build from:',
...HOSTS.map(host => ` - ${host[0]}`),
...DOMAIN_LISTS.map(domainList => ` - ${domainList[0]}`),
...ADGUARD_FILTERS.map(filter => ` - ${Array.isArray(filter) ? filter[0] : filter}`)
]);
const rejectExtraOutput = new DomainsetOutput(span, 'reject_extra')
.withTitle('Sukka\'s Ruleset - Reject Extra')
.withDescription([
...SHARED_DESCRIPTION,
'',
'The domainset supports AD blocking, tracking protection, privacy protection, anti-phishing, anti-mining',
'',
'Build from:',
...HOSTS_EXTRA.map(host => ` - ${host[0]}`),
...DOMAIN_LISTS_EXTRA.map(domainList => ` - ${domainList[0]}`),
...ADGUARD_FILTERS_EXTRA.map(filter => ` - ${Array.isArray(filter) ? filter[0] : filter}`),
...PHISHING_DOMAIN_LISTS_EXTRA.map(domainList => ` - ${domainList[0]}`)
]);
const appendArrayToRejectOutput = rejectOutput.addFromDomainset.bind(rejectOutput);
const appendArrayToRejectExtraOutput = rejectExtraOutput.addFromDomainset.bind(rejectExtraOutput);
/** Whitelists */
const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST);
const domainSets = new Set<string>();
const appendArrayToDomainSets = setAddFromArrayCurried(domainSets);
const domainSetsExtra = new Set<string>();
const appendArrayToDomainSetsExtra = setAddFromArrayCurried(domainSetsExtra);
// Parse from AdGuard Filters
const shouldStop = await span
.traceChild('download and process hosts / adblock filter rules')
@@ -42,11 +63,11 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as
let shouldStop = false;
await Promise.all([
// Parse from remote hosts & domain lists
HOSTS.map(entry => processHosts(childSpan, ...entry).then(appendArrayToDomainSets)),
HOSTS_EXTRA.map(entry => processHosts(childSpan, ...entry).then(appendArrayToDomainSetsExtra)),
HOSTS.map(entry => processHosts(childSpan, ...entry).then(appendArrayToRejectOutput)),
HOSTS_EXTRA.map(entry => processHosts(childSpan, ...entry).then(appendArrayToRejectExtraOutput)),
DOMAIN_LISTS.map(entry => processDomainLists(childSpan, ...entry).then(appendArrayToDomainSets)),
DOMAIN_LISTS_EXTRA.map(entry => processDomainLists(childSpan, ...entry).then(appendArrayToDomainSetsExtra)),
DOMAIN_LISTS.map(entry => processDomainLists(childSpan, ...entry).then(appendArrayToRejectOutput)),
DOMAIN_LISTS_EXTRA.map(entry => processDomainLists(childSpan, ...entry).then(appendArrayToRejectExtraOutput)),
ADGUARD_FILTERS.map(
entry => processFilterRules(childSpan, ...entry)
@@ -57,7 +78,7 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as
// we should not break here, as we want to see full matches from all data source
}
setAddFromArray(filterRuleWhitelistDomainSets, white);
setAddFromArray(domainSets, black);
appendArrayToRejectOutput(black);
})
),
ADGUARD_FILTERS_EXTRA.map(
@@ -69,7 +90,7 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as
// we should not break here, as we want to see full matches from all data source
}
setAddFromArray(filterRuleWhitelistDomainSets, white);
setAddFromArray(domainSetsExtra, black);
appendArrayToRejectExtraOutput(black);
})
),
@@ -82,8 +103,8 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as
setAddFromArray(filterRuleWhitelistDomainSets, black);
})
)),
getPhishingDomains(childSpan).then(appendArrayToDomainSetsExtra),
getRejectSukkaConfPromise.then(appendArrayToDomainSets)
getPhishingDomains(childSpan).then(appendArrayToRejectExtraOutput),
getRejectSukkaConfPromise.then(appendArrayToRejectOutput)
].flat());
// eslint-disable-next-line sukka/no-single-return -- not single return
return shouldStop;
@@ -93,72 +114,23 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as
process.exit(1);
}
console.log(`Import ${domainSets.size} + ${domainSetsExtra.size} rules from Hosts / AdBlock Filter Rules & reject_sukka.conf!`);
// Dedupe domainSets
const domainKeywordsSet = await span.traceChildAsync('collect black keywords/suffixes', async () => {
await span.traceChildAsync('collect black keywords/suffixes', async () => {
/** Collect DOMAIN-KEYWORD from non_ip/reject.conf for deduplication */
const domainKeywordsSet = new Set<string>();
for await (const line of readFileByLine(path.resolve(__dirname, '../Source/non_ip/reject.conf'))) {
const [type, value] = line.split(',');
if (type === 'DOMAIN-KEYWORD') {
domainKeywordsSet.add(value);
rejectOutput.addDomainKeyword(value); // Add for later deduplication
rejectExtraOutput.addDomainKeyword(value); // Add for later deduplication
} else if (type === 'DOMAIN-SUFFIX') {
domainSets.add('.' + value); // Add to domainSets for later deduplication
rejectOutput.addDomainSuffix(value); // Add for later deduplication
}
}
return domainKeywordsSet;
});
const [baseTrie, extraTrie] = span.traceChildSync('create smol trie while deduping black keywords', (childSpan) => {
const baseTrie = createTrie(null, true);
const extraTrie = createTrie(null, true);
const kwfilter = createKeywordFilter(domainKeywordsSet);
childSpan.traceChildSync('add items to trie (extra)', () => {
for (const domain of domainSetsExtra) {
// exclude keyword when creating trie
if (!kwfilter(domain)) {
extraTrie.add(domain);
}
}
});
childSpan.traceChildSync('add items to trie (base) + dedupe extra trie', () => {
for (const domain of domainSets) {
// exclude keyword when creating trie
if (!kwfilter(domain)) {
baseTrie.add(domain);
extraTrie.whitelist(domain);
}
}
});
return [baseTrie, extraTrie] as const;
});
span.traceChildSync('dedupe from white suffixes (base)', () => filterRuleWhitelistDomainSets.forEach(baseTrie.whitelist));
span.traceChildSync('dedupe from white suffixes and base (extra)', () => {
filterRuleWhitelistDomainSets.forEach(extraTrie.whitelist);
});
// Dedupe domainSets
const dedupedDominArray = span.traceChildSync('dedupe from covered subdomain (base)', () => baseTrie.dump());
const dudupedDominArrayExtra = span.traceChildSync('dedupe from covered subdomain (extra)', () => extraTrie.dump());
console.log(`Final size ${dedupedDominArray.length} + ${dudupedDominArrayExtra.length}`);
const {
domainMap: domainArrayMainDomainMap,
subdomainMap: domainArraySubdomainMap
} = span.traceChildSync(
'build map for stat and sort',
() => buildParseDomainMap(dedupedDominArray.concat(dudupedDominArrayExtra))
);
rejectOutput.calcDomainMap();
rejectExtraOutput.calcDomainMap();
// Create reject stats
const rejectDomainsStats: string[] = span
@@ -166,50 +138,15 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as
.traceSyncFn(() => {
const results = [];
results.push('=== base ===');
appendArrayInPlace(results, getStatMap(dedupedDominArray, domainArrayMainDomainMap));
appendArrayInPlace(results, rejectOutput.getStatMap());
results.push('=== extra ===');
appendArrayInPlace(results, getStatMap(dudupedDominArrayExtra, domainArrayMainDomainMap));
appendArrayInPlace(results, rejectExtraOutput.getStatMap());
return results;
});
return Promise.all([
createRuleset(
span,
'Sukka\'s Ruleset - Reject Base',
[
...SHARED_DESCRIPTION,
'',
'The domainset supports AD blocking, tracking protection, privacy protection, anti-phishing, anti-mining',
'',
'Build from:',
...HOSTS.map(host => ` - ${host[0]}`),
...DOMAIN_LISTS.map(domainList => ` - ${domainList[0]}`),
...ADGUARD_FILTERS.map(filter => ` - ${Array.isArray(filter) ? filter[0] : filter}`)
],
new Date(),
span.traceChildSync('sort reject domainset (base)', () => sortDomains(dedupedDominArray, domainArrayMainDomainMap, domainArraySubdomainMap)),
'domainset',
output('reject', 'domainset')
),
createRuleset(
span,
'Sukka\'s Ruleset - Reject Extra',
[
...SHARED_DESCRIPTION,
'',
'The domainset supports AD blocking, tracking protection, privacy protection, anti-phishing, anti-mining',
'',
'Build from:',
...HOSTS_EXTRA.map(host => ` - ${host[0]}`),
...DOMAIN_LISTS_EXTRA.map(domainList => ` - ${domainList[0]}`),
...ADGUARD_FILTERS_EXTRA.map(filter => ` - ${Array.isArray(filter) ? filter[0] : filter}`),
...PHISHING_DOMAIN_LISTS_EXTRA.map(domainList => ` - ${domainList[0]}`)
],
new Date(),
span.traceChildSync('sort reject domainset (extra)', () => sortDomains(dudupedDominArrayExtra, domainArrayMainDomainMap, domainArraySubdomainMap)),
'domainset',
output('reject_extra', 'domainset')
),
rejectOutput.write(),
rejectExtraOutput.write(),
compareAndWriteFile(
span,
rejectDomainsStats,
@@ -217,22 +154,3 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as
)
]);
});
function getStatMap(domains: string[], domainArrayMainDomainMap: Map<string, string>): string[] {
return Array.from(
(
domains.reduce<Map<string, number>>((acc, cur) => {
const suffix = domainArrayMainDomainMap.get(cur);
if (suffix) {
acc.set(suffix, (acc.get(suffix) ?? 0) + 1);
}
return acc;
}, new Map())
).entries()
)
.filter(a => a[1] > 9)
.sort(
(a, b) => (b[1] - a[1]) || a[0].localeCompare(b[0])
)
.map(([domain, count]) => `${domain}${' '.repeat(100 - domain.length)}${count}`);
};