GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2025-12-12 09:10:35 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Chore: faster reject domainset builder

Browse Source
This commit is contained in:
SukkaW 2022-07-05 00:06:14 +08:00
parent d2c3907af5
commit ed3ba14e0e
3 changed files with 21 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Build/build-reject-domainset.js
View File

@ -71,7 +71,8 @@ const threads = require('os').cpus().length - 1;
'heapanalytics.com',
'segment.com',
'segmentify.com',
't.co' // pgl yoyo add t.co to the blacklist
't.co', // pgl yoyo add t.co to the blacklist
'survicate.com' // AdGuardDNSFilter
]);
(await Promise.all([

25
Build/lib/parse-filter.js
View File

@ -84,28 +84,33 @@ async function processFilterRules(filterRulesUrl) {
const filterRules = (await (await fetch(filterRulesUrl)).text()).split('\n').map(line => line.trim());
filterRules.forEach(line => {
const lineStartsWithDoubleVerticalBar = line.startsWith('||');
if (
line === ''
|| line.includes('#')
|| line.includes('!')
|| line.includes('*')
|| line.includes('/')
|| line.includes('$') && !line.startsWith('||')
|| line.trim() === ''
|| line.includes('$') && !lineStartsWithDoubleVerticalBar
|| line === ''
|| isIP(line) !== 0
) {
return;
}
if (line.startsWith('||') && line.endsWith('^$badfilter')) {
const lineEndsWithCaret = line.endsWith('^');
const lineEndsWithCaretVerticalBar = line.endsWith('^|');
if (lineStartsWithDoubleVerticalBar && line.endsWith('^$badfilter')) {
const domain = line.replace('||', '').replace('^$badfilter', '').trim();
if (rDomain.test(domain)) {
whitelistDomainSets.add(domain);
}
} else if (line.startsWith('@@||')
&& (
line.endsWith('^')
|| line.endsWith('^|')
lineEndsWithCaret
|| lineEndsWithCaretVerticalBar
|| line.endsWith('^$badfilter')
|| line.endsWith('^$1p')
)
@ -121,10 +126,10 @@ async function processFilterRules(filterRulesUrl) {
whitelistDomainSets.add(domain);
}
} else if (
line.startsWith('||')
lineStartsWithDoubleVerticalBar
&& (
line.endsWith('^')
|| line.endsWith('^|')
lineEndsWithCaret
|| lineEndsWithCaretVerticalBar
|| line.endsWith('^$all')
)
) {
@ -139,8 +144,8 @@ async function processFilterRules(filterRulesUrl) {
}
} else if (line.startsWith('://')
&& (
line.endsWith('^')
|| line.endsWith('^|')
lineEndsWithCaret
|| lineEndsWithCaretVerticalBar
)
) {
const domain = `${line.replaceAll('://', '').replaceAll('^|', '').replaceAll('^', '')}`.trim();

7
Build/worker/build-reject-domainset-worker.js
View File

@ -3,14 +3,15 @@ const { workerData } = require('piscina');
exports.dedupe = ({ chunk }) => {
const outputToBeRemoved = new Set();
for (const domainFromInput of chunk) {
for (let i = 0, l = chunk.length; i < l; i++) {
const domainFromInput = chunk[i];
for (const domainFromFullSet of workerData) {
if (domainFromFullSet === domainFromInput) continue;
if (domainFromFullSet.charAt(0) !== '.') continue;
if (
`.${domainFromInput}` === domainFromFullSet
|| domainFromInput.endsWith(domainFromFullSet)
// `.${domainFromInput}` === domainFromFullSet
domainFromInput.endsWith(domainFromFullSet)
) {
outputToBeRemoved.add(domainFromInput);
break;