GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2025-12-12 17:20:35 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Chore: faster reject domainset builder

Browse Source
This commit is contained in:
SukkaW 2022-07-05 00:06:14 +08:00
parent d2c3907af5
commit ed3ba14e0e
3 changed files with 21 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Build/build-reject-domainset.js
View File

@ -71,7 +71,8 @@ const threads = require('os').cpus().length - 1;
'heapanalytics.com', 'heapanalytics.com',
'segment.com', 'segment.com',
'segmentify.com', 'segmentify.com',
't.co' // pgl yoyo add t.co to the blacklist 't.co', // pgl yoyo add t.co to the blacklist
'survicate.com' // AdGuardDNSFilter
]); ]);
(await Promise.all([ (await Promise.all([

25
Build/lib/parse-filter.js
View File

@ -84,28 +84,33 @@ async function processFilterRules(filterRulesUrl) {
const filterRules = (await (await fetch(filterRulesUrl)).text()).split('\n').map(line => line.trim()); const filterRules = (await (await fetch(filterRulesUrl)).text()).split('\n').map(line => line.trim());
filterRules.forEach(line => { filterRules.forEach(line => {
const lineStartsWithDoubleVerticalBar = line.startsWith('||');
if ( if (
line === '' line === ''
|| line.includes('#') || line.includes('#')
|| line.includes('!') || line.includes('!')
|| line.includes('*') || line.includes('*')
|| line.includes('/') || line.includes('/')
|| line.includes('$') && !line.startsWith('||') || line.includes('$') && !lineStartsWithDoubleVerticalBar
|| line.trim() === '' || line === ''
|| isIP(line) !== 0 || isIP(line) !== 0
) { ) {
return; return;
} }
if (line.startsWith('||') && line.endsWith('^$badfilter')) { const lineEndsWithCaret = line.endsWith('^');
const lineEndsWithCaretVerticalBar = line.endsWith('^|');
if (lineStartsWithDoubleVerticalBar && line.endsWith('^$badfilter')) {
const domain = line.replace('||', '').replace('^$badfilter', '').trim(); const domain = line.replace('||', '').replace('^$badfilter', '').trim();
if (rDomain.test(domain)) { if (rDomain.test(domain)) {
whitelistDomainSets.add(domain); whitelistDomainSets.add(domain);
} }
} else if (line.startsWith('@@||') } else if (line.startsWith('@@||')
&& ( && (
line.endsWith('^') lineEndsWithCaret
|| line.endsWith('^|') || lineEndsWithCaretVerticalBar
|| line.endsWith('^$badfilter') || line.endsWith('^$badfilter')
|| line.endsWith('^$1p') || line.endsWith('^$1p')
) )
@ -121,10 +126,10 @@ async function processFilterRules(filterRulesUrl) {
whitelistDomainSets.add(domain); whitelistDomainSets.add(domain);
} }
} else if ( } else if (
line.startsWith('||') lineStartsWithDoubleVerticalBar
&& ( && (
line.endsWith('^') lineEndsWithCaret
|| line.endsWith('^|') || lineEndsWithCaretVerticalBar
|| line.endsWith('^$all') || line.endsWith('^$all')
) )
) { ) {
@ -139,8 +144,8 @@ async function processFilterRules(filterRulesUrl) {
} }
} else if (line.startsWith('://') } else if (line.startsWith('://')
&& ( && (
line.endsWith('^') lineEndsWithCaret
|| line.endsWith('^|') || lineEndsWithCaretVerticalBar
) )
) { ) {
const domain = `${line.replaceAll('://', '').replaceAll('^|', '').replaceAll('^', '')}`.trim(); const domain = `${line.replaceAll('://', '').replaceAll('^|', '').replaceAll('^', '')}`.trim();

7
Build/worker/build-reject-domainset-worker.js
View File

@ -3,14 +3,15 @@ const { workerData } = require('piscina');
exports.dedupe = ({ chunk }) => { exports.dedupe = ({ chunk }) => {
const outputToBeRemoved = new Set(); const outputToBeRemoved = new Set();
for (const domainFromInput of chunk) { for (let i = 0, l = chunk.length; i < l; i++) {
const domainFromInput = chunk[i];
for (const domainFromFullSet of workerData) { for (const domainFromFullSet of workerData) {
if (domainFromFullSet === domainFromInput) continue; if (domainFromFullSet === domainFromInput) continue;
if (domainFromFullSet.charAt(0) !== '.') continue; if (domainFromFullSet.charAt(0) !== '.') continue;
if ( if (
`.${domainFromInput}` === domainFromFullSet // `.${domainFromInput}` === domainFromFullSet
|| domainFromInput.endsWith(domainFromFullSet) domainFromInput.endsWith(domainFromFullSet)
) { ) {
outputToBeRemoved.add(domainFromInput); outputToBeRemoved.add(domainFromInput);
break; break;