mirror of
https://github.com/SukkaW/Surge.git
synced 2025-12-12 01:00:34 +08:00
Chore: introduce eslint to build script
This commit is contained in:
parent
aeb90e36ca
commit
09961fad44
11
.eslintrc.json
Normal file
11
.eslintrc.json
Normal file
@ -0,0 +1,11 @@
|
||||
{
|
||||
"root": true,
|
||||
"extends": ["sukka/node"],
|
||||
"rules": {
|
||||
"no-console": "off"
|
||||
},
|
||||
"parserOptions": {
|
||||
"ecmaVersion": "latest",
|
||||
"sourceType": "module"
|
||||
}
|
||||
}
|
||||
@ -34,7 +34,7 @@ const { fetchRemoteTextAndCreateReadlineInterface } = require('./lib/fetch-remot
|
||||
'This file contains Apple\'s domains using their China mainland CDN servers.',
|
||||
'',
|
||||
'Data from:',
|
||||
' - https://github.com/felixonmars/dnsmasq-china-list',
|
||||
' - https://github.com/felixonmars/dnsmasq-china-list'
|
||||
],
|
||||
new Date(),
|
||||
res.map(domain => `DOMAIN-SUFFIX,${domain}`)
|
||||
@ -52,14 +52,14 @@ const { fetchRemoteTextAndCreateReadlineInterface } = require('./lib/fetch-remot
|
||||
'This file contains Apple\'s domains using their China mainland CDN servers.',
|
||||
'',
|
||||
'Data from:',
|
||||
' - https://github.com/felixonmars/dnsmasq-china-list',
|
||||
' - https://github.com/felixonmars/dnsmasq-china-list'
|
||||
],
|
||||
new Date(),
|
||||
res.map(i => `.${i}`)
|
||||
),
|
||||
path.resolve(__dirname, '../List/domainset/apple_cdn.conf')
|
||||
)
|
||||
])
|
||||
]);
|
||||
|
||||
console.timeEnd('Total Time - build-apple-cdn-conf');
|
||||
})();
|
||||
|
||||
@ -20,11 +20,11 @@ const { minifyRules } = require('./lib/minify-rules');
|
||||
line.endsWith('.amazonaws.com')
|
||||
|| line.endsWith('.scw.cloud')
|
||||
)
|
||||
&& !line.includes('cn-')
|
||||
&& !line.includes('cn-');
|
||||
}
|
||||
|
||||
return false;
|
||||
})
|
||||
});
|
||||
|
||||
const filePath = path.resolve(__dirname, '../Source/non_ip/cdn.conf');
|
||||
const resultPath = path.resolve(__dirname, '../List/non_ip/cdn.conf');
|
||||
@ -48,7 +48,7 @@ const { minifyRules } = require('./lib/minify-rules');
|
||||
minifyRules(content.split('\n'))
|
||||
),
|
||||
resultPath
|
||||
)
|
||||
);
|
||||
|
||||
console.timeEnd('Total Time - build-cdn-conf');
|
||||
})();
|
||||
|
||||
@ -15,7 +15,7 @@ const { compareAndWriteFile } = require('./lib/string-array-compare');
|
||||
console.log('Before Merge:', cidr.length);
|
||||
const filteredCidr = mergeCidrs(cidr.filter(line => {
|
||||
if (line) {
|
||||
return !line.startsWith('#')
|
||||
return !line.startsWith('#');
|
||||
}
|
||||
|
||||
return false;
|
||||
@ -30,13 +30,13 @@ const { compareAndWriteFile } = require('./lib/string-array-compare');
|
||||
'Homepage: https://ruleset.skk.moe',
|
||||
'GitHub: https://github.com/SukkaW/Surge',
|
||||
'',
|
||||
'Data from https://misaka.io (misakaio @ GitHub)',
|
||||
'Data from https://misaka.io (misakaio @ GitHub)'
|
||||
],
|
||||
new Date(),
|
||||
filteredCidr.map(i => `IP-CIDR,${i}`)
|
||||
),
|
||||
pathResolve(__dirname, '../List/ip/china_ip.conf')
|
||||
)
|
||||
);
|
||||
|
||||
console.timeEnd('Total Time - build-chnroutes-cidr');
|
||||
})();
|
||||
|
||||
@ -36,7 +36,7 @@ const tldts = require('tldts');
|
||||
console.warn('[drop line from domainset]', line);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* @param {string} ruleSetPath
|
||||
@ -49,14 +49,14 @@ const tldts = require('tldts');
|
||||
})
|
||||
) {
|
||||
if (line.startsWith('DOMAIN-SUFFIX,')) {
|
||||
addApexDomain(line.replace('DOMAIN-SUFFIX,', ''))
|
||||
addApexDomain(line.replace('DOMAIN-SUFFIX,', ''));
|
||||
} else if (line.startsWith('DOMAIN,')) {
|
||||
addApexDomain(line.replace('DOMAIN,', ''));
|
||||
} else if (!line.startsWith('#') && line.trim() !== '') {
|
||||
console.warn('[drop line from ruleset]', line);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
await processLocalRuleSet(path.resolve(__dirname, '../List/non_ip/cdn.conf'));
|
||||
await processLocalRuleSet(path.resolve(__dirname, '../List/non_ip/global.conf'));
|
||||
@ -71,6 +71,6 @@ const tldts = require('tldts');
|
||||
await fse.ensureDir(path.resolve(__dirname, '../List/internal'));
|
||||
await fs.promises.writeFile(
|
||||
path.resolve(__dirname, '../List/internal/cdn.txt'),
|
||||
Array.from(set).map(i => `SUFFIX,${i}`).join('\n') + '\n'
|
||||
`${Array.from(set).map(i => `SUFFIX,${i}`).join('\n')}\n`
|
||||
);
|
||||
})();
|
||||
|
||||
@ -116,7 +116,7 @@ const PRESET_MITM_HOSTNAMES = [
|
||||
&& !(i !== '*.meituan.net' && i.endsWith('.meituan.net'))
|
||||
&& !i.startsWith('.')
|
||||
&& !i.endsWith('.')
|
||||
&& !i.endsWith('*')
|
||||
&& !i.endsWith('*');
|
||||
});
|
||||
|
||||
const mitmDomainsRegExpArray = mitmDomains.map(i => {
|
||||
@ -124,7 +124,7 @@ const PRESET_MITM_HOSTNAMES = [
|
||||
escapeRegExp(i)
|
||||
.replaceAll('{www or not}', '(www.)?')
|
||||
.replaceAll('\\*', '(.*)')
|
||||
)
|
||||
);
|
||||
});
|
||||
|
||||
const parsedDomainsData = [];
|
||||
@ -141,7 +141,7 @@ const PRESET_MITM_HOSTNAMES = [
|
||||
});
|
||||
|
||||
console.log('Mitm Hostnames:');
|
||||
console.log('hostname = %APPEND% ' + mitmDomains.join(', '));
|
||||
console.log(`hostname = %APPEND% ${mitmDomains.join(', ')}`);
|
||||
console.log('--------------------');
|
||||
console.log('Parsed Sucessed:');
|
||||
console.log(table.table(parsedDomainsData, {
|
||||
@ -164,11 +164,11 @@ function parseDomain(input) {
|
||||
return {
|
||||
success: true,
|
||||
hostname: url.hostname
|
||||
}
|
||||
};
|
||||
} catch {
|
||||
return {
|
||||
success: false
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -110,7 +110,7 @@ const BLACK_TLD = Array.from(new Set([
|
||||
if (
|
||||
count >= 5
|
||||
) {
|
||||
results.push('.' + domain);
|
||||
results.push(`.${domain}`);
|
||||
}
|
||||
});
|
||||
|
||||
@ -132,5 +132,5 @@ const BLACK_TLD = Array.from(new Set([
|
||||
results
|
||||
),
|
||||
path.resolve(__dirname, '../List/domainset/reject_phishing.conf')
|
||||
)
|
||||
);
|
||||
})();
|
||||
|
||||
@ -69,5 +69,5 @@ function template(urlList) {
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
`
|
||||
`;
|
||||
}
|
||||
|
||||
@ -1,6 +1,8 @@
|
||||
// @ts-check
|
||||
const { promises: fsPromises } = require('fs');
|
||||
const fs = require('fs');
|
||||
const fse = require('fs-extra');
|
||||
const readline = require('readline');
|
||||
|
||||
const { resolve: pathResolve } = require('path');
|
||||
const { processHosts, processFilterRules, preprocessFullDomainSetBeforeUsedAsWorkerData } = require('./lib/parse-filter');
|
||||
const { getDomain } = require('tldts');
|
||||
@ -23,32 +25,25 @@ const domainSuffixSet = new Set();
|
||||
/** @type Set<string> */
|
||||
const domainSets = new Set();
|
||||
|
||||
console.log('Downloading hosts file...');
|
||||
console.time('* Download and process Hosts');
|
||||
// Parse from AdGuard Filters
|
||||
console.time('* Download and process Hosts / AdBlock Filter Rules');
|
||||
|
||||
let shouldStop = false;
|
||||
|
||||
await Promise.all([
|
||||
// Parse from remote hosts & domain lists
|
||||
(await Promise.all(HOSTS.map(entry => processHosts(entry[0], entry[1]))))
|
||||
.forEach(hosts => {
|
||||
Promise.all(HOSTS.map(entry => processHosts(entry[0], entry[1])))
|
||||
.then(r => r.forEach(hosts => {
|
||||
hosts.forEach(host => {
|
||||
if (host) {
|
||||
domainSets.add(host);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
console.timeEnd('* Download and process Hosts');
|
||||
|
||||
let previousSize = domainSets.size;
|
||||
console.log(`Import ${previousSize} rules from hosts files!`);
|
||||
|
||||
// Parse from AdGuard Filters
|
||||
console.time('* Download and process AdBlock Filter Rules');
|
||||
|
||||
let shouldStop = false;
|
||||
await Promise.all(ADGUARD_FILTERS.map(input => {
|
||||
})),
|
||||
Promise.all(ADGUARD_FILTERS.map(input => {
|
||||
const promise = typeof input === 'string'
|
||||
? processFilterRules(input, undefined, false)
|
||||
: processFilterRules(input[0], input[1] || undefined, input[2] ?? false)
|
||||
: processFilterRules(input[0], input[1] || undefined, input[2] ?? false);
|
||||
|
||||
return promise.then((i) => {
|
||||
if (i) {
|
||||
@ -57,9 +52,9 @@ const domainSuffixSet = new Set();
|
||||
shouldStop = true;
|
||||
}
|
||||
white.forEach(i => {
|
||||
if (PREDEFINED_ENFORCED_BACKLIST.some(j => i.endsWith(j))) {
|
||||
return;
|
||||
}
|
||||
// if (PREDEFINED_ENFORCED_BACKLIST.some(j => i.endsWith(j))) {
|
||||
// return;
|
||||
// }
|
||||
filterRuleWhitelistDomainSets.add(i);
|
||||
});
|
||||
black.forEach(i => domainSets.add(i));
|
||||
@ -67,9 +62,8 @@ const domainSuffixSet = new Set();
|
||||
process.exit(1);
|
||||
}
|
||||
});
|
||||
}));
|
||||
|
||||
await Promise.all([
|
||||
})),
|
||||
Promise.all([
|
||||
'https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt',
|
||||
'https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt'
|
||||
].map(
|
||||
@ -77,57 +71,67 @@ const domainSuffixSet = new Set();
|
||||
if (i) {
|
||||
const { white, black } = i;
|
||||
white.forEach(i => {
|
||||
if (PREDEFINED_ENFORCED_BACKLIST.some(j => i.endsWith(j))) {
|
||||
return;
|
||||
}
|
||||
filterRuleWhitelistDomainSets.add(i)
|
||||
// if (PREDEFINED_ENFORCED_BACKLIST.some(j => i.endsWith(j))) {
|
||||
// return;
|
||||
// }
|
||||
filterRuleWhitelistDomainSets.add(i);
|
||||
});
|
||||
black.forEach(i => {
|
||||
if (PREDEFINED_ENFORCED_BACKLIST.some(j => i.endsWith(j))) {
|
||||
return;
|
||||
}
|
||||
filterRuleWhitelistDomainSets.add(i)
|
||||
// if (PREDEFINED_ENFORCED_BACKLIST.some(j => i.endsWith(j))) {
|
||||
// return;
|
||||
// }
|
||||
filterRuleWhitelistDomainSets.add(i);
|
||||
});
|
||||
} else {
|
||||
process.exit(1);
|
||||
}
|
||||
})
|
||||
));
|
||||
))
|
||||
]);
|
||||
|
||||
console.timeEnd('* Download and process AdBlock Filter Rules');
|
||||
const trie0 = Trie.from(Array.from(filterRuleWhitelistDomainSets));
|
||||
PREDEFINED_ENFORCED_BACKLIST.forEach(enforcedBlack => {
|
||||
trie0.find(enforcedBlack).forEach(found => filterRuleWhitelistDomainSets.delete(found));
|
||||
});
|
||||
|
||||
console.timeEnd('* Download and process Hosts / AdBlock Filter Rules');
|
||||
|
||||
if (shouldStop) {
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
previousSize = domainSets.size - previousSize;
|
||||
console.log(`Import ${previousSize} rules from adguard filters!`);
|
||||
let previousSize = domainSets.size;
|
||||
console.log(`Import ${previousSize} rules from Hosts / AdBlock Filter Rules!`);
|
||||
|
||||
await fsPromises.readFile(pathResolve(__dirname, '../Source/domainset/reject_sukka.conf'), { encoding: 'utf-8' }).then(data => {
|
||||
data.split('\n').forEach(line => {
|
||||
const trimmed = line.trim();
|
||||
const rl1 = readline.createInterface({
|
||||
input: fs.createReadStream(pathResolve(__dirname, '../Source/domainset/reject_sukka.conf'), { encoding: 'utf-8' }),
|
||||
crlfDelay: Infinity
|
||||
});
|
||||
|
||||
for await (const line of rl1) {
|
||||
if (
|
||||
line.startsWith('#')
|
||||
|| line.startsWith(' ')
|
||||
|| line.startsWith('\r')
|
||||
|| line.startsWith('\n')
|
||||
|| trimmed === ''
|
||||
) {
|
||||
return;
|
||||
continue;
|
||||
}
|
||||
|
||||
const trimmed = line.trim();
|
||||
if (trimmed === '') continue;
|
||||
|
||||
domainSets.add(trimmed);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
previousSize = domainSets.size - previousSize;
|
||||
console.log(`Import ${previousSize} rules from reject_sukka.conf!`);
|
||||
|
||||
await Promise.all([
|
||||
// Copy reject_sukka.conf for backward compatibility
|
||||
fse.copy(pathResolve(__dirname, '../Source/domainset/reject_sukka.conf'), pathResolve(__dirname, '../List/domainset/reject_sukka.conf')),
|
||||
fsPromises.readFile(pathResolve(__dirname, '../List/non_ip/reject.conf'), { encoding: 'utf-8' }).then(data => {
|
||||
data.split('\n').forEach(line => {
|
||||
const rl2 = readline.createInterface({
|
||||
input: fs.createReadStream(pathResolve(__dirname, '../List/non_ip/reject.conf'), { encoding: 'utf-8' }),
|
||||
crlfDelay: Infinity
|
||||
});
|
||||
for await (const line of rl2) {
|
||||
if (line.startsWith('DOMAIN-KEYWORD')) {
|
||||
const [, ...keywords] = line.split(',');
|
||||
domainKeywordsSet.add(keywords.join(',').trim());
|
||||
@ -135,33 +139,34 @@ const domainSuffixSet = new Set();
|
||||
const [, ...keywords] = line.split(',');
|
||||
domainSuffixSet.add(keywords.join(',').trim());
|
||||
}
|
||||
}
|
||||
|
||||
const rl3 = readline.createInterface({
|
||||
input: fs.createReadStream(pathResolve(__dirname, '../List/domainset/reject_phishing.conf'), { encoding: 'utf-8' }),
|
||||
crlfDelay: Infinity
|
||||
});
|
||||
}),
|
||||
// Read Special Phishing Suffix list
|
||||
fsPromises.readFile(pathResolve(__dirname, '../List/domainset/reject_phishing.conf'), { encoding: 'utf-8' }).then(data => {
|
||||
data.split('\n').forEach(line => {
|
||||
const trimmed = line.trim();
|
||||
for await (const line of rl3) {
|
||||
if (
|
||||
line.startsWith('#')
|
||||
|| line.startsWith(' ')
|
||||
|| line.startsWith('\r')
|
||||
|| line.startsWith('\n')
|
||||
|| trimmed === ''
|
||||
) {
|
||||
return;
|
||||
continue;
|
||||
}
|
||||
|
||||
const trimmed = line.trim();
|
||||
if (trimmed === '') continue;
|
||||
|
||||
domainSuffixSet.add(trimmed);
|
||||
});
|
||||
})
|
||||
]);
|
||||
}
|
||||
|
||||
console.log(`Import ${domainKeywordsSet.size} black keywords and ${domainSuffixSet.size} black suffixes!`);
|
||||
|
||||
previousSize = domainSets.size;
|
||||
// Dedupe domainSets
|
||||
console.log(`Start deduping from black keywords/suffixes! (${previousSize})`);
|
||||
console.time(`* Dedupe from black keywords/suffixes`);
|
||||
console.time('* Dedupe from black keywords/suffixes');
|
||||
|
||||
const trie1 = Trie.from(Array.from(domainSets));
|
||||
domainSuffixSet.forEach(suffix => {
|
||||
@ -174,16 +179,14 @@ const domainSuffixSet = new Set();
|
||||
// Build whitelist trie, to handle case like removing `g.msn.com` due to white `.g.msn.com` (`@@||g.msn.com`)
|
||||
const trieWhite = Trie.from(Array.from(filterRuleWhitelistDomainSets));
|
||||
for (const domain of domainSets) {
|
||||
if (domain[0] !== '.' && trieWhite.has(`.${domain}`)) {
|
||||
domainSets.delete(domain);
|
||||
continue;
|
||||
}
|
||||
if (domain[0] === '.') {
|
||||
const found = trieWhite.find(domain);
|
||||
if (found.length > 0) {
|
||||
if (trieWhite.contains(domain)) {
|
||||
domainSets.delete(domain);
|
||||
continue;
|
||||
}
|
||||
} else if (trieWhite.has(`.${domain}`)) {
|
||||
domainSets.delete(domain);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Remove keyword
|
||||
@ -192,7 +195,7 @@ const domainSuffixSet = new Set();
|
||||
}
|
||||
}
|
||||
|
||||
console.timeEnd(`* Dedupe from black keywords/suffixes`);
|
||||
console.timeEnd('* Dedupe from black keywords/suffixes');
|
||||
console.log(`Deduped ${previousSize} - ${domainSets.size} = ${previousSize - domainSets.size} from black keywords and suffixes!`);
|
||||
|
||||
previousSize = domainSets.size;
|
||||
@ -212,7 +215,7 @@ const domainSuffixSet = new Set();
|
||||
if (found.length) {
|
||||
found.forEach(f => {
|
||||
domainSets.delete(f);
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
const a = domainStartsWithADotAndFromFullSet.slice(1);
|
||||
@ -237,12 +240,10 @@ const domainSuffixSet = new Set();
|
||||
};
|
||||
const sortedDomainSets = Array.from(domainSets)
|
||||
.map((v) => {
|
||||
return { v, domain: getDomain(v.charCodeAt(0) === 46 ? v.slice(1) : v)?.toLowerCase() || v };
|
||||
return { v, domain: getDomain(v.charCodeAt(0) === 46 ? v.slice(1) : v) || v };
|
||||
})
|
||||
.sort(sorter)
|
||||
.map((i) => {
|
||||
return i.v;
|
||||
});
|
||||
.map((i) => i.v);
|
||||
|
||||
await compareAndWriteFile(
|
||||
withBannerArray(
|
||||
@ -256,7 +257,7 @@ const domainSuffixSet = new Set();
|
||||
'',
|
||||
'Build from:',
|
||||
...HOSTS.map(host => ` - ${host[0]}`),
|
||||
...ADGUARD_FILTERS.map(filter => ` - ${Array.isArray(filter) ? filter[0] : filter}`),
|
||||
...ADGUARD_FILTERS.map(filter => ` - ${Array.isArray(filter) ? filter[0] : filter}`)
|
||||
],
|
||||
new Date(),
|
||||
sortedDomainSets
|
||||
@ -264,6 +265,9 @@ const domainSuffixSet = new Set();
|
||||
pathResolve(__dirname, '../List/domainset/reject.conf')
|
||||
);
|
||||
|
||||
// Copy reject_sukka.conf for backward compatibility
|
||||
await fse.copy(pathResolve(__dirname, '../Source/domainset/reject_sukka.conf'), pathResolve(__dirname, '../List/domainset/reject_sukka.conf'));
|
||||
|
||||
console.timeEnd('* Write reject.conf');
|
||||
|
||||
console.timeEnd('Total Time - build-reject-domain-set');
|
||||
|
||||
@ -34,7 +34,7 @@ const { withBanner } = require('./lib/with-banner');
|
||||
date,
|
||||
res.map(ip => {
|
||||
const [subnet] = ip.split('/');
|
||||
console.log(' - ' + ip + ': ' + subnet);
|
||||
console.log(` - ${ip}: ${subnet}`);
|
||||
if (isIPv4(subnet)) {
|
||||
return `IP-CIDR,${ip},no-resolve`;
|
||||
}
|
||||
@ -44,7 +44,7 @@ const { withBanner } = require('./lib/with-banner');
|
||||
return '';
|
||||
})
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
console.timeEnd('Total Time - build-telegram-cidr');
|
||||
})();
|
||||
|
||||
@ -41,7 +41,7 @@ const fileExists = (path) => {
|
||||
file: tempFile,
|
||||
cwd: extractedPath,
|
||||
filter: (p) => {
|
||||
return p.split('/')[1] === 'List'
|
||||
return p.split('/')[1] === 'List';
|
||||
}
|
||||
});
|
||||
|
||||
@ -51,7 +51,7 @@ const fileExists = (path) => {
|
||||
{
|
||||
overwrite: true
|
||||
}
|
||||
)))
|
||||
)));
|
||||
|
||||
await fs.promises.unlink(tempFile).catch(() => { });
|
||||
await fs.promises.unlink(extractedPath).catch(() => { });
|
||||
|
||||
@ -17,4 +17,4 @@ module.exports.fetchRemoteTextAndCreateReadlineInterface = async (url, opt) => {
|
||||
input: Readable.fromWeb(resp.body),
|
||||
crlfDelay: Infinity
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
@ -23,7 +23,7 @@ module.exports.normalizeDomain = (domain) => {
|
||||
|
||||
if (isIcann || isPrivate) {
|
||||
return hostname;
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
@ -15,7 +15,7 @@ const warnOnce = (url, isWhite, ...message) => {
|
||||
}
|
||||
warnOnceUrl.add(key);
|
||||
console.warn(url, isWhite ? '(white)' : '(black)', ...message);
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* @param {string | URL} domainListsUrl
|
||||
@ -99,6 +99,9 @@ async function processHosts(hostsUrl, includeAllSubDomain = false) {
|
||||
return domainSets;
|
||||
}
|
||||
|
||||
const R_KNOWN_NOT_NETWORK_FILTER_PATTERN = /[#&%~=]/;
|
||||
const R_KNOWN_NOT_NETWORK_FILTER_PATTERN_2 = /(\$popup|\$removeparam|\$popunder)/;
|
||||
|
||||
/**
|
||||
* @param {string | URL} filterRulesUrl
|
||||
* @param {readonly (string | URL)[] | undefined} [fallbackUrls]
|
||||
@ -130,7 +133,7 @@ async function processFilterRules(filterRulesUrl, fallbackUrls, includeThirdPart
|
||||
foundDebugDomain = true;
|
||||
}
|
||||
whitelistDomainSets.add(domainToBeAddedToWhite);
|
||||
}
|
||||
};
|
||||
|
||||
let filterRules;
|
||||
try {
|
||||
@ -143,7 +146,7 @@ async function processFilterRules(filterRulesUrl, fallbackUrls, includeThirdPart
|
||||
)
|
||||
).split('\n').map(line => line.trim());
|
||||
} catch (e) {
|
||||
console.log('Download Rule for [' + filterRulesUrl + '] failed');
|
||||
console.log(`Download Rule for [${filterRulesUrl}] failed`);
|
||||
throw e;
|
||||
}
|
||||
|
||||
@ -154,31 +157,34 @@ async function processFilterRules(filterRulesUrl, fallbackUrls, includeThirdPart
|
||||
|
||||
if (
|
||||
line === ''
|
||||
|| line.startsWith('/')
|
||||
|| R_KNOWN_NOT_NETWORK_FILTER_PATTERN.test(line)
|
||||
// doesn't include
|
||||
|| !line.includes('.') // rule with out dot can not be a domain
|
||||
// includes
|
||||
|| line.includes('#')
|
||||
// || line.includes('#')
|
||||
|| line.includes('!')
|
||||
|| line.includes('?')
|
||||
|| line.includes('*')
|
||||
|| line.includes('=')
|
||||
// || line.includes('=')
|
||||
|| line.includes('[')
|
||||
|| line.includes('(')
|
||||
|| line.includes(']')
|
||||
|| line.includes(')')
|
||||
|| line.includes(',')
|
||||
|| line.includes('~')
|
||||
|| line.includes('&')
|
||||
|| line.includes('%')
|
||||
// || line.includes('~')
|
||||
// || line.includes('&')
|
||||
// || line.includes('%')
|
||||
|| ((line.includes('/') || line.includes(':')) && !line.includes('://'))
|
||||
// ends with
|
||||
|| line.endsWith('.')
|
||||
|| line.endsWith('-')
|
||||
|| line.endsWith('_')
|
||||
// special modifier
|
||||
|| line.includes('$popup')
|
||||
|| line.includes('$removeparam')
|
||||
|| line.includes('$popunder')
|
||||
|| R_KNOWN_NOT_NETWORK_FILTER_PATTERN_2.test(line)
|
||||
// || line.includes('$popup')
|
||||
// || line.includes('$removeparam')
|
||||
// || line.includes('$popunder')
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
@ -401,7 +407,6 @@ function preprocessFullDomainSetBeforeUsedAsWorkerData(data) {
|
||||
.sort((a, b) => a.length - b.length);
|
||||
}
|
||||
|
||||
|
||||
module.exports.processDomainLists = processDomainLists;
|
||||
module.exports.processHosts = processHosts;
|
||||
module.exports.processFilterRules = processFilterRules;
|
||||
|
||||
@ -6,7 +6,7 @@ const HOSTS = [
|
||||
['https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/spy.txt', false],
|
||||
['https://raw.githubusercontent.com/jerryn70/GoodbyeAds/master/Extension/GoodbyeAds-Xiaomi-Extension.txt', false],
|
||||
['https://raw.githubusercontent.com/jdlingyu/ad-wars/master/hosts', false]
|
||||
]
|
||||
];
|
||||
|
||||
const ADGUARD_FILTERS = /** @type {const} */([
|
||||
// Easy List
|
||||
@ -142,7 +142,7 @@ const ADGUARD_FILTERS = /** @type {const} */([
|
||||
[
|
||||
'https://curbengh.github.io/urlhaus-filter/urlhaus-filter-agh-online.txt',
|
||||
[
|
||||
'https://urlhaus-filter.pages.dev/urlhaus-filter-agh-online.txt',
|
||||
'https://urlhaus-filter.pages.dev/urlhaus-filter-agh-online.txt'
|
||||
// Prefer mirror, since malware-filter.gitlab.io has not been updated for a while
|
||||
// 'https://malware-filter.gitlab.io/urlhaus-filter/urlhaus-filter-agh-online.txt'
|
||||
],
|
||||
@ -152,7 +152,7 @@ const ADGUARD_FILTERS = /** @type {const} */([
|
||||
[
|
||||
'https://curbengh.github.io/phishing-filter/phishing-filter-agh.txt',
|
||||
[
|
||||
'https://phishing-filter.pages.dev/phishing-filter-agh.txt',
|
||||
'https://phishing-filter.pages.dev/phishing-filter-agh.txt'
|
||||
// Prefer mirror, since malware-filter.gitlab.io has not been updated for a while
|
||||
// 'https://malware-filter.gitlab.io/malware-filter/phishing-filter-agh.txt'
|
||||
],
|
||||
@ -162,7 +162,7 @@ const ADGUARD_FILTERS = /** @type {const} */([
|
||||
[
|
||||
'https://curbengh.github.io/pup-filter/pup-filter-agh.txt',
|
||||
[
|
||||
'https://pup-filter.pages.dev/pup-filter-agh.txt',
|
||||
'https://pup-filter.pages.dev/pup-filter-agh.txt'
|
||||
// Prefer mirror, since malware-filter.gitlab.io has not been updated for a while
|
||||
// 'https://malware-filter.gitlab.io/malware-filter/pup-filter-agh.txt'
|
||||
],
|
||||
@ -241,7 +241,7 @@ const PREDEFINED_ENFORCED_WHITELIST = [
|
||||
'repl.co',
|
||||
'w3s.link',
|
||||
'translate.goog'
|
||||
]
|
||||
];
|
||||
|
||||
module.exports.HOSTS = HOSTS;
|
||||
module.exports.ADGUARD_FILTERS = ADGUARD_FILTERS;
|
||||
|
||||
@ -13,7 +13,7 @@ async function compareAndWriteFile(linesA, filePath) {
|
||||
filePath,
|
||||
linesA.join('\n'),
|
||||
{ encoding: 'utf-8' }
|
||||
)
|
||||
);
|
||||
} else {
|
||||
console.log(`Same Content, bail out writing: ${filePath}`);
|
||||
}
|
||||
@ -23,7 +23,7 @@ async function compareAndWriteFile(linesA, filePath) {
|
||||
* @param {string[]} linesA
|
||||
* @param {string[]} linesB
|
||||
*/
|
||||
function stringArrayCompare (linesA, linesB) {
|
||||
function stringArrayCompare(linesA, linesB) {
|
||||
if (linesA.length !== linesB.length) return false;
|
||||
|
||||
for (let i = 0; i < linesA.length; i++) {
|
||||
|
||||
@ -31,6 +31,24 @@ class Trie {
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} suffix
|
||||
*/
|
||||
contains(suffix) {
|
||||
let node = this.root;
|
||||
let token;
|
||||
|
||||
for (let i = suffix.length - 1; i >= 0; i--) {
|
||||
token = suffix[i];
|
||||
|
||||
node = node[token];
|
||||
|
||||
if (node == null) return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Method used to retrieve every item in the trie with the given prefix.
|
||||
*
|
||||
@ -42,8 +60,6 @@ class Trie {
|
||||
let node = this.root;
|
||||
const matches = [];
|
||||
let token;
|
||||
let i;
|
||||
let l;
|
||||
|
||||
for (let i = suffix.length - 1; i >= 0; i--) {
|
||||
token = suffix[i];
|
||||
|
||||
@ -72,10 +72,10 @@ const validateRuleset = async (filePath) => {
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
(async () => {
|
||||
const [domainsetFiles, rulesetFiles] = await Promise.all([
|
||||
const [domainsetFiles, _rulesetFiles] = await Promise.all([
|
||||
listDir(path.resolve(__dirname, '../List/domainset')),
|
||||
listDir(path.resolve(__dirname, '../List/non_ip'))
|
||||
]);
|
||||
@ -84,4 +84,3 @@ const validateRuleset = async (filePath) => {
|
||||
// rulesetFiles.map(file => validateRuleset(file))
|
||||
);
|
||||
})();
|
||||
|
||||
|
||||
@ -127,11 +127,13 @@
|
||||
"devDependencies": {
|
||||
"@types/mocha": "^10.0.1",
|
||||
"chai": "^4.3.7",
|
||||
"eslint-config-sukka": "^1.8.6",
|
||||
"eslint-plugin-import": "npm:eslint-plugin-i@2.27.5-4",
|
||||
"eslint-plugin-node": "^11.1.0",
|
||||
"mocha": "^10.2.0",
|
||||
"wireit": "^0.9.5"
|
||||
},
|
||||
"engines": {
|
||||
"node": "> 18.0.0"
|
||||
"node": ">=18.0.0"
|
||||
}
|
||||
}
|
||||
|
||||
72
pnpm-lock.yaml
generated
72
pnpm-lock.yaml
generated
@ -49,9 +49,15 @@ devDependencies:
|
||||
chai:
|
||||
specifier: ^4.3.7
|
||||
version: 4.3.7
|
||||
eslint-config-sukka:
|
||||
specifier: ^1.8.6
|
||||
version: 1.8.6(eslint@8.44.0)
|
||||
eslint-plugin-import:
|
||||
specifier: npm:eslint-plugin-i@2.27.5-4
|
||||
version: /eslint-plugin-i@2.27.5-4(eslint@8.44.0)
|
||||
eslint-plugin-node:
|
||||
specifier: ^11.1.0
|
||||
version: 11.1.0(eslint@8.44.0)
|
||||
mocha:
|
||||
specifier: ^10.2.0
|
||||
version: 10.2.0
|
||||
@ -130,6 +136,15 @@ packages:
|
||||
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
|
||||
dev: true
|
||||
|
||||
/@fluffyfox/eslint-plugin@0.1.0(eslint@8.44.0):
|
||||
resolution: {integrity: sha512-akpFZb5VGXaja0zrndWNsp8pjxGVNvhmT34sk9MsXAVrVlv7LB4pqfSGJ684y6kR2FIDsT1cVcPIbGuD5W00Sg==}
|
||||
engines: {node: '>= 16'}
|
||||
peerDependencies:
|
||||
eslint: '>= 8'
|
||||
dependencies:
|
||||
eslint: 8.44.0
|
||||
dev: true
|
||||
|
||||
/@humanwhocodes/config-array@0.11.10:
|
||||
resolution: {integrity: sha512-KVVjQmNUepDVGXNuoRRdmmEjruj0KfiGSbS8LVc12LMsWDQzRXJ0qdhN8L8uUigKpfEHRhlaQFY0ib1tnUbNeQ==}
|
||||
engines: {node: '>=10.10.0'}
|
||||
@ -541,6 +556,15 @@ packages:
|
||||
engines: {node: '>=10'}
|
||||
dev: true
|
||||
|
||||
/eslint-config-sukka@1.8.6(eslint@8.44.0):
|
||||
resolution: {integrity: sha512-xt6/Knl7Fuwvpecbt1ysMF5JrR2Cmya2zRbI7+SVFxI37pcjbcePusHKqwfm6yFstXdYaC5uBKIYostCsNv82A==}
|
||||
engines: {node: '>= 8.3.0'}
|
||||
dependencies:
|
||||
'@fluffyfox/eslint-plugin': 0.1.0(eslint@8.44.0)
|
||||
transitivePeerDependencies:
|
||||
- eslint
|
||||
dev: true
|
||||
|
||||
/eslint-import-resolver-node@0.3.7:
|
||||
resolution: {integrity: sha512-gozW2blMLJCeFpBwugLTGyvVjNoeo1knonXAcatC6bjPBZitotxdWf7Gimr25N4c0AAOo4eOUfaG82IJPDpqCA==}
|
||||
dependencies:
|
||||
@ -579,6 +603,17 @@ packages:
|
||||
- supports-color
|
||||
dev: true
|
||||
|
||||
/eslint-plugin-es@3.0.1(eslint@8.44.0):
|
||||
resolution: {integrity: sha512-GUmAsJaN4Fc7Gbtl8uOBlayo2DqhwWvEzykMHSCZHU3XdJ+NSzzZcVhXh3VxX5icqQ+oQdIEawXX8xkR3mIFmQ==}
|
||||
engines: {node: '>=8.10.0'}
|
||||
peerDependencies:
|
||||
eslint: '>=4.19.1'
|
||||
dependencies:
|
||||
eslint: 8.44.0
|
||||
eslint-utils: 2.1.0
|
||||
regexpp: 3.2.0
|
||||
dev: true
|
||||
|
||||
/eslint-plugin-i@2.27.5-4(eslint@8.44.0):
|
||||
resolution: {integrity: sha512-X3Z+dp9nZw7d/y41EDO6JyFw4WVMOT91SFuoJvL0C0/4M1l6NxQ5mLTjXHuYhq0AazW75pAmj25yMk5wPMzjsw==}
|
||||
engines: {node: '>=12'}
|
||||
@ -602,6 +637,21 @@ packages:
|
||||
- supports-color
|
||||
dev: true
|
||||
|
||||
/eslint-plugin-node@11.1.0(eslint@8.44.0):
|
||||
resolution: {integrity: sha512-oUwtPJ1W0SKD0Tr+wqu92c5xuCeQqB3hSCHasn/ZgjFdA9iDGNkNf2Zi9ztY7X+hNuMib23LNGRm6+uN+KLE3g==}
|
||||
engines: {node: '>=8.10.0'}
|
||||
peerDependencies:
|
||||
eslint: '>=5.16.0'
|
||||
dependencies:
|
||||
eslint: 8.44.0
|
||||
eslint-plugin-es: 3.0.1(eslint@8.44.0)
|
||||
eslint-utils: 2.1.0
|
||||
ignore: 5.2.4
|
||||
minimatch: 3.1.2
|
||||
resolve: 1.22.3
|
||||
semver: 6.3.1
|
||||
dev: true
|
||||
|
||||
/eslint-scope@7.2.0:
|
||||
resolution: {integrity: sha512-DYj5deGlHBfMt15J7rdtyKNq/Nqlv5KfU4iodrQ019XESsRnwXH9KAE0y3cwtUHDo2ob7CypAnCqefh6vioWRw==}
|
||||
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
|
||||
@ -610,6 +660,18 @@ packages:
|
||||
estraverse: 5.3.0
|
||||
dev: true
|
||||
|
||||
/eslint-utils@2.1.0:
|
||||
resolution: {integrity: sha512-w94dQYoauyvlDc43XnGB8lU3Zt713vNChgt4EWwhXAP2XkBvndfxF0AgIqKOOasjPIPzj9JqgwkwbCYD0/V3Zg==}
|
||||
engines: {node: '>=6'}
|
||||
dependencies:
|
||||
eslint-visitor-keys: 1.3.0
|
||||
dev: true
|
||||
|
||||
/eslint-visitor-keys@1.3.0:
|
||||
resolution: {integrity: sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==}
|
||||
engines: {node: '>=4'}
|
||||
dev: true
|
||||
|
||||
/eslint-visitor-keys@3.4.1:
|
||||
resolution: {integrity: sha512-pZnmmLwYzf+kWaM/Qgrvpen51upAktaaiI01nsJD/Yr3lMOdNtq0cxkrrg16w64VtisN6okbs7Q8AfGqj4c9fA==}
|
||||
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
|
||||
@ -1313,6 +1375,11 @@ packages:
|
||||
picomatch: 2.3.1
|
||||
dev: true
|
||||
|
||||
/regexpp@3.2.0:
|
||||
resolution: {integrity: sha512-pq2bWo9mVD43nbts2wGv17XLiNLya+GklZ8kaDLV2Z08gDCsGpnKn9BFMepvWuHCbyVvY7J5o5+BVvoQbmlJLg==}
|
||||
engines: {node: '>=8'}
|
||||
dev: true
|
||||
|
||||
/require-directory@2.1.1:
|
||||
resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==}
|
||||
engines: {node: '>=0.10.0'}
|
||||
@ -1373,6 +1440,11 @@ packages:
|
||||
resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==}
|
||||
dev: true
|
||||
|
||||
/semver@6.3.1:
|
||||
resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==}
|
||||
hasBin: true
|
||||
dev: true
|
||||
|
||||
/semver@7.5.3:
|
||||
resolution: {integrity: sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ==}
|
||||
engines: {node: '>=10'}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user