diff --git a/Build/build-reject-domainset.js b/Build/build-reject-domainset.js index 2321e02e..2af52de4 100644 --- a/Build/build-reject-domainset.js +++ b/Build/build-reject-domainset.js @@ -1,156 +1,8 @@ -const { default: got } = require('got-cjs'); const { promises: fsPromises } = require('fs'); const { resolve: pathResolve } = require('path'); -const { cpus } = require('os'); -const { isIP } = require('net'); - -const threads = Math.max(cpus().length, 12); - -const rDomain = /^(((?!\-))(xn\-\-)?[a-z0-9\-_]{0,61}[a-z0-9]{1,1}\.)*(xn\-\-)?([a-z0-9\-]{1,61}|[a-z0-9\-]{1,30})\.[a-z]{2,}$/m - const Piscina = require('piscina'); - -/** - * @param {string | URL} domainListsUrl - */ -async function processDomainLists(domainListsUrl) { - if (typeof domainListsUrl === 'string') { - domainListsUrl = new URL(domainListsUrl); - } - - /** @type Set */ - const domainSets = new Set(); - /** @type string[] */ - const domains = (await got(domainListsUrl).text()).split('\n'); - domains.forEach(line => { - if ( - line.startsWith('#') - || line.startsWith('!') - || line.startsWith(' ') - || line === '' - || line.startsWith('\r') - || line.startsWith('\n') - ) { - return; - } - domainSets.add(line.trim()); - }); - - return [...domainSets]; -} - -/** - * @param {string | URL} hostsUrl - */ -async function processHosts(hostsUrl, includeAllSubDomain = false) { - if (typeof hostsUrl === 'string') { - hostsUrl = new URL(hostsUrl); - } - - /** @type Set */ - const domainSets = new Set(); - - /** @type string[] */ - const hosts = (await got(hostsUrl).text()).split('\n'); - hosts.forEach(line => { - if (line.includes('#')) { - return; - } - if (line.startsWith(' ') || line.startsWith('\r') || line.startsWith('\n') || line.trim() === '') { - return; - } - const [, ...domains] = line.split(' '); - const domain = domains.join(' ').trim(); - if (rDomain.test(domain)) { - if (includeAllSubDomain) { - domainSets.add(`.${domain}`); - } else { - domainSets.add(domain); - } - } - }); - - return [...domainSets]; -} - -/** - * @param {string | URL} filterRulesUrl - * @returns {Promise<{ white: Set, black: Set }>} - */ -async function processFilterRules(filterRulesUrl) { - if (typeof filterRulesUrl === 'string') { - filterRulesUrl = new URL(filterRulesUrl); - } - - /** @type Set */ - const whitelistDomainSets = new Set(); - /** @type Set */ - const blacklistDomainSets = new Set(); - - /** @type string[] */ - const filterRules = (await got(filterRulesUrl).text()).split('\n').map(line => line.trim()); - - filterRules.forEach(line => { - if ( - line === '' - || line.includes('#') - || line.includes('!') - || line.includes('*') - || line.includes('/') - || line.includes('$') && !line.startsWith('@@') - || line.trim() === '' - || isIP(line) !== 0 - ) { - return; - } - - if (line.startsWith('@@||') - && ( - line.endsWith('^') - || line.endsWith('^|') - || line.endsWith('^$badfilter') - || line.endsWith('^$1p') - ) - ) { - const domain = line - .replaceAll('@@||', '') - .replaceAll('^$badfilter', '') - .replaceAll('^$1p', '') - .replaceAll('^|', '') - .replaceAll('^', '') - .trim(); - if (rDomain.test(domain)) { - whitelistDomainSets.add(domain); - } - } else if ( - line.startsWith('||') - && ( - line.endsWith('^') - || line.endsWith('^|') - ) - ) { - const domain = `${line.replaceAll('||', '').replaceAll('^|', '').replaceAll('^', '')}`.trim(); - if (rDomain.test(domain)) { - blacklistDomainSets.add(`.${domain}`); - } - } else if (line.startsWith('://') - && ( - line.endsWith('^') - || line.endsWith('^|') - ) - ) { - const domain = `${line.replaceAll('://', '').replaceAll('^|', '').replaceAll('^', '')}`.trim(); - if (rDomain.test(domain)) { - blacklistDomainSets.add(domain); - } - } - }); - - return { - white: whitelistDomainSets, - black: blacklistDomainSets - }; -} +const { processHosts, processFilterRules } = require('./lib/parse-filter'); +const threads = Math.max(require('os').cpus().length, 12); (async () => { /** @type Set */ diff --git a/Build/lib/parse-filter.js b/Build/lib/parse-filter.js new file mode 100644 index 00000000..1666ce0e --- /dev/null +++ b/Build/lib/parse-filter.js @@ -0,0 +1,150 @@ +const { isIP } = require('net'); +const { default: got } = require('got-cjs'); + +const rDomain = /^(((?!\-))(xn\-\-)?[a-z0-9\-_]{0,61}[a-z0-9]{1,1}\.)*(xn\-\-)?([a-z0-9\-]{1,61}|[a-z0-9\-]{1,30})\.[a-z]{2,}$/m + +/** + * @param {string | URL} domainListsUrl + */ +async function processDomainLists(domainListsUrl) { + if (typeof domainListsUrl === 'string') { + domainListsUrl = new URL(domainListsUrl); + } + + /** @type Set */ + const domainSets = new Set(); + /** @type string[] */ + const domains = (await got(domainListsUrl).text()).split('\n'); + domains.forEach(line => { + if ( + line.startsWith('#') + || line.startsWith('!') + || line.startsWith(' ') + || line === '' + || line.startsWith('\r') + || line.startsWith('\n') + ) { + return; + } + domainSets.add(line.trim()); + }); + + return [...domainSets]; +} + +/** + * @param {string | URL} hostsUrl + */ +async function processHosts(hostsUrl, includeAllSubDomain = false) { + if (typeof hostsUrl === 'string') { + hostsUrl = new URL(hostsUrl); + } + + /** @type Set */ + const domainSets = new Set(); + + /** @type string[] */ + const hosts = (await got(hostsUrl).text()).split('\n'); + hosts.forEach(line => { + if (line.includes('#')) { + return; + } + if (line.startsWith(' ') || line.startsWith('\r') || line.startsWith('\n') || line.trim() === '') { + return; + } + const [, ...domains] = line.split(' '); + const domain = domains.join(' ').trim(); + if (rDomain.test(domain)) { + if (includeAllSubDomain) { + domainSets.add(`.${domain}`); + } else { + domainSets.add(domain); + } + } + }); + + return [...domainSets]; +} + +/** + * @param {string | URL} filterRulesUrl + * @returns {Promise<{ white: Set, black: Set }>} + */ +async function processFilterRules(filterRulesUrl) { + if (typeof filterRulesUrl === 'string') { + filterRulesUrl = new URL(filterRulesUrl); + } + + /** @type Set */ + const whitelistDomainSets = new Set(); + /** @type Set */ + const blacklistDomainSets = new Set(); + + /** @type string[] */ + const filterRules = (await got(filterRulesUrl).text()).split('\n').map(line => line.trim()); + + filterRules.forEach(line => { + if ( + line === '' + || line.includes('#') + || line.includes('!') + || line.includes('*') + || line.includes('/') + || line.includes('$') && !line.startsWith('@@') + || line.trim() === '' + || isIP(line) !== 0 + ) { + return; + } + + if (line.startsWith('@@||') + && ( + line.endsWith('^') + || line.endsWith('^|') + || line.endsWith('^$badfilter') + || line.endsWith('^$1p') + ) + ) { + const domain = line + .replaceAll('@@||', '') + .replaceAll('^$badfilter', '') + .replaceAll('^$1p', '') + .replaceAll('^|', '') + .replaceAll('^', '') + .trim(); + if (rDomain.test(domain)) { + whitelistDomainSets.add(domain); + } + } else if ( + line.startsWith('||') + && ( + line.endsWith('^') + || line.endsWith('^|') + ) + ) { + const domain = `${line.replaceAll('||', '').replaceAll('^|', '').replaceAll('^', '')}`.trim(); + if (rDomain.test(domain)) { + blacklistDomainSets.add(`.${domain}`); + } + } else if (line.startsWith('://') + && ( + line.endsWith('^') + || line.endsWith('^|') + ) + ) { + const domain = `${line.replaceAll('://', '').replaceAll('^|', '').replaceAll('^', '')}`.trim(); + if (rDomain.test(domain)) { + blacklistDomainSets.add(domain); + } + } + }); + + return { + white: whitelistDomainSets, + black: blacklistDomainSets + }; +} + +module.exports.processDomainLists = processDomainLists; +module.exports.processHosts = processHosts; +module.exports.processFilterRules = processFilterRules;