mirror of
https://github.com/SukkaW/Surge.git
synced 2025-12-12 01:00:34 +08:00
Update Reject Filter Parsing
This commit is contained in:
parent
14494a0919
commit
44aeb217d8
@ -40,17 +40,19 @@ const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST);
|
||||
|
||||
// Parse from AdGuard Filters
|
||||
console.time('* Download and process AdBlock Filter Rules');
|
||||
|
||||
let shouldStop = false;
|
||||
await Promise.all(ADGUARD_FILTERS.map(input => {
|
||||
const promise = Array.isArray(input) && input.length === 2
|
||||
? processFilterRules(input[0], input[1])
|
||||
: processFilterRules(input);
|
||||
const promise = typeof input === 'string'
|
||||
? processFilterRules(input, undefined, false)
|
||||
: processFilterRules(input[0], input[1] ?? undefined, input[2] ?? false)
|
||||
|
||||
return promise.then((i) => {
|
||||
if (i) {
|
||||
const { white, black, foundDebugDomain } = i;
|
||||
if (foundDebugDomain) {
|
||||
process.exit(1);
|
||||
};
|
||||
shouldStop = true;
|
||||
}
|
||||
white.forEach(i => filterRuleWhitelistDomainSets.add(i));
|
||||
black.forEach(i => domainSets.add(i));
|
||||
} else {
|
||||
@ -61,6 +63,10 @@ const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST);
|
||||
|
||||
console.timeEnd('* Download and process AdBlock Filter Rules');
|
||||
|
||||
if (shouldStop) {
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
previousSize = domainSets.size - previousSize;
|
||||
console.log(`Import ${previousSize} rules from adguard filters!`);
|
||||
|
||||
@ -246,6 +252,11 @@ function isInWhiteList (domain) {
|
||||
if (domain === white || domain.endsWith(white)) {
|
||||
return true;
|
||||
}
|
||||
if (white.endsWith(domain)) {
|
||||
// If a whole domain is in blacklist but a subdomain is in whitelist
|
||||
// We have no choice but to remove the whole domain from blacklist
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
const { fetchWithRetry } = require('./fetch-retry');
|
||||
const { NetworkFilter } = require('@cliqz/adblocker');
|
||||
const { normalizeDomain } = require('./is-domain-loose');
|
||||
|
||||
const DEBUG_DOMAIN_TO_FIND = null; // example.com | null
|
||||
@ -99,9 +100,9 @@ async function processHosts (hostsUrl, includeAllSubDomain = false) {
|
||||
/**
|
||||
* @param {string | URL} filterRulesUrl
|
||||
* @param {(string | URL)[] | undefined} fallbackUrls
|
||||
* @returns {Promise<{ white: Set<string>, black: Set<string>, foundDebugDomain: boolean }>}
|
||||
* @returns {Promise<{ white: Set<string>, black: Set<string>, foundDebugDomain: boolean, parseFailed: boolean }>}
|
||||
*/
|
||||
async function processFilterRules (filterRulesUrl, fallbackUrls) {
|
||||
async function processFilterRules (filterRulesUrl, fallbackUrls, includeThirdParties = false) {
|
||||
console.time(` - processFilterRules: ${filterRulesUrl}`);
|
||||
|
||||
/** @type Set<string> */
|
||||
@ -109,6 +110,26 @@ async function processFilterRules (filterRulesUrl, fallbackUrls) {
|
||||
/** @type Set<string> */
|
||||
const blacklistDomainSets = new Set();
|
||||
|
||||
const addToBlackList = (domainToBeAddedToBlack, isSubDomain) => {
|
||||
if (DEBUG_DOMAIN_TO_FIND && domainToBeAddedToBlack.includes(DEBUG_DOMAIN_TO_FIND)) {
|
||||
warnOnce(filterRulesUrl.toString(), false, DEBUG_DOMAIN_TO_FIND);
|
||||
foundDebugDomain = true;
|
||||
}
|
||||
|
||||
if (isSubDomain && !domainToBeAddedToBlack.startsWith('.')) {
|
||||
blacklistDomainSets.add(`.${domainToBeAddedToBlack}`);
|
||||
} else {
|
||||
blacklistDomainSets.add(domainToBeAddedToBlack);
|
||||
}
|
||||
};
|
||||
const addToWhiteList = (domainToBeAddedToWhite) => {
|
||||
if (DEBUG_DOMAIN_TO_FIND && domainToBeAddedToWhite.includes(DEBUG_DOMAIN_TO_FIND)) {
|
||||
warnOnce(filterRulesUrl.toString(), false, DEBUG_DOMAIN_TO_FIND);
|
||||
foundDebugDomain = true;
|
||||
}
|
||||
whitelistDomainSets.add(domainToBeAddedToWhite);
|
||||
}
|
||||
|
||||
let filterRules;
|
||||
try {
|
||||
/** @type string[] */
|
||||
@ -124,117 +145,236 @@ async function processFilterRules (filterRulesUrl, fallbackUrls) {
|
||||
throw e;
|
||||
}
|
||||
|
||||
for (let i = 0, len = filterRules.length; i < len; i++) {
|
||||
const line = filterRules[i];
|
||||
let hasParseFailed = false;
|
||||
|
||||
const lineStartsWithDoubleVerticalBar = line.startsWith('||');
|
||||
for (let i = 0, len = filterRules.length; i < len; i++) {
|
||||
const line = filterRules[i].trim();
|
||||
|
||||
if (
|
||||
line === ''
|
||||
// doesn't include
|
||||
|| !line.includes('.') // rule with out dot can not be a domain
|
||||
// includes
|
||||
|| line.includes('#')
|
||||
|| line.includes('!')
|
||||
|| line.includes('?')
|
||||
|| line.includes('*')
|
||||
|| line.includes('/')
|
||||
|| line.includes('=')
|
||||
|| line.includes('[')
|
||||
|| line.includes('(')
|
||||
|| line.includes('$') && !lineStartsWithDoubleVerticalBar
|
||||
|| line.includes(']')
|
||||
|| line.includes(')')
|
||||
|| line.includes(',')
|
||||
|| line.includes('~')
|
||||
|| line.includes('&')
|
||||
|| line.includes('%')
|
||||
|| ((line.includes('/') || line.includes(':')) && !line.includes('://'))
|
||||
// ends with
|
||||
|| line.endsWith('.')
|
||||
|| line.endsWith('-')
|
||||
|| line.endsWith('_')
|
||||
// special modifier
|
||||
|| line.includes('$popup')
|
||||
|| line.includes('$removeparam')
|
||||
|| line.includes('$popunder')
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const filter = NetworkFilter.parse(line);
|
||||
if (filter) {
|
||||
if (
|
||||
filter.isElemHide()
|
||||
|| filter.isGenericHide()
|
||||
|| filter.isSpecificHide()
|
||||
|| filter.isRedirect()
|
||||
|| filter.isRedirectRule()
|
||||
|| filter.hasDomains()
|
||||
|| filter.isCSP() // must not be csp rule
|
||||
|| (!filter.fromAny() && !filter.fromDocument())
|
||||
) {
|
||||
// not supported type
|
||||
continue;
|
||||
}
|
||||
|
||||
if (
|
||||
filter.hasHostname() // must have
|
||||
&& filter.isPlain()
|
||||
&& (!filter.isRegex())
|
||||
&& (!filter.isFullRegex())
|
||||
) {
|
||||
const hostname = filter.getHostname();
|
||||
if (hostname) {
|
||||
if (filter.isException() || filter.isBadFilter()) {
|
||||
addToWhiteList(hostname);
|
||||
continue;
|
||||
}
|
||||
if (filter.firstParty() === filter.thirdParty()) {
|
||||
addToBlackList(hostname, true);
|
||||
continue;
|
||||
}
|
||||
if (filter.thirdParty()) {
|
||||
if (includeThirdParties) {
|
||||
addToBlackList(hostname, true);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (filter.firstParty()) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (line.includes('$third-party') || line.includes('$frame')) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const lineEndsWithCaret = line.endsWith('^');
|
||||
const lineEndsWithCaretVerticalBar = line.endsWith('^|');
|
||||
|
||||
if (lineStartsWithDoubleVerticalBar && line.endsWith('^$badfilter')) {
|
||||
const _domain = line.replace('||', '').replace('^$badfilter', '').trim();
|
||||
const domain = normalizeDomain(_domain);
|
||||
if (domain) {
|
||||
if (DEBUG_DOMAIN_TO_FIND && domain.includes(DEBUG_DOMAIN_TO_FIND)) {
|
||||
warnOnce(filterRulesUrl.toString(), true, DEBUG_DOMAIN_TO_FIND);
|
||||
foundDebugDomain = true;
|
||||
if (line.startsWith('@@')) {
|
||||
if (line.endsWith('$cname')) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (
|
||||
(line.startsWith('@@|') || line.startsWith('@@.'))
|
||||
&& (
|
||||
lineEndsWithCaret
|
||||
|| lineEndsWithCaretVerticalBar
|
||||
|| line.endsWith('$genericblock')
|
||||
|| line.endsWith('$document')
|
||||
)
|
||||
) {
|
||||
const _domain = line
|
||||
.replace('@@||', '')
|
||||
.replace('@@|', '')
|
||||
.replace('@@.', '')
|
||||
.replace('^|', '')
|
||||
.replace('^$genericblock', '')
|
||||
.replace('$genericblock', '')
|
||||
.replace('^$document', '')
|
||||
.replace('$document', '')
|
||||
.replaceAll('^', '')
|
||||
.trim();
|
||||
|
||||
const domain = normalizeDomain(_domain);
|
||||
if (domain) {
|
||||
addToWhiteList(domain);
|
||||
} else {
|
||||
console.warn(' * [parse-filter E0001] (black) invalid domain:', _domain);
|
||||
}
|
||||
|
||||
whitelistDomainSets.add(domain);
|
||||
} else {
|
||||
console.warn(' * [parse-filter white] ' + _domain + ' is not a valid domain');
|
||||
continue;
|
||||
}
|
||||
} else if (line.startsWith('@@||')
|
||||
}
|
||||
|
||||
if (
|
||||
line.startsWith('||')
|
||||
&& (
|
||||
lineEndsWithCaret
|
||||
|| lineEndsWithCaretVerticalBar
|
||||
|| line.endsWith('^$badfilter')
|
||||
|| line.endsWith('^$1p')
|
||||
|| line.endsWith('$cname')
|
||||
)
|
||||
) {
|
||||
const _domain = line
|
||||
.replaceAll('@@||', '')
|
||||
.replaceAll('^$badfilter', '')
|
||||
.replaceAll('^$1p', '')
|
||||
.replaceAll('^|', '')
|
||||
.replace('||', '')
|
||||
.replace('^|', '')
|
||||
.replace('$cname', '')
|
||||
.replaceAll('^', '')
|
||||
.trim();
|
||||
|
||||
const domain = normalizeDomain(_domain);
|
||||
|
||||
if (domain) {
|
||||
if (DEBUG_DOMAIN_TO_FIND && domain.includes(DEBUG_DOMAIN_TO_FIND)) {
|
||||
warnOnce(filterRulesUrl.toString(), true, DEBUG_DOMAIN_TO_FIND);
|
||||
foundDebugDomain = true;
|
||||
}
|
||||
|
||||
whitelistDomainSets.add(domain);
|
||||
addToBlackList(domain, true);
|
||||
} else {
|
||||
console.warn(' * [parse-filter white] ' + _domain + ' is not a valid domain');
|
||||
console.warn(' * [parse-filter E0002] (black) invalid domain:', _domain);
|
||||
}
|
||||
} else if (
|
||||
lineStartsWithDoubleVerticalBar
|
||||
continue;
|
||||
}
|
||||
|
||||
const lineStartsWithSingleDot = line.startsWith('.');
|
||||
if (
|
||||
lineStartsWithSingleDot
|
||||
&& (
|
||||
lineEndsWithCaret
|
||||
|| lineEndsWithCaretVerticalBar
|
||||
|| line.endsWith('^$all')
|
||||
|| line.endsWith('^$doc')
|
||||
|| line.endsWith('^$document')
|
||||
)
|
||||
) {
|
||||
const _domain = line
|
||||
.replaceAll('||', '')
|
||||
.replaceAll('^|', '')
|
||||
.replaceAll('^$all', '')
|
||||
.replaceAll('^$document', '')
|
||||
.replaceAll('^$doc', '')
|
||||
.replace('^|', '')
|
||||
.replaceAll('^', '')
|
||||
.slice(1)
|
||||
.trim();
|
||||
|
||||
const domain = normalizeDomain(_domain);
|
||||
if (domain) {
|
||||
addToBlackList(domain, true);
|
||||
} else {
|
||||
console.warn(' * [parse-filter E0003] (black) invalid domain:', _domain);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
(
|
||||
line.startsWith('://')
|
||||
|| line.startsWith('http://')
|
||||
|| line.startsWith('https://')
|
||||
|| line.startsWith('|http://')
|
||||
|| line.startsWith('|https://')
|
||||
)
|
||||
&& (
|
||||
lineEndsWithCaret
|
||||
|| lineEndsWithCaretVerticalBar
|
||||
)
|
||||
) {
|
||||
const _domain = line
|
||||
.replace('|https://', '')
|
||||
.replace('https://', '')
|
||||
.replace('|http://', '')
|
||||
.replace('http://', '')
|
||||
.replace('://', '')
|
||||
.replace('^|', '')
|
||||
.replaceAll('^', '')
|
||||
.trim();
|
||||
|
||||
const domain = normalizeDomain(_domain);
|
||||
|
||||
if (domain) {
|
||||
if (DEBUG_DOMAIN_TO_FIND && domain.includes(DEBUG_DOMAIN_TO_FIND)) {
|
||||
warnOnce(filterRulesUrl.toString(), false, DEBUG_DOMAIN_TO_FIND);
|
||||
foundDebugDomain = true;
|
||||
}
|
||||
|
||||
blacklistDomainSets.add(`.${domain}`);
|
||||
addToBlackList(domain, false);
|
||||
} else {
|
||||
console.warn(' * [parse-filter E0004] (black) invalid domain:', _domain);
|
||||
}
|
||||
} else if (
|
||||
line.startsWith('://')
|
||||
&& (
|
||||
lineEndsWithCaret
|
||||
|| lineEndsWithCaretVerticalBar
|
||||
)
|
||||
) {
|
||||
const _domain = `${line.replaceAll('://', '').replaceAll('^|', '').replaceAll('^', '')}`.trim();
|
||||
continue;
|
||||
}
|
||||
if (!line.startsWith('|') && lineEndsWithCaret) {
|
||||
const _domain = line.slice(0, -1);
|
||||
const domain = normalizeDomain(_domain);
|
||||
if (domain) {
|
||||
if (DEBUG_DOMAIN_TO_FIND && domain.includes(DEBUG_DOMAIN_TO_FIND)) {
|
||||
warnOnce(filterRulesUrl.toString(), false, DEBUG_DOMAIN_TO_FIND);
|
||||
foundDebugDomain = true;
|
||||
}
|
||||
|
||||
blacklistDomainSets.add(domain);
|
||||
addToBlackList(domain, false);
|
||||
} else {
|
||||
console.warn(' * [parse-filter E0005] (black) invalid domain:', _domain);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const tryNormalizeDomain = normalizeDomain(lineStartsWithSingleDot ? line.slice(1) : line);
|
||||
if (
|
||||
tryNormalizeDomain
|
||||
&& (
|
||||
lineStartsWithSingleDot
|
||||
? tryNormalizeDomain.length === line.length - 1
|
||||
: tryNormalizeDomain === line
|
||||
)
|
||||
) {
|
||||
addToBlackList(line, true);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (
|
||||
!line.endsWith('.js')
|
||||
) {
|
||||
hasParseFailed = true;
|
||||
console.warn(' * [parse-filter E0010] can not parse:', line);
|
||||
}
|
||||
}
|
||||
|
||||
@ -243,7 +383,8 @@ async function processFilterRules (filterRulesUrl, fallbackUrls) {
|
||||
return {
|
||||
white: whitelistDomainSets,
|
||||
black: blacklistDomainSets,
|
||||
foundDebugDomain
|
||||
foundDebugDomain,
|
||||
parseFailed: hasParseFailed
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@ -15,7 +15,18 @@ const ADGUARD_FILTERS = [
|
||||
'https://easylist-downloads.adblockplus.org/easylist.txt',
|
||||
'https://raw.githubusercontent.com/easylist/easylist/gh-pages/easylist.txt',
|
||||
'https://secure.fanboy.co.nz/easylist.txt'
|
||||
]
|
||||
],
|
||||
true
|
||||
],
|
||||
// Easy Privacy
|
||||
[
|
||||
'https://easylist.to/easylist/easyprivacy.txt',
|
||||
[
|
||||
'https://secure.fanboy.co.nz/easyprivacy.txt',
|
||||
'https://raw.githubusercontent.com/easylist/easylist/gh-pages/easyprivacy.txt',
|
||||
'https://easylist-downloads.adblockplus.org/easyprivacy.txt'
|
||||
],
|
||||
true
|
||||
],
|
||||
// AdGuard DNS Filter
|
||||
'https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt',
|
||||
@ -27,28 +38,32 @@ const ADGUARD_FILTERS = [
|
||||
[
|
||||
'https://ublockorigin.github.io/uAssetsCDN/filters/filters.txt',
|
||||
'https://ublockorigin.pages.dev/filters/filters.txt'
|
||||
]
|
||||
],
|
||||
false
|
||||
],
|
||||
[
|
||||
'https://ublockorigin.github.io/uAssets/filters/filters-2020.txt',
|
||||
[
|
||||
'https://ublockorigin.github.io/uAssetsCDN/filters/filters-2020.txt',
|
||||
'https://ublockorigin.pages.dev/filters/filters-2020.txt'
|
||||
]
|
||||
],
|
||||
false
|
||||
],
|
||||
[
|
||||
'https://ublockorigin.github.io/uAssets/filters/filters-2021.txt',
|
||||
[
|
||||
'https://ublockorigin.github.io/uAssetsCDN/filters/filters-2021.txt',
|
||||
'https://ublockorigin.pages.dev/filters/filters-2021.txt'
|
||||
]
|
||||
],
|
||||
false
|
||||
],
|
||||
[
|
||||
'https://ublockorigin.github.io/uAssets/filters/filters-2022.txt',
|
||||
[
|
||||
'https://ublockorigin.github.io/uAssetsCDN/filters/filters-2022.txt',
|
||||
'https://ublockorigin.pages.dev/filters/filters-2022.txt'
|
||||
]
|
||||
],
|
||||
false
|
||||
],
|
||||
// uBlock Origin Badware Risk List
|
||||
[
|
||||
@ -56,7 +71,8 @@ const ADGUARD_FILTERS = [
|
||||
[
|
||||
'https://ublockorigin.github.io/uAssetsCDN/filters/badware.txt',
|
||||
'https://ublockorigin.pages.dev/filters/badware.txt'
|
||||
]
|
||||
],
|
||||
false
|
||||
],
|
||||
// uBlock Origin Privacy List
|
||||
[
|
||||
@ -64,7 +80,8 @@ const ADGUARD_FILTERS = [
|
||||
[
|
||||
'https://ublockorigin.github.io/uAssetsCDN/filters/privacy.txt',
|
||||
'https://ublockorigin.pages.dev/filters/privacy.txt'
|
||||
]
|
||||
],
|
||||
false
|
||||
],
|
||||
// uBlock Origin Resource Abuse
|
||||
[
|
||||
@ -72,7 +89,8 @@ const ADGUARD_FILTERS = [
|
||||
[
|
||||
'https://ublockorigin.github.io/uAssetsCDN/filters/resource-abuse.txt',
|
||||
'https://ublockorigin.pages.dev/filters/resource-abuse.txt'
|
||||
]
|
||||
],
|
||||
false
|
||||
],
|
||||
// uBlock Origin Unbreak
|
||||
[
|
||||
@ -80,7 +98,8 @@ const ADGUARD_FILTERS = [
|
||||
[
|
||||
'https://ublockorigin.github.io/uAssetsCDN/filters/unbreak.txt',
|
||||
'https://ublockorigin.pages.dev/filters/unbreak.txt'
|
||||
]
|
||||
],
|
||||
false
|
||||
],
|
||||
// AdGuard Base Filter
|
||||
'https://filters.adtidy.org/extension/ublock/filters/2_without_easylist.txt',
|
||||
@ -99,16 +118,8 @@ const ADGUARD_FILTERS = [
|
||||
'https://easylist.to/easylistgermany/easylistgermany.txt',
|
||||
[
|
||||
'https://easylist-downloads.adblockplus.org/easylistgermany.txt'
|
||||
]
|
||||
],
|
||||
// Easy Privacy
|
||||
[
|
||||
'https://easylist.to/easylist/easyprivacy.txt',
|
||||
[
|
||||
'https://secure.fanboy.co.nz/easyprivacy.txt',
|
||||
'https://raw.githubusercontent.com/easylist/easylist/gh-pages/easyprivacy.txt',
|
||||
'https://easylist-downloads.adblockplus.org/easyprivacy.txt'
|
||||
]
|
||||
],
|
||||
false
|
||||
],
|
||||
// Curben's UrlHaus Malicious URL Blocklist
|
||||
[
|
||||
@ -117,7 +128,8 @@ const ADGUARD_FILTERS = [
|
||||
'https://urlhaus-filter.pages.dev/urlhaus-filter-agh-online.txt',
|
||||
// Prefer mirror, since malware-filter.gitlab.io has not been updated for a while
|
||||
// 'https://malware-filter.gitlab.io/urlhaus-filter/urlhaus-filter-agh-online.txt'
|
||||
]
|
||||
],
|
||||
false
|
||||
],
|
||||
// Curben's Phishing URL Blocklist
|
||||
[
|
||||
@ -126,7 +138,8 @@ const ADGUARD_FILTERS = [
|
||||
'https://phishing-filter.pages.dev/phishing-filter-agh.txt',
|
||||
// Prefer mirror, since malware-filter.gitlab.io has not been updated for a while
|
||||
// 'https://malware-filter.gitlab.io/malware-filter/phishing-filter-agh.txt'
|
||||
]
|
||||
],
|
||||
false
|
||||
],
|
||||
// Curben's PUP Domains Blocklist
|
||||
[
|
||||
@ -135,7 +148,8 @@ const ADGUARD_FILTERS = [
|
||||
'https://pup-filter.pages.dev/pup-filter-agh.txt',
|
||||
// Prefer mirror, since malware-filter.gitlab.io has not been updated for a while
|
||||
// 'https://malware-filter.gitlab.io/malware-filter/pup-filter-agh.txt'
|
||||
]
|
||||
],
|
||||
false
|
||||
],
|
||||
// GameConsoleAdblockList
|
||||
'https://raw.githubusercontent.com/DandelionSprout/adfilt/master/GameConsoleAdblockList.txt',
|
||||
@ -147,7 +161,8 @@ const ADGUARD_FILTERS = [
|
||||
'https://paulgb.github.io/BarbBlock/blacklists/ublock-origin.txt',
|
||||
// Brave First Party & First Party CNAME
|
||||
'https://raw.githubusercontent.com/brave/adblock-lists/master/brave-lists/brave-firstparty.txt',
|
||||
'https://raw.githubusercontent.com/brave/adblock-lists/master/brave-lists/brave-firstparty-cname.txt'
|
||||
'https://raw.githubusercontent.com/brave/adblock-lists/master/brave-lists/brave-firstparty-cname.txt',
|
||||
'https://raw.githubusercontent.com/brave/adblock-lists/master/coin-miners.txt'
|
||||
];
|
||||
|
||||
const PREDEFINED_WHITELIST = [
|
||||
|
||||
@ -1,8 +1,11 @@
|
||||
const Piscina = require('piscina');
|
||||
const { isCI } = require('ci-info');
|
||||
|
||||
const fullsetDomainStartsWithADot = Piscina.workerData
|
||||
const totalLen = fullsetDomainStartsWithADot.length;
|
||||
|
||||
const log = isCI ? () => { } : console.log.bind(console);
|
||||
|
||||
module.exports.dedupe = ({ chunk }) => {
|
||||
const chunkLength = chunk.length;
|
||||
const outputToBeRemoved = new Int8Array(chunkLength);
|
||||
@ -34,6 +37,7 @@ module.exports.dedupe = ({ chunk }) => {
|
||||
|
||||
if (shouldBeRemoved) {
|
||||
outputToBeRemoved[i] = 1;
|
||||
log(domainFromInput, domainFromFullSet)
|
||||
break;
|
||||
}
|
||||
}
|
||||
@ -42,6 +46,7 @@ module.exports.dedupe = ({ chunk }) => {
|
||||
// domainFromInput is now startsWith a "."
|
||||
if (domainFromInput.endsWith(domainFromFullSet)) {
|
||||
outputToBeRemoved[i] = 1;
|
||||
log(domainFromInput, domainFromFullSet)
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@ -50,6 +50,10 @@ AND,((DOMAIN-KEYWORD,f-log), (DOMAIN-SUFFIX,grammarly.io))
|
||||
DOMAIN,stun.smartgslb.com
|
||||
AND,((DOMAIN-SUFFIX,prod.hosts.ooklaserver.net), (DOMAIN-KEYWORD,.ad.))
|
||||
|
||||
DOMAIN-KEYWORD,_vmind.qqvideo.tc.qq.com
|
||||
DOMAIN-KEYWORD,-logging.nextmedia.com
|
||||
DOMAIN-KEYWORD,-spiky.clevertap-prod.com
|
||||
|
||||
# Important: Force add the following domains without whitelisting
|
||||
DOMAIN-SUFFIX,openx.net
|
||||
# DOMAIN-SUFFIX,mmstat.com
|
||||
|
||||
@ -15,6 +15,8 @@ optimus-ads.amap.com
|
||||
.rjno1.com
|
||||
.taio.app
|
||||
.headcaptcha.live
|
||||
# fake pirate bay
|
||||
.thepiratebay3.com
|
||||
|
||||
# >> Qihoo 360
|
||||
hot.m.shouji.360tpcdn.com
|
||||
@ -565,6 +567,10 @@ adcloud.jp
|
||||
events.jianshu.io
|
||||
.mobileadtrading.com
|
||||
|
||||
t.yesware.com
|
||||
.charlestownwyllie.oaklawnnonantum.com
|
||||
.auwrtou.cn
|
||||
|
||||
# >> Tracking
|
||||
tracking.yorg.app
|
||||
.taginstall.com
|
||||
@ -618,6 +624,7 @@ analytics.slashdotmedia.com
|
||||
.tagtic.cn
|
||||
sa.holopin.io
|
||||
analytics.infomaniak.com
|
||||
.prtrackings.com
|
||||
|
||||
.adjust.io
|
||||
.airbrake.io
|
||||
@ -957,6 +964,7 @@ videopush.baidu.com
|
||||
.toolbar.baidu.com
|
||||
.tracker.baidu.com
|
||||
.tuijian.baidu.com
|
||||
.ubmcmm.baidustatic.com
|
||||
|
||||
# >> Qihoo 360
|
||||
.stat.360safe.com
|
||||
|
||||
@ -102,6 +102,7 @@
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"@cliqz/adblocker": "^1.25.1",
|
||||
"@sukka/listdir": "^0.2.0",
|
||||
"@vercel/fetch-retry": "^5.1.3",
|
||||
"ci-info": "^3.7.0",
|
||||
|
||||
89
pnpm-lock.yaml
generated
89
pnpm-lock.yaml
generated
@ -1,6 +1,7 @@
|
||||
lockfileVersion: 5.4
|
||||
|
||||
specifiers:
|
||||
'@cliqz/adblocker': ^1.25.1
|
||||
'@sukka/listdir': ^0.2.0
|
||||
'@vercel/fetch-retry': ^5.1.3
|
||||
ci-info: ^3.7.0
|
||||
@ -15,6 +16,7 @@ specifiers:
|
||||
wireit: ^0.9.1
|
||||
|
||||
dependencies:
|
||||
'@cliqz/adblocker': 1.25.1
|
||||
'@sukka/listdir': 0.2.0
|
||||
'@vercel/fetch-retry': 5.1.3
|
||||
ci-info: 3.7.0
|
||||
@ -36,6 +38,29 @@ packages:
|
||||
resolution: {integrity: sha512-H71nDOOL8Y7kWRLqf6Sums+01Q5msqBW2KhDUTemh1tvY04eSkSXrK0uj/4mmY0Xr16/3zyZmsrxN7CKuRbNRg==}
|
||||
dev: false
|
||||
|
||||
/@cliqz/adblocker-content/1.25.1:
|
||||
resolution: {integrity: sha512-7gl2VdNPBfj7aPoq34B5miwGcnda/7LCr+BqnpcSOjdLV6jjT2FrNSAKGFvcH23q0HM1IFhYDV6ydTgsdWFCnA==}
|
||||
dependencies:
|
||||
'@cliqz/adblocker-extended-selectors': 1.25.1
|
||||
dev: false
|
||||
|
||||
/@cliqz/adblocker-extended-selectors/1.25.1:
|
||||
resolution: {integrity: sha512-4MdMe/YfIok5d8WYVcLR3Ak7vGrmeUV47frgmXEe945luY93vwlzk1NiLYW1JM5Gdm+VePweoS9cJ1/QUTmv+Q==}
|
||||
dev: false
|
||||
|
||||
/@cliqz/adblocker/1.25.1:
|
||||
resolution: {integrity: sha512-1C1/ELI94/XewdUj/o1+Q4ziOigMvTZQA05UERfDoKqpJ+0cbrEF/UImrzpX7n+kYsR7xTJvmf+iNM3zS0tfsg==}
|
||||
dependencies:
|
||||
'@cliqz/adblocker-content': 1.25.1
|
||||
'@cliqz/adblocker-extended-selectors': 1.25.1
|
||||
'@remusao/guess-url-type': 1.2.1
|
||||
'@remusao/small': 1.2.1
|
||||
'@remusao/smaz': 1.9.1
|
||||
'@types/chrome': 0.0.197
|
||||
'@types/firefox-webext-browser': 94.0.1
|
||||
tldts-experimental: 5.7.103
|
||||
dev: false
|
||||
|
||||
/@nodelib/fs.scandir/2.1.5:
|
||||
resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==}
|
||||
engines: {node: '>= 8'}
|
||||
@ -57,10 +82,64 @@ packages:
|
||||
fastq: 1.13.0
|
||||
dev: true
|
||||
|
||||
/@remusao/guess-url-type/1.2.1:
|
||||
resolution: {integrity: sha512-rbOqre2jW8STjheOsOaQHLgYBaBZ9Owbdt8NO7WvNZftJlaG3y/K9oOkl8ZUpuFBisIhmBuMEW6c+YrQl5inRA==}
|
||||
dev: false
|
||||
|
||||
/@remusao/small/1.2.1:
|
||||
resolution: {integrity: sha512-7MjoGt0TJMVw1GPKgWq6SJPws1SLsUXQRa43Umht+nkyw2jnpy3WpiLNqGdwo5rHr5Wp9B2W/Pm5RQp656UJdw==}
|
||||
dev: false
|
||||
|
||||
/@remusao/smaz-compress/1.9.1:
|
||||
resolution: {integrity: sha512-E2f48TwloQu3r6BdLOGF2aczeH7bJ/32oJGqvzT9SKur0cuUnLcZ7ZXP874E2fwmdE+cXzfC7bKzp79cDnmeyw==}
|
||||
dependencies:
|
||||
'@remusao/trie': 1.4.1
|
||||
dev: false
|
||||
|
||||
/@remusao/smaz-decompress/1.9.1:
|
||||
resolution: {integrity: sha512-TfjKKprYe3n47od8auhvJ/Ikj9kQTbDTe71ynKlxslrvvUhlIV3VQSuwYuMWMbdz1fIs0H/fxCN1Z8/H3km6/A==}
|
||||
dev: false
|
||||
|
||||
/@remusao/smaz/1.9.1:
|
||||
resolution: {integrity: sha512-e6BLuP8oaXCZ9+v46Is4ilAZ/Vq6YLgmBP204Ixgk1qTjXmqvFYG7+AS7v9nsZdGOy96r9DWGFbbDVgMxwu1rA==}
|
||||
dependencies:
|
||||
'@remusao/smaz-compress': 1.9.1
|
||||
'@remusao/smaz-decompress': 1.9.1
|
||||
dev: false
|
||||
|
||||
/@remusao/trie/1.4.1:
|
||||
resolution: {integrity: sha512-yvwa+aCyYI/UjeD39BnpMypG8N06l86wIDW1/PAc6ihBRnodIfZDwccxQN3n1t74wduzaz74m4ZMHZnB06567Q==}
|
||||
dev: false
|
||||
|
||||
/@sukka/listdir/0.2.0:
|
||||
resolution: {integrity: sha512-UyVirNhAOXKwjiDehjUaGtpfk0QwNHyiXrlLb/FmWMtI+BGhaEvB9MypSfEAtiiMI3g6QTfG38ayNAorEuz5ow==}
|
||||
dev: false
|
||||
|
||||
/@types/chrome/0.0.197:
|
||||
resolution: {integrity: sha512-m1NfS5bOjaypyqQfaX6CxmJodZVcvj5+Mt/K94EBHkflYjPNmXHAzbxfifdLMa0YM3PDyOxohoTS5ug/e6p5jA==}
|
||||
dependencies:
|
||||
'@types/filesystem': 0.0.32
|
||||
'@types/har-format': 1.2.10
|
||||
dev: false
|
||||
|
||||
/@types/filesystem/0.0.32:
|
||||
resolution: {integrity: sha512-Yuf4jR5YYMR2DVgwuCiP11s0xuVRyPKmz8vo6HBY3CGdeMj8af93CFZX+T82+VD1+UqHOxTq31lO7MI7lepBtQ==}
|
||||
dependencies:
|
||||
'@types/filewriter': 0.0.29
|
||||
dev: false
|
||||
|
||||
/@types/filewriter/0.0.29:
|
||||
resolution: {integrity: sha512-BsPXH/irW0ht0Ji6iw/jJaK8Lj3FJemon2gvEqHKpCdDCeemHa+rI3WBGq5z7cDMZgoLjY40oninGxqk+8NzNQ==}
|
||||
dev: false
|
||||
|
||||
/@types/firefox-webext-browser/94.0.1:
|
||||
resolution: {integrity: sha512-I6iHRQJSTZ+gYt2IxdH2RRAMvcUyK8v5Ig7fHQR0IwUNYP7hz9+cziBVIKxLCO6XI7fiyRsNOWObfl3/4Js2Lg==}
|
||||
dev: false
|
||||
|
||||
/@types/har-format/1.2.10:
|
||||
resolution: {integrity: sha512-o0J30wqycjF5miWDKYKKzzOU1ZTLuA42HZ4HE7/zqTOc/jTLdQ5NhYWvsRQo45Nfi1KHoRdNhteSI4BAxTF1Pg==}
|
||||
dev: false
|
||||
|
||||
/@vercel/fetch-retry/5.1.3:
|
||||
resolution: {integrity: sha512-UIbFc4VsEZHOr6dWuE+kxY4NxnOLXFMCWm0fSKRRHUEtrIzaJLzHpWk2QskCXTSzFgFvhkLAvSrBK2XZg7NSzg==}
|
||||
peerDependencies:
|
||||
@ -552,6 +631,16 @@ packages:
|
||||
resolution: {integrity: sha512-EFhrqqb0abo/9KZ4ZxHSF4e8CYKk/SouOLbNUMZUw+5OjC0hUJbFmHXtHNWYGaqnmwjnvdwFQ1a29Tflpz5Ufg==}
|
||||
dev: false
|
||||
|
||||
/tldts-core/5.7.103:
|
||||
resolution: {integrity: sha512-MdSolgnhJwr2SH6a+5KFAYB8znpYdRLoOFTJmrWslsec9Ne/V3DBrw+BbS1YYQJKGTin6U02Y9CSYxnOpg3vwg==}
|
||||
dev: false
|
||||
|
||||
/tldts-experimental/5.7.103:
|
||||
resolution: {integrity: sha512-oo9QI0TjXdrlZnDSJMazRJe+nnd0OwXgzRmHcsnyp4k6UxmmlaWEA1iq3RY3EDSKwYEJ+lDnwQeAaGRleU/LEQ==}
|
||||
dependencies:
|
||||
tldts-core: 5.7.103
|
||||
dev: false
|
||||
|
||||
/tldts/5.7.102:
|
||||
resolution: {integrity: sha512-FvqzkTsADAntBc4EPmcmIs4GkjBBtDKZKHJJU6i1TzL9d+ijBearWEgGNoE8YO9T8pEe+szYE+KFMmh6ixexYA==}
|
||||
hasBin: true
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user