diff --git a/Build/build-reject-domainset.ts b/Build/build-reject-domainset.ts index 55603c33..ec7c7705 100644 --- a/Build/build-reject-domainset.ts +++ b/Build/build-reject-domainset.ts @@ -85,21 +85,29 @@ export const buildRejectDomainSet = task(require.main === module, __filename)(as domainListsExtraDownloads.map(task => task(childSpan).then(appendArrayToRejectExtraOutput)), adguardFiltersDownloads.map( - task => task(childSpan).then(({ white, black }) => { - addArrayElementsToSet(filterRuleWhitelistDomainSets, white); - appendArrayToRejectOutput(black); + task => task(childSpan).then(({ whiteDomains, whiteDomainSuffixes, blackDomains, blackDomainSuffixes }) => { + addArrayElementsToSet(filterRuleWhitelistDomainSets, whiteDomains); + addArrayElementsToSet(filterRuleWhitelistDomainSets, whiteDomainSuffixes, suffix => '.' + suffix); + + rejectOutput.bulkAddDomain(blackDomains); + rejectOutput.bulkAddDomainSuffix(blackDomainSuffixes); }) ), adguardFiltersExtraDownloads.map( - task => task(childSpan).then(({ white, black }) => { - addArrayElementsToSet(filterRuleWhitelistDomainSets, white); - appendArrayToRejectExtraOutput(black); + task => task(childSpan).then(({ whiteDomains, whiteDomainSuffixes, blackDomains, blackDomainSuffixes }) => { + addArrayElementsToSet(filterRuleWhitelistDomainSets, whiteDomains); + addArrayElementsToSet(filterRuleWhitelistDomainSets, whiteDomainSuffixes, suffix => '.' + suffix); + + rejectOutput.bulkAddDomain(blackDomains); + rejectOutput.bulkAddDomainSuffix(blackDomainSuffixes); }) ), adguardFiltersWhitelistsDownloads.map( - task => task(childSpan).then(({ white, black }) => { - addArrayElementsToSet(filterRuleWhitelistDomainSets, white); - addArrayElementsToSet(filterRuleWhitelistDomainSets, black); + task => task(childSpan).then(({ whiteDomains, whiteDomainSuffixes, blackDomains, blackDomainSuffixes }) => { + addArrayElementsToSet(filterRuleWhitelistDomainSets, whiteDomains); + addArrayElementsToSet(filterRuleWhitelistDomainSets, whiteDomainSuffixes, suffix => '.' + suffix); + addArrayElementsToSet(filterRuleWhitelistDomainSets, blackDomains); + addArrayElementsToSet(filterRuleWhitelistDomainSets, blackDomainSuffixes, suffix => '.' + suffix); }) ), getPhishingDomains(childSpan).then(appendArrayToRejectExtraOutput), diff --git a/Build/lib/parse-filter/filters.ts b/Build/lib/parse-filter/filters.ts index 51ac3e3a..dfebf7a9 100644 --- a/Build/lib/parse-filter/filters.ts +++ b/Build/lib/parse-filter/filters.ts @@ -27,11 +27,14 @@ export function processFilterRulesWithPreload( ) { const downloadPromise = fetchAssets(filterRulesUrl, fallbackUrls); - return (span: Span) => span.traceChildAsync<{ white: string[], black: string[] }>(`process filter rules: ${filterRulesUrl}`, async (span) => { + return (span: Span) => span.traceChildAsync>(`process filter rules: ${filterRulesUrl}`, async (span) => { const text = await span.traceChildPromise('download', downloadPromise); - const whitelistDomainSets = new Set(); - const blacklistDomainSets = new Set(); + const whiteDomains = new Set(); + const whiteDomainSuffixes = new Set(); + + const blackDomains = new Set(); + const blackDomainSuffixes = new Set(); const warningMessages: string[] = []; @@ -60,24 +63,16 @@ export function processFilterRulesWithPreload( switch (flag) { case ParseType.WhiteIncludeSubdomain: - if (hostname[0] === '.') { - whitelistDomainSets.add(hostname); - } else { - whitelistDomainSets.add(`.${hostname}`); - } + whiteDomainSuffixes.add(hostname); break; case ParseType.WhiteAbsolute: - whitelistDomainSets.add(hostname); + whiteDomains.add(hostname); break; case ParseType.BlackIncludeSubdomain: - if (hostname[0] === '.') { - blacklistDomainSets.add(hostname); - } else { - blacklistDomainSets.add(`.${hostname}`); - } + blackDomainSuffixes.add(hostname); break; case ParseType.BlackAbsolute: - blacklistDomainSets.add(hostname); + blackDomains.add(hostname); break; case ParseType.ErrorMessage: warningMessages.push(hostname); @@ -105,118 +100,19 @@ export function processFilterRulesWithPreload( console.log( picocolors.gray('[process filter]'), picocolors.gray(filterRulesUrl), - picocolors.gray(`white: ${whitelistDomainSets.size}`), - picocolors.gray(`black: ${blacklistDomainSets.size}`) + picocolors.gray(`white: ${whiteDomains.size + whiteDomainSuffixes.size}`), + picocolors.gray(`black: ${blackDomains.size + blackDomainSuffixes.size}`) ); return { - white: Array.from(whitelistDomainSets), - black: Array.from(blacklistDomainSets) + whiteDomains: Array.from(whiteDomains), + whiteDomainSuffixes: Array.from(whiteDomainSuffixes), + blackDomains: Array.from(blackDomains), + blackDomainSuffixes: Array.from(blackDomainSuffixes) }; }); } -export async function processFilterRules( - parentSpan: Span, - filterRulesUrl: string, - fallbackUrls?: string[] | null, - includeThirdParty = false -): Promise<{ white: string[], black: string[] }> { - const [white, black, warningMessages] = await parentSpan.traceChild(`process filter rules: ${filterRulesUrl}`).traceAsyncFn(async (span) => { - const text = await span.traceChildAsync('download', () => fetchAssets(filterRulesUrl, fallbackUrls)); - - const whitelistDomainSets = new Set(); - const blacklistDomainSets = new Set(); - - const warningMessages: string[] = []; - - const MUTABLE_PARSE_LINE_RESULT: [string, ParseType] = ['', ParseType.NotParsed]; - /** - * @param {string} line - */ - const lineCb = (line: string) => { - const result = parse(line, MUTABLE_PARSE_LINE_RESULT, includeThirdParty); - const flag = result[1]; - - if (flag === ParseType.NotParsed) { - throw new Error(`Didn't parse line: ${line}`); - } - if (flag === ParseType.Null) { - return; - } - - const hostname = result[0]; - - if (flag === ParseType.WhiteIncludeSubdomain || flag === ParseType.WhiteAbsolute) { - onWhiteFound(hostname, filterRulesUrl); - } else { - onBlackFound(hostname, filterRulesUrl); - } - - switch (flag) { - case ParseType.WhiteIncludeSubdomain: - if (hostname[0] === '.') { - whitelistDomainSets.add(hostname); - } else { - whitelistDomainSets.add(`.${hostname}`); - } - break; - case ParseType.WhiteAbsolute: - whitelistDomainSets.add(hostname); - break; - case ParseType.BlackIncludeSubdomain: - if (hostname[0] === '.') { - blacklistDomainSets.add(hostname); - } else { - blacklistDomainSets.add(`.${hostname}`); - } - break; - case ParseType.BlackAbsolute: - blacklistDomainSets.add(hostname); - break; - case ParseType.ErrorMessage: - warningMessages.push(hostname); - break; - default: - break; - } - }; - - const filterRules = text.split('\n'); - - span.traceChild('parse adguard filter').traceSyncFn(() => { - for (let i = 0, len = filterRules.length; i < len; i++) { - lineCb(filterRules[i]); - } - }); - - return [ - Array.from(whitelistDomainSets), - Array.from(blacklistDomainSets), - warningMessages - ] as const; - }); - - for (let i = 0, len = warningMessages.length; i < len; i++) { - console.warn( - picocolors.yellow(warningMessages[i]), - picocolors.gray(picocolors.underline(filterRulesUrl)) - ); - } - - console.log( - picocolors.gray('[process filter]'), - picocolors.gray(filterRulesUrl), - picocolors.gray(`white: ${white.length}`), - picocolors.gray(`black: ${black.length}`) - ); - - return { - white, - black - }; -} - // const R_KNOWN_NOT_NETWORK_FILTER_PATTERN_2 = /(\$popup|\$removeparam|\$popunder|\$cname)/; // cname exceptional filter can not be parsed by NetworkFilter // Surge / Clash can't handle CNAME either, so we just ignore them diff --git a/package.json b/package.json index 881162e2..a7e6e11e 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "fast-cidr-tools": "^0.3.1", "fast-fifo": "^1.3.2", "fdir": "^6.4.3", - "foxts": "^1.1.6", + "foxts": "^1.1.7", "hash-wasm": "^4.12.0", "json-stringify-pretty-compact": "^3.0.0", "picocolors": "^1.1.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f9a5b19c..ccbf4360 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -53,8 +53,8 @@ importers: specifier: ^6.4.3 version: 6.4.3(picomatch@4.0.2) foxts: - specifier: ^1.1.6 - version: 1.1.6 + specifier: ^1.1.7 + version: 1.1.7 hash-wasm: specifier: ^4.12.0 version: 4.12.0 @@ -1136,8 +1136,8 @@ packages: resolution: {integrity: sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==} engines: {node: '>= 6'} - foxts@1.1.6: - resolution: {integrity: sha512-O2UR/MDLo0w4igcFHwLn2KyXUD84P6bE3U4OpVsxvcYrWLFvvDO8zKLBS/o++tFJTCq7p/3USR48E8/dF2vAAQ==} + foxts@1.1.7: + resolution: {integrity: sha512-Pw7S1yI20GY8gfj6RXt9usRE5TdQ/lgAqpy2EaWKUVNARC+jW0hxx/MQH8xkNlT3NSpt0X1P99CJTEvh3kVdUQ==} fs-constants@1.0.0: resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==} @@ -2934,7 +2934,7 @@ snapshots: combined-stream: 1.0.8 mime-types: 2.1.35 - foxts@1.1.6: {} + foxts@1.1.7: {} fs-constants@1.0.0: {}