Feat: support wildcard from adguard filter

Build/lib/parse-filter/filters.ts

@@ -15,6 +15,9 @@ const enum ParseType {
   BlackIncludeSubdomain = 2,
   ErrorMessage = 10,
   BlackIP = 20,
+  BlackWildcard = 30,
+  BlackKeyword = 40,
+  WhiteKeyword = 50,
   Null = 1000,
   NotParsed = 2000
 }
@@ -28,7 +31,19 @@ export function processFilterRulesWithPreload(
 ) {
   const downloadPromise = fetchAssets(filterRulesUrl, fallbackUrls);
 
-  return (span: Span) => span.traceChildAsync<Record<'whiteDomains' | 'whiteDomainSuffixes' | 'blackDomains' | 'blackDomainSuffixes' | 'blackIPs', string[]>>(`process filter rules: ${filterRulesUrl}`, async (span) => {
+  return (span: Span) => span.traceChildAsync<
+    Record<
+      'whiteDomains'
+      | 'whiteDomainSuffixes'
+      | 'blackDomains'
+      | 'blackDomainSuffixes'
+      | 'blackIPs'
+      | 'blackWildcard'
+      | 'whiteKeyword'
+      | 'blackKeyword',
+      string[]
+    >
+  >(`process filter rules: ${filterRulesUrl}`, async (span) => {
     const filterRules = await span.traceChildPromise('download', downloadPromise);
 
     const whiteDomains = new Set<string>();
@@ -40,6 +55,10 @@ export function processFilterRulesWithPreload(
     const warningMessages: string[] = [];
 
     const blackIPs: string[] = [];
+    const blackWildcard = new Set<string>();
+
+    const whiteKeyword = new Set<string>();
+    const blackKeyword = new Set<string>();
 
     const MUTABLE_PARSE_LINE_RESULT: [string, ParseType] = ['', ParseType.NotParsed];
     /**
@@ -83,6 +102,15 @@ export function processFilterRulesWithPreload(
         case ParseType.BlackIP:
           blackIPs.push(hostname);
           break;
+        case ParseType.BlackWildcard:
+          blackWildcard.add(hostname);
+          break;
+        case ParseType.BlackKeyword:
+          blackKeyword.add(hostname);
+          break;
+        case ParseType.WhiteKeyword:
+          whiteKeyword.add(hostname);
+          break;
         default:
           break;
       }
@@ -113,7 +141,10 @@ export function processFilterRulesWithPreload(
       whiteDomainSuffixes: Array.from(whiteDomainSuffixes),
       blackDomains: Array.from(blackDomains),
       blackDomainSuffixes: Array.from(blackDomainSuffixes),
-      blackIPs
+      blackIPs,
+      blackWildcard: Array.from(blackWildcard),
+      whiteKeyword: Array.from(whiteKeyword),
+      blackKeyword: Array.from(blackKeyword)
     };
   });
 }
@@ -482,6 +513,7 @@ export function parse($line: string, result: [string, ParseType], includeThirdPa
   }
 
   const parsed = tldts.parse(sliced, looseTldtsOpt);
+  const hostname = parsed.hostname;
 
   /**
    * We can exclude wildcard in TLD
@@ -495,15 +527,21 @@ export function parse($line: string, result: [string, ParseType], includeThirdPa
    *
    * This also exclude non standard TLD like `.tor`, `.onion`, `.dn42`, etc.
    */
-  if (!parsed.publicSuffix || !parsed.isIcann || !parsed.hostname || !parsed.domain) {
+  if (!parsed.publicSuffix || !parsed.isIcann || !hostname || !parsed.domain) {
     result[1] = ParseType.Null;
     return result;
   }
 
   // no wildcard, we can safely normalize it˝
-  if (!parsed.hostname.includes('*')) {
+  if (!hostname.includes('*')) {
+    if (hostname.charCodeAt(0) === 45) { // 45 `-`
+      result[0] = hostname;
+      result[1] = white ? ParseType.WhiteKeyword : ParseType.BlackKeyword;
+      return result;
+    }
+
     if (white) {
-      result[0] = parsed.hostname;
+      result[0] = hostname;
       result[1] = includeAllSubDomain ? ParseType.WhiteIncludeSubdomain : ParseType.WhiteAbsolute;
       return result;
     }
@@ -522,14 +560,46 @@ export function parse($line: string, result: [string, ParseType], includeThirdPa
       }
     }
 
-    result[0] = parsed.hostname;
+    result[0] = hostname;
     result[1] = includeAllSubDomain ? ParseType.BlackIncludeSubdomain : ParseType.BlackAbsolute;
     return result;
   }
 
-  result[0] = `[parse-filter E0010] (${white ? 'white' : 'black'}) invalid domain: ${JSON.stringify({
-    line, sliced, sliceStart, sliceEnd, parsed
-  })}`;
-  result[1] = ParseType.ErrorMessage;
+  // now we only have wildcard domain left
+  if (white) {
+    // we don't support wildcard in whitelist
+    // result[1] = ParseType.Null;
+    // return result;
+    result[0] = `[parse-filter E0021] wildcard whitelist not supported: ${JSON.stringify({
+      line, sliced, sliceStart, sliceEnd, parsed
+    })}`;
+    result[1] = ParseType.ErrorMessage;
+    return result;
+  }
+
+  for (let i = 0, len = hostname.length; i < len; i++) {
+    const char = hostname.charCodeAt(i);
+
+    if (
+      (char >= 97 && char <= 122) // 97-122 `a-z`
+      || char === 46 // 46 `.`
+      || char === 45 // 45 `-`
+      || (char >= 48 && char <= 57) // 48-57 `0-9`
+      || char === 42 // 42 `*`
+      || char === 95 // 95 `_`
+      // || (char >= 65 && char <= 90) // 65-90 `A-Z`
+    ) {
+      continue;
+    }
+
+    result[0] = `[parse-filter E0020] (black) invalid wildcard domain: ${JSON.stringify({
+      line, sliced, sliceStart, sliceEnd, parsed
+    })}`;
+    result[1] = ParseType.ErrorMessage;
+    return result;
+  }
+
+  result[0] = hostname;
+  result[1] = ParseType.BlackWildcard;
   return result;
 }

Build/lib/rules/base.ts

@@ -17,8 +17,12 @@ export class FileOutput {
   protected strategies: BaseWriteStrategy[] = [];
 
   public domainTrie = new HostnameSmolTrie(null);
+  public wildcardTrie: HostnameSmolTrie = new HostnameSmolTrie(null);
+
   protected domainKeywords = new Set<string>();
-  protected domainWildcard = new Set<string>();
+
+  private whitelistKeywords = new Set<string>();
+
   protected userAgent = new Set<string>();
   protected processName = new Set<string>();
   protected processPath = new Set<string>();
@@ -43,6 +47,12 @@ export class FileOutput {
 
   whitelistDomain = (domain: string) => {
     this.domainTrie.whitelist(domain);
+    this.wildcardTrie.whitelist(domain);
+    return this;
+  };
+
+  whitelistKeyword = (keyword: string) => {
+    this.whitelistKeywords.add(keyword);
     return this;
   };
 
@@ -112,6 +122,20 @@ export class FileOutput {
     return this;
   }
 
+  bulkAddDomainKeyword(keywords: string[]) {
+    for (let i = 0, len = keywords.length; i < len; i++) {
+      this.domainKeywords.add(keywords[i]);
+    }
+    return this;
+  }
+
+  bulkAddDomainWildcard(domains: string[]) {
+    for (let i = 0, len = domains.length; i < len; i++) {
+      this.wildcardTrie.add(domains[i]);
+    }
+    return this;
+  }
+
   addIPASN(asn: string) {
     this.ipasn.add(asn);
     return this;
@@ -161,7 +185,7 @@ export class FileOutput {
           this.addDomainKeyword(value);
           break;
         case 'DOMAIN-WILDCARD':
-          this.domainWildcard.add(value);
+          this.wildcardTrie.add(value);
           break;
         case 'USER-AGENT':
           this.userAgent.add(value);
@@ -318,7 +342,11 @@ export class FileOutput {
 
     this.strategiesWritten = true;
 
-    const kwfilter = createKeywordFilter(Array.from(this.domainKeywords));
+    // We use both DOMAIN-KEYWORD and whitelisted keyword to whitelist DOMAIN and DOMAIN-SUFFIX
+    const kwfilter = createKeywordFilter(
+      Array.from(this.domainKeywords)
+        .concat(Array.from(this.whitelistKeywords))
+    );
 
     if (this.strategies.filter(not(false)).length === 0) {
       throw new Error('No strategies to write ' + this.id);
@@ -330,6 +358,8 @@ export class FileOutput {
         return;
       }
 
+      this.wildcardTrie.whitelist(domain, includeAllSubdomain);
+
       for (let i = 0; i < strategiesLen; i++) {
         const strategy = this.strategies[i];
         if (includeAllSubdomain) {
@@ -340,14 +370,27 @@ export class FileOutput {
       }
     }, true);
 
-    for (let i = 0, len = this.strategies.length; i < len; i++) {
+    // Now, we whitelisted out DOMAIN-KEYWORD
+    const whiteKwfilter = createKeywordFilter(Array.from(this.whitelistKeywords));
+    const whitelistedKeywords = Array.from(this.domainKeywords).filter(kw => !whiteKwfilter(kw));
+
+    this.wildcardTrie.dumpWithoutDot((wildcard) => {
+      if (kwfilter(wildcard)) {
+        return;
+      }
+
+      for (let i = 0; i < strategiesLen; i++) {
+        const strategy = this.strategies[i];
+        strategy.writeDomainWildcard(wildcard);
+      }
+    });
+
+    for (let i = 0; i < strategiesLen; i++) {
       const strategy = this.strategies[i];
-      if (this.domainKeywords.size) {
+      if (whitelistedKeywords.length) {
         strategy.writeDomainKeywords(this.domainKeywords);
       }
-      if (this.domainWildcard.size) {
-        strategy.writeDomainWildcards(this.domainWildcard);
-      }
+
       if (this.protocol.size) {
         strategy.writeProtocols(this.protocol);
       }

Build/lib/writing-strategy/adguardhome.ts

@@ -52,14 +52,12 @@ export class AdGuardHome extends BaseWriteStrategy {
     }
   }
 
-  writeDomainWildcards(wildcards: Set<string>): void {
-    for (const wildcard of wildcards) {
-      const processed = wildcard.replaceAll('?', '*');
-      if (processed.startsWith('*.')) {
-        this.result.push(`||${processed.slice(2)}^`);
-      } else {
-        this.result.push(`|${processed}^`);
-      }
+  writeDomainWildcard(wildcard: string): void {
+    const processed = wildcard.replaceAll('?', '*');
+    if (processed.startsWith('*.')) {
+      this.result.push(`||${processed.slice(2)}^`);
+    } else {
+      this.result.push(`|${processed}^`);
     }
   }
 

Build/lib/writing-strategy/base.ts

@@ -30,7 +30,7 @@ export abstract class BaseWriteStrategy {
   abstract writeDomain(domain: string): void;
   abstract writeDomainSuffix(domain: string): void;
   abstract writeDomainKeywords(keyword: Set<string>): void;
-  abstract writeDomainWildcards(wildcard: Set<string>): void;
+  abstract writeDomainWildcard(wildcard: string): void;
   abstract writeUserAgents(userAgent: Set<string>): void;
   abstract writeProcessNames(processName: Set<string>): void;
   abstract writeProcessPaths(processPath: Set<string>): void;

Build/lib/writing-strategy/clash.ts

@@ -30,7 +30,7 @@ export class ClashDomainSet extends BaseWriteStrategy {
   }
 
   writeDomainKeywords = noop;
-  writeDomainWildcards = noop;
+  writeDomainWildcard = noop;
   writeUserAgents = noop;
   writeProcessNames = noop;
   writeProcessPaths = noop;
@@ -64,7 +64,7 @@ export class ClashIPSet extends BaseWriteStrategy {
   writeDomain = notSupported('writeDomain');
   writeDomainSuffix = notSupported('writeDomainSuffix');
   writeDomainKeywords = notSupported('writeDomainKeywords');
-  writeDomainWildcards = notSupported('writeDomainWildcards');
+  writeDomainWildcard = notSupported('writeDomainWildcards');
   writeUserAgents = notSupported('writeUserAgents');
   writeProcessNames = notSupported('writeProcessNames');
   writeProcessPaths = notSupported('writeProcessPaths');
@@ -111,8 +111,8 @@ export class ClashClassicRuleSet extends BaseWriteStrategy {
     appendSetElementsToArray(this.result, keyword, i => `DOMAIN-KEYWORD,${i}`);
   }
 
-  writeDomainWildcards(wildcard: Set<string>): void {
-    appendSetElementsToArray(this.result, wildcard, i => `DOMAIN-REGEX,${ClashClassicRuleSet.domainWildCardToRegex(i)}`);
+  writeDomainWildcard(wildcard: string): void {
+    this.result.push(`DOMAIN-REGEX,${ClashClassicRuleSet.domainWildCardToRegex(wildcard)}`);
   }
 
   writeUserAgents = noop;

Build/lib/writing-strategy/legacy-clash-premium.ts

@@ -14,6 +14,6 @@ export class LegacyClashPremiumClassicRuleSet extends ClashClassicRuleSet {
     super(type, outputDir);
   }
 
-  override writeDomainWildcards = noop;
+  override writeDomainWildcard = noop;
   override writeIpAsns = noop;
 }

Build/lib/writing-strategy/singbox.ts

@@ -71,11 +71,9 @@ export class SingboxSource extends BaseWriteStrategy {
     );
   }
 
-  writeDomainWildcards(wildcard: Set<string>): void {
-    appendArrayInPlace(
-      this.singbox.domain_regex ??= [],
-      Array.from(wildcard, SingboxSource.domainWildCardToRegex)
-    );
+  writeDomainWildcard(wildcard: string): void {
+    this.singbox.domain_regex ??= [];
+    this.singbox.domain_regex.push(SingboxSource.domainWildCardToRegex(wildcard));
   }
 
   writeUserAgents = noop;

Build/lib/writing-strategy/surfboard.ts

@@ -12,7 +12,7 @@ export class SurfboardRuleSet extends SurgeRuleSet {
     super(type, outputDir);
   }
 
-  override writeDomainWildcards = noop;
+  override writeDomainWildcard = noop;
   override writeUserAgents = noop;
   override writeUrlRegexes = noop;
   override writeIpAsns = noop;

Build/lib/writing-strategy/surge.ts

@@ -33,7 +33,7 @@ export class SurgeDomainSet extends BaseWriteStrategy {
   }
 
   writeDomainKeywords = noop;
-  writeDomainWildcards = noop;
+  writeDomainWildcard = noop;
   writeUserAgents = noop;
   writeProcessNames = noop;
   writeProcessPaths = noop;
@@ -78,8 +78,8 @@ export class SurgeRuleSet extends BaseWriteStrategy {
     appendSetElementsToArray(this.result, keyword, i => `DOMAIN-KEYWORD,${i}`);
   }
 
-  writeDomainWildcards(wildcard: Set<string>): void {
-    appendSetElementsToArray(this.result, wildcard, i => `DOMAIN-WILDCARD,${i}`);
+  writeDomainWildcard(wildcard: string): void {
+    this.result.push(`DOMAIN-WILDCARD,${wildcard}`);
   }
 
   writeUserAgents(userAgent: Set<string>): void {
@@ -176,7 +176,7 @@ export class SurgeMitmSgmodule extends BaseWriteStrategy {
   writeDomainSuffix = noop;
 
   writeDomainKeywords = noop;
-  writeDomainWildcards = noop;
+  writeDomainWildcard = noop;
   writeUserAgents = noop;
   writeProcessNames = noop;
   writeProcessPaths = noop;