Chore: minor changes

Build/lib/create-file.worker.ts

@@ -31,7 +31,7 @@ const pool = new Worktank({
         isEqual = await fileEqual(linesA, readFileByLine(filePath));
       } else {
         console.log(`${filePath} does not exists, writing...`);
-        isEqual = false;
+        // isEqual = false; // isEqual is false by default anyway
       }
 
       if (isEqual) {

Build/lib/parse-filter/hosts.ts

@@ -5,25 +5,42 @@ import { onBlackFound } from './shared';
 
 const rSpace = /\s+/;
 
-function hostsLineCb(line: string, set: string[], includeAllSubDomain: boolean, meta: string) {
+function hostsLineCb(line: string, set: string[], meta: string) {
-  const _domain = line.split(rSpace, 3)[1]?.trim();
+  const _domain = line.split(rSpace, 3)[1];
   if (!_domain) {
     return;
   }
-  const domain = fastNormalizeDomainWithoutWww(_domain);
+  const domain = fastNormalizeDomainWithoutWww(_domain.trim());
   if (!domain) {
     return;
   }
 
   onBlackFound(domain, meta);
 
-  set.push(includeAllSubDomain ? `.${domain}` : domain);
+  set.push(domain);
+}
+
+function hostsLineCbIncludeAllSubdomain(line: string, set: string[], meta: string) {
+  const _domain = line.split(rSpace, 3)[1];
+  if (!_domain) {
+    return;
+  }
+  const domain = fastNormalizeDomainWithoutWww(_domain.trim());
+  if (!domain) {
+    return;
+  }
+
+  onBlackFound(domain, meta);
+
+  set.push('.' + domain);
 }
 
 export function processHosts(
   span: Span,
   hostsUrl: string, mirrors: string[] | null, includeAllSubDomain = false
 ) {
+  const cb = includeAllSubDomain ? hostsLineCbIncludeAllSubdomain : hostsLineCb;
+
   return span.traceChildAsync(`process hosts: ${hostsUrl}`, async (span) => {
     const filterRules = await span.traceChild('download').traceAsyncFn(() => fetchAssets(hostsUrl, mirrors, true));
 
@@ -31,7 +48,7 @@ export function processHosts(
 
     span.traceChild('parse hosts').traceSyncFn(() => {
       for (let i = 0, len = filterRules.length; i < len; i++) {
-        hostsLineCb(filterRules[i], domainSets, includeAllSubDomain, hostsUrl);
+        cb(filterRules[i], domainSets, hostsUrl);
       }
     });
 
@@ -41,6 +58,7 @@ export function processHosts(
 
 export function processHostsWithPreload(hostsUrl: string, mirrors: string[] | null, includeAllSubDomain = false) {
   const downloadPromise = fetchAssets(hostsUrl, mirrors, true);
+  const cb = includeAllSubDomain ? hostsLineCbIncludeAllSubdomain : hostsLineCb;
 
   return (span: Span) => span.traceChildAsync(`process hosts: ${hostsUrl}`, async (span) => {
     const filterRules = await span.traceChild('download').tracePromise(downloadPromise);
@@ -49,7 +67,7 @@ export function processHostsWithPreload(hostsUrl: string, mirrors: string[] | nu
 
     span.traceChild('parse hosts').traceSyncFn(() => {
       for (let i = 0, len = filterRules.length; i < len; i++) {
-        hostsLineCb(filterRules[i], domainSets, includeAllSubDomain, hostsUrl);
+        cb(filterRules[i], domainSets, hostsUrl);
       }
     });
 

Build/tools-migrate-domains.ts

@@ -6,17 +6,17 @@ import { processLine } from './lib/process-line';
 import { HostnameSmolTrie } from './lib/trie';
 import { dummySpan } from './trace';
 import { SOURCE_DIR } from './constants/dir';
+import { PREDEFINED_WHITELIST } from './constants/reject-data-source';
 
 (async () => {
   const trie = new HostnameSmolTrie();
 
-  writeFiltersToTrie(trie, 'https://cdn.jsdelivr.net/gh/DandelionSprout/adfilt@master/GameConsoleAdblockList.txt', true);
+  await writeHostsToTrie(trie, 'https://cdn.jsdelivr.net/gh/crazy-max/WindowsSpyBlocker@master/data/hosts/spy.txt', true);
 
-  for await (const line of readFileByLine(path.join(SOURCE_DIR, 'domainset', 'reject.conf'))) {
+  await runWhiteOnSource(path.join(SOURCE_DIR, 'domainset', 'reject.conf'), trie);
-    const l = processLine(line);
+
-    if (l) {
+  for (let i = 0, len = PREDEFINED_WHITELIST.length; i < len; i++) {
-      trie.whitelist(l);
+    trie.whitelist(PREDEFINED_WHITELIST[i]);
-    }
   }
 
   console.log('---------------------------');
@@ -24,15 +24,24 @@ import { SOURCE_DIR } from './constants/dir';
   console.log('---------------------------');
 })();
 
-// eslint-disable-next-line unused-imports/no-unused-vars -- ready to use function
+async function runWhiteOnSource(sourceFile: string, trie: HostnameSmolTrie) {
+  for await (const line of readFileByLine(sourceFile)) {
+    const l = processLine(line);
+    if (l) {
+      trie.whitelist(l);
+    }
+  }
+}
+
 async function writeHostsToTrie(trie: HostnameSmolTrie, hostsUrl: string, includeAllSubDomain = false) {
-  const hosts = await processHosts(dummySpan, 'https://cdn.jsdelivr.net/gh/DandelionSprout/adfilt@master/GameConsoleAdblockList.txt', [], includeAllSubDomain);
+  const hosts = await processHosts(dummySpan, hostsUrl, [], includeAllSubDomain);
 
   for (let i = 0, len = hosts.length; i < len; i++) {
     trie.add(hosts[i]);
   }
 }
 
+// eslint-disable-next-line unused-imports/no-unused-vars -- ready to use function
 async function writeFiltersToTrie(trie: HostnameSmolTrie, filterUrl: string, includeThirdParty = false) {
   const { whiteDomainSuffixes, whiteDomains, blackDomainSuffixes, blackDomains } = await processFilterRulesWithPreload(filterUrl, [], includeThirdParty)(dummySpan);
   for (let i = 0, len = blackDomainSuffixes.length; i < len; i++) {