From 9c7ecd8b9a55fc932b069ddee43e0bc1a3156d0d Mon Sep 17 00:00:00 2001
From: SukkaW <isukkaw@gmail.com>
Date: Wed, 16 Apr 2025 21:59:01 +0800
Subject: [PATCH] Chore: dedupe tools now supports whitelist

---
 Build/tools-dedupe-src.ts | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/Build/tools-dedupe-src.ts b/Build/tools-dedupe-src.ts
index c1fc6191..3d6e6b25 100644
--- a/Build/tools-dedupe-src.ts
+++ b/Build/tools-dedupe-src.ts
@@ -4,6 +4,8 @@ import fsp from 'node:fs/promises';
 import { SOURCE_DIR } from './constants/dir';
 import { readFileByLine } from './lib/fetch-text-by-line';
 
+const WHITELIST: string[] = ['packages.argotunnel.com', 'compass-ssl.xbox.com', 'static.agilebits.com', 'ntp.api.bz', 'softwareupdate.vmware.com', 'ftp.apache.org', 'ftp.cuhk.edu.hk', 'apache.belnet.be', 'mirrors.viethosting.com', 'apache.01link.hk', 'artfiles.org.org', 'mirror.synyx.de', 'apache.mediamirrors.org', 'wwwftp.ciril.fr', 'mirror.dkd.de', 'apache.javapipe.com', 'ftp.heikorichter.name', 'apache.panu.it', 'mirrors.supportex.net', 'apache.forsale.plus', 'apache.spinellicreations.com', 'ftp.itu.edu.tr', 'mirror1.spango.com', 'apache.oshte.net', 'mirrors.koehn.com', 'apache.dattatec.com', 'download.nextag.com', 'mirror.jre655.com', 'mirror.kiu.ac.ug', 'apache.cp.if.ua', 'mirrors.sorengard.com', 'ftp.igh.cnrs.fr', 'mirrors.hostingromania.ro', 'mirror.bhoovd.com', 'download.xs4all.nl', 'cpan.panu.it', 'cpan.nctu.edu.tw', 'mirror.serverbeheren.nl', 'cpan.llarian.net', 'cpan.etla.org', 'mirrors.syringanetworks.net', 'mirror.met.hu', 'cpan.cs.uu.nl', 'mirror.teklinks.com', 'mirror.rasanegar.com', 'ctan.kako-dev.de', 'ctan.ijs.si', 'mirrors.chevalier.io', 'mirror.yongbok.net', '1-mirrors.in.sahilister.net', '2-mirrors.in.sahilister.net', 'cc.uoc.gr', 'mirror.sergal.org', 'mirrors.mi.ras.ru', 'ctan.cs.uu.nl', 'mirrors.tripadvisor.com', 'gnu.spinellicreations.com', 'ftp.neowiz.com', 'mirror.rackdc.com', 'mirror.veriportal.com', 'ftp.pbone.net', 'downloader.cursor.sh', 'redrockdigimark.commirror', 'nimiq.by', 'aaxdetect.com', 'ctan.epst-tlemcen.dz', 'udahce.com', 'rs-staticart.ybcdn.net', 'doumpaq.com', 'c.medialytics.com', 'keybut.com', 'adserver.ubiyoo.com', 'kaspa-classic.com', 'minafacil.com', 'jiandanpool.com', 'xn--blockchan-n5a.com', 'alphax.pro', 'crypto-pool.online', 'bbqpool.org', 'nyxcoin.org', 'lpool.name', 'tsfpool.xyz', 'ltcmaster.xyz', '8282.space', 'myminingpool.uk', 'binance.live', 'mining.garden', 'scaleway.ovh', 'atpool.party', 'nimiq.by', 'binance.directory', 'onyx.run', 'lucky-pool.co.uk', 'ra7.xyz'];
+
 (async () => {
   const files = await new Fdir()
     .withFullPaths()
@@ -34,11 +36,25 @@ async function dedupeFile(file: string) {
       continue;
     }
     if (set.has(line)) {
-      // do nothing
-    } else {
-      set.add(line);
-      result.push(line);
+      continue;
     }
+
+    // We can't use a trie here since we need to keep the order
+    if (WHITELIST.some((item) => {
+      if (item.length > line.length) {
+        return false;
+      }
+
+      return (
+        item === line // exact match
+        || line.endsWith('.' + item) // the whitelist is considered as a domain-suffix
+      );
+    })) {
+      continue;
+    }
+
+    set.add(line);
+    result.push(line);
   }
 
   return fsp.writeFile(file, result.join('\n') + '\n');