From 88af6ddee2b5ef711569b3ee3203a9f024146d58 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Fri, 15 Dec 2023 17:25:28 +0800 Subject: [PATCH] Update Download / Global Hosts --- Build/lib/normalize-domain.ts | 15 +++++ Build/lib/parse-filter.ts | 19 +----- Build/validate-gfwlist.ts | 117 +++++++++++++++++++++++++++++++++ Source/domainset/download.conf | 1 + Source/non_ip/global.conf | 10 --- Source/non_ip/global_plus.conf | 35 ++++++++++ bun.lockb | Bin 88290 -> 88634 bytes package.json | 1 + 8 files changed, 172 insertions(+), 26 deletions(-) create mode 100644 Build/lib/normalize-domain.ts create mode 100644 Build/validate-gfwlist.ts diff --git a/Build/lib/normalize-domain.ts b/Build/lib/normalize-domain.ts new file mode 100644 index 00000000..140ebe27 --- /dev/null +++ b/Build/lib/normalize-domain.ts @@ -0,0 +1,15 @@ +import * as tldts from './cached-tld-parse'; +import { isProbablyIpv4 } from './is-fast-ip'; +export const normalizeDomain = (domain: string) => { + if (!domain) return null; + if (isProbablyIpv4(domain)) return null; + + const parsed = tldts.parse2(domain); + if (parsed.isIp) return null; + if (!parsed.isIcann && !parsed.isPrivate) return null; + + const h = parsed.hostname; + if (!h) return null; + + return h[0] === '.' ? h.slice(1) : h; +}; diff --git a/Build/lib/parse-filter.ts b/Build/lib/parse-filter.ts index 5c9bb77a..f1e7d4b4 100644 --- a/Build/lib/parse-filter.ts +++ b/Build/lib/parse-filter.ts @@ -1,14 +1,15 @@ // @ts-check import { defaultRequestInit, fetchWithRetry } from './fetch-retry'; -import * as tldts from './cached-tld-parse'; + import { fetchRemoteTextAndReadByLine } from './fetch-text-by-line'; import { NetworkFilter } from '@cliqz/adblocker'; import { processLine } from './process-line'; import { getGorhillPublicSuffixPromise } from './get-gorhill-publicsuffix'; import type { PublicSuffixList } from 'gorhill-publicsuffixlist'; -import { isProbablyIpv4 } from './is-fast-ip'; + import { traceAsync } from './trace-runner'; import picocolors from 'picocolors'; +import { normalizeDomain } from './normalize-domain'; const DEBUG_DOMAIN_TO_FIND: string | null = null; // example.com | null let foundDebugDomain = false; @@ -23,20 +24,6 @@ const warnOnce = (url: string, isWhite: boolean, ...message: any[]) => { console.warn(url, isWhite ? '(white)' : '(black)', ...message); }; -const normalizeDomain = (domain: string) => { - if (!domain) return null; - if (isProbablyIpv4(domain)) return null; - - const parsed = tldts.parse2(domain); - if (parsed.isIp) return null; - if (!parsed.isIcann && !parsed.isPrivate) return null; - - const h = parsed.hostname; - if (!h) return null; - - return h[0] === '.' ? h.slice(1) : h; -}; - export function processDomainLists(domainListsUrl: string, includeAllSubDomain = false) { return traceAsync(`- processDomainLists: ${domainListsUrl}`, async () => { const domainSets = new Set(); diff --git a/Build/validate-gfwlist.ts b/Build/validate-gfwlist.ts new file mode 100644 index 00000000..70d13e86 --- /dev/null +++ b/Build/validate-gfwlist.ts @@ -0,0 +1,117 @@ +import { processLine } from './lib/process-line'; +import { normalizeDomain } from './lib/normalize-domain'; +import { createTrie } from './lib/trie'; +import { Readable } from 'stream'; +import { parse } from 'csv-parse'; +import { readFileByLine } from './lib/fetch-text-by-line'; +import path from 'path'; + +export const parseGfwList = async () => { + const whiteSet = new Set(); + const blackSet = new Set(); + + const text = await (await fetch('https://raw.githubusercontent.com/gfwlist/gfwlist/master/gfwlist.txt')).text(); + for (const l of atob(text).split('\n')) { + const line = processLine(l); + if (!line) continue; + if (line[0] === '[') { + continue; + } + if (line.includes('.*')) { + continue; + } + if (line.includes('*')) { + continue; + } + if (line.startsWith('@@||')) { + whiteSet.add(line.slice(4)); + continue; + } + if (line.startsWith('@@|http://')) { + whiteSet.add(line.slice(8)); + continue; + } + if (line.startsWith('@@|https://')) { + whiteSet.add(line.slice(9)); + continue; + } + if (line.startsWith('||')) { + blackSet.add(line.slice(2)); + continue; + } + if (line.startsWith('|')) { + blackSet.add(line.slice(1)); + continue; + } + if (line.startsWith('.')) { + blackSet.add(line.slice(1)); + continue; + } + const d = normalizeDomain(line); + if (d) { + blackSet.add(d); + continue; + } + } + + const top500Gfwed = new Set(); + + const res = await fetch('https://radar.cloudflare.com/charts/LargerTopDomainsTable/attachment?id=843&top=1000'); + const stream = Readable.fromWeb(res.body!).pipe(parse()); + + const trie = createTrie(blackSet); + + for await (const [domain] of stream) { + if (trie.has(domain)) { + top500Gfwed.add(domain); + } + } + + const notIncludedTop500Gfwed = new Set(top500Gfwed); + + const runAgainstRuleset = async (ruleset: string) => { + for await (const l of readFileByLine(ruleset)) { + const line = processLine(l); + if (!line) continue; + const [type, domain] = line.split(','); + if (type === 'DOMAIN-SUFFIX') { + if (top500Gfwed.has(domain)) { + notIncludedTop500Gfwed.delete(domain); + } + } else if (type === 'DOMAIN-KEYWORD') { + for (const d of top500Gfwed) { + if (d.includes(domain)) { + notIncludedTop500Gfwed.delete(d); + } + } + } + } + }; + + await Promise.all([ + runAgainstRuleset(path.resolve(import.meta.dir, '../Source/non_ip/global_plus.conf')), + runAgainstRuleset(path.resolve(import.meta.dir, '../List/non_ip/stream.conf')) + ]); + + // for await (const l of readFileByLine(path.resolve(import.meta.dir, '../List/non_ip/stream.conf'))) { + // const line = processLine(l); + // if (!line) continue; + // const domain = line[0] === '.' ? line.slice(1) : line; + // if (top500Gfwed.has(domain)) { + // notIncludedTop500Gfwed.delete(domain); + // } + // } + + console.log(notIncludedTop500Gfwed); + + return [ + whiteSet, + blackSet, + trie, + top500Gfwed + ] as const; +}; + +if (import.meta.main) { + parseGfwList(); +} diff --git a/Source/domainset/download.conf b/Source/domainset/download.conf index 10b0a83e..4c5128d1 100644 --- a/Source/domainset/download.conf +++ b/Source/domainset/download.conf @@ -169,6 +169,7 @@ fedorapeople.org mirrors.edge.kernel.org .mirrors.dotsrc.org .mirror.clarkson.edu +.mirror.constant.com # WhatPulse releases.whatpulse.org # GIMP diff --git a/Source/non_ip/global.conf b/Source/non_ip/global.conf index 55bd9865..fd643745 100644 --- a/Source/non_ip/global.conf +++ b/Source/non_ip/global.conf @@ -16,8 +16,6 @@ DOMAIN,news-edge.apple.com # >> Google DOMAIN-SUFFIX,abc.xyz -DOMAIN-SUFFIX,ampproject.org -DOMAIN-SUFFIX,android.com DOMAIN-SUFFIX,androidify.com DOMAIN-SUFFIX,autodraw.com DOMAIN-SUFFIX,capitalg.com @@ -35,7 +33,6 @@ DOMAIN-SUFFIX,getmdl.io DOMAIN-SUFFIX,gmodules.com DOMAIN-SUFFIX,godoc.org DOMAIN-SUFFIX,golang.org -DOMAIN-SUFFIX,gstatic.com DOMAIN-SUFFIX,gv.com DOMAIN-SUFFIX,gwtproject.org DOMAIN-SUFFIX,itasoftware.com @@ -68,13 +65,6 @@ DOMAIN-SUFFIX,bing.com # >> Oracle DOMAIN-SUFFIX,oracle.com -# >> Snapchat -DOMAIN-SUFFIX,sc-cdn.net -DOMAIN-SUFFIX,snap-dev.net -DOMAIN-SUFFIX,snap.com -DOMAIN-SUFFIX,snapchat.com -DOMAIN-SUFFIX,snapkit.co - # >> 451 DOMAIN-SUFFIX,aicoin.com DOMAIN-SUFFIX,aimoon.com diff --git a/Source/non_ip/global_plus.conf b/Source/non_ip/global_plus.conf index 25001aa7..5599e8c9 100644 --- a/Source/non_ip/global_plus.conf +++ b/Source/non_ip/global_plus.conf @@ -5,9 +5,13 @@ DOMAIN-SUFFIX,cloudflareresolve.com # >> Google +DOMAIN-SUFFIX,ampproject.org +DOMAIN-SUFFIX,android.com DOMAIN-SUFFIX,appspot.com DOMAIN-SUFFIX,blogger.com DOMAIN-SUFFIX,blogblog.com +DOMAIN-SUFFIX,cloudfunctions.net +DOMAIN-SUFFIX,firebaseio.com DOMAIN-SUFFIX,getoutline.org DOMAIN-SUFFIX,gvt0.com DOMAIN-SUFFIX,gvt1.com @@ -15,6 +19,7 @@ DOMAIN-SUFFIX,gvt2.com DOMAIN-SUFFIX,gvt3.com DOMAIN-SUFFIX,googleapis.cn DOMAIN-KEYWORD,google +DOMAIN-SUFFIX,gstatic.com DOMAIN-SUFFIX,gmail.com DOMAIN-SUFFIX,ggpht.com DOMAIN-KEYWORD,blogspot @@ -22,6 +27,10 @@ DOMAIN-SUFFIX,youtu.be DOMAIN-SUFFIX,yt.be DOMAIN-SUFFIX,ytimg.com DOMAIN-SUFFIX,g.co +DOMAIN-SUFFIX,goo.gl +DOMAIN-SUFFIX,dns.google +# ocsp.pki.goog is available in Mainland China +DOMAIN,pki.goog # >> Facebook DOMAIN-SUFFIX,cdninstagram.com @@ -111,6 +120,19 @@ DOMAIN-SUFFIX,archive.ph DOMAIN-SUFFIX,archive.is DOMAIN-SUFFIX,archive.today +# >> Snapchat +DOMAIN-SUFFIX,sc-cdn.net +DOMAIN-SUFFIX,snap-dev.net +DOMAIN-SUFFIX,snap.com +DOMAIN-SUFFIX,snapchat.com +DOMAIN-SUFFIX,snapkit.co + +# Signal +DOMAIN-SUFFIX,signal.art +DOMAIN-SUFFIX,signal.org +DOMAIN-SUFFIX,signalusers.org +DOMAIN-SUFFIX,whispersystems.org + # >> Other DOMAIN-SUFFIX,abc.net.au DOMAIN-SUFFIX,amazon.co.jp @@ -126,6 +148,7 @@ DOMAIN-SUFFIX,bibox.com DOMAIN-SUFFIX,binance.com DOMAIN-SUFFIX,bitfinex.com DOMAIN-SUFFIX,booklive.jp +DOMAIN-SUFFIX,brave.com DOMAIN-SUFFIX,bwh1.net DOMAIN-SUFFIX,cbc.ca DOMAIN-SUFFIX,character.ai @@ -142,7 +165,9 @@ DOMAIN-SUFFIX,exhentai.org DOMAIN-SUFFIX,feedly.com DOMAIN-SUFFIX,flickr.com DOMAIN-SUFFIX,gate.io +DOMAIN-SUFFIX,godaddy.com DOMAIN-SUFFIX,goodreads.com +DOMAIN-SUFFIX,homedepot.com DOMAIN-SUFFIX,huggingface.co DOMAIN-SUFFIX,initiummall.com DOMAIN-SUFFIX,issuu.com @@ -171,6 +196,7 @@ DOMAIN-SUFFIX,pixiv.net DOMAIN-SUFFIX,pixiv.org DOMAIN-SUFFIX,pximg.net DOMAIN-SUFFIX,pornhub.com +DOMAIN-SUFFIX,phncdn.com DOMAIN-SUFFIX,quora.com DOMAIN-SUFFIX,quoracdn.net DOMAIN-SUFFIX,redd.it @@ -185,10 +211,12 @@ DOMAIN-SUFFIX,shadowsocks.org DOMAIN-SUFFIX,slideshare.net DOMAIN-SUFFIX,soundcloud.com DOMAIN-SUFFIX,steamcommunity.com +DOMAIN-SUFFIX,surfshark.com DOMAIN-SUFFIX,theinitium.com DOMAIN-SUFFIX,theguardian.com DOMAIN-SUFFIX,thisav.com DOMAIN-SUFFIX,tineye.com +DOMAIN-SUFFIX,tradingview.com DOMAIN-SUFFIX,tumblr.com DOMAIN-SUFFIX,turbobit.net DOMAIN-SUFFIX,twitch.tv @@ -198,11 +226,13 @@ DOMAIN-SUFFIX,v2fly.org DOMAIN-SUFFIX,v2ray.com DOMAIN-SUFFIX,vimeo.com DOMAIN-SUFFIX,vine.co +DOMAIN-SUFFIX,viber.com DOMAIN-SUFFIX,voachinese.com DOMAIN-SUFFIX,washingtonpost.com DOMAIN-SUFFIX,whoer.net DOMAIN-SUFFIX,wikibooks.org DOMAIN-SUFFIX,wikidata.org +DOMAIN-SUFFIX,wikimedia.org DOMAIN-SUFFIX,wikinews.org DOMAIN-SUFFIX,wikipedia.org DOMAIN-SUFFIX,wikiquote.org @@ -210,9 +240,14 @@ DOMAIN-SUFFIX,wikisource.org DOMAIN-SUFFIX,wikiversity.org DOMAIN-SUFFIX,wikivoyage.org DOMAIN-SUFFIX,wiktionary.org +DOMAIN-SUFFIX,wordpress.com DOMAIN-SUFFIX,wsj.com DOMAIN-SUFFIX,wsj.net +DOMAIN-SUFFIX,xfinity.com +DOMAIN-SUFFIX,xhamster.com +DOMAIN-SUFFIX,xnxx.com DOMAIN-SUFFIX,xvideos.com +DOMAIN-SUFFIX,xvideos-cdn.com DOMAIN-SUFFIX,yahoo.com DOMAIN,search.yahoo.co.jp diff --git a/bun.lockb b/bun.lockb index bc477b2f42a6f1ae722c106edc9c668cb548f48c..27cc62190ac630cf62fda7e5cb6e956c37d2bc1e 100755 GIT binary patch delta 14171 zcmeI3iGLMEw#U0JxsV0|0RqV-!32;^NXSA6NyxpdVrT<`BCAS(009CaVG9@%Kok&A zEb)SD0;4R7ir`>y0Yn{9XBs({QC5-_H*h~ z)$L00*>3Otz1|0tI;_5b^u~3!wO=0R8*}K%d5v-!?)*Bl>6IZ3KRIfRc`i57zjL@U zB>ZdD(D>{RPTKE0Vn9Q~sGKon+>BD*H<7C@e3&asXOtA=7tJxoc%1lFt9C~Zno!Fj zyIeUES)W4wFvEyOrXlMg2f69VUMIZ`QsVa`Bamior=DKgr{eGU9M*)`_mNS^*O1b+ z0d7}%Bc*gv;nc$7{L<17!wtiPXAv)Q8xKi8hx8%OyQ+PGH1!IVR#8PK8~?X{F3-)k`FHBW{^%)iD7)G7kHv8Mo?5@2RC(e&L2Ci zWGc0vc5@e$PAQx^!!UAbURn;g@**-4KBZumWH-KbmlEI|YmN z=33DmiW406CfP(k)zYcHq_C(Yf4pJXZKCfV^~qZ88a+v>G&;GUY?fHz7VQg*_hn!2 zlzE9dM3qBu>B@FJC@fxW(9^@B&Gu~_dtBD5!lEnowsVx4m|vPWeoFp?{7H6oM#yuE z+8YK2tGt&Q`Byp1?cMnN(izifx1?ZZY2o-X<4^}@G#|RbY54?FtiBK_+6 zd`wCH*n(2S_yb(xzeI{tbnohFx&aB1i&oH3~U3Mmb~g~UlKJM?nWXXF>>kC{?nEQE_Co*`X2f?@3X*7bH2 zuGPm`vF=1-08i!lJxlAvtMJ6j+mB!9baUCB?cul7R#S9!gMfF3GK?I3r9rkjt;-q) zR7YLKlQf>&b!KEheWuHJM(V1_fH|EtygMz{)+#c?+~+c{F!jAIYZNetgc%0;WiM>9K2SqBENW)IeRvbG5GGc}Z9E zOwyTA0X1HiMFq^gbr=XrD5=X67OpQv`OW@p0@8pTsqWQfO#`Z0SMf~I)jX%@%w_@g zoGxqDuaSuhh>@|K9pdzR&DyDeu5K2HSWH?LY4pnQ>O+k?>T?ak%>%^Zi*EOS)Me2D zPf9(#KiaRR>FVf!xrISt>?;l3Gb%%Us>@;m-nSX>Jh87QCPH6|@vE6S(-%-5=rUix z6VpKN_xV+kt|n@?&TJm=#xP(5bx!ka?*!f~c{302hR>mvqTY)-Gd5tR;V^l2T+hDU z!XSmZJSN+FhPPYg&Ah3RV+*hD$>7|h%i;p+n68Qoc#H78{`%#(Z1tMXY!UFpH_>xj z_{{>WfyYZLz6|eUup1?lr+$>qy3ViW>ayzs=F`k9u}p1yK)-Q(_yomidK+Ha0$ zia6DX&u)SBu`@2fmZx=f>wvdy9ILRt(lFb6f;UGY)hxsO7{(gnc%>(*g}!vX-yFso z<7R1-VXlEmY<`ChqR^sWFQxQ?pV$D8sxFCI)eOSgFg}2Gk*4)iz-MOp27k zrf8kT5+(I}Z42KElQetEn6J6avA!q4>7J7_3nt}iY3cern3ThkuqxP1FlXA9OP}2-^r_0FM zGQ<1~CjGEyq?yp#sReDW%P>d6B#j>9p=)(j$AI}RaiRqaL7NPZ_j;Yx$#3?(-s!Mo z$OW(-WT~T7bcXjWSeAadakd%O##s>@Q{4uWPTA{_TCS@S1LiBlb#f|iond|nljf3w2qFED=>H0kh5x zjzLIin&HX0L1(4<&5cBg3GD3VH!#Qd7c|N+(>fXk`;1tSX>v0nhGC%`lX2o#9;y>t(Mny_5C+IKR0T#~W=|#FB0%vYp=!!)IeM z%wAa8^OK zPL~-nfP8M~xjBBdSy$x*%!c%4JQl1<4UFn1HzmpEJ}Cd2y^tc&ga8Egx^?7VbW zmFdhtz&t{n42WIc)2O@NAMmTYbaf!0R_V+h0rM=T5Ur?;6{sEi;;l{SR7>}a&oDDlSG<@N zCndvN0b^8Te#B;|!@8_*!1VO4p^$e7%-$`{oxJrUZ>0PNFr)B*{!Xp28DWC54HJ#ZnyCTN0f72QY$$n=Ck+nb^@PO-pgx`{0;#&hLe?5>OQp&S9*`cPC zcvcTPAV=5*Su$+Ns$hqjGK?6;!%k;{+Ts5rJ-X8Xo0xkxB_SO0Eq+^nVNp-wdR?+kga-62Dytp{AragX0HD?33~!h?Lw<2_cA-K0FO1{uv-a zr0{25xf>}#WG%g7aCC*F{!yX`HKo+{0+947AVH*z-OEAt8 z{xFb$w0*5+gl(iZoOGk6lo5E>jTb2uzUOk0Qt{thUQBV}rQ;_?&5SZ7pD z5)lEu1+~EsKnnf@B#5j7t^jH1Dv*3(LjFUFbDG3U`FfJBuiWa`xq+Koq`WqCWuzNl zQ%dQ^#7lRZyXhjO_IQ_ztPAge6yNIP#@Cb*m8cB8D9=~OtI?SViA#263R22;ad|pY zG|NUxA9CFE9!Loy#RvMh{I{giErvK6_j7ZJ6io&qrB<0Dk~Ely4D3iZ{tk&G)RfX% zKJg+8koAzoNXa@~9`2|L-N=&)|H}e9FUIu);>VA9WSPC4W#__K!Qjq z-%bdjrj+#VgyznjoxboYYeyT=`U zu6d8j$8TIy@O|NsduQFU__dGAmVN#49f2>d|D<2syS>LfmFW4f`9&O08CM%#Woow8 z!}Evg@G%kk<$O!E(dL+;`UP0g7)!O&Rj`R;BXo<-oPr3QSYWBnx~yQRZa+Rke*#O<9mWsU$6zbQTPjT-hb@^9p>rl!DqSy~ zFjRM&7@@y`W$4U_L-iN14HGSusn5XH6h`R0LQ7@qwS`0VO_L(@6aB-QLBAsOgZ0&B5&FT3iY(PnSHUJuMZc+* z%GHHa(Qg|1!7S~YhJLW}X_m^Y=m*CTdGtahb<{b zzjBL{?4{-CHy{0AvvuZt^n-1fZ*fR)2DYXG{VFUqPp_>&ze@CjmFwI}^n>lHv{Z$z zhHYPfehVzMK<`+9ehblWp`{k-;S14k5&FRvYjY9$!HO1H>ONfsn|KfU-D9bxy6_(K zTa13N`?YT|`oYQGN4_5x5rFQ8fusN&HZdZ~(2ivg8Qpfcf*qX=C?=ee#q}M)%ew)z`_KD8jjDE0Pn=Sr~Qw`g` z1^u>I>ZIPW1^u?7-&RYV(!;k-^NwL9ts5LTOzAy;eevO1`k&hlGN(3gpY6TUlucez zqWJ?-m>K$e{F{4v?g%KgX3v71(X}En+B$!Z?1_vFRg(}VCG@E0!o81V^1%>_U*pj? z_3K1CKX~1jI&Jj91}*Exe8##?h&!cE{?lJ^^_mkT>SG|k!vqpSTs|J?pdCV=lk$7E zznk#{@A4{NB{%aT@1FuGCwq1-kZ=-6n*8>Yj}g)<4C#(U)&W!Hg>VYUt9(Y?=EnWY zjg#L(L)^H3yK(YiID$Co)M+3M%U0Ra&G!ZG@>(Ap1=6WAz#}m?No#~3PV1!4=BGyU zDxY97SYyFBAfI$Mfm|R;TR`rY+z9$8ed4nbvZ|fRh`5)+Hv!SOzb-!&U(ua6PJRsz zl#LXS3gSTmXbI$FtvQGV@)0+lDkp%6U=o-Nih!6#&SY)|^2K-;5YtKL#SCA8v)~-K zl{7IMW~#7l&=>MH34Z~HfgEPM0``L!!An36MD_tWE_n{@0dkZg$1S_SFfbgf2XYLw z20RQjkn^XNK+c<%fTdtLXf0Qfwf>2 zcnI{Qu-q7uqkt3e-k=BISiu;Iyb7*?T5vh|s0-vo=2g%U>;#X2M}Zu9RLH5wX2=$> z6+8}}0P8?~(sm#tkTP$s<6YcX&N>pnQFtP_9mrWm?HXxvmJ!OA@HrU>7r%C7-R?8f zQvC^C5=sko;Il@FqR5Bf10bCh%ZQg|f^HxKi1pinHhR$Mtrcx~>k86A$a_2UE-H#C zWt==fde;fa_@#hkARaE|S_AQHk#B*=fFERmY>)$FcJu?{0}_8D=p(B_$nHIf=;bE# zbrS7N8+n(plY)bQs3UPQuHv1KfRJ~JC*BX1fW@E!l!Lp$JWvK`)NXS&a+c)q@lpuJ zf-zt;xC`6~?f@gf2rvu`1vdj3r@=rv)E}527l^lt*NgXy_ex$1NPNhf`J0C^1l$IO zOAkkZd>{=KfN@|vm;fe%Nnj=@1(U%PPy(idVjvxv3W~rq_g&-+FbB+arM%Aq^T9n} zA*iV2u>dRrk>Fl%9}uf714}_u@Bml=R)UAXgJ2a{4aCIK2{F68%aD!+V!^dQEGOw} zz&h|Kkj2Fwe;H4KjN=jT64(Mp13wUF+3d;}kz0Y}-vIW3{on=gIFQCeLnlMi18fI> z08&QsKLNG@CqMR=1c{_KwhUFY11>|-14tzq;E!N0cnXMiPlBC5EV3NPQYvw~z%$@! z@GRI3o&!?f^FZ2@b~eiRcL!CVX775yrDM`l(M~jzPDtDv;7#x+@H#jQq+&7g%b*94 zdzp6$g2dLgE-Vf_XqJ(Hn>^I>znA>U$@vy0c4Ca#AuT z(bLcR_=>xqT%oJABh>njZoIImm|W@PO5yj1K0q$N?s?A5%U68FIbUq(lI^L~9`&+5 z?58P_>eTDx-LFTID|F@dyQ2 zT&wzy9V3_ZtLprE^m4oA)UHV>saf?O`aXtl;9ci^v7sxqADsI6=}#8SJ?i8Vy%*|} z=Y4$3T_#uP0`F&2wtU&|^mV5tSBmsAwMuvX&X>N45voheNwwOHYO}rLJkq;LYN`>s z;+yuu>nBfd+&9jnZg7YDurB}37aO|z`;>oB@}&*eR{Wa&rQZFW&to*y?|c^%7rFtw zCH>~)oxzv;+TH9T-4ta~afh_;)6zrN10x!#drG4h>?K!n62r)1)7ZJ+xAO7Qtz1Q!}4E z@%67Mr*(@Te6gWB#O3Ecc=f5=usyV&nuN2lF;Rb-8le~ekf%P>@ADk2TmI-HCi};Z z<|%dr8Mr0-{Exo$&?V9NU7j}oHg@6bwiYQ#sr+c8XZ)-DZ}lgOGgSo-)l$CTrFtr= zj#xg~u

Fjn?&liir!|TP>J>W_HQ&liTc$*- zscE4*u~{kBZOs>a`V=WL@K}D9Uh$KUD}dX6%HxCl$Di_Iw?{j7tygY-ZpfALcZZPI zooX-XJ1(GYa92G}sCHiH!o9eYSDS^UvGVHb3u#=NOT3uI$6h|quk~u47xa;fX&!Hk zzIrh=E_Ca8YIWwtyh(#&?TWjKNxH@aBO9q0vx-oW;nYa=uvia(3nL z1K)brpZKP|ZIYBEHnaiyJ!%=HzoN#KI{vaxy`VEMr*Q$Ii1&H=zRP)W#R>K&R%PdH z{)49%k14l%g-Q9)YN^j%uH@)p`pOi!WIAgJB@gKbuK419Gu_rV`+EJ_mGPeTZFQ%sH>Y22>o|DbGecVh9$S8ideUs4 zG1_>Yz76R7>K5NyhoeYgl1U4D^{%UF_Ne*z^1gi4$4NlbYiXQD4 z&Hm=7FWv4JKFGEmY@w7dJ#@|b>x*MPxvNQd7rT4*QXaZ+{Z;)0U%iEGQh!xxL~ycF z9X$!1gKLyZhz;H3Zsr}{VnB;UjP0G>lG0_3csuxVL*0fdZrp4ST!B;378@dpF za>++Cb`N-q`Yw<0i^-dJ-J%RRf&n(>!Tig8%ZWRFA()Fs8N|uNDR?YNJ%> zKK_AD>;92FB&I)Fvz{d}`;P_p)JB`ngP$WjiRr<(Iw~eMbZL_pA|6H)0ic*cn-GT|>4BT%h|Et!n zEgZ`SzYSMhH@EE^Y-Os(;UVJ$BkQT=!P`xeZwXE}>Coih{UYWEA2)yT@8n?Jx+=+B zmg(F`KNjp+mn^4)BSoAG&aA7_Vng@i*R6f4`ShX}>e+^}m;S2Y;ks%9qc5d`ed?(J z|Ho?Uth4G=@MJxer}hRr*QbdK!QA?4fy;vbs!wkR1+S5k9=bGdPV4^0mxCUCk@YS; ziC+p`jCMVo&JMqI^W8t^FJmA=yZ1A}yaqX@BHP<=C|CLM2B5}{ZCK)HRY|~ zIw|@Ub<1XE&FcQm`t?)#d6d1c;1vJ+6tuVL*sGKjm;N}@=ihaC)n9+z%(n(xG(gpZ z!HfnfFE(_kf8yD_X9stg(Sa^7)nq}O9Na{{^w9sGND2I*&E1O{H77$F^OeaK`VR_C z8|01{JMm~1DKbx)$@T~|4SE`?ZXP2!*u5c6v@kfNA)^=i?-F$e%$r@@;oDx0%CdaQ z#>VaYN7d$V**5#K4RB@;L^~&T{v|0f zJt;ky++MZZH=%_p%Bnle{U30nhX+p#Rjv4KKvMMXdqQ`x4b_(~em z%6w}b^I3|erlx6~#Gafo^Hdf+EIX&9G{y39e`7t=+wI)%{&Vm5-S0c|^S8(R&G#5{ z%(>Q@Okdq=|7y2=MQq2QJ_BDF=6L8kiXW%e^=PWrU)v=<`0iNf!uju|J$Zb>%^|~u z#Nq48%&64UU+QuEgagea$y-t|wxk&Gr%}oue2^)NONw%{C(V$uY(~1Ha(CF^M|}+P zoGFJuHbJE%Ur7ps>;xGA+271>Z#VK=L2~+T$UsP`ksTYVcfKM18jP%Mk{5IF0rKAlV|{As*Iqh5?D3u6kVG_l`_H{()7IioIKRY zFV39-4Ww^j0JhowdyL60%r41`LBlzD*##4kKb3a-hB+*6@63!_vyCFk_ae$om7yBz#x8?W-lmG7%MeJ;e+cQ=4!qx(ye~M6`xZ@Gexwa8vG(JND=EP1HWge0>6!^CPnw z9)3m#dZL(36)hA|Pc``MKyph~D+9kmiaZJ)i8=O4G&s~P)=*8jVqb(A^{^#vr<%|JN4iL$3Wp*p zsm7t$P6d*GM7roi84-$@OcfD|{S=&V5G{*Hwcm~9u5wJCg&5onS_Pxsm#M<3*z1wz zp=Higdm4NrpWY2mwf_aNp&XO#m|nK2od#nLdQeR(MO2X-sn|Qg69!OVWU81;6_JYV zvk*ENnJ%}1((rI}G1%d@j|A(>g={s=X>sdx;ij6_iaZf33CjZW%(M7B*e!F|J`JVC zQRyO_YN8Z*9G*&CQ19M$yaf%2PPbidL5ri)#Zy!ht%x9!?^4A5lyR40do!FC-<2*0 zIwUC$l_b_uP6J~NB&uxfwwEzuZCU#^ihL8C`-Y{PyX7QopI9V3>#g15G0JGG*xrbs z#ck8&wpaj0Gwhn9z&MglEmLE*;Ta{bfN?z^;-M!;-x|q%$$lXH^$yqq zC%|~fD0r7!_Qm$WgT%5#=gD9^0gXA&J`b$FZkM;sbbavh?`9n^)ala3vQljew>;it z$V2Pv!B{mgnBa3TZrzt^Tf61(wniSt0SA4E}$m z_hfI1$8-WM3r)3mH)0{F_9Ddk(lRvtCSpCU*6fYjqiuROELBzz!y^v*aEs+&z4aE^ zuE)8Qo~Lg^?dfD(y4)Mjc^;$T-z&i$0mI&g8LI~y1cnuUms==Q6VF?OoS?{WbTo`2 z2nM$O){zz`q{|O?GR&YCm$!lS)hky9qbV>&Z<5?8*sm7(qKZ6f8x(#yM zdUmFh(dqJZSca_%<1}~MKL#79t9Uow4LIFL_u<(l)4kDt7_6u69)7WgZyUbX6D$LD zP%7FjF9923Fid&_*g!C>khX4n-!6I!*kq(Q^2Z+Z5Zq(5-W~hP4`2_0HKM`cZuw@s z<;BeA!SD~>$~z6l`Wq!9U6SP$DlE;W2~QvU|b682X$TsV~@}ugR%{d&FwYT z*TQ{Z?0?uF;X5ZJ)WxKP_DjLnG{9 zMv?I*lX&uMe@&*7Y3brolG7FWofJtLhl2XTjOyF1OW z1UB~0(A{Ll{2T@28G*B7L;VE|PQcy3E!)6gvE;!%{QzQyTk>LAK{Y)U``bwCMS(3+ zg@ZDBDNwWmzw4={mtw0|$iH{Gy%XxdVIzYx#i1X z?7^701h+f@%3}_oO06@QYWgVh5v1Hw#~#v)Hwv^Wk4LN@N(S-MU#Tk< z14~X{Wy)twxf+rKOU_>du-@xUz5$Ze`3t~-C70X61mRA}{cY7hgN=w@a2uCG&?R}) zs2*HwC&10@0ywZ_{-P;gg5>%)4}$d~SvGSc2N(^+zZADEmaH(YD-J0+L<(9Hh` zlGn~>CjT6g&G01<2>b}A)2HPk2v__S;J~sma1G#&t^+)MK1}`~*?s+x&h`Dxe3l#y zFl7@{267?-OO7^WC*w#n{`rIC27{5#8aT{+mfT@$le6>(?*Pe~-D{@bDLJV#eptqs z`7Aj~A9zCZITD8-Zop+`Bto*reB8(bxzEg3AUUvP-{@uXKa-r*8$T@h_);Rzz_4;w zXCN52`hb}`7?LM^gqi*bB!@dC_cj{oEXP6yKu&?=qJ{ipO3p7a(>aYFBn&PR&f5$J zM09nXkUWq`fWx0jws#wV8)yq~V9DmXhY7-+lGE`N(F04acl%+IV`TiVK22zC|FC~Q zKnw@|_XEW6z<)nLu<-u<0Qvv_0LlG7d4N=39rjKDMUGKLTgo4kNqJ)e>0PjP zahbGeTp;}j7EdYTGbwp|Agvv*;?Vaz*!N(bJXIvos=Q2kCNGe#gSjaqKa=|B2hz@b zRiw~Wum-S^6I78(TPI}F3ljn!VXdoWM2Du&UjV(3>4{lGFQqXhbu zK)(`IWYJZy2C$LSRPiuvod*4;LBCQJN8DMZ(61EwfsG=0I`o?k{idrTn+|~O2aBAc z;zT=t2K1W&{lId`Q3n0WpkJ9P##0U039y)%s>r9Znb2=0^aGnn9Up^!k3ql3R56Lp zfPDg%HcJ&#sA3lMo7I$lnl(U7rIgvwZ#MLst>U=;JlOYOo;j+RMyuvPzd6tkY&vC> zL%(w9SFVaOx(e0+Hqxu&sCKIt`gx(>ToottS#zP^T<8Z@PV(c>?{Vn&xQfH-17Q2X zBIl`M9_7!2e)FIo*aC9Qhko;+-+Wapq#CdjU@;3+Q9)%3px*-M2eyPdJ^}rnfPPP? zVi}zQ`vfd)p(>uFiiOZ`A?6RPl2R5yzeUh*kt&GJgMAO?sZeo{zN!NHRX{&5jWQNP zzs1mRu_{*4Rj>xIkxNvunzk;1eoLU=QdO*_tfkOzDf9zdPx3P8w+#9%Q^iI)0Ja}2 za=D6=>HOu;Z#nb>d!8ImLcb@W-;=7?LN#C~z+zUY_^UVtKLh=qfqu`Z;&nOzwjV5V zl`4)?{wnCV3i^S)Nsecs-?Px~SyjA6HDD*eVpgl7hRRk$ztzwW>}~3}2Kudmerr_m zE}a4U1T1Z>D&D7xwa{-Z^aHD6&zfI5&>cDM^5W}iH+rxbPFSRrN z0C}Tb_2ga0e8te}`0Aoq|L8j7rTOl!=xklO@dkh6rV%*jQ-I&H85}ImZ}y>j%I$AJ zeml=F3-b3MNBNUMMTFyL0j|ey{rv$BUjm%RA4HM-1;GXC%tZc9oW!{ZUjZEDN7PU= z?Q1iQzi%Ei)4nm&_|q*5X*{TJ0dAMSYog6^-yzP?Ccwu459<41{BXesxHp7`Z|TOl zHq`&?M-oP(a5gXo$N~BS{eb=eD?I@40Q@z^Uu>f#`uOV+Vg|*3;|}bHwpil~Ad#kh z6IISfQhZDm2gC!dfhZsv;4dir;feoRuYX?UqPcOvc!0ksCjbQin`R}-@QD(C&OHj$ zAhSk9f1L7nct@ZUa6j@qKy$pgSt}1o%6h-MS~B0Q}+q5ae~>24Dl{(+U~jx8c`; zPQX@RJ-`QBO_9eZ7!x*tYy>s|&jHT^eBu>=%oiZ}ScBJ1B;xGGd>Ro2u&Z_kh5@nQ z_B-A%mFxyfe`zgo|0UjS4gB>#&a06WdHuyc{`EGN0JH(QJM8I;^ zSj1T)wj5834dB7t3vd^4Ko@`=n(MUz*oj%z0BnUcAO+|K+y|rreF2_wPVWQs;Sz1K@VK z&r)C-??etn@_;cwHZTf!6c`CS0*nB%fK1>aKm~YO1_L~x3_u3@1MHlG0CskERxUdL z;B?D*hamnSFccV${`C{w(U9EHSRe<;1;zp6fqb9@C;}z`6M?BfAut8tflLAlfXQZ@ zWic=vC^IF;rvbBpdB9wtoCn|q9tVPe1;7&kTcrY61hfE_151HrKqc@bumV^Ku!(sP zY<7;bdD)6=L0(~OInLLBXMt4!?o*D(%0&E7j4wru(_zPf^hx(ksi5QJ8t%hCz=P9uo>5BNvKsB%( zVC}X6TLHGn5`cG3PTK+O0(Jr~0xto(0d8+E!2NMQYtg^HH1KL_^=F&1oC|R!c8os2On@!HbI7LT z4&MPz0`Hmew;{PKOHO|ms0IE8d<1+3@HDa6KLJhyJgq#H9|P8uq3v>AvolC!P1vlg z4G)Sn;?B9^r@-F0juvj&lM z05_-rasR{*LP&@+YmMx7hc+9n>2i*=>McEuleum9x0XJOa{4?gR1FSx|h^zcOo?YR(vKQD^F2wB^#!z;7#kNGho?LS)dYQBFAy&kw(0$MJtra7 z1?$k-dWX}xPdh@a`Lyqq^@Ye!Ld|$=>r`9sz$fuP@T-@_A6lb+b~vq@wO=Q659s!B zKSGTJqc5ZEZkmKr{QBmPertzs`yk`e`fqJwPJC>9Y$9ffHva4oC+Q%5&(f)%JxSJ8 z)A~1N7p^(|&WAQ}$j`VZX5G6jZnR={&NJO+*u=-ki9oySDDxMW?TY|f`b)Uex(qwj z_C%XryFys~_}GN5vGEBJbmSLDl686Zqpz>+`h4!p(?&ikXp3Z$ zb3X-MbU3Z+yI&P-`k~*s*5^1sJ|5~Olv0n24r6+p)@5GT{x|E+t=l)&COWZOAb9D? zi;k|=CEp$CgX4Z%dvn?CeCwL;mnrGqZhsvbe!Ks}P3i2#@K)CS-%UyP$8FON_t6U^ z@&K;{P{^eOI7^>Pt|aTyUtqA9UmP}fFN(#*Vj?BUx<$GC`QptHy$3yS6YVjsKn%^7 zc3yIbmfC6`;SeQMcgbNp975MFg*&Y~!Y9rr?wIz{*?-<1?`G=qtHWvC6`o!H(XkyF zzSZb0Ar=mYTY`FPH0#$05k}gt9(*c&`>V%kT^4?3W%oBe+CQAtOp0~ICg5#WJJLk7 zq@2qw_AnXWtiDT>#sO) zZgb#@M~tAx4IXE1m~o?e`NloNuFrmd7|NQ9t&FBLIE1c}iw-p;;xqMUBrKqSt1f)J zEBM`@t@jmizA)8R+5*0pKE3L)9SftN-x69`SD7bYO=<8<7~<4h?#gDm7)Fo&7A{z6 zhts-I9bG!$rJ~RQM^P*;Hs$u3wr*YzJ5zUWNR&EL&&L|YBbvVW&E>RiTJP_^{ykMX z^eghQT)6&riuv7z&+zQu9fHd`oYpPv{!&w?=jeNqgRY|&mEx~CaAc!ibK$_Q9PwmYb0lBiYD82ugIf$*F82^ny~xVxRVw5H8jXbL~9d8DB404uf{q zPXF<=BMFsVs0^1Jt97vnN0N1+`kxJBJ|ETGFHs+_ZeZ(b^*@?KI|3eWm!MbGbHX&( z^IqHMvD!A9h_=sBWUPThTbYcaNgR^}aOZ}n0fmHc%Zj}Q+!w&=+??E|#uv@UTfIxW+z9jhyG!fTSd0^jR@oR0 z^3KxI7ptyHd$TcAcuuRt0Y#N5lFT6wUrC;ivm+-N94vhJ*}D?JpG|2KP)-i5v`{)q+p+|t@K zh%o_A``@y)mgp}+Y!~9S{{Ety)4HFYyR>=gQ*T#~F()6+-$C2mOoTQ{W=BZX_W9rP zd97QB2-jK#h*-HF%JF+km3Fh4Xkllk*{n?pfSxwYuSr29o93xKpTYuN!EW$IPuPVKSVt6Xe{(KUG|}#)zD})7?-o(Rh>4P3*B9?xGD0g5ecy zd=R{Lwze8uhiOt{BXNo0noltD%rXA!ed3mvMu}b8`e3w^svSU0+tFUyPr>NKnAS!K zSn$TS$SyL_Y5o6!<7WoN2RAKxAA@um8;jmzV?56A;nPxk4gO@jNoe&UBGG?z2wut& zHiT%g%|%aba&wUoaTQCRXLf#a(mP^C(;FX{h2F^A_iA&|RP3t_6)(yA4z?Br-TWo< g|FroH*W$B8n~;jMna(52H!M#!(}xV_Pk|5q1L?dQVgLXD diff --git a/package.json b/package.json index 30368955..1b8d59da 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "async-retry": "^1.3.3", "async-sema": "^3.1.1", "ci-info": "^4.0.0", + "csv-parse": "^5.5.3", "fast-cidr-tools": "^0.2.2", "gorhill-publicsuffixlist": "github:gorhill/publicsuffixlist.js", "mnemonist": "^0.39.6",