From 99589cf2fc77badde8ad26b56c1e5f618c740fa9 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Wed, 15 Nov 2023 16:26:46 +0800 Subject: [PATCH] Refactor: continues to rewrite to TS --- ...s-domain.js => build-anti-bogus-domain.ts} | 16 ++-- ...{build-apple-cdn.js => build-apple-cdn.ts} | 13 ++-- .../{build-chn-cidr.js => build-chn-cidr.ts} | 16 ++-- Build/{build-common.js => build-common.ts} | 36 +++------ ...c-ruleset.js => build-domestic-ruleset.ts} | 20 +++-- ...n-rules.js => build-internal-cdn-rules.ts} | 39 ++++------ ...mains.js => build-internal-chn-domains.ts} | 15 ++-- ....js => build-internal-reverse-chn-cidr.ts} | 15 ++-- Build/build-mitm-hostname.js | 2 +- ...mainset.js => build-phishing-domainset.ts} | 31 ++++---- Build/{build-public.js => build-public.ts} | 18 ++--- ...ect-module.js => build-redirect-module.ts} | 14 ++-- ...domainset.js => build-reject-domainset.ts} | 43 +++++------ ...ainset.js => build-speedtest-domainset.ts} | 30 +++----- ...eam-service.js => build-stream-service.ts} | 34 +++----- ...elegram-cidr.js => build-telegram-cidr.ts} | 18 ++--- .../lib/{aho-corasick.js => aho-corasick.ts} | 50 ++++-------- Build/lib/{cache-apply.js => cache-apply.ts} | 22 +----- Build/lib/cached-tld-parse.js | 24 ------ Build/lib/cached-tld-parse.ts | 17 ++++ Build/lib/{clash.js => clash.ts} | 15 +--- Build/lib/{create-file.js => create-file.ts} | 37 ++------- Build/lib/domain-deduper.ts | 2 +- Build/lib/fetch-retry.js | 3 - Build/lib/fetch-retry.ts | 4 + ...csuffix.js => get-gorhill-publicsuffix.ts} | 16 ++-- Build/lib/parse-dnsmasq.ts | 4 +- .../lib/{parse-filter.js => parse-filter.ts} | 77 ++++++------------- ...e-sort-domain.js => stable-sort-domain.ts} | 40 +++------- .../lib/{trace-runner.js => trace-runner.ts} | 35 ++------- ...ate-domainset.js => validate-domainset.ts} | 21 +++-- tsconfig.json | 2 +- 32 files changed, 258 insertions(+), 471 deletions(-) rename Build/{build-anti-bogus-domain.js => build-anti-bogus-domain.ts} (78%) rename Build/{build-apple-cdn.js => build-apple-cdn.ts} (79%) rename Build/{build-chn-cidr.js => build-chn-cidr.ts} (69%) rename Build/{build-common.js => build-common.ts} (77%) rename Build/{build-domestic-ruleset.js => build-domestic-ruleset.ts} (73%) rename Build/{build-internal-cdn-rules.js => build-internal-cdn-rules.ts} (74%) rename Build/{build-internal-chn-domains.js => build-internal-chn-domains.ts} (55%) rename Build/{build-internal-reverse-chn-cidr.js => build-internal-reverse-chn-cidr.ts} (70%) rename Build/{build-phishing-domainset.js => build-phishing-domainset.ts} (83%) rename Build/{build-public.js => build-public.ts} (88%) rename Build/{build-redirect-module.js => build-redirect-module.ts} (92%) rename Build/{build-reject-domainset.js => build-reject-domainset.ts} (85%) rename Build/{build-speedtest-domainset.js => build-speedtest-domainset.ts} (82%) rename Build/{build-stream-service.js => build-stream-service.ts} (65%) rename Build/{build-telegram-cidr.js => build-telegram-cidr.ts} (71%) rename Build/lib/{aho-corasick.js => aho-corasick.ts} (69%) rename Build/lib/{cache-apply.js => cache-apply.ts} (56%) delete mode 100644 Build/lib/cached-tld-parse.js create mode 100644 Build/lib/cached-tld-parse.ts rename Build/lib/{clash.js => clash.ts} (50%) rename Build/lib/{create-file.js => create-file.ts} (67%) delete mode 100644 Build/lib/fetch-retry.js create mode 100644 Build/lib/fetch-retry.ts rename Build/lib/{get-gorhill-publicsuffix.js => get-gorhill-publicsuffix.ts} (70%) rename Build/lib/{parse-filter.js => parse-filter.ts} (88%) rename Build/lib/{stable-sort-domain.js => stable-sort-domain.ts} (58%) rename Build/lib/{trace-runner.js => trace-runner.ts} (51%) rename Build/{validate-domainset.js => validate-domainset.ts} (78%) diff --git a/Build/build-anti-bogus-domain.js b/Build/build-anti-bogus-domain.ts similarity index 78% rename from Build/build-anti-bogus-domain.js rename to Build/build-anti-bogus-domain.ts index 89be4388..f62df805 100644 --- a/Build/build-anti-bogus-domain.js +++ b/Build/build-anti-bogus-domain.ts @@ -1,10 +1,10 @@ // @ts-check -const path = require('path'); -const { isIPv4, isIPv6 } = require('net'); -const { createRuleset } = require('./lib/create-file'); -const { fetchRemoteTextAndCreateReadlineInterface, readFileByLine } = require('./lib/fetch-remote-text-by-line'); -const { processLine } = require('./lib/process-line'); -const { task } = require('./lib/trace-runner'); +import path from 'path'; +import { isIPv4, isIPv6 } from 'net'; +import { createRuleset } from './lib/create-file'; +import { fetchRemoteTextAndCreateReadlineInterface, readFileByLine } from './lib/fetch-remote-text-by-line'; +import { processLine } from './lib/process-line'; +import { task } from './lib/trace-runner'; const getBogusNxDomainIPs = async () => { /** @type {string[]} */ @@ -22,7 +22,7 @@ const getBogusNxDomainIPs = async () => { return result; }; -const buildAntiBogusDomain = task(__filename, async () => { +export const buildAntiBogusDomain = task(__filename, async () => { const bogusIpPromise = getBogusNxDomainIPs(); /** @type {string[]} */ @@ -61,8 +61,6 @@ const buildAntiBogusDomain = task(__filename, async () => { )); }); -module.exports.buildAntiBogusDomain = buildAntiBogusDomain; - if (import.meta.main) { buildAntiBogusDomain(); } diff --git a/Build/build-apple-cdn.js b/Build/build-apple-cdn.ts similarity index 79% rename from Build/build-apple-cdn.js rename to Build/build-apple-cdn.ts index 5ca897e8..7bddca33 100644 --- a/Build/build-apple-cdn.js +++ b/Build/build-apple-cdn.ts @@ -1,9 +1,10 @@ -const path = require('path'); -const { createRuleset } = require('./lib/create-file'); -const { parseFelixDnsmasq } = require('./lib/parse-dnsmasq'); -const { task } = require('./lib/trace-runner'); +// @ts-check +import path from 'path'; +import { createRuleset } from './lib/create-file'; +import { parseFelixDnsmasq } from './lib/parse-dnsmasq'; +import { task } from './lib/trace-runner'; -const buildAppleCdn = task(__filename, async () => { +export const buildAppleCdn = task(__filename, async () => { const res = await parseFelixDnsmasq('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf'); const description = [ @@ -42,8 +43,6 @@ const buildAppleCdn = task(__filename, async () => { ]); }); -module.exports.buildAppleCdn = buildAppleCdn; - if (import.meta.main) { buildAppleCdn(); } diff --git a/Build/build-chn-cidr.js b/Build/build-chn-cidr.ts similarity index 69% rename from Build/build-chn-cidr.js rename to Build/build-chn-cidr.ts index 9cc1023f..0ada565e 100644 --- a/Build/build-chn-cidr.js +++ b/Build/build-chn-cidr.ts @@ -1,10 +1,8 @@ -// @ts-check -const { fetchRemoteTextAndCreateReadlineInterface } = require('./lib/fetch-remote-text-by-line'); -const { resolve: pathResolve } = require('path'); -// This should not use `createRuleset` API since we are going to generate ipcidr for Clash -const { compareAndWriteFile, withBannerArray } = require('./lib/create-file'); -const { processLineFromReadline } = require('./lib/process-line'); -const { task } = require('./lib/trace-runner'); +import { fetchRemoteTextAndCreateReadlineInterface } from './lib/fetch-remote-text-by-line'; +import { resolve as pathResolve } from 'path'; +import { compareAndWriteFile, withBannerArray } from './lib/create-file'; +import { processLineFromReadline } from './lib/process-line'; +import { task } from './lib/trace-runner'; // https://github.com/misakaio/chnroutes2/issues/25 const EXCLUDE_CIDRS = [ @@ -12,7 +10,7 @@ const EXCLUDE_CIDRS = [ '223.120.0.0/15' ]; -const buildChnCidr = task(__filename, async () => { +export const buildChnCidr = task(__filename, async () => { const [{ exclude: excludeCidrs }, cidr] = await Promise.all([ import('cidr-tools-wasm'), processLineFromReadline(await fetchRemoteTextAndCreateReadlineInterface('https://raw.githubusercontent.com/misakaio/chnroutes2/master/chnroutes.txt')) @@ -50,8 +48,6 @@ const buildChnCidr = task(__filename, async () => { ]); }); -module.exports.buildChnCidr = buildChnCidr; - if (import.meta.main) { buildChnCidr(); } diff --git a/Build/build-common.js b/Build/build-common.ts similarity index 77% rename from Build/build-common.js rename to Build/build-common.ts index a757789b..6ca627c9 100644 --- a/Build/build-common.js +++ b/Build/build-common.ts @@ -1,12 +1,12 @@ // @ts-check -const path = require('path'); -const { PathScurry } = require('path-scurry'); -const { readFileByLine } = require('./lib/fetch-remote-text-by-line'); -const { processLine } = require('./lib/process-line'); -const { createRuleset } = require('./lib/create-file'); -const { domainDeduper } = require('./lib/domain-deduper'); -const { task } = require('./lib/trace-runner'); +import * as path from 'path'; +import { PathScurry } from 'path-scurry'; +import { readFileByLine } from './lib/fetch-remote-text-by-line'; +import { processLine } from './lib/process-line'; +import { createRuleset } from './lib/create-file'; +import { domainDeduper } from './lib/domain-deduper'; +import { task } from './lib/trace-runner'; const MAGIC_COMMAND_SKIP = '# $ custom_build_script'; const MAGIC_COMMAND_TITLE = '# $ meta_title '; @@ -16,7 +16,7 @@ const sourceDir = path.resolve(__dirname, '../Source'); const outputSurgeDir = path.resolve(__dirname, '../List'); const outputClashDir = path.resolve(__dirname, '../Clash'); -const buildCommon = task(__filename, async () => { +export const buildCommon = task(__filename, async () => { /** @type {Promise[]} */ const promises = []; @@ -45,16 +45,11 @@ const buildCommon = task(__filename, async () => { return Promise.all(promises); }); -module.exports.buildCommon = buildCommon; - if (import.meta.main) { buildCommon(); } -/** - * @param {string} sourcePath - */ -const processFile = async (sourcePath) => { +const processFile = async (sourcePath: string) => { /** @type {string[]} */ const lines = []; @@ -83,14 +78,10 @@ const processFile = async (sourcePath) => { } } - return /** @type {const} */ ([title, descriptions, lines]); + return [title, descriptions, lines] as const; }; -/** - * @param {string} sourcePath - * @param {string} relativePath - */ -async function transformDomainset(sourcePath, relativePath) { +async function transformDomainset(sourcePath: string, relativePath: string) { const res = await processFile(sourcePath); if (!res) return; const [title, descriptions, lines] = res; @@ -120,11 +111,8 @@ async function transformDomainset(sourcePath, relativePath) { /** * Output Surge RULE-SET and Clash classical text format - * - * @param {string} sourcePath - * @param {string} relativePath */ -async function transformRuleset(sourcePath, relativePath) { +async function transformRuleset(sourcePath: string, relativePath: string) { const res = await processFile(sourcePath); if (!res) return; const [title, descriptions, lines] = res; diff --git a/Build/build-domestic-ruleset.js b/Build/build-domestic-ruleset.ts similarity index 73% rename from Build/build-domestic-ruleset.js rename to Build/build-domestic-ruleset.ts index d839a39b..2c51021f 100644 --- a/Build/build-domestic-ruleset.js +++ b/Build/build-domestic-ruleset.ts @@ -1,22 +1,22 @@ // @ts-check -const path = require('path'); -const { DOMESTICS } = require('../Source/non_ip/domestic'); -const { readFileByLine } = require('./lib/fetch-remote-text-by-line'); -const { processLineFromReadline } = require('./lib/process-line'); -const { compareAndWriteFile, createRuleset } = require('./lib/create-file'); -const { task } = require('./lib/trace-runner'); +import path from 'path'; +import { DOMESTICS } from '../Source/non_ip/domestic'; +import { readFileByLine } from './lib/fetch-remote-text-by-line'; +import { processLineFromReadline } from './lib/process-line'; +import { compareAndWriteFile, createRuleset } from './lib/create-file'; +import { task } from './lib/trace-runner'; -const buildDomesticRuleset = task(__filename, async () => { +export const buildDomesticRuleset = task(__filename, async () => { const results = await processLineFromReadline(readFileByLine(path.resolve(__dirname, '../Source/non_ip/domestic.conf'))); results.push( ...Object.entries(DOMESTICS) - .reduce( + .reduce( (acc, [key, { domains }]) => { if (key === 'SYSTEM') return acc; return [...acc, ...domains]; }, - /** @type {string[]} */([]) + [] ) .map((domain) => `DOMAIN-SUFFIX,${domain}`) ); @@ -58,8 +58,6 @@ const buildDomesticRuleset = task(__filename, async () => { ]); }); -module.exports.buildDomesticRuleset = buildDomesticRuleset; - if (import.meta.main) { buildDomesticRuleset(); } diff --git a/Build/build-internal-cdn-rules.js b/Build/build-internal-cdn-rules.ts similarity index 74% rename from Build/build-internal-cdn-rules.js rename to Build/build-internal-cdn-rules.ts index 02d5473e..7438eab9 100644 --- a/Build/build-internal-cdn-rules.js +++ b/Build/build-internal-cdn-rules.ts @@ -1,28 +1,25 @@ // @ts-check -const fsp = require('fs/promises'); -const path = require('path'); -const tldts = require('tldts'); -const { processLine } = require('./lib/process-line'); -const { readFileByLine } = require('./lib/fetch-remote-text-by-line'); -const { createDomainSorter } = require('./lib/stable-sort-domain'); -const { task } = require('./lib/trace-runner'); -const { compareAndWriteFile } = require('./lib/create-file'); -const { getGorhillPublicSuffixPromise } = require('./lib/get-gorhill-publicsuffix'); +import fsp from 'fs/promises' +import path from 'path'; +import * as tldts from 'tldts'; +import { processLine } from './lib/process-line'; +import { readFileByLine } from './lib/fetch-remote-text-by-line'; +import { createDomainSorter } from './lib/stable-sort-domain'; +import { task } from './lib/trace-runner'; +import { compareAndWriteFile } from './lib/create-file'; +import { getGorhillPublicSuffixPromise } from './lib/get-gorhill-publicsuffix'; // const { createCachedGorhillGetDomain } = require('./lib/cached-tld-parse'); const escapeRegExp = (string = '') => string.replaceAll(/[$()*+.?[\\\]^{|}]/g, '\\$&'); -const buildInternalCDNDomains = task(__filename, async () => { - const set = new Set(); +export const buildInternalCDNDomains = task(__filename, async () => { + const set = new Set(); const keywords = new Set(); const gorhill = await getGorhillPublicSuffixPromise(); const domainSorter = createDomainSorter(gorhill); - /** - * @param {string} input - */ - const addApexDomain = (input) => { + const addApexDomain = (input: string) => { // We are including the private domains themselves const d = tldts.getDomain(input, { allowPrivateDomains: false }); if (d) { @@ -30,10 +27,7 @@ const buildInternalCDNDomains = task(__filename, async () => { } }; - /** - * @param {string} domainSetPath - */ - const processLocalDomainSet = async (domainSetPath) => { + const processLocalDomainSet = async (domainSetPath: string) => { for await (const line of readFileByLine(domainSetPath)) { // console.log({ line }); @@ -52,10 +46,7 @@ const buildInternalCDNDomains = task(__filename, async () => { } }; - /** - * @param {string} ruleSetPath - */ - const processLocalRuleSet = async (ruleSetPath) => { + const processLocalRuleSet = async (ruleSetPath: string) => { for await (const line of readFileByLine(ruleSetPath)) { if (line.startsWith('DOMAIN-SUFFIX,')) { addApexDomain(line.replace('DOMAIN-SUFFIX,', '')); @@ -93,8 +84,6 @@ const buildInternalCDNDomains = task(__filename, async () => { ); }); -module.exports.buildInternalCDNDomains = buildInternalCDNDomains; - if (import.meta.main) { buildInternalCDNDomains(); } diff --git a/Build/build-internal-chn-domains.js b/Build/build-internal-chn-domains.ts similarity index 55% rename from Build/build-internal-chn-domains.js rename to Build/build-internal-chn-domains.ts index 710a88a0..da8c55a9 100644 --- a/Build/build-internal-chn-domains.js +++ b/Build/build-internal-chn-domains.ts @@ -1,11 +1,10 @@ -// @ts-check -const path = require('path'); -const fsp = require('fs/promises'); -const { parseFelixDnsmasq } = require('./lib/parse-dnsmasq'); -const { task } = require('./lib/trace-runner'); -const { compareAndWriteFile } = require('./lib/create-file'); +import path from 'path'; +import fsp from 'fs/promises' +import { parseFelixDnsmasq } from './lib/parse-dnsmasq'; +import { task } from './lib/trace-runner'; +import { compareAndWriteFile } from './lib/create-file'; -const buildInternalChnDomains = task(__filename, async () => { +export const buildInternalChnDomains = task(__filename, async () => { const [result] = await Promise.all([ parseFelixDnsmasq('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf'), fsp.mkdir(path.resolve(__dirname, '../List/internal'), { recursive: true }) @@ -17,8 +16,6 @@ const buildInternalChnDomains = task(__filename, async () => { ); }); -module.exports.buildInternalChnDomains = buildInternalChnDomains; - if (import.meta.main) { buildInternalChnDomains(); } diff --git a/Build/build-internal-reverse-chn-cidr.js b/Build/build-internal-reverse-chn-cidr.ts similarity index 70% rename from Build/build-internal-reverse-chn-cidr.js rename to Build/build-internal-reverse-chn-cidr.ts index 57f25de1..d094dc83 100644 --- a/Build/build-internal-reverse-chn-cidr.js +++ b/Build/build-internal-reverse-chn-cidr.ts @@ -1,9 +1,8 @@ -// @ts-check -const { fetchRemoteTextAndCreateReadlineInterface } = require('./lib/fetch-remote-text-by-line'); -const { processLineFromReadline } = require('./lib/process-line'); -const path = require('path'); -const fsp = require('fs/promises'); -const { task } = require('./lib/trace-runner'); +import { fetchRemoteTextAndCreateReadlineInterface } from './lib/fetch-remote-text-by-line'; +import { processLineFromReadline } from './lib/process-line'; +import path from 'path'; +import fsp from 'fs/promises' +import { task } from './lib/trace-runner'; const RESERVED_IPV4_CIDR = [ '0.0.0.0/8', @@ -23,7 +22,7 @@ const RESERVED_IPV4_CIDR = [ '240.0.0.0/4' ]; -const buildInternalReverseChnCIDR = task(__filename, async () => { +export const buildInternalReverseChnCIDR = task(__filename, async () => { const [{ exclude }, cidr] = await Promise.all([ import('cidr-tools-wasm'), processLineFromReadline(await fetchRemoteTextAndCreateReadlineInterface('https://raw.githubusercontent.com/misakaio/chnroutes2/master/chnroutes.txt')), @@ -44,8 +43,6 @@ const buildInternalReverseChnCIDR = task(__filename, async () => { return Bun.write(path.resolve(__dirname, '../List/internal/reversed-chn-cidr.txt'), `${reversedCidr.join('\n')}\n`); }); -module.exports.buildInternalReverseChnCIDR = buildInternalReverseChnCIDR; - if (import.meta.main) { buildInternalReverseChnCIDR(); } diff --git a/Build/build-mitm-hostname.js b/Build/build-mitm-hostname.js index e1563736..eac384b7 100644 --- a/Build/build-mitm-hostname.js +++ b/Build/build-mitm-hostname.js @@ -1,7 +1,7 @@ const fsPromises = require('fs').promises; const pathFn = require('path'); const table = require('table'); -const listDir = require('@sukka/listdir'); +import listDir from '@sukka/listdir'; const { green, yellow } = require('picocolors'); const PRESET_MITM_HOSTNAMES = [ diff --git a/Build/build-phishing-domainset.js b/Build/build-phishing-domainset.ts similarity index 83% rename from Build/build-phishing-domainset.js rename to Build/build-phishing-domainset.ts index 0f7e8e18..862b9023 100644 --- a/Build/build-phishing-domainset.js +++ b/Build/build-phishing-domainset.ts @@ -1,14 +1,13 @@ -// @ts-check -const { processFilterRules } = require('./lib/parse-filter.js'); -const path = require('path'); -const { createRuleset } = require('./lib/create-file'); -const { processLine } = require('./lib/process-line.js'); -const { createDomainSorter } = require('./lib/stable-sort-domain'); -const { traceSync, task } = require('./lib/trace-runner.js'); -const createTrie = require('./lib/trie.js'); -const { getGorhillPublicSuffixPromise } = require('./lib/get-gorhill-publicsuffix.js'); -const { createCachedGorhillGetDomain } = require('./lib/cached-tld-parse.js'); -const tldts = require('tldts'); +import { processFilterRules } from './lib/parse-filter'; +import path from 'path'; +import { createRuleset } from './lib/create-file'; +import { processLine } from './lib/process-line'; +import { createDomainSorter } from './lib/stable-sort-domain'; +import { traceSync, task } from './lib/trace-runner'; +import createTrie from './lib/trie'; +import { getGorhillPublicSuffixPromise } from './lib/get-gorhill-publicsuffix'; +import { createCachedGorhillGetDomain } from './lib/cached-tld-parse'; +import * as tldts from 'tldts'; const WHITELIST_DOMAIN = new Set([ 'w3s.link', @@ -64,7 +63,7 @@ const BLACK_TLD = new Set([ 'com.cn' ]); -const buildPhishingDomainSet = task(__filename, async () => { +export const buildPhishingDomainSet = task(__filename, async () => { const [{ black: domainSet }, gorhill] = await Promise.all([ processFilterRules( 'https://curbengh.github.io/phishing-filter/phishing-filter-agh.txt', @@ -87,7 +86,7 @@ const buildPhishingDomainSet = task(__filename, async () => { }); }); - const domainCountMap = {}; + const domainCountMap: Record = {}; const getDomain = createCachedGorhillGetDomain(gorhill); traceSync('* process domain set', () => { @@ -154,12 +153,12 @@ const buildPhishingDomainSet = task(__filename, async () => { const domainSorter = createDomainSorter(gorhill); const results = traceSync('* get final results', () => Object.entries(domainCountMap) - .reduce((acc, [apexDomain, count]) => { + .reduce((acc, [apexDomain, count]) => { if (count >= 5) { acc.push(`.${apexDomain}`); } return acc; - }, /** @type {string[]} */([])) + }, []) .sort(domainSorter)); const description = [ @@ -183,8 +182,6 @@ const buildPhishingDomainSet = task(__filename, async () => { )); }); -module.exports.buildPhishingDomainSet = buildPhishingDomainSet; - if (import.meta.main) { buildPhishingDomainSet(); } diff --git a/Build/build-public.js b/Build/build-public.ts similarity index 88% rename from Build/build-public.js rename to Build/build-public.ts index ff38a5b7..07751707 100644 --- a/Build/build-public.js +++ b/Build/build-public.ts @@ -1,7 +1,7 @@ -const listDir = require('@sukka/listdir'); -const path = require('path'); -const fsp = require('fs/promises'); -const { task } = require('./lib/trace-runner'); +import listDir from '@sukka/listdir'; +import path from 'path'; +import fsp from 'fs/promises' +import { task } from './lib/trace-runner'; const rootPath = path.resolve(__dirname, '../'); const publicPath = path.resolve(__dirname, '../public'); @@ -15,7 +15,7 @@ const folderAndFilesToBeDeployed = [ 'LICENSE' ]; -const buildPublicHtml = task(__filename, async () => { +export const buildPublicHtml = task(__filename, async () => { await fsp.mkdir(publicPath, { recursive: true }); await Promise.all(folderAndFilesToBeDeployed.map(dir => fsp.cp( path.resolve(rootPath, dir), @@ -33,17 +33,11 @@ const buildPublicHtml = task(__filename, async () => { return Bun.write(path.join(publicPath, 'index.html'), html); }); -module.exports.buildPublicHtml = buildPublicHtml; - if (import.meta.main) { buildPublicHtml(); } -/** - * @param {string[]} urlList - * @returns {string} - */ -function template(urlList) { +function template(urlList: string[]) { return ` diff --git a/Build/build-redirect-module.js b/Build/build-redirect-module.ts similarity index 92% rename from Build/build-redirect-module.js rename to Build/build-redirect-module.ts index 19eef59d..c9f1cd3f 100644 --- a/Build/build-redirect-module.js +++ b/Build/build-redirect-module.ts @@ -1,9 +1,7 @@ -// @ts-check - -const path = require('path'); -const { task } = require('./lib/trace-runner'); -const { compareAndWriteFile } = require('./lib/create-file'); -const tldts = require('tldts'); +import path from 'path'; +import { task } from './lib/trace-runner'; +import { compareAndWriteFile } from './lib/create-file'; +import * as tldts from 'tldts'; function escapeRegExp(string = '') { const reRegExpChar = /[$()*+.?[\\\]^{|}]/g; @@ -73,7 +71,7 @@ const REDIRECT = /** @type {const} */ ([ ['googleajax.wp-china-yes.net/', 'https://ajax.googleapis.com/'] ]); -const buildRedirectModule = task(__filename, async () => { +export const buildRedirectModule = task(__filename, async () => { const domains = Array.from(new Set(REDIRECT.map(([from]) => tldts.getHostname(from, { detectIp: false })))).filter(Boolean); return compareAndWriteFile( @@ -94,8 +92,6 @@ const buildRedirectModule = task(__filename, async () => { ); }); -module.exports.buildRedirectModule = buildRedirectModule; - if (import.meta.main) { buildRedirectModule(); } diff --git a/Build/build-reject-domainset.js b/Build/build-reject-domainset.ts similarity index 85% rename from Build/build-reject-domainset.js rename to Build/build-reject-domainset.ts index a1ce3a51..c9517996 100644 --- a/Build/build-reject-domainset.js +++ b/Build/build-reject-domainset.ts @@ -1,31 +1,31 @@ // @ts-check -const fsp = require('fs/promises'); -const path = require('path'); +import fsp from 'fs/promises' +import path from 'path'; -const { processHosts, processFilterRules } = require('./lib/parse-filter'); -const createTrie = require('./lib/trie'); +import { processHosts, processFilterRules } from './lib/parse-filter'; +import createTrie from './lib/trie'; -const { HOSTS, ADGUARD_FILTERS, PREDEFINED_WHITELIST, PREDEFINED_ENFORCED_BACKLIST } = require('./lib/reject-data-source'); -const { createRuleset, compareAndWriteFile } = require('./lib/create-file'); -const { processLine } = require('./lib/process-line'); -const { domainDeduper } = require('./lib/domain-deduper'); -const createKeywordFilter = require('./lib/aho-corasick'); -const { readFileByLine } = require('./lib/fetch-remote-text-by-line'); -const { createDomainSorter } = require('./lib/stable-sort-domain'); -const { traceSync, task } = require('./lib/trace-runner'); -const { getGorhillPublicSuffixPromise } = require('./lib/get-gorhill-publicsuffix'); -const tldts = require('tldts'); +import { HOSTS, ADGUARD_FILTERS, PREDEFINED_WHITELIST, PREDEFINED_ENFORCED_BACKLIST } from './lib/reject-data-source'; +import { createRuleset, compareAndWriteFile } from './lib/create-file'; +import { processLine } from './lib/process-line'; +import { domainDeduper } from './lib/domain-deduper'; +import createKeywordFilter from './lib/aho-corasick'; +import { readFileByLine } from './lib/fetch-remote-text-by-line'; +import { createDomainSorter } from './lib/stable-sort-domain'; +import { traceSync, task } from './lib/trace-runner'; +import { getGorhillPublicSuffixPromise } from './lib/get-gorhill-publicsuffix'; +import * as tldts from 'tldts'; /** Whitelists */ const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST); /** @type {Set} Dedupe domains inclued by DOMAIN-KEYWORD */ -const domainKeywordsSet = new Set(); +const domainKeywordsSet: Set = new Set(); /** @type {Set} Dedupe domains included by DOMAIN-SUFFIX */ -const domainSuffixSet = new Set(); +const domainSuffixSet: Set = new Set(); -const buildRejectDomainSet = task(__filename, async () => { +export const buildRejectDomainSet = task(__filename, async () => { /** @type Set */ - const domainSets = new Set(); + const domainSets: Set = new Set(); // Parse from AdGuard Filters console.time('* Download and process Hosts / AdBlock Filter Rules'); @@ -172,11 +172,10 @@ const buildRejectDomainSet = task(__filename, async () => { console.log(`Deduped ${previousSize - dudupedDominArray.length} rules!`); // Create reject stats - /** @type {[string, number][]} */ - const rejectDomainsStats = traceSync( + const rejectDomainsStats: [string, number][] = traceSync( '* Collect reject domain stats', () => Object.entries( - dudupedDominArray.reduce((acc, cur) => { + dudupedDominArray.reduce>((acc, cur) => { const suffix = tldts.getDomain(cur, { allowPrivateDomains: false, detectIp: false }); if (suffix) { acc[suffix] = (acc[suffix] ?? 0) + 1; @@ -231,8 +230,6 @@ const buildRejectDomainSet = task(__filename, async () => { ]); }); -module.exports.buildRejectDomainSet = buildRejectDomainSet; - if (import.meta.main) { buildRejectDomainSet(); } diff --git a/Build/build-speedtest-domainset.js b/Build/build-speedtest-domainset.ts similarity index 82% rename from Build/build-speedtest-domainset.js rename to Build/build-speedtest-domainset.ts index ae8fb282..9298f99d 100644 --- a/Build/build-speedtest-domainset.js +++ b/Build/build-speedtest-domainset.ts @@ -1,18 +1,14 @@ -const { domainDeduper } = require('./lib/domain-deduper'); -const path = require('path'); -const { createRuleset } = require('./lib/create-file'); -const domainSorter = require('./lib/stable-sort-domain'); +import { domainDeduper } from './lib/domain-deduper'; +import path from 'path'; +import { createRuleset } from './lib/create-file'; +import domainSorter from './lib/stable-sort-domain'; -const { Sema } = require('async-sema'); -const tldts = require('tldts'); -const { task } = require('./lib/trace-runner'); +import { Sema } from 'async-sema'; +import * as tldts from 'tldts'; +import { task } from './lib/trace-runner'; const s = new Sema(2); -/** - * @param {string} keyword - * @returns {string[]} - */ -const querySpeedtestApi = async (keyword) => { +const querySpeedtestApi = async (keyword: string): Promise<(string | null)[]> => { await s.acquire(); try { @@ -36,19 +32,19 @@ const querySpeedtestApi = async (keyword) => { throw new Error(text); } - /** @type {{ url: string }[]} */ - const json = await res.json(); + const json = await res.json() as { url: string; }[]; s.release(); return json.map(({ url }) => tldts.getHostname(url, { detectIp: false })); } catch (e) { s.release(); console.log(e); + return []; } }; -const buildSpeedtestDomainSet = task(__filename, async () => { +export const buildSpeedtestDomainSet = task(__filename, async () => { /** @type {Set} */ - const domains = new Set([ + const domains: Set = new Set([ '.speedtest.net', '.speedtestcustom.com', '.ooklaserver.net', @@ -140,8 +136,6 @@ const buildSpeedtestDomainSet = task(__filename, async () => { )); }); -module.exports.buildSpeedtestDomainSet = buildSpeedtestDomainSet; - if (import.meta.main) { buildSpeedtestDomainSet(); } diff --git a/Build/build-stream-service.js b/Build/build-stream-service.ts similarity index 65% rename from Build/build-stream-service.js rename to Build/build-stream-service.ts index 51377342..d9d51d80 100644 --- a/Build/build-stream-service.js +++ b/Build/build-stream-service.ts @@ -1,20 +1,12 @@ // @ts-check -const { task } = require('./lib/trace-runner'); +import { task } from './lib/trace-runner'; -const path = require('path'); -const { createRuleset } = require('./lib/create-file'); +import path from 'path'; +import { createRuleset } from './lib/create-file'; -const { - ALL, NORTH_AMERICA, EU, HK, TW, JP, KR - // SOUTH_EAST_ASIA, AU -} = require('../Source/stream'); +import { ALL, NORTH_AMERICA, EU, HK, TW, JP, KR } from '../Source/stream'; -/** - * @param {string} fileId - * @param {string} title - * @param {import('../Source/stream').StreamService[]} streamServices - */ -const createRulesetForStreamService = (fileId, title, streamServices) => { +const createRulesetForStreamService = (fileId: string, title: string, streamServices: import('../Source/stream').StreamService[]) => { return [ // Domains ...createRuleset( @@ -24,10 +16,10 @@ const createRulesetForStreamService = (fileId, title, streamServices) => { 'Homepage: https://ruleset.skk.moe', 'GitHub: https://github.com/SukkaW/Surge', '', - ...streamServices.map(i => `- ${i.name}`) + ...streamServices.map((i: { name: any; }) => `- ${i.name}`) ], new Date(), - streamServices.flatMap(i => i.rules), + streamServices.flatMap((i: { rules: any; }) => i.rules), 'ruleset', path.resolve(__dirname, `../List/non_ip/${fileId}.conf`), path.resolve(__dirname, `../Clash/non_ip/${fileId}.txt`) @@ -40,14 +32,14 @@ const createRulesetForStreamService = (fileId, title, streamServices) => { 'Homepage: https://ruleset.skk.moe', 'GitHub: https://github.com/SukkaW/Surge', '', - ...streamServices.map(i => `- ${i.name}`) + ...streamServices.map((i: { name: any; }) => `- ${i.name}`) ], new Date(), - streamServices.flatMap(i => ( + streamServices.flatMap((i) => ( i.ip ? [ - ...i.ip.v4.map(ip => `IP-CIDR,${ip},no-resolve`), - ...i.ip.v6.map(ip => `IP-CIDR6,${ip},no-resolve`) + ...i.ip.v4.map((ip: any) => `IP-CIDR,${ip},no-resolve`), + ...i.ip.v6.map((ip: any) => `IP-CIDR6,${ip},no-resolve`) ] : [] )), @@ -58,7 +50,7 @@ const createRulesetForStreamService = (fileId, title, streamServices) => { ]; }; -const buildStreamService = task(__filename, async () => { +export const buildStreamService = task(__filename, async () => { return Promise.all([ ...createRulesetForStreamService('stream', 'All', ALL), ...createRulesetForStreamService('stream_us', 'North America', NORTH_AMERICA), @@ -72,8 +64,6 @@ const buildStreamService = task(__filename, async () => { ]); }); -module.exports.buildStreamService = buildStreamService; - if (import.meta.main) { buildStreamService(); } diff --git a/Build/build-telegram-cidr.js b/Build/build-telegram-cidr.ts similarity index 71% rename from Build/build-telegram-cidr.js rename to Build/build-telegram-cidr.ts index 769f1077..191084e8 100644 --- a/Build/build-telegram-cidr.js +++ b/Build/build-telegram-cidr.ts @@ -1,13 +1,13 @@ // @ts-check -const { fetchWithRetry } = require('./lib/fetch-retry'); -const { createReadlineInterfaceFromResponse } = require('./lib/fetch-remote-text-by-line'); -const path = require('path'); -const { isIPv4, isIPv6 } = require('net'); -const { processLine } = require('./lib/process-line'); -const { createRuleset } = require('./lib/create-file'); -const { task } = require('./lib/trace-runner'); +import { fetchWithRetry } from './lib/fetch-retry'; +import { createReadlineInterfaceFromResponse } from './lib/fetch-remote-text-by-line'; +import path from 'path'; +import { isIPv4, isIPv6 } from 'net'; +import { processLine } from './lib/process-line'; +import { createRuleset } from './lib/create-file'; +import { task } from './lib/trace-runner'; -const buildTelegramCIDR = task(__filename, async () => { +export const buildTelegramCIDR = task(__filename, async () => { /** @type {Response} */ const resp = await fetchWithRetry('https://core.telegram.org/resources/cidr.txt'); const lastModified = resp.headers.get('last-modified'); @@ -52,8 +52,6 @@ const buildTelegramCIDR = task(__filename, async () => { )); }); -module.exports.buildTelegramCIDR = buildTelegramCIDR; - if (import.meta.main) { buildTelegramCIDR(); } diff --git a/Build/lib/aho-corasick.js b/Build/lib/aho-corasick.ts similarity index 69% rename from Build/lib/aho-corasick.js rename to Build/lib/aho-corasick.ts index b527ffc2..585cb3fb 100644 --- a/Build/lib/aho-corasick.js +++ b/Build/lib/aho-corasick.ts @@ -1,19 +1,15 @@ -/** - * @typedef {Object} Node - * @prop {number} [depth = 0] - * @prop {string} key - * @prop {boolean} [word = false] - * @prop {Record} [children={}] - * @prop {Node} [fail] - * @prop {number} [count=0] - */ +interface Node { + /** @default 0 */ + depth?: number; + key: string; + /** @default false */ + word?: boolean; + children: Record; + fail?: Node; + count: number; +} -/** - * @param {string} key - * @param {number} depth - * @returns {Node} - */ -const createNode = (key, depth = 0) => ({ +const createNode = (key: string, depth = 0): Node => ({ depth, key, word: false, @@ -22,15 +18,11 @@ const createNode = (key, depth = 0) => ({ count: 0 }); -/** - * @param {string[] | Set} keys - */ -const createKeywordFilter = (keys) => { +const createKeywordFilter = (keys: string[] | Set) => { const root = createNode('root'); const build = () => { - /** @type {Node[]} */ - const queue = []; + const queue: Node[] = []; queue.push(root); let idx = 0; @@ -57,11 +49,7 @@ const createKeywordFilter = (keys) => { } }; - /** - * @param {string} key - * @param {number} len - */ - const put = (key, len) => { + const put = (key: string, len: number) => { let node = root; const lastIdx = len - 1; node.count++; @@ -91,12 +79,8 @@ const createKeywordFilter = (keys) => { build(); - /** - * @param {string} text - * @returns {boolean} - */ - const search = (text) => { - let node = root; + const search = (text: string) => { + let node: Node | undefined = root; for (let i = 0, textLen = text.length; i < textLen; i++) { // const key = text.charAt(i); @@ -120,4 +104,4 @@ const createKeywordFilter = (keys) => { }; }; -module.exports = createKeywordFilter; +export default createKeywordFilter; diff --git a/Build/lib/cache-apply.js b/Build/lib/cache-apply.ts similarity index 56% rename from Build/lib/cache-apply.js rename to Build/lib/cache-apply.ts index 012f02a7..fe63cec4 100644 --- a/Build/lib/cache-apply.js +++ b/Build/lib/cache-apply.ts @@ -1,7 +1,4 @@ -/** - * @param {string} [namespace] - */ -const createCache = (namespace, printStats = false) => { +export const createCache = (namespace?: string, printStats = false) => { const cache = new Map(); let hit = 0; @@ -12,13 +9,7 @@ const createCache = (namespace, printStats = false) => { } return { - /** - * @template T - * @param {string} key - * @param {() => T} fn - * @returns {T} - */ - sync(key, fn) { + sync(key: string, fn: () => T): T { if (cache.has(key)) { hit++; return cache.get(key); @@ -27,13 +18,7 @@ const createCache = (namespace, printStats = false) => { cache.set(key, value); return value; }, - /** - * @template T - * @param {string} key - * @param {() => Promise} fn - * @returns {Promise} - */ - async async(key, fn) { + async async(key: string, fn: () => Promise): Promise { if (cache.has(key)) { hit++; return cache.get(key); @@ -44,4 +29,3 @@ const createCache = (namespace, printStats = false) => { } }; }; -module.exports.createCache = createCache; diff --git a/Build/lib/cached-tld-parse.js b/Build/lib/cached-tld-parse.js deleted file mode 100644 index 9630bab4..00000000 --- a/Build/lib/cached-tld-parse.js +++ /dev/null @@ -1,24 +0,0 @@ -const tldts = require('tldts'); -const { createCache } = require('./cache-apply'); - -const cache = createCache('cached-tld-parse', true); - -const sharedConfig = { allowPrivateDomains: true }; - -/** - * @param {string} domain - * @returns {ReturnType} - */ -module.exports.parse = (domain) => cache.sync(domain, () => tldts.parse(domain, sharedConfig)); - -let gothillGetDomainCache = null; -/** - * @param {import('gorhill-publicsuffixlist').default | null} gorhill - */ -module.exports.createCachedGorhillGetDomain = (gorhill) => { - gothillGetDomainCache ||= createCache('cached-gorhill-get-domain', true); - /** - * @param {string} domain - */ - return (domain) => (/** @type {ReturnType} */ (gothillGetDomainCache)).sync(domain, () => gorhill.getDomain(domain[0] === '.' ? domain.slice(1) : domain)); -}; diff --git a/Build/lib/cached-tld-parse.ts b/Build/lib/cached-tld-parse.ts new file mode 100644 index 00000000..01d75ffd --- /dev/null +++ b/Build/lib/cached-tld-parse.ts @@ -0,0 +1,17 @@ +import tldts from 'tldts'; +import { createCache } from './cache-apply'; +import { PublicSuffixList } from 'gorhill-publicsuffixlist'; + +const cache = createCache('cached-tld-parse', true); + +const sharedConfig = { allowPrivateDomains: true }; + +export const parse = (domain: string) => cache.sync(domain, () => tldts.parse(domain, sharedConfig)); + +let gothillGetDomainCache: ReturnType | null = null; +export const createCachedGorhillGetDomain = (gorhill: PublicSuffixList) => { + return (domain: string) => { + gothillGetDomainCache ??= createCache('cached-gorhill-get-domain', true); + return gothillGetDomainCache.sync(domain, () => gorhill.getDomain(domain[0] === '.' ? domain.slice(1) : domain)) + }; +}; diff --git a/Build/lib/clash.js b/Build/lib/clash.ts similarity index 50% rename from Build/lib/clash.js rename to Build/lib/clash.ts index e9353fb9..7995a362 100644 --- a/Build/lib/clash.js +++ b/Build/lib/clash.ts @@ -1,6 +1,5 @@ // @ts-check -const _Trie = require('mnemonist/trie'); -const Trie = _Trie.default || _Trie; +import Trie from 'mnemonist/trie'; // https://dreamacro.github.io/clash/configuration/rules.html const CLASH_SUPPORTED_RULE_TYPE = [ @@ -17,21 +16,13 @@ const CLASH_SUPPORTED_RULE_TYPE = [ 'PROCESS-PATH' ]; -/** - * @param {string[] | Set} rules - */ -const surgeRulesetToClashClassicalTextRuleset = (rules) => { +export const surgeRulesetToClashClassicalTextRuleset = (rules: string[] | Set) => { const trie = Trie.from(rules); return CLASH_SUPPORTED_RULE_TYPE.flatMap( type => trie.find(`${type},`) ); }; -module.exports.surgeRulesetToClashClassicalTextRuleset = surgeRulesetToClashClassicalTextRuleset; -/** - * @param {string[]} domainset - */ -const surgeDomainsetToClashDomainset = (domainset) => { +export const surgeDomainsetToClashDomainset = (domainset: string[]) => { return domainset.map(i => (i[0] === '.' ? `+${i}` : i)); }; -module.exports.surgeDomainsetToClashDomainset = surgeDomainsetToClashDomainset; diff --git a/Build/lib/create-file.js b/Build/lib/create-file.ts similarity index 67% rename from Build/lib/create-file.js rename to Build/lib/create-file.ts index ffc0e53e..7be6f7b7 100644 --- a/Build/lib/create-file.js +++ b/Build/lib/create-file.ts @@ -1,12 +1,8 @@ // @ts-check -const { readFileByLine } = require('./fetch-remote-text-by-line'); -const { surgeDomainsetToClashDomainset, surgeRulesetToClashClassicalTextRuleset } = require('./clash'); +import { readFileByLine } from './fetch-remote-text-by-line'; +import { surgeDomainsetToClashDomainset, surgeRulesetToClashClassicalTextRuleset } from './clash'; -/** - * @param {string[]} linesA - * @param {string} filePath - */ -async function compareAndWriteFile(linesA, filePath) { +export async function compareAndWriteFile(linesA: string[], filePath: string) { let isEqual = true; const file = Bun.file(filePath); @@ -57,16 +53,8 @@ async function compareAndWriteFile(linesA, filePath) { console.log(`Same Content, bail out writing: ${filePath}`); } -module.exports.compareAndWriteFile = compareAndWriteFile; -/** - * @param {string} title - * @param {string[]} description - * @param {Date} date - * @param {string[]} content - * @returns {string[]} - */ -const withBannerArray = (title, description, date, content) => { +export const withBannerArray = (title: string, description: string[], date: Date, content: string[]) => { return [ '########################################', `# ${title}`, @@ -78,20 +66,10 @@ const withBannerArray = (title, description, date, content) => { '################# END ###################' ]; }; -module.exports.withBannerArray = withBannerArray; -/** - * @param {string} title - * @param {string[]} description - * @param {Date} date - * @param {string[]} content - * @param {'ruleset' | 'domainset'} type - * @param {string} surgePath - * @param {string} clashPath - */ -const createRuleset = ( - title, description, date, content, - type, surgePath, clashPath +export const createRuleset = ( + title: string, description: string[], date: Date, content: string[], + type: 'ruleset' | 'domainset', surgePath: string, clashPath: string ) => { const surgeContent = withBannerArray(title, description, date, content); @@ -114,4 +92,3 @@ const createRuleset = ( compareAndWriteFile(clashContent, clashPath) ]; }; -module.exports.createRuleset = createRuleset; diff --git a/Build/lib/domain-deduper.ts b/Build/lib/domain-deduper.ts index 75545035..875853f5 100644 --- a/Build/lib/domain-deduper.ts +++ b/Build/lib/domain-deduper.ts @@ -1,6 +1,6 @@ import createTrie from './trie'; -const domainDeduper = (inputDomains: string[]): string[] => { +export const domainDeduper = (inputDomains: string[]): string[] => { const trie = createTrie(inputDomains); const sets = new Set(inputDomains); diff --git a/Build/lib/fetch-retry.js b/Build/lib/fetch-retry.js deleted file mode 100644 index 4ee7b99b..00000000 --- a/Build/lib/fetch-retry.js +++ /dev/null @@ -1,3 +0,0 @@ -// @ts-check -const fetchWithRetry = require('@vercel/fetch-retry')(fetch); -module.exports.fetchWithRetry = fetchWithRetry; diff --git a/Build/lib/fetch-retry.ts b/Build/lib/fetch-retry.ts new file mode 100644 index 00000000..1bba35ea --- /dev/null +++ b/Build/lib/fetch-retry.ts @@ -0,0 +1,4 @@ +// @ts-expect-error -- missing types +import createFetchRetry from '@vercel/fetch-retry'; + +export const fetchWithRetry: typeof fetch = createFetchRetry(fetch); diff --git a/Build/lib/get-gorhill-publicsuffix.js b/Build/lib/get-gorhill-publicsuffix.ts similarity index 70% rename from Build/lib/get-gorhill-publicsuffix.js rename to Build/lib/get-gorhill-publicsuffix.ts index 517bc7f5..fa0058ce 100644 --- a/Build/lib/get-gorhill-publicsuffix.js +++ b/Build/lib/get-gorhill-publicsuffix.ts @@ -1,13 +1,12 @@ -const { toASCII } = require('punycode/'); -const path = require('path'); -const { traceAsync } = require('./trace-runner'); +import { toASCII } from 'punycode'; +import path from 'path'; +import { traceAsync } from './trace-runner'; +import type { PublicSuffixList } from 'gorhill-publicsuffixlist'; const publicSuffixPath = path.resolve(__dirname, '../../node_modules/.cache/public_suffix_list_dat.txt'); const getGorhillPublicSuffix = () => traceAsync('create gorhill public suffix instance', async () => { - const customFetch = async (url) => { - return Bun.file(url); - }; + const customFetch = async (url: string | URL) => Bun.file(url); const publicSuffixFile = Bun.file(publicSuffixPath); @@ -27,9 +26,8 @@ const getGorhillPublicSuffix = () => traceAsync('create gorhill public suffix in return gorhill; }); -/** @type {Promise | null} */ -let gorhillPublicSuffixPromise = null; -module.exports.getGorhillPublicSuffixPromise = () => { +let gorhillPublicSuffixPromise: Promise | null = null; +export const getGorhillPublicSuffixPromise = () => { gorhillPublicSuffixPromise ||= getGorhillPublicSuffix(); return gorhillPublicSuffixPromise; }; diff --git a/Build/lib/parse-dnsmasq.ts b/Build/lib/parse-dnsmasq.ts index d8c3a8d0..eb4b764c 100644 --- a/Build/lib/parse-dnsmasq.ts +++ b/Build/lib/parse-dnsmasq.ts @@ -6,7 +6,7 @@ const isDomainLoose = (domain: string): boolean => { return !!(!isIp && (isIcann || isPrivate)); }; -const parseFelixDnsmasq = async (url: string | URL): Promise => { +export const parseFelixDnsmasq = async (url: string | URL): Promise => { const res: string[] = []; for await (const line of await fetchRemoteTextAndCreateReadlineInterface(url)) { if (line.startsWith('server=/') && line.endsWith('/114.114.114.114')) { @@ -19,5 +19,3 @@ const parseFelixDnsmasq = async (url: string | URL): Promise => { return res; }; - -export { parseFelixDnsmasq }; diff --git a/Build/lib/parse-filter.js b/Build/lib/parse-filter.ts similarity index 88% rename from Build/lib/parse-filter.js rename to Build/lib/parse-filter.ts index aa77c11c..89550c73 100644 --- a/Build/lib/parse-filter.js +++ b/Build/lib/parse-filter.ts @@ -1,24 +1,18 @@ // @ts-check -const { fetchWithRetry } = require('./fetch-retry'); -const tldts = require('./cached-tld-parse'); -const { fetchRemoteTextAndCreateReadlineInterface } = require('./fetch-remote-text-by-line'); -const { NetworkFilter } = require('@cliqz/adblocker'); -const { processLine } = require('./process-line'); -const { performance } = require('perf_hooks'); -const { getGorhillPublicSuffixPromise } = require('./get-gorhill-publicsuffix'); +import { fetchWithRetry } from './fetch-retry'; +import * as tldts from './cached-tld-parse'; +import { fetchRemoteTextAndCreateReadlineInterface } from './fetch-remote-text-by-line'; +import { NetworkFilter } from '@cliqz/adblocker'; +import { processLine } from './process-line'; +import { performance } from 'perf_hooks'; +import { getGorhillPublicSuffixPromise } from './get-gorhill-publicsuffix'; +import type { PublicSuffixList } from 'gorhill-publicsuffixlist'; const DEBUG_DOMAIN_TO_FIND = null; // example.com | null let foundDebugDomain = false; -const warnOnceUrl = new Set(); -/** - * - * @param {string} url - * @param {boolean} isWhite - * @param {...any} message - * @returns - */ -const warnOnce = (url, isWhite, ...message) => { +const warnOnceUrl = new Set(); +const warnOnce = (url: string, isWhite: boolean, ...message: any[]) => { const key = `${url}${isWhite ? 'white' : 'black'}`; if (warnOnceUrl.has(key)) { return; @@ -27,10 +21,7 @@ const warnOnce = (url, isWhite, ...message) => { console.warn(url, isWhite ? '(white)' : '(black)', ...message); }; -/** - * @param {string} domain - */ -const normalizeDomain = (domain) => { +const normalizeDomain = (domain: string) => { if (!domain) return null; const parsed = tldts.parse(domain); @@ -47,10 +38,7 @@ const normalizeDomain = (domain) => { return null; }; -/** - * @param {string | URL} domainListsUrl - */ -async function processDomainLists(domainListsUrl) { +export async function processDomainLists(domainListsUrl: string | URL) { if (typeof domainListsUrl === 'string') { domainListsUrl = new URL(domainListsUrl); } @@ -79,18 +67,14 @@ async function processDomainLists(domainListsUrl) { return domainSets; } -/** - * @param {string | URL} hostsUrl - */ -async function processHosts(hostsUrl, includeAllSubDomain = false) { +export async function processHosts(hostsUrl: string | URL, includeAllSubDomain = false) { console.time(` - processHosts: ${hostsUrl}`); if (typeof hostsUrl === 'string') { hostsUrl = new URL(hostsUrl); } - /** @type Set */ - const domainSets = new Set(); + const domainSets = new Set(); for await (const l of await fetchRemoteTextAndCreateReadlineInterface(hostsUrl)) { const line = processLine(l); @@ -121,24 +105,20 @@ async function processHosts(hostsUrl, includeAllSubDomain = false) { return domainSets; } -/** - * @param {string | URL} filterRulesUrl - * @param {readonly (string | URL)[] | undefined} [fallbackUrls] - * @returns {Promise<{ white: Set, black: Set, foundDebugDomain: boolean }>} - */ -async function processFilterRules(filterRulesUrl, fallbackUrls) { +export async function processFilterRules( + filterRulesUrl: string | URL, + fallbackUrls?: readonly (string | URL)[] | undefined +): Promise<{ white: Set, black: Set, foundDebugDomain: boolean }> { const runStart = performance.now(); - /** @type Set */ - const whitelistDomainSets = new Set(); - /** @type Set */ - const blacklistDomainSets = new Set(); + const whitelistDomainSets = new Set(); + const blacklistDomainSets = new Set(); /** * @param {string} domainToBeAddedToBlack * @param {boolean} isSubDomain */ - const addToBlackList = (domainToBeAddedToBlack, isSubDomain) => { + const addToBlackList = (domainToBeAddedToBlack: string, isSubDomain: boolean) => { if (isSubDomain && domainToBeAddedToBlack[0] !== '.') { blacklistDomainSets.add(`.${domainToBeAddedToBlack}`); } else { @@ -149,7 +129,7 @@ async function processFilterRules(filterRulesUrl, fallbackUrls) { * @param {string} domainToBeAddedToWhite * @param {boolean} [isSubDomain] */ - const addToWhiteList = (domainToBeAddedToWhite, isSubDomain = true) => { + const addToWhiteList = (domainToBeAddedToWhite: string, isSubDomain = true) => { if (isSubDomain && domainToBeAddedToWhite[0] !== '.') { whitelistDomainSets.add(`.${domainToBeAddedToWhite}`); } else { @@ -163,7 +143,7 @@ async function processFilterRules(filterRulesUrl, fallbackUrls) { /** * @param {string} line */ - const lineCb = (line) => { + const lineCb = (line: string) => { const result = parse(line, gorhill); if (result) { const flag = result[1]; @@ -250,12 +230,7 @@ async function processFilterRules(filterRulesUrl, fallbackUrls) { const R_KNOWN_NOT_NETWORK_FILTER_PATTERN = /[#%&=~]/; const R_KNOWN_NOT_NETWORK_FILTER_PATTERN_2 = /(\$popup|\$removeparam|\$popunder)/; -/** - * @param {string} $line - * @param {import('gorhill-publicsuffixlist').default} gorhill - * @returns {null | [hostname: string, flag: 0 | 1 | 2 | -1]} - 0 white include subdomain, 1 black abosulte, 2 black include subdomain, -1 white - */ -function parse($line, gorhill) { +function parse($line: string, gorhill: PublicSuffixList): null | [hostname: string, flag: 0 | 1 | 2 | -1] { if ( // doesn't include !$line.includes('.') // rule with out dot can not be a domain @@ -615,7 +590,3 @@ function parse($line, gorhill) { return null; } - -module.exports.processDomainLists = processDomainLists; -module.exports.processHosts = processHosts; -module.exports.processFilterRules = processFilterRules; diff --git a/Build/lib/stable-sort-domain.js b/Build/lib/stable-sort-domain.ts similarity index 58% rename from Build/lib/stable-sort-domain.js rename to Build/lib/stable-sort-domain.ts index cb3e4746..354895d2 100644 --- a/Build/lib/stable-sort-domain.js +++ b/Build/lib/stable-sort-domain.ts @@ -1,10 +1,7 @@ -// @ts-check -/** - * @param {string | null} a - * @param {string | null} b - * @returns {0 | 1 | -1} - */ -const compare = (a, b) => { +import type { PublicSuffixList } from 'gorhill-publicsuffixlist'; +import { createCachedGorhillGetDomain } from './cached-tld-parse'; + +const compare = (a: string | null, b: string | null) => { if (a === b) return 0; if (b == null) { return 1; @@ -38,22 +35,11 @@ const compare = (a, b) => { return 0; }; -/** - * @param {import('gorhill-publicsuffixlist').default | null} [gorhill] - */ -const createDomainSorter = (gorhill = null) => { +const createDomainSorter = (gorhill: PublicSuffixList | null = null) => { if (gorhill) { - /** - * @param {string} input - */ - const getDomain = require('./cached-tld-parse').createCachedGorhillGetDomain(gorhill); + const getDomain = createCachedGorhillGetDomain(gorhill); - /** - * @param {string} a - * @param {string} b - * @returns {0 | 1 | -1} - */ - return (a, b) => { + return (a: string, b: string) => { if (a === b) return 0; const aDomain = getDomain(a); @@ -65,12 +51,8 @@ const createDomainSorter = (gorhill = null) => { } const tldts = require('./cached-tld-parse'); - /** - * @param {string} a - * @param {string} b - * @returns {0 | 1 | -1} - */ - return (a, b) => { + + return (a: string, b: string) => { if (a === b) return 0; const aDomain = tldts.parse(a).domain; @@ -81,5 +63,5 @@ const createDomainSorter = (gorhill = null) => { }; }; -module.exports = createDomainSorter(); -module.exports.createDomainSorter = createDomainSorter; +export default createDomainSorter(); +export { createDomainSorter }; diff --git a/Build/lib/trace-runner.js b/Build/lib/trace-runner.ts similarity index 51% rename from Build/lib/trace-runner.js rename to Build/lib/trace-runner.ts index fd7ce413..3a0920a5 100644 --- a/Build/lib/trace-runner.js +++ b/Build/lib/trace-runner.ts @@ -1,44 +1,24 @@ -// @ts-check -const path = require('path'); -const { performance } = require('perf_hooks'); +import path from 'path'; -/** - * @template T - * @param {string} prefix - * @param {() => T} fn - * @returns {T} - */ -const traceSync = (prefix, fn) => { +const traceSync = (prefix: string, fn: () => T): T => { const start = performance.now(); const result = fn(); const end = performance.now(); console.log(`${prefix}: ${(end - start).toFixed(3)}ms`); return result; }; -module.exports.traceSync = traceSync; +export { traceSync }; -/** - * @template T - * @param {string} prefix - * @param {() => Promise} fn - * @returns {Promise} - */ -const traceAsync = async (prefix, fn) => { +const traceAsync = async (prefix: string, fn: () => Promise): Promise => { const start = performance.now(); const result = await fn(); const end = performance.now(); console.log(`${prefix}: ${(end - start).toFixed(3)}ms`); return result; }; -module.exports.traceAsync = traceAsync; +export { traceAsync }; -/** - * @template T - * @param {string} __filename - * @param {() => Promise} fn - * @param {string | null} [customname] - */ -module.exports.task = (__filename, fn, customname = null) => { +const task = (__filename: string, fn: () => Promise, customname: string | null = null) => { const taskName = customname ?? path.basename(__filename, path.extname(__filename)); return async () => { console.log(`🏃 [${taskName}] Start executing`); @@ -47,6 +27,7 @@ module.exports.task = (__filename, fn, customname = null) => { const end = performance.now(); console.log(`✅ [${taskName}] Executed successfully: ${(end - start).toFixed(3)}ms`); - return { start, end, taskName }; + return { start, end, taskName } as const; }; }; +export { task }; diff --git a/Build/validate-domainset.js b/Build/validate-domainset.ts similarity index 78% rename from Build/validate-domainset.js rename to Build/validate-domainset.ts index 70613501..5faf1400 100644 --- a/Build/validate-domainset.js +++ b/Build/validate-domainset.ts @@ -1,12 +1,12 @@ // Surge Domain Set can not include root domain from public suffix list. -const tldts = require('tldts'); // hit ratio way too low, dont cache -const picocolors = require('picocolors'); -const path = require('path'); -const listDir = require('@sukka/listdir'); -const { readFileByLine } = require('./lib/fetch-remote-text-by-line'); -const { processLine } = require('./lib/process-line'); -const { task } = require('./lib/trace-runner'); +import * as tldts from 'tldts'; // hit ratio way too low, dont cache +import picocolors from 'picocolors'; +import path from 'path'; +import listDir from '@sukka/listdir'; +import { readFileByLine } from './lib/fetch-remote-text-by-line'; +import { processLine } from './lib/process-line'; +import { task } from './lib/trace-runner'; const SPECIAL_SUFFIXES = new Set([ 'linodeobjects.com', // only *.linodeobjects.com are public suffix @@ -14,7 +14,7 @@ const SPECIAL_SUFFIXES = new Set([ 'dweb.link' // only *.dweb.link are public suffix ]); -const validateDomainSet = async (filePath) => { +const validateDomainSet = async (filePath: string) => { for await (const l of readFileByLine(path.resolve(__dirname, '../List/domainset', filePath))) { // starts with # const line = processLine(l); @@ -35,7 +35,7 @@ const validateDomainSet = async (filePath) => { } }; -const _validateRuleset = async (filePath) => { +const _validateRuleset = async (filePath: string) => { console.log(`[${filePath}]`); for await (const l of readFileByLine(path.resolve(__dirname, '../List/non_ip', filePath))) { @@ -58,7 +58,7 @@ const _validateRuleset = async (filePath) => { } }; -const validate = task(__filename, async () => { +export const validate = task(__filename, async () => { // const [domainsetFiles, _rulesetFiles] = await Promise.all([ // listDir(path.resolve(__dirname, '../List/domainset')), // listDir(path.resolve(__dirname, '../List/non_ip')) @@ -69,7 +69,6 @@ const validate = task(__filename, async () => { // rulesetFiles.map(file => validateRuleset(file)) ]); }); -module.exports.validate = validate; if (import.meta.main) { validate(); diff --git a/tsconfig.json b/tsconfig.json index 0d90f2b7..cd4fa176 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -16,7 +16,7 @@ "skipLibCheck": true }, "include": [ - "./Build/**/*.js", + "./Source/**/*.js", "./Build/**/*.ts" ] }