GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2025-12-12 01:00:34 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Refactor: continues to rewrite to TS

Browse Source
This commit is contained in:
SukkaW 2023-11-15 16:26:46 +08:00
parent ec338a659f
commit 99589cf2fc
32 changed files with 258 additions and 471 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
Build/build-anti-bogus-domain.js → Build/build-anti-bogus-domain.ts
View File

@ -1,10 +1,10 @@
// @ts-check // @ts-check
const path = require('path'); import path from 'path';
const { isIPv4, isIPv6 } = require('net'); import { isIPv4, isIPv6 } from 'net';
const { createRuleset } = require('./lib/create-file'); import { createRuleset } from './lib/create-file';
const { fetchRemoteTextAndCreateReadlineInterface, readFileByLine } = require('./lib/fetch-remote-text-by-line'); import { fetchRemoteTextAndCreateReadlineInterface, readFileByLine } from './lib/fetch-remote-text-by-line';
const { processLine } = require('./lib/process-line'); import { processLine } from './lib/process-line';
const { task } = require('./lib/trace-runner'); import { task } from './lib/trace-runner';
const getBogusNxDomainIPs = async () => { const getBogusNxDomainIPs = async () => {
/** @type {string[]} */ /** @type {string[]} */
@ -22,7 +22,7 @@ const getBogusNxDomainIPs = async () => {
return result; return result;
}; };
const buildAntiBogusDomain = task(__filename, async () => { export const buildAntiBogusDomain = task(__filename, async () => {
const bogusIpPromise = getBogusNxDomainIPs(); const bogusIpPromise = getBogusNxDomainIPs();
/** @type {string[]} */ /** @type {string[]} */
@ -61,8 +61,6 @@ const buildAntiBogusDomain = task(__filename, async () => {
)); ));
}); });
module.exports.buildAntiBogusDomain = buildAntiBogusDomain;
if (import.meta.main) { if (import.meta.main) {
buildAntiBogusDomain(); buildAntiBogusDomain();
} }

13
Build/build-apple-cdn.js → Build/build-apple-cdn.ts
View File

@ -1,9 +1,10 @@
const path = require('path'); // @ts-check
const { createRuleset } = require('./lib/create-file'); import path from 'path';
const { parseFelixDnsmasq } = require('./lib/parse-dnsmasq'); import { createRuleset } from './lib/create-file';
const { task } = require('./lib/trace-runner'); import { parseFelixDnsmasq } from './lib/parse-dnsmasq';
import { task } from './lib/trace-runner';
const buildAppleCdn = task(__filename, async () => { export const buildAppleCdn = task(__filename, async () => {
const res = await parseFelixDnsmasq('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf'); const res = await parseFelixDnsmasq('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf');
const description = [ const description = [
@ -42,8 +43,6 @@ const buildAppleCdn = task(__filename, async () => {
]); ]);
}); });
module.exports.buildAppleCdn = buildAppleCdn;
if (import.meta.main) { if (import.meta.main) {
buildAppleCdn(); buildAppleCdn();
} }

16
Build/build-chn-cidr.js → Build/build-chn-cidr.ts
View File

@ -1,10 +1,8 @@
// @ts-check import { fetchRemoteTextAndCreateReadlineInterface } from './lib/fetch-remote-text-by-line';
const { fetchRemoteTextAndCreateReadlineInterface } = require('./lib/fetch-remote-text-by-line'); import { resolve as pathResolve } from 'path';
const { resolve: pathResolve } = require('path'); import { compareAndWriteFile, withBannerArray } from './lib/create-file';
// This should not use `createRuleset` API since we are going to generate ipcidr for Clash import { processLineFromReadline } from './lib/process-line';
const { compareAndWriteFile, withBannerArray } = require('./lib/create-file'); import { task } from './lib/trace-runner';
const { processLineFromReadline } = require('./lib/process-line');
const { task } = require('./lib/trace-runner');
// https://github.com/misakaio/chnroutes2/issues/25 // https://github.com/misakaio/chnroutes2/issues/25
const EXCLUDE_CIDRS = [ const EXCLUDE_CIDRS = [
@ -12,7 +10,7 @@ const EXCLUDE_CIDRS = [
'223.120.0.0/15' '223.120.0.0/15'
]; ];
const buildChnCidr = task(__filename, async () => { export const buildChnCidr = task(__filename, async () => {
const [{ exclude: excludeCidrs }, cidr] = await Promise.all([ const [{ exclude: excludeCidrs }, cidr] = await Promise.all([
import('cidr-tools-wasm'), import('cidr-tools-wasm'),
processLineFromReadline(await fetchRemoteTextAndCreateReadlineInterface('https://raw.githubusercontent.com/misakaio/chnroutes2/master/chnroutes.txt')) processLineFromReadline(await fetchRemoteTextAndCreateReadlineInterface('https://raw.githubusercontent.com/misakaio/chnroutes2/master/chnroutes.txt'))
@ -50,8 +48,6 @@ const buildChnCidr = task(__filename, async () => {
]); ]);
}); });
module.exports.buildChnCidr = buildChnCidr;
if (import.meta.main) { if (import.meta.main) {
buildChnCidr(); buildChnCidr();
} }

36
Build/build-common.js → Build/build-common.ts
View File

@ -1,12 +1,12 @@
// @ts-check // @ts-check
const path = require('path'); import * as path from 'path';
const { PathScurry } = require('path-scurry'); import { PathScurry } from 'path-scurry';
const { readFileByLine } = require('./lib/fetch-remote-text-by-line'); import { readFileByLine } from './lib/fetch-remote-text-by-line';
const { processLine } = require('./lib/process-line'); import { processLine } from './lib/process-line';
const { createRuleset } = require('./lib/create-file'); import { createRuleset } from './lib/create-file';
const { domainDeduper } = require('./lib/domain-deduper'); import { domainDeduper } from './lib/domain-deduper';
const { task } = require('./lib/trace-runner'); import { task } from './lib/trace-runner';
const MAGIC_COMMAND_SKIP = '# $ custom_build_script'; const MAGIC_COMMAND_SKIP = '# $ custom_build_script';
const MAGIC_COMMAND_TITLE = '# $ meta_title '; const MAGIC_COMMAND_TITLE = '# $ meta_title ';
@ -16,7 +16,7 @@ const sourceDir = path.resolve(__dirname, '../Source');
const outputSurgeDir = path.resolve(__dirname, '../List'); const outputSurgeDir = path.resolve(__dirname, '../List');
const outputClashDir = path.resolve(__dirname, '../Clash'); const outputClashDir = path.resolve(__dirname, '../Clash');
const buildCommon = task(__filename, async () => { export const buildCommon = task(__filename, async () => {
/** @type {Promise<unknown>[]} */ /** @type {Promise<unknown>[]} */
const promises = []; const promises = [];
@ -45,16 +45,11 @@ const buildCommon = task(__filename, async () => {
return Promise.all(promises); return Promise.all(promises);
}); });
module.exports.buildCommon = buildCommon;
if (import.meta.main) { if (import.meta.main) {
buildCommon(); buildCommon();
} }
/** const processFile = async (sourcePath: string) => {
* @param {string} sourcePath
*/
const processFile = async (sourcePath) => {
/** @type {string[]} */ /** @type {string[]} */
const lines = []; const lines = [];
@ -83,14 +78,10 @@ const processFile = async (sourcePath) => {
} }
} }
return /** @type {const} */ ([title, descriptions, lines]); return [title, descriptions, lines] as const;
}; };
/** async function transformDomainset(sourcePath: string, relativePath: string) {
* @param {string} sourcePath
* @param {string} relativePath
*/
async function transformDomainset(sourcePath, relativePath) {
const res = await processFile(sourcePath); const res = await processFile(sourcePath);
if (!res) return; if (!res) return;
const [title, descriptions, lines] = res; const [title, descriptions, lines] = res;
@ -120,11 +111,8 @@ async function transformDomainset(sourcePath, relativePath) {
/** /**
* Output Surge RULE-SET and Clash classical text format * Output Surge RULE-SET and Clash classical text format
*
* @param {string} sourcePath
* @param {string} relativePath
*/ */
async function transformRuleset(sourcePath, relativePath) { async function transformRuleset(sourcePath: string, relativePath: string) {
const res = await processFile(sourcePath); const res = await processFile(sourcePath);
if (!res) return; if (!res) return;
const [title, descriptions, lines] = res; const [title, descriptions, lines] = res;

20
Build/build-domestic-ruleset.js → Build/build-domestic-ruleset.ts
View File

@ -1,22 +1,22 @@
// @ts-check // @ts-check
const path = require('path'); import path from 'path';
const { DOMESTICS } = require('../Source/non_ip/domestic'); import { DOMESTICS } from '../Source/non_ip/domestic';
const { readFileByLine } = require('./lib/fetch-remote-text-by-line'); import { readFileByLine } from './lib/fetch-remote-text-by-line';
const { processLineFromReadline } = require('./lib/process-line'); import { processLineFromReadline } from './lib/process-line';
const { compareAndWriteFile, createRuleset } = require('./lib/create-file'); import { compareAndWriteFile, createRuleset } from './lib/create-file';
const { task } = require('./lib/trace-runner'); import { task } from './lib/trace-runner';
const buildDomesticRuleset = task(__filename, async () => { export const buildDomesticRuleset = task(__filename, async () => {
const results = await processLineFromReadline(readFileByLine(path.resolve(__dirname, '../Source/non_ip/domestic.conf'))); const results = await processLineFromReadline(readFileByLine(path.resolve(__dirname, '../Source/non_ip/domestic.conf')));
results.push( results.push(
...Object.entries(DOMESTICS) ...Object.entries(DOMESTICS)
.reduce( .reduce<string[]>(
(acc, [key, { domains }]) => { (acc, [key, { domains }]) => {
if (key === 'SYSTEM') return acc; if (key === 'SYSTEM') return acc;
return [...acc, ...domains]; return [...acc, ...domains];
}, },
/** @type {string[]} */([]) []
) )
.map((domain) => `DOMAIN-SUFFIX,${domain}`) .map((domain) => `DOMAIN-SUFFIX,${domain}`)
); );
@ -58,8 +58,6 @@ const buildDomesticRuleset = task(__filename, async () => {
]); ]);
}); });
module.exports.buildDomesticRuleset = buildDomesticRuleset;
if (import.meta.main) { if (import.meta.main) {
buildDomesticRuleset(); buildDomesticRuleset();
} }

39
Build/build-internal-cdn-rules.js → Build/build-internal-cdn-rules.ts
View File

@ -1,28 +1,25 @@
// @ts-check // @ts-check
const fsp = require('fs/promises'); import fsp from 'fs/promises'
const path = require('path'); import path from 'path';
const tldts = require('tldts'); import * as tldts from 'tldts';
const { processLine } = require('./lib/process-line'); import { processLine } from './lib/process-line';
const { readFileByLine } = require('./lib/fetch-remote-text-by-line'); import { readFileByLine } from './lib/fetch-remote-text-by-line';
const { createDomainSorter } = require('./lib/stable-sort-domain'); import { createDomainSorter } from './lib/stable-sort-domain';
const { task } = require('./lib/trace-runner'); import { task } from './lib/trace-runner';
const { compareAndWriteFile } = require('./lib/create-file'); import { compareAndWriteFile } from './lib/create-file';
const { getGorhillPublicSuffixPromise } = require('./lib/get-gorhill-publicsuffix'); import { getGorhillPublicSuffixPromise } from './lib/get-gorhill-publicsuffix';
// const { createCachedGorhillGetDomain } = require('./lib/cached-tld-parse'); // const { createCachedGorhillGetDomain } = require('./lib/cached-tld-parse');
const escapeRegExp = (string = '') => string.replaceAll(/[$()*+.?[\\\]^{|}]/g, '\\$&'); const escapeRegExp = (string = '') => string.replaceAll(/[$()*+.?[\\\]^{|}]/g, '\\$&');
const buildInternalCDNDomains = task(__filename, async () => { export const buildInternalCDNDomains = task(__filename, async () => {
const set = new Set(); const set = new Set<string>();
const keywords = new Set(); const keywords = new Set();
const gorhill = await getGorhillPublicSuffixPromise(); const gorhill = await getGorhillPublicSuffixPromise();
const domainSorter = createDomainSorter(gorhill); const domainSorter = createDomainSorter(gorhill);
/** const addApexDomain = (input: string) => {
* @param {string} input
*/
const addApexDomain = (input) => {
// We are including the private domains themselves // We are including the private domains themselves
const d = tldts.getDomain(input, { allowPrivateDomains: false }); const d = tldts.getDomain(input, { allowPrivateDomains: false });
if (d) { if (d) {
@ -30,10 +27,7 @@ const buildInternalCDNDomains = task(__filename, async () => {
} }
}; };
/** const processLocalDomainSet = async (domainSetPath: string) => {
* @param {string} domainSetPath
*/
const processLocalDomainSet = async (domainSetPath) => {
for await (const line of readFileByLine(domainSetPath)) { for await (const line of readFileByLine(domainSetPath)) {
// console.log({ line }); // console.log({ line });
@ -52,10 +46,7 @@ const buildInternalCDNDomains = task(__filename, async () => {
} }
}; };
/** const processLocalRuleSet = async (ruleSetPath: string) => {
* @param {string} ruleSetPath
*/
const processLocalRuleSet = async (ruleSetPath) => {
for await (const line of readFileByLine(ruleSetPath)) { for await (const line of readFileByLine(ruleSetPath)) {
if (line.startsWith('DOMAIN-SUFFIX,')) { if (line.startsWith('DOMAIN-SUFFIX,')) {
addApexDomain(line.replace('DOMAIN-SUFFIX,', '')); addApexDomain(line.replace('DOMAIN-SUFFIX,', ''));
@ -93,8 +84,6 @@ const buildInternalCDNDomains = task(__filename, async () => {
); );
}); });
module.exports.buildInternalCDNDomains = buildInternalCDNDomains;
if (import.meta.main) { if (import.meta.main) {
buildInternalCDNDomains(); buildInternalCDNDomains();
} }

15
Build/build-internal-chn-domains.js → Build/build-internal-chn-domains.ts
View File

@ -1,11 +1,10 @@
// @ts-check import path from 'path';
const path = require('path'); import fsp from 'fs/promises'
const fsp = require('fs/promises'); import { parseFelixDnsmasq } from './lib/parse-dnsmasq';
const { parseFelixDnsmasq } = require('./lib/parse-dnsmasq'); import { task } from './lib/trace-runner';
const { task } = require('./lib/trace-runner'); import { compareAndWriteFile } from './lib/create-file';
const { compareAndWriteFile } = require('./lib/create-file');
const buildInternalChnDomains = task(__filename, async () => { export const buildInternalChnDomains = task(__filename, async () => {
const [result] = await Promise.all([ const [result] = await Promise.all([
parseFelixDnsmasq('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf'), parseFelixDnsmasq('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf'),
fsp.mkdir(path.resolve(__dirname, '../List/internal'), { recursive: true }) fsp.mkdir(path.resolve(__dirname, '../List/internal'), { recursive: true })
@ -17,8 +16,6 @@ const buildInternalChnDomains = task(__filename, async () => {
); );
}); });
module.exports.buildInternalChnDomains = buildInternalChnDomains;
if (import.meta.main) { if (import.meta.main) {
buildInternalChnDomains(); buildInternalChnDomains();
} }

15
Build/build-internal-reverse-chn-cidr.js → Build/build-internal-reverse-chn-cidr.ts
View File

@ -1,9 +1,8 @@
// @ts-check import { fetchRemoteTextAndCreateReadlineInterface } from './lib/fetch-remote-text-by-line';
const { fetchRemoteTextAndCreateReadlineInterface } = require('./lib/fetch-remote-text-by-line'); import { processLineFromReadline } from './lib/process-line';
const { processLineFromReadline } = require('./lib/process-line'); import path from 'path';
const path = require('path'); import fsp from 'fs/promises'
const fsp = require('fs/promises'); import { task } from './lib/trace-runner';
const { task } = require('./lib/trace-runner');
const RESERVED_IPV4_CIDR = [ const RESERVED_IPV4_CIDR = [
'0.0.0.0/8', '0.0.0.0/8',
@ -23,7 +22,7 @@ const RESERVED_IPV4_CIDR = [
'240.0.0.0/4' '240.0.0.0/4'
]; ];
const buildInternalReverseChnCIDR = task(__filename, async () => { export const buildInternalReverseChnCIDR = task(__filename, async () => {
const [{ exclude }, cidr] = await Promise.all([ const [{ exclude }, cidr] = await Promise.all([
import('cidr-tools-wasm'), import('cidr-tools-wasm'),
processLineFromReadline(await fetchRemoteTextAndCreateReadlineInterface('https://raw.githubusercontent.com/misakaio/chnroutes2/master/chnroutes.txt')), processLineFromReadline(await fetchRemoteTextAndCreateReadlineInterface('https://raw.githubusercontent.com/misakaio/chnroutes2/master/chnroutes.txt')),
@ -44,8 +43,6 @@ const buildInternalReverseChnCIDR = task(__filename, async () => {
return Bun.write(path.resolve(__dirname, '../List/internal/reversed-chn-cidr.txt'), `${reversedCidr.join('\n')}\n`); return Bun.write(path.resolve(__dirname, '../List/internal/reversed-chn-cidr.txt'), `${reversedCidr.join('\n')}\n`);
}); });
module.exports.buildInternalReverseChnCIDR = buildInternalReverseChnCIDR;
if (import.meta.main) { if (import.meta.main) {
buildInternalReverseChnCIDR(); buildInternalReverseChnCIDR();
} }

2
Build/build-mitm-hostname.js
View File

@ -1,7 +1,7 @@
const fsPromises = require('fs').promises; const fsPromises = require('fs').promises;
const pathFn = require('path'); const pathFn = require('path');
const table = require('table'); const table = require('table');
const listDir = require('@sukka/listdir'); import listDir from '@sukka/listdir';
const { green, yellow } = require('picocolors'); const { green, yellow } = require('picocolors');
const PRESET_MITM_HOSTNAMES = [ const PRESET_MITM_HOSTNAMES = [

31
Build/build-phishing-domainset.js → Build/build-phishing-domainset.ts
View File

@ -1,14 +1,13 @@
// @ts-check import { processFilterRules } from './lib/parse-filter';
const { processFilterRules } = require('./lib/parse-filter.js'); import path from 'path';
const path = require('path'); import { createRuleset } from './lib/create-file';
const { createRuleset } = require('./lib/create-file'); import { processLine } from './lib/process-line';
const { processLine } = require('./lib/process-line.js'); import { createDomainSorter } from './lib/stable-sort-domain';
const { createDomainSorter } = require('./lib/stable-sort-domain'); import { traceSync, task } from './lib/trace-runner';
const { traceSync, task } = require('./lib/trace-runner.js'); import createTrie from './lib/trie';
const createTrie = require('./lib/trie.js'); import { getGorhillPublicSuffixPromise } from './lib/get-gorhill-publicsuffix';
const { getGorhillPublicSuffixPromise } = require('./lib/get-gorhill-publicsuffix.js'); import { createCachedGorhillGetDomain } from './lib/cached-tld-parse';
const { createCachedGorhillGetDomain } = require('./lib/cached-tld-parse.js'); import * as tldts from 'tldts';
const tldts = require('tldts');
const WHITELIST_DOMAIN = new Set([ const WHITELIST_DOMAIN = new Set([
'w3s.link', 'w3s.link',
@ -64,7 +63,7 @@ const BLACK_TLD = new Set([
'com.cn' 'com.cn'
]); ]);
const buildPhishingDomainSet = task(__filename, async () => { export const buildPhishingDomainSet = task(__filename, async () => {
const [{ black: domainSet }, gorhill] = await Promise.all([ const [{ black: domainSet }, gorhill] = await Promise.all([
processFilterRules( processFilterRules(
'https://curbengh.github.io/phishing-filter/phishing-filter-agh.txt', 'https://curbengh.github.io/phishing-filter/phishing-filter-agh.txt',
@ -87,7 +86,7 @@ const buildPhishingDomainSet = task(__filename, async () => {
}); });
}); });
const domainCountMap = {}; const domainCountMap: Record<string, number> = {};
const getDomain = createCachedGorhillGetDomain(gorhill); const getDomain = createCachedGorhillGetDomain(gorhill);
traceSync('* process domain set', () => { traceSync('* process domain set', () => {
@ -154,12 +153,12 @@ const buildPhishingDomainSet = task(__filename, async () => {
const domainSorter = createDomainSorter(gorhill); const domainSorter = createDomainSorter(gorhill);
const results = traceSync('* get final results', () => Object.entries(domainCountMap) const results = traceSync('* get final results', () => Object.entries(domainCountMap)
.reduce((acc, [apexDomain, count]) => { .reduce<string[]>((acc, [apexDomain, count]) => {
if (count >= 5) { if (count >= 5) {
acc.push(`.${apexDomain}`); acc.push(`.${apexDomain}`);
} }
return acc; return acc;
}, /** @type {string[]} */([])) }, [])
.sort(domainSorter)); .sort(domainSorter));
const description = [ const description = [
@ -183,8 +182,6 @@ const buildPhishingDomainSet = task(__filename, async () => {
)); ));
}); });
module.exports.buildPhishingDomainSet = buildPhishingDomainSet;
if (import.meta.main) { if (import.meta.main) {
buildPhishingDomainSet(); buildPhishingDomainSet();
} }

18
Build/build-public.js → Build/build-public.ts
View File

@ -1,7 +1,7 @@
const listDir = require('@sukka/listdir'); import listDir from '@sukka/listdir';
const path = require('path'); import path from 'path';
const fsp = require('fs/promises'); import fsp from 'fs/promises'
const { task } = require('./lib/trace-runner'); import { task } from './lib/trace-runner';
const rootPath = path.resolve(__dirname, '../'); const rootPath = path.resolve(__dirname, '../');
const publicPath = path.resolve(__dirname, '../public'); const publicPath = path.resolve(__dirname, '../public');
@ -15,7 +15,7 @@ const folderAndFilesToBeDeployed = [
'LICENSE' 'LICENSE'
]; ];
const buildPublicHtml = task(__filename, async () => { export const buildPublicHtml = task(__filename, async () => {
await fsp.mkdir(publicPath, { recursive: true }); await fsp.mkdir(publicPath, { recursive: true });
await Promise.all(folderAndFilesToBeDeployed.map(dir => fsp.cp( await Promise.all(folderAndFilesToBeDeployed.map(dir => fsp.cp(
path.resolve(rootPath, dir), path.resolve(rootPath, dir),
@ -33,17 +33,11 @@ const buildPublicHtml = task(__filename, async () => {
return Bun.write(path.join(publicPath, 'index.html'), html); return Bun.write(path.join(publicPath, 'index.html'), html);
}); });
module.exports.buildPublicHtml = buildPublicHtml;
if (import.meta.main) { if (import.meta.main) {
buildPublicHtml(); buildPublicHtml();
} }
/** function template(urlList: string[]) {
* @param {string[]} urlList
* @returns {string}
*/
function template(urlList) {
return ` return `
<!DOCTYPE html> <!DOCTYPE html>
<html lang="en"> <html lang="en">

14
Build/build-redirect-module.js → Build/build-redirect-module.ts
View File

@ -1,9 +1,7 @@
// @ts-check import path from 'path';
import { task } from './lib/trace-runner';
const path = require('path'); import { compareAndWriteFile } from './lib/create-file';
const { task } = require('./lib/trace-runner'); import * as tldts from 'tldts';
const { compareAndWriteFile } = require('./lib/create-file');
const tldts = require('tldts');
function escapeRegExp(string = '') { function escapeRegExp(string = '') {
const reRegExpChar = /[$()*+.?[\\\]^{|}]/g; const reRegExpChar = /[$()*+.?[\\\]^{|}]/g;
@ -73,7 +71,7 @@ const REDIRECT = /** @type {const} */ ([
['googleajax.wp-china-yes.net/', 'https://ajax.googleapis.com/'] ['googleajax.wp-china-yes.net/', 'https://ajax.googleapis.com/']
]); ]);
const buildRedirectModule = task(__filename, async () => { export const buildRedirectModule = task(__filename, async () => {
const domains = Array.from(new Set(REDIRECT.map(([from]) => tldts.getHostname(from, { detectIp: false })))).filter(Boolean); const domains = Array.from(new Set(REDIRECT.map(([from]) => tldts.getHostname(from, { detectIp: false })))).filter(Boolean);
return compareAndWriteFile( return compareAndWriteFile(
@ -94,8 +92,6 @@ const buildRedirectModule = task(__filename, async () => {
); );
}); });
module.exports.buildRedirectModule = buildRedirectModule;
if (import.meta.main) { if (import.meta.main) {
buildRedirectModule(); buildRedirectModule();
} }

43
Build/build-reject-domainset.js → Build/build-reject-domainset.ts
View File

@ -1,31 +1,31 @@
// @ts-check // @ts-check
const fsp = require('fs/promises'); import fsp from 'fs/promises'
const path = require('path'); import path from 'path';
const { processHosts, processFilterRules } = require('./lib/parse-filter'); import { processHosts, processFilterRules } from './lib/parse-filter';
const createTrie = require('./lib/trie'); import createTrie from './lib/trie';
const { HOSTS, ADGUARD_FILTERS, PREDEFINED_WHITELIST, PREDEFINED_ENFORCED_BACKLIST } = require('./lib/reject-data-source'); import { HOSTS, ADGUARD_FILTERS, PREDEFINED_WHITELIST, PREDEFINED_ENFORCED_BACKLIST } from './lib/reject-data-source';
const { createRuleset, compareAndWriteFile } = require('./lib/create-file'); import { createRuleset, compareAndWriteFile } from './lib/create-file';
const { processLine } = require('./lib/process-line'); import { processLine } from './lib/process-line';
const { domainDeduper } = require('./lib/domain-deduper'); import { domainDeduper } from './lib/domain-deduper';
const createKeywordFilter = require('./lib/aho-corasick'); import createKeywordFilter from './lib/aho-corasick';
const { readFileByLine } = require('./lib/fetch-remote-text-by-line'); import { readFileByLine } from './lib/fetch-remote-text-by-line';
const { createDomainSorter } = require('./lib/stable-sort-domain'); import { createDomainSorter } from './lib/stable-sort-domain';
const { traceSync, task } = require('./lib/trace-runner'); import { traceSync, task } from './lib/trace-runner';
const { getGorhillPublicSuffixPromise } = require('./lib/get-gorhill-publicsuffix'); import { getGorhillPublicSuffixPromise } from './lib/get-gorhill-publicsuffix';
const tldts = require('tldts'); import * as tldts from 'tldts';
/** Whitelists */ /** Whitelists */
const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST); const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST);
/** @type {Set<string>} Dedupe domains inclued by DOMAIN-KEYWORD */ /** @type {Set<string>} Dedupe domains inclued by DOMAIN-KEYWORD */
const domainKeywordsSet = new Set(); const domainKeywordsSet: Set<string> = new Set();
/** @type {Set<string>} Dedupe domains included by DOMAIN-SUFFIX */ /** @type {Set<string>} Dedupe domains included by DOMAIN-SUFFIX */
const domainSuffixSet = new Set(); const domainSuffixSet: Set<string> = new Set();
const buildRejectDomainSet = task(__filename, async () => { export const buildRejectDomainSet = task(__filename, async () => {
/** @type Set<string> */ /** @type Set<string> */
const domainSets = new Set(); const domainSets: Set<string> = new Set();
// Parse from AdGuard Filters // Parse from AdGuard Filters
console.time('* Download and process Hosts / AdBlock Filter Rules'); console.time('* Download and process Hosts / AdBlock Filter Rules');
@ -172,11 +172,10 @@ const buildRejectDomainSet = task(__filename, async () => {
console.log(`Deduped ${previousSize - dudupedDominArray.length} rules!`); console.log(`Deduped ${previousSize - dudupedDominArray.length} rules!`);
// Create reject stats // Create reject stats
/** @type {[string, number][]} */ const rejectDomainsStats: [string, number][] = traceSync(
const rejectDomainsStats = traceSync(
'* Collect reject domain stats', '* Collect reject domain stats',
() => Object.entries( () => Object.entries(
dudupedDominArray.reduce((acc, cur) => { dudupedDominArray.reduce<Record<string, number>>((acc, cur) => {
const suffix = tldts.getDomain(cur, { allowPrivateDomains: false, detectIp: false }); const suffix = tldts.getDomain(cur, { allowPrivateDomains: false, detectIp: false });
if (suffix) { if (suffix) {
acc[suffix] = (acc[suffix] ?? 0) + 1; acc[suffix] = (acc[suffix] ?? 0) + 1;
@ -231,8 +230,6 @@ const buildRejectDomainSet = task(__filename, async () => {
]); ]);
}); });
module.exports.buildRejectDomainSet = buildRejectDomainSet;
if (import.meta.main) { if (import.meta.main) {
buildRejectDomainSet(); buildRejectDomainSet();
} }

30
Build/build-speedtest-domainset.js → Build/build-speedtest-domainset.ts
View File

@ -1,18 +1,14 @@
const { domainDeduper } = require('./lib/domain-deduper'); import { domainDeduper } from './lib/domain-deduper';
const path = require('path'); import path from 'path';
const { createRuleset } = require('./lib/create-file'); import { createRuleset } from './lib/create-file';
const domainSorter = require('./lib/stable-sort-domain'); import domainSorter from './lib/stable-sort-domain';
const { Sema } = require('async-sema'); import { Sema } from 'async-sema';
const tldts = require('tldts'); import * as tldts from 'tldts';
const { task } = require('./lib/trace-runner'); import { task } from './lib/trace-runner';
const s = new Sema(2); const s = new Sema(2);
/** const querySpeedtestApi = async (keyword: string): Promise<(string | null)[]> => {
* @param {string} keyword
* @returns {string[]}
*/
const querySpeedtestApi = async (keyword) => {
await s.acquire(); await s.acquire();
try { try {
@ -36,19 +32,19 @@ const querySpeedtestApi = async (keyword) => {
throw new Error(text); throw new Error(text);
} }
/** @type {{ url: string }[]} */ const json = await res.json() as { url: string; }[];
const json = await res.json();
s.release(); s.release();
return json.map(({ url }) => tldts.getHostname(url, { detectIp: false })); return json.map(({ url }) => tldts.getHostname(url, { detectIp: false }));
} catch (e) { } catch (e) {
s.release(); s.release();
console.log(e); console.log(e);
return [];
} }
}; };
const buildSpeedtestDomainSet = task(__filename, async () => { export const buildSpeedtestDomainSet = task(__filename, async () => {
/** @type {Set<string>} */ /** @type {Set<string>} */
const domains = new Set([ const domains: Set<string> = new Set([
'.speedtest.net', '.speedtest.net',
'.speedtestcustom.com', '.speedtestcustom.com',
'.ooklaserver.net', '.ooklaserver.net',
@ -140,8 +136,6 @@ const buildSpeedtestDomainSet = task(__filename, async () => {
)); ));
}); });
module.exports.buildSpeedtestDomainSet = buildSpeedtestDomainSet;
if (import.meta.main) { if (import.meta.main) {
buildSpeedtestDomainSet(); buildSpeedtestDomainSet();
} }

34
Build/build-stream-service.js → Build/build-stream-service.ts
View File

@ -1,20 +1,12 @@
// @ts-check // @ts-check
const { task } = require('./lib/trace-runner'); import { task } from './lib/trace-runner';
const path = require('path'); import path from 'path';
const { createRuleset } = require('./lib/create-file'); import { createRuleset } from './lib/create-file';
const { import { ALL, NORTH_AMERICA, EU, HK, TW, JP, KR } from '../Source/stream';
ALL, NORTH_AMERICA, EU, HK, TW, JP, KR
// SOUTH_EAST_ASIA, AU
} = require('../Source/stream');
/** const createRulesetForStreamService = (fileId: string, title: string, streamServices: import('../Source/stream').StreamService[]) => {
* @param {string} fileId
* @param {string} title
* @param {import('../Source/stream').StreamService[]} streamServices
*/
const createRulesetForStreamService = (fileId, title, streamServices) => {
return [ return [
// Domains // Domains
...createRuleset( ...createRuleset(
@ -24,10 +16,10 @@ const createRulesetForStreamService = (fileId, title, streamServices) => {
'Homepage: https://ruleset.skk.moe', 'Homepage: https://ruleset.skk.moe',
'GitHub: https://github.com/SukkaW/Surge', 'GitHub: https://github.com/SukkaW/Surge',
'', '',
...streamServices.map(i => `- ${i.name}`) ...streamServices.map((i: { name: any; }) => `- ${i.name}`)
], ],
new Date(), new Date(),
streamServices.flatMap(i => i.rules), streamServices.flatMap((i: { rules: any; }) => i.rules),
'ruleset', 'ruleset',
path.resolve(__dirname, `../List/non_ip/${fileId}.conf`), path.resolve(__dirname, `../List/non_ip/${fileId}.conf`),
path.resolve(__dirname, `../Clash/non_ip/${fileId}.txt`) path.resolve(__dirname, `../Clash/non_ip/${fileId}.txt`)
@ -40,14 +32,14 @@ const createRulesetForStreamService = (fileId, title, streamServices) => {
'Homepage: https://ruleset.skk.moe', 'Homepage: https://ruleset.skk.moe',
'GitHub: https://github.com/SukkaW/Surge', 'GitHub: https://github.com/SukkaW/Surge',
'', '',
...streamServices.map(i => `- ${i.name}`) ...streamServices.map((i: { name: any; }) => `- ${i.name}`)
], ],
new Date(), new Date(),
streamServices.flatMap(i => ( streamServices.flatMap((i) => (
i.ip i.ip
? [ ? [
...i.ip.v4.map(ip => `IP-CIDR,${ip},no-resolve`), ...i.ip.v4.map((ip: any) => `IP-CIDR,${ip},no-resolve`),
...i.ip.v6.map(ip => `IP-CIDR6,${ip},no-resolve`) ...i.ip.v6.map((ip: any) => `IP-CIDR6,${ip},no-resolve`)
] ]
: [] : []
)), )),
@ -58,7 +50,7 @@ const createRulesetForStreamService = (fileId, title, streamServices) => {
]; ];
}; };
const buildStreamService = task(__filename, async () => { export const buildStreamService = task(__filename, async () => {
return Promise.all([ return Promise.all([
...createRulesetForStreamService('stream', 'All', ALL), ...createRulesetForStreamService('stream', 'All', ALL),
...createRulesetForStreamService('stream_us', 'North America', NORTH_AMERICA), ...createRulesetForStreamService('stream_us', 'North America', NORTH_AMERICA),
@ -72,8 +64,6 @@ const buildStreamService = task(__filename, async () => {
]); ]);
}); });
module.exports.buildStreamService = buildStreamService;
if (import.meta.main) { if (import.meta.main) {
buildStreamService(); buildStreamService();
} }

18
Build/build-telegram-cidr.js → Build/build-telegram-cidr.ts
View File

@ -1,13 +1,13 @@
// @ts-check // @ts-check
const { fetchWithRetry } = require('./lib/fetch-retry'); import { fetchWithRetry } from './lib/fetch-retry';
const { createReadlineInterfaceFromResponse } = require('./lib/fetch-remote-text-by-line'); import { createReadlineInterfaceFromResponse } from './lib/fetch-remote-text-by-line';
const path = require('path'); import path from 'path';
const { isIPv4, isIPv6 } = require('net'); import { isIPv4, isIPv6 } from 'net';
const { processLine } = require('./lib/process-line'); import { processLine } from './lib/process-line';
const { createRuleset } = require('./lib/create-file'); import { createRuleset } from './lib/create-file';
const { task } = require('./lib/trace-runner'); import { task } from './lib/trace-runner';
const buildTelegramCIDR = task(__filename, async () => { export const buildTelegramCIDR = task(__filename, async () => {
/** @type {Response} */ /** @type {Response} */
const resp = await fetchWithRetry('https://core.telegram.org/resources/cidr.txt'); const resp = await fetchWithRetry('https://core.telegram.org/resources/cidr.txt');
const lastModified = resp.headers.get('last-modified'); const lastModified = resp.headers.get('last-modified');
@ -52,8 +52,6 @@ const buildTelegramCIDR = task(__filename, async () => {
)); ));
}); });
module.exports.buildTelegramCIDR = buildTelegramCIDR;
if (import.meta.main) { if (import.meta.main) {
buildTelegramCIDR(); buildTelegramCIDR();
} }

50
Build/lib/aho-corasick.js → Build/lib/aho-corasick.ts
View File

@ -1,19 +1,15 @@
/** interface Node {
* @typedef {Object} Node /** @default 0 */
* @prop {number} [depth = 0] depth?: number;
* @prop {string} key key: string;
* @prop {boolean} [word = false] /** @default false */
* @prop {Record<string, Node>} [children={}] word?: boolean;
* @prop {Node} [fail] children: Record<string, Node>;
* @prop {number} [count=0] fail?: Node;
*/ count: number;
}
/** const createNode = (key: string, depth = 0): Node => ({
* @param {string} key
* @param {number} depth
* @returns {Node}
*/
const createNode = (key, depth = 0) => ({
depth, depth,
key, key,
word: false, word: false,
@ -22,15 +18,11 @@ const createNode = (key, depth = 0) => ({
count: 0 count: 0
}); });
/** const createKeywordFilter = (keys: string[] | Set<string>) => {
* @param {string[] | Set<string>} keys
*/
const createKeywordFilter = (keys) => {
const root = createNode('root'); const root = createNode('root');
const build = () => { const build = () => {
/** @type {Node[]} */ const queue: Node[] = [];
const queue = [];
queue.push(root); queue.push(root);
let idx = 0; let idx = 0;
@ -57,11 +49,7 @@ const createKeywordFilter = (keys) => {
} }
}; };
/** const put = (key: string, len: number) => {
* @param {string} key
* @param {number} len
*/
const put = (key, len) => {
let node = root; let node = root;
const lastIdx = len - 1; const lastIdx = len - 1;
node.count++; node.count++;
@ -91,12 +79,8 @@ const createKeywordFilter = (keys) => {
build(); build();
/** const search = (text: string) => {
* @param {string} text let node: Node | undefined = root;
* @returns {boolean}
*/
const search = (text) => {
let node = root;
for (let i = 0, textLen = text.length; i < textLen; i++) { for (let i = 0, textLen = text.length; i < textLen; i++) {
// const key = text.charAt(i); // const key = text.charAt(i);
@ -120,4 +104,4 @@ const createKeywordFilter = (keys) => {
}; };
}; };
module.exports = createKeywordFilter; export default createKeywordFilter;

22
Build/lib/cache-apply.js → Build/lib/cache-apply.ts
View File

@ -1,7 +1,4 @@
/** export const createCache = (namespace?: string, printStats = false) => {
* @param {string} [namespace]
*/
const createCache = (namespace, printStats = false) => {
const cache = new Map(); const cache = new Map();
let hit = 0; let hit = 0;
@ -12,13 +9,7 @@ const createCache = (namespace, printStats = false) => {
} }
return { return {
/** sync<T>(key: string, fn: () => T): T {
* @template T
* @param {string} key
* @param {() => T} fn
* @returns {T}
*/
sync(key, fn) {
if (cache.has(key)) { if (cache.has(key)) {
hit++; hit++;
return cache.get(key); return cache.get(key);
@ -27,13 +18,7 @@ const createCache = (namespace, printStats = false) => {
cache.set(key, value); cache.set(key, value);
return value; return value;
}, },
/** async async<T>(key: string, fn: () => Promise<T>): Promise<T> {
* @template T
* @param {string} key
* @param {() => Promise<T>} fn
* @returns {Promise<T>}
*/
async async(key, fn) {
if (cache.has(key)) { if (cache.has(key)) {
hit++; hit++;
return cache.get(key); return cache.get(key);
@ -44,4 +29,3 @@ const createCache = (namespace, printStats = false) => {
} }
}; };
}; };
module.exports.createCache = createCache;

24
Build/lib/cached-tld-parse.js
View File

@ -1,24 +0,0 @@
const tldts = require('tldts');
const { createCache } = require('./cache-apply');
const cache = createCache('cached-tld-parse', true);
const sharedConfig = { allowPrivateDomains: true };
/**
* @param {string} domain
* @returns {ReturnType<import('tldts').parse>}
*/
module.exports.parse = (domain) => cache.sync(domain, () => tldts.parse(domain, sharedConfig));
let gothillGetDomainCache = null;
/**
* @param {import('gorhill-publicsuffixlist').default | null} gorhill
*/
module.exports.createCachedGorhillGetDomain = (gorhill) => {
gothillGetDomainCache ||= createCache('cached-gorhill-get-domain', true);
/**
* @param {string} domain
*/
return (domain) => (/** @type {ReturnType<typeof createCache>} */ (gothillGetDomainCache)).sync(domain, () => gorhill.getDomain(domain[0] === '.' ? domain.slice(1) : domain));
};

17
Build/lib/cached-tld-parse.ts Normal file
View File

@ -0,0 +1,17 @@
import tldts from 'tldts';
import { createCache } from './cache-apply';
import { PublicSuffixList } from 'gorhill-publicsuffixlist';
const cache = createCache('cached-tld-parse', true);
const sharedConfig = { allowPrivateDomains: true };
export const parse = (domain: string) => cache.sync(domain, () => tldts.parse(domain, sharedConfig));
let gothillGetDomainCache: ReturnType<typeof createCache> | null = null;
export const createCachedGorhillGetDomain = (gorhill: PublicSuffixList) => {
return (domain: string) => {
gothillGetDomainCache ??= createCache('cached-gorhill-get-domain', true);
return gothillGetDomainCache.sync(domain, () => gorhill.getDomain(domain[0] === '.' ? domain.slice(1) : domain))
};
};

15
Build/lib/clash.js → Build/lib/clash.ts
View File

@ -1,6 +1,5 @@
// @ts-check // @ts-check
const _Trie = require('mnemonist/trie'); import Trie from 'mnemonist/trie';
const Trie = _Trie.default || _Trie;
// https://dreamacro.github.io/clash/configuration/rules.html // https://dreamacro.github.io/clash/configuration/rules.html
const CLASH_SUPPORTED_RULE_TYPE = [ const CLASH_SUPPORTED_RULE_TYPE = [
@ -17,21 +16,13 @@ const CLASH_SUPPORTED_RULE_TYPE = [
'PROCESS-PATH' 'PROCESS-PATH'
]; ];
/** export const surgeRulesetToClashClassicalTextRuleset = (rules: string[] | Set<string>) => {
* @param {string[] | Set<string>} rules
*/
const surgeRulesetToClashClassicalTextRuleset = (rules) => {
const trie = Trie.from(rules); const trie = Trie.from(rules);
return CLASH_SUPPORTED_RULE_TYPE.flatMap( return CLASH_SUPPORTED_RULE_TYPE.flatMap(
type => trie.find(`${type},`) type => trie.find(`${type},`)
); );
}; };
module.exports.surgeRulesetToClashClassicalTextRuleset = surgeRulesetToClashClassicalTextRuleset;
/** export const surgeDomainsetToClashDomainset = (domainset: string[]) => {
* @param {string[]} domainset
*/
const surgeDomainsetToClashDomainset = (domainset) => {
return domainset.map(i => (i[0] === '.' ? `+${i}` : i)); return domainset.map(i => (i[0] === '.' ? `+${i}` : i));
}; };
module.exports.surgeDomainsetToClashDomainset = surgeDomainsetToClashDomainset;

37
Build/lib/create-file.js → Build/lib/create-file.ts
View File

@ -1,12 +1,8 @@
// @ts-check // @ts-check
const { readFileByLine } = require('./fetch-remote-text-by-line'); import { readFileByLine } from './fetch-remote-text-by-line';
const { surgeDomainsetToClashDomainset, surgeRulesetToClashClassicalTextRuleset } = require('./clash'); import { surgeDomainsetToClashDomainset, surgeRulesetToClashClassicalTextRuleset } from './clash';
/** export async function compareAndWriteFile(linesA: string[], filePath: string) {
* @param {string[]} linesA
* @param {string} filePath
*/
async function compareAndWriteFile(linesA, filePath) {
let isEqual = true; let isEqual = true;
const file = Bun.file(filePath); const file = Bun.file(filePath);
@ -57,16 +53,8 @@ async function compareAndWriteFile(linesA, filePath) {
console.log(`Same Content, bail out writing: ${filePath}`); console.log(`Same Content, bail out writing: ${filePath}`);
} }
module.exports.compareAndWriteFile = compareAndWriteFile;
/** export const withBannerArray = (title: string, description: string[], date: Date, content: string[]) => {
* @param {string} title
* @param {string[]} description
* @param {Date} date
* @param {string[]} content
* @returns {string[]}
*/
const withBannerArray = (title, description, date, content) => {
return [ return [
'########################################', '########################################',
`# ${title}`, `# ${title}`,
@ -78,20 +66,10 @@ const withBannerArray = (title, description, date, content) => {
'################# END ###################' '################# END ###################'
]; ];
}; };
module.exports.withBannerArray = withBannerArray;
/** export const createRuleset = (
* @param {string} title title: string, description: string[], date: Date, content: string[],
* @param {string[]} description type: 'ruleset' | 'domainset', surgePath: string, clashPath: string
* @param {Date} date
* @param {string[]} content
* @param {'ruleset' | 'domainset'} type
* @param {string} surgePath
* @param {string} clashPath
*/
const createRuleset = (
title, description, date, content,
type, surgePath, clashPath
) => { ) => {
const surgeContent = withBannerArray(title, description, date, content); const surgeContent = withBannerArray(title, description, date, content);
@ -114,4 +92,3 @@ const createRuleset = (
compareAndWriteFile(clashContent, clashPath) compareAndWriteFile(clashContent, clashPath)
]; ];
}; };
module.exports.createRuleset = createRuleset;

2
Build/lib/domain-deduper.ts
View File

@ -1,6 +1,6 @@
import createTrie from './trie'; import createTrie from './trie';
const domainDeduper = (inputDomains: string[]): string[] => { export const domainDeduper = (inputDomains: string[]): string[] => {
const trie = createTrie(inputDomains); const trie = createTrie(inputDomains);
const sets = new Set(inputDomains); const sets = new Set(inputDomains);

3
Build/lib/fetch-retry.js
View File

@ -1,3 +0,0 @@
// @ts-check
const fetchWithRetry = require('@vercel/fetch-retry')(fetch);
module.exports.fetchWithRetry = fetchWithRetry;

4
Build/lib/fetch-retry.ts Normal file
View File

@ -0,0 +1,4 @@
// @ts-expect-error -- missing types
import createFetchRetry from '@vercel/fetch-retry';
export const fetchWithRetry: typeof fetch = createFetchRetry(fetch);

16
Build/lib/get-gorhill-publicsuffix.js → Build/lib/get-gorhill-publicsuffix.ts
View File

@ -1,13 +1,12 @@
const { toASCII } = require('punycode/'); import { toASCII } from 'punycode';
const path = require('path'); import path from 'path';
const { traceAsync } = require('./trace-runner'); import { traceAsync } from './trace-runner';
import type { PublicSuffixList } from 'gorhill-publicsuffixlist';
const publicSuffixPath = path.resolve(__dirname, '../../node_modules/.cache/public_suffix_list_dat.txt'); const publicSuffixPath = path.resolve(__dirname, '../../node_modules/.cache/public_suffix_list_dat.txt');
const getGorhillPublicSuffix = () => traceAsync('create gorhill public suffix instance', async () => { const getGorhillPublicSuffix = () => traceAsync('create gorhill public suffix instance', async () => {
const customFetch = async (url) => { const customFetch = async (url: string | URL) => Bun.file(url);
return Bun.file(url);
};
const publicSuffixFile = Bun.file(publicSuffixPath); const publicSuffixFile = Bun.file(publicSuffixPath);
@ -27,9 +26,8 @@ const getGorhillPublicSuffix = () => traceAsync('create gorhill public suffix in
return gorhill; return gorhill;
}); });
/** @type {Promise<import('gorhill-publicsuffixlist').default> | null} */ let gorhillPublicSuffixPromise: Promise<PublicSuffixList> | null = null;
let gorhillPublicSuffixPromise = null; export const getGorhillPublicSuffixPromise = () => {
module.exports.getGorhillPublicSuffixPromise = () => {
gorhillPublicSuffixPromise ||= getGorhillPublicSuffix(); gorhillPublicSuffixPromise ||= getGorhillPublicSuffix();
return gorhillPublicSuffixPromise; return gorhillPublicSuffixPromise;
}; };

4
Build/lib/parse-dnsmasq.ts
View File

@ -6,7 +6,7 @@ const isDomainLoose = (domain: string): boolean => {
return !!(!isIp && (isIcann || isPrivate)); return !!(!isIp && (isIcann || isPrivate));
}; };
const parseFelixDnsmasq = async (url: string | URL): Promise<string[]> => { export const parseFelixDnsmasq = async (url: string | URL): Promise<string[]> => {
const res: string[] = []; const res: string[] = [];
for await (const line of await fetchRemoteTextAndCreateReadlineInterface(url)) { for await (const line of await fetchRemoteTextAndCreateReadlineInterface(url)) {
if (line.startsWith('server=/') && line.endsWith('/114.114.114.114')) { if (line.startsWith('server=/') && line.endsWith('/114.114.114.114')) {
@ -19,5 +19,3 @@ const parseFelixDnsmasq = async (url: string | URL): Promise<string[]> => {
return res; return res;
}; };
export { parseFelixDnsmasq };

77
Build/lib/parse-filter.js → Build/lib/parse-filter.ts
View File

@ -1,24 +1,18 @@
// @ts-check // @ts-check
const { fetchWithRetry } = require('./fetch-retry'); import { fetchWithRetry } from './fetch-retry';
const tldts = require('./cached-tld-parse'); import * as tldts from './cached-tld-parse';
const { fetchRemoteTextAndCreateReadlineInterface } = require('./fetch-remote-text-by-line'); import { fetchRemoteTextAndCreateReadlineInterface } from './fetch-remote-text-by-line';
const { NetworkFilter } = require('@cliqz/adblocker'); import { NetworkFilter } from '@cliqz/adblocker';
const { processLine } = require('./process-line'); import { processLine } from './process-line';
const { performance } = require('perf_hooks'); import { performance } from 'perf_hooks';
const { getGorhillPublicSuffixPromise } = require('./get-gorhill-publicsuffix'); import { getGorhillPublicSuffixPromise } from './get-gorhill-publicsuffix';
import type { PublicSuffixList } from 'gorhill-publicsuffixlist';
const DEBUG_DOMAIN_TO_FIND = null; // example.com | null const DEBUG_DOMAIN_TO_FIND = null; // example.com | null
let foundDebugDomain = false; let foundDebugDomain = false;
const warnOnceUrl = new Set(); const warnOnceUrl = new Set<string>();
/** const warnOnce = (url: string, isWhite: boolean, ...message: any[]) => {
*
* @param {string} url
* @param {boolean} isWhite
* @param {...any} message
* @returns
*/
const warnOnce = (url, isWhite, ...message) => {
const key = `${url}${isWhite ? 'white' : 'black'}`; const key = `${url}${isWhite ? 'white' : 'black'}`;
if (warnOnceUrl.has(key)) { if (warnOnceUrl.has(key)) {
return; return;
@ -27,10 +21,7 @@ const warnOnce = (url, isWhite, ...message) => {
console.warn(url, isWhite ? '(white)' : '(black)', ...message); console.warn(url, isWhite ? '(white)' : '(black)', ...message);
}; };
/** const normalizeDomain = (domain: string) => {
* @param {string} domain
*/
const normalizeDomain = (domain) => {
if (!domain) return null; if (!domain) return null;
const parsed = tldts.parse(domain); const parsed = tldts.parse(domain);
@ -47,10 +38,7 @@ const normalizeDomain = (domain) => {
return null; return null;
}; };
/** export async function processDomainLists(domainListsUrl: string | URL) {
* @param {string | URL} domainListsUrl
*/
async function processDomainLists(domainListsUrl) {
if (typeof domainListsUrl === 'string') { if (typeof domainListsUrl === 'string') {
domainListsUrl = new URL(domainListsUrl); domainListsUrl = new URL(domainListsUrl);
} }
@ -79,18 +67,14 @@ async function processDomainLists(domainListsUrl) {
return domainSets; return domainSets;
} }
/** export async function processHosts(hostsUrl: string | URL, includeAllSubDomain = false) {
* @param {string | URL} hostsUrl
*/
async function processHosts(hostsUrl, includeAllSubDomain = false) {
console.time(` - processHosts: ${hostsUrl}`); console.time(` - processHosts: ${hostsUrl}`);
if (typeof hostsUrl === 'string') { if (typeof hostsUrl === 'string') {
hostsUrl = new URL(hostsUrl); hostsUrl = new URL(hostsUrl);
} }
/** @type Set<string> */ const domainSets = new Set<string>();
const domainSets = new Set();
for await (const l of await fetchRemoteTextAndCreateReadlineInterface(hostsUrl)) { for await (const l of await fetchRemoteTextAndCreateReadlineInterface(hostsUrl)) {
const line = processLine(l); const line = processLine(l);
@ -121,24 +105,20 @@ async function processHosts(hostsUrl, includeAllSubDomain = false) {
return domainSets; return domainSets;
} }
/** export async function processFilterRules(
* @param {string | URL} filterRulesUrl filterRulesUrl: string | URL,
* @param {readonly (string | URL)[] | undefined} [fallbackUrls] fallbackUrls?: readonly (string | URL)[] | undefined
* @returns {Promise<{ white: Set<string>, black: Set<string>, foundDebugDomain: boolean }>} ): Promise<{ white: Set<string>, black: Set<string>, foundDebugDomain: boolean }> {
*/
async function processFilterRules(filterRulesUrl, fallbackUrls) {
const runStart = performance.now(); const runStart = performance.now();
/** @type Set<string> */ const whitelistDomainSets = new Set<string>();
const whitelistDomainSets = new Set(); const blacklistDomainSets = new Set<string>();
/** @type Set<string> */
const blacklistDomainSets = new Set();
/** /**
* @param {string} domainToBeAddedToBlack * @param {string} domainToBeAddedToBlack
* @param {boolean} isSubDomain * @param {boolean} isSubDomain
*/ */
const addToBlackList = (domainToBeAddedToBlack, isSubDomain) => { const addToBlackList = (domainToBeAddedToBlack: string, isSubDomain: boolean) => {
if (isSubDomain && domainToBeAddedToBlack[0] !== '.') { if (isSubDomain && domainToBeAddedToBlack[0] !== '.') {
blacklistDomainSets.add(`.${domainToBeAddedToBlack}`); blacklistDomainSets.add(`.${domainToBeAddedToBlack}`);
} else { } else {
@ -149,7 +129,7 @@ async function processFilterRules(filterRulesUrl, fallbackUrls) {
* @param {string} domainToBeAddedToWhite * @param {string} domainToBeAddedToWhite
* @param {boolean} [isSubDomain] * @param {boolean} [isSubDomain]
*/ */
const addToWhiteList = (domainToBeAddedToWhite, isSubDomain = true) => { const addToWhiteList = (domainToBeAddedToWhite: string, isSubDomain = true) => {
if (isSubDomain && domainToBeAddedToWhite[0] !== '.') { if (isSubDomain && domainToBeAddedToWhite[0] !== '.') {
whitelistDomainSets.add(`.${domainToBeAddedToWhite}`); whitelistDomainSets.add(`.${domainToBeAddedToWhite}`);
} else { } else {
@ -163,7 +143,7 @@ async function processFilterRules(filterRulesUrl, fallbackUrls) {
/** /**
* @param {string} line * @param {string} line
*/ */
const lineCb = (line) => { const lineCb = (line: string) => {
const result = parse(line, gorhill); const result = parse(line, gorhill);
if (result) { if (result) {
const flag = result[1]; const flag = result[1];
@ -250,12 +230,7 @@ async function processFilterRules(filterRulesUrl, fallbackUrls) {
const R_KNOWN_NOT_NETWORK_FILTER_PATTERN = /[#%&=~]/; const R_KNOWN_NOT_NETWORK_FILTER_PATTERN = /[#%&=~]/;
const R_KNOWN_NOT_NETWORK_FILTER_PATTERN_2 = /(\$popup|\$removeparam|\$popunder)/; const R_KNOWN_NOT_NETWORK_FILTER_PATTERN_2 = /(\$popup|\$removeparam|\$popunder)/;
/** function parse($line: string, gorhill: PublicSuffixList): null | [hostname: string, flag: 0 | 1 | 2 | -1] {
* @param {string} $line
* @param {import('gorhill-publicsuffixlist').default} gorhill
* @returns {null | [hostname: string, flag: 0 | 1 | 2 | -1]} - 0 white include subdomain, 1 black abosulte, 2 black include subdomain, -1 white
*/
function parse($line, gorhill) {
if ( if (
// doesn't include // doesn't include
!$line.includes('.') // rule with out dot can not be a domain !$line.includes('.') // rule with out dot can not be a domain
@ -615,7 +590,3 @@ function parse($line, gorhill) {
return null; return null;
} }
module.exports.processDomainLists = processDomainLists;
module.exports.processHosts = processHosts;
module.exports.processFilterRules = processFilterRules;

40
Build/lib/stable-sort-domain.js → Build/lib/stable-sort-domain.ts
View File

@ -1,10 +1,7 @@
// @ts-check import type { PublicSuffixList } from 'gorhill-publicsuffixlist';
/** import { createCachedGorhillGetDomain } from './cached-tld-parse';
* @param {string | null} a
* @param {string | null} b const compare = (a: string | null, b: string | null) => {
* @returns {0 | 1 | -1}
*/
const compare = (a, b) => {
if (a === b) return 0; if (a === b) return 0;
if (b == null) { if (b == null) {
return 1; return 1;
@ -38,22 +35,11 @@ const compare = (a, b) => {
return 0; return 0;
}; };
/** const createDomainSorter = (gorhill: PublicSuffixList | null = null) => {
* @param {import('gorhill-publicsuffixlist').default | null} [gorhill]
*/
const createDomainSorter = (gorhill = null) => {
if (gorhill) { if (gorhill) {
/** const getDomain = createCachedGorhillGetDomain(gorhill);
* @param {string} input
*/
const getDomain = require('./cached-tld-parse').createCachedGorhillGetDomain(gorhill);
/** return (a: string, b: string) => {
* @param {string} a
* @param {string} b
* @returns {0 | 1 | -1}
*/
return (a, b) => {
if (a === b) return 0; if (a === b) return 0;
const aDomain = getDomain(a); const aDomain = getDomain(a);
@ -65,12 +51,8 @@ const createDomainSorter = (gorhill = null) => {
} }
const tldts = require('./cached-tld-parse'); const tldts = require('./cached-tld-parse');
/**
* @param {string} a return (a: string, b: string) => {
* @param {string} b
* @returns {0 | 1 | -1}
*/
return (a, b) => {
if (a === b) return 0; if (a === b) return 0;
const aDomain = tldts.parse(a).domain; const aDomain = tldts.parse(a).domain;
@ -81,5 +63,5 @@ const createDomainSorter = (gorhill = null) => {
}; };
}; };
module.exports = createDomainSorter(); export default createDomainSorter();
module.exports.createDomainSorter = createDomainSorter; export { createDomainSorter };

35
Build/lib/trace-runner.js → Build/lib/trace-runner.ts
View File

@ -1,44 +1,24 @@
// @ts-check import path from 'path';
const path = require('path');
const { performance } = require('perf_hooks');
/** const traceSync = <T>(prefix: string, fn: () => T): T => {
* @template T
* @param {string} prefix
* @param {() => T} fn
* @returns {T}
*/
const traceSync = (prefix, fn) => {
const start = performance.now(); const start = performance.now();
const result = fn(); const result = fn();
const end = performance.now(); const end = performance.now();
console.log(`${prefix}: ${(end - start).toFixed(3)}ms`); console.log(`${prefix}: ${(end - start).toFixed(3)}ms`);
return result; return result;
}; };
module.exports.traceSync = traceSync; export { traceSync };
/** const traceAsync = async <T>(prefix: string, fn: () => Promise<T>): Promise<T> => {
* @template T
* @param {string} prefix
* @param {() => Promise<T>} fn
* @returns {Promise<T>}
*/
const traceAsync = async (prefix, fn) => {
const start = performance.now(); const start = performance.now();
const result = await fn(); const result = await fn();
const end = performance.now(); const end = performance.now();
console.log(`${prefix}: ${(end - start).toFixed(3)}ms`); console.log(`${prefix}: ${(end - start).toFixed(3)}ms`);
return result; return result;
}; };
module.exports.traceAsync = traceAsync; export { traceAsync };
/** const task = <T>(__filename: string, fn: () => Promise<T>, customname: string | null = null) => {
* @template T
* @param {string} __filename
* @param {() => Promise<T>} fn
* @param {string | null} [customname]
*/
module.exports.task = (__filename, fn, customname = null) => {
const taskName = customname ?? path.basename(__filename, path.extname(__filename)); const taskName = customname ?? path.basename(__filename, path.extname(__filename));
return async () => { return async () => {
console.log(`🏃 [${taskName}] Start executing`); console.log(`🏃 [${taskName}] Start executing`);
@ -47,6 +27,7 @@ module.exports.task = (__filename, fn, customname = null) => {
const end = performance.now(); const end = performance.now();
console.log(`✅ [${taskName}] Executed successfully: ${(end - start).toFixed(3)}ms`); console.log(`✅ [${taskName}] Executed successfully: ${(end - start).toFixed(3)}ms`);
return { start, end, taskName }; return { start, end, taskName } as const;
}; };
}; };
export { task };

21
Build/validate-domainset.js → Build/validate-domainset.ts
View File

@ -1,12 +1,12 @@
// Surge Domain Set can not include root domain from public suffix list. // Surge Domain Set can not include root domain from public suffix list.
const tldts = require('tldts'); // hit ratio way too low, dont cache import * as tldts from 'tldts'; // hit ratio way too low, dont cache
const picocolors = require('picocolors'); import picocolors from 'picocolors';
const path = require('path'); import path from 'path';
const listDir = require('@sukka/listdir'); import listDir from '@sukka/listdir';
const { readFileByLine } = require('./lib/fetch-remote-text-by-line'); import { readFileByLine } from './lib/fetch-remote-text-by-line';
const { processLine } = require('./lib/process-line'); import { processLine } from './lib/process-line';
const { task } = require('./lib/trace-runner'); import { task } from './lib/trace-runner';
const SPECIAL_SUFFIXES = new Set([ const SPECIAL_SUFFIXES = new Set([
'linodeobjects.com', // only *.linodeobjects.com are public suffix 'linodeobjects.com', // only *.linodeobjects.com are public suffix
@ -14,7 +14,7 @@ const SPECIAL_SUFFIXES = new Set([
'dweb.link' // only *.dweb.link are public suffix 'dweb.link' // only *.dweb.link are public suffix
]); ]);
const validateDomainSet = async (filePath) => { const validateDomainSet = async (filePath: string) => {
for await (const l of readFileByLine(path.resolve(__dirname, '../List/domainset', filePath))) { for await (const l of readFileByLine(path.resolve(__dirname, '../List/domainset', filePath))) {
// starts with # // starts with #
const line = processLine(l); const line = processLine(l);
@ -35,7 +35,7 @@ const validateDomainSet = async (filePath) => {
} }
}; };
const _validateRuleset = async (filePath) => { const _validateRuleset = async (filePath: string) => {
console.log(`[${filePath}]`); console.log(`[${filePath}]`);
for await (const l of readFileByLine(path.resolve(__dirname, '../List/non_ip', filePath))) { for await (const l of readFileByLine(path.resolve(__dirname, '../List/non_ip', filePath))) {
@ -58,7 +58,7 @@ const _validateRuleset = async (filePath) => {
} }
}; };
const validate = task(__filename, async () => { export const validate = task(__filename, async () => {
// const [domainsetFiles, _rulesetFiles] = await Promise.all([ // const [domainsetFiles, _rulesetFiles] = await Promise.all([
// listDir(path.resolve(__dirname, '../List/domainset')), // listDir(path.resolve(__dirname, '../List/domainset')),
// listDir(path.resolve(__dirname, '../List/non_ip')) // listDir(path.resolve(__dirname, '../List/non_ip'))
@ -69,7 +69,6 @@ const validate = task(__filename, async () => {
// rulesetFiles.map(file => validateRuleset(file)) // rulesetFiles.map(file => validateRuleset(file))
]); ]);
}); });
module.exports.validate = validate;
if (import.meta.main) { if (import.meta.main) {
validate(); validate();

2
tsconfig.json
View File

@ -16,7 +16,7 @@
"skipLibCheck": true "skipLibCheck": true
}, },
"include": [ "include": [
"./Build/**/*.js", "./Source/**/*.js",
"./Build/**/*.ts" "./Build/**/*.ts"
] ]
} }