mirror of
https://github.com/SukkaW/Surge.git
synced 2025-12-12 01:00:34 +08:00
Refactor: continues to rewrite to TS
This commit is contained in:
SukkaW
parent
ec338a659f
commit
99589cf2fc
@ -1,10 +1,10 @@
|
||||
// @ts-check
|
||||
const path = require('path');
|
||||
const { isIPv4, isIPv6 } = require('net');
|
||||
const { createRuleset } = require('./lib/create-file');
|
||||
const { fetchRemoteTextAndCreateReadlineInterface, readFileByLine } = require('./lib/fetch-remote-text-by-line');
|
||||
const { processLine } = require('./lib/process-line');
|
||||
const { task } = require('./lib/trace-runner');
|
||||
import path from 'path';
|
||||
import { isIPv4, isIPv6 } from 'net';
|
||||
import { createRuleset } from './lib/create-file';
|
||||
import { fetchRemoteTextAndCreateReadlineInterface, readFileByLine } from './lib/fetch-remote-text-by-line';
|
||||
import { processLine } from './lib/process-line';
|
||||
import { task } from './lib/trace-runner';
|
||||
|
||||
const getBogusNxDomainIPs = async () => {
|
||||
/** @type {string[]} */
|
||||
@ -22,7 +22,7 @@ const getBogusNxDomainIPs = async () => {
|
||||
return result;
|
||||
};
|
||||
|
||||
const buildAntiBogusDomain = task(__filename, async () => {
|
||||
export const buildAntiBogusDomain = task(__filename, async () => {
|
||||
const bogusIpPromise = getBogusNxDomainIPs();
|
||||
|
||||
/** @type {string[]} */
|
||||
@ -61,8 +61,6 @@ const buildAntiBogusDomain = task(__filename, async () => {
|
||||
));
|
||||
});
|
||||
|
||||
module.exports.buildAntiBogusDomain = buildAntiBogusDomain;
|
||||
|
||||
if (import.meta.main) {
|
||||
buildAntiBogusDomain();
|
||||
}
|
||||
@ -1,9 +1,10 @@
|
||||
const path = require('path');
|
||||
const { createRuleset } = require('./lib/create-file');
|
||||
const { parseFelixDnsmasq } = require('./lib/parse-dnsmasq');
|
||||
const { task } = require('./lib/trace-runner');
|
||||
// @ts-check
|
||||
import path from 'path';
|
||||
import { createRuleset } from './lib/create-file';
|
||||
import { parseFelixDnsmasq } from './lib/parse-dnsmasq';
|
||||
import { task } from './lib/trace-runner';
|
||||
|
||||
const buildAppleCdn = task(__filename, async () => {
|
||||
export const buildAppleCdn = task(__filename, async () => {
|
||||
const res = await parseFelixDnsmasq('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf');
|
||||
|
||||
const description = [
|
||||
@ -42,8 +43,6 @@ const buildAppleCdn = task(__filename, async () => {
|
||||
]);
|
||||
});
|
||||
|
||||
module.exports.buildAppleCdn = buildAppleCdn;
|
||||
|
||||
if (import.meta.main) {
|
||||
buildAppleCdn();
|
||||
}
|
||||
@ -1,10 +1,8 @@
|
||||
// @ts-check
|
||||
const { fetchRemoteTextAndCreateReadlineInterface } = require('./lib/fetch-remote-text-by-line');
|
||||
const { resolve: pathResolve } = require('path');
|
||||
// This should not use `createRuleset` API since we are going to generate ipcidr for Clash
|
||||
const { compareAndWriteFile, withBannerArray } = require('./lib/create-file');
|
||||
const { processLineFromReadline } = require('./lib/process-line');
|
||||
const { task } = require('./lib/trace-runner');
|
||||
import { fetchRemoteTextAndCreateReadlineInterface } from './lib/fetch-remote-text-by-line';
|
||||
import { resolve as pathResolve } from 'path';
|
||||
import { compareAndWriteFile, withBannerArray } from './lib/create-file';
|
||||
import { processLineFromReadline } from './lib/process-line';
|
||||
import { task } from './lib/trace-runner';
|
||||
|
||||
// https://github.com/misakaio/chnroutes2/issues/25
|
||||
const EXCLUDE_CIDRS = [
|
||||
@ -12,7 +10,7 @@ const EXCLUDE_CIDRS = [
|
||||
'223.120.0.0/15'
|
||||
];
|
||||
|
||||
const buildChnCidr = task(__filename, async () => {
|
||||
export const buildChnCidr = task(__filename, async () => {
|
||||
const [{ exclude: excludeCidrs }, cidr] = await Promise.all([
|
||||
import('cidr-tools-wasm'),
|
||||
processLineFromReadline(await fetchRemoteTextAndCreateReadlineInterface('https://raw.githubusercontent.com/misakaio/chnroutes2/master/chnroutes.txt'))
|
||||
@ -50,8 +48,6 @@ const buildChnCidr = task(__filename, async () => {
|
||||
]);
|
||||
});
|
||||
|
||||
module.exports.buildChnCidr = buildChnCidr;
|
||||
|
||||
if (import.meta.main) {
|
||||
buildChnCidr();
|
||||
}
|
||||
@ -1,12 +1,12 @@
|
||||
// @ts-check
|
||||
|
||||
const path = require('path');
|
||||
const { PathScurry } = require('path-scurry');
|
||||
const { readFileByLine } = require('./lib/fetch-remote-text-by-line');
|
||||
const { processLine } = require('./lib/process-line');
|
||||
const { createRuleset } = require('./lib/create-file');
|
||||
const { domainDeduper } = require('./lib/domain-deduper');
|
||||
const { task } = require('./lib/trace-runner');
|
||||
import * as path from 'path';
|
||||
import { PathScurry } from 'path-scurry';
|
||||
import { readFileByLine } from './lib/fetch-remote-text-by-line';
|
||||
import { processLine } from './lib/process-line';
|
||||
import { createRuleset } from './lib/create-file';
|
||||
import { domainDeduper } from './lib/domain-deduper';
|
||||
import { task } from './lib/trace-runner';
|
||||
|
||||
const MAGIC_COMMAND_SKIP = '# $ custom_build_script';
|
||||
const MAGIC_COMMAND_TITLE = '# $ meta_title ';
|
||||
@ -16,7 +16,7 @@ const sourceDir = path.resolve(__dirname, '../Source');
|
||||
const outputSurgeDir = path.resolve(__dirname, '../List');
|
||||
const outputClashDir = path.resolve(__dirname, '../Clash');
|
||||
|
||||
const buildCommon = task(__filename, async () => {
|
||||
export const buildCommon = task(__filename, async () => {
|
||||
/** @type {Promise<unknown>[]} */
|
||||
const promises = [];
|
||||
|
||||
@ -45,16 +45,11 @@ const buildCommon = task(__filename, async () => {
|
||||
return Promise.all(promises);
|
||||
});
|
||||
|
||||
module.exports.buildCommon = buildCommon;
|
||||
|
||||
if (import.meta.main) {
|
||||
buildCommon();
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} sourcePath
|
||||
*/
|
||||
const processFile = async (sourcePath) => {
|
||||
const processFile = async (sourcePath: string) => {
|
||||
/** @type {string[]} */
|
||||
const lines = [];
|
||||
|
||||
@ -83,14 +78,10 @@ const processFile = async (sourcePath) => {
|
||||
}
|
||||
}
|
||||
|
||||
return /** @type {const} */ ([title, descriptions, lines]);
|
||||
return [title, descriptions, lines] as const;
|
||||
};
|
||||
|
||||
/**
|
||||
* @param {string} sourcePath
|
||||
* @param {string} relativePath
|
||||
*/
|
||||
async function transformDomainset(sourcePath, relativePath) {
|
||||
async function transformDomainset(sourcePath: string, relativePath: string) {
|
||||
const res = await processFile(sourcePath);
|
||||
if (!res) return;
|
||||
const [title, descriptions, lines] = res;
|
||||
@ -120,11 +111,8 @@ async function transformDomainset(sourcePath, relativePath) {
|
||||
|
||||
/**
|
||||
* Output Surge RULE-SET and Clash classical text format
|
||||
*
|
||||
* @param {string} sourcePath
|
||||
* @param {string} relativePath
|
||||
*/
|
||||
async function transformRuleset(sourcePath, relativePath) {
|
||||
async function transformRuleset(sourcePath: string, relativePath: string) {
|
||||
const res = await processFile(sourcePath);
|
||||
if (!res) return;
|
||||
const [title, descriptions, lines] = res;
|
||||
@ -1,22 +1,22 @@
|
||||
// @ts-check
|
||||
const path = require('path');
|
||||
const { DOMESTICS } = require('../Source/non_ip/domestic');
|
||||
const { readFileByLine } = require('./lib/fetch-remote-text-by-line');
|
||||
const { processLineFromReadline } = require('./lib/process-line');
|
||||
const { compareAndWriteFile, createRuleset } = require('./lib/create-file');
|
||||
const { task } = require('./lib/trace-runner');
|
||||
import path from 'path';
|
||||
import { DOMESTICS } from '../Source/non_ip/domestic';
|
||||
import { readFileByLine } from './lib/fetch-remote-text-by-line';
|
||||
import { processLineFromReadline } from './lib/process-line';
|
||||
import { compareAndWriteFile, createRuleset } from './lib/create-file';
|
||||
import { task } from './lib/trace-runner';
|
||||
|
||||
const buildDomesticRuleset = task(__filename, async () => {
|
||||
export const buildDomesticRuleset = task(__filename, async () => {
|
||||
const results = await processLineFromReadline(readFileByLine(path.resolve(__dirname, '../Source/non_ip/domestic.conf')));
|
||||
|
||||
results.push(
|
||||
...Object.entries(DOMESTICS)
|
||||
.reduce(
|
||||
.reduce<string[]>(
|
||||
(acc, [key, { domains }]) => {
|
||||
if (key === 'SYSTEM') return acc;
|
||||
return [...acc, ...domains];
|
||||
},
|
||||
/** @type {string[]} */([])
|
||||
[]
|
||||
)
|
||||
.map((domain) => `DOMAIN-SUFFIX,${domain}`)
|
||||
);
|
||||
@ -58,8 +58,6 @@ const buildDomesticRuleset = task(__filename, async () => {
|
||||
]);
|
||||
});
|
||||
|
||||
module.exports.buildDomesticRuleset = buildDomesticRuleset;
|
||||
|
||||
if (import.meta.main) {
|
||||
buildDomesticRuleset();
|
||||
}
|
||||
@ -1,28 +1,25 @@
|
||||
// @ts-check
|
||||
const fsp = require('fs/promises');
|
||||
const path = require('path');
|
||||
const tldts = require('tldts');
|
||||
const { processLine } = require('./lib/process-line');
|
||||
const { readFileByLine } = require('./lib/fetch-remote-text-by-line');
|
||||
const { createDomainSorter } = require('./lib/stable-sort-domain');
|
||||
const { task } = require('./lib/trace-runner');
|
||||
const { compareAndWriteFile } = require('./lib/create-file');
|
||||
const { getGorhillPublicSuffixPromise } = require('./lib/get-gorhill-publicsuffix');
|
||||
import fsp from 'fs/promises'
|
||||
import path from 'path';
|
||||
import * as tldts from 'tldts';
|
||||
import { processLine } from './lib/process-line';
|
||||
import { readFileByLine } from './lib/fetch-remote-text-by-line';
|
||||
import { createDomainSorter } from './lib/stable-sort-domain';
|
||||
import { task } from './lib/trace-runner';
|
||||
import { compareAndWriteFile } from './lib/create-file';
|
||||
import { getGorhillPublicSuffixPromise } from './lib/get-gorhill-publicsuffix';
|
||||
// const { createCachedGorhillGetDomain } = require('./lib/cached-tld-parse');
|
||||
|
||||
const escapeRegExp = (string = '') => string.replaceAll(/[$()*+.?[\\\]^{|}]/g, '\\$&');
|
||||
|
||||
const buildInternalCDNDomains = task(__filename, async () => {
|
||||
const set = new Set();
|
||||
export const buildInternalCDNDomains = task(__filename, async () => {
|
||||
const set = new Set<string>();
|
||||
const keywords = new Set();
|
||||
|
||||
const gorhill = await getGorhillPublicSuffixPromise();
|
||||
const domainSorter = createDomainSorter(gorhill);
|
||||
|
||||
/**
|
||||
* @param {string} input
|
||||
*/
|
||||
const addApexDomain = (input) => {
|
||||
const addApexDomain = (input: string) => {
|
||||
// We are including the private domains themselves
|
||||
const d = tldts.getDomain(input, { allowPrivateDomains: false });
|
||||
if (d) {
|
||||
@ -30,10 +27,7 @@ const buildInternalCDNDomains = task(__filename, async () => {
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* @param {string} domainSetPath
|
||||
*/
|
||||
const processLocalDomainSet = async (domainSetPath) => {
|
||||
const processLocalDomainSet = async (domainSetPath: string) => {
|
||||
for await (const line of readFileByLine(domainSetPath)) {
|
||||
// console.log({ line });
|
||||
|
||||
@ -52,10 +46,7 @@ const buildInternalCDNDomains = task(__filename, async () => {
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* @param {string} ruleSetPath
|
||||
*/
|
||||
const processLocalRuleSet = async (ruleSetPath) => {
|
||||
const processLocalRuleSet = async (ruleSetPath: string) => {
|
||||
for await (const line of readFileByLine(ruleSetPath)) {
|
||||
if (line.startsWith('DOMAIN-SUFFIX,')) {
|
||||
addApexDomain(line.replace('DOMAIN-SUFFIX,', ''));
|
||||
@ -93,8 +84,6 @@ const buildInternalCDNDomains = task(__filename, async () => {
|
||||
);
|
||||
});
|
||||
|
||||
module.exports.buildInternalCDNDomains = buildInternalCDNDomains;
|
||||
|
||||
if (import.meta.main) {
|
||||
buildInternalCDNDomains();
|
||||
}
|
||||
@ -1,11 +1,10 @@
|
||||
// @ts-check
|
||||
const path = require('path');
|
||||
const fsp = require('fs/promises');
|
||||
const { parseFelixDnsmasq } = require('./lib/parse-dnsmasq');
|
||||
const { task } = require('./lib/trace-runner');
|
||||
const { compareAndWriteFile } = require('./lib/create-file');
|
||||
import path from 'path';
|
||||
import fsp from 'fs/promises'
|
||||
import { parseFelixDnsmasq } from './lib/parse-dnsmasq';
|
||||
import { task } from './lib/trace-runner';
|
||||
import { compareAndWriteFile } from './lib/create-file';
|
||||
|
||||
const buildInternalChnDomains = task(__filename, async () => {
|
||||
export const buildInternalChnDomains = task(__filename, async () => {
|
||||
const [result] = await Promise.all([
|
||||
parseFelixDnsmasq('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf'),
|
||||
fsp.mkdir(path.resolve(__dirname, '../List/internal'), { recursive: true })
|
||||
@ -17,8 +16,6 @@ const buildInternalChnDomains = task(__filename, async () => {
|
||||
);
|
||||
});
|
||||
|
||||
module.exports.buildInternalChnDomains = buildInternalChnDomains;
|
||||
|
||||
if (import.meta.main) {
|
||||
buildInternalChnDomains();
|
||||
}
|
||||
@ -1,9 +1,8 @@
|
||||
// @ts-check
|
||||
const { fetchRemoteTextAndCreateReadlineInterface } = require('./lib/fetch-remote-text-by-line');
|
||||
const { processLineFromReadline } = require('./lib/process-line');
|
||||
const path = require('path');
|
||||
const fsp = require('fs/promises');
|
||||
const { task } = require('./lib/trace-runner');
|
||||
import { fetchRemoteTextAndCreateReadlineInterface } from './lib/fetch-remote-text-by-line';
|
||||
import { processLineFromReadline } from './lib/process-line';
|
||||
import path from 'path';
|
||||
import fsp from 'fs/promises'
|
||||
import { task } from './lib/trace-runner';
|
||||
|
||||
const RESERVED_IPV4_CIDR = [
|
||||
'0.0.0.0/8',
|
||||
@ -23,7 +22,7 @@ const RESERVED_IPV4_CIDR = [
|
||||
'240.0.0.0/4'
|
||||
];
|
||||
|
||||
const buildInternalReverseChnCIDR = task(__filename, async () => {
|
||||
export const buildInternalReverseChnCIDR = task(__filename, async () => {
|
||||
const [{ exclude }, cidr] = await Promise.all([
|
||||
import('cidr-tools-wasm'),
|
||||
processLineFromReadline(await fetchRemoteTextAndCreateReadlineInterface('https://raw.githubusercontent.com/misakaio/chnroutes2/master/chnroutes.txt')),
|
||||
@ -44,8 +43,6 @@ const buildInternalReverseChnCIDR = task(__filename, async () => {
|
||||
return Bun.write(path.resolve(__dirname, '../List/internal/reversed-chn-cidr.txt'), `${reversedCidr.join('\n')}\n`);
|
||||
});
|
||||
|
||||
module.exports.buildInternalReverseChnCIDR = buildInternalReverseChnCIDR;
|
||||
|
||||
if (import.meta.main) {
|
||||
buildInternalReverseChnCIDR();
|
||||
}
|
||||
@ -1,7 +1,7 @@
|
||||
const fsPromises = require('fs').promises;
|
||||
const pathFn = require('path');
|
||||
const table = require('table');
|
||||
const listDir = require('@sukka/listdir');
|
||||
import listDir from '@sukka/listdir';
|
||||
const { green, yellow } = require('picocolors');
|
||||
|
||||
const PRESET_MITM_HOSTNAMES = [
|
||||
|
||||
@ -1,14 +1,13 @@
|
||||
// @ts-check
|
||||
const { processFilterRules } = require('./lib/parse-filter.js');
|
||||
const path = require('path');
|
||||
const { createRuleset } = require('./lib/create-file');
|
||||
const { processLine } = require('./lib/process-line.js');
|
||||
const { createDomainSorter } = require('./lib/stable-sort-domain');
|
||||
const { traceSync, task } = require('./lib/trace-runner.js');
|
||||
const createTrie = require('./lib/trie.js');
|
||||
const { getGorhillPublicSuffixPromise } = require('./lib/get-gorhill-publicsuffix.js');
|
||||
const { createCachedGorhillGetDomain } = require('./lib/cached-tld-parse.js');
|
||||
const tldts = require('tldts');
|
||||
import { processFilterRules } from './lib/parse-filter';
|
||||
import path from 'path';
|
||||
import { createRuleset } from './lib/create-file';
|
||||
import { processLine } from './lib/process-line';
|
||||
import { createDomainSorter } from './lib/stable-sort-domain';
|
||||
import { traceSync, task } from './lib/trace-runner';
|
||||
import createTrie from './lib/trie';
|
||||
import { getGorhillPublicSuffixPromise } from './lib/get-gorhill-publicsuffix';
|
||||
import { createCachedGorhillGetDomain } from './lib/cached-tld-parse';
|
||||
import * as tldts from 'tldts';
|
||||
|
||||
const WHITELIST_DOMAIN = new Set([
|
||||
'w3s.link',
|
||||
@ -64,7 +63,7 @@ const BLACK_TLD = new Set([
|
||||
'com.cn'
|
||||
]);
|
||||
|
||||
const buildPhishingDomainSet = task(__filename, async () => {
|
||||
export const buildPhishingDomainSet = task(__filename, async () => {
|
||||
const [{ black: domainSet }, gorhill] = await Promise.all([
|
||||
processFilterRules(
|
||||
'https://curbengh.github.io/phishing-filter/phishing-filter-agh.txt',
|
||||
@ -87,7 +86,7 @@ const buildPhishingDomainSet = task(__filename, async () => {
|
||||
});
|
||||
});
|
||||
|
||||
const domainCountMap = {};
|
||||
const domainCountMap: Record<string, number> = {};
|
||||
const getDomain = createCachedGorhillGetDomain(gorhill);
|
||||
|
||||
traceSync('* process domain set', () => {
|
||||
@ -154,12 +153,12 @@ const buildPhishingDomainSet = task(__filename, async () => {
|
||||
const domainSorter = createDomainSorter(gorhill);
|
||||
|
||||
const results = traceSync('* get final results', () => Object.entries(domainCountMap)
|
||||
.reduce((acc, [apexDomain, count]) => {
|
||||
.reduce<string[]>((acc, [apexDomain, count]) => {
|
||||
if (count >= 5) {
|
||||
acc.push(`.${apexDomain}`);
|
||||
}
|
||||
return acc;
|
||||
}, /** @type {string[]} */([]))
|
||||
}, [])
|
||||
.sort(domainSorter));
|
||||
|
||||
const description = [
|
||||
@ -183,8 +182,6 @@ const buildPhishingDomainSet = task(__filename, async () => {
|
||||
));
|
||||
});
|
||||
|
||||
module.exports.buildPhishingDomainSet = buildPhishingDomainSet;
|
||||
|
||||
if (import.meta.main) {
|
||||
buildPhishingDomainSet();
|
||||
}
|
||||
@ -1,7 +1,7 @@
|
||||
const listDir = require('@sukka/listdir');
|
||||
const path = require('path');
|
||||
const fsp = require('fs/promises');
|
||||
const { task } = require('./lib/trace-runner');
|
||||
import listDir from '@sukka/listdir';
|
||||
import path from 'path';
|
||||
import fsp from 'fs/promises'
|
||||
import { task } from './lib/trace-runner';
|
||||
|
||||
const rootPath = path.resolve(__dirname, '../');
|
||||
const publicPath = path.resolve(__dirname, '../public');
|
||||
@ -15,7 +15,7 @@ const folderAndFilesToBeDeployed = [
|
||||
'LICENSE'
|
||||
];
|
||||
|
||||
const buildPublicHtml = task(__filename, async () => {
|
||||
export const buildPublicHtml = task(__filename, async () => {
|
||||
await fsp.mkdir(publicPath, { recursive: true });
|
||||
await Promise.all(folderAndFilesToBeDeployed.map(dir => fsp.cp(
|
||||
path.resolve(rootPath, dir),
|
||||
@ -33,17 +33,11 @@ const buildPublicHtml = task(__filename, async () => {
|
||||
return Bun.write(path.join(publicPath, 'index.html'), html);
|
||||
});
|
||||
|
||||
module.exports.buildPublicHtml = buildPublicHtml;
|
||||
|
||||
if (import.meta.main) {
|
||||
buildPublicHtml();
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string[]} urlList
|
||||
* @returns {string}
|
||||
*/
|
||||
function template(urlList) {
|
||||
function template(urlList: string[]) {
|
||||
return `
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
@ -1,9 +1,7 @@
|
||||
// @ts-check
|
||||
|
||||
const path = require('path');
|
||||
const { task } = require('./lib/trace-runner');
|
||||
const { compareAndWriteFile } = require('./lib/create-file');
|
||||
const tldts = require('tldts');
|
||||
import path from 'path';
|
||||
import { task } from './lib/trace-runner';
|
||||
import { compareAndWriteFile } from './lib/create-file';
|
||||
import * as tldts from 'tldts';
|
||||
|
||||
function escapeRegExp(string = '') {
|
||||
const reRegExpChar = /[$()*+.?[\\\]^{|}]/g;
|
||||
@ -73,7 +71,7 @@ const REDIRECT = /** @type {const} */ ([
|
||||
['googleajax.wp-china-yes.net/', 'https://ajax.googleapis.com/']
|
||||
]);
|
||||
|
||||
const buildRedirectModule = task(__filename, async () => {
|
||||
export const buildRedirectModule = task(__filename, async () => {
|
||||
const domains = Array.from(new Set(REDIRECT.map(([from]) => tldts.getHostname(from, { detectIp: false })))).filter(Boolean);
|
||||
|
||||
return compareAndWriteFile(
|
||||
@ -94,8 +92,6 @@ const buildRedirectModule = task(__filename, async () => {
|
||||
);
|
||||
});
|
||||
|
||||
module.exports.buildRedirectModule = buildRedirectModule;
|
||||
|
||||
if (import.meta.main) {
|
||||
buildRedirectModule();
|
||||
}
|
||||
@ -1,31 +1,31 @@
|
||||
// @ts-check
|
||||
const fsp = require('fs/promises');
|
||||
const path = require('path');
|
||||
import fsp from 'fs/promises'
|
||||
import path from 'path';
|
||||
|
||||
const { processHosts, processFilterRules } = require('./lib/parse-filter');
|
||||
const createTrie = require('./lib/trie');
|
||||
import { processHosts, processFilterRules } from './lib/parse-filter';
|
||||
import createTrie from './lib/trie';
|
||||
|
||||
const { HOSTS, ADGUARD_FILTERS, PREDEFINED_WHITELIST, PREDEFINED_ENFORCED_BACKLIST } = require('./lib/reject-data-source');
|
||||
const { createRuleset, compareAndWriteFile } = require('./lib/create-file');
|
||||
const { processLine } = require('./lib/process-line');
|
||||
const { domainDeduper } = require('./lib/domain-deduper');
|
||||
const createKeywordFilter = require('./lib/aho-corasick');
|
||||
const { readFileByLine } = require('./lib/fetch-remote-text-by-line');
|
||||
const { createDomainSorter } = require('./lib/stable-sort-domain');
|
||||
const { traceSync, task } = require('./lib/trace-runner');
|
||||
const { getGorhillPublicSuffixPromise } = require('./lib/get-gorhill-publicsuffix');
|
||||
const tldts = require('tldts');
|
||||
import { HOSTS, ADGUARD_FILTERS, PREDEFINED_WHITELIST, PREDEFINED_ENFORCED_BACKLIST } from './lib/reject-data-source';
|
||||
import { createRuleset, compareAndWriteFile } from './lib/create-file';
|
||||
import { processLine } from './lib/process-line';
|
||||
import { domainDeduper } from './lib/domain-deduper';
|
||||
import createKeywordFilter from './lib/aho-corasick';
|
||||
import { readFileByLine } from './lib/fetch-remote-text-by-line';
|
||||
import { createDomainSorter } from './lib/stable-sort-domain';
|
||||
import { traceSync, task } from './lib/trace-runner';
|
||||
import { getGorhillPublicSuffixPromise } from './lib/get-gorhill-publicsuffix';
|
||||
import * as tldts from 'tldts';
|
||||
|
||||
/** Whitelists */
|
||||
const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST);
|
||||
/** @type {Set<string>} Dedupe domains inclued by DOMAIN-KEYWORD */
|
||||
const domainKeywordsSet = new Set();
|
||||
const domainKeywordsSet: Set<string> = new Set();
|
||||
/** @type {Set<string>} Dedupe domains included by DOMAIN-SUFFIX */
|
||||
const domainSuffixSet = new Set();
|
||||
const domainSuffixSet: Set<string> = new Set();
|
||||
|
||||
const buildRejectDomainSet = task(__filename, async () => {
|
||||
export const buildRejectDomainSet = task(__filename, async () => {
|
||||
/** @type Set<string> */
|
||||
const domainSets = new Set();
|
||||
const domainSets: Set<string> = new Set();
|
||||
|
||||
// Parse from AdGuard Filters
|
||||
console.time('* Download and process Hosts / AdBlock Filter Rules');
|
||||
@ -172,11 +172,10 @@ const buildRejectDomainSet = task(__filename, async () => {
|
||||
console.log(`Deduped ${previousSize - dudupedDominArray.length} rules!`);
|
||||
|
||||
// Create reject stats
|
||||
/** @type {[string, number][]} */
|
||||
const rejectDomainsStats = traceSync(
|
||||
const rejectDomainsStats: [string, number][] = traceSync(
|
||||
'* Collect reject domain stats',
|
||||
() => Object.entries(
|
||||
dudupedDominArray.reduce((acc, cur) => {
|
||||
dudupedDominArray.reduce<Record<string, number>>((acc, cur) => {
|
||||
const suffix = tldts.getDomain(cur, { allowPrivateDomains: false, detectIp: false });
|
||||
if (suffix) {
|
||||
acc[suffix] = (acc[suffix] ?? 0) + 1;
|
||||
@ -231,8 +230,6 @@ const buildRejectDomainSet = task(__filename, async () => {
|
||||
]);
|
||||
});
|
||||
|
||||
module.exports.buildRejectDomainSet = buildRejectDomainSet;
|
||||
|
||||
if (import.meta.main) {
|
||||
buildRejectDomainSet();
|
||||
}
|
||||
@ -1,18 +1,14 @@
|
||||
const { domainDeduper } = require('./lib/domain-deduper');
|
||||
const path = require('path');
|
||||
const { createRuleset } = require('./lib/create-file');
|
||||
const domainSorter = require('./lib/stable-sort-domain');
|
||||
import { domainDeduper } from './lib/domain-deduper';
|
||||
import path from 'path';
|
||||
import { createRuleset } from './lib/create-file';
|
||||
import domainSorter from './lib/stable-sort-domain';
|
||||
|
||||
const { Sema } = require('async-sema');
|
||||
const tldts = require('tldts');
|
||||
const { task } = require('./lib/trace-runner');
|
||||
import { Sema } from 'async-sema';
|
||||
import * as tldts from 'tldts';
|
||||
import { task } from './lib/trace-runner';
|
||||
const s = new Sema(2);
|
||||
|
||||
/**
|
||||
* @param {string} keyword
|
||||
* @returns {string[]}
|
||||
*/
|
||||
const querySpeedtestApi = async (keyword) => {
|
||||
const querySpeedtestApi = async (keyword: string): Promise<(string | null)[]> => {
|
||||
await s.acquire();
|
||||
|
||||
try {
|
||||
@ -36,19 +32,19 @@ const querySpeedtestApi = async (keyword) => {
|
||||
throw new Error(text);
|
||||
}
|
||||
|
||||
/** @type {{ url: string }[]} */
|
||||
const json = await res.json();
|
||||
const json = await res.json() as { url: string; }[];
|
||||
s.release();
|
||||
return json.map(({ url }) => tldts.getHostname(url, { detectIp: false }));
|
||||
} catch (e) {
|
||||
s.release();
|
||||
console.log(e);
|
||||
return [];
|
||||
}
|
||||
};
|
||||
|
||||
const buildSpeedtestDomainSet = task(__filename, async () => {
|
||||
export const buildSpeedtestDomainSet = task(__filename, async () => {
|
||||
/** @type {Set<string>} */
|
||||
const domains = new Set([
|
||||
const domains: Set<string> = new Set([
|
||||
'.speedtest.net',
|
||||
'.speedtestcustom.com',
|
||||
'.ooklaserver.net',
|
||||
@ -140,8 +136,6 @@ const buildSpeedtestDomainSet = task(__filename, async () => {
|
||||
));
|
||||
});
|
||||
|
||||
module.exports.buildSpeedtestDomainSet = buildSpeedtestDomainSet;
|
||||
|
||||
if (import.meta.main) {
|
||||
buildSpeedtestDomainSet();
|
||||
}
|
||||
@ -1,20 +1,12 @@
|
||||
// @ts-check
|
||||
const { task } = require('./lib/trace-runner');
|
||||
import { task } from './lib/trace-runner';
|
||||
|
||||
const path = require('path');
|
||||
const { createRuleset } = require('./lib/create-file');
|
||||
import path from 'path';
|
||||
import { createRuleset } from './lib/create-file';
|
||||
|
||||
const {
|
||||
ALL, NORTH_AMERICA, EU, HK, TW, JP, KR
|
||||
// SOUTH_EAST_ASIA, AU
|
||||
} = require('../Source/stream');
|
||||
import { ALL, NORTH_AMERICA, EU, HK, TW, JP, KR } from '../Source/stream';
|
||||
|
||||
/**
|
||||
* @param {string} fileId
|
||||
* @param {string} title
|
||||
* @param {import('../Source/stream').StreamService[]} streamServices
|
||||
*/
|
||||
const createRulesetForStreamService = (fileId, title, streamServices) => {
|
||||
const createRulesetForStreamService = (fileId: string, title: string, streamServices: import('../Source/stream').StreamService[]) => {
|
||||
return [
|
||||
// Domains
|
||||
...createRuleset(
|
||||
@ -24,10 +16,10 @@ const createRulesetForStreamService = (fileId, title, streamServices) => {
|
||||
'Homepage: https://ruleset.skk.moe',
|
||||
'GitHub: https://github.com/SukkaW/Surge',
|
||||
'',
|
||||
...streamServices.map(i => `- ${i.name}`)
|
||||
...streamServices.map((i: { name: any; }) => `- ${i.name}`)
|
||||
],
|
||||
new Date(),
|
||||
streamServices.flatMap(i => i.rules),
|
||||
streamServices.flatMap((i: { rules: any; }) => i.rules),
|
||||
'ruleset',
|
||||
path.resolve(__dirname, `../List/non_ip/${fileId}.conf`),
|
||||
path.resolve(__dirname, `../Clash/non_ip/${fileId}.txt`)
|
||||
@ -40,14 +32,14 @@ const createRulesetForStreamService = (fileId, title, streamServices) => {
|
||||
'Homepage: https://ruleset.skk.moe',
|
||||
'GitHub: https://github.com/SukkaW/Surge',
|
||||
'',
|
||||
...streamServices.map(i => `- ${i.name}`)
|
||||
...streamServices.map((i: { name: any; }) => `- ${i.name}`)
|
||||
],
|
||||
new Date(),
|
||||
streamServices.flatMap(i => (
|
||||
streamServices.flatMap((i) => (
|
||||
i.ip
|
||||
? [
|
||||
...i.ip.v4.map(ip => `IP-CIDR,${ip},no-resolve`),
|
||||
...i.ip.v6.map(ip => `IP-CIDR6,${ip},no-resolve`)
|
||||
...i.ip.v4.map((ip: any) => `IP-CIDR,${ip},no-resolve`),
|
||||
...i.ip.v6.map((ip: any) => `IP-CIDR6,${ip},no-resolve`)
|
||||
]
|
||||
: []
|
||||
)),
|
||||
@ -58,7 +50,7 @@ const createRulesetForStreamService = (fileId, title, streamServices) => {
|
||||
];
|
||||
};
|
||||
|
||||
const buildStreamService = task(__filename, async () => {
|
||||
export const buildStreamService = task(__filename, async () => {
|
||||
return Promise.all([
|
||||
...createRulesetForStreamService('stream', 'All', ALL),
|
||||
...createRulesetForStreamService('stream_us', 'North America', NORTH_AMERICA),
|
||||
@ -72,8 +64,6 @@ const buildStreamService = task(__filename, async () => {
|
||||
]);
|
||||
});
|
||||
|
||||
module.exports.buildStreamService = buildStreamService;
|
||||
|
||||
if (import.meta.main) {
|
||||
buildStreamService();
|
||||
}
|
||||
@ -1,13 +1,13 @@
|
||||
// @ts-check
|
||||
const { fetchWithRetry } = require('./lib/fetch-retry');
|
||||
const { createReadlineInterfaceFromResponse } = require('./lib/fetch-remote-text-by-line');
|
||||
const path = require('path');
|
||||
const { isIPv4, isIPv6 } = require('net');
|
||||
const { processLine } = require('./lib/process-line');
|
||||
const { createRuleset } = require('./lib/create-file');
|
||||
const { task } = require('./lib/trace-runner');
|
||||
import { fetchWithRetry } from './lib/fetch-retry';
|
||||
import { createReadlineInterfaceFromResponse } from './lib/fetch-remote-text-by-line';
|
||||
import path from 'path';
|
||||
import { isIPv4, isIPv6 } from 'net';
|
||||
import { processLine } from './lib/process-line';
|
||||
import { createRuleset } from './lib/create-file';
|
||||
import { task } from './lib/trace-runner';
|
||||
|
||||
const buildTelegramCIDR = task(__filename, async () => {
|
||||
export const buildTelegramCIDR = task(__filename, async () => {
|
||||
/** @type {Response} */
|
||||
const resp = await fetchWithRetry('https://core.telegram.org/resources/cidr.txt');
|
||||
const lastModified = resp.headers.get('last-modified');
|
||||
@ -52,8 +52,6 @@ const buildTelegramCIDR = task(__filename, async () => {
|
||||
));
|
||||
});
|
||||
|
||||
module.exports.buildTelegramCIDR = buildTelegramCIDR;
|
||||
|
||||
if (import.meta.main) {
|
||||
buildTelegramCIDR();
|
||||
}
|
||||
@ -1,19 +1,15 @@
|
||||
/**
|
||||
* @typedef {Object} Node
|
||||
* @prop {number} [depth = 0]
|
||||
* @prop {string} key
|
||||
* @prop {boolean} [word = false]
|
||||
* @prop {Record<string, Node>} [children={}]
|
||||
* @prop {Node} [fail]
|
||||
* @prop {number} [count=0]
|
||||
*/
|
||||
interface Node {
|
||||
/** @default 0 */
|
||||
depth?: number;
|
||||
key: string;
|
||||
/** @default false */
|
||||
word?: boolean;
|
||||
children: Record<string, Node>;
|
||||
fail?: Node;
|
||||
count: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} key
|
||||
* @param {number} depth
|
||||
* @returns {Node}
|
||||
*/
|
||||
const createNode = (key, depth = 0) => ({
|
||||
const createNode = (key: string, depth = 0): Node => ({
|
||||
depth,
|
||||
key,
|
||||
word: false,
|
||||
@ -22,15 +18,11 @@ const createNode = (key, depth = 0) => ({
|
||||
count: 0
|
||||
});
|
||||
|
||||
/**
|
||||
* @param {string[] | Set<string>} keys
|
||||
*/
|
||||
const createKeywordFilter = (keys) => {
|
||||
const createKeywordFilter = (keys: string[] | Set<string>) => {
|
||||
const root = createNode('root');
|
||||
|
||||
const build = () => {
|
||||
/** @type {Node[]} */
|
||||
const queue = [];
|
||||
const queue: Node[] = [];
|
||||
queue.push(root);
|
||||
|
||||
let idx = 0;
|
||||
@ -57,11 +49,7 @@ const createKeywordFilter = (keys) => {
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* @param {string} key
|
||||
* @param {number} len
|
||||
*/
|
||||
const put = (key, len) => {
|
||||
const put = (key: string, len: number) => {
|
||||
let node = root;
|
||||
const lastIdx = len - 1;
|
||||
node.count++;
|
||||
@ -91,12 +79,8 @@ const createKeywordFilter = (keys) => {
|
||||
|
||||
build();
|
||||
|
||||
/**
|
||||
* @param {string} text
|
||||
* @returns {boolean}
|
||||
*/
|
||||
const search = (text) => {
|
||||
let node = root;
|
||||
const search = (text: string) => {
|
||||
let node: Node | undefined = root;
|
||||
|
||||
for (let i = 0, textLen = text.length; i < textLen; i++) {
|
||||
// const key = text.charAt(i);
|
||||
@ -120,4 +104,4 @@ const createKeywordFilter = (keys) => {
|
||||
};
|
||||
};
|
||||
|
||||
module.exports = createKeywordFilter;
|
||||
export default createKeywordFilter;
|
||||
@ -1,7 +1,4 @@
|
||||
/**
|
||||
* @param {string} [namespace]
|
||||
*/
|
||||
const createCache = (namespace, printStats = false) => {
|
||||
export const createCache = (namespace?: string, printStats = false) => {
|
||||
const cache = new Map();
|
||||
|
||||
let hit = 0;
|
||||
@ -12,13 +9,7 @@ const createCache = (namespace, printStats = false) => {
|
||||
}
|
||||
|
||||
return {
|
||||
/**
|
||||
* @template T
|
||||
* @param {string} key
|
||||
* @param {() => T} fn
|
||||
* @returns {T}
|
||||
*/
|
||||
sync(key, fn) {
|
||||
sync<T>(key: string, fn: () => T): T {
|
||||
if (cache.has(key)) {
|
||||
hit++;
|
||||
return cache.get(key);
|
||||
@ -27,13 +18,7 @@ const createCache = (namespace, printStats = false) => {
|
||||
cache.set(key, value);
|
||||
return value;
|
||||
},
|
||||
/**
|
||||
* @template T
|
||||
* @param {string} key
|
||||
* @param {() => Promise<T>} fn
|
||||
* @returns {Promise<T>}
|
||||
*/
|
||||
async async(key, fn) {
|
||||
async async<T>(key: string, fn: () => Promise<T>): Promise<T> {
|
||||
if (cache.has(key)) {
|
||||
hit++;
|
||||
return cache.get(key);
|
||||
@ -44,4 +29,3 @@ const createCache = (namespace, printStats = false) => {
|
||||
}
|
||||
};
|
||||
};
|
||||
module.exports.createCache = createCache;
|
||||
@ -1,24 +0,0 @@
|
||||
const tldts = require('tldts');
|
||||
const { createCache } = require('./cache-apply');
|
||||
|
||||
const cache = createCache('cached-tld-parse', true);
|
||||
|
||||
const sharedConfig = { allowPrivateDomains: true };
|
||||
|
||||
/**
|
||||
* @param {string} domain
|
||||
* @returns {ReturnType<import('tldts').parse>}
|
||||
*/
|
||||
module.exports.parse = (domain) => cache.sync(domain, () => tldts.parse(domain, sharedConfig));
|
||||
|
||||
let gothillGetDomainCache = null;
|
||||
/**
|
||||
* @param {import('gorhill-publicsuffixlist').default | null} gorhill
|
||||
*/
|
||||
module.exports.createCachedGorhillGetDomain = (gorhill) => {
|
||||
gothillGetDomainCache ||= createCache('cached-gorhill-get-domain', true);
|
||||
/**
|
||||
* @param {string} domain
|
||||
*/
|
||||
return (domain) => (/** @type {ReturnType<typeof createCache>} */ (gothillGetDomainCache)).sync(domain, () => gorhill.getDomain(domain[0] === '.' ? domain.slice(1) : domain));
|
||||
};
|
||||
17
Build/lib/cached-tld-parse.ts
Normal file
17
Build/lib/cached-tld-parse.ts
Normal file
@ -0,0 +1,17 @@
|
||||
import tldts from 'tldts';
|
||||
import { createCache } from './cache-apply';
|
||||
import { PublicSuffixList } from 'gorhill-publicsuffixlist';
|
||||
|
||||
const cache = createCache('cached-tld-parse', true);
|
||||
|
||||
const sharedConfig = { allowPrivateDomains: true };
|
||||
|
||||
export const parse = (domain: string) => cache.sync(domain, () => tldts.parse(domain, sharedConfig));
|
||||
|
||||
let gothillGetDomainCache: ReturnType<typeof createCache> | null = null;
|
||||
export const createCachedGorhillGetDomain = (gorhill: PublicSuffixList) => {
|
||||
return (domain: string) => {
|
||||
gothillGetDomainCache ??= createCache('cached-gorhill-get-domain', true);
|
||||
return gothillGetDomainCache.sync(domain, () => gorhill.getDomain(domain[0] === '.' ? domain.slice(1) : domain))
|
||||
};
|
||||
};
|
||||
@ -1,6 +1,5 @@
|
||||
// @ts-check
|
||||
const _Trie = require('mnemonist/trie');
|
||||
const Trie = _Trie.default || _Trie;
|
||||
import Trie from 'mnemonist/trie';
|
||||
|
||||
// https://dreamacro.github.io/clash/configuration/rules.html
|
||||
const CLASH_SUPPORTED_RULE_TYPE = [
|
||||
@ -17,21 +16,13 @@ const CLASH_SUPPORTED_RULE_TYPE = [
|
||||
'PROCESS-PATH'
|
||||
];
|
||||
|
||||
/**
|
||||
* @param {string[] | Set<string>} rules
|
||||
*/
|
||||
const surgeRulesetToClashClassicalTextRuleset = (rules) => {
|
||||
export const surgeRulesetToClashClassicalTextRuleset = (rules: string[] | Set<string>) => {
|
||||
const trie = Trie.from(rules);
|
||||
return CLASH_SUPPORTED_RULE_TYPE.flatMap(
|
||||
type => trie.find(`${type},`)
|
||||
);
|
||||
};
|
||||
module.exports.surgeRulesetToClashClassicalTextRuleset = surgeRulesetToClashClassicalTextRuleset;
|
||||
|
||||
/**
|
||||
* @param {string[]} domainset
|
||||
*/
|
||||
const surgeDomainsetToClashDomainset = (domainset) => {
|
||||
export const surgeDomainsetToClashDomainset = (domainset: string[]) => {
|
||||
return domainset.map(i => (i[0] === '.' ? `+${i}` : i));
|
||||
};
|
||||
module.exports.surgeDomainsetToClashDomainset = surgeDomainsetToClashDomainset;
|
||||
@ -1,12 +1,8 @@
|
||||
// @ts-check
|
||||
const { readFileByLine } = require('./fetch-remote-text-by-line');
|
||||
const { surgeDomainsetToClashDomainset, surgeRulesetToClashClassicalTextRuleset } = require('./clash');
|
||||
import { readFileByLine } from './fetch-remote-text-by-line';
|
||||
import { surgeDomainsetToClashDomainset, surgeRulesetToClashClassicalTextRuleset } from './clash';
|
||||
|
||||
/**
|
||||
* @param {string[]} linesA
|
||||
* @param {string} filePath
|
||||
*/
|
||||
async function compareAndWriteFile(linesA, filePath) {
|
||||
export async function compareAndWriteFile(linesA: string[], filePath: string) {
|
||||
let isEqual = true;
|
||||
const file = Bun.file(filePath);
|
||||
|
||||
@ -57,16 +53,8 @@ async function compareAndWriteFile(linesA, filePath) {
|
||||
|
||||
console.log(`Same Content, bail out writing: ${filePath}`);
|
||||
}
|
||||
module.exports.compareAndWriteFile = compareAndWriteFile;
|
||||
|
||||
/**
|
||||
* @param {string} title
|
||||
* @param {string[]} description
|
||||
* @param {Date} date
|
||||
* @param {string[]} content
|
||||
* @returns {string[]}
|
||||
*/
|
||||
const withBannerArray = (title, description, date, content) => {
|
||||
export const withBannerArray = (title: string, description: string[], date: Date, content: string[]) => {
|
||||
return [
|
||||
'########################################',
|
||||
`# ${title}`,
|
||||
@ -78,20 +66,10 @@ const withBannerArray = (title, description, date, content) => {
|
||||
'################# END ###################'
|
||||
];
|
||||
};
|
||||
module.exports.withBannerArray = withBannerArray;
|
||||
|
||||
/**
|
||||
* @param {string} title
|
||||
* @param {string[]} description
|
||||
* @param {Date} date
|
||||
* @param {string[]} content
|
||||
* @param {'ruleset' | 'domainset'} type
|
||||
* @param {string} surgePath
|
||||
* @param {string} clashPath
|
||||
*/
|
||||
const createRuleset = (
|
||||
title, description, date, content,
|
||||
type, surgePath, clashPath
|
||||
export const createRuleset = (
|
||||
title: string, description: string[], date: Date, content: string[],
|
||||
type: 'ruleset' | 'domainset', surgePath: string, clashPath: string
|
||||
) => {
|
||||
const surgeContent = withBannerArray(title, description, date, content);
|
||||
|
||||
@ -114,4 +92,3 @@ const createRuleset = (
|
||||
compareAndWriteFile(clashContent, clashPath)
|
||||
];
|
||||
};
|
||||
module.exports.createRuleset = createRuleset;
|
||||
@ -1,6 +1,6 @@
|
||||
import createTrie from './trie';
|
||||
|
||||
const domainDeduper = (inputDomains: string[]): string[] => {
|
||||
export const domainDeduper = (inputDomains: string[]): string[] => {
|
||||
const trie = createTrie(inputDomains);
|
||||
const sets = new Set(inputDomains);
|
||||
|
||||
|
||||
@ -1,3 +0,0 @@
|
||||
// @ts-check
|
||||
const fetchWithRetry = require('@vercel/fetch-retry')(fetch);
|
||||
module.exports.fetchWithRetry = fetchWithRetry;
|
||||
4
Build/lib/fetch-retry.ts
Normal file
4
Build/lib/fetch-retry.ts
Normal file
@ -0,0 +1,4 @@
|
||||
// @ts-expect-error -- missing types
|
||||
import createFetchRetry from '@vercel/fetch-retry';
|
||||
|
||||
export const fetchWithRetry: typeof fetch = createFetchRetry(fetch);
|
||||
@ -1,13 +1,12 @@
|
||||
const { toASCII } = require('punycode/');
|
||||
const path = require('path');
|
||||
const { traceAsync } = require('./trace-runner');
|
||||
import { toASCII } from 'punycode';
|
||||
import path from 'path';
|
||||
import { traceAsync } from './trace-runner';
|
||||
import type { PublicSuffixList } from 'gorhill-publicsuffixlist';
|
||||
|
||||
const publicSuffixPath = path.resolve(__dirname, '../../node_modules/.cache/public_suffix_list_dat.txt');
|
||||
|
||||
const getGorhillPublicSuffix = () => traceAsync('create gorhill public suffix instance', async () => {
|
||||
const customFetch = async (url) => {
|
||||
return Bun.file(url);
|
||||
};
|
||||
const customFetch = async (url: string | URL) => Bun.file(url);
|
||||
|
||||
const publicSuffixFile = Bun.file(publicSuffixPath);
|
||||
|
||||
@ -27,9 +26,8 @@ const getGorhillPublicSuffix = () => traceAsync('create gorhill public suffix in
|
||||
return gorhill;
|
||||
});
|
||||
|
||||
/** @type {Promise<import('gorhill-publicsuffixlist').default> | null} */
|
||||
let gorhillPublicSuffixPromise = null;
|
||||
module.exports.getGorhillPublicSuffixPromise = () => {
|
||||
let gorhillPublicSuffixPromise: Promise<PublicSuffixList> | null = null;
|
||||
export const getGorhillPublicSuffixPromise = () => {
|
||||
gorhillPublicSuffixPromise ||= getGorhillPublicSuffix();
|
||||
return gorhillPublicSuffixPromise;
|
||||
};
|
||||
@ -6,7 +6,7 @@ const isDomainLoose = (domain: string): boolean => {
|
||||
return !!(!isIp && (isIcann || isPrivate));
|
||||
};
|
||||
|
||||
const parseFelixDnsmasq = async (url: string | URL): Promise<string[]> => {
|
||||
export const parseFelixDnsmasq = async (url: string | URL): Promise<string[]> => {
|
||||
const res: string[] = [];
|
||||
for await (const line of await fetchRemoteTextAndCreateReadlineInterface(url)) {
|
||||
if (line.startsWith('server=/') && line.endsWith('/114.114.114.114')) {
|
||||
@ -19,5 +19,3 @@ const parseFelixDnsmasq = async (url: string | URL): Promise<string[]> => {
|
||||
|
||||
return res;
|
||||
};
|
||||
|
||||
export { parseFelixDnsmasq };
|
||||
|
||||
@ -1,24 +1,18 @@
|
||||
// @ts-check
|
||||
const { fetchWithRetry } = require('./fetch-retry');
|
||||
const tldts = require('./cached-tld-parse');
|
||||
const { fetchRemoteTextAndCreateReadlineInterface } = require('./fetch-remote-text-by-line');
|
||||
const { NetworkFilter } = require('@cliqz/adblocker');
|
||||
const { processLine } = require('./process-line');
|
||||
const { performance } = require('perf_hooks');
|
||||
const { getGorhillPublicSuffixPromise } = require('./get-gorhill-publicsuffix');
|
||||
import { fetchWithRetry } from './fetch-retry';
|
||||
import * as tldts from './cached-tld-parse';
|
||||
import { fetchRemoteTextAndCreateReadlineInterface } from './fetch-remote-text-by-line';
|
||||
import { NetworkFilter } from '@cliqz/adblocker';
|
||||
import { processLine } from './process-line';
|
||||
import { performance } from 'perf_hooks';
|
||||
import { getGorhillPublicSuffixPromise } from './get-gorhill-publicsuffix';
|
||||
import type { PublicSuffixList } from 'gorhill-publicsuffixlist';
|
||||
|
||||
const DEBUG_DOMAIN_TO_FIND = null; // example.com | null
|
||||
let foundDebugDomain = false;
|
||||
|
||||
const warnOnceUrl = new Set();
|
||||
/**
|
||||
*
|
||||
* @param {string} url
|
||||
* @param {boolean} isWhite
|
||||
* @param {...any} message
|
||||
* @returns
|
||||
*/
|
||||
const warnOnce = (url, isWhite, ...message) => {
|
||||
const warnOnceUrl = new Set<string>();
|
||||
const warnOnce = (url: string, isWhite: boolean, ...message: any[]) => {
|
||||
const key = `${url}${isWhite ? 'white' : 'black'}`;
|
||||
if (warnOnceUrl.has(key)) {
|
||||
return;
|
||||
@ -27,10 +21,7 @@ const warnOnce = (url, isWhite, ...message) => {
|
||||
console.warn(url, isWhite ? '(white)' : '(black)', ...message);
|
||||
};
|
||||
|
||||
/**
|
||||
* @param {string} domain
|
||||
*/
|
||||
const normalizeDomain = (domain) => {
|
||||
const normalizeDomain = (domain: string) => {
|
||||
if (!domain) return null;
|
||||
|
||||
const parsed = tldts.parse(domain);
|
||||
@ -47,10 +38,7 @@ const normalizeDomain = (domain) => {
|
||||
return null;
|
||||
};
|
||||
|
||||
/**
|
||||
* @param {string | URL} domainListsUrl
|
||||
*/
|
||||
async function processDomainLists(domainListsUrl) {
|
||||
export async function processDomainLists(domainListsUrl: string | URL) {
|
||||
if (typeof domainListsUrl === 'string') {
|
||||
domainListsUrl = new URL(domainListsUrl);
|
||||
}
|
||||
@ -79,18 +67,14 @@ async function processDomainLists(domainListsUrl) {
|
||||
return domainSets;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string | URL} hostsUrl
|
||||
*/
|
||||
async function processHosts(hostsUrl, includeAllSubDomain = false) {
|
||||
export async function processHosts(hostsUrl: string | URL, includeAllSubDomain = false) {
|
||||
console.time(` - processHosts: ${hostsUrl}`);
|
||||
|
||||
if (typeof hostsUrl === 'string') {
|
||||
hostsUrl = new URL(hostsUrl);
|
||||
}
|
||||
|
||||
/** @type Set<string> */
|
||||
const domainSets = new Set();
|
||||
const domainSets = new Set<string>();
|
||||
|
||||
for await (const l of await fetchRemoteTextAndCreateReadlineInterface(hostsUrl)) {
|
||||
const line = processLine(l);
|
||||
@ -121,24 +105,20 @@ async function processHosts(hostsUrl, includeAllSubDomain = false) {
|
||||
return domainSets;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string | URL} filterRulesUrl
|
||||
* @param {readonly (string | URL)[] | undefined} [fallbackUrls]
|
||||
* @returns {Promise<{ white: Set<string>, black: Set<string>, foundDebugDomain: boolean }>}
|
||||
*/
|
||||
async function processFilterRules(filterRulesUrl, fallbackUrls) {
|
||||
export async function processFilterRules(
|
||||
filterRulesUrl: string | URL,
|
||||
fallbackUrls?: readonly (string | URL)[] | undefined
|
||||
): Promise<{ white: Set<string>, black: Set<string>, foundDebugDomain: boolean }> {
|
||||
const runStart = performance.now();
|
||||
|
||||
/** @type Set<string> */
|
||||
const whitelistDomainSets = new Set();
|
||||
/** @type Set<string> */
|
||||
const blacklistDomainSets = new Set();
|
||||
const whitelistDomainSets = new Set<string>();
|
||||
const blacklistDomainSets = new Set<string>();
|
||||
|
||||
/**
|
||||
* @param {string} domainToBeAddedToBlack
|
||||
* @param {boolean} isSubDomain
|
||||
*/
|
||||
const addToBlackList = (domainToBeAddedToBlack, isSubDomain) => {
|
||||
const addToBlackList = (domainToBeAddedToBlack: string, isSubDomain: boolean) => {
|
||||
if (isSubDomain && domainToBeAddedToBlack[0] !== '.') {
|
||||
blacklistDomainSets.add(`.${domainToBeAddedToBlack}`);
|
||||
} else {
|
||||
@ -149,7 +129,7 @@ async function processFilterRules(filterRulesUrl, fallbackUrls) {
|
||||
* @param {string} domainToBeAddedToWhite
|
||||
* @param {boolean} [isSubDomain]
|
||||
*/
|
||||
const addToWhiteList = (domainToBeAddedToWhite, isSubDomain = true) => {
|
||||
const addToWhiteList = (domainToBeAddedToWhite: string, isSubDomain = true) => {
|
||||
if (isSubDomain && domainToBeAddedToWhite[0] !== '.') {
|
||||
whitelistDomainSets.add(`.${domainToBeAddedToWhite}`);
|
||||
} else {
|
||||
@ -163,7 +143,7 @@ async function processFilterRules(filterRulesUrl, fallbackUrls) {
|
||||
/**
|
||||
* @param {string} line
|
||||
*/
|
||||
const lineCb = (line) => {
|
||||
const lineCb = (line: string) => {
|
||||
const result = parse(line, gorhill);
|
||||
if (result) {
|
||||
const flag = result[1];
|
||||
@ -250,12 +230,7 @@ async function processFilterRules(filterRulesUrl, fallbackUrls) {
|
||||
const R_KNOWN_NOT_NETWORK_FILTER_PATTERN = /[#%&=~]/;
|
||||
const R_KNOWN_NOT_NETWORK_FILTER_PATTERN_2 = /(\$popup|\$removeparam|\$popunder)/;
|
||||
|
||||
/**
|
||||
* @param {string} $line
|
||||
* @param {import('gorhill-publicsuffixlist').default} gorhill
|
||||
* @returns {null | [hostname: string, flag: 0 | 1 | 2 | -1]} - 0 white include subdomain, 1 black abosulte, 2 black include subdomain, -1 white
|
||||
*/
|
||||
function parse($line, gorhill) {
|
||||
function parse($line: string, gorhill: PublicSuffixList): null | [hostname: string, flag: 0 | 1 | 2 | -1] {
|
||||
if (
|
||||
// doesn't include
|
||||
!$line.includes('.') // rule with out dot can not be a domain
|
||||
@ -615,7 +590,3 @@ function parse($line, gorhill) {
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
module.exports.processDomainLists = processDomainLists;
|
||||
module.exports.processHosts = processHosts;
|
||||
module.exports.processFilterRules = processFilterRules;
|
||||
@ -1,10 +1,7 @@
|
||||
// @ts-check
|
||||
/**
|
||||
* @param {string | null} a
|
||||
* @param {string | null} b
|
||||
* @returns {0 | 1 | -1}
|
||||
*/
|
||||
const compare = (a, b) => {
|
||||
import type { PublicSuffixList } from 'gorhill-publicsuffixlist';
|
||||
import { createCachedGorhillGetDomain } from './cached-tld-parse';
|
||||
|
||||
const compare = (a: string | null, b: string | null) => {
|
||||
if (a === b) return 0;
|
||||
if (b == null) {
|
||||
return 1;
|
||||
@ -38,22 +35,11 @@ const compare = (a, b) => {
|
||||
return 0;
|
||||
};
|
||||
|
||||
/**
|
||||
* @param {import('gorhill-publicsuffixlist').default | null} [gorhill]
|
||||
*/
|
||||
const createDomainSorter = (gorhill = null) => {
|
||||
const createDomainSorter = (gorhill: PublicSuffixList | null = null) => {
|
||||
if (gorhill) {
|
||||
/**
|
||||
* @param {string} input
|
||||
*/
|
||||
const getDomain = require('./cached-tld-parse').createCachedGorhillGetDomain(gorhill);
|
||||
const getDomain = createCachedGorhillGetDomain(gorhill);
|
||||
|
||||
/**
|
||||
* @param {string} a
|
||||
* @param {string} b
|
||||
* @returns {0 | 1 | -1}
|
||||
*/
|
||||
return (a, b) => {
|
||||
return (a: string, b: string) => {
|
||||
if (a === b) return 0;
|
||||
|
||||
const aDomain = getDomain(a);
|
||||
@ -65,12 +51,8 @@ const createDomainSorter = (gorhill = null) => {
|
||||
}
|
||||
|
||||
const tldts = require('./cached-tld-parse');
|
||||
/**
|
||||
* @param {string} a
|
||||
* @param {string} b
|
||||
* @returns {0 | 1 | -1}
|
||||
*/
|
||||
return (a, b) => {
|
||||
|
||||
return (a: string, b: string) => {
|
||||
if (a === b) return 0;
|
||||
|
||||
const aDomain = tldts.parse(a).domain;
|
||||
@ -81,5 +63,5 @@ const createDomainSorter = (gorhill = null) => {
|
||||
};
|
||||
};
|
||||
|
||||
module.exports = createDomainSorter();
|
||||
module.exports.createDomainSorter = createDomainSorter;
|
||||
export default createDomainSorter();
|
||||
export { createDomainSorter };
|
||||
@ -1,44 +1,24 @@
|
||||
// @ts-check
|
||||
const path = require('path');
|
||||
const { performance } = require('perf_hooks');
|
||||
import path from 'path';
|
||||
|
||||
/**
|
||||
* @template T
|
||||
* @param {string} prefix
|
||||
* @param {() => T} fn
|
||||
* @returns {T}
|
||||
*/
|
||||
const traceSync = (prefix, fn) => {
|
||||
const traceSync = <T>(prefix: string, fn: () => T): T => {
|
||||
const start = performance.now();
|
||||
const result = fn();
|
||||
const end = performance.now();
|
||||
console.log(`${prefix}: ${(end - start).toFixed(3)}ms`);
|
||||
return result;
|
||||
};
|
||||
module.exports.traceSync = traceSync;
|
||||
export { traceSync };
|
||||
|
||||
/**
|
||||
* @template T
|
||||
* @param {string} prefix
|
||||
* @param {() => Promise<T>} fn
|
||||
* @returns {Promise<T>}
|
||||
*/
|
||||
const traceAsync = async (prefix, fn) => {
|
||||
const traceAsync = async <T>(prefix: string, fn: () => Promise<T>): Promise<T> => {
|
||||
const start = performance.now();
|
||||
const result = await fn();
|
||||
const end = performance.now();
|
||||
console.log(`${prefix}: ${(end - start).toFixed(3)}ms`);
|
||||
return result;
|
||||
};
|
||||
module.exports.traceAsync = traceAsync;
|
||||
export { traceAsync };
|
||||
|
||||
/**
|
||||
* @template T
|
||||
* @param {string} __filename
|
||||
* @param {() => Promise<T>} fn
|
||||
* @param {string | null} [customname]
|
||||
*/
|
||||
module.exports.task = (__filename, fn, customname = null) => {
|
||||
const task = <T>(__filename: string, fn: () => Promise<T>, customname: string | null = null) => {
|
||||
const taskName = customname ?? path.basename(__filename, path.extname(__filename));
|
||||
return async () => {
|
||||
console.log(`🏃 [${taskName}] Start executing`);
|
||||
@ -47,6 +27,7 @@ module.exports.task = (__filename, fn, customname = null) => {
|
||||
const end = performance.now();
|
||||
console.log(`✅ [${taskName}] Executed successfully: ${(end - start).toFixed(3)}ms`);
|
||||
|
||||
return { start, end, taskName };
|
||||
return { start, end, taskName } as const;
|
||||
};
|
||||
};
|
||||
export { task };
|
||||
@ -1,12 +1,12 @@
|
||||
// Surge Domain Set can not include root domain from public suffix list.
|
||||
|
||||
const tldts = require('tldts'); // hit ratio way too low, dont cache
|
||||
const picocolors = require('picocolors');
|
||||
const path = require('path');
|
||||
const listDir = require('@sukka/listdir');
|
||||
const { readFileByLine } = require('./lib/fetch-remote-text-by-line');
|
||||
const { processLine } = require('./lib/process-line');
|
||||
const { task } = require('./lib/trace-runner');
|
||||
import * as tldts from 'tldts'; // hit ratio way too low, dont cache
|
||||
import picocolors from 'picocolors';
|
||||
import path from 'path';
|
||||
import listDir from '@sukka/listdir';
|
||||
import { readFileByLine } from './lib/fetch-remote-text-by-line';
|
||||
import { processLine } from './lib/process-line';
|
||||
import { task } from './lib/trace-runner';
|
||||
|
||||
const SPECIAL_SUFFIXES = new Set([
|
||||
'linodeobjects.com', // only *.linodeobjects.com are public suffix
|
||||
@ -14,7 +14,7 @@ const SPECIAL_SUFFIXES = new Set([
|
||||
'dweb.link' // only *.dweb.link are public suffix
|
||||
]);
|
||||
|
||||
const validateDomainSet = async (filePath) => {
|
||||
const validateDomainSet = async (filePath: string) => {
|
||||
for await (const l of readFileByLine(path.resolve(__dirname, '../List/domainset', filePath))) {
|
||||
// starts with #
|
||||
const line = processLine(l);
|
||||
@ -35,7 +35,7 @@ const validateDomainSet = async (filePath) => {
|
||||
}
|
||||
};
|
||||
|
||||
const _validateRuleset = async (filePath) => {
|
||||
const _validateRuleset = async (filePath: string) => {
|
||||
console.log(`[${filePath}]`);
|
||||
|
||||
for await (const l of readFileByLine(path.resolve(__dirname, '../List/non_ip', filePath))) {
|
||||
@ -58,7 +58,7 @@ const _validateRuleset = async (filePath) => {
|
||||
}
|
||||
};
|
||||
|
||||
const validate = task(__filename, async () => {
|
||||
export const validate = task(__filename, async () => {
|
||||
// const [domainsetFiles, _rulesetFiles] = await Promise.all([
|
||||
// listDir(path.resolve(__dirname, '../List/domainset')),
|
||||
// listDir(path.resolve(__dirname, '../List/non_ip'))
|
||||
@ -69,7 +69,6 @@ const validate = task(__filename, async () => {
|
||||
// rulesetFiles.map(file => validateRuleset(file))
|
||||
]);
|
||||
});
|
||||
module.exports.validate = validate;
|
||||
|
||||
if (import.meta.main) {
|
||||
validate();
|
||||
@ -16,7 +16,7 @@
|
||||
"skipLibCheck": true
|
||||
},
|
||||
"include": [
|
||||
"./Build/**/*.js",
|
||||
"./Source/**/*.js",
|
||||
"./Build/**/*.ts"
|
||||
]
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user