diff --git a/Build/build-internal-cdn-rules.ts b/Build/build-internal-cdn-rules.ts deleted file mode 100644 index b2359fb9..00000000 --- a/Build/build-internal-cdn-rules.ts +++ /dev/null @@ -1,64 +0,0 @@ -import path from 'path'; -import { processLine } from './lib/process-line'; -import { readFileByLine } from './lib/fetch-text-by-line'; -import { sortDomains } from './lib/stable-sort-domain'; -import { task } from './trace'; -import { compareAndWriteFile } from './lib/create-file'; -import { domainDeduper } from './lib/domain-deduper'; -import { sort } from './lib/timsort'; - -const escapeRegExp = (string = '') => string.replaceAll(/[$()*+.?[\\\]^{|}]/g, '\\$&'); - -const processLocalDomainSet = async (domainSetPath: string, set: Set) => { - for await (const l of readFileByLine(domainSetPath)) { - const line = processLine(l); - if (line) { - set.add(line[0] === '.' ? line.slice(1) : line); - } - } -}; - -const processLocalRuleSet = async (ruleSetPath: string, set: Set, keywords: Set) => { - for await (const line of readFileByLine(ruleSetPath)) { - if (line.startsWith('DOMAIN-SUFFIX,')) { - set.add(line.slice(14)); - } else if (line.startsWith('DOMAIN,')) { - set.add(line.slice(7)); - } else if (line.startsWith('DOMAIN-KEYWORD')) { - keywords.add(escapeRegExp(line.slice(15))); - } else if (line.includes('USER-AGENT,') || line.includes('PROCESS-NAME,') || line.includes('URL-REGEX,') || line.includes('DOMAIN-WILDCARD')) { - // do nothing - } else if (processLine(line)) { - console.warn('[drop line from ruleset]', line); - } - } -}; - -export const buildInternalCDNDomains = task(import.meta.path, async (span) => { - const proxySet = new Set(); - const proxyKeywords = new Set(); - - await Promise.all([ - processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/cdn.conf'), proxySet, proxyKeywords), - processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/global.conf'), proxySet, proxyKeywords), - processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/my_proxy.conf'), proxySet, proxyKeywords), - processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/my_plus.conf'), proxySet, proxyKeywords), - processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/stream.conf'), proxySet, proxyKeywords), - processLocalRuleSet(path.resolve(import.meta.dir, '../List/non_ip/telegram.conf'), proxySet, proxyKeywords), - processLocalDomainSet(path.resolve(import.meta.dir, '../List/domainset/cdn.conf'), proxySet), - processLocalDomainSet(path.resolve(import.meta.dir, '../List/domainset/download.conf'), proxySet) - ]); - - return compareAndWriteFile( - span, - [ - ...sortDomains(domainDeduper(Array.from(proxySet))).map(i => `SUFFIX,${i}`), - ...sort(Array.from(proxyKeywords)).map(i => `REGEX,${i}`) - ], - path.resolve(import.meta.dir, '../Internal/cdn.txt') - ); -}); - -if (import.meta.main) { - buildInternalCDNDomains(); -} diff --git a/Build/index.ts b/Build/index.ts index 9711cef0..570e6c0c 100644 --- a/Build/index.ts +++ b/Build/index.ts @@ -9,7 +9,6 @@ import { buildRejectDomainSet } from './build-reject-domainset'; import { buildTelegramCIDR } from './build-telegram-cidr'; import { buildChnCidr } from './build-chn-cidr'; import { buildSpeedtestDomainSet } from './build-speedtest-domainset'; -import { buildInternalCDNDomains } from './build-internal-cdn-rules'; import { buildInternalReverseChnCIDR } from './build-internal-reverse-chn-cidr'; import { buildDomesticRuleset } from './build-domestic-ruleset'; import { buildStreamService } from './build-stream-service'; @@ -51,10 +50,6 @@ process.on('unhandledRejection', (reason) => { const buildTelegramCIDRPromise = downloadPreviousBuildPromise.then(() => buildTelegramCIDR(rootSpan)); const buildChnCidrPromise = downloadPreviousBuildPromise.then(() => buildChnCidr(rootSpan)); const buildSpeedtestDomainSetPromise = downloadPreviousBuildPromise.then(() => buildSpeedtestDomainSet(rootSpan)); - const buildInternalCDNDomainsPromise = Promise.all([ - buildCommonPromise, - buildCdnConfPromise - ]).then(() => buildInternalCDNDomains(rootSpan)); const buildInternalReverseChnCIDRPromise = buildInternalReverseChnCIDR(rootSpan); @@ -86,7 +81,6 @@ process.on('unhandledRejection', (reason) => { buildTelegramCIDRPromise, buildChnCidrPromise, buildSpeedtestDomainSetPromise, - buildInternalCDNDomainsPromise, buildInternalReverseChnCIDRPromise, buildInternalReverseChnCIDRPromise, // buildInternalChnDomainsPromise, diff --git a/Build/lib/get-phishing-domains.ts b/Build/lib/get-phishing-domains.ts index 8db20d33..00498ef0 100644 --- a/Build/lib/get-phishing-domains.ts +++ b/Build/lib/get-phishing-domains.ts @@ -118,20 +118,18 @@ export const getPhishingDomains = (parentSpan: Span) => parentSpan.traceChild('g for (let i = 0, len = domainArr.length; i < len; i++) { const line = domainArr[i]; - const safeGorhillLine = line[0] === '.' ? line.slice(1) : line; - const { publicSuffix: tld, domain: apexDomain, subdomain - } = parse(safeGorhillLine, looseTldtsOpt); + } = parse(line, looseTldtsOpt); if (!tld) { - console.log(picocolors.yellow('[phishing domains] E0001'), 'missing tld', { line, safeGorhillLine, tld }); + console.log(picocolors.yellow('[phishing domains] E0001'), 'missing tld', { line, tld }); continue; } if (!apexDomain) { - console.log(picocolors.yellow('[phishing domains] E0002'), 'missing domain', { line, safeGorhillLine, apexDomain }); + console.log(picocolors.yellow('[phishing domains] E0002'), 'missing domain', { line, apexDomain }); continue; }