GithubFollow/Surge_by_SukkaW
1
0
Fork 0
mirror of https://github.com/SukkaW/Surge.git synced 2026-01-29 01:51:52 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Perf: replace psl with tldts (it is blazing fast!)

Browse Source
This commit is contained in:
SukkaW
2022-11-02 01:23:36 +08:00
parent 2ba69aa167
commit b69f4a75c1
5 changed files with 259 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
Build/build-phishing-domainset.js
View File

@@ -1,4 +1,4 @@
const psl = require('psl');
const tldts = require('tldts');
const { processFilterRules } = require('./lib/parse-filter.js');
const fs = require('fs');
const path = require('path');
@@ -55,19 +55,20 @@ const BLACK_TLD = Array.from(new Set([
const domain = line.charCodeAt(0) === 46 ? line.slice(1) : line;
if (line.length > 25) {
const parsed = psl.parse(domain);
const parsed = tldts.parse(domain, { allowPrivateDomains: true });
if (parsed.input === parsed.tld) {
if (parsed.isIp || domain === parsed.publicSuffix) {
continue;
}
const apexDomain = parsed.domain
const apexDomain = parsed.domain;
if (apexDomain) {
if (WHITELIST_DOMAIN.has(apexDomain)) {
continue;
}
if (WHITELIST_DOMAIN.has(apexDomain)) {
continue;
domainCountMap[apexDomain] ||= 0;
domainCountMap[apexDomain] += 1;
}
domainCountMap[apexDomain] ||= 0;
domainCountMap[apexDomain] += 1;
}
}

25
Build/validate-cdn-conf.js
View File

@@ -1,9 +1,15 @@
const psl = require('psl');
const tldts = require('tldts');
const picocolors = require('picocolors');
const fs = require('fs');
const path = require('path');
const SPECIAL_SUFFIXES = new Set([
'linodeobjects.com', // only *.linodeobjects.com are public suffix
'vultrobjects.com', // only *.vultrobjects.com are public suffix
'dweb.link' // only *.dweb.link are public suffix
]);
(async () => {
const domainSetContent = await fs.promises.readFile(
path.resolve(__dirname, '../List/domainset/cdn.conf'),
@@ -22,9 +28,14 @@ const path = require('path');
}
const domain = line.charCodeAt(0) === 46 ? line.slice(1) : line;
const parsed = psl.parse(domain);
const parsed = tldts.parse(domain, { allowPrivateDomains: true });
if (parsed.listed && parsed.input === parsed.tld) {
if (
(
parsed.isPrivate
|| parsed.isIcann
) && domain === parsed.publicSuffix
) {
console.error('Domain', picocolors.yellow(domain), picocolors.red('is in public suffix list!'));
}
}
@@ -47,10 +58,12 @@ const path = require('path');
if (line.startsWith('DOMAIN-SUFFIX')) {
const domain = line.slice(14);
const parsed = psl.parse(domain);
const parsed = tldts.parse(domain, { allowPrivateDomains: true });
if (parsed.input !== parsed.tld) {
console.error('Domain', picocolors.yellow(domain), picocolors.green('is not in public suffix list!'));
if (domain !== parsed.publicSuffix) {
if (!SPECIAL_SUFFIXES.has(domain)) {
console.error('Domain', picocolors.yellow(domain), picocolors.green('is not in public suffix list!'));
}
}
}
}