mirror of
https://github.com/SukkaW/Surge.git
synced 2025-12-21 05:40:29 +08:00
376 lines
12 KiB
JavaScript
376 lines
12 KiB
JavaScript
'use strict';Object.defineProperty(exports,Symbol.toStringTag,{value:'Module'});const getPhishingDomains=require('../../_virtual/get-phishing-domains.cjs'),hosts=require('./parse-filter/hosts.cjs'),domainlists=require('./parse-filter/domainlists.cjs'),require$$5=require('tldts-experimental'),index=require('../trace/index.cjs'),appendArrayInPlace=require('./append-array-in-place.cjs'),rejectDataSource=require('../constants/reject-data-source.cjs'),looseTldtsOpt=require('../constants/loose-tldts-opt.cjs'),require$$0=require('picocolors'),require$$3=require('foxts/retrie'),cacheFilesystem=require('./cache-filesystem.cjs'),fsMemo=require('./fs-memo.cjs'),require$$11=require('ci-info');var hasRequiredGetPhishingDomains;
|
|
|
|
function requireGetPhishingDomains () {
|
|
if (hasRequiredGetPhishingDomains) return getPhishingDomains.__module.exports;
|
|
hasRequiredGetPhishingDomains = 1;
|
|
(function (module, exports) {
|
|
Object.defineProperty(exports, "__esModule", {
|
|
value: true
|
|
});
|
|
function _export(target, all) {
|
|
for(var name in all)Object.defineProperty(target, name, {
|
|
enumerable: true,
|
|
get: all[name]
|
|
});
|
|
}
|
|
_export(exports, {
|
|
calcDomainAbuseScore: function() {
|
|
return calcDomainAbuseScore;
|
|
},
|
|
getPhishingDomains: function() {
|
|
return getPhishingDomains;
|
|
}
|
|
});
|
|
const _hosts = /*@__PURE__*/ hosts.__require();
|
|
const _domainlists = /*@__PURE__*/ domainlists.__require();
|
|
const _tldtsexperimental = /*#__PURE__*/ _interop_require_wildcard(require$$5);
|
|
const _trace = /*@__PURE__*/ index.__require();
|
|
const _appendarrayinplace = /*@__PURE__*/ appendArrayInPlace.__require();
|
|
const _rejectdatasource = /*@__PURE__*/ rejectDataSource.__require();
|
|
const _loosetldtsopt = /*@__PURE__*/ looseTldtsOpt.__require();
|
|
const _picocolors = /*#__PURE__*/ _interop_require_default(require$$0);
|
|
const _retrie = require$$3;
|
|
const _cachefilesystem = /*@__PURE__*/ cacheFilesystem.__require();
|
|
const _fsmemo = /*@__PURE__*/ fsMemo.__require();
|
|
const _ciinfo = require$$11;
|
|
function _interop_require_default(obj) {
|
|
return obj && obj.__esModule ? obj : {
|
|
default: obj
|
|
};
|
|
}
|
|
function _getRequireWildcardCache(nodeInterop) {
|
|
if (typeof WeakMap !== "function") return null;
|
|
var cacheBabelInterop = new WeakMap();
|
|
var cacheNodeInterop = new WeakMap();
|
|
return (_getRequireWildcardCache = function(nodeInterop) {
|
|
return nodeInterop ? cacheNodeInterop : cacheBabelInterop;
|
|
})(nodeInterop);
|
|
}
|
|
function _interop_require_wildcard(obj, nodeInterop) {
|
|
if (obj && obj.__esModule) {
|
|
return obj;
|
|
}
|
|
if (obj === null || typeof obj !== "object" && typeof obj !== "function") {
|
|
return {
|
|
default: obj
|
|
};
|
|
}
|
|
var cache = _getRequireWildcardCache(nodeInterop);
|
|
if (cache && cache.has(obj)) {
|
|
return cache.get(obj);
|
|
}
|
|
var newObj = {
|
|
__proto__: null
|
|
};
|
|
var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor;
|
|
for(var key in obj){
|
|
if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) {
|
|
var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null;
|
|
if (desc && (desc.get || desc.set)) {
|
|
Object.defineProperty(newObj, key, desc);
|
|
} else {
|
|
newObj[key] = obj[key];
|
|
}
|
|
}
|
|
}
|
|
newObj.default = obj;
|
|
if (cache) {
|
|
cache.set(obj, newObj);
|
|
}
|
|
return newObj;
|
|
}
|
|
const BLACK_TLD = new Set([
|
|
'accountant',
|
|
'art',
|
|
'autos',
|
|
'bar',
|
|
'beauty',
|
|
'bid',
|
|
'bio',
|
|
'biz',
|
|
'bond',
|
|
'business',
|
|
'buzz',
|
|
'cc',
|
|
'cf',
|
|
'cfd',
|
|
'click',
|
|
'cloud',
|
|
'club',
|
|
'cn',
|
|
'codes',
|
|
'co.uk',
|
|
'co.in',
|
|
'com.br',
|
|
'com.cn',
|
|
'com.pl',
|
|
'com.vn',
|
|
'cool',
|
|
'cricket',
|
|
'cyou',
|
|
'date',
|
|
'design',
|
|
'digital',
|
|
'download',
|
|
'faith',
|
|
'fit',
|
|
'fun',
|
|
'ga',
|
|
'gd',
|
|
'gives',
|
|
'gq',
|
|
'group',
|
|
'host',
|
|
'icu',
|
|
'id',
|
|
'info',
|
|
'ink',
|
|
'lat',
|
|
'life',
|
|
'live',
|
|
'link',
|
|
'loan',
|
|
'lol',
|
|
'ltd',
|
|
'me',
|
|
'men',
|
|
'ml',
|
|
'mobi',
|
|
'mom',
|
|
'monster',
|
|
'net.pl',
|
|
'one',
|
|
'online',
|
|
'party',
|
|
'pro',
|
|
'pl',
|
|
'pw',
|
|
'racing',
|
|
'rest',
|
|
'review',
|
|
'rf.gd',
|
|
'sa.com',
|
|
'sbs',
|
|
'science',
|
|
'shop',
|
|
'site',
|
|
'skin',
|
|
'space',
|
|
'store',
|
|
'stream',
|
|
'su',
|
|
'surf',
|
|
'tech',
|
|
'tk',
|
|
'tokyo',
|
|
'top',
|
|
'trade',
|
|
'vip',
|
|
'vn',
|
|
'webcam',
|
|
'website',
|
|
'win',
|
|
'xyz',
|
|
'za.com'
|
|
]);
|
|
const WHITELIST_MAIN_DOMAINS = new Set([
|
|
// 'w3s.link', // ipfs gateway
|
|
// 'dweb.link', // ipfs gateway
|
|
// 'nftstorage.link', // ipfs gateway
|
|
'fleek.cool',
|
|
'flk-ipfs.xyz',
|
|
'business.site',
|
|
'page.link',
|
|
// 'notion.site',
|
|
// 'vercel.app',
|
|
'gitbook.io',
|
|
'zendesk.com',
|
|
'ipfs.eth.aragon.network',
|
|
'wordpress.com'
|
|
]);
|
|
const leathalKeywords = (0, _retrie.createRetrieKeywordFilter)([
|
|
'vinted-',
|
|
'inpost-pl',
|
|
'vlnted-'
|
|
]);
|
|
const sensitiveKeywords = (0, _retrie.createRetrieKeywordFilter)([
|
|
'.amazon-',
|
|
'-amazon',
|
|
'fb-com',
|
|
'facebook-com',
|
|
'-facebook',
|
|
'facebook-',
|
|
'focebaak',
|
|
'.facebook.',
|
|
'metamask',
|
|
'www.apple',
|
|
'-coinbase',
|
|
'coinbase-',
|
|
'booking-com',
|
|
'booking.com-',
|
|
'booking-eu',
|
|
'vinted-',
|
|
'inpost-pl',
|
|
'login.microsoft',
|
|
'login-microsoft',
|
|
'microsoftonline',
|
|
'google.com-',
|
|
'minecraft',
|
|
'staemco',
|
|
'oferta'
|
|
]);
|
|
const lowKeywords = (0, _retrie.createRetrieKeywordFilter)([
|
|
'transactions-',
|
|
'payment',
|
|
'wallet',
|
|
'-transactions',
|
|
'-faceb',
|
|
'.faceb',
|
|
'facebook',
|
|
'virus-',
|
|
'icloud-',
|
|
'apple-',
|
|
'-roblox',
|
|
'-co-jp',
|
|
'customer.',
|
|
'customer-',
|
|
'.www-',
|
|
'.www.',
|
|
'.www2',
|
|
'instagram',
|
|
'microsof',
|
|
'passwordreset',
|
|
'.google-',
|
|
'recover',
|
|
'banking'
|
|
]);
|
|
const processPhihsingDomains = (0, _fsmemo.cache)(function processPhihsingDomains(domainArr) {
|
|
const domainCountMap = new Map();
|
|
const domainScoreMap = {};
|
|
let line = '';
|
|
let tld = '';
|
|
let apexDomain = '';
|
|
let subdomain = '';
|
|
// const set = new Set<string>();
|
|
// let duplicateCount = 0;
|
|
for(let i = 0, len = domainArr.length; i < len; i++){
|
|
line = domainArr[i];
|
|
// if (set.has(line)) {
|
|
// duplicateCount++;
|
|
// } else {
|
|
// set.add(line);
|
|
// }
|
|
const parsed = _tldtsexperimental.parse(line, _loosetldtsopt.loosTldOptWithPrivateDomains);
|
|
if (parsed.isPrivate) {
|
|
continue;
|
|
}
|
|
tld = parsed.publicSuffix;
|
|
apexDomain = parsed.domain;
|
|
if (!tld) {
|
|
console.log(_picocolors.default.yellow('[phishing domains] E0001'), 'missing tld', {
|
|
line,
|
|
tld
|
|
});
|
|
continue;
|
|
}
|
|
if (!apexDomain) {
|
|
console.log(_picocolors.default.yellow('[phishing domains] E0002'), 'missing domain', {
|
|
line,
|
|
apexDomain
|
|
});
|
|
continue;
|
|
}
|
|
domainCountMap.set(apexDomain, domainCountMap.has(apexDomain) ? domainCountMap.get(apexDomain) + 1 : 1);
|
|
if (!(apexDomain in domainScoreMap)) {
|
|
domainScoreMap[apexDomain] = 0;
|
|
if (BLACK_TLD.has(tld)) {
|
|
domainScoreMap[apexDomain] += 3;
|
|
} else if (tld.length > 6) {
|
|
domainScoreMap[apexDomain] += 2;
|
|
}
|
|
if (apexDomain.length >= 18) {
|
|
domainScoreMap[apexDomain] += 0.5;
|
|
}
|
|
}
|
|
subdomain = parsed.subdomain;
|
|
if (subdomain && !WHITELIST_MAIN_DOMAINS.has(apexDomain)) {
|
|
domainScoreMap[apexDomain] += calcDomainAbuseScore(subdomain, line);
|
|
}
|
|
}
|
|
domainCountMap.forEach((count, apexDomain)=>{
|
|
if (// !WHITELIST_MAIN_DOMAINS.has(apexDomain)
|
|
domainScoreMap[apexDomain] >= 24 || domainScoreMap[apexDomain] >= 16 && count >= 7 || domainScoreMap[apexDomain] >= 13 && count >= 11 || domainScoreMap[apexDomain] >= 5 && count >= 14 || domainScoreMap[apexDomain] >= 3 && count >= 21 || domainScoreMap[apexDomain] >= 1 && count >= 60) {
|
|
domainArr.push('.' + apexDomain);
|
|
}
|
|
});
|
|
// console.log({
|
|
// score: domainScoreMap['awicksin.com'],
|
|
// count: domainCountMap.get('awicksin.com')
|
|
// });
|
|
// console.log({ duplicateCount, domainArrLen: domainArr.length });
|
|
return domainArr;
|
|
}, {
|
|
serializer: _cachefilesystem.serializeArray,
|
|
deserializer: _cachefilesystem.deserializeArray,
|
|
temporaryBypass: !_ciinfo.isCI || _rejectdatasource.DEBUG_DOMAIN_TO_FIND !== null
|
|
});
|
|
const downloads = [
|
|
..._rejectdatasource.PHISHING_DOMAIN_LISTS_EXTRA.map((entry)=>(0, _domainlists.processDomainListsWithPreload)(...entry)),
|
|
..._rejectdatasource.PHISHING_HOSTS_EXTRA.map((entry)=>(0, _hosts.processHostsWithPreload)(...entry))
|
|
];
|
|
function getPhishingDomains(parentSpan) {
|
|
return parentSpan.traceChild('get phishing domains').traceAsyncFn(async (span)=>{
|
|
const domainArr = await span.traceChildAsync('download/parse/merge phishing domains', async (curSpan)=>{
|
|
const domainArr = [];
|
|
const domainGroups = await Promise.all(downloads.map((task)=>task(curSpan)));
|
|
domainGroups.forEach((0, _appendarrayinplace.appendArrayInPlaceCurried)(domainArr));
|
|
return domainArr;
|
|
});
|
|
return span.traceChildAsync('process phishing domain set', ()=>processPhihsingDomains(domainArr));
|
|
});
|
|
}
|
|
function calcDomainAbuseScore(subdomain, fullDomain = subdomain) {
|
|
if (leathalKeywords(fullDomain)) {
|
|
return 100;
|
|
}
|
|
let weight = 0;
|
|
const hitLowKeywords = lowKeywords(fullDomain);
|
|
const sensitiveKeywordsHit = sensitiveKeywords(fullDomain);
|
|
if (sensitiveKeywordsHit) {
|
|
weight += 10;
|
|
if (hitLowKeywords) {
|
|
weight += 6;
|
|
}
|
|
} else if (hitLowKeywords) {
|
|
weight += 1.7;
|
|
}
|
|
const subdomainLength = subdomain.length;
|
|
if (subdomainLength > 6) {
|
|
weight += 0.015;
|
|
if (subdomainLength > 13) {
|
|
weight += 0.2;
|
|
if (subdomainLength > 20) {
|
|
weight += 1;
|
|
if (subdomainLength > 30) {
|
|
weight += 5;
|
|
if (subdomainLength > 40) {
|
|
weight += 10;
|
|
}
|
|
}
|
|
}
|
|
if (subdomain.indexOf('.', 1) > 1) {
|
|
weight += 1;
|
|
}
|
|
}
|
|
}
|
|
return weight;
|
|
}
|
|
if (require.main === module) {
|
|
getPhishingDomains(_trace.dummySpan).catch(console.error).finally(()=>{
|
|
_trace.dummySpan.stop();
|
|
(0, _trace.printTraceResult)(_trace.dummySpan.traceResult);
|
|
});
|
|
}
|
|
} (getPhishingDomains.__module, getPhishingDomains.__module.exports));
|
|
return getPhishingDomains.__module.exports;
|
|
}exports.__require=requireGetPhishingDomains; |