From 94293d98c9244ec1da8b3edda60bd4de36ca4dd5 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Fri, 7 Feb 2025 17:52:36 +0800 Subject: [PATCH] Chore: maintainance --- Build/build-apple-cdn.ts | 2 +- Build/build-cdn-download-conf.ts | 2 +- Build/build-chn-cidr.ts | 2 +- Build/build-cloudmounter-rules.ts | 2 +- Build/build-common.ts | 3 +- ...c-direct-lan-ruleset-dns-mapping-module.ts | 2 +- Build/build-microsoft-cdn.ts | 2 +- Build/build-reject-domainset.ts | 2 +- Build/build-reject-ip-list.ts | 3 +- Build/build-speedtest-domainset.ts | 2 +- Build/build-stream-service.ts | 2 +- Build/build-telegram-cidr.ts | 2 +- Build/lib/create-file.ts | 113 +++++++++++++++++- Build/lib/rules/base.ts | 108 ----------------- 14 files changed, 123 insertions(+), 124 deletions(-) diff --git a/Build/build-apple-cdn.ts b/Build/build-apple-cdn.ts index e9d30a96..c9710e1c 100644 --- a/Build/build-apple-cdn.ts +++ b/Build/build-apple-cdn.ts @@ -2,7 +2,7 @@ import { parseFelixDnsmasqFromResp } from './lib/parse-dnsmasq'; import { task } from './trace'; import { SHARED_DESCRIPTION } from './constants/description'; import { createMemoizedPromise } from './lib/memo-promise'; -import { DomainsetOutput } from './lib/create-file'; +import { DomainsetOutput } from './lib/rules/domainset'; import { $$fetch } from './lib/fetch-retry'; export const getAppleCdnDomainsPromise = createMemoizedPromise(() => $$fetch('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf').then(parseFelixDnsmasqFromResp)); diff --git a/Build/build-cdn-download-conf.ts b/Build/build-cdn-download-conf.ts index 7f407f4e..21d945f5 100644 --- a/Build/build-cdn-download-conf.ts +++ b/Build/build-cdn-download-conf.ts @@ -5,7 +5,7 @@ import { task } from './trace'; import { SHARED_DESCRIPTION } from './constants/description'; import { appendArrayInPlace } from './lib/append-array-in-place'; import { SOURCE_DIR } from './constants/dir'; -import { DomainsetOutput } from './lib/create-file'; +import { DomainsetOutput } from './lib/rules/domainset'; import { CRASHLYTICS_WHITELIST } from './constants/reject-data-source'; import { appendSetElementsToArray } from 'foxts/append-set-elements-to-array'; diff --git a/Build/build-chn-cidr.ts b/Build/build-chn-cidr.ts index e75465b2..b0b3d5b8 100644 --- a/Build/build-chn-cidr.ts +++ b/Build/build-chn-cidr.ts @@ -5,7 +5,7 @@ import { contains as containsCidr, exclude as excludeCidr } from 'fast-cidr-tool import { createMemoizedPromise } from './lib/memo-promise'; import { CN_CIDR_MISSING_IN_CHNROUTE, NON_CN_CIDR_INCLUDED_IN_CHNROUTE } from './constants/cidr'; import { appendArrayInPlace } from './lib/append-array-in-place'; -import { IPListOutput } from './lib/create-file'; +import { IPListOutput } from './lib/rules/ip'; import { cachedOnlyFail } from './lib/fs-memo'; import { createFileDescription } from './constants/description'; diff --git a/Build/build-cloudmounter-rules.ts b/Build/build-cloudmounter-rules.ts index a60b718f..5650473c 100644 --- a/Build/build-cloudmounter-rules.ts +++ b/Build/build-cloudmounter-rules.ts @@ -1,7 +1,7 @@ import { DOMAINS, PROCESS_NAMES } from '../Source/non_ip/cloudmounter'; import { SHARED_DESCRIPTION } from './constants/description'; import { task } from './trace'; -import { RulesetOutput } from './lib/create-file'; +import { RulesetOutput } from './lib/rules/ruleset'; export const buildCloudMounterRules = task(require.main === module, __filename)(async (span) => { // AND,((SRC-IP,192.168.1.110), (DOMAIN, example.com)) diff --git a/Build/build-common.ts b/Build/build-common.ts index 96abfdf4..37475fe7 100644 --- a/Build/build-common.ts +++ b/Build/build-common.ts @@ -9,7 +9,8 @@ import { SHARED_DESCRIPTION } from './constants/description'; import { fdir as Fdir } from 'fdir'; import { appendArrayInPlace } from './lib/append-array-in-place'; import { SOURCE_DIR } from './constants/dir'; -import { DomainsetOutput, RulesetOutput } from './lib/create-file'; +import { DomainsetOutput } from './lib/rules/domainset'; +import { RulesetOutput } from './lib/rules/ruleset'; const MAGIC_COMMAND_SKIP = '# $ custom_build_script'; const MAGIC_COMMAND_TITLE = '# $ meta_title '; diff --git a/Build/build-domestic-direct-lan-ruleset-dns-mapping-module.ts b/Build/build-domestic-direct-lan-ruleset-dns-mapping-module.ts index 8119e43f..7f3f88ee 100644 --- a/Build/build-domestic-direct-lan-ruleset-dns-mapping-module.ts +++ b/Build/build-domestic-direct-lan-ruleset-dns-mapping-module.ts @@ -11,7 +11,7 @@ import { createMemoizedPromise } from './lib/memo-promise'; import * as yaml from 'yaml'; import { appendArrayInPlace } from './lib/append-array-in-place'; import { OUTPUT_INTERNAL_DIR, OUTPUT_MODULES_DIR, OUTPUT_MODULES_RULES_DIR, SOURCE_DIR } from './constants/dir'; -import { RulesetOutput } from './lib/create-file'; +import { RulesetOutput } from './lib/rules/ruleset'; import { SurgeOnlyRulesetOutput } from './lib/rules/ruleset'; export function createGetDnsMappingRule(allowWildcard: boolean) { diff --git a/Build/build-microsoft-cdn.ts b/Build/build-microsoft-cdn.ts index 64e1b6f5..2139a1ce 100644 --- a/Build/build-microsoft-cdn.ts +++ b/Build/build-microsoft-cdn.ts @@ -4,7 +4,7 @@ import { HostnameSmolTrie } from './lib/trie'; import { SHARED_DESCRIPTION } from './constants/description'; import { createMemoizedPromise } from './lib/memo-promise'; import { extractDomainsFromFelixDnsmasq } from './lib/parse-dnsmasq'; -import { RulesetOutput } from './lib/create-file'; +import { RulesetOutput } from './lib/rules/ruleset'; import { appendArrayInPlace } from './lib/append-array-in-place'; const PROBE_DOMAINS = ['.microsoft.com', '.windows.net', '.windows.com', '.windowsupdate.com', '.windowssearch.com', '.office.net']; diff --git a/Build/build-reject-domainset.ts b/Build/build-reject-domainset.ts index c2b13636..9f5d98e3 100644 --- a/Build/build-reject-domainset.ts +++ b/Build/build-reject-domainset.ts @@ -17,7 +17,7 @@ import { getPhishingDomains } from './lib/get-phishing-domains'; import { addArrayElementsToSet } from 'foxts/add-array-elements-to-set'; import { OUTPUT_INTERNAL_DIR, SOURCE_DIR } from './constants/dir'; -import { DomainsetOutput } from './lib/create-file'; +import { DomainsetOutput } from './lib/rules/domainset'; import { foundDebugDomain } from './lib/parse-filter/shared'; import { AdGuardHomeOutput } from './lib/rules/domainset'; diff --git a/Build/build-reject-ip-list.ts b/Build/build-reject-ip-list.ts index a09a8222..e72e7593 100644 --- a/Build/build-reject-ip-list.ts +++ b/Build/build-reject-ip-list.ts @@ -3,12 +3,13 @@ import path from 'node:path'; import { createReadlineInterfaceFromResponse, readFileIntoProcessedArray } from './lib/fetch-text-by-line'; import { task } from './trace'; import { SHARED_DESCRIPTION } from './constants/description'; -import { compareAndWriteFile, RulesetOutput } from './lib/create-file'; +import { compareAndWriteFile } from './lib/create-file'; import { OUTPUT_INTERNAL_DIR, SOURCE_DIR } from './constants/dir'; import { $$fetch } from './lib/fetch-retry'; import { fetchAssets } from './lib/fetch-assets'; import { fastIpVersion } from './lib/misc'; import { AUGUST_ASN, HUIZE_ASN } from '../Source/ip/badboy_asn'; +import { RulesetOutput } from './lib/rules/ruleset'; const BOGUS_NXDOMAIN_URL = 'https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/bogus-nxdomain.china.conf'; const getBogusNxDomainIPsPromise: Promise<[ipv4: string[], ipv6: string[]]> = $$fetch(BOGUS_NXDOMAIN_URL).then(async (resp) => { diff --git a/Build/build-speedtest-domainset.ts b/Build/build-speedtest-domainset.ts index 1dd0b532..a1a3d6c4 100644 --- a/Build/build-speedtest-domainset.ts +++ b/Build/build-speedtest-domainset.ts @@ -5,7 +5,7 @@ import { task } from './trace'; import { SHARED_DESCRIPTION } from './constants/description'; import { readFileIntoProcessedArray } from './lib/fetch-text-by-line'; -import { DomainsetOutput } from './lib/create-file'; +import { DomainsetOutput } from './lib/rules/domainset'; import { OUTPUT_SURGE_DIR, SOURCE_DIR } from './constants/dir'; import { newQueue } from '@henrygd/queue'; import { $$fetch } from './lib/fetch-retry'; diff --git a/Build/build-stream-service.ts b/Build/build-stream-service.ts index a008ca1d..f8b7fab2 100644 --- a/Build/build-stream-service.ts +++ b/Build/build-stream-service.ts @@ -4,7 +4,7 @@ import { task } from './trace'; import { ALL, NORTH_AMERICA, EU, HK, TW, JP, KR } from '../Source/stream'; import { SHARED_DESCRIPTION } from './constants/description'; -import { RulesetOutput } from './lib/create-file'; +import { RulesetOutput } from './lib/rules/ruleset'; function createRulesetForStreamService( span: Span, diff --git a/Build/build-telegram-cidr.ts b/Build/build-telegram-cidr.ts index 7dbb561d..875d5573 100644 --- a/Build/build-telegram-cidr.ts +++ b/Build/build-telegram-cidr.ts @@ -3,7 +3,7 @@ import { createReadlineInterfaceFromResponse } from './lib/fetch-text-by-line'; import { task } from './trace'; import { SHARED_DESCRIPTION } from './constants/description'; import { createMemoizedPromise } from './lib/memo-promise'; -import { RulesetOutput } from './lib/create-file'; +import { RulesetOutput } from './lib/rules/ruleset'; import { $$fetch } from './lib/fetch-retry'; import { fastIpVersion } from './lib/misc'; diff --git a/Build/lib/create-file.ts b/Build/lib/create-file.ts index a42c92c9..8fda87b3 100644 --- a/Build/lib/create-file.ts +++ b/Build/lib/create-file.ts @@ -1,4 +1,109 @@ -export { DomainsetOutput } from './rules/domainset'; -export { IPListOutput } from './rules/ip'; -export { RulesetOutput } from './rules/ruleset'; -export { fileEqual, compareAndWriteFile } from './rules/base'; +import { asyncWriteToStream } from 'foxts/async-write-to-stream'; +import { fastStringArrayJoin } from 'foxts/fast-string-array-join'; +import fs from 'node:fs'; +import picocolors from 'picocolors'; +import type { Span } from '../trace'; +import { readFileByLine } from './fetch-text-by-line'; +import { writeFile } from './misc'; + +export async function fileEqual(linesA: string[], source: AsyncIterable | Iterable): Promise { + if (linesA.length === 0) { + return false; + } + + const linesABound = linesA.length - 1; + + let index = -1; + for await (const lineB of source) { + index++; + + if (index > linesABound) { + return (index === linesA.length && lineB.length === 0); + } + + const lineA = linesA[index]; + + if (lineA.length === 0) { + if (lineB.length === 0) { + // both lines are empty, check next line + continue; + } + // lineA is empty but lineB is not + return false; + } + // now lineA can not be empty + if (lineB.length === 0) { + // lineB is empty but lineA is not + return false; + } + + // now both lines can not be empty + + const firstCharA = lineA.charCodeAt(0); + const firstCharB = lineB.charCodeAt(0); + + if (firstCharA !== firstCharB) { + return false; + } + + // now firstCharA is equal to firstCharB, we only need to check the first char + if (firstCharA === 35 /* # */) { + continue; + } + // adguard conf + if (firstCharA === 33 /* ! */) { + continue; + } + + if ( + firstCharA === 47 /* / */ + && lineA[1] === '/' && lineB[1] === '/' + && lineA[3] === '#' && lineB[3] === '#' + ) { + continue; + } + + if (lineA !== lineB) { + return false; + } + } + + // The file becomes larger + return !(index < linesABound); +} + +export async function compareAndWriteFile(span: Span, linesA: string[], filePath: string) { + const linesALen = linesA.length; + + const isEqual = await span.traceChildAsync(`compare ${filePath}`, async () => { + if (fs.existsSync(filePath)) { + return fileEqual(linesA, readFileByLine(filePath)); + } + + console.log(`${filePath} does not exists, writing...`); + return false; + }); + + if (isEqual) { + console.log(picocolors.gray(picocolors.dim(`same content, bail out writing: ${filePath}`))); + return; + } + + await span.traceChildAsync(`writing ${filePath}`, async () => { + // The default highwater mark is normally 16384, + // So we make sure direct write to file if the content is + // most likely less than 500 lines + if (linesALen < 500) { + return writeFile(filePath, fastStringArrayJoin(linesA, '\n') + '\n'); + } + + const writeStream = fs.createWriteStream(filePath); + for (let i = 0; i < linesALen; i++) { + const p = asyncWriteToStream(writeStream, linesA[i] + '\n'); + // eslint-disable-next-line no-await-in-loop -- stream high water mark + if (p) await p; + } + + writeStream.end(); + }); +} diff --git a/Build/lib/rules/base.ts b/Build/lib/rules/base.ts index 2ba1426a..44b48e53 100644 --- a/Build/lib/rules/base.ts +++ b/Build/lib/rules/base.ts @@ -1,13 +1,7 @@ import type { Span } from '../../trace'; import { HostnameSmolTrie } from '../trie'; import { invariant, not } from 'foxts/guard'; -import picocolors from 'picocolors'; -import fs from 'node:fs'; -import { writeFile } from '../misc'; import type { MaybePromise } from '../misc'; -import { fastStringArrayJoin } from 'foxts/fast-string-array-join'; -import { readFileByLine } from '../fetch-text-by-line'; -import { asyncWriteToStream } from 'foxts/async-write-to-stream'; import type { BaseWriteStrategy } from '../writing-strategy/base'; import { merge as mergeCidr } from 'fast-cidr-tools'; import { createRetrieKeywordFilter as createKeywordFilter } from 'foxts/retrie'; @@ -496,105 +490,3 @@ export class FileOutput { return this; } } - -export async function fileEqual(linesA: string[], source: AsyncIterable | Iterable): Promise { - if (linesA.length === 0) { - return false; - } - - const linesABound = linesA.length - 1; - - let index = -1; - for await (const lineB of source) { - index++; - - if (index > linesABound) { - return (index === linesA.length && lineB.length === 0); - } - - const lineA = linesA[index]; - - if (lineA.length === 0) { - if (lineB.length === 0) { - // both lines are empty, check next line - continue; - } - // lineA is empty but lineB is not - return false; - } - // now lineA can not be empty - if (lineB.length === 0) { - // lineB is empty but lineA is not - return false; - } - - // now both lines can not be empty - - const firstCharA = lineA.charCodeAt(0); - const firstCharB = lineB.charCodeAt(0); - - if (firstCharA !== firstCharB) { - return false; - } - - // now firstCharA is equal to firstCharB, we only need to check the first char - if (firstCharA === 35 /* # */) { - continue; - } - // adguard conf - if (firstCharA === 33 /* ! */) { - continue; - } - - if ( - firstCharA === 47 /* / */ - && lineA[1] === '/' && lineB[1] === '/' - && lineA[3] === '#' && lineB[3] === '#' - ) { - continue; - } - - if (lineA !== lineB) { - return false; - } - } - - // The file becomes larger - return !(index < linesABound); -} - -export async function compareAndWriteFile(span: Span, linesA: string[], filePath: string) { - const linesALen = linesA.length; - - const isEqual = await span.traceChildAsync(`compare ${filePath}`, async () => { - if (fs.existsSync(filePath)) { - return fileEqual(linesA, readFileByLine(filePath)); - } - - console.log(`${filePath} does not exists, writing...`); - return false; - }); - - if (isEqual) { - console.log(picocolors.gray(picocolors.dim(`same content, bail out writing: ${filePath}`))); - return; - } - - await span.traceChildAsync(`writing ${filePath}`, async () => { - // The default highwater mark is normally 16384, - // So we make sure direct write to file if the content is - // most likely less than 500 lines - if (linesALen < 500) { - return writeFile(filePath, fastStringArrayJoin(linesA, '\n') + '\n'); - } - - const writeStream = fs.createWriteStream(filePath); - for (let i = 0; i < linesALen; i++) { - const p = asyncWriteToStream(writeStream, linesA[i] + '\n'); - // eslint-disable-next-line no-await-in-loop -- stream high water mark - if (p) await p; - } - - writeStream.end(); - }); -}