New build infra

Build/build-anti-bogus-domain.js

@@ -2,6 +2,8 @@ const { fetchWithRetry } = require('./lib/fetch-retry');
 const fs = require('fs');
 const path = require('path');
 const { isIPv4, isIPv6 } = require('net');
+const { compareAndWriteFile } = require('./lib/string-array-compare');
+const { withBannerArray } = require('./lib/with-banner');
 
 (async () => {
   console.time('Total Time - build-anti-bogus-domain');
@@ -35,6 +37,24 @@ const { isIPv4, isIPv6 } = require('net');
       }).join('\n')
     );
 
-  await fs.promises.writeFile(resultPath, content, 'utf-8');
+  await compareAndWriteFile(
+    withBannerArray(
+      'Sukka\'s Surge Rules - Telegram IP CIDR',
+      [
+        'License: AGPL 3.0',
+        'Homepage: https://ruleset.skk.moe',
+        'GitHub: https://github.com/SukkaW/Surge',
+        '',
+        'This file contains known addresses that are hijacking NXDOMAIN results returned by DNS servers.',
+        '',
+        'Data from:',
+        ' - https://github.com/felixonmars/dnsmasq-china-list'
+      ],
+      new Date(),
+      content.split('\n')
+    ),
+    resultPath
+  )
+
   console.timeEnd('Total Time - build-anti-bogus-domain');
 })();

Build/build-apple-cdn.js

@@ -3,6 +3,8 @@ const fs = require('fs');
 const path = require('path');
 
 const { isDomainLoose } = require('./lib/is-domain-loose');
+const { compareAndWriteFile } = require('./lib/string-array-compare');
+const { withBannerArray } = require('./lib/with-banner');
 
 (async () => {
   console.time('Total Time - build-apple-cdn-conf');
@@ -19,15 +21,41 @@ const { isDomainLoose } = require('./lib/is-domain-loose');
     .filter(domain => typeof domain === 'string' && isDomainLoose(domain));
 
   await Promise.all([
-    fs.promises.writeFile(
-      path.resolve(__dirname, '../List/non_ip/apple_cdn.conf'),
-      res.map(domain => `DOMAIN-SUFFIX,${domain}`).join('\n') + '\n',
-      'utf-8'
+    compareAndWriteFile(
+      withBannerArray(
+        'Sukka\'s Surge Rules - Apple CDN',
+        [
+          'License: AGPL 3.0',
+          'Homepage: https://ruleset.skk.moe',
+          'GitHub: https://github.com/SukkaW/Surge',
+          '',
+          'This file contains Apple\'s domains using their China mainland CDN servers.',
+          '',
+          'Data from:',
+          ' - https://github.com/felixonmars/dnsmasq-china-list',
+        ],
+        new Date(),
+        res.map(domain => `DOMAIN-SUFFIX,${domain}`)
+      ),
+      path.resolve(__dirname, '../List/non_ip/apple_cdn.conf')
     ),
-    fs.promises.writeFile(
-      path.resolve(__dirname, '../List/domainset/apple_cdn.conf'),
-      res.map(i => `.${i}`).join('\n') + '\n',
-      'utf-8'
+    compareAndWriteFile(
+      withBannerArray(
+        'Sukka\'s Surge Rules - Apple CDN',
+        [
+          'License: AGPL 3.0',
+          'Homepage: https://ruleset.skk.moe',
+          'GitHub: https://github.com/SukkaW/Surge',
+          '',
+          'This file contains Apple\'s domains using their China mainland CDN servers.',
+          '',
+          'Data from:',
+          ' - https://github.com/felixonmars/dnsmasq-china-list',
+        ],
+        new Date(),
+        res.map(i => `.${i}`)
+      ),
+      path.resolve(__dirname, '../List/domainset/apple_cdn.conf')
     )
   ])
 

Build/build-cdn-conf.js

@@ -1,6 +1,8 @@
 const { fetchWithRetry } = require('./lib/fetch-retry');
 const fs = require('fs');
 const path = require('path');
+const { compareAndWriteFile } = require('./lib/string-array-compare');
+const { withBannerArray } = require('./lib/with-banner');
 
 (async () => {
   console.time('Total Time - build-cdn-conf');
@@ -31,7 +33,21 @@ const path = require('path');
       S3OSSDomains.map(domain => `DOMAIN-SUFFIX,${domain}`).join('\n')
     );
 
-  await fs.promises.writeFile(resultPath, content, 'utf-8');
+  await compareAndWriteFile(
+    withBannerArray(
+      'Sukka\'s Surge Rules - CDN Domains',
+        [
+          'License: AGPL 3.0',
+          'Homepage: https://ruleset.skk.moe',
+          'GitHub: https://github.com/SukkaW/Surge',
+          '',
+          'This file contains object storage and static assets CDN domains.'
+        ],
+        new Date(),
+        content.split('\n')
+    ),
+    resultPath
+  )
 
   console.timeEnd('Total Time - build-cdn-conf');
 })();

Build/build-chn-cidr.js

@@ -1,7 +1,7 @@
 const { fetchWithRetry } = require('./lib/fetch-retry');
-const { withBanner } = require('./lib/with-banner');
-const { promises: fsPromises } = require('fs');
+const { withBannerArray } = require('./lib/with-banner');
 const { resolve: pathResolve } = require('path');
+const { compareAndWriteFile } = require('./lib/string-array-compare');
 
 (async () => {
   console.time('Total Time - build-chnroutes-cidr');
@@ -22,18 +22,21 @@ const { resolve: pathResolve } = require('path');
   }));
   console.log('After Merge:', filteredCidr.length);
 
-  await fsPromises.writeFile(pathResolve(__dirname, '../List/ip/china_ip.conf'), makeCidrList(filteredCidr), { encoding: 'utf-8' });
+  await compareAndWriteFile(
+    withBannerArray(
+      'Sukka\'s Surge Rules - Mainland China IPv4 CIDR',
+      [
+        'License: CC BY-SA 2.0',
+        'Homepage: https://ruleset.skk.moe',
+        'GitHub: https://github.com/SukkaW/Surge',
+        '',
+        'Data from https://misaka.io (misakaio @ GitHub)',
+      ],
+      new Date(),
+      cidr.map(i => `IP-CIDR,${i}`)
+    ),
+    pathResolve(__dirname, '../List/ip/china_ip.conf')
+  )
 
   console.timeEnd('Total Time - build-chnroutes-cidr');
 })();
-
-function makeCidrList(cidr) {
-  const date = new Date();
-
-  return withBanner(
-    'Mainland China IPv4 CIDR',
-    ['Data from misaka.io (misakaio @ GitHub)', 'License: CC BY-SA 2.0'],
-    date,
-    cidr.map(i => `IP-CIDR,${i}`)
-  );
-};

Build/build-phishing-domainset.js

@@ -2,7 +2,8 @@ const tldts = require('tldts');
 const { processFilterRules } = require('./lib/parse-filter.js');
 const fs = require('fs');
 const path = require('path');
-const { withBanner } = require('./lib/with-banner.js');
+const { withBannerArray } = require('./lib/with-banner.js');
+const { stringArrayCompare, compareAndWriteFile } = require('./lib/string-array-compare');
 
 const WHITELIST_DOMAIN = new Set([
   'w3s.link',
@@ -97,19 +98,23 @@ const BLACK_TLD = Array.from(new Set([
     }
   });
 
-  const filePath = path.resolve(__dirname, '../List/domainset/reject_phishing.conf');
-  await fs.promises.writeFile(
-    filePath,
-    withBanner(
-      'Reject Domain Set for Surge',
+  results.sort();
+
+  await compareAndWriteFile(
+    withBannerArray(
+      'Sukka\'s Surge Rules - Reject Phishing',
       [
-        '(Enhanced Phishing Protection)',
+        'License: AGPL 3.0',
+        'Homepage: https://ruleset.skk.moe',
+        'GitHub: https://github.com/SukkaW/Surge',
+        '',
+        'The domainset supports enhanced phishing protection',
         'Build from:',
-        '- https://gitlab.com/malware-filter/phishing-filter'
+        ' - https://gitlab.com/malware-filter/phishing-filter'
       ],
       new Date(),
       results
     ),
-    'utf-8'
-  );
+    path.resolve(__dirname, '../List/domainset/reject_phishing.conf')
+  )
 })();

Build/build-reject-domainset.js

@@ -8,7 +8,8 @@ const { isCI } = require('ci-info');
 const threads = isCI ? cpuCount : cpuCount / 2;
 
 const { HOSTS, ADGUARD_FILTERS, PREDEFINED_WHITELIST } = require('./lib/reject-data-source');
-const { withBanner } = require('./lib/with-banner');
+const { withBannerArray } = require('./lib/with-banner');
+const { compareAndWriteFile } = require('./lib/string-array-compare');
 
 const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST);
 
@@ -208,25 +209,30 @@ const filterRuleWhitelistDomainSets = new Set(PREDEFINED_WHITELIST);
   console.log(`* Dedupe from covered subdomain - ${(Date.now() - START_TIME) / 1000}s`);
   console.log(`Deduped ${previousSize - domainSets.size} rules!`);
 
+  await piscina.destroy();
+
   console.time('* Write reject.conf');
-  await Promise.all([
-    fsPromises.writeFile(
-      pathResolve(__dirname, '../List/domainset/reject.conf'),
-      withBanner(
-        'Reject Domain Set for Surge',
-        [
-          '(AdBlock, Tracking Protection, Privacy Protection, Anti-Phishing, Anti-Mining)',
-          'Build from:',
-          ...HOSTS.map(host => `- ${host[0]}`),
-          ...ADGUARD_FILTERS.map(filter => `- ${Array.isArray(filter) ? filter[0] : filter}`),
-        ],
-        new Date(),
-        [...domainSets].sort()
-      ),
-      { encoding: 'utf-8' }
+
+  await compareAndWriteFile(
+    withBannerArray(
+      'Sukka\'s Surge Rules - Reject Base',
+      [
+        'License: AGPL 3.0',
+        'Homepage: https://ruleset.skk.moe',
+        'GitHub: https://github.com/SukkaW/Surge',
+        '',
+        'The domainset supports AD blocking, tracking protection, privacy protection, anti-phishing, anti-mining',
+        '',
+        'Build from:',
+        ...HOSTS.map(host => ` - ${host[0]}`),
+        ...ADGUARD_FILTERS.map(filter => ` - ${Array.isArray(filter) ? filter[0] : filter}`),
+      ],
+      new Date(),
+      [...domainSets].sort()
     ),
-    piscina.destroy()
-  ]);
+    pathResolve(__dirname, '../List/domainset/reject.conf')
+  );
+
   console.timeEnd('* Write reject.conf');
 
   console.timeEnd('Total Time - build-reject-domain-set');

Build/build-telegram-cidr.js

@@ -2,12 +2,16 @@ const { fetchWithRetry } = require('./lib/fetch-retry');
 const fs = require('fs');
 const path = require('path');
 const { isIPv4, isIPv6 } = require('net');
+const { compareAndWriteFile } = require('./lib/string-array-compare');
+const { withBannerArray } = require('./lib/with-banner');
 
 (async () => {
   console.time('Total Time - build-telegram-cidr');
 
+  /** @type {Response} */
   const resp = await fetchWithRetry('https://core.telegram.org/resources/cidr.txt');
-  const lastModified = new Date(resp.headers.get('last-modified'));
+  const lastModified = resp.headers.get('last-modified');
+  const date = lastModified ? new Date(lastModified) : new Date();
 
   const res = (await resp.text())
     .split('\n')
@@ -17,23 +21,31 @@ const { isIPv4, isIPv6 } = require('net');
     throw new Error('Failed to fetch data!');
   }
 
-  await fs.promises.writeFile(
-    path.resolve(__dirname, '../List/ip/telegram.conf'),
-    '# Telegram CIDR (https://core.telegram.org/resources/cidr.txt)' + '\n' +
-    '# Last Updated: ' + lastModified.toISOString() + '\n' +
-    res.map(ip => {
-      const [subnet] = ip.split('/');
-      console.log('  - ' + ip + ': ' + subnet);
-      if (isIPv4(subnet)) {
-        return `IP-CIDR,${ip},no-resolve`;
-      }
-      if (isIPv6(subnet)) {
-        return `IP-CIDR6,${ip},no-resolve`;
-      }
-      return '';
-    }).join('\n') + '\n',
-    'utf-8'
-  );
+  await compareAndWriteFile(
+    withBannerArray(
+      'Sukka\'s Surge Rules - Telegram IP CIDR',
+      [
+        'License: AGPL 3.0',
+        'Homepage: https://ruleset.skk.moe',
+        'GitHub: https://github.com/SukkaW/Surge',
+        'Data from:',
+        ' - https://core.telegram.org/resources/cidr.txt'
+      ],
+      date,
+      res.map(ip => {
+        const [subnet] = ip.split('/');
+        console.log('  - ' + ip + ': ' + subnet);
+        if (isIPv4(subnet)) {
+          return `IP-CIDR,${ip},no-resolve`;
+        }
+        if (isIPv6(subnet)) {
+          return `IP-CIDR6,${ip},no-resolve`;
+        }
+        return '';
+      })
+    ),
+    path.resolve(__dirname, '../List/ip/telegram.conf')
+  )
 
   console.timeEnd('Total Time - build-telegram-cidr');
 })();

Build/download-previous-build.js

@@ -0,0 +1,46 @@
+const { fetch } = require('undici');
+const tar = require('tar');
+const fs = require('fs');
+const fse = require('fs-extra');
+const { join, resolve } = require('path');
+const { tmpdir } = require('os');
+const { Stream, Readable } = require('stream');
+const { promisify } = require('util');
+const pipeline = promisify(Stream.pipeline);
+
+(async () => {
+  const filesList = (
+    await fs.promises.readFile(resolve(__dirname, '../.gitignore'), { encoding: 'utf-8' })
+  )
+    .split('\n')
+    .filter(p => p.startsWith('List/'));
+
+  const tempFile = join(tmpdir(), `sukka-surge-last-build-tar-${Date.now()}`);
+  const resp = await fetch('https://codeload.github.com/sukkaw/surge/tar.gz/gh-pages');
+  const readableNodeStream = Readable.fromWeb(resp.body);
+  await pipeline(
+    readableNodeStream,
+    fs.createWriteStream(tempFile)
+  );
+
+  const extractedPath = join(tmpdir(), `sukka-surge-last-build-extracted-${Date.now()}`);
+  await fse.ensureDir(extractedPath);
+  await tar.x({
+    file: tempFile,
+    cwd: extractedPath,
+    filter: (p) => {
+      return p.split('/')[1] === 'List'
+    }
+  });
+
+  await Promise.all(filesList.map(p => fse.copy(
+    join(extractedPath, 'Surge-gh-pages', p),
+    join(__dirname, '..', p),
+    {
+      overwrite: true
+    }
+  )))
+
+  await fs.promises.unlink(tempFile).catch(() => {});
+  await fs.promises.unlink(extractedPath).catch(() => {});
+})();

Build/lib/string-array-compare.js

@@ -0,0 +1,35 @@
+const { promises: fsPromises } = require('fs');
+
+async function compareAndWriteFile(linesA, filePath) {
+  const linesB = (await fsPromises.readFile(filePath, { encoding: 'utf-8' })).split('\n');
+
+  if (!stringArrayCompare(linesA, linesB)) {
+    await fsPromises.writeFile(
+      filePath,
+      linesA.join('\n'),
+      { encoding: 'utf-8' }
+    )
+  } else {
+    console.log(`Same Content, bail out writing: ${filePath}`);
+  }
+}
+
+function stringArrayCompare (linesA, linesB) {
+  if (linesA.length !== linesB.length) return false;
+
+  for (let i = 0; i < linesA.length; i++) {
+    const lineA = linesA[i];
+    const lineB = linesB[i];
+    if (lineA.startsWith('#') && lineB.startsWith('#')) {
+      continue;
+    }
+    if (lineA !== lineB) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+module.exports.stringArrayCompare = stringArrayCompare;
+module.exports.compareAndWriteFile = compareAndWriteFile;

Build/lib/with-banner.js

@@ -6,12 +6,33 @@
  * @returns {string}
  */
 const withBanner = (title, description, date, content) => {
-    return `########################################
+  return `########################################
 # ${title}
-${description.map(line => `# ${line}`).join('\n')}
 # Last Updated: ${date.toISOString()}
 # Size: ${content.length}
+${description.map(line => line ? `# ${line}` : '#').join('\n')}
 ########################################\n` + content.join('\n') + '\n################# END ###################\n';
 };
+/**
+ * @param {string} title
+ * @param {string[]} description
+ * @param {Date} date
+ * @param {string[]} content
+ * @returns {string[]}
+ */
+const withBannerArray = (title, description, date, content) => {
+  return [
+    `########################################`,
+    `# ${title}`,
+    `# Last Updated: ${date.toISOString()}`,
+    `# Size: ${content.length}`,
+    ...description.map(line => line ? `# ${line}` : '#'),
+    `########################################`,
+    ...content,
+    `################# END ###################`,
+    ''
+  ];
+};
 
 module.exports.withBanner = withBanner;
+module.exports.withBannerArray = withBannerArray;