add backup url for China IPs

console-api/package-lock.json

@@ -18,6 +18,7 @@
         "@nestjs/config": "^1.0.1",
         "@nestjs/core": "^8.0.0",
         "@nestjs/platform-express": "^8.0.0",
+        "@nestjs/schedule": "^3.0.3",
         "@nestjs/serve-static": "^2.2.2",
         "@nestjs/swagger": "^5.0.9",
         "@nestjs/typeorm": "^8.0.2",
@@ -29,6 +30,7 @@
         "delay": "^5.0.0",
         "ioredis": "^4.28.5",
         "ip6addr": "^0.2.5",
+        "ipaddr.js": "^2.2.0",
         "lodash": "^4.17.21",
         "moment": "^2.29.1",
         "mustache": "^4.2.0",
@@ -2849,6 +2851,30 @@
         "node": ">= 0.8.0"
       }
     },
+    "node_modules/@nestjs/schedule": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/@nestjs/schedule/-/schedule-3.0.3.tgz",
+      "integrity": "sha512-xsMA4dmP3LcW3rt2iMPfm88bDbCj/hLuDsLrKmJQlbnxyCYtBwLtmu/4cSfZELLM7pTDT+E8QDAqGwhYyUUjxg==",
+      "license": "MIT",
+      "dependencies": {
+        "cron": "2.4.1",
+        "uuid": "9.0.0"
+      },
+      "peerDependencies": {
+        "@nestjs/common": "^8.0.0 || ^9.0.0 || ^10.0.0",
+        "@nestjs/core": "^8.0.0 || ^9.0.0 || ^10.0.0",
+        "reflect-metadata": "^0.1.12"
+      }
+    },
+    "node_modules/@nestjs/schedule/node_modules/uuid": {
+      "version": "9.0.0",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz",
+      "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==",
+      "license": "MIT",
+      "bin": {
+        "uuid": "dist/bin/uuid"
+      }
+    },
     "node_modules/@nestjs/schematics": {
       "version": "8.0.2",
       "resolved": "https://registry.npmjs.org/@nestjs/schematics/-/schematics-8.0.2.tgz",
@@ -4906,6 +4932,15 @@
       "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==",
       "dev": true
     },
+    "node_modules/cron": {
+      "version": "2.4.1",
+      "resolved": "https://registry.npmjs.org/cron/-/cron-2.4.1.tgz",
+      "integrity": "sha512-ty0hUSPuENwDtIShDFxUxWEIsqiu2vhoFtt6Vwrbg4lHGtJX2/cV2p0hH6/qaEM9Pj+i6mQoau48BO5wBpkP4w==",
+      "license": "MIT",
+      "dependencies": {
+        "luxon": "^3.2.1"
+      }
+    },
     "node_modules/cross-spawn": {
       "version": "7.0.3",
       "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
@@ -6750,11 +6785,12 @@
       }
     },
     "node_modules/ipaddr.js": {
-      "version": "1.9.1",
-      "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
-      "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==",
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.2.0.tgz",
+      "integrity": "sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA==",
+      "license": "MIT",
       "engines": {
-        "node": ">= 0.10"
+        "node": ">= 10"
       }
     },
     "node_modules/is-arrayish": {
@@ -8480,6 +8516,15 @@
         "node": ">=10"
       }
     },
+    "node_modules/luxon": {
+      "version": "3.7.1",
+      "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.7.1.tgz",
+      "integrity": "sha512-RkRWjA926cTvz5rAb1BqyWkKbbjzCGchDUIKMCUvNi17j6f6j8uHGDV82Aqcqtzd+icoYpELmG3ksgGiFNNcNg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=12"
+      }
+    },
     "node_modules/macos-release": {
       "version": "2.5.0",
       "resolved": "https://registry.npmjs.org/macos-release/-/macos-release-2.5.0.tgz",
@@ -9447,6 +9492,15 @@
         "node": ">= 0.10"
       }
     },
+    "node_modules/proxy-addr/node_modules/ipaddr.js": {
+      "version": "1.9.1",
+      "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
+      "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.10"
+      }
+    },
     "node_modules/psl": {
       "version": "1.8.0",
       "resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz",
@@ -14102,6 +14156,22 @@
         }
       }
     },
+    "@nestjs/schedule": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/@nestjs/schedule/-/schedule-3.0.3.tgz",
+      "integrity": "sha512-xsMA4dmP3LcW3rt2iMPfm88bDbCj/hLuDsLrKmJQlbnxyCYtBwLtmu/4cSfZELLM7pTDT+E8QDAqGwhYyUUjxg==",
+      "requires": {
+        "cron": "2.4.1",
+        "uuid": "9.0.0"
+      },
+      "dependencies": {
+        "uuid": {
+          "version": "9.0.0",
+          "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz",
+          "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg=="
+        }
+      }
+    },
     "@nestjs/schematics": {
       "version": "8.0.2",
       "resolved": "https://registry.npmjs.org/@nestjs/schematics/-/schematics-8.0.2.tgz",
@@ -15705,6 +15775,14 @@
       "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==",
       "dev": true
     },
+    "cron": {
+      "version": "2.4.1",
+      "resolved": "https://registry.npmjs.org/cron/-/cron-2.4.1.tgz",
+      "integrity": "sha512-ty0hUSPuENwDtIShDFxUxWEIsqiu2vhoFtt6Vwrbg4lHGtJX2/cV2p0hH6/qaEM9Pj+i6mQoau48BO5wBpkP4w==",
+      "requires": {
+        "luxon": "^3.2.1"
+      }
+    },
     "cross-spawn": {
       "version": "7.0.3",
       "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
@@ -17073,9 +17151,9 @@
       }
     },
     "ipaddr.js": {
-      "version": "1.9.1",
-      "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
-      "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.2.0.tgz",
+      "integrity": "sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA=="
     },
     "is-arrayish": {
       "version": "0.2.1",
@@ -18399,6 +18477,11 @@
         "yallist": "^4.0.0"
       }
     },
+    "luxon": {
+      "version": "3.7.1",
+      "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.7.1.tgz",
+      "integrity": "sha512-RkRWjA926cTvz5rAb1BqyWkKbbjzCGchDUIKMCUvNi17j6f6j8uHGDV82Aqcqtzd+icoYpELmG3ksgGiFNNcNg=="
+    },
     "macos-release": {
       "version": "2.5.0",
       "resolved": "https://registry.npmjs.org/macos-release/-/macos-release-2.5.0.tgz",
@@ -19114,6 +19197,13 @@
       "requires": {
         "forwarded": "0.2.0",
         "ipaddr.js": "1.9.1"
+      },
+      "dependencies": {
+        "ipaddr.js": {
+          "version": "1.9.1",
+          "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
+          "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="
+        }
       }
     },
     "psl": {

console-api/package.json

@@ -30,6 +30,7 @@
     "@nestjs/config": "^1.0.1",
     "@nestjs/core": "^8.0.0",
     "@nestjs/platform-express": "^8.0.0",
+    "@nestjs/schedule": "^3.0.3",
     "@nestjs/serve-static": "^2.2.2",
     "@nestjs/swagger": "^5.0.9",
     "@nestjs/typeorm": "^8.0.2",
@@ -41,6 +42,7 @@
     "delay": "^5.0.0",
     "ioredis": "^4.28.5",
     "ip6addr": "^0.2.5",
+    "ipaddr.js": "^2.2.0",
     "lodash": "^4.17.21",
     "moment": "^2.29.1",
     "mustache": "^4.2.0",

console-api/src/app.module.ts

@@ -21,6 +21,8 @@ import { MirrorService } from './mirror/mirror.service';
 import { HttpModule } from '@nestjs/axios';
 import { RedisModule } from '@nestjs-modules/ioredis';
 // import { LockService } from './lock/lock.service';
+import { ChnrouteService } from './chnroute/chnroute.service';
+import { ScheduleModule } from '@nestjs/schedule';
 
 @Module({
   imports: [
@@ -58,8 +60,17 @@ import { RedisModule } from '@nestjs-modules/ioredis';
         },
       }),
     }),
+    ScheduleModule.forRoot(),
   ],
   controllers: [AppController, AdminController, UpdateController],
-  providers: [AppService, PackagerService, AssetsS3Service, PackageS3Service, UpdateService, MirrorService /*LockService*/],
+  providers: [
+    AppService,
+    PackagerService,
+    AssetsS3Service,
+    PackageS3Service,
+    UpdateService,
+    MirrorService,
+    ChnrouteService /*LockService*/,
+  ],
 })
 export class AppModule {}

console-api/src/chnroute/chnroute.service.spec.ts

+import { Test, TestingModule } from '@nestjs/testing';
+import { ChnrouteService } from './chnroute.service';
+
+describe('ChnrouteService', () => {
+  let service: ChnrouteService;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      providers: [ChnrouteService],
+    }).compile();
+
+    service = module.get<ChnrouteService>(ChnrouteService);
+  });
+
+  it('should be defined', () => {
+    expect(service).toBeDefined();
+  });
+});

console-api/src/chnroute/chnroute.service.ts

+import { ConsoleLogger, Injectable } from '@nestjs/common';
+import { HttpService } from '@nestjs/axios';
+import Address from 'ipaddr.js';
+import { lastValueFrom } from 'rxjs';
+import { Interval } from '@nestjs/schedule';
+
+@Injectable()
+export class ChnrouteService extends ConsoleLogger {
+  constructor(private http: HttpService) {
+    super('ChnrouteService');
+  }
+
+  private misakaUrl = 'https://raw.githubusercontent.com/misakaio/chnroutes2/master/chnroutes.txt';
+  private chinaIpRnages: [Address.IPv4, number][] = [];
+
+  async onApplicationBootstrap() {
+    this.updateChnroute().then();
+  }
+
+  @Interval(3600000)
+  async updateChnroute() {
+    try {
+      const { data } = await lastValueFrom(
+        this.http.get<string>(this.misakaUrl, {
+          responseType: 'text',
+        })
+      );
+      const ranges = data
+        .trim()
+        .split('\n')
+        .map((line) => line.trim())
+        .filter((line) => line && !line.startsWith('#'));
+      this.chinaIpRnages = ranges.map((range) => Address.parseCIDR(range) as [Address.IPv4, number]);
+      this.log(`update chnroute success: ${this.chinaIpRnages.length}`);
+    } catch (e) {
+      this.error(`update chnroute error: ${e.message}`);
+    }
+  }
+
+  isChina(ip: string) {
+    if (ip === '::1') return false;
+    if (this.chinaIpRnages.length) {
+      const ipObj = Address.parse(ip);
+      if (ipObj.kind() === 'ipv6') {
+        return false; // IPv6 is not supported in this service
+      }
+      return this.chinaIpRnages.some(([range, mask]) => {
+        return ipObj.match(range, mask);
+      });
+    } else {
+      return false;
+    }
+  }
+}

console-api/src/entities/Archive.entity.ts

@@ -50,16 +50,18 @@ export class Archive extends TimeBase {
     return 5 + (this.files?.length || 0);
   }
 
-  toMetalinkView() {
-    return {
-      name: this.archiveFullPath,
-    };
-  }
-
   url: string;
+  backupUrl: string;
 
-  fillUrl(cdnUrl: string, cdnUrlOversize: string) {
-    const useCdn = this.size > 536870912 ? cdnUrlOversize : cdnUrl;
-    this.url = `${useCdn}/${this.path}.tar.zst`;
+  fillUrl(cdnUrl: string, cdnUrlOversize: string, addBackup = false) {
+    const overSize = this.size > 536870912;
+    if (overSize) {
+      this.url = `${cdnUrlOversize}/${this.archiveFullPath}`;
+    } else {
+      this.url = `${cdnUrl}/${this.archiveFullPath}`;
+      if (addBackup) {
+        this.backupUrl = `${cdnUrlOversize}/${this.archiveFullPath}`;
+      }
+    }
   }
 }

console-api/src/update/update.service.ts

@@ -12,6 +12,7 @@ import { AppsJson } from '../utility/apps-json-type';
 import { MirrorService } from '../mirror/mirror.service';
 import moment from 'moment';
 import Platform = AppsJson.Platform;
+import { ChnrouteService } from '../chnroute/chnroute.service';
 
 @Injectable()
 export class UpdateService extends ConsoleLogger {
@@ -21,7 +22,8 @@ export class UpdateService extends ConsoleLogger {
     @InjectConnection('app')
     private db: Connection,
     packageS3: PackageS3Service,
-    private mirror: MirrorService
+    private mirror: MirrorService,
+    private chnroute: ChnrouteService
   ) {
     super('update');
     this.cdnUrl = packageS3.cdnUrl;
@@ -142,6 +144,13 @@ export class UpdateService extends ConsoleLogger {
     };
   }
 
+  private async prepareUrl(archives: Archive[], ip: string) {
+    await this.mirror.lookForArchivesMirror(archives, ip);
+    const isChina = this.chnroute.isChina(ip);
+    archives.forEach((a) => a.fillUrl(this.cdnUrl, this.cdnUrlOversize, isChina));
+    return archives;
+  }
+
   async getSinglePackageMetalink(path: string, ip: string) {
     const archives = await this.db.getRepository(Archive).find({
       where: { path },
@@ -151,10 +160,8 @@ export class UpdateService extends ConsoleLogger {
     if (!archives.length) {
       throw new BlankReturnMessageDto(404, 'Archive not found').toException();
     }
-    await this.mirror.lookForArchivesMirror(archives, ip);
-    archives.forEach((a) => a.fillUrl(this.cdnUrl, this.cdnUrlOversize));
     return {
-      archives,
+      archives: await this.prepareUrl(archives, ip),
     };
   }
 
@@ -171,10 +178,9 @@ export class UpdateService extends ConsoleLogger {
     if (!archives.length) {
       archives = await tryRole(ArchiveType.Full);
     }
-    await this.mirror.lookForArchivesMirror(archives, ip);
-    archives.forEach((a) => a.fillUrl(this.cdnUrl, this.cdnUrlOversize));
+
     return {
-      archives: archives,
+      archives: await this.prepareUrl(archives, ip),
     };
   }
 
@@ -264,10 +270,8 @@ export class UpdateService extends ConsoleLogger {
       // If single update archive satisfies all requested files, return it.
       if (!remainingFiles.length) {
         // console.log('exact', bestUpdateArchive);updateArchives
-        await this.mirror.lookForArchivesMirror([bestUpdateArchive], ip);
-        bestUpdateArchive.fillUrl(this.cdnUrl, this.cdnUrlOversize);
         return {
-          archives: [bestUpdateArchive],
+          archives: await this.prepareUrl([bestUpdateArchive], ip),
         };
       }
     }
@@ -314,10 +318,8 @@ export class UpdateService extends ConsoleLogger {
 
     const archives = _.minBy(packagePlans, (plan) => this.getCostOfArchives(plan));
     // console.log('use', archives);
-    await this.mirror.lookForArchivesMirror(archives, ip);
-    archives.forEach((a) => a.fillUrl(this.cdnUrl, this.cdnUrlOversize));
     return {
-      archives,
+      archives: await this.prepareUrl(archives, ip),
     };
   }
 }