one package togo

console-api/package-lock.json

@@ -23,6 +23,7 @@
         "lodash": "^4.17.21",
         "moment": "^2.29.1",
         "mustache": "^4.2.0",
+        "mustache-express": "^1.3.1",
         "pg": "^8.7.1",
         "readdirp": "^3.6.0",
         "reflect-metadata": "^0.1.13",
@@ -41,6 +42,7 @@
         "@types/lodash": "^4.14.172",
         "@types/multer": "^1.4.7",
         "@types/mustache": "^4.1.2",
+        "@types/mustache-express": "^1.2.1",
         "@types/node": "^16.0.0",
         "@types/supertest": "^2.0.11",
         "@typescript-eslint/eslint-plugin": "^4.28.2",
@@ -3215,6 +3217,12 @@
       "integrity": "sha512-/BHF5HAx3em7/KkzVKm3LrsD6HZAXuXO1AJZQ3cRRBZj4oHZDviWPYu0aEplAqDFNHZPW6d3G7KN+ONcCCC7pw==",
       "dev": true
     },
+    "node_modules/@types/lru-cache": {
+      "version": "4.1.3",
+      "resolved": "https://registry.npmjs.org/@types/lru-cache/-/lru-cache-4.1.3.tgz",
+      "integrity": "sha512-QjCOmf5kYwekcsfEKhcEHMK8/SvgnneuSDXFERBuC/DPca2KJIO/gpChTsVb35BoWLBpEAJWz1GFVEArSdtKtw==",
+      "dev": true
+    },
     "node_modules/@types/mime": {
       "version": "1.3.2",
       "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz",
@@ -3236,6 +3244,15 @@
       "integrity": "sha512-c4OVMMcyodKQ9dpwBwh3ofK9P6U9ZktKU9S+p33UqwMNN1vlv2P0zJZUScTshnx7OEoIIRcCFNQ904sYxZz8kg==",
       "dev": true
     },
+    "node_modules/@types/mustache-express": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/@types/mustache-express/-/mustache-express-1.2.1.tgz",
+      "integrity": "sha512-wZ/XqSpJjlOf7qI8ch/oaD/zHfC8K/TM+kSdL/CVdiPp1xu7DoI4jk/4vR4vkEYKeS2zV54q8Le9C0NebtvoSg==",
+      "dev": true,
+      "dependencies": {
+        "@types/lru-cache": "^4"
+      }
+    },
     "node_modules/@types/node": {
       "version": "16.6.1",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-16.6.1.tgz",
@@ -3870,6 +3887,11 @@
         "node": ">=8"
       }
     },
+    "node_modules/async": {
+      "version": "3.2.1",
+      "resolved": "https://registry.npmjs.org/async/-/async-3.2.1.tgz",
+      "integrity": "sha512-XdD5lRO/87udXCMC9meWdYiR+Nq6ZjUfXidViUZGu2F1MO4T3XwZ1et0hb2++BgLfhyJwy44BGB/yx80ABx8hg=="
+    },
     "node_modules/asynckit": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
@@ -8363,6 +8385,32 @@
         "mustache": "bin/mustache"
       }
     },
+    "node_modules/mustache-express": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/mustache-express/-/mustache-express-1.3.1.tgz",
+      "integrity": "sha512-RSSzrvM+CVAk9217dkWSNYyl6c2JnesNn6zaZ8+FvZSn8aLxY9l4kTnYqIoiE8GxdLyVQL2ak7XlMZS6t/l8YA==",
+      "dependencies": {
+        "async": "~3.2.0",
+        "lru-cache": "~5.1.1",
+        "mustache": "^4.2.0"
+      },
+      "engines": {
+        "node": ">= 0.8.0"
+      }
+    },
+    "node_modules/mustache-express/node_modules/lru-cache": {
+      "version": "5.1.1",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
+      "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==",
+      "dependencies": {
+        "yallist": "^3.0.2"
+      }
+    },
+    "node_modules/mustache-express/node_modules/yallist": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",
+      "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="
+    },
     "node_modules/mute-stream": {
       "version": "0.0.8",
       "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz",
@@ -14099,6 +14147,12 @@
       "integrity": "sha512-/BHF5HAx3em7/KkzVKm3LrsD6HZAXuXO1AJZQ3cRRBZj4oHZDviWPYu0aEplAqDFNHZPW6d3G7KN+ONcCCC7pw==",
       "dev": true
     },
+    "@types/lru-cache": {
+      "version": "4.1.3",
+      "resolved": "https://registry.npmjs.org/@types/lru-cache/-/lru-cache-4.1.3.tgz",
+      "integrity": "sha512-QjCOmf5kYwekcsfEKhcEHMK8/SvgnneuSDXFERBuC/DPca2KJIO/gpChTsVb35BoWLBpEAJWz1GFVEArSdtKtw==",
+      "dev": true
+    },
     "@types/mime": {
       "version": "1.3.2",
       "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz",
@@ -14120,6 +14174,15 @@
       "integrity": "sha512-c4OVMMcyodKQ9dpwBwh3ofK9P6U9ZktKU9S+p33UqwMNN1vlv2P0zJZUScTshnx7OEoIIRcCFNQ904sYxZz8kg==",
       "dev": true
     },
+    "@types/mustache-express": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/@types/mustache-express/-/mustache-express-1.2.1.tgz",
+      "integrity": "sha512-wZ/XqSpJjlOf7qI8ch/oaD/zHfC8K/TM+kSdL/CVdiPp1xu7DoI4jk/4vR4vkEYKeS2zV54q8Le9C0NebtvoSg==",
+      "dev": true,
+      "requires": {
+        "@types/lru-cache": "^4"
+      }
+    },
     "@types/node": {
       "version": "16.6.1",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-16.6.1.tgz",
@@ -14617,6 +14680,11 @@
       "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==",
       "dev": true
     },
+    "async": {
+      "version": "3.2.1",
+      "resolved": "https://registry.npmjs.org/async/-/async-3.2.1.tgz",
+      "integrity": "sha512-XdD5lRO/87udXCMC9meWdYiR+Nq6ZjUfXidViUZGu2F1MO4T3XwZ1et0hb2++BgLfhyJwy44BGB/yx80ABx8hg=="
+    },
     "asynckit": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
@@ -18050,6 +18118,31 @@
       "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz",
       "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ=="
     },
+    "mustache-express": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/mustache-express/-/mustache-express-1.3.1.tgz",
+      "integrity": "sha512-RSSzrvM+CVAk9217dkWSNYyl6c2JnesNn6zaZ8+FvZSn8aLxY9l4kTnYqIoiE8GxdLyVQL2ak7XlMZS6t/l8YA==",
+      "requires": {
+        "async": "~3.2.0",
+        "lru-cache": "~5.1.1",
+        "mustache": "^4.2.0"
+      },
+      "dependencies": {
+        "lru-cache": {
+          "version": "5.1.1",
+          "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
+          "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==",
+          "requires": {
+            "yallist": "^3.0.2"
+          }
+        },
+        "yallist": {
+          "version": "3.1.1",
+          "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",
+          "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="
+        }
+      }
+    },
     "mute-stream": {
       "version": "0.0.8",
       "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz",

console-api/package.json

@@ -36,6 +36,7 @@
     "lodash": "^4.17.21",
     "moment": "^2.29.1",
     "mustache": "^4.2.0",
+    "mustache-express": "^1.3.1",
     "pg": "^8.7.1",
     "readdirp": "^3.6.0",
     "reflect-metadata": "^0.1.13",
@@ -57,6 +58,7 @@
     "@types/lodash": "^4.14.172",
     "@types/multer": "^1.4.7",
     "@types/mustache": "^4.1.2",
+    "@types/mustache-express": "^1.2.1",
     "@types/node": "^16.0.0",
     "@types/supertest": "^2.0.11",
     "@typescript-eslint/eslint-plugin": "^4.28.2",

console-api/src/app.controller.ts

@@ -39,11 +39,6 @@ import { DepotDto } from './dto/Depot.dto';
 export class AppController {
   constructor(private readonly appService: AppService, private readonly s3: AssetsS3Service) {}
 
-  @Get('apps.json')
-  getAppsJson() {
-    return this.appService.getAppsJson();
-  }
-
   @Get('app')
   @ApiOperation({
     summary: '获取 app',

console-api/src/app.module.ts

@@ -12,6 +12,8 @@ import { PackageS3Service } from './package-s3/package-s3.service';
 import { Archive } from './entities/Archive.entity';
 import { Build } from './entities/Build.entity';
 import { Depot } from './entities/Depot.entity';
+import { UpdateController } from './update/update.controller';
+import { UpdateService } from './update/update.service';
 
 const configModule = ConfigModule.forRoot();
 
@@ -36,7 +38,7 @@ const configModule = ConfigModule.forRoot();
       },
     }),
   ],
-  controllers: [AppController, AdminController],
-  providers: [AppService, PackagerService, AssetsS3Service, PackageS3Service],
+  controllers: [AppController, AdminController, UpdateController],
+  providers: [AppService, PackagerService, AssetsS3Service, PackageS3Service, UpdateService],
 })
 export class AppModule {}

console-api/src/app.service.ts

@@ -32,10 +32,6 @@ export class AppService extends ConsoleLogger {
     this.packageVersionPreserveCount = parseInt(config.get('PACKAGE_VERSION_PRESERVE_COUNT')) || 5;
   }
 
-  async getAppsJson() {
-    return (await this.db.getRepository(App).find({ where: { appData: Not(IsNull()), isDeleted: false } })).map((a) => a.appData);
-  }
-
   private async updateResult<T>(f: () => Promise<T>, returnCode = 200) {
     try {
       const result = await f();
@@ -129,7 +125,7 @@ export class AppService extends ConsoleLogger {
     }
     appData.id = id;
     const app = await this.db.getRepository(App).findOne({
-      where: { id: appData.id },
+      where: { id: appData.id, isDeleted: false },
       relations: ['history'],
       select: ['id', 'author', 'appData'],
     });
@@ -171,7 +167,7 @@ export class AppService extends ConsoleLogger {
     }
     //this.log('Build: Checking app.');
     const app = await this.db.getRepository(App).findOne({
-      where: { id },
+      where: { id, isDeleted: false },
       select: ['id', 'packagePrefix', 'author'],
     });
     if (!app) {
@@ -232,7 +228,7 @@ export class AppService extends ConsoleLogger {
 
     query.andWhere(`unusedBuild.id not in ${subQuery.getQuery()}`);
     this.packageReferenceSubQuery(query);
-    return (await query.getRawMany()).map((s) => s.pathToPurge as string);
+    return (await query.getRawMany()).map((s) => `${s.pathToPurge}.tar.gz` as string);
   }
 
   private async getArchivePathsToPurge(buildId: number) {
@@ -244,7 +240,7 @@ export class AppService extends ConsoleLogger {
 
     this.packageReferenceSubQuery(query);
     // this.log(`SQL: ${query.getQueryAndParameters()}`);
-    return (await query.getRawMany()).map((s) => s.pathToPurge as string);
+    return (await query.getRawMany()).map((s) => `${s.pathToPurge}.tar.gz` as string);
   }
 
   async purgeOldArchives() {
@@ -270,7 +266,7 @@ export class AppService extends ConsoleLogger {
       throw new BlankReturnMessageDto(401, 'Needs login').toException();
     }
     const app = await this.db.getRepository(App).findOne({
-      where: { id },
+      where: { id, isDeleted: false },
       select: ['id', 'author'],
     });
     if (!app) {

console-api/src/entities/App.entity.ts

@@ -16,13 +16,13 @@ export class App extends AppBase {
   @Column('int', { nullable: true, array: true })
   author: number[];
 
-  @Column({ nullable: true, select: false })
+  @Column({ default: false, select: false })
   isDeleted: boolean;
 
   @Column('varchar', { length: 128, nullable: true })
   packagePrefix: string;
 
-  @OneToMany(() => Depot, depot => depot.app)
+  @OneToMany(() => Depot, (depot) => depot.app)
   depots: Depot[];
 
   get packageFullPath() {

console-api/src/entities/Archive.entity.ts

@@ -18,7 +18,7 @@ export class Archive extends TimeBase {
   files: string[];
 
   @Index()
-  @Column('varchar', { length: 140 })
+  @Column('varchar', { length: 128 })
   path: string;
 
   @Column('int', { unsigned: true })
@@ -29,4 +29,14 @@ export class Archive extends TimeBase {
 
   @Column({ type: 'enum', enum: ArchiveType })
   role: ArchiveType;
+
+  get archiveFullPath() {
+    return `${this.path}.tar.gz`;
+  }
+
+  toMetalinkView() {
+    return {
+      name: this.archiveFullPath,
+    };
+  }
 }

console-api/src/main.ts

@@ -2,17 +2,25 @@ import { NestFactory } from '@nestjs/core';
 import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
 import { NestExpressApplication } from '@nestjs/platform-express';
 import { AppModule } from './app.module';
+import mustacheExpress from 'mustache-express';
+import path from 'path';
 
 async function bootstrap() {
   const app = await NestFactory.create<NestExpressApplication>(AppModule);
   app.enableCors();
   app.set('trust proxy', ['172.16.0.0/12', 'loopback']);
 
+  app.setBaseViewsDir(path.join(__dirname, '..', 'views'));
+  const engine = mustacheExpress();
+  app.engine('mustache', engine);
+  app.setViewEngine('mustache');
+
   const documentConfig = new DocumentBuilder()
     .setTitle('app')
     .setDescription('The app')
     .setVersion('1.0')
     .addTag('admin', '只有萌卡管理员可以用')
+    .addTag('update', '萌卡客户端使用的')
     .build();
 
   const document = SwaggerModule.createDocument(app, documentConfig);

console-api/src/packager/packager.service.ts

@@ -24,10 +24,13 @@ export interface FileWithHash {
 export class ArchiveTask {
   readonly path: string;
   constructor(public readonly role: ArchiveType, public readonly files: FileWithHash[], public readonly altFiles?: string[]) {
-    this.path =
-      createHash('sha512')
-        .update(files.map((f) => `${f.file.path}${f.hash}`).join(''))
-        .digest('hex') + '.tar.gz';
+    this.path = createHash('sha512')
+      .update(files.map((f) => `${f.file.path}${f.hash}`).join(''))
+      .digest('hex');
+  }
+
+  get archiveFullPath() {
+    return `${this.path}.tar.gz`;
   }
 
   get filePaths() {
@@ -166,7 +169,7 @@ export class PackagerService extends ConsoleLogger {
 
   async archive(root: string, archiveTask: ArchiveTask): Promise<Archive> {
     const archive = archiveTask.archive;
-    const archiveName = archiveTask.path;
+    const archiveName = archiveTask.archiveFullPath;
     const existing = await this.s3.fileExists(archiveName);
     if (existing) {
       archive.size = existing.Size;

console-api/src/update/update.controller.spec.ts

+import { Test, TestingModule } from '@nestjs/testing';
+import { UpdateController } from './update.controller';
+
+describe('UpdateController', () => {
+  let controller: UpdateController;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      controllers: [UpdateController],
+    }).compile();
+
+    controller = module.get<UpdateController>(UpdateController);
+  });
+
+  it('should be defined', () => {
+    expect(controller).toBeDefined();
+  });
+});

console-api/src/update/update.controller.ts

+import { Controller, Get, Param, Query, Render, ValidationPipe } from '@nestjs/common';
+import { UpdateService } from './update.service';
+import { ApiOkResponse, ApiOperation, ApiParam, ApiProperty, ApiQuery, ApiTags } from '@nestjs/swagger';
+import { DepotDto } from '../dto/Depot.dto';
+
+@Controller('update')
+@ApiTags('update')
+export class UpdateController {
+  constructor(private readonly updateService: UpdateService) {}
+  @Get('apps.json')
+  @ApiOperation({ summary: '获取 apps.json', description: '懒得解释这是啥了……' })
+  getAppsJson() {
+    return this.updateService.getAppsJson();
+  }
+
+  @Get('checksums/:id/:version')
+  @Render('checksums')
+  @ApiOperation({ summary: '获取 app 校验和', description: '是 shasum 的格式' })
+  @ApiParam({ name: 'id', description: 'APP 的 id' })
+  @ApiParam({ name: 'version', description: 'APP 的版本号' })
+  @ApiQuery({ type: DepotDto, description: 'APP 的类型' })
+  @ApiOkResponse({ type: String })
+  async getChecksum(
+    @Param('id') id: string,
+    @Query(new ValidationPipe({ transform: true })) depot: DepotDto,
+    @Param('version') version: string
+  ) {
+    return this.updateService.getChecksum(id, depot, version);
+  }
+
+  @Get('metalinks/:id/:version')
+  @Render('metalinks')
+  @ApiOperation({ summary: '获取 app 完整包 metalink', description: '只包含完整包的' })
+  @ApiParam({ name: 'id', description: 'APP 的 id' })
+  @ApiParam({ name: 'version', description: 'APP 的版本号' })
+  @ApiQuery({ type: DepotDto, description: 'APP 的类型' })
+  @ApiOkResponse({ type: String })
+  async getFullPackageMetalink(
+    @Param('id') id: string,
+    @Query(new ValidationPipe({ transform: true })) depot: DepotDto,
+    @Param('version') version: string
+  ) {
+    return this.updateService.getFullPackageMetalink(id, depot, version);
+  }
+}

console-api/src/update/update.service.spec.ts

+import { Test, TestingModule } from '@nestjs/testing';
+import { UpdateService } from './update.service';
+
+describe('UpdateService', () => {
+  let service: UpdateService;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      providers: [UpdateService],
+    }).compile();
+
+    service = module.get<UpdateService>(UpdateService);
+  });
+
+  it('should be defined', () => {
+    expect(service).toBeDefined();
+  });
+});

console-api/src/update/update.service.ts

+import { ConsoleLogger, Injectable } from '@nestjs/common';
+import { InjectConnection } from '@nestjs/typeorm';
+import { Connection, IsNull, Not, SelectQueryBuilder } from 'typeorm';
+import { App } from '../entities/App.entity';
+import { DepotDto } from '../dto/Depot.dto';
+import { Build } from '../entities/Build.entity';
+import { BlankReturnMessageDto } from '../dto/ReturnMessage.dto';
+import { ArchiveType } from '../entities/Archive.entity';
+import { PackageS3Service } from '../package-s3/package-s3.service';
+
+@Injectable()
+export class UpdateService extends ConsoleLogger {
+  private readonly cdnUrl: string;
+  constructor(
+    @InjectConnection('app')
+    private db: Connection,
+    packageS3: PackageS3Service
+  ) {
+    super('update');
+    this.cdnUrl = packageS3.cdnUrl;
+  }
+
+  async getAppsJson() {
+    return (await this.db.getRepository(App).find({ where: { appData: Not(IsNull()), isDeleted: false } })).map((a) => a.appData);
+  }
+
+  private async getBuild(id: string, depotDto: DepotDto, version: string, extraQuery?: (query: SelectQueryBuilder<Build>) => void) {
+    const depotObj = depotDto.toActual;
+    const query = this.db
+      .getRepository(Build)
+      .createQueryBuilder('build')
+      .innerJoin('build.depot', 'depot')
+      .innerJoin('depot.app', 'app')
+      .leftJoinAndSelect('build.archives', 'archive')
+      .where('app.id = :id', { id })
+      .andWhere('app.isDeleted = false')
+      .andWhere('depot.platform = :platform')
+      .andWhere('depot.arch = :arch')
+      .andWhere('depot.locale = :locale')
+      .andWhere('build.version = :version', { version })
+      .setParameters(depotObj);
+    if (extraQuery) {
+      extraQuery(query);
+    }
+    const build = await query.getOne();
+    if (!build) {
+      throw new BlankReturnMessageDto(404, 'Build not found').toException();
+    }
+    return build;
+  }
+
+  async getChecksum(id: string, depotDto: DepotDto, version: string) {
+    const build = await this.getBuild(id, depotDto, version);
+    return {
+      files: Object.entries(build.checksum)
+        // .filter(([name, hash]) => file.length && hash !== null)
+        .map(([name, hash]) => ({ name, hash })),
+    };
+  }
+
+  async getFullPackageMetalink(id: string, depotDto: DepotDto, version: string) {
+    const build = await this.getBuild(id, depotDto, version, (qb) =>
+      qb.andWhere('archive.role = :fullRole', { fullRole: ArchiveType.Full })
+    );
+    return {
+      cdnUrl: this.cdnUrl,
+      archives: build.archives,
+    };
+  }
+}

console-api/views/checksums.mustache

+{{#files}}
+{{&hash}}  {{&name}}
+{{/files}}

console-api/views/metalinks.mustache

+<?xml version="1.0" encoding="UTF-8"?>
+<metalink xmlns="urn:ietf:params:xml:ns:metalink">
+  {{#archives}}
+    <file name="{{path}}.tar.gz">
+      <size>{{size}}</size>
+      <url priority="1">{{&cdnUrl}}/{{path}}.tar.gz</url>
+    </file>
+  {{/archives}}
+</metalink>