reuse package

console-api/src/dto/PackageResult.dto.ts

 export class PackageResult {
-  constructor(public checksum: Record<string, string>, public packages: Record<string, string[]>) {}
+  constructor(public checksum: Record<string, string>, public packages: Record<string, string[]>, public fullPackage: string) {}
 }

console-api/src/packager/packager.service.ts

@@ -13,6 +13,12 @@ import readdirp from 'readdirp';
 import { v4 as uuidv4 } from 'uuid';
 
 import { ConsoleLogger, Injectable } from '@nestjs/common';
+import { createHash } from 'crypto';
+
+export interface FileWithHash {
+  file: readdirp.EntryInfo;
+  hash: string;
+}
 
 @Injectable()
 export class PackagerService extends ConsoleLogger {
@@ -38,7 +44,7 @@ export class PackagerService extends ConsoleLogger {
 
     this.log(`Package extracted to ${extractRoot}.`);
 
-    const packages: Record<string, string[]> = {};
+    const packagesSequence: string[][] = [];
     const entries = await readdirp.promise(root, { alwaysStat: true, type: 'files_directories' });
     const [directories, files] = _.partition(entries, (item) => item.stats.isDirectory());
 
@@ -48,41 +54,39 @@ export class PackagerService extends ConsoleLogger {
       directories.map((d) => d.path),
       files.map((f) => f.path)
     );
-    const promises: Promise<[string, string]>[] = [];
+    const promises: Promise<string>[] = [];
+
+    const filesWithHash: FileWithHash[] = files.map((f) => ({ file: f, hash: checksum[f.path] }));
 
     // 整包
-    const archive = `${uuidv4()}.tar.gz`;
-    packages[archive] = files.map((f) => f.path);
-    promises.push(this.archive(archive, root, tarballRoot, await fs.promises.readdir(root)));
+    packagesSequence.push(files.map((f) => f.path));
+    promises.push(this.archive(root, tarballRoot, filesWithHash, await fs.promises.readdir(root)));
 
     // 散包
-    const buckets: Record<string, [string[], number]> = {};
-    for (const file of files) {
-      if (file.stats.size < this.bucket_enter) {
-        const extname = path.extname(file.basename);
+    const buckets: Record<string, [FileWithHash[], number]> = {};
+    for (const file of filesWithHash) {
+      if (file.file.stats.size < this.bucket_enter) {
+        const extname = path.extname(file.file.basename);
         buckets[extname] ??= [[], 0];
         const bucket = buckets[extname];
-        if (bucket[1] + file.stats.size >= this.bucket_max) {
-          const archive = `${uuidv4()}.tar.gz`;
-          packages[archive] = bucket[0];
-          promises.push(this.archive(archive, root, tarballRoot, bucket[0], checksum));
+        if (bucket[1] + file.file.stats.size >= this.bucket_max) {
+          packagesSequence.push(bucket[0].map((f) => f.file.path));
+          promises.push(this.archive(root, tarballRoot, bucket[0]));
           bucket[0] = [];
           bucket[1] = 0;
         } else {
-          bucket[0].push(file.path);
-          bucket[1] += file.stats.size;
+          bucket[0].push(file);
+          bucket[1] += file.file.stats.size;
         }
       } else {
-        const archive = `${uuidv4()}.tar.gz`;
-        packages[archive] = [file.path];
-        promises.push(this.archive(archive, root, tarballRoot, [file.path], checksum));
+        packagesSequence.push([file.file.path]);
+        promises.push(this.archive(root, tarballRoot, [file]));
       }
     }
     for (const bucket of Object.values(buckets)) {
       if (bucket[0].length) {
-        const archive = `${uuidv4()}.tar.gz`;
-        packages[archive] = bucket[0];
-        promises.push(this.archive(archive, root, tarballRoot, bucket[0], checksum));
+        packagesSequence.push(bucket[0].map((f) => f.file.path));
+        promises.push(this.archive(root, tarballRoot, bucket[0]));
       }
     }
 
@@ -90,16 +94,15 @@ export class PackagerService extends ConsoleLogger {
 
     const gotPackages = await Promise.all(promises); // 这个 await 过后，checksum 和 打包上传都已经跑完了
 
-    for (const differentPackages of gotPackages.filter((p) => p[0] !== p[1])) {
-      const [originalPackage, gotPackage] = differentPackages;
-      packages[gotPackage] = packages[originalPackage];
-      delete packages[originalPackage];
+    const packages: Record<string, string[]> = {};
+    for (let i = 0; i < packagesSequence.length; ++i) {
+      packages[gotPackages[i]] = packagesSequence[i];
     }
 
     this.log({ checksum, packages });
     await fs.promises.rm(root, { recursive: true });
     await fs.promises.rm(tarballRoot, { recursive: true });
-    return new PackageResult(checksum, packages);
+    return new PackageResult(checksum, packages, gotPackages[0]);
   }
 
   async checksum(root: string, directories: string[], files: string[]): Promise<Record<string, string>> {
@@ -111,20 +114,22 @@ export class PackagerService extends ConsoleLogger {
     ]);
   }
 
-  async archive(
-    archive: string = `${uuidv4()}.tar.gz`,
-    root: string,
-    tarballRoot: string,
-    files: string[],
-    checksum: Record<string, string> = {}
-  ): Promise<[string, string]> {
+  async archive(root: string, tarballRoot: string, files: FileWithHash[], altFiles?: string[]): Promise<string> {
+    const archive =
+      createHash('sha512')
+        .update(files.map((f) => `${f.file.path}${f.hash}`).join(''))
+        .digest('hex') + '.tar.gz';
+    if (await this.s3.fileExists(archive)) {
+      return archive;
+    }
     const archivePath = path.join(tarballRoot, archive);
-    await this.spawnAsync('tar', ['-zcvf', archivePath].concat(files), {
+    this.log(`Packaging archive ${archivePath} with ${files.length} files.`);
+    await this.spawnAsync('tar', ['-zcvf', archivePath].concat(altFiles || files.map((f) => f.file.path)), {
       cwd: root,
     });
     const fileSize = (await fs.promises.stat(archivePath)).size;
     await this.s3.uploadFile(archive, fs.createReadStream(archivePath), { ContentType: 'application/tar+gzip', ContentLength: fileSize });
-    return [archive, archive];
+    return archive;
   }
 
   private spawnAsync(command: string, args: string[], options: child_process.SpawnOptions) {

console-api/src/s3/s3.service.ts

@@ -3,6 +3,7 @@ import { ConfigService } from '@nestjs/config';
 import { ListObjectsCommand, PutObjectCommand, PutObjectCommandInput, S3Client } from '@aws-sdk/client-s3';
 import { createHash } from 'crypto';
 import internal from 'stream';
+import Path from 'path';
 
 export class S3Service extends ConsoleLogger {
   private readonly bucket: string;
@@ -33,28 +34,32 @@ export class S3Service extends ConsoleLogger {
   async listObjects(path: string) {
     const command = new ListObjectsCommand({
       Bucket: this.bucket,
-      Prefix: path,
+      Prefix: this.getPathWithPrefix(path),
     });
     return this.s3.send(command);
   }
 
+  async fileExists(path: string) {
+    const objects = await this.listObjects(path);
+    // this.log(objects);
+    return objects.Contents && objects.Contents.some((obj) => obj.Key === this.getPathWithPrefix(path));
+  }
+
   private getPathWithPrefix(filename: string) {
     return this.prefix ? `${this.prefix}/${filename}` : filename;
   }
 
   async uploadAssets(file: Express.Multer.File) {
-    const fileSuffix = file.originalname.split('.').pop();
+    const fileSuffix = Path.extname(file.originalname);
     const hash = createHash('sha512').update(file.buffer).digest('hex');
-    const filename = `${hash}.${fileSuffix}`;
-    const path = this.getPathWithPrefix(filename);
-    const downloadUrl = `${this.cdnUrl}/${filename}`;
-    const checkExisting = await this.listObjects(path);
+    const path = fileSuffix ? `${hash}.${fileSuffix}` : hash;
+    const downloadUrl = `${this.cdnUrl}/${path}`;
     try {
-      if (checkExisting.Contents && checkExisting.Contents.some((obj) => obj.Key === path)) {
+      if (await this.fileExists(path)) {
         // already uploaded
         return downloadUrl;
       } else {
-        return this.uploadFile(filename, file.buffer, { ContentType: file.mimetype });
+        return this.uploadFile(path, file.buffer, { ContentType: file.mimetype });
       }
     } catch (e) {
       this.error(`Failed to upload assets ${path}: ${e.toString()}`);