so far

console-api/package.json

@@ -33,12 +33,15 @@
     "class-validator": "^0.13.1",
     "lodash": "^4.17.21",
     "moment": "^2.29.1",
+    "mustache": "^4.2.0",
+    "p-queue": "6.6.2",
     "pg": "^8.7.1",
     "pg-native": "^3.0.0",
     "reflect-metadata": "^0.1.13",
     "rimraf": "^3.0.2",
     "rxjs": "^7.2.0",
     "swagger-ui-express": "^4.1.6",
+    "tar": "^6.1.8",
     "typeorm": "^0.2.37"
   },
   "devDependencies": {
@@ -49,8 +52,10 @@
     "@types/jest": "^26.0.24",
     "@types/lodash": "^4.14.172",
     "@types/multer": "^1.4.7",
+    "@types/mustache": "^4.1.2",
     "@types/node": "^16.0.0",
     "@types/supertest": "^2.0.11",
+    "@types/tar": "^4.0.5",
     "@typescript-eslint/eslint-plugin": "^4.28.2",
     "@typescript-eslint/parser": "^4.28.2",
     "eslint": "^7.30.0",

console-api/src/admin/admin.controller.ts

@@ -25,6 +25,7 @@ import { AppsJson } from '../utility/apps-json-type';
 import { AppService } from '../app.service';
 import { BlankReturnMessageDto } from '../dto/ReturnMessage.dto';
 import { AssignAppDto } from '../dto/AssignApp.dto';
+import { AppPrefixDto } from '../dto/AppPrefix.dto';
 
 @Controller('api/admin')
 @ApiTags('admin')
@@ -56,14 +57,14 @@ export class AdminController {
   }
 
   @Delete('app/:id')
-  @ApiOperation({ summary: '创建 app' })
+  @ApiOperation({ summary: '删除 app' })
   @ApiOkResponse({ type: BlankReturnMessageDto })
   async deleteApp(@Param('id') id: string) {
     return this.appService.deleteApp(id);
   }
 
   @Post('app/:id/assign')
-  @ApiOperation({ summary: '创建 app' })
+  @ApiOperation({ summary: '设置 app 管理者' })
   @ApiOkResponse({ type: BlankReturnMessageDto })
   async assignApp(
     @Param('id') id: string,
@@ -71,4 +72,14 @@ export class AdminController {
   ) {
     return this.appService.assignApp(id, assignAppData.author);
   }
+
+  @Post('app/:id/prefix')
+  @ApiOperation({ summary: '设置 app 打包前缀' })
+  @ApiOkResponse({ type: BlankReturnMessageDto })
+  async setAppPrefix(
+    @Param('id') id: string,
+    @Body(new ValidationPipe({ transform: true })) appPrefix: AppPrefixDto,
+  ) {
+    return this.appService.setAppPrefix(id, appPrefix._prefix);
+  }
 }

console-api/src/app.module.ts

@@ -7,6 +7,7 @@ import { ConfigModule, ConfigService } from '@nestjs/config';
 import { App } from './entities/App.entity';
 import { AppHistory } from './entities/AppHistory.entity';
 import { S3Service } from './s3/s3.service';
+import { PackagerService } from './packager/packager.service';
 
 const configModule = ConfigModule.forRoot();
 
@@ -32,6 +33,6 @@ const configModule = ConfigModule.forRoot();
     }),
   ],
   controllers: [AppController, AdminController],
-  providers: [AppService, S3Service],
+  providers: [AppService, S3Service, PackagerService],
 })
 export class AppModule {}

console-api/src/app.service.ts

@@ -125,6 +125,14 @@ export class AppService extends ConsoleLogger {
     );
   }
 
+  async setAppPrefix(id: string, prefix: string) {
+    return this.updateResult(
+      () =>
+        this.db.getRepository(App).update({ id }, { packagePrefix: prefix }),
+      201,
+    );
+  }
+
   async updateApp(user: MyCardUser, id: string, appData: AppsJson.App) {
     if (!user) {
       throw new BlankReturnMessageDto(401, 'Needs login').toException();

console-api/src/dto/AppPrefix.dto.ts

+import { ApiProperty } from '@nestjs/swagger';
+
+export class AppPrefixDto {
+  @ApiProperty({ description: '打包起始路径' })
+  prefix: string;
+
+  get _prefix() {
+    return this.prefix && this.prefix.length ? this.prefix : null;
+  }
+}

console-api/src/entities/App.entity.ts

@@ -4,6 +4,7 @@ import { MyCardUser } from '../utility/mycard-auth';
 import { AppBase } from './AppBase.entity';
 import { AppHistory } from './AppHistory.entity';
 import moment from 'moment';
+import { join } from 'path';
 
 @Entity()
 export class App extends AppBase {
@@ -17,6 +18,17 @@ export class App extends AppBase {
   @Column({ nullable: true, select: false })
   isDeleted: boolean;
 
+  @Column('varchar', { length: 128, nullable: true })
+  packagePrefix: string;
+
+  get packageFullPath() {
+    if (this.packagePrefix && this.packagePrefix.length) {
+      return join(this.id, this.packagePrefix);
+    } else {
+      return this.id;
+    }
+  }
+
   isUserCanEditApp(u: MyCardUser) {
     return u.admin || (this.author && this.author.includes(u.id));
   }

console-api/src/my-card-app-maintainer.guard.spec.ts

+import { MyCardAppMaintainerGuard } from './my-card-app-maintainer.guard';
+
+describe('MyCardAppMaintainerGuard', () => {
+  it('should be defined', () => {
+    expect(new MyCardAppMaintainerGuard()).toBeDefined();
+  });
+});

console-api/src/my-card-app-maintainer.guard.ts

+import { CanActivate, ExecutionContext, Injectable } from '@nestjs/common';
+import { AppService } from './app.service';
+import { getUserFromContext } from './utility/mycard-auth';
+import { BlankReturnMessageDto } from './dto/ReturnMessage.dto';
+
+@Injectable()
+export class MyCardAppMaintainerGuard implements CanActivate {
+  constructor(private readonly appService: AppService) {}
+  async canActivate(context: ExecutionContext): Promise<boolean> {
+    const user = await getUserFromContext(context);
+    if (!user) {
+      throw new BlankReturnMessageDto(401, 'Invalid user').toException();
+    }
+    if (!(await this.appService.isUserCanMaintainApp(user))) {
+      throw new BlankReturnMessageDto(403, 'Permission denied').toException();
+    }
+    return true;
+  }
+}

console-api/src/packager/packager.service.spec.ts

+import { Test, TestingModule } from '@nestjs/testing';
+import { PackagerService } from './packager.service';
+
+describe('PackagerService', () => {
+  let service: PackagerService;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      providers: [PackagerService],
+    }).compile();
+
+    service = module.get<PackagerService>(PackagerService);
+  });
+
+  it('should be defined', () => {
+    expect(service).toBeDefined();
+  });
+});

console-api/src/packager/packager.service.ts

+import { ConsoleLogger, Injectable } from '@nestjs/common';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as crypto from 'crypto';
+import * as child_process from 'child_process';
+import * as Mustache from 'mustache';
+import tar from 'tar';
+import { App } from '../entities/App.entity';
+import PQueue from 'p-queue';
+
+// eslint-disable-next-line @typescript-eslint/no-empty-function
+function nothing() {}
+
+@Injectable()
+export class PackagerService extends ConsoleLogger {
+  workingPath: string;
+  releasePath: string;
+  downloadBaseUrl: string;
+  queueIdMap = new Map<string, PQueue>();
+
+  getQueue(id: string) {
+    if (!this.queueIdMap.has(id)) {
+      this.queueIdMap.set(id, new PQueue({ concurrency: 1 }));
+    }
+    return this.queueIdMap.get(id);
+  }
+
+  constructor() {
+    super('packager');
+  }
+
+  private async readdirRecursive(_path: string): Promise<string[]> {
+    const files = await fs.promises.readdir(_path, { encoding: 'utf-8' });
+    let result = files;
+    for (const file of files) {
+      const child = path.join(_path, file);
+      const stat = await fs.promises.stat(child);
+      if (stat.isDirectory()) {
+        result = result.concat(
+          (await this.readdirRecursive(child)).map((_file) =>
+            path.join(file, _file),
+          ),
+        );
+      }
+    }
+    return result;
+  }
+
+  private caculateSHA256(file: string): Promise<string> {
+    return new Promise((resolve, reject) => {
+      const input = fs.createReadStream(file);
+      const hash = crypto.createHash('sha256');
+      hash.on('error', (error: Error) => {
+        reject(error);
+      });
+      input.on('error', (error: Error) => {
+        reject(error);
+      });
+      hash.on('readable', () => {
+        const data = hash.read();
+        if (data) {
+          resolve((<Buffer>data).toString('hex'));
+        }
+      });
+      input.pipe(hash);
+    });
+  }
+
+  private spawnAsync(
+    command: string,
+    args: string[],
+    options: child_process.SpawnOptions,
+  ) {
+    return new Promise<void>((resolve, reject) => {
+      const child = child_process.spawn(command, args, options);
+      child.on('exit', (code) => {
+        if (code == 0) {
+          resolve();
+        } else {
+          reject(code);
+        }
+      });
+      child.on('error', (error) => {
+        reject(error);
+      });
+    });
+  }
+
+  private archive(
+    archive: string,
+    files: string[],
+    directory: string,
+  ): Promise<void> {
+    return tar.c({ gzip: true, C: directory, file: archive }, files);
+  }
+
+  private async createDirectoryIfNotExists(directory: string) {
+    try {
+      await fs.promises.access(directory);
+    } catch (e) {
+      await fs.promises.mkdir(directory, { recursive: true });
+    }
+  }
+
+  private async unarchiveStream(stream: fs.ReadStream, directory: string) {
+    await this.createDirectoryIfNotExists(directory);
+    const unarchiveProcessStream = stream.pipe(tar.x({ C: directory }));
+    return new Promise<void>((resolve, reject) => {
+      unarchiveProcessStream.on('finish', resolve);
+      unarchiveProcessStream.on('error', reject);
+    });
+  }
+
+  private async packageProcess(app: App, stream: fs.ReadStream) {
+    const unarchiveDestination = path.join(
+      this.workingPath,
+      app.packageFullPath,
+    );
+    const packageWorkingRoot = path.join(this.workingPath, app.id);
+  }
+
+  async package(app: App, stream: fs.ReadStream) {
+    const queue = this.getQueue();
+  }
+}

console-api/src/s3/s3.service.spec.ts

+import { Test, TestingModule } from '@nestjs/testing';
+import { S3Service } from './s3.service';
+
+describe('S3Service', () => {
+  let service: S3Service;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      providers: [S3Service],
+    }).compile();
+
+    service = module.get<S3Service>(S3Service);
+  });
+
+  it('should be defined', () => {
+    expect(service).toBeDefined();
+  });
+});

console-api/src/s3/s3.service.ts

+import { ConsoleLogger, Injectable } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
+import {
+  ListObjectsCommand,
+  PutObjectCommand,
+  S3Client,
+} from '@aws-sdk/client-s3';
+import { createHash } from 'crypto';
+
+@Injectable()
+export class S3Service extends ConsoleLogger {
+  bucket: string;
+  prefix: string;
+  cdnUrl: string;
+  s3: S3Client;
+  constructor(config: ConfigService) {
+    super('s3');
+    this.bucket = config.get('S3_BUCKET');
+    this.prefix = config.get('S3_PREFIX');
+    this.cdnUrl =
+      config.get('S3_CDN_URL') ||
+      `${config.get('S3_ENDPOINT')}/${this.bucket}/${this.prefix}`;
+    this.s3 = new S3Client({
+      credentials: {
+        accessKeyId: config.get('S3_KEY'),
+        secretAccessKey: config.get('S3_SECRET'),
+      },
+      region: config.get('S3_REGION') || 'us-west-1',
+      endpoint: config.get('S3_ENDPOINT'),
+      forcePathStyle: true,
+    });
+  }
+
+  private async listObjects(path: string) {
+    const command = new ListObjectsCommand({
+      Bucket: this.bucket,
+      Prefix: path,
+    });
+    return this.s3.send(command);
+  }
+
+  async uploadAssets(file: Express.Multer.File) {
+    const fileSuffix = file.originalname.split('.').pop();
+    const hash = createHash('sha512').update(file.buffer).digest('hex');
+    const filename = `${hash}.${fileSuffix}`;
+    const path = `${this.prefix}/${filename}`;
+    const downloadUrl = `${this.cdnUrl}/${filename}`;
+    const checkExisting = await this.listObjects(path);
+    try {
+      if (
+        checkExisting.Contents &&
+        checkExisting.Contents.some((obj) => obj.Key === path)
+      ) {
+        // already uploaded
+      } else {
+        await this.uploadFile(path, file.buffer, file.mimetype);
+      }
+      return downloadUrl;
+    } catch (e) {
+      this.error(`Failed to assign upload of file ${path}: ${e.toString()}`);
+      return null;
+    }
+  }
+
+  async uploadFile(path: string, buffer: Buffer, mime?: string) {
+    return this.s3.send(
+      new PutObjectCommand({
+        Bucket: this.bucket,
+        Key: path,
+        Body: buffer,
+        ContentType: mime,
+      }),
+    );
+  }
+}