stream thing

console-api/package-lock.json

@@ -16,6 +16,7 @@
         "@nestjs/swagger": "^5.0.9",
         "@nestjs/typeorm": "^8.0.2",
         "axios": "^0.21.1",
+        "busboy": "^0.2.14",
         "class-transformer": "^0.4.0",
         "class-validator": "^0.13.1",
         "lodash": "^4.17.21",
@@ -33,6 +34,7 @@
         "@nestjs/cli": "^8.0.0",
         "@nestjs/schematics": "^8.0.0",
         "@nestjs/testing": "^8.0.0",
+        "@types/busboy": "^0.2.4",
         "@types/express": "^4.17.13",
         "@types/jest": "^26.0.24",
         "@types/lodash": "^4.14.172",
@@ -3067,6 +3069,15 @@
         "@types/node": "*"
       }
     },
+    "node_modules/@types/busboy": {
+      "version": "0.2.4",
+      "resolved": "https://registry.npmjs.org/@types/busboy/-/busboy-0.2.4.tgz",
+      "integrity": "sha512-f+ZCVjlcN8JW/zf3iR0GqO4gjOUlltMTtZjn+YR1mlK+MVu6esTiIecO0/GQlmYQPQLdBnc7+5vG3Xb+SkvFLw==",
+      "dev": true,
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
     "node_modules/@types/connect": {
       "version": "3.4.35",
       "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz",
@@ -13884,6 +13895,15 @@
         "@types/node": "*"
       }
     },
+    "@types/busboy": {
+      "version": "0.2.4",
+      "resolved": "https://registry.npmjs.org/@types/busboy/-/busboy-0.2.4.tgz",
+      "integrity": "sha512-f+ZCVjlcN8JW/zf3iR0GqO4gjOUlltMTtZjn+YR1mlK+MVu6esTiIecO0/GQlmYQPQLdBnc7+5vG3Xb+SkvFLw==",
+      "dev": true,
+      "requires": {
+        "@types/node": "*"
+      }
+    },
     "@types/connect": {
       "version": "3.4.35",
       "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz",

console-api/package.json

@@ -29,6 +29,7 @@
     "@nestjs/swagger": "^5.0.9",
     "@nestjs/typeorm": "^8.0.2",
     "axios": "^0.21.1",
+    "busboy": "^0.2.14",
     "class-transformer": "^0.4.0",
     "class-validator": "^0.13.1",
     "lodash": "^4.17.21",
@@ -49,6 +50,7 @@
     "@nestjs/cli": "^8.0.0",
     "@nestjs/schematics": "^8.0.0",
     "@nestjs/testing": "^8.0.0",
+    "@types/busboy": "^0.2.4",
     "@types/express": "^4.17.13",
     "@types/jest": "^26.0.24",
     "@types/lodash": "^4.14.172",

console-api/src/app.controller.ts

-import {
-  BadRequestException,
-  Body,
-  Controller,
-  Get,
-  Param,
-  Post,
-  Query,
-  UploadedFile,
-  UseGuards,
-  UseInterceptors,
-  ValidationPipe,
-} from '@nestjs/common';
+import { Body, Controller, Get, Param, Post, Query, Req, UploadedFile, UseGuards, UseInterceptors, ValidationPipe } from '@nestjs/common';
 import { AppService } from './app.service';
 import { ApiBody, ApiConsumes, ApiCreatedResponse, ApiOkResponse, ApiOperation, ApiParam, ApiQuery } from '@nestjs/swagger';
 import { BlankReturnMessageDto, GetAppReturnMessageDto, ReturnMessageDto, StringReturnMessageDto } from './dto/ReturnMessage.dto';
@@ -19,10 +7,13 @@ import { AppsJson } from './utility/apps-json-type';
 import { MyCardAppMaintainerGuard } from './my-card-app-maintainer.guard';
 import { FileInterceptor } from '@nestjs/platform-express';
 import { FileUploadDto } from './dto/FileUpload.dto';
-import AppClass = AppsJson.AppClass;
 import { AssetsS3Service } from './assets-s3/assets-s3.service';
-import { MulterDirectEngine } from './packager/MulterStreamEngine';
-import * as fs from 'fs';
+import Busboy from 'busboy';
+import { Request } from 'express';
+import { Stream } from 'stream';
+import { PackageResult } from './dto/PackageResult.dto';
+import { platform } from 'os';
+import AppClass = AppsJson.AppClass;
 
 @Controller('api')
 export class AppController {
@@ -84,16 +75,15 @@ export class AppController {
     summary: '打包文件',
     description: '必须登录用户且必须是管理员或者拥有1个 app 才能上传',
   })
-  @UseInterceptors(FileInterceptor('file', { storage: new MulterDirectEngine() }))
   @ApiConsumes('multipart/form-data')
   @ApiParam({ name: 'id', description: 'APP 的 id' })
   @ApiParam({ name: 'platform', description: 'APP 的 版本号', enum: AppsJson.Platform })
   @ApiParam({ name: 'locale', description: 'APP 的 版本号', enum: AppsJson.Locale })
   @ApiParam({ name: 'version', description: 'APP 的 版本号' })
-  /*@ApiBody({
+  @ApiBody({
     description: 'app 的 tar.gz 文件',
     type: FileUploadDto,
-  })*/
+  })
   @ApiCreatedResponse({ type: BlankReturnMessageDto })
   async makeBuild(
     @FetchMyCardUser() user: MyCardUser,
@@ -101,12 +91,44 @@ export class AppController {
     @Param('id') id: string,
     @Param('platform') platform: AppsJson.Platform,
     @Param('locale') locale: AppsJson.Locale,
-    @Param('version') version: string
+    @Param('version') version: string,
+    @Req() req: Request
   ) {
-    /*console.log(file.stream);
-    if (!file) {
-      throw new BlankReturnMessageDto(400, 'no file').toException();
-    }*/
-    return this.appService.makeBuild(user, fs.createReadStream('/tmp/test1.tar.gz'), id, platform, locale, version);
+    let busboy: busboy.Busboy;
+    try {
+      busboy = new Busboy({ headers: req.headers });
+    } catch (e) {
+      throw new BlankReturnMessageDto(400, `Creation failed: ${e.toString()}`).toException();
+    }
+    const packagePromise = new Promise<ReturnMessageDto<PackageResult>>((resolve, reject) => {
+      let gotFile = false;
+      busboy.on('file', async (fieldname, fileStream, filename, encoding, mimetype) => {
+        if (fieldname !== 'file') {
+          reject(new BlankReturnMessageDto(400, 'invalid field').toException());
+          return;
+        }
+        gotFile = true;
+        // console.log(`got file ${fieldname}`);
+        const stream = new Stream.Readable().wrap(fileStream);
+        try {
+          resolve(await this.appService.makeBuild(user, stream, id, platform, locale, version));
+        } catch (e) {
+          stream.destroy();
+          reject(e);
+        }
+      });
+      busboy.on('error', () => {
+        reject(new BlankReturnMessageDto(500, 'upload error').toException());
+      });
+      busboy.on('finish', () => {
+        //console.log(`finished receiving file`);
+        if (!gotFile) {
+          reject(new BlankReturnMessageDto(400, 'no file').toException());
+        }
+        //resolve();
+      });
+    });
+    req.pipe(busboy);
+    return packagePromise;
   }
 }

console-api/src/app.module.ts

@@ -6,7 +6,6 @@ import { AdminController } from './admin/admin.controller';
 import { ConfigModule, ConfigService } from '@nestjs/config';
 import { App } from './entities/App.entity';
 import { AppHistory } from './entities/AppHistory.entity';
-import { S3Service } from './s3/s3.service';
 import { PackagerService } from './packager/packager.service';
 import { AssetsS3Service } from './assets-s3/assets-s3.service';
 import { PackageS3Service } from './package-s3/package-s3.service';

console-api/src/app.service.ts

 import { Connection, IsNull, Not } from 'typeorm';
 import { InjectConnection } from '@nestjs/typeorm';
-import { ConsoleLogger, Injectable, Param, UploadedFile } from '@nestjs/common';
+import { ConsoleLogger, Injectable } from '@nestjs/common';
 import { AppsJson } from './utility/apps-json-type';
 import { App } from './entities/App.entity';
 import { BlankReturnMessageDto, ReturnMessageDto } from './dto/ReturnMessage.dto';
-import { FetchMyCardUser, MyCardUser } from './utility/mycard-auth';
+import { MyCardUser } from './utility/mycard-auth';
 import { PackagerService } from './packager/packager.service';
 import internal from 'stream';
 

console-api/src/entities/AppHistory.entity.ts

-import {
-  Column,
-  Entity,
-  Index,
-  ManyToOne,
-  PrimaryGeneratedColumn,
-} from 'typeorm';
+import { Column, Entity, Index, ManyToOne, PrimaryGeneratedColumn } from 'typeorm';
 import { AppBase } from './AppBase.entity';
 import { App } from './App.entity';
 

console-api/src/packager/packager.service.ts

@@ -10,7 +10,6 @@ import internal from 'stream';
 import { PackageResult } from '../dto/PackageResult.dto';
 import { PackageS3Service } from '../package-s3/package-s3.service';
 import readdirp from 'readdirp';
-import { v4 as uuidv4 } from 'uuid';
 
 import { ConsoleLogger, Injectable } from '@nestjs/common';
 import { createHash } from 'crypto';
@@ -40,7 +39,8 @@ export class PackagerService extends ConsoleLogger {
       extractRoot = path.join(root, pathPrefix);
       await fs.promises.mkdir(extractRoot, { recursive: true });
     }
-    await this.spawnAsync('tar', ['-zxvf', '-'], { cwd: extractRoot, stdio: [stream, 'ignore', 'ignore'] });
+    this.log(`Extracting package to ${extractRoot}.`);
+    await this.spawnAsync('tar', ['-zxf', '-'], { cwd: extractRoot }, stream);
 
     this.log(`Package extracted to ${extractRoot}.`);
 
@@ -99,7 +99,7 @@ export class PackagerService extends ConsoleLogger {
       packages[gotPackages[i]] = packagesSequence[i];
     }
 
-    this.log({ checksum, packages });
+    // this.log({ checksum, packages });
     await fs.promises.rm(root, { recursive: true });
     await fs.promises.rm(tarballRoot, { recursive: true });
     return new PackageResult(checksum, packages, gotPackages[0]);
@@ -132,9 +132,25 @@ export class PackagerService extends ConsoleLogger {
     return archive;
   }
 
-  private spawnAsync(command: string, args: string[], options: child_process.SpawnOptions) {
+  private spawnAsync(
+    command: string,
+    args: string[],
+    options: child_process.SpawnOptions,
+    stdinStream?: internal.Readable,
+    stdoutStream?: internal.Writable,
+    stderrStream?: internal.Writable
+  ) {
     return new Promise<void>((resolve, reject) => {
       const child = child_process.spawn(command, args, options);
+      if (stdinStream) {
+        stdinStream.pipe(child.stdin);
+      }
+      if (stdoutStream) {
+        child.stdout.pipe(stdoutStream);
+      }
+      if (stderrStream) {
+        child.stderr.pipe(stderrStream);
+      }
       child.on('exit', (code) => {
         if (code == 0) {
           resolve();

console-api/src/s3/s3.service.ts

-import { ConsoleLogger, Injectable } from '@nestjs/common';
+import { ConsoleLogger } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
 import { ListObjectsCommand, PutObjectCommand, PutObjectCommandInput, S3Client } from '@aws-sdk/client-s3';
 import { createHash } from 'crypto';

console-api/src/utility/mycard-auth.ts

@@ -25,22 +25,18 @@ export interface FetchMyCardUserOptions {
 }
 
 const _options = {
-  mycardAccountsUrl:
-    process.env.MYCARD_ACCOUNTS_URL || `https://sapi.moecube.com:444/accounts`,
+  mycardAccountsUrl: process.env.MYCARD_ACCOUNTS_URL || `https://sapi.moecube.com:444/accounts`,
   field: 'sso',
 };
 
 export async function fetchUserWithToken(token: string) {
   let authResult: AxiosResponse<MyCardUser>;
   try {
-    authResult = await axios.get<MyCardUser>(
-      `${_options.mycardAccountsUrl}/authUser`,
-      {
-        responseType: 'json',
-        validateStatus: (s) => true,
-        headers: { Authorization: `Bearer ${token}` },
-      },
-    );
+    authResult = await axios.get<MyCardUser>(`${_options.mycardAccountsUrl}/authUser`, {
+      responseType: 'json',
+      validateStatus: (s) => true,
+      headers: { Authorization: `Bearer ${token}` },
+    });
   } catch (e) {
     return null;
   }
@@ -76,8 +72,6 @@ export function getUserFromContext(context: ExecutionContext) {
   return fetchUserWithToken(token);
 }
 
-export const FetchMyCardUser = createParamDecorator(
-  async (_, context: ExecutionContext) => {
-    return getUserFromContext(context);
-  },
-);
+export const FetchMyCardUser = createParamDecorator(async (_, context: ExecutionContext) => {
+  return getUserFromContext(context);
+});