Commit 5e90acd3 authored by nanahira's avatar nanahira

stream thing

parent 119e0e6b
......@@ -16,6 +16,7 @@
"@nestjs/swagger": "^5.0.9",
"@nestjs/typeorm": "^8.0.2",
"axios": "^0.21.1",
"busboy": "^0.2.14",
"class-transformer": "^0.4.0",
"class-validator": "^0.13.1",
"lodash": "^4.17.21",
......@@ -33,6 +34,7 @@
"@nestjs/cli": "^8.0.0",
"@nestjs/schematics": "^8.0.0",
"@nestjs/testing": "^8.0.0",
"@types/busboy": "^0.2.4",
"@types/express": "^4.17.13",
"@types/jest": "^26.0.24",
"@types/lodash": "^4.14.172",
......@@ -3067,6 +3069,15 @@
"@types/node": "*"
}
},
"node_modules/@types/busboy": {
"version": "0.2.4",
"resolved": "https://registry.npmjs.org/@types/busboy/-/busboy-0.2.4.tgz",
"integrity": "sha512-f+ZCVjlcN8JW/zf3iR0GqO4gjOUlltMTtZjn+YR1mlK+MVu6esTiIecO0/GQlmYQPQLdBnc7+5vG3Xb+SkvFLw==",
"dev": true,
"dependencies": {
"@types/node": "*"
}
},
"node_modules/@types/connect": {
"version": "3.4.35",
"resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz",
......@@ -13884,6 +13895,15 @@
"@types/node": "*"
}
},
"@types/busboy": {
"version": "0.2.4",
"resolved": "https://registry.npmjs.org/@types/busboy/-/busboy-0.2.4.tgz",
"integrity": "sha512-f+ZCVjlcN8JW/zf3iR0GqO4gjOUlltMTtZjn+YR1mlK+MVu6esTiIecO0/GQlmYQPQLdBnc7+5vG3Xb+SkvFLw==",
"dev": true,
"requires": {
"@types/node": "*"
}
},
"@types/connect": {
"version": "3.4.35",
"resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz",
......
......@@ -29,6 +29,7 @@
"@nestjs/swagger": "^5.0.9",
"@nestjs/typeorm": "^8.0.2",
"axios": "^0.21.1",
"busboy": "^0.2.14",
"class-transformer": "^0.4.0",
"class-validator": "^0.13.1",
"lodash": "^4.17.21",
......@@ -49,6 +50,7 @@
"@nestjs/cli": "^8.0.0",
"@nestjs/schematics": "^8.0.0",
"@nestjs/testing": "^8.0.0",
"@types/busboy": "^0.2.4",
"@types/express": "^4.17.13",
"@types/jest": "^26.0.24",
"@types/lodash": "^4.14.172",
......
import {
BadRequestException,
Body,
Controller,
Get,
Param,
Post,
Query,
UploadedFile,
UseGuards,
UseInterceptors,
ValidationPipe,
} from '@nestjs/common';
import { Body, Controller, Get, Param, Post, Query, Req, UploadedFile, UseGuards, UseInterceptors, ValidationPipe } from '@nestjs/common';
import { AppService } from './app.service';
import { ApiBody, ApiConsumes, ApiCreatedResponse, ApiOkResponse, ApiOperation, ApiParam, ApiQuery } from '@nestjs/swagger';
import { BlankReturnMessageDto, GetAppReturnMessageDto, ReturnMessageDto, StringReturnMessageDto } from './dto/ReturnMessage.dto';
......@@ -19,10 +7,13 @@ import { AppsJson } from './utility/apps-json-type';
import { MyCardAppMaintainerGuard } from './my-card-app-maintainer.guard';
import { FileInterceptor } from '@nestjs/platform-express';
import { FileUploadDto } from './dto/FileUpload.dto';
import AppClass = AppsJson.AppClass;
import { AssetsS3Service } from './assets-s3/assets-s3.service';
import { MulterDirectEngine } from './packager/MulterStreamEngine';
import * as fs from 'fs';
import Busboy from 'busboy';
import { Request } from 'express';
import { Stream } from 'stream';
import { PackageResult } from './dto/PackageResult.dto';
import { platform } from 'os';
import AppClass = AppsJson.AppClass;
@Controller('api')
export class AppController {
......@@ -84,16 +75,15 @@ export class AppController {
summary: '打包文件',
description: '必须登录用户且必须是管理员或者拥有1个 app 才能上传',
})
@UseInterceptors(FileInterceptor('file', { storage: new MulterDirectEngine() }))
@ApiConsumes('multipart/form-data')
@ApiParam({ name: 'id', description: 'APP 的 id' })
@ApiParam({ name: 'platform', description: 'APP 的 版本号', enum: AppsJson.Platform })
@ApiParam({ name: 'locale', description: 'APP 的 版本号', enum: AppsJson.Locale })
@ApiParam({ name: 'version', description: 'APP 的 版本号' })
/*@ApiBody({
@ApiBody({
description: 'app 的 tar.gz 文件',
type: FileUploadDto,
})*/
})
@ApiCreatedResponse({ type: BlankReturnMessageDto })
async makeBuild(
@FetchMyCardUser() user: MyCardUser,
......@@ -101,12 +91,44 @@ export class AppController {
@Param('id') id: string,
@Param('platform') platform: AppsJson.Platform,
@Param('locale') locale: AppsJson.Locale,
@Param('version') version: string
@Param('version') version: string,
@Req() req: Request
) {
/*console.log(file.stream);
if (!file) {
throw new BlankReturnMessageDto(400, 'no file').toException();
}*/
return this.appService.makeBuild(user, fs.createReadStream('/tmp/test1.tar.gz'), id, platform, locale, version);
let busboy: busboy.Busboy;
try {
busboy = new Busboy({ headers: req.headers });
} catch (e) {
throw new BlankReturnMessageDto(400, `Creation failed: ${e.toString()}`).toException();
}
const packagePromise = new Promise<ReturnMessageDto<PackageResult>>((resolve, reject) => {
let gotFile = false;
busboy.on('file', async (fieldname, fileStream, filename, encoding, mimetype) => {
if (fieldname !== 'file') {
reject(new BlankReturnMessageDto(400, 'invalid field').toException());
return;
}
gotFile = true;
// console.log(`got file ${fieldname}`);
const stream = new Stream.Readable().wrap(fileStream);
try {
resolve(await this.appService.makeBuild(user, stream, id, platform, locale, version));
} catch (e) {
stream.destroy();
reject(e);
}
});
busboy.on('error', () => {
reject(new BlankReturnMessageDto(500, 'upload error').toException());
});
busboy.on('finish', () => {
//console.log(`finished receiving file`);
if (!gotFile) {
reject(new BlankReturnMessageDto(400, 'no file').toException());
}
//resolve();
});
});
req.pipe(busboy);
return packagePromise;
}
}
......@@ -6,7 +6,6 @@ import { AdminController } from './admin/admin.controller';
import { ConfigModule, ConfigService } from '@nestjs/config';
import { App } from './entities/App.entity';
import { AppHistory } from './entities/AppHistory.entity';
import { S3Service } from './s3/s3.service';
import { PackagerService } from './packager/packager.service';
import { AssetsS3Service } from './assets-s3/assets-s3.service';
import { PackageS3Service } from './package-s3/package-s3.service';
......
import { Connection, IsNull, Not } from 'typeorm';
import { InjectConnection } from '@nestjs/typeorm';
import { ConsoleLogger, Injectable, Param, UploadedFile } from '@nestjs/common';
import { ConsoleLogger, Injectable } from '@nestjs/common';
import { AppsJson } from './utility/apps-json-type';
import { App } from './entities/App.entity';
import { BlankReturnMessageDto, ReturnMessageDto } from './dto/ReturnMessage.dto';
import { FetchMyCardUser, MyCardUser } from './utility/mycard-auth';
import { MyCardUser } from './utility/mycard-auth';
import { PackagerService } from './packager/packager.service';
import internal from 'stream';
......
import {
Column,
Entity,
Index,
ManyToOne,
PrimaryGeneratedColumn,
} from 'typeorm';
import { Column, Entity, Index, ManyToOne, PrimaryGeneratedColumn } from 'typeorm';
import { AppBase } from './AppBase.entity';
import { App } from './App.entity';
......
......@@ -10,7 +10,6 @@ import internal from 'stream';
import { PackageResult } from '../dto/PackageResult.dto';
import { PackageS3Service } from '../package-s3/package-s3.service';
import readdirp from 'readdirp';
import { v4 as uuidv4 } from 'uuid';
import { ConsoleLogger, Injectable } from '@nestjs/common';
import { createHash } from 'crypto';
......@@ -40,7 +39,8 @@ export class PackagerService extends ConsoleLogger {
extractRoot = path.join(root, pathPrefix);
await fs.promises.mkdir(extractRoot, { recursive: true });
}
await this.spawnAsync('tar', ['-zxvf', '-'], { cwd: extractRoot, stdio: [stream, 'ignore', 'ignore'] });
this.log(`Extracting package to ${extractRoot}.`);
await this.spawnAsync('tar', ['-zxf', '-'], { cwd: extractRoot }, stream);
this.log(`Package extracted to ${extractRoot}.`);
......@@ -99,7 +99,7 @@ export class PackagerService extends ConsoleLogger {
packages[gotPackages[i]] = packagesSequence[i];
}
this.log({ checksum, packages });
// this.log({ checksum, packages });
await fs.promises.rm(root, { recursive: true });
await fs.promises.rm(tarballRoot, { recursive: true });
return new PackageResult(checksum, packages, gotPackages[0]);
......@@ -132,9 +132,25 @@ export class PackagerService extends ConsoleLogger {
return archive;
}
private spawnAsync(command: string, args: string[], options: child_process.SpawnOptions) {
private spawnAsync(
command: string,
args: string[],
options: child_process.SpawnOptions,
stdinStream?: internal.Readable,
stdoutStream?: internal.Writable,
stderrStream?: internal.Writable
) {
return new Promise<void>((resolve, reject) => {
const child = child_process.spawn(command, args, options);
if (stdinStream) {
stdinStream.pipe(child.stdin);
}
if (stdoutStream) {
child.stdout.pipe(stdoutStream);
}
if (stderrStream) {
child.stderr.pipe(stderrStream);
}
child.on('exit', (code) => {
if (code == 0) {
resolve();
......
import { ConsoleLogger, Injectable } from '@nestjs/common';
import { ConsoleLogger } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { ListObjectsCommand, PutObjectCommand, PutObjectCommandInput, S3Client } from '@aws-sdk/client-s3';
import { createHash } from 'crypto';
......
......@@ -25,22 +25,18 @@ export interface FetchMyCardUserOptions {
}
const _options = {
mycardAccountsUrl:
process.env.MYCARD_ACCOUNTS_URL || `https://sapi.moecube.com:444/accounts`,
mycardAccountsUrl: process.env.MYCARD_ACCOUNTS_URL || `https://sapi.moecube.com:444/accounts`,
field: 'sso',
};
export async function fetchUserWithToken(token: string) {
let authResult: AxiosResponse<MyCardUser>;
try {
authResult = await axios.get<MyCardUser>(
`${_options.mycardAccountsUrl}/authUser`,
{
responseType: 'json',
validateStatus: (s) => true,
headers: { Authorization: `Bearer ${token}` },
},
);
authResult = await axios.get<MyCardUser>(`${_options.mycardAccountsUrl}/authUser`, {
responseType: 'json',
validateStatus: (s) => true,
headers: { Authorization: `Bearer ${token}` },
});
} catch (e) {
return null;
}
......@@ -76,8 +72,6 @@ export function getUserFromContext(context: ExecutionContext) {
return fetchUserWithToken(token);
}
export const FetchMyCardUser = createParamDecorator(
async (_, context: ExecutionContext) => {
return getUserFromContext(context);
},
);
export const FetchMyCardUser = createParamDecorator(async (_, context: ExecutionContext) => {
return getUserFromContext(context);
});
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment