Commit 5e90acd3 authored by nanahira's avatar nanahira

stream thing

parent 119e0e6b
...@@ -16,6 +16,7 @@ ...@@ -16,6 +16,7 @@
"@nestjs/swagger": "^5.0.9", "@nestjs/swagger": "^5.0.9",
"@nestjs/typeorm": "^8.0.2", "@nestjs/typeorm": "^8.0.2",
"axios": "^0.21.1", "axios": "^0.21.1",
"busboy": "^0.2.14",
"class-transformer": "^0.4.0", "class-transformer": "^0.4.0",
"class-validator": "^0.13.1", "class-validator": "^0.13.1",
"lodash": "^4.17.21", "lodash": "^4.17.21",
...@@ -33,6 +34,7 @@ ...@@ -33,6 +34,7 @@
"@nestjs/cli": "^8.0.0", "@nestjs/cli": "^8.0.0",
"@nestjs/schematics": "^8.0.0", "@nestjs/schematics": "^8.0.0",
"@nestjs/testing": "^8.0.0", "@nestjs/testing": "^8.0.0",
"@types/busboy": "^0.2.4",
"@types/express": "^4.17.13", "@types/express": "^4.17.13",
"@types/jest": "^26.0.24", "@types/jest": "^26.0.24",
"@types/lodash": "^4.14.172", "@types/lodash": "^4.14.172",
...@@ -3067,6 +3069,15 @@ ...@@ -3067,6 +3069,15 @@
"@types/node": "*" "@types/node": "*"
} }
}, },
"node_modules/@types/busboy": {
"version": "0.2.4",
"resolved": "https://registry.npmjs.org/@types/busboy/-/busboy-0.2.4.tgz",
"integrity": "sha512-f+ZCVjlcN8JW/zf3iR0GqO4gjOUlltMTtZjn+YR1mlK+MVu6esTiIecO0/GQlmYQPQLdBnc7+5vG3Xb+SkvFLw==",
"dev": true,
"dependencies": {
"@types/node": "*"
}
},
"node_modules/@types/connect": { "node_modules/@types/connect": {
"version": "3.4.35", "version": "3.4.35",
"resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz", "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz",
...@@ -13884,6 +13895,15 @@ ...@@ -13884,6 +13895,15 @@
"@types/node": "*" "@types/node": "*"
} }
}, },
"@types/busboy": {
"version": "0.2.4",
"resolved": "https://registry.npmjs.org/@types/busboy/-/busboy-0.2.4.tgz",
"integrity": "sha512-f+ZCVjlcN8JW/zf3iR0GqO4gjOUlltMTtZjn+YR1mlK+MVu6esTiIecO0/GQlmYQPQLdBnc7+5vG3Xb+SkvFLw==",
"dev": true,
"requires": {
"@types/node": "*"
}
},
"@types/connect": { "@types/connect": {
"version": "3.4.35", "version": "3.4.35",
"resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz", "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz",
......
...@@ -29,6 +29,7 @@ ...@@ -29,6 +29,7 @@
"@nestjs/swagger": "^5.0.9", "@nestjs/swagger": "^5.0.9",
"@nestjs/typeorm": "^8.0.2", "@nestjs/typeorm": "^8.0.2",
"axios": "^0.21.1", "axios": "^0.21.1",
"busboy": "^0.2.14",
"class-transformer": "^0.4.0", "class-transformer": "^0.4.0",
"class-validator": "^0.13.1", "class-validator": "^0.13.1",
"lodash": "^4.17.21", "lodash": "^4.17.21",
...@@ -49,6 +50,7 @@ ...@@ -49,6 +50,7 @@
"@nestjs/cli": "^8.0.0", "@nestjs/cli": "^8.0.0",
"@nestjs/schematics": "^8.0.0", "@nestjs/schematics": "^8.0.0",
"@nestjs/testing": "^8.0.0", "@nestjs/testing": "^8.0.0",
"@types/busboy": "^0.2.4",
"@types/express": "^4.17.13", "@types/express": "^4.17.13",
"@types/jest": "^26.0.24", "@types/jest": "^26.0.24",
"@types/lodash": "^4.14.172", "@types/lodash": "^4.14.172",
......
import { import { Body, Controller, Get, Param, Post, Query, Req, UploadedFile, UseGuards, UseInterceptors, ValidationPipe } from '@nestjs/common';
BadRequestException,
Body,
Controller,
Get,
Param,
Post,
Query,
UploadedFile,
UseGuards,
UseInterceptors,
ValidationPipe,
} from '@nestjs/common';
import { AppService } from './app.service'; import { AppService } from './app.service';
import { ApiBody, ApiConsumes, ApiCreatedResponse, ApiOkResponse, ApiOperation, ApiParam, ApiQuery } from '@nestjs/swagger'; import { ApiBody, ApiConsumes, ApiCreatedResponse, ApiOkResponse, ApiOperation, ApiParam, ApiQuery } from '@nestjs/swagger';
import { BlankReturnMessageDto, GetAppReturnMessageDto, ReturnMessageDto, StringReturnMessageDto } from './dto/ReturnMessage.dto'; import { BlankReturnMessageDto, GetAppReturnMessageDto, ReturnMessageDto, StringReturnMessageDto } from './dto/ReturnMessage.dto';
...@@ -19,10 +7,13 @@ import { AppsJson } from './utility/apps-json-type'; ...@@ -19,10 +7,13 @@ import { AppsJson } from './utility/apps-json-type';
import { MyCardAppMaintainerGuard } from './my-card-app-maintainer.guard'; import { MyCardAppMaintainerGuard } from './my-card-app-maintainer.guard';
import { FileInterceptor } from '@nestjs/platform-express'; import { FileInterceptor } from '@nestjs/platform-express';
import { FileUploadDto } from './dto/FileUpload.dto'; import { FileUploadDto } from './dto/FileUpload.dto';
import AppClass = AppsJson.AppClass;
import { AssetsS3Service } from './assets-s3/assets-s3.service'; import { AssetsS3Service } from './assets-s3/assets-s3.service';
import { MulterDirectEngine } from './packager/MulterStreamEngine'; import Busboy from 'busboy';
import * as fs from 'fs'; import { Request } from 'express';
import { Stream } from 'stream';
import { PackageResult } from './dto/PackageResult.dto';
import { platform } from 'os';
import AppClass = AppsJson.AppClass;
@Controller('api') @Controller('api')
export class AppController { export class AppController {
...@@ -84,16 +75,15 @@ export class AppController { ...@@ -84,16 +75,15 @@ export class AppController {
summary: '打包文件', summary: '打包文件',
description: '必须登录用户且必须是管理员或者拥有1个 app 才能上传', description: '必须登录用户且必须是管理员或者拥有1个 app 才能上传',
}) })
@UseInterceptors(FileInterceptor('file', { storage: new MulterDirectEngine() }))
@ApiConsumes('multipart/form-data') @ApiConsumes('multipart/form-data')
@ApiParam({ name: 'id', description: 'APP 的 id' }) @ApiParam({ name: 'id', description: 'APP 的 id' })
@ApiParam({ name: 'platform', description: 'APP 的 版本号', enum: AppsJson.Platform }) @ApiParam({ name: 'platform', description: 'APP 的 版本号', enum: AppsJson.Platform })
@ApiParam({ name: 'locale', description: 'APP 的 版本号', enum: AppsJson.Locale }) @ApiParam({ name: 'locale', description: 'APP 的 版本号', enum: AppsJson.Locale })
@ApiParam({ name: 'version', description: 'APP 的 版本号' }) @ApiParam({ name: 'version', description: 'APP 的 版本号' })
/*@ApiBody({ @ApiBody({
description: 'app 的 tar.gz 文件', description: 'app 的 tar.gz 文件',
type: FileUploadDto, type: FileUploadDto,
})*/ })
@ApiCreatedResponse({ type: BlankReturnMessageDto }) @ApiCreatedResponse({ type: BlankReturnMessageDto })
async makeBuild( async makeBuild(
@FetchMyCardUser() user: MyCardUser, @FetchMyCardUser() user: MyCardUser,
...@@ -101,12 +91,44 @@ export class AppController { ...@@ -101,12 +91,44 @@ export class AppController {
@Param('id') id: string, @Param('id') id: string,
@Param('platform') platform: AppsJson.Platform, @Param('platform') platform: AppsJson.Platform,
@Param('locale') locale: AppsJson.Locale, @Param('locale') locale: AppsJson.Locale,
@Param('version') version: string @Param('version') version: string,
@Req() req: Request
) { ) {
/*console.log(file.stream); let busboy: busboy.Busboy;
if (!file) { try {
throw new BlankReturnMessageDto(400, 'no file').toException(); busboy = new Busboy({ headers: req.headers });
}*/ } catch (e) {
return this.appService.makeBuild(user, fs.createReadStream('/tmp/test1.tar.gz'), id, platform, locale, version); throw new BlankReturnMessageDto(400, `Creation failed: ${e.toString()}`).toException();
}
const packagePromise = new Promise<ReturnMessageDto<PackageResult>>((resolve, reject) => {
let gotFile = false;
busboy.on('file', async (fieldname, fileStream, filename, encoding, mimetype) => {
if (fieldname !== 'file') {
reject(new BlankReturnMessageDto(400, 'invalid field').toException());
return;
}
gotFile = true;
// console.log(`got file ${fieldname}`);
const stream = new Stream.Readable().wrap(fileStream);
try {
resolve(await this.appService.makeBuild(user, stream, id, platform, locale, version));
} catch (e) {
stream.destroy();
reject(e);
}
});
busboy.on('error', () => {
reject(new BlankReturnMessageDto(500, 'upload error').toException());
});
busboy.on('finish', () => {
//console.log(`finished receiving file`);
if (!gotFile) {
reject(new BlankReturnMessageDto(400, 'no file').toException());
}
//resolve();
});
});
req.pipe(busboy);
return packagePromise;
} }
} }
...@@ -6,7 +6,6 @@ import { AdminController } from './admin/admin.controller'; ...@@ -6,7 +6,6 @@ import { AdminController } from './admin/admin.controller';
import { ConfigModule, ConfigService } from '@nestjs/config'; import { ConfigModule, ConfigService } from '@nestjs/config';
import { App } from './entities/App.entity'; import { App } from './entities/App.entity';
import { AppHistory } from './entities/AppHistory.entity'; import { AppHistory } from './entities/AppHistory.entity';
import { S3Service } from './s3/s3.service';
import { PackagerService } from './packager/packager.service'; import { PackagerService } from './packager/packager.service';
import { AssetsS3Service } from './assets-s3/assets-s3.service'; import { AssetsS3Service } from './assets-s3/assets-s3.service';
import { PackageS3Service } from './package-s3/package-s3.service'; import { PackageS3Service } from './package-s3/package-s3.service';
......
import { Connection, IsNull, Not } from 'typeorm'; import { Connection, IsNull, Not } from 'typeorm';
import { InjectConnection } from '@nestjs/typeorm'; import { InjectConnection } from '@nestjs/typeorm';
import { ConsoleLogger, Injectable, Param, UploadedFile } from '@nestjs/common'; import { ConsoleLogger, Injectable } from '@nestjs/common';
import { AppsJson } from './utility/apps-json-type'; import { AppsJson } from './utility/apps-json-type';
import { App } from './entities/App.entity'; import { App } from './entities/App.entity';
import { BlankReturnMessageDto, ReturnMessageDto } from './dto/ReturnMessage.dto'; import { BlankReturnMessageDto, ReturnMessageDto } from './dto/ReturnMessage.dto';
import { FetchMyCardUser, MyCardUser } from './utility/mycard-auth'; import { MyCardUser } from './utility/mycard-auth';
import { PackagerService } from './packager/packager.service'; import { PackagerService } from './packager/packager.service';
import internal from 'stream'; import internal from 'stream';
......
import { import { Column, Entity, Index, ManyToOne, PrimaryGeneratedColumn } from 'typeorm';
Column,
Entity,
Index,
ManyToOne,
PrimaryGeneratedColumn,
} from 'typeorm';
import { AppBase } from './AppBase.entity'; import { AppBase } from './AppBase.entity';
import { App } from './App.entity'; import { App } from './App.entity';
......
...@@ -10,7 +10,6 @@ import internal from 'stream'; ...@@ -10,7 +10,6 @@ import internal from 'stream';
import { PackageResult } from '../dto/PackageResult.dto'; import { PackageResult } from '../dto/PackageResult.dto';
import { PackageS3Service } from '../package-s3/package-s3.service'; import { PackageS3Service } from '../package-s3/package-s3.service';
import readdirp from 'readdirp'; import readdirp from 'readdirp';
import { v4 as uuidv4 } from 'uuid';
import { ConsoleLogger, Injectable } from '@nestjs/common'; import { ConsoleLogger, Injectable } from '@nestjs/common';
import { createHash } from 'crypto'; import { createHash } from 'crypto';
...@@ -40,7 +39,8 @@ export class PackagerService extends ConsoleLogger { ...@@ -40,7 +39,8 @@ export class PackagerService extends ConsoleLogger {
extractRoot = path.join(root, pathPrefix); extractRoot = path.join(root, pathPrefix);
await fs.promises.mkdir(extractRoot, { recursive: true }); await fs.promises.mkdir(extractRoot, { recursive: true });
} }
await this.spawnAsync('tar', ['-zxvf', '-'], { cwd: extractRoot, stdio: [stream, 'ignore', 'ignore'] }); this.log(`Extracting package to ${extractRoot}.`);
await this.spawnAsync('tar', ['-zxf', '-'], { cwd: extractRoot }, stream);
this.log(`Package extracted to ${extractRoot}.`); this.log(`Package extracted to ${extractRoot}.`);
...@@ -99,7 +99,7 @@ export class PackagerService extends ConsoleLogger { ...@@ -99,7 +99,7 @@ export class PackagerService extends ConsoleLogger {
packages[gotPackages[i]] = packagesSequence[i]; packages[gotPackages[i]] = packagesSequence[i];
} }
this.log({ checksum, packages }); // this.log({ checksum, packages });
await fs.promises.rm(root, { recursive: true }); await fs.promises.rm(root, { recursive: true });
await fs.promises.rm(tarballRoot, { recursive: true }); await fs.promises.rm(tarballRoot, { recursive: true });
return new PackageResult(checksum, packages, gotPackages[0]); return new PackageResult(checksum, packages, gotPackages[0]);
...@@ -132,9 +132,25 @@ export class PackagerService extends ConsoleLogger { ...@@ -132,9 +132,25 @@ export class PackagerService extends ConsoleLogger {
return archive; return archive;
} }
private spawnAsync(command: string, args: string[], options: child_process.SpawnOptions) { private spawnAsync(
command: string,
args: string[],
options: child_process.SpawnOptions,
stdinStream?: internal.Readable,
stdoutStream?: internal.Writable,
stderrStream?: internal.Writable
) {
return new Promise<void>((resolve, reject) => { return new Promise<void>((resolve, reject) => {
const child = child_process.spawn(command, args, options); const child = child_process.spawn(command, args, options);
if (stdinStream) {
stdinStream.pipe(child.stdin);
}
if (stdoutStream) {
child.stdout.pipe(stdoutStream);
}
if (stderrStream) {
child.stderr.pipe(stderrStream);
}
child.on('exit', (code) => { child.on('exit', (code) => {
if (code == 0) { if (code == 0) {
resolve(); resolve();
......
import { ConsoleLogger, Injectable } from '@nestjs/common'; import { ConsoleLogger } from '@nestjs/common';
import { ConfigService } from '@nestjs/config'; import { ConfigService } from '@nestjs/config';
import { ListObjectsCommand, PutObjectCommand, PutObjectCommandInput, S3Client } from '@aws-sdk/client-s3'; import { ListObjectsCommand, PutObjectCommand, PutObjectCommandInput, S3Client } from '@aws-sdk/client-s3';
import { createHash } from 'crypto'; import { createHash } from 'crypto';
......
...@@ -25,22 +25,18 @@ export interface FetchMyCardUserOptions { ...@@ -25,22 +25,18 @@ export interface FetchMyCardUserOptions {
} }
const _options = { const _options = {
mycardAccountsUrl: mycardAccountsUrl: process.env.MYCARD_ACCOUNTS_URL || `https://sapi.moecube.com:444/accounts`,
process.env.MYCARD_ACCOUNTS_URL || `https://sapi.moecube.com:444/accounts`,
field: 'sso', field: 'sso',
}; };
export async function fetchUserWithToken(token: string) { export async function fetchUserWithToken(token: string) {
let authResult: AxiosResponse<MyCardUser>; let authResult: AxiosResponse<MyCardUser>;
try { try {
authResult = await axios.get<MyCardUser>( authResult = await axios.get<MyCardUser>(`${_options.mycardAccountsUrl}/authUser`, {
`${_options.mycardAccountsUrl}/authUser`, responseType: 'json',
{ validateStatus: (s) => true,
responseType: 'json', headers: { Authorization: `Bearer ${token}` },
validateStatus: (s) => true, });
headers: { Authorization: `Bearer ${token}` },
},
);
} catch (e) { } catch (e) {
return null; return null;
} }
...@@ -76,8 +72,6 @@ export function getUserFromContext(context: ExecutionContext) { ...@@ -76,8 +72,6 @@ export function getUserFromContext(context: ExecutionContext) {
return fetchUserWithToken(token); return fetchUserWithToken(token);
} }
export const FetchMyCardUser = createParamDecorator( export const FetchMyCardUser = createParamDecorator(async (_, context: ExecutionContext) => {
async (_, context: ExecutionContext) => { return getUserFromContext(context);
return getUserFromContext(context); });
},
);
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment