Commit f50c48f4 authored by nanahira's avatar nanahira

why is it stuck

parent 9fd068f1
This diff is collapsed.
......@@ -34,14 +34,12 @@
"lodash": "^4.17.21",
"moment": "^2.29.1",
"mustache": "^4.2.0",
"p-queue": "6.6.2",
"pg": "^8.7.1",
"readdirp": "^3.6.0",
"reflect-metadata": "^0.1.13",
"rimraf": "^3.0.2",
"rxjs": "^7.2.0",
"swagger-ui-express": "^4.1.6",
"tar": "^6.1.8",
"typeorm": "^0.2.37"
},
"optionalDependencies": {
......@@ -58,7 +56,6 @@
"@types/mustache": "^4.1.2",
"@types/node": "^16.0.0",
"@types/supertest": "^2.0.11",
"@types/tar": "^4.0.5",
"@typescript-eslint/eslint-plugin": "^4.28.2",
"@typescript-eslint/parser": "^4.28.2",
"eslint": "^7.30.0",
......
import {
Body,
Controller,
Delete,
Param,
Post,
Put,
UploadedFile,
UseGuards,
UseInterceptors,
ValidationPipe,
} from '@nestjs/common';
import {
ApiBody,
ApiConsumes,
ApiCreatedResponse,
ApiOkResponse,
ApiOperation,
ApiTags,
} from '@nestjs/swagger';
import { Body, Controller, Delete, Param, Post, Put, UploadedFile, UseGuards, UseInterceptors, ValidationPipe } from '@nestjs/common';
import { ApiBody, ApiConsumes, ApiCreatedResponse, ApiOkResponse, ApiOperation, ApiTags } from '@nestjs/swagger';
import { MyCardAdminGuard } from '../my-card-admin.guard';
import { FileInterceptor } from '@nestjs/platform-express';
import { FileUploadDto } from '../dto/FileUpload.dto';
......@@ -42,9 +24,10 @@ export class AdminController {
type: FileUploadDto,
})
@ApiCreatedResponse({ type: BlankReturnMessageDto })
async migrate(
@UploadedFile('file') file: Express.Multer.File,
): Promise<BlankReturnMessageDto> {
async migrate(@UploadedFile('file') file: Express.Multer.File): Promise<BlankReturnMessageDto> {
if (!file) {
throw new BlankReturnMessageDto(400, 'no file').toException();
}
const apps: AppsJson.App[] = JSON.parse(file.buffer.toString());
return this.appService.migrateFromAppsJson(apps);
}
......@@ -66,20 +49,14 @@ export class AdminController {
@Post('app/:id/assign')
@ApiOperation({ summary: '设置 app 管理者' })
@ApiOkResponse({ type: BlankReturnMessageDto })
async assignApp(
@Param('id') id: string,
@Body(new ValidationPipe({ transform: true })) assignAppData: AssignAppDto,
) {
async assignApp(@Param('id') id: string, @Body(new ValidationPipe({ transform: true })) assignAppData: AssignAppDto) {
return this.appService.assignApp(id, assignAppData.author);
}
@Post('app/:id/prefix')
@ApiOperation({ summary: '设置 app 打包前缀' })
@ApiOkResponse({ type: BlankReturnMessageDto })
async setAppPrefix(
@Param('id') id: string,
@Body(new ValidationPipe({ transform: true })) appPrefix: AppPrefixDto,
) {
async setAppPrefix(@Param('id') id: string, @Body(new ValidationPipe({ transform: true })) appPrefix: AppPrefixDto) {
return this.appService.setAppPrefix(id, appPrefix._prefix);
}
}
import {
BadRequestException,
Body,
Controller,
Get,
Param,
Post,
Query,
UploadedFile,
......@@ -10,34 +12,20 @@ import {
ValidationPipe,
} from '@nestjs/common';
import { AppService } from './app.service';
import {
ApiBody,
ApiConsumes,
ApiCreatedResponse,
ApiOkResponse,
ApiOperation,
ApiQuery,
} from '@nestjs/swagger';
import {
BlankReturnMessageDto,
GetAppReturnMessageDto,
ReturnMessageDto,
StringReturnMessageDto,
} from './dto/ReturnMessage.dto';
import { ApiBody, ApiConsumes, ApiCreatedResponse, ApiOkResponse, ApiOperation, ApiParam, ApiQuery } from '@nestjs/swagger';
import { BlankReturnMessageDto, GetAppReturnMessageDto, ReturnMessageDto, StringReturnMessageDto } from './dto/ReturnMessage.dto';
import { FetchMyCardUser, MyCardUser } from './utility/mycard-auth';
import { AppsJson } from './utility/apps-json-type';
import { MyCardAppMaintainerGuard } from './my-card-app-maintainer.guard';
import { S3Service } from './s3/s3.service';
import { FileInterceptor } from '@nestjs/platform-express';
import { FileUploadDto } from './dto/FileUpload.dto';
import AppClass = AppsJson.AppClass;
import { AssetsS3Service } from './assets-s3/assets-s3.service';
import { MulterDirectEngine } from './packager/MulterStreamEngine';
@Controller('api')
export class AppController {
constructor(
private readonly appService: AppService,
private readonly s3: S3Service,
) {}
constructor(private readonly appService: AppService, private readonly s3: AssetsS3Service) {}
@Get('apps.json')
getAppsJson() {
......@@ -61,10 +49,7 @@ export class AppController {
})
@ApiBody({ type: AppsJson.AppClass })
@ApiCreatedResponse({ type: BlankReturnMessageDto })
updateApp(
@FetchMyCardUser() user: MyCardUser,
@Body(new ValidationPipe({ transform: true })) app: AppClass,
) {
updateApp(@FetchMyCardUser() user: MyCardUser, @Body(new ValidationPipe({ transform: true })) app: AppClass) {
return this.appService.updateApp(user, app.id, app);
}
......@@ -82,6 +67,9 @@ export class AppController {
@ApiCreatedResponse({ type: StringReturnMessageDto })
@UseGuards(MyCardAppMaintainerGuard)
async uploadAssets(@UploadedFile() file: Express.Multer.File) {
if (!file) {
throw new BlankReturnMessageDto(400, 'no file').toException();
}
const res = await this.s3.uploadAssets(file);
if (res) {
return new ReturnMessageDto(201, 'success', res);
......@@ -89,4 +77,35 @@ export class AppController {
throw new BlankReturnMessageDto(500, 'upload fail').toException();
}
}
@Post('build/:id/:platform/:locale/:version')
@ApiOperation({
summary: '打包文件',
description: '必须登录用户且必须是管理员或者拥有1个 app 才能上传',
})
@UseInterceptors(FileInterceptor('file', { storage: new MulterDirectEngine() }))
@ApiConsumes('multipart/form-data')
@ApiParam({ name: 'id', description: 'APP 的 id' })
@ApiParam({ name: 'platform', description: 'APP 的 版本号', enum: AppsJson.Platform })
@ApiParam({ name: 'locale', description: 'APP 的 版本号', enum: AppsJson.Locale })
@ApiParam({ name: 'version', description: 'APP 的 版本号' })
@ApiBody({
description: 'app 的 tar.gz 文件',
type: FileUploadDto,
})
@ApiCreatedResponse({ type: BlankReturnMessageDto })
async makeBuild(
@FetchMyCardUser() user: MyCardUser,
@UploadedFile() file: Express.Multer.File,
@Param('id') id: string,
@Param('platform') platform: AppsJson.Platform,
@Param('locale') locale: AppsJson.Locale,
@Param('version') version: string
) {
console.log(file.stream);
if (!file) {
throw new BlankReturnMessageDto(400, 'no file').toException();
}
return this.appService.makeBuild(user, file, id, platform, locale, version);
}
}
......@@ -8,6 +8,8 @@ import { App } from './entities/App.entity';
import { AppHistory } from './entities/AppHistory.entity';
import { S3Service } from './s3/s3.service';
import { PackagerService } from './packager/packager.service';
import { AssetsS3Service } from './assets-s3/assets-s3.service';
import { PackageS3Service } from './package-s3/package-s3.service';
const configModule = ConfigModule.forRoot();
......@@ -33,6 +35,6 @@ const configModule = ConfigModule.forRoot();
}),
],
controllers: [AppController, AdminController],
providers: [AppService, S3Service, PackagerService],
providers: [AppService, PackagerService, AssetsS3Service, PackageS3Service],
})
export class AppModule {}
import { Connection, IsNull, Not } from 'typeorm';
import { InjectConnection } from '@nestjs/typeorm';
import { ConsoleLogger, Injectable } from '@nestjs/common';
import { ConsoleLogger, Injectable, Param, UploadedFile } from '@nestjs/common';
import { AppsJson } from './utility/apps-json-type';
import { App } from './entities/App.entity';
import {
BlankReturnMessageDto,
ReturnMessageDto,
} from './dto/ReturnMessage.dto';
import { MyCardUser } from './utility/mycard-auth';
import { BlankReturnMessageDto, ReturnMessageDto } from './dto/ReturnMessage.dto';
import { FetchMyCardUser, MyCardUser } from './utility/mycard-auth';
import { PackagerService } from './packager/packager.service';
@Injectable()
export class AppService extends ConsoleLogger {
constructor(
@InjectConnection('app')
private db: Connection,
private packager: PackagerService
) {
super('app');
}
async getAppsJson() {
return (
await this.db
.getRepository(App)
.find({ where: { appData: Not(IsNull()), isDeleted: false } })
).map((a) => a.appData);
return (await this.db.getRepository(App).find({ where: { appData: Not(IsNull()), isDeleted: false } })).map((a) => a.appData);
}
private async updateResult<T>(f: () => Promise<T>, returnCode = 200) {
......@@ -39,14 +34,9 @@ export class AppService extends ConsoleLogger {
const targetApps: App[] = [];
for (const appData of apps) {
if (!appData.id) {
throw new BlankReturnMessageDto(
400,
`App ${appData.name} is invalid.`,
).toException();
throw new BlankReturnMessageDto(400, `App ${appData.name} is invalid.`).toException();
}
const checkExistingApp = await this.db
.getRepository(App)
.findOne({ where: { id: appData.id }, relations: ['history'] });
const checkExistingApp = await this.db.getRepository(App).findOne({ where: { id: appData.id }, relations: ['history'] });
//this.error('read');
if (checkExistingApp) {
checkExistingApp.updateApp(appData);
......@@ -70,10 +60,7 @@ export class AppService extends ConsoleLogger {
if (!user) {
throw new BlankReturnMessageDto(401, 'Needs login').toException();
}
const query = this.db
.getRepository(App)
.createQueryBuilder('app')
.where('app.isDeleted = false');
const query = this.db.getRepository(App).createQueryBuilder('app').where('app.isDeleted = false');
if (!user.admin) {
query.andWhere(':uid = ANY(app.author)', { uid: user.id });
}
......@@ -100,18 +87,13 @@ export class AppService extends ConsoleLogger {
}
async createApp(id: string) {
let app = await this.db
.getRepository(App)
.findOne({ where: { id }, select: ['id', 'isDeleted'] });
let app = await this.db.getRepository(App).findOne({ where: { id }, select: ['id', 'isDeleted'] });
if (!app) {
app = new App();
app.id = id;
} else {
if (!app.isDeleted) {
throw new BlankReturnMessageDto(
404,
'App already exists',
).toException();
throw new BlankReturnMessageDto(404, 'App already exists').toException();
}
app.isDeleted = false;
}
......@@ -119,18 +101,11 @@ export class AppService extends ConsoleLogger {
}
async assignApp(id: string, author: number[]) {
return this.updateResult(
() => this.db.getRepository(App).update({ id }, { author }),
201,
);
return this.updateResult(() => this.db.getRepository(App).update({ id }, { author }), 201);
}
async setAppPrefix(id: string, prefix: string) {
return this.updateResult(
() =>
this.db.getRepository(App).update({ id }, { packagePrefix: prefix }),
201,
);
return this.updateResult(() => this.db.getRepository(App).update({ id }, { packagePrefix: prefix }), 201);
}
async updateApp(user: MyCardUser, id: string, appData: AppsJson.App) {
......@@ -156,9 +131,33 @@ export class AppService extends ConsoleLogger {
}, 201);
}
async makeBuild(
user: MyCardUser,
file: Express.Multer.File,
id: string,
platform: AppsJson.Platform,
locale: AppsJson.Locale,
version: string
) {
if (!user) {
throw new BlankReturnMessageDto(401, 'Needs login').toException();
}
this.log('Build: Checking app.');
const app = await this.db.getRepository(App).findOne({
where: { id },
select: ['id', 'packagePrefix', 'author'],
});
if (!app) {
throw new BlankReturnMessageDto(404, 'App not found').toException();
}
if (!app.isUserCanEditApp(user)) {
throw new BlankReturnMessageDto(403, 'Permission denied').toException();
}
const result = await this.packager.build(file.stream, app.packagePrefix);
return new ReturnMessageDto(201, 'success', result);
}
async deleteApp(id: string) {
return this.updateResult(() =>
this.db.getRepository(App).update({ id }, { isDeleted: true }),
);
return this.updateResult(() => this.db.getRepository(App).update({ id }, { isDeleted: true }));
}
}
import { Test, TestingModule } from '@nestjs/testing';
import { S3Service } from './s3.service';
import { AssetsS3Service } from './assets-s3.service';
describe('S3Service', () => {
let service: S3Service;
describe('AssetsS3Service', () => {
let service: AssetsS3Service;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [S3Service],
providers: [AssetsS3Service],
}).compile();
service = module.get<S3Service>(S3Service);
service = module.get<AssetsS3Service>(AssetsS3Service);
});
it('should be defined', () => {
......
import { Injectable } from '@nestjs/common';
import { S3Service } from '../s3/s3.service';
import { ConfigService } from '@nestjs/config';
@Injectable()
export class AssetsS3Service extends S3Service {
constructor(config: ConfigService) {
super('ASSETS', config);
}
}
export class PackageResult {
constructor(public checksum: Record<string, string>, public packages: Record<string, string[]>) {}
}
import { Test, TestingModule } from '@nestjs/testing';
import { PackageS3Service } from './package-s3.service';
describe('PackageS3Service', () => {
let service: PackageS3Service;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [PackageS3Service],
}).compile();
service = module.get<PackageS3Service>(PackageS3Service);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
});
import { Injectable } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { S3Service } from '../s3/s3.service';
@Injectable()
export class PackageS3Service extends S3Service {
constructor(config: ConfigService) {
super('PACKAGE', config);
}
}
import multer from 'multer';
export class MulterDirectEngine implements multer.StorageEngine {
_handleFile(req: Express.Request, file: Express.Multer.File, callback: (error?: any, info?: Partial<Express.Multer.File>) => void) {
callback(null, { stream: file.stream });
}
_removeFile(req: Express.Request, file: Express.Multer.File, callback: (error: Error | null) => void) {
callback(null);
}
}
......@@ -2,32 +2,38 @@ import { ConsoleLogger, Injectable } from '@nestjs/common';
import fs from 'fs';
import path from 'path';
import child_process from 'child_process';
import tar from 'tar';
import os from 'os';
import { S3Service } from '../s3/s3.service';
import { PutObjectCommand } from '@aws-sdk/client-s3';
import util from 'util';
import { v4 as uuidv4 } from 'uuid';
import readdirp from 'readdirp';
import _ from 'lodash';
import { ConfigService } from '@nestjs/config';
import internal from 'stream';
import { PackageResult } from '../dto/PackageResult.dto';
@Injectable()
export class PackagerService extends ConsoleLogger {
// workingPath: string;
// releasePath: string;
// downloadBaseUrl: string;
// queueIdMap = new Map<string, PQueue>();
bucket_max = 10 * 1024 ** 2;
bucket_enter = 1 * 1024 ** 2;
constructor(private s3: S3Service) {
constructor(private s3: S3Service, config: ConfigService) {
super('packager');
this.bucket_max = (parseInt(config.get('PACKAGE_BUCKET_MAX')) || 10) * 1024 ** 2;
this.bucket_enter = (parseInt(config.get('PACKAGE_BUCKET_ENTER')) || 1) * 1024 ** 2;
}
async build(stream: fs.ReadStream): Promise<BuildResult> {
const bucket_max = 10 * 1024 ** 2;
const bucket_enter = 1 * 1024 ** 2;
async build(stream: internal.Readable, pathPrefix?: string): Promise<PackageResult> {
this.log(`Start packaging.`);
const root = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'mycard-console-'));
await this.spawnAsync('tar', ['-zxvf', '-'], { cwd: root, stdio: [stream, 'inherit', 'inherit'] });
let extractRoot = root;
if (pathPrefix) {
extractRoot = path.join(root, pathPrefix);
await fs.promises.mkdir(extractRoot, { recursive: true });
}
await this.spawnAsync('tar', ['-zxvf', '-'], { cwd: extractRoot, stdio: [stream, 'inherit', 'inherit'] });
this.log(`Package extracted to ${extractRoot}.`);
const buckets: Record<string, [string[], number]> = {};
const packages: Record<string, string[]> = {};
......@@ -50,11 +56,11 @@ export class PackagerService extends ConsoleLogger {
// 散包
for (const file of files) {
if (file.stats.size < bucket_enter) {
if (file.stats.size < this.bucket_enter) {
const extname = path.extname(file.basename);
buckets[extname] ??= [[], 0];
const bucket = buckets[extname];
if (bucket[1] + file.stats.size >= bucket_max) {
if (bucket[1] + file.stats.size >= this.bucket_max) {
const archive = `${uuidv4()}.tar.gz`;
packages[archive] = bucket[0];
promises.push(this.archive(archive, root, bucket[0]));
......@@ -81,8 +87,9 @@ export class PackagerService extends ConsoleLogger {
// TODO: 更新包
const [checksum] = await Promise.all(promises); // 这个 await 过后,checksum 和 打包上传都已经跑完了
console.log(checksum, packages);
return { checksum, packages };
this.log({ checksum, packages });
await fs.promises.unlink(root);
return new PackageResult(checksum, packages);
}
async checksum(root: string, directories: string[], files: string[]) {
......@@ -99,13 +106,7 @@ export class PackagerService extends ConsoleLogger {
cwd: root,
stdio: ['ignore', 'pipe', 'inherit'],
});
return this.s3.s3.send(
new PutObjectCommand({
Bucket: this.s3.bucket,
Key: archive,
Body: child.stdout,
})
);
return this.s3.uploadFile(archive, child.stdout, 'application/tar+gzip');
}
private spawnAsync(command: string, args: string[], options: child_process.SpawnOptions) {
......@@ -124,8 +125,3 @@ export class PackagerService extends ConsoleLogger {
});
}
}
export interface BuildResult {
checksum: Record<string, string>;
packages: Record<string, string[]>;
}
import { ConsoleLogger, Injectable } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import {
ListObjectsCommand,
PutObjectCommand,
S3Client,
} from '@aws-sdk/client-s3';
import { ListObjectsCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
import { createHash } from 'crypto';
import internal from 'stream';
@Injectable()
export class S3Service extends ConsoleLogger {
bucket: string;
prefix: string;
cdnUrl: string;
s3: S3Client;
constructor(config: ConfigService) {
super('s3');
this.bucket = config.get('S3_BUCKET');
this.prefix = config.get('S3_PREFIX');
this.cdnUrl =
config.get('S3_CDN_URL') ||
`${config.get('S3_ENDPOINT')}/${this.bucket}/${this.prefix}`;
private readonly bucket: string;
private readonly prefix: string;
private readonly cdnUrl: string;
private readonly s3: S3Client;
private getConfig(field: string) {
return this.config.get(`${this.servicePrefix}_${field}`) || this.config.get(field);
}
constructor(private servicePrefix: string, private config: ConfigService) {
super(`${servicePrefix} s3`);
this.bucket = this.getConfig('S3_BUCKET');
this.prefix = this.getConfig('S3_PREFIX');
this.cdnUrl = this.getConfig('S3_CDN_URL') || `${this.getConfig('S3_ENDPOINT')}/${this.bucket}${this.prefix ? `/${this.prefix}` : ''}`;
this.s3 = new S3Client({
credentials: {
accessKeyId: config.get('S3_KEY'),
secretAccessKey: config.get('S3_SECRET'),
accessKeyId: this.getConfig('S3_KEY'),
secretAccessKey: this.getConfig('S3_SECRET'),
},
region: config.get('S3_REGION') || 'us-west-1',
endpoint: config.get('S3_ENDPOINT'),
region: this.getConfig('S3_REGION') || 'us-west-1',
endpoint: this.getConfig('S3_ENDPOINT'),
forcePathStyle: true,
});
}
private async listObjects(path: string) {
async listObjects(path: string) {
const command = new ListObjectsCommand({
Bucket: this.bucket,
Prefix: path,
......@@ -39,37 +38,39 @@ export class S3Service extends ConsoleLogger {
return this.s3.send(command);
}
private getPathWithPrefix(filename: string) {
return this.prefix ? `${this.prefix}/${filename}` : filename;
}
async uploadAssets(file: Express.Multer.File) {
const fileSuffix = file.originalname.split('.').pop();
const hash = createHash('sha512').update(file.buffer).digest('hex');
const filename = `${hash}.${fileSuffix}`;
const path = `${this.prefix}/${filename}`;
const path = this.getPathWithPrefix(filename);
const downloadUrl = `${this.cdnUrl}/${filename}`;
const checkExisting = await this.listObjects(path);
try {
if (
checkExisting.Contents &&
checkExisting.Contents.some((obj) => obj.Key === path)
) {
if (checkExisting.Contents && checkExisting.Contents.some((obj) => obj.Key === path)) {
// already uploaded
return downloadUrl;
} else {
await this.uploadFile(path, file.buffer, file.mimetype);
return this.uploadFile(filename, file.buffer, file.mimetype);
}
return downloadUrl;
} catch (e) {
this.error(`Failed to assign upload of file ${path}: ${e.toString()}`);
this.error(`Failed to upload assets ${path}: ${e.toString()}`);
return null;
}
}
async uploadFile(path: string, buffer: Buffer, mime?: string) {
return this.s3.send(
async uploadFile(path: string, content: Buffer | internal.Readable, mime?: string) {
await this.s3.send(
new PutObjectCommand({
Bucket: this.bucket,
Key: path,
Body: buffer,
Key: this.getPathWithPrefix(path),
Body: content,
ContentType: mime,
}),
})
);
return `${this.cdnUrl}/${path}`;
}
}
......@@ -2,6 +2,7 @@ import { IsNotEmpty } from 'class-validator';
export namespace AppsJson {
export enum Locale {
generic = 'generic',
zh_CN = 'zh-CN',
en_US = 'en-US',
ja_JP = 'ja-JP',
......@@ -11,6 +12,7 @@ export namespace AppsJson {
zh_TW = 'zh-TW',
}
export enum Platform {
generic = 'generic',
Linux = 'linux',
macOS = 'darwin',
Windows = 'win32',
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment