Commit f50c48f4 authored by nanahira's avatar nanahira

why is it stuck

parent 9fd068f1
This diff is collapsed.
...@@ -34,14 +34,12 @@ ...@@ -34,14 +34,12 @@
"lodash": "^4.17.21", "lodash": "^4.17.21",
"moment": "^2.29.1", "moment": "^2.29.1",
"mustache": "^4.2.0", "mustache": "^4.2.0",
"p-queue": "6.6.2",
"pg": "^8.7.1", "pg": "^8.7.1",
"readdirp": "^3.6.0", "readdirp": "^3.6.0",
"reflect-metadata": "^0.1.13", "reflect-metadata": "^0.1.13",
"rimraf": "^3.0.2", "rimraf": "^3.0.2",
"rxjs": "^7.2.0", "rxjs": "^7.2.0",
"swagger-ui-express": "^4.1.6", "swagger-ui-express": "^4.1.6",
"tar": "^6.1.8",
"typeorm": "^0.2.37" "typeorm": "^0.2.37"
}, },
"optionalDependencies": { "optionalDependencies": {
...@@ -58,7 +56,6 @@ ...@@ -58,7 +56,6 @@
"@types/mustache": "^4.1.2", "@types/mustache": "^4.1.2",
"@types/node": "^16.0.0", "@types/node": "^16.0.0",
"@types/supertest": "^2.0.11", "@types/supertest": "^2.0.11",
"@types/tar": "^4.0.5",
"@typescript-eslint/eslint-plugin": "^4.28.2", "@typescript-eslint/eslint-plugin": "^4.28.2",
"@typescript-eslint/parser": "^4.28.2", "@typescript-eslint/parser": "^4.28.2",
"eslint": "^7.30.0", "eslint": "^7.30.0",
......
import { import { Body, Controller, Delete, Param, Post, Put, UploadedFile, UseGuards, UseInterceptors, ValidationPipe } from '@nestjs/common';
Body, import { ApiBody, ApiConsumes, ApiCreatedResponse, ApiOkResponse, ApiOperation, ApiTags } from '@nestjs/swagger';
Controller,
Delete,
Param,
Post,
Put,
UploadedFile,
UseGuards,
UseInterceptors,
ValidationPipe,
} from '@nestjs/common';
import {
ApiBody,
ApiConsumes,
ApiCreatedResponse,
ApiOkResponse,
ApiOperation,
ApiTags,
} from '@nestjs/swagger';
import { MyCardAdminGuard } from '../my-card-admin.guard'; import { MyCardAdminGuard } from '../my-card-admin.guard';
import { FileInterceptor } from '@nestjs/platform-express'; import { FileInterceptor } from '@nestjs/platform-express';
import { FileUploadDto } from '../dto/FileUpload.dto'; import { FileUploadDto } from '../dto/FileUpload.dto';
...@@ -42,9 +24,10 @@ export class AdminController { ...@@ -42,9 +24,10 @@ export class AdminController {
type: FileUploadDto, type: FileUploadDto,
}) })
@ApiCreatedResponse({ type: BlankReturnMessageDto }) @ApiCreatedResponse({ type: BlankReturnMessageDto })
async migrate( async migrate(@UploadedFile('file') file: Express.Multer.File): Promise<BlankReturnMessageDto> {
@UploadedFile('file') file: Express.Multer.File, if (!file) {
): Promise<BlankReturnMessageDto> { throw new BlankReturnMessageDto(400, 'no file').toException();
}
const apps: AppsJson.App[] = JSON.parse(file.buffer.toString()); const apps: AppsJson.App[] = JSON.parse(file.buffer.toString());
return this.appService.migrateFromAppsJson(apps); return this.appService.migrateFromAppsJson(apps);
} }
...@@ -66,20 +49,14 @@ export class AdminController { ...@@ -66,20 +49,14 @@ export class AdminController {
@Post('app/:id/assign') @Post('app/:id/assign')
@ApiOperation({ summary: '设置 app 管理者' }) @ApiOperation({ summary: '设置 app 管理者' })
@ApiOkResponse({ type: BlankReturnMessageDto }) @ApiOkResponse({ type: BlankReturnMessageDto })
async assignApp( async assignApp(@Param('id') id: string, @Body(new ValidationPipe({ transform: true })) assignAppData: AssignAppDto) {
@Param('id') id: string,
@Body(new ValidationPipe({ transform: true })) assignAppData: AssignAppDto,
) {
return this.appService.assignApp(id, assignAppData.author); return this.appService.assignApp(id, assignAppData.author);
} }
@Post('app/:id/prefix') @Post('app/:id/prefix')
@ApiOperation({ summary: '设置 app 打包前缀' }) @ApiOperation({ summary: '设置 app 打包前缀' })
@ApiOkResponse({ type: BlankReturnMessageDto }) @ApiOkResponse({ type: BlankReturnMessageDto })
async setAppPrefix( async setAppPrefix(@Param('id') id: string, @Body(new ValidationPipe({ transform: true })) appPrefix: AppPrefixDto) {
@Param('id') id: string,
@Body(new ValidationPipe({ transform: true })) appPrefix: AppPrefixDto,
) {
return this.appService.setAppPrefix(id, appPrefix._prefix); return this.appService.setAppPrefix(id, appPrefix._prefix);
} }
} }
import { import {
BadRequestException,
Body, Body,
Controller, Controller,
Get, Get,
Param,
Post, Post,
Query, Query,
UploadedFile, UploadedFile,
...@@ -10,34 +12,20 @@ import { ...@@ -10,34 +12,20 @@ import {
ValidationPipe, ValidationPipe,
} from '@nestjs/common'; } from '@nestjs/common';
import { AppService } from './app.service'; import { AppService } from './app.service';
import { import { ApiBody, ApiConsumes, ApiCreatedResponse, ApiOkResponse, ApiOperation, ApiParam, ApiQuery } from '@nestjs/swagger';
ApiBody, import { BlankReturnMessageDto, GetAppReturnMessageDto, ReturnMessageDto, StringReturnMessageDto } from './dto/ReturnMessage.dto';
ApiConsumes,
ApiCreatedResponse,
ApiOkResponse,
ApiOperation,
ApiQuery,
} from '@nestjs/swagger';
import {
BlankReturnMessageDto,
GetAppReturnMessageDto,
ReturnMessageDto,
StringReturnMessageDto,
} from './dto/ReturnMessage.dto';
import { FetchMyCardUser, MyCardUser } from './utility/mycard-auth'; import { FetchMyCardUser, MyCardUser } from './utility/mycard-auth';
import { AppsJson } from './utility/apps-json-type'; import { AppsJson } from './utility/apps-json-type';
import { MyCardAppMaintainerGuard } from './my-card-app-maintainer.guard'; import { MyCardAppMaintainerGuard } from './my-card-app-maintainer.guard';
import { S3Service } from './s3/s3.service';
import { FileInterceptor } from '@nestjs/platform-express'; import { FileInterceptor } from '@nestjs/platform-express';
import { FileUploadDto } from './dto/FileUpload.dto'; import { FileUploadDto } from './dto/FileUpload.dto';
import AppClass = AppsJson.AppClass; import AppClass = AppsJson.AppClass;
import { AssetsS3Service } from './assets-s3/assets-s3.service';
import { MulterDirectEngine } from './packager/MulterStreamEngine';
@Controller('api') @Controller('api')
export class AppController { export class AppController {
constructor( constructor(private readonly appService: AppService, private readonly s3: AssetsS3Service) {}
private readonly appService: AppService,
private readonly s3: S3Service,
) {}
@Get('apps.json') @Get('apps.json')
getAppsJson() { getAppsJson() {
...@@ -61,10 +49,7 @@ export class AppController { ...@@ -61,10 +49,7 @@ export class AppController {
}) })
@ApiBody({ type: AppsJson.AppClass }) @ApiBody({ type: AppsJson.AppClass })
@ApiCreatedResponse({ type: BlankReturnMessageDto }) @ApiCreatedResponse({ type: BlankReturnMessageDto })
updateApp( updateApp(@FetchMyCardUser() user: MyCardUser, @Body(new ValidationPipe({ transform: true })) app: AppClass) {
@FetchMyCardUser() user: MyCardUser,
@Body(new ValidationPipe({ transform: true })) app: AppClass,
) {
return this.appService.updateApp(user, app.id, app); return this.appService.updateApp(user, app.id, app);
} }
...@@ -82,6 +67,9 @@ export class AppController { ...@@ -82,6 +67,9 @@ export class AppController {
@ApiCreatedResponse({ type: StringReturnMessageDto }) @ApiCreatedResponse({ type: StringReturnMessageDto })
@UseGuards(MyCardAppMaintainerGuard) @UseGuards(MyCardAppMaintainerGuard)
async uploadAssets(@UploadedFile() file: Express.Multer.File) { async uploadAssets(@UploadedFile() file: Express.Multer.File) {
if (!file) {
throw new BlankReturnMessageDto(400, 'no file').toException();
}
const res = await this.s3.uploadAssets(file); const res = await this.s3.uploadAssets(file);
if (res) { if (res) {
return new ReturnMessageDto(201, 'success', res); return new ReturnMessageDto(201, 'success', res);
...@@ -89,4 +77,35 @@ export class AppController { ...@@ -89,4 +77,35 @@ export class AppController {
throw new BlankReturnMessageDto(500, 'upload fail').toException(); throw new BlankReturnMessageDto(500, 'upload fail').toException();
} }
} }
@Post('build/:id/:platform/:locale/:version')
@ApiOperation({
summary: '打包文件',
description: '必须登录用户且必须是管理员或者拥有1个 app 才能上传',
})
@UseInterceptors(FileInterceptor('file', { storage: new MulterDirectEngine() }))
@ApiConsumes('multipart/form-data')
@ApiParam({ name: 'id', description: 'APP 的 id' })
@ApiParam({ name: 'platform', description: 'APP 的 版本号', enum: AppsJson.Platform })
@ApiParam({ name: 'locale', description: 'APP 的 版本号', enum: AppsJson.Locale })
@ApiParam({ name: 'version', description: 'APP 的 版本号' })
@ApiBody({
description: 'app 的 tar.gz 文件',
type: FileUploadDto,
})
@ApiCreatedResponse({ type: BlankReturnMessageDto })
async makeBuild(
@FetchMyCardUser() user: MyCardUser,
@UploadedFile() file: Express.Multer.File,
@Param('id') id: string,
@Param('platform') platform: AppsJson.Platform,
@Param('locale') locale: AppsJson.Locale,
@Param('version') version: string
) {
console.log(file.stream);
if (!file) {
throw new BlankReturnMessageDto(400, 'no file').toException();
}
return this.appService.makeBuild(user, file, id, platform, locale, version);
}
} }
...@@ -8,6 +8,8 @@ import { App } from './entities/App.entity'; ...@@ -8,6 +8,8 @@ import { App } from './entities/App.entity';
import { AppHistory } from './entities/AppHistory.entity'; import { AppHistory } from './entities/AppHistory.entity';
import { S3Service } from './s3/s3.service'; import { S3Service } from './s3/s3.service';
import { PackagerService } from './packager/packager.service'; import { PackagerService } from './packager/packager.service';
import { AssetsS3Service } from './assets-s3/assets-s3.service';
import { PackageS3Service } from './package-s3/package-s3.service';
const configModule = ConfigModule.forRoot(); const configModule = ConfigModule.forRoot();
...@@ -33,6 +35,6 @@ const configModule = ConfigModule.forRoot(); ...@@ -33,6 +35,6 @@ const configModule = ConfigModule.forRoot();
}), }),
], ],
controllers: [AppController, AdminController], controllers: [AppController, AdminController],
providers: [AppService, S3Service, PackagerService], providers: [AppService, PackagerService, AssetsS3Service, PackageS3Service],
}) })
export class AppModule {} export class AppModule {}
import { Connection, IsNull, Not } from 'typeorm'; import { Connection, IsNull, Not } from 'typeorm';
import { InjectConnection } from '@nestjs/typeorm'; import { InjectConnection } from '@nestjs/typeorm';
import { ConsoleLogger, Injectable } from '@nestjs/common'; import { ConsoleLogger, Injectable, Param, UploadedFile } from '@nestjs/common';
import { AppsJson } from './utility/apps-json-type'; import { AppsJson } from './utility/apps-json-type';
import { App } from './entities/App.entity'; import { App } from './entities/App.entity';
import { import { BlankReturnMessageDto, ReturnMessageDto } from './dto/ReturnMessage.dto';
BlankReturnMessageDto, import { FetchMyCardUser, MyCardUser } from './utility/mycard-auth';
ReturnMessageDto, import { PackagerService } from './packager/packager.service';
} from './dto/ReturnMessage.dto';
import { MyCardUser } from './utility/mycard-auth';
@Injectable() @Injectable()
export class AppService extends ConsoleLogger { export class AppService extends ConsoleLogger {
constructor( constructor(
@InjectConnection('app') @InjectConnection('app')
private db: Connection, private db: Connection,
private packager: PackagerService
) { ) {
super('app'); super('app');
} }
async getAppsJson() { async getAppsJson() {
return ( return (await this.db.getRepository(App).find({ where: { appData: Not(IsNull()), isDeleted: false } })).map((a) => a.appData);
await this.db
.getRepository(App)
.find({ where: { appData: Not(IsNull()), isDeleted: false } })
).map((a) => a.appData);
} }
private async updateResult<T>(f: () => Promise<T>, returnCode = 200) { private async updateResult<T>(f: () => Promise<T>, returnCode = 200) {
...@@ -39,14 +34,9 @@ export class AppService extends ConsoleLogger { ...@@ -39,14 +34,9 @@ export class AppService extends ConsoleLogger {
const targetApps: App[] = []; const targetApps: App[] = [];
for (const appData of apps) { for (const appData of apps) {
if (!appData.id) { if (!appData.id) {
throw new BlankReturnMessageDto( throw new BlankReturnMessageDto(400, `App ${appData.name} is invalid.`).toException();
400,
`App ${appData.name} is invalid.`,
).toException();
} }
const checkExistingApp = await this.db const checkExistingApp = await this.db.getRepository(App).findOne({ where: { id: appData.id }, relations: ['history'] });
.getRepository(App)
.findOne({ where: { id: appData.id }, relations: ['history'] });
//this.error('read'); //this.error('read');
if (checkExistingApp) { if (checkExistingApp) {
checkExistingApp.updateApp(appData); checkExistingApp.updateApp(appData);
...@@ -70,10 +60,7 @@ export class AppService extends ConsoleLogger { ...@@ -70,10 +60,7 @@ export class AppService extends ConsoleLogger {
if (!user) { if (!user) {
throw new BlankReturnMessageDto(401, 'Needs login').toException(); throw new BlankReturnMessageDto(401, 'Needs login').toException();
} }
const query = this.db const query = this.db.getRepository(App).createQueryBuilder('app').where('app.isDeleted = false');
.getRepository(App)
.createQueryBuilder('app')
.where('app.isDeleted = false');
if (!user.admin) { if (!user.admin) {
query.andWhere(':uid = ANY(app.author)', { uid: user.id }); query.andWhere(':uid = ANY(app.author)', { uid: user.id });
} }
...@@ -100,18 +87,13 @@ export class AppService extends ConsoleLogger { ...@@ -100,18 +87,13 @@ export class AppService extends ConsoleLogger {
} }
async createApp(id: string) { async createApp(id: string) {
let app = await this.db let app = await this.db.getRepository(App).findOne({ where: { id }, select: ['id', 'isDeleted'] });
.getRepository(App)
.findOne({ where: { id }, select: ['id', 'isDeleted'] });
if (!app) { if (!app) {
app = new App(); app = new App();
app.id = id; app.id = id;
} else { } else {
if (!app.isDeleted) { if (!app.isDeleted) {
throw new BlankReturnMessageDto( throw new BlankReturnMessageDto(404, 'App already exists').toException();
404,
'App already exists',
).toException();
} }
app.isDeleted = false; app.isDeleted = false;
} }
...@@ -119,18 +101,11 @@ export class AppService extends ConsoleLogger { ...@@ -119,18 +101,11 @@ export class AppService extends ConsoleLogger {
} }
async assignApp(id: string, author: number[]) { async assignApp(id: string, author: number[]) {
return this.updateResult( return this.updateResult(() => this.db.getRepository(App).update({ id }, { author }), 201);
() => this.db.getRepository(App).update({ id }, { author }),
201,
);
} }
async setAppPrefix(id: string, prefix: string) { async setAppPrefix(id: string, prefix: string) {
return this.updateResult( return this.updateResult(() => this.db.getRepository(App).update({ id }, { packagePrefix: prefix }), 201);
() =>
this.db.getRepository(App).update({ id }, { packagePrefix: prefix }),
201,
);
} }
async updateApp(user: MyCardUser, id: string, appData: AppsJson.App) { async updateApp(user: MyCardUser, id: string, appData: AppsJson.App) {
...@@ -156,9 +131,33 @@ export class AppService extends ConsoleLogger { ...@@ -156,9 +131,33 @@ export class AppService extends ConsoleLogger {
}, 201); }, 201);
} }
async makeBuild(
user: MyCardUser,
file: Express.Multer.File,
id: string,
platform: AppsJson.Platform,
locale: AppsJson.Locale,
version: string
) {
if (!user) {
throw new BlankReturnMessageDto(401, 'Needs login').toException();
}
this.log('Build: Checking app.');
const app = await this.db.getRepository(App).findOne({
where: { id },
select: ['id', 'packagePrefix', 'author'],
});
if (!app) {
throw new BlankReturnMessageDto(404, 'App not found').toException();
}
if (!app.isUserCanEditApp(user)) {
throw new BlankReturnMessageDto(403, 'Permission denied').toException();
}
const result = await this.packager.build(file.stream, app.packagePrefix);
return new ReturnMessageDto(201, 'success', result);
}
async deleteApp(id: string) { async deleteApp(id: string) {
return this.updateResult(() => return this.updateResult(() => this.db.getRepository(App).update({ id }, { isDeleted: true }));
this.db.getRepository(App).update({ id }, { isDeleted: true }),
);
} }
} }
import { Test, TestingModule } from '@nestjs/testing'; import { Test, TestingModule } from '@nestjs/testing';
import { S3Service } from './s3.service'; import { AssetsS3Service } from './assets-s3.service';
describe('S3Service', () => { describe('AssetsS3Service', () => {
let service: S3Service; let service: AssetsS3Service;
beforeEach(async () => { beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({ const module: TestingModule = await Test.createTestingModule({
providers: [S3Service], providers: [AssetsS3Service],
}).compile(); }).compile();
service = module.get<S3Service>(S3Service); service = module.get<AssetsS3Service>(AssetsS3Service);
}); });
it('should be defined', () => { it('should be defined', () => {
......
import { Injectable } from '@nestjs/common';
import { S3Service } from '../s3/s3.service';
import { ConfigService } from '@nestjs/config';
@Injectable()
export class AssetsS3Service extends S3Service {
constructor(config: ConfigService) {
super('ASSETS', config);
}
}
export class PackageResult {
constructor(public checksum: Record<string, string>, public packages: Record<string, string[]>) {}
}
import { Test, TestingModule } from '@nestjs/testing';
import { PackageS3Service } from './package-s3.service';
describe('PackageS3Service', () => {
let service: PackageS3Service;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [PackageS3Service],
}).compile();
service = module.get<PackageS3Service>(PackageS3Service);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
});
import { Injectable } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { S3Service } from '../s3/s3.service';
@Injectable()
export class PackageS3Service extends S3Service {
constructor(config: ConfigService) {
super('PACKAGE', config);
}
}
import multer from 'multer';
export class MulterDirectEngine implements multer.StorageEngine {
_handleFile(req: Express.Request, file: Express.Multer.File, callback: (error?: any, info?: Partial<Express.Multer.File>) => void) {
callback(null, { stream: file.stream });
}
_removeFile(req: Express.Request, file: Express.Multer.File, callback: (error: Error | null) => void) {
callback(null);
}
}
...@@ -2,32 +2,38 @@ import { ConsoleLogger, Injectable } from '@nestjs/common'; ...@@ -2,32 +2,38 @@ import { ConsoleLogger, Injectable } from '@nestjs/common';
import fs from 'fs'; import fs from 'fs';
import path from 'path'; import path from 'path';
import child_process from 'child_process'; import child_process from 'child_process';
import tar from 'tar';
import os from 'os'; import os from 'os';
import { S3Service } from '../s3/s3.service'; import { S3Service } from '../s3/s3.service';
import { PutObjectCommand } from '@aws-sdk/client-s3';
import util from 'util'; import util from 'util';
import { v4 as uuidv4 } from 'uuid'; import { v4 as uuidv4 } from 'uuid';
import readdirp from 'readdirp'; import readdirp from 'readdirp';
import _ from 'lodash'; import _ from 'lodash';
import { ConfigService } from '@nestjs/config';
import internal from 'stream';
import { PackageResult } from '../dto/PackageResult.dto';
@Injectable() @Injectable()
export class PackagerService extends ConsoleLogger { export class PackagerService extends ConsoleLogger {
// workingPath: string; bucket_max = 10 * 1024 ** 2;
// releasePath: string; bucket_enter = 1 * 1024 ** 2;
// downloadBaseUrl: string;
// queueIdMap = new Map<string, PQueue>();
constructor(private s3: S3Service) { constructor(private s3: S3Service, config: ConfigService) {
super('packager'); super('packager');
this.bucket_max = (parseInt(config.get('PACKAGE_BUCKET_MAX')) || 10) * 1024 ** 2;
this.bucket_enter = (parseInt(config.get('PACKAGE_BUCKET_ENTER')) || 1) * 1024 ** 2;
} }
async build(stream: fs.ReadStream): Promise<BuildResult> { async build(stream: internal.Readable, pathPrefix?: string): Promise<PackageResult> {
const bucket_max = 10 * 1024 ** 2; this.log(`Start packaging.`);
const bucket_enter = 1 * 1024 ** 2;
const root = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'mycard-console-')); const root = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'mycard-console-'));
await this.spawnAsync('tar', ['-zxvf', '-'], { cwd: root, stdio: [stream, 'inherit', 'inherit'] }); let extractRoot = root;
if (pathPrefix) {
extractRoot = path.join(root, pathPrefix);
await fs.promises.mkdir(extractRoot, { recursive: true });
}
await this.spawnAsync('tar', ['-zxvf', '-'], { cwd: extractRoot, stdio: [stream, 'inherit', 'inherit'] });
this.log(`Package extracted to ${extractRoot}.`);
const buckets: Record<string, [string[], number]> = {}; const buckets: Record<string, [string[], number]> = {};
const packages: Record<string, string[]> = {}; const packages: Record<string, string[]> = {};
...@@ -50,11 +56,11 @@ export class PackagerService extends ConsoleLogger { ...@@ -50,11 +56,11 @@ export class PackagerService extends ConsoleLogger {
// 散包 // 散包
for (const file of files) { for (const file of files) {
if (file.stats.size < bucket_enter) { if (file.stats.size < this.bucket_enter) {
const extname = path.extname(file.basename); const extname = path.extname(file.basename);
buckets[extname] ??= [[], 0]; buckets[extname] ??= [[], 0];
const bucket = buckets[extname]; const bucket = buckets[extname];
if (bucket[1] + file.stats.size >= bucket_max) { if (bucket[1] + file.stats.size >= this.bucket_max) {
const archive = `${uuidv4()}.tar.gz`; const archive = `${uuidv4()}.tar.gz`;
packages[archive] = bucket[0]; packages[archive] = bucket[0];
promises.push(this.archive(archive, root, bucket[0])); promises.push(this.archive(archive, root, bucket[0]));
...@@ -81,8 +87,9 @@ export class PackagerService extends ConsoleLogger { ...@@ -81,8 +87,9 @@ export class PackagerService extends ConsoleLogger {
// TODO: 更新包 // TODO: 更新包
const [checksum] = await Promise.all(promises); // 这个 await 过后,checksum 和 打包上传都已经跑完了 const [checksum] = await Promise.all(promises); // 这个 await 过后,checksum 和 打包上传都已经跑完了
console.log(checksum, packages); this.log({ checksum, packages });
return { checksum, packages }; await fs.promises.unlink(root);
return new PackageResult(checksum, packages);
} }
async checksum(root: string, directories: string[], files: string[]) { async checksum(root: string, directories: string[], files: string[]) {
...@@ -99,13 +106,7 @@ export class PackagerService extends ConsoleLogger { ...@@ -99,13 +106,7 @@ export class PackagerService extends ConsoleLogger {
cwd: root, cwd: root,
stdio: ['ignore', 'pipe', 'inherit'], stdio: ['ignore', 'pipe', 'inherit'],
}); });
return this.s3.s3.send( return this.s3.uploadFile(archive, child.stdout, 'application/tar+gzip');
new PutObjectCommand({
Bucket: this.s3.bucket,
Key: archive,
Body: child.stdout,
})
);
} }
private spawnAsync(command: string, args: string[], options: child_process.SpawnOptions) { private spawnAsync(command: string, args: string[], options: child_process.SpawnOptions) {
...@@ -124,8 +125,3 @@ export class PackagerService extends ConsoleLogger { ...@@ -124,8 +125,3 @@ export class PackagerService extends ConsoleLogger {
}); });
} }
} }
export interface BuildResult {
checksum: Record<string, string>;
packages: Record<string, string[]>;
}
import { ConsoleLogger, Injectable } from '@nestjs/common'; import { ConsoleLogger, Injectable } from '@nestjs/common';
import { ConfigService } from '@nestjs/config'; import { ConfigService } from '@nestjs/config';
import { import { ListObjectsCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
ListObjectsCommand,
PutObjectCommand,
S3Client,
} from '@aws-sdk/client-s3';
import { createHash } from 'crypto'; import { createHash } from 'crypto';
import internal from 'stream';
@Injectable()
export class S3Service extends ConsoleLogger { export class S3Service extends ConsoleLogger {
bucket: string; private readonly bucket: string;
prefix: string; private readonly prefix: string;
cdnUrl: string; private readonly cdnUrl: string;
s3: S3Client; private readonly s3: S3Client;
constructor(config: ConfigService) {
super('s3'); private getConfig(field: string) {
this.bucket = config.get('S3_BUCKET'); return this.config.get(`${this.servicePrefix}_${field}`) || this.config.get(field);
this.prefix = config.get('S3_PREFIX'); }
this.cdnUrl =
config.get('S3_CDN_URL') || constructor(private servicePrefix: string, private config: ConfigService) {
`${config.get('S3_ENDPOINT')}/${this.bucket}/${this.prefix}`; super(`${servicePrefix} s3`);
this.bucket = this.getConfig('S3_BUCKET');
this.prefix = this.getConfig('S3_PREFIX');
this.cdnUrl = this.getConfig('S3_CDN_URL') || `${this.getConfig('S3_ENDPOINT')}/${this.bucket}${this.prefix ? `/${this.prefix}` : ''}`;
this.s3 = new S3Client({ this.s3 = new S3Client({
credentials: { credentials: {
accessKeyId: config.get('S3_KEY'), accessKeyId: this.getConfig('S3_KEY'),
secretAccessKey: config.get('S3_SECRET'), secretAccessKey: this.getConfig('S3_SECRET'),
}, },
region: config.get('S3_REGION') || 'us-west-1', region: this.getConfig('S3_REGION') || 'us-west-1',
endpoint: config.get('S3_ENDPOINT'), endpoint: this.getConfig('S3_ENDPOINT'),
forcePathStyle: true, forcePathStyle: true,
}); });
} }
private async listObjects(path: string) { async listObjects(path: string) {
const command = new ListObjectsCommand({ const command = new ListObjectsCommand({
Bucket: this.bucket, Bucket: this.bucket,
Prefix: path, Prefix: path,
...@@ -39,37 +38,39 @@ export class S3Service extends ConsoleLogger { ...@@ -39,37 +38,39 @@ export class S3Service extends ConsoleLogger {
return this.s3.send(command); return this.s3.send(command);
} }
private getPathWithPrefix(filename: string) {
return this.prefix ? `${this.prefix}/${filename}` : filename;
}
async uploadAssets(file: Express.Multer.File) { async uploadAssets(file: Express.Multer.File) {
const fileSuffix = file.originalname.split('.').pop(); const fileSuffix = file.originalname.split('.').pop();
const hash = createHash('sha512').update(file.buffer).digest('hex'); const hash = createHash('sha512').update(file.buffer).digest('hex');
const filename = `${hash}.${fileSuffix}`; const filename = `${hash}.${fileSuffix}`;
const path = `${this.prefix}/${filename}`; const path = this.getPathWithPrefix(filename);
const downloadUrl = `${this.cdnUrl}/${filename}`; const downloadUrl = `${this.cdnUrl}/${filename}`;
const checkExisting = await this.listObjects(path); const checkExisting = await this.listObjects(path);
try { try {
if ( if (checkExisting.Contents && checkExisting.Contents.some((obj) => obj.Key === path)) {
checkExisting.Contents &&
checkExisting.Contents.some((obj) => obj.Key === path)
) {
// already uploaded // already uploaded
return downloadUrl;
} else { } else {
await this.uploadFile(path, file.buffer, file.mimetype); return this.uploadFile(filename, file.buffer, file.mimetype);
} }
return downloadUrl;
} catch (e) { } catch (e) {
this.error(`Failed to assign upload of file ${path}: ${e.toString()}`); this.error(`Failed to upload assets ${path}: ${e.toString()}`);
return null; return null;
} }
} }
async uploadFile(path: string, buffer: Buffer, mime?: string) { async uploadFile(path: string, content: Buffer | internal.Readable, mime?: string) {
return this.s3.send( await this.s3.send(
new PutObjectCommand({ new PutObjectCommand({
Bucket: this.bucket, Bucket: this.bucket,
Key: path, Key: this.getPathWithPrefix(path),
Body: buffer, Body: content,
ContentType: mime, ContentType: mime,
}), })
); );
return `${this.cdnUrl}/${path}`;
} }
} }
...@@ -2,6 +2,7 @@ import { IsNotEmpty } from 'class-validator'; ...@@ -2,6 +2,7 @@ import { IsNotEmpty } from 'class-validator';
export namespace AppsJson { export namespace AppsJson {
export enum Locale { export enum Locale {
generic = 'generic',
zh_CN = 'zh-CN', zh_CN = 'zh-CN',
en_US = 'en-US', en_US = 'en-US',
ja_JP = 'ja-JP', ja_JP = 'ja-JP',
...@@ -11,6 +12,7 @@ export namespace AppsJson { ...@@ -11,6 +12,7 @@ export namespace AppsJson {
zh_TW = 'zh-TW', zh_TW = 'zh-TW',
} }
export enum Platform { export enum Platform {
generic = 'generic',
Linux = 'linux', Linux = 'linux',
macOS = 'darwin', macOS = 'darwin',
Windows = 'win32', Windows = 'win32',
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment