Commit a02cf620 authored by nanahira's avatar nanahira

so far

parent f2ab8518
This diff is collapsed.
...@@ -33,12 +33,15 @@ ...@@ -33,12 +33,15 @@
"class-validator": "^0.13.1", "class-validator": "^0.13.1",
"lodash": "^4.17.21", "lodash": "^4.17.21",
"moment": "^2.29.1", "moment": "^2.29.1",
"mustache": "^4.2.0",
"p-queue": "6.6.2",
"pg": "^8.7.1", "pg": "^8.7.1",
"pg-native": "^3.0.0", "pg-native": "^3.0.0",
"reflect-metadata": "^0.1.13", "reflect-metadata": "^0.1.13",
"rimraf": "^3.0.2", "rimraf": "^3.0.2",
"rxjs": "^7.2.0", "rxjs": "^7.2.0",
"swagger-ui-express": "^4.1.6", "swagger-ui-express": "^4.1.6",
"tar": "^6.1.8",
"typeorm": "^0.2.37" "typeorm": "^0.2.37"
}, },
"devDependencies": { "devDependencies": {
...@@ -49,8 +52,10 @@ ...@@ -49,8 +52,10 @@
"@types/jest": "^26.0.24", "@types/jest": "^26.0.24",
"@types/lodash": "^4.14.172", "@types/lodash": "^4.14.172",
"@types/multer": "^1.4.7", "@types/multer": "^1.4.7",
"@types/mustache": "^4.1.2",
"@types/node": "^16.0.0", "@types/node": "^16.0.0",
"@types/supertest": "^2.0.11", "@types/supertest": "^2.0.11",
"@types/tar": "^4.0.5",
"@typescript-eslint/eslint-plugin": "^4.28.2", "@typescript-eslint/eslint-plugin": "^4.28.2",
"@typescript-eslint/parser": "^4.28.2", "@typescript-eslint/parser": "^4.28.2",
"eslint": "^7.30.0", "eslint": "^7.30.0",
......
...@@ -25,6 +25,7 @@ import { AppsJson } from '../utility/apps-json-type'; ...@@ -25,6 +25,7 @@ import { AppsJson } from '../utility/apps-json-type';
import { AppService } from '../app.service'; import { AppService } from '../app.service';
import { BlankReturnMessageDto } from '../dto/ReturnMessage.dto'; import { BlankReturnMessageDto } from '../dto/ReturnMessage.dto';
import { AssignAppDto } from '../dto/AssignApp.dto'; import { AssignAppDto } from '../dto/AssignApp.dto';
import { AppPrefixDto } from '../dto/AppPrefix.dto';
@Controller('api/admin') @Controller('api/admin')
@ApiTags('admin') @ApiTags('admin')
...@@ -56,14 +57,14 @@ export class AdminController { ...@@ -56,14 +57,14 @@ export class AdminController {
} }
@Delete('app/:id') @Delete('app/:id')
@ApiOperation({ summary: '创建 app' }) @ApiOperation({ summary: '删除 app' })
@ApiOkResponse({ type: BlankReturnMessageDto }) @ApiOkResponse({ type: BlankReturnMessageDto })
async deleteApp(@Param('id') id: string) { async deleteApp(@Param('id') id: string) {
return this.appService.deleteApp(id); return this.appService.deleteApp(id);
} }
@Post('app/:id/assign') @Post('app/:id/assign')
@ApiOperation({ summary: '创建 app' }) @ApiOperation({ summary: '设置 app 管理者' })
@ApiOkResponse({ type: BlankReturnMessageDto }) @ApiOkResponse({ type: BlankReturnMessageDto })
async assignApp( async assignApp(
@Param('id') id: string, @Param('id') id: string,
...@@ -71,4 +72,14 @@ export class AdminController { ...@@ -71,4 +72,14 @@ export class AdminController {
) { ) {
return this.appService.assignApp(id, assignAppData.author); return this.appService.assignApp(id, assignAppData.author);
} }
@Post('app/:id/prefix')
@ApiOperation({ summary: '设置 app 打包前缀' })
@ApiOkResponse({ type: BlankReturnMessageDto })
async setAppPrefix(
@Param('id') id: string,
@Body(new ValidationPipe({ transform: true })) appPrefix: AppPrefixDto,
) {
return this.appService.setAppPrefix(id, appPrefix._prefix);
}
} }
...@@ -7,6 +7,7 @@ import { ConfigModule, ConfigService } from '@nestjs/config'; ...@@ -7,6 +7,7 @@ import { ConfigModule, ConfigService } from '@nestjs/config';
import { App } from './entities/App.entity'; import { App } from './entities/App.entity';
import { AppHistory } from './entities/AppHistory.entity'; import { AppHistory } from './entities/AppHistory.entity';
import { S3Service } from './s3/s3.service'; import { S3Service } from './s3/s3.service';
import { PackagerService } from './packager/packager.service';
const configModule = ConfigModule.forRoot(); const configModule = ConfigModule.forRoot();
...@@ -32,6 +33,6 @@ const configModule = ConfigModule.forRoot(); ...@@ -32,6 +33,6 @@ const configModule = ConfigModule.forRoot();
}), }),
], ],
controllers: [AppController, AdminController], controllers: [AppController, AdminController],
providers: [AppService, S3Service], providers: [AppService, S3Service, PackagerService],
}) })
export class AppModule {} export class AppModule {}
...@@ -125,6 +125,14 @@ export class AppService extends ConsoleLogger { ...@@ -125,6 +125,14 @@ export class AppService extends ConsoleLogger {
); );
} }
async setAppPrefix(id: string, prefix: string) {
return this.updateResult(
() =>
this.db.getRepository(App).update({ id }, { packagePrefix: prefix }),
201,
);
}
async updateApp(user: MyCardUser, id: string, appData: AppsJson.App) { async updateApp(user: MyCardUser, id: string, appData: AppsJson.App) {
if (!user) { if (!user) {
throw new BlankReturnMessageDto(401, 'Needs login').toException(); throw new BlankReturnMessageDto(401, 'Needs login').toException();
......
import { ApiProperty } from '@nestjs/swagger';
export class AppPrefixDto {
@ApiProperty({ description: '打包起始路径' })
prefix: string;
get _prefix() {
return this.prefix && this.prefix.length ? this.prefix : null;
}
}
...@@ -4,6 +4,7 @@ import { MyCardUser } from '../utility/mycard-auth'; ...@@ -4,6 +4,7 @@ import { MyCardUser } from '../utility/mycard-auth';
import { AppBase } from './AppBase.entity'; import { AppBase } from './AppBase.entity';
import { AppHistory } from './AppHistory.entity'; import { AppHistory } from './AppHistory.entity';
import moment from 'moment'; import moment from 'moment';
import { join } from 'path';
@Entity() @Entity()
export class App extends AppBase { export class App extends AppBase {
...@@ -17,6 +18,17 @@ export class App extends AppBase { ...@@ -17,6 +18,17 @@ export class App extends AppBase {
@Column({ nullable: true, select: false }) @Column({ nullable: true, select: false })
isDeleted: boolean; isDeleted: boolean;
@Column('varchar', { length: 128, nullable: true })
packagePrefix: string;
get packageFullPath() {
if (this.packagePrefix && this.packagePrefix.length) {
return join(this.id, this.packagePrefix);
} else {
return this.id;
}
}
isUserCanEditApp(u: MyCardUser) { isUserCanEditApp(u: MyCardUser) {
return u.admin || (this.author && this.author.includes(u.id)); return u.admin || (this.author && this.author.includes(u.id));
} }
......
import { MyCardAppMaintainerGuard } from './my-card-app-maintainer.guard';
describe('MyCardAppMaintainerGuard', () => {
it('should be defined', () => {
expect(new MyCardAppMaintainerGuard()).toBeDefined();
});
});
import { CanActivate, ExecutionContext, Injectable } from '@nestjs/common';
import { AppService } from './app.service';
import { getUserFromContext } from './utility/mycard-auth';
import { BlankReturnMessageDto } from './dto/ReturnMessage.dto';
@Injectable()
export class MyCardAppMaintainerGuard implements CanActivate {
constructor(private readonly appService: AppService) {}
async canActivate(context: ExecutionContext): Promise<boolean> {
const user = await getUserFromContext(context);
if (!user) {
throw new BlankReturnMessageDto(401, 'Invalid user').toException();
}
if (!(await this.appService.isUserCanMaintainApp(user))) {
throw new BlankReturnMessageDto(403, 'Permission denied').toException();
}
return true;
}
}
import { Test, TestingModule } from '@nestjs/testing';
import { PackagerService } from './packager.service';
describe('PackagerService', () => {
let service: PackagerService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [PackagerService],
}).compile();
service = module.get<PackagerService>(PackagerService);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
});
import { ConsoleLogger, Injectable } from '@nestjs/common';
import * as fs from 'fs';
import * as path from 'path';
import * as crypto from 'crypto';
import * as child_process from 'child_process';
import * as Mustache from 'mustache';
import tar from 'tar';
import { App } from '../entities/App.entity';
import PQueue from 'p-queue';
// eslint-disable-next-line @typescript-eslint/no-empty-function
function nothing() {}
@Injectable()
export class PackagerService extends ConsoleLogger {
workingPath: string;
releasePath: string;
downloadBaseUrl: string;
queueIdMap = new Map<string, PQueue>();
getQueue(id: string) {
if (!this.queueIdMap.has(id)) {
this.queueIdMap.set(id, new PQueue({ concurrency: 1 }));
}
return this.queueIdMap.get(id);
}
constructor() {
super('packager');
}
private async readdirRecursive(_path: string): Promise<string[]> {
const files = await fs.promises.readdir(_path, { encoding: 'utf-8' });
let result = files;
for (const file of files) {
const child = path.join(_path, file);
const stat = await fs.promises.stat(child);
if (stat.isDirectory()) {
result = result.concat(
(await this.readdirRecursive(child)).map((_file) =>
path.join(file, _file),
),
);
}
}
return result;
}
private caculateSHA256(file: string): Promise<string> {
return new Promise((resolve, reject) => {
const input = fs.createReadStream(file);
const hash = crypto.createHash('sha256');
hash.on('error', (error: Error) => {
reject(error);
});
input.on('error', (error: Error) => {
reject(error);
});
hash.on('readable', () => {
const data = hash.read();
if (data) {
resolve((<Buffer>data).toString('hex'));
}
});
input.pipe(hash);
});
}
private spawnAsync(
command: string,
args: string[],
options: child_process.SpawnOptions,
) {
return new Promise<void>((resolve, reject) => {
const child = child_process.spawn(command, args, options);
child.on('exit', (code) => {
if (code == 0) {
resolve();
} else {
reject(code);
}
});
child.on('error', (error) => {
reject(error);
});
});
}
private archive(
archive: string,
files: string[],
directory: string,
): Promise<void> {
return tar.c({ gzip: true, C: directory, file: archive }, files);
}
private async createDirectoryIfNotExists(directory: string) {
try {
await fs.promises.access(directory);
} catch (e) {
await fs.promises.mkdir(directory, { recursive: true });
}
}
private async unarchiveStream(stream: fs.ReadStream, directory: string) {
await this.createDirectoryIfNotExists(directory);
const unarchiveProcessStream = stream.pipe(tar.x({ C: directory }));
return new Promise<void>((resolve, reject) => {
unarchiveProcessStream.on('finish', resolve);
unarchiveProcessStream.on('error', reject);
});
}
private async packageProcess(app: App, stream: fs.ReadStream) {
const unarchiveDestination = path.join(
this.workingPath,
app.packageFullPath,
);
const packageWorkingRoot = path.join(this.workingPath, app.id);
}
async package(app: App, stream: fs.ReadStream) {
const queue = this.getQueue();
}
}
import { Test, TestingModule } from '@nestjs/testing';
import { S3Service } from './s3.service';
describe('S3Service', () => {
let service: S3Service;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [S3Service],
}).compile();
service = module.get<S3Service>(S3Service);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
});
import { ConsoleLogger, Injectable } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import {
ListObjectsCommand,
PutObjectCommand,
S3Client,
} from '@aws-sdk/client-s3';
import { createHash } from 'crypto';
@Injectable()
export class S3Service extends ConsoleLogger {
bucket: string;
prefix: string;
cdnUrl: string;
s3: S3Client;
constructor(config: ConfigService) {
super('s3');
this.bucket = config.get('S3_BUCKET');
this.prefix = config.get('S3_PREFIX');
this.cdnUrl =
config.get('S3_CDN_URL') ||
`${config.get('S3_ENDPOINT')}/${this.bucket}/${this.prefix}`;
this.s3 = new S3Client({
credentials: {
accessKeyId: config.get('S3_KEY'),
secretAccessKey: config.get('S3_SECRET'),
},
region: config.get('S3_REGION') || 'us-west-1',
endpoint: config.get('S3_ENDPOINT'),
forcePathStyle: true,
});
}
private async listObjects(path: string) {
const command = new ListObjectsCommand({
Bucket: this.bucket,
Prefix: path,
});
return this.s3.send(command);
}
async uploadAssets(file: Express.Multer.File) {
const fileSuffix = file.originalname.split('.').pop();
const hash = createHash('sha512').update(file.buffer).digest('hex');
const filename = `${hash}.${fileSuffix}`;
const path = `${this.prefix}/${filename}`;
const downloadUrl = `${this.cdnUrl}/${filename}`;
const checkExisting = await this.listObjects(path);
try {
if (
checkExisting.Contents &&
checkExisting.Contents.some((obj) => obj.Key === path)
) {
// already uploaded
} else {
await this.uploadFile(path, file.buffer, file.mimetype);
}
return downloadUrl;
} catch (e) {
this.error(`Failed to assign upload of file ${path}: ${e.toString()}`);
return null;
}
}
async uploadFile(path: string, buffer: Buffer, mime?: string) {
return this.s3.send(
new PutObjectCommand({
Bucket: this.bucket,
Key: path,
Body: buffer,
ContentType: mime,
}),
);
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment