Commit 119e0e6b authored by nanahira's avatar nanahira

reuse package

parent 664c0d45
export class PackageResult {
constructor(public checksum: Record<string, string>, public packages: Record<string, string[]>) {}
constructor(public checksum: Record<string, string>, public packages: Record<string, string[]>, public fullPackage: string) {}
}
......@@ -13,6 +13,12 @@ import readdirp from 'readdirp';
import { v4 as uuidv4 } from 'uuid';
import { ConsoleLogger, Injectable } from '@nestjs/common';
import { createHash } from 'crypto';
export interface FileWithHash {
file: readdirp.EntryInfo;
hash: string;
}
@Injectable()
export class PackagerService extends ConsoleLogger {
......@@ -38,7 +44,7 @@ export class PackagerService extends ConsoleLogger {
this.log(`Package extracted to ${extractRoot}.`);
const packages: Record<string, string[]> = {};
const packagesSequence: string[][] = [];
const entries = await readdirp.promise(root, { alwaysStat: true, type: 'files_directories' });
const [directories, files] = _.partition(entries, (item) => item.stats.isDirectory());
......@@ -48,41 +54,39 @@ export class PackagerService extends ConsoleLogger {
directories.map((d) => d.path),
files.map((f) => f.path)
);
const promises: Promise<[string, string]>[] = [];
const promises: Promise<string>[] = [];
const filesWithHash: FileWithHash[] = files.map((f) => ({ file: f, hash: checksum[f.path] }));
// 整包
const archive = `${uuidv4()}.tar.gz`;
packages[archive] = files.map((f) => f.path);
promises.push(this.archive(archive, root, tarballRoot, await fs.promises.readdir(root)));
packagesSequence.push(files.map((f) => f.path));
promises.push(this.archive(root, tarballRoot, filesWithHash, await fs.promises.readdir(root)));
// 散包
const buckets: Record<string, [string[], number]> = {};
for (const file of files) {
if (file.stats.size < this.bucket_enter) {
const extname = path.extname(file.basename);
const buckets: Record<string, [FileWithHash[], number]> = {};
for (const file of filesWithHash) {
if (file.file.stats.size < this.bucket_enter) {
const extname = path.extname(file.file.basename);
buckets[extname] ??= [[], 0];
const bucket = buckets[extname];
if (bucket[1] + file.stats.size >= this.bucket_max) {
const archive = `${uuidv4()}.tar.gz`;
packages[archive] = bucket[0];
promises.push(this.archive(archive, root, tarballRoot, bucket[0], checksum));
if (bucket[1] + file.file.stats.size >= this.bucket_max) {
packagesSequence.push(bucket[0].map((f) => f.file.path));
promises.push(this.archive(root, tarballRoot, bucket[0]));
bucket[0] = [];
bucket[1] = 0;
} else {
bucket[0].push(file.path);
bucket[1] += file.stats.size;
bucket[0].push(file);
bucket[1] += file.file.stats.size;
}
} else {
const archive = `${uuidv4()}.tar.gz`;
packages[archive] = [file.path];
promises.push(this.archive(archive, root, tarballRoot, [file.path], checksum));
packagesSequence.push([file.file.path]);
promises.push(this.archive(root, tarballRoot, [file]));
}
}
for (const bucket of Object.values(buckets)) {
if (bucket[0].length) {
const archive = `${uuidv4()}.tar.gz`;
packages[archive] = bucket[0];
promises.push(this.archive(archive, root, tarballRoot, bucket[0], checksum));
packagesSequence.push(bucket[0].map((f) => f.file.path));
promises.push(this.archive(root, tarballRoot, bucket[0]));
}
}
......@@ -90,16 +94,15 @@ export class PackagerService extends ConsoleLogger {
const gotPackages = await Promise.all(promises); // 这个 await 过后,checksum 和 打包上传都已经跑完了
for (const differentPackages of gotPackages.filter((p) => p[0] !== p[1])) {
const [originalPackage, gotPackage] = differentPackages;
packages[gotPackage] = packages[originalPackage];
delete packages[originalPackage];
const packages: Record<string, string[]> = {};
for (let i = 0; i < packagesSequence.length; ++i) {
packages[gotPackages[i]] = packagesSequence[i];
}
this.log({ checksum, packages });
await fs.promises.rm(root, { recursive: true });
await fs.promises.rm(tarballRoot, { recursive: true });
return new PackageResult(checksum, packages);
return new PackageResult(checksum, packages, gotPackages[0]);
}
async checksum(root: string, directories: string[], files: string[]): Promise<Record<string, string>> {
......@@ -111,20 +114,22 @@ export class PackagerService extends ConsoleLogger {
]);
}
async archive(
archive: string = `${uuidv4()}.tar.gz`,
root: string,
tarballRoot: string,
files: string[],
checksum: Record<string, string> = {}
): Promise<[string, string]> {
async archive(root: string, tarballRoot: string, files: FileWithHash[], altFiles?: string[]): Promise<string> {
const archive =
createHash('sha512')
.update(files.map((f) => `${f.file.path}${f.hash}`).join(''))
.digest('hex') + '.tar.gz';
if (await this.s3.fileExists(archive)) {
return archive;
}
const archivePath = path.join(tarballRoot, archive);
await this.spawnAsync('tar', ['-zcvf', archivePath].concat(files), {
this.log(`Packaging archive ${archivePath} with ${files.length} files.`);
await this.spawnAsync('tar', ['-zcvf', archivePath].concat(altFiles || files.map((f) => f.file.path)), {
cwd: root,
});
const fileSize = (await fs.promises.stat(archivePath)).size;
await this.s3.uploadFile(archive, fs.createReadStream(archivePath), { ContentType: 'application/tar+gzip', ContentLength: fileSize });
return [archive, archive];
return archive;
}
private spawnAsync(command: string, args: string[], options: child_process.SpawnOptions) {
......
......@@ -3,6 +3,7 @@ import { ConfigService } from '@nestjs/config';
import { ListObjectsCommand, PutObjectCommand, PutObjectCommandInput, S3Client } from '@aws-sdk/client-s3';
import { createHash } from 'crypto';
import internal from 'stream';
import Path from 'path';
export class S3Service extends ConsoleLogger {
private readonly bucket: string;
......@@ -33,28 +34,32 @@ export class S3Service extends ConsoleLogger {
async listObjects(path: string) {
const command = new ListObjectsCommand({
Bucket: this.bucket,
Prefix: path,
Prefix: this.getPathWithPrefix(path),
});
return this.s3.send(command);
}
async fileExists(path: string) {
const objects = await this.listObjects(path);
// this.log(objects);
return objects.Contents && objects.Contents.some((obj) => obj.Key === this.getPathWithPrefix(path));
}
private getPathWithPrefix(filename: string) {
return this.prefix ? `${this.prefix}/${filename}` : filename;
}
async uploadAssets(file: Express.Multer.File) {
const fileSuffix = file.originalname.split('.').pop();
const fileSuffix = Path.extname(file.originalname);
const hash = createHash('sha512').update(file.buffer).digest('hex');
const filename = `${hash}.${fileSuffix}`;
const path = this.getPathWithPrefix(filename);
const downloadUrl = `${this.cdnUrl}/${filename}`;
const checkExisting = await this.listObjects(path);
const path = fileSuffix ? `${hash}.${fileSuffix}` : hash;
const downloadUrl = `${this.cdnUrl}/${path}`;
try {
if (checkExisting.Contents && checkExisting.Contents.some((obj) => obj.Key === path)) {
if (await this.fileExists(path)) {
// already uploaded
return downloadUrl;
} else {
return this.uploadFile(filename, file.buffer, { ContentType: file.mimetype });
return this.uploadFile(path, file.buffer, { ContentType: file.mimetype });
}
} catch (e) {
this.error(`Failed to upload assets ${path}: ${e.toString()}`);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment