Commit c62b2392 authored by nanahira's avatar nanahira

dont put check hash in queue

parent 46c72c45
Pipeline #15368 passed with stages
in 4 minutes and 47 seconds
......@@ -209,9 +209,7 @@ export class PackagerService extends ConsoleLogger {
}
async archive(root: string, archiveTask: ArchiveTask): Promise<Archive> {
return this.redlock.using([`archive:${archiveTask.path}`], 5000, async () =>
this.archiveQueue.add(() => this.archiveProcess(root, archiveTask))
);
return this.redlock.using([`archive:${archiveTask.path}`], 5000, async () => this.archiveProcess(root, archiveTask));
}
private archiveQueue = new PQueue({ concurrency: parseInt(process.env.PACKAGE_COCURRENCY) || os.cpus().length });
......@@ -229,42 +227,44 @@ export class PackagerService extends ConsoleLogger {
return archive;
}
}
const files = archiveTask.filePaths;
this.log(`Packaging archive ${archiveName} with ${archiveTask.exactFilePaths.length} files.`);
return this.archiveQueue.add(async () => {
const files = archiveTask.filePaths;
this.log(`Packaging archive ${archiveName} with ${archiveTask.exactFilePaths.length} files.`);
const child = child_process.spawn('tar', ['--zstd', '-cf', '-'].concat(files), {
cwd: root,
});
const childPromise = new Promise<void>((resolve, reject) => {
child.on('exit', (code) => {
if (code == 0) {
resolve();
} else {
reject(code);
}
const child = child_process.spawn('tar', ['--zstd', '-cf', '-'].concat(files), {
cwd: root,
});
child.on('error', (error) => {
reject(error);
const childPromise = new Promise<void>((resolve, reject) => {
child.on('exit', (code) => {
if (code == 0) {
resolve();
} else {
reject(code);
}
});
child.on('error', (error) => {
reject(error);
});
});
});
const hashObject = createHash('sha256');
child.stdout.on('data', (chunk) => {
hashObject.update(chunk);
});
/* if (existing) {
await childPromise;
archive.hash = hashObject.digest('hex');
return archive;
}*/
const uploadPromise = this.s3.uploadStream(archiveName, child.stdout, {
ContentType: 'application/tar+zstd',
});
const [, { object }] = await Promise.all([childPromise, uploadPromise]);
archive.hash = hashObject.digest('hex');
await this.redis.set(`hash:${archive.path}`, archive.hash, 'EX', 60 * 60 * 24);
archive.size = object.Size;
const hashObject = createHash('sha256');
child.stdout.on('data', (chunk) => {
hashObject.update(chunk);
});
/* if (existing) {
await childPromise;
archive.hash = hashObject.digest('hex');
return archive;
}*/
const uploadPromise = this.s3.uploadStream(archiveName, child.stdout, {
ContentType: 'application/tar+zstd',
});
const [, { object }] = await Promise.all([childPromise, uploadPromise]);
archive.hash = hashObject.digest('hex');
await this.redis.set(`hash:${archive.path}`, archive.hash, 'EX', 60 * 60 * 24);
archive.size = object.Size;
return archive;
return archive;
});
}
private spawnAsync(command: string, args: string[], options: child_process.SpawnOptions, stdinStream?: NodeJS.ReadableStream) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment