Commit 9bccb1d0 authored by nanahira's avatar nanahira

fix

parent c1fdf5ca
...@@ -76,6 +76,10 @@ class Bucket { ...@@ -76,6 +76,10 @@ class Bucket {
this.files.push(file); this.files.push(file);
this.size += file.file.stats.size; this.size += file.file.stats.size;
} }
canFit(file: FileWithHash, limit: number): boolean {
return this.size + file.file.stats.size <= limit;
}
} }
@Injectable() @Injectable()
...@@ -145,11 +149,37 @@ export class PackagerService extends ConsoleLogger { ...@@ -145,11 +149,37 @@ export class PackagerService extends ConsoleLogger {
// 整包 // 整包
new ArchiveTask(ArchiveType.Full, filesWithHash, await fs.promises.readdir(root)).addToTask(archiveTasks); new ArchiveTask(ArchiveType.Full, filesWithHash, await fs.promises.readdir(root)).addToTask(archiveTasks);
// 更新包 if (files.length > 1) {
for (const lastChecksum of lastBuildChecksums) { // 整包(512M 一包)
const changedFiles = filesWithHash.filter((f) => !lastChecksum[f.file.path] || lastChecksum[f.file.path] !== f.hash); const cloudflareLimit = 450 * 1024 ** 2; // Cloudflare 的限制是 512M,留点余地
if (changedFiles.length) { const totalFileSize = filesWithHash.reduce((sum, f) => sum + f.file.stats.size, 0);
new ArchiveTask(ArchiveType.Update, changedFiles).addToTask(archiveTasks); if (totalFileSize > cloudflareLimit) {
this.log(`Total file size is ${totalFileSize}, which is larger than ${cloudflareLimit}. Will create fulls archives.`);
const sortedFiles = _.sortBy(filesWithHash, (f) => -f.file.stats.size); // 按照文件大小降序排序,优先打包大文件
// create multiple fulls archives, with each archive containing files up to cloudflareLimit
const buckets: Bucket[] = [];
for (const file of sortedFiles) {
const useBucket = buckets.find((b) => b.canFit(file, cloudflareLimit));
if (useBucket) {
useBucket.addFile(file);
} else {
const newBucket = new Bucket();
newBucket.addFile(file);
buckets.push(newBucket);
}
}
for (const bucket of buckets) {
new ArchiveTask(ArchiveType.Fulls, bucket.files).addToTask(archiveTasks);
}
this.log(`Created ${buckets.length} fulls archives.`);
}
// 更新包
for (const lastChecksum of lastBuildChecksums) {
const changedFiles = filesWithHash.filter((f) => !lastChecksum[f.file.path] || lastChecksum[f.file.path] !== f.hash);
if (changedFiles.length) {
new ArchiveTask(ArchiveType.Update, changedFiles).addToTask(archiveTasks);
}
} }
} }
...@@ -162,7 +192,7 @@ export class PackagerService extends ConsoleLogger { ...@@ -162,7 +192,7 @@ export class PackagerService extends ConsoleLogger {
if (file.file.stats.size < this.bucket_enter && !this.noGatherExts.has(extname)) { if (file.file.stats.size < this.bucket_enter && !this.noGatherExts.has(extname)) {
buckets[extname] ??= new Bucket(); buckets[extname] ??= new Bucket();
const bucket = buckets[extname]; const bucket = buckets[extname];
if (bucket.size + file.file.stats.size >= this.bucket_max) { if (!bucket.canFit(file, this.bucket_max)) {
new ArchiveTask(ArchiveType.Part, bucket.files).addToTask(pendingPartTasks, true); new ArchiveTask(ArchiveType.Part, bucket.files).addToTask(pendingPartTasks, true);
bucket.empty(); bucket.empty();
} }
......
...@@ -161,7 +161,7 @@ export class UpdateService extends ConsoleLogger { ...@@ -161,7 +161,7 @@ export class UpdateService extends ConsoleLogger {
async getFullPackageMetalink(id: string, depotDto: DepotDto, version: string, ip: string) { async getFullPackageMetalink(id: string, depotDto: DepotDto, version: string, ip: string) {
const tryRole = (role: ArchiveType) => const tryRole = (role: ArchiveType) =>
this.getArchives(id, depotDto, version, (qb) => this.getArchives(id, depotDto, version, (qb) =>
qb.select(['archive.hash', 'archive.path', 'archive.size']).andWhere('archive.role = :fullRole', { fullRole: ArchiveType.Full }) qb.select(['archive.hash', 'archive.path', 'archive.size']).andWhere('archive.role = :role', { role })
); );
let archives = await tryRole(ArchiveType.Fulls); let archives = await tryRole(ArchiveType.Fulls);
if (!archives.length) { if (!archives.length) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment