remove archive for non-existing

console-api/src/app.service.ts

-import { Connection, IsNull, Not, SelectQueryBuilder } from 'typeorm';
+import { Connection, In, IsNull, Not, SelectQueryBuilder } from 'typeorm';
 import { InjectConnection } from '@nestjs/typeorm';
 import { ConsoleLogger, forwardRef, Inject, Injectable } from '@nestjs/common';
 import { AppsJson } from './utility/apps-json-type';
@@ -264,7 +264,8 @@ export class AppService extends ConsoleLogger {
       .createQueryBuilder('unusedBuild')
       .select('distinct(archive.path)', 'pathToPurge')
       .innerJoin('unusedBuild.archives', 'archive')
-      .where('archive.role != :latestRole', { latestRole: ArchiveType.Full });
+      .where('archive.role != :fullRole', { fullRole: ArchiveType.Full })
+      .andWhere('archive.role != :fullsRole', { fullsRole: ArchiveType.Fulls });
 
     const subQuery = query
       .subQuery()
@@ -283,7 +284,7 @@ export class AppService extends ConsoleLogger {
 
     query.andWhere(`unusedBuild.id not in ${subQuery.getQuery()}`).andWhere(`not exists ${packageReferenceQuery.getQuery()}`);
     // this.log(`SQL: ${query.getQueryAndParameters()}`);
-    return (await query.getRawMany()).map((s) => `${s.pathToPurge}.tar.zst` as string);
+    return (await query.getRawMany()).map((s) => s.pathToPurge as string);
   }
 
   private async getArchivePathsToPurge(buildId: number) {
@@ -295,7 +296,20 @@ export class AppService extends ConsoleLogger {
 
     this.packageReferenceSubQuery(query);
     // this.log(`SQL: ${query.getQueryAndParameters()}`);
-    return (await query.getRawMany()).map((s) => `${s.pathToPurge}.tar.zst` as string);
+    return (await query.getRawMany()).map((s) => s.pathToPurge as string);
+  }
+
+  private async removeArchiveEntry(paths: string[]) {
+    if (!paths || !paths.length) {
+      return;
+    }
+    const chunks = _.chunk(paths, 65000);
+    await this.db.transaction(async (edb) => {
+      for (const chunk of chunks) {
+        this.log(`Removing archive entries for ${chunk.length} paths.`);
+        await edb.getRepository(Archive).delete({ path: In(chunk) });
+      }
+    });
   }
 
   async purgeOldArchives() {
@@ -304,7 +318,8 @@ export class AppService extends ConsoleLogger {
     if (!paths.length) {
       return;
     }
-    return this.packageS3.removeObjects(paths);
+    await this.packageS3.removeObjects(paths.map((p) => p + '.tar.zst'));
+    await this.removeArchiveEntry(paths);
   }
 
   private async purgeRelatedArchives(build: Build) {
@@ -313,7 +328,8 @@ export class AppService extends ConsoleLogger {
     if (!paths.length) {
       return;
     }
-    return this.packageS3.removeObjects(paths);
+    await this.packageS3.removeObjects(paths.map((p) => p + '.tar.zst'));
+    await this.removeArchiveEntry(paths);
   }
 
   async removeBuild(user: MyCardUser, id: string, depotDto: DepotDto, version: string) {

console-api/src/s3/s3.service.ts

@@ -72,6 +72,23 @@ export class S3Service extends ConsoleLogger {
     return this.s3.send(command);
   }
 
+  async hasObject(path: string) {
+    try {
+      await this.s3.send(
+        new HeadObjectCommand({
+          Bucket: this.bucket,
+          Key: this.getPathWithPrefix(path),
+        })
+      );
+      return true;
+    } catch (e) {
+      if (e.name === 'NotFound') {
+        return false;
+      }
+      throw e; // rethrow other errors
+    }
+  }
+
   async removeObjects(paths: string[]) {
     const CHUNK_SIZE = 1000;