recalculate md5

console-api/src/s3/s3.service.ts

@@ -3,6 +3,7 @@ import { ConfigService } from '@nestjs/config';
 import {
   _Object,
   DeleteObjectsCommand,
+  GetObjectCommand,
   HeadObjectCommand,
   ListObjectsCommand,
   PutObjectCommand,
@@ -73,27 +74,6 @@ export class S3Service extends ConsoleLogger {
     return this.s3.send(command);
   }
 
-  async getObjectHash(path: string) {
-    try {
-      const head = await this.s3.send(
-        new HeadObjectCommand({
-          Bucket: this.bucket,
-          Key: this.getPathWithPrefix(path),
-        })
-      );
-      const etag = head.ETag;
-      if (etag) {
-        return etag.replace(/"/g, '');
-      }
-      return undefined;
-    } catch (e) {
-      if (e.name === 'NotFound') {
-        return undefined;
-      }
-      throw e; // rethrow other errors
-    }
-  }
-
   async removeObjects(paths: string[]) {
     const CHUNK_SIZE = 1000;
 
@@ -204,7 +184,28 @@ export class S3Service extends ConsoleLogger {
         Key: key,
       })
     );
-    const md5 = object.ETag ? object.ETag.replace(/"/g, '') : undefined;
+    let md5 = object.ETag ? object.ETag.replace(/"/g, '') : undefined;
+    if (!md5 || !md5.match(/^[a-f0-9]{32}$/)) { 
+      // this means the ETag is not an MD5, likely because the object was uploaded in multiple parts
+      this.warn(`ETag for ${key} is not a valid MD5: ${md5}`);
+      // recalculate MD5 by downloading the object again
+      const reworkHash = createHash('md5');
+      const redownload = await this.s3.send(
+        new GetObjectCommand({
+          Bucket: this.bucket,
+          Key: key,
+        })
+      );
+      const redlStream = redownload.Body as internal.Readable;
+      await new Promise<void>((resolve, reject) => {
+        redlStream.on('data', (chunk) => reworkHash.update(chunk));
+        redlStream.on('end', () => resolve());
+        redlStream.on('error', (e) => reject(e));
+      });
+      const recalculatedMd5 = reworkHash.digest('hex');
+      this.warn(`Recalculated MD5 for ${key} (${md5}): ${recalculatedMd5}`); // should not happen normally
+      md5 = recalculatedMd5;
+    }
     return { object, md5, url: this.getCdnUrl(path) };
   }
 }