more checks

basedformer/dataset.py

@@ -10,6 +10,7 @@ import pickle
 from pathlib import Path
 from PIL import Image
 import requests
+import hashlib
 
 # Does this work with other block_sizes? doesn't seem to.
 class FbDataset(data.Dataset):
@@ -216,10 +217,10 @@ class ImageDatasetBuilder():
         else:
             raise Exception("Metadata file not found at {}".format(self.metadata_path))
 
-    def operate(self, operation, batch, identities, metadata=None, executor=concurrent.futures.ThreadPoolExecutor):
+    def operate(self, operation, batch, identities, metadata=None, executor=concurrent.futures.ThreadPoolExecutor, **kwargs):
 
         executor = executor(max_workers=self.threads)
-        futures = executor.map(operation, batch)
+        futures = executor.map(operation, batch, **kwargs)
         futures = list(futures)
             
         for data, identity in zip(futures, identities):
@@ -237,11 +238,18 @@ class ImageDatasetBuilder():
 
         return data
 
-    def url_op(self, url):
+    def url_op(self, url, md5):
         result = requests.get(url)
         for _ in range(5):
             if result.status_code == 200:
                 break
+        
+        if result.status_code != 200:
+            return None
+
+        saved_md5 = hashlib.md5(data)
+        if saved_md5 != md5:
+            return None
 
         data = result.content
         data = self.encode_op(data)