ree.

.gitignore

@@ -132,4 +132,4 @@ models
 gptjconvert
 j6b_vanilla
 wandb
-sigurd_v5_2049.map
\ No newline at end of file
+*.map
\ No newline at end of file

train.py

@@ -26,16 +26,16 @@ model_config = {
 train_config = {
     "lr": 6e-4,
     "end_lr": 6e-4,
-    "warmup_steps": 100,
+    "warmup_steps": 20,
     "bs": 16,
-    "gas": 2,
+    "gas": 16,
     "seed": 69,
 }
 bs = train_config["bs"]
 gas = train_config["gas"]
 model = GPTModel.neox_init(model_config).cuda().bfloat16()
 opt = optimizer.BasedOptimizer(model.parameters(), train_config, "adamw")
-train_dataset = utils.FbDataset(2049, "sigurd_v5_2049.map")
+train_dataset = utils.FbDataset(2049, "/home/xuser/diffusionstorage/datasets/sigurd/map/sigurd_v5_2049.map")
 train_loader = data.DataLoader(train_dataset, batch_size=bs*gas, shuffle=False, num_workers=0)
 wandb.init(project="basedformer-tests", name="sigurd_v5_2049")