Skip to content

B
Basedformer
Commit aa35ad92 authored by Wes Brown's avatar Wes Brown
Browse files
Options

Small fixes.

parent 8b26deda
Hide whitespace changes
Inline Side-by-side
Showing with 3 additions and 3 deletions
hypertrain.py
View file @ aa35ad92
...@@ -214,7 +214,7 @@ parser.add_argument('--optimizer', type=str, help='the optimizer to use', ...@@ -214,7 +214,7 @@ parser.add_argument('--optimizer', type=str, help='the optimizer to use',
parser.add_argument('--lr', type=float, help='learning rate', default=2e-4) parser.add_argument('--lr', type=float, help='learning rate', default=2e-4)
parser.add_argument('--end_lr', type=float, help='end learning rate', parser.add_argument('--end_lr', type=float, help='end learning rate',
default=2e-4) default=2e-4)
parser.add_argument('--warmup', type=int, help='warmup steps') parser.add_argument('--warmup', type=int, help='warmup steps', default=10)
parser.add_argument('--bs', type=int, help='batch size', default=4) parser.add_argument('--bs', type=int, help='batch size', default=4)
parser.add_argument('--gas', type=int, help='gas', default=1) parser.add_argument('--gas', type=int, help='gas', default=1)
parser.add_argument('--seed', type=int, help="Random seed value", parser.add_argument('--seed', type=int, help="Random seed value",
...@@ -247,7 +247,7 @@ if args.output == '': ...@@ -247,7 +247,7 @@ if args.output == '':
# we need 250 batch size to train the small GPT. # we need 250 batch size to train the small GPT.
train_config = { train_config = {
"data_path": args.dataset, "data_path": args.dataset,
"save_path": args.model, "save_path": args.output,
"lm_path": args.model, "lm_path": args.model,
"optimizer": args.optimizer, "optimizer": args.optimizer,
"masked_softmax_fusion": args.masked, "masked_softmax_fusion": args.masked,
...@@ -259,7 +259,7 @@ train_config = { ...@@ -259,7 +259,7 @@ train_config = {
"bs": args.bs, "bs": args.bs,
"gas": args.gas, "gas": args.gas,
"seed": args.seed, "seed": args.seed,
"save_every": args.save_steps0, "save_every": args.save_steps,
"amp": args.amp, "amp": args.amp,
"loss_scale": args.loss_scale, "loss_scale": args.loss_scale,
"eval_every": args.eval_every, "eval_every": args.eval_every,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment