|
@ -56,7 +56,7 @@ search_params = sample(all_params, min(NUM_JOBS, len(all_params))) |
|
|
|
|
|
|
|
|
for idx, params in enumerate(search_params): |
|
|
for idx, params in enumerate(search_params): |
|
|
a, lr, bs, me, s, w, opt = params |
|
|
a, lr, bs, me, s, w, opt = params |
|
|
cmd = f"cd ~/colors && python main.py --alpha {a} --lr {lr} --bs {bs} --max_epochs {me} --seed {s} --width {w}" |
|
|
# cmd = f"cd ~/colors && python main.py --alpha {a} --lr {lr} --bs {bs} --max_epochs {me} --seed {s} --width {w}" |
|
|
cmd = f""" |
|
|
cmd = f""" |
|
|
python newmain.py fit \ |
|
|
python newmain.py fit \ |
|
|
--seed_everything {s} \ |
|
|
--seed_everything {s} \ |
|
@ -65,6 +65,7 @@ python newmain.py fit \ |
|
|
--data.val_size 100000 \ |
|
|
--data.val_size 100000 \ |
|
|
--model.alpha {a} \ |
|
|
--model.alpha {a} \ |
|
|
--model.width {w} \ |
|
|
--model.width {w} \ |
|
|
|
|
|
--model.depth 1 \ |
|
|
--trainer.min_epochs 10 \ |
|
|
--trainer.min_epochs 10 \ |
|
|
--trainer.max_epochs {me} \ |
|
|
--trainer.max_epochs {me} \ |
|
|
--trainer.log_every_n_steps 3 \ |
|
|
--trainer.log_every_n_steps 3 \ |
|
@ -82,13 +83,15 @@ python newmain.py fit \ |
|
|
--lr_scheduler.init_args.cooldown 10 \ |
|
|
--lr_scheduler.init_args.cooldown 10 \ |
|
|
--lr_scheduler.init_args.verbose true |
|
|
--lr_scheduler.init_args.verbose true |
|
|
""" |
|
|
""" |
|
|
test_cmd = cmd + f" --print_config > out/config_v{idx:04d}.txt" |
|
|
test_cmd = f"{cmd.strip()} --print_config > out/config_v{idx:04d}.txt" |
|
|
|
|
|
|
|
|
# job_name = f"color2_{bs}_{a}_{lr:2.2e}" |
|
|
# job_name = f"color2_{bs}_{a}_{lr:2.2e}" |
|
|
# job_plugin.run(cmd, machine=Machine.T4, name=job_name) |
|
|
# job_plugin.run(cmd, machine=Machine.T4, name=job_name) |
|
|
print(f"Running {params}: {cmd}") |
|
|
print(f"Running {params}: {cmd}") |
|
|
|
|
|
cmd = f"{test_cmd.strip()} && {cmd}" |
|
|
try: |
|
|
try: |
|
|
# Run the command and wait for it to complete |
|
|
# Run the command and wait for it to complete |
|
|
subprocess.run(test_cmd, shell=True, check=True) |
|
|
# subprocess.run(test_cmd, shell=True, check=True) |
|
|
subprocess.run(cmd, shell=True, check=True) |
|
|
subprocess.run(cmd, shell=True, check=True) |
|
|
except KeyboardInterrupt: |
|
|
except KeyboardInterrupt: |
|
|
print("Interrupted by user") |
|
|
print("Interrupted by user") |
|
|