Browse Source

experiment loggin setup

new-sep-loss
Michael Pilosov, PhD 10 months ago
parent
commit
49e6260346
  1. 2
      makefile
  2. 9
      newsearch.py

2
makefile

@ -50,6 +50,6 @@ animate:
clean: clean:
rm -rf lightning_logs rm -rf lightning_logs
rm -f out/*.png rm -f out/*.png out/*.txt
rm -rf __pycache__/ rm -rf __pycache__/
cp hsv.png out/ cp hsv.png out/

9
newsearch.py

@ -56,7 +56,7 @@ search_params = sample(all_params, min(NUM_JOBS, len(all_params)))
for idx, params in enumerate(search_params): for idx, params in enumerate(search_params):
a, lr, bs, me, s, w, opt = params a, lr, bs, me, s, w, opt = params
cmd = f"cd ~/colors && python main.py --alpha {a} --lr {lr} --bs {bs} --max_epochs {me} --seed {s} --width {w}" # cmd = f"cd ~/colors && python main.py --alpha {a} --lr {lr} --bs {bs} --max_epochs {me} --seed {s} --width {w}"
cmd = f""" cmd = f"""
python newmain.py fit \ python newmain.py fit \
--seed_everything {s} \ --seed_everything {s} \
@ -65,6 +65,7 @@ python newmain.py fit \
--data.val_size 100000 \ --data.val_size 100000 \
--model.alpha {a} \ --model.alpha {a} \
--model.width {w} \ --model.width {w} \
--model.depth 1 \
--trainer.min_epochs 10 \ --trainer.min_epochs 10 \
--trainer.max_epochs {me} \ --trainer.max_epochs {me} \
--trainer.log_every_n_steps 3 \ --trainer.log_every_n_steps 3 \
@ -82,13 +83,15 @@ python newmain.py fit \
--lr_scheduler.init_args.cooldown 10 \ --lr_scheduler.init_args.cooldown 10 \
--lr_scheduler.init_args.verbose true --lr_scheduler.init_args.verbose true
""" """
test_cmd = cmd + f" --print_config > out/config_v{idx:04d}.txt" test_cmd = f"{cmd.strip()} --print_config > out/config_v{idx:04d}.txt"
# job_name = f"color2_{bs}_{a}_{lr:2.2e}" # job_name = f"color2_{bs}_{a}_{lr:2.2e}"
# job_plugin.run(cmd, machine=Machine.T4, name=job_name) # job_plugin.run(cmd, machine=Machine.T4, name=job_name)
print(f"Running {params}: {cmd}") print(f"Running {params}: {cmd}")
cmd = f"{test_cmd.strip()} && {cmd}"
try: try:
# Run the command and wait for it to complete # Run the command and wait for it to complete
subprocess.run(test_cmd, shell=True, check=True) # subprocess.run(test_cmd, shell=True, check=True)
subprocess.run(cmd, shell=True, check=True) subprocess.run(cmd, shell=True, check=True)
except KeyboardInterrupt: except KeyboardInterrupt:
print("Interrupted by user") print("Interrupted by user")

Loading…
Cancel
Save