colors/scrape.py

import glob
import shutil
from pathlib import Path

from check import make_image


def get_exps(pattern: str, splitter: str = "_", dry_run: bool = True):
    basedir = "/teamspace/jobs/"
    chkpt_basedir = "/work/colors/lightning_logs/"
    location = basedir + pattern
    res = glob.glob(location)
    location = location.replace("*", "")
    H = []  # hyperparams used
    # print(res)
    for r in res:
        d = r.replace(location, "").split(splitter)
        d = list(float(_d) for _d in d)
        d[0] = int(d[0])
        H.append(d)
    for i, r in enumerate(res):
        dir_path = Path(
            f"/teamspace/studios/this_studio/colors/lightning_logs/version_{i}/"
        )
        dir_path.mkdir(parents=True, exist_ok=True)
        g = glob.glob(r + chkpt_basedir + "*")
        logs = glob.glob(g[0] + "/events*")[-1]
        source_path = Path(logs)
        print(logs)
        if not dry_run:
            c = g[0] + "/checkpoints"
            latest_checkpoint = glob.glob(c + "/*")[-1]
            print(latest_checkpoint)
        if not dry_run:
            shutil.copy(source_path, dir_path)
            make_image(latest_checkpoint, f"out/version_{i}")
            # make_image(latest_checkpoint, f"out/version_{i}b", color=False)
        else:
            print("Would copy", source_path, dir_path)
    return H


if __name__ == "__main__":
    D = get_exps("color_*", "_")

    import numpy as np

    D = np.array(D)
    # print(len(D), "\n", D)
    import pandas as pd

    df = pd.DataFrame(D)
    df.columns = ["batch_size", "alpha", "learning_rate"]
    df.to_csv("experiments.csv")
    print(df)