|
| 1 | +import os |
| 2 | +import sys |
| 3 | +from pathlib import Path |
| 4 | +from unittest.mock import patch |
| 5 | + |
| 6 | +from art import Model |
| 7 | + |
| 8 | + |
| 9 | +def test_wandb_creates_separate_runs_per_model(tmp_path: Path): |
| 10 | + class FakeRun: |
| 11 | + def __init__(self, name: str): |
| 12 | + self.name = name |
| 13 | + self.id = name |
| 14 | + self._is_finished = False |
| 15 | + self.defined_metrics: list[tuple[str, str | None]] = [] |
| 16 | + |
| 17 | + def define_metric(self, name: str, *, step_metric: str | None = None) -> None: |
| 18 | + self.defined_metrics.append((name, step_metric)) |
| 19 | + |
| 20 | + class FakeWandb: |
| 21 | + def __init__(self): |
| 22 | + self.init_calls: list[dict] = [] |
| 23 | + self.runs: list[FakeRun] = [] |
| 24 | + |
| 25 | + @staticmethod |
| 26 | + def Settings(**kwargs): |
| 27 | + return kwargs |
| 28 | + |
| 29 | + def init(self, **kwargs): |
| 30 | + self.init_calls.append(kwargs) |
| 31 | + run = FakeRun(kwargs["name"]) |
| 32 | + self.runs.append(run) |
| 33 | + return run |
| 34 | + |
| 35 | + def define_metric(self, *args, **kwargs) -> None: |
| 36 | + raise AssertionError("Model should define metrics on the run object") |
| 37 | + |
| 38 | + fake_wandb = FakeWandb() |
| 39 | + model_one = Model( |
| 40 | + name="run-one", |
| 41 | + project="test-project", |
| 42 | + base_path=str(tmp_path), |
| 43 | + ) |
| 44 | + model_two = Model( |
| 45 | + name="run-two", |
| 46 | + project="test-project", |
| 47 | + base_path=str(tmp_path), |
| 48 | + ) |
| 49 | + |
| 50 | + with patch.dict(os.environ, {"WANDB_API_KEY": "test-key"}): |
| 51 | + with patch.dict(sys.modules, {"wandb": fake_wandb}): |
| 52 | + run_one = model_one._get_wandb_run() |
| 53 | + run_two = model_two._get_wandb_run() |
| 54 | + model_one._define_wandb_step_metrics(["costs/train/custom"]) |
| 55 | + |
| 56 | + assert run_one is not None |
| 57 | + assert run_two is not None |
| 58 | + assert run_one is not run_two |
| 59 | + assert [call["name"] for call in fake_wandb.init_calls] == [ |
| 60 | + "run-one", |
| 61 | + "run-two", |
| 62 | + ] |
| 63 | + assert all(call["reinit"] == "create_new" for call in fake_wandb.init_calls) |
| 64 | + assert run_one.defined_metrics == [ |
| 65 | + ("training_step", None), |
| 66 | + ("time/wall_clock_sec", None), |
| 67 | + ("reward/*", "training_step"), |
| 68 | + ("loss/*", "training_step"), |
| 69 | + ("throughput/*", "training_step"), |
| 70 | + ("costs/*", "training_step"), |
| 71 | + ("time/*", "training_step"), |
| 72 | + ("data/*", "training_step"), |
| 73 | + ("train/*", "training_step"), |
| 74 | + ("val/*", "training_step"), |
| 75 | + ("test/*", "training_step"), |
| 76 | + ("discarded/*", "training_step"), |
| 77 | + ("costs/train/custom", "training_step"), |
| 78 | + ] |
| 79 | + assert run_two.defined_metrics == [ |
| 80 | + ("training_step", None), |
| 81 | + ("time/wall_clock_sec", None), |
| 82 | + ("reward/*", "training_step"), |
| 83 | + ("loss/*", "training_step"), |
| 84 | + ("throughput/*", "training_step"), |
| 85 | + ("costs/*", "training_step"), |
| 86 | + ("time/*", "training_step"), |
| 87 | + ("data/*", "training_step"), |
| 88 | + ("train/*", "training_step"), |
| 89 | + ("val/*", "training_step"), |
| 90 | + ("test/*", "training_step"), |
| 91 | + ("discarded/*", "training_step"), |
| 92 | + ] |
0 commit comments