Log FAD only during test. Use rendered files during test.

README.md

@@ -35,10 +35,11 @@ Ex. `python scripts/train.py +exp=umx_distortion trainer.accelerator='gpu' train
 - `reverb`
 - `all` (choose random effect to apply to each file)
 
+### Testing
+Experiment dictates data, ckpt dictates model
+`python scripts/test.py +exp=umx_distortion.yaml +ckpt_path=test_ckpts/umx_dist.ckpt`
+
 ## Misc.
 By default, files are rendered to `input_dir / processed / train/val/test`.
-To skip rendering files (use previously rendered), add `render_files=False` to the command-line
-To change the rendered location, add `render_root={path/to/dir}` to the command-line
-Test
-Experiment dictates data, ckpt dictates model
-`python scripts/test.py +exp=umx_distortion.yaml +ckpt_path=test_ckpts/umx_dist.ckpt`
+To skip rendering files (use previously rendered), add `render_files=False` to the command-line (added to test by default).
+To change the rendered location, add `render_root={path/to/dir}` to the command-line (use this for train and test)

cfg/config.yaml

@@ -19,7 +19,7 @@ callbacks:
     save_last: True         # additionaly always save model from last epoch
     mode: "min"             # can be "max" or "min"
     verbose: False
-    dirpath: ${logs_dir}/ckpts/${now:%Y-%m-%d-%H-%M-%S}
+    dirpath: ${logs_dir}/ckpts/${now:%Y-%m-%d-%H-%M-%S}-${exp}
     filename: '{epoch:02d}-{valid_loss:.3f}'
 
 datamodule:

remfx/models.py

@@ -79,6 +79,9 @@ class RemFXModel(pl.LightningModule):
                     negate = -1
                 else:
                     negate = 1
+                # Only Log FAD on test set
+                if metric == "FAD" and mode != "test":
+                    continue
                 self.log(
                     f"{mode}_{metric}",
                     negate * self.metrics[metric](output, y),

scripts/test.py

@@ -0,0 +1,55 @@
+import pytorch_lightning as pl
+import hydra
+from omegaconf import DictConfig
+import remfx.utils as utils
+from pytorch_lightning.utilities.model_summary import ModelSummary
+from remfx.models import RemFXModel
+import torch
+
+log = utils.get_logger(__name__)
+
+
+@hydra.main(version_base=None, config_path="../cfg", config_name="config.yaml")
+def main(cfg: DictConfig):
+    # Apply seed for reproducibility
+    if cfg.seed:
+        pl.seed_everything(cfg.seed)
+    cfg.render_files = False
+    log.info(f"Instantiating datamodule <{cfg.datamodule._target_}>.")
+    datamodule = hydra.utils.instantiate(cfg.datamodule, _convert_="partial")
+    log.info(f"Instantiating model <{cfg.model._target_}>.")
+    model = hydra.utils.instantiate(cfg.model, _convert_="partial")
+    state_dict = torch.load(cfg.ckpt_path, map_location=torch.device("cpu"))[
+        "state_dict"
+    ]
+    model.load_state_dict(state_dict)
+
+    # Init all callbacks
+    callbacks = []
+    if "callbacks" in cfg:
+        for _, cb_conf in cfg["callbacks"].items():
+            if "_target_" in cb_conf:
+                log.info(f"Instantiating callback <{cb_conf._target_}>.")
+                callbacks.append(hydra.utils.instantiate(cb_conf, _convert_="partial"))
+
+    logger = hydra.utils.instantiate(cfg.logger, _convert_="partial")
+    log.info(f"Instantiating trainer <{cfg.trainer._target_}>.")
+    trainer = hydra.utils.instantiate(
+        cfg.trainer, callbacks=callbacks, logger=logger, _convert_="partial"
+    )
+    log.info("Logging hyperparameters!")
+    utils.log_hyperparameters(
+        config=cfg,
+        model=model,
+        datamodule=datamodule,
+        trainer=trainer,
+        callbacks=callbacks,
+        logger=logger,
+    )
+    summary = ModelSummary(model)
+    print(summary)
+    trainer.test(model=model, datamodule=datamodule)
+
+
+if __name__ == "__main__":
+    main()