Better error proceeding in code loops

[isaacmg]

When looping through Wanda_sweeps there seems to be an error related to the validation loader.

Traceback (most recent call last):
  File "/usr/lib/python3.6/multiprocessing/process.py", line 258, in _bootstrap
    self.run()
  File "/usr/lib/python3.6/multiprocessing/process.py", line 93, in run
    self._target(*self._args, **self._kwargs)
  File "/usr/local/lib/python3.6/dist-packages/wandb/wandb_agent.py", line 64, in _start
    function()
  File "<ipython-input-14-43e09d654e98>", line 7, in <lambda>
    wandb.agent(sweep_id, lambda: train_function("PyTorch", make_config_file(file_path2, len(region), weight_path=None)))
  File "/content/github_aistream-peelout_flow-forecast/flood_forecast/trainer.py", line 33, in train_function
    train_transformer_style(trained_model, params["training_params"], params["forward_params"])
  File "/content/github_aistream-peelout_flow-forecast/flood_forecast/pytorch_training.py", line 65, in train_transformer_style
    test = compute_validation(test_data_loader, model.model, epoch, model.params["dataset_params"]["forecast_length"], criterion, model.device, decoder_structure=True, use_wandb=use_wandb, val_or_test="test_loss")
  File "/content/github_aistream-peelout_flow-forecast/flood_forecast/pytorch_training.py", line 108, in compute_validation
    for src, targ in validation_loader:
  File "/usr/local/lib/python3.6/dist-packages/torch/utils/data/dataloader.py", line 345, in __next__
    data = self._next_data()
  File "/usr/local/lib/python3.6/dist-packages/torch/utils/data/dataloader.py", line 384, in _next_data
    index = self._next_index()  # may raise StopIteration
  File "/usr/local/lib/python3.6/dist-packages/torch/utils/data/dataloader.py", line 339, in _next_index
    return next(self._sampler_iter)  # may raise StopIteration
  File "/usr/local/lib/python3.6/dist-packages/torch/utils/data/sampler.py", line 200, in __iter__
    for idx in self.sampler:
  File "/usr/local/lib/python3.6/dist-packages/torch/utils/data/sampler.py", line 62, in __iter__
    return iter(range(len(self.data_source)))
ValueError: __len__() should return >= 0