Remove error when test dataloader used in test

CHANGELOG.md

@@ -111,6 +111,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 
 ### Removed
 
+- Removed test for no test dataloader in .fit ([#1495](https://github.com/PyTorchLightning/pytorch-lightning/pull/1495))
 - Removed duplicated module `pytorch_lightning.utilities.arg_parse` for loading CLI arguments ([#1167](https://github.com/PyTorchLightning/pytorch-lightning/issues/1167))
 - Removed wandb logger's `finalize` method ([#1193](https://github.com/PyTorchLightning/pytorch-lightning/pull/1193))
 - Dropped `torchvision` dependency in tests and added own MNIST dataset class instead ([#986](https://github.com/PyTorchLightning/pytorch-lightning/issues/986))

pytorch_lightning/trainer/trainer.py

@@ -944,6 +944,9 @@ def test(self, model: Optional[LightningModule] = None, test_dataloaders: Option
             else:
                 self.__attach_dataloaders(self.model, test_dataloaders=test_dataloaders)
 
+        # give proper warnings if user only passed in loader without hooks
+        self.check_testing_model_configuration(model, test_dataloaders)
+
         if model is not None:
             self.model = model
             self.fit(model)
@@ -1012,10 +1015,26 @@ def check_model_configuration(self, model: LightningModule):
                         'You have defined a `test_dataloader()` and have defined a `test_step()`, you may also want to'
                         ' define `test_epoch_end()` for accumulating stats.', RuntimeWarning
                     )
-        else:
-            if self.is_overriden('test_step', model):
-                raise MisconfigurationException('You have defined `test_step()`,'
-                                                ' but have not passed in a `test_dataloader()`.')
+
+    def check_testing_model_configuration(self, model: LightningModule, test_dataloader: DataLoader):
+
+        has_test_step = self.is_overriden('test_step', model)
+        has_test_epoch_end = self.is_overriden('test_epoch_end', model)
+        gave_test_loader = test_dataloader is not None
+
+        if gave_test_loader and not has_test_step:
+            raise MisconfigurationException('You passed in a `test_dataloader` but did not implement '
+                                            ' `test_step()`')
+
+        if has_test_step and not gave_test_loader:
+            raise MisconfigurationException('You defined `test_step()` but did not implement '
+                                            ' `test_dataloader` nor passed in `.fit(test_dataloaders`.')
+
+        if has_test_step and gave_test_loader and not has_test_epoch_end:
+            rank_zero_warn(
+                'You passed  in a `test_dataloader` and have defined a `test_step()`, you may also want to'
+                ' define `test_epoch_end()` for accumulating stats.', RuntimeWarning
+            )
 
 
 class _PatchDataLoader(object):

tests/base/__init__.py

@@ -3,6 +3,7 @@
 import torch
 
 from tests.base.models import TestModelBase, DictHparamsModel
+from tests.base.template_test_model import TemplateTestModel
 from tests.base.mixins import (
     LightEmptyTestStep,
     LightValidationStepMixin,

tests/base/config_optimizers_variations.py

@@ -0,0 +1,59 @@
+from torch import optim
+
+
+class ConfigureOptimizersVariationsMixin:
+    def configure_optimizers(self):
+        """
+        return whatever optimizers we want here.
+        :return: list of optimizers
+        """
+        optimizer = optim.Adam(self.parameters(), lr=self.hparams.learning_rate)
+        return optimizer
+
+    def configure_optimizers_empty(self):
+        return None
+
+    def configure_optimizers_lbfgs(self):
+        """
+        return whatever optimizers we want here.
+        :return: list of optimizers
+        """
+        optimizer = optim.LBFGS(self.parameters(), lr=self.hparams.learning_rate)
+        return optimizer
+
+    def configure_optimizers_multiple_optimizers(self):
+        """
+        return whatever optimizers we want here.
+        :return: list of optimizers
+        """
+        # try no scheduler for this model (testing purposes)
+        optimizer1 = optim.Adam(self.parameters(), lr=self.hparams.learning_rate)
+        optimizer2 = optim.Adam(self.parameters(), lr=self.hparams.learning_rate)
+        return optimizer1, optimizer2
+
+    def configure_optimizers_single_scheduler(self):
+        optimizer = optim.Adam(self.parameters(), lr=self.hparams.learning_rate)
+        lr_scheduler = optim.lr_scheduler.StepLR(optimizer, 1, gamma=0.1)
+        return [optimizer], [lr_scheduler]
+
+    def configure_optimizers_multiple_schedulers(self):
+        optimizer1 = optim.Adam(self.parameters(), lr=self.hparams.learning_rate)
+        optimizer2 = optim.Adam(self.parameters(), lr=self.hparams.learning_rate)
+        lr_scheduler1 = optim.lr_scheduler.StepLR(optimizer1, 1, gamma=0.1)
+        lr_scheduler2 = optim.lr_scheduler.StepLR(optimizer2, 1, gamma=0.1)
+
+        return [optimizer1, optimizer2], [lr_scheduler1, lr_scheduler2]
+
+    def configure_optimizers_mixed_scheduling(self):
+        optimizer1 = optim.Adam(self.parameters(), lr=self.hparams.learning_rate)
+        optimizer2 = optim.Adam(self.parameters(), lr=self.hparams.learning_rate)
+        lr_scheduler1 = optim.lr_scheduler.StepLR(optimizer1, 4, gamma=0.1)
+        lr_scheduler2 = optim.lr_scheduler.StepLR(optimizer2, 1, gamma=0.1)
+
+        return [optimizer1, optimizer2], \
+            [{'scheduler': lr_scheduler1, 'interval': 'step'}, lr_scheduler2]
+
+    def configure_optimizers_reduce_lr_on_plateau(self):
+        optimizer = optim.Adam(self.parameters(), lr=self.hparams.learning_rate)
+        lr_scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer)
+        return [optimizer], [lr_scheduler]

tests/base/template_test_model.py

@@ -0,0 +1,80 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from tests.base.datasets import TestingMNIST
+from pytorch_lightning.core.lightning import LightningModule
+from tests.base.training_step_variations import TrainingStepVariationsMixin
+from tests.base.test_step_variations import TestStepVariationsMixin
+from tests.base.validation_step_variations import ValidationStepVariationsMixin
+from tests.base.test_epoch_end_variations import TestEpochEndVariationsMixin
+from tests.base.config_optimizers_variations import ConfigureOptimizersVariationsMixin
+from tests.base.val_dataloader_variations import ValDataloaderVariationsMixin
+from tests.base.train_dataloader_variations import TrainDataloaderVariationsMixin
+from tests.base.test_dataloader_variations import TestDataloaderVariationsMixin
+from tests.base.validation_epoch_end_variations import ValidationEpochEndVariationsMixin
+from tests.base.template_test_model_utils import TemplateTestModelUtilsMixin
+
+
+class TemplateTestModel(
+    TrainingStepVariationsMixin,
+    ValidationStepVariationsMixin,
+    ValidationEpochEndVariationsMixin,
+    TestStepVariationsMixin,
+    TestEpochEndVariationsMixin,
+    TrainDataloaderVariationsMixin,
+    ValDataloaderVariationsMixin,
+    TestDataloaderVariationsMixin,
+    ConfigureOptimizersVariationsMixin,
+    TemplateTestModelUtilsMixin,
+    LightningModule
+):
+    """
+    This template houses all  combinations of model  configurations  we want to test
+    """
+    def __init__(self, hparams):
+        """Pass in parsed HyperOptArgumentParser to the model."""
+        # init superclass
+        super().__init__()
+        self.hparams = hparams
+
+        # if you specify an example input, the summary will show input/output for each layer
+        self.example_input_array = torch.rand(5, 28 * 28)
+
+        # build model
+        self.__build_model()
+
+    def __build_model(self):
+        """
+        Simple model for testing
+        :return:
+        """
+        self.c_d1 = nn.Linear(
+            in_features=self.hparams.in_features,
+            out_features=self.hparams.hidden_dim
+        )
+        self.c_d1_bn = nn.BatchNorm1d(self.hparams.hidden_dim)
+        self.c_d1_drop = nn.Dropout(self.hparams.drop_prob)
+
+        self.c_d2 = nn.Linear(
+            in_features=self.hparams.hidden_dim,
+            out_features=self.hparams.out_features
+        )
+
+    def forward(self, x):
+        x = self.c_d1(x)
+        x = torch.tanh(x)
+        x = self.c_d1_bn(x)
+        x = self.c_d1_drop(x)
+
+        x = self.c_d2(x)
+        logits = F.log_softmax(x, dim=1)
+
+        return logits
+
+    def loss(self, labels, logits):
+        nll = F.nll_loss(logits, labels)
+        return nll
+
+    def prepare_data(self):
+        _ = TestingMNIST(root=self.hparams.data_root, train=True, download=True)

tests/base/template_test_model_utils.py

@@ -0,0 +1,23 @@
+from torch.utils.data import DataLoader
+from tests.base.datasets import TestingMNIST
+
+
+class TemplateTestModelUtilsMixin:
+
+    def dataloader(self, train):
+        dataset = TestingMNIST(root=self.hparams.data_root, train=train, download=False)
+
+        loader = DataLoader(
+            dataset=dataset,
+            batch_size=self.hparams.batch_size,
+            shuffle=True
+        )
+
+        return loader
+
+    def get_output_metric(self, output, name):
+        if isinstance(output, dict):
+            val = output[name]
+        else:  # if it is 2level deep -> per dataloader and per batch
+            val = sum(out[name] for out in output) / len(output)
+        return val

tests/base/test_dataloader_variations.py

@@ -0,0 +1,11 @@
+import tests.base.model_utils as mutils
+
+
+class TestDataloaderVariationsMixin:
+
+    def test_dataloader(self):
+        return mutils.dataloader(
+            train=False,
+            data_root=self.hparams.data_root,
+            batch_size=self.hparams.batch_size,
+        )

tests/base/test_epoch_end_variations.py

@@ -0,0 +1,36 @@
+import torch
+
+
+class TestEpochEndVariationsMixin:
+    def test_epoch_end(self, outputs):
+        """
+        Called at the end of validation to aggregate outputs
+        :param outputs: list of individual outputs of each validation step
+        :return:
+        """
+        # if returned a scalar from test_step, outputs is a list of tensor scalars
+        # we return just the average in this case (if we want)
+        # return torch.stack(outputs).mean()
+        test_loss_mean = 0
+        test_acc_mean = 0
+        for output in outputs:
+            test_loss = self.get_output_metric(output, 'test_loss')
+
+            # reduce manually when using dp
+            if self.trainer.use_dp:
+                test_loss = torch.mean(test_loss)
+            test_loss_mean += test_loss
+
+            # reduce manually when using dp
+            test_acc = self.get_output_metric(output, 'test_acc')
+            if self.trainer.use_dp:
+                test_acc = torch.mean(test_acc)
+
+            test_acc_mean += test_acc
+
+        test_loss_mean /= len(outputs)
+        test_acc_mean /= len(outputs)
+
+        metrics_dict = {'test_loss': test_loss_mean.item(), 'test_acc': test_acc_mean.item()}
+        result = {'progress_bar': metrics_dict, 'log': metrics_dict}
+        return result

tests/base/test_step_variations.py

@@ -0,0 +1,50 @@
+from collections import OrderedDict
+import torch
+
+
+class TestStepVariationsMixin:
+    """
+    Houses all variations of test steps
+    """
+    def test_step(self, batch, batch_idx, dataloader_idx, **kwargs):
+        """
+        Default, baseline test_step
+        :param batch:
+        :return:
+        """
+        x, y = batch
+        x = x.view(x.size(0), -1)
+        y_hat = self(x)
+
+        loss_test = self.loss(y, y_hat)
+
+        # acc
+        labels_hat = torch.argmax(y_hat, dim=1)
+        test_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0)
+        test_acc = torch.tensor(test_acc)
+
+        test_acc = test_acc.as_type(x)
+
+        # alternate possible outputs to test
+        if batch_idx % 1 == 0:
+            output = OrderedDict({
+                'test_loss': loss_test,
+                'test_acc': test_acc,
+            })
+            return output
+        if batch_idx % 2 == 0:
+            return test_acc
+
+        if batch_idx % 3 == 0:
+            output = OrderedDict({
+                'test_loss': loss_test,
+                'test_acc': test_acc,
+                'test_dic': {'test_loss_a': loss_test}
+            })
+            return output
+        if batch_idx % 5 == 0:
+            output = OrderedDict({
+                f'test_loss_{dataloader_idx}': loss_test,
+                f'test_acc_{dataloader_idx}': test_acc,
+            })
+            return output

tests/base/train_dataloader_variations.py

@@ -0,0 +1,11 @@
+import tests.base.model_utils as mutils
+
+
+class TrainDataloaderVariationsMixin:
+
+    def train_dataloader(self):
+        return mutils.dataloader(
+            train=True,
+            data_root=self.hparams.data_root,
+            batch_size=self.hparams.batch_size,
+        )

tests/base/training_step_variations.py

@@ -0,0 +1,29 @@
+from collections import OrderedDict
+
+
+class TrainingStepVariationsMixin:
+    """
+    Houses all variations of training steps
+    """
+    def training_step(self, batch, batch_idx, optimizer_idx=None):
+        """Lightning calls this inside the training loop"""
+        # forward pass
+        x, y = batch
+        x = x.view(x.size(0), -1)
+
+        y_hat = self(x)
+
+        # calculate loss
+        loss_val = self.loss(y, y_hat)
+
+        # alternate possible outputs to test
+        if self.trainer.batch_idx % 1 == 0:
+            output = OrderedDict({
+                'loss': loss_val,
+                'progress_bar': {'some_val': loss_val * loss_val},
+                'log': {'train_some_val': loss_val * loss_val},
+            })
+            return output
+
+        if self.trainer.batch_idx % 2 == 0:
+            return loss_val

tests/base/val_dataloader_variations.py

@@ -0,0 +1,11 @@
+import tests.base.model_utils as mutils
+
+
+class ValDataloaderVariationsMixin:
+
+    def val_dataloader(self):
+        return mutils.dataloader(
+            train=False,
+            data_root=self.hparams.data_root,
+            batch_size=self.hparams.batch_size,
+        )