Merge pull request #82 from takluyver/i81

Fail output comparison on unexpected output

Author: takluyver

nbval/plugin.py

@@ -404,17 +404,29 @@ def compare_outputs(self, test, ref, skip_compare=None):
         # The traceback from the comparison will be stored here.
         self.comparison_traceback = []
 
-        for key in reference_outs.keys():
+        ref_keys = set(reference_outs.keys())
+        test_keys = set(testing_outs.keys())
 
-            # Check if they have the same keys
-            if key not in testing_outs.keys():
-                self.comparison_traceback.append(
-                    bcolors.FAIL
-                    + "missing key: TESTING %s != REFERENCE %s"
-                    % (testing_outs.keys(), reference_outs.keys())
-                    + bcolors.ENDC)
-                return False
+        if ref_keys - test_keys:
+            self.comparison_traceback.append(
+                bcolors.FAIL
+                + "Missing output fields from running code: %s"
+                % (ref_keys - test_keys)
+                + bcolors.ENDC
+            )
+            return False
+        elif test_keys - ref_keys:
+            self.comparison_traceback.append(
+                bcolors.FAIL
+                + "Unexpected output fields from running code: %s"
+                % (test_keys - ref_keys)
+                + bcolors.ENDC
+            )
+            return False
+
+        # If we've got to here, the two dicts must have the same set of keys
 
+        for key in reference_outs.keys():
             # Get output values for dictionary entries.
             # We use str() to be sure that the unicode key strings from the
             # reference are also read from the testing dictionary:
@@ -453,9 +465,9 @@ def compare_outputs(self, test, ref, skip_compare=None):
     def format_output_compare(self, key, left, right):
         """Format an output for printing"""
         if isinstance(left, six.string_types):
-            left = _trim_base64(output)
+            left = _trim_base64(left)
         if isinstance(right, six.string_types):
-            right = _trim_base64(output)
+            right = _trim_base64(right)
 
         self.comparison_traceback.append(
             bcolors.OKBLUE
@@ -474,7 +486,7 @@ def format_output_compare(self, key, left, right):
         else:
             # Fallback repr:
             self.comparison_traceback.append(
-                + "  <<<<<<<<<<<< Reference output from ipynb file:"
+                "  <<<<<<<<<<<< Reference output from ipynb file:"
                 + bcolors.ENDC)
             self.comparison_traceback.append(_indent(left))
             self.comparison_traceback.append(
@@ -786,7 +798,6 @@ def coalesce_streams(outputs):
 
     # process \r and \b characters
     for output in streams.values():
-        backspace_pat
         old = output.text
         while len(output.text) < len(old):
             old = output.text

tests/ipynb-test-samples/test-stream-fail-missingoutput.ipynb

@@ -0,0 +1,68 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "test\n"
+     ]
+    }
+   ],
+   "source": [
+    "print('test')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print('test')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "test\n"
+     ]
+    }
+   ],
+   "source": [
+    "print('test')"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

tests/test_unit_tests_in_notebooks.py

@@ -63,10 +63,15 @@ def test_print(filename, correctoutcome):
     else:
         exitcode = subprocess.call(command)
 
-    if correctoutcome is 'pass':
+    if correctoutcome == 'pass':
+        if exitcode != 0:
+            raise AssertionError("Tests failed on ipynb (expected pass)")
         assert exitcode is 0
         print("The call of py.test has not reported errors - this is good.")
-    elif correctoutcome is 'fail':
-        assert exitcode is not 0
+    elif correctoutcome == 'fail':
+        if exitcode == 0:
+            raise AssertionError("Tests passed on ipynb (expected fail)")
         print("The call of py.test has reported errors - " +
               "this is good for this test.")
+    else:
+        raise AssertionError("correctoutcome=%r (not 'pass' or 'fail')" % correctoutcome)