Merge pull request #5 from JMU-CS/more_output_assertions

more output assertions

Author: spragunr

README.md

@@ -11,7 +11,7 @@ Features include:
     * provides a `@required` annotation that makes a test required, so
       that subsequent tests will automatically fail if it fails.
     * provide several additional assertion methods, including an
-      `assertScriptOutputEqual` method that makes it possible to specify
+      `assertOutputEqual` method that makes it possible to specify
       stdin input and check for correct stdout output for a Python
       script.
 *    Some additional utility functions for checking PEP 8 compliance

doc/jmu_test_case.rst

@@ -14,13 +14,3 @@ JmuTestCase
    :members:
    :private-members:
    :undoc-members:
-
-.. automodule:: jmu_gradescope_utils.utils
-   :members:
-   :private-members:
-   :undoc-members:
-
-.. automodule:: jmu_gradescope_utils.coverage_utils
-   :members:
-   :private-members:
-   :undoc-members:

examples/hello_world/tests/test_hello.py

@@ -24,4 +24,5 @@ def test_functionality(self):
 
         string_in = ""
         expected = "Hello World!\n"
-        self.assertScriptOutputEqual('hello_world.py', string_in, expected)
+        self.assertOutputEqual('hello_world.py', string_in, expected)
+        print('Correct output:\n' + expected)

examples/test_output_assertions/config.ini

@@ -0,0 +1,3 @@
+[SUBMIT]
+code: hello_world.py
+tests:

examples/test_output_assertions/configurations/docstring.cfg

@@ -0,0 +1,10 @@
+[flake8]
+select = D,DAR
+ignore = D401
+
+[pydocstyle]
+convention=google
+
+[darglint]
+docstring_style=google
+

examples/test_output_assertions/configurations/flake8.cfg

@@ -0,0 +1,4 @@
+[flake8]
+select = E, F, C90
+ignore = W, E226, E123, W504, N818, E704, E24, E121, E126
+max-line-length = 100

examples/test_output_assertions/sample/hello_world.py

@@ -0,0 +1 @@
+print("Hello World!")

examples/test_output_assertions/tests/test_hello.py

@@ -0,0 +1,73 @@
+import unittest
+from gradescope_utils.autograder_utils.decorators import weight, number
+from jmu_gradescope_utils import JmuTestCase, required, check_submitted_files
+import jmu_gradescope_utils
+
+
+def to_lower(s):
+    return s.lower()
+
+
+class TestHelloWorld(JmuTestCase):
+    @required()
+    @weight(0)
+    def test_submitted_files(self):
+        """Check submitted files"""
+        missing_files = check_submitted_files(["hello_world.py"])
+        for path in missing_files:
+            print("Missing {0}".format(path))
+        self.assertEqual(len(missing_files), 0, "Missing some required files!")
+        print("All required files submitted!")
+
+    @required()
+    @weight(1)
+    def test_output_retrieved(self):
+        """hello_world.py output retrieved."""
+        string_in = ""
+        expected = "Hello World!\n"
+        actual = self.getScriptOutput("hello_world.py", string_in)
+        self.assertEqual(
+            expected, actual["stdout"], "didn't get expected output from script"
+        )
+        print("Correct output equal:\n" + expected)
+
+    @weight(1)
+    def test_output_exactly(self):
+        """hello_world.py output equal."""
+        string_in = ""
+        expected = "Hello World!\n"
+        self.assertOutputEqual("hello_world.py", string_in, expected)
+        print("Correct output equal:\n" + expected)
+
+    @weight(1)
+    def test_output_not_equal(self):
+        """hello_world.py output not equal."""
+        string_in = ""
+        expected = "Incorrect\n"
+        self.assertOutputNotEqual("hello_world.py", string_in, expected)
+        print("Correct output not equal:\n" + expected)
+
+    @weight(1)
+    def test_output_contains(self):
+        """hello_world.py output contains."""
+        string_in = ""
+        expected = "hello"
+        self.assertInOutput("hello_world.py", string_in, expected, processor=to_lower)
+        print("Correct output contains:\n" + expected)
+
+    @weight(1)
+    def test_output_not_containing(self):
+        """hello_world.py output not contains."""
+        string_in = ""
+        expected_not_in = "hola"
+        self.assertNotInOutput("hello_world.py", string_in, expected_not_in)
+        try:
+            expected_in = "orld"
+            self.assertNotInOutput("hello_world.py", string_in, expected_in)
+        except Exception:
+            print("assertNotIn failed correctly")
+        else:
+            self.fail(
+                'assertNotIn must be broken, "orld" should have been considered to be in "Hello World!" and so asserting not in should have failed, but this error means it didnt'
+            )
+        print("Correct output not containing:\n" + expected_not_in)

jmu_gradescope_utils/jmu_test_case.py

@@ -20,7 +20,6 @@
 import sys
 from importlib import import_module
 
-
 _TEST_ORDER = {}
 
 
@@ -50,6 +49,7 @@ def wrapper(self, *args, **kwargs):
         else:
             result = func(self, *args, **kwargs)
         return result
+
     return wrapper
 
 
@@ -58,6 +58,7 @@ def required():
     failed then all of the following methods will fail as well.
 
     """
+
     def decorator(func):
         @wraps(func)
         def wrapper(self, *args, **kwargs):
@@ -68,7 +69,9 @@ def wrapper(self, *args, **kwargs):
                 result = None
                 raise e.__class__(str(e) + "\n This test was required.  All of the following tests will fail automatically.")
             return result
+
         return wrapper
+
     return decorator
 
 
@@ -92,33 +95,33 @@ def __new__(cls, name, bases, local):
 class _JmuTestCase(unittest.TestCase):
     """Additional useful assertions for grading.
 
-    This is the superclass for JmuTestCase.  Users should subclass
+    This is the superclass for ``JmuTestCase``.  Users should subclass
     ``JmuTestCase`` in their test code. """
 
     # counts the number of dynamic modules created
     module_count = 0
 
-    def assertScriptOutputEqual(self, filename, string_in, expected,
-                                variables=None, args="", msg=None,
-                                processor=None):
-        """Assert correct output for the provided Python script.
+    def getScriptOutput(self, filename, string_in, variables=None, args="",
+                        msg=None, processor=None, only_output=False):
+        """Get output for the provided Python script.
 
         Args:
             filename (str): The name of the Python file to test
             string_in (str): A string that will be fed to stdin for the script
-            expected (str): Expected stdout
             variables (dict): A dictionary mapping from variable names to
                 values. The script will be edited with these
                 substitutions before it is executed.
             args (str):  Command line arguments that will be passed to the script.
             msg (str):  Error message that will be printed if the assertion fails.
             processor (func):  A function mapping from string to string that will
-                process the script output before it is compared
-                to the expected output.
+                process the script output before it is returned.
+            only_output (bool): Return only the stdout (rather than also the stderr).
 
-        Raises:
-            AssertionError: If the expected output doesn't match the actual
-                output.
+        Returns:
+            dict: keys include 'stdout', and 'msg' as well as 'stderr' if there
+            there was any output on stderr. 'stdout' and 'stderr' are the script
+            output and 'msg' is a formatted string describing any execution errors
+            as well as the inputs to the script.
 
         """
         tmpdir = None
@@ -145,29 +148,108 @@ def assertScriptOutputEqual(self, filename, string_in, expected,
 
             if processor:
                 actual_text = processor(actual_text)
+                if only_output:
+                    return {"stdout": actual_text}
             stderr_text = stderr.decode()
 
             if len(stderr) > 0:
-                stderr_text = stderr_text.replace(utils.full_source_path() + "/" , '')
+                stderr_text = stderr_text.replace(utils.full_source_path() + "/", '')
                 err_msg = "Error during script execution:\n{}".format(stderr_text)
                 out_msg = "\nOutput before failure:\n{}".format(actual_text)
-                self.fail(err_msg + out_msg)
+                return {"stdout": actual_text, "stderr": stderr_text, "msg": err_msg + out_msg}
 
             show_in = string_in.encode('unicode_escape').decode()
             message = "Input was: '{}'".format(show_in)
             if len(args) > 0:
                 message += "\nCommand line arguments: {}".format(args)
             if msg is not None:
                 message += "\n" + msg
-
-            self.assertEqual(actual_text, expected, message)
+            return {"stdout": actual_text, "msg": message}
         finally:
             if tmpdir is not None:
                 # Restore the original submission in source:
                 shutil.copy(os.path.join(tmpdir, "__tmp_backup.py"),
                             utils.full_source_path(filename))
                 shutil.rmtree(tmpdir)
 
+    def assertScriptOutputEqual(self, filename, string_in, expected,
+                                variables=None, args="", msg=None,
+                                processor=None):
+        """Deprecated wrapper for :meth:`~jmu_gradescope_utils.jmu_test_case._JmuTestCase.assertOutputEqual`."""
+        self.assertOutputEqual(filename, string_in, expected, variables, args,
+                               msg, processor)
+
+    def assertOutputEqual(self, filename, string_in, expected,
+                          variables=None, args="", msg=None,
+                          processor=None):
+        """Assert correct output for the provided Python script.
+
+        Args:
+            filename (str): The name of the Python file to test
+            string_in (str): A string that will be fed to stdin for the script
+            expected (str): Expected stdout
+            variables (dict): A dictionary mapping from variable names to
+                values. The script will be edited with these
+                substitutions before it is executed.
+            args (str):  Command line arguments that will be passed to the script.
+            msg (str):  Error message that will be printed if the assertion fails.
+            processor (func):  A function mapping from string to string that will
+                process the script output before it is compared
+                to the expected output.
+
+        Raises:
+            AssertionError: If the expected output doesn't match the actual
+                output.
+
+        """
+        result = self.getScriptOutput(filename, string_in, variables=variables,
+                                      args=args, msg=msg, processor=processor)
+        if "stderr" in result:
+            self.fail(result["msg"])
+        self.assertEqual(result["stdout"], expected, result["msg"])
+
+    def assertOutputNotEqual(self, filename, string_in, expected,
+                             variables=None, args="", msg=None,
+                             processor=None):
+        """Assert script output is NOT equal to the indicated string.
+
+        See :meth:`~jmu_gradescope_utils.jmu_test_case._JmuTestCase.assertOutputEqual` for
+        description of arguments.
+        """
+        result = self.getScriptOutput(filename, string_in, variables=variables,
+                                      args=args, msg=msg, processor=processor)
+        if "stderr" in result:
+            self.fail(result["msg"])
+        self.assertNotEqual(result["stdout"], expected, result["msg"])
+
+    def assertInOutput(self, filename, string_in, expected,
+                       variables=None, args="", msg=None,
+                       processor=None):
+        """Assert script output contains the indicated string.
+
+        See :meth:`~jmu_gradescope_utils.jmu_test_case._JmuTestCase.assertOutputEqual` for
+        description of arguments.
+        """
+        result = self.getScriptOutput(filename, string_in, variables=variables,
+                                      args=args, msg=msg, processor=processor)
+        if "stderr" in result:
+            self.fail(result["msg"])
+        self.assertIn(expected, result["stdout"], result["msg"])
+
+    def assertNotInOutput(self, filename, string_in, expected,
+                          variables=None, args="", msg=None,
+                          processor=None):
+        """Assert script output does not contain the indicated string.
+
+        See :meth:`~jmu_gradescope_utils.jmu_test_case._JmuTestCase.assertOutputEqual` for
+        description of arguments.
+        """
+        result = self.getScriptOutput(filename, string_in, variables=variables,
+                                      args=args, msg=msg, processor=processor)
+        if "stderr" in result:
+            self.fail(result["msg"])
+        self.assertNotIn(expected, result["stdout"], result["msg"])
+
     def assertNoLoops(self, filename, msg=None):
         """ Assert that the provided script has no for or while loops.
 
@@ -312,13 +394,13 @@ def assertRequiredFilesPresent(self, required_files):
 
     def assertOutputCorrect(self, filename, string_in, expected,
                             variables=None, processor=None):
-        """ Wrapper for assertScriptOutputEqual.
+        """ Wrapper for assertOutputEqual.
 
         I'm not sure why this exists. -NRS
 
         """
-        self.assertScriptOutputEqual(filename, string_in, expected,
-                                     variables=variables, processor=processor)
+        self.assertOutputEqual(filename, string_in, expected,
+                               variables=variables, processor=processor)
         print('Correct output:\n' + expected)
 
     def run_with_substitution(self, filename, variables, func):
@@ -328,7 +410,9 @@ def run_with_substitution(self, filename, variables, func):
         if filename[-3:] == '.py':
             short_filename = filename[0:-3]
         new_module_name = short_filename + "_" + str(_JmuTestCase.module_count)
-        (tmpdir, new_file_name) = utils.replace_variables(filename, variables=variables, new_name=new_module_name + ".py")
+        (tmpdir, new_file_name) = utils.replace_variables(filename,
+                                                          variables=variables,
+                                                          new_name=new_module_name + ".py")
         # insert the new temporary directory into the system module load path
         sys.path.insert(1, tmpdir)
         # load the module
@@ -359,12 +443,11 @@ def assertMatchCount(self, filename, regex, num_matches, msg=None):
 
 class JmuTestCase(_JmuTestCase, metaclass=OrderAllTestsMeta):
     """Test methods declared within subclasses will be executed in the
-    order they are declared as long as the sortTestMethodUsing attrute
+    order they are declared as long as the ``sortTestMethodUsing`` attribute
     of the defaultTestLoader has been set::
+        unittest.defaultTestLoader.sortTestMethodsUsing = test_compare
 
-    unittest.defaultTestLoader.sortTestMethodsUsing = test_compare
-
-    They will also respect the @required decorator.
+    They will also respect the ``@required`` decorator.
 
     """
     pass

jmu_gradescope_utils/utils.py

@@ -121,11 +121,11 @@ def check_submitted_files(paths, base=SUBMISSION_BASE):
 def suppress_IO(in_string):
     """
     Suppresses standard io when running a block of code, feeding in the given in_string as input
-    and squelching all output. Use as
+    and squelching all output. Use as::
 
-    ```with suppress_IO("desired input"):
+        with suppress_IO("desired input"):
             # my code block
-    ```
+
     """
     text_in = io.StringIO(in_string)
     text_out = io.StringIO()