Merge pull request #164 from datacamp/spec-multi-with-gen

Spec multi with gen, add test_not and fail SCTs

Author: Filip Schouwenaars

pythonwhat/check_funcs.py

@@ -2,6 +2,7 @@
 from pythonwhat.Test import Test, EqualTest
 from pythonwhat.Feedback import Feedback
 from pythonwhat.utils import get_ord
+from types import GeneratorType
 from functools import partial
 import copy
 
@@ -136,6 +137,8 @@ def has_equal_part_len(name, insufficient_msg, state=None):
     return state
 
 
+# functions for running multiple sub-tests ------------------------------------
+
 def extend(*args, state=None):
     """Run multiple subtests in sequence, each using the output state of the previous."""
 
@@ -151,30 +154,51 @@ def multi(*args, state=None):
         rep = Reporter.active_reporter
 
         # when input is a single list of subtests
-        args = args[0] if len(args) == 1 and isinstance(args[0], (list, tuple)) else args
+        if len(args) == 1 and isinstance(args[0], (list, tuple, GeneratorType)):
+            args = args[0]
 
         for test in args:
             # assume test is function needing a state argument
             # partial state so reporter can test
-            # TODO: it seems clear the reporter doesn't need to hold state anymore
             closure = partial(test, state=state)
-            # message from parent checks
-            #prefix = state.build_message()
-            ## resetting reporter message until it can be refactored
-            #prev_msg = rep.failure_msg
             rep.do_test(closure, "", state.highlight)
-            #rep.failure_msg = prev_msg
 
     # return original state, so can be chained
     return state
 
+from pythonwhat.Test import TestFail
+
+def test_not(*args, msg, state=None):
+    """Pass if all of the subtests fail"""
+    rep = Reporter.active_reporter
+
+    try: multi(*args, state=state)
+    except TestFail as e:
+        rep.failed_test = False          # protect against old behavior
+        return state
+    
+    _msg = state.build_message(msg)
+    return rep.do_test(Test(msg))
+
+# utility functions -----------------------------------------------------------
+
 def quiet(n = 0, state=None):
     """Turn off prepended messages. Defaults to turning all off."""
     cpy = copy.copy(state)
     hushed = [{**m, 'msg': ""} for m in cpy.messages]
     cpy.messages = hushed
     return cpy
 
+def fail(msg="", state=None):
+    """Fail test with message"""
+    rep = Reporter.active_reporter
+    _msg = state.build_message(msg)
+    rep.do_test(Test(Feedback(msg, state.highlight)))
+
+    return state
+
+# context functions -----------------------------------------------------------
+
 from pythonwhat.tasks import setUpNewEnvInProcess, breakDownNewEnvInProcess
 def with_context(*args, state=None):
     # set up context in processes
@@ -209,18 +233,31 @@ def with_context(*args, state=None):
     return state
 
 def set_context(*args, state=None, **kwargs):
+    """Update context values for student and solution environments.
+    
+    Note that excess args and unmatched kwargs will be unused in the student environment.
+    If an argument is specified both by name and position args, will use named arg.
+    """
     stu_crnt = state.student_context.context
     sol_crnt = state.solution_context.context
-    # set args specified by pos ----
-    upd_stu = stu_crnt.update(dict(zip(sol_crnt.keys(), args)))
+    # set args specified by pos -----------------------------------------------
+    # stop if too many pos args for solution
+    if len(args) > len(sol_crnt): 
+        raise IndexError("Too many positional args. There are {} context vals, but tried to set {}"
+                            .format(len(sol_crnt), len(args)))
+    # set pos args
     upd_sol = sol_crnt.update(dict(zip(stu_crnt.keys(), args)))
+    upd_stu = stu_crnt.update(dict(zip(sol_crnt.keys(), args)))
 
-    # set args specified by keyword ----
+    # set args specified by keyword -------------------------------------------
+    if set(kwargs) - set(upd_sol):
+        raise KeyError("Context val names are {}, but tried to set {}"
+                            .format(upd_sol or "none", kwargs.keys()))
     out_sol = upd_sol.update(kwargs)
     # need to match keys in kwargs with corresponding keys in stu context
     # in case they used, e.g., different loop variable names
     match_keys = dict(zip(sol_crnt.keys(), stu_crnt.keys()))
-    out_stu = upd_stu.update({match_keys[k]: v for k,v in kwargs.items()})
+    out_stu = upd_stu.update({match_keys[k]: v for k,v in kwargs.items() if k in match_keys})
 
     return state.to_child_state(student_subtree = None, solution_subtree = None,
                                 student_context = out_stu, solution_context = out_sol)
@@ -304,8 +341,8 @@ def call(args,
     eval_sol, str_sol = run_call(args, state.solution_parts['node'], state.solution_process, get_func, **kwargs)
 
     if (test == 'error') ^ isinstance(str_sol, Exception):
-        _msg = state.build_message("FMT:Calling for arguments {args} resulted in an error (or not an error if testing for one). Error message: {str_sol}",
-                                   dict(args=args, str_sol=str_sol))
+        _msg = state.build_message("FMT:Calling for arguments {args} resulted in an error (or not an error if testing for one). Error message: {type_err} {str_sol}",
+                                   dict(args=args, type_err=type(str_sol), str_sol=str_sol))
         raise ValueError(_msg)
 
     if isinstance(eval_sol, ReprFail):
@@ -367,7 +404,7 @@ def has_expr(incorrect_msg="FMT:Unexpected expression {test}: expected `{sol_eva
 
     if (test == 'error') ^ isinstance(str_sol, Exception):
         raise ValueError("evaluating expression raised error in solution process (or not an error if testing for one). "
-                         "Error message: %s"%str_sol)
+                         "Error: %s - %s"%(type(str_sol), str_sol))
     if isinstance(eval_sol, ReprFail):
         raise ValueError("Couldn't figure out the value of a default argument: " + eval_sol.info)
 

pythonwhat/check_syntax.py

@@ -45,24 +45,31 @@ class Chain:
     def __init__(self, state):
         self._state = state
         self._crnt_sct = None
+        self._waiting_on_call = False
 
     def __getattr__(self, attr):
         if attr not in ATTR_SCTS: raise AttributeError("No SCT named %s"%attr)
+        elif self._waiting_on_call: 
+            raise AttributeError("Did you forget to call a statement? "
+                                 "e.g. Ex().check_list_comp.check_body()")
         else:
             # make a copy to return, 
             # in case someone does: a = chain.a; b = chain.b
             chain = copy.copy(self)
             chain._crnt_sct = ATTR_SCTS[attr]
+            chain._waiting_on_call = True
             return chain
 
     def __call__(self, *args, **kwargs):
         self._state = self._crnt_sct(state=self._state, *args, **kwargs)
+        self._waiting_on_call = False
         return self
 
 class F(Chain):
     def __init__(self, stack = None):
         self._crnt_sct = None
         self._stack = [] if stack is None else stack
+        self._waiting_on_call = False
 
     def __call__(self, *args, **kwargs):
         if not self._crnt_sct:
@@ -84,7 +91,7 @@ def Ex():
 # Prepare SCTs that may be chained attributes ----------------------
 # decorate functions that may try to run test_* function nodes as subtests
 # so they remove those nodes from the tree
-for k in ['multi', 'with_context']:
+for k in ['multi', 'with_context', 'test_not']:
     ATTR_SCTS[k] = multi_dec(ATTR_SCTS[k])
 # allow test_* functions as chained attributes
 for k in TEST_NAMES:

pythonwhat/check_wrappers.py

@@ -56,7 +56,7 @@
 
 for k in ['set_context', 
           'has_equal_value', 'has_equal_output', 'has_equal_error', 'call',
-          'extend', 'multi',
+          'extend', 'multi', 'test_not', 'fail', 'quiet',
           'with_context',
           'check_args',
           'has_equal_part']:

tests/test_instructor_warnings.py

@@ -10,5 +10,27 @@ def test_converter_err(self):
         data['DC_CODE'] = data['DC_SOLUTION']
         self.assertRaises(ValueError, lambda: helper.run(data))
 
+    def test_check_syntax_double_getattr(self):
+        data = {
+                "DC_SOLUTION": "",
+                "DC_CODE": "",
+                "DC_SCT": """Ex().check_list_comp.check_body()"""
+                }
+        self.assertRaises(AttributeError, lambda: helper.run(data))
+
+    def test_check_syntax_check_index_no_index(self):
+        data = {
+                "DC_SOLUTION": "[i for i in range(1)]",
+                "DC_CODE": "[i for i in range(1)]",
+                "DC_SCT": """Ex().check_list_comp()"""
+                }
+        self.assertRaises(TypeError, lambda: helper.run(data))
+
+    def test_context_vals_wrong_place_in_chain(self):
+        data = {"DC_SOLUTION": "[(i,j) for i,j in enumerate(range(10))]"}
+        data["DC_CODE"] = data["DC_SOLUTION"]
+        data["DC_SCT"] = """Ex().check_list_comp(0).set_context(i=1,j=2).check_iter()"""
+        self.assertRaises(KeyError, lambda: helper.run(data))
+
 if __name__ == "__main__":
     unittest.main()

tests/test_spec.py

@@ -135,7 +135,50 @@ def test_multi_splits_node_and_check(self):
         sct_payload = helper.run(self.data)
         self.assertTrue(sct_payload['correct'])
 
+    def test_multi_generator(self):
+        self.data["DC_SCT"] = """
+Ex().check_list_comp(0).check_body()\
+        .multi(set_context(aa=i).has_equal_value('wrong') for i in range(2))
+"""
+        sct_payload = helper.run(self.data)
+        self.assertTrue(sct_payload['correct'])
+
+class TestTestNot(unittest.TestCase):
+    def setUp(self):
+        self.data = {
+                "DC_SOLUTION": "x = 1",
+                "DC_CODE": "x = 1"
+                }
+
+    def test_pass(self):
+        self.data["DC_SCT"] = """Ex().test_not(check_list_comp(0), msg="no")"""
+        sct_payload = helper.run(self.data)
+        self.assertTrue(sct_payload['correct'])
+
+    def test_fail(self):
+        # obviously this would be a terrible sct...
+        self.data["DC_SCT"] = """Ex().test_not(test_object('x'), msg="no")"""
+        sct_payload = helper.run(self.data)
+        self.assertFalse(sct_payload['correct'])
+
+class TestTestFail(unittest.TestCase):
+    def setUp(self):
+        self.data = {
+                "DC_SOLUTION": "", "DC_CODE": ""
+                }
+
+    def test_fail(self):
+        self.data["DC_SCT"] = """Ex().fail()"""
+        sct_payload = helper.run(self.data)
+        self.assertFalse(sct_payload['correct'])
+
 
+#class TestSetContext(unittest.TestCase):
+#    def setUp(self):
+#        self.data = {
+#                "DC_PEC": "",
+#                "DC_SOLUTION": "[(i,j) for i,j in enumerate(range(10))]"
+#                }
 
 if __name__ == "__main__":
     unittest.main()