Merge pull request #167 from datacamp/fix-test-funciton-highlight

Fix test function highlight *decision needed*

Author: Filip Schouwenaars

pythonwhat/State.py

@@ -102,7 +102,7 @@ def __init__(self,
         self.student_context  = Context(student_context)  if student_context is None else student_context
         self.solution_context = Context(solution_context) if solution_context is None else solution_context
 
-        self.highlight = self.student_tree if highlight is None else highlight
+        self.highlight = self.student_tree if (not highlight) and self.parent_state else highlight
 
         self.converters = get_manual_converters()    # accessed only from root state
 

pythonwhat/test_funcs/test_function.py

@@ -19,6 +19,7 @@ def test_function(name,
                   args_not_specified_msg=None,
                   incorrect_msg=None,
                   add_more=False,
+                  highlight=False,
                   state=None):
     """Test if function calls match.
 
@@ -128,7 +129,7 @@ def test_function(name,
                 if feedback is None:
                     if not args_not_specified_msg:
                         args_not_specified_msg = dflt
-                    feedback = Feedback(args_not_specified_msg, student_call)
+                    feedback = Feedback(args_not_specified_msg, student_call if highlight else None)
                 success = False
                 continue
 
@@ -148,7 +149,8 @@ def test_function(name,
 
                 test = build_test(arg_student, arg_solution,
                                   state,
-                                  do_eval, eq_fun, msg, add_more=add_more)
+                                  do_eval, eq_fun, msg, add_more=add_more,
+                                  highlight=arg_student if highlight else None)
                 test.test()
 
                 if not test.result:
@@ -170,7 +172,8 @@ def test_function(name,
 
                     test = build_test(key_student, key_solution,
                                       state,
-                                      do_eval, eq_fun, msg, add_more=add_more)
+                                      do_eval, eq_fun, msg, add_more=add_more,
+                                      highlight=key_student if highlight else None)
                     test.test()
 
                     if not test.result:
@@ -197,6 +200,7 @@ def test_print(index = 1,
                params_not_matched_msg="Have you correctly called `print()`?",
                params_not_specified_msg="Have you correctly called `print()`?",
                incorrect_msg="Have you printed out the correct object?",
+               highlight=False,
                state=None):
     test_function_v2("print",
                      index=index,
@@ -207,7 +211,8 @@ def test_print(index = 1,
                      not_called_msg=not_called_msg,
                      params_not_matched_msg=params_not_matched_msg,
                      params_not_specified_msg=params_not_specified_msg,
-                     incorrect_msg=incorrect_msg, state=state)
+                     incorrect_msg=incorrect_msg,
+                     highlight=highlight, state=state)
     """Test print() calls
 
     Utility function to test the print() function. For arguments, check test_function_v2()
@@ -224,6 +229,7 @@ def test_function_v2(name,
                      params_not_specified_msg=None,
                      incorrect_msg=None,
                      add_more=False,
+                     highlight=False,
                      state=None):
     """Test if function calls match (v2).
 
@@ -342,7 +348,7 @@ def test_function_v2(name,
         sub_tests = [partial(test_call, name, call_ind, signature, params, do_eval, solution_args, 
                              eq_fun, add_more, index,
                              params_not_specified_msg, params_not_matched_msg, incorrect_msg, 
-                             keywords, state=state)
+                             keywords, state=state, highlight = highlight)
                      for call_ind in call_indices]
         test_or(*sub_tests, state=state)
 
@@ -356,7 +362,7 @@ def test_call(name, call_ind, signature, params, do_eval, solution_args,
               eq_fun, add_more, index,
               params_not_specified_msg, params_not_matched_msg, incorrect_msg, 
               keywords,  # pulled from solution process
-              state):
+              state, highlight):
     #stud_name = get_mapped_name(name, state.student_mappings)
 
     rep = Reporter.active_reporter
@@ -374,7 +380,7 @@ def test_call(name, call_ind, signature, params, do_eval, solution_args,
             params_not_matched_msg = ("Something went wrong in figuring out how you specified the " + \
                 "arguments for `%s()`; have another look at your code and its output.") % stud_name
         _msg = state.build_message(params_not_matched_msg)
-        feedback = Feedback(_msg, student_call)
+        feedback = Feedback(_msg, student_call if highlight else None)
         # run subtest
         rep.do_test(Test(feedback))    # TODO: sub_call
 
@@ -392,7 +398,7 @@ def test_call(name, call_ind, signature, params, do_eval, solution_args,
         else:
             msg = params_not_specified_msg[param_ind]
         _msg = state.build_message(msg)
-        feedback = Feedback(_msg, student_call)
+        feedback = Feedback(_msg, student_call if highlight else None)
         # run subtest
         rep.do_test(Test(feedback))    # TODO: sub_call
 
@@ -405,15 +411,15 @@ def test_call(name, call_ind, signature, params, do_eval, solution_args,
         test_arg(param, do_eval[ind],
                  arg_student, arg_solution, param_kind, stud_name,
                  eq_fun, add_more,
-                 incorrect_msg[ind], state=state)
+                 incorrect_msg[ind], state=state, highlight = arg_student if highlight else None)
 
     # If all is still good, we have a winner!
     state.set_used(name, call_ind, index)
 
 def test_arg(param, do_eval, 
              arg_student, arg_solution, param_kind, stud_name,
              eq_fun, add_more,
-             incorrect_msg, state=None):
+             incorrect_msg, state=None, highlight = None):
     rep = Reporter.active_reporter
 
     if incorrect_msg is None:
@@ -426,7 +432,7 @@ def test_arg(param, do_eval,
 
     test = build_test(arg_student, arg_solution,
                         state,
-                        do_eval, eq_fun, msg, add_more = add_more)
+                        do_eval, eq_fun, msg, add_more = add_more, highlight=highlight)
     # TODO
     rep.do_test(test)
 
@@ -443,7 +449,7 @@ def bind_args(signature, arguments, keyws):
     bound_args = signature.bind(*arguments, **keyws)
     return(bound_args.arguments, signature.parameters)
 
-def build_test(stud, sol, state, do_eval, eq_fun, feedback_msg, add_more):
+def build_test(stud, sol, state, do_eval, eq_fun, feedback_msg, add_more, highlight = False):
     got_error = False
     if do_eval:
 
@@ -472,8 +478,8 @@ def build_test(stud, sol, state, do_eval, eq_fun, feedback_msg, add_more):
         eval_solution = ast.dump(sol)
 
     _msg = state.build_message(feedback_msg)
-    return(Test(Feedback(_msg, stud)) if got_error else
-        eq_fun(eval_student, eval_solution, Feedback(_msg, stud)))
+    return(Test(Feedback(_msg, stud if highlight else None)) if got_error else
+        eq_fun(eval_student, eval_solution, Feedback(_msg, stud if highlight else None)))
 
 
 

tests/test_test_comp.py

@@ -15,8 +15,8 @@ def setUp(self):
                iter_vars_names=True,
                incorrect_iter_vars_msg=None,
                body=lambda: test_expression_result(context_vals = ['a', 2]),
-               ifs=[lambda: test_function_v2('isinstance', params = ['obj'], do_eval = [False]),
-                    lambda: test_function_v2('isinstance', params = ['obj'], do_eval = [False])],
+               ifs=[lambda: test_function_v2('isinstance', params = ['obj'], do_eval = [False], highlight=True),
+                    lambda: test_function_v2('isinstance', params = ['obj'], do_eval = [False], highlight=True)],
                insufficient_ifs_msg=None,
                expand_message=True)
             '''
@@ -96,8 +96,8 @@ def test_pass_mix_lam(self):
                iter_vars_names=True,
                incorrect_iter_vars_msg=None,
                body=test_expression_result(context_vals = ['a', 2]),
-               ifs=[test_function_v2('isinstance', params = ['obj'], do_eval = [False]),
-                    test_function_v2('isinstance', params = ['obj'], do_eval = [False])],
+               ifs=[test_function_v2('isinstance', params = ['obj'], do_eval = [False], highlight=True),
+                    test_function_v2('isinstance', params = ['obj'], do_eval = [False], highlight=True)],
                insufficient_ifs_msg=None,
                expand_message=True)
             '''
@@ -124,8 +124,8 @@ def setUp(self):
                iter_vars_names=True,
                incorrect_iter_vars_msg='incorrectitervars',
                body=lambda: test_expression_result(context_vals = ['a', 2], incorrect_msg = 'bodyincorrect'),
-               ifs=[lambda: test_function_v2('isinstance', params = ['obj'], do_eval = [False], not_called_msg = 'notcalled1', incorrect_msg = 'incorrect2'),
-                    lambda: test_function_v2('isinstance', params = ['obj'], do_eval = [False], not_called_msg = 'notcalled2', incorrect_msg = 'incorrect2')],
+               ifs=[lambda: test_function_v2('isinstance', params = ['obj'], do_eval = [False], not_called_msg = 'notcalled1', incorrect_msg = 'incorrect2', highlight=True),
+                    lambda: test_function_v2('isinstance', params = ['obj'], do_eval = [False], not_called_msg = 'notcalled2', incorrect_msg = 'incorrect2', highlight=True)],
                insufficient_ifs_msg='insufficientifs')
             '''
         }
@@ -311,7 +311,7 @@ def setUp(self):
                incorrect_iter_vars_msg=None,
                key=lambda: test_expression_result(context_vals = ['a']),
                value=lambda: test_expression_result(context_vals = ['a']),
-               ifs=[lambda: test_function_v2('isinstance', params = ['obj'], do_eval = [False])],
+               ifs=[lambda: test_function_v2('isinstance', params = ['obj'], do_eval = [False], highlight=True)],
                insufficient_ifs_msg=None,
                expand_message=True)
             '''
@@ -395,8 +395,8 @@ def setUp(self):
                    iter_vars_names=True,
                    incorrect_iter_vars_msg=None,
                    body=lambda: test_expression_result(context_vals = ['a', 2]),
-                   ifs=[lambda: test_function_v2('isinstance', params = ['obj'], do_eval = [False]),
-                        lambda: test_function_v2('isinstance', params = ['obj'], do_eval = [False])],
+                   ifs=[lambda: test_function_v2('isinstance', params = ['obj'], do_eval = [False], highlight=True),
+                        lambda: test_function_v2('isinstance', params = ['obj'], do_eval = [False], highlight=True)],
                    insufficient_ifs_msg=None,
                    expand_message=True)
             '''

tests/test_test_for_loop.py

@@ -15,7 +15,7 @@ def setUp(self):
 ''',
             "DC_SCT": '''
 test_for_loop(1,
-              lambda: test_function("range"),
+              lambda: test_function("range", highlight=True),
               lambda: test_object_after_expression("size", {"size": 1}, [1]))
 success_msg("Great!")
 '''
@@ -88,7 +88,7 @@ def setUp(self):
             ''',
             "DC_SCT": '''
 msg = "loopinggonewrong"
-test_for_loop(1, for_iter=lambda msg=msg: test_function("enumerate", incorrect_msg = msg))
+test_for_loop(1, for_iter=lambda msg=msg: test_function("enumerate", incorrect_msg = msg, highlight=True))
 
 msg = "blabla"
 test_for_loop(1, body=lambda msg=msg: test_expression_output(incorrect_msg = msg, context_vals = [2, "test"]))
@@ -137,7 +137,7 @@ def test_Pass_spec(self):
 forl = Ex().check_for_loop(0)
 
 forl.check_iter()\
-    .multi(test_function("enumerate", incorrect_msg=msg))
+    .multi(test_function("enumerate", incorrect_msg=msg, highlight=True))
 
 msg = "blabla"
 forl.check_body()\
@@ -165,7 +165,7 @@ def setUp(self):
         .check_for_loop(0)\
         .check_body()\
         .set_context(jj=2)\
-        .multi(test_function('sum', incorrect_msg="wronginnerfor"))
+        .multi(test_function('sum', incorrect_msg="wronginnerfor", highlight=True))
         '''
         }
 

tests/test_test_function.py

@@ -458,7 +458,7 @@ def test_line_numbers1(self):
         self.data = {"DC_PEC": '',
                      "DC_SOLUTION": "round(1.23456, ndigits = 1)",
                      "DC_CODE": "round(1.34567, ndigits = 1)",
-                     "DC_SCT": "test_function('round', index = 1)"}
+                     "DC_SCT": "test_function('round', index = 1, highlight=True)"}
         sct_payload = helper.run(self.data)
         self.assertFalse(sct_payload['correct'])
         self.assertIn("Did you call <code>round()</code> with the correct arguments? The first argument seems to be incorrect.", sct_payload['message'])
@@ -468,7 +468,7 @@ def test_line_numbers(self):
         self.data = {"DC_PEC": '',
                      "DC_SOLUTION": "round(1.23456, ndigits = 1)",
                      "DC_CODE": "round(1.23456, ndigits = 3)",
-                     "DC_SCT": "test_function('round', index = 1)"}
+                     "DC_SCT": "test_function('round', index = 1, highlight=True)"}
         sct_payload = helper.run(self.data)
         self.assertFalse(sct_payload['correct'])
         self.assertIn("Did you call <code>round()</code> with the correct arguments? Keyword <code>ndigits</code> seems to be incorrect.", sct_payload['message'])
@@ -488,7 +488,7 @@ def test_nested_arg2(self):
         self.data = {"DC_PEC": '',
                      "DC_SOLUTION": "print(type([1, 2, 3]))",
                      "DC_CODE": "print(type([1, 2, 4]))",
-                     "DC_SCT": "test_function('type')"}
+                     "DC_SCT": "test_function('type', highlight=True)"}
         sct_payload = helper.run(self.data)
         self.assertFalse(sct_payload['correct'])
         helper.test_lines(self, sct_payload, 1, 1, 12, 20)
@@ -497,15 +497,15 @@ def test_nested_keyw1(self):
         self.data = {"DC_PEC": '',
                      "DC_SOLUTION": "round(1.1234, ndigits = max([1, 2, 3]))",
                      "DC_CODE": "round(1.1234, ndigits = max([1, 2, 3]))",
-                     "DC_SCT": "test_function('max')"}
+                     "DC_SCT": "test_function('max', highlight=True)"}
         sct_payload = helper.run(self.data)
         self.assertTrue(sct_payload['correct'])
 
     def test_nested_keyw2(self):
         self.data = {"DC_PEC": '',
                      "DC_SOLUTION": "round(1.1234, ndigits = max([1, 2, 3]))",
                      "DC_CODE": "round(1.1234, ndigits = max([1, 2]))",
-                     "DC_SCT": "test_function('max')"}
+                     "DC_SCT": "test_function('max', highlight=True)"}
         sct_payload = helper.run(self.data)
         self.assertFalse(sct_payload['correct'])
         helper.test_lines(self, sct_payload, 1, 1, 29, 34)
@@ -558,7 +558,7 @@ def test_do_eval_none_fail1(self):
         self.data = {"DC_PEC": '',
              "DC_SOLUTION": "round(123.123, ndigits = 2)",
              "DC_CODE": "round(123.123)",
-             "DC_SCT": "test_function('round', do_eval = None)"}
+             "DC_SCT": "test_function('round', do_eval = None, highlight=True)"}
         sct_payload = helper.run(self.data)
         self.assertFalse(sct_payload['correct'])
         self.assertEqual("Have you specified all required arguments inside <code>round()</code>?", sct_payload['message'])
@@ -568,7 +568,7 @@ def test_do_eval_none_fail2(self):
         self.data = {"DC_PEC": '',
              "DC_SOLUTION": "round(123.123)", # args = [0]
              "DC_CODE": "round(number = 123.123)", # student args is len 0
-             "DC_SCT": "test_function('round', do_eval = None)"}
+             "DC_SCT": "test_function('round', do_eval = None, highlight=True)"}
         sct_payload = helper.run(self.data)
         self.assertFalse(sct_payload['correct'])
         self.assertEqual("Have you specified all required arguments inside <code>round()</code>?", sct_payload['message'])
@@ -578,7 +578,7 @@ def test_do_eval_none_fail3(self):
         self.data = {"DC_PEC": '',
              "DC_SOLUTION": "round(123.123, 2)", # args = [0, 1]
              "DC_CODE": "round(123.123)", # student_args is len 1
-             "DC_SCT": "test_function('round', do_eval = None)"}
+             "DC_SCT": "test_function('round', do_eval = None, highlight=True)"}
         sct_payload = helper.run(self.data)
         self.assertFalse(sct_payload['correct'])
         self.assertEqual("Have you specified all required arguments inside <code>round()</code>?", sct_payload['message'])
@@ -594,9 +594,9 @@ def setUp(self):
 print([1, 2, 3])
             ''',
              "DC_SCT": '''
-test_function("print", index = 1)
-test_function("print", index = 2)
-test_function("print", index = 3)
+test_function("print", index = 1, highlight=True)
+test_function("print", index = 2, highlight=True)
+test_function("print", index = 3, highlight=True)
             '''
         }
         self.DC_SCT_SPEC2 = '''
@@ -626,6 +626,18 @@ def test_multiple_4(self):
         self.assertIn("Did you call <code>print()</code> with the correct arguments? The first argument seems to be incorrect.", sct_payload['message'])
         helper.test_lines(self, sct_payload, 1, 1, 7, 11)
 
+    def test_multiple_4_no_highlight(self):
+        self.data["DC_CODE"] = 'print("acb")\nprint(1234)\nprint([1, 2, 3])'
+        self.data["DC_SCT"] = """
+test_function("print", index = 1, highlight = False)
+test_function("print", index = 2)
+test_function("print", index = 3)
+        """
+        sct_payload = helper.run(self.data)
+        self.assertFalse(sct_payload['correct'])
+        self.assertIn("Did you call <code>print()</code> with the correct arguments? The first argument seems to be incorrect.", sct_payload['message'])
+        self.assertEqual(sct_payload.get('line_start'), None)
+
     def test_multiple_5(self):
         self.data["DC_CODE"] = 'print("abc")\nprint(1234)\nprint([1, 2, 3])'
         sct_payload = helper.run(self.data)
@@ -640,6 +652,13 @@ def test_multiple_6(self):
         self.assertIn("Did you call <code>print()</code> with the correct arguments? The first argument seems to be incorrect.", sct_payload['message'])
         helper.test_lines(self, sct_payload, 3, 3, 7, 18)
 
+    def test_nohighlight_too_few_calls(self):
+        self.data["DC_SCT"] = 'test_function("print", index = 3, args = [], keywords = [])\n' + self.data["DC_SCT"]
+        self.data["DC_CODE"] = 'print("abc")\nprint(1234)'
+        sct_payload = helper.run(self.data)
+        self.assertFalse(sct_payload['correct'])
+        self.assertEqual(sct_payload.get('line_start'), None)
+
 
 
 if __name__ == "__main__":