add assert given failure

Author: beachdweller

CHANGELOG.md

@@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 * MECE principle in comment generation
+* if failed, assert the error message
 
 
 ### Changed

ai_tutor.py

@@ -88,7 +88,7 @@ def gemini_qna(
         readme_file:pathlib.Path,
         api_key:str,
         explanation_in:str='Korean',
-    ) -> str:
+    ) -> Tuple[int, str]:
     '''
     Queries the Gemini API to provide explanations for failed pytest test cases.
 
@@ -105,7 +105,7 @@ def gemini_qna(
     logging.info(f"Student files: {student_files}")
     logging.info(f"Readme file: {readme_file}")
 
-    consolidated_question = get_prompt(
+    n_failed, consolidated_question = get_prompt(
         report_paths,
         student_files,
         readme_file,
@@ -114,17 +114,20 @@ def gemini_qna(
 
     answers = ask_gemini(consolidated_question, api_key)
 
-    return answers
+    return n_failed, answers
 
 
 def get_prompt(
         report_paths:Tuple[pathlib.Path],
         student_files:Tuple[pathlib.Path],
         readme_file:pathlib.Path,
         explanation_in:str,
-    ) -> str:
+    ) -> Tuple[int, str]:
     pytest_longrepr_list = collect_longrepr_from_multiple_reports(report_paths, explanation_in)
 
+    n_failed_tests = len(pytest_longrepr_list)
+
+
     def get_initial_instruction(questions:List[str],language:str) -> str:
         # Add the main directive or instruction based on whether there are failed tests
         if questions:
@@ -153,7 +156,7 @@ def get_initial_instruction(questions:List[str],language:str) -> str:
     # Join all questions into a single string
     prompt_str = "\n\n".join(prompt_list)
 
-    return prompt_str
+    return n_failed_tests, prompt_str
 
 
 def collect_longrepr_from_multiple_reports(pytest_json_report_paths:Tuple[pathlib.Path], explanation_in:str) -> List[str]:

entrypoint.py

@@ -33,7 +33,7 @@ def main() -> None:
 
     explanation_in = os.environ['INPUT_EXPLANATION-IN']
 
-    feedback = ai_tutor.gemini_qna(
+    n_failed, feedback = ai_tutor.gemini_qna(
         report_files,
         student_files,
         readme_file,
@@ -48,6 +48,8 @@ def main() -> None:
         out_string = f'feedback<<EOF\n{feedback}\nEOF'
         logging.info(f"Writing to GITHUB_OUTPUT: {f.write(out_string)} characters")
 
+    assert n_failed == 0, f'{n_failed} failed tests'
+
 
 def get_path_tuple(report_files_str:str) -> Tuple[pathlib.Path]:
     """

tests/test_ai_tutor.py

@@ -246,22 +246,25 @@ def test_get_prompt__has__homework__msg__instruction(
         student_files=(sample_student_code_path,),
         readme_file=sample_readme_path,
         explanation_in=explanation_in,
-    ).lower()
+    )
+
+    n_failed = result[0]
+    prompe_text = result[1].lower()
 
     assert any(
         map(
-            lambda x: x in result,
+            lambda x: x in prompe_text,
             homework
         )
     )
-    assert msg in result
+    assert msg in prompe_text
 
     assert any(
         map(
-            lambda x: x in result,
+            lambda x: x in prompe_text,
             instruction
         )
-    ), f"Could not find instruction: {instruction} in result: {result}."
+    ), f"Could not find instruction: {instruction} in result: {prompe_text}."
 
 
 def test_load_locale(explanation_in:str, homework:Tuple[str]):

tests/test_integration.py

@@ -43,6 +43,9 @@ def test_main_argument_passing__all_exists(mock_gemini_qna, caplog, tmp_path) ->
     path_readme.touch()
     os.environ['INPUT_README-PATH'] = str(path_readme)
 
+    # mock return value
+    mock_gemini_qna.return_value = (0, "This is the feedback message.") 
+
     entrypoint.main()
 
     mock_gemini_qna.assert_called_once_with(