Merge pull request #265 from sbintuitions/add-flexeval-lm-exit-code

Add exit code to flexeval_lm

Author: moskomule

flexeval/scripts/flexeval_lm.py

@@ -261,6 +261,7 @@ def main() -> None:  # noqa: C901, PLR0912, PLR0915
                 )
             )
 
+    exit_code = os.EX_OK
     # run evaluation
     for eval_setup, eval_setup_config, group in eval_setups_and_metadata:
         logger.info(f"Evaluating with the setup: {eval_setup_config}")
@@ -305,6 +306,9 @@ def main() -> None:  # noqa: C901, PLR0912, PLR0915
             logger.error(
                 f"Error in evaluation:\n{e}\n{stack_trace_str}",
             )
+            exit_code = os.EX_DATAERR
+
+    sys.exit(exit_code)
 
 
 if __name__ == "__main__":

tests/scripts/test_flexeval_lm.py

@@ -234,7 +234,7 @@ def test_if_cli_raises_error_when_save_data_exists(command: list[str]) -> None:
         config_file = Path(f) / CONFIG_FILE_NAME
         config_file.touch()
         result = subprocess.run([*command, "--save_dir", f], check=False)
-        assert result.returncode == os.EX_OK
+        assert result.returncode == os.EX_DATAERR
         # check if config_file is not modified and empty
         assert config_file.read_text() == ""