some fixes

Author: reisaraujo-miguel

Pipfile

@@ -7,6 +7,7 @@ name = "pypi"
 boto3 = "1.42.71"
 docker = "7.1.0"
 psycopg2-binary = "2.9.11"
+boto3-stubs = {extras = ["s3"], version = "*"}
 
 [dev-packages]
 

Pipfile.lock

@@ -1,4 +1,3 @@
 from rcc import main
 
-
-main()
+main()

src/main.py

@@ -1,17 +1,17 @@
 from __future__ import unicode_literals
 
-from six.moves import range
-
 import argparse
 import logging
 import logging.handlers
-import sys
 import multiprocessing as mp
+import sys
+
+from six.moves import range
+
 import rcc.config
 import rcc.engine
 import rcc.provider
 import rcc.provider.data
-import rcc.provider.storage
 import rcc.util
 
 
@@ -26,14 +26,16 @@ def parse_args():
     return parser.parse_args()
 
 
-def setup_logger(name, log_config, slack_config=None):
+def setup_logger(name, log_config):
     # NOTE: should we handle multiprocessing?
     logger = logging.getLogger(name)
     logger.setLevel(logging.DEBUG)
 
     # Console Logger
     console_handler = logging.StreamHandler(sys.stderr)
-    console_fmt = logging.Formatter("[%(asctime)s] %(module)s:%(lineno)d: <%(process)d> %(message)s")
+    console_fmt = logging.Formatter(
+        "[%(asctime)s] %(module)s:%(lineno)d: <%(process)d> %(message)s"
+    )
     console_handler.setFormatter(console_fmt)
     logger.addHandler(console_handler)
 
@@ -47,22 +49,11 @@ def setup_logger(name, log_config, slack_config=None):
         handler.setLevel(log_config["level"])
         handler.setFormatter(formatter)
         logger.addHandler(handler)
-    
-    # Slack Logger
-    if slack_config is not None:
-        fmt = "[%(asctime)s] %(module)s:%(lineno)d: <%(process)d> %(message)s"
-        formatter = logging.Formatter(fmt)
-        handler = rcc.util.SlackLogHandler(
-            slack_config["webhook_url"], slack_config["sender_name"]
-        )
-        handler.setLevel(slack_config["level"])
-        handler.setFormatter(formatter)
-        logger.addHandler(handler)
 
     return logger
 
 
-def commit_filter(commit):
+def commit_filter(_):
     return True
 
 
@@ -73,7 +64,7 @@ def main():
     else:
         cfg = rcc.config.from_json(rcc.config.DEFAULT_CONFIG, args.config)
 
-    logger = setup_logger(rcc.config.DEFAULT_LOGGER, cfg.log, cfg.slack)
+    logger = setup_logger(rcc.config.DEFAULT_LOGGER, cfg.log)
 
     with rcc.util.SingletonContext(cfg.lock_file):
         logger.info("Started")
@@ -89,7 +80,7 @@ def main():
             mp.Process(
                 target=rcc.engine.process_commits, args=(data_provider, task_queue)
             )
-            for i in range(cfg.num_workers)
+            for _ in range(cfg.num_workers)
         ]
 
         # Poll for new commits and put them in our internal processing queue
@@ -111,7 +102,7 @@ def main():
                     task_queue.join()
                     sleeper.reset()
                 sleeper.sleep()
-        except KeyboardInterrupt as e:
+        except KeyboardInterrupt:
             logger.info("Interrupted; waiting for workers")
             try:
                 for worker in engine_workers:
@@ -122,7 +113,7 @@ def main():
                 # Join each worker to ensure all of them are done
                 for worker in engine_workers:
                     worker.join()
-            except KeyboardInterrupt as e:
+            except KeyboardInterrupt:
                 # Give up and terminate everything
                 logger.info("Aborted")
                 for worker in engine_workers:

src/rcc/__init__.py

@@ -15,7 +15,7 @@ def text_cmp(file1, file2):
             return text_cmp(file1, f2)
     try:
         for line1, line2 in zip_longest(file1, file2):
-            if line1 == None or line2 == None:
+            if line1 is None or line2 is None:
                 return False
             if line1.rstrip() != line2.rstrip():
                 return False
@@ -35,15 +35,15 @@ def text_cmp2(file1, file2):
         while True:
             line1 = next(file1, None)
             line2 = next(file2, None)
-            if line1 == None or line2 == None:
+            if line1 is None or line2 is None:
                 break
             while line1.strip() == "":
                 line1 = next(file1, None)
-                if line1 == None:
+                if line1 is None:
                     return False
             while line2.strip() == "":
                 line2 = next(file2, None)
-                if line2 == None:
+                if line2 is None:
                     return False
             if line1 != line2:
                 tokens1 = line1.lower().split()
@@ -75,7 +75,7 @@ def is_float(s):
         raise ValueError("Error must be positive")
     try:
         for line1, line2 in zip_longest(file1, file2):
-            if line1 == None or line2 == None:
+            if line1 is None or line2 is None:
                 return False
             tokens1 = line1.split()
             tokens2 = line2.split()

src/rcc/cmp.py

@@ -1,14 +1,11 @@
 from __future__ import division, print_function, unicode_literals
 
 import asyncio
-import asyncio.subprocess
 import configparser
-import contextlib
 import datetime
 import filecmp
 import itertools as it
 import logging
-import multiprocessing as mp
 import os
 import shutil
 import zipfile
@@ -23,8 +20,6 @@
 import rcc.util
 from rcc.model import Commit, TestCase, TestCaseResult
 
-from .languages import language_from_extension
-
 DEFAULT_MKDIR_PERMISSIONS = 0o777
 
 
@@ -302,11 +297,11 @@ def is_correct(test_result):
 def cleanup_tests(base_dir):
     if os.path.isdir(base_dir):
 
-        def rmtree_handler(func, path, exc_info):
+        def rmtree_handler(_func, _path, exc_info):
             exc_type, exc_value, _ = exc_info
             raise exc_type(exc_value)
 
-        shutil.rmtree(base_dir, onerror=rmtree_handler)
+        shutil.rmtree(base_dir, onexc=rmtree_handler)
 
 
 def process_commit(data_provider, commit):
@@ -321,22 +316,22 @@ def process_commit(data_provider, commit):
 
     try:
         storage_provider = rcc.provider.storage.from_config(cfg)
-    except:
+    except Exception:
         logger.error("[{c.id}] Storage provider error".format(c=commit), exc_info=True)
         commit.status = Commit.STATUS_INTERNAL_ERROR
         data_provider.update_commit(commit)
         return
 
     try:
         test_cases = data_provider.fetch_test_cases(commit)
-    except:
+    except Exception:
         logger.error(
             "[{c.id}] Failed to fetch test cases".format(c=commit), exc_info=True
         )
         commit.status = Commit.STATUS_INTERNAL_ERROR
         data_provider.update_commit(commit)
         if cfg.cleanup_on_error:
-            cleanup_tests(base_dir)
+            cleanup_tests(os.path.join(cfg.exec_dir, "commit_{}".format(commit.id)))
         return
 
     # Delete results already produced by this commit, if any
@@ -355,7 +350,7 @@ def process_commit(data_provider, commit):
         create_container_cfg_file(commit, test_cases, base_dir)
         copy_source_files(data_provider, storage_provider, commit, base_dir)
         copy_test_case_files(storage_provider, test_cases, base_dir)
-    except:
+    except Exception:
         logger.error("[{c.id}] Failed to prepare runs".format(c=commit), exc_info=True)
         commit.status = Commit.STATUS_INTERNAL_ERROR
         data_provider.update_commit(commit)
@@ -368,7 +363,7 @@ def process_commit(data_provider, commit):
         test_results = run_tests(
             data_provider, storage_provider, commit, test_cases, base_dir, remote_dir
         )
-    except:
+    except Exception:
         logger.error("[{c.id}] Failed to run tests".format(c=commit), exc_info=True)
         commit.status = Commit.STATUS_INTERNAL_ERROR
         data_provider.update_commit(commit)
@@ -386,7 +381,7 @@ def process_commit(data_provider, commit):
             output_fname = prepare_output_file(commit, base_dir)
             storage_provider.store_commit_output(commit, output_fname)
         cleanup_tests(base_dir)
-    except:
+    except Exception:
         logger.error(
             "[{c.id}] Could not save results, commit data might be inconsistent".format(
                 c=commit

src/rcc/engine.py

@@ -1,5 +1,6 @@
 from __future__ import unicode_literals
 
+
 class Commit:
     STATUS_IN_QUEUE = 0
     STATUS_COMPILING = 1

src/rcc/model/commit.py

@@ -0,0 +1 @@
+

src/rcc/provider/__init__.py

@@ -2,17 +2,17 @@ class DataProvider(object):
     def fetch_commits_in_queue(self):
         raise NotImplementedError()
 
-    def update_commit(self, commit):
+    def update_commit(self, _commit):
         raise NotImplementedError()
 
-    def store_commit_test_results(self, commit, test_results):
+    def store_commit_test_results(self, _commit, _test_results):
         raise NotImplementedError()
 
-    def delete_commit_test_results(self, commit):
+    def delete_commit_test_results(self, _commit):
         raise NotImplementedError()
 
-    def fetch_exercise_files(self, commit):
+    def fetch_exercise_files(self, _commit):
         raise NotImplementedError()
 
-    def fetch_test_cases(self, commit):
+    def fetch_test_cases(self, _commit):
         raise NotImplementedError()