Add type hints + More tests + Better docs (#69)

* minor edits to client scripts
* add type hints (and some doc edits) to core scripts
* some doc edits to logger script for consistency
* fix project name and copyright in docs conf
* more tests for the client scripts for better coverage
* remove forgotten debug prints
* reformat
* add mypy check to workflows
* automatically get year for copyright
* change master to main
* pad version and edit changelog
* improved datetime import and use

Author: SuperKogito

.github/workflows/test.yml

@@ -24,15 +24,33 @@ jobs:
           black --check urlchecker
 
 
-  testing:
+  type_checking:
     needs: formatting
     runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup mypy environment
+        run: conda create --quiet --name type_checking mypy
+
+      - name: Check types with mypy
+        run: |
+          export PATH="/usr/share/miniconda/bin:$PATH"
+          source activate type_checking
+          pip install mypy
+          pip install types-requests
+          mypy urlchecker
+
+
+  testing:
+    needs: type_checking
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - name: Setup testing environment
         run: conda create --quiet --name testing pytest
 
-      - name: Test 
+      - name: Test
         run: |
           export PATH="/usr/share/miniconda/bin:$PATH"
           source activate testing

CHANGELOG.md

@@ -1,7 +1,7 @@
 # CHANGELOG
 
-This is a manually generated log to track changes to the repository for each release. 
-Each section should include general headers such as **Implemented enhancements** 
+This is a manually generated log to track changes to the repository for each release.
+Each section should include general headers such as **Implemented enhancements**
 and **Merged pull requests**. Critical items to know are:
 
  - renamed commands
@@ -12,6 +12,7 @@ and **Merged pull requests**. Critical items to know are:
 Referenced versions in headers are tagged on Github, in parentheses are for pypi.
 
 ## [vxx](https://github.com/urlstechie/urlschecker-python/tree/master) (master)
+ - adding type hints to code, more tests and logging bug fix (0.0.29)
  - decrease verbosity when filename is None (0.0.28)
  - don't exit and fail if no URLs to check (0.0.27)
  - multiprocessing to speed up checks (0.0.26)

docs/source/conf.py

@@ -12,13 +12,14 @@
 #
 import os
 import sys
+from datetime import datetime
 
 sys.path.insert(0, os.path.abspath("../../"))
 
 # -- Project information -----------------------------------------------------
 
-project = "URLs-checker"
-copyright = "2020, urlstechie"
+project = "urlchecker"
+copyright = "2020-%s, urlstechie" % datetime.now().year
 author = "urlstechie"
 
 # The master toctree document.

tests/test_client_check.py

@@ -1,8 +1,121 @@
 import os
+import sys
 import pytest
-import subprocess
+import argparse
 import tempfile
+import subprocess
 import configparser
+from urlchecker.client import main as init_main
+from urlchecker.client.check import main as check_main
+
+
+@pytest.mark.parametrize("config_fname", ["./tests/_local_test_config.conf"])
+@pytest.mark.parametrize("cleanup", [False, True])
+@pytest.mark.parametrize("print_all", [False])
+@pytest.mark.parametrize("verbose", [True])
+@pytest.mark.parametrize("force_pass", [False, True])
+@pytest.mark.parametrize("rcount", [1])
+@pytest.mark.parametrize("timeout", [3])
+def test_client_init_(
+    config_fname, cleanup, print_all, verbose, force_pass, rcount, timeout
+):
+
+    # init config parser
+    config = configparser.ConfigParser()
+    config.read(config_fname)
+
+    # init env variables
+    path = config["DEFAULT"]["git_path_test_value"]
+    file_types = config["DEFAULT"]["file_types_test_values"]
+    exclude_urls = config["DEFAULT"]["exclude_test_urls"]
+    exclude_patterns = config["DEFAULT"]["exclude_test_patterns"]
+
+    # Generate command
+    cmd = [
+        "urlchecker",
+        "check",
+        "--branch",
+        "main",
+        "--subfolder",
+        "test_files",
+        "--file-types",
+        file_types,
+        "--exclude-files",
+        "conf.py",
+        "--exclude-urls",
+        exclude_urls,
+        "--exclude-patterns",
+        exclude_patterns,
+        "--retry-count",
+        str(rcount),
+        "--timeout",
+        str(timeout),
+    ]
+
+    # Add boolean arguments
+    if cleanup:
+        cmd.append("--cleanup")
+    if print_all:
+        cmd.append("--print-all")
+    if force_pass:
+        cmd.append("--force-pass")
+    if verbose:
+        cmd.append("--verbose")
+
+    # Add final path
+    cmd.append(path)
+
+    # assign args and run main
+    sys.argv = cmd
+    with pytest.raises(SystemExit):
+        init_main()
+
+
+@pytest.mark.parametrize(
+    "command", ["", "--version", "--help", "--unsupported_command"]
+)
+def test_command(command):
+    # assign args and run main
+    sys.argv = ["urlchecker", command]
+    with pytest.raises(SystemExit):
+        init_main()
+
+
+@pytest.mark.parametrize("config_fname", ["./tests/_local_test_config.conf"])
+@pytest.mark.parametrize("cleanup", [False, True])
+@pytest.mark.parametrize("no_print", [True])
+@pytest.mark.parametrize("verbose", [False])
+@pytest.mark.parametrize("force_pass", [False])
+@pytest.mark.parametrize("rcount", [1])
+@pytest.mark.parametrize("timeout", [3])
+def test_arguments_from_cli(
+    config_fname, cleanup, no_print, verbose, force_pass, rcount, timeout
+):
+
+    # init config parser
+    config = configparser.ConfigParser()
+    config.read("./tests/_local_test_config.conf")
+
+    # Generate command
+    cmd = argparse.Namespace(
+        path=config["DEFAULT"]["git_path_test_value"],
+        subfolder="test_files",
+        branch="main",
+        cleanup=str(cleanup),
+        file_types=config["DEFAULT"]["file_types_test_values"],
+        files="",
+        no_print=str(no_print),
+        exclude_urls=config["DEFAULT"]["exclude_test_urls"],
+        exclude_patterns=config["DEFAULT"]["exclude_test_patterns"],
+        exclude_files="conf.py",
+        force_pass=str(force_pass),
+        retry_count=rcount,
+        save=None,
+        timeout=timeout,
+        verbose=verbose,
+    )
+    with pytest.raises(SystemExit):
+        check_main(cmd, None)
 
 
 @pytest.mark.parametrize("config_fname", ["./tests/_local_test_config.conf"])
@@ -12,8 +125,9 @@
 @pytest.mark.parametrize("force_pass", [False, True])
 @pytest.mark.parametrize("rcount", [1, 3])
 @pytest.mark.parametrize("timeout", [3, 5])
-def test_client_general(config_fname, cleanup, print_all, verbose,
-                        force_pass, rcount, timeout):
+def test_client_general(
+    config_fname, cleanup, print_all, verbose, force_pass, rcount, timeout
+):
 
     # init config parser
     config = configparser.ConfigParser()

tests/test_main_github.py

@@ -28,11 +28,11 @@ def test_get_branch():
     """
     test getting branch from environment or default
     """
-    # Unset defaults to master
+    # Unset defaults to main
     os.environ["GITHUB_REF"] = ""
     os.putenv("GITHUB_REF", "")
     branch = get_branch()
-    if branch != "master":
+    if branch != "main":
         raise AssertionError
 
     # Set both GitHub input variable and ref (ref takes priority)

urlchecker/client/__init__.py

@@ -53,8 +53,8 @@ def get_parser():
     check.add_argument(
         "-b",
         "--branch",
-        help="if cloning, specify a branch to use (defaults to master)",
-        default="master",
+        help="if cloning, specify a branch to use (defaults to main)",
+        default="main",
     )
 
     check.add_argument(
@@ -152,12 +152,15 @@ def get_parser():
 
 
 def main():
-    """main is the entrypoint urlchecker-python."""
+    """
+    main is the entrypoint urlchecker-python.
+    """
 
     parser = get_parser()
 
     def help(return_code=0):
-        """print help, including the software version and active client
+        """
+        print help, including the software version and active client
         and exit with return code.
         """
 

urlchecker/client/check.py

@@ -1,6 +1,11 @@
 """
+
 client/github.py: entrypoint for interaction with a GitHub repostiory.
 Copyright (c) 2020-2022 Ayoub Malek and Vanessa Sochat
+
+This source code is licensed under the terms of the MIT license.
+For a copy, see <https://opensource.org/licenses/MIT>.
+
 """
 
 import re
@@ -18,14 +23,14 @@
 
 def main(args, extra):
     """
-    main entrypoint for running a check. We expect an args object with
+    Main entrypoint for running a check. We expect an args object with
     arguments from the main client. From here we determine the path
     to parse (or GitHub url to clone) and call the main check function
     under main/check.py
 
-    Parameters:
-      - args: the argparse ArgParser with parsed args
-      - extra: extra arguments not handled by the parser
+    Args:
+      - args  : the argparse ArgParser with parsed args
+      - extra : extra arguments not handled by the parser
     """
     path = args.path