Merge pull request #28 from urlstechie/add/files-flag

Adding files flag and fixing bug with return code

Author: vsoch

CHANGELOG.md

@@ -12,6 +12,7 @@ and **Merged pull requests**. Critical items to know are:
 Referenced versions in headers are tagged on Github, in parentheses are for pypi.
 
 ## [vxx](https://github.com/urlstechie/urlschecker-python/tree/master) (master)
+ - bug with incorrect return code on fail, add files flag (0.0.16)
  - reverting back to working client (0.0.15)
  - removing unused file variable (0.0.13)
  - adding support for csv export (0.0.12)

README.md

@@ -47,15 +47,15 @@ Your most likely use case will be to check a local directory with static files (
 for files. In this case, you can use urlchecker check:
 
 ```bash
-$ urlchecker check --help
-
 $ urlchecker check --help
 usage: urlchecker check [-h] [-b BRANCH] [--subfolder SUBFOLDER] [--cleanup]
                         [--force-pass] [--no-print] [--file-types FILE_TYPES]
+                        [--files FILES]
                         [--white-listed-urls WHITE_LISTED_URLS]
                         [--white-listed-patterns WHITE_LISTED_PATTERNS]
                         [--white-listed-files WHITE_LISTED_FILES]
-                        [--retry-count RETRY_COUNT] [--timeout TIMEOUT]
+                        [--save SAVE] [--retry-count RETRY_COUNT]
+                        [--timeout TIMEOUT]
                         path
 
 positional arguments:
@@ -78,6 +78,8 @@ optional arguments:
   --file-types FILE_TYPES
                         comma separated list of file extensions to check
                         (defaults to .md,.py)
+  --files FILES         comma separated list of exact files or patterns to
+                        check.
   --white-listed-urls WHITE_LISTED_URLS
                         comma separated list of white listed urls (no spaces)
   --white-listed-patterns WHITE_LISTED_PATTERNS
@@ -86,6 +88,7 @@ optional arguments:
   --white-listed-files WHITE_LISTED_FILES
                         comma separated list of white listed files and
                         patterns for files (no spaces)
+  --save SAVE           Path to a csv file to save results to.
   --retry-count RETRY_COUNT
                         retry count upon failure (defaults to 2, one retry).
   --timeout TIMEOUT     timeout (seconds) to provide to the requests library

urlchecker/client/__init__.py

@@ -16,6 +16,7 @@
 import urlchecker
 import logging
 
+
 def get_parser():
     parser = argparse.ArgumentParser(description="urlchecker python")
 
@@ -39,7 +40,7 @@ def get_parser():
     subparsers = parser.add_subparsers(
         help="urlchecker python actions",
         title="actions",
-        description='actions for urlchecker',
+        description="actions for urlchecker",
         dest="command",
     )
 
@@ -55,8 +56,7 @@ def get_parser():
 
     # supports a clone URL or a path
     check.add_argument(
-        "path",
-        help="the local path or GitHub repository to clone and check",
+        "path", help="the local path or GitHub repository to clone and check",
     )
 
     check.add_argument(
@@ -99,7 +99,14 @@ def get_parser():
         default=".md,.py",
     )
 
-# White listing
+    check.add_argument(
+        "--files",
+        dest="files",
+        help="comma separated list of exact files or patterns to check.",
+        default=None,
+    )
+
+    # White listing
 
     check.add_argument(
         "--white-listed-urls",
@@ -119,15 +126,13 @@ def get_parser():
         default="",
     )
 
-# Saving
+    # Saving
 
     check.add_argument(
-        "--save",
-        help="Path toa csv file to save results to.",
-        default=None,
+        "--save", help="Path to a csv file to save results to.", default=None,
     )
 
-# Timeouts
+    # Timeouts
 
     check.add_argument(
         "--retry-count",
@@ -180,7 +185,7 @@ def help(return_code=0):
     else:
         print("Unsupported command %s" % args.command)
         sys.exit(0)
-    
+
     # Pass on to the correct parser
     return_code = 0
     try:

urlchecker/client/check.py

@@ -13,7 +13,7 @@
 from urlchecker.core.check import run_urlchecker
 from urlchecker.logger import print_success, print_failure
 
-logger = logging.getLogger('urlchecker')
+logger = logging.getLogger("urlchecker")
 
 
 def main(args, extra):
@@ -52,6 +52,7 @@ def main(args, extra):
     white_listed_urls = remove_empty(args.white_listed_urls.split(","))
     white_listed_patterns = remove_empty(args.white_listed_patterns.split(","))
     white_listed_files = remove_empty(args.white_listed_files.split(","))
+    files = remove_empty(args.files.split(","))
 
     # Alert user about settings
     print("  original path: %s" % args.path)
@@ -60,6 +61,7 @@ def main(args, extra):
     print("         branch: %s" % args.branch)
     print("        cleanup: %s" % args.cleanup)
     print("     file types: %s" % file_types)
+    print("          files: %s" % files)
     print("      print all: %s" % (not args.no_print))
     print(" url whitetlist: %s" % white_listed_urls)
     print("   url patterns: %s" % white_listed_patterns)
@@ -70,14 +72,17 @@ def main(args, extra):
     print("        timeout: %s" % args.timeout)
 
     # Run checks, get lookup of results and fails
-    check_results = run_urlchecker(path=path,
-                                   file_types=file_types,
-                                   white_listed_files=white_listed_files,
-                                   white_listed_urls=white_listed_urls,
-                                   white_listed_patterns=white_listed_patterns,
-                                   print_all=not args.no_print,
-                                   retry_count=args.retry_count,
-                                   timeout=args.timeout)
+    check_results = run_urlchecker(
+        path=path,
+        file_types=file_types,
+        include_patterns=files,
+        white_listed_files=white_listed_files,
+        white_listed_urls=white_listed_urls,
+        white_listed_patterns=white_listed_patterns,
+        print_all=not args.no_print,
+        retry_count=args.retry_count,
+        timeout=args.timeout,
+    )
 
     # save results to flie, if save indicated
     if args.save:
@@ -89,17 +94,23 @@ def main(args, extra):
         delete_repo(path)
 
     # Case 1: We didn't find any urls to check
-    if not check_results['failed'] and not check_results['passed']:
+    if not check_results["failed"] and not check_results["passed"]:
         print("\n\nDone. No urls were collected.")
         sys.exit(0)
 
-    # Case 2: We had errors, but force pass is True
-    elif args.force_pass and check_results['failed']:
+    # Case 2: We had errors, print them for the user
+    if check_results["failed"]:
         print("\n\nDone. The following urls did not pass:")
-        for failed_url in check_results['failed']:
+        for failed_url in check_results["failed"]:
             print_failure(failed_url)
+
+    # If we have failures and it's not a force pass, exit with 1
+    if not args.force_pass and check_results["failed"]:
         sys.exit(1)
 
+    # Finally, alert user if we are passing conditionally
+    if check_results["failed"]:
+        print("\n\nConditional pass force pass True.")
     else:
         print("\n\nDone. All URLS passed.")
-        sys.exit(0)
+    sys.exit(0)

urlchecker/core/check.py

@@ -12,15 +12,17 @@
 from urlchecker.core import fileproc, urlproc
 from urlchecker.core.whitelist import white_listed
 
+
 def run_urlchecker(
     path,
     file_types,
     white_listed_files,
     white_listed_urls,
     white_listed_patterns,
     print_all,
+    include_patterns=None,
     retry_count=2,
-    timeout=5
+    timeout=5,
 ):
     """
     Run the url checker given a path, a whitelist for each of url and file
@@ -34,6 +36,7 @@ def run_urlchecker(
         - white_listed_urls     (list) : list of white-listed urls.
         - white_listed_patterns (list) : list of white-listed patterns for urls.
         - white_listed_files    (list) : list of white-listed files and patterns for flies.
+        - include_patterns      (list) : list of files and patterns to check.
         - retry_count            (int) : number of retries on failed first check. Default=2.
         - timeout                (int) : timeout to use when waiting on check feedback. Default=5.
 
@@ -44,6 +47,7 @@ def run_urlchecker(
 
     # get all file paths
     file_paths = fileproc.get_file_paths(
+        include_patterns=include_patterns,
         base_path=path,
         file_types=file_types,
         white_listed_files=white_listed_files,
@@ -83,7 +87,7 @@ def check_files(
         (list) check-results as a list of two lists (successfull checks, failed checks).
     """
     # init results list (first is success, second is issue)
-    check_results = {"passed":[], "failed": []}
+    check_results = {"passed": [], "failed": []}
 
     # Allow for user to skip specifying white listed options
     white_listed_urls = white_listed_urls or []

urlchecker/core/fileproc.py

@@ -34,40 +34,61 @@ def check_file_type(file_path, file_types):
     return False
 
 
-def include_file(file_path, white_list_patterns):
+def include_file(file_path, white_list_patterns=None, include_patterns=None):
     """
     Check a file path for inclusion based on an OR regular expression.
     The user is currently not notified if a file is marked for removal.
 
     Args:
         - file_path            (str) : a file path to check if should be included.
-        - white_list_patterns (list) : list of patterns to whitelist (include).
+        - white_list_patterns (list) : list of patterns to whitelist (not test).
+        - include_patterns    (list) : list of patterns to include.
 
     Returns:
-        (bool) boolean indicating if the URL should be white listed (included).
+        (bool) boolean indicating if the URL should be white listed (not tested).
     """
+    include_patterns = include_patterns or []
+    white_list_patterns = white_list_patterns or []
+
     # No white listed patterns, all files are included
-    if not white_list_patterns:
+    if not white_list_patterns and not include_patterns:
         return True
 
-    # Return False (don't include) if the patterns match
-    regexp = "(%s)" % "|".join(white_list_patterns)
-    return not re.search(regexp, file_path)
+    # Create a regular expression for each
+    whitelist_regexp = "(%s)" % "|".join(white_list_patterns)
+    include_regexp = "(%s)" % "|".join(include_patterns)
+
+    # Return False (don't include) if whitelisted
+    if not include_patterns:
+        return not re.search(whitelist_regexp, file_path)
+
+    # We have an include_patterns only
+    elif not white_list_patterns:
+        return re.search(include_regexp, file_path)
+
+    # If both defined, whitelisting takes preference
+    return re.search(include_regexp, file_path) and not re.search(
+        whitelist_regexp, file_path
+    )
 
 
-def get_file_paths(base_path, file_types, white_listed_files=None):
+def get_file_paths(
+    base_path, file_types, white_listed_files=None, include_patterns=None
+):
     """
     Get path to all files under a give directory and its subfolders.
 
     Args:
         - base_path           (str) : base path.
         - file_types         (list) : list of file extensions to accept.
+        - include_patterns   (list) : list of files and patterns to include.
         - white_listed_files (list) : list of files or patterns to white list
 
     Returns:
         (list) list of file paths.
     """
     white_listed_files = white_listed_files or []
+    include_patterns = include_patterns or []
 
     # init paths
     file_paths = []
@@ -79,7 +100,9 @@ def get_file_paths(base_path, file_types, white_listed_files=None):
             for file in files
             if os.path.isfile(os.path.join(root, file))
             and check_file_type(file, file_types)
-            and include_file(os.path.join(root, file), white_listed_files)
+            and include_file(
+                os.path.join(root, file), white_listed_files, include_patterns
+            )
         ]
     return file_paths
 
@@ -145,7 +168,7 @@ def save_results(check_results, file_path, sep=",", header=None):
     dirname = os.path.dirname(file_path)
 
     if not os.path.exists(dirname):
-        sys.exit("%s does not exist, cannot save %s there." %(dirname, file_path))
+        sys.exit("%s does not exist, cannot save %s there." % (dirname, file_path))
 
     # Ensure the header is provided and correct (length 2)
     if not header:
@@ -157,10 +180,10 @@ def save_results(check_results, file_path, sep=",", header=None):
     print("Saving results to %s" % file_path)
 
     # Write to file after header row
-    with open(file_path, mode='w') as fd:
+    with open(file_path, mode="w") as fd:
         writer = csv.writer(fd, delimiter=sep, quotechar='"', quoting=csv.QUOTE_MINIMAL)
         writer.writerow(header)
         for result, items in check_results.items():
-            [writer.writerow([item, result]) for item in items];
+            [writer.writerow([item, result]) for item in items]
 
     return file_path

urlchecker/core/urlproc.py

@@ -14,6 +14,7 @@
 from urlchecker.core import urlmarker
 from urlchecker.logger import print_success, print_failure
 
+
 def record_response(url, response, check_results):
     """
     Record response status of an input url. This function is run after success,

urlchecker/core/whitelist.py

@@ -7,6 +7,7 @@
 
 """
 
+
 def white_listed(url, white_listed_urls, white_listed_patterns):
     """
     Check if link is in the white listed URLs or patterns to ignore.

urlchecker/logger.py

@@ -9,6 +9,7 @@
 
 import logging
 
+
 def print_failure(message):
     """
     Given a message string, print as a failure in red.
@@ -29,7 +30,7 @@ def print_success(message):
     print("\x1b[32m" + message + "\x1b[0m")
 
 
-def get_logger(name='urlchecker', level=logging.INFO):
+def get_logger(name="urlchecker", level=logging.INFO):
     """
     Get a default logger for the urlchecker library, meaning
     that we use name "urlchecker" and use the default logging
@@ -50,7 +51,9 @@ def get_logger(name='urlchecker', level=logging.INFO):
     ch.setLevel(logging.ERROR)
 
     # formatting
-    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')    
+    formatter = logging.Formatter(
+        "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+    )
     ch.setFormatter(formatter)
     logger.addHandler(ch)
     return logger

urlchecker/main/github.py

@@ -55,7 +55,6 @@ def delete_repo(base_path):
     return result.returncode
 
 
-
 def get_branch(default="master"):
     """
     Derive the selected branch. We first look to the environment variable