Better method to derive User-Agent strings (#76)

* updating user agent string generation to be more random (and get better results)
Signed-off-by: vsoch <vsoch@users.noreply.github.com>

Author: vsoch

.github/workflows/main.yml

@@ -9,7 +9,7 @@ jobs:
   testing-docker:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2      
+      - uses: actions/checkout@v3
       - name: Build container image
         run: |
           docker build -t quay.io/urlstechie/urlchecker .

.github/workflows/test.yml

@@ -11,11 +11,11 @@ jobs:
   formatting:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Setup black environment
         run: conda create --quiet --name black pyflakes
-
+          
       - name: Check formatting with black
         run: |
           export PATH="/usr/share/miniconda/bin:$PATH"
@@ -28,7 +28,7 @@ jobs:
     needs: formatting
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Setup mypy environment
         run: conda create --quiet --name type_checking mypy
@@ -46,7 +46,7 @@ jobs:
     needs: type_checking
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Setup testing environment
         run: conda create --quiet --name testing pytest
 

CHANGELOG.md

@@ -12,6 +12,7 @@ and **Merged pull requests**. Critical items to know are:
 Referenced versions in headers are tagged on Github, in parentheses are for pypi.
 
 ## [vxx](https://github.com/urlstechie/urlschecker-python/tree/master) (master)
+ - use ANSI escape sequences for colors, fake-useragent for agents (0.0.30)
  - adding type hints to code, more tests and logging bug fix (0.0.29)
  - decrease verbosity when filename is None (0.0.28)
  - don't exit and fail if no URLs to check (0.0.27)

mypy.ini

@@ -0,0 +1,2 @@
+[mypy]
+ignore_missing_imports = True

tests/test_core_urlproc.py

@@ -55,8 +55,9 @@ def test_check_urls(file):
 
 
 def test_get_user_agent():
-    user_agent = get_user_agent()
-    assert isinstance(user_agent, str)
+    ua = get_user_agent()    
+    assert isinstance(ua, dict)
+    assert "User-Agent" in ua
 
 
 def test_check_response_status_code():

urlchecker/client/__init__.py

@@ -10,8 +10,9 @@
 """
 
 
-import sys
 import argparse
+import sys
+
 import urlchecker
 
 

urlchecker/client/check.py

@@ -8,15 +8,15 @@
 
 """
 
-import re
+import logging
 import os
+import re
 import sys
-import logging
 
-from urlchecker.main.github import clone_repo, delete_repo
-from urlchecker.core.fileproc import remove_empty
 from urlchecker.core.check import UrlChecker
+from urlchecker.core.fileproc import remove_empty
 from urlchecker.logger import print_failure
+from urlchecker.main.github import clone_repo, delete_repo
 
 logger = logging.getLogger("urlchecker")
 
@@ -114,11 +114,11 @@ def main(args, extra):
                 if result["failed"]:
                     print_failure(file_name + ":")
                     for url in result["failed"]:
-                        print_failure("     " + url)
+                        print_failure("     ❌️ " + url)
         else:
             print("\n\U0001F914 Uh oh... The following urls did not pass:")
             for failed_url in check_results["failed"]:
-                print_failure(failed_url)
+                print_failure("❌️ " + failed_url)
 
     # If we have failures and it's not a force pass, exit with 1
     if not args.force_pass and check_results["failed"]:

urlchecker/core/check.py

@@ -11,10 +11,11 @@
 import os
 import re
 import sys
-from typing import List, Dict
+from typing import Dict, List
+
 from urlchecker.core import fileproc
-from urlchecker.core.worker import Workers
 from urlchecker.core.urlproc import UrlCheckResult
+from urlchecker.core.worker import Workers
 
 
 class UrlChecker:

urlchecker/core/fileproc.py

@@ -8,9 +8,10 @@
 """
 
 import fnmatch
-import re
 import os
+import re
 from typing import List
+
 from urlchecker.core import urlmarker
 
 

urlchecker/core/urlproc.py

@@ -8,13 +8,16 @@
 """
 
 import os
-import time
 import random
+import time
+from typing import Any, Dict, List, Optional
+
 import requests
-from typing import List, Optional
+from fake_useragent import UserAgent
+
 from urlchecker.core import fileproc
 from urlchecker.core.exclude import excluded
-from urlchecker.logger import print_success, print_failure
+from urlchecker.logger import print_failure, print_success
 
 
 def check_response_status_code(
@@ -46,51 +49,48 @@ def check_response_status_code(
     return True
 
 
-def get_user_agent() -> str:
+def get_user_agent() -> dict:
     """
-    Return a randomly chosen user agent for requests
+    Return a randomly chosen user agent and headers for requests
 
     Returns:
-        user agent string to include with User-Agent.
+        headers dict to include with request.
     """
-    agents = [
-        (
-            "Mozilla/5.0 (X11; Linux x86_64) "
-            "AppleWebKit/537.36 (KHTML, like Gecko) "
-            "Chrome/57.0.2987.110 "
-            "Safari/537.36"
-        ),  # chrome
-        (
-            "Mozilla/5.0 (X11; Linux x86_64) "
-            "AppleWebKit/537.36 (KHTML, like Gecko) "
-            "Chrome/61.0.3163.79 "
-            "Safari/537.36"
-        ),  # chrome
-        (
-            "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:55.0) "
-            "Gecko/20100101 "
-            "Firefox/55.0"
-        ),  # firefox
-        (
-            "Mozilla/5.0 (X11; Linux x86_64) "
-            "AppleWebKit/537.36 (KHTML, like Gecko) "
-            "Chrome/61.0.3163.91 "
-            "Safari/537.36"
-        ),  # chrome
-        (
-            "Mozilla/5.0 (X11; Linux x86_64) "
-            "AppleWebKit/537.36 (KHTML, like Gecko) "
-            "Chrome/62.0.3202.89 "
-            "Safari/537.36"
-        ),  # chrome
-        (
-            "Mozilla/5.0 (X11; Linux x86_64) "
-            "AppleWebKit/537.36 (KHTML, like Gecko) "
-            "Chrome/63.0.3239.108 "
-            "Safari/537.36"
-        ),  # chrome
-    ]
-    return random.choice(agents)
+    browser = random.choice(["chrome", "firefox"])
+    headers = get_faux_headers(browser)
+    headers["User-Agent"] = getattr(UserAgent(), browser)
+    return headers
+
+
+def get_faux_headers(browser) -> Dict[Any, Any]:
+    """
+    Get faux headers to populate based on user agent
+    """
+    headers = {
+        "chrome": {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
+            "Accept-Encoding": "gzip, deflate, br",
+            "Accept-Language": "en-US,en;q=0.9",
+            "Sec-Ch-Ua": '"Chromium";v="92", " Not A;Brand";v="99", "Google Chrome";v="92"',
+            "Sec-Ch-Ua-Mobile": "?0",
+            "Sec-Fetch-Dest": "document",
+            "Sec-Fetch-Mode": "navigate",
+            "Sec-Fetch-Site": "none",
+            "Sec-Fetch-User": "?1",
+            "Upgrade-Insecure-Requests": "1",
+        },
+        "firefox": {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+            "Accept-Encoding": "gzip, deflate, br",
+            "Accept-Language": "en-US,en;q=0.5",
+            "Sec-Fetch-Dest": "document",
+            "Sec-Fetch-Mode": "navigate",
+            "Sec-Fetch-Site": "none",
+            "Sec-Fetch-User": "?1",
+            "Upgrade-Insecure-Requests": "1",
+        },
+    }
+    return headers[browser]
 
 
 class UrlCheckResult:
@@ -185,13 +185,12 @@ def check_urls(
         # init seen urls list
         seen = set()
 
-        # Some sites will return 403 if it's not a "human" user agent
-        user_agent = get_user_agent()
-        headers = {"User-Agent": user_agent}
-
         # check links
         for url in [url for url in urls if "http" in url]:
 
+            # Some sites will return 403 if it's not a "human" user agent
+            headers = get_user_agent()
+
             # init do retrails and retrails counts
             do_retry = True
             rcount = retry_count
@@ -211,7 +210,6 @@ def check_urls(
                 response = None
                 try:
                     response = requests.get(url, timeout=pause, headers=headers)
-
                 except requests.exceptions.Timeout as e:
                     print(e)