implemented fake-useragent package (#49)

* Introduced fake-useragent package
* Use fallback Agent in case online-db in not available
* Proper UserAgent logging

The default way the program gets user agents is now from an online, up-to-date database
with the help of the fake-useragent package. Reading useragents from a custom local
file is still available as a parameter to the UserAgentManager class. Solves #28.

Author: la55u

http_request_randomizer/requests/proxy/requestProxy.py

@@ -31,10 +31,10 @@
 
 class RequestProxy:
     def __init__(self, web_proxy_list=[], sustain=False, timeout=5, protocol=Protocol.HTTP):
-        self.userAgent = UserAgentManager()
         self.logger = logging.getLogger()
         self.logger.addHandler(handler)
         self.logger.setLevel(0)
+        self.userAgent = UserAgentManager()
 
         #####
         # Each of the classes below implements a specific URL Parser

http_request_randomizer/requests/useragent/userAgent.py

@@ -1,15 +1,25 @@
 import os
 import random
+from fake_useragent import FakeUserAgent
+import logging
 
+logger = logging.getLogger(__name__)
 
 class UserAgentManager:
-    def __init__(self, agent_file=os.path.join(os.path.dirname(__file__), '../data/user_agents.txt')):
-        self.agent_file = agent_file
-        self.useragents = self.load_user_agents(self.agent_file)
+    def __init__(self, fallback=None, file=None):
+        self.agent_file = file
+        if file is not None:
+            logger.info('Using local file for user agents: '+self.agent_file)
+            self.useragents = self.load_user_agents(self.agent_file)
+        else:
+            logger.info('Using fake-useragent package for user agents.')
+            if fallback is None:
+                fallback = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36'
+            self.fakeuseragent = FakeUserAgent(fallback=fallback, cache=False)
 
     def load_user_agents(self, useragentsfile):
         """
-        useragentfile : string
+        useragentsfile : string
             path to text file of user agents, one per line
         """
         useragents = []
@@ -20,28 +30,41 @@ def load_user_agents(self, useragentsfile):
         return useragents
 
     def get_random_user_agent(self):
-        """
-        useragents : string array of different user agents
-        :param useragents:
-        :return random agent:
-        """
-        user_agent = random.choice(self.useragents)
-        return user_agent.decode('utf-8')
+        if self.agent_file:
+            user_agent = random.choice(self.useragents)
+            return user_agent.decode('utf-8')
+        else:
+            return self.fakeuseragent.random
 
     def get_first_user_agent(self):
-        return self.useragents[0].decode('utf-8')
+        if self.agent_file:
+            return self.useragents[0].decode('utf-8')
+        else:
+            logger.warning('Fake-useragent library does not support operaration get_first - change to user-agent file!')
+            return None
 
     def get_last_user_agent(self):
-        return self.useragents[-1].decode('utf-8')
+        if self.agent_file:
+            return self.useragents[-1].decode('utf-8')
+        else:
+            logger.warning('Fake-useragent library does not support operaration get_last - change to user-agent file!')
+            return None
 
     def get_len_user_agent(self):
-        return len(self.useragents)
+        if self.agent_file:
+            return len(self.useragents)
+        else:
+            logger.warning('Fake-useragent library does not support operaration get_len - change to user-agent file!')
+            return None
 
 
 if __name__ == '__main__':
     ua = UserAgentManager()
-    print("Number of User Agent headers: {0}".format(ua.get_len_user_agent()))
-    print("First User Agent in file: {0}".format(ua.get_first_user_agent()))
-    print("Last User Agent in file: {0}".format(ua.get_last_user_agent()))
+    if ua.agent_file:
+        print("Number of User Agent headers: {0}".format(ua.get_len_user_agent()))
+        print("First User Agent in file: {0}".format(ua.get_first_user_agent()))
+        print("Last User Agent in file: {0}".format(ua.get_last_user_agent()))
+    else:
+        print("Using up-to-date user agents from online databse.")
     print("If you want one random header for a request, you may use the following header:\n")
     print("User-Agent: " + ua.get_random_user_agent() + "\n")

requirements.txt

@@ -7,3 +7,4 @@ pytest-cov >= 2.5.1
 python-dateutil >= 2.6.1
 requests >= 2.18.4
 pyOpenSSL >= 17.5.0
+fake-useragent >= 0.1.10

setup.py

@@ -66,7 +66,8 @@ def run_tests(self):
                       'psutil >= 5.4.3',
                       'python-dateutil >= 2.6.1',
                       'requests >= 2.18.4',
-                      'pyOpenSSL >= 17.5.0'
+                      'pyOpenSSL >= 17.5.0',
+                      'fake-useragent >= 0.1.10'
                       ],
     use_scm_version=True,
     setup_requires=['setuptools-scm', 'pytest-runner'],

tests/test_useragent.py

@@ -13,21 +13,27 @@
 
 class TestBaseProxyParsers(unittest.TestCase):
     def setUp(self):
-        self.ua = UserAgentManager()
+        agentsfile = os.path.join(os.path.dirname(__file__), '../http_request_randomizer/requests/data/user_agents.txt')
+        self.uafile = UserAgentManager(file=agentsfile)
+        self.uafake = UserAgentManager()
 
     def test_agent_size(self):
-        self.assertTrue(self.ua.get_len_user_agent() >= 899)
+        self.assertTrue(self.uafile.get_len_user_agent() >= 899)
+        self.assertIsNone(self.uafake.get_len_user_agent())
 
     def test_fist_user_agent(self):
         expected = "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0"
-        self.assertEquals(self.ua.get_first_user_agent(), expected)
+        self.assertEqual(self.uafile.get_first_user_agent(), expected)
+        self.assertIsNone(self.uafake.get_first_user_agent())
 
     def test_last_user_agent(self):
         expected = "Opera/9.80 (Windows NT 5.1; U; ru) Presto/2.2.15 Version/10.0"
-        self.assertEquals(self.ua.get_last_user_agent(), expected)
+        self.assertEqual(self.uafile.get_last_user_agent(), expected)
+        self.assertIsNone(self.uafake.get_last_user_agent())
 
     def test_random_user_agent(self):
-        self.assertNotEqual(self.ua.get_random_user_agent(), self.ua.get_random_user_agent())
+        self.assertNotEqual(self.uafile.get_random_user_agent(), self.uafile.get_random_user_agent())
+        self.assertNotEqual(self.uafake.get_random_user_agent(), self.uafake.get_random_user_agent())
 
 
 if __name__ == '__main__':