Extra string spaces in proxy attributes made parsers fail. Fixing using strip() - more work to be done in code deduplication

Author: pgaref

http_request_randomizer/requests/parsers/FreeProxyParser.py

@@ -57,17 +57,17 @@ def createProxyObject(self, dataset):
         for field in dataset:
             if field[0] == 'IP Address':
                 # Make sure it is a Valid IP
-                if not UrlParser.valid_ip(field[1]):
-                    logger.debug("IP with Invalid format: {}".format(field[1]))
-                    break
-                else:
-                    ip = field[1]
+                ip = field[1].strip()  # String strip()
+                # TODO @pgaref: Duplicate code!!!
+                if not UrlParser.valid_ip(ip):
+                    logger.debug("IP with Invalid format: {}".format(ip))
+                    return None
             elif field[0] == 'Port':
-                port = field[1]
+                port = field[1].strip()  # String strip()
             elif field[0] == 'Anonymity':
-                anonymity = AnonymityLevel(field[1])
+                anonymity = AnonymityLevel(field[1].strip())  # String strip()
             elif field[0] == 'Country':
-                country = field[1]
+                country = field[1].strip()  # String strip()
         return ProxyObject(source=self.id, ip=ip, port=port, anonymity_level=anonymity, country=country)
 
     def __str__(self):

http_request_randomizer/requests/parsers/ProxyForEuParser.py

@@ -57,18 +57,18 @@ def createProxyObject(self, dataset):
                 if float(field[1]) < self.get_min_bandwidth():
                     return None
             if field[0] == 'IP':
+                ip = field[1].strip()  # String strip()
+                # TODO @pgaref : Dupicate code?
                 # Make sure it is a Valid IP
-                if not UrlParser.valid_ip(field[1]):
-                    logger.debug("IP with Invalid format: {}".format(field[1]))
-                    break
-                else:
-                    ip = field[1]
+                if not UrlParser.valid_ip(ip):
+                    logger.debug("IP with Invalid format: {}".format(ip))
+                    return None
             elif field[0] == 'Port':
-                port = field[1]
+                port = field[1].strip()  # String strip()
             elif field[0] == 'Anon':
-                anonymity = AnonymityLevel(field[1])
+                anonymity = AnonymityLevel(field[1].strip())  # String strip()
             elif field[0] == 'Country':
-                country = field[1]
+                country = field[1].strip()  # String strip()
         return ProxyObject(source=self.id, ip=ip, port=port, anonymity_level=anonymity, country=country)
 
     def __str__(self):

http_request_randomizer/requests/parsers/RebroWeeblyParser.py

@@ -56,6 +56,7 @@ def parse_proxyList(self, use_top15k=False):
 
     def createProxyObject(self, dataset):
         # Provider specific code
+        dataset = dataset.strip()  # String strip()
         ip = dataset.split(":")[0]
         port = dataset.split(":")[1]
         # TODO: Parse extra tables and combine data - Provider seems to be out-of-date

http_request_randomizer/requests/parsers/SamairProxyParser.py

@@ -62,12 +62,13 @@ def parse_proxyList(self):
     def createProxyObject(self, row):
         for td_row in row.findAll("td"):
             if td_row.attrs['data-label'] == 'IP:port ':
-                ip = td_row.text.split(":")[0]
-                port = td_row.text.split(":")[1]
+                text = td_row.text.strip()
+                ip = text.split(":")[0]
+                port = text.split(":")[1]
             elif td_row.attrs['data-label'] == 'Anonymity Type: ':
-                anonymity = AnonymityLevel(td_row.text)
+                anonymity = AnonymityLevel(td_row.text.strip())
             elif td_row.attrs['data-label'] == 'Country: ':
-                country = td_row.text
+                country = td_row.text.strip()
         return ProxyObject(source=self.id, ip=ip, port=port, anonymity_level=anonymity, country=country)
 
     def __str__(self):

http_request_randomizer/requests/proxy/ProxyObject.py

@@ -48,3 +48,5 @@ class AnonymityLevel(MultiValueEnum):
     ANONYMOUS = 'anonymous', 'anonymous proxy', 'high-anonymous'
     ELITE = 'elite', 'elite proxy', 'HIGH'
     UNKNOWN = 'unknown', 'none'
+    # TODO @pgaref: use a default case instead!!!
+    BAD = "bad", "bad1"

tests/mocks.py

@@ -163,7 +163,7 @@ def samair_mock(url, request):
     \n
     <tr class="anon">
         <td data-label="IP:port ">152.251.141.93:8080</td>
-        <td data-label="Anonymity Type: ">elite</td>
+        <td data-label="Anonymity Type: ">elite </td>
         <td data-label="Checked: ">Jul-16, 04:39</td>
         <td data-label="Country: ">Brazil</td>
         <td data-label="City: ">&nbsp;</td>