Fixing failing test after merge - Still need to parse the pagination table in Samair Parser

Author: pgaref

http_request_randomizer/requests/parsers/SamairProxyParser.py

@@ -19,7 +19,7 @@ def __init__(self, id, web_url, timeout=None):
     def parse_proxyList(self):
         curr_proxy_list = []
         # Parse all proxy pages -> format: /list/{num}.htm
-        # TODO: get the pageRange from the 'pagination' table
+        # TODO @pgaref: get the pageRange from the 'pagination' table
         for page in range(1, 21):
             response = requests.get("{0}{num:02d}.htm".format(self.get_URl(), num=page), timeout=self.timeout)
             if not response.ok:
@@ -52,24 +52,24 @@ def parse_proxyList(self):
                 # curr_proxy_list.append('http://' + row.text + ports[row['class'][0]])
                 # Make sure it is a Valid Proxy Address
                 if UrlParser.valid_ip_port(td_row.text):
-                    proxy_obj = self.createProxyObject(td_row)
+                    proxy_obj = self.createProxyObject(row)
                     proxy_obj.print_everything()
                     curr_proxy_list.append(proxy_obj)
                 else:
                     logger.debug("Address with Invalid format: {}".format(td_row.text))
         return curr_proxy_list
 
-    def createProxyObject(self, td_row):
-        ip = td_row.text.split(":")[0]
-        port = td_row.text.split(":")[1]
-        next_td_row = td_row.findNext("td")
-        anonymity = AnonymityLevel(next_td_row.text)
-        next_td_row = next_td_row.findNext("td")
-        next_td_row = next_td_row.findNext("td")
-        country = next_td_row.text
+    def createProxyObject(self, row):
+        for td_row in row.findAll("td"):
+            if td_row.attrs['data-label'] == 'IP:port ':
+                ip = td_row.text.split(":")[0]
+                port = td_row.text.split(":")[1]
+            elif td_row.attrs['data-label'] == 'Anonymity Type: ':
+                anonymity = AnonymityLevel(td_row.text)
+            elif td_row.attrs['data-label'] == 'Country: ':
+                country = td_row.text
         return ProxyObject(source=self.id, ip=ip, port=port, anonymity_level=anonymity, country=country)
 
-
     def __str__(self):
         return "SemairProxy Parser of '{0}' with required bandwidth: '{1}' KBs" \
             .format(self.url, self.minimum_bandwidth_in_KBs)

tests/mocks.py

@@ -144,7 +144,7 @@ def samair_mock(url, request):
     \n
     <tr class="anon">
         <td data-label="IP:port ">191.252.61.28:80</td>
-        <td data-label="Anonymity Type: "high-anonymous</td>
+        <td data-label="Anonymity Type: ">high-anonymous</td>
         <td data-label="Checked: ">Apr-18, 17:18</td>
         <td data-label="Country: ">Brazil</td>
         <td data-label="City: ">S\xe3o Jos\xe9 Dos Campos</td>
@@ -154,7 +154,7 @@ def samair_mock(url, request):
     \n
     <tr class="anon">
         <td data-label="IP:port ">167.114.203.141:8080</td>
-        <td data-label="Anonymity Type: "transparent</td>
+        <td data-label="Anonymity Type: ">transparent</td>
         <td data-label="Checked: ">Apr-18, 13:22</td>
         <td data-label="Country: ">Canada</td>
         <td data-label="City: ">Montr\xe9al (QC)</td>
@@ -163,7 +163,7 @@ def samair_mock(url, request):
     \n
     <tr class="anon">
         <td data-label="IP:port ">152.251.141.93:8080</td>
-        <td data-label="Anonymity Type: ">elite </td>
+        <td data-label="Anonymity Type: ">elite</td>
         <td data-label="Checked: ">Jul-16, 04:39</td>
         <td data-label="Country: ">Brazil</td>
         <td data-label="City: ">&nbsp;</td>

tests/test_providers.py

@@ -50,8 +50,11 @@ def test_SemairProxyParser(self):
         with HTTMock(samair_mock):
             proxy_provider = SamairProxyParser('Samair', 'https://www.premproxy.com')
             proxy_list = proxy_provider.parse_proxyList()
+            proxy_list_addr = []
+            for proxy in proxy_list:
+                proxy_list_addr.append(proxy.getAddress())
             for item in samair_expected:
-                self.assertTrue(item in proxy_list)
+                self.assertTrue(item in proxy_list_addr)
 
 
 if __name__ == '__main__':