More work on (#25)

*  Making sure request response is a valid page before parsing

*  Using the boolean response.ok attribute to make sure request was successful

Author: pgaref

http_request_randomizer/requests/parsers/FreeProxyParser.py

@@ -15,7 +15,13 @@ def __init__(self, web_url, timeout=None):
 
     def parse_proxyList(self):
         curr_proxy_list = []
-        content = requests.get(self.get_URl(), timeout=self.timeout).content
+        response = requests.get(self.get_URl(), timeout=self.timeout)
+
+        if not response.ok:
+            logger.warn("Proxy Provider url failed: {}".format(self.get_URl()))
+            return []
+
+        content = response.content
         soup = BeautifulSoup(content, "html.parser")
         table = soup.find("table", attrs={"class": "display fpltable"})
 

http_request_randomizer/requests/parsers/ProxyForEuParser.py

@@ -15,7 +15,13 @@ def __init__(self, web_url, bandwithdh=None, timeout=None):
 
     def parse_proxyList(self):
         curr_proxy_list = []
-        content = requests.get(self.get_URl(), timeout=self.timeout).content
+        response = requests.get(self.get_URl(), timeout=self.timeout)
+
+        if not response.ok:
+            logger.warn("Proxy Provider url failed: {}".format(self.get_URl()))
+            return []
+
+        content = response.content
         soup = BeautifulSoup(content, "html.parser")
         table = soup.find("table", attrs={"class": "proxy_list"})
 

http_request_randomizer/requests/parsers/RebroWeeblyParser.py

@@ -17,7 +17,13 @@ def __init__(self, web_url, timeout=None):
 
     def parse_proxyList(self, use_top15k=False):
         curr_proxy_list = []
-        content = requests.get(self.get_URl()+"/"+self.top_proxy_path, timeout=self.timeout).content
+        response = requests.get(self.get_URl()+"/"+self.top_proxy_path, timeout=self.timeout)
+
+        if not response.ok:
+            logger.warn("Proxy Provider url failed: {}".format(self.get_URl()))
+            return []
+
+        content = response.content
         soup = BeautifulSoup(content, "html.parser")
         table = soup.find("div", attrs={"class": "paragraph", 'style': "text-align:left;"}).find('font', attrs={
             'color': '#33a27f'})

http_request_randomizer/requests/parsers/SamairProxyParser.py

@@ -15,7 +15,13 @@ def __init__(self, web_url, timeout=None):
 
     def parse_proxyList(self):
         curr_proxy_list = []
-        content = requests.get(self.get_URl(), timeout=self.timeout).content
+        response = requests.get(self.get_URl(), timeout=self.timeout)
+
+        if not response.ok:
+            logger.warn("Proxy Provider url failed: {}".format(self.get_URl()))
+            return []
+
+        content = response.content
         soup = BeautifulSoup(content, "html.parser")
         # css provides the port number so we reverse it
         # for href in soup.findAll('link'):