Finalising url abstraction - fixing printout format and adding author doc

Author: pgaref

project/http/requests/parsers/UrlParser.py

@@ -1,35 +1,37 @@
 from project.http.requests.errors.ParserExceptions import ParserException
 
+__author__ = 'pgaref'
+
 class UrlParser(object):
     """
         An abstract class representing any URL containing Proxy information
         To add an extra Proxy URL just implement this class and provide a 'url specific' parse_proxyList method
 
     Attributes:
         site url (hhtp)
-        bandwidth_limit_in_KBs (to remobe straggling proxies when provided by the url)
+        minimum_bandwidth_in_KBs (to avoid straggling proxies when having the extra info from proxy provider)
     """
 
-    def __init__(self, web_url, limitinKBs=None):
+    def __init__(self, web_url, bandwidthKBs=None):
         self.url = web_url
-        if limitinKBs is not None:
-            self.bandwidth_limit_in_KBs=limitinKBs
+        if bandwidthKBs is not None:
+            self.minimum_bandwidth_in_KBs=bandwidthKBs
         else:
-            self.bandwidth_limit_in_KBs=150
+            self.minimum_bandwidth_in_KBs=150
 
     def get_URl(self):
         if self.url is None:
             raise ParserException("webURL is NONE")
         return self.url
 
-    def get_bandwidthLimit(self):
-        if self.bandwidth_limit_in_KBs <- 0:
-            raise ParserException("invalid bandwidth limit {0} ".format(self.bandwidth_limit_in_KBs))
-        return self.bandwidth_limit_in_KBs
+    def get_min_bandwidth(self):
+        if self.minimum_bandwidth_in_KBs < 0:
+            raise ParserException("invalid minimum bandwidth limit {0} ".format(self.minimum_bandwidth_in_KBs))
+        return self.minimum_bandwidth_in_KBs
 
     def parse_proxyList(self):
         raise ParserException(" abstract method should be implemented by each subclass")
 
     def __str__(self):
-        return "URL Parser of '{0}' with bandwidth limit at '{1}' KBs"\
-            .format(self.url, self.bandwidth_limit_in_KBs)
+        return "URL Parser of '{0}' with required bandwidth: '{1}' KBs"\
+            .format(self.url, self.minimum_bandwidth_in_KBs)

project/http/requests/parsers/freeproxyParser.py

@@ -2,6 +2,7 @@
 import requests
 from bs4 import BeautifulSoup
 
+__author__ = 'pgaref'
 
 class freeproxyParser(UrlParser):
 
@@ -36,6 +37,6 @@ def parse_proxyList(self):
         return curr_proxy_list
 
     def __str__(self):
-        return "FreeProxy Parser of '{0}' with bandwidth limit at '{1}' KBs" \
-            .format(self.url, self.bandwidth_limit_in_KBs)
+        return "FreeProxy Parser of '{0}' with required bandwidth: '{1}' KBs" \
+            .format(self.url, self.minimum_bandwidth_in_KBs)
 

project/http/requests/parsers/proxyforeuParser.py

@@ -2,6 +2,8 @@
 import requests
 from bs4 import BeautifulSoup
 
+__author__ = 'pgaref'
+
 class proxyforeuParser(UrlParser):
 
     def __init__(self, web_url, bandwithdh=None):
@@ -28,7 +30,7 @@ def parse_proxyList(self):
             for field in dataset:
                 # Discard slow proxies! Speed is in KB/s
                 if field[0] == 'Speed':
-                    if float(field[1]) < self.get_bandwidthLimit():
+                    if float(field[1]) < self.get_min_bandwidth():
                         proxy_straggler = True
                 if field[0] == 'IP':
                     proxy = proxy + field[1] + ':'
@@ -42,5 +44,5 @@ def parse_proxyList(self):
         return curr_proxy_list
 
     def __str__(self):
-        return "ProxyForEU Parser of '{0}' with bandwidth limit at '{1}' KBs" \
-            .format(self.url, self.bandwidth_limit_in_KBs)
+        return "ProxyForEU Parser of '{0}' with required bandwidth: '{1}' KBs" \
+            .format(self.url, self.minimum_bandwidth_in_KBs)

project/http/requests/parsers/rebroweeblyParser.py

@@ -2,6 +2,8 @@
 import requests
 from bs4 import BeautifulSoup
 
+__author__ = 'pgaref'
+
 class rebroweeblyParser(UrlParser):
 
     def __init__(self, web_url):
@@ -20,5 +22,5 @@ def parse_proxyList(self):
         return curr_proxy_list
 
     def __str__(self):
-        return "RebroWeebly Parser of '{0}' with bandwidth limit at '{1}' KBs" \
-            .format(self.url, self.bandwidth_limit_in_KBs)
+        return "RebroWeebly Parser of '{0}' with required bandwidth: '{1}' KBs" \
+            .format(self.url, self.minimum_bandwidth_in_KBs)

project/http/requests/parsers/samairproxyParser.py

@@ -2,6 +2,8 @@
 import requests
 from bs4 import BeautifulSoup
 
+__author__ = 'pgaref'
+
 class semairproxyParser(UrlParser):
 
     def __init__(self, web_url):
@@ -34,5 +36,5 @@ def parse_proxyList(self):
         return curr_proxy_list
 
     def __str__(self):
-        return "SemairProxy Parser of '{0}' with bandwidth limit at '{1}' KBs" \
-            .format(self.url, self.bandwidth_limit_in_KBs)
+        return "SemairProxy Parser of '{0}' with required bandwidth: '{1}' KBs" \
+            .format(self.url, self.minimum_bandwidth_in_KBs)

project/http/requests/proxy/requestProxy.py

@@ -2,26 +2,22 @@
 from project.http.requests.parsers.proxyforeuParser import proxyforeuParser
 from project.http.requests.parsers.rebroweeblyParser import rebroweeblyParser
 from project.http.requests.parsers.samairproxyParser import semairproxyParser
-
-__author__ = 'pgaref'
-
 import requests
 from requests.exceptions import ConnectionError
 import random
-import os
 import time
-from bs4 import BeautifulSoup
 from requests.exceptions import ReadTimeout
 
+__author__ = 'pgaref'
 
 class RequestProxy:
     agent_file = '../data/user_agents.txt'
 
     def __init__(self, web_proxy_list=[]):
         self.useragents = self.load_user_agents(RequestProxy.agent_file)
+
         #####
         # Each of the classes below implements a specific URL Parser
-        # http://<USERNAME>:<PASSWORD>@<IP-ADDR>:<PORT>
         #####
         parsers = []
         parsers.append(freeproxyParser('http://free-proxy-list.net'))
@@ -83,24 +79,24 @@ def generate_proxied_request(self, url, params={}, req_timeout=30):
         request = None
         try:
             rand_proxy = random.choice(self.proxy_list)
-            print "Using proxy: " + str(rand_proxy)
+            print "Using proxy: {0}".format(str(rand_proxy))
             request = requests.get(test_url, proxies={"http": rand_proxy},
                                    headers=req_headers, timeout=req_timeout)
         except ConnectionError:
             self.proxy_list.remove(rand_proxy)
-            print "Proxy unreachable - Removed Straggling proxy :", rand_proxy, " PL Size = ",len(self.proxy_list)
+            print "Proxy unreachable - Removed Straggling proxy: {0} PL Size = {1}".format(rand_proxy, len(self.proxy_list))
             pass
         except ReadTimeout:
             self.proxy_list.remove(rand_proxy)
-            print "Read timed out - Removed Straggling proxy :", rand_proxy, " PL Size = ", len(self.proxy_list)
+            print "Read timed out - Removed Straggling proxy: {0} PL Size = {1}".format(rand_proxy, len(self.proxy_list))
             pass
         return request
 
 if __name__ == '__main__':
 
     start = time.time()
     req_proxy = RequestProxy()
-    print "Initialization took: ", (time.time()-start)
+    print "Initialization took: {0} sec".format((time.time()-start))
     print "Size : ", len(req_proxy.get_proxy_list())
     print " ALL = ", req_proxy.get_proxy_list()
 
@@ -109,7 +105,7 @@ def generate_proxied_request(self, url, params={}, req_timeout=30):
     while True:
         start = time.time()
         request = req_proxy.generate_proxied_request(test_url)
-        print "Proxied Request Took: ", (time.time()-start), " => Status: ", request.__str__()
+        print "Proxied Request Took: {0} sec => Status: {1}".format((time.time()-start), request.__str__())
         print "Proxy List Size: ", len(req_proxy.get_proxy_list())
 
         print"-> Going to sleep.."