Handling rare ChunkedEncodingError #6 caused by broken server connection

Author: pgaref

project/http/requests/proxy/requestProxy.py

@@ -9,6 +9,7 @@
 from project.http.requests.useragent.userAgent import UserAgentManager
 import requests
 from requests.exceptions import ConnectionError
+from requests.exceptions import ChunkedEncodingError
 import random
 import time
 from requests.exceptions import ReadTimeout
@@ -77,6 +78,13 @@ def generate_proxied_request(self, url, params={}, req_timeout=30):
                 pass
             print "Read timed out - Removed Straggling proxy: {0} PL Size = {1}".format(rand_proxy,
                                                                                         len(self.proxy_list))
+        except ChunkedEncodingError:
+            try:
+                self.proxy_list.remove(rand_proxy)
+            except ValueError:
+                pass
+            print "Wrong server chunked encoding - Removed Straggling proxy: {0} PL Size = {1}".format(rand_proxy,
+                                                                                        len(self.proxy_list))
 
 
 if __name__ == '__main__':