Attempted at Python3 compatibility and testing

Author: palewire

.travis.yml

@@ -3,6 +3,7 @@ python:
 - '2.5'
 - '2.6'
 - '2.7'
+- '3.3'
 install:
 - pip install -r requirements.txt --use-mirrors
 script:

documentcloud/MultipartPostHandler.py

@@ -41,13 +41,16 @@
 """
 import os
 import sys
-import urllib
-import urllib2
+import six
 import tempfile
 import mimetools
 import mimetypes
 from os import SEEK_END
-from cStringIO import StringIO
+from six.moves import urllib
+try:
+    import io
+except ImportError:
+    import cStringIO as io
 
 
 class Callable:
@@ -59,25 +62,25 @@ def __init__(self, anycallable):
 doseq = 1
 
 
-class MultipartPostHandler(urllib2.BaseHandler):
+class MultipartPostHandler(urllib2.request.BaseHandler):
     # needs to run first
-    handler_order = urllib2.HTTPHandler.handler_order - 10
+    handler_order = urllib2.request.HTTPHandler.handler_order - 10
 
     def http_request(self, request):
         data = request.get_data()
         if data is not None and type(data) != str:
             v_files = []
             v_vars = []
             try:
-                for(key, value) in data.items():
+                for(key, value) in list(data.items()):
                     if hasattr(value, 'read'):
                         v_files.append((key, value))
                     else:
                         v_vars.append((key, value))
             except TypeError:
                 raise TypeError
             if len(v_files) == 0:
-                data = urllib.urlencode(v_vars, doseq)
+                data = urllib.parse.urlencode(v_vars, doseq)
             else:
                 boundary, data = self.multipart_encode(v_vars, v_files)
                 contenttype = 'multipart/form-data; boundary=%s' % boundary
@@ -86,9 +89,11 @@ def http_request(self, request):
                     request.get_header('Content-Type').find(
                         'multipart/form-data') != 0
                 ):
-                    print "Replacing %s with %s" % (
-                        request.get_header('content-type'),
-                        'multipart/form-data'
+                    six.print_(
+                            "Replacing %s with %s" % (
+                            request.get_header('content-type'),
+                            'multipart/form-data'
+                        )
                     )
                 request.add_unredirected_header('Content-Type', contenttype)
             request.add_data(data)
@@ -99,7 +104,7 @@ def multipart_encode(vars, files, boundary=None, buf=None):
         if boundary is None:
             boundary = mimetools.choose_boundary()
         if buf is None:
-            buf = StringIO()
+            buf = io.StringIO()
         for(key, value) in vars:
             buf.write('--%s\r\n' % boundary)
             buf.write('Content-Disposition: form-data; name="%s"' % key)
@@ -141,7 +146,7 @@ def getsize(o_file):
 
 
 def main():
-    opener = urllib2.build_opener(MultipartPostHandler)
+    opener = urllib2.request.build_opener(MultipartPostHandler)
 
     def validateFile(url):
         temp = tempfile.mkstemp(suffix=".html")

documentcloud/__init__.py

@@ -11,18 +11,19 @@
     https://www.documentcloud.org/help/api
 
 """
+from __future__ import absolute_import
 import os
+import six
 import copy
 import base64
-import urllib
-import urllib2
-from toolbox import retry
-from toolbox import DoesNotExistError
-from toolbox import DuplicateObjectError
-from toolbox import credentials_required
-from toolbox import CredentialsFailedError
+from .toolbox import retry
+from six.moves import urllib
+from .toolbox import DoesNotExistError
+from .toolbox import DuplicateObjectError
+from .toolbox import credentials_required
+from .toolbox import CredentialsFailedError
 from dateutil.parser import parse as dateparser
-from MultipartPostHandler import MultipartPostHandler
+from .MultipartPostHandler import MultipartPostHandler
 try:
     import json
 except ImportError:
@@ -50,7 +51,7 @@ def _make_request(self, url, params=None, opener=None):
         """
         # Create the request object
         args = [i for i in [url, params] if i]
-        request = urllib2.Request(*args)
+        request = urllib2.request.Request(*args)
         # If the client has credentials, include them as a header
         if self.username and self.password:
             credentials = '%s:%s' % (self.username, self.password)
@@ -62,14 +63,14 @@ def _make_request(self, url, params=None, opener=None):
         # If the request provides a custom opener, like the upload request,
         # which relies on a multipart request, it is applied here.
         if opener:
-            opener = urllib2.build_opener(opener)
+            opener = urllib2.request.build_opener(opener)
             request_method = opener.open
         else:
-            request_method = urllib2.urlopen
+            request_method = urllib2.request.urlopen
         # Make the request
         try:
             response = request_method(request)
-        except urllib2.HTTPError, e:
+        except urllib2.error.HTTPError as e:
             if e.code == 404:
                 raise DoesNotExistError("The resource you've requested does \
 not exist or is unavailable without the proper credentials.")
@@ -95,7 +96,7 @@ def put(self, method, params):
             # Pull the document_ids out of the params
             document_ids = params.get("document_ids")
             del params['document_ids']
-            params = urllib.urlencode(params, doseq=True)
+            params = urllib.parse.urlencode(params, doseq=True)
             # These need to be specially formatted in the style documentcloud
             # expects arrays. The example they provide is:
             # ?document_ids[]=28-boumediene&document_ids[]=\
@@ -108,15 +109,16 @@ def put(self, method, params):
             # Pull them out of the dict
             data = params.get("data")
             del params['data']
-            params = urllib.urlencode(params, doseq=True)
+            params = urllib.parse.urlencode(params, doseq=True)
             # Format them in the style documentcloud expects
             # ?data['foo']=bar&data['tit']=tat
             params += "".join([
-                '&data[%s]=%s' % (key, value) for key, value in data.items()
+                '&data[%s]=%s' % (key, value) for key, value in
+                list(data.items())
             ])
         else:
             # Otherwise, we can just use the vanilla urllib prep method
-            params = urllib.urlencode(params, doseq=True)
+            params = urllib.parse.urlencode(params, doseq=True)
         # Make the request
         self._make_request(
             self.BASE_URI + method,
@@ -129,7 +131,7 @@ def fetch(self, method, params=None):
         """
         # Encode params if they exist
         if params:
-            params = urllib.urlencode(params, doseq=True)
+            params = urllib.parse.urlencode(params, doseq=True)
         content = self._make_request(
             self.BASE_URI + method,
             params,
@@ -268,7 +270,7 @@ def upload(
         if project:
             params['project'] = project
         if data:
-            for key, value in data.items():
+            for key, value in list(data.items()):
                 is_valid_data_keyword(key)
                 params['data[%s]' % key] = value
         if secure:
@@ -425,7 +427,7 @@ def create(self, title, description=None, document_ids=None):
         }
         if description:
             params['description'] = description
-        params = urllib.urlencode(params, doseq=True)
+        params = urllib.parse.urlencode(params, doseq=True)
         if document_ids:
             # These need to be specially formatted in the style documentcloud
             # expects arrays. The example they provide is:
@@ -489,7 +491,7 @@ def __str__(self):
         return self.__unicode__().encode("utf-8")
 
     def __unicode__(self):
-        return unicode(self.title)
+        return six.u(self.title)
 
 
 class Annotation(BaseAPIObject):
@@ -506,14 +508,14 @@ def __str__(self):
         return self.__unicode__().encode("utf-8")
 
     def __unicode__(self):
-        return u''
+        return six.u('')
 
     def get_location(self):
         """
         Return the location as a good
         """
         image_string = self.__dict__['location']['image']
-        image_ints = map(int, image_string.split(","))
+        image_ints = list(map(int, image_string.split(",")))
         return Location(*image_ints)
     location = property(get_location)
 
@@ -623,7 +625,7 @@ def set_data(self, data):
         if not isinstance(data, type({})):
             raise TypeError("This attribute must be a dictionary.")
         # Validate keywords
-        for keyword in data.keys():
+        for keyword in list(data.keys()):
             is_valid_data_keyword(keyword)
         # Set the attribute
         self.__dict__['data'] = data
@@ -676,7 +678,7 @@ def get_entities(self):
                 "documents/%s/entities.json" % self.id
             ).get("entities")
             obj_list = []
-            for type, entity_list in entities.items():
+            for type, entity_list in list(entities.items()):
                 for entity in entity_list:
                     entity['type'] = type
                     obj = Entity(entity)
@@ -691,10 +693,10 @@ def get_entities(self):
 
     def _get_url(self, url):
         if self.access == 'public':
-            req = urllib2.Request(url)
+            req = urllib2.request.Request(url)
             try:
-                return urllib2.urlopen(req).read()
-            except urllib2.HTTPError:
+                return urllib2.request.urlopen(req).read()
+            except urllib2.error.HTTPError:
                 raise NotImplementedError(
                     "Currently, DocumentCloud only allows you to access this \
 resource on public documents."
@@ -931,7 +933,7 @@ class Entity(BaseAPIObject):
     Keywords and such extracted from the document by OpenCalais.
     """
     def __unicode__(self):
-        return unicode(self.value)
+        return six.u(self.value)
 
 
 class Location(object):
@@ -945,7 +947,7 @@ def __str__(self):
         return self.__unicode__().encode("utf-8")
 
     def __unicode__(self):
-        return u''
+        return six.u('')
 
     def __init__(self, top, right, bottom, left):
         self.top = top
@@ -959,7 +961,7 @@ class Mention(BaseAPIObject):
     A mention of a search found in the document.
     """
     def __unicode__(self):
-        return unicode("Page %s" % (self.page))
+        return six.u("Page %s" % (self.page))
 
 
 class Project(BaseAPIObject):
@@ -1058,7 +1060,7 @@ def __str__(self):
         return self.__unicode__().encode("utf-8")
 
     def __unicode__(self):
-        return u''
+        return six.u('')
 
     def __getattr__(self, name):
         # When these attrs don't exist in the DocumentCloud db,

documentcloud/toolbox.py

@@ -1,6 +1,7 @@
 """
 A few toys the API will use.
 """
+import six
 import time
 from functools import wraps
 
@@ -74,7 +75,7 @@ def f_retry(*args, **kwargs):
                     try_one_last_time = False
                     break
                 except ExceptionToCheck:
-                    print "Retrying in %s seconds" % str(mdelay)
+                    six.print_("Retrying in %s seconds" % str(mdelay))
                     time.sleep(mdelay)
                     mtries -= 1
                     mdelay *= backoff

requirements.txt

@@ -1,4 +1,6 @@
 argparse==1.2.1
 pep8==1.4.6
+pyflakes==0.7.3
 python-dateutil==1.5
 simplejson==3.3.1
+six==1.4.1

setup.py

@@ -88,6 +88,7 @@ def fullsplit(path, result=None):
       install_requires=[
         'python-dateutil==1.5',
         'simplejson==3.3.1',
+        'six==1.4.1',
       ],
 )
 

test.py

@@ -77,6 +77,7 @@ def get_editable_document(self, version):
             "2.5": "351008-lbex-docid-3383445",
             "2.6": "15144-mitchrpt",
             "2.7": "351151-lbex-docid-130036",
+            "3.3": "50986-lbhi_sec07940_755445",
         }
         return version2slug[str(version)]
 
@@ -93,6 +94,7 @@ def get_editable_project(self, version):
             "2.5": 11051,
             "2.6": 11048,
             "2.7": 11047,
+            "3.3": 11070,
         }
         return version2slug[str(version)]