Tried to fix MultiPartEncoder to work with Python3 and BytesIO.

palewire · palewire · commit 1e68505dc38e · 2013-10-17T16:32:15.000-07:00
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,4 @@
+.tox/
 scratch.py
 _sources/*
 sphinx/*
diff --git a/documentcloud/MultipartPostHandler.py b/documentcloud/MultipartPostHandler.py
@@ -47,17 +47,14 @@
 from os import SEEK_END
 if six.PY3:
     import io
-    import urllib
+    import urllib.parse
+    import urllib.request
     from email.generator import _make_boundary as choose_boundary
 else:
     import cStringIO as io
     from six.moves import urllib
     from mimetools import choose_boundary
 
-class Callable:
-    def __init__(self, anycallable):
-        self.__call__ = anycallable
-
 # Controls how sequences are uncoded. If true, elements
 # may be given multiple values byassigning a sequence.
 doseq = 1
@@ -101,36 +98,79 @@ def http_request(self, request):
 
         return request
 
-    def multipart_encode(vars, files, boundary=None, buf=None):
-        if boundary is None:
-            boundary = choose_boundary()
-        if buf is None:
-            buf = io.StringIO()
-        for(key, value) in vars:
-            buf.write('--%s\r\n' % boundary)
-            buf.write('Content-Disposition: form-data; name="%s"' % key)
-            buf.write('\r\n\r\n' + value + '\r\n')
-        for(key, fd) in files:
-            try:
-                filename = fd.name.split('/')[-1]
-            except AttributeError:
-                # Spoof a file name if the object doesn't have one.
-                # This is designed to catch when the user submits
-                # a StringIO object
-                filename = 'temp.pdf'
-            contenttype = mimetypes.guess_type(filename)[0] or \
-                'application/octet-stream'
-            buf.write('--%s\r\n' % boundary)
-            buf.write('Content-Disposition: form-data; \
-name="%s"; filename="%s"\r\n' % (key, filename))
-            buf.write('Content-Type: %s\r\n' % contenttype)
-            # buffer += 'Content-Length: %s\r\n' % file_size
-            fd.seek(0)
-            buf.write('\r\n' + fd.read() + '\r\n')
-        buf.write('--' + boundary + '--\r\n\r\n')
-        buf = buf.getvalue()
-        return boundary, buf
-    multipart_encode = Callable(multipart_encode)
+    def multipart_encode(self, v_vars, files, boundary=None, buf=None):
+        if six.PY3:
+            if boundary is None:
+                boundary = choose_boundary()
+            if buf is None:
+                buf = io.BytesIO()
+            for(key, value) in v_vars:
+                buf.write(b'--' + boundary.encode("utf-8") + b'\r\n')
+                buf.write(
+                    b'Content-Disposition: form-data; name="' +
+                    key.encode("utf-8") +
+                    b'"'
+                )
+                buf.write(b'\r\n\r\n' + value.encode("utf-8") + b'\r\n')
+            for(key, fd) in files:
+                try:
+                    filename = fd.name.split('/')[-1]
+                except AttributeError:
+                    # Spoof a file name if the object doesn't have one.
+                    # This is designed to catch when the user submits
+                    # a StringIO object
+                    filename = 'temp.pdf'
+                contenttype = mimetypes.guess_type(filename)[0] or \
+                    b'application/octet-stream'
+                buf.write(b'--' + boundary.encode("utf-8") + b'\r\n')
+                buf.write(
+                    b'Content-Disposition: form-data; ' +
+                    b'name="' + key.encode("utf-8") + b'"; ' +
+                    b'filename="' + filename.encode("utf-8") + b'"\r\n'
+                )
+                buf.write(
+                    b'Content-Type: ' +
+                    contenttype.encode("utf-8") +
+                    b'\r\n'
+                )
+                fd.seek(0)
+                buf.write(
+                    b'\r\n' + fd.read() + b'\r\n'
+                )
+            buf.write(b'--')
+            buf.write(boundary.encode("utf-8"))
+            buf.write(b'--\r\n\r\n')
+            buf = buf.getvalue()
+            return boundary, buf
+        else:
+            if boundary is None:
+                boundary = choose_boundary()
+            if buf is None:
+                buf = io.StringIO()
+            for(key, value) in v_vars:
+                buf.write('--%s\r\n' % boundary)
+                buf.write('Content-Disposition: form-data; name="%s"' % key)
+                buf.write('\r\n\r\n' + value + '\r\n')
+            for(key, fd) in files:
+                try:
+                    filename = fd.name.split('/')[-1]
+                except AttributeError:
+                    # Spoof a file name if the object doesn't have one.
+                    # This is designed to catch when the user submits
+                    # a StringIO object
+                    filename = 'temp.pdf'
+                contenttype = mimetypes.guess_type(filename)[0] or \
+                    'application/octet-stream'
+                buf.write('--%s\r\n' % boundary)
+                buf.write('Content-Disposition: form-data; \
+    name="%s"; filename="%s"\r\n' % (key, filename))
+                buf.write('Content-Type: %s\r\n' % contenttype)
+                # buffer += 'Content-Length: %s\r\n' % file_size
+                fd.seek(0)
+                buf.write('\r\n' + fd.read() + '\r\n')
+            buf.write('--' + boundary + '--\r\n\r\n')
+            buf = buf.getvalue()
+            return boundary, buf
     https_request = http_request
 
 
diff --git a/documentcloud/__init__.py b/documentcloud/__init__.py
@@ -25,7 +25,9 @@
 from dateutil.parser import parse as dateparser
 from .MultipartPostHandler import MultipartPostHandler
 if six.PY3:
-    import urllib
+    import urllib.parse
+    import urllib.error
+    import urllib.request
 else:
     from six.moves import urllib
 try:
@@ -48,7 +50,7 @@ def __init__(self, username, password):
         self.username = username
         self.password = password
 
-    @retry(Exception, tries=4)
+    @retry(Exception, tries=3)
     def _make_request(self, url, params=None, opener=None):
         """
         Configure a HTTP request, fire it off and return the response.
@@ -60,8 +62,8 @@ def _make_request(self, url, params=None, opener=None):
         if self.username and self.password:
             credentials = '%s:%s' % (self.username, self.password)
             encoded_credentials = base64.encodestring(
-                credentials
-            ).replace("\n", "")
+                credentials.encode("utf-8")
+            ).decode("utf-8").replace("\n", "")
             header = 'Basic %s' % encoded_credentials
             request.add_header('Authorization', header)
         # If the request provides a custom opener, like the upload request,
@@ -124,10 +126,11 @@ def put(self, method, params):
         else:
             # Otherwise, we can just use the vanilla urllib prep method
             params = urllib.parse.urlencode(params, doseq=True)
+
         # Make the request
         self._make_request(
             self.BASE_URI + method,
-            params,
+            params.encode("utf-8"),
         )
 
     def fetch(self, method, params=None):
@@ -136,13 +139,13 @@ def fetch(self, method, params=None):
         """
         # Encode params if they exist
         if params:
-            params = urllib.parse.urlencode(params, doseq=True)
+            params = urllib.parse.urlencode(params, doseq=True).encode("utf-8")
         content = self._make_request(
             self.BASE_URI + method,
             params,
         )
         # Convert its JSON to a Python dictionary and return
-        return json.loads(content)
+        return json.loads(content.decode("utf-8"))
 
 
 class DocumentCloud(BaseDocumentCloudClient):
@@ -286,7 +289,7 @@ def upload(
             params,
             MultipartPostHandler
         )
-        return self.get(json.loads(response)['id'])
+        return self.get(json.loads(response.decode("utf-8"))['id'])
 
     @credentials_required
     def upload_directory(
@@ -441,8 +444,11 @@ def create(self, title, description=None, document_ids=None):
             params += "".join([
                 '&document_ids[]=%s' % id for id in document_ids
             ])
-        response = self._make_request(self.BASE_URI + "projects.json", params)
-        new_id = json.loads(response)['project']['id']
+        response = self._make_request(
+            self.BASE_URI + "projects.json",
+            params.encode("utf-8")
+        )
+        new_id = json.loads(response.decode("utf-8"))['project']['id']
         # If it doesn't exist, that suggests the project already exists
         if not new_id:
             raise DuplicateObjectError("The Project title you tried to create \
diff --git a/documentcloud/toolbox.py b/documentcloud/toolbox.py
@@ -59,7 +59,7 @@ def _checkcredentials(self, *args, **kwargs):
     return wraps(method_func)(_checkcredentials)
 
 
-def retry(ExceptionToCheck, tries=4, delay=3, backoff=2):
+def retry(ExceptionToCheck, tries=3, delay=2, backoff=2):
     """
     Retry decorator published by Saltry Crane.
 
diff --git a/test.py b/test.py
@@ -8,6 +8,7 @@
 """
 import os
 import sys
+import six
 import random
 import string
 import textwrap
@@ -32,7 +33,11 @@ def get_random_string(length=6):
     """
     Generate a random string of letters and numbers
     """
-    return unicode(''.join(random.choice(string.letters + string.digits) for i in xrange(length)))
+    return six.u(''.join(
+        random.choice(string.ascii_letters + string.digits)
+            for i in range(length)
+    ))
+
 
 PANGRAMS = {
     'en': 'The quick brown fox jumps over the lazy dog.',
@@ -80,7 +85,7 @@ def get_editable_document(self, version):
             "2.5": "351008-lbex-docid-3383445",
             "2.6": "15144-mitchrpt",
             "2.7": "351151-lbex-docid-130036",
-            "3.3": "50986-lbhi_sec07940_755445",
+            "3.3": "351029-lbex-docid-149714",
         }
         return version2slug[str(version)]
 
@@ -283,14 +288,14 @@ def test_get_put(self):
         source = get_random_string()
         description = get_random_string()
         data = {get_random_string(): get_random_string()}
-        if obj.resources.related_article == u'http://documents.latimes.com':
-            related_article = u'http://documentcloud.org'
+        if obj.resources.related_article == 'http://documents.latimes.com':
+            related_article = 'http://documentcloud.org'
         else:
-            related_article = u'http://documents.latimes.com'
-        if obj.resources.published_url == u'http://documents.latimes.com':
-            published_url = u'http://documentcloud.org'
+            related_article = 'http://documents.latimes.com'
+        if obj.resources.published_url == 'http://documents.latimes.com':
+            published_url = 'http://documentcloud.org'
         else:
-            published_url = u'http://documents.latimes.com'
+            published_url = 'http://documents.latimes.com'
         # Set the random strings our local object's attributes
         obj.title = title
         obj.source = source
@@ -438,7 +443,10 @@ def test_virtual_file_upload_and_delete(self):
         """
         path = os.path.join(os.path.dirname(__file__), "español.pdf")
         real_file = open(path, 'rb')
-        virtual_file = io.StringIO(real_file.read())
+        if six.PY3:
+            virtual_file = io.BytesIO(real_file.read())
+        else:
+            virtual_file = io.StringIO(real_file.read())
         obj = self.private_client.documents.upload(virtual_file, title='Espanola!')
         self.assertEqual(type(obj), Document)
         # Delete it
@@ -535,7 +543,7 @@ def test_get_document(self):
         Verify that a project can pull a particular document by id
         """
         obj = self.private_client.projects.get('934')
-        doc = obj.get_document(u'25798-pr-01092011-loughner')
+        doc = obj.get_document('25798-pr-01092011-loughner')
         self.assertEqual(type(doc), Document)
 
     def test_put(self):
@@ -545,8 +553,8 @@ def test_put(self):
         # Pull the object we'll deface
         obj = self.private_client.projects.get(self.editable_project)
         # Create random strings we will save to the editable attributes
-        title = u'The Klee Report (%s)' % get_random_string()
-        description = textwrap.dedent(u"""
+        title = 'The Klee Report (%s)' % get_random_string()
+        description = textwrap.dedent("""
         An independent probe into Sam Zell\'s purchase of Tribune Company by 
         investigator Kenneth Klee. Released at the end of July 2010. (%s)
         """)
@@ -565,8 +573,8 @@ def test_put(self):
         self.assertEqual(len(obj.document_list), 0)
         # Now add all the documents back in
         proj_ids = [
-            u'12667-the-klee-report-volume-2',
-            u'12666-the-klee-report-volume-1'
+            '12667-the-klee-report-volume-2',
+            '12666-the-klee-report-volume-1'
         ]
         for id in proj_ids:
             doc = self.private_client.documents.get(id)
@@ -660,7 +668,7 @@ def test_duplicate_object(self):
         Make sure DuplicateObjectError works.
         """
         obj = self.private_client.projects.get("703")
-        doc = self.private_client.documents.get(u'12666-the-klee-report-volume-1')
+        doc = self.private_client.documents.get('12666-the-klee-report-volume-1')
         self.assertRaises(DuplicateObjectError, obj.document_list.append, doc)
 
 
diff --git a/tox.ini b/tox.ini
@@ -1,9 +1,9 @@
 [tox]
-envlist=py25,py26,py27,py32,py33
+envlist=py33
 
 [testenv]
 deps=
     python-dateutil
     simplejson
     six
-commands=python quicktest.py documentcloud
+commands=python test.py

Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,4 @@`
	`1`	`+.tox/`
`1`	`2`	`scratch.py`
`2`	`3`	`_sources/*`
`3`	`4`	`sphinx/*`