Compatibility with Python >= 2.6

erlastic/__init__.py

@@ -15,7 +15,7 @@
 def mailbox_gen():
   while True:
     len_bin = sys.stdin.buffer.read(4)
-    if len(len_bin) != 4: return None
+    if len(len_bin) != 4: return
     (length,) = struct.unpack('!I',len_bin)
     yield decode(sys.stdin.buffer.read(length))
 def port_gen():

erlastic/codec.py

@@ -1,9 +1,11 @@
 
 from __future__ import division
 
+import six
 import struct
 import zlib
 
+from erlastic.compat import *
 from erlastic.constants import *
 from erlastic.types import *
 
@@ -23,25 +25,25 @@ def __init__(self):
                 except: pass
 
     def decode(self, buf, offset=0):
-        version = buf[offset]
+        version = six.indexbytes(buf, offset)
         if version != FORMAT_VERSION:
             raise EncodingError("Bad version number. Expected %d found %d" % (FORMAT_VERSION, version))
         return self.decode_part(buf, offset+1)[0]
 
     def decode_part(self, buf, offset=0):
-        return self.decoders[buf[offset]](buf, offset+1)
+        return self.decoders[six.indexbytes(buf, offset)](buf, offset+1)
 
     def decode_97(self, buf, offset):
         """SMALL_INTEGER_EXT"""
-        return buf[offset], offset+1
+        return six.indexbytes(buf, offset), offset+1
 
     def decode_98(self, buf, offset):
         """INTEGER_EXT"""
         return struct.unpack(">l", buf[offset:offset+4])[0], offset+4
 
     def decode_99(self, buf, offset):
         """FLOAT_EXT"""
-        return float(buf[offset:offset+31].split(b'\x00', 1)[0]), offset+31
+        return float(buf[offset:offset+31].split(six.b('\x00'), 1)[0]), offset+31
 
     def decode_70(self, buf, offset):
         """NEW_FLOAT_EXT"""
@@ -55,13 +57,13 @@ def decode_100(self, buf, offset):
 
     def decode_115(self, buf, offset):
         """SMALL_ATOM_EXT"""
-        atom_len = buf[offset]
+        atom_len = six.intexbytes(buf, offset)
         atom = buf[offset+1:offset+1+atom_len]
         return self.convert_atom(atom), offset+atom_len+1
 
     def decode_104(self, buf, offset):
         """SMALL_TUPLE_EXT"""
-        arity = buf[offset]
+        arity = six.indexbytes(buf, offset)
         offset += 1
 
         items = []
@@ -112,7 +114,7 @@ def decode_109(self, buf, offset):
 
     def decode_110(self, buf, offset):
         """SMALL_BIG_EXT"""
-        n = buf[offset]
+        n = six.indexbytes(buf, offset)
         offset += 1
         return self.decode_bigint(n, buf, offset)
 
@@ -123,12 +125,12 @@ def decode_111(self, buf, offset):
         return self.decode_bigint(n, buf, offset)
 
     def decode_bigint(self, n, buf, offset):
-        sign = buf[offset]
+        sign = six.indexbytes(buf, offset)
         offset += 1
         b = 1
         val = 0
         for i in range(n):
-            val += buf[offset] * b
+            val += six.indexbytes(buf, offset) * b
             b <<= 8
             offset += 1
         if sign != 0:
@@ -149,7 +151,7 @@ def decode_114(self, buf, offset):
         node, offset = self.decode_part(buf, offset+2)
         if not isinstance(node, Atom):
             raise EncodingError("Expected atom while parsing NEW_REFERENCE_EXT, found %r instead" % node)
-        creation = buf[offset]
+        creation = six.indexbytes(buf, offset)
         reference_id = struct.unpack(">%dL" % id_len, buf[offset+1:offset+1+4*id_len])
         return Reference(node, reference_id, creation), offset+1+4*id_len
 
@@ -178,7 +180,7 @@ def decode_113(self, buf, offset):
         if not isinstance(function, Atom):
             raise EncodingError("Expected atom while parsing EXPORT_EXT, found %r instead" % function)
         arity, offset = self.decode_part(buf, offset)
-        if not isinstance(arity, int):
+        if not isinstance(arity, six.integer_types):
             raise EncodingError("Expected integer while parsing EXPORT_EXT, found %r instead" % arity)
         return Export(module, function, arity), offset+1
 
@@ -206,11 +208,11 @@ def encode(self, obj, compressed=False):
         import sys
         import pprint
         #pprint.pprint(self.encode_part(obj),stream=sys.stderr)
-        ubuf = b"".join(self.encode_part(obj))
+        ubuf = six.b('').join(self.encode_part(obj))
         if compressed is True:
             compressed = 6
         if not (compressed is False \
-                    or (isinstance(compressed, int) \
+                    or (isinstance(compressed, six.integer_types) \
                             and compressed >= 0 and compressed <= 9)):
             raise TypeError("compressed must be True, False or "
                             "an integer between 0 and 9")
@@ -219,20 +221,20 @@ def encode(self, obj, compressed=False):
             if len(cbuf) < len(ubuf):
                 usize = struct.pack(">L", len(ubuf))
                 ubuf = "".join([COMPRESSED, usize, cbuf])
-        return bytes([FORMAT_VERSION]) + ubuf
+        return pack_bytes([FORMAT_VERSION]) + ubuf
 
     def encode_part(self, obj):
         if obj is False:
-            return [bytes([ATOM_EXT]), struct.pack(">H", 5), b"false"]
+            return [pack_bytes([ATOM_EXT]), struct.pack(">H", 5), b"false"]
         elif obj is True:
-            return [bytes([ATOM_EXT]), struct.pack(">H", 4), b"true"]
+            return [pack_bytes([ATOM_EXT]), struct.pack(">H", 4), b"true"]
         elif obj is None:
-            return [bytes([ATOM_EXT]), struct.pack(">H", 4), b"none"]
-        elif isinstance(obj, int):
+            return [pack_bytes([ATOM_EXT]), struct.pack(">H", 4), b"none"]
+        elif isinstance(obj, six.integer_types):
             if 0 <= obj <= 255:
-                return [bytes([SMALL_INTEGER_EXT,obj])]
+                return [pack_bytes([SMALL_INTEGER_EXT,obj])]
             elif -2147483648 <= obj <= 2147483647:
-                return [bytes([INTEGER_EXT]), struct.pack(">l", obj)]
+                return [pack_bytes([INTEGER_EXT]), struct.pack(">l", obj)]
             else:
                 sign = obj < 0
                 obj = abs(obj)
@@ -243,54 +245,57 @@ def encode_part(self, obj):
                     obj >>= 8
 
                 if len(big_buf) < 256:
-                    return [bytes([SMALL_BIG_EXT,len(big_buf),sign]),bytes(big_buf)]
+                    return [pack_bytes([SMALL_BIG_EXT,len(big_buf),sign]),
+                            pack_bytes(big_buf)]
                 else:
-                    return [bytes([LARGE_BIG_EXT]), struct.pack(">L", len(big_buf)), bytes([sign]), bytes(big_buf)]
+                    return [pack_bytes([LARGE_BIG_EXT]),
+                            struct.pack(">L", len(big_buf)),
+                            pack_bytes([sign]), pack_bytes(big_buf)]
         elif isinstance(obj, float):
             floatstr = ("%.20e" % obj).encode('ascii')
-            return [bytes([FLOAT_EXT]), floatstr + b"\x00"*(31-len(floatstr))]
+            return [pack_bytes([FLOAT_EXT]), floatstr + b"\x00"*(31-len(floatstr))]
         elif isinstance(obj, Atom):
             st = obj.encode('latin-1')
-            return [bytes([ATOM_EXT]), struct.pack(">H", len(st)), st]
-        elif isinstance(obj, str):
+            return [pack_bytes([ATOM_EXT]), struct.pack(">H", len(st)), st]
+        elif isinstance(obj, six.string_types):
             st = obj.encode('utf-8')
-            return [bytes([BINARY_EXT]), struct.pack(">L", len(st)), st]
-        elif isinstance(obj, bytes):
-            return [bytes([BINARY_EXT]), struct.pack(">L", len(obj)), obj]
+            return [pack_bytes([BINARY_EXT]), struct.pack(">L", len(st)), st]
+        elif isinstance(obj, six.binary_type):
+            return [pack_bytes([BINARY_EXT]), struct.pack(">L", len(obj)), obj]
         elif isinstance(obj, tuple):
             n = len(obj)
             if n < 256:
-                buf = [bytes([SMALL_TUPLE_EXT,n])]
+                buf = [pack_bytes([SMALL_TUPLE_EXT,n])]
             else:
-                buf = [bytes([LARGE_TUPLE_EXT]), struct.pack(">L", n)]
+                buf = [pack_bytes([LARGE_TUPLE_EXT]), struct.pack(">L", n)]
             for item in obj:
                 buf += self.encode_part(item)
             return buf
         elif obj == []:
-            return [bytes([NIL_EXT])]
+            return [pack_bytes([NIL_EXT])]
         elif isinstance(obj, list):
-            buf = [bytes([LIST_EXT]), struct.pack(">L", len(obj))]
+            buf = [pack_bytes([LIST_EXT]), struct.pack(">L", len(obj))]
             for item in obj:
                 buf += self.encode_part(item)
-            buf.append(bytes([NIL_EXT])) # list tail - no such thing in Python
+            buf.append(pack_bytes([NIL_EXT])) # list tail - no such thing in Python
             return buf
         elif isinstance(obj, Reference):
-            return [bytes([NEW_REFERENCE_EXT]),
-                struct.pack(">H", len(obj.ref_id)),
-                bytes([ATOM_EXT]), struct.pack(">H", len(obj.node)), obj.node.encode('latin-1'),
-                bytes([obj.creation]), struct.pack(">%dL" % len(obj.ref_id), *obj.ref_id)]
+            return [pack_bytes([NEW_REFERENCE_EXT]), struct.pack(">H", len(obj.ref_id)),
+                    pack_bytes([ATOM_EXT]), struct.pack(">H", len(obj.node)),
+                    obj.node.encode('latin-1'), pack_bytes([obj.creation]),
+                    struct.pack(">%dL" % len(obj.ref_id), *obj.ref_id)]
         elif isinstance(obj, Port):
-            return [bytes([PORT_EXT]),
-                bytes([ATOM_EXT]), struct.pack(">H", len(obj.node)), obj.node.encode('latin-1'),
-                struct.pack(">LB", obj.port_id, obj.creation)]
+            return [pack_bytes([PORT_EXT]), pack_bytes([ATOM_EXT]),
+                    struct.pack(">H", len(obj.node)), obj.node.encode('latin-1'),
+                    struct.pack(">LB", obj.port_id, obj.creation)]
         elif isinstance(obj, PID):
-           return [bytes([PID_EXT]),
-                bytes([ATOM_EXT]), struct.pack(">H", len(obj.node)), obj.node.encode('latin-1'),
-                struct.pack(">LLB", obj.pid_id, obj.serial, obj.creation)]
+           return [pack_bytes([PID_EXT]), pack_bytes([ATOM_EXT]),
+                   struct.pack(">H", len(obj.node)), obj.node.encode('latin-1'),
+                   struct.pack(">LLB", obj.pid_id, obj.serial, obj.creation)]
         elif isinstance(obj, Export):
-            return [bytes([EXPORT_EXT]),
-                bytes([ATOM_EXT]), struct.pack(">H", len(obj.module)), obj.module.encode('latin-1'),
-                bytes([ATOM_EXT]), struct.pack(">H", len(obj.function)), obj.function.encode('latin-1'),
-                bytes([SMALL_INTEGER_EXT,obj.arity])]
+            return [pack_bytes([EXPORT_EXT]), pack_bytes([ATOM_EXT]),
+                    struct.pack(">H", len(obj.module)), obj.module.encode('latin-1'),
+                    pack_bytes([ATOM_EXT]), struct.pack(">H", len(obj.function)),
+                    obj.function.encode('latin-1'), pack_bytes([SMALL_INTEGER_EXT,obj.arity])]
         else:
             raise NotImplementedError("Unable to serialize %r" % obj)

erlastic/compat.py

@@ -0,0 +1,10 @@
+import sys
+
+from array import array
+
+__all__ = ['pack_bytes']
+
+if sys.version_info < (3,):
+    pack_bytes = lambda a: array('B', a).tostring()
+else:
+    pack_bytes = bytes

setup.py

@@ -12,6 +12,7 @@
     author_email = 'samuel@descolada.com',
     url = 'http://github.com/samuel/python-erlastic',
     packages = ['erlastic'],
+    install_requires=['six'],
     classifiers = [
         'Intended Audience :: Developers',
         'License :: OSI Approved :: BSD License',

tests.py

@@ -1,58 +1,60 @@
 #!/usr/bin/env python
 
+import six
 import unittest
 
 from erlastic import decode, encode
+from erlastic.compat import *
 from erlastic.types import *
 
 erlang_term_binaries = [
     # nil
-    ([], list, b"\x83j"),
+    ([], list, six.b('\x83j')),
     # binary
-    (b"foo", bytes, b'\x83m\x00\x00\x00\x03foo'),
+    (six.b('foo'), six.binary_type, six.b('\x83m\x00\x00\x00\x03foo')),
     # atom
-    (Atom("foo"), Atom, b'\x83d\x00\x03foo'),
+    (Atom("foo"), Atom, six.b('\x83d\x00\x03foo')),
     # atom true
-    (True, bool, b'\x83d\x00\x04true'),
+    (True, bool, six.b('\x83d\x00\x04true')),
     # atom false
-    (False, bool, b'\x83d\x00\x05false'),
+    (False, bool, six.b('\x83d\x00\x05false')),
     # atom none
-    (None, type(None), b'\x83d\x00\x04none'),
+    (None, type(None), six.b('\x83d\x00\x04none')),
     # small integer
-    (123, int, b'\x83a{'),
+    (123, six.integer_types, six.b('\x83a{')),
     # integer
-    (12345, int, b'\x83b\x00\x0009'),
+    (12345, six.integer_types, six.b('\x83b\x00\x0009')),
     # float
-    (1.2345, float, b'\x83c1.23449999999999993072e+00\x00\x00\x00\x00\x00'),
+    (1.2345, float, six.b('\x83c1.23449999999999993072e+00\x00\x00\x00\x00\x00')),
     # tuple
-    ((Atom("foo"), b"test", 123), tuple, b'\x83h\x03d\x00\x03foom\x00\x00\x00\x04testa{'),
+    ((Atom("foo"), b"test", 123), tuple, six.b('\x83h\x03d\x00\x03foom\x00\x00\x00\x04testa{')),
     # list
-    ([1024, b"test", 4.096], list, b'\x83l\x00\x00\x00\x03b\x00\x00\x04\x00m\x00\x00\x00\x04testc4.09600000000000008527e+00\x00\x00\x00\x00\x00j'),
-    ([102, 111, 111], list, b'\x83l\x00\x00\x00\x03afaoaoj'),
+    ([1024, b"test", 4.096], list, six.b('\x83l\x00\x00\x00\x03b\x00\x00\x04\x00m\x00\x00\x00\x04testc4.09600000000000008527e+00\x00\x00\x00\x00\x00j')),
+    ([102, 111, 111], list, six.b('\x83l\x00\x00\x00\x03afaoaoj')),
     # small big
-    (12345678901234567890, int, b'\x83n\x08\x00\xd2\n\x1f\xeb\x8c\xa9T\xab'),
+    (12345678901234567890, six.integer_types, six.b('\x83n\x08\x00\xd2\n\x1f\xeb\x8c\xa9T\xab')),
     # large big
     (123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890,
-     int, b'\x83o\x00\x00\x01D\x00\xd2\n?\xce\x96\xf1\xcf\xacK\xf1{\xefa\x11=$^\x93\xa9\x88\x17\xa0\xc2\x01\xa5%\xb7\xe3Q\x1b\x00\xeb\xe7\xe5\xd5Po\x98\xbd\x90\xf1\xc3\xddR\x83\xd1)\xfc&\xeaH\xc31w\xf1\x07\xf3\xf33\x8f\xb7\x96\x83\x05t\xeci\x9cY"\x98\x98i\xca\x11bY=\xcc\xa1\xb4R\x1bl\x01\x86\x18\xe9\xa23\xaa\x14\xef\x11[}O\x14RU\x18$\xfe\x7f\x96\x94\xcer?\xd7\x8b\x9a\xa7v\xbd\xbb+\x07X\x94x\x7fI\x024.\xa0\xcc\xde\xef:\xa7\x89~\xa4\xafb\xe4\xc1\x07\x1d\xf3cl|0\xc9P`\xbf\xab\x95z\xa2DQf\xf7\xca\xef\xb0\xc4=\x11\x06*:Y\xf58\xaf\x18\xa7\x81\x13\xdf\xbdTl4\xe0\x00\xee\x93\xd6\x83V\xc9<\xe7I\xdf\xa8.\xf5\xfc\xa4$R\x95\xef\xd1\xa7\xd2\x89\xceu!\xf8\x08\xb1Zv\xa6\xd9z\xdb0\x88\x10\xf3\x7f\xd3sc\x98[\x1a\xac6V\x1f\xad0)\xd0\x978\xd1\x02\xe6\xfbH\x149\xdc).\xb5\x92\xf6\x91A\x1b\xcd\xb8`B\xc6\x04\x83L\xc0\xb8\xafN+\x81\xed\xec?;\x1f\xab1\xc1^J\xffO\x1e\x01\x87H\x0f.ZD\x06\xf0\xbak\xaagVH]\x17\xe6I.B\x14a2\xc1;\xd1+\xea.\xe4\x92\x15\x93\xe9\'E\xd0(\xcd\x90\xfb\x10'),
+     six.integer_types, six.b('\x83o\x00\x00\x01D\x00\xd2\n?\xce\x96\xf1\xcf\xacK\xf1{\xefa\x11=$^\x93\xa9\x88\x17\xa0\xc2\x01\xa5%\xb7\xe3Q\x1b\x00\xeb\xe7\xe5\xd5Po\x98\xbd\x90\xf1\xc3\xddR\x83\xd1)\xfc&\xeaH\xc31w\xf1\x07\xf3\xf33\x8f\xb7\x96\x83\x05t\xeci\x9cY"\x98\x98i\xca\x11bY=\xcc\xa1\xb4R\x1bl\x01\x86\x18\xe9\xa23\xaa\x14\xef\x11[}O\x14RU\x18$\xfe\x7f\x96\x94\xcer?\xd7\x8b\x9a\xa7v\xbd\xbb+\x07X\x94x\x7fI\x024.\xa0\xcc\xde\xef:\xa7\x89~\xa4\xafb\xe4\xc1\x07\x1d\xf3cl|0\xc9P`\xbf\xab\x95z\xa2DQf\xf7\xca\xef\xb0\xc4=\x11\x06*:Y\xf58\xaf\x18\xa7\x81\x13\xdf\xbdTl4\xe0\x00\xee\x93\xd6\x83V\xc9<\xe7I\xdf\xa8.\xf5\xfc\xa4$R\x95\xef\xd1\xa7\xd2\x89\xceu!\xf8\x08\xb1Zv\xa6\xd9z\xdb0\x88\x10\xf3\x7f\xd3sc\x98[\x1a\xac6V\x1f\xad0)\xd0\x978\xd1\x02\xe6\xfbH\x149\xdc).\xb5\x92\xf6\x91A\x1b\xcd\xb8`B\xc6\x04\x83L\xc0\xb8\xafN+\x81\xed\xec?;\x1f\xab1\xc1^J\xffO\x1e\x01\x87H\x0f.ZD\x06\xf0\xbak\xaagVH]\x17\xe6I.B\x14a2\xc1;\xd1+\xea.\xe4\x92\x15\x93\xe9\'E\xd0(\xcd\x90\xfb\x10')),
     # reference
-    (Reference('nonode@nohost', [33, 0, 0], 0), Reference, b'\x83r\x00\x03d\x00\rnonode@nohost\x00\x00\x00\x00!\x00\x00\x00\x00\x00\x00\x00\x00'),
+    (Reference('nonode@nohost', [33, 0, 0], 0), Reference, six.b('\x83r\x00\x03d\x00\rnonode@nohost\x00\x00\x00\x00!\x00\x00\x00\x00\x00\x00\x00\x00')),
     # port
-    (Port('nonode@nohost', 455, 0), Port, b'\x83fd\x00\rnonode@nohost\x00\x00\x01\xc7\x00'),
+    (Port('nonode@nohost', 455, 0), Port, six.b('\x83fd\x00\rnonode@nohost\x00\x00\x01\xc7\x00')),
     # pid
-    (PID('nonode@nohost', 31, 0, 0), PID, b'\x83gd\x00\rnonode@nohost\x00\x00\x00\x1f\x00\x00\x00\x00\x00'),
+    (PID('nonode@nohost', 31, 0, 0), PID, six.b('\x83gd\x00\rnonode@nohost\x00\x00\x00\x1f\x00\x00\x00\x00\x00')),
     # function export
-    (Export('jobqueue', 'stats', 0), Export, b'\x83qd\x00\x08jobqueued\x00\x05statsa\x00'),
+    (Export('jobqueue', 'stats', 0), Export, six.b('\x83qd\x00\x08jobqueued\x00\x05statsa\x00')),
 ]
 
 erlang_term_decode = [
     ## ext_string is an optimized way to send list of bytes, so not bijective (no erlang representation), only decode
-    (bytes([102, 111, 111]), bytes, b'\x83k\x00\x03foo')
+    (pack_bytes([102, 111, 111]), six.binary_type, six.b('\x83k\x00\x03foo'))
 ]
 
 
 erlang_term_encode = [
-    ## python3 choice : encode string as binary utf-8, so not bijective (decoded binary utf-8 is of type "bytes()"), only encode
-    ("foo", str, b'\x83m\x00\x00\x00\x03foo')
+    ## python3 choice : encode string as binary utf-8, so not bijective (decoded binary utf-8 is of type "six.binary_type"), only encode
+    ("foo", six.string_types, six.b('\x83m\x00\x00\x00\x03foo'))
 ]
 
 class ErlangTestCase(unittest.TestCase):