Wrap raw::IndexedValue with a C struct to avoid alignment errors on non-x86_64 platforms

Author: jbaiter

build-wheels.sh

@@ -45,8 +45,9 @@ function clean_project {
     # Remove compiled files that might cause conflicts
     pushd /io/
     rm -rf .cache .eggs rust_fst/_ffi.py build *.egg-info
-    find ./ -name "__pycache__" -type d -print0 |xargs rm -rf --
-    find ./ -name "*.pyc" -type f -print0 |xargs rm -rf --
+    find ./ -name "__pycache__" -type d -print0 |xargs -0 rm -rf
+    find ./ -name "*.pyc" -type f -print0 |xargs -0 rm -rf
+    find ./ -name "*.so" -type f -print0 |xargs -0 rm -rf
     popd
 }
 

fstwrapper/src/lib.rs

@@ -79,10 +79,19 @@ macro_rules! mapop_make_next_fn {
         pub extern fn $name(ptr: $t) -> *const MapOpItem {
             let stream = mutref_from_ptr!(ptr);
             match stream.next() {
-                Some((k, vs)) => to_raw_ptr(
-                    MapOpItem { key: ::std::ffi::CString::new(k).unwrap().into_raw(),
-                                num_values: vs.len(),
-                                values: vs.as_ptr()}),
+                Some((k, vs)) => {
+                    let vals: Vec<CIndexedValue> = (0..vs.len()).map(|idx| {
+                        CIndexedValue { index: vs[idx].index,
+                                        value: vs[idx].value }
+                    }).collect();
+                    let mut vals_boxed: Box<[CIndexedValue]> = vals.into_boxed_slice();
+                    let vals_ptr: *const CIndexedValue = vals_boxed.as_ptr();
+                    mem::forget(vals_boxed);
+                    to_raw_ptr(MapOpItem {
+                        key: ::std::ffi::CString::new(k).unwrap().into_raw(),
+                        num_values: vs.len(),
+                        values: vals_ptr })
+                },
                 None         => ::std::ptr::null_mut()
             }
         }

fstwrapper/src/map.rs

@@ -3,10 +3,10 @@ extern crate libc;
 use std::error::Error;
 use std::fs::File;
 use std::io;
+use std::mem;
 use std::ptr;
 use fst::{IntoStreamer, Streamer, Map, MapBuilder};
 use fst::map;
-use fst::raw;
 use fst_levenshtein::Levenshtein;
 use fst_regex::Regex;
 
@@ -27,9 +27,16 @@ pub struct MapItem {
 pub struct MapOpItem {
     key: *const libc::c_char,
     num_values: libc::size_t,
-    values: *const raw::IndexedValue
+    values: *const CIndexedValue
 }
 
+#[repr(C)]
+#[derive(Debug)]
+#[allow(dead_code)]
+pub struct CIndexedValue {
+    index: libc::size_t,
+    value: libc::uint64_t,
+}
 
 pub type FileMapBuilder = MapBuilder<&'static mut io::BufWriter<File>>;
 pub type MemMapBuilder = MapBuilder<Vec<u8>>;

rust_fst/common.py

@@ -84,8 +84,8 @@ def __next__(self):
         key = ffi.string(itm.key).decode('utf8')
         values = []
         for n in range(itm.num_values):
-            v = itm.values[n]
-            values.append(IndexedValue(int(v.index), v.value))
+            rust_val = itm.values[n]
+            values.append(IndexedValue(rust_val.index, rust_val.value))
         lib.fst_string_free(itm.key)
         lib.fst_map_opitem_free(itm)
         return (key, tuple(values))

tests/test_map.py

@@ -92,9 +92,12 @@ def test_map_union():
     b = Map.from_iter({'bar': 32, 'moo': 64})
     u = dict(a.union(b))
     assert len(u) == 3
-    assert u['bar'] == ((0, 8), (1, 32))
-    assert u['baz'] == ((0, 16),)
-    assert u['moo'] == ((1, 64),)
+    bar_itms = [(itm.index, itm.value) for itm in u['bar']]
+    assert bar_itms == [(0, 8), (1, 32)]
+    baz_itms = [(itm.index, itm.value) for itm in u['baz']]
+    assert baz_itms == [(0, 16)]
+    moo_itms = [(itm.index, itm.value) for itm in u['moo']]
+    assert moo_itms == [(1, 64)]
 
 
 def test_map_intersection():