Fix: Avoid x86-only hashes on Arm

Author: ashvardanian

Cargo.lock

@@ -30,15 +30,15 @@ bench_find = [
     "regex",        # Byteset Search
 ]
 bench_hash = [
-    "ahash",       # aHash
-    "xxhash-rust", # xxHash
-    "gxhash",      # gxHash
-    "blake3",      # Blake3
-    "crc32fast",   # CRC32
-    "murmur3",     # MurmurHash3
-    "cityhash",    # CityHash
-    "foldhash",    # FoldHash
-    "bit-set",     # for collision counting
+    "ahash",        # aHash
+    "xxhash-rust",  # xxHash
+    "gxhash",       # gxHash (x86_64 only)
+    "blake3",       # Blake3
+    "crc32fast",    # CRC32
+    "murmurhash32", # MurmurHash32
+    "cityhash",     # CityHash (x86_64 only)
+    "foldhash",     # FoldHash
+    "bit-set",      # for collision counting
 ]
 bench_sequence = [
     "arrow",  # Sorting
@@ -121,22 +121,14 @@ optional = true
 version = "1.3.2"
 optional = true
 
-[dependencies.murmur3]
-version = "0.5"
-optional = true
-
-[dependencies.cityhash]
-version = "0.1"
+[dependencies.murmurhash32]
+version = "0.3"
 optional = true
 
 [dependencies.foldhash]
 version = "0.2"
 optional = true
 
-[dependencies.gxhash]
-version = "3.4.1"
-optional = true
-
 [dependencies.ahash]
 version = "0.8"
 optional = true
@@ -194,6 +186,10 @@ optional = true
 version = "0.8"
 optional = true
 
+[target.'cfg(target_arch = "x86_64")'.dependencies]
+gxhash = { version = "3.4.1", optional = true }
+cityhash = { version = "0.1", optional = true }
+
 [[bench]]
 name = "bench_find"
 path = "bench_find.rs"

Cargo.toml

@@ -40,28 +40,29 @@ Many of them have similar pitfalls:
 StringZilla addresses those issues and seems to provide competitive performance.
 On Intel Sapphire Rapids CPU, on `xlsum.csv` dataset, the following numbers can be expected for hashing individual whitespace-delimited words and newline-delimited lines:
 
-| Library               | Bits  | Ports ¹ |    Short Words |      Long Lines |
-| --------------------- | :---: | :-----: | -------------: | --------------: |
-| Rust 🦀                |       |         |                |                 |
-| `std::hash`           |  64   |    ❌    |     0.43 GiB/s |      3.74 GiB/s |
-| `crc32fast::hash`     |  32   |    ✅    |     0.49 GiB/s |      8.45 GiB/s |
-| `xxh3::xxh3_64`       |  64   |    ✅    |     1.08 GiB/s |      9.48 GiB/s |
-| `aHash::hash_one`     |  64   |    ❌    |     1.23 GiB/s |      8.61 GiB/s |
-| `foldhash::hash_one`  |  64   |    ❌    |     1.02 GiB/s |      8.24 GiB/s |
-| `gxhash::gxhash64`    |  64   |    ❌    | __2.68 GiB/s__ |      9.19 GiB/s |
-| `stringzilla::hash`   |  64   |    ✅    |     1.84 GiB/s | __11.38 GiB/s__ |
-|                       |       |         |                |                 |
-| Python 🐍              |       |         |                |                 |
-| `hash`                | 32/64 |    ❌    |     0.13 GiB/s |      4.27 GiB/s |
-| `xxhash.xxh3_64`      |  64   |    ✅    |     0.04 GiB/s |      6.38 GiB/s |
-| `google_crc32c.value` |  32   |    ✅    |     0.04 GiB/s |      5.96 GiB/s |
-| `mmh3.hash32`         |  32   |    ✅    |     0.05 GiB/s |      2.65 GiB/s |
-| `mmh3.hash64`         |  64   |    ✅    |     0.03 GiB/s |      4.45 GiB/s |
-| `cityhash.CityHash64` |  64   |    ✅    |     0.06 GiB/s |      4.87 GiB/s |
-| `stringzilla.hash`    |  64   |    ✅    | __0.14 GiB/s__ |  __9.19 GiB/s__ |
+| Library               | Bits  | Ports ¹ | Arm ² |    Short Words |      Long Lines |
+| --------------------- | :---: | :-----: | :---: | -------------: | --------------: |
+| Rust 🦀                |       |         |       |                |                 |
+| `std::hash`           |  64   |    ❌    |   ✅   |     0.43 GiB/s |      3.74 GiB/s |
+| `crc32fast::hash`     |  32   |    ✅    |   ✅   |     0.49 GiB/s |      8.45 GiB/s |
+| `xxh3::xxh3_64`       |  64   |    ✅    |   ✅   |     1.08 GiB/s |      9.48 GiB/s |
+| `aHash::hash_one`     |  64   |    ❌    |   ✅   |     1.23 GiB/s |      8.61 GiB/s |
+| `foldhash::hash_one`  |  64   |    ❌    |   ✅   |     1.02 GiB/s |      8.24 GiB/s |
+| `gxhash::gxhash64`    |  64   |    ❌    |   ❌   |     2.68 GiB/s |      9.19 GiB/s |
+| `stringzilla::hash`   |  64   |    ✅    |   ✅   | __1.84 GiB/s__ | __11.38 GiB/s__ |
+|                       |       |         |       |                |
+| Python 🐍              |       |         |       |                |
+| `hash`                | 32/64 |    ❌    |   ✅   |     0.13 GiB/s |      4.27 GiB/s |
+| `xxhash.xxh3_64`      |  64   |    ✅    |   ✅   |     0.04 GiB/s |      6.38 GiB/s |
+| `google_crc32c.value` |  32   |    ✅    |   ✅   |     0.04 GiB/s |      5.96 GiB/s |
+| `mmh3.hash32`         |  32   |    ✅    |   ✅   |     0.05 GiB/s |      2.65 GiB/s |
+| `mmh3.hash64`         |  64   |    ✅    |   ✅   |     0.03 GiB/s |      4.45 GiB/s |
+| `cityhash.CityHash64` |  64   |    ✅    |   ❌   |     0.06 GiB/s |      4.87 GiB/s |
+| `stringzilla.hash`    |  64   |    ✅    |   ✅   | __0.14 GiB/s__ |  __9.19 GiB/s__ |
 
 
 > ¹ Portability means availability in multiple other programming languages, like C, C++, Python, Java, Go, JavaScript, etc.
+> ² Most hash functions work on both x86 and Arm, as well as many other CPU architectures, but gxHash, and many MurMurHash and CityHash implementations don't.
 
 In larger systems, however, we often need the ability to incrementally hash the data.
 This is especially important in distributed systems, where the data is too large to fit into memory at once.

README.md

@@ -12,9 +12,11 @@ The benchmarks compare the performance of different hash functions including:
 - StringZilla (`bytesum`, `hash`, and incremental `hash` function variants)
 - aHash (both incremental and single-entry variants)
 - xxHash (xxh3) through the third-party `xxhash-rust` crate
-- gxhash (gxhash64)
+- gxhash (gxhash64, x86_64 only)
 - FoldHash (a fast hash with good quality)
 - CRC32 (IEEE) via `crc32fast`
+- MurmurHash32 via `murmurhash32`
+- CityHash64 via `cityhash` (x86_64 only)
 - Blake3 (the only cryptographic hash in the comparison, for reference)
 
 ## Usage Examples
@@ -36,26 +38,27 @@ RUSTFLAGS="-C target-cpu=native" \
     cargo criterion --features bench_hash bench_hash --jobs $(nproc)
 ```
 
-For `gxhash`, ensure that your CPU supports the required AES and SSE2 instructions.
+Note: `gxhash` and `cityhash` are only compiled on x86_64 targets as they require x86-specific instructions.
 "#]
 use std::collections::HashSet;
 use std::env;
 use std::error::Error;
 use std::fs;
 use std::hash::{BuildHasher, Hasher};
 use std::hint::black_box;
-use std::io::Cursor;
 
 use bit_set::BitSet;
 use criterion::{Criterion, Throughput};
 
 use ahash::RandomState as AHashState;
 use blake3;
+#[cfg(target_arch = "x86_64")]
 use cityhash;
 use crc32fast;
 use foldhash;
+#[cfg(target_arch = "x86_64")]
 use gxhash;
-use murmur3;
+use murmurhash32;
 use stringzilla::sz;
 use xxhash_rust::xxh3::xxh3_64;
 
@@ -236,7 +239,8 @@ fn bench_stateless(
         print_collision_rate(&unique_tokens, |t| xxh3_64(t));
     }
 
-    // Benchmark: gxhash
+    // Benchmark: gxhash (x86_64 only)
+    #[cfg(target_arch = "x86_64")]
     if should_run("stateless/gxhash::gxhash64") {
         group.bench_function("gxhash::gxhash64", |b| {
             b.iter(|| {
@@ -276,24 +280,21 @@ fn bench_stateless(
         print_collision_rate(&unique_tokens, |t| crc32fast::hash(t) as u64);
     }
 
-    // Benchmark: MurmurHash3 (x64_128) via `murmur3` (stateless)
-    if should_run("stateless/murmur3::x64_128") {
-        group.bench_function("murmur3::x64_128", |b| {
+    // Benchmark: MurmurHash32 via `murmurhash32` (stateless)
+    if should_run("stateless/murmurhash32") {
+        group.bench_function("murmurhash32", |b| {
             b.iter(|| {
                 for token in tokens {
                     let t = black_box(*token);
-                    let mut cursor = Cursor::new(t);
-                    let _ = black_box(murmur3::murmur3_x64_128(&mut cursor, 0).unwrap());
+                    let _ = black_box(murmurhash32::murmurhash3(t) as u64);
                 }
             })
         });
-        print_collision_rate(&unique_tokens, |t| {
-            let mut cursor = Cursor::new(t);
-            murmur3::murmur3_x64_128(&mut cursor, 0).unwrap() as u64
-        });
+        print_collision_rate(&unique_tokens, |t| murmurhash32::murmurhash3(t) as u64);
     }
 
-    // Benchmark: CityHash64 via `cityhash` (stateless)
+    // Benchmark: CityHash64 via `cityhash` (stateless, x86_64 only)
+    #[cfg(target_arch = "x86_64")]
     if should_run("stateless/cityhash::city_hash_64") {
         group.bench_function("cityhash::city_hash_64", |b| {
             b.iter(|| {