From a73dfed51fb1807a6b97116d3f0bf31b1f6f9326 Mon Sep 17 00:00:00 2001 From: Marco Barbone Date: Sun, 5 Jul 2026 17:08:01 -0400 Subject: [PATCH] perf(avx512f): single vpermt2 for zip_lo/zip_hi The AVX-512 zip_lo/zip_hi lowered the interleave as per-128-bit unpacklo/unpackhi followed by a 3x insertf32x4 / extractf32x4 lane-fixup pile (~7 shuffle-port uops) needed to undo AVX-512's in-lane element ordering. A single 2-source cross-lane permute (vpermt2d/vpermt2q for integral, vpermt2ps/vpermt2pd for float/double) produces the same result in one port-5 uop. --- include/xsimd/arch/xsimd_avx512f.hpp | 81 ++++++++-------------------- 1 file changed, 21 insertions(+), 60 deletions(-) diff --git a/include/xsimd/arch/xsimd_avx512f.hpp b/include/xsimd/arch/xsimd_avx512f.hpp index e508f3d0e..b656c35c0 100644 --- a/include/xsimd/arch/xsimd_avx512f.hpp +++ b/include/xsimd/arch/xsimd_avx512f.hpp @@ -2806,7 +2806,6 @@ namespace xsimd XSIMD_INLINE batch zip_hi(batch const& self, batch const& other, requires_arch) noexcept { - __m512i lo, hi; XSIMD_IF_CONSTEXPR(sizeof(T) == 1) { assert(false && "not implemented yet"); @@ -2819,62 +2818,43 @@ namespace xsimd } else XSIMD_IF_CONSTEXPR(sizeof(T) == 4) { - lo = _mm512_unpacklo_epi32(self, other); - hi = _mm512_unpackhi_epi32(self, other); + __m512i idx = _mm512_setr_epi32(8, 24, 9, 25, 10, 26, 11, 27, + 12, 28, 13, 29, 14, 30, 15, 31); + return _mm512_permutex2var_epi32(self, idx, other); } else XSIMD_IF_CONSTEXPR(sizeof(T) == 8) { - lo = _mm512_unpacklo_epi64(self, other); - hi = _mm512_unpackhi_epi64(self, other); + __m512i idx = _mm512_setr_epi64(4, 12, 5, 13, 6, 14, 7, 15); + return _mm512_permutex2var_epi64(self, idx, other); } else { assert(false && "unsupported arch/op combination"); return {}; } - return _mm512_inserti32x4( - _mm512_inserti32x4( - _mm512_inserti32x4(hi, _mm512_extracti32x4_epi32(lo, 2), 0), - _mm512_extracti32x4_epi32(lo, 3), - 2), - _mm512_extracti32x4_epi32(hi, 2), - 1); } template XSIMD_INLINE batch zip_hi(batch const& self, batch const& other, requires_arch) noexcept { - auto lo = _mm512_unpacklo_ps(self, other); - auto hi = _mm512_unpackhi_ps(self, other); - return _mm512_insertf32x4( - _mm512_insertf32x4( - _mm512_insertf32x4(hi, _mm512_extractf32x4_ps(lo, 2), 0), - _mm512_extractf32x4_ps(lo, 3), - 2), - _mm512_extractf32x4_ps(hi, 2), - 1); + __m512i idx = _mm512_setr_epi32(8, 24, 9, 25, 10, 26, 11, 27, + 12, 28, 13, 29, 14, 30, 15, 31); + return _mm512_permutex2var_ps(self, idx, other); } template XSIMD_INLINE batch zip_hi(batch const& self, batch const& other, requires_arch) noexcept { - auto lo = _mm512_castpd_ps(_mm512_unpacklo_pd(self, other)); - auto hi = _mm512_castpd_ps(_mm512_unpackhi_pd(self, other)); - return _mm512_castps_pd(_mm512_insertf32x4( - _mm512_insertf32x4( - _mm512_insertf32x4(hi, _mm512_extractf32x4_ps(lo, 2), 0), - _mm512_extractf32x4_ps(lo, 3), - 2), - _mm512_extractf32x4_ps(hi, 2), - 1)); + __m512i idx = _mm512_setr_epi64(4, 12, 5, 13, 6, 14, 7, 15); + return _mm512_permutex2var_pd(self, idx, other); } // zip_lo + // See zip_hi: one vpermt2{d,q,ps,pd} in place of the unpack+insert128 pile. template ::value>> XSIMD_INLINE batch zip_lo(batch const& self, batch const& other, requires_arch) noexcept { - __m512i lo, hi; XSIMD_IF_CONSTEXPR(sizeof(T) == 1) { assert(false && "not implemented yet"); @@ -2887,54 +2867,35 @@ namespace xsimd } else XSIMD_IF_CONSTEXPR(sizeof(T) == 4) { - lo = _mm512_unpacklo_epi32(self, other); - hi = _mm512_unpackhi_epi32(self, other); + __m512i idx = _mm512_setr_epi32(0, 16, 1, 17, 2, 18, 3, 19, + 4, 20, 5, 21, 6, 22, 7, 23); + return _mm512_permutex2var_epi32(self, idx, other); } else XSIMD_IF_CONSTEXPR(sizeof(T) == 8) { - lo = _mm512_unpacklo_epi64(self, other); - hi = _mm512_unpackhi_epi64(self, other); + __m512i idx = _mm512_setr_epi64(0, 8, 1, 9, 2, 10, 3, 11); + return _mm512_permutex2var_epi64(self, idx, other); } else { assert(false && "unsupported arch/op combination"); return {}; } - return _mm512_inserti32x4( - _mm512_inserti32x4( - _mm512_inserti32x4(lo, _mm512_extracti32x4_epi32(hi, 0), 1), - _mm512_extracti32x4_epi32(hi, 1), - 3), - _mm512_extracti32x4_epi32(lo, 1), - 2); } template XSIMD_INLINE batch zip_lo(batch const& self, batch const& other, requires_arch) noexcept { - auto lo = _mm512_unpacklo_ps(self, other); - auto hi = _mm512_unpackhi_ps(self, other); - return _mm512_insertf32x4( - _mm512_insertf32x4( - _mm512_insertf32x4(lo, _mm512_extractf32x4_ps(hi, 0), 1), - _mm512_extractf32x4_ps(hi, 1), - 3), - _mm512_extractf32x4_ps(lo, 1), - 2); + __m512i idx = _mm512_setr_epi32(0, 16, 1, 17, 2, 18, 3, 19, + 4, 20, 5, 21, 6, 22, 7, 23); + return _mm512_permutex2var_ps(self, idx, other); } template XSIMD_INLINE batch zip_lo(batch const& self, batch const& other, requires_arch) noexcept { - auto lo = _mm512_castpd_ps(_mm512_unpacklo_pd(self, other)); - auto hi = _mm512_castpd_ps(_mm512_unpackhi_pd(self, other)); - return _mm512_castps_pd(_mm512_insertf32x4( - _mm512_insertf32x4( - _mm512_insertf32x4(lo, _mm512_extractf32x4_ps(hi, 0), 1), - _mm512_extractf32x4_ps(hi, 1), - 3), - _mm512_extractf32x4_ps(lo, 1), - 2)); + __m512i idx = _mm512_setr_epi64(0, 8, 1, 9, 2, 10, 3, 11); + return _mm512_permutex2var_pd(self, idx, other); } // widen