Skip to content

ML-DSA AVX512: Add new assembly #11032

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
SparkiDev wants to merge 1 commit into
base: master
Choose a base branch
from
Open

ML-DSA AVX512: Add new assembly #11032

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
308 changes: 154 additions & 154 deletions wolfcrypt/src/aes_gcm_asm.asm
View file
Open in desktop

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions wolfcrypt/src/aes_x86_64_asm.asm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3093,7 +3093,7 @@ AES_CTR_encrypt_vaes PROC
vmovdqu OWORD PTR [rsp+112], xmm13
vmovdqu OWORD PTR [rsp+128], xmm14
vmovdqu OWORD PTR [rsp+144], xmm15
vbroadcasti128 ymm8, ptr_L_aes_ctr_bswap_vaes
vbroadcasti128 ymm8, OWORD PTR L_aes_ctr_bswap_vaes
vbroadcasti128 ymm7, OWORD PTR [r10]
vpshufb ymm7, ymm7, ymm8
vbroadcasti128 ymm12, OWORD PTR [ptr_L_aes_ctr_inc_vaes+32]
Expand Down Expand Up @@ -4623,7 +4623,7 @@ AES_CTR_encrypt_avx512 PROC
vmovdqu OWORD PTR [rsp+112], xmm13
vmovdqu OWORD PTR [rsp+128], xmm14
vmovdqu OWORD PTR [rsp+144], xmm15
vbroadcasti32x4 zmm8, ptr_L_aes_ctr_bswap_avx512
vbroadcasti32x4 zmm8, OWORD PTR L_aes_ctr_bswap_avx512
vbroadcasti32x4 zmm7, OWORD PTR [r10]
vpshufb zmm7, zmm7, zmm8
vbroadcasti32x4 zmm12, OWORD PTR [ptr_L_aes_ctr_inc_avx512+64]
Expand Down
48 changes: 24 additions & 24 deletions wolfcrypt/src/aes_xts_asm.asm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2919,7 +2919,7 @@ AES_XTS_encrypt_vaes PROC
vmovdqu OWORD PTR [rsp+200], xmm14
vmovdqu OWORD PTR [rsp+216], xmm15
vmovdqu xmm12, OWORD PTR L_vaes_aes_xts_gc_xts
vbroadcasti128 ymm13, ptr_L_vaes_aes_xts_poly
vbroadcasti128 ymm13, OWORD PTR L_vaes_aes_xts_poly
vmovdqu ymm14, YMMWORD PTR L_vaes_aes_xts_shl
vmovdqu ymm15, YMMWORD PTR L_vaes_aes_xts_shr
vmovdqu xmm8, OWORD PTR [r12]
Expand Down Expand Up @@ -3433,7 +3433,7 @@ AES_XTS_encrypt_update_vaes PROC
vmovdqu OWORD PTR [rsp+192], xmm14
vmovdqu OWORD PTR [rsp+208], xmm15
vmovdqu xmm12, OWORD PTR L_vaes_aes_xts_gc_xts
vbroadcasti128 ymm13, ptr_L_vaes_aes_xts_poly
vbroadcasti128 ymm13, OWORD PTR L_vaes_aes_xts_poly
vmovdqu ymm14, YMMWORD PTR L_vaes_aes_xts_shl
vmovdqu ymm15, YMMWORD PTR L_vaes_aes_xts_shr
vmovdqu xmm8, OWORD PTR [r8]
Expand Down Expand Up @@ -3914,7 +3914,7 @@ AES_XTS_decrypt_vaes PROC
vmovdqu OWORD PTR [rsp+200], xmm14
vmovdqu OWORD PTR [rsp+216], xmm15
vmovdqu xmm12, OWORD PTR L_vaes_aes_xts_gc_xts
vbroadcasti128 ymm13, ptr_L_vaes_aes_xts_poly
vbroadcasti128 ymm13, OWORD PTR L_vaes_aes_xts_poly
vmovdqu ymm14, YMMWORD PTR L_vaes_aes_xts_shl
vmovdqu ymm15, YMMWORD PTR L_vaes_aes_xts_shr
vmovdqu xmm8, OWORD PTR [r12]
Expand Down Expand Up @@ -4502,7 +4502,7 @@ AES_XTS_decrypt_update_vaes PROC
vmovdqu OWORD PTR [rsp+192], xmm14
vmovdqu OWORD PTR [rsp+208], xmm15
vmovdqu xmm12, OWORD PTR L_vaes_aes_xts_gc_xts
vbroadcasti128 ymm13, ptr_L_vaes_aes_xts_poly
vbroadcasti128 ymm13, OWORD PTR L_vaes_aes_xts_poly
vmovdqu ymm14, YMMWORD PTR L_vaes_aes_xts_shl
vmovdqu ymm15, YMMWORD PTR L_vaes_aes_xts_shr
vmovdqu xmm8, OWORD PTR [r8]
Expand Down Expand Up @@ -5127,9 +5127,9 @@ AES_XTS_encrypt_avx512 PROC
vmovdqu OWORD PTR [rsp+200], xmm14
vmovdqu OWORD PTR [rsp+216], xmm15
vmovdqu xmm12, OWORD PTR L_avx512_aes_xts_gc_xts
vbroadcasti32x4 zmm13, ptr_L_avx512_aes_xts_poly
vmovdqu64 zmm14, ptr_L_avx512_aes_xts_shl
vmovdqu64 zmm15, ptr_L_avx512_aes_xts_shr
vbroadcasti32x4 zmm13, OWORD PTR L_avx512_aes_xts_poly
vmovdqu64 zmm14, ZMMWORD PTR L_avx512_aes_xts_shl
vmovdqu64 zmm15, ZMMWORD PTR L_avx512_aes_xts_shr
vmovdqu xmm8, OWORD PTR [r12]
; aes_enc_block
vpxor xmm8, xmm8, [r9]
Expand Down Expand Up @@ -5452,7 +5452,7 @@ L_AES_XTS_encrypt_avx512_done_64:
; aes_enc_32
lea rcx, QWORD PTR [rdi+r13]
lea rdx, QWORD PTR [rsi+r13]
vmovdqu64 ymm0, [rcx]
vmovdqu64 ymm0, YMMWORD PTR [rcx]
vshufi64x2 zmm5, zmm8, zmm8, 0
vpsrlvq zmm6, zmm5, zmm15
vpclmulqdq zmm7, zmm6, zmm13, 1
Expand Down Expand Up @@ -5484,7 +5484,7 @@ L_AES_XTS_encrypt_avx512_done_64:
L_AES_XTS_encrypt_avx512_aes_enc_32_aes_enc_block_last:
vaesenclast ymm0, ymm0, ymm9
vpxorq ymm0, ymm0, ymm4
vmovdqu64 [rdx], ymm0
vmovdqu64 YMMWORD PTR [rdx], ymm0
vextracti32x4 xmm8, zmm4, 2
add r13d, 32
L_AES_XTS_encrypt_avx512_done_32:
Expand Down Expand Up @@ -5648,9 +5648,9 @@ AES_XTS_encrypt_update_avx512 PROC
vmovdqu OWORD PTR [rsp+192], xmm14
vmovdqu OWORD PTR [rsp+208], xmm15
vmovdqu xmm12, OWORD PTR L_avx512_aes_xts_gc_xts
vbroadcasti32x4 zmm13, ptr_L_avx512_aes_xts_poly
vmovdqu64 zmm14, ptr_L_avx512_aes_xts_shl
vmovdqu64 zmm15, ptr_L_avx512_aes_xts_shr
vbroadcasti32x4 zmm13, OWORD PTR L_avx512_aes_xts_poly
vmovdqu64 zmm14, ZMMWORD PTR L_avx512_aes_xts_shl
vmovdqu64 zmm15, ZMMWORD PTR L_avx512_aes_xts_shr
vmovdqu xmm8, OWORD PTR [r8]
xor r12d, r12d
cmp eax, 32
Expand Down Expand Up @@ -5938,7 +5938,7 @@ L_AES_XTS_encrypt_update_avx512_done_64:
; aes_enc_32
lea rcx, QWORD PTR [rdi+r12]
lea rdx, QWORD PTR [rsi+r12]
vmovdqu64 ymm0, [rcx]
vmovdqu64 ymm0, YMMWORD PTR [rcx]
vshufi64x2 zmm5, zmm8, zmm8, 0
vpsrlvq zmm6, zmm5, zmm15
vpclmulqdq zmm7, zmm6, zmm13, 1
Expand Down Expand Up @@ -5970,7 +5970,7 @@ L_AES_XTS_encrypt_update_avx512_done_64:
L_AES_XTS_encrypt_update_avx512_aes_enc_32_aes_enc_block_last:
vaesenclast ymm0, ymm0, ymm9
vpxorq ymm0, ymm0, ymm4
vmovdqu64 [rdx], ymm0
vmovdqu64 YMMWORD PTR [rdx], ymm0
vextracti32x4 xmm8, zmm4, 2
add r12d, 32
L_AES_XTS_encrypt_update_avx512_done_32:
Expand Down Expand Up @@ -6136,9 +6136,9 @@ AES_XTS_decrypt_avx512 PROC
vmovdqu OWORD PTR [rsp+200], xmm14
vmovdqu OWORD PTR [rsp+216], xmm15
vmovdqu xmm12, OWORD PTR L_avx512_aes_xts_gc_xts
vbroadcasti32x4 zmm13, ptr_L_avx512_aes_xts_poly
vmovdqu64 zmm14, ptr_L_avx512_aes_xts_shl
vmovdqu64 zmm15, ptr_L_avx512_aes_xts_shr
vbroadcasti32x4 zmm13, OWORD PTR L_avx512_aes_xts_poly
vmovdqu64 zmm14, ZMMWORD PTR L_avx512_aes_xts_shl
vmovdqu64 zmm15, ZMMWORD PTR L_avx512_aes_xts_shr
vmovdqu xmm8, OWORD PTR [r12]
; aes_enc_block
vpxor xmm8, xmm8, [r9]
Expand Down Expand Up @@ -6501,7 +6501,7 @@ L_AES_XTS_decrypt_avx512_mul16_32:
; aes_dec_32
lea rcx, QWORD PTR [rdi+r13]
lea rdx, QWORD PTR [rsi+r13]
vmovdqu64 ymm0, [rcx]
vmovdqu64 ymm0, YMMWORD PTR [rcx]
vshufi64x2 zmm5, zmm8, zmm8, 0
vpsrlvq zmm6, zmm5, zmm15
vpclmulqdq zmm7, zmm6, zmm13, 1
Expand Down Expand Up @@ -6533,7 +6533,7 @@ L_AES_XTS_decrypt_avx512_mul16_32:
L_AES_XTS_decrypt_avx512_aes_dec_32_aes_dec_block_last:
vaesdeclast ymm0, ymm0, ymm9
vpxorq ymm0, ymm0, ymm4
vmovdqu64 [rdx], ymm0
vmovdqu64 YMMWORD PTR [rdx], ymm0
vextracti32x4 xmm8, zmm4, 2
add r13d, 32
L_AES_XTS_decrypt_avx512_done_32:
Expand Down Expand Up @@ -6741,9 +6741,9 @@ AES_XTS_decrypt_update_avx512 PROC
vmovdqu OWORD PTR [rsp+192], xmm14
vmovdqu OWORD PTR [rsp+208], xmm15
vmovdqu xmm12, OWORD PTR L_avx512_aes_xts_gc_xts
vbroadcasti32x4 zmm13, ptr_L_avx512_aes_xts_poly
vmovdqu64 zmm14, ptr_L_avx512_aes_xts_shl
vmovdqu64 zmm15, ptr_L_avx512_aes_xts_shr
vbroadcasti32x4 zmm13, OWORD PTR L_avx512_aes_xts_poly
vmovdqu64 zmm14, ZMMWORD PTR L_avx512_aes_xts_shl
vmovdqu64 zmm15, ZMMWORD PTR L_avx512_aes_xts_shr
vmovdqu xmm8, OWORD PTR [r8]
xor r12d, r12d
mov r11d, eax
Expand Down Expand Up @@ -7071,7 +7071,7 @@ L_AES_XTS_decrypt_update_avx512_mul16_32:
; aes_dec_32
lea rcx, QWORD PTR [rdi+r12]
lea rdx, QWORD PTR [rsi+r12]
vmovdqu64 ymm0, [rcx]
vmovdqu64 ymm0, YMMWORD PTR [rcx]
vshufi64x2 zmm5, zmm8, zmm8, 0
vpsrlvq zmm6, zmm5, zmm15
vpclmulqdq zmm7, zmm6, zmm13, 1
Expand Down Expand Up @@ -7103,7 +7103,7 @@ L_AES_XTS_decrypt_update_avx512_mul16_32:
L_AES_XTS_decrypt_update_avx512_aes_dec_32_aes_dec_block_last:
vaesdeclast ymm0, ymm0, ymm9
vpxorq ymm0, ymm0, ymm4
vmovdqu64 [rdx], ymm0
vmovdqu64 YMMWORD PTR [rdx], ymm0
vextracti32x4 xmm8, zmm4, 2
add r12d, 32
L_AES_XTS_decrypt_update_avx512_done_32:
Expand Down
9 changes: 4 additions & 5 deletions wolfcrypt/src/cpuid.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,12 +54,8 @@
new_cpuid_flags |= CPUID_MOVBE;
new_cpuid_flags |= CPUID_BMI1;
#ifdef WOLFSSL_SGX_CPUID_AVX512_VAES
/* CPUID is unavailable inside an SGX enclave, so the integrator
* defines this macro to assert the target CPU implements VAES and
* AVX512. This enables ALL VAES/AVX512-gated code (AES-GCM and the
* rest), not just AES-GCM-SIV - only set it when that is true. */
new_cpuid_flags |= CPUID_VAES;
new_cpuid_flags |= CPUID_AVX512;
new_cpuid_flags |= CPUID_AVX512 | CPUID_AVX512_BW;
#endif

(void)wolfSSL_Atomic_Uint_CompareExchange
Expand Down Expand Up @@ -155,6 +151,9 @@
if (cpuid_flag(7, 0, ECX, 6)) {
new_cpuid_flags |= CPUID_AVX512_VBMI2;
}
if (cpuid_flag(7, 0, EBX, 30)) {
new_cpuid_flags |= CPUID_AVX512_BW;
}
if (cpuid_is_intel()) { new_cpuid_flags |= CPUID_INTEL ; }
if (cpuid_is_amd()) { new_cpuid_flags |= CPUID_AMD ; }
(void)wolfSSL_Atomic_Uint_CompareExchange
Expand Down
8 changes: 1 addition & 7 deletions wolfcrypt/src/sha3.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,13 +136,7 @@
* it is not built - see scripts sha3_avx512.rb.)
* Overrides: WOLFSSL_SHA3_AVX2 forces AVX2 on any vendor with it;
* WOLFSSL_SHA3_NO_AVX2 never uses AVX2. */
#if defined(WOLFSSL_SHA3_NO_AVX2)
#define SHA3_USE_AVX2(f) 0
#elif defined(WOLFSSL_SHA3_AVX2)
#define SHA3_USE_AVX2(f) IS_INTEL_AVX2(f)
#else
#define SHA3_USE_AVX2(f) (IS_INTEL_AVX2(f) && IS_CPU_INTEL(f))
#endif
/* SHA3_USE_AVX2() is defined in sha3.h - shared with ML-DSA. */

/* True when the selected block function uses vector registers and so
* needs the caller to save/restore them. BMI2 and the C block use only
Expand Down
Loading
Loading