Skip to content

Commit 272115d

Browse files
brooniebroonie
committed
2 parents 5e47af3 + 63432fd commit 272115d

8 files changed

Lines changed: 158 additions & 30 deletions

File tree

‎crypto/crc32c.c‎

Lines changed: 1 addition & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
// SPDX-License-Identifier: GPL-2.0-or-later
22
/*
3-
* Cryptographic API.
4-
*
5-
* CRC32C chksum
3+
* crypto_shash support for CRC-32C
64
*
75
*@Article{castagnoli-crc,
86
* author = { Guy Castagnoli and Stefan Braeuer and Martin Herrman},
@@ -15,16 +13,6 @@
1513
* pages = {},
1614
* month = {June},
1715
*}
18-
* Used by the iSCSI driver, possibly others, and derived from
19-
* the iscsi-crc.c module of the linux-iscsi driver at
20-
* http://linux-iscsi.sourceforge.net.
21-
*
22-
* Following the example of lib/crc32, this function is intended to be
23-
* flexible and useful for all users. Modules that currently have their
24-
* own crc32c, but hopefully may be able to use this one are:
25-
* net/sctp (please add all your doco to here if you change to
26-
* use this one!)
27-
* <endoflist>
2816
*
2917
* Copyright (c) 2004 Cisco Systems, Inc.
3018
* Copyright (c) 2008 Herbert Xu <herbert@gondor.apana.org.au>
@@ -49,11 +37,6 @@ struct chksum_desc_ctx {
4937
u32 crc;
5038
};
5139

52-
/*
53-
* Steps through buffer one byte at a time, calculates reflected
54-
* crc using table.
55-
*/
56-
5740
static int chksum_init(struct shash_desc *desc)
5841
{
5942
struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);

‎lib/crc/.kunitconfig‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
CONFIG_KUNIT=y
2+
CONFIG_CRC_ENABLE_ALL_FOR_KUNIT=y
3+
CONFIG_CRC_KUNIT_TEST=y

‎lib/crc/Kconfig‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ config CRC_T10DIF_ARCH
4848
bool
4949
depends on CRC_T10DIF && CRC_OPTIMIZATIONS
5050
default y if ARM && KERNEL_MODE_NEON
51-
default y if ARM64 && KERNEL_MODE_NEON
51+
default y if ARM64
5252
default y if PPC64 && ALTIVEC
5353
default y if RISCV && RISCV_ISA_ZBC
5454
default y if X86
@@ -82,6 +82,7 @@ config CRC64
8282
config CRC64_ARCH
8383
bool
8484
depends on CRC64 && CRC_OPTIMIZATIONS
85+
default y if ARM64
8586
default y if RISCV && RISCV_ISA_ZBC && 64BIT
8687
default y if X86_64
8788

@@ -99,18 +100,27 @@ config CRC_OPTIMIZATIONS
99100

100101
config CRC_KUNIT_TEST
101102
tristate "KUnit tests for CRC functions" if !KUNIT_ALL_TESTS
102-
depends on KUNIT
103+
depends on KUNIT && (CRC7 || CRC16 || CRC_T10DIF || CRC32 || CRC64)
103104
default KUNIT_ALL_TESTS
105+
help
106+
Unit tests for the CRC library functions.
107+
108+
This is intended to help people writing architecture-specific
109+
optimized versions. If unsure, say N.
110+
111+
config CRC_ENABLE_ALL_FOR_KUNIT
112+
tristate "Enable all CRC functions for KUnit test"
113+
depends on KUNIT
104114
select CRC7
105115
select CRC16
106116
select CRC_T10DIF
107117
select CRC32
108118
select CRC64
109119
help
110-
Unit tests for the CRC library functions.
120+
Enable all CRC functions that have test code in CRC_KUNIT_TEST.
111121

112-
This is intended to help people writing architecture-specific
113-
optimized versions. If unsure, say N.
122+
Enable this only if you'd like the CRC KUnit test suite to test all
123+
the CRC variants, even ones that wouldn't otherwise need to be built.
114124

115125
config CRC_BENCHMARK
116126
bool "Benchmark for the CRC functions"

‎lib/crc/Makefile‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,15 @@ obj-$(CONFIG_CRC64) += crc64.o
3838
crc64-y := crc64-main.o
3939
ifeq ($(CONFIG_CRC64_ARCH),y)
4040
CFLAGS_crc64-main.o += -I$(src)/$(SRCARCH)
41+
42+
CFLAGS_REMOVE_arm64/crc64-neon-inner.o += -mgeneral-regs-only
43+
CFLAGS_arm64/crc64-neon-inner.o += -ffreestanding -march=armv8-a+crypto
44+
CFLAGS_arm64/crc64-neon-inner.o += -isystem $(shell $(CC) -print-file-name=include)
45+
crc64-$(CONFIG_ARM64) += arm64/crc64-neon-inner.o
46+
4147
crc64-$(CONFIG_RISCV) += riscv/crc64_lsb.o riscv/crc64_msb.o
4248
crc64-$(CONFIG_X86) += x86/crc64-pclmul.o
43-
endif
49+
endif # CONFIG_CRC64_ARCH
4450

4551
obj-y += tests/
4652

‎lib/crc/arm64/crc64-neon-inner.c‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
// SPDX-License-Identifier: GPL-2.0-only
2+
/*
3+
* Accelerated CRC64 (NVMe) using ARM NEON C intrinsics
4+
*/
5+
6+
#include <linux/types.h>
7+
#include <asm/neon-intrinsics.h>
8+
9+
u64 crc64_nvme_arm64_c(u64 crc, const u8 *p, size_t len);
10+
11+
#define GET_P64_0(v) ((poly64_t)vgetq_lane_u64(vreinterpretq_u64_p64(v), 0))
12+
#define GET_P64_1(v) ((poly64_t)vgetq_lane_u64(vreinterpretq_u64_p64(v), 1))
13+
14+
/* x^191 mod G, x^127 mod G */
15+
static const u64 fold_consts_val[2] = { 0xeadc41fd2ba3d420ULL,
16+
0x21e9761e252621acULL };
17+
/* floor(x^127 / G), (G - x^64) / x */
18+
static const u64 bconsts_val[2] = { 0x27ecfa329aef9f77ULL,
19+
0x34d926535897936aULL };
20+
21+
u64 crc64_nvme_arm64_c(u64 crc, const u8 *p, size_t len)
22+
{
23+
uint64x2_t v0_u64 = { crc, 0 };
24+
poly64x2_t v0 = vreinterpretq_p64_u64(v0_u64);
25+
poly64x2_t fold_consts =
26+
vreinterpretq_p64_u64(vld1q_u64(fold_consts_val));
27+
poly64x2_t v1 = vreinterpretq_p64_u8(vld1q_u8(p));
28+
29+
v0 = vreinterpretq_p64_u8(veorq_u8(vreinterpretq_u8_p64(v0),
30+
vreinterpretq_u8_p64(v1)));
31+
p += 16;
32+
len -= 16;
33+
34+
do {
35+
v1 = vreinterpretq_p64_u8(vld1q_u8(p));
36+
37+
poly128_t v2 = vmull_high_p64(fold_consts, v0);
38+
poly128_t v0_128 =
39+
vmull_p64(GET_P64_0(fold_consts), GET_P64_0(v0));
40+
41+
uint8x16_t x0 = veorq_u8(vreinterpretq_u8_p128(v0_128),
42+
vreinterpretq_u8_p128(v2));
43+
44+
x0 = veorq_u8(x0, vreinterpretq_u8_p64(v1));
45+
v0 = vreinterpretq_p64_u8(x0);
46+
47+
p += 16;
48+
len -= 16;
49+
} while (len >= 16);
50+
51+
/* Multiply the 128-bit value by x^64 and reduce it back to 128 bits. */
52+
poly64x2_t v7 = vreinterpretq_p64_u64((uint64x2_t){ 0, 0 });
53+
poly128_t v1_128 = vmull_p64(GET_P64_1(fold_consts), GET_P64_0(v0));
54+
55+
uint8x16_t ext_v0 =
56+
vextq_u8(vreinterpretq_u8_p64(v0), vreinterpretq_u8_p64(v7), 8);
57+
uint8x16_t x0 = veorq_u8(ext_v0, vreinterpretq_u8_p128(v1_128));
58+
59+
v0 = vreinterpretq_p64_u8(x0);
60+
61+
/* Final Barrett reduction */
62+
poly64x2_t bconsts = vreinterpretq_p64_u64(vld1q_u64(bconsts_val));
63+
64+
v1_128 = vmull_p64(GET_P64_0(bconsts), GET_P64_0(v0));
65+
66+
poly64x2_t v1_64 = vreinterpretq_p64_u8(vreinterpretq_u8_p128(v1_128));
67+
poly128_t v3_128 = vmull_p64(GET_P64_1(bconsts), GET_P64_0(v1_64));
68+
69+
x0 = veorq_u8(vreinterpretq_u8_p64(v0), vreinterpretq_u8_p128(v3_128));
70+
71+
uint8x16_t ext_v2 = vextq_u8(vreinterpretq_u8_p64(v7),
72+
vreinterpretq_u8_p128(v1_128), 8);
73+
74+
x0 = veorq_u8(x0, ext_v2);
75+
76+
v0 = vreinterpretq_p64_u8(x0);
77+
return vgetq_lane_u64(vreinterpretq_u64_p64(v0), 1);
78+
}

‎lib/crc/arm64/crc64.h‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
/* SPDX-License-Identifier: GPL-2.0-only */
2+
/*
3+
* CRC64 using ARM64 PMULL instructions
4+
*/
5+
6+
#include <linux/cpufeature.h>
7+
#include <asm/simd.h>
8+
#include <linux/minmax.h>
9+
#include <linux/sizes.h>
10+
11+
u64 crc64_nvme_arm64_c(u64 crc, const u8 *p, size_t len);
12+
13+
#define crc64_be_arch crc64_be_generic
14+
15+
static inline u64 crc64_nvme_arch(u64 crc, const u8 *p, size_t len)
16+
{
17+
if (len >= 128 && cpu_have_named_feature(PMULL) &&
18+
likely(may_use_simd())) {
19+
do {
20+
size_t chunk = min_t(size_t, len & ~15, SZ_4K);
21+
22+
scoped_ksimd()
23+
crc = crc64_nvme_arm64_c(crc, p, chunk);
24+
25+
p += chunk;
26+
len -= chunk;
27+
} while (len >= 128);
28+
}
29+
return crc64_nvme_generic(crc, p, len);
30+
}

‎lib/crc/tests/crc_kunit.c‎

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -268,8 +268,7 @@ crc_benchmark(struct kunit *test,
268268
}
269269
}
270270

271-
/* crc7_be */
272-
271+
#if IS_REACHABLE(CONFIG_CRC7)
273272
static u64 crc7_be_wrapper(u64 crc, const u8 *p, size_t len)
274273
{
275274
/*
@@ -294,9 +293,9 @@ static void crc7_be_benchmark(struct kunit *test)
294293
{
295294
crc_benchmark(test, crc7_be_wrapper);
296295
}
296+
#endif /* CONFIG_CRC7 */
297297

298-
/* crc16 */
299-
298+
#if IS_REACHABLE(CONFIG_CRC16)
300299
static u64 crc16_wrapper(u64 crc, const u8 *p, size_t len)
301300
{
302301
return crc16(crc, p, len);
@@ -318,9 +317,9 @@ static void crc16_benchmark(struct kunit *test)
318317
{
319318
crc_benchmark(test, crc16_wrapper);
320319
}
320+
#endif /* CONFIG_CRC16 */
321321

322-
/* crc_t10dif */
323-
322+
#if IS_REACHABLE(CONFIG_CRC_T10DIF)
324323
static u64 crc_t10dif_wrapper(u64 crc, const u8 *p, size_t len)
325324
{
326325
return crc_t10dif_update(crc, p, len);
@@ -342,6 +341,9 @@ static void crc_t10dif_benchmark(struct kunit *test)
342341
{
343342
crc_benchmark(test, crc_t10dif_wrapper);
344343
}
344+
#endif /* CONFIG_CRC_T10DIF */
345+
346+
#if IS_REACHABLE(CONFIG_CRC32)
345347

346348
/* crc32_le */
347349

@@ -414,6 +416,9 @@ static void crc32c_benchmark(struct kunit *test)
414416
{
415417
crc_benchmark(test, crc32c_wrapper);
416418
}
419+
#endif /* CONFIG_CRC32 */
420+
421+
#if IS_REACHABLE(CONFIG_CRC64)
417422

418423
/* crc64_be */
419424

@@ -463,24 +468,35 @@ static void crc64_nvme_benchmark(struct kunit *test)
463468
{
464469
crc_benchmark(test, crc64_nvme_wrapper);
465470
}
471+
#endif /* CONFIG_CRC64 */
466472

467473
static struct kunit_case crc_test_cases[] = {
474+
#if IS_REACHABLE(CONFIG_CRC7)
468475
KUNIT_CASE(crc7_be_test),
469476
KUNIT_CASE(crc7_be_benchmark),
477+
#endif
478+
#if IS_REACHABLE(CONFIG_CRC16)
470479
KUNIT_CASE(crc16_test),
471480
KUNIT_CASE(crc16_benchmark),
481+
#endif
482+
#if IS_REACHABLE(CONFIG_CRC_T10DIF)
472483
KUNIT_CASE(crc_t10dif_test),
473484
KUNIT_CASE(crc_t10dif_benchmark),
485+
#endif
486+
#if IS_REACHABLE(CONFIG_CRC32)
474487
KUNIT_CASE(crc32_le_test),
475488
KUNIT_CASE(crc32_le_benchmark),
476489
KUNIT_CASE(crc32_be_test),
477490
KUNIT_CASE(crc32_be_benchmark),
478491
KUNIT_CASE(crc32c_test),
479492
KUNIT_CASE(crc32c_benchmark),
493+
#endif
494+
#if IS_REACHABLE(CONFIG_CRC64)
480495
KUNIT_CASE(crc64_be_test),
481496
KUNIT_CASE(crc64_be_benchmark),
482497
KUNIT_CASE(crc64_nvme_test),
483498
KUNIT_CASE(crc64_nvme_benchmark),
499+
#endif
484500
{},
485501
};
486502

‎tools/testing/kunit/configs/all_tests.config‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,8 @@ CONFIG_CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT=y
4848

4949
CONFIG_PRIME_NUMBERS=y
5050

51+
CONFIG_CRC_ENABLE_ALL_FOR_KUNIT=y
52+
5153
CONFIG_SECURITY=y
5254
CONFIG_SECURITY_APPARMOR=y
5355
CONFIG_SECURITY_LANDLOCK=y

0 commit comments

Comments
 (0)