* [PATCH v2 01/11] lib/crc64-rocksoft: stop wrapping the crypto API
2025-01-30 3:51 [PATCH v2 00/11] CRC64 library rework and x86 CRC optimization Eric Biggers
@ 2025-01-30 3:51 ` Eric Biggers
2025-02-02 14:26 ` Ard Biesheuvel
2025-01-30 3:51 ` [PATCH v2 02/11] crypto: crc64-rocksoft - remove from " Eric Biggers
` (12 subsequent siblings)
13 siblings, 1 reply; 21+ messages in thread
From: Eric Biggers @ 2025-01-30 3:51 UTC (permalink / raw)
To: linux-kernel
Cc: linux-crypto, x86, linux-block, Ard Biesheuvel, Keith Busch,
Kent Overstreet, Martin K . Petersen
From: Eric Biggers <ebiggers@google.com>
Following what was done for the CRC32 and CRC-T10DIF library functions,
get rid of the pointless use of the crypto API and make
crc64_rocksoft_update() call into the library directly. This is faster
and simpler.
Remove crc64_rocksoft() (the version of the function that did not take a
'crc' argument) since it is unused.
Signed-off-by: Eric Biggers <ebiggers@google.com>
---
block/Kconfig | 2 +-
include/linux/crc64.h | 13 ++++-
lib/Kconfig | 9 ---
lib/Makefile | 1 -
lib/crc64-rocksoft.c | 126 ------------------------------------------
lib/crc64.c | 7 ---
6 files changed, 12 insertions(+), 146 deletions(-)
delete mode 100644 lib/crc64-rocksoft.c
diff --git a/block/Kconfig b/block/Kconfig
index 5b623b876d3b..df8973bc0539 100644
--- a/block/Kconfig
+++ b/block/Kconfig
@@ -61,11 +61,11 @@ config BLK_DEV_BSGLIB
If unsure, say N.
config BLK_DEV_INTEGRITY
bool "Block layer data integrity support"
select CRC_T10DIF
- select CRC64_ROCKSOFT
+ select CRC64
help
Some storage devices allow extra information to be
stored/retrieved to help protect the data. The block layer
data integrity option provides hooks which can be used by
filesystems to ensure better data integrity.
diff --git a/include/linux/crc64.h b/include/linux/crc64.h
index e044c60d1e61..0a595b272166 100644
--- a/include/linux/crc64.h
+++ b/include/linux/crc64.h
@@ -10,9 +10,18 @@
#define CRC64_ROCKSOFT_STRING "crc64-rocksoft"
u64 __pure crc64_be(u64 crc, const void *p, size_t len);
u64 __pure crc64_rocksoft_generic(u64 crc, const void *p, size_t len);
-u64 crc64_rocksoft(const unsigned char *buffer, size_t len);
-u64 crc64_rocksoft_update(u64 crc, const unsigned char *buffer, size_t len);
+/**
+ * crc64_rocksoft_update - Calculate bitwise Rocksoft CRC64
+ * @crc: seed value for computation. 0 for a new CRC calculation, or the
+ * previous crc64 value if computing incrementally.
+ * @p: pointer to buffer over which CRC64 is run
+ * @len: length of buffer @p
+ */
+static inline u64 crc64_rocksoft_update(u64 crc, const u8 *p, size_t len)
+{
+ return crc64_rocksoft_generic(crc, p, len);
+}
#endif /* _LINUX_CRC64_H */
diff --git a/lib/Kconfig b/lib/Kconfig
index dccb61b7d698..da07fd39cf97 100644
--- a/lib/Kconfig
+++ b/lib/Kconfig
@@ -166,19 +166,10 @@ config ARCH_HAS_CRC_T10DIF
config CRC_T10DIF_ARCH
tristate
default CRC_T10DIF if ARCH_HAS_CRC_T10DIF && CRC_OPTIMIZATIONS
-config CRC64_ROCKSOFT
- tristate "CRC calculation for the Rocksoft model CRC64"
- select CRC64
- select CRYPTO
- select CRYPTO_CRC64_ROCKSOFT
- help
- This option provides a CRC64 API to a registered crypto driver.
- This is used with the block layer's data integrity subsystem.
-
config CRC_ITU_T
tristate "CRC ITU-T V.41 functions"
help
This option is provided for the case where no in-kernel-tree
modules require CRC ITU-T V.41 functions, but a module built outside
diff --git a/lib/Makefile b/lib/Makefile
index f1c6e9d76a7c..518018b2a5d4 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -164,11 +164,10 @@ obj-$(CONFIG_CRC_ITU_T) += crc-itu-t.o
obj-$(CONFIG_CRC32) += crc32.o
obj-$(CONFIG_CRC64) += crc64.o
obj-$(CONFIG_CRC4) += crc4.o
obj-$(CONFIG_CRC7) += crc7.o
obj-$(CONFIG_CRC8) += crc8.o
-obj-$(CONFIG_CRC64_ROCKSOFT) += crc64-rocksoft.o
obj-$(CONFIG_XXHASH) += xxhash.o
obj-$(CONFIG_GENERIC_ALLOCATOR) += genalloc.o
obj-$(CONFIG_842_COMPRESS) += 842/
obj-$(CONFIG_842_DECOMPRESS) += 842/
diff --git a/lib/crc64-rocksoft.c b/lib/crc64-rocksoft.c
deleted file mode 100644
index fc9ae0da5df7..000000000000
--- a/lib/crc64-rocksoft.c
+++ /dev/null
@@ -1,126 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-
-#include <linux/types.h>
-#include <linux/module.h>
-#include <linux/crc64.h>
-#include <linux/err.h>
-#include <linux/init.h>
-#include <crypto/hash.h>
-#include <crypto/algapi.h>
-#include <linux/static_key.h>
-#include <linux/notifier.h>
-
-static struct crypto_shash __rcu *crc64_rocksoft_tfm;
-static DEFINE_STATIC_KEY_TRUE(crc64_rocksoft_fallback);
-static DEFINE_MUTEX(crc64_rocksoft_mutex);
-static struct work_struct crc64_rocksoft_rehash_work;
-
-static int crc64_rocksoft_notify(struct notifier_block *self, unsigned long val, void *data)
-{
- struct crypto_alg *alg = data;
-
- if (val != CRYPTO_MSG_ALG_LOADED ||
- strcmp(alg->cra_name, CRC64_ROCKSOFT_STRING))
- return NOTIFY_DONE;
-
- schedule_work(&crc64_rocksoft_rehash_work);
- return NOTIFY_OK;
-}
-
-static void crc64_rocksoft_rehash(struct work_struct *work)
-{
- struct crypto_shash *new, *old;
-
- mutex_lock(&crc64_rocksoft_mutex);
- old = rcu_dereference_protected(crc64_rocksoft_tfm,
- lockdep_is_held(&crc64_rocksoft_mutex));
- new = crypto_alloc_shash(CRC64_ROCKSOFT_STRING, 0, 0);
- if (IS_ERR(new)) {
- mutex_unlock(&crc64_rocksoft_mutex);
- return;
- }
- rcu_assign_pointer(crc64_rocksoft_tfm, new);
- mutex_unlock(&crc64_rocksoft_mutex);
-
- if (old) {
- synchronize_rcu();
- crypto_free_shash(old);
- } else {
- static_branch_disable(&crc64_rocksoft_fallback);
- }
-}
-
-static struct notifier_block crc64_rocksoft_nb = {
- .notifier_call = crc64_rocksoft_notify,
-};
-
-u64 crc64_rocksoft_update(u64 crc, const unsigned char *buffer, size_t len)
-{
- struct {
- struct shash_desc shash;
- u64 crc;
- } desc;
- int err;
-
- if (static_branch_unlikely(&crc64_rocksoft_fallback))
- return crc64_rocksoft_generic(crc, buffer, len);
-
- rcu_read_lock();
- desc.shash.tfm = rcu_dereference(crc64_rocksoft_tfm);
- desc.crc = crc;
- err = crypto_shash_update(&desc.shash, buffer, len);
- rcu_read_unlock();
-
- BUG_ON(err);
-
- return desc.crc;
-}
-EXPORT_SYMBOL_GPL(crc64_rocksoft_update);
-
-u64 crc64_rocksoft(const unsigned char *buffer, size_t len)
-{
- return crc64_rocksoft_update(0, buffer, len);
-}
-EXPORT_SYMBOL_GPL(crc64_rocksoft);
-
-static int __init crc64_rocksoft_mod_init(void)
-{
- INIT_WORK(&crc64_rocksoft_rehash_work, crc64_rocksoft_rehash);
- crypto_register_notifier(&crc64_rocksoft_nb);
- crc64_rocksoft_rehash(&crc64_rocksoft_rehash_work);
- return 0;
-}
-
-static void __exit crc64_rocksoft_mod_fini(void)
-{
- crypto_unregister_notifier(&crc64_rocksoft_nb);
- cancel_work_sync(&crc64_rocksoft_rehash_work);
- crypto_free_shash(rcu_dereference_protected(crc64_rocksoft_tfm, 1));
-}
-
-module_init(crc64_rocksoft_mod_init);
-module_exit(crc64_rocksoft_mod_fini);
-
-static int crc64_rocksoft_transform_show(char *buffer, const struct kernel_param *kp)
-{
- struct crypto_shash *tfm;
- int len;
-
- if (static_branch_unlikely(&crc64_rocksoft_fallback))
- return sprintf(buffer, "fallback\n");
-
- rcu_read_lock();
- tfm = rcu_dereference(crc64_rocksoft_tfm);
- len = snprintf(buffer, PAGE_SIZE, "%s\n",
- crypto_shash_driver_name(tfm));
- rcu_read_unlock();
-
- return len;
-}
-
-module_param_call(transform, NULL, crc64_rocksoft_transform_show, NULL, 0444);
-
-MODULE_AUTHOR("Keith Busch <kbusch@kernel.org>");
-MODULE_DESCRIPTION("Rocksoft model CRC64 calculation (library API)");
-MODULE_LICENSE("GPL");
-MODULE_SOFTDEP("pre: crc64");
diff --git a/lib/crc64.c b/lib/crc64.c
index 61ae8dfb6a1c..b5136fb4c199 100644
--- a/lib/crc64.c
+++ b/lib/crc64.c
@@ -61,17 +61,10 @@ u64 __pure crc64_be(u64 crc, const void *p, size_t len)
return crc;
}
EXPORT_SYMBOL_GPL(crc64_be);
-/**
- * crc64_rocksoft_generic - Calculate bitwise Rocksoft CRC64
- * @crc: seed value for computation. 0 for a new CRC calculation, or the
- * previous crc64 value if computing incrementally.
- * @p: pointer to buffer over which CRC64 is run
- * @len: length of buffer @p
- */
u64 __pure crc64_rocksoft_generic(u64 crc, const void *p, size_t len)
{
const unsigned char *_p = p;
size_t i;
--
2.48.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* Re: [PATCH v2 01/11] lib/crc64-rocksoft: stop wrapping the crypto API
2025-01-30 3:51 ` [PATCH v2 01/11] lib/crc64-rocksoft: stop wrapping the crypto API Eric Biggers
@ 2025-02-02 14:26 ` Ard Biesheuvel
0 siblings, 0 replies; 21+ messages in thread
From: Ard Biesheuvel @ 2025-02-02 14:26 UTC (permalink / raw)
To: Eric Biggers
Cc: linux-kernel, linux-crypto, x86, linux-block, Keith Busch,
Kent Overstreet, Martin K . Petersen
On Thu, 30 Jan 2025 at 04:54, Eric Biggers <ebiggers@kernel.org> wrote:
>
> From: Eric Biggers <ebiggers@google.com>
>
> Following what was done for the CRC32 and CRC-T10DIF library functions,
> get rid of the pointless use of the crypto API and make
> crc64_rocksoft_update() call into the library directly. This is faster
> and simpler.
>
> Remove crc64_rocksoft() (the version of the function that did not take a
> 'crc' argument) since it is unused.
>
> Signed-off-by: Eric Biggers <ebiggers@google.com>
> ---
> block/Kconfig | 2 +-
> include/linux/crc64.h | 13 ++++-
> lib/Kconfig | 9 ---
> lib/Makefile | 1 -
> lib/crc64-rocksoft.c | 126 ------------------------------------------
> lib/crc64.c | 7 ---
> 6 files changed, 12 insertions(+), 146 deletions(-)
> delete mode 100644 lib/crc64-rocksoft.c
>
Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
> diff --git a/block/Kconfig b/block/Kconfig
> index 5b623b876d3b..df8973bc0539 100644
> --- a/block/Kconfig
> +++ b/block/Kconfig
> @@ -61,11 +61,11 @@ config BLK_DEV_BSGLIB
> If unsure, say N.
>
> config BLK_DEV_INTEGRITY
> bool "Block layer data integrity support"
> select CRC_T10DIF
> - select CRC64_ROCKSOFT
> + select CRC64
> help
> Some storage devices allow extra information to be
> stored/retrieved to help protect the data. The block layer
> data integrity option provides hooks which can be used by
> filesystems to ensure better data integrity.
> diff --git a/include/linux/crc64.h b/include/linux/crc64.h
> index e044c60d1e61..0a595b272166 100644
> --- a/include/linux/crc64.h
> +++ b/include/linux/crc64.h
> @@ -10,9 +10,18 @@
> #define CRC64_ROCKSOFT_STRING "crc64-rocksoft"
>
> u64 __pure crc64_be(u64 crc, const void *p, size_t len);
> u64 __pure crc64_rocksoft_generic(u64 crc, const void *p, size_t len);
>
> -u64 crc64_rocksoft(const unsigned char *buffer, size_t len);
> -u64 crc64_rocksoft_update(u64 crc, const unsigned char *buffer, size_t len);
> +/**
> + * crc64_rocksoft_update - Calculate bitwise Rocksoft CRC64
> + * @crc: seed value for computation. 0 for a new CRC calculation, or the
> + * previous crc64 value if computing incrementally.
> + * @p: pointer to buffer over which CRC64 is run
> + * @len: length of buffer @p
> + */
> +static inline u64 crc64_rocksoft_update(u64 crc, const u8 *p, size_t len)
> +{
> + return crc64_rocksoft_generic(crc, p, len);
> +}
>
> #endif /* _LINUX_CRC64_H */
> diff --git a/lib/Kconfig b/lib/Kconfig
> index dccb61b7d698..da07fd39cf97 100644
> --- a/lib/Kconfig
> +++ b/lib/Kconfig
> @@ -166,19 +166,10 @@ config ARCH_HAS_CRC_T10DIF
>
> config CRC_T10DIF_ARCH
> tristate
> default CRC_T10DIF if ARCH_HAS_CRC_T10DIF && CRC_OPTIMIZATIONS
>
> -config CRC64_ROCKSOFT
> - tristate "CRC calculation for the Rocksoft model CRC64"
> - select CRC64
> - select CRYPTO
> - select CRYPTO_CRC64_ROCKSOFT
> - help
> - This option provides a CRC64 API to a registered crypto driver.
> - This is used with the block layer's data integrity subsystem.
> -
> config CRC_ITU_T
> tristate "CRC ITU-T V.41 functions"
> help
> This option is provided for the case where no in-kernel-tree
> modules require CRC ITU-T V.41 functions, but a module built outside
> diff --git a/lib/Makefile b/lib/Makefile
> index f1c6e9d76a7c..518018b2a5d4 100644
> --- a/lib/Makefile
> +++ b/lib/Makefile
> @@ -164,11 +164,10 @@ obj-$(CONFIG_CRC_ITU_T) += crc-itu-t.o
> obj-$(CONFIG_CRC32) += crc32.o
> obj-$(CONFIG_CRC64) += crc64.o
> obj-$(CONFIG_CRC4) += crc4.o
> obj-$(CONFIG_CRC7) += crc7.o
> obj-$(CONFIG_CRC8) += crc8.o
> -obj-$(CONFIG_CRC64_ROCKSOFT) += crc64-rocksoft.o
> obj-$(CONFIG_XXHASH) += xxhash.o
> obj-$(CONFIG_GENERIC_ALLOCATOR) += genalloc.o
>
> obj-$(CONFIG_842_COMPRESS) += 842/
> obj-$(CONFIG_842_DECOMPRESS) += 842/
> diff --git a/lib/crc64-rocksoft.c b/lib/crc64-rocksoft.c
> deleted file mode 100644
> index fc9ae0da5df7..000000000000
> --- a/lib/crc64-rocksoft.c
> +++ /dev/null
> @@ -1,126 +0,0 @@
> -// SPDX-License-Identifier: GPL-2.0-only
> -
> -#include <linux/types.h>
> -#include <linux/module.h>
> -#include <linux/crc64.h>
> -#include <linux/err.h>
> -#include <linux/init.h>
> -#include <crypto/hash.h>
> -#include <crypto/algapi.h>
> -#include <linux/static_key.h>
> -#include <linux/notifier.h>
> -
> -static struct crypto_shash __rcu *crc64_rocksoft_tfm;
> -static DEFINE_STATIC_KEY_TRUE(crc64_rocksoft_fallback);
> -static DEFINE_MUTEX(crc64_rocksoft_mutex);
> -static struct work_struct crc64_rocksoft_rehash_work;
> -
> -static int crc64_rocksoft_notify(struct notifier_block *self, unsigned long val, void *data)
> -{
> - struct crypto_alg *alg = data;
> -
> - if (val != CRYPTO_MSG_ALG_LOADED ||
> - strcmp(alg->cra_name, CRC64_ROCKSOFT_STRING))
> - return NOTIFY_DONE;
> -
> - schedule_work(&crc64_rocksoft_rehash_work);
> - return NOTIFY_OK;
> -}
> -
> -static void crc64_rocksoft_rehash(struct work_struct *work)
> -{
> - struct crypto_shash *new, *old;
> -
> - mutex_lock(&crc64_rocksoft_mutex);
> - old = rcu_dereference_protected(crc64_rocksoft_tfm,
> - lockdep_is_held(&crc64_rocksoft_mutex));
> - new = crypto_alloc_shash(CRC64_ROCKSOFT_STRING, 0, 0);
> - if (IS_ERR(new)) {
> - mutex_unlock(&crc64_rocksoft_mutex);
> - return;
> - }
> - rcu_assign_pointer(crc64_rocksoft_tfm, new);
> - mutex_unlock(&crc64_rocksoft_mutex);
> -
> - if (old) {
> - synchronize_rcu();
> - crypto_free_shash(old);
> - } else {
> - static_branch_disable(&crc64_rocksoft_fallback);
> - }
> -}
> -
> -static struct notifier_block crc64_rocksoft_nb = {
> - .notifier_call = crc64_rocksoft_notify,
> -};
> -
> -u64 crc64_rocksoft_update(u64 crc, const unsigned char *buffer, size_t len)
> -{
> - struct {
> - struct shash_desc shash;
> - u64 crc;
> - } desc;
> - int err;
> -
> - if (static_branch_unlikely(&crc64_rocksoft_fallback))
> - return crc64_rocksoft_generic(crc, buffer, len);
> -
> - rcu_read_lock();
> - desc.shash.tfm = rcu_dereference(crc64_rocksoft_tfm);
> - desc.crc = crc;
> - err = crypto_shash_update(&desc.shash, buffer, len);
> - rcu_read_unlock();
> -
> - BUG_ON(err);
> -
> - return desc.crc;
> -}
> -EXPORT_SYMBOL_GPL(crc64_rocksoft_update);
> -
> -u64 crc64_rocksoft(const unsigned char *buffer, size_t len)
> -{
> - return crc64_rocksoft_update(0, buffer, len);
> -}
> -EXPORT_SYMBOL_GPL(crc64_rocksoft);
> -
> -static int __init crc64_rocksoft_mod_init(void)
> -{
> - INIT_WORK(&crc64_rocksoft_rehash_work, crc64_rocksoft_rehash);
> - crypto_register_notifier(&crc64_rocksoft_nb);
> - crc64_rocksoft_rehash(&crc64_rocksoft_rehash_work);
> - return 0;
> -}
> -
> -static void __exit crc64_rocksoft_mod_fini(void)
> -{
> - crypto_unregister_notifier(&crc64_rocksoft_nb);
> - cancel_work_sync(&crc64_rocksoft_rehash_work);
> - crypto_free_shash(rcu_dereference_protected(crc64_rocksoft_tfm, 1));
> -}
> -
> -module_init(crc64_rocksoft_mod_init);
> -module_exit(crc64_rocksoft_mod_fini);
> -
> -static int crc64_rocksoft_transform_show(char *buffer, const struct kernel_param *kp)
> -{
> - struct crypto_shash *tfm;
> - int len;
> -
> - if (static_branch_unlikely(&crc64_rocksoft_fallback))
> - return sprintf(buffer, "fallback\n");
> -
> - rcu_read_lock();
> - tfm = rcu_dereference(crc64_rocksoft_tfm);
> - len = snprintf(buffer, PAGE_SIZE, "%s\n",
> - crypto_shash_driver_name(tfm));
> - rcu_read_unlock();
> -
> - return len;
> -}
> -
> -module_param_call(transform, NULL, crc64_rocksoft_transform_show, NULL, 0444);
> -
> -MODULE_AUTHOR("Keith Busch <kbusch@kernel.org>");
> -MODULE_DESCRIPTION("Rocksoft model CRC64 calculation (library API)");
> -MODULE_LICENSE("GPL");
> -MODULE_SOFTDEP("pre: crc64");
> diff --git a/lib/crc64.c b/lib/crc64.c
> index 61ae8dfb6a1c..b5136fb4c199 100644
> --- a/lib/crc64.c
> +++ b/lib/crc64.c
> @@ -61,17 +61,10 @@ u64 __pure crc64_be(u64 crc, const void *p, size_t len)
>
> return crc;
> }
> EXPORT_SYMBOL_GPL(crc64_be);
>
> -/**
> - * crc64_rocksoft_generic - Calculate bitwise Rocksoft CRC64
> - * @crc: seed value for computation. 0 for a new CRC calculation, or the
> - * previous crc64 value if computing incrementally.
> - * @p: pointer to buffer over which CRC64 is run
> - * @len: length of buffer @p
> - */
> u64 __pure crc64_rocksoft_generic(u64 crc, const void *p, size_t len)
> {
> const unsigned char *_p = p;
> size_t i;
>
> --
> 2.48.1
>
^ permalink raw reply [flat|nested] 21+ messages in thread
* [PATCH v2 02/11] crypto: crc64-rocksoft - remove from crypto API
2025-01-30 3:51 [PATCH v2 00/11] CRC64 library rework and x86 CRC optimization Eric Biggers
2025-01-30 3:51 ` [PATCH v2 01/11] lib/crc64-rocksoft: stop wrapping the crypto API Eric Biggers
@ 2025-01-30 3:51 ` Eric Biggers
2025-02-02 14:27 ` Ard Biesheuvel
2025-01-30 3:51 ` [PATCH v2 03/11] lib/crc64: rename CRC64-Rocksoft to CRC64-NVME Eric Biggers
` (11 subsequent siblings)
13 siblings, 1 reply; 21+ messages in thread
From: Eric Biggers @ 2025-01-30 3:51 UTC (permalink / raw)
To: linux-kernel
Cc: linux-crypto, x86, linux-block, Ard Biesheuvel, Keith Busch,
Kent Overstreet, Martin K . Petersen
From: Eric Biggers <ebiggers@google.com>
Remove crc64-rocksoft from the crypto API. It has no known user now
that the lib is no longer built on top of it. It was also added much
more recently than the longstanding crc32 and crc32c. Unlike crc32 and
crc32c, crc64-rocksoft is also not mentioned in the dm-integrity
documentation and there are no references to it in anywhere in the
cryptsetup git repo, so it is unlikely to have any user there either.
Also, this CRC variant is named incorrectly; it has nothing to do with
Rocksoft and should be called crc64-nvme. That is yet another reason to
remove it from the crypto API; we would not want anyone to start
depending on the current incorrect algorithm name of crc64-rocksoft.
Note that this change temporarily makes this CRC variant not be covered
by any tests, as previously it was relying on the crypto self-tests.
This will be fixed by adding this CRC variant to crc_kunit.
Signed-off-by: Eric Biggers <ebiggers@google.com>
---
crypto/Kconfig | 11 ----
crypto/Makefile | 1 -
crypto/crc64_rocksoft_generic.c | 89 ---------------------------------
crypto/testmgr.c | 7 ---
crypto/testmgr.h | 12 -----
include/linux/crc64.h | 2 -
6 files changed, 122 deletions(-)
delete mode 100644 crypto/crc64_rocksoft_generic.c
diff --git a/crypto/Kconfig b/crypto/Kconfig
index 74ae5f52b784..9ffb59b1aac3 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -1088,21 +1088,10 @@ config CRYPTO_CRCT10DIF
help
CRC16 CRC algorithm used for the T10 (SCSI) Data Integrity Field (DIF)
CRC algorithm used by the SCSI Block Commands standard.
-config CRYPTO_CRC64_ROCKSOFT
- tristate "CRC64 based on Rocksoft Model algorithm"
- depends on CRC64
- select CRYPTO_HASH
- help
- CRC64 CRC algorithm based on the Rocksoft Model CRC Algorithm
-
- Used by the NVMe implementation of T10 DIF (BLK_DEV_INTEGRITY)
-
- See https://zlib.net/crc_v3.txt
-
endmenu
menu "Compression"
config CRYPTO_DEFLATE
diff --git a/crypto/Makefile b/crypto/Makefile
index f67e853c4690..d3b79b8c9022 100644
--- a/crypto/Makefile
+++ b/crypto/Makefile
@@ -155,11 +155,10 @@ obj-$(CONFIG_CRYPTO_CRC32C) += crc32c_generic.o
obj-$(CONFIG_CRYPTO_CRC32) += crc32_generic.o
CFLAGS_crc32c_generic.o += -DARCH=$(ARCH)
CFLAGS_crc32_generic.o += -DARCH=$(ARCH)
obj-$(CONFIG_CRYPTO_CRCT10DIF) += crct10dif_generic.o
CFLAGS_crct10dif_generic.o += -DARCH=$(ARCH)
-obj-$(CONFIG_CRYPTO_CRC64_ROCKSOFT) += crc64_rocksoft_generic.o
obj-$(CONFIG_CRYPTO_AUTHENC) += authenc.o authencesn.o
obj-$(CONFIG_CRYPTO_LZO) += lzo.o lzo-rle.o
obj-$(CONFIG_CRYPTO_LZ4) += lz4.o
obj-$(CONFIG_CRYPTO_LZ4HC) += lz4hc.o
obj-$(CONFIG_CRYPTO_XXHASH) += xxhash_generic.o
diff --git a/crypto/crc64_rocksoft_generic.c b/crypto/crc64_rocksoft_generic.c
deleted file mode 100644
index ce0f3059b912..000000000000
--- a/crypto/crc64_rocksoft_generic.c
+++ /dev/null
@@ -1,89 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-
-#include <linux/crc64.h>
-#include <linux/module.h>
-#include <crypto/internal/hash.h>
-#include <linux/unaligned.h>
-
-static int chksum_init(struct shash_desc *desc)
-{
- u64 *crc = shash_desc_ctx(desc);
-
- *crc = 0;
-
- return 0;
-}
-
-static int chksum_update(struct shash_desc *desc, const u8 *data,
- unsigned int length)
-{
- u64 *crc = shash_desc_ctx(desc);
-
- *crc = crc64_rocksoft_generic(*crc, data, length);
-
- return 0;
-}
-
-static int chksum_final(struct shash_desc *desc, u8 *out)
-{
- u64 *crc = shash_desc_ctx(desc);
-
- put_unaligned_le64(*crc, out);
- return 0;
-}
-
-static int __chksum_finup(u64 crc, const u8 *data, unsigned int len, u8 *out)
-{
- crc = crc64_rocksoft_generic(crc, data, len);
- put_unaligned_le64(crc, out);
- return 0;
-}
-
-static int chksum_finup(struct shash_desc *desc, const u8 *data,
- unsigned int len, u8 *out)
-{
- u64 *crc = shash_desc_ctx(desc);
-
- return __chksum_finup(*crc, data, len, out);
-}
-
-static int chksum_digest(struct shash_desc *desc, const u8 *data,
- unsigned int length, u8 *out)
-{
- return __chksum_finup(0, data, length, out);
-}
-
-static struct shash_alg alg = {
- .digestsize = sizeof(u64),
- .init = chksum_init,
- .update = chksum_update,
- .final = chksum_final,
- .finup = chksum_finup,
- .digest = chksum_digest,
- .descsize = sizeof(u64),
- .base = {
- .cra_name = CRC64_ROCKSOFT_STRING,
- .cra_driver_name = "crc64-rocksoft-generic",
- .cra_priority = 200,
- .cra_blocksize = 1,
- .cra_module = THIS_MODULE,
- }
-};
-
-static int __init crc64_rocksoft_init(void)
-{
- return crypto_register_shash(&alg);
-}
-
-static void __exit crc64_rocksoft_exit(void)
-{
- crypto_unregister_shash(&alg);
-}
-
-module_init(crc64_rocksoft_init);
-module_exit(crc64_rocksoft_exit);
-
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("Rocksoft model CRC64 calculation.");
-MODULE_ALIAS_CRYPTO("crc64-rocksoft");
-MODULE_ALIAS_CRYPTO("crc64-rocksoft-generic");
diff --git a/crypto/testmgr.c b/crypto/testmgr.c
index e61490ba4095..0c1c3a6453b6 100644
--- a/crypto/testmgr.c
+++ b/crypto/testmgr.c
@@ -4757,17 +4757,10 @@ static const struct alg_test_desc alg_test_descs[] = {
.test = alg_test_crc32c,
.fips_allowed = 1,
.suite = {
.hash = __VECS(crc32c_tv_template)
}
- }, {
- .alg = "crc64-rocksoft",
- .test = alg_test_hash,
- .fips_allowed = 1,
- .suite = {
- .hash = __VECS(crc64_rocksoft_tv_template)
- }
}, {
.alg = "crct10dif",
.test = alg_test_hash,
.fips_allowed = 1,
.suite = {
diff --git a/crypto/testmgr.h b/crypto/testmgr.h
index d754ab997186..4ab05046b734 100644
--- a/crypto/testmgr.h
+++ b/crypto/testmgr.h
@@ -6018,22 +6018,10 @@ static const struct hash_testvec rmd160_tv_template[] = {
};
static const u8 zeroes[4096] = { [0 ... 4095] = 0 };
static const u8 ones[4096] = { [0 ... 4095] = 0xff };
-static const struct hash_testvec crc64_rocksoft_tv_template[] = {
- {
- .plaintext = zeroes,
- .psize = 4096,
- .digest = "\x4e\xb6\x22\xeb\x67\xd3\x82\x64",
- }, {
- .plaintext = ones,
- .psize = 4096,
- .digest = "\xac\xa3\xec\x02\x73\xba\xdd\xc0",
- }
-};
-
static const struct hash_testvec crct10dif_tv_template[] = {
{
.plaintext = "abc",
.psize = 3,
.digest = (u8 *)(u16 []){ 0x443b },
diff --git a/include/linux/crc64.h b/include/linux/crc64.h
index 0a595b272166..7880aeab69d6 100644
--- a/include/linux/crc64.h
+++ b/include/linux/crc64.h
@@ -5,12 +5,10 @@
#ifndef _LINUX_CRC64_H
#define _LINUX_CRC64_H
#include <linux/types.h>
-#define CRC64_ROCKSOFT_STRING "crc64-rocksoft"
-
u64 __pure crc64_be(u64 crc, const void *p, size_t len);
u64 __pure crc64_rocksoft_generic(u64 crc, const void *p, size_t len);
/**
* crc64_rocksoft_update - Calculate bitwise Rocksoft CRC64
--
2.48.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* Re: [PATCH v2 02/11] crypto: crc64-rocksoft - remove from crypto API
2025-01-30 3:51 ` [PATCH v2 02/11] crypto: crc64-rocksoft - remove from " Eric Biggers
@ 2025-02-02 14:27 ` Ard Biesheuvel
0 siblings, 0 replies; 21+ messages in thread
From: Ard Biesheuvel @ 2025-02-02 14:27 UTC (permalink / raw)
To: Eric Biggers
Cc: linux-kernel, linux-crypto, x86, linux-block, Keith Busch,
Kent Overstreet, Martin K . Petersen
On Thu, 30 Jan 2025 at 04:54, Eric Biggers <ebiggers@kernel.org> wrote:
>
> From: Eric Biggers <ebiggers@google.com>
>
> Remove crc64-rocksoft from the crypto API. It has no known user now
> that the lib is no longer built on top of it. It was also added much
> more recently than the longstanding crc32 and crc32c. Unlike crc32 and
> crc32c, crc64-rocksoft is also not mentioned in the dm-integrity
> documentation and there are no references to it in anywhere in the
> cryptsetup git repo, so it is unlikely to have any user there either.
>
> Also, this CRC variant is named incorrectly; it has nothing to do with
> Rocksoft and should be called crc64-nvme. That is yet another reason to
> remove it from the crypto API; we would not want anyone to start
> depending on the current incorrect algorithm name of crc64-rocksoft.
>
> Note that this change temporarily makes this CRC variant not be covered
> by any tests, as previously it was relying on the crypto self-tests.
> This will be fixed by adding this CRC variant to crc_kunit.
>
> Signed-off-by: Eric Biggers <ebiggers@google.com>
> ---
> crypto/Kconfig | 11 ----
> crypto/Makefile | 1 -
> crypto/crc64_rocksoft_generic.c | 89 ---------------------------------
> crypto/testmgr.c | 7 ---
> crypto/testmgr.h | 12 -----
> include/linux/crc64.h | 2 -
> 6 files changed, 122 deletions(-)
> delete mode 100644 crypto/crc64_rocksoft_generic.c
>
Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
> diff --git a/crypto/Kconfig b/crypto/Kconfig
> index 74ae5f52b784..9ffb59b1aac3 100644
> --- a/crypto/Kconfig
> +++ b/crypto/Kconfig
> @@ -1088,21 +1088,10 @@ config CRYPTO_CRCT10DIF
> help
> CRC16 CRC algorithm used for the T10 (SCSI) Data Integrity Field (DIF)
>
> CRC algorithm used by the SCSI Block Commands standard.
>
> -config CRYPTO_CRC64_ROCKSOFT
> - tristate "CRC64 based on Rocksoft Model algorithm"
> - depends on CRC64
> - select CRYPTO_HASH
> - help
> - CRC64 CRC algorithm based on the Rocksoft Model CRC Algorithm
> -
> - Used by the NVMe implementation of T10 DIF (BLK_DEV_INTEGRITY)
> -
> - See https://zlib.net/crc_v3.txt
> -
> endmenu
>
> menu "Compression"
>
> config CRYPTO_DEFLATE
> diff --git a/crypto/Makefile b/crypto/Makefile
> index f67e853c4690..d3b79b8c9022 100644
> --- a/crypto/Makefile
> +++ b/crypto/Makefile
> @@ -155,11 +155,10 @@ obj-$(CONFIG_CRYPTO_CRC32C) += crc32c_generic.o
> obj-$(CONFIG_CRYPTO_CRC32) += crc32_generic.o
> CFLAGS_crc32c_generic.o += -DARCH=$(ARCH)
> CFLAGS_crc32_generic.o += -DARCH=$(ARCH)
> obj-$(CONFIG_CRYPTO_CRCT10DIF) += crct10dif_generic.o
> CFLAGS_crct10dif_generic.o += -DARCH=$(ARCH)
> -obj-$(CONFIG_CRYPTO_CRC64_ROCKSOFT) += crc64_rocksoft_generic.o
> obj-$(CONFIG_CRYPTO_AUTHENC) += authenc.o authencesn.o
> obj-$(CONFIG_CRYPTO_LZO) += lzo.o lzo-rle.o
> obj-$(CONFIG_CRYPTO_LZ4) += lz4.o
> obj-$(CONFIG_CRYPTO_LZ4HC) += lz4hc.o
> obj-$(CONFIG_CRYPTO_XXHASH) += xxhash_generic.o
> diff --git a/crypto/crc64_rocksoft_generic.c b/crypto/crc64_rocksoft_generic.c
> deleted file mode 100644
> index ce0f3059b912..000000000000
> --- a/crypto/crc64_rocksoft_generic.c
> +++ /dev/null
> @@ -1,89 +0,0 @@
> -// SPDX-License-Identifier: GPL-2.0-only
> -
> -#include <linux/crc64.h>
> -#include <linux/module.h>
> -#include <crypto/internal/hash.h>
> -#include <linux/unaligned.h>
> -
> -static int chksum_init(struct shash_desc *desc)
> -{
> - u64 *crc = shash_desc_ctx(desc);
> -
> - *crc = 0;
> -
> - return 0;
> -}
> -
> -static int chksum_update(struct shash_desc *desc, const u8 *data,
> - unsigned int length)
> -{
> - u64 *crc = shash_desc_ctx(desc);
> -
> - *crc = crc64_rocksoft_generic(*crc, data, length);
> -
> - return 0;
> -}
> -
> -static int chksum_final(struct shash_desc *desc, u8 *out)
> -{
> - u64 *crc = shash_desc_ctx(desc);
> -
> - put_unaligned_le64(*crc, out);
> - return 0;
> -}
> -
> -static int __chksum_finup(u64 crc, const u8 *data, unsigned int len, u8 *out)
> -{
> - crc = crc64_rocksoft_generic(crc, data, len);
> - put_unaligned_le64(crc, out);
> - return 0;
> -}
> -
> -static int chksum_finup(struct shash_desc *desc, const u8 *data,
> - unsigned int len, u8 *out)
> -{
> - u64 *crc = shash_desc_ctx(desc);
> -
> - return __chksum_finup(*crc, data, len, out);
> -}
> -
> -static int chksum_digest(struct shash_desc *desc, const u8 *data,
> - unsigned int length, u8 *out)
> -{
> - return __chksum_finup(0, data, length, out);
> -}
> -
> -static struct shash_alg alg = {
> - .digestsize = sizeof(u64),
> - .init = chksum_init,
> - .update = chksum_update,
> - .final = chksum_final,
> - .finup = chksum_finup,
> - .digest = chksum_digest,
> - .descsize = sizeof(u64),
> - .base = {
> - .cra_name = CRC64_ROCKSOFT_STRING,
> - .cra_driver_name = "crc64-rocksoft-generic",
> - .cra_priority = 200,
> - .cra_blocksize = 1,
> - .cra_module = THIS_MODULE,
> - }
> -};
> -
> -static int __init crc64_rocksoft_init(void)
> -{
> - return crypto_register_shash(&alg);
> -}
> -
> -static void __exit crc64_rocksoft_exit(void)
> -{
> - crypto_unregister_shash(&alg);
> -}
> -
> -module_init(crc64_rocksoft_init);
> -module_exit(crc64_rocksoft_exit);
> -
> -MODULE_LICENSE("GPL");
> -MODULE_DESCRIPTION("Rocksoft model CRC64 calculation.");
> -MODULE_ALIAS_CRYPTO("crc64-rocksoft");
> -MODULE_ALIAS_CRYPTO("crc64-rocksoft-generic");
> diff --git a/crypto/testmgr.c b/crypto/testmgr.c
> index e61490ba4095..0c1c3a6453b6 100644
> --- a/crypto/testmgr.c
> +++ b/crypto/testmgr.c
> @@ -4757,17 +4757,10 @@ static const struct alg_test_desc alg_test_descs[] = {
> .test = alg_test_crc32c,
> .fips_allowed = 1,
> .suite = {
> .hash = __VECS(crc32c_tv_template)
> }
> - }, {
> - .alg = "crc64-rocksoft",
> - .test = alg_test_hash,
> - .fips_allowed = 1,
> - .suite = {
> - .hash = __VECS(crc64_rocksoft_tv_template)
> - }
> }, {
> .alg = "crct10dif",
> .test = alg_test_hash,
> .fips_allowed = 1,
> .suite = {
> diff --git a/crypto/testmgr.h b/crypto/testmgr.h
> index d754ab997186..4ab05046b734 100644
> --- a/crypto/testmgr.h
> +++ b/crypto/testmgr.h
> @@ -6018,22 +6018,10 @@ static const struct hash_testvec rmd160_tv_template[] = {
> };
>
> static const u8 zeroes[4096] = { [0 ... 4095] = 0 };
> static const u8 ones[4096] = { [0 ... 4095] = 0xff };
>
> -static const struct hash_testvec crc64_rocksoft_tv_template[] = {
> - {
> - .plaintext = zeroes,
> - .psize = 4096,
> - .digest = "\x4e\xb6\x22\xeb\x67\xd3\x82\x64",
> - }, {
> - .plaintext = ones,
> - .psize = 4096,
> - .digest = "\xac\xa3\xec\x02\x73\xba\xdd\xc0",
> - }
> -};
> -
> static const struct hash_testvec crct10dif_tv_template[] = {
> {
> .plaintext = "abc",
> .psize = 3,
> .digest = (u8 *)(u16 []){ 0x443b },
> diff --git a/include/linux/crc64.h b/include/linux/crc64.h
> index 0a595b272166..7880aeab69d6 100644
> --- a/include/linux/crc64.h
> +++ b/include/linux/crc64.h
> @@ -5,12 +5,10 @@
> #ifndef _LINUX_CRC64_H
> #define _LINUX_CRC64_H
>
> #include <linux/types.h>
>
> -#define CRC64_ROCKSOFT_STRING "crc64-rocksoft"
> -
> u64 __pure crc64_be(u64 crc, const void *p, size_t len);
> u64 __pure crc64_rocksoft_generic(u64 crc, const void *p, size_t len);
>
> /**
> * crc64_rocksoft_update - Calculate bitwise Rocksoft CRC64
> --
> 2.48.1
>
^ permalink raw reply [flat|nested] 21+ messages in thread
* [PATCH v2 03/11] lib/crc64: rename CRC64-Rocksoft to CRC64-NVME
2025-01-30 3:51 [PATCH v2 00/11] CRC64 library rework and x86 CRC optimization Eric Biggers
2025-01-30 3:51 ` [PATCH v2 01/11] lib/crc64-rocksoft: stop wrapping the crypto API Eric Biggers
2025-01-30 3:51 ` [PATCH v2 02/11] crypto: crc64-rocksoft - remove from " Eric Biggers
@ 2025-01-30 3:51 ` Eric Biggers
2025-02-02 14:28 ` Ard Biesheuvel
2025-02-08 18:59 ` Eric Biggers
2025-01-30 3:51 ` [PATCH v2 04/11] lib/crc_kunit.c: add test and benchmark for CRC64-NVME Eric Biggers
` (10 subsequent siblings)
13 siblings, 2 replies; 21+ messages in thread
From: Eric Biggers @ 2025-01-30 3:51 UTC (permalink / raw)
To: linux-kernel
Cc: linux-crypto, x86, linux-block, Ard Biesheuvel, Keith Busch,
Kent Overstreet, Martin K . Petersen
From: Eric Biggers <ebiggers@google.com>
This CRC64 variant comes from the NVME NVM Command Set Specification
(https://nvmexpress.org/wp-content/uploads/NVM-Express-NVM-Command-Set-Specification-1.0e-2024.07.29-Ratified.pdf).
The "Rocksoft Model CRC Algorithm", published in 1993 and available at
https://www.zlib.net/crc_v3.txt, is a generalized CRC algorithm that can
calculate any variant of CRC, given a list of parameters such as
polynomial, bit order, etc. It is not a CRC variant.
The NVME NVM Command Set Specification has a table that gives the
"Rocksoft Model Parameters" for the CRC variant it uses. When support
for this CRC variant was added to Linux, this table seems to have been
misinterpreted as naming the CRC variant the "Rocksoft" CRC. In fact,
the CRC variant is not explicitly named by the NVME spec.
Most implementations of this CRC variant outside Linux have been calling
it CRC64-NVME. Therefore, update Linux to match.
While at it, remove the superfluous "update" from the function name, so
crc64_rocksoft_update() is now just crc64_nvme(), matching most of the
other CRC library functions.
Signed-off-by: Eric Biggers <ebiggers@google.com>
---
block/t10-pi.c | 2 +-
include/linux/crc64.h | 11 +++++++----
lib/crc64.c | 10 +++++-----
lib/gen_crc64table.c | 10 +++++-----
4 files changed, 18 insertions(+), 15 deletions(-)
diff --git a/block/t10-pi.c b/block/t10-pi.c
index 2d05421f0fa5..2577114ff20c 100644
--- a/block/t10-pi.c
+++ b/block/t10-pi.c
@@ -208,11 +208,11 @@ static void t10_pi_type1_complete(struct request *rq, unsigned int nr_bytes)
}
}
static __be64 ext_pi_crc64(u64 crc, void *data, unsigned int len)
{
- return cpu_to_be64(crc64_rocksoft_update(crc, data, len));
+ return cpu_to_be64(crc64_nvme(crc, data, len));
}
static void ext_pi_crc64_generate(struct blk_integrity_iter *iter,
struct blk_integrity *bi)
{
diff --git a/include/linux/crc64.h b/include/linux/crc64.h
index 7880aeab69d6..17cf5af3e78e 100644
--- a/include/linux/crc64.h
+++ b/include/linux/crc64.h
@@ -6,20 +6,23 @@
#define _LINUX_CRC64_H
#include <linux/types.h>
u64 __pure crc64_be(u64 crc, const void *p, size_t len);
-u64 __pure crc64_rocksoft_generic(u64 crc, const void *p, size_t len);
+u64 __pure crc64_nvme_generic(u64 crc, const void *p, size_t len);
/**
- * crc64_rocksoft_update - Calculate bitwise Rocksoft CRC64
+ * crc64_nvme - Calculate CRC64-NVME
* @crc: seed value for computation. 0 for a new CRC calculation, or the
* previous crc64 value if computing incrementally.
* @p: pointer to buffer over which CRC64 is run
* @len: length of buffer @p
+ *
+ * This computes the CRC64 defined in the NVME NVM Command Set Specification,
+ * *including the bitwise inversion at the beginning and end*.
*/
-static inline u64 crc64_rocksoft_update(u64 crc, const u8 *p, size_t len)
+static inline u64 crc64_nvme(u64 crc, const u8 *p, size_t len)
{
- return crc64_rocksoft_generic(crc, p, len);
+ return crc64_nvme_generic(crc, p, len);
}
#endif /* _LINUX_CRC64_H */
diff --git a/lib/crc64.c b/lib/crc64.c
index b5136fb4c199..d6f3f245eede 100644
--- a/lib/crc64.c
+++ b/lib/crc64.c
@@ -20,12 +20,12 @@
* x^64 + x^62 + x^57 + x^55 + x^54 + x^53 + x^52 + x^47 + x^46 + x^45 +
* x^40 + x^39 + x^38 + x^37 + x^35 + x^33 + x^32 + x^31 + x^29 + x^27 +
* x^24 + x^23 + x^22 + x^21 + x^19 + x^17 + x^13 + x^12 + x^10 + x^9 +
* x^7 + x^4 + x + 1
*
- * crc64rocksoft[256] table is from the Rocksoft specification polynomial
- * defined as,
+ * crc64nvmetable[256] uses the CRC64 polynomial from the NVME NVM Command Set
+ * Specification and uses least-significant-bit first bit order:
*
* x^64 + x^63 + x^61 + x^59 + x^58 + x^56 + x^55 + x^52 + x^49 + x^48 + x^47 +
* x^46 + x^44 + x^41 + x^37 + x^36 + x^34 + x^32 + x^31 + x^28 + x^26 + x^23 +
* x^22 + x^19 + x^16 + x^13 + x^12 + x^10 + x^9 + x^6 + x^4 + x^3 + 1
*
@@ -61,18 +61,18 @@ u64 __pure crc64_be(u64 crc, const void *p, size_t len)
return crc;
}
EXPORT_SYMBOL_GPL(crc64_be);
-u64 __pure crc64_rocksoft_generic(u64 crc, const void *p, size_t len)
+u64 __pure crc64_nvme_generic(u64 crc, const void *p, size_t len)
{
const unsigned char *_p = p;
size_t i;
crc = ~crc;
for (i = 0; i < len; i++)
- crc = (crc >> 8) ^ crc64rocksofttable[(crc & 0xff) ^ *_p++];
+ crc = (crc >> 8) ^ crc64nvmetable[(crc & 0xff) ^ *_p++];
return ~crc;
}
-EXPORT_SYMBOL_GPL(crc64_rocksoft_generic);
+EXPORT_SYMBOL_GPL(crc64_nvme_generic);
diff --git a/lib/gen_crc64table.c b/lib/gen_crc64table.c
index 55e222acd0b8..e05a4230a0a0 100644
--- a/lib/gen_crc64table.c
+++ b/lib/gen_crc64table.c
@@ -15,14 +15,14 @@
*/
#include <inttypes.h>
#include <stdio.h>
#define CRC64_ECMA182_POLY 0x42F0E1EBA9EA3693ULL
-#define CRC64_ROCKSOFT_POLY 0x9A6C9329AC4BC9B5ULL
+#define CRC64_NVME_POLY 0x9A6C9329AC4BC9B5ULL
static uint64_t crc64_table[256] = {0};
-static uint64_t crc64_rocksoft_table[256] = {0};
+static uint64_t crc64_nvme_table[256] = {0};
static void generate_reflected_crc64_table(uint64_t table[256], uint64_t poly)
{
uint64_t i, j, c, crc;
@@ -80,16 +80,16 @@ static void print_crc64_tables(void)
printf("#include <linux/types.h>\n");
printf("#include <linux/cache.h>\n\n");
printf("static const u64 ____cacheline_aligned crc64table[256] = {\n");
output_table(crc64_table);
- printf("\nstatic const u64 ____cacheline_aligned crc64rocksofttable[256] = {\n");
- output_table(crc64_rocksoft_table);
+ printf("\nstatic const u64 ____cacheline_aligned crc64nvmetable[256] = {\n");
+ output_table(crc64_nvme_table);
}
int main(int argc, char *argv[])
{
generate_crc64_table(crc64_table, CRC64_ECMA182_POLY);
- generate_reflected_crc64_table(crc64_rocksoft_table, CRC64_ROCKSOFT_POLY);
+ generate_reflected_crc64_table(crc64_nvme_table, CRC64_NVME_POLY);
print_crc64_tables();
return 0;
}
--
2.48.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* Re: [PATCH v2 03/11] lib/crc64: rename CRC64-Rocksoft to CRC64-NVME
2025-01-30 3:51 ` [PATCH v2 03/11] lib/crc64: rename CRC64-Rocksoft to CRC64-NVME Eric Biggers
@ 2025-02-02 14:28 ` Ard Biesheuvel
2025-02-08 18:59 ` Eric Biggers
1 sibling, 0 replies; 21+ messages in thread
From: Ard Biesheuvel @ 2025-02-02 14:28 UTC (permalink / raw)
To: Eric Biggers
Cc: linux-kernel, linux-crypto, x86, linux-block, Keith Busch,
Kent Overstreet, Martin K . Petersen
On Thu, 30 Jan 2025 at 04:54, Eric Biggers <ebiggers@kernel.org> wrote:
>
> From: Eric Biggers <ebiggers@google.com>
>
> This CRC64 variant comes from the NVME NVM Command Set Specification
> (https://nvmexpress.org/wp-content/uploads/NVM-Express-NVM-Command-Set-Specification-1.0e-2024.07.29-Ratified.pdf).
>
> The "Rocksoft Model CRC Algorithm", published in 1993 and available at
> https://www.zlib.net/crc_v3.txt, is a generalized CRC algorithm that can
> calculate any variant of CRC, given a list of parameters such as
> polynomial, bit order, etc. It is not a CRC variant.
>
> The NVME NVM Command Set Specification has a table that gives the
> "Rocksoft Model Parameters" for the CRC variant it uses. When support
> for this CRC variant was added to Linux, this table seems to have been
> misinterpreted as naming the CRC variant the "Rocksoft" CRC. In fact,
> the CRC variant is not explicitly named by the NVME spec.
>
> Most implementations of this CRC variant outside Linux have been calling
> it CRC64-NVME. Therefore, update Linux to match.
>
> While at it, remove the superfluous "update" from the function name, so
> crc64_rocksoft_update() is now just crc64_nvme(), matching most of the
> other CRC library functions.
>
> Signed-off-by: Eric Biggers <ebiggers@google.com>
> ---
> block/t10-pi.c | 2 +-
> include/linux/crc64.h | 11 +++++++----
> lib/crc64.c | 10 +++++-----
> lib/gen_crc64table.c | 10 +++++-----
> 4 files changed, 18 insertions(+), 15 deletions(-)
>
Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
> diff --git a/block/t10-pi.c b/block/t10-pi.c
> index 2d05421f0fa5..2577114ff20c 100644
> --- a/block/t10-pi.c
> +++ b/block/t10-pi.c
> @@ -208,11 +208,11 @@ static void t10_pi_type1_complete(struct request *rq, unsigned int nr_bytes)
> }
> }
>
> static __be64 ext_pi_crc64(u64 crc, void *data, unsigned int len)
> {
> - return cpu_to_be64(crc64_rocksoft_update(crc, data, len));
> + return cpu_to_be64(crc64_nvme(crc, data, len));
> }
>
> static void ext_pi_crc64_generate(struct blk_integrity_iter *iter,
> struct blk_integrity *bi)
> {
> diff --git a/include/linux/crc64.h b/include/linux/crc64.h
> index 7880aeab69d6..17cf5af3e78e 100644
> --- a/include/linux/crc64.h
> +++ b/include/linux/crc64.h
> @@ -6,20 +6,23 @@
> #define _LINUX_CRC64_H
>
> #include <linux/types.h>
>
> u64 __pure crc64_be(u64 crc, const void *p, size_t len);
> -u64 __pure crc64_rocksoft_generic(u64 crc, const void *p, size_t len);
> +u64 __pure crc64_nvme_generic(u64 crc, const void *p, size_t len);
>
> /**
> - * crc64_rocksoft_update - Calculate bitwise Rocksoft CRC64
> + * crc64_nvme - Calculate CRC64-NVME
> * @crc: seed value for computation. 0 for a new CRC calculation, or the
> * previous crc64 value if computing incrementally.
> * @p: pointer to buffer over which CRC64 is run
> * @len: length of buffer @p
> + *
> + * This computes the CRC64 defined in the NVME NVM Command Set Specification,
> + * *including the bitwise inversion at the beginning and end*.
> */
> -static inline u64 crc64_rocksoft_update(u64 crc, const u8 *p, size_t len)
> +static inline u64 crc64_nvme(u64 crc, const u8 *p, size_t len)
> {
> - return crc64_rocksoft_generic(crc, p, len);
> + return crc64_nvme_generic(crc, p, len);
> }
>
> #endif /* _LINUX_CRC64_H */
> diff --git a/lib/crc64.c b/lib/crc64.c
> index b5136fb4c199..d6f3f245eede 100644
> --- a/lib/crc64.c
> +++ b/lib/crc64.c
> @@ -20,12 +20,12 @@
> * x^64 + x^62 + x^57 + x^55 + x^54 + x^53 + x^52 + x^47 + x^46 + x^45 +
> * x^40 + x^39 + x^38 + x^37 + x^35 + x^33 + x^32 + x^31 + x^29 + x^27 +
> * x^24 + x^23 + x^22 + x^21 + x^19 + x^17 + x^13 + x^12 + x^10 + x^9 +
> * x^7 + x^4 + x + 1
> *
> - * crc64rocksoft[256] table is from the Rocksoft specification polynomial
> - * defined as,
> + * crc64nvmetable[256] uses the CRC64 polynomial from the NVME NVM Command Set
> + * Specification and uses least-significant-bit first bit order:
> *
> * x^64 + x^63 + x^61 + x^59 + x^58 + x^56 + x^55 + x^52 + x^49 + x^48 + x^47 +
> * x^46 + x^44 + x^41 + x^37 + x^36 + x^34 + x^32 + x^31 + x^28 + x^26 + x^23 +
> * x^22 + x^19 + x^16 + x^13 + x^12 + x^10 + x^9 + x^6 + x^4 + x^3 + 1
> *
> @@ -61,18 +61,18 @@ u64 __pure crc64_be(u64 crc, const void *p, size_t len)
>
> return crc;
> }
> EXPORT_SYMBOL_GPL(crc64_be);
>
> -u64 __pure crc64_rocksoft_generic(u64 crc, const void *p, size_t len)
> +u64 __pure crc64_nvme_generic(u64 crc, const void *p, size_t len)
> {
> const unsigned char *_p = p;
> size_t i;
>
> crc = ~crc;
>
> for (i = 0; i < len; i++)
> - crc = (crc >> 8) ^ crc64rocksofttable[(crc & 0xff) ^ *_p++];
> + crc = (crc >> 8) ^ crc64nvmetable[(crc & 0xff) ^ *_p++];
>
> return ~crc;
> }
> -EXPORT_SYMBOL_GPL(crc64_rocksoft_generic);
> +EXPORT_SYMBOL_GPL(crc64_nvme_generic);
> diff --git a/lib/gen_crc64table.c b/lib/gen_crc64table.c
> index 55e222acd0b8..e05a4230a0a0 100644
> --- a/lib/gen_crc64table.c
> +++ b/lib/gen_crc64table.c
> @@ -15,14 +15,14 @@
> */
> #include <inttypes.h>
> #include <stdio.h>
>
> #define CRC64_ECMA182_POLY 0x42F0E1EBA9EA3693ULL
> -#define CRC64_ROCKSOFT_POLY 0x9A6C9329AC4BC9B5ULL
> +#define CRC64_NVME_POLY 0x9A6C9329AC4BC9B5ULL
>
> static uint64_t crc64_table[256] = {0};
> -static uint64_t crc64_rocksoft_table[256] = {0};
> +static uint64_t crc64_nvme_table[256] = {0};
>
> static void generate_reflected_crc64_table(uint64_t table[256], uint64_t poly)
> {
> uint64_t i, j, c, crc;
>
> @@ -80,16 +80,16 @@ static void print_crc64_tables(void)
> printf("#include <linux/types.h>\n");
> printf("#include <linux/cache.h>\n\n");
> printf("static const u64 ____cacheline_aligned crc64table[256] = {\n");
> output_table(crc64_table);
>
> - printf("\nstatic const u64 ____cacheline_aligned crc64rocksofttable[256] = {\n");
> - output_table(crc64_rocksoft_table);
> + printf("\nstatic const u64 ____cacheline_aligned crc64nvmetable[256] = {\n");
> + output_table(crc64_nvme_table);
> }
>
> int main(int argc, char *argv[])
> {
> generate_crc64_table(crc64_table, CRC64_ECMA182_POLY);
> - generate_reflected_crc64_table(crc64_rocksoft_table, CRC64_ROCKSOFT_POLY);
> + generate_reflected_crc64_table(crc64_nvme_table, CRC64_NVME_POLY);
> print_crc64_tables();
> return 0;
> }
> --
> 2.48.1
>
^ permalink raw reply [flat|nested] 21+ messages in thread* Re: [PATCH v2 03/11] lib/crc64: rename CRC64-Rocksoft to CRC64-NVME
2025-01-30 3:51 ` [PATCH v2 03/11] lib/crc64: rename CRC64-Rocksoft to CRC64-NVME Eric Biggers
2025-02-02 14:28 ` Ard Biesheuvel
@ 2025-02-08 18:59 ` Eric Biggers
1 sibling, 0 replies; 21+ messages in thread
From: Eric Biggers @ 2025-02-08 18:59 UTC (permalink / raw)
To: linux-kernel
Cc: linux-crypto, x86, linux-block, Ard Biesheuvel, Keith Busch,
Kent Overstreet, Martin K . Petersen
On Wed, Jan 29, 2025 at 07:51:22PM -0800, Eric Biggers wrote:
> The NVME NVM Command Set Specification has a table that gives the
> "Rocksoft Model Parameters" for the CRC variant it uses. When support
> for this CRC variant was added to Linux, this table seems to have been
> misinterpreted as naming the CRC variant the "Rocksoft" CRC. In fact,
> the CRC variant is not explicitly named by the NVME spec.
Making a small correction to this commit message: the spec actually does name
this CRC variant, as "NVM Express 64b CRC". (The name is one of the listed
Rocksoft Model Parameters.) So that further supports using nvme in the name.
- Eric
^ permalink raw reply [flat|nested] 21+ messages in thread
* [PATCH v2 04/11] lib/crc_kunit.c: add test and benchmark for CRC64-NVME
2025-01-30 3:51 [PATCH v2 00/11] CRC64 library rework and x86 CRC optimization Eric Biggers
` (2 preceding siblings ...)
2025-01-30 3:51 ` [PATCH v2 03/11] lib/crc64: rename CRC64-Rocksoft to CRC64-NVME Eric Biggers
@ 2025-01-30 3:51 ` Eric Biggers
2025-02-02 14:29 ` Ard Biesheuvel
2025-01-30 3:51 ` [PATCH v2 05/11] lib/crc64: add support for arch-optimized implementations Eric Biggers
` (9 subsequent siblings)
13 siblings, 1 reply; 21+ messages in thread
From: Eric Biggers @ 2025-01-30 3:51 UTC (permalink / raw)
To: linux-kernel
Cc: linux-crypto, x86, linux-block, Ard Biesheuvel, Keith Busch,
Kent Overstreet, Martin K . Petersen
From: Eric Biggers <ebiggers@google.com>
Wire up crc64_nvme() to the new CRC unit test and benchmark.
This replaces and improves on the test coverage that was lost by
removing this CRC variant from the crypto API.
Signed-off-by: Eric Biggers <ebiggers@google.com>
---
lib/crc_kunit.c | 30 +++++++++++++++++++++++++++++-
1 file changed, 29 insertions(+), 1 deletion(-)
diff --git a/lib/crc_kunit.c b/lib/crc_kunit.c
index 6a61d4b5fd45..1e82fcf9489e 100644
--- a/lib/crc_kunit.c
+++ b/lib/crc_kunit.c
@@ -30,11 +30,12 @@ static size_t test_buflen;
* polynomial coefficients in each byte), false if it's a "big endian" CRC
* (natural mapping between bits and polynomial coefficients in each byte)
* @poly: The generator polynomial with the highest-order term omitted.
* Bit-reversed if @le is true.
* @func: The function to compute a CRC. The type signature uses u64 so that it
- * can fit any CRC up to CRC-64.
+ * can fit any CRC up to CRC-64. The function is expected to *not*
+ * invert the CRC at the beginning and end.
* @combine_func: Optional function to combine two CRCs.
*/
struct crc_variant {
int bits;
bool le;
@@ -405,10 +406,35 @@ static void crc64_be_test(struct kunit *test)
static void crc64_be_benchmark(struct kunit *test)
{
crc_benchmark(test, crc64_be_wrapper);
}
+/* crc64_nvme */
+
+static u64 crc64_nvme_wrapper(u64 crc, const u8 *p, size_t len)
+{
+ /* The inversions that crc64_nvme() does have to be undone here. */
+ return ~crc64_nvme(~crc, p, len);
+}
+
+static const struct crc_variant crc_variant_crc64_nvme = {
+ .bits = 64,
+ .le = true,
+ .poly = 0x9a6c9329ac4bc9b5,
+ .func = crc64_nvme_wrapper,
+};
+
+static void crc64_nvme_test(struct kunit *test)
+{
+ crc_test(test, &crc_variant_crc64_nvme);
+}
+
+static void crc64_nvme_benchmark(struct kunit *test)
+{
+ crc_benchmark(test, crc64_nvme_wrapper);
+}
+
static struct kunit_case crc_test_cases[] = {
KUNIT_CASE(crc16_test),
KUNIT_CASE(crc16_benchmark),
KUNIT_CASE(crc_t10dif_test),
KUNIT_CASE(crc_t10dif_benchmark),
@@ -418,10 +444,12 @@ static struct kunit_case crc_test_cases[] = {
KUNIT_CASE(crc32_be_benchmark),
KUNIT_CASE(crc32c_test),
KUNIT_CASE(crc32c_benchmark),
KUNIT_CASE(crc64_be_test),
KUNIT_CASE(crc64_be_benchmark),
+ KUNIT_CASE(crc64_nvme_test),
+ KUNIT_CASE(crc64_nvme_benchmark),
{},
};
static struct kunit_suite crc_test_suite = {
.name = "crc",
--
2.48.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* Re: [PATCH v2 04/11] lib/crc_kunit.c: add test and benchmark for CRC64-NVME
2025-01-30 3:51 ` [PATCH v2 04/11] lib/crc_kunit.c: add test and benchmark for CRC64-NVME Eric Biggers
@ 2025-02-02 14:29 ` Ard Biesheuvel
0 siblings, 0 replies; 21+ messages in thread
From: Ard Biesheuvel @ 2025-02-02 14:29 UTC (permalink / raw)
To: Eric Biggers
Cc: linux-kernel, linux-crypto, x86, linux-block, Keith Busch,
Kent Overstreet, Martin K . Petersen
On Thu, 30 Jan 2025 at 04:54, Eric Biggers <ebiggers@kernel.org> wrote:
>
> From: Eric Biggers <ebiggers@google.com>
>
> Wire up crc64_nvme() to the new CRC unit test and benchmark.
>
> This replaces and improves on the test coverage that was lost by
> removing this CRC variant from the crypto API.
>
> Signed-off-by: Eric Biggers <ebiggers@google.com>
> ---
> lib/crc_kunit.c | 30 +++++++++++++++++++++++++++++-
> 1 file changed, 29 insertions(+), 1 deletion(-)
>
Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
> diff --git a/lib/crc_kunit.c b/lib/crc_kunit.c
> index 6a61d4b5fd45..1e82fcf9489e 100644
> --- a/lib/crc_kunit.c
> +++ b/lib/crc_kunit.c
> @@ -30,11 +30,12 @@ static size_t test_buflen;
> * polynomial coefficients in each byte), false if it's a "big endian" CRC
> * (natural mapping between bits and polynomial coefficients in each byte)
> * @poly: The generator polynomial with the highest-order term omitted.
> * Bit-reversed if @le is true.
> * @func: The function to compute a CRC. The type signature uses u64 so that it
> - * can fit any CRC up to CRC-64.
> + * can fit any CRC up to CRC-64. The function is expected to *not*
> + * invert the CRC at the beginning and end.
> * @combine_func: Optional function to combine two CRCs.
> */
> struct crc_variant {
> int bits;
> bool le;
> @@ -405,10 +406,35 @@ static void crc64_be_test(struct kunit *test)
> static void crc64_be_benchmark(struct kunit *test)
> {
> crc_benchmark(test, crc64_be_wrapper);
> }
>
> +/* crc64_nvme */
> +
> +static u64 crc64_nvme_wrapper(u64 crc, const u8 *p, size_t len)
> +{
> + /* The inversions that crc64_nvme() does have to be undone here. */
> + return ~crc64_nvme(~crc, p, len);
> +}
> +
> +static const struct crc_variant crc_variant_crc64_nvme = {
> + .bits = 64,
> + .le = true,
> + .poly = 0x9a6c9329ac4bc9b5,
> + .func = crc64_nvme_wrapper,
> +};
> +
> +static void crc64_nvme_test(struct kunit *test)
> +{
> + crc_test(test, &crc_variant_crc64_nvme);
> +}
> +
> +static void crc64_nvme_benchmark(struct kunit *test)
> +{
> + crc_benchmark(test, crc64_nvme_wrapper);
> +}
> +
> static struct kunit_case crc_test_cases[] = {
> KUNIT_CASE(crc16_test),
> KUNIT_CASE(crc16_benchmark),
> KUNIT_CASE(crc_t10dif_test),
> KUNIT_CASE(crc_t10dif_benchmark),
> @@ -418,10 +444,12 @@ static struct kunit_case crc_test_cases[] = {
> KUNIT_CASE(crc32_be_benchmark),
> KUNIT_CASE(crc32c_test),
> KUNIT_CASE(crc32c_benchmark),
> KUNIT_CASE(crc64_be_test),
> KUNIT_CASE(crc64_be_benchmark),
> + KUNIT_CASE(crc64_nvme_test),
> + KUNIT_CASE(crc64_nvme_benchmark),
> {},
> };
>
> static struct kunit_suite crc_test_suite = {
> .name = "crc",
> --
> 2.48.1
>
^ permalink raw reply [flat|nested] 21+ messages in thread
* [PATCH v2 05/11] lib/crc64: add support for arch-optimized implementations
2025-01-30 3:51 [PATCH v2 00/11] CRC64 library rework and x86 CRC optimization Eric Biggers
` (3 preceding siblings ...)
2025-01-30 3:51 ` [PATCH v2 04/11] lib/crc_kunit.c: add test and benchmark for CRC64-NVME Eric Biggers
@ 2025-01-30 3:51 ` Eric Biggers
2025-02-02 14:31 ` Ard Biesheuvel
2025-01-30 3:51 ` [PATCH v2 06/11] x86: move ZMM exclusion list into CPU feature flag Eric Biggers
` (8 subsequent siblings)
13 siblings, 1 reply; 21+ messages in thread
From: Eric Biggers @ 2025-01-30 3:51 UTC (permalink / raw)
To: linux-kernel
Cc: linux-crypto, x86, linux-block, Ard Biesheuvel, Keith Busch,
Kent Overstreet, Martin K . Petersen
From: Eric Biggers <ebiggers@google.com>
Add support for architecture-optimized implementations of the CRC64
library functions, following the approach taken for the CRC32 and
CRC-T10DIF library functions.
Also take the opportunity to tweak the function prototypes:
- Use 'const void *' for the lib entry points (since this is easier for
users) but 'const u8 *' for the underlying arch and generic functions
(since this is easier for the implementations of these functions).
- Don't bother with __pure. It's an unusual optimization that doesn't
help properly written code. It's a weird quirk we can do without.
Signed-off-by: Eric Biggers <ebiggers@google.com>
---
include/linux/crc64.h | 26 ++++++++++++++++++++++----
lib/Kconfig | 7 +++++++
lib/crc64.c | 36 ++++++++----------------------------
3 files changed, 37 insertions(+), 32 deletions(-)
diff --git a/include/linux/crc64.h b/include/linux/crc64.h
index 17cf5af3e78e..41de30b907df 100644
--- a/include/linux/crc64.h
+++ b/include/linux/crc64.h
@@ -5,12 +5,28 @@
#ifndef _LINUX_CRC64_H
#define _LINUX_CRC64_H
#include <linux/types.h>
-u64 __pure crc64_be(u64 crc, const void *p, size_t len);
-u64 __pure crc64_nvme_generic(u64 crc, const void *p, size_t len);
+u64 crc64_be_arch(u64 crc, const u8 *p, size_t len);
+u64 crc64_be_generic(u64 crc, const u8 *p, size_t len);
+u64 crc64_nvme_arch(u64 crc, const u8 *p, size_t len);
+u64 crc64_nvme_generic(u64 crc, const u8 *p, size_t len);
+
+/**
+ * crc64_be - Calculate bitwise big-endian ECMA-182 CRC64
+ * @crc: seed value for computation. 0 or (u64)~0 for a new CRC calculation,
+ * or the previous crc64 value if computing incrementally.
+ * @p: pointer to buffer over which CRC64 is run
+ * @len: length of buffer @p
+ */
+static inline u64 crc64_be(u64 crc, const void *p, size_t len)
+{
+ if (IS_ENABLED(CONFIG_CRC64_ARCH))
+ return crc64_be_arch(crc, p, len);
+ return crc64_be_generic(crc, p, len);
+}
/**
* crc64_nvme - Calculate CRC64-NVME
* @crc: seed value for computation. 0 for a new CRC calculation, or the
* previous crc64 value if computing incrementally.
@@ -18,11 +34,13 @@ u64 __pure crc64_nvme_generic(u64 crc, const void *p, size_t len);
* @len: length of buffer @p
*
* This computes the CRC64 defined in the NVME NVM Command Set Specification,
* *including the bitwise inversion at the beginning and end*.
*/
-static inline u64 crc64_nvme(u64 crc, const u8 *p, size_t len)
+static inline u64 crc64_nvme(u64 crc, const void *p, size_t len)
{
- return crc64_nvme_generic(crc, p, len);
+ if (IS_ENABLED(CONFIG_CRC64_ARCH))
+ return ~crc64_nvme_arch(~crc, p, len);
+ return ~crc64_nvme_generic(~crc, p, len);
}
#endif /* _LINUX_CRC64_H */
diff --git a/lib/Kconfig b/lib/Kconfig
index da07fd39cf97..67bbf4f64dd9 100644
--- a/lib/Kconfig
+++ b/lib/Kconfig
@@ -199,10 +199,17 @@ config CRC64
This option is provided for the case where no in-kernel-tree
modules require CRC64 functions, but a module built outside
the kernel tree does. Such modules that use library CRC64
functions require M here.
+config ARCH_HAS_CRC64
+ bool
+
+config CRC64_ARCH
+ tristate
+ default CRC64 if ARCH_HAS_CRC64 && CRC_OPTIMIZATIONS
+
config CRC4
tristate "CRC4 functions"
help
This option is provided for the case where no in-kernel-tree
modules require CRC4 functions, but a module built outside
diff --git a/lib/crc64.c b/lib/crc64.c
index d6f3f245eede..5b1b17057f0a 100644
--- a/lib/crc64.c
+++ b/lib/crc64.c
@@ -39,40 +39,20 @@
#include "crc64table.h"
MODULE_DESCRIPTION("CRC64 calculations");
MODULE_LICENSE("GPL v2");
-/**
- * crc64_be - Calculate bitwise big-endian ECMA-182 CRC64
- * @crc: seed value for computation. 0 or (u64)~0 for a new CRC calculation,
- * or the previous crc64 value if computing incrementally.
- * @p: pointer to buffer over which CRC64 is run
- * @len: length of buffer @p
- */
-u64 __pure crc64_be(u64 crc, const void *p, size_t len)
+u64 crc64_be_generic(u64 crc, const u8 *p, size_t len)
{
- size_t i, t;
-
- const unsigned char *_p = p;
-
- for (i = 0; i < len; i++) {
- t = ((crc >> 56) ^ (*_p++)) & 0xFF;
- crc = crc64table[t] ^ (crc << 8);
- }
-
+ while (len--)
+ crc = (crc << 8) ^ crc64table[(crc >> 56) ^ *p++];
return crc;
}
-EXPORT_SYMBOL_GPL(crc64_be);
+EXPORT_SYMBOL_GPL(crc64_be_generic);
-u64 __pure crc64_nvme_generic(u64 crc, const void *p, size_t len)
+u64 crc64_nvme_generic(u64 crc, const u8 *p, size_t len)
{
- const unsigned char *_p = p;
- size_t i;
-
- crc = ~crc;
-
- for (i = 0; i < len; i++)
- crc = (crc >> 8) ^ crc64nvmetable[(crc & 0xff) ^ *_p++];
-
- return ~crc;
+ while (len--)
+ crc = (crc >> 8) ^ crc64nvmetable[(crc & 0xff) ^ *p++];
+ return crc;
}
EXPORT_SYMBOL_GPL(crc64_nvme_generic);
--
2.48.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* Re: [PATCH v2 05/11] lib/crc64: add support for arch-optimized implementations
2025-01-30 3:51 ` [PATCH v2 05/11] lib/crc64: add support for arch-optimized implementations Eric Biggers
@ 2025-02-02 14:31 ` Ard Biesheuvel
0 siblings, 0 replies; 21+ messages in thread
From: Ard Biesheuvel @ 2025-02-02 14:31 UTC (permalink / raw)
To: Eric Biggers
Cc: linux-kernel, linux-crypto, x86, linux-block, Keith Busch,
Kent Overstreet, Martin K . Petersen
On Thu, 30 Jan 2025 at 04:54, Eric Biggers <ebiggers@kernel.org> wrote:
>
> From: Eric Biggers <ebiggers@google.com>
>
> Add support for architecture-optimized implementations of the CRC64
> library functions, following the approach taken for the CRC32 and
> CRC-T10DIF library functions.
>
> Also take the opportunity to tweak the function prototypes:
> - Use 'const void *' for the lib entry points (since this is easier for
> users) but 'const u8 *' for the underlying arch and generic functions
> (since this is easier for the implementations of these functions).
> - Don't bother with __pure. It's an unusual optimization that doesn't
> help properly written code. It's a weird quirk we can do without.
>
> Signed-off-by: Eric Biggers <ebiggers@google.com>
> ---
> include/linux/crc64.h | 26 ++++++++++++++++++++++----
> lib/Kconfig | 7 +++++++
> lib/crc64.c | 36 ++++++++----------------------------
> 3 files changed, 37 insertions(+), 32 deletions(-)
>
Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
> diff --git a/include/linux/crc64.h b/include/linux/crc64.h
> index 17cf5af3e78e..41de30b907df 100644
> --- a/include/linux/crc64.h
> +++ b/include/linux/crc64.h
> @@ -5,12 +5,28 @@
> #ifndef _LINUX_CRC64_H
> #define _LINUX_CRC64_H
>
> #include <linux/types.h>
>
> -u64 __pure crc64_be(u64 crc, const void *p, size_t len);
> -u64 __pure crc64_nvme_generic(u64 crc, const void *p, size_t len);
> +u64 crc64_be_arch(u64 crc, const u8 *p, size_t len);
> +u64 crc64_be_generic(u64 crc, const u8 *p, size_t len);
> +u64 crc64_nvme_arch(u64 crc, const u8 *p, size_t len);
> +u64 crc64_nvme_generic(u64 crc, const u8 *p, size_t len);
> +
> +/**
> + * crc64_be - Calculate bitwise big-endian ECMA-182 CRC64
> + * @crc: seed value for computation. 0 or (u64)~0 for a new CRC calculation,
> + * or the previous crc64 value if computing incrementally.
> + * @p: pointer to buffer over which CRC64 is run
> + * @len: length of buffer @p
> + */
> +static inline u64 crc64_be(u64 crc, const void *p, size_t len)
> +{
> + if (IS_ENABLED(CONFIG_CRC64_ARCH))
> + return crc64_be_arch(crc, p, len);
> + return crc64_be_generic(crc, p, len);
> +}
>
> /**
> * crc64_nvme - Calculate CRC64-NVME
> * @crc: seed value for computation. 0 for a new CRC calculation, or the
> * previous crc64 value if computing incrementally.
> @@ -18,11 +34,13 @@ u64 __pure crc64_nvme_generic(u64 crc, const void *p, size_t len);
> * @len: length of buffer @p
> *
> * This computes the CRC64 defined in the NVME NVM Command Set Specification,
> * *including the bitwise inversion at the beginning and end*.
> */
> -static inline u64 crc64_nvme(u64 crc, const u8 *p, size_t len)
> +static inline u64 crc64_nvme(u64 crc, const void *p, size_t len)
> {
> - return crc64_nvme_generic(crc, p, len);
> + if (IS_ENABLED(CONFIG_CRC64_ARCH))
> + return ~crc64_nvme_arch(~crc, p, len);
> + return ~crc64_nvme_generic(~crc, p, len);
> }
>
> #endif /* _LINUX_CRC64_H */
> diff --git a/lib/Kconfig b/lib/Kconfig
> index da07fd39cf97..67bbf4f64dd9 100644
> --- a/lib/Kconfig
> +++ b/lib/Kconfig
> @@ -199,10 +199,17 @@ config CRC64
> This option is provided for the case where no in-kernel-tree
> modules require CRC64 functions, but a module built outside
> the kernel tree does. Such modules that use library CRC64
> functions require M here.
>
> +config ARCH_HAS_CRC64
> + bool
> +
> +config CRC64_ARCH
> + tristate
> + default CRC64 if ARCH_HAS_CRC64 && CRC_OPTIMIZATIONS
> +
> config CRC4
> tristate "CRC4 functions"
> help
> This option is provided for the case where no in-kernel-tree
> modules require CRC4 functions, but a module built outside
> diff --git a/lib/crc64.c b/lib/crc64.c
> index d6f3f245eede..5b1b17057f0a 100644
> --- a/lib/crc64.c
> +++ b/lib/crc64.c
> @@ -39,40 +39,20 @@
> #include "crc64table.h"
>
> MODULE_DESCRIPTION("CRC64 calculations");
> MODULE_LICENSE("GPL v2");
>
> -/**
> - * crc64_be - Calculate bitwise big-endian ECMA-182 CRC64
> - * @crc: seed value for computation. 0 or (u64)~0 for a new CRC calculation,
> - * or the previous crc64 value if computing incrementally.
> - * @p: pointer to buffer over which CRC64 is run
> - * @len: length of buffer @p
> - */
> -u64 __pure crc64_be(u64 crc, const void *p, size_t len)
> +u64 crc64_be_generic(u64 crc, const u8 *p, size_t len)
> {
> - size_t i, t;
> -
> - const unsigned char *_p = p;
> -
> - for (i = 0; i < len; i++) {
> - t = ((crc >> 56) ^ (*_p++)) & 0xFF;
> - crc = crc64table[t] ^ (crc << 8);
> - }
> -
> + while (len--)
> + crc = (crc << 8) ^ crc64table[(crc >> 56) ^ *p++];
> return crc;
> }
> -EXPORT_SYMBOL_GPL(crc64_be);
> +EXPORT_SYMBOL_GPL(crc64_be_generic);
>
> -u64 __pure crc64_nvme_generic(u64 crc, const void *p, size_t len)
> +u64 crc64_nvme_generic(u64 crc, const u8 *p, size_t len)
> {
> - const unsigned char *_p = p;
> - size_t i;
> -
> - crc = ~crc;
> -
> - for (i = 0; i < len; i++)
> - crc = (crc >> 8) ^ crc64nvmetable[(crc & 0xff) ^ *_p++];
> -
> - return ~crc;
> + while (len--)
> + crc = (crc >> 8) ^ crc64nvmetable[(crc & 0xff) ^ *p++];
> + return crc;
> }
> EXPORT_SYMBOL_GPL(crc64_nvme_generic);
> --
> 2.48.1
>
^ permalink raw reply [flat|nested] 21+ messages in thread
* [PATCH v2 06/11] x86: move ZMM exclusion list into CPU feature flag
2025-01-30 3:51 [PATCH v2 00/11] CRC64 library rework and x86 CRC optimization Eric Biggers
` (4 preceding siblings ...)
2025-01-30 3:51 ` [PATCH v2 05/11] lib/crc64: add support for arch-optimized implementations Eric Biggers
@ 2025-01-30 3:51 ` Eric Biggers
2025-01-30 3:51 ` [PATCH v2 07/11] scripts/gen-crc-consts: add gen-crc-consts.py Eric Biggers
` (7 subsequent siblings)
13 siblings, 0 replies; 21+ messages in thread
From: Eric Biggers @ 2025-01-30 3:51 UTC (permalink / raw)
To: linux-kernel
Cc: linux-crypto, x86, linux-block, Ard Biesheuvel, Keith Busch,
Kent Overstreet, Martin K . Petersen, Ingo Molnar
From: Eric Biggers <ebiggers@google.com>
Lift zmm_exclusion_list in aesni-intel_glue.c into the x86 CPU setup
code, and add a new x86 CPU feature flag X86_FEATURE_PREFER_YMM that is
set when the CPU is on this list.
This allows other code in arch/x86/, such as the CRC library code, to
apply the same exclusion list when deciding whether to execute 256-bit
or 512-bit optimized functions.
Note that full AVX512 support including ZMM registers is still exposed
to userspace and is still supported for in-kernel use. This flag just
indicates whether in-kernel code should prefer to use YMM registers.
Acked-by: Ard Biesheuvel <ardb@kernel.org>
Acked-by: Ingo Molnar <mingo@kernel.org>
Signed-off-by: Eric Biggers <ebiggers@google.com>
---
arch/x86/crypto/aesni-intel_glue.c | 22 +---------------------
arch/x86/include/asm/cpufeatures.h | 1 +
arch/x86/kernel/cpu/intel.c | 22 ++++++++++++++++++++++
3 files changed, 24 insertions(+), 21 deletions(-)
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index 11e95fc62636..3e9ab5cdade4 100644
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -1534,30 +1534,10 @@ DEFINE_GCM_ALGS(vaes_avx10_256, FLAG_AVX10_256,
DEFINE_GCM_ALGS(vaes_avx10_512, FLAG_AVX10_512,
"generic-gcm-vaes-avx10_512", "rfc4106-gcm-vaes-avx10_512",
AES_GCM_KEY_AVX10_SIZE, 800);
#endif /* CONFIG_AS_VAES && CONFIG_AS_VPCLMULQDQ */
-/*
- * This is a list of CPU models that are known to suffer from downclocking when
- * zmm registers (512-bit vectors) are used. On these CPUs, the AES mode
- * implementations with zmm registers won't be used by default. Implementations
- * with ymm registers (256-bit vectors) will be used by default instead.
- */
-static const struct x86_cpu_id zmm_exclusion_list[] = {
- X86_MATCH_VFM(INTEL_SKYLAKE_X, 0),
- X86_MATCH_VFM(INTEL_ICELAKE_X, 0),
- X86_MATCH_VFM(INTEL_ICELAKE_D, 0),
- X86_MATCH_VFM(INTEL_ICELAKE, 0),
- X86_MATCH_VFM(INTEL_ICELAKE_L, 0),
- X86_MATCH_VFM(INTEL_ICELAKE_NNPI, 0),
- X86_MATCH_VFM(INTEL_TIGERLAKE_L, 0),
- X86_MATCH_VFM(INTEL_TIGERLAKE, 0),
- /* Allow Rocket Lake and later, and Sapphire Rapids and later. */
- /* Also allow AMD CPUs (starting with Zen 4, the first with AVX-512). */
- {},
-};
-
static int __init register_avx_algs(void)
{
int err;
if (!boot_cpu_has(X86_FEATURE_AVX))
@@ -1598,11 +1578,11 @@ static int __init register_avx_algs(void)
ARRAY_SIZE(aes_gcm_algs_vaes_avx10_256),
aes_gcm_simdalgs_vaes_avx10_256);
if (err)
return err;
- if (x86_match_cpu(zmm_exclusion_list)) {
+ if (boot_cpu_has(X86_FEATURE_PREFER_YMM)) {
int i;
aes_xts_alg_vaes_avx10_512.base.cra_priority = 1;
for (i = 0; i < ARRAY_SIZE(aes_gcm_algs_vaes_avx10_512); i++)
aes_gcm_algs_vaes_avx10_512[i].base.cra_priority = 1;
diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h
index 508c0dad116b..99334026a26c 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -481,10 +481,11 @@
#define X86_FEATURE_CLEAR_BHB_HW (21*32+ 3) /* BHI_DIS_S HW control enabled */
#define X86_FEATURE_CLEAR_BHB_LOOP_ON_VMEXIT (21*32+ 4) /* Clear branch history at vmexit using SW loop */
#define X86_FEATURE_AMD_FAST_CPPC (21*32 + 5) /* Fast CPPC */
#define X86_FEATURE_AMD_HETEROGENEOUS_CORES (21*32 + 6) /* Heterogeneous Core Topology */
#define X86_FEATURE_AMD_WORKLOAD_CLASS (21*32 + 7) /* Workload Classification */
+#define X86_FEATURE_PREFER_YMM (21*32 + 8) /* Avoid ZMM registers due to downclocking */
/*
* BUG word(s)
*/
#define X86_BUG(x) (NCAPINTS*32 + (x))
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index 3dce22f00dc3..c3005c4ec46a 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -519,10 +519,29 @@ static void init_intel_misc_features(struct cpuinfo_x86 *c)
msr = this_cpu_read(msr_misc_features_shadow);
wrmsrl(MSR_MISC_FEATURES_ENABLES, msr);
}
+/*
+ * This is a list of Intel CPUs that are known to suffer from downclocking when
+ * ZMM registers (512-bit vectors) are used. On these CPUs, when the kernel
+ * executes SIMD-optimized code such as cryptography functions or CRCs, it
+ * should prefer 256-bit (YMM) code to 512-bit (ZMM) code.
+ */
+static const struct x86_cpu_id zmm_exclusion_list[] = {
+ X86_MATCH_VFM(INTEL_SKYLAKE_X, 0),
+ X86_MATCH_VFM(INTEL_ICELAKE_X, 0),
+ X86_MATCH_VFM(INTEL_ICELAKE_D, 0),
+ X86_MATCH_VFM(INTEL_ICELAKE, 0),
+ X86_MATCH_VFM(INTEL_ICELAKE_L, 0),
+ X86_MATCH_VFM(INTEL_ICELAKE_NNPI, 0),
+ X86_MATCH_VFM(INTEL_TIGERLAKE_L, 0),
+ X86_MATCH_VFM(INTEL_TIGERLAKE, 0),
+ /* Allow Rocket Lake and later, and Sapphire Rapids and later. */
+ {},
+};
+
static void init_intel(struct cpuinfo_x86 *c)
{
early_init_intel(c);
intel_workarounds(c);
@@ -599,10 +618,13 @@ static void init_intel(struct cpuinfo_x86 *c)
if (p)
strcpy(c->x86_model_id, p);
}
#endif
+ if (x86_match_cpu(zmm_exclusion_list))
+ set_cpu_cap(c, X86_FEATURE_PREFER_YMM);
+
/* Work around errata */
srat_detect_node(c);
init_ia32_feat_ctl(c);
--
2.48.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v2 07/11] scripts/gen-crc-consts: add gen-crc-consts.py
2025-01-30 3:51 [PATCH v2 00/11] CRC64 library rework and x86 CRC optimization Eric Biggers
` (5 preceding siblings ...)
2025-01-30 3:51 ` [PATCH v2 06/11] x86: move ZMM exclusion list into CPU feature flag Eric Biggers
@ 2025-01-30 3:51 ` Eric Biggers
2025-01-30 3:51 ` [PATCH v2 08/11] x86/crc: add "template" for [V]PCLMULQDQ based CRC functions Eric Biggers
` (6 subsequent siblings)
13 siblings, 0 replies; 21+ messages in thread
From: Eric Biggers @ 2025-01-30 3:51 UTC (permalink / raw)
To: linux-kernel
Cc: linux-crypto, x86, linux-block, Ard Biesheuvel, Keith Busch,
Kent Overstreet, Martin K . Petersen
From: Eric Biggers <ebiggers@google.com>
Add a Python script that generates constants for computing the given CRC
variant(s) using x86's pclmulqdq or vpclmulqdq instructions.
This is specifically tuned for x86's crc-pclmul-template.S. However,
other architectures with a 64x64 => 128-bit carryless multiplication
instruction should be able to use the generated constants too. (Some
tweaks may be warranted based on the exact instructions available on
each arch, so the script may grow an arch argument in the future.)
The script also supports generating the tables needed for table-based
CRC computation. Thus, it can also be used to reproduce the tables like
t10_dif_crc_table[] and crc16_table[] that are currently hardcoded in
the source with no generation script explicitly documented.
Python is used rather than C since it enables implementing the CRC math
in the simplest way possible, using arbitrary precision integers. The
outputs of this script are intended to be checked into the repo, so
Python will continue to not be required to build the kernel, and the
script has been optimized for simplicity rather than performance.
Acked-by: Ard Biesheuvel <ardb@kernel.org>
Signed-off-by: Eric Biggers <ebiggers@google.com>
---
MAINTAINERS | 1 +
scripts/gen-crc-consts.py | 214 ++++++++++++++++++++++++++++++++++++++
2 files changed, 215 insertions(+)
create mode 100755 scripts/gen-crc-consts.py
diff --git a/MAINTAINERS b/MAINTAINERS
index bc8ce7af3303..aabd17a1cc6c 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -6129,10 +6129,11 @@ S: Maintained
T: git https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git crc-next
F: Documentation/staging/crc*
F: arch/*/lib/crc*
F: include/linux/crc*
F: lib/crc*
+F: scripts/gen-crc-consts.py
CREATIVE SB0540
M: Bastien Nocera <hadess@hadess.net>
L: linux-input@vger.kernel.org
S: Maintained
diff --git a/scripts/gen-crc-consts.py b/scripts/gen-crc-consts.py
new file mode 100755
index 000000000000..43d044ada7be
--- /dev/null
+++ b/scripts/gen-crc-consts.py
@@ -0,0 +1,214 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-or-later
+#
+# Script that generates constants for computing the given CRC variant(s).
+#
+# Copyright 2025 Google LLC
+#
+# Author: Eric Biggers <ebiggers@google.com>
+
+import sys
+
+# XOR (add) an iterable of polynomials.
+def xor(iterable):
+ res = 0
+ for val in iterable:
+ res ^= val
+ return res
+
+# Multiply two polynomials.
+def clmul(a, b):
+ return xor(a << i for i in range(b.bit_length()) if (b & (1 << i)) != 0)
+
+# Polynomial division floor(a / b).
+def div(a, b):
+ q = 0
+ while a.bit_length() >= b.bit_length():
+ q ^= 1 << (a.bit_length() - b.bit_length())
+ a ^= b << (a.bit_length() - b.bit_length())
+ return q
+
+# Reduce the polynomial 'a' modulo the polynomial 'b'.
+def reduce(a, b):
+ return a ^ clmul(div(a, b), b)
+
+# Pretty-print a polynomial.
+def pprint_poly(prefix, poly):
+ terms = ['1' if i == 0 else 'x' if i == 1 else f'x^{i}'
+ for i in reversed(range(poly.bit_length()))
+ if (poly & (1 << i)) != 0]
+ j = 0
+ while j < len(terms):
+ s = prefix + terms[j] + (' +' if j < len(terms) - 1 else '')
+ j += 1
+ while j < len(terms) and len(s) < 72:
+ s += ' ' + terms[j] + (' +' if j < len(terms) - 1 else '')
+ j += 1
+ print(s)
+ prefix = ' * ' + (' ' * (len(prefix) - 3))
+
+# Reverse the bits of a polynomial.
+def bitreverse(poly, num_bits):
+ assert poly.bit_length() <= num_bits
+ return xor(1 << (num_bits - 1 - i) for i in range(num_bits)
+ if (poly & (1 << i)) != 0)
+
+# Format a polynomial as hex. Bit-reflect it if the CRC is LSB-first.
+def fmt_poly(variant, poly, num_bits):
+ if variant.lsb:
+ poly = bitreverse(poly, num_bits)
+ return f'0x{poly:0{2*num_bits//8}x}'
+
+# Print a comment describing constants generated for the given CRC variant.
+def print_header(variant, what):
+ print('/*')
+ s = f'{"least" if variant.lsb else "most"}-significant-bit-first CRC-{variant.bits}'
+ print(f' * {what} generated for {s} using')
+ pprint_poly(' * G(x) = ', variant.G)
+ print(' */')
+
+# Print a polynomial as hex, but drop a term if needed to keep it in 64 bits.
+def print_poly_truncate65thbit(variant, poly, num_bits, desc):
+ if num_bits > 64:
+ assert num_bits == 65
+ if variant.lsb:
+ assert (poly & 1) != 0
+ poly >>= 1
+ desc += ' - 1'
+ else:
+ poly ^= 1 << 64
+ desc += ' - x^64'
+ num_bits = 64
+ print(f'\t\t{fmt_poly(variant, poly, num_bits)},\t/* {desc} */')
+
+class CrcVariant:
+ def __init__(self, bits, generator_poly, bit_order):
+ self.bits = bits
+ if bit_order not in ['lsb', 'msb']:
+ raise ValueError('Invalid value for bit_order')
+ self.lsb = bit_order == 'lsb'
+ self.name = f'crc{bits}_{bit_order}_0x{generator_poly:0{(2*bits+7)//8}x}'
+ if self.lsb:
+ generator_poly = bitreverse(generator_poly, bits)
+ self.G = generator_poly ^ (1 << bits)
+
+# Generate tables for CRC computation using the "slice-by-N" method.
+# N=1 corresponds to the traditional byte-at-a-time table.
+def gen_slicebyN_tables(variants, n):
+ for v in variants:
+ print('')
+ print_header(v, f'Slice-by-{n} CRC table')
+ print(f'static const u{v.bits} __maybe_unused {v.name}_table[{256*n}] = {{')
+ s = ''
+ for i in range(256 * n):
+ # The i'th table entry is the CRC of the message consisting of byte
+ # i % 256 followed by i // 256 zero bytes.
+ poly = (bitreverse(i % 256, 8) if v.lsb else (i % 256)) << (v.bits + 8*(i//256))
+ next_entry = fmt_poly(v, reduce(poly, v.G), v.bits) + ','
+ if len(s + next_entry) > 71:
+ print(f'\t{s}')
+ s = ''
+ s += (' ' if s else '') + next_entry
+ if s:
+ print(f'\t{s}')
+ print('};')
+
+# Generate constants for carryless multiplication based CRC computation.
+def gen_x86_pclmul_consts(variants):
+ # These are the distances, in bits, to generate folding constants for.
+ FOLD_DISTANCES = [2048, 1024, 512, 256, 128]
+
+ for v in variants:
+ print('')
+ print_header(v, 'CRC folding constants')
+ print('static const struct {')
+ if not v.lsb:
+ print('\tu8 bswap_mask[16];')
+ for i in FOLD_DISTANCES:
+ print(f'\tu64 fold_across_{i}_bits_consts[2];')
+ print('\tu8 shuf_table[48];')
+ print('\tu64 barrett_reduction_consts[2];')
+ print(f'}} {v.name}_consts ____cacheline_aligned __maybe_unused = {{')
+
+ # Byte-reflection mask, needed for MSB CRCs
+ if not v.lsb:
+ print('\t.bswap_mask = {' + ', '.join(str(i) for i in reversed(range(16))) + '},')
+
+ # Fold constants for all distances down to 128 bits
+ k = v.bits - 65 if v.lsb else 0
+ for i in FOLD_DISTANCES:
+ print(f'\t.fold_across_{i}_bits_consts = {{')
+ for j in [64, 0] if v.lsb else [0, 64]:
+ if i + j + k == 128 and not v.lsb:
+ # Special case: for MSB CRCs, store
+ # (x^(64 + v.bits) mod G) * x^(64 - v.bits) instead of
+ # x^128 mod G. These values are congruent to each other and
+ # are equivalent for the usual folding across 128 bits, but
+ # the former value happens to also be needed during the
+ # final reduction. It generates a fold across '64 + v.bits'
+ # bits combined with a left shift by '64 - v.bits' bits.
+ const = reduce(1 << (64 + v.bits), v.G) << (64 - v.bits)
+ print(f'\t\t{fmt_poly(v, const, v.bits)},\t/* x^{64+v.bits} mod G(x) * x^{64 - v.bits} */')
+ continue
+ const = reduce(1 << (i + j + k), v.G)
+ pow_desc = f'{i}{"+" if j >= 0 else "-"}{abs(j)}'
+ if k != 0:
+ pow_desc += f'{"+" if k >= 0 else "-"}{abs(k)}'
+ print(f'\t\t{fmt_poly(v, const, v.bits)},\t/* x^({pow_desc}) mod G(x) */')
+ print('\t},')
+
+ # Shuffle table for handling 1..15 bytes at end
+ print('\t.shuf_table = {')
+ print('\t\t' + (16*'-1, ').rstrip())
+ print('\t\t' + ''.join(f'{i:2}, ' for i in range(16)).rstrip())
+ print('\t\t' + (16*'-1, ').rstrip())
+ print('\t},')
+
+ # Barrett reduction constants for reducing 128 bits to the final CRC
+ m = 63 if v.lsb else 64
+ print('\t.barrett_reduction_consts = {')
+ print_poly_truncate65thbit(v, div(1<<(m+v.bits), v.G), m+1,
+ f'floor(x^{m+v.bits} / G(x))')
+ print_poly_truncate65thbit(v, v.G, v.bits+1, 'G(x)')
+ print('\t},')
+
+ print('};')
+
+def parse_crc_variants(vars_string):
+ variants = []
+ for var_string in vars_string.split(','):
+ bits, bit_order, generator_poly = var_string.split('_')
+ assert bits.startswith('crc')
+ bits = int(bits.removeprefix('crc'))
+ assert generator_poly.startswith('0x')
+ generator_poly = generator_poly.removeprefix('0x')
+ assert len(generator_poly) % 2 == 0
+ generator_poly = int(generator_poly, 16)
+ variants.append(CrcVariant(bits, generator_poly, bit_order))
+ return variants
+
+if len(sys.argv) != 3:
+ sys.stderr.write(f'Usage: {sys.argv[0]} CONSTS_TYPE[,CONSTS_TYPE]... CRC_VARIANT[,CRC_VARIANT]...\n')
+ sys.stderr.write(' CONSTS_TYPE can be sliceby[1-8] or x86_pclmul\n')
+ sys.stderr.write(' CRC_VARIANT is crc${num_bits}_${bit_order}_${generator_poly_as_hex}\n')
+ sys.stderr.write(' E.g. crc16_msb_0x8bb7 or crc32_lsb_0xedb88320\n')
+ sys.stderr.write(' Polynomial must use the given bit_order and exclude x^{num_bits}\n')
+ sys.exit(1)
+
+print('/* SPDX-License-Identifier: GPL-2.0-or-later */')
+print('/*')
+print(' * CRC constants generated by:')
+print(' *')
+print(f' *\t{sys.argv[0]} {" ".join(sys.argv[1:])}')
+print(' *')
+print(' * Do not edit manually.')
+print(' */')
+consts_types = sys.argv[1].split(',')
+variants = parse_crc_variants(sys.argv[2])
+for consts_type in consts_types:
+ if consts_type.startswith('sliceby'):
+ gen_slicebyN_tables(variants, int(consts_type.removeprefix('sliceby')))
+ elif consts_type == 'x86_pclmul':
+ gen_x86_pclmul_consts(variants)
+ else:
+ raise ValueError(f'Unknown consts_type: {consts_type}')
--
2.48.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v2 08/11] x86/crc: add "template" for [V]PCLMULQDQ based CRC functions
2025-01-30 3:51 [PATCH v2 00/11] CRC64 library rework and x86 CRC optimization Eric Biggers
` (6 preceding siblings ...)
2025-01-30 3:51 ` [PATCH v2 07/11] scripts/gen-crc-consts: add gen-crc-consts.py Eric Biggers
@ 2025-01-30 3:51 ` Eric Biggers
2025-01-30 3:51 ` [PATCH v2 09/11] x86/crc32: implement crc32_le using new template Eric Biggers
` (5 subsequent siblings)
13 siblings, 0 replies; 21+ messages in thread
From: Eric Biggers @ 2025-01-30 3:51 UTC (permalink / raw)
To: linux-kernel
Cc: linux-crypto, x86, linux-block, Ard Biesheuvel, Keith Busch,
Kent Overstreet, Martin K . Petersen
From: Eric Biggers <ebiggers@google.com>
The Linux kernel implements many variants of CRC, such as crc16,
crc_t10dif, crc32_le, crc32c, crc32_be, crc64_nvme, and crc64_be. On
x86, except for crc32c which has special scalar instructions, the
fastest way to compute any of these CRCs on any message of length
roughly >= 16 bytes is to use the SIMD carryless multiplication
instructions PCLMULQDQ or VPCLMULQDQ. Depending on the available CPU
features this can mean PCLMULQDQ+SSE4.1, VPCLMULQDQ+AVX2,
VPCLMULQDQ+AVX10/256, or VPCLMULQDQ+AVX10/512 (or the AVX512 equivalents
to AVX10/*). This results in a total of 20+ CRC implementations being
potentially needed to properly optimize all CRCs that someone cares
about for x86. Besides crc32c, currently only crc32_le and crc_t10dif
are actually optimized for x86, and they only use PCLMULQDQ, which means
they can be 2-4x slower than what is possible with VPCLMULQDQ.
Fortunately, at a high level the code that is needed for any
[V]PCLMULQDQ based CRC implementation is mostly the same. Therefore,
this patch introduces an assembly macro that expands into the body of a
[V]PCLMULQDQ based CRC function for a given number of bits (8, 16, 32,
or 64), bit order (LSB or MSB-first), vector length, and AVX level.
The function expects to be passed a constants table, specific to the
polynomial desired, that was generated by the script previously added.
When two CRC variants share the same number of bits and bit order, the
same functions can be reused, with only the constants table differing.
A new C header is also added to make it easy to integrate the new
assembly code using a static call.
The result is that it becomes straightforward to wire up an optimized
implementation of any CRC-8, CRC-16, CRC-32, or CRC-64 for x86. Later
patches will wire up specific CRC variants.
Although this new template allows easily generating many functions, care
was taken to still keep the binary size fairly low. Each generated
function is only 550 to 850 bytes depending on the CRC variant and
target CPU features. And only one function per CRC variant is actually
used at runtime (since all functions support all lengths >= 16 bytes).
Note that a similar approach should also work for other architectures
that have carryless multiplication instructions, such as arm64.
Acked-by: Ard Biesheuvel <ardb@kernel.org>
Signed-off-by: Eric Biggers <ebiggers@google.com>
---
arch/x86/lib/crc-pclmul-template.S | 578 +++++++++++++++++++++++++++++
arch/x86/lib/crc-pclmul-template.h | 81 ++++
2 files changed, 659 insertions(+)
create mode 100644 arch/x86/lib/crc-pclmul-template.S
create mode 100644 arch/x86/lib/crc-pclmul-template.h
diff --git a/arch/x86/lib/crc-pclmul-template.S b/arch/x86/lib/crc-pclmul-template.S
new file mode 100644
index 000000000000..aeeff26a876a
--- /dev/null
+++ b/arch/x86/lib/crc-pclmul-template.S
@@ -0,0 +1,578 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+//
+// Template to generate [V]PCLMULQDQ-based CRC functions for x86
+//
+// Copyright 2025 Google LLC
+//
+// Author: Eric Biggers <ebiggers@google.com>
+
+#include <linux/linkage.h>
+
+// Offsets within the generated constants table
+.set OFFSETOF_BSWAP_MASK, -5*16 // only used for MSB CRC
+.set OFFSETOF_FOLD_ACROSS_2048BIT_CONSTS, -4*16 // must precede next
+.set OFFSETOF_FOLD_ACROSS_1024BIT_CONSTS, -3*16 // must precede next
+.set OFFSETOF_FOLD_ACROSS_512BIT_CONSTS, -2*16 // must precede next
+.set OFFSETOF_FOLD_ACROSS_256BIT_CONSTS, -1*16 // must precede next
+.set OFFSETOF_FOLD_ACROSS_128BIT_CONSTS, 0*16 // must be 0 offset
+.set OFFSETOF_SHUF_TABLE, 1*16
+.set OFFSETOF_BARRETT_REDUCTION_CONSTS, 4*16
+
+// Emit a VEX (or EVEX) coded instruction if allowed, or emulate it using the
+// corresponding non-VEX instruction plus any needed moves. The supported
+// instruction formats are:
+//
+// - Two-arg [src, dst], where the non-VEX form is the same.
+// - Three-arg [src1, src2, dst] where the non-VEX form is
+// [src1, src2_and_dst]. If src2 != dst, then src1 must != dst too.
+//
+// \insn gives the instruction without a "v" prefix and including any immediate
+// argument if needed to make the instruction follow one of the above formats.
+// If \unaligned_mem_tmp is given, then the emitted non-VEX code moves \arg1 to
+// it first; this is needed when \arg1 is an unaligned mem operand.
+.macro _cond_vex insn:req, arg1:req, arg2:req, arg3, unaligned_mem_tmp
+.if AVX_LEVEL == 0
+ // VEX not allowed. Emulate it.
+ .ifnb \arg3 // Three-arg [src1, src2, dst]
+ .ifc "\arg2", "\arg3" // src2 == dst?
+ .ifnb \unaligned_mem_tmp
+ movdqu \arg1, \unaligned_mem_tmp
+ \insn \unaligned_mem_tmp, \arg3
+ .else
+ \insn \arg1, \arg3
+ .endif
+ .else // src2 != dst
+ .ifc "\arg1", "\arg3"
+ .error "Can't have src1 == dst when src2 != dst"
+ .endif
+ .ifnb \unaligned_mem_tmp
+ movdqu \arg1, \unaligned_mem_tmp
+ movdqa \arg2, \arg3
+ \insn \unaligned_mem_tmp, \arg3
+ .else
+ movdqa \arg2, \arg3
+ \insn \arg1, \arg3
+ .endif
+ .endif
+ .else // Two-arg [src, dst]
+ .ifnb \unaligned_mem_tmp
+ movdqu \arg1, \unaligned_mem_tmp
+ \insn \unaligned_mem_tmp, \arg2
+ .else
+ \insn \arg1, \arg2
+ .endif
+ .endif
+.else
+ // VEX is allowed. Emit the desired instruction directly.
+ .ifnb \arg3
+ v\insn \arg1, \arg2, \arg3
+ .else
+ v\insn \arg1, \arg2
+ .endif
+.endif
+.endm
+
+// Broadcast an aligned 128-bit mem operand to all 128-bit lanes of a vector
+// register of length VL.
+.macro _vbroadcast src, dst
+.if VL == 16
+ _cond_vex movdqa, \src, \dst
+.elseif VL == 32
+ vbroadcasti128 \src, \dst
+.else
+ vbroadcasti32x4 \src, \dst
+.endif
+.endm
+
+// Load \vl bytes from the unaligned mem operand \src into \dst, and if the CRC
+// is MSB-first use \bswap_mask to reflect the bytes within each 128-bit lane.
+.macro _load_data vl, src, bswap_mask, dst
+.if \vl < 64
+ _cond_vex movdqu, "\src", \dst
+.else
+ vmovdqu8 \src, \dst
+.endif
+.if !LSB_CRC
+ _cond_vex pshufb, \bswap_mask, \dst, \dst
+.endif
+.endm
+
+.macro _prepare_v0 vl, v0, v1, bswap_mask
+.if LSB_CRC
+ .if \vl < 64
+ _cond_vex pxor, (BUF), \v0, \v0, unaligned_mem_tmp=\v1
+ .else
+ vpxorq (BUF), \v0, \v0
+ .endif
+.else
+ _load_data \vl, (BUF), \bswap_mask, \v1
+ .if \vl < 64
+ _cond_vex pxor, \v1, \v0, \v0
+ .else
+ vpxorq \v1, \v0, \v0
+ .endif
+.endif
+.endm
+
+// Fold \acc into \data and store the result back into \acc. \data can be an
+// unaligned mem operand if using VEX is allowed and the CRC is LSB-first so no
+// byte-reflection is needed; otherwise it must be a vector register. \consts
+// is a vector register containing the needed fold constants, and \tmp is a
+// temporary vector register. All arguments must be the same length.
+.macro _fold_vec acc, data, consts, tmp
+ _cond_vex "pclmulqdq $0x00,", \consts, \acc, \tmp
+ _cond_vex "pclmulqdq $0x11,", \consts, \acc, \acc
+.if AVX_LEVEL < 10
+ _cond_vex pxor, \data, \tmp, \tmp
+ _cond_vex pxor, \tmp, \acc, \acc
+.else
+ vpternlogq $0x96, \data, \tmp, \acc
+.endif
+.endm
+
+// Fold \acc into \data and store the result back into \acc. \data is an
+// unaligned mem operand, \consts is a vector register containing the needed
+// fold constants, \bswap_mask is a vector register containing the
+// byte-reflection table if the CRC is MSB-first, and \tmp1 and \tmp2 are
+// temporary vector registers. All arguments must have length \vl.
+.macro _fold_vec_mem vl, acc, data, consts, bswap_mask, tmp1, tmp2
+.if AVX_LEVEL == 0 || !LSB_CRC
+ _load_data \vl, \data, \bswap_mask, \tmp1
+ _fold_vec \acc, \tmp1, \consts, \tmp2
+.else
+ _fold_vec \acc, \data, \consts, \tmp1
+.endif
+.endm
+
+// Load the constants for folding across 2**i vectors of length VL at a time
+// into all 128-bit lanes of the vector register CONSTS.
+.macro _load_vec_folding_consts i
+ _vbroadcast OFFSETOF_FOLD_ACROSS_128BIT_CONSTS+(4-LOG2_VL-\i)*16(CONSTS_PTR), \
+ CONSTS
+.endm
+
+// Given vector registers \v0 and \v1 of length \vl, fold \v0 into \v1 and store
+// the result back into \v0. If the remaining length mod \vl is nonzero, also
+// fold \vl bytes from (BUF). For both operations the fold distance is \vl.
+// \consts must be a register of length \vl containing the fold constants.
+.macro _fold_vec_final vl, v0, v1, consts, bswap_mask, tmp1, tmp2
+ _fold_vec \v0, \v1, \consts, \tmp1
+ test $\vl, LEN8
+ jz .Lfold_vec_final_done\@
+ _fold_vec_mem \vl, \v0, (BUF), \consts, \bswap_mask, \tmp1, \tmp2
+ add $\vl, BUF
+.Lfold_vec_final_done\@:
+.endm
+
+// This macro generates the body of a CRC function with the following prototype:
+//
+// crc_t crc_func(crc_t crc, const u8 *buf, size_t len, const void *consts);
+//
+// |crc| is the initial CRC, and crc_t is a data type wide enough to hold it.
+// |buf| is the data to checksum. |len| is the data length in bytes, which must
+// be at least 16. |consts| is a pointer to the fold_across_128_bits_consts
+// field of the constants table that was generated for the chosen CRC variant.
+//
+// Moving onto the macro parameters, \crc_bits is the number of bits in the CRC,
+// e.g. 32 for a CRC-32. Currently the supported values are 8, 16, 32, and 64.
+// If the file is compiled in i386 mode, then the maximum supported value is 32.
+//
+// \lsb_crc is 1 if the CRC processes the least significant bit of each byte
+// first, i.e. maps bit0 to x^7, bit1 to x^6, ..., bit7 to x^0. \lsb_crc is 0
+// if the CRC processes the most significant bit of each byte first, i.e. maps
+// bit0 to x^0, bit1 to x^1, bit7 to x^7.
+//
+// \vl is the maximum length of vector register to use in bytes: 16, 32, or 64.
+//
+// \avx_level is the level of AVX support to use: 0 for SSE only, 2 for AVX2, or
+// 10 for AVX10 or AVX512.
+//
+// If \vl == 16 && \avx_level == 0, the generated code requires:
+// PCLMULQDQ && SSE4.1. (Note: all known CPUs with PCLMULQDQ also have SSE4.1.)
+//
+// If \vl == 32 && \avx_level == 2, the generated code requires:
+// VPCLMULQDQ && AVX2.
+//
+// If \vl == 32 && \avx_level == 10, the generated code requires:
+// VPCLMULQDQ && (AVX10/256 || (AVX512BW && AVX512VL))
+//
+// If \vl == 64 && \avx_level == 10, the generated code requires:
+// VPCLMULQDQ && (AVX10/512 || (AVX512BW && AVX512VL))
+//
+// Other \vl and \avx_level combinations are either not supported or not useful.
+.macro _crc_pclmul crc_bits, lsb_crc, vl, avx_level
+ .set LSB_CRC, \lsb_crc
+ .set VL, \vl
+ .set AVX_LEVEL, \avx_level
+
+ // Define aliases for the xmm, ymm, or zmm registers according to VL.
+.irp i, 0,1,2,3,4,5,6,7
+ .if VL == 16
+ .set V\i, %xmm\i
+ .set LOG2_VL, 4
+ .elseif VL == 32
+ .set V\i, %ymm\i
+ .set LOG2_VL, 5
+ .elseif VL == 64
+ .set V\i, %zmm\i
+ .set LOG2_VL, 6
+ .else
+ .error "Unsupported vector length"
+ .endif
+.endr
+ // Define aliases for the function parameters.
+ // Note: when crc_t is shorter than u32, zero-extension to 32 bits is
+ // guaranteed by the ABI. Zero-extension to 64 bits is *not* guaranteed
+ // when crc_t is shorter than u64.
+#ifdef __x86_64__
+.if \crc_bits <= 32
+ .set CRC, %edi
+.else
+ .set CRC, %rdi
+.endif
+ .set BUF, %rsi
+ .set LEN, %rdx
+ .set LEN32, %edx
+ .set LEN8, %dl
+ .set CONSTS_PTR, %rcx
+#else
+ // 32-bit support, assuming -mregparm=3 and not including support for
+ // CRC-64 (which would use both eax and edx to pass the crc parameter).
+ .set CRC, %eax
+ .set BUF, %edx
+ .set LEN, %ecx
+ .set LEN32, %ecx
+ .set LEN8, %cl
+ .set CONSTS_PTR, %ebx // Passed on stack
+#endif
+
+ // Define aliases for some local variables. V0-V5 are used without
+ // aliases (for accumulators, data, temporary values, etc). Staying
+ // within the first 8 vector registers keeps the code 32-bit SSE
+ // compatible and reduces the size of 64-bit SSE code slightly.
+ .set BSWAP_MASK, V6
+ .set BSWAP_MASK_YMM, %ymm6
+ .set BSWAP_MASK_XMM, %xmm6
+ .set CONSTS, V7
+ .set CONSTS_YMM, %ymm7
+ .set CONSTS_XMM, %xmm7
+
+#ifdef __i386__
+ push CONSTS_PTR
+ mov 8(%esp), CONSTS_PTR
+#endif
+
+ // Create a 128-bit vector that contains the initial CRC in the end
+ // representing the high-order polynomial coefficients, and the rest 0.
+.if \crc_bits <= 32
+ _cond_vex movd, CRC, %xmm0
+.else
+ _cond_vex movq, CRC, %xmm0
+.endif
+.if !LSB_CRC
+ _cond_vex pslldq, $(128-\crc_bits)/8, %xmm0, %xmm0
+ _vbroadcast OFFSETOF_BSWAP_MASK(CONSTS_PTR), BSWAP_MASK
+.endif
+
+ // Load the first vector of data and XOR the initial CRC into the
+ // appropriate end of the first 128-bit lane of data. If LEN < VL, then
+ // use a short vector and jump to the end to do the final reduction.
+ // (LEN >= 16 is guaranteed here but not necessarily LEN >= VL.)
+.if VL >= 32
+ cmp $VL, LEN
+ jae 2f
+ .if VL == 64
+ cmp $32, LEN32
+ jb 1f
+ _prepare_v0 32, %ymm0, %ymm1, BSWAP_MASK_YMM
+ add $32, BUF
+ jmp .Lreduce_256bits_to_128bits\@
+1:
+ .endif
+ _prepare_v0 16, %xmm0, %xmm1, BSWAP_MASK_XMM
+ add $16, BUF
+ vmovdqa OFFSETOF_FOLD_ACROSS_128BIT_CONSTS(CONSTS_PTR), CONSTS_XMM
+ jmp .Lcheck_for_partial_block\@
+2:
+.endif
+ _prepare_v0 VL, V0, V1, BSWAP_MASK
+
+ // Handle VL <= LEN < 4*VL.
+ cmp $4*VL-1, LEN
+ ja .Lfold_4vecs_prepare\@
+ add $VL, BUF
+ // If VL <= LEN < 2*VL, then jump to the code at the end that handles
+ // the reduction from 1 vector. If VL==16 then
+ // fold_across_128bit_consts must be loaded first, as the final
+ // reduction depends on it and it won't be loaded anywhere else.
+ cmp $2*VL-1, LEN32
+.if VL == 16
+ _cond_vex movdqa, OFFSETOF_FOLD_ACROSS_128BIT_CONSTS(CONSTS_PTR), CONSTS_XMM
+.endif
+ jbe .Lreduce_1vec_to_128bits\@
+ // Otherwise 2*VL <= LEN < 4*VL. Load one more vector and jump to the
+ // code at the end that handles the reduction from 2 vectors.
+ _load_data VL, (BUF), BSWAP_MASK, V1
+ add $VL, BUF
+ jmp .Lreduce_2vecs_to_1\@
+
+.Lfold_4vecs_prepare\@:
+ // Load 3 more vectors of data.
+ _load_data VL, 1*VL(BUF), BSWAP_MASK, V1
+ _load_data VL, 2*VL(BUF), BSWAP_MASK, V2
+ _load_data VL, 3*VL(BUF), BSWAP_MASK, V3
+ sub $-4*VL, BUF // Shorter than 'add 4*VL' when VL=32
+ add $-4*VL, LEN // Shorter than 'sub 4*VL' when VL=32
+
+ // While >= 4 vectors of data remain, fold the 4 vectors V0-V3 into the
+ // next 4 vectors of data and write the result back to V0-V3.
+ cmp $4*VL-1, LEN // Shorter than 'cmp 4*VL' when VL=32
+ jbe .Lreduce_4vecs_to_2\@
+ _load_vec_folding_consts 2
+.Lfold_4vecs_loop\@:
+ _fold_vec_mem VL, V0, 0*VL(BUF), CONSTS, BSWAP_MASK, V4, V5
+ _fold_vec_mem VL, V1, 1*VL(BUF), CONSTS, BSWAP_MASK, V4, V5
+ _fold_vec_mem VL, V2, 2*VL(BUF), CONSTS, BSWAP_MASK, V4, V5
+ _fold_vec_mem VL, V3, 3*VL(BUF), CONSTS, BSWAP_MASK, V4, V5
+ sub $-4*VL, BUF
+ add $-4*VL, LEN
+ cmp $4*VL-1, LEN
+ ja .Lfold_4vecs_loop\@
+
+ // Fold V0,V1 into V2,V3 and write the result back to V0,V1.
+ // Then fold two vectors of data, if at least that much remains.
+.Lreduce_4vecs_to_2\@:
+ _load_vec_folding_consts 1
+ _fold_vec V0, V2, CONSTS, V4
+ _fold_vec V1, V3, CONSTS, V4
+ test $2*VL, LEN8
+ jz .Lreduce_2vecs_to_1\@
+ _fold_vec_mem VL, V0, 0*VL(BUF), CONSTS, BSWAP_MASK, V4, V5
+ _fold_vec_mem VL, V1, 1*VL(BUF), CONSTS, BSWAP_MASK, V4, V5
+ sub $-2*VL, BUF
+
+ // Fold V0 into V1 and write the result back to V0.
+ // Then fold one vector of data, if at least that much remains.
+.Lreduce_2vecs_to_1\@:
+ _load_vec_folding_consts 0
+ _fold_vec_final VL, V0, V1, CONSTS, BSWAP_MASK, V4, V5
+
+.Lreduce_1vec_to_128bits\@:
+ // Reduce V0 to 128 bits xmm0.
+.if VL == 64
+ // zmm0 => ymm0
+ vbroadcasti128 OFFSETOF_FOLD_ACROSS_256BIT_CONSTS(CONSTS_PTR), CONSTS_YMM
+ vextracti64x4 $1, %zmm0, %ymm1
+ _fold_vec_final 32, %ymm0, %ymm1, CONSTS_YMM, BSWAP_MASK_YMM, %ymm4, %ymm5
+.endif
+.if VL >= 32
+.Lreduce_256bits_to_128bits\@:
+ // ymm0 => xmm0
+ vmovdqa OFFSETOF_FOLD_ACROSS_128BIT_CONSTS(CONSTS_PTR), CONSTS_XMM
+ vextracti128 $1, %ymm0, %xmm1
+ _fold_vec_final 16, %xmm0, %xmm1, CONSTS_XMM, BSWAP_MASK_XMM, %xmm4, %xmm5
+.endif
+
+.Lcheck_for_partial_block\@:
+ and $15, LEN32
+ jz .Lpartial_block_done\@
+
+ // 1 <= LEN <= 15 data bytes remain. The polynomial is now
+ // A*(x^(8*LEN)) + B, where A = xmm0 and B is the polynomial of the
+ // remaining LEN bytes. To reduce this to 128 bits without needing fold
+ // constants for each possible LEN, rearrange this expression into
+ // C1*(x^128) + C2, where C1 = floor(A / x^(128 - 8*LEN)) and
+ // C2 = A*x^(8*LEN) + B mod x^128. Then fold C1 into C2, which is just
+ // another fold across 128 bits.
+
+.if !LSB_CRC || AVX_LEVEL == 0
+ // Load the last 16 bytes.
+ _load_data 16, "-16(BUF,LEN)", BSWAP_MASK_XMM, %xmm1
+.endif // Else will use vpblendvb mem operand later.
+.if !LSB_CRC
+ neg LEN // Needed for indexing shuf_table
+.endif
+
+ // tmp = A*x^(8*LEN) mod x^128
+ // LSB CRC: pshufb by [LEN, LEN+1, ..., 15, -1, -1, ..., -1]
+ // i.e. right-shift by LEN bytes.
+ // MSB CRC: pshufb by [-1, -1, ..., -1, 0, 1, ..., 15-LEN]
+ // i.e. left-shift by LEN bytes.
+ _cond_vex movdqu, "OFFSETOF_SHUF_TABLE+16(CONSTS_PTR,LEN)", %xmm3
+ _cond_vex pshufb, %xmm3, %xmm0, %xmm2
+
+ // C1 = floor(A / x^(128 - 8*LEN))
+ // LSB CRC: pshufb by [-1, -1, ..., -1, 0, 1, ..., LEN-1]
+ // i.e. left-shift by 16-LEN bytes.
+ // MSB CRC: pshufb by [16-LEN, 16-LEN+1, ..., 15, -1, -1, ..., -1]
+ // i.e. right-shift by 16-LEN bytes.
+ _cond_vex pshufb, "OFFSETOF_SHUF_TABLE+32*!LSB_CRC(CONSTS_PTR,LEN)", \
+ %xmm0, %xmm0, unaligned_mem_tmp=%xmm4
+
+ // C2 = tmp + B
+ // LSB CRC: blend 1=last16bytes,0=tmp by [LEN, LEN+1, ..., 15, -1, -1, ..., -1]
+ // MSB CRC: blend 1=last16bytes,0=tmp by [16-LEN, 16-LEN+1, ..., 15, -1, -1, ..., -1]
+.if AVX_LEVEL == 0
+ movdqa %xmm0, %xmm4
+ movdqa %xmm3, %xmm0
+ pblendvb %xmm1, %xmm2 // uses %xmm0 as implicit operand
+ movdqa %xmm4, %xmm0
+.else
+ .if LSB_CRC
+ vpblendvb %xmm3, -16(BUF,LEN), %xmm2, %xmm2
+ .else
+ vpblendvb %xmm3, %xmm1, %xmm2, %xmm2
+ .endif
+.endif
+
+ // Fold C1 into C2 and store the 128-bit result in xmm0.
+ _fold_vec %xmm0, %xmm2, CONSTS_XMM, %xmm4
+
+.Lpartial_block_done\@:
+ // Compute the final CRC 'A * x^n mod G', where n=\crc_bits and A
+ // denotes the 128-bit polynomial stored in xmm0. CONSTS_XMM contains
+ // fold_across_128_bits_consts.
+
+ // First, multiply by x^n and reduce to 64+n bits:
+ //
+ // t0 := ((x^(64+n) mod G) * floor(A / x^64)) + (x^n * (A mod x^64))
+ //
+ // Store the resulting 64+n bit polynomial t0 in the physically low bits
+ // of xmm0 for LSB-first CRCs (with the physically high 64-n bits
+ // zeroed), or in the physically high bits of xmm0 for MSB-first CRCs
+ // (with the physically low 64-n bits zeroed). In the MSB-first case,
+ // accomplish this by multiplying by an extra factor of x^(64-n). This
+ // allows the next pclmulqdq to easily select floor(t0 / x^n), i.e. the
+ // high 64 terms of t0, without needing an extra shift to prepare it.
+.if LSB_CRC
+ _cond_vex "pclmulqdq $0x10,", CONSTS_XMM, %xmm0, %xmm1
+ _cond_vex psrldq, $8, %xmm0, %xmm0
+ _cond_vex pxor, %xmm1, %xmm0, %xmm0
+.else
+ _cond_vex "pclmulqdq $0x01,", CONSTS_XMM, %xmm0, %xmm1
+ _cond_vex pslldq, $8, %xmm0, %xmm0
+ _cond_vex pxor, %xmm1, %xmm0, %xmm0
+.endif
+
+ // Compute floor(t0 / G). This is the polynomial by which G needs to be
+ // multiplied to cancel out the x^n and higher terms of t0. First do:
+ //
+ // t1 := floor(x^(m+n) / G) * floor(t0 / x^n)
+ //
+ // Then the desired value floor(t0 / G) is floor(t1 / x^m).
+ //
+ // m >= 63 is required, since floor(t0 / G) has max degree 63. For
+ // LSB-first CRCs, use m=63. floor(x^(m+n) / G) then has degree 63 and
+ // the multiplication is of two 64-bit values producing a 127-bit result
+ // t1 in bits 0..126 of xmm1 using LSB-first order; the physically low
+ // qword of xmm1 then contains the desired value floor(t1 / x^63).
+ //
+ // For MSB-first CRCs, use m=64. m=63 would cause the desired value
+ // floor(t1 / x^63) to be in bits 63..126, which crosses qwords and
+ // would need several instructions to fix. Instead, use m=64 to get
+ // this value in bits 64..127 instead. This does mean that the
+ // multiplication becomes a 65-bit by 64-bit multiplication, which
+ // requires an extra XOR to handle multiplying by the x^64 term.
+ _cond_vex movdqa, OFFSETOF_BARRETT_REDUCTION_CONSTS(CONSTS_PTR), CONSTS_XMM
+.if LSB_CRC
+ _cond_vex "pclmulqdq $0x00,", CONSTS_XMM, %xmm0, %xmm1
+.else
+ _cond_vex "pclmulqdq $0x01,", CONSTS_XMM, %xmm0, %xmm1
+ _cond_vex pxor, %xmm0, %xmm1, %xmm1
+.endif
+
+ // Cancel out the x^n and higher terms of t0 by subtracting the needed
+ // multiple of G. This gives the final CRC:
+ //
+ // crc := t0 - (G * floor(t1 / x^m))
+.if LSB_CRC
+ // For LSB-first CRCs, floor(t1 / x^m) is in bits 0..63 of xmm1.
+ // Multiplying by the n+1 bit constant G produces a 64+n bit polynomial
+ // in physical bits 0..(64+n-1) of xmm1, aligned with t0 which is a 64+n
+ // bit polynomial in physical bits 0..(64+n-1) of xmm0. XOR'ing in t0
+ // cancels out the x^n and above terms, leaving the final CRC in
+ // physical bits 64..(64+n-1), with the higher bits guaranteed be zero.
+ //
+ // When n=64, G is 65-bit and the multiplication has to be broken up
+ // into a 64-bit part and a 1-bit part. The pclmulqdq handles
+ // multiplying by the physically low 64 bits of G which represent terms
+ // x^1 through x^64. The multiplication by physical bit 64 of G, which
+ // represents the x^0 term and which we assume is nonzero, is handled
+ // separately using a shuffle and an XOR.
+ //
+ // Note that the x^64 term of G does not affect the result, so an
+ // alternative would be to just multiply by the x^0 through x^63 terms.
+ // But that would leave the result in bits 63..(63+n-1) which is
+ // misaligned, and several instructions would be needed to fix it up.
+ .if \crc_bits == 64
+ _cond_vex punpcklqdq, %xmm1, %xmm1, %xmm2
+ _cond_vex "pclmulqdq $0x10,", CONSTS_XMM, %xmm1, %xmm1
+ .if AVX_LEVEL < 10
+ _cond_vex pxor, %xmm1, %xmm0, %xmm0
+ _cond_vex pxor, %xmm2, %xmm0, %xmm0
+ .else
+ vpternlogq $0x96, %xmm2, %xmm1, %xmm0
+ .endif
+ .else
+ _cond_vex "pclmulqdq $0x10,", CONSTS_XMM, %xmm1, %xmm1
+ _cond_vex pxor, %xmm1, %xmm0, %xmm0
+ .endif
+ .if \crc_bits <= 32
+ _cond_vex "pextrd $2,", %xmm0, %eax
+ .else
+ _cond_vex "pextrq $1,", %xmm0, %rax
+ .endif
+.else
+ // This is the same calculation crc := t0 - (G * floor(t1 / x^m)), but
+ // now in the MSB-first case. In this case the result is generated in
+ // bits 0..(n-1), where bits n..63 are guaranteed to be zero. Bit 64 of
+ // G (present when n=64) does not affect the result, so it is ignored
+ // and the multiplication is by just the x^0 through x^63 terms of G.
+ //
+ // When n < 64, a right-shift by 64-n bits is needed to move the low n
+ // terms of t0 into the proper position, considering the extra
+ // multiplication by x^(64-n) that was done earlier.
+ .if \crc_bits < 64
+ _cond_vex psrlq $64-\crc_bits, %xmm0, %xmm0
+ .endif
+ _cond_vex "pclmulqdq $0x11,", CONSTS_XMM, %xmm1, %xmm1
+ _cond_vex pxor, %xmm1, %xmm0, %xmm0
+ .if \crc_bits <= 32
+ _cond_vex movd, %xmm0, %eax
+ .else
+ _cond_vex movq, %xmm0, %rax
+ .endif
+.endif
+
+.if VL > 16
+ vzeroupper // Needed when ymm or zmm registers may have been used.
+.endif
+#ifdef __i386__
+ pop CONSTS_PTR
+#endif
+ RET
+.endm
+
+#ifdef CONFIG_AS_VPCLMULQDQ
+#define DEFINE_CRC_PCLMUL_FUNCS(prefix, bits, lsb) \
+SYM_FUNC_START(prefix##_pclmul_sse); \
+ _crc_pclmul crc_bits=bits, lsb_crc=lsb, vl=16, avx_level=0; \
+SYM_FUNC_END(prefix##_pclmul_sse); \
+ \
+SYM_FUNC_START(prefix##_vpclmul_avx2); \
+ _crc_pclmul crc_bits=bits, lsb_crc=lsb, vl=32, avx_level=2; \
+SYM_FUNC_END(prefix##_vpclmul_avx2); \
+ \
+SYM_FUNC_START(prefix##_vpclmul_avx10_256); \
+ _crc_pclmul crc_bits=bits, lsb_crc=lsb, vl=32, avx_level=10;\
+SYM_FUNC_END(prefix##_vpclmul_avx10_256); \
+ \
+SYM_FUNC_START(prefix##_vpclmul_avx10_512); \
+ _crc_pclmul crc_bits=bits, lsb_crc=lsb, vl=64, avx_level=10;\
+SYM_FUNC_END(prefix##_vpclmul_avx10_512);
+#else
+#define DEFINE_CRC_PCLMUL_FUNCS(prefix, bits, lsb) \
+SYM_FUNC_START(prefix##_pclmul_sse); \
+ _crc_pclmul crc_bits=bits, lsb_crc=lsb, vl=16, avx_level=0; \
+SYM_FUNC_END(prefix##_pclmul_sse);
+#endif // !CONFIG_AS_VPCLMULQDQ
diff --git a/arch/x86/lib/crc-pclmul-template.h b/arch/x86/lib/crc-pclmul-template.h
new file mode 100644
index 000000000000..7b89f0edbc17
--- /dev/null
+++ b/arch/x86/lib/crc-pclmul-template.h
@@ -0,0 +1,81 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Macros for accessing the [V]PCLMULQDQ-based CRC functions that are
+ * instantiated by crc-pclmul-template.S
+ *
+ * Copyright 2025 Google LLC
+ *
+ * Author: Eric Biggers <ebiggers@google.com>
+ */
+#ifndef _CRC_PCLMUL_TEMPLATE_H
+#define _CRC_PCLMUL_TEMPLATE_H
+
+#include <asm/cpufeatures.h>
+#include <asm/simd.h>
+#include <crypto/internal/simd.h>
+#include <linux/static_call.h>
+#include "crc-pclmul-consts.h"
+
+#define DECLARE_CRC_PCLMUL_FUNCS(prefix, crc_t) \
+crc_t prefix##_pclmul_sse(crc_t crc, const u8 *p, size_t len, \
+ const void *consts_ptr); \
+crc_t prefix##_vpclmul_avx2(crc_t crc, const u8 *p, size_t len, \
+ const void *consts_ptr); \
+crc_t prefix##_vpclmul_avx10_256(crc_t crc, const u8 *p, size_t len, \
+ const void *consts_ptr); \
+crc_t prefix##_vpclmul_avx10_512(crc_t crc, const u8 *p, size_t len, \
+ const void *consts_ptr); \
+DEFINE_STATIC_CALL(prefix##_pclmul, prefix##_pclmul_sse)
+
+#define INIT_CRC_PCLMUL(prefix) \
+do { \
+ if (IS_ENABLED(CONFIG_AS_VPCLMULQDQ) && \
+ boot_cpu_has(X86_FEATURE_VPCLMULQDQ) && \
+ boot_cpu_has(X86_FEATURE_AVX2) && \
+ cpu_has_xfeatures(XFEATURE_MASK_YMM, NULL)) { \
+ if (boot_cpu_has(X86_FEATURE_AVX512BW) && \
+ boot_cpu_has(X86_FEATURE_AVX512VL) && \
+ cpu_has_xfeatures(XFEATURE_MASK_AVX512, NULL)) { \
+ if (boot_cpu_has(X86_FEATURE_PREFER_YMM)) \
+ static_call_update(prefix##_pclmul, \
+ prefix##_vpclmul_avx10_256); \
+ else \
+ static_call_update(prefix##_pclmul, \
+ prefix##_vpclmul_avx10_512); \
+ } else { \
+ static_call_update(prefix##_pclmul, \
+ prefix##_vpclmul_avx2); \
+ } \
+ } \
+} while (0)
+
+/*
+ * Call a [V]PCLMULQDQ optimized CRC function if the data length is at least 16
+ * bytes, the CPU has PCLMULQDQ support, and the current context may use SIMD.
+ *
+ * 16 bytes is the minimum length supported by the [V]PCLMULQDQ functions.
+ * There is overhead associated with kernel_fpu_begin() and kernel_fpu_end(),
+ * varying by CPU and factors such as which parts of the "FPU" state userspace
+ * has touched, which could result in a larger cutoff being better. Indeed, a
+ * larger cutoff is usually better for a *single* message. However, the
+ * overhead of the FPU section gets amortized if multiple FPU sections get
+ * executed before returning to userspace, since the XSAVE and XRSTOR occur only
+ * once. Considering that and the fact that the [V]PCLMULQDQ code is lighter on
+ * the dcache than the table-based code is, a 16-byte cutoff seems to work well.
+ */
+#define CRC_PCLMUL(crc, p, len, prefix, consts, have_pclmulqdq) \
+do { \
+ if ((len) >= 16 && static_branch_likely(&(have_pclmulqdq)) && \
+ crypto_simd_usable()) { \
+ const void *consts_ptr; \
+ \
+ consts_ptr = (consts).fold_across_128_bits_consts; \
+ kernel_fpu_begin(); \
+ crc = static_call(prefix##_pclmul)((crc), (p), (len), \
+ consts_ptr); \
+ kernel_fpu_end(); \
+ return crc; \
+ } \
+} while (0)
+
+#endif /* _CRC_PCLMUL_TEMPLATE_H */
--
2.48.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v2 09/11] x86/crc32: implement crc32_le using new template
2025-01-30 3:51 [PATCH v2 00/11] CRC64 library rework and x86 CRC optimization Eric Biggers
` (7 preceding siblings ...)
2025-01-30 3:51 ` [PATCH v2 08/11] x86/crc: add "template" for [V]PCLMULQDQ based CRC functions Eric Biggers
@ 2025-01-30 3:51 ` Eric Biggers
2025-01-30 3:51 ` [PATCH v2 10/11] x86/crc-t10dif: implement crc_t10dif " Eric Biggers
` (4 subsequent siblings)
13 siblings, 0 replies; 21+ messages in thread
From: Eric Biggers @ 2025-01-30 3:51 UTC (permalink / raw)
To: linux-kernel
Cc: linux-crypto, x86, linux-block, Ard Biesheuvel, Keith Busch,
Kent Overstreet, Martin K . Petersen
From: Eric Biggers <ebiggers@google.com>
Instantiate crc-pclmul-template.S for crc32_le, and delete the original
PCLMULQDQ optimized implementation. This has the following advantages:
- Less CRC-variant-specific code.
- VPCLMULQDQ support, greatly improving performance on sufficiently long
messages on newer CPUs.
- A faster reduction from 128 bits to the final CRC.
- Support for lengths not a multiple of 16 bytes, improving performance
for such lengths.
- Support for misaligned buffers, improving performance in such cases.
Benchmark results on AMD Ryzen 9 9950X (Zen 5) using crc_kunit:
Length Before After
------ ------ -----
1 427 MB/s 605 MB/s
16 710 MB/s 3631 MB/s
64 704 MB/s 7615 MB/s
127 3610 MB/s 9710 MB/s
128 8759 MB/s 12702 MB/s
200 7083 MB/s 15343 MB/s
256 17284 MB/s 22904 MB/s
511 10919 MB/s 27309 MB/s
512 19849 MB/s 48900 MB/s
1024 21216 MB/s 62630 MB/s
3173 22150 MB/s 72437 MB/s
4096 22496 MB/s 79593 MB/s
16384 22018 MB/s 85106 MB/s
Acked-by: Ard Biesheuvel <ardb@kernel.org>
Signed-off-by: Eric Biggers <ebiggers@google.com>
---
arch/x86/lib/crc-pclmul-consts.h | 53 ++++++++
arch/x86/lib/crc32-glue.c | 37 ++----
arch/x86/lib/crc32-pclmul.S | 219 +------------------------------
3 files changed, 65 insertions(+), 244 deletions(-)
create mode 100644 arch/x86/lib/crc-pclmul-consts.h
diff --git a/arch/x86/lib/crc-pclmul-consts.h b/arch/x86/lib/crc-pclmul-consts.h
new file mode 100644
index 000000000000..ee22cf221c35
--- /dev/null
+++ b/arch/x86/lib/crc-pclmul-consts.h
@@ -0,0 +1,53 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * CRC constants generated by:
+ *
+ * ./scripts/gen-crc-consts.py x86_pclmul crc32_lsb_0xedb88320
+ *
+ * Do not edit manually.
+ */
+
+/*
+ * CRC folding constants generated for least-significant-bit-first CRC-32 using
+ * G(x) = x^32 + x^26 + x^23 + x^22 + x^16 + x^12 + x^11 + x^10 + x^8 + x^7 +
+ * x^5 + x^4 + x^2 + x + 1
+ */
+static const struct {
+ u64 fold_across_2048_bits_consts[2];
+ u64 fold_across_1024_bits_consts[2];
+ u64 fold_across_512_bits_consts[2];
+ u64 fold_across_256_bits_consts[2];
+ u64 fold_across_128_bits_consts[2];
+ u8 shuf_table[48];
+ u64 barrett_reduction_consts[2];
+} crc32_lsb_0xedb88320_consts ____cacheline_aligned __maybe_unused = {
+ .fold_across_2048_bits_consts = {
+ 0xce3371cb, /* x^(2048+64-33) mod G(x) */
+ 0xe95c1271, /* x^(2048+0-33) mod G(x) */
+ },
+ .fold_across_1024_bits_consts = {
+ 0x33fff533, /* x^(1024+64-33) mod G(x) */
+ 0x910eeec1, /* x^(1024+0-33) mod G(x) */
+ },
+ .fold_across_512_bits_consts = {
+ 0x8f352d95, /* x^(512+64-33) mod G(x) */
+ 0x1d9513d7, /* x^(512+0-33) mod G(x) */
+ },
+ .fold_across_256_bits_consts = {
+ 0xf1da05aa, /* x^(256+64-33) mod G(x) */
+ 0x81256527, /* x^(256+0-33) mod G(x) */
+ },
+ .fold_across_128_bits_consts = {
+ 0xae689191, /* x^(128+64-33) mod G(x) */
+ 0xccaa009e, /* x^(128+0-33) mod G(x) */
+ },
+ .shuf_table = {
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ },
+ .barrett_reduction_consts = {
+ 0xb4e5b025f7011641, /* floor(x^95 / G(x)) */
+ 0x1db710641, /* G(x) */
+ },
+};
diff --git a/arch/x86/lib/crc32-glue.c b/arch/x86/lib/crc32-glue.c
index 2dd18a886ded..1a579a4fcded 100644
--- a/arch/x86/lib/crc32-glue.c
+++ b/arch/x86/lib/crc32-glue.c
@@ -5,47 +5,24 @@
* Copyright (C) 2008 Intel Corporation
* Copyright 2012 Xyratex Technology Limited
* Copyright 2024 Google LLC
*/
-#include <asm/cpufeatures.h>
-#include <asm/simd.h>
-#include <crypto/internal/simd.h>
#include <linux/crc32.h>
-#include <linux/linkage.h>
#include <linux/module.h>
-
-/* minimum size of buffer for crc32_pclmul_le_16 */
-#define CRC32_PCLMUL_MIN_LEN 64
+#include "crc-pclmul-template.h"
static DEFINE_STATIC_KEY_FALSE(have_crc32);
static DEFINE_STATIC_KEY_FALSE(have_pclmulqdq);
-u32 crc32_pclmul_le_16(u32 crc, const u8 *buffer, size_t len);
+DECLARE_CRC_PCLMUL_FUNCS(crc32_lsb, u32);
u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
{
- if (len >= CRC32_PCLMUL_MIN_LEN + 15 &&
- static_branch_likely(&have_pclmulqdq) && crypto_simd_usable()) {
- size_t n = -(uintptr_t)p & 15;
-
- /* align p to 16-byte boundary */
- if (n) {
- crc = crc32_le_base(crc, p, n);
- p += n;
- len -= n;
- }
- n = round_down(len, 16);
- kernel_fpu_begin();
- crc = crc32_pclmul_le_16(crc, p, n);
- kernel_fpu_end();
- p += n;
- len -= n;
- }
- if (len)
- crc = crc32_le_base(crc, p, len);
- return crc;
+ CRC_PCLMUL(crc, p, len, crc32_lsb, crc32_lsb_0xedb88320_consts,
+ have_pclmulqdq);
+ return crc32_le_base(crc, p, len);
}
EXPORT_SYMBOL(crc32_le_arch);
#ifdef CONFIG_X86_64
#define CRC32_INST "crc32q %1, %q0"
@@ -95,12 +72,14 @@ EXPORT_SYMBOL(crc32_be_arch);
static int __init crc32_x86_init(void)
{
if (boot_cpu_has(X86_FEATURE_XMM4_2))
static_branch_enable(&have_crc32);
- if (boot_cpu_has(X86_FEATURE_PCLMULQDQ))
+ if (boot_cpu_has(X86_FEATURE_PCLMULQDQ)) {
static_branch_enable(&have_pclmulqdq);
+ INIT_CRC_PCLMUL(crc32_lsb);
+ }
return 0;
}
arch_initcall(crc32_x86_init);
static void __exit crc32_x86_exit(void)
diff --git a/arch/x86/lib/crc32-pclmul.S b/arch/x86/lib/crc32-pclmul.S
index f9637789cac1..f20f40fb0172 100644
--- a/arch/x86/lib/crc32-pclmul.S
+++ b/arch/x86/lib/crc32-pclmul.S
@@ -1,217 +1,6 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright 2012 Xyratex Technology Limited
- *
- * Using hardware provided PCLMULQDQ instruction to accelerate the CRC32
- * calculation.
- * CRC32 polynomial:0x04c11db7(BE)/0xEDB88320(LE)
- * PCLMULQDQ is a new instruction in Intel SSE4.2, the reference can be found
- * at:
- * http://www.intel.com/products/processor/manuals/
- * Intel(R) 64 and IA-32 Architectures Software Developer's Manual
- * Volume 2B: Instruction Set Reference, N-Z
- *
- * Authors: Gregory Prestas <Gregory_Prestas@us.xyratex.com>
- * Alexander Boyko <Alexander_Boyko@xyratex.com>
- */
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+// Copyright 2025 Google LLC
-#include <linux/linkage.h>
+#include "crc-pclmul-template.S"
-
-.section .rodata
-.align 16
-/*
- * [x4*128+32 mod P(x) << 32)]' << 1 = 0x154442bd4
- * #define CONSTANT_R1 0x154442bd4LL
- *
- * [(x4*128-32 mod P(x) << 32)]' << 1 = 0x1c6e41596
- * #define CONSTANT_R2 0x1c6e41596LL
- */
-.Lconstant_R2R1:
- .octa 0x00000001c6e415960000000154442bd4
-/*
- * [(x128+32 mod P(x) << 32)]' << 1 = 0x1751997d0
- * #define CONSTANT_R3 0x1751997d0LL
- *
- * [(x128-32 mod P(x) << 32)]' << 1 = 0x0ccaa009e
- * #define CONSTANT_R4 0x0ccaa009eLL
- */
-.Lconstant_R4R3:
- .octa 0x00000000ccaa009e00000001751997d0
-/*
- * [(x64 mod P(x) << 32)]' << 1 = 0x163cd6124
- * #define CONSTANT_R5 0x163cd6124LL
- */
-.Lconstant_R5:
- .octa 0x00000000000000000000000163cd6124
-.Lconstant_mask32:
- .octa 0x000000000000000000000000FFFFFFFF
-/*
- * #define CRCPOLY_TRUE_LE_FULL 0x1DB710641LL
- *
- * Barrett Reduction constant (u64`) = u` = (x**64 / P(x))` = 0x1F7011641LL
- * #define CONSTANT_RU 0x1F7011641LL
- */
-.Lconstant_RUpoly:
- .octa 0x00000001F701164100000001DB710641
-
-#define CONSTANT %xmm0
-
-#ifdef __x86_64__
-#define CRC %edi
-#define BUF %rsi
-#define LEN %rdx
-#else
-#define CRC %eax
-#define BUF %edx
-#define LEN %ecx
-#endif
-
-
-
-.text
-/**
- * Calculate crc32
- * CRC - initial crc32
- * BUF - buffer (16 bytes aligned)
- * LEN - sizeof buffer (16 bytes aligned), LEN should be greater than 63
- * return %eax crc32
- * u32 crc32_pclmul_le_16(u32 crc, const u8 *buffer, size_t len);
- */
-
-SYM_FUNC_START(crc32_pclmul_le_16) /* buffer and buffer size are 16 bytes aligned */
- movdqa (BUF), %xmm1
- movdqa 0x10(BUF), %xmm2
- movdqa 0x20(BUF), %xmm3
- movdqa 0x30(BUF), %xmm4
- movd CRC, CONSTANT
- pxor CONSTANT, %xmm1
- sub $0x40, LEN
- add $0x40, BUF
- cmp $0x40, LEN
- jb .Lless_64
-
-#ifdef __x86_64__
- movdqa .Lconstant_R2R1(%rip), CONSTANT
-#else
- movdqa .Lconstant_R2R1, CONSTANT
-#endif
-
-.Lloop_64:/* 64 bytes Full cache line folding */
- prefetchnta 0x40(BUF)
- movdqa %xmm1, %xmm5
- movdqa %xmm2, %xmm6
- movdqa %xmm3, %xmm7
-#ifdef __x86_64__
- movdqa %xmm4, %xmm8
-#endif
- pclmulqdq $0x00, CONSTANT, %xmm1
- pclmulqdq $0x00, CONSTANT, %xmm2
- pclmulqdq $0x00, CONSTANT, %xmm3
-#ifdef __x86_64__
- pclmulqdq $0x00, CONSTANT, %xmm4
-#endif
- pclmulqdq $0x11, CONSTANT, %xmm5
- pclmulqdq $0x11, CONSTANT, %xmm6
- pclmulqdq $0x11, CONSTANT, %xmm7
-#ifdef __x86_64__
- pclmulqdq $0x11, CONSTANT, %xmm8
-#endif
- pxor %xmm5, %xmm1
- pxor %xmm6, %xmm2
- pxor %xmm7, %xmm3
-#ifdef __x86_64__
- pxor %xmm8, %xmm4
-#else
- /* xmm8 unsupported for x32 */
- movdqa %xmm4, %xmm5
- pclmulqdq $0x00, CONSTANT, %xmm4
- pclmulqdq $0x11, CONSTANT, %xmm5
- pxor %xmm5, %xmm4
-#endif
-
- pxor (BUF), %xmm1
- pxor 0x10(BUF), %xmm2
- pxor 0x20(BUF), %xmm3
- pxor 0x30(BUF), %xmm4
-
- sub $0x40, LEN
- add $0x40, BUF
- cmp $0x40, LEN
- jge .Lloop_64
-.Lless_64:/* Folding cache line into 128bit */
-#ifdef __x86_64__
- movdqa .Lconstant_R4R3(%rip), CONSTANT
-#else
- movdqa .Lconstant_R4R3, CONSTANT
-#endif
- prefetchnta (BUF)
-
- movdqa %xmm1, %xmm5
- pclmulqdq $0x00, CONSTANT, %xmm1
- pclmulqdq $0x11, CONSTANT, %xmm5
- pxor %xmm5, %xmm1
- pxor %xmm2, %xmm1
-
- movdqa %xmm1, %xmm5
- pclmulqdq $0x00, CONSTANT, %xmm1
- pclmulqdq $0x11, CONSTANT, %xmm5
- pxor %xmm5, %xmm1
- pxor %xmm3, %xmm1
-
- movdqa %xmm1, %xmm5
- pclmulqdq $0x00, CONSTANT, %xmm1
- pclmulqdq $0x11, CONSTANT, %xmm5
- pxor %xmm5, %xmm1
- pxor %xmm4, %xmm1
-
- cmp $0x10, LEN
- jb .Lfold_64
-.Lloop_16:/* Folding rest buffer into 128bit */
- movdqa %xmm1, %xmm5
- pclmulqdq $0x00, CONSTANT, %xmm1
- pclmulqdq $0x11, CONSTANT, %xmm5
- pxor %xmm5, %xmm1
- pxor (BUF), %xmm1
- sub $0x10, LEN
- add $0x10, BUF
- cmp $0x10, LEN
- jge .Lloop_16
-
-.Lfold_64:
- /* perform the last 64 bit fold, also adds 32 zeroes
- * to the input stream */
- pclmulqdq $0x01, %xmm1, CONSTANT /* R4 * xmm1.low */
- psrldq $0x08, %xmm1
- pxor CONSTANT, %xmm1
-
- /* final 32-bit fold */
- movdqa %xmm1, %xmm2
-#ifdef __x86_64__
- movdqa .Lconstant_R5(%rip), CONSTANT
- movdqa .Lconstant_mask32(%rip), %xmm3
-#else
- movdqa .Lconstant_R5, CONSTANT
- movdqa .Lconstant_mask32, %xmm3
-#endif
- psrldq $0x04, %xmm2
- pand %xmm3, %xmm1
- pclmulqdq $0x00, CONSTANT, %xmm1
- pxor %xmm2, %xmm1
-
- /* Finish up with the bit-reversed barrett reduction 64 ==> 32 bits */
-#ifdef __x86_64__
- movdqa .Lconstant_RUpoly(%rip), CONSTANT
-#else
- movdqa .Lconstant_RUpoly, CONSTANT
-#endif
- movdqa %xmm1, %xmm2
- pand %xmm3, %xmm1
- pclmulqdq $0x10, CONSTANT, %xmm1
- pand %xmm3, %xmm1
- pclmulqdq $0x00, CONSTANT, %xmm1
- pxor %xmm2, %xmm1
- pextrd $0x01, %xmm1, %eax
-
- RET
-SYM_FUNC_END(crc32_pclmul_le_16)
+DEFINE_CRC_PCLMUL_FUNCS(crc32_lsb, /* bits= */ 32, /* lsb= */ 1)
--
2.48.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v2 10/11] x86/crc-t10dif: implement crc_t10dif using new template
2025-01-30 3:51 [PATCH v2 00/11] CRC64 library rework and x86 CRC optimization Eric Biggers
` (8 preceding siblings ...)
2025-01-30 3:51 ` [PATCH v2 09/11] x86/crc32: implement crc32_le using new template Eric Biggers
@ 2025-01-30 3:51 ` Eric Biggers
2025-01-30 3:51 ` [PATCH v2 11/11] x86/crc64: implement crc64_be and crc64_nvme " Eric Biggers
` (3 subsequent siblings)
13 siblings, 0 replies; 21+ messages in thread
From: Eric Biggers @ 2025-01-30 3:51 UTC (permalink / raw)
To: linux-kernel
Cc: linux-crypto, x86, linux-block, Ard Biesheuvel, Keith Busch,
Kent Overstreet, Martin K . Petersen
From: Eric Biggers <ebiggers@google.com>
Instantiate crc-pclmul-template.S for crc_t10dif and delete the original
PCLMULQDQ optimized implementation. This has the following advantages:
- Less CRC-variant-specific code.
- VPCLMULQDQ support, greatly improving performance on sufficiently long
messages on newer CPUs.
- A faster reduction from 128 bits to the final CRC.
- Support for i386.
Benchmark results on AMD Ryzen 9 9950X (Zen 5) using crc_kunit:
Length Before After
------ ------ -----
1 440 MB/s 386 MB/s
16 1865 MB/s 2008 MB/s
64 4343 MB/s 6917 MB/s
127 5440 MB/s 8909 MB/s
128 5533 MB/s 12150 MB/s
200 5908 MB/s 14423 MB/s
256 15870 MB/s 21288 MB/s
511 14219 MB/s 25840 MB/s
512 18361 MB/s 37797 MB/s
1024 19941 MB/s 61374 MB/s
3173 20461 MB/s 74909 MB/s
4096 21310 MB/s 78919 MB/s
16384 21663 MB/s 85012 MB/s
Acked-by: Ard Biesheuvel <ardb@kernel.org>
Signed-off-by: Eric Biggers <ebiggers@google.com>
---
arch/x86/Kconfig | 2 +-
arch/x86/lib/Makefile | 2 +-
arch/x86/lib/crc-pclmul-consts.h | 48 +++-
arch/x86/lib/crc-t10dif-glue.c | 23 +-
arch/x86/lib/crc16-msb-pclmul.S | 6 +
arch/x86/lib/crct10dif-pcl-asm_64.S | 332 ----------------------------
6 files changed, 64 insertions(+), 349 deletions(-)
create mode 100644 arch/x86/lib/crc16-msb-pclmul.S
delete mode 100644 arch/x86/lib/crct10dif-pcl-asm_64.S
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 87198d957e2f..7f59d73201ce 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -75,11 +75,11 @@ config X86
select ARCH_HAS_CACHE_LINE_SIZE
select ARCH_HAS_CPU_CACHE_INVALIDATE_MEMREGION
select ARCH_HAS_CPU_FINALIZE_INIT
select ARCH_HAS_CPU_PASID if IOMMU_SVA
select ARCH_HAS_CRC32
- select ARCH_HAS_CRC_T10DIF if X86_64
+ select ARCH_HAS_CRC_T10DIF
select ARCH_HAS_CURRENT_STACK_POINTER
select ARCH_HAS_DEBUG_VIRTUAL
select ARCH_HAS_DEBUG_VM_PGTABLE if !X86_PAE
select ARCH_HAS_DEVMEM_IS_ALLOWED
select ARCH_HAS_DMA_OPS if GART_IOMMU || XEN
diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile
index 8a59c61624c2..08496e221a7d 100644
--- a/arch/x86/lib/Makefile
+++ b/arch/x86/lib/Makefile
@@ -41,11 +41,11 @@ lib-$(CONFIG_MITIGATION_RETPOLINE) += retpoline.o
obj-$(CONFIG_CRC32_ARCH) += crc32-x86.o
crc32-x86-y := crc32-glue.o crc32-pclmul.o
crc32-x86-$(CONFIG_64BIT) += crc32c-3way.o
obj-$(CONFIG_CRC_T10DIF_ARCH) += crc-t10dif-x86.o
-crc-t10dif-x86-y := crc-t10dif-glue.o crct10dif-pcl-asm_64.o
+crc-t10dif-x86-y := crc-t10dif-glue.o crc16-msb-pclmul.o
obj-y += msr.o msr-reg.o msr-reg-export.o hweight.o
obj-y += iomem.o
ifeq ($(CONFIG_X86_32),y)
diff --git a/arch/x86/lib/crc-pclmul-consts.h b/arch/x86/lib/crc-pclmul-consts.h
index ee22cf221c35..d7beee26d158 100644
--- a/arch/x86/lib/crc-pclmul-consts.h
+++ b/arch/x86/lib/crc-pclmul-consts.h
@@ -1,14 +1,60 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* CRC constants generated by:
*
- * ./scripts/gen-crc-consts.py x86_pclmul crc32_lsb_0xedb88320
+ * ./scripts/gen-crc-consts.py x86_pclmul crc16_msb_0x8bb7,crc32_lsb_0xedb88320
*
* Do not edit manually.
*/
+/*
+ * CRC folding constants generated for most-significant-bit-first CRC-16 using
+ * G(x) = x^16 + x^15 + x^11 + x^9 + x^8 + x^7 + x^5 + x^4 + x^2 + x + 1
+ */
+static const struct {
+ u8 bswap_mask[16];
+ u64 fold_across_2048_bits_consts[2];
+ u64 fold_across_1024_bits_consts[2];
+ u64 fold_across_512_bits_consts[2];
+ u64 fold_across_256_bits_consts[2];
+ u64 fold_across_128_bits_consts[2];
+ u8 shuf_table[48];
+ u64 barrett_reduction_consts[2];
+} crc16_msb_0x8bb7_consts ____cacheline_aligned __maybe_unused = {
+ .bswap_mask = {15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0},
+ .fold_across_2048_bits_consts = {
+ 0x22c6, /* x^(2048+0) mod G(x) */
+ 0x9f16, /* x^(2048+64) mod G(x) */
+ },
+ .fold_across_1024_bits_consts = {
+ 0x6123, /* x^(1024+0) mod G(x) */
+ 0x2295, /* x^(1024+64) mod G(x) */
+ },
+ .fold_across_512_bits_consts = {
+ 0x1069, /* x^(512+0) mod G(x) */
+ 0xdd31, /* x^(512+64) mod G(x) */
+ },
+ .fold_across_256_bits_consts = {
+ 0x857d, /* x^(256+0) mod G(x) */
+ 0x7acc, /* x^(256+64) mod G(x) */
+ },
+ .fold_across_128_bits_consts = {
+ 0x2d56000000000000, /* x^80 mod G(x) * x^48 */
+ 0x1faa, /* x^(128+64) mod G(x) */
+ },
+ .shuf_table = {
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ },
+ .barrett_reduction_consts = {
+ 0xf65a57f81d33a48a, /* floor(x^80 / G(x)) - x^64 */
+ 0x18bb7, /* G(x) */
+ },
+};
+
/*
* CRC folding constants generated for least-significant-bit-first CRC-32 using
* G(x) = x^32 + x^26 + x^23 + x^22 + x^16 + x^12 + x^11 + x^10 + x^8 + x^7 +
* x^5 + x^4 + x^2 + x + 1
*/
diff --git a/arch/x86/lib/crc-t10dif-glue.c b/arch/x86/lib/crc-t10dif-glue.c
index 13f07ddc9122..6b09374b8355 100644
--- a/arch/x86/lib/crc-t10dif-glue.c
+++ b/arch/x86/lib/crc-t10dif-glue.c
@@ -1,39 +1,34 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/*
- * CRC-T10DIF using PCLMULQDQ instructions
+ * CRC-T10DIF using [V]PCLMULQDQ instructions
*
* Copyright 2024 Google LLC
*/
-#include <asm/cpufeatures.h>
-#include <asm/simd.h>
-#include <crypto/internal/simd.h>
#include <linux/crc-t10dif.h>
#include <linux/module.h>
+#include "crc-pclmul-template.h"
static DEFINE_STATIC_KEY_FALSE(have_pclmulqdq);
-asmlinkage u16 crc_t10dif_pcl(u16 init_crc, const u8 *buf, size_t len);
+DECLARE_CRC_PCLMUL_FUNCS(crc16_msb, u16);
u16 crc_t10dif_arch(u16 crc, const u8 *p, size_t len)
{
- if (len >= 16 &&
- static_key_enabled(&have_pclmulqdq) && crypto_simd_usable()) {
- kernel_fpu_begin();
- crc = crc_t10dif_pcl(crc, p, len);
- kernel_fpu_end();
- return crc;
- }
+ CRC_PCLMUL(crc, p, len, crc16_msb, crc16_msb_0x8bb7_consts,
+ have_pclmulqdq);
return crc_t10dif_generic(crc, p, len);
}
EXPORT_SYMBOL(crc_t10dif_arch);
static int __init crc_t10dif_x86_init(void)
{
- if (boot_cpu_has(X86_FEATURE_PCLMULQDQ))
+ if (boot_cpu_has(X86_FEATURE_PCLMULQDQ)) {
static_branch_enable(&have_pclmulqdq);
+ INIT_CRC_PCLMUL(crc16_msb);
+ }
return 0;
}
arch_initcall(crc_t10dif_x86_init);
static void __exit crc_t10dif_x86_exit(void)
@@ -45,7 +40,7 @@ bool crc_t10dif_is_optimized(void)
{
return static_key_enabled(&have_pclmulqdq);
}
EXPORT_SYMBOL(crc_t10dif_is_optimized);
-MODULE_DESCRIPTION("CRC-T10DIF using PCLMULQDQ instructions");
+MODULE_DESCRIPTION("CRC-T10DIF using [V]PCLMULQDQ instructions");
MODULE_LICENSE("GPL");
diff --git a/arch/x86/lib/crc16-msb-pclmul.S b/arch/x86/lib/crc16-msb-pclmul.S
new file mode 100644
index 000000000000..e9fe248093a8
--- /dev/null
+++ b/arch/x86/lib/crc16-msb-pclmul.S
@@ -0,0 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+// Copyright 2025 Google LLC
+
+#include "crc-pclmul-template.S"
+
+DEFINE_CRC_PCLMUL_FUNCS(crc16_msb, /* bits= */ 16, /* lsb= */ 0)
diff --git a/arch/x86/lib/crct10dif-pcl-asm_64.S b/arch/x86/lib/crct10dif-pcl-asm_64.S
deleted file mode 100644
index 5286db5b8165..000000000000
--- a/arch/x86/lib/crct10dif-pcl-asm_64.S
+++ /dev/null
@@ -1,332 +0,0 @@
-########################################################################
-# Implement fast CRC-T10DIF computation with SSE and PCLMULQDQ instructions
-#
-# Copyright (c) 2013, Intel Corporation
-#
-# Authors:
-# Erdinc Ozturk <erdinc.ozturk@intel.com>
-# Vinodh Gopal <vinodh.gopal@intel.com>
-# James Guilford <james.guilford@intel.com>
-# Tim Chen <tim.c.chen@linux.intel.com>
-#
-# This software is available to you under a choice of one of two
-# licenses. You may choose to be licensed under the terms of the GNU
-# General Public License (GPL) Version 2, available from the file
-# COPYING in the main directory of this source tree, or the
-# OpenIB.org BSD license below:
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-# * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-#
-# * Redistributions in binary form must reproduce the above copyright
-# notice, this list of conditions and the following disclaimer in the
-# documentation and/or other materials provided with the
-# distribution.
-#
-# * Neither the name of the Intel Corporation nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-#
-# THIS SOFTWARE IS PROVIDED BY INTEL CORPORATION ""AS IS"" AND ANY
-# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL INTEL CORPORATION OR
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
-# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
-# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-#
-# Reference paper titled "Fast CRC Computation for Generic
-# Polynomials Using PCLMULQDQ Instruction"
-# URL: http://www.intel.com/content/dam/www/public/us/en/documents
-# /white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf
-#
-
-#include <linux/linkage.h>
-
-.text
-
-#define init_crc %edi
-#define buf %rsi
-#define len %rdx
-
-#define FOLD_CONSTS %xmm10
-#define BSWAP_MASK %xmm11
-
-# Fold reg1, reg2 into the next 32 data bytes, storing the result back into
-# reg1, reg2.
-.macro fold_32_bytes offset, reg1, reg2
- movdqu \offset(buf), %xmm9
- movdqu \offset+16(buf), %xmm12
- pshufb BSWAP_MASK, %xmm9
- pshufb BSWAP_MASK, %xmm12
- movdqa \reg1, %xmm8
- movdqa \reg2, %xmm13
- pclmulqdq $0x00, FOLD_CONSTS, \reg1
- pclmulqdq $0x11, FOLD_CONSTS, %xmm8
- pclmulqdq $0x00, FOLD_CONSTS, \reg2
- pclmulqdq $0x11, FOLD_CONSTS, %xmm13
- pxor %xmm9 , \reg1
- xorps %xmm8 , \reg1
- pxor %xmm12, \reg2
- xorps %xmm13, \reg2
-.endm
-
-# Fold src_reg into dst_reg.
-.macro fold_16_bytes src_reg, dst_reg
- movdqa \src_reg, %xmm8
- pclmulqdq $0x11, FOLD_CONSTS, \src_reg
- pclmulqdq $0x00, FOLD_CONSTS, %xmm8
- pxor %xmm8, \dst_reg
- xorps \src_reg, \dst_reg
-.endm
-
-#
-# u16 crc_t10dif_pcl(u16 init_crc, const *u8 buf, size_t len);
-#
-# Assumes len >= 16.
-#
-SYM_FUNC_START(crc_t10dif_pcl)
-
- movdqa .Lbswap_mask(%rip), BSWAP_MASK
-
- # For sizes less than 256 bytes, we can't fold 128 bytes at a time.
- cmp $256, len
- jl .Lless_than_256_bytes
-
- # Load the first 128 data bytes. Byte swapping is necessary to make the
- # bit order match the polynomial coefficient order.
- movdqu 16*0(buf), %xmm0
- movdqu 16*1(buf), %xmm1
- movdqu 16*2(buf), %xmm2
- movdqu 16*3(buf), %xmm3
- movdqu 16*4(buf), %xmm4
- movdqu 16*5(buf), %xmm5
- movdqu 16*6(buf), %xmm6
- movdqu 16*7(buf), %xmm7
- add $128, buf
- pshufb BSWAP_MASK, %xmm0
- pshufb BSWAP_MASK, %xmm1
- pshufb BSWAP_MASK, %xmm2
- pshufb BSWAP_MASK, %xmm3
- pshufb BSWAP_MASK, %xmm4
- pshufb BSWAP_MASK, %xmm5
- pshufb BSWAP_MASK, %xmm6
- pshufb BSWAP_MASK, %xmm7
-
- # XOR the first 16 data *bits* with the initial CRC value.
- pxor %xmm8, %xmm8
- pinsrw $7, init_crc, %xmm8
- pxor %xmm8, %xmm0
-
- movdqa .Lfold_across_128_bytes_consts(%rip), FOLD_CONSTS
-
- # Subtract 128 for the 128 data bytes just consumed. Subtract another
- # 128 to simplify the termination condition of the following loop.
- sub $256, len
-
- # While >= 128 data bytes remain (not counting xmm0-7), fold the 128
- # bytes xmm0-7 into them, storing the result back into xmm0-7.
-.Lfold_128_bytes_loop:
- fold_32_bytes 0, %xmm0, %xmm1
- fold_32_bytes 32, %xmm2, %xmm3
- fold_32_bytes 64, %xmm4, %xmm5
- fold_32_bytes 96, %xmm6, %xmm7
- add $128, buf
- sub $128, len
- jge .Lfold_128_bytes_loop
-
- # Now fold the 112 bytes in xmm0-xmm6 into the 16 bytes in xmm7.
-
- # Fold across 64 bytes.
- movdqa .Lfold_across_64_bytes_consts(%rip), FOLD_CONSTS
- fold_16_bytes %xmm0, %xmm4
- fold_16_bytes %xmm1, %xmm5
- fold_16_bytes %xmm2, %xmm6
- fold_16_bytes %xmm3, %xmm7
- # Fold across 32 bytes.
- movdqa .Lfold_across_32_bytes_consts(%rip), FOLD_CONSTS
- fold_16_bytes %xmm4, %xmm6
- fold_16_bytes %xmm5, %xmm7
- # Fold across 16 bytes.
- movdqa .Lfold_across_16_bytes_consts(%rip), FOLD_CONSTS
- fold_16_bytes %xmm6, %xmm7
-
- # Add 128 to get the correct number of data bytes remaining in 0...127
- # (not counting xmm7), following the previous extra subtraction by 128.
- # Then subtract 16 to simplify the termination condition of the
- # following loop.
- add $128-16, len
-
- # While >= 16 data bytes remain (not counting xmm7), fold the 16 bytes
- # xmm7 into them, storing the result back into xmm7.
- jl .Lfold_16_bytes_loop_done
-.Lfold_16_bytes_loop:
- movdqa %xmm7, %xmm8
- pclmulqdq $0x11, FOLD_CONSTS, %xmm7
- pclmulqdq $0x00, FOLD_CONSTS, %xmm8
- pxor %xmm8, %xmm7
- movdqu (buf), %xmm0
- pshufb BSWAP_MASK, %xmm0
- pxor %xmm0 , %xmm7
- add $16, buf
- sub $16, len
- jge .Lfold_16_bytes_loop
-
-.Lfold_16_bytes_loop_done:
- # Add 16 to get the correct number of data bytes remaining in 0...15
- # (not counting xmm7), following the previous extra subtraction by 16.
- add $16, len
- je .Lreduce_final_16_bytes
-
-.Lhandle_partial_segment:
- # Reduce the last '16 + len' bytes where 1 <= len <= 15 and the first 16
- # bytes are in xmm7 and the rest are the remaining data in 'buf'. To do
- # this without needing a fold constant for each possible 'len', redivide
- # the bytes into a first chunk of 'len' bytes and a second chunk of 16
- # bytes, then fold the first chunk into the second.
-
- movdqa %xmm7, %xmm2
-
- # xmm1 = last 16 original data bytes
- movdqu -16(buf, len), %xmm1
- pshufb BSWAP_MASK, %xmm1
-
- # xmm2 = high order part of second chunk: xmm7 left-shifted by 'len' bytes.
- lea .Lbyteshift_table+16(%rip), %rax
- sub len, %rax
- movdqu (%rax), %xmm0
- pshufb %xmm0, %xmm2
-
- # xmm7 = first chunk: xmm7 right-shifted by '16-len' bytes.
- pxor .Lmask1(%rip), %xmm0
- pshufb %xmm0, %xmm7
-
- # xmm1 = second chunk: 'len' bytes from xmm1 (low-order bytes),
- # then '16-len' bytes from xmm2 (high-order bytes).
- pblendvb %xmm2, %xmm1 #xmm0 is implicit
-
- # Fold the first chunk into the second chunk, storing the result in xmm7.
- movdqa %xmm7, %xmm8
- pclmulqdq $0x11, FOLD_CONSTS, %xmm7
- pclmulqdq $0x00, FOLD_CONSTS, %xmm8
- pxor %xmm8, %xmm7
- pxor %xmm1, %xmm7
-
-.Lreduce_final_16_bytes:
- # Reduce the 128-bit value M(x), stored in xmm7, to the final 16-bit CRC
-
- # Load 'x^48 * (x^48 mod G(x))' and 'x^48 * (x^80 mod G(x))'.
- movdqa .Lfinal_fold_consts(%rip), FOLD_CONSTS
-
- # Fold the high 64 bits into the low 64 bits, while also multiplying by
- # x^64. This produces a 128-bit value congruent to x^64 * M(x) and
- # whose low 48 bits are 0.
- movdqa %xmm7, %xmm0
- pclmulqdq $0x11, FOLD_CONSTS, %xmm7 # high bits * x^48 * (x^80 mod G(x))
- pslldq $8, %xmm0
- pxor %xmm0, %xmm7 # + low bits * x^64
-
- # Fold the high 32 bits into the low 96 bits. This produces a 96-bit
- # value congruent to x^64 * M(x) and whose low 48 bits are 0.
- movdqa %xmm7, %xmm0
- pand .Lmask2(%rip), %xmm0 # zero high 32 bits
- psrldq $12, %xmm7 # extract high 32 bits
- pclmulqdq $0x00, FOLD_CONSTS, %xmm7 # high 32 bits * x^48 * (x^48 mod G(x))
- pxor %xmm0, %xmm7 # + low bits
-
- # Load G(x) and floor(x^48 / G(x)).
- movdqa .Lbarrett_reduction_consts(%rip), FOLD_CONSTS
-
- # Use Barrett reduction to compute the final CRC value.
- movdqa %xmm7, %xmm0
- pclmulqdq $0x11, FOLD_CONSTS, %xmm7 # high 32 bits * floor(x^48 / G(x))
- psrlq $32, %xmm7 # /= x^32
- pclmulqdq $0x00, FOLD_CONSTS, %xmm7 # *= G(x)
- psrlq $48, %xmm0
- pxor %xmm7, %xmm0 # + low 16 nonzero bits
- # Final CRC value (x^16 * M(x)) mod G(x) is in low 16 bits of xmm0.
-
- pextrw $0, %xmm0, %eax
- RET
-
-.align 16
-.Lless_than_256_bytes:
- # Checksumming a buffer of length 16...255 bytes
-
- # Load the first 16 data bytes.
- movdqu (buf), %xmm7
- pshufb BSWAP_MASK, %xmm7
- add $16, buf
-
- # XOR the first 16 data *bits* with the initial CRC value.
- pxor %xmm0, %xmm0
- pinsrw $7, init_crc, %xmm0
- pxor %xmm0, %xmm7
-
- movdqa .Lfold_across_16_bytes_consts(%rip), FOLD_CONSTS
- cmp $16, len
- je .Lreduce_final_16_bytes # len == 16
- sub $32, len
- jge .Lfold_16_bytes_loop # 32 <= len <= 255
- add $16, len
- jmp .Lhandle_partial_segment # 17 <= len <= 31
-SYM_FUNC_END(crc_t10dif_pcl)
-
-.section .rodata, "a", @progbits
-.align 16
-
-# Fold constants precomputed from the polynomial 0x18bb7
-# G(x) = x^16 + x^15 + x^11 + x^9 + x^8 + x^7 + x^5 + x^4 + x^2 + x^1 + x^0
-.Lfold_across_128_bytes_consts:
- .quad 0x0000000000006123 # x^(8*128) mod G(x)
- .quad 0x0000000000002295 # x^(8*128+64) mod G(x)
-.Lfold_across_64_bytes_consts:
- .quad 0x0000000000001069 # x^(4*128) mod G(x)
- .quad 0x000000000000dd31 # x^(4*128+64) mod G(x)
-.Lfold_across_32_bytes_consts:
- .quad 0x000000000000857d # x^(2*128) mod G(x)
- .quad 0x0000000000007acc # x^(2*128+64) mod G(x)
-.Lfold_across_16_bytes_consts:
- .quad 0x000000000000a010 # x^(1*128) mod G(x)
- .quad 0x0000000000001faa # x^(1*128+64) mod G(x)
-.Lfinal_fold_consts:
- .quad 0x1368000000000000 # x^48 * (x^48 mod G(x))
- .quad 0x2d56000000000000 # x^48 * (x^80 mod G(x))
-.Lbarrett_reduction_consts:
- .quad 0x0000000000018bb7 # G(x)
- .quad 0x00000001f65a57f8 # floor(x^48 / G(x))
-
-.section .rodata.cst16.mask1, "aM", @progbits, 16
-.align 16
-.Lmask1:
- .octa 0x80808080808080808080808080808080
-
-.section .rodata.cst16.mask2, "aM", @progbits, 16
-.align 16
-.Lmask2:
- .octa 0x00000000FFFFFFFFFFFFFFFFFFFFFFFF
-
-.section .rodata.cst16.bswap_mask, "aM", @progbits, 16
-.align 16
-.Lbswap_mask:
- .octa 0x000102030405060708090A0B0C0D0E0F
-
-.section .rodata.cst32.byteshift_table, "aM", @progbits, 32
-.align 16
-# For 1 <= len <= 15, the 16-byte vector beginning at &byteshift_table[16 - len]
-# is the index vector to shift left by 'len' bytes, and is also {0x80, ...,
-# 0x80} XOR the index vector to shift right by '16 - len' bytes.
-.Lbyteshift_table:
- .byte 0x0, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87
- .byte 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f
- .byte 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7
- .byte 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe , 0x0
--
2.48.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH v2 11/11] x86/crc64: implement crc64_be and crc64_nvme using new template
2025-01-30 3:51 [PATCH v2 00/11] CRC64 library rework and x86 CRC optimization Eric Biggers
` (9 preceding siblings ...)
2025-01-30 3:51 ` [PATCH v2 10/11] x86/crc-t10dif: implement crc_t10dif " Eric Biggers
@ 2025-01-30 3:51 ` Eric Biggers
2025-01-30 15:09 ` [PATCH v2 00/11] CRC64 library rework and x86 CRC optimization Keith Busch
` (2 subsequent siblings)
13 siblings, 0 replies; 21+ messages in thread
From: Eric Biggers @ 2025-01-30 3:51 UTC (permalink / raw)
To: linux-kernel
Cc: linux-crypto, x86, linux-block, Ard Biesheuvel, Keith Busch,
Kent Overstreet, Martin K . Petersen
From: Eric Biggers <ebiggers@google.com>
Add x86_64 [V]PCLMULQDQ optimized implementations of crc64_be() and
crc64_nvme() by wiring them up to crc-pclmul-template.S.
crc64_be() is used by bcache and bcachefs, and crc64_nvme() is used by
blk-integrity. Both features can CRC large amounts of data, and the
developers of both features have expressed interest in having these CRCs
be optimized. So this optimization should be worthwhile. (See
https://lore.kernel.org/r/v36sousjd5ukqlkpdxslvpu7l37zbu7d7slgc2trjjqwty2bny@qgzew34feo2r
and
https://lore.kernel.org/r/20220222163144.1782447-11-kbusch@kernel.org)
Benchmark results on AMD Ryzen 9 9950X (Zen 5) using crc_kunit:
crc64_be:
Length Before After
------ ------ -----
1 633 MB/s 477 MB/s
16 717 MB/s 2517 MB/s
64 715 MB/s 7525 MB/s
127 714 MB/s 10002 MB/s
128 713 MB/s 13344 MB/s
200 715 MB/s 15752 MB/s
256 714 MB/s 22933 MB/s
511 715 MB/s 28025 MB/s
512 714 MB/s 49772 MB/s
1024 715 MB/s 65261 MB/s
3173 714 MB/s 78773 MB/s
4096 714 MB/s 83315 MB/s
16384 714 MB/s 89487 MB/s
crc64_nvme:
Length Before After
------ ------ -----
1 716 MB/s 474 MB/s
16 717 MB/s 3303 MB/s
64 713 MB/s 7940 MB/s
127 715 MB/s 9867 MB/s
128 714 MB/s 13698 MB/s
200 715 MB/s 15995 MB/s
256 714 MB/s 23479 MB/s
511 714 MB/s 28013 MB/s
512 715 MB/s 51533 MB/s
1024 715 MB/s 66788 MB/s
3173 715 MB/s 79182 MB/s
4096 715 MB/s 83966 MB/s
16384 715 MB/s 89739 MB/s
Signed-off-by: Eric Biggers <ebiggers@google.com>
---
arch/x86/Kconfig | 1 +
arch/x86/lib/Makefile | 3 +
arch/x86/lib/crc-pclmul-consts.h | 98 +++++++++++++++++++++++++++++++-
arch/x86/lib/crc64-glue.c | 50 ++++++++++++++++
arch/x86/lib/crc64-pclmul.S | 7 +++
5 files changed, 158 insertions(+), 1 deletion(-)
create mode 100644 arch/x86/lib/crc64-glue.c
create mode 100644 arch/x86/lib/crc64-pclmul.S
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 7f59d73201ce..aa7c9d57e4d3 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -75,10 +75,11 @@ config X86
select ARCH_HAS_CACHE_LINE_SIZE
select ARCH_HAS_CPU_CACHE_INVALIDATE_MEMREGION
select ARCH_HAS_CPU_FINALIZE_INIT
select ARCH_HAS_CPU_PASID if IOMMU_SVA
select ARCH_HAS_CRC32
+ select ARCH_HAS_CRC64 if X86_64
select ARCH_HAS_CRC_T10DIF
select ARCH_HAS_CURRENT_STACK_POINTER
select ARCH_HAS_DEBUG_VIRTUAL
select ARCH_HAS_DEBUG_VM_PGTABLE if !X86_PAE
select ARCH_HAS_DEVMEM_IS_ALLOWED
diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile
index 08496e221a7d..71c14329fd79 100644
--- a/arch/x86/lib/Makefile
+++ b/arch/x86/lib/Makefile
@@ -40,10 +40,13 @@ lib-$(CONFIG_MITIGATION_RETPOLINE) += retpoline.o
obj-$(CONFIG_CRC32_ARCH) += crc32-x86.o
crc32-x86-y := crc32-glue.o crc32-pclmul.o
crc32-x86-$(CONFIG_64BIT) += crc32c-3way.o
+obj-$(CONFIG_CRC64_ARCH) += crc64-x86.o
+crc64-x86-y := crc64-glue.o crc64-pclmul.o
+
obj-$(CONFIG_CRC_T10DIF_ARCH) += crc-t10dif-x86.o
crc-t10dif-x86-y := crc-t10dif-glue.o crc16-msb-pclmul.o
obj-y += msr.o msr-reg.o msr-reg-export.o hweight.o
obj-y += iomem.o
diff --git a/arch/x86/lib/crc-pclmul-consts.h b/arch/x86/lib/crc-pclmul-consts.h
index d7beee26d158..1ea26b99289a 100644
--- a/arch/x86/lib/crc-pclmul-consts.h
+++ b/arch/x86/lib/crc-pclmul-consts.h
@@ -1,10 +1,10 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* CRC constants generated by:
*
- * ./scripts/gen-crc-consts.py x86_pclmul crc16_msb_0x8bb7,crc32_lsb_0xedb88320
+ * ./scripts/gen-crc-consts.py x86_pclmul crc16_msb_0x8bb7,crc32_lsb_0xedb88320,crc64_msb_0x42f0e1eba9ea3693,crc64_lsb_0x9a6c9329ac4bc9b5
*
* Do not edit manually.
*/
/*
@@ -95,5 +95,101 @@ static const struct {
.barrett_reduction_consts = {
0xb4e5b025f7011641, /* floor(x^95 / G(x)) */
0x1db710641, /* G(x) */
},
};
+
+/*
+ * CRC folding constants generated for most-significant-bit-first CRC-64 using
+ * G(x) = x^64 + x^62 + x^57 + x^55 + x^54 + x^53 + x^52 + x^47 + x^46 +
+ * x^45 + x^40 + x^39 + x^38 + x^37 + x^35 + x^33 + x^32 + x^31 +
+ * x^29 + x^27 + x^24 + x^23 + x^22 + x^21 + x^19 + x^17 + x^13 +
+ * x^12 + x^10 + x^9 + x^7 + x^4 + x + 1
+ */
+static const struct {
+ u8 bswap_mask[16];
+ u64 fold_across_2048_bits_consts[2];
+ u64 fold_across_1024_bits_consts[2];
+ u64 fold_across_512_bits_consts[2];
+ u64 fold_across_256_bits_consts[2];
+ u64 fold_across_128_bits_consts[2];
+ u8 shuf_table[48];
+ u64 barrett_reduction_consts[2];
+} crc64_msb_0x42f0e1eba9ea3693_consts ____cacheline_aligned __maybe_unused = {
+ .bswap_mask = {15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0},
+ .fold_across_2048_bits_consts = {
+ 0x7f52691a60ddc70d, /* x^(2048+0) mod G(x) */
+ 0x7036b0389f6a0c82, /* x^(2048+64) mod G(x) */
+ },
+ .fold_across_1024_bits_consts = {
+ 0x05cf79dea9ac37d6, /* x^(1024+0) mod G(x) */
+ 0x001067e571d7d5c2, /* x^(1024+64) mod G(x) */
+ },
+ .fold_across_512_bits_consts = {
+ 0x5f6843ca540df020, /* x^(512+0) mod G(x) */
+ 0xddf4b6981205b83f, /* x^(512+64) mod G(x) */
+ },
+ .fold_across_256_bits_consts = {
+ 0x571bee0a227ef92b, /* x^(256+0) mod G(x) */
+ 0x44bef2a201b5200c, /* x^(256+64) mod G(x) */
+ },
+ .fold_across_128_bits_consts = {
+ 0x05f5c3c7eb52fab6, /* x^128 mod G(x) * x^0 */
+ 0x4eb938a7d257740e, /* x^(128+64) mod G(x) */
+ },
+ .shuf_table = {
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ },
+ .barrett_reduction_consts = {
+ 0x578d29d06cc4f872, /* floor(x^128 / G(x)) - x^64 */
+ 0x42f0e1eba9ea3693, /* G(x) - x^64 */
+ },
+};
+
+/*
+ * CRC folding constants generated for least-significant-bit-first CRC-64 using
+ * G(x) = x^64 + x^63 + x^61 + x^59 + x^58 + x^56 + x^55 + x^52 + x^49 +
+ * x^48 + x^47 + x^46 + x^44 + x^41 + x^37 + x^36 + x^34 + x^32 +
+ * x^31 + x^28 + x^26 + x^23 + x^22 + x^19 + x^16 + x^13 + x^12 +
+ * x^10 + x^9 + x^6 + x^4 + x^3 + 1
+ */
+static const struct {
+ u64 fold_across_2048_bits_consts[2];
+ u64 fold_across_1024_bits_consts[2];
+ u64 fold_across_512_bits_consts[2];
+ u64 fold_across_256_bits_consts[2];
+ u64 fold_across_128_bits_consts[2];
+ u8 shuf_table[48];
+ u64 barrett_reduction_consts[2];
+} crc64_lsb_0x9a6c9329ac4bc9b5_consts ____cacheline_aligned __maybe_unused = {
+ .fold_across_2048_bits_consts = {
+ 0x37ccd3e14069cabc, /* x^(2048+64-1) mod G(x) */
+ 0xa043808c0f782663, /* x^(2048+0-1) mod G(x) */
+ },
+ .fold_across_1024_bits_consts = {
+ 0xa1ca681e733f9c40, /* x^(1024+64-1) mod G(x) */
+ 0x5f852fb61e8d92dc, /* x^(1024+0-1) mod G(x) */
+ },
+ .fold_across_512_bits_consts = {
+ 0x0c32cdb31e18a84a, /* x^(512+64-1) mod G(x) */
+ 0x62242240ace5045a, /* x^(512+0-1) mod G(x) */
+ },
+ .fold_across_256_bits_consts = {
+ 0xb0bc2e589204f500, /* x^(256+64-1) mod G(x) */
+ 0xe1e0bb9d45d7a44c, /* x^(256+0-1) mod G(x) */
+ },
+ .fold_across_128_bits_consts = {
+ 0xeadc41fd2ba3d420, /* x^(128+64-1) mod G(x) */
+ 0x21e9761e252621ac, /* x^(128+0-1) mod G(x) */
+ },
+ .shuf_table = {
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ },
+ .barrett_reduction_consts = {
+ 0x27ecfa329aef9f77, /* floor(x^127 / G(x)) */
+ 0x34d926535897936b, /* G(x) - 1 */
+ },
+};
diff --git a/arch/x86/lib/crc64-glue.c b/arch/x86/lib/crc64-glue.c
new file mode 100644
index 000000000000..b0e1b719ecbf
--- /dev/null
+++ b/arch/x86/lib/crc64-glue.c
@@ -0,0 +1,50 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * CRC64 using [V]PCLMULQDQ instructions
+ *
+ * Copyright 2025 Google LLC
+ */
+
+#include <linux/crc64.h>
+#include <linux/module.h>
+#include "crc-pclmul-template.h"
+
+static DEFINE_STATIC_KEY_FALSE(have_pclmulqdq);
+
+DECLARE_CRC_PCLMUL_FUNCS(crc64_msb, u64);
+DECLARE_CRC_PCLMUL_FUNCS(crc64_lsb, u64);
+
+u64 crc64_be_arch(u64 crc, const u8 *p, size_t len)
+{
+ CRC_PCLMUL(crc, p, len, crc64_msb, crc64_msb_0x42f0e1eba9ea3693_consts,
+ have_pclmulqdq);
+ return crc64_be_generic(crc, p, len);
+}
+EXPORT_SYMBOL_GPL(crc64_be_arch);
+
+u64 crc64_nvme_arch(u64 crc, const u8 *p, size_t len)
+{
+ CRC_PCLMUL(crc, p, len, crc64_lsb, crc64_lsb_0x9a6c9329ac4bc9b5_consts,
+ have_pclmulqdq);
+ return crc64_nvme_generic(crc, p, len);
+}
+EXPORT_SYMBOL_GPL(crc64_nvme_arch);
+
+static int __init crc64_x86_init(void)
+{
+ if (boot_cpu_has(X86_FEATURE_PCLMULQDQ)) {
+ static_branch_enable(&have_pclmulqdq);
+ INIT_CRC_PCLMUL(crc64_msb);
+ INIT_CRC_PCLMUL(crc64_lsb);
+ }
+ return 0;
+}
+arch_initcall(crc64_x86_init);
+
+static void __exit crc64_x86_exit(void)
+{
+}
+module_exit(crc64_x86_exit);
+
+MODULE_DESCRIPTION("CRC64 using [V]PCLMULQDQ instructions");
+MODULE_LICENSE("GPL");
diff --git a/arch/x86/lib/crc64-pclmul.S b/arch/x86/lib/crc64-pclmul.S
new file mode 100644
index 000000000000..4173051b5197
--- /dev/null
+++ b/arch/x86/lib/crc64-pclmul.S
@@ -0,0 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+// Copyright 2025 Google LLC
+
+#include "crc-pclmul-template.S"
+
+DEFINE_CRC_PCLMUL_FUNCS(crc64_msb, /* bits= */ 64, /* lsb= */ 0)
+DEFINE_CRC_PCLMUL_FUNCS(crc64_lsb, /* bits= */ 64, /* lsb= */ 1)
--
2.48.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* Re: [PATCH v2 00/11] CRC64 library rework and x86 CRC optimization
2025-01-30 3:51 [PATCH v2 00/11] CRC64 library rework and x86 CRC optimization Eric Biggers
` (10 preceding siblings ...)
2025-01-30 3:51 ` [PATCH v2 11/11] x86/crc64: implement crc64_be and crc64_nvme " Eric Biggers
@ 2025-01-30 15:09 ` Keith Busch
2025-01-30 15:20 ` Martin K. Petersen
2025-02-04 19:54 ` Eric Biggers
13 siblings, 0 replies; 21+ messages in thread
From: Keith Busch @ 2025-01-30 15:09 UTC (permalink / raw)
To: Eric Biggers
Cc: linux-kernel, linux-crypto, x86, linux-block, Ard Biesheuvel,
Kent Overstreet, Martin K . Petersen
On Wed, Jan 29, 2025 at 07:51:19PM -0800, Eric Biggers wrote:
> Patch 11 wires up the same optimization to crc64_be() and crc64_nvme()
> (a.k.a. the old "crc64_rocksoft") which previously were unoptimized,
Yes, I mistakenly thought the name of the table in the spec was the
name of the CRC, but that was just referring to format of how the
parameters were displayed. It really should have been crc64_nvme, so
thank you for clearing up the naming confusion.
> improving the performance of those CRC functions by as much as 100x.
Awesome!
Looks good:
Acked-by: Keith Busch <kbusch@kernel.org>
^ permalink raw reply [flat|nested] 21+ messages in thread* Re: [PATCH v2 00/11] CRC64 library rework and x86 CRC optimization
2025-01-30 3:51 [PATCH v2 00/11] CRC64 library rework and x86 CRC optimization Eric Biggers
` (11 preceding siblings ...)
2025-01-30 15:09 ` [PATCH v2 00/11] CRC64 library rework and x86 CRC optimization Keith Busch
@ 2025-01-30 15:20 ` Martin K. Petersen
2025-02-04 19:54 ` Eric Biggers
13 siblings, 0 replies; 21+ messages in thread
From: Martin K. Petersen @ 2025-01-30 15:20 UTC (permalink / raw)
To: Eric Biggers
Cc: linux-kernel, linux-crypto, x86, linux-block, Ard Biesheuvel,
Keith Busch, Kent Overstreet, Martin K . Petersen
Eric,
> Patches 1-5 rework the CRC64 library along the lines of what I did for
> CRC32 and CRC-T10DIF in 6.14. They add direct support for
> architecture-specific optimizations, fix the naming of the NVME CRC64
> variant, and eliminate a pointless use of the crypto API.
>
> Patches 6-10 replace the existing x86 PCLMULQDQ optimized CRC code
> with new code that is shared among the different CRC variants and also
> adds VPCLMULQDQ support, greatly improving performance on recent CPUs.
> Patch 11 wires up the same optimization to crc64_be() and crc64_nvme()
> (a.k.a. the old "crc64_rocksoft") which previously were unoptimized,
> improving the performance of those CRC functions by as much as 100x.
> crc64_be is used by bcachefs, and crc64_nvme is used by blk-integrity.
Very nice!
Reviewed-by: Martin K. Petersen <martin.petersen@oracle.com>
--
Martin K. Petersen Oracle Linux Engineering
^ permalink raw reply [flat|nested] 21+ messages in thread* Re: [PATCH v2 00/11] CRC64 library rework and x86 CRC optimization
2025-01-30 3:51 [PATCH v2 00/11] CRC64 library rework and x86 CRC optimization Eric Biggers
` (12 preceding siblings ...)
2025-01-30 15:20 ` Martin K. Petersen
@ 2025-02-04 19:54 ` Eric Biggers
13 siblings, 0 replies; 21+ messages in thread
From: Eric Biggers @ 2025-02-04 19:54 UTC (permalink / raw)
To: linux-kernel
Cc: linux-crypto, x86, linux-block, Ard Biesheuvel, Keith Busch,
Kent Overstreet, Martin K . Petersen
On Wed, Jan 29, 2025 at 07:51:19PM -0800, Eric Biggers wrote:
> This patchset applies to commit 72deda0abee6e7 and is also available at:
>
> git fetch https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git crc-x86-v2
>
> This is the next major set of CRC library improvements, targeting 6.15.
>
> Patches 1-5 rework the CRC64 library along the lines of what I did for
> CRC32 and CRC-T10DIF in 6.14. They add direct support for
> architecture-specific optimizations, fix the naming of the NVME CRC64
> variant, and eliminate a pointless use of the crypto API.
>
> Patches 6-10 replace the existing x86 PCLMULQDQ optimized CRC code with
> new code that is shared among the different CRC variants and also adds
> VPCLMULQDQ support, greatly improving performance on recent CPUs.
> Patch 11 wires up the same optimization to crc64_be() and crc64_nvme()
> (a.k.a. the old "crc64_rocksoft") which previously were unoptimized,
> improving the performance of those CRC functions by as much as 100x.
> crc64_be is used by bcachefs, and crc64_nvme is used by blk-integrity.
Applied patches 1-5 to
https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git/log/?h=crc-next
I'm still working a bit on the x86 assembly and am planning to do a v3 for
patches 6-11.
- Eric
^ permalink raw reply [flat|nested] 21+ messages in thread