From: Eric Biggers <ebiggers@kernel.org>
To: linux-crypto@vger.kernel.org
Cc: Ard Biesheuvel <ardb@kernel.org>,
"Jason A . Donenfeld" <Jason@zx2c4.com>,
Herbert Xu <herbert@gondor.apana.org.au>,
Martin Willi <martin@strongswan.org>,
Antonio Quartulli <antonio@mandelbit.com>,
Eric Biggers <ebiggers@kernel.org>
Subject: [PATCH] lib/crypto: x86/chacha: Simplify chacha_dosimd()
Date: Wed, 4 Nov 2026 00:16:04 -0800 [thread overview]
Message-ID: <20261104081604.302414-1-ebiggers@kernel.org> (raw)
- Use if-else statements to make the logic clearer, eliminate excessive
not-taken branches for certain lengths (e.g., 16-block aligned
lengths), and consolidate a lot of the counter increments.
- Eliminate the unnecessary chacha_advance() function.
DIV_ROUND_UP(bytes, CHACHA_BLOCK_SIZE) is just as good.
The min() clamp did nothing.
- Don't pass a length greater than N blocks to the assembly functions
that process N blocks. They all accepted it and just processed the
first N blocks anyway, but there's no need to rely on that.
The behavior is unchanged for every length.
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
lib/crypto/x86/chacha.h | 128 +++++++++++++++++-----------------------
1 file changed, 54 insertions(+), 74 deletions(-)
diff --git a/lib/crypto/x86/chacha.h b/lib/crypto/x86/chacha.h
index ed39b3fdda84..57d644f89431 100644
--- a/lib/crypto/x86/chacha.h
+++ b/lib/crypto/x86/chacha.h
@@ -48,101 +48,81 @@ static __ro_after_init DEFINE_STATIC_KEY_FALSE(chacha_use_avx2);
static __ro_after_init DEFINE_STATIC_KEY_FALSE(chacha_use_avx512vl);
static __ro_after_init DEFINE_STATIC_KEY_FALSE(chacha_use_avx512);
-static size_t chacha_advance(size_t len, size_t maxblocks)
-{
- len = min(len, maxblocks * CHACHA_BLOCK_SIZE);
- return round_up(len, CHACHA_BLOCK_SIZE) / CHACHA_BLOCK_SIZE;
-}
-
static void chacha_dosimd(struct chacha_state *state, u8 *dst, const u8 *src,
size_t bytes, int nrounds)
{
- if (static_branch_likely(&chacha_use_avx512)) {
- while (bytes >= CHACHA_BLOCK_SIZE * 16) {
- chacha_16block_xor_avx512(state, dst, src, bytes,
- nrounds);
- bytes -= CHACHA_BLOCK_SIZE * 16;
- src += CHACHA_BLOCK_SIZE * 16;
- dst += CHACHA_BLOCK_SIZE * 16;
- state->x[12] += 16;
- }
- if (bytes > CHACHA_BLOCK_SIZE * 8) {
- chacha_16block_xor_avx512(state, dst, src, bytes,
- nrounds);
- state->x[12] += chacha_advance(bytes, 16);
- return;
- }
- }
-
if (static_branch_likely(&chacha_use_avx512vl)) {
- while (bytes >= CHACHA_BLOCK_SIZE * 8) {
- chacha_8block_xor_avx512vl(state, dst, src, bytes,
- nrounds);
- bytes -= CHACHA_BLOCK_SIZE * 8;
- src += CHACHA_BLOCK_SIZE * 8;
- dst += CHACHA_BLOCK_SIZE * 8;
- state->x[12] += 8;
+ if (static_branch_likely(&chacha_use_avx512)) {
+ while (bytes >= CHACHA_BLOCK_SIZE * 16) {
+ chacha_16block_xor_avx512(
+ state, dst, src, CHACHA_BLOCK_SIZE * 16,
+ nrounds);
+ bytes -= CHACHA_BLOCK_SIZE * 16;
+ src += CHACHA_BLOCK_SIZE * 16;
+ dst += CHACHA_BLOCK_SIZE * 16;
+ state->x[12] += 16;
+ }
+ if (bytes > CHACHA_BLOCK_SIZE * 8) {
+ chacha_16block_xor_avx512(state, dst, src,
+ bytes, nrounds);
+ state->x[12] +=
+ DIV_ROUND_UP(bytes, CHACHA_BLOCK_SIZE);
+ return;
+ }
+ } else {
+ while (bytes >= CHACHA_BLOCK_SIZE * 8) {
+ chacha_8block_xor_avx512vl(
+ state, dst, src, CHACHA_BLOCK_SIZE * 8,
+ nrounds);
+ bytes -= CHACHA_BLOCK_SIZE * 8;
+ src += CHACHA_BLOCK_SIZE * 8;
+ dst += CHACHA_BLOCK_SIZE * 8;
+ state->x[12] += 8;
+ }
}
- if (bytes > CHACHA_BLOCK_SIZE * 4) {
+ if (bytes > CHACHA_BLOCK_SIZE * 4)
chacha_8block_xor_avx512vl(state, dst, src, bytes,
nrounds);
- state->x[12] += chacha_advance(bytes, 8);
- return;
- }
- if (bytes > CHACHA_BLOCK_SIZE * 2) {
+ else if (bytes > CHACHA_BLOCK_SIZE * 2)
chacha_4block_xor_avx512vl(state, dst, src, bytes,
nrounds);
- state->x[12] += chacha_advance(bytes, 4);
- return;
- }
- if (bytes) {
+ else if (bytes)
chacha_2block_xor_avx512vl(state, dst, src, bytes,
nrounds);
- state->x[12] += chacha_advance(bytes, 2);
- return;
- }
- }
-
- if (static_branch_likely(&chacha_use_avx2)) {
+ state->x[12] += DIV_ROUND_UP(bytes, CHACHA_BLOCK_SIZE);
+ } else if (static_branch_likely(&chacha_use_avx2)) {
while (bytes >= CHACHA_BLOCK_SIZE * 8) {
- chacha_8block_xor_avx2(state, dst, src, bytes, nrounds);
+ chacha_8block_xor_avx2(state, dst, src,
+ CHACHA_BLOCK_SIZE * 8, nrounds);
bytes -= CHACHA_BLOCK_SIZE * 8;
src += CHACHA_BLOCK_SIZE * 8;
dst += CHACHA_BLOCK_SIZE * 8;
state->x[12] += 8;
}
- if (bytes > CHACHA_BLOCK_SIZE * 4) {
+ if (bytes > CHACHA_BLOCK_SIZE * 4)
chacha_8block_xor_avx2(state, dst, src, bytes, nrounds);
- state->x[12] += chacha_advance(bytes, 8);
- return;
- }
- if (bytes > CHACHA_BLOCK_SIZE * 2) {
+ else if (bytes > CHACHA_BLOCK_SIZE * 2)
chacha_4block_xor_avx2(state, dst, src, bytes, nrounds);
- state->x[12] += chacha_advance(bytes, 4);
- return;
- }
- if (bytes > CHACHA_BLOCK_SIZE) {
+ else if (bytes > CHACHA_BLOCK_SIZE)
chacha_2block_xor_avx2(state, dst, src, bytes, nrounds);
- state->x[12] += chacha_advance(bytes, 2);
- return;
+ else if (bytes)
+ chacha_block_xor_ssse3(state, dst, src, bytes, nrounds);
+ state->x[12] += DIV_ROUND_UP(bytes, CHACHA_BLOCK_SIZE);
+ } else {
+ while (bytes >= CHACHA_BLOCK_SIZE * 4) {
+ chacha_4block_xor_ssse3(state, dst, src,
+ CHACHA_BLOCK_SIZE * 4, nrounds);
+ bytes -= CHACHA_BLOCK_SIZE * 4;
+ src += CHACHA_BLOCK_SIZE * 4;
+ dst += CHACHA_BLOCK_SIZE * 4;
+ state->x[12] += 4;
}
- }
-
- while (bytes >= CHACHA_BLOCK_SIZE * 4) {
- chacha_4block_xor_ssse3(state, dst, src, bytes, nrounds);
- bytes -= CHACHA_BLOCK_SIZE * 4;
- src += CHACHA_BLOCK_SIZE * 4;
- dst += CHACHA_BLOCK_SIZE * 4;
- state->x[12] += 4;
- }
- if (bytes > CHACHA_BLOCK_SIZE) {
- chacha_4block_xor_ssse3(state, dst, src, bytes, nrounds);
- state->x[12] += chacha_advance(bytes, 4);
- return;
- }
- if (bytes) {
- chacha_block_xor_ssse3(state, dst, src, bytes, nrounds);
- state->x[12]++;
+ if (bytes > CHACHA_BLOCK_SIZE)
+ chacha_4block_xor_ssse3(state, dst, src, bytes,
+ nrounds);
+ else if (bytes)
+ chacha_block_xor_ssse3(state, dst, src, bytes, nrounds);
+ state->x[12] += DIV_ROUND_UP(bytes, CHACHA_BLOCK_SIZE);
}
}
base-commit: 9fbbbaefd5f95551da18426c200cf41e1b4255a1
prerequisite-patch-id: 890b33260038bff2e3adfe7867af5919666677fc
prerequisite-patch-id: 36cadf8c358a9778d745637c00368df82a93ce0d
prerequisite-patch-id: b9ef92f4abcdfe8946d645db12187f6293a46381
prerequisite-patch-id: 85904a5ef5aa8a48c311b2e87ed4d2403bd9e351
--
2.56.0
reply other threads:[~2026-11-04 8:16 UTC|newest]
Thread overview: [no followups] expand[flat|nested] mbox.gz Atom feed
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261104081604.302414-1-ebiggers@kernel.org \
--to=ebiggers@kernel.org \
--cc=Jason@zx2c4.com \
--cc=antonio@mandelbit.com \
--cc=ardb@kernel.org \
--cc=herbert@gondor.apana.org.au \
--cc=linux-crypto@vger.kernel.org \
--cc=martin@strongswan.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox