From: Ard Biesheuvel <ard.biesheuvel@linaro.org>
To: linux-kernel@vger.kernel.org
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>,
Dave Martin <Dave.Martin@arm.com>,
Russell King - ARM Linux <linux@armlinux.org.uk>,
Sebastian Andrzej Siewior <bigeasy@linutronix.de>,
Mark Rutland <mark.rutland@arm.com>,
linux-rt-users@vger.kernel.org,
Peter Zijlstra <peterz@infradead.org>,
Catalin Marinas <catalin.marinas@arm.com>,
Will Deacon <will.deacon@arm.com>,
Steven Rostedt <rostedt@goodmis.org>,
Thomas Gleixner <tglx@linutronix.de>
Subject: [PATCH v4 18/20] crypto: arm64/crc32-ce - yield NEON after every block of input
Date: Tue, 26 Dec 2017 10:29:38 +0000 [thread overview]
Message-ID: <20171226102940.26908-19-ard.biesheuvel@linaro.org> (raw)
In-Reply-To: <20171226102940.26908-1-ard.biesheuvel@linaro.org>
Avoid excessive scheduling delays under a preemptible kernel by
yielding the NEON after every block of input.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
arch/arm64/crypto/crc32-ce-core.S | 40 +++++++++++++++-----
1 file changed, 30 insertions(+), 10 deletions(-)
diff --git a/arch/arm64/crypto/crc32-ce-core.S b/arch/arm64/crypto/crc32-ce-core.S
index 18f5a8442276..29a601bb810e 100644
--- a/arch/arm64/crypto/crc32-ce-core.S
+++ b/arch/arm64/crypto/crc32-ce-core.S
@@ -100,9 +100,10 @@
dCONSTANT .req d0
qCONSTANT .req q0
- BUF .req x0
- LEN .req x1
- CRC .req x2
+ BUF .req x19
+ LEN .req x20
+ CRC .req x21
+ CONST .req x22
vzr .req v9
@@ -122,7 +123,14 @@ ENTRY(crc32_pmull_le)
ENTRY(crc32c_pmull_le)
adr x3, .Lcrc32c_constants
-0: bic LEN, LEN, #15
+0: frame_push 4, 64
+
+ mov BUF, x0
+ mov LEN, x1
+ mov CRC, x2
+ mov CONST, x3
+
+ bic LEN, LEN, #15
ld1 {v1.16b-v4.16b}, [BUF], #0x40
movi vzr.16b, #0
fmov dCONSTANT, CRC
@@ -131,7 +139,7 @@ ENTRY(crc32c_pmull_le)
cmp LEN, #0x40
b.lt less_64
- ldr qCONSTANT, [x3]
+ ldr qCONSTANT, [CONST]
loop_64: /* 64 bytes Full cache line folding */
sub LEN, LEN, #0x40
@@ -161,10 +169,21 @@ loop_64: /* 64 bytes Full cache line folding */
eor v4.16b, v4.16b, v8.16b
cmp LEN, #0x40
- b.ge loop_64
+ b.lt less_64
+
+ if_will_cond_yield_neon
+ stp q1, q2, [sp, #.Lframe_local_offset]
+ stp q3, q4, [sp, #.Lframe_local_offset + 32]
+ do_cond_yield_neon
+ ldp q1, q2, [sp, #.Lframe_local_offset]
+ ldp q3, q4, [sp, #.Lframe_local_offset + 32]
+ ldr qCONSTANT, [CONST]
+ movi vzr.16b, #0
+ endif_yield_neon
+ b loop_64
less_64: /* Folding cache line into 128bit */
- ldr qCONSTANT, [x3, #16]
+ ldr qCONSTANT, [CONST, #16]
pmull2 v5.1q, v1.2d, vCONSTANT.2d
pmull v1.1q, v1.1d, vCONSTANT.1d
@@ -203,8 +222,8 @@ fold_64:
eor v1.16b, v1.16b, v2.16b
/* final 32-bit fold */
- ldr dCONSTANT, [x3, #32]
- ldr d3, [x3, #40]
+ ldr dCONSTANT, [CONST, #32]
+ ldr d3, [CONST, #40]
ext v2.16b, v1.16b, vzr.16b, #4
and v1.16b, v1.16b, v3.16b
@@ -212,7 +231,7 @@ fold_64:
eor v1.16b, v1.16b, v2.16b
/* Finish up with the bit-reversed barrett reduction 64 ==> 32 bits */
- ldr qCONSTANT, [x3, #48]
+ ldr qCONSTANT, [CONST, #48]
and v2.16b, v1.16b, v3.16b
ext v2.16b, vzr.16b, v2.16b, #8
@@ -222,6 +241,7 @@ fold_64:
eor v1.16b, v1.16b, v2.16b
mov w0, v1.s[1]
+ frame_pop
ret
ENDPROC(crc32_pmull_le)
ENDPROC(crc32c_pmull_le)
--
2.11.0
next prev parent reply other threads:[~2017-12-26 10:29 UTC|newest]
Thread overview: 23+ messages / expand[flat|nested] mbox.gz Atom feed top
2017-12-26 10:29 [PATCH v4 00/20] crypto: arm64 - play nice with CONFIG_PREEMPT Ard Biesheuvel
2017-12-26 10:29 ` [PATCH v4 01/20] crypto: testmgr - add a new test case for CRC-T10DIF Ard Biesheuvel
2017-12-26 10:29 ` [PATCH v4 02/20] crypto: arm64/aes-ce-ccm - move kernel mode neon en/disable into loop Ard Biesheuvel
2017-12-26 10:29 ` [PATCH v4 03/20] crypto: arm64/aes-blk " Ard Biesheuvel
2017-12-26 10:29 ` [PATCH v4 04/20] crypto: arm64/aes-bs " Ard Biesheuvel
2017-12-26 10:29 ` [PATCH v4 05/20] crypto: arm64/chacha20 " Ard Biesheuvel
2017-12-26 10:29 ` [PATCH v4 06/20] crypto: arm64/aes-blk - remove configurable interleave Ard Biesheuvel
2017-12-26 10:29 ` [PATCH v4 07/20] crypto: arm64/aes-blk - add 4 way interleave to CBC encrypt path Ard Biesheuvel
2017-12-26 10:29 ` [PATCH v4 08/20] crypto: arm64/aes-blk - add 4 way interleave to CBC-MAC " Ard Biesheuvel
2017-12-26 10:29 ` [PATCH v4 09/20] crypto: arm64/sha256-neon - play nice with CONFIG_PREEMPT kernels Ard Biesheuvel
2017-12-26 10:29 ` [PATCH v4 10/20] arm64: assembler: add utility macros to push/pop stack frames Ard Biesheuvel
2017-12-26 10:29 ` [PATCH v4 11/20] arm64: assembler: add macros to conditionally yield the NEON under PREEMPT Ard Biesheuvel
2017-12-26 10:29 ` [PATCH v4 12/20] crypto: arm64/sha1-ce - yield NEON after every block of input Ard Biesheuvel
2017-12-26 10:29 ` [PATCH v4 13/20] crypto: arm64/sha2-ce " Ard Biesheuvel
2017-12-26 10:29 ` [PATCH v4 14/20] crypto: arm64/aes-ccm " Ard Biesheuvel
2017-12-26 10:29 ` [PATCH v4 15/20] crypto: arm64/aes-blk " Ard Biesheuvel
2017-12-26 10:29 ` [PATCH v4 16/20] crypto: arm64/aes-bs " Ard Biesheuvel
2017-12-26 10:29 ` [PATCH v4 17/20] crypto: arm64/aes-ghash " Ard Biesheuvel
2017-12-26 10:29 ` Ard Biesheuvel [this message]
2017-12-26 10:29 ` [PATCH v4 19/20] crypto: arm64/crct10dif-ce " Ard Biesheuvel
2017-12-26 10:29 ` [PATCH v4 20/20] DO NOT MERGE Ard Biesheuvel
2018-02-09 18:02 ` [PATCH v4 00/20] crypto: arm64 - play nice with CONFIG_PREEMPT Sebastian Andrzej Siewior
2018-02-09 19:28 ` Ard Biesheuvel
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20171226102940.26908-19-ard.biesheuvel@linaro.org \
--to=ard.biesheuvel@linaro.org \
--cc=Dave.Martin@arm.com \
--cc=bigeasy@linutronix.de \
--cc=catalin.marinas@arm.com \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-rt-users@vger.kernel.org \
--cc=linux@armlinux.org.uk \
--cc=mark.rutland@arm.com \
--cc=peterz@infradead.org \
--cc=rostedt@goodmis.org \
--cc=tglx@linutronix.de \
--cc=will.deacon@arm.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).