* [PATCH v6 1/3] drivers: crypto: Add Support for Octeon-tx CPT Engine
From: George Cherian @ 2017-02-07 14:51 UTC (permalink / raw)
To: herbert, davem
Cc: david.daney, clabbe.montjoie, smueller, levinsasha928,
linux-kernel, linux-crypto, George Cherian
In-Reply-To: <1486479075-1813-1-git-send-email-george.cherian@cavium.com>
Enable the Physical Function driver for the Cavium Crypto Engine (CPT)
found in Octeon-tx series of SoC's. CPT is the Cryptographic Accelaration
Unit. CPT includes microcoded GigaCypher symmetric engines (SEs) and
asymmetric engines (AEs).
Signed-off-by: George Cherian <george.cherian@cavium.com>
Reviewed-by: David Daney <david.daney@cavium.com>
---
drivers/crypto/cavium/cpt/Kconfig | 16 +
drivers/crypto/cavium/cpt/Makefile | 2 +
drivers/crypto/cavium/cpt/cpt_common.h | 158 +++++++
drivers/crypto/cavium/cpt/cpt_hw_types.h | 658 ++++++++++++++++++++++++++++
drivers/crypto/cavium/cpt/cptpf.h | 69 +++
drivers/crypto/cavium/cpt/cptpf_main.c | 708 +++++++++++++++++++++++++++++++
drivers/crypto/cavium/cpt/cptpf_mbox.c | 163 +++++++
7 files changed, 1774 insertions(+)
create mode 100644 drivers/crypto/cavium/cpt/Kconfig
create mode 100644 drivers/crypto/cavium/cpt/Makefile
create mode 100644 drivers/crypto/cavium/cpt/cpt_common.h
create mode 100644 drivers/crypto/cavium/cpt/cpt_hw_types.h
create mode 100644 drivers/crypto/cavium/cpt/cptpf.h
create mode 100644 drivers/crypto/cavium/cpt/cptpf_main.c
create mode 100644 drivers/crypto/cavium/cpt/cptpf_mbox.c
diff --git a/drivers/crypto/cavium/cpt/Kconfig b/drivers/crypto/cavium/cpt/Kconfig
new file mode 100644
index 0000000..247f1cb
--- /dev/null
+++ b/drivers/crypto/cavium/cpt/Kconfig
@@ -0,0 +1,16 @@
+#
+# Cavium crypto device configuration
+#
+
+config CRYPTO_DEV_CPT
+ tristate
+
+config CAVIUM_CPT
+ tristate "Cavium Cryptographic Accelerator driver"
+ depends on ARCH_THUNDER
+ select CRYPTO_DEV_CPT
+ help
+ Support for Cavium CPT block found in octeon-tx series of
+ processors.
+
+ To compile this as a module, choose M here.
diff --git a/drivers/crypto/cavium/cpt/Makefile b/drivers/crypto/cavium/cpt/Makefile
new file mode 100644
index 0000000..fe3d454
--- /dev/null
+++ b/drivers/crypto/cavium/cpt/Makefile
@@ -0,0 +1,2 @@
+obj-$(CONFIG_CAVIUM_CPT) += cptpf.o
+cptpf-objs := cptpf_main.o cptpf_mbox.o
diff --git a/drivers/crypto/cavium/cpt/cpt_common.h b/drivers/crypto/cavium/cpt/cpt_common.h
new file mode 100644
index 0000000..ede612f
--- /dev/null
+++ b/drivers/crypto/cavium/cpt/cpt_common.h
@@ -0,0 +1,158 @@
+/*
+ * Copyright (C) 2016 Cavium, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License
+ * as published by the Free Software Foundation.
+ */
+
+#ifndef __CPT_COMMON_H
+#define __CPT_COMMON_H
+
+#include <asm/byteorder.h>
+#include <linux/delay.h>
+#include <linux/pci.h>
+
+#include "cpt_hw_types.h"
+
+/* Device ID */
+#define CPT_81XX_PCI_PF_DEVICE_ID 0xa040
+#define CPT_81XX_PCI_VF_DEVICE_ID 0xa041
+
+/* flags to indicate the features supported */
+#define CPT_FLAG_MSIX_ENABLED BIT(0)
+#define CPT_FLAG_SRIOV_ENABLED BIT(1)
+#define CPT_FLAG_VF_DRIVER BIT(2)
+#define CPT_FLAG_DEVICE_READY BIT(3)
+
+#define cpt_msix_enabled(cpt) ((cpt)->flags & CPT_FLAG_MSIX_ENABLED)
+#define cpt_sriov_enabled(cpt) ((cpt)->flags & CPT_FLAG_SRIOV_ENABLED)
+#define cpt_vf_driver(cpt) ((cpt)->flags & CPT_FLAG_VF_DRIVER)
+#define cpt_device_ready(cpt) ((cpt)->flags & CPT_FLAG_DEVICE_READY)
+
+#define CPT_MBOX_MSG_TYPE_ACK 1
+#define CPT_MBOX_MSG_TYPE_NACK 2
+#define CPT_MBOX_MSG_TIMEOUT 2000
+#define VF_STATE_DOWN 0
+#define VF_STATE_UP 1
+
+/*
+ * CPT Registers map for 81xx
+ */
+
+/* PF registers */
+#define CPTX_PF_CONSTANTS(a) (0x0ll + ((u64)(a) << 36))
+#define CPTX_PF_RESET(a) (0x100ll + ((u64)(a) << 36))
+#define CPTX_PF_DIAG(a) (0x120ll + ((u64)(a) << 36))
+#define CPTX_PF_BIST_STATUS(a) (0x160ll + ((u64)(a) << 36))
+#define CPTX_PF_ECC0_CTL(a) (0x200ll + ((u64)(a) << 36))
+#define CPTX_PF_ECC0_FLIP(a) (0x210ll + ((u64)(a) << 36))
+#define CPTX_PF_ECC0_INT(a) (0x220ll + ((u64)(a) << 36))
+#define CPTX_PF_ECC0_INT_W1S(a) (0x230ll + ((u64)(a) << 36))
+#define CPTX_PF_ECC0_ENA_W1S(a) (0x240ll + ((u64)(a) << 36))
+#define CPTX_PF_ECC0_ENA_W1C(a) (0x250ll + ((u64)(a) << 36))
+#define CPTX_PF_MBOX_INTX(a, b) \
+ (0x400ll + ((u64)(a) << 36) + ((b) << 3))
+#define CPTX_PF_MBOX_INT_W1SX(a, b) \
+ (0x420ll + ((u64)(a) << 36) + ((b) << 3))
+#define CPTX_PF_MBOX_ENA_W1CX(a, b) \
+ (0x440ll + ((u64)(a) << 36) + ((b) << 3))
+#define CPTX_PF_MBOX_ENA_W1SX(a, b) \
+ (0x460ll + ((u64)(a) << 36) + ((b) << 3))
+#define CPTX_PF_EXEC_INT(a) (0x500ll + 0x1000000000ll * ((a) & 0x1))
+#define CPTX_PF_EXEC_INT_W1S(a) (0x520ll + ((u64)(a) << 36))
+#define CPTX_PF_EXEC_ENA_W1C(a) (0x540ll + ((u64)(a) << 36))
+#define CPTX_PF_EXEC_ENA_W1S(a) (0x560ll + ((u64)(a) << 36))
+#define CPTX_PF_GX_EN(a, b) \
+ (0x600ll + ((u64)(a) << 36) + ((b) << 3))
+#define CPTX_PF_EXEC_INFO(a) (0x700ll + ((u64)(a) << 36))
+#define CPTX_PF_EXEC_BUSY(a) (0x800ll + ((u64)(a) << 36))
+#define CPTX_PF_EXEC_INFO0(a) (0x900ll + ((u64)(a) << 36))
+#define CPTX_PF_EXEC_INFO1(a) (0x910ll + ((u64)(a) << 36))
+#define CPTX_PF_INST_REQ_PC(a) (0x10000ll + ((u64)(a) << 36))
+#define CPTX_PF_INST_LATENCY_PC(a) \
+ (0x10020ll + ((u64)(a) << 36))
+#define CPTX_PF_RD_REQ_PC(a) (0x10040ll + ((u64)(a) << 36))
+#define CPTX_PF_RD_LATENCY_PC(a) (0x10060ll + ((u64)(a) << 36))
+#define CPTX_PF_RD_UC_PC(a) (0x10080ll + ((u64)(a) << 36))
+#define CPTX_PF_ACTIVE_CYCLES_PC(a) (0x10100ll + ((u64)(a) << 36))
+#define CPTX_PF_EXE_CTL(a) (0x4000000ll + ((u64)(a) << 36))
+#define CPTX_PF_EXE_STATUS(a) (0x4000008ll + ((u64)(a) << 36))
+#define CPTX_PF_EXE_CLK(a) (0x4000010ll + ((u64)(a) << 36))
+#define CPTX_PF_EXE_DBG_CTL(a) (0x4000018ll + ((u64)(a) << 36))
+#define CPTX_PF_EXE_DBG_DATA(a) (0x4000020ll + ((u64)(a) << 36))
+#define CPTX_PF_EXE_BIST_STATUS(a) (0x4000028ll + ((u64)(a) << 36))
+#define CPTX_PF_EXE_REQ_TIMER(a) (0x4000030ll + ((u64)(a) << 36))
+#define CPTX_PF_EXE_MEM_CTL(a) (0x4000038ll + ((u64)(a) << 36))
+#define CPTX_PF_EXE_PERF_CTL(a) (0x4001000ll + ((u64)(a) << 36))
+#define CPTX_PF_EXE_DBG_CNTX(a, b) \
+ (0x4001100ll + ((u64)(a) << 36) + ((b) << 3))
+#define CPTX_PF_EXE_PERF_EVENT_CNT(a) (0x4001180ll + ((u64)(a) << 36))
+#define CPTX_PF_EXE_EPCI_INBX_CNT(a, b) \
+ (0x4001200ll + ((u64)(a) << 36) + ((b) << 3))
+#define CPTX_PF_EXE_EPCI_OUTBX_CNT(a, b) \
+ (0x4001240ll + ((u64)(a) << 36) + ((b) << 3))
+#define CPTX_PF_ENGX_UCODE_BASE(a, b) \
+ (0x4002000ll + ((u64)(a) << 36) + ((b) << 3))
+#define CPTX_PF_QX_CTL(a, b) \
+ (0x8000000ll + ((u64)(a) << 36) + ((b) << 20))
+#define CPTX_PF_QX_GMCTL(a, b) \
+ (0x8000020ll + ((u64)(a) << 36) + ((b) << 20))
+#define CPTX_PF_QX_CTL2(a, b) \
+ (0x8000100ll + ((u64)(a) << 36) + ((b) << 20))
+#define CPTX_PF_VFX_MBOXX(a, b, c) \
+ (0x8001000ll + ((u64)(a) << 36) + ((b) << 20) + ((c) << 8))
+
+/* VF registers */
+#define CPTX_VQX_CTL(a, b) (0x100ll + ((u64)(a) << 36) + ((b) << 20))
+#define CPTX_VQX_SADDR(a, b) (0x200ll + ((u64)(a) << 36) + ((b) << 20))
+#define CPTX_VQX_DONE_WAIT(a, b) (0x400ll + ((u64)(a) << 36) + ((b) << 20))
+#define CPTX_VQX_INPROG(a, b) (0x410ll + ((u64)(a) << 36) + ((b) << 20))
+#define CPTX_VQX_DONE(a, b) (0x420ll + ((u64)(a) << 36) + ((b) << 20))
+#define CPTX_VQX_DONE_ACK(a, b) (0x440ll + ((u64)(a) << 36) + ((b) << 20))
+#define CPTX_VQX_DONE_INT_W1S(a, b) (0x460ll + ((u64)(a) << 36) + ((b) << 20))
+#define CPTX_VQX_DONE_INT_W1C(a, b) (0x468ll + ((u64)(a) << 36) + ((b) << 20))
+#define CPTX_VQX_DONE_ENA_W1S(a, b) (0x470ll + ((u64)(a) << 36) + ((b) << 20))
+#define CPTX_VQX_DONE_ENA_W1C(a, b) (0x478ll + ((u64)(a) << 36) + ((b) << 20))
+#define CPTX_VQX_MISC_INT(a, b) (0x500ll + ((u64)(a) << 36) + ((b) << 20))
+#define CPTX_VQX_MISC_INT_W1S(a, b) (0x508ll + ((u64)(a) << 36) + ((b) << 20))
+#define CPTX_VQX_MISC_ENA_W1S(a, b) (0x510ll + ((u64)(a) << 36) + ((b) << 20))
+#define CPTX_VQX_MISC_ENA_W1C(a, b) (0x518ll + ((u64)(a) << 36) + ((b) << 20))
+#define CPTX_VQX_DOORBELL(a, b) (0x600ll + ((u64)(a) << 36) + ((b) << 20))
+#define CPTX_VFX_PF_MBOXX(a, b, c) \
+ (0x1000ll + ((u64)(a) << 36) + ((b) << 20) + ((c) << 3))
+
+enum vftype {
+ AE_TYPES = 1,
+ SE_TYPES = 2,
+ BAD_CPT_TYPES,
+};
+
+/* Max CPT devices supported */
+enum cpt_mbox_opcode {
+ CPT_MSG_VF_UP = 1,
+ CPT_MSG_VF_DOWN,
+ CPT_MSG_READY,
+ CPT_MSG_QLEN,
+ CPT_MSG_QBIND_GRP,
+ CPT_MSG_VQ_PRIORITY,
+};
+
+/* CPT mailbox structure */
+struct cpt_mbox {
+ u64 msg; /* Message type MBOX[0] */
+ u64 data;/* Data MBOX[1] */
+};
+
+/* Register read/write APIs */
+static inline void cpt_write_csr64(u8 __iomem *hw_addr, u64 offset,
+ u64 val)
+{
+ writeq(val, hw_addr + offset);
+}
+
+static inline u64 cpt_read_csr64(u8 __iomem *hw_addr, u64 offset)
+{
+ return readq(hw_addr + offset);
+}
+#endif /* __CPT_COMMON_H */
diff --git a/drivers/crypto/cavium/cpt/cpt_hw_types.h b/drivers/crypto/cavium/cpt/cpt_hw_types.h
new file mode 100644
index 0000000..2796694
--- /dev/null
+++ b/drivers/crypto/cavium/cpt/cpt_hw_types.h
@@ -0,0 +1,658 @@
+/*
+ * Copyright (C) 2016 Cavium, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License
+ * as published by the Free Software Foundation.
+ */
+
+#ifndef __CPT_HW_TYPES_H
+#define __CPT_HW_TYPES_H
+
+#include "cpt_common.h"
+
+/**
+ * Enumeration cpt_comp_e
+ *
+ * CPT Completion Enumeration
+ * Enumerates the values of CPT_RES_S[COMPCODE].
+ */
+enum cpt_comp_e {
+ CPT_COMP_E_NOTDONE = 0x00,
+ CPT_COMP_E_GOOD = 0x01,
+ CPT_COMP_E_FAULT = 0x02,
+ CPT_COMP_E_SWERR = 0x03,
+ CPT_COMP_E_LAST_ENTRY = 0xFF
+};
+
+/**
+ * Structure cpt_inst_s
+ *
+ * CPT Instruction Structure
+ * This structure specifies the instruction layout. Instructions are
+ * stored in memory as little-endian unless CPT()_PF_Q()_CTL[INST_BE] is set.
+ * cpt_inst_s_s
+ * Word 0
+ * doneint:1 Done interrupt.
+ * 0 = No interrupts related to this instruction.
+ * 1 = When the instruction completes, CPT()_VQ()_DONE[DONE] will be
+ * incremented,and based on the rules described there an interrupt may
+ * occur.
+ * Word 1
+ * res_addr [127: 64] Result IOVA.
+ * If nonzero, specifies where to write CPT_RES_S.
+ * If zero, no result structure will be written.
+ * Address must be 16-byte aligned.
+ * Bits <63:49> are ignored by hardware; software should use a
+ * sign-extended bit <48> for forward compatibility.
+ * Word 2
+ * grp:10 [171:162] If [WQ_PTR] is nonzero, the SSO guest-group to use when
+ * CPT submits work SSO.
+ * For the SSO to not discard the add-work request, FPA_PF_MAP() must map
+ * [GRP] and CPT()_PF_Q()_GMCTL[GMID] as valid.
+ * tt:2 [161:160] If [WQ_PTR] is nonzero, the SSO tag type to use when CPT
+ * submits work to SSO
+ * tag:32 [159:128] If [WQ_PTR] is nonzero, the SSO tag to use when CPT
+ * submits work to SSO.
+ * Word 3
+ * wq_ptr [255:192] If [WQ_PTR] is nonzero, it is a pointer to a
+ * work-queue entry that CPT submits work to SSO after all context,
+ * output data, and result write operations are visible to other
+ * CNXXXX units and the cores. Bits <2:0> must be zero.
+ * Bits <63:49> are ignored by hardware; software should
+ * use a sign-extended bit <48> for forward compatibility.
+ * Internal:
+ * Bits <63:49>, <2:0> are ignored by hardware, treated as always 0x0.
+ * Word 4
+ * ei0; [319:256] Engine instruction word 0. Passed to the AE/SE.
+ * Word 5
+ * ei1; [383:320] Engine instruction word 1. Passed to the AE/SE.
+ * Word 6
+ * ei2; [447:384] Engine instruction word 1. Passed to the AE/SE.
+ * Word 7
+ * ei3; [511:448] Engine instruction word 1. Passed to the AE/SE.
+ *
+ */
+union cpt_inst_s {
+ u64 u[8];
+ struct cpt_inst_s_s {
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
+ u64 reserved_17_63:47;
+ u64 doneint:1;
+ u64 reserved_0_1:16;
+#else /* Word 0 - Little Endian */
+ u64 reserved_0_15:16;
+ u64 doneint:1;
+ u64 reserved_17_63:47;
+#endif /* Word 0 - End */
+ u64 res_addr;
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 2 - Big Endian */
+ u64 reserved_172_19:20;
+ u64 grp:10;
+ u64 tt:2;
+ u64 tag:32;
+#else /* Word 2 - Little Endian */
+ u64 tag:32;
+ u64 tt:2;
+ u64 grp:10;
+ u64 reserved_172_191:20;
+#endif /* Word 2 - End */
+ u64 wq_ptr;
+ u64 ei0;
+ u64 ei1;
+ u64 ei2;
+ u64 ei3;
+ } s;
+};
+
+/**
+ * Structure cpt_res_s
+ *
+ * CPT Result Structure
+ * The CPT coprocessor writes the result structure after it completes a
+ * CPT_INST_S instruction. The result structure is exactly 16 bytes, and
+ * each instruction completion produces exactly one result structure.
+ *
+ * This structure is stored in memory as little-endian unless
+ * CPT()_PF_Q()_CTL[INST_BE] is set.
+ * cpt_res_s_s
+ * Word 0
+ * doneint:1 [16:16] Done interrupt. This bit is copied from the
+ * corresponding instruction's CPT_INST_S[DONEINT].
+ * compcode:8 [7:0] Indicates completion/error status of the CPT coprocessor
+ * for the associated instruction, as enumerated by CPT_COMP_E.
+ * Core software may write the memory location containing [COMPCODE] to
+ * 0x0 before ringing the doorbell, and then poll for completion by
+ * checking for a nonzero value.
+ * Once the core observes a nonzero [COMPCODE] value in this case,the CPT
+ * coprocessor will have also completed L2/DRAM write operations.
+ * Word 1
+ * reserved
+ *
+ */
+union cpt_res_s {
+ u64 u[2];
+ struct cpt_res_s_s {
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
+ u64 reserved_17_63:47;
+ u64 doneint:1;
+ u64 reserved_8_15:8;
+ u64 compcode:8;
+#else /* Word 0 - Little Endian */
+ u64 compcode:8;
+ u64 reserved_8_15:8;
+ u64 doneint:1;
+ u64 reserved_17_63:47;
+#endif /* Word 0 - End */
+ u64 reserved_64_127;
+ } s;
+};
+
+/**
+ * Register (NCB) cpt#_pf_bist_status
+ *
+ * CPT PF Control Bist Status Register
+ * This register has the BIST status of memories. Each bit is the BIST result
+ * of an individual memory (per bit, 0 = pass and 1 = fail).
+ * cptx_pf_bist_status_s
+ * Word0
+ * bstatus [29:0](RO/H) BIST status. One bit per memory, enumerated by
+ * CPT_RAMS_E.
+ */
+union cptx_pf_bist_status {
+ u64 u;
+ struct cptx_pf_bist_status_s {
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
+ u64 reserved_30_63:34;
+ u64 bstatus:30;
+#else /* Word 0 - Little Endian */
+ u64 bstatus:30;
+ u64 reserved_30_63:34;
+#endif /* Word 0 - End */
+ } s;
+};
+
+/**
+ * Register (NCB) cpt#_pf_constants
+ *
+ * CPT PF Constants Register
+ * This register contains implementation-related parameters of CPT in CNXXXX.
+ * cptx_pf_constants_s
+ * Word 0
+ * reserved_40_63:24 [63:40] Reserved.
+ * epcis:8 [39:32](RO) Number of EPCI busses.
+ * grps:8 [31:24](RO) Number of engine groups implemented.
+ * ae:8 [23:16](RO/H) Number of AEs. In CNXXXX, for CPT0 returns 0x0,
+ * for CPT1 returns 0x18, or less if there are fuse-disables.
+ * se:8 [15:8](RO/H) Number of SEs. In CNXXXX, for CPT0 returns 0x30,
+ * or less if there are fuse-disables, for CPT1 returns 0x0.
+ * vq:8 [7:0](RO) Number of VQs.
+ */
+union cptx_pf_constants {
+ u64 u;
+ struct cptx_pf_constants_s {
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
+ u64 reserved_40_63:24;
+ u64 epcis:8;
+ u64 grps:8;
+ u64 ae:8;
+ u64 se:8;
+ u64 vq:8;
+#else /* Word 0 - Little Endian */
+ u64 vq:8;
+ u64 se:8;
+ u64 ae:8;
+ u64 grps:8;
+ u64 epcis:8;
+ u64 reserved_40_63:24;
+#endif /* Word 0 - End */
+ } s;
+};
+
+/**
+ * Register (NCB) cpt#_pf_exe_bist_status
+ *
+ * CPT PF Engine Bist Status Register
+ * This register has the BIST status of each engine. Each bit is the
+ * BIST result of an individual engine (per bit, 0 = pass and 1 = fail).
+ * cptx_pf_exe_bist_status_s
+ * Word0
+ * reserved_48_63:16 [63:48] reserved
+ * bstatus:48 [47:0](RO/H) BIST status. One bit per engine.
+ *
+ */
+union cptx_pf_exe_bist_status {
+ u64 u;
+ struct cptx_pf_exe_bist_status_s {
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
+ u64 reserved_48_63:16;
+ u64 bstatus:48;
+#else /* Word 0 - Little Endian */
+ u64 bstatus:48;
+ u64 reserved_48_63:16;
+#endif /* Word 0 - End */
+ } s;
+};
+
+/**
+ * Register (NCB) cpt#_pf_q#_ctl
+ *
+ * CPT Queue Control Register
+ * This register configures queues. This register should be changed only
+ * when quiescent (see CPT()_VQ()_INPROG[INFLIGHT]).
+ * cptx_pf_qx_ctl_s
+ * Word0
+ * reserved_60_63:4 [63:60] reserved.
+ * aura:12; [59:48](R/W) Guest-aura for returning this queue's
+ * instruction-chunk buffers to FPA. Only used when [INST_FREE] is set.
+ * For the FPA to not discard the request, FPA_PF_MAP() must map
+ * [AURA] and CPT()_PF_Q()_GMCTL[GMID] as valid.
+ * reserved_45_47:3 [47:45] reserved.
+ * size:13 [44:32](R/W) Command-buffer size, in number of 64-bit words per
+ * command buffer segment. Must be 8*n + 1, where n is the number of
+ * instructions per buffer segment.
+ * reserved_11_31:21 [31:11] Reserved.
+ * cont_err:1 [10:10](R/W) Continue on error.
+ * 0 = When CPT()_VQ()_MISC_INT[NWRP], CPT()_VQ()_MISC_INT[IRDE] or
+ * CPT()_VQ()_MISC_INT[DOVF] are set by hardware or software via
+ * CPT()_VQ()_MISC_INT_W1S, then CPT()_VQ()_CTL[ENA] is cleared. Due to
+ * pipelining, additional instructions may have been processed between the
+ * instruction causing the error and the next instruction in the disabled
+ * queue (the instruction at CPT()_VQ()_SADDR).
+ * 1 = Ignore errors and continue processing instructions.
+ * For diagnostic use only.
+ * inst_free:1 [9:9](R/W) Instruction FPA free. When set, when CPT reaches the
+ * end of an instruction chunk, that chunk will be freed to the FPA.
+ * inst_be:1 [8:8](R/W) Instruction big-endian control. When set, instructions,
+ * instruction next chunk pointers, and result structures are stored in
+ * big-endian format in memory.
+ * iqb_ldwb:1 [7:7](R/W) Instruction load don't write back.
+ * 0 = The hardware issues NCB transient load (LDT) towards the cache,
+ * which if the line hits and is is dirty will cause the line to be
+ * written back before being replaced.
+ * 1 = The hardware issues NCB LDWB read-and-invalidate command towards
+ * the cache when fetching the last word of instructions; as a result the
+ * line will not be written back when replaced. This improves
+ * performance, but software must not read the instructions after they are
+ * posted to the hardware. Reads that do not consume the last word of a
+ * cache line always use LDI.
+ * reserved_4_6:3 [6:4] Reserved.
+ * grp:3; [3:1](R/W) Engine group.
+ * pri:1; [0:0](R/W) Queue priority.
+ * 1 = This queue has higher priority. Round-robin between higher
+ * priority queues.
+ * 0 = This queue has lower priority. Round-robin between lower
+ * priority queues.
+ */
+union cptx_pf_qx_ctl {
+ u64 u;
+ struct cptx_pf_qx_ctl_s {
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
+ u64 reserved_60_63:4;
+ u64 aura:12;
+ u64 reserved_45_47:3;
+ u64 size:13;
+ u64 reserved_11_31:21;
+ u64 cont_err:1;
+ u64 inst_free:1;
+ u64 inst_be:1;
+ u64 iqb_ldwb:1;
+ u64 reserved_4_6:3;
+ u64 grp:3;
+ u64 pri:1;
+#else /* Word 0 - Little Endian */
+ u64 pri:1;
+ u64 grp:3;
+ u64 reserved_4_6:3;
+ u64 iqb_ldwb:1;
+ u64 inst_be:1;
+ u64 inst_free:1;
+ u64 cont_err:1;
+ u64 reserved_11_31:21;
+ u64 size:13;
+ u64 reserved_45_47:3;
+ u64 aura:12;
+ u64 reserved_60_63:4;
+#endif /* Word 0 - End */
+ } s;
+};
+
+/**
+ * Register (NCB) cpt#_vq#_saddr
+ *
+ * CPT Queue Starting Buffer Address Registers
+ * These registers set the instruction buffer starting address.
+ * cptx_vqx_saddr_s
+ * Word0
+ * reserved_49_63:15 [63:49] Reserved.
+ * ptr:43 [48:6](R/W/H) Instruction buffer IOVA <48:6> (64-byte aligned).
+ * When written, it is the initial buffer starting address; when read,
+ * it is the next read pointer to be requested from L2C. The PTR field
+ * is overwritten with the next pointer each time that the command buffer
+ * segment is exhausted. New commands will then be read from the newly
+ * specified command buffer pointer.
+ * reserved_0_5:6 [5:0] Reserved.
+ *
+ */
+union cptx_vqx_saddr {
+ u64 u;
+ struct cptx_vqx_saddr_s {
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
+ u64 reserved_49_63:15;
+ u64 ptr:43;
+ u64 reserved_0_5:6;
+#else /* Word 0 - Little Endian */
+ u64 reserved_0_5:6;
+ u64 ptr:43;
+ u64 reserved_49_63:15;
+#endif /* Word 0 - End */
+ } s;
+};
+
+/**
+ * Register (NCB) cpt#_vq#_misc_ena_w1s
+ *
+ * CPT Queue Misc Interrupt Enable Set Register
+ * This register sets interrupt enable bits.
+ * cptx_vqx_misc_ena_w1s_s
+ * Word0
+ * reserved_5_63:59 [63:5] Reserved.
+ * swerr:1 [4:4](R/W1S/H) Reads or sets enable for
+ * CPT(0..1)_VQ(0..63)_MISC_INT[SWERR].
+ * nwrp:1 [3:3](R/W1S/H) Reads or sets enable for
+ * CPT(0..1)_VQ(0..63)_MISC_INT[NWRP].
+ * irde:1 [2:2](R/W1S/H) Reads or sets enable for
+ * CPT(0..1)_VQ(0..63)_MISC_INT[IRDE].
+ * dovf:1 [1:1](R/W1S/H) Reads or sets enable for
+ * CPT(0..1)_VQ(0..63)_MISC_INT[DOVF].
+ * mbox:1 [0:0](R/W1S/H) Reads or sets enable for
+ * CPT(0..1)_VQ(0..63)_MISC_INT[MBOX].
+ *
+ */
+union cptx_vqx_misc_ena_w1s {
+ u64 u;
+ struct cptx_vqx_misc_ena_w1s_s {
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
+ u64 reserved_5_63:59;
+ u64 swerr:1;
+ u64 nwrp:1;
+ u64 irde:1;
+ u64 dovf:1;
+ u64 mbox:1;
+#else /* Word 0 - Little Endian */
+ u64 mbox:1;
+ u64 dovf:1;
+ u64 irde:1;
+ u64 nwrp:1;
+ u64 swerr:1;
+ u64 reserved_5_63:59;
+#endif /* Word 0 - End */
+ } s;
+};
+
+/**
+ * Register (NCB) cpt#_vq#_doorbell
+ *
+ * CPT Queue Doorbell Registers
+ * Doorbells for the CPT instruction queues.
+ * cptx_vqx_doorbell_s
+ * Word0
+ * reserved_20_63:44 [63:20] Reserved.
+ * dbell_cnt:20 [19:0](R/W/H) Number of instruction queue 64-bit words to add
+ * to the CPT instruction doorbell count. Readback value is the the
+ * current number of pending doorbell requests. If counter overflows
+ * CPT()_VQ()_MISC_INT[DBELL_DOVF] is set. To reset the count back to
+ * zero, write one to clear CPT()_VQ()_MISC_INT_ENA_W1C[DBELL_DOVF],
+ * then write a value of 2^20 minus the read [DBELL_CNT], then write one
+ * to CPT()_VQ()_MISC_INT_W1C[DBELL_DOVF] and
+ * CPT()_VQ()_MISC_INT_ENA_W1S[DBELL_DOVF]. Must be a multiple of 8.
+ * All CPT instructions are 8 words and require a doorbell count of
+ * multiple of 8.
+ */
+union cptx_vqx_doorbell {
+ u64 u;
+ struct cptx_vqx_doorbell_s {
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
+ u64 reserved_20_63:44;
+ u64 dbell_cnt:20;
+#else /* Word 0 - Little Endian */
+ u64 dbell_cnt:20;
+ u64 reserved_20_63:44;
+#endif /* Word 0 - End */
+ } s;
+};
+
+/**
+ * Register (NCB) cpt#_vq#_inprog
+ *
+ * CPT Queue In Progress Count Registers
+ * These registers contain the per-queue instruction in flight registers.
+ * cptx_vqx_inprog_s
+ * Word0
+ * reserved_8_63:56 [63:8] Reserved.
+ * inflight:8 [7:0](RO/H) Inflight count. Counts the number of instructions
+ * for the VF for which CPT is fetching, executing or responding to
+ * instructions. However this does not include any interrupts that are
+ * awaiting software handling (CPT()_VQ()_DONE[DONE] != 0x0).
+ * A queue may not be reconfigured until:
+ * 1. CPT()_VQ()_CTL[ENA] is cleared by software.
+ * 2. [INFLIGHT] is polled until equals to zero.
+ */
+union cptx_vqx_inprog {
+ u64 u;
+ struct cptx_vqx_inprog_s {
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
+ u64 reserved_8_63:56;
+ u64 inflight:8;
+#else /* Word 0 - Little Endian */
+ u64 inflight:8;
+ u64 reserved_8_63:56;
+#endif /* Word 0 - End */
+ } s;
+};
+
+/**
+ * Register (NCB) cpt#_vq#_misc_int
+ *
+ * CPT Queue Misc Interrupt Register
+ * These registers contain the per-queue miscellaneous interrupts.
+ * cptx_vqx_misc_int_s
+ * Word 0
+ * reserved_5_63:59 [63:5] Reserved.
+ * swerr:1 [4:4](R/W1C/H) Software error from engines.
+ * nwrp:1 [3:3](R/W1C/H) NCB result write response error.
+ * irde:1 [2:2](R/W1C/H) Instruction NCB read response error.
+ * dovf:1 [1:1](R/W1C/H) Doorbell overflow.
+ * mbox:1 [0:0](R/W1C/H) PF to VF mailbox interrupt. Set when
+ * CPT()_VF()_PF_MBOX(0) is written.
+ *
+ */
+union cptx_vqx_misc_int {
+ u64 u;
+ struct cptx_vqx_misc_int_s {
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
+ u64 reserved_5_63:59;
+ u64 swerr:1;
+ u64 nwrp:1;
+ u64 irde:1;
+ u64 dovf:1;
+ u64 mbox:1;
+#else /* Word 0 - Little Endian */
+ u64 mbox:1;
+ u64 dovf:1;
+ u64 irde:1;
+ u64 nwrp:1;
+ u64 swerr:1;
+ u64 reserved_5_63:59;
+#endif /* Word 0 - End */
+ } s;
+};
+
+/**
+ * Register (NCB) cpt#_vq#_done_ack
+ *
+ * CPT Queue Done Count Ack Registers
+ * This register is written by software to acknowledge interrupts.
+ * cptx_vqx_done_ack_s
+ * Word0
+ * reserved_20_63:44 [63:20] Reserved.
+ * done_ack:20 [19:0](R/W/H) Number of decrements to CPT()_VQ()_DONE[DONE].
+ * Reads CPT()_VQ()_DONE[DONE]. Written by software to acknowledge
+ * interrupts. If CPT()_VQ()_DONE[DONE] is still nonzero the interrupt
+ * will be re-sent if the conditions described in CPT()_VQ()_DONE[DONE]
+ * are satisfied.
+ *
+ */
+union cptx_vqx_done_ack {
+ u64 u;
+ struct cptx_vqx_done_ack_s {
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
+ u64 reserved_20_63:44;
+ u64 done_ack:20;
+#else /* Word 0 - Little Endian */
+ u64 done_ack:20;
+ u64 reserved_20_63:44;
+#endif /* Word 0 - End */
+ } s;
+};
+
+/**
+ * Register (NCB) cpt#_vq#_done
+ *
+ * CPT Queue Done Count Registers
+ * These registers contain the per-queue instruction done count.
+ * cptx_vqx_done_s
+ * Word0
+ * reserved_20_63:44 [63:20] Reserved.
+ * done:20 [19:0](R/W/H) Done count. When CPT_INST_S[DONEINT] set and that
+ * instruction completes, CPT()_VQ()_DONE[DONE] is incremented when the
+ * instruction finishes. Write to this field are for diagnostic use only;
+ * instead software writes CPT()_VQ()_DONE_ACK with the number of
+ * decrements for this field.
+ * Interrupts are sent as follows:
+ * * When CPT()_VQ()_DONE[DONE] = 0, then no results are pending, the
+ * interrupt coalescing timer is held to zero, and an interrupt is not
+ * sent.
+ * * When CPT()_VQ()_DONE[DONE] != 0, then the interrupt coalescing timer
+ * counts. If the counter is >= CPT()_VQ()_DONE_WAIT[TIME_WAIT]*1024, or
+ * CPT()_VQ()_DONE[DONE] >= CPT()_VQ()_DONE_WAIT[NUM_WAIT], i.e. enough
+ * time has passed or enough results have arrived, then the interrupt is
+ * sent.
+ * * When CPT()_VQ()_DONE_ACK is written (or CPT()_VQ()_DONE is written
+ * but this is not typical), the interrupt coalescing timer restarts.
+ * Note after decrementing this interrupt equation is recomputed,
+ * for example if CPT()_VQ()_DONE[DONE] >= CPT()_VQ()_DONE_WAIT[NUM_WAIT]
+ * and because the timer is zero, the interrupt will be resent immediately.
+ * (This covers the race case between software acknowledging an interrupt
+ * and a result returning.)
+ * * When CPT()_VQ()_DONE_ENA_W1S[DONE] = 0, interrupts are not sent,
+ * but the counting described above still occurs.
+ * Since CPT instructions complete out-of-order, if software is using
+ * completion interrupts the suggested scheme is to request a DONEINT on
+ * each request, and when an interrupt arrives perform a "greedy" scan for
+ * completions; even if a later command is acknowledged first this will
+ * not result in missing a completion.
+ * Software is responsible for making sure [DONE] does not overflow;
+ * for example by insuring there are not more than 2^20-1 instructions in
+ * flight that may request interrupts.
+ *
+ */
+union cptx_vqx_done {
+ u64 u;
+ struct cptx_vqx_done_s {
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
+ u64 reserved_20_63:44;
+ u64 done:20;
+#else /* Word 0 - Little Endian */
+ u64 done:20;
+ u64 reserved_20_63:44;
+#endif /* Word 0 - End */
+ } s;
+};
+
+/**
+ * Register (NCB) cpt#_vq#_done_wait
+ *
+ * CPT Queue Done Interrupt Coalescing Wait Registers
+ * Specifies the per queue interrupt coalescing settings.
+ * cptx_vqx_done_wait_s
+ * Word0
+ * reserved_48_63:16 [63:48] Reserved.
+ * time_wait:16; [47:32](R/W) Time hold-off. When CPT()_VQ()_DONE[DONE] = 0
+ * or CPT()_VQ()_DONE_ACK is written a timer is cleared. When the timer
+ * reaches [TIME_WAIT]*1024 then interrupt coalescing ends.
+ * see CPT()_VQ()_DONE[DONE]. If 0x0, time coalescing is disabled.
+ * reserved_20_31:12 [31:20] Reserved.
+ * num_wait:20 [19:0](R/W) Number of messages hold-off.
+ * When CPT()_VQ()_DONE[DONE] >= [NUM_WAIT] then interrupt coalescing ends
+ * see CPT()_VQ()_DONE[DONE]. If 0x0, same behavior as 0x1.
+ *
+ */
+union cptx_vqx_done_wait {
+ u64 u;
+ struct cptx_vqx_done_wait_s {
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
+ u64 reserved_48_63:16;
+ u64 time_wait:16;
+ u64 reserved_20_31:12;
+ u64 num_wait:20;
+#else /* Word 0 - Little Endian */
+ u64 num_wait:20;
+ u64 reserved_20_31:12;
+ u64 time_wait:16;
+ u64 reserved_48_63:16;
+#endif /* Word 0 - End */
+ } s;
+};
+
+/**
+ * Register (NCB) cpt#_vq#_done_ena_w1s
+ *
+ * CPT Queue Done Interrupt Enable Set Registers
+ * Write 1 to these registers will enable the DONEINT interrupt for the queue.
+ * cptx_vqx_done_ena_w1s_s
+ * Word0
+ * reserved_1_63:63 [63:1] Reserved.
+ * done:1 [0:0](R/W1S/H) Write 1 will enable DONEINT for this queue.
+ * Write 0 has no effect. Read will return the enable bit.
+ */
+union cptx_vqx_done_ena_w1s {
+ u64 u;
+ struct cptx_vqx_done_ena_w1s_s {
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
+ u64 reserved_1_63:63;
+ u64 done:1;
+#else /* Word 0 - Little Endian */
+ u64 done:1;
+ u64 reserved_1_63:63;
+#endif /* Word 0 - End */
+ } s;
+};
+
+/**
+ * Register (NCB) cpt#_vq#_ctl
+ *
+ * CPT VF Queue Control Registers
+ * This register configures queues. This register should be changed (other than
+ * clearing [ENA]) only when quiescent (see CPT()_VQ()_INPROG[INFLIGHT]).
+ * cptx_vqx_ctl_s
+ * Word0
+ * reserved_1_63:63 [63:1] Reserved.
+ * ena:1 [0:0](R/W/H) Enables the logical instruction queue.
+ * See also CPT()_PF_Q()_CTL[CONT_ERR] and CPT()_VQ()_INPROG[INFLIGHT].
+ * 1 = Queue is enabled.
+ * 0 = Queue is disabled.
+ */
+union cptx_vqx_ctl {
+ u64 u;
+ struct cptx_vqx_ctl_s {
+#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
+ u64 reserved_1_63:63;
+ u64 ena:1;
+#else /* Word 0 - Little Endian */
+ u64 ena:1;
+ u64 reserved_1_63:63;
+#endif /* Word 0 - End */
+ } s;
+};
+#endif /*__CPT_HW_TYPES_H*/
diff --git a/drivers/crypto/cavium/cpt/cptpf.h b/drivers/crypto/cavium/cpt/cptpf.h
new file mode 100644
index 0000000..8a2a8e5
--- /dev/null
+++ b/drivers/crypto/cavium/cpt/cptpf.h
@@ -0,0 +1,69 @@
+/*
+ * Copyright (C) 2016 Cavium, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License
+ * as published by the Free Software Foundation.
+ */
+
+#ifndef __CPTPF_H
+#define __CPTPF_H
+
+#include "cpt_common.h"
+
+#define CSR_DELAY 30
+#define CPT_MAX_CORE_GROUPS 8
+#define CPT_MAX_SE_CORES 10
+#define CPT_MAX_AE_CORES 6
+#define CPT_MAX_TOTAL_CORES (CPT_MAX_SE_CORES + CPT_MAX_AE_CORES)
+#define CPT_MAX_VF_NUM 16
+#define CPT_PF_MSIX_VECTORS 3
+#define CPT_PF_INT_VEC_E_MBOXX(a) (0x02 + (a))
+#define CPT_UCODE_VERSION_SZ 32
+struct cpt_device;
+
+struct microcode {
+ u8 is_mc_valid;
+ u8 is_ae;
+ u8 group;
+ u8 num_cores;
+ u32 code_size;
+ u64 core_mask;
+ u8 version[CPT_UCODE_VERSION_SZ];
+ /* Base info */
+ dma_addr_t phys_base;
+ void *code;
+};
+
+struct cpt_vf_info {
+ u8 state;
+ u8 priority;
+ u8 id;
+ u32 qlen;
+};
+
+/**
+ * cpt device structure
+ */
+struct cpt_device {
+ u16 flags; /* Flags to hold device status bits */
+ u8 num_vf_en; /* Number of VFs enabled (0...CPT_MAX_VF_NUM) */
+ struct cpt_vf_info vfinfo[CPT_MAX_VF_NUM]; /* Per VF info */
+
+ void __iomem *reg_base; /* Register start address */
+ /* MSI-X */
+ u8 num_vec;
+ bool msix_enabled;
+ struct msix_entry msix_entries[CPT_PF_MSIX_VECTORS];
+ bool irq_allocated[CPT_PF_MSIX_VECTORS];
+ struct pci_dev *pdev; /* pci device handle */
+
+ struct microcode mcode[CPT_MAX_CORE_GROUPS];
+ u8 next_mc_idx; /* next microcode index */
+ u8 next_group;
+ u8 max_se_cores;
+ u8 max_ae_cores;
+};
+
+void cpt_mbox_intr_handler(struct cpt_device *cpt, int mbx);
+#endif /* __CPTPF_H */
diff --git a/drivers/crypto/cavium/cpt/cptpf_main.c b/drivers/crypto/cavium/cpt/cptpf_main.c
new file mode 100644
index 0000000..682d57a
--- /dev/null
+++ b/drivers/crypto/cavium/cpt/cptpf_main.c
@@ -0,0 +1,708 @@
+/*
+ * Copyright (C) 2016 Cavium, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License
+ * as published by the Free Software Foundation.
+ */
+
+#include <linux/device.h>
+#include <linux/firmware.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/pci.h>
+#include <linux/printk.h>
+#include <linux/version.h>
+
+#include "cptpf.h"
+
+#define DRV_NAME "thunder-cpt"
+#define DRV_VERSION "1.0"
+
+static u32 num_vfs = 4; /* Default 4 VF enabled */
+module_param(num_vfs, uint, 0444);
+MODULE_PARM_DESC(num_vfs, "Number of VFs to enable(1-16)");
+
+/*
+ * Disable cores specified by coremask
+ */
+static void cpt_disable_cores(struct cpt_device *cpt, u64 coremask,
+ u8 type, u8 grp)
+{
+ u64 pf_exe_ctl;
+ u32 timeout = 100;
+ u64 grpmask = 0;
+ struct device *dev = &cpt->pdev->dev;
+
+ if (type == AE_TYPES)
+ coremask = (coremask << cpt->max_se_cores);
+
+ /* Disengage the cores from groups */
+ grpmask = cpt_read_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp));
+ cpt_write_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp),
+ (grpmask & ~coremask));
+ udelay(CSR_DELAY);
+ grp = cpt_read_csr64(cpt->reg_base, CPTX_PF_EXEC_BUSY(0));
+ while (grp & coremask) {
+ dev_err(dev, "Cores still busy %llx", coremask);
+ grp = cpt_read_csr64(cpt->reg_base,
+ CPTX_PF_EXEC_BUSY(0));
+ if (timeout--)
+ break;
+
+ udelay(CSR_DELAY);
+ }
+
+ /* Disable the cores */
+ pf_exe_ctl = cpt_read_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0));
+ cpt_write_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0),
+ (pf_exe_ctl & ~coremask));
+ udelay(CSR_DELAY);
+}
+
+/*
+ * Enable cores specified by coremask
+ */
+static void cpt_enable_cores(struct cpt_device *cpt, u64 coremask,
+ u8 type)
+{
+ u64 pf_exe_ctl;
+
+ if (type == AE_TYPES)
+ coremask = (coremask << cpt->max_se_cores);
+
+ pf_exe_ctl = cpt_read_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0));
+ cpt_write_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0),
+ (pf_exe_ctl | coremask));
+ udelay(CSR_DELAY);
+}
+
+static void cpt_configure_group(struct cpt_device *cpt, u8 grp,
+ u64 coremask, u8 type)
+{
+ u64 pf_gx_en = 0;
+
+ if (type == AE_TYPES)
+ coremask = (coremask << cpt->max_se_cores);
+
+ pf_gx_en = cpt_read_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp));
+ cpt_write_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp),
+ (pf_gx_en | coremask));
+ udelay(CSR_DELAY);
+}
+
+static void cpt_disable_mbox_interrupts(struct cpt_device *cpt)
+{
+ /* Clear mbox(0) interupts for all vfs */
+ cpt_write_csr64(cpt->reg_base, CPTX_PF_MBOX_ENA_W1CX(0, 0), ~0ull);
+}
+
+static void cpt_disable_ecc_interrupts(struct cpt_device *cpt)
+{
+ /* Clear ecc(0) interupts for all vfs */
+ cpt_write_csr64(cpt->reg_base, CPTX_PF_ECC0_ENA_W1C(0), ~0ull);
+}
+
+static void cpt_disable_exec_interrupts(struct cpt_device *cpt)
+{
+ /* Clear exec interupts for all vfs */
+ cpt_write_csr64(cpt->reg_base, CPTX_PF_EXEC_ENA_W1C(0), ~0ull);
+}
+
+static void cpt_disable_all_interrupts(struct cpt_device *cpt)
+{
+ cpt_disable_mbox_interrupts(cpt);
+ cpt_disable_ecc_interrupts(cpt);
+ cpt_disable_exec_interrupts(cpt);
+}
+
+static void cpt_enable_mbox_interrupts(struct cpt_device *cpt)
+{
+ /* Set mbox(0) interupts for all vfs */
+ cpt_write_csr64(cpt->reg_base, CPTX_PF_MBOX_ENA_W1SX(0, 0), ~0ull);
+}
+
+static int cpt_load_microcode(struct cpt_device *cpt, struct microcode *mcode)
+{
+ int ret = 0, core = 0, shift = 0;
+ u32 total_cores = 0;
+ struct device *dev = &cpt->pdev->dev;
+
+ if (!mcode || !mcode->code) {
+ dev_err(dev, "Either the mcode is null or data is NULL\n");
+ return -EINVAL;
+ }
+
+ if (mcode->code_size == 0) {
+ dev_err(dev, "microcode size is 0\n");
+ return -EINVAL;
+ }
+
+ /* Assumes 0-9 are SE cores for UCODE_BASE registers and
+ * AE core bases follow
+ */
+ if (mcode->is_ae) {
+ core = CPT_MAX_SE_CORES; /* start couting from 10 */
+ total_cores = CPT_MAX_TOTAL_CORES; /* upto 15 */
+ } else {
+ core = 0; /* start couting from 0 */
+ total_cores = CPT_MAX_SE_CORES; /* upto 9 */
+ }
+
+ /* Point to microcode for each core of the group */
+ for (; core < total_cores ; core++, shift++) {
+ if (mcode->core_mask & (1 << shift)) {
+ cpt_write_csr64(cpt->reg_base,
+ CPTX_PF_ENGX_UCODE_BASE(0, core),
+ (u64)mcode->phys_base);
+ }
+ }
+ return ret;
+}
+
+static int do_cpt_init(struct cpt_device *cpt, struct microcode *mcode)
+{
+ int ret = 0;
+ struct device *dev = &cpt->pdev->dev;
+
+ /* Make device not ready */
+ cpt->flags &= ~CPT_FLAG_DEVICE_READY;
+ /* Disable All PF interrupts */
+ cpt_disable_all_interrupts(cpt);
+ /* Calculate mcode group and coremasks */
+ if (mcode->is_ae) {
+ if (mcode->num_cores > cpt->max_ae_cores) {
+ dev_err(dev, "Requested for more cores than available AE cores\n");
+ ret = -EINVAL;
+ goto cpt_init_fail;
+ }
+
+ if (cpt->next_group >= CPT_MAX_CORE_GROUPS) {
+ dev_err(dev, "Can't load, all eight microcode groups in use");
+ return -ENFILE;
+ }
+
+ mcode->group = cpt->next_group;
+ /* Convert requested cores to mask */
+ mcode->core_mask = GENMASK(mcode->num_cores, 0);
+ cpt_disable_cores(cpt, mcode->core_mask, AE_TYPES,
+ mcode->group);
+ /* Load microcode for AE engines */
+ ret = cpt_load_microcode(cpt, mcode);
+ if (ret) {
+ dev_err(dev, "Microcode load Failed for %s\n",
+ mcode->version);
+ goto cpt_init_fail;
+ }
+ cpt->next_group++;
+ /* Configure group mask for the mcode */
+ cpt_configure_group(cpt, mcode->group, mcode->core_mask,
+ AE_TYPES);
+ /* Enable AE cores for the group mask */
+ cpt_enable_cores(cpt, mcode->core_mask, AE_TYPES);
+ } else {
+ if (mcode->num_cores > cpt->max_se_cores) {
+ dev_err(dev, "Requested for more cores than available SE cores\n");
+ ret = -EINVAL;
+ goto cpt_init_fail;
+ }
+ if (cpt->next_group >= CPT_MAX_CORE_GROUPS) {
+ dev_err(dev, "Can't load, all eight microcode groups in use");
+ return -ENFILE;
+ }
+
+ mcode->group = cpt->next_group;
+ /* Covert requested cores to mask */
+ mcode->core_mask = GENMASK(mcode->num_cores, 0);
+ cpt_disable_cores(cpt, mcode->core_mask, SE_TYPES,
+ mcode->group);
+ /* Load microcode for SE engines */
+ ret = cpt_load_microcode(cpt, mcode);
+ if (ret) {
+ dev_err(dev, "Microcode load Failed for %s\n",
+ mcode->version);
+ goto cpt_init_fail;
+ }
+ cpt->next_group++;
+ /* Configure group mask for the mcode */
+ cpt_configure_group(cpt, mcode->group, mcode->core_mask,
+ SE_TYPES);
+ /* Enable SE cores for the group mask */
+ cpt_enable_cores(cpt, mcode->core_mask, SE_TYPES);
+ }
+
+ /* Enabled PF mailbox interrupts */
+ cpt_enable_mbox_interrupts(cpt);
+ cpt->flags |= CPT_FLAG_DEVICE_READY;
+
+ return ret;
+
+cpt_init_fail:
+ /* Enabled PF mailbox interrupts */
+ cpt_enable_mbox_interrupts(cpt);
+
+ return ret;
+}
+
+struct ucode_header {
+ u8 version[CPT_UCODE_VERSION_SZ];
+ u32 code_length;
+ u32 data_length;
+ u64 sram_address;
+};
+
+static int cpt_ucode_load_fw(struct cpt_device *cpt, const u8 *fw, bool is_ae)
+{
+ const struct firmware *fw_entry;
+ struct device *dev = &cpt->pdev->dev;
+ struct ucode_header *ucode;
+ struct microcode *mcode;
+ int j, ret = 0;
+
+ ret = request_firmware(&fw_entry, fw, dev);
+ if (ret)
+ return ret;
+
+ ucode = (struct ucode_header *)fw_entry->data;
+ mcode = &cpt->mcode[cpt->next_mc_idx];
+ memcpy(mcode->version, (u8 *)fw_entry->data, CPT_UCODE_VERSION_SZ);
+ mcode->code_size = ntohl(ucode->code_length) * 2;
+ if (!mcode->code_size)
+ return -EINVAL;
+
+ mcode->is_ae = is_ae;
+ mcode->core_mask = 0ULL;
+ mcode->num_cores = is_ae ? 6 : 10;
+
+ /* Allocate DMAable space */
+ mcode->code = dma_zalloc_coherent(&cpt->pdev->dev, mcode->code_size,
+ &mcode->phys_base, GFP_KERNEL);
+ if (!mcode->code) {
+ dev_err(dev, "Unable to allocate space for microcode");
+ return -ENOMEM;
+ }
+
+ memcpy((void *)mcode->code, (void *)(fw_entry->data + sizeof(*ucode)),
+ mcode->code_size);
+
+ /* Byte swap 64-bit */
+ for (j = 0; j < (mcode->code_size / 8); j++)
+ ((u64 *)mcode->code)[j] = cpu_to_be64(((u64 *)mcode->code)[j]);
+ /* MC needs 16-bit swap */
+ for (j = 0; j < (mcode->code_size / 2); j++)
+ ((u16 *)mcode->code)[j] = cpu_to_be16(((u16 *)mcode->code)[j]);
+
+ dev_dbg(dev, "mcode->code_size = %u\n", mcode->code_size);
+ dev_dbg(dev, "mcode->is_ae = %u\n", mcode->is_ae);
+ dev_dbg(dev, "mcode->num_cores = %u\n", mcode->num_cores);
+ dev_dbg(dev, "mcode->code = %llx\n", (u64)mcode->code);
+ dev_dbg(dev, "mcode->phys_base = %llx\n", mcode->phys_base);
+
+ ret = do_cpt_init(cpt, mcode);
+ if (ret) {
+ dev_err(dev, "do_cpt_init failed with ret: %d\n", ret);
+ return ret;
+ }
+
+ dev_info(dev, "Microcode Loaded %s\n", mcode->version);
+ mcode->is_mc_valid = 1;
+ cpt->next_mc_idx++;
+ release_firmware(fw_entry);
+
+ return ret;
+}
+
+static int cpt_ucode_load(struct cpt_device *cpt)
+{
+ int ret = 0;
+ struct device *dev = &cpt->pdev->dev;
+
+ ret = cpt_ucode_load_fw(cpt, "cpt8x-mc-ae.out", true);
+ if (ret) {
+ dev_err(dev, "ae:cpt_ucode_load failed with ret: %d\n", ret);
+ return ret;
+ }
+ ret = cpt_ucode_load_fw(cpt, "cpt8x-mc-se.out", false);
+ if (ret) {
+ dev_err(dev, "se:cpt_ucode_load failed with ret: %d\n", ret);
+ return ret;
+ }
+
+ return ret;
+}
+
+static int cpt_enable_msix(struct cpt_device *cpt)
+{
+ int i, ret;
+
+ cpt->num_vec = CPT_PF_MSIX_VECTORS;
+
+ for (i = 0; i < cpt->num_vec; i++)
+ cpt->msix_entries[i].entry = i;
+
+ ret = pci_enable_msix(cpt->pdev, cpt->msix_entries, cpt->num_vec);
+ if (ret) {
+ dev_err(&cpt->pdev->dev, "Request for #%d msix vectors failed\n",
+ cpt->num_vec);
+ return ret;
+ }
+
+ cpt->msix_enabled = 1;
+ return 0;
+}
+
+static irqreturn_t cpt_mbx0_intr_handler(int irq, void *cpt_irq)
+{
+ struct cpt_device *cpt = (struct cpt_device *)cpt_irq;
+
+ cpt_mbox_intr_handler(cpt, 0);
+
+ return IRQ_HANDLED;
+}
+
+static void cpt_disable_msix(struct cpt_device *cpt)
+{
+ if (cpt->msix_enabled) {
+ pci_disable_msix(cpt->pdev);
+ cpt->msix_enabled = 0;
+ cpt->num_vec = 0;
+ }
+}
+
+static void cpt_free_all_interrupts(struct cpt_device *cpt)
+{
+ int irq;
+
+ for (irq = 0; irq < cpt->num_vec; irq++) {
+ if (cpt->irq_allocated[irq])
+ free_irq(cpt->msix_entries[irq].vector, cpt);
+ cpt->irq_allocated[irq] = false;
+ }
+}
+
+static void cpt_reset(struct cpt_device *cpt)
+{
+ cpt_write_csr64(cpt->reg_base, CPTX_PF_RESET(0), 1);
+}
+
+static void cpt_find_max_enabled_cores(struct cpt_device *cpt)
+{
+ union cptx_pf_constants pf_cnsts = {0};
+
+ pf_cnsts.u = cpt_read_csr64(cpt->reg_base, CPTX_PF_CONSTANTS(0));
+ cpt->max_se_cores = pf_cnsts.s.se;
+ cpt->max_ae_cores = pf_cnsts.s.ae;
+}
+
+static u32 cpt_check_bist_status(struct cpt_device *cpt)
+{
+ union cptx_pf_bist_status bist_sts = {0};
+
+ bist_sts.u = cpt_read_csr64(cpt->reg_base,
+ CPTX_PF_BIST_STATUS(0));
+
+ return bist_sts.u;
+}
+
+static u64 cpt_check_exe_bist_status(struct cpt_device *cpt)
+{
+ union cptx_pf_exe_bist_status bist_sts = {0};
+
+ bist_sts.u = cpt_read_csr64(cpt->reg_base,
+ CPTX_PF_EXE_BIST_STATUS(0));
+
+ return bist_sts.u;
+}
+
+static void cpt_disable_all_cores(struct cpt_device *cpt)
+{
+ u32 grp, timeout = 100;
+ struct device *dev = &cpt->pdev->dev;
+
+ /* Disengage the cores from groups */
+ for (grp = 0; grp < CPT_MAX_CORE_GROUPS; grp++) {
+ cpt_write_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp), 0);
+ udelay(CSR_DELAY);
+ }
+
+ grp = cpt_read_csr64(cpt->reg_base, CPTX_PF_EXEC_BUSY(0));
+ while (grp) {
+ dev_err(dev, "Cores still busy");
+ grp = cpt_read_csr64(cpt->reg_base,
+ CPTX_PF_EXEC_BUSY(0));
+ if (timeout--)
+ break;
+
+ udelay(CSR_DELAY);
+ }
+ /* Disable the cores */
+ cpt_write_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0), 0);
+}
+
+/**
+ * Ensure all cores are disengaged from all groups by
+ * calling cpt_disable_all_cores() before calling this
+ * function.
+ */
+static void cpt_unload_microcode(struct cpt_device *cpt)
+{
+ u32 grp = 0, core;
+
+ /* Free microcode bases and reset group masks */
+ for (grp = 0; grp < CPT_MAX_CORE_GROUPS; grp++) {
+ struct microcode *mcode = &cpt->mcode[grp];
+
+ if (cpt->mcode[grp].code)
+ dma_free_coherent(&cpt->pdev->dev, mcode->code_size,
+ mcode->code, mcode->phys_base);
+ mcode->code = NULL;
+ }
+ /* Clear UCODE_BASE registers for all engines */
+ for (core = 0; core < CPT_MAX_TOTAL_CORES; core++)
+ cpt_write_csr64(cpt->reg_base,
+ CPTX_PF_ENGX_UCODE_BASE(0, core), 0ull);
+}
+
+static int cpt_device_init(struct cpt_device *cpt)
+{
+ u64 bist;
+ struct device *dev = &cpt->pdev->dev;
+
+ /* Reset the PF when probed first */
+ cpt_reset(cpt);
+ mdelay(100);
+
+ /*Check BIST status*/
+ bist = (u64)cpt_check_bist_status(cpt);
+ if (bist) {
+ dev_err(dev, "RAM BIST failed with code 0x%llx", bist);
+ return -ENODEV;
+ }
+
+ bist = cpt_check_exe_bist_status(cpt);
+ if (bist) {
+ dev_err(dev, "Engine BIST failed with code 0x%llx", bist);
+ return -ENODEV;
+ }
+
+ /*Get CLK frequency*/
+ /*Get max enabled cores */
+ cpt_find_max_enabled_cores(cpt);
+ /*Disable all cores*/
+ cpt_disable_all_cores(cpt);
+ /*Reset device parameters*/
+ cpt->next_mc_idx = 0;
+ cpt->next_group = 0;
+ /* PF is ready */
+ cpt->flags |= CPT_FLAG_DEVICE_READY;
+
+ return 0;
+}
+
+static int cpt_register_interrupts(struct cpt_device *cpt)
+{
+ int ret;
+ struct device *dev = &cpt->pdev->dev;
+
+ /* Enable MSI-X */
+ ret = cpt_enable_msix(cpt);
+ if (ret)
+ return ret;
+
+ /* Register mailbox interrupt handlers */
+ ret = request_irq(cpt->msix_entries[CPT_PF_INT_VEC_E_MBOXX(0)].vector,
+ cpt_mbx0_intr_handler, 0, "CPT Mbox0", cpt);
+ if (ret)
+ goto fail;
+
+ cpt->irq_allocated[CPT_PF_INT_VEC_E_MBOXX(0)] = true;
+
+ /* Enable mailbox interrupt */
+ cpt_enable_mbox_interrupts(cpt);
+ return 0;
+
+fail:
+ dev_err(dev, "Request irq failed\n");
+ cpt_free_all_interrupts(cpt);
+ return ret;
+}
+
+static void cpt_unregister_interrupts(struct cpt_device *cpt)
+{
+ cpt_free_all_interrupts(cpt);
+ cpt_disable_msix(cpt);
+}
+
+static int cpt_sriov_init(struct cpt_device *cpt, int num_vfs)
+{
+ int pos = 0;
+ int err;
+ u16 total_vf_cnt;
+ struct pci_dev *pdev = cpt->pdev;
+
+ pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV);
+ if (!pos) {
+ dev_err(&pdev->dev, "SRIOV capability is not found in PCIe config space\n");
+ return -ENODEV;
+ }
+
+ cpt->num_vf_en = num_vfs; /* User requested VFs */
+ pci_read_config_word(pdev, (pos + PCI_SRIOV_TOTAL_VF), &total_vf_cnt);
+ if (total_vf_cnt < cpt->num_vf_en)
+ cpt->num_vf_en = total_vf_cnt;
+
+ if (!total_vf_cnt)
+ return 0;
+
+ /*Enabled the available VFs */
+ err = pci_enable_sriov(pdev, cpt->num_vf_en);
+ if (err) {
+ dev_err(&pdev->dev, "SRIOV enable failed, num VF is %d\n",
+ cpt->num_vf_en);
+ cpt->num_vf_en = 0;
+ return err;
+ }
+
+ /* TODO: Optionally enable static VQ priorities feature */
+
+ dev_info(&pdev->dev, "SRIOV enabled, number of VF available %d\n",
+ cpt->num_vf_en);
+
+ cpt->flags |= CPT_FLAG_SRIOV_ENABLED;
+
+ return 0;
+}
+
+static int cpt_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
+{
+ struct device *dev = &pdev->dev;
+ struct cpt_device *cpt;
+ int err;
+
+ if (num_vfs > 16 || num_vfs < 4) {
+ dev_warn(dev, "Invalid vf count %d, Resetting it to 4(default)\n",
+ num_vfs);
+ num_vfs = 4;
+ }
+
+ cpt = devm_kzalloc(dev, sizeof(*cpt), GFP_KERNEL);
+ if (!cpt)
+ return -ENOMEM;
+
+ pci_set_drvdata(pdev, cpt);
+ cpt->pdev = pdev;
+ err = pci_enable_device(pdev);
+ if (err) {
+ dev_err(dev, "Failed to enable PCI device\n");
+ pci_set_drvdata(pdev, NULL);
+ return err;
+ }
+
+ err = pci_request_regions(pdev, DRV_NAME);
+ if (err) {
+ dev_err(dev, "PCI request regions failed 0x%x\n", err);
+ goto cpt_err_disable_device;
+ }
+
+ err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48));
+ if (err) {
+ dev_err(dev, "Unable to get usable DMA configuration\n");
+ goto cpt_err_release_regions;
+ }
+
+ err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48));
+ if (err) {
+ dev_err(dev, "Unable to get 48-bit DMA for consistent allocations\n");
+ goto cpt_err_release_regions;
+ }
+
+ /* MAP PF's configuration registers */
+ cpt->reg_base = pcim_iomap(pdev, 0, 0);
+ if (!cpt->reg_base) {
+ dev_err(dev, "Cannot map config register space, aborting\n");
+ err = -ENOMEM;
+ goto cpt_err_release_regions;
+ }
+
+ /* CPT device HW initialization */
+ cpt_device_init(cpt);
+
+ /* Register interrupts */
+ err = cpt_register_interrupts(cpt);
+ if (err)
+ goto cpt_err_release_regions;
+
+ err = cpt_ucode_load(cpt);
+ if (err)
+ goto cpt_err_unregister_interrupts;
+
+ /* Configure SRIOV */
+ err = cpt_sriov_init(cpt, num_vfs);
+ if (err)
+ goto cpt_err_unregister_interrupts;
+
+ return 0;
+
+cpt_err_unregister_interrupts:
+ cpt_unregister_interrupts(cpt);
+cpt_err_release_regions:
+ pci_release_regions(pdev);
+cpt_err_disable_device:
+ pci_disable_device(pdev);
+ pci_set_drvdata(pdev, NULL);
+ return err;
+}
+
+static void cpt_remove(struct pci_dev *pdev)
+{
+ struct cpt_device *cpt = pci_get_drvdata(pdev);
+
+ /* Disengage SE and AE cores from all groups*/
+ cpt_disable_all_cores(cpt);
+ /* Unload microcodes */
+ cpt_unload_microcode(cpt);
+ cpt_unregister_interrupts(cpt);
+ pci_disable_sriov(pdev);
+ pci_release_regions(pdev);
+ pci_disable_device(pdev);
+ pci_set_drvdata(pdev, NULL);
+}
+
+static void cpt_shutdown(struct pci_dev *pdev)
+{
+ struct cpt_device *cpt = pci_get_drvdata(pdev);
+
+ if (!cpt)
+ return;
+
+ dev_info(&pdev->dev, "Shutdown device %x:%x.\n",
+ (u32)pdev->vendor, (u32)pdev->device);
+
+ cpt_unregister_interrupts(cpt);
+ pci_release_regions(pdev);
+ pci_disable_device(pdev);
+ pci_set_drvdata(pdev, NULL);
+}
+
+/* Supported devices */
+static const struct pci_device_id cpt_id_table[] = {
+ { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, CPT_81XX_PCI_PF_DEVICE_ID) },
+ { 0, } /* end of table */
+};
+
+static struct pci_driver cpt_pci_driver = {
+ .name = DRV_NAME,
+ .id_table = cpt_id_table,
+ .probe = cpt_probe,
+ .remove = cpt_remove,
+ .shutdown = cpt_shutdown,
+};
+
+module_pci_driver(cpt_pci_driver);
+
+MODULE_AUTHOR("George Cherian <george.cherian@cavium.com>");
+MODULE_DESCRIPTION("Cavium Thunder CPT Physical Function Driver");
+MODULE_LICENSE("GPL v2");
+MODULE_VERSION(DRV_VERSION);
+MODULE_DEVICE_TABLE(pci, cpt_id_table);
diff --git a/drivers/crypto/cavium/cpt/cptpf_mbox.c b/drivers/crypto/cavium/cpt/cptpf_mbox.c
new file mode 100644
index 0000000..5818b41
--- /dev/null
+++ b/drivers/crypto/cavium/cpt/cptpf_mbox.c
@@ -0,0 +1,163 @@
+/*
+ * Copyright (C) 2016 Cavium, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License
+ * as published by the Free Software Foundation.
+ */
+#include <linux/module.h>
+#include "cptpf.h"
+
+static void cpt_send_msg_to_vf(struct cpt_device *cpt, int vf,
+ struct cpt_mbox *mbx)
+{
+ /* Writing mbox(0) causes interrupt */
+ cpt_write_csr64(cpt->reg_base, CPTX_PF_VFX_MBOXX(0, vf, 1),
+ mbx->data);
+ cpt_write_csr64(cpt->reg_base, CPTX_PF_VFX_MBOXX(0, vf, 0), mbx->msg);
+}
+
+/* ACKs VF's mailbox message
+ * @vf: VF to which ACK to be sent
+ */
+static void cpt_mbox_send_ack(struct cpt_device *cpt, int vf,
+ struct cpt_mbox *mbx)
+{
+ mbx->data = 0ull;
+ mbx->msg = CPT_MBOX_MSG_TYPE_ACK;
+ cpt_send_msg_to_vf(cpt, vf, mbx);
+}
+
+static void cpt_clear_mbox_intr(struct cpt_device *cpt, u32 vf)
+{
+ /* W1C for the VF */
+ cpt_write_csr64(cpt->reg_base, CPTX_PF_MBOX_INTX(0, 0), (1 << vf));
+}
+
+/*
+ * Configure QLEN/Chunk sizes for VF
+ */
+static void cpt_cfg_qlen_for_vf(struct cpt_device *cpt, int vf, u32 size)
+{
+ union cptx_pf_qx_ctl pf_qx_ctl;
+
+ pf_qx_ctl.u = cpt_read_csr64(cpt->reg_base, CPTX_PF_QX_CTL(0, vf));
+ pf_qx_ctl.s.size = size;
+ pf_qx_ctl.s.cont_err = true;
+ cpt_write_csr64(cpt->reg_base, CPTX_PF_QX_CTL(0, vf), pf_qx_ctl.u);
+}
+
+/*
+ * Configure VQ priority
+ */
+static void cpt_cfg_vq_priority(struct cpt_device *cpt, int vf, u32 pri)
+{
+ union cptx_pf_qx_ctl pf_qx_ctl;
+
+ pf_qx_ctl.u = cpt_read_csr64(cpt->reg_base, CPTX_PF_QX_CTL(0, vf));
+ pf_qx_ctl.s.pri = pri;
+ cpt_write_csr64(cpt->reg_base, CPTX_PF_QX_CTL(0, vf), pf_qx_ctl.u);
+}
+
+static u8 cpt_bind_vq_to_grp(struct cpt_device *cpt, u8 q, u8 grp)
+{
+ struct microcode *mcode = cpt->mcode;
+ union cptx_pf_qx_ctl pf_qx_ctl;
+ struct device *dev = &cpt->pdev->dev;
+
+ if (q >= CPT_MAX_VF_NUM) {
+ dev_err(dev, "Queues are more than cores in the group");
+ return -EINVAL;
+ }
+ if (grp >= CPT_MAX_CORE_GROUPS) {
+ dev_err(dev, "Request group is more than possible groups");
+ return -EINVAL;
+ }
+ if (grp >= cpt->next_mc_idx) {
+ dev_err(dev, "Request group is higher than available functional groups");
+ return -EINVAL;
+ }
+ pf_qx_ctl.u = cpt_read_csr64(cpt->reg_base, CPTX_PF_QX_CTL(0, q));
+ pf_qx_ctl.s.grp = mcode[grp].group;
+ cpt_write_csr64(cpt->reg_base, CPTX_PF_QX_CTL(0, q), pf_qx_ctl.u);
+ dev_dbg(dev, "VF %d TYPE %s", q, (mcode[grp].is_ae ? "AE" : "SE"));
+
+ return mcode[grp].is_ae ? AE_TYPES : SE_TYPES;
+}
+
+/* Interrupt handler to handle mailbox messages from VFs */
+static void cpt_handle_mbox_intr(struct cpt_device *cpt, int vf)
+{
+ struct cpt_vf_info *vfx = &cpt->vfinfo[vf];
+ struct cpt_mbox mbx = {};
+ u8 vftype;
+ struct device *dev = &cpt->pdev->dev;
+ /*
+ * MBOX[0] contains msg
+ * MBOX[1] contains data
+ */
+ mbx.msg = cpt_read_csr64(cpt->reg_base, CPTX_PF_VFX_MBOXX(0, vf, 0));
+ mbx.data = cpt_read_csr64(cpt->reg_base, CPTX_PF_VFX_MBOXX(0, vf, 1));
+ dev_dbg(dev, "%s: Mailbox msg 0x%llx from VF%d", __func__, mbx.msg, vf);
+ switch (mbx.msg) {
+ case CPT_MSG_VF_UP:
+ vfx->state = VF_STATE_UP;
+ try_module_get(THIS_MODULE);
+ cpt_mbox_send_ack(cpt, vf, &mbx);
+ break;
+ case CPT_MSG_READY:
+ mbx.msg = CPT_MSG_READY;
+ mbx.data = vf;
+ cpt_send_msg_to_vf(cpt, vf, &mbx);
+ break;
+ case CPT_MSG_VF_DOWN:
+ /* First msg in VF teardown sequence */
+ vfx->state = VF_STATE_DOWN;
+ module_put(THIS_MODULE);
+ cpt_mbox_send_ack(cpt, vf, &mbx);
+ break;
+ case CPT_MSG_QLEN:
+ vfx->qlen = mbx.data;
+ cpt_cfg_qlen_for_vf(cpt, vf, vfx->qlen);
+ cpt_mbox_send_ack(cpt, vf, &mbx);
+ break;
+ case CPT_MSG_QBIND_GRP:
+ vftype = cpt_bind_vq_to_grp(cpt, vf, (u8)mbx.data);
+ if ((vftype != AE_TYPES) && (vftype != SE_TYPES))
+ dev_err(dev, "Queue %d binding to group %llu failed",
+ vf, mbx.data);
+ else {
+ dev_dbg(dev, "Queue %d binding to group %llu successful",
+ vf, mbx.data);
+ mbx.msg = CPT_MSG_QBIND_GRP;
+ mbx.data = vftype;
+ cpt_send_msg_to_vf(cpt, vf, &mbx);
+ }
+ break;
+ case CPT_MSG_VQ_PRIORITY:
+ vfx->priority = mbx.data;
+ cpt_cfg_vq_priority(cpt, vf, vfx->priority);
+ cpt_mbox_send_ack(cpt, vf, &mbx);
+ break;
+ default:
+ dev_err(&cpt->pdev->dev, "Invalid msg from VF%d, msg 0x%llx\n",
+ vf, mbx.msg);
+ break;
+ }
+}
+
+void cpt_mbox_intr_handler (struct cpt_device *cpt, int mbx)
+{
+ u64 intr;
+ u8 vf;
+
+ intr = cpt_read_csr64(cpt->reg_base, CPTX_PF_MBOX_INTX(0, 0));
+ dev_dbg(&cpt->pdev->dev, "PF interrupt Mbox%d 0x%llx\n", mbx, intr);
+ for (vf = 0; vf < CPT_MAX_VF_NUM; vf++) {
+ if (intr & (1ULL << vf)) {
+ dev_dbg(&cpt->pdev->dev, "Intr from VF %d\n", vf);
+ cpt_handle_mbox_intr(cpt, vf);
+ cpt_clear_mbox_intr(cpt, vf);
+ }
+ }
+}
--
2.1.4
^ permalink raw reply related
* [PATCH v6 3/3] drivers: crypto: Enable CPT options crypto for build
From: George Cherian @ 2017-02-07 14:51 UTC (permalink / raw)
To: herbert, davem
Cc: david.daney, clabbe.montjoie, smueller, levinsasha928,
linux-kernel, linux-crypto, George Cherian
In-Reply-To: <1486479075-1813-1-git-send-email-george.cherian@cavium.com>
Add the CPT options in crypto Kconfig and update the
crypto Makefile
Update the MAINTAINERS file too.
Signed-off-by: George Cherian <george.cherian@cavium.com>
Reviewed-by: David Daney <david.daney@cavium.com>
---
MAINTAINERS | 7 +++++++
drivers/crypto/Kconfig | 1 +
drivers/crypto/Makefile | 1 +
3 files changed, 9 insertions(+)
diff --git a/MAINTAINERS b/MAINTAINERS
index 187b961..386fc34 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -2990,6 +2990,13 @@ W: http://www.cavium.com
S: Supported
F: drivers/net/ethernet/cavium/liquidio/
+CAVIUM OCTEON-TX CRYPTO DRIVER
+M: George Cherian <george.cherian@cavium.com>
+L: linux-crypto@vger.kernel.org
+W: http://www.cavium.com
+S: Supported
+F: drivers/crypto/cavium/cpt/
+
CC2520 IEEE-802.15.4 RADIO DRIVER
M: Varka Bhadram <varkabhadram@gmail.com>
L: linux-wpan@vger.kernel.org
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig
index 7956478..1603ac3 100644
--- a/drivers/crypto/Kconfig
+++ b/drivers/crypto/Kconfig
@@ -484,6 +484,7 @@ config CRYPTO_DEV_MXS_DCP
will be called mxs-dcp.
source "drivers/crypto/qat/Kconfig"
+source "drivers/crypto/cavium/cpt/Kconfig"
config CRYPTO_DEV_QCE
tristate "Qualcomm crypto engine accelerator"
diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile
index bc53cb8..7add7fd 100644
--- a/drivers/crypto/Makefile
+++ b/drivers/crypto/Makefile
@@ -3,6 +3,7 @@ obj-$(CONFIG_CRYPTO_DEV_ATMEL_SHA) += atmel-sha.o
obj-$(CONFIG_CRYPTO_DEV_ATMEL_TDES) += atmel-tdes.o
obj-$(CONFIG_CRYPTO_DEV_BFIN_CRC) += bfin_crc.o
obj-$(CONFIG_CRYPTO_DEV_CCP) += ccp/
+obj-$(CONFIG_CRYPTO_DEV_CPT) += cavium/cpt/
obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam/
obj-$(CONFIG_CRYPTO_DEV_GEODE) += geode-aes.o
obj-$(CONFIG_CRYPTO_DEV_HIFN_795X) += hifn_795x.o
--
2.1.4
^ permalink raw reply related
* [PATCH v6 0/3] Add Support for Cavium Cryptographic Acceleration Unit
From: George Cherian @ 2017-02-07 14:51 UTC (permalink / raw)
To: herbert, davem
Cc: david.daney, clabbe.montjoie, smueller, levinsasha928,
linux-kernel, linux-crypto, George Cherian
This series adds the support for Cavium Cryptographic Accelerarion Unit (CPT)
CPT is available in Cavium's Octeon-Tx SoC series.
The series was tested with ecryptfs and dm-crypt for in kernel cryptographic
offload operations. This driver needs a firmware to work, I will be sending the
firmware to linux-firmware once the driver is accepted.
Cahnges v5 -> v6
-- Addressed Sasha Levin's comments.
- Remove open coding of hlist_for_each_entry_safe, instead use
the same.
- Remove the direct access of hlist members
- Remove unwanted argument checks.
Changes v4 -> v5
--Addressed Stephan Muller's comments.
- Fix XTS key sizes.
- Introduce a new fuction for xts setkey.
- There was a key2 misalignment issue, the same is also fixed
for aes_xts.
Changes v3 -> v4
--Addressed Corentin Labbe's comments
- Convert all pr_x to dev_x.
- Fix Typo errors.
- Fix the Double unlock.
- Use sg_virt.
Changes v2 -> v3
-- Addressed David Daney's comments
- There is not much difference in performance readq/writeq vs
readq_relaxed/writeq_relaxed, so switching to readq/writeq variant.
- Removed the useless bitfield definitions.
- Use GENMASK,dev_to_node() instead of custome functions.
- Use module_pci_driver instead of module_init/exit.
Changes v1 -> v2
-- Addressed a crash issue when more gather components are passed.
-- Redo the cptvf request manager.
- Get rid of the un necessary buffer copies.
-- s/uint*_t/u*
-- Remove unwanted Macro definitions
-- Remove the redundant ROUNDUP* macros and use kernel function
-- Select proper config option in Kconfig file.
-- Removed some of the unwanted header file inclusions
-- Miscellaneous Cleanup
George Cherian (3):
drivers: crypto: Add Support for Octeon-tx CPT Engine
drivers: crypto: Add the Virtual Function driver for CPT
drivers: crypto: Enable CPT options crypto for build
MAINTAINERS | 7 +
drivers/crypto/Kconfig | 1 +
drivers/crypto/Makefile | 1 +
drivers/crypto/cavium/cpt/Kconfig | 16 +
drivers/crypto/cavium/cpt/Makefile | 3 +
drivers/crypto/cavium/cpt/cpt_common.h | 158 +++++
drivers/crypto/cavium/cpt/cpt_hw_types.h | 658 +++++++++++++++++++
drivers/crypto/cavium/cpt/cptpf.h | 69 ++
drivers/crypto/cavium/cpt/cptpf_main.c | 708 ++++++++++++++++++++
drivers/crypto/cavium/cpt/cptpf_mbox.c | 163 +++++
drivers/crypto/cavium/cpt/cptvf.h | 135 ++++
drivers/crypto/cavium/cpt/cptvf_algs.c | 444 +++++++++++++
drivers/crypto/cavium/cpt/cptvf_algs.h | 113 ++++
drivers/crypto/cavium/cpt/cptvf_main.c | 936 +++++++++++++++++++++++++++
drivers/crypto/cavium/cpt/cptvf_mbox.c | 211 ++++++
drivers/crypto/cavium/cpt/cptvf_reqmanager.c | 593 +++++++++++++++++
drivers/crypto/cavium/cpt/request_manager.h | 147 +++++
17 files changed, 4363 insertions(+)
create mode 100644 drivers/crypto/cavium/cpt/Kconfig
create mode 100644 drivers/crypto/cavium/cpt/Makefile
create mode 100644 drivers/crypto/cavium/cpt/cpt_common.h
create mode 100644 drivers/crypto/cavium/cpt/cpt_hw_types.h
create mode 100644 drivers/crypto/cavium/cpt/cptpf.h
create mode 100644 drivers/crypto/cavium/cpt/cptpf_main.c
create mode 100644 drivers/crypto/cavium/cpt/cptpf_mbox.c
create mode 100644 drivers/crypto/cavium/cpt/cptvf.h
create mode 100644 drivers/crypto/cavium/cpt/cptvf_algs.c
create mode 100644 drivers/crypto/cavium/cpt/cptvf_algs.h
create mode 100644 drivers/crypto/cavium/cpt/cptvf_main.c
create mode 100644 drivers/crypto/cavium/cpt/cptvf_mbox.c
create mode 100644 drivers/crypto/cavium/cpt/cptvf_reqmanager.c
create mode 100644 drivers/crypto/cavium/cpt/request_manager.h
--
2.1.4
^ permalink raw reply
* [PATCH v6 2/3] drivers: crypto: Add the Virtual Function driver for CPT
From: George Cherian @ 2017-02-07 14:51 UTC (permalink / raw)
To: herbert, davem
Cc: david.daney, clabbe.montjoie, smueller, levinsasha928,
linux-kernel, linux-crypto, George Cherian
In-Reply-To: <1486479075-1813-1-git-send-email-george.cherian@cavium.com>
Enable the CPT VF driver. CPT is the cryptographic Acceleration Unit
in Octeon-tx series of processors.
Signed-off-by: George Cherian <george.cherian@cavium.com>
Reviewed-by: David Daney <david.daney@cavium.com>
---
drivers/crypto/cavium/cpt/Makefile | 3 +-
drivers/crypto/cavium/cpt/cptvf.h | 135 ++++
drivers/crypto/cavium/cpt/cptvf_algs.c | 444 +++++++++++++
drivers/crypto/cavium/cpt/cptvf_algs.h | 113 ++++
drivers/crypto/cavium/cpt/cptvf_main.c | 936 +++++++++++++++++++++++++++
drivers/crypto/cavium/cpt/cptvf_mbox.c | 211 ++++++
drivers/crypto/cavium/cpt/cptvf_reqmanager.c | 593 +++++++++++++++++
drivers/crypto/cavium/cpt/request_manager.h | 147 +++++
8 files changed, 2581 insertions(+), 1 deletion(-)
create mode 100644 drivers/crypto/cavium/cpt/cptvf.h
create mode 100644 drivers/crypto/cavium/cpt/cptvf_algs.c
create mode 100644 drivers/crypto/cavium/cpt/cptvf_algs.h
create mode 100644 drivers/crypto/cavium/cpt/cptvf_main.c
create mode 100644 drivers/crypto/cavium/cpt/cptvf_mbox.c
create mode 100644 drivers/crypto/cavium/cpt/cptvf_reqmanager.c
create mode 100644 drivers/crypto/cavium/cpt/request_manager.h
diff --git a/drivers/crypto/cavium/cpt/Makefile b/drivers/crypto/cavium/cpt/Makefile
index fe3d454..dbf055e 100644
--- a/drivers/crypto/cavium/cpt/Makefile
+++ b/drivers/crypto/cavium/cpt/Makefile
@@ -1,2 +1,3 @@
-obj-$(CONFIG_CAVIUM_CPT) += cptpf.o
+obj-$(CONFIG_CAVIUM_CPT) += cptpf.o cptvf.o
cptpf-objs := cptpf_main.o cptpf_mbox.o
+cptvf-objs := cptvf_main.o cptvf_reqmanager.o cptvf_mbox.o cptvf_algs.o
diff --git a/drivers/crypto/cavium/cpt/cptvf.h b/drivers/crypto/cavium/cpt/cptvf.h
new file mode 100644
index 0000000..1cc04aa
--- /dev/null
+++ b/drivers/crypto/cavium/cpt/cptvf.h
@@ -0,0 +1,135 @@
+/*
+ * Copyright (C) 2016 Cavium, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License
+ * as published by the Free Software Foundation.
+ */
+
+#ifndef __CPTVF_H
+#define __CPTVF_H
+
+#include <linux/list.h>
+#include "cpt_common.h"
+
+/* Default command queue length */
+#define CPT_CMD_QLEN 2046
+#define CPT_CMD_QCHUNK_SIZE 1023
+
+/* Default command timeout in seconds */
+#define CPT_COMMAND_TIMEOUT 4
+#define CPT_TIMER_THOLD 0xFFFF
+#define CPT_NUM_QS_PER_VF 1
+#define CPT_INST_SIZE 64
+#define CPT_NEXT_CHUNK_PTR_SIZE 8
+
+#define CPT_VF_MSIX_VECTORS 2
+#define CPT_VF_INTR_MBOX_MASK BIT(0)
+#define CPT_VF_INTR_DOVF_MASK BIT(1)
+#define CPT_VF_INTR_IRDE_MASK BIT(2)
+#define CPT_VF_INTR_NWRP_MASK BIT(3)
+#define CPT_VF_INTR_SERR_MASK BIT(4)
+#define DMA_DIRECT_DIRECT 0 /* Input DIRECT, Output DIRECT */
+#define DMA_GATHER_SCATTER 1
+#define FROM_DPTR 1
+
+/**
+ * Enumeration cpt_vf_int_vec_e
+ *
+ * CPT VF MSI-X Vector Enumeration
+ * Enumerates the MSI-X interrupt vectors.
+ */
+enum cpt_vf_int_vec_e {
+ CPT_VF_INT_VEC_E_MISC = 0x00,
+ CPT_VF_INT_VEC_E_DONE = 0x01
+};
+
+struct command_chunk {
+ u8 *head;
+ dma_addr_t dma_addr;
+ u32 size; /* Chunk size, max CPT_INST_CHUNK_MAX_SIZE */
+ struct hlist_node nextchunk;
+};
+
+struct command_queue {
+ spinlock_t lock; /* command queue lock */
+ u32 idx; /* Command queue host write idx */
+ u32 nchunks; /* Number of command chunks */
+ struct command_chunk *qhead; /* Command queue head, instructions
+ * are inserted here
+ */
+ struct hlist_head chead;
+};
+
+struct command_qinfo {
+ u32 cmd_size;
+ u32 qchunksize; /* Command queue chunk size */
+ struct command_queue queue[CPT_NUM_QS_PER_VF];
+};
+
+struct pending_entry {
+ u8 busy; /* Entry status (free/busy) */
+
+ volatile u64 *completion_addr; /* Completion address */
+ void *post_arg;
+ void (*callback)(int, void *); /* Kernel ASYNC request callabck */
+ void *callback_arg; /* Kernel ASYNC request callabck arg */
+};
+
+struct pending_queue {
+ struct pending_entry *head; /* head of the queue */
+ u32 front; /* Process work from here */
+ u32 rear; /* Append new work here */
+ atomic64_t pending_count;
+ spinlock_t lock; /* Queue lock */
+};
+
+struct pending_qinfo {
+ u32 nr_queues; /* Number of queues supported */
+ u32 qlen; /* Queue length */
+ struct pending_queue queue[CPT_NUM_QS_PER_VF];
+};
+
+#define for_each_pending_queue(qinfo, q, i) \
+ for (i = 0, q = &qinfo->queue[i]; i < qinfo->nr_queues; i++, \
+ q = &qinfo->queue[i])
+
+struct cpt_vf {
+ u16 flags; /* Flags to hold device status bits */
+ u8 vfid; /* Device Index 0...CPT_MAX_VF_NUM */
+ u8 vftype; /* VF type of SE_TYPE(1) or AE_TYPE(1) */
+ u8 vfgrp; /* VF group (0 - 8) */
+ u8 node; /* Operating node: Bits (46:44) in BAR0 address */
+ u8 priority; /* VF priority ring: 1-High proirity round
+ * robin ring;0-Low priority round robin ring;
+ */
+ struct pci_dev *pdev; /* pci device handle */
+ void __iomem *reg_base; /* Register start address */
+ void *wqe_info; /* BH worker info */
+ /* MSI-X */
+ bool msix_enabled;
+ struct msix_entry msix_entries[CPT_VF_MSIX_VECTORS];
+ bool irq_allocated[CPT_VF_MSIX_VECTORS];
+ cpumask_var_t affinity_mask[CPT_VF_MSIX_VECTORS];
+ /* Command and Pending queues */
+ u32 qsize;
+ u32 nr_queues;
+ struct command_qinfo cqinfo; /* Command queue information */
+ struct pending_qinfo pqinfo; /* Pending queue information */
+ /* VF-PF mailbox communication */
+ bool pf_acked;
+ bool pf_nacked;
+};
+
+int cptvf_send_vf_up(struct cpt_vf *cptvf);
+int cptvf_send_vf_down(struct cpt_vf *cptvf);
+int cptvf_send_vf_to_grp_msg(struct cpt_vf *cptvf);
+int cptvf_send_vf_priority_msg(struct cpt_vf *cptvf);
+int cptvf_send_vq_size_msg(struct cpt_vf *cptvf);
+int cptvf_check_pf_ready(struct cpt_vf *cptvf);
+void cptvf_handle_mbox_intr(struct cpt_vf *cptvf);
+void cvm_crypto_exit(void);
+int cvm_crypto_init(struct cpt_vf *cptvf);
+void vq_post_process(struct cpt_vf *cptvf, u32 qno);
+void cptvf_write_vq_doorbell(struct cpt_vf *cptvf, u32 val);
+#endif /* __CPTVF_H */
diff --git a/drivers/crypto/cavium/cpt/cptvf_algs.c b/drivers/crypto/cavium/cpt/cptvf_algs.c
new file mode 100644
index 0000000..cc853f9
--- /dev/null
+++ b/drivers/crypto/cavium/cpt/cptvf_algs.c
@@ -0,0 +1,444 @@
+
+/*
+ * Copyright (C) 2016 Cavium, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License
+ * as published by the Free Software Foundation.
+ */
+
+#include <crypto/aes.h>
+#include <crypto/algapi.h>
+#include <crypto/authenc.h>
+#include <crypto/cryptd.h>
+#include <crypto/crypto_wq.h>
+#include <crypto/des.h>
+#include <crypto/xts.h>
+#include <linux/crypto.h>
+#include <linux/err.h>
+#include <linux/list.h>
+#include <linux/scatterlist.h>
+
+#include "cptvf.h"
+#include "cptvf_algs.h"
+
+struct cpt_device_handle {
+ void *cdev[MAX_DEVICES];
+ u32 dev_count;
+};
+
+static struct cpt_device_handle dev_handle;
+
+static void cvm_callback(u32 status, void *arg)
+{
+ struct crypto_async_request *req = (struct crypto_async_request *)arg;
+
+ req->complete(req, !status);
+}
+
+static inline void update_input_iv(struct cpt_request_info *req_info,
+ u8 *iv, u32 enc_iv_len,
+ u32 *argcnt)
+{
+ /* Setting the iv information */
+ req_info->in[*argcnt].vptr = (void *)iv;
+ req_info->in[*argcnt].size = enc_iv_len;
+ req_info->req.dlen += enc_iv_len;
+
+ ++(*argcnt);
+}
+
+static inline void update_output_iv(struct cpt_request_info *req_info,
+ u8 *iv, u32 enc_iv_len,
+ u32 *argcnt)
+{
+ /* Setting the iv information */
+ req_info->out[*argcnt].vptr = (void *)iv;
+ req_info->out[*argcnt].size = enc_iv_len;
+ req_info->rlen += enc_iv_len;
+
+ ++(*argcnt);
+}
+
+static inline void update_input_data(struct cpt_request_info *req_info,
+ struct scatterlist *inp_sg,
+ u32 nbytes, u32 *argcnt)
+{
+ req_info->req.dlen += nbytes;
+
+ while (nbytes) {
+ u32 len = min(nbytes, inp_sg->length);
+ u8 *ptr = sg_virt(inp_sg);
+
+ req_info->in[*argcnt].vptr = (void *)ptr;
+ req_info->in[*argcnt].size = len;
+ nbytes -= len;
+
+ ++(*argcnt);
+ ++inp_sg;
+ }
+}
+
+static inline void update_output_data(struct cpt_request_info *req_info,
+ struct scatterlist *outp_sg,
+ u32 nbytes, u32 *argcnt)
+{
+ req_info->rlen += nbytes;
+
+ while (nbytes) {
+ u32 len = min(nbytes, outp_sg->length);
+ u8 *ptr = sg_virt(outp_sg);
+
+ req_info->out[*argcnt].vptr = (void *)ptr;
+ req_info->out[*argcnt].size = len;
+ nbytes -= len;
+ ++(*argcnt);
+ ++outp_sg;
+ }
+}
+
+static inline u32 create_ctx_hdr(struct ablkcipher_request *req, u32 enc,
+ u32 cipher_type, u32 aes_key_type,
+ u32 *argcnt)
+{
+ struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
+ struct cvm_enc_ctx *ctx = crypto_ablkcipher_ctx(tfm);
+ struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req);
+ struct fc_context *fctx = &rctx->fctx;
+ u64 *offset_control = &rctx->control_word;
+ u32 enc_iv_len = crypto_ablkcipher_ivsize(tfm);
+ struct cpt_request_info *req_info = &rctx->cpt_req;
+ u64 *ctrl_flags = NULL;
+
+ req_info->ctrl.s.grp = 0;
+ req_info->ctrl.s.dma_mode = DMA_GATHER_SCATTER;
+ req_info->ctrl.s.se_req = SE_CORE_REQ;
+
+ req_info->req.opcode.s.major = MAJOR_OP_FC |
+ DMA_MODE_FLAG(DMA_GATHER_SCATTER);
+ if (enc)
+ req_info->req.opcode.s.minor = 2;
+ else
+ req_info->req.opcode.s.minor = 3;
+
+ req_info->req.param1 = req->nbytes; /* Encryption Data length */
+ req_info->req.param2 = 0; /*Auth data length */
+
+ fctx->enc.enc_ctrl.e.enc_cipher = cipher_type;
+ fctx->enc.enc_ctrl.e.aes_key = aes_key_type;
+ fctx->enc.enc_ctrl.e.iv_source = FROM_DPTR;
+
+ if (cipher_type == AES_XTS)
+ memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2);
+ else
+ memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len);
+ ctrl_flags = (u64 *)&fctx->enc.enc_ctrl.flags;
+ *ctrl_flags = cpu_to_be64(*ctrl_flags);
+
+ *offset_control = cpu_to_be64(((u64)(enc_iv_len) << 16));
+ /* Storing Packet Data Information in offset
+ * Control Word First 8 bytes
+ */
+ req_info->in[*argcnt].vptr = (u8 *)offset_control;
+ req_info->in[*argcnt].size = CONTROL_WORD_LEN;
+ req_info->req.dlen += CONTROL_WORD_LEN;
+ ++(*argcnt);
+
+ req_info->in[*argcnt].vptr = (u8 *)fctx;
+ req_info->in[*argcnt].size = sizeof(struct fc_context);
+ req_info->req.dlen += sizeof(struct fc_context);
+
+ ++(*argcnt);
+
+ return 0;
+}
+
+static inline u32 create_input_list(struct ablkcipher_request *req, u32 enc,
+ u32 cipher_type, u32 aes_key_type,
+ u32 enc_iv_len)
+{
+ struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req);
+ struct cpt_request_info *req_info = &rctx->cpt_req;
+ u32 argcnt = 0;
+
+ create_ctx_hdr(req, enc, cipher_type, aes_key_type, &argcnt);
+ update_input_iv(req_info, req->info, enc_iv_len, &argcnt);
+ update_input_data(req_info, req->src, req->nbytes, &argcnt);
+ req_info->incnt = argcnt;
+
+ return 0;
+}
+
+static inline void store_cb_info(struct ablkcipher_request *req,
+ struct cpt_request_info *req_info)
+{
+ req_info->callback = (void *)cvm_callback;
+ req_info->callback_arg = (void *)&req->base;
+}
+
+static inline void create_output_list(struct ablkcipher_request *req,
+ u32 cipher_type,
+ u32 enc_iv_len)
+{
+ struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req);
+ struct cpt_request_info *req_info = &rctx->cpt_req;
+ u32 argcnt = 0;
+
+ /* OUTPUT Buffer Processing
+ * AES encryption/decryption output would be
+ * received in the following format
+ *
+ * ------IV--------|------ENCRYPTED/DECRYPTED DATA-----|
+ * [ 16 Bytes/ [ Request Enc/Dec/ DATA Len AES CBC ]
+ */
+ /* Reading IV information */
+ update_output_iv(req_info, req->info, enc_iv_len, &argcnt);
+ update_output_data(req_info, req->dst, req->nbytes, &argcnt);
+ req_info->outcnt = argcnt;
+}
+
+static inline int cvm_enc_dec(struct ablkcipher_request *req, u32 enc,
+ u32 cipher_type)
+{
+ struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
+ struct cvm_enc_ctx *ctx = crypto_ablkcipher_ctx(tfm);
+ u32 key_type = AES_128_BIT;
+ struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req);
+ u32 enc_iv_len = crypto_ablkcipher_ivsize(tfm);
+ struct fc_context *fctx = &rctx->fctx;
+ struct cpt_request_info *req_info = &rctx->cpt_req;
+ void *cdev = NULL;
+ int status;
+
+ switch (ctx->key_len) {
+ case 16:
+ key_type = AES_128_BIT;
+ break;
+ case 24:
+ key_type = AES_192_BIT;
+ break;
+ case 32:
+ if (cipher_type == AES_XTS)
+ key_type = AES_128_BIT;
+ else
+ key_type = AES_256_BIT;
+ break;
+ case 64:
+ if (cipher_type == AES_XTS)
+ key_type = AES_256_BIT;
+ else
+ return -EINVAL;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ if (cipher_type == DES3_CBC)
+ key_type = 0;
+
+ memset(req_info, 0, sizeof(struct cpt_request_info));
+ memset(fctx, 0, sizeof(struct fc_context));
+ create_input_list(req, enc, cipher_type, key_type, enc_iv_len);
+ create_output_list(req, cipher_type, enc_iv_len);
+ store_cb_info(req, req_info);
+ cdev = dev_handle.cdev[smp_processor_id()];
+ status = cptvf_do_request(cdev, req_info);
+ /* We perform an asynchronous send and once
+ * the request is completed the driver would
+ * intimate through registered call back functions
+ */
+
+ if (status)
+ return status;
+ else
+ return -EINPROGRESS;
+}
+
+int cvm_des3_encrypt_cbc(struct ablkcipher_request *req)
+{
+ return cvm_enc_dec(req, true, DES3_CBC);
+}
+
+int cvm_des3_decrypt_cbc(struct ablkcipher_request *req)
+{
+ return cvm_enc_dec(req, false, DES3_CBC);
+}
+
+int cvm_aes_encrypt_xts(struct ablkcipher_request *req)
+{
+ return cvm_enc_dec(req, true, AES_XTS);
+}
+
+int cvm_aes_decrypt_xts(struct ablkcipher_request *req)
+{
+ return cvm_enc_dec(req, false, AES_XTS);
+}
+
+int cvm_aes_encrypt_cbc(struct ablkcipher_request *req)
+{
+ return cvm_enc_dec(req, true, AES_CBC);
+}
+
+int cvm_aes_decrypt_cbc(struct ablkcipher_request *req)
+{
+ return cvm_enc_dec(req, false, AES_CBC);
+}
+
+int cvm_xts_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
+ u32 keylen)
+{
+ struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
+ struct cvm_enc_ctx *ctx = crypto_tfm_ctx(tfm);
+ int err;
+ const u8 *key1 = key;
+ const u8 *key2 = key + (keylen / 2);
+
+ err = xts_check_key(tfm, key, keylen);
+ if (err)
+ return err;
+ ctx->key_len = keylen;
+ memcpy(ctx->enc_key, key1, keylen / 2);
+ memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2);
+
+ return 0;
+}
+
+int cvm_enc_dec_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
+ u32 keylen)
+{
+ struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
+ struct cvm_enc_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ if ((keylen == 16) || (keylen == 24) || (keylen == 32)) {
+ ctx->key_len = keylen;
+ memcpy(ctx->enc_key, key, keylen);
+ return 0;
+ }
+ crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
+
+ return -EINVAL;
+}
+
+int cvm_enc_dec_init(struct crypto_tfm *tfm)
+{
+ struct cvm_enc_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ memset(ctx, 0, sizeof(*ctx));
+ tfm->crt_ablkcipher.reqsize = sizeof(struct cvm_req_ctx) +
+ sizeof(struct ablkcipher_request);
+ /* Additional memory for ablkcipher_request is
+ * allocated since the cryptd daemon uses
+ * this memory for request_ctx information
+ */
+
+ return 0;
+}
+
+struct crypto_alg algs[] = { {
+ .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct cvm_enc_ctx),
+ .cra_alignmask = 7,
+ .cra_priority = 4001,
+ .cra_name = "xts(aes)",
+ .cra_driver_name = "cavium-xts-aes",
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_u = {
+ .ablkcipher = {
+ .ivsize = AES_BLOCK_SIZE,
+ .min_keysize = 2 * AES_MIN_KEY_SIZE,
+ .max_keysize = 2 * AES_MAX_KEY_SIZE,
+ .setkey = cvm_xts_setkey,
+ .encrypt = cvm_aes_encrypt_xts,
+ .decrypt = cvm_aes_decrypt_xts,
+ },
+ },
+ .cra_init = cvm_enc_dec_init,
+ .cra_module = THIS_MODULE,
+}, {
+ .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct cvm_enc_ctx),
+ .cra_alignmask = 7,
+ .cra_priority = 4001,
+ .cra_name = "cbc(aes)",
+ .cra_driver_name = "cavium-cbc-aes",
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_u = {
+ .ablkcipher = {
+ .ivsize = AES_BLOCK_SIZE,
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .setkey = cvm_enc_dec_setkey,
+ .encrypt = cvm_aes_encrypt_cbc,
+ .decrypt = cvm_aes_decrypt_cbc,
+ },
+ },
+ .cra_init = cvm_enc_dec_init,
+ .cra_module = THIS_MODULE,
+}, {
+ .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
+ .cra_blocksize = DES3_EDE_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct cvm_des3_ctx),
+ .cra_alignmask = 7,
+ .cra_priority = 4001,
+ .cra_name = "cbc(des3_ede)",
+ .cra_driver_name = "cavium-cbc-des3_ede",
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_u = {
+ .ablkcipher = {
+ .min_keysize = DES3_EDE_KEY_SIZE,
+ .max_keysize = DES3_EDE_KEY_SIZE,
+ .ivsize = DES_BLOCK_SIZE,
+ .setkey = cvm_enc_dec_setkey,
+ .encrypt = cvm_des3_encrypt_cbc,
+ .decrypt = cvm_des3_decrypt_cbc,
+ },
+ },
+ .cra_init = cvm_enc_dec_init,
+ .cra_module = THIS_MODULE,
+} };
+
+static inline int cav_register_algs(void)
+{
+ int err = 0;
+
+ err = crypto_register_algs(algs, ARRAY_SIZE(algs));
+ if (err)
+ return err;
+
+ return 0;
+}
+
+static inline void cav_unregister_algs(void)
+{
+ crypto_unregister_algs(algs, ARRAY_SIZE(algs));
+}
+
+int cvm_crypto_init(struct cpt_vf *cptvf)
+{
+ struct pci_dev *pdev = cptvf->pdev;
+ u32 dev_count;
+
+ dev_count = dev_handle.dev_count;
+ dev_handle.cdev[dev_count] = cptvf;
+ dev_handle.dev_count++;
+
+ if (dev_count == 3) {
+ if (cav_register_algs()) {
+ dev_err(&pdev->dev, "Error in registering crypto algorithms\n");
+ return -EINVAL;
+ }
+ }
+
+ return 0;
+}
+
+void cvm_crypto_exit(void)
+{
+ u32 dev_count;
+
+ dev_count = --dev_handle.dev_count;
+ if (!dev_count)
+ cav_unregister_algs();
+}
diff --git a/drivers/crypto/cavium/cpt/cptvf_algs.h b/drivers/crypto/cavium/cpt/cptvf_algs.h
new file mode 100644
index 0000000..a12050d
--- /dev/null
+++ b/drivers/crypto/cavium/cpt/cptvf_algs.h
@@ -0,0 +1,113 @@
+/*
+ * Copyright (C) 2016 Cavium, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License
+ * as published by the Free Software Foundation.
+ */
+
+#ifndef _CPTVF_ALGS_H_
+#define _CPTVF_ALGS_H_
+
+#include "request_manager.h"
+
+#define MAX_DEVICES 16
+#define MAJOR_OP_FC 0x33
+#define MAX_ENC_KEY_SIZE 32
+#define MAX_HASH_KEY_SIZE 64
+#define MAX_KEY_SIZE (MAX_ENC_KEY_SIZE + MAX_HASH_KEY_SIZE)
+#define CONTROL_WORD_LEN 8
+#define KEY2_OFFSET 48
+
+#define DMA_MODE_FLAG(dma_mode) \
+ (((dma_mode) == DMA_GATHER_SCATTER) ? (1 << 7) : 0)
+
+enum req_type {
+ AE_CORE_REQ,
+ SE_CORE_REQ,
+};
+
+enum cipher_type {
+ DES3_CBC = 0x1,
+ DES3_ECB = 0x2,
+ AES_CBC = 0x3,
+ AES_ECB = 0x4,
+ AES_CFB = 0x5,
+ AES_CTR = 0x6,
+ AES_GCM = 0x7,
+ AES_XTS = 0x8
+};
+
+enum aes_type {
+ AES_128_BIT = 0x1,
+ AES_192_BIT = 0x2,
+ AES_256_BIT = 0x3
+};
+
+union encr_ctrl {
+ u64 flags;
+ struct {
+#if defined(__BIG_ENDIAN_BITFIELD)
+ u64 enc_cipher:4;
+ u64 reserved1:1;
+ u64 aes_key:2;
+ u64 iv_source:1;
+ u64 hash_type:4;
+ u64 reserved2:3;
+ u64 auth_input_type:1;
+ u64 mac_len:8;
+ u64 reserved3:8;
+ u64 encr_offset:16;
+ u64 iv_offset:8;
+ u64 auth_offset:8;
+#else
+ u64 auth_offset:8;
+ u64 iv_offset:8;
+ u64 encr_offset:16;
+ u64 reserved3:8;
+ u64 mac_len:8;
+ u64 auth_input_type:1;
+ u64 reserved2:3;
+ u64 hash_type:4;
+ u64 iv_source:1;
+ u64 aes_key:2;
+ u64 reserved1:1;
+ u64 enc_cipher:4;
+#endif
+ } e;
+};
+
+struct enc_context {
+ union encr_ctrl enc_ctrl;
+ u8 encr_key[32];
+ u8 encr_iv[16];
+};
+
+struct fchmac_context {
+ u8 ipad[64];
+ u8 opad[64]; /* or OPAD */
+};
+
+struct fc_context {
+ struct enc_context enc;
+ struct fchmac_context hmac;
+};
+
+struct cvm_enc_ctx {
+ u32 key_len;
+ u8 enc_key[MAX_KEY_SIZE];
+};
+
+struct cvm_des3_ctx {
+ u32 key_len;
+ u8 des3_key[MAX_KEY_SIZE];
+};
+
+struct cvm_req_ctx {
+ struct cpt_request_info cpt_req;
+ u64 control_word;
+ struct fc_context fctx;
+};
+
+int cptvf_do_request(void *cptvf, struct cpt_request_info *req);
+#endif /*_CPTVF_ALGS_H_*/
diff --git a/drivers/crypto/cavium/cpt/cptvf_main.c b/drivers/crypto/cavium/cpt/cptvf_main.c
new file mode 100644
index 0000000..527bdc3
--- /dev/null
+++ b/drivers/crypto/cavium/cpt/cptvf_main.c
@@ -0,0 +1,936 @@
+/*
+ * Copyright (C) 2016 Cavium, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License
+ * as published by the Free Software Foundation.
+ */
+
+#include <linux/interrupt.h>
+#include <linux/module.h>
+
+#include "cptvf.h"
+
+#define DRV_NAME "thunder-cptvf"
+#define DRV_VERSION "1.0"
+
+struct cptvf_wqe {
+ struct tasklet_struct twork;
+ void *cptvf;
+ u32 qno;
+};
+
+struct cptvf_wqe_info {
+ struct cptvf_wqe vq_wqe[CPT_NUM_QS_PER_VF];
+};
+
+static void vq_work_handler(unsigned long data)
+{
+ struct cptvf_wqe_info *cwqe_info = (struct cptvf_wqe_info *)data;
+ struct cptvf_wqe *cwqe = &cwqe_info->vq_wqe[0];
+
+ vq_post_process(cwqe->cptvf, cwqe->qno);
+}
+
+static int init_worker_threads(struct cpt_vf *cptvf)
+{
+ struct pci_dev *pdev = cptvf->pdev;
+ struct cptvf_wqe_info *cwqe_info;
+ int i;
+
+ cwqe_info = kzalloc(sizeof(*cwqe_info), GFP_KERNEL);
+ if (!cwqe_info)
+ return -ENOMEM;
+
+ if (cptvf->nr_queues) {
+ dev_info(&pdev->dev, "Creating VQ worker threads (%d)\n",
+ cptvf->nr_queues);
+ }
+
+ for (i = 0; i < cptvf->nr_queues; i++) {
+ tasklet_init(&cwqe_info->vq_wqe[i].twork, vq_work_handler,
+ (u64)cwqe_info);
+ cwqe_info->vq_wqe[i].qno = i;
+ cwqe_info->vq_wqe[i].cptvf = cptvf;
+ }
+
+ cptvf->wqe_info = cwqe_info;
+
+ return 0;
+}
+
+static void cleanup_worker_threads(struct cpt_vf *cptvf)
+{
+ struct cptvf_wqe_info *cwqe_info;
+ struct pci_dev *pdev = cptvf->pdev;
+ int i;
+
+ cwqe_info = (struct cptvf_wqe_info *)cptvf->wqe_info;
+ if (!cwqe_info)
+ return;
+
+ if (cptvf->nr_queues) {
+ dev_info(&pdev->dev, "Cleaning VQ worker threads (%u)\n",
+ cptvf->nr_queues);
+ }
+
+ for (i = 0; i < cptvf->nr_queues; i++)
+ tasklet_kill(&cwqe_info->vq_wqe[i].twork);
+
+ kzfree(cwqe_info);
+ cptvf->wqe_info = NULL;
+}
+
+static void free_pending_queues(struct pending_qinfo *pqinfo)
+{
+ int i;
+ struct pending_queue *queue;
+
+ for_each_pending_queue(pqinfo, queue, i) {
+ if (!queue->head)
+ continue;
+
+ /* free single queue */
+ kzfree((queue->head));
+
+ queue->front = 0;
+ queue->rear = 0;
+
+ return;
+ }
+
+ pqinfo->qlen = 0;
+ pqinfo->nr_queues = 0;
+}
+
+static int alloc_pending_queues(struct pending_qinfo *pqinfo, u32 qlen,
+ u32 nr_queues)
+{
+ u32 i;
+ size_t size;
+ int ret;
+ struct pending_queue *queue = NULL;
+
+ pqinfo->nr_queues = nr_queues;
+ pqinfo->qlen = qlen;
+
+ size = (qlen * sizeof(struct pending_entry));
+
+ for_each_pending_queue(pqinfo, queue, i) {
+ queue->head = kzalloc((size), GFP_KERNEL);
+ if (!queue->head) {
+ ret = -ENOMEM;
+ goto pending_qfail;
+ }
+
+ queue->front = 0;
+ queue->rear = 0;
+ atomic64_set((&queue->pending_count), (0));
+
+ /* init queue spin lock */
+ spin_lock_init(&queue->lock);
+ }
+
+ return 0;
+
+pending_qfail:
+ free_pending_queues(pqinfo);
+
+ return ret;
+}
+
+static int init_pending_queues(struct cpt_vf *cptvf, u32 qlen, u32 nr_queues)
+{
+ struct pci_dev *pdev = cptvf->pdev;
+ int ret;
+
+ if (!nr_queues)
+ return 0;
+
+ ret = alloc_pending_queues(&cptvf->pqinfo, qlen, nr_queues);
+ if (ret) {
+ dev_err(&pdev->dev, "failed to setup pending queues (%u)\n",
+ nr_queues);
+ return ret;
+ }
+
+ return 0;
+}
+
+static void cleanup_pending_queues(struct cpt_vf *cptvf)
+{
+ struct pci_dev *pdev = cptvf->pdev;
+
+ if (!cptvf->nr_queues)
+ return;
+
+ dev_info(&pdev->dev, "Cleaning VQ pending queue (%u)\n",
+ cptvf->nr_queues);
+ free_pending_queues(&cptvf->pqinfo);
+}
+
+static void free_command_queues(struct cpt_vf *cptvf,
+ struct command_qinfo *cqinfo)
+{
+ int i;
+ struct command_queue *queue = NULL;
+ struct command_chunk *chunk = NULL;
+ struct pci_dev *pdev = cptvf->pdev;
+ struct hlist_node *node;
+
+ /* clean up for each queue */
+ for (i = 0; i < cptvf->nr_queues; i++) {
+ queue = &cqinfo->queue[i];
+ if (hlist_empty(&cqinfo->queue[i].chead))
+ continue;
+
+ hlist_for_each_entry_safe(chunk, node, &cqinfo->queue[i].chead,
+ nextchunk) {
+ dma_free_coherent(&pdev->dev, chunk->size,
+ chunk->head,
+ chunk->dma_addr);
+ chunk->head = NULL;
+ chunk->dma_addr = 0;
+ hlist_del(&chunk->nextchunk);
+ kzfree(chunk);
+ }
+
+ queue->nchunks = 0;
+ queue->idx = 0;
+ }
+
+ /* common cleanup */
+ cqinfo->cmd_size = 0;
+}
+
+static int alloc_command_queues(struct cpt_vf *cptvf,
+ struct command_qinfo *cqinfo, size_t cmd_size,
+ u32 qlen)
+{
+ int i;
+ size_t q_size;
+ struct command_queue *queue = NULL;
+ struct pci_dev *pdev = cptvf->pdev;
+
+ /* common init */
+ cqinfo->cmd_size = cmd_size;
+ /* Qsize in dwords, needed for SADDR config, 1-next chunk pointer */
+ cptvf->qsize = min(qlen, cqinfo->qchunksize) *
+ CPT_NEXT_CHUNK_PTR_SIZE + 1;
+ /* Qsize in bytes to create space for alignment */
+ q_size = qlen * cqinfo->cmd_size;
+
+ /* per queue initialization */
+ for (i = 0; i < cptvf->nr_queues; i++) {
+ size_t c_size = 0;
+ size_t rem_q_size = q_size;
+ struct command_chunk *curr = NULL, *first = NULL, *last = NULL;
+ u32 qcsize_bytes = cqinfo->qchunksize * cqinfo->cmd_size;
+
+ queue = &cqinfo->queue[i];
+ INIT_HLIST_HEAD(&cqinfo->queue[i].chead);
+ do {
+ curr = kzalloc(sizeof(*curr), GFP_KERNEL);
+ if (!curr)
+ goto cmd_qfail;
+
+ c_size = (rem_q_size > qcsize_bytes) ? qcsize_bytes :
+ rem_q_size;
+ curr->head = (u8 *)dma_zalloc_coherent(&pdev->dev,
+ c_size + CPT_NEXT_CHUNK_PTR_SIZE,
+ &curr->dma_addr, GFP_KERNEL);
+ if (!curr->head) {
+ dev_err(&pdev->dev, "Command Q (%d) chunk (%d) allocation failed\n",
+ i, queue->nchunks);
+ goto cmd_qfail;
+ }
+
+ curr->size = c_size;
+ if (queue->nchunks == 0) {
+ hlist_add_head(&curr->nextchunk,
+ &cqinfo->queue[i].chead);
+ first = curr;
+ } else {
+ hlist_add_behind(&curr->nextchunk,
+ &last->nextchunk);
+ }
+
+ queue->nchunks++;
+ rem_q_size -= c_size;
+ if (last)
+ *((u64 *)(&last->head[last->size])) = (u64)curr->dma_addr;
+
+ last = curr;
+ } while (rem_q_size);
+
+ /* Make the queue circular */
+ /* Tie back last chunk entry to head */
+ curr = first;
+ *((u64 *)(&last->head[last->size])) = (u64)curr->dma_addr;
+ queue->qhead = curr;
+ spin_lock_init(&queue->lock);
+ }
+ return 0;
+
+cmd_qfail:
+ free_command_queues(cptvf, cqinfo);
+ return -ENOMEM;
+}
+
+static int init_command_queues(struct cpt_vf *cptvf, u32 qlen)
+{
+ struct pci_dev *pdev = cptvf->pdev;
+ int ret;
+
+ /* setup AE command queues */
+ ret = alloc_command_queues(cptvf, &cptvf->cqinfo, CPT_INST_SIZE,
+ qlen);
+ if (ret) {
+ dev_err(&pdev->dev, "failed to allocate AE command queues (%u)\n",
+ cptvf->nr_queues);
+ return ret;
+ }
+
+ return ret;
+}
+
+static void cleanup_command_queues(struct cpt_vf *cptvf)
+{
+ struct pci_dev *pdev = cptvf->pdev;
+
+ if (!cptvf->nr_queues)
+ return;
+
+ dev_info(&pdev->dev, "Cleaning VQ command queue (%u)\n",
+ cptvf->nr_queues);
+ free_command_queues(cptvf, &cptvf->cqinfo);
+}
+
+static void cptvf_sw_cleanup(struct cpt_vf *cptvf)
+{
+ cleanup_worker_threads(cptvf);
+ cleanup_pending_queues(cptvf);
+ cleanup_command_queues(cptvf);
+}
+
+static int cptvf_sw_init(struct cpt_vf *cptvf, u32 qlen, u32 nr_queues)
+{
+ struct pci_dev *pdev = cptvf->pdev;
+ int ret = 0;
+ u32 max_dev_queues = 0;
+
+ max_dev_queues = CPT_NUM_QS_PER_VF;
+ /* possible cpus */
+ nr_queues = min_t(u32, nr_queues, max_dev_queues);
+ cptvf->nr_queues = nr_queues;
+
+ ret = init_command_queues(cptvf, qlen);
+ if (ret) {
+ dev_err(&pdev->dev, "Failed to setup command queues (%u)\n",
+ nr_queues);
+ return ret;
+ }
+
+ ret = init_pending_queues(cptvf, qlen, nr_queues);
+ if (ret) {
+ dev_err(&pdev->dev, "Failed to setup pending queues (%u)\n",
+ nr_queues);
+ goto setup_pqfail;
+ }
+
+ /* Create worker threads for BH processing */
+ ret = init_worker_threads(cptvf);
+ if (ret) {
+ dev_err(&pdev->dev, "Failed to setup worker threads\n");
+ goto init_work_fail;
+ }
+
+ return 0;
+
+init_work_fail:
+ cleanup_worker_threads(cptvf);
+ cleanup_pending_queues(cptvf);
+
+setup_pqfail:
+ cleanup_command_queues(cptvf);
+
+ return ret;
+}
+
+static void cptvf_disable_msix(struct cpt_vf *cptvf)
+{
+ if (cptvf->msix_enabled) {
+ pci_disable_msix(cptvf->pdev);
+ cptvf->msix_enabled = 0;
+ }
+}
+
+static int cptvf_enable_msix(struct cpt_vf *cptvf)
+{
+ int i, ret;
+
+ for (i = 0; i < CPT_VF_MSIX_VECTORS; i++)
+ cptvf->msix_entries[i].entry = i;
+
+ ret = pci_enable_msix(cptvf->pdev, cptvf->msix_entries,
+ CPT_VF_MSIX_VECTORS);
+ if (ret) {
+ dev_err(&cptvf->pdev->dev, "Request for #%d msix vectors failed\n",
+ CPT_VF_MSIX_VECTORS);
+ return ret;
+ }
+
+ cptvf->msix_enabled = 1;
+ /* Mark MSIX enabled */
+ cptvf->flags |= CPT_FLAG_MSIX_ENABLED;
+
+ return 0;
+}
+
+static void cptvf_free_all_interrupts(struct cpt_vf *cptvf)
+{
+ int irq;
+
+ for (irq = 0; irq < CPT_VF_MSIX_VECTORS; irq++) {
+ if (cptvf->irq_allocated[irq])
+ irq_set_affinity_hint(cptvf->msix_entries[irq].vector,
+ NULL);
+ free_cpumask_var(cptvf->affinity_mask[irq]);
+ free_irq(cptvf->msix_entries[irq].vector, cptvf);
+ cptvf->irq_allocated[irq] = false;
+ }
+}
+
+static void cptvf_write_vq_ctl(struct cpt_vf *cptvf, bool val)
+{
+ union cptx_vqx_ctl vqx_ctl;
+
+ vqx_ctl.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_CTL(0, 0));
+ vqx_ctl.s.ena = val;
+ cpt_write_csr64(cptvf->reg_base, CPTX_VQX_CTL(0, 0), vqx_ctl.u);
+}
+
+void cptvf_write_vq_doorbell(struct cpt_vf *cptvf, u32 val)
+{
+ union cptx_vqx_doorbell vqx_dbell;
+
+ vqx_dbell.u = cpt_read_csr64(cptvf->reg_base,
+ CPTX_VQX_DOORBELL(0, 0));
+ vqx_dbell.s.dbell_cnt = val * 8; /* Num of Instructions * 8 words */
+ cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DOORBELL(0, 0),
+ vqx_dbell.u);
+}
+
+static void cptvf_write_vq_inprog(struct cpt_vf *cptvf, u8 val)
+{
+ union cptx_vqx_inprog vqx_inprg;
+
+ vqx_inprg.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_INPROG(0, 0));
+ vqx_inprg.s.inflight = val;
+ cpt_write_csr64(cptvf->reg_base, CPTX_VQX_INPROG(0, 0), vqx_inprg.u);
+}
+
+static void cptvf_write_vq_done_numwait(struct cpt_vf *cptvf, u32 val)
+{
+ union cptx_vqx_done_wait vqx_dwait;
+
+ vqx_dwait.u = cpt_read_csr64(cptvf->reg_base,
+ CPTX_VQX_DONE_WAIT(0, 0));
+ vqx_dwait.s.num_wait = val;
+ cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_WAIT(0, 0),
+ vqx_dwait.u);
+}
+
+static void cptvf_write_vq_done_timewait(struct cpt_vf *cptvf, u16 time)
+{
+ union cptx_vqx_done_wait vqx_dwait;
+
+ vqx_dwait.u = cpt_read_csr64(cptvf->reg_base,
+ CPTX_VQX_DONE_WAIT(0, 0));
+ vqx_dwait.s.time_wait = time;
+ cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_WAIT(0, 0),
+ vqx_dwait.u);
+}
+
+static void cptvf_enable_swerr_interrupts(struct cpt_vf *cptvf)
+{
+ union cptx_vqx_misc_ena_w1s vqx_misc_ena;
+
+ vqx_misc_ena.u = cpt_read_csr64(cptvf->reg_base,
+ CPTX_VQX_MISC_ENA_W1S(0, 0));
+ /* Set mbox(0) interupts for the requested vf */
+ vqx_misc_ena.s.swerr = 1;
+ cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_ENA_W1S(0, 0),
+ vqx_misc_ena.u);
+}
+
+static void cptvf_enable_mbox_interrupts(struct cpt_vf *cptvf)
+{
+ union cptx_vqx_misc_ena_w1s vqx_misc_ena;
+
+ vqx_misc_ena.u = cpt_read_csr64(cptvf->reg_base,
+ CPTX_VQX_MISC_ENA_W1S(0, 0));
+ /* Set mbox(0) interupts for the requested vf */
+ vqx_misc_ena.s.mbox = 1;
+ cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_ENA_W1S(0, 0),
+ vqx_misc_ena.u);
+}
+
+static void cptvf_enable_done_interrupts(struct cpt_vf *cptvf)
+{
+ union cptx_vqx_done_ena_w1s vqx_done_ena;
+
+ vqx_done_ena.u = cpt_read_csr64(cptvf->reg_base,
+ CPTX_VQX_DONE_ENA_W1S(0, 0));
+ /* Set DONE interrupt for the requested vf */
+ vqx_done_ena.s.done = 1;
+ cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_ENA_W1S(0, 0),
+ vqx_done_ena.u);
+}
+
+static void cptvf_clear_dovf_intr(struct cpt_vf *cptvf)
+{
+ union cptx_vqx_misc_int vqx_misc_int;
+
+ vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
+ CPTX_VQX_MISC_INT(0, 0));
+ /* W1C for the VF */
+ vqx_misc_int.s.dovf = 1;
+ cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0),
+ vqx_misc_int.u);
+}
+
+static void cptvf_clear_irde_intr(struct cpt_vf *cptvf)
+{
+ union cptx_vqx_misc_int vqx_misc_int;
+
+ vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
+ CPTX_VQX_MISC_INT(0, 0));
+ /* W1C for the VF */
+ vqx_misc_int.s.irde = 1;
+ cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0),
+ vqx_misc_int.u);
+}
+
+static void cptvf_clear_nwrp_intr(struct cpt_vf *cptvf)
+{
+ union cptx_vqx_misc_int vqx_misc_int;
+
+ vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
+ CPTX_VQX_MISC_INT(0, 0));
+ /* W1C for the VF */
+ vqx_misc_int.s.nwrp = 1;
+ cpt_write_csr64(cptvf->reg_base,
+ CPTX_VQX_MISC_INT(0, 0), vqx_misc_int.u);
+}
+
+static void cptvf_clear_mbox_intr(struct cpt_vf *cptvf)
+{
+ union cptx_vqx_misc_int vqx_misc_int;
+
+ vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
+ CPTX_VQX_MISC_INT(0, 0));
+ /* W1C for the VF */
+ vqx_misc_int.s.mbox = 1;
+ cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0),
+ vqx_misc_int.u);
+}
+
+static void cptvf_clear_swerr_intr(struct cpt_vf *cptvf)
+{
+ union cptx_vqx_misc_int vqx_misc_int;
+
+ vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
+ CPTX_VQX_MISC_INT(0, 0));
+ /* W1C for the VF */
+ vqx_misc_int.s.swerr = 1;
+ cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0),
+ vqx_misc_int.u);
+}
+
+static u64 cptvf_read_vf_misc_intr_status(struct cpt_vf *cptvf)
+{
+ return cpt_read_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0));
+}
+
+static irqreturn_t cptvf_misc_intr_handler(int irq, void *cptvf_irq)
+{
+ struct cpt_vf *cptvf = (struct cpt_vf *)cptvf_irq;
+ struct pci_dev *pdev = cptvf->pdev;
+ u64 intr;
+
+ intr = cptvf_read_vf_misc_intr_status(cptvf);
+ /*Check for MISC interrupt types*/
+ if (likely(intr & CPT_VF_INTR_MBOX_MASK)) {
+ dev_err(&pdev->dev, "Mailbox interrupt 0x%llx on CPT VF %d\n",
+ intr, cptvf->vfid);
+ cptvf_handle_mbox_intr(cptvf);
+ cptvf_clear_mbox_intr(cptvf);
+ } else if (unlikely(intr & CPT_VF_INTR_DOVF_MASK)) {
+ cptvf_clear_dovf_intr(cptvf);
+ /*Clear doorbell count*/
+ cptvf_write_vq_doorbell(cptvf, 0);
+ dev_err(&pdev->dev, "Doorbell overflow error interrupt 0x%llx on CPT VF %d\n",
+ intr, cptvf->vfid);
+ } else if (unlikely(intr & CPT_VF_INTR_IRDE_MASK)) {
+ cptvf_clear_irde_intr(cptvf);
+ dev_err(&pdev->dev, "Instruction NCB read error interrupt 0x%llx on CPT VF %d\n",
+ intr, cptvf->vfid);
+ } else if (unlikely(intr & CPT_VF_INTR_NWRP_MASK)) {
+ cptvf_clear_nwrp_intr(cptvf);
+ dev_err(&pdev->dev, "NCB response write error interrupt 0x%llx on CPT VF %d\n",
+ intr, cptvf->vfid);
+ } else if (unlikely(intr & CPT_VF_INTR_SERR_MASK)) {
+ cptvf_clear_swerr_intr(cptvf);
+ dev_err(&pdev->dev, "Software error interrupt 0x%llx on CPT VF %d\n",
+ intr, cptvf->vfid);
+ } else {
+ dev_err(&pdev->dev, "Unhandled interrupt in CPT VF %d\n",
+ cptvf->vfid);
+ }
+
+ return IRQ_HANDLED;
+}
+
+static inline struct cptvf_wqe *get_cptvf_vq_wqe(struct cpt_vf *cptvf,
+ int qno)
+{
+ struct cptvf_wqe_info *nwqe_info;
+
+ if (unlikely(qno >= cptvf->nr_queues))
+ return NULL;
+ nwqe_info = (struct cptvf_wqe_info *)cptvf->wqe_info;
+
+ return &nwqe_info->vq_wqe[qno];
+}
+
+static inline u32 cptvf_read_vq_done_count(struct cpt_vf *cptvf)
+{
+ union cptx_vqx_done vqx_done;
+
+ vqx_done.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_DONE(0, 0));
+ return vqx_done.s.done;
+}
+
+static inline void cptvf_write_vq_done_ack(struct cpt_vf *cptvf,
+ u32 ackcnt)
+{
+ union cptx_vqx_done_ack vqx_dack_cnt;
+
+ vqx_dack_cnt.u = cpt_read_csr64(cptvf->reg_base,
+ CPTX_VQX_DONE_ACK(0, 0));
+ vqx_dack_cnt.s.done_ack = ackcnt;
+ cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_ACK(0, 0),
+ vqx_dack_cnt.u);
+}
+
+static irqreturn_t cptvf_done_intr_handler(int irq, void *cptvf_irq)
+{
+ struct cpt_vf *cptvf = (struct cpt_vf *)cptvf_irq;
+ struct pci_dev *pdev = cptvf->pdev;
+ /* Read the number of completions */
+ u32 intr = cptvf_read_vq_done_count(cptvf);
+
+ if (intr) {
+ struct cptvf_wqe *wqe;
+
+ /* Acknowledge the number of
+ * scheduled completions for processing
+ */
+ cptvf_write_vq_done_ack(cptvf, intr);
+ wqe = get_cptvf_vq_wqe(cptvf, 0);
+ if (unlikely(!wqe)) {
+ dev_err(&pdev->dev, "No work to schedule for VF (%d)",
+ cptvf->vfid);
+ return IRQ_NONE;
+ }
+ tasklet_hi_schedule(&wqe->twork);
+ }
+
+ return IRQ_HANDLED;
+}
+
+static int cptvf_register_misc_intr(struct cpt_vf *cptvf)
+{
+ struct pci_dev *pdev = cptvf->pdev;
+ int ret;
+
+ /* Register misc interrupt handlers */
+ ret = request_irq(cptvf->msix_entries[CPT_VF_INT_VEC_E_MISC].vector,
+ cptvf_misc_intr_handler, 0, "CPT VF misc intr",
+ cptvf);
+ if (ret)
+ goto fail;
+
+ cptvf->irq_allocated[CPT_VF_INT_VEC_E_MISC] = true;
+
+ /* Enable mailbox interrupt */
+ cptvf_enable_mbox_interrupts(cptvf);
+ cptvf_enable_swerr_interrupts(cptvf);
+
+ return 0;
+
+fail:
+ dev_err(&pdev->dev, "Request misc irq failed");
+ cptvf_free_all_interrupts(cptvf);
+ return ret;
+}
+
+static int cptvf_register_done_intr(struct cpt_vf *cptvf)
+{
+ struct pci_dev *pdev = cptvf->pdev;
+ int ret;
+
+ /* Register DONE interrupt handlers */
+ ret = request_irq(cptvf->msix_entries[CPT_VF_INT_VEC_E_DONE].vector,
+ cptvf_done_intr_handler, 0, "CPT VF done intr",
+ cptvf);
+ if (ret)
+ goto fail;
+
+ cptvf->irq_allocated[CPT_VF_INT_VEC_E_DONE] = true;
+
+ /* Enable mailbox interrupt */
+ cptvf_enable_done_interrupts(cptvf);
+ return 0;
+
+fail:
+ dev_err(&pdev->dev, "Request done irq failed\n");
+ cptvf_free_all_interrupts(cptvf);
+ return ret;
+}
+
+static void cptvf_unregister_interrupts(struct cpt_vf *cptvf)
+{
+ cptvf_free_all_interrupts(cptvf);
+ cptvf_disable_msix(cptvf);
+}
+
+static void cptvf_set_irq_affinity(struct cpt_vf *cptvf)
+{
+ struct pci_dev *pdev = cptvf->pdev;
+ int vec, cpu;
+ int irqnum;
+
+ for (vec = 0; vec < CPT_VF_MSIX_VECTORS; vec++) {
+ if (!cptvf->irq_allocated[vec])
+ continue;
+
+ if (!zalloc_cpumask_var(&cptvf->affinity_mask[vec],
+ GFP_KERNEL)) {
+ dev_err(&pdev->dev, "Allocation failed for affinity_mask for VF %d",
+ cptvf->vfid);
+ return;
+ }
+
+ cpu = cptvf->vfid % num_online_cpus();
+ cpumask_set_cpu(cpumask_local_spread(cpu, cptvf->node),
+ cptvf->affinity_mask[vec]);
+ irqnum = cptvf->msix_entries[vec].vector;
+ irq_set_affinity_hint(irqnum, cptvf->affinity_mask[vec]);
+ }
+}
+
+static void cptvf_write_vq_saddr(struct cpt_vf *cptvf, u64 val)
+{
+ union cptx_vqx_saddr vqx_saddr;
+
+ vqx_saddr.u = val;
+ cpt_write_csr64(cptvf->reg_base, CPTX_VQX_SADDR(0, 0), vqx_saddr.u);
+}
+
+void cptvf_device_init(struct cpt_vf *cptvf)
+{
+ u64 base_addr = 0;
+
+ /* Disable the VQ */
+ cptvf_write_vq_ctl(cptvf, 0);
+ /* Reset the doorbell */
+ cptvf_write_vq_doorbell(cptvf, 0);
+ /* Clear inflight */
+ cptvf_write_vq_inprog(cptvf, 0);
+ /* Write VQ SADDR */
+ /* TODO: for now only one queue, so hard coded */
+ base_addr = (u64)(cptvf->cqinfo.queue[0].qhead->dma_addr);
+ cptvf_write_vq_saddr(cptvf, base_addr);
+ /* Configure timerhold / coalescence */
+ cptvf_write_vq_done_timewait(cptvf, CPT_TIMER_THOLD);
+ cptvf_write_vq_done_numwait(cptvf, 1);
+ /* Enable the VQ */
+ cptvf_write_vq_ctl(cptvf, 1);
+ /* Flag the VF ready */
+ cptvf->flags |= CPT_FLAG_DEVICE_READY;
+}
+
+static int cptvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
+{
+ struct device *dev = &pdev->dev;
+ struct cpt_vf *cptvf;
+ int err;
+
+ cptvf = devm_kzalloc(dev, sizeof(*cptvf), GFP_KERNEL);
+ if (!cptvf)
+ return -ENOMEM;
+
+ pci_set_drvdata(pdev, cptvf);
+ cptvf->pdev = pdev;
+ err = pci_enable_device(pdev);
+ if (err) {
+ dev_err(dev, "Failed to enable PCI device\n");
+ pci_set_drvdata(pdev, NULL);
+ return err;
+ }
+
+ err = pci_request_regions(pdev, DRV_NAME);
+ if (err) {
+ dev_err(dev, "PCI request regions failed 0x%x\n", err);
+ goto cptvf_err_disable_device;
+ }
+ /* Mark as VF driver */
+ cptvf->flags |= CPT_FLAG_VF_DRIVER;
+ err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48));
+ if (err) {
+ dev_err(dev, "Unable to get usable DMA configuration\n");
+ goto cptvf_err_release_regions;
+ }
+
+ err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48));
+ if (err) {
+ dev_err(dev, "Unable to get 48-bit DMA for consistent allocations\n");
+ goto cptvf_err_release_regions;
+ }
+
+ /* MAP PF's configuration registers */
+ cptvf->reg_base = pcim_iomap(pdev, 0, 0);
+ if (!cptvf->reg_base) {
+ dev_err(dev, "Cannot map config register space, aborting\n");
+ err = -ENOMEM;
+ goto cptvf_err_release_regions;
+ }
+
+ cptvf->node = dev_to_node(&pdev->dev);
+ /* Enable MSI-X */
+ err = cptvf_enable_msix(cptvf);
+ if (err) {
+ dev_err(dev, "cptvf_enable_msix() failed");
+ goto cptvf_err_release_regions;
+ }
+
+ /* Register mailbox interrupts */
+ cptvf_register_misc_intr(cptvf);
+
+ /* Check ready with PF */
+ /* Gets chip ID / device Id from PF if ready */
+ err = cptvf_check_pf_ready(cptvf);
+ if (err) {
+ dev_err(dev, "PF not responding to READY msg");
+ goto cptvf_err_release_regions;
+ }
+
+ /* CPT VF software resources initialization */
+ cptvf->cqinfo.qchunksize = CPT_CMD_QCHUNK_SIZE;
+ err = cptvf_sw_init(cptvf, CPT_CMD_QLEN, CPT_NUM_QS_PER_VF);
+ if (err) {
+ dev_err(dev, "cptvf_sw_init() failed");
+ goto cptvf_err_release_regions;
+ }
+ /* Convey VQ LEN to PF */
+ err = cptvf_send_vq_size_msg(cptvf);
+ if (err) {
+ dev_err(dev, "PF not responding to QLEN msg");
+ goto cptvf_err_release_regions;
+ }
+
+ /* CPT VF device initialization */
+ cptvf_device_init(cptvf);
+ /* Send msg to PF to assign currnet Q to required group */
+ cptvf->vfgrp = 1;
+ err = cptvf_send_vf_to_grp_msg(cptvf);
+ if (err) {
+ dev_err(dev, "PF not responding to VF_GRP msg");
+ goto cptvf_err_release_regions;
+ }
+
+ cptvf->priority = 1;
+ err = cptvf_send_vf_priority_msg(cptvf);
+ if (err) {
+ dev_err(dev, "PF not responding to VF_PRIO msg");
+ goto cptvf_err_release_regions;
+ }
+ /* Register DONE interrupts */
+ err = cptvf_register_done_intr(cptvf);
+ if (err)
+ goto cptvf_err_release_regions;
+
+ /* Set irq affinity masks */
+ cptvf_set_irq_affinity(cptvf);
+ /* Convey UP to PF */
+ err = cptvf_send_vf_up(cptvf);
+ if (err) {
+ dev_err(dev, "PF not responding to UP msg");
+ goto cptvf_up_fail;
+ }
+ err = cvm_crypto_init(cptvf);
+ if (err) {
+ dev_err(dev, "Algorithm register failed\n");
+ goto cptvf_up_fail;
+ }
+ return 0;
+
+cptvf_up_fail:
+ cptvf_unregister_interrupts(cptvf);
+cptvf_err_release_regions:
+ pci_release_regions(pdev);
+cptvf_err_disable_device:
+ pci_disable_device(pdev);
+ pci_set_drvdata(pdev, NULL);
+
+ return err;
+}
+
+static void cptvf_remove(struct pci_dev *pdev)
+{
+ struct cpt_vf *cptvf = pci_get_drvdata(pdev);
+
+ if (!cptvf)
+ dev_err(&pdev->dev, "Invalid CPT-VF device\n");
+
+ /* Convey DOWN to PF */
+ if (cptvf_send_vf_down(cptvf)) {
+ dev_err(&pdev->dev, "PF not responding to DOWN msg");
+ } else {
+ cptvf_unregister_interrupts(cptvf);
+ cptvf_sw_cleanup(cptvf);
+ pci_set_drvdata(pdev, NULL);
+ pci_release_regions(pdev);
+ pci_disable_device(pdev);
+ cvm_crypto_exit();
+ }
+}
+
+static void cptvf_shutdown(struct pci_dev *pdev)
+{
+ cptvf_remove(pdev);
+}
+
+/* Supported devices */
+static const struct pci_device_id cptvf_id_table[] = {
+ {PCI_VDEVICE(CAVIUM, CPT_81XX_PCI_VF_DEVICE_ID), 0},
+ { 0, } /* end of table */
+};
+
+static struct pci_driver cptvf_pci_driver = {
+ .name = DRV_NAME,
+ .id_table = cptvf_id_table,
+ .probe = cptvf_probe,
+ .remove = cptvf_remove,
+ .shutdown = cptvf_shutdown,
+};
+
+module_pci_driver(cptvf_pci_driver);
+
+MODULE_AUTHOR("George Cherian <george.cherian@cavium.com>");
+MODULE_DESCRIPTION("Cavium Thunder CPT Virtual Function Driver");
+MODULE_LICENSE("GPL v2");
+MODULE_VERSION(DRV_VERSION);
+MODULE_DEVICE_TABLE(pci, cptvf_id_table);
diff --git a/drivers/crypto/cavium/cpt/cptvf_mbox.c b/drivers/crypto/cavium/cpt/cptvf_mbox.c
new file mode 100644
index 0000000..d5ec3b8
--- /dev/null
+++ b/drivers/crypto/cavium/cpt/cptvf_mbox.c
@@ -0,0 +1,211 @@
+/*
+ * Copyright (C) 2016 Cavium, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License
+ * as published by the Free Software Foundation.
+ */
+
+#include "cptvf.h"
+
+static void cptvf_send_msg_to_pf(struct cpt_vf *cptvf, struct cpt_mbox *mbx)
+{
+ /* Writing mbox(1) causes interrupt */
+ cpt_write_csr64(cptvf->reg_base, CPTX_VFX_PF_MBOXX(0, 0, 0),
+ mbx->msg);
+ cpt_write_csr64(cptvf->reg_base, CPTX_VFX_PF_MBOXX(0, 0, 1),
+ mbx->data);
+}
+
+/* ACKs PF's mailbox message
+ */
+void cptvf_mbox_send_ack(struct cpt_vf *cptvf, struct cpt_mbox *mbx)
+{
+ mbx->msg = CPT_MBOX_MSG_TYPE_ACK;
+ cptvf_send_msg_to_pf(cptvf, mbx);
+}
+
+/* NACKs PF's mailbox message that VF is not able to
+ * complete the action
+ */
+void cptvf_mbox_send_nack(struct cpt_vf *cptvf, struct cpt_mbox *mbx)
+{
+ mbx->msg = CPT_MBOX_MSG_TYPE_NACK;
+ cptvf_send_msg_to_pf(cptvf, mbx);
+}
+
+/* Interrupt handler to handle mailbox messages from VFs */
+void cptvf_handle_mbox_intr(struct cpt_vf *cptvf)
+{
+ struct cpt_mbox mbx = {};
+
+ /*
+ * MBOX[0] contains msg
+ * MBOX[1] contains data
+ */
+ mbx.msg = cpt_read_csr64(cptvf->reg_base, CPTX_VFX_PF_MBOXX(0, 0, 0));
+ mbx.data = cpt_read_csr64(cptvf->reg_base, CPTX_VFX_PF_MBOXX(0, 0, 1));
+ dev_dbg(&cptvf->pdev->dev, "%s: Mailbox msg 0x%llx from PF\n",
+ __func__, mbx.msg);
+ switch (mbx.msg) {
+ case CPT_MSG_READY:
+ {
+ cptvf->pf_acked = true;
+ cptvf->vfid = mbx.data;
+ dev_dbg(&cptvf->pdev->dev, "Received VFID %d\n", cptvf->vfid);
+ break;
+ }
+ case CPT_MSG_QBIND_GRP:
+ cptvf->pf_acked = true;
+ cptvf->vftype = mbx.data;
+ dev_dbg(&cptvf->pdev->dev, "VF %d type %s group %d\n",
+ cptvf->vfid, ((mbx.data == SE_TYPES) ? "SE" : "AE"),
+ cptvf->vfgrp);
+ break;
+ case CPT_MBOX_MSG_TYPE_ACK:
+ cptvf->pf_acked = true;
+ break;
+ case CPT_MBOX_MSG_TYPE_NACK:
+ cptvf->pf_nacked = true;
+ break;
+ default:
+ dev_err(&cptvf->pdev->dev, "Invalid msg from PF, msg 0x%llx\n",
+ mbx.msg);
+ break;
+ }
+}
+
+static int cptvf_send_msg_to_pf_timeout(struct cpt_vf *cptvf,
+ struct cpt_mbox *mbx)
+{
+ int timeout = CPT_MBOX_MSG_TIMEOUT;
+ int sleep = 10;
+
+ cptvf->pf_acked = false;
+ cptvf->pf_nacked = false;
+ cptvf_send_msg_to_pf(cptvf, mbx);
+ /* Wait for previous message to be acked, timeout 2sec */
+ while (!cptvf->pf_acked) {
+ if (cptvf->pf_nacked)
+ return -EINVAL;
+ msleep(sleep);
+ if (cptvf->pf_acked)
+ break;
+ timeout -= sleep;
+ if (!timeout) {
+ dev_err(&cptvf->pdev->dev, "PF didn't ack to mbox msg %llx from VF%u\n",
+ (mbx->msg & 0xFF), cptvf->vfid);
+ return -EBUSY;
+ }
+ }
+
+ return 0;
+}
+
+/*
+ * Checks if VF is able to comminicate with PF
+ * and also gets the CPT number this VF is associated to.
+ */
+int cptvf_check_pf_ready(struct cpt_vf *cptvf)
+{
+ struct pci_dev *pdev = cptvf->pdev;
+ struct cpt_mbox mbx = {};
+
+ mbx.msg = CPT_MSG_READY;
+ if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) {
+ dev_err(&pdev->dev, "PF didn't respond to READY msg\n");
+ return -EBUSY;
+ }
+
+ return 0;
+}
+
+/*
+ * Communicate VQs size to PF to program CPT(0)_PF_Q(0-15)_CTL of the VF.
+ * Must be ACKed.
+ */
+int cptvf_send_vq_size_msg(struct cpt_vf *cptvf)
+{
+ struct pci_dev *pdev = cptvf->pdev;
+ struct cpt_mbox mbx = {};
+
+ mbx.msg = CPT_MSG_QLEN;
+ mbx.data = cptvf->qsize;
+ if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) {
+ dev_err(&pdev->dev, "PF didn't respond to vq_size msg\n");
+ return -EBUSY;
+ }
+
+ return 0;
+}
+
+/*
+ * Communicate VF group required to PF and get the VQ binded to that group
+ */
+int cptvf_send_vf_to_grp_msg(struct cpt_vf *cptvf)
+{
+ struct pci_dev *pdev = cptvf->pdev;
+ struct cpt_mbox mbx = {};
+
+ mbx.msg = CPT_MSG_QBIND_GRP;
+ /* Convey group of the VF */
+ mbx.data = cptvf->vfgrp;
+ if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) {
+ dev_err(&pdev->dev, "PF didn't respond to vf_type msg\n");
+ return -EBUSY;
+ }
+
+ return 0;
+}
+
+/*
+ * Communicate VF group required to PF and get the VQ binded to that group
+ */
+int cptvf_send_vf_priority_msg(struct cpt_vf *cptvf)
+{
+ struct pci_dev *pdev = cptvf->pdev;
+ struct cpt_mbox mbx = {};
+
+ mbx.msg = CPT_MSG_VQ_PRIORITY;
+ /* Convey group of the VF */
+ mbx.data = cptvf->priority;
+ if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) {
+ dev_err(&pdev->dev, "PF didn't respond to vf_type msg\n");
+ return -EBUSY;
+ }
+ return 0;
+}
+
+/*
+ * Communicate to PF that VF is UP and running
+ */
+int cptvf_send_vf_up(struct cpt_vf *cptvf)
+{
+ struct pci_dev *pdev = cptvf->pdev;
+ struct cpt_mbox mbx = {};
+
+ mbx.msg = CPT_MSG_VF_UP;
+ if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) {
+ dev_err(&pdev->dev, "PF didn't respond to UP msg\n");
+ return -EBUSY;
+ }
+
+ return 0;
+}
+
+/*
+ * Communicate to PF that VF is DOWN and running
+ */
+int cptvf_send_vf_down(struct cpt_vf *cptvf)
+{
+ struct pci_dev *pdev = cptvf->pdev;
+ struct cpt_mbox mbx = {};
+
+ mbx.msg = CPT_MSG_VF_DOWN;
+ if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) {
+ dev_err(&pdev->dev, "PF didn't respond to DOWN msg\n");
+ return -EBUSY;
+ }
+
+ return 0;
+}
diff --git a/drivers/crypto/cavium/cpt/cptvf_reqmanager.c b/drivers/crypto/cavium/cpt/cptvf_reqmanager.c
new file mode 100644
index 0000000..7f57f30
--- /dev/null
+++ b/drivers/crypto/cavium/cpt/cptvf_reqmanager.c
@@ -0,0 +1,593 @@
+/*
+ * Copyright (C) 2016 Cavium, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License
+ * as published by the Free Software Foundation.
+ */
+
+#include "cptvf.h"
+#include "request_manager.h"
+
+/**
+ * get_free_pending_entry - get free entry from pending queue
+ * @param pqinfo: pending_qinfo structure
+ * @param qno: queue number
+ */
+static struct pending_entry *get_free_pending_entry(struct pending_queue *q,
+ int qlen)
+{
+ struct pending_entry *ent = NULL;
+
+ ent = &q->head[q->rear];
+ if (unlikely(ent->busy)) {
+ ent = NULL;
+ goto no_free_entry;
+ }
+
+ q->rear++;
+ if (unlikely(q->rear == qlen))
+ q->rear = 0;
+
+no_free_entry:
+ return ent;
+}
+
+static inline void pending_queue_inc_front(struct pending_qinfo *pqinfo,
+ int qno)
+{
+ struct pending_queue *queue = &pqinfo->queue[qno];
+
+ queue->front++;
+ if (unlikely(queue->front == pqinfo->qlen))
+ queue->front = 0;
+}
+
+static int setup_sgio_components(struct cpt_vf *cptvf, struct buf_ptr *list,
+ int buf_count, u8 *buffer)
+{
+ int ret = 0, i, j;
+ int components;
+ struct sglist_component *sg_ptr = NULL;
+ struct pci_dev *pdev = cptvf->pdev;
+
+ if (unlikely(!list)) {
+ dev_err(&pdev->dev, "Input List pointer is NULL\n");
+ return -EFAULT;
+ }
+
+ for (i = 0; i < buf_count; i++) {
+ if (likely(list[i].vptr)) {
+ list[i].dma_addr = dma_map_single(&pdev->dev,
+ list[i].vptr,
+ list[i].size,
+ DMA_BIDIRECTIONAL);
+ if (unlikely(dma_mapping_error(&pdev->dev,
+ list[i].dma_addr))) {
+ dev_err(&pdev->dev, "DMA map kernel buffer failed for component: %d\n",
+ i);
+ ret = -EIO;
+ goto sg_cleanup;
+ }
+ }
+ }
+
+ components = buf_count / 4;
+ sg_ptr = (struct sglist_component *)buffer;
+ for (i = 0; i < components; i++) {
+ sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size);
+ sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size);
+ sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size);
+ sg_ptr->u.s.len3 = cpu_to_be16(list[i * 4 + 3].size);
+ sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr);
+ sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr);
+ sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr);
+ sg_ptr->ptr3 = cpu_to_be64(list[i * 4 + 3].dma_addr);
+ sg_ptr++;
+ }
+
+ components = buf_count % 4;
+
+ switch (components) {
+ case 3:
+ sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size);
+ sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr);
+ /* Fall through */
+ case 2:
+ sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size);
+ sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr);
+ /* Fall through */
+ case 1:
+ sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size);
+ sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr);
+ break;
+ default:
+ break;
+ }
+
+ return ret;
+
+sg_cleanup:
+ for (j = 0; j < i; j++) {
+ if (list[j].dma_addr) {
+ dma_unmap_single(&pdev->dev, list[i].dma_addr,
+ list[i].size, DMA_BIDIRECTIONAL);
+ }
+
+ list[j].dma_addr = 0;
+ }
+
+ return ret;
+}
+
+static inline int setup_sgio_list(struct cpt_vf *cptvf,
+ struct cpt_info_buffer *info,
+ struct cpt_request_info *req)
+{
+ u16 g_sz_bytes = 0, s_sz_bytes = 0;
+ int ret = 0;
+ struct pci_dev *pdev = cptvf->pdev;
+
+ if (req->incnt > MAX_SG_IN_CNT || req->outcnt > MAX_SG_OUT_CNT) {
+ dev_err(&pdev->dev, "Request SG components are higher than supported\n");
+ ret = -EINVAL;
+ goto scatter_gather_clean;
+ }
+
+ /* Setup gather (input) components */
+ g_sz_bytes = ((req->incnt + 3) / 4) * sizeof(struct sglist_component);
+ info->gather_components = kzalloc(g_sz_bytes, GFP_KERNEL);
+ if (!info->gather_components) {
+ ret = -ENOMEM;
+ goto scatter_gather_clean;
+ }
+
+ ret = setup_sgio_components(cptvf, req->in,
+ req->incnt,
+ info->gather_components);
+ if (ret) {
+ dev_err(&pdev->dev, "Failed to setup gather list\n");
+ ret = -EFAULT;
+ goto scatter_gather_clean;
+ }
+
+ /* Setup scatter (output) components */
+ s_sz_bytes = ((req->outcnt + 3) / 4) * sizeof(struct sglist_component);
+ info->scatter_components = kzalloc(s_sz_bytes, GFP_KERNEL);
+ if (!info->scatter_components) {
+ ret = -ENOMEM;
+ goto scatter_gather_clean;
+ }
+
+ ret = setup_sgio_components(cptvf, req->out,
+ req->outcnt,
+ info->scatter_components);
+ if (ret) {
+ dev_err(&pdev->dev, "Failed to setup gather list\n");
+ ret = -EFAULT;
+ goto scatter_gather_clean;
+ }
+
+ /* Create and initialize DPTR */
+ info->dlen = g_sz_bytes + s_sz_bytes + SG_LIST_HDR_SIZE;
+ info->in_buffer = kzalloc(info->dlen, GFP_KERNEL);
+ if (!info->in_buffer) {
+ ret = -ENOMEM;
+ goto scatter_gather_clean;
+ }
+
+ ((u16 *)info->in_buffer)[0] = req->outcnt;
+ ((u16 *)info->in_buffer)[1] = req->incnt;
+ ((u16 *)info->in_buffer)[2] = 0;
+ ((u16 *)info->in_buffer)[3] = 0;
+ *(u64 *)info->in_buffer = cpu_to_be64p((u64 *)info->in_buffer);
+
+ memcpy(&info->in_buffer[8], info->gather_components,
+ g_sz_bytes);
+ memcpy(&info->in_buffer[8 + g_sz_bytes],
+ info->scatter_components, s_sz_bytes);
+
+ info->dptr_baddr = dma_map_single(&pdev->dev,
+ (void *)info->in_buffer,
+ info->dlen,
+ DMA_BIDIRECTIONAL);
+ if (dma_mapping_error(&pdev->dev, info->dptr_baddr)) {
+ dev_err(&pdev->dev, "Mapping DPTR Failed %d\n", info->dlen);
+ ret = -EIO;
+ goto scatter_gather_clean;
+ }
+
+ /* Create and initialize RPTR */
+ info->out_buffer = kzalloc(COMPLETION_CODE_SIZE, GFP_KERNEL);
+ if (!info->out_buffer) {
+ ret = -ENOMEM;
+ goto scatter_gather_clean;
+ }
+
+ *((u64 *)info->out_buffer) = ~((u64)COMPLETION_CODE_INIT);
+ info->alternate_caddr = (u64 *)info->out_buffer;
+ info->rptr_baddr = dma_map_single(&pdev->dev,
+ (void *)info->out_buffer,
+ COMPLETION_CODE_SIZE,
+ DMA_BIDIRECTIONAL);
+ if (dma_mapping_error(&pdev->dev, info->rptr_baddr)) {
+ dev_err(&pdev->dev, "Mapping RPTR Failed %d\n",
+ COMPLETION_CODE_SIZE);
+ ret = -EIO;
+ goto scatter_gather_clean;
+ }
+
+ return 0;
+
+scatter_gather_clean:
+ return ret;
+}
+
+int send_cpt_command(struct cpt_vf *cptvf, union cpt_inst_s *cmd,
+ u32 qno)
+{
+ struct pci_dev *pdev = cptvf->pdev;
+ struct command_qinfo *qinfo = NULL;
+ struct command_queue *queue;
+ struct command_chunk *chunk;
+ u8 *ent;
+ int ret = 0;
+
+ if (unlikely(qno >= cptvf->nr_queues)) {
+ dev_err(&pdev->dev, "Invalid queue (qno: %d, nr_queues: %d)\n",
+ qno, cptvf->nr_queues);
+ return -EINVAL;
+ }
+
+ qinfo = &cptvf->cqinfo;
+ queue = &qinfo->queue[qno];
+ /* lock commad queue */
+ spin_lock(&queue->lock);
+ ent = &queue->qhead->head[queue->idx * qinfo->cmd_size];
+ memcpy(ent, (void *)cmd, qinfo->cmd_size);
+
+ if (++queue->idx >= queue->qhead->size / 64) {
+ struct hlist_node *node;
+
+ hlist_for_each(node, &queue->chead) {
+ chunk = hlist_entry(node, struct command_chunk,
+ nextchunk);
+ if (chunk == queue->qhead) {
+ continue;
+ } else {
+ queue->qhead = chunk;
+ break;
+ }
+ }
+ queue->idx = 0;
+ }
+ /* make sure all memory stores are done before ringing doorbell */
+ smp_wmb();
+ cptvf_write_vq_doorbell(cptvf, 1);
+ /* unlock command queue */
+ spin_unlock(&queue->lock);
+
+ return ret;
+}
+
+void do_request_cleanup(struct cpt_vf *cptvf,
+ struct cpt_info_buffer *info)
+{
+ int i;
+ struct pci_dev *pdev = cptvf->pdev;
+ struct cpt_request_info *req;
+
+ if (info->dptr_baddr)
+ dma_unmap_single(&pdev->dev, info->dptr_baddr,
+ info->dlen, DMA_BIDIRECTIONAL);
+
+ if (info->rptr_baddr)
+ dma_unmap_single(&pdev->dev, info->rptr_baddr,
+ COMPLETION_CODE_SIZE, DMA_BIDIRECTIONAL);
+
+ if (info->comp_baddr)
+ dma_unmap_single(&pdev->dev, info->comp_baddr,
+ sizeof(union cpt_res_s), DMA_BIDIRECTIONAL);
+
+ if (info->req) {
+ req = info->req;
+ for (i = 0; i < req->outcnt; i++) {
+ if (req->out[i].dma_addr)
+ dma_unmap_single(&pdev->dev,
+ req->out[i].dma_addr,
+ req->out[i].size,
+ DMA_BIDIRECTIONAL);
+ }
+
+ for (i = 0; i < req->incnt; i++) {
+ if (req->in[i].dma_addr)
+ dma_unmap_single(&pdev->dev,
+ req->in[i].dma_addr,
+ req->in[i].size,
+ DMA_BIDIRECTIONAL);
+ }
+ }
+
+ if (info->scatter_components)
+ kzfree(info->scatter_components);
+
+ if (info->gather_components)
+ kzfree(info->gather_components);
+
+ if (info->out_buffer)
+ kzfree(info->out_buffer);
+
+ if (info->in_buffer)
+ kzfree(info->in_buffer);
+
+ if (info->completion_addr)
+ kzfree((void *)info->completion_addr);
+
+ kzfree(info);
+}
+
+void do_post_process(struct cpt_vf *cptvf, struct cpt_info_buffer *info)
+{
+ struct pci_dev *pdev = cptvf->pdev;
+
+ if (!info || !cptvf) {
+ dev_err(&pdev->dev, "Input params are incorrect for post processing\n");
+ return;
+ }
+
+ do_request_cleanup(cptvf, info);
+}
+
+static inline void process_pending_queue(struct cpt_vf *cptvf,
+ struct pending_qinfo *pqinfo,
+ int qno)
+{
+ struct pci_dev *pdev = cptvf->pdev;
+ struct pending_queue *pqueue = &pqinfo->queue[qno];
+ struct pending_entry *pentry = NULL;
+ struct cpt_info_buffer *info = NULL;
+ union cpt_res_s *status = NULL;
+ unsigned char ccode;
+
+ while (1) {
+ spin_lock_bh(&pqueue->lock);
+ pentry = &pqueue->head[pqueue->front];
+ if (unlikely(!pentry->busy)) {
+ spin_unlock_bh(&pqueue->lock);
+ break;
+ }
+
+ info = (struct cpt_info_buffer *)pentry->post_arg;
+ if (unlikely(!info)) {
+ dev_err(&pdev->dev, "Pending Entry post arg NULL\n");
+ pending_queue_inc_front(pqinfo, qno);
+ spin_unlock_bh(&pqueue->lock);
+ continue;
+ }
+
+ status = (union cpt_res_s *)pentry->completion_addr;
+ ccode = status->s.compcode;
+ if ((status->s.compcode == CPT_COMP_E_FAULT) ||
+ (status->s.compcode == CPT_COMP_E_SWERR)) {
+ dev_err(&pdev->dev, "Request failed with %s\n",
+ (status->s.compcode == CPT_COMP_E_FAULT) ?
+ "DMA Fault" : "Software error");
+ pentry->completion_addr = NULL;
+ pentry->busy = false;
+ atomic64_dec((&pqueue->pending_count));
+ pentry->post_arg = NULL;
+ pending_queue_inc_front(pqinfo, qno);
+ do_request_cleanup(cptvf, info);
+ spin_unlock_bh(&pqueue->lock);
+ break;
+ } else if (status->s.compcode == COMPLETION_CODE_INIT) {
+ /* check for timeout */
+ if (time_after_eq(jiffies,
+ (info->time_in +
+ (CPT_COMMAND_TIMEOUT * HZ)))) {
+ dev_err(&pdev->dev, "Request timed out");
+ pentry->completion_addr = NULL;
+ pentry->busy = false;
+ atomic64_dec((&pqueue->pending_count));
+ pentry->post_arg = NULL;
+ pending_queue_inc_front(pqinfo, qno);
+ do_request_cleanup(cptvf, info);
+ spin_unlock_bh(&pqueue->lock);
+ break;
+ } else if ((*info->alternate_caddr ==
+ (~COMPLETION_CODE_INIT)) &&
+ (info->extra_time < TIME_IN_RESET_COUNT)) {
+ info->time_in = jiffies;
+ info->extra_time++;
+ spin_unlock_bh(&pqueue->lock);
+ break;
+ }
+ }
+
+ pentry->completion_addr = NULL;
+ pentry->busy = false;
+ pentry->post_arg = NULL;
+ atomic64_dec((&pqueue->pending_count));
+ pending_queue_inc_front(pqinfo, qno);
+ spin_unlock_bh(&pqueue->lock);
+
+ do_post_process(info->cptvf, info);
+ /*
+ * Calling callback after we find
+ * that the request has been serviced
+ */
+ pentry->callback(ccode, pentry->callback_arg);
+ }
+}
+
+int process_request(struct cpt_vf *cptvf, struct cpt_request_info *req)
+{
+ int ret = 0, clear = 0, queue = 0;
+ struct cpt_info_buffer *info = NULL;
+ struct cptvf_request *cpt_req = NULL;
+ union ctrl_info *ctrl = NULL;
+ union cpt_res_s *result = NULL;
+ struct pending_entry *pentry = NULL;
+ struct pending_queue *pqueue = NULL;
+ struct pci_dev *pdev = cptvf->pdev;
+ u8 group = 0;
+ struct cpt_vq_command vq_cmd;
+ union cpt_inst_s cptinst;
+
+ info = kzalloc(sizeof(*info), GFP_KERNEL);
+ if (unlikely(!info)) {
+ dev_err(&pdev->dev, "Unable to allocate memory for info_buffer\n");
+ return -ENOMEM;
+ }
+
+ cpt_req = (struct cptvf_request *)&req->req;
+ ctrl = (union ctrl_info *)&req->ctrl;
+
+ info->cptvf = cptvf;
+ group = ctrl->s.grp;
+ ret = setup_sgio_list(cptvf, info, req);
+ if (ret) {
+ dev_err(&pdev->dev, "Setting up SG list failed");
+ goto request_cleanup;
+ }
+
+ cpt_req->dlen = info->dlen;
+ /*
+ * Get buffer for union cpt_res_s response
+ * structure and its physical address
+ */
+ info->completion_addr = kzalloc(sizeof(union cpt_res_s), GFP_KERNEL);
+ if (unlikely(!info->completion_addr)) {
+ dev_err(&pdev->dev, "Unable to allocate memory for completion_addr\n");
+ return -ENOMEM;
+ }
+
+ result = (union cpt_res_s *)info->completion_addr;
+ result->s.compcode = COMPLETION_CODE_INIT;
+ info->comp_baddr = dma_map_single(&pdev->dev,
+ (void *)info->completion_addr,
+ sizeof(union cpt_res_s),
+ DMA_BIDIRECTIONAL);
+ if (dma_mapping_error(&pdev->dev, info->comp_baddr)) {
+ dev_err(&pdev->dev, "mapping compptr Failed %lu\n",
+ sizeof(union cpt_res_s));
+ ret = -EFAULT;
+ goto request_cleanup;
+ }
+
+ /* Fill the VQ command */
+ vq_cmd.cmd.u64 = 0;
+ vq_cmd.cmd.s.opcode = cpu_to_be16(cpt_req->opcode.flags);
+ vq_cmd.cmd.s.param1 = cpu_to_be16(cpt_req->param1);
+ vq_cmd.cmd.s.param2 = cpu_to_be16(cpt_req->param2);
+ vq_cmd.cmd.s.dlen = cpu_to_be16(cpt_req->dlen);
+
+ /* 64-bit swap for microcode data reads, not needed for addresses*/
+ vq_cmd.cmd.u64 = cpu_to_be64(vq_cmd.cmd.u64);
+ vq_cmd.dptr = info->dptr_baddr;
+ vq_cmd.rptr = info->rptr_baddr;
+ vq_cmd.cptr.u64 = 0;
+ vq_cmd.cptr.s.grp = group;
+ /* Get Pending Entry to submit command */
+ /* Always queue 0, because 1 queue per VF */
+ queue = 0;
+ pqueue = &cptvf->pqinfo.queue[queue];
+
+ if (atomic64_read(&pqueue->pending_count) > PENDING_THOLD) {
+ dev_err(&pdev->dev, "pending threshold reached\n");
+ process_pending_queue(cptvf, &cptvf->pqinfo, queue);
+ }
+
+get_pending_entry:
+ spin_lock_bh(&pqueue->lock);
+ pentry = get_free_pending_entry(pqueue, cptvf->pqinfo.qlen);
+ if (unlikely(!pentry)) {
+ spin_unlock_bh(&pqueue->lock);
+ if (clear == 0) {
+ process_pending_queue(cptvf, &cptvf->pqinfo, queue);
+ clear = 1;
+ goto get_pending_entry;
+ }
+ dev_err(&pdev->dev, "Get free entry failed\n");
+ dev_err(&pdev->dev, "queue: %d, rear: %d, front: %d\n",
+ queue, pqueue->rear, pqueue->front);
+ ret = -EFAULT;
+ goto request_cleanup;
+ }
+
+ pentry->completion_addr = info->completion_addr;
+ pentry->post_arg = (void *)info;
+ pentry->callback = req->callback;
+ pentry->callback_arg = req->callback_arg;
+ info->pentry = pentry;
+ pentry->busy = true;
+ atomic64_inc(&pqueue->pending_count);
+
+ /* Send CPT command */
+ info->pentry = pentry;
+ info->time_in = jiffies;
+ info->req = req;
+
+ /* Create the CPT_INST_S type command for HW intrepretation */
+ cptinst.s.doneint = true;
+ cptinst.s.res_addr = (u64)info->comp_baddr;
+ cptinst.s.tag = 0;
+ cptinst.s.grp = 0;
+ cptinst.s.wq_ptr = 0;
+ cptinst.s.ei0 = vq_cmd.cmd.u64;
+ cptinst.s.ei1 = vq_cmd.dptr;
+ cptinst.s.ei2 = vq_cmd.rptr;
+ cptinst.s.ei3 = vq_cmd.cptr.u64;
+
+ ret = send_cpt_command(cptvf, &cptinst, queue);
+ spin_unlock_bh(&pqueue->lock);
+ if (unlikely(ret)) {
+ dev_err(&pdev->dev, "Send command failed for AE\n");
+ ret = -EFAULT;
+ goto request_cleanup;
+ }
+
+ return 0;
+
+request_cleanup:
+ dev_dbg(&pdev->dev, "Failed to submit CPT command\n");
+ do_request_cleanup(cptvf, info);
+
+ return ret;
+}
+
+void vq_post_process(struct cpt_vf *cptvf, u32 qno)
+{
+ struct pci_dev *pdev = cptvf->pdev;
+
+ if (unlikely(qno > cptvf->nr_queues)) {
+ dev_err(&pdev->dev, "Request for post processing on invalid pending queue: %u\n",
+ qno);
+ return;
+ }
+
+ process_pending_queue(cptvf, &cptvf->pqinfo, qno);
+}
+
+int cptvf_do_request(void *vfdev, struct cpt_request_info *req)
+{
+ struct cpt_vf *cptvf = (struct cpt_vf *)vfdev;
+ struct pci_dev *pdev = cptvf->pdev;
+
+ if (!cpt_device_ready(cptvf)) {
+ dev_err(&pdev->dev, "CPT Device is not ready");
+ return -ENODEV;
+ }
+
+ if ((cptvf->vftype == SE_TYPES) && (!req->ctrl.s.se_req)) {
+ dev_err(&pdev->dev, "CPTVF-%d of SE TYPE got AE request",
+ cptvf->vfid);
+ return -EINVAL;
+ } else if ((cptvf->vftype == AE_TYPES) && (req->ctrl.s.se_req)) {
+ dev_err(&pdev->dev, "CPTVF-%d of AE TYPE got SE request",
+ cptvf->vfid);
+ return -EINVAL;
+ }
+
+ return process_request(cptvf, req);
+}
diff --git a/drivers/crypto/cavium/cpt/request_manager.h b/drivers/crypto/cavium/cpt/request_manager.h
new file mode 100644
index 0000000..80ee074
--- /dev/null
+++ b/drivers/crypto/cavium/cpt/request_manager.h
@@ -0,0 +1,147 @@
+/*
+ * Copyright (C) 2016 Cavium, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License
+ * as published by the Free Software Foundation.
+ */
+
+#ifndef __REQUEST_MANAGER_H
+#define __REQUEST_MANAGER_H
+
+#include "cpt_common.h"
+
+#define TIME_IN_RESET_COUNT 5
+#define COMPLETION_CODE_SIZE 8
+#define COMPLETION_CODE_INIT 0
+#define PENDING_THOLD 100
+#define MAX_SG_IN_CNT 12
+#define MAX_SG_OUT_CNT 13
+#define SG_LIST_HDR_SIZE 8
+#define MAX_BUF_CNT 16
+
+union ctrl_info {
+ u32 flags;
+ struct {
+#if defined(__BIG_ENDIAN_BITFIELD)
+ u32 reserved0:26;
+ u32 grp:3; /* Group bits */
+ u32 dma_mode:2; /* DMA mode */
+ u32 se_req:1;/* To SE core */
+#else
+ u32 se_req:1; /* To SE core */
+ u32 dma_mode:2; /* DMA mode */
+ u32 grp:3; /* Group bits */
+ u32 reserved0:26;
+#endif
+ } s;
+};
+
+union opcode_info {
+ u16 flags;
+ struct {
+ u8 major;
+ u8 minor;
+ } s;
+};
+
+struct cptvf_request {
+ union opcode_info opcode;
+ u16 param1;
+ u16 param2;
+ u16 dlen;
+};
+
+struct buf_ptr {
+ u8 *vptr;
+ dma_addr_t dma_addr;
+ u16 size;
+};
+
+struct cpt_request_info {
+ u8 incnt; /* Number of input buffers */
+ u8 outcnt; /* Number of output buffers */
+ u16 rlen; /* Output length */
+ union ctrl_info ctrl; /* User control information */
+ struct cptvf_request req; /* Request Information (Core specific) */
+
+ struct buf_ptr in[MAX_BUF_CNT];
+ struct buf_ptr out[MAX_BUF_CNT];
+
+ void (*callback)(int, void *); /* Kernel ASYNC request callabck */
+ void *callback_arg; /* Kernel ASYNC request callabck arg */
+};
+
+struct sglist_component {
+ union {
+ u64 len;
+ struct {
+ u16 len0;
+ u16 len1;
+ u16 len2;
+ u16 len3;
+ } s;
+ } u;
+ u64 ptr0;
+ u64 ptr1;
+ u64 ptr2;
+ u64 ptr3;
+};
+
+struct cpt_info_buffer {
+ struct cpt_vf *cptvf;
+ unsigned long time_in;
+ u8 extra_time;
+
+ struct cpt_request_info *req;
+ dma_addr_t dptr_baddr;
+ u32 dlen;
+ dma_addr_t rptr_baddr;
+ dma_addr_t comp_baddr;
+ u8 *in_buffer;
+ u8 *out_buffer;
+ u8 *gather_components;
+ u8 *scatter_components;
+
+ struct pending_entry *pentry;
+ volatile u64 *completion_addr;
+ volatile u64 *alternate_caddr;
+};
+
+/*
+ * CPT_INST_S software command definitions
+ * Words EI (0-3)
+ */
+union vq_cmd_word0 {
+ u64 u64;
+ struct {
+ u16 opcode;
+ u16 param1;
+ u16 param2;
+ u16 dlen;
+ } s;
+};
+
+union vq_cmd_word3 {
+ u64 u64;
+ struct {
+#if defined(__BIG_ENDIAN_BITFIELD)
+ u64 grp:3;
+ u64 cptr:61;
+#else
+ u64 cptr:61;
+ u64 grp:3;
+#endif
+ } s;
+};
+
+struct cpt_vq_command {
+ union vq_cmd_word0 cmd;
+ u64 dptr;
+ u64 rptr;
+ union vq_cmd_word3 cptr;
+};
+
+void vq_post_process(struct cpt_vf *cptvf, u32 qno);
+int process_request(struct cpt_vf *cptvf, struct cpt_request_info *req);
+#endif /* __REQUEST_MANAGER_H */
--
2.1.4
^ permalink raw reply related
* Re: [PATCH v2 2/5] async_tx: Handle DMA devices having support for fewer PQ coefficients
From: Vinod Koul @ 2017-02-07 16:42 UTC (permalink / raw)
To: Anup Patel
Cc: Dan Williams, Rob Herring, Mark Rutland, Herbert Xu,
David S . Miller, Jassi Brar, Ray Jui, Scott Branden, Jon Mason,
Rob Rice, BCM Kernel Feedback, dmaengine@vger.kernel.org,
Device Tree, linux-arm-kernel@lists.infradead.org,
linux-kernel@vger.kernel.org, linux-crypto, linux-raid
In-Reply-To: <CAALAos_yhXufdt79KWY0eugjF5oyJ8+NcPh2E-dEXnSQu2WhhA@mail.gmail.com>
On Tue, Feb 07, 2017 at 02:32:15PM +0530, Anup Patel wrote:
> On Tue, Feb 7, 2017 at 1:57 PM, Dan Williams <dan.j.williams@intel.com> wrote:
> > On Tue, Feb 7, 2017 at 12:16 AM, Anup Patel <anup.patel@broadcom.com> wrote:
> >> The DMAENGINE framework assumes that if PQ offload is supported by a
> >> DMA device then all 256 PQ coefficients are supported. This assumption
> >> does not hold anymore because we now have BCM-SBA-RAID offload engine
> >> which supports PQ offload with limited number of PQ coefficients.
> >>
> >> This patch extends async_tx APIs to handle DMA devices with support
> >> for fewer PQ coefficients.
> >>
> >> Signed-off-by: Anup Patel <anup.patel@broadcom.com>
> >> Reviewed-by: Scott Branden <scott.branden@broadcom.com>
> >
> > I don't like this approach. Define an interface for md to query the
> > offload engine once at the beginning of time. We should not be adding
> > any new extensions to async_tx.
>
> Even if we do capability checks in Linux MD, we still need a way
> for DMAENGINE drivers to advertise number of PQ coefficients
> handled by the HW.
If the question is only for advertising caps, then why not do as done
for dma_get_slave_caps(). you can add dma_get_pq_caps() so that clients (md)
in this case would know the HW capability.
> I agree capability checks should be done once in Linux MD but I don't
> see why this has to be part of BCM-SBA-RAID driver patches. We need
> separate patchsets to address limitations of async_tx framework.
>
> Regards,
> Anup
--
~Vinod
^ permalink raw reply
* Re: [bug report] crypto: atmel-sha - update request queue management to make it more generic
From: Cyrille Pitchen @ 2017-02-07 17:31 UTC (permalink / raw)
To: Dan Carpenter; +Cc: linux-crypto
In-Reply-To: <20170207105614.GA7777@mwanda>
Hi Dan,
Le 07/02/2017 à 11:56, Dan Carpenter a écrit :
> Hello Cyrille Pitchen,
>
> The patch a29af939b24d: "crypto: atmel-sha - update request queue
> management to make it more generic" from Jan 26, 2017, leads to the
> following static checker warning:
>
> drivers/crypto/atmel-sha.c:673 atmel_sha_xmit_dma()
> error: we previously assumed 'in_desc' could be null (see line 670)
>
> drivers/crypto/atmel-sha.c
> 652
> 653 dmaengine_slave_config(dd->dma_lch_in.chan, &dd->dma_lch_in.dma_conf);
> 654
> 655 if (length2) {
> 656 sg_init_table(sg, 2);
> 657 sg_dma_address(&sg[0]) = dma_addr1;
> 658 sg_dma_len(&sg[0]) = length1;
> 659 sg_dma_address(&sg[1]) = dma_addr2;
> 660 sg_dma_len(&sg[1]) = length2;
> 661 in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, sg, 2,
> 662 DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
> 663 } else {
> 664 sg_init_table(sg, 1);
> 665 sg_dma_address(&sg[0]) = dma_addr1;
> 666 sg_dma_len(&sg[0]) = length1;
> 667 in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, sg, 1,
> 668 DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
> 669 }
> 670 if (!in_desc)
> 671 atmel_sha_complete(dd, -EINVAL);
>
> Did you mean return atmel_sha_complete(dd, -EINVAL);??? That patch
> change a bunch of returns to just call atmel_sha_complete(). Someone
> should probably review it again to make sure there aren't other bugs
> as well.
Indeed, you're totally right: the "return -EINVAL" lines updated by the
patch should have been replaced by "return atmel_sha_complete(dd, -EINVAL);".
Sorry for that :(
and thanks for having caught it :)
I will prepare a fix for it.
Best regards,
Cyrille
>
> 672
> 673 in_desc->callback = atmel_sha_dma_callback;
> ^^^^^^^^^^^^^^^^^
> NULL dereference.
>
> 674 in_desc->callback_param = dd;
> 675
> 676 atmel_sha_write_ctrl(dd, 1);
> 677
>
> regards,
> dan carpenter
>
^ permalink raw reply
* Re: [PATCH v2 2/5] async_tx: Handle DMA devices having support for fewer PQ coefficients
From: Dan Williams @ 2017-02-07 18:16 UTC (permalink / raw)
To: Anup Patel
Cc: Vinod Koul, Rob Herring, Mark Rutland, Herbert Xu,
David S . Miller, Jassi Brar, Ray Jui, Scott Branden, Jon Mason,
Rob Rice, BCM Kernel Feedback, dmaengine@vger.kernel.org,
Device Tree, linux-arm-kernel@lists.infradead.org,
linux-kernel@vger.kernel.org, linux-crypto, linux-raid
In-Reply-To: <CAALAos_yhXufdt79KWY0eugjF5oyJ8+NcPh2E-dEXnSQu2WhhA@mail.gmail.com>
On Tue, Feb 7, 2017 at 1:02 AM, Anup Patel <anup.patel@broadcom.com> wrote:
> On Tue, Feb 7, 2017 at 1:57 PM, Dan Williams <dan.j.williams@intel.com> wrote:
>> On Tue, Feb 7, 2017 at 12:16 AM, Anup Patel <anup.patel@broadcom.com> wrote:
>>> The DMAENGINE framework assumes that if PQ offload is supported by a
>>> DMA device then all 256 PQ coefficients are supported. This assumption
>>> does not hold anymore because we now have BCM-SBA-RAID offload engine
>>> which supports PQ offload with limited number of PQ coefficients.
>>>
>>> This patch extends async_tx APIs to handle DMA devices with support
>>> for fewer PQ coefficients.
>>>
>>> Signed-off-by: Anup Patel <anup.patel@broadcom.com>
>>> Reviewed-by: Scott Branden <scott.branden@broadcom.com>
>>
>> I don't like this approach. Define an interface for md to query the
>> offload engine once at the beginning of time. We should not be adding
>> any new extensions to async_tx.
>
> Even if we do capability checks in Linux MD, we still need a way
> for DMAENGINE drivers to advertise number of PQ coefficients
> handled by the HW.
>
> I agree capability checks should be done once in Linux MD but I don't
> see why this has to be part of BCM-SBA-RAID driver patches. We need
> separate patchsets to address limitations of async_tx framework.
Right, separate enabling before we pile on new hardware support to a
known broken framework.
^ permalink raw reply
* [PATCH] Revert "hwrng: core - zeroize buffers with random data"
From: David Daney @ 2017-02-08 0:23 UTC (permalink / raw)
To: linux-crypto, Matt Mackall, Herbert Xu, Stephan Mueller,
Linus Torvalds
Cc: linux-kernel, David Daney
This reverts commit 2cc751545854d7bd7eedf4d7e377bb52e176cd07.
With this commit in place I get on a Cavium ThunderX (arm64) system:
$ if=/dev/hwrng bs=256 count=1 | od -t x1 -A x -v > rng-bad.txt
1+0 records in
1+0 records out
256 bytes (256 B) copied, 9.1171e-05 s, 2.8 MB/s
$ dd if=/dev/hwrng bs=256 count=1 | od -t x1 -A x -v >> rng-bad.txt
1+0 records in
1+0 records out
256 bytes (256 B) copied, 9.6141e-05 s, 2.7 MB/s
$ cat rng-bad.txt
000000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
000020 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
000040 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
000050 00 00 00 00 37 20 46 ae d0 fc 1c 55 25 6e b0 b8
000060 7c 7e d7 d4 00 0f 6f b2 91 1e 30 a8 fa 3e 52 0e
000070 06 2d 53 30 be a1 20 0f aa 56 6e 0e 44 6e f4 35
000080 b7 6a fe d2 52 70 7e 58 56 02 41 ea d1 9c 6a 6a
000090 d1 bd d8 4c da 35 45 ef 89 55 fc 59 d5 cd 57 ba
0000a0 4e 3e 02 1c 12 76 43 37 23 e1 9f 7a 9f 9e 99 24
0000b0 47 b2 de e3 79 85 f6 55 7e ad 76 13 4f a0 b5 41
0000c0 c6 92 42 01 d9 12 de 8f b4 7b 6e ae d7 24 fc 65
0000d0 4d af 0a aa 36 d9 17 8d 0e 8b 7a 3b b6 5f 96 47
0000e0 46 f7 d8 ce 0b e8 3e c6 13 a6 2c b6 d6 cc 17 26
0000f0 e3 c3 17 8e 9e 45 56 1e 41 ef 29 1a a8 65 c8 3a
000100
000000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
000020 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
000040 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
000050 00 00 00 00 f4 90 65 aa 8b f2 5e 31 01 53 b4 d4
000060 06 c0 23 a2 99 3d 01 e4 b0 c1 b1 55 0f 80 63 cf
000070 33 24 d8 3a 1d 5e cd 2c ba c0 d0 18 6f bc 97 46
000080 1e 19 51 b1 90 15 af 80 5e d1 08 0d eb b0 6c ab
000090 6a b4 fe 62 37 c5 e1 ee 93 c3 58 78 91 2a d5 23
0000a0 63 50 eb 1f 3b 84 35 18 cf b2 a4 b8 46 69 9e cf
0000b0 0c 95 af 03 51 45 a8 42 f1 64 c9 55 fc 69 76 63
0000c0 98 9d 82 fa 76 85 24 da 80 07 29 fe 4e 76 0c 61
0000d0 ff 23 94 4f c8 5c ce 0b 50 e8 31 bc 9d ce f4 ca
0000e0 be ca 28 da e6 fa cc 64 1c ec a8 41 db fe 42 bd
0000f0 a0 e2 4b 32 b4 52 ba 03 70 8e c1 8e d0 50 3a c6
000100
To my untrained mental entropy detector, the first several bytes of
each read from /dev/hwrng seem to not be very random (i.e. all zero).
When I revert the patch (apply this patch), I get back to what we have
in v4.9, which looks like (much more random appearing):
$ dd if=/dev/hwrng bs=256 count=1 | od -t x1 -A x -v > rng-good.txt
1+0 records in
1+0 records out
256 bytes (256 B) copied, 0.000252233 s, 1.0 MB/s
$ dd if=/dev/hwrng bs=256 count=1 | od -t x1 -A x -v >> rng-good.txt
1+0 records in
1+0 records out
256 bytes (256 B) copied, 0.000113571 s, 2.3 MB/s
$ cat rng-good.txt
000000 75 d1 2d 19 68 1f d2 26 a1 49 22 61 66 e8 09 e5
000010 e0 4e 10 d0 1a 2c 45 5d 59 04 79 8e e2 b7 2c 2e
000020 e8 ad da 34 d5 56 51 3d 58 29 c7 7a 8e ed 22 67
000030 f9 25 b9 fb c6 b7 9c 35 1f 84 21 35 c1 1d 48 34
000040 45 7c f6 f1 57 63 1a 88 38 e8 81 f0 a9 63 ad 0e
000050 be 5d 3e 74 2e 4e cb 36 c2 01 a8 14 e1 38 e1 bb
000060 23 79 09 56 77 19 ff 98 e8 44 f3 27 eb 6e 0a cb
000070 c9 36 e3 2a 96 13 07 a0 90 3f 3b bd 1d 04 1d 67
000080 be 33 14 f8 02 c2 a4 02 ab 8b 5b 74 86 17 f0 5e
000090 a1 d7 aa ef a6 21 7b 93 d1 85 86 eb 4e 8c d0 4c
0000a0 56 ac e4 45 27 44 84 9f 71 db 36 b9 f7 47 d7 b3
0000b0 f2 9c 62 41 a3 46 2b 5b e3 80 63 a4 35 b5 3c f4
0000c0 bc 1e 3a ad e4 59 4a 98 6c e8 8d ff 1b 16 f8 52
0000d0 05 5c 2f 52 2a 0f 45 5b 51 fb 93 97 a4 49 4f 06
0000e0 f3 a0 d1 1e ba 3d ed a7 60 8f bb 84 2c 21 94 2d
0000f0 b3 66 a6 61 1e 58 30 24 85 f8 c8 18 c3 77 00 22
000100
000000 73 ca cc a1 d9 bb 21 8d c3 5c f3 ab 43 6d a7 a4
000010 4a fd c5 f4 9c ba 4a 0f b1 2e 19 15 4e 84 26 e0
000020 67 c9 f2 52 4d 65 1f 81 b7 8b 6d 2b 56 7b 99 75
000030 2e cd d0 db 08 0c 4b df f3 83 c6 83 00 2e 2b b8
000040 0f af 61 1d f2 02 35 74 b5 a4 6f 28 f3 a1 09 12
000050 f2 53 b5 d2 da 45 01 e5 12 d6 46 f8 0b db ed 51
000060 7b f4 0d 54 e0 63 ea 22 e2 1d d0 d6 d0 e7 7e e0
000070 93 91 fb 87 95 43 41 28 de 3d 8b a3 a8 8f c4 9e
000080 30 95 12 7a b2 27 28 ff 37 04 2e 09 7c dd 7c 12
000090 e1 50 60 fb 6d 5f a8 65 14 40 89 e3 4c d2 87 8f
0000a0 34 76 7e 66 7a 8e 6b a3 fc cf 38 52 2e f9 26 f0
0000b0 98 63 15 06 34 99 b2 88 4f aa d8 14 88 71 f1 81
0000c0 be 51 11 2b f4 7e a0 1e 12 b2 44 2e f6 8d 84 ea
0000d0 63 82 2b 66 b3 9a fd 08 73 5a c2 cc ab 5a af b1
0000e0 88 e3 a6 80 4b fc db ed 71 e0 ae c0 0a a4 8c 35
0000f0 eb 89 f9 8a 4b 52 59 6f 09 7c 01 3f 56 e7 c7 bf
000100
Signed-off-by: David Daney <david.daney@cavium.com>
---
drivers/char/hw_random/core.c | 3 ---
1 file changed, 3 deletions(-)
diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c
index 6ce5ce8..87fba42 100644
--- a/drivers/char/hw_random/core.c
+++ b/drivers/char/hw_random/core.c
@@ -92,7 +92,6 @@ static void add_early_randomness(struct hwrng *rng)
mutex_unlock(&reading_mutex);
if (bytes_read > 0)
add_device_randomness(rng_buffer, bytes_read);
- memset(rng_buffer, 0, size);
}
static inline void cleanup_rng(struct kref *kref)
@@ -288,7 +287,6 @@ static ssize_t rng_dev_read(struct file *filp, char __user *buf,
}
}
out:
- memset(rng_buffer, 0, rng_buffer_size());
return ret ? : err;
out_unlock_reading:
@@ -427,7 +425,6 @@ static int hwrng_fillfn(void *unused)
/* Outside lock, sure, but y'know: randomness. */
add_hwgenerator_randomness((void *)rng_fillbuf, rc,
rc * current_quality * 8 >> 10);
- memset(rng_fillbuf, 0, rng_buffer_size());
}
hwrng_fill = NULL;
return 0;
--
1.8.3.1
^ permalink raw reply related
* Re: [PATCH v2 2/5] async_tx: Handle DMA devices having support for fewer PQ coefficients
From: Anup Patel @ 2017-02-08 6:37 UTC (permalink / raw)
To: Vinod Koul
Cc: Dan Williams, Rob Herring, Mark Rutland, Herbert Xu,
David S . Miller, Jassi Brar, Ray Jui, Scott Branden, Jon Mason,
Rob Rice, BCM Kernel Feedback, dmaengine@vger.kernel.org,
Device Tree, linux-arm-kernel@lists.infradead.org,
linux-kernel@vger.kernel.org, linux-crypto, linux-raid
In-Reply-To: <20170207164207.GI19244@localhost>
On Tue, Feb 7, 2017 at 10:12 PM, Vinod Koul <vinod.koul@intel.com> wrote:
> On Tue, Feb 07, 2017 at 02:32:15PM +0530, Anup Patel wrote:
>> On Tue, Feb 7, 2017 at 1:57 PM, Dan Williams <dan.j.williams@intel.com> wrote:
>> > On Tue, Feb 7, 2017 at 12:16 AM, Anup Patel <anup.patel@broadcom.com> wrote:
>> >> The DMAENGINE framework assumes that if PQ offload is supported by a
>> >> DMA device then all 256 PQ coefficients are supported. This assumption
>> >> does not hold anymore because we now have BCM-SBA-RAID offload engine
>> >> which supports PQ offload with limited number of PQ coefficients.
>> >>
>> >> This patch extends async_tx APIs to handle DMA devices with support
>> >> for fewer PQ coefficients.
>> >>
>> >> Signed-off-by: Anup Patel <anup.patel@broadcom.com>
>> >> Reviewed-by: Scott Branden <scott.branden@broadcom.com>
>> >
>> > I don't like this approach. Define an interface for md to query the
>> > offload engine once at the beginning of time. We should not be adding
>> > any new extensions to async_tx.
>>
>> Even if we do capability checks in Linux MD, we still need a way
>> for DMAENGINE drivers to advertise number of PQ coefficients
>> handled by the HW.
>
> If the question is only for advertising caps, then why not do as done
> for dma_get_slave_caps(). you can add dma_get_pq_caps() so that clients (md)
> in this case would know the HW capability.
We have large number of possible capabilities for
DMA slave such as src_addr_widths, dst_addr_widths,
directions, max_burst, residue_granularity, and
descriptor_resue.
The possible capabilities of PQ offload are:
1. Number of PQ sources handled by PQ offload
(Represented by "max_pq" member of "struct dma_device")
2. Number of PQ coefficients handled by PQ offload
The above two PQ capabilities are good enough for
current PQ HW and future PQ HW so we just need a
way to specify number of PQ coefficients.
Till now all of the PQ HW always supported all 256
PQ coefficients so we never felt the need of capability
to specify PQ coefficients. The BCM-SBA-RAID is the
only HW (as far as I know) which does not support all
256 PQ coefficients.
Currently, DMAENGINE drivers use dma_set_maxpq() to
specify number of PQ sources handled by PQ HW and
Linux Async Tx uses dma_maxpq() to get number of
PQ sources.
On similar lines, we added dma_set_maxpqcoef() to
specify number of PQ coefficients and Linux Async Tx
uses dma_maxpqcoef() to get number of PQ coefficients.
If DMAENGINE driver does not specify PQ coefficients
then dma_maxpqcoef() will return 256 assuming all
PQ coefficients are supported. This approach is
backward compatible to existing DMAENGINE APIs
and will not break existing DMAENGINE drivers.
If we add dma_get_pq_caps() similar to the
dma_get_slave_caps() for PQ capabilities then we
will have to use this new method for both of the above
PQ capabilities and we have to change all DMAENGINE
drivers to use new method of specifying PQ capabilities.
I think this is too intrusive and bit overkill because its
very very unlikely to see anymore additions to
PQ capabilities.
Regards,
Anup
^ permalink raw reply
* Re: [RFC PATCH v4] IV Generation algorithms for dm-crypt
From: Gilad Ben-Yossef @ 2017-02-08 7:32 UTC (permalink / raw)
To: Binoy Jayan
Cc: Oded, Ofir, Herbert Xu, David S. Miller, linux-crypto, Mark Brown,
Arnd Bergmann, Linux kernel mailing list, Alasdair Kergon,
Mike Snitzer, dm-devel, Shaohua Li, linux-raid, Rajendra,
Milan Broz
In-Reply-To: <1486463731-6224-1-git-send-email-binoy.jayan@linaro.org>
On Tue, Feb 7, 2017 at 12:35 PM, Binoy Jayan <binoy.jayan@linaro.org> wrote:
> ===============================================================================
> dm-crypt optimization for larger block sizes
> ===============================================================================
>
> Currently, the iv generation algorithms are implemented in dm-crypt.c. The goal
> is to move these algorithms from the dm layer to the kernel crypto layer by
> implementing them as template ciphers so they can be used in relation with
> algorithms like aes, and with multiple modes like cbc, ecb etc. As part of this
> patchset, the iv-generation code is moved from the dm layer to the crypto layer
> and adapt the dm-layer to send a whole 'bio' (as defined in the block layer)
> at a time. Each bio contains the in memory representation of physically
> contiguous disk blocks. Since the bio itself may not be contiguous in main
> memory, the dm layer sets up a chained scatterlist of these blocks split into
> physically contiguous segments in memory so that DMA can be performed.
...
> Binoy Jayan (1):
> crypto: Add IV generation algorithms
>
> drivers/md/dm-crypt.c | 1894 ++++++++++++++++++++++++++++++++++--------------
> include/crypto/geniv.h | 47 ++
> 2 files changed, 1402 insertions(+), 539 deletions(-)
> create mode 100644 include/crypto/geniv.h
Ran Bonnie++ on it last night (Luks mode, plain64, Qemu Virt platform
Arm64) and it works just fine.
Tested-by: Gilad Ben-Yossef <gilad@benyossef.com>
Gilad
--
Gilad Ben-Yossef
Chief Coffee Drinker
"If you take a class in large-scale robotics, can you end up in a
situation where the homework eats your dog?"
-- Jean-Baptiste Queru
^ permalink raw reply
* Re: [PATCH v2 2/5] async_tx: Handle DMA devices having support for fewer PQ coefficients
From: Anup Patel @ 2017-02-08 8:57 UTC (permalink / raw)
To: Dan Williams
Cc: Vinod Koul, Rob Herring, Mark Rutland, Herbert Xu,
David S . Miller, Jassi Brar, Ray Jui, Scott Branden, Jon Mason,
Rob Rice, BCM Kernel Feedback, dmaengine@vger.kernel.org,
Device Tree, linux-arm-kernel@lists.infradead.org,
linux-kernel@vger.kernel.org, linux-crypto, linux-raid
In-Reply-To: <CAPcyv4hK_1f8bryq6smPQy7vTnP+QEzmT9wnAEE1xpxq7EAnjQ@mail.gmail.com>
On Tue, Feb 7, 2017 at 11:46 PM, Dan Williams <dan.j.williams@intel.com> wrote:
> On Tue, Feb 7, 2017 at 1:02 AM, Anup Patel <anup.patel@broadcom.com> wrote:
>> On Tue, Feb 7, 2017 at 1:57 PM, Dan Williams <dan.j.williams@intel.com> wrote:
>>> On Tue, Feb 7, 2017 at 12:16 AM, Anup Patel <anup.patel@broadcom.com> wrote:
>>>> The DMAENGINE framework assumes that if PQ offload is supported by a
>>>> DMA device then all 256 PQ coefficients are supported. This assumption
>>>> does not hold anymore because we now have BCM-SBA-RAID offload engine
>>>> which supports PQ offload with limited number of PQ coefficients.
>>>>
>>>> This patch extends async_tx APIs to handle DMA devices with support
>>>> for fewer PQ coefficients.
>>>>
>>>> Signed-off-by: Anup Patel <anup.patel@broadcom.com>
>>>> Reviewed-by: Scott Branden <scott.branden@broadcom.com>
>>>
>>> I don't like this approach. Define an interface for md to query the
>>> offload engine once at the beginning of time. We should not be adding
>>> any new extensions to async_tx.
>>
>> Even if we do capability checks in Linux MD, we still need a way
>> for DMAENGINE drivers to advertise number of PQ coefficients
>> handled by the HW.
>>
>> I agree capability checks should be done once in Linux MD but I don't
>> see why this has to be part of BCM-SBA-RAID driver patches. We need
>> separate patchsets to address limitations of async_tx framework.
>
> Right, separate enabling before we pile on new hardware support to a
> known broken framework.
Linux Async Tx not broken framework. The issue is:
1. Its not complete enough
2. Its not optimized for very high through-put offload engines
Regards,
Anup
^ permalink raw reply
* [PATCH] crypto: arm/aes-ce: assign err return conditionally
From: Nicholas Mc Guire @ 2017-02-08 13:36 UTC (permalink / raw)
To: Herbert Xu
Cc: linux-kernel, Russell King, Nicholas Mc Guire, linux-crypto,
David S. Miller, linux-arm-kernel
As the err value is not used unless there was an error it can be assigned
conditionally here.
Signed-off-by: Nicholas Mc Guire <der.herr@hofr.at>
---
Not sure if this is really relevant and worth changing, effectively it
is practically no change as gcc would move the err = PTR_ERR(simd);
below unregister_simds: anyway (based on inspection of .lst/.s files)
- so it is more of an adjust C-level to object level for readability.
Patch was compile-tested with multi_v7_defconfig
(implies CONFIG_CRYPTO_AES_ARM_CE=m)
Patch is against 4.10-rc7 (localversion-next is next-20170208)
arch/arm/crypto/aes-ce-glue.c | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/arch/arm/crypto/aes-ce-glue.c b/arch/arm/crypto/aes-ce-glue.c
index 883b84d..8f65030 100644
--- a/arch/arm/crypto/aes-ce-glue.c
+++ b/arch/arm/crypto/aes-ce-glue.c
@@ -437,9 +437,10 @@ static int __init aes_init(void)
drvname = aes_algs[i].base.cra_driver_name + 2;
basename = aes_algs[i].base.cra_driver_name;
simd = simd_skcipher_create_compat(algname, drvname, basename);
- err = PTR_ERR(simd);
- if (IS_ERR(simd))
+ if (IS_ERR(simd)) {
+ err = PTR_ERR(simd);
goto unregister_simds;
+ }
aes_simd_algs[i] = simd;
}
--
2.1.4
^ permalink raw reply related
* Re: [PATCH v2 2/5] async_tx: Handle DMA devices having support for fewer PQ coefficients
From: Dan Williams @ 2017-02-08 16:24 UTC (permalink / raw)
To: Anup Patel
Cc: Vinod Koul, Rob Herring, Mark Rutland, Herbert Xu,
David S . Miller, Jassi Brar, Ray Jui, Scott Branden, Jon Mason,
Rob Rice, BCM Kernel Feedback, dmaengine@vger.kernel.org,
Device Tree, linux-arm-kernel@lists.infradead.org,
linux-kernel@vger.kernel.org, linux-crypto, linux-raid
In-Reply-To: <CAALAos9ovab4x=waRgM7G2adRC7syHMkY0Rhf-UPB_+h4w3yNQ@mail.gmail.com>
On Wed, Feb 8, 2017 at 12:57 AM, Anup Patel <anup.patel@broadcom.com> wrote:
> On Tue, Feb 7, 2017 at 11:46 PM, Dan Williams <dan.j.williams@intel.com> wrote:
>> On Tue, Feb 7, 2017 at 1:02 AM, Anup Patel <anup.patel@broadcom.com> wrote:
>>> On Tue, Feb 7, 2017 at 1:57 PM, Dan Williams <dan.j.williams@intel.com> wrote:
>>>> On Tue, Feb 7, 2017 at 12:16 AM, Anup Patel <anup.patel@broadcom.com> wrote:
>>>>> The DMAENGINE framework assumes that if PQ offload is supported by a
>>>>> DMA device then all 256 PQ coefficients are supported. This assumption
>>>>> does not hold anymore because we now have BCM-SBA-RAID offload engine
>>>>> which supports PQ offload with limited number of PQ coefficients.
>>>>>
>>>>> This patch extends async_tx APIs to handle DMA devices with support
>>>>> for fewer PQ coefficients.
>>>>>
>>>>> Signed-off-by: Anup Patel <anup.patel@broadcom.com>
>>>>> Reviewed-by: Scott Branden <scott.branden@broadcom.com>
>>>>
>>>> I don't like this approach. Define an interface for md to query the
>>>> offload engine once at the beginning of time. We should not be adding
>>>> any new extensions to async_tx.
>>>
>>> Even if we do capability checks in Linux MD, we still need a way
>>> for DMAENGINE drivers to advertise number of PQ coefficients
>>> handled by the HW.
>>>
>>> I agree capability checks should be done once in Linux MD but I don't
>>> see why this has to be part of BCM-SBA-RAID driver patches. We need
>>> separate patchsets to address limitations of async_tx framework.
>>
>> Right, separate enabling before we pile on new hardware support to a
>> known broken framework.
>
> Linux Async Tx not broken framework. The issue is:
> 1. Its not complete enough
> 2. Its not optimized for very high through-put offload engines
I'm not understanding your point. I'm nak'ing this change to add yet
more per-transaction capability checking to async_tx. I don't like the
DMA_HAS_FEWER_PQ_COEF flag, especially since it is equal to
DMA_HAS_PQ_CONTINUE. I'm not asking for all of async_tx's problems to
be fixed before this new hardware support, I'm simply saying we should
start the process of moving offload-engine capability checking to the
raid code.
^ permalink raw reply
* [PATCH] crypto: ccp - Set the AES size field for all modes
From: Gary R Hook @ 2017-02-08 19:07 UTC (permalink / raw)
To: linux-crypto; +Cc: thomas.lendacky, herbert, davem
Ensure that the size field is correctly populated for
all AES modes.
Signed-off-by: Gary R Hook <gary.hook@amd.com>
---
drivers/crypto/ccp/ccp-dev-v5.c | 3 +--
drivers/crypto/ccp/ccp-dev.h | 1 +
drivers/crypto/ccp/ccp-ops.c | 8 ++++++++
3 files changed, 10 insertions(+), 2 deletions(-)
diff --git a/drivers/crypto/ccp/ccp-dev-v5.c b/drivers/crypto/ccp/ccp-dev-v5.c
index 612898b..9c6ff8b8 100644
--- a/drivers/crypto/ccp/ccp-dev-v5.c
+++ b/drivers/crypto/ccp/ccp-dev-v5.c
@@ -284,8 +284,7 @@ static int ccp5_perform_aes(struct ccp_op *op)
CCP_AES_ENCRYPT(&function) = op->u.aes.action;
CCP_AES_MODE(&function) = op->u.aes.mode;
CCP_AES_TYPE(&function) = op->u.aes.type;
- if (op->u.aes.mode == CCP_AES_MODE_CFB)
- CCP_AES_SIZE(&function) = 0x7f;
+ CCP_AES_SIZE(&function) = op->u.aes.size;
CCP5_CMD_FUNCTION(&desc) = function.raw;
diff --git a/drivers/crypto/ccp/ccp-dev.h b/drivers/crypto/ccp/ccp-dev.h
index 649e561..2b5c01f 100644
--- a/drivers/crypto/ccp/ccp-dev.h
+++ b/drivers/crypto/ccp/ccp-dev.h
@@ -467,6 +467,7 @@ struct ccp_aes_op {
enum ccp_aes_type type;
enum ccp_aes_mode mode;
enum ccp_aes_action action;
+ unsigned int size;
};
struct ccp_xts_aes_op {
diff --git a/drivers/crypto/ccp/ccp-ops.c b/drivers/crypto/ccp/ccp-ops.c
index 50fae44..6878160 100644
--- a/drivers/crypto/ccp/ccp-ops.c
+++ b/drivers/crypto/ccp/ccp-ops.c
@@ -692,6 +692,14 @@ static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
goto e_ctx;
}
}
+ switch (aes->mode) {
+ case CCP_AES_MODE_CFB: /* CFB128 only */
+ case CCP_AES_MODE_CTR:
+ op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1;
+ break;
+ default:
+ op.u.aes.size = 0;
+ }
/* Prepare the input and output data workareas. For in-place
* operations we need to set the dma direction to BIDIRECTIONAL
^ permalink raw reply related
* [PATCH v2 0/2] crypto: AF_ALG memory management fix
From: Stephan Müller @ 2017-02-08 20:14 UTC (permalink / raw)
To: herbert; +Cc: linux-crypto
Hi Herbert,
Changes v2:
* import fix from Harsh Jain <harsh@chelsio.com> to remove SG
from list before freeing
* fix return code used for ki_complete to match AIO behavior
with sync behavior
* rename variable list -> tsgl_list
* update the algif_aead patch to include a dynamic TX SGL
allocation similar to what algif_skcipher does. This allows
concurrent continuous read/write operations to the extent
you requested. Although I have not implemented "pairs of
TX/RX SGLs" as I think that is even more overhead, the
implementation conceptually defines such pairs. The recvmsg
call defines how much from the input data is processed.
The caller can have arbitrary number of sendmsg calls
where the data is added to the TX SGL before an recvmsg
asks the kernel to process a given amount (or all) of the
TX SGL.
With the changes, you will see a lot of code duplication now
as I deliberately tried to use the same struct and variable names,
the same function names and even the same oder of functions.
If you agree to this patch, I volunteer to provide a followup
patch that will extract the code duplication into common
functions.
Please find attached memory management updates to
- simplify the code: the old AIO memory management is very
complex and seemingly very fragile -- the update now
eliminates all reported bugs in the skcipher and AEAD
interfaces which allowed the kernel to be crashed by
an unprivileged user
- streamline the code: there is one code path for AIO and sync
operation; the code between algif_skcipher and algif_aead
is very similar (if that patch set is accepted, I volunteer
to reduce code duplication by moving service operations
into af_alg.c and to further unify the TX SGL handling)
- unify the AIO and sync operation which only differ in the
kernel crypto API callback and whether to wait for the
crypto operation or not
- fix all reported bugs regarding the handling of multiple
IOCBs.
The following testing was performed:
- stress testing to verify that no memleaks exist
- testing using Tadeusz Struck AIO test tool (see
https://github.com/tstruk/afalg_async_test) -- the AEAD test
is not applicable any more due to the changed user space
interface; the skcipher test works once the user space
interface change is honored in the test code
- using the libkcapi test suite, all tests including the
originally failing ones (AIO with multiple IOCBs) work now --
the current libkcapi code artificially limits the AEAD
operation to one IOCB. After altering the libkcapi code
to allow multiple IOCBs, the testing works flawless.
Stephan Mueller (2):
crypto: skcipher AF_ALG - overhaul memory management
crypto: aead AF_ALG - overhaul memory management
crypto/algif_aead.c | 668 +++++++++++++++++++++++++-----------------------
crypto/algif_skcipher.c | 472 ++++++++++++++--------------------
2 files changed, 544 insertions(+), 596 deletions(-)
--
2.9.3
^ permalink raw reply
* [PATCH v2 1/2] crypto: skcipher AF_ALG - overhaul memory management
From: Stephan Müller @ 2017-02-08 20:15 UTC (permalink / raw)
To: herbert; +Cc: linux-crypto
In-Reply-To: <2626546.tAxUg930Nc@positron.chronox.de>
The updated memory management is described in the top part of the code.
As one benefit of the changed memory management, the AIO and synchronous
operation is now implemented in one common function. The AF_ALG
operation uses the async kernel crypto API interface for each cipher
operation. Thus, the only difference between the AIO and sync operation
types visible from user space is:
1. the callback function to be invoked when the asynchronous operation
is completed
2. whether to wait for the completion of the kernel crypto API operation
or not
In addition, the code structure is adjusted to match the structure of
algif_aead for easier code assessment.
The user space interface changed slightly as follows: the old AIO
operation returned zero upon success and < 0 in case of an error to user
space. As all other AF_ALG interfaces (including the sync skcipher
interface) returned the number of processed bytes upon success and < 0
in case of an error, the new skcipher interface (regardless of AIO or
sync) returns the number of processed bytes in case of success.
Signed-off-by: Stephan Mueller <smueller@chronox.de>
---
crypto/algif_skcipher.c | 472 ++++++++++++++++++++----------------------------
1 file changed, 193 insertions(+), 279 deletions(-)
diff --git a/crypto/algif_skcipher.c b/crypto/algif_skcipher.c
index a9e79d8..e65ebc4 100644
--- a/crypto/algif_skcipher.c
+++ b/crypto/algif_skcipher.c
@@ -10,6 +10,25 @@
* Software Foundation; either version 2 of the License, or (at your option)
* any later version.
*
+ * The following concept of the memory management is used:
+ *
+ * The kernel maintains two SGLs, the TX SGL and the RX SGL. The TX SGL is
+ * filled by user space with the data submitted via sendpage/sendmsg. Filling
+ * up the TX SGL does not cause a crypto operation -- the data will only be
+ * tracked by the kernel. Upon receipt of one recvmsg call, the caller must
+ * provide a buffer which is tracked with the RX SGL.
+ *
+ * During the processing of the recvmsg operation, the cipher request is
+ * allocated and prepared. To support multiple recvmsg operations operating
+ * on one TX SGL, an offset pointer into the TX SGL is maintained. The TX SGL
+ * that is used for the crypto request is scatterwalk_ffwd by the offset
+ * pointer to obtain the start address the crypto operation shall use for
+ * the input data.
+ *
+ * After the completion of the crypto operation, the RX SGL and the cipher
+ * request is released. The processed TX SGL parts are released together with
+ * the RX SGL release and the offset pointer is reduced by the released
+ * data.
*/
#include <crypto/scatterwalk.h>
@@ -31,78 +50,50 @@ struct skcipher_sg_list {
struct scatterlist sg[0];
};
+struct skcipher_rsgl {
+ struct af_alg_sgl sgl;
+ struct list_head list;
+};
+
+struct skcipher_async_req {
+ struct kiocb *iocb;
+ struct sock *sk;
+
+ struct skcipher_rsgl first_sgl;
+ struct list_head rsgl_list;
+
+ unsigned int areqlen;
+ struct skcipher_request req;
+};
+
struct skcipher_tfm {
struct crypto_skcipher *skcipher;
bool has_key;
};
struct skcipher_ctx {
- struct list_head tsgl;
- struct af_alg_sgl rsgl;
+ struct list_head tsgl_list;
void *iv;
struct af_alg_completion completion;
- atomic_t inflight;
+ unsigned int inflight;
size_t used;
+ size_t processed;
- unsigned int len;
bool more;
bool merge;
bool enc;
- struct skcipher_request req;
-};
-
-struct skcipher_async_rsgl {
- struct af_alg_sgl sgl;
- struct list_head list;
+ unsigned int len;
};
-struct skcipher_async_req {
- struct kiocb *iocb;
- struct skcipher_async_rsgl first_sgl;
- struct list_head list;
- struct scatterlist *tsg;
- atomic_t *inflight;
- struct skcipher_request req;
-};
+static DECLARE_WAIT_QUEUE_HEAD(skcipher_aio_finish_wait);
#define MAX_SGL_ENTS ((4096 - sizeof(struct skcipher_sg_list)) / \
sizeof(struct scatterlist) - 1)
-static void skcipher_free_async_sgls(struct skcipher_async_req *sreq)
-{
- struct skcipher_async_rsgl *rsgl, *tmp;
- struct scatterlist *sgl;
- struct scatterlist *sg;
- int i, n;
-
- list_for_each_entry_safe(rsgl, tmp, &sreq->list, list) {
- af_alg_free_sg(&rsgl->sgl);
- if (rsgl != &sreq->first_sgl)
- kfree(rsgl);
- }
- sgl = sreq->tsg;
- n = sg_nents(sgl);
- for_each_sg(sgl, sg, n, i)
- put_page(sg_page(sg));
-
- kfree(sreq->tsg);
-}
-
-static void skcipher_async_cb(struct crypto_async_request *req, int err)
-{
- struct skcipher_async_req *sreq = req->data;
- struct kiocb *iocb = sreq->iocb;
-
- atomic_dec(sreq->inflight);
- skcipher_free_async_sgls(sreq);
- kzfree(sreq);
- iocb->ki_complete(iocb, err, err);
-}
-
static inline int skcipher_sndbuf(struct sock *sk)
{
struct alg_sock *ask = alg_sk(sk);
@@ -117,15 +108,15 @@ static inline bool skcipher_writable(struct sock *sk)
return PAGE_SIZE <= skcipher_sndbuf(sk);
}
-static int skcipher_alloc_sgl(struct sock *sk)
+static int skcipher_alloc_tsgl(struct sock *sk)
{
struct alg_sock *ask = alg_sk(sk);
struct skcipher_ctx *ctx = ask->private;
struct skcipher_sg_list *sgl;
struct scatterlist *sg = NULL;
- sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list);
- if (!list_empty(&ctx->tsgl))
+ sgl = list_entry(ctx->tsgl_list.prev, struct skcipher_sg_list, list);
+ if (!list_empty(&ctx->tsgl_list))
sg = sgl->sg;
if (!sg || sgl->cur >= MAX_SGL_ENTS) {
@@ -141,13 +132,13 @@ static int skcipher_alloc_sgl(struct sock *sk)
if (sg)
sg_chain(sg, MAX_SGL_ENTS + 1, sgl->sg);
- list_add_tail(&sgl->list, &ctx->tsgl);
+ list_add_tail(&sgl->list, &ctx->tsgl_list);
}
return 0;
}
-static void skcipher_pull_sgl(struct sock *sk, size_t used, int put)
+static void skcipher_pull_tsgl(struct sock *sk, size_t used)
{
struct alg_sock *ask = alg_sk(sk);
struct skcipher_ctx *ctx = ask->private;
@@ -155,8 +146,8 @@ static void skcipher_pull_sgl(struct sock *sk, size_t used, int put)
struct scatterlist *sg;
int i;
- while (!list_empty(&ctx->tsgl)) {
- sgl = list_first_entry(&ctx->tsgl, struct skcipher_sg_list,
+ while (!list_empty(&ctx->tsgl_list)) {
+ sgl = list_first_entry(&ctx->tsgl_list, struct skcipher_sg_list,
list);
sg = sgl->sg;
@@ -171,30 +162,35 @@ static void skcipher_pull_sgl(struct sock *sk, size_t used, int put)
used -= plen;
ctx->used -= plen;
+ ctx->processed -= plen;
if (sg[i].length)
return;
- if (put)
- put_page(sg_page(sg + i));
+
+ put_page(sg_page(sg + i));
sg_assign_page(sg + i, NULL);
}
list_del(&sgl->list);
- sock_kfree_s(sk, sgl,
- sizeof(*sgl) + sizeof(sgl->sg[0]) *
- (MAX_SGL_ENTS + 1));
+ sock_kfree_s(sk, sgl, sizeof(*sgl) + sizeof(sgl->sg[0]) *
+ (MAX_SGL_ENTS + 1));
}
if (!ctx->used)
ctx->merge = 0;
}
-static void skcipher_free_sgl(struct sock *sk)
+static void skcipher_free_rsgl(struct skcipher_async_req *areq)
{
- struct alg_sock *ask = alg_sk(sk);
- struct skcipher_ctx *ctx = ask->private;
+ struct sock *sk = areq->sk;
+ struct skcipher_rsgl *rsgl, *tmp;
- skcipher_pull_sgl(sk, ctx->used, 1);
+ list_for_each_entry_safe(rsgl, tmp, &areq->rsgl_list, list) {
+ af_alg_free_sg(&rsgl->sgl);
+ list_del(&rsgl->list);
+ if (rsgl != &areq->first_sgl)
+ sock_kfree_s(sk, rsgl, sizeof(*rsgl));
+ }
}
static int skcipher_wait_for_wmem(struct sock *sk, unsigned flags)
@@ -348,7 +344,7 @@ static int skcipher_sendmsg(struct socket *sock, struct msghdr *msg,
size_t plen;
if (ctx->merge) {
- sgl = list_entry(ctx->tsgl.prev,
+ sgl = list_entry(ctx->tsgl_list.prev,
struct skcipher_sg_list, list);
sg = sgl->sg + sgl->cur - 1;
len = min_t(unsigned long, len,
@@ -378,11 +374,12 @@ static int skcipher_sendmsg(struct socket *sock, struct msghdr *msg,
len = min_t(unsigned long, len, skcipher_sndbuf(sk));
- err = skcipher_alloc_sgl(sk);
+ err = skcipher_alloc_tsgl(sk);
if (err)
goto unlock;
- sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list);
+ sgl = list_entry(ctx->tsgl_list.prev, struct skcipher_sg_list,
+ list);
sg = sgl->sg;
if (sgl->cur)
sg_unmark_end(sg + sgl->cur - 1);
@@ -453,12 +450,12 @@ static ssize_t skcipher_sendpage(struct socket *sock, struct page *page,
goto unlock;
}
- err = skcipher_alloc_sgl(sk);
+ err = skcipher_alloc_tsgl(sk);
if (err)
goto unlock;
ctx->merge = 0;
- sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list);
+ sgl = list_entry(ctx->tsgl_list.prev, struct skcipher_sg_list, list);
if (sgl->cur)
sg_unmark_end(sgl->sg + sgl->cur - 1);
@@ -479,25 +476,37 @@ static ssize_t skcipher_sendpage(struct socket *sock, struct page *page,
return err ?: size;
}
-static int skcipher_all_sg_nents(struct skcipher_ctx *ctx)
+static void skcipher_async_cb(struct crypto_async_request *req, int err)
{
- struct skcipher_sg_list *sgl;
- struct scatterlist *sg;
- int nents = 0;
+ struct skcipher_async_req *areq = req->data;
+ struct sock *sk = areq->sk;
+ struct alg_sock *ask = alg_sk(sk);
+ struct skcipher_ctx *ctx = ask->private;
+ struct kiocb *iocb = areq->iocb;
+ unsigned int resultlen;
- list_for_each_entry(sgl, &ctx->tsgl, list) {
- sg = sgl->sg;
+ lock_sock(sk);
- while (!sg->length)
- sg++;
+ BUG_ON(!ctx->inflight);
- nents += sg_nents(sg);
- }
- return nents;
+ /* Buffer size written by crypto operation. */
+ resultlen = areq->req.cryptlen;
+
+ skcipher_free_rsgl(areq);
+ skcipher_pull_tsgl(sk, areq->req.cryptlen);
+ sock_kfree_s(sk, areq, areq->areqlen);
+ __sock_put(sk);
+ ctx->inflight--;
+
+ iocb->ki_complete(iocb, err ? err : resultlen, 0);
+
+ release_sock(sk);
+
+ wake_up_interruptible(&skcipher_aio_finish_wait);
}
-static int skcipher_recvmsg_async(struct socket *sock, struct msghdr *msg,
- int flags)
+static int skcipher_recvmsg(struct socket *sock, struct msghdr *msg,
+ size_t ignored, int flags)
{
struct sock *sk = sock->sk;
struct alg_sock *ask = alg_sk(sk);
@@ -506,215 +515,131 @@ static int skcipher_recvmsg_async(struct socket *sock, struct msghdr *msg,
struct skcipher_ctx *ctx = ask->private;
struct skcipher_tfm *skc = pask->private;
struct crypto_skcipher *tfm = skc->skcipher;
- struct skcipher_sg_list *sgl;
- struct scatterlist *sg;
- struct skcipher_async_req *sreq;
- struct skcipher_request *req;
- struct skcipher_async_rsgl *last_rsgl = NULL;
- unsigned int txbufs = 0, len = 0, tx_nents;
- unsigned int reqsize = crypto_skcipher_reqsize(tfm);
- unsigned int ivsize = crypto_skcipher_ivsize(tfm);
+ unsigned int bs = crypto_skcipher_blocksize(tfm);
+ unsigned int areqlen = sizeof(struct skcipher_async_req) +
+ crypto_skcipher_reqsize(tfm);
+ struct skcipher_sg_list *tsgl;
+ struct skcipher_async_req *areq;
+ struct skcipher_rsgl *last_rsgl = NULL;
+ struct scatterlist tsgl_head[2], *tsgl_head_p;
int err = -ENOMEM;
- bool mark = false;
- char *iv;
-
- sreq = kzalloc(sizeof(*sreq) + reqsize + ivsize, GFP_KERNEL);
- if (unlikely(!sreq))
- goto out;
-
- req = &sreq->req;
- iv = (char *)(req + 1) + reqsize;
- sreq->iocb = msg->msg_iocb;
- INIT_LIST_HEAD(&sreq->list);
- sreq->inflight = &ctx->inflight;
+ size_t len = 0;
lock_sock(sk);
- tx_nents = skcipher_all_sg_nents(ctx);
- sreq->tsg = kcalloc(tx_nents, sizeof(*sg), GFP_KERNEL);
- if (unlikely(!sreq->tsg))
+
+ /* Allocate cipher request for current operation. */
+ areq = sock_kmalloc(sk, areqlen, GFP_KERNEL);
+ if (unlikely(!areq))
goto unlock;
- sg_init_table(sreq->tsg, tx_nents);
- memcpy(iv, ctx->iv, ivsize);
- skcipher_request_set_tfm(req, tfm);
- skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP,
- skcipher_async_cb, sreq);
+ areq->areqlen = areqlen;
+ areq->sk = sk;
+ INIT_LIST_HEAD(&areq->rsgl_list);
- while (iov_iter_count(&msg->msg_iter)) {
- struct skcipher_async_rsgl *rsgl;
- int used;
+ /* convert iovecs of output buffers into RX SGL */
+ while (len < ctx->used && iov_iter_count(&msg->msg_iter)) {
+ struct skcipher_rsgl *rsgl;
+ size_t seglen;
if (!ctx->used) {
err = skcipher_wait_for_data(sk, flags);
if (err)
goto free;
}
- sgl = list_first_entry(&ctx->tsgl,
- struct skcipher_sg_list, list);
- sg = sgl->sg;
- while (!sg->length)
- sg++;
-
- used = min_t(unsigned long, ctx->used,
- iov_iter_count(&msg->msg_iter));
- used = min_t(unsigned long, used, sg->length);
-
- if (txbufs == tx_nents) {
- struct scatterlist *tmp;
- int x;
- /* Ran out of tx slots in async request
- * need to expand */
- tmp = kcalloc(tx_nents * 2, sizeof(*tmp),
- GFP_KERNEL);
- if (!tmp) {
- err = -ENOMEM;
- goto free;
- }
+ seglen = min_t(size_t, ctx->used,
+ iov_iter_count(&msg->msg_iter));
- sg_init_table(tmp, tx_nents * 2);
- for (x = 0; x < tx_nents; x++)
- sg_set_page(&tmp[x], sg_page(&sreq->tsg[x]),
- sreq->tsg[x].length,
- sreq->tsg[x].offset);
- kfree(sreq->tsg);
- sreq->tsg = tmp;
- tx_nents *= 2;
- mark = true;
- }
- /* Need to take over the tx sgl from ctx
- * to the asynch req - these sgls will be freed later */
- sg_set_page(sreq->tsg + txbufs++, sg_page(sg), sg->length,
- sg->offset);
-
- if (list_empty(&sreq->list)) {
- rsgl = &sreq->first_sgl;
- list_add_tail(&rsgl->list, &sreq->list);
+ if (list_empty(&areq->rsgl_list)) {
+ rsgl = &areq->first_sgl;
} else {
- rsgl = kmalloc(sizeof(*rsgl), GFP_KERNEL);
+ rsgl = sock_kmalloc(sk, sizeof(*rsgl), GFP_KERNEL);
if (!rsgl) {
err = -ENOMEM;
goto free;
}
- list_add_tail(&rsgl->list, &sreq->list);
}
- used = af_alg_make_sg(&rsgl->sgl, &msg->msg_iter, used);
- err = used;
- if (used < 0)
+ rsgl->sgl.npages = 0;
+ list_add_tail(&rsgl->list, &areq->rsgl_list);
+
+ /* make one iovec available as scatterlist */
+ err = af_alg_make_sg(&rsgl->sgl, &msg->msg_iter, seglen);
+ if (err < 0)
goto free;
+
+ /* chain the new scatterlist with previous one */
if (last_rsgl)
af_alg_link_sg(&last_rsgl->sgl, &rsgl->sgl);
last_rsgl = rsgl;
- len += used;
- skcipher_pull_sgl(sk, used, 0);
- iov_iter_advance(&msg->msg_iter, used);
+ len += err;
+ iov_iter_advance(&msg->msg_iter, err);
}
- if (mark)
- sg_mark_end(sreq->tsg + txbufs - 1);
+ /* Process only as much RX buffers for which we have TX data */
+ if (len > ctx->used)
+ len = ctx->used;
+
+ /*
+ * If more buffers are to be expected to be processed, process only
+ * full block size buffers.
+ */
+ if (ctx->more || len < ctx->used)
+ len -= len % bs;
+
+ tsgl = list_first_entry(&ctx->tsgl_list, struct skcipher_sg_list, list);
+ /* Get the head of the SGL we want to process */
+ tsgl_head_p = scatterwalk_ffwd(tsgl_head, tsgl->sg, ctx->processed);
+ BUG_ON(!tsgl_head_p);
+
+ /* Initialize the crypto operation */
+ skcipher_request_set_tfm(&areq->req, tfm);
+ skcipher_request_set_crypt(&areq->req, tsgl_head_p,
+ areq->first_sgl.sgl.sg, len, ctx->iv);
+
+ if (msg->msg_iocb && !is_sync_kiocb(msg->msg_iocb)) {
+ /* AIO operation */
+ areq->iocb = msg->msg_iocb;
+ skcipher_request_set_callback(&areq->req,
+ CRYPTO_TFM_REQ_MAY_SLEEP,
+ skcipher_async_cb, areq);
+ err = ctx->enc ? crypto_skcipher_encrypt(&areq->req) :
+ crypto_skcipher_decrypt(&areq->req);
+ } else {
+ /* Synchronous operation */
+ skcipher_request_set_callback(&areq->req,
+ CRYPTO_TFM_REQ_MAY_SLEEP |
+ CRYPTO_TFM_REQ_MAY_BACKLOG,
+ af_alg_complete,
+ &ctx->completion);
+ err = af_alg_wait_for_completion(ctx->enc ?
+ crypto_skcipher_encrypt(&areq->req) :
+ crypto_skcipher_decrypt(&areq->req),
+ &ctx->completion);
+ }
- skcipher_request_set_crypt(req, sreq->tsg, sreq->first_sgl.sgl.sg,
- len, iv);
- err = ctx->enc ? crypto_skcipher_encrypt(req) :
- crypto_skcipher_decrypt(req);
+ /* AIO operation in progress */
if (err == -EINPROGRESS) {
- atomic_inc(&ctx->inflight);
+ sock_hold(sk);
err = -EIOCBQUEUED;
- sreq = NULL;
+ ctx->inflight++;
+ /* Remember the TX bytes that were processed. */
+ ctx->processed += len;
goto unlock;
- }
-free:
- skcipher_free_async_sgls(sreq);
-unlock:
- skcipher_wmem_wakeup(sk);
- release_sock(sk);
- kzfree(sreq);
-out:
- return err;
-}
-
-static int skcipher_recvmsg_sync(struct socket *sock, struct msghdr *msg,
- int flags)
-{
- struct sock *sk = sock->sk;
- struct alg_sock *ask = alg_sk(sk);
- struct sock *psk = ask->parent;
- struct alg_sock *pask = alg_sk(psk);
- struct skcipher_ctx *ctx = ask->private;
- struct skcipher_tfm *skc = pask->private;
- struct crypto_skcipher *tfm = skc->skcipher;
- unsigned bs = crypto_skcipher_blocksize(tfm);
- struct skcipher_sg_list *sgl;
- struct scatterlist *sg;
- int err = -EAGAIN;
- int used;
- long copied = 0;
-
- lock_sock(sk);
- while (msg_data_left(msg)) {
- if (!ctx->used) {
- err = skcipher_wait_for_data(sk, flags);
- if (err)
- goto unlock;
- }
-
- used = min_t(unsigned long, ctx->used, msg_data_left(msg));
-
- used = af_alg_make_sg(&ctx->rsgl, &msg->msg_iter, used);
- err = used;
- if (err < 0)
- goto unlock;
-
- if (ctx->more || used < ctx->used)
- used -= used % bs;
-
- err = -EINVAL;
- if (!used)
- goto free;
-
- sgl = list_first_entry(&ctx->tsgl,
- struct skcipher_sg_list, list);
- sg = sgl->sg;
-
- while (!sg->length)
- sg++;
-
- skcipher_request_set_crypt(&ctx->req, sg, ctx->rsgl.sg, used,
- ctx->iv);
-
- err = af_alg_wait_for_completion(
- ctx->enc ?
- crypto_skcipher_encrypt(&ctx->req) :
- crypto_skcipher_decrypt(&ctx->req),
- &ctx->completion);
+ } else if (!err)
+ /* Remember the TX bytes that were processed. */
+ ctx->processed += len;
free:
- af_alg_free_sg(&ctx->rsgl);
-
- if (err)
- goto unlock;
-
- copied += used;
- skcipher_pull_sgl(sk, used, 1);
- iov_iter_advance(&msg->msg_iter, used);
- }
-
- err = 0;
+ skcipher_free_rsgl(areq);
+ if (areq)
+ sock_kfree_s(sk, areq, areqlen);
+ skcipher_pull_tsgl(sk, len);
unlock:
skcipher_wmem_wakeup(sk);
release_sock(sk);
-
- return copied ?: err;
-}
-
-static int skcipher_recvmsg(struct socket *sock, struct msghdr *msg,
- size_t ignored, int flags)
-{
- return (msg->msg_iocb && !is_sync_kiocb(msg->msg_iocb)) ?
- skcipher_recvmsg_async(sock, msg, flags) :
- skcipher_recvmsg_sync(sock, msg, flags);
+ return err ? err : len;
}
static unsigned int skcipher_poll(struct file *file, struct socket *sock,
@@ -723,10 +648,9 @@ static unsigned int skcipher_poll(struct file *file, struct socket *sock,
struct sock *sk = sock->sk;
struct alg_sock *ask = alg_sk(sk);
struct skcipher_ctx *ctx = ask->private;
- unsigned int mask;
+ unsigned int mask = 0;
sock_poll_wait(file, sk_sleep(sk), wait);
- mask = 0;
if (ctx->used)
mask |= POLLIN | POLLRDNORM;
@@ -894,26 +818,20 @@ static int skcipher_setkey(void *private, const u8 *key, unsigned int keylen)
return err;
}
-static void skcipher_wait(struct sock *sk)
-{
- struct alg_sock *ask = alg_sk(sk);
- struct skcipher_ctx *ctx = ask->private;
- int ctr = 0;
-
- while (atomic_read(&ctx->inflight) && ctr++ < 100)
- msleep(100);
-}
-
static void skcipher_sock_destruct(struct sock *sk)
{
struct alg_sock *ask = alg_sk(sk);
struct skcipher_ctx *ctx = ask->private;
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(&ctx->req);
+ struct sock *psk = ask->parent;
+ struct alg_sock *pask = alg_sk(psk);
+ struct skcipher_tfm *skc = pask->private;
+ struct crypto_skcipher *tfm = skc->skcipher;
- if (atomic_read(&ctx->inflight))
- skcipher_wait(sk);
+ /* Suspend caller if AIO operations are in flight. */
+ wait_event_interruptible(skcipher_aio_finish_wait,
+ (ctx->inflight == 0));
- skcipher_free_sgl(sk);
+ skcipher_pull_tsgl(sk, ctx->used);
sock_kzfree_s(sk, ctx->iv, crypto_skcipher_ivsize(tfm));
sock_kfree_s(sk, ctx, ctx->len);
af_alg_release_parent(sk);
@@ -925,7 +843,7 @@ static int skcipher_accept_parent_nokey(void *private, struct sock *sk)
struct alg_sock *ask = alg_sk(sk);
struct skcipher_tfm *tfm = private;
struct crypto_skcipher *skcipher = tfm->skcipher;
- unsigned int len = sizeof(*ctx) + crypto_skcipher_reqsize(skcipher);
+ unsigned int len = sizeof(*ctx);
ctx = sock_kmalloc(sk, len, GFP_KERNEL);
if (!ctx)
@@ -940,22 +858,18 @@ static int skcipher_accept_parent_nokey(void *private, struct sock *sk)
memset(ctx->iv, 0, crypto_skcipher_ivsize(skcipher));
- INIT_LIST_HEAD(&ctx->tsgl);
+ INIT_LIST_HEAD(&ctx->tsgl_list);
ctx->len = len;
ctx->used = 0;
+ ctx->processed = 0;
+ ctx->inflight = 0;
ctx->more = 0;
ctx->merge = 0;
ctx->enc = 0;
- atomic_set(&ctx->inflight, 0);
af_alg_init_completion(&ctx->completion);
ask->private = ctx;
- skcipher_request_set_tfm(&ctx->req, skcipher);
- skcipher_request_set_callback(&ctx->req, CRYPTO_TFM_REQ_MAY_SLEEP |
- CRYPTO_TFM_REQ_MAY_BACKLOG,
- af_alg_complete, &ctx->completion);
-
sk->sk_destruct = skcipher_sock_destruct;
return 0;
--
2.9.3
^ permalink raw reply related
* [PATCH v2 2/2] crypto: aead AF_ALG - overhaul memory management
From: Stephan Müller @ 2017-02-08 20:16 UTC (permalink / raw)
To: herbert; +Cc: linux-crypto
In-Reply-To: <2626546.tAxUg930Nc@positron.chronox.de>
The updated memory management is described in the top part of the code.
As one benefit of the changed memory management, the AIO and synchronous
operation is now implemented in one common function. The AF_ALG
operation uses the async kernel crypto API interface for each cipher
operation. Thus, the only difference between the AIO and sync operation
types visible from user space is:
1. the callback function to be invoked when the asynchronous operation
is completed
2. whether to wait for the completion of the kernel crypto API operation
or not
The change includes the overhaul of the TX and RX SGL handling. The TX
SGL holding the data sent from user space to the kernel is now dynamic
similar to algif_skcipher. This dynamic nature allows a continuous
operation of a thread sending data and a second thread receiving the
data. These threads do not need to synchronize as the kernel processes
as much data from the TX SGL to fill the RX SGL.
The caller reading the data from the kernel defines the amount of data
to be processed. Considering that the interface covers AEAD
authenticating ciphers, the reader must provide the buffer in the
correct size. Thus the reader defines the encryption size.
Signed-off-by: Stephan Mueller <smueller@chronox.de>
---
crypto/algif_aead.c | 668 +++++++++++++++++++++++++++-------------------------
1 file changed, 351 insertions(+), 317 deletions(-)
diff --git a/crypto/algif_aead.c b/crypto/algif_aead.c
index 533265f..6c24c81 100644
--- a/crypto/algif_aead.c
+++ b/crypto/algif_aead.c
@@ -11,6 +11,26 @@
* under the terms of the GNU General Public License as published by the Free
* Software Foundation; either version 2 of the License, or (at your option)
* any later version.
+ *
+ * The following concept of the memory management is used:
+ *
+ * The kernel maintains two SGLs, the TX SGL and the RX SGL. The TX SGL is
+ * filled by user space with the data submitted via sendpage/sendmsg. Filling
+ * up the TX SGL does not cause a crypto operation -- the data will only be
+ * tracked by the kernel. Upon receipt of one recvmsg call, the caller must
+ * provide a buffer which is tracked with the RX SGL.
+ *
+ * During the processing of the recvmsg operation, the cipher request is
+ * allocated and prepared. To support multiple recvmsg operations operating
+ * on one TX SGL, an offset pointer into the TX SGL is maintained. The TX SGL
+ * that is used for the crypto request is scatterwalk_ffwd by the offset
+ * pointer to obtain the start address the crypto operation shall use for
+ * the input data.
+ *
+ * After the completion of the crypto operation, the RX SGL and the cipher
+ * request is released. The processed TX SGL parts are released together with
+ * the RX SGL release and the offset pointer is reduced by the released
+ * data.
*/
#include <crypto/internal/aead.h>
@@ -24,45 +44,55 @@
#include <linux/net.h>
#include <net/sock.h>
-struct aead_sg_list {
- unsigned int cur;
- struct scatterlist sg[ALG_MAX_PAGES];
+struct aead_tsgl {
+ struct list_head list;
+ unsigned int cur; /* Last processed SG entry */
+ struct scatterlist sg[0]; /* Array of SGs forming the SGL */
};
-struct aead_async_rsgl {
+struct aead_rsgl {
struct af_alg_sgl sgl;
struct list_head list;
};
struct aead_async_req {
- struct scatterlist *tsgl;
- struct aead_async_rsgl first_rsgl;
- struct list_head list;
struct kiocb *iocb;
- unsigned int tsgls;
- char iv[];
+ struct sock *sk;
+
+ struct aead_rsgl first_rsgl; /* First RX SG */
+ struct list_head rsgl_list; /* Track RX SGs */
+
+ unsigned int outlen; /* Filled output buf length */
+
+ unsigned int areqlen; /* Length of this data struct */
+ struct aead_request aead_req; /* req ctx trails this struct */
};
struct aead_ctx {
- struct aead_sg_list tsgl;
- struct aead_async_rsgl first_rsgl;
- struct list_head list;
+ struct list_head tsgl_list; /* Link to TX SGL */
void *iv;
+ size_t aead_assoclen;
- struct af_alg_completion completion;
+ struct af_alg_completion completion; /* sync work queue */
- unsigned long used;
+ unsigned int inflight; /* Outstanding AIO ops */
+ size_t used; /* TX bytes sent to kernel */
+ size_t processed; /* Processed TX bytes */
- unsigned int len;
- bool more;
- bool merge;
- bool enc;
+ bool more; /* More data to be expected? */
+ bool merge; /* Merge new data into existing SG */
+ bool enc; /* Crypto operation: enc, dec */
- size_t aead_assoclen;
- struct aead_request aead_req;
+ unsigned int len; /* Length of allocated memory for this struct */
+ struct crypto_aead *aead_tfm;
};
+static DECLARE_WAIT_QUEUE_HEAD(aead_aio_finish_wait);
+
+#define MAX_SGL_ENTS ((4096 - sizeof(struct aead_tsgl)) / \
+ sizeof(struct scatterlist) - 1)
+
static inline int aead_sndbuf(struct sock *sk)
{
struct alg_sock *ask = alg_sk(sk);
@@ -79,42 +109,128 @@ static inline bool aead_writable(struct sock *sk)
static inline bool aead_sufficient_data(struct aead_ctx *ctx)
{
- unsigned as = crypto_aead_authsize(crypto_aead_reqtfm(&ctx->aead_req));
+ unsigned int as = crypto_aead_authsize(ctx->aead_tfm);
/*
* The minimum amount of memory needed for an AEAD cipher is
* the AAD and in case of decryption the tag.
+ *
+ * Also, sufficient data must be available after disregarding the
+ * already processed data.
*/
- return ctx->used >= ctx->aead_assoclen + (ctx->enc ? 0 : as);
+ return ctx->used >= ctx->aead_assoclen + (ctx->enc ? 0 : as) +
+ ctx->processed;
}
-static void aead_reset_ctx(struct aead_ctx *ctx)
+static int aead_alloc_tsgl(struct sock *sk)
{
- struct aead_sg_list *sgl = &ctx->tsgl;
+ struct alg_sock *ask = alg_sk(sk);
+ struct aead_ctx *ctx = ask->private;
+ struct aead_tsgl *sgl;
+ struct scatterlist *sg = NULL;
- sg_init_table(sgl->sg, ALG_MAX_PAGES);
- sgl->cur = 0;
- ctx->used = 0;
- ctx->more = 0;
- ctx->merge = 0;
+ sgl = list_entry(ctx->tsgl_list.prev, struct aead_tsgl, list);
+ if (!list_empty(&ctx->tsgl_list))
+ sg = sgl->sg;
+
+ if (!sg || sgl->cur >= MAX_SGL_ENTS) {
+ sgl = sock_kmalloc(sk, sizeof(*sgl) +
+ sizeof(sgl->sg[0]) * (MAX_SGL_ENTS + 1),
+ GFP_KERNEL);
+ if (!sgl)
+ return -ENOMEM;
+
+ sg_init_table(sgl->sg, MAX_SGL_ENTS + 1);
+ sgl->cur = 0;
+
+ if (sg)
+ sg_chain(sg, MAX_SGL_ENTS + 1, sgl->sg);
+
+ list_add_tail(&sgl->list, &ctx->tsgl_list);
+ }
+
+ return 0;
}
-static void aead_put_sgl(struct sock *sk)
+static void aead_pull_tsgl(struct sock *sk, size_t used)
{
struct alg_sock *ask = alg_sk(sk);
struct aead_ctx *ctx = ask->private;
- struct aead_sg_list *sgl = &ctx->tsgl;
- struct scatterlist *sg = sgl->sg;
+ struct aead_tsgl *sgl;
+ struct scatterlist *sg;
unsigned int i;
- for (i = 0; i < sgl->cur; i++) {
- if (!sg_page(sg + i))
- continue;
+ while (!list_empty(&ctx->tsgl_list)) {
+ sgl = list_first_entry(&ctx->tsgl_list, struct aead_tsgl,
+ list);
+ sg = sgl->sg;
+
+ for (i = 0; i < sgl->cur; i++) {
+ size_t plen = min_t(size_t, used, sg[i].length);
+
+ if (!sg_page(sg + i))
+ continue;
+
+ sg[i].length -= plen;
+ sg[i].offset += plen;
+
+ used -= plen;
+ ctx->used -= plen;
+ ctx->processed -= plen;
- put_page(sg_page(sg + i));
- sg_assign_page(sg + i, NULL);
+ if (sg[i].length)
+ return;
+
+ put_page(sg_page(sg + i));
+ sg_assign_page(sg + i, NULL);
+ }
+
+ list_del(&sgl->list);
+ sock_kfree_s(sk, sgl, sizeof(*sgl) + sizeof(sgl->sg[0]) *
+ (MAX_SGL_ENTS + 1));
+ }
+
+ if (!ctx->used)
+ ctx->merge = 0;
+}
+
+static void aead_free_rsgl(struct aead_async_req *areq)
+{
+ struct sock *sk = areq->sk;
+ struct aead_rsgl *rsgl, *tmp;
+
+ list_for_each_entry_safe(rsgl, tmp, &areq->rsgl_list, list) {
+ af_alg_free_sg(&rsgl->sgl);
+ list_del(&rsgl->list);
+ if (rsgl != &areq->first_rsgl)
+ sock_kfree_s(sk, rsgl, sizeof(*rsgl));
+ }
+}
+
+static int aead_wait_for_wmem(struct sock *sk, unsigned flags)
+{
+ DEFINE_WAIT_FUNC(wait, woken_wake_function);
+ int err = -ERESTARTSYS;
+ long timeout;
+
+ if (flags & MSG_DONTWAIT)
+ return -EAGAIN;
+
+ sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
+
+ add_wait_queue(sk_sleep(sk), &wait);
+ for (;;) {
+ if (signal_pending(current))
+ break;
+ timeout = MAX_SCHEDULE_TIMEOUT;
+ if (sk_wait_event(sk, &timeout, aead_writable(sk), &wait)) {
+ err = 0;
+ break;
+ }
}
- aead_reset_ctx(ctx);
+ remove_wait_queue(sk_sleep(sk), &wait);
+
+ return err;
}
static void aead_wmem_wakeup(struct sock *sk)
@@ -146,6 +262,7 @@ static int aead_wait_for_data(struct sock *sk, unsigned flags)
return -EAGAIN;
sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
+
add_wait_queue(sk_sleep(sk), &wait);
for (;;) {
if (signal_pending(current))
@@ -169,8 +286,6 @@ static void aead_data_wakeup(struct sock *sk)
struct aead_ctx *ctx = ask->private;
struct socket_wq *wq;
- if (ctx->more)
- return;
if (!ctx->used)
return;
@@ -189,14 +304,13 @@ static int aead_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
struct sock *sk = sock->sk;
struct alg_sock *ask = alg_sk(sk);
struct aead_ctx *ctx = ask->private;
- unsigned ivsize =
- crypto_aead_ivsize(crypto_aead_reqtfm(&ctx->aead_req));
- struct aead_sg_list *sgl = &ctx->tsgl;
+ unsigned int ivsize = crypto_aead_ivsize(ctx->aead_tfm);
+ struct aead_tsgl *sgl;
struct af_alg_control con = {};
long copied = 0;
bool enc = 0;
bool init = 0;
- int err = -EINVAL;
+ int err = 0;
if (msg->msg_controllen) {
err = af_alg_cmsg_send(msg, &con);
@@ -220,8 +334,10 @@ static int aead_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
}
lock_sock(sk);
- if (!ctx->more && ctx->used)
+ if (!ctx->more && ctx->used) {
+ err = -EINVAL;
goto unlock;
+ }
if (init) {
ctx->enc = enc;
@@ -232,11 +348,14 @@ static int aead_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
}
while (size) {
+ struct scatterlist *sg;
size_t len = size;
- struct scatterlist *sg = NULL;
+ size_t plen;
/* use the existing memory in an allocated page */
if (ctx->merge) {
+ sgl = list_entry(ctx->tsgl_list.prev,
+ struct aead_tsgl, list);
sg = sgl->sg + sgl->cur - 1;
len = min_t(unsigned long, len,
PAGE_SIZE - sg->offset - sg->length);
@@ -257,57 +376,60 @@ static int aead_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
}
if (!aead_writable(sk)) {
- /* user space sent too much data */
- aead_put_sgl(sk);
- err = -EMSGSIZE;
- goto unlock;
+ err = aead_wait_for_wmem(sk, msg->msg_flags);
+ if (err)
+ goto unlock;
}
/* allocate a new page */
len = min_t(unsigned long, size, aead_sndbuf(sk));
- while (len) {
- size_t plen = 0;
- if (sgl->cur >= ALG_MAX_PAGES) {
- aead_put_sgl(sk);
- err = -E2BIG;
- goto unlock;
- }
+ err = aead_alloc_tsgl(sk);
+ if (err)
+ goto unlock;
+
+ sgl = list_entry(ctx->tsgl_list.prev, struct aead_tsgl,
+ list);
+ sg = sgl->sg;
+ if (sgl->cur)
+ sg_unmark_end(sg + sgl->cur - 1);
+
+ do {
+ unsigned int i = sgl->cur;
- sg = sgl->sg + sgl->cur;
plen = min_t(size_t, len, PAGE_SIZE);
- sg_assign_page(sg, alloc_page(GFP_KERNEL));
- err = -ENOMEM;
- if (!sg_page(sg))
+ sg_assign_page(sg + i, alloc_page(GFP_KERNEL));
+ if (!sg_page(sg + i)) {
+ err = -ENOMEM;
goto unlock;
+ }
- err = memcpy_from_msg(page_address(sg_page(sg)),
+ err = memcpy_from_msg(page_address(sg_page(sg + i)),
msg, plen);
if (err) {
- __free_page(sg_page(sg));
- sg_assign_page(sg, NULL);
+ __free_page(sg_page(sg + i));
+ sg_assign_page(sg + i, NULL);
goto unlock;
}
- sg->offset = 0;
- sg->length = plen;
+ sg[i].length = plen;
len -= plen;
ctx->used += plen;
copied += plen;
- sgl->cur++;
size -= plen;
- ctx->merge = plen & (PAGE_SIZE - 1);
- }
+ sgl->cur++;
+ } while (len && sgl->cur < MAX_SGL_ENTS);
+
+ if (!size)
+ sg_mark_end(sg + sgl->cur - 1);
+
+ ctx->merge = plen & (PAGE_SIZE - 1);
}
err = 0;
ctx->more = msg->msg_flags & MSG_MORE;
- if (!ctx->more && !aead_sufficient_data(ctx)) {
- aead_put_sgl(sk);
- err = -EMSGSIZE;
- }
unlock:
aead_data_wakeup(sk);
@@ -322,15 +444,12 @@ static ssize_t aead_sendpage(struct socket *sock, struct page *page,
struct sock *sk = sock->sk;
struct alg_sock *ask = alg_sk(sk);
struct aead_ctx *ctx = ask->private;
- struct aead_sg_list *sgl = &ctx->tsgl;
+ struct aead_tsgl *sgl;
int err = -EINVAL;
if (flags & MSG_SENDPAGE_NOTLAST)
flags |= MSG_MORE;
- if (sgl->cur >= ALG_MAX_PAGES)
- return -E2BIG;
-
lock_sock(sk);
if (!ctx->more && ctx->used)
goto unlock;
@@ -339,13 +458,22 @@ static ssize_t aead_sendpage(struct socket *sock, struct page *page,
goto done;
if (!aead_writable(sk)) {
- /* user space sent too much data */
- aead_put_sgl(sk);
- err = -EMSGSIZE;
- goto unlock;
+ err = aead_wait_for_wmem(sk, flags);
+ if (err)
+ goto unlock;
}
+ err = aead_alloc_tsgl(sk);
+ if (err)
+ goto unlock;
+
ctx->merge = 0;
+ sgl = list_entry(ctx->tsgl_list.prev, struct aead_tsgl, list);
+
+ if (sgl->cur)
+ sg_unmark_end(sgl->sg + sgl->cur - 1);
+
+ sg_mark_end(sgl->sg + sgl->cur);
get_page(page);
sg_set_page(sgl->sg + sgl->cur, page, size, offset);
@@ -356,11 +484,6 @@ static ssize_t aead_sendpage(struct socket *sock, struct page *page,
done:
ctx->more = flags & MSG_MORE;
- if (!ctx->more && !aead_sufficient_data(ctx)) {
- aead_put_sgl(sk);
- err = -EMSGSIZE;
- }
-
unlock:
aead_data_wakeup(sk);
release_sock(sk);
@@ -368,206 +491,62 @@ static ssize_t aead_sendpage(struct socket *sock, struct page *page,
return err ?: size;
}
-#define GET_ASYM_REQ(req, tfm) (struct aead_async_req *) \
- ((char *)req + sizeof(struct aead_request) + \
- crypto_aead_reqsize(tfm))
-
- #define GET_REQ_SIZE(tfm) sizeof(struct aead_async_req) + \
- crypto_aead_reqsize(tfm) + crypto_aead_ivsize(tfm) + \
- sizeof(struct aead_request)
-
static void aead_async_cb(struct crypto_async_request *_req, int err)
{
- struct sock *sk = _req->data;
+ struct aead_async_req *areq = _req->data;
+ struct sock *sk = areq->sk;
struct alg_sock *ask = alg_sk(sk);
struct aead_ctx *ctx = ask->private;
- struct crypto_aead *tfm = crypto_aead_reqtfm(&ctx->aead_req);
- struct aead_request *req = aead_request_cast(_req);
- struct aead_async_req *areq = GET_ASYM_REQ(req, tfm);
- struct scatterlist *sg = areq->tsgl;
- struct aead_async_rsgl *rsgl;
struct kiocb *iocb = areq->iocb;
- unsigned int i, reqlen = GET_REQ_SIZE(tfm);
-
- list_for_each_entry(rsgl, &areq->list, list) {
- af_alg_free_sg(&rsgl->sgl);
- if (rsgl != &areq->first_rsgl)
- sock_kfree_s(sk, rsgl, sizeof(*rsgl));
- }
-
- for (i = 0; i < areq->tsgls; i++)
- put_page(sg_page(sg + i));
-
- sock_kfree_s(sk, areq->tsgl, sizeof(*areq->tsgl) * areq->tsgls);
- sock_kfree_s(sk, req, reqlen);
- __sock_put(sk);
- iocb->ki_complete(iocb, err, err);
-}
-
-static int aead_recvmsg_async(struct socket *sock, struct msghdr *msg,
- int flags)
-{
- struct sock *sk = sock->sk;
- struct alg_sock *ask = alg_sk(sk);
- struct aead_ctx *ctx = ask->private;
- struct crypto_aead *tfm = crypto_aead_reqtfm(&ctx->aead_req);
- struct aead_async_req *areq;
- struct aead_request *req = NULL;
- struct aead_sg_list *sgl = &ctx->tsgl;
- struct aead_async_rsgl *last_rsgl = NULL, *rsgl;
- unsigned int as = crypto_aead_authsize(tfm);
- unsigned int i, reqlen = GET_REQ_SIZE(tfm);
- int err = -ENOMEM;
- unsigned long used;
- size_t outlen = 0;
- size_t usedpages = 0;
+ unsigned int resultlen;
lock_sock(sk);
- if (ctx->more) {
- err = aead_wait_for_data(sk, flags);
- if (err)
- goto unlock;
- }
-
- if (!aead_sufficient_data(ctx))
- goto unlock;
-
- used = ctx->used;
- if (ctx->enc)
- outlen = used + as;
- else
- outlen = used - as;
-
- req = sock_kmalloc(sk, reqlen, GFP_KERNEL);
- if (unlikely(!req))
- goto unlock;
-
- areq = GET_ASYM_REQ(req, tfm);
- memset(&areq->first_rsgl, '\0', sizeof(areq->first_rsgl));
- INIT_LIST_HEAD(&areq->list);
- areq->iocb = msg->msg_iocb;
- memcpy(areq->iv, ctx->iv, crypto_aead_ivsize(tfm));
- aead_request_set_tfm(req, tfm);
- aead_request_set_ad(req, ctx->aead_assoclen);
- aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
- aead_async_cb, sk);
- used -= ctx->aead_assoclen;
-
- /* take over all tx sgls from ctx */
- areq->tsgl = sock_kmalloc(sk,
- sizeof(*areq->tsgl) * max_t(u32, sgl->cur, 1),
- GFP_KERNEL);
- if (unlikely(!areq->tsgl))
- goto free;
-
- sg_init_table(areq->tsgl, max_t(u32, sgl->cur, 1));
- for (i = 0; i < sgl->cur; i++)
- sg_set_page(&areq->tsgl[i], sg_page(&sgl->sg[i]),
- sgl->sg[i].length, sgl->sg[i].offset);
-
- areq->tsgls = sgl->cur;
-
- /* create rx sgls */
- while (outlen > usedpages && iov_iter_count(&msg->msg_iter)) {
- size_t seglen = min_t(size_t, iov_iter_count(&msg->msg_iter),
- (outlen - usedpages));
-
- if (list_empty(&areq->list)) {
- rsgl = &areq->first_rsgl;
-
- } else {
- rsgl = sock_kmalloc(sk, sizeof(*rsgl), GFP_KERNEL);
- if (unlikely(!rsgl)) {
- err = -ENOMEM;
- goto free;
- }
- }
- rsgl->sgl.npages = 0;
- list_add_tail(&rsgl->list, &areq->list);
-
- /* make one iovec available as scatterlist */
- err = af_alg_make_sg(&rsgl->sgl, &msg->msg_iter, seglen);
- if (err < 0)
- goto free;
- usedpages += err;
-
- /* chain the new scatterlist with previous one */
- if (last_rsgl)
- af_alg_link_sg(&last_rsgl->sgl, &rsgl->sgl);
+ BUG_ON(!ctx->inflight);
- last_rsgl = rsgl;
+ /* Buffer size written by crypto operation. */
+ resultlen = areq->outlen;
- iov_iter_advance(&msg->msg_iter, err);
- }
+ aead_free_rsgl(areq);
+ aead_pull_tsgl(sk, areq->aead_req.cryptlen);
+ sock_kfree_s(sk, areq, areq->areqlen);
+ __sock_put(sk);
+ ctx->inflight--;
- /* ensure output buffer is sufficiently large */
- if (usedpages < outlen) {
- err = -EINVAL;
- goto unlock;
- }
+ iocb->ki_complete(iocb, err ? err : resultlen, 0);
- aead_request_set_crypt(req, areq->tsgl, areq->first_rsgl.sgl.sg, used,
- areq->iv);
- err = ctx->enc ? crypto_aead_encrypt(req) : crypto_aead_decrypt(req);
- if (err) {
- if (err == -EINPROGRESS) {
- sock_hold(sk);
- err = -EIOCBQUEUED;
- aead_reset_ctx(ctx);
- goto unlock;
- } else if (err == -EBADMSG) {
- aead_put_sgl(sk);
- }
- goto free;
- }
- aead_put_sgl(sk);
-
-free:
- list_for_each_entry(rsgl, &areq->list, list) {
- af_alg_free_sg(&rsgl->sgl);
- if (rsgl != &areq->first_rsgl)
- sock_kfree_s(sk, rsgl, sizeof(*rsgl));
- }
- if (areq->tsgl)
- sock_kfree_s(sk, areq->tsgl, sizeof(*areq->tsgl) * areq->tsgls);
- if (req)
- sock_kfree_s(sk, req, reqlen);
-unlock:
- aead_wmem_wakeup(sk);
release_sock(sk);
- return err ? err : outlen;
+
+ wake_up_interruptible(&aead_aio_finish_wait);
}
-static int aead_recvmsg_sync(struct socket *sock, struct msghdr *msg, int flags)
+static int aead_recvmsg(struct socket *sock, struct msghdr *msg, size_t ignored,
+ int flags)
{
struct sock *sk = sock->sk;
struct alg_sock *ask = alg_sk(sk);
struct aead_ctx *ctx = ask->private;
- unsigned as = crypto_aead_authsize(crypto_aead_reqtfm(&ctx->aead_req));
- struct aead_sg_list *sgl = &ctx->tsgl;
- struct aead_async_rsgl *last_rsgl = NULL;
- struct aead_async_rsgl *rsgl, *tmp;
+ struct crypto_aead *tfm = ctx->aead_tfm;
+ unsigned int as = crypto_aead_authsize(tfm);
+ unsigned int areqlen =
+ sizeof(struct aead_async_req) + crypto_aead_reqsize(tfm);
+ struct aead_tsgl *tsgl;
+ struct aead_async_req *areq;
+ struct aead_rsgl *last_rsgl = NULL;
+ struct scatterlist tsgl_head[2], *tsgl_head_p;
int err = -EINVAL;
- unsigned long used = 0;
- size_t outlen = 0;
- size_t usedpages = 0;
+ size_t used = 0; /* [in] TX bufs to be en/decrypted */
+ size_t outlen = 0; /* [out] RX bufs produced by kernel */
+ size_t usedpages = 0; /* [in] RX bufs to be used from user */
+ size_t processed = 0;
lock_sock(sk);
/*
- * Please see documentation of aead_request_set_crypt for the
- * description of the AEAD memory structure expected from the caller.
+ * Data length provided by caller via sendmsg/sendpage that has not
+ * yet been processed.
*/
-
- if (ctx->more) {
- err = aead_wait_for_data(sk, flags);
- if (err)
- goto unlock;
- }
-
- /* data length provided by caller via sendmsg/sendpage */
- used = ctx->used;
+ used = ctx->used - ctx->processed;
/*
* Make sure sufficient data is present -- note, the same check is
@@ -600,96 +579,151 @@ static int aead_recvmsg_sync(struct socket *sock, struct msghdr *msg, int flags)
*/
used -= ctx->aead_assoclen;
- /* convert iovecs of output buffers into scatterlists */
+ /* Allocate cipher request for current operation. */
+ areq = sock_kmalloc(sk, areqlen, GFP_KERNEL);
+ if (unlikely(!areq)) {
+ err = -ENOMEM;
+ goto unlock;
+ }
+ areq->areqlen = areqlen;
+ areq->sk = sk;
+ INIT_LIST_HEAD(&areq->rsgl_list);
+
+ /* convert iovecs of output buffers into RX SGL */
while (outlen > usedpages && iov_iter_count(&msg->msg_iter)) {
- size_t seglen = min_t(size_t, iov_iter_count(&msg->msg_iter),
- (outlen - usedpages));
+ struct aead_rsgl *rsgl;
+ size_t seglen;
+
+ if (!ctx->used) {
+ err = aead_wait_for_data(sk, flags);
+ if (err)
+ goto free;
+ }
+
+ seglen = min_t(size_t, (outlen - usedpages),
+ iov_iter_count(&msg->msg_iter));
- if (list_empty(&ctx->list)) {
- rsgl = &ctx->first_rsgl;
+ if (list_empty(&areq->rsgl_list)) {
+ rsgl = &areq->first_rsgl;
} else {
rsgl = sock_kmalloc(sk, sizeof(*rsgl), GFP_KERNEL);
if (unlikely(!rsgl)) {
err = -ENOMEM;
- goto unlock;
+ goto free;
}
}
+
rsgl->sgl.npages = 0;
- list_add_tail(&rsgl->list, &ctx->list);
+ list_add_tail(&rsgl->list, &areq->rsgl_list);
/* make one iovec available as scatterlist */
err = af_alg_make_sg(&rsgl->sgl, &msg->msg_iter, seglen);
if (err < 0)
- goto unlock;
- usedpages += err;
+ goto free;
+
/* chain the new scatterlist with previous one */
if (last_rsgl)
af_alg_link_sg(&last_rsgl->sgl, &rsgl->sgl);
last_rsgl = rsgl;
-
+ usedpages += err;
iov_iter_advance(&msg->msg_iter, err);
}
- /* ensure output buffer is sufficiently large */
+ /*
+ * Ensure output buffer is sufficiently large. If the caller provides
+ * less buffer space, only use the relative required input size. This
+ * allows AIO operation where the caller sent all data to be processed
+ * and the AIO operation performs the operation on the different chunks
+ * of the input data.
+ */
if (usedpages < outlen) {
- err = -EINVAL;
- goto unlock;
- }
+ size_t less = outlen - usedpages;
- sg_mark_end(sgl->sg + sgl->cur - 1);
- aead_request_set_crypt(&ctx->aead_req, sgl->sg, ctx->first_rsgl.sgl.sg,
- used, ctx->iv);
- aead_request_set_ad(&ctx->aead_req, ctx->aead_assoclen);
+ if (used < less) {
+ err = -EINVAL;
+ goto free;
+ }
+ used -= less;
+ outlen -= less;
+ }
- err = af_alg_wait_for_completion(ctx->enc ?
- crypto_aead_encrypt(&ctx->aead_req) :
- crypto_aead_decrypt(&ctx->aead_req),
+ tsgl = list_first_entry(&ctx->tsgl_list, struct aead_tsgl, list);
+ /* Get the head of the SGL we want to process */
+ tsgl_head_p = scatterwalk_ffwd(tsgl_head, tsgl->sg, ctx->processed);
+ BUG_ON(!tsgl_head_p);
+
+ /* Initialize the crypto operation */
+ aead_request_set_crypt(&areq->aead_req, tsgl_head_p,
+ areq->first_rsgl.sgl.sg, used, ctx->iv);
+ aead_request_set_ad(&areq->aead_req, ctx->aead_assoclen);
+ aead_request_set_tfm(&areq->aead_req, tfm);
+
+ if (msg->msg_iocb && !is_sync_kiocb(msg->msg_iocb)) {
+ /* AIO operation */
+ areq->iocb = msg->msg_iocb;
+ aead_request_set_callback(&areq->aead_req,
+ CRYPTO_TFM_REQ_MAY_BACKLOG,
+ aead_async_cb, areq);
+ err = ctx->enc ? crypto_aead_encrypt(&areq->aead_req) :
+ crypto_aead_decrypt(&areq->aead_req);
+ } else {
+ /* Synchronous operation */
+ aead_request_set_callback(&areq->aead_req,
+ CRYPTO_TFM_REQ_MAY_BACKLOG,
+ af_alg_complete, &ctx->completion);
+ err = af_alg_wait_for_completion(ctx->enc ?
+ crypto_aead_encrypt(&areq->aead_req) :
+ crypto_aead_decrypt(&areq->aead_req),
&ctx->completion);
+ }
if (err) {
- /* EBADMSG implies a valid cipher operation took place */
- if (err == -EBADMSG)
- aead_put_sgl(sk);
+ /* AIO operation in progress */
+ if (err == -EINPROGRESS) {
+ sock_hold(sk);
+ err = -EIOCBQUEUED;
- goto unlock;
+ /* Remember the TX bytes that were processed. */
+ ctx->processed += ctx->enc ? (outlen - as) :
+ (outlen + as);
+ ctx->inflight++;
+
+ /* Remember output size that will be generated. */
+ areq->outlen = outlen;
+
+ goto unlock;
+ }
+ /* EBADMSG implies a valid cipher operation took place */
+ else if (err != -EBADMSG)
+ goto free;
}
- aead_put_sgl(sk);
- err = 0;
+ processed = ctx->enc ? (outlen - as) : (outlen + as);
+ /* Remember the TX bytes that were processed. */
+ ctx->processed += processed;
+
+free:
+ aead_free_rsgl(areq);
+ if (areq)
+ sock_kfree_s(sk, areq, areqlen);
+ aead_pull_tsgl(sk, processed);
unlock:
- list_for_each_entry_safe(rsgl, tmp, &ctx->list, list) {
- af_alg_free_sg(&rsgl->sgl);
- list_del(&rsgl->list);
- if (rsgl != &ctx->first_rsgl)
- sock_kfree_s(sk, rsgl, sizeof(*rsgl));
- }
- INIT_LIST_HEAD(&ctx->list);
aead_wmem_wakeup(sk);
release_sock(sk);
-
return err ? err : outlen;
}
-static int aead_recvmsg(struct socket *sock, struct msghdr *msg, size_t ignored,
- int flags)
-{
- return (msg->msg_iocb && !is_sync_kiocb(msg->msg_iocb)) ?
- aead_recvmsg_async(sock, msg, flags) :
- aead_recvmsg_sync(sock, msg, flags);
-}
-
static unsigned int aead_poll(struct file *file, struct socket *sock,
poll_table *wait)
{
struct sock *sk = sock->sk;
struct alg_sock *ask = alg_sk(sk);
struct aead_ctx *ctx = ask->private;
- unsigned int mask;
+ unsigned int mask = 0;
sock_poll_wait(file, sk_sleep(sk), wait);
- mask = 0;
if (!ctx->more)
mask |= POLLIN | POLLRDNORM;
@@ -746,11 +780,14 @@ static void aead_sock_destruct(struct sock *sk)
{
struct alg_sock *ask = alg_sk(sk);
struct aead_ctx *ctx = ask->private;
- unsigned int ivlen = crypto_aead_ivsize(
- crypto_aead_reqtfm(&ctx->aead_req));
+ unsigned int ivlen = crypto_aead_ivsize(ctx->aead_tfm);
WARN_ON(atomic_read(&sk->sk_refcnt) != 0);
- aead_put_sgl(sk);
+
+ /* Suspend caller if AIO operations are in flight. */
+ wait_event_interruptible(aead_aio_finish_wait, (ctx->inflight == 0));
+
+ aead_pull_tsgl(sk, ctx->used);
sock_kzfree_s(sk, ctx->iv, ivlen);
sock_kfree_s(sk, ctx, ctx->len);
af_alg_release_parent(sk);
@@ -760,7 +797,7 @@ static int aead_accept_parent(void *private, struct sock *sk)
{
struct aead_ctx *ctx;
struct alg_sock *ask = alg_sk(sk);
- unsigned int len = sizeof(*ctx) + crypto_aead_reqsize(private);
+ unsigned int len = sizeof(*ctx);
unsigned int ivlen = crypto_aead_ivsize(private);
ctx = sock_kmalloc(sk, len, GFP_KERNEL);
@@ -775,23 +812,20 @@ static int aead_accept_parent(void *private, struct sock *sk)
}
memset(ctx->iv, 0, ivlen);
+ INIT_LIST_HEAD(&ctx->tsgl_list);
ctx->len = len;
ctx->used = 0;
+ ctx->processed = 0;
ctx->more = 0;
ctx->merge = 0;
ctx->enc = 0;
- ctx->tsgl.cur = 0;
+ ctx->inflight = 0;
ctx->aead_assoclen = 0;
af_alg_init_completion(&ctx->completion);
- sg_init_table(ctx->tsgl.sg, ALG_MAX_PAGES);
- INIT_LIST_HEAD(&ctx->list);
+ ctx->aead_tfm = private;
ask->private = ctx;
- aead_request_set_tfm(&ctx->aead_req, private);
- aead_request_set_callback(&ctx->aead_req, CRYPTO_TFM_REQ_MAY_BACKLOG,
- af_alg_complete, &ctx->completion);
-
sk->sk_destruct = aead_sock_destruct;
return 0;
--
2.9.3
^ permalink raw reply related
* Re: [PATCH v7 0/5] Update LZ4 compressor module
From: Minchan Kim @ 2017-02-08 23:31 UTC (permalink / raw)
To: Sven Schmidt
Cc: akpm, bongkyu.kim, rsalvaterra, sergey.senozhatsky, gregkh,
linux-kernel, herbert, davem, linux-crypto, anton, ccross,
keescook, tony.luck
In-Reply-To: <1486321748-19085-1-git-send-email-4sschmid@informatik.uni-hamburg.de>
Hello Sven,
On Sun, Feb 05, 2017 at 08:09:03PM +0100, Sven Schmidt wrote:
>
> This patchset is for updating the LZ4 compression module to a version based
> on LZ4 v1.7.3 allowing to use the fast compression algorithm aka LZ4 fast
> which provides an "acceleration" parameter as a tradeoff between
> high compression ratio and high compression speed.
>
> We want to use LZ4 fast in order to support compression in lustre
> and (mostly, based on that) investigate data reduction techniques in behalf of
> storage systems.
>
> Also, it will be useful for other users of LZ4 compression, as with LZ4 fast
> it is possible to enable applications to use fast and/or high compression
> depending on the usecase.
> For instance, ZRAM is offering a LZ4 backend and could benefit from an updated
> LZ4 in the kernel.
>
> LZ4 homepage: http://www.lz4.org/
> LZ4 source repository: https://github.com/lz4/lz4
> Source version: 1.7.3
>
> Benchmark (taken from [1], Core i5-4300U @1.9GHz):
> ----------------|--------------|----------------|----------
> Compressor | Compression | Decompression | Ratio
> ----------------|--------------|----------------|----------
> memcpy | 4200 MB/s | 4200 MB/s | 1.000
> LZ4 fast 50 | 1080 MB/s | 2650 MB/s | 1.375
> LZ4 fast 17 | 680 MB/s | 2220 MB/s | 1.607
> LZ4 fast 5 | 475 MB/s | 1920 MB/s | 1.886
> LZ4 default | 385 MB/s | 1850 MB/s | 2.101
>
> [1] http://fastcompression.blogspot.de/2015/04/sampling-or-faster-lz4.html
>
> [PATCH 1/5] lib: Update LZ4 compressor module
> [PATCH 2/5] lib/decompress_unlz4: Change module to work with new LZ4 module version
> [PATCH 3/5] crypto: Change LZ4 modules to work with new LZ4 module version
> [PATCH 4/5] fs/pstore: fs/squashfs: Change usage of LZ4 to work with new LZ4 version
> [PATCH 5/5] lib/lz4: Remove back-compat wrappers
Today, I did zram-lz4 performance test with fio in current mmotm and
found it makes regression about 20%.
"lz4-update" means current mmots(git://git.cmpxchg.org/linux-mmots.git) so
applied your 5 patches. (But now sure current mmots has recent uptodate
patches)
"revert" means I reverted your 5 patches in current mmots.
revert lz4-update
seq-write 1547 1339 86.55%
rand-write 22775 19381 85.10%
seq-read 7035 5589 79.45%
rand-read 78556 68479 87.17%
mixed-seq(R) 1305 1066 81.69%
mixed-seq(W) 1205 984 81.66%
mixed-rand(R) 17421 14993 86.06%
mixed-rand(W) 17391 14968 86.07%
My fio description file
[global]
bs=4k
ioengine=sync
size=100m
numjobs=1
group_reporting
buffer_compress_percentage=30
scramble_buffers=0
filename=/dev/zram0
loops=10
fsync_on_close=1
[seq-write]
bs=64k
rw=write
stonewall
[rand-write]
rw=randwrite
stonewall
[seq-read]
bs=64k
rw=read
stonewall
[rand-read]
rw=randread
stonewall
[mixed-seq]
bs=64k
rw=rw
stonewall
[mixed-rand]
rw=randrw
stonewall
^ permalink raw reply
* Re: [PATCH v7 0/5] Update LZ4 compressor module
From: Eric Biggers @ 2017-02-09 0:24 UTC (permalink / raw)
To: Minchan Kim
Cc: Sven Schmidt, akpm, bongkyu.kim, rsalvaterra, sergey.senozhatsky,
gregkh, linux-kernel, herbert, davem, linux-crypto, anton, ccross,
keescook, tony.luck
In-Reply-To: <20170208233121.GA16728@bbox>
On Thu, Feb 09, 2017 at 08:31:21AM +0900, Minchan Kim wrote:
>
> Today, I did zram-lz4 performance test with fio in current mmotm and
> found it makes regression about 20%.
>
This may or may not be the cause of the specific regression you're observing,
but I just noticed that the proposed patch drops a lot of FORCEINLINE
annotations from upstream LZ4. The FORCEINLINE's are there for a reason,
especially for the main decompression and compression functions which are
basically "templates" that take in different sets of constant parameters, and
should be left in. We should #define FORCEINLINE to __always_inline somewhere,
or just do a s/FORCEINLINE/__always_inline/g.
Note that the upstream LZ4 code is very carefully optimized, so we should not,
in general, be changing things like when functions are force-inlined, what the
hash table size is, etc.
[Also, for some reason linux-crypto is apparently still not receiving patch 1/5
in the series. It's missing from the linux-crypto archive at
http://www.spinics.net/lists/linux-crypto/, so it's not just me.]
Thanks!
Eric
^ permalink raw reply
* Re: [PATCH] Revert "hwrng: core - zeroize buffers with random data"
From: Linus Torvalds @ 2017-02-09 1:57 UTC (permalink / raw)
To: David Daney
Cc: Linux Crypto Mailing List, Matt Mackall, Herbert Xu,
Stephan Mueller, Linux Kernel Mailing List
In-Reply-To: <20170208002331.29408-1-david.daney@cavium.com>
Stephan, Herbert? The zeroes in /dev/hwrng output are obviously
complete crap, so there's something badly wrong somewhere.
The locking, for example, is completely buggered. There's even a
comment about it, but that comment makes the correct observation of
"but y'know: randomness". But the memset() also being outside the lock
makes a complete joke of the whole thing.
Is the hwrng thing even worth maintaining? Compared to something like
/dev/urandom, it clearly does not do a very good job.
So I'm inclined to take the revert, but I'm also somewhat inclined to
simply mark this crud broken when we have other things that clearly do
a lot better.
Linus
On Tue, Feb 7, 2017 at 4:23 PM, David Daney <david.daney@cavium.com> wrote:
> This reverts commit 2cc751545854d7bd7eedf4d7e377bb52e176cd07.
^ permalink raw reply
* [PATCH] random: Don't overwrite CRNG state in crng_initialize()
From: Alden Tondettar @ 2017-02-09 3:31 UTC (permalink / raw)
To: Theodore Ts'o
Cc: Alden Tondettar, Arnd Bergmann, Greg Kroah-Hartman, linux-crypto,
linux-kernel
The new non-blocking system introduced in commit e192be9d9a30 ("random:
replace non-blocking pool with a Chacha20-based CRNG") can under
some circumstances report itself initialized while it still contains
dangerously little entropy, as follows:
Approximately every 64th call to add_interrupt_randomness(), the "fast"
pool of interrupt-timing-based entropy is fed into one of two places. At
calls numbered <= 256, the fast pool is XORed into the primary CRNG state.
At call 256, the CRNG is deemed initialized, getrandom(2) is unblocked,
and reading from /dev/urandom no longer gives warnings.
At calls > 256, the fast pool is fed into the input pool, leaving the CRNG
untouched.
The problem arises between call number 256 and 320. If crng_initialize()
is called at this time, it will overwrite the _entire_ CRNG state with
48 bytes generated from the input pool. But the add_interrupt_randomness()
entropy was never _in_ the input pool, so instead we destroy all of
add_interrupt_randomness()'s hard work and replace it with the possibly
feeble entropy from a few calls to add_device_randomness(),
init_std_data(), etc.
Nevertheless crng_ready() will happily inform us that getrandom(2) and
/dev/urandom are ready to go. This state of affairs will continue until
the next call to crng_reseed() dumps more entropy into the CRNG and _that_
won't happen until the input pool entropy estimate exceeds 128 bits. On a
system with no rotational drives and little or no user input it could be
a long wait (minutes).
Dumping /var/foo/random-seed into /dev/urandom won't help here because
that only adds entropy to the pool without increasing the estimate.
In short, the situation is:
A) No usable hardware RNG or arch_get_random() (or we don't trust it...)
B) add_interrupt_randomness() called 256-320 times but other
add_*_randomness() functions aren't adding much entropy.
C) then crng_initialize() is called
D) not enough calls to add_*_randomness() to push the entropy
estimate over 128 (yet)
E) getrandom(2) or /dev/urandom used for something important
Based on a few experiments with VMs, A) through D) can occur easily in
practice. And with no HDD we have a window of about a minute or two for
E) to happen before add_interrupt_randomness() finally pushes the
estimate over 128 on its own.
The fix is simple enough: XOR the input pool randomness into the CRNG state
instead of overwriting it.
Fixes: e192be9d9a30 ("random: replace non-blocking pool with a Chacha20-based CRNG")
Signed-off-by: Alden Tondettar <alden.tondettar@gmail.com>
---
drivers/char/random.c | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/drivers/char/random.c b/drivers/char/random.c
index 1ef2640..bda30df 100644
--- a/drivers/char/random.c
+++ b/drivers/char/random.c
@@ -777,20 +777,22 @@ static void crng_initialize(struct crng_state *crng)
{
int i;
unsigned long rv;
+ __u32 tmp[12];
memcpy(&crng->state[0], "expand 32-byte k", 16);
if (crng == &primary_crng)
- _extract_entropy(&input_pool, &crng->state[4],
- sizeof(__u32) * 12, 0);
+ _extract_entropy(&input_pool, tmp, sizeof(__u32) * 12, 0);
else
- get_random_bytes(&crng->state[4], sizeof(__u32) * 12);
+ get_random_bytes(tmp, sizeof(__u32) * 12);
for (i = 4; i < 16; i++) {
if (!arch_get_random_seed_long(&rv) &&
!arch_get_random_long(&rv))
rv = random_get_entropy();
- crng->state[i] ^= rv;
+ crng->state[i] ^= tmp[i - 4] ^ rv;
}
crng->init_time = jiffies - CRNG_RESEED_INTERVAL - 1;
+
+ memzero_explicit(tmp, sizeof(tmp));
}
static int crng_fast_load(const char *cp, size_t len)
--
2.1.4
^ permalink raw reply related
* Re: [PATCH] random: Don't overwrite CRNG state in crng_initialize()
From: Theodore Ts'o @ 2017-02-09 4:19 UTC (permalink / raw)
To: Alden Tondettar
Cc: Arnd Bergmann, Greg Kroah-Hartman, linux-crypto, linux-kernel
In-Reply-To: <1486611086-2290-1-git-send-email-alden.tondettar@gmail.com>
On Wed, Feb 08, 2017 at 08:31:26PM -0700, Alden Tondettar wrote:
> The new non-blocking system introduced in commit e192be9d9a30 ("random:
> replace non-blocking pool with a Chacha20-based CRNG") can under
> some circumstances report itself initialized while it still contains
> dangerously little entropy, as follows:
>
> Approximately every 64th call to add_interrupt_randomness(), the "fast"
> pool of interrupt-timing-based entropy is fed into one of two places. At
> calls numbered <= 256, the fast pool is XORed into the primary CRNG state.
> At call 256, the CRNG is deemed initialized, getrandom(2) is unblocked,
> and reading from /dev/urandom no longer gives warnings.
>
> At calls > 256, the fast pool is fed into the input pool, leaving the CRNG
> untouched.
>
> The problem arises between call number 256 and 320. If crng_initialize()
> is called at this time, it will overwrite the _entire_ CRNG state with
> 48 bytes generated from the input pool.
So in practice this isn't a problem because crng_initialize is called
in early init. For reference, the ordering of init calls are:
"early", <--- crng_initialize is here()
"core", <---- ftrace is initialized here()
"postcore",
"arch",
"subsys", <---- acpi_init is here()
"fs",
"device", <---- device probing is here
"late",
So in practice, call 256 typically happens **well** after
crng_initialize. You can see where it is the boot messages, which is
after 2.5 seconds into the boot:
[ 2.570733] rtc_cmos 00:02: alarms up to one month, y3k, 114 bytes nvram, hpet irqs
[ 2.570863] usbcore: registered new interface driver i2c-tiny-usb
[ 2.571035] device-mapper: uevent: version 1.0.3
[ 2.571215] random: fast init done <-------------
[ 2.571316] device-mapper: ioctl: 4.35.0-ioctl (2016-06-23) initialised: dm-devel@redhat.com
[ 2.571678] device-mapper: multipath round-robin: version 1.1.0 loaded
[ 2.571728] intel_pstate: Intel P-state driver initializing
[ 2.572331] input: AT Translated Set 2 keyboard as /devices/platform/i8042/serio0/input/input3
[ 2.572462] intel_pstate: HWP enabled
[ 2.572464] sdhci: Secure Digital Host Controller Interface driver
When is crng_initialize() called? Sometime *before* 0.05 seconds into
the boot on my laptop:
[ 0.054529] ftrace: allocating 29140 entries in 114 pages
> In short, the situation is:
>
> A) No usable hardware RNG or arch_get_random() (or we don't trust it...)
> B) add_interrupt_randomness() called 256-320 times but other
> add_*_randomness() functions aren't adding much entropy.
> C) then crng_initialize() is called
> D) not enough calls to add_*_randomness() to push the entropy
> estimate over 128 (yet)
> E) getrandom(2) or /dev/urandom used for something important
>
> Based on a few experiments with VMs, A) through D) can occur easily in
> practice. And with no HDD we have a window of about a minute or two for
> E) to happen before add_interrupt_randomness() finally pushes the
> estimate over 128 on its own.
How did you determine when crng_initialize() was being called? On a
VM generally there are fewer interrupts than on real hardware. On
KVM, for I see the random: fast_init message being printed 3.6 seconds
into the boot.
On Google Compute Engine, the fast_init message happens 52 seconds into the
boot.
So what VM where you using? I'm trying to figure out whether this is
hypothetical or real problem, and on what systems.
- Ted
^ permalink raw reply
* Re: [PATCH v7 0/5] Update LZ4 compressor module
From: Eric Biggers @ 2017-02-09 5:24 UTC (permalink / raw)
To: Minchan Kim
Cc: Sven Schmidt, akpm, bongkyu.kim, rsalvaterra, sergey.senozhatsky,
gregkh, linux-kernel, herbert, davem, linux-crypto, anton, ccross,
keescook, tony.luck
In-Reply-To: <20170209002436.GA103792@gmail.com>
Also I noticed another bug, this time in LZ4_count():
> #if defined(CONFIG_64BIT)
> #define LZ4_ARCH64 1
> #else
> #define LZ4_ARCH64 0
> #endif
...
> #ifdef LZ4_ARCH64
> if ((pIn < (pInLimit-3))
> && (LZ4_read32(pMatch) == LZ4_read32(pIn))) {
> pIn += 4; pMatch += 4;
> }
> #endif
Because of how LZ4_ARCH64 is defined, it needs to be '#if LZ4_ARCH64'.
But I also think the way upstream LZ4 does 64-bit detection could have just been
left as-is; it has a function which gets inlined:
static unsigned LZ4_64bits(void) { return sizeof(void*)==8; }
Eric
^ permalink raw reply
* Re: [PATCH] random: Don't overwrite CRNG state in crng_initialize()
From: Greg Kroah-Hartman @ 2017-02-09 6:47 UTC (permalink / raw)
To: Alden Tondettar
Cc: Theodore Ts'o, Arnd Bergmann, linux-crypto, linux-kernel
In-Reply-To: <1486611086-2290-1-git-send-email-alden.tondettar@gmail.com>
On Wed, Feb 08, 2017 at 08:31:26PM -0700, Alden Tondettar wrote:
> In short, the situation is:
>
> A) No usable hardware RNG or arch_get_random() (or we don't trust it...)
Wait, why would you not trust arch_get_random()? Is it broken somehow
on some arches? If so, why not fix that as well?
thanks,
greg k-h
^ permalink raw reply
* Re: [PATCH] random: Don't overwrite CRNG state in crng_initialize()
From: Alden Tondettar @ 2017-02-09 8:13 UTC (permalink / raw)
To: Theodore Ts'o
Cc: Arnd Bergmann, Greg Kroah-Hartman, linux-crypto, linux-kernel
In-Reply-To: <20170209041931.xgkmysquazppiewx@thunk.org>
On Wed, Feb 08, 2017 at 11:19:31PM -0500, Theodore Ts'o wrote:
> How did you determine when crng_initialize() was being called? On a
> VM generally there are fewer interrupts than on real hardware. On
> KVM, for I see the random: fast_init message being printed 3.6 seconds
> into the boot.
>
> On Google Compute Engine, the fast_init message happens 52 seconds into the
> boot.
>
> So what VM where you using? I'm trying to figure out whether this is
> hypothetical or real problem, and on what systems.
Adding a few printk()s to the latest kernel:
@@ -778,6 +778,8 @@ static void crng_initialize(struct crng_state *crng)
int i;
unsigned long rv;
+ printk("crng_initialize called\n");
+
memcpy(&crng->state[0], "expand 32-byte k", 16);
if (crng == &primary_crng)
_extract_entropy(&input_pool, &crng->state[4],
@@ -1149,6 +1151,9 @@ void add_interrupt_randomness(int irq, int irq_flags)
fast_mix(fast_pool);
add_interrupt_bench(cycles);
+ if (fast_pool->count >= 64)
+ printk("add_interrupt_randomness: fast_pool->count >= 64, dumping entropy");
+
if (!crng_ready()) {
if ((fast_pool->count >= 64) &&
crng_fast_load((char *) fast_pool->pool,
And using:
$ qemu-system-x86_64 --version
QEMU emulator version 2.1.2 (Debian 1:2.1+dfsg-12+deb8u6), Copyright (c) 2003-2008 Fabrice Bellard
$ qemu-system-x86_64 -nographic -enable-kvm -m 1024M -kernel bzImage -append "root=/dev/sda1 loglevel=3 console=ttyS0" hd3
I get:
[ 0.010247] mce: CPU supports 10 MCE banks
[ 0.010317] Last level iTLB entries: 4KB 0, 2MB 0, 4MB 0
[ 0.010318] Last level dTLB entries: 4KB 0, 2MB 0, 4MB 0, 1GB 0
[ 0.064002] add_interrupt_randomness: fast_pool->count >= 64, dumping entropy
[ 0.128003] add_interrupt_randomness: fast_pool->count >= 64, dumping entropy
[ 0.160364] Freeing SMP alternatives memory: 36K
[ 0.160428] ftrace: allocating 35771 entries in 140 pages
[ 0.172384] smpboot: Max logical packages: 1
[ 0.173964] ..TIMER: vector=0x30 apic1=0 pin1=2 apic2=-1 pin2=-1
[ 0.184000] add_interrupt_randomness: fast_pool->count >= 64, dumping entropy
[ 0.184000] add_interrupt_randomness: fast_pool->count >= 64, dumping entropy
[ 0.184000] random: fast init done
[ 0.184000] smpboot: CPU0: Intel QEMU Virtual CPU version 2.1.2 (family: 0x6,
model: 0x6, stepping: 0x3)
[ 0.184000] Performance Events: PMU not available due to virtualization, usin
g software events only.
[ 0.184000] crng_initialize called
[ 0.184000] crng_initialize called
[ 0.184000] smp: Bringing up secondary CPUs ...
[ 0.184000] smp: Brought up 1 node, 1 CPU
Sometimes I get three add_interrupt_randomness lines instead of four which
is fine but still cutting things awfully close.
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox