* [PATCH 1/3] cxgb4: Add Chelsio LLD support Chelsio Crypto ULD
From: Yeshaswi M R Gowda @ 2016-07-11 18:28 UTC (permalink / raw)
To: hariprasad, netdev, linux-kernel, herbert, davem, linux-crypto,
jlulla, atul.gupta, harsh
Cc: Yeshaswi M R Gowda
In-Reply-To: <1468261688-24525-1-git-send-email-yeshaswi@chelsio.com>
The Chelsio crypto driver is an Upper Layer Driver (ULD), making use
of the Chelsio Lower Layer Driver (LLD - cxgb4). The LLD facilitates
the basic infrastructure services of the ULD. These services include
queue allocation, deallocation and registration with LLD. The queues
are used for sending the crypto requests to the Chelsio's hardware
and for receiving the responses from the hardware.
This patch enables the services mentioned for the Chelsio's crypto
driver.
Signed-off-by: Yeshaswi M R Gowda <yeshaswi@chelsio.com>
---
drivers/net/ethernet/chelsio/cxgb4/cxgb4.h | 22 +-
drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c | 71 +++-
drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h | 10 +
drivers/net/ethernet/chelsio/cxgb4/sge.c | 64 ++++
drivers/net/ethernet/chelsio/cxgb4/t4_msg.h | 437 +++++++++++++++++++++++
drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h | 125 +++++++
6 files changed, 721 insertions(+), 8 deletions(-)
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
index b4fceb9..14b26dd 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
@@ -346,6 +346,8 @@ struct adapter_params {
unsigned int max_ordird_qp; /* Max read depth per RDMA QP */
unsigned int max_ird_adapter; /* Max read depth per adapter */
+
+ unsigned char ulp_crypto_lookaside; /* crypto lookaside support */
};
/* State needed to monitor the forward progress of SGE Ingress DMA activities
@@ -435,7 +437,7 @@ enum {
MAX_CTRL_QUEUES = NCHAN, /* # of control Tx queues */
MAX_RDMA_QUEUES = NCHAN, /* # of streaming RDMA Rx queues */
MAX_RDMA_CIQS = 32, /* # of RDMA concentrator IQs */
-
+ MAX_CRYPTO_QUEUES = 32, /* # of crypto queues */
/* # of streaming iSCSIT Rx queues */
MAX_ISCSIT_QUEUES = MAX_OFLD_QSETS,
};
@@ -455,7 +457,8 @@ enum {
INGQ_EXTRAS = 2, /* firmware event queue and */
/* forwarded interrupts */
MAX_INGQ = MAX_ETH_QSETS + MAX_OFLD_QSETS + MAX_RDMA_QUEUES +
- MAX_RDMA_CIQS + MAX_ISCSIT_QUEUES + INGQ_EXTRAS,
+ MAX_RDMA_CIQS + MAX_ISCSIT_QUEUES + INGQ_EXTRAS +
+ MAX_CRYPTO_QUEUES,
};
struct adapter;
@@ -509,6 +512,10 @@ enum { /* adapter flags */
FW_OFLD_CONN = (1 << 9),
};
+enum {
+ ULP_CRYPTO_LOOKASIDE = 1 << 0,
+};
+
struct rx_sw_desc;
struct sge_fl { /* SGE free-buffer queue state */
@@ -682,10 +689,12 @@ struct sge_ctrl_txq { /* state for an SGE control Tx queue */
struct sge {
struct sge_eth_txq ethtxq[MAX_ETH_QSETS];
struct sge_ofld_txq ofldtxq[MAX_OFLD_QSETS];
+ struct sge_ofld_txq cryptotxq[MAX_CRYPTO_QUEUES];
struct sge_ctrl_txq ctrlq[MAX_CTRL_QUEUES];
struct sge_eth_rxq ethrxq[MAX_ETH_QSETS];
struct sge_ofld_rxq iscsirxq[MAX_OFLD_QSETS];
+ struct sge_ofld_rxq cryptorxq[MAX_CRYPTO_QUEUES];
struct sge_ofld_rxq iscsitrxq[MAX_ISCSIT_QUEUES];
struct sge_ofld_rxq rdmarxq[MAX_RDMA_QUEUES];
struct sge_ofld_rxq rdmaciq[MAX_RDMA_CIQS];
@@ -699,10 +708,12 @@ struct sge {
u16 ethtxq_rover; /* Tx queue to clean up next */
u16 iscsiqsets; /* # of active iSCSI queue sets */
u16 niscsitq; /* # of available iSCST Rx queues */
+ u16 ncryptoq; /* # of available lookaside crypto queues */
u16 rdmaqs; /* # of available RDMA Rx queues */
u16 rdmaciqs; /* # of available RDMA concentrator IQs */
u16 iscsi_rxq[MAX_OFLD_QSETS];
u16 iscsit_rxq[MAX_ISCSIT_QUEUES];
+ u16 crypto_rxq[MAX_CRYPTO_QUEUES];
u16 rdma_rxq[MAX_RDMA_QUEUES];
u16 rdma_ciq[MAX_RDMA_CIQS];
u16 timer_val[SGE_NTIMERS];
@@ -732,6 +743,7 @@ struct sge {
#define for_each_iscsitrxq(sge, i) for (i = 0; i < (sge)->niscsitq; i++)
#define for_each_rdmarxq(sge, i) for (i = 0; i < (sge)->rdmaqs; i++)
#define for_each_rdmaciq(sge, i) for (i = 0; i < (sge)->rdmaciqs; i++)
+#define for_each_cryptorxq(sge, i) for (i = 0; i < (sge)->ncryptoq; i++)
struct l2t_data;
@@ -1441,7 +1453,7 @@ int t4_fw_bye(struct adapter *adap, unsigned int mbox);
int t4_early_init(struct adapter *adap, unsigned int mbox);
int t4_fw_reset(struct adapter *adap, unsigned int mbox, int reset);
int t4_fixup_host_params(struct adapter *adap, unsigned int page_size,
- unsigned int cache_line_size);
+ unsigned int cache_line_size);
int t4_fw_initialize(struct adapter *adap, unsigned int mbox);
int t4_query_params(struct adapter *adap, unsigned int mbox, unsigned int pf,
unsigned int vf, unsigned int nparams, const u32 *params,
@@ -1468,8 +1480,8 @@ int t4_free_vi(struct adapter *adap, unsigned int mbox,
unsigned int pf, unsigned int vf,
unsigned int viid);
int t4_set_rxmode(struct adapter *adap, unsigned int mbox, unsigned int viid,
- int mtu, int promisc, int all_multi, int bcast, int vlanex,
- bool sleep_ok);
+ int mtu, int promisc, int all_multi, int bcast, int vlanex,
+ bool sleep_ok);
int t4_alloc_mac_filt(struct adapter *adap, unsigned int mbox,
unsigned int viid, bool free, unsigned int naddr,
const u8 **addr, u16 *idx, u64 *hash, bool sleep_ok);
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
index 477db47..565fa1b 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
@@ -228,7 +228,7 @@ static DEFINE_MUTEX(uld_mutex);
static LIST_HEAD(adap_rcu_list);
static DEFINE_SPINLOCK(adap_rcu_lock);
static struct cxgb4_uld_info ulds[CXGB4_ULD_MAX];
-static const char *const uld_str[] = { "RDMA", "iSCSI", "iSCSIT" };
+static const char *const uld_str[] = { "RDMA", "iSCSI", "iSCSIT", "CRYPTO" };
static void link_report(struct net_device *dev)
{
@@ -786,6 +786,10 @@ static void name_msix_vecs(struct adapter *adap)
snprintf(adap->msix_info[msi_idx++].desc, n, "%s-iscsi%d",
adap->port[0]->name, i);
+ for_each_cryptorxq(&adap->sge, i)
+ snprintf(adap->msix_info[msi_idx++].desc, n, "%s-crypto%d",
+ adap->port[0]->name, i);
+
for_each_iscsitrxq(&adap->sge, i)
snprintf(adap->msix_info[msi_idx++].desc, n, "%s-iSCSIT%d",
adap->port[0]->name, i);
@@ -803,7 +807,7 @@ static int request_msix_queue_irqs(struct adapter *adap)
{
struct sge *s = &adap->sge;
int err, ethqidx, iscsiqidx = 0, rdmaqidx = 0, rdmaciqqidx = 0;
- int iscsitqidx = 0;
+ int iscsitqidx = 0, cryptoqidx = 0;
int msi_index = 2;
err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0,
@@ -838,6 +842,15 @@ static int request_msix_queue_irqs(struct adapter *adap)
goto unwind;
msi_index++;
}
+ for_each_cryptorxq(s, cryptoqidx) {
+ err = request_irq(adap->msix_info[msi_index].vec,
+ t4_sge_intr_msix, 0,
+ adap->msix_info[msi_index].desc,
+ &s->cryptorxq[cryptoqidx].rspq);
+ if (err)
+ goto unwind;
+ msi_index++;
+ }
for_each_rdmarxq(s, rdmaqidx) {
err = request_irq(adap->msix_info[msi_index].vec,
t4_sge_intr_msix, 0,
@@ -874,6 +887,9 @@ unwind:
while (--ethqidx >= 0)
free_irq(adap->msix_info[--msi_index].vec,
&s->ethrxq[ethqidx].rspq);
+ while (--cryptoqidx >= 0)
+ free_irq(adap->msix_info[--msi_index].vec,
+ &s->cryptorxq[cryptoqidx].rspq);
free_irq(adap->msix_info[1].vec, &s->fw_evtq);
return err;
}
@@ -892,6 +908,9 @@ static void free_msix_queue_irqs(struct adapter *adap)
for_each_iscsitrxq(s, i)
free_irq(adap->msix_info[msi_index++].vec,
&s->iscsitrxq[i].rspq);
+ for_each_cryptorxq(s, i)
+ free_irq(adap->msix_info[msi_index++].vec,
+ &s->cryptorxq[i].rspq);
for_each_rdmarxq(s, i)
free_irq(adap->msix_info[msi_index++].vec, &s->rdmarxq[i].rspq);
for_each_rdmaciq(s, i)
@@ -1155,6 +1174,8 @@ freeout: t4_free_sge_resources(adap);
ALLOC_OFLD_RXQS(s->rdmarxq, s->rdmaqs, 1, s->rdma_rxq, false);
j = s->rdmaciqs / adap->params.nports; /* rdmaq queues per channel */
ALLOC_OFLD_RXQS(s->rdmaciq, s->rdmaciqs, j, s->rdma_ciq, false);
+ j = s->ncryptoq / adap->params.nports;
+ ALLOC_OFLD_RXQS(s->cryptorxq, s->ncryptoq, j, s->crypto_rxq, 0);
#undef ALLOC_OFLD_RXQS
@@ -1170,6 +1191,18 @@ freeout: t4_free_sge_resources(adap);
goto freeout;
}
+ j = s->ncryptoq / adap->params.nports;
+ for_each_cryptorxq(s, i) {
+ struct sge_eth_txq *t;
+
+ t = (struct sge_eth_txq *)&s->cryptotxq[i];
+ err = t4_sge_alloc_ofld_txq(adap, &s->cryptotxq[i],
+ adap->port[i / j],
+ s->fw_evtq.cntxt_id);
+ if (err)
+ goto freeout;
+ }
+
t4_write_reg(adap, is_t4(adap->params.chip) ?
MPS_TRC_RSS_CONTROL_A :
MPS_T5_TRC_RSS_CONTROL_A,
@@ -2601,7 +2634,7 @@ static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state)
int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p)
{
int ret = 0;
- struct adapter *adap;
+ struct adapter *adap = NULL;
if (type >= CXGB4_ULD_MAX)
return -EINVAL;
@@ -4131,6 +4164,8 @@ static int adap_init0(struct adapter *adap)
adap->vres.iscsi.start = val[0];
adap->vres.iscsi.size = val[1] - val[0] + 1;
}
+ if (caps_cmd.cryptocaps)
+ adap->params.ulp_crypto_lookaside |= ULP_CRYPTO_LOOKASIDE;
#undef FW_PARAM_PFVF
#undef FW_PARAM_DEV
@@ -4406,6 +4441,12 @@ static void cfg_queues(struct adapter *adap)
if (!is_t4(adap->params.chip))
s->niscsitq = s->iscsiqsets;
}
+ if (adap->params.ulp_crypto_lookaside & ULP_CRYPTO_LOOKASIDE) {
+ s->ncryptoq = min_t(int, MAX_CRYPTO_QUEUES, num_online_cpus());
+ s->ncryptoq = (s->ncryptoq / adap->params.nports) *
+ adap->params.nports;
+ s->ncryptoq = max_t(int, s->ncryptoq, adap->params.nports);
+ }
for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
struct sge_eth_rxq *r = &s->ethrxq[i];
@@ -4420,6 +4461,9 @@ static void cfg_queues(struct adapter *adap)
for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++)
s->ctrlq[i].q.size = 512;
+ for (i = 0; i < ARRAY_SIZE(s->cryptotxq); i++)
+ s->cryptotxq[i].q.size = 1024;
+
for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++)
s->ofldtxq[i].q.size = 1024;
@@ -4462,6 +4506,14 @@ static void cfg_queues(struct adapter *adap)
r->rspq.uld = CXGB4_ULD_RDMA;
}
+ for (i = 0; i < ARRAY_SIZE(s->cryptorxq); i++) {
+ struct sge_ofld_rxq *r = &s->cryptorxq[i];
+
+ init_rspq(adap, &r->rspq, 5, 1, 1024, 64);
+ r->rspq.uld = CXGB4_ULD_CRYPTO;
+ r->fl.size = 72;
+ }
+
init_rspq(adap, &s->fw_evtq, 0, 1, 1024, 64);
init_rspq(adap, &s->intrq, 0, 1, 2 * MAX_INGQ, 64);
}
@@ -4520,9 +4572,14 @@ static int enable_msix(struct adapter *adap)
/* need nchan for each possible ULD */
if (is_t4(adap->params.chip))
ofld_need = 3 * nchan;
+ else if (is_t6(adap->params.chip))
+ ofld_need = 5 * nchan;
else
ofld_need = 4 * nchan;
}
+ if (adap->params.ulp_crypto_lookaside & ULP_CRYPTO_LOOKASIDE)
+ want += s->ncryptoq;
+
#ifdef CONFIG_CHELSIO_T4_DCB
/* For Data Center Bridging we need 8 Ethernet TX Priority Queues for
* each port.
@@ -4549,6 +4606,10 @@ static int enable_msix(struct adapter *adap)
if (i < s->ethqsets)
reduce_ethqs(adap, i);
}
+ if (adap->params.ulp_crypto_lookaside & ULP_CRYPTO_LOOKASIDE) {
+ if (allocated < want)
+ s->ncryptoq = nchan;
+ }
if (is_offload(adap)) {
if (allocated < want) {
s->rdmaqs = nchan;
@@ -4563,6 +4624,8 @@ static int enable_msix(struct adapter *adap)
s->rdmaqs - s->rdmaciqs - s->niscsitq;
s->iscsiqsets = (i / nchan) * nchan; /* round down */
+ if (adap->params.ulp_crypto_lookaside & ULP_CRYPTO_LOOKASIDE)
+ i -= s->ncryptoq;
}
for (i = 0; i < allocated; ++i)
adap->msix_info[i].vec = entries[i].vector;
@@ -4570,6 +4633,8 @@ static int enable_msix(struct adapter *adap)
"nic %d iscsi %d rdma cpl %d rdma ciq %d\n",
allocated, s->max_ethqsets, s->iscsiqsets, s->rdmaqs,
s->rdmaciqs);
+ if (adap->params.ulp_crypto_lookaside & ULP_CRYPTO_LOOKASIDE)
+ dev_info(adap->pdev_dev, " crypto %d\n", s->ncryptoq);
kfree(entries);
return 0;
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h
index f3c58aa..963e03b 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h
@@ -40,6 +40,7 @@
#include <linux/skbuff.h>
#include <linux/inetdevice.h>
#include <linux/atomic.h>
+#include <linux/pci.h>
#include "cxgb4.h"
/* CPL message priority levels */
@@ -192,6 +193,7 @@ enum cxgb4_uld {
CXGB4_ULD_RDMA,
CXGB4_ULD_ISCSI,
CXGB4_ULD_ISCSIT,
+ CXGB4_ULD_CRYPTO,
CXGB4_ULD_MAX
};
@@ -280,6 +282,11 @@ struct cxgb4_lld_info {
unsigned int iscsi_llimit; /* chip's iscsi region llimit */
void **iscsi_ppm; /* iscsi page pod manager */
int nodeid; /* device numa node id */
+ unsigned int ulp_crypto; /* crypto lookaside support */
+};
+
+enum {
+ ULD_CRYPTO_LOOKASIDE = 1 << 0,
};
struct cxgb4_uld_info {
@@ -322,6 +329,9 @@ int cxgb4_flush_eq_cache(struct net_device *dev);
int cxgb4_read_tpte(struct net_device *dev, u32 stag, __be32 *tpte);
u64 cxgb4_read_sge_timestamp(struct net_device *dev);
+int cxgb4_is_crypto_q_full(struct net_device *dev, unsigned int idx);
+int cxgb4_crypto_send(struct net_device *dev, struct sk_buff *skb);
+
enum cxgb4_bar2_qtype { CXGB4_BAR2_QTYPE_EGRESS, CXGB4_BAR2_QTYPE_INGRESS };
int cxgb4_bar2_sge_qregs(struct net_device *dev,
unsigned int qid,
diff --git a/drivers/net/ethernet/chelsio/cxgb4/sge.c b/drivers/net/ethernet/chelsio/cxgb4/sge.c
index bad253b..6ce1362 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/sge.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/sge.c
@@ -1813,6 +1813,48 @@ int cxgb4_ofld_send(struct net_device *dev, struct sk_buff *skb)
}
EXPORT_SYMBOL(cxgb4_ofld_send);
+static inline int crypto_send(struct adapter *adap, struct sk_buff *skb)
+{
+ unsigned int idx = skb_txq(skb);
+
+ if (unlikely(is_ctrl_pkt(skb)))
+ return ctrl_xmit(&adap->sge.ctrlq[idx], skb);
+ return ofld_xmit(&adap->sge.cryptotxq[idx], skb);
+}
+
+int t4_crypto_send(struct adapter *adap, struct sk_buff *skb)
+{
+ int ret;
+
+ local_bh_disable();
+ ret = crypto_send(adap, skb);
+ local_bh_enable();
+ return ret;
+}
+
+int cxgb4_crypto_send(struct net_device *dev, struct sk_buff *skb)
+{
+ return t4_crypto_send(netdev2adap(dev), skb);
+}
+EXPORT_SYMBOL(cxgb4_crypto_send);
+
+int cxgb4_is_crypto_q_full(struct net_device *dev, unsigned int idx)
+{
+ int ret = 0;
+ struct sge_ofld_txq *q;
+ struct adapter *adap = netdev2adap(dev);
+
+ local_bh_disable();
+ q = &adap->sge.cryptotxq[idx];
+ spin_lock(&q->sendq.lock);
+ if (q->full)
+ ret = -1;
+ spin_unlock(&q->sendq.lock);
+ local_bh_enable();
+ return ret;
+}
+EXPORT_SYMBOL(cxgb4_is_crypto_q_full);
+
static inline void copy_frags(struct sk_buff *skb,
const struct pkt_gl *gl, unsigned int offset)
{
@@ -3019,6 +3061,7 @@ void t4_free_sge_resources(struct adapter *adap)
t4_free_ofld_rxqs(adap, adap->sge.niscsitq, adap->sge.iscsitrxq);
t4_free_ofld_rxqs(adap, adap->sge.rdmaqs, adap->sge.rdmarxq);
t4_free_ofld_rxqs(adap, adap->sge.rdmaciqs, adap->sge.rdmaciq);
+ t4_free_ofld_rxqs(adap, adap->sge.ncryptoq, adap->sge.cryptorxq);
/* clean up offload Tx queues */
for (i = 0; i < ARRAY_SIZE(adap->sge.ofldtxq); i++) {
@@ -3035,6 +3078,21 @@ void t4_free_sge_resources(struct adapter *adap)
}
}
+ /* clean up crypto queues */
+ for (i = 0; i < ARRAY_SIZE(adap->sge.cryptotxq); i++) {
+ struct sge_ofld_txq *q = &adap->sge.cryptotxq[i];
+
+ if (q->q.desc) {
+ tasklet_kill(&q->qresume_tsk);
+ t4_ctrl_eq_free(adap, adap->mbox, adap->pf,
+ 0, q->q.cntxt_id);
+ free_tx_desc(adap, &q->q, q->q.in_use, false);
+ kfree(q->q.sdesc);
+ __skb_queue_purge(&q->sendq);
+ free_txq(adap, &q->q);
+ }
+ }
+
/* clean up control Tx queues */
for (i = 0; i < ARRAY_SIZE(adap->sge.ctrlq); i++) {
struct sge_ctrl_txq *cq = &adap->sge.ctrlq[i];
@@ -3093,6 +3151,12 @@ void t4_sge_stop(struct adapter *adap)
if (q->q.desc)
tasklet_kill(&q->qresume_tsk);
}
+ for (i = 0; i < ARRAY_SIZE(s->cryptotxq); i++) {
+ struct sge_ofld_txq *q = &s->cryptotxq[i];
+
+ if (q->q.desc)
+ tasklet_kill(&q->qresume_tsk);
+ }
for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++) {
struct sge_ctrl_txq *cq = &s->ctrlq[i];
diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h b/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h
index 4705e2d..cf0cc6c 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h
@@ -61,6 +61,7 @@ enum {
CPL_ABORT_REQ_RSS = 0x2B,
CPL_ABORT_RPL_RSS = 0x2D,
+ CPL_RX_PHYS_ADDR = 0x30,
CPL_CLOSE_CON_RPL = 0x32,
CPL_ISCSI_HDR = 0x33,
CPL_RDMA_CQE = 0x35,
@@ -83,6 +84,10 @@ enum {
CPL_PASS_OPEN_REQ6 = 0x81,
CPL_ACT_OPEN_REQ6 = 0x83,
+ CPL_TX_TLS_PDU = 0x88,
+ CPL_TX_SEC_PDU = 0x8A,
+ CPL_TX_TLS_ACK = 0x8B,
+
CPL_RDMA_TERMINATE = 0xA2,
CPL_RDMA_WRITE = 0xA4,
CPL_SGE_EGR_UPDATE = 0xA5,
@@ -94,6 +99,8 @@ enum {
CPL_FW4_PLD = 0xC1,
CPL_FW4_ACK = 0xC3,
+ CPL_RX_PHYS_DSGL = 0xD0,
+
CPL_FW6_MSG = 0xE0,
CPL_FW6_PLD = 0xE1,
CPL_TX_PKT_LSO = 0xED,
@@ -1360,6 +1367,15 @@ struct ulptx_idata {
__be32 len;
};
+struct ulp_txpkt {
+ __be32 cmd_dest;
+ __be32 len;
+};
+
+#define S_ULPTX_CMD 24
+#define M_ULPTX_CMD 0xFF
+#define V_ULPTX_CMD(x) ((x) << S_ULPTX_CMD)
+
#define ULPTX_NSGE_S 0
#define ULPTX_NSGE_V(x) ((x) << ULPTX_NSGE_S)
@@ -1367,6 +1383,22 @@ struct ulptx_idata {
#define ULPTX_MORE_V(x) ((x) << ULPTX_MORE_S)
#define ULPTX_MORE_F ULPTX_MORE_V(1U)
+#define S_ULP_TXPKT_DEST 16
+#define M_ULP_TXPKT_DEST 0x3
+#define V_ULP_TXPKT_DEST(x) ((x) << S_ULP_TXPKT_DEST)
+
+#define S_ULP_TXPKT_FID 4
+#define M_ULP_TXPKT_FID 0x7ff
+#define V_ULP_TXPKT_FID(x) ((x) << S_ULP_TXPKT_FID)
+
+#define S_ULP_TXPKT_RO 3
+#define V_ULP_TXPKT_RO(x) ((x) << S_ULP_TXPKT_RO)
+#define F_ULP_TXPKT_RO V_ULP_TXPKT_RO(1U)
+
+#define S_ULP_TX_SC_MORE 23
+#define V_ULP_TX_SC_MORE(x) ((x) << S_ULP_TX_SC_MORE)
+#define F_ULP_TX_SC_MORE V_ULP_TX_SC_MORE(1U)
+
struct ulp_mem_io {
WR_HDR;
__be32 cmd;
@@ -1404,4 +1436,409 @@ struct ulp_mem_io {
#define ULP_MEMIO_DATA_LEN_S 0
#define ULP_MEMIO_DATA_LEN_V(x) ((x) << ULP_MEMIO_DATA_LEN_S)
+#define S_ULPTX_NSGE 0
+#define M_ULPTX_NSGE 0xFFFF
+#define V_ULPTX_NSGE(x) ((x) << S_ULPTX_NSGE)
+#define G_ULPTX_NSGE(x) (((x) >> S_ULPTX_NSGE) & M_ULPTX_NSGE)
+
+struct ulptx_sc_memrd {
+ __be32 cmd_to_len;
+ __be32 addr;
+};
+
+#define S_ULP_TXPKT_DATAMODIFY 23
+#define M_ULP_TXPKT_DATAMODIFY 0x1
+#define V_ULP_TXPKT_DATAMODIFY(x) ((x) << S_ULP_TXPKT_DATAMODIFY)
+#define G_ULP_TXPKT_DATAMODIFY(x) \
+ (((x) >> S_ULP_TXPKT_DATAMODIFY) & M_ULP_TXPKT_DATAMODIFY_)
+#define F_ULP_TXPKT_DATAMODIFY V_ULP_TXPKT_DATAMODIFY(1U)
+
+#define S_ULP_TXPKT_CHANNELID 22
+#define M_ULP_TXPKT_CHANNELID 0x1
+#define V_ULP_TXPKT_CHANNELID(x) ((x) << S_ULP_TXPKT_CHANNELID)
+#define G_ULP_TXPKT_CHANNELID(x) \
+ (((x) >> S_ULP_TXPKT_CHANNELID) & M_ULP_TXPKT_CHANNELID)
+#define F_ULP_TXPKT_CHANNELID V_ULP_TXPKT_CHANNELID(1U)
+
+#define S_SCMD_SEQ_NO_CTRL 29
+#define M_SCMD_SEQ_NO_CTRL 0x3
+#define V_SCMD_SEQ_NO_CTRL(x) ((x) << S_SCMD_SEQ_NO_CTRL)
+#define G_SCMD_SEQ_NO_CTRL(x) \
+ (((x) >> S_SCMD_SEQ_NO_CTRL) & M_SCMD_SEQ_NO_CTRL)
+
+/* StsFieldPrsnt- Status field at the end of the TLS PDU */
+#define S_SCMD_STATUS_PRESENT 28
+#define M_SCMD_STATUS_PRESENT 0x1
+#define V_SCMD_STATUS_PRESENT(x) ((x) << S_SCMD_STATUS_PRESENT)
+#define G_SCMD_STATUS_PRESENT(x) \
+ (((x) >> S_SCMD_STATUS_PRESENT) & M_SCMD_STATUS_PRESENT)
+#define F_SCMD_STATUS_PRESENT V_SCMD_STATUS_PRESENT(1U)
+
+/* ProtoVersion - Protocol Version 0: 1.2, 1:1.1, 2:DTLS, 3:Generic,
+ * 3-15: Reserved.
+ */
+#define S_SCMD_PROTO_VERSION 24
+#define M_SCMD_PROTO_VERSION 0xf
+#define V_SCMD_PROTO_VERSION(x) ((x) << S_SCMD_PROTO_VERSION)
+#define G_SCMD_PROTO_VERSION(x) \
+ (((x) >> S_SCMD_PROTO_VERSION) & M_SCMD_PROTO_VERSION)
+
+/* EncDecCtrl - Encryption/Decryption Control. 0: Encrypt, 1: Decrypt */
+#define S_SCMD_ENC_DEC_CTRL 23
+#define M_SCMD_ENC_DEC_CTRL 0x1
+#define V_SCMD_ENC_DEC_CTRL(x) ((x) << S_SCMD_ENC_DEC_CTRL)
+#define G_SCMD_ENC_DEC_CTRL(x) \
+ (((x) >> S_SCMD_ENC_DEC_CTRL) & M_SCMD_ENC_DEC_CTRL)
+#define F_SCMD_ENC_DEC_CTRL V_SCMD_ENC_DEC_CTRL(1U)
+
+/* CipherAuthSeqCtrl - Cipher Authentication Sequence Control. */
+#define S_SCMD_CIPH_AUTH_SEQ_CTRL 22
+#define M_SCMD_CIPH_AUTH_SEQ_CTRL 0x1
+#define V_SCMD_CIPH_AUTH_SEQ_CTRL(x) \
+ ((x) << S_SCMD_CIPH_AUTH_SEQ_CTRL)
+#define G_SCMD_CIPH_AUTH_SEQ_CTRL(x) \
+ (((x) >> S_SCMD_CIPH_AUTH_SEQ_CTRL) & M_SCMD_CIPH_AUTH_SEQ_CTRL)
+#define F_SCMD_CIPH_AUTH_SEQ_CTRL V_SCMD_CIPH_AUTH_SEQ_CTRL(1U)
+
+/* CiphMode - Cipher Mode. 0: NOP, 1:AES-CBC, 2:AES-GCM, 3:AES-CTR,
+ * 4:Generic-AES, 5-15: Reserved.
+ */
+#define S_SCMD_CIPH_MODE 18
+#define M_SCMD_CIPH_MODE 0xf
+#define V_SCMD_CIPH_MODE(x) ((x) << S_SCMD_CIPH_MODE)
+#define G_SCMD_CIPH_MODE(x) \
+ (((x) >> S_SCMD_CIPH_MODE) & M_SCMD_CIPH_MODE)
+
+/* AuthMode - Auth Mode. 0: NOP, 1:SHA1, 2:SHA2-224, 3:SHA2-256
+ * 4-15: Reserved
+ */
+#define S_SCMD_AUTH_MODE 14
+#define M_SCMD_AUTH_MODE 0xf
+#define V_SCMD_AUTH_MODE(x) ((x) << S_SCMD_AUTH_MODE)
+#define G_SCMD_AUTH_MODE(x) \
+ (((x) >> S_SCMD_AUTH_MODE) & M_SCMD_AUTH_MODE)
+
+/* HmacCtrl - HMAC Control. 0:NOP, 1:No truncation, 2:Support HMAC Truncation
+ * per RFC 4366, 3:IPSec 96 bits, 4-7:Reserved
+ */
+#define S_SCMD_HMAC_CTRL 11
+#define M_SCMD_HMAC_CTRL 0x7
+#define V_SCMD_HMAC_CTRL(x) ((x) << S_SCMD_HMAC_CTRL)
+#define G_SCMD_HMAC_CTRL(x) \
+ (((x) >> S_SCMD_HMAC_CTRL) & M_SCMD_HMAC_CTRL)
+
+/* IvSize - IV size in units of 2 bytes */
+#define S_SCMD_IV_SIZE 7
+#define M_SCMD_IV_SIZE 0xf
+#define V_SCMD_IV_SIZE(x) ((x) << S_SCMD_IV_SIZE)
+#define G_SCMD_IV_SIZE(x) \
+ (((x) >> S_SCMD_IV_SIZE) & M_SCMD_IV_SIZE)
+
+/* NumIVs - Number of IVs */
+#define S_SCMD_NUM_IVS 0
+#define M_SCMD_NUM_IVS 0x7f
+#define V_SCMD_NUM_IVS(x) ((x) << S_SCMD_NUM_IVS)
+#define G_SCMD_NUM_IVS(x) \
+ (((x) >> S_SCMD_NUM_IVS) & M_SCMD_NUM_IVS)
+
+/* EnbDbgId - If this is enabled upper 20 (63:44) bits if SeqNumber
+ * (below) are used as Cid (connection id for debug status), these
+ * bits are padded to zero for forming the 64 bit
+ * sequence number for TLS
+ */
+#define S_SCMD_ENB_DBGID 31
+#define M_SCMD_ENB_DBGID 0x1
+#define V_SCMD_ENB_DBGID(x) ((x) << S_SCMD_ENB_DBGID)
+#define G_SCMD_ENB_DBGID(x) \
+ (((x) >> S_SCMD_ENB_DBGID) & M_SCMD_ENB_DBGID)
+
+/* IV generation in SW. */
+#define S_SCMD_IV_GEN_CTRL 30
+#define M_SCMD_IV_GEN_CTRL 0x1
+#define V_SCMD_IV_GEN_CTRL(x) ((x) << S_SCMD_IV_GEN_CTRL)
+#define G_SCMD_IV_GEN_CTRL(x) \
+ (((x) >> S_SCMD_IV_GEN_CTRL) & M_SCMD_IV_GEN_CTRL)
+#define F_SCMD_IV_GEN_CTRL V_SCMD_IV_GEN_CTRL(1U)
+
+/* More frags */
+#define S_SCMD_MORE_FRAGS 20
+#define M_SCMD_MORE_FRAGS 0x1
+#define V_SCMD_MORE_FRAGS(x) ((x) << S_SCMD_MORE_FRAGS)
+#define G_SCMD_MORE_FRAGS(x) (((x) >> S_SCMD_MORE_FRAGS) & M_SCMD_MORE_FRAGS)
+
+/*last frag */
+#define S_SCMD_LAST_FRAG 19
+#define M_SCMD_LAST_FRAG 0x1
+#define V_SCMD_LAST_FRAG(x) ((x) << S_SCMD_LAST_FRAG)
+#define G_SCMD_LAST_FRAG(x) (((x) >> S_SCMD_LAST_FRAG) & M_SCMD_LAST_FRAG)
+
+/* TlsCompPdu */
+#define S_SCMD_TLS_COMPPDU 18
+#define M_SCMD_TLS_COMPPDU 0x1
+#define V_SCMD_TLS_COMPPDU(x) ((x) << S_SCMD_TLS_COMPPDU)
+#define G_SCMD_TLS_COMPPDU(x) (((x) >> S_SCMD_TLS_COMPPDU) & M_SCMD_TLS_COMPPDU)
+
+/* KeyCntxtInline - Key context inline after the scmd OR PayloadOnly*/
+#define S_SCMD_KEY_CTX_INLINE 17
+#define M_SCMD_KEY_CTX_INLINE 0x1
+#define V_SCMD_KEY_CTX_INLINE(x) ((x) << S_SCMD_KEY_CTX_INLINE)
+#define G_SCMD_KEY_CTX_INLINE(x) \
+ (((x) >> S_SCMD_KEY_CTX_INLINE) & M_SCMD_KEY_CTX_INLINE)
+#define F_SCMD_KEY_CTX_INLINE V_SCMD_KEY_CTX_INLINE(1U)
+
+/* TLSFragEnable - 0: Host created TLS PDUs, 1: TLS Framgmentation in ASIC */
+#define S_SCMD_TLS_FRAG_ENABLE 16
+#define M_SCMD_TLS_FRAG_ENABLE 0x1
+#define V_SCMD_TLS_FRAG_ENABLE(x) ((x) << S_SCMD_TLS_FRAG_ENABLE)
+#define G_SCMD_TLS_FRAG_ENABLE(x) \
+ (((x) >> S_SCMD_TLS_FRAG_ENABLE) & M_SCMD_TLS_FRAG_ENABLE)
+#define F_SCMD_TLS_FRAG_ENABLE V_SCMD_TLS_FRAG_ENABLE(1U)
+
+/* MacOnly - Only send the MAC and discard PDU. This is valid for hash only
+ * modes, in this case TLS_TX will drop the PDU and only
+ * send back the MAC bytes.
+ */
+#define S_SCMD_MAC_ONLY 15
+#define M_SCMD_MAC_ONLY 0x1
+#define V_SCMD_MAC_ONLY(x) ((x) << S_SCMD_MAC_ONLY)
+#define G_SCMD_MAC_ONLY(x) \
+ (((x) >> S_SCMD_MAC_ONLY) & M_SCMD_MAC_ONLY)
+#define F_SCMD_MAC_ONLY V_SCMD_MAC_ONLY(1U)
+
+/* AadIVDrop - Drop the AAD and IV fields. Useful in protocols
+ * which have complex AAD and IV formations Eg:AES-CCM
+ */
+#define S_SCMD_AADIVDROP 14
+#define M_SCMD_AADIVDROP 0x1
+#define V_SCMD_AADIVDROP(x) ((x) << S_SCMD_AADIVDROP)
+#define G_SCMD_AADIVDROP(x) \
+ (((x) >> S_SCMD_AADIVDROP) & M_SCMD_AADIVDROP)
+#define F_SCMD_AADIVDROP V_SCMD_AADIVDROP(1U)
+
+/* HdrLength - Length of all headers excluding TLS header
+ * present before start of crypto PDU/payload.
+ */
+#define S_SCMD_HDR_LEN 0
+#define M_SCMD_HDR_LEN 0x3fff
+#define V_SCMD_HDR_LEN(x) ((x) << S_SCMD_HDR_LEN)
+#define G_SCMD_HDR_LEN(x) \
+ (((x) >> S_SCMD_HDR_LEN) & M_SCMD_HDR_LEN)
+
+struct cpl_tx_sec_pdu {
+ __be32 op_ivinsrtofst;
+ __be32 pldlen;
+ __be32 aadstart_cipherstop_hi;
+ __be32 cipherstop_lo_authinsert;
+ __be32 seqno_numivs;
+ __be32 ivgen_hdrlen;
+ __be64 scmd1;
+};
+
+#define S_CPL_TX_SEC_PDU_OPCODE 24
+#define M_CPL_TX_SEC_PDU_OPCODE 0xff
+#define V_CPL_TX_SEC_PDU_OPCODE(x) ((x) << S_CPL_TX_SEC_PDU_OPCODE)
+#define G_CPL_TX_SEC_PDU_OPCODE(x) \
+ (((x) >> S_CPL_TX_SEC_PDU_OPCODE) & M_CPL_TX_SEC_PDU_OPCODE)
+
+/* RX Channel Id */
+#define S_CPL_TX_SEC_PDU_RXCHID 22
+#define M_CPL_TX_SEC_PDU_RXCHID 0x1
+#define V_CPL_TX_SEC_PDU_RXCHID(x) ((x) << S_CPL_TX_SEC_PDU_RXCHID)
+#define G_CPL_TX_SEC_PDU_RXCHID(x) \
+ (((x) >> S_CPL_TX_SEC_PDU_RXCHID) & M_CPL_TX_SEC_PDU_RXCHID)
+#define F_CPL_TX_SEC_PDU_RXCHID V_CPL_TX_SEC_PDU_RXCHID(1U)
+
+/* Ack Follows */
+#define S_CPL_TX_SEC_PDU_ACKFOLLOWS 21
+#define M_CPL_TX_SEC_PDU_ACKFOLLOWS 0x1
+#define V_CPL_TX_SEC_PDU_ACKFOLLOWS(x) ((x) << S_CPL_TX_SEC_PDU_ACKFOLLOWS)
+#define G_CPL_TX_SEC_PDU_ACKFOLLOWS(x) \
+ (((x) >> S_CPL_TX_SEC_PDU_ACKFOLLOWS) & M_CPL_TX_SEC_PDU_ACKFOLLOWS)
+#define F_CPL_TX_SEC_PDU_ACKFOLLOWS V_CPL_TX_SEC_PDU_ACKFOLLOWS(1U)
+
+/* Loopback bit in cpl_tx_sec_pdu */
+#define S_CPL_TX_SEC_PDU_ULPTXLPBK 20
+#define M_CPL_TX_SEC_PDU_ULPTXLPBK 0x1
+#define V_CPL_TX_SEC_PDU_ULPTXLPBK(x) ((x) << S_CPL_TX_SEC_PDU_ULPTXLPBK)
+#define G_CPL_TX_SEC_PDU_ULPTXLPBK(x) \
+ (((x) >> S_CPL_TX_SEC_PDU_ULPTXLPBK) & M_CPL_TX_SEC_PDU_ULPTXLPBK)
+#define F_CPL_TX_SEC_PDU_ULPTXLPBK V_CPL_TX_SEC_PDU_ULPTXLPBK(1U)
+
+/* Length of cpl header encapsulated */
+#define S_CPL_TX_SEC_PDU_CPLLEN 16
+#define M_CPL_TX_SEC_PDU_CPLLEN 0xf
+#define V_CPL_TX_SEC_PDU_CPLLEN(x) ((x) << S_CPL_TX_SEC_PDU_CPLLEN)
+#define G_CPL_TX_SEC_PDU_CPLLEN(x) \
+ (((x) >> S_CPL_TX_SEC_PDU_CPLLEN) & M_CPL_TX_SEC_PDU_CPLLEN)
+
+/* PlaceHolder */
+#define S_CPL_TX_SEC_PDU_PLACEHOLDER 10
+#define M_CPL_TX_SEC_PDU_PLACEHOLDER 0x1
+#define V_CPL_TX_SEC_PDU_PLACEHOLDER(x) ((x) << S_CPL_TX_SEC_PDU_PLACEHOLDER)
+#define G_CPL_TX_SEC_PDU_PLACEHOLDER(x) \
+ (((x) >> S_CPL_TX_SEC_PDU_PLACEHOLDER) & \
+ M_CPL_TX_SEC_PDU_PLACEHOLDER)
+
+/* IvInsrtOffset: Insertion location for IV */
+#define S_CPL_TX_SEC_PDU_IVINSRTOFST 0
+#define M_CPL_TX_SEC_PDU_IVINSRTOFST 0x3ff
+#define V_CPL_TX_SEC_PDU_IVINSRTOFST(x) ((x) << S_CPL_TX_SEC_PDU_IVINSRTOFST)
+#define G_CPL_TX_SEC_PDU_IVINSRTOFST(x) \
+ (((x) >> S_CPL_TX_SEC_PDU_IVINSRTOFST) & \
+ M_CPL_TX_SEC_PDU_IVINSRTOFST)
+
+/* AadStartOffset: Offset in bytes for AAD start from
+ * the first byte following the pkt headers (0-255 bytes)
+ */
+#define S_CPL_TX_SEC_PDU_AADSTART 24
+#define M_CPL_TX_SEC_PDU_AADSTART 0xff
+#define V_CPL_TX_SEC_PDU_AADSTART(x) ((x) << S_CPL_TX_SEC_PDU_AADSTART)
+#define G_CPL_TX_SEC_PDU_AADSTART(x) \
+ (((x) >> S_CPL_TX_SEC_PDU_AADSTART) & \
+ M_CPL_TX_SEC_PDU_AADSTART)
+
+/* AadStopOffset: offset in bytes for AAD stop/end from the first byte following
+ * the pkt headers (0-511 bytes)
+ */
+#define S_CPL_TX_SEC_PDU_AADSTOP 15
+#define M_CPL_TX_SEC_PDU_AADSTOP 0x1ff
+#define V_CPL_TX_SEC_PDU_AADSTOP(x) ((x) << S_CPL_TX_SEC_PDU_AADSTOP)
+#define G_CPL_TX_SEC_PDU_AADSTOP(x) \
+ (((x) >> S_CPL_TX_SEC_PDU_AADSTOP) & M_CPL_TX_SEC_PDU_AADSTOP)
+
+/* CipherStartOffset: offset in bytes for encryption/decryption start from the
+ * first byte following the pkt headers (0-1023 bytes)
+ */
+#define S_CPL_TX_SEC_PDU_CIPHERSTART 5
+#define M_CPL_TX_SEC_PDU_CIPHERSTART 0x3ff
+#define V_CPL_TX_SEC_PDU_CIPHERSTART(x) ((x) << S_CPL_TX_SEC_PDU_CIPHERSTART)
+#define G_CPL_TX_SEC_PDU_CIPHERSTART(x) \
+ (((x) >> S_CPL_TX_SEC_PDU_CIPHERSTART) & \
+ M_CPL_TX_SEC_PDU_CIPHERSTART)
+
+/* CipherStopOffset: offset in bytes for encryption/decryption end
+ * from end of the payload of this command (0-511 bytes)
+ */
+#define S_CPL_TX_SEC_PDU_CIPHERSTOP_HI 0
+#define M_CPL_TX_SEC_PDU_CIPHERSTOP_HI 0x1f
+#define V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(x) \
+ ((x) << S_CPL_TX_SEC_PDU_CIPHERSTOP_HI)
+#define G_CPL_TX_SEC_PDU_CIPHERSTOP_HI(x) \
+ (((x) >> S_CPL_TX_SEC_PDU_CIPHERSTOP_HI) & \
+ M_CPL_TX_SEC_PDU_CIPHERSTOP_HI)
+
+#define S_CPL_TX_SEC_PDU_CIPHERSTOP_LO 28
+#define M_CPL_TX_SEC_PDU_CIPHERSTOP_LO 0xf
+#define V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(x) \
+ ((x) << S_CPL_TX_SEC_PDU_CIPHERSTOP_LO)
+#define G_CPL_TX_SEC_PDU_CIPHERSTOP_LO(x) \
+ (((x) >> S_CPL_TX_SEC_PDU_CIPHERSTOP_LO) & \
+ M_CPL_TX_SEC_PDU_CIPHERSTOP_LO)
+
+/* AuthStartOffset: offset in bytes for authentication start from
+ * the first byte following the pkt headers (0-1023)
+ */
+#define S_CPL_TX_SEC_PDU_AUTHSTART 18
+#define M_CPL_TX_SEC_PDU_AUTHSTART 0x3ff
+#define V_CPL_TX_SEC_PDU_AUTHSTART(x) ((x) << S_CPL_TX_SEC_PDU_AUTHSTART)
+#define G_CPL_TX_SEC_PDU_AUTHSTART(x) \
+ (((x) >> S_CPL_TX_SEC_PDU_AUTHSTART) & \
+ M_CPL_TX_SEC_PDU_AUTHSTART)
+
+/* AuthStopOffset: offset in bytes for authentication
+ * end from end of the payload of this command (0-511 Bytes)
+ */
+#define S_CPL_TX_SEC_PDU_AUTHSTOP 9
+#define M_CPL_TX_SEC_PDU_AUTHSTOP 0x1ff
+#define V_CPL_TX_SEC_PDU_AUTHSTOP(x) ((x) << S_CPL_TX_SEC_PDU_AUTHSTOP)
+#define G_CPL_TX_SEC_PDU_AUTHSTOP(x) \
+ (((x) >> S_CPL_TX_SEC_PDU_AUTHSTOP) & \
+ M_CPL_TX_SEC_PDU_AUTHSTOP)
+
+/* AuthInsrtOffset: offset in bytes for authentication insertion
+ * from end of the payload of this command (0-511 bytes)
+ */
+#define S_CPL_TX_SEC_PDU_AUTHINSERT 0
+#define M_CPL_TX_SEC_PDU_AUTHINSERT 0x1ff
+#define V_CPL_TX_SEC_PDU_AUTHINSERT(x) ((x) << S_CPL_TX_SEC_PDU_AUTHINSERT)
+#define G_CPL_TX_SEC_PDU_AUTHINSERT(x) \
+ (((x) >> S_CPL_TX_SEC_PDU_AUTHINSERT) & \
+ M_CPL_TX_SEC_PDU_AUTHINSERT)
+
+struct cpl_rx_phys_dsgl {
+ __be32 op_to_tid;
+ __be32 pcirlxorder_to_noofsgentr;
+ struct rss_header rss_hdr_int;
+};
+
+#define S_CPL_RX_PHYS_DSGL_OPCODE 24
+#define M_CPL_RX_PHYS_DSGL_OPCODE 0xff
+#define V_CPL_RX_PHYS_DSGL_OPCODE(x) ((x) << S_CPL_RX_PHYS_DSGL_OPCODE)
+#define G_CPL_RX_PHYS_DSGL_OPCODE(x) \
+ (((x) >> S_CPL_RX_PHYS_DSGL_OPCODE) & M_CPL_RX_PHYS_DSGL_OPCODE)
+
+#define S_CPL_RX_PHYS_DSGL_ISRDMA 23
+#define M_CPL_RX_PHYS_DSGL_ISRDMA 0x1
+#define V_CPL_RX_PHYS_DSGL_ISRDMA(x) ((x) << S_CPL_RX_PHYS_DSGL_ISRDMA)
+#define G_CPL_RX_PHYS_DSGL_ISRDMA(x) \
+ (((x) >> S_CPL_RX_PHYS_DSGL_ISRDMA) & M_CPL_RX_PHYS_DSGL_ISRDMA)
+#define F_CPL_RX_PHYS_DSGL_ISRDMA V_CPL_RX_PHYS_DSGL_ISRDMA(1U)
+
+#define S_CPL_RX_PHYS_DSGL_RSVD1 20
+#define M_CPL_RX_PHYS_DSGL_RSVD1 0x7
+#define V_CPL_RX_PHYS_DSGL_RSVD1(x) ((x) << S_CPL_RX_PHYS_DSGL_RSVD1)
+#define G_CPL_RX_PHYS_DSGL_RSVD1(x) \
+ (((x) >> S_CPL_RX_PHYS_DSGL_RSVD1) & \
+ M_CPL_RX_PHYS_DSGL_RSVD1)
+
+#define S_CPL_RX_PHYS_DSGL_PCIRLXORDER 31
+#define M_CPL_RX_PHYS_DSGL_PCIRLXORDER 0x1
+#define V_CPL_RX_PHYS_DSGL_PCIRLXORDER(x) \
+ ((x) << S_CPL_RX_PHYS_DSGL_PCIRLXORDER)
+#define G_CPL_RX_PHYS_DSGL_PCIRLXORDER(x) \
+ (((x) >> S_CPL_RX_PHYS_DSGL_PCIRLXORDER) & \
+ M_CPL_RX_PHYS_DSGL_PCIRLXORDER)
+#define F_CPL_RX_PHYS_DSGL_PCIRLXORDER V_CPL_RX_PHYS_DSGL_PCIRLXORDER(1U)
+
+#define S_CPL_RX_PHYS_DSGL_PCINOSNOOP 30
+#define M_CPL_RX_PHYS_DSGL_PCINOSNOOP 0x1
+#define V_CPL_RX_PHYS_DSGL_PCINOSNOOP(x) \
+ ((x) << S_CPL_RX_PHYS_DSGL_PCINOSNOOP)
+#define G_CPL_RX_PHYS_DSGL_PCINOSNOOP(x) \
+ (((x) >> S_CPL_RX_PHYS_DSGL_PCINOSNOOP) & \
+ M_CPL_RX_PHYS_DSGL_PCINOSNOOP)
+
+#define F_CPL_RX_PHYS_DSGL_PCINOSNOOP V_CPL_RX_PHYS_DSGL_PCINOSNOOP(1U)
+
+#define S_CPL_RX_PHYS_DSGL_PCITPHNTENB 29
+#define M_CPL_RX_PHYS_DSGL_PCITPHNTENB 0x1
+#define V_CPL_RX_PHYS_DSGL_PCITPHNTENB(x) \
+ ((x) << S_CPL_RX_PHYS_DSGL_PCITPHNTENB)
+#define G_CPL_RX_PHYS_DSGL_PCITPHNTENB(x) \
+ (((x) >> S_CPL_RX_PHYS_DSGL_PCITPHNTENB) & \
+ M_CPL_RX_PHYS_DSGL_PCITPHNTENB)
+#define F_CPL_RX_PHYS_DSGL_PCITPHNTENB V_CPL_RX_PHYS_DSGL_PCITPHNTENB(1U)
+
+#define S_CPL_RX_PHYS_DSGL_PCITPHNT 27
+#define M_CPL_RX_PHYS_DSGL_PCITPHNT 0x3
+#define V_CPL_RX_PHYS_DSGL_PCITPHNT(x) ((x) << S_CPL_RX_PHYS_DSGL_PCITPHNT)
+#define G_CPL_RX_PHYS_DSGL_PCITPHNT(x) \
+ (((x) >> S_CPL_RX_PHYS_DSGL_PCITPHNT) & \
+ M_CPL_RX_PHYS_DSGL_PCITPHNT)
+
+#define S_CPL_RX_PHYS_DSGL_DCAID 16
+#define M_CPL_RX_PHYS_DSGL_DCAID 0x7ff
+#define V_CPL_RX_PHYS_DSGL_DCAID(x) ((x) << S_CPL_RX_PHYS_DSGL_DCAID)
+#define G_CPL_RX_PHYS_DSGL_DCAID(x) \
+ (((x) >> S_CPL_RX_PHYS_DSGL_DCAID) & \
+ M_CPL_RX_PHYS_DSGL_DCAID)
+
+#define S_CPL_RX_PHYS_DSGL_NOOFSGENTR 0
+#define M_CPL_RX_PHYS_DSGL_NOOFSGENTR 0xffff
+#define V_CPL_RX_PHYS_DSGL_NOOFSGENTR(x) \
+ ((x) << S_CPL_RX_PHYS_DSGL_NOOFSGENTR)
+#define G_CPL_RX_PHYS_DSGL_NOOFSGENTR(x) \
+ (((x) >> S_CPL_RX_PHYS_DSGL_NOOFSGENTR) & \
+ M_CPL_RX_PHYS_DSGL_NOOFSGENTR)
+
#endif /* __T4_MSG_H */
diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h b/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h
index 392d664..d6c5819 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h
@@ -102,6 +102,7 @@ enum fw_wr_opcodes {
FW_RI_FR_NSMR_WR = 0x19,
FW_RI_INV_LSTAG_WR = 0x1a,
FW_ISCSI_TX_DATA_WR = 0x45,
+ FW_CRYPTO_LOOKASIDE_WR = 0X6d,
FW_LASTC2E_WR = 0x70
};
@@ -1060,6 +1061,7 @@ struct fw_caps_config_cmd {
__be16 niccaps;
__be16 ofldcaps;
__be16 rdmacaps;
+ __be16 cryptocaps;
__be16 r4;
__be16 iscsicaps;
__be16 fcoecaps;
@@ -3243,4 +3245,127 @@ struct fw_devlog_cmd {
#define PCIE_FW_PF_DEVLOG_MEMTYPE_G(x) \
(((x) >> PCIE_FW_PF_DEVLOG_MEMTYPE_S) & PCIE_FW_PF_DEVLOG_MEMTYPE_M)
+#define MAX_IMM_OFLD_TX_DATA_WR_LEN (0xff + sizeof(struct fw_ofld_tx_data_wr))
+
+struct fw_crypto_lookaside_wr {
+ __be32 op_to_cctx_size;
+ __be32 len16_pkd;
+ __be32 session_id;
+ __be32 rx_chid_to_rx_q_id;
+ __be32 key_addr;
+ __be32 pld_size_hash_size;
+ __be64 cookie;
+};
+
+#define S_FW_CRYPTO_LOOKASIDE_WR_OPCODE 24
+#define M_FW_CRYPTO_LOOKASIDE_WR_OPCODE 0xff
+#define V_FW_CRYPTO_LOOKASIDE_WR_OPCODE(x) \
+ ((x) << S_FW_CRYPTO_LOOKASIDE_WR_OPCODE)
+#define G_FW_CRYPTO_LOOKASIDE_WR_OPCODE(x) \
+ (((x) >> S_FW_CRYPTO_LOOKASIDE_WR_OPCODE) & \
+ M_FW_CRYPTO_LOOKASIDE_WR_OPCODE)
+
+#define S_FW_CRYPTO_LOOKASIDE_WR_COMPL 23
+#define M_FW_CRYPTO_LOOKASIDE_WR_COMPL 0x1
+#define V_FW_CRYPTO_LOOKASIDE_WR_COMPL(x) \
+ ((x) << S_FW_CRYPTO_LOOKASIDE_WR_COMPL)
+#define G_FW_CRYPTO_LOOKASIDE_WR_COMPL(x) \
+ (((x) >> S_FW_CRYPTO_LOOKASIDE_WR_COMPL) & \
+ M_FW_CRYPTO_LOOKASIDE_WR_COMPL)
+#define F_FW_CRYPTO_LOOKASIDE_WR_COMPL V_FW_CRYPTO_LOOKASIDE_WR_COMPL(1U)
+
+#define S_FW_CRYPTO_LOOKASIDE_WR_IMM_LEN 15
+#define M_FW_CRYPTO_LOOKASIDE_WR_IMM_LEN 0xff
+#define V_FW_CRYPTO_LOOKASIDE_WR_IMM_LEN(x) \
+ ((x) << S_FW_CRYPTO_LOOKASIDE_WR_IMM_LEN)
+#define G_FW_CRYPTO_LOOKASIDE_WR_IMM_LEN(x) \
+ (((x) >> S_FW_CRYPTO_LOOKASIDE_WR_IMM_LEN) & \
+ M_FW_CRYPTO_LOOKASIDE_WR_IMM_LEN)
+
+#define S_FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC 5
+#define M_FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC 0x3
+#define V_FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC(x) \
+ ((x) << S_FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC)
+#define G_FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC(x) \
+ (((x) >> S_FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC) & \
+ M_FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC)
+
+#define S_FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE 0
+#define M_FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE 0x1f
+#define V_FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE(x) \
+ ((x) << S_FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE)
+#define G_FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE(x) \
+ (((x) >> S_FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE) & \
+ M_FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE)
+
+#define S_FW_CRYPTO_LOOKASIDE_WR_LEN16 0
+#define M_FW_CRYPTO_LOOKASIDE_WR_LEN16 0xff
+#define V_FW_CRYPTO_LOOKASIDE_WR_LEN16(x) \
+ ((x) << S_FW_CRYPTO_LOOKASIDE_WR_LEN16)
+#define G_FW_CRYPTO_LOOKASIDE_WR_LEN16(x) \
+ (((x) >> S_FW_CRYPTO_LOOKASIDE_WR_LEN16) & \
+ M_FW_CRYPTO_LOOKASIDE_WR_LEN16)
+
+#define S_FW_CRYPTO_LOOKASIDE_WR_RX_CHID 29
+#define M_FW_CRYPTO_LOOKASIDE_WR_RX_CHID 0x3
+#define V_FW_CRYPTO_LOOKASIDE_WR_RX_CHID(x) \
+ ((x) << S_FW_CRYPTO_LOOKASIDE_WR_RX_CHID)
+#define G_FW_CRYPTO_LOOKASIDE_WR_RX_CHID(x) \
+ (((x) >> S_FW_CRYPTO_LOOKASIDE_WR_RX_CHID) & \
+ M_FW_CRYPTO_LOOKASIDE_WR_RX_CHID)
+
+#define S_FW_CRYPTO_LOOKASIDE_WR_LCB 27
+#define M_FW_CRYPTO_LOOKASIDE_WR_LCB 0x3
+#define V_FW_CRYPTO_LOOKASIDE_WR_LCB(x) \
+ ((x) << S_FW_CRYPTO_LOOKASIDE_WR_LCB)
+#define G_FW_CRYPTO_LOOKASIDE_WR_LCB(x) \
+ (((x) >> S_FW_CRYPTO_LOOKASIDE_WR_LCB) & M_FW_CRYPTO_LOOKASIDE_WR_LCB)
+
+#define S_FW_CRYPTO_LOOKASIDE_WR_PHASH 25
+#define M_FW_CRYPTO_LOOKASIDE_WR_PHASH 0x3
+#define V_FW_CRYPTO_LOOKASIDE_WR_PHASH(x) \
+ ((x) << S_FW_CRYPTO_LOOKASIDE_WR_PHASH)
+#define G_FW_CRYPTO_LOOKASIDE_WR_PHASH(x) \
+ (((x) >> S_FW_CRYPTO_LOOKASIDE_WR_PHASH) & \
+ M_FW_CRYPTO_LOOKASIDE_WR_PHASH)
+
+#define S_FW_CRYPTO_LOOKASIDE_WR_IV 23
+#define M_FW_CRYPTO_LOOKASIDE_WR_IV 0x3
+#define V_FW_CRYPTO_LOOKASIDE_WR_IV(x) \
+ ((x) << S_FW_CRYPTO_LOOKASIDE_WR_IV)
+#define G_FW_CRYPTO_LOOKASIDE_WR_IV(x) \
+ (((x) >> S_FW_CRYPTO_LOOKASIDE_WR_IV) & M_FW_CRYPTO_LOOKASIDE_WR_IV)
+
+#define S_FW_CRYPTO_LOOKASIDE_WR_TX_CH 10
+#define M_FW_CRYPTO_LOOKASIDE_WR_TX_CH 0x3
+#define V_FW_CRYPTO_LOOKASIDE_WR_TX_CH(x) \
+ ((x) << S_FW_CRYPTO_LOOKASIDE_WR_TX_CH)
+#define G_FW_CRYPTO_LOOKASIDE_WR_TX_CH(x) \
+ (((x) >> S_FW_CRYPTO_LOOKASIDE_WR_TX_CH) & \
+ M_FW_CRYPTO_LOOKASIDE_WR_TX_CH)
+
+#define S_FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID 0
+#define M_FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID 0x3ff
+#define V_FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID(x) \
+ ((x) << S_FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID)
+#define G_FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID(x) \
+ (((x) >> S_FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID) & \
+ M_FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID)
+
+#define S_FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE 24
+#define M_FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE 0xff
+#define V_FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE(x) \
+ ((x) << S_FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE)
+#define G_FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE(x) \
+ (((x) >> S_FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE) & \
+ M_FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE)
+
+#define S_FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE 17
+#define M_FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE 0x7f
+#define V_FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE(x) \
+ ((x) << S_FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE)
+#define G_FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE(x) \
+ (((x) >> S_FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE) & \
+ M_FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE)
+
#endif /* _T4FW_INTERFACE_H_ */
--
1.7.10.1
^ permalink raw reply related
* [PATCH 2/3] chcr: Support for Chelsio's Crypto Hardware
From: Yeshaswi M R Gowda @ 2016-07-11 18:28 UTC (permalink / raw)
To: hariprasad, netdev, linux-kernel, herbert, davem, linux-crypto,
jlulla, atul.gupta, harsh
Cc: Yeshaswi M R Gowda
In-Reply-To: <1468261688-24525-1-git-send-email-yeshaswi@chelsio.com>
The Chelsio's Crypto Hardware can perform the following operations:
SHA1, SHA224, SHA256, SHA384 and SHA512, HMAC(SHA1), HMAC(SHA224),
HMAC(SHA256), HMAC(SHA384), HAMC(SHA512), AES-128-CBC, AES-192-CBC,
AES-256-CBC, AES-128-XTS, AES-256-XTS
This patch implements the driver for above mentioned features. This
driver is an Upper Layer Driver which is attached to Chelsio's LLD
(cxgb4) and uses the queue allocated by the LLD for sending the crypto
requests to the Hardware and receiving the responses from it.
The crypto operations can be performed by Chelsio's hardware from the
userspace applications and/or from within the kernel space using the
kernel's crypto API.
The above mentioned crypto features have been tested using kernel's
tests mentioned in testmgr.h. They also have been tested from user
space using libkcapi and Openssl.
Signed-off-by: Yeshaswi M R Gowda <yeshaswi@chelsio.com>
---
drivers/crypto/chelsio/Kconfig | 19 +
drivers/crypto/chelsio/Makefile | 4 +
drivers/crypto/chelsio/chcr_algo.c | 1531 ++++++++++++++++++++++++++++++++++
drivers/crypto/chelsio/chcr_algo.h | 502 +++++++++++
drivers/crypto/chelsio/chcr_core.c | 273 ++++++
drivers/crypto/chelsio/chcr_core.h | 85 ++
drivers/crypto/chelsio/chcr_crypto.h | 255 ++++++
7 files changed, 2669 insertions(+)
create mode 100644 drivers/crypto/chelsio/Kconfig
create mode 100644 drivers/crypto/chelsio/Makefile
create mode 100644 drivers/crypto/chelsio/chcr_algo.c
create mode 100644 drivers/crypto/chelsio/chcr_algo.h
create mode 100644 drivers/crypto/chelsio/chcr_core.c
create mode 100644 drivers/crypto/chelsio/chcr_core.h
create mode 100644 drivers/crypto/chelsio/chcr_crypto.h
diff --git a/drivers/crypto/chelsio/Kconfig b/drivers/crypto/chelsio/Kconfig
new file mode 100644
index 0000000..5684a55
--- /dev/null
+++ b/drivers/crypto/chelsio/Kconfig
@@ -0,0 +1,19 @@
+config CRYPTO_DEV_CHELSIO
+ tristate "Chelsio Crypto Co-processor Driver"
+ select CHELSIO_T4
+ select CRYPTO_SHA1
+ select CRYPTO_SHA256
+ select CRYPTO_SHA512
+ ---help---
+ The Chelsio Crypto Co-processor driver for T6 adapters.
+
+ For general information about Chelsio and our products, visit
+ our website at <http://www.chelsio.com>.
+
+ For customer support, please visit our customer support page at
+ <http://www.chelsio.com/support.html>.
+
+ Please send feedback to <linux-bugs@chelsio.com>.
+
+ To compile this driver as a module, choose M here: the module
+ will be called chcr.
diff --git a/drivers/crypto/chelsio/Makefile b/drivers/crypto/chelsio/Makefile
new file mode 100644
index 0000000..7e4fda5
--- /dev/null
+++ b/drivers/crypto/chelsio/Makefile
@@ -0,0 +1,4 @@
+ ccflags-y := -Idrivers/net/ethernet/chelsio/cxgb4
+
+ obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chcr.o
+ chcr-objs := chcr_core.o chcr_algo.o
\ No newline at end of file
diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c
new file mode 100644
index 0000000..b070486
--- /dev/null
+++ b/drivers/crypto/chelsio/chcr_algo.c
@@ -0,0 +1,1531 @@
+/*
+ * This file is part of the Chelsio T6 Crypto driver for Linux.
+ *
+ * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Written and Maintained by:
+ * Manoj Malviya (manojmalviya@chelsio.com)
+ * Atul Gupta (atul.gupta@chelsio.com)
+ * Jitendra Lulla (jlulla@chelsio.com)
+ * Yeshaswi M R Gowda (yeshaswi@chelsio.com)
+ * Harsh Jain (harsh@chelsio.com)
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/crypto.h>
+#include <linux/cryptohash.h>
+#include <linux/skbuff.h>
+#include <linux/rtnetlink.h>
+#include <linux/highmem.h>
+#include <linux/scatterlist.h>
+
+#include <crypto/aes.h>
+#include <crypto/algapi.h>
+#include <crypto/hash.h>
+#include <crypto/sha.h>
+#include <crypto/authenc.h>
+#include <crypto/internal/aead.h>
+#include <crypto/internal/hash.h>
+#include <crypto/aead.h>
+#include <crypto/scatterwalk.h>
+
+#include "t4fw_api.h"
+#include "t4_msg.h"
+#include "chcr_core.h"
+#include "chcr_algo.h"
+#include "chcr_crypto.h"
+
+static inline struct aead_ctx *AEAD_CTX(struct chcr_context *ctx)
+{
+ return ctx->crypto_ctx->aeadctx;
+}
+
+static inline struct ablk_ctx *ABLK_CTX(struct chcr_context *ctx)
+{
+ return ctx->crypto_ctx->ablkctx;
+}
+
+static inline struct hmac_ctx *HMAC_CTX(struct chcr_context *ctx)
+{
+ return ctx->crypto_ctx->hmacctx;
+}
+
+static inline struct uld_ctx *ULD_CTX(struct chcr_context *ctx)
+{
+ return ctx->dev->u_ctx;
+}
+
+static inline int is_ofld_imm(const struct sk_buff *skb)
+{
+ return skb->len <= MAX_IMM_OFLD_TX_DATA_WR_LEN;
+}
+
+/*
+ * sgl_len - calculates the size of an SGL of the given capacity
+ * @n: the number of SGL entries
+ * Calculates the number of flits needed for a scatter/gather list that
+ * can hold the given number of entries.
+ */
+static inline unsigned int sgl_len(unsigned int n)
+{
+ n--;
+ return (3 * n) / 2 + (n & 1) + 2;
+}
+
+/*
+ * chcr_handle_resp - Unmap the DMA buffers associated with the request
+ * @req: crypto request
+ */
+int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input,
+ int error_status)
+{
+ struct crypto_tfm *tfm = req->tfm;
+ struct chcr_context *ctx = crypto_tfm_ctx(tfm);
+ struct uld_ctx *u_ctx = ULD_CTX(ctx);
+ struct chcr_req_ctx ctx_req;
+ struct cpl_fw6_pld *fw6_pld;
+ unsigned int digestsize, updated_digestsize;
+
+ switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) {
+ case CRYPTO_ALG_TYPE_AEAD:
+ ctx_req.req.aead_req = (struct aead_request *)req;
+ ctx_req.ctx.aead_ctx =
+ aead_request_ctx((struct aead_request *)req);
+ dma_unmap_sg(&u_ctx->lldi.pdev->dev, ctx_req.req.aead_req->dst,
+ AEAD_CTX(ctx)->dst_nents, DMA_FROM_DEVICE);
+ if (ctx_req.ctx.aead_ctx->skb) {
+ kfree_skb(ctx_req.ctx.aead_ctx->skb);
+ ctx_req.ctx.aead_ctx->skb = NULL;
+ }
+ break;
+
+ case CRYPTO_ALG_TYPE_BLKCIPHER:
+ ctx_req.req.ablk_req = (struct ablkcipher_request *)req;
+ ctx_req.ctx.ablk_ctx =
+ ablkcipher_request_ctx(ctx_req.req.ablk_req);
+ if (error_status)
+ goto dma_unmap_blkcipher;
+ fw6_pld = (struct cpl_fw6_pld *)input;
+ memcpy(ctx_req.req.ablk_req->info, &fw6_pld->data[2],
+ AES_BLOCK_SIZE);
+dma_unmap_blkcipher:
+ dma_unmap_sg(&u_ctx->lldi.pdev->dev, ctx_req.req.ablk_req->dst,
+ ABLK_CTX(ctx)->dst_nents, DMA_FROM_DEVICE);
+ if (ctx_req.ctx.ablk_ctx->skb) {
+ kfree_skb(ctx_req.ctx.ablk_ctx->skb);
+ ctx_req.ctx.ablk_ctx->skb = NULL;
+ }
+ break;
+
+ case CRYPTO_ALG_TYPE_AHASH:
+ ctx_req.req.ahash_req = (struct ahash_request *)req;
+ ctx_req.ctx.ahash_ctx =
+ ahash_request_ctx(ctx_req.req.ahash_req);
+ digestsize =
+ crypto_ahash_digestsize(crypto_ahash_reqtfm(
+ ctx_req.req.ahash_req));
+ updated_digestsize = digestsize;
+ if (digestsize == SHA224_DIGEST_SIZE)
+ updated_digestsize = SHA256_DIGEST_SIZE;
+ else if (digestsize == SHA384_DIGEST_SIZE)
+ updated_digestsize = SHA512_DIGEST_SIZE;
+ if (ctx_req.ctx.ahash_ctx->skb)
+ ctx_req.ctx.ahash_ctx->skb = NULL;
+ if (ctx_req.ctx.ahash_ctx->result == 1) {
+ ctx_req.ctx.ahash_ctx->result = 0;
+ memcpy(ctx_req.req.ahash_req->result, input +
+ sizeof(struct cpl_fw6_pld),
+ digestsize);
+ } else {
+ memcpy(ctx_req.ctx.ahash_ctx->partial_hash, input +
+ sizeof(struct cpl_fw6_pld),
+ updated_digestsize);
+ }
+ kfree(ctx_req.ctx.ahash_ctx->dummy_payload_ptr);
+ ctx_req.ctx.ahash_ctx->dummy_payload_ptr = NULL;
+ break;
+ }
+ return 0;
+}
+
+/*
+ * calc_tx_flits_ofld - calculate # of flits for an offload packet
+ * @skb: the packet
+ * Returns the number of flits needed for the given offload packet.
+ * These packets are already fully constructed and no additional headers
+ * will be added.
+ */
+static inline unsigned int calc_tx_flits_ofld(const struct sk_buff *skb)
+{
+ unsigned int flits, cnt;
+
+ if (is_ofld_imm(skb))
+ return DIV_ROUND_UP(skb->len, 8);
+
+ flits = skb_transport_offset(skb) / 8; /* headers */
+ cnt = skb_shinfo(skb)->nr_frags;
+ if (skb_tail_pointer(skb) != skb_transport_header(skb))
+ cnt++;
+ return flits + sgl_len(cnt);
+}
+
+static struct shash_desc *chcr_alloc_shash(unsigned int ds)
+{
+ struct crypto_shash *base_hash = NULL;
+ struct shash_desc *desc;
+
+ switch (ds) {
+ case SHA1_DIGEST_SIZE:
+ base_hash = crypto_alloc_shash("sha1-generic", 0, 0);
+ break;
+ case SHA224_DIGEST_SIZE:
+ base_hash = crypto_alloc_shash("sha224-generic", 0, 0);
+ break;
+ case SHA256_DIGEST_SIZE:
+ base_hash = crypto_alloc_shash("sha256-generic", 0, 0);
+ break;
+ case SHA384_DIGEST_SIZE:
+ base_hash = crypto_alloc_shash("sha384-generic", 0, 0);
+ break;
+ case SHA512_DIGEST_SIZE:
+ base_hash = crypto_alloc_shash("sha512-generic", 0, 0);
+ break;
+ }
+ if (IS_ERR(base_hash)) {
+ pr_err("Can not allocate sha-generic algo.\n");
+ return (void *)base_hash;
+ }
+
+ desc = kmalloc(sizeof(*desc) + crypto_shash_descsize(base_hash),
+ GFP_KERNEL);
+ if (!desc)
+ return ERR_PTR(-ENOMEM);
+ desc->tfm = base_hash;
+ desc->flags = crypto_shash_get_flags(base_hash);
+ return desc;
+}
+
+static int chcr_compute_partial_hash(struct shash_desc *desc,
+ char *iopad, char *result_hash,
+ int digest_size)
+{
+ struct sha1_state sha1_st;
+ struct sha256_state sha256_st;
+ struct sha512_state sha512_st;
+ int error;
+
+ if (digest_size == SHA1_DIGEST_SIZE) {
+ error = crypto_shash_init(desc) ?:
+ crypto_shash_update(desc, iopad, SHA1_BLOCK_SIZE) ?:
+ crypto_shash_export(desc, (void *)&sha1_st);
+ memcpy(result_hash, sha1_st.state, SHA1_DIGEST_SIZE);
+ } else if (digest_size == SHA224_DIGEST_SIZE) {
+ error = crypto_shash_init(desc) ?:
+ crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?:
+ crypto_shash_export(desc, (void *)&sha256_st);
+ memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE);
+
+ } else if (digest_size == SHA256_DIGEST_SIZE) {
+ error = crypto_shash_init(desc) ?:
+ crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?:
+ crypto_shash_export(desc, (void *)&sha256_st);
+ memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE);
+
+ } else if (digest_size == SHA384_DIGEST_SIZE) {
+ error = crypto_shash_init(desc) ?:
+ crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?:
+ crypto_shash_export(desc, (void *)&sha512_st);
+ memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE);
+
+ } else if (digest_size == SHA512_DIGEST_SIZE) {
+ error = crypto_shash_init(desc) ?:
+ crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?:
+ crypto_shash_export(desc, (void *)&sha512_st);
+ memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE);
+ } else {
+ error = -EINVAL;
+ pr_err("Unknown digest size %d\n", digest_size);
+ }
+ return error;
+}
+
+static void chcr_change_order(char *buf, int ds)
+{
+ int i;
+
+ if (ds == SHA512_DIGEST_SIZE) {
+ for (i = 0; i < (ds / sizeof(u64)); i++)
+ *((__be64 *)buf + i) =
+ cpu_to_be64(*((u64 *)buf + i));
+ } else {
+ for (i = 0; i < (ds / sizeof(u32)); i++)
+ *((__be32 *)buf + i) =
+ cpu_to_be32(*((u32 *)buf + i));
+ }
+}
+
+static inline int is_hmac(struct crypto_tfm *tfm)
+{
+ struct crypto_alg *alg = tfm->__crt_alg;
+ struct chcr_alg_template *chcr_crypto_alg =
+ container_of(__crypto_ahash_alg(alg), struct chcr_alg_template,
+ alg.hash);
+ if ((chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK) ==
+ CRYPTO_ALG_SUB_TYPE_HASH_HMAC)
+ return 1;
+ return 0;
+}
+
+static inline unsigned int ch_nents(struct scatterlist *sg,
+ unsigned int *total_size)
+{
+ unsigned int nents;
+
+ for (nents = 0, *total_size = 0; sg; sg = sg_next(sg)) {
+ nents++;
+ *total_size += sg->length;
+ }
+ return nents;
+}
+
+static void write_phys_cpl(struct cpl_rx_phys_dsgl *phys_cpl,
+ struct scatterlist *sg,
+ struct phys_sge_parm *sg_param)
+{
+ struct phys_sge_pairs *to;
+ unsigned int out_buf_size = sg_param->obsize;
+ unsigned int nents = sg_param->nents, i, j, tot_len = 0;
+
+ phys_cpl->op_to_tid = htonl(V_CPL_RX_PHYS_DSGL_OPCODE(CPL_RX_PHYS_DSGL)
+ | V_CPL_RX_PHYS_DSGL_ISRDMA(0));
+ phys_cpl->pcirlxorder_to_noofsgentr =
+ htonl(V_CPL_RX_PHYS_DSGL_PCIRLXORDER(0) |
+ V_CPL_RX_PHYS_DSGL_PCINOSNOOP(0) |
+ V_CPL_RX_PHYS_DSGL_PCITPHNTENB(0) |
+ V_CPL_RX_PHYS_DSGL_PCITPHNT(0) |
+ V_CPL_RX_PHYS_DSGL_DCAID(0) |
+ V_CPL_RX_PHYS_DSGL_NOOFSGENTR(nents));
+ phys_cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR;
+ phys_cpl->rss_hdr_int.qid = htons(sg_param->qid);
+ phys_cpl->rss_hdr_int.hash_val = 0;
+ to = (struct phys_sge_pairs *)((unsigned char *)phys_cpl +
+ sizeof(struct cpl_rx_phys_dsgl));
+
+ for (i = 0; nents; to++) {
+ for (j = i; (nents && (j < (8 + i))); j++, nents--) {
+ to->len[j] = htons(sg->length);
+ to->addr[j] = cpu_to_be64(sg_dma_address(sg));
+ if (out_buf_size) {
+ if (tot_len + sg_dma_len(sg) >= out_buf_size) {
+ to->len[j] = htons(out_buf_size -
+ tot_len);
+ return;
+ }
+ tot_len += sg_dma_len(sg);
+ }
+ sg = sg_next(sg);
+ }
+ }
+}
+
+static inline unsigned
+int map_writesg_phys_cpl(struct device *dev, struct cpl_rx_phys_dsgl *phys_cpl,
+ struct scatterlist *sg, struct phys_sge_parm *sg_param)
+{
+ if (!sg || !sg_param->nents)
+ return 0;
+
+ sg_param->nents = dma_map_sg(dev, sg, sg_param->nents, DMA_FROM_DEVICE);
+ if (sg_param->nents == 0) {
+ pr_err("CHCR : DMA mapping failed\n");
+ return -EINVAL;
+ }
+ write_phys_cpl(phys_cpl, sg, sg_param);
+ return 0;
+}
+
+static inline int get_cryptoalg_subtype(struct crypto_tfm *tfm)
+{
+ struct crypto_alg *alg = tfm->__crt_alg;
+ struct chcr_alg_template *chcr_crypto_alg =
+ container_of(alg, struct chcr_alg_template, alg.crypto);
+
+ return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK;
+}
+
+static inline int
+write_sg_data_page_desc(struct sk_buff *skb, unsigned int *frags,
+ struct scatterlist *sg, unsigned int count)
+{
+ struct page *spage;
+ unsigned int page_len;
+
+ skb->len += count;
+ skb->data_len += count;
+ skb->truesize += count;
+ while (count > 0) {
+ if (sg && !sg->length)
+ break;
+ spage = sg_page(sg);
+ get_page(spage);
+ page_len = min(sg->length, count);
+ skb_fill_page_desc(skb, *frags, spage, sg->offset, page_len);
+ (*frags)++;
+ count -= page_len;
+ sg = sg_next(sg);
+ }
+ return 0;
+}
+
+static int generate_copy_rrkey(struct ablk_ctx *ablkctx,
+ struct _key_ctx *key_ctx)
+{
+ if (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) {
+ get_aes_decrypt_key(key_ctx->key, ablkctx->key,
+ ablkctx->enckey_len << 3);
+ memset(key_ctx->key + ablkctx->enckey_len, 0,
+ CHCR_AES_MAX_KEY_LEN - ablkctx->enckey_len);
+ } else {
+ memcpy(key_ctx->key,
+ ablkctx->key + (ablkctx->enckey_len >> 1),
+ ablkctx->enckey_len >> 1);
+ get_aes_decrypt_key(key_ctx->key + (ablkctx->enckey_len >> 1),
+ ablkctx->key, ablkctx->enckey_len << 2);
+ }
+ return 0;
+}
+
+static inline void create_wreq(struct chcr_context *ctx,
+ struct fw_crypto_lookaside_wr *wreq,
+ void *req, struct sk_buff *skb,
+ int kctx_len, int hash_sz,
+ unsigned int phys_dsgl)
+{
+ struct uld_ctx *u_ctx = ULD_CTX(ctx);
+ struct ulp_txpkt *ulptx = (struct ulp_txpkt *)(wreq + 1);
+ struct ulptx_idata *sc_imm = (struct ulptx_idata *)(ulptx + 1);
+ int iv_loc = IV_DSGL;
+ int qid = u_ctx->lldi.rxq_ids[ctx->tx_channel_id];
+ unsigned int immdatalen = 0, nr_frags = 0;
+
+ if (is_ofld_imm(skb)) {
+ immdatalen = skb->data_len;
+ iv_loc = IV_IMMEDIATE;
+ } else {
+ nr_frags = skb_shinfo(skb)->nr_frags;
+ }
+
+ wreq->op_to_cctx_size = FILL_WR_OP_CCTX_SIZE(immdatalen,
+ (kctx_len >> 4));
+ wreq->pld_size_hash_size =
+ htonl(V_FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE(sgl_lengths[nr_frags]) |
+ V_FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE(hash_sz));
+ wreq->len16_pkd = htonl(V_FW_CRYPTO_LOOKASIDE_WR_LEN16(DIV_ROUND_UP(
+ (calc_tx_flits_ofld(skb) * 8), 16)));
+ wreq->cookie = cpu_to_be64((u64)req);
+ wreq->rx_chid_to_rx_q_id =
+ FILL_WR_RX_Q_ID(ctx->dev->tx_channel_id, qid,
+ (hash_sz) ? IV_NOP : iv_loc);
+
+ ulptx->cmd_dest = FILL_ULPTX_CMD_DEST(ctx->dev->tx_channel_id);
+ ulptx->len = htonl((DIV_ROUND_UP((calc_tx_flits_ofld(skb) * 8),
+ 16) - ((sizeof(*wreq)) >> 4)));
+
+ sc_imm->cmd_more = FILL_CMD_MORE(immdatalen);
+ sc_imm->len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) + kctx_len +
+ ((hash_sz) ? DUMMY_BYTES :
+ (sizeof(struct cpl_rx_phys_dsgl) +
+ phys_dsgl)) + immdatalen);
+}
+
+/**
+ * create_cipher_wr - form the WR for cipher operations
+ * @req: cipher req.
+ * @ctx: crypto driver context of the request.
+ * @qid: ingress qid where response of this WR should be received.
+ * @op_type: encryption or decryption
+ */
+static struct sk_buff
+*create_cipher_wr(struct crypto_async_request *req_base,
+ struct chcr_context *ctx, unsigned short qid,
+ unsigned short op_type)
+{
+ struct ablkcipher_request *req = (struct ablkcipher_request *)req_base;
+ struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
+ struct uld_ctx *u_ctx = ULD_CTX(ctx);
+ struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
+ struct sk_buff *skb = NULL;
+ struct _key_ctx *key_ctx;
+ struct fw_crypto_lookaside_wr *wreq;
+ struct cpl_tx_sec_pdu *sec_cpl;
+ struct cpl_rx_phys_dsgl *phys_cpl;
+ struct chcr_blkcipher_req_ctx *req_ctx = ablkcipher_request_ctx(req);
+ struct phys_sge_parm sg_param;
+ unsigned int frags = 0, transhdr_len, phys_dsgl, dst_bufsize = 0;
+ unsigned int ivsize = crypto_ablkcipher_ivsize(tfm), kctx_len;
+
+ if (!req->info)
+ return ERR_PTR(-EINVAL);
+ ablkctx->dst_nents = ch_nents(req->dst, &dst_bufsize);
+ ablkctx->enc = op_type;
+
+ if ((ablkctx->enckey_len == 0) || (ivsize > AES_BLOCK_SIZE) ||
+ (req->nbytes <= 0) || (req->nbytes % AES_BLOCK_SIZE))
+ return ERR_PTR(-EINVAL);
+
+ phys_dsgl = get_space_for_phys_dsgl(ablkctx->dst_nents);
+
+ kctx_len = sizeof(*key_ctx) +
+ (DIV_ROUND_UP(ablkctx->enckey_len, 16) * 16);
+ transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl);
+ skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)),
+ GFP_ATOMIC);
+ if (!skb)
+ return ERR_PTR(-ENOMEM);
+ skb_reserve(skb, sizeof(struct sge_opaque_hdr));
+ wreq = (struct fw_crypto_lookaside_wr *)__skb_put(skb, transhdr_len);
+
+ sec_cpl = (struct cpl_tx_sec_pdu *)((u8 *)wreq + SEC_CPL_OFFSET);
+ sec_cpl->op_ivinsrtofst =
+ FILL_SEC_CPL_OP_IVINSR(ctx->dev->tx_channel_id, 2, 1, 1);
+
+ sec_cpl->pldlen = htonl(ivsize + req->nbytes);
+ sec_cpl->aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(0, 0,
+ ivsize + 1, 0);
+
+ sec_cpl->cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(0, 0,
+ 0, 0);
+ sec_cpl->seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, 0,
+ ablkctx->ciph_mode,
+ 0, 0, ivsize >> 1, 1);
+ sec_cpl->ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 0,
+ 0, 1, phys_dsgl);
+
+ key_ctx = (struct _key_ctx *)((u8 *)sec_cpl + sizeof(*sec_cpl));
+ key_ctx->ctx_hdr = ablkctx->key_ctx_hdr;
+ if ((op_type == CHCR_DECRYPT_OP) &&
+ (!(get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
+ CRYPTO_ALG_SUB_TYPE_CTR))) {
+ if (generate_copy_rrkey(ablkctx, key_ctx))
+ goto map_fail1;
+ } else {
+ if ((ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) ||
+ (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CTR)) {
+ memcpy(key_ctx->key, ablkctx->key, ablkctx->enckey_len);
+ } else {
+ memcpy(key_ctx->key, ablkctx->key +
+ (ablkctx->enckey_len >> 1),
+ ablkctx->enckey_len >> 1);
+ memcpy(key_ctx->key +
+ (ablkctx->enckey_len >> 1),
+ ablkctx->key,
+ ablkctx->enckey_len >> 1);
+ }
+ }
+ phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)key_ctx + kctx_len);
+
+ memcpy(ablkctx->iv, req->info, ivsize);
+ sg_init_table(&ablkctx->iv_sg, 1);
+ sg_set_buf(&ablkctx->iv_sg, ablkctx->iv, ivsize);
+ sg_param.nents = ablkctx->dst_nents;
+ sg_param.obsize = dst_bufsize;
+ sg_param.qid = qid;
+ sg_param.align = 1;
+ if (map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, req->dst,
+ &sg_param))
+ goto map_fail1;
+
+ skb_set_transport_header(skb, transhdr_len);
+ write_sg_data_page_desc(skb, &frags, &ablkctx->iv_sg, ivsize);
+ write_sg_data_page_desc(skb, &frags, req->src, req->nbytes);
+ create_wreq(ctx, wreq, req, skb, kctx_len, 0, phys_dsgl);
+ req_ctx->skb = skb;
+ skb_get(skb);
+ return skb;
+map_fail1:
+ kfree_skb(skb);
+ return ERR_PTR(-ENOMEM);
+}
+
+static int chcr_aes_cbc_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
+ struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
+ struct ablkcipher_alg *alg = crypto_ablkcipher_alg(tfm);
+ unsigned int ck_size, context_size;
+ u16 alignment = 0;
+
+ if ((keylen < alg->min_keysize) || (keylen > alg->max_keysize))
+ goto badkey_err;
+
+ memcpy(ablkctx->key, key, keylen);
+ ablkctx->enckey_len = keylen;
+ if (keylen == AES_KEYSIZE_128) {
+ ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
+ } else if (keylen == AES_KEYSIZE_192) {
+ alignment = 8;
+ ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
+ } else if (keylen == AES_KEYSIZE_256) {
+ ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
+ } else {
+ goto badkey_err;
+ }
+
+ context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
+ keylen + alignment) >> 4;
+
+ ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
+ 0, 0, context_size);
+ if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
+ CRYPTO_ALG_SUB_TYPE_CTR)
+ ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR;
+ else
+ ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CBC;
+ return 0;
+badkey_err:
+ crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ ablkctx->enckey_len = 0;
+ return -EINVAL;
+}
+
+static int chcr_aes_encrypt(struct ablkcipher_request *req)
+{
+ struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
+ struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
+ struct crypto_async_request *req_base = &req->base;
+ struct uld_ctx *u_ctx = ULD_CTX(ctx);
+ struct sk_buff *skb;
+
+ if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], ctx->tx_channel_id))
+ return -EBUSY;
+
+ skb = create_cipher_wr(req_base, ctx,
+ u_ctx->lldi.rxq_ids[ctx->tx_channel_id],
+ CHCR_ENCRYPT_OP);
+ if (IS_ERR(skb)) {
+ pr_err("chcr : %s : Failed to form WR. No memory\n", __func__);
+ return PTR_ERR(skb);
+ }
+ skb->dev = u_ctx->lldi.ports[0];
+ set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id);
+ chcr_send_wr(skb);
+ return -EINPROGRESS;
+}
+
+static int chcr_aes_decrypt(struct ablkcipher_request *req)
+{
+ struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
+ struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
+ struct crypto_async_request *req_base = &req->base;
+ struct uld_ctx *u_ctx = ULD_CTX(ctx);
+ struct sk_buff *skb;
+
+ if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], ctx->tx_channel_id))
+ return -EBUSY;
+
+ skb = create_cipher_wr(req_base, ctx, u_ctx->lldi.rxq_ids[0],
+ CHCR_DECRYPT_OP);
+ if (IS_ERR(skb)) {
+ pr_err("chcr : %s : Failed to form WR. No memory\n", __func__);
+ return PTR_ERR(skb);
+ }
+ skb->dev = u_ctx->lldi.ports[0];
+ set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id);
+ chcr_send_wr(skb);
+ return -EINPROGRESS;
+}
+
+static int chcr_device_init(struct chcr_context *ctx)
+{
+ struct uld_ctx *u_ctx;
+ unsigned int id;
+ int err = 0, rxq_perchan, rxq_idx;
+
+ id = smp_processor_id();
+ if (!ctx->dev) {
+ err = assign_chcr_device(&ctx->dev);
+ if (err) {
+ pr_err("chcr device assignment fails\n");
+ goto out;
+ }
+ u_ctx = ULD_CTX(ctx);
+ rxq_perchan = u_ctx->lldi.nrxq / u_ctx->lldi.nchan;
+ ctx->dev->tx_channel_id = 0;
+ rxq_idx = ctx->dev->tx_channel_id * rxq_perchan;
+ rxq_idx += id % rxq_perchan;
+ spin_lock(&ctx->dev->lock_chcr_dev);
+ ctx->tx_channel_id = rxq_idx;
+ spin_unlock(&ctx->dev->lock_chcr_dev);
+ }
+out:
+ return err;
+}
+
+static int chcr_cra_init(struct crypto_tfm *tfm)
+{
+ tfm->crt_ablkcipher.reqsize = sizeof(struct chcr_blkcipher_req_ctx);
+ return chcr_device_init(crypto_tfm_ctx(tfm));
+}
+
+static int get_alg_config(struct algo_param *params,
+ unsigned int auth_size)
+{
+ switch (auth_size) {
+ case SHA1_DIGEST_SIZE:
+ params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160;
+ params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA1;
+ params->result_size = SHA1_DIGEST_SIZE;
+ break;
+ case SHA224_DIGEST_SIZE:
+ params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
+ params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA224;
+ params->result_size = SHA256_DIGEST_SIZE;
+ break;
+ case SHA256_DIGEST_SIZE:
+ params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
+ params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA256;
+ params->result_size = SHA256_DIGEST_SIZE;
+ break;
+ case SHA384_DIGEST_SIZE:
+ params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
+ params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_384;
+ params->result_size = SHA512_DIGEST_SIZE;
+ break;
+ case SHA512_DIGEST_SIZE:
+ params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
+ params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_512;
+ params->result_size = SHA512_DIGEST_SIZE;
+ break;
+ default:
+ pr_err("chcr : ERROR, unsupported digest size\n");
+ return -EINVAL;
+ }
+ return 0;
+}
+
+static inline int
+write_buffer_data_page_desc(struct chcr_ahash_req_ctx *req_ctx,
+ struct sk_buff *skb, unsigned int *frags, char *bfr,
+ u8 bfr_len)
+{
+ void *page_ptr = NULL;
+
+ skb->len += bfr_len;
+ skb->data_len += bfr_len;
+ skb->truesize += bfr_len;
+ page_ptr = kmalloc(CHCR_HASH_MAX_BLOCK_SIZE_128, GFP_ATOMIC | GFP_DMA);
+ if (!page_ptr)
+ return -ENOMEM;
+ get_page(virt_to_page(page_ptr));
+ req_ctx->dummy_payload_ptr = page_ptr;
+ memcpy(page_ptr, bfr, bfr_len);
+ skb_fill_page_desc(skb, *frags, virt_to_page(page_ptr),
+ offset_in_page(page_ptr), bfr_len);
+ (*frags)++;
+ return 0;
+}
+
+/**
+ * create_final_hash_wr - Create hash work request
+ * @req - Cipher req base
+ */
+static struct sk_buff *create_final_hash_wr(struct ahash_request *req,
+ struct hash_wr_param *param)
+{
+ struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
+ struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
+ struct sk_buff *skb = NULL;
+ struct _key_ctx *key_ctx;
+ struct fw_crypto_lookaside_wr *wreq;
+ struct cpl_tx_sec_pdu *sec_cpl;
+ unsigned int frags = 0, transhdr_len, iopad_alignment = 0;
+ unsigned int digestsize = crypto_ahash_digestsize(tfm);
+ unsigned int kctx_len = sizeof(*key_ctx);
+ u8 hash_size_in_response = 0;
+
+ iopad_alignment = KEYCTX_ALIGN_PAD(digestsize);
+ kctx_len += param->alg_prm.result_size + iopad_alignment;
+ if (param->opad_needed)
+ kctx_len += param->alg_prm.result_size + iopad_alignment;
+
+ if (req_ctx->result)
+ hash_size_in_response = digestsize;
+ else
+ hash_size_in_response = param->alg_prm.result_size;
+ transhdr_len = HASH_TRANSHDR_SIZE(kctx_len);
+ skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)),
+ GFP_ATOMIC);
+ if (!skb)
+ return skb;
+
+ skb_reserve(skb, sizeof(struct sge_opaque_hdr));
+ wreq = (struct fw_crypto_lookaside_wr *)__skb_put(skb, transhdr_len);
+ memset(wreq, 0, transhdr_len);
+
+ sec_cpl = (struct cpl_tx_sec_pdu *)((u8 *)wreq + SEC_CPL_OFFSET);
+ sec_cpl->op_ivinsrtofst =
+ FILL_SEC_CPL_OP_IVINSR(ctx->dev->tx_channel_id, 2, 0, 0);
+ sec_cpl->pldlen = htonl(param->bfr_len + param->sg_len);
+
+ sec_cpl->aadstart_cipherstop_hi =
+ FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, 0, 0);
+ sec_cpl->cipherstop_lo_authinsert =
+ FILL_SEC_CPL_AUTHINSERT(0, 1, 0, 0);
+ sec_cpl->seqno_numivs =
+ FILL_SEC_CPL_SCMD0_SEQNO(0, 0, 0, param->alg_prm.auth_mode,
+ param->opad_needed, 0, 0);
+
+ sec_cpl->ivgen_hdrlen =
+ FILL_SEC_CPL_IVGEN_HDRLEN(param->last, param->more, 0, 1, 0, 0);
+
+ key_ctx = (struct _key_ctx *)((u8 *)sec_cpl + sizeof(*sec_cpl));
+ memcpy(key_ctx->key, req_ctx->partial_hash, param->alg_prm.result_size);
+
+ if (param->opad_needed)
+ memcpy(key_ctx->key + ((param->alg_prm.result_size <= 32) ? 32 :
+ CHCR_HASH_MAX_DIGEST_SIZE),
+ hmacctx->opad, param->alg_prm.result_size);
+
+ key_ctx->ctx_hdr = FILL_KEY_CTX_HDR(CHCR_KEYCTX_NO_KEY,
+ param->alg_prm.mk_size, 0,
+ param->opad_needed,
+ (kctx_len >> 4));
+ sec_cpl->scmd1 = cpu_to_be64((u64)param->scmd1);
+
+ skb_set_transport_header(skb, transhdr_len);
+ if (param->bfr_len != 0)
+ write_buffer_data_page_desc(req_ctx, skb, &frags, req_ctx->bfr,
+ param->bfr_len);
+ if (param->sg_len != 0)
+ write_sg_data_page_desc(skb, &frags, req->src, param->sg_len);
+
+ create_wreq(ctx, wreq, req, skb, kctx_len, hash_size_in_response,
+ 0);
+ req_ctx->skb = skb;
+ skb_get(skb);
+ return skb;
+}
+
+static int chcr_ahash_update(struct ahash_request *req)
+{
+ struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
+ struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
+ struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
+ struct uld_ctx *u_ctx = NULL;
+ struct sk_buff *skb;
+ u8 remainder = 0, bs;
+ unsigned int nbytes = req->nbytes;
+ struct hash_wr_param params;
+
+ bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
+
+ u_ctx = ULD_CTX(ctx);
+ if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], ctx->tx_channel_id))
+ return -EBUSY;
+
+ if (nbytes + req_ctx->bfr_len >= bs) {
+ remainder = (nbytes + req_ctx->bfr_len) % bs;
+ nbytes = nbytes + req_ctx->bfr_len - remainder;
+ } else {
+ sg_pcopy_to_buffer(req->src, sg_nents(req->src), req_ctx->bfr +
+ req_ctx->bfr_len, nbytes, 0);
+ req_ctx->bfr_len += nbytes;
+ return 0;
+ }
+
+ params.opad_needed = 0;
+ params.more = 1;
+ params.last = 0;
+ params.sg_len = nbytes - req_ctx->bfr_len;
+ params.bfr_len = req_ctx->bfr_len;
+ params.scmd1 = 0;
+ get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm));
+ req_ctx->result = 0;
+ req_ctx->data_len += params.sg_len + params.bfr_len;
+ skb = create_final_hash_wr(req, ¶ms);
+ if (!skb)
+ return -ENOMEM;
+
+ req_ctx->bfr_len = remainder;
+ if (remainder)
+ sg_pcopy_to_buffer(req->src, sg_nents(req->src),
+ req_ctx->bfr, remainder, req->nbytes -
+ remainder);
+ skb->dev = u_ctx->lldi.ports[0];
+ set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id);
+ chcr_send_wr(skb);
+
+ return -EINPROGRESS;
+}
+
+static void create_last_hash_block(char *bfr_ptr, unsigned int bs, u64 scmd1)
+{
+ memset(bfr_ptr, 0, bs);
+ *bfr_ptr = 0x80;
+ if (bs == 64)
+ *(__be64 *)(bfr_ptr + 56) = cpu_to_be64(scmd1 << 3);
+ else
+ *(__be64 *)(bfr_ptr + 120) = cpu_to_be64(scmd1 << 3);
+}
+
+static int chcr_ahash_final(struct ahash_request *req)
+{
+ struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
+ struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
+ struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
+ struct hash_wr_param params;
+ struct sk_buff *skb;
+ struct uld_ctx *u_ctx = NULL;
+ u8 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
+
+ u_ctx = ULD_CTX(ctx);
+ if (is_hmac(crypto_ahash_tfm(rtfm)))
+ params.opad_needed = 1;
+ else
+ params.opad_needed = 0;
+ params.sg_len = 0;
+ get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm));
+ req_ctx->result = 1;
+ params.bfr_len = req_ctx->bfr_len;
+ req_ctx->data_len += params.bfr_len + params.sg_len;
+ if (req_ctx->bfr && (req_ctx->bfr_len == 0)) {
+ create_last_hash_block(req_ctx->bfr, bs, req_ctx->data_len);
+ params.last = 0;
+ params.more = 1;
+ params.scmd1 = 0;
+ params.bfr_len = bs;
+
+ } else {
+ params.scmd1 = req_ctx->data_len;
+ params.last = 1;
+ params.more = 0;
+ }
+ skb = create_final_hash_wr(req, ¶ms);
+ skb->dev = u_ctx->lldi.ports[0];
+ set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id);
+ chcr_send_wr(skb);
+ return -EINPROGRESS;
+}
+
+static int chcr_ahash_finup(struct ahash_request *req)
+{
+ struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
+ struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
+ struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
+ struct uld_ctx *u_ctx = NULL;
+ struct sk_buff *skb;
+ struct hash_wr_param params;
+ u8 bs;
+
+ bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
+ u_ctx = ULD_CTX(ctx);
+
+ if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], ctx->tx_channel_id))
+ return -EBUSY;
+
+ if (is_hmac(crypto_ahash_tfm(rtfm)))
+ params.opad_needed = 1;
+ else
+ params.opad_needed = 0;
+
+ params.sg_len = req->nbytes;
+ params.bfr_len = req_ctx->bfr_len;
+ get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm));
+ req_ctx->data_len += params.bfr_len + params.sg_len;
+ req_ctx->result = 1;
+ if (req_ctx->bfr && (req_ctx->bfr_len + req->nbytes) == 0) {
+ create_last_hash_block(req_ctx->bfr, bs, req_ctx->data_len);
+ params.last = 0;
+ params.more = 1;
+ params.scmd1 = 0;
+ params.bfr_len = bs;
+ } else {
+ params.scmd1 = req_ctx->data_len;
+ params.last = 1;
+ params.more = 0;
+ }
+
+ skb = create_final_hash_wr(req, ¶ms);
+ if (!skb)
+ return -ENOMEM;
+ skb->dev = u_ctx->lldi.ports[0];
+ set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id);
+ chcr_send_wr(skb);
+
+ return -EINPROGRESS;
+}
+
+static int chcr_ahash_digest(struct ahash_request *req)
+{
+ struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
+ struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
+ struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
+ struct uld_ctx *u_ctx = NULL;
+ struct sk_buff *skb;
+ struct hash_wr_param params;
+ u8 bs;
+
+ rtfm->init(req);
+ bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
+
+ u_ctx = ULD_CTX(ctx);
+ if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], ctx->tx_channel_id))
+ return -EBUSY;
+
+ if (is_hmac(crypto_ahash_tfm(rtfm)))
+ params.opad_needed = 1;
+ else
+ params.opad_needed = 0;
+
+ params.last = 0;
+ params.more = 0;
+ params.sg_len = req->nbytes;
+ params.bfr_len = 0;
+ params.scmd1 = 0;
+ get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm));
+ req_ctx->result = 1;
+ req_ctx->data_len += params.bfr_len + params.sg_len;
+
+ if (req_ctx->bfr && req->nbytes == 0) {
+ create_last_hash_block(req_ctx->bfr, bs, 0);
+ params.more = 1;
+ params.bfr_len = bs;
+ }
+
+ skb = create_final_hash_wr(req, ¶ms);
+ if (!skb)
+ return -ENOMEM;
+
+ skb->dev = u_ctx->lldi.ports[0];
+ set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id);
+ chcr_send_wr(skb);
+ return -EINPROGRESS;
+}
+
+static int chcr_ahash_export(struct ahash_request *areq, void *out)
+{
+ struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
+ struct chcr_ahash_req_ctx *state = out;
+
+ state->bfr_len = req_ctx->bfr_len;
+ state->data_len = req_ctx->data_len;
+ memcpy(state->bfr, req_ctx->bfr, CHCR_HASH_MAX_BLOCK_SIZE_128);
+ memcpy(state->partial_hash, req_ctx->partial_hash,
+ CHCR_HASH_MAX_DIGEST_SIZE);
+ return 0;
+}
+
+static int chcr_ahash_import(struct ahash_request *areq, const void *in)
+{
+ struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
+ struct chcr_ahash_req_ctx *state = (struct chcr_ahash_req_ctx *)in;
+
+ req_ctx->bfr_len = state->bfr_len;
+ req_ctx->data_len = state->data_len;
+ req_ctx->dummy_payload_ptr = NULL;
+ memcpy(req_ctx->bfr, state->bfr, CHCR_HASH_MAX_BLOCK_SIZE_128);
+ memcpy(req_ctx->partial_hash, state->partial_hash,
+ CHCR_HASH_MAX_DIGEST_SIZE);
+ return 0;
+}
+
+static int chcr_ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
+ struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
+ unsigned int digestsize = crypto_ahash_digestsize(tfm);
+ unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
+ unsigned int i, err = 0, updated_digestsize;
+
+ /*
+ * use the key to calculate the ipad and opad. ipad will sent with the
+ * first request's data. opad will be sent with the final hash result
+ * ipad in hmacctx->ipad and opad in hmacctx->opad location
+ */
+ if (!hmacctx->desc)
+ return -EINVAL;
+ if (keylen > bs) {
+ err = crypto_shash_digest(hmacctx->desc, key, keylen,
+ hmacctx->ipad);
+ if (err)
+ goto out;
+ keylen = digestsize;
+ } else {
+ memcpy(hmacctx->ipad, key, keylen);
+ }
+ memset(hmacctx->ipad + keylen, 0, bs - keylen);
+ memcpy(hmacctx->opad, hmacctx->ipad, bs);
+
+ for (i = 0; i < bs / sizeof(int); i++) {
+ *((unsigned int *)(&hmacctx->ipad) + i) ^= IPAD_DATA;
+ *((unsigned int *)(&hmacctx->opad) + i) ^= OPAD_DATA;
+ }
+
+ updated_digestsize = digestsize;
+ if (digestsize == SHA224_DIGEST_SIZE)
+ updated_digestsize = SHA256_DIGEST_SIZE;
+ else if (digestsize == SHA384_DIGEST_SIZE)
+ updated_digestsize = SHA512_DIGEST_SIZE;
+ err = chcr_compute_partial_hash(hmacctx->desc, hmacctx->ipad,
+ hmacctx->ipad, digestsize);
+ if (err)
+ goto out;
+ chcr_change_order(hmacctx->ipad, updated_digestsize);
+
+ err = chcr_compute_partial_hash(hmacctx->desc, hmacctx->opad,
+ hmacctx->opad, digestsize);
+ if (err)
+ goto out;
+ chcr_change_order(hmacctx->opad, updated_digestsize);
+out:
+ return err;
+}
+
+static int chcr_aes_xts_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
+ unsigned int key_len)
+{
+ struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
+ struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
+ int status = 0;
+ unsigned short context_size = 0;
+
+ if ((key_len == (AES_KEYSIZE_128 << 1)) ||
+ (key_len == (AES_KEYSIZE_256 << 1))) {
+ memcpy(ablkctx->key, key, key_len);
+ ablkctx->enckey_len = key_len;
+ context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + key_len) >> 4;
+ ablkctx->key_ctx_hdr =
+ FILL_KEY_CTX_HDR((key_len == AES_KEYSIZE_256) ?
+ CHCR_KEYCTX_CIPHER_KEY_SIZE_128 :
+ CHCR_KEYCTX_CIPHER_KEY_SIZE_256,
+ CHCR_KEYCTX_NO_KEY, 1,
+ 0, context_size);
+ ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_XTS;
+ } else {
+ crypto_tfm_set_flags((struct crypto_tfm *)tfm,
+ CRYPTO_TFM_RES_BAD_KEY_LEN);
+ ablkctx->enckey_len = 0;
+ status = -EINVAL;
+ }
+ return status;
+}
+
+static int chcr_sha_init(struct ahash_request *areq)
+{
+ struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
+ int digestsize = crypto_ahash_digestsize(tfm);
+
+ req_ctx->data_len = 0;
+ req_ctx->dummy_payload_ptr = NULL;
+ req_ctx->bfr_len = 0;
+ req_ctx->skb = NULL;
+ req_ctx->result = 0;
+ copy_hash_init_values(req_ctx->partial_hash, digestsize);
+ return 0;
+}
+
+static int chcr_sha_cra_init(struct crypto_tfm *tfm)
+{
+ crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
+ sizeof(struct chcr_ahash_req_ctx));
+ return chcr_device_init(crypto_tfm_ctx(tfm));
+}
+
+static int chcr_hmac_init(struct ahash_request *areq)
+{
+ struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
+ struct crypto_ahash *rtfm = crypto_ahash_reqtfm(areq);
+ struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
+ struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
+ unsigned int digestsize = crypto_ahash_digestsize(rtfm);
+ unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
+
+ chcr_sha_init(areq);
+ req_ctx->data_len = bs;
+ if (is_hmac(crypto_ahash_tfm(rtfm))) {
+ if (digestsize == SHA224_DIGEST_SIZE)
+ memcpy(req_ctx->partial_hash, hmacctx->ipad,
+ SHA256_DIGEST_SIZE);
+ else if (digestsize == SHA384_DIGEST_SIZE)
+ memcpy(req_ctx->partial_hash, hmacctx->ipad,
+ SHA512_DIGEST_SIZE);
+ else
+ memcpy(req_ctx->partial_hash, hmacctx->ipad,
+ digestsize);
+ }
+ return 0;
+}
+
+static int chcr_hmac_cra_init(struct crypto_tfm *tfm)
+{
+ struct chcr_context *ctx = crypto_tfm_ctx(tfm);
+ struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
+ unsigned int digestsize =
+ crypto_ahash_digestsize(__crypto_ahash_cast(tfm));
+
+ crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
+ sizeof(struct chcr_ahash_req_ctx));
+ hmacctx->desc = chcr_alloc_shash(digestsize);
+ if (IS_ERR(hmacctx->desc))
+ return PTR_ERR(hmacctx->desc);
+ return chcr_device_init(crypto_tfm_ctx(tfm));
+}
+
+static void chcr_free_shash(struct shash_desc *desc)
+{
+ crypto_free_shash(desc->tfm);
+ kfree(desc);
+}
+
+static void chcr_hmac_cra_exit(struct crypto_tfm *tfm)
+{
+ struct chcr_context *ctx = crypto_tfm_ctx(tfm);
+ struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
+
+ if (hmacctx->desc) {
+ chcr_free_shash(hmacctx->desc);
+ hmacctx->desc = NULL;
+ }
+}
+
+static struct chcr_alg_template driver_algs[] = {
+ /* AES-CBC */
+ {
+ .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
+ .is_registered = 0,
+ .alg.crypto = {
+ .cra_name = "cbc(aes)",
+ .cra_driver_name = "cbc(aes-chcr)",
+ .cra_priority = CHCR_CRA_PRIORITY,
+ .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
+ CRYPTO_ALG_ASYNC,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct chcr_context)
+ + sizeof(struct ablk_ctx),
+ .cra_alignmask = 0,
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_init = chcr_cra_init,
+ .cra_exit = NULL,
+ .cra_u.ablkcipher = {
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = chcr_aes_cbc_setkey,
+ .encrypt = chcr_aes_encrypt,
+ .decrypt = chcr_aes_decrypt,
+ }
+ }
+ },
+ {
+ .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
+ .is_registered = 0,
+ .alg.crypto = {
+ .cra_name = "xts(aes)",
+ .cra_driver_name = "xts(aes-chcr)",
+ .cra_priority = CHCR_CRA_PRIORITY,
+ .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
+ CRYPTO_ALG_ASYNC,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct chcr_context) +
+ sizeof(struct ablk_ctx),
+ .cra_alignmask = 0,
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_init = chcr_cra_init,
+ .cra_exit = NULL,
+ .cra_u = {
+ .ablkcipher = {
+ .min_keysize = 2 * AES_MIN_KEY_SIZE,
+ .max_keysize = 2 * AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = chcr_aes_xts_setkey,
+ .encrypt = chcr_aes_encrypt,
+ .decrypt = chcr_aes_decrypt,
+ }
+ }
+ }
+ },
+ /* SHA */
+ {
+ .type = CRYPTO_ALG_TYPE_AHASH,
+ .is_registered = 0,
+ .alg.hash = {
+ .halg.digestsize = SHA1_DIGEST_SIZE,
+ .halg.base = {
+ .cra_name = "sha1",
+ .cra_driver_name = "sha1-chcr",
+ .cra_blocksize = SHA1_BLOCK_SIZE,
+ }
+ }
+ },
+ {
+ .type = CRYPTO_ALG_TYPE_AHASH,
+ .is_registered = 0,
+ .alg.hash = {
+ .halg.digestsize = SHA256_DIGEST_SIZE,
+ .halg.base = {
+ .cra_name = "sha256",
+ .cra_driver_name = "sha256-chcr",
+ .cra_blocksize = SHA256_BLOCK_SIZE,
+ }
+ }
+ },
+ {
+ .type = CRYPTO_ALG_TYPE_AHASH,
+ .is_registered = 0,
+ .alg.hash = {
+ .halg.digestsize = SHA224_DIGEST_SIZE,
+ .halg.base = {
+ .cra_name = "sha224",
+ .cra_driver_name = "sha224-chcr",
+ .cra_blocksize = SHA224_BLOCK_SIZE,
+ }
+ }
+ },
+ {
+ .type = CRYPTO_ALG_TYPE_AHASH,
+ .is_registered = 0,
+ .alg.hash = {
+ .halg.digestsize = SHA384_DIGEST_SIZE,
+ .halg.base = {
+ .cra_name = "sha384",
+ .cra_driver_name = "sha384-chcr",
+ .cra_blocksize = SHA384_BLOCK_SIZE,
+ }
+ }
+ },
+ {
+ .type = CRYPTO_ALG_TYPE_AHASH,
+ .is_registered = 0,
+ .alg.hash = {
+ .halg.digestsize = SHA512_DIGEST_SIZE,
+ .halg.base = {
+ .cra_name = "sha512",
+ .cra_driver_name = "sha512-chcr",
+ .cra_blocksize = SHA512_BLOCK_SIZE,
+ }
+ }
+ },
+ /* HMAC */
+ {
+ .type = CRYPTO_ALG_TYPE_HMAC,
+ .is_registered = 0,
+ .alg.hash = {
+ .halg.digestsize = SHA1_DIGEST_SIZE,
+ .halg.base = {
+ .cra_name = "hmac(sha1)",
+ .cra_driver_name = "hmac(sha1-chcr)",
+ .cra_blocksize = SHA1_BLOCK_SIZE,
+ }
+ }
+ },
+ {
+ .type = CRYPTO_ALG_TYPE_HMAC,
+ .is_registered = 0,
+ .alg.hash = {
+ .halg.digestsize = SHA224_DIGEST_SIZE,
+ .halg.base = {
+ .cra_name = "hmac(sha224)",
+ .cra_driver_name = "hmac(sha224-chcr)",
+ .cra_blocksize = SHA224_BLOCK_SIZE,
+ }
+ }
+ },
+ {
+ .type = CRYPTO_ALG_TYPE_HMAC,
+ .is_registered = 0,
+ .alg.hash = {
+ .halg.digestsize = SHA256_DIGEST_SIZE,
+ .halg.base = {
+ .cra_name = "hmac(sha256)",
+ .cra_driver_name = "hmac(sha256-chcr)",
+ .cra_blocksize = SHA256_BLOCK_SIZE,
+ }
+ }
+ },
+ {
+ .type = CRYPTO_ALG_TYPE_HMAC,
+ .is_registered = 0,
+ .alg.hash = {
+ .halg.digestsize = SHA384_DIGEST_SIZE,
+ .halg.base = {
+ .cra_name = "hmac(sha384)",
+ .cra_driver_name = "hmac(sha384-chcr)",
+ .cra_blocksize = SHA384_BLOCK_SIZE,
+ }
+ }
+ },
+ {
+ .type = CRYPTO_ALG_TYPE_HMAC,
+ .is_registered = 0,
+ .alg.hash = {
+ .halg.digestsize = SHA512_DIGEST_SIZE,
+ .halg.base = {
+ .cra_name = "hmac(sha512)",
+ .cra_driver_name = "hmac(sha512-chcr)",
+ .cra_blocksize = SHA512_BLOCK_SIZE,
+ }
+ }
+ },
+};
+
+/*
+ * chcr_unregister_alg - Deregister crypto algorithms with
+ * kernel framework.
+ */
+static int chcr_unregister_alg(void)
+{
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
+ switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) {
+ case CRYPTO_ALG_TYPE_ABLKCIPHER:
+ if (driver_algs[i].is_registered)
+ crypto_unregister_alg(
+ &driver_algs[i].alg.crypto);
+ break;
+ case CRYPTO_ALG_TYPE_AHASH:
+ if (driver_algs[i].is_registered)
+ crypto_unregister_ahash(
+ &driver_algs[i].alg.hash);
+ break;
+ }
+ driver_algs[i].is_registered = 0;
+ }
+ return 0;
+}
+
+/*
+ * chcr_register_alg - Register crypto algorithms with kernel framework.
+ */
+static int chcr_register_alg(void)
+{
+ struct crypto_alg ai;
+ int err = 0, i;
+ char *name = NULL;
+
+ for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
+ if (driver_algs[i].is_registered)
+ continue;
+ switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) {
+ case CRYPTO_ALG_TYPE_ABLKCIPHER:
+ err = crypto_register_alg(&driver_algs[i].alg.crypto);
+ name = driver_algs[i].alg.crypto.cra_driver_name;
+ break;
+ case CRYPTO_ALG_TYPE_AHASH:
+ driver_algs[i].alg.hash.update = chcr_ahash_update;
+ driver_algs[i].alg.hash.final = chcr_ahash_final;
+ driver_algs[i].alg.hash.finup = chcr_ahash_finup;
+ driver_algs[i].alg.hash.digest = chcr_ahash_digest;
+ driver_algs[i].alg.hash.export = chcr_ahash_export;
+ driver_algs[i].alg.hash.import = chcr_ahash_import;
+ driver_algs[i].alg.hash.halg.statesize =
+ sizeof(struct chcr_ahash_req_ctx);
+ driver_algs[i].alg.hash.halg.base.cra_priority =
+ CHCR_CRA_PRIORITY;
+ driver_algs[i].alg.hash.halg.base.cra_module =
+ THIS_MODULE;
+ driver_algs[i].alg.hash.halg.base.cra_flags =
+ CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC;
+ driver_algs[i].alg.hash.halg.base.cra_alignmask = 0;
+ driver_algs[i].alg.hash.halg.base.cra_exit = NULL;
+ driver_algs[i].alg.hash.halg.base.cra_type =
+ &crypto_ahash_type;
+
+ if (driver_algs[i].type == CRYPTO_ALG_TYPE_HMAC) {
+ driver_algs[i].alg.hash.halg.base.cra_init =
+ driver_algs[i].alg.hash.halg.base.cra_init =
+ chcr_hmac_cra_init;
+ driver_algs[i].alg.hash.halg.base.cra_exit =
+ chcr_hmac_cra_exit;
+
+ driver_algs[i].alg.hash.init = chcr_hmac_init;
+
+ driver_algs[i].alg.hash.setkey =
+ chcr_ahash_setkey;
+ driver_algs[i].alg.hash.halg.base.cra_ctxsize =
+ sizeof(struct chcr_context) +
+ sizeof(struct hmac_ctx);
+ } else {
+ driver_algs[i].alg.hash.halg.base.cra_init =
+ chcr_sha_cra_init;
+ driver_algs[i].alg.hash.init = chcr_sha_init;
+
+ driver_algs[i].alg.hash.halg.base.cra_ctxsize =
+ sizeof(struct chcr_context);
+ }
+ err = crypto_register_ahash(&driver_algs[i].alg.hash);
+ ai = driver_algs[i].alg.hash.halg.base;
+ name = ai.cra_driver_name;
+ break;
+ }
+ if (err) {
+ pr_err("chcr : %s : Algorithm registration failed\n",
+ name);
+ goto register_err;
+ } else {
+ driver_algs[i].is_registered = 1;
+ }
+ }
+ return 0;
+
+register_err:
+ chcr_unregister_alg();
+ return err;
+}
+
+/*
+ * start_crypto - Register the crypto algorithms.
+ * This should called once when the first device comesup. After this
+ * kernel will start calling driver APIs for crypto operations.
+ */
+int start_crypto(void)
+{
+ return chcr_register_alg();
+}
+
+/*
+ * stop_crypto - Deregister all the crypto algorithms with kernel.
+ * This should be called once when the last device goes down. After this
+ * kernel will not call the driver API for crypto operations.
+ */
+int stop_crypto(void)
+{
+ chcr_unregister_alg();
+ return 0;
+}
diff --git a/drivers/crypto/chelsio/chcr_algo.h b/drivers/crypto/chelsio/chcr_algo.h
new file mode 100644
index 0000000..34ded07
--- /dev/null
+++ b/drivers/crypto/chelsio/chcr_algo.h
@@ -0,0 +1,502 @@
+/*
+ * This file is part of the Chelsio T6 Crypto driver for Linux.
+ *
+ * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#ifndef __CHCR_ALGO_H__
+#define __CHCR_ALGO_H__
+
+/* Crypto key context */
+#define S_KEY_CONTEXT_CTX_LEN 24
+#define M_KEY_CONTEXT_CTX_LEN 0xff
+#define V_KEY_CONTEXT_CTX_LEN(x) ((x) << S_KEY_CONTEXT_CTX_LEN)
+#define G_KEY_CONTEXT_CTX_LEN(x) \
+ (((x) >> S_KEY_CONTEXT_CTX_LEN) & M_KEY_CONTEXT_CTX_LEN)
+
+#define S_KEY_CONTEXT_DUAL_CK 12
+#define M_KEY_CONTEXT_DUAL_CK 0x1
+#define V_KEY_CONTEXT_DUAL_CK(x) ((x) << S_KEY_CONTEXT_DUAL_CK)
+#define G_KEY_CONTEXT_DUAL_CK(x) \
+(((x) >> S_KEY_CONTEXT_DUAL_CK) & M_KEY_CONTEXT_DUAL_CK)
+#define F_KEY_CONTEXT_DUAL_CK V_KEY_CONTEXT_DUAL_CK(1U)
+
+#define S_KEY_CONTEXT_SALT_PRESENT 10
+#define M_KEY_CONTEXT_SALT_PRESENT 0x1
+#define V_KEY_CONTEXT_SALT_PRESENT(x) ((x) << S_KEY_CONTEXT_SALT_PRESENT)
+#define G_KEY_CONTEXT_SALT_PRESENT(x) \
+ (((x) >> S_KEY_CONTEXT_SALT_PRESENT) & \
+ M_KEY_CONTEXT_SALT_PRESENT)
+#define F_KEY_CONTEXT_SALT_PRESENT V_KEY_CONTEXT_SALT_PRESENT(1U)
+
+#define S_KEY_CONTEXT_VALID 0
+#define M_KEY_CONTEXT_VALID 0x1
+#define V_KEY_CONTEXT_VALID(x) ((x) << S_KEY_CONTEXT_VALID)
+#define G_KEY_CONTEXT_VALID(x) \
+ (((x) >> S_KEY_CONTEXT_VALID) & \
+ M_KEY_CONTEXT_VALID)
+#define F_KEY_CONTEXT_VALID V_KEY_CONTEXT_VALID(1U)
+
+#define S_KEY_CONTEXT_CK_SIZE 6
+#define M_KEY_CONTEXT_CK_SIZE 0xf
+#define V_KEY_CONTEXT_CK_SIZE(x) ((x) << S_KEY_CONTEXT_CK_SIZE)
+#define G_KEY_CONTEXT_CK_SIZE(x) \
+ (((x) >> S_KEY_CONTEXT_CK_SIZE) & M_KEY_CONTEXT_CK_SIZE)
+
+#define S_KEY_CONTEXT_MK_SIZE 2
+#define M_KEY_CONTEXT_MK_SIZE 0xf
+#define V_KEY_CONTEXT_MK_SIZE(x) ((x) << S_KEY_CONTEXT_MK_SIZE)
+#define G_KEY_CONTEXT_MK_SIZE(x) \
+ (((x) >> S_KEY_CONTEXT_MK_SIZE) & M_KEY_CONTEXT_MK_SIZE)
+
+#define S_KEY_CONTEXT_OPAD_PRESENT 11
+#define M_KEY_CONTEXT_OPAD_PRESENT 0x1
+#define V_KEY_CONTEXT_OPAD_PRESENT(x) ((x) << S_KEY_CONTEXT_OPAD_PRESENT)
+#define G_KEY_CONTEXT_OPAD_PRESENT(x) \
+ (((x) >> S_KEY_CONTEXT_OPAD_PRESENT) & \
+ M_KEY_CONTEXT_OPAD_PRESENT)
+#define F_KEY_CONTEXT_OPAD_PRESENT V_KEY_CONTEXT_OPAD_PRESENT(1U)
+
+#define CHCR_HASH_MAX_DIGEST_SIZE 64
+#define CHCR_MAX_SHA_DIGEST_SIZE 64
+
+#define IPSEC_TRUNCATED_ICV_SIZE 12
+#define TLS_TRUNCATED_HMAC_SIZE 10
+#define CBCMAC_DIGEST_SIZE 16
+#define MAX_HASH_NAME 20
+
+#define SHA1_INIT_STATE_5X4B 5
+#define SHA256_INIT_STATE_8X4B 8
+#define SHA512_INIT_STATE_8X8B 8
+#define SHA1_INIT_STATE SHA1_INIT_STATE_5X4B
+#define SHA224_INIT_STATE SHA256_INIT_STATE_8X4B
+#define SHA256_INIT_STATE SHA256_INIT_STATE_8X4B
+#define SHA384_INIT_STATE SHA512_INIT_STATE_8X8B
+#define SHA512_INIT_STATE SHA512_INIT_STATE_8X8B
+
+#define DUMMY_BYTES 16
+
+#define IPAD_DATA 0x36363636
+#define OPAD_DATA 0x5c5c5c5c
+
+#define TRANSHDR_SIZE(alignedkctx_len)\
+ (sizeof(struct ulptx_idata) +\
+ sizeof(struct ulp_txpkt) +\
+ sizeof(struct fw_crypto_lookaside_wr) +\
+ sizeof(struct cpl_tx_sec_pdu) +\
+ (alignedkctx_len))
+#define CIPHER_TRANSHDR_SIZE(alignedkctx_len, sge_pairs) \
+ (TRANSHDR_SIZE(alignedkctx_len) + sge_pairs +\
+ sizeof(struct cpl_rx_phys_dsgl))
+#define HASH_TRANSHDR_SIZE(alignedkctx_len)\
+ (TRANSHDR_SIZE(alignedkctx_len) + DUMMY_BYTES)
+
+#define SEC_CPL_OFFSET (sizeof(struct fw_crypto_lookaside_wr) + \
+ sizeof(struct ulp_txpkt) + \
+ sizeof(struct ulptx_idata))
+
+#define FILL_SEC_CPL_OP_IVINSR(id, len, hldr, ofst) \
+ htonl( \
+ V_CPL_TX_SEC_PDU_OPCODE(CPL_TX_SEC_PDU) | \
+ V_CPL_TX_SEC_PDU_RXCHID((id)) | \
+ V_CPL_TX_SEC_PDU_ACKFOLLOWS(0) | \
+ V_CPL_TX_SEC_PDU_ULPTXLPBK(1) | \
+ V_CPL_TX_SEC_PDU_CPLLEN((len)) | \
+ V_CPL_TX_SEC_PDU_PLACEHOLDER((hldr)) | \
+ V_CPL_TX_SEC_PDU_IVINSRTOFST((ofst)))
+
+#define FILL_SEC_CPL_CIPHERSTOP_HI(a_start, a_stop, c_start, c_stop_hi) \
+ htonl( \
+ V_CPL_TX_SEC_PDU_AADSTART((a_start)) | \
+ V_CPL_TX_SEC_PDU_AADSTOP((a_stop)) | \
+ V_CPL_TX_SEC_PDU_CIPHERSTART((c_start)) | \
+ V_CPL_TX_SEC_PDU_CIPHERSTOP_HI((c_stop_hi)))
+
+#define FILL_SEC_CPL_AUTHINSERT(c_stop_lo, a_start, a_stop, a_inst) \
+ htonl( \
+ V_CPL_TX_SEC_PDU_CIPHERSTOP_LO((c_stop_lo)) | \
+ V_CPL_TX_SEC_PDU_AUTHSTART((a_start)) | \
+ V_CPL_TX_SEC_PDU_AUTHSTOP((a_stop)) | \
+ V_CPL_TX_SEC_PDU_AUTHINSERT((a_inst)))
+
+#define FILL_SEC_CPL_SCMD0_SEQNO(ctrl, seq, cmode, amode, opad, size, nivs) \
+ htonl( \
+ V_SCMD_SEQ_NO_CTRL(0) | \
+ V_SCMD_STATUS_PRESENT(0) | \
+ V_SCMD_PROTO_VERSION(CHCR_SCMD_PROTO_VERSION_GENERIC) | \
+ V_SCMD_ENC_DEC_CTRL((ctrl)) | \
+ V_SCMD_CIPH_AUTH_SEQ_CTRL((seq)) | \
+ V_SCMD_CIPH_MODE((cmode)) | \
+ V_SCMD_AUTH_MODE((amode)) | \
+ V_SCMD_HMAC_CTRL((opad)) | \
+ V_SCMD_IV_SIZE((size)) | \
+ V_SCMD_NUM_IVS((nivs)))
+
+#define FILL_SEC_CPL_IVGEN_HDRLEN(last, more, ctx_in, mac, ivdrop, len) htonl( \
+ V_SCMD_ENB_DBGID(0) | \
+ V_SCMD_IV_GEN_CTRL(0) | \
+ V_SCMD_LAST_FRAG((last)) | \
+ V_SCMD_MORE_FRAGS((more)) | \
+ V_SCMD_TLS_COMPPDU(0) | \
+ V_SCMD_KEY_CTX_INLINE((ctx_in)) | \
+ V_SCMD_TLS_FRAG_ENABLE(0) | \
+ V_SCMD_MAC_ONLY((mac)) | \
+ V_SCMD_AADIVDROP((ivdrop)) | \
+ V_SCMD_HDR_LEN((len)))
+
+#define FILL_KEY_CTX_HDR(ck_size, mk_size, d_ck, opad, ctx_len) \
+ htonl(V_KEY_CONTEXT_VALID(1) | \
+ V_KEY_CONTEXT_CK_SIZE((ck_size)) | \
+ V_KEY_CONTEXT_MK_SIZE(mk_size) | \
+ V_KEY_CONTEXT_DUAL_CK((d_ck)) | \
+ V_KEY_CONTEXT_OPAD_PRESENT((opad)) | \
+ V_KEY_CONTEXT_SALT_PRESENT(1) | \
+ V_KEY_CONTEXT_CTX_LEN((ctx_len)))
+
+#define FILL_WR_OP_CCTX_SIZE(len, ctx_len) \
+ htonl( \
+ V_FW_CRYPTO_LOOKASIDE_WR_OPCODE( \
+ FW_CRYPTO_LOOKASIDE_WR) | \
+ V_FW_CRYPTO_LOOKASIDE_WR_COMPL(0) | \
+ V_FW_CRYPTO_LOOKASIDE_WR_IMM_LEN((len)) | \
+ V_FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC(1) | \
+ V_FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE((ctx_len)))
+
+#define FILL_WR_RX_Q_ID(cid, qid, wr_iv) \
+ htonl( \
+ V_FW_CRYPTO_LOOKASIDE_WR_RX_CHID((cid)) | \
+ V_FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID((qid)) | \
+ V_FW_CRYPTO_LOOKASIDE_WR_LCB(0) | \
+ V_FW_CRYPTO_LOOKASIDE_WR_IV((wr_iv)))
+
+#define FILL_ULPTX_CMD_DEST(cid) \
+ htonl(V_ULPTX_CMD(ULP_TX_PKT) | \
+ V_ULP_TXPKT_DEST(0) | \
+ V_ULP_TXPKT_DATAMODIFY(0) | \
+ V_ULP_TXPKT_CHANNELID((cid)) | \
+ V_ULP_TXPKT_RO(1) | \
+ V_ULP_TXPKT_FID(0))
+
+#define KEYCTX_ALIGN_PAD(bs) ({unsigned int _bs = (bs);\
+ _bs == SHA1_DIGEST_SIZE ? 12 : 0; })
+
+#define FILL_PLD_SIZE_HASH_SIZE(payload_sgl_len, sgl_lengths, total_frags) \
+ htonl(V_FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE(payload_sgl_len ? \
+ sgl_lengths[total_frags] : 0) |\
+ V_FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE(0))
+
+#define FILL_LEN_PKD(calc_tx_flits_ofld, skb) \
+ htonl(V_FW_CRYPTO_LOOKASIDE_WR_LEN16(DIV_ROUND_UP((\
+ calc_tx_flits_ofld(skb) * 8), 16)))
+
+#define FILL_CMD_MORE(immdatalen) htonl(V_ULPTX_CMD(ULP_TX_SC_IMM) |\
+ V_ULP_TX_SC_MORE((immdatalen) ? 0 : 1))
+
+#define MAX_NK 8
+
+struct algo_param {
+ unsigned int auth_mode;
+ unsigned int mk_size;
+ unsigned int result_size;
+};
+
+struct hash_wr_param {
+ unsigned int opad_needed;
+ unsigned int more;
+ unsigned int last;
+ struct algo_param alg_prm;
+ unsigned int sg_len;
+ unsigned int bfr_len;
+ u64 scmd1;
+};
+
+enum {
+ AES_KEYLENGTH_128BIT = 128,
+ AES_KEYLENGTH_192BIT = 192,
+ AES_KEYLENGTH_256BIT = 256
+};
+
+enum {
+ KEYLENGTH_3BYTES = 3,
+ KEYLENGTH_4BYTES = 4,
+ KEYLENGTH_6BYTES = 6,
+ KEYLENGTH_8BYTES = 8
+};
+
+enum {
+ NUMBER_OF_ROUNDS_10 = 10,
+ NUMBER_OF_ROUNDS_12 = 12,
+ NUMBER_OF_ROUNDS_14 = 14,
+};
+
+/*
+ * CCM defines values of 4, 6, 8, 10, 12, 14, and 16 octets,
+ * where they indicate the size of the integrity check value (ICV)
+ */
+enum {
+ AES_CCM_ICV_4 = 4,
+ AES_CCM_ICV_6 = 6,
+ AES_CCM_ICV_8 = 8,
+ AES_CCM_ICV_10 = 10,
+ AES_CCM_ICV_12 = 12,
+ AES_CCM_ICV_14 = 14,
+ AES_CCM_ICV_16 = 16
+};
+
+struct hash_op_params {
+ unsigned char mk_size;
+ unsigned char pad_align;
+ unsigned char auth_mode;
+ char hash_name[MAX_HASH_NAME];
+ unsigned short block_size;
+ unsigned short word_size;
+ unsigned short ipad_size;
+};
+
+struct phys_sge_pairs {
+ __be16 len[8];
+ __be64 addr[8];
+};
+
+struct phys_sge_parm {
+ unsigned int nents;
+ unsigned int obsize;
+ unsigned short qid;
+ unsigned char align;
+};
+
+struct crypto_result {
+ struct completion completion;
+ int err;
+};
+
+static const u32 sha1_init[SHA1_DIGEST_SIZE / 4] = {
+ SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4,
+};
+
+static const u32 sha224_init[SHA256_DIGEST_SIZE / 4] = {
+ SHA224_H0, SHA224_H1, SHA224_H2, SHA224_H3,
+ SHA224_H4, SHA224_H5, SHA224_H6, SHA224_H7,
+};
+
+static const u32 sha256_init[SHA256_DIGEST_SIZE / 4] = {
+ SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3,
+ SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7,
+};
+
+static const u64 sha384_init[SHA512_DIGEST_SIZE / 8] = {
+ SHA384_H0, SHA384_H1, SHA384_H2, SHA384_H3,
+ SHA384_H4, SHA384_H5, SHA384_H6, SHA384_H7,
+};
+
+static const u64 sha512_init[SHA512_DIGEST_SIZE / 8] = {
+ SHA512_H0, SHA512_H1, SHA512_H2, SHA512_H3,
+ SHA512_H4, SHA512_H5, SHA512_H6, SHA512_H7,
+};
+
+static inline void copy_hash_init_values(char *key, int digestsize)
+{
+ u8 i;
+ __be32 *dkey = (__be32 *)key;
+ u64 *ldkey = (u64 *)key;
+ __be64 *sha384 = (__be64 *)sha384_init;
+ __be64 *sha512 = (__be64 *)sha512_init;
+
+ switch (digestsize) {
+ case SHA1_DIGEST_SIZE:
+ for (i = 0; i < SHA1_INIT_STATE; i++)
+ dkey[i] = cpu_to_be32(sha1_init[i]);
+ break;
+ case SHA224_DIGEST_SIZE:
+ for (i = 0; i < SHA224_INIT_STATE; i++)
+ dkey[i] = cpu_to_be32(sha224_init[i]);
+ break;
+ case SHA256_DIGEST_SIZE:
+ for (i = 0; i < SHA256_INIT_STATE; i++)
+ dkey[i] = cpu_to_be32(sha256_init[i]);
+ break;
+ case SHA384_DIGEST_SIZE:
+ for (i = 0; i < SHA384_INIT_STATE; i++)
+ ldkey[i] = be64_to_cpu(sha384[i]);
+ break;
+ case SHA512_DIGEST_SIZE:
+ for (i = 0; i < SHA512_INIT_STATE; i++)
+ ldkey[i] = be64_to_cpu(sha512[i]);
+ break;
+ }
+}
+
+static const u8 sgl_lengths[20] = {
+ 0, 1, 2, 3, 4, 4, 5, 6, 7, 7, 8, 9, 10, 10, 11, 12, 13, 13, 14, 15
+};
+
+/* Number of len fields(8) * size of one addr field */
+#define PHYSDSGL_MAX_LEN_SIZE 16
+
+static inline u16 get_space_for_phys_dsgl(unsigned int sgl_entr)
+{
+ /* len field size + addr field size */
+ return ((sgl_entr >> 3) + ((sgl_entr % 8) ?
+ 1 : 0)) * PHYSDSGL_MAX_LEN_SIZE +
+ (sgl_entr << 3) + ((sgl_entr % 2 ? 1 : 0) << 3);
+}
+
+/* The AES s-transform matrix (s-box). */
+static const u8 aes_sbox[256] = {
+ 99, 124, 119, 123, 242, 107, 111, 197, 48, 1, 103, 43, 254, 215,
+ 171, 118, 202, 130, 201, 125, 250, 89, 71, 240, 173, 212, 162, 175,
+ 156, 164, 114, 192, 183, 253, 147, 38, 54, 63, 247, 204, 52, 165,
+ 229, 241, 113, 216, 49, 21, 4, 199, 35, 195, 24, 150, 5, 154, 7,
+ 18, 128, 226, 235, 39, 178, 117, 9, 131, 44, 26, 27, 110, 90,
+ 160, 82, 59, 214, 179, 41, 227, 47, 132, 83, 209, 0, 237, 32,
+ 252, 177, 91, 106, 203, 190, 57, 74, 76, 88, 207, 208, 239, 170,
+ 251, 67, 77, 51, 133, 69, 249, 2, 127, 80, 60, 159, 168, 81,
+ 163, 64, 143, 146, 157, 56, 245, 188, 182, 218, 33, 16, 255, 243,
+ 210, 205, 12, 19, 236, 95, 151, 68, 23, 196, 167, 126, 61, 100,
+ 93, 25, 115, 96, 129, 79, 220, 34, 42, 144, 136, 70, 238, 184,
+ 20, 222, 94, 11, 219, 224, 50, 58, 10, 73, 6, 36, 92, 194,
+ 211, 172, 98, 145, 149, 228, 121, 231, 200, 55, 109, 141, 213, 78,
+ 169, 108, 86, 244, 234, 101, 122, 174, 8, 186, 120, 37, 46, 28, 166,
+ 180, 198, 232, 221, 116, 31, 75, 189, 139, 138, 112, 62, 181, 102,
+ 72, 3, 246, 14, 97, 53, 87, 185, 134, 193, 29, 158, 225, 248,
+ 152, 17, 105, 217, 142, 148, 155, 30, 135, 233, 206, 85, 40, 223,
+ 140, 161, 137, 13, 191, 230, 66, 104, 65, 153, 45, 15, 176, 84,
+ 187, 22
+};
+
+static u32 aes_ks_subword(const u32 w)
+{
+ u8 bytes[4];
+
+ *(u32 *)(&bytes[0]) = w;
+ bytes[0] = aes_sbox[bytes[0]];
+ bytes[1] = aes_sbox[bytes[1]];
+ bytes[2] = aes_sbox[bytes[2]];
+ bytes[3] = aes_sbox[bytes[3]];
+ return *(u32 *)(&bytes[0]);
+}
+
+static u32 round_constant[11] = {
+ 0x01000000, 0x02000000, 0x04000000, 0x08000000,
+ 0x10000000, 0x20000000, 0x40000000, 0x80000000,
+ 0x1B000000, 0x36000000, 0x6C000000
+};
+
+/* dec_key - OUTPUT - Reverse round key
+ * key - INPUT - key
+ * keylength - INPUT - length of the key in number of bits
+ */
+static inline void get_aes_decrypt_key(unsigned char *dec_key,
+ const unsigned char *key,
+ unsigned int keylength)
+{
+ u32 temp;
+ __be32 val;
+ u32 w_ring[MAX_NK];
+ u8 w_last_ix;
+ int i, j, k = 0, flag = 0, start = 1, t1 = 0;
+ u8 nr, nk;
+
+ switch (keylength) {
+ case AES_KEYLENGTH_128BIT:
+ nk = KEYLENGTH_4BYTES;
+ nr = NUMBER_OF_ROUNDS_10;
+ start = 4;
+ break;
+
+ case AES_KEYLENGTH_192BIT:
+ nk = KEYLENGTH_6BYTES;
+ nr = NUMBER_OF_ROUNDS_12;
+ start = 2;
+ break;
+ case AES_KEYLENGTH_256BIT:
+ nk = KEYLENGTH_8BYTES;
+ nr = NUMBER_OF_ROUNDS_14;
+ start = 0;
+ break;
+ default:
+ return;
+ }
+
+ j = keylength >> KEYLENGTH_3BYTES;
+
+ /*
+ * Need to do host byte order correction here since key is byte
+ * oriented and the kx algorithm is word (u32) oriented.
+ */
+ for (i = 0; i < nk; i += 1)
+ w_ring[i] = be32_to_cpu(*((__be32 *)&key[4 * i]));
+
+ i = (int)nk;
+ w_last_ix = i - 1;
+
+ while (i < (4 * (nr + 2))) {
+ temp = w_ring[w_last_ix];
+ if (!(i % nk)) {
+ temp = (temp << 8) | (temp >> 24);
+ temp = aes_ks_subword(temp);
+ temp ^= round_constant[i / nk - 1];
+ } else if ((nk > 6) && ((i % nk) == 4)) {
+ temp = aes_ks_subword(temp);
+ }
+ w_last_ix = (w_last_ix + 1) % nk;
+ temp ^= w_ring[w_last_ix];
+ w_ring[w_last_ix] = temp;
+ /* We need the round keys for round Nr+1 and Nr+2 (round key
+ * Nr+2 is the round key beyond the last one used when
+ * encrypting). Rounds are numbered starting from 0, Nr=10
+ * implies 11 rounds are used in encryption/decryption.
+ */
+ if (i >= (4 * (nr - 1))) {
+ if (t1 >= start) {
+ if (j >= 0)
+ j -= 4;
+ if ((j < 0) && !flag) {
+ k = (keylength >> KEYLENGTH_3BYTES) - 4;
+ flag = 1;
+ }
+ if (k && flag)
+ k += 4;
+ if (j < 0)
+ j = 0;
+ val = cpu_to_be32(temp);
+ memcpy(dec_key + j + k, (void *)&val,
+ sizeof(u32));
+ } else {
+ t1++;
+ }
+ }
+ ++i;
+ }
+}
+
+#endif /* __CHCR_ALGO_H__ */
diff --git a/drivers/crypto/chelsio/chcr_core.c b/drivers/crypto/chelsio/chcr_core.c
new file mode 100644
index 0000000..80c5dba
--- /dev/null
+++ b/drivers/crypto/chelsio/chcr_core.c
@@ -0,0 +1,273 @@
+/*
+ * This file is part of the Chelsio T6 Crypto driver for Linux.
+ *
+ * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Written and Maintained by:
+ * Manoj Malviya (manojmalviya@chelsio.com)
+ * Atul Gupta (atul.gupta@chelsio.com)
+ * Jitendra Lulla (jlulla@chelsio.com)
+ * Yeshaswi M R Gowda (yeshaswi@chelsio.com)
+ * Harsh Jain (harsh@chelsio.com)
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/skbuff.h>
+
+#include <crypto/aes.h>
+#include <crypto/hash.h>
+
+#include "t4_msg.h"
+#include "chcr_core.h"
+
+static LIST_HEAD(uld_ctx_list);
+static DEFINE_MUTEX(dev_mutex);
+static atomic_t dev_count;
+
+typedef int (*chcr_handler_func)(struct chcr_dev *dev, unsigned char *input);
+static int cpl_fw6_pld_handler(struct chcr_dev *dev, unsigned char *input);
+static int chcr_uld_state_change(void *handle, enum cxgb4_state state);
+static void *chcr_uld_add(const struct cxgb4_lld_info *lld);
+static int chcr_uld_control(void *handle, enum cxgb4_control control, ...);
+
+static chcr_handler_func work_handlers[NUM_CPL_CMDS] = {
+ [CPL_FW6_PLD] = cpl_fw6_pld_handler,
+};
+
+static const struct cxgb4_uld_info chcr_uld_info = {
+ .name = DRV_MODULE_NAME,
+ .add = chcr_uld_add,
+ .rx_handler = chcr_uld_rx_handler,
+ .state_change = chcr_uld_state_change,
+ .control = chcr_uld_control,
+};
+
+int assign_chcr_device(struct chcr_dev **dev)
+{
+ struct uld_ctx *u_ctx;
+
+ /*
+ * Which device to use if multiple devices are available TODO
+ * May be select the device based on round robin. One session
+ * must go to the same device to maintain the ordering.
+ */
+ mutex_lock(&dev_mutex); /* TODO ? */
+ u_ctx = list_first_entry(&uld_ctx_list, struct uld_ctx, entry);
+ if (!u_ctx) {
+ mutex_unlock(&dev_mutex);
+ goto err;
+ }
+
+ *dev = u_ctx->dev;
+ mutex_unlock(&dev_mutex);
+ return 0;
+err:
+ return -ENXIO;
+}
+
+static int chcr_dev_add(struct uld_ctx *u_ctx)
+{
+ struct chcr_dev *dev;
+
+ dev = kzalloc(sizeof(*dev), GFP_KERNEL);
+ if (!dev)
+ goto err;
+
+ spin_lock_init(&dev->lock_chcr_dev);
+ u_ctx->dev = dev;
+ dev->u_ctx = u_ctx;
+ atomic_inc(&dev_count);
+ return 0;
+err:
+ pr_err("CHCR Cannot allocate crypto dev\n");
+ return -ENXIO;
+}
+
+static int chcr_dev_remove(struct uld_ctx *u_ctx)
+{
+ kfree(u_ctx->dev);
+ u_ctx->dev = NULL;
+ atomic_dec(&dev_count);
+ return 0;
+}
+
+static int cpl_fw6_pld_handler(struct chcr_dev *dev,
+ unsigned char *input)
+{
+ struct crypto_async_request *req;
+ struct cpl_fw6_pld *fw6_pld;
+ u64 cookie;
+ u32 ack_err_status = 0;
+ int error_status = 0;
+
+ fw6_pld = (struct cpl_fw6_pld *)input;
+ cookie = be64_to_cpu(fw6_pld->data[1]);
+ req = (struct crypto_async_request *)cookie;
+
+ ack_err_status =
+ ntohl(*(__be32 *)((unsigned char *)&fw6_pld->data[0] + 4));
+ if (ack_err_status) {
+ if (CHK_MAC_ERR_BIT(ack_err_status) ||
+ CHK_PAD_ERR_BIT(ack_err_status))
+ error_status = -EINVAL;
+ }
+ /* call completion callback with failure status */
+ if (req) {
+ if (!chcr_handle_resp(req, input, error_status))
+ req->complete(req, error_status);
+ else
+ return -EINVAL;
+ } else {
+ pr_err("Incorrect request address from the firmware\n");
+ return -EFAULT;
+ }
+ return 0;
+}
+
+int chcr_send_wr(struct sk_buff *skb)
+{
+ return cxgb4_crypto_send(skb->dev, skb);
+}
+
+static void *chcr_uld_add(const struct cxgb4_lld_info *lld)
+{
+ struct uld_ctx *u_ctx;
+ /* Create the device and add it in the device list */
+ u_ctx = kzalloc(sizeof(*u_ctx), GFP_KERNEL);
+ if (!u_ctx) {
+ u_ctx = ERR_PTR(-ENOMEM);
+ goto out;
+ }
+ if (!(lld->ulp_crypto & ULP_CRYPTO_LOOKASIDE)) {
+ u_ctx = ERR_PTR(-ENOMEM);
+ goto out;
+ }
+ u_ctx->lldi = *lld;
+ mutex_lock(&dev_mutex);
+ list_add_tail(&u_ctx->entry, &uld_ctx_list);
+ mutex_unlock(&dev_mutex);
+out:
+ return u_ctx;
+}
+
+int chcr_uld_rx_handler(void *handle, const __be64 *rsp,
+ const struct pkt_gl *pgl)
+{
+ struct uld_ctx *u_ctx = (struct uld_ctx *)handle;
+ struct chcr_dev *dev = u_ctx->dev;
+ const struct cpl_act_establish *rpl = (struct cpl_act_establish
+ *)rsp;
+
+ if (rpl->ot.opcode != CPL_FW6_PLD) {
+ pr_err("Unsupported opcode\n");
+ return 0;
+ }
+
+ if (!pgl)
+ work_handlers[rpl->ot.opcode](dev, (unsigned char *)&rsp[1]);
+ else
+ work_handlers[rpl->ot.opcode](dev, pgl->va);
+ return 0;
+}
+
+static int chcr_uld_state_change(void *handle, enum cxgb4_state state)
+{
+ struct uld_ctx *u_ctx = handle;
+ int ret = 0;
+
+ switch (state) {
+ case CXGB4_STATE_UP:
+ if (!u_ctx->dev) {
+ ret = chcr_dev_add(u_ctx);
+ if (ret != 0)
+ return ret;
+ }
+ if (atomic_read(&dev_count) == 1)
+ ret = start_crypto();
+ break;
+
+ case CXGB4_STATE_DETACH:
+ if (u_ctx->dev) {
+ mutex_lock(&dev_mutex);
+ chcr_dev_remove(u_ctx);
+ mutex_unlock(&dev_mutex);
+ }
+ if (!atomic_read(&dev_count))
+ stop_crypto();
+ break;
+
+ case CXGB4_STATE_START_RECOVERY:
+ case CXGB4_STATE_DOWN:
+ default:
+ break;
+ }
+ return ret;
+}
+
+static int chcr_uld_control(void *handle, enum cxgb4_control control, ...)
+{
+ return 0;
+}
+
+static int __init chcr_crypto_init(void)
+{
+ if (cxgb4_register_uld(CXGB4_ULD_CRYPTO, &chcr_uld_info))
+ pr_err("ULD register fail: No chcr crypto support in cxgb4");
+
+ return 0;
+}
+
+static void __exit chcr_crypto_exit(void)
+{
+ struct uld_ctx *u_ctx, *tmp;
+
+ if (atomic_read(&dev_count))
+ stop_crypto();
+
+ /* Remove all devices from list */
+ mutex_lock(&dev_mutex);
+ list_for_each_entry_safe(u_ctx, tmp, &uld_ctx_list, entry) {
+ if (u_ctx->dev)
+ chcr_dev_remove(u_ctx);
+ kfree(u_ctx);
+ }
+ mutex_unlock(&dev_mutex);
+ cxgb4_unregister_uld(CXGB4_ULD_CRYPTO);
+}
+
+module_init(chcr_crypto_init);
+module_exit(chcr_crypto_exit);
+
+MODULE_DESCRIPTION("Crypto Co-processor for Chelsio Terminator cards.");
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Chelsio Communications");
+MODULE_VERSION(DRV_VERSION);
diff --git a/drivers/crypto/chelsio/chcr_core.h b/drivers/crypto/chelsio/chcr_core.h
new file mode 100644
index 0000000..35b7cf24
--- /dev/null
+++ b/drivers/crypto/chelsio/chcr_core.h
@@ -0,0 +1,85 @@
+/*
+ * This file is part of the Chelsio T6 Crypto driver for Linux.
+ *
+ * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#ifndef __CHCR_CORE_H__
+#define __CHCR_CORE_H__
+
+#include <crypto/algapi.h>
+#include "t4_hw.h"
+#include "cxgb4.h"
+#include "cxgb4_uld.h"
+
+#define DRV_MODULE_NAME "chcr"
+#define DRV_VERSION "1.0.0.0"
+
+#define MAX_PENDING_REQ_TO_HW 20
+#define CHCR_TEST_RESPONSE_TIMEOUT 1000
+
+#define PAD_ERROR_BIT 1
+#define CHK_PAD_ERR_BIT(x) (((x) >> PAD_ERROR_BIT) & 1)
+
+#define MAC_ERROR_BIT 0
+#define CHK_MAC_ERR_BIT(x) (((x) >> MAC_ERROR_BIT) & 1)
+
+struct uld_ctx;
+
+struct chcr_dev {
+ /* Request submited to h/w and waiting for response. */
+ spinlock_t lock_chcr_dev;
+ struct crypto_queue pending_queue;
+ struct uld_ctx *u_ctx;
+ unsigned char tx_channel_id;
+};
+
+struct uld_ctx {
+ struct list_head entry;
+ struct cxgb4_lld_info lldi;
+ struct chcr_dev *dev;
+};
+
+void *chcr_dequeue_request(struct crypto_queue *queue);
+int assign_chcr_device(struct chcr_dev **dev);
+int chcr_send_wr(struct sk_buff *skb);
+int start_crypto(void);
+int stop_crypto(void);
+int chcr_enqueue_request(struct crypto_queue *queue,
+ struct crypto_async_request *req);
+void chcr_delete_request(struct crypto_queue *queue,
+ struct crypto_async_request *req);
+int chcr_uld_rx_handler(void *handle, const __be64 *rsp,
+ const struct pkt_gl *pgl);
+int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input,
+ int err);
+#endif /* __CHCR_CORE_H__ */
diff --git a/drivers/crypto/chelsio/chcr_crypto.h b/drivers/crypto/chelsio/chcr_crypto.h
new file mode 100644
index 0000000..d673c11
--- /dev/null
+++ b/drivers/crypto/chelsio/chcr_crypto.h
@@ -0,0 +1,255 @@
+/*
+ * This file is part of the Chelsio T6 Crypto driver for Linux.
+ *
+ * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#ifndef __CHCR_CRYPTO_H__
+#define __CHCR_CRYPTO_H__
+
+#define GHASH_BLOCK_SIZE 16
+#define GHASH_DIGEST_SIZE 16
+
+#define CCM_B0_SIZE 16
+#define CCM_AAD_FIELD_SIZE 2
+#define T5_MAX_AAD_SIZE 512
+
+/* Define following if h/w is not dropping the AAD and IV data before
+ * giving the processed data
+ */
+#define CHCR_NO_DATA_DROP_IN_HW
+
+#define CHCR_CRA_PRIORITY 300
+
+#define CHCR_AES_MAX_KEY_LEN (2 * (AES_MAX_KEY_SIZE)) /* consider xts */
+#define CHCR_MAX_CRYPTO_IV_LEN 16 /* AES IV len */
+
+#define CHCR_MAX_AUTHENC_AES_KEY_LEN 32 /* max aes key length*/
+#define CHCR_MAX_AUTHENC_SHA_KEY_LEN 128 /* max sha key length*/
+
+#define CHCR_GIVENCRYPT_OP 2
+/* CPL/SCMD parameters */
+
+#define CHCR_ENCRYPT_OP 0
+#define CHCR_DECRYPT_OP 1
+
+#define CHCR_SCMD_SEQ_NO_CTRL_32BIT 1
+#define CHCR_SCMD_SEQ_NO_CTRL_48BIT 2
+#define CHCR_SCMD_SEQ_NO_CTRL_64BIT 3
+
+#define CHCR_SCMD_PROTO_VERSION_GENERIC 4
+
+#define CHCR_SCMD_AUTH_CTRL_AUTH_CIPHER 0
+#define CHCR_SCMD_AUTH_CTRL_CIPHER_AUTH 1
+
+#define CHCR_SCMD_CIPHER_MODE_NOP 0
+#define CHCR_SCMD_CIPHER_MODE_AES_CBC 1
+#define CHCR_SCMD_CIPHER_MODE_AES_GCM 2
+#define CHCR_SCMD_CIPHER_MODE_AES_CTR 3
+#define CHCR_SCMD_CIPHER_MODE_GENERIC_AES 4
+#define CHCR_SCMD_CIPHER_MODE_IPSEC_ESP 5
+#define CHCR_SCMD_CIPHER_MODE_AES_XTS 6
+#define CHCR_SCMD_CIPHER_MODE_AES_CCM 7
+
+#define CHCR_SCMD_AUTH_MODE_NOP 0
+#define CHCR_SCMD_AUTH_MODE_SHA1 1
+#define CHCR_SCMD_AUTH_MODE_SHA224 2
+#define CHCR_SCMD_AUTH_MODE_SHA256 3
+#define CHCR_SCMD_AUTH_MODE_GHASH 4
+#define CHCR_SCMD_AUTH_MODE_SHA512_224 5
+#define CHCR_SCMD_AUTH_MODE_SHA512_256 6
+#define CHCR_SCMD_AUTH_MODE_SHA512_384 7
+#define CHCR_SCMD_AUTH_MODE_SHA512_512 8
+#define CHCR_SCMD_AUTH_MODE_CBCMAC 9
+#define CHCR_SCMD_AUTH_MODE_CMAC 10
+
+#define CHCR_SCMD_HMAC_CTRL_NOP 0
+#define CHCR_SCMD_HMAC_CTRL_NO_TRUNC 1
+#define CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366 2
+#define CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT 3
+#define CHCR_SCMD_HMAC_CTRL_IPSEC_64BIT 4
+
+#define CHCR_SCMD_IVGEN_CTRL_HW 0
+#define CHCR_SCMD_IVGEN_CTRL_SW 1
+/* This are not really mac key size. They are intermediate values
+ * of sha engine and its size
+ */
+#define CHCR_KEYCTX_MAC_KEY_SIZE_128 0
+#define CHCR_KEYCTX_MAC_KEY_SIZE_160 1
+#define CHCR_KEYCTX_MAC_KEY_SIZE_192 2
+#define CHCR_KEYCTX_MAC_KEY_SIZE_256 3
+#define CHCR_KEYCTX_MAC_KEY_SIZE_512 4
+#define CHCR_KEYCTX_CIPHER_KEY_SIZE_128 0
+#define CHCR_KEYCTX_CIPHER_KEY_SIZE_192 1
+#define CHCR_KEYCTX_CIPHER_KEY_SIZE_256 2
+#define CHCR_KEYCTX_NO_KEY 15
+
+#define CHCR_CPL_FW4_PLD_IV_OFFSET (5 * 64) /* bytes. flt #5 and #6 */
+#define CHCR_CPL_FW4_PLD_HASH_RESULT_OFFSET (7 * 64) /* bytes. flt #7 */
+#define CHCR_CPL_FW4_PLD_DATA_SIZE (4 * 64) /* bytes. flt #4 to #7 */
+
+#define KEY_CONTEXT_HDR_SALT_AND_PAD 16
+#define flits_to_bytes(x) (x * 8)
+
+#define IV_NOP 0
+#define IV_IMMEDIATE 1
+#define IV_DSGL 2
+#define AEAD_H_SIZE 16
+
+#define CRYPTO_ALG_SUB_TYPE_MASK 0x0f000000
+#define CRYPTO_ALG_SUB_TYPE_HASH_HMAC 0x01000000
+#define CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106 0x02000000
+#define CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC 0x03000000
+#define CRYPTO_ALG_SUB_TYPE_AEAD_CCM 0x04000000
+#define CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309 0x05000000
+#define CRYPTO_ALG_SUB_TYPE_CTR 0x06000000
+
+#define CRYPTO_ALG_TYPE_HMAC (CRYPTO_ALG_TYPE_AHASH |\
+ CRYPTO_ALG_SUB_TYPE_HASH_HMAC)
+
+#define MAX_SALT 4
+#define MAX_SCRATCH_PAD_SIZE 32
+#define MAX_AEAD_IV_LENGTH 20
+
+#define CHCR_HASH_MAX_BLOCK_SIZE_64 64
+#define CHCR_HASH_MAX_BLOCK_SIZE_128 128
+
+/* Aligned to 128 bit boundary */
+struct _key_ctx {
+ __be32 ctx_hdr;
+ u8 salt[MAX_SALT];
+ __be64 reserverd;
+ unsigned char key[0];
+};
+
+struct ablk_ctx {
+ u8 enc;
+ unsigned int processed_len;
+ __be32 key_ctx_hdr;
+ unsigned int enckey_len;
+ unsigned int dst_nents;
+ struct scatterlist iv_sg;
+ u8 key[CHCR_AES_MAX_KEY_LEN];
+ u8 iv[CHCR_MAX_CRYPTO_IV_LEN];
+ unsigned char ciph_mode;
+};
+
+struct aead_ctx {
+ __be32 key_ctx_hdr;
+ unsigned int enckey_len;
+ unsigned int authkey_len;
+ unsigned int dst_nents;
+ struct scatterlist iv_sg;
+ unsigned char auth_mode;
+ u8 salt[MAX_SALT];
+ u8 key[128];
+ u8 iv[CHCR_MAX_CRYPTO_IV_LEN];
+ u8 ghash_h[AEAD_H_SIZE]; /* used only in GCM(AES) algorithms */
+ u8 h_iopad[2 * CHCR_HASH_MAX_DIGEST_SIZE];
+ unsigned char *dec_rrkey;
+};
+
+struct hmac_ctx {
+ struct shash_desc *desc;
+ u8 ipad[CHCR_HASH_MAX_BLOCK_SIZE_128];
+ u8 opad[CHCR_HASH_MAX_BLOCK_SIZE_128];
+};
+
+struct __crypto_ctx {
+ struct hmac_ctx hmacctx[0];
+ struct ablk_ctx ablkctx[0];
+ struct aead_ctx aeadctx[0];
+};
+
+struct chcr_context {
+ struct chcr_dev *dev;
+ unsigned char tx_channel_id;
+ struct __crypto_ctx crypto_ctx[0];
+};
+
+struct chcr_ahash_req_ctx {
+ u32 result;
+ char bfr[CHCR_HASH_MAX_BLOCK_SIZE_128];
+ u8 bfr_len;
+ /* DMA the partial hash in it */
+ u8 partial_hash[CHCR_HASH_MAX_DIGEST_SIZE];
+ u64 data_len; /* Data len till time */
+ void *dummy_payload_ptr;
+ /* SKB which is being sent to the hardware for processing */
+ struct sk_buff *skb;
+};
+
+struct chcr_aead_req_ctx {
+ unsigned char iv[MAX_AEAD_IV_LENGTH];
+ unsigned char scratch_pad[MAX_SCRATCH_PAD_SIZE];
+ struct sk_buff *skb;
+};
+
+struct chcr_blkcipher_req_ctx {
+ struct sk_buff *skb;
+};
+
+struct chcr_alg_template {
+ u32 type;
+ u32 is_registered;
+ union {
+ struct crypto_alg crypto;
+ struct ahash_alg hash;
+ struct aead_alg aead;
+ } alg;
+};
+
+struct chcr_req_ctx {
+ union {
+ struct ahash_request *ahash_req;
+ struct aead_request *aead_req;
+ struct ablkcipher_request *ablk_req;
+ } req;
+ union {
+ struct chcr_ahash_req_ctx *ahash_ctx;
+ struct chcr_aead_req_ctx *aead_ctx;
+ struct chcr_blkcipher_req_ctx *ablk_ctx;
+ } ctx;
+};
+
+struct sge_opaque_hdr {
+ void *dev;
+ dma_addr_t addr[MAX_SKB_FRAGS + 1];
+};
+
+typedef struct sk_buff *(*create_wr_t)(struct crypto_async_request *req,
+ struct chcr_context *ctx,
+ unsigned short qid,
+ unsigned short op_type);
+
+#endif /* __CHCR_CRYPTO_H__ */
+
--
1.7.10.1
^ permalink raw reply related
* [PATCH 3/3] crypto: Added Chelsio Menu to the Kconfig file
From: Yeshaswi M R Gowda @ 2016-07-11 18:28 UTC (permalink / raw)
To: hariprasad, netdev, linux-kernel, herbert, davem, linux-crypto,
jlulla, atul.gupta, harsh
Cc: Yeshaswi M R Gowda
In-Reply-To: <1468261688-24525-1-git-send-email-yeshaswi@chelsio.com>
Adds the config entry for the Chelsio Crypto Driver, Makefile changes
for the same.
Signed-off-by: Yeshaswi M R Gowda <yeshaswi@chelsio.com>
---
drivers/crypto/Kconfig | 2 ++
drivers/crypto/Makefile | 1 +
2 files changed, 3 insertions(+)
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig
index d77ba2f..b44faf0 100644
--- a/drivers/crypto/Kconfig
+++ b/drivers/crypto/Kconfig
@@ -537,4 +537,6 @@ config CRYPTO_DEV_ROCKCHIP
This driver interfaces with the hardware crypto accelerator.
Supporting cbc/ecb chainmode, and aes/des/des3_ede cipher mode.
+source "drivers/crypto/chelsio/Kconfig"
+
endif # CRYPTO_HW
diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile
index 3c6432d..ad7250f 100644
--- a/drivers/crypto/Makefile
+++ b/drivers/crypto/Makefile
@@ -31,3 +31,4 @@ obj-$(CONFIG_CRYPTO_DEV_QCE) += qce/
obj-$(CONFIG_CRYPTO_DEV_VMX) += vmx/
obj-$(CONFIG_CRYPTO_DEV_SUN4I_SS) += sunxi-ss/
obj-$(CONFIG_CRYPTO_DEV_ROCKCHIP) += rockchip/
+obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chelsio/
--
1.7.10.1
^ permalink raw reply related
* Re: [PATCH 2/3] chcr: Support for Chelsio's Crypto Hardware
From: Joe Perches @ 2016-07-11 18:57 UTC (permalink / raw)
To: Yeshaswi M R Gowda, hariprasad, netdev, linux-kernel, herbert,
davem, linux-crypto, jlulla, atul.gupta, harsh
In-Reply-To: <1468261688-24525-3-git-send-email-yeshaswi@chelsio.com>
On Mon, 2016-07-11 at 11:28 -0700, Yeshaswi M R Gowda wrote:
> The Chelsio's Crypto Hardware can perform the following operations:
> SHA1, SHA224, SHA256, SHA384 and SHA512, HMAC(SHA1), HMAC(SHA224),
> HMAC(SHA256), HMAC(SHA384), HAMC(SHA512), AES-128-CBC, AES-192-CBC,
> AES-256-CBC, AES-128-XTS, AES-256-XTS
>
> This patch implements the driver for above mentioned features.
trivial notes:
> diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c
[]
> +int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input,
> + int error_status)
> +{
[]
> + case CRYPTO_ALG_TYPE_BLKCIPHER:
> + ctx_req.req.ablk_req = (struct ablkcipher_request *)req;
> + ctx_req.ctx.ablk_ctx =
> + ablkcipher_request_ctx(ctx_req.req.ablk_req);
> + if (error_status)
> + goto dma_unmap_blkcipher;
> + fw6_pld = (struct cpl_fw6_pld *)input;
> + memcpy(ctx_req.req.ablk_req->info, &fw6_pld->data[2],
> + AES_BLOCK_SIZE);
> +dma_unmap_blkcipher:
> + dma_unmap_sg(&u_ctx->lldi.pdev->dev, ctx_req.req.ablk_req->dst,
> + ABLK_CTX(ctx)->dst_nents, DMA_FROM_DEVICE);
> + if (ctx_req.ctx.ablk_ctx->skb) {
> + kfree_skb(ctx_req.ctx.ablk_ctx->skb);
> + ctx_req.ctx.ablk_ctx->skb = NULL;
> + }
> + break;
This case label is only used here right?
This would be better without the goto
[]
> + if (IS_ERR(base_hash)) {
> + pr_err("Can not allocate sha-generic algo.\n");
> + return (void *)base_hash;
> + }
Please add
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
before any #include to prefix any pr_<level> uses.
[]
> +/*
> + * chcr_register_alg - Register crypto algorithms with kernel framework.
> + */
> +static int chcr_register_alg(void)
> +{
> + struct crypto_alg ai;
> + int err = 0, i;
> + char *name = NULL;
> +
> + for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
> + if (driver_algs[i].is_registered)
> + continue;
> + switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) {
> + case CRYPTO_ALG_TYPE_ABLKCIPHER:
> + err = crypto_register_alg(&driver_algs[i].alg.crypto);
> + name = driver_algs[i].alg.crypto.cra_driver_name;
> + break;
> + case CRYPTO_ALG_TYPE_AHASH:
This could be clearer with a temporary for
driver_algs[i].alg.hash
<whatever type *> hash = &driver_algs[i].alg.hash;
> + driver_algs[i].alg.hash.update = chcr_ahash_update;
hash->update = chcr_ahash_update;
etc...
> + driver_algs[i].alg.hash.final = chcr_ahash_final;
> + driver_algs[i].alg.hash.finup = chcr_ahash_finup;
> + driver_algs[i].alg.hash.digest = chcr_ahash_digest;
> + driver_algs[i].alg.hash.export = chcr_ahash_export;
> + driver_algs[i].alg.hash.import = chcr_ahash_import;
> + driver_algs[i].alg.hash.halg.statesize =
> + sizeof(struct chcr_ahash_req_ctx);
Even with this sort of change, a lot of barely >80 column lines
are split making the code a bit less readable.
It might be better to avoid splitting these long lines and
ignore the >80 column limits occasionally.
^ permalink raw reply
* [PATCH 2/2] crypto: vmx - Adding support for XTS
From: Paulo Flabiano Smorigo @ 2016-07-11 19:07 UTC (permalink / raw)
To: linux-kernel
Cc: Leonidas S. Barbosa, Paulo Flabiano Smorigo,
Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman,
Herbert Xu, David S. Miller,
open list:IBM Power VMX Cryptographic instructions,
open list:LINUX FOR POWERPC (32-BIT AND 64-BIT)
In-Reply-To: <1468264060-22769-1-git-send-email-pfsmorigo@linux.vnet.ibm.com>
From: "Leonidas S. Barbosa" <leosilva@linux.vnet.ibm.com>
This patch add XTS support using VMX-crypto driver.
Signed-off-by: Leonidas S. Barbosa <leosilva@linux.vnet.ibm.com>
Signed-off-by: Paulo Flabiano Smorigo <pfsmorigo@linux.vnet.ibm.com>
---
drivers/crypto/vmx/Makefile | 2 +-
drivers/crypto/vmx/aes_xts.c | 187 +++++++++++++++++++++++++++++++++++++++++++
drivers/crypto/vmx/vmx.c | 2 +
3 files changed, 190 insertions(+), 1 deletion(-)
create mode 100644 drivers/crypto/vmx/aes_xts.c
diff --git a/drivers/crypto/vmx/Makefile b/drivers/crypto/vmx/Makefile
index d28ab96..de6e241 100644
--- a/drivers/crypto/vmx/Makefile
+++ b/drivers/crypto/vmx/Makefile
@@ -1,5 +1,5 @@
obj-$(CONFIG_CRYPTO_DEV_VMX_ENCRYPT) += vmx-crypto.o
-vmx-crypto-objs := vmx.o aesp8-ppc.o ghashp8-ppc.o aes.o aes_cbc.o aes_ctr.o ghash.o
+vmx-crypto-objs := vmx.o aesp8-ppc.o ghashp8-ppc.o aes.o aes_cbc.o aes_ctr.o aes_xts.o ghash.o
ifeq ($(CONFIG_CPU_LITTLE_ENDIAN),y)
TARGET := linux-ppc64le
diff --git a/drivers/crypto/vmx/aes_xts.c b/drivers/crypto/vmx/aes_xts.c
new file mode 100644
index 0000000..62e698c
--- /dev/null
+++ b/drivers/crypto/vmx/aes_xts.c
@@ -0,0 +1,187 @@
+/**
+ * AES XTS routines supporting VMX In-core instructions on Power 8
+ *
+ * Copyright (C) 2015 International Business Machines Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundations; version 2 only.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY of FITNESS FOR A PARTICUPAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * Author: Leonidas S. Barbosa <leosilva@linux.vnet.ibm.com>
+ */
+
+#include <linux/types.h>
+#include <linux/err.h>
+#include <linux/crypto.h>
+#include <linux/delay.h>
+#include <linux/hardirq.h>
+#include <asm/switch_to.h>
+#include <crypto/aes.h>
+#include <crypto/scatterwalk.h>
+
+#include "aesp8-ppc.h"
+
+struct p8_aes_xts_ctx {
+ struct crypto_blkcipher *fallback;
+ struct aes_key enc_key;
+ struct aes_key dec_key;
+ struct aes_key tweak_key;
+};
+
+static int p8_aes_xts_init(struct crypto_tfm *tfm)
+{
+ const char *alg;
+ struct crypto_blkcipher *fallback;
+ struct p8_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ if (!(alg = crypto_tfm_alg_name(tfm))) {
+ printk(KERN_ERR "Failed to get algorithm name.\n");
+ return -ENOENT;
+ }
+
+ fallback =
+ crypto_alloc_blkcipher(alg, 0, CRYPTO_ALG_NEED_FALLBACK);
+ if (IS_ERR(fallback)) {
+ printk(KERN_ERR
+ "Failed to allocate transformation for '%s': %ld\n",
+ alg, PTR_ERR(fallback));
+ return PTR_ERR(fallback);
+ }
+ printk(KERN_INFO "Using '%s' as fallback implementation.\n",
+ crypto_tfm_alg_driver_name((struct crypto_tfm *) fallback));
+
+ crypto_blkcipher_set_flags(
+ fallback,
+ crypto_blkcipher_get_flags((struct crypto_blkcipher *)tfm));
+ ctx->fallback = fallback;
+
+ return 0;
+}
+
+static void p8_aes_xts_exit(struct crypto_tfm *tfm)
+{
+ struct p8_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ if (ctx->fallback) {
+ crypto_free_blkcipher(ctx->fallback);
+ ctx->fallback = NULL;
+ }
+}
+
+static int p8_aes_xts_setkey(struct crypto_tfm *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ int ret;
+ struct p8_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ if (keylen % 2)
+ return -EINVAL;
+
+ preempt_disable();
+ pagefault_disable();
+ enable_kernel_vsx();
+ ret = aes_p8_set_encrypt_key(key + keylen/2, (keylen/2) * 8, &ctx->tweak_key);
+ ret += aes_p8_set_encrypt_key(key, (keylen/2) * 8, &ctx->enc_key);
+ ret += aes_p8_set_decrypt_key(key, (keylen/2) * 8, &ctx->dec_key);
+ disable_kernel_vsx();
+ pagefault_enable();
+ preempt_enable();
+
+ ret += crypto_blkcipher_setkey(ctx->fallback, key, keylen);
+ return ret;
+}
+
+static int p8_aes_xts_crypt(struct blkcipher_desc *desc,
+ struct scatterlist *dst,
+ struct scatterlist *src,
+ unsigned int nbytes, int enc)
+{
+ int ret;
+ u8 tweak[AES_BLOCK_SIZE];
+ u8 *iv;
+ struct blkcipher_walk walk;
+ struct p8_aes_xts_ctx *ctx =
+ crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm));
+ struct blkcipher_desc fallback_desc = {
+ .tfm = ctx->fallback,
+ .info = desc->info,
+ .flags = desc->flags
+ };
+
+ if (in_interrupt()) {
+ ret = enc ? crypto_blkcipher_encrypt(&fallback_desc, dst, src, nbytes) :
+ crypto_blkcipher_decrypt(&fallback_desc, dst, src, nbytes);
+ } else {
+ preempt_disable();
+ pagefault_disable();
+ enable_kernel_vsx();
+
+ blkcipher_walk_init(&walk, dst, src, nbytes);
+
+ iv = (u8 *)walk.iv;
+ ret = blkcipher_walk_virt(desc, &walk);
+ aes_p8_encrypt(iv, tweak, &ctx->tweak_key);
+
+ while ((nbytes = walk.nbytes)) {
+ if (enc)
+ aes_p8_xts_encrypt(walk.src.virt.addr, walk.dst.virt.addr,
+ nbytes & AES_BLOCK_MASK, &ctx->enc_key, NULL, tweak);
+ else
+ aes_p8_xts_decrypt(walk.src.virt.addr, walk.dst.virt.addr,
+ nbytes & AES_BLOCK_MASK, &ctx->dec_key, NULL, tweak);
+
+ nbytes &= AES_BLOCK_SIZE - 1;
+ ret = blkcipher_walk_done(desc, &walk, nbytes);
+ }
+
+ disable_kernel_vsx();
+ pagefault_enable();
+ preempt_enable();
+ }
+ return ret;
+}
+
+static int p8_aes_xts_encrypt(struct blkcipher_desc *desc,
+ struct scatterlist *dst,
+ struct scatterlist *src, unsigned int nbytes)
+{
+ return p8_aes_xts_crypt(desc, dst, src, nbytes, 1);
+}
+
+static int p8_aes_xts_decrypt(struct blkcipher_desc *desc,
+ struct scatterlist *dst,
+ struct scatterlist *src, unsigned int nbytes)
+{
+ return p8_aes_xts_crypt(desc, dst, src, nbytes, 0);
+}
+
+struct crypto_alg p8_aes_xts_alg = {
+ .cra_name = "xts(aes)",
+ .cra_driver_name = "p8_aes_xts",
+ .cra_module = THIS_MODULE,
+ .cra_priority = 2000,
+ .cra_type = &crypto_blkcipher_type,
+ .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_NEED_FALLBACK,
+ .cra_alignmask = 0,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct p8_aes_xts_ctx),
+ .cra_init = p8_aes_xts_init,
+ .cra_exit = p8_aes_xts_exit,
+ .cra_blkcipher = {
+ .ivsize = AES_BLOCK_SIZE,
+ .min_keysize = 2 * AES_MIN_KEY_SIZE,
+ .max_keysize = 2 * AES_MAX_KEY_SIZE,
+ .setkey = p8_aes_xts_setkey,
+ .encrypt = p8_aes_xts_encrypt,
+ .decrypt = p8_aes_xts_decrypt,
+ }
+};
diff --git a/drivers/crypto/vmx/vmx.c b/drivers/crypto/vmx/vmx.c
index e163d57..f688c32 100644
--- a/drivers/crypto/vmx/vmx.c
+++ b/drivers/crypto/vmx/vmx.c
@@ -31,10 +31,12 @@ extern struct shash_alg p8_ghash_alg;
extern struct crypto_alg p8_aes_alg;
extern struct crypto_alg p8_aes_cbc_alg;
extern struct crypto_alg p8_aes_ctr_alg;
+extern struct crypto_alg p8_aes_xts_alg;
static struct crypto_alg *algs[] = {
&p8_aes_alg,
&p8_aes_cbc_alg,
&p8_aes_ctr_alg,
+ &p8_aes_xts_alg,
NULL,
};
--
2.5.5
^ permalink raw reply related
* [PATCH 1/2] crypto: vmx - Adding asm subroutines for XTS
From: Paulo Flabiano Smorigo @ 2016-07-11 19:07 UTC (permalink / raw)
To: linux-kernel
Cc: Paulo Flabiano Smorigo, Leonidas S. Barbosa,
Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman,
Herbert Xu, David S. Miller,
open list:IBM Power VMX Cryptographic instructions,
open list:LINUX FOR POWERPC (32-BIT AND 64-BIT)
This patch add XTS subroutines using VMX-crypto driver.
It gives a boost of 20 times using XTS.
These code has been adopted from OpenSSL project in collaboration
with the original author (Andy Polyakov <appro@openssl.org>).
Signed-off-by: Leonidas S. Barbosa <leosilva@linux.vnet.ibm.com>
Signed-off-by: Paulo Flabiano Smorigo <pfsmorigo@linux.vnet.ibm.com>
---
drivers/crypto/vmx/aesp8-ppc.h | 4 +
drivers/crypto/vmx/aesp8-ppc.pl | 1865 ++++++++++++++++++++++++++++++++++++++-
2 files changed, 1867 insertions(+), 2 deletions(-)
diff --git a/drivers/crypto/vmx/aesp8-ppc.h b/drivers/crypto/vmx/aesp8-ppc.h
index 4cd34ee..01972e1 100644
--- a/drivers/crypto/vmx/aesp8-ppc.h
+++ b/drivers/crypto/vmx/aesp8-ppc.h
@@ -19,3 +19,7 @@ void aes_p8_cbc_encrypt(const u8 *in, u8 *out, size_t len,
void aes_p8_ctr32_encrypt_blocks(const u8 *in, u8 *out,
size_t len, const struct aes_key *key,
const u8 *iv);
+void aes_p8_xts_encrypt(const u8 *in, u8 *out, size_t len,
+ const struct aes_key *key1, const struct aes_key *key2, u8 *iv);
+void aes_p8_xts_decrypt(const u8 *in, u8 *out, size_t len,
+ const struct aes_key *key1, const struct aes_key *key2, u8 *iv);
diff --git a/drivers/crypto/vmx/aesp8-ppc.pl b/drivers/crypto/vmx/aesp8-ppc.pl
index 2280539..813ffcc 100644
--- a/drivers/crypto/vmx/aesp8-ppc.pl
+++ b/drivers/crypto/vmx/aesp8-ppc.pl
@@ -1,4 +1,11 @@
-#!/usr/bin/env perl
+#! /usr/bin/env perl
+# Copyright 2014-2016 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the OpenSSL license (the "License"). You may not use
+# this file except in compliance with the License. You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
@@ -20,6 +27,19 @@
# instructions are interleaved. It's reckoned that eventual
# misalignment penalties at page boundaries are in average lower
# than additional overhead in pure AltiVec approach.
+#
+# May 2016
+#
+# Add XTS subroutine, 9x on little- and 12x improvement on big-endian
+# systems were measured.
+#
+######################################################################
+# Current large-block performance in cycles per byte processed with
+# 128-bit key (less is better).
+#
+# CBC en-/decrypt CTR XTS
+# POWER8[le] 3.96/0.72 0.74 1.1
+# POWER8[be] 3.75/0.65 0.66 1.0
$flavour = shift;
@@ -1875,6 +1895,1847 @@ Lctr32_enc8x_done:
___
}} }}}
+#########################################################################
+{{{ # XTS procedures #
+# int aes_p8_xts_[en|de]crypt(const char *inp, char *out, size_t len, #
+# const AES_KEY *key1, const AES_KEY *key2, #
+# [const] unsigned char iv[16]); #
+# If $key2 is NULL, then a "tweak chaining" mode is engaged, in which #
+# input tweak value is assumed to be encrypted already, and last tweak #
+# value, one suitable for consecutive call on same chunk of data, is #
+# written back to original buffer. In addition, in "tweak chaining" #
+# mode only complete input blocks are processed. #
+
+my ($inp,$out,$len,$key1,$key2,$ivp,$rounds,$idx) = map("r$_",(3..10));
+my ($rndkey0,$rndkey1,$inout) = map("v$_",(0..2));
+my ($output,$inptail,$inpperm,$leperm,$keyperm) = map("v$_",(3..7));
+my ($tweak,$seven,$eighty7,$tmp,$tweak1) = map("v$_",(8..12));
+my $taillen = $key2;
+
+ ($inp,$idx) = ($idx,$inp); # reassign
+
+$code.=<<___;
+.globl .${prefix}_xts_encrypt
+ mr $inp,r3 # reassign
+ li r3,-1
+ ${UCMP}i $len,16
+ bltlr-
+
+ lis r0,0xfff0
+ mfspr r12,256 # save vrsave
+ li r11,0
+ mtspr 256,r0
+
+ vspltisb $seven,0x07 # 0x070707..07
+ le?lvsl $leperm,r11,r11
+ le?vspltisb $tmp,0x0f
+ le?vxor $leperm,$leperm,$seven
+
+ li $idx,15
+ lvx $tweak,0,$ivp # load [unaligned] iv
+ lvsl $inpperm,0,$ivp
+ lvx $inptail,$idx,$ivp
+ le?vxor $inpperm,$inpperm,$tmp
+ vperm $tweak,$tweak,$inptail,$inpperm
+
+ neg r11,$inp
+ lvsr $inpperm,0,r11 # prepare for unaligned load
+ lvx $inout,0,$inp
+ addi $inp,$inp,15 # 15 is not typo
+ le?vxor $inpperm,$inpperm,$tmp
+
+ ${UCMP}i $key2,0 # key2==NULL?
+ beq Lxts_enc_no_key2
+
+ ?lvsl $keyperm,0,$key2 # prepare for unaligned key
+ lwz $rounds,240($key2)
+ srwi $rounds,$rounds,1
+ subi $rounds,$rounds,1
+ li $idx,16
+
+ lvx $rndkey0,0,$key2
+ lvx $rndkey1,$idx,$key2
+ addi $idx,$idx,16
+ ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
+ vxor $tweak,$tweak,$rndkey0
+ lvx $rndkey0,$idx,$key2
+ addi $idx,$idx,16
+ mtctr $rounds
+
+Ltweak_xts_enc:
+ ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
+ vcipher $tweak,$tweak,$rndkey1
+ lvx $rndkey1,$idx,$key2
+ addi $idx,$idx,16
+ ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
+ vcipher $tweak,$tweak,$rndkey0
+ lvx $rndkey0,$idx,$key2
+ addi $idx,$idx,16
+ bdnz Ltweak_xts_enc
+
+ ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
+ vcipher $tweak,$tweak,$rndkey1
+ lvx $rndkey1,$idx,$key2
+ ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
+ vcipherlast $tweak,$tweak,$rndkey0
+
+ li $ivp,0 # don't chain the tweak
+ b Lxts_enc
+
+Lxts_enc_no_key2:
+ li $idx,-16
+ and $len,$len,$idx # in "tweak chaining"
+ # mode only complete
+ # blocks are processed
+Lxts_enc:
+ lvx $inptail,0,$inp
+ addi $inp,$inp,16
+
+ ?lvsl $keyperm,0,$key1 # prepare for unaligned key
+ lwz $rounds,240($key1)
+ srwi $rounds,$rounds,1
+ subi $rounds,$rounds,1
+ li $idx,16
+
+ vslb $eighty7,$seven,$seven # 0x808080..80
+ vor $eighty7,$eighty7,$seven # 0x878787..87
+ vspltisb $tmp,1 # 0x010101..01
+ vsldoi $eighty7,$eighty7,$tmp,15 # 0x870101..01
+
+ ${UCMP}i $len,96
+ bge _aesp8_xts_encrypt6x
+
+ andi. $taillen,$len,15
+ subic r0,$len,32
+ subi $taillen,$taillen,16
+ subfe r0,r0,r0
+ and r0,r0,$taillen
+ add $inp,$inp,r0
+
+ lvx $rndkey0,0,$key1
+ lvx $rndkey1,$idx,$key1
+ addi $idx,$idx,16
+ vperm $inout,$inout,$inptail,$inpperm
+ ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
+ vxor $inout,$inout,$tweak
+ vxor $inout,$inout,$rndkey0
+ lvx $rndkey0,$idx,$key1
+ addi $idx,$idx,16
+ mtctr $rounds
+ b Loop_xts_enc
+
+.align 5
+Loop_xts_enc:
+ ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
+ vcipher $inout,$inout,$rndkey1
+ lvx $rndkey1,$idx,$key1
+ addi $idx,$idx,16
+ ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
+ vcipher $inout,$inout,$rndkey0
+ lvx $rndkey0,$idx,$key1
+ addi $idx,$idx,16
+ bdnz Loop_xts_enc
+
+ ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
+ vcipher $inout,$inout,$rndkey1
+ lvx $rndkey1,$idx,$key1
+ li $idx,16
+ ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
+ vxor $rndkey0,$rndkey0,$tweak
+ vcipherlast $output,$inout,$rndkey0
+
+ le?vperm $tmp,$output,$output,$leperm
+ be?nop
+ le?stvx_u $tmp,0,$out
+ be?stvx_u $output,0,$out
+ addi $out,$out,16
+
+ subic. $len,$len,16
+ beq Lxts_enc_done
+
+ vmr $inout,$inptail
+ lvx $inptail,0,$inp
+ addi $inp,$inp,16
+ lvx $rndkey0,0,$key1
+ lvx $rndkey1,$idx,$key1
+ addi $idx,$idx,16
+
+ subic r0,$len,32
+ subfe r0,r0,r0
+ and r0,r0,$taillen
+ add $inp,$inp,r0
+
+ vsrab $tmp,$tweak,$seven # next tweak value
+ vaddubm $tweak,$tweak,$tweak
+ vsldoi $tmp,$tmp,$tmp,15
+ vand $tmp,$tmp,$eighty7
+ vxor $tweak,$tweak,$tmp
+
+ vperm $inout,$inout,$inptail,$inpperm
+ ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
+ vxor $inout,$inout,$tweak
+ vxor $output,$output,$rndkey0 # just in case $len<16
+ vxor $inout,$inout,$rndkey0
+ lvx $rndkey0,$idx,$key1
+ addi $idx,$idx,16
+
+ mtctr $rounds
+ ${UCMP}i $len,16
+ bge Loop_xts_enc
+
+ vxor $output,$output,$tweak
+ lvsr $inpperm,0,$len # $inpperm is no longer needed
+ vxor $inptail,$inptail,$inptail # $inptail is no longer needed
+ vspltisb $tmp,-1
+ vperm $inptail,$inptail,$tmp,$inpperm
+ vsel $inout,$inout,$output,$inptail
+
+ subi r11,$out,17
+ subi $out,$out,16
+ mtctr $len
+ li $len,16
+Loop_xts_enc_steal:
+ lbzu r0,1(r11)
+ stb r0,16(r11)
+ bdnz Loop_xts_enc_steal
+
+ mtctr $rounds
+ b Loop_xts_enc # one more time...
+
+Lxts_enc_done:
+ ${UCMP}i $ivp,0
+ beq Lxts_enc_ret
+
+ vsrab $tmp,$tweak,$seven # next tweak value
+ vaddubm $tweak,$tweak,$tweak
+ vsldoi $tmp,$tmp,$tmp,15
+ vand $tmp,$tmp,$eighty7
+ vxor $tweak,$tweak,$tmp
+
+ le?vperm $tweak,$tweak,$tweak,$leperm
+ stvx_u $tweak,0,$ivp
+
+Lxts_enc_ret:
+ mtspr 256,r12 # restore vrsave
+ li r3,0
+ blr
+ .long 0
+ .byte 0,12,0x04,0,0x80,6,6,0
+ .long 0
+.size .${prefix}_xts_encrypt,.-.${prefix}_xts_encrypt
+
+.globl .${prefix}_xts_decrypt
+.align 5
+.${prefix}_xts_decrypt:
+ mr $inp,r3 # reassign
+ li r3,-1
+ ${UCMP}i $len,16
+ bltlr-
+
+ lis r0,0xfff8
+ mfspr r12,256 # save vrsave
+ li r11,0
+ mtspr 256,r0
+
+ andi. r0,$len,15
+ neg r0,r0
+ andi. r0,r0,16
+ sub $len,$len,r0
+
+ vspltisb $seven,0x07 # 0x070707..07
+ le?lvsl $leperm,r11,r11
+ le?vspltisb $tmp,0x0f
+ le?vxor $leperm,$leperm,$seven
+
+ li $idx,15
+ lvx $tweak,0,$ivp # load [unaligned] iv
+ lvsl $inpperm,0,$ivp
+ lvx $inptail,$idx,$ivp
+ le?vxor $inpperm,$inpperm,$tmp
+ vperm $tweak,$tweak,$inptail,$inpperm
+
+ neg r11,$inp
+ lvsr $inpperm,0,r11 # prepare for unaligned load
+ lvx $inout,0,$inp
+ addi $inp,$inp,15 # 15 is not typo
+ le?vxor $inpperm,$inpperm,$tmp
+
+ ${UCMP}i $key2,0 # key2==NULL?
+ beq Lxts_dec_no_key2
+
+ ?lvsl $keyperm,0,$key2 # prepare for unaligned key
+ lwz $rounds,240($key2)
+ srwi $rounds,$rounds,1
+ subi $rounds,$rounds,1
+ li $idx,16
+
+ lvx $rndkey0,0,$key2
+ lvx $rndkey1,$idx,$key2
+ addi $idx,$idx,16
+ ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
+ vxor $tweak,$tweak,$rndkey0
+ lvx $rndkey0,$idx,$key2
+ addi $idx,$idx,16
+ mtctr $rounds
+
+Ltweak_xts_dec:
+ ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
+ vcipher $tweak,$tweak,$rndkey1
+ lvx $rndkey1,$idx,$key2
+ addi $idx,$idx,16
+ ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
+ vcipher $tweak,$tweak,$rndkey0
+ lvx $rndkey0,$idx,$key2
+ addi $idx,$idx,16
+ bdnz Ltweak_xts_dec
+
+ ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
+ vcipher $tweak,$tweak,$rndkey1
+ lvx $rndkey1,$idx,$key2
+ ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
+ vcipherlast $tweak,$tweak,$rndkey0
+
+ li $ivp,0 # don't chain the tweak
+ b Lxts_dec
+
+Lxts_dec_no_key2:
+ neg $idx,$len
+ andi. $idx,$idx,15
+ add $len,$len,$idx # in "tweak chaining"
+ # mode only complete
+ # blocks are processed
+Lxts_dec:
+ lvx $inptail,0,$inp
+ addi $inp,$inp,16
+
+ ?lvsl $keyperm,0,$key1 # prepare for unaligned key
+ lwz $rounds,240($key1)
+ srwi $rounds,$rounds,1
+ subi $rounds,$rounds,1
+ li $idx,16
+
+ vslb $eighty7,$seven,$seven # 0x808080..80
+ vor $eighty7,$eighty7,$seven # 0x878787..87
+ vspltisb $tmp,1 # 0x010101..01
+ vsldoi $eighty7,$eighty7,$tmp,15 # 0x870101..01
+
+ ${UCMP}i $len,96
+ bge _aesp8_xts_decrypt6x
+
+ lvx $rndkey0,0,$key1
+ lvx $rndkey1,$idx,$key1
+ addi $idx,$idx,16
+ vperm $inout,$inout,$inptail,$inpperm
+ ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
+ vxor $inout,$inout,$tweak
+ vxor $inout,$inout,$rndkey0
+ lvx $rndkey0,$idx,$key1
+ addi $idx,$idx,16
+ mtctr $rounds
+
+ ${UCMP}i $len,16
+ blt Ltail_xts_dec
+ be?b Loop_xts_dec
+
+.align 5
+Loop_xts_dec:
+ ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
+ vncipher $inout,$inout,$rndkey1
+ lvx $rndkey1,$idx,$key1
+ addi $idx,$idx,16
+ ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
+ vncipher $inout,$inout,$rndkey0
+ lvx $rndkey0,$idx,$key1
+ addi $idx,$idx,16
+ bdnz Loop_xts_dec
+
+ ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
+ vncipher $inout,$inout,$rndkey1
+ lvx $rndkey1,$idx,$key1
+ li $idx,16
+ ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
+ vxor $rndkey0,$rndkey0,$tweak
+ vncipherlast $output,$inout,$rndkey0
+
+ le?vperm $tmp,$output,$output,$leperm
+ be?nop
+ le?stvx_u $tmp,0,$out
+ be?stvx_u $output,0,$out
+ addi $out,$out,16
+
+ subic. $len,$len,16
+ beq Lxts_dec_done
+
+ vmr $inout,$inptail
+ lvx $inptail,0,$inp
+ addi $inp,$inp,16
+ lvx $rndkey0,0,$key1
+ lvx $rndkey1,$idx,$key1
+ addi $idx,$idx,16
+
+ vsrab $tmp,$tweak,$seven # next tweak value
+ vaddubm $tweak,$tweak,$tweak
+ vsldoi $tmp,$tmp,$tmp,15
+ vand $tmp,$tmp,$eighty7
+ vxor $tweak,$tweak,$tmp
+
+ vperm $inout,$inout,$inptail,$inpperm
+ ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
+ vxor $inout,$inout,$tweak
+ vxor $inout,$inout,$rndkey0
+ lvx $rndkey0,$idx,$key1
+ addi $idx,$idx,16
+
+ mtctr $rounds
+ ${UCMP}i $len,16
+ bge Loop_xts_dec
+
+Ltail_xts_dec:
+ vsrab $tmp,$tweak,$seven # next tweak value
+ vaddubm $tweak1,$tweak,$tweak
+ vsldoi $tmp,$tmp,$tmp,15
+ vand $tmp,$tmp,$eighty7
+ vxor $tweak1,$tweak1,$tmp
+
+ subi $inp,$inp,16
+ add $inp,$inp,$len
+
+ vxor $inout,$inout,$tweak # :-(
+ vxor $inout,$inout,$tweak1 # :-)
+
+Loop_xts_dec_short:
+ ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
+ vncipher $inout,$inout,$rndkey1
+ lvx $rndkey1,$idx,$key1
+ addi $idx,$idx,16
+ ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
+ vncipher $inout,$inout,$rndkey0
+ lvx $rndkey0,$idx,$key1
+ addi $idx,$idx,16
+ bdnz Loop_xts_dec_short
+
+ ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
+ vncipher $inout,$inout,$rndkey1
+ lvx $rndkey1,$idx,$key1
+ li $idx,16
+ ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
+ vxor $rndkey0,$rndkey0,$tweak1
+ vncipherlast $output,$inout,$rndkey0
+
+ le?vperm $tmp,$output,$output,$leperm
+ be?nop
+ le?stvx_u $tmp,0,$out
+ be?stvx_u $output,0,$out
+
+ vmr $inout,$inptail
+ lvx $inptail,0,$inp
+ #addi $inp,$inp,16
+ lvx $rndkey0,0,$key1
+ lvx $rndkey1,$idx,$key1
+ addi $idx,$idx,16
+ vperm $inout,$inout,$inptail,$inpperm
+ ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
+
+ lvsr $inpperm,0,$len # $inpperm is no longer needed
+ vxor $inptail,$inptail,$inptail # $inptail is no longer needed
+ vspltisb $tmp,-1
+ vperm $inptail,$inptail,$tmp,$inpperm
+ vsel $inout,$inout,$output,$inptail
+
+ vxor $rndkey0,$rndkey0,$tweak
+ vxor $inout,$inout,$rndkey0
+ lvx $rndkey0,$idx,$key1
+ addi $idx,$idx,16
+
+ subi r11,$out,1
+ mtctr $len
+ li $len,16
+Loop_xts_dec_steal:
+ lbzu r0,1(r11)
+ stb r0,16(r11)
+ bdnz Loop_xts_dec_steal
+
+ mtctr $rounds
+ b Loop_xts_dec # one more time...
+
+Lxts_dec_done:
+ ${UCMP}i $ivp,0
+ beq Lxts_dec_ret
+
+ vsrab $tmp,$tweak,$seven # next tweak value
+ vaddubm $tweak,$tweak,$tweak
+ vsldoi $tmp,$tmp,$tmp,15
+ vand $tmp,$tmp,$eighty7
+ vxor $tweak,$tweak,$tmp
+
+ le?vperm $tweak,$tweak,$tweak,$leperm
+ stvx_u $tweak,0,$ivp
+
+Lxts_dec_ret:
+ mtspr 256,r12 # restore vrsave
+ li r3,0
+ blr
+ .long 0
+ .byte 0,12,0x04,0,0x80,6,6,0
+ .long 0
+.size .${prefix}_xts_decrypt,.-.${prefix}_xts_decrypt
+___
+#########################################################################
+{{ # Optimized XTS procedures #
+my $key_=$key2;
+my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,3,26..31));
+ $x00=0 if ($flavour =~ /osx/);
+my ($in0, $in1, $in2, $in3, $in4, $in5 )=map("v$_",(0..5));
+my ($out0, $out1, $out2, $out3, $out4, $out5)=map("v$_",(7,12..16));
+my ($twk0, $twk1, $twk2, $twk3, $twk4, $twk5)=map("v$_",(17..22));
+my $rndkey0="v23"; # v24-v25 rotating buffer for first found keys
+ # v26-v31 last 6 round keys
+my ($keyperm)=($out0); # aliases with "caller", redundant assignment
+my $taillen=$x70;
+
+$code.=<<___;
+.align 5
+_aesp8_xts_encrypt6x:
+ $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
+ mflr r11
+ li r7,`$FRAME+8*16+15`
+ li r3,`$FRAME+8*16+31`
+ $PUSH r11,`$FRAME+21*16+6*$SIZE_T+$LRSAVE`($sp)
+ stvx v20,r7,$sp # ABI says so
+ addi r7,r7,32
+ stvx v21,r3,$sp
+ addi r3,r3,32
+ stvx v22,r7,$sp
+ addi r7,r7,32
+ stvx v23,r3,$sp
+ addi r3,r3,32
+ stvx v24,r7,$sp
+ addi r7,r7,32
+ stvx v25,r3,$sp
+ addi r3,r3,32
+ stvx v26,r7,$sp
+ addi r7,r7,32
+ stvx v27,r3,$sp
+ addi r3,r3,32
+ stvx v28,r7,$sp
+ addi r7,r7,32
+ stvx v29,r3,$sp
+ addi r3,r3,32
+ stvx v30,r7,$sp
+ stvx v31,r3,$sp
+ li r0,-1
+ stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave
+ li $x10,0x10
+ $PUSH r26,`$FRAME+21*16+0*$SIZE_T`($sp)
+ li $x20,0x20
+ $PUSH r27,`$FRAME+21*16+1*$SIZE_T`($sp)
+ li $x30,0x30
+ $PUSH r28,`$FRAME+21*16+2*$SIZE_T`($sp)
+ li $x40,0x40
+ $PUSH r29,`$FRAME+21*16+3*$SIZE_T`($sp)
+ li $x50,0x50
+ $PUSH r30,`$FRAME+21*16+4*$SIZE_T`($sp)
+ li $x60,0x60
+ $PUSH r31,`$FRAME+21*16+5*$SIZE_T`($sp)
+ li $x70,0x70
+ mtspr 256,r0
+
+ subi $rounds,$rounds,3 # -4 in total
+
+ lvx $rndkey0,$x00,$key1 # load key schedule
+ lvx v30,$x10,$key1
+ addi $key1,$key1,0x20
+ lvx v31,$x00,$key1
+ ?vperm $rndkey0,$rndkey0,v30,$keyperm
+ addi $key_,$sp,$FRAME+15
+ mtctr $rounds
+
+Load_xts_enc_key:
+ ?vperm v24,v30,v31,$keyperm
+ lvx v30,$x10,$key1
+ addi $key1,$key1,0x20
+ stvx v24,$x00,$key_ # off-load round[1]
+ ?vperm v25,v31,v30,$keyperm
+ lvx v31,$x00,$key1
+ stvx v25,$x10,$key_ # off-load round[2]
+ addi $key_,$key_,0x20
+ bdnz Load_xts_enc_key
+
+ lvx v26,$x10,$key1
+ ?vperm v24,v30,v31,$keyperm
+ lvx v27,$x20,$key1
+ stvx v24,$x00,$key_ # off-load round[3]
+ ?vperm v25,v31,v26,$keyperm
+ lvx v28,$x30,$key1
+ stvx v25,$x10,$key_ # off-load round[4]
+ addi $key_,$sp,$FRAME+15 # rewind $key_
+ ?vperm v26,v26,v27,$keyperm
+ lvx v29,$x40,$key1
+ ?vperm v27,v27,v28,$keyperm
+ lvx v30,$x50,$key1
+ ?vperm v28,v28,v29,$keyperm
+ lvx v31,$x60,$key1
+ ?vperm v29,v29,v30,$keyperm
+ lvx $twk5,$x70,$key1 # borrow $twk5
+ ?vperm v30,v30,v31,$keyperm
+ lvx v24,$x00,$key_ # pre-load round[1]
+ ?vperm v31,v31,$twk5,$keyperm
+ lvx v25,$x10,$key_ # pre-load round[2]
+
+ vperm $in0,$inout,$inptail,$inpperm
+ subi $inp,$inp,31 # undo "caller"
+ vxor $twk0,$tweak,$rndkey0
+ vsrab $tmp,$tweak,$seven # next tweak value
+ vaddubm $tweak,$tweak,$tweak
+ vsldoi $tmp,$tmp,$tmp,15
+ vand $tmp,$tmp,$eighty7
+ vxor $out0,$in0,$twk0
+ vxor $tweak,$tweak,$tmp
+
+ lvx_u $in1,$x10,$inp
+ vxor $twk1,$tweak,$rndkey0
+ vsrab $tmp,$tweak,$seven # next tweak value
+ vaddubm $tweak,$tweak,$tweak
+ vsldoi $tmp,$tmp,$tmp,15
+ le?vperm $in1,$in1,$in1,$leperm
+ vand $tmp,$tmp,$eighty7
+ vxor $out1,$in1,$twk1
+ vxor $tweak,$tweak,$tmp
+
+ lvx_u $in2,$x20,$inp
+ andi. $taillen,$len,15
+ vxor $twk2,$tweak,$rndkey0
+ vsrab $tmp,$tweak,$seven # next tweak value
+ vaddubm $tweak,$tweak,$tweak
+ vsldoi $tmp,$tmp,$tmp,15
+ le?vperm $in2,$in2,$in2,$leperm
+ vand $tmp,$tmp,$eighty7
+ vxor $out2,$in2,$twk2
+ vxor $tweak,$tweak,$tmp
+
+ lvx_u $in3,$x30,$inp
+ sub $len,$len,$taillen
+ vxor $twk3,$tweak,$rndkey0
+ vsrab $tmp,$tweak,$seven # next tweak value
+ vaddubm $tweak,$tweak,$tweak
+ vsldoi $tmp,$tmp,$tmp,15
+ le?vperm $in3,$in3,$in3,$leperm
+ vand $tmp,$tmp,$eighty7
+ vxor $out3,$in3,$twk3
+ vxor $tweak,$tweak,$tmp
+
+ lvx_u $in4,$x40,$inp
+ subi $len,$len,0x60
+ vxor $twk4,$tweak,$rndkey0
+ vsrab $tmp,$tweak,$seven # next tweak value
+ vaddubm $tweak,$tweak,$tweak
+ vsldoi $tmp,$tmp,$tmp,15
+ le?vperm $in4,$in4,$in4,$leperm
+ vand $tmp,$tmp,$eighty7
+ vxor $out4,$in4,$twk4
+ vxor $tweak,$tweak,$tmp
+
+ lvx_u $in5,$x50,$inp
+ addi $inp,$inp,0x60
+ vxor $twk5,$tweak,$rndkey0
+ vsrab $tmp,$tweak,$seven # next tweak value
+ vaddubm $tweak,$tweak,$tweak
+ vsldoi $tmp,$tmp,$tmp,15
+ le?vperm $in5,$in5,$in5,$leperm
+ vand $tmp,$tmp,$eighty7
+ vxor $out5,$in5,$twk5
+ vxor $tweak,$tweak,$tmp
+
+ vxor v31,v31,$rndkey0
+ mtctr $rounds
+ b Loop_xts_enc6x
+
+.align 5
+Loop_xts_enc6x:
+ vcipher $out0,$out0,v24
+ vcipher $out1,$out1,v24
+ vcipher $out2,$out2,v24
+ vcipher $out3,$out3,v24
+ vcipher $out4,$out4,v24
+ vcipher $out5,$out5,v24
+ lvx v24,$x20,$key_ # round[3]
+ addi $key_,$key_,0x20
+
+ vcipher $out0,$out0,v25
+ vcipher $out1,$out1,v25
+ vcipher $out2,$out2,v25
+ vcipher $out3,$out3,v25
+ vcipher $out4,$out4,v25
+ vcipher $out5,$out5,v25
+ lvx v25,$x10,$key_ # round[4]
+ bdnz Loop_xts_enc6x
+
+ subic $len,$len,96 # $len-=96
+ vxor $in0,$twk0,v31 # xor with last round key
+ vcipher $out0,$out0,v24
+ vcipher $out1,$out1,v24
+ vsrab $tmp,$tweak,$seven # next tweak value
+ vxor $twk0,$tweak,$rndkey0
+ vaddubm $tweak,$tweak,$tweak
+ vcipher $out2,$out2,v24
+ vcipher $out3,$out3,v24
+ vsldoi $tmp,$tmp,$tmp,15
+ vcipher $out4,$out4,v24
+ vcipher $out5,$out5,v24
+
+ subfe. r0,r0,r0 # borrow?-1:0
+ vand $tmp,$tmp,$eighty7
+ vcipher $out0,$out0,v25
+ vcipher $out1,$out1,v25
+ vxor $tweak,$tweak,$tmp
+ vcipher $out2,$out2,v25
+ vcipher $out3,$out3,v25
+ vxor $in1,$twk1,v31
+ vsrab $tmp,$tweak,$seven # next tweak value
+ vxor $twk1,$tweak,$rndkey0
+ vcipher $out4,$out4,v25
+ vcipher $out5,$out5,v25
+
+ and r0,r0,$len
+ vaddubm $tweak,$tweak,$tweak
+ vsldoi $tmp,$tmp,$tmp,15
+ vcipher $out0,$out0,v26
+ vcipher $out1,$out1,v26
+ vand $tmp,$tmp,$eighty7
+ vcipher $out2,$out2,v26
+ vcipher $out3,$out3,v26
+ vxor $tweak,$tweak,$tmp
+ vcipher $out4,$out4,v26
+ vcipher $out5,$out5,v26
+
+ add $inp,$inp,r0 # $inp is adjusted in such
+ # way that at exit from the
+ # loop inX-in5 are loaded
+ # with last "words"
+ vxor $in2,$twk2,v31
+ vsrab $tmp,$tweak,$seven # next tweak value
+ vxor $twk2,$tweak,$rndkey0
+ vaddubm $tweak,$tweak,$tweak
+ vcipher $out0,$out0,v27
+ vcipher $out1,$out1,v27
+ vsldoi $tmp,$tmp,$tmp,15
+ vcipher $out2,$out2,v27
+ vcipher $out3,$out3,v27
+ vand $tmp,$tmp,$eighty7
+ vcipher $out4,$out4,v27
+ vcipher $out5,$out5,v27
+
+ addi $key_,$sp,$FRAME+15 # rewind $key_
+ vxor $tweak,$tweak,$tmp
+ vcipher $out0,$out0,v28
+ vcipher $out1,$out1,v28
+ vxor $in3,$twk3,v31
+ vsrab $tmp,$tweak,$seven # next tweak value
+ vxor $twk3,$tweak,$rndkey0
+ vcipher $out2,$out2,v28
+ vcipher $out3,$out3,v28
+ vaddubm $tweak,$tweak,$tweak
+ vsldoi $tmp,$tmp,$tmp,15
+ vcipher $out4,$out4,v28
+ vcipher $out5,$out5,v28
+ lvx v24,$x00,$key_ # re-pre-load round[1]
+ vand $tmp,$tmp,$eighty7
+
+ vcipher $out0,$out0,v29
+ vcipher $out1,$out1,v29
+ vxor $tweak,$tweak,$tmp
+ vcipher $out2,$out2,v29
+ vcipher $out3,$out3,v29
+ vxor $in4,$twk4,v31
+ vsrab $tmp,$tweak,$seven # next tweak value
+ vxor $twk4,$tweak,$rndkey0
+ vcipher $out4,$out4,v29
+ vcipher $out5,$out5,v29
+ lvx v25,$x10,$key_ # re-pre-load round[2]
+ vaddubm $tweak,$tweak,$tweak
+ vsldoi $tmp,$tmp,$tmp,15
+
+ vcipher $out0,$out0,v30
+ vcipher $out1,$out1,v30
+ vand $tmp,$tmp,$eighty7
+ vcipher $out2,$out2,v30
+ vcipher $out3,$out3,v30
+ vxor $tweak,$tweak,$tmp
+ vcipher $out4,$out4,v30
+ vcipher $out5,$out5,v30
+ vxor $in5,$twk5,v31
+ vsrab $tmp,$tweak,$seven # next tweak value
+ vxor $twk5,$tweak,$rndkey0
+
+ vcipherlast $out0,$out0,$in0
+ lvx_u $in0,$x00,$inp # load next input block
+ vaddubm $tweak,$tweak,$tweak
+ vsldoi $tmp,$tmp,$tmp,15
+ vcipherlast $out1,$out1,$in1
+ lvx_u $in1,$x10,$inp
+ vcipherlast $out2,$out2,$in2
+ le?vperm $in0,$in0,$in0,$leperm
+ lvx_u $in2,$x20,$inp
+ vand $tmp,$tmp,$eighty7
+ vcipherlast $out3,$out3,$in3
+ le?vperm $in1,$in1,$in1,$leperm
+ lvx_u $in3,$x30,$inp
+ vcipherlast $out4,$out4,$in4
+ le?vperm $in2,$in2,$in2,$leperm
+ lvx_u $in4,$x40,$inp
+ vxor $tweak,$tweak,$tmp
+ vcipherlast $tmp,$out5,$in5 # last block might be needed
+ # in stealing mode
+ le?vperm $in3,$in3,$in3,$leperm
+ lvx_u $in5,$x50,$inp
+ addi $inp,$inp,0x60
+ le?vperm $in4,$in4,$in4,$leperm
+ le?vperm $in5,$in5,$in5,$leperm
+
+ le?vperm $out0,$out0,$out0,$leperm
+ le?vperm $out1,$out1,$out1,$leperm
+ stvx_u $out0,$x00,$out # store output
+ vxor $out0,$in0,$twk0
+ le?vperm $out2,$out2,$out2,$leperm
+ stvx_u $out1,$x10,$out
+ vxor $out1,$in1,$twk1
+ le?vperm $out3,$out3,$out3,$leperm
+ stvx_u $out2,$x20,$out
+ vxor $out2,$in2,$twk2
+ le?vperm $out4,$out4,$out4,$leperm
+ stvx_u $out3,$x30,$out
+ vxor $out3,$in3,$twk3
+ le?vperm $out5,$tmp,$tmp,$leperm
+ stvx_u $out4,$x40,$out
+ vxor $out4,$in4,$twk4
+ le?stvx_u $out5,$x50,$out
+ be?stvx_u $tmp, $x50,$out
+ vxor $out5,$in5,$twk5
+ addi $out,$out,0x60
+
+ mtctr $rounds
+ beq Loop_xts_enc6x # did $len-=96 borrow?
+
+ addic. $len,$len,0x60
+ beq Lxts_enc6x_zero
+ cmpwi $len,0x20
+ blt Lxts_enc6x_one
+ nop
+ beq Lxts_enc6x_two
+ cmpwi $len,0x40
+ blt Lxts_enc6x_three
+ nop
+ beq Lxts_enc6x_four
+
+Lxts_enc6x_five:
+ vxor $out0,$in1,$twk0
+ vxor $out1,$in2,$twk1
+ vxor $out2,$in3,$twk2
+ vxor $out3,$in4,$twk3
+ vxor $out4,$in5,$twk4
+
+ bl _aesp8_xts_enc5x
+
+ le?vperm $out0,$out0,$out0,$leperm
+ vmr $twk0,$twk5 # unused tweak
+ le?vperm $out1,$out1,$out1,$leperm
+ stvx_u $out0,$x00,$out # store output
+ le?vperm $out2,$out2,$out2,$leperm
+ stvx_u $out1,$x10,$out
+ le?vperm $out3,$out3,$out3,$leperm
+ stvx_u $out2,$x20,$out
+ vxor $tmp,$out4,$twk5 # last block prep for stealing
+ le?vperm $out4,$out4,$out4,$leperm
+ stvx_u $out3,$x30,$out
+ stvx_u $out4,$x40,$out
+ addi $out,$out,0x50
+ bne Lxts_enc6x_steal
+ b Lxts_enc6x_done
+
+.align 4
+Lxts_enc6x_four:
+ vxor $out0,$in2,$twk0
+ vxor $out1,$in3,$twk1
+ vxor $out2,$in4,$twk2
+ vxor $out3,$in5,$twk3
+ vxor $out4,$out4,$out4
+
+ bl _aesp8_xts_enc5x
+
+ le?vperm $out0,$out0,$out0,$leperm
+ vmr $twk0,$twk4 # unused tweak
+ le?vperm $out1,$out1,$out1,$leperm
+ stvx_u $out0,$x00,$out # store output
+ le?vperm $out2,$out2,$out2,$leperm
+ stvx_u $out1,$x10,$out
+ vxor $tmp,$out3,$twk4 # last block prep for stealing
+ le?vperm $out3,$out3,$out3,$leperm
+ stvx_u $out2,$x20,$out
+ stvx_u $out3,$x30,$out
+ addi $out,$out,0x40
+ bne Lxts_enc6x_steal
+ b Lxts_enc6x_done
+
+.align 4
+Lxts_enc6x_three:
+ vxor $out0,$in3,$twk0
+ vxor $out1,$in4,$twk1
+ vxor $out2,$in5,$twk2
+ vxor $out3,$out3,$out3
+ vxor $out4,$out4,$out4
+
+ bl _aesp8_xts_enc5x
+
+ le?vperm $out0,$out0,$out0,$leperm
+ vmr $twk0,$twk3 # unused tweak
+ le?vperm $out1,$out1,$out1,$leperm
+ stvx_u $out0,$x00,$out # store output
+ vxor $tmp,$out2,$twk3 # last block prep for stealing
+ le?vperm $out2,$out2,$out2,$leperm
+ stvx_u $out1,$x10,$out
+ stvx_u $out2,$x20,$out
+ addi $out,$out,0x30
+ bne Lxts_enc6x_steal
+ b Lxts_enc6x_done
+
+.align 4
+Lxts_enc6x_two:
+ vxor $out0,$in4,$twk0
+ vxor $out1,$in5,$twk1
+ vxor $out2,$out2,$out2
+ vxor $out3,$out3,$out3
+ vxor $out4,$out4,$out4
+
+ bl _aesp8_xts_enc5x
+
+ le?vperm $out0,$out0,$out0,$leperm
+ vmr $twk0,$twk2 # unused tweak
+ vxor $tmp,$out1,$twk2 # last block prep for stealing
+ le?vperm $out1,$out1,$out1,$leperm
+ stvx_u $out0,$x00,$out # store output
+ stvx_u $out1,$x10,$out
+ addi $out,$out,0x20
+ bne Lxts_enc6x_steal
+ b Lxts_enc6x_done
+
+.align 4
+Lxts_enc6x_one:
+ vxor $out0,$in5,$twk0
+ nop
+Loop_xts_enc1x:
+ vcipher $out0,$out0,v24
+ lvx v24,$x20,$key_ # round[3]
+ addi $key_,$key_,0x20
+
+ vcipher $out0,$out0,v25
+ lvx v25,$x10,$key_ # round[4]
+ bdnz Loop_xts_enc1x
+
+ add $inp,$inp,$taillen
+ cmpwi $taillen,0
+ vcipher $out0,$out0,v24
+
+ subi $inp,$inp,16
+ vcipher $out0,$out0,v25
+
+ lvsr $inpperm,0,$taillen
+ vcipher $out0,$out0,v26
+
+ lvx_u $in0,0,$inp
+ vcipher $out0,$out0,v27
+
+ addi $key_,$sp,$FRAME+15 # rewind $key_
+ vcipher $out0,$out0,v28
+ lvx v24,$x00,$key_ # re-pre-load round[1]
+
+ vcipher $out0,$out0,v29
+ lvx v25,$x10,$key_ # re-pre-load round[2]
+ vxor $twk0,$twk0,v31
+
+ le?vperm $in0,$in0,$in0,$leperm
+ vcipher $out0,$out0,v30
+
+ vperm $in0,$in0,$in0,$inpperm
+ vcipherlast $out0,$out0,$twk0
+
+ vmr $twk0,$twk1 # unused tweak
+ vxor $tmp,$out0,$twk1 # last block prep for stealing
+ le?vperm $out0,$out0,$out0,$leperm
+ stvx_u $out0,$x00,$out # store output
+ addi $out,$out,0x10
+ bne Lxts_enc6x_steal
+ b Lxts_enc6x_done
+
+.align 4
+Lxts_enc6x_zero:
+ cmpwi $taillen,0
+ beq Lxts_enc6x_done
+
+ add $inp,$inp,$taillen
+ subi $inp,$inp,16
+ lvx_u $in0,0,$inp
+ lvsr $inpperm,0,$taillen # $in5 is no more
+ le?vperm $in0,$in0,$in0,$leperm
+ vperm $in0,$in0,$in0,$inpperm
+ vxor $tmp,$tmp,$twk0
+Lxts_enc6x_steal:
+ vxor $in0,$in0,$twk0
+ vxor $out0,$out0,$out0
+ vspltisb $out1,-1
+ vperm $out0,$out0,$out1,$inpperm
+ vsel $out0,$in0,$tmp,$out0 # $tmp is last block, remember?
+
+ subi r30,$out,17
+ subi $out,$out,16
+ mtctr $taillen
+Loop_xts_enc6x_steal:
+ lbzu r0,1(r30)
+ stb r0,16(r30)
+ bdnz Loop_xts_enc6x_steal
+
+ li $taillen,0
+ mtctr $rounds
+ b Loop_xts_enc1x # one more time...
+
+.align 4
+Lxts_enc6x_done:
+ ${UCMP}i $ivp,0
+ beq Lxts_enc6x_ret
+
+ vxor $tweak,$twk0,$rndkey0
+ le?vperm $tweak,$tweak,$tweak,$leperm
+ stvx_u $tweak,0,$ivp
+
+Lxts_enc6x_ret:
+ mtlr r11
+ li r10,`$FRAME+15`
+ li r11,`$FRAME+31`
+ stvx $seven,r10,$sp # wipe copies of round keys
+ addi r10,r10,32
+ stvx $seven,r11,$sp
+ addi r11,r11,32
+ stvx $seven,r10,$sp
+ addi r10,r10,32
+ stvx $seven,r11,$sp
+ addi r11,r11,32
+ stvx $seven,r10,$sp
+ addi r10,r10,32
+ stvx $seven,r11,$sp
+ addi r11,r11,32
+ stvx $seven,r10,$sp
+ addi r10,r10,32
+ stvx $seven,r11,$sp
+ addi r11,r11,32
+
+ mtspr 256,$vrsave
+ lvx v20,r10,$sp # ABI says so
+ addi r10,r10,32
+ lvx v21,r11,$sp
+ addi r11,r11,32
+ lvx v22,r10,$sp
+ addi r10,r10,32
+ lvx v23,r11,$sp
+ addi r11,r11,32
+ lvx v24,r10,$sp
+ addi r10,r10,32
+ lvx v25,r11,$sp
+ addi r11,r11,32
+ lvx v26,r10,$sp
+ addi r10,r10,32
+ lvx v27,r11,$sp
+ addi r11,r11,32
+ lvx v28,r10,$sp
+ addi r10,r10,32
+ lvx v29,r11,$sp
+ addi r11,r11,32
+ lvx v30,r10,$sp
+ lvx v31,r11,$sp
+ $POP r26,`$FRAME+21*16+0*$SIZE_T`($sp)
+ $POP r27,`$FRAME+21*16+1*$SIZE_T`($sp)
+ $POP r28,`$FRAME+21*16+2*$SIZE_T`($sp)
+ $POP r29,`$FRAME+21*16+3*$SIZE_T`($sp)
+ $POP r30,`$FRAME+21*16+4*$SIZE_T`($sp)
+ $POP r31,`$FRAME+21*16+5*$SIZE_T`($sp)
+ addi $sp,$sp,`$FRAME+21*16+6*$SIZE_T`
+ blr
+ .long 0
+ .byte 0,12,0x04,1,0x80,6,6,0
+ .long 0
+
+.align 5
+_aesp8_xts_enc5x:
+ vcipher $out0,$out0,v24
+ vcipher $out1,$out1,v24
+ vcipher $out2,$out2,v24
+ vcipher $out3,$out3,v24
+ vcipher $out4,$out4,v24
+ lvx v24,$x20,$key_ # round[3]
+ addi $key_,$key_,0x20
+
+ vcipher $out0,$out0,v25
+ vcipher $out1,$out1,v25
+ vcipher $out2,$out2,v25
+ vcipher $out3,$out3,v25
+ vcipher $out4,$out4,v25
+ lvx v25,$x10,$key_ # round[4]
+ bdnz _aesp8_xts_enc5x
+
+ add $inp,$inp,$taillen
+ cmpwi $taillen,0
+ vcipher $out0,$out0,v24
+ vcipher $out1,$out1,v24
+ vcipher $out2,$out2,v24
+ vcipher $out3,$out3,v24
+ vcipher $out4,$out4,v24
+
+ subi $inp,$inp,16
+ vcipher $out0,$out0,v25
+ vcipher $out1,$out1,v25
+ vcipher $out2,$out2,v25
+ vcipher $out3,$out3,v25
+ vcipher $out4,$out4,v25
+ vxor $twk0,$twk0,v31
+
+ vcipher $out0,$out0,v26
+ lvsr $inpperm,r0,$taillen # $in5 is no more
+ vcipher $out1,$out1,v26
+ vcipher $out2,$out2,v26
+ vcipher $out3,$out3,v26
+ vcipher $out4,$out4,v26
+ vxor $in1,$twk1,v31
+
+ vcipher $out0,$out0,v27
+ lvx_u $in0,0,$inp
+ vcipher $out1,$out1,v27
+ vcipher $out2,$out2,v27
+ vcipher $out3,$out3,v27
+ vcipher $out4,$out4,v27
+ vxor $in2,$twk2,v31
+
+ addi $key_,$sp,$FRAME+15 # rewind $key_
+ vcipher $out0,$out0,v28
+ vcipher $out1,$out1,v28
+ vcipher $out2,$out2,v28
+ vcipher $out3,$out3,v28
+ vcipher $out4,$out4,v28
+ lvx v24,$x00,$key_ # re-pre-load round[1]
+ vxor $in3,$twk3,v31
+
+ vcipher $out0,$out0,v29
+ le?vperm $in0,$in0,$in0,$leperm
+ vcipher $out1,$out1,v29
+ vcipher $out2,$out2,v29
+ vcipher $out3,$out3,v29
+ vcipher $out4,$out4,v29
+ lvx v25,$x10,$key_ # re-pre-load round[2]
+ vxor $in4,$twk4,v31
+
+ vcipher $out0,$out0,v30
+ vperm $in0,$in0,$in0,$inpperm
+ vcipher $out1,$out1,v30
+ vcipher $out2,$out2,v30
+ vcipher $out3,$out3,v30
+ vcipher $out4,$out4,v30
+
+ vcipherlast $out0,$out0,$twk0
+ vcipherlast $out1,$out1,$in1
+ vcipherlast $out2,$out2,$in2
+ vcipherlast $out3,$out3,$in3
+ vcipherlast $out4,$out4,$in4
+ blr
+ .long 0
+ .byte 0,12,0x14,0,0,0,0,0
+
+.align 5
+_aesp8_xts_decrypt6x:
+ $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
+ mflr r11
+ li r7,`$FRAME+8*16+15`
+ li r3,`$FRAME+8*16+31`
+ $PUSH r11,`$FRAME+21*16+6*$SIZE_T+$LRSAVE`($sp)
+ stvx v20,r7,$sp # ABI says so
+ addi r7,r7,32
+ stvx v21,r3,$sp
+ addi r3,r3,32
+ stvx v22,r7,$sp
+ addi r7,r7,32
+ stvx v23,r3,$sp
+ addi r3,r3,32
+ stvx v24,r7,$sp
+ addi r7,r7,32
+ stvx v25,r3,$sp
+ addi r3,r3,32
+ stvx v26,r7,$sp
+ addi r7,r7,32
+ stvx v27,r3,$sp
+ addi r3,r3,32
+ stvx v28,r7,$sp
+ addi r7,r7,32
+ stvx v29,r3,$sp
+ addi r3,r3,32
+ stvx v30,r7,$sp
+ stvx v31,r3,$sp
+ li r0,-1
+ stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave
+ li $x10,0x10
+ $PUSH r26,`$FRAME+21*16+0*$SIZE_T`($sp)
+ li $x20,0x20
+ $PUSH r27,`$FRAME+21*16+1*$SIZE_T`($sp)
+ li $x30,0x30
+ $PUSH r28,`$FRAME+21*16+2*$SIZE_T`($sp)
+ li $x40,0x40
+ $PUSH r29,`$FRAME+21*16+3*$SIZE_T`($sp)
+ li $x50,0x50
+ $PUSH r30,`$FRAME+21*16+4*$SIZE_T`($sp)
+ li $x60,0x60
+ $PUSH r31,`$FRAME+21*16+5*$SIZE_T`($sp)
+ li $x70,0x70
+ mtspr 256,r0
+
+ subi $rounds,$rounds,3 # -4 in total
+
+ lvx $rndkey0,$x00,$key1 # load key schedule
+ lvx v30,$x10,$key1
+ addi $key1,$key1,0x20
+ lvx v31,$x00,$key1
+ ?vperm $rndkey0,$rndkey0,v30,$keyperm
+ addi $key_,$sp,$FRAME+15
+ mtctr $rounds
+
+Load_xts_dec_key:
+ ?vperm v24,v30,v31,$keyperm
+ lvx v30,$x10,$key1
+ addi $key1,$key1,0x20
+ stvx v24,$x00,$key_ # off-load round[1]
+ ?vperm v25,v31,v30,$keyperm
+ lvx v31,$x00,$key1
+ stvx v25,$x10,$key_ # off-load round[2]
+ addi $key_,$key_,0x20
+ bdnz Load_xts_dec_key
+
+ lvx v26,$x10,$key1
+ ?vperm v24,v30,v31,$keyperm
+ lvx v27,$x20,$key1
+ stvx v24,$x00,$key_ # off-load round[3]
+ ?vperm v25,v31,v26,$keyperm
+ lvx v28,$x30,$key1
+ stvx v25,$x10,$key_ # off-load round[4]
+ addi $key_,$sp,$FRAME+15 # rewind $key_
+ ?vperm v26,v26,v27,$keyperm
+ lvx v29,$x40,$key1
+ ?vperm v27,v27,v28,$keyperm
+ lvx v30,$x50,$key1
+ ?vperm v28,v28,v29,$keyperm
+ lvx v31,$x60,$key1
+ ?vperm v29,v29,v30,$keyperm
+ lvx $twk5,$x70,$key1 # borrow $twk5
+ ?vperm v30,v30,v31,$keyperm
+ lvx v24,$x00,$key_ # pre-load round[1]
+ ?vperm v31,v31,$twk5,$keyperm
+ lvx v25,$x10,$key_ # pre-load round[2]
+
+ vperm $in0,$inout,$inptail,$inpperm
+ subi $inp,$inp,31 # undo "caller"
+ vxor $twk0,$tweak,$rndkey0
+ vsrab $tmp,$tweak,$seven # next tweak value
+ vaddubm $tweak,$tweak,$tweak
+ vsldoi $tmp,$tmp,$tmp,15
+ vand $tmp,$tmp,$eighty7
+ vxor $out0,$in0,$twk0
+ vxor $tweak,$tweak,$tmp
+
+ lvx_u $in1,$x10,$inp
+ vxor $twk1,$tweak,$rndkey0
+ vsrab $tmp,$tweak,$seven # next tweak value
+ vaddubm $tweak,$tweak,$tweak
+ vsldoi $tmp,$tmp,$tmp,15
+ le?vperm $in1,$in1,$in1,$leperm
+ vand $tmp,$tmp,$eighty7
+ vxor $out1,$in1,$twk1
+ vxor $tweak,$tweak,$tmp
+
+ lvx_u $in2,$x20,$inp
+ andi. $taillen,$len,15
+ vxor $twk2,$tweak,$rndkey0
+ vsrab $tmp,$tweak,$seven # next tweak value
+ vaddubm $tweak,$tweak,$tweak
+ vsldoi $tmp,$tmp,$tmp,15
+ le?vperm $in2,$in2,$in2,$leperm
+ vand $tmp,$tmp,$eighty7
+ vxor $out2,$in2,$twk2
+ vxor $tweak,$tweak,$tmp
+
+ lvx_u $in3,$x30,$inp
+ sub $len,$len,$taillen
+ vxor $twk3,$tweak,$rndkey0
+ vsrab $tmp,$tweak,$seven # next tweak value
+ vaddubm $tweak,$tweak,$tweak
+ vsldoi $tmp,$tmp,$tmp,15
+ le?vperm $in3,$in3,$in3,$leperm
+ vand $tmp,$tmp,$eighty7
+ vxor $out3,$in3,$twk3
+ vxor $tweak,$tweak,$tmp
+
+ lvx_u $in4,$x40,$inp
+ subi $len,$len,0x60
+ vxor $twk4,$tweak,$rndkey0
+ vsrab $tmp,$tweak,$seven # next tweak value
+ vaddubm $tweak,$tweak,$tweak
+ vsldoi $tmp,$tmp,$tmp,15
+ le?vperm $in4,$in4,$in4,$leperm
+ vand $tmp,$tmp,$eighty7
+ vxor $out4,$in4,$twk4
+ vxor $tweak,$tweak,$tmp
+
+ lvx_u $in5,$x50,$inp
+ addi $inp,$inp,0x60
+ vxor $twk5,$tweak,$rndkey0
+ vsrab $tmp,$tweak,$seven # next tweak value
+ vaddubm $tweak,$tweak,$tweak
+ vsldoi $tmp,$tmp,$tmp,15
+ le?vperm $in5,$in5,$in5,$leperm
+ vand $tmp,$tmp,$eighty7
+ vxor $out5,$in5,$twk5
+ vxor $tweak,$tweak,$tmp
+
+ vxor v31,v31,$rndkey0
+ mtctr $rounds
+ b Loop_xts_dec6x
+
+.align 5
+Loop_xts_dec6x:
+ vncipher $out0,$out0,v24
+ vncipher $out1,$out1,v24
+ vncipher $out2,$out2,v24
+ vncipher $out3,$out3,v24
+ vncipher $out4,$out4,v24
+ vncipher $out5,$out5,v24
+ lvx v24,$x20,$key_ # round[3]
+ addi $key_,$key_,0x20
+
+ vncipher $out0,$out0,v25
+ vncipher $out1,$out1,v25
+ vncipher $out2,$out2,v25
+ vncipher $out3,$out3,v25
+ vncipher $out4,$out4,v25
+ vncipher $out5,$out5,v25
+ lvx v25,$x10,$key_ # round[4]
+ bdnz Loop_xts_dec6x
+
+ subic $len,$len,96 # $len-=96
+ vxor $in0,$twk0,v31 # xor with last round key
+ vncipher $out0,$out0,v24
+ vncipher $out1,$out1,v24
+ vsrab $tmp,$tweak,$seven # next tweak value
+ vxor $twk0,$tweak,$rndkey0
+ vaddubm $tweak,$tweak,$tweak
+ vncipher $out2,$out2,v24
+ vncipher $out3,$out3,v24
+ vsldoi $tmp,$tmp,$tmp,15
+ vncipher $out4,$out4,v24
+ vncipher $out5,$out5,v24
+
+ subfe. r0,r0,r0 # borrow?-1:0
+ vand $tmp,$tmp,$eighty7
+ vncipher $out0,$out0,v25
+ vncipher $out1,$out1,v25
+ vxor $tweak,$tweak,$tmp
+ vncipher $out2,$out2,v25
+ vncipher $out3,$out3,v25
+ vxor $in1,$twk1,v31
+ vsrab $tmp,$tweak,$seven # next tweak value
+ vxor $twk1,$tweak,$rndkey0
+ vncipher $out4,$out4,v25
+ vncipher $out5,$out5,v25
+
+ and r0,r0,$len
+ vaddubm $tweak,$tweak,$tweak
+ vsldoi $tmp,$tmp,$tmp,15
+ vncipher $out0,$out0,v26
+ vncipher $out1,$out1,v26
+ vand $tmp,$tmp,$eighty7
+ vncipher $out2,$out2,v26
+ vncipher $out3,$out3,v26
+ vxor $tweak,$tweak,$tmp
+ vncipher $out4,$out4,v26
+ vncipher $out5,$out5,v26
+
+ add $inp,$inp,r0 # $inp is adjusted in such
+ # way that at exit from the
+ # loop inX-in5 are loaded
+ # with last "words"
+ vxor $in2,$twk2,v31
+ vsrab $tmp,$tweak,$seven # next tweak value
+ vxor $twk2,$tweak,$rndkey0
+ vaddubm $tweak,$tweak,$tweak
+ vncipher $out0,$out0,v27
+ vncipher $out1,$out1,v27
+ vsldoi $tmp,$tmp,$tmp,15
+ vncipher $out2,$out2,v27
+ vncipher $out3,$out3,v27
+ vand $tmp,$tmp,$eighty7
+ vncipher $out4,$out4,v27
+ vncipher $out5,$out5,v27
+
+ addi $key_,$sp,$FRAME+15 # rewind $key_
+ vxor $tweak,$tweak,$tmp
+ vncipher $out0,$out0,v28
+ vncipher $out1,$out1,v28
+ vxor $in3,$twk3,v31
+ vsrab $tmp,$tweak,$seven # next tweak value
+ vxor $twk3,$tweak,$rndkey0
+ vncipher $out2,$out2,v28
+ vncipher $out3,$out3,v28
+ vaddubm $tweak,$tweak,$tweak
+ vsldoi $tmp,$tmp,$tmp,15
+ vncipher $out4,$out4,v28
+ vncipher $out5,$out5,v28
+ lvx v24,$x00,$key_ # re-pre-load round[1]
+ vand $tmp,$tmp,$eighty7
+
+ vncipher $out0,$out0,v29
+ vncipher $out1,$out1,v29
+ vxor $tweak,$tweak,$tmp
+ vncipher $out2,$out2,v29
+ vncipher $out3,$out3,v29
+ vxor $in4,$twk4,v31
+ vsrab $tmp,$tweak,$seven # next tweak value
+ vxor $twk4,$tweak,$rndkey0
+ vncipher $out4,$out4,v29
+ vncipher $out5,$out5,v29
+ lvx v25,$x10,$key_ # re-pre-load round[2]
+ vaddubm $tweak,$tweak,$tweak
+ vsldoi $tmp,$tmp,$tmp,15
+
+ vncipher $out0,$out0,v30
+ vncipher $out1,$out1,v30
+ vand $tmp,$tmp,$eighty7
+ vncipher $out2,$out2,v30
+ vncipher $out3,$out3,v30
+ vxor $tweak,$tweak,$tmp
+ vncipher $out4,$out4,v30
+ vncipher $out5,$out5,v30
+ vxor $in5,$twk5,v31
+ vsrab $tmp,$tweak,$seven # next tweak value
+ vxor $twk5,$tweak,$rndkey0
+
+ vncipherlast $out0,$out0,$in0
+ lvx_u $in0,$x00,$inp # load next input block
+ vaddubm $tweak,$tweak,$tweak
+ vsldoi $tmp,$tmp,$tmp,15
+ vncipherlast $out1,$out1,$in1
+ lvx_u $in1,$x10,$inp
+ vncipherlast $out2,$out2,$in2
+ le?vperm $in0,$in0,$in0,$leperm
+ lvx_u $in2,$x20,$inp
+ vand $tmp,$tmp,$eighty7
+ vncipherlast $out3,$out3,$in3
+ le?vperm $in1,$in1,$in1,$leperm
+ lvx_u $in3,$x30,$inp
+ vncipherlast $out4,$out4,$in4
+ le?vperm $in2,$in2,$in2,$leperm
+ lvx_u $in4,$x40,$inp
+ vxor $tweak,$tweak,$tmp
+ vncipherlast $out5,$out5,$in5
+ le?vperm $in3,$in3,$in3,$leperm
+ lvx_u $in5,$x50,$inp
+ addi $inp,$inp,0x60
+ le?vperm $in4,$in4,$in4,$leperm
+ le?vperm $in5,$in5,$in5,$leperm
+
+ le?vperm $out0,$out0,$out0,$leperm
+ le?vperm $out1,$out1,$out1,$leperm
+ stvx_u $out0,$x00,$out # store output
+ vxor $out0,$in0,$twk0
+ le?vperm $out2,$out2,$out2,$leperm
+ stvx_u $out1,$x10,$out
+ vxor $out1,$in1,$twk1
+ le?vperm $out3,$out3,$out3,$leperm
+ stvx_u $out2,$x20,$out
+ vxor $out2,$in2,$twk2
+ le?vperm $out4,$out4,$out4,$leperm
+ stvx_u $out3,$x30,$out
+ vxor $out3,$in3,$twk3
+ le?vperm $out5,$out5,$out5,$leperm
+ stvx_u $out4,$x40,$out
+ vxor $out4,$in4,$twk4
+ stvx_u $out5,$x50,$out
+ vxor $out5,$in5,$twk5
+ addi $out,$out,0x60
+
+ mtctr $rounds
+ beq Loop_xts_dec6x # did $len-=96 borrow?
+
+ addic. $len,$len,0x60
+ beq Lxts_dec6x_zero
+ cmpwi $len,0x20
+ blt Lxts_dec6x_one
+ nop
+ beq Lxts_dec6x_two
+ cmpwi $len,0x40
+ blt Lxts_dec6x_three
+ nop
+ beq Lxts_dec6x_four
+
+Lxts_dec6x_five:
+ vxor $out0,$in1,$twk0
+ vxor $out1,$in2,$twk1
+ vxor $out2,$in3,$twk2
+ vxor $out3,$in4,$twk3
+ vxor $out4,$in5,$twk4
+
+ bl _aesp8_xts_dec5x
+
+ le?vperm $out0,$out0,$out0,$leperm
+ vmr $twk0,$twk5 # unused tweak
+ vxor $twk1,$tweak,$rndkey0
+ le?vperm $out1,$out1,$out1,$leperm
+ stvx_u $out0,$x00,$out # store output
+ vxor $out0,$in0,$twk1
+ le?vperm $out2,$out2,$out2,$leperm
+ stvx_u $out1,$x10,$out
+ le?vperm $out3,$out3,$out3,$leperm
+ stvx_u $out2,$x20,$out
+ le?vperm $out4,$out4,$out4,$leperm
+ stvx_u $out3,$x30,$out
+ stvx_u $out4,$x40,$out
+ addi $out,$out,0x50
+ bne Lxts_dec6x_steal
+ b Lxts_dec6x_done
+
+.align 4
+Lxts_dec6x_four:
+ vxor $out0,$in2,$twk0
+ vxor $out1,$in3,$twk1
+ vxor $out2,$in4,$twk2
+ vxor $out3,$in5,$twk3
+ vxor $out4,$out4,$out4
+
+ bl _aesp8_xts_dec5x
+
+ le?vperm $out0,$out0,$out0,$leperm
+ vmr $twk0,$twk4 # unused tweak
+ vmr $twk1,$twk5
+ le?vperm $out1,$out1,$out1,$leperm
+ stvx_u $out0,$x00,$out # store output
+ vxor $out0,$in0,$twk5
+ le?vperm $out2,$out2,$out2,$leperm
+ stvx_u $out1,$x10,$out
+ le?vperm $out3,$out3,$out3,$leperm
+ stvx_u $out2,$x20,$out
+ stvx_u $out3,$x30,$out
+ addi $out,$out,0x40
+ bne Lxts_dec6x_steal
+ b Lxts_dec6x_done
+
+.align 4
+Lxts_dec6x_three:
+ vxor $out0,$in3,$twk0
+ vxor $out1,$in4,$twk1
+ vxor $out2,$in5,$twk2
+ vxor $out3,$out3,$out3
+ vxor $out4,$out4,$out4
+
+ bl _aesp8_xts_dec5x
+
+ le?vperm $out0,$out0,$out0,$leperm
+ vmr $twk0,$twk3 # unused tweak
+ vmr $twk1,$twk4
+ le?vperm $out1,$out1,$out1,$leperm
+ stvx_u $out0,$x00,$out # store output
+ vxor $out0,$in0,$twk4
+ le?vperm $out2,$out2,$out2,$leperm
+ stvx_u $out1,$x10,$out
+ stvx_u $out2,$x20,$out
+ addi $out,$out,0x30
+ bne Lxts_dec6x_steal
+ b Lxts_dec6x_done
+
+.align 4
+Lxts_dec6x_two:
+ vxor $out0,$in4,$twk0
+ vxor $out1,$in5,$twk1
+ vxor $out2,$out2,$out2
+ vxor $out3,$out3,$out3
+ vxor $out4,$out4,$out4
+
+ bl _aesp8_xts_dec5x
+
+ le?vperm $out0,$out0,$out0,$leperm
+ vmr $twk0,$twk2 # unused tweak
+ vmr $twk1,$twk3
+ le?vperm $out1,$out1,$out1,$leperm
+ stvx_u $out0,$x00,$out # store output
+ vxor $out0,$in0,$twk3
+ stvx_u $out1,$x10,$out
+ addi $out,$out,0x20
+ bne Lxts_dec6x_steal
+ b Lxts_dec6x_done
+
+.align 4
+Lxts_dec6x_one:
+ vxor $out0,$in5,$twk0
+ nop
+Loop_xts_dec1x:
+ vncipher $out0,$out0,v24
+ lvx v24,$x20,$key_ # round[3]
+ addi $key_,$key_,0x20
+
+ vncipher $out0,$out0,v25
+ lvx v25,$x10,$key_ # round[4]
+ bdnz Loop_xts_dec1x
+
+ subi r0,$taillen,1
+ vncipher $out0,$out0,v24
+
+ andi. r0,r0,16
+ cmpwi $taillen,0
+ vncipher $out0,$out0,v25
+
+ sub $inp,$inp,r0
+ vncipher $out0,$out0,v26
+
+ lvx_u $in0,0,$inp
+ vncipher $out0,$out0,v27
+
+ addi $key_,$sp,$FRAME+15 # rewind $key_
+ vncipher $out0,$out0,v28
+ lvx v24,$x00,$key_ # re-pre-load round[1]
+
+ vncipher $out0,$out0,v29
+ lvx v25,$x10,$key_ # re-pre-load round[2]
+ vxor $twk0,$twk0,v31
+
+ le?vperm $in0,$in0,$in0,$leperm
+ vncipher $out0,$out0,v30
+
+ mtctr $rounds
+ vncipherlast $out0,$out0,$twk0
+
+ vmr $twk0,$twk1 # unused tweak
+ vmr $twk1,$twk2
+ le?vperm $out0,$out0,$out0,$leperm
+ stvx_u $out0,$x00,$out # store output
+ addi $out,$out,0x10
+ vxor $out0,$in0,$twk2
+ bne Lxts_dec6x_steal
+ b Lxts_dec6x_done
+
+.align 4
+Lxts_dec6x_zero:
+ cmpwi $taillen,0
+ beq Lxts_dec6x_done
+
+ lvx_u $in0,0,$inp
+ le?vperm $in0,$in0,$in0,$leperm
+ vxor $out0,$in0,$twk1
+Lxts_dec6x_steal:
+ vncipher $out0,$out0,v24
+ lvx v24,$x20,$key_ # round[3]
+ addi $key_,$key_,0x20
+
+ vncipher $out0,$out0,v25
+ lvx v25,$x10,$key_ # round[4]
+ bdnz Lxts_dec6x_steal
+
+ add $inp,$inp,$taillen
+ vncipher $out0,$out0,v24
+
+ cmpwi $taillen,0
+ vncipher $out0,$out0,v25
+
+ lvx_u $in0,0,$inp
+ vncipher $out0,$out0,v26
+
+ lvsr $inpperm,0,$taillen # $in5 is no more
+ vncipher $out0,$out0,v27
+
+ addi $key_,$sp,$FRAME+15 # rewind $key_
+ vncipher $out0,$out0,v28
+ lvx v24,$x00,$key_ # re-pre-load round[1]
+
+ vncipher $out0,$out0,v29
+ lvx v25,$x10,$key_ # re-pre-load round[2]
+ vxor $twk1,$twk1,v31
+
+ le?vperm $in0,$in0,$in0,$leperm
+ vncipher $out0,$out0,v30
+
+ vperm $in0,$in0,$in0,$inpperm
+ vncipherlast $tmp,$out0,$twk1
+
+ le?vperm $out0,$tmp,$tmp,$leperm
+ le?stvx_u $out0,0,$out
+ be?stvx_u $tmp,0,$out
+
+ vxor $out0,$out0,$out0
+ vspltisb $out1,-1
+ vperm $out0,$out0,$out1,$inpperm
+ vsel $out0,$in0,$tmp,$out0
+ vxor $out0,$out0,$twk0
+
+ subi r30,$out,1
+ mtctr $taillen
+Loop_xts_dec6x_steal:
+ lbzu r0,1(r30)
+ stb r0,16(r30)
+ bdnz Loop_xts_dec6x_steal
+
+ li $taillen,0
+ mtctr $rounds
+ b Loop_xts_dec1x # one more time...
+
+.align 4
+Lxts_dec6x_done:
+ ${UCMP}i $ivp,0
+ beq Lxts_dec6x_ret
+
+ vxor $tweak,$twk0,$rndkey0
+ le?vperm $tweak,$tweak,$tweak,$leperm
+ stvx_u $tweak,0,$ivp
+
+Lxts_dec6x_ret:
+ mtlr r11
+ li r10,`$FRAME+15`
+ li r11,`$FRAME+31`
+ stvx $seven,r10,$sp # wipe copies of round keys
+ addi r10,r10,32
+ stvx $seven,r11,$sp
+ addi r11,r11,32
+ stvx $seven,r10,$sp
+ addi r10,r10,32
+ stvx $seven,r11,$sp
+ addi r11,r11,32
+ stvx $seven,r10,$sp
+ addi r10,r10,32
+ stvx $seven,r11,$sp
+ addi r11,r11,32
+ stvx $seven,r10,$sp
+ addi r10,r10,32
+ stvx $seven,r11,$sp
+ addi r11,r11,32
+
+ mtspr 256,$vrsave
+ lvx v20,r10,$sp # ABI says so
+ addi r10,r10,32
+ lvx v21,r11,$sp
+ addi r11,r11,32
+ lvx v22,r10,$sp
+ addi r10,r10,32
+ lvx v23,r11,$sp
+ addi r11,r11,32
+ lvx v24,r10,$sp
+ addi r10,r10,32
+ lvx v25,r11,$sp
+ addi r11,r11,32
+ lvx v26,r10,$sp
+ addi r10,r10,32
+ lvx v27,r11,$sp
+ addi r11,r11,32
+ lvx v28,r10,$sp
+ addi r10,r10,32
+ lvx v29,r11,$sp
+ addi r11,r11,32
+ lvx v30,r10,$sp
+ lvx v31,r11,$sp
+ $POP r26,`$FRAME+21*16+0*$SIZE_T`($sp)
+ $POP r27,`$FRAME+21*16+1*$SIZE_T`($sp)
+ $POP r28,`$FRAME+21*16+2*$SIZE_T`($sp)
+ $POP r29,`$FRAME+21*16+3*$SIZE_T`($sp)
+ $POP r30,`$FRAME+21*16+4*$SIZE_T`($sp)
+ $POP r31,`$FRAME+21*16+5*$SIZE_T`($sp)
+ addi $sp,$sp,`$FRAME+21*16+6*$SIZE_T`
+ blr
+ .long 0
+ .byte 0,12,0x04,1,0x80,6,6,0
+ .long 0
+
+.align 5
+_aesp8_xts_dec5x:
+ vncipher $out0,$out0,v24
+ vncipher $out1,$out1,v24
+ vncipher $out2,$out2,v24
+ vncipher $out3,$out3,v24
+ vncipher $out4,$out4,v24
+ lvx v24,$x20,$key_ # round[3]
+ addi $key_,$key_,0x20
+
+ vncipher $out0,$out0,v25
+ vncipher $out1,$out1,v25
+ vncipher $out2,$out2,v25
+ vncipher $out3,$out3,v25
+ vncipher $out4,$out4,v25
+ lvx v25,$x10,$key_ # round[4]
+ bdnz _aesp8_xts_dec5x
+
+ subi r0,$taillen,1
+ vncipher $out0,$out0,v24
+ vncipher $out1,$out1,v24
+ vncipher $out2,$out2,v24
+ vncipher $out3,$out3,v24
+ vncipher $out4,$out4,v24
+
+ andi. r0,r0,16
+ cmpwi $taillen,0
+ vncipher $out0,$out0,v25
+ vncipher $out1,$out1,v25
+ vncipher $out2,$out2,v25
+ vncipher $out3,$out3,v25
+ vncipher $out4,$out4,v25
+ vxor $twk0,$twk0,v31
+
+ sub $inp,$inp,r0
+ vncipher $out0,$out0,v26
+ vncipher $out1,$out1,v26
+ vncipher $out2,$out2,v26
+ vncipher $out3,$out3,v26
+ vncipher $out4,$out4,v26
+ vxor $in1,$twk1,v31
+
+ vncipher $out0,$out0,v27
+ lvx_u $in0,0,$inp
+ vncipher $out1,$out1,v27
+ vncipher $out2,$out2,v27
+ vncipher $out3,$out3,v27
+ vncipher $out4,$out4,v27
+ vxor $in2,$twk2,v31
+
+ addi $key_,$sp,$FRAME+15 # rewind $key_
+ vncipher $out0,$out0,v28
+ vncipher $out1,$out1,v28
+ vncipher $out2,$out2,v28
+ vncipher $out3,$out3,v28
+ vncipher $out4,$out4,v28
+ lvx v24,$x00,$key_ # re-pre-load round[1]
+ vxor $in3,$twk3,v31
+
+ vncipher $out0,$out0,v29
+ le?vperm $in0,$in0,$in0,$leperm
+ vncipher $out1,$out1,v29
+ vncipher $out2,$out2,v29
+ vncipher $out3,$out3,v29
+ vncipher $out4,$out4,v29
+ lvx v25,$x10,$key_ # re-pre-load round[2]
+ vxor $in4,$twk4,v31
+
+ vncipher $out0,$out0,v30
+ vncipher $out1,$out1,v30
+ vncipher $out2,$out2,v30
+ vncipher $out3,$out3,v30
+ vncipher $out4,$out4,v30
+
+ vncipherlast $out0,$out0,$twk0
+ vncipherlast $out1,$out1,$in1
+ vncipherlast $out2,$out2,$in2
+ vncipherlast $out3,$out3,$in3
+ vncipherlast $out4,$out4,$in4
+ mtctr $rounds
+ blr
+ .long 0
+ .byte 0,12,0x14,0,0,0,0,0
+___
+}} }}}
+
my $consts=1;
foreach(split("\n",$code)) {
s/\`([^\`]*)\`/eval($1)/geo;
@@ -1898,7 +3759,7 @@ foreach(split("\n",$code)) {
if ($flavour =~ /le$/o) {
SWITCH: for($conv) {
/\?inv/ && do { @bytes=map($_^0xf,@bytes); last; };
- /\?rev/ && do { @bytes=reverse(@bytes); last; };
+ /\?rev/ && do { @bytes=reverse(@bytes); last; };
}
}
--
2.5.5
^ permalink raw reply related
* Re: [PATCH 3/3] crypto: Added Chelsio Menu to the Kconfig file
From: kbuild test robot @ 2016-07-11 19:30 UTC (permalink / raw)
To: Yeshaswi M R Gowda
Cc: kbuild-all, hariprasad, netdev, linux-kernel, herbert, davem,
linux-crypto, jlulla, atul.gupta, harsh, Yeshaswi M R Gowda
In-Reply-To: <1468261688-24525-4-git-send-email-yeshaswi@chelsio.com>
[-- Attachment #1: Type: text/plain, Size: 1211 bytes --]
Hi,
[auto build test WARNING on net-next/master]
[also build test WARNING on v4.7-rc7 next-20160711]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system]
url: https://github.com/0day-ci/linux/commits/Yeshaswi-M-R-Gowda/crypto-chcr-Add-Chelsio-Crypto-Driver/20160712-023513
config: sh-allmodconfig (attached as .config)
compiler: sh4-linux-gnu-gcc (Debian 5.3.1-8) 5.3.1 20160205
reproduce:
wget https://git.kernel.org/cgit/linux/kernel/git/wfg/lkp-tests.git/plain/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# save the attached .config to linux build tree
make.cross ARCH=sh
All warnings (new ones prefixed by >>):
warning: (ISCSI_TARGET_CXGB4) selects CHELSIO_T4_UWIRE which has unmet direct dependencies (NETDEVICES && ETHERNET && NET_VENDOR_CHELSIO && CHELSIO_T4)
warning: (SCSI_CXGB4_ISCSI && CRYPTO_DEV_CHELSIO) selects CHELSIO_T4 which has unmet direct dependencies (NETDEVICES && ETHERNET && NET_VENDOR_CHELSIO && PCI && (IPV6 || IPV6=n))
---
0-DAY kernel test infrastructure Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all Intel Corporation
[-- Attachment #2: .config.gz --]
[-- Type: application/octet-stream, Size: 40190 bytes --]
^ permalink raw reply
* Re: [PATCH 2/2] crypto: vmx - Adding support for XTS
From: Stephan Mueller @ 2016-07-11 20:10 UTC (permalink / raw)
To: Paulo Flabiano Smorigo
Cc: linux-kernel, Leonidas S. Barbosa, Benjamin Herrenschmidt,
Paul Mackerras, Michael Ellerman, Herbert Xu, David S. Miller,
open list:IBM Power VMX Cryptographic instructions,
open list:LINUX FOR POWERPC (32-BIT AND 64-BIT)
In-Reply-To: <1468264060-22769-2-git-send-email-pfsmorigo@linux.vnet.ibm.com>
Am Montag, 11. Juli 2016, 16:07:40 CEST schrieb Paulo Flabiano Smorigo:
Hi Paulo,
> From: "Leonidas S. Barbosa" <leosilva@linux.vnet.ibm.com>
>
> This patch add XTS support using VMX-crypto driver.
>
> Signed-off-by: Leonidas S. Barbosa <leosilva@linux.vnet.ibm.com>
> Signed-off-by: Paulo Flabiano Smorigo <pfsmorigo@linux.vnet.ibm.com>
> ---
> drivers/crypto/vmx/Makefile | 2 +-
> drivers/crypto/vmx/aes_xts.c | 187
> +++++++++++++++++++++++++++++++++++++++++++ drivers/crypto/vmx/vmx.c |
> 2 +
> 3 files changed, 190 insertions(+), 1 deletion(-)
> create mode 100644 drivers/crypto/vmx/aes_xts.c
>
> diff --git a/drivers/crypto/vmx/Makefile b/drivers/crypto/vmx/Makefile
> index d28ab96..de6e241 100644
> --- a/drivers/crypto/vmx/Makefile
> +++ b/drivers/crypto/vmx/Makefile
> @@ -1,5 +1,5 @@
> obj-$(CONFIG_CRYPTO_DEV_VMX_ENCRYPT) += vmx-crypto.o
> -vmx-crypto-objs := vmx.o aesp8-ppc.o ghashp8-ppc.o aes.o aes_cbc.o
> aes_ctr.o ghash.o +vmx-crypto-objs := vmx.o aesp8-ppc.o ghashp8-ppc.o aes.o
> aes_cbc.o aes_ctr.o aes_xts.o ghash.o
>
> ifeq ($(CONFIG_CPU_LITTLE_ENDIAN),y)
> TARGET := linux-ppc64le
> diff --git a/drivers/crypto/vmx/aes_xts.c b/drivers/crypto/vmx/aes_xts.c
> new file mode 100644
> index 0000000..62e698c
> --- /dev/null
> +++ b/drivers/crypto/vmx/aes_xts.c
> @@ -0,0 +1,187 @@
> +/**
> + * AES XTS routines supporting VMX In-core instructions on Power 8
> + *
> + * Copyright (C) 2015 International Business Machines Inc.
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License as published by
> + * the Free Software Foundations; version 2 only.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY of FITNESS FOR A PARTICUPAR PURPOSE. See the
> + * GNU General Public License for more details.
> + *
> + * You should have received a copy of the GNU General Public License
> + * along with this program; if not, write to the Free Software
> + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
> + *
> + * Author: Leonidas S. Barbosa <leosilva@linux.vnet.ibm.com>
> + */
> +
> +#include <linux/types.h>
> +#include <linux/err.h>
> +#include <linux/crypto.h>
> +#include <linux/delay.h>
> +#include <linux/hardirq.h>
> +#include <asm/switch_to.h>
> +#include <crypto/aes.h>
> +#include <crypto/scatterwalk.h>
> +
> +#include "aesp8-ppc.h"
> +
> +struct p8_aes_xts_ctx {
> + struct crypto_blkcipher *fallback;
> + struct aes_key enc_key;
> + struct aes_key dec_key;
> + struct aes_key tweak_key;
> +};
> +
> +static int p8_aes_xts_init(struct crypto_tfm *tfm)
> +{
> + const char *alg;
> + struct crypto_blkcipher *fallback;
> + struct p8_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm);
> +
> + if (!(alg = crypto_tfm_alg_name(tfm))) {
> + printk(KERN_ERR "Failed to get algorithm name.\n");
> + return -ENOENT;
> + }
> +
> + fallback =
> + crypto_alloc_blkcipher(alg, 0, CRYPTO_ALG_NEED_FALLBACK);
> + if (IS_ERR(fallback)) {
> + printk(KERN_ERR
> + "Failed to allocate transformation for '%s': %ld\n",
> + alg, PTR_ERR(fallback));
> + return PTR_ERR(fallback);
> + }
> + printk(KERN_INFO "Using '%s' as fallback implementation.\n",
> + crypto_tfm_alg_driver_name((struct crypto_tfm *) fallback));
> +
> + crypto_blkcipher_set_flags(
> + fallback,
> + crypto_blkcipher_get_flags((struct crypto_blkcipher *)tfm));
> + ctx->fallback = fallback;
> +
> + return 0;
> +}
> +
> +static void p8_aes_xts_exit(struct crypto_tfm *tfm)
> +{
> + struct p8_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm);
> +
> + if (ctx->fallback) {
> + crypto_free_blkcipher(ctx->fallback);
> + ctx->fallback = NULL;
> + }
> +}
> +
> +static int p8_aes_xts_setkey(struct crypto_tfm *tfm, const u8 *key,
> + unsigned int keylen)
> +{
> + int ret;
> + struct p8_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm);
> +
> + if (keylen % 2)
> + return -EINVAL;
Please use xts_check_key instead of this check.
> +
> + preempt_disable();
> + pagefault_disable();
> + enable_kernel_vsx();
> + ret = aes_p8_set_encrypt_key(key + keylen/2, (keylen/2) * 8,
> &ctx->tweak_key); + ret += aes_p8_set_encrypt_key(key, (keylen/2) * 8,
> &ctx->enc_key); + ret += aes_p8_set_decrypt_key(key, (keylen/2) * 8,
> &ctx->dec_key); + disable_kernel_vsx();
> + pagefault_enable();
> + preempt_enable();
> +
> + ret += crypto_blkcipher_setkey(ctx->fallback, key, keylen);
> + return ret;
> +}
> +
> +static int p8_aes_xts_crypt(struct blkcipher_desc *desc,
> + struct scatterlist *dst,
> + struct scatterlist *src,
> + unsigned int nbytes, int enc)
> +{
> + int ret;
> + u8 tweak[AES_BLOCK_SIZE];
> + u8 *iv;
> + struct blkcipher_walk walk;
> + struct p8_aes_xts_ctx *ctx =
> + crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm));
> + struct blkcipher_desc fallback_desc = {
> + .tfm = ctx->fallback,
> + .info = desc->info,
> + .flags = desc->flags
> + };
> +
> + if (in_interrupt()) {
> + ret = enc ? crypto_blkcipher_encrypt(&fallback_desc, dst, src, nbytes)
:
> + crypto_blkcipher_decrypt(&fallback_desc, dst,
> src, nbytes); + } else {
> + preempt_disable();
> + pagefault_disable();
> + enable_kernel_vsx();
> +
> + blkcipher_walk_init(&walk, dst, src, nbytes);
> +
> + iv = (u8 *)walk.iv;
> + ret = blkcipher_walk_virt(desc, &walk);
> + aes_p8_encrypt(iv, tweak, &ctx->tweak_key);
> +
> + while ((nbytes = walk.nbytes)) {
> + if (enc)
> + aes_p8_xts_encrypt(walk.src.virt.addr, walk.dst.virt.addr,
> + nbytes & AES_BLOCK_MASK, &ctx->enc_key, NULL, tweak);
> + else
> + aes_p8_xts_decrypt(walk.src.virt.addr, walk.dst.virt.addr,
> + nbytes & AES_BLOCK_MASK, &ctx->dec_key, NULL, tweak);
> +
> + nbytes &= AES_BLOCK_SIZE - 1;
> + ret = blkcipher_walk_done(desc, &walk, nbytes);
> + }
> +
> + disable_kernel_vsx();
> + pagefault_enable();
> + preempt_enable();
memzero_explicit(tweak) ?
> + }
> + return ret;
> +}
> +
> +static int p8_aes_xts_encrypt(struct blkcipher_desc *desc,
> + struct scatterlist *dst,
> + struct scatterlist *src, unsigned int nbytes)
> +{
> + return p8_aes_xts_crypt(desc, dst, src, nbytes, 1);
> +}
> +
> +static int p8_aes_xts_decrypt(struct blkcipher_desc *desc,
> + struct scatterlist *dst,
> + struct scatterlist *src, unsigned int nbytes)
> +{
> + return p8_aes_xts_crypt(desc, dst, src, nbytes, 0);
> +}
> +
> +struct crypto_alg p8_aes_xts_alg = {
> + .cra_name = "xts(aes)",
> + .cra_driver_name = "p8_aes_xts",
> + .cra_module = THIS_MODULE,
> + .cra_priority = 2000,
> + .cra_type = &crypto_blkcipher_type,
> + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_NEED_FALLBACK,
> + .cra_alignmask = 0,
> + .cra_blocksize = AES_BLOCK_SIZE,
> + .cra_ctxsize = sizeof(struct p8_aes_xts_ctx),
> + .cra_init = p8_aes_xts_init,
> + .cra_exit = p8_aes_xts_exit,
> + .cra_blkcipher = {
> + .ivsize = AES_BLOCK_SIZE,
> + .min_keysize = 2 * AES_MIN_KEY_SIZE,
> + .max_keysize = 2 * AES_MAX_KEY_SIZE,
> + .setkey = p8_aes_xts_setkey,
> + .encrypt = p8_aes_xts_encrypt,
> + .decrypt = p8_aes_xts_decrypt,
> + }
> +};
> diff --git a/drivers/crypto/vmx/vmx.c b/drivers/crypto/vmx/vmx.c
> index e163d57..f688c32 100644
> --- a/drivers/crypto/vmx/vmx.c
> +++ b/drivers/crypto/vmx/vmx.c
> @@ -31,10 +31,12 @@ extern struct shash_alg p8_ghash_alg;
> extern struct crypto_alg p8_aes_alg;
> extern struct crypto_alg p8_aes_cbc_alg;
> extern struct crypto_alg p8_aes_ctr_alg;
> +extern struct crypto_alg p8_aes_xts_alg;
> static struct crypto_alg *algs[] = {
> &p8_aes_alg,
> &p8_aes_cbc_alg,
> &p8_aes_ctr_alg,
> + &p8_aes_xts_alg,
> NULL,
> };
Ciao
Stephan
^ permalink raw reply
* Re: [PATCH 1/2] crypto: vmx - Adding asm subroutines for XTS
From: Stephen Rothwell @ 2016-07-11 23:37 UTC (permalink / raw)
To: Paulo Flabiano Smorigo
Cc: linux-kernel, Leonidas S. Barbosa, Herbert Xu, Paul Mackerras,
open list:IBM Power VMX Cryptographic instructions,
open list:LINUX FOR POWERPC 32-BIT AND 64-BIT, David S. Miller
In-Reply-To: <1468264060-22769-1-git-send-email-pfsmorigo@linux.vnet.ibm.com>
Hi Paulo,
On Mon, 11 Jul 2016 16:07:39 -0300 Paulo Flabiano Smorigo <pfsmorigo@linux.vnet.ibm.com> wrote:
>
> diff --git a/drivers/crypto/vmx/aesp8-ppc.pl b/drivers/crypto/vmx/aesp8-ppc.pl
> index 2280539..813ffcc 100644
> --- a/drivers/crypto/vmx/aesp8-ppc.pl
> +++ b/drivers/crypto/vmx/aesp8-ppc.pl
> @@ -1,4 +1,11 @@
> -#!/usr/bin/env perl
> +#! /usr/bin/env perl
> +# Copyright 2014-2016 The OpenSSL Project Authors. All Rights Reserved.
> +#
> +# Licensed under the OpenSSL license (the "License"). You may not use
> +# this file except in compliance with the License. You can obtain a copy
> +# in the file LICENSE in the source distribution or at
> +# https://www.openssl.org/source/license.html
So, I assume that this license is compatible with the GPLv2?
--
Cheers,
Stephen Rothwell
^ permalink raw reply
* Re: [PATCH 3/3] crypto: Added Chelsio Menu to the Kconfig file
From: kbuild test robot @ 2016-07-12 0:14 UTC (permalink / raw)
To: Yeshaswi M R Gowda
Cc: kbuild-all, hariprasad, netdev, linux-kernel, herbert, davem,
linux-crypto, jlulla, atul.gupta, harsh, Yeshaswi M R Gowda
In-Reply-To: <1468261688-24525-4-git-send-email-yeshaswi@chelsio.com>
[-- Attachment #1: Type: text/plain, Size: 4873 bytes --]
Hi,
[auto build test WARNING on net-next/master]
[also build test WARNING on v4.7-rc7 next-20160711]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system]
url: https://github.com/0day-ci/linux/commits/Yeshaswi-M-R-Gowda/crypto-chcr-Add-Chelsio-Crypto-Driver/20160712-023513
config: i386-allmodconfig (attached as .config)
compiler: gcc-6 (Debian 6.1.1-1) 6.1.1 20160430
reproduce:
# save the attached .config to linux build tree
make ARCH=i386
All warnings (new ones prefixed by >>):
drivers/crypto/chelsio/chcr_core.c: In function 'cpl_fw6_pld_handler':
>> drivers/crypto/chelsio/chcr_core.c:134:8: warning: cast to pointer from integer of different size [-Wint-to-pointer-cast]
req = (struct crypto_async_request *)cookie;
^
--
In file included from include/linux/swab.h:4:0,
from include/uapi/linux/byteorder/little_endian.h:12,
from include/linux/byteorder/little_endian.h:4,
from arch/x86/include/uapi/asm/byteorder.h:4,
from include/asm-generic/bitops/le.h:5,
from arch/x86/include/asm/bitops.h:504,
from include/linux/bitops.h:36,
from include/linux/kernel.h:10,
from drivers/crypto/chelsio/chcr_algo.c:42:
drivers/crypto/chelsio/chcr_algo.c: In function 'create_wreq':
>> drivers/crypto/chelsio/chcr_algo.c:454:29: warning: cast from pointer to integer of different size [-Wpointer-to-int-cast]
wreq->cookie = cpu_to_be64((u64)req);
^
include/uapi/linux/swab.h:126:54: note: in definition of macro '__swab64'
#define __swab64(x) (__u64)__builtin_bswap64((__u64)(x))
^
>> include/linux/byteorder/generic.h:91:21: note: in expansion of macro '__cpu_to_be64'
#define cpu_to_be64 __cpu_to_be64
^~~~~~~~~~~~~
>> drivers/crypto/chelsio/chcr_algo.c:454:17: note: in expansion of macro 'cpu_to_be64'
wreq->cookie = cpu_to_be64((u64)req);
^~~~~~~~~~~
drivers/crypto/chelsio/chcr_algo.c: In function 'chcr_register_alg':
>> drivers/crypto/chelsio/chcr_algo.c:1471:48: warning: operation on 'driver_algs[i].alg.hash.halg.base.cra_init' may be undefined [-Wsequence-point]
driver_algs[i].alg.hash.halg.base.cra_init =
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
driver_algs[i].alg.hash.halg.base.cra_init =
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
chcr_hmac_cra_init;
~~~~~~~~~~~~~~~~~~
vim +134 drivers/crypto/chelsio/chcr_core.c
5c923415 Yeshaswi M R Gowda 2016-07-11 118 u_ctx->dev = NULL;
5c923415 Yeshaswi M R Gowda 2016-07-11 119 atomic_dec(&dev_count);
5c923415 Yeshaswi M R Gowda 2016-07-11 120 return 0;
5c923415 Yeshaswi M R Gowda 2016-07-11 121 }
5c923415 Yeshaswi M R Gowda 2016-07-11 122
5c923415 Yeshaswi M R Gowda 2016-07-11 123 static int cpl_fw6_pld_handler(struct chcr_dev *dev,
5c923415 Yeshaswi M R Gowda 2016-07-11 124 unsigned char *input)
5c923415 Yeshaswi M R Gowda 2016-07-11 125 {
5c923415 Yeshaswi M R Gowda 2016-07-11 126 struct crypto_async_request *req;
5c923415 Yeshaswi M R Gowda 2016-07-11 127 struct cpl_fw6_pld *fw6_pld;
5c923415 Yeshaswi M R Gowda 2016-07-11 128 u64 cookie;
5c923415 Yeshaswi M R Gowda 2016-07-11 129 u32 ack_err_status = 0;
5c923415 Yeshaswi M R Gowda 2016-07-11 130 int error_status = 0;
5c923415 Yeshaswi M R Gowda 2016-07-11 131
5c923415 Yeshaswi M R Gowda 2016-07-11 132 fw6_pld = (struct cpl_fw6_pld *)input;
5c923415 Yeshaswi M R Gowda 2016-07-11 133 cookie = be64_to_cpu(fw6_pld->data[1]);
5c923415 Yeshaswi M R Gowda 2016-07-11 @134 req = (struct crypto_async_request *)cookie;
5c923415 Yeshaswi M R Gowda 2016-07-11 135
5c923415 Yeshaswi M R Gowda 2016-07-11 136 ack_err_status =
5c923415 Yeshaswi M R Gowda 2016-07-11 137 ntohl(*(__be32 *)((unsigned char *)&fw6_pld->data[0] + 4));
5c923415 Yeshaswi M R Gowda 2016-07-11 138 if (ack_err_status) {
5c923415 Yeshaswi M R Gowda 2016-07-11 139 if (CHK_MAC_ERR_BIT(ack_err_status) ||
5c923415 Yeshaswi M R Gowda 2016-07-11 140 CHK_PAD_ERR_BIT(ack_err_status))
5c923415 Yeshaswi M R Gowda 2016-07-11 141 error_status = -EINVAL;
5c923415 Yeshaswi M R Gowda 2016-07-11 142 }
:::::: The code at line 134 was first introduced by commit
:::::: 5c9234157776103907606c9f4c93a311467e246f chcr: Support for Chelsio's Crypto Hardware
:::::: TO: Yeshaswi M R Gowda <yeshaswi@chelsio.com>
:::::: CC: 0day robot <fengguang.wu@intel.com>
---
0-DAY kernel test infrastructure Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all Intel Corporation
[-- Attachment #2: .config.gz --]
[-- Type: application/octet-stream, Size: 55097 bytes --]
^ permalink raw reply
* Re: [PATCH 3/3] crypto: Added Chelsio Menu to the Kconfig file
From: kbuild test robot @ 2016-07-12 4:36 UTC (permalink / raw)
To: Yeshaswi M R Gowda
Cc: kbuild-all, hariprasad, netdev, linux-kernel, herbert, davem,
linux-crypto, jlulla, atul.gupta, harsh, Yeshaswi M R Gowda
In-Reply-To: <1468261688-24525-4-git-send-email-yeshaswi@chelsio.com>
[-- Attachment #1: Type: text/plain, Size: 12832 bytes --]
Hi,
[auto build test WARNING on net-next/master]
[also build test WARNING on v4.7-rc7 next-20160711]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system]
url: https://github.com/0day-ci/linux/commits/Yeshaswi-M-R-Gowda/crypto-chcr-Add-Chelsio-Crypto-Driver/20160712-023513
config: sh-allyesconfig (attached as .config)
compiler: sh4-linux-gnu-gcc (Debian 5.3.1-8) 5.3.1 20160205
reproduce:
wget https://git.kernel.org/cgit/linux/kernel/git/wfg/lkp-tests.git/plain/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# save the attached .config to linux build tree
make.cross ARCH=sh
All warnings (new ones prefixed by >>):
In file included from include/linux/swab.h:4:0,
from include/uapi/linux/byteorder/little_endian.h:12,
from include/linux/byteorder/little_endian.h:4,
from arch/sh/include/uapi/asm/byteorder.h:5,
from arch/sh/include/asm/bitops.h:11,
from include/linux/bitops.h:36,
from include/linux/kernel.h:10,
from drivers/crypto/chelsio/chcr_algo.c:42:
drivers/crypto/chelsio/chcr_algo.c: In function 'create_wreq':
drivers/crypto/chelsio/chcr_algo.c:454:29: warning: cast from pointer to integer of different size [-Wpointer-to-int-cast]
wreq->cookie = cpu_to_be64((u64)req);
^
include/uapi/linux/swab.h:129:32: note: in definition of macro '__swab64'
(__builtin_constant_p((__u64)(x)) ? \
^
include/linux/byteorder/generic.h:91:21: note: in expansion of macro '__cpu_to_be64'
#define cpu_to_be64 __cpu_to_be64
^
drivers/crypto/chelsio/chcr_algo.c:454:17: note: in expansion of macro 'cpu_to_be64'
wreq->cookie = cpu_to_be64((u64)req);
^
drivers/crypto/chelsio/chcr_algo.c:454:29: warning: cast from pointer to integer of different size [-Wpointer-to-int-cast]
wreq->cookie = cpu_to_be64((u64)req);
^
include/uapi/linux/swab.h:23:12: note: in definition of macro '___constant_swab64'
(((__u64)(x) & (__u64)0x00000000000000ffULL) << 56) | \
^
>> include/uapi/linux/byteorder/little_endian.h:36:43: note: in expansion of macro '__swab64'
#define __cpu_to_be64(x) ((__force __be64)__swab64((x)))
^
include/linux/byteorder/generic.h:91:21: note: in expansion of macro '__cpu_to_be64'
#define cpu_to_be64 __cpu_to_be64
^
drivers/crypto/chelsio/chcr_algo.c:454:17: note: in expansion of macro 'cpu_to_be64'
wreq->cookie = cpu_to_be64((u64)req);
^
drivers/crypto/chelsio/chcr_algo.c:454:29: warning: cast from pointer to integer of different size [-Wpointer-to-int-cast]
wreq->cookie = cpu_to_be64((u64)req);
^
include/uapi/linux/swab.h:24:12: note: in definition of macro '___constant_swab64'
(((__u64)(x) & (__u64)0x000000000000ff00ULL) << 40) | \
^
>> include/uapi/linux/byteorder/little_endian.h:36:43: note: in expansion of macro '__swab64'
#define __cpu_to_be64(x) ((__force __be64)__swab64((x)))
^
include/linux/byteorder/generic.h:91:21: note: in expansion of macro '__cpu_to_be64'
#define cpu_to_be64 __cpu_to_be64
^
drivers/crypto/chelsio/chcr_algo.c:454:17: note: in expansion of macro 'cpu_to_be64'
wreq->cookie = cpu_to_be64((u64)req);
^
drivers/crypto/chelsio/chcr_algo.c:454:29: warning: cast from pointer to integer of different size [-Wpointer-to-int-cast]
wreq->cookie = cpu_to_be64((u64)req);
^
include/uapi/linux/swab.h:25:12: note: in definition of macro '___constant_swab64'
(((__u64)(x) & (__u64)0x0000000000ff0000ULL) << 24) | \
^
>> include/uapi/linux/byteorder/little_endian.h:36:43: note: in expansion of macro '__swab64'
#define __cpu_to_be64(x) ((__force __be64)__swab64((x)))
^
include/linux/byteorder/generic.h:91:21: note: in expansion of macro '__cpu_to_be64'
#define cpu_to_be64 __cpu_to_be64
^
drivers/crypto/chelsio/chcr_algo.c:454:17: note: in expansion of macro 'cpu_to_be64'
wreq->cookie = cpu_to_be64((u64)req);
^
drivers/crypto/chelsio/chcr_algo.c:454:29: warning: cast from pointer to integer of different size [-Wpointer-to-int-cast]
wreq->cookie = cpu_to_be64((u64)req);
^
include/uapi/linux/swab.h:26:12: note: in definition of macro '___constant_swab64'
(((__u64)(x) & (__u64)0x00000000ff000000ULL) << 8) | \
^
>> include/uapi/linux/byteorder/little_endian.h:36:43: note: in expansion of macro '__swab64'
#define __cpu_to_be64(x) ((__force __be64)__swab64((x)))
^
include/linux/byteorder/generic.h:91:21: note: in expansion of macro '__cpu_to_be64'
#define cpu_to_be64 __cpu_to_be64
^
drivers/crypto/chelsio/chcr_algo.c:454:17: note: in expansion of macro 'cpu_to_be64'
wreq->cookie = cpu_to_be64((u64)req);
^
drivers/crypto/chelsio/chcr_algo.c:454:29: warning: cast from pointer to integer of different size [-Wpointer-to-int-cast]
wreq->cookie = cpu_to_be64((u64)req);
^
include/uapi/linux/swab.h:27:12: note: in definition of macro '___constant_swab64'
(((__u64)(x) & (__u64)0x000000ff00000000ULL) >> 8) | \
^
>> include/uapi/linux/byteorder/little_endian.h:36:43: note: in expansion of macro '__swab64'
#define __cpu_to_be64(x) ((__force __be64)__swab64((x)))
^
include/linux/byteorder/generic.h:91:21: note: in expansion of macro '__cpu_to_be64'
#define cpu_to_be64 __cpu_to_be64
^
drivers/crypto/chelsio/chcr_algo.c:454:17: note: in expansion of macro 'cpu_to_be64'
wreq->cookie = cpu_to_be64((u64)req);
^
drivers/crypto/chelsio/chcr_algo.c:454:29: warning: cast from pointer to integer of different size [-Wpointer-to-int-cast]
wreq->cookie = cpu_to_be64((u64)req);
^
include/uapi/linux/swab.h:28:12: note: in definition of macro '___constant_swab64'
(((__u64)(x) & (__u64)0x0000ff0000000000ULL) >> 24) | \
^
>> include/uapi/linux/byteorder/little_endian.h:36:43: note: in expansion of macro '__swab64'
#define __cpu_to_be64(x) ((__force __be64)__swab64((x)))
^
include/linux/byteorder/generic.h:91:21: note: in expansion of macro '__cpu_to_be64'
#define cpu_to_be64 __cpu_to_be64
^
drivers/crypto/chelsio/chcr_algo.c:454:17: note: in expansion of macro 'cpu_to_be64'
wreq->cookie = cpu_to_be64((u64)req);
^
drivers/crypto/chelsio/chcr_algo.c:454:29: warning: cast from pointer to integer of different size [-Wpointer-to-int-cast]
wreq->cookie = cpu_to_be64((u64)req);
^
include/uapi/linux/swab.h:29:12: note: in definition of macro '___constant_swab64'
(((__u64)(x) & (__u64)0x00ff000000000000ULL) >> 40) | \
^
>> include/uapi/linux/byteorder/little_endian.h:36:43: note: in expansion of macro '__swab64'
#define __cpu_to_be64(x) ((__force __be64)__swab64((x)))
^
include/linux/byteorder/generic.h:91:21: note: in expansion of macro '__cpu_to_be64'
#define cpu_to_be64 __cpu_to_be64
^
drivers/crypto/chelsio/chcr_algo.c:454:17: note: in expansion of macro 'cpu_to_be64'
wreq->cookie = cpu_to_be64((u64)req);
^
drivers/crypto/chelsio/chcr_algo.c:454:29: warning: cast from pointer to integer of different size [-Wpointer-to-int-cast]
wreq->cookie = cpu_to_be64((u64)req);
^
include/uapi/linux/swab.h:30:12: note: in definition of macro '___constant_swab64'
(((__u64)(x) & (__u64)0xff00000000000000ULL) >> 56)))
^
>> include/uapi/linux/byteorder/little_endian.h:36:43: note: in expansion of macro '__swab64'
#define __cpu_to_be64(x) ((__force __be64)__swab64((x)))
^
include/linux/byteorder/generic.h:91:21: note: in expansion of macro '__cpu_to_be64'
#define cpu_to_be64 __cpu_to_be64
^
drivers/crypto/chelsio/chcr_algo.c:454:17: note: in expansion of macro 'cpu_to_be64'
wreq->cookie = cpu_to_be64((u64)req);
^
drivers/crypto/chelsio/chcr_algo.c:454:29: warning: cast from pointer to integer of different size [-Wpointer-to-int-cast]
wreq->cookie = cpu_to_be64((u64)req);
^
include/uapi/linux/swab.h:131:12: note: in definition of macro '__swab64'
__fswab64(x))
^
include/linux/byteorder/generic.h:91:21: note: in expansion of macro '__cpu_to_be64'
#define cpu_to_be64 __cpu_to_be64
^
drivers/crypto/chelsio/chcr_algo.c:454:17: note: in expansion of macro 'cpu_to_be64'
wreq->cookie = cpu_to_be64((u64)req);
^
drivers/crypto/chelsio/chcr_algo.c: In function 'chcr_register_alg':
drivers/crypto/chelsio/chcr_algo.c:1471:48: warning: operation on 'driver_algs[i].alg.hash.halg.base.cra_init' may be undefined [-Wsequence-point]
driver_algs[i].alg.hash.halg.base.cra_init =
^
vim +/__swab64 +36 include/uapi/linux/byteorder/little_endian.h
5921e6f8 David Howells 2012-10-13 20 #define __constant_cpu_to_le32(x) ((__force __le32)(__u32)(x))
5921e6f8 David Howells 2012-10-13 21 #define __constant_le32_to_cpu(x) ((__force __u32)(__le32)(x))
5921e6f8 David Howells 2012-10-13 22 #define __constant_cpu_to_le16(x) ((__force __le16)(__u16)(x))
5921e6f8 David Howells 2012-10-13 23 #define __constant_le16_to_cpu(x) ((__force __u16)(__le16)(x))
5921e6f8 David Howells 2012-10-13 24 #define __constant_cpu_to_be64(x) ((__force __be64)___constant_swab64((x)))
5921e6f8 David Howells 2012-10-13 25 #define __constant_be64_to_cpu(x) ___constant_swab64((__force __u64)(__be64)(x))
5921e6f8 David Howells 2012-10-13 26 #define __constant_cpu_to_be32(x) ((__force __be32)___constant_swab32((x)))
5921e6f8 David Howells 2012-10-13 27 #define __constant_be32_to_cpu(x) ___constant_swab32((__force __u32)(__be32)(x))
5921e6f8 David Howells 2012-10-13 28 #define __constant_cpu_to_be16(x) ((__force __be16)___constant_swab16((x)))
5921e6f8 David Howells 2012-10-13 29 #define __constant_be16_to_cpu(x) ___constant_swab16((__force __u16)(__be16)(x))
5921e6f8 David Howells 2012-10-13 30 #define __cpu_to_le64(x) ((__force __le64)(__u64)(x))
5921e6f8 David Howells 2012-10-13 31 #define __le64_to_cpu(x) ((__force __u64)(__le64)(x))
5921e6f8 David Howells 2012-10-13 32 #define __cpu_to_le32(x) ((__force __le32)(__u32)(x))
5921e6f8 David Howells 2012-10-13 33 #define __le32_to_cpu(x) ((__force __u32)(__le32)(x))
5921e6f8 David Howells 2012-10-13 34 #define __cpu_to_le16(x) ((__force __le16)(__u16)(x))
5921e6f8 David Howells 2012-10-13 35 #define __le16_to_cpu(x) ((__force __u16)(__le16)(x))
5921e6f8 David Howells 2012-10-13 @36 #define __cpu_to_be64(x) ((__force __be64)__swab64((x)))
5921e6f8 David Howells 2012-10-13 37 #define __be64_to_cpu(x) __swab64((__force __u64)(__be64)(x))
5921e6f8 David Howells 2012-10-13 38 #define __cpu_to_be32(x) ((__force __be32)__swab32((x)))
5921e6f8 David Howells 2012-10-13 39 #define __be32_to_cpu(x) __swab32((__force __u32)(__be32)(x))
5921e6f8 David Howells 2012-10-13 40 #define __cpu_to_be16(x) ((__force __be16)__swab16((x)))
5921e6f8 David Howells 2012-10-13 41 #define __be16_to_cpu(x) __swab16((__force __u16)(__be16)(x))
5921e6f8 David Howells 2012-10-13 42
bc27fb68 Denys Vlasenko 2016-03-17 43 static __always_inline __le64 __cpu_to_le64p(const __u64 *p)
5921e6f8 David Howells 2012-10-13 44 {
:::::: The code at line 36 was first introduced by commit
:::::: 5921e6f8809b1616932ca4afd40fe449faa8fd88 UAPI: (Scripted) Disintegrate include/linux/byteorder
:::::: TO: David Howells <dhowells@redhat.com>
:::::: CC: David Howells <dhowells@redhat.com>
---
0-DAY kernel test infrastructure Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all Intel Corporation
[-- Attachment #2: .config.gz --]
[-- Type: application/octet-stream, Size: 40314 bytes --]
^ permalink raw reply
* [PATCH 0/30] crypto: skcipher - skcipher algorithm conversion part 2
From: Herbert Xu @ 2016-07-12 5:15 UTC (permalink / raw)
To: Linux Crypto Mailing List
Hi:
This patch series continues the task of converting blkcipher and
ablkcipher implementations over to the unified skcipher interface.
It includes a repost of patches in the previous series which have
not been included because of the addition of the chunksize parameter
(block size for stream ciphers and the like which still need to be
processed in blocks except for the last one).
This series also removes the top-level blkcipher/ablkcipher entry
points, meaning that while they can still be used by internal crypto
entities such as cryptd, they can no longer be allocated from the
outside.
Finally this series introduces a new skcipher walk interface that
combines both blkcipher walk and ablkcipher walk into one. This
is then used in the conversion of the lrw algorithm to skcipher.
Cheers,
--
Email: Herbert Xu <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
^ permalink raw reply
* [PATCH 1/30] crypto: skcipher - Add low-level skcipher interface
From: Herbert Xu @ 2016-07-12 5:17 UTC (permalink / raw)
To: Linux Crypto Mailing List
In-Reply-To: <20160712051554.GA28324@gondor.apana.org.au>
This patch allows skcipher algorithms and instances to be created
and registered with the crypto API. They are accessible through
the top-level skcipher interface, along with ablkcipher/blkcipher
algorithms and instances.
This patch also introduces a new parameter called chunk size
which is meant for ciphers such as CTR and CTS which ostensibly
can handle arbitrary lengths, but still behave like block ciphers
in that you can only process a partial block at the very end.
For these ciphers the block size will continue to be set to 1
as it is now while the chunk size will be set to the underlying
block size.
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
---
crypto/skcipher.c | 196 +++++++++++++++++++++++++++++++++++--
include/crypto/internal/skcipher.h | 87 ++++++++++++++++
include/crypto/skcipher.h | 130 ++++++++++++++++++++++++
include/linux/crypto.h | 1
4 files changed, 407 insertions(+), 7 deletions(-)
diff --git a/crypto/skcipher.c b/crypto/skcipher.c
index 69230e9..d248008 100644
--- a/crypto/skcipher.c
+++ b/crypto/skcipher.c
@@ -16,7 +16,11 @@
#include <crypto/internal/skcipher.h>
#include <linux/bug.h>
+#include <linux/cryptouser.h>
#include <linux/module.h>
+#include <linux/rtnetlink.h>
+#include <linux/seq_file.h>
+#include <net/netlink.h>
#include "internal.h"
@@ -25,10 +29,11 @@ static unsigned int crypto_skcipher_extsize(struct crypto_alg *alg)
if (alg->cra_type == &crypto_blkcipher_type)
return sizeof(struct crypto_blkcipher *);
- BUG_ON(alg->cra_type != &crypto_ablkcipher_type &&
- alg->cra_type != &crypto_givcipher_type);
+ if (alg->cra_type == &crypto_ablkcipher_type ||
+ alg->cra_type == &crypto_givcipher_type)
+ return sizeof(struct crypto_ablkcipher *);
- return sizeof(struct crypto_ablkcipher *);
+ return crypto_alg_extsize(alg);
}
static int skcipher_setkey_blkcipher(struct crypto_skcipher *tfm,
@@ -216,26 +221,118 @@ static int crypto_init_skcipher_ops_ablkcipher(struct crypto_tfm *tfm)
return 0;
}
+static void crypto_skcipher_exit_tfm(struct crypto_tfm *tfm)
+{
+ struct crypto_skcipher *skcipher = __crypto_skcipher_cast(tfm);
+ struct skcipher_alg *alg = crypto_skcipher_alg(skcipher);
+
+ alg->exit(skcipher);
+}
+
static int crypto_skcipher_init_tfm(struct crypto_tfm *tfm)
{
+ struct crypto_skcipher *skcipher = __crypto_skcipher_cast(tfm);
+ struct skcipher_alg *alg = crypto_skcipher_alg(skcipher);
+
if (tfm->__crt_alg->cra_type == &crypto_blkcipher_type)
return crypto_init_skcipher_ops_blkcipher(tfm);
- BUG_ON(tfm->__crt_alg->cra_type != &crypto_ablkcipher_type &&
- tfm->__crt_alg->cra_type != &crypto_givcipher_type);
+ if (tfm->__crt_alg->cra_type == &crypto_ablkcipher_type ||
+ tfm->__crt_alg->cra_type == &crypto_givcipher_type)
+ return crypto_init_skcipher_ops_ablkcipher(tfm);
+
+ skcipher->setkey = alg->setkey;
+ skcipher->encrypt = alg->encrypt;
+ skcipher->decrypt = alg->decrypt;
+ skcipher->ivsize = alg->ivsize;
+ skcipher->keysize = alg->max_keysize;
+
+ if (alg->exit)
+ skcipher->base.exit = crypto_skcipher_exit_tfm;
- return crypto_init_skcipher_ops_ablkcipher(tfm);
+ if (alg->init)
+ return alg->init(skcipher);
+
+ return 0;
+}
+
+static void crypto_skcipher_free_instance(struct crypto_instance *inst)
+{
+ struct skcipher_instance *skcipher =
+ container_of(inst, struct skcipher_instance, s.base);
+
+ skcipher->free(skcipher);
+}
+
+static void crypto_skcipher_show(struct seq_file *m, struct crypto_alg *alg)
+ __attribute__ ((unused));
+static void crypto_skcipher_show(struct seq_file *m, struct crypto_alg *alg)
+{
+ struct skcipher_alg *skcipher = container_of(alg, struct skcipher_alg,
+ base);
+
+ seq_printf(m, "type : skcipher\n");
+ seq_printf(m, "async : %s\n",
+ alg->cra_flags & CRYPTO_ALG_ASYNC ? "yes" : "no");
+ seq_printf(m, "blocksize : %u\n", alg->cra_blocksize);
+ seq_printf(m, "min keysize : %u\n", skcipher->min_keysize);
+ seq_printf(m, "max keysize : %u\n", skcipher->max_keysize);
+ seq_printf(m, "ivsize : %u\n", skcipher->ivsize);
+ seq_printf(m, "chunksize : %u\n", skcipher->chunksize);
}
+#ifdef CONFIG_NET
+static int crypto_skcipher_report(struct sk_buff *skb, struct crypto_alg *alg)
+{
+ struct crypto_report_blkcipher rblkcipher;
+ struct skcipher_alg *skcipher = container_of(alg, struct skcipher_alg,
+ base);
+
+ strncpy(rblkcipher.type, "skcipher", sizeof(rblkcipher.type));
+ strncpy(rblkcipher.geniv, "<none>", sizeof(rblkcipher.geniv));
+
+ rblkcipher.blocksize = alg->cra_blocksize;
+ rblkcipher.min_keysize = skcipher->min_keysize;
+ rblkcipher.max_keysize = skcipher->max_keysize;
+ rblkcipher.ivsize = skcipher->ivsize;
+
+ if (nla_put(skb, CRYPTOCFGA_REPORT_BLKCIPHER,
+ sizeof(struct crypto_report_blkcipher), &rblkcipher))
+ goto nla_put_failure;
+ return 0;
+
+nla_put_failure:
+ return -EMSGSIZE;
+}
+#else
+static int crypto_skcipher_report(struct sk_buff *skb, struct crypto_alg *alg)
+{
+ return -ENOSYS;
+}
+#endif
+
static const struct crypto_type crypto_skcipher_type2 = {
.extsize = crypto_skcipher_extsize,
.init_tfm = crypto_skcipher_init_tfm,
+ .free = crypto_skcipher_free_instance,
+#ifdef CONFIG_PROC_FS
+ .show = crypto_skcipher_show,
+#endif
+ .report = crypto_skcipher_report,
.maskclear = ~CRYPTO_ALG_TYPE_MASK,
.maskset = CRYPTO_ALG_TYPE_BLKCIPHER_MASK,
- .type = CRYPTO_ALG_TYPE_BLKCIPHER,
+ .type = CRYPTO_ALG_TYPE_SKCIPHER,
.tfmsize = offsetof(struct crypto_skcipher, base),
};
+int crypto_grab_skcipher2(struct crypto_skcipher_spawn *spawn,
+ const char *name, u32 type, u32 mask)
+{
+ spawn->base.frontend = &crypto_skcipher_type2;
+ return crypto_grab_spawn(&spawn->base, name, type, mask);
+}
+EXPORT_SYMBOL_GPL(crypto_grab_skcipher2);
+
struct crypto_skcipher *crypto_alloc_skcipher(const char *alg_name,
u32 type, u32 mask)
{
@@ -243,5 +340,90 @@ struct crypto_skcipher *crypto_alloc_skcipher(const char *alg_name,
}
EXPORT_SYMBOL_GPL(crypto_alloc_skcipher);
+int crypto_has_skcipher2(const char *alg_name, u32 type, u32 mask)
+{
+ return crypto_type_has_alg(alg_name, &crypto_skcipher_type2,
+ type, mask);
+}
+EXPORT_SYMBOL_GPL(crypto_has_skcipher2);
+
+static int skcipher_prepare_alg(struct skcipher_alg *alg)
+{
+ struct crypto_alg *base = &alg->base;
+
+ if (alg->ivsize > PAGE_SIZE / 8 || alg->chunksize > PAGE_SIZE / 8)
+ return -EINVAL;
+
+ if (!alg->chunksize)
+ alg->chunksize = base->cra_blocksize;
+
+ base->cra_type = &crypto_skcipher_type2;
+ base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK;
+ base->cra_flags |= CRYPTO_ALG_TYPE_SKCIPHER;
+
+ return 0;
+}
+
+int crypto_register_skcipher(struct skcipher_alg *alg)
+{
+ struct crypto_alg *base = &alg->base;
+ int err;
+
+ err = skcipher_prepare_alg(alg);
+ if (err)
+ return err;
+
+ return crypto_register_alg(base);
+}
+EXPORT_SYMBOL_GPL(crypto_register_skcipher);
+
+void crypto_unregister_skcipher(struct skcipher_alg *alg)
+{
+ crypto_unregister_alg(&alg->base);
+}
+EXPORT_SYMBOL_GPL(crypto_unregister_skcipher);
+
+int crypto_register_skciphers(struct skcipher_alg *algs, int count)
+{
+ int i, ret;
+
+ for (i = 0; i < count; i++) {
+ ret = crypto_register_skcipher(&algs[i]);
+ if (ret)
+ goto err;
+ }
+
+ return 0;
+
+err:
+ for (--i; i >= 0; --i)
+ crypto_unregister_skcipher(&algs[i]);
+
+ return ret;
+}
+EXPORT_SYMBOL_GPL(crypto_register_skciphers);
+
+void crypto_unregister_skciphers(struct skcipher_alg *algs, int count)
+{
+ int i;
+
+ for (i = count - 1; i >= 0; --i)
+ crypto_unregister_skcipher(&algs[i]);
+}
+EXPORT_SYMBOL_GPL(crypto_unregister_skciphers);
+
+int skcipher_register_instance(struct crypto_template *tmpl,
+ struct skcipher_instance *inst)
+{
+ int err;
+
+ err = skcipher_prepare_alg(&inst->alg);
+ if (err)
+ return err;
+
+ return crypto_register_instance(tmpl, skcipher_crypto_instance(inst));
+}
+EXPORT_SYMBOL_GPL(skcipher_register_instance);
+
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Symmetric key cipher type");
diff --git a/include/crypto/internal/skcipher.h b/include/crypto/internal/skcipher.h
index 2cf7a61..ce6619c 100644
--- a/include/crypto/internal/skcipher.h
+++ b/include/crypto/internal/skcipher.h
@@ -19,12 +19,46 @@
struct rtattr;
+struct skcipher_instance {
+ void (*free)(struct skcipher_instance *inst);
+ union {
+ struct {
+ char head[offsetof(struct skcipher_alg, base)];
+ struct crypto_instance base;
+ } s;
+ struct skcipher_alg alg;
+ };
+};
+
struct crypto_skcipher_spawn {
struct crypto_spawn base;
};
extern const struct crypto_type crypto_givcipher_type;
+static inline struct crypto_instance *skcipher_crypto_instance(
+ struct skcipher_instance *inst)
+{
+ return &inst->s.base;
+}
+
+static inline struct skcipher_instance *skcipher_alg_instance(
+ struct crypto_skcipher *skcipher)
+{
+ return container_of(crypto_skcipher_alg(skcipher),
+ struct skcipher_instance, alg);
+}
+
+static inline void *skcipher_instance_ctx(struct skcipher_instance *inst)
+{
+ return crypto_instance_ctx(skcipher_crypto_instance(inst));
+}
+
+static inline void skcipher_request_complete(struct skcipher_request *req, int err)
+{
+ req->base.complete(&req->base, err);
+}
+
static inline void crypto_set_skcipher_spawn(
struct crypto_skcipher_spawn *spawn, struct crypto_instance *inst)
{
@@ -33,6 +67,8 @@ static inline void crypto_set_skcipher_spawn(
int crypto_grab_skcipher(struct crypto_skcipher_spawn *spawn, const char *name,
u32 type, u32 mask);
+int crypto_grab_skcipher2(struct crypto_skcipher_spawn *spawn,
+ const char *name, u32 type, u32 mask);
struct crypto_alg *crypto_lookup_skcipher(const char *name, u32 type, u32 mask);
@@ -47,6 +83,12 @@ static inline struct crypto_alg *crypto_skcipher_spawn_alg(
return spawn->base.alg;
}
+static inline struct skcipher_alg *crypto_spawn_skcipher_alg(
+ struct crypto_skcipher_spawn *spawn)
+{
+ return container_of(spawn->base.alg, struct skcipher_alg, base);
+}
+
static inline struct crypto_ablkcipher *crypto_spawn_skcipher(
struct crypto_skcipher_spawn *spawn)
{
@@ -55,6 +97,25 @@ static inline struct crypto_ablkcipher *crypto_spawn_skcipher(
crypto_skcipher_mask(0)));
}
+static inline struct crypto_skcipher *crypto_spawn_skcipher2(
+ struct crypto_skcipher_spawn *spawn)
+{
+ return crypto_spawn_tfm2(&spawn->base);
+}
+
+static inline void crypto_skcipher_set_reqsize(
+ struct crypto_skcipher *skcipher, unsigned int reqsize)
+{
+ skcipher->reqsize = reqsize;
+}
+
+int crypto_register_skcipher(struct skcipher_alg *alg);
+void crypto_unregister_skcipher(struct skcipher_alg *alg);
+int crypto_register_skciphers(struct skcipher_alg *algs, int count);
+void crypto_unregister_skciphers(struct skcipher_alg *algs, int count);
+int skcipher_register_instance(struct crypto_template *tmpl,
+ struct skcipher_instance *inst);
+
int skcipher_null_givencrypt(struct skcipher_givcrypt_request *req);
int skcipher_null_givdecrypt(struct skcipher_givcrypt_request *req);
const char *crypto_default_geniv(const struct crypto_alg *alg);
@@ -122,5 +183,31 @@ static inline u32 skcipher_request_flags(struct skcipher_request *req)
return req->base.flags;
}
+static inline unsigned int crypto_skcipher_alg_min_keysize(
+ struct skcipher_alg *alg)
+{
+ if ((alg->base.cra_flags & CRYPTO_ALG_TYPE_MASK) ==
+ CRYPTO_ALG_TYPE_BLKCIPHER)
+ return alg->base.cra_blkcipher.min_keysize;
+
+ if (alg->base.cra_ablkcipher.encrypt)
+ return alg->base.cra_ablkcipher.min_keysize;
+
+ return alg->min_keysize;
+}
+
+static inline unsigned int crypto_skcipher_alg_max_keysize(
+ struct skcipher_alg *alg)
+{
+ if ((alg->base.cra_flags & CRYPTO_ALG_TYPE_MASK) ==
+ CRYPTO_ALG_TYPE_BLKCIPHER)
+ return alg->base.cra_blkcipher.max_keysize;
+
+ if (alg->base.cra_ablkcipher.encrypt)
+ return alg->base.cra_ablkcipher.max_keysize;
+
+ return alg->max_keysize;
+}
+
#endif /* _CRYPTO_INTERNAL_SKCIPHER_H */
diff --git a/include/crypto/skcipher.h b/include/crypto/skcipher.h
index 0f987f5..a381f57 100644
--- a/include/crypto/skcipher.h
+++ b/include/crypto/skcipher.h
@@ -65,6 +65,75 @@ struct crypto_skcipher {
struct crypto_tfm base;
};
+/**
+ * struct skcipher_alg - symmetric key cipher definition
+ * @min_keysize: Minimum key size supported by the transformation. This is the
+ * smallest key length supported by this transformation algorithm.
+ * This must be set to one of the pre-defined values as this is
+ * not hardware specific. Possible values for this field can be
+ * found via git grep "_MIN_KEY_SIZE" include/crypto/
+ * @max_keysize: Maximum key size supported by the transformation. This is the
+ * largest key length supported by this transformation algorithm.
+ * This must be set to one of the pre-defined values as this is
+ * not hardware specific. Possible values for this field can be
+ * found via git grep "_MAX_KEY_SIZE" include/crypto/
+ * @setkey: Set key for the transformation. This function is used to either
+ * program a supplied key into the hardware or store the key in the
+ * transformation context for programming it later. Note that this
+ * function does modify the transformation context. This function can
+ * be called multiple times during the existence of the transformation
+ * object, so one must make sure the key is properly reprogrammed into
+ * the hardware. This function is also responsible for checking the key
+ * length for validity. In case a software fallback was put in place in
+ * the @cra_init call, this function might need to use the fallback if
+ * the algorithm doesn't support all of the key sizes.
+ * @encrypt: Encrypt a scatterlist of blocks. This function is used to encrypt
+ * the supplied scatterlist containing the blocks of data. The crypto
+ * API consumer is responsible for aligning the entries of the
+ * scatterlist properly and making sure the chunks are correctly
+ * sized. In case a software fallback was put in place in the
+ * @cra_init call, this function might need to use the fallback if
+ * the algorithm doesn't support all of the key sizes. In case the
+ * key was stored in transformation context, the key might need to be
+ * re-programmed into the hardware in this function. This function
+ * shall not modify the transformation context, as this function may
+ * be called in parallel with the same transformation object.
+ * @decrypt: Decrypt a single block. This is a reverse counterpart to @encrypt
+ * and the conditions are exactly the same.
+ * @init: Initialize the cryptographic transformation object. This function
+ * is used to initialize the cryptographic transformation object.
+ * This function is called only once at the instantiation time, right
+ * after the transformation context was allocated. In case the
+ * cryptographic hardware has some special requirements which need to
+ * be handled by software, this function shall check for the precise
+ * requirement of the transformation and put any software fallbacks
+ * in place.
+ * @exit: Deinitialize the cryptographic transformation object. This is a
+ * counterpart to @init, used to remove various changes set in
+ * @init.
+ * @ivsize: IV size applicable for transformation. The consumer must provide an
+ * IV of exactly that size to perform the encrypt or decrypt operation.
+ * @chunksize: Equal to the block size except for stream ciphers such as
+ * CTR where it is set to the underlying block size.
+ *
+ * All fields except @ivsize are mandatory and must be filled.
+ */
+struct skcipher_alg {
+ int (*setkey)(struct crypto_skcipher *tfm, const u8 *key,
+ unsigned int keylen);
+ int (*encrypt)(struct skcipher_request *req);
+ int (*decrypt)(struct skcipher_request *req);
+ int (*init)(struct crypto_skcipher *tfm);
+ void (*exit)(struct crypto_skcipher *tfm);
+
+ unsigned int min_keysize;
+ unsigned int max_keysize;
+ unsigned int ivsize;
+ unsigned int chunksize;
+
+ struct crypto_alg base;
+};
+
#define SKCIPHER_REQUEST_ON_STACK(name, tfm) \
char __##name##_desc[sizeof(struct skcipher_request) + \
crypto_skcipher_reqsize(tfm)] CRYPTO_MINALIGN_ATTR; \
@@ -231,12 +300,43 @@ static inline int crypto_has_skcipher(const char *alg_name, u32 type,
crypto_skcipher_mask(mask));
}
+/**
+ * crypto_has_skcipher2() - Search for the availability of an skcipher.
+ * @alg_name: is the cra_name / name or cra_driver_name / driver name of the
+ * skcipher
+ * @type: specifies the type of the skcipher
+ * @mask: specifies the mask for the skcipher
+ *
+ * Return: true when the skcipher is known to the kernel crypto API; false
+ * otherwise
+ */
+int crypto_has_skcipher2(const char *alg_name, u32 type, u32 mask);
+
static inline const char *crypto_skcipher_driver_name(
struct crypto_skcipher *tfm)
{
return crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm));
}
+static inline struct skcipher_alg *crypto_skcipher_alg(
+ struct crypto_skcipher *tfm)
+{
+ return container_of(crypto_skcipher_tfm(tfm)->__crt_alg,
+ struct skcipher_alg, base);
+}
+
+static inline unsigned int crypto_skcipher_alg_ivsize(struct skcipher_alg *alg)
+{
+ if ((alg->base.cra_flags & CRYPTO_ALG_TYPE_MASK) ==
+ CRYPTO_ALG_TYPE_BLKCIPHER)
+ return alg->base.cra_blkcipher.ivsize;
+
+ if (alg->base.cra_ablkcipher.encrypt)
+ return alg->base.cra_ablkcipher.ivsize;
+
+ return alg->ivsize;
+}
+
/**
* crypto_skcipher_ivsize() - obtain IV size
* @tfm: cipher handle
@@ -251,6 +351,36 @@ static inline unsigned int crypto_skcipher_ivsize(struct crypto_skcipher *tfm)
return tfm->ivsize;
}
+static inline unsigned int crypto_skcipher_alg_chunksize(
+ struct skcipher_alg *alg)
+{
+ if ((alg->base.cra_flags & CRYPTO_ALG_TYPE_MASK) ==
+ CRYPTO_ALG_TYPE_BLKCIPHER)
+ return alg->base.cra_blocksize;
+
+ if (alg->base.cra_ablkcipher.encrypt)
+ return alg->base.cra_blocksize;
+
+ return alg->chunksize;
+}
+
+/**
+ * crypto_skcipher_chunksize() - obtain chunk size
+ * @tfm: cipher handle
+ *
+ * The block size is set to one for ciphers such as CTR. However,
+ * you still need to provide incremental updates in multiples of
+ * the underlying block size as the IV does not have sub-block
+ * granularity. This is known in this API as the chunk size.
+ *
+ * Return: chunk size in bytes
+ */
+static inline unsigned int crypto_skcipher_chunksize(
+ struct crypto_skcipher *tfm)
+{
+ return crypto_skcipher_alg_chunksize(crypto_skcipher_alg(tfm));
+}
+
/**
* crypto_skcipher_blocksize() - obtain block size of cipher
* @tfm: cipher handle
diff --git a/include/linux/crypto.h b/include/linux/crypto.h
index 992cfc2..37a652d 100644
--- a/include/linux/crypto.h
+++ b/include/linux/crypto.h
@@ -47,6 +47,7 @@
#define CRYPTO_ALG_TYPE_AEAD 0x00000003
#define CRYPTO_ALG_TYPE_BLKCIPHER 0x00000004
#define CRYPTO_ALG_TYPE_ABLKCIPHER 0x00000005
+#define CRYPTO_ALG_TYPE_SKCIPHER 0x00000005
#define CRYPTO_ALG_TYPE_GIVCIPHER 0x00000006
#define CRYPTO_ALG_TYPE_KPP 0x00000008
#define CRYPTO_ALG_TYPE_RNG 0x0000000c
^ permalink raw reply related
* [PATCH 2/30] crypto: null - Add new default null skcipher
From: Herbert Xu @ 2016-07-12 5:17 UTC (permalink / raw)
To: Linux Crypto Mailing List
In-Reply-To: <20160712051554.GA28324@gondor.apana.org.au>
Current the default null skcipher is actually a crypto_blkcipher.
This patch creates a synchronous crypto_skcipher version of the
null cipher which unfortunately has to settle for the name skcipher2.
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
---
crypto/crypto_null.c | 38 ++++++++++++++++++++++++++++++++++++++
include/crypto/null.h | 2 ++
2 files changed, 40 insertions(+)
diff --git a/crypto/crypto_null.c b/crypto/crypto_null.c
index 941c9a4..c3f6839 100644
--- a/crypto/crypto_null.c
+++ b/crypto/crypto_null.c
@@ -28,6 +28,8 @@
static DEFINE_MUTEX(crypto_default_null_skcipher_lock);
static struct crypto_blkcipher *crypto_default_null_skcipher;
static int crypto_default_null_skcipher_refcnt;
+static struct crypto_skcipher *crypto_default_null_skcipher2;
+static int crypto_default_null_skcipher2_refcnt;
static int null_compress(struct crypto_tfm *tfm, const u8 *src,
unsigned int slen, u8 *dst, unsigned int *dlen)
@@ -188,6 +190,42 @@ void crypto_put_default_null_skcipher(void)
}
EXPORT_SYMBOL_GPL(crypto_put_default_null_skcipher);
+struct crypto_skcipher *crypto_get_default_null_skcipher2(void)
+{
+ struct crypto_skcipher *tfm;
+
+ mutex_lock(&crypto_default_null_skcipher_lock);
+ tfm = crypto_default_null_skcipher2;
+
+ if (!tfm) {
+ tfm = crypto_alloc_skcipher("ecb(cipher_null)",
+ 0, CRYPTO_ALG_ASYNC);
+ if (IS_ERR(tfm))
+ goto unlock;
+
+ crypto_default_null_skcipher2 = tfm;
+ }
+
+ crypto_default_null_skcipher2_refcnt++;
+
+unlock:
+ mutex_unlock(&crypto_default_null_skcipher_lock);
+
+ return tfm;
+}
+EXPORT_SYMBOL_GPL(crypto_get_default_null_skcipher2);
+
+void crypto_put_default_null_skcipher2(void)
+{
+ mutex_lock(&crypto_default_null_skcipher_lock);
+ if (!--crypto_default_null_skcipher2_refcnt) {
+ crypto_free_skcipher(crypto_default_null_skcipher2);
+ crypto_default_null_skcipher2 = NULL;
+ }
+ mutex_unlock(&crypto_default_null_skcipher_lock);
+}
+EXPORT_SYMBOL_GPL(crypto_put_default_null_skcipher2);
+
static int __init crypto_null_mod_init(void)
{
int ret = 0;
diff --git a/include/crypto/null.h b/include/crypto/null.h
index 06dc30d..dda87cb 100644
--- a/include/crypto/null.h
+++ b/include/crypto/null.h
@@ -10,5 +10,7 @@
struct crypto_blkcipher *crypto_get_default_null_skcipher(void);
void crypto_put_default_null_skcipher(void);
+struct crypto_skcipher *crypto_get_default_null_skcipher2(void);
+void crypto_put_default_null_skcipher2(void);
#endif
^ permalink raw reply related
* [PATCH 3/30] crypto: aead - Add chunk size
From: Herbert Xu @ 2016-07-12 5:17 UTC (permalink / raw)
To: Linux Crypto Mailing List
In-Reply-To: <20160712051554.GA28324@gondor.apana.org.au>
This patch adds a chunk size parameter to aead algorithms, just
like the chunk size for skcipher algorithms.
However, unlike skcipher we do not currently export this to AEAD
users. It is only meant to be used by AEAD implementors for now.
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
---
crypto/aead.c | 6 +++++-
include/crypto/aead.h | 12 +++++++-----
include/crypto/internal/aead.h | 21 +++++++++++++++++++++
3 files changed, 33 insertions(+), 6 deletions(-)
diff --git a/crypto/aead.c b/crypto/aead.c
index 9b18a1e..b155cbc 100644
--- a/crypto/aead.c
+++ b/crypto/aead.c
@@ -346,9 +346,13 @@ static int aead_prepare_alg(struct aead_alg *alg)
{
struct crypto_alg *base = &alg->base;
- if (max(alg->maxauthsize, alg->ivsize) > PAGE_SIZE / 8)
+ if (max3(alg->maxauthsize, alg->ivsize, alg->chunksize) >
+ PAGE_SIZE / 8)
return -EINVAL;
+ if (!alg->chunksize)
+ alg->chunksize = base->cra_blocksize;
+
base->cra_type = &crypto_aead_type;
base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK;
base->cra_flags |= CRYPTO_ALG_TYPE_AEAD;
diff --git a/include/crypto/aead.h b/include/crypto/aead.h
index 75174f8..12f8432 100644
--- a/include/crypto/aead.h
+++ b/include/crypto/aead.h
@@ -112,11 +112,12 @@ struct aead_request {
* supplied during the decryption operation. This function is also
* responsible for checking the authentication tag size for
* validity.
- * @setkey: see struct ablkcipher_alg
- * @encrypt: see struct ablkcipher_alg
- * @decrypt: see struct ablkcipher_alg
- * @geniv: see struct ablkcipher_alg
- * @ivsize: see struct ablkcipher_alg
+ * @setkey: see struct skcipher_alg
+ * @encrypt: see struct skcipher_alg
+ * @decrypt: see struct skcipher_alg
+ * @geniv: see struct skcipher_alg
+ * @ivsize: see struct skcipher_alg
+ * @chunksize: see struct skcipher_alg
* @init: Initialize the cryptographic transformation object. This function
* is used to initialize the cryptographic transformation object.
* This function is called only once at the instantiation time, right
@@ -145,6 +146,7 @@ struct aead_alg {
unsigned int ivsize;
unsigned int maxauthsize;
+ unsigned int chunksize;
struct crypto_alg base;
};
diff --git a/include/crypto/internal/aead.h b/include/crypto/internal/aead.h
index da38649..6ad8e31 100644
--- a/include/crypto/internal/aead.h
+++ b/include/crypto/internal/aead.h
@@ -159,6 +159,27 @@ static inline struct aead_request *aead_get_backlog(struct aead_queue *queue)
return req ? container_of(req, struct aead_request, base) : NULL;
}
+static inline unsigned int crypto_aead_alg_chunksize(struct aead_alg *alg)
+{
+ return alg->chunksize;
+}
+
+/**
+ * crypto_aead_chunksize() - obtain chunk size
+ * @tfm: cipher handle
+ *
+ * The block size is set to one for ciphers such as CCM. However,
+ * you still need to provide incremental updates in multiples of
+ * the underlying block size as the IV does not have sub-block
+ * granularity. This is known in this API as the chunk size.
+ *
+ * Return: chunk size in bytes
+ */
+static inline unsigned int crypto_aead_chunksize(struct crypto_aead *tfm)
+{
+ return crypto_aead_alg_chunksize(crypto_aead_alg(tfm));
+}
+
int crypto_register_aead(struct aead_alg *alg);
void crypto_unregister_aead(struct aead_alg *alg);
int crypto_register_aeads(struct aead_alg *algs, int count);
^ permalink raw reply related
* [PATCH 4/30] crypto: authenc - Use skcipher
From: Herbert Xu @ 2016-07-12 5:17 UTC (permalink / raw)
To: Linux Crypto Mailing List
In-Reply-To: <20160712051554.GA28324@gondor.apana.org.au>
This patch converts authenc to use the new skcipher interface as
opposed to ablkcipher.
It also fixes a little bug where if a sync version of authenc
is requested we may still end up using an async ahash. This should
have no effect as none of the authenc users can request for a
sync authenc.
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
---
crypto/authenc.c | 107 ++++++++++++++++++++++++++++---------------------------
1 file changed, 56 insertions(+), 51 deletions(-)
diff --git a/crypto/authenc.c b/crypto/authenc.c
index 309fbc1..a7e1ac7 100644
--- a/crypto/authenc.c
+++ b/crypto/authenc.c
@@ -32,8 +32,8 @@ struct authenc_instance_ctx {
struct crypto_authenc_ctx {
struct crypto_ahash *auth;
- struct crypto_ablkcipher *enc;
- struct crypto_blkcipher *null;
+ struct crypto_skcipher *enc;
+ struct crypto_skcipher *null;
};
struct authenc_request_ctx {
@@ -83,7 +83,7 @@ static int crypto_authenc_setkey(struct crypto_aead *authenc, const u8 *key,
{
struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc);
struct crypto_ahash *auth = ctx->auth;
- struct crypto_ablkcipher *enc = ctx->enc;
+ struct crypto_skcipher *enc = ctx->enc;
struct crypto_authenc_keys keys;
int err = -EINVAL;
@@ -100,11 +100,11 @@ static int crypto_authenc_setkey(struct crypto_aead *authenc, const u8 *key,
if (err)
goto out;
- crypto_ablkcipher_clear_flags(enc, CRYPTO_TFM_REQ_MASK);
- crypto_ablkcipher_set_flags(enc, crypto_aead_get_flags(authenc) &
- CRYPTO_TFM_REQ_MASK);
- err = crypto_ablkcipher_setkey(enc, keys.enckey, keys.enckeylen);
- crypto_aead_set_flags(authenc, crypto_ablkcipher_get_flags(enc) &
+ crypto_skcipher_clear_flags(enc, CRYPTO_TFM_REQ_MASK);
+ crypto_skcipher_set_flags(enc, crypto_aead_get_flags(authenc) &
+ CRYPTO_TFM_REQ_MASK);
+ err = crypto_skcipher_setkey(enc, keys.enckey, keys.enckeylen);
+ crypto_aead_set_flags(authenc, crypto_skcipher_get_flags(enc) &
CRYPTO_TFM_RES_MASK);
out:
@@ -184,12 +184,15 @@ static int crypto_authenc_copy_assoc(struct aead_request *req)
{
struct crypto_aead *authenc = crypto_aead_reqtfm(req);
struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc);
- struct blkcipher_desc desc = {
- .tfm = ctx->null,
- };
+ SKCIPHER_REQUEST_ON_STACK(skreq, ctx->null);
- return crypto_blkcipher_encrypt(&desc, req->dst, req->src,
- req->assoclen);
+ skcipher_request_set_tfm(skreq, ctx->null);
+ skcipher_request_set_callback(skreq, aead_request_flags(req),
+ NULL, NULL);
+ skcipher_request_set_crypt(skreq, req->src, req->dst, req->assoclen,
+ NULL);
+
+ return crypto_skcipher_encrypt(skreq);
}
static int crypto_authenc_encrypt(struct aead_request *req)
@@ -199,10 +202,10 @@ static int crypto_authenc_encrypt(struct aead_request *req)
struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc);
struct authenc_instance_ctx *ictx = aead_instance_ctx(inst);
struct authenc_request_ctx *areq_ctx = aead_request_ctx(req);
- struct crypto_ablkcipher *enc = ctx->enc;
+ struct crypto_skcipher *enc = ctx->enc;
unsigned int cryptlen = req->cryptlen;
- struct ablkcipher_request *abreq = (void *)(areq_ctx->tail +
- ictx->reqoff);
+ struct skcipher_request *skreq = (void *)(areq_ctx->tail +
+ ictx->reqoff);
struct scatterlist *src, *dst;
int err;
@@ -217,12 +220,12 @@ static int crypto_authenc_encrypt(struct aead_request *req)
dst = scatterwalk_ffwd(areq_ctx->dst, req->dst, req->assoclen);
}
- ablkcipher_request_set_tfm(abreq, enc);
- ablkcipher_request_set_callback(abreq, aead_request_flags(req),
- crypto_authenc_encrypt_done, req);
- ablkcipher_request_set_crypt(abreq, src, dst, cryptlen, req->iv);
+ skcipher_request_set_tfm(skreq, enc);
+ skcipher_request_set_callback(skreq, aead_request_flags(req),
+ crypto_authenc_encrypt_done, req);
+ skcipher_request_set_crypt(skreq, src, dst, cryptlen, req->iv);
- err = crypto_ablkcipher_encrypt(abreq);
+ err = crypto_skcipher_encrypt(skreq);
if (err)
return err;
@@ -238,8 +241,8 @@ static int crypto_authenc_decrypt_tail(struct aead_request *req,
struct authenc_instance_ctx *ictx = aead_instance_ctx(inst);
struct authenc_request_ctx *areq_ctx = aead_request_ctx(req);
struct ahash_request *ahreq = (void *)(areq_ctx->tail + ictx->reqoff);
- struct ablkcipher_request *abreq = (void *)(areq_ctx->tail +
- ictx->reqoff);
+ struct skcipher_request *skreq = (void *)(areq_ctx->tail +
+ ictx->reqoff);
unsigned int authsize = crypto_aead_authsize(authenc);
u8 *ihash = ahreq->result + authsize;
struct scatterlist *src, *dst;
@@ -255,13 +258,13 @@ static int crypto_authenc_decrypt_tail(struct aead_request *req,
if (req->src != req->dst)
dst = scatterwalk_ffwd(areq_ctx->dst, req->dst, req->assoclen);
- ablkcipher_request_set_tfm(abreq, ctx->enc);
- ablkcipher_request_set_callback(abreq, aead_request_flags(req),
- req->base.complete, req->base.data);
- ablkcipher_request_set_crypt(abreq, src, dst,
- req->cryptlen - authsize, req->iv);
+ skcipher_request_set_tfm(skreq, ctx->enc);
+ skcipher_request_set_callback(skreq, aead_request_flags(req),
+ req->base.complete, req->base.data);
+ skcipher_request_set_crypt(skreq, src, dst,
+ req->cryptlen - authsize, req->iv);
- return crypto_ablkcipher_decrypt(abreq);
+ return crypto_skcipher_decrypt(skreq);
}
static void authenc_verify_ahash_done(struct crypto_async_request *areq,
@@ -313,20 +316,20 @@ static int crypto_authenc_init_tfm(struct crypto_aead *tfm)
struct authenc_instance_ctx *ictx = aead_instance_ctx(inst);
struct crypto_authenc_ctx *ctx = crypto_aead_ctx(tfm);
struct crypto_ahash *auth;
- struct crypto_ablkcipher *enc;
- struct crypto_blkcipher *null;
+ struct crypto_skcipher *enc;
+ struct crypto_skcipher *null;
int err;
auth = crypto_spawn_ahash(&ictx->auth);
if (IS_ERR(auth))
return PTR_ERR(auth);
- enc = crypto_spawn_skcipher(&ictx->enc);
+ enc = crypto_spawn_skcipher2(&ictx->enc);
err = PTR_ERR(enc);
if (IS_ERR(enc))
goto err_free_ahash;
- null = crypto_get_default_null_skcipher();
+ null = crypto_get_default_null_skcipher2();
err = PTR_ERR(null);
if (IS_ERR(null))
goto err_free_skcipher;
@@ -342,13 +345,13 @@ static int crypto_authenc_init_tfm(struct crypto_aead *tfm)
max_t(unsigned int,
crypto_ahash_reqsize(auth) +
sizeof(struct ahash_request),
- sizeof(struct ablkcipher_request) +
- crypto_ablkcipher_reqsize(enc)));
+ sizeof(struct skcipher_request) +
+ crypto_skcipher_reqsize(enc)));
return 0;
err_free_skcipher:
- crypto_free_ablkcipher(enc);
+ crypto_free_skcipher(enc);
err_free_ahash:
crypto_free_ahash(auth);
return err;
@@ -359,8 +362,8 @@ static void crypto_authenc_exit_tfm(struct crypto_aead *tfm)
struct crypto_authenc_ctx *ctx = crypto_aead_ctx(tfm);
crypto_free_ahash(ctx->auth);
- crypto_free_ablkcipher(ctx->enc);
- crypto_put_default_null_skcipher();
+ crypto_free_skcipher(ctx->enc);
+ crypto_put_default_null_skcipher2();
}
static void crypto_authenc_free(struct aead_instance *inst)
@@ -379,7 +382,7 @@ static int crypto_authenc_create(struct crypto_template *tmpl,
struct aead_instance *inst;
struct hash_alg_common *auth;
struct crypto_alg *auth_base;
- struct crypto_alg *enc;
+ struct skcipher_alg *enc;
struct authenc_instance_ctx *ctx;
const char *enc_name;
int err;
@@ -417,38 +420,40 @@ static int crypto_authenc_create(struct crypto_template *tmpl,
goto err_free_inst;
crypto_set_skcipher_spawn(&ctx->enc, aead_crypto_instance(inst));
- err = crypto_grab_skcipher(&ctx->enc, enc_name, 0,
- crypto_requires_sync(algt->type,
- algt->mask));
+ err = crypto_grab_skcipher2(&ctx->enc, enc_name, 0,
+ crypto_requires_sync(algt->type,
+ algt->mask));
if (err)
goto err_drop_auth;
- enc = crypto_skcipher_spawn_alg(&ctx->enc);
+ enc = crypto_spawn_skcipher_alg(&ctx->enc);
ctx->reqoff = ALIGN(2 * auth->digestsize + auth_base->cra_alignmask,
auth_base->cra_alignmask + 1);
err = -ENAMETOOLONG;
if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,
- "authenc(%s,%s)", auth_base->cra_name, enc->cra_name) >=
+ "authenc(%s,%s)", auth_base->cra_name,
+ enc->base.cra_name) >=
CRYPTO_MAX_ALG_NAME)
goto err_drop_enc;
if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
"authenc(%s,%s)", auth_base->cra_driver_name,
- enc->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
+ enc->base.cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
goto err_drop_enc;
- inst->alg.base.cra_flags = (auth_base->cra_flags | enc->cra_flags) &
- CRYPTO_ALG_ASYNC;
- inst->alg.base.cra_priority = enc->cra_priority * 10 +
+ inst->alg.base.cra_flags = (auth_base->cra_flags |
+ enc->base.cra_flags) & CRYPTO_ALG_ASYNC;
+ inst->alg.base.cra_priority = enc->base.cra_priority * 10 +
auth_base->cra_priority;
- inst->alg.base.cra_blocksize = enc->cra_blocksize;
+ inst->alg.base.cra_blocksize = enc->base.cra_blocksize;
inst->alg.base.cra_alignmask = auth_base->cra_alignmask |
- enc->cra_alignmask;
+ enc->base.cra_alignmask;
inst->alg.base.cra_ctxsize = sizeof(struct crypto_authenc_ctx);
- inst->alg.ivsize = enc->cra_ablkcipher.ivsize;
+ inst->alg.ivsize = crypto_skcipher_alg_ivsize(enc);
+ inst->alg.chunksize = crypto_skcipher_alg_chunksize(enc);
inst->alg.maxauthsize = auth->digestsize;
inst->alg.init = crypto_authenc_init_tfm;
^ permalink raw reply related
* [PATCH 5/30] crypto: authencesn - Use skcipher
From: Herbert Xu @ 2016-07-12 5:17 UTC (permalink / raw)
To: Linux Crypto Mailing List
In-Reply-To: <20160712051554.GA28324@gondor.apana.org.au>
This patch converts authencesn to use the new skcipher interface as
opposed to ablkcipher.
It also fixes a little bug where if a sync version of authencesn
is requested we may still end up using an async ahash. This should
have no effect as none of the authencesn users can request for a
sync authencesn.
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
---
crypto/authencesn.c | 104 +++++++++++++++++++++++++++-------------------------
1 file changed, 54 insertions(+), 50 deletions(-)
diff --git a/crypto/authencesn.c b/crypto/authencesn.c
index 0662b18..121010a 100644
--- a/crypto/authencesn.c
+++ b/crypto/authencesn.c
@@ -35,8 +35,8 @@ struct authenc_esn_instance_ctx {
struct crypto_authenc_esn_ctx {
unsigned int reqoff;
struct crypto_ahash *auth;
- struct crypto_ablkcipher *enc;
- struct crypto_blkcipher *null;
+ struct crypto_skcipher *enc;
+ struct crypto_skcipher *null;
};
struct authenc_esn_request_ctx {
@@ -65,7 +65,7 @@ static int crypto_authenc_esn_setkey(struct crypto_aead *authenc_esn, const u8 *
{
struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn);
struct crypto_ahash *auth = ctx->auth;
- struct crypto_ablkcipher *enc = ctx->enc;
+ struct crypto_skcipher *enc = ctx->enc;
struct crypto_authenc_keys keys;
int err = -EINVAL;
@@ -82,11 +82,11 @@ static int crypto_authenc_esn_setkey(struct crypto_aead *authenc_esn, const u8 *
if (err)
goto out;
- crypto_ablkcipher_clear_flags(enc, CRYPTO_TFM_REQ_MASK);
- crypto_ablkcipher_set_flags(enc, crypto_aead_get_flags(authenc_esn) &
+ crypto_skcipher_clear_flags(enc, CRYPTO_TFM_REQ_MASK);
+ crypto_skcipher_set_flags(enc, crypto_aead_get_flags(authenc_esn) &
CRYPTO_TFM_REQ_MASK);
- err = crypto_ablkcipher_setkey(enc, keys.enckey, keys.enckeylen);
- crypto_aead_set_flags(authenc_esn, crypto_ablkcipher_get_flags(enc) &
+ err = crypto_skcipher_setkey(enc, keys.enckey, keys.enckeylen);
+ crypto_aead_set_flags(authenc_esn, crypto_skcipher_get_flags(enc) &
CRYPTO_TFM_RES_MASK);
out:
@@ -182,11 +182,14 @@ static int crypto_authenc_esn_copy(struct aead_request *req, unsigned int len)
{
struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req);
struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn);
- struct blkcipher_desc desc = {
- .tfm = ctx->null,
- };
+ SKCIPHER_REQUEST_ON_STACK(skreq, ctx->null);
- return crypto_blkcipher_encrypt(&desc, req->dst, req->src, len);
+ skcipher_request_set_tfm(skreq, ctx->null);
+ skcipher_request_set_callback(skreq, aead_request_flags(req),
+ NULL, NULL);
+ skcipher_request_set_crypt(skreq, req->src, req->dst, len, NULL);
+
+ return crypto_skcipher_encrypt(skreq);
}
static int crypto_authenc_esn_encrypt(struct aead_request *req)
@@ -194,9 +197,9 @@ static int crypto_authenc_esn_encrypt(struct aead_request *req)
struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req);
struct authenc_esn_request_ctx *areq_ctx = aead_request_ctx(req);
struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn);
- struct ablkcipher_request *abreq = (void *)(areq_ctx->tail
- + ctx->reqoff);
- struct crypto_ablkcipher *enc = ctx->enc;
+ struct skcipher_request *skreq = (void *)(areq_ctx->tail +
+ ctx->reqoff);
+ struct crypto_skcipher *enc = ctx->enc;
unsigned int assoclen = req->assoclen;
unsigned int cryptlen = req->cryptlen;
struct scatterlist *src, *dst;
@@ -215,12 +218,12 @@ static int crypto_authenc_esn_encrypt(struct aead_request *req)
dst = scatterwalk_ffwd(areq_ctx->dst, req->dst, assoclen);
}
- ablkcipher_request_set_tfm(abreq, enc);
- ablkcipher_request_set_callback(abreq, aead_request_flags(req),
- crypto_authenc_esn_encrypt_done, req);
- ablkcipher_request_set_crypt(abreq, src, dst, cryptlen, req->iv);
+ skcipher_request_set_tfm(skreq, enc);
+ skcipher_request_set_callback(skreq, aead_request_flags(req),
+ crypto_authenc_esn_encrypt_done, req);
+ skcipher_request_set_crypt(skreq, src, dst, cryptlen, req->iv);
- err = crypto_ablkcipher_encrypt(abreq);
+ err = crypto_skcipher_encrypt(skreq);
if (err)
return err;
@@ -234,8 +237,8 @@ static int crypto_authenc_esn_decrypt_tail(struct aead_request *req,
unsigned int authsize = crypto_aead_authsize(authenc_esn);
struct authenc_esn_request_ctx *areq_ctx = aead_request_ctx(req);
struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn);
- struct ablkcipher_request *abreq = (void *)(areq_ctx->tail
- + ctx->reqoff);
+ struct skcipher_request *skreq = (void *)(areq_ctx->tail +
+ ctx->reqoff);
struct crypto_ahash *auth = ctx->auth;
u8 *ohash = PTR_ALIGN((u8 *)areq_ctx->tail,
crypto_ahash_alignmask(auth) + 1);
@@ -256,12 +259,12 @@ static int crypto_authenc_esn_decrypt_tail(struct aead_request *req,
sg_init_table(areq_ctx->dst, 2);
dst = scatterwalk_ffwd(areq_ctx->dst, dst, assoclen);
- ablkcipher_request_set_tfm(abreq, ctx->enc);
- ablkcipher_request_set_callback(abreq, flags,
- req->base.complete, req->base.data);
- ablkcipher_request_set_crypt(abreq, dst, dst, cryptlen, req->iv);
+ skcipher_request_set_tfm(skreq, ctx->enc);
+ skcipher_request_set_callback(skreq, flags,
+ req->base.complete, req->base.data);
+ skcipher_request_set_crypt(skreq, dst, dst, cryptlen, req->iv);
- return crypto_ablkcipher_decrypt(abreq);
+ return crypto_skcipher_decrypt(skreq);
}
static void authenc_esn_verify_ahash_done(struct crypto_async_request *areq,
@@ -331,20 +334,20 @@ static int crypto_authenc_esn_init_tfm(struct crypto_aead *tfm)
struct authenc_esn_instance_ctx *ictx = aead_instance_ctx(inst);
struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(tfm);
struct crypto_ahash *auth;
- struct crypto_ablkcipher *enc;
- struct crypto_blkcipher *null;
+ struct crypto_skcipher *enc;
+ struct crypto_skcipher *null;
int err;
auth = crypto_spawn_ahash(&ictx->auth);
if (IS_ERR(auth))
return PTR_ERR(auth);
- enc = crypto_spawn_skcipher(&ictx->enc);
+ enc = crypto_spawn_skcipher2(&ictx->enc);
err = PTR_ERR(enc);
if (IS_ERR(enc))
goto err_free_ahash;
- null = crypto_get_default_null_skcipher();
+ null = crypto_get_default_null_skcipher2();
err = PTR_ERR(null);
if (IS_ERR(null))
goto err_free_skcipher;
@@ -361,15 +364,15 @@ static int crypto_authenc_esn_init_tfm(struct crypto_aead *tfm)
sizeof(struct authenc_esn_request_ctx) +
ctx->reqoff +
max_t(unsigned int,
- crypto_ahash_reqsize(auth) +
- sizeof(struct ahash_request),
- sizeof(struct skcipher_givcrypt_request) +
- crypto_ablkcipher_reqsize(enc)));
+ crypto_ahash_reqsize(auth) +
+ sizeof(struct ahash_request),
+ sizeof(struct skcipher_request) +
+ crypto_skcipher_reqsize(enc)));
return 0;
err_free_skcipher:
- crypto_free_ablkcipher(enc);
+ crypto_free_skcipher(enc);
err_free_ahash:
crypto_free_ahash(auth);
return err;
@@ -380,8 +383,8 @@ static void crypto_authenc_esn_exit_tfm(struct crypto_aead *tfm)
struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(tfm);
crypto_free_ahash(ctx->auth);
- crypto_free_ablkcipher(ctx->enc);
- crypto_put_default_null_skcipher();
+ crypto_free_skcipher(ctx->enc);
+ crypto_put_default_null_skcipher2();
}
static void crypto_authenc_esn_free(struct aead_instance *inst)
@@ -400,7 +403,7 @@ static int crypto_authenc_esn_create(struct crypto_template *tmpl,
struct aead_instance *inst;
struct hash_alg_common *auth;
struct crypto_alg *auth_base;
- struct crypto_alg *enc;
+ struct skcipher_alg *enc;
struct authenc_esn_instance_ctx *ctx;
const char *enc_name;
int err;
@@ -438,35 +441,36 @@ static int crypto_authenc_esn_create(struct crypto_template *tmpl,
goto err_free_inst;
crypto_set_skcipher_spawn(&ctx->enc, aead_crypto_instance(inst));
- err = crypto_grab_skcipher(&ctx->enc, enc_name, 0,
- crypto_requires_sync(algt->type,
- algt->mask));
+ err = crypto_grab_skcipher2(&ctx->enc, enc_name, 0,
+ crypto_requires_sync(algt->type,
+ algt->mask));
if (err)
goto err_drop_auth;
- enc = crypto_skcipher_spawn_alg(&ctx->enc);
+ enc = crypto_spawn_skcipher_alg(&ctx->enc);
err = -ENAMETOOLONG;
if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,
"authencesn(%s,%s)", auth_base->cra_name,
- enc->cra_name) >= CRYPTO_MAX_ALG_NAME)
+ enc->base.cra_name) >= CRYPTO_MAX_ALG_NAME)
goto err_drop_enc;
if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
"authencesn(%s,%s)", auth_base->cra_driver_name,
- enc->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
+ enc->base.cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
goto err_drop_enc;
- inst->alg.base.cra_flags = (auth_base->cra_flags | enc->cra_flags) &
- CRYPTO_ALG_ASYNC;
- inst->alg.base.cra_priority = enc->cra_priority * 10 +
+ inst->alg.base.cra_flags = (auth_base->cra_flags |
+ enc->base.cra_flags) & CRYPTO_ALG_ASYNC;
+ inst->alg.base.cra_priority = enc->base.cra_priority * 10 +
auth_base->cra_priority;
- inst->alg.base.cra_blocksize = enc->cra_blocksize;
+ inst->alg.base.cra_blocksize = enc->base.cra_blocksize;
inst->alg.base.cra_alignmask = auth_base->cra_alignmask |
- enc->cra_alignmask;
+ enc->base.cra_alignmask;
inst->alg.base.cra_ctxsize = sizeof(struct crypto_authenc_esn_ctx);
- inst->alg.ivsize = enc->cra_ablkcipher.ivsize;
+ inst->alg.ivsize = crypto_skcipher_alg_ivsize(enc);
+ inst->alg.chunksize = crypto_skcipher_alg_chunksize(enc);
inst->alg.maxauthsize = auth->digestsize;
inst->alg.init = crypto_authenc_esn_init_tfm;
^ permalink raw reply related
* [PATCH 6/30] crypto: ctr - Use skcipher in rfc3686
From: Herbert Xu @ 2016-07-12 5:17 UTC (permalink / raw)
To: Linux Crypto Mailing List
In-Reply-To: <20160712051554.GA28324@gondor.apana.org.au>
This patch converts rfc3686 to use the new skcipher interface as
opposed to ablkcipher.
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
---
crypto/ctr.c | 183 ++++++++++++++++++++++++++++++-----------------------------
1 file changed, 94 insertions(+), 89 deletions(-)
diff --git a/crypto/ctr.c b/crypto/ctr.c
index 2386f73..ff4d21e 100644
--- a/crypto/ctr.c
+++ b/crypto/ctr.c
@@ -26,13 +26,13 @@ struct crypto_ctr_ctx {
};
struct crypto_rfc3686_ctx {
- struct crypto_ablkcipher *child;
+ struct crypto_skcipher *child;
u8 nonce[CTR_RFC3686_NONCE_SIZE];
};
struct crypto_rfc3686_req_ctx {
u8 iv[CTR_RFC3686_BLOCK_SIZE];
- struct ablkcipher_request subreq CRYPTO_MINALIGN_ATTR;
+ struct skcipher_request subreq CRYPTO_MINALIGN_ATTR;
};
static int crypto_ctr_setkey(struct crypto_tfm *parent, const u8 *key,
@@ -249,11 +249,11 @@ static struct crypto_template crypto_ctr_tmpl = {
.module = THIS_MODULE,
};
-static int crypto_rfc3686_setkey(struct crypto_ablkcipher *parent,
+static int crypto_rfc3686_setkey(struct crypto_skcipher *parent,
const u8 *key, unsigned int keylen)
{
- struct crypto_rfc3686_ctx *ctx = crypto_ablkcipher_ctx(parent);
- struct crypto_ablkcipher *child = ctx->child;
+ struct crypto_rfc3686_ctx *ctx = crypto_skcipher_ctx(parent);
+ struct crypto_skcipher *child = ctx->child;
int err;
/* the nonce is stored in bytes at end of key */
@@ -265,173 +265,178 @@ static int crypto_rfc3686_setkey(struct crypto_ablkcipher *parent,
keylen -= CTR_RFC3686_NONCE_SIZE;
- crypto_ablkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
- crypto_ablkcipher_set_flags(child, crypto_ablkcipher_get_flags(parent) &
- CRYPTO_TFM_REQ_MASK);
- err = crypto_ablkcipher_setkey(child, key, keylen);
- crypto_ablkcipher_set_flags(parent, crypto_ablkcipher_get_flags(child) &
- CRYPTO_TFM_RES_MASK);
+ crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
+ crypto_skcipher_set_flags(child, crypto_skcipher_get_flags(parent) &
+ CRYPTO_TFM_REQ_MASK);
+ err = crypto_skcipher_setkey(child, key, keylen);
+ crypto_skcipher_set_flags(parent, crypto_skcipher_get_flags(child) &
+ CRYPTO_TFM_RES_MASK);
return err;
}
-static int crypto_rfc3686_crypt(struct ablkcipher_request *req)
+static int crypto_rfc3686_crypt(struct skcipher_request *req)
{
- struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
- struct crypto_rfc3686_ctx *ctx = crypto_ablkcipher_ctx(tfm);
- struct crypto_ablkcipher *child = ctx->child;
- unsigned long align = crypto_ablkcipher_alignmask(tfm);
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ struct crypto_rfc3686_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct crypto_skcipher *child = ctx->child;
+ unsigned long align = crypto_skcipher_alignmask(tfm);
struct crypto_rfc3686_req_ctx *rctx =
- (void *)PTR_ALIGN((u8 *)ablkcipher_request_ctx(req), align + 1);
- struct ablkcipher_request *subreq = &rctx->subreq;
+ (void *)PTR_ALIGN((u8 *)skcipher_request_ctx(req), align + 1);
+ struct skcipher_request *subreq = &rctx->subreq;
u8 *iv = rctx->iv;
/* set up counter block */
memcpy(iv, ctx->nonce, CTR_RFC3686_NONCE_SIZE);
- memcpy(iv + CTR_RFC3686_NONCE_SIZE, req->info, CTR_RFC3686_IV_SIZE);
+ memcpy(iv + CTR_RFC3686_NONCE_SIZE, req->iv, CTR_RFC3686_IV_SIZE);
/* initialize counter portion of counter block */
*(__be32 *)(iv + CTR_RFC3686_NONCE_SIZE + CTR_RFC3686_IV_SIZE) =
cpu_to_be32(1);
- ablkcipher_request_set_tfm(subreq, child);
- ablkcipher_request_set_callback(subreq, req->base.flags,
- req->base.complete, req->base.data);
- ablkcipher_request_set_crypt(subreq, req->src, req->dst, req->nbytes,
- iv);
+ skcipher_request_set_tfm(subreq, child);
+ skcipher_request_set_callback(subreq, req->base.flags,
+ req->base.complete, req->base.data);
+ skcipher_request_set_crypt(subreq, req->src, req->dst,
+ req->cryptlen, iv);
- return crypto_ablkcipher_encrypt(subreq);
+ return crypto_skcipher_encrypt(subreq);
}
-static int crypto_rfc3686_init_tfm(struct crypto_tfm *tfm)
+static int crypto_rfc3686_init_tfm(struct crypto_skcipher *tfm)
{
- struct crypto_instance *inst = (void *)tfm->__crt_alg;
- struct crypto_skcipher_spawn *spawn = crypto_instance_ctx(inst);
- struct crypto_rfc3686_ctx *ctx = crypto_tfm_ctx(tfm);
- struct crypto_ablkcipher *cipher;
+ struct skcipher_instance *inst = skcipher_alg_instance(tfm);
+ struct crypto_skcipher_spawn *spawn = skcipher_instance_ctx(inst);
+ struct crypto_rfc3686_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct crypto_skcipher *cipher;
unsigned long align;
+ unsigned int reqsize;
- cipher = crypto_spawn_skcipher(spawn);
+ cipher = crypto_spawn_skcipher2(spawn);
if (IS_ERR(cipher))
return PTR_ERR(cipher);
ctx->child = cipher;
- align = crypto_tfm_alg_alignmask(tfm);
+ align = crypto_skcipher_alignmask(tfm);
align &= ~(crypto_tfm_ctx_alignment() - 1);
- tfm->crt_ablkcipher.reqsize = align +
- sizeof(struct crypto_rfc3686_req_ctx) +
- crypto_ablkcipher_reqsize(cipher);
+ reqsize = align + sizeof(struct crypto_rfc3686_req_ctx) +
+ crypto_skcipher_reqsize(cipher);
+ crypto_skcipher_set_reqsize(tfm, reqsize);
return 0;
}
-static void crypto_rfc3686_exit_tfm(struct crypto_tfm *tfm)
+static void crypto_rfc3686_exit_tfm(struct crypto_skcipher *tfm)
{
- struct crypto_rfc3686_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct crypto_rfc3686_ctx *ctx = crypto_skcipher_ctx(tfm);
+
+ crypto_free_skcipher(ctx->child);
+}
- crypto_free_ablkcipher(ctx->child);
+static void crypto_rfc3686_free(struct skcipher_instance *inst)
+{
+ struct crypto_skcipher_spawn *spawn = skcipher_instance_ctx(inst);
+
+ crypto_drop_skcipher(spawn);
+ kfree(inst);
}
-static struct crypto_instance *crypto_rfc3686_alloc(struct rtattr **tb)
+static int crypto_rfc3686_create(struct crypto_template *tmpl,
+ struct rtattr **tb)
{
struct crypto_attr_type *algt;
- struct crypto_instance *inst;
- struct crypto_alg *alg;
+ struct skcipher_instance *inst;
+ struct skcipher_alg *alg;
struct crypto_skcipher_spawn *spawn;
const char *cipher_name;
int err;
algt = crypto_get_attr_type(tb);
if (IS_ERR(algt))
- return ERR_CAST(algt);
+ return PTR_ERR(algt);
- if ((algt->type ^ CRYPTO_ALG_TYPE_BLKCIPHER) & algt->mask)
- return ERR_PTR(-EINVAL);
+ if ((algt->type ^ CRYPTO_ALG_TYPE_SKCIPHER) & algt->mask)
+ return -EINVAL;
cipher_name = crypto_attr_alg_name(tb[1]);
if (IS_ERR(cipher_name))
- return ERR_CAST(cipher_name);
+ return PTR_ERR(cipher_name);
inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL);
if (!inst)
- return ERR_PTR(-ENOMEM);
+ return -ENOMEM;
- spawn = crypto_instance_ctx(inst);
+ spawn = skcipher_instance_ctx(inst);
- crypto_set_skcipher_spawn(spawn, inst);
- err = crypto_grab_skcipher(spawn, cipher_name, 0,
- crypto_requires_sync(algt->type,
- algt->mask));
+ crypto_set_skcipher_spawn(spawn, skcipher_crypto_instance(inst));
+ err = crypto_grab_skcipher2(spawn, cipher_name, 0,
+ crypto_requires_sync(algt->type,
+ algt->mask));
if (err)
goto err_free_inst;
- alg = crypto_skcipher_spawn_alg(spawn);
+ alg = crypto_spawn_skcipher_alg(spawn);
/* We only support 16-byte blocks. */
err = -EINVAL;
- if (alg->cra_ablkcipher.ivsize != CTR_RFC3686_BLOCK_SIZE)
+ if (crypto_skcipher_alg_ivsize(alg) != CTR_RFC3686_BLOCK_SIZE)
goto err_drop_spawn;
/* Not a stream cipher? */
- if (alg->cra_blocksize != 1)
+ if (alg->base.cra_blocksize != 1)
goto err_drop_spawn;
err = -ENAMETOOLONG;
- if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME, "rfc3686(%s)",
- alg->cra_name) >= CRYPTO_MAX_ALG_NAME)
+ if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,
+ "rfc3686(%s)", alg->base.cra_name) >= CRYPTO_MAX_ALG_NAME)
goto err_drop_spawn;
- if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
- "rfc3686(%s)", alg->cra_driver_name) >=
- CRYPTO_MAX_ALG_NAME)
+ if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
+ "rfc3686(%s)", alg->base.cra_driver_name) >=
+ CRYPTO_MAX_ALG_NAME)
goto err_drop_spawn;
- inst->alg.cra_priority = alg->cra_priority;
- inst->alg.cra_blocksize = 1;
- inst->alg.cra_alignmask = alg->cra_alignmask;
+ inst->alg.base.cra_priority = alg->base.cra_priority;
+ inst->alg.base.cra_blocksize = 1;
+ inst->alg.base.cra_alignmask = alg->base.cra_alignmask;
- inst->alg.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
- (alg->cra_flags & CRYPTO_ALG_ASYNC);
- inst->alg.cra_type = &crypto_ablkcipher_type;
+ inst->alg.base.cra_flags = alg->base.cra_flags & CRYPTO_ALG_ASYNC;
- inst->alg.cra_ablkcipher.ivsize = CTR_RFC3686_IV_SIZE;
- inst->alg.cra_ablkcipher.min_keysize =
- alg->cra_ablkcipher.min_keysize + CTR_RFC3686_NONCE_SIZE;
- inst->alg.cra_ablkcipher.max_keysize =
- alg->cra_ablkcipher.max_keysize + CTR_RFC3686_NONCE_SIZE;
+ inst->alg.ivsize = CTR_RFC3686_IV_SIZE;
+ inst->alg.chunksize = crypto_skcipher_alg_chunksize(alg);
+ inst->alg.min_keysize = crypto_skcipher_alg_min_keysize(alg) +
+ CTR_RFC3686_NONCE_SIZE;
+ inst->alg.max_keysize = crypto_skcipher_alg_max_keysize(alg) +
+ CTR_RFC3686_NONCE_SIZE;
- inst->alg.cra_ablkcipher.geniv = "seqiv";
+ inst->alg.setkey = crypto_rfc3686_setkey;
+ inst->alg.encrypt = crypto_rfc3686_crypt;
+ inst->alg.decrypt = crypto_rfc3686_crypt;
- inst->alg.cra_ablkcipher.setkey = crypto_rfc3686_setkey;
- inst->alg.cra_ablkcipher.encrypt = crypto_rfc3686_crypt;
- inst->alg.cra_ablkcipher.decrypt = crypto_rfc3686_crypt;
+ inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc3686_ctx);
- inst->alg.cra_ctxsize = sizeof(struct crypto_rfc3686_ctx);
+ inst->alg.init = crypto_rfc3686_init_tfm;
+ inst->alg.exit = crypto_rfc3686_exit_tfm;
- inst->alg.cra_init = crypto_rfc3686_init_tfm;
- inst->alg.cra_exit = crypto_rfc3686_exit_tfm;
+ inst->free = crypto_rfc3686_free;
- return inst;
+ err = skcipher_register_instance(tmpl, inst);
+ if (err)
+ goto err_drop_spawn;
+
+out:
+ return err;
err_drop_spawn:
crypto_drop_skcipher(spawn);
err_free_inst:
kfree(inst);
- return ERR_PTR(err);
-}
-
-static void crypto_rfc3686_free(struct crypto_instance *inst)
-{
- struct crypto_skcipher_spawn *spawn = crypto_instance_ctx(inst);
-
- crypto_drop_skcipher(spawn);
- kfree(inst);
+ goto out;
}
static struct crypto_template crypto_rfc3686_tmpl = {
.name = "rfc3686",
- .alloc = crypto_rfc3686_alloc,
- .free = crypto_rfc3686_free,
+ .create = crypto_rfc3686_create,
.module = THIS_MODULE,
};
^ permalink raw reply related
* [PATCH 7/30] crypto: ccm - Use skcipher
From: Herbert Xu @ 2016-07-12 5:17 UTC (permalink / raw)
To: Linux Crypto Mailing List
In-Reply-To: <20160712051554.GA28324@gondor.apana.org.au>
This patch converts ccm to use the new skcipher interface as opposed
to ablkcipher.
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
---
crypto/ccm.c | 72 ++++++++++++++++++++++++++++++-----------------------------
1 file changed, 37 insertions(+), 35 deletions(-)
diff --git a/crypto/ccm.c b/crypto/ccm.c
index cc31ea4..006d857 100644
--- a/crypto/ccm.c
+++ b/crypto/ccm.c
@@ -28,7 +28,7 @@ struct ccm_instance_ctx {
struct crypto_ccm_ctx {
struct crypto_cipher *cipher;
- struct crypto_ablkcipher *ctr;
+ struct crypto_skcipher *ctr;
};
struct crypto_rfc4309_ctx {
@@ -50,7 +50,7 @@ struct crypto_ccm_req_priv_ctx {
u32 flags;
struct scatterlist src[3];
struct scatterlist dst[3];
- struct ablkcipher_request abreq;
+ struct skcipher_request skreq;
};
static inline struct crypto_ccm_req_priv_ctx *crypto_ccm_reqctx(
@@ -83,15 +83,15 @@ static int crypto_ccm_setkey(struct crypto_aead *aead, const u8 *key,
unsigned int keylen)
{
struct crypto_ccm_ctx *ctx = crypto_aead_ctx(aead);
- struct crypto_ablkcipher *ctr = ctx->ctr;
+ struct crypto_skcipher *ctr = ctx->ctr;
struct crypto_cipher *tfm = ctx->cipher;
int err = 0;
- crypto_ablkcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK);
- crypto_ablkcipher_set_flags(ctr, crypto_aead_get_flags(aead) &
- CRYPTO_TFM_REQ_MASK);
- err = crypto_ablkcipher_setkey(ctr, key, keylen);
- crypto_aead_set_flags(aead, crypto_ablkcipher_get_flags(ctr) &
+ crypto_skcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK);
+ crypto_skcipher_set_flags(ctr, crypto_aead_get_flags(aead) &
+ CRYPTO_TFM_REQ_MASK);
+ err = crypto_skcipher_setkey(ctr, key, keylen);
+ crypto_aead_set_flags(aead, crypto_skcipher_get_flags(ctr) &
CRYPTO_TFM_RES_MASK);
if (err)
goto out;
@@ -347,7 +347,7 @@ static int crypto_ccm_encrypt(struct aead_request *req)
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct crypto_ccm_ctx *ctx = crypto_aead_ctx(aead);
struct crypto_ccm_req_priv_ctx *pctx = crypto_ccm_reqctx(req);
- struct ablkcipher_request *abreq = &pctx->abreq;
+ struct skcipher_request *skreq = &pctx->skreq;
struct scatterlist *dst;
unsigned int cryptlen = req->cryptlen;
u8 *odata = pctx->odata;
@@ -366,11 +366,11 @@ static int crypto_ccm_encrypt(struct aead_request *req)
if (req->src != req->dst)
dst = pctx->dst;
- ablkcipher_request_set_tfm(abreq, ctx->ctr);
- ablkcipher_request_set_callback(abreq, pctx->flags,
- crypto_ccm_encrypt_done, req);
- ablkcipher_request_set_crypt(abreq, pctx->src, dst, cryptlen + 16, iv);
- err = crypto_ablkcipher_encrypt(abreq);
+ skcipher_request_set_tfm(skreq, ctx->ctr);
+ skcipher_request_set_callback(skreq, pctx->flags,
+ crypto_ccm_encrypt_done, req);
+ skcipher_request_set_crypt(skreq, pctx->src, dst, cryptlen + 16, iv);
+ err = crypto_skcipher_encrypt(skreq);
if (err)
return err;
@@ -407,7 +407,7 @@ static int crypto_ccm_decrypt(struct aead_request *req)
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct crypto_ccm_ctx *ctx = crypto_aead_ctx(aead);
struct crypto_ccm_req_priv_ctx *pctx = crypto_ccm_reqctx(req);
- struct ablkcipher_request *abreq = &pctx->abreq;
+ struct skcipher_request *skreq = &pctx->skreq;
struct scatterlist *dst;
unsigned int authsize = crypto_aead_authsize(aead);
unsigned int cryptlen = req->cryptlen;
@@ -429,11 +429,11 @@ static int crypto_ccm_decrypt(struct aead_request *req)
if (req->src != req->dst)
dst = pctx->dst;
- ablkcipher_request_set_tfm(abreq, ctx->ctr);
- ablkcipher_request_set_callback(abreq, pctx->flags,
- crypto_ccm_decrypt_done, req);
- ablkcipher_request_set_crypt(abreq, pctx->src, dst, cryptlen + 16, iv);
- err = crypto_ablkcipher_decrypt(abreq);
+ skcipher_request_set_tfm(skreq, ctx->ctr);
+ skcipher_request_set_callback(skreq, pctx->flags,
+ crypto_ccm_decrypt_done, req);
+ skcipher_request_set_crypt(skreq, pctx->src, dst, cryptlen + 16, iv);
+ err = crypto_skcipher_decrypt(skreq);
if (err)
return err;
@@ -454,7 +454,7 @@ static int crypto_ccm_init_tfm(struct crypto_aead *tfm)
struct ccm_instance_ctx *ictx = aead_instance_ctx(inst);
struct crypto_ccm_ctx *ctx = crypto_aead_ctx(tfm);
struct crypto_cipher *cipher;
- struct crypto_ablkcipher *ctr;
+ struct crypto_skcipher *ctr;
unsigned long align;
int err;
@@ -462,7 +462,7 @@ static int crypto_ccm_init_tfm(struct crypto_aead *tfm)
if (IS_ERR(cipher))
return PTR_ERR(cipher);
- ctr = crypto_spawn_skcipher(&ictx->ctr);
+ ctr = crypto_spawn_skcipher2(&ictx->ctr);
err = PTR_ERR(ctr);
if (IS_ERR(ctr))
goto err_free_cipher;
@@ -475,7 +475,7 @@ static int crypto_ccm_init_tfm(struct crypto_aead *tfm)
crypto_aead_set_reqsize(
tfm,
align + sizeof(struct crypto_ccm_req_priv_ctx) +
- crypto_ablkcipher_reqsize(ctr));
+ crypto_skcipher_reqsize(ctr));
return 0;
@@ -489,7 +489,7 @@ static void crypto_ccm_exit_tfm(struct crypto_aead *tfm)
struct crypto_ccm_ctx *ctx = crypto_aead_ctx(tfm);
crypto_free_cipher(ctx->cipher);
- crypto_free_ablkcipher(ctx->ctr);
+ crypto_free_skcipher(ctx->ctr);
}
static void crypto_ccm_free(struct aead_instance *inst)
@@ -509,7 +509,7 @@ static int crypto_ccm_create_common(struct crypto_template *tmpl,
{
struct crypto_attr_type *algt;
struct aead_instance *inst;
- struct crypto_alg *ctr;
+ struct skcipher_alg *ctr;
struct crypto_alg *cipher;
struct ccm_instance_ctx *ictx;
int err;
@@ -544,39 +544,40 @@ static int crypto_ccm_create_common(struct crypto_template *tmpl,
goto err_free_inst;
crypto_set_skcipher_spawn(&ictx->ctr, aead_crypto_instance(inst));
- err = crypto_grab_skcipher(&ictx->ctr, ctr_name, 0,
- crypto_requires_sync(algt->type,
- algt->mask));
+ err = crypto_grab_skcipher2(&ictx->ctr, ctr_name, 0,
+ crypto_requires_sync(algt->type,
+ algt->mask));
if (err)
goto err_drop_cipher;
- ctr = crypto_skcipher_spawn_alg(&ictx->ctr);
+ ctr = crypto_spawn_skcipher_alg(&ictx->ctr);
/* Not a stream cipher? */
err = -EINVAL;
- if (ctr->cra_blocksize != 1)
+ if (ctr->base.cra_blocksize != 1)
goto err_drop_ctr;
/* We want the real thing! */
- if (ctr->cra_ablkcipher.ivsize != 16)
+ if (crypto_skcipher_alg_ivsize(ctr) != 16)
goto err_drop_ctr;
err = -ENAMETOOLONG;
if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
- "ccm_base(%s,%s)", ctr->cra_driver_name,
+ "ccm_base(%s,%s)", ctr->base.cra_driver_name,
cipher->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
goto err_drop_ctr;
memcpy(inst->alg.base.cra_name, full_name, CRYPTO_MAX_ALG_NAME);
- inst->alg.base.cra_flags = ctr->cra_flags & CRYPTO_ALG_ASYNC;
+ inst->alg.base.cra_flags = ctr->base.cra_flags & CRYPTO_ALG_ASYNC;
inst->alg.base.cra_priority = (cipher->cra_priority +
- ctr->cra_priority) / 2;
+ ctr->base.cra_priority) / 2;
inst->alg.base.cra_blocksize = 1;
inst->alg.base.cra_alignmask = cipher->cra_alignmask |
- ctr->cra_alignmask |
+ ctr->base.cra_alignmask |
(__alignof__(u32) - 1);
inst->alg.ivsize = 16;
+ inst->alg.chunksize = crypto_skcipher_alg_chunksize(ctr);
inst->alg.maxauthsize = 16;
inst->alg.base.cra_ctxsize = sizeof(struct crypto_ccm_ctx);
inst->alg.init = crypto_ccm_init_tfm;
@@ -863,6 +864,7 @@ static int crypto_rfc4309_create(struct crypto_template *tmpl,
inst->alg.base.cra_alignmask = alg->base.cra_alignmask;
inst->alg.ivsize = 8;
+ inst->alg.chunksize = crypto_aead_alg_chunksize(alg);
inst->alg.maxauthsize = 16;
inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4309_ctx);
^ permalink raw reply related
* [PATCH 9/30] crypto: chacha20poly1305 - Use skcipher
From: Herbert Xu @ 2016-07-12 5:17 UTC (permalink / raw)
To: Linux Crypto Mailing List
In-Reply-To: <20160712051554.GA28324@gondor.apana.org.au>
This patch converts chacha20poly1305 to use the new skcipher
interface as opposed to ablkcipher.
It also fixes a buglet where we may end up with an async poly1305
when the user asks for a async algorithm. This shouldn't be a
problem yet as there aren't any async implementations of poly1305
out there.
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
---
crypto/chacha20poly1305.c | 89 +++++++++++++++++++++++-----------------------
1 file changed, 46 insertions(+), 43 deletions(-)
diff --git a/crypto/chacha20poly1305.c b/crypto/chacha20poly1305.c
index 7b6b935..e899ef5 100644
--- a/crypto/chacha20poly1305.c
+++ b/crypto/chacha20poly1305.c
@@ -31,7 +31,7 @@ struct chachapoly_instance_ctx {
};
struct chachapoly_ctx {
- struct crypto_ablkcipher *chacha;
+ struct crypto_skcipher *chacha;
struct crypto_ahash *poly;
/* key bytes we use for the ChaCha20 IV */
unsigned int saltlen;
@@ -53,7 +53,7 @@ struct poly_req {
struct chacha_req {
u8 iv[CHACHA20_IV_SIZE];
struct scatterlist src[1];
- struct ablkcipher_request req; /* must be last member */
+ struct skcipher_request req; /* must be last member */
};
struct chachapoly_req_ctx {
@@ -144,12 +144,12 @@ static int chacha_decrypt(struct aead_request *req)
dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen);
}
- ablkcipher_request_set_callback(&creq->req, aead_request_flags(req),
- chacha_decrypt_done, req);
- ablkcipher_request_set_tfm(&creq->req, ctx->chacha);
- ablkcipher_request_set_crypt(&creq->req, src, dst,
- rctx->cryptlen, creq->iv);
- err = crypto_ablkcipher_decrypt(&creq->req);
+ skcipher_request_set_callback(&creq->req, aead_request_flags(req),
+ chacha_decrypt_done, req);
+ skcipher_request_set_tfm(&creq->req, ctx->chacha);
+ skcipher_request_set_crypt(&creq->req, src, dst,
+ rctx->cryptlen, creq->iv);
+ err = crypto_skcipher_decrypt(&creq->req);
if (err)
return err;
@@ -393,13 +393,13 @@ static int poly_genkey(struct aead_request *req)
chacha_iv(creq->iv, req, 0);
- ablkcipher_request_set_callback(&creq->req, aead_request_flags(req),
- poly_genkey_done, req);
- ablkcipher_request_set_tfm(&creq->req, ctx->chacha);
- ablkcipher_request_set_crypt(&creq->req, creq->src, creq->src,
- POLY1305_KEY_SIZE, creq->iv);
+ skcipher_request_set_callback(&creq->req, aead_request_flags(req),
+ poly_genkey_done, req);
+ skcipher_request_set_tfm(&creq->req, ctx->chacha);
+ skcipher_request_set_crypt(&creq->req, creq->src, creq->src,
+ POLY1305_KEY_SIZE, creq->iv);
- err = crypto_ablkcipher_decrypt(&creq->req);
+ err = crypto_skcipher_decrypt(&creq->req);
if (err)
return err;
@@ -433,12 +433,12 @@ static int chacha_encrypt(struct aead_request *req)
dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen);
}
- ablkcipher_request_set_callback(&creq->req, aead_request_flags(req),
- chacha_encrypt_done, req);
- ablkcipher_request_set_tfm(&creq->req, ctx->chacha);
- ablkcipher_request_set_crypt(&creq->req, src, dst,
- req->cryptlen, creq->iv);
- err = crypto_ablkcipher_encrypt(&creq->req);
+ skcipher_request_set_callback(&creq->req, aead_request_flags(req),
+ chacha_encrypt_done, req);
+ skcipher_request_set_tfm(&creq->req, ctx->chacha);
+ skcipher_request_set_crypt(&creq->req, src, dst,
+ req->cryptlen, creq->iv);
+ err = crypto_skcipher_encrypt(&creq->req);
if (err)
return err;
@@ -500,13 +500,13 @@ static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key,
keylen -= ctx->saltlen;
memcpy(ctx->salt, key + keylen, ctx->saltlen);
- crypto_ablkcipher_clear_flags(ctx->chacha, CRYPTO_TFM_REQ_MASK);
- crypto_ablkcipher_set_flags(ctx->chacha, crypto_aead_get_flags(aead) &
- CRYPTO_TFM_REQ_MASK);
+ crypto_skcipher_clear_flags(ctx->chacha, CRYPTO_TFM_REQ_MASK);
+ crypto_skcipher_set_flags(ctx->chacha, crypto_aead_get_flags(aead) &
+ CRYPTO_TFM_REQ_MASK);
- err = crypto_ablkcipher_setkey(ctx->chacha, key, keylen);
- crypto_aead_set_flags(aead, crypto_ablkcipher_get_flags(ctx->chacha) &
- CRYPTO_TFM_RES_MASK);
+ err = crypto_skcipher_setkey(ctx->chacha, key, keylen);
+ crypto_aead_set_flags(aead, crypto_skcipher_get_flags(ctx->chacha) &
+ CRYPTO_TFM_RES_MASK);
return err;
}
@@ -524,7 +524,7 @@ static int chachapoly_init(struct crypto_aead *tfm)
struct aead_instance *inst = aead_alg_instance(tfm);
struct chachapoly_instance_ctx *ictx = aead_instance_ctx(inst);
struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm);
- struct crypto_ablkcipher *chacha;
+ struct crypto_skcipher *chacha;
struct crypto_ahash *poly;
unsigned long align;
@@ -532,7 +532,7 @@ static int chachapoly_init(struct crypto_aead *tfm)
if (IS_ERR(poly))
return PTR_ERR(poly);
- chacha = crypto_spawn_skcipher(&ictx->chacha);
+ chacha = crypto_spawn_skcipher2(&ictx->chacha);
if (IS_ERR(chacha)) {
crypto_free_ahash(poly);
return PTR_ERR(chacha);
@@ -548,8 +548,8 @@ static int chachapoly_init(struct crypto_aead *tfm)
tfm,
align + offsetof(struct chachapoly_req_ctx, u) +
max(offsetof(struct chacha_req, req) +
- sizeof(struct ablkcipher_request) +
- crypto_ablkcipher_reqsize(chacha),
+ sizeof(struct skcipher_request) +
+ crypto_skcipher_reqsize(chacha),
offsetof(struct poly_req, req) +
sizeof(struct ahash_request) +
crypto_ahash_reqsize(poly)));
@@ -562,7 +562,7 @@ static void chachapoly_exit(struct crypto_aead *tfm)
struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm);
crypto_free_ahash(ctx->poly);
- crypto_free_ablkcipher(ctx->chacha);
+ crypto_free_skcipher(ctx->chacha);
}
static void chachapoly_free(struct aead_instance *inst)
@@ -579,7 +579,7 @@ static int chachapoly_create(struct crypto_template *tmpl, struct rtattr **tb,
{
struct crypto_attr_type *algt;
struct aead_instance *inst;
- struct crypto_alg *chacha;
+ struct skcipher_alg *chacha;
struct crypto_alg *poly;
struct hash_alg_common *poly_hash;
struct chachapoly_instance_ctx *ctx;
@@ -605,7 +605,9 @@ static int chachapoly_create(struct crypto_template *tmpl, struct rtattr **tb,
poly = crypto_find_alg(poly_name, &crypto_ahash_type,
CRYPTO_ALG_TYPE_HASH,
- CRYPTO_ALG_TYPE_AHASH_MASK);
+ CRYPTO_ALG_TYPE_AHASH_MASK |
+ crypto_requires_sync(algt->type,
+ algt->mask));
if (IS_ERR(poly))
return PTR_ERR(poly);
@@ -623,20 +625,20 @@ static int chachapoly_create(struct crypto_template *tmpl, struct rtattr **tb,
goto err_free_inst;
crypto_set_skcipher_spawn(&ctx->chacha, aead_crypto_instance(inst));
- err = crypto_grab_skcipher(&ctx->chacha, chacha_name, 0,
- crypto_requires_sync(algt->type,
- algt->mask));
+ err = crypto_grab_skcipher2(&ctx->chacha, chacha_name, 0,
+ crypto_requires_sync(algt->type,
+ algt->mask));
if (err)
goto err_drop_poly;
- chacha = crypto_skcipher_spawn_alg(&ctx->chacha);
+ chacha = crypto_spawn_skcipher_alg(&ctx->chacha);
err = -EINVAL;
/* Need 16-byte IV size, including Initial Block Counter value */
- if (chacha->cra_ablkcipher.ivsize != CHACHA20_IV_SIZE)
+ if (crypto_skcipher_alg_ivsize(chacha) != CHACHA20_IV_SIZE)
goto out_drop_chacha;
/* Not a stream cipher? */
- if (chacha->cra_blocksize != 1)
+ if (chacha->base.cra_blocksize != 1)
goto out_drop_chacha;
err = -ENAMETOOLONG;
@@ -645,20 +647,21 @@ static int chachapoly_create(struct crypto_template *tmpl, struct rtattr **tb,
poly_name) >= CRYPTO_MAX_ALG_NAME)
goto out_drop_chacha;
if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
- "%s(%s,%s)", name, chacha->cra_driver_name,
+ "%s(%s,%s)", name, chacha->base.cra_driver_name,
poly->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
goto out_drop_chacha;
- inst->alg.base.cra_flags = (chacha->cra_flags | poly->cra_flags) &
+ inst->alg.base.cra_flags = (chacha->base.cra_flags | poly->cra_flags) &
CRYPTO_ALG_ASYNC;
- inst->alg.base.cra_priority = (chacha->cra_priority +
+ inst->alg.base.cra_priority = (chacha->base.cra_priority +
poly->cra_priority) / 2;
inst->alg.base.cra_blocksize = 1;
- inst->alg.base.cra_alignmask = chacha->cra_alignmask |
+ inst->alg.base.cra_alignmask = chacha->base.cra_alignmask |
poly->cra_alignmask;
inst->alg.base.cra_ctxsize = sizeof(struct chachapoly_ctx) +
ctx->saltlen;
inst->alg.ivsize = ivsize;
+ inst->alg.chunksize = crypto_skcipher_alg_chunksize(chacha);
inst->alg.maxauthsize = POLY1305_DIGEST_SIZE;
inst->alg.init = chachapoly_init;
inst->alg.exit = chachapoly_exit;
^ permalink raw reply related
* [PATCH 11/30] crypto: aead - Add skcipher null for IV generators
From: Herbert Xu @ 2016-07-12 5:17 UTC (permalink / raw)
To: Linux Crypto Mailing List
In-Reply-To: <20160712051554.GA28324@gondor.apana.org.au>
This patch adds an skcipher null object alongside the existing
null blkcipher so that IV generators using it can switch over
to skcipher.
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
---
crypto/aead.c | 10 +++++++++-
include/crypto/internal/geniv.h | 1 +
2 files changed, 10 insertions(+), 1 deletion(-)
diff --git a/crypto/aead.c b/crypto/aead.c
index b155cbc..a5d9a83 100644
--- a/crypto/aead.c
+++ b/crypto/aead.c
@@ -294,10 +294,15 @@ int aead_init_geniv(struct crypto_aead *aead)
if (err)
goto out;
+ ctx->sknull = crypto_get_default_null_skcipher2();
+ err = PTR_ERR(ctx->sknull);
+ if (IS_ERR(ctx->sknull))
+ goto out;
+
ctx->null = crypto_get_default_null_skcipher();
err = PTR_ERR(ctx->null);
if (IS_ERR(ctx->null))
- goto out;
+ goto drop_sknull;
child = crypto_spawn_aead(aead_instance_ctx(inst));
err = PTR_ERR(child);
@@ -315,6 +320,8 @@ out:
drop_null:
crypto_put_default_null_skcipher();
+drop_sknull:
+ crypto_put_default_null_skcipher2();
goto out;
}
EXPORT_SYMBOL_GPL(aead_init_geniv);
@@ -325,6 +332,7 @@ void aead_exit_geniv(struct crypto_aead *tfm)
crypto_free_aead(ctx->child);
crypto_put_default_null_skcipher();
+ crypto_put_default_null_skcipher2();
}
EXPORT_SYMBOL_GPL(aead_exit_geniv);
diff --git a/include/crypto/internal/geniv.h b/include/crypto/internal/geniv.h
index 5933363..e8447c9 100644
--- a/include/crypto/internal/geniv.h
+++ b/include/crypto/internal/geniv.h
@@ -21,6 +21,7 @@ struct aead_geniv_ctx {
spinlock_t lock;
struct crypto_aead *child;
struct crypto_blkcipher *null;
+ struct crypto_skcipher *sknull;
u8 salt[] __attribute__ ((aligned(__alignof__(u32))));
};
^ permalink raw reply related
* [PATCH 12/30] crypto: echainiv - Use skcipher
From: Herbert Xu @ 2016-07-12 5:17 UTC (permalink / raw)
To: Linux Crypto Mailing List
In-Reply-To: <20160712051554.GA28324@gondor.apana.org.au>
This patch replaces use of the obsolete blkcipher with skcipher.
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
---
crypto/echainiv.c | 16 ++++++++++------
1 file changed, 10 insertions(+), 6 deletions(-)
diff --git a/crypto/echainiv.c b/crypto/echainiv.c
index b96a8456..1b01fe9 100644
--- a/crypto/echainiv.c
+++ b/crypto/echainiv.c
@@ -20,6 +20,7 @@
#include <crypto/internal/geniv.h>
#include <crypto/scatterwalk.h>
+#include <crypto/skcipher.h>
#include <linux/err.h>
#include <linux/init.h>
#include <linux/kernel.h>
@@ -112,13 +113,16 @@ static int echainiv_encrypt(struct aead_request *req)
info = req->iv;
if (req->src != req->dst) {
- struct blkcipher_desc desc = {
- .tfm = ctx->null,
- };
+ SKCIPHER_REQUEST_ON_STACK(nreq, ctx->sknull);
- err = crypto_blkcipher_encrypt(
- &desc, req->dst, req->src,
- req->assoclen + req->cryptlen);
+ skcipher_request_set_tfm(nreq, ctx->sknull);
+ skcipher_request_set_callback(nreq, req->base.flags,
+ NULL, NULL);
+ skcipher_request_set_crypt(nreq, req->src, req->dst,
+ req->assoclen + req->cryptlen,
+ NULL);
+
+ err = crypto_skcipher_encrypt(nreq);
if (err)
return err;
}
^ permalink raw reply related
* [PATCH 8/30] crypto: gcm - Use skcipher
From: Herbert Xu @ 2016-07-12 5:17 UTC (permalink / raw)
To: Linux Crypto Mailing List
In-Reply-To: <20160712051554.GA28324@gondor.apana.org.au>
This patch converts gcm to use the new skcipher interface as opposed
to ablkcipher.
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
---
crypto/gcm.c | 111 ++++++++++++++++++++++++++++++-----------------------------
1 file changed, 58 insertions(+), 53 deletions(-)
diff --git a/crypto/gcm.c b/crypto/gcm.c
index d9ea5f9..70a892e 100644
--- a/crypto/gcm.c
+++ b/crypto/gcm.c
@@ -29,7 +29,7 @@ struct gcm_instance_ctx {
};
struct crypto_gcm_ctx {
- struct crypto_ablkcipher *ctr;
+ struct crypto_skcipher *ctr;
struct crypto_ahash *ghash;
};
@@ -50,7 +50,7 @@ struct crypto_rfc4543_instance_ctx {
struct crypto_rfc4543_ctx {
struct crypto_aead *child;
- struct crypto_blkcipher *null;
+ struct crypto_skcipher *null;
u8 nonce[4];
};
@@ -74,7 +74,7 @@ struct crypto_gcm_req_priv_ctx {
struct crypto_gcm_ghash_ctx ghash_ctx;
union {
struct ahash_request ahreq;
- struct ablkcipher_request abreq;
+ struct skcipher_request skreq;
} u;
};
@@ -114,7 +114,7 @@ static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key,
{
struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
struct crypto_ahash *ghash = ctx->ghash;
- struct crypto_ablkcipher *ctr = ctx->ctr;
+ struct crypto_skcipher *ctr = ctx->ctr;
struct {
be128 hash;
u8 iv[8];
@@ -122,35 +122,35 @@ static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key,
struct crypto_gcm_setkey_result result;
struct scatterlist sg[1];
- struct ablkcipher_request req;
+ struct skcipher_request req;
} *data;
int err;
- crypto_ablkcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK);
- crypto_ablkcipher_set_flags(ctr, crypto_aead_get_flags(aead) &
- CRYPTO_TFM_REQ_MASK);
- err = crypto_ablkcipher_setkey(ctr, key, keylen);
- crypto_aead_set_flags(aead, crypto_ablkcipher_get_flags(ctr) &
+ crypto_skcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK);
+ crypto_skcipher_set_flags(ctr, crypto_aead_get_flags(aead) &
+ CRYPTO_TFM_REQ_MASK);
+ err = crypto_skcipher_setkey(ctr, key, keylen);
+ crypto_aead_set_flags(aead, crypto_skcipher_get_flags(ctr) &
CRYPTO_TFM_RES_MASK);
if (err)
return err;
- data = kzalloc(sizeof(*data) + crypto_ablkcipher_reqsize(ctr),
+ data = kzalloc(sizeof(*data) + crypto_skcipher_reqsize(ctr),
GFP_KERNEL);
if (!data)
return -ENOMEM;
init_completion(&data->result.completion);
sg_init_one(data->sg, &data->hash, sizeof(data->hash));
- ablkcipher_request_set_tfm(&data->req, ctr);
- ablkcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP |
- CRYPTO_TFM_REQ_MAY_BACKLOG,
- crypto_gcm_setkey_done,
- &data->result);
- ablkcipher_request_set_crypt(&data->req, data->sg, data->sg,
- sizeof(data->hash), data->iv);
-
- err = crypto_ablkcipher_encrypt(&data->req);
+ skcipher_request_set_tfm(&data->req, ctr);
+ skcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP |
+ CRYPTO_TFM_REQ_MAY_BACKLOG,
+ crypto_gcm_setkey_done,
+ &data->result);
+ skcipher_request_set_crypt(&data->req, data->sg, data->sg,
+ sizeof(data->hash), data->iv);
+
+ err = crypto_skcipher_encrypt(&data->req);
if (err == -EINPROGRESS || err == -EBUSY) {
err = wait_for_completion_interruptible(
&data->result.completion);
@@ -223,13 +223,13 @@ static void crypto_gcm_init_crypt(struct aead_request *req,
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
- struct ablkcipher_request *ablk_req = &pctx->u.abreq;
+ struct skcipher_request *skreq = &pctx->u.skreq;
struct scatterlist *dst;
dst = req->src == req->dst ? pctx->src : pctx->dst;
- ablkcipher_request_set_tfm(ablk_req, ctx->ctr);
- ablkcipher_request_set_crypt(ablk_req, pctx->src, dst,
+ skcipher_request_set_tfm(skreq, ctx->ctr);
+ skcipher_request_set_crypt(skreq, pctx->src, dst,
cryptlen + sizeof(pctx->auth_tag),
pctx->iv);
}
@@ -494,14 +494,14 @@ out:
static int crypto_gcm_encrypt(struct aead_request *req)
{
struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
- struct ablkcipher_request *abreq = &pctx->u.abreq;
+ struct skcipher_request *skreq = &pctx->u.skreq;
u32 flags = aead_request_flags(req);
crypto_gcm_init_common(req);
crypto_gcm_init_crypt(req, req->cryptlen);
- ablkcipher_request_set_callback(abreq, flags, gcm_encrypt_done, req);
+ skcipher_request_set_callback(skreq, flags, gcm_encrypt_done, req);
- return crypto_ablkcipher_encrypt(abreq) ?:
+ return crypto_skcipher_encrypt(skreq) ?:
gcm_encrypt_continue(req, flags);
}
@@ -533,12 +533,12 @@ static void gcm_decrypt_done(struct crypto_async_request *areq, int err)
static int gcm_dec_hash_continue(struct aead_request *req, u32 flags)
{
struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
- struct ablkcipher_request *abreq = &pctx->u.abreq;
+ struct skcipher_request *skreq = &pctx->u.skreq;
struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
crypto_gcm_init_crypt(req, gctx->cryptlen);
- ablkcipher_request_set_callback(abreq, flags, gcm_decrypt_done, req);
- return crypto_ablkcipher_decrypt(abreq) ?: crypto_gcm_verify(req);
+ skcipher_request_set_callback(skreq, flags, gcm_decrypt_done, req);
+ return crypto_skcipher_decrypt(skreq) ?: crypto_gcm_verify(req);
}
static int crypto_gcm_decrypt(struct aead_request *req)
@@ -566,7 +566,7 @@ static int crypto_gcm_init_tfm(struct crypto_aead *tfm)
struct aead_instance *inst = aead_alg_instance(tfm);
struct gcm_instance_ctx *ictx = aead_instance_ctx(inst);
struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm);
- struct crypto_ablkcipher *ctr;
+ struct crypto_skcipher *ctr;
struct crypto_ahash *ghash;
unsigned long align;
int err;
@@ -575,7 +575,7 @@ static int crypto_gcm_init_tfm(struct crypto_aead *tfm)
if (IS_ERR(ghash))
return PTR_ERR(ghash);
- ctr = crypto_spawn_skcipher(&ictx->ctr);
+ ctr = crypto_spawn_skcipher2(&ictx->ctr);
err = PTR_ERR(ctr);
if (IS_ERR(ctr))
goto err_free_hash;
@@ -587,8 +587,8 @@ static int crypto_gcm_init_tfm(struct crypto_aead *tfm)
align &= ~(crypto_tfm_ctx_alignment() - 1);
crypto_aead_set_reqsize(tfm,
align + offsetof(struct crypto_gcm_req_priv_ctx, u) +
- max(sizeof(struct ablkcipher_request) +
- crypto_ablkcipher_reqsize(ctr),
+ max(sizeof(struct skcipher_request) +
+ crypto_skcipher_reqsize(ctr),
sizeof(struct ahash_request) +
crypto_ahash_reqsize(ghash)));
@@ -604,7 +604,7 @@ static void crypto_gcm_exit_tfm(struct crypto_aead *tfm)
struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm);
crypto_free_ahash(ctx->ghash);
- crypto_free_ablkcipher(ctx->ctr);
+ crypto_free_skcipher(ctx->ctr);
}
static void crypto_gcm_free(struct aead_instance *inst)
@@ -624,7 +624,7 @@ static int crypto_gcm_create_common(struct crypto_template *tmpl,
{
struct crypto_attr_type *algt;
struct aead_instance *inst;
- struct crypto_alg *ctr;
+ struct skcipher_alg *ctr;
struct crypto_alg *ghash_alg;
struct hash_alg_common *ghash;
struct gcm_instance_ctx *ctx;
@@ -663,41 +663,42 @@ static int crypto_gcm_create_common(struct crypto_template *tmpl,
goto err_drop_ghash;
crypto_set_skcipher_spawn(&ctx->ctr, aead_crypto_instance(inst));
- err = crypto_grab_skcipher(&ctx->ctr, ctr_name, 0,
- crypto_requires_sync(algt->type,
- algt->mask));
+ err = crypto_grab_skcipher2(&ctx->ctr, ctr_name, 0,
+ crypto_requires_sync(algt->type,
+ algt->mask));
if (err)
goto err_drop_ghash;
- ctr = crypto_skcipher_spawn_alg(&ctx->ctr);
+ ctr = crypto_spawn_skcipher_alg(&ctx->ctr);
/* We only support 16-byte blocks. */
- if (ctr->cra_ablkcipher.ivsize != 16)
+ if (crypto_skcipher_alg_ivsize(ctr) != 16)
goto out_put_ctr;
/* Not a stream cipher? */
err = -EINVAL;
- if (ctr->cra_blocksize != 1)
+ if (ctr->base.cra_blocksize != 1)
goto out_put_ctr;
err = -ENAMETOOLONG;
if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
- "gcm_base(%s,%s)", ctr->cra_driver_name,
+ "gcm_base(%s,%s)", ctr->base.cra_driver_name,
ghash_alg->cra_driver_name) >=
CRYPTO_MAX_ALG_NAME)
goto out_put_ctr;
memcpy(inst->alg.base.cra_name, full_name, CRYPTO_MAX_ALG_NAME);
- inst->alg.base.cra_flags = (ghash->base.cra_flags | ctr->cra_flags) &
- CRYPTO_ALG_ASYNC;
+ inst->alg.base.cra_flags = (ghash->base.cra_flags |
+ ctr->base.cra_flags) & CRYPTO_ALG_ASYNC;
inst->alg.base.cra_priority = (ghash->base.cra_priority +
- ctr->cra_priority) / 2;
+ ctr->base.cra_priority) / 2;
inst->alg.base.cra_blocksize = 1;
inst->alg.base.cra_alignmask = ghash->base.cra_alignmask |
- ctr->cra_alignmask;
+ ctr->base.cra_alignmask;
inst->alg.base.cra_ctxsize = sizeof(struct crypto_gcm_ctx);
inst->alg.ivsize = 12;
+ inst->alg.chunksize = crypto_skcipher_alg_chunksize(ctr);
inst->alg.maxauthsize = 16;
inst->alg.init = crypto_gcm_init_tfm;
inst->alg.exit = crypto_gcm_exit_tfm;
@@ -982,6 +983,7 @@ static int crypto_rfc4106_create(struct crypto_template *tmpl,
inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4106_ctx);
inst->alg.ivsize = 8;
+ inst->alg.chunksize = crypto_aead_alg_chunksize(alg);
inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg);
inst->alg.init = crypto_rfc4106_init_tfm;
@@ -1086,11 +1088,13 @@ static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc)
unsigned int authsize = crypto_aead_authsize(aead);
unsigned int nbytes = req->assoclen + req->cryptlen -
(enc ? 0 : authsize);
- struct blkcipher_desc desc = {
- .tfm = ctx->null,
- };
+ SKCIPHER_REQUEST_ON_STACK(nreq, ctx->null);
+
+ skcipher_request_set_tfm(nreq, ctx->null);
+ skcipher_request_set_callback(nreq, req->base.flags, NULL, NULL);
+ skcipher_request_set_crypt(nreq, req->src, req->dst, nbytes, NULL);
- return crypto_blkcipher_encrypt(&desc, req->dst, req->src, nbytes);
+ return crypto_skcipher_encrypt(nreq);
}
static int crypto_rfc4543_encrypt(struct aead_request *req)
@@ -1110,7 +1114,7 @@ static int crypto_rfc4543_init_tfm(struct crypto_aead *tfm)
struct crypto_aead_spawn *spawn = &ictx->aead;
struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm);
struct crypto_aead *aead;
- struct crypto_blkcipher *null;
+ struct crypto_skcipher *null;
unsigned long align;
int err = 0;
@@ -1118,7 +1122,7 @@ static int crypto_rfc4543_init_tfm(struct crypto_aead *tfm)
if (IS_ERR(aead))
return PTR_ERR(aead);
- null = crypto_get_default_null_skcipher();
+ null = crypto_get_default_null_skcipher2();
err = PTR_ERR(null);
if (IS_ERR(null))
goto err_free_aead;
@@ -1146,7 +1150,7 @@ static void crypto_rfc4543_exit_tfm(struct crypto_aead *tfm)
struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm);
crypto_free_aead(ctx->child);
- crypto_put_default_null_skcipher();
+ crypto_put_default_null_skcipher2();
}
static void crypto_rfc4543_free(struct aead_instance *inst)
@@ -1221,6 +1225,7 @@ static int crypto_rfc4543_create(struct crypto_template *tmpl,
inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4543_ctx);
inst->alg.ivsize = 8;
+ inst->alg.chunksize = crypto_aead_alg_chunksize(alg);
inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg);
inst->alg.init = crypto_rfc4543_init_tfm;
^ permalink raw reply related
* [PATCH 14/30] crypto: aead - Remove blkcipher null for IV generators
From: Herbert Xu @ 2016-07-12 5:17 UTC (permalink / raw)
To: Linux Crypto Mailing List
In-Reply-To: <20160712051554.GA28324@gondor.apana.org.au>
The blkcipher null object is no longer used and can now be removed.
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
---
crypto/aead.c | 8 --------
include/crypto/internal/geniv.h | 1 -
2 files changed, 9 deletions(-)
diff --git a/crypto/aead.c b/crypto/aead.c
index a5d9a83..3f5c5ff 100644
--- a/crypto/aead.c
+++ b/crypto/aead.c
@@ -299,11 +299,6 @@ int aead_init_geniv(struct crypto_aead *aead)
if (IS_ERR(ctx->sknull))
goto out;
- ctx->null = crypto_get_default_null_skcipher();
- err = PTR_ERR(ctx->null);
- if (IS_ERR(ctx->null))
- goto drop_sknull;
-
child = crypto_spawn_aead(aead_instance_ctx(inst));
err = PTR_ERR(child);
if (IS_ERR(child))
@@ -319,8 +314,6 @@ out:
return err;
drop_null:
- crypto_put_default_null_skcipher();
-drop_sknull:
crypto_put_default_null_skcipher2();
goto out;
}
@@ -331,7 +324,6 @@ void aead_exit_geniv(struct crypto_aead *tfm)
struct aead_geniv_ctx *ctx = crypto_aead_ctx(tfm);
crypto_free_aead(ctx->child);
- crypto_put_default_null_skcipher();
crypto_put_default_null_skcipher2();
}
EXPORT_SYMBOL_GPL(aead_exit_geniv);
diff --git a/include/crypto/internal/geniv.h b/include/crypto/internal/geniv.h
index e8447c9..2bcfb93 100644
--- a/include/crypto/internal/geniv.h
+++ b/include/crypto/internal/geniv.h
@@ -20,7 +20,6 @@
struct aead_geniv_ctx {
spinlock_t lock;
struct crypto_aead *child;
- struct crypto_blkcipher *null;
struct crypto_skcipher *sknull;
u8 salt[] __attribute__ ((aligned(__alignof__(u32))));
};
^ permalink raw reply related
* [PATCH 13/30] crypto: seqiv - Use skcipher
From: Herbert Xu @ 2016-07-12 5:17 UTC (permalink / raw)
To: Linux Crypto Mailing List
In-Reply-To: <20160712051554.GA28324@gondor.apana.org.au>
This patch replaces use of the obsolete blkcipher with skcipher.
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
---
crypto/seqiv.c | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)
diff --git a/crypto/seqiv.c b/crypto/seqiv.c
index 15a749a..a859b3a 100644
--- a/crypto/seqiv.c
+++ b/crypto/seqiv.c
@@ -165,12 +165,16 @@ static int seqiv_aead_encrypt(struct aead_request *req)
info = req->iv;
if (req->src != req->dst) {
- struct blkcipher_desc desc = {
- .tfm = ctx->null,
- };
+ SKCIPHER_REQUEST_ON_STACK(nreq, ctx->sknull);
- err = crypto_blkcipher_encrypt(&desc, req->dst, req->src,
- req->assoclen + req->cryptlen);
+ skcipher_request_set_tfm(nreq, ctx->sknull);
+ skcipher_request_set_callback(nreq, req->base.flags,
+ NULL, NULL);
+ skcipher_request_set_crypt(nreq, req->src, req->dst,
+ req->assoclen + req->cryptlen,
+ NULL);
+
+ err = crypto_skcipher_encrypt(nreq);
if (err)
return err;
}
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox