* [PATCH 6/7] crypto:chelsio:Move DMA un/mapping to chcr from lld cxgb4 driver
From: Harsh Jain @ 2017-10-03 6:46 UTC (permalink / raw)
To: herbert, linux-crypto, netdev; +Cc: Harsh Jain, Ganesh Goudar
In-Reply-To: <cover.1507010612.git.harsh@chelsio.com>
Allow chcr to do DMA mapping/Unmapping instead of lld cxgb4.
It moves "Copy AAD to dst buffer" requirement from driver to
firmware.
Signed-off-by: Ganesh Goudar <ganeshgr@chelsio.com>
Signed-off-by: Harsh Jain <harsh@chelsio.com>
---
drivers/crypto/chelsio/chcr_algo.c | 1645 ++++++++++++++++++------------
drivers/crypto/chelsio/chcr_algo.h | 44 +-
drivers/crypto/chelsio/chcr_crypto.h | 114 ++-
drivers/net/ethernet/chelsio/cxgb4/sge.c | 8 +-
4 files changed, 1116 insertions(+), 695 deletions(-)
diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c
index b13991d..646dfff 100644
--- a/drivers/crypto/chelsio/chcr_algo.c
+++ b/drivers/crypto/chelsio/chcr_algo.c
@@ -70,6 +70,8 @@
#include "chcr_algo.h"
#include "chcr_crypto.h"
+#define IV AES_BLOCK_SIZE
+
static inline struct chcr_aead_ctx *AEAD_CTX(struct chcr_context *ctx)
{
return ctx->crypto_ctx->aeadctx;
@@ -102,7 +104,7 @@ static inline struct uld_ctx *ULD_CTX(struct chcr_context *ctx)
static inline int is_ofld_imm(const struct sk_buff *skb)
{
- return (skb->len <= CRYPTO_MAX_IMM_TX_PKT_LEN);
+ return (skb->len <= SGE_MAX_WR_LEN);
}
/*
@@ -117,21 +119,92 @@ static inline unsigned int sgl_len(unsigned int n)
return (3 * n) / 2 + (n & 1) + 2;
}
-static int dstsg_2k(struct scatterlist *sgl, unsigned int reqlen)
+static int sg_nents_xlen(struct scatterlist *sg, unsigned int reqlen,
+ unsigned int entlen,
+ unsigned int skip)
{
int nents = 0;
unsigned int less;
+ unsigned int skip_len = 0;
- while (sgl && reqlen) {
- less = min(reqlen, sgl->length);
- nents += DIV_ROUND_UP(less, CHCR_SG_SIZE);
- reqlen -= less;
- sgl = sg_next(sgl);
+ while (sg && skip) {
+ if (sg_dma_len(sg) <= skip) {
+ skip -= sg_dma_len(sg);
+ skip_len = 0;
+ sg = sg_next(sg);
+ } else {
+ skip_len = skip;
+ skip = 0;
+ }
}
+ while (sg && reqlen) {
+ less = min(reqlen, sg_dma_len(sg) - skip_len);
+ nents += DIV_ROUND_UP(less, entlen);
+ reqlen -= less;
+ skip_len = 0;
+ sg = sg_next(sg);
+ }
return nents;
}
+static inline void chcr_handle_ahash_resp(struct ahash_request *req,
+ unsigned char *input,
+ int err)
+{
+ struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req);
+ int digestsize, updated_digestsize;
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct uld_ctx *u_ctx = ULD_CTX(h_ctx(tfm));
+
+ if (input == NULL)
+ goto out;
+ reqctx = ahash_request_ctx(req);
+ digestsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(req));
+ if (reqctx->is_sg_map)
+ chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req);
+ if (reqctx->dma_addr)
+ dma_unmap_single(&u_ctx->lldi.pdev->dev, reqctx->dma_addr,
+ reqctx->dma_len, DMA_TO_DEVICE);
+ reqctx->dma_addr = 0;
+ updated_digestsize = digestsize;
+ if (digestsize == SHA224_DIGEST_SIZE)
+ updated_digestsize = SHA256_DIGEST_SIZE;
+ else if (digestsize == SHA384_DIGEST_SIZE)
+ updated_digestsize = SHA512_DIGEST_SIZE;
+ if (reqctx->result == 1) {
+ reqctx->result = 0;
+ memcpy(req->result, input + sizeof(struct cpl_fw6_pld),
+ digestsize);
+ } else {
+ memcpy(reqctx->partial_hash, input + sizeof(struct cpl_fw6_pld),
+ updated_digestsize);
+ }
+out:
+ req->base.complete(&req->base, err);
+
+ }
+
+static inline void chcr_handle_aead_resp(struct aead_request *req,
+ unsigned char *input,
+ int err)
+{
+ struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
+ struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+ struct uld_ctx *u_ctx = ULD_CTX(a_ctx(tfm));
+
+
+ chcr_aead_dma_unmap(&u_ctx->lldi.pdev->dev, req, reqctx->op);
+ if (reqctx->b0_dma)
+ dma_unmap_single(&u_ctx->lldi.pdev->dev, reqctx->b0_dma,
+ reqctx->b0_len, DMA_BIDIRECTIONAL);
+ if (reqctx->verify == VERIFY_SW) {
+ chcr_verify_tag(req, input, &err);
+ reqctx->verify = VERIFY_HW;
+}
+ req->base.complete(&req->base, err);
+
+}
static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err)
{
u8 temp[SHA512_DIGEST_SIZE];
@@ -166,27 +239,11 @@ int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input,
{
struct crypto_tfm *tfm = req->tfm;
struct chcr_context *ctx = crypto_tfm_ctx(tfm);
- struct uld_ctx *u_ctx = ULD_CTX(ctx);
- struct chcr_req_ctx ctx_req;
- unsigned int digestsize, updated_digestsize;
struct adapter *adap = padap(ctx->dev);
switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) {
case CRYPTO_ALG_TYPE_AEAD:
- ctx_req.req.aead_req = aead_request_cast(req);
- ctx_req.ctx.reqctx = aead_request_ctx(ctx_req.req.aead_req);
- dma_unmap_sg(&u_ctx->lldi.pdev->dev, ctx_req.ctx.reqctx->dst,
- ctx_req.ctx.reqctx->dst_nents, DMA_FROM_DEVICE);
- if (ctx_req.ctx.reqctx->skb) {
- kfree_skb(ctx_req.ctx.reqctx->skb);
- ctx_req.ctx.reqctx->skb = NULL;
- }
- if (ctx_req.ctx.reqctx->verify == VERIFY_SW) {
- chcr_verify_tag(ctx_req.req.aead_req, input,
- &err);
- ctx_req.ctx.reqctx->verify = VERIFY_HW;
- }
- ctx_req.req.aead_req->base.complete(req, err);
+ chcr_handle_aead_resp(aead_request_cast(req), input, err);
break;
case CRYPTO_ALG_TYPE_ABLKCIPHER:
@@ -195,60 +252,13 @@ int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input,
break;
case CRYPTO_ALG_TYPE_AHASH:
- ctx_req.req.ahash_req = ahash_request_cast(req);
- ctx_req.ctx.ahash_ctx =
- ahash_request_ctx(ctx_req.req.ahash_req);
- digestsize =
- crypto_ahash_digestsize(crypto_ahash_reqtfm(
- ctx_req.req.ahash_req));
- updated_digestsize = digestsize;
- if (digestsize == SHA224_DIGEST_SIZE)
- updated_digestsize = SHA256_DIGEST_SIZE;
- else if (digestsize == SHA384_DIGEST_SIZE)
- updated_digestsize = SHA512_DIGEST_SIZE;
- if (ctx_req.ctx.ahash_ctx->skb) {
- kfree_skb(ctx_req.ctx.ahash_ctx->skb);
- ctx_req.ctx.ahash_ctx->skb = NULL;
- }
- if (ctx_req.ctx.ahash_ctx->result == 1) {
- ctx_req.ctx.ahash_ctx->result = 0;
- memcpy(ctx_req.req.ahash_req->result, input +
- sizeof(struct cpl_fw6_pld),
- digestsize);
- } else {
- memcpy(ctx_req.ctx.ahash_ctx->partial_hash, input +
- sizeof(struct cpl_fw6_pld),
- updated_digestsize);
+ chcr_handle_ahash_resp(ahash_request_cast(req), input, err);
}
- ctx_req.req.ahash_req->base.complete(req, err);
- break;
- }
atomic_inc(&adap->chcr_stats.complete);
return err;
}
-/*
- * calc_tx_flits_ofld - calculate # of flits for an offload packet
- * @skb: the packet
- * Returns the number of flits needed for the given offload packet.
- * These packets are already fully constructed and no additional headers
- * will be added.
- */
-static inline unsigned int calc_tx_flits_ofld(const struct sk_buff *skb)
-{
- unsigned int flits, cnt;
-
- if (is_ofld_imm(skb))
- return DIV_ROUND_UP(skb->len, 8);
-
- flits = skb_transport_offset(skb) / 8; /* headers */
- cnt = skb_shinfo(skb)->nr_frags;
- if (skb_tail_pointer(skb) != skb_transport_header(skb))
- cnt++;
- return flits + sgl_len(cnt);
-}
-
-static inline void get_aes_decrypt_key(unsigned char *dec_key,
+static void get_aes_decrypt_key(unsigned char *dec_key,
const unsigned char *key,
unsigned int keylength)
{
@@ -395,64 +405,193 @@ static inline int is_hmac(struct crypto_tfm *tfm)
return 0;
}
-static void write_phys_cpl(struct cpl_rx_phys_dsgl *phys_cpl,
- struct scatterlist *sg,
- struct phys_sge_parm *sg_param)
+static inline void dsgl_walk_init(struct dsgl_walk *walk,
+ struct cpl_rx_phys_dsgl *dsgl)
{
- struct phys_sge_pairs *to;
- unsigned int len = 0, left_size = sg_param->obsize;
- unsigned int j = 0;
- int offset, ent_len;
+ walk->dsgl = dsgl;
+ walk->nents = 0;
+ walk->to = (struct phys_sge_pairs *)(dsgl + 1);
+}
+
+static inline void dsgl_walk_end(struct dsgl_walk *walk, unsigned short qid)
+{
+ struct cpl_rx_phys_dsgl *phys_cpl;
+
+ phys_cpl = walk->dsgl;
phys_cpl->op_to_tid = htonl(CPL_RX_PHYS_DSGL_OPCODE_V(CPL_RX_PHYS_DSGL)
| CPL_RX_PHYS_DSGL_ISRDMA_V(0));
- to = (struct phys_sge_pairs *)((unsigned char *)phys_cpl +
- sizeof(struct cpl_rx_phys_dsgl));
+ phys_cpl->pcirlxorder_to_noofsgentr =
+ htonl(CPL_RX_PHYS_DSGL_PCIRLXORDER_V(0) |
+ CPL_RX_PHYS_DSGL_PCINOSNOOP_V(0) |
+ CPL_RX_PHYS_DSGL_PCITPHNTENB_V(0) |
+ CPL_RX_PHYS_DSGL_PCITPHNT_V(0) |
+ CPL_RX_PHYS_DSGL_DCAID_V(0) |
+ CPL_RX_PHYS_DSGL_NOOFSGENTR_V(walk->nents));
+ phys_cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR;
+ phys_cpl->rss_hdr_int.qid = htons(qid);
+ phys_cpl->rss_hdr_int.hash_val = 0;
+}
+
+static inline void dsgl_walk_add_page(struct dsgl_walk *walk,
+ size_t size,
+ dma_addr_t *addr)
+{
+ int j;
+
+ if (!size)
+ return;
+ j = walk->nents;
+ walk->to->len[j % 8] = htons(size);
+ walk->to->addr[j % 8] = cpu_to_be64(*addr);
+ j++;
+ if ((j % 8) == 0)
+ walk->to++;
+ walk->nents = j;
+}
+
+static void dsgl_walk_add_sg(struct dsgl_walk *walk,
+ struct scatterlist *sg,
+ unsigned int slen,
+ unsigned int skip)
+{
+ int skip_len = 0;
+ unsigned int left_size = slen, len = 0;
+ unsigned int j = walk->nents;
+ int offset, ent_len;
+
+ if (!slen)
+ return;
+ while (sg && skip) {
+ if (sg_dma_len(sg) <= skip) {
+ skip -= sg_dma_len(sg);
+ skip_len = 0;
+ sg = sg_next(sg);
+ } else {
+ skip_len = skip;
+ skip = 0;
+ }
+ }
+
while (left_size && sg) {
- len = min_t(u32, left_size, sg_dma_len(sg));
+ len = min_t(u32, left_size, sg_dma_len(sg) - skip_len);
offset = 0;
while (len) {
- ent_len = min_t(u32, len, CHCR_SG_SIZE);
- to->len[j % 8] = htons(ent_len);
- to->addr[j % 8] = cpu_to_be64(sg_dma_address(sg) +
- offset);
+ ent_len = min_t(u32, len, CHCR_DST_SG_SIZE);
+ walk->to->len[j % 8] = htons(ent_len);
+ walk->to->addr[j % 8] = cpu_to_be64(sg_dma_address(sg) +
+ offset + skip_len);
offset += ent_len;
len -= ent_len;
j++;
if ((j % 8) == 0)
- to++;
+ walk->to++;
}
- left_size -= min(left_size, sg_dma_len(sg));
+ walk->last_sg = sg;
+ walk->last_sg_len = min_t(u32, left_size, sg_dma_len(sg) -
+ skip_len) + skip_len;
+ left_size -= min_t(u32, left_size, sg_dma_len(sg) - skip_len);
+ skip_len = 0;
sg = sg_next(sg);
}
- phys_cpl->pcirlxorder_to_noofsgentr =
- htonl(CPL_RX_PHYS_DSGL_PCIRLXORDER_V(0) |
- CPL_RX_PHYS_DSGL_PCINOSNOOP_V(0) |
- CPL_RX_PHYS_DSGL_PCITPHNTENB_V(0) |
- CPL_RX_PHYS_DSGL_PCITPHNT_V(0) |
- CPL_RX_PHYS_DSGL_DCAID_V(0) |
- CPL_RX_PHYS_DSGL_NOOFSGENTR_V(j));
- phys_cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR;
- phys_cpl->rss_hdr_int.qid = htons(sg_param->qid);
- phys_cpl->rss_hdr_int.hash_val = 0;
+ walk->nents = j;
+}
+
+static inline void ulptx_walk_init(struct ulptx_walk *walk,
+ struct ulptx_sgl *ulp)
+{
+ walk->sgl = ulp;
+ walk->nents = 0;
+ walk->pair_idx = 0;
+ walk->pair = ulp->sge;
+ walk->last_sg = NULL;
+ walk->last_sg_len = 0;
+}
+
+static inline void ulptx_walk_end(struct ulptx_walk *walk)
+{
+ walk->sgl->cmd_nsge = htonl(ULPTX_CMD_V(ULP_TX_SC_DSGL) |
+ ULPTX_NSGE_V(walk->nents));
+}
+
+static inline void ulptx_walk_add_page(struct ulptx_walk *walk,
+ size_t size,
+ dma_addr_t *addr)
+{
+ if (!size)
+ return;
+
+ if (walk->nents == 0) {
+ walk->sgl->len0 = cpu_to_be32(size);
+ walk->sgl->addr0 = cpu_to_be64(*addr);
+ } else {
+ walk->pair->addr[walk->pair_idx] = cpu_to_be64(*addr);
+ walk->pair->len[walk->pair_idx] = cpu_to_be32(size);
+ walk->pair_idx = !walk->pair_idx;
+ if (!walk->pair_idx)
+ walk->pair++;
+ }
+ walk->nents++;
}
-static inline int map_writesg_phys_cpl(struct device *dev,
- struct cpl_rx_phys_dsgl *phys_cpl,
+static void ulptx_walk_add_sg(struct ulptx_walk *walk,
struct scatterlist *sg,
- struct phys_sge_parm *sg_param)
+ unsigned int len,
+ unsigned int skip)
{
- if (!sg || !sg_param->nents)
- return -EINVAL;
+ int small;
+ int skip_len = 0;
+ unsigned int sgmin;
- sg_param->nents = dma_map_sg(dev, sg, sg_param->nents, DMA_FROM_DEVICE);
- if (sg_param->nents == 0) {
- pr_err("CHCR : DMA mapping failed\n");
- return -EINVAL;
+ if (!len)
+ return;
+
+ while (sg && skip) {
+ if (sg_dma_len(sg) <= skip) {
+ skip -= sg_dma_len(sg);
+ skip_len = 0;
+ sg = sg_next(sg);
+ } else {
+ skip_len = skip;
+ skip = 0;
+ }
+ }
+ if (walk->nents == 0) {
+ small = min_t(unsigned int, sg_dma_len(sg) - skip_len, len);
+ sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE);
+ walk->sgl->len0 = cpu_to_be32(sgmin);
+ walk->sgl->addr0 = cpu_to_be64(sg_dma_address(sg) + skip_len);
+ walk->nents++;
+ len -= sgmin;
+ walk->last_sg = sg;
+ walk->last_sg_len = sgmin + skip_len;
+ skip_len += sgmin;
+ if (sg_dma_len(sg) == skip_len) {
+ sg = sg_next(sg);
+ skip_len = 0;
+ }
+ }
+
+ while (sg && len) {
+ small = min(sg_dma_len(sg) - skip_len, len);
+ sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE);
+ walk->pair->len[walk->pair_idx] = cpu_to_be32(sgmin);
+ walk->pair->addr[walk->pair_idx] =
+ cpu_to_be64(sg_dma_address(sg) + skip_len);
+ walk->pair_idx = !walk->pair_idx;
+ walk->nents++;
+ if (!walk->pair_idx)
+ walk->pair++;
+ len -= sgmin;
+ skip_len += sgmin;
+ walk->last_sg = sg;
+ walk->last_sg_len = skip_len;
+ if (sg_dma_len(sg) == skip_len) {
+ sg = sg_next(sg);
+ skip_len = 0;
+ }
}
- write_phys_cpl(phys_cpl, sg, sg_param);
- return 0;
}
static inline int get_aead_subtype(struct crypto_aead *aead)
@@ -472,45 +611,6 @@ static inline int get_cryptoalg_subtype(struct crypto_tfm *tfm)
return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK;
}
-static inline void write_buffer_to_skb(struct sk_buff *skb,
- unsigned int *frags,
- char *bfr,
- u8 bfr_len)
-{
- skb->len += bfr_len;
- skb->data_len += bfr_len;
- skb->truesize += bfr_len;
- get_page(virt_to_page(bfr));
- skb_fill_page_desc(skb, *frags, virt_to_page(bfr),
- offset_in_page(bfr), bfr_len);
- (*frags)++;
-}
-
-
-static inline void
-write_sg_to_skb(struct sk_buff *skb, unsigned int *frags,
- struct scatterlist *sg, unsigned int count)
-{
- struct page *spage;
- unsigned int page_len;
-
- skb->len += count;
- skb->data_len += count;
- skb->truesize += count;
-
- while (count > 0) {
- if (!sg || (!(sg->length)))
- break;
- spage = sg_page(sg);
- get_page(spage);
- page_len = min(sg->length, count);
- skb_fill_page_desc(skb, *frags, spage, sg->offset, page_len);
- (*frags)++;
- count -= page_len;
- sg = sg_next(sg);
- }
-}
-
static int cxgb4_is_crypto_q_full(struct net_device *dev, unsigned int idx)
{
struct adapter *adap = netdev2adap(dev);
@@ -546,32 +646,46 @@ static int generate_copy_rrkey(struct ablk_ctx *ablkctx,
static int chcr_sg_ent_in_wr(struct scatterlist *src,
struct scatterlist *dst,
unsigned int minsg,
- unsigned int space)
+ unsigned int space,
+ unsigned int srcskip,
+ unsigned int dstskip)
{
int srclen = 0, dstlen = 0;
- int srcsg = minsg, dstsg = 0;
+ int srcsg = minsg, dstsg = minsg;
int offset = 0, less;
- while (src && dst && ((srcsg + 1) <= MAX_SKB_FRAGS) &&
+ if (sg_dma_len(src) == srcskip) {
+ src = sg_next(src);
+ srcskip = 0;
+ }
+
+ if (sg_dma_len(dst) == dstskip) {
+ dst = sg_next(dst);
+ dstskip = 0;
+ }
+
+ while (src && dst &&
space > (sgl_ent_len[srcsg + 1] + dsgl_ent_len[dstsg])) {
- srclen += src->length;
+ srclen += (sg_dma_len(src) - srcskip);
srcsg++;
offset = 0;
while (dst && ((dstsg + 1) <= MAX_DSGL_ENT) &&
space > (sgl_ent_len[srcsg] + dsgl_ent_len[dstsg + 1])) {
if (srclen <= dstlen)
break;
- less = min_t(unsigned int, dst->length - offset,
- CHCR_SG_SIZE);
+ less = min_t(unsigned int, sg_dma_len(dst) - offset -
+ dstskip, CHCR_DST_SG_SIZE);
dstlen += less;
offset += less;
- if (offset == dst->length) {
+ if (offset == sg_dma_len(dst)) {
dst = sg_next(dst);
offset = 0;
}
dstsg++;
+ dstskip = 0;
}
src = sg_next(src);
+ srcskip = 0;
}
return min(srclen, dstlen);
}
@@ -601,24 +715,22 @@ static int chcr_cipher_fallback(struct crypto_skcipher *cipher,
}
static inline void create_wreq(struct chcr_context *ctx,
struct chcr_wr *chcr_req,
- void *req, struct sk_buff *skb,
+ struct crypto_async_request *req,
+ unsigned int imm,
int hash_sz,
+ unsigned int len16,
unsigned int sc_len,
unsigned int lcb)
{
struct uld_ctx *u_ctx = ULD_CTX(ctx);
int qid = u_ctx->lldi.rxq_ids[ctx->rx_qidx];
- unsigned int immdatalen = 0;
- if (is_ofld_imm(skb))
- immdatalen = skb->data_len;
chcr_req->wreq.op_to_cctx_size = FILL_WR_OP_CCTX_SIZE;
chcr_req->wreq.pld_size_hash_size =
htonl(FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz));
chcr_req->wreq.len16_pkd =
- htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP(
- (calc_tx_flits_ofld(skb) * 8), 16)));
+ htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP(len16, 16)));
chcr_req->wreq.cookie = cpu_to_be64((uintptr_t)req);
chcr_req->wreq.rx_chid_to_rx_q_id =
FILL_WR_RX_Q_ID(ctx->dev->rx_channel_id, qid,
@@ -626,13 +738,12 @@ static inline void create_wreq(struct chcr_context *ctx,
chcr_req->ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(ctx->dev->tx_channel_id,
qid);
- chcr_req->ulptx.len = htonl((DIV_ROUND_UP((calc_tx_flits_ofld(skb) * 8),
- 16) - ((sizeof(chcr_req->wreq)) >> 4)));
+ chcr_req->ulptx.len = htonl((DIV_ROUND_UP(len16, 16) -
+ ((sizeof(chcr_req->wreq)) >> 4)));
- chcr_req->sc_imm.cmd_more = FILL_CMD_MORE(immdatalen);
+ chcr_req->sc_imm.cmd_more = FILL_CMD_MORE(!imm);
chcr_req->sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) +
- sizeof(chcr_req->key_ctx) +
- sc_len + immdatalen);
+ sizeof(chcr_req->key_ctx) + sc_len);
}
/**
@@ -645,49 +756,52 @@ static inline void create_wreq(struct chcr_context *ctx,
static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam)
{
struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(wrparam->req);
- struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
- struct uld_ctx *u_ctx = ULD_CTX(ctx);
- struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
+ struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm));
struct sk_buff *skb = NULL;
struct chcr_wr *chcr_req;
struct cpl_rx_phys_dsgl *phys_cpl;
+ struct ulptx_sgl *ulptx;
struct chcr_blkcipher_req_ctx *reqctx =
ablkcipher_request_ctx(wrparam->req);
- struct phys_sge_parm sg_param;
- unsigned int frags = 0, transhdr_len, phys_dsgl;
+ unsigned int temp = 0, transhdr_len, dst_size;
int error;
int nents;
- unsigned int ivsize = AES_BLOCK_SIZE, kctx_len;
+ unsigned int kctx_len;
gfp_t flags = wrparam->req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
GFP_KERNEL : GFP_ATOMIC;
- struct adapter *adap = padap(ctx->dev);
+ struct adapter *adap = padap(c_ctx(tfm)->dev);
- reqctx->dst_nents = sg_nents_for_len(reqctx->dst, wrparam->bytes);
- nents = dstsg_2k(reqctx->dst, wrparam->bytes);
- phys_dsgl = get_space_for_phys_dsgl(nents);
+ nents = sg_nents_xlen(reqctx->dstsg, wrparam->bytes, CHCR_DST_SG_SIZE,
+ reqctx->dst_ofst);
+ dst_size = get_space_for_phys_dsgl(nents + 1);
kctx_len = (DIV_ROUND_UP(ablkctx->enckey_len, 16) * 16);
- transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl);
- skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
+ transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
+ nents = sg_nents_xlen(reqctx->srcsg, wrparam->bytes,
+ CHCR_SRC_SG_SIZE, reqctx->src_ofst);
+ temp = reqctx->imm ? (DIV_ROUND_UP((IV + wrparam->req->nbytes), 16)
+ * 16) : (sgl_len(nents + MIN_CIPHER_SG) * 8);
+ transhdr_len += temp;
+ transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16;
+ skb = alloc_skb(SGE_MAX_WR_LEN, flags);
if (!skb) {
error = -ENOMEM;
goto err;
}
- skb_reserve(skb, sizeof(struct sge_opaque_hdr));
chcr_req = __skb_put_zero(skb, transhdr_len);
chcr_req->sec_cpl.op_ivinsrtofst =
- FILL_SEC_CPL_OP_IVINSR(ctx->dev->rx_channel_id, 2, 1);
+ FILL_SEC_CPL_OP_IVINSR(c_ctx(tfm)->dev->rx_channel_id, 2, 1);
- chcr_req->sec_cpl.pldlen = htonl(ivsize + wrparam->bytes);
+ chcr_req->sec_cpl.pldlen = htonl(IV + wrparam->bytes);
chcr_req->sec_cpl.aadstart_cipherstop_hi =
- FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, ivsize + 1, 0);
+ FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, IV + 1, 0);
chcr_req->sec_cpl.cipherstop_lo_authinsert =
FILL_SEC_CPL_AUTHINSERT(0, 0, 0, 0);
chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, 0,
ablkctx->ciph_mode,
- 0, 0, ivsize >> 1);
+ 0, 0, IV >> 1);
chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 0,
- 0, 1, phys_dsgl);
+ 0, 0, dst_size);
chcr_req->key_ctx.ctx_hdr = ablkctx->key_ctx_hdr;
if ((reqctx->op == CHCR_DECRYPT_OP) &&
@@ -712,26 +826,18 @@ static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam)
}
}
phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
- sg_param.nents = reqctx->dst_nents;
- sg_param.obsize = wrparam->bytes;
- sg_param.qid = wrparam->qid;
- error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl,
- reqctx->dst, &sg_param);
- if (error)
- goto map_fail1;
+ ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size);
+ chcr_add_cipher_src_ent(wrparam->req, ulptx, wrparam);
+ chcr_add_cipher_dst_ent(wrparam->req, phys_cpl, wrparam, wrparam->qid);
- skb_set_transport_header(skb, transhdr_len);
- write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize);
- write_sg_to_skb(skb, &frags, wrparam->srcsg, wrparam->bytes);
atomic_inc(&adap->chcr_stats.cipher_rqst);
- create_wreq(ctx, chcr_req, &(wrparam->req->base), skb, 0,
- sizeof(struct cpl_rx_phys_dsgl) + phys_dsgl + kctx_len,
+ temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len
+ +(reqctx->imm ? (IV + wrparam->bytes) : 0);
+ create_wreq(c_ctx(tfm), chcr_req, &(wrparam->req->base), reqctx->imm, 0,
+ transhdr_len, temp,
ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC);
reqctx->skb = skb;
- skb_get(skb);
return skb;
-map_fail1:
- kfree_skb(skb);
err:
return ERR_PTR(error);
}
@@ -756,8 +862,7 @@ static int chcr_cipher_fallback_setkey(struct crypto_ablkcipher *cipher,
unsigned int keylen)
{
struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
- struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher);
- struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
+ struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
int err = 0;
crypto_skcipher_clear_flags(ablkctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
@@ -775,8 +880,7 @@ static int chcr_aes_cbc_setkey(struct crypto_ablkcipher *cipher,
const u8 *key,
unsigned int keylen)
{
- struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher);
- struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
+ struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
unsigned int ck_size, context_size;
u16 alignment = 0;
int err;
@@ -808,8 +912,7 @@ static int chcr_aes_ctr_setkey(struct crypto_ablkcipher *cipher,
const u8 *key,
unsigned int keylen)
{
- struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher);
- struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
+ struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
unsigned int ck_size, context_size;
u16 alignment = 0;
int err;
@@ -840,8 +943,7 @@ static int chcr_aes_rfc3686_setkey(struct crypto_ablkcipher *cipher,
const u8 *key,
unsigned int keylen)
{
- struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher);
- struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
+ struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
unsigned int ck_size, context_size;
u16 alignment = 0;
int err;
@@ -908,8 +1010,7 @@ static unsigned int adjust_ctr_overflow(u8 *iv, u32 bytes)
static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv)
{
struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
- struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
- struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
+ struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm));
struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
struct crypto_cipher *cipher;
int ret, i;
@@ -926,8 +1027,7 @@ static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv)
ret = crypto_cipher_setkey(cipher, key, keylen);
if (ret)
goto out;
-
- crypto_cipher_encrypt_one(cipher, iv, iv);
+ /*H/W sends the encrypted IV in dsgl when AADIVDROP bit is 0*/
for (i = 0; i < round8; i++)
gf128mul_x8_ble((le128 *)iv, (le128 *)iv);
@@ -1005,64 +1105,60 @@ static int chcr_handle_cipher_resp(struct ablkcipher_request *req,
unsigned char *input, int err)
{
struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
- struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
- struct uld_ctx *u_ctx = ULD_CTX(ctx);
- struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
+ struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm));
+ struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm));
struct sk_buff *skb;
struct cpl_fw6_pld *fw6_pld = (struct cpl_fw6_pld *)input;
struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
struct cipher_wr_param wrparam;
int bytes;
- dma_unmap_sg(&u_ctx->lldi.pdev->dev, reqctx->dst, reqctx->dst_nents,
- DMA_FROM_DEVICE);
-
- if (reqctx->skb) {
- kfree_skb(reqctx->skb);
- reqctx->skb = NULL;
- }
if (err)
- goto complete;
-
+ goto unmap;
if (req->nbytes == reqctx->processed) {
+ chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev,
+ req);
err = chcr_final_cipher_iv(req, fw6_pld, req->info);
goto complete;
}
if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
- ctx->tx_qidx))) {
+ c_ctx(tfm)->tx_qidx))) {
if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
err = -EBUSY;
- goto complete;
+ goto unmap;
}
}
- wrparam.srcsg = scatterwalk_ffwd(reqctx->srcffwd, req->src,
- reqctx->processed);
- reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, reqctx->dstsg,
- reqctx->processed);
- if (!wrparam.srcsg || !reqctx->dst) {
- pr_err("Input sg list length less that nbytes\n");
- err = -EINVAL;
- goto complete;
- }
- bytes = chcr_sg_ent_in_wr(wrparam.srcsg, reqctx->dst, 1,
- SPACE_LEFT(ablkctx->enckey_len));
+ if (!reqctx->imm) {
+ bytes = chcr_sg_ent_in_wr(reqctx->srcsg, reqctx->dstsg, 1,
+ SPACE_LEFT(ablkctx->enckey_len),
+ reqctx->src_ofst, reqctx->dst_ofst);
if ((bytes + reqctx->processed) >= req->nbytes)
bytes = req->nbytes - reqctx->processed;
else
bytes = ROUND_16(bytes);
+ } else {
+ /*CTR mode counter overfloa*/
+ bytes = req->nbytes - reqctx->processed;
+ }
+ dma_sync_single_for_cpu(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev,
+ reqctx->iv_dma, IV, DMA_BIDIRECTIONAL);
err = chcr_update_cipher_iv(req, fw6_pld, reqctx->iv);
+ dma_sync_single_for_device(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev,
+ reqctx->iv_dma, IV, DMA_BIDIRECTIONAL);
if (err)
- goto complete;
+ goto unmap;
if (unlikely(bytes == 0)) {
+ chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev,
+ req);
err = chcr_cipher_fallback(ablkctx->sw_cipher,
req->base.flags,
- wrparam.srcsg,
- reqctx->dst,
- req->nbytes - reqctx->processed,
- reqctx->iv,
+ req->src,
+ req->dst,
+ req->nbytes,
+ req->info,
reqctx->op);
goto complete;
}
@@ -1070,21 +1166,23 @@ static int chcr_handle_cipher_resp(struct ablkcipher_request *req,
if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
CRYPTO_ALG_SUB_TYPE_CTR)
bytes = adjust_ctr_overflow(reqctx->iv, bytes);
- reqctx->processed += bytes;
- reqctx->last_req_len = bytes;
- wrparam.qid = u_ctx->lldi.rxq_ids[ctx->rx_qidx];
+ wrparam.qid = u_ctx->lldi.rxq_ids[c_ctx(tfm)->rx_qidx];
wrparam.req = req;
wrparam.bytes = bytes;
skb = create_cipher_wr(&wrparam);
if (IS_ERR(skb)) {
pr_err("chcr : %s : Failed to form WR. No memory\n", __func__);
err = PTR_ERR(skb);
- goto complete;
+ goto unmap;
}
skb->dev = u_ctx->lldi.ports[0];
- set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
+ set_wr_txq(skb, CPL_PRIORITY_DATA, c_ctx(tfm)->tx_qidx);
chcr_send_wr(skb);
+ reqctx->last_req_len = bytes;
+ reqctx->processed += bytes;
return 0;
+unmap:
+ chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req);
complete:
req->base.complete(&req->base, err);
return err;
@@ -1098,8 +1196,7 @@ static int process_cipher(struct ablkcipher_request *req,
struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
unsigned int ivsize = crypto_ablkcipher_ivsize(tfm);
struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
- struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
- struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
+ struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm));
struct cipher_wr_param wrparam;
int bytes, err = -EINVAL;
@@ -1113,16 +1210,41 @@ static int process_cipher(struct ablkcipher_request *req,
ablkctx->enckey_len, req->nbytes, ivsize);
goto error;
}
- wrparam.srcsg = req->src;
- reqctx->dstsg = req->dst;
- bytes = chcr_sg_ent_in_wr(wrparam.srcsg, reqctx->dstsg, MIN_CIPHER_SG,
- SPACE_LEFT(ablkctx->enckey_len));
+ chcr_cipher_dma_map(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req);
+ if (req->nbytes < (SGE_MAX_WR_LEN - (sizeof(struct chcr_wr) +
+ AES_MIN_KEY_SIZE +
+ sizeof(struct cpl_rx_phys_dsgl) +
+ /*Min dsgl size*/
+ 32))) {
+ /* Can be sent as Imm*/
+ unsigned int dnents = 0, transhdr_len, phys_dsgl, kctx_len;
+
+ dnents = sg_nents_xlen(req->dst, req->nbytes,
+ CHCR_DST_SG_SIZE, 0);
+ dnents += 1; // IV
+ phys_dsgl = get_space_for_phys_dsgl(dnents);
+ kctx_len = (DIV_ROUND_UP(ablkctx->enckey_len, 16) * 16);
+ transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl);
+ reqctx->imm = (transhdr_len + IV + req->nbytes) <=
+ SGE_MAX_WR_LEN;
+ bytes = IV + req->nbytes;
+
+ } else {
+ reqctx->imm = 0;
+ }
+
+ if (!reqctx->imm) {
+ bytes = chcr_sg_ent_in_wr(req->src, req->dst,
+ MIN_CIPHER_SG,
+ SPACE_LEFT(ablkctx->enckey_len),
+ 0, 0);
if ((bytes + reqctx->processed) >= req->nbytes)
bytes = req->nbytes - reqctx->processed;
else
bytes = ROUND_16(bytes);
- if (unlikely(bytes > req->nbytes))
+ } else {
bytes = req->nbytes;
+ }
if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
CRYPTO_ALG_SUB_TYPE_CTR) {
bytes = adjust_ctr_overflow(req->info, bytes);
@@ -1139,9 +1261,11 @@ static int process_cipher(struct ablkcipher_request *req,
} else {
- memcpy(reqctx->iv, req->info, ivsize);
+ memcpy(reqctx->iv, req->info, IV);
}
if (unlikely(bytes == 0)) {
+ chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev,
+ req);
err = chcr_cipher_fallback(ablkctx->sw_cipher,
req->base.flags,
req->src,
@@ -1151,20 +1275,25 @@ static int process_cipher(struct ablkcipher_request *req,
op_type);
goto error;
}
- reqctx->processed = bytes;
- reqctx->last_req_len = bytes;
- reqctx->dst = reqctx->dstsg;
reqctx->op = op_type;
+ reqctx->srcsg = req->src;
+ reqctx->dstsg = req->dst;
+ reqctx->src_ofst = 0;
+ reqctx->dst_ofst = 0;
wrparam.qid = qid;
wrparam.req = req;
wrparam.bytes = bytes;
*skb = create_cipher_wr(&wrparam);
if (IS_ERR(*skb)) {
err = PTR_ERR(*skb);
- goto error;
+ goto unmap;
}
+ reqctx->processed = bytes;
+ reqctx->last_req_len = bytes;
return 0;
+unmap:
+ chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req);
error:
return err;
}
@@ -1172,23 +1301,22 @@ static int process_cipher(struct ablkcipher_request *req,
static int chcr_aes_encrypt(struct ablkcipher_request *req)
{
struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
- struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
struct sk_buff *skb = NULL;
int err;
- struct uld_ctx *u_ctx = ULD_CTX(ctx);
+ struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm));
if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
- ctx->tx_qidx))) {
+ c_ctx(tfm)->tx_qidx))) {
if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
return -EBUSY;
}
- err = process_cipher(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx], &skb,
- CHCR_ENCRYPT_OP);
+ err = process_cipher(req, u_ctx->lldi.rxq_ids[c_ctx(tfm)->rx_qidx],
+ &skb, CHCR_ENCRYPT_OP);
if (err || !skb)
return err;
skb->dev = u_ctx->lldi.ports[0];
- set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
+ set_wr_txq(skb, CPL_PRIORITY_DATA, c_ctx(tfm)->tx_qidx);
chcr_send_wr(skb);
return -EINPROGRESS;
}
@@ -1196,23 +1324,22 @@ static int chcr_aes_encrypt(struct ablkcipher_request *req)
static int chcr_aes_decrypt(struct ablkcipher_request *req)
{
struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
- struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
- struct uld_ctx *u_ctx = ULD_CTX(ctx);
+ struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm));
struct sk_buff *skb = NULL;
int err;
if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
- ctx->tx_qidx))) {
+ c_ctx(tfm)->tx_qidx))) {
if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
return -EBUSY;
}
- err = process_cipher(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx], &skb,
- CHCR_DECRYPT_OP);
+ err = process_cipher(req, u_ctx->lldi.rxq_ids[c_ctx(tfm)->rx_qidx],
+ &skb, CHCR_DECRYPT_OP);
if (err || !skb)
return err;
skb->dev = u_ctx->lldi.ports[0];
- set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
+ set_wr_txq(skb, CPL_PRIORITY_DATA, c_ctx(tfm)->tx_qidx);
chcr_send_wr(skb);
return -EINPROGRESS;
}
@@ -1360,17 +1487,19 @@ static struct sk_buff *create_hash_wr(struct ahash_request *req,
{
struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
- struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
- struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
+ struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(tfm));
struct sk_buff *skb = NULL;
+ struct uld_ctx *u_ctx = ULD_CTX(h_ctx(tfm));
struct chcr_wr *chcr_req;
- unsigned int frags = 0, transhdr_len, iopad_alignment = 0;
+ struct ulptx_sgl *ulptx;
+ unsigned int nents = 0, transhdr_len, iopad_alignment = 0;
unsigned int digestsize = crypto_ahash_digestsize(tfm);
- unsigned int kctx_len = 0;
+ unsigned int kctx_len = 0, temp = 0;
u8 hash_size_in_response = 0;
gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
GFP_ATOMIC;
- struct adapter *adap = padap(ctx->dev);
+ struct adapter *adap = padap(h_ctx(tfm)->dev);
+ int error = 0;
iopad_alignment = KEYCTX_ALIGN_PAD(digestsize);
kctx_len = param->alg_prm.result_size + iopad_alignment;
@@ -1382,15 +1511,22 @@ static struct sk_buff *create_hash_wr(struct ahash_request *req,
else
hash_size_in_response = param->alg_prm.result_size;
transhdr_len = HASH_TRANSHDR_SIZE(kctx_len);
- skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
+ req_ctx->imm = (transhdr_len + param->bfr_len + param->sg_len) <=
+ SGE_MAX_WR_LEN;
+ nents = sg_nents_xlen(req->src, param->sg_len, CHCR_SRC_SG_SIZE, 0);
+ nents += param->bfr_len ? 1 : 0;
+ transhdr_len += req_ctx->imm ? (DIV_ROUND_UP((param->bfr_len +
+ param->sg_len), 16) * 16) :
+ (sgl_len(nents) * 8);
+ transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16;
+
+ skb = alloc_skb(SGE_MAX_WR_LEN, flags);
if (!skb)
- return skb;
-
- skb_reserve(skb, sizeof(struct sge_opaque_hdr));
+ return ERR_PTR(-ENOMEM);
chcr_req = __skb_put_zero(skb, transhdr_len);
chcr_req->sec_cpl.op_ivinsrtofst =
- FILL_SEC_CPL_OP_IVINSR(ctx->dev->rx_channel_id, 2, 0);
+ FILL_SEC_CPL_OP_IVINSR(h_ctx(tfm)->dev->rx_channel_id, 2, 0);
chcr_req->sec_cpl.pldlen = htonl(param->bfr_len + param->sg_len);
chcr_req->sec_cpl.aadstart_cipherstop_hi =
@@ -1419,37 +1555,52 @@ static struct sk_buff *create_hash_wr(struct ahash_request *req,
((kctx_len +
sizeof(chcr_req->key_ctx)) >> 4));
chcr_req->sec_cpl.scmd1 = cpu_to_be64((u64)param->scmd1);
-
- skb_set_transport_header(skb, transhdr_len);
- if (param->bfr_len != 0)
- write_buffer_to_skb(skb, &frags, req_ctx->reqbfr,
- param->bfr_len);
- if (param->sg_len != 0)
- write_sg_to_skb(skb, &frags, req->src, param->sg_len);
+ ulptx = (struct ulptx_sgl *)((u8 *)(chcr_req + 1) + kctx_len +
+ DUMMY_BYTES);
+ if (param->bfr_len != 0) {
+ req_ctx->dma_addr = dma_map_single(&u_ctx->lldi.pdev->dev,
+ req_ctx->reqbfr, param->bfr_len,
+ DMA_TO_DEVICE);
+ if (dma_mapping_error(&u_ctx->lldi.pdev->dev,
+ req_ctx->dma_addr)) {
+ error = -ENOMEM;
+ goto err;
+ }
+ req_ctx->dma_len = param->bfr_len;
+ } else {
+ req_ctx->dma_addr = 0;
+ }
+ chcr_add_hash_src_ent(req, ulptx, param);
+ /* Request upto max wr size */
+ temp = kctx_len + DUMMY_BYTES + (req_ctx->imm ? (param->sg_len
+ + param->bfr_len) : 0);
atomic_inc(&adap->chcr_stats.digest_rqst);
- create_wreq(ctx, chcr_req, &req->base, skb, hash_size_in_response,
- DUMMY_BYTES + kctx_len, 0);
+ create_wreq(h_ctx(tfm), chcr_req, &req->base, req_ctx->imm,
+ hash_size_in_response, transhdr_len,
+ temp, 0);
req_ctx->skb = skb;
- skb_get(skb);
return skb;
+err:
+ kfree_skb(skb);
+ return ERR_PTR(error);
}
static int chcr_ahash_update(struct ahash_request *req)
{
struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
- struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
struct uld_ctx *u_ctx = NULL;
struct sk_buff *skb;
u8 remainder = 0, bs;
unsigned int nbytes = req->nbytes;
struct hash_wr_param params;
+ int error;
bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
- u_ctx = ULD_CTX(ctx);
+ u_ctx = ULD_CTX(h_ctx(rtfm));
if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
- ctx->tx_qidx))) {
+ h_ctx(rtfm)->tx_qidx))) {
if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
return -EBUSY;
}
@@ -1463,7 +1614,9 @@ static int chcr_ahash_update(struct ahash_request *req)
req_ctx->reqlen += nbytes;
return 0;
}
-
+ error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req);
+ if (error)
+ return -ENOMEM;
params.opad_needed = 0;
params.more = 1;
params.last = 0;
@@ -1474,8 +1627,10 @@ static int chcr_ahash_update(struct ahash_request *req)
req_ctx->result = 0;
req_ctx->data_len += params.sg_len + params.bfr_len;
skb = create_hash_wr(req, ¶ms);
- if (IS_ERR(skb))
- return PTR_ERR(skb);
+ if (IS_ERR(skb)) {
+ error = PTR_ERR(skb);
+ goto unmap;
+ }
if (remainder) {
u8 *temp;
@@ -1489,10 +1644,13 @@ static int chcr_ahash_update(struct ahash_request *req)
}
req_ctx->reqlen = remainder;
skb->dev = u_ctx->lldi.ports[0];
- set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
+ set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx);
chcr_send_wr(skb);
return -EINPROGRESS;
+unmap:
+ chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req);
+ return error;
}
static void create_last_hash_block(char *bfr_ptr, unsigned int bs, u64 scmd1)
@@ -1509,13 +1667,12 @@ static int chcr_ahash_final(struct ahash_request *req)
{
struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
- struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
struct hash_wr_param params;
struct sk_buff *skb;
struct uld_ctx *u_ctx = NULL;
u8 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
- u_ctx = ULD_CTX(ctx);
+ u_ctx = ULD_CTX(h_ctx(rtfm));
if (is_hmac(crypto_ahash_tfm(rtfm)))
params.opad_needed = 1;
else
@@ -1542,7 +1699,7 @@ static int chcr_ahash_final(struct ahash_request *req)
return PTR_ERR(skb);
skb->dev = u_ctx->lldi.ports[0];
- set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
+ set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx);
chcr_send_wr(skb);
return -EINPROGRESS;
}
@@ -1551,17 +1708,17 @@ static int chcr_ahash_finup(struct ahash_request *req)
{
struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
- struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
struct uld_ctx *u_ctx = NULL;
struct sk_buff *skb;
struct hash_wr_param params;
u8 bs;
+ int error;
bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
- u_ctx = ULD_CTX(ctx);
+ u_ctx = ULD_CTX(h_ctx(rtfm));
if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
- ctx->tx_qidx))) {
+ h_ctx(rtfm)->tx_qidx))) {
if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
return -EBUSY;
}
@@ -1587,34 +1744,41 @@ static int chcr_ahash_finup(struct ahash_request *req)
params.last = 1;
params.more = 0;
}
+ error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req);
+ if (error)
+ return -ENOMEM;
skb = create_hash_wr(req, ¶ms);
- if (IS_ERR(skb))
- return PTR_ERR(skb);
-
+ if (IS_ERR(skb)) {
+ error = PTR_ERR(skb);
+ goto unmap;
+ }
skb->dev = u_ctx->lldi.ports[0];
- set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
+ set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx);
chcr_send_wr(skb);
return -EINPROGRESS;
+unmap:
+ chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req);
+ return error;
}
static int chcr_ahash_digest(struct ahash_request *req)
{
struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
- struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
struct uld_ctx *u_ctx = NULL;
struct sk_buff *skb;
struct hash_wr_param params;
u8 bs;
+ int error;
rtfm->init(req);
bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
- u_ctx = ULD_CTX(ctx);
+ u_ctx = ULD_CTX(h_ctx(rtfm));
if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
- ctx->tx_qidx))) {
+ h_ctx(rtfm)->tx_qidx))) {
if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
return -EBUSY;
}
@@ -1623,6 +1787,9 @@ static int chcr_ahash_digest(struct ahash_request *req)
params.opad_needed = 1;
else
params.opad_needed = 0;
+ error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req);
+ if (error)
+ return -ENOMEM;
params.last = 0;
params.more = 0;
@@ -1640,13 +1807,17 @@ static int chcr_ahash_digest(struct ahash_request *req)
}
skb = create_hash_wr(req, ¶ms);
- if (IS_ERR(skb))
- return PTR_ERR(skb);
-
+ if (IS_ERR(skb)) {
+ error = PTR_ERR(skb);
+ goto unmap;
+ }
skb->dev = u_ctx->lldi.ports[0];
- set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
+ set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx);
chcr_send_wr(skb);
return -EINPROGRESS;
+unmap:
+ chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req);
+ return error;
}
static int chcr_ahash_export(struct ahash_request *areq, void *out)
@@ -1656,6 +1827,8 @@ static int chcr_ahash_export(struct ahash_request *areq, void *out)
state->reqlen = req_ctx->reqlen;
state->data_len = req_ctx->data_len;
+ state->is_sg_map = 0;
+ state->result = 0;
memcpy(state->bfr1, req_ctx->reqbfr, req_ctx->reqlen);
memcpy(state->partial_hash, req_ctx->partial_hash,
CHCR_HASH_MAX_DIGEST_SIZE);
@@ -1671,6 +1844,8 @@ static int chcr_ahash_import(struct ahash_request *areq, const void *in)
req_ctx->data_len = state->data_len;
req_ctx->reqbfr = req_ctx->bfr1;
req_ctx->skbfr = req_ctx->bfr2;
+ req_ctx->is_sg_map = 0;
+ req_ctx->result = 0;
memcpy(req_ctx->bfr1, state->bfr1, CHCR_HASH_MAX_BLOCK_SIZE_128);
memcpy(req_ctx->partial_hash, state->partial_hash,
CHCR_HASH_MAX_DIGEST_SIZE);
@@ -1680,8 +1855,7 @@ static int chcr_ahash_import(struct ahash_request *areq, const void *in)
static int chcr_ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
unsigned int keylen)
{
- struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
- struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
+ struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(tfm));
unsigned int digestsize = crypto_ahash_digestsize(tfm);
unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
unsigned int i, err = 0, updated_digestsize;
@@ -1734,8 +1908,7 @@ static int chcr_ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
static int chcr_aes_xts_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
unsigned int key_len)
{
- struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher);
- struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
+ struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
unsigned short context_size = 0;
int err;
@@ -1774,6 +1947,7 @@ static int chcr_sha_init(struct ahash_request *areq)
req_ctx->skbfr = req_ctx->bfr2;
req_ctx->skb = NULL;
req_ctx->result = 0;
+ req_ctx->is_sg_map = 0;
copy_hash_init_values(req_ctx->partial_hash, digestsize);
return 0;
}
@@ -1789,8 +1963,7 @@ static int chcr_hmac_init(struct ahash_request *areq)
{
struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
struct crypto_ahash *rtfm = crypto_ahash_reqtfm(areq);
- struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
- struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
+ struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(rtfm));
unsigned int digestsize = crypto_ahash_digestsize(rtfm);
unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
@@ -1836,29 +2009,48 @@ static void chcr_hmac_cra_exit(struct crypto_tfm *tfm)
}
}
-static int chcr_copy_assoc(struct aead_request *req,
- struct chcr_aead_ctx *ctx)
+static int chcr_aead_common_init(struct aead_request *req,
+ unsigned short op_type)
{
- SKCIPHER_REQUEST_ON_STACK(skreq, ctx->null);
-
- skcipher_request_set_tfm(skreq, ctx->null);
- skcipher_request_set_callback(skreq, aead_request_flags(req),
- NULL, NULL);
- skcipher_request_set_crypt(skreq, req->src, req->dst, req->assoclen,
- NULL);
+ struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+ struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
+ struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
+ int error = -EINVAL;
+ unsigned int dst_size;
+ unsigned int authsize = crypto_aead_authsize(tfm);
- return crypto_skcipher_encrypt(skreq);
+ dst_size = req->assoclen + req->cryptlen + (op_type ?
+ -authsize : authsize);
+ /* validate key size */
+ if (aeadctx->enckey_len == 0)
+ goto err;
+ if (op_type && req->cryptlen < authsize)
+ goto err;
+ error = chcr_aead_dma_map(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req,
+ op_type);
+ if (error) {
+ error = -ENOMEM;
+ goto err;
+ }
+ reqctx->aad_nents = sg_nents_xlen(req->src, req->assoclen,
+ CHCR_SRC_SG_SIZE, 0);
+ reqctx->src_nents = sg_nents_xlen(req->src, req->cryptlen,
+ CHCR_SRC_SG_SIZE, req->assoclen);
+ return 0;
+err:
+ return error;
}
-static int chcr_aead_need_fallback(struct aead_request *req, int src_nent,
+
+static int chcr_aead_need_fallback(struct aead_request *req, int dst_nents,
int aadmax, int wrlen,
unsigned short op_type)
{
unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req));
if (((req->cryptlen - (op_type ? authsize : 0)) == 0) ||
+ dst_nents > MAX_DSGL_ENT ||
(req->assoclen > aadmax) ||
- (src_nent > MAX_SKB_FRAGS) ||
- (wrlen > MAX_WR_SIZE))
+ (wrlen > SGE_MAX_WR_LEN))
return 1;
return 0;
}
@@ -1866,8 +2058,7 @@ static int chcr_aead_need_fallback(struct aead_request *req, int src_nent,
static int chcr_aead_fallback(struct aead_request *req, unsigned short op_type)
{
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
- struct chcr_context *ctx = crypto_aead_ctx(tfm);
- struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
+ struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
struct aead_request *subreq = aead_request_ctx(req);
aead_request_set_tfm(subreq, aeadctx->sw_cipher);
@@ -1886,84 +2077,75 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req,
unsigned short op_type)
{
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
- struct chcr_context *ctx = crypto_aead_ctx(tfm);
- struct uld_ctx *u_ctx = ULD_CTX(ctx);
- struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
+ struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
struct sk_buff *skb = NULL;
struct chcr_wr *chcr_req;
struct cpl_rx_phys_dsgl *phys_cpl;
- struct phys_sge_parm sg_param;
- struct scatterlist *src;
- unsigned int frags = 0, transhdr_len;
- unsigned int ivsize = crypto_aead_ivsize(tfm), dst_size = 0;
- unsigned int kctx_len = 0, nents;
- unsigned short stop_offset = 0;
+ struct ulptx_sgl *ulptx;
+ unsigned int transhdr_len;
+ unsigned int dst_size = 0, temp;
+ unsigned int kctx_len = 0, dnents;
unsigned int assoclen = req->assoclen;
unsigned int authsize = crypto_aead_authsize(tfm);
- int error = -EINVAL, src_nent;
+ int error = -EINVAL;
int null = 0;
gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
GFP_ATOMIC;
- struct adapter *adap = padap(ctx->dev);
+ struct adapter *adap = padap(a_ctx(tfm)->dev);
- dst_size = req->assoclen + req->cryptlen + (op_type ? -authsize :
- authsize);
- if (aeadctx->enckey_len == 0 || (req->cryptlen <= 0))
- goto err;
+ if (req->cryptlen == 0)
+ return NULL;
- if (op_type && req->cryptlen < crypto_aead_authsize(tfm))
- goto err;
- src_nent = sg_nents_for_len(req->src, req->assoclen + req->cryptlen);
- if (src_nent < 0)
- goto err;
- src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen);
- reqctx->dst = src;
- if (req->src != req->dst) {
- error = chcr_copy_assoc(req, aeadctx);
- if (error)
- return ERR_PTR(error);
- reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst,
- req->assoclen);
- }
+ reqctx->b0_dma = 0;
if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_NULL) {
null = 1;
assoclen = 0;
}
- reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen +
- (op_type ? -authsize : authsize));
- if (reqctx->dst_nents < 0) {
- pr_err("AUTHENC:Invalid Destination sg entries\n");
- error = -EINVAL;
- goto err;
+ dst_size = assoclen + req->cryptlen + (op_type ? -authsize :
+ authsize);
+ error = chcr_aead_common_init(req, op_type);
+ if (error)
+ return ERR_PTR(error);
+ if (dst_size) {
+ dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0);
+ dnents += sg_nents_xlen(req->dst, req->cryptlen +
+ (op_type ? -authsize : authsize), CHCR_DST_SG_SIZE,
+ req->assoclen);
+ dnents += MIN_AUTH_SG; // For IV
+ } else {
+ dnents = 0;
}
- nents = dst_size ? dstsg_2k(reqctx->dst, req->cryptlen +
- (op_type ? -authsize : authsize)) : 0;
- dst_size = get_space_for_phys_dsgl(nents);
+
+ dst_size = get_space_for_phys_dsgl(dnents);
kctx_len = (ntohl(KEY_CONTEXT_CTX_LEN_V(aeadctx->key_ctx_hdr)) << 4)
- sizeof(chcr_req->key_ctx);
transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
- if (chcr_aead_need_fallback(req, src_nent + MIN_AUTH_SG,
- T6_MAX_AAD_SIZE,
- transhdr_len + (sgl_len(src_nent + MIN_AUTH_SG) * 8),
- op_type)) {
+ reqctx->imm = (transhdr_len + assoclen + IV + req->cryptlen) <
+ SGE_MAX_WR_LEN;
+ temp = reqctx->imm ? (DIV_ROUND_UP((assoclen + IV + req->cryptlen), 16)
+ * 16) : (sgl_len(reqctx->src_nents + reqctx->aad_nents
+ + MIN_GCM_SG) * 8);
+ transhdr_len += temp;
+ transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16;
+
+ if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE,
+ transhdr_len, op_type)) {
atomic_inc(&adap->chcr_stats.fallback);
+ chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req,
+ op_type);
return ERR_PTR(chcr_aead_fallback(req, op_type));
}
- skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
+ skb = alloc_skb(SGE_MAX_WR_LEN, flags);
if (!skb) {
error = -ENOMEM;
goto err;
}
- /* LLD is going to write the sge hdr. */
- skb_reserve(skb, sizeof(struct sge_opaque_hdr));
-
- /* Write WR */
chcr_req = __skb_put_zero(skb, transhdr_len);
- stop_offset = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize;
+ temp = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize;
/*
* Input order is AAD,IV and Payload. where IV should be included as
@@ -1971,24 +2153,24 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req,
* to the hardware spec
*/
chcr_req->sec_cpl.op_ivinsrtofst =
- FILL_SEC_CPL_OP_IVINSR(ctx->dev->rx_channel_id, 2,
- (ivsize ? (assoclen + 1) : 0));
- chcr_req->sec_cpl.pldlen = htonl(assoclen + ivsize + req->cryptlen);
+ FILL_SEC_CPL_OP_IVINSR(a_ctx(tfm)->dev->rx_channel_id, 2,
+ assoclen + 1);
+ chcr_req->sec_cpl.pldlen = htonl(assoclen + IV + req->cryptlen);
chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
assoclen ? 1 : 0, assoclen,
- assoclen + ivsize + 1,
- (stop_offset & 0x1F0) >> 4);
+ assoclen + IV + 1,
+ (temp & 0x1F0) >> 4);
chcr_req->sec_cpl.cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(
- stop_offset & 0xF,
- null ? 0 : assoclen + ivsize + 1,
- stop_offset, stop_offset);
+ temp & 0xF,
+ null ? 0 : assoclen + IV + 1,
+ temp, temp);
chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type,
(op_type == CHCR_ENCRYPT_OP) ? 1 : 0,
CHCR_SCMD_CIPHER_MODE_AES_CBC,
actx->auth_mode, aeadctx->hmac_ctrl,
- ivsize >> 1);
+ IV >> 1);
chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
- 0, 1, dst_size);
+ 0, 0, dst_size);
chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
if (op_type == CHCR_ENCRYPT_OP)
@@ -2001,39 +2183,312 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req,
memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) <<
4), actx->h_iopad, kctx_len -
(DIV_ROUND_UP(aeadctx->enckey_len, 16) << 4));
-
+ memcpy(reqctx->iv, req->iv, IV);
phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
- sg_param.nents = reqctx->dst_nents;
- sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize);
- sg_param.qid = qid;
- error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl,
- reqctx->dst, &sg_param);
- if (error)
- goto dstmap_fail;
-
- skb_set_transport_header(skb, transhdr_len);
-
- if (assoclen) {
- /* AAD buffer in */
- write_sg_to_skb(skb, &frags, req->src, assoclen);
-
- }
- write_buffer_to_skb(skb, &frags, req->iv, ivsize);
- write_sg_to_skb(skb, &frags, src, req->cryptlen);
+ ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size);
+ chcr_add_aead_dst_ent(req, phys_cpl, assoclen, op_type, qid);
+ chcr_add_aead_src_ent(req, ulptx, assoclen, op_type);
atomic_inc(&adap->chcr_stats.cipher_rqst);
- create_wreq(ctx, chcr_req, &req->base, skb, size,
- sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len, 0);
+ temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size +
+ kctx_len + (reqctx->imm ? (assoclen + IV + req->cryptlen) : 0);
+ create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, size,
+ transhdr_len, temp, 0);
reqctx->skb = skb;
- skb_get(skb);
+ reqctx->op = op_type;
return skb;
-dstmap_fail:
- /* ivmap_fail: */
- kfree_skb(skb);
err:
+ chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req,
+ op_type);
+
return ERR_PTR(error);
}
+static int chcr_aead_dma_map(struct device *dev,
+ struct aead_request *req,
+ unsigned short op_type)
+{
+ int error;
+ struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
+ struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+ unsigned int authsize = crypto_aead_authsize(tfm);
+ int dst_size;
+
+ dst_size = req->assoclen + req->cryptlen + (op_type ?
+ -authsize : authsize);
+ if (!req->cryptlen || !dst_size)
+ return 0;
+ reqctx->iv_dma = dma_map_single(dev, reqctx->iv, IV,
+ DMA_BIDIRECTIONAL);
+ if (dma_mapping_error(dev, reqctx->iv_dma))
+ return -ENOMEM;
+
+ if (req->src == req->dst) {
+ error = dma_map_sg(dev, req->src, sg_nents(req->src),
+ DMA_BIDIRECTIONAL);
+ if (!error)
+ goto err;
+ } else {
+ error = dma_map_sg(dev, req->src, sg_nents(req->src),
+ DMA_TO_DEVICE);
+ if (!error)
+ goto err;
+ error = dma_map_sg(dev, req->dst, sg_nents(req->dst),
+ DMA_FROM_DEVICE);
+ if (!error) {
+ dma_unmap_sg(dev, req->src, sg_nents(req->src),
+ DMA_TO_DEVICE);
+ goto err;
+ }
+ }
+
+ return 0;
+err:
+ dma_unmap_single(dev, reqctx->iv_dma, IV, DMA_BIDIRECTIONAL);
+ return -ENOMEM;
+}
+
+static void chcr_aead_dma_unmap(struct device *dev,
+ struct aead_request *req,
+ unsigned short op_type)
+{
+ struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
+ struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+ unsigned int authsize = crypto_aead_authsize(tfm);
+ int dst_size;
+
+ dst_size = req->assoclen + req->cryptlen + (op_type ?
+ -authsize : authsize);
+ if (!req->cryptlen || !dst_size)
+ return;
+
+ dma_unmap_single(dev, reqctx->iv_dma, IV,
+ DMA_BIDIRECTIONAL);
+ if (req->src == req->dst) {
+ dma_unmap_sg(dev, req->src, sg_nents(req->src),
+ DMA_BIDIRECTIONAL);
+ } else {
+ dma_unmap_sg(dev, req->src, sg_nents(req->src),
+ DMA_TO_DEVICE);
+ dma_unmap_sg(dev, req->dst, sg_nents(req->dst),
+ DMA_FROM_DEVICE);
+ }
+}
+
+static inline void chcr_add_aead_src_ent(struct aead_request *req,
+ struct ulptx_sgl *ulptx,
+ unsigned int assoclen,
+ unsigned short op_type)
+{
+ struct ulptx_walk ulp_walk;
+ struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
+
+ if (reqctx->imm) {
+ u8 *buf = (u8 *)ulptx;
+
+ if (reqctx->b0_dma) {
+ memcpy(buf, reqctx->scratch_pad, reqctx->b0_len);
+ buf += reqctx->b0_len;
+ }
+ sg_pcopy_to_buffer(req->src, sg_nents(req->src),
+ buf, assoclen, 0);
+ buf += assoclen;
+ memcpy(buf, reqctx->iv, IV);
+ buf += IV;
+ sg_pcopy_to_buffer(req->src, sg_nents(req->src),
+ buf, req->cryptlen, req->assoclen);
+ } else {
+ ulptx_walk_init(&ulp_walk, ulptx);
+ if (reqctx->b0_dma)
+ ulptx_walk_add_page(&ulp_walk, reqctx->b0_len,
+ &reqctx->b0_dma);
+ ulptx_walk_add_sg(&ulp_walk, req->src, assoclen, 0);
+ ulptx_walk_add_page(&ulp_walk, IV, &reqctx->iv_dma);
+ ulptx_walk_add_sg(&ulp_walk, req->src, req->cryptlen,
+ req->assoclen);
+ ulptx_walk_end(&ulp_walk);
+ }
+}
+
+static inline void chcr_add_aead_dst_ent(struct aead_request *req,
+ struct cpl_rx_phys_dsgl *phys_cpl,
+ unsigned int assoclen,
+ unsigned short op_type,
+ unsigned short qid)
+{
+ struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
+ struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+ struct dsgl_walk dsgl_walk;
+ unsigned int authsize = crypto_aead_authsize(tfm);
+ u32 temp;
+
+ dsgl_walk_init(&dsgl_walk, phys_cpl);
+ if (reqctx->b0_dma)
+ dsgl_walk_add_page(&dsgl_walk, reqctx->b0_len, &reqctx->b0_dma);
+ dsgl_walk_add_sg(&dsgl_walk, req->dst, assoclen, 0);
+ dsgl_walk_add_page(&dsgl_walk, IV, &reqctx->iv_dma);
+ temp = req->cryptlen + (op_type ? -authsize : authsize);
+ dsgl_walk_add_sg(&dsgl_walk, req->dst, temp, req->assoclen);
+ dsgl_walk_end(&dsgl_walk, qid);
+}
+
+static inline void chcr_add_cipher_src_ent(struct ablkcipher_request *req,
+ struct ulptx_sgl *ulptx,
+ struct cipher_wr_param *wrparam)
+{
+ struct ulptx_walk ulp_walk;
+ struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
+
+ if (reqctx->imm) {
+ u8 *buf = (u8 *)ulptx;
+
+ memcpy(buf, reqctx->iv, IV);
+ buf += IV;
+ sg_pcopy_to_buffer(req->src, sg_nents(req->src),
+ buf, wrparam->bytes, reqctx->processed);
+ } else {
+ ulptx_walk_init(&ulp_walk, ulptx);
+ ulptx_walk_add_page(&ulp_walk, IV, &reqctx->iv_dma);
+ ulptx_walk_add_sg(&ulp_walk, reqctx->srcsg, wrparam->bytes,
+ reqctx->src_ofst);
+ reqctx->srcsg = ulp_walk.last_sg;
+ reqctx->src_ofst = ulp_walk.last_sg_len;
+ ulptx_walk_end(&ulp_walk);
+ }
+}
+
+static inline void chcr_add_cipher_dst_ent(struct ablkcipher_request *req,
+ struct cpl_rx_phys_dsgl *phys_cpl,
+ struct cipher_wr_param *wrparam,
+ unsigned short qid)
+{
+ struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
+ struct dsgl_walk dsgl_walk;
+
+ dsgl_walk_init(&dsgl_walk, phys_cpl);
+ dsgl_walk_add_page(&dsgl_walk, IV, &reqctx->iv_dma);
+ dsgl_walk_add_sg(&dsgl_walk, reqctx->dstsg, wrparam->bytes,
+ reqctx->dst_ofst);
+ reqctx->dstsg = dsgl_walk.last_sg;
+ reqctx->dst_ofst = dsgl_walk.last_sg_len;
+
+ dsgl_walk_end(&dsgl_walk, qid);
+}
+
+static inline void chcr_add_hash_src_ent(struct ahash_request *req,
+ struct ulptx_sgl *ulptx,
+ struct hash_wr_param *param)
+{
+ struct ulptx_walk ulp_walk;
+ struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req);
+
+ if (reqctx->imm) {
+ u8 *buf = (u8 *)ulptx;
+
+ if (param->bfr_len) {
+ memcpy(buf, reqctx->reqbfr, param->bfr_len);
+ buf += param->bfr_len;
+ }
+ sg_pcopy_to_buffer(req->src, sg_nents(req->src),
+ buf, param->sg_len, 0);
+ } else {
+ ulptx_walk_init(&ulp_walk, ulptx);
+ if (param->bfr_len)
+ ulptx_walk_add_page(&ulp_walk, param->bfr_len,
+ &reqctx->dma_addr);
+ ulptx_walk_add_sg(&ulp_walk, req->src, param->sg_len,
+ 0);
+// reqctx->srcsg = ulp_walk.last_sg;
+// reqctx->src_ofst = ulp_walk.last_sg_len;
+ ulptx_walk_end(&ulp_walk);
+ }
+}
+
+
+static inline int chcr_hash_dma_map(struct device *dev,
+ struct ahash_request *req)
+{
+ struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
+ int error = 0;
+
+ if (!req->nbytes)
+ return 0;
+ error = dma_map_sg(dev, req->src, sg_nents(req->src),
+ DMA_TO_DEVICE);
+ if (!error)
+ return error;
+ req_ctx->is_sg_map = 1;
+ return 0;
+}
+
+static inline void chcr_hash_dma_unmap(struct device *dev,
+ struct ahash_request *req)
+{
+ struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
+
+ if (!req->nbytes)
+ return;
+
+ dma_unmap_sg(dev, req->src, sg_nents(req->src),
+ DMA_TO_DEVICE);
+ req_ctx->is_sg_map = 0;
+
+}
+
+
+static int chcr_cipher_dma_map(struct device *dev,
+ struct ablkcipher_request *req)
+{
+ int error;
+ struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
+
+ reqctx->iv_dma = dma_map_single(dev, reqctx->iv, IV,
+ DMA_BIDIRECTIONAL);
+ if (dma_mapping_error(dev, reqctx->iv_dma))
+ return -ENOMEM;
+
+ if (req->src == req->dst) {
+ error = dma_map_sg(dev, req->src, sg_nents(req->src),
+ DMA_BIDIRECTIONAL);
+ if (!error)
+ goto err;
+ } else {
+ error = dma_map_sg(dev, req->src, sg_nents(req->src),
+ DMA_TO_DEVICE);
+ if (!error)
+ goto err;
+ error = dma_map_sg(dev, req->dst, sg_nents(req->dst),
+ DMA_FROM_DEVICE);
+ if (!error) {
+ dma_unmap_sg(dev, req->src, sg_nents(req->src),
+ DMA_TO_DEVICE);
+ goto err;
+ }
+ }
+
+ return 0;
+err:
+ dma_unmap_single(dev, reqctx->iv_dma, IV, DMA_BIDIRECTIONAL);
+ return -ENOMEM;
+}
+static void chcr_cipher_dma_unmap(struct device *dev,
+ struct ablkcipher_request *req)
+{
+ struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
+
+ dma_unmap_single(dev, reqctx->iv_dma, IV,
+ DMA_BIDIRECTIONAL);
+ if (req->src == req->dst) {
+ dma_unmap_sg(dev, req->src, sg_nents(req->src),
+ DMA_BIDIRECTIONAL);
+ } else {
+ dma_unmap_sg(dev, req->src, sg_nents(req->src),
+ DMA_TO_DEVICE);
+ dma_unmap_sg(dev, req->dst, sg_nents(req->dst),
+ DMA_FROM_DEVICE);
+ }
+}
+
static int set_msg_len(u8 *block, unsigned int msglen, int csize)
{
__be32 data;
@@ -2118,15 +2573,13 @@ static int ccm_format_packet(struct aead_request *req,
static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl,
unsigned int dst_size,
struct aead_request *req,
- unsigned short op_type,
- struct chcr_context *chcrctx)
+ unsigned short op_type)
{
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
- struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
- unsigned int ivsize = AES_BLOCK_SIZE;
+ struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
unsigned int cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_CCM;
unsigned int mac_mode = CHCR_SCMD_AUTH_MODE_CBCMAC;
- unsigned int c_id = chcrctx->dev->rx_channel_id;
+ unsigned int c_id = a_ctx(tfm)->dev->rx_channel_id;
unsigned int ccm_xtra;
unsigned char tag_offset = 0, auth_offset = 0;
unsigned int assoclen;
@@ -2139,7 +2592,7 @@ static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl,
((assoclen) ? CCM_AAD_FIELD_SIZE : 0);
auth_offset = req->cryptlen ?
- (assoclen + ivsize + 1 + ccm_xtra) : 0;
+ (assoclen + IV + 1 + ccm_xtra) : 0;
if (op_type == CHCR_DECRYPT_OP) {
if (crypto_aead_authsize(tfm) != req->cryptlen)
tag_offset = crypto_aead_authsize(tfm);
@@ -2149,14 +2602,13 @@ static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl,
sec_cpl->op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(c_id,
- 2, (ivsize ? (assoclen + 1) : 0) +
- ccm_xtra);
+ 2, assoclen + 1 + ccm_xtra);
sec_cpl->pldlen =
- htonl(assoclen + ivsize + req->cryptlen + ccm_xtra);
+ htonl(assoclen + IV + req->cryptlen + ccm_xtra);
/* For CCM there wil be b0 always. So AAD start will be 1 always */
sec_cpl->aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
1, assoclen + ccm_xtra, assoclen
- + ivsize + 1 + ccm_xtra, 0);
+ + IV + 1 + ccm_xtra, 0);
sec_cpl->cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(0,
auth_offset, tag_offset,
@@ -2165,10 +2617,10 @@ static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl,
sec_cpl->seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type,
(op_type == CHCR_ENCRYPT_OP) ? 0 : 1,
cipher_mode, mac_mode,
- aeadctx->hmac_ctrl, ivsize >> 1);
+ aeadctx->hmac_ctrl, IV >> 1);
sec_cpl->ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 0,
- 1, dst_size);
+ 0, dst_size);
}
int aead_ccm_validate_input(unsigned short op_type,
@@ -2188,119 +2640,83 @@ int aead_ccm_validate_input(unsigned short op_type,
return -EINVAL;
}
}
- if (aeadctx->enckey_len == 0) {
- pr_err("CCM: Encryption key not set\n");
- return -EINVAL;
- }
return 0;
}
-unsigned int fill_aead_req_fields(struct sk_buff *skb,
- struct aead_request *req,
- struct scatterlist *src,
- unsigned int ivsize,
- struct chcr_aead_ctx *aeadctx)
-{
- unsigned int frags = 0;
- struct crypto_aead *tfm = crypto_aead_reqtfm(req);
- struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
- /* b0 and aad length(if available) */
-
- write_buffer_to_skb(skb, &frags, reqctx->scratch_pad, CCM_B0_SIZE +
- (req->assoclen ? CCM_AAD_FIELD_SIZE : 0));
- if (req->assoclen) {
- if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309)
- write_sg_to_skb(skb, &frags, req->src,
- req->assoclen - 8);
- else
- write_sg_to_skb(skb, &frags, req->src, req->assoclen);
- }
- write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize);
- if (req->cryptlen)
- write_sg_to_skb(skb, &frags, src, req->cryptlen);
-
- return frags;
-}
-
static struct sk_buff *create_aead_ccm_wr(struct aead_request *req,
unsigned short qid,
int size,
unsigned short op_type)
{
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
- struct chcr_context *ctx = crypto_aead_ctx(tfm);
- struct uld_ctx *u_ctx = ULD_CTX(ctx);
- struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
+ struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
struct sk_buff *skb = NULL;
struct chcr_wr *chcr_req;
struct cpl_rx_phys_dsgl *phys_cpl;
- struct phys_sge_parm sg_param;
- struct scatterlist *src;
- unsigned int frags = 0, transhdr_len, ivsize = AES_BLOCK_SIZE;
- unsigned int dst_size = 0, kctx_len, nents;
- unsigned int sub_type;
+ struct ulptx_sgl *ulptx;
+ unsigned int transhdr_len;
+ unsigned int dst_size = 0, kctx_len, dnents, temp;
+ unsigned int sub_type, assoclen = req->assoclen;
unsigned int authsize = crypto_aead_authsize(tfm);
- int error = -EINVAL, src_nent;
+ int error = -EINVAL;
gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
GFP_ATOMIC;
- struct adapter *adap = padap(ctx->dev);
+ struct adapter *adap = padap(a_ctx(tfm)->dev);
- dst_size = req->assoclen + req->cryptlen + (op_type ? -authsize :
+ reqctx->b0_dma = 0;
+ sub_type = get_aead_subtype(tfm);
+ if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309)
+ assoclen -= 8;
+ dst_size = assoclen + req->cryptlen + (op_type ? -authsize :
authsize);
- if (op_type && req->cryptlen < crypto_aead_authsize(tfm))
- goto err;
- src_nent = sg_nents_for_len(req->src, req->assoclen + req->cryptlen);
- if (src_nent < 0)
- goto err;
+ error = chcr_aead_common_init(req, op_type);
+ if (error)
+ return ERR_PTR(error);
- sub_type = get_aead_subtype(tfm);
- src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen);
- reqctx->dst = src;
- if (req->src != req->dst) {
- error = chcr_copy_assoc(req, aeadctx);
- if (error) {
- pr_err("AAD copy to destination buffer fails\n");
- return ERR_PTR(error);
- }
- reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst,
- req->assoclen);
- }
- reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen +
- (op_type ? -authsize : authsize));
- if (reqctx->dst_nents < 0) {
- pr_err("CCM:Invalid Destination sg entries\n");
- error = -EINVAL;
- goto err;
- }
+
+ reqctx->b0_len = CCM_B0_SIZE + (assoclen ? CCM_AAD_FIELD_SIZE : 0);
error = aead_ccm_validate_input(op_type, req, aeadctx, sub_type);
if (error)
goto err;
- nents = dst_size ? dstsg_2k(reqctx->dst, req->cryptlen +
- (op_type ? -authsize : authsize)) : 0;
- dst_size = get_space_for_phys_dsgl(nents);
+ if (dst_size) {
+ dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0);
+ dnents += sg_nents_xlen(req->dst, req->cryptlen
+ + (op_type ? -authsize : authsize),
+ CHCR_DST_SG_SIZE, req->assoclen);
+ dnents += MIN_CCM_SG; // For IV and B0
+ } else {
+ dnents = 0;
+ }
+ dst_size = get_space_for_phys_dsgl(dnents);
kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) * 2;
transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
- if (chcr_aead_need_fallback(req, src_nent + MIN_CCM_SG,
- T6_MAX_AAD_SIZE - 18,
- transhdr_len + (sgl_len(src_nent + MIN_CCM_SG) * 8),
- op_type)) {
+ reqctx->imm = (transhdr_len + assoclen + IV + req->cryptlen +
+ reqctx->b0_len) <= SGE_MAX_WR_LEN;
+ temp = reqctx->imm ? (DIV_ROUND_UP((assoclen + IV + req->cryptlen +
+ reqctx->b0_len), 16) * 16) :
+ (sgl_len(reqctx->src_nents + reqctx->aad_nents +
+ MIN_CCM_SG) * 8);
+ transhdr_len += temp;
+ transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16;
+
+ if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE -
+ reqctx->b0_len, transhdr_len, op_type)) {
atomic_inc(&adap->chcr_stats.fallback);
+ chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req,
+ op_type);
return ERR_PTR(chcr_aead_fallback(req, op_type));
}
-
- skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
+ skb = alloc_skb(SGE_MAX_WR_LEN, flags);
if (!skb) {
error = -ENOMEM;
goto err;
}
- skb_reserve(skb, sizeof(struct sge_opaque_hdr));
+ chcr_req = (struct chcr_wr *) __skb_put_zero(skb, transhdr_len);
- chcr_req = __skb_put_zero(skb, transhdr_len);
-
- fill_sec_cpl_for_aead(&chcr_req->sec_cpl, dst_size, req, op_type, ctx);
+ fill_sec_cpl_for_aead(&chcr_req->sec_cpl, dst_size, req, op_type);
chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len);
@@ -2308,29 +2724,37 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req,
16), aeadctx->key, aeadctx->enckey_len);
phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
+ ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size);
error = ccm_format_packet(req, aeadctx, sub_type, op_type);
if (error)
goto dstmap_fail;
- sg_param.nents = reqctx->dst_nents;
- sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize);
- sg_param.qid = qid;
- error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl,
- reqctx->dst, &sg_param);
- if (error)
+ reqctx->b0_dma = dma_map_single(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev,
+ &reqctx->scratch_pad, reqctx->b0_len,
+ DMA_BIDIRECTIONAL);
+ if (dma_mapping_error(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev,
+ reqctx->b0_dma)) {
+ error = -ENOMEM;
goto dstmap_fail;
+ }
+
+ chcr_add_aead_dst_ent(req, phys_cpl, assoclen, op_type, qid);
+ chcr_add_aead_src_ent(req, ulptx, assoclen, op_type);
- skb_set_transport_header(skb, transhdr_len);
- frags = fill_aead_req_fields(skb, req, src, ivsize, aeadctx);
atomic_inc(&adap->chcr_stats.aead_rqst);
- create_wreq(ctx, chcr_req, &req->base, skb, 0,
- sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len, 0);
+ temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size +
+ kctx_len + (reqctx->imm ? (assoclen + IV + req->cryptlen +
+ reqctx->b0_len) : 0);
+ create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, 0,
+ transhdr_len, temp, 0);
reqctx->skb = skb;
- skb_get(skb);
+ reqctx->op = op_type;
+
return skb;
dstmap_fail:
kfree_skb(skb);
err:
+ chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, op_type);
return ERR_PTR(error);
}
@@ -2340,101 +2764,84 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req,
unsigned short op_type)
{
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
- struct chcr_context *ctx = crypto_aead_ctx(tfm);
- struct uld_ctx *u_ctx = ULD_CTX(ctx);
- struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
+ struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
struct sk_buff *skb = NULL;
struct chcr_wr *chcr_req;
struct cpl_rx_phys_dsgl *phys_cpl;
- struct phys_sge_parm sg_param;
- struct scatterlist *src;
- unsigned int frags = 0, transhdr_len;
- unsigned int ivsize = AES_BLOCK_SIZE;
- unsigned int dst_size = 0, kctx_len, nents, assoclen = req->assoclen;
- unsigned char tag_offset = 0;
+ struct ulptx_sgl *ulptx;
+ unsigned int transhdr_len, dnents = 0;
+ unsigned int dst_size = 0, temp = 0, kctx_len, assoclen = req->assoclen;
unsigned int authsize = crypto_aead_authsize(tfm);
- int error = -EINVAL, src_nent;
+ int error = -EINVAL;
gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
GFP_ATOMIC;
- struct adapter *adap = padap(ctx->dev);
-
- dst_size = assoclen + req->cryptlen + (op_type ? -authsize :
- authsize);
- /* validate key size */
- if (aeadctx->enckey_len == 0)
- goto err;
+ struct adapter *adap = padap(a_ctx(tfm)->dev);
- if (op_type && req->cryptlen < crypto_aead_authsize(tfm))
- goto err;
- src_nent = sg_nents_for_len(req->src, assoclen + req->cryptlen);
- if (src_nent < 0)
- goto err;
+ if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106)
+ assoclen = req->assoclen - 8;
- src = scatterwalk_ffwd(reqctx->srcffwd, req->src, assoclen);
- reqctx->dst = src;
- if (req->src != req->dst) {
- error = chcr_copy_assoc(req, aeadctx);
+ reqctx->b0_dma = 0;
+ dst_size = assoclen + req->cryptlen + (op_type ? -authsize : authsize);
+ error = chcr_aead_common_init(req, op_type);
if (error)
return ERR_PTR(error);
- reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst,
- req->assoclen);
- }
- reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen +
- (op_type ? -authsize : authsize));
- if (reqctx->dst_nents < 0) {
- pr_err("GCM:Invalid Destination sg entries\n");
- error = -EINVAL;
- goto err;
+ if (dst_size) {
+ dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0);
+ dnents += sg_nents_xlen(req->dst,
+ req->cryptlen + (op_type ? -authsize : authsize),
+ CHCR_DST_SG_SIZE, req->assoclen);
+ dnents += MIN_GCM_SG; // For IV
+ } else {
+ dnents = 0;
}
-
- nents = dst_size ? dstsg_2k(reqctx->dst, req->cryptlen +
- (op_type ? -authsize : authsize)) : 0;
- dst_size = get_space_for_phys_dsgl(nents);
+ dst_size = get_space_for_phys_dsgl(dnents);
kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) +
AEAD_H_SIZE;
transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
- if (chcr_aead_need_fallback(req, src_nent + MIN_GCM_SG,
- T6_MAX_AAD_SIZE,
- transhdr_len + (sgl_len(src_nent + MIN_GCM_SG) * 8),
- op_type)) {
+ reqctx->imm = (transhdr_len + assoclen + IV + req->cryptlen) <=
+ SGE_MAX_WR_LEN;
+ temp = reqctx->imm ? (DIV_ROUND_UP((assoclen + IV +
+ req->cryptlen), 16) * 16) : (sgl_len(reqctx->src_nents +
+ reqctx->aad_nents + MIN_GCM_SG) * 8);
+ transhdr_len += temp;
+ transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16;
+ if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE,
+ transhdr_len, op_type)) {
atomic_inc(&adap->chcr_stats.fallback);
+ chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req,
+ op_type);
return ERR_PTR(chcr_aead_fallback(req, op_type));
}
- skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
+ skb = alloc_skb(SGE_MAX_WR_LEN, flags);
if (!skb) {
error = -ENOMEM;
goto err;
}
- /* NIC driver is going to write the sge hdr. */
- skb_reserve(skb, sizeof(struct sge_opaque_hdr));
-
chcr_req = __skb_put_zero(skb, transhdr_len);
- if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106)
- assoclen = req->assoclen - 8;
-
- tag_offset = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize;
+ //Offset of tag from end
+ temp = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize;
chcr_req->sec_cpl.op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(
- ctx->dev->rx_channel_id, 2, (ivsize ?
- (assoclen + 1) : 0));
+ a_ctx(tfm)->dev->rx_channel_id, 2,
+ (assoclen + 1));
chcr_req->sec_cpl.pldlen =
- htonl(assoclen + ivsize + req->cryptlen);
+ htonl(assoclen + IV + req->cryptlen);
chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
assoclen ? 1 : 0, assoclen,
- assoclen + ivsize + 1, 0);
+ assoclen + IV + 1, 0);
chcr_req->sec_cpl.cipherstop_lo_authinsert =
- FILL_SEC_CPL_AUTHINSERT(0, assoclen + ivsize + 1,
- tag_offset, tag_offset);
+ FILL_SEC_CPL_AUTHINSERT(0, assoclen + IV + 1,
+ temp, temp);
chcr_req->sec_cpl.seqno_numivs =
FILL_SEC_CPL_SCMD0_SEQNO(op_type, (op_type ==
CHCR_ENCRYPT_OP) ? 1 : 0,
CHCR_SCMD_CIPHER_MODE_AES_GCM,
CHCR_SCMD_AUTH_MODE_GHASH,
- aeadctx->hmac_ctrl, ivsize >> 1);
+ aeadctx->hmac_ctrl, IV >> 1);
chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
- 0, 1, dst_size);
+ 0, 0, dst_size);
chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len);
memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) *
@@ -2452,30 +2859,21 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req,
*((unsigned int *)(reqctx->iv + 12)) = htonl(0x01);
phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
- sg_param.nents = reqctx->dst_nents;
- sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize);
- sg_param.qid = qid;
- error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl,
- reqctx->dst, &sg_param);
- if (error)
- goto dstmap_fail;
+ ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size);
- skb_set_transport_header(skb, transhdr_len);
- write_sg_to_skb(skb, &frags, req->src, assoclen);
- write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize);
- write_sg_to_skb(skb, &frags, src, req->cryptlen);
+ chcr_add_aead_dst_ent(req, phys_cpl, assoclen, op_type, qid);
+ chcr_add_aead_src_ent(req, ulptx, assoclen, op_type);
atomic_inc(&adap->chcr_stats.aead_rqst);
- create_wreq(ctx, chcr_req, &req->base, skb, size,
- sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len,
- reqctx->verify);
+ temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size +
+ kctx_len + (reqctx->imm ? (assoclen + IV + req->cryptlen) : 0);
+ create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, size,
+ transhdr_len, temp, reqctx->verify);
reqctx->skb = skb;
- skb_get(skb);
+ reqctx->op = op_type;
return skb;
-dstmap_fail:
- /* ivmap_fail: */
- kfree_skb(skb);
err:
+ chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, op_type);
return ERR_PTR(error);
}
@@ -2483,8 +2881,7 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req,
static int chcr_aead_cra_init(struct crypto_aead *tfm)
{
- struct chcr_context *ctx = crypto_aead_ctx(tfm);
- struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
+ struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
struct aead_alg *alg = crypto_aead_alg(tfm);
aeadctx->sw_cipher = crypto_alloc_aead(alg->base.cra_name, 0,
@@ -2495,25 +2892,20 @@ static int chcr_aead_cra_init(struct crypto_aead *tfm)
crypto_aead_set_reqsize(tfm, max(sizeof(struct chcr_aead_reqctx),
sizeof(struct aead_request) +
crypto_aead_reqsize(aeadctx->sw_cipher)));
- aeadctx->null = crypto_get_default_null_skcipher();
- if (IS_ERR(aeadctx->null))
- return PTR_ERR(aeadctx->null);
- return chcr_device_init(ctx);
+ return chcr_device_init(a_ctx(tfm));
}
static void chcr_aead_cra_exit(struct crypto_aead *tfm)
{
- struct chcr_context *ctx = crypto_aead_ctx(tfm);
- struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
+ struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
- crypto_put_default_null_skcipher();
crypto_free_aead(aeadctx->sw_cipher);
}
static int chcr_authenc_null_setauthsize(struct crypto_aead *tfm,
unsigned int authsize)
{
- struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
+ struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NOP;
aeadctx->mayverify = VERIFY_HW;
@@ -2522,7 +2914,7 @@ static int chcr_authenc_null_setauthsize(struct crypto_aead *tfm,
static int chcr_authenc_setauthsize(struct crypto_aead *tfm,
unsigned int authsize)
{
- struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
+ struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
u32 maxauth = crypto_aead_maxauthsize(tfm);
/*SHA1 authsize in ipsec is 12 instead of 10 i.e maxauthsize / 2 is not
@@ -2560,7 +2952,7 @@ static int chcr_authenc_setauthsize(struct crypto_aead *tfm,
static int chcr_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
{
- struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
+ struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
switch (authsize) {
case ICV_4:
@@ -2600,7 +2992,7 @@ static int chcr_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
static int chcr_4106_4309_setauthsize(struct crypto_aead *tfm,
unsigned int authsize)
{
- struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
+ struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
switch (authsize) {
case ICV_8:
@@ -2626,7 +3018,7 @@ static int chcr_4106_4309_setauthsize(struct crypto_aead *tfm,
static int chcr_ccm_setauthsize(struct crypto_aead *tfm,
unsigned int authsize)
{
- struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
+ struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
switch (authsize) {
case ICV_4:
@@ -2669,8 +3061,7 @@ static int chcr_ccm_common_setkey(struct crypto_aead *aead,
const u8 *key,
unsigned int keylen)
{
- struct chcr_context *ctx = crypto_aead_ctx(aead);
- struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
+ struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead));
unsigned char ck_size, mk_size;
int key_ctx_size = 0;
@@ -2703,8 +3094,7 @@ static int chcr_aead_ccm_setkey(struct crypto_aead *aead,
const u8 *key,
unsigned int keylen)
{
- struct chcr_context *ctx = crypto_aead_ctx(aead);
- struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
+ struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead));
int error;
crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
@@ -2722,8 +3112,7 @@ static int chcr_aead_ccm_setkey(struct crypto_aead *aead,
static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key,
unsigned int keylen)
{
- struct chcr_context *ctx = crypto_aead_ctx(aead);
- struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
+ struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead));
int error;
if (keylen < 3) {
@@ -2749,8 +3138,7 @@ static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key,
static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key,
unsigned int keylen)
{
- struct chcr_context *ctx = crypto_aead_ctx(aead);
- struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
+ struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead));
struct chcr_gcm_ctx *gctx = GCM_CTX(aeadctx);
struct crypto_cipher *cipher;
unsigned int ck_size;
@@ -2822,8 +3210,7 @@ static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key,
static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key,
unsigned int keylen)
{
- struct chcr_context *ctx = crypto_aead_ctx(authenc);
- struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
+ struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(authenc));
struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
/* it contains auth and cipher key both*/
struct crypto_authenc_keys keys;
@@ -2943,8 +3330,7 @@ static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key,
static int chcr_aead_digest_null_setkey(struct crypto_aead *authenc,
const u8 *key, unsigned int keylen)
{
- struct chcr_context *ctx = crypto_aead_ctx(authenc);
- struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
+ struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(authenc));
struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
struct crypto_authenc_keys keys;
int err;
@@ -3016,7 +3402,7 @@ static int chcr_aead_encrypt(struct aead_request *req)
static int chcr_aead_decrypt(struct aead_request *req)
{
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
- struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
+ struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
int size;
@@ -3049,30 +3435,29 @@ static int chcr_aead_op(struct aead_request *req,
create_wr_t create_wr_fn)
{
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
- struct chcr_context *ctx = crypto_aead_ctx(tfm);
struct uld_ctx *u_ctx;
struct sk_buff *skb;
- if (!ctx->dev) {
+ if (!a_ctx(tfm)->dev) {
pr_err("chcr : %s : No crypto device.\n", __func__);
return -ENXIO;
}
- u_ctx = ULD_CTX(ctx);
+ u_ctx = ULD_CTX(a_ctx(tfm));
if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
- ctx->tx_qidx)) {
+ a_ctx(tfm)->tx_qidx)) {
if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
return -EBUSY;
}
/* Form a WR from req */
- skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx], size,
+ skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[a_ctx(tfm)->rx_qidx], size,
op_type);
if (IS_ERR(skb) || !skb)
return PTR_ERR(skb);
skb->dev = u_ctx->lldi.ports[0];
- set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
+ set_wr_txq(skb, CPL_PRIORITY_DATA, a_ctx(tfm)->tx_qidx);
chcr_send_wr(skb);
return -EINPROGRESS;
}
diff --git a/drivers/crypto/chelsio/chcr_algo.h b/drivers/crypto/chelsio/chcr_algo.h
index d1a787b..96c9335 100644
--- a/drivers/crypto/chelsio/chcr_algo.h
+++ b/drivers/crypto/chelsio/chcr_algo.h
@@ -214,27 +214,22 @@
calc_tx_flits_ofld(skb) * 8), 16)))
#define FILL_CMD_MORE(immdatalen) htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) |\
- ULP_TX_SC_MORE_V((immdatalen) ? 0 : 1))
-
+ ULP_TX_SC_MORE_V((immdatalen)))
#define MAX_NK 8
-#define CRYPTO_MAX_IMM_TX_PKT_LEN 256
-#define MAX_WR_SIZE 512
#define ROUND_16(bytes) ((bytes) & 0xFFFFFFF0)
#define MAX_DSGL_ENT 32
-#define MAX_DIGEST_SKB_SGE (MAX_SKB_FRAGS - 1)
#define MIN_CIPHER_SG 1 /* IV */
-#define MIN_AUTH_SG 2 /*IV + AAD*/
-#define MIN_GCM_SG 2 /* IV + AAD*/
+#define MIN_AUTH_SG 1 /* IV */
+#define MIN_GCM_SG 1 /* IV */
#define MIN_DIGEST_SG 1 /*Partial Buffer*/
-#define MIN_CCM_SG 3 /*IV+AAD+B0*/
+#define MIN_CCM_SG 2 /*IV+B0*/
#define SPACE_LEFT(len) \
- ((MAX_WR_SIZE - WR_MIN_LEN - (len)))
+ ((SGE_MAX_WR_LEN - WR_MIN_LEN - (len)))
-unsigned int sgl_ent_len[] = {0, 0, 16, 24, 40,
- 48, 64, 72, 88,
- 96, 112, 120, 136,
- 144, 160, 168, 184,
- 192};
+unsigned int sgl_ent_len[] = {0, 0, 16, 24, 40, 48, 64, 72, 88,
+ 96, 112, 120, 136, 144, 160, 168, 184,
+ 192, 208, 216, 232, 240, 256, 264, 280,
+ 288, 304, 312, 328, 336, 352, 360, 376};
unsigned int dsgl_ent_len[] = {0, 32, 32, 48, 48, 64, 64, 80, 80,
112, 112, 128, 128, 144, 144, 160, 160,
192, 192, 208, 208, 224, 224, 240, 240,
@@ -258,7 +253,6 @@ struct hash_wr_param {
struct cipher_wr_param {
struct ablkcipher_request *req;
- struct scatterlist *srcsg;
char *iv;
int bytes;
unsigned short qid;
@@ -298,31 +292,11 @@ enum {
ICV_16 = 16
};
-struct hash_op_params {
- unsigned char mk_size;
- unsigned char pad_align;
- unsigned char auth_mode;
- char hash_name[MAX_HASH_NAME];
- unsigned short block_size;
- unsigned short word_size;
- unsigned short ipad_size;
-};
-
struct phys_sge_pairs {
__be16 len[8];
__be64 addr[8];
};
-struct phys_sge_parm {
- unsigned int nents;
- unsigned int obsize;
- unsigned short qid;
-};
-
-struct crypto_result {
- struct completion completion;
- int err;
-};
static const u32 sha1_init[SHA1_DIGEST_SIZE / 4] = {
SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4,
diff --git a/drivers/crypto/chelsio/chcr_crypto.h b/drivers/crypto/chelsio/chcr_crypto.h
index 8f436f8..94a87e3 100644
--- a/drivers/crypto/chelsio/chcr_crypto.h
+++ b/drivers/crypto/chelsio/chcr_crypto.h
@@ -149,9 +149,23 @@
#define CHCR_HASH_MAX_BLOCK_SIZE_64 64
#define CHCR_HASH_MAX_BLOCK_SIZE_128 128
-#define CHCR_SG_SIZE 2048
+#define CHCR_SRC_SG_SIZE (0x10000 - sizeof(int))
+#define CHCR_DST_SG_SIZE 2048
-/* Aligned to 128 bit boundary */
+static inline struct chcr_context *a_ctx(struct crypto_aead *tfm)
+{
+ return crypto_aead_ctx(tfm);
+}
+
+static inline struct chcr_context *c_ctx(struct crypto_ablkcipher *tfm)
+{
+ return crypto_ablkcipher_ctx(tfm);
+}
+
+static inline struct chcr_context *h_ctx(struct crypto_ahash *tfm)
+{
+ return crypto_tfm_ctx(crypto_ahash_tfm(tfm));
+}
struct ablk_ctx {
struct crypto_skcipher *sw_cipher;
@@ -165,15 +179,39 @@ struct ablk_ctx {
};
struct chcr_aead_reqctx {
struct sk_buff *skb;
- struct scatterlist *dst;
- struct scatterlist srcffwd[2];
- struct scatterlist dstffwd[2];
+ dma_addr_t iv_dma;
+ dma_addr_t b0_dma;
+ unsigned int b0_len;
+ unsigned int op;
+ short int aad_nents;
+ short int src_nents;
short int dst_nents;
+ u16 imm;
u16 verify;
u8 iv[CHCR_MAX_CRYPTO_IV_LEN];
unsigned char scratch_pad[MAX_SCRATCH_PAD_SIZE];
};
+struct ulptx_walk {
+ struct ulptx_sgl *sgl;
+ unsigned int nents;
+ unsigned int pair_idx;
+ unsigned int last_sg_len;
+ struct scatterlist *last_sg;
+ struct ulptx_sge_pair *pair;
+
+};
+
+struct dsgl_walk {
+ unsigned int nents;
+ unsigned int last_sg_len;
+ struct scatterlist *last_sg;
+ struct cpl_rx_phys_dsgl *dsgl;
+ struct phys_sge_pairs *to;
+};
+
+
+
struct chcr_gcm_ctx {
u8 ghash_h[AEAD_H_SIZE];
};
@@ -194,7 +232,6 @@ struct __aead_ctx {
struct chcr_aead_ctx {
__be32 key_ctx_hdr;
unsigned int enckey_len;
- struct crypto_skcipher *null;
struct crypto_aead *sw_cipher;
u8 salt[MAX_SALT];
u8 key[CHCR_AES_MAX_KEY_LEN];
@@ -230,8 +267,11 @@ struct chcr_ahash_req_ctx {
u8 bfr2[CHCR_HASH_MAX_BLOCK_SIZE_128];
u8 *reqbfr;
u8 *skbfr;
+ dma_addr_t dma_addr;
+ u32 dma_len;
u8 reqlen;
- /* DMA the partial hash in it */
+ u8 imm;
+ u8 is_sg_map;
u8 partial_hash[CHCR_HASH_MAX_DIGEST_SIZE];
u64 data_len; /* Data len till time */
/* SKB which is being sent to the hardware for processing */
@@ -240,14 +280,15 @@ struct chcr_ahash_req_ctx {
struct chcr_blkcipher_req_ctx {
struct sk_buff *skb;
- struct scatterlist srcffwd[2];
- struct scatterlist dstffwd[2];
struct scatterlist *dstsg;
- struct scatterlist *dst;
unsigned int processed;
unsigned int last_req_len;
+ struct scatterlist *srcsg;
+ unsigned int src_ofst;
+ unsigned int dst_ofst;
unsigned int op;
- short int dst_nents;
+ dma_addr_t iv_dma;
+ u16 imm;
u8 iv[CHCR_MAX_CRYPTO_IV_LEN];
};
@@ -261,24 +302,6 @@ struct chcr_alg_template {
} alg;
};
-struct chcr_req_ctx {
- union {
- struct ahash_request *ahash_req;
- struct aead_request *aead_req;
- struct ablkcipher_request *ablk_req;
- } req;
- union {
- struct chcr_ahash_req_ctx *ahash_ctx;
- struct chcr_aead_reqctx *reqctx;
- struct chcr_blkcipher_req_ctx *ablk_ctx;
- } ctx;
-};
-
-struct sge_opaque_hdr {
- void *dev;
- dma_addr_t addr[MAX_SKB_FRAGS + 1];
-};
-
typedef struct sk_buff *(*create_wr_t)(struct aead_request *req,
unsigned short qid,
int size,
@@ -291,4 +314,37 @@ static int chcr_aead_op(struct aead_request *req_base,
static inline int get_aead_subtype(struct crypto_aead *aead);
static int chcr_handle_cipher_resp(struct ablkcipher_request *req,
unsigned char *input, int err);
+static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err);
+static int chcr_aead_dma_map(struct device *dev, struct aead_request *req,
+ unsigned short op_type);
+static void chcr_aead_dma_unmap(struct device *dev, struct aead_request
+ *req, unsigned short op_type);
+static inline void chcr_add_aead_dst_ent(struct aead_request *req,
+ struct cpl_rx_phys_dsgl *phys_cpl,
+ unsigned int assoclen,
+ unsigned short op_type,
+ unsigned short qid);
+static inline void chcr_add_aead_src_ent(struct aead_request *req,
+ struct ulptx_sgl *ulptx,
+ unsigned int assoclen,
+ unsigned short op_type);
+static inline void chcr_add_cipher_src_ent(struct ablkcipher_request *req,
+ struct ulptx_sgl *ulptx,
+ struct cipher_wr_param *wrparam);
+static int chcr_cipher_dma_map(struct device *dev,
+ struct ablkcipher_request *req);
+static void chcr_cipher_dma_unmap(struct device *dev,
+ struct ablkcipher_request *req);
+static inline void chcr_add_cipher_dst_ent(struct ablkcipher_request *req,
+ struct cpl_rx_phys_dsgl *phys_cpl,
+ struct cipher_wr_param *wrparam,
+ unsigned short qid);
+int sg_nents_len_skip(struct scatterlist *sg, u64 len, u64 skip);
+static inline void chcr_add_hash_src_ent(struct ahash_request *req,
+ struct ulptx_sgl *ulptx,
+ struct hash_wr_param *param);
+static inline int chcr_hash_dma_map(struct device *dev,
+ struct ahash_request *req);
+static inline void chcr_hash_dma_unmap(struct device *dev,
+ struct ahash_request *req);
#endif /* __CHCR_CRYPTO_H__ */
diff --git a/drivers/net/ethernet/chelsio/cxgb4/sge.c b/drivers/net/ethernet/chelsio/cxgb4/sge.c
index ede1220..1c1cc48 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/sge.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/sge.c
@@ -1537,7 +1537,13 @@ int t4_mgmt_tx(struct adapter *adap, struct sk_buff *skb)
*/
static inline int is_ofld_imm(const struct sk_buff *skb)
{
- return skb->len <= MAX_IMM_TX_PKT_LEN;
+ struct work_request_hdr *req = (struct work_request_hdr *)skb->data;
+ unsigned long opcode = FW_WR_OP_G(ntohl(req->wr_hi));
+
+ if (opcode == FW_CRYPTO_LOOKASIDE_WR)
+ return skb->len <= SGE_MAX_WR_LEN;
+ else
+ return skb->len <= MAX_IMM_TX_PKT_LEN;
}
/**
--
2.1.4
^ permalink raw reply related
* [PATCH 7/7] crypto:chelsio: Fix memory leak
From: Harsh Jain @ 2017-10-03 6:46 UTC (permalink / raw)
To: herbert, linux-crypto, netdev; +Cc: Harsh Jain
In-Reply-To: <cover.1507010612.git.harsh@chelsio.com>
Fix memory leak when device does not support crypto.
Reported-by: Dan Carpenter <dan.carpenter@oracle.com>
Signed-off-by: Harsh Jain <harsh@chelsio.com>
---
drivers/crypto/chelsio/chcr_core.c | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/drivers/crypto/chelsio/chcr_core.c b/drivers/crypto/chelsio/chcr_core.c
index b6dd9cb..4f677b3 100644
--- a/drivers/crypto/chelsio/chcr_core.c
+++ b/drivers/crypto/chelsio/chcr_core.c
@@ -154,15 +154,15 @@ static void *chcr_uld_add(const struct cxgb4_lld_info *lld)
struct uld_ctx *u_ctx;
/* Create the device and add it in the device list */
+ if (!(lld->ulp_crypto & ULP_CRYPTO_LOOKASIDE))
+ return ERR_PTR(-EOPNOTSUPP);
+
+ /* Create the device and add it in the device list */
u_ctx = kzalloc(sizeof(*u_ctx), GFP_KERNEL);
if (!u_ctx) {
u_ctx = ERR_PTR(-ENOMEM);
goto out;
}
- if (!(lld->ulp_crypto & ULP_CRYPTO_LOOKASIDE)) {
- u_ctx = ERR_PTR(-ENOMEM);
- goto out;
- }
u_ctx->lldi = *lld;
out:
return u_ctx;
--
2.1.4
^ permalink raw reply related
* [PATCH 5/7] crypto:chelsio:Remove allocation of sg list to implement 2K limit of dsgl header
From: Harsh Jain @ 2017-10-03 6:46 UTC (permalink / raw)
To: herbert, linux-crypto, netdev; +Cc: Harsh Jain
In-Reply-To: <cover.1507010612.git.harsh@chelsio.com>
Update DMA address index instead of allocating new sg list to impose 2k size limit for each entry.
Signed-off-by: Harsh Jain <harsh@chelsio.com>
---
drivers/crypto/chelsio/chcr_algo.c | 237 +++++++++++------------------------
drivers/crypto/chelsio/chcr_algo.h | 3 +-
drivers/crypto/chelsio/chcr_core.h | 2 +-
drivers/crypto/chelsio/chcr_crypto.h | 6 -
4 files changed, 76 insertions(+), 172 deletions(-)
diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c
index e0ab34a..b13991d 100644
--- a/drivers/crypto/chelsio/chcr_algo.c
+++ b/drivers/crypto/chelsio/chcr_algo.c
@@ -117,6 +117,21 @@ static inline unsigned int sgl_len(unsigned int n)
return (3 * n) / 2 + (n & 1) + 2;
}
+static int dstsg_2k(struct scatterlist *sgl, unsigned int reqlen)
+{
+ int nents = 0;
+ unsigned int less;
+
+ while (sgl && reqlen) {
+ less = min(reqlen, sgl->length);
+ nents += DIV_ROUND_UP(less, CHCR_SG_SIZE);
+ reqlen -= less;
+ sgl = sg_next(sgl);
+ }
+
+ return nents;
+}
+
static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err)
{
u8 temp[SHA512_DIGEST_SIZE];
@@ -166,8 +181,6 @@ int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input,
kfree_skb(ctx_req.ctx.reqctx->skb);
ctx_req.ctx.reqctx->skb = NULL;
}
- free_new_sg(ctx_req.ctx.reqctx->newdstsg);
- ctx_req.ctx.reqctx->newdstsg = NULL;
if (ctx_req.ctx.reqctx->verify == VERIFY_SW) {
chcr_verify_tag(ctx_req.req.aead_req, input,
&err);
@@ -388,31 +401,41 @@ static void write_phys_cpl(struct cpl_rx_phys_dsgl *phys_cpl,
{
struct phys_sge_pairs *to;
unsigned int len = 0, left_size = sg_param->obsize;
- unsigned int nents = sg_param->nents, i, j = 0;
+ unsigned int j = 0;
+ int offset, ent_len;
phys_cpl->op_to_tid = htonl(CPL_RX_PHYS_DSGL_OPCODE_V(CPL_RX_PHYS_DSGL)
| CPL_RX_PHYS_DSGL_ISRDMA_V(0));
+ to = (struct phys_sge_pairs *)((unsigned char *)phys_cpl +
+ sizeof(struct cpl_rx_phys_dsgl));
+ while (left_size && sg) {
+ len = min_t(u32, left_size, sg_dma_len(sg));
+ offset = 0;
+ while (len) {
+ ent_len = min_t(u32, len, CHCR_SG_SIZE);
+ to->len[j % 8] = htons(ent_len);
+ to->addr[j % 8] = cpu_to_be64(sg_dma_address(sg) +
+ offset);
+ offset += ent_len;
+ len -= ent_len;
+ j++;
+ if ((j % 8) == 0)
+ to++;
+ }
+ left_size -= min(left_size, sg_dma_len(sg));
+ sg = sg_next(sg);
+ }
phys_cpl->pcirlxorder_to_noofsgentr =
htonl(CPL_RX_PHYS_DSGL_PCIRLXORDER_V(0) |
CPL_RX_PHYS_DSGL_PCINOSNOOP_V(0) |
CPL_RX_PHYS_DSGL_PCITPHNTENB_V(0) |
CPL_RX_PHYS_DSGL_PCITPHNT_V(0) |
CPL_RX_PHYS_DSGL_DCAID_V(0) |
- CPL_RX_PHYS_DSGL_NOOFSGENTR_V(nents));
+ CPL_RX_PHYS_DSGL_NOOFSGENTR_V(j));
phys_cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR;
phys_cpl->rss_hdr_int.qid = htons(sg_param->qid);
phys_cpl->rss_hdr_int.hash_val = 0;
- to = (struct phys_sge_pairs *)((unsigned char *)phys_cpl +
- sizeof(struct cpl_rx_phys_dsgl));
- for (i = 0; nents && left_size; to++) {
- for (j = 0; j < 8 && nents && left_size; j++, nents--) {
- len = min(left_size, sg_dma_len(sg));
- to->len[j] = htons(len);
- to->addr[j] = cpu_to_be64(sg_dma_address(sg));
- left_size -= len;
- sg = sg_next(sg);
- }
- }
+
}
static inline int map_writesg_phys_cpl(struct device *dev,
@@ -523,31 +546,33 @@ static int generate_copy_rrkey(struct ablk_ctx *ablkctx,
static int chcr_sg_ent_in_wr(struct scatterlist *src,
struct scatterlist *dst,
unsigned int minsg,
- unsigned int space,
- short int *sent,
- short int *dent)
+ unsigned int space)
{
int srclen = 0, dstlen = 0;
int srcsg = minsg, dstsg = 0;
+ int offset = 0, less;
- *sent = 0;
- *dent = 0;
while (src && dst && ((srcsg + 1) <= MAX_SKB_FRAGS) &&
space > (sgl_ent_len[srcsg + 1] + dsgl_ent_len[dstsg])) {
srclen += src->length;
srcsg++;
+ offset = 0;
while (dst && ((dstsg + 1) <= MAX_DSGL_ENT) &&
space > (sgl_ent_len[srcsg] + dsgl_ent_len[dstsg + 1])) {
if (srclen <= dstlen)
break;
- dstlen += dst->length;
- dst = sg_next(dst);
+ less = min_t(unsigned int, dst->length - offset,
+ CHCR_SG_SIZE);
+ dstlen += less;
+ offset += less;
+ if (offset == dst->length) {
+ dst = sg_next(dst);
+ offset = 0;
+ }
dstsg++;
}
src = sg_next(src);
}
- *sent = srcsg - minsg;
- *dent = dstsg;
return min(srclen, dstlen);
}
@@ -631,13 +656,15 @@ static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam)
struct phys_sge_parm sg_param;
unsigned int frags = 0, transhdr_len, phys_dsgl;
int error;
+ int nents;
unsigned int ivsize = AES_BLOCK_SIZE, kctx_len;
gfp_t flags = wrparam->req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
GFP_KERNEL : GFP_ATOMIC;
struct adapter *adap = padap(ctx->dev);
- phys_dsgl = get_space_for_phys_dsgl(reqctx->dst_nents);
-
+ reqctx->dst_nents = sg_nents_for_len(reqctx->dst, wrparam->bytes);
+ nents = dstsg_2k(reqctx->dst, wrparam->bytes);
+ phys_dsgl = get_space_for_phys_dsgl(nents);
kctx_len = (DIV_ROUND_UP(ablkctx->enckey_len, 16) * 16);
transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl);
skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
@@ -1020,8 +1047,7 @@ static int chcr_handle_cipher_resp(struct ablkcipher_request *req,
goto complete;
}
bytes = chcr_sg_ent_in_wr(wrparam.srcsg, reqctx->dst, 1,
- SPACE_LEFT(ablkctx->enckey_len),
- &wrparam.snent, &reqctx->dst_nents);
+ SPACE_LEFT(ablkctx->enckey_len));
if ((bytes + reqctx->processed) >= req->nbytes)
bytes = req->nbytes - reqctx->processed;
else
@@ -1060,8 +1086,6 @@ static int chcr_handle_cipher_resp(struct ablkcipher_request *req,
chcr_send_wr(skb);
return 0;
complete:
- free_new_sg(reqctx->newdstsg);
- reqctx->newdstsg = NULL;
req->base.complete(&req->base, err);
return err;
}
@@ -1077,9 +1101,8 @@ static int process_cipher(struct ablkcipher_request *req,
struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
struct cipher_wr_param wrparam;
- int bytes, nents, err = -EINVAL;
+ int bytes, err = -EINVAL;
- reqctx->newdstsg = NULL;
reqctx->processed = 0;
if (!req->info)
goto error;
@@ -1091,18 +1114,9 @@ static int process_cipher(struct ablkcipher_request *req,
goto error;
}
wrparam.srcsg = req->src;
- if (is_newsg(req->dst, &nents)) {
- reqctx->newdstsg = alloc_new_sg(req->dst, nents);
- if (IS_ERR(reqctx->newdstsg))
- return PTR_ERR(reqctx->newdstsg);
- reqctx->dstsg = reqctx->newdstsg;
- } else {
reqctx->dstsg = req->dst;
- }
bytes = chcr_sg_ent_in_wr(wrparam.srcsg, reqctx->dstsg, MIN_CIPHER_SG,
- SPACE_LEFT(ablkctx->enckey_len),
- &wrparam.snent,
- &reqctx->dst_nents);
+ SPACE_LEFT(ablkctx->enckey_len));
if ((bytes + reqctx->processed) >= req->nbytes)
bytes = req->nbytes - reqctx->processed;
else
@@ -1152,8 +1166,6 @@ static int process_cipher(struct ablkcipher_request *req,
return 0;
error:
- free_new_sg(reqctx->newdstsg);
- reqctx->newdstsg = NULL;
return err;
}
@@ -1824,63 +1836,6 @@ static void chcr_hmac_cra_exit(struct crypto_tfm *tfm)
}
}
-static int is_newsg(struct scatterlist *sgl, unsigned int *newents)
-{
- int nents = 0;
- int ret = 0;
-
- while (sgl) {
- if (sgl->length > CHCR_SG_SIZE)
- ret = 1;
- nents += DIV_ROUND_UP(sgl->length, CHCR_SG_SIZE);
- sgl = sg_next(sgl);
- }
- *newents = nents;
- return ret;
-}
-
-static inline void free_new_sg(struct scatterlist *sgl)
-{
- kfree(sgl);
-}
-
-static struct scatterlist *alloc_new_sg(struct scatterlist *sgl,
- unsigned int nents)
-{
- struct scatterlist *newsg, *sg;
- int i, len, processed = 0;
- struct page *spage;
- int offset;
-
- newsg = kmalloc_array(nents, sizeof(struct scatterlist), GFP_KERNEL);
- if (!newsg)
- return ERR_PTR(-ENOMEM);
- sg = newsg;
- sg_init_table(sg, nents);
- offset = sgl->offset;
- spage = sg_page(sgl);
- for (i = 0; i < nents; i++) {
- len = min_t(u32, sgl->length - processed, CHCR_SG_SIZE);
- sg_set_page(sg, spage, len, offset);
- processed += len;
- offset += len;
- if (offset >= PAGE_SIZE) {
- offset = offset % PAGE_SIZE;
- spage++;
- }
- if (processed == sgl->length) {
- processed = 0;
- sgl = sg_next(sgl);
- if (!sgl)
- break;
- spage = sg_page(sgl);
- offset = sgl->offset;
- }
- sg = sg_next(sg);
- }
- return newsg;
-}
-
static int chcr_copy_assoc(struct aead_request *req,
struct chcr_aead_ctx *ctx)
{
@@ -1953,7 +1908,6 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req,
GFP_ATOMIC;
struct adapter *adap = padap(ctx->dev);
- reqctx->newdstsg = NULL;
dst_size = req->assoclen + req->cryptlen + (op_type ? -authsize :
authsize);
if (aeadctx->enckey_len == 0 || (req->cryptlen <= 0))
@@ -1965,24 +1919,13 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req,
if (src_nent < 0)
goto err;
src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen);
-
+ reqctx->dst = src;
if (req->src != req->dst) {
error = chcr_copy_assoc(req, aeadctx);
if (error)
return ERR_PTR(error);
- }
- if (dst_size && is_newsg(req->dst, &nents)) {
- reqctx->newdstsg = alloc_new_sg(req->dst, nents);
- if (IS_ERR(reqctx->newdstsg))
- return ERR_CAST(reqctx->newdstsg);
- reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd,
- reqctx->newdstsg, req->assoclen);
- } else {
- if (req->src == req->dst)
- reqctx->dst = src;
- else
- reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd,
- req->dst, req->assoclen);
+ reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst,
+ req->assoclen);
}
if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_NULL) {
null = 1;
@@ -1995,7 +1938,9 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req,
error = -EINVAL;
goto err;
}
- dst_size = get_space_for_phys_dsgl(reqctx->dst_nents);
+ nents = dst_size ? dstsg_2k(reqctx->dst, req->cryptlen +
+ (op_type ? -authsize : authsize)) : 0;
+ dst_size = get_space_for_phys_dsgl(nents);
kctx_len = (ntohl(KEY_CONTEXT_CTX_LEN_V(aeadctx->key_ctx_hdr)) << 4)
- sizeof(chcr_req->key_ctx);
transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
@@ -2004,8 +1949,6 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req,
transhdr_len + (sgl_len(src_nent + MIN_AUTH_SG) * 8),
op_type)) {
atomic_inc(&adap->chcr_stats.fallback);
- free_new_sg(reqctx->newdstsg);
- reqctx->newdstsg = NULL;
return ERR_PTR(chcr_aead_fallback(req, op_type));
}
skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
@@ -2088,8 +2031,6 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req,
/* ivmap_fail: */
kfree_skb(skb);
err:
- free_new_sg(reqctx->newdstsg);
- reqctx->newdstsg = NULL;
return ERR_PTR(error);
}
@@ -2307,7 +2248,6 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req,
dst_size = req->assoclen + req->cryptlen + (op_type ? -authsize :
authsize);
- reqctx->newdstsg = NULL;
if (op_type && req->cryptlen < crypto_aead_authsize(tfm))
goto err;
src_nent = sg_nents_for_len(req->src, req->assoclen + req->cryptlen);
@@ -2316,25 +2256,15 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req,
sub_type = get_aead_subtype(tfm);
src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen);
+ reqctx->dst = src;
if (req->src != req->dst) {
error = chcr_copy_assoc(req, aeadctx);
if (error) {
pr_err("AAD copy to destination buffer fails\n");
return ERR_PTR(error);
}
- }
- if (dst_size && is_newsg(req->dst, &nents)) {
- reqctx->newdstsg = alloc_new_sg(req->dst, nents);
- if (IS_ERR(reqctx->newdstsg))
- return ERR_CAST(reqctx->newdstsg);
- reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd,
- reqctx->newdstsg, req->assoclen);
- } else {
- if (req->src == req->dst)
- reqctx->dst = src;
- else
- reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd,
- req->dst, req->assoclen);
+ reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst,
+ req->assoclen);
}
reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen +
(op_type ? -authsize : authsize));
@@ -2346,8 +2276,9 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req,
error = aead_ccm_validate_input(op_type, req, aeadctx, sub_type);
if (error)
goto err;
-
- dst_size = get_space_for_phys_dsgl(reqctx->dst_nents);
+ nents = dst_size ? dstsg_2k(reqctx->dst, req->cryptlen +
+ (op_type ? -authsize : authsize)) : 0;
+ dst_size = get_space_for_phys_dsgl(nents);
kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) * 2;
transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
if (chcr_aead_need_fallback(req, src_nent + MIN_CCM_SG,
@@ -2355,8 +2286,6 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req,
transhdr_len + (sgl_len(src_nent + MIN_CCM_SG) * 8),
op_type)) {
atomic_inc(&adap->chcr_stats.fallback);
- free_new_sg(reqctx->newdstsg);
- reqctx->newdstsg = NULL;
return ERR_PTR(chcr_aead_fallback(req, op_type));
}
@@ -2402,8 +2331,6 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req,
dstmap_fail:
kfree_skb(skb);
err:
- free_new_sg(reqctx->newdstsg);
- reqctx->newdstsg = NULL;
return ERR_PTR(error);
}
@@ -2432,7 +2359,6 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req,
GFP_ATOMIC;
struct adapter *adap = padap(ctx->dev);
- reqctx->newdstsg = NULL;
dst_size = assoclen + req->cryptlen + (op_type ? -authsize :
authsize);
/* validate key size */
@@ -2446,26 +2372,14 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req,
goto err;
src = scatterwalk_ffwd(reqctx->srcffwd, req->src, assoclen);
+ reqctx->dst = src;
if (req->src != req->dst) {
error = chcr_copy_assoc(req, aeadctx);
if (error)
return ERR_PTR(error);
+ reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst,
+ req->assoclen);
}
-
- if (dst_size && is_newsg(req->dst, &nents)) {
- reqctx->newdstsg = alloc_new_sg(req->dst, nents);
- if (IS_ERR(reqctx->newdstsg))
- return ERR_CAST(reqctx->newdstsg);
- reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd,
- reqctx->newdstsg, assoclen);
- } else {
- if (req->src == req->dst)
- reqctx->dst = src;
- else
- reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd,
- req->dst, assoclen);
- }
-
reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen +
(op_type ? -authsize : authsize));
if (reqctx->dst_nents < 0) {
@@ -2474,8 +2388,9 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req,
goto err;
}
-
- dst_size = get_space_for_phys_dsgl(reqctx->dst_nents);
+ nents = dst_size ? dstsg_2k(reqctx->dst, req->cryptlen +
+ (op_type ? -authsize : authsize)) : 0;
+ dst_size = get_space_for_phys_dsgl(nents);
kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) +
AEAD_H_SIZE;
transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
@@ -2484,8 +2399,6 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req,
transhdr_len + (sgl_len(src_nent + MIN_GCM_SG) * 8),
op_type)) {
atomic_inc(&adap->chcr_stats.fallback);
- free_new_sg(reqctx->newdstsg);
- reqctx->newdstsg = NULL;
return ERR_PTR(chcr_aead_fallback(req, op_type));
}
skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
@@ -2563,8 +2476,6 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req,
/* ivmap_fail: */
kfree_skb(skb);
err:
- free_new_sg(reqctx->newdstsg);
- reqctx->newdstsg = NULL;
return ERR_PTR(error);
}
diff --git a/drivers/crypto/chelsio/chcr_algo.h b/drivers/crypto/chelsio/chcr_algo.h
index 54851ec..d1a787b 100644
--- a/drivers/crypto/chelsio/chcr_algo.h
+++ b/drivers/crypto/chelsio/chcr_algo.h
@@ -221,7 +221,7 @@
#define MAX_WR_SIZE 512
#define ROUND_16(bytes) ((bytes) & 0xFFFFFFF0)
#define MAX_DSGL_ENT 32
-#define MAX_DIGEST_SKB_SGE (MAX_SKB_FRAGS - 2)
+#define MAX_DIGEST_SKB_SGE (MAX_SKB_FRAGS - 1)
#define MIN_CIPHER_SG 1 /* IV */
#define MIN_AUTH_SG 2 /*IV + AAD*/
#define MIN_GCM_SG 2 /* IV + AAD*/
@@ -261,7 +261,6 @@ struct cipher_wr_param {
struct scatterlist *srcsg;
char *iv;
int bytes;
- short int snent;
unsigned short qid;
};
enum {
diff --git a/drivers/crypto/chelsio/chcr_core.h b/drivers/crypto/chelsio/chcr_core.h
index c9a19b2..94e7412 100644
--- a/drivers/crypto/chelsio/chcr_core.h
+++ b/drivers/crypto/chelsio/chcr_core.h
@@ -89,7 +89,7 @@ struct uld_ctx {
struct chcr_dev *dev;
};
-struct uld_ctx * assign_chcr_device(void);
+struct uld_ctx *assign_chcr_device(void);
int chcr_send_wr(struct sk_buff *skb);
int start_crypto(void);
int stop_crypto(void);
diff --git a/drivers/crypto/chelsio/chcr_crypto.h b/drivers/crypto/chelsio/chcr_crypto.h
index b3722b3..8f436f8 100644
--- a/drivers/crypto/chelsio/chcr_crypto.h
+++ b/drivers/crypto/chelsio/chcr_crypto.h
@@ -166,7 +166,6 @@ struct ablk_ctx {
struct chcr_aead_reqctx {
struct sk_buff *skb;
struct scatterlist *dst;
- struct scatterlist *newdstsg;
struct scatterlist srcffwd[2];
struct scatterlist dstffwd[2];
short int dst_nents;
@@ -245,7 +244,6 @@ struct chcr_blkcipher_req_ctx {
struct scatterlist dstffwd[2];
struct scatterlist *dstsg;
struct scatterlist *dst;
- struct scatterlist *newdstsg;
unsigned int processed;
unsigned int last_req_len;
unsigned int op;
@@ -291,10 +289,6 @@ static int chcr_aead_op(struct aead_request *req_base,
int size,
create_wr_t create_wr_fn);
static inline int get_aead_subtype(struct crypto_aead *aead);
-static int is_newsg(struct scatterlist *sgl, unsigned int *newents);
-static struct scatterlist *alloc_new_sg(struct scatterlist *sgl,
- unsigned int nents);
-static inline void free_new_sg(struct scatterlist *sgl);
static int chcr_handle_cipher_resp(struct ablkcipher_request *req,
unsigned char *input, int err);
#endif /* __CHCR_CRYPTO_H__ */
--
2.1.4
^ permalink raw reply related
* [PATCH 4/7] crypto:chelsio:Use x8_ble gf multiplication to calculate IV.
From: Harsh Jain @ 2017-10-03 6:46 UTC (permalink / raw)
To: herbert, linux-crypto, netdev; +Cc: Harsh Jain
In-Reply-To: <cover.1507010612.git.harsh@chelsio.com>
gf128mul_x8_ble() will reduce gf Multiplication iteration by 8.
Signed-off-by: Harsh Jain <harsh@chelsio.com>
---
drivers/crypto/chelsio/chcr_algo.c | 11 +++++++++--
drivers/crypto/chelsio/chcr_crypto.h | 1 +
2 files changed, 10 insertions(+), 2 deletions(-)
diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c
index e4bf32d..e0ab34a 100644
--- a/drivers/crypto/chelsio/chcr_algo.c
+++ b/drivers/crypto/chelsio/chcr_algo.c
@@ -888,9 +888,11 @@ static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv)
int ret, i;
u8 *key;
unsigned int keylen;
+ int round = reqctx->last_req_len / AES_BLOCK_SIZE;
+ int round8 = round / 8;
cipher = ablkctx->aes_generic;
- memcpy(iv, req->info, AES_BLOCK_SIZE);
+ memcpy(iv, reqctx->iv, AES_BLOCK_SIZE);
keylen = ablkctx->enckey_len / 2;
key = ablkctx->key + keylen;
@@ -899,7 +901,10 @@ static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv)
goto out;
crypto_cipher_encrypt_one(cipher, iv, iv);
- for (i = 0; i < (reqctx->processed / AES_BLOCK_SIZE); i++)
+ for (i = 0; i < round8; i++)
+ gf128mul_x8_ble((le128 *)iv, (le128 *)iv);
+
+ for (i = 0; i < (round % 8); i++)
gf128mul_x_ble((le128 *)iv, (le128 *)iv);
crypto_cipher_decrypt_one(cipher, iv, iv);
@@ -1040,6 +1045,7 @@ static int chcr_handle_cipher_resp(struct ablkcipher_request *req,
CRYPTO_ALG_SUB_TYPE_CTR)
bytes = adjust_ctr_overflow(reqctx->iv, bytes);
reqctx->processed += bytes;
+ reqctx->last_req_len = bytes;
wrparam.qid = u_ctx->lldi.rxq_ids[ctx->rx_qidx];
wrparam.req = req;
wrparam.bytes = bytes;
@@ -1132,6 +1138,7 @@ static int process_cipher(struct ablkcipher_request *req,
goto error;
}
reqctx->processed = bytes;
+ reqctx->last_req_len = bytes;
reqctx->dst = reqctx->dstsg;
reqctx->op = op_type;
wrparam.qid = qid;
diff --git a/drivers/crypto/chelsio/chcr_crypto.h b/drivers/crypto/chelsio/chcr_crypto.h
index 30af1ee..b3722b3 100644
--- a/drivers/crypto/chelsio/chcr_crypto.h
+++ b/drivers/crypto/chelsio/chcr_crypto.h
@@ -247,6 +247,7 @@ struct chcr_blkcipher_req_ctx {
struct scatterlist *dst;
struct scatterlist *newdstsg;
unsigned int processed;
+ unsigned int last_req_len;
unsigned int op;
short int dst_nents;
u8 iv[CHCR_MAX_CRYPTO_IV_LEN];
--
2.1.4
^ permalink raw reply related
* [PATCH 3/7] crypto:gf128mul: The x8_ble multiplication functions
From: Harsh Jain @ 2017-10-03 6:46 UTC (permalink / raw)
To: herbert, linux-crypto, netdev; +Cc: Harsh Jain
In-Reply-To: <cover.1507010612.git.harsh@chelsio.com>
It multiply GF(2^128) elements in the ble format.
It will be used by chelsio driver to fasten gf multiplication.
Signed-off-by: Harsh Jain <harsh@chelsio.com>
---
crypto/gf128mul.c | 13 +++++++++++++
include/crypto/gf128mul.h | 2 +-
2 files changed, 14 insertions(+), 1 deletion(-)
diff --git a/crypto/gf128mul.c b/crypto/gf128mul.c
index dc01212..24e6019 100644
--- a/crypto/gf128mul.c
+++ b/crypto/gf128mul.c
@@ -156,6 +156,19 @@ static void gf128mul_x8_bbe(be128 *x)
x->b = cpu_to_be64((b << 8) ^ _tt);
}
+void gf128mul_x8_ble(le128 *r, const le128 *x)
+{
+ u64 a = le64_to_cpu(x->a);
+ u64 b = le64_to_cpu(x->b);
+
+ /* equivalent to gf128mul_table_be[b >> 63] (see crypto/gf128mul.c): */
+ u64 _tt = gf128mul_table_be[a >> 56];
+
+ r->a = cpu_to_le64((a << 8) | (b >> 56));
+ r->b = cpu_to_le64((b << 8) ^ _tt);
+}
+EXPORT_SYMBOL(gf128mul_x8_ble);
+
void gf128mul_lle(be128 *r, const be128 *b)
{
be128 p[8];
diff --git a/include/crypto/gf128mul.h b/include/crypto/gf128mul.h
index 0977fb1..fa0a63d 100644
--- a/include/crypto/gf128mul.h
+++ b/include/crypto/gf128mul.h
@@ -227,7 +227,7 @@ struct gf128mul_4k *gf128mul_init_4k_lle(const be128 *g);
struct gf128mul_4k *gf128mul_init_4k_bbe(const be128 *g);
void gf128mul_4k_lle(be128 *a, const struct gf128mul_4k *t);
void gf128mul_4k_bbe(be128 *a, const struct gf128mul_4k *t);
-
+void gf128mul_x8_ble(le128 *r, const le128 *x);
static inline void gf128mul_free_4k(struct gf128mul_4k *t)
{
kzfree(t);
--
2.1.4
^ permalink raw reply related
* [PATCH 1/7] crypto:chelsio: Remove unused parameter
From: Harsh Jain @ 2017-10-03 6:46 UTC (permalink / raw)
To: herbert, linux-crypto, netdev; +Cc: Harsh Jain
In-Reply-To: <cover.1507010612.git.harsh@chelsio.com>
Remove unused parameter sent to latest fw.
Signed-off-by: Harsh Jain <harsh@chelsio.com>
---
drivers/crypto/chelsio/chcr_algo.c | 43 +++++++++++++++-----------------------
drivers/crypto/chelsio/chcr_algo.h | 12 +++++------
2 files changed, 23 insertions(+), 32 deletions(-)
diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c
index 0e81607..bdb1014 100644
--- a/drivers/crypto/chelsio/chcr_algo.c
+++ b/drivers/crypto/chelsio/chcr_algo.c
@@ -577,36 +577,27 @@ static int chcr_cipher_fallback(struct crypto_skcipher *cipher,
static inline void create_wreq(struct chcr_context *ctx,
struct chcr_wr *chcr_req,
void *req, struct sk_buff *skb,
- int kctx_len, int hash_sz,
- int is_iv,
+ int hash_sz,
unsigned int sc_len,
unsigned int lcb)
{
struct uld_ctx *u_ctx = ULD_CTX(ctx);
- int iv_loc = IV_DSGL;
int qid = u_ctx->lldi.rxq_ids[ctx->rx_qidx];
- unsigned int immdatalen = 0, nr_frags = 0;
+ unsigned int immdatalen = 0;
- if (is_ofld_imm(skb)) {
+ if (is_ofld_imm(skb))
immdatalen = skb->data_len;
- iv_loc = IV_IMMEDIATE;
- } else {
- nr_frags = skb_shinfo(skb)->nr_frags;
- }
- chcr_req->wreq.op_to_cctx_size = FILL_WR_OP_CCTX_SIZE(immdatalen,
- ((sizeof(chcr_req->key_ctx) + kctx_len) >> 4));
+ chcr_req->wreq.op_to_cctx_size = FILL_WR_OP_CCTX_SIZE;
chcr_req->wreq.pld_size_hash_size =
- htonl(FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE_V(sgl_lengths[nr_frags]) |
- FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz));
+ htonl(FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz));
chcr_req->wreq.len16_pkd =
htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP(
(calc_tx_flits_ofld(skb) * 8), 16)));
chcr_req->wreq.cookie = cpu_to_be64((uintptr_t)req);
chcr_req->wreq.rx_chid_to_rx_q_id =
FILL_WR_RX_Q_ID(ctx->dev->rx_channel_id, qid,
- is_iv ? iv_loc : IV_NOP, !!lcb,
- ctx->tx_qidx);
+ !!lcb, ctx->tx_qidx);
chcr_req->ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(ctx->dev->tx_channel_id,
qid);
@@ -616,7 +607,7 @@ static inline void create_wreq(struct chcr_context *ctx,
chcr_req->sc_imm.cmd_more = FILL_CMD_MORE(immdatalen);
chcr_req->sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) +
sizeof(chcr_req->key_ctx) +
- kctx_len + sc_len + immdatalen);
+ sc_len + immdatalen);
}
/**
@@ -706,8 +697,8 @@ static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam)
write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize);
write_sg_to_skb(skb, &frags, wrparam->srcsg, wrparam->bytes);
atomic_inc(&adap->chcr_stats.cipher_rqst);
- create_wreq(ctx, chcr_req, &(wrparam->req->base), skb, kctx_len, 0, 1,
- sizeof(struct cpl_rx_phys_dsgl) + phys_dsgl,
+ create_wreq(ctx, chcr_req, &(wrparam->req->base), skb, 0,
+ sizeof(struct cpl_rx_phys_dsgl) + phys_dsgl + kctx_len,
ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC);
reqctx->skb = skb;
skb_get(skb);
@@ -1417,8 +1408,8 @@ static struct sk_buff *create_hash_wr(struct ahash_request *req,
if (param->sg_len != 0)
write_sg_to_skb(skb, &frags, req->src, param->sg_len);
atomic_inc(&adap->chcr_stats.digest_rqst);
- create_wreq(ctx, chcr_req, &req->base, skb, kctx_len,
- hash_size_in_response, 0, DUMMY_BYTES, 0);
+ create_wreq(ctx, chcr_req, &req->base, skb, hash_size_in_response,
+ DUMMY_BYTES + kctx_len, 0);
req_ctx->skb = skb;
skb_get(skb);
return skb;
@@ -2080,8 +2071,8 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req,
write_buffer_to_skb(skb, &frags, req->iv, ivsize);
write_sg_to_skb(skb, &frags, src, req->cryptlen);
atomic_inc(&adap->chcr_stats.cipher_rqst);
- create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, size, 1,
- sizeof(struct cpl_rx_phys_dsgl) + dst_size, 0);
+ create_wreq(ctx, chcr_req, &req->base, skb, size,
+ sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len, 0);
reqctx->skb = skb;
skb_get(skb);
@@ -2396,8 +2387,8 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req,
skb_set_transport_header(skb, transhdr_len);
frags = fill_aead_req_fields(skb, req, src, ivsize, aeadctx);
atomic_inc(&adap->chcr_stats.aead_rqst);
- create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, 0, 1,
- sizeof(struct cpl_rx_phys_dsgl) + dst_size, 0);
+ create_wreq(ctx, chcr_req, &req->base, skb, 0,
+ sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len, 0);
reqctx->skb = skb;
skb_get(skb);
return skb;
@@ -2554,8 +2545,8 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req,
write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize);
write_sg_to_skb(skb, &frags, src, req->cryptlen);
atomic_inc(&adap->chcr_stats.aead_rqst);
- create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, size, 1,
- sizeof(struct cpl_rx_phys_dsgl) + dst_size,
+ create_wreq(ctx, chcr_req, &req->base, skb, size,
+ sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len,
reqctx->verify);
reqctx->skb = skb;
skb_get(skb);
diff --git a/drivers/crypto/chelsio/chcr_algo.h b/drivers/crypto/chelsio/chcr_algo.h
index 583008d..54851ec 100644
--- a/drivers/crypto/chelsio/chcr_algo.h
+++ b/drivers/crypto/chelsio/chcr_algo.h
@@ -176,21 +176,21 @@
KEY_CONTEXT_SALT_PRESENT_V(1) | \
KEY_CONTEXT_CTX_LEN_V((ctx_len)))
-#define FILL_WR_OP_CCTX_SIZE(len, ctx_len) \
+#define FILL_WR_OP_CCTX_SIZE \
htonl( \
FW_CRYPTO_LOOKASIDE_WR_OPCODE_V( \
FW_CRYPTO_LOOKASIDE_WR) | \
FW_CRYPTO_LOOKASIDE_WR_COMPL_V(0) | \
- FW_CRYPTO_LOOKASIDE_WR_IMM_LEN_V((len)) | \
- FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC_V(1) | \
- FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE_V((ctx_len)))
+ FW_CRYPTO_LOOKASIDE_WR_IMM_LEN_V((0)) | \
+ FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC_V(0) | \
+ FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE_V(0))
-#define FILL_WR_RX_Q_ID(cid, qid, wr_iv, lcb, fid) \
+#define FILL_WR_RX_Q_ID(cid, qid, lcb, fid) \
htonl( \
FW_CRYPTO_LOOKASIDE_WR_RX_CHID_V((cid)) | \
FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID_V((qid)) | \
FW_CRYPTO_LOOKASIDE_WR_LCB_V((lcb)) | \
- FW_CRYPTO_LOOKASIDE_WR_IV_V((wr_iv)) | \
+ FW_CRYPTO_LOOKASIDE_WR_IV_V((IV_NOP)) | \
FW_CRYPTO_LOOKASIDE_WR_FQIDX_V(fid))
#define FILL_ULPTX_CMD_DEST(cid, qid) \
--
2.1.4
^ permalink raw reply related
* [PATCH 0/7]crypto:chelsio: Bugs fixes
From: Harsh Jain @ 2017-10-03 6:46 UTC (permalink / raw)
To: herbert, linux-crypto, netdev; +Cc: Harsh Jain
It includes bug fix and performance improvement changes.
Harsh Jain (7):
crypto:gf128mul: The x8_ble multiplication functions
crypto:chelsio:Use x8_ble gf multiplication to calculate IV.
crypto:chelsio:Remove allocation of sg list to implement 2K limit of
dsgl header
crypto:chelsio:Move DMA un/mapping to chcr from lld cxgb4 driver
crypto:chelsio: Fix memory leak
crypto:chelsio: Remove unused parameter
crypto:chelsio: Check error code with IS_ERR macro
crypto/gf128mul.c | 13 +
drivers/crypto/chelsio/chcr_algo.c | 1784 +++++++++++++++++-------------
drivers/crypto/chelsio/chcr_algo.h | 57 +-
drivers/crypto/chelsio/chcr_core.c | 8 +-
drivers/crypto/chelsio/chcr_core.h | 2 +-
drivers/crypto/chelsio/chcr_crypto.h | 121 +-
drivers/net/ethernet/chelsio/cxgb4/sge.c | 8 +-
include/crypto/gf128mul.h | 2 +-
8 files changed, 1166 insertions(+), 829 deletions(-)
--
2.1.4
^ permalink raw reply
* [PATCH 2/7] crypto:chelsio: Check error code with IS_ERR macro
From: Harsh Jain @ 2017-10-03 6:46 UTC (permalink / raw)
To: herbert, linux-crypto, netdev; +Cc: Harsh Jain
In-Reply-To: <cover.1507010612.git.harsh@chelsio.com>
Check and return proper error code.
Signed-off-by: Harsh Jain <harsh@chelsio.com>
---
drivers/crypto/chelsio/chcr_algo.c | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c
index bdb1014..e4bf32d 100644
--- a/drivers/crypto/chelsio/chcr_algo.c
+++ b/drivers/crypto/chelsio/chcr_algo.c
@@ -1455,8 +1455,8 @@ static int chcr_ahash_update(struct ahash_request *req)
req_ctx->result = 0;
req_ctx->data_len += params.sg_len + params.bfr_len;
skb = create_hash_wr(req, ¶ms);
- if (!skb)
- return -ENOMEM;
+ if (IS_ERR(skb))
+ return PTR_ERR(skb);
if (remainder) {
u8 *temp;
@@ -1519,8 +1519,8 @@ static int chcr_ahash_final(struct ahash_request *req)
params.more = 0;
}
skb = create_hash_wr(req, ¶ms);
- if (!skb)
- return -ENOMEM;
+ if (IS_ERR(skb))
+ return PTR_ERR(skb);
skb->dev = u_ctx->lldi.ports[0];
set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
@@ -1570,8 +1570,8 @@ static int chcr_ahash_finup(struct ahash_request *req)
}
skb = create_hash_wr(req, ¶ms);
- if (!skb)
- return -ENOMEM;
+ if (IS_ERR(skb))
+ return PTR_ERR(skb);
skb->dev = u_ctx->lldi.ports[0];
set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
@@ -1621,8 +1621,8 @@ static int chcr_ahash_digest(struct ahash_request *req)
}
skb = create_hash_wr(req, ¶ms);
- if (!skb)
- return -ENOMEM;
+ if (IS_ERR(skb))
+ return PTR_ERR(skb);
skb->dev = u_ctx->lldi.ports[0];
set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
--
2.1.4
^ permalink raw reply related
* [PATCH net-next] cxgb4: Update comment for min_mtu
From: Ganesh Goudar @ 2017-10-03 6:13 UTC (permalink / raw)
To: netdev, davem
Cc: nirranjan, indranil, venkatesh, Arjun Vynipadath, Ganesh Goudar
From: Arjun Vynipadath <arjun@chelsio.com>
We have lost a comment for minimum mtu value set for netdevice with
'commit d894be57ca92 ("ethernet: use net core MTU range checking in
more drivers"). Updating it accordingly.
Signed-off-by: Arjun Vynipadath <arjun@chelsio.com>
Signed-off-by: Ganesh Goudar <ganeshgr@chelsio.com>
---
drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
index 13b636b..fe4cbe2 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
@@ -5024,7 +5024,7 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
netdev->priv_flags |= IFF_UNICAST_FLT;
/* MTU range: 81 - 9600 */
- netdev->min_mtu = 81;
+ netdev->min_mtu = 81; /* accommodate SACK */
netdev->max_mtu = MAX_MTU;
netdev->netdev_ops = &cxgb4_netdev_ops;
--
2.1.0
^ permalink raw reply related
* Re: [PATCH net-next 3/3] bridge: suppress nd pkts on BR_NEIGH_SUPPRESS ports
From: kbuild test robot @ 2017-10-03 5:53 UTC (permalink / raw)
To: Roopa Prabhu; +Cc: kbuild-all, davem, netdev, nikolay, stephen, bridge
In-Reply-To: <1506919018-27875-4-git-send-email-roopa@cumulusnetworks.com>
[-- Attachment #1: Type: text/plain, Size: 4879 bytes --]
Hi Roopa,
[auto build test ERROR on net-next/master]
url: https://github.com/0day-ci/linux/commits/Roopa-Prabhu/bridge-neigh-msg-proxy-and-flood-suppression-support/20171003-124610
config: tile-allyesconfig (attached as .config)
compiler: tilegx-linux-gcc (GCC) 4.6.2
reproduce:
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# save the attached .config to linux build tree
make.cross ARCH=tile
All errors (new ones prefixed by >>):
net/bridge/br_arp_nd_proxy.c: In function 'br_nd_send':
>> net/bridge/br_arp_nd_proxy.c:310:2: error: implicit declaration of function 'csum_ipv6_magic'
cc1: some warnings being treated as errors
vim +/csum_ipv6_magic +310 net/bridge/br_arp_nd_proxy.c
232
233 static void br_nd_send(struct net_bridge_port *p, struct sk_buff *request,
234 struct neighbour *n, __be16 vlan_proto, u16 vlan_tci,
235 struct nd_msg *ns)
236 {
237 struct net_device *dev = request->dev;
238 struct sk_buff *reply;
239 struct nd_msg *na;
240 struct ipv6hdr *pip6;
241 u8 *daddr;
242 int na_olen = 8; /* opt hdr + ETH_ALEN for target */
243 int ns_olen;
244 int i, len;
245
246 if (!dev)
247 return;
248
249 len = LL_RESERVED_SPACE(dev) + sizeof(struct ipv6hdr) +
250 sizeof(*na) + na_olen + dev->needed_tailroom;
251
252 reply = alloc_skb(len, GFP_ATOMIC);
253 if (!reply)
254 return;
255
256 reply->protocol = htons(ETH_P_IPV6);
257 reply->dev = dev;
258 skb_reserve(reply, LL_RESERVED_SPACE(dev));
259 skb_push(reply, sizeof(struct ethhdr));
260 skb_set_mac_header(reply, 0);
261
262 daddr = eth_hdr(request)->h_source;
263
264 /* Do we need option processing ? */
265 ns_olen = request->len - (skb_network_offset(request) +
266 sizeof(struct ipv6hdr)) - sizeof(*ns);
267 for (i = 0; i < ns_olen - 1; i += (ns->opt[i + 1] << 3)) {
268 if (ns->opt[i] == ND_OPT_SOURCE_LL_ADDR) {
269 daddr = ns->opt + i + sizeof(struct nd_opt_hdr);
270 break;
271 }
272 }
273
274 /* Ethernet header */
275 ether_addr_copy(eth_hdr(reply)->h_dest, daddr);
276 ether_addr_copy(eth_hdr(reply)->h_source, n->ha);
277 eth_hdr(reply)->h_proto = htons(ETH_P_IPV6);
278 reply->protocol = htons(ETH_P_IPV6);
279
280 skb_pull(reply, sizeof(struct ethhdr));
281 skb_set_network_header(reply, 0);
282 skb_put(reply, sizeof(struct ipv6hdr));
283
284 /* IPv6 header */
285 pip6 = ipv6_hdr(reply);
286 memset(pip6, 0, sizeof(struct ipv6hdr));
287 pip6->version = 6;
288 pip6->priority = ipv6_hdr(request)->priority;
289 pip6->nexthdr = IPPROTO_ICMPV6;
290 pip6->hop_limit = 255;
291 pip6->daddr = ipv6_hdr(request)->saddr;
292 pip6->saddr = *(struct in6_addr *)n->primary_key;
293
294 skb_pull(reply, sizeof(struct ipv6hdr));
295 skb_set_transport_header(reply, 0);
296
297 na = (struct nd_msg *)skb_put(reply, sizeof(*na) + na_olen);
298
299 /* Neighbor Advertisement */
300 memset(na, 0, sizeof(*na) + na_olen);
301 na->icmph.icmp6_type = NDISC_NEIGHBOUR_ADVERTISEMENT;
302 na->icmph.icmp6_router = 0; /* XXX: should be 1 ? */
303 na->icmph.icmp6_override = 1;
304 na->icmph.icmp6_solicited = 1;
305 na->target = ns->target;
306 ether_addr_copy(&na->opt[2], n->ha);
307 na->opt[0] = ND_OPT_TARGET_LL_ADDR;
308 na->opt[1] = na_olen >> 3;
309
> 310 na->icmph.icmp6_cksum = csum_ipv6_magic(&pip6->saddr,
311 &pip6->daddr,
312 sizeof(*na) + na_olen,
313 IPPROTO_ICMPV6,
314 csum_partial(na, sizeof(*na) + na_olen, 0));
315
316 pip6->payload_len = htons(sizeof(*na) + na_olen);
317
318 skb_push(reply, sizeof(struct ipv6hdr));
319 skb_push(reply, sizeof(struct ethhdr));
320
321 reply->ip_summed = CHECKSUM_UNNECESSARY;
322
323 if (p) {
324 struct net_bridge_vlan_group *vg;
325 u16 pvid;
326
327 vg = nbp_vlan_group_rcu(p);
328 pvid = br_get_pvid(vg);
329 if (pvid && pvid == vlan_tci)
330 vlan_tci = 0;
331 }
332
333 if (vlan_tci != 0) {
334 reply = vlan_insert_tag_set_proto(reply, vlan_proto, vlan_tci);
335 if (!reply) {
336 net_err_ratelimited("evpn: failed to insert VLAN tag\n");
337 return;
338 }
339 }
340
341 netdev_dbg(dev, "nd send dev %s dst %pI6 dst_hw %pM src %pI6 src_hw %pM\n",
342 dev->name, &pip6->daddr, daddr, &pip6->saddr, n->ha);
343
344 dev_queue_xmit(reply);
345 }
346
---
0-DAY kernel test infrastructure Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all Intel Corporation
[-- Attachment #2: .config.gz --]
[-- Type: application/gzip, Size: 50505 bytes --]
^ permalink raw reply
* [PATCH v2 net-next 8/8] samples/bpf: use bpf_prog_query() interface
From: Alexei Starovoitov @ 2017-10-03 5:50 UTC (permalink / raw)
To: David S . Miller
Cc: Daniel Borkmann, Tejun Heo, David Ahern, netdev, kernel-team
In-Reply-To: <20171003055028.1294791-1-ast@fb.com>
use BPF_PROG_QUERY command to strengthen test coverage
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Acked-by: Daniel Borkmann <daniel@iogearbox.net>
---
samples/bpf/test_cgrp2_attach2.c | 36 ++++++++++++++++++++++++++++++++++++
1 file changed, 36 insertions(+)
diff --git a/samples/bpf/test_cgrp2_attach2.c b/samples/bpf/test_cgrp2_attach2.c
index 9a9f6836e5e9..3e8232cc04a8 100644
--- a/samples/bpf/test_cgrp2_attach2.c
+++ b/samples/bpf/test_cgrp2_attach2.c
@@ -236,6 +236,7 @@ static int prog_load_cnt(int verdict, int val)
static int test_multiprog(void)
{
+ __u32 prog_ids[4], prog_cnt = 0, attach_flags, saved_prog_id;
int cg1 = 0, cg2 = 0, cg3 = 0, cg4 = 0, cg5 = 0, key = 0;
int drop_prog, allow_prog[6] = {}, rc = 0;
unsigned long long value;
@@ -304,6 +305,32 @@ static int test_multiprog(void)
assert(bpf_map_lookup_elem(map_fd, &key, &value) == 0);
assert(value == 1 + 2 + 8 + 32);
+ /* query the number of effective progs in cg5 */
+ assert(bpf_prog_query(cg5, BPF_CGROUP_INET_EGRESS, BPF_F_QUERY_EFFECTIVE,
+ NULL, NULL, &prog_cnt) == 0);
+ assert(prog_cnt == 4);
+ /* retrieve prog_ids of effective progs in cg5 */
+ assert(bpf_prog_query(cg5, BPF_CGROUP_INET_EGRESS, BPF_F_QUERY_EFFECTIVE,
+ &attach_flags, prog_ids, &prog_cnt) == 0);
+ assert(prog_cnt == 4);
+ assert(attach_flags == 0);
+ saved_prog_id = prog_ids[0];
+ /* check enospc handling */
+ prog_ids[0] = 0;
+ prog_cnt = 2;
+ assert(bpf_prog_query(cg5, BPF_CGROUP_INET_EGRESS, BPF_F_QUERY_EFFECTIVE,
+ &attach_flags, prog_ids, &prog_cnt) == -1 &&
+ errno == ENOSPC);
+ assert(prog_cnt == 4);
+ /* check that prog_ids are returned even when buffer is too small */
+ assert(prog_ids[0] == saved_prog_id);
+ /* retrieve prog_id of single attached prog in cg5 */
+ prog_ids[0] = 0;
+ assert(bpf_prog_query(cg5, BPF_CGROUP_INET_EGRESS, 0,
+ NULL, prog_ids, &prog_cnt) == 0);
+ assert(prog_cnt == 1);
+ assert(prog_ids[0] == saved_prog_id);
+
/* detach bottom program and ping again */
if (bpf_prog_detach2(-1, cg5, BPF_CGROUP_INET_EGRESS)) {
log_err("Detaching prog from cg5");
@@ -341,6 +368,15 @@ static int test_multiprog(void)
assert(system(PING_CMD) == 0);
assert(bpf_map_lookup_elem(map_fd, &key, &value) == 0);
assert(value == 1 + 2 + 4);
+
+ prog_cnt = 4;
+ assert(bpf_prog_query(cg5, BPF_CGROUP_INET_EGRESS, BPF_F_QUERY_EFFECTIVE,
+ &attach_flags, prog_ids, &prog_cnt) == 0);
+ assert(prog_cnt == 3);
+ assert(attach_flags == 0);
+ assert(bpf_prog_query(cg5, BPF_CGROUP_INET_EGRESS, 0,
+ NULL, prog_ids, &prog_cnt) == 0);
+ assert(prog_cnt == 0);
goto out;
err:
rc = 1;
--
2.9.5
^ permalink raw reply related
* [PATCH v2 net-next 1/8] bpf: multi program support for cgroup+bpf
From: Alexei Starovoitov @ 2017-10-03 5:50 UTC (permalink / raw)
To: David S . Miller
Cc: Daniel Borkmann, Tejun Heo, David Ahern, netdev, kernel-team
In-Reply-To: <20171003055028.1294791-1-ast@fb.com>
introduce BPF_F_ALLOW_MULTI flag that can be used to attach multiple
bpf programs to a cgroup.
The difference between three possible flags for BPF_PROG_ATTACH command:
- NONE(default): No further bpf programs allowed in the subtree.
- BPF_F_ALLOW_OVERRIDE: If a sub-cgroup installs some bpf program,
the program in this cgroup yields to sub-cgroup program.
- BPF_F_ALLOW_MULTI: If a sub-cgroup installs some bpf program,
that cgroup program gets run in addition to the program in this cgroup.
NONE and BPF_F_ALLOW_OVERRIDE existed before. This patch doesn't
change their behavior. It only clarifies the semantics in relation
to new flag.
Only one program is allowed to be attached to a cgroup with
NONE or BPF_F_ALLOW_OVERRIDE flag.
Multiple programs are allowed to be attached to a cgroup with
BPF_F_ALLOW_MULTI flag. They are executed in FIFO order
(those that were attached first, run first)
The programs of sub-cgroup are executed first, then programs of
this cgroup and then programs of parent cgroup.
All eligible programs are executed regardless of return code from
earlier programs.
To allow efficient execution of multiple programs attached to a cgroup
and to avoid penalizing cgroups without any programs attached
introduce 'struct bpf_prog_array' which is RCU protected array
of pointers to bpf programs.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Acked-by: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Martin KaFai Lau <kafai@fb.com>
for cgroup bits
Acked-by: Tejun Heo <tj@kernel.org>
---
include/linux/bpf-cgroup.h | 46 +++--
include/linux/bpf.h | 32 ++++
include/linux/filter.h | 2 +-
include/uapi/linux/bpf.h | 42 +++-
kernel/bpf/cgroup.c | 467 ++++++++++++++++++++++++++++++++-------------
kernel/bpf/core.c | 31 +++
kernel/bpf/syscall.c | 37 ++--
kernel/cgroup/cgroup.c | 28 ++-
8 files changed, 516 insertions(+), 169 deletions(-)
diff --git a/include/linux/bpf-cgroup.h b/include/linux/bpf-cgroup.h
index d41d40ac3efd..102e56fbb6de 100644
--- a/include/linux/bpf-cgroup.h
+++ b/include/linux/bpf-cgroup.h
@@ -14,27 +14,42 @@ struct bpf_sock_ops_kern;
extern struct static_key_false cgroup_bpf_enabled_key;
#define cgroup_bpf_enabled static_branch_unlikely(&cgroup_bpf_enabled_key)
+struct bpf_prog_list {
+ struct list_head node;
+ struct bpf_prog *prog;
+};
+
+struct bpf_prog_array;
+
struct cgroup_bpf {
- /*
- * Store two sets of bpf_prog pointers, one for programs that are
- * pinned directly to this cgroup, and one for those that are effective
- * when this cgroup is accessed.
+ /* array of effective progs in this cgroup */
+ struct bpf_prog_array __rcu *effective[MAX_BPF_ATTACH_TYPE];
+
+ /* attached progs to this cgroup and attach flags
+ * when flags == 0 or BPF_F_ALLOW_OVERRIDE the progs list will
+ * have either zero or one element
+ * when BPF_F_ALLOW_MULTI the list can have up to BPF_CGROUP_MAX_PROGS
*/
- struct bpf_prog *prog[MAX_BPF_ATTACH_TYPE];
- struct bpf_prog __rcu *effective[MAX_BPF_ATTACH_TYPE];
- bool disallow_override[MAX_BPF_ATTACH_TYPE];
+ struct list_head progs[MAX_BPF_ATTACH_TYPE];
+ u32 flags[MAX_BPF_ATTACH_TYPE];
+
+ /* temp storage for effective prog array used by prog_attach/detach */
+ struct bpf_prog_array __rcu *inactive;
};
void cgroup_bpf_put(struct cgroup *cgrp);
-void cgroup_bpf_inherit(struct cgroup *cgrp, struct cgroup *parent);
+int cgroup_bpf_inherit(struct cgroup *cgrp);
-int __cgroup_bpf_update(struct cgroup *cgrp, struct cgroup *parent,
- struct bpf_prog *prog, enum bpf_attach_type type,
- bool overridable);
+int __cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *prog,
+ enum bpf_attach_type type, u32 flags);
+int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
+ enum bpf_attach_type type, u32 flags);
-/* Wrapper for __cgroup_bpf_update() protected by cgroup_mutex */
-int cgroup_bpf_update(struct cgroup *cgrp, struct bpf_prog *prog,
- enum bpf_attach_type type, bool overridable);
+/* Wrapper for __cgroup_bpf_*() protected by cgroup_mutex */
+int cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *prog,
+ enum bpf_attach_type type, u32 flags);
+int cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
+ enum bpf_attach_type type, u32 flags);
int __cgroup_bpf_run_filter_skb(struct sock *sk,
struct sk_buff *skb,
@@ -96,8 +111,7 @@ int __cgroup_bpf_run_filter_sock_ops(struct sock *sk,
struct cgroup_bpf {};
static inline void cgroup_bpf_put(struct cgroup *cgrp) {}
-static inline void cgroup_bpf_inherit(struct cgroup *cgrp,
- struct cgroup *parent) {}
+static inline int cgroup_bpf_inherit(struct cgroup *cgrp) { return 0; }
#define BPF_CGROUP_RUN_PROG_INET_INGRESS(sk,skb) ({ 0; })
#define BPF_CGROUP_RUN_PROG_INET_EGRESS(sk,skb) ({ 0; })
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 252f4bc9eb25..a6964b75f070 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -241,6 +241,38 @@ int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,
int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
union bpf_attr __user *uattr);
+/* an array of programs to be executed under rcu_lock.
+ *
+ * Typical usage:
+ * ret = BPF_PROG_RUN_ARRAY(&bpf_prog_array, ctx, BPF_PROG_RUN);
+ *
+ * the structure returned by bpf_prog_array_alloc() should be populated
+ * with program pointers and the last pointer must be NULL.
+ * The user has to keep refcnt on the program and make sure the program
+ * is removed from the array before bpf_prog_put().
+ * The 'struct bpf_prog_array *' should only be replaced with xchg()
+ * since other cpus are walking the array of pointers in parallel.
+ */
+struct bpf_prog_array {
+ struct rcu_head rcu;
+ struct bpf_prog *progs[0];
+};
+
+struct bpf_prog_array __rcu *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags);
+void bpf_prog_array_free(struct bpf_prog_array __rcu *progs);
+
+#define BPF_PROG_RUN_ARRAY(array, ctx, func) \
+ ({ \
+ struct bpf_prog **_prog; \
+ u32 _ret = 1; \
+ rcu_read_lock(); \
+ _prog = rcu_dereference(array)->progs; \
+ for (; *_prog; _prog++) \
+ _ret &= func(*_prog, ctx); \
+ rcu_read_unlock(); \
+ _ret; \
+ })
+
#ifdef CONFIG_BPF_SYSCALL
DECLARE_PER_CPU(int, bpf_prog_active);
diff --git a/include/linux/filter.h b/include/linux/filter.h
index 911d454af107..2d2db394b0ca 100644
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -481,7 +481,7 @@ struct sk_filter {
struct bpf_prog *prog;
};
-#define BPF_PROG_RUN(filter, ctx) (*filter->bpf_func)(ctx, filter->insnsi)
+#define BPF_PROG_RUN(filter, ctx) (*(filter)->bpf_func)(ctx, (filter)->insnsi)
#define BPF_SKB_CB_LEN QDISC_CB_PRIV_LEN
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 6d2137b4cf38..762f74bc6c47 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -143,11 +143,47 @@ enum bpf_attach_type {
#define MAX_BPF_ATTACH_TYPE __MAX_BPF_ATTACH_TYPE
-/* If BPF_F_ALLOW_OVERRIDE flag is used in BPF_PROG_ATTACH command
- * to the given target_fd cgroup the descendent cgroup will be able to
- * override effective bpf program that was inherited from this cgroup
+/* cgroup-bpf attach flags used in BPF_PROG_ATTACH command
+ *
+ * NONE(default): No further bpf programs allowed in the subtree.
+ *
+ * BPF_F_ALLOW_OVERRIDE: If a sub-cgroup installs some bpf program,
+ * the program in this cgroup yields to sub-cgroup program.
+ *
+ * BPF_F_ALLOW_MULTI: If a sub-cgroup installs some bpf program,
+ * that cgroup program gets run in addition to the program in this cgroup.
+ *
+ * Only one program is allowed to be attached to a cgroup with
+ * NONE or BPF_F_ALLOW_OVERRIDE flag.
+ * Attaching another program on top of NONE or BPF_F_ALLOW_OVERRIDE will
+ * release old program and attach the new one. Attach flags has to match.
+ *
+ * Multiple programs are allowed to be attached to a cgroup with
+ * BPF_F_ALLOW_MULTI flag. They are executed in FIFO order
+ * (those that were attached first, run first)
+ * The programs of sub-cgroup are executed first, then programs of
+ * this cgroup and then programs of parent cgroup.
+ * When children program makes decision (like picking TCP CA or sock bind)
+ * parent program has a chance to override it.
+ *
+ * A cgroup with MULTI or OVERRIDE flag allows any attach flags in sub-cgroups.
+ * A cgroup with NONE doesn't allow any programs in sub-cgroups.
+ * Ex1:
+ * cgrp1 (MULTI progs A, B) ->
+ * cgrp2 (OVERRIDE prog C) ->
+ * cgrp3 (MULTI prog D) ->
+ * cgrp4 (OVERRIDE prog E) ->
+ * cgrp5 (NONE prog F)
+ * the event in cgrp5 triggers execution of F,D,A,B in that order.
+ * if prog F is detached, the execution is E,D,A,B
+ * if prog F and D are detached, the execution is E,A,B
+ * if prog F, E and D are detached, the execution is C,A,B
+ *
+ * All eligible programs are executed regardless of return code from
+ * earlier programs.
*/
#define BPF_F_ALLOW_OVERRIDE (1U << 0)
+#define BPF_F_ALLOW_MULTI (1U << 1)
/* If BPF_F_STRICT_ALIGNMENT is used in BPF_PROG_LOAD command, the
* verifier will perform strict alignment checking as if the kernel
diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c
index 546113430049..6b7500bbdb53 100644
--- a/kernel/bpf/cgroup.c
+++ b/kernel/bpf/cgroup.c
@@ -27,129 +27,361 @@ void cgroup_bpf_put(struct cgroup *cgrp)
{
unsigned int type;
- for (type = 0; type < ARRAY_SIZE(cgrp->bpf.prog); type++) {
- struct bpf_prog *prog = cgrp->bpf.prog[type];
-
- if (prog) {
- bpf_prog_put(prog);
+ for (type = 0; type < ARRAY_SIZE(cgrp->bpf.progs); type++) {
+ struct list_head *progs = &cgrp->bpf.progs[type];
+ struct bpf_prog_list *pl, *tmp;
+
+ list_for_each_entry_safe(pl, tmp, progs, node) {
+ list_del(&pl->node);
+ bpf_prog_put(pl->prog);
+ kfree(pl);
static_branch_dec(&cgroup_bpf_enabled_key);
}
+ bpf_prog_array_free(cgrp->bpf.effective[type]);
+ }
+}
+
+/* count number of elements in the list.
+ * it's slow but the list cannot be long
+ */
+static u32 prog_list_length(struct list_head *head)
+{
+ struct bpf_prog_list *pl;
+ u32 cnt = 0;
+
+ list_for_each_entry(pl, head, node) {
+ if (!pl->prog)
+ continue;
+ cnt++;
}
+ return cnt;
+}
+
+/* if parent has non-overridable prog attached,
+ * disallow attaching new programs to the descendent cgroup.
+ * if parent has overridable or multi-prog, allow attaching
+ */
+static bool hierarchy_allows_attach(struct cgroup *cgrp,
+ enum bpf_attach_type type,
+ u32 new_flags)
+{
+ struct cgroup *p;
+
+ p = cgroup_parent(cgrp);
+ if (!p)
+ return true;
+ do {
+ u32 flags = p->bpf.flags[type];
+ u32 cnt;
+
+ if (flags & BPF_F_ALLOW_MULTI)
+ return true;
+ cnt = prog_list_length(&p->bpf.progs[type]);
+ WARN_ON_ONCE(cnt > 1);
+ if (cnt == 1)
+ return !!(flags & BPF_F_ALLOW_OVERRIDE);
+ p = cgroup_parent(p);
+ } while (p);
+ return true;
+}
+
+/* compute a chain of effective programs for a given cgroup:
+ * start from the list of programs in this cgroup and add
+ * all parent programs.
+ * Note that parent's F_ALLOW_OVERRIDE-type program is yielding
+ * to programs in this cgroup
+ */
+static int compute_effective_progs(struct cgroup *cgrp,
+ enum bpf_attach_type type,
+ struct bpf_prog_array __rcu **array)
+{
+ struct bpf_prog_array __rcu *progs;
+ struct bpf_prog_list *pl;
+ struct cgroup *p = cgrp;
+ int cnt = 0;
+
+ /* count number of effective programs by walking parents */
+ do {
+ if (cnt == 0 || (p->bpf.flags[type] & BPF_F_ALLOW_MULTI))
+ cnt += prog_list_length(&p->bpf.progs[type]);
+ p = cgroup_parent(p);
+ } while (p);
+
+ progs = bpf_prog_array_alloc(cnt, GFP_KERNEL);
+ if (!progs)
+ return -ENOMEM;
+
+ /* populate the array with effective progs */
+ cnt = 0;
+ p = cgrp;
+ do {
+ if (cnt == 0 || (p->bpf.flags[type] & BPF_F_ALLOW_MULTI))
+ list_for_each_entry(pl,
+ &p->bpf.progs[type], node) {
+ if (!pl->prog)
+ continue;
+ rcu_dereference_protected(progs, 1)->
+ progs[cnt++] = pl->prog;
+ }
+ p = cgroup_parent(p);
+ } while (p);
+
+ *array = progs;
+ return 0;
+}
+
+static void activate_effective_progs(struct cgroup *cgrp,
+ enum bpf_attach_type type,
+ struct bpf_prog_array __rcu *array)
+{
+ struct bpf_prog_array __rcu *old_array;
+
+ old_array = xchg(&cgrp->bpf.effective[type], array);
+ /* free prog array after grace period, since __cgroup_bpf_run_*()
+ * might be still walking the array
+ */
+ bpf_prog_array_free(old_array);
}
/**
* cgroup_bpf_inherit() - inherit effective programs from parent
* @cgrp: the cgroup to modify
- * @parent: the parent to inherit from
*/
-void cgroup_bpf_inherit(struct cgroup *cgrp, struct cgroup *parent)
+int cgroup_bpf_inherit(struct cgroup *cgrp)
{
- unsigned int type;
+/* has to use marco instead of const int, since compiler thinks
+ * that array below is variable length
+ */
+#define NR ARRAY_SIZE(cgrp->bpf.effective)
+ struct bpf_prog_array __rcu *arrays[NR] = {};
+ int i;
- for (type = 0; type < ARRAY_SIZE(cgrp->bpf.effective); type++) {
- struct bpf_prog *e;
+ for (i = 0; i < NR; i++)
+ INIT_LIST_HEAD(&cgrp->bpf.progs[i]);
- e = rcu_dereference_protected(parent->bpf.effective[type],
- lockdep_is_held(&cgroup_mutex));
- rcu_assign_pointer(cgrp->bpf.effective[type], e);
- cgrp->bpf.disallow_override[type] = parent->bpf.disallow_override[type];
- }
+ for (i = 0; i < NR; i++)
+ if (compute_effective_progs(cgrp, i, &arrays[i]))
+ goto cleanup;
+
+ for (i = 0; i < NR; i++)
+ activate_effective_progs(cgrp, i, arrays[i]);
+
+ return 0;
+cleanup:
+ for (i = 0; i < NR; i++)
+ bpf_prog_array_free(arrays[i]);
+ return -ENOMEM;
}
+#define BPF_CGROUP_MAX_PROGS 64
+
/**
- * __cgroup_bpf_update() - Update the pinned program of a cgroup, and
+ * __cgroup_bpf_attach() - Attach the program to a cgroup, and
* propagate the change to descendants
* @cgrp: The cgroup which descendants to traverse
- * @parent: The parent of @cgrp, or %NULL if @cgrp is the root
- * @prog: A new program to pin
- * @type: Type of pinning operation (ingress/egress)
- *
- * Each cgroup has a set of two pointers for bpf programs; one for eBPF
- * programs it owns, and which is effective for execution.
- *
- * If @prog is not %NULL, this function attaches a new program to the cgroup
- * and releases the one that is currently attached, if any. @prog is then made
- * the effective program of type @type in that cgroup.
- *
- * If @prog is %NULL, the currently attached program of type @type is released,
- * and the effective program of the parent cgroup (if any) is inherited to
- * @cgrp.
- *
- * Then, the descendants of @cgrp are walked and the effective program for
- * each of them is set to the effective program of @cgrp unless the
- * descendant has its own program attached, in which case the subbranch is
- * skipped. This ensures that delegated subcgroups with own programs are left
- * untouched.
+ * @prog: A program to attach
+ * @type: Type of attach operation
*
* Must be called with cgroup_mutex held.
*/
-int __cgroup_bpf_update(struct cgroup *cgrp, struct cgroup *parent,
- struct bpf_prog *prog, enum bpf_attach_type type,
- bool new_overridable)
+int __cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *prog,
+ enum bpf_attach_type type, u32 flags)
{
- struct bpf_prog *old_prog, *effective = NULL;
- struct cgroup_subsys_state *pos;
- bool overridable = true;
-
- if (parent) {
- overridable = !parent->bpf.disallow_override[type];
- effective = rcu_dereference_protected(parent->bpf.effective[type],
- lockdep_is_held(&cgroup_mutex));
- }
-
- if (prog && effective && !overridable)
- /* if parent has non-overridable prog attached, disallow
- * attaching new programs to descendent cgroup
- */
+ struct list_head *progs = &cgrp->bpf.progs[type];
+ struct bpf_prog *old_prog = NULL;
+ struct cgroup_subsys_state *css;
+ struct bpf_prog_list *pl;
+ bool pl_was_allocated;
+ u32 old_flags;
+ int err;
+
+ if ((flags & BPF_F_ALLOW_OVERRIDE) && (flags & BPF_F_ALLOW_MULTI))
+ /* invalid combination */
+ return -EINVAL;
+
+ if (!hierarchy_allows_attach(cgrp, type, flags))
return -EPERM;
- if (prog && effective && overridable != new_overridable)
- /* if parent has overridable prog attached, only
- * allow overridable programs in descendent cgroup
+ if (!list_empty(progs) && cgrp->bpf.flags[type] != flags)
+ /* Disallow attaching non-overridable on top
+ * of existing overridable in this cgroup.
+ * Disallow attaching multi-prog if overridable or none
*/
return -EPERM;
- old_prog = cgrp->bpf.prog[type];
-
- if (prog) {
- overridable = new_overridable;
- effective = prog;
- if (old_prog &&
- cgrp->bpf.disallow_override[type] == new_overridable)
- /* disallow attaching non-overridable on top
- * of existing overridable in this cgroup
- * and vice versa
- */
- return -EPERM;
+ if (prog_list_length(progs) >= BPF_CGROUP_MAX_PROGS)
+ return -E2BIG;
+
+ if (flags & BPF_F_ALLOW_MULTI) {
+ list_for_each_entry(pl, progs, node)
+ if (pl->prog == prog)
+ /* disallow attaching the same prog twice */
+ return -EINVAL;
+
+ pl = kmalloc(sizeof(*pl), GFP_KERNEL);
+ if (!pl)
+ return -ENOMEM;
+ pl_was_allocated = true;
+ pl->prog = prog;
+ list_add_tail(&pl->node, progs);
+ } else {
+ if (list_empty(progs)) {
+ pl = kmalloc(sizeof(*pl), GFP_KERNEL);
+ if (!pl)
+ return -ENOMEM;
+ pl_was_allocated = true;
+ list_add_tail(&pl->node, progs);
+ } else {
+ pl = list_first_entry(progs, typeof(*pl), node);
+ old_prog = pl->prog;
+ pl_was_allocated = false;
+ }
+ pl->prog = prog;
}
- if (!prog && !old_prog)
- /* report error when trying to detach and nothing is attached */
- return -ENOENT;
+ old_flags = cgrp->bpf.flags[type];
+ cgrp->bpf.flags[type] = flags;
- cgrp->bpf.prog[type] = prog;
+ /* allocate and recompute effective prog arrays */
+ css_for_each_descendant_pre(css, &cgrp->self) {
+ struct cgroup *desc = container_of(css, struct cgroup, self);
- css_for_each_descendant_pre(pos, &cgrp->self) {
- struct cgroup *desc = container_of(pos, struct cgroup, self);
-
- /* skip the subtree if the descendant has its own program */
- if (desc->bpf.prog[type] && desc != cgrp) {
- pos = css_rightmost_descendant(pos);
- } else {
- rcu_assign_pointer(desc->bpf.effective[type],
- effective);
- desc->bpf.disallow_override[type] = !overridable;
- }
+ err = compute_effective_progs(desc, type, &desc->bpf.inactive);
+ if (err)
+ goto cleanup;
}
- if (prog)
- static_branch_inc(&cgroup_bpf_enabled_key);
+ /* all allocations were successful. Activate all prog arrays */
+ css_for_each_descendant_pre(css, &cgrp->self) {
+ struct cgroup *desc = container_of(css, struct cgroup, self);
+ activate_effective_progs(desc, type, desc->bpf.inactive);
+ desc->bpf.inactive = NULL;
+ }
+
+ static_branch_inc(&cgroup_bpf_enabled_key);
if (old_prog) {
bpf_prog_put(old_prog);
static_branch_dec(&cgroup_bpf_enabled_key);
}
return 0;
+
+cleanup:
+ /* oom while computing effective. Free all computed effective arrays
+ * since they were not activated
+ */
+ css_for_each_descendant_pre(css, &cgrp->self) {
+ struct cgroup *desc = container_of(css, struct cgroup, self);
+
+ bpf_prog_array_free(desc->bpf.inactive);
+ desc->bpf.inactive = NULL;
+ }
+
+ /* and cleanup the prog list */
+ pl->prog = old_prog;
+ if (pl_was_allocated) {
+ list_del(&pl->node);
+ kfree(pl);
+ }
+ return err;
+}
+
+/**
+ * __cgroup_bpf_detach() - Detach the program from a cgroup, and
+ * propagate the change to descendants
+ * @cgrp: The cgroup which descendants to traverse
+ * @prog: A program to detach or NULL
+ * @type: Type of detach operation
+ *
+ * Must be called with cgroup_mutex held.
+ */
+int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
+ enum bpf_attach_type type, u32 unused_flags)
+{
+ struct list_head *progs = &cgrp->bpf.progs[type];
+ u32 flags = cgrp->bpf.flags[type];
+ struct bpf_prog *old_prog = NULL;
+ struct cgroup_subsys_state *css;
+ struct bpf_prog_list *pl;
+ int err;
+
+ if (flags & BPF_F_ALLOW_MULTI) {
+ if (!prog)
+ /* to detach MULTI prog the user has to specify valid FD
+ * of the program to be detached
+ */
+ return -EINVAL;
+ } else {
+ if (list_empty(progs))
+ /* report error when trying to detach and nothing is attached */
+ return -ENOENT;
+ }
+
+ if (flags & BPF_F_ALLOW_MULTI) {
+ /* find the prog and detach it */
+ list_for_each_entry(pl, progs, node) {
+ if (pl->prog != prog)
+ continue;
+ old_prog = prog;
+ /* mark it deleted, so it's ignored while
+ * recomputing effective
+ */
+ pl->prog = NULL;
+ break;
+ }
+ if (!old_prog)
+ return -ENOENT;
+ } else {
+ /* to maintain backward compatibility NONE and OVERRIDE cgroups
+ * allow detaching with invalid FD (prog==NULL)
+ */
+ pl = list_first_entry(progs, typeof(*pl), node);
+ old_prog = pl->prog;
+ pl->prog = NULL;
+ }
+
+ /* allocate and recompute effective prog arrays */
+ css_for_each_descendant_pre(css, &cgrp->self) {
+ struct cgroup *desc = container_of(css, struct cgroup, self);
+
+ err = compute_effective_progs(desc, type, &desc->bpf.inactive);
+ if (err)
+ goto cleanup;
+ }
+
+ /* all allocations were successful. Activate all prog arrays */
+ css_for_each_descendant_pre(css, &cgrp->self) {
+ struct cgroup *desc = container_of(css, struct cgroup, self);
+
+ activate_effective_progs(desc, type, desc->bpf.inactive);
+ desc->bpf.inactive = NULL;
+ }
+
+ /* now can actually delete it from this cgroup list */
+ list_del(&pl->node);
+ kfree(pl);
+ if (list_empty(progs))
+ /* last program was detached, reset flags to zero */
+ cgrp->bpf.flags[type] = 0;
+
+ bpf_prog_put(old_prog);
+ static_branch_dec(&cgroup_bpf_enabled_key);
+ return 0;
+
+cleanup:
+ /* oom while computing effective. Free all computed effective arrays
+ * since they were not activated
+ */
+ css_for_each_descendant_pre(css, &cgrp->self) {
+ struct cgroup *desc = container_of(css, struct cgroup, self);
+
+ bpf_prog_array_free(desc->bpf.inactive);
+ desc->bpf.inactive = NULL;
+ }
+
+ /* and restore back old_prog */
+ pl->prog = old_prog;
+ return err;
}
/**
@@ -171,36 +403,26 @@ int __cgroup_bpf_run_filter_skb(struct sock *sk,
struct sk_buff *skb,
enum bpf_attach_type type)
{
- struct bpf_prog *prog;
+ unsigned int offset = skb->data - skb_network_header(skb);
+ struct sock *save_sk;
struct cgroup *cgrp;
- int ret = 0;
+ int ret;
if (!sk || !sk_fullsock(sk))
return 0;
- if (sk->sk_family != AF_INET &&
- sk->sk_family != AF_INET6)
+ if (sk->sk_family != AF_INET && sk->sk_family != AF_INET6)
return 0;
cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
-
- rcu_read_lock();
-
- prog = rcu_dereference(cgrp->bpf.effective[type]);
- if (prog) {
- unsigned int offset = skb->data - skb_network_header(skb);
- struct sock *save_sk = skb->sk;
-
- skb->sk = sk;
- __skb_push(skb, offset);
- ret = bpf_prog_run_save_cb(prog, skb) == 1 ? 0 : -EPERM;
- __skb_pull(skb, offset);
- skb->sk = save_sk;
- }
-
- rcu_read_unlock();
-
- return ret;
+ save_sk = skb->sk;
+ skb->sk = sk;
+ __skb_push(skb, offset);
+ ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], skb,
+ bpf_prog_run_save_cb);
+ __skb_pull(skb, offset);
+ skb->sk = save_sk;
+ return ret == 1 ? 0 : -EPERM;
}
EXPORT_SYMBOL(__cgroup_bpf_run_filter_skb);
@@ -221,19 +443,10 @@ int __cgroup_bpf_run_filter_sk(struct sock *sk,
enum bpf_attach_type type)
{
struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
- struct bpf_prog *prog;
- int ret = 0;
-
-
- rcu_read_lock();
-
- prog = rcu_dereference(cgrp->bpf.effective[type]);
- if (prog)
- ret = BPF_PROG_RUN(prog, sk) == 1 ? 0 : -EPERM;
+ int ret;
- rcu_read_unlock();
-
- return ret;
+ ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], sk, BPF_PROG_RUN);
+ return ret == 1 ? 0 : -EPERM;
}
EXPORT_SYMBOL(__cgroup_bpf_run_filter_sk);
@@ -258,18 +471,10 @@ int __cgroup_bpf_run_filter_sock_ops(struct sock *sk,
enum bpf_attach_type type)
{
struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
- struct bpf_prog *prog;
- int ret = 0;
-
-
- rcu_read_lock();
-
- prog = rcu_dereference(cgrp->bpf.effective[type]);
- if (prog)
- ret = BPF_PROG_RUN(prog, sock_ops) == 1 ? 0 : -EPERM;
-
- rcu_read_unlock();
+ int ret;
- return ret;
+ ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], sock_ops,
+ BPF_PROG_RUN);
+ return ret == 1 ? 0 : -EPERM;
}
EXPORT_SYMBOL(__cgroup_bpf_run_filter_sock_ops);
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 917cc04a0a94..6b49e1991ae7 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -1381,6 +1381,37 @@ struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err)
}
EXPORT_SYMBOL_GPL(bpf_prog_select_runtime);
+/* to avoid allocating empty bpf_prog_array for cgroups that
+ * don't have bpf program attached use one global 'empty_prog_array'
+ * It will not be modified the caller of bpf_prog_array_alloc()
+ * (since caller requested prog_cnt == 0)
+ * that pointer should be 'freed' by bpf_prog_array_free()
+ */
+static struct {
+ struct bpf_prog_array hdr;
+ struct bpf_prog *null_prog;
+} empty_prog_array = {
+ .null_prog = NULL,
+};
+
+struct bpf_prog_array __rcu *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags)
+{
+ if (prog_cnt)
+ return kzalloc(sizeof(struct bpf_prog_array) +
+ sizeof(struct bpf_prog *) * (prog_cnt + 1),
+ flags);
+
+ return &empty_prog_array.hdr;
+}
+
+void bpf_prog_array_free(struct bpf_prog_array __rcu *progs)
+{
+ if (!progs ||
+ progs == (struct bpf_prog_array __rcu *)&empty_prog_array.hdr)
+ return;
+ kfree_rcu(progs, rcu);
+}
+
static void bpf_prog_free_deferred(struct work_struct *work)
{
struct bpf_prog_aux *aux;
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index b927da66f653..51bee695d32c 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -1168,6 +1168,9 @@ static int sockmap_get_from_fd(const union bpf_attr *attr, bool attach)
return 0;
}
+#define BPF_F_ATTACH_MASK \
+ (BPF_F_ALLOW_OVERRIDE | BPF_F_ALLOW_MULTI)
+
static int bpf_prog_attach(const union bpf_attr *attr)
{
enum bpf_prog_type ptype;
@@ -1181,7 +1184,7 @@ static int bpf_prog_attach(const union bpf_attr *attr)
if (CHECK_ATTR(BPF_PROG_ATTACH))
return -EINVAL;
- if (attr->attach_flags & ~BPF_F_ALLOW_OVERRIDE)
+ if (attr->attach_flags & ~BPF_F_ATTACH_MASK)
return -EINVAL;
switch (attr->attach_type) {
@@ -1212,8 +1215,8 @@ static int bpf_prog_attach(const union bpf_attr *attr)
return PTR_ERR(cgrp);
}
- ret = cgroup_bpf_update(cgrp, prog, attr->attach_type,
- attr->attach_flags & BPF_F_ALLOW_OVERRIDE);
+ ret = cgroup_bpf_attach(cgrp, prog, attr->attach_type,
+ attr->attach_flags);
if (ret)
bpf_prog_put(prog);
cgroup_put(cgrp);
@@ -1225,6 +1228,8 @@ static int bpf_prog_attach(const union bpf_attr *attr)
static int bpf_prog_detach(const union bpf_attr *attr)
{
+ enum bpf_prog_type ptype;
+ struct bpf_prog *prog;
struct cgroup *cgrp;
int ret;
@@ -1237,23 +1242,33 @@ static int bpf_prog_detach(const union bpf_attr *attr)
switch (attr->attach_type) {
case BPF_CGROUP_INET_INGRESS:
case BPF_CGROUP_INET_EGRESS:
+ ptype = BPF_PROG_TYPE_CGROUP_SKB;
+ break;
case BPF_CGROUP_INET_SOCK_CREATE:
+ ptype = BPF_PROG_TYPE_CGROUP_SOCK;
+ break;
case BPF_CGROUP_SOCK_OPS:
- cgrp = cgroup_get_from_fd(attr->target_fd);
- if (IS_ERR(cgrp))
- return PTR_ERR(cgrp);
-
- ret = cgroup_bpf_update(cgrp, NULL, attr->attach_type, false);
- cgroup_put(cgrp);
+ ptype = BPF_PROG_TYPE_SOCK_OPS;
break;
case BPF_SK_SKB_STREAM_PARSER:
case BPF_SK_SKB_STREAM_VERDICT:
- ret = sockmap_get_from_fd(attr, false);
- break;
+ return sockmap_get_from_fd(attr, false);
default:
return -EINVAL;
}
+ cgrp = cgroup_get_from_fd(attr->target_fd);
+ if (IS_ERR(cgrp))
+ return PTR_ERR(cgrp);
+
+ prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype);
+ if (IS_ERR(prog))
+ prog = NULL;
+
+ ret = cgroup_bpf_detach(cgrp, prog, attr->attach_type, 0);
+ if (prog)
+ bpf_prog_put(prog);
+ cgroup_put(cgrp);
return ret;
}
diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c
index d6551cd45238..57eb866ae78d 100644
--- a/kernel/cgroup/cgroup.c
+++ b/kernel/cgroup/cgroup.c
@@ -1896,6 +1896,9 @@ int cgroup_setup_root(struct cgroup_root *root, u16 ss_mask, int ref_flags)
if (ret)
goto destroy_root;
+ ret = cgroup_bpf_inherit(root_cgrp);
+ WARN_ON_ONCE(ret);
+
trace_cgroup_setup_root(root);
/*
@@ -4713,6 +4716,9 @@ static struct cgroup *cgroup_create(struct cgroup *parent)
cgrp->self.parent = &parent->self;
cgrp->root = root;
cgrp->level = level;
+ ret = cgroup_bpf_inherit(cgrp);
+ if (ret)
+ goto out_idr_free;
for (tcgrp = cgrp; tcgrp; tcgrp = cgroup_parent(tcgrp)) {
cgrp->ancestor_ids[tcgrp->level] = tcgrp->id;
@@ -4747,13 +4753,12 @@ static struct cgroup *cgroup_create(struct cgroup *parent)
if (!cgroup_on_dfl(cgrp))
cgrp->subtree_control = cgroup_control(cgrp);
- if (parent)
- cgroup_bpf_inherit(cgrp, parent);
-
cgroup_propagate_control(cgrp);
return cgrp;
+out_idr_free:
+ cgroup_idr_remove(&root->cgroup_idr, cgrp->id);
out_cancel_ref:
percpu_ref_exit(&cgrp->self.refcnt);
out_free_cgrp:
@@ -5736,14 +5741,23 @@ void cgroup_sk_free(struct sock_cgroup_data *skcd)
#endif /* CONFIG_SOCK_CGROUP_DATA */
#ifdef CONFIG_CGROUP_BPF
-int cgroup_bpf_update(struct cgroup *cgrp, struct bpf_prog *prog,
- enum bpf_attach_type type, bool overridable)
+int cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *prog,
+ enum bpf_attach_type type, u32 flags)
+{
+ int ret;
+
+ mutex_lock(&cgroup_mutex);
+ ret = __cgroup_bpf_attach(cgrp, prog, type, flags);
+ mutex_unlock(&cgroup_mutex);
+ return ret;
+}
+int cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
+ enum bpf_attach_type type, u32 flags)
{
- struct cgroup *parent = cgroup_parent(cgrp);
int ret;
mutex_lock(&cgroup_mutex);
- ret = __cgroup_bpf_update(cgrp, parent, prog, type, overridable);
+ ret = __cgroup_bpf_detach(cgrp, prog, type, flags);
mutex_unlock(&cgroup_mutex);
return ret;
}
--
2.9.5
^ permalink raw reply related
* [PATCH v2 net-next 7/8] libbpf: add support for BPF_PROG_QUERY
From: Alexei Starovoitov @ 2017-10-03 5:50 UTC (permalink / raw)
To: David S . Miller
Cc: Daniel Borkmann, Tejun Heo, David Ahern, netdev, kernel-team
In-Reply-To: <20171003055028.1294791-1-ast@fb.com>
add support for BPF_PROG_QUERY command to libbpf
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Acked-by: Daniel Borkmann <daniel@iogearbox.net>
---
tools/lib/bpf/bpf.c | 20 ++++++++++++++++++++
tools/lib/bpf/bpf.h | 3 ++-
2 files changed, 22 insertions(+), 1 deletion(-)
diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c
index d4b6ba8292ee..5128677e4117 100644
--- a/tools/lib/bpf/bpf.c
+++ b/tools/lib/bpf/bpf.c
@@ -303,6 +303,26 @@ int bpf_prog_detach2(int prog_fd, int target_fd, enum bpf_attach_type type)
return sys_bpf(BPF_PROG_DETACH, &attr, sizeof(attr));
}
+int bpf_prog_query(int target_fd, enum bpf_attach_type type, __u32 query_flags,
+ __u32 *attach_flags, __u32 *prog_ids, __u32 *prog_cnt)
+{
+ union bpf_attr attr;
+ int ret;
+
+ bzero(&attr, sizeof(attr));
+ attr.query.target_fd = target_fd;
+ attr.query.attach_type = type;
+ attr.query.query_flags = query_flags;
+ attr.query.prog_cnt = *prog_cnt;
+ attr.query.prog_ids = ptr_to_u64(prog_ids);
+
+ ret = sys_bpf(BPF_PROG_QUERY, &attr, sizeof(attr));
+ if (attach_flags)
+ *attach_flags = attr.query.attach_flags;
+ *prog_cnt = attr.query.prog_cnt;
+ return ret;
+}
+
int bpf_prog_test_run(int prog_fd, int repeat, void *data, __u32 size,
void *data_out, __u32 *size_out, __u32 *retval,
__u32 *duration)
diff --git a/tools/lib/bpf/bpf.h b/tools/lib/bpf/bpf.h
index afd64727c9cf..6534889e2b2f 100644
--- a/tools/lib/bpf/bpf.h
+++ b/tools/lib/bpf/bpf.h
@@ -75,5 +75,6 @@ int bpf_map_get_next_id(__u32 start_id, __u32 *next_id);
int bpf_prog_get_fd_by_id(__u32 id);
int bpf_map_get_fd_by_id(__u32 id);
int bpf_obj_get_info_by_fd(int prog_fd, void *info, __u32 *info_len);
-
+int bpf_prog_query(int target_fd, enum bpf_attach_type type, __u32 query_flags,
+ __u32 *attach_flags, __u32 *prog_ids, __u32 *prog_cnt);
#endif
--
2.9.5
^ permalink raw reply related
* [PATCH v2 net-next 5/8] samples/bpf: add multi-prog cgroup test case
From: Alexei Starovoitov @ 2017-10-03 5:50 UTC (permalink / raw)
To: David S . Miller
Cc: Daniel Borkmann, Tejun Heo, David Ahern, netdev, kernel-team
In-Reply-To: <20171003055028.1294791-1-ast@fb.com>
create 5 cgroups, attach 6 progs and check that progs are executed as:
cgrp1 (MULTI progs A, B) ->
cgrp2 (OVERRIDE prog C) ->
cgrp3 (MULTI prog D) ->
cgrp4 (OVERRIDE prog E) ->
cgrp5 (NONE prog F)
the event in cgrp5 triggers execution of F,D,A,B in that order.
if prog F is detached, the execution is E,D,A,B
if prog F and D are detached, the execution is E,A,B
if prog F, E and D are detached, the execution is C,A,B
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Acked-by: Daniel Borkmann <daniel@iogearbox.net>
---
samples/bpf/cgroup_helpers.c | 4 +-
samples/bpf/test_cgrp2_attach2.c | 188 +++++++++++++++++++++++++++++++++++++--
2 files changed, 185 insertions(+), 7 deletions(-)
diff --git a/samples/bpf/cgroup_helpers.c b/samples/bpf/cgroup_helpers.c
index 9d1be9426401..88bdcf4b1670 100644
--- a/samples/bpf/cgroup_helpers.c
+++ b/samples/bpf/cgroup_helpers.c
@@ -56,7 +56,7 @@ int setup_cgroup_environment(void)
return 1;
}
- if (mount("none", CGROUP_MOUNT_PATH, "cgroup2", 0, NULL)) {
+ if (mount("none", CGROUP_MOUNT_PATH, "cgroup2", 0, NULL) && errno != EBUSY) {
log_err("mount cgroup2");
return 1;
}
@@ -163,7 +163,7 @@ int create_and_get_cgroup(char *path)
format_cgroup_path(cgroup_path, path);
if (mkdir(cgroup_path, 0777) && errno != EEXIST) {
- log_err("mkdiring cgroup");
+ log_err("mkdiring cgroup %s .. %s", path, cgroup_path);
return 0;
}
diff --git a/samples/bpf/test_cgrp2_attach2.c b/samples/bpf/test_cgrp2_attach2.c
index 3049b1f26267..9a9f6836e5e9 100644
--- a/samples/bpf/test_cgrp2_attach2.c
+++ b/samples/bpf/test_cgrp2_attach2.c
@@ -30,7 +30,7 @@
#define FOO "/foo"
#define BAR "/foo/bar/"
-#define PING_CMD "ping -c1 -w1 127.0.0.1"
+#define PING_CMD "ping -c1 -w1 127.0.0.1 > /dev/null"
char bpf_log_buf[BPF_LOG_BUF_SIZE];
@@ -55,8 +55,7 @@ static int prog_load(int verdict)
return ret;
}
-
-int main(int argc, char **argv)
+static int test_foo_bar(void)
{
int drop_prog, allow_prog, foo = 0, bar = 0, rc = 0;
@@ -189,8 +188,187 @@ int main(int argc, char **argv)
close(bar);
cleanup_cgroup_environment();
if (!rc)
- printf("PASS\n");
+ printf("### override:PASS\n");
+ else
+ printf("### override:FAIL\n");
+ return rc;
+}
+
+static int map_fd = -1;
+
+static int prog_load_cnt(int verdict, int val)
+{
+ if (map_fd < 0)
+ map_fd = bpf_create_map(BPF_MAP_TYPE_ARRAY, 4, 8, 1, 0);
+ if (map_fd < 0) {
+ printf("failed to create map '%s'\n", strerror(errno));
+ return -1;
+ }
+
+ struct bpf_insn prog[] = {
+ BPF_MOV32_IMM(BPF_REG_0, 0),
+ BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_0, -4), /* *(u32 *)(fp - 4) = r0 */
+ BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
+ BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4), /* r2 = fp - 4 */
+ BPF_LD_MAP_FD(BPF_REG_1, map_fd),
+ BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem),
+ BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 2),
+ BPF_MOV64_IMM(BPF_REG_1, val), /* r1 = 1 */
+ BPF_RAW_INSN(BPF_STX | BPF_XADD | BPF_DW, BPF_REG_0, BPF_REG_1, 0, 0), /* xadd r0 += r1 */
+ BPF_MOV64_IMM(BPF_REG_0, verdict), /* r0 = verdict */
+ BPF_EXIT_INSN(),
+ };
+ size_t insns_cnt = sizeof(prog) / sizeof(struct bpf_insn);
+ int ret;
+
+ ret = bpf_load_program(BPF_PROG_TYPE_CGROUP_SKB,
+ prog, insns_cnt, "GPL", 0,
+ bpf_log_buf, BPF_LOG_BUF_SIZE);
+
+ if (ret < 0) {
+ log_err("Loading program");
+ printf("Output from verifier:\n%s\n-------\n", bpf_log_buf);
+ return 0;
+ }
+ return ret;
+}
+
+
+static int test_multiprog(void)
+{
+ int cg1 = 0, cg2 = 0, cg3 = 0, cg4 = 0, cg5 = 0, key = 0;
+ int drop_prog, allow_prog[6] = {}, rc = 0;
+ unsigned long long value;
+ int i = 0;
+
+ for (i = 0; i < 6; i++) {
+ allow_prog[i] = prog_load_cnt(1, 1 << i);
+ if (!allow_prog[i])
+ goto err;
+ }
+ drop_prog = prog_load_cnt(0, 1);
+ if (!drop_prog)
+ goto err;
+
+ if (setup_cgroup_environment())
+ goto err;
+
+ cg1 = create_and_get_cgroup("/cg1");
+ if (!cg1)
+ goto err;
+ cg2 = create_and_get_cgroup("/cg1/cg2");
+ if (!cg2)
+ goto err;
+ cg3 = create_and_get_cgroup("/cg1/cg2/cg3");
+ if (!cg3)
+ goto err;
+ cg4 = create_and_get_cgroup("/cg1/cg2/cg3/cg4");
+ if (!cg4)
+ goto err;
+ cg5 = create_and_get_cgroup("/cg1/cg2/cg3/cg4/cg5");
+ if (!cg5)
+ goto err;
+
+ if (join_cgroup("/cg1/cg2/cg3/cg4/cg5"))
+ goto err;
+
+ if (bpf_prog_attach(allow_prog[0], cg1, BPF_CGROUP_INET_EGRESS, 2)) {
+ log_err("Attaching prog to cg1");
+ goto err;
+ }
+ if (!bpf_prog_attach(allow_prog[0], cg1, BPF_CGROUP_INET_EGRESS, 2)) {
+ log_err("Unexpected success attaching the same prog to cg1");
+ goto err;
+ }
+ if (bpf_prog_attach(allow_prog[1], cg1, BPF_CGROUP_INET_EGRESS, 2)) {
+ log_err("Attaching prog2 to cg1");
+ goto err;
+ }
+ if (bpf_prog_attach(allow_prog[2], cg2, BPF_CGROUP_INET_EGRESS, 1)) {
+ log_err("Attaching prog to cg2");
+ goto err;
+ }
+ if (bpf_prog_attach(allow_prog[3], cg3, BPF_CGROUP_INET_EGRESS, 2)) {
+ log_err("Attaching prog to cg3");
+ goto err;
+ }
+ if (bpf_prog_attach(allow_prog[4], cg4, BPF_CGROUP_INET_EGRESS, 1)) {
+ log_err("Attaching prog to cg4");
+ goto err;
+ }
+ if (bpf_prog_attach(allow_prog[5], cg5, BPF_CGROUP_INET_EGRESS, 0)) {
+ log_err("Attaching prog to cg5");
+ goto err;
+ }
+ assert(system(PING_CMD) == 0);
+ assert(bpf_map_lookup_elem(map_fd, &key, &value) == 0);
+ assert(value == 1 + 2 + 8 + 32);
+
+ /* detach bottom program and ping again */
+ if (bpf_prog_detach2(-1, cg5, BPF_CGROUP_INET_EGRESS)) {
+ log_err("Detaching prog from cg5");
+ goto err;
+ }
+ value = 0;
+ assert(bpf_map_update_elem(map_fd, &key, &value, 0) == 0);
+ assert(system(PING_CMD) == 0);
+ assert(bpf_map_lookup_elem(map_fd, &key, &value) == 0);
+ assert(value == 1 + 2 + 8 + 16);
+
+ /* detach 3rd from bottom program and ping again */
+ errno = 0;
+ if (!bpf_prog_detach2(0, cg3, BPF_CGROUP_INET_EGRESS)) {
+ log_err("Unexpected success on detach from cg3");
+ goto err;
+ }
+ if (bpf_prog_detach2(allow_prog[3], cg3, BPF_CGROUP_INET_EGRESS)) {
+ log_err("Detaching from cg3");
+ goto err;
+ }
+ value = 0;
+ assert(bpf_map_update_elem(map_fd, &key, &value, 0) == 0);
+ assert(system(PING_CMD) == 0);
+ assert(bpf_map_lookup_elem(map_fd, &key, &value) == 0);
+ assert(value == 1 + 2 + 16);
+
+ /* detach 2nd from bottom program and ping again */
+ if (bpf_prog_detach2(-1, cg4, BPF_CGROUP_INET_EGRESS)) {
+ log_err("Detaching prog from cg4");
+ goto err;
+ }
+ value = 0;
+ assert(bpf_map_update_elem(map_fd, &key, &value, 0) == 0);
+ assert(system(PING_CMD) == 0);
+ assert(bpf_map_lookup_elem(map_fd, &key, &value) == 0);
+ assert(value == 1 + 2 + 4);
+ goto out;
+err:
+ rc = 1;
+
+out:
+ for (i = 0; i < 6; i++)
+ if (allow_prog[i] > 0)
+ close(allow_prog[i]);
+ close(cg1);
+ close(cg2);
+ close(cg3);
+ close(cg4);
+ close(cg5);
+ cleanup_cgroup_environment();
+ if (!rc)
+ printf("### multi:PASS\n");
else
- printf("FAIL\n");
+ printf("### multi:FAIL\n");
return rc;
}
+
+int main(int argc, char **argv)
+{
+ int rc = 0;
+
+ rc = test_foo_bar();
+ if (rc)
+ return rc;
+
+ return test_multiprog();
+}
--
2.9.5
^ permalink raw reply related
* [PATCH v2 net-next 4/8] libbpf: introduce bpf_prog_detach2()
From: Alexei Starovoitov @ 2017-10-03 5:50 UTC (permalink / raw)
To: David S . Miller
Cc: Daniel Borkmann, Tejun Heo, David Ahern, netdev, kernel-team
In-Reply-To: <20171003055028.1294791-1-ast@fb.com>
introduce bpf_prog_detach2() that takes one more argument prog_fd
vs bpf_prog_detach() that takes only attach_fd and type.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Acked-by: Daniel Borkmann <daniel@iogearbox.net>
---
tools/lib/bpf/bpf.c | 12 ++++++++++++
tools/lib/bpf/bpf.h | 1 +
2 files changed, 13 insertions(+)
diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c
index daf624e4c720..d4b6ba8292ee 100644
--- a/tools/lib/bpf/bpf.c
+++ b/tools/lib/bpf/bpf.c
@@ -291,6 +291,18 @@ int bpf_prog_detach(int target_fd, enum bpf_attach_type type)
return sys_bpf(BPF_PROG_DETACH, &attr, sizeof(attr));
}
+int bpf_prog_detach2(int prog_fd, int target_fd, enum bpf_attach_type type)
+{
+ union bpf_attr attr;
+
+ bzero(&attr, sizeof(attr));
+ attr.target_fd = target_fd;
+ attr.attach_bpf_fd = prog_fd;
+ attr.attach_type = type;
+
+ return sys_bpf(BPF_PROG_DETACH, &attr, sizeof(attr));
+}
+
int bpf_prog_test_run(int prog_fd, int repeat, void *data, __u32 size,
void *data_out, __u32 *size_out, __u32 *retval,
__u32 *duration)
diff --git a/tools/lib/bpf/bpf.h b/tools/lib/bpf/bpf.h
index 118d00535a0d..afd64727c9cf 100644
--- a/tools/lib/bpf/bpf.h
+++ b/tools/lib/bpf/bpf.h
@@ -66,6 +66,7 @@ int bpf_obj_get(const char *pathname);
int bpf_prog_attach(int prog_fd, int attachable_fd, enum bpf_attach_type type,
unsigned int flags);
int bpf_prog_detach(int attachable_fd, enum bpf_attach_type type);
+int bpf_prog_detach2(int prog_fd, int attachable_fd, enum bpf_attach_type type);
int bpf_prog_test_run(int prog_fd, int repeat, void *data, __u32 size,
void *data_out, __u32 *size_out, __u32 *retval,
__u32 *duration);
--
2.9.5
^ permalink raw reply related
* [PATCH v2 net-next 3/8] bpf: enforce return code for cgroup-bpf programs
From: Alexei Starovoitov @ 2017-10-03 5:50 UTC (permalink / raw)
To: David S . Miller
Cc: Daniel Borkmann, Tejun Heo, David Ahern, netdev, kernel-team
In-Reply-To: <20171003055028.1294791-1-ast@fb.com>
with addition of tnum logic the verifier got smart enough and
we can enforce return codes at program load time.
For now do so for cgroup-bpf program types.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Acked-by: Daniel Borkmann <daniel@iogearbox.net>
---
kernel/bpf/verifier.c | 40 ++++++++++++++++
tools/testing/selftests/bpf/test_verifier.c | 72 +++++++++++++++++++++++++++++
2 files changed, 112 insertions(+)
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 4cf9b72c59a0..52b022310f6a 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -3073,6 +3073,43 @@ static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn)
return 0;
}
+static int check_return_code(struct bpf_verifier_env *env)
+{
+ struct bpf_reg_state *reg;
+ struct tnum range = tnum_range(0, 1);
+
+ switch (env->prog->type) {
+ case BPF_PROG_TYPE_CGROUP_SKB:
+ case BPF_PROG_TYPE_CGROUP_SOCK:
+ case BPF_PROG_TYPE_SOCK_OPS:
+ break;
+ default:
+ return 0;
+ }
+
+ reg = &env->cur_state.regs[BPF_REG_0];
+ if (reg->type != SCALAR_VALUE) {
+ verbose("At program exit the register R0 is not a known value (%s)\n",
+ reg_type_str[reg->type]);
+ return -EINVAL;
+ }
+
+ if (!tnum_in(range, reg->var_off)) {
+ verbose("At program exit the register R0 ");
+ if (!tnum_is_unknown(reg->var_off)) {
+ char tn_buf[48];
+
+ tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
+ verbose("has value %s", tn_buf);
+ } else {
+ verbose("has unknown scalar value");
+ }
+ verbose(" should have been 0 or 1\n");
+ return -EINVAL;
+ }
+ return 0;
+}
+
/* non-recursive DFS pseudo code
* 1 procedure DFS-iterative(G,v):
* 2 label v as discovered
@@ -3863,6 +3900,9 @@ static int do_check(struct bpf_verifier_env *env)
return -EACCES;
}
+ err = check_return_code(env);
+ if (err)
+ return err;
process_bpf_exit:
insn_idx = pop_stack(env, &prev_insn_idx);
if (insn_idx < 0) {
diff --git a/tools/testing/selftests/bpf/test_verifier.c b/tools/testing/selftests/bpf/test_verifier.c
index 290d5056c165..cc91d0159f43 100644
--- a/tools/testing/selftests/bpf/test_verifier.c
+++ b/tools/testing/selftests/bpf/test_verifier.c
@@ -6892,6 +6892,78 @@ static struct bpf_test tests[] = {
.result = ACCEPT,
.prog_type = BPF_PROG_TYPE_XDP,
},
+ {
+ "bpf_exit with invalid return code. test1",
+ .insns = {
+ BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1, 0),
+ BPF_EXIT_INSN(),
+ },
+ .errstr = "R0 has value (0x0; 0xffffffff)",
+ .result = REJECT,
+ .prog_type = BPF_PROG_TYPE_CGROUP_SOCK,
+ },
+ {
+ "bpf_exit with invalid return code. test2",
+ .insns = {
+ BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1, 0),
+ BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 1),
+ BPF_EXIT_INSN(),
+ },
+ .result = ACCEPT,
+ .prog_type = BPF_PROG_TYPE_CGROUP_SOCK,
+ },
+ {
+ "bpf_exit with invalid return code. test3",
+ .insns = {
+ BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1, 0),
+ BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 3),
+ BPF_EXIT_INSN(),
+ },
+ .errstr = "R0 has value (0x0; 0x3)",
+ .result = REJECT,
+ .prog_type = BPF_PROG_TYPE_CGROUP_SOCK,
+ },
+ {
+ "bpf_exit with invalid return code. test4",
+ .insns = {
+ BPF_MOV64_IMM(BPF_REG_0, 1),
+ BPF_EXIT_INSN(),
+ },
+ .result = ACCEPT,
+ .prog_type = BPF_PROG_TYPE_CGROUP_SOCK,
+ },
+ {
+ "bpf_exit with invalid return code. test5",
+ .insns = {
+ BPF_MOV64_IMM(BPF_REG_0, 2),
+ BPF_EXIT_INSN(),
+ },
+ .errstr = "R0 has value (0x2; 0x0)",
+ .result = REJECT,
+ .prog_type = BPF_PROG_TYPE_CGROUP_SOCK,
+ },
+ {
+ "bpf_exit with invalid return code. test6",
+ .insns = {
+ BPF_MOV64_REG(BPF_REG_0, BPF_REG_1),
+ BPF_EXIT_INSN(),
+ },
+ .errstr = "R0 is not a known value (ctx)",
+ .result = REJECT,
+ .prog_type = BPF_PROG_TYPE_CGROUP_SOCK,
+ },
+ {
+ "bpf_exit with invalid return code. test7",
+ .insns = {
+ BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1, 0),
+ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, 4),
+ BPF_ALU64_REG(BPF_MUL, BPF_REG_0, BPF_REG_2),
+ BPF_EXIT_INSN(),
+ },
+ .errstr = "R0 has unknown scalar value",
+ .result = REJECT,
+ .prog_type = BPF_PROG_TYPE_CGROUP_SOCK,
+ },
};
static int probe_filter_length(const struct bpf_insn *fp)
--
2.9.5
^ permalink raw reply related
* [PATCH v2 net-next 6/8] libbpf: sync bpf.h
From: Alexei Starovoitov @ 2017-10-03 5:50 UTC (permalink / raw)
To: David S . Miller
Cc: Daniel Borkmann, Tejun Heo, David Ahern, netdev, kernel-team
In-Reply-To: <20171003055028.1294791-1-ast@fb.com>
tools/include/uapi/linux/bpf.h got out of sync with actual kernel header.
Update it.
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Acked-by: Daniel Borkmann <daniel@iogearbox.net>
---
tools/include/uapi/linux/bpf.h | 55 +++++++++++++++++++++++++++++++++++++++---
1 file changed, 52 insertions(+), 3 deletions(-)
diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index 6d2137b4cf38..cb2b9f95160a 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -92,6 +92,7 @@ enum bpf_cmd {
BPF_PROG_GET_FD_BY_ID,
BPF_MAP_GET_FD_BY_ID,
BPF_OBJ_GET_INFO_BY_FD,
+ BPF_PROG_QUERY,
};
enum bpf_map_type {
@@ -143,11 +144,47 @@ enum bpf_attach_type {
#define MAX_BPF_ATTACH_TYPE __MAX_BPF_ATTACH_TYPE
-/* If BPF_F_ALLOW_OVERRIDE flag is used in BPF_PROG_ATTACH command
- * to the given target_fd cgroup the descendent cgroup will be able to
- * override effective bpf program that was inherited from this cgroup
+/* cgroup-bpf attach flags used in BPF_PROG_ATTACH command
+ *
+ * NONE(default): No further bpf programs allowed in the subtree.
+ *
+ * BPF_F_ALLOW_OVERRIDE: If a sub-cgroup installs some bpf program,
+ * the program in this cgroup yields to sub-cgroup program.
+ *
+ * BPF_F_ALLOW_MULTI: If a sub-cgroup installs some bpf program,
+ * that cgroup program gets run in addition to the program in this cgroup.
+ *
+ * Only one program is allowed to be attached to a cgroup with
+ * NONE or BPF_F_ALLOW_OVERRIDE flag.
+ * Attaching another program on top of NONE or BPF_F_ALLOW_OVERRIDE will
+ * release old program and attach the new one. Attach flags has to match.
+ *
+ * Multiple programs are allowed to be attached to a cgroup with
+ * BPF_F_ALLOW_MULTI flag. They are executed in FIFO order
+ * (those that were attached first, run first)
+ * The programs of sub-cgroup are executed first, then programs of
+ * this cgroup and then programs of parent cgroup.
+ * When children program makes decision (like picking TCP CA or sock bind)
+ * parent program has a chance to override it.
+ *
+ * A cgroup with MULTI or OVERRIDE flag allows any attach flags in sub-cgroups.
+ * A cgroup with NONE doesn't allow any programs in sub-cgroups.
+ * Ex1:
+ * cgrp1 (MULTI progs A, B) ->
+ * cgrp2 (OVERRIDE prog C) ->
+ * cgrp3 (MULTI prog D) ->
+ * cgrp4 (OVERRIDE prog E) ->
+ * cgrp5 (NONE prog F)
+ * the event in cgrp5 triggers execution of F,D,A,B in that order.
+ * if prog F is detached, the execution is E,D,A,B
+ * if prog F and D are detached, the execution is E,A,B
+ * if prog F, E and D are detached, the execution is C,A,B
+ *
+ * All eligible programs are executed regardless of return code from
+ * earlier programs.
*/
#define BPF_F_ALLOW_OVERRIDE (1U << 0)
+#define BPF_F_ALLOW_MULTI (1U << 1)
/* If BPF_F_STRICT_ALIGNMENT is used in BPF_PROG_LOAD command, the
* verifier will perform strict alignment checking as if the kernel
@@ -175,6 +212,9 @@ enum bpf_attach_type {
/* Specify numa node during map creation */
#define BPF_F_NUMA_NODE (1U << 2)
+/* flags for BPF_PROG_QUERY */
+#define BPF_F_QUERY_EFFECTIVE (1U << 0)
+
#define BPF_OBJ_NAME_LEN 16U
union bpf_attr {
@@ -253,6 +293,15 @@ union bpf_attr {
__u32 info_len;
__aligned_u64 info;
} info;
+
+ struct { /* anonymous struct used by BPF_PROG_QUERY command */
+ __u32 target_fd; /* container object to query */
+ __u32 attach_type;
+ __u32 query_flags;
+ __u32 attach_flags;
+ __aligned_u64 prog_ids;
+ __u32 prog_cnt;
+ } query;
} __attribute__((aligned(8)));
/* BPF helper function descriptions:
--
2.9.5
^ permalink raw reply related
* [PATCH v2 net-next 2/8] bpf: introduce BPF_PROG_QUERY command
From: Alexei Starovoitov @ 2017-10-03 5:50 UTC (permalink / raw)
To: David S . Miller
Cc: Daniel Borkmann, Tejun Heo, David Ahern, netdev, kernel-team
In-Reply-To: <20171003055028.1294791-1-ast@fb.com>
introduce BPF_PROG_QUERY command to retrieve a set of either
attached programs to given cgroup or a set of effective programs
that will execute for events within a cgroup
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Acked-by: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Martin KaFai Lau <kafai@fb.com>
for cgroup bits
Acked-by: Tejun Heo <tj@kernel.org>
---
include/linux/bpf-cgroup.h | 4 ++++
include/linux/bpf.h | 3 +++
include/uapi/linux/bpf.h | 13 +++++++++++++
kernel/bpf/cgroup.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++
kernel/bpf/core.c | 38 ++++++++++++++++++++++++++++++++++++++
kernel/bpf/syscall.c | 34 ++++++++++++++++++++++++++++++++++
kernel/cgroup/cgroup.c | 10 ++++++++++
7 files changed, 148 insertions(+)
diff --git a/include/linux/bpf-cgroup.h b/include/linux/bpf-cgroup.h
index 102e56fbb6de..359b6f5d3d90 100644
--- a/include/linux/bpf-cgroup.h
+++ b/include/linux/bpf-cgroup.h
@@ -44,12 +44,16 @@ int __cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *prog,
enum bpf_attach_type type, u32 flags);
int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
enum bpf_attach_type type, u32 flags);
+int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
+ union bpf_attr __user *uattr);
/* Wrapper for __cgroup_bpf_*() protected by cgroup_mutex */
int cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *prog,
enum bpf_attach_type type, u32 flags);
int cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
enum bpf_attach_type type, u32 flags);
+int cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
+ union bpf_attr __user *uattr);
int __cgroup_bpf_run_filter_skb(struct sock *sk,
struct sk_buff *skb,
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index a6964b75f070..a67daea731ab 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -260,6 +260,9 @@ struct bpf_prog_array {
struct bpf_prog_array __rcu *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags);
void bpf_prog_array_free(struct bpf_prog_array __rcu *progs);
+int bpf_prog_array_length(struct bpf_prog_array __rcu *progs);
+int bpf_prog_array_copy_to_user(struct bpf_prog_array __rcu *progs,
+ __u32 __user *prog_ids, u32 cnt);
#define BPF_PROG_RUN_ARRAY(array, ctx, func) \
({ \
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 762f74bc6c47..cb2b9f95160a 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -92,6 +92,7 @@ enum bpf_cmd {
BPF_PROG_GET_FD_BY_ID,
BPF_MAP_GET_FD_BY_ID,
BPF_OBJ_GET_INFO_BY_FD,
+ BPF_PROG_QUERY,
};
enum bpf_map_type {
@@ -211,6 +212,9 @@ enum bpf_attach_type {
/* Specify numa node during map creation */
#define BPF_F_NUMA_NODE (1U << 2)
+/* flags for BPF_PROG_QUERY */
+#define BPF_F_QUERY_EFFECTIVE (1U << 0)
+
#define BPF_OBJ_NAME_LEN 16U
union bpf_attr {
@@ -289,6 +293,15 @@ union bpf_attr {
__u32 info_len;
__aligned_u64 info;
} info;
+
+ struct { /* anonymous struct used by BPF_PROG_QUERY command */
+ __u32 target_fd; /* container object to query */
+ __u32 attach_type;
+ __u32 query_flags;
+ __u32 attach_flags;
+ __aligned_u64 prog_ids;
+ __u32 prog_cnt;
+ } query;
} __attribute__((aligned(8)));
/* BPF helper function descriptions:
diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c
index 6b7500bbdb53..e88abc0865d5 100644
--- a/kernel/bpf/cgroup.c
+++ b/kernel/bpf/cgroup.c
@@ -384,6 +384,52 @@ int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
return err;
}
+/* Must be called with cgroup_mutex held to avoid races. */
+int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
+ union bpf_attr __user *uattr)
+{
+ __u32 __user *prog_ids = u64_to_user_ptr(attr->query.prog_ids);
+ enum bpf_attach_type type = attr->query.attach_type;
+ struct list_head *progs = &cgrp->bpf.progs[type];
+ u32 flags = cgrp->bpf.flags[type];
+ int cnt, ret = 0, i;
+
+ if (attr->query.query_flags & BPF_F_QUERY_EFFECTIVE)
+ cnt = bpf_prog_array_length(cgrp->bpf.effective[type]);
+ else
+ cnt = prog_list_length(progs);
+
+ if (copy_to_user(&uattr->query.attach_flags, &flags, sizeof(flags)))
+ return -EFAULT;
+ if (copy_to_user(&uattr->query.prog_cnt, &cnt, sizeof(cnt)))
+ return -EFAULT;
+ if (attr->query.prog_cnt == 0 || !prog_ids || !cnt)
+ /* return early if user requested only program count + flags */
+ return 0;
+ if (attr->query.prog_cnt < cnt) {
+ cnt = attr->query.prog_cnt;
+ ret = -ENOSPC;
+ }
+
+ if (attr->query.query_flags & BPF_F_QUERY_EFFECTIVE) {
+ return bpf_prog_array_copy_to_user(cgrp->bpf.effective[type],
+ prog_ids, cnt);
+ } else {
+ struct bpf_prog_list *pl;
+ u32 id;
+
+ i = 0;
+ list_for_each_entry(pl, progs, node) {
+ id = pl->prog->aux->id;
+ if (copy_to_user(prog_ids + i, &id, sizeof(id)))
+ return -EFAULT;
+ if (++i == cnt)
+ break;
+ }
+ }
+ return ret;
+}
+
/**
* __cgroup_bpf_run_filter_skb() - Run a program for packet filtering
* @sk: The socket sending or receiving traffic
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 6b49e1991ae7..eba966c09053 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -1412,6 +1412,44 @@ void bpf_prog_array_free(struct bpf_prog_array __rcu *progs)
kfree_rcu(progs, rcu);
}
+int bpf_prog_array_length(struct bpf_prog_array __rcu *progs)
+{
+ struct bpf_prog **prog;
+ u32 cnt = 0;
+
+ rcu_read_lock();
+ prog = rcu_dereference(progs)->progs;
+ for (; *prog; prog++)
+ cnt++;
+ rcu_read_unlock();
+ return cnt;
+}
+
+int bpf_prog_array_copy_to_user(struct bpf_prog_array __rcu *progs,
+ __u32 __user *prog_ids, u32 cnt)
+{
+ struct bpf_prog **prog;
+ u32 i = 0, id;
+
+ rcu_read_lock();
+ prog = rcu_dereference(progs)->progs;
+ for (; *prog; prog++) {
+ id = (*prog)->aux->id;
+ if (copy_to_user(prog_ids + i, &id, sizeof(id))) {
+ rcu_read_unlock();
+ return -EFAULT;
+ }
+ if (++i == cnt) {
+ prog++;
+ break;
+ }
+ }
+ rcu_read_unlock();
+ if (*prog)
+ return -ENOSPC;
+ return 0;
+}
+
static void bpf_prog_free_deferred(struct work_struct *work)
{
struct bpf_prog_aux *aux;
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 51bee695d32c..0048cb24ba7b 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -1272,6 +1272,37 @@ static int bpf_prog_detach(const union bpf_attr *attr)
return ret;
}
+#define BPF_PROG_QUERY_LAST_FIELD query.prog_cnt
+
+static int bpf_prog_query(const union bpf_attr *attr,
+ union bpf_attr __user *uattr)
+{
+ struct cgroup *cgrp;
+ int ret;
+
+ if (!capable(CAP_NET_ADMIN))
+ return -EPERM;
+ if (CHECK_ATTR(BPF_PROG_QUERY))
+ return -EINVAL;
+ if (attr->query.query_flags & ~BPF_F_QUERY_EFFECTIVE)
+ return -EINVAL;
+
+ switch (attr->query.attach_type) {
+ case BPF_CGROUP_INET_INGRESS:
+ case BPF_CGROUP_INET_EGRESS:
+ case BPF_CGROUP_INET_SOCK_CREATE:
+ case BPF_CGROUP_SOCK_OPS:
+ break;
+ default:
+ return -EINVAL;
+ }
+ cgrp = cgroup_get_from_fd(attr->query.target_fd);
+ if (IS_ERR(cgrp))
+ return PTR_ERR(cgrp);
+ ret = cgroup_bpf_query(cgrp, attr, uattr);
+ cgroup_put(cgrp);
+ return ret;
+}
#endif /* CONFIG_CGROUP_BPF */
#define BPF_PROG_TEST_RUN_LAST_FIELD test.duration
@@ -1568,6 +1599,9 @@ SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, siz
case BPF_PROG_DETACH:
err = bpf_prog_detach(&attr);
break;
+ case BPF_PROG_QUERY:
+ err = bpf_prog_query(&attr, uattr);
+ break;
#endif
case BPF_PROG_TEST_RUN:
err = bpf_prog_test_run(&attr, uattr);
diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c
index 57eb866ae78d..269512b94a94 100644
--- a/kernel/cgroup/cgroup.c
+++ b/kernel/cgroup/cgroup.c
@@ -5761,4 +5761,14 @@ int cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
mutex_unlock(&cgroup_mutex);
return ret;
}
+int cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
+ union bpf_attr __user *uattr)
+{
+ int ret;
+
+ mutex_lock(&cgroup_mutex);
+ ret = __cgroup_bpf_query(cgrp, attr, uattr);
+ mutex_unlock(&cgroup_mutex);
+ return ret;
+}
#endif /* CONFIG_CGROUP_BPF */
--
2.9.5
^ permalink raw reply related
* [PATCH v2 net-next 0/8] bpf: muli prog support for cgroup-bpf
From: Alexei Starovoitov @ 2017-10-03 5:50 UTC (permalink / raw)
To: David S . Miller
Cc: Daniel Borkmann, Tejun Heo, David Ahern, netdev, kernel-team
v1->v2:
- fixed accidentally swapped two lines which caused static_key not going to zero
- addressed Martin's feedback and changed prog_query to be consistent
with verifier output: return -enospc and fill supplied buffer instead
of just returning -enospc when buffer is too small to fit all prog_ids
v1:
cgroup-bpf use cases are getting more advanced and running only
one program per cgroup is no longer enough. Therefore introduce
support for attaching multiple programs per cgroup and running
a set of effective programs.
These patches introduces BPF_F_ALLOW_MULTI flag for BPF_PROG_ATTACH cmd.
The default is still NONE and behavior of BPF_F_ALLOW_OVERRIDE flag
is unchanged.
The difference between three possible flags for BPF_PROG_ATTACH command:
- NONE(default): No further bpf programs allowed in the subtree.
- BPF_F_ALLOW_OVERRIDE: If a sub-cgroup installs some bpf program,
the program in this cgroup yields to sub-cgroup program.
- BPF_F_ALLOW_MULTI: If a sub-cgroup installs some bpf program,
that cgroup program gets run in addition to the program in this cgroup.
Most of the logic is in patch 1. Even when cgroup doesn't have
any programs attached its set of effective program can be non-empty.
To quickly execute them and avoid penalizing cgroups without
any effective programs introduce 'struct bpf_prog_array'
which has an optimization for cgroups with zero effective programs.
Patch 2 introduces BPF_PROG_QUERY command for introspection
Patch 3 makes verifier more strict for cgroup-bpf program types.
Patch 4+ are tests.
More details in individual patches
Alexei Starovoitov (8):
bpf: multi program support for cgroup+bpf
bpf: introduce BPF_PROG_QUERY command
bpf: enforce return code for cgroup-bpf programs
libbpf: introduce bpf_prog_detach2()
samples/bpf: add multi-prog cgroup test case
libbpf: sync bpf.h
libbpf: add support for BPF_PROG_QUERY
samples/bpf: use bpf_prog_query() interface
include/linux/bpf-cgroup.h | 54 ++-
include/linux/bpf.h | 35 ++
include/linux/filter.h | 2 +-
include/uapi/linux/bpf.h | 55 ++-
kernel/bpf/cgroup.c | 513 +++++++++++++++++++++-------
kernel/bpf/core.c | 69 ++++
kernel/bpf/syscall.c | 71 +++-
kernel/bpf/verifier.c | 40 +++
kernel/cgroup/cgroup.c | 38 ++-
samples/bpf/cgroup_helpers.c | 4 +-
samples/bpf/test_cgrp2_attach2.c | 224 +++++++++++-
tools/include/uapi/linux/bpf.h | 55 ++-
tools/lib/bpf/bpf.c | 32 ++
tools/lib/bpf/bpf.h | 4 +-
tools/testing/selftests/bpf/test_verifier.c | 72 ++++
15 files changed, 1086 insertions(+), 182 deletions(-)
--
2.9.5
^ permalink raw reply
* [PATCH net-next] cxgb4: add new T6 pci device id's
From: Ganesh Goudar @ 2017-10-03 5:40 UTC (permalink / raw)
To: netdev, davem; +Cc: nirranjan, indranil, venkatesh, Ganesh Goudar
Add 0x6085 T6 device id.
Signed-off-by: Ganesh Goudar <ganeshgr@chelsio.com>
---
drivers/net/ethernet/chelsio/cxgb4/t4_pci_id_tbl.h | 1 +
1 file changed, 1 insertion(+)
diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4_pci_id_tbl.h b/drivers/net/ethernet/chelsio/cxgb4/t4_pci_id_tbl.h
index 37d90d6..633e975 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/t4_pci_id_tbl.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/t4_pci_id_tbl.h
@@ -202,6 +202,7 @@ CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN
CH_PCI_ID_TABLE_FENTRY(0x6082), /* Custom T6225-CR SFP28 */
CH_PCI_ID_TABLE_FENTRY(0x6083), /* Custom T62100-CR QSFP28 */
CH_PCI_ID_TABLE_FENTRY(0x6084), /* Custom T64100-CR QSFP28 */
+ CH_PCI_ID_TABLE_FENTRY(0x6085), /* Custom T6240-SO */
CH_PCI_DEVICE_ID_TABLE_DEFINE_END;
#endif /* __T4_PCI_ID_TBL_H__ */
--
2.1.0
^ permalink raw reply related
* Re: [PATCH V2] Fix a sleep-in-atomic bug in shash_setkey_unaligned
From: Herbert Xu @ 2017-10-03 5:26 UTC (permalink / raw)
To: Andy Lutomirski
Cc: Jia-Ju Bai, David S. Miller, Neil Horman,
vyasevich-Re5JQEeQqe8AvxtiuMwx3w, Kalle Valo,
Linux Crypto Mailing List, Network Development,
linux-sctp-u79uwXL29TY76Z2rM5mHXA, Linux Wireless List
In-Reply-To: <CALCETrWdXjTTTywbb3duCEsLYNxkeGx7bf3SM4PYKeErCyiUNQ-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org>
On Mon, Oct 02, 2017 at 09:18:24PM -0700, Andy Lutomirski wrote:
> > On Oct 2, 2017, at 7:25 PM, Jia-Ju Bai <baijiaju1990-9Onoh4P/yGk@public.gmane.org> wrote:
> >
> > The SCTP program may sleep under a spinlock, and the function call path is:
> > sctp_generate_t3_rtx_event (acquire the spinlock)
> > sctp_do_sm
> > sctp_side_effects
> > sctp_cmd_interpreter
> > sctp_make_init_ack
> > sctp_pack_cookie
> > crypto_shash_setkey
> > shash_setkey_unaligned
> > kmalloc(GFP_KERNEL)
>
> I'm going to go out on a limb here: why on Earth is out crypto API so
> full of indirection that we allocate memory at all here?
The crypto API operates on a one key per-tfm basis. So normally
tfm allocation and key setting is done once only and not done on
the data path.
I have looked at the SCTP code and it appears to fit this paradigm.
That is, we should be able to allocate the tfm and set the key when
the key is actually generated via get_random_bytes, rather than every
time the key is used which is not only a waste but as you see runs
into API issues.
Usually if you're invoking setkey from a non-sleeping code-path
you're probably doing something wrong.
As someone else noted recently, there is no single forum for
reviewing code that uses the crypto API so buggy code like this
is not surprising.
> We're synchronously computing a hash of a small amount of data using
> either HMAC-SHA1 or HMAC-SHA256 (determined at runtime) if I read it
> right. There's a sane way to do this that doesn't need kmalloc,
> alloca, or fancy indirection. And then there's crypto_shash_xyz().
There are some legitimate cases where you want to use a different
key for every hashing operation. But so far these are uses have
been very few so there has been no need to provide an API for them.
Cheers,
--
Email: Herbert Xu <herbert-lOAM2aK0SrRLBo1qDEOMRrpzq4S04n8Q@public.gmane.org>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
^ permalink raw reply
* RE: [PATCH v2 3/6] staging: fsl-dpaa2/ethsw: Add ethtool support
From: Razvan Stefanescu @ 2017-10-03 5:22 UTC (permalink / raw)
To: Andrew Lunn
Cc: devel@driverdev.osuosl.org, arnd@arndb.de,
gregkh@linuxfoundation.org, Alexandru Marginean, agraf@suse.de,
linux-kernel@vger.kernel.org, stuyoder@gmail.com,
netdev@vger.kernel.org, Bogdan Purcareata, Laurentiu Tudor
In-Reply-To: <20171002153718.GJ17713@lunn.ch>
> -----Original Message-----
> From: Andrew Lunn [mailto:andrew@lunn.ch]
> Sent: Monday, October 02, 2017 18:37
> To: Razvan Stefanescu <razvan.stefanescu@nxp.com>
> Cc: gregkh@linuxfoundation.org; devel@driverdev.osuosl.org; linux-
> kernel@vger.kernel.org; netdev@vger.kernel.org; agraf@suse.de;
> arnd@arndb.de; Alexandru Marginean <alexandru.marginean@nxp.com>;
> Bogdan Purcareata <bogdan.purcareata@nxp.com>; Ruxandra Ioana Radulescu
> <ruxandra.radulescu@nxp.com>; Laurentiu Tudor <laurentiu.tudor@nxp.com>;
> stuyoder@gmail.com
> Subject: Re: [PATCH v2 3/6] staging: fsl-dpaa2/ethsw: Add ethtool support
>
> Hi Razvan
>
> > +static void ethsw_get_drvinfo(struct net_device *netdev,
> > + struct ethtool_drvinfo *drvinfo)
> > +{
> > + struct ethsw_port_priv *port_priv = netdev_priv(netdev);
> > + u16 version_major, version_minor;
> > + int err;
> > +
> > + strlcpy(drvinfo->driver, KBUILD_MODNAME, sizeof(drvinfo->driver));
> > + strlcpy(drvinfo->version, ethsw_drv_version, sizeof(drvinfo->version));
>
> Software driver versions are mostly useless. I would suggest you
> remove this.
>
> Andrew
Thank you. I'll remove it in v3.
Best regards,
Razvan S.
^ permalink raw reply
* Re: [PATCH net-next 1/8] bpf: multi program support for cgroup+bpf
From: Alexei Starovoitov @ 2017-10-03 5:00 UTC (permalink / raw)
To: David Ahern, David S . Miller
Cc: Daniel Borkmann, Tejun Heo, netdev, kernel-team
In-Reply-To: <1211d4c0-b812-af08-cbd4-72eb9259ee78@cumulusnetworks.com>
On 10/2/17 9:26 PM, David Ahern wrote:
> On 10/2/17 9:21 PM, Alexei Starovoitov wrote:
>>
>> i'm not sure what you're trying to say.
>> The first loop quoted above is inside cgroup_bpf_put()
>> which is called when cgroup is destroyed. At this point
>> we're detaching and prog_put all attached programs.
>> While there is only one static_branch_inc() in __cgroup_bpf_attach()
>> that is called every time the prog is attached to a cgroup.
>> So what's the concern?
>
> just asking if cgroup_bpf_enabled_key is 0 when all programs are removed
> -- ie., that the inc's and dec's are equal. Reviewing this patch it was
> not clear that they are.
after some debugging turned out there is a typo in attach code
that leaks prog in the case of override.
Strangely kmemleak didn't catch it.
Will respin.
^ permalink raw reply
* Re: [PATCH net] net: br: Fix igmp snooping offload with CONFIG_BRIDGE_VLAN_FILTERING
From: Toshiaki Makita @ 2017-10-03 3:29 UTC (permalink / raw)
To: Andrew Lunn; +Cc: David Miller, Vivien Didelot, netdev
In-Reply-To: <1506992111-25004-1-git-send-email-andrew@lunn.ch>
On 2017/10/03 9:55, Andrew Lunn wrote:
> With CONFIG_BRIDGE_VLAN_FILTERING enabled, but the feature not enabled
> via /sys/class/net/brX/bridge/vlan_filtering, mdb offloaded to the
> kernel have the wrong VID.
>
> When an interface is added to the bridge, switchdev is first used to
> notify the hardware that a port has joined a bridge. This is
> immediately followed by the default_pvid, 1, being added to the
> interface via another switchdev call.
>
> The bridge will then perform IGMP snooping, and offload an mdb entries
> to the switch as needed. With vlan filtering disabled, the vid is left
> as 0. This causes the switch to put the static mdb into the wrong
> vlan, and so frames are not forwarded by the mdb entry.
>
> If vlan filtering is disable, use the default_pvid, not 0.
>
> Fixes: f1fecb1d10ec ("bridge: Reflect MDB entries to hardware")
> Signed-off-by: Andrew Lunn <andrew@lunn.ch>
> ---
> net/bridge/br_vlan.c | 1 +
> 1 file changed, 1 insertion(+)
>
> diff --git a/net/bridge/br_vlan.c b/net/bridge/br_vlan.c
> index 233a30040c91..aa3589891797 100644
> --- a/net/bridge/br_vlan.c
> +++ b/net/bridge/br_vlan.c
> @@ -492,6 +492,7 @@ bool br_allowed_ingress(const struct net_bridge *br,
> */
> if (!br->vlan_enabled) {
> BR_INPUT_SKB_CB(skb)->vlan_filtered = false;
> + *vid = br_get_pvid(vg);
> return true;
> }
>
This does not look correct.
This will update fdb with vid which is not 0.
Pvid can be different between each port even when vlan_filtering is
disabled so unicast forwarding (fdb learning) will break.
Also, fdb is visible to userspace so this can break userspace which
expects fdb entries with 0 as well.
Why does the switch driver use pvid while vlan_filtering is disabled?
The (software) bridge does not use pvid for forwarding and use fdb/mdb
entires with vid 0 when vlan_filtering is disabled even if pvid has been
configured.
--
Toshiaki Makita
^ permalink raw reply
* Re: [PATCH net-next v2 3/3] tools: bpftool: add documentation
From: Alexei Starovoitov @ 2017-10-03 4:29 UTC (permalink / raw)
To: Jakub Kicinski; +Cc: netdev, daniel, oss-drivers, David Beckett
In-Reply-To: <20171002183509.76b2cc65@cakuba>
On Mon, Oct 02, 2017 at 06:35:09PM -0700, Jakub Kicinski wrote:
> > will pretty print them as verifier output as well?
>
> We tried to use LLVM as a library for this but the interface is
> painfully unstable and it's a heavy dependency. The current thinking
> is to try to put the instruction printing code in some higher level
> library, but I would rather leave that as a follow up.
follow up, of course.
Not depending on llvm is must have for this tool.
I think we need tiny and simple tools first.
Since you're using gpl+bsd license for this tool I think
it would be fine to copy-paste verifier's pretty print code into it.
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox