From mboxrd@z Thu Jan 1 00:00:00 1970 From: Ido Shamay Subject: Re: [PATCH net-next 09/11] net/mlx5: Ethernet Datapath files Date: Sun, 12 Apr 2015 14:52:21 +0300 Message-ID: <552A5C75.4020901@dev.mellanox.co.il> References: <1428504685-8945-1-git-send-email-amirv@mellanox.com> <1428504685-8945-10-git-send-email-amirv@mellanox.com> Mime-Version: 1.0 Content-Type: text/plain; charset=windows-1252; format=flowed Content-Transfer-Encoding: 7bit Cc: netdev@vger.kernel.org, Yevgeny Petrilin , Saeed Mahameed , Or Gerlitz , Achiad Shochat , Ido Shamay To: Amir Vadai , "David S. Miller" Return-path: Received: from mail-wi0-f176.google.com ([209.85.212.176]:32907 "EHLO mail-wi0-f176.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1751408AbbDLLw0 (ORCPT ); Sun, 12 Apr 2015 07:52:26 -0400 Received: by wiax7 with SMTP id x7so45279886wia.0 for ; Sun, 12 Apr 2015 04:52:25 -0700 (PDT) In-Reply-To: <1428504685-8945-10-git-send-email-amirv@mellanox.com> Sender: netdev-owner@vger.kernel.org List-ID: On 4/8/2015 5:51 PM, Amir Vadai wrote: > Signed-off-by: Amir Vadai > --- > drivers/net/ethernet/mellanox/mlx5/core/en_rx.c | 249 +++++++++++++++ > drivers/net/ethernet/mellanox/mlx5/core/en_tx.c | 350 ++++++++++++++++++++++ > drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c | 107 +++++++ > 3 files changed, 706 insertions(+) > create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en_rx.c > create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en_tx.c > create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c > > diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c > new file mode 100644 > index 0000000..e567046 > --- /dev/null > +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c > @@ -0,0 +1,249 @@ > +/* > + * Copyright (c) 2015, Mellanox Technologies. All rights reserved. > + * > + * This software is available to you under a choice of one of two > + * licenses. You may choose to be licensed under the terms of the GNU > + * General Public License (GPL) Version 2, available from the file > + * COPYING in the main directory of this source tree, or the > + * OpenIB.org BSD license below: > + * > + * Redistribution and use in source and binary forms, with or > + * without modification, are permitted provided that the following > + * conditions are met: > + * > + * - Redistributions of source code must retain the above > + * copyright notice, this list of conditions and the following > + * disclaimer. > + * > + * - Redistributions in binary form must reproduce the above > + * copyright notice, this list of conditions and the following > + * disclaimer in the documentation and/or other materials > + * provided with the distribution. > + * > + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, > + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF > + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND > + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS > + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN > + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN > + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE > + * SOFTWARE. > + */ > + > +#include > +#include > +#include > +#include "en.h" > + > +static inline int mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, > + struct mlx5e_rx_wqe *wqe, u16 ix) > +{ > + struct sk_buff *skb; > + dma_addr_t dma_addr; > + > + skb = netdev_alloc_skb(rq->netdev, rq->wqe_sz); > + if (unlikely(!skb)) > + return -ENOMEM; > + > + skb_reserve(skb, MLX5E_NET_IP_ALIGN); > + > + dma_addr = dma_map_single(rq->pdev, > + /* hw start padding */ > + skb->data - MLX5E_NET_IP_ALIGN, > + /* hw end padding */ > + skb_end_offset(skb), > + DMA_FROM_DEVICE); skb_end_offset depends on NET_SKBUFF_DATA_USES_OFFSET, and may be a pointer. Can use rq->wqe_sz instead. > + > + if (unlikely(dma_mapping_error(rq->pdev, dma_addr))) > + goto err_free_skb; > + > + *((dma_addr_t *)skb->cb) = dma_addr; > + wqe->data.addr = cpu_to_be64(dma_addr + MLX5E_NET_IP_ALIGN); > + > + rq->skb[ix] = skb; > + > + return 0; > + > +err_free_skb: > + dev_kfree_skb(skb); > + > + return -ENOMEM; > +} > + > +bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq) > +{ > + struct mlx5_wq_ll *wq = &rq->wq; > + > + if (unlikely(!test_bit(MLX5E_RQ_STATE_POST_WQES_ENABLE, &rq->state))) > + return false; > + > + while (!mlx5_wq_ll_is_full(wq)) { > + struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(wq, wq->head); > + > + if (unlikely(mlx5e_alloc_rx_wqe(rq, wqe, wq->head))) > + break; > + > + mlx5_wq_ll_push(wq, be16_to_cpu(wqe->next.next_wqe_index)); > + } > + > + /* ensure wqes are visible to device before updating doorbell record */ > + wmb(); > + > + mlx5_wq_ll_update_db_record(wq); > + > + return !mlx5_wq_ll_is_full(wq); > +} > + > +static void mlx5e_lro_update_hdr(struct sk_buff *skb, struct mlx5_cqe64 *cqe) > +{ > + struct ethhdr *eth = (struct ethhdr *)(skb->data); > + struct iphdr *ipv4 = (struct iphdr *)(skb->data + ETH_HLEN); > + struct ipv6hdr *ipv6 = (struct ipv6hdr *)(skb->data + ETH_HLEN); > + struct tcphdr *tcp; > + > + u8 l4_hdr_type = get_cqe_l4_hdr_type(cqe); > + int tcp_ack = ((CQE_L4_HDR_TYPE_TCP_ACK_NO_DATA == l4_hdr_type) || > + (CQE_L4_HDR_TYPE_TCP_ACK_AND_DATA == l4_hdr_type)); > + > + u16 tot_len = be32_to_cpu(cqe->byte_cnt) - ETH_HLEN; > + > + if (eth->h_proto == htons(ETH_P_IP)) { > + tcp = (struct tcphdr *)(skb->data + ETH_HLEN + > + sizeof(struct iphdr)); > + ipv6 = NULL; > + } else { > + tcp = (struct tcphdr *)(skb->data + ETH_HLEN + > + sizeof(struct ipv6hdr)); > + ipv4 = NULL; > + } > + > + if (get_cqe_lro_tcppsh(cqe)) > + tcp->psh = 1; > + > + if (tcp_ack) { > + tcp->ack = 1; > + tcp->ack_seq = cqe->lro_ack_seq_num; > + tcp->window = cqe->lro_tcp_win; > + } > + > + if (ipv4) { > + ipv4->ttl = cqe->lro_min_ttl; > + ipv4->tot_len = cpu_to_be16(tot_len); > + ipv4->check = 0; > + ipv4->check = ip_fast_csum((unsigned char *)ipv4, > + ipv4->ihl); > + } else { > + ipv6->hop_limit = cqe->lro_min_ttl; > + ipv6->payload_len = cpu_to_be16(tot_len - > + sizeof(struct ipv6hdr)); > + } > +} > + > +static inline void mlx5e_skb_set_hash(struct mlx5_cqe64 *cqe, > + struct sk_buff *skb) > +{ > + u8 cht = cqe->rss_hash_type; > + int ht = (cht & CQE_RSS_HTYPE_L4) ? PKT_HASH_TYPE_L4 : > + (cht & CQE_RSS_HTYPE_IP) ? PKT_HASH_TYPE_L3 : > + PKT_HASH_TYPE_NONE; > + skb_set_hash(skb, be32_to_cpu(cqe->rss_hash_result), ht); > +} > + > +static inline void mlx5e_build_rx_skb(struct mlx5_cqe64 *cqe, > + struct mlx5e_rq *rq, > + struct sk_buff *skb) > +{ > + struct net_device *netdev = rq->netdev; > + u32 cqe_bcnt = be32_to_cpu(cqe->byte_cnt); > + int lro_num_seg; > + > + skb_put(skb, cqe_bcnt); > + > + lro_num_seg = be32_to_cpu(cqe->srqn) >> 24; > + if (lro_num_seg > 1) { > + mlx5e_lro_update_hdr(skb, cqe); > + skb_shinfo(skb)->gso_size = MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ; > + rq->stats.lro_packets++; > + rq->stats.lro_bytes += cqe_bcnt; > + } > + > + if (likely(netdev->features & NETIF_F_RXCSUM) && > + (cqe->hds_ip_ext & CQE_L2_OK) && > + (cqe->hds_ip_ext & CQE_L3_OK) && > + (cqe->hds_ip_ext & CQE_L4_OK)) { > + skb->ip_summed = CHECKSUM_UNNECESSARY; > + } else { > + skb->ip_summed = CHECKSUM_NONE; > + rq->stats.csum_none++; > + } > + > + skb->protocol = eth_type_trans(skb, netdev); > + > + skb_record_rx_queue(skb, rq->ix); > + > + if (likely(netdev->features & NETIF_F_RXHASH)) > + mlx5e_skb_set_hash(cqe, skb); > + > + if (cqe_has_vlan(cqe)) > + __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), > + be16_to_cpu(cqe->vlan_info)); > +} > + > +bool mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget) > +{ > + struct mlx5e_rq *rq = cq->sqrq; > + int i; > + > + /* avoid accessing cq (dma coherent memory) if not needed */ > + if (!test_and_clear_bit(MLX5E_CQ_HAS_CQES, &cq->flags)) > + return false; > + > + for (i = 0; i < budget; i++) { > + struct mlx5e_rx_wqe *wqe; > + struct mlx5_cqe64 *cqe; > + struct sk_buff *skb; > + __be16 wqe_counter_be; > + u16 wqe_counter; > + > + cqe = mlx5e_get_cqe(cq); > + if (!cqe) > + break; > + > + wqe_counter_be = cqe->wqe_counter; > + wqe_counter = be16_to_cpu(wqe_counter_be); > + wqe = mlx5_wq_ll_get_wqe(&rq->wq, wqe_counter); > + skb = rq->skb[wqe_counter]; > + rq->skb[wqe_counter] = NULL; > + > + dma_unmap_single(rq->pdev, > + *((dma_addr_t *)skb->cb), > + skb_end_offset(skb), > + DMA_FROM_DEVICE); > + > + if (unlikely((cqe->op_own >> 4) != MLX5_CQE_RESP_SEND)) { > + rq->stats.wqe_err++; > + dev_kfree_skb(skb); > + goto wq_ll_pop; > + } > + > + mlx5e_build_rx_skb(cqe, rq, skb); > + rq->stats.packets++; > + napi_gro_receive(cq->napi, skb); > + > +wq_ll_pop: > + mlx5_wq_ll_pop(&rq->wq, wqe_counter_be, > + &wqe->next.next_wqe_index); > + } > + > + mlx5_cqwq_update_db_record(&cq->wq); > + > + /* ensure cq space is freed before enabling more cqes */ > + wmb(); > + > + if (i == budget) { > + set_bit(MLX5E_CQ_HAS_CQES, &cq->flags); > + return true; > + } > + > + return false; > +} > diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c > new file mode 100644 > index 0000000..1bd2027 > --- /dev/null > +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c > @@ -0,0 +1,350 @@ > +/* > + * Copyright (c) 2015, Mellanox Technologies. All rights reserved. > + * > + * This software is available to you under a choice of one of two > + * licenses. You may choose to be licensed under the terms of the GNU > + * General Public License (GPL) Version 2, available from the file > + * COPYING in the main directory of this source tree, or the > + * OpenIB.org BSD license below: > + * > + * Redistribution and use in source and binary forms, with or > + * without modification, are permitted provided that the following > + * conditions are met: > + * > + * - Redistributions of source code must retain the above > + * copyright notice, this list of conditions and the following > + * disclaimer. > + * > + * - Redistributions in binary form must reproduce the above > + * copyright notice, this list of conditions and the following > + * disclaimer in the documentation and/or other materials > + * provided with the distribution. > + * > + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, > + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF > + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND > + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS > + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN > + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN > + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE > + * SOFTWARE. > + */ > + > +#include > +#include > +#include "en.h" > + > +static void mlx5e_dma_pop_last_pushed(struct mlx5e_sq *sq, dma_addr_t *addr, > + u32 *size) > +{ > + sq->dma_fifo_pc--; > + *addr = sq->dma_fifo[sq->dma_fifo_pc & sq->dma_fifo_mask].addr; > + *size = sq->dma_fifo[sq->dma_fifo_pc & sq->dma_fifo_mask].size; > +} > + > +static void mlx5e_dma_unmap_wqe_err(struct mlx5e_sq *sq, struct sk_buff *skb) > +{ > + dma_addr_t addr; > + u32 size; > + int i; > + > + for (i = 0; i < MLX5E_TX_SKB_CB(skb)->num_dma; i++) { > + mlx5e_dma_pop_last_pushed(sq, &addr, &size); > + dma_unmap_single(sq->pdev, addr, size, DMA_TO_DEVICE); > + } > +} > + > +static inline void mlx5e_dma_push(struct mlx5e_sq *sq, dma_addr_t addr, > + u32 size) > +{ > + sq->dma_fifo[sq->dma_fifo_pc & sq->dma_fifo_mask].addr = addr; > + sq->dma_fifo[sq->dma_fifo_pc & sq->dma_fifo_mask].size = size; > + sq->dma_fifo_pc++; > +} > + > +static inline void mlx5e_dma_get(struct mlx5e_sq *sq, u32 i, dma_addr_t *addr, > + u32 *size) > +{ > + *addr = sq->dma_fifo[i & sq->dma_fifo_mask].addr; > + *size = sq->dma_fifo[i & sq->dma_fifo_mask].size; > +} > + > +u16 mlx5e_select_queue(struct net_device *dev, struct sk_buff *skb, > + void *accel_priv, select_queue_fallback_t fallback) > +{ > + struct mlx5e_priv *priv = netdev_priv(dev); > + int channel_ix = fallback(dev, skb); > + int up = skb_vlan_tag_present(skb) ? > + skb->vlan_tci >> VLAN_PRIO_SHIFT : > + priv->default_vlan_prio; > + int tc = netdev_get_prio_tc_map(dev, up); > + > + return (tc << priv->order_base_2_num_channels) | channel_ix; > +} > + > +static inline u16 mlx5e_get_inline_hdr_size(struct mlx5e_sq *sq, > + struct sk_buff *skb) > +{ > +#define MLX5E_MIN_INLINE 16 /* eth header with vlan (w/o next ethertype) */ > +#define MLX5E_MAX_INLINE (128 - sizeof(struct mlx5e_tx_wqe) +\ > + 2/*sizeof(eseg->inline_hdr_start)*/) > + > + if (!skb_shinfo(skb)->nr_frags && > + (skb_headlen(skb) <= MLX5E_MAX_INLINE)) > + return skb_headlen(skb); > + > + return MLX5E_MIN_INLINE; > +} > + > +static inline void mlx5e_insert_vlan(void *start, struct sk_buff *skb, u16 ihs) > +{ > + struct vlan_ethhdr *vhdr = (struct vlan_ethhdr *)start; > + int cpy1_sz = 2 * ETH_ALEN; > + int cpy2_sz = ihs - cpy1_sz - VLAN_HLEN; > + > + skb_copy_from_linear_data(skb, vhdr, cpy1_sz); > + skb_pull_inline(skb, cpy1_sz); > + vhdr->h_vlan_proto = skb->vlan_proto; > + vhdr->h_vlan_TCI = cpu_to_be16(skb_vlan_tag_get(skb)); > + skb_copy_from_linear_data(skb, &vhdr->h_vlan_encapsulated_proto, > + cpy2_sz); > + skb_pull_inline(skb, cpy2_sz); > +} > + > +static netdev_tx_t mlx5e_sq_xmit(struct mlx5e_sq *sq, struct sk_buff *skb) > +{ > + struct mlx5_wq_cyc *wq = &sq->wq; > + > + u16 pi = sq->pc & wq->sz_m1; > + struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(wq, pi); > + > + struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; > + struct mlx5_wqe_eth_seg *eseg = &wqe->eth; > + struct mlx5_wqe_data_seg *dseg; > + > + u8 opcode = MLX5_OPCODE_SEND; > + dma_addr_t dma_addr = 0; > + u16 headlen; > + u16 ds_cnt; > + u16 ihs; > + int i; > + > + memset(wqe, 0, sizeof(*wqe)); > + > + if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) > + eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM | MLX5_ETH_WQE_L4_CSUM; > + else > + sq->stats.csum_offload_none++; > + > + if (skb_is_gso(skb)) { > + u32 payload_len; > + int num_pkts; > + > + eseg->mss = cpu_to_be16(skb_shinfo(skb)->gso_size); > + opcode = MLX5_OPCODE_LSO; > + ihs = skb_transport_offset(skb) + tcp_hdrlen(skb); > + payload_len = skb->len - ihs; > + num_pkts = (payload_len / skb_shinfo(skb)->gso_size) + > + !!(payload_len % skb_shinfo(skb)->gso_size); > + MLX5E_TX_SKB_CB(skb)->num_bytes = skb->len + > + (num_pkts - 1) * ihs; > + sq->stats.tso_packets++; > + sq->stats.tso_bytes += payload_len; > + } else { > + ihs = mlx5e_get_inline_hdr_size(sq, skb); > + MLX5E_TX_SKB_CB(skb)->num_bytes = max_t(unsigned int, skb->len, > + ETH_ZLEN); > + } > + > + if (skb_vlan_tag_present(skb)) { > + mlx5e_insert_vlan(eseg->inline_hdr_start, skb, ihs); > + } else { > + skb_copy_from_linear_data(skb, eseg->inline_hdr_start, ihs); > + skb_pull_inline(skb, ihs); > + } > + > + eseg->inline_hdr_sz = cpu_to_be16(ihs); > + > + ds_cnt = sizeof(*wqe) / MLX5_SEND_WQE_DS; > + ds_cnt += DIV_ROUND_UP(ihs - sizeof(eseg->inline_hdr_start), > + MLX5_SEND_WQE_DS); > + dseg = (struct mlx5_wqe_data_seg *)cseg + ds_cnt; > + > + MLX5E_TX_SKB_CB(skb)->num_dma = 0; > + > + headlen = skb_headlen(skb); > + if (headlen) { > + dma_addr = dma_map_single(sq->pdev, skb->data, headlen, > + DMA_TO_DEVICE); > + if (unlikely(dma_mapping_error(sq->pdev, dma_addr))) > + goto dma_unmap_wqe_err; > + > + dseg->addr = cpu_to_be64(dma_addr); > + dseg->lkey = sq->mkey_be; > + dseg->byte_count = cpu_to_be32(headlen); > + > + mlx5e_dma_push(sq, dma_addr, headlen); > + MLX5E_TX_SKB_CB(skb)->num_dma++; > + > + dseg++; > + } > + > + for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { > + struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i]; > + int fsz = skb_frag_size(frag); > + > + dma_addr = skb_frag_dma_map(sq->pdev, frag, 0, fsz, > + DMA_TO_DEVICE); > + if (unlikely(dma_mapping_error(sq->pdev, dma_addr))) > + goto dma_unmap_wqe_err; > + > + dseg->addr = cpu_to_be64(dma_addr); > + dseg->lkey = sq->mkey_be; > + dseg->byte_count = cpu_to_be32(fsz); > + > + mlx5e_dma_push(sq, dma_addr, fsz); > + MLX5E_TX_SKB_CB(skb)->num_dma++; > + > + dseg++; > + } > + > + ds_cnt += MLX5E_TX_SKB_CB(skb)->num_dma; > + > + cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | opcode); > + cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt); > + cseg->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE; > + > + sq->skb[pi] = skb; > + > + MLX5E_TX_SKB_CB(skb)->num_wqebbs = DIV_ROUND_UP(ds_cnt, > + MLX5_SEND_WQEBB_NUM_DS); > + sq->pc += MLX5E_TX_SKB_CB(skb)->num_wqebbs; > + > + netdev_tx_sent_queue(sq->txq, MLX5E_TX_SKB_CB(skb)->num_bytes); > + > + if (unlikely(!mlx5e_sq_has_room_for(sq, MLX5_SEND_WQE_MAX_WQEBBS))) { > + netif_tx_stop_queue(sq->txq); > + sq->stats.stopped++; > + } > + > + if (!skb->xmit_more || netif_xmit_stopped(sq->txq)) > + mlx5e_tx_notify_hw(sq, wqe); > + > + sq->stats.packets++; > + return NETDEV_TX_OK; > + > +dma_unmap_wqe_err: > + sq->stats.dropped++; > + mlx5e_dma_unmap_wqe_err(sq, skb); > + > + dev_kfree_skb_any(skb); > + > + return NETDEV_TX_OK; > +} > + > +netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev) > +{ > + struct mlx5e_priv *priv = netdev_priv(dev); > + int ix = skb->queue_mapping; > + int tc = 0; > + struct mlx5e_channel *c = priv->channel[ix]; > + struct mlx5e_sq *sq = &c->sq[tc]; > + > + return mlx5e_sq_xmit(sq, skb); > +} > + > +netdev_tx_t mlx5e_xmit_multi_tc(struct sk_buff *skb, struct net_device *dev) > +{ > + struct mlx5e_priv *priv = netdev_priv(dev); > + int ix = skb->queue_mapping & priv->queue_mapping_channel_mask; > + int tc = skb->queue_mapping >> priv->order_base_2_num_channels; > + struct mlx5e_channel *c = priv->channel[ix]; > + struct mlx5e_sq *sq = &c->sq[tc]; > + > + return mlx5e_sq_xmit(sq, skb); > +} > + > +bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq) > +{ > + struct mlx5e_sq *sq; > + u32 dma_fifo_cc; > + u32 nbytes; > + u16 npkts; > + u16 sqcc; > + int i; > + > + /* avoid accessing cq (dma coherent memory) if not needed */ > + if (!test_and_clear_bit(MLX5E_CQ_HAS_CQES, &cq->flags)) > + return false; > + > + sq = cq->sqrq; > + > + npkts = 0; > + nbytes = 0; > + > + /* sq->cc must be updated only after mlx5_cqwq_update_db_record(), > + * otherwise a cq overrun may occur */ > + sqcc = sq->cc; > + > + /* avoid dirtying sq cache line every cqe */ > + dma_fifo_cc = sq->dma_fifo_cc; > + > + for (i = 0; i < MLX5E_TX_CQ_POLL_BUDGET; i++) { > + struct mlx5_cqe64 *cqe; > + struct sk_buff *skb; > + u16 ci; > + int j; > + > + cqe = mlx5e_get_cqe(cq); > + if (!cqe) > + break; > + > + ci = sqcc & sq->wq.sz_m1; > + skb = sq->skb[ci]; > + > + if (unlikely(!skb)) { /* nop */ > + sq->stats.nop++; > + sqcc++; > + goto free_skb; > + } > + > + for (j = 0; j < MLX5E_TX_SKB_CB(skb)->num_dma; j++) { > + dma_addr_t addr; > + u32 size; > + > + mlx5e_dma_get(sq, dma_fifo_cc, &addr, &size); > + dma_fifo_cc++; > + dma_unmap_single(sq->pdev, addr, size, DMA_TO_DEVICE); > + } > + > + npkts++; > + nbytes += MLX5E_TX_SKB_CB(skb)->num_bytes; > + sqcc += MLX5E_TX_SKB_CB(skb)->num_wqebbs; > + > +free_skb: > + dev_kfree_skb(skb); > + } > + > + mlx5_cqwq_update_db_record(&cq->wq); > + > + /* ensure cq space is freed before enabling more cqes */ > + wmb(); > + > + sq->dma_fifo_cc = dma_fifo_cc; > + sq->cc = sqcc; > + > + netdev_tx_completed_queue(sq->txq, npkts, nbytes); > + > + if (netif_tx_queue_stopped(sq->txq) && > + mlx5e_sq_has_room_for(sq, MLX5_SEND_WQE_MAX_WQEBBS) && > + likely(test_bit(MLX5E_SQ_STATE_WAKE_TXQ_ENABLE, &sq->state))) { > + netif_tx_wake_queue(sq->txq); > + sq->stats.wake++; > + } > + if (i == MLX5E_TX_CQ_POLL_BUDGET) { > + set_bit(MLX5E_CQ_HAS_CQES, &cq->flags); > + return true; > + } > + > + return false; > +} > diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c > new file mode 100644 > index 0000000..088bc42 > --- /dev/null > +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c > @@ -0,0 +1,107 @@ > +/* > + * Copyright (c) 2015, Mellanox Technologies. All rights reserved. > + * > + * This software is available to you under a choice of one of two > + * licenses. You may choose to be licensed under the terms of the GNU > + * General Public License (GPL) Version 2, available from the file > + * COPYING in the main directory of this source tree, or the > + * OpenIB.org BSD license below: > + * > + * Redistribution and use in source and binary forms, with or > + * without modification, are permitted provided that the following > + * conditions are met: > + * > + * - Redistributions of source code must retain the above > + * copyright notice, this list of conditions and the following > + * disclaimer. > + * > + * - Redistributions in binary form must reproduce the above > + * copyright notice, this list of conditions and the following > + * disclaimer in the documentation and/or other materials > + * provided with the distribution. > + * > + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, > + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF > + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND > + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS > + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN > + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN > + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE > + * SOFTWARE. > + */ > + > +#include "en.h" > + > +struct mlx5_cqe64 *mlx5e_get_cqe(struct mlx5e_cq *cq) > +{ > + struct mlx5_cqwq *wq = &cq->wq; > + u32 ci = mlx5_cqwq_get_ci(wq); > + struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(wq, ci); > + int cqe_ownership_bit = cqe->op_own & MLX5_CQE_OWNER_MASK; > + int sw_ownership_val = mlx5_cqwq_get_wrap_cnt(wq) & 1; > + > + if (cqe_ownership_bit != sw_ownership_val) > + return NULL; > + > + mlx5_cqwq_pop(wq); > + > + /* ensure cqe content is read after cqe ownership bit */ > + rmb(); > + > + return cqe; > +} > + > +int mlx5e_napi_poll(struct napi_struct *napi, int budget) > +{ > + struct mlx5e_channel *c = container_of(napi, struct mlx5e_channel, > + napi); > + bool busy = false; > + int i; > + > + clear_bit(MLX5E_CHANNEL_NAPI_SCHED, &c->flags); > + > + for (i = 0; i < c->num_tc; i++) > + busy |= mlx5e_poll_tx_cq(&c->sq[i].cq); > + > + busy |= mlx5e_poll_rx_cq(&c->rq.cq, budget); > + > + busy |= mlx5e_post_rx_wqes(c->rq.cq.sqrq); > + > + if (busy) > + return budget; > + > + napi_complete(napi); > + > + /* avoid losing completion event during/after polling cqs */ > + if (test_bit(MLX5E_CHANNEL_NAPI_SCHED, &c->flags)) { > + napi_schedule(napi); > + return 0; > + } > + > + for (i = 0; i < c->num_tc; i++) > + mlx5e_cq_arm(&c->sq[i].cq); > + mlx5e_cq_arm(&c->rq.cq); > + > + return 0; > +} > + > +void mlx5e_completion_event(struct mlx5_core_cq *mcq) > +{ > + struct mlx5e_cq *cq = container_of(mcq, struct mlx5e_cq, mcq); > + > + set_bit(MLX5E_CQ_HAS_CQES, &cq->flags); > + set_bit(MLX5E_CHANNEL_NAPI_SCHED, &cq->channel->flags); > + barrier(); > + napi_schedule(cq->napi); > +} > + > +void mlx5e_cq_error_event(struct mlx5_core_cq *mcq, enum mlx5_event event) > +{ > + struct mlx5e_cq *cq = container_of(mcq, struct mlx5e_cq, mcq); > + struct mlx5e_channel *c = cq->channel; > + struct mlx5e_priv *priv = c->priv; > + struct net_device *netdev = priv->netdev; > + > + netdev_err(netdev, "%s: cqn=0x%.6x event=0x%.2x\n", > + __func__, mcq->cqn, event); > +}