Replenish is the last open-path hypervisor call that still needs per-queue awareness before MQ is turned on. Today ibmveth_replenish_buffer_pool() calls h_add_logical_lan_buffer() or h_add_logical_lan_buffers() directly; MQ posts via H_ADD_LOGICAL_LAN_BUFFERS_QUEUE against adapter->queue_handle[]. Add ibmveth_add_logical_lan_buffers() to pick the hcall: multi_queue uses h_add_logical_lan_buffers_queue() (up to 12 buffers; IOBAs packed two per word: even indices in the high 32 bits, odd in the low); legacy uses the existing single- and multi-buffer hcalls. Count add_buf/add_bufs/add_bufs_queue in hcall_stats. Thread queue_index through the RX helpers used by poll and replenish, and update open/poll/netpoll callers so arity stays consistent. Add per-queue replenish_lock so later concurrent NAPI/resize paths can serialize buffer posting (producer and harvest/remove consumer). Until MQ enablement, callers still pass queue 0 and legacy hcalls remain the live path. H_FUNCTION handling is split: - multi_queue: schedule adapter reset after dropping replenish_lock (do not printk under the lock - netconsole can re-enter replenish) - legacy multi-buffer LPM fallback: set rx_buffers_per_hcall = 1 and break so the next replenish re-samples batch as 1 Signed-off-by: Mingming Cao Reviewed-by: Dave Marquardt Tested-by: Shaik Abdulla --- Changes in v5: - On MQ buffer-add H_FUNCTION: schedule adapter reset after dropping replenish_lock (v4 logged/broke with no recovery; can permanently dry the pool) - Move replenish fail logging / reset scheduling out from under replenish_lock so netconsole cannot deadlock re-entering replenish - Serialize harvest/remove with per-queue replenish_lock (netpoll replenish vs NAPI consumer; v4 locked producer only) - Fail logs use real wrapper names: h_add_logical_lan_buffers[_queue] / h_add_logical_lan_buffer (v4 interpolated broken lan[_queue] strings) - Document replenish_lock + no-printk-under-lock for netconsole (first lock use); outcomes enum + ibmveth_replenish_fail defined here - Defer adapter-global counter atomics and irqsave critical-section shorten to cover follow-up - Bad queue_index poll path: napi_complete before return lands with poll harden (not claimed fully here) - Keep pool active/size/threshold across free_buffer_pool (probe/sysfs geometry); only clear runtime allocations + available (ifdown/up reopen must still see active pools) - get_buffer: use correlator_valid (drop WARN_ON; keep schedule_work until poll skip owns reset) - Introduce ibmveth_rxq_correlator_valid / ibmveth_rxq_advance at first remove/harvest use; init replenish_lock in remove_buffer KUnit - On RESET_MAP/RESET_MQ, stop remaining pool walks (goto unlock) Changes in v4: - Introduce queue-aware replenish/poll helpers with their first callers in the same patch; do not leave a 2-arg replenish call ahead of the signature change. - Restore the pre-MQ LPM H_FUNCTION break instead of continue; do not loop forever on a stale local batch size. - Fold per-queue replenish_lock into this patch. - Update kdoc for MQ parameters on remove_buffer_from_pool / rxq_harvest_buffer. drivers/net/ethernet/ibm/ibmveth.c | 523 +++++++++++++++++++++-------- drivers/net/ethernet/ibm/ibmveth.h | 6 +- 2 files changed, 387 insertions(+), 142 deletions(-) diff --git a/drivers/net/ethernet/ibm/ibmveth.c b/drivers/net/ethernet/ibm/ibmveth.c index 8519cad50322..58a639a962a6 100644 --- a/drivers/net/ethernet/ibm/ibmveth.c +++ b/drivers/net/ethernet/ibm/ibmveth.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -101,49 +102,58 @@ static struct ibmveth_stat ibmveth_stats[] = { }; /* simple methods of getting data from the current rxq entry */ -static inline u32 ibmveth_rxq_flags(struct ibmveth_adapter *adapter) +static inline u32 ibmveth_rxq_flags(struct ibmveth_adapter *adapter, + int queue_index) { - struct ibmveth_rx_q *rxq = &adapter->rx_queue[0]; + struct ibmveth_rx_q *rxq = &adapter->rx_queue[queue_index]; return be32_to_cpu(rxq->queue_addr[rxq->index].flags_off); } -static inline int ibmveth_rxq_toggle(struct ibmveth_adapter *adapter) +static inline int ibmveth_rxq_toggle(struct ibmveth_adapter *adapter, + int queue_index) { - return (ibmveth_rxq_flags(adapter) & IBMVETH_RXQ_TOGGLE) >> - IBMVETH_RXQ_TOGGLE_SHIFT; + return (ibmveth_rxq_flags(adapter, queue_index) & IBMVETH_RXQ_TOGGLE) >> + IBMVETH_RXQ_TOGGLE_SHIFT; } -static inline int ibmveth_rxq_pending_buffer(struct ibmveth_adapter *adapter) +static inline int ibmveth_rxq_pending_buffer(struct ibmveth_adapter *adapter, + int queue_index) { - return ibmveth_rxq_toggle(adapter) == adapter->rx_queue[0].toggle; + return ibmveth_rxq_toggle(adapter, queue_index) == + adapter->rx_queue[queue_index].toggle; } -static inline int ibmveth_rxq_buffer_valid(struct ibmveth_adapter *adapter) +static inline int ibmveth_rxq_buffer_valid(struct ibmveth_adapter *adapter, + int queue_index) { - return ibmveth_rxq_flags(adapter) & IBMVETH_RXQ_VALID; + return ibmveth_rxq_flags(adapter, queue_index) & IBMVETH_RXQ_VALID; } -static inline int ibmveth_rxq_frame_offset(struct ibmveth_adapter *adapter) +static inline int ibmveth_rxq_frame_offset(struct ibmveth_adapter *adapter, + int queue_index) { - return ibmveth_rxq_flags(adapter) & IBMVETH_RXQ_OFF_MASK; + return ibmveth_rxq_flags(adapter, queue_index) & IBMVETH_RXQ_OFF_MASK; } -static inline int ibmveth_rxq_large_packet(struct ibmveth_adapter *adapter) +static inline int ibmveth_rxq_large_packet(struct ibmveth_adapter *adapter, + int queue_index) { - return ibmveth_rxq_flags(adapter) & IBMVETH_RXQ_LRG_PKT; + return ibmveth_rxq_flags(adapter, queue_index) & IBMVETH_RXQ_LRG_PKT; } -static inline int ibmveth_rxq_frame_length(struct ibmveth_adapter *adapter) +static inline int ibmveth_rxq_frame_length(struct ibmveth_adapter *adapter, + int queue_index) { - struct ibmveth_rx_q *rxq = &adapter->rx_queue[0]; + struct ibmveth_rx_q *rxq = &adapter->rx_queue[queue_index]; return be32_to_cpu(rxq->queue_addr[rxq->index].length); } -static inline int ibmveth_rxq_csum_good(struct ibmveth_adapter *adapter) +static inline int ibmveth_rxq_csum_good(struct ibmveth_adapter *adapter, + int queue_index) { - return ibmveth_rxq_flags(adapter) & IBMVETH_RXQ_CSUM_GOOD; + return ibmveth_rxq_flags(adapter, queue_index) & IBMVETH_RXQ_CSUM_GOOD; } static unsigned int ibmveth_real_max_tx_queues(void) @@ -262,6 +272,7 @@ ibmveth_alloc_rx_queues(struct ibmveth_adapter *adapter, int rxq_entries) adapter->rx_queue[i].index = 0; adapter->rx_queue[i].num_slots = rxq_entries; adapter->rx_queue[i].toggle = 1; + spin_lock_init(&adapter->rx_queue[i].replenish_lock); netdev_dbg(netdev, "queue %d: buffer_list @ 0x%p (DMA: 0x%llx), rx_queue @ 0x%p (DMA: 0x%llx), %llu entries\n", i, adapter->buffer_list_addr[i], @@ -696,11 +707,92 @@ static inline void ibmveth_flush_buffer(void *addr, unsigned long length) asm("dcbf %0,%1,1" :: "b" (addr), "r" (offset)); } -/* replenish the buffers for a pool. note that we don't need to - * skb_reserve these since they are used for incoming... +/** + * ibmveth_add_logical_lan_buffers - Add receive buffers to hypervisor + * @adapter: ibmveth adapter structure + * @descs: array of buffer descriptors to add + * @filled: number of valid descriptors in the array + * @buff_size: size of each buffer (multi-queue mode only) + * @queue_index: RX queue index + * + * Return: hypervisor return code */ -static void ibmveth_replenish_buffer_pool(struct ibmveth_adapter *adapter, - struct ibmveth_buff_pool *pool) +static long ibmveth_add_logical_lan_buffers(struct ibmveth_adapter *adapter, + union ibmveth_buf_desc *descs, + int filled, + unsigned long buff_size, + int queue_index) +{ + struct vio_dev *vdev = adapter->vdev; + unsigned long rc; + + if (adapter->multi_queue) { + unsigned long buffersznum = (buff_size << 32) | filled; + unsigned long ioba[IBMVETH_MAX_RX_PER_HCALL / 2] = {0}; + unsigned long handle = adapter->queue_handle[queue_index]; + int i; + + /* Pack descriptor addresses into ioba pairs. + * Each ioba holds two 32-bit addresses packed into 64 bits: + * - Even descriptors (0,2,4...) go in high 32 bits + * - Odd descriptors (1,3,5...) go in low 32 bits + */ + for (i = 0; i < filled && i < IBMVETH_MAX_RX_PER_HCALL; i++) { + int pair_idx = i / 2; + int is_high = (i % 2 == 0); + + if (is_high) + ioba[pair_idx] = (unsigned long) + descs[i].fields.address << 32; + else + ioba[pair_idx] |= descs[i].fields.address; + } + + rc = h_add_logical_lan_buffers_queue(vdev->unit_address, + handle, + buffersznum, + ioba[0], ioba[1], ioba[2], + ioba[3], ioba[4], ioba[5]); + adapter->hcall_stats.add_bufs_queue++; + } else if (filled == 1) { + rc = h_add_logical_lan_buffer(vdev->unit_address, + descs[0].desc); + adapter->hcall_stats.add_buf++; + } else { + rc = h_add_logical_lan_buffers(vdev->unit_address, + descs[0].desc, descs[1].desc, + descs[2].desc, descs[3].desc, + descs[4].desc, descs[5].desc, + descs[6].desc, descs[7].desc); + adapter->hcall_stats.add_bufs++; + } + + return rc; +} + +/* Outcomes for ibmveth_replenish_buffer_pool(); logged after unlock. */ +enum { + IBMVETH_REPLENISH_OK = 0, + IBMVETH_REPLENISH_RESET_MAP, + IBMVETH_REPLENISH_RESET_MQ, + IBMVETH_REPLENISH_HCALL_FAIL, + IBMVETH_REPLENISH_BATCH_FALLBACK, +}; + +struct ibmveth_replenish_fail { + unsigned long lpar_rc; + u32 filled; + u32 batch; +}; + +/* Replenish the buffers for a pool. + * Caller must hold the per-queue replenish_lock. Do not printk here — + * netconsole on the same device can re-enter replenish_task. + */ +static int ibmveth_replenish_buffer_pool(struct ibmveth_adapter *adapter, + struct ibmveth_buff_pool *pool, + int queue_index, + struct ibmveth_replenish_fail *fail) { union ibmveth_buf_desc descs[IBMVETH_MAX_RX_PER_HCALL] = {0}; u32 remaining = pool->size - atomic_read(&pool->available); @@ -712,6 +804,7 @@ static void ibmveth_replenish_buffer_pool(struct ibmveth_adapter *adapter, dma_addr_t dma_addr; struct device *dev; u32 index; + int outcome = IBMVETH_REPLENISH_OK; vdev = adapter->vdev; dev = &vdev->dev; @@ -726,12 +819,9 @@ static void ibmveth_replenish_buffer_pool(struct ibmveth_adapter *adapter, /* Fill a batch of descriptors */ for (filled = 0; filled < min(remaining, batch); filled++) { index = pool->free_map[free_index]; - if (WARN_ON(index == IBM_VETH_INVALID_MAP)) { + if (index == IBM_VETH_INVALID_MAP) { adapter->replenish_add_buff_failure++; - netdev_info(adapter->netdev, - "Invalid map index %u, reset\n", - index); - schedule_work(&adapter->work); + outcome = IBMVETH_REPLENISH_RESET_MAP; break; } @@ -783,28 +873,21 @@ static void ibmveth_replenish_buffer_pool(struct ibmveth_adapter *adapter, free_index = 0; } + if (outcome != IBMVETH_REPLENISH_OK) + break; + if (!filled) break; - /* single buffer case*/ - if (filled == 1) - lpar_rc = h_add_logical_lan_buffer(vdev->unit_address, - descs[0].desc); - else - /* Multi-buffer hcall */ - lpar_rc = h_add_logical_lan_buffers(vdev->unit_address, - descs[0].desc, - descs[1].desc, - descs[2].desc, - descs[3].desc, - descs[4].desc, - descs[5].desc, - descs[6].desc, - descs[7].desc); + lpar_rc = ibmveth_add_logical_lan_buffers(adapter, descs, + filled, + pool->buff_size, + queue_index); + if (lpar_rc != H_SUCCESS) { - dev_warn_ratelimited(dev, - "RX h_add_logical_lan failed: filled=%u, rc=%lu, batch=%u\n", - filled, lpar_rc, batch); + fail->lpar_rc = lpar_rc; + fail->filled = filled; + fail->batch = batch; goto hcall_failure; } @@ -844,30 +927,35 @@ static void ibmveth_replenish_buffer_pool(struct ibmveth_adapter *adapter, } adapter->replenish_add_buff_failure += filled; - /* - * If multi rx buffers hcall is no longer supported by FW - * e.g. in the case of Live Partition Migration - */ - if (batch > 1 && lpar_rc == H_FUNCTION) { - /* - * Instead of retry submit single buffer individually - * here just set the max rx buffer per hcall to 1 - * buffers will be respleshed next time - * when ibmveth_replenish_buffer_pool() is called again - * with single-buffer case - */ - netdev_info(adapter->netdev, - "RX Multi buffers not supported by FW, rc=%lu\n", - lpar_rc); - adapter->rx_buffers_per_hcall = 1; - netdev_info(adapter->netdev, - "Next rx replesh will fall back to single-buffer hcall\n"); + if (lpar_rc == H_FUNCTION) { + if (adapter->multi_queue) { + /* + * LPM / firmware may drop MQ buffer hcalls. + * Schedule reset so we do not sit forever in + * no-buffer with the link still up. + */ + outcome = IBMVETH_REPLENISH_RESET_MQ; + } else if (batch > 1) { + /* + * Live Partition Migration may drop multi- + * buffer support. Fall back to single-buffer + * on the next replenish; do not continue with + * a stale local batch size (infinite loop). + */ + adapter->rx_buffers_per_hcall = 1; + outcome = IBMVETH_REPLENISH_BATCH_FALLBACK; + } else { + outcome = IBMVETH_REPLENISH_HCALL_FAIL; + } + } else { + outcome = IBMVETH_REPLENISH_HCALL_FAIL; } break; } mb(); atomic_add(buffers_added, &(pool->available)); + return outcome; } /* @@ -883,21 +971,85 @@ static void ibmveth_update_rx_no_buffer(struct ibmveth_adapter *adapter) } /* replenish routine */ -static void ibmveth_replenish_task(struct ibmveth_adapter *adapter) +static void ibmveth_replenish_task(struct ibmveth_adapter *adapter, + int queue_index) { - int i; + struct ibmveth_rx_q *rxq = &adapter->rx_queue[queue_index]; + struct ibmveth_replenish_fail fail = {}; + unsigned long flags; + int i, rc; + int need_reset = 0; + int batch_fallback = 0; + int hcall_fail = 0; + + if (queue_index >= adapter->num_rx_queues) { + netdev_dbg(adapter->netdev, + "Skipping replenish for freed queue %d (num_queues=%d)\n", + queue_index, adapter->num_rx_queues); + return; + } adapter->replenish_task_cycles++; - for (i = (IBMVETH_NUM_BUFF_POOLS - 1); i >= 0; i--) { - struct ibmveth_buff_pool *pool = &adapter->rx_buff_pool[0][i]; + spin_lock_irqsave(&rxq->replenish_lock, flags); - if (pool->active && - (atomic_read(&pool->available) < pool->threshold)) - ibmveth_replenish_buffer_pool(adapter, pool); + for (i = (IBMVETH_NUM_BUFF_POOLS - 1); i >= 0; i--) { + struct ibmveth_buff_pool *pool = + &adapter->rx_buff_pool[queue_index][i]; + + if (pool->active && pool->free_map && + (atomic_read(&pool->available) < pool->threshold)) { + rc = ibmveth_replenish_buffer_pool(adapter, pool, + queue_index, &fail); + switch (rc) { + case IBMVETH_REPLENISH_RESET_MAP: + case IBMVETH_REPLENISH_RESET_MQ: + need_reset = rc; + goto out_unlock; + case IBMVETH_REPLENISH_BATCH_FALLBACK: + batch_fallback = 1; + break; + case IBMVETH_REPLENISH_HCALL_FAIL: + hcall_fail = 1; + break; + default: + break; + } + } } +out_unlock: ibmveth_update_rx_no_buffer(adapter); + + spin_unlock_irqrestore(&rxq->replenish_lock, flags); + + /* Log and schedule reset only after dropping replenish_lock. */ + if (need_reset == IBMVETH_REPLENISH_RESET_MAP) { + netdev_info(adapter->netdev, + "Invalid RX free_map entry on queue %d, reset\n", + queue_index); + schedule_work(&adapter->work); + } else if (need_reset == IBMVETH_REPLENISH_RESET_MQ) { + dev_err_ratelimited(&adapter->netdev->dev, + "MQ buffer add H_FUNCTION (q=%d, batch=%u), reset\n", + queue_index, fail.batch); + schedule_work(&adapter->work); + } + + if (batch_fallback) + dev_warn_ratelimited(&adapter->netdev->dev, + "Legacy batch add H_FUNCTION (batch=%u), fallback\n", + fail.batch); + + if (hcall_fail) + dev_warn_ratelimited(&adapter->netdev->dev, + "RX %s failed: filled=%u, rc=%lu, batch=%u\n", + adapter->multi_queue ? + "h_add_logical_lan_buffers_queue" : + (fail.batch == 1 ? + "h_add_logical_lan_buffer" : + "h_add_logical_lan_buffers"), + fail.filled, fail.lpar_rc, fail.batch); } /* empty and free ana buffer pool - also used to do cleanup in error paths */ @@ -932,6 +1084,14 @@ static void ibmveth_free_buffer_pool(struct ibmveth_adapter *adapter, kfree(pool->skbuff); pool->skbuff = NULL; } + + /* + * Keep probe/sysfs geometry (active, size, buff_size, threshold). + * Clearing active here was a v3 ifdown/up regression: open skips + * !active pools, so reopen posted no RX buffers (TX OK, ARP/RX + * dead) at any queue count, including RX=8 with no -L. + */ + atomic_set(&pool->available, 0); } /** @@ -1070,35 +1230,75 @@ ibmveth_free_buffer_pools(struct ibmveth_adapter *adapter) adapter->num_rx_queues); } +static bool ibmveth_rxq_correlator_valid(struct ibmveth_adapter *adapter, + int queue_index, u64 correlator) +{ + unsigned int pool = correlator >> 32; + unsigned int index = correlator & 0xffffffffUL; + struct ibmveth_buff_pool *bpool; + + if (pool >= IBMVETH_NUM_BUFF_POOLS) + return false; + + bpool = &adapter->rx_buff_pool[queue_index][pool]; + + /* init_buffer_pool() sets size for inactive pools; free_buffer_pool() + * clears skbuff but used to leave size/active set. Require a live + * pool with allocated arrays before indexing. + */ + if (!bpool->active || !bpool->skbuff || !bpool->free_map) + return false; + + return index < bpool->size; +} + +static void ibmveth_rxq_advance(struct ibmveth_rx_q *rxq) +{ + if (++rxq->index == rxq->num_slots) { + rxq->index = 0; + rxq->toggle = !rxq->toggle; + } +} + /** * ibmveth_remove_buffer_from_pool - remove a buffer from a pool * @adapter: adapter instance * @correlator: identifies pool and index + * @queue_index: RX queue index (0..num_rx_queues-1) * @reuse: whether to reuse buffer * + * Context: may run concurrently with netpoll replenish_task on the same + * queue; takes per-queue replenish_lock to serialize free_map / + * producer_index / available against the producer. + * * Return: * * %0 - success * * %-EINVAL - correlator maps to pool or index out of range * * %-EFAULT - pool and index map to null skb */ static int ibmveth_remove_buffer_from_pool(struct ibmveth_adapter *adapter, - u64 correlator, bool reuse) + u64 correlator, int queue_index, + bool reuse) { + struct ibmveth_rx_q *rxq = &adapter->rx_queue[queue_index]; unsigned int pool = correlator >> 32; unsigned int index = correlator & 0xffffffffUL; unsigned int free_index; struct sk_buff *skb; + unsigned long flags; + int rc = 0; - if (WARN_ON(pool >= IBMVETH_NUM_BUFF_POOLS) || - WARN_ON(index >= adapter->rx_buff_pool[0][pool].size)) { - schedule_work(&adapter->work); - return -EINVAL; + spin_lock_irqsave(&rxq->replenish_lock, flags); + + if (!ibmveth_rxq_correlator_valid(adapter, queue_index, correlator)) { + rc = -EINVAL; + goto out_unlock; } - skb = adapter->rx_buff_pool[0][pool].skbuff[index]; - if (WARN_ON(!skb)) { - schedule_work(&adapter->work); - return -EFAULT; + skb = adapter->rx_buff_pool[queue_index][pool].skbuff[index]; + if (!skb) { + rc = -EFAULT; + goto out_unlock; } /* if we are going to reuse the buffer then keep the pointers around @@ -1109,75 +1309,88 @@ static int ibmveth_remove_buffer_from_pool(struct ibmveth_adapter *adapter, /* remove the skb pointer to mark free. actual freeing is done * by upper level networking after gro_receive */ - adapter->rx_buff_pool[0][pool].skbuff[index] = NULL; + struct ibmveth_buff_pool *bpool = + &adapter->rx_buff_pool[queue_index][pool]; + + bpool->skbuff[index] = NULL; dma_unmap_single(&adapter->vdev->dev, - adapter->rx_buff_pool[0][pool].dma_addr[index], - adapter->rx_buff_pool[0][pool].buff_size, + bpool->dma_addr[index], + bpool->buff_size, DMA_FROM_DEVICE); } - free_index = adapter->rx_buff_pool[0][pool].producer_index; - adapter->rx_buff_pool[0][pool].producer_index++; - if (adapter->rx_buff_pool[0][pool].producer_index >= - adapter->rx_buff_pool[0][pool].size) - adapter->rx_buff_pool[0][pool].producer_index = 0; - adapter->rx_buff_pool[0][pool].free_map[free_index] = index; + free_index = adapter->rx_buff_pool[queue_index][pool].producer_index; + adapter->rx_buff_pool[queue_index][pool].producer_index++; + if (adapter->rx_buff_pool[queue_index][pool].producer_index >= + adapter->rx_buff_pool[queue_index][pool].size) + adapter->rx_buff_pool[queue_index][pool].producer_index = 0; + adapter->rx_buff_pool[queue_index][pool].free_map[free_index] = index; mb(); - atomic_dec(&adapter->rx_buff_pool[0][pool].available); + atomic_dec(&adapter->rx_buff_pool[queue_index][pool].available); - return 0; +out_unlock: + spin_unlock_irqrestore(&rxq->replenish_lock, flags); + return rc; } /* get the current buffer on the rx queue */ -static inline struct sk_buff *ibmveth_rxq_get_buffer(struct ibmveth_adapter *adapter) +static inline struct sk_buff * +ibmveth_rxq_get_buffer(struct ibmveth_adapter *adapter, + int queue_index) { - struct ibmveth_rx_q *rxq = &adapter->rx_queue[0]; + struct ibmveth_rx_q *rxq = &adapter->rx_queue[queue_index]; u64 correlator = rxq->queue_addr[rxq->index].correlator; unsigned int pool = correlator >> 32; unsigned int index = correlator & 0xffffffffUL; - if (WARN_ON(pool >= IBMVETH_NUM_BUFF_POOLS) || - WARN_ON(index >= adapter->rx_buff_pool[0][pool].size)) { + if (!ibmveth_rxq_correlator_valid(adapter, queue_index, correlator)) { schedule_work(&adapter->work); return NULL; } - return adapter->rx_buff_pool[0][pool].skbuff[index]; + return adapter->rx_buff_pool[queue_index][pool].skbuff[index]; } /** * ibmveth_rxq_harvest_buffer - Harvest buffer from pool * * @adapter: pointer to adapter + * @queue_index: RX queue index to harvest from * @reuse: whether to reuse buffer * * Context: called from ibmveth_poll * + * On a bad correlator (-EINVAL/-EFAULT) the ring is still advanced so poll + * cannot spin forever on one slot. The error is still returned: callers must + * not treat it as a successful take from the pool (especially reuse=false, + * which would hand the SKB to the stack while it remains pool-owned). + * * Return: - * * %0 - success - * * other - non-zero return from ibmveth_remove_buffer_from_pool + * * %0 - buffer removed from pool (or marked for reuse) and ring advanced + * * other - non-zero return from ibmveth_remove_buffer_from_pool; ring has + * still been advanced for -EINVAL/-EFAULT */ static int ibmveth_rxq_harvest_buffer(struct ibmveth_adapter *adapter, - bool reuse) + int queue_index, bool reuse) { + struct ibmveth_rx_q *rxq = &adapter->rx_queue[queue_index]; u64 cor; int rc; - struct ibmveth_rx_q *rxq = &adapter->rx_queue[0]; - cor = rxq->queue_addr[rxq->index].correlator; - rc = ibmveth_remove_buffer_from_pool(adapter, cor, reuse); - if (unlikely(rc)) + rc = ibmveth_remove_buffer_from_pool(adapter, cor, queue_index, reuse); + if (unlikely(rc)) { + /* Skip a corrupt slot without claiming pool ownership. */ + if (rc == -EINVAL || rc == -EFAULT) + ibmveth_rxq_advance(rxq); return rc; - - if (++adapter->rx_queue[0].index == adapter->rx_queue[0].num_slots) { - adapter->rx_queue[0].index = 0; - adapter->rx_queue[0].toggle = !adapter->rx_queue[0].toggle; } + ibmveth_rxq_advance(rxq); + return 0; } @@ -2127,34 +2340,41 @@ static void ibmveth_rx_csum_helper(struct sk_buff *skb, static int ibmveth_poll(struct napi_struct *napi, int budget) { - struct ibmveth_adapter *adapter = - container_of(napi, struct ibmveth_adapter, napi[0]); - struct net_device *netdev = adapter->netdev; + struct net_device *netdev = napi->dev; + struct ibmveth_adapter *adapter = netdev_priv(netdev); int frames_processed = 0; - int rc; + int queue_index, rc; u16 mss = 0; + queue_index = napi - adapter->napi; + restart_poll: while (frames_processed < budget) { - if (!ibmveth_rxq_pending_buffer(adapter)) + if (!ibmveth_rxq_pending_buffer(adapter, queue_index)) break; smp_rmb(); - if (!ibmveth_rxq_buffer_valid(adapter)) { + if (!ibmveth_rxq_buffer_valid(adapter, queue_index)) { wmb(); /* suggested by larson1 */ adapter->rx_invalid_buffer++; netdev_dbg(netdev, "recycling invalid buffer\n"); - if (unlikely(ibmveth_rxq_harvest_buffer(adapter, true))) + rc = ibmveth_rxq_harvest_buffer(adapter, + queue_index, true); + if (unlikely(rc)) break; } else { struct sk_buff *skb, *new_skb; - int length = ibmveth_rxq_frame_length(adapter); - int offset = ibmveth_rxq_frame_offset(adapter); - int csum_good = ibmveth_rxq_csum_good(adapter); - int lrg_pkt = ibmveth_rxq_large_packet(adapter); + int length = ibmveth_rxq_frame_length(adapter, + queue_index); + int offset = ibmveth_rxq_frame_offset(adapter, + queue_index); + int csum_good = ibmveth_rxq_csum_good(adapter, + queue_index); + int lrg_pkt = ibmveth_rxq_large_packet(adapter, + queue_index); __sum16 iph_check = 0; - skb = ibmveth_rxq_get_buffer(adapter); + skb = ibmveth_rxq_get_buffer(adapter, queue_index); if (unlikely(!skb)) break; @@ -2179,12 +2399,18 @@ static int ibmveth_poll(struct napi_struct *napi, int budget) length); if (rx_flush) ibmveth_flush_buffer(skb->data, - length + offset); - if (unlikely(ibmveth_rxq_harvest_buffer(adapter, true))) + length + offset); + rc = ibmveth_rxq_harvest_buffer(adapter, + queue_index, + true); + if (unlikely(rc)) break; skb = new_skb; } else { - if (unlikely(ibmveth_rxq_harvest_buffer(adapter, false))) + rc = ibmveth_rxq_harvest_buffer(adapter, + queue_index, + false); + if (unlikely(rc)) break; skb_reserve(skb, offset); } @@ -2220,7 +2446,7 @@ static int ibmveth_poll(struct napi_struct *napi, int budget) } } - ibmveth_replenish_task(adapter); + ibmveth_replenish_task(adapter, queue_index); if (frames_processed == budget) goto out; @@ -2231,14 +2457,18 @@ static int ibmveth_poll(struct napi_struct *napi, int budget) /* We think we are done - reenable interrupts, * then check once more to make sure we are done. */ - rc = ibmveth_enable_irq(adapter, 0); - if (WARN_ON(rc)) { + rc = ibmveth_enable_irq(adapter, queue_index); + if (rc) { + netdev_err(netdev, + "Failed to enable IRQ for queue %d (rc=%d), scheduling reset\n", + queue_index, rc); schedule_work(&adapter->work); goto out; } - if (ibmveth_rxq_pending_buffer(adapter) && napi_schedule(napi)) { - rc = ibmveth_disable_irq(adapter, 0); + if (ibmveth_rxq_pending_buffer(adapter, queue_index) && + napi_schedule(napi)) { + rc = ibmveth_disable_irq(adapter, queue_index); WARN_ON(rc); goto restart_poll; } @@ -2369,7 +2599,7 @@ static void ibmveth_poll_controller(struct net_device *dev) { struct ibmveth_adapter *adapter = netdev_priv(dev); - ibmveth_replenish_task(adapter); + ibmveth_replenish_task(adapter, 0); ibmveth_schedule_rx_queue(adapter, 0); } #endif @@ -2568,7 +2798,7 @@ static int ibmveth_probe(struct vio_dev *dev, const struct vio_device_id *id) if (ret == H_SUCCESS && (ret_attr & IBMVETH_ILLAN_RX_MULTI_BUFF_SUPPORT)) { - adapter->rx_buffers_per_hcall = IBMVETH_MAX_RX_PER_HCALL; + adapter->rx_buffers_per_hcall = IBMVETH_MAX_RX_REGULAR; netdev_dbg(netdev, "RX Multi-buffer hcall supported by FW, batch set to %u\n", adapter->rx_buffers_per_hcall); @@ -2889,8 +3119,7 @@ static void ibmveth_reset_kunit(struct work_struct *w) * @test: pointer to kunit structure * * Tests the error returns from ibmveth_remove_buffer_from_pool. - * ibmveth_remove_buffer_from_pool also calls WARN_ON, so dmesg should be - * checked to see that these warnings happened. + * Bad correlators return -EINVAL/-EFAULT (no WARN_ON). * * Return: void */ @@ -2904,6 +3133,8 @@ static void ibmveth_remove_buffer_from_pool_test(struct kunit *test) INIT_WORK(&adapter->work, ibmveth_reset_kunit); + spin_lock_init(&adapter->rx_queue[0].replenish_lock); + /* Set sane values for buffer pools */ for (int i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) ibmveth_init_buffer_pool(&adapter->rx_buff_pool[0][i], i, @@ -2915,17 +3146,29 @@ static void ibmveth_remove_buffer_from_pool_test(struct kunit *test) KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pool->skbuff); correlator = ((u64)IBMVETH_NUM_BUFF_POOLS << 32) | 0; - KUNIT_EXPECT_EQ(test, -EINVAL, ibmveth_remove_buffer_from_pool(adapter, correlator, false)); - KUNIT_EXPECT_EQ(test, -EINVAL, ibmveth_remove_buffer_from_pool(adapter, correlator, true)); + KUNIT_EXPECT_EQ(test, -EINVAL, + ibmveth_remove_buffer_from_pool(adapter, + correlator, 0, false)); + KUNIT_EXPECT_EQ(test, -EINVAL, + ibmveth_remove_buffer_from_pool(adapter, + correlator, 0, true)); correlator = ((u64)0 << 32) | adapter->rx_buff_pool[0][0].size; - KUNIT_EXPECT_EQ(test, -EINVAL, ibmveth_remove_buffer_from_pool(adapter, correlator, false)); - KUNIT_EXPECT_EQ(test, -EINVAL, ibmveth_remove_buffer_from_pool(adapter, correlator, true)); + KUNIT_EXPECT_EQ(test, -EINVAL, + ibmveth_remove_buffer_from_pool(adapter, + correlator, 0, false)); + KUNIT_EXPECT_EQ(test, -EINVAL, + ibmveth_remove_buffer_from_pool(adapter, + correlator, 0, true)); correlator = (u64)0 | 0; pool->skbuff[0] = NULL; - KUNIT_EXPECT_EQ(test, -EFAULT, ibmveth_remove_buffer_from_pool(adapter, correlator, false)); - KUNIT_EXPECT_EQ(test, -EFAULT, ibmveth_remove_buffer_from_pool(adapter, correlator, true)); + KUNIT_EXPECT_EQ(test, -EFAULT, + ibmveth_remove_buffer_from_pool(adapter, + correlator, 0, false)); + KUNIT_EXPECT_EQ(test, -EFAULT, + ibmveth_remove_buffer_from_pool(adapter, + correlator, 0, true)); flush_work(&adapter->work); } @@ -2934,9 +3177,7 @@ static void ibmveth_remove_buffer_from_pool_test(struct kunit *test) * ibmveth_rxq_get_buffer_test - unit test for ibmveth_rxq_get_buffer * @test: pointer to kunit structure * - * Tests ibmveth_rxq_get_buffer. ibmveth_rxq_get_buffer also calls WARN_ON for - * the NULL returns, so dmesg should be checked to see that these warnings - * happened. + * Tests ibmveth_rxq_get_buffer invalid correlator returns NULL without WARN. * * Return: void */ @@ -2970,15 +3211,15 @@ static void ibmveth_rxq_get_buffer_test(struct kunit *test) adapter->rx_queue[0].queue_addr[0].correlator = (u64)IBMVETH_NUM_BUFF_POOLS << 32 | 0; - KUNIT_EXPECT_PTR_EQ(test, NULL, ibmveth_rxq_get_buffer(adapter)); + KUNIT_EXPECT_PTR_EQ(test, NULL, ibmveth_rxq_get_buffer(adapter, 0)); adapter->rx_queue[0].queue_addr[0].correlator = (u64)0 << 32 | adapter->rx_buff_pool[0][0].size; - KUNIT_EXPECT_PTR_EQ(test, NULL, ibmveth_rxq_get_buffer(adapter)); + KUNIT_EXPECT_PTR_EQ(test, NULL, ibmveth_rxq_get_buffer(adapter, 0)); pool->skbuff[0] = skb; adapter->rx_queue[0].queue_addr[0].correlator = (u64)0 << 32 | 0; - KUNIT_EXPECT_PTR_EQ(test, skb, ibmveth_rxq_get_buffer(adapter)); + KUNIT_EXPECT_PTR_EQ(test, skb, ibmveth_rxq_get_buffer(adapter, 0)); flush_work(&adapter->work); } diff --git a/drivers/net/ethernet/ibm/ibmveth.h b/drivers/net/ethernet/ibm/ibmveth.h index fae3473cc498..d02444d5b3b8 100644 --- a/drivers/net/ethernet/ibm/ibmveth.h +++ b/drivers/net/ethernet/ibm/ibmveth.h @@ -14,6 +14,8 @@ #ifndef _IBMVETH_H #define _IBMVETH_H +#include + /* constants for H_MULTICAST_CTRL */ #define IbmVethMcastReceptionModifyBit 0x80000UL #define IbmVethMcastReceptionEnableBit 0x20000UL @@ -259,7 +261,8 @@ static inline long h_illan_attributes(unsigned long unit_address, #define IBMVETH_DEFAULT_QUEUES 8U #define IBMVETH_MAX_RX_QUEUES 1U #define IBMVETH_DEFAULT_RX_QUEUES 1U -#define IBMVETH_MAX_RX_PER_HCALL 8U +#define IBMVETH_MAX_RX_REGULAR 8U +#define IBMVETH_MAX_RX_PER_HCALL 12U static int pool_size[] = { 512, 1024 * 2, 1024 * 16, 1024 * 32, 1024 * 64 }; static int pool_count[] = { 256, 512, 256, 256, 256 }; @@ -301,6 +304,7 @@ struct ibmveth_rx_q { dma_addr_t queue_dma; u32 queue_len; struct ibmveth_rx_q_entry *queue_addr; + spinlock_t replenish_lock; /* per-queue buffer replenish */ }; struct ibmveth_adapter { -- 2.50.1 (Apple Git-155)