From: "Bryan O'Sullivan" <bos@pathscale.com>
To: rdreier@cisco.com
Cc: linux-kernel@vger.kernel.org, openib-general@openib.org
Subject: [PATCH 1 of 28] IB/ipath - limit # of packets sent without an ACK received
Date: Thu, 28 Sep 2006 08:59:57 -0700 [thread overview]
Message-ID: <c46292ccb0f54abc77f7.1159459197@eng-12.pathscale.com> (raw)
In-Reply-To: <patchbomb.1159459196@eng-12.pathscale.com>
The sender requests an ACK every 1/2 MB to avoid retransmit timeouts that
were causing MVAPICH mod_bw to fail after a predictable number of sends.
Signed-off-by: Bryan O'Sullivan <bryan.osullivan@qlogic.com>
diff -r f1b431dca1f9 -r c46292ccb0f5 drivers/infiniband/hw/ipath/ipath_qp.c
--- a/drivers/infiniband/hw/ipath/ipath_qp.c Thu Sep 28 08:57:12 2006 -0700
+++ b/drivers/infiniband/hw/ipath/ipath_qp.c Thu Sep 28 08:57:12 2006 -0700
@@ -342,6 +342,7 @@ static void ipath_reset_qp(struct ipath_
qp->s_last = 0;
qp->s_ssn = 1;
qp->s_lsn = 0;
+ qp->s_wait_credit = 0;
if (qp->r_rq.wq) {
qp->r_rq.wq->head = 0;
qp->r_rq.wq->tail = 0;
@@ -516,7 +517,7 @@ int ipath_modify_qp(struct ib_qp *ibqp,
qp->remote_qpn = attr->dest_qp_num;
if (attr_mask & IB_QP_SQ_PSN) {
- qp->s_next_psn = attr->sq_psn;
+ qp->s_psn = qp->s_next_psn = attr->sq_psn;
qp->s_last_psn = qp->s_next_psn - 1;
}
diff -r f1b431dca1f9 -r c46292ccb0f5 drivers/infiniband/hw/ipath/ipath_rc.c
--- a/drivers/infiniband/hw/ipath/ipath_rc.c Thu Sep 28 08:57:12 2006 -0700
+++ b/drivers/infiniband/hw/ipath/ipath_rc.c Thu Sep 28 08:57:12 2006 -0700
@@ -201,6 +201,18 @@ int ipath_make_rc_req(struct ipath_qp *q
qp->s_rnr_timeout)
goto done;
+ /* Limit the number of packets sent without an ACK. */
+ if (ipath_cmp24(qp->s_psn, qp->s_last_psn + IPATH_PSN_CREDIT) > 0) {
+ qp->s_wait_credit = 1;
+ dev->n_rc_stalls++;
+ spin_lock(&dev->pending_lock);
+ if (list_empty(&qp->timerwait))
+ list_add_tail(&qp->timerwait,
+ &dev->pending[dev->pending_index]);
+ spin_unlock(&dev->pending_lock);
+ goto done;
+ }
+
/* header size in 32-bit words LRH+BTH = (8+12)/4. */
hwords = 5;
bth0 = 0;
@@ -221,7 +233,7 @@ int ipath_make_rc_req(struct ipath_qp *q
/* Check if send work queue is empty. */
if (qp->s_tail == qp->s_head)
goto done;
- qp->s_psn = wqe->psn = qp->s_next_psn;
+ wqe->psn = qp->s_next_psn;
newreq = 1;
}
/*
@@ -393,12 +405,6 @@ int ipath_make_rc_req(struct ipath_qp *q
ss = &qp->s_sge;
len = qp->s_len;
if (len > pmtu) {
- /*
- * Request an ACK every 1/2 MB to avoid retransmit
- * timeouts.
- */
- if (((wqe->length - len) % (512 * 1024)) == 0)
- bth2 |= 1 << 31;
len = pmtu;
break;
}
@@ -435,12 +441,6 @@ int ipath_make_rc_req(struct ipath_qp *q
ss = &qp->s_sge;
len = qp->s_len;
if (len > pmtu) {
- /*
- * Request an ACK every 1/2 MB to avoid retransmit
- * timeouts.
- */
- if (((wqe->length - len) % (512 * 1024)) == 0)
- bth2 |= 1 << 31;
len = pmtu;
break;
}
@@ -498,6 +498,8 @@ int ipath_make_rc_req(struct ipath_qp *q
*/
goto done;
}
+ if (ipath_cmp24(qp->s_psn, qp->s_last_psn + IPATH_PSN_CREDIT - 1) >= 0)
+ bth2 |= 1 << 31; /* Request ACK. */
qp->s_len -= len;
qp->s_hdrwords = hwords;
qp->s_cur_sge = ss;
@@ -737,6 +739,15 @@ bail:
return;
}
+static inline void update_last_psn(struct ipath_qp *qp, u32 psn)
+{
+ if (qp->s_wait_credit) {
+ qp->s_wait_credit = 0;
+ tasklet_hi_schedule(&qp->s_task);
+ }
+ qp->s_last_psn = psn;
+}
+
/**
* do_rc_ack - process an incoming RC ACK
* @qp: the QP the ACK came in on
@@ -805,7 +816,7 @@ static int do_rc_ack(struct ipath_qp *qp
* The last valid PSN seen is the previous
* request's.
*/
- qp->s_last_psn = wqe->psn - 1;
+ update_last_psn(qp, wqe->psn - 1);
/* Retry this request. */
ipath_restart_rc(qp, wqe->psn, &wc);
/*
@@ -864,7 +875,7 @@ static int do_rc_ack(struct ipath_qp *qp
ipath_get_credit(qp, aeth);
qp->s_rnr_retry = qp->s_rnr_retry_cnt;
qp->s_retry = qp->s_retry_cnt;
- qp->s_last_psn = psn;
+ update_last_psn(qp, psn);
ret = 1;
goto bail;
@@ -883,7 +894,7 @@ static int do_rc_ack(struct ipath_qp *qp
goto bail;
/* The last valid PSN is the previous PSN. */
- qp->s_last_psn = psn - 1;
+ update_last_psn(qp, psn - 1);
dev->n_rc_resends += (int)qp->s_psn - (int)psn;
@@ -898,7 +909,7 @@ static int do_rc_ack(struct ipath_qp *qp
case 3: /* NAK */
/* The last valid PSN seen is the previous request's. */
if (qp->s_last != qp->s_tail)
- qp->s_last_psn = wqe->psn - 1;
+ update_last_psn(qp, wqe->psn - 1);
switch ((aeth >> IPATH_AETH_CREDIT_SHIFT) &
IPATH_AETH_CREDIT_MASK) {
case 0: /* PSN sequence error */
@@ -1071,7 +1082,7 @@ static inline void ipath_rc_rcv_resp(str
* since we don't want s_sge modified.
*/
qp->s_len -= pmtu;
- qp->s_last_psn = psn;
+ update_last_psn(qp, psn);
spin_unlock_irqrestore(&qp->s_lock, flags);
ipath_copy_sge(&qp->s_sge, data, pmtu);
goto bail;
diff -r f1b431dca1f9 -r c46292ccb0f5 drivers/infiniband/hw/ipath/ipath_verbs.c
--- a/drivers/infiniband/hw/ipath/ipath_verbs.c Thu Sep 28 08:57:12 2006 -0700
+++ b/drivers/infiniband/hw/ipath/ipath_verbs.c Thu Sep 28 08:57:12 2006 -0700
@@ -1683,6 +1683,7 @@ static ssize_t show_stats(struct class_d
"RC OTH NAKs %d\n"
"RC timeouts %d\n"
"RC RDMA dup %d\n"
+ "RC stalls %d\n"
"piobuf wait %d\n"
"no piobuf %d\n"
"PKT drops %d\n"
@@ -1690,7 +1691,7 @@ static ssize_t show_stats(struct class_d
dev->n_rc_resends, dev->n_rc_qacks, dev->n_rc_acks,
dev->n_seq_naks, dev->n_rdma_seq, dev->n_rnr_naks,
dev->n_other_naks, dev->n_timeouts,
- dev->n_rdma_dup_busy, dev->n_piowait,
+ dev->n_rdma_dup_busy, dev->n_rc_stalls, dev->n_piowait,
dev->n_no_piobuf, dev->n_pkt_drops, dev->n_wqe_errs);
for (i = 0; i < ARRAY_SIZE(dev->opstats); i++) {
const struct ipath_opcode_stats *si = &dev->opstats[i];
diff -r f1b431dca1f9 -r c46292ccb0f5 drivers/infiniband/hw/ipath/ipath_verbs.h
--- a/drivers/infiniband/hw/ipath/ipath_verbs.h Thu Sep 28 08:57:12 2006 -0700
+++ b/drivers/infiniband/hw/ipath/ipath_verbs.h Thu Sep 28 08:57:12 2006 -0700
@@ -370,6 +370,7 @@ struct ipath_qp {
u8 s_rnr_retry_cnt;
u8 s_retry; /* requester retry counter */
u8 s_rnr_retry; /* requester RNR retry counter */
+ u8 s_wait_credit; /* limit number of unacked packets sent */
u8 s_pkey_index; /* PKEY index to use */
u8 timeout; /* Timeout for this QP */
enum ib_mtu path_mtu;
@@ -392,6 +393,8 @@ struct ipath_qp {
*/
#define IPATH_S_BUSY 0
#define IPATH_S_SIGNAL_REQ_WR 1
+
+#define IPATH_PSN_CREDIT 2048
/*
* Since struct ipath_swqe is not a fixed size, we can't simply index into
@@ -521,6 +524,7 @@ struct ipath_ibdev {
u32 n_rnr_naks;
u32 n_other_naks;
u32 n_timeouts;
+ u32 n_rc_stalls;
u32 n_pkt_drops;
u32 n_vl15_dropped;
u32 n_wqe_errs;
next prev parent reply other threads:[~2006-09-28 16:01 UTC|newest]
Thread overview: 39+ messages / expand[flat|nested] mbox.gz Atom feed top
2006-09-28 15:59 [PATCH 0 of 28] ipath patches for 2.6.19 Bryan O'Sullivan
2006-09-28 15:59 ` Bryan O'Sullivan [this message]
2006-09-28 18:16 ` [PATCH 1 of 28] IB/ipath - limit # of packets sent without an ACK received Roland Dreier
2006-09-28 18:33 ` Bryan O'Sullivan
2006-09-28 15:59 ` [PATCH 2 of 28] IB/ipath - fix memory leak if allocation fails Bryan O'Sullivan
2006-09-28 15:59 ` [PATCH 3 of 28] IB/ipath - driver support for userspace sharing of HW contexts Bryan O'Sullivan
2006-09-28 16:00 ` [PATCH 4 of 28] IB/ipath - support revision 2 InfiniPath PCIE devices Bryan O'Sullivan
2006-09-28 18:15 ` Roland Dreier
2006-09-28 16:00 ` [PATCH 5 of 28] IB/ipath - unregister from IB core early Bryan O'Sullivan
2006-09-28 16:00 ` [PATCH 6 of 28] IB/ipath - clean up handling of GUID 0 Bryan O'Sullivan
2006-09-28 16:00 ` [PATCH 7 of 28] IB/ipath - lock and count allocated CQs properly Bryan O'Sullivan
2006-09-28 16:00 ` [PATCH 8 of 28] IB/ipath - count SRQs properly Bryan O'Sullivan
2006-09-28 16:00 ` [PATCH 9 of 28] IB/ipath - only allow complete writes to flash Bryan O'Sullivan
2006-09-28 16:00 ` [PATCH 10 of 28] IB/ipath - RC and UC should validate SLID and DLID Bryan O'Sullivan
2006-09-28 16:00 ` [PATCH 11 of 28] IB/ipath - ensure that PD of MR matches PD of QP checking the Rkey Bryan O'Sullivan
2006-09-28 16:00 ` [PATCH 12 of 28] IB/ipath - print more informative parity error messages Bryan O'Sullivan
2006-09-28 16:00 ` [PATCH 13 of 28] IB/ipath - fix compiler warnings and errors on non-x86_64 systems Bryan O'Sullivan
2006-09-28 16:00 ` [PATCH 14 of 28] IB/ipath - Fix mismatch in shifts and masks for printing debug info Bryan O'Sullivan
2006-09-28 16:00 ` [PATCH 15 of 28] IB/ipath - support multiple simultaneous devices of different types Bryan O'Sullivan
2006-09-28 16:00 ` [PATCH 16 of 28] IB/ipath - drop unnecessary "(void *)" casts Bryan O'Sullivan
2006-09-28 16:00 ` [PATCH 17 of 28] IB/ipath - improved support for powerpc Bryan O'Sullivan
2006-09-28 16:00 ` [PATCH 18 of 28] IB/ipath - flush RWQEs if access error or invalid error seen Bryan O'Sullivan
2006-09-28 16:00 ` [PATCH 19 of 28] IB/ipath - call mtrr_del with correct arguments Bryan O'Sullivan
2006-09-28 16:00 ` [PATCH 20 of 28] IB/ipath - clean up module exit code Bryan O'Sullivan
2006-09-28 16:00 ` [PATCH 21 of 28] IB/ipath - change HT CRC message to indicate how to resolve problem Bryan O'Sullivan
2006-09-28 16:00 ` [PATCH 22 of 28] IB/ipath - fix and recover TXE piobuf and PBC parity errors Bryan O'Sullivan
2006-09-28 16:00 ` [PATCH 23 of 28] IB/ipath - fix EEPROM read when driver is compiled with -Os Bryan O'Sullivan
2006-09-28 16:00 ` [PATCH 24 of 28] IB/mthca - Fix compiler warnings with gcc4 on possible unitialized variables Bryan O'Sullivan
2006-09-28 18:14 ` Roland Dreier
2006-09-28 20:46 ` Jeremy Fitzhardinge
2006-09-28 16:00 ` [PATCH 25 of 28] IB/ipath - Set CPU affinity early Bryan O'Sullivan
2006-09-28 18:15 ` Roland Dreier
2006-09-28 16:00 ` [PATCH 26 of 28] IB/ipath - support new PCIE device, QLE7142 Bryan O'Sullivan
2006-09-28 16:00 ` [PATCH 27 of 28] IB/ipath - fix races with ib_resize_cq() Bryan O'Sullivan
2006-09-28 16:00 ` [PATCH 28 of 28] IB/ipath - fix lockdep error upon "ifconfig ibN down" Bryan O'Sullivan
2006-09-30 0:36 ` [PATCH 0 of 28] ipath patches for 2.6.19 Eric W. Biederman
2006-10-02 18:14 ` Bryan O'Sullivan
2006-10-03 1:22 ` Eric W. Biederman
2006-10-03 15:25 ` Bryan O'Sullivan
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=c46292ccb0f54abc77f7.1159459197@eng-12.pathscale.com \
--to=bos@pathscale.com \
--cc=linux-kernel@vger.kernel.org \
--cc=openib-general@openib.org \
--cc=rdreier@cisco.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.