[PATCH] opensm: fix SMInfo delayed response sending due to multiple timeouts in the fabric.

[Alex Netes <alexne-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>]

The issue is resolved by adding High priority unicast FIFO queue to the VL15
poller thread.

Signed-off-by: Vladimir Koushnir <vladimirk-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
Signed-off-by: Alex Netes <alexne-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
---
 include/opensm/osm_sm.h       |    7 +++++--
 include/opensm/osm_vl15intf.h |   15 +++++++++++++--
 opensm/osm_req.c              |    6 +++---
 opensm/osm_resp.c             |    5 +++--
 opensm/osm_sm_mad_ctrl.c      |    2 +-
 opensm/osm_sminfo_rcv.c       |    8 ++++----
 opensm/osm_trap_rcv.c         |    2 +-
 opensm/osm_vl15intf.c         |   36 +++++++++++++++++++++++++++++-------
 8 files changed, 59 insertions(+), 22 deletions(-)

diff --git a/include/opensm/osm_sm.h b/include/opensm/osm_sm.h
index 4c7b120..17d6afd 100644
--- a/include/opensm/osm_sm.h
+++ b/include/opensm/osm_sm.h
@@ -495,7 +495,8 @@ ib_api_status_t osm_req_set(IN osm_sm_t * sm, IN const osm_dr_path_t * p_path,
 ib_api_status_t osm_resp_send(IN osm_sm_t * sm,
 			      IN const osm_madw_t * p_req_madw,
 			      IN ib_net16_t status,
-			      IN const uint8_t * p_payload);
+			      IN const uint8_t * p_payload,
+			      IN boolean_t force_flag);
 /*
 * PARAMETERS
 *	p_resp
@@ -510,7 +511,9 @@ ib_api_status_t osm_resp_send(IN osm_sm_t * sm,
 *
 *	p_payload
 *		[in] Pointer to the payload of the response MAD.
-*
+*	force_flag
+*		[in] If TRUE, indicates that the response should be processed immediately
+*		by VL15 poller thread
 * RETURN VALUES
 *	IB_SUCCESS if the response was successful.
 *
diff --git a/include/opensm/osm_vl15intf.h b/include/opensm/osm_vl15intf.h
index e621c68..a600de6 100644
--- a/include/opensm/osm_vl15intf.h
+++ b/include/opensm/osm_vl15intf.h
@@ -123,6 +123,7 @@ typedef struct osm_vl15 {
 	cl_thread_t poller;
 	cl_qlist_t rfifo;
 	cl_qlist_t ufifo;
+	cl_qlist_t hp_ufifo;
 	cl_spinlock_t lock;
 	osm_vendor_t *p_vend;
 	osm_log_t *p_log;
@@ -156,6 +157,10 @@ typedef struct osm_vl15 {
 *		First-in First-out queue for outbound VL15 MADs for which
 *		no response is expected, aka the "unicast fifo".
 *
+*	hp_ufifo
+*		First-in First-out high priority queue for outbound VL15 MADs for which
+*		no response is expected. SMInfo response MADs are placed in this queue
+*
 *	poller
 *		Worker thread pool that services the fifo to transmit VL15 MADs
 *
@@ -298,7 +303,7 @@ ib_api_status_t osm_vl15_init(IN osm_vl15_t * p_vl15, IN osm_vendor_t * p_vend,
 *
 * SYNOPSIS
 */
-void osm_vl15_post(IN osm_vl15_t * p_vl15, IN osm_madw_t * p_madw);
+void osm_vl15_post(IN osm_vl15_t * p_vl15, IN osm_madw_t * p_madw, IN boolean_t force_flag);
 /*
 * PARAMETERS
 *	p_vl15
@@ -307,6 +312,9 @@ void osm_vl15_post(IN osm_vl15_t * p_vl15, IN osm_madw_t * p_madw);
 *	p_madw
 *		[in] Pointer to a MAD wrapper structure containing the MAD.
 *
+*	force_flag
+*		[in] If set to TRUE, high priority ucast queue should be used
+*
 * RETURN VALUES
 *	This function does not return a value.
 *
@@ -327,12 +335,15 @@ void osm_vl15_post(IN osm_vl15_t * p_vl15, IN osm_madw_t * p_madw);
 *
 * SYNOPSIS
 */
-void osm_vl15_poll(IN osm_vl15_t * p_vl);
+void osm_vl15_poll(IN osm_vl15_t * p_vl, IN boolean_t force_flag);
 /*
 * PARAMETERS
 *	p_vl15
 *		[in] Pointer to an osm_vl15_t object.
 *
+*	force_flag
+*		[in] If set to TRUE, poller thread should be signalled immediately
+*
 * RETURN VALUES
 *	None.
 *
diff --git a/opensm/osm_req.c b/opensm/osm_req.c
index 7e9e664..3728327 100644
--- a/opensm/osm_req.c
+++ b/opensm/osm_req.c
@@ -117,7 +117,7 @@ ib_api_status_t osm_req_get(IN osm_sm_t * sm, IN const osm_dr_path_t * p_path,
 	if (p_context)
 		p_madw->context = *p_context;
 
-	osm_vl15_post(sm->p_vl15, p_madw);
+	osm_vl15_post(sm->p_vl15, p_madw, FALSE);
 
 Exit:
 	OSM_LOG_EXIT(sm->p_log);
@@ -189,7 +189,7 @@ ib_api_status_t osm_req_set(IN osm_sm_t * sm, IN const osm_dr_path_t * p_path,
 
 	memcpy(osm_madw_get_smp_ptr(p_madw)->data, p_payload, payload_size);
 
-	osm_vl15_post(sm->p_vl15, p_madw);
+	osm_vl15_post(sm->p_vl15, p_madw, FALSE);
 
 Exit:
 	OSM_LOG_EXIT(sm->p_log);
@@ -267,7 +267,7 @@ int osm_send_trap144(osm_sm_t * sm, ib_net16_t local)
 		"Sending Trap 144, TID 0x%" PRIx64 " to SM lid %u\n",
 		cl_ntoh64(smp->trans_id), cl_ntoh16(madw->mad_addr.dest_lid));
 
-	osm_vl15_post(sm->p_vl15, madw);
+	osm_vl15_post(sm->p_vl15, madw, FALSE);
 
 	return 0;
 }
diff --git a/opensm/osm_resp.c b/opensm/osm_resp.c
index ee391b6..cc52b68 100644
--- a/opensm/osm_resp.c
+++ b/opensm/osm_resp.c
@@ -95,7 +95,8 @@ Exit:
 ib_api_status_t osm_resp_send(IN osm_sm_t * sm,
 			      IN const osm_madw_t * p_req_madw,
 			      IN ib_net16_t mad_status,
-			      IN const uint8_t * p_payload)
+			      IN const uint8_t * p_payload,
+			      IN boolean_t force_flag)
 {
 	const ib_smp_t *p_req_smp;
 	ib_smp_t *p_smp;
@@ -142,7 +143,7 @@ ib_api_status_t osm_resp_send(IN osm_sm_t * sm,
 		ib_get_sm_attr_str(p_smp->attr_id), cl_ntoh16(p_smp->attr_id),
 		cl_ntoh32(p_smp->attr_mod), cl_ntoh64(p_smp->trans_id));
 
-	osm_vl15_post(sm->p_vl15, p_madw);
+	osm_vl15_post(sm->p_vl15, p_madw, force_flag);
 
 Exit:
 	OSM_LOG_EXIT(sm->p_log);
diff --git a/opensm/osm_sm_mad_ctrl.c b/opensm/osm_sm_mad_ctrl.c
index f0bcff2..2ccfbfa 100644
--- a/opensm/osm_sm_mad_ctrl.c
+++ b/opensm/osm_sm_mad_ctrl.c
@@ -163,7 +163,7 @@ static void sm_mad_ctrl_update_wire_stats(IN osm_sm_mad_ctrl_t * p_ctrl)
 	   We can signal the VL15 controller to send another MAD
 	   if any are waiting for transmission.
 	 */
-	osm_vl15_poll(p_ctrl->p_vl15);
+	osm_vl15_poll(p_ctrl->p_vl15, FALSE);
 	OSM_LOG_EXIT(p_ctrl->p_log);
 }
 
diff --git a/opensm/osm_sminfo_rcv.c b/opensm/osm_sminfo_rcv.c
index e45d14e..6f0b099 100644
--- a/opensm/osm_sminfo_rcv.c
+++ b/opensm/osm_sminfo_rcv.c
@@ -111,7 +111,7 @@ static void smi_rcv_process_get_request(IN osm_sm_t * sm,
 		p_smi->sm_key = 0;
 	}
 
-	status = osm_resp_send(sm, p_madw, 0, payload);
+	status = osm_resp_send(sm, p_madw, 0, payload, TRUE);
 	if (status != IB_SUCCESS) {
 		OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 2F02: "
 			"Error sending response (%s)\n",
@@ -208,7 +208,7 @@ static void smi_rcv_process_set_request(IN osm_sm_t * sm,
 			"Check legality failed. AttributeModifier:0x%X RemoteState:%s\n",
 			p_smp->attr_mod,
 			osm_get_sm_mgr_state_str(ib_sminfo_get_state(sm_smi)));
-		status = osm_resp_send(sm, p_madw, 7, payload);
+		status = osm_resp_send(sm, p_madw, 7, payload, TRUE);
 		if (status != IB_SUCCESS)
 			OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 2F05: "
 				"Error sending response (%s)\n",
@@ -253,7 +253,7 @@ static void smi_rcv_process_set_request(IN osm_sm_t * sm,
 			"AttributeModifier:0x%X RemoteState:%s\n",
 			p_smp->attr_mod,
 			osm_get_sm_mgr_state_str(ib_sminfo_get_state(sm_smi)));
-		status = osm_resp_send(sm, p_madw, 7, payload);
+		status = osm_resp_send(sm, p_madw, 7, payload, TRUE);
 		if (status != IB_SUCCESS)
 			OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 2F08: "
 				"Error sending response (%s)\n",
@@ -263,7 +263,7 @@ static void smi_rcv_process_set_request(IN osm_sm_t * sm,
 	}
 
 	/* the SubnSet(SMInfo) command is ok. Send a response. */
-	status = osm_resp_send(sm, p_madw, 0, payload);
+	status = osm_resp_send(sm, p_madw, 0, payload, TRUE);
 	if (status != IB_SUCCESS)
 		OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 2F09: "
 			"Error sending response (%s)\n",
diff --git a/opensm/osm_trap_rcv.c b/opensm/osm_trap_rcv.c
index 2aaaba4..132c510 100644
--- a/opensm/osm_trap_rcv.c
+++ b/opensm/osm_trap_rcv.c
@@ -410,7 +410,7 @@ static void trap_rcv_process_request(IN osm_sm_t * sm,
 		OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3809: "
 			"Failed to find source physical port for trap\n");
 
-	status = osm_resp_send(sm, &tmp_madw, 0, payload);
+	status = osm_resp_send(sm, &tmp_madw, 0, payload, FALSE);
 	if (status != IB_SUCCESS) {
 		OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3802: "
 			"Error sending response (%s)\n",
diff --git a/opensm/osm_vl15intf.c b/opensm/osm_vl15intf.c
index c845597..5019f64 100644
--- a/opensm/osm_vl15intf.c
+++ b/opensm/osm_vl15intf.c
@@ -131,9 +131,12 @@ static void vl15_poller(IN void *p_ptr)
 		   The unicast FIFO has priority, since somebody is waiting
 		   for a timely response.
 		 */
+send_next:
 		cl_spinlock_acquire(&p_vl->lock);
 
-		if (cl_qlist_count(&p_vl->ufifo) != 0)
+		if (cl_qlist_count(&p_vl->hp_ufifo) != 0)
+			p_fifo = &p_vl->hp_ufifo;
+		else if (cl_qlist_count(&p_vl->ufifo) != 0)
 			p_fifo = &p_vl->ufifo;
 		else
 			p_fifo = &p_vl->rfifo;
@@ -151,6 +154,9 @@ static void vl15_poller(IN void *p_ptr)
 						OSM_LOG_FRAMES);
 
 			vl15_send_mad(p_vl, p_madw);
+			/* we should always drain the high priority queue first */
+			if (p_fifo == &p_vl->hp_ufifo)
+				goto send_next;
 		} else
 			/*
 			   The VL15 FIFO is empty, so we have nothing left to do.
@@ -195,6 +201,7 @@ void osm_vl15_construct(IN osm_vl15_t * p_vl)
 	cl_spinlock_construct(&p_vl->lock);
 	cl_qlist_init(&p_vl->rfifo);
 	cl_qlist_init(&p_vl->ufifo);
+	cl_qlist_init(&p_vl->hp_ufifo);
 	cl_thread_construct(&p_vl->poller);
 }
 
@@ -232,6 +239,10 @@ void osm_vl15_destroy(IN osm_vl15_t * p_vl, IN struct osm_mad_pool *p_pool)
 		p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->ufifo);
 		osm_mad_pool_put(p_pool, p_madw);
 	}
+	while (!cl_is_qlist_empty(&p_vl->hp_ufifo)) {
+		p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->hp_ufifo);
+		osm_mad_pool_put(p_pool, p_madw);
+	}
 
 	cl_spinlock_release(&p_vl->lock);
 
@@ -281,7 +292,7 @@ Exit:
 	return status;
 }
 
-void osm_vl15_poll(IN osm_vl15_t * p_vl)
+void osm_vl15_poll(IN osm_vl15_t * p_vl, IN boolean_t force_flag)
 {
 	OSM_LOG_ENTER(p_vl->p_log);
 
@@ -296,8 +307,8 @@ void osm_vl15_poll(IN osm_vl15_t * p_vl)
 	   the event here.  To cover this rare case, the poller
 	   thread checks for a spurious wake-up.
 	 */
-	if (p_vl->p_stats->qp0_mads_outstanding_on_wire <
-	    (int32_t) p_vl->max_wire_smps) {
+	if (force_flag || (p_vl->p_stats->qp0_mads_outstanding_on_wire <
+	    (int32_t) p_vl->max_wire_smps)) {
 		OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG,
 			"Signalling poller thread\n");
 		cl_event_signal(&p_vl->signal);
@@ -306,7 +317,7 @@ void osm_vl15_poll(IN osm_vl15_t * p_vl)
 	OSM_LOG_EXIT(p_vl->p_log);
 }
 
-void osm_vl15_post(IN osm_vl15_t * p_vl, IN osm_madw_t * p_madw)
+void osm_vl15_post(IN osm_vl15_t * p_vl, IN osm_madw_t * p_madw, boolean_t force_flag)
 {
 	OSM_LOG_ENTER(p_vl->p_log);
 
@@ -322,7 +333,8 @@ void osm_vl15_post(IN osm_vl15_t * p_vl, IN osm_madw_t * p_madw)
 		cl_qlist_insert_tail(&p_vl->rfifo, &p_madw->list_item);
 		osm_stats_inc_qp0_outstanding(p_vl->p_stats);
 	} else
-		cl_qlist_insert_tail(&p_vl->ufifo, &p_madw->list_item);
+		cl_qlist_insert_tail(force_flag ? &p_vl->hp_ufifo : &p_vl->ufifo,
+				     &p_madw->list_item);
 	cl_spinlock_release(&p_vl->lock);
 
 	OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG,
@@ -330,7 +342,7 @@ void osm_vl15_post(IN osm_vl15_t * p_vl, IN osm_madw_t * p_madw)
 		p_vl->p_stats->qp0_mads_outstanding_on_wire,
 		p_vl->p_stats->qp0_mads_outstanding);
 
-	osm_vl15_poll(p_vl);
+	osm_vl15_poll(p_vl, force_flag);
 
 	OSM_LOG_EXIT(p_vl->p_log);
 }
@@ -350,6 +362,16 @@ void osm_vl15_shutdown(IN osm_vl15_t * p_vl, IN osm_mad_pool_t * p_mad_pool)
 	/* go over all outstanding MADs and retire their transactions */
 
 	/* first we handle the list of response MADs */
+	p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->hp_ufifo);
+	while (p_madw != (osm_madw_t *) cl_qlist_end(&p_vl->hp_ufifo)) {
+		OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG,
+			"Releasing High Priority Response p_madw = %p\n", p_madw);
+
+		osm_mad_pool_put(p_mad_pool, p_madw);
+
+		p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->hp_ufifo);
+	}
+
 	p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->ufifo);
 	while (p_madw != (osm_madw_t *) cl_qlist_end(&p_vl->ufifo)) {
 		OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG,
-- 
1.7.7.6


-- 

-- Alex
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html