[PATCH 2/7] RapidIO: Add switch locking during discovery

[PATCH 2/7] RapidIO: Add switch locking during discovery

[Alexandre Bounine]

From: Alexandre Bounine <alexandre.bounine@idt.com>

Add switch access locking during RapidIO discovery. Access lock is
required when reading switch routing table contents due to indexed mechanism
of RT addressing.

Signed-off-by: Alexandre Bounine <alexandre.bounine@idt.com>
Tested-by: Thomas Moll <thomas.moll@sysgo.com>
---

 rio-scan.c |   88 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-------
 1 files changed, 79 insertions(+), 9 deletions(-)

diff -x '*.pj' -X dontdiff_2.6.32-rc5 -pNur w33r7a/drivers/rapidio/rio-scan.c w33r7b/drivers/rapidio/rio-scan.c
--- w33r7a/drivers/rapidio/rio-scan.c	2010-02-09 15:11:09.141972000 -0500
+++ w33r7b/drivers/rapidio/rio-scan.c	2010-02-10 16:13:56.382802000 -0500
@@ -750,6 +750,7 @@ rio_disc_peer(struct rio_net *net, struc
 	int num_ports;
 	struct rio_dev *rdev;
 	u16 ndestid;
+	u32 result;
 
 	/* Setup new RIO device */
 	if ((rdev = rio_setup_device(net, port, destid, hopcount, 0))) {
@@ -778,6 +779,27 @@ rio_disc_peer(struct rio_net *net, struc
 				pr_debug(
 				    "RIO: scanning device on port %d\n",
 				    port_num);
+
+				/* Attempt to acquire device lock */
+				rio_mport_write_config_32(port, destid,
+							  hopcount,
+							  RIO_HOST_DID_LOCK_CSR,
+							  port->host_deviceid);
+				rio_mport_read_config_32(port, destid, hopcount,
+					RIO_HOST_DID_LOCK_CSR, &result);
+				while (result != port->host_deviceid) {
+					/* Delay a bit */
+					mdelay(1);
+					/* Try to acquire device lock again */
+					rio_mport_write_config_32(port, destid,
+						hopcount,
+						RIO_HOST_DID_LOCK_CSR,
+						port->host_deviceid);
+					rio_mport_read_config_32(port, destid,
+						hopcount,
+						RIO_HOST_DID_LOCK_CSR, &result);
+				}
+
 				for (ndestid = 0;
 				     ndestid < RIO_ANY_DESTID(port->sys_size);
 				     ndestid++) {
@@ -789,6 +811,19 @@ rio_disc_peer(struct rio_net *net, struc
 						break;
 				}
 
+				/* Release device lock */
+				rio_mport_write_config_32(port, destid,
+							  hopcount,
+							  RIO_HOST_DID_LOCK_CSR,
+							  port->host_deviceid);
+				rio_mport_read_config_32(port, destid, hopcount,
+					RIO_HOST_DID_LOCK_CSR, &result);
+				if ((result & 0xffff) != 0xffff) {
+					pr_info("RIO: badness when releasing " \
+						"host lock on %s\n",
+						rio_name(rdev));
+				}
+
 				if (rio_disc_peer
 				    (net, port, ndestid, hopcount + 1) < 0)
 					return -1;
@@ -958,17 +993,45 @@ static void rio_build_route_tables(void)
 	struct rio_dev *rdev;
 	int i;
 	u8 sport;
+	u32 result;
 
 	list_for_each_entry(rdev, &rio_devices, global_list)
-	    if (rio_is_switch(rdev))
-		for (i = 0;
-		     i < RIO_MAX_ROUTE_ENTRIES(rdev->net->hport->sys_size);
-		     i++) {
-			if (rio_route_get_entry
-			    (rdev->net->hport, rdev->rswitch, RIO_GLOBAL_TABLE,
-			     i, &sport) < 0)
-				continue;
-			rdev->rswitch->route_table[i] = sport;
+		if (rio_is_switch(rdev)) {
+			/* Attempt to acquire device lock */
+			rio_write_config_32(rdev, RIO_HOST_DID_LOCK_CSR,
+					rdev->net->hport->host_deviceid);
+			rio_read_config_32(rdev,
+					RIO_HOST_DID_LOCK_CSR, &result);
+			while (result != rdev->net->hport->host_deviceid) {
+				/* Delay a bit */
+				mdelay(1);
+				/* Attempt to acquire device lock again */
+				rio_write_config_32(rdev, RIO_HOST_DID_LOCK_CSR,
+					rdev->net->hport->host_deviceid);
+				rio_read_config_32(rdev,
+					RIO_HOST_DID_LOCK_CSR, &result);
+			}
+
+			for (i = 0;
+			     i < RIO_MAX_ROUTE_ENTRIES(rdev->net->hport->sys_size);
+			     i++) {
+				if (rio_route_get_entry
+				    (rdev->net->hport, rdev->rswitch,
+				     RIO_GLOBAL_TABLE, i, &sport) < 0)
+					continue;
+				rdev->rswitch->route_table[i] = sport;
+			}
+
+			/* Release device lock */
+			rio_write_config_32(rdev, RIO_HOST_DID_LOCK_CSR,
+					rdev->net->hport->host_deviceid);
+			rio_read_config_32(rdev,
+					RIO_HOST_DID_LOCK_CSR, &result);
+			if ((result & 0xffff) != 0xffff) {
+				pr_info("RIO: badness when releasing " \
+					"host lock on %s\n", rio_name(rdev));
+			}
+
 		}
 }
 
@@ -1027,6 +1090,13 @@ int __devinit rio_disc_mport(struct rio_
 		del_timer_sync(&rio_enum_timer);
 
 		pr_debug("done\n");
+
+		/* Read DestID assigned by enumerator */
+		rio_local_read_config_32(mport, RIO_DID_CSR,
+					 &mport->host_deviceid);
+		mport->host_deviceid = RIO_GET_DID(mport->sys_size,
+						   mport->host_deviceid);
+
 		if (rio_disc_peer(net, mport, RIO_ANY_DESTID(mport->sys_size),
 					0) < 0) {
 			printk(KERN_INFO

---

Important Notice: This message is intended for the use of the individual to whom it is addressed and may contain information which is privileged, confidential and/or exempt from disclosure under applicable law. If the reader of this message is not the intended recipient, or is not the employee or agent responsible for delivering the message to the intended recipient, you are hereby notified that any dissemination, distribution, or copying of this communication is strictly prohibited. If you have received this communication in error, please notify the sender immediately by telephone or return e-mail and delete the original message from your systems. Thank you. 

Re: [PATCH 2/7] RapidIO: Add switch locking during discovery

[Micha Nelissen]

Alexandre Bounine wrote:
> +				/* Attempt to acquire device lock */
> +				rio_mport_write_config_32(port, destid,
> +							  hopcount,
> +							  RIO_HOST_DID_LOCK_CSR,
> +							  port->host_deviceid);
> +				rio_mport_read_config_32(port, destid, hopcount,
> +					RIO_HOST_DID_LOCK_CSR, &result);
> +				while (result != port->host_deviceid) {

It's better to abstract the locking of a device into a new function,
rio_lock_device / rio_unlock_device.

Then you can use those in rio_get_route_entry and rio_add_route_entry.

> @@ -1027,6 +1090,13 @@ int __devinit rio_disc_mport(struct rio_
> +
> +		/* Read DestID assigned by enumerator */
> +		rio_local_read_config_32(mport, RIO_DID_CSR,
> +					 &mport->host_deviceid);
> +		mport->host_deviceid = RIO_GET_DID(mport->sys_size,
> +						   mport->host_deviceid);
> +

This fixes something else, should be a separate patch.

Regards, Micha

RE: [PATCH 2/7] RapidIO: Add switch locking during discovery

[Bounine, Alexandre]

Micha Nelissen wrote:
>=20
> Alexandre Bounine wrote:
> > +				/* Attempt to acquire device lock */
> > +				rio_mport_write_config_32(port, destid,
> > +							  hopcount,
> > +
RIO_HOST_DID_LOCK_CSR,
> > +
port->host_deviceid);
> > +				rio_mport_read_config_32(port, destid,
hopcount,
> > +					RIO_HOST_DID_LOCK_CSR, &result);
> > +				while (result !=3D port->host_deviceid) {
>=20
> It's better to abstract the locking of a device into a new function,
> rio_lock_device / rio_unlock_device.
>=20
> Then you can use those in rio_get_route_entry and rio_add_route_entry.

Agree. Plus, adding a "lock" parameter to
rio_get_route_entry/rio_add_route_entry to avoid excessive locking
requests when scanning entire table. I will do it in next version or as
additional patch: I have to address locking anyway for future changes.
   =20
>=20
> > @@ -1027,6 +1090,13 @@ int __devinit rio_disc_mport(struct rio_
> > +
> > +		/* Read DestID assigned by enumerator */
> > +		rio_local_read_config_32(mport, RIO_DID_CSR,
> > +					 &mport->host_deviceid);
> > +		mport->host_deviceid =3D RIO_GET_DID(mport->sys_size,
> > +
mport->host_deviceid);
> > +
>=20
> This fixes something else, should be a separate patch.

This sets an ID used for locking during discovery. On a startup only
enumerator's ID is set to the specified value. All discovering agents
have this ID set to -1. After enumeration is completed it is safe to
initialize host_deviceid for agents as well.

Alex.

Re: [PATCH 2/7] RapidIO: Add switch locking during discovery

[Micha Nelissen]

Bounine, Alexandre wrote:
> Micha Nelissen wrote:
>> Alexandre Bounine wrote:
>>> +		mport->host_deviceid = RIO_GET_DID(mport->sys_size,
>> This fixes something else, should be a separate patch.
> 
> This sets an ID used for locking during discovery. On a startup only
> enumerator's ID is set to the specified value. All discovering agents

Ah I see. Isn't it a bit ugly to have "host_deviceid" being the device's
own destid?

I have removed mport->host_deviceid, and added rdev_host and rdev_self
to rio_net. I also added code to create a rio device for the "self"
instance; then the locking would always use net->rdev_self->destid.

Micha