[PATCH] aic94xx: make use of the new sas_port

[PATCH] aic94xx: make use of the new sas_port

[James Bottomley]

This one is slightly tricky.  The aic94xx driver has a natural sas_port
at the HBA level, but it doesn't have any concept of a sas_port at the
expander level.  Since this is just a proof of concept I just rammed one
in there; however a bit more careful work will have to be done to make
this clean.

James

diff --git a/drivers/scsi/sas/sas_discover.c b/drivers/scsi/sas/sas_discover.c
index 9dd72c5..b9018b7 100644
--- a/drivers/scsi/sas/sas_discover.c
+++ b/drivers/scsi/sas/sas_discover.c
@@ -237,13 +237,15 @@ static int sas_get_port_device(struct as
 
 	switch (dev->dev_type) {
 	case SAS_END_DEV:
-		rphy = sas_end_device_alloc(phy->phy);
+		rphy = sas_end_device_alloc(port->port);
 		break;
 	case EDGE_DEV:
-		rphy = sas_expander_alloc(phy->phy, SAS_EDGE_EXPANDER_DEVICE);
+		rphy = sas_expander_alloc(port->port,
+					  SAS_EDGE_EXPANDER_DEVICE);
 		break;
 	case FANOUT_DEV:
-		rphy = sas_expander_alloc(phy->phy, SAS_FANOUT_EXPANDER_DEVICE);
+		rphy = sas_expander_alloc(port->port,
+					  SAS_FANOUT_EXPANDER_DEVICE);
 		break;
 	default:
 		printk("ERROR: Unidentified device type %d\n", dev->dev_type);
@@ -580,10 +582,6 @@ int sas_discover_end_dev(struct domain_d
 	if (res)
 		return res;
 
-
-	if (!res) {
-	}
-
 	res = sas_rphy_add(dev->rphy);
 	if (res)
 		goto out_err;
diff --git a/drivers/scsi/sas/sas_expander.c b/drivers/scsi/sas/sas_expander.c
index e716d50..5e3ba2f 100644
--- a/drivers/scsi/sas/sas_expander.c
+++ b/drivers/scsi/sas/sas_expander.c
@@ -151,7 +151,6 @@ static void sas_set_ex_phy(struct domain
 	struct sas_rphy *rphy = dev->rphy;
 
 	phy->phy = sas_phy_alloc(&rphy->dev, phy_id);
-	dev_printk(KERN_ERR, &phy->phy->dev, "ALLOCATED\n\n");
 
 	/* FIXME: error_handling */
 	BUG_ON(!phy->phy);
@@ -271,18 +270,24 @@ out_err:
 static int sas_expander_discover(struct domain_device *dev)
 {
 	struct expander_device *ex = &dev->ex_dev;
-	int res;
+	int res = -ENOMEM;
 
 	ex->ex_phy = kzalloc(sizeof(*ex->ex_phy)*ex->num_phys, GFP_KERNEL);
 	if (!ex->ex_phy)
 		return -ENOMEM;
+	ex->ex_port = kzalloc(sizeof(void *)*ex->num_phys, GFP_KERNEL);
+	if (!ex->ex_port)
+		goto out_free_phys;
 
 	res = sas_ex_phy_discover(dev, -1);
 	if (res)
 		goto out_err;
 
 	return 0;
-out_err:
+ out_err:
+	kfree(ex->ex_port);
+	ex->ex_port = NULL;
+ out_free_phys:
 	kfree(ex->ex_phy);
 	ex->ex_phy = NULL;
 	return res;
@@ -513,10 +518,20 @@ static inline void sas_ex_get_linkrate(s
 				       struct ex_phy *parent_phy)
 {
 	struct expander_device *parent_ex = &parent->ex_dev;
+	struct sas_port *port;
 	int i;
 
 	child->pathways = 0;
 
+	for (i = 0; i < parent_ex->num_phys; i++)
+		if (!parent_ex->ex_port[i])
+			break;
+
+	parent_ex->ex_port[i] = port = sas_port_alloc(&parent->rphy->dev, i);
+	BUG_ON(!port);
+	/* FIXME: better error handling */
+	BUG_ON(sas_port_add(port) != 0);
+		
 	for (i = 0; i < parent_ex->num_phys; i++) {
 		struct ex_phy *phy = &parent_ex->ex_phy[i];
 
@@ -532,6 +547,9 @@ static inline void sas_ex_get_linkrate(s
 			child->max_linkrate = max(parent->max_linkrate,
 						  phy->linkrate);
 			child->pathways++;
+			BUG_ON(phy->port != NULL);
+			phy->port = port;
+			sas_port_add_phy(port, phy->phy);
 		}
 	}
 	child->linkrate = min(parent_phy->linkrate, child->max_linkrate);
@@ -590,7 +608,7 @@ static struct domain_device *sas_ex_disc
 		}
 	} else if (phy->attached_tproto & SAS_PROTO_SSP) {
 		child->dev_type = SAS_END_DEV;
-		rphy = sas_end_device_alloc(phy->phy);
+		rphy = sas_end_device_alloc(phy->port);
 		/* FIXME: error handling */
 		BUG_ON(!rphy);
 		child->tproto = phy->attached_tproto;
@@ -613,7 +631,8 @@ static struct domain_device *sas_ex_disc
 				    "at %016llx:0x%x returned 0x%x\n",
 				    SAS_ADDR(child->sas_addr),
 				    SAS_ADDR(parent->sas_addr), phy_id, res);
-			kfree(child);
+			/* FIXME: this kfrees list elements without removing them */
+			//kfree(child);
 			return NULL;
 		}
 	} else {
@@ -649,10 +668,12 @@ static struct domain_device *sas_ex_disc
 		return NULL;
 	switch (phy->attached_dev_type) {
 	case EDGE_DEV:
-		rphy = sas_expander_alloc(phy->phy, SAS_EDGE_EXPANDER_DEVICE);
+		rphy = sas_expander_alloc(phy->port,
+					  SAS_EDGE_EXPANDER_DEVICE);
 		break;
 	case FANOUT_DEV:
-		rphy = sas_expander_alloc(phy->phy, SAS_FANOUT_EXPANDER_DEVICE);
+		rphy = sas_expander_alloc(phy->port,
+					  SAS_FANOUT_EXPANDER_DEVICE);
 		break;
 	default:
 		rphy = NULL;	/* shut gcc up */
diff --git a/drivers/scsi/sas/sas_port.c b/drivers/scsi/sas/sas_port.c
index 1a042f9..da4599d 100644
--- a/drivers/scsi/sas/sas_port.c
+++ b/drivers/scsi/sas/sas_port.c
@@ -84,13 +84,19 @@ static void sas_form_port(struct asd_sas
 		return;
 	}
 
+	if (!port->port) {
+		port->port = sas_port_alloc(phy->phy->dev.parent, port->id);
+		BUG_ON(!port->port);
+		sas_port_add(port->port);
+	}
+	sas_port_add_phy(port->port, phy->phy);
+
 	/* add the phy to the port */
 	list_add_tail(&phy->port_phy_el, &port->phy_list);
 	phy->port = port;
 	port->num_phys++;
 	port->phy_mask |= (1U << phy->id);
 
-	phy->phy->port_identifier = port->id;
 	if (!port->phy)
 		port->phy = phy->phy;
 
@@ -141,6 +147,7 @@ void sas_deform_port(struct asd_sas_phy 
 		port->port_dev->pathways--;
 
 	if (port->num_phys == 1) {
+		sas_port_delete(port->port);
 		init_completion(&port->port_gone_completion);
 		sas_discover_event(port, DISCE_PORT_GONE);
 		wait_for_completion(&port->port_gone_completion);
@@ -149,6 +156,8 @@ void sas_deform_port(struct asd_sas_phy 
 	if (si->dft->lldd_port_deformed)
 		si->dft->lldd_port_deformed(phy);
 
+	sas_port_delete_phy(port->port, phy->phy);
+
 	spin_lock(&sas_ha->phy_port_lock);
 	spin_lock(&port->phy_list_lock);
 
diff --git a/include/scsi/sas/sas_class.h b/include/scsi/sas/sas_class.h
index 0c3aae5..4b56259 100644
--- a/include/scsi/sas/sas_class.h
+++ b/include/scsi/sas/sas_class.h
@@ -180,6 +180,8 @@ struct asd_sas_port {
 
 	struct sas_ha_struct *ha;
 
+	struct sas_port	*port;
+
 	void *lldd_port;	  /* not touched by the sas class code */
 };
 
diff --git a/include/scsi/sas/sas_expander.h b/include/scsi/sas/sas_expander.h
index e83b41c..6ce6163 100644
--- a/include/scsi/sas/sas_expander.h
+++ b/include/scsi/sas/sas_expander.h
@@ -72,6 +72,7 @@ struct ex_phy {
 	int  last_da_index;
 
 	struct sas_phy *phy;
+	struct sas_port *port;
 };
 
 struct expander_device {
@@ -85,6 +86,7 @@ struct expander_device {
 	u8     enclosure_logical_id[8];
 
 	struct ex_phy *ex_phy;
+	struct sas_port **ex_port;
 };
 
 int  sas_discover_root_expander(struct domain_device *dev);

Re: [PATCH] aic94xx: make use of the new sas_port

[Alexis Bruemmer]

These patches look good and are running on our x260 and the ppc64
(power).  I was just wondering about a few things under the sys/class/:

there is a sas_device, sas_end_device, and a sas_expander dir.
sas_device contains the same info as sas_end_device, and a sas_expander
can we eliminate one or the other?  Or was this a design choice?

Also during the boot process on the x260 with an expander we see these
printk's:
sas: phy[n] matched wide port0
If we are going to print this then we better be sure that this is truly
a wide port.  It seems under the sys/class/ we have both a single
port0:0 and then also port-0:0:[n]-- a port[n] for each phy[n].  and
although each phy seems to have the same sas_address the end_device
attached to that phy have unique addresses.  My understanding is that a
wide port has multiple phys with the same sas address and are attached
to one end device.  I believe that the expander situation is not the
same as wide port.  If this is the case then it is as simple as changing
the printk statement to not say "wide port"

I am more than happy to make these changes if everyone agrees that they
should be made.

Thanks,

Alexis 


On Tue, 2006-05-30 at 23:09 -0500, James Bottomley wrote:
> This one is slightly tricky.  The aic94xx driver has a natural sas_port
> at the HBA level, but it doesn't have any concept of a sas_port at the
> expander level.  Since this is just a proof of concept I just rammed one
> in there; however a bit more careful work will have to be done to make
> this clean.
> 
> James
> 
> diff --git a/drivers/scsi/sas/sas_discover.c b/drivers/scsi/sas/sas_discover.c
> index 9dd72c5..b9018b7 100644
> --- a/drivers/scsi/sas/sas_discover.c
> +++ b/drivers/scsi/sas/sas_discover.c
> @@ -237,13 +237,15 @@ static int sas_get_port_device(struct as
>  
>  	switch (dev->dev_type) {
>  	case SAS_END_DEV:
> -		rphy = sas_end_device_alloc(phy->phy);
> +		rphy = sas_end_device_alloc(port->port);
>  		break;
>  	case EDGE_DEV:
> -		rphy = sas_expander_alloc(phy->phy, SAS_EDGE_EXPANDER_DEVICE);
> +		rphy = sas_expander_alloc(port->port,
> +					  SAS_EDGE_EXPANDER_DEVICE);
>  		break;
>  	case FANOUT_DEV:
> -		rphy = sas_expander_alloc(phy->phy, SAS_FANOUT_EXPANDER_DEVICE);
> +		rphy = sas_expander_alloc(port->port,
> +					  SAS_FANOUT_EXPANDER_DEVICE);
>  		break;
>  	default:
>  		printk("ERROR: Unidentified device type %d\n", dev->dev_type);
> @@ -580,10 +582,6 @@ int sas_discover_end_dev(struct domain_d
>  	if (res)
>  		return res;
>  
> -
> -	if (!res) {
> -	}
> -
>  	res = sas_rphy_add(dev->rphy);
>  	if (res)
>  		goto out_err;
> diff --git a/drivers/scsi/sas/sas_expander.c b/drivers/scsi/sas/sas_expander.c
> index e716d50..5e3ba2f 100644
> --- a/drivers/scsi/sas/sas_expander.c
> +++ b/drivers/scsi/sas/sas_expander.c
> @@ -151,7 +151,6 @@ static void sas_set_ex_phy(struct domain
>  	struct sas_rphy *rphy = dev->rphy;
>  
>  	phy->phy = sas_phy_alloc(&rphy->dev, phy_id);
> -	dev_printk(KERN_ERR, &phy->phy->dev, "ALLOCATED\n\n");
>  
>  	/* FIXME: error_handling */
>  	BUG_ON(!phy->phy);
> @@ -271,18 +270,24 @@ out_err:
>  static int sas_expander_discover(struct domain_device *dev)
>  {
>  	struct expander_device *ex = &dev->ex_dev;
> -	int res;
> +	int res = -ENOMEM;
>  
>  	ex->ex_phy = kzalloc(sizeof(*ex->ex_phy)*ex->num_phys, GFP_KERNEL);
>  	if (!ex->ex_phy)
>  		return -ENOMEM;
> +	ex->ex_port = kzalloc(sizeof(void *)*ex->num_phys, GFP_KERNEL);
> +	if (!ex->ex_port)
> +		goto out_free_phys;
>  
>  	res = sas_ex_phy_discover(dev, -1);
>  	if (res)
>  		goto out_err;
>  
>  	return 0;
> -out_err:
> + out_err:
> +	kfree(ex->ex_port);
> +	ex->ex_port = NULL;
> + out_free_phys:
>  	kfree(ex->ex_phy);
>  	ex->ex_phy = NULL;
>  	return res;
> @@ -513,10 +518,20 @@ static inline void sas_ex_get_linkrate(s
>  				       struct ex_phy *parent_phy)
>  {
>  	struct expander_device *parent_ex = &parent->ex_dev;
> +	struct sas_port *port;
>  	int i;
>  
>  	child->pathways = 0;
>  
> +	for (i = 0; i < parent_ex->num_phys; i++)
> +		if (!parent_ex->ex_port[i])
> +			break;
> +
> +	parent_ex->ex_port[i] = port = sas_port_alloc(&parent->rphy->dev, i);
> +	BUG_ON(!port);
> +	/* FIXME: better error handling */
> +	BUG_ON(sas_port_add(port) != 0);
> +		
>  	for (i = 0; i < parent_ex->num_phys; i++) {
>  		struct ex_phy *phy = &parent_ex->ex_phy[i];
>  
> @@ -532,6 +547,9 @@ static inline void sas_ex_get_linkrate(s
>  			child->max_linkrate = max(parent->max_linkrate,
>  						  phy->linkrate);
>  			child->pathways++;
> +			BUG_ON(phy->port != NULL);
> +			phy->port = port;
> +			sas_port_add_phy(port, phy->phy);
>  		}
>  	}
>  	child->linkrate = min(parent_phy->linkrate, child->max_linkrate);
> @@ -590,7 +608,7 @@ static struct domain_device *sas_ex_disc
>  		}
>  	} else if (phy->attached_tproto & SAS_PROTO_SSP) {
>  		child->dev_type = SAS_END_DEV;
> -		rphy = sas_end_device_alloc(phy->phy);
> +		rphy = sas_end_device_alloc(phy->port);
>  		/* FIXME: error handling */
>  		BUG_ON(!rphy);
>  		child->tproto = phy->attached_tproto;
> @@ -613,7 +631,8 @@ static struct domain_device *sas_ex_disc
>  				    "at %016llx:0x%x returned 0x%x\n",
>  				    SAS_ADDR(child->sas_addr),
>  				    SAS_ADDR(parent->sas_addr), phy_id, res);
> -			kfree(child);
> +			/* FIXME: this kfrees list elements without removing them */
> +			//kfree(child);
>  			return NULL;
>  		}
>  	} else {
> @@ -649,10 +668,12 @@ static struct domain_device *sas_ex_disc
>  		return NULL;
>  	switch (phy->attached_dev_type) {
>  	case EDGE_DEV:
> -		rphy = sas_expander_alloc(phy->phy, SAS_EDGE_EXPANDER_DEVICE);
> +		rphy = sas_expander_alloc(phy->port,
> +					  SAS_EDGE_EXPANDER_DEVICE);
>  		break;
>  	case FANOUT_DEV:
> -		rphy = sas_expander_alloc(phy->phy, SAS_FANOUT_EXPANDER_DEVICE);
> +		rphy = sas_expander_alloc(phy->port,
> +					  SAS_FANOUT_EXPANDER_DEVICE);
>  		break;
>  	default:
>  		rphy = NULL;	/* shut gcc up */
> diff --git a/drivers/scsi/sas/sas_port.c b/drivers/scsi/sas/sas_port.c
> index 1a042f9..da4599d 100644
> --- a/drivers/scsi/sas/sas_port.c
> +++ b/drivers/scsi/sas/sas_port.c
> @@ -84,13 +84,19 @@ static void sas_form_port(struct asd_sas
>  		return;
>  	}
>  
> +	if (!port->port) {
> +		port->port = sas_port_alloc(phy->phy->dev.parent, port->id);
> +		BUG_ON(!port->port);
> +		sas_port_add(port->port);
> +	}
> +	sas_port_add_phy(port->port, phy->phy);
> +
>  	/* add the phy to the port */
>  	list_add_tail(&phy->port_phy_el, &port->phy_list);
>  	phy->port = port;
>  	port->num_phys++;
>  	port->phy_mask |= (1U << phy->id);
>  
> -	phy->phy->port_identifier = port->id;
>  	if (!port->phy)
>  		port->phy = phy->phy;
>  
> @@ -141,6 +147,7 @@ void sas_deform_port(struct asd_sas_phy 
>  		port->port_dev->pathways--;
>  
>  	if (port->num_phys == 1) {
> +		sas_port_delete(port->port);
>  		init_completion(&port->port_gone_completion);
>  		sas_discover_event(port, DISCE_PORT_GONE);
>  		wait_for_completion(&port->port_gone_completion);
> @@ -149,6 +156,8 @@ void sas_deform_port(struct asd_sas_phy 
>  	if (si->dft->lldd_port_deformed)
>  		si->dft->lldd_port_deformed(phy);
>  
> +	sas_port_delete_phy(port->port, phy->phy);
> +
>  	spin_lock(&sas_ha->phy_port_lock);
>  	spin_lock(&port->phy_list_lock);
>  
> diff --git a/include/scsi/sas/sas_class.h b/include/scsi/sas/sas_class.h
> index 0c3aae5..4b56259 100644
> --- a/include/scsi/sas/sas_class.h
> +++ b/include/scsi/sas/sas_class.h
> @@ -180,6 +180,8 @@ struct asd_sas_port {
>  
>  	struct sas_ha_struct *ha;
>  
> +	struct sas_port	*port;
> +
>  	void *lldd_port;	  /* not touched by the sas class code */
>  };
>  
> diff --git a/include/scsi/sas/sas_expander.h b/include/scsi/sas/sas_expander.h
> index e83b41c..6ce6163 100644
> --- a/include/scsi/sas/sas_expander.h
> +++ b/include/scsi/sas/sas_expander.h
> @@ -72,6 +72,7 @@ struct ex_phy {
>  	int  last_da_index;
>  
>  	struct sas_phy *phy;
> +	struct sas_port *port;
>  };
>  
>  struct expander_device {
> @@ -85,6 +86,7 @@ struct expander_device {
>  	u8     enclosure_logical_id[8];
>  
>  	struct ex_phy *ex_phy;
> +	struct sas_port **ex_port;
>  };
>  
>  int  sas_discover_root_expander(struct domain_device *dev);
> 
> 
> -
> To unsubscribe from this list: send the line "unsubscribe linux-scsi" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH] aic94xx: make use of the new sas_port

[James Bottomley]

On Wed, 2006-05-31 at 12:46 -0700, Alexis Bruemmer wrote:
> there is a sas_device, sas_end_device, and a sas_expander dir.
> sas_device contains the same info as sas_end_device, and a sas_expander
> can we eliminate one or the other?  Or was this a design choice?

Not really.  This is a design choice given to us by the interface.
Everything that's connected to a port is a sas_device.  As such, they
all share a set of characteristics (initiator and target roles, sas
address and so forth).  The specific devices (sas_expander and
sas_end_device) contain only parameters that are specific to the device,
so the expander shows the manufacturer and component info.  The
end_device shows the SAS mandated parameters like ready led and nexus
loss timeout).

> Also during the boot process on the x260 with an expander we see these
> printk's:
> sas: phy[n] matched wide port0
> If we are going to print this then we better be sure that this is truly
> a wide port.  It seems under the sys/class/ we have both a single
> port0:0 and then also port-0:0:[n]-- a port[n] for each phy[n].  and
> although each phy seems to have the same sas_address the end_device
> attached to that phy have unique addresses.  My understanding is that a
> wide port has multiple phys with the same sas address and are attached
> to one end device.  I believe that the expander situation is not the
> same as wide port.  If this is the case then it is as simple as changing
> the printk statement to not say "wide port"

As I read the code, it only prints that when it actually finds a wide
port (as in a port that already has a phy attached) ... do an ls -R of
your tree and I'll see if I can figure out what your topology is.

James

Re: [PATCH] aic94xx: make use of the new sas_port

[Douglas Gilbert]

Alexis Bruemmer wrote:
> These patches look good and are running on our x260 and the ppc64
> (power).  I was just wondering about a few things under the sys/class/:
> 
> there is a sas_device, sas_end_device, and a sas_expander dir.
> sas_device contains the same info as sas_end_device, and a sas_expander
> can we eliminate one or the other?  Or was this a design choice?
> 
> Also during the boot process on the x260 with an expander we see these
> printk's:
> sas: phy[n] matched wide port0
> If we are going to print this then we better be sure that this is truly
> a wide port.  It seems under the sys/class/ we have both a single
> port0:0 and then also port-0:0:[n]-- a port[n] for each phy[n].  and
> although each phy seems to have the same sas_address the end_device
> attached to that phy have unique addresses.  My understanding is that a
> wide port has multiple phys with the same sas address and are attached
> to one end device.

A wide port is an abstraction just above the link
layer in which two or more links running between
the same two devices have phys with the same SAS
addresses associated with them.

Assume we have two phys on a HBA connected to two
phys on an expander, then two other phys on the
expander connected to a dual ported SAS disk.
That is one wide port connection (between the
at HBA and the expander) and two narrow
port connections (between the expander and each
port on the SAS disk).

This comes about because phys on a HBA share a
SAS address, phys on a expander share a SAS address
but phys of a dual ported SAS disk have distinct
SAS addresses.
[Could someone suggest an instructive diagram url
of this as the sas drafts are a bit complex.]

You are probably aware of this distinction but my
guess is a lot of other people aren't.

Put another way, seen from a LLD or discover process when
multiple phys on the same device have this tuple
<SAS_address, attached_SAS_address> equal then there is
a wide port connection.

This also means that it is possible to talk about
a narrow port in the absence of an established
link but not possible to talk about a wide port
until there are two or more established links.

It may also be worth pointing out that there is not data
aggregation on a wide port, but there is command aggregation.
So a 2 phy wide port connection will not run a single SCSI
command twice as fast, but can run two co-incident SCSI
commands at full link speed (i.e. up to 3 Gbps currently).

  I believe that the expander situation is not the
> same as wide port.  If this is the case then it is as simple as changing
> the printk statement to not say "wide port"
> 
> I am more than happy to make these changes if everyone agrees that they
> should be made.

It may also be useful to use the term "narrow"
sometimes, especially if there is a transition
from wide to narrow (e.g. due to a faulty cable
or a link reset on one of the links).

Doug Gilbert

Re: [PATCH] aic94xx: make use of the new sas_port

[Alexis Bruemmer]

On Wed, 2006-05-31 at 15:57 -0500, James Bottomley wrote:
> On Wed, 2006-05-31 at 12:46 -0700, Alexis Bruemmer wrote:
> > there is a sas_device, sas_end_device, and a sas_expander dir.
> > sas_device contains the same info as sas_end_device, and a sas_expander
> > can we eliminate one or the other?  Or was this a design choice?
> 
> Not really.  This is a design choice given to us by the interface.
> Everything that's connected to a port is a sas_device.  As such, they
> all share a set of characteristics (initiator and target roles, sas
> address and so forth).  The specific devices (sas_expander and
> sas_end_device) contain only parameters that are specific to the device,
> so the expander shows the manufacturer and component info.  The
> end_device shows the SAS mandated parameters like ready led and nexus
> loss timeout).
OK, makes sense

> 
> > Also during the boot process on the x260 with an expander we see these
> > printk's:
> > sas: phy[n] matched wide port0
> > If we are going to print this then we better be sure that this is truly
> > a wide port.  It seems under the sys/class/ we have both a single
> > port0:0 and then also port-0:0:[n]-- a port[n] for each phy[n].  and
> > although each phy seems to have the same sas_address the end_device
> > attached to that phy have unique addresses.  My understanding is that a
> > wide port has multiple phys with the same sas address and are attached
> > to one end device.  I believe that the expander situation is not the
> > same as wide port.  If this is the case then it is as simple as changing
> > the printk statement to not say "wide port"
> 
> As I read the code, it only prints that when it actually finds a wide
> port (as in a port that already has a phy attached) ... do an ls -R of
> your tree and I'll see if I can figure out what your topology is.
Below is a ls -R of my tree on this machine.  It seems that it sees the
expander as a wide port (port-0:0) and then the expander has a series of
phys attached to end devices.  What I am curious about is the
port-0:0:[0...4]-- the way this is laid out it seems that there is one
port for each phy which is attached to one end device, which would be a
narrow port.  Does your x260 create the same topology?

--Alexis


tree:
sas_device:
end_device-0:0-0  end_device-0:1-1  end_device-0:2-2  end_device-0:3-3
end_device-0:4-4  expander-0:0

sas_device/end_device-0:0-0:
device  device_type  initiator_port_protocols  phy_identifier
sas_address  target_port_protocols  uevent

sas_device/end_device-0:1-1:
device  device_type  initiator_port_protocols  phy_identifier
sas_address  target_port_protocols  uevent

sas_device/end_device-0:2-2:
device  device_type  initiator_port_protocols  phy_identifier
sas_address  target_port_protocols  uevent

sas_device/end_device-0:3-3:
device  device_type  initiator_port_protocols  phy_identifier
sas_address  target_port_protocols  uevent

sas_device/end_device-0:4-4:
device  device_type  initiator_port_protocols  phy_identifier
sas_address  target_port_protocols  uevent

sas_device/expander-0:0:
device  device_type  initiator_port_protocols  phy_identifier
sas_address  target_port_protocols  uevent

sas_end_device:
end_device-0:0-0  end_device-0:1-1  end_device-0:2-2  end_device-0:3-3
end_device-0:4-4

sas_end_device/end_device-0:0-0:
device  initiator_response_timeout  I_T_nexus_loss_timeout
ready_led_meaning  uevent

sas_end_device/end_device-0:1-1:
device  initiator_response_timeout  I_T_nexus_loss_timeout
ready_led_meaning  uevent

sas_end_device/end_device-0:2-2:
device  initiator_response_timeout  I_T_nexus_loss_timeout
ready_led_meaning  uevent

sas_end_device/end_device-0:3-3:
device  initiator_response_timeout  I_T_nexus_loss_timeout
ready_led_meaning  uevent

sas_end_device/end_device-0:4-4:
device  initiator_response_timeout  I_T_nexus_loss_timeout
ready_led_meaning  uevent

sas_expander:
expander-0:0

sas_expander/expander-0:0:
component_id  component_revision_id  component_vendor_id  device  level
product_id  product_rev  uevent  vendor_id

sas_host:
host0

sas_host/host0:
device  uevent

sas_phy:
phy-0:0    phy-0-0:1   phy-0-0:11  phy-0-0:2  phy-0-0:4  phy-0-0:6
phy-0-0:8  phy-0:1  phy-0:3  phy-0:5  phy-0:7
phy-0-0:0  phy-0-0:10  phy-0-0:12  phy-0-0:3  phy-0-0:5  phy-0-0:7
phy-0-0:9  phy-0:2  phy-0:4  phy-0:6

sas_phy/phy-0:0:
device                    invalid_dword_count       maximum_linkrate_hw
negotiated_linkrate      running_disparity_error_count  uevent
device_type               loss_of_dword_sync_count  minimum_linkrate
phy_identifier           sas_address
initiator_port_protocols  maximum_linkrate          minimum_linkrate_hw
phy_reset_problem_count  target_port_protocols

sas_phy/phy-0-0:0:
device                    invalid_dword_count       maximum_linkrate_hw
negotiated_linkrate      running_disparity_error_count  uevent
device_type               loss_of_dword_sync_count  minimum_linkrate
phy_identifier           sas_address
initiator_port_protocols  maximum_linkrate          minimum_linkrate_hw
phy_reset_problem_count  target_port_protocols

sas_phy/phy-0-0:1:
device                    invalid_dword_count       maximum_linkrate_hw
negotiated_linkrate      running_disparity_error_count  uevent
device_type               loss_of_dword_sync_count  minimum_linkrate
phy_identifier           sas_address
initiator_port_protocols  maximum_linkrate          minimum_linkrate_hw
phy_reset_problem_count  target_port_protocols

sas_phy/phy-0-0:10:
device                    invalid_dword_count       maximum_linkrate_hw
negotiated_linkrate      running_disparity_error_count  uevent
device_type               loss_of_dword_sync_count  minimum_linkrate
phy_identifier           sas_address
initiator_port_protocols  maximum_linkrate          minimum_linkrate_hw
phy_reset_problem_count  target_port_protocols

sas_phy/phy-0-0:11:
device                    invalid_dword_count       maximum_linkrate_hw
negotiated_linkrate      running_disparity_error_count  uevent
device_type               loss_of_dword_sync_count  minimum_linkrate
phy_identifier           sas_address
initiator_port_protocols  maximum_linkrate          minimum_linkrate_hw
phy_reset_problem_count  target_port_protocols

sas_phy/phy-0-0:12:
device                    invalid_dword_count       maximum_linkrate_hw
negotiated_linkrate      running_disparity_error_count  uevent
device_type               loss_of_dword_sync_count  minimum_linkrate
phy_identifier           sas_address
initiator_port_protocols  maximum_linkrate          minimum_linkrate_hw
phy_reset_problem_count  target_port_protocols

sas_phy/phy-0-0:2:
device                    invalid_dword_count       maximum_linkrate_hw
negotiated_linkrate      running_disparity_error_count  uevent
device_type               loss_of_dword_sync_count  minimum_linkrate
phy_identifier           sas_address
initiator_port_protocols  maximum_linkrate          minimum_linkrate_hw
phy_reset_problem_count  target_port_protocols

sas_phy/phy-0-0:3:
device                    invalid_dword_count       maximum_linkrate_hw
negotiated_linkrate      running_disparity_error_count  uevent
device_type               loss_of_dword_sync_count  minimum_linkrate
phy_identifier           sas_address
initiator_port_protocols  maximum_linkrate          minimum_linkrate_hw
phy_reset_problem_count  target_port_protocols

sas_phy/phy-0-0:4:
device                    invalid_dword_count       maximum_linkrate_hw
negotiated_linkrate      running_disparity_error_count  uevent
device_type               loss_of_dword_sync_count  minimum_linkrate
phy_identifier           sas_address
initiator_port_protocols  maximum_linkrate          minimum_linkrate_hw
phy_reset_problem_count  target_port_protocols

sas_phy/phy-0-0:5:
device                    invalid_dword_count       maximum_linkrate_hw
negotiated_linkrate      running_disparity_error_count  uevent
device_type               loss_of_dword_sync_count  minimum_linkrate
phy_identifier           sas_address
initiator_port_protocols  maximum_linkrate          minimum_linkrate_hw
phy_reset_problem_count  target_port_protocols

sas_phy/phy-0-0:6:
device                    invalid_dword_count       maximum_linkrate_hw
negotiated_linkrate      running_disparity_error_count  uevent
device_type               loss_of_dword_sync_count  minimum_linkrate
phy_identifier           sas_address
initiator_port_protocols  maximum_linkrate          minimum_linkrate_hw
phy_reset_problem_count  target_port_protocols

sas_phy/phy-0-0:7:
device                    invalid_dword_count       maximum_linkrate_hw
negotiated_linkrate      running_disparity_error_count  uevent
device_type               loss_of_dword_sync_count  minimum_linkrate
phy_identifier           sas_address
initiator_port_protocols  maximum_linkrate          minimum_linkrate_hw
phy_reset_problem_count  target_port_protocols

sas_phy/phy-0-0:8:
device                    invalid_dword_count       maximum_linkrate_hw
negotiated_linkrate      running_disparity_error_count  uevent
device_type               loss_of_dword_sync_count  minimum_linkrate
phy_identifier           sas_address
initiator_port_protocols  maximum_linkrate          minimum_linkrate_hw
phy_reset_problem_count  target_port_protocols

sas_phy/phy-0-0:9:
device                    invalid_dword_count       maximum_linkrate_hw
negotiated_linkrate      running_disparity_error_count  uevent
device_type               loss_of_dword_sync_count  minimum_linkrate
phy_identifier           sas_address
initiator_port_protocols  maximum_linkrate          minimum_linkrate_hw
phy_reset_problem_count  target_port_protocols

sas_phy/phy-0:1:
device                    invalid_dword_count       maximum_linkrate_hw
negotiated_linkrate      running_disparity_error_count  uevent
device_type               loss_of_dword_sync_count  minimum_linkrate
phy_identifier           sas_address
initiator_port_protocols  maximum_linkrate          minimum_linkrate_hw
phy_reset_problem_count  target_port_protocols

sas_phy/phy-0:2:
device                    invalid_dword_count       maximum_linkrate_hw
negotiated_linkrate      running_disparity_error_count  uevent
device_type               loss_of_dword_sync_count  minimum_linkrate
phy_identifier           sas_address
initiator_port_protocols  maximum_linkrate          minimum_linkrate_hw
phy_reset_problem_count  target_port_protocols

sas_phy/phy-0:3:
device                    invalid_dword_count       maximum_linkrate_hw
negotiated_linkrate      running_disparity_error_count  uevent
device_type               loss_of_dword_sync_count  minimum_linkrate
phy_identifier           sas_address
initiator_port_protocols  maximum_linkrate          minimum_linkrate_hw
phy_reset_problem_count  target_port_protocols

sas_phy/phy-0:4:
device                    invalid_dword_count       maximum_linkrate_hw
negotiated_linkrate      running_disparity_error_count  uevent
device_type               loss_of_dword_sync_count  minimum_linkrate
phy_identifier           sas_address
initiator_port_protocols  maximum_linkrate          minimum_linkrate_hw
phy_reset_problem_count  target_port_protocols

sas_phy/phy-0:5:
device                    invalid_dword_count       maximum_linkrate_hw
negotiated_linkrate      running_disparity_error_count  uevent
device_type               loss_of_dword_sync_count  minimum_linkrate
phy_identifier           sas_address
initiator_port_protocols  maximum_linkrate          minimum_linkrate_hw
phy_reset_problem_count  target_port_protocols

sas_phy/phy-0:6:
device                    invalid_dword_count       maximum_linkrate_hw
negotiated_linkrate      running_disparity_error_count  uevent
device_type               loss_of_dword_sync_count  minimum_linkrate
phy_identifier           sas_address
initiator_port_protocols  maximum_linkrate          minimum_linkrate_hw
phy_reset_problem_count  target_port_protocols

sas_phy/phy-0:7:
device                    invalid_dword_count       maximum_linkrate_hw
negotiated_linkrate      running_disparity_error_count  uevent
device_type               loss_of_dword_sync_count  minimum_linkrate
phy_identifier           sas_address
initiator_port_protocols  maximum_linkrate          minimum_linkrate_hw
phy_reset_problem_count  target_port_protocols

sas_port:
port-0:0  port-0:0:0  port-0:0:1  port-0:0:2  port-0:0:3  port-0:0:4

sas_port/port-0:0:
device  num_phys  uevent

sas_port/port-0:0:0:
device  num_phys  uevent

sas_port/port-0:0:1:
device  num_phys  uevent

sas_port/port-0:0:2:
device  num_phys  uevent

sas_port/port-0:0:3:
device  num_phys  uevent

sas_port/port-0:0:4:
device  num_phys  uevent

Re: [PATCH] aic94xx: make use of the new sas_port

[James Bottomley]

On Wed, 2006-05-31 at 15:00 -0700, Alexis Bruemmer wrote:
> Below is a ls -R of my tree on this machine.  It seems that it sees
> the
> expander as a wide port (port-0:0) and then the expander has a series
> of
> phys attached to end devices.  What I am curious about is the
> port-0:0:[0...4]-- the way this is laid out it seems that there is one
> port for each phy which is attached to one end device, which would be
> a
> narrow port.  Does your x260 create the same topology?

Yes, that's by design.

Now every start point for a device must be a port (either narrow or
wide).  The port-0:0:X are ports formed on the expander.  And, since you
only have narrow devices attached, that's all it shows  you'll see this
if you look at the tree under /sys/devices rather than /sys/class (since
the latter is flat).  All of the port-0:0:x will show up under
expander-0:0 (just do ls /sys/class/sas_expander/expander-0:0/device/ to
see this).

James

Re: [PATCH] aic94xx: make use of the new sas_port

[Alexis Bruemmer]

On Wed, 2006-05-31 at 17:32 -0400, Douglas Gilbert wrote:
> Alexis Bruemmer wrote:
> > These patches look good and are running on our x260 and the ppc64
> > (power).  I was just wondering about a few things under the sys/class/:
> > 
> > there is a sas_device, sas_end_device, and a sas_expander dir.
> > sas_device contains the same info as sas_end_device, and a sas_expander
> > can we eliminate one or the other?  Or was this a design choice?
> > 
> > Also during the boot process on the x260 with an expander we see these
> > printk's:
> > sas: phy[n] matched wide port0
> > If we are going to print this then we better be sure that this is truly
> > a wide port.  It seems under the sys/class/ we have both a single
> > port0:0 and then also port-0:0:[n]-- a port[n] for each phy[n].  and
> > although each phy seems to have the same sas_address the end_device
> > attached to that phy have unique addresses.  My understanding is that a
> > wide port has multiple phys with the same sas address and are attached
> > to one end device.
> 
> A wide port is an abstraction just above the link
> layer in which two or more links running between
> the same two devices have phys with the same SAS
> addresses associated with them.
> 
> Assume we have two phys on a HBA connected to two
> phys on an expander, then two other phys on the
> expander connected to a dual ported SAS disk.
> That is one wide port connection (between the
> at HBA and the expander) and two narrow
> port connections (between the expander and each
> port on the SAS disk).
> 
> This comes about because phys on a HBA share a
> SAS address, phys on a expander share a SAS address
> but phys of a dual ported SAS disk have distinct
> SAS addresses.
> [Could someone suggest an instructive diagram url
> of this as the sas drafts are a bit complex.]

This would be assume!

> 
> You are probably aware of this distinction but my
> guess is a lot of other people aren't.
> 
> Put another way, seen from a LLD or discover process when
> multiple phys on the same device have this tuple
> <SAS_address, attached_SAS_address> equal then there is
> a wide port connection.

If this is the case then according to the below print out we do not have
a wide port on this machine:

sas: phy0 added to port0, phy_mask:0x1
[  155.328494] sas: phy1 matched wide port0
[  155.328499] sas: phy1 added to port0, phy_mask:0x3
[  155.328516] sas: phy2 matched wide port0
[  155.328520] sas: phy2 added to port0, phy_mask:0x7
[  155.328541] sas: phy3 matched wide port0
[  155.328548] sas: phy3 added to port0, phy_mask:0xf
[  155.328578] sas: DOING DISCOVERY on port 0, pid:1785
[  155.328917] sas: ex 5005076a00000a40 phy00:T attached: 50010b9000021585
[  155.328996] sas: ex 5005076a00000a40 phy01:T attached: 50010b9000021575
[  155.329081] sas: ex 5005076a00000a40 phy02:T attached: 50010b900004b87e
[  155.329169] sas: ex 5005076a00000a40 phy03:S attached: 5005076a011061c0

The sas_address and the sas_attached_address are different

> 
> This also means that it is possible to talk about
> a narrow port in the absence of an established
> link but not possible to talk about a wide port
> until there are two or more established links.
> 
> It may also be worth pointing out that there is not data
> aggregation on a wide port, but there is command aggregation.
> So a 2 phy wide port connection will not run a single SCSI
> command twice as fast, but can run two co-incident SCSI
> commands at full link speed (i.e. up to 3 Gbps currently).
> 
>   I believe that the expander situation is not the
> > same as wide port.  If this is the case then it is as simple as changing
> > the printk statement to not say "wide port"
> > 
> > I am more than happy to make these changes if everyone agrees that they
> > should be made.
> 
> It may also be useful to use the term "narrow"
> sometimes, especially if there is a transition
> from wide to narrow (e.g. due to a faulty cable
> or a link reset on one of the links).
> 
> Doug Gilbert
> -
> To unsubscribe from this list: send the line "unsubscribe linux-scsi" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH] aic94xx: make use of the new sas_port

[James Bottomley]

On Wed, 2006-05-31 at 15:30 -0700, Alexis Bruemmer wrote:
> sas: phy0 added to port0, phy_mask:0x1
> [  155.328494] sas: phy1 matched wide port0
> [  155.328499] sas: phy1 added to port0, phy_mask:0x3
> [  155.328516] sas: phy2 matched wide port0
> [  155.328520] sas: phy2 added to port0, phy_mask:0x7
> [  155.328541] sas: phy3 matched wide port0
> [  155.328548] sas: phy3 added to port0, phy_mask:0xf
> [  155.328578] sas: DOING DISCOVERY on port 0, pid:1785
> [  155.328917] sas: ex 5005076a00000a40 phy00:T attached:
> 50010b9000021585
> [  155.328996] sas: ex 5005076a00000a40 phy01:T attached:
> 50010b9000021575
> [  155.329081] sas: ex 5005076a00000a40 phy02:T attached:
> 50010b900004b87e
> [  155.329169] sas: ex 5005076a00000a40 phy03:S attached:
> 5005076a011061c0

No.  Look again.  the first four, which are a wide port, are phys 0-3 on
the HBA.

The second four, which have separate attached devices are phys 0-3 on
the expander (that's what the sightly cryptic ex 5005076a00000a40 phy<n>
means).

You can see this better in the transport class tree

James

Re: [PATCH] aic94xx: make use of the new sas_port

[Alexis Bruemmer]

On Wed, 2006-05-31 at 17:37 -0500, James Bottomley wrote:
> On Wed, 2006-05-31 at 15:30 -0700, Alexis Bruemmer wrote:
> > sas: phy0 added to port0, phy_mask:0x1
> > [  155.328494] sas: phy1 matched wide port0
> > [  155.328499] sas: phy1 added to port0, phy_mask:0x3
> > [  155.328516] sas: phy2 matched wide port0
> > [  155.328520] sas: phy2 added to port0, phy_mask:0x7
> > [  155.328541] sas: phy3 matched wide port0
> > [  155.328548] sas: phy3 added to port0, phy_mask:0xf
> > [  155.328578] sas: DOING DISCOVERY on port 0, pid:1785
> > [  155.328917] sas: ex 5005076a00000a40 phy00:T attached:
> > 50010b9000021585
> > [  155.328996] sas: ex 5005076a00000a40 phy01:T attached:
> > 50010b9000021575
> > [  155.329081] sas: ex 5005076a00000a40 phy02:T attached:
> > 50010b900004b87e
> > [  155.329169] sas: ex 5005076a00000a40 phy03:S attached:
> > 5005076a011061c0
> 
> No.  Look again.  the first four, which are a wide port, are phys 0-3 on
> the HBA.
> 
> The second four, which have separate attached devices are phys 0-3 on
> the expander (that's what the sightly cryptic ex 5005076a00000a40 phy<n>
> means).
> 
> You can see this better in the transport class tree
> 
> James

OK, that makes more sense, thank you for explaining. 

--Alexis
> 
> 
> -
> To unsubscribe from this list: send the line "unsubscribe linux-scsi" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH] aic94xx: make use of the new sas_port

[Douglas Gilbert]

[-- Attachment #1: Type: text/plain, Size: 3112 bytes --]

Douglas Gilbert wrote:
> Alexis Bruemmer wrote:
> 
>>These patches look good and are running on our x260 and the ppc64
>>(power).  I was just wondering about a few things under the sys/class/:
>>
>>there is a sas_device, sas_end_device, and a sas_expander dir.
>>sas_device contains the same info as sas_end_device, and a sas_expander
>>can we eliminate one or the other?  Or was this a design choice?
>>
>>Also during the boot process on the x260 with an expander we see these
>>printk's:
>>sas: phy[n] matched wide port0
>>If we are going to print this then we better be sure that this is truly
>>a wide port.  It seems under the sys/class/ we have both a single
>>port0:0 and then also port-0:0:[n]-- a port[n] for each phy[n].  and
>>although each phy seems to have the same sas_address the end_device
>>attached to that phy have unique addresses.  My understanding is that a
>>wide port has multiple phys with the same sas address and are attached
>>to one end device.
> 
> 
> A wide port is an abstraction just above the link
> layer in which two or more links running between
> the same two devices have phys with the same SAS
> addresses associated with them.
> 
> Assume we have two phys on a HBA connected to two
> phys on an expander, then two other phys on the
> expander connected to a dual ported SAS disk.
> That is one wide port connection (between the
> at HBA and the expander) and two narrow
> port connections (between the expander and each
> port on the SAS disk).
> 
> This comes about because phys on a HBA share a
> SAS address, phys on a expander share a SAS address
> but phys of a dual ported SAS disk have distinct
> SAS addresses.
> [Could someone suggest an instructive diagram url
> of this as the sas drafts are a bit complex.]
> 
> You are probably aware of this distinction but my
> guess is a lot of other people aren't.
> 
> Put another way, seen from a LLD or discover process when
> multiple phys on the same device have this tuple
> <SAS_address, attached_SAS_address> equal then there is
> a wide port connection.

I don't have a diagram, but I do have some real
output. The hardware is set up as in my example
above with a LSI SAS HBA (0x500605b00006f260).
There is a twelve port expander (0x500605b000000af0)
with phy ids 3 and 4 connected to the HBA, and phy
ids 8 and 9 connected to a dual ported SAS disk.

sg_vpd shows the naa-5 addresses for /dev/sdb
and /dev/sdc . So many to choose from! The first
is the lu name, next is the port identifier and the
last two are different renderings of the target device
name. Only the target port identifiers differ (the
secondary port is usually one higher than the primary).

[Aside: if lsscsi was to show a device's wwn then I
 believe it should show the lu name (if there is one).
 Thoughts?]

Doing a SMP discover on the expander's connected phys
shows a 2 link wide port to the HBA because the
two <sas_addres, attached_sas_address> tuples match. Note
that the attached phy ids (i.e. on the HBA) are not
consecutive. There are also two narrow port links to the
primary and secondary ports of the SAS disk.

Doug Gilbert



[-- Attachment #2: wide_narrow_ex.txt --]
[-- Type: text/plain, Size: 2057 bytes --]

[root@sas ~]# lsscsi
[0:0:0:0]    disk    ATA      WDC WD360GD-00FL 33.0  /dev/sda
[4:2:0:0]    disk    SEAGATE  ST336754SS       0003  /dev/sdb
[4:2:1:0]    disk    SEAGATE  ST336754SS       0003  /dev/sdc

[root@sas ~]# sg_vpd /dev/sdb -iq
0x5000c50000520a2b
0x5000c50000520a2a
0x5000c50000520a28
naa.5000C50000520A28

[root@sas ~]# sg_vpd /dev/sdc -iq
0x5000c50000520a2b
0x5000c50000520a29
0x5000c50000520a28
naa.5000C50000520A28

[root@sas ~]# smp_discover -s 0x500605b000000af0 /dev/mptctl -b -p 3
Discover response:
  phy identifier: 3
  attached device type: end device
  negotiated physical link rate: phy enabled; 3 Gbps
  attached initiator: ssp=1 stp=1 smp=1 sata_host=0
  attached target: ssp=0 stp=0 smp=0 sata_device=0
  SAS address: 0x500605b000000af0
  attached SAS address: 0x500605b00006f260
  attached phy identifier: 0

[root@sas ~]# smp_discover -s 0x500605b000000af0 /dev/mptctl -b -p 4
Discover response:
  phy identifier: 4
  attached device type: end device
  negotiated physical link rate: phy enabled; 3 Gbps
  attached initiator: ssp=1 stp=1 smp=1 sata_host=0
  attached target: ssp=0 stp=0 smp=0 sata_device=0
  SAS address: 0x500605b000000af0
  attached SAS address: 0x500605b00006f260
  attached phy identifier: 3

[root@sas ~]# smp_discover -s 0x500605b000000af0 /dev/mptctl -b -p 8
Discover response:
  phy identifier: 8
  attached device type: end device
  negotiated physical link rate: phy enabled; 3 Gbps
  attached initiator: ssp=0 stp=0 smp=0 sata_host=0
  attached target: ssp=1 stp=0 smp=0 sata_device=0
  SAS address: 0x500605b000000af0
  attached SAS address: 0x5000c50000520a2a
  attached phy identifier: 1

[root@sas ~]# smp_discover -s 0x500605b000000af0 /dev/mptctl -b -p 9
Discover response:
  phy identifier: 9
  attached device type: end device
  negotiated physical link rate: phy enabled; 3 Gbps
  attached initiator: ssp=0 stp=0 smp=0 sata_host=0
  attached target: ssp=1 stp=0 smp=0 sata_device=0
  SAS address: 0x500605b000000af0
  attached SAS address: 0x5000c50000520a29
  attached phy identifier: 0

Re: [PATCH] aic94xx: make use of the new sas_port

[James Bottomley]

On Thu, 2006-06-01 at 12:21 -0400, Douglas Gilbert wrote:
> >>Also during the boot process on the x260 with an expander we see these
> >>printk's:
> >>sas: phy[n] matched wide port0
> >>If we are going to print this then we better be sure that this is truly
> >>a wide port.  It seems under the sys/class/ we have both a single
> >>port0:0 and then also port-0:0:[n]-- a port[n] for each phy[n].  and
> >>although each phy seems to have the same sas_address the end_device
> >>attached to that phy have unique addresses.  My understanding is that a
> >>wide port has multiple phys with the same sas address and are attached
> >>to one end device.

Just for reference, this is what my x260 looks like:

./
  +-- phy-0:0/
  |   +-- port@ -> ../../../../../devices/pci0000:01/0000:01:02.0/host0/port-0:0
  |   +-- power/
  |   |   +-- state
  |   |   +-- wakeup
  |   +-- sas_phy:phy-0:0@ -> ../../../../../class/sas_phy/phy-0:0
  |   +-- uevent
  +-- phy-0:1/
  |   +-- port@ -> ../../../../../devices/pci0000:01/0000:01:02.0/host0/port-0:0
  |   +-- power/
  |   |   +-- state
  |   |   +-- wakeup
  |   +-- sas_phy:phy-0:1@ -> ../../../../../class/sas_phy/phy-0:1
  |   +-- uevent
  +-- phy-0:2/
  |   +-- port@ -> ../../../../../devices/pci0000:01/0000:01:02.0/host0/port-0:0
  |   +-- power/
  |   |   +-- state
  |   |   +-- wakeup
  |   +-- sas_phy:phy-0:2@ -> ../../../../../class/sas_phy/phy-0:2
  |   +-- uevent
  +-- phy-0:3/
  |   +-- port@ -> ../../../../../devices/pci0000:01/0000:01:02.0/host0/port-0:0
  |   +-- power/
  |   |   +-- state
  |   |   +-- wakeup
  |   +-- sas_phy:phy-0:3@ -> ../../../../../class/sas_phy/phy-0:3
  |   +-- uevent
  +-- phy-0:4/
  |   +-- port@ -> ../../../../../devices/pci0000:01/0000:01:02.0/host0/port-0:1
  |   +-- power/
  |   |   +-- state
  |   |   +-- wakeup
  |   +-- sas_phy:phy-0:4@ -> ../../../../../class/sas_phy/phy-0:4
  |   +-- uevent
  +-- phy-0:5/
  |   +-- port@ -> ../../../../../devices/pci0000:01/0000:01:02.0/host0/port-0:1
  |   +-- power/
  |   |   +-- state
  |   |   +-- wakeup
  |   +-- sas_phy:phy-0:5@ -> ../../../../../class/sas_phy/phy-0:5
  |   +-- uevent
  +-- phy-0:6/
  |   +-- port@ -> ../../../../../devices/pci0000:01/0000:01:02.0/host0/port-0:1
  |   +-- power/
  |   |   +-- state
  |   |   +-- wakeup
  |   +-- sas_phy:phy-0:6@ -> ../../../../../class/sas_phy/phy-0:6
  |   +-- uevent
  +-- phy-0:7/
  |   +-- port@ -> ../../../../../devices/pci0000:01/0000:01:02.0/host0/port-0:1
  |   +-- power/
  |   |   +-- state
  |   |   +-- wakeup
  |   +-- sas_phy:phy-0:7@ -> ../../../../../class/sas_phy/phy-0:7
  |   +-- uevent
  +-- port-0:0/
  |   +-- expander-0:0/
  |   |   +-- phy-0-0:0/
  |   |   |   +-- power/
  |   |   |   |   +-- state
  |   |   |   |   +-- wakeup
  |   |   |   +-- sas_phy:phy-0-0:0@ -> ../../../../../../../class/sas_phy/phy-0-0:0
  |   |   |   +-- uevent
  |   |   +-- phy-0-0:1/
  |   |   |   +-- power/
  |   |   |   |   +-- state
  |   |   |   |   +-- wakeup
  |   |   |   +-- sas_phy:phy-0-0:1@ -> ../../../../../../../class/sas_phy/phy-0-0:1
  |   |   |   +-- uevent
  |   |   +-- phy-0-0:10/
  |   |   |   +-- power/
  |   |   |   |   +-- state
  |   |   |   |   +-- wakeup
  |   |   |   +-- sas_phy:phy-0-0:10@ -> ../../../../../../../class/sas_phy/phy-0-0:10
  |   |   |   +-- uevent
  |   |   +-- phy-0-0:11/
  |   |   |   +-- power/
  |   |   |   |   +-- state
  |   |   |   |   +-- wakeup
  |   |   |   +-- sas_phy:phy-0-0:11@ -> ../../../../../../../class/sas_phy/phy-0-0:11
  |   |   |   +-- uevent
  |   |   +-- phy-0-0:12/
  |   |   |   +-- port@ -> ../../../../../../../devices/pci0000:01/0000:01:02.0/host0/port-0:0/expander-0:0/port-0:0:0
  |   |   |   +-- power/
  |   |   |   |   +-- state
  |   |   |   |   +-- wakeup
  |   |   |   +-- sas_phy:phy-0-0:12@ -> ../../../../../../../class/sas_phy/phy-0-0:12
  |   |   |   +-- uevent
  |   |   +-- phy-0-0:2/
  |   |   |   +-- power/
  |   |   |   |   +-- state
  |   |   |   |   +-- wakeup
  |   |   |   +-- sas_phy:phy-0-0:2@ -> ../../../../../../../class/sas_phy/phy-0-0:2
  |   |   |   +-- uevent
  |   |   +-- phy-0-0:3/
  |   |   |   +-- power/
  |   |   |   |   +-- state
  |   |   |   |   +-- wakeup
  |   |   |   +-- sas_phy:phy-0-0:3@ -> ../../../../../../../class/sas_phy/phy-0-0:3
  |   |   |   +-- uevent
  |   |   +-- phy-0-0:4/
  |   |   |   +-- power/
  |   |   |   |   +-- state
  |   |   |   |   +-- wakeup
  |   |   |   +-- sas_phy:phy-0-0:4@ -> ../../../../../../../class/sas_phy/phy-0-0:4
  |   |   |   +-- uevent
  |   |   +-- phy-0-0:5/
  |   |   |   +-- power/
  |   |   |   |   +-- state
  |   |   |   |   +-- wakeup
  |   |   |   +-- sas_phy:phy-0-0:5@ -> ../../../../../../../class/sas_phy/phy-0-0:5
  |   |   |   +-- uevent
  |   |   +-- phy-0-0:6/
  |   |   |   +-- power/
  |   |   |   |   +-- state
  |   |   |   |   +-- wakeup
  |   |   |   +-- sas_phy:phy-0-0:6@ -> ../../../../../../../class/sas_phy/phy-0-0:6
  |   |   |   +-- uevent
  |   |   +-- phy-0-0:7/
  |   |   |   +-- power/
  |   |   |   |   +-- state
  |   |   |   |   +-- wakeup
  |   |   |   +-- sas_phy:phy-0-0:7@ -> ../../../../../../../class/sas_phy/phy-0-0:7
  |   |   |   +-- uevent
  |   |   +-- phy-0-0:8/
  |   |   |   +-- power/
  |   |   |   |   +-- state
  |   |   |   |   +-- wakeup
  |   |   |   +-- sas_phy:phy-0-0:8@ -> ../../../../../../../class/sas_phy/phy-0-0:8
  |   |   |   +-- uevent
  |   |   +-- phy-0-0:9/
  |   |   |   +-- power/
  |   |   |   |   +-- state
  |   |   |   |   +-- wakeup
  |   |   |   +-- sas_phy:phy-0-0:9@ -> ../../../../../../../class/sas_phy/phy-0-0:9
  |   |   |   +-- uevent
  |   |   +-- port-0:0:0/
  |   |   |   +-- end_device-0:0-0/
  |   |   |   |   +-- power/
  |   |   |   |   |   +-- state
  |   |   |   |   |   +-- wakeup
  |   |   |   |   +-- sas_device:end_device-0:0-0@ -> ../../../../../../../../class/sas_device/end_device-0:0-0
  |   |   |   |   +-- sas_end_device:end_device-0:0-0@ -> ../../../../../../../../class/sas_end_device/end_device-0:0-0
  |   |   |   |   +-- target0:0:0/
  |   |   |   |   |   +-- 0:0:0:0/
  |   |   |   |   |   |   +-- bus@ -> ../../../../../../../../../../bus/scsi
  |   |   |   |   |   |   +-- delete
  |   |   |   |   |   |   +-- device_blocked
  |   |   |   |   |   |   +-- iocounterbits
  |   |   |   |   |   |   +-- iodone_cnt
  |   |   |   |   |   |   +-- ioerr_cnt
  |   |   |   |   |   |   +-- iorequest_cnt
  |   |   |   |   |   |   +-- model
  |   |   |   |   |   |   +-- power/
  |   |   |   |   |   |   |   +-- state
  |   |   |   |   |   |   |   +-- wakeup
  |   |   |   |   |   |   +-- queue_depth
  |   |   |   |   |   |   +-- queue_type
  |   |   |   |   |   |   +-- rescan
  |   |   |   |   |   |   +-- rev
  |   |   |   |   |   |   +-- scsi_device:0:0:0:0@ -> ../../../../../../../../../../class/scsi_device/0:0:0:0
  |   |   |   |   |   |   +-- scsi_level
  |   |   |   |   |   |   +-- state
  |   |   |   |   |   |   +-- timeout
  |   |   |   |   |   |   +-- type
  |   |   |   |   |   |   +-- uevent
  |   |   |   |   |   |   +-- vendor
  |   |   |   |   |   +-- power/
  |   |   |   |   |   |   +-- state
  |   |   |   |   |   |   +-- wakeup
  |   |   |   |   |   +-- uevent
  |   |   |   |   +-- uevent
  |   |   |   +-- phy-0-0:12@ -> ../../../../../../../devices/pci0000:01/0000:01:02.0/host0/port-0:0/expander-0:0/phy-0-0:12
  |   |   |   +-- power/
  |   |   |   |   +-- state
  |   |   |   |   +-- wakeup
  |   |   |   +-- sas_port:port-0:0:0@ -> ../../../../../../../class/sas_port/port-0:0:0
  |   |   |   +-- uevent
  |   |   +-- power/
  |   |   |   +-- state
  |   |   |   +-- wakeup
  |   |   +-- sas_device:expander-0:0@ -> ../../../../../../class/sas_device/expander-0:0
  |   |   +-- sas_expander:expander-0:0@ -> ../../../../../../class/sas_expander/expander-0:0
  |   |   +-- uevent
  |   +-- phy-0:0@ -> ../../../../../devices/pci0000:01/0000:01:02.0/host0/phy-0:0
  |   +-- phy-0:1@ -> ../../../../../devices/pci0000:01/0000:01:02.0/host0/phy-0:1
  |   +-- phy-0:2@ -> ../../../../../devices/pci0000:01/0000:01:02.0/host0/phy-0:2
  |   +-- phy-0:3@ -> ../../../../../devices/pci0000:01/0000:01:02.0/host0/phy-0:3
  |   +-- power/
  |   |   +-- state
  |   |   +-- wakeup
  |   +-- sas_port:port-0:0@ -> ../../../../../class/sas_port/port-0:0
  |   +-- uevent
  +-- port-0:1/
  |   +-- expander-0:1/
  |   |   +-- phy-0-1:0/
  |   |   |   +-- port@ -> ../../../../../../../devices/pci0000:01/0000:01:02.0/host0/port-0:1/expander-0:1/port-0:1:0
  |   |   |   +-- power/
  |   |   |   |   +-- state
  |   |   |   |   +-- wakeup
  |   |   |   +-- sas_phy:phy-0-1:0@ -> ../../../../../../../class/sas_phy/phy-0-1:0
  |   |   |   +-- uevent
  |   |   +-- phy-0-1:1/
  |   |   |   +-- power/
  |   |   |   |   +-- state
  |   |   |   |   +-- wakeup
  |   |   |   +-- sas_phy:phy-0-1:1@ -> ../../../../../../../class/sas_phy/phy-0-1:1
  |   |   |   +-- uevent
  |   |   +-- phy-0-1:10/
  |   |   |   +-- power/
  |   |   |   |   +-- state
  |   |   |   |   +-- wakeup
  |   |   |   +-- sas_phy:phy-0-1:10@ -> ../../../../../../../class/sas_phy/phy-0-1:10
  |   |   |   +-- uevent
  |   |   +-- phy-0-1:11/
  |   |   |   +-- power/
  |   |   |   |   +-- state
  |   |   |   |   +-- wakeup
  |   |   |   +-- sas_phy:phy-0-1:11@ -> ../../../../../../../class/sas_phy/phy-0-1:11
  |   |   |   +-- uevent
  |   |   +-- phy-0-1:12/
  |   |   |   +-- port@ -> ../../../../../../../devices/pci0000:01/0000:01:02.0/host0/port-0:1/expander-0:1/port-0:1:1
  |   |   |   +-- power/
  |   |   |   |   +-- state
  |   |   |   |   +-- wakeup
  |   |   |   +-- sas_phy:phy-0-1:12@ -> ../../../../../../../class/sas_phy/phy-0-1:12
  |   |   |   +-- uevent
  |   |   +-- phy-0-1:2/
  |   |   |   +-- power/
  |   |   |   |   +-- state
  |   |   |   |   +-- wakeup
  |   |   |   +-- sas_phy:phy-0-1:2@ -> ../../../../../../../class/sas_phy/phy-0-1:2
  |   |   |   +-- uevent
  |   |   +-- phy-0-1:3/
  |   |   |   +-- power/
  |   |   |   |   +-- state
  |   |   |   |   +-- wakeup
  |   |   |   +-- sas_phy:phy-0-1:3@ -> ../../../../../../../class/sas_phy/phy-0-1:3
  |   |   |   +-- uevent
  |   |   +-- phy-0-1:4/
  |   |   |   +-- power/
  |   |   |   |   +-- state
  |   |   |   |   +-- wakeup
  |   |   |   +-- sas_phy:phy-0-1:4@ -> ../../../../../../../class/sas_phy/phy-0-1:4
  |   |   |   +-- uevent
  |   |   +-- phy-0-1:5/
  |   |   |   +-- power/
  |   |   |   |   +-- state
  |   |   |   |   +-- wakeup
  |   |   |   +-- sas_phy:phy-0-1:5@ -> ../../../../../../../class/sas_phy/phy-0-1:5
  |   |   |   +-- uevent
  |   |   +-- phy-0-1:6/
  |   |   |   +-- power/
  |   |   |   |   +-- state
  |   |   |   |   +-- wakeup
  |   |   |   +-- sas_phy:phy-0-1:6@ -> ../../../../../../../class/sas_phy/phy-0-1:6
  |   |   |   +-- uevent
  |   |   +-- phy-0-1:7/
  |   |   |   +-- power/
  |   |   |   |   +-- state
  |   |   |   |   +-- wakeup
  |   |   |   +-- sas_phy:phy-0-1:7@ -> ../../../../../../../class/sas_phy/phy-0-1:7
  |   |   |   +-- uevent
  |   |   +-- phy-0-1:8/
  |   |   |   +-- power/
  |   |   |   |   +-- state
  |   |   |   |   +-- wakeup
  |   |   |   +-- sas_phy:phy-0-1:8@ -> ../../../../../../../class/sas_phy/phy-0-1:8
  |   |   |   +-- uevent
  |   |   +-- phy-0-1:9/
  |   |   |   +-- power/
  |   |   |   |   +-- state
  |   |   |   |   +-- wakeup
  |   |   |   +-- sas_phy:phy-0-1:9@ -> ../../../../../../../class/sas_phy/phy-0-1:9
  |   |   |   +-- uevent
  |   |   +-- port-0:1:0/
  |   |   |   +-- end_device-0:0-0/
  |   |   |   |   +-- power/
  |   |   |   |   |   +-- state
  |   |   |   |   |   +-- wakeup
  |   |   |   |   +-- target0:0:1/
  |   |   |   |   |   +-- 0:0:1:0/
  |   |   |   |   |   |   +-- block:sda@ -> ../../../../../../../../../../block/sda
  |   |   |   |   |   |   +-- bus@ -> ../../../../../../../../../../bus/scsi
  |   |   |   |   |   |   +-- delete
  |   |   |   |   |   |   +-- device_blocked
  |   |   |   |   |   |   +-- driver@ -> ../../../../../../../../../../bus/scsi/drivers/sd
  |   |   |   |   |   |   +-- iocounterbits
  |   |   |   |   |   |   +-- iodone_cnt
  |   |   |   |   |   |   +-- ioerr_cnt
  |   |   |   |   |   |   +-- iorequest_cnt
  |   |   |   |   |   |   +-- model
  |   |   |   |   |   |   +-- power/
  |   |   |   |   |   |   |   +-- state
  |   |   |   |   |   |   |   +-- wakeup
  |   |   |   |   |   |   +-- queue_depth
  |   |   |   |   |   |   +-- queue_type
  |   |   |   |   |   |   +-- rescan
  |   |   |   |   |   |   +-- rev
  |   |   |   |   |   |   +-- scsi_device:0:0:1:0@ -> ../../../../../../../../../../class/scsi_device/0:0:1:0
  |   |   |   |   |   |   +-- scsi_disk:0:0:1:0@ -> ../../../../../../../../../../class/scsi_disk/0:0:1:0
  |   |   |   |   |   |   +-- scsi_level
  |   |   |   |   |   |   +-- state
  |   |   |   |   |   |   +-- timeout
  |   |   |   |   |   |   +-- type
  |   |   |   |   |   |   +-- uevent
  |   |   |   |   |   |   +-- vendor
  |   |   |   |   |   +-- power/
  |   |   |   |   |   |   +-- state
  |   |   |   |   |   |   +-- wakeup
  |   |   |   |   |   +-- uevent
  |   |   |   |   +-- uevent
  |   |   |   +-- phy-0-1:0@ -> ../../../../../../../devices/pci0000:01/0000:01:02.0/host0/port-0:1/expander-0:1/phy-0-1:0
  |   |   |   +-- power/
  |   |   |   |   +-- state
  |   |   |   |   +-- wakeup
  |   |   |   +-- sas_port:port-0:1:0@ -> ../../../../../../../class/sas_port/port-0:1:0
  |   |   |   +-- uevent
  |   |   +-- port-0:1:1/
  |   |   |   +-- end_device-0:1-1/
  |   |   |   |   +-- power/
  |   |   |   |   |   +-- state
  |   |   |   |   |   +-- wakeup
  |   |   |   |   +-- sas_device:end_device-0:1-1@ -> ../../../../../../../../class/sas_device/end_device-0:1-1
  |   |   |   |   +-- sas_end_device:end_device-0:1-1@ -> ../../../../../../../../class/sas_end_device/end_device-0:1-1
  |   |   |   |   +-- target0:1:2/
  |   |   |   |   |   +-- 0:1:2:0/
  |   |   |   |   |   |   +-- bus@ -> ../../../../../../../../../../bus/scsi
  |   |   |   |   |   |   +-- delete
  |   |   |   |   |   |   +-- device_blocked
  |   |   |   |   |   |   +-- iocounterbits
  |   |   |   |   |   |   +-- iodone_cnt
  |   |   |   |   |   |   +-- ioerr_cnt
  |   |   |   |   |   |   +-- iorequest_cnt
  |   |   |   |   |   |   +-- model
  |   |   |   |   |   |   +-- power/
  |   |   |   |   |   |   |   +-- state
  |   |   |   |   |   |   |   +-- wakeup
  |   |   |   |   |   |   +-- queue_depth
  |   |   |   |   |   |   +-- queue_type
  |   |   |   |   |   |   +-- rescan
  |   |   |   |   |   |   +-- rev
  |   |   |   |   |   |   +-- scsi_device:0:1:2:0@ -> ../../../../../../../../../../class/scsi_device/0:1:2:0
  |   |   |   |   |   |   +-- scsi_level
  |   |   |   |   |   |   +-- state
  |   |   |   |   |   |   +-- timeout
  |   |   |   |   |   |   +-- type
  |   |   |   |   |   |   +-- uevent
  |   |   |   |   |   |   +-- vendor
  |   |   |   |   |   +-- power/
  |   |   |   |   |   |   +-- state
  |   |   |   |   |   |   +-- wakeup
  |   |   |   |   |   +-- uevent
  |   |   |   |   +-- uevent
  |   |   |   +-- phy-0-1:12@ -> ../../../../../../../devices/pci0000:01/0000:01:02.0/host0/port-0:1/expander-0:1/phy-0-1:12
  |   |   |   +-- power/
  |   |   |   |   +-- state
  |   |   |   |   +-- wakeup
  |   |   |   +-- sas_port:port-0:1:1@ -> ../../../../../../../class/sas_port/port-0:1:1
  |   |   |   +-- uevent
  |   |   +-- power/
  |   |   |   +-- state
  |   |   |   +-- wakeup
  |   |   +-- sas_device:expander-0:1@ -> ../../../../../../class/sas_device/expander-0:1
  |   |   +-- sas_expander:expander-0:1@ -> ../../../../../../class/sas_expander/expander-0:1
  |   |   +-- uevent
  |   +-- phy-0:4@ -> ../../../../../devices/pci0000:01/0000:01:02.0/host0/phy-0:4
  |   +-- phy-0:5@ -> ../../../../../devices/pci0000:01/0000:01:02.0/host0/phy-0:5
  |   +-- phy-0:6@ -> ../../../../../devices/pci0000:01/0000:01:02.0/host0/phy-0:6
  |   +-- phy-0:7@ -> ../../../../../devices/pci0000:01/0000:01:02.0/host0/phy-0:7
  |   +-- power/
  |   |   +-- state
  |   |   +-- wakeup
  |   +-- sas_port:port-0:1@ -> ../../../../../class/sas_port/port-0:1
  |   +-- uevent
  +-- power/
  |   +-- state
  |   +-- wakeup
  +-- sas_host:host0@ -> ../../../../class/sas_host/host0
  +-- scsi_host:host0@ -> ../../../../class/scsi_host/host0
  +-- uevent

As you see: two wide ports over 8 phys on the HBA then three single
ports on the expanders for the two SES monitors and the one disk.

James