[PATCH net-next 2.6] myri10ge: DCA update

[PATCH net-next 2.6] myri10ge: DCA update

[Andrew Gallatin]

[-- Attachment #1: Type: text/plain, Size: 758 bytes --]

This patch contains the following DCA improvements to myri10ge:

1) Finally move myri10ge to use dca3 API

2) Disable PCIe relaxed ordering when enabling DCA on
     myri10ge.  This provides a performance boost on Nehalem
     based Xeons

3) Make sure to properly initialize NIC's DCA state when it is enabled,
     rather than giving the NIC a bogus tag (0) and waiting for
     the first received packet to trigger an update.  Not using a
     real tag can cause hardware exceptions on some motherboards
     when a CPU socket is empty.

3) Always update the cached CPU when our interrupt affinity changes
     so as to avoid excessive calls to dca3_get_tag()

Signed-off-by: Andrew Gallatin <gallatin@myri.com>
Signed-off-by: Loic Prylli <loic@myri.com>

[-- Attachment #2: myri10ge_dca.diff --]
[-- Type: text/x-diff, Size: 2222 bytes --]

diff --git a/drivers/net/myri10ge/myri10ge.c b/drivers/net/myri10ge/myri10ge.c
index 4f3a3c0..545d481 100644
--- a/drivers/net/myri10ge/myri10ge.c
+++ b/drivers/net/myri10ge/myri10ge.c
@@ -225,6 +225,7 @@ struct myri10ge_priv {
 	struct msix_entry *msix_vectors;
 #ifdef CONFIG_MYRI10GE_DCA
 	int dca_enabled;
+	int relaxed_order;
 #endif
 	u32 link_state;
 	unsigned int rdma_tags_available;
@@ -1074,10 +1075,29 @@ static int myri10ge_reset(struct myri10ge_priv *mgp)
 }
 
 #ifdef CONFIG_MYRI10GE_DCA
+static int
+myri10ge_toggle_relaxed(struct pci_dev *pdev, int on)
+{
+	int ret, cap, err;
+	u16 ctl;
+
+	cap = pci_find_capability(pdev, PCI_CAP_ID_EXP);
+	if (!cap)
+		return (0);
+
+	err = pci_read_config_word(pdev, cap + PCI_EXP_DEVCTL, &ctl);
+	ret = (ctl & PCI_EXP_DEVCTL_RELAX_EN) >> 4;
+	if (ret != on) {
+		ctl &= ~PCI_EXP_DEVCTL_RELAX_EN;
+		ctl |= (on << 4);
+		pci_write_config_word(pdev, cap + PCI_EXP_DEVCTL, ctl);
+	}
+	return (ret);	
+}
+
 static void
 myri10ge_write_dca(struct myri10ge_slice_state *ss, int cpu, int tag)
 {
-	ss->cpu = cpu;
 	ss->cached_dca_tag = tag;
 	put_be32(htonl(tag), ss->dca_tag);
 }
@@ -1088,9 +1108,10 @@ static inline void myri10ge_update_dca(struct myri10ge_slice_state *ss)
 	int tag;
 
 	if (cpu != ss->cpu) {
-		tag = dca_get_tag(cpu);
+		tag = dca3_get_tag(&ss->mgp->pdev->dev, cpu);
 		if (ss->cached_dca_tag != tag)
 			myri10ge_write_dca(ss, cpu, tag);
+		ss->cpu = cpu;
 	}
 	put_cpu();
 }
@@ -1113,9 +1134,13 @@ static void myri10ge_setup_dca(struct myri10ge_priv *mgp)
 				"dca_add_requester() failed, err=%d\n", err);
 		return;
 	}
+	mgp->relaxed_order = myri10ge_toggle_relaxed(pdev, 0);
 	mgp->dca_enabled = 1;
-	for (i = 0; i < mgp->num_slices; i++)
-		myri10ge_write_dca(&mgp->ss[i], -1, 0);
+	for (i = 0; i < mgp->num_slices; i++) {
+		mgp->ss[i].cpu = -1;
+		mgp->ss[i].cached_dca_tag = -1;
+		myri10ge_update_dca(&mgp->ss[i]);
+	 }
 }
 
 static void myri10ge_teardown_dca(struct myri10ge_priv *mgp)
@@ -1126,6 +1151,8 @@ static void myri10ge_teardown_dca(struct myri10ge_priv *mgp)
 	if (!mgp->dca_enabled)
 		return;
 	mgp->dca_enabled = 0;
+	if (mgp->relaxed_order)
+		myri10ge_toggle_relaxed(pdev, 1);
 	err = dca_remove_requester(&pdev->dev);
 }
 

Re: [PATCH net-next 2.6] myri10ge: DCA update

[Eric Dumazet]

Le mardi 28 septembre 2010 à 09:41 -0400, Andrew Gallatin a écrit :
> This patch contains the following DCA improvements to myri10ge:
> 
> 1) Finally move myri10ge to use dca3 API
> 
> 2) Disable PCIe relaxed ordering when enabling DCA on
>      myri10ge.  This provides a performance boost on Nehalem
>      based Xeons
> 
> 3) Make sure to properly initialize NIC's DCA state when it is enabled,
>      rather than giving the NIC a bogus tag (0) and waiting for
>      the first received packet to trigger an update.  Not using a
>      real tag can cause hardware exceptions on some motherboards
>      when a CPU socket is empty.
> 
> 3) Always update the cached CPU when our interrupt affinity changes
>      so as to avoid excessive calls to dca3_get_tag()
> 
> Signed-off-by: Andrew Gallatin <gallatin@myri.com>
> Signed-off-by: Loic Prylli <loic@myri.com>

ERROR: return is not a function, parentheses are not required
#99: FILE: drivers/net/myri10ge/myri10ge.c:1086:
+		return (0);

ERROR: trailing whitespace
#108: FILE: drivers/net/myri10ge/myri10ge.c:1095:
+^Ireturn (ret);^I$

ERROR: return is not a function, parentheses are not required
#108: FILE: drivers/net/myri10ge/myri10ge.c:1095:
+	return (ret);	

total: 3 errors, 0 warnings, 71 lines checked

Re: [PATCH net-next 2.6] myri10ge: DCA update

[Andrew Gallatin]

[-- Attachment #1: Type: text/plain, Size: 780 bytes --]


> total: 3 errors, 0 warnings, 71 lines checked

Sorry! From googling, this seems to be checkpatch.pl output,
which I now know that I'm supposed to run before submitting
a patch.

Please accept my apologies.  Brice is no longer maintaining the
in-kernel version of myri10ge, and I'm trying to get some improvements
and fixes we've done for our vendor driver merged, so the in-kernel
version does not lag too far behind.  I'm pretty new to dealing
directly with the linux kernel lists, so I apologize for my
mistakes.

I've corrected the style problems pointed out by checkpatch, and
I've attached a new diff.  Is that sufficient, or do I start a
new thread, or..?

Thank you,

Drew

Signed-off-by: Andrew Gallatin <gallatin@myri.com>
Signed-off-by: Loic Prylli <loic@myri.com>



[-- Attachment #2: myri10ge_dca_2.diff --]
[-- Type: text/x-diff, Size: 2216 bytes --]

diff --git a/drivers/net/myri10ge/myri10ge.c b/drivers/net/myri10ge/myri10ge.c
index 4f3a3c0..02dd92e 100644
--- a/drivers/net/myri10ge/myri10ge.c
+++ b/drivers/net/myri10ge/myri10ge.c
@@ -225,6 +225,7 @@ struct myri10ge_priv {
 	struct msix_entry *msix_vectors;
 #ifdef CONFIG_MYRI10GE_DCA
 	int dca_enabled;
+	int relaxed_order;
 #endif
 	u32 link_state;
 	unsigned int rdma_tags_available;
@@ -1074,10 +1075,28 @@ static int myri10ge_reset(struct myri10ge_priv *mgp)
 }
 
 #ifdef CONFIG_MYRI10GE_DCA
+static int myri10ge_toggle_relaxed(struct pci_dev *pdev, int on)
+{
+	int ret, cap, err;
+	u16 ctl;
+
+	cap = pci_find_capability(pdev, PCI_CAP_ID_EXP);
+	if (!cap)
+		return 0;
+
+	err = pci_read_config_word(pdev, cap + PCI_EXP_DEVCTL, &ctl);
+	ret = (ctl & PCI_EXP_DEVCTL_RELAX_EN) >> 4;
+	if (ret != on) {
+		ctl &= ~PCI_EXP_DEVCTL_RELAX_EN;
+		ctl |= (on << 4);
+		pci_write_config_word(pdev, cap + PCI_EXP_DEVCTL, ctl);
+	}
+	return ret;
+}
+
 static void
 myri10ge_write_dca(struct myri10ge_slice_state *ss, int cpu, int tag)
 {
-	ss->cpu = cpu;
 	ss->cached_dca_tag = tag;
 	put_be32(htonl(tag), ss->dca_tag);
 }
@@ -1088,9 +1107,10 @@ static inline void myri10ge_update_dca(struct myri10ge_slice_state *ss)
 	int tag;
 
 	if (cpu != ss->cpu) {
-		tag = dca_get_tag(cpu);
+		tag = dca3_get_tag(&ss->mgp->pdev->dev, cpu);
 		if (ss->cached_dca_tag != tag)
 			myri10ge_write_dca(ss, cpu, tag);
+		ss->cpu = cpu;
 	}
 	put_cpu();
 }
@@ -1113,9 +1133,13 @@ static void myri10ge_setup_dca(struct myri10ge_priv *mgp)
 				"dca_add_requester() failed, err=%d\n", err);
 		return;
 	}
+	mgp->relaxed_order = myri10ge_toggle_relaxed(pdev, 0);
 	mgp->dca_enabled = 1;
-	for (i = 0; i < mgp->num_slices; i++)
-		myri10ge_write_dca(&mgp->ss[i], -1, 0);
+	for (i = 0; i < mgp->num_slices; i++) {
+		mgp->ss[i].cpu = -1;
+		mgp->ss[i].cached_dca_tag = -1;
+		myri10ge_update_dca(&mgp->ss[i]);
+	 }
 }
 
 static void myri10ge_teardown_dca(struct myri10ge_priv *mgp)
@@ -1126,6 +1150,8 @@ static void myri10ge_teardown_dca(struct myri10ge_priv *mgp)
 	if (!mgp->dca_enabled)
 		return;
 	mgp->dca_enabled = 0;
+	if (mgp->relaxed_order)
+		myri10ge_toggle_relaxed(pdev, 1);
 	err = dca_remove_requester(&pdev->dev);
 }
 

Re: [PATCH net-next 2.6] myri10ge: DCA update

[David Miller]

From: Andrew Gallatin <gallatin@myri.com>
Date: Tue, 28 Sep 2010 11:00:12 -0400

> I've corrected the style problems pointed out by checkpatch, and
> I've attached a new diff.  Is that sufficient, or do I start a
> new thread, or..?

Please send a new fresh patch submission, or as you term it
"start a new thread" :-)