[RFC PATCH v2 0/2] nb8800 suspend/resume support

[RFC PATCH v2 0/2] nb8800 suspend/resume support

[Mason]

Hello,

I need suspend/resume support in the nb8800 driver.
On tango platforms, suspend loses all context (MMIO registers).
To make the task easy, we just close the device on suspend,
and open it again on resume. This requires properly resetting
the HW on resume.

Patch 1 moves all the HW init to nb8800_init()
Patch 2 adds suspend/resume support

Regards.

[RFC PATCH v2 1/2] net: ethernet: nb8800: Reset HW block in ndo_open

[Mason]

Move all HW initializations to nb8800_init.
This provides the basis for suspend/resume support.
---
 drivers/net/ethernet/aurora/nb8800.c | 50 +++++++++++++++++-------------------
 drivers/net/ethernet/aurora/nb8800.h |  1 +
 2 files changed, 25 insertions(+), 26 deletions(-)

diff --git a/drivers/net/ethernet/aurora/nb8800.c b/drivers/net/ethernet/aurora/nb8800.c
index e94159507847..aa18ea25d91f 100644
--- a/drivers/net/ethernet/aurora/nb8800.c
+++ b/drivers/net/ethernet/aurora/nb8800.c
@@ -39,6 +39,7 @@
 
 #include "nb8800.h"
 
+static void nb8800_init(struct net_device *dev);
 static void nb8800_tx_done(struct net_device *dev);
 static int nb8800_dma_stop(struct net_device *dev);
 
@@ -957,6 +958,8 @@ static int nb8800_open(struct net_device *dev)
 	struct phy_device *phydev;
 	int err;
 
+	nb8800_init(dev);
+
 	/* clear any pending interrupts */
 	nb8800_writel(priv, NB8800_RXC_SR, 0xf);
 	nb8800_writel(priv, NB8800_TXC_SR, 0xf);
@@ -1246,11 +1249,6 @@ static int nb8800_hw_init(struct net_device *dev)
 	nb8800_writeb(priv, NB8800_PQ1, val >> 8);
 	nb8800_writeb(priv, NB8800_PQ2, val & 0xff);
 
-	/* Auto-negotiate by default */
-	priv->pause_aneg = true;
-	priv->pause_rx = true;
-	priv->pause_tx = true;
-
 	nb8800_mc_init(dev, 0);
 
 	return 0;
@@ -1350,6 +1348,20 @@ static const struct of_device_id nb8800_dt_ids[] = {
 };
 MODULE_DEVICE_TABLE(of, nb8800_dt_ids);
 
+static void nb8800_init(struct net_device *dev)
+{
+	struct nb8800_priv *priv = netdev_priv(dev);
+	const struct nb8800_ops *ops = priv->ops;
+
+	if (ops && ops->reset)
+		ops->reset(dev);
+	nb8800_hw_init(dev);
+	if (ops && ops->init)
+		ops->init(dev);
+	nb8800_update_mac_addr(dev);
+	priv->speed = 0;
+}
+
 static int nb8800_probe(struct platform_device *pdev)
 {
 	const struct of_device_id *match;
@@ -1389,6 +1401,7 @@ static int nb8800_probe(struct platform_device *pdev)
 
 	priv = netdev_priv(dev);
 	priv->base = base;
+	priv->ops = ops;
 
 	priv->phy_mode = of_get_phy_mode(pdev->dev.of_node);
 	if (priv->phy_mode < 0)
@@ -1407,12 +1420,6 @@ static int nb8800_probe(struct platform_device *pdev)
 
 	spin_lock_init(&priv->tx_lock);
 
-	if (ops && ops->reset) {
-		ret = ops->reset(dev);
-		if (ret)
-			goto err_disable_clk;
-	}
-
 	bus = devm_mdiobus_alloc(&pdev->dev);
 	if (!bus) {
 		ret = -ENOMEM;
@@ -1454,21 +1461,16 @@ static int nb8800_probe(struct platform_device *pdev)
 
 	priv->mii_bus = bus;
 
-	ret = nb8800_hw_init(dev);
-	if (ret)
-		goto err_deregister_fixed_link;
-
-	if (ops && ops->init) {
-		ret = ops->init(dev);
-		if (ret)
-			goto err_deregister_fixed_link;
-	}
-
 	dev->netdev_ops = &nb8800_netdev_ops;
 	dev->ethtool_ops = &nb8800_ethtool_ops;
 	dev->flags |= IFF_MULTICAST;
 	dev->irq = irq;
 
+	/* Auto-negotiate by default */
+	priv->pause_aneg = true;
+	priv->pause_rx = true;
+	priv->pause_tx = true;
+
 	mac = of_get_mac_address(pdev->dev.of_node);
 	if (mac)
 		ether_addr_copy(dev->dev_addr, mac);
@@ -1476,14 +1478,12 @@ static int nb8800_probe(struct platform_device *pdev)
 	if (!is_valid_ether_addr(dev->dev_addr))
 		eth_hw_addr_random(dev);
 
-	nb8800_update_mac_addr(dev);
-
 	netif_carrier_off(dev);
 
 	ret = register_netdev(dev);
 	if (ret) {
 		netdev_err(dev, "failed to register netdev\n");
-		goto err_free_dma;
+		goto err_deregister_fixed_link;
 	}
 
 	netif_napi_add(dev, &priv->napi, nb8800_poll, NAPI_POLL_WEIGHT);
@@ -1492,8 +1492,6 @@ static int nb8800_probe(struct platform_device *pdev)
 
 	return 0;
 
-err_free_dma:
-	nb8800_dma_free(dev);
 err_deregister_fixed_link:
 	if (of_phy_is_fixed_link(pdev->dev.of_node))
 		of_phy_deregister_fixed_link(pdev->dev.of_node);
diff --git a/drivers/net/ethernet/aurora/nb8800.h b/drivers/net/ethernet/aurora/nb8800.h
index 6ec4a956e1e5..d5f4481a2c7b 100644
--- a/drivers/net/ethernet/aurora/nb8800.h
+++ b/drivers/net/ethernet/aurora/nb8800.h
@@ -305,6 +305,7 @@ struct nb8800_priv {
 	dma_addr_t			tx_desc_dma;
 
 	struct clk			*clk;
+	const struct nb8800_ops		*ops;
 };
 
 struct nb8800_ops {
-- 
2.11.0

Re: [RFC PATCH v2 1/2] net: ethernet: nb8800: Reset HW block in ndo_open

[Måns Rullgård]

Mason <slash.tmp@free.fr> writes:

> Move all HW initializations to nb8800_init.
> This provides the basis for suspend/resume support.
> ---
>  drivers/net/ethernet/aurora/nb8800.c | 50 +++++++++++++++++-------------------
>  drivers/net/ethernet/aurora/nb8800.h |  1 +
>  2 files changed, 25 insertions(+), 26 deletions(-)

You're still not doing anything to preserve flow control and multicast
configuration, and those are probably not the only ones.

Worse, you're now never tearing down dma properly, which means the
hardware can be left with active pointers to memory no longer allocated
to it.

Finally, you still haven't explained why the hw needs to be reset in
ndo_open().  Whatever is causing your lockup can almost certainly be
triggered in some other way too.  I will not accept this side-stepping
of the issue.

> diff --git a/drivers/net/ethernet/aurora/nb8800.c b/drivers/net/ethernet/aurora/nb8800.c
> index e94159507847..aa18ea25d91f 100644
> --- a/drivers/net/ethernet/aurora/nb8800.c
> +++ b/drivers/net/ethernet/aurora/nb8800.c
> @@ -39,6 +39,7 @@
>
>  #include "nb8800.h"
>
> +static void nb8800_init(struct net_device *dev);
>  static void nb8800_tx_done(struct net_device *dev);
>  static int nb8800_dma_stop(struct net_device *dev);
>
> @@ -957,6 +958,8 @@ static int nb8800_open(struct net_device *dev)
>  	struct phy_device *phydev;
>  	int err;
>
> +	nb8800_init(dev);
> +
>  	/* clear any pending interrupts */
>  	nb8800_writel(priv, NB8800_RXC_SR, 0xf);
>  	nb8800_writel(priv, NB8800_TXC_SR, 0xf);
> @@ -1246,11 +1249,6 @@ static int nb8800_hw_init(struct net_device *dev)
>  	nb8800_writeb(priv, NB8800_PQ1, val >> 8);
>  	nb8800_writeb(priv, NB8800_PQ2, val & 0xff);
>
> -	/* Auto-negotiate by default */
> -	priv->pause_aneg = true;
> -	priv->pause_rx = true;
> -	priv->pause_tx = true;
> -
>  	nb8800_mc_init(dev, 0);
>
>  	return 0;
> @@ -1350,6 +1348,20 @@ static const struct of_device_id nb8800_dt_ids[] = {
>  };
>  MODULE_DEVICE_TABLE(of, nb8800_dt_ids);
>
> +static void nb8800_init(struct net_device *dev)
> +{
> +	struct nb8800_priv *priv = netdev_priv(dev);
> +	const struct nb8800_ops *ops = priv->ops;
> +
> +	if (ops && ops->reset)
> +		ops->reset(dev);
> +	nb8800_hw_init(dev);
> +	if (ops && ops->init)
> +		ops->init(dev);
> +	nb8800_update_mac_addr(dev);
> +	priv->speed = 0;
> +}
> +
>  static int nb8800_probe(struct platform_device *pdev)
>  {
>  	const struct of_device_id *match;
> @@ -1389,6 +1401,7 @@ static int nb8800_probe(struct platform_device *pdev)
>
>  	priv = netdev_priv(dev);
>  	priv->base = base;
> +	priv->ops = ops;
>
>  	priv->phy_mode = of_get_phy_mode(pdev->dev.of_node);
>  	if (priv->phy_mode < 0)
> @@ -1407,12 +1420,6 @@ static int nb8800_probe(struct platform_device *pdev)
>
>  	spin_lock_init(&priv->tx_lock);
>
> -	if (ops && ops->reset) {
> -		ret = ops->reset(dev);
> -		if (ret)
> -			goto err_disable_clk;
> -	}
> -
>  	bus = devm_mdiobus_alloc(&pdev->dev);
>  	if (!bus) {
>  		ret = -ENOMEM;
> @@ -1454,21 +1461,16 @@ static int nb8800_probe(struct platform_device *pdev)
>
>  	priv->mii_bus = bus;
>
> -	ret = nb8800_hw_init(dev);
> -	if (ret)
> -		goto err_deregister_fixed_link;
> -
> -	if (ops && ops->init) {
> -		ret = ops->init(dev);
> -		if (ret)
> -			goto err_deregister_fixed_link;
> -	}
> -
>  	dev->netdev_ops = &nb8800_netdev_ops;
>  	dev->ethtool_ops = &nb8800_ethtool_ops;
>  	dev->flags |= IFF_MULTICAST;
>  	dev->irq = irq;
>
> +	/* Auto-negotiate by default */
> +	priv->pause_aneg = true;
> +	priv->pause_rx = true;
> +	priv->pause_tx = true;
> +
>  	mac = of_get_mac_address(pdev->dev.of_node);
>  	if (mac)
>  		ether_addr_copy(dev->dev_addr, mac);
> @@ -1476,14 +1478,12 @@ static int nb8800_probe(struct platform_device *pdev)
>  	if (!is_valid_ether_addr(dev->dev_addr))
>  		eth_hw_addr_random(dev);
>
> -	nb8800_update_mac_addr(dev);
> -
>  	netif_carrier_off(dev);
>
>  	ret = register_netdev(dev);
>  	if (ret) {
>  		netdev_err(dev, "failed to register netdev\n");
> -		goto err_free_dma;
> +		goto err_deregister_fixed_link;
>  	}
>
>  	netif_napi_add(dev, &priv->napi, nb8800_poll, NAPI_POLL_WEIGHT);
> @@ -1492,8 +1492,6 @@ static int nb8800_probe(struct platform_device *pdev)
>
>  	return 0;
>
> -err_free_dma:
> -	nb8800_dma_free(dev);
>  err_deregister_fixed_link:
>  	if (of_phy_is_fixed_link(pdev->dev.of_node))
>  		of_phy_deregister_fixed_link(pdev->dev.of_node);
> diff --git a/drivers/net/ethernet/aurora/nb8800.h b/drivers/net/ethernet/aurora/nb8800.h
> index 6ec4a956e1e5..d5f4481a2c7b 100644
> --- a/drivers/net/ethernet/aurora/nb8800.h
> +++ b/drivers/net/ethernet/aurora/nb8800.h
> @@ -305,6 +305,7 @@ struct nb8800_priv {
>  	dma_addr_t			tx_desc_dma;
>
>  	struct clk			*clk;
> +	const struct nb8800_ops		*ops;
>  };
>
>  struct nb8800_ops {
> -- 
> 2.11.0

-- 
Måns Rullgård

Re: [RFC PATCH v2 1/2] net: ethernet: nb8800: Reset HW block in ndo_open

[Mason]

On 02/08/2017 13:02, Måns Rullgård wrote:

> Mason wrote:
> 
>> Move all HW initializations to nb8800_init.
>> This provides the basis for suspend/resume support.
>> ---
>>  drivers/net/ethernet/aurora/nb8800.c | 50 +++++++++++++++++-------------------
>>  drivers/net/ethernet/aurora/nb8800.h |  1 +
>>  2 files changed, 25 insertions(+), 26 deletions(-)
> 
> You're still not doing anything to preserve flow control and multicast
> configuration, and those are probably not the only ones.

I did handle flow control (as far as I can tell).
The current settings are stored in:
priv->pause_aneg, priv->pause_rx, priv->pause_tx
and nb8800_pause_config() is called from nb8800_link_reconfigure()

I'll take a closer look at multicast settings.

> Worse, you're now never tearing down dma properly, which means the
> hardware can be left with active pointers to memory no longer allocated
> to it.

You're making it sound like there is a risk those pointers
might be used somehow. There is no such risk AFAICT.
The PHY is disconnected, NAPI is stopped, RX and TX have
been disabled, and the ISR has been removed.

I have considered putting the HW block in reset (clock gated)
at the end of nb8800_stop() to save power, which would make
the issue you point out moot.

The reason I removed nb8800_dma_stop() in nb8800_stop()
is because it hangs when called from nb8800_suspend().
(It requires interrupts to make progress, and interrupts
seem to be disabled when nb8800_suspend() is called.)


Broader question for netdev:

I've been wondering about costly close operations.
If closing a device is complex (in terms of code), at which
point does it become simpler to just reset the block, and
avoid writing/maintaining/debugging the code performing
said close operation?

> Finally, you still haven't explained why the hw needs to be reset in
> ndo_open().  Whatever is causing your lockup can almost certainly be
> triggered in some other way too.  I will not accept this side-stepping
> of the issue.

(I was not aware that you were the final authority on which
patches are accepted upstream.)

FWIW, I have placed a formal request for the HW dev to
investigate, as I could not reproduce on tango4, despite
several days wasted on this issue.

In the mean time, the driver in its current form does not
support suspend/resume. How would *you* implement it?
(IMO, my way is great in its simplicity and code reuse.)

Regards.

Re: [RFC PATCH v2 1/2] net: ethernet: nb8800: Reset HW block in ndo_open

[Andrew Lunn]

> > Finally, you still haven't explained why the hw needs to be reset in
> > ndo_open().  Whatever is causing your lockup can almost certainly be
> > triggered in some other way too.  I will not accept this side-stepping
> > of the issue.
> 
> (I was not aware that you were the final authority on which
> patches are accepted upstream.)

./scripts/get_maintainer.pl -f drivers/net/ethernet/aurora/nb8800.c
"David S. Miller" <davem@davemloft.net> (commit_signer:6/6=100%)
Mans Rullgard <mans@mansr.com> (commit_signer:2/6=33%)
Jarod Wilson <jarod@redhat.com> (commit_signer:1/6=17%,authored:1/6=17%,removed_lines:1/17=6%)
Stephen Hemminger <stephen@networkplumber.org> (commit_signer:1/6=17%)
Florian Fainelli <f.fainelli@gmail.com> (commit_signer:1/6=17%,added_lines:1/14=7%,removed_lines:11/17=65%)
Javier Martinez Canillas <javier@osg.samsung.com> (authored:1/6=17%,added_lines:1/14=7%)
Johan Hovold <johan@kernel.org> (authored:1/6=17%,added_lines:7/14=50%,removed_lines:2/17=12%)
Johannes Berg <johannes.berg@intel.com> (authored:1/6=17%,added_lines:2/14=14%,removed_lines:2/17=12%)
Wei Yongjun <weiyongjun1@huawei.com> (authored:1/6=17%,added_lines:3/14=21%,removed_lines:1/17=6%)

David is very likely to listen to Mans's recommendations, since Mans
has done most of the review work for this driver.

      Andrew

Re: [RFC PATCH v2 1/2] net: ethernet: nb8800: Reset HW block in ndo_open

[Måns Rullgård]

Mason <slash.tmp@free.fr> writes:

> On 02/08/2017 13:02, Måns Rullgård wrote:
>
>> Mason wrote:
>> 
>>> Move all HW initializations to nb8800_init.
>>> This provides the basis for suspend/resume support.
>>> ---
>>>  drivers/net/ethernet/aurora/nb8800.c | 50 +++++++++++++++++-------------------
>>>  drivers/net/ethernet/aurora/nb8800.h |  1 +
>>>  2 files changed, 25 insertions(+), 26 deletions(-)
>> 
>> You're still not doing anything to preserve flow control and multicast
>> configuration, and those are probably not the only ones.
>
> I did handle flow control (as far as I can tell).
> The current settings are stored in:
> priv->pause_aneg, priv->pause_rx, priv->pause_tx
> and nb8800_pause_config() is called from nb8800_link_reconfigure()

I think the whole pause setup needs to be revisited.  It's a mess.
Maybe I'll find the time one day.

> I'll take a closer look at multicast settings.

You're currently losing all multicast setup as well as promiscuous mode
if that has been enabled.

>> Worse, you're now never tearing down dma properly, which means the
>> hardware can be left with active pointers to memory no longer allocated
>> to it.
>
> You're making it sound like there is a risk those pointers
> might be used somehow. There is no such risk AFAICT.
> The PHY is disconnected, NAPI is stopped, RX and TX have
> been disabled, and the ISR has been removed.

The interrupt handler and NAPI have nothing to do with it since they
only get involved *after* the hw has filled the buffers.  If you've
given the hardware control of a memory buffer, you should damn well take
it back before reusing that memory for something else.  Otherwise you'll
eventually have a very difficult to debug problem and a security risk.

> I have considered putting the HW block in reset (clock gated)
> at the end of nb8800_stop() to save power, which would make
> the issue you point out moot.

That's not necessarily possible.  It is on Sigma SoCs, but it doesn't
have to be.

> The reason I removed nb8800_dma_stop() in nb8800_stop()
> is because it hangs when called from nb8800_suspend().
> (It requires interrupts to make progress, and interrupts
> seem to be disabled when nb8800_suspend() is called.)

It shouldn't require interrupts.  If it does for some reason, that
should be fixed.

> Broader question for netdev:
>
> I've been wondering about costly close operations.
> If closing a device is complex (in terms of code), at which
> point does it become simpler to just reset the block, and
> avoid writing/maintaining/debugging the code performing
> said close operation?
>
>> Finally, you still haven't explained why the hw needs to be reset in
>> ndo_open().  Whatever is causing your lockup can almost certainly be
>> triggered in some other way too.  I will not accept this side-stepping
>> of the issue.
>
> (I was not aware that you were the final authority on which
> patches are accepted upstream.)
>
> FWIW, I have placed a formal request for the HW dev to
> investigate, as I could not reproduce on tango4, despite
> several days wasted on this issue.
>
> In the mean time, the driver in its current form does not
> support suspend/resume. How would *you* implement it?

I'm not saying it's a bad idea for suspend.  It might even be the only
way.

-- 
Måns Rullgård

[RFC PATCH v2 2/2] net: ethernet: nb8800: Add suspend/resume support

[Mason]

Wrappers around nb8800_stop and nb8800_open.
---
 drivers/net/ethernet/aurora/nb8800.c | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/drivers/net/ethernet/aurora/nb8800.c b/drivers/net/ethernet/aurora/nb8800.c
index aa18ea25d91f..607064a6d7a1 100644
--- a/drivers/net/ethernet/aurora/nb8800.c
+++ b/drivers/net/ethernet/aurora/nb8800.c
@@ -1012,7 +1012,6 @@ static int nb8800_stop(struct net_device *dev)
 	netif_stop_queue(dev);
 	napi_disable(&priv->napi);
 
-	nb8800_dma_stop(dev);
 	nb8800_mac_rx(dev, false);
 	nb8800_mac_tx(dev, false);
 
@@ -1526,6 +1525,26 @@ static int nb8800_remove(struct platform_device *pdev)
 	return 0;
 }
 
+static int nb8800_suspend(struct platform_device *pdev, pm_message_t state)
+{
+	struct net_device *dev = platform_get_drvdata(pdev);
+
+	if (netif_running(dev))
+		nb8800_stop(dev);
+
+	return 0;
+}
+
+static int nb8800_resume(struct platform_device *pdev)
+{
+	struct net_device *dev = platform_get_drvdata(pdev);
+
+	if (netif_running(dev))
+		nb8800_open(dev);
+
+	return 0;
+}
+
 static struct platform_driver nb8800_driver = {
 	.driver = {
 		.name		= "nb8800",
@@ -1533,6 +1552,8 @@ static struct platform_driver nb8800_driver = {
 	},
 	.probe	= nb8800_probe,
 	.remove	= nb8800_remove,
+	.suspend = nb8800_suspend,
+	.resume = nb8800_resume,
 };
 
 module_platform_driver(nb8800_driver);
-- 
2.11.0

Re: [RFC PATCH v2 0/2] nb8800 suspend/resume support

[Mason]

On 01/08/2017 18:32, Mason wrote:

> I need suspend/resume support in the nb8800 driver.
> On tango platforms, suspend loses all context (MMIO registers).
> To make the task easy, we just close the device on suspend,
> and open it again on resume. This requires properly resetting
> the HW on resume.
> 
> Patch 1 moves all the HW init to nb8800_init()
> Patch 2 adds suspend/resume support

I have now confirmed that the "flow control" issue I reported
in another thread has nothing to do with flow control per se.

The problem is that nb8800_pause_config() calls nb8800_dma_stop()
and when it does, RX is borked.

On a GigE switch:
[   21.444268] ENTER nb8800_pause_config
[   21.448604] rxcr=06100a8f pause_rx=1 pause_tx=0 pause=1 asym_pause=1
[   21.455020] nb8800 26000.ethernet eth0: Link is Up - 1Gbps/Full - flow control rx/tx

In this case, pause_tx and RCR_FL match, so we skip the
silly dance.

On a FastE switch:
[   11.611613] ENTER nb8800_pause_config
[   11.615942] rxcr=06100a8f pause_rx=1 pause_tx=1 pause=1 asym_pause=0
[   11.825535] nb8800 26000.ethernet eth0: Link is Up - 100Mbps/Full - flow control rx/tx

In this case pause_tx and RCR_FL don't match, so we hit
nb8800_dma_stop() and the HW block becomes deaf.

In conclusion, two issues I've been discussing:
A) RX wedged after ndo_close
B) flow control breaks on FastE switches
are in fact two symptoms of the same root issue.

Regards.

Re: [RFC PATCH v2 0/2] nb8800 suspend/resume support

[Mason]

On 02/08/2017 16:41, Mason wrote:

> On 01/08/2017 18:32, Mason wrote:
> 
>> I need suspend/resume support in the nb8800 driver.
>> On tango platforms, suspend loses all context (MMIO registers).
>> To make the task easy, we just close the device on suspend,
>> and open it again on resume. This requires properly resetting
>> the HW on resume.
>>
>> Patch 1 moves all the HW init to nb8800_init()
>> Patch 2 adds suspend/resume support
> 
> I have now confirmed that the "flow control" issue I reported
> in another thread has nothing to do with flow control per se.
> 
> The problem is that nb8800_pause_config() calls nb8800_dma_stop()
> and when it does, RX is borked.
> 
> On a GigE switch:
> [   21.444268] ENTER nb8800_pause_config
> [   21.448604] rxcr=06100a8f pause_rx=1 pause_tx=0 pause=1 asym_pause=1
> [   21.455020] nb8800 26000.ethernet eth0: Link is Up - 1Gbps/Full - flow control rx/tx
> 
> In this case, pause_tx and RCR_FL match, so we skip the
> silly dance.

The documentation states:

Receive Channel Control Register
Description:
The Receive Channel Control Register holds channel enable, mode, endian,
DMA control, and interrupt control information.  This register can only
be written when the Receive DMA Channel is idle - the Enable bit in it is "0".
Register Number:  0x200
Access:  read/write
Reset Value: le = AMBA_LE; all other bits = 0x0
Fields:

fl:  Flow control enable.  "1" indicates flow control is enabled. 
When flow control is enabled and a Receive FIFO overrun occurs,
the Ethernet 10/100/1000 Subsystem will send PAUSE frames if in
full duplex mode.  This continues until the Receive FIFO is emptied.

en:  Receive DMA Channel enable.  "1" indicates that the Receive
DMA Channel is being configured from a descriptor, or that the DMA
operation is in progress.  The Receive DMA Channel is idle when
this enable bit is "0".  Software sets this bit to "1" to start the
configuration from a descriptor and DMA operation.  When the DMA
operation is finished, this bit is automatically reset to "0".


> Receive DMA Channel Disabling
>
> When the entire receive frame has been read from the Receive FIFO and
> sent over the AMBA bus, the DMA operation ends, and the Receive DMA
> Channel is automatically disabled.  To do this, hardware resets the
> Enable bit in the Receive Channel Control Register to "0" after the
> last data has been read from the Receive FIFO and sent over the AMBA
> bus.
> 
> When operating in descriptor mode, upon completion of a receive frame
> DMA operation, if the descriptor chain has not ended when a receive
> frame DMA operation completes, the next receive frame DMA operation
> begins.  The last descriptor in a descriptor chain is indicated by
> having its End Of Chain- EOC, flag set to "1".  If this EOC flag is
> "0", to begin the next receive frame DMA operation, the next
> descriptor is automatically retrieved and used to configure the
> Receive DMA Channel.  The Receive DMA Channel is then automatically
> re-enabled and the next receive frame DMA operation begins.
> 
> In descriptor mode, an AMBA bus error can occur when reading receive
> descriptor data.  If this happens, receive descriptor processing ends
> and the Receive DMA Channel is turned off.  The Descriptor Error bit
> in the Receive Status Register is set to "1".

Hmmm, I guess this is what Maxime/Mans did...


Looking at the tango-specific integration, I note this nugget:

1.5.4 Stopping & Starting the DMA

This feature has been added to allow the software to stop and start
the DMA without any issues.

Procedure:
1- STOP:
2- Stop RX core;
3- Set OWN bit of all descriptor of the chain to 1;
4- Stop DMA by writing dma_stop bit to 1 in  RX_DMA_Stop register
5- Wait around 100 clock cycles.

The pending packets are held until the system will re-start.

RE-START:
1- Clear dma_stop bit (note that if at the time of stopping the DMA,
the next packet in the FIFO was an UDP packet, when clearing dma_stop,
this packet will directly start being written in the DRAM since UDP
packets are not controlled by the descriptor mechanism);
2- Program a new chain of descriptor;
3- Re-enable DMA (rx_ctrl register)

rx_dma_stop:
Software control to stop the Rx DMA.
A write to this bit with “1” will gracefully stop the Rx DMA by after
transferring the current packet. If more packets are pending they will
be held until the software clears this bit.


Hmmm, what do you think? This looks promising...

Regards.

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH v2 0/2] nb8800 suspend/resume support

[Måns Rullgård]

Mason <slash.tmp@free.fr> writes:

> On 02/08/2017 16:41, Mason wrote:
>
>> On 01/08/2017 18:32, Mason wrote:
>> 
>>> I need suspend/resume support in the nb8800 driver.
>>> On tango platforms, suspend loses all context (MMIO registers).
>>> To make the task easy, we just close the device on suspend,
>>> and open it again on resume. This requires properly resetting
>>> the HW on resume.
>>>
>>> Patch 1 moves all the HW init to nb8800_init()
>>> Patch 2 adds suspend/resume support
>> 
>> I have now confirmed that the "flow control" issue I reported
>> in another thread has nothing to do with flow control per se.
>> 
>> The problem is that nb8800_pause_config() calls nb8800_dma_stop()
>> and when it does, RX is borked.
>> 
>> On a GigE switch:
>> [   21.444268] ENTER nb8800_pause_config
>> [   21.448604] rxcr=06100a8f pause_rx=1 pause_tx=0 pause=1 asym_pause=1
>> [   21.455020] nb8800 26000.ethernet eth0: Link is Up - 1Gbps/Full - flow control rx/tx
>> 
>> In this case, pause_tx and RCR_FL match, so we skip the
>> silly dance.
>
> The documentation states:
>
> Receive Channel Control Register
> Description:
> The Receive Channel Control Register holds channel enable, mode, endian,
> DMA control, and interrupt control information.  This register can only
> be written when the Receive DMA Channel is idle - the Enable bit in it is "0".
> Register Number:  0x200
> Access:  read/write
> Reset Value: le = AMBA_LE; all other bits = 0x0
> Fields:
>
> fl:  Flow control enable.  "1" indicates flow control is enabled. 
> When flow control is enabled and a Receive FIFO overrun occurs,
> the Ethernet 10/100/1000 Subsystem will send PAUSE frames if in
> full duplex mode.  This continues until the Receive FIFO is emptied.
>
> en:  Receive DMA Channel enable.  "1" indicates that the Receive
> DMA Channel is being configured from a descriptor, or that the DMA
> operation is in progress.  The Receive DMA Channel is idle when
> this enable bit is "0".  Software sets this bit to "1" to start the
> configuration from a descriptor and DMA operation.  When the DMA
> operation is finished, this bit is automatically reset to "0".

Here's the problem.  Once started, there is no way to forcibly stop rx
dma.  It keeps going until it hits a descriptor with the end of chain
flag set.

>> Receive DMA Channel Disabling
>>
>> When the entire receive frame has been read from the Receive FIFO and
>> sent over the AMBA bus, the DMA operation ends, and the Receive DMA
>> Channel is automatically disabled.  To do this, hardware resets the
>> Enable bit in the Receive Channel Control Register to "0" after the
>> last data has been read from the Receive FIFO and sent over the AMBA
>> bus.
>> 
>> When operating in descriptor mode, upon completion of a receive frame
>> DMA operation, if the descriptor chain has not ended when a receive
>> frame DMA operation completes, the next receive frame DMA operation
>> begins.  The last descriptor in a descriptor chain is indicated by
>> having its End Of Chain- EOC, flag set to "1".  If this EOC flag is
>> "0", to begin the next receive frame DMA operation, the next
>> descriptor is automatically retrieved and used to configure the
>> Receive DMA Channel.  The Receive DMA Channel is then automatically
>> re-enabled and the next receive frame DMA operation begins.
>> 
>> In descriptor mode, an AMBA bus error can occur when reading receive
>> descriptor data.  If this happens, receive descriptor processing ends
>> and the Receive DMA Channel is turned off.  The Descriptor Error bit
>> in the Receive Status Register is set to "1".
>
> Hmmm, I guess this is what Maxime/Mans did...
>
> Looking at the tango-specific integration, I note this nugget:
>
> 1.5.4 Stopping & Starting the DMA
>
> This feature has been added to allow the software to stop and start
> the DMA without any issues.
>
> Procedure:
> 1- STOP:
> 2- Stop RX core;
> 3- Set OWN bit of all descriptor of the chain to 1;
> 4- Stop DMA by writing dma_stop bit to 1 in  RX_DMA_Stop register
> 5- Wait around 100 clock cycles.
>
> The pending packets are held until the system will re-start.
>
> RE-START:
> 1- Clear dma_stop bit (note that if at the time of stopping the DMA,
> the next packet in the FIFO was an UDP packet, when clearing dma_stop,
> this packet will directly start being written in the DRAM since UDP
> packets are not controlled by the descriptor mechanism);
> 2- Program a new chain of descriptor;
> 3- Re-enable DMA (rx_ctrl register)
>
> rx_dma_stop:
> Software control to stop the Rx DMA.
> A write to this bit with “1” will gracefully stop the Rx DMA by after
> transferring the current packet. If more packets are pending they will
> be held until the software clears this bit.
>
> Hmmm, what do you think? This looks promising...

This is only available in the more recent Sigma versions.  Although it
is nicer, I didn't think it was worth the trouble to support both
methods since the older method should work on all chips.

-- 
Måns Rullgård

Re: [RFC PATCH v2 0/2] nb8800 suspend/resume support

[Mason]

On 02/08/2017 17:36, Måns Rullgård wrote:

> Mason wrote:
> 
>> Looking at the tango-specific integration, I note this nugget:
>>
>> 1.5.4 Stopping & Starting the DMA
>>
>> This feature has been added to allow the software to stop and start
>> the DMA without any issues.
>>
>> Procedure:
>> 1- STOP:
>> 2- Stop RX core;
>> 3- Set OWN bit of all descriptor of the chain to 1;
>> 4- Stop DMA by writing dma_stop bit to 1 in  RX_DMA_Stop register
>> 5- Wait around 100 clock cycles.
>>
>> The pending packets are held until the system will re-start.
>>
>> RE-START:
>> 1- Clear dma_stop bit (note that if at the time of stopping the DMA,
>> the next packet in the FIFO was an UDP packet, when clearing dma_stop,
>> this packet will directly start being written in the DRAM since UDP
>> packets are not controlled by the descriptor mechanism);
>> 2- Program a new chain of descriptor;
>> 3- Re-enable DMA (rx_ctrl register)
>>
>> rx_dma_stop:
>> Software control to stop the Rx DMA.
>> A write to this bit with "1" will gracefully stop the Rx DMA by after
>> transferring the current packet. If more packets are pending they will
>> be held until the software clears this bit.
>>
>> Hmmm, what do you think? This looks promising...
> 
> This is only available in the more recent Sigma versions.  Although it
> is nicer, I didn't think it was worth the trouble to support both
> methods since the older method should work on all chips.

Well, from my perspective, the older method does not
work on newer chips :-)

The "new" method is available on the SMP8670 onward
(released Jan 2011). I might try implementing it for
"sigma,smp8734-ethernet" boards.

Regards.

Re: [RFC PATCH v2 0/2] nb8800 suspend/resume support

[Måns Rullgård]

Mason <slash.tmp@free.fr> writes:

> On 02/08/2017 17:36, Måns Rullgård wrote:
>
>> Mason wrote:
>> 
>>> Looking at the tango-specific integration, I note this nugget:
>>>
>>> 1.5.4 Stopping & Starting the DMA
>>>
>>> This feature has been added to allow the software to stop and start
>>> the DMA without any issues.
>>>
>>> Procedure:
>>> 1- STOP:
>>> 2- Stop RX core;
>>> 3- Set OWN bit of all descriptor of the chain to 1;
>>> 4- Stop DMA by writing dma_stop bit to 1 in  RX_DMA_Stop register
>>> 5- Wait around 100 clock cycles.
>>>
>>> The pending packets are held until the system will re-start.
>>>
>>> RE-START:
>>> 1- Clear dma_stop bit (note that if at the time of stopping the DMA,
>>> the next packet in the FIFO was an UDP packet, when clearing dma_stop,
>>> this packet will directly start being written in the DRAM since UDP
>>> packets are not controlled by the descriptor mechanism);
>>> 2- Program a new chain of descriptor;
>>> 3- Re-enable DMA (rx_ctrl register)
>>>
>>> rx_dma_stop:
>>> Software control to stop the Rx DMA.
>>> A write to this bit with "1" will gracefully stop the Rx DMA by after
>>> transferring the current packet. If more packets are pending they will
>>> be held until the software clears this bit.
>>>
>>> Hmmm, what do you think? This looks promising...
>> 
>> This is only available in the more recent Sigma versions.  Although it
>> is nicer, I didn't think it was worth the trouble to support both
>> methods since the older method should work on all chips.
>
> Well, from my perspective, the older method does not work on newer
> chips :-)

It does work on tango4.

What does the tango5 do if you flood it with packets faster than the
kernel can keep up with?  That would make it hit the end of the rx
chain, which is apparently what makes it miserable with the current dma
stop code.

-- 
Måns Rullgård

Re: [RFC PATCH v2 0/2] nb8800 suspend/resume support

[Mason]

On 02/08/2017 17:56, Måns Rullgård wrote:

> Mason writes:
> 
>> From my perspective, the older method does not work on newer chips :-)
> 
> It does work on tango4.

Agreed.

> What does the tango5 do if you flood it with packets faster than the
> kernel can keep up with?  That would make it hit the end of the rx
> chain, which is apparently what makes it miserable with the current dma
> stop code.

The simplest way to test this would be sending tiny packets
as fast as possible, right? So ping -f on a GigE link should
fit the bill?

Regards.

Re: [RFC PATCH v2 0/2] nb8800 suspend/resume support

[Måns Rullgård]

Mason <slash.tmp@free.fr> writes:

> On 02/08/2017 17:56, Måns Rullgård wrote:
>
>> Mason writes:
>> 
>>> From my perspective, the older method does not work on newer chips :-)
>> 
>> It does work on tango4.
>
> Agreed.
>
>> What does the tango5 do if you flood it with packets faster than the
>> kernel can keep up with?  That would make it hit the end of the rx
>> chain, which is apparently what makes it miserable with the current dma
>> stop code.
>
> The simplest way to test this would be sending tiny packets
> as fast as possible, right? So ping -f on a GigE link should
> fit the bill?

ping -f is limited to 100 packets per second.  Use something like iperf
in UDP mode instead.

-- 
Måns Rullgård

RE: [RFC PATCH v2 0/2] nb8800 suspend/resume support

[David Laight]

From: Måns Rullgård
> Sent: 02 August 2017 17:10
...
> ping -f is limited to 100 packets per second.  Use something like iperf
> in UDP mode instead.

Or break a MAC driver so it just saturates the network.

You might actually need bursts of traffic - otherwise the receiver
will be continuously out of rx buffers and no dma will be active.

	David

Re: [RFC PATCH v2 0/2] nb8800 suspend/resume support

[Mason]

On 02/08/2017 18:10, Måns Rullgård wrote:

> Mason writes:
> 
>> On 02/08/2017 17:56, Måns Rullgård wrote:
>>
>>> What does the tango5 do if you flood it with packets faster than the
>>> kernel can keep up with?  That would make it hit the end of the rx
>>> chain, which is apparently what makes it miserable with the current dma
>>> stop code.
>>
>> The simplest way to test this would be sending tiny packets
>> as fast as possible, right? So ping -f on a GigE link should
>> fit the bill?
> 
> ping -f is limited to 100 packets per second.  Use something like iperf
> in UDP mode instead.

ping -f can go 100 times faster than 100 pps:

# ping -f -q -c 150000 -s 300 172.27.64.45
PING 172.27.64.45 (172.27.64.45) 300(328) bytes of data.

--- 172.27.64.45 ping statistics ---
150000 packets transmitted, 150000 received, 0% packet loss, time 15035ms
rtt min/avg/max/mdev = 0.065/0.084/0.537/0.014 ms, ipg/ewma 0.100/0.087 ms


150,000 packets in 15 seconds = 10,000 pps

(172.27.64.45 is the tango5 board)

Ergo, dealing with 10,000 packets per second does not hose RX.

Regards.

Re: [RFC PATCH v2 0/2] nb8800 suspend/resume support

[Måns Rullgård]

Mason <slash.tmp@free.fr> writes:

> On 02/08/2017 18:10, Måns Rullgård wrote:
>
>> Mason writes:
>> 
>>> On 02/08/2017 17:56, Måns Rullgård wrote:
>>>
>>>> What does the tango5 do if you flood it with packets faster than the
>>>> kernel can keep up with?  That would make it hit the end of the rx
>>>> chain, which is apparently what makes it miserable with the current dma
>>>> stop code.
>>>
>>> The simplest way to test this would be sending tiny packets
>>> as fast as possible, right? So ping -f on a GigE link should
>>> fit the bill?
>> 
>> ping -f is limited to 100 packets per second.  Use something like iperf
>> in UDP mode instead.
>
> ping -f can go 100 times faster than 100 pps:
>
> # ping -f -q -c 150000 -s 300 172.27.64.45
> PING 172.27.64.45 (172.27.64.45) 300(328) bytes of data.
>
> --- 172.27.64.45 ping statistics ---
> 150000 packets transmitted, 150000 received, 0% packet loss, time 15035ms
> rtt min/avg/max/mdev = 0.065/0.084/0.537/0.014 ms, ipg/ewma 0.100/0.087 ms
>
> 150,000 packets in 15 seconds = 10,000 pps
>
> (172.27.64.45 is the tango5 board)
>
> Ergo, dealing with 10,000 packets per second does not hose RX.

ping -f goes as fast as the other end replies or 100 per second,
whichever is higher, so says the man page.

-- 
Måns Rullgård

Re: [RFC PATCH v2 0/2] nb8800 suspend/resume support

[Mason]

On 02/08/2017 18:10, Måns Rullgård wrote:

> ping -f is limited to 100 packets per second.
> Use something like iperf in UDP mode instead.

CLIENT: DESKTOP
# iperf3 -c 172.27.64.45 -u -b 950M
Connecting to host 172.27.64.45, port 5201
[  4] local 172.27.64.1 port 55533 connected to 172.27.64.45 port 5201
[ ID] Interval           Transfer     Bandwidth       Total Datagrams
[  4]   0.00-1.00   sec   102 MBytes   858 Mbits/sec  13091  
[  4]   1.00-2.00   sec   114 MBytes   953 Mbits/sec  14541  
[  4]   2.00-3.00   sec   114 MBytes   953 Mbits/sec  14542  
[  4]   3.00-4.00   sec   114 MBytes   953 Mbits/sec  14541  
[  4]   4.00-5.00   sec   114 MBytes   953 Mbits/sec  14542  
[  4]   5.00-6.00   sec   114 MBytes   953 Mbits/sec  14541  
[  4]   6.00-7.00   sec   114 MBytes   953 Mbits/sec  14535  
[  4]   7.00-8.00   sec   114 MBytes   953 Mbits/sec  14536  
[  4]   8.00-9.00   sec   114 MBytes   953 Mbits/sec  14543  
[  4]   9.00-10.00  sec   114 MBytes   953 Mbits/sec  14541  
- - - - - - - - - - - - - - - - - - - - - - - - -
[ ID] Interval           Transfer     Bandwidth       Jitter    Lost/Total Datagrams
[  4]   0.00-10.00  sec  1.10 GBytes   943 Mbits/sec  0.247 ms  132176/143788 (92%)  
[  4] Sent 143788 datagrams

iperf Done.


SERVER: TANGO BOARD
# iperf3 -s
Accepted connection from 172.27.64.1, port 44995
[  5] local 172.27.64.45 port 5201 connected to 172.27.64.1 port 55533
[ ID] Interval           Transfer     Bandwidth       Jitter    Lost/Total Datagrams
[  5]   0.00-1.00   sec  7.90 MBytes  66.2 Mbits/sec  0.218 ms  11365/12376 (92%)  
[  5]   1.00-2.00   sec  9.13 MBytes  76.6 Mbits/sec  0.230 ms  13381/14550 (92%)  
[  5]   2.00-3.00   sec  9.12 MBytes  76.5 Mbits/sec  0.290 ms  13372/14540 (92%)  
[  5]   3.00-4.00   sec  9.11 MBytes  76.4 Mbits/sec  0.292 ms  13369/14535 (92%)  
[  5]   4.00-5.00   sec  9.18 MBytes  77.0 Mbits/sec  0.178 ms  13374/14549 (92%)  
[  5]   5.00-6.00   sec  9.10 MBytes  76.3 Mbits/sec  0.228 ms  13367/14532 (92%)  
[  5]   6.00-7.00   sec  9.26 MBytes  77.7 Mbits/sec  0.607 ms  13356/14541 (92%)  
[  5]   7.00-8.00   sec  9.23 MBytes  77.4 Mbits/sec  0.507 ms  13364/14545 (92%)  
[  5]   8.00-9.00   sec  9.20 MBytes  77.1 Mbits/sec  0.215 ms  13351/14528 (92%)  
[  5]   9.00-10.00  sec  9.16 MBytes  76.9 Mbits/sec  0.188 ms  13356/14529 (92%)  
[  5]  10.00-10.04  sec   336 KBytes  72.2 Mbits/sec  0.247 ms  521/563 (93%)  
- - - - - - - - - - - - - - - - - - - - - - - - -
[ ID] Interval           Transfer     Bandwidth       Jitter    Lost/Total Datagrams
[  5]   0.00-10.04  sec  0.00 Bytes  0.00 bits/sec  0.247 ms  132176/143788 (92%)  


There is some packet loss, but the board doesn't lose connectivity.

With smaller packets...

# iperf3 -c 172.27.64.45 -u -b 950M -l 800
Connecting to host 172.27.64.45, port 5201
[  4] local 172.27.64.1 port 35197 connected to 172.27.64.45 port 5201
[ ID] Interval           Transfer     Bandwidth       Total Datagrams
[  4]   0.00-1.00   sec  90.6 MBytes   760 Mbits/sec  118724  
[  4]   1.00-2.00   sec   107 MBytes   894 Mbits/sec  139718  
[  4]   2.00-3.00   sec   106 MBytes   889 Mbits/sec  138918  
[  4]   3.00-4.00   sec   107 MBytes   895 Mbits/sec  139768  
[  4]   4.00-5.00   sec   106 MBytes   891 Mbits/sec  139275  
[  4]   5.00-6.00   sec   107 MBytes   895 Mbits/sec  139862  
[  4]   6.00-7.00   sec   107 MBytes   895 Mbits/sec  139825  
[  4]   7.00-8.00   sec   107 MBytes   895 Mbits/sec  139775  
[  4]   8.00-9.00   sec   107 MBytes   895 Mbits/sec  139849  
[  4]   9.00-10.00  sec   107 MBytes   895 Mbits/sec  139835  
- - - - - - - - - - - - - - - - - - - - - - - - -
[ ID] Interval           Transfer     Bandwidth       Jitter    Lost/Total Datagrams
[  4]   0.00-10.00  sec  1.02 GBytes   880 Mbits/sec  0.009 ms  564117/1375520 (41%)  
[  4] Sent 1375520 datagrams

iperf Done.


Accepted connection from 172.27.64.1, port 46151
[  5] local 172.27.64.45 port 5201 connected to 172.27.64.1 port 35197
[ ID] Interval           Transfer     Bandwidth       Jitter    Lost/Total Datagrams
[  5]   0.00-1.00   sec  60.8 MBytes   510 Mbits/sec  0.004 ms  33508/113252 (30%)  
iperf3: OUT OF ORDER - incoming packet = 147146 and received packet = 0 AND SP = 147497
iperf3: OUT OF ORDER - incoming packet = 146128 and received packet = 0 AND SP = 147690
iperf3: OUT OF ORDER - incoming packet = 146067 and received packet = 0 AND SP = 147863
iperf3: OUT OF ORDER - incoming packet = 147242 and received packet = 0 AND SP = 148039
iperf3: OUT OF ORDER - incoming packet = 163837 and received packet = 0 AND SP = 164363
[  5]   1.00-2.00   sec  61.0 MBytes   511 Mbits/sec  0.198 ms  59635/139649 (43%)  
iperf3: OUT OF ORDER - incoming packet = 286437 and received packet = 0 AND SP = 287226
iperf3: OUT OF ORDER - incoming packet = 302990 and received packet = 0 AND SP = 305710
[  5]   2.00-3.00   sec  61.5 MBytes   517 Mbits/sec  0.005 ms  58369/138944 (42%)  
iperf3: OUT OF ORDER - incoming packet = 427697 and received packet = 0 AND SP = 428089
iperf3: OUT OF ORDER - incoming packet = 441087 and received packet = 0 AND SP = 442963
iperf3: OUT OF ORDER - incoming packet = 453844 and received packet = 0 AND SP = 454238
[  5]   3.00-4.00   sec  61.7 MBytes   517 Mbits/sec  0.008 ms  58975/139795 (42%)  
[  5]   4.00-5.00   sec  61.3 MBytes   514 Mbits/sec  0.010 ms  58453/138795 (42%)  
[  5]   5.00-6.00   sec  61.2 MBytes   513 Mbits/sec  0.014 ms  60141/140338 (43%)  
[  5]   6.00-7.00   sec  61.3 MBytes   514 Mbits/sec  0.009 ms  59055/139384 (42%)  
iperf3: OUT OF ORDER - incoming packet = 993646 and received packet = 0 AND SP = 994370
[  5]   7.00-8.00   sec  61.0 MBytes   512 Mbits/sec  0.007 ms  60265/140210 (43%)  
iperf3: OUT OF ORDER - incoming packet = 1134516 and received packet = 0 AND SP = 1135942
[  5]   8.00-9.00   sec  62.6 MBytes   523 Mbits/sec  0.198 ms  57805/139795 (41%)  
iperf3: OUT OF ORDER - incoming packet = 1275612 and received packet = 0 AND SP = 1277716
[  5]   9.00-10.00  sec  64.0 MBytes   538 Mbits/sec  0.007 ms  56029/139888 (40%)  
[  5]  10.00-10.04  sec  2.74 MBytes   589 Mbits/sec  0.009 ms  1882/5470 (34%)  
- - - - - - - - - - - - - - - - - - - - - - - - -
[ ID] Interval           Transfer     Bandwidth       Jitter    Lost/Total Datagrams
[  5]   0.00-10.04  sec  0.00 Bytes  0.00 bits/sec  0.009 ms  564117/1375520 (41%)  
[SUM]  0.0-10.0 sec  13 datagrams received out-of-order


A lot more packet loss, but RX holds on.

Regards.

Re: [RFC PATCH v2 0/2] nb8800 suspend/resume support

[Mason]

On 02/08/2017 19:31, Mason wrote:

> # iperf3 -c 172.27.64.45 -u -b 950M
> Connecting to host 172.27.64.45, port 5201
> [  4] local 172.27.64.1 port 55533 connected to 172.27.64.45 port 5201
> [ ID] Interval           Transfer     Bandwidth       Total Datagrams
> [  4]   0.00-1.00   sec   102 MBytes   858 Mbits/sec  13091  
> [  4]   1.00-2.00   sec   114 MBytes   953 Mbits/sec  14541  

114 MB in 14541 packets => 7840 bytes per packet
Is iperf3 sending jumbo frames??
In the nb8800 driver, RX_BUF_SIZE is only 1552,
how would it deal with jumbo frames... truncate?

> # iperf3 -c 172.27.64.45 -u -b 950M -l 800
> Connecting to host 172.27.64.45, port 5201
> [  4] local 172.27.64.1 port 35197 connected to 172.27.64.45 port 5201
> [ ID] Interval           Transfer     Bandwidth       Total Datagrams
> [  4]   0.00-1.00   sec  90.6 MBytes   760 Mbits/sec  118724  
> [  4]   1.00-2.00   sec   107 MBytes   894 Mbits/sec  139718  

107 MB in 139718 packets => 766 bytes per packet
800 - 8 (UDP) - 20 (IPv4) = 772 bytes per packet
I suppose that's close enough...
772 * 139718 = 107.86 MB

I need to run the test slightly slower, to prevent
packet loss at the sender.

Perhaps -b 0 or -b 800M

Regards.

Re: [RFC PATCH v2 0/2] nb8800 suspend/resume support

[Mason]

On 02/08/2017 22:02, Mason wrote:

> I need to run the test slightly slower, to prevent packet loss
> at the sender.

# iperf3 -c 172.27.64.45 -u -b 0 -l 1000
Connecting to host 172.27.64.45, port 5201
[  4] local 172.27.64.1 port 42607 connected to 172.27.64.45 port 5201
[ ID] Interval           Transfer     Bandwidth       Total Datagrams
[  4]   0.00-1.00   sec   111 MBytes   931 Mbits/sec  116420  
[  4]   1.00-2.00   sec   111 MBytes   931 Mbits/sec  116390  
[  4]   2.00-3.00   sec   111 MBytes   930 Mbits/sec  116220  
[  4]   3.00-4.00   sec   111 MBytes   930 Mbits/sec  116310  
[  4]   4.00-5.00   sec   111 MBytes   931 Mbits/sec  116380  
[  4]   5.00-6.00   sec   111 MBytes   930 Mbits/sec  116280  
[  4]   6.00-7.00   sec   111 MBytes   931 Mbits/sec  116390  
[  4]   7.00-8.00   sec   111 MBytes   931 Mbits/sec  116370  
[  4]   8.00-9.00   sec   111 MBytes   931 Mbits/sec  116340  
[  4]   9.00-10.00  sec   111 MBytes   930 Mbits/sec  116310  
- - - - - - - - - - - - - - - - - - - - - - - - -
[ ID] Interval           Transfer     Bandwidth       Jitter    Lost/Total Datagrams
[  4]   0.00-10.00  sec  1.08 GBytes   931 Mbits/sec  0.009 ms  278644/1163363 (24%)  
[  4] Sent 1163363 datagrams

iperf Done.


# iperf3 -s
Accepted connection from 172.27.64.1, port 42966
[  5] local 172.27.64.45 port 5201 connected to 172.27.64.1 port 42607
[ ID] Interval           Transfer     Bandwidth       Jitter    Lost/Total Datagrams
[  5]   0.00-1.00   sec  81.1 MBytes   681 Mbits/sec  0.017 ms  26834/111909 (24%)  
[  5]   1.00-2.00   sec  84.2 MBytes   706 Mbits/sec  0.019 ms  28127/116384 (24%)  
[  5]   2.00-3.00   sec  84.2 MBytes   706 Mbits/sec  0.013 ms  27946/116204 (24%)  
[  5]   3.00-4.00   sec  84.5 MBytes   709 Mbits/sec  0.013 ms  27674/116311 (24%)  
[  5]   4.00-5.00   sec  84.6 MBytes   709 Mbits/sec  0.015 ms  27712/116387 (24%)  
[  5]   5.00-6.00   sec  84.5 MBytes   709 Mbits/sec  0.010 ms  27649/116265 (24%)  
[  5]   6.00-7.00   sec  84.3 MBytes   707 Mbits/sec  0.011 ms  27995/116382 (24%)  
[  5]   7.00-8.00   sec  84.3 MBytes   707 Mbits/sec  0.013 ms  27972/116387 (24%)  
[  5]   8.00-9.00   sec  84.3 MBytes   708 Mbits/sec  0.020 ms  27899/116343 (24%)  
[  5]   9.00-10.00  sec  84.4 MBytes   708 Mbits/sec  0.014 ms  27759/116305 (24%)  
[  5]  10.00-10.04  sec  3.25 MBytes   710 Mbits/sec  0.009 ms  1077/4486 (24%)  
- - - - - - - - - - - - - - - - - - - - - - - - -
[ ID] Interval           Transfer     Bandwidth       Jitter    Lost/Total Datagrams
[  5]   0.00-10.04  sec  0.00 Bytes  0.00 bits/sec  0.009 ms  278644/1163363 (24%)  


IIUC, sender (desktop system) sends datagrams as fast as possible.
Receiver (tango board) drops around 24% of all datagrams.
I think this invalidates the theory that exhausting RX descriptors
wedges RX DMA.

Regards.

Re: [RFC PATCH v2 0/2] nb8800 suspend/resume support

[Måns Rullgård]

Mason <slash.tmp@free.fr> writes:

> IIUC, sender (desktop system) sends datagrams as fast as possible.
> Receiver (tango board) drops around 24% of all datagrams.
> I think this invalidates the theory that exhausting RX descriptors
> wedges RX DMA.

No, it doesn't.  The bottleneck causing packet loss could be somewhere
else.

-- 
Måns Rullgård

Re: [RFC PATCH v2 0/2] nb8800 suspend/resume support

[Måns Rullgård]

Mason <slash.tmp@free.fr> writes:

> On 02/08/2017 19:31, Mason wrote:
>
>> # iperf3 -c 172.27.64.45 -u -b 950M
>> Connecting to host 172.27.64.45, port 5201
>> [  4] local 172.27.64.1 port 55533 connected to 172.27.64.45 port 5201
>> [ ID] Interval           Transfer     Bandwidth       Total Datagrams
>> [  4]   0.00-1.00   sec   102 MBytes   858 Mbits/sec  13091  
>> [  4]   1.00-2.00   sec   114 MBytes   953 Mbits/sec  14541  
>
> 114 MB in 14541 packets => 7840 bytes per packet
> Is iperf3 sending jumbo frames??

It's probably sending fragmented packets.

-- 
Måns Rullgård