* [PATCH v2 2/9] net: pxa168_eth: Provide phy_interface mode on platform_data
From: Sebastian Hesselbarth @ 2014-10-22 18:26 UTC (permalink / raw)
To: Sebastian Hesselbarth
Cc: Florian Fainelli, Eric Miao, netdev, Antoine Ténart,
linux-kernel, Haojian Zhuang, David S. Miller, linux-arm-kernel
In-Reply-To: <1414002412-13615-1-git-send-email-sebastian.hesselbarth@gmail.com>
The PXA168 Ethernet IP support MII and RMII connection to its PHY.
Currently, pxa168 platform_data does not provide a way to pass that
and there is one user of pxa168 platform_data (mach-mmp/gplug).
Given the pinctrl settings of gplug it uses RMII, so add and pass
a corresponding phy_interface_t.
Tested-by: Antoine Ténart <antoine.tenart@free-electrons.com>
Reviewed-by: Florian Fainelli <f.fainelli@gmail.com>
Signed-off-by: Sebastian Hesselbarth <sebastian.hesselbarth@gmail.com>
---
Cc: "David S. Miller" <davem@davemloft.net>
Cc: "Antoine Ténart" <antoine.tenart@free-electrons.com>
Cc: Florian Fainelli <f.fainelli@gmail.com>
Cc: Eric Miao <eric.y.miao@gmail.com>
Cc: Haojian Zhuang <haojian.zhuang@gmail.com>
Cc: linux-arm-kernel@lists.infradead.org
Cc: netdev@vger.kernel.org
Cc: linux-kernel@vger.kernel.org
---
arch/arm/mach-mmp/gplugd.c | 2 ++
include/linux/pxa168_eth.h | 1 +
2 files changed, 3 insertions(+)
diff --git a/arch/arm/mach-mmp/gplugd.c b/arch/arm/mach-mmp/gplugd.c
index d81b2475e67e..3b5794cd0357 100644
--- a/arch/arm/mach-mmp/gplugd.c
+++ b/arch/arm/mach-mmp/gplugd.c
@@ -12,6 +12,7 @@
#include <linux/platform_device.h>
#include <linux/gpio.h>
#include <linux/gpio-pxa.h>
+#include <linux/phy.h>
#include <asm/mach/arch.h>
#include <asm/mach-types.h>
@@ -158,6 +159,7 @@ struct pxa168_eth_platform_data gplugd_eth_platform_data = {
.port_number = 0,
.phy_addr = 0,
.speed = 0, /* Autonagotiation */
+ .intf = PHY_INTERFACE_MODE_RMII,
.init = gplugd_eth_init,
};
diff --git a/include/linux/pxa168_eth.h b/include/linux/pxa168_eth.h
index 18d75e795606..37c381120bc8 100644
--- a/include/linux/pxa168_eth.h
+++ b/include/linux/pxa168_eth.h
@@ -13,6 +13,7 @@ struct pxa168_eth_platform_data {
*/
int speed; /* 0, SPEED_10, SPEED_100 */
int duplex; /* DUPLEX_HALF or DUPLEX_FULL */
+ phy_interface_t intf;
/*
* Override default RX/TX queue sizes if nonzero.
--
2.1.1
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related
* [PATCH v2 1/9] phy: marvell: Add support for 88E3016 FastEthernet PHY
From: Sebastian Hesselbarth @ 2014-10-22 18:26 UTC (permalink / raw)
To: Sebastian Hesselbarth
Cc: David S. Miller, Antoine Ténart, Florian Fainelli,
linux-arm-kernel, netdev, linux-kernel
In-Reply-To: <1414002412-13615-1-git-send-email-sebastian.hesselbarth@gmail.com>
Marvell 88E3016 is a FastEthernet PHY that also can be found in Marvell
Berlin SoCs as integrated PHY.
Tested-by: Antoine Ténart <antoine.tenart@free-electrons.com>
Reviewed-by: Florian Fainelli <f.fainelli@gmail.com>
Signed-off-by: Sebastian Hesselbarth <sebastian.hesselbarth@gmail.com>
---
Cc: "David S. Miller" <davem@davemloft.net>
Cc: "Antoine Ténart" <antoine.tenart@free-electrons.com>
Cc: Florian Fainelli <f.fainelli@gmail.com>
Cc: linux-arm-kernel@lists.infradead.org
Cc: netdev@vger.kernel.org
Cc: linux-kernel@vger.kernel.org
---
drivers/net/phy/marvell.c | 46 +++++++++++++++++++++++++++++++++++++++++++++
include/linux/marvell_phy.h | 1 +
2 files changed, 47 insertions(+)
diff --git a/drivers/net/phy/marvell.c b/drivers/net/phy/marvell.c
index bd37e45c89c0..d2b2f2f795d5 100644
--- a/drivers/net/phy/marvell.c
+++ b/drivers/net/phy/marvell.c
@@ -118,6 +118,9 @@
#define MII_M1116R_CONTROL_REG_MAC 21
+#define MII_88E3016_PHY_SPEC_CTRL 0x10
+#define MII_88E3016_DISABLE_SCRAMBLER 0x0200
+#define MII_88E3016_AUTO_MDIX_CROSSOVER 0x0030
MODULE_DESCRIPTION("Marvell PHY driver");
MODULE_AUTHOR("Andy Fleming");
@@ -434,6 +437,25 @@ static int m88e1116r_config_init(struct phy_device *phydev)
return 0;
}
+static int m88e3016_config_init(struct phy_device *phydev)
+{
+ int reg;
+
+ /* Enable Scrambler and Auto-Crossover */
+ reg = phy_read(phydev, MII_88E3016_PHY_SPEC_CTRL);
+ if (reg < 0)
+ return reg;
+
+ reg &= ~MII_88E3016_DISABLE_SCRAMBLER;
+ reg |= MII_88E3016_AUTO_MDIX_CROSSOVER;
+
+ reg = phy_write(phydev, MII_88E3016_PHY_SPEC_CTRL, reg);
+ if (reg < 0)
+ return reg;
+
+ return 0;
+}
+
static int m88e1111_config_init(struct phy_device *phydev)
{
int err;
@@ -770,6 +792,12 @@ static int marvell_read_status(struct phy_device *phydev)
return 0;
}
+static int marvell_aneg_done(struct phy_device *phydev)
+{
+ int retval = phy_read(phydev, MII_M1011_PHY_STATUS);
+ return (retval < 0) ? retval : (retval & MII_M1011_PHY_STATUS_RESOLVED);
+}
+
static int m88e1121_did_interrupt(struct phy_device *phydev)
{
int imask;
@@ -1050,6 +1078,23 @@ static struct phy_driver marvell_drivers[] = {
.suspend = &genphy_suspend,
.driver = { .owner = THIS_MODULE },
},
+ {
+ .phy_id = MARVELL_PHY_ID_88E3016,
+ .phy_id_mask = MARVELL_PHY_ID_MASK,
+ .name = "Marvell 88E3016",
+ .features = PHY_BASIC_FEATURES,
+ .flags = PHY_HAS_INTERRUPT,
+ .config_aneg = &genphy_config_aneg,
+ .config_init = &m88e3016_config_init,
+ .aneg_done = &marvell_aneg_done,
+ .read_status = &marvell_read_status,
+ .ack_interrupt = &marvell_ack_interrupt,
+ .config_intr = &marvell_config_intr,
+ .did_interrupt = &m88e1121_did_interrupt,
+ .resume = &genphy_resume,
+ .suspend = &genphy_suspend,
+ .driver = { .owner = THIS_MODULE },
+ },
};
static int __init marvell_init(void)
@@ -1079,6 +1124,7 @@ static struct mdio_device_id __maybe_unused marvell_tbl[] = {
{ MARVELL_PHY_ID_88E1318S, MARVELL_PHY_ID_MASK },
{ MARVELL_PHY_ID_88E1116R, MARVELL_PHY_ID_MASK },
{ MARVELL_PHY_ID_88E1510, MARVELL_PHY_ID_MASK },
+ { MARVELL_PHY_ID_88E3016, MARVELL_PHY_ID_MASK },
{ }
};
diff --git a/include/linux/marvell_phy.h b/include/linux/marvell_phy.h
index 8e9a029e093d..e6982ac3200d 100644
--- a/include/linux/marvell_phy.h
+++ b/include/linux/marvell_phy.h
@@ -16,6 +16,7 @@
#define MARVELL_PHY_ID_88E1318S 0x01410e90
#define MARVELL_PHY_ID_88E1116R 0x01410e40
#define MARVELL_PHY_ID_88E1510 0x01410dd0
+#define MARVELL_PHY_ID_88E3016 0x01410e60
/* struct phy_device dev_flags definitions */
#define MARVELL_PHY_M1145_FLAGS_RESISTANCE 0x00000001
--
2.1.1
^ permalink raw reply related
* [PATCH v2 0/9] Marvell PXA168 libphy handling and Berlin Ethernet
From: Sebastian Hesselbarth @ 2014-10-22 18:26 UTC (permalink / raw)
To: Sebastian Hesselbarth
Cc: devicetree, Florian Fainelli, Eric Miao, netdev,
Antoine Ténart, linux-kernel, Haojian Zhuang,
David S. Miller, linux-arm-kernel
This patch series deals with a removing a IP feature that can be found
on all currently supported Marvell Ethernet IP (pxa168_eth, mv643xx_eth,
mvneta). The MAC IP allows to automatically perform PHY auto-negotiation
without software interaction.
However, this feature (a) fundamentally clashes with the way libphy works
and (b) is unable to deal with quirky PHYs that require special treatment.
In this series, pxa168_eth driver is rewritten to completely disable that
feature and properly deal with libphy provided PHYs.
As usual, a branch on top of v3.18-rc1 can be found at
git://git.infradead.org/users/hesselba/linux-berlin.git devel/bg2-bg2cd-eth-v2
Patches 1-5 should go through David's net tree, I'll pick up the DT patches
6-9.
There have been some changes,
compared to the RFT
- added phy-connection-type property to BG2Q PHY DT node
- bail out from pxa168_eth_adjust_link when there is no change in
PHY parameters. Also, add a call to phy_print_status.
compared to v1
- move phy-connection-type to ethernet node instead of PHY node
Patch 1 adds support for Marvell 88E3016 FastEthernet PHY that is also
integrated in Marvell Berlin BG2/BG2CD SoCs.
Patch 2 allows to pass phy_interface_t on pxa168_eth platform_data that
is only used by mach-mmp/gplug. From the board setup, I guessed gplug's
PHY is connected via RMII. The patch still isn't even compile tested.
Patches 3-5 prepare proper libphy handling and finally remove all in-driver
PHY mangling related to the feature explained above.
Patches 6-9 add corresponding ethernet DT nodes to BG2, BG2CD, add a
phy-connection-type property to BG2Q and enable ethernet on BG2-based Sony
NSZ-GS7. I have tested all this on GS7 successfully with ip=dhcp on 100M FD.
Antoine Ténart (1):
ARM: berlin: Add phy-connection-type to BG2Q Ethernet
Sebastian Hesselbarth (8):
phy: marvell: Add support for 88E3016 FastEthernet PHY
net: pxa168_eth: Provide phy_interface mode on platform_data
net: pxa168_eth: Prepare proper libphy handling
net: pxa168_eth: Remove HW auto-negotiaion
net: pxa168_eth: Remove in-driver PHY mangling
ARM: berlin: Add BG2 ethernet DT nodes
ARM: berlin: Add BG2CD ethernet DT nodes
ARM: berlin: Enable ethernet on Sony NSZ-GS7
arch/arm/boot/dts/berlin2-sony-nsz-gs7.dts | 2 +
arch/arm/boot/dts/berlin2.dtsi | 36 +++++
arch/arm/boot/dts/berlin2cd.dtsi | 36 +++++
arch/arm/boot/dts/berlin2q.dtsi | 1 +
arch/arm/mach-mmp/gplugd.c | 2 +
drivers/net/ethernet/marvell/pxa168_eth.c | 248 ++++++++++++-----------------
drivers/net/phy/marvell.c | 46 ++++++
include/linux/marvell_phy.h | 1 +
include/linux/pxa168_eth.h | 1 +
9 files changed, 225 insertions(+), 148 deletions(-)
---
Cc: "David S. Miller" <davem@davemloft.net>
Cc: "Antoine Ténart" <antoine.tenart@free-electrons.com>
Cc: Florian Fainelli <f.fainelli@gmail.com>
Cc: Eric Miao <eric.y.miao@gmail.com>
Cc: Haojian Zhuang <haojian.zhuang@gmail.com>
Cc: linux-arm-kernel@lists.infradead.org
Cc: netdev@vger.kernel.org
Cc: devicetree@vger.kernel.org
Cc: linux-kernel@vger.kernel.org
--
2.1.1
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply
* Re: localed stuck in recent 3.18 git in copy_net_ns?
From: Eric W. Biederman @ 2014-10-22 18:25 UTC (permalink / raw)
To: paulmck
Cc: Cong Wang, Josh Boyer, Kevin Fenzi, netdev,
Linux-Kernel@Vger. Kernel. Org
In-Reply-To: <20141022181135.GH4977@linux.vnet.ibm.com>
"Paul E. McKenney" <paulmck@linux.vnet.ibm.com> writes:
> On Wed, Oct 22, 2014 at 12:53:24PM -0500, Eric W. Biederman wrote:
>> Cong Wang <cwang@twopensource.com> writes:
>>
>> > (Adding Paul and Eric in Cc)
>> >
>> >
>> > On Wed, Oct 22, 2014 at 10:12 AM, Josh Boyer <jwboyer@fedoraproject.org> wrote:
>> >>
>> >> Someone else is seeing this when they try and modprobe ppp_generic:
>> >>
>> >> [ 240.599195] INFO: task kworker/u16:5:100 blocked for more than 120 seconds.
>> >> [ 240.599338] Not tainted 3.18.0-0.rc1.git2.1.fc22.x86_64 #1
>> >> [ 240.599446] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs"
>> >> disables this message.
>> >> [ 240.599583] kworker/u16:5 D ffff8802202db480 12400 100 2 0x00000000
>> >> [ 240.599744] Workqueue: netns cleanup_net
>> >> [ 240.599823] ffff8802202eb9e8 0000000000000096 ffff8802202db480
>> >> 00000000001d5f00
>> >> [ 240.600066] ffff8802202ebfd8 00000000001d5f00 ffff8800368c3480
>> >> ffff8802202db480
>> >> [ 240.600228] ffffffff81ee2690 7fffffffffffffff ffffffff81ee2698
>> >> ffffffff81ee2690
>> >> [ 240.600386] Call Trace:
>> >> [ 240.600445] [<ffffffff8185e239>] schedule+0x29/0x70
>> >> [ 240.600541] [<ffffffff8186345c>] schedule_timeout+0x26c/0x410
>> >> [ 240.600651] [<ffffffff81865ef7>] ? retint_restore_args+0x13/0x13
>> >> [ 240.600765] [<ffffffff818644e4>] ? _raw_spin_unlock_irq+0x34/0x50
>> >> [ 240.600879] [<ffffffff8185fc6c>] wait_for_completion+0x10c/0x150
>> >> [ 240.601025] [<ffffffff810e53e0>] ? wake_up_state+0x20/0x20
>> >> [ 240.601133] [<ffffffff8112a749>] _rcu_barrier+0x159/0x200
>> >> [ 240.601237] [<ffffffff8112a845>] rcu_barrier+0x15/0x20
>> >> [ 240.601335] [<ffffffff81718ebf>] netdev_run_todo+0x6f/0x310
>> >> [ 240.601442] [<ffffffff8170da85>] ? rollback_registered_many+0x265/0x2e0
>> >> [ 240.601564] [<ffffffff81725f2e>] rtnl_unlock+0xe/0x10
>> >> [ 240.601660] [<ffffffff8170f8e6>] default_device_exit_batch+0x156/0x180
>> >> [ 240.601781] [<ffffffff810fd8a0>] ? abort_exclusive_wait+0xb0/0xb0
>> >> [ 240.601895] [<ffffffff81707993>] ops_exit_list.isra.1+0x53/0x60
>> >> [ 240.602028] [<ffffffff81708540>] cleanup_net+0x100/0x1f0
>> >> [ 240.602131] [<ffffffff810ccfa8>] process_one_work+0x218/0x850
>> >> [ 240.602241] [<ffffffff810ccf0f>] ? process_one_work+0x17f/0x850
>> >> [ 240.602350] [<ffffffff810cd6c7>] ? worker_thread+0xe7/0x4a0
>> >> [ 240.602454] [<ffffffff810cd64b>] worker_thread+0x6b/0x4a0
>> >> [ 240.602555] [<ffffffff810cd5e0>] ? process_one_work+0x850/0x850
>> >> [ 240.602665] [<ffffffff810d399b>] kthread+0x10b/0x130
>> >> [ 240.602762] [<ffffffff81028cc9>] ? sched_clock+0x9/0x10
>> >> [ 240.602862] [<ffffffff810d3890>] ? kthread_create_on_node+0x250/0x250
>> >> [ 240.603004] [<ffffffff818651fc>] ret_from_fork+0x7c/0xb0
>> >> [ 240.603106] [<ffffffff810d3890>] ? kthread_create_on_node+0x250/0x250
>> >> [ 240.603224] 4 locks held by kworker/u16:5/100:
>> >> [ 240.603304] #0: ("%s""netns"){.+.+.+}, at: [<ffffffff810ccf0f>]
>> >> process_one_work+0x17f/0x850
>> >> [ 240.603495] #1: (net_cleanup_work){+.+.+.}, at:
>> >> [<ffffffff810ccf0f>] process_one_work+0x17f/0x850
>> >> [ 240.603691] #2: (net_mutex){+.+.+.}, at: [<ffffffff817084cc>]
>> >> cleanup_net+0x8c/0x1f0
>> >> [ 240.603869] #3: (rcu_sched_state.barrier_mutex){+.+...}, at:
>> >> [<ffffffff8112a625>] _rcu_barrier+0x35/0x200
>> >> [ 240.604211] INFO: task modprobe:1387 blocked for more than 120 seconds.
>> >> [ 240.604329] Not tainted 3.18.0-0.rc1.git2.1.fc22.x86_64 #1
>> >> [ 240.604434] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs"
>> >> disables this message.
>> >> [ 240.604570] modprobe D ffff8800cb4f1a40 13112 1387 1386 0x00000080
>> >> [ 240.604719] ffff8800cafbbbe8 0000000000000096 ffff8800cb4f1a40
>> >> 00000000001d5f00
>> >> [ 240.604878] ffff8800cafbbfd8 00000000001d5f00 ffff880223280000
>> >> ffff8800cb4f1a40
>> >> [ 240.605068] ffff8800cb4f1a40 ffffffff81f8fb48 0000000000000246
>> >> ffff8800cb4f1a40
>> >> [ 240.605228] Call Trace:
>> >> [ 240.605283] [<ffffffff8185e7e1>] schedule_preempt_disabled+0x31/0x80
>> >> [ 240.605400] [<ffffffff81860033>] mutex_lock_nested+0x183/0x440
>> >> [ 240.605510] [<ffffffff8170835f>] ? register_pernet_subsys+0x1f/0x50
>> >> [ 240.605626] [<ffffffff8170835f>] ? register_pernet_subsys+0x1f/0x50
>> >> [ 240.605757] [<ffffffffa0701000>] ? 0xffffffffa0701000
>> >> [ 240.605854] [<ffffffff8170835f>] register_pernet_subsys+0x1f/0x50
>> >> [ 240.606005] [<ffffffffa0701048>] br_init+0x48/0xd3 [bridge]
>> >> [ 240.606112] [<ffffffff81002148>] do_one_initcall+0xd8/0x210
>> >> [ 240.606224] [<ffffffff81153c02>] load_module+0x20c2/0x2870
>> >> [ 240.606327] [<ffffffff8114ebe0>] ? store_uevent+0x70/0x70
>> >> [ 240.606433] [<ffffffff8110ac26>] ? lock_release_non_nested+0x3c6/0x3d0
>> >> [ 240.606557] [<ffffffff81154497>] SyS_init_module+0xe7/0x140
>> >> [ 240.606664] [<ffffffff818652a9>] system_call_fastpath+0x12/0x17
>> >> [ 240.606773] 1 lock held by modprobe/1387:
>> >> [ 240.606845] #0: (net_mutex){+.+.+.}, at: [<ffffffff8170835f>]
>> >> register_pernet_subsys+0x1f/0x50
>> >> [ 240.607114] INFO: task modprobe:1466 blocked for more than 120 seconds.
>> >> [ 240.607231] Not tainted 3.18.0-0.rc1.git2.1.fc22.x86_64 #1
>> >> [ 240.607337] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs"
>> >> disables this message.
>> >> [ 240.607473] modprobe D ffff88020fbab480 13096 1466 1399 0x00000084
>> >> [ 240.607622] ffff88020d1bbbe8 0000000000000096 ffff88020fbab480
>> >> 00000000001d5f00
>> >> [ 240.607791] ffff88020d1bbfd8 00000000001d5f00 ffffffff81e1b580
>> >> ffff88020fbab480
>> >> [ 240.607949] ffff88020fbab480 ffffffff81f8fb48 0000000000000246
>> >> ffff88020fbab480
>> >> [ 240.608138] Call Trace:
>> >> [ 240.608193] [<ffffffff8185e7e1>] schedule_preempt_disabled+0x31/0x80
>> >> [ 240.608316] [<ffffffff81860033>] mutex_lock_nested+0x183/0x440
>> >> [ 240.608425] [<ffffffff817083ad>] ? register_pernet_device+0x1d/0x70
>> >> [ 240.608542] [<ffffffff817083ad>] ? register_pernet_device+0x1d/0x70
>> >> [ 240.608662] [<ffffffffa071d000>] ? 0xffffffffa071d000
>> >> [ 240.608759] [<ffffffff817083ad>] register_pernet_device+0x1d/0x70
>> >> [ 240.608881] [<ffffffffa071d020>] ppp_init+0x20/0x1000 [ppp_generic]
>> >> [ 240.609021] [<ffffffff81002148>] do_one_initcall+0xd8/0x210
>> >> [ 240.609131] [<ffffffff81153c02>] load_module+0x20c2/0x2870
>> >> [ 240.609235] [<ffffffff8114ebe0>] ? store_uevent+0x70/0x70
>> >> [ 240.609339] [<ffffffff8110ac26>] ? lock_release_non_nested+0x3c6/0x3d0
>> >> [ 240.609462] [<ffffffff81154497>] SyS_init_module+0xe7/0x140
>> >> [ 240.609568] [<ffffffff818652a9>] system_call_fastpath+0x12/0x17
>> >> [ 240.609677] 1 lock held by modprobe/1466:
>> >> [ 240.609749] #0: (net_mutex){+.+.+.}, at: [<ffffffff817083ad>]
>> >> register_pernet_device+0x1d/0x70
>> >>
>> >> Looks like contention on net_mutex or something, but I honestly have
>> >> no idea yet. I can't recreate it myself at the moment or I would
>> >> bisect.
>> >>
>> >> Has nobody else run into this with the pre-3.18 kernels? Fedora isn't
>> >> carrying any patches in this area.
>>
>> > I am not aware of any change in net/core/dev.c related here,
>> > so I guess it's a bug in rcu_barrier().
>>
>> >From the limited trace data I see in this email I have to agree.
>>
>> It looks like for some reason rcu_barrier is taking forever
>> while the rtnl_lock is held in cleanup_net. Because the
>> rtnl_lock is held modprobe of the ppp driver is getting stuck.
>>
>> Is it possible we have an AB BA deadlock between the rtnl_lock
>> and rcu. With something the module loading code assumes?
>
> I am not aware of RCU ever acquiring rtnl_lock, not directly, anyway.
Does the module loading code do something strange with rcu? Perhaps
blocking an rcu grace period until the module loading completes?
If the module loading somehow blocks an rcu grace period that would
create an AB deadlock because loading the ppp module grabs the
rtnl_lock. And elsewhere we have the rtnl_lock waiting for an rcu grace
period.
I would think trying and failing to get the rtnl_lock would sleep and
thus let any rcu grace period happen but shrug.
It looks like something is holding up the rcu grace period, and causing
this. Although it is possible that something is causing cleanup_net
to run slowly and we are just seeing that slowness show up in
rcu_barrier as that is one of the slower bits. With a single trace I
can't definitely same that the rcu barrier is getting stuck but it
certainly looks that way.
Eric
^ permalink raw reply
* Re: localed stuck in recent 3.18 git in copy_net_ns?
From: Paul E. McKenney @ 2014-10-22 18:11 UTC (permalink / raw)
To: Eric W. Biederman
Cc: Cong Wang, Josh Boyer, Kevin Fenzi, netdev,
Linux-Kernel@Vger. Kernel. Org
In-Reply-To: <8738aghtyj.fsf@x220.int.ebiederm.org>
On Wed, Oct 22, 2014 at 12:53:24PM -0500, Eric W. Biederman wrote:
> Cong Wang <cwang@twopensource.com> writes:
>
> > (Adding Paul and Eric in Cc)
> >
> >
> > On Wed, Oct 22, 2014 at 10:12 AM, Josh Boyer <jwboyer@fedoraproject.org> wrote:
> >>
> >> Someone else is seeing this when they try and modprobe ppp_generic:
> >>
> >> [ 240.599195] INFO: task kworker/u16:5:100 blocked for more than 120 seconds.
> >> [ 240.599338] Not tainted 3.18.0-0.rc1.git2.1.fc22.x86_64 #1
> >> [ 240.599446] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs"
> >> disables this message.
> >> [ 240.599583] kworker/u16:5 D ffff8802202db480 12400 100 2 0x00000000
> >> [ 240.599744] Workqueue: netns cleanup_net
> >> [ 240.599823] ffff8802202eb9e8 0000000000000096 ffff8802202db480
> >> 00000000001d5f00
> >> [ 240.600066] ffff8802202ebfd8 00000000001d5f00 ffff8800368c3480
> >> ffff8802202db480
> >> [ 240.600228] ffffffff81ee2690 7fffffffffffffff ffffffff81ee2698
> >> ffffffff81ee2690
> >> [ 240.600386] Call Trace:
> >> [ 240.600445] [<ffffffff8185e239>] schedule+0x29/0x70
> >> [ 240.600541] [<ffffffff8186345c>] schedule_timeout+0x26c/0x410
> >> [ 240.600651] [<ffffffff81865ef7>] ? retint_restore_args+0x13/0x13
> >> [ 240.600765] [<ffffffff818644e4>] ? _raw_spin_unlock_irq+0x34/0x50
> >> [ 240.600879] [<ffffffff8185fc6c>] wait_for_completion+0x10c/0x150
> >> [ 240.601025] [<ffffffff810e53e0>] ? wake_up_state+0x20/0x20
> >> [ 240.601133] [<ffffffff8112a749>] _rcu_barrier+0x159/0x200
> >> [ 240.601237] [<ffffffff8112a845>] rcu_barrier+0x15/0x20
> >> [ 240.601335] [<ffffffff81718ebf>] netdev_run_todo+0x6f/0x310
> >> [ 240.601442] [<ffffffff8170da85>] ? rollback_registered_many+0x265/0x2e0
> >> [ 240.601564] [<ffffffff81725f2e>] rtnl_unlock+0xe/0x10
> >> [ 240.601660] [<ffffffff8170f8e6>] default_device_exit_batch+0x156/0x180
> >> [ 240.601781] [<ffffffff810fd8a0>] ? abort_exclusive_wait+0xb0/0xb0
> >> [ 240.601895] [<ffffffff81707993>] ops_exit_list.isra.1+0x53/0x60
> >> [ 240.602028] [<ffffffff81708540>] cleanup_net+0x100/0x1f0
> >> [ 240.602131] [<ffffffff810ccfa8>] process_one_work+0x218/0x850
> >> [ 240.602241] [<ffffffff810ccf0f>] ? process_one_work+0x17f/0x850
> >> [ 240.602350] [<ffffffff810cd6c7>] ? worker_thread+0xe7/0x4a0
> >> [ 240.602454] [<ffffffff810cd64b>] worker_thread+0x6b/0x4a0
> >> [ 240.602555] [<ffffffff810cd5e0>] ? process_one_work+0x850/0x850
> >> [ 240.602665] [<ffffffff810d399b>] kthread+0x10b/0x130
> >> [ 240.602762] [<ffffffff81028cc9>] ? sched_clock+0x9/0x10
> >> [ 240.602862] [<ffffffff810d3890>] ? kthread_create_on_node+0x250/0x250
> >> [ 240.603004] [<ffffffff818651fc>] ret_from_fork+0x7c/0xb0
> >> [ 240.603106] [<ffffffff810d3890>] ? kthread_create_on_node+0x250/0x250
> >> [ 240.603224] 4 locks held by kworker/u16:5/100:
> >> [ 240.603304] #0: ("%s""netns"){.+.+.+}, at: [<ffffffff810ccf0f>]
> >> process_one_work+0x17f/0x850
> >> [ 240.603495] #1: (net_cleanup_work){+.+.+.}, at:
> >> [<ffffffff810ccf0f>] process_one_work+0x17f/0x850
> >> [ 240.603691] #2: (net_mutex){+.+.+.}, at: [<ffffffff817084cc>]
> >> cleanup_net+0x8c/0x1f0
> >> [ 240.603869] #3: (rcu_sched_state.barrier_mutex){+.+...}, at:
> >> [<ffffffff8112a625>] _rcu_barrier+0x35/0x200
> >> [ 240.604211] INFO: task modprobe:1387 blocked for more than 120 seconds.
> >> [ 240.604329] Not tainted 3.18.0-0.rc1.git2.1.fc22.x86_64 #1
> >> [ 240.604434] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs"
> >> disables this message.
> >> [ 240.604570] modprobe D ffff8800cb4f1a40 13112 1387 1386 0x00000080
> >> [ 240.604719] ffff8800cafbbbe8 0000000000000096 ffff8800cb4f1a40
> >> 00000000001d5f00
> >> [ 240.604878] ffff8800cafbbfd8 00000000001d5f00 ffff880223280000
> >> ffff8800cb4f1a40
> >> [ 240.605068] ffff8800cb4f1a40 ffffffff81f8fb48 0000000000000246
> >> ffff8800cb4f1a40
> >> [ 240.605228] Call Trace:
> >> [ 240.605283] [<ffffffff8185e7e1>] schedule_preempt_disabled+0x31/0x80
> >> [ 240.605400] [<ffffffff81860033>] mutex_lock_nested+0x183/0x440
> >> [ 240.605510] [<ffffffff8170835f>] ? register_pernet_subsys+0x1f/0x50
> >> [ 240.605626] [<ffffffff8170835f>] ? register_pernet_subsys+0x1f/0x50
> >> [ 240.605757] [<ffffffffa0701000>] ? 0xffffffffa0701000
> >> [ 240.605854] [<ffffffff8170835f>] register_pernet_subsys+0x1f/0x50
> >> [ 240.606005] [<ffffffffa0701048>] br_init+0x48/0xd3 [bridge]
> >> [ 240.606112] [<ffffffff81002148>] do_one_initcall+0xd8/0x210
> >> [ 240.606224] [<ffffffff81153c02>] load_module+0x20c2/0x2870
> >> [ 240.606327] [<ffffffff8114ebe0>] ? store_uevent+0x70/0x70
> >> [ 240.606433] [<ffffffff8110ac26>] ? lock_release_non_nested+0x3c6/0x3d0
> >> [ 240.606557] [<ffffffff81154497>] SyS_init_module+0xe7/0x140
> >> [ 240.606664] [<ffffffff818652a9>] system_call_fastpath+0x12/0x17
> >> [ 240.606773] 1 lock held by modprobe/1387:
> >> [ 240.606845] #0: (net_mutex){+.+.+.}, at: [<ffffffff8170835f>]
> >> register_pernet_subsys+0x1f/0x50
> >> [ 240.607114] INFO: task modprobe:1466 blocked for more than 120 seconds.
> >> [ 240.607231] Not tainted 3.18.0-0.rc1.git2.1.fc22.x86_64 #1
> >> [ 240.607337] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs"
> >> disables this message.
> >> [ 240.607473] modprobe D ffff88020fbab480 13096 1466 1399 0x00000084
> >> [ 240.607622] ffff88020d1bbbe8 0000000000000096 ffff88020fbab480
> >> 00000000001d5f00
> >> [ 240.607791] ffff88020d1bbfd8 00000000001d5f00 ffffffff81e1b580
> >> ffff88020fbab480
> >> [ 240.607949] ffff88020fbab480 ffffffff81f8fb48 0000000000000246
> >> ffff88020fbab480
> >> [ 240.608138] Call Trace:
> >> [ 240.608193] [<ffffffff8185e7e1>] schedule_preempt_disabled+0x31/0x80
> >> [ 240.608316] [<ffffffff81860033>] mutex_lock_nested+0x183/0x440
> >> [ 240.608425] [<ffffffff817083ad>] ? register_pernet_device+0x1d/0x70
> >> [ 240.608542] [<ffffffff817083ad>] ? register_pernet_device+0x1d/0x70
> >> [ 240.608662] [<ffffffffa071d000>] ? 0xffffffffa071d000
> >> [ 240.608759] [<ffffffff817083ad>] register_pernet_device+0x1d/0x70
> >> [ 240.608881] [<ffffffffa071d020>] ppp_init+0x20/0x1000 [ppp_generic]
> >> [ 240.609021] [<ffffffff81002148>] do_one_initcall+0xd8/0x210
> >> [ 240.609131] [<ffffffff81153c02>] load_module+0x20c2/0x2870
> >> [ 240.609235] [<ffffffff8114ebe0>] ? store_uevent+0x70/0x70
> >> [ 240.609339] [<ffffffff8110ac26>] ? lock_release_non_nested+0x3c6/0x3d0
> >> [ 240.609462] [<ffffffff81154497>] SyS_init_module+0xe7/0x140
> >> [ 240.609568] [<ffffffff818652a9>] system_call_fastpath+0x12/0x17
> >> [ 240.609677] 1 lock held by modprobe/1466:
> >> [ 240.609749] #0: (net_mutex){+.+.+.}, at: [<ffffffff817083ad>]
> >> register_pernet_device+0x1d/0x70
> >>
> >> Looks like contention on net_mutex or something, but I honestly have
> >> no idea yet. I can't recreate it myself at the moment or I would
> >> bisect.
> >>
> >> Has nobody else run into this with the pre-3.18 kernels? Fedora isn't
> >> carrying any patches in this area.
>
> > I am not aware of any change in net/core/dev.c related here,
> > so I guess it's a bug in rcu_barrier().
>
> >From the limited trace data I see in this email I have to agree.
>
> It looks like for some reason rcu_barrier is taking forever
> while the rtnl_lock is held in cleanup_net. Because the
> rtnl_lock is held modprobe of the ppp driver is getting stuck.
>
> Is it possible we have an AB BA deadlock between the rtnl_lock
> and rcu. With something the module loading code assumes?
I am not aware of RCU ever acquiring rtnl_lock, not directly, anyway.
Thanx, Paul
^ permalink raw reply
* Re: localed stuck in recent 3.18 git in copy_net_ns?
From: Josh Boyer @ 2014-10-22 18:03 UTC (permalink / raw)
To: Paul McKenney
Cc: Cong Wang, Kevin Fenzi, netdev, Linux-Kernel@Vger. Kernel. Org,
Eric W. Biederman
In-Reply-To: <20141022175908.GG4977@linux.vnet.ibm.com>
On Wed, Oct 22, 2014 at 1:59 PM, Paul E. McKenney
<paulmck@linux.vnet.ibm.com> wrote:
> On Wed, Oct 22, 2014 at 10:37:53AM -0700, Cong Wang wrote:
>> (Adding Paul and Eric in Cc)
>>
>> I am not aware of any change in net/core/dev.c related here,
>> so I guess it's a bug in rcu_barrier().
>>
>> Thanks.
>
> Does commit 789cbbeca4e (workqueue: Add quiescent state between work items)
> and 3e28e3772 (workqueue: Use cond_resched_rcu_qs macro) help this?
I don't believe so. The output below is from a post 3.18-rc1 kernel
(Linux v3.18-rc1-221-gc3351dfabf5c to be exact), and both of those
commits are included in that if I'm reading the git output correctly.
josh
>> On Wed, Oct 22, 2014 at 10:12 AM, Josh Boyer <jwboyer@fedoraproject.org> wrote:
>> >
>> > Someone else is seeing this when they try and modprobe ppp_generic:
>> >
>> > [ 240.599195] INFO: task kworker/u16:5:100 blocked for more than 120 seconds.
>> > [ 240.599338] Not tainted 3.18.0-0.rc1.git2.1.fc22.x86_64 #1
>> > [ 240.599446] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs"
>> > disables this message.
>> > [ 240.599583] kworker/u16:5 D ffff8802202db480 12400 100 2 0x00000000
>> > [ 240.599744] Workqueue: netns cleanup_net
>> > [ 240.599823] ffff8802202eb9e8 0000000000000096 ffff8802202db480
>> > 00000000001d5f00
>> > [ 240.600066] ffff8802202ebfd8 00000000001d5f00 ffff8800368c3480
>> > ffff8802202db480
>> > [ 240.600228] ffffffff81ee2690 7fffffffffffffff ffffffff81ee2698
>> > ffffffff81ee2690
>> > [ 240.600386] Call Trace:
>> > [ 240.600445] [<ffffffff8185e239>] schedule+0x29/0x70
>> > [ 240.600541] [<ffffffff8186345c>] schedule_timeout+0x26c/0x410
>> > [ 240.600651] [<ffffffff81865ef7>] ? retint_restore_args+0x13/0x13
>> > [ 240.600765] [<ffffffff818644e4>] ? _raw_spin_unlock_irq+0x34/0x50
>> > [ 240.600879] [<ffffffff8185fc6c>] wait_for_completion+0x10c/0x150
>> > [ 240.601025] [<ffffffff810e53e0>] ? wake_up_state+0x20/0x20
>> > [ 240.601133] [<ffffffff8112a749>] _rcu_barrier+0x159/0x200
>> > [ 240.601237] [<ffffffff8112a845>] rcu_barrier+0x15/0x20
>> > [ 240.601335] [<ffffffff81718ebf>] netdev_run_todo+0x6f/0x310
>> > [ 240.601442] [<ffffffff8170da85>] ? rollback_registered_many+0x265/0x2e0
>> > [ 240.601564] [<ffffffff81725f2e>] rtnl_unlock+0xe/0x10
>> > [ 240.601660] [<ffffffff8170f8e6>] default_device_exit_batch+0x156/0x180
>> > [ 240.601781] [<ffffffff810fd8a0>] ? abort_exclusive_wait+0xb0/0xb0
>> > [ 240.601895] [<ffffffff81707993>] ops_exit_list.isra.1+0x53/0x60
>> > [ 240.602028] [<ffffffff81708540>] cleanup_net+0x100/0x1f0
>> > [ 240.602131] [<ffffffff810ccfa8>] process_one_work+0x218/0x850
>> > [ 240.602241] [<ffffffff810ccf0f>] ? process_one_work+0x17f/0x850
>> > [ 240.602350] [<ffffffff810cd6c7>] ? worker_thread+0xe7/0x4a0
>> > [ 240.602454] [<ffffffff810cd64b>] worker_thread+0x6b/0x4a0
>> > [ 240.602555] [<ffffffff810cd5e0>] ? process_one_work+0x850/0x850
>> > [ 240.602665] [<ffffffff810d399b>] kthread+0x10b/0x130
>> > [ 240.602762] [<ffffffff81028cc9>] ? sched_clock+0x9/0x10
>> > [ 240.602862] [<ffffffff810d3890>] ? kthread_create_on_node+0x250/0x250
>> > [ 240.603004] [<ffffffff818651fc>] ret_from_fork+0x7c/0xb0
>> > [ 240.603106] [<ffffffff810d3890>] ? kthread_create_on_node+0x250/0x250
>> > [ 240.603224] 4 locks held by kworker/u16:5/100:
>> > [ 240.603304] #0: ("%s""netns"){.+.+.+}, at: [<ffffffff810ccf0f>]
>> > process_one_work+0x17f/0x850
>> > [ 240.603495] #1: (net_cleanup_work){+.+.+.}, at:
>> > [<ffffffff810ccf0f>] process_one_work+0x17f/0x850
>> > [ 240.603691] #2: (net_mutex){+.+.+.}, at: [<ffffffff817084cc>]
>> > cleanup_net+0x8c/0x1f0
>> > [ 240.603869] #3: (rcu_sched_state.barrier_mutex){+.+...}, at:
>> > [<ffffffff8112a625>] _rcu_barrier+0x35/0x200
>> > [ 240.604211] INFO: task modprobe:1387 blocked for more than 120 seconds.
>> > [ 240.604329] Not tainted 3.18.0-0.rc1.git2.1.fc22.x86_64 #1
>> > [ 240.604434] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs"
>> > disables this message.
>> > [ 240.604570] modprobe D ffff8800cb4f1a40 13112 1387 1386 0x00000080
>> > [ 240.604719] ffff8800cafbbbe8 0000000000000096 ffff8800cb4f1a40
>> > 00000000001d5f00
>> > [ 240.604878] ffff8800cafbbfd8 00000000001d5f00 ffff880223280000
>> > ffff8800cb4f1a40
>> > [ 240.605068] ffff8800cb4f1a40 ffffffff81f8fb48 0000000000000246
>> > ffff8800cb4f1a40
>> > [ 240.605228] Call Trace:
>> > [ 240.605283] [<ffffffff8185e7e1>] schedule_preempt_disabled+0x31/0x80
>> > [ 240.605400] [<ffffffff81860033>] mutex_lock_nested+0x183/0x440
>> > [ 240.605510] [<ffffffff8170835f>] ? register_pernet_subsys+0x1f/0x50
>> > [ 240.605626] [<ffffffff8170835f>] ? register_pernet_subsys+0x1f/0x50
>> > [ 240.605757] [<ffffffffa0701000>] ? 0xffffffffa0701000
>> > [ 240.605854] [<ffffffff8170835f>] register_pernet_subsys+0x1f/0x50
>> > [ 240.606005] [<ffffffffa0701048>] br_init+0x48/0xd3 [bridge]
>> > [ 240.606112] [<ffffffff81002148>] do_one_initcall+0xd8/0x210
>> > [ 240.606224] [<ffffffff81153c02>] load_module+0x20c2/0x2870
>> > [ 240.606327] [<ffffffff8114ebe0>] ? store_uevent+0x70/0x70
>> > [ 240.606433] [<ffffffff8110ac26>] ? lock_release_non_nested+0x3c6/0x3d0
>> > [ 240.606557] [<ffffffff81154497>] SyS_init_module+0xe7/0x140
>> > [ 240.606664] [<ffffffff818652a9>] system_call_fastpath+0x12/0x17
>> > [ 240.606773] 1 lock held by modprobe/1387:
>> > [ 240.606845] #0: (net_mutex){+.+.+.}, at: [<ffffffff8170835f>]
>> > register_pernet_subsys+0x1f/0x50
>> > [ 240.607114] INFO: task modprobe:1466 blocked for more than 120 seconds.
>> > [ 240.607231] Not tainted 3.18.0-0.rc1.git2.1.fc22.x86_64 #1
>> > [ 240.607337] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs"
>> > disables this message.
>> > [ 240.607473] modprobe D ffff88020fbab480 13096 1466 1399 0x00000084
>> > [ 240.607622] ffff88020d1bbbe8 0000000000000096 ffff88020fbab480
>> > 00000000001d5f00
>> > [ 240.607791] ffff88020d1bbfd8 00000000001d5f00 ffffffff81e1b580
>> > ffff88020fbab480
>> > [ 240.607949] ffff88020fbab480 ffffffff81f8fb48 0000000000000246
>> > ffff88020fbab480
>> > [ 240.608138] Call Trace:
>> > [ 240.608193] [<ffffffff8185e7e1>] schedule_preempt_disabled+0x31/0x80
>> > [ 240.608316] [<ffffffff81860033>] mutex_lock_nested+0x183/0x440
>> > [ 240.608425] [<ffffffff817083ad>] ? register_pernet_device+0x1d/0x70
>> > [ 240.608542] [<ffffffff817083ad>] ? register_pernet_device+0x1d/0x70
>> > [ 240.608662] [<ffffffffa071d000>] ? 0xffffffffa071d000
>> > [ 240.608759] [<ffffffff817083ad>] register_pernet_device+0x1d/0x70
>> > [ 240.608881] [<ffffffffa071d020>] ppp_init+0x20/0x1000 [ppp_generic]
>> > [ 240.609021] [<ffffffff81002148>] do_one_initcall+0xd8/0x210
>> > [ 240.609131] [<ffffffff81153c02>] load_module+0x20c2/0x2870
>> > [ 240.609235] [<ffffffff8114ebe0>] ? store_uevent+0x70/0x70
>> > [ 240.609339] [<ffffffff8110ac26>] ? lock_release_non_nested+0x3c6/0x3d0
>> > [ 240.609462] [<ffffffff81154497>] SyS_init_module+0xe7/0x140
>> > [ 240.609568] [<ffffffff818652a9>] system_call_fastpath+0x12/0x17
>> > [ 240.609677] 1 lock held by modprobe/1466:
>> > [ 240.609749] #0: (net_mutex){+.+.+.}, at: [<ffffffff817083ad>]
>> > register_pernet_device+0x1d/0x70
>> >
>> > Looks like contention on net_mutex or something, but I honestly have
>> > no idea yet. I can't recreate it myself at the moment or I would
>> > bisect.
>> >
>> > Has nobody else run into this with the pre-3.18 kernels? Fedora isn't
>> > carrying any patches in this area.
>> >
>> > josh
>> > --
>> > To unsubscribe from this list: send the line "unsubscribe netdev" in
>> > the body of a message to majordomo@vger.kernel.org
>> > More majordomo info at http://vger.kernel.org/majordomo-info.html
>>
>
^ permalink raw reply
* Re: localed stuck in recent 3.18 git in copy_net_ns?
From: Paul E. McKenney @ 2014-10-22 17:59 UTC (permalink / raw)
To: Cong Wang
Cc: Josh Boyer, Kevin Fenzi, netdev, Linux-Kernel@Vger. Kernel. Org,
Eric W. Biederman
In-Reply-To: <CAHA+R7OUHy8XFPoip5gPvr1uqwkxgKxoSMf_pSgB1aFx=XCs8g@mail.gmail.com>
On Wed, Oct 22, 2014 at 10:37:53AM -0700, Cong Wang wrote:
> (Adding Paul and Eric in Cc)
>
> I am not aware of any change in net/core/dev.c related here,
> so I guess it's a bug in rcu_barrier().
>
> Thanks.
Does commit 789cbbeca4e (workqueue: Add quiescent state between work items)
and 3e28e3772 (workqueue: Use cond_resched_rcu_qs macro) help this?
Thanx, Paul
> On Wed, Oct 22, 2014 at 10:12 AM, Josh Boyer <jwboyer@fedoraproject.org> wrote:
> >
> > Someone else is seeing this when they try and modprobe ppp_generic:
> >
> > [ 240.599195] INFO: task kworker/u16:5:100 blocked for more than 120 seconds.
> > [ 240.599338] Not tainted 3.18.0-0.rc1.git2.1.fc22.x86_64 #1
> > [ 240.599446] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs"
> > disables this message.
> > [ 240.599583] kworker/u16:5 D ffff8802202db480 12400 100 2 0x00000000
> > [ 240.599744] Workqueue: netns cleanup_net
> > [ 240.599823] ffff8802202eb9e8 0000000000000096 ffff8802202db480
> > 00000000001d5f00
> > [ 240.600066] ffff8802202ebfd8 00000000001d5f00 ffff8800368c3480
> > ffff8802202db480
> > [ 240.600228] ffffffff81ee2690 7fffffffffffffff ffffffff81ee2698
> > ffffffff81ee2690
> > [ 240.600386] Call Trace:
> > [ 240.600445] [<ffffffff8185e239>] schedule+0x29/0x70
> > [ 240.600541] [<ffffffff8186345c>] schedule_timeout+0x26c/0x410
> > [ 240.600651] [<ffffffff81865ef7>] ? retint_restore_args+0x13/0x13
> > [ 240.600765] [<ffffffff818644e4>] ? _raw_spin_unlock_irq+0x34/0x50
> > [ 240.600879] [<ffffffff8185fc6c>] wait_for_completion+0x10c/0x150
> > [ 240.601025] [<ffffffff810e53e0>] ? wake_up_state+0x20/0x20
> > [ 240.601133] [<ffffffff8112a749>] _rcu_barrier+0x159/0x200
> > [ 240.601237] [<ffffffff8112a845>] rcu_barrier+0x15/0x20
> > [ 240.601335] [<ffffffff81718ebf>] netdev_run_todo+0x6f/0x310
> > [ 240.601442] [<ffffffff8170da85>] ? rollback_registered_many+0x265/0x2e0
> > [ 240.601564] [<ffffffff81725f2e>] rtnl_unlock+0xe/0x10
> > [ 240.601660] [<ffffffff8170f8e6>] default_device_exit_batch+0x156/0x180
> > [ 240.601781] [<ffffffff810fd8a0>] ? abort_exclusive_wait+0xb0/0xb0
> > [ 240.601895] [<ffffffff81707993>] ops_exit_list.isra.1+0x53/0x60
> > [ 240.602028] [<ffffffff81708540>] cleanup_net+0x100/0x1f0
> > [ 240.602131] [<ffffffff810ccfa8>] process_one_work+0x218/0x850
> > [ 240.602241] [<ffffffff810ccf0f>] ? process_one_work+0x17f/0x850
> > [ 240.602350] [<ffffffff810cd6c7>] ? worker_thread+0xe7/0x4a0
> > [ 240.602454] [<ffffffff810cd64b>] worker_thread+0x6b/0x4a0
> > [ 240.602555] [<ffffffff810cd5e0>] ? process_one_work+0x850/0x850
> > [ 240.602665] [<ffffffff810d399b>] kthread+0x10b/0x130
> > [ 240.602762] [<ffffffff81028cc9>] ? sched_clock+0x9/0x10
> > [ 240.602862] [<ffffffff810d3890>] ? kthread_create_on_node+0x250/0x250
> > [ 240.603004] [<ffffffff818651fc>] ret_from_fork+0x7c/0xb0
> > [ 240.603106] [<ffffffff810d3890>] ? kthread_create_on_node+0x250/0x250
> > [ 240.603224] 4 locks held by kworker/u16:5/100:
> > [ 240.603304] #0: ("%s""netns"){.+.+.+}, at: [<ffffffff810ccf0f>]
> > process_one_work+0x17f/0x850
> > [ 240.603495] #1: (net_cleanup_work){+.+.+.}, at:
> > [<ffffffff810ccf0f>] process_one_work+0x17f/0x850
> > [ 240.603691] #2: (net_mutex){+.+.+.}, at: [<ffffffff817084cc>]
> > cleanup_net+0x8c/0x1f0
> > [ 240.603869] #3: (rcu_sched_state.barrier_mutex){+.+...}, at:
> > [<ffffffff8112a625>] _rcu_barrier+0x35/0x200
> > [ 240.604211] INFO: task modprobe:1387 blocked for more than 120 seconds.
> > [ 240.604329] Not tainted 3.18.0-0.rc1.git2.1.fc22.x86_64 #1
> > [ 240.604434] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs"
> > disables this message.
> > [ 240.604570] modprobe D ffff8800cb4f1a40 13112 1387 1386 0x00000080
> > [ 240.604719] ffff8800cafbbbe8 0000000000000096 ffff8800cb4f1a40
> > 00000000001d5f00
> > [ 240.604878] ffff8800cafbbfd8 00000000001d5f00 ffff880223280000
> > ffff8800cb4f1a40
> > [ 240.605068] ffff8800cb4f1a40 ffffffff81f8fb48 0000000000000246
> > ffff8800cb4f1a40
> > [ 240.605228] Call Trace:
> > [ 240.605283] [<ffffffff8185e7e1>] schedule_preempt_disabled+0x31/0x80
> > [ 240.605400] [<ffffffff81860033>] mutex_lock_nested+0x183/0x440
> > [ 240.605510] [<ffffffff8170835f>] ? register_pernet_subsys+0x1f/0x50
> > [ 240.605626] [<ffffffff8170835f>] ? register_pernet_subsys+0x1f/0x50
> > [ 240.605757] [<ffffffffa0701000>] ? 0xffffffffa0701000
> > [ 240.605854] [<ffffffff8170835f>] register_pernet_subsys+0x1f/0x50
> > [ 240.606005] [<ffffffffa0701048>] br_init+0x48/0xd3 [bridge]
> > [ 240.606112] [<ffffffff81002148>] do_one_initcall+0xd8/0x210
> > [ 240.606224] [<ffffffff81153c02>] load_module+0x20c2/0x2870
> > [ 240.606327] [<ffffffff8114ebe0>] ? store_uevent+0x70/0x70
> > [ 240.606433] [<ffffffff8110ac26>] ? lock_release_non_nested+0x3c6/0x3d0
> > [ 240.606557] [<ffffffff81154497>] SyS_init_module+0xe7/0x140
> > [ 240.606664] [<ffffffff818652a9>] system_call_fastpath+0x12/0x17
> > [ 240.606773] 1 lock held by modprobe/1387:
> > [ 240.606845] #0: (net_mutex){+.+.+.}, at: [<ffffffff8170835f>]
> > register_pernet_subsys+0x1f/0x50
> > [ 240.607114] INFO: task modprobe:1466 blocked for more than 120 seconds.
> > [ 240.607231] Not tainted 3.18.0-0.rc1.git2.1.fc22.x86_64 #1
> > [ 240.607337] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs"
> > disables this message.
> > [ 240.607473] modprobe D ffff88020fbab480 13096 1466 1399 0x00000084
> > [ 240.607622] ffff88020d1bbbe8 0000000000000096 ffff88020fbab480
> > 00000000001d5f00
> > [ 240.607791] ffff88020d1bbfd8 00000000001d5f00 ffffffff81e1b580
> > ffff88020fbab480
> > [ 240.607949] ffff88020fbab480 ffffffff81f8fb48 0000000000000246
> > ffff88020fbab480
> > [ 240.608138] Call Trace:
> > [ 240.608193] [<ffffffff8185e7e1>] schedule_preempt_disabled+0x31/0x80
> > [ 240.608316] [<ffffffff81860033>] mutex_lock_nested+0x183/0x440
> > [ 240.608425] [<ffffffff817083ad>] ? register_pernet_device+0x1d/0x70
> > [ 240.608542] [<ffffffff817083ad>] ? register_pernet_device+0x1d/0x70
> > [ 240.608662] [<ffffffffa071d000>] ? 0xffffffffa071d000
> > [ 240.608759] [<ffffffff817083ad>] register_pernet_device+0x1d/0x70
> > [ 240.608881] [<ffffffffa071d020>] ppp_init+0x20/0x1000 [ppp_generic]
> > [ 240.609021] [<ffffffff81002148>] do_one_initcall+0xd8/0x210
> > [ 240.609131] [<ffffffff81153c02>] load_module+0x20c2/0x2870
> > [ 240.609235] [<ffffffff8114ebe0>] ? store_uevent+0x70/0x70
> > [ 240.609339] [<ffffffff8110ac26>] ? lock_release_non_nested+0x3c6/0x3d0
> > [ 240.609462] [<ffffffff81154497>] SyS_init_module+0xe7/0x140
> > [ 240.609568] [<ffffffff818652a9>] system_call_fastpath+0x12/0x17
> > [ 240.609677] 1 lock held by modprobe/1466:
> > [ 240.609749] #0: (net_mutex){+.+.+.}, at: [<ffffffff817083ad>]
> > register_pernet_device+0x1d/0x70
> >
> > Looks like contention on net_mutex or something, but I honestly have
> > no idea yet. I can't recreate it myself at the moment or I would
> > bisect.
> >
> > Has nobody else run into this with the pre-3.18 kernels? Fedora isn't
> > carrying any patches in this area.
> >
> > josh
> > --
> > To unsubscribe from this list: send the line "unsubscribe netdev" in
> > the body of a message to majordomo@vger.kernel.org
> > More majordomo info at http://vger.kernel.org/majordomo-info.html
>
^ permalink raw reply
* Re: localed stuck in recent 3.18 git in copy_net_ns?
From: Eric W. Biederman @ 2014-10-22 17:53 UTC (permalink / raw)
To: Cong Wang
Cc: Josh Boyer, Kevin Fenzi, netdev, Linux-Kernel@Vger. Kernel. Org,
Paul McKenney
In-Reply-To: <CAHA+R7OUHy8XFPoip5gPvr1uqwkxgKxoSMf_pSgB1aFx=XCs8g@mail.gmail.com>
Cong Wang <cwang@twopensource.com> writes:
> (Adding Paul and Eric in Cc)
>
>
> On Wed, Oct 22, 2014 at 10:12 AM, Josh Boyer <jwboyer@fedoraproject.org> wrote:
>>
>> Someone else is seeing this when they try and modprobe ppp_generic:
>>
>> [ 240.599195] INFO: task kworker/u16:5:100 blocked for more than 120 seconds.
>> [ 240.599338] Not tainted 3.18.0-0.rc1.git2.1.fc22.x86_64 #1
>> [ 240.599446] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs"
>> disables this message.
>> [ 240.599583] kworker/u16:5 D ffff8802202db480 12400 100 2 0x00000000
>> [ 240.599744] Workqueue: netns cleanup_net
>> [ 240.599823] ffff8802202eb9e8 0000000000000096 ffff8802202db480
>> 00000000001d5f00
>> [ 240.600066] ffff8802202ebfd8 00000000001d5f00 ffff8800368c3480
>> ffff8802202db480
>> [ 240.600228] ffffffff81ee2690 7fffffffffffffff ffffffff81ee2698
>> ffffffff81ee2690
>> [ 240.600386] Call Trace:
>> [ 240.600445] [<ffffffff8185e239>] schedule+0x29/0x70
>> [ 240.600541] [<ffffffff8186345c>] schedule_timeout+0x26c/0x410
>> [ 240.600651] [<ffffffff81865ef7>] ? retint_restore_args+0x13/0x13
>> [ 240.600765] [<ffffffff818644e4>] ? _raw_spin_unlock_irq+0x34/0x50
>> [ 240.600879] [<ffffffff8185fc6c>] wait_for_completion+0x10c/0x150
>> [ 240.601025] [<ffffffff810e53e0>] ? wake_up_state+0x20/0x20
>> [ 240.601133] [<ffffffff8112a749>] _rcu_barrier+0x159/0x200
>> [ 240.601237] [<ffffffff8112a845>] rcu_barrier+0x15/0x20
>> [ 240.601335] [<ffffffff81718ebf>] netdev_run_todo+0x6f/0x310
>> [ 240.601442] [<ffffffff8170da85>] ? rollback_registered_many+0x265/0x2e0
>> [ 240.601564] [<ffffffff81725f2e>] rtnl_unlock+0xe/0x10
>> [ 240.601660] [<ffffffff8170f8e6>] default_device_exit_batch+0x156/0x180
>> [ 240.601781] [<ffffffff810fd8a0>] ? abort_exclusive_wait+0xb0/0xb0
>> [ 240.601895] [<ffffffff81707993>] ops_exit_list.isra.1+0x53/0x60
>> [ 240.602028] [<ffffffff81708540>] cleanup_net+0x100/0x1f0
>> [ 240.602131] [<ffffffff810ccfa8>] process_one_work+0x218/0x850
>> [ 240.602241] [<ffffffff810ccf0f>] ? process_one_work+0x17f/0x850
>> [ 240.602350] [<ffffffff810cd6c7>] ? worker_thread+0xe7/0x4a0
>> [ 240.602454] [<ffffffff810cd64b>] worker_thread+0x6b/0x4a0
>> [ 240.602555] [<ffffffff810cd5e0>] ? process_one_work+0x850/0x850
>> [ 240.602665] [<ffffffff810d399b>] kthread+0x10b/0x130
>> [ 240.602762] [<ffffffff81028cc9>] ? sched_clock+0x9/0x10
>> [ 240.602862] [<ffffffff810d3890>] ? kthread_create_on_node+0x250/0x250
>> [ 240.603004] [<ffffffff818651fc>] ret_from_fork+0x7c/0xb0
>> [ 240.603106] [<ffffffff810d3890>] ? kthread_create_on_node+0x250/0x250
>> [ 240.603224] 4 locks held by kworker/u16:5/100:
>> [ 240.603304] #0: ("%s""netns"){.+.+.+}, at: [<ffffffff810ccf0f>]
>> process_one_work+0x17f/0x850
>> [ 240.603495] #1: (net_cleanup_work){+.+.+.}, at:
>> [<ffffffff810ccf0f>] process_one_work+0x17f/0x850
>> [ 240.603691] #2: (net_mutex){+.+.+.}, at: [<ffffffff817084cc>]
>> cleanup_net+0x8c/0x1f0
>> [ 240.603869] #3: (rcu_sched_state.barrier_mutex){+.+...}, at:
>> [<ffffffff8112a625>] _rcu_barrier+0x35/0x200
>> [ 240.604211] INFO: task modprobe:1387 blocked for more than 120 seconds.
>> [ 240.604329] Not tainted 3.18.0-0.rc1.git2.1.fc22.x86_64 #1
>> [ 240.604434] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs"
>> disables this message.
>> [ 240.604570] modprobe D ffff8800cb4f1a40 13112 1387 1386 0x00000080
>> [ 240.604719] ffff8800cafbbbe8 0000000000000096 ffff8800cb4f1a40
>> 00000000001d5f00
>> [ 240.604878] ffff8800cafbbfd8 00000000001d5f00 ffff880223280000
>> ffff8800cb4f1a40
>> [ 240.605068] ffff8800cb4f1a40 ffffffff81f8fb48 0000000000000246
>> ffff8800cb4f1a40
>> [ 240.605228] Call Trace:
>> [ 240.605283] [<ffffffff8185e7e1>] schedule_preempt_disabled+0x31/0x80
>> [ 240.605400] [<ffffffff81860033>] mutex_lock_nested+0x183/0x440
>> [ 240.605510] [<ffffffff8170835f>] ? register_pernet_subsys+0x1f/0x50
>> [ 240.605626] [<ffffffff8170835f>] ? register_pernet_subsys+0x1f/0x50
>> [ 240.605757] [<ffffffffa0701000>] ? 0xffffffffa0701000
>> [ 240.605854] [<ffffffff8170835f>] register_pernet_subsys+0x1f/0x50
>> [ 240.606005] [<ffffffffa0701048>] br_init+0x48/0xd3 [bridge]
>> [ 240.606112] [<ffffffff81002148>] do_one_initcall+0xd8/0x210
>> [ 240.606224] [<ffffffff81153c02>] load_module+0x20c2/0x2870
>> [ 240.606327] [<ffffffff8114ebe0>] ? store_uevent+0x70/0x70
>> [ 240.606433] [<ffffffff8110ac26>] ? lock_release_non_nested+0x3c6/0x3d0
>> [ 240.606557] [<ffffffff81154497>] SyS_init_module+0xe7/0x140
>> [ 240.606664] [<ffffffff818652a9>] system_call_fastpath+0x12/0x17
>> [ 240.606773] 1 lock held by modprobe/1387:
>> [ 240.606845] #0: (net_mutex){+.+.+.}, at: [<ffffffff8170835f>]
>> register_pernet_subsys+0x1f/0x50
>> [ 240.607114] INFO: task modprobe:1466 blocked for more than 120 seconds.
>> [ 240.607231] Not tainted 3.18.0-0.rc1.git2.1.fc22.x86_64 #1
>> [ 240.607337] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs"
>> disables this message.
>> [ 240.607473] modprobe D ffff88020fbab480 13096 1466 1399 0x00000084
>> [ 240.607622] ffff88020d1bbbe8 0000000000000096 ffff88020fbab480
>> 00000000001d5f00
>> [ 240.607791] ffff88020d1bbfd8 00000000001d5f00 ffffffff81e1b580
>> ffff88020fbab480
>> [ 240.607949] ffff88020fbab480 ffffffff81f8fb48 0000000000000246
>> ffff88020fbab480
>> [ 240.608138] Call Trace:
>> [ 240.608193] [<ffffffff8185e7e1>] schedule_preempt_disabled+0x31/0x80
>> [ 240.608316] [<ffffffff81860033>] mutex_lock_nested+0x183/0x440
>> [ 240.608425] [<ffffffff817083ad>] ? register_pernet_device+0x1d/0x70
>> [ 240.608542] [<ffffffff817083ad>] ? register_pernet_device+0x1d/0x70
>> [ 240.608662] [<ffffffffa071d000>] ? 0xffffffffa071d000
>> [ 240.608759] [<ffffffff817083ad>] register_pernet_device+0x1d/0x70
>> [ 240.608881] [<ffffffffa071d020>] ppp_init+0x20/0x1000 [ppp_generic]
>> [ 240.609021] [<ffffffff81002148>] do_one_initcall+0xd8/0x210
>> [ 240.609131] [<ffffffff81153c02>] load_module+0x20c2/0x2870
>> [ 240.609235] [<ffffffff8114ebe0>] ? store_uevent+0x70/0x70
>> [ 240.609339] [<ffffffff8110ac26>] ? lock_release_non_nested+0x3c6/0x3d0
>> [ 240.609462] [<ffffffff81154497>] SyS_init_module+0xe7/0x140
>> [ 240.609568] [<ffffffff818652a9>] system_call_fastpath+0x12/0x17
>> [ 240.609677] 1 lock held by modprobe/1466:
>> [ 240.609749] #0: (net_mutex){+.+.+.}, at: [<ffffffff817083ad>]
>> register_pernet_device+0x1d/0x70
>>
>> Looks like contention on net_mutex or something, but I honestly have
>> no idea yet. I can't recreate it myself at the moment or I would
>> bisect.
>>
>> Has nobody else run into this with the pre-3.18 kernels? Fedora isn't
>> carrying any patches in this area.
> I am not aware of any change in net/core/dev.c related here,
> so I guess it's a bug in rcu_barrier().
>From the limited trace data I see in this email I have to agree.
It looks like for some reason rcu_barrier is taking forever
while the rtnl_lock is held in cleanup_net. Because the
rtnl_lock is held modprobe of the ppp driver is getting stuck.
Is it possible we have an AB BA deadlock between the rtnl_lock
and rcu. With something the module loading code assumes?
Eric
^ permalink raw reply
* Re: localed stuck in recent 3.18 git in copy_net_ns?
From: Josh Boyer @ 2014-10-22 17:49 UTC (permalink / raw)
To: Cong Wang
Cc: Kevin Fenzi, netdev, Linux-Kernel@Vger. Kernel. Org,
Paul McKenney, Eric W. Biederman
In-Reply-To: <CAHA+R7OUHy8XFPoip5gPvr1uqwkxgKxoSMf_pSgB1aFx=XCs8g@mail.gmail.com>
On Wed, Oct 22, 2014 at 1:37 PM, Cong Wang <cwang@twopensource.com> wrote:
> (Adding Paul and Eric in Cc)
>
> I am not aware of any change in net/core/dev.c related here,
> so I guess it's a bug in rcu_barrier().
Possibly. The person that reported the issue below said it showed up
between Linux v3.17-7872-g5ff0b9e1a1da and Linux
v3.17-8307-gf1d0d14120a8 for them. Which is a slightly older window
than the on that Kevin reported. I haven't had a chance to dig
through the commits yet.
josh
> On Wed, Oct 22, 2014 at 10:12 AM, Josh Boyer <jwboyer@fedoraproject.org> wrote:
>>
>> Someone else is seeing this when they try and modprobe ppp_generic:
>>
>> [ 240.599195] INFO: task kworker/u16:5:100 blocked for more than 120 seconds.
>> [ 240.599338] Not tainted 3.18.0-0.rc1.git2.1.fc22.x86_64 #1
>> [ 240.599446] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs"
>> disables this message.
>> [ 240.599583] kworker/u16:5 D ffff8802202db480 12400 100 2 0x00000000
>> [ 240.599744] Workqueue: netns cleanup_net
>> [ 240.599823] ffff8802202eb9e8 0000000000000096 ffff8802202db480
>> 00000000001d5f00
>> [ 240.600066] ffff8802202ebfd8 00000000001d5f00 ffff8800368c3480
>> ffff8802202db480
>> [ 240.600228] ffffffff81ee2690 7fffffffffffffff ffffffff81ee2698
>> ffffffff81ee2690
>> [ 240.600386] Call Trace:
>> [ 240.600445] [<ffffffff8185e239>] schedule+0x29/0x70
>> [ 240.600541] [<ffffffff8186345c>] schedule_timeout+0x26c/0x410
>> [ 240.600651] [<ffffffff81865ef7>] ? retint_restore_args+0x13/0x13
>> [ 240.600765] [<ffffffff818644e4>] ? _raw_spin_unlock_irq+0x34/0x50
>> [ 240.600879] [<ffffffff8185fc6c>] wait_for_completion+0x10c/0x150
>> [ 240.601025] [<ffffffff810e53e0>] ? wake_up_state+0x20/0x20
>> [ 240.601133] [<ffffffff8112a749>] _rcu_barrier+0x159/0x200
>> [ 240.601237] [<ffffffff8112a845>] rcu_barrier+0x15/0x20
>> [ 240.601335] [<ffffffff81718ebf>] netdev_run_todo+0x6f/0x310
>> [ 240.601442] [<ffffffff8170da85>] ? rollback_registered_many+0x265/0x2e0
>> [ 240.601564] [<ffffffff81725f2e>] rtnl_unlock+0xe/0x10
>> [ 240.601660] [<ffffffff8170f8e6>] default_device_exit_batch+0x156/0x180
>> [ 240.601781] [<ffffffff810fd8a0>] ? abort_exclusive_wait+0xb0/0xb0
>> [ 240.601895] [<ffffffff81707993>] ops_exit_list.isra.1+0x53/0x60
>> [ 240.602028] [<ffffffff81708540>] cleanup_net+0x100/0x1f0
>> [ 240.602131] [<ffffffff810ccfa8>] process_one_work+0x218/0x850
>> [ 240.602241] [<ffffffff810ccf0f>] ? process_one_work+0x17f/0x850
>> [ 240.602350] [<ffffffff810cd6c7>] ? worker_thread+0xe7/0x4a0
>> [ 240.602454] [<ffffffff810cd64b>] worker_thread+0x6b/0x4a0
>> [ 240.602555] [<ffffffff810cd5e0>] ? process_one_work+0x850/0x850
>> [ 240.602665] [<ffffffff810d399b>] kthread+0x10b/0x130
>> [ 240.602762] [<ffffffff81028cc9>] ? sched_clock+0x9/0x10
>> [ 240.602862] [<ffffffff810d3890>] ? kthread_create_on_node+0x250/0x250
>> [ 240.603004] [<ffffffff818651fc>] ret_from_fork+0x7c/0xb0
>> [ 240.603106] [<ffffffff810d3890>] ? kthread_create_on_node+0x250/0x250
>> [ 240.603224] 4 locks held by kworker/u16:5/100:
>> [ 240.603304] #0: ("%s""netns"){.+.+.+}, at: [<ffffffff810ccf0f>]
>> process_one_work+0x17f/0x850
>> [ 240.603495] #1: (net_cleanup_work){+.+.+.}, at:
>> [<ffffffff810ccf0f>] process_one_work+0x17f/0x850
>> [ 240.603691] #2: (net_mutex){+.+.+.}, at: [<ffffffff817084cc>]
>> cleanup_net+0x8c/0x1f0
>> [ 240.603869] #3: (rcu_sched_state.barrier_mutex){+.+...}, at:
>> [<ffffffff8112a625>] _rcu_barrier+0x35/0x200
>> [ 240.604211] INFO: task modprobe:1387 blocked for more than 120 seconds.
>> [ 240.604329] Not tainted 3.18.0-0.rc1.git2.1.fc22.x86_64 #1
>> [ 240.604434] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs"
>> disables this message.
>> [ 240.604570] modprobe D ffff8800cb4f1a40 13112 1387 1386 0x00000080
>> [ 240.604719] ffff8800cafbbbe8 0000000000000096 ffff8800cb4f1a40
>> 00000000001d5f00
>> [ 240.604878] ffff8800cafbbfd8 00000000001d5f00 ffff880223280000
>> ffff8800cb4f1a40
>> [ 240.605068] ffff8800cb4f1a40 ffffffff81f8fb48 0000000000000246
>> ffff8800cb4f1a40
>> [ 240.605228] Call Trace:
>> [ 240.605283] [<ffffffff8185e7e1>] schedule_preempt_disabled+0x31/0x80
>> [ 240.605400] [<ffffffff81860033>] mutex_lock_nested+0x183/0x440
>> [ 240.605510] [<ffffffff8170835f>] ? register_pernet_subsys+0x1f/0x50
>> [ 240.605626] [<ffffffff8170835f>] ? register_pernet_subsys+0x1f/0x50
>> [ 240.605757] [<ffffffffa0701000>] ? 0xffffffffa0701000
>> [ 240.605854] [<ffffffff8170835f>] register_pernet_subsys+0x1f/0x50
>> [ 240.606005] [<ffffffffa0701048>] br_init+0x48/0xd3 [bridge]
>> [ 240.606112] [<ffffffff81002148>] do_one_initcall+0xd8/0x210
>> [ 240.606224] [<ffffffff81153c02>] load_module+0x20c2/0x2870
>> [ 240.606327] [<ffffffff8114ebe0>] ? store_uevent+0x70/0x70
>> [ 240.606433] [<ffffffff8110ac26>] ? lock_release_non_nested+0x3c6/0x3d0
>> [ 240.606557] [<ffffffff81154497>] SyS_init_module+0xe7/0x140
>> [ 240.606664] [<ffffffff818652a9>] system_call_fastpath+0x12/0x17
>> [ 240.606773] 1 lock held by modprobe/1387:
>> [ 240.606845] #0: (net_mutex){+.+.+.}, at: [<ffffffff8170835f>]
>> register_pernet_subsys+0x1f/0x50
>> [ 240.607114] INFO: task modprobe:1466 blocked for more than 120 seconds.
>> [ 240.607231] Not tainted 3.18.0-0.rc1.git2.1.fc22.x86_64 #1
>> [ 240.607337] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs"
>> disables this message.
>> [ 240.607473] modprobe D ffff88020fbab480 13096 1466 1399 0x00000084
>> [ 240.607622] ffff88020d1bbbe8 0000000000000096 ffff88020fbab480
>> 00000000001d5f00
>> [ 240.607791] ffff88020d1bbfd8 00000000001d5f00 ffffffff81e1b580
>> ffff88020fbab480
>> [ 240.607949] ffff88020fbab480 ffffffff81f8fb48 0000000000000246
>> ffff88020fbab480
>> [ 240.608138] Call Trace:
>> [ 240.608193] [<ffffffff8185e7e1>] schedule_preempt_disabled+0x31/0x80
>> [ 240.608316] [<ffffffff81860033>] mutex_lock_nested+0x183/0x440
>> [ 240.608425] [<ffffffff817083ad>] ? register_pernet_device+0x1d/0x70
>> [ 240.608542] [<ffffffff817083ad>] ? register_pernet_device+0x1d/0x70
>> [ 240.608662] [<ffffffffa071d000>] ? 0xffffffffa071d000
>> [ 240.608759] [<ffffffff817083ad>] register_pernet_device+0x1d/0x70
>> [ 240.608881] [<ffffffffa071d020>] ppp_init+0x20/0x1000 [ppp_generic]
>> [ 240.609021] [<ffffffff81002148>] do_one_initcall+0xd8/0x210
>> [ 240.609131] [<ffffffff81153c02>] load_module+0x20c2/0x2870
>> [ 240.609235] [<ffffffff8114ebe0>] ? store_uevent+0x70/0x70
>> [ 240.609339] [<ffffffff8110ac26>] ? lock_release_non_nested+0x3c6/0x3d0
>> [ 240.609462] [<ffffffff81154497>] SyS_init_module+0xe7/0x140
>> [ 240.609568] [<ffffffff818652a9>] system_call_fastpath+0x12/0x17
>> [ 240.609677] 1 lock held by modprobe/1466:
>> [ 240.609749] #0: (net_mutex){+.+.+.}, at: [<ffffffff817083ad>]
>> register_pernet_device+0x1d/0x70
>>
>> Looks like contention on net_mutex or something, but I honestly have
>> no idea yet. I can't recreate it myself at the moment or I would
>> bisect.
>>
>> Has nobody else run into this with the pre-3.18 kernels? Fedora isn't
>> carrying any patches in this area.
>>
>> josh
>> --
>> To unsubscribe from this list: send the line "unsubscribe netdev" in
>> the body of a message to majordomo@vger.kernel.org
>> More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* Re: [PATCH] net: fix saving TX flow hash in sock for outgoing connections
From: Eric Dumazet @ 2014-10-22 17:39 UTC (permalink / raw)
To: Sathya Perla; +Cc: netdev, therbert
In-Reply-To: <1413994321-20435-1-git-send-email-sathya.perla@emulex.com>
On Wed, 2014-10-22 at 21:42 +0530, Sathya Perla wrote:
> The commit "net: Save TX flow hash in sock and set in skbuf on xmit"
> introduced the inet_set_txhash() and ip6_set_txhash() routines to calculate
> and record flow hash(sk_txhash) in the socket structure. sk_txhash is used
> to set skb->hash which is used to spread flows across multiple TXQs.
>
> But, the above routines are invoked before the source port of the connection
> is created. Because of this all outgoing connections that just differ in the
> source port get hashed into the same TXQ.
Acked-by: Eric Dumazet <edumazet@google.com>
Are you really using the socket/flow hash to select a TXQ ?
Even with this patch, you have a good probability of multiple
cpus hitting same TXQ.
^ permalink raw reply
* Re: localed stuck in recent 3.18 git in copy_net_ns?
From: Cong Wang @ 2014-10-22 17:37 UTC (permalink / raw)
To: Josh Boyer
Cc: Kevin Fenzi, netdev, Linux-Kernel@Vger. Kernel. Org,
Paul McKenney, Eric W. Biederman
In-Reply-To: <CA+5PVA7Ro_ejBUqsZ9StWVeu59==fGnj6e4Gx8zM4_3+Lq5s4A@mail.gmail.com>
(Adding Paul and Eric in Cc)
I am not aware of any change in net/core/dev.c related here,
so I guess it's a bug in rcu_barrier().
Thanks.
On Wed, Oct 22, 2014 at 10:12 AM, Josh Boyer <jwboyer@fedoraproject.org> wrote:
>
> Someone else is seeing this when they try and modprobe ppp_generic:
>
> [ 240.599195] INFO: task kworker/u16:5:100 blocked for more than 120 seconds.
> [ 240.599338] Not tainted 3.18.0-0.rc1.git2.1.fc22.x86_64 #1
> [ 240.599446] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs"
> disables this message.
> [ 240.599583] kworker/u16:5 D ffff8802202db480 12400 100 2 0x00000000
> [ 240.599744] Workqueue: netns cleanup_net
> [ 240.599823] ffff8802202eb9e8 0000000000000096 ffff8802202db480
> 00000000001d5f00
> [ 240.600066] ffff8802202ebfd8 00000000001d5f00 ffff8800368c3480
> ffff8802202db480
> [ 240.600228] ffffffff81ee2690 7fffffffffffffff ffffffff81ee2698
> ffffffff81ee2690
> [ 240.600386] Call Trace:
> [ 240.600445] [<ffffffff8185e239>] schedule+0x29/0x70
> [ 240.600541] [<ffffffff8186345c>] schedule_timeout+0x26c/0x410
> [ 240.600651] [<ffffffff81865ef7>] ? retint_restore_args+0x13/0x13
> [ 240.600765] [<ffffffff818644e4>] ? _raw_spin_unlock_irq+0x34/0x50
> [ 240.600879] [<ffffffff8185fc6c>] wait_for_completion+0x10c/0x150
> [ 240.601025] [<ffffffff810e53e0>] ? wake_up_state+0x20/0x20
> [ 240.601133] [<ffffffff8112a749>] _rcu_barrier+0x159/0x200
> [ 240.601237] [<ffffffff8112a845>] rcu_barrier+0x15/0x20
> [ 240.601335] [<ffffffff81718ebf>] netdev_run_todo+0x6f/0x310
> [ 240.601442] [<ffffffff8170da85>] ? rollback_registered_many+0x265/0x2e0
> [ 240.601564] [<ffffffff81725f2e>] rtnl_unlock+0xe/0x10
> [ 240.601660] [<ffffffff8170f8e6>] default_device_exit_batch+0x156/0x180
> [ 240.601781] [<ffffffff810fd8a0>] ? abort_exclusive_wait+0xb0/0xb0
> [ 240.601895] [<ffffffff81707993>] ops_exit_list.isra.1+0x53/0x60
> [ 240.602028] [<ffffffff81708540>] cleanup_net+0x100/0x1f0
> [ 240.602131] [<ffffffff810ccfa8>] process_one_work+0x218/0x850
> [ 240.602241] [<ffffffff810ccf0f>] ? process_one_work+0x17f/0x850
> [ 240.602350] [<ffffffff810cd6c7>] ? worker_thread+0xe7/0x4a0
> [ 240.602454] [<ffffffff810cd64b>] worker_thread+0x6b/0x4a0
> [ 240.602555] [<ffffffff810cd5e0>] ? process_one_work+0x850/0x850
> [ 240.602665] [<ffffffff810d399b>] kthread+0x10b/0x130
> [ 240.602762] [<ffffffff81028cc9>] ? sched_clock+0x9/0x10
> [ 240.602862] [<ffffffff810d3890>] ? kthread_create_on_node+0x250/0x250
> [ 240.603004] [<ffffffff818651fc>] ret_from_fork+0x7c/0xb0
> [ 240.603106] [<ffffffff810d3890>] ? kthread_create_on_node+0x250/0x250
> [ 240.603224] 4 locks held by kworker/u16:5/100:
> [ 240.603304] #0: ("%s""netns"){.+.+.+}, at: [<ffffffff810ccf0f>]
> process_one_work+0x17f/0x850
> [ 240.603495] #1: (net_cleanup_work){+.+.+.}, at:
> [<ffffffff810ccf0f>] process_one_work+0x17f/0x850
> [ 240.603691] #2: (net_mutex){+.+.+.}, at: [<ffffffff817084cc>]
> cleanup_net+0x8c/0x1f0
> [ 240.603869] #3: (rcu_sched_state.barrier_mutex){+.+...}, at:
> [<ffffffff8112a625>] _rcu_barrier+0x35/0x200
> [ 240.604211] INFO: task modprobe:1387 blocked for more than 120 seconds.
> [ 240.604329] Not tainted 3.18.0-0.rc1.git2.1.fc22.x86_64 #1
> [ 240.604434] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs"
> disables this message.
> [ 240.604570] modprobe D ffff8800cb4f1a40 13112 1387 1386 0x00000080
> [ 240.604719] ffff8800cafbbbe8 0000000000000096 ffff8800cb4f1a40
> 00000000001d5f00
> [ 240.604878] ffff8800cafbbfd8 00000000001d5f00 ffff880223280000
> ffff8800cb4f1a40
> [ 240.605068] ffff8800cb4f1a40 ffffffff81f8fb48 0000000000000246
> ffff8800cb4f1a40
> [ 240.605228] Call Trace:
> [ 240.605283] [<ffffffff8185e7e1>] schedule_preempt_disabled+0x31/0x80
> [ 240.605400] [<ffffffff81860033>] mutex_lock_nested+0x183/0x440
> [ 240.605510] [<ffffffff8170835f>] ? register_pernet_subsys+0x1f/0x50
> [ 240.605626] [<ffffffff8170835f>] ? register_pernet_subsys+0x1f/0x50
> [ 240.605757] [<ffffffffa0701000>] ? 0xffffffffa0701000
> [ 240.605854] [<ffffffff8170835f>] register_pernet_subsys+0x1f/0x50
> [ 240.606005] [<ffffffffa0701048>] br_init+0x48/0xd3 [bridge]
> [ 240.606112] [<ffffffff81002148>] do_one_initcall+0xd8/0x210
> [ 240.606224] [<ffffffff81153c02>] load_module+0x20c2/0x2870
> [ 240.606327] [<ffffffff8114ebe0>] ? store_uevent+0x70/0x70
> [ 240.606433] [<ffffffff8110ac26>] ? lock_release_non_nested+0x3c6/0x3d0
> [ 240.606557] [<ffffffff81154497>] SyS_init_module+0xe7/0x140
> [ 240.606664] [<ffffffff818652a9>] system_call_fastpath+0x12/0x17
> [ 240.606773] 1 lock held by modprobe/1387:
> [ 240.606845] #0: (net_mutex){+.+.+.}, at: [<ffffffff8170835f>]
> register_pernet_subsys+0x1f/0x50
> [ 240.607114] INFO: task modprobe:1466 blocked for more than 120 seconds.
> [ 240.607231] Not tainted 3.18.0-0.rc1.git2.1.fc22.x86_64 #1
> [ 240.607337] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs"
> disables this message.
> [ 240.607473] modprobe D ffff88020fbab480 13096 1466 1399 0x00000084
> [ 240.607622] ffff88020d1bbbe8 0000000000000096 ffff88020fbab480
> 00000000001d5f00
> [ 240.607791] ffff88020d1bbfd8 00000000001d5f00 ffffffff81e1b580
> ffff88020fbab480
> [ 240.607949] ffff88020fbab480 ffffffff81f8fb48 0000000000000246
> ffff88020fbab480
> [ 240.608138] Call Trace:
> [ 240.608193] [<ffffffff8185e7e1>] schedule_preempt_disabled+0x31/0x80
> [ 240.608316] [<ffffffff81860033>] mutex_lock_nested+0x183/0x440
> [ 240.608425] [<ffffffff817083ad>] ? register_pernet_device+0x1d/0x70
> [ 240.608542] [<ffffffff817083ad>] ? register_pernet_device+0x1d/0x70
> [ 240.608662] [<ffffffffa071d000>] ? 0xffffffffa071d000
> [ 240.608759] [<ffffffff817083ad>] register_pernet_device+0x1d/0x70
> [ 240.608881] [<ffffffffa071d020>] ppp_init+0x20/0x1000 [ppp_generic]
> [ 240.609021] [<ffffffff81002148>] do_one_initcall+0xd8/0x210
> [ 240.609131] [<ffffffff81153c02>] load_module+0x20c2/0x2870
> [ 240.609235] [<ffffffff8114ebe0>] ? store_uevent+0x70/0x70
> [ 240.609339] [<ffffffff8110ac26>] ? lock_release_non_nested+0x3c6/0x3d0
> [ 240.609462] [<ffffffff81154497>] SyS_init_module+0xe7/0x140
> [ 240.609568] [<ffffffff818652a9>] system_call_fastpath+0x12/0x17
> [ 240.609677] 1 lock held by modprobe/1466:
> [ 240.609749] #0: (net_mutex){+.+.+.}, at: [<ffffffff817083ad>]
> register_pernet_device+0x1d/0x70
>
> Looks like contention on net_mutex or something, but I honestly have
> no idea yet. I can't recreate it myself at the moment or I would
> bisect.
>
> Has nobody else run into this with the pre-3.18 kernels? Fedora isn't
> carrying any patches in this area.
>
> josh
> --
> To unsubscribe from this list: send the line "unsubscribe netdev" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* Re: localed stuck in recent 3.18 git in copy_net_ns?
From: Josh Boyer @ 2014-10-22 17:12 UTC (permalink / raw)
To: Kevin Fenzi; +Cc: netdev, Linux-Kernel@Vger. Kernel. Org
In-Reply-To: <20141021151225.5df96645@voldemort.scrye.com>
On Tue, Oct 21, 2014 at 5:12 PM, Kevin Fenzi <kevin@scrye.com> wrote:
> On Mon, 20 Oct 2014 14:53:59 -0600
> Kevin Fenzi <kevin@scrye.com> wrote:
>
>> On Mon, 20 Oct 2014 16:43:26 -0400
>> Dave Jones <davej@redhat.com> wrote:
>>
>> > I've seen similar soft lockup traces from the sys_unshare path when
>> > running my fuzz tester. It seems that if you create enough network
>> > namespaces, it can take a huge amount of time for them to be
>> > iterated. (Running trinity with '-c unshare' you can see the slow
>> > down happen. In some cases, it takes so long that the watchdog
>> > process kills it -- though the SIGKILL won't get delivered until
>> > the unshare() completes)
>> >
>> > Any idea what this machine had been doing prior to this that may
>> > have involved creating lots of namespaces ?
>>
>> That was right after boot. ;)
>>
>> This is my main rawhide running laptop.
>>
>> A 'ip netns list' shows nothing.
>
> Some more information:
>
> The problem started between:
>
> v3.17-7872-g5ff0b9e1a1da and v3.17-8307-gf1d0d14120a8
>
> (I can try and do a bisect, but have to head out on a trip tomorrow)
>
> In all the kernels with the problem, there is a kworker process in D.
>
> sysrq-t says:
> Showing all locks held in the system:
> Oct 21 15:06:31 voldemort.scrye.com kernel: 4 locks held by kworker/u16:0/6:
> Oct 21 15:06:31 voldemort.scrye.com kernel: #0: ("%s""netns"){.+.+.+}, at: [<ffffffff810ccbff>] process_one_work+0x17f/0x850
> Oct 21 15:06:31 voldemort.scrye.com kernel: #1: (net_cleanup_work){+.+.+.}, at: [<ffffffff810ccbff>] process_one_work+0x17f/0x850
> Oct 21 15:06:31 voldemort.scrye.com kernel: #2: (net_mutex){+.+.+.}, at: [<ffffffff817069fc>] cleanup_net+0x8c/0x1f0
> Oct 21 15:06:31 voldemort.scrye.com kernel: #3:
> (rcu_sched_state.barrier_mutex){+.+...}, at: [<ffffffff8112a395>]
> _rcu_barrier+0x35/0x200
>
> On first running any of the systemd units that use PrivateNetwork, then
> run ok, but they are also set to timeout after a minute. On sucessive
> runs they hang in D also.
Someone else is seeing this when they try and modprobe ppp_generic:
[ 240.599195] INFO: task kworker/u16:5:100 blocked for more than 120 seconds.
[ 240.599338] Not tainted 3.18.0-0.rc1.git2.1.fc22.x86_64 #1
[ 240.599446] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs"
disables this message.
[ 240.599583] kworker/u16:5 D ffff8802202db480 12400 100 2 0x00000000
[ 240.599744] Workqueue: netns cleanup_net
[ 240.599823] ffff8802202eb9e8 0000000000000096 ffff8802202db480
00000000001d5f00
[ 240.600066] ffff8802202ebfd8 00000000001d5f00 ffff8800368c3480
ffff8802202db480
[ 240.600228] ffffffff81ee2690 7fffffffffffffff ffffffff81ee2698
ffffffff81ee2690
[ 240.600386] Call Trace:
[ 240.600445] [<ffffffff8185e239>] schedule+0x29/0x70
[ 240.600541] [<ffffffff8186345c>] schedule_timeout+0x26c/0x410
[ 240.600651] [<ffffffff81865ef7>] ? retint_restore_args+0x13/0x13
[ 240.600765] [<ffffffff818644e4>] ? _raw_spin_unlock_irq+0x34/0x50
[ 240.600879] [<ffffffff8185fc6c>] wait_for_completion+0x10c/0x150
[ 240.601025] [<ffffffff810e53e0>] ? wake_up_state+0x20/0x20
[ 240.601133] [<ffffffff8112a749>] _rcu_barrier+0x159/0x200
[ 240.601237] [<ffffffff8112a845>] rcu_barrier+0x15/0x20
[ 240.601335] [<ffffffff81718ebf>] netdev_run_todo+0x6f/0x310
[ 240.601442] [<ffffffff8170da85>] ? rollback_registered_many+0x265/0x2e0
[ 240.601564] [<ffffffff81725f2e>] rtnl_unlock+0xe/0x10
[ 240.601660] [<ffffffff8170f8e6>] default_device_exit_batch+0x156/0x180
[ 240.601781] [<ffffffff810fd8a0>] ? abort_exclusive_wait+0xb0/0xb0
[ 240.601895] [<ffffffff81707993>] ops_exit_list.isra.1+0x53/0x60
[ 240.602028] [<ffffffff81708540>] cleanup_net+0x100/0x1f0
[ 240.602131] [<ffffffff810ccfa8>] process_one_work+0x218/0x850
[ 240.602241] [<ffffffff810ccf0f>] ? process_one_work+0x17f/0x850
[ 240.602350] [<ffffffff810cd6c7>] ? worker_thread+0xe7/0x4a0
[ 240.602454] [<ffffffff810cd64b>] worker_thread+0x6b/0x4a0
[ 240.602555] [<ffffffff810cd5e0>] ? process_one_work+0x850/0x850
[ 240.602665] [<ffffffff810d399b>] kthread+0x10b/0x130
[ 240.602762] [<ffffffff81028cc9>] ? sched_clock+0x9/0x10
[ 240.602862] [<ffffffff810d3890>] ? kthread_create_on_node+0x250/0x250
[ 240.603004] [<ffffffff818651fc>] ret_from_fork+0x7c/0xb0
[ 240.603106] [<ffffffff810d3890>] ? kthread_create_on_node+0x250/0x250
[ 240.603224] 4 locks held by kworker/u16:5/100:
[ 240.603304] #0: ("%s""netns"){.+.+.+}, at: [<ffffffff810ccf0f>]
process_one_work+0x17f/0x850
[ 240.603495] #1: (net_cleanup_work){+.+.+.}, at:
[<ffffffff810ccf0f>] process_one_work+0x17f/0x850
[ 240.603691] #2: (net_mutex){+.+.+.}, at: [<ffffffff817084cc>]
cleanup_net+0x8c/0x1f0
[ 240.603869] #3: (rcu_sched_state.barrier_mutex){+.+...}, at:
[<ffffffff8112a625>] _rcu_barrier+0x35/0x200
[ 240.604211] INFO: task modprobe:1387 blocked for more than 120 seconds.
[ 240.604329] Not tainted 3.18.0-0.rc1.git2.1.fc22.x86_64 #1
[ 240.604434] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs"
disables this message.
[ 240.604570] modprobe D ffff8800cb4f1a40 13112 1387 1386 0x00000080
[ 240.604719] ffff8800cafbbbe8 0000000000000096 ffff8800cb4f1a40
00000000001d5f00
[ 240.604878] ffff8800cafbbfd8 00000000001d5f00 ffff880223280000
ffff8800cb4f1a40
[ 240.605068] ffff8800cb4f1a40 ffffffff81f8fb48 0000000000000246
ffff8800cb4f1a40
[ 240.605228] Call Trace:
[ 240.605283] [<ffffffff8185e7e1>] schedule_preempt_disabled+0x31/0x80
[ 240.605400] [<ffffffff81860033>] mutex_lock_nested+0x183/0x440
[ 240.605510] [<ffffffff8170835f>] ? register_pernet_subsys+0x1f/0x50
[ 240.605626] [<ffffffff8170835f>] ? register_pernet_subsys+0x1f/0x50
[ 240.605757] [<ffffffffa0701000>] ? 0xffffffffa0701000
[ 240.605854] [<ffffffff8170835f>] register_pernet_subsys+0x1f/0x50
[ 240.606005] [<ffffffffa0701048>] br_init+0x48/0xd3 [bridge]
[ 240.606112] [<ffffffff81002148>] do_one_initcall+0xd8/0x210
[ 240.606224] [<ffffffff81153c02>] load_module+0x20c2/0x2870
[ 240.606327] [<ffffffff8114ebe0>] ? store_uevent+0x70/0x70
[ 240.606433] [<ffffffff8110ac26>] ? lock_release_non_nested+0x3c6/0x3d0
[ 240.606557] [<ffffffff81154497>] SyS_init_module+0xe7/0x140
[ 240.606664] [<ffffffff818652a9>] system_call_fastpath+0x12/0x17
[ 240.606773] 1 lock held by modprobe/1387:
[ 240.606845] #0: (net_mutex){+.+.+.}, at: [<ffffffff8170835f>]
register_pernet_subsys+0x1f/0x50
[ 240.607114] INFO: task modprobe:1466 blocked for more than 120 seconds.
[ 240.607231] Not tainted 3.18.0-0.rc1.git2.1.fc22.x86_64 #1
[ 240.607337] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs"
disables this message.
[ 240.607473] modprobe D ffff88020fbab480 13096 1466 1399 0x00000084
[ 240.607622] ffff88020d1bbbe8 0000000000000096 ffff88020fbab480
00000000001d5f00
[ 240.607791] ffff88020d1bbfd8 00000000001d5f00 ffffffff81e1b580
ffff88020fbab480
[ 240.607949] ffff88020fbab480 ffffffff81f8fb48 0000000000000246
ffff88020fbab480
[ 240.608138] Call Trace:
[ 240.608193] [<ffffffff8185e7e1>] schedule_preempt_disabled+0x31/0x80
[ 240.608316] [<ffffffff81860033>] mutex_lock_nested+0x183/0x440
[ 240.608425] [<ffffffff817083ad>] ? register_pernet_device+0x1d/0x70
[ 240.608542] [<ffffffff817083ad>] ? register_pernet_device+0x1d/0x70
[ 240.608662] [<ffffffffa071d000>] ? 0xffffffffa071d000
[ 240.608759] [<ffffffff817083ad>] register_pernet_device+0x1d/0x70
[ 240.608881] [<ffffffffa071d020>] ppp_init+0x20/0x1000 [ppp_generic]
[ 240.609021] [<ffffffff81002148>] do_one_initcall+0xd8/0x210
[ 240.609131] [<ffffffff81153c02>] load_module+0x20c2/0x2870
[ 240.609235] [<ffffffff8114ebe0>] ? store_uevent+0x70/0x70
[ 240.609339] [<ffffffff8110ac26>] ? lock_release_non_nested+0x3c6/0x3d0
[ 240.609462] [<ffffffff81154497>] SyS_init_module+0xe7/0x140
[ 240.609568] [<ffffffff818652a9>] system_call_fastpath+0x12/0x17
[ 240.609677] 1 lock held by modprobe/1466:
[ 240.609749] #0: (net_mutex){+.+.+.}, at: [<ffffffff817083ad>]
register_pernet_device+0x1d/0x70
Looks like contention on net_mutex or something, but I honestly have
no idea yet. I can't recreate it myself at the moment or I would
bisect.
Has nobody else run into this with the pre-3.18 kernels? Fedora isn't
carrying any patches in this area.
josh
^ permalink raw reply
* Re: [PATCH net-next] tcp: Add TCP_FREEZE socket option
From: Kristian Evensen @ 2014-10-22 17:11 UTC (permalink / raw)
To: Cong Wang; +Cc: netdev
In-Reply-To: <CAHA+R7OTmmeUf=_LaN1_isL1ZKiqcS0pYE0eKpRbPhaPeX-2jQ@mail.gmail.com>
Hi,
On Wed, Oct 22, 2014 at 6:56 PM, Cong Wang <cwang@twopensource.com> wrote:
> At least split TCP is transparent to applications, while your approach is not.
> I don't understand why you said it typically operates on some ports, since
> TCP is stateful.
I see that I might have used the wrong word here. I am use to calling
them TCP splitters, but I see that the devices are also referred to as
transparent TCP proxies. Anyhow, they are still transparent, but they
violate end-to-end (even though I guess that is pretty common
now-a-days).
What I mean by the port-comment is that only connections to some ports
are proxied/split. For example, one of the operators in Norway only
proxy port 80, so any HTTPS transfer risk getting stuck after a
temporary disconnect.
-Kristian
^ permalink raw reply
* Re: [PATCH net-next] tcp: Add TCP_FREEZE socket option
From: Kristian Evensen @ 2014-10-22 17:08 UTC (permalink / raw)
To: David Miller; +Cc: Network Development
In-Reply-To: <20141022.121418.1477947755244353588.davem@davemloft.net>
Hi,
On Wed, Oct 22, 2014 at 6:14 PM, David Miller <davem@davemloft.net> wrote:
> Instead, I would expect the device layer to trigger a notification
> during a "technology change" or whatever you want to call losing
> connectivity, whichi TCP can receive and use to start sending zero
> windows over all TCP connections using that path.
I totally agree that this is ideally something that should be
controlled by the device layer. However, these temporary disconnects
are not visible through any normal link events (like link down, loss
of address, ...). The only way to detect the events is to parse meta
data coming from devices and look at traffic statistics. This would
involve for example adding parsing of the different mobile broadband
protocols (QMI, MBIM, and so on) to the device layer. When looking at
for example the commits for the QMI driver, parsing QMI messages seems
to have intentionally been left up to user space applications to avoid
bloating driver.
> And therefore there should be a global option that turns this on for
> the entire system by default.
>
> This requires a lot more work than you have done here, you need to
> add all the notification handling, the logic in TCP to look at the
> attached route on send and trigger zero window probes if the device
> event has happened, etc.
Another approach I designed was to have a separate TCP Freeze module
and trigger the freeze/unfreeze through genetlink-messages. A user
space application will be responsible for monitoring the devices and
decide when to trigger the ZWAs. Would a design like that be
acceptable?
-Kristian
^ permalink raw reply
* Re: [PATCH net-next] tcp: Add TCP_FREEZE socket option
From: Cong Wang @ 2014-10-22 16:56 UTC (permalink / raw)
To: Kristian Evensen; +Cc: netdev
In-Reply-To: <1413992196-4891-1-git-send-email-kristian.evensen@gmail.com>
On Wed, Oct 22, 2014 at 8:36 AM, Kristian Evensen
<kristian.evensen@gmail.com> wrote:
> From: Kristian Evensen <kristian.evensen@gmail.com>
>
> This patch introduces support for Freeze-TCP [1].
>
> Devices that are mobile frequently experience temporary disconnects, for example
> due to signal fading or a technology change. These changes can last for a
> substantial amount of time (>10 seconds), potentially causing multiple RTOs to
> expire and the sender to enter slow start. Even though a device has
> reconnected, it can take a long time for the TCP connection to recover.
>
> Operators of mobile broadband networks mitigate this issue by placing TCP
> splitters at the edge of their networks. However, the splitters typically only
> operate on some ports (mostly only port 80) and violate the end-to-end
> principle. The operator's TCP splitter receives a notification when a temporary
> disconnect occurs and starts sending Zero Window Announcements (ZWA) to the
> remote part of the connection. When a devices regains connectivity, the window
> is reopened.
At least split TCP is transparent to applications, while your approach is not.
I don't understand why you said it typically operates on some ports, since
TCP is stateful.
BTW, AFAIK Linux doesn't support split TCP.
^ permalink raw reply
* Re: [PATCH] net: tso: fix unaligned access to crafted TCP header in helper API
From: David Miller @ 2014-10-22 16:53 UTC (permalink / raw)
To: karl.beldan; +Cc: netdev, karl.beldan, ezequiel.garcia
In-Reply-To: <1413900365-12289-1-git-send-email-karl.beldan@gmail.com>
From: Karl Beldan <karl.beldan@gmail.com>
Date: Tue, 21 Oct 2014 16:06:05 +0200
> From: Karl Beldan <karl.beldan@rivierawaves.com>
>
> The crafted header start address is from a driver supplied buffer, which
> one can reasonably expect to be aligned on a 4-bytes boundary.
> However ATM the TSO helper API is only used by ethernet drivers and
> the tcp header will then be aligned to a 2-bytes only boundary from the
> header start address.
>
> Signed-off-by: Karl Beldan <karl.beldan@rivierawaves.com>
Applied, thank you.
^ permalink raw reply
* Re: [PATCH net] sfc: remove incorrect EFX_BUG_ON_PARANOID check
From: David Miller @ 2014-10-22 16:51 UTC (permalink / raw)
To: ecree; +Cc: netdev, sshah, jcooper, linux-net-drivers
In-Reply-To: <alpine.LFD.2.03.1410211353290.26215@solarflare.com>
From: Edward Cree <ecree@solarflare.com>
Date: Tue, 21 Oct 2014 14:50:29 +0100
> From: Jon Cooper <jcooper@solarflare.com>
>
> write_count and insert_count can wrap around, making > check invalid.
>
> Fixes: 70b33fb0ddec827cbbd14cdc664fc27b2ef4a6b6 ("sfc: add support for
> skb->xmit_more").
>
> Signed-off-by: Edward Cree <ecree@solarflare.com>
Applied, thanks.
^ permalink raw reply
* Re: [net-next 1/2] ip6_tunnel: put ip6tnl0 FB device into 'any' mode
From: David Miller @ 2014-10-22 16:47 UTC (permalink / raw)
To: alan; +Cc: netdev, edumazet
In-Reply-To: <1413879088-13513-1-git-send-email-alan@al-an.info>
From: "Alexey Andriyanov" <alan@al-an.info>
Date: Tue, 21 Oct 2014 12:11:27 +0400
> The fallback device is in ipv6 mode by default.
> The mode can not be changed in runtime, so there
> is no way to decapsulate ip4in6 packets coming from
> various sources without creating the specific tunnel
> ifaces for each peer.
>
> Cc: David Miller <davem@davemloft.net>
> Cc: Eric Dumazet <edumazet@google.com>
> Signed-off-by: Alexey Andriyanov <alan@al-an.info>
I don't think you can legitimately change this after all
these years. You'll break someone's setup somehow.
You're going to have to find a way to achieve what you
want whilst keeping the default unchanged.
Sorry.
^ permalink raw reply
* Re: [PATCH] netfilter: log: protect nf_log_register against double registering
From: Pablo Neira Ayuso @ 2014-10-22 16:44 UTC (permalink / raw)
To: Marcelo Ricardo Leitner; +Cc: netfilter-devel, netdev
In-Reply-To: <5447A429.3030608@redhat.com>
On Wed, Oct 22, 2014 at 10:33:45AM -0200, Marcelo Ricardo Leitner wrote:
> On 22-10-2014 10:02, Pablo Neira Ayuso wrote:
> >On Mon, Oct 20, 2014 at 07:58:03PM -0200, Marcelo Ricardo Leitner wrote:
> >>Currently, despite the comment right before the function,
> >>nf_log_register allows registering two loggers on with the same type and
> >>end up overwriting the previous register.
> >>
> >>Not a real issue today as current tree doesn't have two loggers for the
> >>same type but it's better to get this protected.
> >>
> >>Also make sure that all of its callers do error checking.
> >
> >No major objetions to this sanity check. Some comment below.
> >
> >>Signed-off-by: Marcelo Ricardo Leitner <mleitner@redhat.com>
> >>---
> >>
> >>Notes:
> >> Please let me know if you have any issues with the identation on
> >> nf_log_register. I just couldn't find a better one.
> >
> >You can split nf_log_register() in two functions to avoid this.
>
> Sorry but I don't follow this one. You mean having the check on
> nf_log_register() and then calling a __nf_log_register() to actually
> register it?
>
> Now I'm thinking on wrapping
> rcu_dereference_protected(loggers[i][logger->type],
> + lockdep_is_held(&nf_log_mutex))
> into a macro or something like that, because that's the issue in
> there and this construction is called several times. Something like:
>
> #define logger_deref_protected(pf, type) \
> rcu_dereference_protected(loggers[pf][type], \
> lockdep_is_held(&nf_log_mutex));
>
> WDYT?
Seems OK, I think this can be:
#define nft_log_dereference(logger)
So you can use this both from net->nf.nf_loggers[x] and
loggers[x][y] and we have one single macro and we avoid the indent
issues.
Thanks.
^ permalink raw reply
* [PATCH net v1 0/2] amd-xgbe: AMD XGBE driver fixes 2014-10-22
From: Tom Lendacky @ 2014-10-22 16:26 UTC (permalink / raw)
To: netdev; +Cc: davem
The following series of patches includes fixes to the driver.
- Properly handle feature changes via ethtool by using correctly sized
variables
- Perform proper napi packet counting and budget checking
This patch series is based on net.
---
Tom Lendacky (2):
amd-xgbe: Properly handle feature changes via ethtool
amd-xgbe: Fix napi Rx budget accounting
drivers/net/ethernet/amd/xgbe/xgbe-drv.c | 22 +++++++++++++---------
1 file changed, 13 insertions(+), 9 deletions(-)
--
Tom Lendacky
^ permalink raw reply
* [PATCH net v1 2/2] amd-xgbe: Fix napi Rx budget accounting
From: Tom Lendacky @ 2014-10-22 16:26 UTC (permalink / raw)
To: netdev; +Cc: davem
In-Reply-To: <20141022162605.31495.98889.stgit@tlendack-t1.amdoffice.net>
Currently the amd-xgbe driver increments the packets processed counter
each time a descriptor is processed. Since a packet can be represented
by more than one descriptor incrementing the counter in this way is not
appropriate. Also, since multiple descriptors cause the budget check
to be short circuited, sometimes the returned value from the poll
function would be larger than the budget value resulting in a WARN_ONCE
being triggered.
Update the polling logic to properly account for the number of packets
processed and exit when the budget value is reached.
Signed-off-by: Tom Lendacky <thomas.lendacky@amd.com>
---
drivers/net/ethernet/amd/xgbe/xgbe-drv.c | 20 ++++++++++++--------
1 file changed, 12 insertions(+), 8 deletions(-)
diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c
index a480b23..2349ea9 100644
--- a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c
+++ b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c
@@ -1598,7 +1598,8 @@ static int xgbe_rx_poll(struct xgbe_channel *channel, int budget)
struct skb_shared_hwtstamps *hwtstamps;
unsigned int incomplete, error, context_next, context;
unsigned int len, put_len, max_len;
- int received = 0;
+ unsigned int received = 0;
+ int packet_count = 0;
DBGPR("-->xgbe_rx_poll: budget=%d\n", budget);
@@ -1608,7 +1609,7 @@ static int xgbe_rx_poll(struct xgbe_channel *channel, int budget)
rdata = XGBE_GET_DESC_DATA(ring, ring->cur);
packet = &ring->packet_data;
- while (received < budget) {
+ while (packet_count < budget) {
DBGPR(" cur = %d\n", ring->cur);
/* First time in loop see if we need to restore state */
@@ -1662,7 +1663,7 @@ read_again:
if (packet->errors)
DBGPR("Error in received packet\n");
dev_kfree_skb(skb);
- continue;
+ goto next_packet;
}
if (!context) {
@@ -1677,7 +1678,7 @@ read_again:
}
dev_kfree_skb(skb);
- continue;
+ goto next_packet;
}
memcpy(skb_tail_pointer(skb), rdata->skb->data,
put_len);
@@ -1694,7 +1695,7 @@ read_again:
/* Stray Context Descriptor? */
if (!skb)
- continue;
+ goto next_packet;
/* Be sure we don't exceed the configured MTU */
max_len = netdev->mtu + ETH_HLEN;
@@ -1705,7 +1706,7 @@ read_again:
if (skb->len > max_len) {
DBGPR("packet length exceeds configured MTU\n");
dev_kfree_skb(skb);
- continue;
+ goto next_packet;
}
#ifdef XGMAC_ENABLE_RX_PKT_DUMP
@@ -1739,6 +1740,9 @@ read_again:
netdev->last_rx = jiffies;
napi_gro_receive(&pdata->napi, skb);
+
+next_packet:
+ packet_count++;
}
/* Check if we need to save state before leaving */
@@ -1752,9 +1756,9 @@ read_again:
rdata->state.error = error;
}
- DBGPR("<--xgbe_rx_poll: received = %d\n", received);
+ DBGPR("<--xgbe_rx_poll: packet_count = %d\n", packet_count);
- return received;
+ return packet_count;
}
static int xgbe_poll(struct napi_struct *napi, int budget)
^ permalink raw reply related
* [PATCH net v1 1/2] amd-xgbe: Properly handle feature changes via ethtool
From: Tom Lendacky @ 2014-10-22 16:26 UTC (permalink / raw)
To: netdev; +Cc: davem
In-Reply-To: <20141022162605.31495.98889.stgit@tlendack-t1.amdoffice.net>
The ndo_set_features callback function was improperly using an unsigned
int to save the current feature value for features such as NETIF_F_RXCSUM.
Since that feature is in the upper 32 bits of a 64 bit variable the
result was always 0 making it not possible to actually turn off the
hardware RX checksum support. Change the unsigned int type to the
netdev_features_t type in order to properly capture the current value
and perform the proper operation.
Signed-off-by: Tom Lendacky <thomas.lendacky@amd.com>
---
drivers/net/ethernet/amd/xgbe/xgbe-drv.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c
index 2955499..a480b23 100644
--- a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c
+++ b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c
@@ -1465,7 +1465,7 @@ static int xgbe_set_features(struct net_device *netdev,
{
struct xgbe_prv_data *pdata = netdev_priv(netdev);
struct xgbe_hw_if *hw_if = &pdata->hw_if;
- unsigned int rxcsum, rxvlan, rxvlan_filter;
+ netdev_features_t rxcsum, rxvlan, rxvlan_filter;
rxcsum = pdata->netdev_features & NETIF_F_RXCSUM;
rxvlan = pdata->netdev_features & NETIF_F_HW_VLAN_CTAG_RX;
^ permalink raw reply related
* Re: [PATCH net-next] tcp: Add TCP_FREEZE socket option
From: David Miller @ 2014-10-22 16:14 UTC (permalink / raw)
To: kristian.evensen; +Cc: netdev
In-Reply-To: <1413992196-4891-1-git-send-email-kristian.evensen@gmail.com>
From: Kristian Evensen <kristian.evensen@gmail.com>
Date: Wed, 22 Oct 2014 17:36:36 +0200
> This patch introduces support for Freeze-TCP [1].
By your description I would not expect the application to get involved
with the actual final zero window advertisement decision at all.
Instead, I would expect the device layer to trigger a notification
during a "technology change" or whatever you want to call losing
connectivity, whichi TCP can receive and use to start sending zero
windows over all TCP connections using that path.
So the socket option enables or disables the facility, but doesn't
actually trigger the zero window advertisement. A real device based
event does that.
The application has no business watching for the loss of connectivity,
and I am certain you do not want that logice in every application in
order for it to take advantage of this.
And therefore there should be a global option that turns this on for
the entire system by default.
This requires a lot more work than you have done here, you need to
add all the notification handling, the logic in TCP to look at the
attached route on send and trigger zero window probes if the device
event has happened, etc.
^ permalink raw reply
* Re: [PATCH net-next] tcp: Add TCP_FREEZE socket option
From: Kristian Evensen @ 2014-10-22 16:10 UTC (permalink / raw)
To: Eric Dumazet; +Cc: Network Development
In-Reply-To: <1413993272.9031.3.camel@edumazet-glaptop2.roam.corp.google.com>
Hi,
On Wed, Oct 22, 2014 at 5:54 PM, Eric Dumazet <eric.dumazet@gmail.com> wrote:
> This asymmetry looks strange
>
> Following sequence should be allowed :
>
> getsockopt(... TCP_FREEZE, &val, ...)
> setsockopt(... TCP_FREEZE, &val, ...)
>
> So setsockopt() should accept val = 0
Thanks for you comment and I agree. The reasoning behind my original
ordering was that I wanted the values to be in the order which made
most logical sense to me, which is Enable (1), Disable (2) and Disable
with TR-ACK (3). However, I see now that when using the option and
when combined with getsockopt(), this does not make much sense. I will
wait for some more feedback and send a revised version tomorrow with
the following ordering: Disable (0), Enable (1), Disable with TR-ACK
(2).
Thanks again,
Kristian
^ permalink raw reply
* [PATCH] net: fix saving TX flow hash in sock for outgoing connections
From: Sathya Perla @ 2014-10-22 16:12 UTC (permalink / raw)
To: netdev; +Cc: therbert
The commit "net: Save TX flow hash in sock and set in skbuf on xmit"
introduced the inet_set_txhash() and ip6_set_txhash() routines to calculate
and record flow hash(sk_txhash) in the socket structure. sk_txhash is used
to set skb->hash which is used to spread flows across multiple TXQs.
But, the above routines are invoked before the source port of the connection
is created. Because of this all outgoing connections that just differ in the
source port get hashed into the same TXQ.
This patch fixes this problem for IPv4/6 by invoking the the above routines
after the source port is available for the socket.
Fixes: b73c3d0e4("net: Save TX flow hash in sock and set in skbuf on xmit")
Signed-off-by: Sathya Perla <sathya.perla@emulex.com>
---
net/ipv4/tcp_ipv4.c | 4 ++--
net/ipv6/tcp_ipv6.c | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
index 94d1a77..9c7d762 100644
--- a/net/ipv4/tcp_ipv4.c
+++ b/net/ipv4/tcp_ipv4.c
@@ -206,8 +206,6 @@ int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
inet->inet_dport = usin->sin_port;
inet->inet_daddr = daddr;
- inet_set_txhash(sk);
-
inet_csk(sk)->icsk_ext_hdr_len = 0;
if (inet_opt)
inet_csk(sk)->icsk_ext_hdr_len = inet_opt->opt.optlen;
@@ -224,6 +222,8 @@ int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
if (err)
goto failure;
+ inet_set_txhash(sk);
+
rt = ip_route_newports(fl4, rt, orig_sport, orig_dport,
inet->inet_sport, inet->inet_dport, sk);
if (IS_ERR(rt)) {
diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c
index 8314955..ace29b6 100644
--- a/net/ipv6/tcp_ipv6.c
+++ b/net/ipv6/tcp_ipv6.c
@@ -200,8 +200,6 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr,
sk->sk_v6_daddr = usin->sin6_addr;
np->flow_label = fl6.flowlabel;
- ip6_set_txhash(sk);
-
/*
* TCP over IPv4
*/
@@ -297,6 +295,8 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr,
if (err)
goto late_failure;
+ ip6_set_txhash(sk);
+
if (!tp->write_seq && likely(!tp->repair))
tp->write_seq = secure_tcpv6_sequence_number(np->saddr.s6_addr32,
sk->sk_v6_daddr.s6_addr32,
--
1.7.1
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox