3.5 bridging regression

[Willy Tarreau <w@1wt.eu>]

[-- Attachment #1: Type: text/plain, Size: 4372 bytes --]

Hi Eric,

Since 3.5, I was getting very quick panic when setting up a bridge
on my guruplug (dual-gig equipped ARM system). This week-end I could
bisect the issue and found that it is this patch which introduced the
issue :

  a1c7fff7e18f59e684e07b0f9a770561cd39f395  net: netdev_alloc_skb() use build_skb()

I can reliably reproduce the issue by installing my laptop behind this
bridge and running "find" on an NFS mount. I don't understand why this
patch can cause this, I was thinking that maybe we free the same page
twice or something like this but I don't see any such thing there.

I set up the bridge using this script :

	brctl addbr br0
	brctl addif br0 eth0
	brctl addif br0 eth1
	ip a f dev eth0
	ip a f dev eth1
	ip li set eth0 up
	ip li set eth1 up
	ip li set br0 up

The network driver is mv643xx. I don't know if this is important, but
since this issue is still present in 3.6.2 and nobody has yet reported
a panic on bridge, I suspect that it may contribute to the issue.

I'm also the config (only non-disabled options), I could strip it down
enough to reduce the possibilities. This config causes the panic with
the patch and does not without. I don't know what else to look for,
I'm open to any idea you might have.

The panic looks like this :

------------[ cut here ]------------
kernel BUG at mm/slab.c:505!
Internal error: Oops - BUG: 0 [#1] ARM
Modules linked in:
CPU: 0    Not tainted  (3.5.0-fail #17)
PC is at kfree+0x8c/0xa8
LR is at __kfree_skb+0x14/0xc8
pc : [<8008bee8>]    lr : [<801bad84>]    psr: 40000093
sp : 8035fe08  ip : 00000000  fp : ffdf8480
r10: 803691a8  r9 : 00f82802  r8 : 80403900
r7 : 00000001  r6 : a0000013  r5 : 9fb776e0  r4 : 8040b000
r3 : 80801ee0  r2 : 00000000  r1 : 00000000  r0 : 00000000
Flags: nZcv  IRQs off  FIQs on  Mode SVC_32  ISA ARM  Segment kernel
Control: 0005397f  Table: 1f074000  DAC: 00000017
Process swapper (pid: 0, stack limit = 0x8035e270)
Stack: (0x8035fe08 to 0x80360000)
fe00:                   9fb749e0 9fb749e0 9f9b0400 801bad84 9f9b1918 801aa740
fe20: 807ed620 00000010 00000000 9f96e400 00000002 9f9b0400 00000000 9f9b1918
fe40: 00000038 803804e0 00000000 00000000 00000001 801ab1c8 622af000 ffffffff
fe60: 0000001d 00000080 00000010 00001514 9f9b04d4 001312cf af6a5400 ffffffff
fe80: 00000000 00989680 001e8480 801aaef8 9f9b04d4 00000080 0000012c 803804e0
fea0: 803804e8 8036f1f0 ffff91f1 801c61bc 00000000 803804e0 00000000 00000001
fec0: 8040166c 8035e000 00000100 80401660 80370138 80401640 0000000a 80024cbc
fee0: 00000000 80370e88 0000000f 00000003 8035ff64 8035e000 0000000f 00000000
ff00: 8035ff64 00000000 56251311 8036b1f8 00000000 800250e8 8037608c 8000fd90
ff20: 801b1150 20000013 fed20200 8000eaf4 8035ff78 20000013 0002dffa 00000004
ff40: 3e957f9a 00000004 8036b470 00000000 00000000 56251311 8036b1f8 00000000
ff60: 9fffffae 8035ff78 80044de0 801b1150 20000013 ffffffff 3e985f94 00000004
ff80: 00000000 8035e000 00000000 00000000 00000000 8036b470 804070e8 8036b1f8
ffa0: 003592b0 801b0f74 8035e000 803806c8 80369e54 80369e4c 00004000 800100c8
ffc0: 80366108 8035a678 8080c0c0 80340738 00000000 00000000 803401d4 00000000
ffe0: 00000000 8035a678 00053975 80366044 8035a674 00008040 00000000 00000000
[<8008bee8>] (kfree+0x8c/0xa8) from [<801bad84>] (__kfree_skb+0x14/0xc8)
[<801bad84>] (__kfree_skb+0x14/0xc8) from [<801aa740>] (txq_reclaim+0x198/0x244)
[<801aa740>] (txq_reclaim+0x198/0x244) from [<801ab1c8>] (mv643xx_eth_poll+0x2d0/0x71c)
[<801ab1c8>] (mv643xx_eth_poll+0x2d0/0x71c) from [<801c61bc>] (net_rx_action+0xb0/0x188)
[<801c61bc>] (net_rx_action+0xb0/0x188) from [<80024cbc>] (__do_softirq+0x90/0x120)
[<80024cbc>] (__do_softirq+0x90/0x120) from [<800250e8>] (irq_exit+0x7c/0x84)
[<800250e8>] (irq_exit+0x7c/0x84) from [<8000fd90>] (handle_IRQ+0x34/0x84)
[<8000fd90>] (handle_IRQ+0x34/0x84) from [<8000eaf4>] (__irq_svc+0x34/0x80)
[<8000eaf4>] (__irq_svc+0x34/0x80) from [<801b1150>] (cpuidle_wrap_enter+0x54/0x9c)
[<801b1150>] (cpuidle_wrap_enter+0x54/0x9c) from [<801b0f74>] (cpuidle_idle_call+0x9c/0x130)
[<801b0f74>] (cpuidle_idle_call+0x9c/0x130) from [<800100c8>] (cpu_idle+0x88/0xd4)
[<800100c8>] (cpu_idle+0x88/0xd4) from [<80340738>] (start_kernel+0x298/0x2ec)
Code: e7845101 e5840000 e121f006 e8bd8070 (e7f001f2)
---[ end trace 879b0e636889a6d4 ]---
Kernel panic - not syncing: Fatal exception in interrupt
Rebooting in 1 seconds..

Thanks,
Willy


[-- Attachment #2: config-arm-crash.txt --]
[-- Type: text/plain, Size: 8312 bytes --]

CONFIG_ARM=y
CONFIG_SYS_SUPPORTS_APM_EMULATION=y
CONFIG_GENERIC_GPIO=y
CONFIG_GENERIC_CLOCKEVENTS=y
CONFIG_KTIME_SCALAR=y
CONFIG_HAVE_PROC_CPU=y
CONFIG_STACKTRACE_SUPPORT=y
CONFIG_HAVE_LATENCYTOP_SUPPORT=y
CONFIG_LOCKDEP_SUPPORT=y
CONFIG_TRACE_IRQFLAGS_SUPPORT=y
CONFIG_HARDIRQS_SW_RESEND=y
CONFIG_GENERIC_IRQ_PROBE=y
CONFIG_RWSEM_GENERIC_SPINLOCK=y
CONFIG_ARCH_HAS_CPU_IDLE_WAIT=y
CONFIG_GENERIC_HWEIGHT=y
CONFIG_GENERIC_CALIBRATE_DELAY=y
CONFIG_NEED_DMA_MAP_STATE=y
CONFIG_VECTORS_BASE=0xffff0000
CONFIG_ARM_PATCH_PHYS_VIRT=y
CONFIG_NEED_MACH_IO_H=y
CONFIG_GENERIC_BUG=y
CONFIG_DEFCONFIG_LIST="/lib/modules/$UNAME_RELEASE/.config"
CONFIG_HAVE_IRQ_WORK=y
CONFIG_IRQ_WORK=y
CONFIG_EXPERIMENTAL=y
CONFIG_BROKEN_ON_SMP=y
CONFIG_INIT_ENV_ARG_LIMIT=32
CONFIG_CROSS_COMPILE=""
CONFIG_LOCALVERSION=""
CONFIG_HAVE_KERNEL_GZIP=y
CONFIG_HAVE_KERNEL_LZMA=y
CONFIG_HAVE_KERNEL_XZ=y
CONFIG_HAVE_KERNEL_LZO=y
CONFIG_KERNEL_GZIP=y
CONFIG_DEFAULT_HOSTNAME="dockstar"
CONFIG_SYSVIPC=y
CONFIG_SYSVIPC_SYSCTL=y
CONFIG_POSIX_MQUEUE=y
CONFIG_POSIX_MQUEUE_SYSCTL=y
CONFIG_BSD_PROCESS_ACCT=y
CONFIG_HAVE_GENERIC_HARDIRQS=y
CONFIG_GENERIC_HARDIRQS=y
CONFIG_GENERIC_IRQ_SHOW=y
CONFIG_GENERIC_IRQ_CHIP=y
CONFIG_TINY_RCU=y
CONFIG_IKCONFIG=y
CONFIG_IKCONFIG_PROC=y
CONFIG_LOG_BUF_SHIFT=19
CONFIG_SYSFS_DEPRECATED=y
CONFIG_SYSFS_DEPRECATED_V2=y
CONFIG_BLK_DEV_INITRD=y
CONFIG_INITRAMFS_SOURCE=""
CONFIG_RD_GZIP=y
CONFIG_RD_LZO=y
CONFIG_SYSCTL=y
CONFIG_ANON_INODES=y
CONFIG_EXPERT=y
CONFIG_UID16=y
CONFIG_SYSCTL_SYSCALL=y
CONFIG_KALLSYMS=y
CONFIG_HOTPLUG=y
CONFIG_PRINTK=y
CONFIG_BUG=y
CONFIG_ELF_CORE=y
CONFIG_BASE_FULL=y
CONFIG_FUTEX=y
CONFIG_EPOLL=y
CONFIG_SIGNALFD=y
CONFIG_TIMERFD=y
CONFIG_EVENTFD=y
CONFIG_SHMEM=y
CONFIG_AIO=y
CONFIG_EMBEDDED=y
CONFIG_HAVE_PERF_EVENTS=y
CONFIG_PERF_USE_VMALLOC=y
CONFIG_PERF_EVENTS=y
CONFIG_VM_EVENT_COUNTERS=y
CONFIG_COMPAT_BRK=y
CONFIG_SLAB=y
CONFIG_PROFILING=y
CONFIG_HAVE_OPROFILE=y
CONFIG_JUMP_LABEL=y
CONFIG_HAVE_KPROBES=y
CONFIG_HAVE_KRETPROBES=y
CONFIG_HAVE_REGS_AND_STACK_ACCESS_API=y
CONFIG_HAVE_DMA_API_DEBUG=y
CONFIG_HAVE_ARCH_JUMP_LABEL=y
CONFIG_HAVE_GENERIC_DMA_COHERENT=y
CONFIG_SLABINFO=y
CONFIG_RT_MUTEXES=y
CONFIG_BASE_SMALL=0
CONFIG_MODULES=y
CONFIG_MODULE_FORCE_LOAD=y
CONFIG_MODULE_UNLOAD=y
CONFIG_MODULE_FORCE_UNLOAD=y
CONFIG_BLOCK=y
CONFIG_LBDAF=y
CONFIG_BLK_DEV_BSG=y
CONFIG_MSDOS_PARTITION=y
CONFIG_IOSCHED_NOOP=y
CONFIG_IOSCHED_CFQ=y
CONFIG_DEFAULT_CFQ=y
CONFIG_DEFAULT_IOSCHED="cfq"
CONFIG_INLINE_SPIN_UNLOCK_IRQ=y
CONFIG_INLINE_READ_UNLOCK=y
CONFIG_INLINE_READ_UNLOCK_IRQ=y
CONFIG_INLINE_WRITE_UNLOCK=y
CONFIG_INLINE_WRITE_UNLOCK_IRQ=y
CONFIG_MMU=y
CONFIG_ARCH_KIRKWOOD=y
CONFIG_MACH_GURUPLUG=y
CONFIG_MACH_DOCKSTAR=y
CONFIG_PLAT_ORION=y
CONFIG_CPU_FEROCEON=y
CONFIG_CPU_32v5=y
CONFIG_CPU_ABRT_EV5T=y
CONFIG_CPU_PABRT_LEGACY=y
CONFIG_CPU_CACHE_VIVT=y
CONFIG_CPU_COPY_FEROCEON=y
CONFIG_CPU_TLB_FEROCEON=y
CONFIG_CPU_CP15=y
CONFIG_CPU_CP15_MMU=y
CONFIG_CPU_USE_DOMAINS=y
CONFIG_OUTER_CACHE=y
CONFIG_CACHE_FEROCEON_L2=y
CONFIG_ARM_L1_CACHE_SHIFT=5
CONFIG_ARM_NR_BANKS=8
CONFIG_PCI=y
CONFIG_PCI_SYSCALL=y
CONFIG_TICK_ONESHOT=y
CONFIG_NO_HZ=y
CONFIG_HIGH_RES_TIMERS=y
CONFIG_GENERIC_CLOCKEVENTS_BUILD=y
CONFIG_VMSPLIT_2G=y
CONFIG_PAGE_OFFSET=0x80000000
CONFIG_ARCH_NR_GPIO=0
CONFIG_PREEMPT_NONE=y
CONFIG_HZ=100
CONFIG_AEABI=y
CONFIG_OABI_COMPAT=y
CONFIG_HAVE_ARCH_PFN_VALID=y
CONFIG_SELECT_MEMORY_MODEL=y
CONFIG_FLATMEM_MANUAL=y
CONFIG_FLATMEM=y
CONFIG_FLAT_NODE_MEM_MAP=y
CONFIG_HAVE_MEMBLOCK=y
CONFIG_PAGEFLAGS_EXTENDED=y
CONFIG_SPLIT_PTLOCK_CPUS=999999
CONFIG_ZONE_DMA_FLAG=0
CONFIG_VIRT_TO_BUS=y
CONFIG_DEFAULT_MMAP_MIN_ADDR=32768
CONFIG_NEED_PER_CPU_KM=y
CONFIG_FORCE_MAX_ZONEORDER=11
CONFIG_ALIGNMENT_TRAP=y
CONFIG_UACCESS_WITH_MEMCPY=y
CONFIG_ZBOOT_ROM_TEXT=0x0
CONFIG_ZBOOT_ROM_BSS=0x0
CONFIG_CMDLINE=""
CONFIG_CPU_IDLE=y
CONFIG_CPU_IDLE_GOV_LADDER=y
CONFIG_CPU_IDLE_GOV_MENU=y
CONFIG_FPE_NWFPE=y
CONFIG_FPE_FASTFPE=y
CONFIG_BINFMT_ELF=y
CONFIG_ARCH_BINFMT_ELF_RANDOMIZE_PIE=y
CONFIG_HAVE_AOUT=y
CONFIG_PM_RUNTIME=y
CONFIG_PM=y
CONFIG_CPU_PM=y
CONFIG_NET=y
CONFIG_PACKET=y
CONFIG_UNIX=y
CONFIG_UNIX_DIAG=y
CONFIG_XFRM=y
CONFIG_INET=y
CONFIG_IP_MULTICAST=y
CONFIG_IP_ADVANCED_ROUTER=y
CONFIG_IP_MULTIPLE_TABLES=y
CONFIG_IP_ROUTE_MULTIPATH=y
CONFIG_NET_IPIP=y
CONFIG_SYN_COOKIES=y
CONFIG_INET_TUNNEL=y
CONFIG_INET_LRO=y
CONFIG_INET_DIAG=y
CONFIG_INET_TCP_DIAG=y
CONFIG_INET_UDP_DIAG=y
CONFIG_TCP_CONG_ADVANCED=y
CONFIG_TCP_CONG_BIC=y
CONFIG_TCP_CONG_CUBIC=y
CONFIG_TCP_CONG_HTCP=y
CONFIG_TCP_CONG_HYBLA=y
CONFIG_TCP_CONG_VEGAS=y
CONFIG_DEFAULT_CUBIC=y
CONFIG_DEFAULT_TCP_CONG="cubic"
CONFIG_IPV6=y
CONFIG_INET6_XFRM_MODE_TRANSPORT=y
CONFIG_INET6_XFRM_MODE_TUNNEL=y
CONFIG_INET6_XFRM_MODE_BEET=y
CONFIG_IPV6_SIT=y
CONFIG_IPV6_NDISC_NODETYPE=y
CONFIG_NETFILTER=y
CONFIG_NETFILTER_ADVANCED=y
CONFIG_BRIDGE_NETFILTER=y
CONFIG_STP=y
CONFIG_BRIDGE=y
CONFIG_VLAN_8021Q=y
CONFIG_LLC=y
CONFIG_NET_SCHED=y
CONFIG_NET_CLS=y
CONFIG_NET_EMATCH=y
CONFIG_NET_EMATCH_STACK=32
CONFIG_NET_CLS_ACT=y
CONFIG_NET_SCH_FIFO=y
CONFIG_BQL=y
CONFIG_HAVE_BPF_JIT=y
CONFIG_BPF_JIT=y
CONFIG_FIB_RULES=y
CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
CONFIG_DEVTMPFS=y
CONFIG_DEVTMPFS_MOUNT=y
CONFIG_STANDALONE=y
CONFIG_PREVENT_FIRMWARE_BUILD=y
CONFIG_FW_LOADER=y
CONFIG_FIRMWARE_IN_KERNEL=y
CONFIG_EXTRA_FIRMWARE=""
CONFIG_MTD=y
CONFIG_MTD_CMDLINE_PARTS=y
CONFIG_MTD_CHAR=y
CONFIG_MTD_BLKDEVS=y
CONFIG_MTD_BLOCK=y
CONFIG_MTD_CFI=y
CONFIG_MTD_JEDECPROBE=y
CONFIG_MTD_GEN_PROBE=y
CONFIG_MTD_CFI_ADV_OPTIONS=y
CONFIG_MTD_CFI_NOSWAP=y
CONFIG_MTD_CFI_GEOMETRY=y
CONFIG_MTD_MAP_BANK_WIDTH_1=y
CONFIG_MTD_MAP_BANK_WIDTH_2=y
CONFIG_MTD_CFI_I1=y
CONFIG_MTD_CFI_I2=y
CONFIG_MTD_CFI_INTELEXT=y
CONFIG_MTD_CFI_STAA=y
CONFIG_MTD_CFI_UTIL=y
CONFIG_MTD_PHYSMAP=y
CONFIG_MTD_NAND_ECC=y
CONFIG_MTD_NAND=y
CONFIG_MTD_NAND_IDS=y
CONFIG_MTD_NAND_ORION=y
CONFIG_MTD_UBI=y
CONFIG_MTD_UBI_WL_THRESHOLD=4096
CONFIG_MTD_UBI_BEB_RESERVE=1
CONFIG_BLK_DEV=y
CONFIG_BLK_DEV_LOOP=y
CONFIG_BLK_DEV_LOOP_MIN_COUNT=8
CONFIG_BLK_DEV_RAM=y
CONFIG_BLK_DEV_RAM_COUNT=16
CONFIG_BLK_DEV_RAM_SIZE=16384
CONFIG_HAVE_IDE=y
CONFIG_SCSI_MOD=y
CONFIG_NETDEVICES=y
CONFIG_NET_CORE=y
CONFIG_MII=y
CONFIG_TUN=y
CONFIG_ETHERNET=y
CONFIG_NET_VENDOR_MARVELL=y
CONFIG_MV643XX_ETH=y
CONFIG_PHYLIB=y
CONFIG_MARVELL_PHY=y
CONFIG_UNIX98_PTYS=y
CONFIG_LEGACY_PTYS=y
CONFIG_LEGACY_PTY_COUNT=16
CONFIG_DEVKMEM=y
CONFIG_SERIAL_8250=y
CONFIG_SERIAL_8250_CONSOLE=y
CONFIG_SERIAL_8250_NR_UARTS=4
CONFIG_SERIAL_8250_RUNTIME_UARTS=2
CONFIG_SERIAL_CORE=y
CONFIG_SERIAL_CORE_CONSOLE=y
CONFIG_DEVPORT=y
CONFIG_ARCH_REQUIRE_GPIOLIB=y
CONFIG_GPIOLIB=y
CONFIG_GPIO_SYSFS=y
CONFIG_GPIO_GENERIC=y
CONFIG_GPIO_GENERIC_PLATFORM=y
CONFIG_SSB_POSSIBLE=y
CONFIG_BCMA_POSSIBLE=y
CONFIG_USB_ARCH_HAS_OHCI=y
CONFIG_USB_ARCH_HAS_EHCI=y
CONFIG_USB_ARCH_HAS_XHCI=y
CONFIG_RTC_LIB=y
CONFIG_RTC_CLASS=y
CONFIG_RTC_HCTOSYS=y
CONFIG_RTC_HCTOSYS_DEVICE="rtc0"
CONFIG_RTC_INTF_SYSFS=y
CONFIG_RTC_INTF_PROC=y
CONFIG_RTC_INTF_DEV=y
CONFIG_RTC_DRV_MV=y
CONFIG_DMADEVICES=y
CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH=y
CONFIG_MV_XOR=y
CONFIG_DMA_ENGINE=y
CONFIG_STAGING=y
CONFIG_CLKSRC_MMIO=y
CONFIG_FILE_LOCKING=y
CONFIG_FSNOTIFY=y
CONFIG_DNOTIFY=y
CONFIG_INOTIFY_USER=y
CONFIG_PROC_FS=y
CONFIG_PROC_SYSCTL=y
CONFIG_PROC_PAGE_MONITOR=y
CONFIG_SYSFS=y
CONFIG_TMPFS=y
CONFIG_CONFIGFS_FS=y
CONFIG_MISC_FILESYSTEMS=y
CONFIG_UBIFS_FS=y
CONFIG_UBIFS_FS_LZO=y
CONFIG_UBIFS_FS_ZLIB=y
CONFIG_DEFAULT_MESSAGE_LOGLEVEL=4
CONFIG_FRAME_WARN=1024
CONFIG_MAGIC_SYSRQ=y
CONFIG_STRIP_ASM_SYMS=y
CONFIG_DEBUG_KERNEL=y
CONFIG_DEBUG_BUGVERBOSE=y
CONFIG_DEBUG_MEMORY_INIT=y
CONFIG_HAVE_FUNCTION_TRACER=y
CONFIG_HAVE_FUNCTION_GRAPH_TRACER=y
CONFIG_HAVE_DYNAMIC_FTRACE=y
CONFIG_HAVE_FTRACE_MCOUNT_RECORD=y
CONFIG_HAVE_C_RECORDMCOUNT=y
CONFIG_TRACING_SUPPORT=y
CONFIG_HAVE_ARCH_KGDB=y
CONFIG_ARM_UNWIND=y
CONFIG_DEBUG_USER=y
CONFIG_DEBUG_LL=y
CONFIG_DEBUG_LL_UART_NONE=y
CONFIG_EARLY_PRINTK=y
CONFIG_DEFAULT_SECURITY_DAC=y
CONFIG_DEFAULT_SECURITY=""
CONFIG_CRYPTO=y
CONFIG_CRYPTO_ALGAPI=y
CONFIG_CRYPTO_ALGAPI2=y
CONFIG_CRYPTO_DEFLATE=y
CONFIG_CRYPTO_LZO=y
CONFIG_BITREVERSE=y
CONFIG_GENERIC_PCI_IOMAP=y
CONFIG_GENERIC_IO=y
CONFIG_CRC16=y
CONFIG_CRC32=y
CONFIG_CRC32_SLICEBY8=y
CONFIG_ZLIB_INFLATE=y
CONFIG_ZLIB_DEFLATE=y
CONFIG_LZO_COMPRESS=y
CONFIG_LZO_DECOMPRESS=y
CONFIG_DECOMPRESS_GZIP=y
CONFIG_DECOMPRESS_LZO=y
CONFIG_HAS_IOMEM=y
CONFIG_HAS_IOPORT=y
CONFIG_HAS_DMA=y
CONFIG_DQL=y
CONFIG_NLATTR=y
CONFIG_GENERIC_ATOMIC64=y