[RFC PATCH] pch_uart: Add eg20t_port lock field, avoid recursive spinlocks

[RFC PATCH] pch_uart: Add eg20t_port lock field, avoid recursive spinlocks

[Darren Hart]

pch_uart_interrupt() takes priv->port.lock which leads to two recursive
spinlock calls if low_latency==1 or CONFIG_PREEMPT_RT_FULL=y (one
otherwise):

pch_uart_interrupt
  spin_lock_irqsave(priv->port.lock, flags)
  case PCH_UART_IID_RDR_TO (data ready)
  handle_rx_to
    push_rx
      tty_port_tty_get
        spin_lock_irqsave(&port->lock, flags) <--- already hold this lock
        ...
      tty_flip_buffer_push
        ...
        flush_to_ldisc
          spin_lock_irqsave(&tty->buf.lock)
            spin_lock_irqsave(&tty->buf.lock)
            disc->ops->receive_buf(tty, char_buf)
              n_tty_receive_buf
                tty->ops->flush_chars()
                uart_flush_chars
                  uart_start
                    spin_lock_irqsave(&port->lock) <--- already hold this lock

Avoid this by using a dedicated lock to protect the eg20t_port structure
and IO access to its membase. This is more consistent with the 8250
driver.  Ensure priv->lock is always take prior to priv->port.lock when
taken at the same time.

Tomoya, I have attempted to protect the eg20t_port structure and the membase
with the new eg20t_port.lock field, however I believe the current locking to be
a bit coarse. I imagine the priv->lock could be held a bit less. What are your
thoughts?

Signed-off-by: Darren Hart <dvhart@linux.intel.com>
CC: Tomoya MORINAGA <tomoya.rohm@gmail.com>
CC: Feng Tang <feng.tang@intel.com>
CC: Alexander Stein <alexander.stein@systec-electronic.com>
CC: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
CC: Alan Cox <alan@linux.intel.com>
CC: linux-serial@vger.kernel.org
---
 drivers/tty/serial/pch_uart.c |   22 ++++++++++++++++------
 1 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/drivers/tty/serial/pch_uart.c b/drivers/tty/serial/pch_uart.c
index 4fdec6a..c90a8cd 100644
--- a/drivers/tty/serial/pch_uart.c
+++ b/drivers/tty/serial/pch_uart.c
@@ -253,6 +253,9 @@ struct eg20t_port {
 	dma_addr_t			rx_buf_dma;
 
 	struct dentry	*debugfs;
+
+	/* protect the eg20t_port private structure and io access to membase */
+	spinlock_t lock;
 };
 
 /**
@@ -1058,7 +1061,7 @@ static irqreturn_t pch_uart_interrupt(int irq, void *dev_id)
 	int next = 1;
 	u8 msr;
 
-	spin_lock_irqsave(&priv->port.lock, flags);
+	spin_lock_irqsave(&priv->lock, flags);
 	handled = 0;
 	while (next) {
 		iid = pch_uart_hal_get_iid(priv);
@@ -1116,7 +1119,7 @@ static irqreturn_t pch_uart_interrupt(int irq, void *dev_id)
 		handled |= (unsigned int)ret;
 	}
 
-	spin_unlock_irqrestore(&priv->port.lock, flags);
+	spin_unlock_irqrestore(&priv->lock, flags);
 	return IRQ_RETVAL(handled);
 }
 
@@ -1226,9 +1229,9 @@ static void pch_uart_break_ctl(struct uart_port *port, int ctl)
 	unsigned long flags;
 
 	priv = container_of(port, struct eg20t_port, port);
-	spin_lock_irqsave(&port->lock, flags);
+	spin_lock_irqsave(&priv->lock, flags);
 	pch_uart_hal_set_break(priv, ctl);
-	spin_unlock_irqrestore(&port->lock, flags);
+	spin_unlock_irqrestore(&priv->lock, flags);
 }
 
 /* Grab any interrupt resources and initialise any low level driver state. */
@@ -1376,7 +1379,8 @@ static void pch_uart_set_termios(struct uart_port *port,
 
 	baud = uart_get_baud_rate(port, termios, old, 0, port->uartclk / 16);
 
-	spin_lock_irqsave(&port->lock, flags);
+	spin_lock_irqsave(&priv->lock, flags);
+	spin_lock(&port->lock);
 
 	uart_update_timeout(port, termios->c_cflag, baud);
 	rtn = pch_uart_hal_set_line(priv, baud, parity, bits, stb);
@@ -1389,7 +1393,8 @@ static void pch_uart_set_termios(struct uart_port *port,
 		tty_termios_encode_baud_rate(termios, baud, baud);
 
 out:
-	spin_unlock_irqrestore(&port->lock, flags);
+	spin_unlock(&port->lock);
+	spin_unlock_irqrestore(&priv->lock, flags);
 }
 
 static const char *pch_uart_type(struct uart_port *port)
@@ -1546,6 +1551,7 @@ pch_console_write(struct console *co, const char *s, unsigned int count)
 	touch_nmi_watchdog();
 
 	local_irq_save(flags);
+	spin_lock(&priv->lock);
 	if (priv->port.sysrq) {
 		/* serial8250_handle_port() already took the lock */
 		locked = 0;
@@ -1572,7 +1578,9 @@ pch_console_write(struct console *co, const char *s, unsigned int count)
 
 	if (locked)
 		spin_unlock(&priv->port.lock);
+	spin_unlock(&priv->lock);
 	local_irq_restore(flags);
+
 }
 
 static int __init pch_console_setup(struct console *co, char *options)
@@ -1669,6 +1677,8 @@ static struct eg20t_port *pch_uart_init_port(struct pci_dev *pdev,
 	pci_enable_msi(pdev);
 	pci_set_master(pdev);
 
+	spin_lock_init(&priv->lock);
+
 	iobase = pci_resource_start(pdev, 0);
 	mapbase = pci_resource_start(pdev, 1);
 	priv->mapbase = mapbase;
-- 
1.7.5.4

Re: [RFC PATCH] pch_uart: Add eg20t_port lock field, avoid recursive spinlocks

[Tomoya MORINAGA]

On Thu, May 31, 2012 at 5:54 PM, Darren Hart <dvhart@linux.intel.com> wrote:
> @@ -1376,7 +1379,8 @@ static void pch_uart_set_termios(struct uart_port *port,
>
>        baud = uart_get_baud_rate(port, termios, old, 0, port->uartclk / 16);
>
> -       spin_lock_irqsave(&port->lock, flags);
> +       spin_lock_irqsave(&priv->lock, flags);
> +       spin_lock(&port->lock);
>
>        uart_update_timeout(port, termios->c_cflag, baud);
>        rtn = pch_uart_hal_set_line(priv, baud, parity, bits, stb);
> @@ -1389,7 +1393,8 @@ static void pch_uart_set_termios(struct uart_port *port,
>                tty_termios_encode_baud_rate(termios, baud, baud);
>
>  out:
> -       spin_unlock_irqrestore(&port->lock, flags);
> +       spin_unlock(&port->lock);
> +       spin_unlock_irqrestore(&priv->lock, flags);
>  }

Are both port->lock and priv->lock really necessary ?


> @@ -1572,7 +1578,9 @@ pch_console_write(struct console *co, const char *s, unsigned int count)
>
>        if (locked)
>                spin_unlock(&priv->port.lock);
> +       spin_unlock(&priv->lock);
>        local_irq_restore(flags);
> +
>  }

Looks spare blank line.

thanks.
-- 
ROHM Co., Ltd.
tomoya

Re: [RFC PATCH] pch_uart: Add eg20t_port lock field, avoid recursive spinlocks

[Darren Hart]

On 06/01/2012 01:30 AM, Tomoya MORINAGA wrote:
> On Thu, May 31, 2012 at 5:54 PM, Darren Hart <dvhart@linux.intel.com> wrote:
>> @@ -1376,7 +1379,8 @@ static void pch_uart_set_termios(struct uart_port *port,
>>
>>        baud = uart_get_baud_rate(port, termios, old, 0, port->uartclk / 16);
>>
>> -       spin_lock_irqsave(&port->lock, flags);
>> +       spin_lock_irqsave(&priv->lock, flags);
>> +       spin_lock(&port->lock);
>>
>>        uart_update_timeout(port, termios->c_cflag, baud);
>>        rtn = pch_uart_hal_set_line(priv, baud, parity, bits, stb);
>> @@ -1389,7 +1393,8 @@ static void pch_uart_set_termios(struct uart_port *port,
>>                tty_termios_encode_baud_rate(termios, baud, baud);
>>
>>  out:
>> -       spin_unlock_irqrestore(&port->lock, flags);
>> +       spin_unlock(&port->lock);
>> +       spin_unlock_irqrestore(&priv->lock, flags);
>>  }
> 
> Are both port->lock and priv->lock really necessary ?

The priv lock protects the pch_uart_hal* calls and the io access.

The port lock protects the uart_update_timeout call. I'm assuming the
8250.c driver is correct in holding the port lock before making this
call and making other modifcations to the port struct.

So yes, I believe both are required. The port->lock was used as the lock
to protect the private data in the interrupt handler,
pch_uart_interrupt. If we could avoid holding that lock across the
entire function, limiting it to just around the pch_uart_hal calls
(perhaps by adding it to the hal calls and adding lockless __pch_uart*
calls) we could avoid the recursive lock that occurs with handle_rx. I
still think a priv-lock is a good idea though, even if just to clarify
what is being protected.

Thoughts?

> 
> 
>> @@ -1572,7 +1578,9 @@ pch_console_write(struct console *co, const char *s, unsigned int count)
>>
>>        if (locked)
>>                spin_unlock(&priv->port.lock);
>> +       spin_unlock(&priv->lock);
>>        local_irq_restore(flags);
>> +
>>  }
> 
> Looks spare blank line.

Thanks, will fix for V2 after this discussion wraps up.

> 
> thanks.

-- 
Darren Hart
Intel Open Source Technology Center
Yocto Project - Linux Kernel

Re: [RFC PATCH] pch_uart: Add eg20t_port lock field, avoid recursive spinlocks

[Darren Hart]

On 06/01/2012 11:36 AM, Darren Hart wrote:
> 
> 
> On 06/01/2012 01:30 AM, Tomoya MORINAGA wrote:
>> On Thu, May 31, 2012 at 5:54 PM, Darren Hart <dvhart@linux.intel.com> wrote:
>>> @@ -1376,7 +1379,8 @@ static void pch_uart_set_termios(struct uart_port *port,
>>>
>>>        baud = uart_get_baud_rate(port, termios, old, 0, port->uartclk / 16);
>>>
>>> -       spin_lock_irqsave(&port->lock, flags);
>>> +       spin_lock_irqsave(&priv->lock, flags);
>>> +       spin_lock(&port->lock);
>>>
>>>        uart_update_timeout(port, termios->c_cflag, baud);
>>>        rtn = pch_uart_hal_set_line(priv, baud, parity, bits, stb);
>>> @@ -1389,7 +1393,8 @@ static void pch_uart_set_termios(struct uart_port *port,
>>>                tty_termios_encode_baud_rate(termios, baud, baud);
>>>
>>>  out:
>>> -       spin_unlock_irqrestore(&port->lock, flags);
>>> +       spin_unlock(&port->lock);
>>> +       spin_unlock_irqrestore(&priv->lock, flags);
>>>  }
>>
>> Are both port->lock and priv->lock really necessary ?
> 
> The priv lock protects the pch_uart_hal* calls and the io access.
> 
> The port lock protects the uart_update_timeout call. I'm assuming the
> 8250.c driver is correct in holding the port lock before making this
> call and making other modifcations to the port struct.
> 
> So yes, I believe both are required. The port->lock was used as the lock
> to protect the private data in the interrupt handler,
> pch_uart_interrupt. If we could avoid holding that lock across the
> entire function, limiting it to just around the pch_uart_hal calls
> (perhaps by adding it to the hal calls and adding lockless __pch_uart*
> calls) we could avoid the recursive lock that occurs with handle_rx. I
> still think a priv-lock is a good idea though, even if just to clarify
> what is being protected.
> 
> Thoughts?

Are there still concerns about the additional lock? I'll resend V2
tomorrow with the single whitespace fix if I don't hear anything back today.

Thanks,

Darren

> 
>>
>>
>>> @@ -1572,7 +1578,9 @@ pch_console_write(struct console *co, const char *s, unsigned int count)
>>>
>>>        if (locked)
>>>                spin_unlock(&priv->port.lock);
>>> +       spin_unlock(&priv->lock);
>>>        local_irq_restore(flags);
>>> +
>>>  }
>>
>> Looks spare blank line.
> 
> Thanks, will fix for V2 after this discussion wraps up.
> 
>>
>> thanks.
> 

-- 
Darren Hart
Intel Open Source Technology Center
Yocto Project - Linux Kernel

Re: [RFC PATCH] pch_uart: Add eg20t_port lock field, avoid recursive spinlocks

[Tomoya MORINAGA]

On Wed, Jun 6, 2012 at 7:07 AM, Darren Hart <dvhart@linux.intel.com> wrote:
> Are there still concerns about the additional lock? I'll resend V2
> tomorrow with the single whitespace fix if I don't hear anything back today.

I understand your saying. Looks good.
However, I am not expert of linux-uart core system.
So, I'd like UART maintainer to give us your opinion.

thanks.
tomoya

Re: [RFC PATCH] pch_uart: Add eg20t_port lock field, avoid recursive spinlocks

[Darren Hart]

On 06/05/2012 04:48 PM, Tomoya MORINAGA wrote:
> On Wed, Jun 6, 2012 at 7:07 AM, Darren Hart <dvhart@linux.intel.com> wrote:
>> Are there still concerns about the additional lock? I'll resend V2
>> tomorrow with the single whitespace fix if I don't hear anything back today.
> 
> I understand your saying. Looks good.
> However, I am not expert of linux-uart core system.
> So, I'd like UART maintainer to give us your opinion.

Greg, Alan,

any concerns with the locking approach I've adopted in the patch?

Thanks,

-- 
Darren Hart
Intel Open Source Technology Center
Yocto Project - Linux Kernel

Re: [RFC PATCH] pch_uart: Add eg20t_port lock field, avoid recursive spinlocks

[Greg Kroah-Hartman]

On Mon, Jun 18, 2012 at 02:41:46PM -0700, Darren Hart wrote:
> 
> 
> On 06/05/2012 04:48 PM, Tomoya MORINAGA wrote:
> > On Wed, Jun 6, 2012 at 7:07 AM, Darren Hart <dvhart@linux.intel.com> wrote:
> >> Are there still concerns about the additional lock? I'll resend V2
> >> tomorrow with the single whitespace fix if I don't hear anything back today.
> > 
> > I understand your saying. Looks good.
> > However, I am not expert of linux-uart core system.
> > So, I'd like UART maintainer to give us your opinion.
> 
> Greg, Alan,
> 
> any concerns with the locking approach I've adopted in the patch?

Care to resend the patch, as it was a RFC one, it's no longer in my
queue.

thanks,

greg k-h

Re: [RFC PATCH] pch_uart: Add eg20t_port lock field, avoid recursive spinlocks

[Alan Cox]

On Mon, 18 Jun 2012 14:41:46 -0700
Darren Hart <dvhart@linux.intel.com> wrote:

> 
> 
> On 06/05/2012 04:48 PM, Tomoya MORINAGA wrote:
> > On Wed, Jun 6, 2012 at 7:07 AM, Darren Hart <dvhart@linux.intel.com> wrote:
> >> Are there still concerns about the additional lock? I'll resend V2
> >> tomorrow with the single whitespace fix if I don't hear anything back today.
> > 
> > I understand your saying. Looks good.
> > However, I am not expert of linux-uart core system.
> > So, I'd like UART maintainer to give us your opinion.
> 
> Greg, Alan,
> 
> any concerns with the locking approach I've adopted in the patch?

Only the one I noted in my reply the first time around which is that you
can't permit tty->low_latency=1 unless your tty receive path is not an
IRQ path. From a locking point of view the change makes sense anyway.

Going back over it your console locking also needs care - an oops or
printk within the areas the private lock covers will hang the box. That
should also probably be a trylock style lock as with the other lock on
that path

Alan

Re: [RFC PATCH] pch_uart: Add eg20t_port lock field, avoid recursive spinlocks

[Darren Hart]

On 06/19/2012 02:14 AM, Alan Cox wrote:
> On Mon, 18 Jun 2012 14:41:46 -0700
> Darren Hart <dvhart@linux.intel.com> wrote:
> 
>>
>>
>> On 06/05/2012 04:48 PM, Tomoya MORINAGA wrote:
>>> On Wed, Jun 6, 2012 at 7:07 AM, Darren Hart <dvhart@linux.intel.com> wrote:
>>>> Are there still concerns about the additional lock? I'll resend V2
>>>> tomorrow with the single whitespace fix if I don't hear anything back today.
>>>
>>> I understand your saying. Looks good.
>>> However, I am not expert of linux-uart core system.
>>> So, I'd like UART maintainer to give us your opinion.
>>
>> Greg, Alan,
>>
>> any concerns with the locking approach I've adopted in the patch?
> 
> Only the one I noted in my reply the first time around

Hi Alan,

I've hunted, but I can't seem to find this reply. :-/

> which is that you
> can't permit tty->low_latency=1 unless your tty receive path is not an
> IRQ path. From a locking point of view the change makes sense anyway.

I ran into this on the PREEMPT_RT kernel which always uses
tty->low_latency and converts the interrupt handler into a thread.

There is a follow-on patch for RT only to address a sleeping while
atomic bug in pch_console_write(), but I felt _this_ locking structure
change was appropriate for mainline.

> 
> Going back over it your console locking also needs care - an oops or
> printk within the areas the private lock covers will hang the box. That
> should also probably be a trylock style lock as with the other lock on
> that path

I presume you are referring to pch_console_write()?

> static void
> pch_console_write(struct console *co, const char *s, unsigned int count)
> {
> 	struct eg20t_port *priv;
> 	unsigned long flags;
> 	u8 ier;
> 	int locked = 1;
> 
> 	priv = pch_uart_ports[co->index];
> 
> 	touch_nmi_watchdog();
> 
> 	local_irq_save(flags);
> 	spin_lock(&priv->lock);
> 	if (priv->port.sysrq) {
> 		/* serial8250_handle_port() already took the lock */
> 		locked = 0;
> 	} else if (oops_in_progress) {
> 		locked = spin_trylock(&priv->port.lock);
> 	} else
> 		spin_lock(&priv->port.lock);


I see, the oops_in_progress test right? My thinking was that the
oops_in_progress was only relevant to the port.lock as that could be
taken outside of the pch_uart driver, while the priv.lock is only used
within the driver. But, as the oops uses the pch_console_write itself, I
can see the recursive spinlock failure case there.

As for the printk, it seems the 8250 driver would also suffer from that
in the serial8250_console_write function on the port.lock, and it does
not make any allowances for printk.

I would like to hold the priv.lock for a smaller window, but ordering
requires that I take it prior to the port.lock.

So I can test for oops_in_progress on the priv->lock too, but that won't
address the printk issue. Is the oops the bigger concern?

-- 
Darren Hart
Intel Open Source Technology Center
Yocto Project - Linux Kernel

Re: [RFC PATCH] pch_uart: Add eg20t_port lock field, avoid recursive spinlocks

[Alan Cox]

> I see, the oops_in_progress test right? My thinking was that the
> oops_in_progress was only relevant to the port.lock as that could be
> taken outside of the pch_uart driver, while the priv.lock is only used
> within the driver. But, as the oops uses the pch_console_write itself, I
> can see the recursive spinlock failure case there.

Until your driver crashes...

> As for the printk, it seems the 8250 driver would also suffer from that
> in the serial8250_console_write function on the port.lock, and it does
> not make any allowances for printk.

I think 8250 probably wants fixing too then!

> 
> I would like to hold the priv.lock for a smaller window, but ordering
> requires that I take it prior to the port.lock.
> 
> So I can test for oops_in_progress on the priv->lock too, but that won't
> address the printk issue. Is the oops the bigger concern?

the oops is the main one - a printk would have to be in driver as a
screwup, and you can force an oops on a stall so pick it up later

Alan