* [PATCH 2/2] nvme: export and use previously added helper
2024-02-12 4:26 ` [PATCH 1/2] nvme-core: add ctrl state transition debug helper Chaitanya Kulkarni
@ 2024-02-12 4:26 ` Chaitanya Kulkarni
2024-02-12 6:54 ` Nitesh Shetty
2024-03-07 8:03 ` Sagi Grimberg
2024-02-12 6:53 ` [PATCH 1/2] nvme-core: add ctrl state transition debug helper Nitesh Shetty
` (3 subsequent siblings)
4 siblings, 2 replies; 11+ messages in thread
From: Chaitanya Kulkarni @ 2024-02-12 4:26 UTC (permalink / raw)
To: linux-nvme; +Cc: kbusch, hch, sagi, Chaitanya Kulkarni
Export nvme_ctrl_state_str() and use it in the host/sysfs.c:
nvme_sysfs_show_state().
Signed-off-by: Chaitanya Kulkarni <kch@nvidia.com>
---
drivers/nvme/host/core.c | 2 +-
drivers/nvme/host/nvme.h | 1 +
drivers/nvme/host/sysfs.c | 16 ++--------------
3 files changed, 4 insertions(+), 15 deletions(-)
diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index daeb2409f989..acde9e377c00 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -544,7 +544,7 @@ void nvme_cancel_admin_tagset(struct nvme_ctrl *ctrl)
}
EXPORT_SYMBOL_GPL(nvme_cancel_admin_tagset);
-static const char *nvme_ctrl_state_str(enum nvme_ctrl_state st)
+const char *nvme_ctrl_state_str(enum nvme_ctrl_state st)
{
static const char *const str[] = {
[NVME_CTRL_NEW] = "new",
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index 3ca486fe927c..3584c3861d6c 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -1094,6 +1094,7 @@ void nvme_passthru_end(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u32 effects,
struct nvme_ctrl *nvme_ctrl_from_file(struct file *file);
struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid);
void nvme_put_ns(struct nvme_ns *ns);
+const char *nvme_ctrl_state_str(enum nvme_ctrl_state st);
static inline bool nvme_multi_css(struct nvme_ctrl *ctrl)
{
diff --git a/drivers/nvme/host/sysfs.c b/drivers/nvme/host/sysfs.c
index f2832f70e7e0..a067feaba5ae 100644
--- a/drivers/nvme/host/sysfs.c
+++ b/drivers/nvme/host/sysfs.c
@@ -368,21 +368,9 @@ static ssize_t nvme_sysfs_show_state(struct device *dev,
char *buf)
{
struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
- unsigned state = (unsigned)nvme_ctrl_state(ctrl);
- static const char *const state_name[] = {
- [NVME_CTRL_NEW] = "new",
- [NVME_CTRL_LIVE] = "live",
- [NVME_CTRL_RESETTING] = "resetting",
- [NVME_CTRL_CONNECTING] = "connecting",
- [NVME_CTRL_DELETING] = "deleting",
- [NVME_CTRL_DELETING_NOIO]= "deleting (no IO)",
- [NVME_CTRL_DEAD] = "dead",
- };
-
- if (state < ARRAY_SIZE(state_name) && state_name[state])
- return sysfs_emit(buf, "%s\n", state_name[state]);
- return sysfs_emit(buf, "unknown state\n");
+ return sysfs_emit(buf, "%s\n",
+ nvme_ctrl_state_str(nvme_ctrl_state(ctrl)));
}
static DEVICE_ATTR(state, S_IRUGO, nvme_sysfs_show_state, NULL);
--
2.40.0
^ permalink raw reply related [flat|nested] 11+ messages in thread* Re: [PATCH 2/2] nvme: export and use previously added helper
2024-02-12 4:26 ` [PATCH 2/2] nvme: export and use previously added helper Chaitanya Kulkarni
@ 2024-02-12 6:54 ` Nitesh Shetty
2024-03-07 8:03 ` Sagi Grimberg
1 sibling, 0 replies; 11+ messages in thread
From: Nitesh Shetty @ 2024-02-12 6:54 UTC (permalink / raw)
To: Chaitanya Kulkarni; +Cc: linux-nvme, kbusch, hch, sagi
[-- Attachment #1: Type: text/plain, Size: 2535 bytes --]
On 11/02/24 08:26PM, Chaitanya Kulkarni wrote:
>Export nvme_ctrl_state_str() and use it in the host/sysfs.c:
>nvme_sysfs_show_state().
>
>Signed-off-by: Chaitanya Kulkarni <kch@nvidia.com>
>---
> drivers/nvme/host/core.c | 2 +-
> drivers/nvme/host/nvme.h | 1 +
> drivers/nvme/host/sysfs.c | 16 ++--------------
> 3 files changed, 4 insertions(+), 15 deletions(-)
>
>diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
>index daeb2409f989..acde9e377c00 100644
>--- a/drivers/nvme/host/core.c
>+++ b/drivers/nvme/host/core.c
>@@ -544,7 +544,7 @@ void nvme_cancel_admin_tagset(struct nvme_ctrl *ctrl)
> }
> EXPORT_SYMBOL_GPL(nvme_cancel_admin_tagset);
>
>-static const char *nvme_ctrl_state_str(enum nvme_ctrl_state st)
>+const char *nvme_ctrl_state_str(enum nvme_ctrl_state st)
> {
> static const char *const str[] = {
> [NVME_CTRL_NEW] = "new",
>diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
>index 3ca486fe927c..3584c3861d6c 100644
>--- a/drivers/nvme/host/nvme.h
>+++ b/drivers/nvme/host/nvme.h
>@@ -1094,6 +1094,7 @@ void nvme_passthru_end(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u32 effects,
> struct nvme_ctrl *nvme_ctrl_from_file(struct file *file);
> struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid);
> void nvme_put_ns(struct nvme_ns *ns);
>+const char *nvme_ctrl_state_str(enum nvme_ctrl_state st);
>
> static inline bool nvme_multi_css(struct nvme_ctrl *ctrl)
> {
>diff --git a/drivers/nvme/host/sysfs.c b/drivers/nvme/host/sysfs.c
>index f2832f70e7e0..a067feaba5ae 100644
>--- a/drivers/nvme/host/sysfs.c
>+++ b/drivers/nvme/host/sysfs.c
>@@ -368,21 +368,9 @@ static ssize_t nvme_sysfs_show_state(struct device *dev,
> char *buf)
> {
> struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
>- unsigned state = (unsigned)nvme_ctrl_state(ctrl);
>- static const char *const state_name[] = {
>- [NVME_CTRL_NEW] = "new",
>- [NVME_CTRL_LIVE] = "live",
>- [NVME_CTRL_RESETTING] = "resetting",
>- [NVME_CTRL_CONNECTING] = "connecting",
>- [NVME_CTRL_DELETING] = "deleting",
>- [NVME_CTRL_DELETING_NOIO]= "deleting (no IO)",
>- [NVME_CTRL_DEAD] = "dead",
>- };
>-
>- if (state < ARRAY_SIZE(state_name) && state_name[state])
>- return sysfs_emit(buf, "%s\n", state_name[state]);
>
>- return sysfs_emit(buf, "unknown state\n");
>+ return sysfs_emit(buf, "%s\n",
>+ nvme_ctrl_state_str(nvme_ctrl_state(ctrl)));
> }
>
> static DEVICE_ATTR(state, S_IRUGO, nvme_sysfs_show_state, NULL);
>--
>2.40.0
>
>
Reviewed-by: Nitesh Shetty <nj.shetty@samsung.com>
[-- Attachment #2: Type: text/plain, Size: 0 bytes --]
^ permalink raw reply [flat|nested] 11+ messages in thread* Re: [PATCH 2/2] nvme: export and use previously added helper
2024-02-12 4:26 ` [PATCH 2/2] nvme: export and use previously added helper Chaitanya Kulkarni
2024-02-12 6:54 ` Nitesh Shetty
@ 2024-03-07 8:03 ` Sagi Grimberg
1 sibling, 0 replies; 11+ messages in thread
From: Sagi Grimberg @ 2024-03-07 8:03 UTC (permalink / raw)
To: Chaitanya Kulkarni, linux-nvme; +Cc: kbusch, hch
Reviewed-by: Sagi Grimberg <sagi@grimberg.me>
^ permalink raw reply [flat|nested] 11+ messages in thread
* Re: [PATCH 1/2] nvme-core: add ctrl state transition debug helper
2024-02-12 4:26 ` [PATCH 1/2] nvme-core: add ctrl state transition debug helper Chaitanya Kulkarni
2024-02-12 4:26 ` [PATCH 2/2] nvme: export and use previously added helper Chaitanya Kulkarni
@ 2024-02-12 6:53 ` Nitesh Shetty
2024-02-21 5:34 ` Chaitanya Kulkarni
` (2 subsequent siblings)
4 siblings, 0 replies; 11+ messages in thread
From: Nitesh Shetty @ 2024-02-12 6:53 UTC (permalink / raw)
To: Chaitanya Kulkarni; +Cc: linux-nvme, kbusch, hch, sagi
[-- Attachment #1: Type: text/plain, Size: 3107 bytes --]
On 11/02/24 08:26PM, Chaitanya Kulkarni wrote:
>NVMe controller state machine has total 7 states and 13 state transition
>arcs. Debugging NVMeOF problems in the field is not straight-froward,
>since it involves complex combination of connect/reconnect/kato/timeout
>handlers etc scenarios.
>We already have a helper in sysfs.c that reads the controller state, but
>one has to constantly read the state in order understand the complete
>state transition which is very inconvenient.
>
>It is often helpful to know full state transition of controller when
>dealing with NVMeOF issues at the time of analyzing the trace.
>
>Add a helper that allows us to decode and print each controller state
>transition :-
>
>blktests (master) # dmesg -c | grep nvme_change_ctrl_state
>blktests (master) # nvme_trtype=tcp ./check nvme/048
>nvme/048 (Test queue count changes on reconnect) [passed]
> runtime 6.264s ... 5.240s
>blktests (master) # dmesg -c | grep nvme_change_ctrl_state
>[17080.988689] nvme nvme0: nvme_change_ctrl_state new -> connecting
>[17081.006623] nvme nvme0: nvme_change_ctrl_state connecting -> live
>[17081.038313] nvme nvme0: nvme_change_ctrl_state live -> resetting
>[17081.040730] nvme nvme0: nvme_change_ctrl_state resetting -> connecting
>[17083.056750] nvme nvme0: nvme_change_ctrl_state connecting -> live
>[17083.075906] nvme nvme0: nvme_change_ctrl_state live -> resetting
>[17083.076112] nvme nvme0: nvme_change_ctrl_state resetting -> connecting
>[17085.105270] nvme nvme0: nvme_change_ctrl_state connecting -> live
>[17086.126484] nvme nvme0: nvme_change_ctrl_state live -> deleting
>[17086.126506] nvme nvme0: nvme_change_ctrl_state deleting -> deleting (no IO)
>blktests (master) #
>
>Signed-off-by: Chaitanya Kulkarni <kch@nvidia.com>
>---
>
> drivers/nvme/host/core.c | 18 ++++++++++++++++++
> 1 file changed, 18 insertions(+)
>
>diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
>index 20c0c141fcc0..daeb2409f989 100644
>--- a/drivers/nvme/host/core.c
>+++ b/drivers/nvme/host/core.c
>@@ -544,6 +544,21 @@ void nvme_cancel_admin_tagset(struct nvme_ctrl *ctrl)
> }
> EXPORT_SYMBOL_GPL(nvme_cancel_admin_tagset);
>
>+static const char *nvme_ctrl_state_str(enum nvme_ctrl_state st)
>+{
>+ static const char *const str[] = {
>+ [NVME_CTRL_NEW] = "new",
>+ [NVME_CTRL_LIVE] = "live",
>+ [NVME_CTRL_RESETTING] = "resetting",
>+ [NVME_CTRL_CONNECTING] = "connecting",
>+ [NVME_CTRL_DELETING] = "deleting",
>+ [NVME_CTRL_DELETING_NOIO] = "deleting (no IO)",
>+ [NVME_CTRL_DEAD] = "dead",
>+ };
>+
>+ return st < ARRAY_SIZE(str) && str[st] ? str[st] : "unknown state";
>+}
>+
> bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
> enum nvme_ctrl_state new_state)
> {
>@@ -621,6 +636,9 @@ bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
> }
>
> if (changed) {
>+ dev_dbg(ctrl->device, "%s %s -> %s\n", __func__,
>+ nvme_ctrl_state_str(old_state),
>+ nvme_ctrl_state_str(new_state));
> WRITE_ONCE(ctrl->state, new_state);
> wake_up_all(&ctrl->state_wq);
> }
>--
>2.40.0
>
>
Revieved-by: Nitesh Shetty <nj.shetty@samsung.com>
[-- Attachment #2: Type: text/plain, Size: 0 bytes --]
^ permalink raw reply [flat|nested] 11+ messages in thread* Re: [PATCH 1/2] nvme-core: add ctrl state transition debug helper
2024-02-12 4:26 ` [PATCH 1/2] nvme-core: add ctrl state transition debug helper Chaitanya Kulkarni
2024-02-12 4:26 ` [PATCH 2/2] nvme: export and use previously added helper Chaitanya Kulkarni
2024-02-12 6:53 ` [PATCH 1/2] nvme-core: add ctrl state transition debug helper Nitesh Shetty
@ 2024-02-21 5:34 ` Chaitanya Kulkarni
2024-03-05 4:34 ` Chaitanya Kulkarni
2024-03-05 16:34 ` Keith Busch
2024-03-07 8:03 ` Sagi Grimberg
4 siblings, 1 reply; 11+ messages in thread
From: Chaitanya Kulkarni @ 2024-02-21 5:34 UTC (permalink / raw)
To: kbusch@kernel.org, sagi@grimberg.me, hch@lst.de
Cc: linux-nvme@lists.infradead.org, Chaitanya Kulkarni
On 2/11/24 20:26, Chaitanya Kulkarni wrote:
> NVMe controller state machine has total 7 states and 13 state transition
> arcs. Debugging NVMeOF problems in the field is not straight-froward,
> since it involves complex combination of connect/reconnect/kato/timeout
> handlers etc scenarios.
> We already have a helper in sysfs.c that reads the controller state, but
> one has to constantly read the state in order understand the complete
> state transition which is very inconvenient.
>
> It is often helpful to know full state transition of controller when
> dealing with NVMeOF issues at the time of analyzing the trace.
>
> Add a helper that allows us to decode and print each controller state
> transition :-
>
> blktests (master) # dmesg -c | grep nvme_change_ctrl_state
> blktests (master) # nvme_trtype=tcp ./check nvme/048
> nvme/048 (Test queue count changes on reconnect) [passed]
> runtime 6.264s ... 5.240s
> blktests (master) # dmesg -c | grep nvme_change_ctrl_state
> [17080.988689] nvme nvme0: nvme_change_ctrl_state new -> connecting
> [17081.006623] nvme nvme0: nvme_change_ctrl_state connecting -> live
> [17081.038313] nvme nvme0: nvme_change_ctrl_state live -> resetting
> [17081.040730] nvme nvme0: nvme_change_ctrl_state resetting -> connecting
> [17083.056750] nvme nvme0: nvme_change_ctrl_state connecting -> live
> [17083.075906] nvme nvme0: nvme_change_ctrl_state live -> resetting
> [17083.076112] nvme nvme0: nvme_change_ctrl_state resetting -> connecting
> [17085.105270] nvme nvme0: nvme_change_ctrl_state connecting -> live
> [17086.126484] nvme nvme0: nvme_change_ctrl_state live -> deleting
> [17086.126506] nvme nvme0: nvme_change_ctrl_state deleting -> deleting (no IO)
> blktests (master) #
>
> Signed-off-by: Chaitanya Kulkarni <kch@nvidia.com>
> ---
>
gentle ping ...
-ck
^ permalink raw reply [flat|nested] 11+ messages in thread
* Re: [PATCH 1/2] nvme-core: add ctrl state transition debug helper
2024-02-21 5:34 ` Chaitanya Kulkarni
@ 2024-03-05 4:34 ` Chaitanya Kulkarni
0 siblings, 0 replies; 11+ messages in thread
From: Chaitanya Kulkarni @ 2024-03-05 4:34 UTC (permalink / raw)
To: kbusch@kernel.org, sagi@grimberg.me, hch@lst.de
Cc: linux-nvme@lists.infradead.org
On 2/20/24 21:34, Chaitanya Kulkarni wrote:
> On 2/11/24 20:26, Chaitanya Kulkarni wrote:
>> NVMe controller state machine has total 7 states and 13 state transition
>> arcs. Debugging NVMeOF problems in the field is not straight-froward,
>> since it involves complex combination of connect/reconnect/kato/timeout
>> handlers etc scenarios.
>> We already have a helper in sysfs.c that reads the controller state, but
>> one has to constantly read the state in order understand the complete
>> state transition which is very inconvenient.
>>
>> It is often helpful to know full state transition of controller when
>> dealing with NVMeOF issues at the time of analyzing the trace.
>>
>> Add a helper that allows us to decode and print each controller state
>> transition :-
>>
>> blktests (master) # dmesg -c | grep nvme_change_ctrl_state
>> blktests (master) # nvme_trtype=tcp ./check nvme/048
>> nvme/048 (Test queue count changes on reconnect) [passed]
>> runtime 6.264s ... 5.240s
>> blktests (master) # dmesg -c | grep nvme_change_ctrl_state
>> [17080.988689] nvme nvme0: nvme_change_ctrl_state new -> connecting
>> [17081.006623] nvme nvme0: nvme_change_ctrl_state connecting -> live
>> [17081.038313] nvme nvme0: nvme_change_ctrl_state live -> resetting
>> [17081.040730] nvme nvme0: nvme_change_ctrl_state resetting -> connecting
>> [17083.056750] nvme nvme0: nvme_change_ctrl_state connecting -> live
>> [17083.075906] nvme nvme0: nvme_change_ctrl_state live -> resetting
>> [17083.076112] nvme nvme0: nvme_change_ctrl_state resetting -> connecting
>> [17085.105270] nvme nvme0: nvme_change_ctrl_state connecting -> live
>> [17086.126484] nvme nvme0: nvme_change_ctrl_state live -> deleting
>> [17086.126506] nvme nvme0: nvme_change_ctrl_state deleting -> deleting (no IO)
>> blktests (master) #
>>
>> Signed-off-by: Chaitanya Kulkarni <kch@nvidia.com>
>> ---
>>
> gentle ping ...
>
> -ck
gentle ping ...
-ck
>
>
^ permalink raw reply [flat|nested] 11+ messages in thread
* Re: [PATCH 1/2] nvme-core: add ctrl state transition debug helper
2024-02-12 4:26 ` [PATCH 1/2] nvme-core: add ctrl state transition debug helper Chaitanya Kulkarni
` (2 preceding siblings ...)
2024-02-21 5:34 ` Chaitanya Kulkarni
@ 2024-03-05 16:34 ` Keith Busch
2024-03-06 5:21 ` Chaitanya Kulkarni
2024-03-07 8:03 ` Sagi Grimberg
2024-03-07 8:03 ` Sagi Grimberg
4 siblings, 2 replies; 11+ messages in thread
From: Keith Busch @ 2024-03-05 16:34 UTC (permalink / raw)
To: Chaitanya Kulkarni; +Cc: linux-nvme, hch, sagi
On Sun, Feb 11, 2024 at 08:26:40PM -0800, Chaitanya Kulkarni wrote:
> @@ -621,6 +636,9 @@ bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
> }
>
> if (changed) {
> + dev_dbg(ctrl->device, "%s %s -> %s\n", __func__,
> + nvme_ctrl_state_str(old_state),
> + nvme_ctrl_state_str(new_state));
> WRITE_ONCE(ctrl->state, new_state);
> wake_up_all(&ctrl->state_wq);
> }
Turing the system logs up to debug level might get you a whole lot of
unrelated stuff. Could a new trace_event get you what you need instead?
There's more fine grain control on those.
^ permalink raw reply [flat|nested] 11+ messages in thread* Re: [PATCH 1/2] nvme-core: add ctrl state transition debug helper
2024-03-05 16:34 ` Keith Busch
@ 2024-03-06 5:21 ` Chaitanya Kulkarni
2024-03-07 8:03 ` Sagi Grimberg
1 sibling, 0 replies; 11+ messages in thread
From: Chaitanya Kulkarni @ 2024-03-06 5:21 UTC (permalink / raw)
To: Keith Busch, Chaitanya Kulkarni
Cc: linux-nvme@lists.infradead.org, hch@lst.de, sagi@grimberg.me
On 3/5/24 08:34, Keith Busch wrote:
> On Sun, Feb 11, 2024 at 08:26:40PM -0800, Chaitanya Kulkarni wrote:
>> @@ -621,6 +636,9 @@ bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
>> }
>>
>> if (changed) {
>> + dev_dbg(ctrl->device, "%s %s -> %s\n", __func__,
>> + nvme_ctrl_state_str(old_state),
>> + nvme_ctrl_state_str(new_state));
>> WRITE_ONCE(ctrl->state, new_state);
>> wake_up_all(&ctrl->state_wq);
>> }
> Turing the system logs up to debug level might get you a whole lot of
> unrelated stuff. Could a new trace_event get you what you need instead?
> There's more fine grain control on those.
totally agree with you, I have patches to add few more tracepoints
I'll add v2 to that series and post soon, thanks for the suggestion ...
-ck
^ permalink raw reply [flat|nested] 11+ messages in thread* Re: [PATCH 1/2] nvme-core: add ctrl state transition debug helper
2024-03-05 16:34 ` Keith Busch
2024-03-06 5:21 ` Chaitanya Kulkarni
@ 2024-03-07 8:03 ` Sagi Grimberg
1 sibling, 0 replies; 11+ messages in thread
From: Sagi Grimberg @ 2024-03-07 8:03 UTC (permalink / raw)
To: Keith Busch, Chaitanya Kulkarni; +Cc: linux-nvme, hch
On 05/03/2024 18:34, Keith Busch wrote:
> On Sun, Feb 11, 2024 at 08:26:40PM -0800, Chaitanya Kulkarni wrote:
>> @@ -621,6 +636,9 @@ bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
>> }
>>
>> if (changed) {
>> + dev_dbg(ctrl->device, "%s %s -> %s\n", __func__,
>> + nvme_ctrl_state_str(old_state),
>> + nvme_ctrl_state_str(new_state));
>> WRITE_ONCE(ctrl->state, new_state);
>> wake_up_all(&ctrl->state_wq);
>> }
> Turing the system logs up to debug level might get you a whole lot of
> unrelated stuff. Could a new trace_event get you what you need instead?
> There's more fine grain control on those.
>
I actually think its useful here as is, dynamic debug can also turn on
debug per function.
^ permalink raw reply [flat|nested] 11+ messages in thread
* Re: [PATCH 1/2] nvme-core: add ctrl state transition debug helper
2024-02-12 4:26 ` [PATCH 1/2] nvme-core: add ctrl state transition debug helper Chaitanya Kulkarni
` (3 preceding siblings ...)
2024-03-05 16:34 ` Keith Busch
@ 2024-03-07 8:03 ` Sagi Grimberg
4 siblings, 0 replies; 11+ messages in thread
From: Sagi Grimberg @ 2024-03-07 8:03 UTC (permalink / raw)
To: Chaitanya Kulkarni, linux-nvme; +Cc: kbusch, hch
Reviewed-by: Sagi Grimberg <sagi@grimberg.me>
^ permalink raw reply [flat|nested] 11+ messages in thread