* [PATCH net-next 1/2] fs/crashdd: add API to collect hardware dump in second kernel
2018-03-23 8:30 [PATCH net-next 0/2] kernel: add support to collect hardware logs in crash recovery kernel Rahul Lakkireddy
@ 2018-03-23 8:31 ` Rahul Lakkireddy
2018-03-24 12:29 ` kbuild test robot
2018-03-24 12:42 ` kbuild test robot
2018-03-23 8:31 ` [PATCH net-next 2/2] cxgb4: " Rahul Lakkireddy
2018-03-23 12:46 ` [PATCH net-next 0/2] kernel: add support to collect hardware logs in crash recovery kernel Andrew Lunn
2 siblings, 2 replies; 6+ messages in thread
From: Rahul Lakkireddy @ 2018-03-23 8:31 UTC (permalink / raw)
To: netdev, linux-fsdevel, kexec, linux-kernel
Cc: davem, viro, ebiederm, stephen, akpm, torvalds, ganeshgr,
nirranjan, indranil, Rahul Lakkireddy
Add a new module crashdd that exports the /sys/kernel/crashdd/
directory in second kernel, containing collected hardware/firmware
dumps.
The sequence of actions done by device drivers to append their device
specific hardware/firmware logs to /sys/kernel/crashdd/ directory are
as follows:
1. During probe (before hardware is initialized), device drivers
register to the crashdd module (via crashdd_add_dump()), with
callback function, along with buffer size and log name needed for
firmware/hardware log collection.
2. Crashdd creates a driver's directory under
/sys/kernel/crashdd/<driver>. Then, it allocates the buffer with
requested size and invokes the device driver's registered callback
function.
3. Device driver collects all hardware/firmware logs into the buffer
and returns control back to crashdd.
4. Crashdd exposes the buffer as a binary file via
/sys/kernel/crashdd/<driver>/<dump_file>.
Suggested-by: Eric Biederman <ebiederm@xmission.com>.
Suggested-by: Stephen Hemminger <stephen@networkplumber.org>
Signed-off-by: Rahul Lakkireddy <rahul.lakkireddy@chelsio.com>
Signed-off-by: Ganesh Goudar <ganeshgr@chelsio.com>
---
Changes since rfc v2:
- Moved exporting crashdd from procfs to sysfs. Suggested by
Stephen Hemminger <stephen@networkplumber.org>
- Moved code from fs/proc/crashdd.c to fs/crashdd/ directory.
- Replaced all proc API with sysfs API and updated comments.
- Calling driver callback before creating the binary file under
crashdd sysfs.
- Changed binary dump file permission from S_IRUSR to S_IRUGO.
- Changed module name from CRASH_DRIVER_DUMP to CRASH_DEVICE_DUMP.
rfc v2:
- Collecting logs in 2nd kernel instead of during kernel panic.
Suggested by Eric Biederman <ebiederm@xmission.com>.
- Patch added in this series.
fs/Kconfig | 1 +
fs/Makefile | 1 +
fs/crashdd/Kconfig | 10 ++
fs/crashdd/Makefile | 3 +
fs/crashdd/crashdd.c | 234 ++++++++++++++++++++++++++++++++++++++++++
fs/crashdd/crashdd_internal.h | 24 +++++
include/linux/crashdd.h | 24 +++++
7 files changed, 297 insertions(+)
create mode 100644 fs/crashdd/Kconfig
create mode 100644 fs/crashdd/Makefile
create mode 100644 fs/crashdd/crashdd.c
create mode 100644 fs/crashdd/crashdd_internal.h
create mode 100644 include/linux/crashdd.h
diff --git a/fs/Kconfig b/fs/Kconfig
index bc821a86d965..aae1c55a7dad 100644
--- a/fs/Kconfig
+++ b/fs/Kconfig
@@ -208,6 +208,7 @@ config ARCH_HAS_GIGANTIC_PAGE
source "fs/configfs/Kconfig"
source "fs/efivarfs/Kconfig"
+source "fs/crashdd/Kconfig"
endmenu
diff --git a/fs/Makefile b/fs/Makefile
index add789ea270a..ff398a44f611 100644
--- a/fs/Makefile
+++ b/fs/Makefile
@@ -128,3 +128,4 @@ obj-y += exofs/ # Multiple modules
obj-$(CONFIG_CEPH_FS) += ceph/
obj-$(CONFIG_PSTORE) += pstore/
obj-$(CONFIG_EFIVAR_FS) += efivarfs/
+obj-$(CONFIG_CRASH_DEVICE_DUMP) += crashdd/
diff --git a/fs/crashdd/Kconfig b/fs/crashdd/Kconfig
new file mode 100644
index 000000000000..5db9c7c98c17
--- /dev/null
+++ b/fs/crashdd/Kconfig
@@ -0,0 +1,10 @@
+config CRASH_DEVICE_DUMP
+ bool "Crash Kernel Device Hardware/Firmware Logs"
+ depends on SYSFS && CRASH_DUMP
+ default y
+ ---help---
+ Device drivers can collect the device specific snapshot of
+ their hardware or firmware before they are initialized in
+ crash recovery kernel. If you say Y here a tree of device
+ specific dumps will be made available under /sys/kernel/crashdd/
+ directory.
diff --git a/fs/crashdd/Makefile b/fs/crashdd/Makefile
new file mode 100644
index 000000000000..8dbf946c0ea4
--- /dev/null
+++ b/fs/crashdd/Makefile
@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: GPL-2.0
+
+obj-y := crashdd.o
diff --git a/fs/crashdd/crashdd.c b/fs/crashdd/crashdd.c
new file mode 100644
index 000000000000..73882ff7722e
--- /dev/null
+++ b/fs/crashdd/crashdd.c
@@ -0,0 +1,234 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2018 Chelsio Communications, Inc. All rights reserved. */
+
+#include <linux/vmalloc.h>
+#include <linux/crash_dump.h>
+#include <linux/crashdd.h>
+
+#include "crashdd_internal.h"
+
+static LIST_HEAD(crashdd_list);
+static DEFINE_MUTEX(crashdd_mutex);
+
+#define CRASHDD_SYSFS_MODE 444 /* S_IRUGO */
+static struct kobject *crashdd_kobj;
+
+static ssize_t crashdd_read(struct file *filp, struct kobject *kobj,
+ struct bin_attribute *bin_attr,
+ char *buf, loff_t fpos, size_t count)
+{
+ struct crashdd_dump_node *dump = bin_attr->private;
+
+ memcpy(buf, dump->buf + fpos, count);
+ return count;
+}
+
+static struct kobject *crashdd_mkdir(const char *name)
+{
+ return kobject_create_and_add(name, crashdd_kobj);
+}
+
+static int crashdd_add_file(struct kobject *kobj, const char *name,
+ struct crashdd_dump_node *dump)
+{
+ dump->bin_attr.attr.name = name;
+ dump->bin_attr.attr.mode = CRASHDD_SYSFS_MODE;
+ dump->bin_attr.size = dump->size;
+ dump->bin_attr.read = crashdd_read;
+ dump->bin_attr.private = dump;
+
+ return sysfs_create_bin_file(kobj, &dump->bin_attr);
+}
+
+static void crashdd_rmdir(struct kobject *kobj)
+{
+ kobject_put(kobj);
+}
+
+/**
+ * crashdd_init_driver - create a sysfs driver entry.
+ * @name: Name of the directory.
+ *
+ * Creates a directory under /sys/kernel/crashdd/ with @name. Allocates
+ * and saves the sysfs entry. The sysfs entry is added to the global
+ * list and then returned to the caller. On failure, returns NULL.
+ */
+static struct crashdd_driver_node *crashdd_init_driver(const char *name)
+{
+ struct crashdd_driver_node *node;
+
+ node = vzalloc(sizeof(*node));
+ if (!node)
+ return NULL;
+
+ /* Create a driver's directory under /sys/kernel/crashdd/ */
+ node->kobj = crashdd_mkdir(name);
+ if (!node->kobj) {
+ vfree(node);
+ return NULL;
+ }
+
+ atomic_set(&node->refcnt, 1);
+
+ /* Initialize the list of dumps that go under this driver's
+ * directory.
+ */
+ INIT_LIST_HEAD(&node->dump_list);
+
+ /* Add the driver's entry to global list */
+ mutex_lock(&crashdd_mutex);
+ list_add_tail(&node->list, &crashdd_list);
+ mutex_unlock(&crashdd_mutex);
+
+ return node;
+}
+
+/**
+ * crashdd_get_driver - get an exisiting sysfs driver entry.
+ * @name: Name of the directory.
+ *
+ * Searches and fetches a sysfs entry having @name. If @name is
+ * found, then the reference count is incremented and the entry
+ * is returned. If @name is not found, NULL is returned.
+ */
+static struct crashdd_driver_node *crashdd_get_driver(const char *name)
+{
+ struct crashdd_driver_node *node;
+ int found = 0;
+
+ /* Search for an existing driver sysfs entry having @name */
+ mutex_lock(&crashdd_mutex);
+ list_for_each_entry(node, &crashdd_list, list) {
+ if (!strcmp(node->kobj->name, name)) {
+ atomic_inc(&node->refcnt);
+ found = 1;
+ break;
+ }
+ }
+ mutex_unlock(&crashdd_mutex);
+
+ if (found)
+ return node;
+
+ /* No driver with @name found */
+ return NULL;
+}
+
+/**
+ * crashdd_put_driver - put an exisiting sysfs driver entry.
+ * @node: driver sysfs entry.
+ *
+ * Decrement @node reference count. If there are no dumps left under it,
+ * delete the sysfs directory and remove it from the global list.
+ */
+static void crashdd_put_driver(struct crashdd_driver_node *node)
+{
+ mutex_lock(&crashdd_mutex);
+ if (atomic_dec_and_test(&node->refcnt)) {
+ /* Delete @node driver entry if it has no dumps under it */
+ crashdd_rmdir(node->kobj);
+ list_del(&node->list);
+ }
+ mutex_unlock(&crashdd_mutex);
+}
+
+/**
+ * crashdd_add_dump - Allocate a directory under /sys/kernel/crashdd/ and
+ * add the dump to it.
+ * @driver_name: directory name under which the dump should be added.
+ * @data: dump info.
+ *
+ * Search for /sys/kernel/crashdd/<@driver_name>/ directory. If not found,
+ * allocate a new directory under /sys/kernel/crashdd/ with @driver_name.
+ * Allocate the dump file's context and invoke the calling driver's dump
+ * collect routine. Once collection is done, add the dump under
+ * /sys/kernel/crashdd/<@driver_name>/ directory.
+ */
+int crashdd_add_dump(const char *driver_name, struct crashdd_data *data)
+{
+ struct crashdd_driver_node *node;
+ struct crashdd_dump_node *dump;
+ void *buf = NULL;
+ int ret;
+
+ if (!driver_name || !strlen(driver_name) ||
+ !data || !strlen(data->name) ||
+ !data->crashdd_callback || !data->size)
+ return -EINVAL;
+
+ /* Get a driver sysfs entry with specified name. */
+ node = crashdd_get_driver(driver_name);
+ if (!node) {
+ /* No driver sysfs entry found with specified name.
+ * So create a new one
+ */
+ node = crashdd_init_driver(driver_name);
+ if (!node)
+ return -ENOMEM;
+ }
+
+ dump = vzalloc(sizeof(*dump));
+ if (!dump) {
+ ret = -ENOMEM;
+ goto out_err;
+ }
+
+ /* Allocate buffer for driver's to write their dumps */
+ buf = vzalloc(data->size);
+ if (!buf) {
+ ret = -ENOMEM;
+ goto out_err;
+ }
+
+ /* Invoke the driver's dump collection routing */
+ ret = data->crashdd_callback(data, buf);
+ if (ret)
+ goto out_err;
+
+ dump->buf = buf;
+ dump->size = data->size;
+
+ /* Add a binary file under /sys/kernel/crashdd/@driver_name/ */
+ ret = crashdd_add_file(node->kobj, data->name, dump);
+ if (ret)
+ goto out_err;
+
+ /* Add the dump to driver sysfs list */
+ mutex_lock(&crashdd_mutex);
+ list_add_tail(&dump->list, &node->dump_list);
+ atomic_inc(&node->refcnt);
+ mutex_unlock(&crashdd_mutex);
+
+ /* Return back the driver sysfs reference */
+ crashdd_put_driver(node);
+ return 0;
+
+out_err:
+ if (buf)
+ vfree(buf);
+
+ if (dump)
+ vfree(dump);
+
+ crashdd_put_driver(node);
+ return ret;
+}
+EXPORT_SYMBOL(crashdd_add_dump);
+
+/* Init function for crash device dump module. */
+static int __init crashdd_init(void)
+{
+ /*
+ * Only export this directory in 2nd kernel.
+ */
+ if (!is_kdump_kernel())
+ return 0;
+
+ /* Create /sys/kernel/crashdd/ directory */
+ crashdd_kobj = kobject_create_and_add("crashdd", kernel_kobj);
+ if (!crashdd_kobj)
+ return -ENOMEM;
+
+ return 0;
+}
+fs_initcall(crashdd_init);
diff --git a/fs/crashdd/crashdd_internal.h b/fs/crashdd/crashdd_internal.h
new file mode 100644
index 000000000000..9162d1a4264b
--- /dev/null
+++ b/fs/crashdd/crashdd_internal.h
@@ -0,0 +1,24 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2018 Chelsio Communications, Inc. All rights reserved. */
+
+#ifndef CRASH_DEVICE_DUMP_INTERNAL_H
+#define CRASH_DEVICE_DUMP_INTERNAL_H
+
+/* Binary dump file's context internal to crashdd */
+struct crashdd_dump_node {
+ /* Pointer to list of dumps under the driver sysfs entry */
+ struct list_head list;
+ void *buf; /* Buffer containing device's dump */
+ unsigned long size; /* Size of the buffer */
+ struct bin_attribute bin_attr; /* Binary dump file's attributes */
+};
+
+/* Driver sysfs entry internal to crashdd */
+struct crashdd_driver_node {
+ /* Pointer to global list of driver sysfs entries */
+ struct list_head list;
+ struct list_head dump_list; /* List of dumps under this driver */
+ atomic_t refcnt; /* Number of dumps under this directory */
+ struct kobject *kobj; /* Pointer to driver sysfs kobject */
+};
+#endif /* CRASH_DEVICE_DUMP_INTERNAL_H */
diff --git a/include/linux/crashdd.h b/include/linux/crashdd.h
new file mode 100644
index 000000000000..edaba8424019
--- /dev/null
+++ b/include/linux/crashdd.h
@@ -0,0 +1,24 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2018 Chelsio Communications, Inc. All rights reserved. */
+
+#ifndef CRASH_DEVICE_DUMP_H
+#define CRASH_DEVICE_DUMP_H
+
+/* Max dump name length */
+#define CRASHDD_NAME_LENGTH 32
+
+/* Device Dump information to be filled by drivers */
+struct crashdd_data {
+ char name[CRASHDD_NAME_LENGTH]; /* Unique name of the dump */
+ unsigned long size; /* Size of the dump */
+ /* Driver's registered callback to be invoked to collect dump */
+ int (*crashdd_callback)(struct crashdd_data *data, void *buf);
+};
+
+#ifdef CONFIG_CRASH_DEVICE_DUMP
+int crashdd_add_dump(const char *driver_name, struct crashdd_data *data);
+#else
+#define crashdd_add_dump(x, y) 0
+#endif /* CONFIG_CRASH_DEVICE_DUMP */
+
+#endif /* CRASH_DEVICE_DUMP_H */
--
2.14.1
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH net-next 2/2] cxgb4: collect hardware dump in second kernel
2018-03-23 8:30 [PATCH net-next 0/2] kernel: add support to collect hardware logs in crash recovery kernel Rahul Lakkireddy
2018-03-23 8:31 ` [PATCH net-next 1/2] fs/crashdd: add API to collect hardware dump in second kernel Rahul Lakkireddy
@ 2018-03-23 8:31 ` Rahul Lakkireddy
2018-03-23 12:46 ` [PATCH net-next 0/2] kernel: add support to collect hardware logs in crash recovery kernel Andrew Lunn
2 siblings, 0 replies; 6+ messages in thread
From: Rahul Lakkireddy @ 2018-03-23 8:31 UTC (permalink / raw)
To: netdev, linux-fsdevel, kexec, linux-kernel
Cc: davem, viro, ebiederm, stephen, akpm, torvalds, ganeshgr,
nirranjan, indranil, Rahul Lakkireddy
Register callback to collect hardware/firmware dumps in second kernel
before hardware/firmware is initialized. The dumps for each device
will be available under /sys/kernel/crashdd/cxgb4/ directory in second
kernel.
Signed-off-by: Rahul Lakkireddy <rahul.lakkireddy@chelsio.com>
Signed-off-by: Ganesh Goudar <ganeshgr@chelsio.com>
---
Changes since rfc v2:
- Update comments and commit message for sysfs change.
rfc v2:
- Updated dump registration to the new API in patch 1.
drivers/net/ethernet/chelsio/cxgb4/cxgb4.h | 4 ++++
drivers/net/ethernet/chelsio/cxgb4/cxgb4_cudbg.c | 25 ++++++++++++++++++++++++
drivers/net/ethernet/chelsio/cxgb4/cxgb4_cudbg.h | 3 +++
drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c | 12 ++++++++++++
4 files changed, 44 insertions(+)
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
index 688f95440af2..79a123bfb628 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
@@ -50,6 +50,7 @@
#include <linux/net_tstamp.h>
#include <linux/ptp_clock_kernel.h>
#include <linux/ptp_classify.h>
+#include <linux/crashdd.h>
#include <asm/io.h>
#include "t4_chip_type.h"
#include "cxgb4_uld.h"
@@ -964,6 +965,9 @@ struct adapter {
struct hma_data hma;
struct srq_data *srq;
+
+ /* Dump buffer for collecting logs in kdump kernel */
+ struct crashdd_data crashdd_buf;
};
/* Support for "sched-class" command to allow a TX Scheduling Class to be
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_cudbg.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_cudbg.c
index 143686c60234..ce9f544781af 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_cudbg.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_cudbg.c
@@ -488,3 +488,28 @@ void cxgb4_init_ethtool_dump(struct adapter *adapter)
adapter->eth_dump.version = adapter->params.fw_vers;
adapter->eth_dump.len = 0;
}
+
+static int cxgb4_cudbg_crashdd_collect(struct crashdd_data *data, void *buf)
+{
+ struct adapter *adap = container_of(data, struct adapter, crashdd_buf);
+ u32 len = data->size;
+
+ return cxgb4_cudbg_collect(adap, buf, &len, CXGB4_ETH_DUMP_ALL);
+}
+
+int cxgb4_cudbg_crashdd_add_dump(struct adapter *adap)
+{
+ struct crashdd_data *data = &adap->crashdd_buf;
+ u32 len;
+
+ len = sizeof(struct cudbg_hdr) +
+ sizeof(struct cudbg_entity_hdr) * CUDBG_MAX_ENTITY;
+ len += CUDBG_DUMP_BUFF_SIZE;
+
+ data->size = len;
+ snprintf(data->name, sizeof(data->name), "%s_%s", cxgb4_driver_name,
+ adap->name);
+ data->crashdd_callback = cxgb4_cudbg_crashdd_collect;
+
+ return crashdd_add_dump(cxgb4_driver_name, data);
+}
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_cudbg.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_cudbg.h
index ce1ac9a1c878..095c6f04357e 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_cudbg.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_cudbg.h
@@ -41,8 +41,11 @@ enum CXGB4_ETHTOOL_DUMP_FLAGS {
CXGB4_ETH_DUMP_HW = (1 << 1), /* various FW and HW dumps */
};
+#define CXGB4_ETH_DUMP_ALL (CXGB4_ETH_DUMP_MEM | CXGB4_ETH_DUMP_HW)
+
u32 cxgb4_get_dump_length(struct adapter *adap, u32 flag);
int cxgb4_cudbg_collect(struct adapter *adap, void *buf, u32 *buf_size,
u32 flag);
void cxgb4_init_ethtool_dump(struct adapter *adapter);
+int cxgb4_cudbg_crashdd_add_dump(struct adapter *adap);
#endif /* __CXGB4_CUDBG_H__ */
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
index 99c9b88d6d34..c4e0e4705cda 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
@@ -5526,6 +5526,18 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
if (err)
goto out_free_adapter;
+ if (is_kdump_kernel()) {
+ /* Collect hardware state and append to
+ * /sys/kernel/crashdd/cxgb4/ directory
+ */
+ err = cxgb4_cudbg_crashdd_add_dump(adapter);
+ if (err) {
+ dev_warn(adapter->pdev_dev,
+ "Fail collecting crash device dump, err: %d. Continuing\n",
+ err);
+ err = 0;
+ }
+ }
if (!is_t4(adapter->params.chip)) {
s_qpp = (QUEUESPERPAGEPF0_S +
--
2.14.1
^ permalink raw reply related [flat|nested] 6+ messages in thread