* [PATCH v2 09/10] Revert "cxl: Allow a default context to be associated with an external pci_dev"
From: Frederic Barrat @ 2018-06-28 10:05 UTC (permalink / raw)
To: alastair, andrew.donnellan, vaibhav, clombard, felix,
linuxppc-dev; +Cc: huyn
In-Reply-To: <20180628100509.17413-1-fbarrat@linux.ibm.com>
Remove abandonned capi support for the Mellanox CX4.
This reverts commit a19bd79e31769626d288cc016e21a31b6f47bf6f.
Signed-off-by: Frederic Barrat <fbarrat@linux.ibm.com>
---
drivers/misc/cxl/Makefile | 2 +-
drivers/misc/cxl/base.c | 35 -------------------------------
drivers/misc/cxl/cxl.h | 6 ------
drivers/misc/cxl/main.c | 2 --
drivers/misc/cxl/phb.c | 44 ---------------------------------------
drivers/misc/cxl/vphb.c | 30 +++++++++++++++++++++++---
include/misc/cxl-base.h | 6 ------
7 files changed, 28 insertions(+), 97 deletions(-)
delete mode 100644 drivers/misc/cxl/phb.c
diff --git a/drivers/misc/cxl/Makefile b/drivers/misc/cxl/Makefile
index 502d41fc9ea5..5eea61b9584f 100644
--- a/drivers/misc/cxl/Makefile
+++ b/drivers/misc/cxl/Makefile
@@ -4,7 +4,7 @@ ccflags-$(CONFIG_PPC_WERROR) += -Werror
cxl-y += main.o file.o irq.o fault.o native.o
cxl-y += context.o sysfs.o pci.o trace.o
-cxl-y += vphb.o phb.o api.o cxllib.o
+cxl-y += vphb.o api.o cxllib.o
cxl-$(CONFIG_PPC_PSERIES) += flash.o guest.o of.o hcalls.o
cxl-$(CONFIG_DEBUG_FS) += debugfs.o
obj-$(CONFIG_CXL) += cxl.o
diff --git a/drivers/misc/cxl/base.c b/drivers/misc/cxl/base.c
index e1e80cb99ad9..7557835cdfcd 100644
--- a/drivers/misc/cxl/base.c
+++ b/drivers/misc/cxl/base.c
@@ -106,41 +106,6 @@ int cxl_update_properties(struct device_node *dn,
}
EXPORT_SYMBOL_GPL(cxl_update_properties);
-/*
- * API calls into the driver that may be called from the PHB code and must be
- * built in.
- */
-bool cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu)
-{
- bool ret;
- struct cxl_calls *calls;
-
- calls = cxl_calls_get();
- if (!calls)
- return false;
-
- ret = calls->cxl_pci_associate_default_context(dev, afu);
-
- cxl_calls_put(calls);
-
- return ret;
-}
-EXPORT_SYMBOL_GPL(cxl_pci_associate_default_context);
-
-void cxl_pci_disable_device(struct pci_dev *dev)
-{
- struct cxl_calls *calls;
-
- calls = cxl_calls_get();
- if (!calls)
- return;
-
- calls->cxl_pci_disable_device(dev);
-
- cxl_calls_put(calls);
-}
-EXPORT_SYMBOL_GPL(cxl_pci_disable_device);
-
static int __init cxl_base_init(void)
{
struct device_node *np;
diff --git a/drivers/misc/cxl/cxl.h b/drivers/misc/cxl/cxl.h
index d95c2c98f2ab..aa453448201d 100644
--- a/drivers/misc/cxl/cxl.h
+++ b/drivers/misc/cxl/cxl.h
@@ -867,15 +867,9 @@ static inline bool cxl_is_power9_dd1(void)
ssize_t cxl_pci_afu_read_err_buffer(struct cxl_afu *afu, char *buf,
loff_t off, size_t count);
-/* Internal functions wrapped in cxl_base to allow PHB to call them */
-bool _cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu);
-void _cxl_pci_disable_device(struct pci_dev *dev);
struct cxl_calls {
void (*cxl_slbia)(struct mm_struct *mm);
- bool (*cxl_pci_associate_default_context)(struct pci_dev *dev, struct cxl_afu *afu);
- void (*cxl_pci_disable_device)(struct pci_dev *dev);
-
struct module *owner;
};
int register_cxl_calls(struct cxl_calls *calls);
diff --git a/drivers/misc/cxl/main.c b/drivers/misc/cxl/main.c
index a7e83624034b..334223b802ee 100644
--- a/drivers/misc/cxl/main.c
+++ b/drivers/misc/cxl/main.c
@@ -104,8 +104,6 @@ static inline void cxl_slbia_core(struct mm_struct *mm)
static struct cxl_calls cxl_calls = {
.cxl_slbia = cxl_slbia_core,
- .cxl_pci_associate_default_context = _cxl_pci_associate_default_context,
- .cxl_pci_disable_device = _cxl_pci_disable_device,
.owner = THIS_MODULE,
};
diff --git a/drivers/misc/cxl/phb.c b/drivers/misc/cxl/phb.c
deleted file mode 100644
index 6ec69ada19f4..000000000000
--- a/drivers/misc/cxl/phb.c
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Copyright 2014-2016 IBM Corp.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version
- * 2 of the License, or (at your option) any later version.
- */
-
-#include <linux/pci.h>
-#include "cxl.h"
-
-bool _cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu)
-{
- struct cxl_context *ctx;
-
- /*
- * Allocate a context to do cxl things to. This is used for interrupts
- * in the peer model using a real phb, and if we eventually do DMA ops
- * in the virtual phb, we'll need a default context to attach them to.
- */
- ctx = cxl_dev_context_init(dev);
- if (IS_ERR(ctx))
- return false;
- dev->dev.archdata.cxl_ctx = ctx;
-
- return (cxl_ops->afu_check_and_enable(afu) == 0);
-}
-/* exported via cxl_base */
-
-void _cxl_pci_disable_device(struct pci_dev *dev)
-{
- struct cxl_context *ctx = cxl_get_context(dev);
-
- if (ctx) {
- if (ctx->status == STARTED) {
- dev_err(&dev->dev, "Default context started\n");
- return;
- }
- dev->dev.archdata.cxl_ctx = NULL;
- cxl_release_context(ctx);
- }
-}
-/* exported via cxl_base */
diff --git a/drivers/misc/cxl/vphb.c b/drivers/misc/cxl/vphb.c
index 1a99c9c7a6fb..7908633d9204 100644
--- a/drivers/misc/cxl/vphb.c
+++ b/drivers/misc/cxl/vphb.c
@@ -44,6 +44,7 @@ static bool cxl_pci_enable_device_hook(struct pci_dev *dev)
{
struct pci_controller *phb;
struct cxl_afu *afu;
+ struct cxl_context *ctx;
phb = pci_bus_to_host(dev->bus);
afu = (struct cxl_afu *)phb->private_data;
@@ -56,7 +57,30 @@ static bool cxl_pci_enable_device_hook(struct pci_dev *dev)
set_dma_ops(&dev->dev, &dma_nommu_ops);
set_dma_offset(&dev->dev, PAGE_OFFSET);
- return _cxl_pci_associate_default_context(dev, afu);
+ /*
+ * Allocate a context to do cxl things too. If we eventually do real
+ * DMA ops, we'll need a default context to attach them to
+ */
+ ctx = cxl_dev_context_init(dev);
+ if (IS_ERR(ctx))
+ return false;
+ dev->dev.archdata.cxl_ctx = ctx;
+
+ return (cxl_ops->afu_check_and_enable(afu) == 0);
+}
+
+static void cxl_pci_disable_device(struct pci_dev *dev)
+{
+ struct cxl_context *ctx = cxl_get_context(dev);
+
+ if (ctx) {
+ if (ctx->status == STARTED) {
+ dev_err(&dev->dev, "Default context started\n");
+ return;
+ }
+ dev->dev.archdata.cxl_ctx = NULL;
+ cxl_release_context(ctx);
+ }
}
static resource_size_t cxl_pci_window_alignment(struct pci_bus *bus,
@@ -190,8 +214,8 @@ static struct pci_controller_ops cxl_pci_controller_ops =
{
.probe_mode = cxl_pci_probe_mode,
.enable_device_hook = cxl_pci_enable_device_hook,
- .disable_device = _cxl_pci_disable_device,
- .release_device = _cxl_pci_disable_device,
+ .disable_device = cxl_pci_disable_device,
+ .release_device = cxl_pci_disable_device,
.window_alignment = cxl_pci_window_alignment,
.reset_secondary_bus = cxl_pci_reset_secondary_bus,
.setup_msi_irqs = cxl_setup_msi_irqs,
diff --git a/include/misc/cxl-base.h b/include/misc/cxl-base.h
index bb7e629ae492..f53808fa638a 100644
--- a/include/misc/cxl-base.h
+++ b/include/misc/cxl-base.h
@@ -10,8 +10,6 @@
#ifndef _MISC_CXL_BASE_H
#define _MISC_CXL_BASE_H
-#include <misc/cxl.h>
-
#ifdef CONFIG_CXL_BASE
#define CXL_IRQ_RANGES 4
@@ -41,8 +39,6 @@ static inline void cxl_ctx_put(void)
struct cxl_afu *cxl_afu_get(struct cxl_afu *afu);
void cxl_afu_put(struct cxl_afu *afu);
void cxl_slbia(struct mm_struct *mm);
-bool cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu);
-void cxl_pci_disable_device(struct pci_dev *dev);
#else /* CONFIG_CXL_BASE */
@@ -50,8 +46,6 @@ static inline bool cxl_ctx_in_use(void) { return false; }
static inline struct cxl_afu *cxl_afu_get(struct cxl_afu *afu) { return NULL; }
static inline void cxl_afu_put(struct cxl_afu *afu) {}
static inline void cxl_slbia(struct mm_struct *mm) {}
-static inline bool cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu) { return false; }
-static inline void cxl_pci_disable_device(struct pci_dev *dev) {}
#endif /* CONFIG_CXL_BASE */
--
2.17.1
^ permalink raw reply related
* [PATCH v2 10/10] cxl: Remove abandonned capi support for the Mellanox CX4, final cleanup
From: Frederic Barrat @ 2018-06-28 10:05 UTC (permalink / raw)
To: alastair, andrew.donnellan, vaibhav, clombard, felix,
linuxppc-dev; +Cc: huyn
In-Reply-To: <20180628100509.17413-1-fbarrat@linux.ibm.com>
Remove a few XSL/CX4 oddities which are no longer needed. A simple
revert of the initial commits was not possible (or not worth it) due
to the history of the code.
Signed-off-by: Frederic Barrat <fbarrat@linux.ibm.com>
---
drivers/misc/cxl/context.c | 2 +-
drivers/misc/cxl/cxl.h | 12 ------
drivers/misc/cxl/debugfs.c | 5 ---
drivers/misc/cxl/pci.c | 75 +++-----------------------------------
4 files changed, 7 insertions(+), 87 deletions(-)
diff --git a/drivers/misc/cxl/context.c b/drivers/misc/cxl/context.c
index 0355d42d367f..5fe529b43ebe 100644
--- a/drivers/misc/cxl/context.c
+++ b/drivers/misc/cxl/context.c
@@ -95,7 +95,7 @@ int cxl_context_init(struct cxl_context *ctx, struct cxl_afu *afu, bool master)
*/
mutex_lock(&afu->contexts_lock);
idr_preload(GFP_KERNEL);
- i = idr_alloc(&ctx->afu->contexts_idr, ctx, ctx->afu->adapter->min_pe,
+ i = idr_alloc(&ctx->afu->contexts_idr, ctx, 0,
ctx->afu->num_procs, GFP_NOWAIT);
idr_preload_end();
mutex_unlock(&afu->contexts_lock);
diff --git a/drivers/misc/cxl/cxl.h b/drivers/misc/cxl/cxl.h
index aa453448201d..44bcfafbb579 100644
--- a/drivers/misc/cxl/cxl.h
+++ b/drivers/misc/cxl/cxl.h
@@ -93,11 +93,6 @@ static const cxl_p1_reg_t CXL_PSL_FIR_CNTL = {0x0148};
static const cxl_p1_reg_t CXL_PSL_DSNDCTL = {0x0150};
static const cxl_p1_reg_t CXL_PSL_SNWRALLOC = {0x0158};
static const cxl_p1_reg_t CXL_PSL_TRACE = {0x0170};
-/* XSL registers (Mellanox CX4) */
-static const cxl_p1_reg_t CXL_XSL_Timebase = {0x0100};
-static const cxl_p1_reg_t CXL_XSL_TB_CTLSTAT = {0x0108};
-static const cxl_p1_reg_t CXL_XSL_FEC = {0x0158};
-static const cxl_p1_reg_t CXL_XSL_DSNCTL = {0x0168};
/* PSL registers - CAIA 2 */
static const cxl_p1_reg_t CXL_PSL9_CONTROL = {0x0020};
static const cxl_p1_reg_t CXL_XSL9_INV = {0x0110};
@@ -695,7 +690,6 @@ struct cxl {
struct bin_attribute cxl_attr;
int adapter_num;
int user_irqs;
- int min_pe;
u64 ps_size;
u16 psl_rev;
u16 base_image;
@@ -934,7 +928,6 @@ int cxl_debugfs_afu_add(struct cxl_afu *afu);
void cxl_debugfs_afu_remove(struct cxl_afu *afu);
void cxl_debugfs_add_adapter_regs_psl9(struct cxl *adapter, struct dentry *dir);
void cxl_debugfs_add_adapter_regs_psl8(struct cxl *adapter, struct dentry *dir);
-void cxl_debugfs_add_adapter_regs_xsl(struct cxl *adapter, struct dentry *dir);
void cxl_debugfs_add_afu_regs_psl9(struct cxl_afu *afu, struct dentry *dir);
void cxl_debugfs_add_afu_regs_psl8(struct cxl_afu *afu, struct dentry *dir);
@@ -977,11 +970,6 @@ static inline void cxl_debugfs_add_adapter_regs_psl8(struct cxl *adapter,
{
}
-static inline void cxl_debugfs_add_adapter_regs_xsl(struct cxl *adapter,
- struct dentry *dir)
-{
-}
-
static inline void cxl_debugfs_add_afu_regs_psl9(struct cxl_afu *afu, struct dentry *dir)
{
}
diff --git a/drivers/misc/cxl/debugfs.c b/drivers/misc/cxl/debugfs.c
index 1643850d2302..a1921d81593a 100644
--- a/drivers/misc/cxl/debugfs.c
+++ b/drivers/misc/cxl/debugfs.c
@@ -58,11 +58,6 @@ void cxl_debugfs_add_adapter_regs_psl8(struct cxl *adapter, struct dentry *dir)
debugfs_create_io_x64("trace", S_IRUSR | S_IWUSR, dir, _cxl_p1_addr(adapter, CXL_PSL_TRACE));
}
-void cxl_debugfs_add_adapter_regs_xsl(struct cxl *adapter, struct dentry *dir)
-{
- debugfs_create_io_x64("fec", S_IRUSR, dir, _cxl_p1_addr(adapter, CXL_XSL_FEC));
-}
-
int cxl_debugfs_adapter_add(struct cxl *adapter)
{
struct dentry *dir;
diff --git a/drivers/misc/cxl/pci.c b/drivers/misc/cxl/pci.c
index 0ca818396524..6dfb4ed345d3 100644
--- a/drivers/misc/cxl/pci.c
+++ b/drivers/misc/cxl/pci.c
@@ -593,27 +593,7 @@ static int init_implementation_adapter_regs_psl8(struct cxl *adapter, struct pci
return 0;
}
-static int init_implementation_adapter_regs_xsl(struct cxl *adapter, struct pci_dev *dev)
-{
- u64 xsl_dsnctl;
- u64 chipid;
- u32 phb_index;
- u64 capp_unit_id;
- int rc;
-
- rc = cxl_calc_capp_routing(dev, &chipid, &phb_index, &capp_unit_id);
- if (rc)
- return rc;
-
- /* Tell XSL where to route data to */
- xsl_dsnctl = 0x0000600000000000ULL | (chipid << (63-5));
- xsl_dsnctl |= (capp_unit_id << (63-13));
- cxl_p1_write(adapter, CXL_XSL_DSNCTL, xsl_dsnctl);
-
- return 0;
-}
-
-/* PSL & XSL */
+/* PSL */
#define TBSYNC_CAL(n) (((u64)n & 0x7) << (63-3))
#define TBSYNC_CNT(n) (((u64)n & 0x7) << (63-6))
/* For the PSL this is a multiple for 0 < n <= 7: */
@@ -625,21 +605,6 @@ static void write_timebase_ctrl_psl8(struct cxl *adapter)
TBSYNC_CNT(2 * PSL_2048_250MHZ_CYCLES));
}
-/* XSL */
-#define TBSYNC_ENA (1ULL << 63)
-/* For the XSL this is 2**n * 2000 clocks for 0 < n <= 6: */
-#define XSL_2000_CLOCKS 1
-#define XSL_4000_CLOCKS 2
-#define XSL_8000_CLOCKS 3
-
-static void write_timebase_ctrl_xsl(struct cxl *adapter)
-{
- cxl_p1_write(adapter, CXL_XSL_TB_CTLSTAT,
- TBSYNC_ENA |
- TBSYNC_CAL(3) |
- TBSYNC_CNT(XSL_4000_CLOCKS));
-}
-
static u64 timebase_read_psl9(struct cxl *adapter)
{
return cxl_p1_read(adapter, CXL_PSL9_Timebase);
@@ -650,11 +615,6 @@ static u64 timebase_read_psl8(struct cxl *adapter)
return cxl_p1_read(adapter, CXL_PSL_Timebase);
}
-static u64 timebase_read_xsl(struct cxl *adapter)
-{
- return cxl_p1_read(adapter, CXL_XSL_Timebase);
-}
-
static void cxl_setup_psl_timebase(struct cxl *adapter, struct pci_dev *dev)
{
struct device_node *np;
@@ -1671,37 +1631,14 @@ static const struct cxl_service_layer_ops psl8_ops = {
.needs_reset_before_disable = true,
};
-static const struct cxl_service_layer_ops xsl_ops = {
- .adapter_regs_init = init_implementation_adapter_regs_xsl,
- .invalidate_all = cxl_invalidate_all_psl8,
- .sanitise_afu_regs = sanitise_afu_regs_psl8,
- .handle_interrupt = cxl_irq_psl8,
- .fail_irq = cxl_fail_irq_psl,
- .activate_dedicated_process = cxl_activate_dedicated_process_psl8,
- .attach_afu_directed = cxl_attach_afu_directed_psl8,
- .attach_dedicated_process = cxl_attach_dedicated_process_psl8,
- .update_dedicated_ivtes = cxl_update_dedicated_ivtes_psl8,
- .debugfs_add_adapter_regs = cxl_debugfs_add_adapter_regs_xsl,
- .write_timebase_ctrl = write_timebase_ctrl_xsl,
- .timebase_read = timebase_read_xsl,
- .capi_mode = OPAL_PHB_CAPI_MODE_DMA,
-};
-
static void set_sl_ops(struct cxl *adapter, struct pci_dev *dev)
{
- if (dev->vendor == PCI_VENDOR_ID_MELLANOX && dev->device == 0x1013) {
- /* Mellanox CX-4 */
- dev_info(&dev->dev, "Device uses an XSL\n");
- adapter->native->sl_ops = &xsl_ops;
- adapter->min_pe = 1; /* Workaround for CX-4 hardware bug */
+ if (cxl_is_power8()) {
+ dev_info(&dev->dev, "Device uses a PSL8\n");
+ adapter->native->sl_ops = &psl8_ops;
} else {
- if (cxl_is_power8()) {
- dev_info(&dev->dev, "Device uses a PSL8\n");
- adapter->native->sl_ops = &psl8_ops;
- } else {
- dev_info(&dev->dev, "Device uses a PSL9\n");
- adapter->native->sl_ops = &psl9_ops;
- }
+ dev_info(&dev->dev, "Device uses a PSL9\n");
+ adapter->native->sl_ops = &psl9_ops;
}
}
--
2.17.1
^ permalink raw reply related
* [PATCH v2 01/10] Revert "cxl: Add kernel API to allow a context to operate with relocate disabled"
From: Frederic Barrat @ 2018-06-28 10:05 UTC (permalink / raw)
To: alastair, andrew.donnellan, vaibhav, clombard, felix,
linuxppc-dev; +Cc: huyn
In-Reply-To: <20180628100509.17413-1-fbarrat@linux.ibm.com>
From: Alastair D'Silva <alastair@d-silva.org>
Remove abandonned capi support for the Mellanox CX4.
The symbol 'cxl_set_translation_mode' is never called, so
ctx->real_mode is always false.
This reverts commit 7a0d85d313c2066712e530e668bc02bb741a685c.
Signed-off-by: Alastair D'Silva <alastair@d-silva.org>
---
drivers/misc/cxl/api.c | 19 -------------------
drivers/misc/cxl/cxl.h | 1 -
drivers/misc/cxl/guest.c | 3 ---
drivers/misc/cxl/native.c | 3 ++-
include/misc/cxl.h | 8 --------
5 files changed, 2 insertions(+), 32 deletions(-)
diff --git a/drivers/misc/cxl/api.c b/drivers/misc/cxl/api.c
index 753b1a698fc4..21d620e29fea 100644
--- a/drivers/misc/cxl/api.c
+++ b/drivers/misc/cxl/api.c
@@ -324,7 +324,6 @@ int cxl_start_context(struct cxl_context *ctx, u64 wed,
if (task) {
ctx->pid = get_task_pid(task, PIDTYPE_PID);
kernel = false;
- ctx->real_mode = false;
/* acquire a reference to the task's mm */
ctx->mm = get_task_mm(current);
@@ -388,24 +387,6 @@ void cxl_set_master(struct cxl_context *ctx)
}
EXPORT_SYMBOL_GPL(cxl_set_master);
-int cxl_set_translation_mode(struct cxl_context *ctx, bool real_mode)
-{
- if (ctx->status == STARTED) {
- /*
- * We could potentially update the PE and issue an update LLCMD
- * to support this, but it doesn't seem to have a good use case
- * since it's trivial to just create a second kernel context
- * with different translation modes, so until someone convinces
- * me otherwise:
- */
- return -EBUSY;
- }
-
- ctx->real_mode = real_mode;
- return 0;
-}
-EXPORT_SYMBOL_GPL(cxl_set_translation_mode);
-
/* wrappers around afu_* file ops which are EXPORTED */
int cxl_fd_open(struct inode *inode, struct file *file)
{
diff --git a/drivers/misc/cxl/cxl.h b/drivers/misc/cxl/cxl.h
index 918d4fb742d1..af8794719956 100644
--- a/drivers/misc/cxl/cxl.h
+++ b/drivers/misc/cxl/cxl.h
@@ -613,7 +613,6 @@ struct cxl_context {
bool pe_inserted;
bool master;
bool kernel;
- bool real_mode;
bool pending_irq;
bool pending_fault;
bool pending_afu_err;
diff --git a/drivers/misc/cxl/guest.c b/drivers/misc/cxl/guest.c
index 4644f16606a3..f5dc740fcd13 100644
--- a/drivers/misc/cxl/guest.c
+++ b/drivers/misc/cxl/guest.c
@@ -623,9 +623,6 @@ static int guest_attach_process(struct cxl_context *ctx, bool kernel, u64 wed, u
{
pr_devel("in %s\n", __func__);
- if (ctx->real_mode)
- return -EPERM;
-
ctx->kernel = kernel;
if (ctx->afu->current_mode == CXL_MODE_DIRECTED)
return attach_afu_directed(ctx, wed, amr);
diff --git a/drivers/misc/cxl/native.c b/drivers/misc/cxl/native.c
index 98f867fcef24..c9d5d82dce8e 100644
--- a/drivers/misc/cxl/native.c
+++ b/drivers/misc/cxl/native.c
@@ -605,6 +605,7 @@ u64 cxl_calculate_sr(bool master, bool kernel, bool real_mode, bool p9)
sr |= CXL_PSL_SR_An_MP;
if (mfspr(SPRN_LPCR) & LPCR_TC)
sr |= CXL_PSL_SR_An_TC;
+
if (kernel) {
if (!real_mode)
sr |= CXL_PSL_SR_An_R;
@@ -629,7 +630,7 @@ u64 cxl_calculate_sr(bool master, bool kernel, bool real_mode, bool p9)
static u64 calculate_sr(struct cxl_context *ctx)
{
- return cxl_calculate_sr(ctx->master, ctx->kernel, ctx->real_mode,
+ return cxl_calculate_sr(ctx->master, ctx->kernel, false,
cxl_is_power9());
}
diff --git a/include/misc/cxl.h b/include/misc/cxl.h
index b712be544f8c..82cc6ffafe2d 100644
--- a/include/misc/cxl.h
+++ b/include/misc/cxl.h
@@ -173,14 +173,6 @@ int cxl_afu_reset(struct cxl_context *ctx);
*/
void cxl_set_master(struct cxl_context *ctx);
-/*
- * Sets the context to use real mode memory accesses to operate with
- * translation disabled. Note that this only makes sense for kernel contexts
- * under bare metal, and will not work with virtualisation. May only be
- * performed on stopped contexts.
- */
-int cxl_set_translation_mode(struct cxl_context *ctx, bool real_mode);
-
/*
* Map and unmap the AFU Problem Space area. The amount and location mapped
* depends on if this context is a master or slave.
--
2.17.1
^ permalink raw reply related
* [PATCH v4 0/6] powerpc/pseries: Machien check handler improvements.
From: Mahesh J Salgaonkar @ 2018-06-28 11:10 UTC (permalink / raw)
To: linuxppc-dev
Cc: Michael Ellerman, Nicholas Piggin, stable, Aneesh Kumar K.V,
Nicholas Piggin, Aneesh Kumar K.V, Laurent Dufour
This patch series includes some improvement to Machine check handler
for pseries. Patch 1 fixes an issue where machine check handler crashes
kernel while accessing vmalloc-ed buffer while in nmi context.
Patch 2 fixes endain bug while restoring of r3 in MCE handler.
Patch 4 implements a real mode mce handler and flushes the SLBs on SLB error.
Patch 5 display's the MCE error details on console.
Patch 6 saves and dumps the SLB contents on SLB MCE errors to improve the
debugability.
Change in V4:
- Flush the SLBs in real mode mce handler to handle SLB errors for entry 0.
- Allocate buffers per cpu to hold rtas error log and old slb contents.
- Defer the logging of rtas error log to irq work queue.
Change in V3:
- Moved patch 5 to patch 2
Change in V2:
- patch 3: Display additional info (NIP and task info) in MCE error details.
- patch 5: Fix endain bug while restoring of r3 in MCE handler.
---
Mahesh Salgaonkar (6):
powerpc/pseries: Defer the logging of rtas error to irq work queue.
powerpc/pseries: Fix endainness while restoring of r3 in MCE handler.
powerpc/pseries: Define MCE error event section.
powerpc/pseries: flush SLB contents on SLB MCE errors.
powerpc/pseries: Display machine check error details.
powerpc/pseries: Dump the SLB contents on SLB MCE errors.
arch/powerpc/include/asm/book3s/64/mmu-hash.h | 8 +
arch/powerpc/include/asm/machdep.h | 1
arch/powerpc/include/asm/paca.h | 4
arch/powerpc/include/asm/rtas.h | 116 ++++++++++++
arch/powerpc/kernel/exceptions-64s.S | 42 ++++
arch/powerpc/kernel/mce.c | 16 +-
arch/powerpc/mm/slb.c | 63 +++++++
arch/powerpc/platforms/powernv/opal.c | 1
arch/powerpc/platforms/pseries/pseries.h | 1
arch/powerpc/platforms/pseries/ras.c | 236 +++++++++++++++++++++++--
arch/powerpc/platforms/pseries/setup.c | 27 +++
11 files changed, 497 insertions(+), 18 deletions(-)
--
Signature
^ permalink raw reply
* [PATCH v4 1/6] powerpc/pseries: Defer the logging of rtas error to irq work queue.
From: Mahesh J Salgaonkar @ 2018-06-28 11:10 UTC (permalink / raw)
To: linuxppc-dev; +Cc: stable, Nicholas Piggin, Aneesh Kumar K.V, Laurent Dufour
In-Reply-To: <153018397703.31589.3105355336278825912.stgit@jupiter.in.ibm.com>
From: Mahesh Salgaonkar <mahesh@linux.vnet.ibm.com>
rtas_log_buf is a buffer to hold RTAS event data that are communicated
to kernel by hypervisor. This buffer is then used to pass RTAS event
data to user through proc fs. This buffer is allocated from vmalloc
(non-linear mapping) area.
On Machine check interrupt, register r3 points to RTAS extended event
log passed by hypervisor that contains the MCE event. The pseries
machine check handler then logs this error into rtas_log_buf. The
rtas_log_buf is a vmalloc-ed (non-linear) buffer we end up taking up a
page fault (vector 0x300) while accessing it. Since machine check
interrupt handler runs in NMI context we can not afford to take any
page fault. Page faults are not honored in NMI context and causes
kernel panic. Apart from that, as Nick pointed out, pSeries_log_error()
also takes a spin_lock while logging error which is not safe in NMI
context. It may endup in deadlock if we get another MCE before releasing
the lock. Fix this by deferring the logging of rtas error to irq work queue.
Current implementation uses two different buffers to hold rtas error log
depending on whether extended log is provided or not. This makes bit
difficult to identify which buffer has valid data that needs to logged
later in irq work. Simplify this using single buffer, one per paca, and
copy rtas log to it irrespective of whether extended log is provided or
not. Allocate this buffer below RMA region so that it can be accessed
in real mode mce handler.
Fixes: b96672dd840f ("powerpc: Machine check interrupt is a non-maskable interrupt")
Cc: stable@vger.kernel.org
Signed-off-by: Mahesh Salgaonkar <mahesh@linux.vnet.ibm.com>
---
arch/powerpc/include/asm/paca.h | 3 ++
arch/powerpc/platforms/pseries/ras.c | 39 +++++++++++++++++++++-----------
arch/powerpc/platforms/pseries/setup.c | 16 +++++++++++++
3 files changed, 45 insertions(+), 13 deletions(-)
diff --git a/arch/powerpc/include/asm/paca.h b/arch/powerpc/include/asm/paca.h
index 3f109a3e3edb..b441fef53077 100644
--- a/arch/powerpc/include/asm/paca.h
+++ b/arch/powerpc/include/asm/paca.h
@@ -251,6 +251,9 @@ struct paca_struct {
void *rfi_flush_fallback_area;
u64 l1d_flush_size;
#endif
+#ifdef CONFIG_PPC_PSERIES
+ u8 *mce_data_buf; /* buffer to hold per cpu rtas errlog */
+#endif /* CONFIG_PPC_PSERIES */
} ____cacheline_aligned;
extern void copy_mm_to_paca(struct mm_struct *mm);
diff --git a/arch/powerpc/platforms/pseries/ras.c b/arch/powerpc/platforms/pseries/ras.c
index 5e1ef9150182..f6ba9a2a4f84 100644
--- a/arch/powerpc/platforms/pseries/ras.c
+++ b/arch/powerpc/platforms/pseries/ras.c
@@ -22,6 +22,7 @@
#include <linux/of.h>
#include <linux/fs.h>
#include <linux/reboot.h>
+#include <linux/irq_work.h>
#include <asm/machdep.h>
#include <asm/rtas.h>
@@ -32,11 +33,13 @@
static unsigned char ras_log_buf[RTAS_ERROR_LOG_MAX];
static DEFINE_SPINLOCK(ras_log_buf_lock);
-static char global_mce_data_buf[RTAS_ERROR_LOG_MAX];
-static DEFINE_PER_CPU(__u64, mce_data_buf);
-
static int ras_check_exception_token;
+static void mce_process_errlog_event(struct irq_work *work);
+static struct irq_work mce_errlog_process_work = {
+ .func = mce_process_errlog_event,
+};
+
#define EPOW_SENSOR_TOKEN 9
#define EPOW_SENSOR_INDEX 0
@@ -336,10 +339,9 @@ static irqreturn_t ras_error_interrupt(int irq, void *dev_id)
* the actual r3 if possible, and a ptr to the error log entry
* will be returned if found.
*
- * If the RTAS error is not of the extended type, then we put it in a per
- * cpu 64bit buffer. If it is the extended type we use global_mce_data_buf.
+ * Use one buffer mce_data_buf per cpu to store RTAS error.
*
- * The global_mce_data_buf does not have any locks or protection around it,
+ * The mce_data_buf does not have any locks or protection around it,
* if a second machine check comes in, or a system reset is done
* before we have logged the error, then we will get corruption in the
* error log. This is preferable over holding off on calling
@@ -362,20 +364,19 @@ static struct rtas_error_log *fwnmi_get_errinfo(struct pt_regs *regs)
savep = __va(regs->gpr[3]);
regs->gpr[3] = savep[0]; /* restore original r3 */
- /* If it isn't an extended log we can use the per cpu 64bit buffer */
h = (struct rtas_error_log *)&savep[1];
+ /* Use the per cpu buffer from paca to store rtas error log */
+ memset(local_paca->mce_data_buf, 0, RTAS_ERROR_LOG_MAX);
if (!rtas_error_extended(h)) {
- memcpy(this_cpu_ptr(&mce_data_buf), h, sizeof(__u64));
- errhdr = (struct rtas_error_log *)this_cpu_ptr(&mce_data_buf);
+ memcpy(local_paca->mce_data_buf, h, sizeof(__u64));
} else {
int len, error_log_length;
error_log_length = 8 + rtas_error_extended_log_length(h);
len = max_t(int, error_log_length, RTAS_ERROR_LOG_MAX);
- memset(global_mce_data_buf, 0, RTAS_ERROR_LOG_MAX);
- memcpy(global_mce_data_buf, h, len);
- errhdr = (struct rtas_error_log *)global_mce_data_buf;
+ memcpy(local_paca->mce_data_buf, h, len);
}
+ errhdr = (struct rtas_error_log *)local_paca->mce_data_buf;
return errhdr;
}
@@ -422,6 +423,17 @@ int pSeries_system_reset_exception(struct pt_regs *regs)
return 0; /* need to perform reset */
}
+/*
+ * Process MCE rtas errlog event.
+ */
+static void mce_process_errlog_event(struct irq_work *work)
+{
+ struct rtas_error_log *err;
+
+ err = fwnmi_get_errlog();
+ log_error((char *)err, ERR_TYPE_RTAS_LOG, 0);
+}
+
/*
* See if we can recover from a machine check exception.
* This is only called on power4 (or above) and only via
@@ -466,7 +478,8 @@ static int recover_mce(struct pt_regs *regs, struct rtas_error_log *err)
recovered = 1;
}
- log_error((char *)err, ERR_TYPE_RTAS_LOG, 0);
+ /* Queue irq work to log this rtas event later. */
+ irq_work_queue(&mce_errlog_process_work);
return recovered;
}
diff --git a/arch/powerpc/platforms/pseries/setup.c b/arch/powerpc/platforms/pseries/setup.c
index fdb32e056ef4..60a067a6e743 100644
--- a/arch/powerpc/platforms/pseries/setup.c
+++ b/arch/powerpc/platforms/pseries/setup.c
@@ -41,6 +41,7 @@
#include <linux/root_dev.h>
#include <linux/of.h>
#include <linux/of_pci.h>
+#include <linux/memblock.h>
#include <asm/mmu.h>
#include <asm/processor.h>
@@ -101,6 +102,9 @@ static void pSeries_show_cpuinfo(struct seq_file *m)
static void __init fwnmi_init(void)
{
unsigned long system_reset_addr, machine_check_addr;
+ u8 *mce_data_buf;
+ unsigned int i;
+ int nr_cpus = num_possible_cpus();
int ibm_nmi_register = rtas_token("ibm,nmi-register");
if (ibm_nmi_register == RTAS_UNKNOWN_SERVICE)
@@ -114,6 +118,18 @@ static void __init fwnmi_init(void)
if (0 == rtas_call(ibm_nmi_register, 2, 1, NULL, system_reset_addr,
machine_check_addr))
fwnmi_active = 1;
+
+ /*
+ * Allocate a chunk for per cpu buffer to hold rtas errorlog.
+ * It will be used in real mode mce handler, hence it needs to be
+ * below RMA.
+ */
+ mce_data_buf = __va(memblock_alloc_base(RTAS_ERROR_LOG_MAX * nr_cpus,
+ RTAS_ERROR_LOG_MAX, ppc64_rma_size));
+ for_each_possible_cpu(i) {
+ paca_ptrs[i]->mce_data_buf = mce_data_buf +
+ (RTAS_ERROR_LOG_MAX * i);
+ }
}
static void pseries_8259_cascade(struct irq_desc *desc)
^ permalink raw reply related
* [PATCH v4 2/6] powerpc/pseries: Fix endainness while restoring of r3 in MCE handler.
From: Mahesh J Salgaonkar @ 2018-06-28 11:11 UTC (permalink / raw)
To: linuxppc-dev
Cc: stable, Nicholas Piggin, Nicholas Piggin, Aneesh Kumar K.V,
Laurent Dufour
In-Reply-To: <153018397703.31589.3105355336278825912.stgit@jupiter.in.ibm.com>
From: Mahesh Salgaonkar <mahesh@linux.vnet.ibm.com>
During Machine Check interrupt on pseries platform, register r3 points
RTAS extended event log passed by hypervisor. Since hypervisor uses r3
to pass pointer to rtas log, it stores the original r3 value at the
start of the memory (first 8 bytes) pointed by r3. Since hypervisor
stores this info and rtas log is in BE format, linux should make
sure to restore r3 value in correct endian format.
Without this patch when MCE handler, after recovery, returns to code that
that caused the MCE may end up with Data SLB access interrupt for invalid
address followed by kernel panic or hang.
[ 62.878965] Severe Machine check interrupt [Recovered]
[ 62.878968] NIP [d00000000ca301b8]: init_module+0x1b8/0x338 [bork_kernel]
[ 62.878969] Initiator: CPU
[ 62.878970] Error type: SLB [Multihit]
[ 62.878971] Effective address: d00000000ca70000
cpu 0xa: Vector: 380 (Data SLB Access) at [c0000000fc7775b0]
pc: c0000000009694c0: vsnprintf+0x80/0x480
lr: c0000000009698e0: vscnprintf+0x20/0x60
sp: c0000000fc777830
msr: 8000000002009033
dar: a803a30c000000d0
current = 0xc00000000bc9ef00
paca = 0xc00000001eca5c00 softe: 3 irq_happened: 0x01
pid = 8860, comm = insmod
[c0000000fc7778b0] c0000000009698e0 vscnprintf+0x20/0x60
[c0000000fc7778e0] c00000000016b6c4 vprintk_emit+0xb4/0x4b0
[c0000000fc777960] c00000000016d40c vprintk_func+0x5c/0xd0
[c0000000fc777980] c00000000016cbb4 printk+0x38/0x4c
[c0000000fc7779a0] d00000000ca301c0 init_module+0x1c0/0x338 [bork_kernel]
[c0000000fc777a40] c00000000000d9c4 do_one_initcall+0x54/0x230
[c0000000fc777b00] c0000000001b3b74 do_init_module+0x8c/0x248
[c0000000fc777b90] c0000000001b2478 load_module+0x12b8/0x15b0
[c0000000fc777d30] c0000000001b29e8 sys_finit_module+0xa8/0x110
[c0000000fc777e30] c00000000000b204 system_call+0x58/0x6c
--- Exception: c00 (System Call) at 00007fff8bda0644
SP (7fffdfbfe980) is in userspace
This patch fixes this issue.
Fixes: a08a53ea4c97 ("powerpc/le: Enable RTAS events support")
Cc: stable@vger.kernel.org
Reviewed-by: Nicholas Piggin <npiggin@gmail.com>
Signed-off-by: Mahesh Salgaonkar <mahesh@linux.vnet.ibm.com>
---
arch/powerpc/platforms/pseries/ras.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/arch/powerpc/platforms/pseries/ras.c b/arch/powerpc/platforms/pseries/ras.c
index f6ba9a2a4f84..e3bd849141de 100644
--- a/arch/powerpc/platforms/pseries/ras.c
+++ b/arch/powerpc/platforms/pseries/ras.c
@@ -362,7 +362,7 @@ static struct rtas_error_log *fwnmi_get_errinfo(struct pt_regs *regs)
}
savep = __va(regs->gpr[3]);
- regs->gpr[3] = savep[0]; /* restore original r3 */
+ regs->gpr[3] = be64_to_cpu(savep[0]); /* restore original r3 */
h = (struct rtas_error_log *)&savep[1];
/* Use the per cpu buffer from paca to store rtas error log */
^ permalink raw reply related
* [PATCH v4 3/6] powerpc/pseries: Define MCE error event section.
From: Mahesh J Salgaonkar @ 2018-06-28 11:11 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin, Aneesh Kumar K.V, Laurent Dufour
In-Reply-To: <153018397703.31589.3105355336278825912.stgit@jupiter.in.ibm.com>
From: Mahesh Salgaonkar <mahesh@linux.vnet.ibm.com>
On pseries, the machine check error details are part of RTAS extended
event log passed under Machine check exception section. This patch adds
the definition of rtas MCE event section and related helper
functions.
Signed-off-by: Mahesh Salgaonkar <mahesh@linux.vnet.ibm.com>
---
arch/powerpc/include/asm/rtas.h | 111 +++++++++++++++++++++++++++++++++++++++
1 file changed, 111 insertions(+)
diff --git a/arch/powerpc/include/asm/rtas.h b/arch/powerpc/include/asm/rtas.h
index ec9dd79398ee..ceeed2dd489b 100644
--- a/arch/powerpc/include/asm/rtas.h
+++ b/arch/powerpc/include/asm/rtas.h
@@ -185,6 +185,13 @@ static inline uint8_t rtas_error_disposition(const struct rtas_error_log *elog)
return (elog->byte1 & 0x18) >> 3;
}
+static inline
+void rtas_set_disposition_recovered(struct rtas_error_log *elog)
+{
+ elog->byte1 &= ~0x18;
+ elog->byte1 |= (RTAS_DISP_FULLY_RECOVERED << 3);
+}
+
static inline uint8_t rtas_error_extended(const struct rtas_error_log *elog)
{
return (elog->byte1 & 0x04) >> 2;
@@ -275,6 +282,7 @@ inline uint32_t rtas_ext_event_company_id(struct rtas_ext_event_log_v6 *ext_log)
#define PSERIES_ELOG_SECT_ID_CALL_HOME (('C' << 8) | 'H')
#define PSERIES_ELOG_SECT_ID_USER_DEF (('U' << 8) | 'D')
#define PSERIES_ELOG_SECT_ID_HOTPLUG (('H' << 8) | 'P')
+#define PSERIES_ELOG_SECT_ID_MCE (('M' << 8) | 'C')
/* Vendor specific Platform Event Log Format, Version 6, section header */
struct pseries_errorlog {
@@ -326,6 +334,109 @@ struct pseries_hp_errorlog {
#define PSERIES_HP_ELOG_ID_DRC_COUNT 3
#define PSERIES_HP_ELOG_ID_DRC_IC 4
+/* RTAS pseries MCE errorlog section */
+#pragma pack(push, 1)
+struct pseries_mc_errorlog {
+ __be32 fru_id;
+ __be32 proc_id;
+ uint8_t error_type;
+ union {
+ struct {
+ uint8_t ue_err_type;
+ /* XXXXXXXX
+ * X 1: Permanent or Transient UE.
+ * X 1: Effective address provided.
+ * X 1: Logical address provided.
+ * XX 2: Reserved.
+ * XXX 3: Type of UE error.
+ */
+ uint8_t reserved_1[6];
+ __be64 effective_address;
+ __be64 logical_address;
+ } ue_error;
+ struct {
+ uint8_t soft_err_type;
+ /* XXXXXXXX
+ * X 1: Effective address provided.
+ * XXXXX 5: Reserved.
+ * XX 2: Type of SLB/ERAT/TLB error.
+ */
+ uint8_t reserved_1[6];
+ __be64 effective_address;
+ uint8_t reserved_2[8];
+ } soft_error;
+ } u;
+};
+#pragma pack(pop)
+
+/* RTAS pseries MCE error types */
+#define PSERIES_MC_ERROR_TYPE_UE 0x00
+#define PSERIES_MC_ERROR_TYPE_SLB 0x01
+#define PSERIES_MC_ERROR_TYPE_ERAT 0x02
+#define PSERIES_MC_ERROR_TYPE_TLB 0x04
+#define PSERIES_MC_ERROR_TYPE_D_CACHE 0x05
+#define PSERIES_MC_ERROR_TYPE_I_CACHE 0x07
+
+/* RTAS pseries MCE error sub types */
+#define PSERIES_MC_ERROR_UE_INDETERMINATE 0
+#define PSERIES_MC_ERROR_UE_IFETCH 1
+#define PSERIES_MC_ERROR_UE_PAGE_TABLE_WALK_IFETCH 2
+#define PSERIES_MC_ERROR_UE_LOAD_STORE 3
+#define PSERIES_MC_ERROR_UE_PAGE_TABLE_WALK_LOAD_STORE 4
+
+#define PSERIES_MC_ERROR_SLB_PARITY 0
+#define PSERIES_MC_ERROR_SLB_MULTIHIT 1
+#define PSERIES_MC_ERROR_SLB_INDETERMINATE 2
+
+#define PSERIES_MC_ERROR_ERAT_PARITY 1
+#define PSERIES_MC_ERROR_ERAT_MULTIHIT 2
+#define PSERIES_MC_ERROR_ERAT_INDETERMINATE 3
+
+#define PSERIES_MC_ERROR_TLB_PARITY 1
+#define PSERIES_MC_ERROR_TLB_MULTIHIT 2
+#define PSERIES_MC_ERROR_TLB_INDETERMINATE 3
+
+static inline uint8_t rtas_mc_error_type(const struct pseries_mc_errorlog *mlog)
+{
+ return mlog->error_type;
+}
+
+static inline uint8_t rtas_mc_error_sub_type(
+ const struct pseries_mc_errorlog *mlog)
+{
+ switch (mlog->error_type) {
+ case PSERIES_MC_ERROR_TYPE_UE:
+ return (mlog->u.ue_error.ue_err_type & 0x07);
+ case PSERIES_MC_ERROR_TYPE_SLB:
+ case PSERIES_MC_ERROR_TYPE_ERAT:
+ case PSERIES_MC_ERROR_TYPE_TLB:
+ return (mlog->u.soft_error.soft_err_type & 0x03);
+ default:
+ return 0;
+ }
+}
+
+static inline uint64_t rtas_mc_get_effective_addr(
+ const struct pseries_mc_errorlog *mlog)
+{
+ uint64_t addr = 0;
+
+ switch (mlog->error_type) {
+ case PSERIES_MC_ERROR_TYPE_UE:
+ if (mlog->u.ue_error.ue_err_type & 0x40)
+ addr = mlog->u.ue_error.effective_address;
+ break;
+ case PSERIES_MC_ERROR_TYPE_SLB:
+ case PSERIES_MC_ERROR_TYPE_ERAT:
+ case PSERIES_MC_ERROR_TYPE_TLB:
+ if (mlog->u.soft_error.soft_err_type & 0x80)
+ addr = mlog->u.soft_error.effective_address;
+ default:
+ break;
+ }
+ return be64_to_cpu(addr);
+}
+
struct pseries_errorlog *get_pseries_errorlog(struct rtas_error_log *log,
uint16_t section_id);
^ permalink raw reply related
* [PATCH v4 4/6] powerpc/pseries: flush SLB contents on SLB MCE errors.
From: Mahesh J Salgaonkar @ 2018-06-28 11:11 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin, Aneesh Kumar K.V, Laurent Dufour
In-Reply-To: <153018397703.31589.3105355336278825912.stgit@jupiter.in.ibm.com>
From: Mahesh Salgaonkar <mahesh@linux.vnet.ibm.com>
On pseries, as of today system crashes if we get a machine check
exceptions due to SLB errors. These are soft errors and can be fixed by
flushing the SLBs so the kernel can continue to function instead of
system crash. We do this in real mode before turning on MMU. Otherwise
we would run into nested machine checks. This patch now fetches the
rtas error log in real mode and flushes the SLBs on SLB errors.
Signed-off-by: Mahesh Salgaonkar <mahesh@linux.vnet.ibm.com>
---
arch/powerpc/include/asm/book3s/64/mmu-hash.h | 1
arch/powerpc/include/asm/machdep.h | 1
arch/powerpc/kernel/exceptions-64s.S | 42 +++++++++++++++++++
arch/powerpc/kernel/mce.c | 16 ++++++-
arch/powerpc/mm/slb.c | 6 +++
arch/powerpc/platforms/powernv/opal.c | 1
arch/powerpc/platforms/pseries/pseries.h | 1
arch/powerpc/platforms/pseries/ras.c | 56 +++++++++++++++++++++++++
arch/powerpc/platforms/pseries/setup.c | 1
9 files changed, 121 insertions(+), 4 deletions(-)
diff --git a/arch/powerpc/include/asm/book3s/64/mmu-hash.h b/arch/powerpc/include/asm/book3s/64/mmu-hash.h
index 50ed64fba4ae..cc00a7088cf3 100644
--- a/arch/powerpc/include/asm/book3s/64/mmu-hash.h
+++ b/arch/powerpc/include/asm/book3s/64/mmu-hash.h
@@ -487,6 +487,7 @@ extern void hpte_init_native(void);
extern void slb_initialize(void);
extern void slb_flush_and_rebolt(void);
+extern void slb_flush_and_rebolt_realmode(void);
extern void slb_vmalloc_update(void);
extern void slb_set_size(u16 size);
diff --git a/arch/powerpc/include/asm/machdep.h b/arch/powerpc/include/asm/machdep.h
index ffe7c71e1132..fe447e0d4140 100644
--- a/arch/powerpc/include/asm/machdep.h
+++ b/arch/powerpc/include/asm/machdep.h
@@ -108,6 +108,7 @@ struct machdep_calls {
/* Early exception handlers called in realmode */
int (*hmi_exception_early)(struct pt_regs *regs);
+ int (*machine_check_early)(struct pt_regs *regs);
/* Called during machine check exception to retrive fixup address. */
bool (*mce_check_early_recovery)(struct pt_regs *regs);
diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index f283958129f2..0038596b7906 100644
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -332,6 +332,9 @@ TRAMP_REAL_BEGIN(machine_check_pSeries)
machine_check_fwnmi:
SET_SCRATCH0(r13) /* save r13 */
EXCEPTION_PROLOG_0(PACA_EXMC)
+BEGIN_FTR_SECTION
+ b machine_check_pSeries_early
+END_FTR_SECTION_IFCLR(CPU_FTR_HVMODE)
machine_check_pSeries_0:
EXCEPTION_PROLOG_1(PACA_EXMC, KVMTEST_PR, 0x200)
/*
@@ -343,6 +346,45 @@ machine_check_pSeries_0:
TRAMP_KVM_SKIP(PACA_EXMC, 0x200)
+TRAMP_REAL_BEGIN(machine_check_pSeries_early)
+BEGIN_FTR_SECTION
+ EXCEPTION_PROLOG_1(PACA_EXMC, NOTEST, 0x200)
+ mr r10,r1 /* Save r1 */
+ ld r1,PACAMCEMERGSP(r13) /* Use MC emergency stack */
+ subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */
+ mfspr r11,SPRN_SRR0 /* Save SRR0 */
+ mfspr r12,SPRN_SRR1 /* Save SRR1 */
+ EXCEPTION_PROLOG_COMMON_1()
+ EXCEPTION_PROLOG_COMMON_2(PACA_EXMC)
+ EXCEPTION_PROLOG_COMMON_3(0x200)
+ addi r3,r1,STACK_FRAME_OVERHEAD
+ BRANCH_LINK_TO_FAR(machine_check_early) /* Function call ABI */
+
+ /* Move original SRR0 and SRR1 into the respective regs */
+ ld r9,_MSR(r1)
+ mtspr SPRN_SRR1,r9
+ ld r3,_NIP(r1)
+ mtspr SPRN_SRR0,r3
+ ld r9,_CTR(r1)
+ mtctr r9
+ ld r9,_XER(r1)
+ mtxer r9
+ ld r9,_LINK(r1)
+ mtlr r9
+ REST_GPR(0, r1)
+ REST_8GPRS(2, r1)
+ REST_GPR(10, r1)
+ ld r11,_CCR(r1)
+ mtcr r11
+ REST_GPR(11, r1)
+ REST_2GPRS(12, r1)
+ /* restore original r1. */
+ ld r1,GPR1(r1)
+ SET_SCRATCH0(r13) /* save r13 */
+ EXCEPTION_PROLOG_0(PACA_EXMC)
+ b machine_check_pSeries_0
+END_FTR_SECTION_IFCLR(CPU_FTR_HVMODE)
+
EXC_COMMON_BEGIN(machine_check_common)
/*
* Machine check is different because we use a different
diff --git a/arch/powerpc/kernel/mce.c b/arch/powerpc/kernel/mce.c
index efdd16a79075..221271c96a57 100644
--- a/arch/powerpc/kernel/mce.c
+++ b/arch/powerpc/kernel/mce.c
@@ -488,9 +488,21 @@ long machine_check_early(struct pt_regs *regs)
{
long handled = 0;
- __this_cpu_inc(irq_stat.mce_exceptions);
+ /*
+ * For pSeries we count mce when we go into virtual mode machine
+ * check handler. Hence skip it. Also, We can't access per cpu
+ * variables in real mode for LPAR.
+ */
+ if (early_cpu_has_feature(CPU_FTR_HVMODE))
+ __this_cpu_inc(irq_stat.mce_exceptions);
- if (cur_cpu_spec && cur_cpu_spec->machine_check_early)
+ /*
+ * See if platform is capable of handling machine check.
+ * Otherwise fallthrough and allow CPU to handle this machine check.
+ */
+ if (ppc_md.machine_check_early)
+ handled = ppc_md.machine_check_early(regs);
+ else if (cur_cpu_spec && cur_cpu_spec->machine_check_early)
handled = cur_cpu_spec->machine_check_early(regs);
return handled;
}
diff --git a/arch/powerpc/mm/slb.c b/arch/powerpc/mm/slb.c
index 66577cc66dc9..5b1813b98358 100644
--- a/arch/powerpc/mm/slb.c
+++ b/arch/powerpc/mm/slb.c
@@ -145,6 +145,12 @@ void slb_flush_and_rebolt(void)
get_paca()->slb_cache_ptr = 0;
}
+void slb_flush_and_rebolt_realmode(void)
+{
+ __slb_flush_and_rebolt();
+ get_paca()->slb_cache_ptr = 0;
+}
+
void slb_vmalloc_update(void)
{
unsigned long vflags;
diff --git a/arch/powerpc/platforms/powernv/opal.c b/arch/powerpc/platforms/powernv/opal.c
index 48fbb41af5d1..ed548d40a9e1 100644
--- a/arch/powerpc/platforms/powernv/opal.c
+++ b/arch/powerpc/platforms/powernv/opal.c
@@ -417,7 +417,6 @@ static int opal_recover_mce(struct pt_regs *regs,
if (!(regs->msr & MSR_RI)) {
/* If MSR_RI isn't set, we cannot recover */
- pr_err("Machine check interrupt unrecoverable: MSR(RI=0)\n");
recovered = 0;
} else if (evt->disposition == MCE_DISPOSITION_RECOVERED) {
/* Platform corrected itself */
diff --git a/arch/powerpc/platforms/pseries/pseries.h b/arch/powerpc/platforms/pseries/pseries.h
index 60db2ee511fb..3611db5dd583 100644
--- a/arch/powerpc/platforms/pseries/pseries.h
+++ b/arch/powerpc/platforms/pseries/pseries.h
@@ -24,6 +24,7 @@ struct pt_regs;
extern int pSeries_system_reset_exception(struct pt_regs *regs);
extern int pSeries_machine_check_exception(struct pt_regs *regs);
+extern int pSeries_machine_check_realmode(struct pt_regs *regs);
#ifdef CONFIG_SMP
extern void smp_init_pseries(void);
diff --git a/arch/powerpc/platforms/pseries/ras.c b/arch/powerpc/platforms/pseries/ras.c
index e3bd849141de..ae08263daa24 100644
--- a/arch/powerpc/platforms/pseries/ras.c
+++ b/arch/powerpc/platforms/pseries/ras.c
@@ -333,6 +333,11 @@ static irqreturn_t ras_error_interrupt(int irq, void *dev_id)
((((A) >= 0x7000) && ((A) < 0x7ff0)) || \
(((A) >= rtas.base) && ((A) < (rtas.base + rtas.size - 16))))
+static inline struct rtas_error_log *fwnmi_get_errlog(void)
+{
+ return (struct rtas_error_log *)local_paca->mce_data_buf;
+}
+
/*
* Get the error information for errors coming through the
* FWNMI vectors. The pt_regs' r3 will be updated to reflect
@@ -423,6 +428,35 @@ int pSeries_system_reset_exception(struct pt_regs *regs)
return 0; /* need to perform reset */
}
+static int mce_handle_error(struct rtas_error_log *errp)
+{
+ struct pseries_errorlog *pseries_log;
+ struct pseries_mc_errorlog *mce_log;
+ int disposition = rtas_error_disposition(errp);
+ uint8_t error_type;
+
+ if (!rtas_error_extended(errp))
+ goto out;
+
+ pseries_log = get_pseries_errorlog(errp, PSERIES_ELOG_SECT_ID_MCE);
+ if (pseries_log == NULL)
+ goto out;
+
+ mce_log = (struct pseries_mc_errorlog *)pseries_log->data;
+ error_type = rtas_mc_error_type(mce_log);
+
+ if ((disposition == RTAS_DISP_NOT_RECOVERED) &&
+ (error_type == PSERIES_MC_ERROR_TYPE_SLB)) {
+ /* Store the old slb content someplace. */
+ slb_flush_and_rebolt_realmode();
+ disposition = RTAS_DISP_FULLY_RECOVERED;
+ rtas_set_disposition_recovered(errp);
+ }
+
+out:
+ return disposition;
+}
+
/*
* Process MCE rtas errlog event.
*/
@@ -499,11 +533,31 @@ int pSeries_machine_check_exception(struct pt_regs *regs)
struct rtas_error_log *errp;
if (fwnmi_active) {
- errp = fwnmi_get_errinfo(regs);
fwnmi_release_errinfo();
+ errp = fwnmi_get_errlog();
if (errp && recover_mce(regs, errp))
return 1;
}
return 0;
}
+
+int pSeries_machine_check_realmode(struct pt_regs *regs)
+{
+ struct rtas_error_log *errp;
+ int disposition;
+
+ if (fwnmi_active) {
+ errp = fwnmi_get_errinfo(regs);
+ /*
+ * Call to fwnmi_release_errinfo() in real mode causes kernel
+ * to panic. Hence we will call it as soon as we go into
+ * virtual mode.
+ */
+ disposition = mce_handle_error(errp);
+ if (disposition == RTAS_DISP_FULLY_RECOVERED)
+ return 1;
+ }
+
+ return 0;
+}
diff --git a/arch/powerpc/platforms/pseries/setup.c b/arch/powerpc/platforms/pseries/setup.c
index 60a067a6e743..249b02bc5c41 100644
--- a/arch/powerpc/platforms/pseries/setup.c
+++ b/arch/powerpc/platforms/pseries/setup.c
@@ -999,6 +999,7 @@ define_machine(pseries) {
.calibrate_decr = generic_calibrate_decr,
.progress = rtas_progress,
.system_reset_exception = pSeries_system_reset_exception,
+ .machine_check_early = pSeries_machine_check_realmode,
.machine_check_exception = pSeries_machine_check_exception,
#ifdef CONFIG_KEXEC_CORE
.machine_kexec = pSeries_machine_kexec,
^ permalink raw reply related
* [PATCH v4 5/6] powerpc/pseries: Display machine check error details.
From: Mahesh J Salgaonkar @ 2018-06-28 11:13 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin, Aneesh Kumar K.V, Laurent Dufour
In-Reply-To: <153018397703.31589.3105355336278825912.stgit@jupiter.in.ibm.com>
From: Mahesh Salgaonkar <mahesh@linux.vnet.ibm.com>
Extract the MCE error details from RTAS extended log and display it to
console.
With this patch you should now see mce logs like below:
[ 142.371818] Severe Machine check interrupt [Recovered]
[ 142.371822] NIP [d00000000ca301b8]: init_module+0x1b8/0x338 [bork_kernel]
[ 142.371822] Initiator: CPU
[ 142.371823] Error type: SLB [Multihit]
[ 142.371824] Effective address: d00000000ca70000
Signed-off-by: Mahesh Salgaonkar <mahesh@linux.vnet.ibm.com>
---
arch/powerpc/include/asm/rtas.h | 5 +
arch/powerpc/platforms/pseries/ras.c | 131 ++++++++++++++++++++++++++++++++++
2 files changed, 136 insertions(+)
diff --git a/arch/powerpc/include/asm/rtas.h b/arch/powerpc/include/asm/rtas.h
index ceeed2dd489b..26bc3d5c4992 100644
--- a/arch/powerpc/include/asm/rtas.h
+++ b/arch/powerpc/include/asm/rtas.h
@@ -197,6 +197,11 @@ static inline uint8_t rtas_error_extended(const struct rtas_error_log *elog)
return (elog->byte1 & 0x04) >> 2;
}
+static inline uint8_t rtas_error_initiator(const struct rtas_error_log *elog)
+{
+ return (elog->byte2 & 0xf0) >> 4;
+}
+
#define rtas_error_type(x) ((x)->byte3)
static inline
diff --git a/arch/powerpc/platforms/pseries/ras.c b/arch/powerpc/platforms/pseries/ras.c
index ae08263daa24..be665eeb97df 100644
--- a/arch/powerpc/platforms/pseries/ras.c
+++ b/arch/powerpc/platforms/pseries/ras.c
@@ -428,6 +428,135 @@ int pSeries_system_reset_exception(struct pt_regs *regs)
return 0; /* need to perform reset */
}
+#define VAL_TO_STRING(ar, val) ((val < ARRAY_SIZE(ar)) ? ar[val] : "Unknown")
+
+static void pseries_print_mce_info(struct pt_regs *regs,
+ struct rtas_error_log *errp)
+{
+ const char *level, *sevstr;
+ struct pseries_errorlog *pseries_log;
+ struct pseries_mc_errorlog *mce_log;
+ uint8_t error_type, err_sub_type;
+ uint64_t addr;
+ uint8_t initiator = rtas_error_initiator(errp);
+ int disposition = rtas_error_disposition(errp);
+
+ static const char * const initiators[] = {
+ "Unknown",
+ "CPU",
+ "PCI",
+ "ISA",
+ "Memory",
+ "Power Mgmt",
+ };
+ static const char * const mc_err_types[] = {
+ "UE",
+ "SLB",
+ "ERAT",
+ "TLB",
+ "D-Cache",
+ "Unknown",
+ "I-Cache",
+ };
+ static const char * const mc_ue_types[] = {
+ "Indeterminate",
+ "Instruction fetch",
+ "Page table walk ifetch",
+ "Load/Store",
+ "Page table walk Load/Store",
+ };
+
+ /* SLB sub errors valid values are 0x0, 0x1, 0x2 */
+ static const char * const mc_slb_types[] = {
+ "Parity",
+ "Multihit",
+ "Indeterminate",
+ };
+
+ /* TLB and ERAT sub errors valid values are 0x1, 0x2, 0x3 */
+ static const char * const mc_soft_types[] = {
+ "Unknown",
+ "Parity",
+ "Multihit",
+ "Indeterminate",
+ };
+
+ if (!rtas_error_extended(errp)) {
+ pr_err("Machine check interrupt: Missing extended error log\n");
+ return;
+ }
+
+ pseries_log = get_pseries_errorlog(errp, PSERIES_ELOG_SECT_ID_MCE);
+ if (pseries_log == NULL)
+ return;
+
+ mce_log = (struct pseries_mc_errorlog *)pseries_log->data;
+
+ error_type = rtas_mc_error_type(mce_log);
+ err_sub_type = rtas_mc_error_sub_type(mce_log);
+
+ switch (rtas_error_severity(errp)) {
+ case RTAS_SEVERITY_NO_ERROR:
+ level = KERN_INFO;
+ sevstr = "Harmless";
+ break;
+ case RTAS_SEVERITY_WARNING:
+ level = KERN_WARNING;
+ sevstr = "";
+ break;
+ case RTAS_SEVERITY_ERROR:
+ case RTAS_SEVERITY_ERROR_SYNC:
+ level = KERN_ERR;
+ sevstr = "Severe";
+ break;
+ case RTAS_SEVERITY_FATAL:
+ default:
+ level = KERN_ERR;
+ sevstr = "Fatal";
+ break;
+ }
+
+ printk("%s%s Machine check interrupt [%s]\n", level, sevstr,
+ disposition == RTAS_DISP_FULLY_RECOVERED ?
+ "Recovered" : "Not recovered");
+ if (user_mode(regs)) {
+ printk("%s NIP: [%016lx] PID: %d Comm: %s\n", level,
+ regs->nip, current->pid, current->comm);
+ } else {
+ printk("%s NIP [%016lx]: %pS\n", level, regs->nip,
+ (void *)regs->nip);
+ }
+ printk("%s Initiator: %s\n", level,
+ VAL_TO_STRING(initiators, initiator));
+
+ switch (error_type) {
+ case PSERIES_MC_ERROR_TYPE_UE:
+ printk("%s Error type: %s [%s]\n", level,
+ VAL_TO_STRING(mc_err_types, error_type),
+ VAL_TO_STRING(mc_ue_types, err_sub_type));
+ break;
+ case PSERIES_MC_ERROR_TYPE_SLB:
+ printk("%s Error type: %s [%s]\n", level,
+ VAL_TO_STRING(mc_err_types, error_type),
+ VAL_TO_STRING(mc_slb_types, err_sub_type));
+ break;
+ case PSERIES_MC_ERROR_TYPE_ERAT:
+ case PSERIES_MC_ERROR_TYPE_TLB:
+ printk("%s Error type: %s [%s]\n", level,
+ VAL_TO_STRING(mc_err_types, error_type),
+ VAL_TO_STRING(mc_soft_types, err_sub_type));
+ break;
+ default:
+ printk("%s Error type: %s\n", level,
+ VAL_TO_STRING(mc_err_types, error_type));
+ break;
+ }
+
+ addr = rtas_mc_get_effective_addr(mce_log);
+ if (addr)
+ printk("%s Effective address: %016llx\n", level, addr);
+}
+
static int mce_handle_error(struct rtas_error_log *errp)
{
struct pseries_errorlog *pseries_log;
@@ -482,6 +611,8 @@ static int recover_mce(struct pt_regs *regs, struct rtas_error_log *err)
int recovered = 0;
int disposition = rtas_error_disposition(err);
+ pseries_print_mce_info(regs, err);
+
if (!(regs->msr & MSR_RI)) {
/* If MSR_RI isn't set, we cannot recover */
recovered = 0;
^ permalink raw reply related
* [PATCH v4 6/6] powerpc/pseries: Dump the SLB contents on SLB MCE errors.
From: Mahesh J Salgaonkar @ 2018-06-28 11:13 UTC (permalink / raw)
To: linuxppc-dev
Cc: Aneesh Kumar K.V, Michael Ellerman, Nicholas Piggin,
Aneesh Kumar K.V, Laurent Dufour
In-Reply-To: <153018397703.31589.3105355336278825912.stgit@jupiter.in.ibm.com>
From: Mahesh Salgaonkar <mahesh@linux.vnet.ibm.com>
If we get a machine check exceptions due to SLB errors then dump the
current SLB contents which will be very much helpful in debugging the
root cause of SLB errors. Introduce an exclusive buffer per cpu to hold
faulty SLB entries. In real mode mce handler saves the old SLB contents
into this buffer accessible through paca and print it out later in virtual
mode.
With this patch the console will log SLB contents like below on SLB MCE
errors:
[ 3022.938065] SLB contents of cpu 0x3
[ 3022.938066] 00 c000000008000000 400ea1b217000500
[ 3022.938067] 1T ESID= c00000 VSID= ea1b217 LLP:100
[ 3022.938068] 01 d000000008000000 400d43642f000510
[ 3022.938069] 1T ESID= d00000 VSID= d43642f LLP:110
[ 3022.938070] 05 f000000008000000 400a86c85f000500
[ 3022.938071] 1T ESID= f00000 VSID= a86c85f LLP:100
[ 3022.938072] 06 00007f0008000000 400a628b13000d90
[ 3022.938073] 1T ESID= 7f VSID= a628b13 LLP:110
[ 3022.938074] 07 0000000018000000 000b7979f523fd90
[ 3022.938075] 256M ESID= 1 VSID= b7979f523f LLP:110
[ 3022.938076] 08 c000000008000000 400ea1b217000510
[ 3022.938076] 1T ESID= c00000 VSID= ea1b217 LLP:110
[ 3022.938077] 09 c000000008000000 400ea1b217000510
[ 3022.938078] 1T ESID= c00000 VSID= ea1b217 LLP:110
Suggested-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Suggested-by: Michael Ellerman <mpe@ellerman.id.au>
Signed-off-by: Mahesh Salgaonkar <mahesh@linux.vnet.ibm.com>
---
arch/powerpc/include/asm/book3s/64/mmu-hash.h | 7 +++
arch/powerpc/include/asm/paca.h | 1
arch/powerpc/mm/slb.c | 57 +++++++++++++++++++++++++
arch/powerpc/platforms/pseries/ras.c | 10 ++++
arch/powerpc/platforms/pseries/setup.c | 10 ++++
5 files changed, 84 insertions(+), 1 deletion(-)
diff --git a/arch/powerpc/include/asm/book3s/64/mmu-hash.h b/arch/powerpc/include/asm/book3s/64/mmu-hash.h
index cc00a7088cf3..5a3fe282076d 100644
--- a/arch/powerpc/include/asm/book3s/64/mmu-hash.h
+++ b/arch/powerpc/include/asm/book3s/64/mmu-hash.h
@@ -485,9 +485,16 @@ static inline void hpte_init_pseries(void) { }
extern void hpte_init_native(void);
+struct slb_entry {
+ u64 esid;
+ u64 vsid;
+};
+
extern void slb_initialize(void);
extern void slb_flush_and_rebolt(void);
extern void slb_flush_and_rebolt_realmode(void);
+extern void slb_save_contents(struct slb_entry *slb_ptr);
+extern void slb_dump_contents(struct slb_entry *slb_ptr);
extern void slb_vmalloc_update(void);
extern void slb_set_size(u16 size);
diff --git a/arch/powerpc/include/asm/paca.h b/arch/powerpc/include/asm/paca.h
index b441fef53077..653f87c69423 100644
--- a/arch/powerpc/include/asm/paca.h
+++ b/arch/powerpc/include/asm/paca.h
@@ -253,6 +253,7 @@ struct paca_struct {
#endif
#ifdef CONFIG_PPC_PSERIES
u8 *mce_data_buf; /* buffer to hold per cpu rtas errlog */
+ struct slb_entry *mce_faulty_slbs;
#endif /* CONFIG_PPC_PSERIES */
} ____cacheline_aligned;
diff --git a/arch/powerpc/mm/slb.c b/arch/powerpc/mm/slb.c
index 5b1813b98358..476ab0b1d4e8 100644
--- a/arch/powerpc/mm/slb.c
+++ b/arch/powerpc/mm/slb.c
@@ -151,6 +151,63 @@ void slb_flush_and_rebolt_realmode(void)
get_paca()->slb_cache_ptr = 0;
}
+void slb_save_contents(struct slb_entry *slb_ptr)
+{
+ int i;
+ unsigned long e, v;
+
+ if (!slb_ptr)
+ return;
+
+ for (i = 0; i < mmu_slb_size; i++) {
+ asm volatile("slbmfee %0,%1" : "=r" (e) : "r" (i));
+ asm volatile("slbmfev %0,%1" : "=r" (v) : "r" (i));
+ slb_ptr->esid = e;
+ slb_ptr->vsid = v;
+ slb_ptr++;
+ }
+}
+
+void slb_dump_contents(struct slb_entry *slb_ptr)
+{
+ int i;
+ unsigned long e, v;
+ unsigned long llp;
+
+ if (!slb_ptr)
+ return;
+
+ pr_err("SLB contents of cpu 0x%x\n", smp_processor_id());
+
+ for (i = 0; i < mmu_slb_size; i++) {
+ e = slb_ptr->esid;
+ v = slb_ptr->vsid;
+ slb_ptr++;
+
+ if (!e && !v)
+ continue;
+
+ pr_err("%02d %016lx %016lx\n", i, e, v);
+
+ if (!(e & SLB_ESID_V)) {
+ pr_err("\n");
+ continue;
+ }
+ llp = v & SLB_VSID_LLP;
+ if (v & SLB_VSID_B_1T) {
+ pr_err(" 1T ESID=%9lx VSID=%13lx LLP:%3lx\n",
+ GET_ESID_1T(e),
+ (v & ~SLB_VSID_B) >> SLB_VSID_SHIFT_1T,
+ llp);
+ } else {
+ pr_err(" 256M ESID=%9lx VSID=%13lx LLP:%3lx\n",
+ GET_ESID(e),
+ (v & ~SLB_VSID_B) >> SLB_VSID_SHIFT,
+ llp);
+ }
+ }
+}
+
void slb_vmalloc_update(void)
{
unsigned long vflags;
diff --git a/arch/powerpc/platforms/pseries/ras.c b/arch/powerpc/platforms/pseries/ras.c
index be665eeb97df..58254d713e89 100644
--- a/arch/powerpc/platforms/pseries/ras.c
+++ b/arch/powerpc/platforms/pseries/ras.c
@@ -516,6 +516,10 @@ static void pseries_print_mce_info(struct pt_regs *regs,
break;
}
+ /* Display faulty slb contents for SLB errors. */
+ if (error_type == PSERIES_MC_ERROR_TYPE_SLB)
+ slb_dump_contents(local_paca->mce_faulty_slbs);
+
printk("%s%s Machine check interrupt [%s]\n", level, sevstr,
disposition == RTAS_DISP_FULLY_RECOVERED ?
"Recovered" : "Not recovered");
@@ -576,7 +580,11 @@ static int mce_handle_error(struct rtas_error_log *errp)
if ((disposition == RTAS_DISP_NOT_RECOVERED) &&
(error_type == PSERIES_MC_ERROR_TYPE_SLB)) {
- /* Store the old slb content someplace. */
+ /*
+ * Store the old slb content in paca before flushing. Print
+ * this when we go to virtual mode.
+ */
+ slb_save_contents(local_paca->mce_faulty_slbs);
slb_flush_and_rebolt_realmode();
disposition = RTAS_DISP_FULLY_RECOVERED;
rtas_set_disposition_recovered(errp);
diff --git a/arch/powerpc/platforms/pseries/setup.c b/arch/powerpc/platforms/pseries/setup.c
index 249b02bc5c41..76d15e46a152 100644
--- a/arch/powerpc/platforms/pseries/setup.c
+++ b/arch/powerpc/platforms/pseries/setup.c
@@ -105,6 +105,9 @@ static void __init fwnmi_init(void)
u8 *mce_data_buf;
unsigned int i;
int nr_cpus = num_possible_cpus();
+ struct slb_entry *slb_ptr;
+ size_t size;
+
int ibm_nmi_register = rtas_token("ibm,nmi-register");
if (ibm_nmi_register == RTAS_UNKNOWN_SERVICE)
@@ -130,6 +133,13 @@ static void __init fwnmi_init(void)
paca_ptrs[i]->mce_data_buf = mce_data_buf +
(RTAS_ERROR_LOG_MAX * i);
}
+
+ /* Allocate per cpu slb area to save old slb contents during MCE */
+ size = sizeof(struct slb_entry) * mmu_slb_size * nr_cpus;
+ slb_ptr = __va(memblock_alloc_base(size, sizeof(struct slb_entry),
+ ppc64_rma_size));
+ for_each_possible_cpu(i)
+ paca_ptrs[i]->mce_faulty_slbs = slb_ptr + (mmu_slb_size * i);
}
static void pseries_8259_cascade(struct irq_desc *desc)
^ permalink raw reply related
* Re: [PATCH v4 1/6] powerpc/pseries: Defer the logging of rtas error to irq work queue.
From: Laurent Dufour @ 2018-06-28 13:19 UTC (permalink / raw)
To: Mahesh J Salgaonkar, linuxppc-dev
Cc: stable, Nicholas Piggin, Aneesh Kumar K.V
In-Reply-To: <153018423713.31589.14195888319570120941.stgit@jupiter.in.ibm.com>
On 28/06/2018 13:10, Mahesh J Salgaonkar wrote:
> From: Mahesh Salgaonkar <mahesh@linux.vnet.ibm.com>
>
> rtas_log_buf is a buffer to hold RTAS event data that are communicated
> to kernel by hypervisor. This buffer is then used to pass RTAS event
> data to user through proc fs. This buffer is allocated from vmalloc
> (non-linear mapping) area.
>
> On Machine check interrupt, register r3 points to RTAS extended event
> log passed by hypervisor that contains the MCE event. The pseries
> machine check handler then logs this error into rtas_log_buf. The
> rtas_log_buf is a vmalloc-ed (non-linear) buffer we end up taking up a
> page fault (vector 0x300) while accessing it. Since machine check
> interrupt handler runs in NMI context we can not afford to take any
> page fault. Page faults are not honored in NMI context and causes
> kernel panic. Apart from that, as Nick pointed out, pSeries_log_error()
> also takes a spin_lock while logging error which is not safe in NMI
> context. It may endup in deadlock if we get another MCE before releasing
> the lock. Fix this by deferring the logging of rtas error to irq work queue.
>
> Current implementation uses two different buffers to hold rtas error log
> depending on whether extended log is provided or not. This makes bit
> difficult to identify which buffer has valid data that needs to logged
> later in irq work. Simplify this using single buffer, one per paca, and
> copy rtas log to it irrespective of whether extended log is provided or
> not. Allocate this buffer below RMA region so that it can be accessed
> in real mode mce handler.
>
> Fixes: b96672dd840f ("powerpc: Machine check interrupt is a non-maskable interrupt")
> Cc: stable@vger.kernel.org
> Signed-off-by: Mahesh Salgaonkar <mahesh@linux.vnet.ibm.com>
> ---
> arch/powerpc/include/asm/paca.h | 3 ++
> arch/powerpc/platforms/pseries/ras.c | 39 +++++++++++++++++++++-----------
> arch/powerpc/platforms/pseries/setup.c | 16 +++++++++++++
> 3 files changed, 45 insertions(+), 13 deletions(-)
>
> diff --git a/arch/powerpc/include/asm/paca.h b/arch/powerpc/include/asm/paca.h
> index 3f109a3e3edb..b441fef53077 100644
> --- a/arch/powerpc/include/asm/paca.h
> +++ b/arch/powerpc/include/asm/paca.h
> @@ -251,6 +251,9 @@ struct paca_struct {
> void *rfi_flush_fallback_area;
> u64 l1d_flush_size;
> #endif
> +#ifdef CONFIG_PPC_PSERIES
> + u8 *mce_data_buf; /* buffer to hold per cpu rtas errlog */
> +#endif /* CONFIG_PPC_PSERIES */
> } ____cacheline_aligned;
>
> extern void copy_mm_to_paca(struct mm_struct *mm);
> diff --git a/arch/powerpc/platforms/pseries/ras.c b/arch/powerpc/platforms/pseries/ras.c
> index 5e1ef9150182..f6ba9a2a4f84 100644
> --- a/arch/powerpc/platforms/pseries/ras.c
> +++ b/arch/powerpc/platforms/pseries/ras.c
> @@ -22,6 +22,7 @@
> #include <linux/of.h>
> #include <linux/fs.h>
> #include <linux/reboot.h>
> +#include <linux/irq_work.h>
>
> #include <asm/machdep.h>
> #include <asm/rtas.h>
> @@ -32,11 +33,13 @@
> static unsigned char ras_log_buf[RTAS_ERROR_LOG_MAX];
> static DEFINE_SPINLOCK(ras_log_buf_lock);
>
> -static char global_mce_data_buf[RTAS_ERROR_LOG_MAX];
> -static DEFINE_PER_CPU(__u64, mce_data_buf);
> -
> static int ras_check_exception_token;
>
> +static void mce_process_errlog_event(struct irq_work *work);
> +static struct irq_work mce_errlog_process_work = {
> + .func = mce_process_errlog_event,
> +};
> +
> #define EPOW_SENSOR_TOKEN 9
> #define EPOW_SENSOR_INDEX 0
>
> @@ -336,10 +339,9 @@ static irqreturn_t ras_error_interrupt(int irq, void *dev_id)
> * the actual r3 if possible, and a ptr to the error log entry
> * will be returned if found.
> *
> - * If the RTAS error is not of the extended type, then we put it in a per
> - * cpu 64bit buffer. If it is the extended type we use global_mce_data_buf.
> + * Use one buffer mce_data_buf per cpu to store RTAS error.
> *
> - * The global_mce_data_buf does not have any locks or protection around it,
> + * The mce_data_buf does not have any locks or protection around it,
> * if a second machine check comes in, or a system reset is done
> * before we have logged the error, then we will get corruption in the
> * error log. This is preferable over holding off on calling
> @@ -362,20 +364,19 @@ static struct rtas_error_log *fwnmi_get_errinfo(struct pt_regs *regs)
> savep = __va(regs->gpr[3]);
> regs->gpr[3] = savep[0]; /* restore original r3 */
>
> - /* If it isn't an extended log we can use the per cpu 64bit buffer */
> h = (struct rtas_error_log *)&savep[1];
> + /* Use the per cpu buffer from paca to store rtas error log */
> + memset(local_paca->mce_data_buf, 0, RTAS_ERROR_LOG_MAX);
> if (!rtas_error_extended(h)) {
> - memcpy(this_cpu_ptr(&mce_data_buf), h, sizeof(__u64));
> - errhdr = (struct rtas_error_log *)this_cpu_ptr(&mce_data_buf);
> + memcpy(local_paca->mce_data_buf, h, sizeof(__u64));
> } else {
> int len, error_log_length;
>
> error_log_length = 8 + rtas_error_extended_log_length(h);
> len = max_t(int, error_log_length, RTAS_ERROR_LOG_MAX);
> - memset(global_mce_data_buf, 0, RTAS_ERROR_LOG_MAX);
> - memcpy(global_mce_data_buf, h, len);
> - errhdr = (struct rtas_error_log *)global_mce_data_buf;
> + memcpy(local_paca->mce_data_buf, h, len);
> }
> + errhdr = (struct rtas_error_log *)local_paca->mce_data_buf;
You should drop errhdr in this function and simply do
return (struct rtas_error_log *)local_paca->mce_data_buf;
>
> return errhdr;
> }
> @@ -422,6 +423,17 @@ int pSeries_system_reset_exception(struct pt_regs *regs)
> return 0; /* need to perform reset */
> }
>
> +/*
> + * Process MCE rtas errlog event.
> + */
> +static void mce_process_errlog_event(struct irq_work *work)
> +{
> + struct rtas_error_log *err;
> +
> + err = fwnmi_get_errlog();
> + log_error((char *)err, ERR_TYPE_RTAS_LOG, 0);
> +}
> +
> /*
> * See if we can recover from a machine check exception.
> * This is only called on power4 (or above) and only via
> @@ -466,7 +478,8 @@ static int recover_mce(struct pt_regs *regs, struct rtas_error_log *err)
> recovered = 1;
> }
>
> - log_error((char *)err, ERR_TYPE_RTAS_LOG, 0);
> + /* Queue irq work to log this rtas event later. */
> + irq_work_queue(&mce_errlog_process_work);
>
> return recovered;
> }
> diff --git a/arch/powerpc/platforms/pseries/setup.c b/arch/powerpc/platforms/pseries/setup.c
> index fdb32e056ef4..60a067a6e743 100644
> --- a/arch/powerpc/platforms/pseries/setup.c
> +++ b/arch/powerpc/platforms/pseries/setup.c
> @@ -41,6 +41,7 @@
> #include <linux/root_dev.h>
> #include <linux/of.h>
> #include <linux/of_pci.h>
> +#include <linux/memblock.h>
>
> #include <asm/mmu.h>
> #include <asm/processor.h>
> @@ -101,6 +102,9 @@ static void pSeries_show_cpuinfo(struct seq_file *m)
> static void __init fwnmi_init(void)
> {
> unsigned long system_reset_addr, machine_check_addr;
> + u8 *mce_data_buf;
> + unsigned int i;
> + int nr_cpus = num_possible_cpus();
>
> int ibm_nmi_register = rtas_token("ibm,nmi-register");
> if (ibm_nmi_register == RTAS_UNKNOWN_SERVICE)
> @@ -114,6 +118,18 @@ static void __init fwnmi_init(void)
> if (0 == rtas_call(ibm_nmi_register, 2, 1, NULL, system_reset_addr,
> machine_check_addr))
> fwnmi_active = 1;
> +
> + /*
> + * Allocate a chunk for per cpu buffer to hold rtas errorlog.
> + * It will be used in real mode mce handler, hence it needs to be
> + * below RMA.
> + */
> + mce_data_buf = __va(memblock_alloc_base(RTAS_ERROR_LOG_MAX * nr_cpus,
> + RTAS_ERROR_LOG_MAX, ppc64_rma_size));
> + for_each_possible_cpu(i) {
> + paca_ptrs[i]->mce_data_buf = mce_data_buf +
> + (RTAS_ERROR_LOG_MAX * i);
> + }
> }
>
> static void pseries_8259_cascade(struct irq_desc *desc)
>
^ permalink raw reply
* Re: [PATCH 2/3] drivers/base: reorder consumer and its children behind suppliers
From: Pingfan Liu @ 2018-06-28 13:47 UTC (permalink / raw)
To: dan.carpenter
Cc: kbuild, kbuild-all, linux-kernel, Greg Kroah-Hartman,
Grygorii Strashko, Christoph Hellwig, Bjorn Helgaas, Dave Young,
linux-pci, linuxppc-dev
In-Reply-To: <20180627083445.jsu2ffyz5cl4wbjn@mwanda>
On Wed, Jun 27, 2018 at 4:35 PM Dan Carpenter <dan.carpenter@oracle.com> wrote:
>
> On Wed, Jun 27, 2018 at 10:34:54AM +0800, Pingfan Liu wrote:
> > > 1b2a1e63 Pingfan Liu 2018-06-25 243 }
> > > 1b2a1e63 Pingfan Liu 2018-06-25 244 }
> > > 1b2a1e63 Pingfan Liu 2018-06-25 @245 BUG_ON(!ret);
> > >
> > > If the list is empty then "ret" can be unitialized. We test a different
> > > list "dev->links.suppliers" to see if that's empty. I wrote a bunch of
> > > code to make Smatch try to understand about empty lists, but I don't
> > > think it works...
> > >
> > Yes, if list_empty, then the code can not touch ret. But ret is
> > useless in this scene. Does it matter?
> >
>
> I'm not sure I understand what you're asking? Of course, it matters?
>
Oh, I misunderstood your original comment. Yes, you are right. I will
fix it in next version, if this code section is still used.
Thanks and regards,
Pingfan
^ permalink raw reply
* Oops in kmem_cache_free() via bioset_exit() (was Re: [next-20180601][nvme][ppc] Kernel Oops is triggered when creating lvm snapshots on nvme disks)
From: Michael Ellerman @ 2018-06-28 14:42 UTC (permalink / raw)
To: Abdul Haleem
Cc: linuxppc-dev, linux-fsdevel, linux-next, linux-kernel, linux-scsi,
Stephen Rothwell, sachinp, sim, manvanth, Brian King, linux-block,
Kent Overstreet, Jens Axboe
In-Reply-To: <1530176707.24245.12.camel@abdul.in.ibm.com>
Kent, Jens,
This looks like it might be related to the recent bioset changes?
cheers
Abdul Haleem <abdhalee@linux.vnet.ibm.com> writes:
> On Tue, 2018-06-26 at 23:36 +1000, Michael Ellerman wrote:
>> Abdul Haleem <abdhalee@linux.vnet.ibm.com> writes:
...
> I was able to reproduce again with slub_debug=FZP and DEBUG_INFO enabled
> on 4.17.0-rc7-next-20180601, but not much traces other than the Oops stack trace
Are you still testing on that revision? It's nearly a month old.
Please try to reproduce on mainline or today's linux-next.
> the faulty instruction points to below code path :
>
> gdb -batch vmlinux -ex 'list *(0xc000000000304fe0)'
> 0xc000000000304fe0 is in kmem_cache_free (mm/slab.h:231).
> 226 }
> 227
> 228 static inline bool slab_equal_or_root(struct kmem_cache *s,
> 229 struct kmem_cache *p)
> 230 {
> 231 return p == s || p == s->memcg_params.root_cache;
> 232 }
And s is NULL.
Called via:
kmem_cache_free+0x210/0x2a0
mempool_free_slab+0x24/0x40
mempool_exit+0x50/0x90
bioset_exit+0x40/0x1d0
dm_io_client_destroy+0x2c/0x50
dm_bufio_client_destroy+0x1fc/0x2d0 [dm_bufio]
persistent_read_metadata+0x430/0x660 [dm_snapshot]
snapshot_ctr+0x5c8/0x7a0 [dm_snapshot]
dm_table_add_target+0x19c/0x3c0
table_load+0x104/0x450
ctl_ioctl+0x1f8/0x570
dm_ctl_ioctl+0x18/0x30
do_vfs_ioctl+0xcc/0x9e0
ksys_ioctl+0x5c/0xe0
sys_ioctl+0x20/0x80
system_call+0x58/0x6c
So looks like we did:
kmem_cache_free(NULL, x)
Probably a bad error path that frees before the cache has been allocated.
mempool_init_node() calls mempool_exit() on a partially initialised
mempool, which looks fishy, though you're not hitting that patch AFAICS.
This patch should hopefully catch it earlier:
diff --git a/mm/mempool.c b/mm/mempool.c
index b54f2c20e5e0..6e23d7a119d4 100644
--- a/mm/mempool.c
+++ b/mm/mempool.c
@@ -508,6 +508,10 @@ EXPORT_SYMBOL(mempool_alloc_slab);
void mempool_free_slab(void *element, void *pool_data)
{
struct kmem_cache *mem = pool_data;
+
+ if (WARN_ON_ONCE(!mem))
+ return;
+
kmem_cache_free(mem, element);
}
EXPORT_SYMBOL(mempool_free_slab);
cheers
> [ 0.000000] dt-cpu-ftrs: setup for ISA 3000
> [ 0.000000] dt-cpu-ftrs: not enabling: system-call-vectored (disabled or unsupported by kernel)
> [ 0.000000] dt-cpu-ftrs: final cpu/mmu features = 0x0000786f8f5fb1a7 0x3c006041
> [ 0.000000] radix-mmu: Page sizes from device-tree:
> [ 0.000000] radix-mmu: Page size shift = 12 AP=0x0
> [ 0.000000] radix-mmu: Page size shift = 16 AP=0x5
> [ 0.000000] radix-mmu: Page size shift = 21 AP=0x1
> [ 0.000000] radix-mmu: Page size shift = 30 AP=0x2
> [ 0.000000] radix-mmu: Initializing Radix MMU
> [ 0.000000] radix-mmu: Partition table (____ptrval____)
> [ 0.000000] radix-mmu: Mapped 0x0000000000000000-0x0000001000000000 with 1.00 GiB pages
> [ 0.000000] radix-mmu: Mapped 0x0000200000000000-0x0000201000000000 with 1.00 GiB pages
> [ 0.000000] radix-mmu: Process table (____ptrval____) and radix root for kernel: (____ptrval____)
> [ 0.000000] Linux version 4.17.0-rc7-next-20180601-autotest (root@ltc-boston21) (gcc version 7.3.0 (Ubuntu 7.3.0-16ubuntu3)) #3 SMP Thu Jun 28 03:01:06 CDT 2018
> [ 0.000000] Found initrd at 0xc000000002d50000:0xc000000009265921
> [ 0.000000] OPAL: Found memory mapped LPC bus on chip 0
> [ 0.000000] ISA: Non-PCI bridge is /lpcm-opb@6030000000000/lpc@0
> [ 0.000000] Using PowerNV machine description
> [ 0.000000] bootconsole [udbg0] enabled
> [ 0.000000] CPU maps initialized for 4 threads per core
> [ 0.000000] (thread shift is 2)
> [ 0.000000] Allocated 4352 bytes for 128 pacas
> [ 0.000000] -----------------------------------------------------
> [ 0.000000] ppc64_pft_size = 0x0
> [ 0.000000] phys_mem_size = 0x2000000000
> [ 0.000000] dcache_bsize = 0x80
> [ 0.000000] icache_bsize = 0x80
> [ 0.000000] cpu_features = 0x0000786f8f5fb1a7
> [ 0.000000] possible = 0x00007fffcf5fb1a7
> [ 0.000000] always = 0x00000003800081a1
> [ 0.000000] cpu_user_features = 0xdc0065c2 0xaee00000
> [ 0.000000] mmu_features = 0x3c006041
> [ 0.000000] firmware_features = 0x0000000110000000
> [ 0.000000] -----------------------------------------------------
> [ 0.000000] cma: Reserved 6560 MiB at 0x0000200e62000000
> [ 0.000000] numa: NODE_DATA [mem 0xfffabe300-0xfffac7fff]
> [ 0.000000] numa: NODE_DATA [mem 0x200fff1a0300-0x200fff1a9fff]
> [ 0.000000] rfi-flush: mttrig type flush available
> [ 0.000000] rfi-flush: patched 9 locations (mttrig type flush)
> [ 0.000000] stf-barrier: eieio barrier available
> [ 0.000000] stf-barrier: patched 61 entry locations (eieio barrier)
> [ 0.000000] stf-barrier: patched 9 exit locations (eieio barrier)
> [ 0.000000] Initializing IODA2 PHB (/pciex@600c3c0000000)
> [ 0.000000] PCI host bridge /pciex@600c3c0000000 (primary) ranges:
> [ 0.000000] MEM 0x000600c000000000..0x000600c07ffeffff -> 0x0000000080000000
> [ 0.000000] MEM 0x0006000000000000..0x0006003fffffffff -> 0x0006000000000000 (M64 #1..31)
> [ 0.000000] Using M64 #31 as default window
> [ 0.000000] 512 (511) PE's M32: 0x80000000 [segment=0x400000]
> [ 0.000000] M64: 0x4000000000 [segment=0x20000000]
> [ 0.000000] Allocated bitmap for 4088 MSIs (base IRQ 0xfe000)
> [ 0.000000] Initializing IODA2 PHB (/pciex@600c3c0100000)
> [ 0.000000] PCI host bridge /pciex@600c3c0100000 ranges:
> [ 0.000000] MEM 0x000600c080000000..0x000600c0fffeffff -> 0x0000000080000000
> [ 0.000000] MEM 0x0006004000000000..0x0006007fffffffff -> 0x0006004000000000 (M64 #1..15)
> [ 0.000000] Using M64 #15 as default window
> [ 0.000000] 256 (255) PE's M32: 0x80000000 [segment=0x800000]
> [ 0.000000] M64: 0x4000000000 [segment=0x40000000]
> [ 0.000000] Allocated bitmap for 2040 MSIs (base IRQ 0xfd800)
> [ 0.000000] Initializing IODA2 PHB (/pciex@600c3c0200000)
> [ 0.000000] PCI host bridge /pciex@600c3c0200000 ranges:
> [ 0.000000] MEM 0x000600c100000000..0x000600c17ffeffff -> 0x0000000080000000
> [ 0.000000] MEM 0x0006008000000000..0x000600bfffffffff -> 0x0006008000000000 (M64 #1..15)
> [ 0.000000] Using M64 #15 as default window
> [ 0.000000] 256 (255) PE's M32: 0x80000000 [segment=0x800000]
> [ 0.000000] M64: 0x4000000000 [segment=0x40000000]
> [ 0.000000] Allocated bitmap for 2040 MSIs (base IRQ 0xfd000)
> [ 0.000000] Initializing IODA2 PHB (/pciex@600c3c0300000)
> [ 0.000000] PCI host bridge /pciex@600c3c0300000 ranges:
> [ 0.000000] MEM 0x000600c180000000..0x000600c1fffeffff -> 0x0000000080000000
> [ 0.000000] MEM 0x0006020000000000..0x0006023fffffffff -> 0x0006020000000000 (M64 #1..31)
> [ 0.000000] Using M64 #31 as default window
> [ 0.000000] 512 (511) PE's M32: 0x80000000 [segment=0x400000]
> [ 0.000000] M64: 0x4000000000 [segment=0x20000000]
> [ 0.000000] Allocated bitmap for 4088 MSIs (base IRQ 0xfc000)
> [ 0.000000] Initializing IODA2 PHB (/pciex@600c3c0400000)
> [ 0.000000] PCI host bridge /pciex@600c3c0400000 ranges:
> [ 0.000000] MEM 0x000600c200000000..0x000600c27ffeffff -> 0x0000000080000000
> [ 0.000000] MEM 0x0006024000000000..0x0006027fffffffff -> 0x0006024000000000 (M64 #1..15)
> [ 0.000000] Using M64 #15 as default window
> [ 0.000000] 256 (255) PE's M32: 0x80000000 [segment=0x800000]
> [ 0.000000] M64: 0x4000000000 [segment=0x40000000]
> [ 0.000000] Allocated bitmap for 2040 MSIs (base IRQ 0xfb800)
> [ 0.000000] Initializing IODA2 PHB (/pciex@600c3c0500000)
> [ 0.000000] PCI host bridge /pciex@600c3c0500000 ranges:
> [ 0.000000] MEM 0x000600c280000000..0x000600c2fffeffff -> 0x0000000080000000
> [ 0.000000] MEM 0x0006028000000000..0x000602bfffffffff -> 0x0006028000000000 (M64 #1..15)
> [ 0.000000] Using M64 #15 as default window
> [ 0.000000] 256 (255) PE's M32: 0x80000000 [segment=0x800000]
> [ 0.000000] M64: 0x4000000000 [segment=0x40000000]
> [ 0.000000] Allocated bitmap for 2040 MSIs (base IRQ 0xfb000)
> [ 0.000000] Initializing IODA2 PHB (/pciex@620c3c0000000)
> [ 0.000000] PCI host bridge /pciex@620c3c0000000 ranges:
> [ 0.000000] MEM 0x000620c000000000..0x000620c07ffeffff -> 0x0000000080000000
> [ 0.000000] MEM 0x0006200000000000..0x0006203fffffffff -> 0x0006200000000000 (M64 #1..31)
> [ 0.000000] Using M64 #31 as default window
> [ 0.000000] 512 (511) PE's M32: 0x80000000 [segment=0x400000]
> [ 0.000000] M64: 0x4000000000 [segment=0x20000000]
> [ 0.000000] Allocated bitmap for 4088 MSIs (base IRQ 0x1fe000)
> [ 0.000000] Initializing IODA2 PHB (/pciex@620c3c0100000)
> [ 0.000000] PCI host bridge /pciex@620c3c0100000 ranges:
> [ 0.000000] MEM 0x000620c080000000..0x000620c0fffeffff -> 0x0000000080000000
> [ 0.000000] MEM 0x0006204000000000..0x0006207fffffffff -> 0x0006204000000000 (M64 #1..15)
> [ 0.000000] Using M64 #15 as default window
> [ 0.000000] 256 (255) PE's M32: 0x80000000 [segment=0x800000]
> [ 0.000000] M64: 0x4000000000 [segment=0x40000000]
> [ 0.000000] Allocated bitmap for 2040 MSIs (base IRQ 0x1fd800)
> [ 0.000000] Initializing IODA2 PHB (/pciex@620c3c0200000)
> [ 0.000000] PCI host bridge /pciex@620c3c0200000 ranges:
> [ 0.000000] MEM 0x000620c100000000..0x000620c17ffeffff -> 0x0000000080000000
> [ 0.000000] MEM 0x0006208000000000..0x000620bfffffffff -> 0x0006208000000000 (M64 #1..15)
> [ 0.000000] Using M64 #15 as default window
> [ 0.000000] 256 (255) PE's M32: 0x80000000 [segment=0x800000]
> [ 0.000000] M64: 0x4000000000 [segment=0x40000000]
> [ 0.000000] Allocated bitmap for 2040 MSIs (base IRQ 0x1fd000)
> [ 0.000000] Initializing IODA2 PHB (/pciex@620c3c0300000)
> [ 0.000000] PCI host bridge /pciex@620c3c0300000 ranges:
> [ 0.000000] MEM 0x000620c180000000..0x000620c1fffeffff -> 0x0000000080000000
> [ 0.000000] MEM 0x0006220000000000..0x0006223fffffffff -> 0x0006220000000000 (M64 #1..31)
> [ 0.000000] Using M64 #31 as default window
> [ 0.000000] 512 (511) PE's M32: 0x80000000 [segment=0x400000]
> [ 0.000000] M64: 0x4000000000 [segment=0x20000000]
> [ 0.000000] Allocated bitmap for 4088 MSIs (base IRQ 0x1fc000)
> [ 0.000000] Initializing IODA2 PHB (/pciex@620c3c0400000)
> [ 0.000000] PCI host bridge /pciex@620c3c0400000 ranges:
> [ 0.000000] MEM 0x000620c200000000..0x000620c27ffeffff -> 0x0000000080000000
> [ 0.000000] MEM 0x0006224000000000..0x0006227fffffffff -> 0x0006224000000000 (M64 #1..15)
> [ 0.000000] Using M64 #15 as default window
> [ 0.000000] 256 (255) PE's M32: 0x80000000 [segment=0x800000]
> [ 0.000000] M64: 0x4000000000 [segment=0x40000000]
> [ 0.000000] Allocated bitmap for 2040 MSIs (base IRQ 0x1fb800)
> [ 0.000000] OPAL nvram setup, 589824 bytes
> [ 0.000000] Top of RAM: 0x201000000000, Total RAM: 0x2000000000
> [ 0.000000] Memory hole size: 33488896MB
> [ 0.000000] Zone ranges:
> [ 0.000000] DMA [mem 0x0000000000000000-0x0000200fffffffff]
> [ 0.000000] DMA32 empty
> [ 0.000000] Normal empty
> [ 0.000000] Movable zone start for each node
> [ 0.000000] Early memory node ranges
> [ 0.000000] node 0: [mem 0x0000000000000000-0x0000000fffffffff]
> [ 0.000000] node 8: [mem 0x0000200000000000-0x0000200fffffffff]
> [ 0.000000] Initmem setup node 0 [mem 0x0000000000000000-0x0000000fffffffff]
> [ 0.000000] On node 0 totalpages: 1048576
> [ 0.000000] DMA zone: 1024 pages used for memmap
> [ 0.000000] DMA zone: 0 pages reserved
> [ 0.000000] DMA zone: 1048576 pages, LIFO batch:1
> [ 0.000000] Initmem setup node 8 [mem 0x0000200000000000-0x0000200fffffffff]
> [ 0.000000] On node 8 totalpages: 1048576
> [ 0.000000] DMA zone: 1024 pages used for memmap
> [ 0.000000] DMA zone: 0 pages reserved
> [ 0.000000] DMA zone: 1048576 pages, LIFO batch:1
> [ 0.000000] percpu: Embedded 4 pages/cpu @(____ptrval____) s166808 r0 d95336 u262144
> [ 0.000000] pcpu-alloc: s166808 r0 d95336 u262144 alloc=4*65536
> [ 0.000000] pcpu-alloc: [0] 000 [0] 001 [0] 002 [0] 003
> [ 0.000000] pcpu-alloc: [0] 004 [0] 005 [0] 006 [0] 007
> [ 0.000000] pcpu-alloc: [0] 008 [0] 009 [0] 010 [0] 011
> [ 0.000000] pcpu-alloc: [0] 012 [0] 013 [0] 014 [0] 015
> [ 0.000000] pcpu-alloc: [0] 016 [0] 017 [0] 018 [0] 019
> [ 0.000000] pcpu-alloc: [0] 020 [0] 021 [0] 022 [0] 023
> [ 0.000000] pcpu-alloc: [0] 024 [0] 025 [0] 026 [0] 027
> [ 0.000000] pcpu-alloc: [0] 028 [0] 029 [0] 030 [0] 031
> [ 0.000000] pcpu-alloc: [0] 032 [0] 033 [0] 034 [0] 035
> [ 0.000000] pcpu-alloc: [0] 036 [0] 037 [0] 038 [0] 039
> [ 0.000000] pcpu-alloc: [0] 040 [0] 041 [0] 042 [0] 043
> [ 0.000000] pcpu-alloc: [0] 044 [0] 045 [0] 046 [0] 047
> [ 0.000000] pcpu-alloc: [0] 048 [0] 049 [0] 050 [0] 051
> [ 0.000000] pcpu-alloc: [0] 052 [0] 053 [0] 054 [0] 055
> [ 0.000000] pcpu-alloc: [0] 056 [0] 057 [0] 058 [0] 059
> [ 0.000000] pcpu-alloc: [0] 060 [0] 061 [0] 062 [0] 063
> [ 0.000000] pcpu-alloc: [1] 064 [1] 065 [1] 066 [1] 067
> [ 0.000000] pcpu-alloc: [1] 068 [1] 069 [1] 070 [1] 071
> [ 0.000000] pcpu-alloc: [1] 072 [1] 073 [1] 074 [1] 075
> [ 0.000000] pcpu-alloc: [1] 076 [1] 077 [1] 078 [1] 079
> [ 0.000000] pcpu-alloc: [1] 080 [1] 081 [1] 082 [1] 083
> [ 0.000000] pcpu-alloc: [1] 084 [1] 085 [1] 086 [1] 087
> [ 0.000000] pcpu-alloc: [1] 088 [1] 089 [1] 090 [1] 091
> [ 0.000000] pcpu-alloc: [1] 092 [1] 093 [1] 094 [1] 095
> [ 0.000000] pcpu-alloc: [1] 096 [1] 097 [1] 098 [1] 099
> [ 0.000000] pcpu-alloc: [1] 100 [1] 101 [1] 102 [1] 103
> [ 0.000000] pcpu-alloc: [1] 104 [1] 105 [1] 106 [1] 107
> [ 0.000000] pcpu-alloc: [1] 108 [1] 109 [1] 110 [1] 111
> [ 0.000000] pcpu-alloc: [1] 112 [1] 113 [1] 114 [1] 115
> [ 0.000000] pcpu-alloc: [1] 116 [1] 117 [1] 118 [1] 119
> [ 0.000000] pcpu-alloc: [1] 120 [1] 121 [1] 122 [1] 123
> [ 0.000000] pcpu-alloc: [1] 124 [1] 125 [1] 126 [1] 127
> [ 0.000000] Built 2 zonelists, mobility grouping on. Total pages: 2095104
> [ 0.000000] Policy zone: DMA
> [ 0.000000] Kernel command line: rw,slub_debug=FZP root=UUID=e62c58bb-2824-4075-a31d-455f1bb62504
> [ 0.000000] log_buf_len individual max cpu contribution: 8192 bytes
> [ 0.000000] log_buf_len total cpu_extra contributions: 1040384 bytes
> [ 0.000000] log_buf_len min size: 262144 bytes
> [ 0.000000] log_buf_len: 2097152 bytes
> [ 0.000000] early log buf free: 249640(95%)
> [ 0.000000] Memory: 126937280K/134217728K available (10624K kernel code, 1600K rwdata, 2816K rodata, 2752K init, 1391K bss, 563008K reserved, 6717440K cma-reserved)
> [ 0.000000] SLUB: HWalign=128, Order=0-3, MinObjects=0, CPUs=128, Nodes=9
> [ 0.000000] rcu: Hierarchical RCU implementation.
> [ 0.000000] rcu: RCU restricting CPUs from NR_CPUS=2048 to nr_cpu_ids=128.
> [ 0.000000] Tasks RCU enabled.
> [ 0.000000] rcu: Adjusting geometry for rcu_fanout_leaf=16, nr_cpu_ids=128
> [ 0.000000] NR_IRQS: 512, nr_irqs: 512, preallocated irqs: 16
> [ 0.000000] xive: Interrupt handling initialized with native backend
> [ 0.000000] xive: Using priority 7 for all interrupts
> [ 0.000000] xive: Using 64kB queues
> [ 0.000000] time_init: decrementer frequency = 512.000000 MHz
> [ 0.000000] time_init: processor frequency = 2250.000000 MHz
> [ 0.000003] time_init: 56 bit decrementer (max: 7fffffffffffff)
> [ 0.004131] clocksource: timebase: mask: 0xffffffffffffffff max_cycles: 0x761537d007, max_idle_ns: 440795202126 ns
> [ 0.015127] clocksource: timebase mult[1f40000] shift[24] registered
> [ 0.020765] clockevent: decrementer mult[83126e98] shift[32] cpu[0]
> [ 0.020996] Console: colour dummy device 80x25
> [ 0.026183] console [hvc0] enabled
> [ 0.031888] bootconsole [udbg0] disabled
> [ 0.040355] mempolicy: Enabling automatic NUMA balancing. Configure with numa_balancing= or the kernel.numa_balancing sysctl
> [ 0.051161] pid_max: default: 131072 minimum: 1024
> [ 0.064324] Dentry cache hash table entries: 8388608 (order: 10, 67108864 bytes)
> [ 0.072553] Inode-cache hash table entries: 4194304 (order: 9, 33554432 bytes)
> [ 0.078288] Mount-cache hash table entries: 131072 (order: 4, 1048576 bytes)
> [ 0.085039] Mountpoint-cache hash table entries: 131072 (order: 4, 1048576 bytes)
> [ 0.094088] EEH: PowerNV platform initialized
> [ 0.098132] POWER9 performance monitor hardware support registered
> [ 0.103745] rcu: Hierarchical SRCU implementation.
> [ 0.109204] smp: Bringing up secondary CPUs ...
> [ 0.497962] smp: Brought up 2 nodes, 128 CPUs
> [ 0.500638] numa: Node 0 CPUs: 0-63
> [ 0.503403] numa: Node 8 CPUs: 64-127
> [ 0.507553] Using shared cache scheduler topology
> [ 0.527795] devtmpfs: initialized
> [ 0.579056] random: get_random_u32 called from bucket_table_alloc+0xa4/0x220 with crng_init=0
> [ 0.586622] kworker/u257:0 (654) used greatest stack depth: 12432 bytes left
> [ 0.586944] clocksource: jiffies: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 19112604462750000 ns
> [ 0.602577] futex hash table entries: 32768 (order: 6, 4194304 bytes)
> [ 0.614833] NET: Registered protocol family 16
> [ 0.617639] EEH: No capable adapters found
> [ 0.621858] audit: initializing netlink subsys (disabled)
> [ 0.627301] audit: type=2000 audit(1530175670.570:1): state=initialized audit_enabled=0 res=1
> [ 0.627458] kworker/u257:0 (663) used greatest stack depth: 12384 bytes left
> [ 0.642475] cpuidle: using governor menu
> [ 0.646772] pstore: Registered nvram as persistent store backend
> [ 0.696312] PCI: Probing PCI hardware
> [ 0.696410] PCI host bridge to bus 0000:00
> [ 0.696454] pci_bus 0000:00: root bus resource [mem 0x600c000000000-0x600c07ffeffff] (bus address [0x80000000-0xfffeffff])
> [ 0.696564] pci_bus 0000:00: root bus resource [mem 0x6000000000000-0x6003fbfffffff 64bit pref]
> [ 0.696648] pci_bus 0000:00: root bus resource [bus 00-ff]
> [ 0.696700] pci_bus 0000:00: busn_res: [bus 00-ff] end is updated to ff
> [ 0.696738] pci 0000:00:00.0: [1014:04c1] type 01 class 0x060400
> [ 0.696940] pci 0000:00:00.0: PME# supported from D0 D3hot D3cold
> [ 0.702791] pci 0000:01:00.0: [10b5:9733] type 01 class 0x060400
> [ 0.702863] pci 0000:01:00.0: reg 0x10: [mem 0x600c001800000-0x600c00183ffff]
> [ 0.703113] pci 0000:01:00.0: PME# supported from D0 D3hot D3cold
> [ 0.708739] pci 0000:00:00.0: PCI bridge to [bus 01-13]
> [ 0.708797] pci 0000:00:00.0: bridge window [io 0x0000-0x0fff]
> [ 0.709172] pci 0000:02:01.0: [10b5:9733] type 01 class 0x060400
> [ 0.709479] pci 0000:02:01.0: PME# supported from D0 D3hot D3cold
> [ 0.710138] pci 0000:02:04.0: [10b5:9733] type 01 class 0x060400
> [ 0.710447] pci 0000:02:04.0: PME# supported from D0 D3hot D3cold
> [ 0.710762] pci 0000:02:05.0: [10b5:9733] type 01 class 0x060400
> [ 0.711071] pci 0000:02:05.0: PME# supported from D0 D3hot D3cold
> [ 0.711386] pci 0000:02:06.0: [10b5:9733] type 01 class 0x060400
> [ 0.711697] pci 0000:02:06.0: PME# supported from D0 D3hot D3cold
> [ 0.712025] pci 0000:02:07.0: [10b5:9733] type 01 class 0x060400
> [ 0.712338] pci 0000:02:07.0: PME# supported from D0 D3hot D3cold
> [ 0.716738] pci 0000:01:00.0: PCI bridge to [bus 02-13]
> [ 0.722562] pci 0000:02:01.0: PCI bridge to [bus 03-07]
> [ 0.728220] pci 0000:02:04.0: PCI bridge to [bus 08-0c]
> [ 0.733861] pci 0000:02:05.0: PCI bridge to [bus 0d-11]
> [ 0.734078] pci 0000:12:00.0: [144d:a804] type 00 class 0x010802
> [ 0.734152] pci 0000:12:00.0: reg 0x10: [mem 0x600c000820000-0x600c000823fff 64bit]
> [ 0.734242] pci 0000:12:00.0: reg 0x30: [mem 0x00000000-0x0001ffff pref]
> [ 0.734273] pci 0000:12:00.0: BAR0 [mem size 0x00004000 64bit]: requesting alignment to 0x10000
> [ 0.740076] pci 0000:02:06.0: PCI bridge to [bus 12]
> [ 0.740299] pci 0000:13:00.0: [144d:a804] type 00 class 0x010802
> [ 0.740374] pci 0000:13:00.0: reg 0x10: [mem 0x600c000c20000-0x600c000c23fff 64bit]
> [ 0.740464] pci 0000:13:00.0: reg 0x30: [mem 0x00000000-0x0001ffff pref]
> [ 0.740493] pci 0000:13:00.0: BAR0 [mem size 0x00004000 64bit]: requesting alignment to 0x10000
> [ 0.746296] pci 0000:02:07.0: PCI bridge to [bus 13]
> [ 0.746463] pci_bus 0000:00: busn_res: [bus 00-ff] end is updated to 13
> [ 0.746525] PCI host bridge to bus 0001:00
> [ 0.746567] pci_bus 0001:00: root bus resource [mem 0x600c080000000-0x600c0fffeffff] (bus address [0x80000000-0xfffeffff])
> [ 0.753399] pci_bus 0001:00: root bus resource [mem 0x6004000000000-0x6007f7fffffff 64bit pref]
> [ 0.761727] pci_bus 0001:00: root bus resource [bus 00-ff]
> [ 0.767269] pci_bus 0001:00: busn_res: [bus 00-ff] end is updated to ff
> [ 0.767299] pci 0001:00:00.0: [1014:04c1] type 01 class 0x060400
> [ 0.767481] pci 0001:00:00.0: PME# supported from D0 D3hot D3cold
> [ 0.778662] pci 0001:00:00.0: PCI bridge to [bus 01-ff]
> [ 0.778720] pci 0001:00:00.0: bridge window [io 0x0000-0x0fff]
> [ 0.778752] pci_bus 0001:00: busn_res: [bus 00-ff] end is updated to ff
> [ 0.778806] PCI host bridge to bus 0002:00
> [ 0.778848] pci_bus 0002:00: root bus resource [mem 0x600c100000000-0x600c17ffeffff] (bus address [0x80000000-0xfffeffff])
> [ 0.788071] pci_bus 0002:00: root bus resource [mem 0x6008000000000-0x600bf7fffffff 64bit pref]
> [ 0.796394] pci_bus 0002:00: root bus resource [bus 00-ff]
> [ 0.801936] pci_bus 0002:00: busn_res: [bus 00-ff] end is updated to ff
> [ 0.801966] pci 0002:00:00.0: [1014:04c1] type 01 class 0x060400
> [ 0.802157] pci 0002:00:00.0: PME# supported from D0 D3hot D3cold
> [ 0.807857] pci 0002:01:00.0: [8086:1589] type 00 class 0x020000
> [ 0.807935] pci 0002:01:00.0: reg 0x10: [mem 0x6008008000000-0x60080087fffff 64bit pref]
> [ 0.807985] pci 0002:01:00.0: reg 0x1c: [mem 0x600800a000000-0x600800a007fff 64bit pref]
> [ 0.808023] pci 0002:01:00.0: reg 0x30: [mem 0x00000000-0x0007ffff pref]
> [ 0.808055] pci 0002:01:00.0: BAR3 [mem size 0x00008000 64bit pref]: requesting alignment to 0x10000
> [ 0.810427] pci 0002:01:00.0: PME# supported from D0 D3hot
> [ 0.810582] pci 0002:01:00.1: [8086:1589] type 00 class 0x020000
> [ 0.810658] pci 0002:01:00.1: reg 0x10: [mem 0x6008008800000-0x6008008ffffff 64bit pref]
> [ 0.810708] pci 0002:01:00.1: reg 0x1c: [mem 0x600800a010000-0x600800a017fff 64bit pref]
> [ 0.810746] pci 0002:01:00.1: reg 0x30: [mem 0x00000000-0x0007ffff pref]
> [ 0.810779] pci 0002:01:00.1: BAR3 [mem size 0x00008000 64bit pref]: requesting alignment to 0x10000
> [ 0.820113] pci 0002:01:00.1: PME# supported from D0 D3hot
> [ 0.820254] pci 0002:01:00.2: [8086:1589] type 00 class 0x020000
> [ 0.820330] pci 0002:01:00.2: reg 0x10: [mem 0x6008009000000-0x60080097fffff 64bit pref]
> [ 0.820380] pci 0002:01:00.2: reg 0x1c: [mem 0x600800a020000-0x600800a027fff 64bit pref]
> [ 0.820420] pci 0002:01:00.2: reg 0x30: [mem 0x00000000-0x0007ffff pref]
> [ 0.820452] pci 0002:01:00.2: BAR3 [mem size 0x00008000 64bit pref]: requesting alignment to 0x10000
> [ 0.829818] pci 0002:01:00.2: PME# supported from D0 D3hot
> [ 0.829967] pci 0002:01:00.3: [8086:1589] type 00 class 0x020000
> [ 0.830044] pci 0002:01:00.3: reg 0x10: [mem 0x6008009800000-0x6008009ffffff 64bit pref]
> [ 0.830095] pci 0002:01:00.3: reg 0x1c: [mem 0x600800a030000-0x600800a037fff 64bit pref]
> [ 0.830135] pci 0002:01:00.3: reg 0x30: [mem 0x00000000-0x0007ffff pref]
> [ 0.830168] pci 0002:01:00.3: BAR3 [mem size 0x00008000 64bit pref]: requesting alignment to 0x10000
> [ 0.838152] pci 0002:01:00.3: PME# supported from D0 D3hot
> [ 0.843758] pci 0002:00:00.0: PCI bridge to [bus 01]
> [ 0.843815] pci 0002:00:00.0: bridge window [io 0x0000-0x0fff]
> [ 0.843903] pci_bus 0002:00: busn_res: [bus 00-ff] end is updated to 01
> [ 0.843960] PCI host bridge to bus 0003:00
> [ 0.847698] pci_bus 0003:00: root bus resource [mem 0x600c180000000-0x600c1fffeffff] (bus address [0x80000000-0xfffeffff])
> [ 0.858793] pci_bus 0003:00: root bus resource [mem 0x6020000000000-0x6023fbfffffff 64bit pref]
> [ 0.867110] pci_bus 0003:00: root bus resource [bus 00-ff]
> [ 0.872660] pci_bus 0003:00: busn_res: [bus 00-ff] end is updated to ff
> [ 0.872689] pci 0003:00:00.0: [1014:04c1] type 01 class 0x060400
> [ 0.872880] pci 0003:00:00.0: PME# supported from D0 D3hot D3cold
> [ 0.878583] pci 0003:01:00.0: [9005:028d] type 00 class 0x010700
> [ 0.878645] pci 0003:01:00.0: reg 0x10: [mem 0x600c180000000-0x600c1800fffff 64bit]
> [ 0.878676] pci 0003:01:00.0: reg 0x18: [mem 0x600c180180000-0x600c1801803ff 64bit]
> [ 0.878696] pci 0003:01:00.0: reg 0x20: [io 0x0000-0x00ff]
> [ 0.878733] pci 0003:01:00.0: reg 0x30: [mem 0x00000000-0x0007ffff pref]
> [ 0.878764] pci 0003:01:00.0: BAR2 [mem size 0x00000400 64bit]: requesting alignment to 0x10000
> [ 0.881098] pci 0003:01:00.0: PME# supported from D0 D1 D3hot
> [ 0.886729] pci 0003:00:00.0: PCI bridge to [bus 01]
> [ 0.886787] pci 0003:00:00.0: bridge window [io 0x0000-0x0fff]
> [ 0.886855] pci_bus 0003:00: busn_res: [bus 00-ff] end is updated to 01
> [ 0.886908] PCI host bridge to bus 0004:00
> [ 0.889311] pci_bus 0004:00: root bus resource [mem 0x600c200000000-0x600c27ffeffff] (bus address [0x80000000-0xfffeffff])
> [ 0.900398] pci_bus 0004:00: root bus resource [mem 0x6024000000000-0x6027f7fffffff 64bit pref]
> [ 0.910089] pci_bus 0004:00: root bus resource [bus 00-ff]
> [ 0.915642] pci_bus 0004:00: busn_res: [bus 00-ff] end is updated to ff
> [ 0.915672] pci 0004:00:00.0: [1014:04c1] type 01 class 0x060400
> [ 0.915865] pci 0004:00:00.0: PME# supported from D0 D3hot D3cold
> [ 0.921568] pci 0004:01:00.0: [1a03:1150] type 01 class 0x060400
> [ 0.921830] pci 0004:01:00.0: supports D1 D2
> [ 0.921834] pci 0004:01:00.0: PME# supported from D0 D1 D2 D3hot D3cold
> [ 0.927451] pci 0004:00:00.0: PCI bridge to [bus 01-02]
> [ 0.927509] pci 0004:00:00.0: bridge window [io 0x0000-0x0fff]
> [ 0.927649] pci_bus 0004:02: extended config space not accessible
> [ 0.927776] pci 0004:02:00.0: [1a03:2000] type 00 class 0x030000
> [ 0.927817] pci 0004:02:00.0: reg 0x10: [mem 0x600c200000000-0x600c200ffffff]
> [ 0.927839] pci 0004:02:00.0: reg 0x14: [mem 0x600c201000000-0x600c20101ffff]
> [ 0.927861] pci 0004:02:00.0: reg 0x18: [io 0x0000-0x007f]
> [ 0.928030] pci 0004:02:00.0: supports D1 D2
> [ 0.928033] pci 0004:02:00.0: PME# supported from D0 D1 D2 D3hot D3cold
> [ 0.933609] pci 0004:01:00.0: PCI bridge to [bus 02]
> [ 0.933744] pci_bus 0004:00: busn_res: [bus 00-ff] end is updated to 02
> [ 0.933799] PCI host bridge to bus 0005:00
> [ 0.935061] pci_bus 0005:00: root bus resource [mem 0x600c280000000-0x600c2fffeffff] (bus address [0x80000000-0xfffeffff])
> [ 0.946162] pci_bus 0005:00: root bus resource [mem 0x6028000000000-0x602bf7fffffff 64bit pref]
> [ 0.954481] pci_bus 0005:00: root bus resource [bus 00-ff]
> [ 0.960027] pci_bus 0005:00: busn_res: [bus 00-ff] end is updated to ff
> [ 0.960056] pci 0005:00:00.0: [1014:04c1] type 01 class 0x060400
> [ 0.960248] pci 0005:00:00.0: PME# supported from D0 D3hot D3cold
> [ 0.965956] pci 0005:01:00.0: [104c:8241] type 00 class 0x0c0330
> [ 0.966021] pci 0005:01:00.0: reg 0x10: [mem 0x600c280000000-0x600c28000ffff 64bit]
> [ 0.966052] pci 0005:01:00.0: reg 0x18: [mem 0x600c280010000-0x600c280011fff 64bit]
> [ 0.966135] pci 0005:01:00.0: BAR2 [mem size 0x00002000 64bit]: requesting alignment to 0x10000
> [ 0.969850] pci 0005:01:00.0: supports D1 D2
> [ 0.969853] pci 0005:01:00.0: PME# supported from D0 D1 D2 D3hot
> [ 0.975472] pci 0005:00:00.0: PCI bridge to [bus 01]
> [ 0.975530] pci 0005:00:00.0: bridge window [io 0x0000-0x0fff]
> [ 0.975598] pci_bus 0005:00: busn_res: [bus 00-ff] end is updated to 01
> [ 0.975652] PCI host bridge to bus 0030:00
> [ 0.978049] pci_bus 0030:00: root bus resource [mem 0x620c000000000-0x620c07ffeffff] (bus address [0x80000000-0xfffeffff])
> [ 0.989146] pci_bus 0030:00: root bus resource [mem 0x6200000000000-0x6203fbfffffff 64bit pref]
> [ 0.997464] pci_bus 0030:00: root bus resource [bus 00-ff]
> [ 1.003014] pci_bus 0030:00: busn_res: [bus 00-ff] end is updated to ff
> [ 1.003048] pci 0030:00:00.0: [1014:04c1] type 01 class 0x060400
> [ 1.003280] pci 0030:00:00.0: PME# supported from D0 D3hot D3cold
> [ 1.007158] random: fast init done
> [ 1.008995] pci 0030:01:00.0: [1000:0097] type 00 class 0x010700
> [ 1.009065] pci 0030:01:00.0: reg 0x10: [io 0x0000-0x00ff]
> [ 1.009101] pci 0030:01:00.0: reg 0x14: [mem 0x620c000140000-0x620c00014ffff 64bit]
> [ 1.009137] pci 0030:01:00.0: reg 0x1c: [mem 0x620c000100000-0x620c00013ffff 64bit]
> [ 1.009181] pci 0030:01:00.0: reg 0x30: [mem 0x00000000-0x000fffff pref]
> [ 1.009203] pci 0030:01:00.0: enabling Extended Tags
> [ 1.011548] pci 0030:01:00.0: supports D1 D2
> [ 1.017179] pci 0030:00:00.0: PCI bridge to [bus 01]
> [ 1.017237] pci 0030:00:00.0: bridge window [io 0x0000-0x0fff]
> [ 1.017310] pci_bus 0030:00: busn_res: [bus 00-ff] end is updated to 01
> [ 1.017365] PCI host bridge to bus 0031:00
> [ 1.021030] pci_bus 0031:00: root bus resource [mem 0x620c080000000-0x620c0fffeffff] (bus address [0x80000000-0xfffeffff])
> [ 1.032133] pci_bus 0031:00: root bus resource [mem 0x6204000000000-0x6207f7fffffff 64bit pref]
> [ 1.040454] pci_bus 0031:00: root bus resource [bus 00-ff]
> [ 1.045996] pci_bus 0031:00: busn_res: [bus 00-ff] end is updated to ff
> [ 1.046030] pci 0031:00:00.0: [1014:04c1] type 01 class 0x060400
> [ 1.046263] pci 0031:00:00.0: PME# supported from D0 D3hot D3cold
> [ 1.051986] pci 0031:01:00.0: [14e4:16a1] type 00 class 0x020000
> [ 1.052085] pci 0031:01:00.0: reg 0x10: [mem 0x620400c000000-0x620400c7fffff 64bit pref]
> [ 1.052123] pci 0031:01:00.0: reg 0x18: [mem 0x620400c800000-0x620400cffffff 64bit pref]
> [ 1.052161] pci 0031:01:00.0: reg 0x20: [mem 0x6204010000000-0x620401000ffff 64bit pref]
> [ 1.052186] pci 0031:01:00.0: reg 0x30: [mem 0x00000000-0x0007ffff pref]
> [ 1.052436] pci 0031:01:00.0: PME# supported from D0 D3hot
> [ 1.052614] pci 0031:01:00.1: [14e4:16a1] type 00 class 0x020000
> [ 1.052710] pci 0031:01:00.1: reg 0x10: [mem 0x620400d000000-0x620400d7fffff 64bit pref]
> [ 1.052747] pci 0031:01:00.1: reg 0x18: [mem 0x620400d800000-0x620400dffffff 64bit pref]
> [ 1.052785] pci 0031:01:00.1: reg 0x20: [mem 0x6204010010000-0x620401001ffff 64bit pref]
> [ 1.052810] pci 0031:01:00.1: reg 0x30: [mem 0x00000000-0x0007ffff pref]
> [ 1.053055] pci 0031:01:00.1: PME# supported from D0 D3hot
> [ 1.053207] pci 0031:01:00.2: [14e4:16a1] type 00 class 0x020000
> [ 1.053303] pci 0031:01:00.2: reg 0x10: [mem 0x620400e000000-0x620400e7fffff 64bit pref]
> [ 1.053341] pci 0031:01:00.2: reg 0x18: [mem 0x620400e800000-0x620400effffff 64bit pref]
> [ 1.053379] pci 0031:01:00.2: reg 0x20: [mem 0x6204010020000-0x620401002ffff 64bit pref]
> [ 1.053405] pci 0031:01:00.2: reg 0x30: [mem 0x00000000-0x0007ffff pref]
> [ 1.053652] pci 0031:01:00.2: PME# supported from D0 D3hot
> [ 1.053804] pci 0031:01:00.3: [14e4:16a1] type 00 class 0x020000
> [ 1.053901] pci 0031:01:00.3: reg 0x10: [mem 0x620400f000000-0x620400f7fffff 64bit pref]
> [ 1.053940] pci 0031:01:00.3: reg 0x18: [mem 0x620400f800000-0x620400fffffff 64bit pref]
> [ 1.053978] pci 0031:01:00.3: reg 0x20: [mem 0x6204010030000-0x620401003ffff 64bit pref]
> [ 1.054003] pci 0031:01:00.3: reg 0x30: [mem 0x00000000-0x0007ffff pref]
> [ 1.054252] pci 0031:01:00.3: PME# supported from D0 D3hot
> [ 1.059857] pci 0031:00:00.0: PCI bridge to [bus 01]
> [ 1.059916] pci 0031:00:00.0: bridge window [io 0x0000-0x0fff]
> [ 1.060060] pci_bus 0031:00: busn_res: [bus 00-ff] end is updated to 01
> [ 1.060117] PCI host bridge to bus 0032:00
> [ 1.060158] pci_bus 0032:00: root bus resource [mem 0x620c100000000-0x620c17ffeffff] (bus address [0x80000000-0xfffeffff])
> [ 1.065411] pci_bus 0032:00: root bus resource [mem 0x6208000000000-0x620bf7fffffff 64bit pref]
> [ 1.075121] pci_bus 0032:00: root bus resource [bus 00-ff]
> [ 1.079282] pci_bus 0032:00: busn_res: [bus 00-ff] end is updated to ff
> [ 1.079316] pci 0032:00:00.0: [1014:04c1] type 01 class 0x060400
> [ 1.079535] pci 0032:00:00.0: PME# supported from D0 D3hot D3cold
> [ 1.090700] pci 0032:00:00.0: PCI bridge to [bus 01-ff]
> [ 1.090758] pci 0032:00:00.0: bridge window [io 0x0000-0x0fff]
> [ 1.090795] pci_bus 0032:00: busn_res: [bus 00-ff] end is updated to ff
> [ 1.090849] PCI host bridge to bus 0033:00
> [ 1.090891] pci_bus 0033:00: root bus resource [mem 0x620c180000000-0x620c1fffeffff] (bus address [0x80000000-0xfffeffff])
> [ 1.100083] pci_bus 0033:00: root bus resource [mem 0x6220000000000-0x6223fbfffffff 64bit pref]
> [ 1.108409] pci_bus 0033:00: root bus resource [bus 00-ff]
> [ 1.113951] pci_bus 0033:00: busn_res: [bus 00-ff] end is updated to ff
> [ 1.113985] pci 0033:00:00.0: [1014:04c1] type 01 class 0x060400
> [ 1.114219] pci 0033:00:00.0: PME# supported from D0 D3hot D3cold
> [ 1.119946] pci 0033:01:00.0: [8086:10fb] type 00 class 0x020000
> [ 1.120028] pci 0033:01:00.0: reg 0x10: [mem 0x6220008000000-0x622000807ffff 64bit pref]
> [ 1.120053] pci 0033:01:00.0: reg 0x18: [io 0x0000-0x001f]
> [ 1.120110] pci 0033:01:00.0: reg 0x20: [mem 0x6220008100000-0x6220008103fff 64bit pref]
> [ 1.120135] pci 0033:01:00.0: reg 0x30: [mem 0x00000000-0x0007ffff pref]
> [ 1.120168] pci 0033:01:00.0: BAR4 [mem size 0x00004000 64bit pref]: requesting alignment to 0x10000
> [ 1.123815] pci 0033:01:00.0: PME# supported from D0 D3hot D3cold
> [ 1.123995] pci 0033:01:00.1: [8086:10fb] type 00 class 0x020000
> [ 1.124077] pci 0033:01:00.1: reg 0x10: [mem 0x6220008080000-0x62200080fffff 64bit pref]
> [ 1.124101] pci 0033:01:00.1: reg 0x18: [io 0x0000-0x001f]
> [ 1.124160] pci 0033:01:00.1: reg 0x20: [mem 0x6220008110000-0x6220008113fff 64bit pref]
> [ 1.124185] pci 0033:01:00.1: reg 0x30: [mem 0x00000000-0x0007ffff pref]
> [ 1.124218] pci 0033:01:00.1: BAR4 [mem size 0x00004000 64bit pref]: requesting alignment to 0x10000
> [ 1.132150] pci 0033:01:00.1: PME# supported from D0 D3hot D3cold
> [ 1.137756] pci 0033:00:00.0: PCI bridge to [bus 01]
> [ 1.137814] pci 0033:00:00.0: bridge window [io 0x0000-0x0fff]
> [ 1.137920] pci_bus 0033:00: busn_res: [bus 00-ff] end is updated to 01
> [ 1.137976] PCI host bridge to bus 0034:00
> [ 1.141684] pci_bus 0034:00: root bus resource [mem 0x620c200000000-0x620c27ffeffff] (bus address [0x80000000-0xfffeffff])
> [ 1.152777] pci_bus 0034:00: root bus resource [mem 0x6224000000000-0x6227f7fffffff 64bit pref]
> [ 1.161104] pci_bus 0034:00: root bus resource [bus 00-ff]
> [ 1.166649] pci_bus 0034:00: busn_res: [bus 00-ff] end is updated to ff
> [ 1.166683] pci 0034:00:00.0: [1014:04c1] type 01 class 0x060400
> [ 1.166919] pci 0034:00:00.0: PME# supported from D0 D3hot D3cold
> [ 1.172639] pci 0034:01:00.0: [1000:005d] type 00 class 0x010400
> [ 1.172706] pci 0034:01:00.0: reg 0x10: [io 0x0000-0x00ff]
> [ 1.172742] pci 0034:01:00.0: reg 0x14: [mem 0x620c200200000-0x620c20020ffff 64bit]
> [ 1.172778] pci 0034:01:00.0: reg 0x1c: [mem 0x620c200000000-0x620c2000fffff 64bit]
> [ 1.172822] pci 0034:01:00.0: reg 0x30: [mem 0x00000000-0x000fffff pref]
> [ 1.173163] pci 0034:01:00.0: supports D1 D2
> [ 1.178815] pci 0034:00:00.0: PCI bridge to [bus 01]
> [ 1.178873] pci 0034:00:00.0: bridge window [io 0x0000-0x0fff]
> [ 1.178947] pci_bus 0034:00: busn_res: [bus 00-ff] end is updated to 01
> [ 1.178998] pci 0000:02:04.0: bridge window [io 0x1000-0x0fff] to [bus 08-0c] add_size 1000
> [ 1.179003] pci 0000:02:04.0: bridge window [mem 0x20000000-0x1fffffff 64bit pref] to [bus 08-0c] add_size 20000000 add_align 20000000
> [ 1.179008] pci 0000:02:04.0: bridge window [mem 0x00400000-0x003fffff] to [bus 08-0c] add_size 400000 add_align 400000
> [ 1.179027] pci 0000:02:05.0: bridge window [io 0x1000-0x0fff] to [bus 0d-11] add_size 1000
> [ 1.179032] pci 0000:02:05.0: bridge window [mem 0x20000000-0x1fffffff 64bit pref] to [bus 0d-11] add_size 20000000 add_align 20000000
> [ 1.179037] pci 0000:02:05.0: bridge window [mem 0x00400000-0x003fffff] to [bus 0d-11] add_size 400000 add_align 400000
> [ 1.179056] pci 0000:02:06.0: bridge window [io 0x1000-0x0fff] to [bus 12] add_size 1000
> [ 1.179061] pci 0000:02:06.0: bridge window [mem 0x20000000-0x1fffffff 64bit pref] to [bus 12] add_size 20000000 add_align 20000000
> [ 1.179081] pci 0000:02:07.0: bridge window [io 0x1000-0x0fff] to [bus 13] add_size 1000
> [ 1.179086] pci 0000:02:07.0: bridge window [mem 0x20000000-0x1fffffff 64bit pref] to [bus 13] add_size 20000000 add_align 20000000
> [ 1.179108] pci 0000:01:00.0: bridge window [io 0x1000-0x0fff] to [bus 02-13] add_size 4000
> [ 1.179113] pci 0000:01:00.0: bridge window [mem 0x20000000-0x9fffffff 64bit pref] to [bus 02-13] add_size 80000000 add_align 20000000
> [ 1.179119] pci 0000:01:00.0: bridge window [mem 0x00400000-0x013fffff] to [bus 02-13] add_size 800000 add_align 400000
> [ 1.179143] pci 0000:00:00.0: bridge window [mem 0x20000000-0x9fffffff 64bit pref] to [bus 01-13] add_size 80000000 add_align 20000000
> [ 1.179148] pci 0000:00:00.0: bridge window [mem 0x00400000-0x017fffff] to [bus 01-13] add_size 800000 add_align 400000
> [ 1.179157] pci 0000:00:00.0: BAR 9: assigned [mem 0x6000000000000-0x60000ffffffff 64bit pref]
> [ 1.180520] pci 0000:00:00.0: BAR 8: assigned [mem 0x600c000000000-0x600c001bfffff]
> [ 1.187468] pci 0000:01:00.0: BAR 9: assigned [mem 0x6000000000000-0x60000ffffffff 64bit pref]
> [ 1.195767] pci 0000:01:00.0: BAR 8: assigned [mem 0x600c000000000-0x600c0017fffff]
> [ 1.204088] pci 0000:01:00.0: BAR 0: assigned [mem 0x600c001800000-0x600c00183ffff]
> [ 1.211037] pci 0000:01:00.0: BAR 7: no space for [io size 0x4000]
> [ 1.217956] pci 0000:01:00.0: BAR 7: failed to assign [io size 0x4000]
> [ 1.224886] pci 0000:01:00.0: BAR 7: no space for [io size 0x4000]
> [ 1.230441] pci 0000:01:00.0: BAR 7: failed to assign [io size 0x4000]
> [ 1.237366] pci 0000:02:04.0: BAR 9: assigned [mem 0x6000000000000-0x600001fffffff 64bit pref]
> [ 1.245705] pci 0000:02:05.0: BAR 9: assigned [mem 0x6000020000000-0x600003fffffff 64bit pref]
> [ 1.254010] pci 0000:02:06.0: BAR 9: assigned [mem 0x6000040000000-0x600005fffffff 64bit pref]
> [ 1.263714] pci 0000:02:07.0: BAR 9: assigned [mem 0x6000060000000-0x600007fffffff 64bit pref]
> [ 1.272041] pci 0000:02:04.0: BAR 8: assigned [mem 0x600c000000000-0x600c0003fffff]
> [ 1.278978] pci 0000:02:05.0: BAR 8: assigned [mem 0x600c000400000-0x600c0007fffff]
> [ 1.287287] pci 0000:02:06.0: BAR 8: assigned [mem 0x600c000800000-0x600c000bfffff]
> [ 1.294236] pci 0000:02:07.0: BAR 8: assigned [mem 0x600c000c00000-0x600c000ffffff]
> [ 1.302545] pci 0000:02:04.0: BAR 7: no space for [io size 0x1000]
> [ 1.308098] pci 0000:02:04.0: BAR 7: failed to assign [io size 0x1000]
> [ 1.315024] pci 0000:02:05.0: BAR 7: no space for [io size 0x1000]
> [ 1.320586] pci 0000:02:05.0: BAR 7: failed to assign [io size 0x1000]
> [ 1.327501] pci 0000:02:06.0: BAR 7: no space for [io size 0x1000]
> [ 1.334437] pci 0000:02:06.0: BAR 7: failed to assign [io size 0x1000]
> [ 1.339995] pci 0000:02:07.0: BAR 7: no space for [io size 0x1000]
> [ 1.346922] pci 0000:02:07.0: BAR 7: failed to assign [io size 0x1000]
> [ 1.353851] pci 0000:02:07.0: BAR 7: no space for [io size 0x1000]
> [ 1.359414] pci 0000:02:07.0: BAR 7: failed to assign [io size 0x1000]
> [ 1.366326] pci 0000:02:06.0: BAR 7: no space for [io size 0x1000]
> [ 1.371888] pci 0000:02:06.0: BAR 7: failed to assign [io size 0x1000]
> [ 1.378814] pci 0000:02:05.0: BAR 7: no space for [io size 0x1000]
> [ 1.385753] pci 0000:02:05.0: BAR 7: failed to assign [io size 0x1000]
> [ 1.391301] pci 0000:02:04.0: BAR 7: no space for [io size 0x1000]
> [ 1.398232] pci 0000:02:04.0: BAR 7: failed to assign [io size 0x1000]
> [ 1.405163] pci 0000:00 : [PE# 1fe] Secondary bus 0 associated with PE#1fe
> [ 1.412384] pci 0000:02:01.0: PCI bridge to [bus 03-07]
> [ 1.417654] pci 0000:02:04.0: PCI bridge to [bus 08-0c]
> [ 1.421819] pci 0000:02:04.0: bridge window [mem 0x600c000000000-0x600c0003fffff]
> [ 1.430122] pci 0000:02:04.0: bridge window [mem 0x6000000000000-0x600001fffffff 64bit pref]
> [ 1.438460] pci 0000:02:05.0: PCI bridge to [bus 0d-11]
> [ 1.443999] pci 0000:02:05.0: bridge window [mem 0x600c000400000-0x600c0007fffff]
> [ 1.450937] pci 0000:02:05.0: bridge window [mem 0x6000020000000-0x600003fffffff 64bit pref]
> [ 1.460642] pci 0000:12:00.0: BAR 6: assigned [mem 0x600c000800000-0x600c00081ffff pref]
> [ 1.467573] pci 0000:12:00.0: BAR 0: assigned [mem 0x600c000820000-0x600c000823fff 64bit]
> [ 1.475913] pci 0000:12 : [PE# 1fd] Secondary bus 18 associated with PE#1fd
> [ 1.483113] pci 0000:12 : [PE# 1fd] Setting up 32-bit TCE table at 0..80000000
> [ 1.498061] IOMMU table initialized, virtual merging enabled
> [ 1.498125] pci 0000:12 : [PE# 1fd] Setting up window#0 0..7fffffff pg=1000
> [ 1.503635] pci 0000:12 : [PE# 1fd] Enabling 64-bit DMA bypass
> [ 1.510560] iommu: Adding device 0000:12:00.0 to group 0
> [ 1.514728] pci 0000:02:06.0: PCI bridge to [bus 12]
> [ 1.520266] pci 0000:02:06.0: bridge window [mem 0x600c000800000-0x600c000bfffff]
> [ 1.528579] pci 0000:02:06.0: bridge window [mem 0x6000040000000-0x600005fffffff 64bit pref]
> [ 1.536914] pci 0000:13:00.0: BAR 6: assigned [mem 0x600c000c00000-0x600c000c1ffff pref]
> [ 1.545225] pci 0000:13:00.0: BAR 0: assigned [mem 0x600c000c20000-0x600c000c23fff 64bit]
> [ 1.552179] pci 0000:13 : [PE# 1fc] Secondary bus 19 associated with PE#1fc
> [ 1.560762] pci 0000:13 : [PE# 1fc] Setting up 32-bit TCE table at 0..80000000
> [ 1.574326] pci 0000:13 : [PE# 1fc] Setting up window#0 0..7fffffff pg=1000
> [ 1.574404] pci 0000:13 : [PE# 1fc] Enabling 64-bit DMA bypass
> [ 1.581277] iommu: Adding device 0000:13:00.0 to group 1
> [ 1.586822] pci 0000:02:07.0: PCI bridge to [bus 13]
> [ 1.590998] pci 0000:02:07.0: bridge window [mem 0x600c000c00000-0x600c000ffffff]
> [ 1.599309] pci 0000:02:07.0: bridge window [mem 0x6000060000000-0x600007fffffff 64bit pref]
> [ 1.607648] pci 0000:02 : [PE# 1fb] Secondary bus 2 associated with PE#1fb
> [ 1.614855] pci 0000:01:00.0: PCI bridge to [bus 02-13]
> [ 1.620112] pci 0000:01:00.0: bridge window [mem 0x600c000000000-0x600c07fefffff]
> [ 1.627060] pci 0000:01:00.0: bridge window [mem 0x6000000000000-0x6003fbff0ffff 64bit pref]
> [ 1.635375] pci 0000:01 : [PE# 1fa] Secondary bus 1 associated with PE#1fa
> [ 1.643957] pci 0000:00:00.0: PCI bridge to [bus 01-13]
> [ 1.647856] pci 0000:00:00.0: bridge window [mem 0x600c000000000-0x600c07fefffff]
> [ 1.656165] pci 0000:00:00.0: bridge window [mem 0x6000000000000-0x6003fbff0ffff 64bit pref]
> [ 1.664492] pci_bus 0000:00: resource 4 [mem 0x600c000000000-0x600c07ffeffff]
> [ 1.664496] pci_bus 0000:00: resource 5 [mem 0x6000000000000-0x6003fbfffffff 64bit pref]
> [ 1.664500] pci_bus 0000:01: resource 1 [mem 0x600c000000000-0x600c07fefffff]
> [ 1.664504] pci_bus 0000:01: resource 2 [mem 0x6000000000000-0x6003fbff0ffff 64bit pref]
> [ 1.664507] pci_bus 0000:02: resource 1 [mem 0x600c000000000-0x600c07fefffff]
> [ 1.664511] pci_bus 0000:02: resource 2 [mem 0x6000000000000-0x6003fbff0ffff 64bit pref]
> [ 1.664514] pci_bus 0000:08: resource 1 [mem 0x600c000000000-0x600c0003fffff]
> [ 1.664518] pci_bus 0000:08: resource 2 [mem 0x6000000000000-0x600001fffffff 64bit pref]
> [ 1.664521] pci_bus 0000:0d: resource 1 [mem 0x600c000400000-0x600c0007fffff]
> [ 1.664525] pci_bus 0000:0d: resource 2 [mem 0x6000020000000-0x600003fffffff 64bit pref]
> [ 1.664528] pci_bus 0000:12: resource 1 [mem 0x600c000800000-0x600c000bfffff]
> [ 1.664532] pci_bus 0000:12: resource 2 [mem 0x6000040000000-0x600005fffffff 64bit pref]
> [ 1.664535] pci_bus 0000:13: resource 1 [mem 0x600c000c00000-0x600c000ffffff]
> [ 1.664538] pci_bus 0000:13: resource 2 [mem 0x6000060000000-0x600007fffffff 64bit pref]
> [ 1.664566] pci 0001:00 : [PE# fe] Secondary bus 0 associated with PE#fe
> [ 1.671703] pci 0001:00:00.0: PCI bridge to [bus 01-ff]
> [ 1.676978] pci_bus 0001:00: resource 4 [mem 0x600c080000000-0x600c0fffeffff]
> [ 1.676982] pci_bus 0001:00: resource 5 [mem 0x6004000000000-0x6007f7fffffff 64bit pref]
> [ 1.677011] pci 0002:00:00.0: BAR 9: assigned [mem 0x6008000000000-0x600803fffffff 64bit pref]
> [ 1.685293] pci 0002:00:00.0: BAR 8: assigned [mem 0x600c100000000-0x600c1007fffff]
> [ 1.693601] pci 0002:01:00.0: BAR 0: assigned [mem 0x6008000000000-0x60080007fffff 64bit pref]
> [ 1.701946] pci 0002:01:00.1: BAR 0: assigned [mem 0x6008000800000-0x6008000ffffff 64bit pref]
> [ 1.710270] pci 0002:01:00.2: BAR 0: assigned [mem 0x6008001000000-0x60080017fffff 64bit pref]
> [ 1.718590] pci 0002:01:00.3: BAR 0: assigned [mem 0x6008001800000-0x6008001ffffff 64bit pref]
> [ 1.726905] pci 0002:01:00.0: BAR 6: assigned [mem 0x600c100000000-0x600c10007ffff pref]
> [ 1.735202] pci 0002:01:00.1: BAR 6: assigned [mem 0x600c100080000-0x600c1000fffff pref]
> [ 1.743516] pci 0002:01:00.2: BAR 6: assigned [mem 0x600c100100000-0x600c10017ffff pref]
> [ 1.751842] pci 0002:01:00.3: BAR 6: assigned [mem 0x600c100180000-0x600c1001fffff pref]
> [ 1.760166] pci 0002:01:00.0: BAR 3: assigned [mem 0x6008002000000-0x6008002007fff 64bit pref]
> [ 1.768517] pci 0002:01:00.1: BAR 3: assigned [mem 0x6008002010000-0x6008002017fff 64bit pref]
> [ 1.776833] pci 0002:01:00.2: BAR 3: assigned [mem 0x6008002020000-0x6008002027fff 64bit pref]
> [ 1.785154] pci 0002:01:00.3: BAR 3: assigned [mem 0x6008002030000-0x6008002037fff 64bit pref]
> [ 1.793475] pci 0002:00 : [PE# fe] Secondary bus 0 associated with PE#fe
> [ 1.800684] pci 0002:01 : [PE# 00] Secondary bus 1 associated with PE#0
> [ 1.807601] pci 0002:01 : [PE# 00] Setting up 32-bit TCE table at 0..80000000
> [ 1.822536] pci 0002:01 : [PE# 00] Setting up window#0 0..7fffffff pg=1000
> [ 1.822621] pci 0002:01 : [PE# 00] Enabling 64-bit DMA bypass
> [ 1.828133] iommu: Adding device 0002:01:00.0 to group 2
> [ 1.833663] iommu: Adding device 0002:01:00.1 to group 2
> [ 1.839212] iommu: Adding device 0002:01:00.2 to group 2
> [ 1.844755] iommu: Adding device 0002:01:00.3 to group 2
> [ 1.850297] pci 0002:00:00.0: PCI bridge to [bus 01]
> [ 1.854483] pci 0002:00:00.0: bridge window [mem 0x600c100000000-0x600c17fefffff]
> [ 1.862784] pci 0002:00:00.0: bridge window [mem 0x6008000000000-0x600bf7ff0ffff 64bit pref]
> [ 1.871117] pci_bus 0002:00: resource 4 [mem 0x600c100000000-0x600c17ffeffff]
> [ 1.871120] pci_bus 0002:00: resource 5 [mem 0x6008000000000-0x600bf7fffffff 64bit pref]
> [ 1.871124] pci_bus 0002:01: resource 1 [mem 0x600c100000000-0x600c17fefffff]
> [ 1.871128] pci_bus 0002:01: resource 2 [mem 0x6008000000000-0x600bf7ff0ffff 64bit pref]
> [ 1.871157] pci 0003:00:00.0: BAR 8: assigned [mem 0x600c180000000-0x600c1803fffff]
> [ 1.878055] pci 0003:01:00.0: BAR 0: assigned [mem 0x600c180000000-0x600c1800fffff 64bit]
> [ 1.886371] pci 0003:01:00.0: BAR 6: assigned [mem 0x600c180100000-0x600c18017ffff pref]
> [ 1.894675] pci 0003:01:00.0: BAR 2: assigned [mem 0x600c180180000-0x600c1801803ff 64bit]
> [ 1.903015] pci 0003:01:00.0: BAR 4: no space for [io size 0x0100]
> [ 1.908557] pci 0003:01:00.0: BAR 4: failed to assign [io size 0x0100]
> [ 1.915473] pci 0003:00 : [PE# 1fe] Secondary bus 0 associated with PE#1fe
> [ 1.922710] pci 0003:01 : [PE# 1fd] Secondary bus 1 associated with PE#1fd
> [ 1.931025] pci 0003:01 : [PE# 1fd] Setting up 32-bit TCE table at 0..80000000
> [ 1.944583] pci 0003:01 : [PE# 1fd] Setting up window#0 0..7fffffff pg=1000
> [ 1.944660] pci 0003:01 : [PE# 1fd] Enabling 64-bit DMA bypass
> [ 1.951541] iommu: Adding device 0003:01:00.0 to group 3
> [ 1.955697] pci 0003:00:00.0: PCI bridge to [bus 01]
> [ 1.961249] pci 0003:00:00.0: bridge window [mem 0x600c180000000-0x600c1ffefffff]
> [ 1.969576] pci_bus 0003:00: Some PCI device resources are unassigned, try booting with pci=realloc
> [ 1.977889] pci_bus 0003:00: resource 4 [mem 0x600c180000000-0x600c1fffeffff]
> [ 1.977892] pci_bus 0003:00: resource 5 [mem 0x6020000000000-0x6023fbfffffff 64bit pref]
> [ 1.977896] pci_bus 0003:01: resource 1 [mem 0x600c180000000-0x600c1ffefffff]
> [ 1.977943] pci 0004:00:00.0: BAR 8: assigned [mem 0x600c200000000-0x600c2017fffff]
> [ 1.986206] pci 0004:01:00.0: BAR 8: assigned [mem 0x600c200000000-0x600c2017fffff]
> [ 1.993146] pci 0004:01:00.0: BAR 7: no space for [io size 0x1000]
> [ 2.000066] pci 0004:01:00.0: BAR 7: failed to assign [io size 0x1000]
> [ 2.005619] pci 0004:02:00.0: BAR 0: assigned [mem 0x600c200000000-0x600c200ffffff]
> [ 2.013939] pci 0004:02:00.0: BAR 1: assigned [mem 0x600c201000000-0x600c20101ffff]
> [ 2.020885] pci 0004:02:00.0: BAR 2: no space for [io size 0x0080]
> [ 2.027801] pci 0004:02:00.0: BAR 2: failed to assign [io size 0x0080]
> [ 2.034737] pci 0004:00 : [PE# fe] Secondary bus 0 associated with PE#fe
> [ 2.041967] pci 0004:02 : [PE# fd] Secondary bus 2..2 associated with PE#fd
> [ 2.048904] pci 0004:02 : [PE# fd] Setting up 32-bit TCE table at 0..80000000
> [ 2.062465] pci 0004:02 : [PE# fd] Setting up window#0 0..7fffffff pg=1000
> [ 2.062541] pci 0004:02 : [PE# fd] Enabling 64-bit DMA bypass
> [ 2.069415] iommu: Adding device 0004:02:00.0 to group 4
> [ 2.074954] pci 0004:01:00.0: PCI bridge to [bus 02]
> [ 2.079128] pci 0004:01:00.0: bridge window [mem 0x600c200000000-0x600c27fefffff]
> [ 2.087457] pci 0004:01 : [PE# fc] Secondary bus 1 associated with PE#fc
> [ 2.094647] pci 0004:00:00.0: PCI bridge to [bus 01-02]
> [ 2.098541] pci 0004:00:00.0: bridge window [mem 0x600c200000000-0x600c27fefffff]
> [ 2.106849] pci_bus 0004:00: Some PCI device resources are unassigned, try booting with pci=realloc
> [ 2.115179] pci_bus 0004:00: resource 4 [mem 0x600c200000000-0x600c27ffeffff]
> [ 2.115183] pci_bus 0004:00: resource 5 [mem 0x6024000000000-0x6027f7fffffff 64bit pref]
> [ 2.115186] pci_bus 0004:01: resource 1 [mem 0x600c200000000-0x600c27fefffff]
> [ 2.115190] pci_bus 0004:02: resource 1 [mem 0x600c200000000-0x600c27fefffff]
> [ 2.115219] pci 0005:00:00.0: BAR 8: assigned [mem 0x600c280000000-0x600c2807fffff]
> [ 2.123490] pci 0005:01:00.0: BAR 0: assigned [mem 0x600c280000000-0x600c28000ffff 64bit]
> [ 2.131828] pci 0005:01:00.0: BAR 2: assigned [mem 0x600c280010000-0x600c280011fff 64bit]
> [ 2.140145] pci 0005:00 : [PE# fe] Secondary bus 0 associated with PE#fe
> [ 2.147353] pci 0005:01 : [PE# fd] Secondary bus 1 associated with PE#fd
> [ 2.154290] pci 0005:01 : [PE# fd] Setting up 32-bit TCE table at 0..80000000
> [ 2.167852] pci 0005:01 : [PE# fd] Setting up window#0 0..7fffffff pg=1000
> [ 2.167928] pci 0005:01 : [PE# fd] Enabling 64-bit DMA bypass
> [ 2.174805] iommu: Adding device 0005:01:00.0 to group 5
> [ 2.180335] pci 0005:00:00.0: PCI bridge to [bus 01]
> [ 2.184521] pci 0005:00:00.0: bridge window [mem 0x600c280000000-0x600c2ffefffff]
> [ 2.192839] pci_bus 0005:00: resource 4 [mem 0x600c280000000-0x600c2fffeffff]
> [ 2.192843] pci_bus 0005:00: resource 5 [mem 0x6028000000000-0x602bf7fffffff 64bit pref]
> [ 2.192846] pci_bus 0005:01: resource 1 [mem 0x600c280000000-0x600c2ffefffff]
> [ 2.192879] pci 0030:00:00.0: BAR 8: assigned [mem 0x620c000000000-0x620c0003fffff]
> [ 2.199769] pci 0030:01:00.0: BAR 6: assigned [mem 0x620c000000000-0x620c0000fffff pref]
> [ 2.208086] pci 0030:01:00.0: BAR 3: assigned [mem 0x620c000100000-0x620c00013ffff 64bit]
> [ 2.216413] pci 0030:01:00.0: BAR 1: assigned [mem 0x620c000140000-0x620c00014ffff 64bit]
> [ 2.224741] pci 0030:01:00.0: BAR 0: no space for [io size 0x0100]
> [ 2.230273] pci 0030:01:00.0: BAR 0: failed to assign [io size 0x0100]
> [ 2.237199] pci 0030:00 : [PE# 1fe] Secondary bus 0 associated with PE#1fe
> [ 2.244425] pci 0030:01 : [PE# 1fd] Secondary bus 1 associated with PE#1fd
> [ 2.251375] pci 0030:01 : [PE# 1fd] Setting up 32-bit TCE table at 0..80000000
> [ 2.266304] pci 0030:01 : [PE# 1fd] Setting up window#0 0..7fffffff pg=1000
> [ 2.266438] pci 0030:01 : [PE# 1fd] Enabling 64-bit DMA bypass
> [ 2.271883] iommu: Adding device 0030:01:00.0 to group 6
> [ 2.277416] pci 0030:00:00.0: PCI bridge to [bus 01]
> [ 2.282977] pci 0030:00:00.0: bridge window [mem 0x620c000000000-0x620c07fefffff]
> [ 2.289917] pci_bus 0030:00: Some PCI device resources are unassigned, try booting with pci=realloc
> [ 2.299602] pci_bus 0030:00: resource 4 [mem 0x620c000000000-0x620c07ffeffff]
> [ 2.299606] pci_bus 0030:00: resource 5 [mem 0x6200000000000-0x6203fbfffffff 64bit pref]
> [ 2.299609] pci_bus 0030:01: resource 1 [mem 0x620c000000000-0x620c07fefffff]
> [ 2.299646] pci 0031:00:00.0: BAR 9: assigned [mem 0x6204000000000-0x620403fffffff 64bit pref]
> [ 2.307936] pci 0031:00:00.0: BAR 8: assigned [mem 0x620c080000000-0x620c0807fffff]
> [ 2.316253] pci 0031:01:00.0: BAR 0: assigned [mem 0x6204000000000-0x62040007fffff 64bit pref]
> [ 2.324597] pci 0031:01:00.0: BAR 2: assigned [mem 0x6204000800000-0x6204000ffffff 64bit pref]
> [ 2.332919] pci 0031:01:00.1: BAR 0: assigned [mem 0x6204001000000-0x62040017fffff 64bit pref]
> [ 2.341240] pci 0031:01:00.1: BAR 2: assigned [mem 0x6204001800000-0x6204001ffffff 64bit pref]
> [ 2.349563] pci 0031:01:00.2: BAR 0: assigned [mem 0x6204002000000-0x62040027fffff 64bit pref]
> [ 2.357877] pci 0031:01:00.2: BAR 2: assigned [mem 0x6204002800000-0x6204002ffffff 64bit pref]
> [ 2.367570] pci 0031:01:00.3: BAR 0: assigned [mem 0x6204003000000-0x62040037fffff 64bit pref]
> [ 2.375903] pci 0031:01:00.3: BAR 2: assigned [mem 0x6204003800000-0x6204003ffffff 64bit pref]
> [ 2.384232] pci 0031:01:00.0: BAR 6: assigned [mem 0x620c080000000-0x620c08007ffff pref]
> [ 2.392506] pci 0031:01:00.1: BAR 6: assigned [mem 0x620c080080000-0x620c0800fffff pref]
> [ 2.400831] pci 0031:01:00.2: BAR 6: assigned [mem 0x620c080100000-0x620c08017ffff pref]
> [ 2.407778] pci 0031:01:00.3: BAR 6: assigned [mem 0x620c080180000-0x620c0801fffff pref]
> [ 2.416097] pci 0031:01:00.0: BAR 4: assigned [mem 0x6204004000000-0x620400400ffff 64bit pref]
> [ 2.424432] pci 0031:01:00.1: BAR 4: assigned [mem 0x6204004010000-0x620400401ffff 64bit pref]
> [ 2.434139] pci 0031:01:00.2: BAR 4: assigned [mem 0x6204004020000-0x620400402ffff 64bit pref]
> [ 2.442464] pci 0031:01:00.3: BAR 4: assigned [mem 0x6204004030000-0x620400403ffff 64bit pref]
> [ 2.450787] pci 0031:00 : [PE# fe] Secondary bus 0 associated with PE#fe
> [ 2.458012] pci 0031:01 : [PE# 00] Secondary bus 1 associated with PE#0
> [ 2.464921] pci 0031:01 : [PE# 00] Setting up 32-bit TCE table at 0..80000000
> [ 2.478491] pci 0031:01 : [PE# 00] Setting up window#0 0..7fffffff pg=1000
> [ 2.479883] pci 0031:01 : [PE# 00] Enabling 64-bit DMA bypass
> [ 2.485447] iommu: Adding device 0031:01:00.0 to group 7
> [ 2.490967] iommu: Adding device 0031:01:00.1 to group 7
> [ 2.496528] iommu: Adding device 0031:01:00.2 to group 7
> [ 2.500694] iommu: Adding device 0031:01:00.3 to group 7
> [ 2.506223] pci 0031:00:00.0: PCI bridge to [bus 01]
> [ 2.511786] pci 0031:00:00.0: bridge window [mem 0x620c080000000-0x620c0ffefffff]
> [ 2.518720] pci 0031:00:00.0: bridge window [mem 0x6204000000000-0x6207f7ff0ffff 64bit pref]
> [ 2.528432] pci_bus 0031:00: resource 4 [mem 0x620c080000000-0x620c0fffeffff]
> [ 2.528435] pci_bus 0031:00: resource 5 [mem 0x6204000000000-0x6207f7fffffff 64bit pref]
> [ 2.528439] pci_bus 0031:01: resource 1 [mem 0x620c080000000-0x620c0ffefffff]
> [ 2.528443] pci_bus 0031:01: resource 2 [mem 0x6204000000000-0x6207f7ff0ffff 64bit pref]
> [ 2.528475] pci 0032:00 : [PE# fe] Secondary bus 0 associated with PE#fe
> [ 2.535633] pci 0032:00:00.0: PCI bridge to [bus 01-ff]
> [ 2.539535] pci_bus 0032:00: resource 4 [mem 0x620c100000000-0x620c17ffeffff]
> [ 2.539539] pci_bus 0032:00: resource 5 [mem 0x6208000000000-0x620bf7fffffff 64bit pref]
> [ 2.539574] pci 0033:00:00.0: BAR 9: assigned [mem 0x6220000000000-0x622001fffffff 64bit pref]
> [ 2.549214] pci 0033:00:00.0: BAR 8: assigned [mem 0x620c180000000-0x620c1803fffff]
> [ 2.556161] pci 0033:01:00.0: BAR 0: assigned [mem 0x6220000000000-0x622000007ffff 64bit pref]
> [ 2.564501] pci 0033:01:00.0: BAR 6: assigned [mem 0x620c180000000-0x620c18007ffff pref]
> [ 2.572794] pci 0033:01:00.1: BAR 0: assigned [mem 0x6220000080000-0x62200000fffff 64bit pref]
> [ 2.581142] pci 0033:01:00.1: BAR 6: assigned [mem 0x620c180080000-0x620c1800fffff pref]
> [ 2.589428] pci 0033:01:00.0: BAR 4: assigned [mem 0x6220000100000-0x6220000103fff 64bit pref]
> [ 2.597775] pci 0033:01:00.1: BAR 4: assigned [mem 0x6220000110000-0x6220000113fff 64bit pref]
> [ 2.606109] pci 0033:01:00.0: BAR 2: no space for [io size 0x0020]
> [ 2.612994] pci 0033:01:00.0: BAR 2: failed to assign [io size 0x0020]
> [ 2.619942] pci 0033:01:00.1: BAR 2: no space for [io size 0x0020]
> [ 2.625495] pci 0033:01:00.1: BAR 2: failed to assign [io size 0x0020]
> [ 2.632416] pci 0033:00 : [PE# 1fe] Secondary bus 0 associated with PE#1fe
> [ 2.639655] pci 0033:01 : [PE# 00] Secondary bus 1 associated with PE#0
> [ 2.646582] pci 0033:01 : [PE# 00] Setting up 32-bit TCE table at 0..80000000
> [ 2.660136] pci 0033:01 : [PE# 00] Setting up window#0 0..7fffffff pg=1000
> [ 2.661536] pci 0033:01 : [PE# 00] Enabling 64-bit DMA bypass
> [ 2.667101] iommu: Adding device 0033:01:00.0 to group 8
> [ 2.672638] iommu: Adding device 0033:01:00.1 to group 8
> [ 2.678179] pci 0033:00:00.0: PCI bridge to [bus 01]
> [ 2.682361] pci 0033:00:00.0: bridge window [mem 0x620c180000000-0x620c1ffefffff]
> [ 2.690669] pci 0033:00:00.0: bridge window [mem 0x6220000000000-0x6223fbff0ffff 64bit pref]
> [ 2.698995] pci_bus 0033:00: Some PCI device resources are unassigned, try booting with pci=realloc
> [ 2.707311] pci_bus 0033:00: resource 4 [mem 0x620c180000000-0x620c1fffeffff]
> [ 2.707314] pci_bus 0033:00: resource 5 [mem 0x6220000000000-0x6223fbfffffff 64bit pref]
> [ 2.707318] pci_bus 0033:01: resource 1 [mem 0x620c180000000-0x620c1ffefffff]
> [ 2.707321] pci_bus 0033:01: resource 2 [mem 0x6220000000000-0x6223fbff0ffff 64bit pref]
> [ 2.707354] pci 0034:00:00.0: BAR 8: assigned [mem 0x620c200000000-0x620c2007fffff]
> [ 2.715615] pci 0034:01:00.0: BAR 3: assigned [mem 0x620c200000000-0x620c2000fffff 64bit]
> [ 2.723969] pci 0034:01:00.0: BAR 6: assigned [mem 0x620c200100000-0x620c2001fffff pref]
> [ 2.732266] pci 0034:01:00.0: BAR 1: assigned [mem 0x620c200200000-0x620c20020ffff 64bit]
> [ 2.739229] pci 0034:01:00.0: BAR 0: no space for [io size 0x0100]
> [ 2.746131] pci 0034:01:00.0: BAR 0: failed to assign [io size 0x0100]
> [ 2.753063] pci 0034:00 : [PE# fe] Secondary bus 0 associated with PE#fe
> [ 2.760285] pci 0034:01 : [PE# fd] Secondary bus 1 associated with PE#fd
> [ 2.767230] pci 0034:01 : [PE# fd] Setting up 32-bit TCE table at 0..80000000
> [ 2.780798] pci 0034:01 : [PE# fd] Setting up window#0 0..7fffffff pg=1000
> [ 2.780876] pci 0034:01 : [PE# fd] Enabling 64-bit DMA bypass
> [ 2.787740] iommu: Adding device 0034:01:00.0 to group 9
> [ 2.793272] pci 0034:00:00.0: PCI bridge to [bus 01]
> [ 2.797460] pci 0034:00:00.0: bridge window [mem 0x620c200000000-0x620c27fefffff]
> [ 2.805780] pci_bus 0034:00: Some PCI device resources are unassigned, try booting with pci=realloc
> [ 2.814087] pci_bus 0034:00: resource 4 [mem 0x620c200000000-0x620c27ffeffff]
> [ 2.814090] pci_bus 0034:00: resource 5 [mem 0x6224000000000-0x6227f7fffffff 64bit pref]
> [ 2.814094] pci_bus 0034:01: resource 1 [mem 0x620c200000000-0x620c27fefffff]
> [ 2.816741] PCI: Probing PCI hardware done
> [ 2.819487] cpuidle-powernv: Default stop: psscr = 0x0000000000000330,mask=0x00000000003003ff
> [ 2.822400] cpuidle-powernv: Deepest stop: psscr = 0x0000000000300375,mask=0x00000000003003ff
> [ 2.830725] cpuidle-powernv: Requested Level (RL) value of first deep stop = 0x4
> [ 2.846774] HugeTLB registered 2.00 MiB page size, pre-allocated 0 pages
> [ 2.846835] HugeTLB registered 1.00 GiB page size, pre-allocated 0 pages
> [ 2.854067] pci 0004:02:00.0: vgaarb: VGA device added: decodes=io+mem,owns=none,locks=none
> [ 2.859842] pci 0004:02:00.0: vgaarb: bridge control possible
> [ 2.865399] pci 0004:02:00.0: vgaarb: setting as boot device (VGA legacy resources not available)
> [ 2.875089] vgaarb: loaded
> [ 2.878169] SCSI subsystem initialized
> [ 2.880663] libata version 3.00 loaded.
> [ 2.880744] usbcore: registered new interface driver usbfs
> [ 2.886202] usbcore: registered new interface driver hub
> [ 2.892209] usbcore: registered new device driver usb
> [ 2.897345] pps_core: LinuxPPS API ver. 1 registered
> [ 2.901451] pps_core: Software ver. 5.3.6 - Copyright 2005-2007 Rodolfo Giometti <giometti@linux.it>
> [ 2.911156] PTP clock support registered
> [ 2.916569] clocksource: Switched to clocksource timebase
> [ 2.941126] NET: Registered protocol family 2
> [ 2.941760] tcp_listen_portaddr_hash hash table entries: 65536 (order: 4, 1048576 bytes)
> [ 2.942163] TCP established hash table entries: 524288 (order: 6, 4194304 bytes)
> [ 2.943790] TCP bind hash table entries: 65536 (order: 4, 1048576 bytes)
> [ 2.947497] TCP: Hash tables configured (established 524288 bind 65536)
> [ 2.952962] UDP hash table entries: 65536 (order: 5, 2097152 bytes)
> [ 2.960350] UDP-Lite hash table entries: 65536 (order: 5, 2097152 bytes)
> [ 2.967614] NET: Registered protocol family 1
> [ 2.971339] RPC: Registered named UNIX socket transport module.
> [ 2.976328] RPC: Registered udp transport module.
> [ 2.981880] RPC: Registered tcp transport module.
> [ 2.986032] RPC: Registered tcp NFSv4.1 backchannel transport module.
> [ 2.993072] pci 0005:00:00.0: enabling device (0101 -> 0103)
> [ 2.998526] pci 0005:01:00.0: enabling device (0140 -> 0142)
> [ 3.020212] pci 0005:01:00.0: xHCI HW did not halt within 16000 usec status = 0x0
> [ 3.020290] pci 0005:01:00.0: quirk_usb_early_handoff+0x0/0xd90 took 26582 usecs
> [ 3.020417] pci 0034:01:00.0: [Firmware Bug]: disabling VPD access (can't determine size of non-standard VPD format)
> [ 3.029020] PCI: CLS 64 bytes, default 128
> [ 3.029090] Trying to unpack rootfs image as initramfs...
> [ 5.595479] Freeing initrd memory: 103488K
> [ 5.598429] nest_capp0_imc performance monitor hardware support registered
> [ 5.598555] nest_capp1_imc performance monitor hardware support registered
> [ 5.601683] core_imc performance monitor hardware support registered
> [ 5.601786] nest_mba0_imc performance monitor hardware support registered
> [ 5.601881] nest_mba1_imc performance monitor hardware support registered
> [ 5.601991] nest_mba2_imc performance monitor hardware support registered
> [ 5.602086] nest_mba3_imc performance monitor hardware support registered
> [ 5.602181] nest_mba4_imc performance monitor hardware support registered
> [ 5.602275] nest_mba5_imc performance monitor hardware support registered
> [ 5.602369] nest_mba6_imc performance monitor hardware support registered
> [ 5.602463] nest_mba7_imc performance monitor hardware support registered
> [ 5.602571] nest_mcs01_imc performance monitor hardware support registered
> [ 5.602678] nest_mcs23_imc performance monitor hardware support registered
> [ 5.602827] nest_nvlink0_imc performance monitor hardware support registered
> [ 5.602971] nest_nvlink1_imc performance monitor hardware support registered
> [ 5.603114] nest_nvlink2_imc performance monitor hardware support registered
> [ 5.603255] nest_nvlink3_imc performance monitor hardware support registered
> [ 5.603416] nest_nvlink4_imc performance monitor hardware support registered
> [ 5.603557] nest_nvlink5_imc performance monitor hardware support registered
> [ 5.603677] nest_nx_imc performance monitor hardware support registered
> [ 5.603774] nest_phb0_imc performance monitor hardware support registered
> [ 5.603872] nest_phb1_imc performance monitor hardware support registered
> [ 5.603968] nest_phb2_imc performance monitor hardware support registered
> [ 5.604066] nest_phb3_imc performance monitor hardware support registered
> [ 5.604162] nest_phb4_imc performance monitor hardware support registered
> [ 5.604259] nest_phb5_imc performance monitor hardware support registered
> [ 5.604399] nest_powerbus0_imc performance monitor hardware support registered
> [ 5.607620] thread_imc performance monitor hardware support registered
> [ 5.607735] nest_xlink0_imc performance monitor hardware support registered
> [ 5.607846] nest_xlink1_imc performance monitor hardware support registered
> [ 5.607956] nest_xlink2_imc performance monitor hardware support registered
> [ 5.621574] workingset: timestamp_bits=38 max_order=21 bucket_order=0
> [ 5.628670] NFS: Registering the id_resolver key type
> [ 5.628726] Key type id_resolver registered
> [ 5.628761] Key type id_legacy registered
> [ 5.628808] pstore: using deflate compression
> [ 5.629724] Block layer SCSI generic (bsg) driver version 0.4 loaded (major 250)
> [ 5.629795] io scheduler noop registered
> [ 5.629830] io scheduler deadline registered
> [ 5.630244] io scheduler cfq registered (default)
> [ 5.630291] io scheduler mq-deadline registered
> [ 5.630336] io scheduler kyber registered
> [ 5.632492] pci 0004:00:00.0: enabling device (0101 -> 0103)
> [ 5.632562] pci 0004:01:00.0: enabling device (0141 -> 0143)
> [ 5.632628] pci 0004:02:00.0: enabling device (0141 -> 0143)
> [ 5.632692] Using unsupported 1024x768 vga at 600c200010000, depth=32, pitch=4096
> [ 5.715647] Console: switching to colour frame buffer device 128x48
> [ 5.797006] fb0: Open Firmware frame buffer device on /pciex@600c3c0400000/pci@0/pci@0/vga@0
> [ 5.814705] hvc0: raw protocol on /ibm,opal/consoles/serial@0 (boot console)
> [ 5.814787] hvc0: No interrupts property, using OPAL event
> [ 5.815227] Serial: 8250/16550 driver, 4 ports, IRQ sharing disabled
> [ 5.819996] brd: module loaded
> [ 5.833910] loop: module loaded
> [ 5.833948] Uniform Multi-Platform E-IDE driver
> [ 5.834109] ide-gd driver 1.18
> [ 5.834156] ide-cd driver 5.00
> [ 5.834257] ipr: IBM Power RAID SCSI Device Driver version: 2.6.4 (March 14, 2017)
> [ 5.834704] libphy: Fixed MDIO Bus: probed
> [ 5.834804] ibmveth: IBM Power Virtual Ethernet Driver 1.06
> [ 5.834822] e100: Intel(R) PRO/100 Network Driver, 3.5.24-k2-NAPI
> [ 5.834882] e100: Copyright(c) 1999-2006 Intel Corporation
> [ 5.834976] e1000: Intel(R) PRO/1000 Network Driver - version 7.3.21-k8-NAPI
> [ 5.835045] e1000: Copyright (c) 1999-2006 Intel Corporation.
> [ 5.835132] e1000e: Intel(R) PRO/1000 Network Driver - 3.2.6-k
> [ 5.835189] e1000e: Copyright(c) 1999 - 2015 Intel Corporation.
> [ 5.835289] ehci_hcd: USB 2.0 'Enhanced' Host Controller (EHCI) Driver
> [ 5.835347] ehci-pci: EHCI PCI platform driver
> [ 5.835415] ohci_hcd: USB 1.1 'Open' Host Controller (OHCI) Driver
> [ 5.835477] ohci-pci: OHCI PCI platform driver
> [ 5.836146] mousedev: PS/2 mouse device common for all mice
> [ 5.856571] rtc-opal opal-rtc: rtc core: registered rtc-opal as rtc0
> [ 5.859287] device-mapper: uevent: version 1.0.3
> [ 5.859586] device-mapper: ioctl: 4.39.0-ioctl (2018-04-03) initialised: dm-devel@redhat.com
> [ 5.859676] powernv-cpufreq: cpufreq pstate min 0x62 nominal 0x5d max 0x0
> [ 5.859732] powernv-cpufreq: Workload Optimized Frequency is enabled in the platform
> [ 5.872773] powernv_idle_driver registered
> [ 5.880389] usbcore: registered new interface driver usbhid
> [ 5.880428] usbhid: USB HID core driver
> [ 5.880457] oprofile: using timer interrupt.
> [ 5.880744] ipip: IPv4 and MPLS over IPv4 tunneling driver
> [ 5.880960] NET: Registered protocol family 17
> [ 5.881023] Key type dns_resolver registered
> [ 5.881100] drmem: No dynamic reconfiguration memory found
> [ 5.881136] Running code patching self-tests ...
> [ 5.881487] Running feature fixup self-tests ...
> [ 5.881492] Running MSI bitmap self-tests ...
> [ 5.881702] registered taskstats version 1
> [ 5.882018] console [netcon0] enabled
> [ 5.882062] netconsole: network logging started
> [ 5.916590] rtc-opal opal-rtc: setting system clock to 2018-06-28 08:47:56 UTC (1530175676)
> [ 5.917441] Freeing unused kernel memory: 2752K
> [ 5.917478] This architecture does not have kernel memory protection.
> [ 5.921631] exe (2390) used greatest stack depth: 12368 bytes left
> [ 5.926988] exe (2396) used greatest stack depth: 12032 bytes left
> [ 6.097730] synth uevent: /devices/vio: failed to send uevent
> [ 6.097733] vio vio: uevent: failed to send synthetic uevent
> [ 6.103991] udevadm (2412) used greatest stack depth: 11120 bytes left
> [ 6.114419] nvme nvme0: pci function 0000:12:00.0
> [ 6.114499] pci 0000:00:00.0: enabling device (0101 -> 0103)
> [ 6.114567] pci 0000:01:00.0: enabling device (0141 -> 0143)
> [ 6.114588] nvme nvme1: pci function 0000:13:00.0
> [ 6.114636] pci 0000:02:06.0: enabling device (0141 -> 0143)
> [ 6.114691] pci 0000:02:07.0: enabling device (0141 -> 0143)
> [ 6.114730] nvme 0000:12:00.0: enabling device (0140 -> 0142)
> [ 6.114778] nvme 0000:13:00.0: enabling device (0140 -> 0142)
> [ 6.114827] nvme 0000:12:00.0: Using 64-bit DMA iommu bypass
> [ 6.114870] nvme 0000:13:00.0: Using 64-bit DMA iommu bypass
> [ 6.154159] mpt3sas version 25.100.00.00 loaded
> [ 6.155139] pci 0030:00:00.0: enabling device (0101 -> 0103)
> [ 6.155215] mpt3sas 0030:01:00.0: enabling device (0140 -> 0142)
> [ 6.155308] mpt3sas 0030:01:00.0: Using 64-bit DMA iommu bypass
> [ 6.155368] mpt3sas_cm0: 64 BIT PCI BUS DMA ADDRESSING SUPPORTED, total mem (133760960 kB)
> [ 6.183349] bnx2x: QLogic 5771x/578xx 10/20-Gigabit Ethernet Driver bnx2x 1.712.30-0 (2014/02/10)
> [ 6.183616] bnx2x 0031:01:00.0: msix capability found
> [ 6.183699] pci 0031:00:00.0: enabling device (0101 -> 0103)
> [ 6.183755] bnx2x 0031:01:00.0: enabling device (0140 -> 0142)
> [ 6.183895] bnx2x 0031:01:00.0: Using 64-bit DMA iommu bypass
> [ 6.184003] bnx2x 0031:01:00.0: part number 394D4342-34383735-30345430-473030
> [ 6.185339] i40e: Intel(R) Ethernet Connection XL710 Network Driver - version 2.3.2-k
> [ 6.185391] i40e: Copyright (c) 2013 - 2014 Intel Corporation.
> [ 6.185711] pci 0002:00:00.0: enabling device (0101 -> 0103)
> [ 6.185789] i40e 0002:01:00.0: enabling device (0140 -> 0142)
> [ 6.185845] i40e 0002:01:00.0: Using 64-bit DMA iommu bypass
> [ 6.196410] ixgbe: Intel(R) 10 Gigabit PCI Express Network Driver - version 5.1.0-k
> [ 6.196460] ixgbe: Copyright (c) 1999-2016 Intel Corporation.
> [ 6.196693] pci 0033:00:00.0: enabling device (0101 -> 0103)
> [ 6.196750] ixgbe 0033:01:00.0: enabling device (0140 -> 0142)
> [ 6.196804] ixgbe 0033:01:00.0: Using 64-bit DMA iommu bypass
> [ 6.199342] i40e 0002:01:00.0: fw 5.1.40981 api 1.5 nvm 5.05 0x800029e1 1.1313.0
> [ 6.230662] mpt3sas_cm0: CurrentHostPageSize is 0: Setting default host page size to 4k
> [ 6.230731] mpt3sas_cm0: MSI-X vectors supported: 96, no of cores: 128, max_msix_vectors: -1
> [ 6.232963] mpt3sas0-msix0: PCI-MSI-X enabled: IRQ 64
> [ 6.232998] mpt3sas0-msix1: PCI-MSI-X enabled: IRQ 65
> [ 6.233030] mpt3sas0-msix2: PCI-MSI-X enabled: IRQ 66
> [ 6.233063] mpt3sas0-msix3: PCI-MSI-X enabled: IRQ 67
> [ 6.233095] mpt3sas0-msix4: PCI-MSI-X enabled: IRQ 68
> [ 6.233128] mpt3sas0-msix5: PCI-MSI-X enabled: IRQ 69
> [ 6.233160] mpt3sas0-msix6: PCI-MSI-X enabled: IRQ 70
> [ 6.233192] mpt3sas0-msix7: PCI-MSI-X enabled: IRQ 71
> [ 6.233225] mpt3sas0-msix8: PCI-MSI-X enabled: IRQ 75
> [ 6.233257] mpt3sas0-msix9: PCI-MSI-X enabled: IRQ 76
> [ 6.233289] mpt3sas0-msix10: PCI-MSI-X enabled: IRQ 77
> [ 6.233321] mpt3sas0-msix11: PCI-MSI-X enabled: IRQ 78
> [ 6.233354] mpt3sas0-msix12: PCI-MSI-X enabled: IRQ 79
> [ 6.233386] mpt3sas0-msix13: PCI-MSI-X enabled: IRQ 81
> [ 6.233418] mpt3sas0-msix14: PCI-MSI-X enabled: IRQ 82
> [ 6.233450] mpt3sas0-msix15: PCI-MSI-X enabled: IRQ 83
> [ 6.233483] mpt3sas0-msix16: PCI-MSI-X enabled: IRQ 84
> [ 6.233515] mpt3sas0-msix17: PCI-MSI-X enabled: IRQ 85
> [ 6.233548] mpt3sas0-msix18: PCI-MSI-X enabled: IRQ 86
> [ 6.233580] mpt3sas0-msix19: PCI-MSI-X enabled: IRQ 87
> [ 6.233612] mpt3sas0-msix20: PCI-MSI-X enabled: IRQ 89
> [ 6.233644] mpt3sas0-msix21: PCI-MSI-X enabled: IRQ 90
> [ 6.233677] mpt3sas0-msix22: PCI-MSI-X enabled: IRQ 91
> [ 6.233709] mpt3sas0-msix23: PCI-MSI-X enabled: IRQ 92
> [ 6.233741] mpt3sas0-msix24: PCI-MSI-X enabled: IRQ 93
> [ 6.233773] mpt3sas0-msix25: PCI-MSI-X enabled: IRQ 94
> [ 6.233805] mpt3sas0-msix26: PCI-MSI-X enabled: IRQ 95
> [ 6.233838] mpt3sas0-msix27: PCI-MSI-X enabled: IRQ 96
> [ 6.233870] mpt3sas0-msix28: PCI-MSI-X enabled: IRQ 97
> [ 6.233902] mpt3sas0-msix29: PCI-MSI-X enabled: IRQ 98
> [ 6.233934] mpt3sas0-msix30: PCI-MSI-X enabled: IRQ 99
> [ 6.233967] mpt3sas0-msix31: PCI-MSI-X enabled: IRQ 100
> [ 6.233999] mpt3sas0-msix32: PCI-MSI-X enabled: IRQ 101
> [ 6.234032] mpt3sas0-msix33: PCI-MSI-X enabled: IRQ 102
> [ 6.234064] mpt3sas0-msix34: PCI-MSI-X enabled: IRQ 103
> [ 6.234097] mpt3sas0-msix35: PCI-MSI-X enabled: IRQ 104
> [ 6.234129] mpt3sas0-msix36: PCI-MSI-X enabled: IRQ 105
> [ 6.234161] mpt3sas0-msix37: PCI-MSI-X enabled: IRQ 106
> [ 6.234193] mpt3sas0-msix38: PCI-MSI-X enabled: IRQ 107
> [ 6.234226] mpt3sas0-msix39: PCI-MSI-X enabled: IRQ 108
> [ 6.234258] mpt3sas0-msix40: PCI-MSI-X enabled: IRQ 109
> [ 6.234309] mpt3sas0-msix41: PCI-MSI-X enabled: IRQ 110
> [ 6.234341] mpt3sas0-msix42: PCI-MSI-X enabled: IRQ 111
> [ 6.234374] mpt3sas0-msix43: PCI-MSI-X enabled: IRQ 112
> [ 6.234406] mpt3sas0-msix44: PCI-MSI-X enabled: IRQ 113
> [ 6.234439] mpt3sas0-msix45: PCI-MSI-X enabled: IRQ 114
> [ 6.234471] mpt3sas0-msix46: PCI-MSI-X enabled: IRQ 115
> [ 6.238459] mpt3sas0-msix47: PCI-MSI-X enabled: IRQ 116
> [ 6.244003] mpt3sas0-msix48: PCI-MSI-X enabled: IRQ 117
> [ 6.249554] mpt3sas0-msix49: PCI-MSI-X enabled: IRQ 118
> [ 6.253715] mpt3sas0-msix50: PCI-MSI-X enabled: IRQ 119
> [ 6.259260] mpt3sas0-msix51: PCI-MSI-X enabled: IRQ 120
> [ 6.264799] mpt3sas0-msix52: PCI-MSI-X enabled: IRQ 121
> [ 6.270353] mpt3sas0-msix53: PCI-MSI-X enabled: IRQ 122
> [ 6.274514] mpt3sas0-msix54: PCI-MSI-X enabled: IRQ 123
> [ 6.280057] mpt3sas0-msix55: PCI-MSI-X enabled: IRQ 124
> [ 6.285604] mpt3sas0-msix56: PCI-MSI-X enabled: IRQ 125
> [ 6.291154] mpt3sas0-msix57: PCI-MSI-X enabled: IRQ 126
> [ 6.295315] mpt3sas0-msix58: PCI-MSI-X enabled: IRQ 127
> [ 6.300857] mpt3sas0-msix59: PCI-MSI-X enabled: IRQ 128
> [ 6.306405] mpt3sas0-msix60: PCI-MSI-X enabled: IRQ 129
> [ 6.311956] mpt3sas0-msix61: PCI-MSI-X enabled: IRQ 130
> [ 6.316119] mpt3sas0-msix62: PCI-MSI-X enabled: IRQ 131
> [ 6.321659] mpt3sas0-msix63: PCI-MSI-X enabled: IRQ 132
> [ 6.327209] mpt3sas0-msix64: PCI-MSI-X enabled: IRQ 133
> [ 6.332752] mpt3sas0-msix65: PCI-MSI-X enabled: IRQ 134
> [ 6.336923] mpt3sas0-msix66: PCI-MSI-X enabled: IRQ 135
> [ 6.342460] mpt3sas0-msix67: PCI-MSI-X enabled: IRQ 136
> [ 6.348011] mpt3sas0-msix68: PCI-MSI-X enabled: IRQ 137
> [ 6.349732] nvme nvme0: Shutdown timeout set to 8 seconds
> [ 6.349757] nvme nvme1: Shutdown timeout set to 8 seconds
> [ 6.353553] mpt3sas0-msix69: PCI-MSI-X enabled: IRQ 138
> [ 6.358569] nvme1n1: p1 p2
> [ 6.363262] mpt3sas0-msix70: PCI-MSI-X enabled: IRQ 139
> [ 6.363265] mpt3sas0-msix71: PCI-MSI-X enabled: IRQ 140
> [ 6.382674] mpt3sas0-msix72: PCI-MSI-X enabled: IRQ 141
> [ 6.386847] mpt3sas0-msix73: PCI-MSI-X enabled: IRQ 142
> [ 6.392377] mpt3sas0-msix74: PCI-MSI-X enabled: IRQ 143
> [ 6.397930] mpt3sas0-msix75: PCI-MSI-X enabled: IRQ 144
> [ 6.403474] mpt3sas0-msix76: PCI-MSI-X enabled: IRQ 145
> [ 6.407648] mpt3sas0-msix77: PCI-MSI-X enabled: IRQ 146
> [ 6.413182] mpt3sas0-msix78: PCI-MSI-X enabled: IRQ 147
> [ 6.418735] mpt3sas0-msix79: PCI-MSI-X enabled: IRQ 148
> [ 6.424277] mpt3sas0-msix80: PCI-MSI-X enabled: IRQ 149
> [ 6.428446] mpt3sas0-msix81: PCI-MSI-X enabled: IRQ 150
> [ 6.433985] mpt3sas0-msix82: PCI-MSI-X enabled: IRQ 151
> [ 6.439533] mpt3sas0-msix83: PCI-MSI-X enabled: IRQ 152
> [ 6.443073] i40e 0002:01:00.0: MAC address: 0c:c4:7a:f2:d5:b6
> [ 6.445077] mpt3sas0-msix84: PCI-MSI-X enabled: IRQ 153
> [ 6.445080] mpt3sas0-msix85: PCI-MSI-X enabled: IRQ 154
> [ 6.458393] i40e 0002:01:00.0 eth0: NIC Link is Up, 1000 Mbps Full Duplex, Flow Control: None
> [ 6.460335] mpt3sas0-msix86: PCI-MSI-X enabled: IRQ 155
> [ 6.470320] i40e 0002:01:00.0: PCI-Express: Speed 8.0GT/s Width x8
> [ 6.474199] mpt3sas0-msix87: PCI-MSI-X enabled: IRQ 156
> [ 6.480273] i40e 0002:01:00.0: Features: PF-id[0] VSIs: 34 QP: 119 RSS FD_ATR FD_SB NTUPLE VxLAN Geneve PTP VEPA
> [ 6.485295] mpt3sas0-msix88: PCI-MSI-X enabled: IRQ 157
> [ 6.496450] i40e 0002:01:00.1: enabling device (0140 -> 0142)
> [ 6.500555] mpt3sas0-msix89: PCI-MSI-X enabled: IRQ 158
> [ 6.500557] mpt3sas0-msix90: PCI-MSI-X enabled: IRQ 159
> [ 6.506103] i40e 0002:01:00.1: Using 64-bit DMA iommu bypass
> [ 6.511640] mpt3sas0-msix91: PCI-MSI-X enabled: IRQ 160
> [ 6.511643] mpt3sas0-msix92: PCI-MSI-X enabled: IRQ 161
> [ 6.530569] i40e 0002:01:00.1: fw 5.1.40981 api 1.5 nvm 5.05 0x800029e1 1.1313.0
> [ 6.532447] mpt3sas0-msix93: PCI-MSI-X enabled: IRQ 162
> [ 6.532449] mpt3sas0-msix94: PCI-MSI-X enabled: IRQ 163
> [ 6.550480] mpt3sas0-msix95: PCI-MSI-X enabled: IRQ 164
> [ 6.556017] mpt3sas_cm0: iomem(0x000620c000140000), mapped(0x(____ptrval____)), size(65536)
> [ 6.564340] mpt3sas_cm0: ioport(0x0000000000000000), size(0)
> [ 6.571410] bnx2x 0031:01:00.0: 63.008 Gb/s available PCIe bandwidth (8 GT/s x8 link)
> [ 6.578387] bnx2x 0031:01:00.1: msix capability found
> [ 6.582388] bnx2x 0031:01:00.1: enabling device (0140 -> 0142)
> [ 6.589362] bnx2x 0031:01:00.1: Using 64-bit DMA iommu bypass
> [ 6.594890] bnx2x 0031:01:00.1: part number 394D4342-34383735-30345430-473030
> [ 6.654106] mpt3sas_cm0: CurrentHostPageSize is 0: Setting default host page size to 4k
> [ 6.667832] bnx2x 0031:01:00.1: 63.008 Gb/s available PCIe bandwidth (8 GT/s x8 link)
> [ 6.667997] bnx2x 0031:01:00.2: msix capability found
> [ 6.668059] bnx2x 0031:01:00.2: enabling device (0140 -> 0142)
> [ 6.668186] bnx2x 0031:01:00.2: Using 64-bit DMA iommu bypass
> [ 6.668284] bnx2x 0031:01:00.2: part number 394D4342-34383735-30345430-473030
> [ 6.686274] mpt3sas_cm0: Allocated physical memory: size(6324 kB)
> [ 6.686325] mpt3sas_cm0: Current Controller Queue Depth(2936),Max Controller Queue Depth(3072)
> [ 6.686408] mpt3sas_cm0: Scatter Gather Elements per IO(128)
> [ 6.714317] ixgbe 0033:01:00.0: Multiqueue Enabled: Rx Queue count = 63, Tx Queue count = 63 XDP Queue count = 0
> [ 6.714702] ixgbe 0033:01:00.0: 32.000 Gb/s available PCIe bandwidth, limited by 5 GT/s x8 link at 0033:00:00.0 (capable of 63.008 Gb/s with 8 GT/s x8 link)
> [ 6.714891] ixgbe 0033:01:00.0: MAC: 2, PHY: 15, SFP+: 7, PBA No: 0210FF-0FF
> [ 6.714949] ixgbe 0033:01:00.0: 0c:c4:7a:87:0d:6a
> [ 6.717983] ixgbe 0033:01:00.0: Intel(R) 10 Gigabit Network Connection
> [ 6.718193] ixgbe 0033:01:00.1: enabling device (0140 -> 0142)
> [ 6.718261] ixgbe 0033:01:00.1: Using 64-bit DMA iommu bypass
> [ 6.747341] bnx2x 0031:01:00.2: 63.008 Gb/s available PCIe bandwidth (8 GT/s x8 link)
> [ 6.747509] bnx2x 0031:01:00.3: msix capability found
> [ 6.747558] bnx2x 0031:01:00.3: enabling device (0140 -> 0142)
> [ 6.747661] bnx2x 0031:01:00.3: Using 64-bit DMA iommu bypass
> [ 6.747744] bnx2x 0031:01:00.3: part number 394D4342-34383735-30345430-473030
> [ 6.781582] i40e 0002:01:00.1: MAC address: 0c:c4:7a:f2:d5:b7
> [ 6.790039] i40e 0002:01:00.1: PCI-Express: Speed 8.0GT/s Width x8
> [ 6.790594] i40e 0002:01:00.1: Features: PF-id[1] VSIs: 34 QP: 119 RSS FD_ATR FD_SB NTUPLE VxLAN Geneve PTP VEPA
> [ 6.791297] i40e 0002:01:00.2: enabling device (0140 -> 0142)
> [ 6.791339] i40e 0002:01:00.2: Using 64-bit DMA iommu bypass
> [ 6.804672] i40e 0002:01:00.2: fw 5.1.40981 api 1.5 nvm 5.05 0x800029e1 1.1313.0
> [ 6.818439] bnx2x 0031:01:00.3: 63.008 Gb/s available PCIe bandwidth (8 GT/s x8 link)
> [ 6.819731] bnx2x 0031:01:00.2 enP49p1s0f2: renamed from eth4
> [ 6.883258] ixgbe 0033:01:00.1: Multiqueue Enabled: Rx Queue count = 63, Tx Queue count = 63 XDP Queue count = 0
> [ 6.883660] ixgbe 0033:01:00.1: 32.000 Gb/s available PCIe bandwidth, limited by 5 GT/s x8 link at 0033:00:00.0 (capable of 63.008 Gb/s with 8 GT/s x8 link)
> [ 6.883849] ixgbe 0033:01:00.1: MAC: 2, PHY: 15, SFP+: 8, PBA No: 0210FF-0FF
> [ 6.883906] ixgbe 0033:01:00.1: 0c:c4:7a:87:0d:6b
> [ 6.934407] mpt3sas_cm0: _base_display_fwpkg_version: complete
> [ 6.948061] mpt3sas_cm0: LSISAS3008: FWVersion(15.00.03.00), ChipRevision(0x02), BiosVersion(08.25.00.00)
> [ 6.948133] mpt3sas_cm0: Protocol=(
> [ 6.948135] Initiator
> [ 6.948165] ),
> [ 6.948186] Capabilities=(
> [ 6.948206] Raid
> [ 6.948227] ,TLR
> [ 6.948248] ,EEDP
> [ 6.948269] ,Snapshot Buffer
> [ 6.948290] ,Diag Trace Buffer
> [ 6.948320] ,Task Set Full
> [ 6.948350] ,NCQ
> [ 6.948370] )
> [ 6.949068] scsi host0: Fusion MPT SAS Host
> [ 6.949434] mpt3sas_cm0: sending port enable !!
> [ 7.045381] i40e 0002:01:00.2: MAC address: 0c:c4:7a:f2:d5:b8
> [ 7.076822] bnx2x 0031:01:00.1 enP49p1s0f1: renamed from eth2
> [ 7.160608] ixgbe 0033:01:00.1: Intel(R) 10 Gigabit Network Connection
> [ 7.160634] bnx2x 0031:01:00.3 enP49p1s0f3: renamed from eth6
> [ 7.162885] i40e 0002:01:00.2 eth4: NIC Link is Up, 1000 Mbps Full Duplex, Flow Control: None
> [ 7.164716] i40e 0002:01:00.2: PCI-Express: Speed 8.0GT/s Width x8
> [ 7.165282] i40e 0002:01:00.2: Features: PF-id[2] VSIs: 34 QP: 119 RSS FD_ATR FD_SB NTUPLE VxLAN Geneve PTP VEPA
> [ 7.165425] i40e 0002:01:00.3: enabling device (0140 -> 0142)
> [ 7.165480] i40e 0002:01:00.3: Using 64-bit DMA iommu bypass
> [ 7.179037] i40e 0002:01:00.3: fw 5.1.40981 api 1.5 nvm 5.05 0x800029e1 1.1313.0
> [ 7.421957] i40e 0002:01:00.3: MAC address: 0c:c4:7a:f2:d5:b9
> [ 7.539606] ixgbe 0033:01:00.1 enP51p1s0f1: renamed from eth2
> [ 7.543966] i40e 0002:01:00.3: PCI-Express: Speed 8.0GT/s Width x8
> [ 7.544533] i40e 0002:01:00.3: Features: PF-id[3] VSIs: 34 QP: 119 RSS FD_ATR FD_SB NTUPLE VxLAN Geneve PTP VEPA
> [ 7.676583] bnx2x 0031:01:00.0 enP49p1s0f0: renamed from eth1
> [ 7.746597] ixgbe 0033:01:00.0 enP51p1s0f0: renamed from eth3
> [ 7.836758] i40e 0002:01:00.3 enP2p1s0f3: renamed from eth6
> [ 7.907249] i40e 0002:01:00.1 enP2p1s0f1: renamed from eth5
> [ 7.987051] i40e 0002:01:00.2 enP2p1s0f2: renamed from eth4
> [ 8.056615] i40e 0002:01:00.0 enP2p1s0f0: renamed from eth0
> [ 8.813855] mpt3sas_cm0: host_add: handle(0x0001), sas_addr(0x500304801bdb3a01), phys(8)
> [ 9.312205] mpt3sas_cm0: expander_add: handle(0x0009), parent(0x0001), sas_addr(0x500304801f1006bf), phys(31)
> [ 14.047021] mpt3sas_cm0: port enable: SUCCESS
> [ 14.049371] scsi 0:1:0:0: Direct-Access LSI Logical Volume 3000 PQ: 0 ANSI: 6
> [ 14.054263] scsi 0:1:0:0: RAID1E: handle(0x0143), wwid(0x0340f942695ee6fc), pd_count(3), type(SSP)
> [ 14.054353] scsi 0:1:0:0: Set queue's max_sector to: 8192
> [ 14.058431] sd 0:1:0:0: Attached scsi generic sg0 type 0
> [ 14.063374] sd 0:1:0:0: [sda] 20480000 512-byte logical blocks: (10.5 GB/9.77 GiB)
> [ 14.063439] sd 0:1:0:0: [sda] 4096-byte physical blocks
> [ 14.064573] sd 0:1:0:0: [sda] Write Protect is off
> [ 14.064633] sd 0:1:0:0: [sda] Mode Sense: 67 00 00 08
> [ 14.066804] sd 0:1:0:0: [sda] Write cache: disabled, read cache: enabled, doesn't support DPO or FUA
> [ 14.094960] sda: sda1
> [ 14.095053] scsi 0:0:0:0: Direct-Access SEAGATE ST2000NM0045 N003 PQ: 0 ANSI: 6
> [ 14.096360] scsi 0:0:0:0: SSP: handle(0x000a), sas_addr(0x5000c50086a57ee5), phy(0), device_name(0x5000c50086a57ee4)
> [ 14.096458] scsi 0:0:0:0: enclosure logical id (0x500304801f1006bf), slot(0)
> [ 14.096521] scsi 0:0:0:0: enclosure level(0x0000), connector name( )
> [ 14.097284] sd 0:1:0:0: [sda] Attached SCSI disk
> [ 14.101326] scsi 0:0:0:0: Attached scsi generic sg1 type 0
> [ 14.114064] scsi 0:0:1:0: Direct-Access SEAGATE ST2000NM0135 E002 PQ: 0 ANSI: 6
> [ 14.114768] scsi 0:0:1:0: SSP: handle(0x000b), sas_addr(0x5000c50093a254fd), phy(1), device_name(0x5000c50093a254fc)
> [ 14.114863] scsi 0:0:1:0: enclosure logical id (0x500304801f1006bf), slot(1)
> [ 14.114925] scsi 0:0:1:0: enclosure level(0x0000), connector name( )
> [ 14.120976] scsi 0:0:1:0: Attached scsi generic sg2 type 0
> [ 14.134248] scsi 0:0:2:0: Direct-Access SEAGATE ST2000NM0135 E002 PQ: 0 ANSI: 6
> [ 14.134952] scsi 0:0:2:0: SSP: handle(0x000c), sas_addr(0x5000c50085d5ceb5), phy(2), device_name(0x5000c50085d5ceb4)
> [ 14.135047] scsi 0:0:2:0: enclosure logical id (0x500304801f1006bf), slot(2)
> [ 14.135106] scsi 0:0:2:0: enclosure level(0x0000), connector name( )
> [ 14.141689] scsi 0:0:2:0: Attached scsi generic sg3 type 0
> [ 14.148503] scsi 0:0:3:0: Direct-Access ATA ST2000NM0125-1YZ SN02 PQ: 0 ANSI: 6
> [ 14.148569] scsi 0:0:3:0: SATA: handle(0x000d), sas_addr(0x500304801f100683), phy(3), device_name(0x5000c50092b8dbb7)
> [ 14.148648] scsi 0:0:3:0: enclosure logical id (0x500304801f1006bf), slot(3)
> [ 14.148707] scsi 0:0:3:0: enclosure level(0x0000), connector name( )
> [ 14.149309] scsi 0:0:3:0: atapi(n), ncq(y), asyn_notify(n), smart(y), fua(y), sw_preserve(y)
> [ 14.155164] sd 0:0:3:0: Attached scsi generic sg4 type 0
> [ 14.164875] sd 0:0:3:0: [sdb] 3907029168 512-byte logical blocks: (2.00 TB/1.82 TiB)
> [ 14.164953] sd 0:0:3:0: [sdb] 4096-byte physical blocks
> [ 14.168178] scsi 0:0:4:0: Direct-Access SEAGATE ST4000NM0075 KT02 PQ: 0 ANSI: 6
> [ 14.168258] scsi 0:0:4:0: SSP: handle(0x000e), sas_addr(0x5000c500863b9e25), phy(8), device_name(0x5000c500863b9e24)
> [ 14.168335] scsi 0:0:4:0: enclosure logical id (0x500304801f1006bf), slot(8)
> [ 14.168393] scsi 0:0:4:0: enclosure level(0x0000), connector name( )
> [ 14.174737] sd 0:0:4:0: Attached scsi generic sg5 type 0
> [ 14.181347] scsi 0:0:5:0: Enclosure LSI SAS3x28 0601 PQ: 0 ANSI: 5
> [ 14.181428] scsi 0:0:5:0: set ignore_delay_remove for handle(0x000f)
> [ 14.181481] scsi 0:0:5:0: SES: handle(0x000f), sas_addr(0x500304801f1006bd), phy(28), device_name(0x500304801f1006bd)
> [ 14.181558] scsi 0:0:5:0: enclosure logical id (0x500304801f1006bf), slot(12)
> [ 14.181618] scsi 0:0:5:0: enclosure level(0x0000), connector name( )
> [ 14.183325] sd 0:0:4:0: [sdc] 976754646 4096-byte logical blocks: (4.00 TB/3.64 TiB)
> [ 14.184665] scsi 0:0:5:0: Attached scsi generic sg6 type 13
> [ 14.185122] sd 0:0:4:0: [sdc] Write Protect is off
> [ 14.185166] sd 0:0:4:0: [sdc] Mode Sense: db 00 10 08
> [ 14.188399] sd 0:0:4:0: [sdc] Write cache: disabled, read cache: enabled, supports DPO and FUA
> [ 14.213574] sd 0:0:4:0: [sdc] Attached SCSI disk
> [ 14.413688] sd 0:0:3:0: [sdb] Write Protect is off
> [ 14.413734] sd 0:0:3:0: [sdb] Mode Sense: 9b 00 10 08
> [ 14.417045] sd 0:0:3:0: [sdb] Write cache: enabled, read cache: enabled, supports DPO and FUA
> [ 14.454663] random: crng init done
> [ 14.454813] sdb: sdb1 sdb2 sdb3
> [ 14.485918] sd 0:0:3:0: [sdb] Attached SCSI disk
> [ 15.494282] setfont (2758) used greatest stack depth: 10864 bytes left
> [ 15.796480] raid6: altivecx1 gen() 7680 MB/s
> [ 15.966480] raid6: altivecx2 gen() 12356 MB/s
> [ 16.136483] raid6: altivecx4 gen() 15549 MB/s
> [ 16.306484] raid6: altivecx8 gen() 13776 MB/s
> [ 16.476484] raid6: vpermxor1 gen() 7682 MB/s
> [ 16.646483] raid6: vpermxor2 gen() 12353 MB/s
> [ 16.816479] raid6: vpermxor4 gen() 15535 MB/s
> [ 16.986479] raid6: vpermxor8 gen() 17241 MB/s
> [ 17.156480] raid6: int64x1 gen() 5098 MB/s
> [ 17.326497] raid6: int64x1 xor() 1581 MB/s
> [ 17.496485] raid6: int64x2 gen() 8888 MB/s
> [ 17.666485] raid6: int64x2 xor() 3232 MB/s
> [ 17.836479] raid6: int64x4 gen() 10741 MB/s
> [ 18.006483] raid6: int64x4 xor() 4083 MB/s
> [ 18.176488] raid6: int64x8 gen() 6282 MB/s
> [ 18.346481] raid6: int64x8 xor() 2571 MB/s
> [ 18.346526] raid6: using algorithm vpermxor8 gen() 17241 MB/s
> [ 18.346563] raid6: using intx1 recovery algorithm
> [ 18.348327] xor: measuring software checksum speed
> [ 18.446478] 8regs : 17683.200 MB/sec
> [ 18.546477] 8regs_prefetch: 15008.000 MB/sec
> [ 18.646481] 32regs : 18534.400 MB/sec
> [ 18.746477] 32regs_prefetch: 16371.200 MB/sec
> [ 18.846482] altivec : 21139.200 MB/sec
> [ 18.846513] xor: using function: altivec (21139.200 MB/sec)
> [ 19.350206] Btrfs loaded, crc32c=crc32c-generic
> [ 19.424799] BTRFS: device fsid 4eba18bc-a2a1-4032-8b44-521c87720061 devid 1 transid 14 /dev/sda1
> [ 19.425013] btrfs (2849) used greatest stack depth: 9952 bytes left
> [ 19.491388] EXT4-fs (nvme1n1p2): mounted filesystem with ordered data mode. Opts: (null)
> [ 19.632803] systemd[1]: Configuration file /etc/systemd/system.conf.d/htxsystemdcustom.conf is marked executable. Please remove executable permission bits. Proceeding anyway.
> [ 19.633556] systemd[1]: systemd 237 running in system mode. (+PAM +AUDIT +SELINUX +IMA +APPARMOR +SMACK +SYSVINIT +UTMP +LIBCRYPTSETUP +GCRYPT +GNUTLS +ACL +XZ +LZ4 +SECCOMP +BLKID +ELFUTILS +KMOD -IDN2 +IDN -PCRE2 default-hierarchy=hybrid)
> [ 19.644997] systemd[1]: Detected architecture ppc64-le.
> [ 19.646311] systemd[1]: Set hostname to <ltc-boston21>.
> [ 19.679595] systemd[1]: File /lib/systemd/system/systemd-journald.service:36 configures an IP firewall (IPAddressDeny=any), but the local system does not support BPF/cgroup based firewalling.
> [ 19.679705] systemd[1]: Proceeding WITHOUT firewalling in effect! (This warning is only shown for the first loaded unit using IP firewalling.)
> [ 19.725501] systemd[1]: Created slice System Slice.
> [ 19.725830] systemd[1]: Listening on udev Kernel Socket.
> [ 19.726085] systemd[1]: Listening on Journal Socket (/dev/log).
> [ 19.726364] systemd[1]: Listening on Device-mapper event daemon FIFOs.
> [ 19.726902] systemd[1]: Created slice User and Session Slice.
> [ 19.734241] ufw-init (2924) used greatest stack depth: 9792 bytes left
> [ 19.751794] Loading iSCSI transport class v2.0-870.
> [ 19.755053] EXT4-fs (nvme1n1p2): re-mounted. Opts: errors=remount-ro
> [ 19.927200] Adding 2097088k swap on /swapfile. Priority:-2 extents:6 across:2260928k SS
> [ 19.929834] Installing knfsd (copyright (C) 1996 okir@monad.swb.de).
> [ 19.987644] iscsi: registered transport (iser)
> [ 20.016987] systemd-journald[2933]: Received request to flush runtime journal from PID 1
> [ 20.018749] systemd-journald[2933]: File /var/log/journal/03421d269c0e44b28ec9bd3f7152d10b/system.journal corrupted or uncleanly shut down, renaming and replacing.
> [ 20.021582] synth uevent: /devices/vio: failed to send uevent
> [ 20.021586] vio vio: uevent: failed to send synthetic uevent
> [ 20.107304] crypto_register_alg 'aes' = 0
> [ 20.107558] crypto_register_alg 'cbc(aes)' = 0
> [ 20.107833] crypto_register_alg 'ctr(aes)' = 0
> [ 20.108078] crypto_register_alg 'xts(aes)' = 0
> [ 20.525406] EXT4-fs (sda1): VFS: Can't find ext4 filesystem
> [ 20.594897] Unable to handle kernel paging request for data at address 0x000000d0
> [ 20.594976] Faulting instruction address: 0xc000000000304fe0
> [ 20.595031] Oops: Kernel access of bad area, sig: 11 [#1]
> [ 20.595073] LE SMP NR_CPUS=2048 NUMA PowerNV
> [ 20.595119] Modules linked in: dm_snapshot dm_bufio bridge vmx_crypto stp llc ib_iser rdma_cm iw_cm ib_cm nfsd ib_core libiscsi scsi_transport_iscsi ip_tables x_tables autofs4 btrfs zstd_decompress zstd_compress xxhash lzo_compress raid10 raid456 async_raid6_recov async_memcpy async_pq async_xor async_tx xor raid6_pq multipath ixgbe i40e bnx2x mpt3sas nvme raid_class nvme_core scsi_transport_sas mdio libcrc32c
> [ 20.595452] CPU: 92 PID: 5368 Comm: lvm Not tainted 4.17.0-rc7-next-20180601-autotest #3
> [ 20.595514] NIP: c000000000304fe0 LR: c0000000002641b4 CTR: c000000000264190
> [ 20.595577] REGS: c000200e49587370 TRAP: 0300 Not tainted (4.17.0-rc7-next-20180601-autotest)
> [ 20.595648] MSR: 900000000280b033 <SF,HV,VEC,VSX,EE,FP,ME,IR,DR,RI,LE> CR: 24022424 XER: 20040000
> [ 20.595734] CFAR: c000000000304df4 DAR: 00000000000000d0 DSISR: 40000000 SOFTE: 0
> GPR00: c0000000002642b0 c000200e495875f0 c000000001109100 c000000ff901b000
> GPR04: 0000000000000000 c000200e4f960300 0000000000000000 0000200e60560000
> GPR08: 0000000000000000 0000000000000000 c00a000000000008 c0080000083f3950
> GPR12: c000000000264190 c000200fff6c8b00 00007fff96bd2910 00007fff96baf658
> GPR16: 00000001289a9960 00000001289774d0 00007fff96bafb88 00007fff96ba5f20
> GPR20: c000200e525c08c0 c000200e32350000 c000200e325c1000 0000000000000001
> GPR24: 0000000000000001 fffffffffffff000 c000200e43451800 c000200e52710820
> GPR28: c0000000002641b4 0000000000000000 5deadbeef0000200 c000000ff901b000
> [ 20.597990] NIP [c000000000304fe0] kmem_cache_free+0x210/0x2a0
> [ 20.603563] LR [c0000000002641b4] mempool_free_slab+0x24/0x40
> [ 20.609091] Call Trace:
> [ 20.609096] [c000200e495875f0] [c000200e49587680] 0xc000200e49587680 (unreliable)
> [ 20.609099] [c000200e49587680] [c000200e49031cf0] 0xc000200e49031cf0
> [ 20.609107] [c000200e495876a0] [c0000000002642b0] mempool_exit+0x50/0x90
> [ 20.632672] [c000200e495876d0] [c0000000005360d0] bioset_exit+0x40/0x1d0
> [ 20.638210] [c000200e49587750] [c00000000087c84c] dm_io_client_destroy+0x2c/0x50
> [ 20.646524] [c000200e49587780] [c0080000083f2f44] dm_bufio_client_destroy+0x1fc/0x2d0 [dm_bufio]
> [ 20.654849] [c000200e49587810] [c0080000089667d8] persistent_read_metadata+0x430/0x660 [dm_snapshot]
> [ 20.664547] [c000200e495878d0] [c0080000089643a0] snapshot_ctr+0x5c8/0x7a0 [dm_snapshot]
> [ 20.671487] [c000200e495879b0] [c000000000874b0c] dm_table_add_target+0x19c/0x3c0
> [ 20.679799] [c000200e49587a60] [c00000000087a344] table_load+0x104/0x450
> [ 20.686726] [c000200e49587b10] [c00000000087b1b8] ctl_ioctl+0x1f8/0x570
> [ 20.692286] [c000200e49587d00] [c00000000087b548] dm_ctl_ioctl+0x18/0x30
> [ 20.699215] [c000200e49587d20] [c000000000354e2c] do_vfs_ioctl+0xcc/0x9e0
> [ 20.706144] [c000200e49587dc0] [c00000000035579c] ksys_ioctl+0x5c/0xe0
> [ 20.713080] [c000200e49587e10] [c000000000355840] sys_ioctl+0x20/0x80
> [ 20.718637] [c000200e49587e30] [c00000000000b9e0] system_call+0x58/0x6c
> [ 20.725560] Instruction dump:
> [ 20.728341] 3d02000a 39089150 7ba98502 792a3664 e9280000 7d295214 e9490008 71480001
> [ 20.736652] 40820090 e9290018 7fbf4840 419efe28 <e94900d0> 7fbf5000 419e0068 7c0802a6
> [ 20.744976] ---[ end trace 076e4b6d277fc734 ]---
>
> [ 22.347952] bnx2x 0031:01:00.3 enP49p1s0f3: using MSI-X IRQs: sp 524 fp[0] 526 ... fp[7] 533
> [ 22.566968] bnx2x 0031:01:00.3 enP49p1s0f3: NIC Link is Up, 10000 Mbps full duplex, Flow control: ON - receive & transmit
> [ 22.767776] bnx2x 0031:01:00.1 enP49p1s0f1: using MSI-X IRQs: sp 310 fp[0] 312 ... fp[7] 319
> [ 23.508053] bnx2x 0031:01:00.0 enP49p1s0f0: using MSI-X IRQs: sp 300 fp[0] 302 ... fp[7] 309
> [ 33.797102] NFSD: starting 90-second grace period (net f000001f)
^ permalink raw reply related
* Re: Oops in kmem_cache_free() via bioset_exit() (was Re: [next-20180601][nvme][ppc] Kernel Oops is triggered when creating lvm snapshots on nvme disks)
From: Jens Axboe @ 2018-06-28 19:57 UTC (permalink / raw)
To: Michael Ellerman, Abdul Haleem
Cc: linuxppc-dev, linux-fsdevel, linux-next, linux-kernel, linux-scsi,
Stephen Rothwell, sachinp, sim, manvanth, Brian King, linux-block,
Kent Overstreet
In-Reply-To: <87a7rf55vy.fsf@concordia.ellerman.id.au>
On 6/28/18 8:42 AM, Michael Ellerman wrote:
> Kent, Jens,
>
> This looks like it might be related to the recent bioset changes?
>
> cheers
>
> Abdul Haleem <abdhalee@linux.vnet.ibm.com> writes:
>> On Tue, 2018-06-26 at 23:36 +1000, Michael Ellerman wrote:
>>> Abdul Haleem <abdhalee@linux.vnet.ibm.com> writes:
> ...
>> I was able to reproduce again with slub_debug=FZP and DEBUG_INFO enabled
>> on 4.17.0-rc7-next-20180601, but not much traces other than the Oops stack trace
>
> Are you still testing on that revision? It's nearly a month old.
>
> Please try to reproduce on mainline or today's linux-next.
>
>
>> the faulty instruction points to below code path :
>>
>> gdb -batch vmlinux -ex 'list *(0xc000000000304fe0)'
>> 0xc000000000304fe0 is in kmem_cache_free (mm/slab.h:231).
>> 226 }
>> 227
>> 228 static inline bool slab_equal_or_root(struct kmem_cache *s,
>> 229 struct kmem_cache *p)
>> 230 {
>> 231 return p == s || p == s->memcg_params.root_cache;
>> 232 }
>
> And s is NULL.
>
> Called via:
> kmem_cache_free+0x210/0x2a0
> mempool_free_slab+0x24/0x40
> mempool_exit+0x50/0x90
> bioset_exit+0x40/0x1d0
> dm_io_client_destroy+0x2c/0x50
> dm_bufio_client_destroy+0x1fc/0x2d0 [dm_bufio]
> persistent_read_metadata+0x430/0x660 [dm_snapshot]
> snapshot_ctr+0x5c8/0x7a0 [dm_snapshot]
> dm_table_add_target+0x19c/0x3c0
> table_load+0x104/0x450
> ctl_ioctl+0x1f8/0x570
> dm_ctl_ioctl+0x18/0x30
> do_vfs_ioctl+0xcc/0x9e0
> ksys_ioctl+0x5c/0xe0
> sys_ioctl+0x20/0x80
> system_call+0x58/0x6c
>
> So looks like we did:
>
> kmem_cache_free(NULL
>
> Probably a bad error path that frees before the cache has been allocated.
>
> mempool_init_node() calls mempool_exit() on a partially initialised
> mempool, which looks fishy, though you're not hitting that patch AFAICS.
The slab cache is setup elsewhere, it's pending_cache. So if pending_cache
is NULL, then yeah and exit there will barf. I'd try something like the
below, but from the trace, we already basically see the path.
diff --git a/include/linux/mempool.h b/include/linux/mempool.h
index 0c964ac107c2..ebfa2f89ffdd 100644
--- a/include/linux/mempool.h
+++ b/include/linux/mempool.h
@@ -59,6 +59,7 @@ void mempool_free_slab(void *element, void *pool_data);
static inline int
mempool_init_slab_pool(mempool_t *pool, int min_nr, struct kmem_cache *kc)
{
+ BUG_ON(!kc);
return mempool_init(pool, min_nr, mempool_alloc_slab,
mempool_free_slab, (void *) kc);
}
diff --git a/mm/mempool.c b/mm/mempool.c
index b54f2c20e5e0..060f44acd0df 100644
--- a/mm/mempool.c
+++ b/mm/mempool.c
@@ -508,7 +508,9 @@ EXPORT_SYMBOL(mempool_alloc_slab);
void mempool_free_slab(void *element, void *pool_data)
{
struct kmem_cache *mem = pool_data;
- kmem_cache_free(mem, element);
+
+ if (!WARN_ON(!mem))
+ kmem_cache_free(mem, element);
}
EXPORT_SYMBOL(mempool_free_slab);
--
Jens Axboe
^ permalink raw reply related
* Re: [PATCH v4 1/6] powerpc/pseries: Defer the logging of rtas error to irq work queue.
From: kbuild test robot @ 2018-06-28 21:05 UTC (permalink / raw)
To: Mahesh J Salgaonkar
Cc: kbuild-all, linuxppc-dev, Laurent Dufour, Nicholas Piggin, stable,
Aneesh Kumar K.V
In-Reply-To: <153018423713.31589.14195888319570120941.stgit@jupiter.in.ibm.com>
[-- Attachment #1: Type: text/plain, Size: 2182 bytes --]
Hi Mahesh,
Thank you for the patch! Yet something to improve:
[auto build test ERROR on powerpc/next]
[also build test ERROR on v4.18-rc2 next-20180628]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system]
url: https://github.com/0day-ci/linux/commits/Mahesh-J-Salgaonkar/powerpc-pseries-Defer-the-logging-of-rtas-error-to-irq-work-queue/20180628-224101
base: https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git next
config: powerpc-defconfig (attached as .config)
compiler: powerpc64-linux-gnu-gcc (Debian 7.2.0-11) 7.2.0
reproduce:
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# save the attached .config to linux build tree
GCC_VERSION=7.2.0 make.cross ARCH=powerpc
Note: the linux-review/Mahesh-J-Salgaonkar/powerpc-pseries-Defer-the-logging-of-rtas-error-to-irq-work-queue/20180628-224101 HEAD 3496ae1afd6528103d508528e25bfca82c60f4ee builds fine.
It only hurts bisectibility.
All errors (new ones prefixed by >>):
arch/powerpc/platforms/pseries/ras.c: In function 'mce_process_errlog_event':
>> arch/powerpc/platforms/pseries/ras.c:433:8: error: implicit declaration of function 'fwnmi_get_errlog'; did you mean 'fwnmi_get_errinfo'? [-Werror=implicit-function-declaration]
err = fwnmi_get_errlog();
^~~~~~~~~~~~~~~~
fwnmi_get_errinfo
>> arch/powerpc/platforms/pseries/ras.c:433:6: error: assignment makes pointer from integer without a cast [-Werror=int-conversion]
err = fwnmi_get_errlog();
^
cc1: all warnings being treated as errors
vim +433 arch/powerpc/platforms/pseries/ras.c
425
426 /*
427 * Process MCE rtas errlog event.
428 */
429 static void mce_process_errlog_event(struct irq_work *work)
430 {
431 struct rtas_error_log *err;
432
> 433 err = fwnmi_get_errlog();
434 log_error((char *)err, ERR_TYPE_RTAS_LOG, 0);
435 }
436
---
0-DAY kernel test infrastructure Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all Intel Corporation
[-- Attachment #2: .config.gz --]
[-- Type: application/gzip, Size: 23378 bytes --]
^ permalink raw reply
* Re: [PATCH v2 01/10] Revert "cxl: Add kernel API to allow a context to operate with relocate disabled"
From: Andrew Donnellan @ 2018-06-28 23:44 UTC (permalink / raw)
To: Frederic Barrat, alastair, vaibhav, clombard, felix, linuxppc-dev; +Cc: huyn
In-Reply-To: <20180628100509.17413-2-fbarrat@linux.ibm.com>
On 28/06/18 20:05, Frederic Barrat wrote:
> From: Alastair D'Silva <alastair@d-silva.org>
>
> Remove abandonned capi support for the Mellanox CX4.
> The symbol 'cxl_set_translation_mode' is never called, so
> ctx->real_mode is always false.
>
> This reverts commit 7a0d85d313c2066712e530e668bc02bb741a685c.
>
> Signed-off-by: Alastair D'Silva <alastair@d-silva.org>
Acked-by: Andrew Donnellan <andrew.donnellan@au1.ibm.com>
> ---
> drivers/misc/cxl/api.c | 19 -------------------
> drivers/misc/cxl/cxl.h | 1 -
> drivers/misc/cxl/guest.c | 3 ---
> drivers/misc/cxl/native.c | 3 ++-
> include/misc/cxl.h | 8 --------
> 5 files changed, 2 insertions(+), 32 deletions(-)
>
> diff --git a/drivers/misc/cxl/api.c b/drivers/misc/cxl/api.c
> index 753b1a698fc4..21d620e29fea 100644
> --- a/drivers/misc/cxl/api.c
> +++ b/drivers/misc/cxl/api.c
> @@ -324,7 +324,6 @@ int cxl_start_context(struct cxl_context *ctx, u64 wed,
> if (task) {
> ctx->pid = get_task_pid(task, PIDTYPE_PID);
> kernel = false;
> - ctx->real_mode = false;
>
> /* acquire a reference to the task's mm */
> ctx->mm = get_task_mm(current);
> @@ -388,24 +387,6 @@ void cxl_set_master(struct cxl_context *ctx)
> }
> EXPORT_SYMBOL_GPL(cxl_set_master);
>
> -int cxl_set_translation_mode(struct cxl_context *ctx, bool real_mode)
> -{
> - if (ctx->status == STARTED) {
> - /*
> - * We could potentially update the PE and issue an update LLCMD
> - * to support this, but it doesn't seem to have a good use case
> - * since it's trivial to just create a second kernel context
> - * with different translation modes, so until someone convinces
> - * me otherwise:
> - */
> - return -EBUSY;
> - }
> -
> - ctx->real_mode = real_mode;
> - return 0;
> -}
> -EXPORT_SYMBOL_GPL(cxl_set_translation_mode);
> -
> /* wrappers around afu_* file ops which are EXPORTED */
> int cxl_fd_open(struct inode *inode, struct file *file)
> {
> diff --git a/drivers/misc/cxl/cxl.h b/drivers/misc/cxl/cxl.h
> index 918d4fb742d1..af8794719956 100644
> --- a/drivers/misc/cxl/cxl.h
> +++ b/drivers/misc/cxl/cxl.h
> @@ -613,7 +613,6 @@ struct cxl_context {
> bool pe_inserted;
> bool master;
> bool kernel;
> - bool real_mode;
> bool pending_irq;
> bool pending_fault;
> bool pending_afu_err;
> diff --git a/drivers/misc/cxl/guest.c b/drivers/misc/cxl/guest.c
> index 4644f16606a3..f5dc740fcd13 100644
> --- a/drivers/misc/cxl/guest.c
> +++ b/drivers/misc/cxl/guest.c
> @@ -623,9 +623,6 @@ static int guest_attach_process(struct cxl_context *ctx, bool kernel, u64 wed, u
> {
> pr_devel("in %s\n", __func__);
>
> - if (ctx->real_mode)
> - return -EPERM;
> -
> ctx->kernel = kernel;
> if (ctx->afu->current_mode == CXL_MODE_DIRECTED)
> return attach_afu_directed(ctx, wed, amr);
> diff --git a/drivers/misc/cxl/native.c b/drivers/misc/cxl/native.c
> index 98f867fcef24..c9d5d82dce8e 100644
> --- a/drivers/misc/cxl/native.c
> +++ b/drivers/misc/cxl/native.c
> @@ -605,6 +605,7 @@ u64 cxl_calculate_sr(bool master, bool kernel, bool real_mode, bool p9)
> sr |= CXL_PSL_SR_An_MP;
> if (mfspr(SPRN_LPCR) & LPCR_TC)
> sr |= CXL_PSL_SR_An_TC;
> +
> if (kernel) {
> if (!real_mode)
> sr |= CXL_PSL_SR_An_R;
> @@ -629,7 +630,7 @@ u64 cxl_calculate_sr(bool master, bool kernel, bool real_mode, bool p9)
>
> static u64 calculate_sr(struct cxl_context *ctx)
> {
> - return cxl_calculate_sr(ctx->master, ctx->kernel, ctx->real_mode,
> + return cxl_calculate_sr(ctx->master, ctx->kernel, false,
> cxl_is_power9());
> }
>
> diff --git a/include/misc/cxl.h b/include/misc/cxl.h
> index b712be544f8c..82cc6ffafe2d 100644
> --- a/include/misc/cxl.h
> +++ b/include/misc/cxl.h
> @@ -173,14 +173,6 @@ int cxl_afu_reset(struct cxl_context *ctx);
> */
> void cxl_set_master(struct cxl_context *ctx);
>
> -/*
> - * Sets the context to use real mode memory accesses to operate with
> - * translation disabled. Note that this only makes sense for kernel contexts
> - * under bare metal, and will not work with virtualisation. May only be
> - * performed on stopped contexts.
> - */
> -int cxl_set_translation_mode(struct cxl_context *ctx, bool real_mode);
> -
> /*
> * Map and unmap the AFU Problem Space area. The amount and location mapped
> * depends on if this context is a master or slave.
>
--
Andrew Donnellan OzLabs, ADL Canberra
andrew.donnellan@au1.ibm.com IBM Australia Limited
^ permalink raw reply
* Re: [PATCH v2 02/10] Revert "cxl: Add support for interrupts on the Mellanox CX4"
From: Andrew Donnellan @ 2018-06-28 23:45 UTC (permalink / raw)
To: Frederic Barrat, alastair, vaibhav, clombard, felix, linuxppc-dev; +Cc: huyn
In-Reply-To: <20180628100509.17413-3-fbarrat@linux.ibm.com>
On 28/06/18 20:05, Frederic Barrat wrote:
> From: Alastair D'Silva <alastair@d-silva.org>
>
> Remove abandonned capi support for the Mellanox CX4.
>
> This reverts commit a2f67d5ee8d950caaa7a6144cf0bfb256500b73e.
>
> Signed-off-by: Alastair D'Silva <alastair@d-silva.org>
Acked-by: Andrew Donnellan <andrew.donnellan@au1.ibm.com>
> ---
> arch/powerpc/platforms/powernv/pci-cxl.c | 84 -----------------------
> arch/powerpc/platforms/powernv/pci-ioda.c | 4 --
> arch/powerpc/platforms/powernv/pci.h | 2 -
> drivers/misc/cxl/api.c | 71 -------------------
> drivers/misc/cxl/base.c | 31 ---------
> drivers/misc/cxl/cxl.h | 4 --
> drivers/misc/cxl/main.c | 2 -
> include/misc/cxl-base.h | 4 --
> 8 files changed, 202 deletions(-)
>
> diff --git a/arch/powerpc/platforms/powernv/pci-cxl.c b/arch/powerpc/platforms/powernv/pci-cxl.c
> index cee003de63af..c447b7f03c09 100644
> --- a/arch/powerpc/platforms/powernv/pci-cxl.c
> +++ b/arch/powerpc/platforms/powernv/pci-cxl.c
> @@ -8,7 +8,6 @@
> */
>
> #include <linux/module.h>
> -#include <linux/msi.h>
> #include <asm/pci-bridge.h>
> #include <asm/pnv-pci.h>
> #include <asm/opal.h>
> @@ -292,86 +291,3 @@ void pnv_cxl_disable_device(struct pci_dev *dev)
> cxl_pci_disable_device(dev);
> cxl_afu_put(afu);
> }
> -
> -/*
> - * This is a special version of pnv_setup_msi_irqs for cards in cxl mode. This
> - * function handles setting up the IVTE entries for the XSL to use.
> - *
> - * We are currently not filling out the MSIX table, since the only currently
> - * supported adapter (CX4) uses a custom MSIX table format in cxl mode and it
> - * is up to their driver to fill that out. In the future we may fill out the
> - * MSIX table (and change the IVTE entries to be an index to the MSIX table)
> - * for adapters implementing the Full MSI-X mode described in the CAIA.
> - */
> -int pnv_cxl_cx4_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type)
> -{
> - struct pci_controller *hose = pci_bus_to_host(pdev->bus);
> - struct pnv_phb *phb = hose->private_data;
> - struct msi_desc *entry;
> - struct cxl_context *ctx = NULL;
> - unsigned int virq;
> - int hwirq;
> - int afu_irq = 0;
> - int rc;
> -
> - if (WARN_ON(!phb) || !phb->msi_bmp.bitmap)
> - return -ENODEV;
> -
> - if (pdev->no_64bit_msi && !phb->msi32_support)
> - return -ENODEV;
> -
> - rc = cxl_cx4_setup_msi_irqs(pdev, nvec, type);
> - if (rc)
> - return rc;
> -
> - for_each_pci_msi_entry(entry, pdev) {
> - if (!entry->msi_attrib.is_64 && !phb->msi32_support) {
> - pr_warn("%s: Supports only 64-bit MSIs\n",
> - pci_name(pdev));
> - return -ENXIO;
> - }
> -
> - hwirq = cxl_next_msi_hwirq(pdev, &ctx, &afu_irq);
> - if (WARN_ON(hwirq <= 0))
> - return (hwirq ? hwirq : -ENOMEM);
> -
> - virq = irq_create_mapping(NULL, hwirq);
> - if (!virq) {
> - pr_warn("%s: Failed to map cxl mode MSI to linux irq\n",
> - pci_name(pdev));
> - return -ENOMEM;
> - }
> -
> - rc = pnv_cxl_ioda_msi_setup(pdev, hwirq, virq);
> - if (rc) {
> - pr_warn("%s: Failed to setup cxl mode MSI\n", pci_name(pdev));
> - irq_dispose_mapping(virq);
> - return rc;
> - }
> -
> - irq_set_msi_desc(virq, entry);
> - }
> -
> - return 0;
> -}
> -
> -void pnv_cxl_cx4_teardown_msi_irqs(struct pci_dev *pdev)
> -{
> - struct pci_controller *hose = pci_bus_to_host(pdev->bus);
> - struct pnv_phb *phb = hose->private_data;
> - struct msi_desc *entry;
> - irq_hw_number_t hwirq;
> -
> - if (WARN_ON(!phb))
> - return;
> -
> - for_each_pci_msi_entry(entry, pdev) {
> - if (!entry->irq)
> - continue;
> - hwirq = virq_to_hw(entry->irq);
> - irq_set_msi_desc(entry->irq, NULL);
> - irq_dispose_mapping(entry->irq);
> - }
> -
> - cxl_cx4_teardown_msi_irqs(pdev);
> -}
> diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
> index 5bd0eb6681bc..41f8f0ff4a55 100644
> --- a/arch/powerpc/platforms/powernv/pci-ioda.c
> +++ b/arch/powerpc/platforms/powernv/pci-ioda.c
> @@ -3847,10 +3847,6 @@ static const struct pci_controller_ops pnv_npu_ocapi_ioda_controller_ops = {
> const struct pci_controller_ops pnv_cxl_cx4_ioda_controller_ops = {
> .dma_dev_setup = pnv_pci_dma_dev_setup,
> .dma_bus_setup = pnv_pci_dma_bus_setup,
> -#ifdef CONFIG_PCI_MSI
> - .setup_msi_irqs = pnv_cxl_cx4_setup_msi_irqs,
> - .teardown_msi_irqs = pnv_cxl_cx4_teardown_msi_irqs,
> -#endif
> .enable_device_hook = pnv_cxl_enable_device_hook,
> .disable_device = pnv_cxl_disable_device,
> .release_device = pnv_pci_release_device,
> diff --git a/arch/powerpc/platforms/powernv/pci.h b/arch/powerpc/platforms/powernv/pci.h
> index eada4b6068cb..ba41913c7e21 100644
> --- a/arch/powerpc/platforms/powernv/pci.h
> +++ b/arch/powerpc/platforms/powernv/pci.h
> @@ -265,8 +265,6 @@ extern int pnv_npu2_init(struct pnv_phb *phb);
> /* cxl functions */
> extern bool pnv_cxl_enable_device_hook(struct pci_dev *dev);
> extern void pnv_cxl_disable_device(struct pci_dev *dev);
> -extern int pnv_cxl_cx4_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type);
> -extern void pnv_cxl_cx4_teardown_msi_irqs(struct pci_dev *pdev);
>
>
> /* phb ops (cxl switches these when enabling the kernel api on the phb) */
> diff --git a/drivers/misc/cxl/api.c b/drivers/misc/cxl/api.c
> index 21d620e29fea..2e5862b7a074 100644
> --- a/drivers/misc/cxl/api.c
> +++ b/drivers/misc/cxl/api.c
> @@ -11,7 +11,6 @@
> #include <linux/slab.h>
> #include <linux/file.h>
> #include <misc/cxl.h>
> -#include <linux/msi.h>
> #include <linux/module.h>
> #include <linux/mount.h>
> #include <linux/sched/mm.h>
> @@ -595,73 +594,3 @@ int cxl_get_max_irqs_per_process(struct pci_dev *dev)
> return afu->irqs_max;
> }
> EXPORT_SYMBOL_GPL(cxl_get_max_irqs_per_process);
> -
> -/*
> - * This is a special interrupt allocation routine called from the PHB's MSI
> - * setup function. When capi interrupts are allocated in this manner they must
> - * still be associated with a running context, but since the MSI APIs have no
> - * way to specify this we use the default context associated with the device.
> - *
> - * The Mellanox CX4 has a hardware limitation that restricts the maximum AFU
> - * interrupt number, so in order to overcome this their driver informs us of
> - * the restriction by setting the maximum interrupts per context, and we
> - * allocate additional contexts as necessary so that we can keep the AFU
> - * interrupt number within the supported range.
> - */
> -int _cxl_cx4_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type)
> -{
> - struct cxl_context *ctx, *new_ctx, *default_ctx;
> - int remaining;
> - int rc;
> -
> - ctx = default_ctx = cxl_get_context(pdev);
> - if (WARN_ON(!default_ctx))
> - return -ENODEV;
> -
> - remaining = nvec;
> - while (remaining > 0) {
> - rc = cxl_allocate_afu_irqs(ctx, min(remaining, ctx->afu->irqs_max));
> - if (rc) {
> - pr_warn("%s: Failed to find enough free MSIs\n", pci_name(pdev));
> - return rc;
> - }
> - remaining -= ctx->afu->irqs_max;
> -
> - if (ctx != default_ctx && default_ctx->status == STARTED) {
> - WARN_ON(cxl_start_context(ctx,
> - be64_to_cpu(default_ctx->elem->common.wed),
> - NULL));
> - }
> -
> - if (remaining > 0) {
> - new_ctx = cxl_dev_context_init(pdev);
> - if (IS_ERR(new_ctx)) {
> - pr_warn("%s: Failed to allocate enough contexts for MSIs\n", pci_name(pdev));
> - return -ENOSPC;
> - }
> - list_add(&new_ctx->extra_irq_contexts, &ctx->extra_irq_contexts);
> - ctx = new_ctx;
> - }
> - }
> -
> - return 0;
> -}
> -/* Exported via cxl_base */
> -
> -void _cxl_cx4_teardown_msi_irqs(struct pci_dev *pdev)
> -{
> - struct cxl_context *ctx, *pos, *tmp;
> -
> - ctx = cxl_get_context(pdev);
> - if (WARN_ON(!ctx))
> - return;
> -
> - cxl_free_afu_irqs(ctx);
> - list_for_each_entry_safe(pos, tmp, &ctx->extra_irq_contexts, extra_irq_contexts) {
> - cxl_stop_context(pos);
> - cxl_free_afu_irqs(pos);
> - list_del(&pos->extra_irq_contexts);
> - cxl_release_context(pos);
> - }
> -}
> -/* Exported via cxl_base */
> diff --git a/drivers/misc/cxl/base.c b/drivers/misc/cxl/base.c
> index cd54ce6f6230..fe90f895bb10 100644
> --- a/drivers/misc/cxl/base.c
> +++ b/drivers/misc/cxl/base.c
> @@ -158,37 +158,6 @@ int cxl_next_msi_hwirq(struct pci_dev *pdev, struct cxl_context **ctx, int *afu_
> }
> EXPORT_SYMBOL_GPL(cxl_next_msi_hwirq);
>
> -int cxl_cx4_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type)
> -{
> - int ret;
> - struct cxl_calls *calls;
> -
> - calls = cxl_calls_get();
> - if (!calls)
> - return false;
> -
> - ret = calls->cxl_cx4_setup_msi_irqs(pdev, nvec, type);
> -
> - cxl_calls_put(calls);
> -
> - return ret;
> -}
> -EXPORT_SYMBOL_GPL(cxl_cx4_setup_msi_irqs);
> -
> -void cxl_cx4_teardown_msi_irqs(struct pci_dev *pdev)
> -{
> - struct cxl_calls *calls;
> -
> - calls = cxl_calls_get();
> - if (!calls)
> - return;
> -
> - calls->cxl_cx4_teardown_msi_irqs(pdev);
> -
> - cxl_calls_put(calls);
> -}
> -EXPORT_SYMBOL_GPL(cxl_cx4_teardown_msi_irqs);
> -
> static int __init cxl_base_init(void)
> {
> struct device_node *np;
> diff --git a/drivers/misc/cxl/cxl.h b/drivers/misc/cxl/cxl.h
> index af8794719956..9688fe8b4d80 100644
> --- a/drivers/misc/cxl/cxl.h
> +++ b/drivers/misc/cxl/cxl.h
> @@ -879,16 +879,12 @@ ssize_t cxl_pci_afu_read_err_buffer(struct cxl_afu *afu, char *buf,
> bool _cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu);
> void _cxl_pci_disable_device(struct pci_dev *dev);
> int _cxl_next_msi_hwirq(struct pci_dev *pdev, struct cxl_context **ctx, int *afu_irq);
> -int _cxl_cx4_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type);
> -void _cxl_cx4_teardown_msi_irqs(struct pci_dev *pdev);
>
> struct cxl_calls {
> void (*cxl_slbia)(struct mm_struct *mm);
> bool (*cxl_pci_associate_default_context)(struct pci_dev *dev, struct cxl_afu *afu);
> void (*cxl_pci_disable_device)(struct pci_dev *dev);
> int (*cxl_next_msi_hwirq)(struct pci_dev *pdev, struct cxl_context **ctx, int *afu_irq);
> - int (*cxl_cx4_setup_msi_irqs)(struct pci_dev *pdev, int nvec, int type);
> - void (*cxl_cx4_teardown_msi_irqs)(struct pci_dev *pdev);
>
> struct module *owner;
> };
> diff --git a/drivers/misc/cxl/main.c b/drivers/misc/cxl/main.c
> index c1ba0d42cbc8..59a904efd104 100644
> --- a/drivers/misc/cxl/main.c
> +++ b/drivers/misc/cxl/main.c
> @@ -107,8 +107,6 @@ static struct cxl_calls cxl_calls = {
> .cxl_pci_associate_default_context = _cxl_pci_associate_default_context,
> .cxl_pci_disable_device = _cxl_pci_disable_device,
> .cxl_next_msi_hwirq = _cxl_next_msi_hwirq,
> - .cxl_cx4_setup_msi_irqs = _cxl_cx4_setup_msi_irqs,
> - .cxl_cx4_teardown_msi_irqs = _cxl_cx4_teardown_msi_irqs,
> .owner = THIS_MODULE,
> };
>
> diff --git a/include/misc/cxl-base.h b/include/misc/cxl-base.h
> index b2ebc91fe09a..bb7e629ae492 100644
> --- a/include/misc/cxl-base.h
> +++ b/include/misc/cxl-base.h
> @@ -43,8 +43,6 @@ void cxl_afu_put(struct cxl_afu *afu);
> void cxl_slbia(struct mm_struct *mm);
> bool cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu);
> void cxl_pci_disable_device(struct pci_dev *dev);
> -int cxl_cx4_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type);
> -void cxl_cx4_teardown_msi_irqs(struct pci_dev *pdev);
>
> #else /* CONFIG_CXL_BASE */
>
> @@ -54,8 +52,6 @@ static inline void cxl_afu_put(struct cxl_afu *afu) {}
> static inline void cxl_slbia(struct mm_struct *mm) {}
> static inline bool cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu) { return false; }
> static inline void cxl_pci_disable_device(struct pci_dev *dev) {}
> -static inline int cxl_cx4_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) { return -ENODEV; }
> -static inline void cxl_cx4_teardown_msi_irqs(struct pci_dev *pdev) {}
>
> #endif /* CONFIG_CXL_BASE */
>
>
--
Andrew Donnellan OzLabs, ADL Canberra
andrew.donnellan@au1.ibm.com IBM Australia Limited
^ permalink raw reply
* Re: [PATCH v2 03/10] Revert "cxl: Add preliminary workaround for CX4 interrupt limitation"
From: Andrew Donnellan @ 2018-06-28 23:45 UTC (permalink / raw)
To: Frederic Barrat, alastair, vaibhav, clombard, felix, linuxppc-dev; +Cc: huyn
In-Reply-To: <20180628100509.17413-4-fbarrat@linux.ibm.com>
On 28/06/18 20:05, Frederic Barrat wrote:
> From: Alastair D'Silva <alastair@d-silva.org>
>
> Remove abandonned capi support for the Mellanox CX4.
>
> This reverts commit cbce0917e2e47d4bf5aa3b5fd6b1247f33e1a126.
>
> Signed-off-by: Alastair D'Silva <alastair@d-silva.org>
Acked-by: Andrew Donnellan <andrew.donnellan@au1.ibm.com>
> ---
> drivers/misc/cxl/api.c | 15 ---------------
> drivers/misc/cxl/base.c | 17 -----------------
> drivers/misc/cxl/context.c | 1 -
> drivers/misc/cxl/cxl.h | 10 ----------
> drivers/misc/cxl/main.c | 1 -
> include/misc/cxl.h | 20 --------------------
> 6 files changed, 64 deletions(-)
>
> diff --git a/drivers/misc/cxl/api.c b/drivers/misc/cxl/api.c
> index 2e5862b7a074..34ba67bc41bd 100644
> --- a/drivers/misc/cxl/api.c
> +++ b/drivers/misc/cxl/api.c
> @@ -181,21 +181,6 @@ static irq_hw_number_t cxl_find_afu_irq(struct cxl_context *ctx, int num)
> return 0;
> }
>
> -int _cxl_next_msi_hwirq(struct pci_dev *pdev, struct cxl_context **ctx, int *afu_irq)
> -{
> - if (*ctx == NULL || *afu_irq == 0) {
> - *afu_irq = 1;
> - *ctx = cxl_get_context(pdev);
> - } else {
> - (*afu_irq)++;
> - if (*afu_irq > cxl_get_max_irqs_per_process(pdev)) {
> - *ctx = list_next_entry(*ctx, extra_irq_contexts);
> - *afu_irq = 1;
> - }
> - }
> - return cxl_find_afu_irq(*ctx, *afu_irq);
> -}
> -/* Exported via cxl_base */
>
> int cxl_set_priv(struct cxl_context *ctx, void *priv)
> {
> diff --git a/drivers/misc/cxl/base.c b/drivers/misc/cxl/base.c
> index fe90f895bb10..e1e80cb99ad9 100644
> --- a/drivers/misc/cxl/base.c
> +++ b/drivers/misc/cxl/base.c
> @@ -141,23 +141,6 @@ void cxl_pci_disable_device(struct pci_dev *dev)
> }
> EXPORT_SYMBOL_GPL(cxl_pci_disable_device);
>
> -int cxl_next_msi_hwirq(struct pci_dev *pdev, struct cxl_context **ctx, int *afu_irq)
> -{
> - int ret;
> - struct cxl_calls *calls;
> -
> - calls = cxl_calls_get();
> - if (!calls)
> - return -EBUSY;
> -
> - ret = calls->cxl_next_msi_hwirq(pdev, ctx, afu_irq);
> -
> - cxl_calls_put(calls);
> -
> - return ret;
> -}
> -EXPORT_SYMBOL_GPL(cxl_next_msi_hwirq);
> -
> static int __init cxl_base_init(void)
> {
> struct device_node *np;
> diff --git a/drivers/misc/cxl/context.c b/drivers/misc/cxl/context.c
> index c6ec872800a2..0355d42d367f 100644
> --- a/drivers/misc/cxl/context.c
> +++ b/drivers/misc/cxl/context.c
> @@ -74,7 +74,6 @@ int cxl_context_init(struct cxl_context *ctx, struct cxl_afu *afu, bool master)
> ctx->pending_afu_err = false;
>
> INIT_LIST_HEAD(&ctx->irq_names);
> - INIT_LIST_HEAD(&ctx->extra_irq_contexts);
>
> /*
> * When we have to destroy all contexts in cxl_context_detach_all() we
> diff --git a/drivers/misc/cxl/cxl.h b/drivers/misc/cxl/cxl.h
> index 9688fe8b4d80..d95c2c98f2ab 100644
> --- a/drivers/misc/cxl/cxl.h
> +++ b/drivers/misc/cxl/cxl.h
> @@ -623,14 +623,6 @@ struct cxl_context {
>
> struct rcu_head rcu;
>
> - /*
> - * Only used when more interrupts are allocated via
> - * pci_enable_msix_range than are supported in the default context, to
> - * use additional contexts to overcome the limitation. i.e. Mellanox
> - * CX4 only:
> - */
> - struct list_head extra_irq_contexts;
> -
> struct mm_struct *mm;
>
> u16 tidr;
> @@ -878,13 +870,11 @@ ssize_t cxl_pci_afu_read_err_buffer(struct cxl_afu *afu, char *buf,
> /* Internal functions wrapped in cxl_base to allow PHB to call them */
> bool _cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu);
> void _cxl_pci_disable_device(struct pci_dev *dev);
> -int _cxl_next_msi_hwirq(struct pci_dev *pdev, struct cxl_context **ctx, int *afu_irq);
>
> struct cxl_calls {
> void (*cxl_slbia)(struct mm_struct *mm);
> bool (*cxl_pci_associate_default_context)(struct pci_dev *dev, struct cxl_afu *afu);
> void (*cxl_pci_disable_device)(struct pci_dev *dev);
> - int (*cxl_next_msi_hwirq)(struct pci_dev *pdev, struct cxl_context **ctx, int *afu_irq);
>
> struct module *owner;
> };
> diff --git a/drivers/misc/cxl/main.c b/drivers/misc/cxl/main.c
> index 59a904efd104..a7e83624034b 100644
> --- a/drivers/misc/cxl/main.c
> +++ b/drivers/misc/cxl/main.c
> @@ -106,7 +106,6 @@ static struct cxl_calls cxl_calls = {
> .cxl_slbia = cxl_slbia_core,
> .cxl_pci_associate_default_context = _cxl_pci_associate_default_context,
> .cxl_pci_disable_device = _cxl_pci_disable_device,
> - .cxl_next_msi_hwirq = _cxl_next_msi_hwirq,
> .owner = THIS_MODULE,
> };
>
> diff --git a/include/misc/cxl.h b/include/misc/cxl.h
> index 82cc6ffafe2d..6a3711a2e217 100644
> --- a/include/misc/cxl.h
> +++ b/include/misc/cxl.h
> @@ -183,26 +183,6 @@ void cxl_psa_unmap(void __iomem *addr);
> /* Get the process element for this context */
> int cxl_process_element(struct cxl_context *ctx);
>
> -/*
> - * Limit the number of interrupts that a single context can allocate via
> - * cxl_start_work. If using the api with a real phb, this may be used to
> - * request that additional default contexts be created when allocating
> - * interrupts via pci_enable_msix_range. These will be set to the same running
> - * state as the default context, and if that is running it will reuse the
> - * parameters previously passed to cxl_start_context for the default context.
> - */
> -int cxl_set_max_irqs_per_process(struct pci_dev *dev, int irqs);
> -int cxl_get_max_irqs_per_process(struct pci_dev *dev);
> -
> -/*
> - * Use to simultaneously iterate over hardware interrupt numbers, contexts and
> - * afu interrupt numbers allocated for the device via pci_enable_msix_range and
> - * is a useful convenience function when working with hardware that has
> - * limitations on the number of interrupts per process. *ctx and *afu_irq
> - * should be NULL and 0 to start the iteration.
> - */
> -int cxl_next_msi_hwirq(struct pci_dev *pdev, struct cxl_context **ctx, int *afu_irq);
> -
> /*
> * These calls allow drivers to create their own file descriptors and make them
> * identical to the cxl file descriptor user API. An example use case:
>
--
Andrew Donnellan OzLabs, ADL Canberra
andrew.donnellan@au1.ibm.com IBM Australia Limited
^ permalink raw reply
* Re: [PATCH v2 04/10] Revert "cxl: Add kernel APIs to get & set the max irqs per context"
From: Andrew Donnellan @ 2018-06-28 23:48 UTC (permalink / raw)
To: Frederic Barrat, alastair, vaibhav, clombard, felix, linuxppc-dev; +Cc: huyn
In-Reply-To: <20180628100509.17413-5-fbarrat@linux.ibm.com>
On 28/06/18 20:05, Frederic Barrat wrote:
> From: Alastair D'Silva <alastair@d-silva.org>
>
> Remove abandonned capi support for the Mellanox CX4.
>
> This reverts commit 79384e4b71240abf50c375eea56060b0d79c242a.
>
> Signed-off-by: Alastair D'Silva <alastair@d-silva.org>
Acked-by: Andrew Donnellan <andrew.donnellan@au1.ibm.com>
> ---
> drivers/misc/cxl/api.c | 27 ---------------------------
> 1 file changed, 27 deletions(-)
>
> diff --git a/drivers/misc/cxl/api.c b/drivers/misc/cxl/api.c
> index 34ba67bc41bd..a535c1e6aa92 100644
> --- a/drivers/misc/cxl/api.c
> +++ b/drivers/misc/cxl/api.c
> @@ -552,30 +552,3 @@ ssize_t cxl_read_adapter_vpd(struct pci_dev *dev, void *buf, size_t count)
> return cxl_ops->read_adapter_vpd(afu->adapter, buf, count);
> }
> EXPORT_SYMBOL_GPL(cxl_read_adapter_vpd);
> -
> -int cxl_set_max_irqs_per_process(struct pci_dev *dev, int irqs)
> -{
> - struct cxl_afu *afu = cxl_pci_to_afu(dev);
> - if (IS_ERR(afu))
> - return -ENODEV;
> -
> - if (irqs > afu->adapter->user_irqs)
> - return -EINVAL;
> -
> - /* Limit user_irqs to prevent the user increasing this via sysfs */
> - afu->adapter->user_irqs = irqs;
> - afu->irqs_max = irqs;
> -
> - return 0;
> -}
> -EXPORT_SYMBOL_GPL(cxl_set_max_irqs_per_process);
> -
> -int cxl_get_max_irqs_per_process(struct pci_dev *dev)
> -{
> - struct cxl_afu *afu = cxl_pci_to_afu(dev);
> - if (IS_ERR(afu))
> - return -ENODEV;
> -
> - return afu->irqs_max;
> -}
> -EXPORT_SYMBOL_GPL(cxl_get_max_irqs_per_process);
>
--
Andrew Donnellan OzLabs, ADL Canberra
andrew.donnellan@au1.ibm.com IBM Australia Limited
^ permalink raw reply
* Re: [PATCH v2 05/10] Revert "cxl: Add cxl_check_and_switch_mode() API to switch bi-modal cards"
From: Andrew Donnellan @ 2018-06-28 23:50 UTC (permalink / raw)
To: Frederic Barrat, alastair, vaibhav, clombard, felix, linuxppc-dev; +Cc: huyn
In-Reply-To: <20180628100509.17413-6-fbarrat@linux.ibm.com>
On 28/06/18 20:05, Frederic Barrat wrote:
> From: Alastair D'Silva <alastair@d-silva.org>
>
> Remove abandonned capi support for the Mellanox CX4.
>
> This reverts commit b0b5e5918ad1babfd1d43d98c7281926a7b57b9f.
>
> Signed-off-by: Alastair D'Silva <alastair@d-silva.org>
I was kinda proud at how dodgy this was and yet how it actually worked...
(Hmm, I should go back and see if there's anything we can rip out of
pnv_php now...)
Acked-by: Andrew Donnellan <andrew.donnellan@au1.ibm.com>
> ---
> drivers/misc/cxl/Kconfig | 8 --
> drivers/misc/cxl/pci.c | 236 +++------------------------------------
> include/misc/cxl.h | 25 -----
> 3 files changed, 18 insertions(+), 251 deletions(-)
>
> diff --git a/drivers/misc/cxl/Kconfig b/drivers/misc/cxl/Kconfig
> index 93397cb05b15..3ce933707828 100644
> --- a/drivers/misc/cxl/Kconfig
> +++ b/drivers/misc/cxl/Kconfig
> @@ -33,11 +33,3 @@ config CXL
> CAPI adapters are found in POWER8 based systems.
>
> If unsure, say N.
> -
> -config CXL_BIMODAL
> - bool "Support for bi-modal CAPI cards"
> - depends on HOTPLUG_PCI_POWERNV = y && CXL || HOTPLUG_PCI_POWERNV = m && CXL = m
> - default y
> - help
> - Select this option to enable support for bi-modal CAPI cards, such as
> - the Mellanox CX-4.
> diff --git a/drivers/misc/cxl/pci.c b/drivers/misc/cxl/pci.c
> index 429d6de1dde7..9c5a21fee835 100644
> --- a/drivers/misc/cxl/pci.c
> +++ b/drivers/misc/cxl/pci.c
> @@ -55,8 +55,6 @@
> pci_read_config_byte(dev, vsec + 0xa, dest)
> #define CXL_WRITE_VSEC_MODE_CONTROL(dev, vsec, val) \
> pci_write_config_byte(dev, vsec + 0xa, val)
> -#define CXL_WRITE_VSEC_MODE_CONTROL_BUS(bus, devfn, vsec, val) \
> - pci_bus_write_config_byte(bus, devfn, vsec + 0xa, val)
> #define CXL_VSEC_PROTOCOL_MASK 0xe0
> #define CXL_VSEC_PROTOCOL_1024TB 0x80
> #define CXL_VSEC_PROTOCOL_512TB 0x40
> @@ -800,234 +798,36 @@ static int setup_cxl_bars(struct pci_dev *dev)
> return 0;
> }
>
> -#ifdef CONFIG_CXL_BIMODAL
> -
> -struct cxl_switch_work {
> - struct pci_dev *dev;
> - struct work_struct work;
> - int vsec;
> - int mode;
> -};
> -
> -static void switch_card_to_cxl(struct work_struct *work)
> +/* pciex node: ibm,opal-m64-window = <0x3d058 0x0 0x3d058 0x0 0x8 0x0>; */
> +static int switch_card_to_cxl(struct pci_dev *dev)
> {
> - struct cxl_switch_work *switch_work =
> - container_of(work, struct cxl_switch_work, work);
> - struct pci_dev *dev = switch_work->dev;
> - struct pci_bus *bus = dev->bus;
> - struct pci_controller *hose = pci_bus_to_host(bus);
> - struct pci_dev *bridge;
> - struct pnv_php_slot *php_slot;
> - unsigned int devfn;
> + int vsec;
> u8 val;
> int rc;
>
> - dev_info(&bus->dev, "cxl: Preparing for mode switch...\n");
> - bridge = list_first_entry_or_null(&hose->bus->devices, struct pci_dev,
> - bus_list);
> - if (!bridge) {
> - dev_WARN(&bus->dev, "cxl: Couldn't find root port!\n");
> - goto err_dev_put;
> - }
> + dev_info(&dev->dev, "switch card to CXL\n");
>
> - php_slot = pnv_php_find_slot(pci_device_to_OF_node(bridge));
> - if (!php_slot) {
> - dev_err(&bus->dev, "cxl: Failed to find slot hotplug "
> - "information. You may need to upgrade "
> - "skiboot. Aborting.\n");
> - goto err_dev_put;
> - }
> -
> - rc = CXL_READ_VSEC_MODE_CONTROL(dev, switch_work->vsec, &val);
> - if (rc) {
> - dev_err(&bus->dev, "cxl: Failed to read CAPI mode control: %i\n", rc);
> - goto err_dev_put;
> - }
> - devfn = dev->devfn;
> -
> - /* Release the reference obtained in cxl_check_and_switch_mode() */
> - pci_dev_put(dev);
> -
> - dev_dbg(&bus->dev, "cxl: Removing PCI devices from kernel\n");
> - pci_lock_rescan_remove();
> - pci_hp_remove_devices(bridge->subordinate);
> - pci_unlock_rescan_remove();
> -
> - /* Switch the CXL protocol on the card */
> - if (switch_work->mode == CXL_BIMODE_CXL) {
> - dev_info(&bus->dev, "cxl: Switching card to CXL mode\n");
> - val &= ~CXL_VSEC_PROTOCOL_MASK;
> - val |= CXL_VSEC_PROTOCOL_256TB | CXL_VSEC_PROTOCOL_ENABLE;
> - rc = pnv_cxl_enable_phb_kernel_api(hose, true);
> - if (rc) {
> - dev_err(&bus->dev, "cxl: Failed to enable kernel API"
> - " on real PHB, aborting\n");
> - goto err_free_work;
> - }
> - } else {
> - dev_WARN(&bus->dev, "cxl: Switching card to PCI mode not supported!\n");
> - goto err_free_work;
> - }
> -
> - rc = CXL_WRITE_VSEC_MODE_CONTROL_BUS(bus, devfn, switch_work->vsec, val);
> - if (rc) {
> - dev_err(&bus->dev, "cxl: Failed to configure CXL protocol: %i\n", rc);
> - goto err_free_work;
> - }
> -
> - /*
> - * The CAIA spec (v1.1, Section 10.6 Bi-modal Device Support) states
> - * we must wait 100ms after this mode switch before touching PCIe config
> - * space.
> - */
> - msleep(100);
> -
> - /*
> - * Hot reset to cause the card to come back in cxl mode. A
> - * OPAL_RESET_PCI_LINK would be sufficient, but currently lacks support
> - * in skiboot, so we use a hot reset instead.
> - *
> - * We call pci_set_pcie_reset_state() on the bridge, as a CAPI card is
> - * guaranteed to sit directly under the root port, and setting the reset
> - * state on a device directly under the root port is equivalent to doing
> - * it on the root port iself.
> - */
> - dev_info(&bus->dev, "cxl: Configuration write complete, resetting card\n");
> - pci_set_pcie_reset_state(bridge, pcie_hot_reset);
> - pci_set_pcie_reset_state(bridge, pcie_deassert_reset);
> -
> - dev_dbg(&bus->dev, "cxl: Offlining slot\n");
> - rc = pnv_php_set_slot_power_state(&php_slot->slot, OPAL_PCI_SLOT_OFFLINE);
> - if (rc) {
> - dev_err(&bus->dev, "cxl: OPAL offlining call failed: %i\n", rc);
> - goto err_free_work;
> - }
> -
> - dev_dbg(&bus->dev, "cxl: Onlining and probing slot\n");
> - rc = pnv_php_set_slot_power_state(&php_slot->slot, OPAL_PCI_SLOT_ONLINE);
> - if (rc) {
> - dev_err(&bus->dev, "cxl: OPAL onlining call failed: %i\n", rc);
> - goto err_free_work;
> - }
> -
> - pci_lock_rescan_remove();
> - pci_hp_add_devices(bridge->subordinate);
> - pci_unlock_rescan_remove();
> -
> - dev_info(&bus->dev, "cxl: CAPI mode switch completed\n");
> - kfree(switch_work);
> - return;
> -
> -err_dev_put:
> - /* Release the reference obtained in cxl_check_and_switch_mode() */
> - pci_dev_put(dev);
> -err_free_work:
> - kfree(switch_work);
> -}
> -
> -int cxl_check_and_switch_mode(struct pci_dev *dev, int mode, int vsec)
> -{
> - struct cxl_switch_work *work;
> - u8 val;
> - int rc;
> -
> - if (!cpu_has_feature(CPU_FTR_HVMODE))
> + if (!(vsec = find_cxl_vsec(dev))) {
> + dev_err(&dev->dev, "ABORTING: CXL VSEC not found!\n");
> return -ENODEV;
> -
> - if (!vsec) {
> - vsec = find_cxl_vsec(dev);
> - if (!vsec) {
> - dev_info(&dev->dev, "CXL VSEC not found\n");
> - return -ENODEV;
> - }
> }
>
> - rc = CXL_READ_VSEC_MODE_CONTROL(dev, vsec, &val);
> - if (rc) {
> - dev_err(&dev->dev, "Failed to read current mode control: %i", rc);
> + if ((rc = CXL_READ_VSEC_MODE_CONTROL(dev, vsec, &val))) {
> + dev_err(&dev->dev, "failed to read current mode control: %i", rc);
> return rc;
> }
> -
> - if (mode == CXL_BIMODE_PCI) {
> - if (!(val & CXL_VSEC_PROTOCOL_ENABLE)) {
> - dev_info(&dev->dev, "Card is already in PCI mode\n");
> - return 0;
> - }
> - /*
> - * TODO: Before it's safe to switch the card back to PCI mode
> - * we need to disable the CAPP and make sure any cachelines the
> - * card holds have been flushed out. Needs skiboot support.
> - */
> - dev_WARN(&dev->dev, "CXL mode switch to PCI unsupported!\n");
> - return -EIO;
> - }
> -
> - if (val & CXL_VSEC_PROTOCOL_ENABLE) {
> - dev_info(&dev->dev, "Card is already in CXL mode\n");
> - return 0;
> + val &= ~CXL_VSEC_PROTOCOL_MASK;
> + val |= CXL_VSEC_PROTOCOL_256TB | CXL_VSEC_PROTOCOL_ENABLE;
> + if ((rc = CXL_WRITE_VSEC_MODE_CONTROL(dev, vsec, val))) {
> + dev_err(&dev->dev, "failed to enable CXL protocol: %i", rc);
> + return rc;
> }
> -
> - dev_info(&dev->dev, "Card is in PCI mode, scheduling kernel thread "
> - "to switch to CXL mode\n");
> -
> - work = kmalloc(sizeof(struct cxl_switch_work), GFP_KERNEL);
> - if (!work)
> - return -ENOMEM;
> -
> - pci_dev_get(dev);
> - work->dev = dev;
> - work->vsec = vsec;
> - work->mode = mode;
> - INIT_WORK(&work->work, switch_card_to_cxl);
> -
> - schedule_work(&work->work);
> -
> /*
> - * We return a failure now to abort the driver init. Once the
> - * link has been cycled and the card is in cxl mode we will
> - * come back (possibly using the generic cxl driver), but
> - * return success as the card should then be in cxl mode.
> - *
> - * TODO: What if the card comes back in PCI mode even after
> - * the switch? Don't want to spin endlessly.
> + * The CAIA spec (v0.12 11.6 Bi-modal Device Support) states
> + * we must wait 100ms after this mode switch before touching
> + * PCIe config space.
> */
> - return -EBUSY;
> -}
> -EXPORT_SYMBOL_GPL(cxl_check_and_switch_mode);
> -
> -#endif /* CONFIG_CXL_BIMODAL */
> -
> -static int setup_cxl_protocol_area(struct pci_dev *dev)
> -{
> - u8 val;
> - int rc;
> - int vsec = find_cxl_vsec(dev);
> -
> - if (!vsec) {
> - dev_info(&dev->dev, "CXL VSEC not found\n");
> - return -ENODEV;
> - }
> -
> - rc = CXL_READ_VSEC_MODE_CONTROL(dev, vsec, &val);
> - if (rc) {
> - dev_err(&dev->dev, "Failed to read current mode control: %i\n", rc);
> - return rc;
> - }
> -
> - if (!(val & CXL_VSEC_PROTOCOL_ENABLE)) {
> - dev_err(&dev->dev, "Card not in CAPI mode!\n");
> - return -EIO;
> - }
> -
> - if ((val & CXL_VSEC_PROTOCOL_MASK) != CXL_VSEC_PROTOCOL_256TB) {
> - val &= ~CXL_VSEC_PROTOCOL_MASK;
> - val |= CXL_VSEC_PROTOCOL_256TB;
> - rc = CXL_WRITE_VSEC_MODE_CONTROL(dev, vsec, val);
> - if (rc) {
> - dev_err(&dev->dev, "Failed to set CXL protocol area: %i\n", rc);
> - return rc;
> - }
> - }
> + msleep(100);
>
> return 0;
> }
> @@ -1724,7 +1524,7 @@ static int cxl_configure_adapter(struct cxl *adapter, struct pci_dev *dev)
> if ((rc = setup_cxl_bars(dev)))
> return rc;
>
> - if ((rc = setup_cxl_protocol_area(dev)))
> + if ((rc = switch_card_to_cxl(dev)))
> return rc;
>
> if ((rc = cxl_update_image_control(adapter)))
> diff --git a/include/misc/cxl.h b/include/misc/cxl.h
> index 6a3711a2e217..74da2e440763 100644
> --- a/include/misc/cxl.h
> +++ b/include/misc/cxl.h
> @@ -39,31 +39,6 @@
> bool cxl_slot_is_supported(struct pci_dev *dev, int flags);
>
>
> -#define CXL_BIMODE_CXL 1
> -#define CXL_BIMODE_PCI 2
> -
> -/*
> - * Check the mode that the given bi-modal CXL adapter is currently in and
> - * change it if necessary. This does not apply to AFU drivers.
> - *
> - * If the mode matches the requested mode this function will return 0 - if the
> - * driver was expecting the generic CXL driver to have bound to the adapter and
> - * it gets this return value it should fail the probe function to give the CXL
> - * driver a chance to probe it.
> - *
> - * If the mode does not match it will start a background task to unplug the
> - * device from Linux and switch its mode, and will return -EBUSY. At this
> - * point the calling driver should make sure it has released the device and
> - * fail its probe function.
> - *
> - * The offset of the CXL VSEC can be provided to this function. If 0 is passed,
> - * this function will search for a CXL VSEC with ID 0x1280 and return -ENODEV
> - * if it is not found.
> - */
> -#ifdef CONFIG_CXL_BIMODAL
> -int cxl_check_and_switch_mode(struct pci_dev *dev, int mode, int vsec);
> -#endif
> -
> /* Get the AFU associated with a pci_dev */
> struct cxl_afu *cxl_pci_to_afu(struct pci_dev *dev);
>
>
--
Andrew Donnellan OzLabs, ADL Canberra
andrew.donnellan@au1.ibm.com IBM Australia Limited
^ permalink raw reply
* Re: [PATCH v2 06/10] Revert "cxl: Add support for using the kernel API with a real PHB"
From: Andrew Donnellan @ 2018-06-28 23:50 UTC (permalink / raw)
To: Frederic Barrat, alastair, vaibhav, clombard, felix, linuxppc-dev; +Cc: huyn
In-Reply-To: <20180628100509.17413-7-fbarrat@linux.ibm.com>
On 28/06/18 20:05, Frederic Barrat wrote:
> From: Alastair D'Silva <alastair@d-silva.org>
>
> Remove abandonned capi support for the Mellanox CX4.
>
> This reverts commit 317f5ef1b363417b6f1e93b90dfd2ffd6be6e867.
>
> Signed-off-by: Alastair D'Silva <alastair@d-silva.org>
Acked-by: Andrew Donnellan <andrew.donnellan@au1.ibm.com>
> ---
> drivers/misc/cxl/pci.c | 3 ---
> drivers/misc/cxl/vphb.c | 16 ++--------------
> 2 files changed, 2 insertions(+), 17 deletions(-)
>
> diff --git a/drivers/misc/cxl/pci.c b/drivers/misc/cxl/pci.c
> index 9c5a21fee835..193ff22f610b 100644
> --- a/drivers/misc/cxl/pci.c
> +++ b/drivers/misc/cxl/pci.c
> @@ -1886,9 +1886,6 @@ static int cxl_probe(struct pci_dev *dev, const struct pci_device_id *id)
> dev_err(&dev->dev, "AFU %i failed to start: %i\n", slice, rc);
> }
>
> - if (pnv_pci_on_cxl_phb(dev) && adapter->slices >= 1)
> - pnv_cxl_phb_set_peer_afu(dev, adapter->afu[0]);
> -
> return 0;
> }
>
> diff --git a/drivers/misc/cxl/vphb.c b/drivers/misc/cxl/vphb.c
> index 7fd0bdc1436a..1a99c9c7a6fb 100644
> --- a/drivers/misc/cxl/vphb.c
> +++ b/drivers/misc/cxl/vphb.c
> @@ -9,7 +9,6 @@
>
> #include <linux/pci.h>
> #include <misc/cxl.h>
> -#include <asm/pnv-pci.h>
> #include "cxl.h"
>
> static int cxl_dma_set_mask(struct pci_dev *pdev, u64 dma_mask)
> @@ -284,18 +283,13 @@ void cxl_pci_vphb_remove(struct cxl_afu *afu)
> */
> }
>
> -static bool _cxl_pci_is_vphb_device(struct pci_controller *phb)
> -{
> - return (phb->ops == &cxl_pcie_pci_ops);
> -}
> -
> bool cxl_pci_is_vphb_device(struct pci_dev *dev)
> {
> struct pci_controller *phb;
>
> phb = pci_bus_to_host(dev->bus);
>
> - return _cxl_pci_is_vphb_device(phb);
> + return (phb->ops == &cxl_pcie_pci_ops);
> }
>
> struct cxl_afu *cxl_pci_to_afu(struct pci_dev *dev)
> @@ -304,13 +298,7 @@ struct cxl_afu *cxl_pci_to_afu(struct pci_dev *dev)
>
> phb = pci_bus_to_host(dev->bus);
>
> - if (_cxl_pci_is_vphb_device(phb))
> - return (struct cxl_afu *)phb->private_data;
> -
> - if (pnv_pci_on_cxl_phb(dev))
> - return pnv_cxl_phb_to_afu(phb);
> -
> - return ERR_PTR(-ENODEV);
> + return (struct cxl_afu *)phb->private_data;
> }
> EXPORT_SYMBOL_GPL(cxl_pci_to_afu);
>
>
--
Andrew Donnellan OzLabs, ADL Canberra
andrew.donnellan@au1.ibm.com IBM Australia Limited
^ permalink raw reply
* Re: [PATCH v2 07/10] Revert "powerpc/powernv: Add support for the cxl kernel api on the real phb"
From: Andrew Donnellan @ 2018-06-28 23:50 UTC (permalink / raw)
To: Frederic Barrat, alastair, vaibhav, clombard, felix, linuxppc-dev; +Cc: huyn
In-Reply-To: <20180628100509.17413-8-fbarrat@linux.ibm.com>
On 28/06/18 20:05, Frederic Barrat wrote:
> From: Alastair D'Silva <alastair@d-silva.org>
>
> Remove abandonned capi support for the Mellanox CX4.
>
> This reverts commit 4361b03430d685610e5feea3ec7846e8b9ae795f.
>
> Signed-off-by: Alastair D'Silva <alastair@d-silva.org>
Acked-by: Andrew Donnellan <andrew.donnellan@au1.ibm.com>
> ---
> arch/powerpc/include/asm/pnv-pci.h | 7 --
> arch/powerpc/platforms/powernv/pci-cxl.c | 115 ----------------------
> arch/powerpc/platforms/powernv/pci-ioda.c | 18 +---
> arch/powerpc/platforms/powernv/pci.h | 13 ---
> 4 files changed, 1 insertion(+), 152 deletions(-)
>
> diff --git a/arch/powerpc/include/asm/pnv-pci.h b/arch/powerpc/include/asm/pnv-pci.h
> index d2d8c28db336..7f627e3f4da4 100644
> --- a/arch/powerpc/include/asm/pnv-pci.h
> +++ b/arch/powerpc/include/asm/pnv-pci.h
> @@ -50,13 +50,6 @@ int pnv_cxl_alloc_hwirq_ranges(struct cxl_irq_ranges *irqs,
> struct pci_dev *dev, int num);
> void pnv_cxl_release_hwirq_ranges(struct cxl_irq_ranges *irqs,
> struct pci_dev *dev);
> -
> -/* Support for the cxl kernel api on the real PHB (instead of vPHB) */
> -int pnv_cxl_enable_phb_kernel_api(struct pci_controller *hose, bool enable);
> -bool pnv_pci_on_cxl_phb(struct pci_dev *dev);
> -struct cxl_afu *pnv_cxl_phb_to_afu(struct pci_controller *hose);
> -void pnv_cxl_phb_set_peer_afu(struct pci_dev *dev, struct cxl_afu *afu);
> -
> #endif
>
> struct pnv_php_slot {
> diff --git a/arch/powerpc/platforms/powernv/pci-cxl.c b/arch/powerpc/platforms/powernv/pci-cxl.c
> index c447b7f03c09..1b18111453d7 100644
> --- a/arch/powerpc/platforms/powernv/pci-cxl.c
> +++ b/arch/powerpc/platforms/powernv/pci-cxl.c
> @@ -8,10 +8,8 @@
> */
>
> #include <linux/module.h>
> -#include <asm/pci-bridge.h>
> #include <asm/pnv-pci.h>
> #include <asm/opal.h>
> -#include <misc/cxl.h>
>
> #include "pci.h"
>
> @@ -178,116 +176,3 @@ static inline int get_cxl_module(void)
> #else
> static inline int get_cxl_module(void) { return 0; }
> #endif
> -
> -/*
> - * Sets flags and switches the controller ops to enable the cxl kernel api.
> - * Originally the cxl kernel API operated on a virtual PHB, but certain cards
> - * such as the Mellanox CX4 use a peer model instead and for these cards the
> - * cxl kernel api will operate on the real PHB.
> - */
> -int pnv_cxl_enable_phb_kernel_api(struct pci_controller *hose, bool enable)
> -{
> - struct pnv_phb *phb = hose->private_data;
> - int rc;
> -
> - if (!enable) {
> - /*
> - * Once cxl mode is enabled on the PHB, there is currently no
> - * known safe method to disable it again, and trying risks a
> - * checkstop. If we can find a way to safely disable cxl mode
> - * in the future we can revisit this, but for now the only sane
> - * thing to do is to refuse to disable cxl mode:
> - */
> - return -EPERM;
> - }
> -
> - /*
> - * Hold a reference to the cxl module since several PHB operations now
> - * depend on it, and it would be insane to allow it to be removed so
> - * long as we are in this mode (and since we can't safely disable this
> - * mode once enabled...).
> - */
> - rc = get_cxl_module();
> - if (rc)
> - return rc;
> -
> - phb->flags |= PNV_PHB_FLAG_CXL;
> - hose->controller_ops = pnv_cxl_cx4_ioda_controller_ops;
> -
> - return 0;
> -}
> -EXPORT_SYMBOL_GPL(pnv_cxl_enable_phb_kernel_api);
> -
> -bool pnv_pci_on_cxl_phb(struct pci_dev *dev)
> -{
> - struct pci_controller *hose = pci_bus_to_host(dev->bus);
> - struct pnv_phb *phb = hose->private_data;
> -
> - return !!(phb->flags & PNV_PHB_FLAG_CXL);
> -}
> -EXPORT_SYMBOL_GPL(pnv_pci_on_cxl_phb);
> -
> -struct cxl_afu *pnv_cxl_phb_to_afu(struct pci_controller *hose)
> -{
> - struct pnv_phb *phb = hose->private_data;
> -
> - return (struct cxl_afu *)phb->cxl_afu;
> -}
> -EXPORT_SYMBOL_GPL(pnv_cxl_phb_to_afu);
> -
> -void pnv_cxl_phb_set_peer_afu(struct pci_dev *dev, struct cxl_afu *afu)
> -{
> - struct pci_controller *hose = pci_bus_to_host(dev->bus);
> - struct pnv_phb *phb = hose->private_data;
> -
> - phb->cxl_afu = afu;
> -}
> -EXPORT_SYMBOL_GPL(pnv_cxl_phb_set_peer_afu);
> -
> -/*
> - * In the peer cxl model, the XSL/PSL is physical function 0, and will be used
> - * by other functions on the device for memory access and interrupts. When the
> - * other functions are enabled we explicitly take a reference on the cxl
> - * function since they will use it, and allocate a default context associated
> - * with that function just like the vPHB model of the cxl kernel API.
> - */
> -bool pnv_cxl_enable_device_hook(struct pci_dev *dev)
> -{
> - struct pci_controller *hose = pci_bus_to_host(dev->bus);
> - struct pnv_phb *phb = hose->private_data;
> - struct cxl_afu *afu = phb->cxl_afu;
> -
> - if (!pnv_pci_enable_device_hook(dev))
> - return false;
> -
> -
> - /* No special handling for the cxl function, which is always PF 0 */
> - if (PCI_FUNC(dev->devfn) == 0)
> - return true;
> -
> - if (!afu) {
> - dev_WARN(&dev->dev, "Attempted to enable function > 0 on CXL PHB without a peer AFU\n");
> - return false;
> - }
> -
> - dev_info(&dev->dev, "Enabling function on CXL enabled PHB with peer AFU\n");
> -
> - /* Make sure the peer AFU can't go away while this device is active */
> - cxl_afu_get(afu);
> -
> - return cxl_pci_associate_default_context(dev, afu);
> -}
> -
> -void pnv_cxl_disable_device(struct pci_dev *dev)
> -{
> - struct pci_controller *hose = pci_bus_to_host(dev->bus);
> - struct pnv_phb *phb = hose->private_data;
> - struct cxl_afu *afu = phb->cxl_afu;
> -
> - /* No special handling for cxl function: */
> - if (PCI_FUNC(dev->devfn) == 0)
> - return;
> -
> - cxl_pci_disable_device(dev);
> - cxl_afu_put(afu);
> -}
> diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
> index 41f8f0ff4a55..770c67c4e8f7 100644
> --- a/arch/powerpc/platforms/powernv/pci-ioda.c
> +++ b/arch/powerpc/platforms/powernv/pci-ioda.c
> @@ -3575,7 +3575,7 @@ static resource_size_t pnv_pci_iov_resource_alignment(struct pci_dev *pdev,
> /* Prevent enabling devices for which we couldn't properly
> * assign a PE
> */
> -bool pnv_pci_enable_device_hook(struct pci_dev *dev)
> +static bool pnv_pci_enable_device_hook(struct pci_dev *dev)
> {
> struct pci_controller *hose = pci_bus_to_host(dev->bus);
> struct pnv_phb *phb = hose->private_data;
> @@ -3843,22 +3843,6 @@ static const struct pci_controller_ops pnv_npu_ocapi_ioda_controller_ops = {
> .shutdown = pnv_pci_ioda_shutdown,
> };
>
> -#ifdef CONFIG_CXL_BASE
> -const struct pci_controller_ops pnv_cxl_cx4_ioda_controller_ops = {
> - .dma_dev_setup = pnv_pci_dma_dev_setup,
> - .dma_bus_setup = pnv_pci_dma_bus_setup,
> - .enable_device_hook = pnv_cxl_enable_device_hook,
> - .disable_device = pnv_cxl_disable_device,
> - .release_device = pnv_pci_release_device,
> - .window_alignment = pnv_pci_window_alignment,
> - .setup_bridge = pnv_pci_setup_bridge,
> - .reset_secondary_bus = pnv_pci_reset_secondary_bus,
> - .dma_set_mask = pnv_pci_ioda_dma_set_mask,
> - .dma_get_required_mask = pnv_pci_ioda_dma_get_required_mask,
> - .shutdown = pnv_pci_ioda_shutdown,
> -};
> -#endif
> -
> static void __init pnv_pci_init_ioda_phb(struct device_node *np,
> u64 hub_id, int ioda_type)
> {
> diff --git a/arch/powerpc/platforms/powernv/pci.h b/arch/powerpc/platforms/powernv/pci.h
> index ba41913c7e21..44dfbc37f547 100644
> --- a/arch/powerpc/platforms/powernv/pci.h
> +++ b/arch/powerpc/platforms/powernv/pci.h
> @@ -88,7 +88,6 @@ struct pnv_ioda_pe {
> };
>
> #define PNV_PHB_FLAG_EEH (1 << 0)
> -#define PNV_PHB_FLAG_CXL (1 << 1) /* Real PHB supporting the cxl kernel API */
>
> struct pnv_phb {
> struct pci_controller *hose;
> @@ -194,9 +193,6 @@ struct pnv_phb {
> bool nmmu_flush;
> } npu;
>
> -#ifdef CONFIG_CXL_BASE
> - struct cxl_afu *cxl_afu;
> -#endif
> int p2p_target_count;
> };
>
> @@ -238,7 +234,6 @@ extern int pnv_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type);
> extern void pnv_teardown_msi_irqs(struct pci_dev *pdev);
> extern struct pnv_ioda_pe *pnv_ioda_get_pe(struct pci_dev *dev);
> extern void pnv_set_msi_irq_chip(struct pnv_phb *phb, unsigned int virq);
> -extern bool pnv_pci_enable_device_hook(struct pci_dev *dev);
> extern void pnv_pci_ioda2_set_bypass(struct pnv_ioda_pe *pe, bool enable);
> extern int pnv_eeh_post_init(void);
>
> @@ -262,12 +257,4 @@ extern void pnv_npu_take_ownership(struct pnv_ioda_pe *npe);
> extern void pnv_npu_release_ownership(struct pnv_ioda_pe *npe);
> extern int pnv_npu2_init(struct pnv_phb *phb);
>
> -/* cxl functions */
> -extern bool pnv_cxl_enable_device_hook(struct pci_dev *dev);
> -extern void pnv_cxl_disable_device(struct pci_dev *dev);
> -
> -
> -/* phb ops (cxl switches these when enabling the kernel api on the phb) */
> -extern const struct pci_controller_ops pnv_cxl_cx4_ioda_controller_ops;
> -
> #endif /* __POWERNV_PCI_H */
>
--
Andrew Donnellan OzLabs, ADL Canberra
andrew.donnellan@au1.ibm.com IBM Australia Limited
^ permalink raw reply
* Re: [PATCH v2 08/10] Revert "cxl: Add cxl_slot_is_supported API"
From: Andrew Donnellan @ 2018-06-28 23:50 UTC (permalink / raw)
To: Frederic Barrat, alastair, vaibhav, clombard, felix, linuxppc-dev; +Cc: huyn
In-Reply-To: <20180628100509.17413-9-fbarrat@linux.ibm.com>
On 28/06/18 20:05, Frederic Barrat wrote:
> Remove abandonned capi support for the Mellanox CX4.
>
> This reverts commit 4e56f858bdde5cbfb70f61baddfaa56a8ed851bf.
>
> Signed-off-by: Frederic Barrat <fbarrat@linux.ibm.com>
Acked-by: Andrew Donnellan <andrew.donnellan@au1.ibm.com>
> ---
> drivers/misc/cxl/pci.c | 37 -------------------------------------
> include/misc/cxl.h | 15 ---------------
> 2 files changed, 52 deletions(-)
>
> diff --git a/drivers/misc/cxl/pci.c b/drivers/misc/cxl/pci.c
> index 193ff22f610b..0ca818396524 100644
> --- a/drivers/misc/cxl/pci.c
> +++ b/drivers/misc/cxl/pci.c
> @@ -1808,43 +1808,6 @@ int cxl_slot_is_switched(struct pci_dev *dev)
> return (depth > CXL_MAX_PCIEX_PARENT);
> }
>
> -bool cxl_slot_is_supported(struct pci_dev *dev, int flags)
> -{
> - if (!cpu_has_feature(CPU_FTR_HVMODE))
> - return false;
> -
> - if ((flags & CXL_SLOT_FLAG_DMA) && (!pvr_version_is(PVR_POWER8NVL))) {
> - /*
> - * CAPP DMA mode is technically supported on regular P8, but
> - * will EEH if the card attempts to access memory < 4GB, which
> - * we cannot realistically avoid. We might be able to work
> - * around the issue, but until then return unsupported:
> - */
> - return false;
> - }
> -
> - if (cxl_slot_is_switched(dev))
> - return false;
> -
> - /*
> - * XXX: This gets a little tricky on regular P8 (not POWER8NVL) since
> - * the CAPP can be connected to PHB 0, 1 or 2 on a first come first
> - * served basis, which is racy to check from here. If we need to
> - * support this in future we might need to consider having this
> - * function effectively reserve it ahead of time.
> - *
> - * Currently, the only user of this API is the Mellanox CX4, which is
> - * only supported on P8NVL due to the above mentioned limitation of
> - * CAPP DMA mode and therefore does not need to worry about this. If the
> - * issue with CAPP DMA mode is later worked around on P8 we might need
> - * to revisit this.
> - */
> -
> - return true;
> -}
> -EXPORT_SYMBOL_GPL(cxl_slot_is_supported);
> -
> -
> static int cxl_probe(struct pci_dev *dev, const struct pci_device_id *id)
> {
> struct cxl *adapter;
> diff --git a/include/misc/cxl.h b/include/misc/cxl.h
> index 74da2e440763..ea9ff4a1a9ca 100644
> --- a/include/misc/cxl.h
> +++ b/include/misc/cxl.h
> @@ -24,21 +24,6 @@
> * generic PCI API. This API is agnostic to the actual AFU.
> */
>
> -#define CXL_SLOT_FLAG_DMA 0x1
> -
> -/*
> - * Checks if the given card is in a cxl capable slot. Pass CXL_SLOT_FLAG_DMA if
> - * the card requires CAPP DMA mode to also check if the system supports it.
> - * This is intended to be used by bi-modal devices to determine if they can use
> - * cxl mode or if they should continue running in PCI mode.
> - *
> - * Note that this only checks if the slot is cxl capable - it does not
> - * currently check if the CAPP is currently available for chips where it can be
> - * assigned to different PHBs on a first come first serve basis (i.e. P8)
> - */
> -bool cxl_slot_is_supported(struct pci_dev *dev, int flags);
> -
> -
> /* Get the AFU associated with a pci_dev */
> struct cxl_afu *cxl_pci_to_afu(struct pci_dev *dev);
>
>
--
Andrew Donnellan OzLabs, ADL Canberra
andrew.donnellan@au1.ibm.com IBM Australia Limited
^ permalink raw reply
* Re: [PATCH v2 09/10] Revert "cxl: Allow a default context to be associated with an external pci_dev"
From: Andrew Donnellan @ 2018-06-28 23:51 UTC (permalink / raw)
To: Frederic Barrat, alastair, vaibhav, clombard, felix, linuxppc-dev; +Cc: huyn
In-Reply-To: <20180628100509.17413-10-fbarrat@linux.ibm.com>
On 28/06/18 20:05, Frederic Barrat wrote:
> Remove abandonned capi support for the Mellanox CX4.
>
> This reverts commit a19bd79e31769626d288cc016e21a31b6f47bf6f.
>
> Signed-off-by: Frederic Barrat <fbarrat@linux.ibm.com>
Acked-by: Andrew Donnellan <andrew.donnellan@au1.ibm.com>
> ---
> drivers/misc/cxl/Makefile | 2 +-
> drivers/misc/cxl/base.c | 35 -------------------------------
> drivers/misc/cxl/cxl.h | 6 ------
> drivers/misc/cxl/main.c | 2 --
> drivers/misc/cxl/phb.c | 44 ---------------------------------------
> drivers/misc/cxl/vphb.c | 30 +++++++++++++++++++++++---
> include/misc/cxl-base.h | 6 ------
> 7 files changed, 28 insertions(+), 97 deletions(-)
> delete mode 100644 drivers/misc/cxl/phb.c
>
> diff --git a/drivers/misc/cxl/Makefile b/drivers/misc/cxl/Makefile
> index 502d41fc9ea5..5eea61b9584f 100644
> --- a/drivers/misc/cxl/Makefile
> +++ b/drivers/misc/cxl/Makefile
> @@ -4,7 +4,7 @@ ccflags-$(CONFIG_PPC_WERROR) += -Werror
>
> cxl-y += main.o file.o irq.o fault.o native.o
> cxl-y += context.o sysfs.o pci.o trace.o
> -cxl-y += vphb.o phb.o api.o cxllib.o
> +cxl-y += vphb.o api.o cxllib.o
> cxl-$(CONFIG_PPC_PSERIES) += flash.o guest.o of.o hcalls.o
> cxl-$(CONFIG_DEBUG_FS) += debugfs.o
> obj-$(CONFIG_CXL) += cxl.o
> diff --git a/drivers/misc/cxl/base.c b/drivers/misc/cxl/base.c
> index e1e80cb99ad9..7557835cdfcd 100644
> --- a/drivers/misc/cxl/base.c
> +++ b/drivers/misc/cxl/base.c
> @@ -106,41 +106,6 @@ int cxl_update_properties(struct device_node *dn,
> }
> EXPORT_SYMBOL_GPL(cxl_update_properties);
>
> -/*
> - * API calls into the driver that may be called from the PHB code and must be
> - * built in.
> - */
> -bool cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu)
> -{
> - bool ret;
> - struct cxl_calls *calls;
> -
> - calls = cxl_calls_get();
> - if (!calls)
> - return false;
> -
> - ret = calls->cxl_pci_associate_default_context(dev, afu);
> -
> - cxl_calls_put(calls);
> -
> - return ret;
> -}
> -EXPORT_SYMBOL_GPL(cxl_pci_associate_default_context);
> -
> -void cxl_pci_disable_device(struct pci_dev *dev)
> -{
> - struct cxl_calls *calls;
> -
> - calls = cxl_calls_get();
> - if (!calls)
> - return;
> -
> - calls->cxl_pci_disable_device(dev);
> -
> - cxl_calls_put(calls);
> -}
> -EXPORT_SYMBOL_GPL(cxl_pci_disable_device);
> -
> static int __init cxl_base_init(void)
> {
> struct device_node *np;
> diff --git a/drivers/misc/cxl/cxl.h b/drivers/misc/cxl/cxl.h
> index d95c2c98f2ab..aa453448201d 100644
> --- a/drivers/misc/cxl/cxl.h
> +++ b/drivers/misc/cxl/cxl.h
> @@ -867,15 +867,9 @@ static inline bool cxl_is_power9_dd1(void)
> ssize_t cxl_pci_afu_read_err_buffer(struct cxl_afu *afu, char *buf,
> loff_t off, size_t count);
>
> -/* Internal functions wrapped in cxl_base to allow PHB to call them */
> -bool _cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu);
> -void _cxl_pci_disable_device(struct pci_dev *dev);
>
> struct cxl_calls {
> void (*cxl_slbia)(struct mm_struct *mm);
> - bool (*cxl_pci_associate_default_context)(struct pci_dev *dev, struct cxl_afu *afu);
> - void (*cxl_pci_disable_device)(struct pci_dev *dev);
> -
> struct module *owner;
> };
> int register_cxl_calls(struct cxl_calls *calls);
> diff --git a/drivers/misc/cxl/main.c b/drivers/misc/cxl/main.c
> index a7e83624034b..334223b802ee 100644
> --- a/drivers/misc/cxl/main.c
> +++ b/drivers/misc/cxl/main.c
> @@ -104,8 +104,6 @@ static inline void cxl_slbia_core(struct mm_struct *mm)
>
> static struct cxl_calls cxl_calls = {
> .cxl_slbia = cxl_slbia_core,
> - .cxl_pci_associate_default_context = _cxl_pci_associate_default_context,
> - .cxl_pci_disable_device = _cxl_pci_disable_device,
> .owner = THIS_MODULE,
> };
>
> diff --git a/drivers/misc/cxl/phb.c b/drivers/misc/cxl/phb.c
> deleted file mode 100644
> index 6ec69ada19f4..000000000000
> --- a/drivers/misc/cxl/phb.c
> +++ /dev/null
> @@ -1,44 +0,0 @@
> -/*
> - * Copyright 2014-2016 IBM Corp.
> - *
> - * This program is free software; you can redistribute it and/or
> - * modify it under the terms of the GNU General Public License
> - * as published by the Free Software Foundation; either version
> - * 2 of the License, or (at your option) any later version.
> - */
> -
> -#include <linux/pci.h>
> -#include "cxl.h"
> -
> -bool _cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu)
> -{
> - struct cxl_context *ctx;
> -
> - /*
> - * Allocate a context to do cxl things to. This is used for interrupts
> - * in the peer model using a real phb, and if we eventually do DMA ops
> - * in the virtual phb, we'll need a default context to attach them to.
> - */
> - ctx = cxl_dev_context_init(dev);
> - if (IS_ERR(ctx))
> - return false;
> - dev->dev.archdata.cxl_ctx = ctx;
> -
> - return (cxl_ops->afu_check_and_enable(afu) == 0);
> -}
> -/* exported via cxl_base */
> -
> -void _cxl_pci_disable_device(struct pci_dev *dev)
> -{
> - struct cxl_context *ctx = cxl_get_context(dev);
> -
> - if (ctx) {
> - if (ctx->status == STARTED) {
> - dev_err(&dev->dev, "Default context started\n");
> - return;
> - }
> - dev->dev.archdata.cxl_ctx = NULL;
> - cxl_release_context(ctx);
> - }
> -}
> -/* exported via cxl_base */
> diff --git a/drivers/misc/cxl/vphb.c b/drivers/misc/cxl/vphb.c
> index 1a99c9c7a6fb..7908633d9204 100644
> --- a/drivers/misc/cxl/vphb.c
> +++ b/drivers/misc/cxl/vphb.c
> @@ -44,6 +44,7 @@ static bool cxl_pci_enable_device_hook(struct pci_dev *dev)
> {
> struct pci_controller *phb;
> struct cxl_afu *afu;
> + struct cxl_context *ctx;
>
> phb = pci_bus_to_host(dev->bus);
> afu = (struct cxl_afu *)phb->private_data;
> @@ -56,7 +57,30 @@ static bool cxl_pci_enable_device_hook(struct pci_dev *dev)
> set_dma_ops(&dev->dev, &dma_nommu_ops);
> set_dma_offset(&dev->dev, PAGE_OFFSET);
>
> - return _cxl_pci_associate_default_context(dev, afu);
> + /*
> + * Allocate a context to do cxl things too. If we eventually do real
> + * DMA ops, we'll need a default context to attach them to
> + */
> + ctx = cxl_dev_context_init(dev);
> + if (IS_ERR(ctx))
> + return false;
> + dev->dev.archdata.cxl_ctx = ctx;
> +
> + return (cxl_ops->afu_check_and_enable(afu) == 0);
> +}
> +
> +static void cxl_pci_disable_device(struct pci_dev *dev)
> +{
> + struct cxl_context *ctx = cxl_get_context(dev);
> +
> + if (ctx) {
> + if (ctx->status == STARTED) {
> + dev_err(&dev->dev, "Default context started\n");
> + return;
> + }
> + dev->dev.archdata.cxl_ctx = NULL;
> + cxl_release_context(ctx);
> + }
> }
>
> static resource_size_t cxl_pci_window_alignment(struct pci_bus *bus,
> @@ -190,8 +214,8 @@ static struct pci_controller_ops cxl_pci_controller_ops =
> {
> .probe_mode = cxl_pci_probe_mode,
> .enable_device_hook = cxl_pci_enable_device_hook,
> - .disable_device = _cxl_pci_disable_device,
> - .release_device = _cxl_pci_disable_device,
> + .disable_device = cxl_pci_disable_device,
> + .release_device = cxl_pci_disable_device,
> .window_alignment = cxl_pci_window_alignment,
> .reset_secondary_bus = cxl_pci_reset_secondary_bus,
> .setup_msi_irqs = cxl_setup_msi_irqs,
> diff --git a/include/misc/cxl-base.h b/include/misc/cxl-base.h
> index bb7e629ae492..f53808fa638a 100644
> --- a/include/misc/cxl-base.h
> +++ b/include/misc/cxl-base.h
> @@ -10,8 +10,6 @@
> #ifndef _MISC_CXL_BASE_H
> #define _MISC_CXL_BASE_H
>
> -#include <misc/cxl.h>
> -
> #ifdef CONFIG_CXL_BASE
>
> #define CXL_IRQ_RANGES 4
> @@ -41,8 +39,6 @@ static inline void cxl_ctx_put(void)
> struct cxl_afu *cxl_afu_get(struct cxl_afu *afu);
> void cxl_afu_put(struct cxl_afu *afu);
> void cxl_slbia(struct mm_struct *mm);
> -bool cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu);
> -void cxl_pci_disable_device(struct pci_dev *dev);
>
> #else /* CONFIG_CXL_BASE */
>
> @@ -50,8 +46,6 @@ static inline bool cxl_ctx_in_use(void) { return false; }
> static inline struct cxl_afu *cxl_afu_get(struct cxl_afu *afu) { return NULL; }
> static inline void cxl_afu_put(struct cxl_afu *afu) {}
> static inline void cxl_slbia(struct mm_struct *mm) {}
> -static inline bool cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu) { return false; }
> -static inline void cxl_pci_disable_device(struct pci_dev *dev) {}
>
> #endif /* CONFIG_CXL_BASE */
>
>
--
Andrew Donnellan OzLabs, ADL Canberra
andrew.donnellan@au1.ibm.com IBM Australia Limited
^ permalink raw reply
* Re: [PATCH v2 10/10] cxl: Remove abandonned capi support for the Mellanox CX4, final cleanup
From: Andrew Donnellan @ 2018-06-28 23:53 UTC (permalink / raw)
To: Frederic Barrat, alastair, vaibhav, clombard, felix, linuxppc-dev; +Cc: huyn
In-Reply-To: <20180628100509.17413-11-fbarrat@linux.ibm.com>
On 28/06/18 20:05, Frederic Barrat wrote:
> Remove a few XSL/CX4 oddities which are no longer needed. A simple
> revert of the initial commits was not possible (or not worth it) due
> to the history of the code.
>
> Signed-off-by: Frederic Barrat <fbarrat@linux.ibm.com>
Acked-by: Andrew Donnellan <andrew.donnellan@au1.ibm.com>
> ---
> drivers/misc/cxl/context.c | 2 +-
> drivers/misc/cxl/cxl.h | 12 ------
> drivers/misc/cxl/debugfs.c | 5 ---
> drivers/misc/cxl/pci.c | 75 +++-----------------------------------
> 4 files changed, 7 insertions(+), 87 deletions(-)
>
> diff --git a/drivers/misc/cxl/context.c b/drivers/misc/cxl/context.c
> index 0355d42d367f..5fe529b43ebe 100644
> --- a/drivers/misc/cxl/context.c
> +++ b/drivers/misc/cxl/context.c
> @@ -95,7 +95,7 @@ int cxl_context_init(struct cxl_context *ctx, struct cxl_afu *afu, bool master)
> */
> mutex_lock(&afu->contexts_lock);
> idr_preload(GFP_KERNEL);
> - i = idr_alloc(&ctx->afu->contexts_idr, ctx, ctx->afu->adapter->min_pe,
> + i = idr_alloc(&ctx->afu->contexts_idr, ctx, 0,
> ctx->afu->num_procs, GFP_NOWAIT);
> idr_preload_end();
> mutex_unlock(&afu->contexts_lock);
> diff --git a/drivers/misc/cxl/cxl.h b/drivers/misc/cxl/cxl.h
> index aa453448201d..44bcfafbb579 100644
> --- a/drivers/misc/cxl/cxl.h
> +++ b/drivers/misc/cxl/cxl.h
> @@ -93,11 +93,6 @@ static const cxl_p1_reg_t CXL_PSL_FIR_CNTL = {0x0148};
> static const cxl_p1_reg_t CXL_PSL_DSNDCTL = {0x0150};
> static const cxl_p1_reg_t CXL_PSL_SNWRALLOC = {0x0158};
> static const cxl_p1_reg_t CXL_PSL_TRACE = {0x0170};
> -/* XSL registers (Mellanox CX4) */
> -static const cxl_p1_reg_t CXL_XSL_Timebase = {0x0100};
> -static const cxl_p1_reg_t CXL_XSL_TB_CTLSTAT = {0x0108};
> -static const cxl_p1_reg_t CXL_XSL_FEC = {0x0158};
> -static const cxl_p1_reg_t CXL_XSL_DSNCTL = {0x0168};
> /* PSL registers - CAIA 2 */
> static const cxl_p1_reg_t CXL_PSL9_CONTROL = {0x0020};
> static const cxl_p1_reg_t CXL_XSL9_INV = {0x0110};
> @@ -695,7 +690,6 @@ struct cxl {
> struct bin_attribute cxl_attr;
> int adapter_num;
> int user_irqs;
> - int min_pe;
> u64 ps_size;
> u16 psl_rev;
> u16 base_image;
> @@ -934,7 +928,6 @@ int cxl_debugfs_afu_add(struct cxl_afu *afu);
> void cxl_debugfs_afu_remove(struct cxl_afu *afu);
> void cxl_debugfs_add_adapter_regs_psl9(struct cxl *adapter, struct dentry *dir);
> void cxl_debugfs_add_adapter_regs_psl8(struct cxl *adapter, struct dentry *dir);
> -void cxl_debugfs_add_adapter_regs_xsl(struct cxl *adapter, struct dentry *dir);
> void cxl_debugfs_add_afu_regs_psl9(struct cxl_afu *afu, struct dentry *dir);
> void cxl_debugfs_add_afu_regs_psl8(struct cxl_afu *afu, struct dentry *dir);
>
> @@ -977,11 +970,6 @@ static inline void cxl_debugfs_add_adapter_regs_psl8(struct cxl *adapter,
> {
> }
>
> -static inline void cxl_debugfs_add_adapter_regs_xsl(struct cxl *adapter,
> - struct dentry *dir)
> -{
> -}
> -
> static inline void cxl_debugfs_add_afu_regs_psl9(struct cxl_afu *afu, struct dentry *dir)
> {
> }
> diff --git a/drivers/misc/cxl/debugfs.c b/drivers/misc/cxl/debugfs.c
> index 1643850d2302..a1921d81593a 100644
> --- a/drivers/misc/cxl/debugfs.c
> +++ b/drivers/misc/cxl/debugfs.c
> @@ -58,11 +58,6 @@ void cxl_debugfs_add_adapter_regs_psl8(struct cxl *adapter, struct dentry *dir)
> debugfs_create_io_x64("trace", S_IRUSR | S_IWUSR, dir, _cxl_p1_addr(adapter, CXL_PSL_TRACE));
> }
>
> -void cxl_debugfs_add_adapter_regs_xsl(struct cxl *adapter, struct dentry *dir)
> -{
> - debugfs_create_io_x64("fec", S_IRUSR, dir, _cxl_p1_addr(adapter, CXL_XSL_FEC));
> -}
> -
> int cxl_debugfs_adapter_add(struct cxl *adapter)
> {
> struct dentry *dir;
> diff --git a/drivers/misc/cxl/pci.c b/drivers/misc/cxl/pci.c
> index 0ca818396524..6dfb4ed345d3 100644
> --- a/drivers/misc/cxl/pci.c
> +++ b/drivers/misc/cxl/pci.c
> @@ -593,27 +593,7 @@ static int init_implementation_adapter_regs_psl8(struct cxl *adapter, struct pci
> return 0;
> }
>
> -static int init_implementation_adapter_regs_xsl(struct cxl *adapter, struct pci_dev *dev)
> -{
> - u64 xsl_dsnctl;
> - u64 chipid;
> - u32 phb_index;
> - u64 capp_unit_id;
> - int rc;
> -
> - rc = cxl_calc_capp_routing(dev, &chipid, &phb_index, &capp_unit_id);
> - if (rc)
> - return rc;
> -
> - /* Tell XSL where to route data to */
> - xsl_dsnctl = 0x0000600000000000ULL | (chipid << (63-5));
> - xsl_dsnctl |= (capp_unit_id << (63-13));
> - cxl_p1_write(adapter, CXL_XSL_DSNCTL, xsl_dsnctl);
> -
> - return 0;
> -}
> -
> -/* PSL & XSL */
> +/* PSL */
> #define TBSYNC_CAL(n) (((u64)n & 0x7) << (63-3))
> #define TBSYNC_CNT(n) (((u64)n & 0x7) << (63-6))
> /* For the PSL this is a multiple for 0 < n <= 7: */
> @@ -625,21 +605,6 @@ static void write_timebase_ctrl_psl8(struct cxl *adapter)
> TBSYNC_CNT(2 * PSL_2048_250MHZ_CYCLES));
> }
>
> -/* XSL */
> -#define TBSYNC_ENA (1ULL << 63)
> -/* For the XSL this is 2**n * 2000 clocks for 0 < n <= 6: */
> -#define XSL_2000_CLOCKS 1
> -#define XSL_4000_CLOCKS 2
> -#define XSL_8000_CLOCKS 3
> -
> -static void write_timebase_ctrl_xsl(struct cxl *adapter)
> -{
> - cxl_p1_write(adapter, CXL_XSL_TB_CTLSTAT,
> - TBSYNC_ENA |
> - TBSYNC_CAL(3) |
> - TBSYNC_CNT(XSL_4000_CLOCKS));
> -}
> -
> static u64 timebase_read_psl9(struct cxl *adapter)
> {
> return cxl_p1_read(adapter, CXL_PSL9_Timebase);
> @@ -650,11 +615,6 @@ static u64 timebase_read_psl8(struct cxl *adapter)
> return cxl_p1_read(adapter, CXL_PSL_Timebase);
> }
>
> -static u64 timebase_read_xsl(struct cxl *adapter)
> -{
> - return cxl_p1_read(adapter, CXL_XSL_Timebase);
> -}
> -
> static void cxl_setup_psl_timebase(struct cxl *adapter, struct pci_dev *dev)
> {
> struct device_node *np;
> @@ -1671,37 +1631,14 @@ static const struct cxl_service_layer_ops psl8_ops = {
> .needs_reset_before_disable = true,
> };
>
> -static const struct cxl_service_layer_ops xsl_ops = {
> - .adapter_regs_init = init_implementation_adapter_regs_xsl,
> - .invalidate_all = cxl_invalidate_all_psl8,
> - .sanitise_afu_regs = sanitise_afu_regs_psl8,
> - .handle_interrupt = cxl_irq_psl8,
> - .fail_irq = cxl_fail_irq_psl,
> - .activate_dedicated_process = cxl_activate_dedicated_process_psl8,
> - .attach_afu_directed = cxl_attach_afu_directed_psl8,
> - .attach_dedicated_process = cxl_attach_dedicated_process_psl8,
> - .update_dedicated_ivtes = cxl_update_dedicated_ivtes_psl8,
> - .debugfs_add_adapter_regs = cxl_debugfs_add_adapter_regs_xsl,
> - .write_timebase_ctrl = write_timebase_ctrl_xsl,
> - .timebase_read = timebase_read_xsl,
> - .capi_mode = OPAL_PHB_CAPI_MODE_DMA,
> -};
> -
> static void set_sl_ops(struct cxl *adapter, struct pci_dev *dev)
> {
> - if (dev->vendor == PCI_VENDOR_ID_MELLANOX && dev->device == 0x1013) {
> - /* Mellanox CX-4 */
> - dev_info(&dev->dev, "Device uses an XSL\n");
> - adapter->native->sl_ops = &xsl_ops;
> - adapter->min_pe = 1; /* Workaround for CX-4 hardware bug */
> + if (cxl_is_power8()) {
> + dev_info(&dev->dev, "Device uses a PSL8\n");
> + adapter->native->sl_ops = &psl8_ops;
> } else {
> - if (cxl_is_power8()) {
> - dev_info(&dev->dev, "Device uses a PSL8\n");
> - adapter->native->sl_ops = &psl8_ops;
> - } else {
> - dev_info(&dev->dev, "Device uses a PSL9\n");
> - adapter->native->sl_ops = &psl9_ops;
> - }
> + dev_info(&dev->dev, "Device uses a PSL9\n");
> + adapter->native->sl_ops = &psl9_ops;
> }
> }
>
>
--
Andrew Donnellan OzLabs, ADL Canberra
andrew.donnellan@au1.ibm.com IBM Australia Limited
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox