* detect and warn about ahci-remapped NVMe devices
@ 2016-12-02 18:31 ` Christoph Hellwig
0 siblings, 0 replies; 14+ messages in thread
From: Christoph Hellwig @ 2016-12-02 18:31 UTC (permalink / raw)
Just a small band-aid to detect the AHCI-remapped NVMe devices
for now. Hopefully we'll eventually get a fake PCIe root port
driver ala VMD for them.
^ permalink raw reply [flat|nested] 14+ messages in thread
* [PATCH 1/3] nvme: move NVMe class code to pci_ids.h
2016-12-02 18:31 ` Christoph Hellwig
@ 2016-12-02 18:31 ` Christoph Hellwig
-1 siblings, 0 replies; 14+ messages in thread
From: Christoph Hellwig @ 2016-12-02 18:31 UTC (permalink / raw)
To: tj; +Cc: dan.j.williams, linux-ide, linux-nvme
We'll need to check for it in the AHCI drivers (yes, really) soon.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
drivers/nvme/host/pci.c | 3 ---
include/linux/pci_ids.h | 2 ++
2 files changed, 2 insertions(+), 3 deletions(-)
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index 0fc99f0..2c58f15 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -2064,9 +2064,6 @@ static const struct pci_error_handlers nvme_err_handler = {
.reset_notify = nvme_reset_notify,
};
-/* Move to pci_ids.h later */
-#define PCI_CLASS_STORAGE_EXPRESS 0x010802
-
static const struct pci_device_id nvme_id_table[] = {
{ PCI_VDEVICE(INTEL, 0x0953),
.driver_data = NVME_QUIRK_STRIPE_SIZE |
diff --git a/include/linux/pci_ids.h b/include/linux/pci_ids.h
index c58752f..a5e6c7b 100644
--- a/include/linux/pci_ids.h
+++ b/include/linux/pci_ids.h
@@ -23,8 +23,10 @@
#define PCI_CLASS_STORAGE_SATA 0x0106
#define PCI_CLASS_STORAGE_SATA_AHCI 0x010601
#define PCI_CLASS_STORAGE_SAS 0x0107
+#define PCI_CLASS_STORAGE_EXPRESS 0x010802
#define PCI_CLASS_STORAGE_OTHER 0x0180
+
#define PCI_BASE_CLASS_NETWORK 0x02
#define PCI_CLASS_NETWORK_ETHERNET 0x0200
#define PCI_CLASS_NETWORK_TOKEN_RING 0x0201
--
2.1.4
^ permalink raw reply related [flat|nested] 14+ messages in thread* [PATCH 1/3] nvme: move NVMe class code to pci_ids.h
@ 2016-12-02 18:31 ` Christoph Hellwig
0 siblings, 0 replies; 14+ messages in thread
From: Christoph Hellwig @ 2016-12-02 18:31 UTC (permalink / raw)
We'll need to check for it in the AHCI drivers (yes, really) soon.
Signed-off-by: Christoph Hellwig <hch at lst.de>
---
drivers/nvme/host/pci.c | 3 ---
include/linux/pci_ids.h | 2 ++
2 files changed, 2 insertions(+), 3 deletions(-)
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index 0fc99f0..2c58f15 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -2064,9 +2064,6 @@ static const struct pci_error_handlers nvme_err_handler = {
.reset_notify = nvme_reset_notify,
};
-/* Move to pci_ids.h later */
-#define PCI_CLASS_STORAGE_EXPRESS 0x010802
-
static const struct pci_device_id nvme_id_table[] = {
{ PCI_VDEVICE(INTEL, 0x0953),
.driver_data = NVME_QUIRK_STRIPE_SIZE |
diff --git a/include/linux/pci_ids.h b/include/linux/pci_ids.h
index c58752f..a5e6c7b 100644
--- a/include/linux/pci_ids.h
+++ b/include/linux/pci_ids.h
@@ -23,8 +23,10 @@
#define PCI_CLASS_STORAGE_SATA 0x0106
#define PCI_CLASS_STORAGE_SATA_AHCI 0x010601
#define PCI_CLASS_STORAGE_SAS 0x0107
+#define PCI_CLASS_STORAGE_EXPRESS 0x010802
#define PCI_CLASS_STORAGE_OTHER 0x0180
+
#define PCI_BASE_CLASS_NETWORK 0x02
#define PCI_CLASS_NETWORK_ETHERNET 0x0200
#define PCI_CLASS_NETWORK_TOKEN_RING 0x0201
--
2.1.4
^ permalink raw reply related [flat|nested] 14+ messages in thread
* [PATCH 2/3] ahci-remap.h: add ahci remapping definitions
2016-12-02 18:31 ` Christoph Hellwig
@ 2016-12-02 18:31 ` Christoph Hellwig
-1 siblings, 0 replies; 14+ messages in thread
From: Christoph Hellwig @ 2016-12-02 18:31 UTC (permalink / raw)
To: tj; +Cc: dan.j.williams, linux-ide, linux-nvme
From: Dan Williams <dan.j.williams@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
[hch: split into a separate header and commit]
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
include/linux/ahci-remap.h | 29 +++++++++++++++++++++++++++++
1 file changed, 29 insertions(+)
create mode 100644 include/linux/ahci-remap.h
diff --git a/include/linux/ahci-remap.h b/include/linux/ahci-remap.h
new file mode 100644
index 0000000..883d366
--- /dev/null
+++ b/include/linux/ahci-remap.h
@@ -0,0 +1,29 @@
+#ifndef _LINUX_AHCI_REMAP_H
+#define _LINUX_AHCI_REMAP_H
+
+#include <linux/sizes.h>
+
+#define AHCI_VSCAP 0xa4
+#define AHCI_REMAP_CAP 0x800
+#define AHCI_VSCAP 0xa4
+
+/* device class code */
+#define AHCI_REMAP_N_DCC 0x880
+
+/* remap-device base relative to ahci-bar */
+#define AHCI_REMAP_N_OFFSET SZ_16K
+#define AHCI_REMAP_N_SIZE SZ_16K
+
+#define AHCI_MAX_REMAP 3
+
+static inline unsigned int ahci_remap_dcc(int i)
+{
+ return AHCI_REMAP_N_DCC + i * 0x80;
+}
+
+static inline unsigned int ahci_remap_base(int i)
+{
+ return AHCI_REMAP_N_OFFSET + i * AHCI_REMAP_N_SIZE;
+}
+
+#endif /* _LINUX_AHCI_REMAP_H */
--
2.1.4
^ permalink raw reply related [flat|nested] 14+ messages in thread* [PATCH 2/3] ahci-remap.h: add ahci remapping definitions
@ 2016-12-02 18:31 ` Christoph Hellwig
0 siblings, 0 replies; 14+ messages in thread
From: Christoph Hellwig @ 2016-12-02 18:31 UTC (permalink / raw)
From: Dan Williams <dan.j.williams@intel.com>
Signed-off-by: Dan Williams <dan.j.williams at intel.com>
[hch: split into a separate header and commit]
Signed-off-by: Christoph Hellwig <hch at lst.de>
---
include/linux/ahci-remap.h | 29 +++++++++++++++++++++++++++++
1 file changed, 29 insertions(+)
create mode 100644 include/linux/ahci-remap.h
diff --git a/include/linux/ahci-remap.h b/include/linux/ahci-remap.h
new file mode 100644
index 0000000..883d366
--- /dev/null
+++ b/include/linux/ahci-remap.h
@@ -0,0 +1,29 @@
+#ifndef _LINUX_AHCI_REMAP_H
+#define _LINUX_AHCI_REMAP_H
+
+#include <linux/sizes.h>
+
+#define AHCI_VSCAP 0xa4
+#define AHCI_REMAP_CAP 0x800
+#define AHCI_VSCAP 0xa4
+
+/* device class code */
+#define AHCI_REMAP_N_DCC 0x880
+
+/* remap-device base relative to ahci-bar */
+#define AHCI_REMAP_N_OFFSET SZ_16K
+#define AHCI_REMAP_N_SIZE SZ_16K
+
+#define AHCI_MAX_REMAP 3
+
+static inline unsigned int ahci_remap_dcc(int i)
+{
+ return AHCI_REMAP_N_DCC + i * 0x80;
+}
+
+static inline unsigned int ahci_remap_base(int i)
+{
+ return AHCI_REMAP_N_OFFSET + i * AHCI_REMAP_N_SIZE;
+}
+
+#endif /* _LINUX_AHCI_REMAP_H */
--
2.1.4
^ permalink raw reply related [flat|nested] 14+ messages in thread* Re: [PATCH 2/3] ahci-remap.h: add ahci remapping definitions
2016-12-02 18:31 ` Christoph Hellwig
@ 2016-12-03 10:57 ` Sergei Shtylyov
-1 siblings, 0 replies; 14+ messages in thread
From: Sergei Shtylyov @ 2016-12-03 10:57 UTC (permalink / raw)
To: Christoph Hellwig, tj; +Cc: dan.j.williams, linux-ide, linux-nvme
Hello.
On 12/2/2016 9:31 PM, Christoph Hellwig wrote:
> From: Dan Williams <dan.j.williams@intel.com>
>
> Signed-off-by: Dan Williams <dan.j.williams@intel.com>
> [hch: split into a separate header and commit]
> Signed-off-by: Christoph Hellwig <hch@lst.de>
> ---
> include/linux/ahci-remap.h | 29 +++++++++++++++++++++++++++++
> 1 file changed, 29 insertions(+)
> create mode 100644 include/linux/ahci-remap.h
>
> diff --git a/include/linux/ahci-remap.h b/include/linux/ahci-remap.h
> new file mode 100644
> index 0000000..883d366
> --- /dev/null
> +++ b/include/linux/ahci-remap.h
> @@ -0,0 +1,29 @@
> +#ifndef _LINUX_AHCI_REMAP_H
> +#define _LINUX_AHCI_REMAP_H
> +
> +#include <linux/sizes.h>
> +
> +#define AHCI_VSCAP 0xa4
> +#define AHCI_REMAP_CAP 0x800
> +#define AHCI_VSCAP 0xa4
Why do it twice?
MBR, Sergei
^ permalink raw reply [flat|nested] 14+ messages in thread* [PATCH 2/3] ahci-remap.h: add ahci remapping definitions
@ 2016-12-03 10:57 ` Sergei Shtylyov
0 siblings, 0 replies; 14+ messages in thread
From: Sergei Shtylyov @ 2016-12-03 10:57 UTC (permalink / raw)
Hello.
On 12/2/2016 9:31 PM, Christoph Hellwig wrote:
> From: Dan Williams <dan.j.williams at intel.com>
>
> Signed-off-by: Dan Williams <dan.j.williams at intel.com>
> [hch: split into a separate header and commit]
> Signed-off-by: Christoph Hellwig <hch at lst.de>
> ---
> include/linux/ahci-remap.h | 29 +++++++++++++++++++++++++++++
> 1 file changed, 29 insertions(+)
> create mode 100644 include/linux/ahci-remap.h
>
> diff --git a/include/linux/ahci-remap.h b/include/linux/ahci-remap.h
> new file mode 100644
> index 0000000..883d366
> --- /dev/null
> +++ b/include/linux/ahci-remap.h
> @@ -0,0 +1,29 @@
> +#ifndef _LINUX_AHCI_REMAP_H
> +#define _LINUX_AHCI_REMAP_H
> +
> +#include <linux/sizes.h>
> +
> +#define AHCI_VSCAP 0xa4
> +#define AHCI_REMAP_CAP 0x800
> +#define AHCI_VSCAP 0xa4
Why do it twice?
MBR, Sergei
^ permalink raw reply [flat|nested] 14+ messages in thread* Re: [PATCH 2/3] ahci-remap.h: add ahci remapping definitions
2016-12-03 10:57 ` Sergei Shtylyov
@ 2016-12-05 15:31 ` Christoph Hellwig
-1 siblings, 0 replies; 14+ messages in thread
From: Christoph Hellwig @ 2016-12-05 15:31 UTC (permalink / raw)
To: Sergei Shtylyov
Cc: Christoph Hellwig, tj, dan.j.williams, linux-ide, linux-nvme
On Sat, Dec 03, 2016 at 01:57:01PM +0300, Sergei Shtylyov wrote:
>> +#ifndef _LINUX_AHCI_REMAP_H
>> +#define _LINUX_AHCI_REMAP_H
>> +
>> +#include <linux/sizes.h>
>> +
>> +#define AHCI_VSCAP 0xa4
>> +#define AHCI_REMAP_CAP 0x800
>> +#define AHCI_VSCAP 0xa4
>
> Why do it twice?
No good reason, probably just me messing up the move..
^ permalink raw reply [flat|nested] 14+ messages in thread
* [PATCH 2/3] ahci-remap.h: add ahci remapping definitions
@ 2016-12-05 15:31 ` Christoph Hellwig
0 siblings, 0 replies; 14+ messages in thread
From: Christoph Hellwig @ 2016-12-05 15:31 UTC (permalink / raw)
On Sat, Dec 03, 2016@01:57:01PM +0300, Sergei Shtylyov wrote:
>> +#ifndef _LINUX_AHCI_REMAP_H
>> +#define _LINUX_AHCI_REMAP_H
>> +
>> +#include <linux/sizes.h>
>> +
>> +#define AHCI_VSCAP 0xa4
>> +#define AHCI_REMAP_CAP 0x800
>> +#define AHCI_VSCAP 0xa4
>
> Why do it twice?
No good reason, probably just me messing up the move..
^ permalink raw reply [flat|nested] 14+ messages in thread
* [PATCH 3/3] ahci: warn about remapped NVMe devices
2016-12-02 18:31 ` Christoph Hellwig
@ 2016-12-02 18:31 ` Christoph Hellwig
-1 siblings, 0 replies; 14+ messages in thread
From: Christoph Hellwig @ 2016-12-02 18:31 UTC (permalink / raw)
To: tj; +Cc: dan.j.williams, linux-ide, linux-nvme
Some Intel ahci implementations have a completely broken remapping mode
where they hide one or more NVMe devices behind the bar of an AHCI device.
Intel refuses to let the OS reprogram the BIOS to switch out of this
mode at runtime, and so far we're not come up with another good way
to undo the mess that the Chipset people created. So for now the only
thing we can do is to alert users about this situation and switch to the
faster and much saner so called "AHCI" mode insted of the RAID mode in
the BIOS so that the BIOS does not hide the NVMe devices from us.
The sitation is even worse as at least one vendor (thanks a lot Lenovo..)
has started hardcoding their BIOS into the "RAID" mode even for laptops
that don't use AHCI _at all_ and just have a single NVMe device. For now
there is an unspported Linux-only BIOS that undoes this braindamage,
but we'll have to see if things are getting better or worse from here.
Based on an earlier patch from Dan Williams <dan.j.williams@intel.com>.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
drivers/ata/ahci.c | 39 +++++++++++++++++++++++++++++++++++++++
1 file changed, 39 insertions(+)
diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c
index ba5f11c..516a689 100644
--- a/drivers/ata/ahci.c
+++ b/drivers/ata/ahci.c
@@ -46,6 +46,8 @@
#include <scsi/scsi_host.h>
#include <scsi/scsi_cmnd.h>
#include <linux/libata.h>
+#include <linux/ahci-remap.h>
+#include <linux/io-64-nonatomic-lo-hi.h>
#include "ahci.h"
#define DRV_NAME "ahci"
@@ -1400,6 +1402,40 @@ static irqreturn_t ahci_thunderx_irq_handler(int irq, void *dev_instance)
}
#endif
+static void ahci_remap_check(struct pci_dev *pdev, int bar,
+ struct ahci_host_priv *hpriv)
+{
+ int i, count = 0;
+ u32 cap;
+
+ /*
+ * Check if this device might have remapped nvme devices.
+ */
+ if (pdev->vendor != PCI_VENDOR_ID_INTEL ||
+ pci_resource_len(pdev, bar) < SZ_512K ||
+ bar != AHCI_PCI_BAR_STANDARD ||
+ !(readl(hpriv->mmio + AHCI_VSCAP) & 1))
+ return;
+
+ cap = readq(hpriv->mmio + AHCI_REMAP_CAP);
+ for (i = 0; i < AHCI_MAX_REMAP; i++) {
+ if ((cap & (1 << i)) == 0)
+ continue;
+ if (readl(hpriv->mmio + ahci_remap_dcc(i))
+ != PCI_CLASS_STORAGE_EXPRESS)
+ continue;
+
+ /* We've found a remapped device */
+ count++;
+ }
+
+ if (!count)
+ return;
+
+ dev_warn(&pdev->dev, "Found %d remapped NVMe devices.\n", count);
+ dev_warn(&pdev->dev, "Switch your BIOS from RAID to AHCI mode to use them.\n");
+}
+
static int ahci_get_irq_vector(struct ata_host *host, int port)
{
return pci_irq_vector(to_pci_dev(host->dev), port);
@@ -1545,6 +1581,9 @@ static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
hpriv->mmio = pcim_iomap_table(pdev)[ahci_pci_bar];
+ /* detect remapped nvme devices */
+ ahci_remap_check(pdev, ahci_pci_bar, hpriv);
+
/* must set flag prior to save config in order to take effect */
if (ahci_broken_devslp(pdev))
hpriv->flags |= AHCI_HFLAG_NO_DEVSLP;
--
2.1.4
^ permalink raw reply related [flat|nested] 14+ messages in thread* [PATCH 3/3] ahci: warn about remapped NVMe devices
@ 2016-12-02 18:31 ` Christoph Hellwig
0 siblings, 0 replies; 14+ messages in thread
From: Christoph Hellwig @ 2016-12-02 18:31 UTC (permalink / raw)
Some Intel ahci implementations have a completely broken remapping mode
where they hide one or more NVMe devices behind the bar of an AHCI device.
Intel refuses to let the OS reprogram the BIOS to switch out of this
mode at runtime, and so far we're not come up with another good way
to undo the mess that the Chipset people created. So for now the only
thing we can do is to alert users about this situation and switch to the
faster and much saner so called "AHCI" mode insted of the RAID mode in
the BIOS so that the BIOS does not hide the NVMe devices from us.
The sitation is even worse as at least one vendor (thanks a lot Lenovo..)
has started hardcoding their BIOS into the "RAID" mode even for laptops
that don't use AHCI _at all_ and just have a single NVMe device. For now
there is an unspported Linux-only BIOS that undoes this braindamage,
but we'll have to see if things are getting better or worse from here.
Based on an earlier patch from Dan Williams <dan.j.williams at intel.com>.
Signed-off-by: Christoph Hellwig <hch at lst.de>
---
drivers/ata/ahci.c | 39 +++++++++++++++++++++++++++++++++++++++
1 file changed, 39 insertions(+)
diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c
index ba5f11c..516a689 100644
--- a/drivers/ata/ahci.c
+++ b/drivers/ata/ahci.c
@@ -46,6 +46,8 @@
#include <scsi/scsi_host.h>
#include <scsi/scsi_cmnd.h>
#include <linux/libata.h>
+#include <linux/ahci-remap.h>
+#include <linux/io-64-nonatomic-lo-hi.h>
#include "ahci.h"
#define DRV_NAME "ahci"
@@ -1400,6 +1402,40 @@ static irqreturn_t ahci_thunderx_irq_handler(int irq, void *dev_instance)
}
#endif
+static void ahci_remap_check(struct pci_dev *pdev, int bar,
+ struct ahci_host_priv *hpriv)
+{
+ int i, count = 0;
+ u32 cap;
+
+ /*
+ * Check if this device might have remapped nvme devices.
+ */
+ if (pdev->vendor != PCI_VENDOR_ID_INTEL ||
+ pci_resource_len(pdev, bar) < SZ_512K ||
+ bar != AHCI_PCI_BAR_STANDARD ||
+ !(readl(hpriv->mmio + AHCI_VSCAP) & 1))
+ return;
+
+ cap = readq(hpriv->mmio + AHCI_REMAP_CAP);
+ for (i = 0; i < AHCI_MAX_REMAP; i++) {
+ if ((cap & (1 << i)) == 0)
+ continue;
+ if (readl(hpriv->mmio + ahci_remap_dcc(i))
+ != PCI_CLASS_STORAGE_EXPRESS)
+ continue;
+
+ /* We've found a remapped device */
+ count++;
+ }
+
+ if (!count)
+ return;
+
+ dev_warn(&pdev->dev, "Found %d remapped NVMe devices.\n", count);
+ dev_warn(&pdev->dev, "Switch your BIOS from RAID to AHCI mode to use them.\n");
+}
+
static int ahci_get_irq_vector(struct ata_host *host, int port)
{
return pci_irq_vector(to_pci_dev(host->dev), port);
@@ -1545,6 +1581,9 @@ static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
hpriv->mmio = pcim_iomap_table(pdev)[ahci_pci_bar];
+ /* detect remapped nvme devices */
+ ahci_remap_check(pdev, ahci_pci_bar, hpriv);
+
/* must set flag prior to save config in order to take effect */
if (ahci_broken_devslp(pdev))
hpriv->flags |= AHCI_HFLAG_NO_DEVSLP;
--
2.1.4
^ permalink raw reply related [flat|nested] 14+ messages in thread
* Re: detect and warn about ahci-remapped NVMe devices
2016-12-02 18:31 ` Christoph Hellwig
@ 2016-12-05 19:33 ` Tejun Heo
-1 siblings, 0 replies; 14+ messages in thread
From: Tejun Heo @ 2016-12-05 19:33 UTC (permalink / raw)
To: Christoph Hellwig; +Cc: dan.j.williams, linux-ide, linux-nvme
Hello,
On Fri, Dec 02, 2016 at 07:31:00PM +0100, Christoph Hellwig wrote:
> Just a small band-aid to detect the AHCI-remapped NVMe devices
> for now. Hopefully we'll eventually get a fake PCIe root port
> driver ala VMD for them.
Applied 1-3 to libata/for-4.10. Fixed the dup constant definition in
0002.
Thanks.
--
tejun
^ permalink raw reply [flat|nested] 14+ messages in thread
* detect and warn about ahci-remapped NVMe devices
@ 2016-12-05 19:33 ` Tejun Heo
0 siblings, 0 replies; 14+ messages in thread
From: Tejun Heo @ 2016-12-05 19:33 UTC (permalink / raw)
Hello,
On Fri, Dec 02, 2016@07:31:00PM +0100, Christoph Hellwig wrote:
> Just a small band-aid to detect the AHCI-remapped NVMe devices
> for now. Hopefully we'll eventually get a fake PCIe root port
> driver ala VMD for them.
Applied 1-3 to libata/for-4.10. Fixed the dup constant definition in
0002.
Thanks.
--
tejun
^ permalink raw reply [flat|nested] 14+ messages in thread