* [PATCH] Implement NVMe native disk support
@ 2024-05-16 18:49 Vladimir Serbinenko
0 siblings, 0 replies; only message in thread
From: Vladimir Serbinenko @ 2024-05-16 18:49 UTC (permalink / raw)
To: grub-devel; +Cc: Vladimir Serbinenko
This is useful 2-fold:
1) On coreboot port it allows to boot from NVMe devices
2) On older systems you can install NVMe via PCIe adapter and boot from
it
Signed-off-by: Vladimir Serbinenko <phcoder@gmail.com>
---
grub-core/Makefile.core.def | 6 +
grub-core/commands/nativedisk.c | 3 +-
grub-core/disk/nvme.c | 642 ++++++++++++++++++++++++++++++++
include/grub/disk.h | 1 +
4 files changed, 651 insertions(+), 1 deletion(-)
create mode 100644 grub-core/disk/nvme.c
diff --git a/grub-core/Makefile.core.def b/grub-core/Makefile.core.def
index 8e1b1d9f3..47ac8bf3f 100644
--- a/grub-core/Makefile.core.def
+++ b/grub-core/Makefile.core.def
@@ -1305,6 +1305,12 @@ module = {
enable = pci;
};
+module = {
+ name = nvme;
+ common = disk/nvme.c;
+ enable = pci;
+};
+
module = {
name = pata;
common = disk/pata.c;
diff --git a/grub-core/commands/nativedisk.c b/grub-core/commands/nativedisk.c
index 580c8d3b0..a431a066a 100644
--- a/grub-core/commands/nativedisk.c
+++ b/grub-core/commands/nativedisk.c
@@ -34,7 +34,7 @@ GRUB_MOD_LICENSE ("GPLv3+");
static const char *modnames_def[] = {
/* FIXME: autogenerate this. */
#if defined (__i386__) || defined (__x86_64__) || defined (GRUB_MACHINE_MIPS_LOONGSON)
- "pata", "ahci", "usbms", "ohci", "uhci", "ehci"
+ "pata", "ahci", "usbms", "ohci", "uhci", "ehci", "nvme"
#elif defined (GRUB_MACHINE_MIPS_QEMU_MIPS)
"pata"
#else
@@ -77,6 +77,7 @@ get_uuid (const char *name, char **uuid, int getnative)
/* Native disks. */
case GRUB_DISK_DEVICE_ATA_ID:
case GRUB_DISK_DEVICE_SCSI_ID:
+ case GRUB_DISK_DEVICE_NVME_ID:
case GRUB_DISK_DEVICE_XEN:
if (getnative)
break;
diff --git a/grub-core/disk/nvme.c b/grub-core/disk/nvme.c
new file mode 100644
index 000000000..b2949308d
--- /dev/null
+++ b/grub-core/disk/nvme.c
@@ -0,0 +1,642 @@
+/*
+ * GRUB -- GRand Unified Bootloader
+ *
+ * Copyright (C) 2019 secunet Security Networks AG
+ * Copyright (C) 2024 Free Software Foundation, Inc.
+ *
+ * GRUB is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Additionally this file can be distributed under 3-clause BSD license.
+ *
+ * GRUB is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with GRUB. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <grub/dl.h>
+#include <grub/disk.h>
+#include <grub/mm.h>
+#include <grub/time.h>
+#include <grub/pci.h>
+#include <grub/misc.h>
+#include <grub/list.h>
+#include <grub/loader.h>
+
+GRUB_MOD_LICENSE ("GPLv3+");
+
+#define NVME_CC_EN (1 << 0)
+#define NVME_CC_CSS (0 << 4)
+#define NVME_CC_MPS (0 << 7)
+#define NVME_CC_AMS (0 << 11)
+#define NVME_CC_SHN (0 << 14)
+#define NVME_CC_IOSQES (6 << 16)
+#define NVME_CC_IOCQES (4 << 20)
+
+#define NVME_QUEUE_SIZE 2
+#define NVME_SQ_ENTRY_SIZE 64
+#define NVME_CQ_ENTRY_SIZE 16
+
+struct grub_nvme_mmio_reg
+{
+ /* 0 */ grub_uint64_t cap;
+ /* 8 */ grub_uint32_t vs;
+ /* c */ grub_uint32_t intms;
+ /* 10 */ grub_uint32_t intmc;
+ /* 14 */ grub_uint32_t controller_config;
+ /* 18 */ grub_uint32_t reserved1;
+ /* 1c */ grub_uint32_t controller_status;
+ /* 20 */ grub_uint32_t nssr;
+ /* 24 */ grub_uint32_t aqa;
+ /* 28 */ grub_uint64_t asq;
+ /* 30 */ grub_uint64_t acq;
+};
+
+struct nvme_ident_block
+{
+ /* 0 */ grub_uint64_t nsze;
+ /* 8 */ grub_uint64_t ncap;
+ /* 10 */ grub_uint64_t nuse;
+ /* 18 */ grub_uint8_t nsfeat;
+ /* 19 */ grub_uint8_t nlbaf;
+ /* 1a */ grub_uint8_t flbas;
+ /* 1b */ grub_uint8_t mc;
+ /* 1c */ grub_uint32_t fill[(0x80 - 0x1c) / 4];
+ /* 80 */ grub_uint32_t lbaf[64];
+};
+
+struct grub_nvme_device
+{
+ struct grub_nvme_device *next;
+ struct grub_nvme_device **prev;
+ volatile struct grub_nvme_mmio_reg *regs;
+ struct grub_pci_dma_chunk *prp_list;
+ struct grub_pci_dma_chunk *sq_buffer;
+ struct grub_pci_dma_chunk *cq_buffer;
+ struct grub_pci_dma_chunk *ioc_buffer;
+ struct grub_pci_dma_chunk *ios_buffer;
+ struct grub_pci_dma_chunk *ident_buffer;
+ int num;
+ grub_uint64_t total_sectors;
+ int log_sector_size;
+
+ struct {
+ volatile void *base;
+ volatile grub_uint32_t *bell;
+ grub_uint16_t idx; // bool pos 0 or 1
+ grub_uint16_t round; // bool round 0 or 1+0xd
+ } queue[4];
+};
+
+struct nvme_s_queue_entry {
+ grub_uint32_t dw[16];
+};
+
+struct nvme_c_queue_entry {
+ grub_uint32_t dw[4];
+};
+
+static struct grub_nvme_device *grub_nvme_devices;
+static int numdevs;
+
+enum nvme_queue {
+ NVME_ADMIN_QUEUE = 0,
+ ads = 0,
+ adc = 1,
+ NVME_IO_QUEUE = 2,
+ ios = 2,
+ ioc = 3,
+};
+
+static int
+nvme_cmd(struct grub_nvme_device *nvme, enum nvme_queue q, const struct nvme_s_queue_entry *cmd)
+{
+ int sq = q, cq = q+1;
+
+ void *s_entry = (char *) nvme->queue[sq].base + (nvme->queue[sq].idx * NVME_SQ_ENTRY_SIZE);
+ grub_memcpy(s_entry, cmd, NVME_SQ_ENTRY_SIZE);
+ nvme->queue[sq].idx = (nvme->queue[sq].idx + 1) & (NVME_QUEUE_SIZE - 1);
+ *nvme->queue[sq].bell = nvme->queue[sq].idx;
+
+ struct nvme_c_queue_entry *c_entry = (struct nvme_c_queue_entry *)
+ ((char *) nvme->queue[cq].base + (nvme->queue[cq].idx * NVME_CQ_ENTRY_SIZE));
+ grub_uint64_t endtime = grub_get_time_ms () + 100;
+ while (((*(volatile grub_uint32_t *)(&c_entry->dw[3]) >> 16) & 0x1) == nvme->queue[cq].round)
+ {
+ if (grub_get_time_ms () > endtime)
+ {
+ grub_dprintf("nvme", "command timed out");
+ return -1;
+ }
+ }
+ nvme->queue[cq].idx = (nvme->queue[cq].idx + 1) & (NVME_QUEUE_SIZE - 1);
+ *nvme->queue[cq].bell = nvme->queue[cq].idx;
+ if (nvme->queue[cq].idx == 0)
+ nvme->queue[cq].round = (nvme->queue[cq].round + 1) & 1;
+ return c_entry->dw[3] >> 17;
+}
+
+static int
+create_admin_queues(struct grub_nvme_device *nvme)
+{
+ grub_uint8_t cap_dstrd = (nvme->regs->cap >> 32) & 0xf;
+ nvme->regs->aqa = (NVME_QUEUE_SIZE - 1) << 16 | (NVME_QUEUE_SIZE - 1);
+
+ nvme->sq_buffer = grub_memalign_dma32(0x1000, NVME_SQ_ENTRY_SIZE * NVME_QUEUE_SIZE);
+ if (!nvme->sq_buffer)
+ {
+ grub_dprintf("nvme", "NVMe ERROR: Failed to allocated memory for admin submission queue\n");
+ return -1;
+ }
+ grub_memset((void *) grub_dma_get_virt(nvme->sq_buffer), 0, NVME_SQ_ENTRY_SIZE * NVME_QUEUE_SIZE);
+ nvme->regs->asq = grub_dma_get_phys(nvme->sq_buffer);
+
+ nvme->queue[ads].base = grub_dma_get_virt(nvme->sq_buffer);
+ nvme->queue[ads].bell = (volatile grub_uint32_t *) nvme->regs + 0x1000 / 4 + (ads * (1 << cap_dstrd));
+ nvme->queue[ads].idx = 0;
+
+ nvme->cq_buffer = grub_memalign_dma32(0x1000, NVME_CQ_ENTRY_SIZE * NVME_QUEUE_SIZE);
+ if (!nvme->cq_buffer)
+ {
+ grub_dprintf("nvme", "NVMe ERROR: Failed to allocate memory for admin completion queue\n");
+ grub_dma_free(nvme->sq_buffer);
+ return -1;
+ }
+ grub_memset((void *) grub_dma_get_virt(nvme->cq_buffer), 0, NVME_CQ_ENTRY_SIZE * NVME_QUEUE_SIZE);
+ nvme->regs->acq = grub_dma_get_phys(nvme->cq_buffer);
+
+ nvme->queue[adc].base = nvme->cq_buffer;
+ nvme->queue[adc].bell = (volatile grub_uint32_t *) nvme->regs + 0x1000 / 4 + (adc * (1 << cap_dstrd));
+ nvme->queue[adc].idx = 0;
+ nvme->queue[adc].round = 0;
+
+ return 0;
+}
+
+static int create_io_submission_queue(struct grub_nvme_device *nvme)
+{
+ nvme->ios_buffer = grub_memalign_dma32(0x1000, NVME_SQ_ENTRY_SIZE * NVME_QUEUE_SIZE);
+ if (!nvme->ios_buffer)
+ {
+ grub_dprintf("nvme", "NVMe ERROR: Failed to allocate memory for io submission queue.\n");
+ return -1;
+ }
+ grub_memset((void *) grub_dma_get_virt(nvme->ios_buffer), 0, NVME_SQ_ENTRY_SIZE * NVME_QUEUE_SIZE);
+
+ struct nvme_s_queue_entry e = {
+ .dw[0] = 0x01,
+ .dw[6] = grub_dma_get_phys(nvme->ios_buffer),
+ .dw[10] = ((NVME_QUEUE_SIZE - 1) << 16) | ios >> 1,
+ .dw[11] = (1 << 16) | 1,
+ };
+
+ int res = nvme_cmd(nvme, NVME_ADMIN_QUEUE, &e);
+ if (res) {
+ grub_dprintf("nvme", "NVMe ERROR: nvme_cmd returned with %i.\n", res);
+ grub_dma_free(nvme->ios_buffer);
+ return res;
+ }
+
+ grub_uint8_t cap_dstrd = (nvme->regs->cap >> 32) & 0xf;
+ nvme->queue[ios].base = nvme->ios_buffer;
+ nvme->queue[ios].bell = (volatile grub_uint32_t *) nvme->regs + 0x1000 / 4 + (ios * (1 << cap_dstrd));
+ nvme->queue[ios].idx = 0;
+ return 0;
+}
+
+static int create_io_completion_queue(struct grub_nvme_device *nvme)
+{
+ nvme->ioc_buffer = grub_memalign_dma32(0x1000, NVME_CQ_ENTRY_SIZE * NVME_QUEUE_SIZE);
+ if (!nvme->ioc_buffer) {
+ grub_dprintf("nvme", "NVMe ERROR: Failed to allocate memory for io completion queue.\n");
+ return -1;
+ }
+ grub_memset((void *) grub_dma_get_virt(nvme->ioc_buffer), 0, NVME_CQ_ENTRY_SIZE * NVME_QUEUE_SIZE);
+
+ const struct nvme_s_queue_entry e = {
+ .dw[0] = 0x05,
+ .dw[6] = grub_dma_get_phys(nvme->ioc_buffer),
+ .dw[10] = ((NVME_QUEUE_SIZE - 1) << 16) | ioc >> 1,
+ .dw[11] = 1,
+ };
+
+ int res = nvme_cmd(nvme, NVME_ADMIN_QUEUE, &e);
+ if (res)
+ {
+ grub_dprintf("nvme", "NVMe ERROR: nvme_cmd returned with %i.\n", res);
+ grub_dma_free(nvme->ioc_buffer);
+ return res;
+ }
+
+ grub_uint8_t cap_dstrd = (nvme->regs->cap >> 32) & 0xf;
+ nvme->queue[ioc].base = nvme->ioc_buffer;
+ nvme->queue[ioc].bell = (volatile grub_uint32_t *) nvme->regs + 0x1000 / 4 + (ioc * (1 << cap_dstrd));
+ nvme->queue[ioc].idx = 0;
+ nvme->queue[ioc].round = 0;
+
+ return 0;
+}
+
+static int identify(struct grub_nvme_device *nvme)
+{
+ nvme->ident_buffer = grub_memalign_dma32(0x1000, 0x1000);
+ if (!nvme->ident_buffer) {
+ grub_dprintf("nvme", "NVMe ERROR: Failed to allocate ident buffer.\n");
+ return -1;
+ }
+ grub_memset((void *) grub_dma_get_virt(nvme->ident_buffer), 0, 0x1000);
+
+ const struct nvme_s_queue_entry e = {
+ .dw[0] = 0x06,
+ .dw[1] = 0x01,
+ .dw[2] = 0x00,
+ .dw[6] = grub_dma_get_phys(nvme->ident_buffer),
+ .dw[10] = 0x00,
+ };
+
+ int res = nvme_cmd(nvme, NVME_ADMIN_QUEUE, &e);
+ if (res)
+ {
+ grub_dprintf("nvme", "NVMe ERROR: nvme_cmd returned with %i.\n", res);
+ grub_memset((void *) grub_dma_get_virt(nvme->ident_buffer), 0, 0x1000);
+ return res;
+ }
+
+ struct nvme_ident_block *nvme_ident = (void *) grub_dma_get_virt(nvme->ident_buffer);
+ nvme->total_sectors = nvme_ident->nsze;
+ int selected_lbaf = nvme_ident->flbas & 0xf;
+ nvme->log_sector_size = (nvme_ident->lbaf[selected_lbaf] >> 16) & 0xff;
+
+ grub_dprintf("nvme", "Detected disk with %lld sectors of 2^%d bytes each\n", (long long) nvme->total_sectors, nvme->log_sector_size);
+
+ return 0;
+}
+
+static int delete_io_submission_queue(struct grub_nvme_device *nvme)
+{
+ const struct nvme_s_queue_entry e = {
+ .dw[0] = 0,
+ .dw[10] = ios,
+ };
+
+ int res = nvme_cmd(nvme, NVME_ADMIN_QUEUE, &e);
+
+ grub_dma_free(nvme->ios_buffer);
+ nvme->queue[ios].base = NULL;
+ nvme->queue[ios].bell = NULL;
+ nvme->queue[ios].idx = 0;
+ return res;
+}
+
+static int delete_io_completion_queue(struct grub_nvme_device *nvme)
+{
+ const struct nvme_s_queue_entry e = {
+ .dw[0] = 1,
+ .dw[10] = ioc,
+ };
+
+ int res = nvme_cmd(nvme, NVME_ADMIN_QUEUE, &e);
+ grub_dma_free(nvme->ioc_buffer);
+
+ nvme->queue[ioc].base = NULL;
+ nvme->queue[ioc].bell = NULL;
+ nvme->queue[ioc].idx = 0;
+ nvme->queue[ioc].round = 0;
+ return res;
+}
+
+static int
+grub_nvme_pciinit (grub_pci_device_t dev,
+ grub_pci_id_t pciid __attribute__ ((unused)),
+ void *data __attribute__ ((unused)))
+{
+ grub_pci_address_t addr;
+ grub_uint32_t class;
+ grub_uint64_t bar;
+ volatile struct grub_nvme_mmio_reg *mmio_reg;
+
+ /* Read class. */
+ addr = grub_pci_make_address (dev, GRUB_PCI_REG_CLASS);
+ class = grub_pci_read (addr);
+
+ /* Check if this class ID matches that of a PCI NVMe Controller. */
+ if (class >> 8 != 0x010802)
+ return 0;
+
+ addr = grub_pci_make_address (dev, GRUB_PCI_REG_ADDRESS_REG0);
+
+ bar = grub_pci_read (addr);
+
+ if ((bar & (GRUB_PCI_ADDR_SPACE_MASK | GRUB_PCI_ADDR_MEM_TYPE_MASK
+ | GRUB_PCI_ADDR_MEM_PREFETCH))
+ != (GRUB_PCI_ADDR_SPACE_MEMORY | GRUB_PCI_ADDR_MEM_TYPE_64))
+ return 0;
+
+ addr = grub_pci_make_address (dev, GRUB_PCI_REG_ADDRESS_REG1);
+
+ bar |= ((grub_uint64_t) grub_pci_read (addr)) << 32;
+
+ addr = grub_pci_make_address (dev, GRUB_PCI_REG_COMMAND);
+ grub_pci_write_word (addr, grub_pci_read_word (addr)
+ | GRUB_PCI_COMMAND_MEM_ENABLED);
+
+ mmio_reg = grub_pci_device_map_range (dev, bar & ~0xfULL,
+ sizeof (*mmio_reg));
+ grub_dprintf ("nvme", "dev: %x:%x.%x\n", dev.bus, dev.device, dev.function);
+
+ if (!(mmio_reg->cap & (1LL << 37))) {
+ grub_dprintf ("nvme", "nvme command set not supported\n");
+ return 0;
+ }
+
+ mmio_reg->controller_config = 0;
+
+ struct grub_nvme_device *nvmedev = grub_malloc (sizeof (*nvmedev));
+ if (!nvmedev)
+ return 0;
+
+ nvmedev->regs = mmio_reg;
+ nvmedev->num = numdevs++;
+
+ grub_int32_t max_timeout_ms = ((mmio_reg->cap >> 24) & 0xff) * 500;
+ grub_int32_t timeout_ms = max_timeout_ms;
+ while (1)
+ {
+ grub_uint8_t status = mmio_reg->controller_status & 0x3;
+ if (status == 0x2)
+ {
+ grub_dprintf("nvme", "NVMe ERROR: Failed to disable controller. FATAL ERROR\n");
+ return 0;
+ }
+ if (status == 0)
+ break;
+ if (timeout_ms < 0)
+ {
+ grub_dprintf("nvme", "NVMe ERROR: Failed to disable controller. Timeout.\n");
+ return 0;
+ }
+ timeout_ms -= 10;
+ grub_millisleep(10);
+ }
+
+ timeout_ms = max_timeout_ms;
+ if (create_admin_queues(nvmedev))
+ {
+ grub_dprintf("nvme", "NVMe ERROR: Failed to create admin queues. FATAL ERROR\n");
+ return 0;
+ }
+
+ mmio_reg->controller_config = NVME_CC_EN | NVME_CC_CSS | NVME_CC_MPS | NVME_CC_AMS | NVME_CC_SHN
+ | NVME_CC_IOSQES | NVME_CC_IOCQES;
+ while (1)
+ {
+ grub_uint8_t status = mmio_reg->controller_status & 0x3;
+ if (status == 0x2) {
+ grub_dprintf("nvme", "NVMe ERROR: Failed to enable controller. FATAL ERROR\n");
+ mmio_reg->controller_config = 0;
+ return 0;
+ }
+ if (status == 1)
+ break;
+ if (timeout_ms < 0) {
+ grub_dprintf("nvme", "NVMe ERROR: Failed to enable controller. Timeout.\n");
+ mmio_reg->controller_config = 0;
+ return 0;
+ }
+ timeout_ms -= 10;
+ grub_millisleep(10);
+ }
+
+ nvmedev->prp_list = grub_memalign_dma32(0x1000, 0x1000);
+ if (!nvmedev->prp_list) {
+ mmio_reg->controller_config = 0;
+ grub_free (nvmedev);
+ return 0;
+ }
+
+ addr = grub_pci_make_address (dev, GRUB_PCI_REG_COMMAND);
+ grub_pci_write_word (addr, grub_pci_read_word (addr) | GRUB_PCI_COMMAND_BUS_MASTER);
+
+ create_io_completion_queue(nvmedev);
+ create_io_submission_queue(nvmedev);
+
+ identify(nvmedev);
+
+ grub_list_push (GRUB_AS_LIST_P (&grub_nvme_devices),
+ GRUB_AS_LIST (nvmedev));
+ return 0;
+}
+
+static grub_err_t
+grub_nvme_initialize (void)
+{
+ grub_pci_iterate (grub_nvme_pciinit, NULL);
+ return grub_errno;
+}
+
+static grub_err_t
+grub_nvme_fini_hw (int noreturn __attribute__ ((unused)))
+{
+ struct grub_nvme_device *dev;
+
+ for (dev = grub_nvme_devices; dev; dev = dev->next)
+ {
+ delete_io_submission_queue(dev);
+ delete_io_completion_queue(dev);
+ dev->regs->controller_config = 0;
+ /* TODO: wait for completition. */
+ }
+ return GRUB_ERR_NONE;
+}
+
+
+static grub_err_t
+grub_nvme_restore_hw (void)
+{
+ struct grub_nvme_device **pdev;
+
+ for (pdev = &grub_nvme_devices; *pdev; pdev = &((*pdev)->next))
+ {
+ (*pdev)->regs->controller_config = NVME_CC_EN | NVME_CC_CSS | NVME_CC_MPS | NVME_CC_AMS | NVME_CC_SHN
+ | NVME_CC_IOSQES | NVME_CC_IOCQES;
+ create_io_completion_queue(*pdev);
+ create_io_submission_queue(*pdev);
+ /* TODO: Error handling. */
+ }
+ return GRUB_ERR_NONE;
+}
+
+
+\f
+
+static int
+grub_nvme_iterate (grub_disk_dev_iterate_hook_t hook, void *hook_data,
+ grub_disk_pull_t pull)
+{
+ struct grub_nvme_device *dev;
+
+ if (pull != GRUB_DISK_PULL_NONE)
+ return 0;
+
+ FOR_LIST_ELEMENTS(dev, grub_nvme_devices)
+ {
+ char devname[40];
+ /* TODO: Other namespaces. */
+ grub_snprintf (devname, sizeof (devname),
+ "nvme%dn%d", dev->num, 1);
+ if (hook (devname, hook_data))
+ return 1;
+ }
+
+ return 0;
+}
+
+static grub_err_t
+grub_nvme_open (const char *name, grub_disk_t disk)
+{
+ const char *rest;
+ if (grub_memcmp(name, "nvme", 4) != 0 || !grub_isdigit(name[4]))
+ return grub_error (GRUB_ERR_UNKNOWN_DEVICE, "not an NVMe disk");
+ int devnum = grub_strtoul (name + 4, &rest, 0);
+ if (*rest != 'n')
+ return grub_error (GRUB_ERR_UNKNOWN_DEVICE, "not an NVMe disk");
+ int namespace = grub_strtoul (rest + 1, 0, 0);
+
+ struct grub_nvme_device *dev;
+
+ FOR_LIST_ELEMENTS(dev, grub_nvme_devices)
+ if (dev->num == devnum)
+ {
+ if (namespace != 1)
+ return grub_error (GRUB_ERR_UNKNOWN_DEVICE, "unknown NVMe namespace");
+
+ disk->total_sectors = dev->total_sectors;
+ disk->max_agglomerate = 512;
+
+ disk->log_sector_size = dev->log_sector_size;
+ disk->id = (devnum << 8) | namespace;
+ disk->data = dev;
+ return 0;
+ }
+
+ return grub_error (GRUB_ERR_UNKNOWN_DEVICE, "unknown NVMe disk");
+}
+
+static void
+grub_nvme_close (grub_disk_t disk)
+{
+ (void) disk;
+}
+
+static grub_err_t
+nvme_readwrite (struct grub_nvme_device *dev, int namespace,
+ grub_disk_addr_t sector, grub_size_t size, char *buf, int is_write)
+{
+ (void) namespace;
+
+ if (size == 0)
+ return 0;
+
+ if (size > 512)
+ return grub_error(GRUB_ERR_BAD_ARGUMENT, "overlong nvme read");
+
+ /* This assumes virt == phys which is true on platforms where we support nvme. */
+ grub_uint64_t buffer_phys = (grub_addr_t) buf;
+
+ struct nvme_s_queue_entry e = {
+ .dw[0] = is_write ? 0x01 : 0x02,
+ .dw[1] = 0x1,
+ .dw[6] = (grub_addr_t) buffer_phys,
+ .dw[7] = (grub_addr_t) (buffer_phys >> 32),
+ .dw[10] = sector,
+ .dw[11] = sector >> 32,
+ .dw[12] = size - 1,
+ };
+
+ const grub_uint64_t start_page = buffer_phys >> 12;
+ const grub_uint64_t end_page = (buffer_phys + (size << dev->log_sector_size) - 1) >> 12;
+ if (end_page == start_page) {
+ /* No page crossing, PRP2 is reserved */
+ } else if (end_page == start_page + 1) {
+ /* Crossing exactly one page boundary, PRP2 is second page */
+ e.dw[8] = (buffer_phys + 0x1000) & ~0xfff;
+ } else {
+ /* Use a single page as PRP list, PRP2 points to the list */
+ unsigned int i;
+ volatile grub_uint64_t *prp_list = grub_dma_get_virt(dev->prp_list);
+ for (i = 0; i < end_page - start_page; ++i) {
+ buffer_phys += 0x1000;
+ prp_list[i] = buffer_phys & ~0xfff;
+ }
+ e.dw[8] = grub_dma_get_phys(dev->prp_list);
+ }
+
+ int io_err = nvme_cmd(dev, ios, &e);
+ if (io_err)
+ return grub_error(GRUB_ERR_IO, "NVMe error %d", io_err);
+ return GRUB_ERR_NONE;
+}
+
+static grub_err_t
+grub_nvme_read (grub_disk_t disk, grub_disk_addr_t sector,
+ grub_size_t size, char *buf)
+{
+ struct grub_nvme_device *dev = disk->data;
+ int namespace = disk->id & 0xff;
+
+ return nvme_readwrite(dev, namespace, sector, size, buf, 0);
+}
+
+static grub_err_t
+grub_nvme_write (grub_disk_t disk, grub_disk_addr_t sector,
+ grub_size_t size, const char *buf)
+{
+ struct grub_nvme_device *dev = disk->data;
+ int namespace = disk->id & 0xff;
+
+ return nvme_readwrite(dev, namespace, sector, size, (char *)buf, 1);
+}
+
+static struct grub_disk_dev grub_nvme_dev =
+ {
+ .name = "nvme",
+ .id = GRUB_DISK_DEVICE_NVME_ID,
+ .disk_iterate = grub_nvme_iterate,
+ .disk_open = grub_nvme_open,
+ .disk_close = grub_nvme_close,
+ .disk_read = grub_nvme_read,
+ .disk_write = grub_nvme_write,
+ .next = 0
+ };
+
+\f
+
+static struct grub_preboot *fini_hnd;
+
+GRUB_MOD_INIT(nvme)
+{
+ grub_stop_disk_firmware ();
+
+ /* NVMe initialization. */
+ grub_nvme_initialize ();
+
+ grub_disk_dev_register (&grub_nvme_dev);
+
+ fini_hnd = grub_loader_register_preboot_hook (grub_nvme_fini_hw,
+ grub_nvme_restore_hw,
+ GRUB_LOADER_PREBOOT_HOOK_PRIO_DISK);
+}
+
+GRUB_MOD_FINI(nvme)
+{
+ grub_nvme_fini_hw (0);
+ grub_loader_unregister_preboot_hook (fini_hnd);
+
+ grub_disk_dev_unregister (&grub_nvme_dev);
+}
diff --git a/include/grub/disk.h b/include/grub/disk.h
index fbf23df7f..fce52c207 100644
--- a/include/grub/disk.h
+++ b/include/grub/disk.h
@@ -52,6 +52,7 @@ enum grub_disk_dev_id
GRUB_DISK_DEVICE_UBOOTDISK_ID,
GRUB_DISK_DEVICE_XEN,
GRUB_DISK_DEVICE_OBDISK_ID,
+ GRUB_DISK_DEVICE_NVME_ID,
};
struct grub_disk;
--
2.39.2
_______________________________________________
Grub-devel mailing list
Grub-devel@gnu.org
https://lists.gnu.org/mailman/listinfo/grub-devel
^ permalink raw reply related [flat|nested] only message in thread
only message in thread, other threads:[~2024-05-16 18:50 UTC | newest]
Thread overview: (only message) (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2024-05-16 18:49 [PATCH] Implement NVMe native disk support Vladimir Serbinenko
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.