* [SPDK] [RFC PATCH 1/4] changed vhost to support multi-process access same target
@ 2018-05-11 3:25 Ming Lin
0 siblings, 0 replies; only message in thread
From: Ming Lin @ 2018-05-11 3:25 UTC (permalink / raw)
To: spdk
[-- Attachment #1: Type: text/plain, Size: 34341 bytes --]
From: Sheng Qiu <sheng.qiu(a)alibaba-inc.com>
Signed-off-by: Sheng Qiu <sheng.qiu(a)alibaba-inc.com>
---
lib/vhost/vhost.c | 69 +++++++-
lib/vhost/vhost_internal.h | 7 +
lib/vhost/vhost_scsi.c | 411 +++++++++++++++++++++++++++------------------
3 files changed, 316 insertions(+), 171 deletions(-)
diff --git a/lib/vhost/vhost.c b/lib/vhost/vhost.c
index 0d49f59..bd3473e 100644
--- a/lib/vhost/vhost.c
+++ b/lib/vhost/vhost.c
@@ -456,6 +456,29 @@ spdk_vhost_dev_find_id(const char *ctrlr_name)
return -1;
}
+static int
+spdk_vhost_dev_find_id_all(const char *ctrlr_name, int *all_id) {
+ unsigned i, vdev_num = 0;
+ size_t dev_dirname_len = strlen(dev_dirname);
+
+ if (strncmp(ctrlr_name, dev_dirname, dev_dirname_len) == 0) {
+ ctrlr_name += dev_dirname_len;
+ }
+
+ for (i = 0; i < MAX_VHOST_DEVICES; i++) {
+ if (g_spdk_vhost_devices[i] == NULL) {
+ continue;
+ }
+
+ if (strcmp(g_spdk_vhost_devices[i]->name, ctrlr_name) == 0) {
+ all_id[vdev_num++] = i;
+ assert(vdev_num <= MAX_CONNECT_DEVS);
+ }
+ }
+
+ return vdev_num;
+}
+
struct spdk_vhost_dev *
spdk_vhost_dev_find(const char *ctrlr_name)
{
@@ -469,6 +492,22 @@ spdk_vhost_dev_find(const char *ctrlr_name)
return g_spdk_vhost_devices[id];
}
+int
+spdk_vhost_dev_find_all(const char *ctrlr_name, struct spdk_vhost_dev **vdevs)
+{
+ int all_id[MAX_CONNECT_DEVS];
+ int vdev_num = 0, i;
+
+ vdev_num = spdk_vhost_dev_find_id_all(ctrlr_name, all_id);
+ assert(vdev_num <= MAX_CONNECT_DEVS);
+
+ for (i = 0; i < vdev_num; i++) {
+ vdevs[i] = g_spdk_vhost_devices[all_id[i]];
+ }
+
+ return vdev_num;
+}
+
static int
spdk_vhost_parse_core_mask(const char *mask, uint64_t *cpumask)
{
@@ -509,6 +548,7 @@ spdk_vhost_dev_construct(struct spdk_vhost_dev *vdev, const char *name, const ch
struct stat file_stat;
char buf[64];
uint64_t cpumask;
+ bool socket_started = false;
assert(vdev);
@@ -523,10 +563,11 @@ spdk_vhost_dev_construct(struct spdk_vhost_dev *vdev, const char *name, const ch
return -EINVAL;
}
+
if (spdk_vhost_dev_find(name)) {
- SPDK_ERRLOG("vhost controller %s already exists.\n", name);
- return -EEXIST;
- }
+ socket_started = true;
+ }
+
for (ctrlr_num = 0; ctrlr_num < MAX_VHOST_DEVICES; ctrlr_num++) {
if (g_spdk_vhost_devices[ctrlr_num] == NULL) {
@@ -545,6 +586,7 @@ spdk_vhost_dev_construct(struct spdk_vhost_dev *vdev, const char *name, const ch
return -EINVAL;
}
+ if (!socket_started) {
/* Register vhost driver to handle vhost messages. */
if (stat(path, &file_stat) != -1) {
if (!S_ISSOCK(file_stat.st_mode)) {
@@ -578,6 +620,7 @@ spdk_vhost_dev_construct(struct spdk_vhost_dev *vdev, const char *name, const ch
SPDK_ERRLOG("Couldn't register callbacks for controller %s\n", name);
return -EIO;
}
+ }
vdev->name = strdup(name);
vdev->path = strdup(path);
@@ -595,6 +638,7 @@ spdk_vhost_dev_construct(struct spdk_vhost_dev *vdev, const char *name, const ch
g_spdk_vhost_devices[ctrlr_num] = vdev;
+ if (!socket_started) {
if (rte_vhost_driver_start(path) != 0) {
spdk_strerror_r(errno, buf, sizeof(buf));
SPDK_ERRLOG("Failed to start vhost driver for controller %s (%d): %s\n", name, errno,
@@ -604,6 +648,7 @@ spdk_vhost_dev_construct(struct spdk_vhost_dev *vdev, const char *name, const ch
}
SPDK_NOTICELOG("Controller %s: new controller added\n", vdev->name);
+ }
return 0;
}
@@ -935,6 +980,10 @@ start_device(int vid)
}
out:
+ if (rc == 0) {
+ assert(vdev->status & 1);
+ vdev->status |= 2;
+ }
pthread_mutex_unlock(&g_spdk_vhost_mutex);
return rc;
}
@@ -1058,14 +1107,18 @@ new_connection(int vid)
return -1;
}
- /* since pollers are not running it safe not to use spdk_event here */
if (vdev->vid != -1) {
- SPDK_ERRLOG("Device with vid %d is already connected.\n", vid);
- pthread_mutex_unlock(&g_spdk_vhost_mutex);
- return -1;
- }
+ vdev = spdk_vhost_dev_find_next(vdev);
+ if (!vdev) {
+ SPDK_ERRLOG("Couldn't find unused vdev for vid %d to create connection for.\n", vid);
+ pthread_mutex_unlock(&g_spdk_vhost_mutex);
+ return -1;
+ }
+ }
vdev->vid = vid;
+ assert(vdev->status == 0);
+ vdev->status |= 1;
pthread_mutex_unlock(&g_spdk_vhost_mutex);
return 0;
}
diff --git a/lib/vhost/vhost_internal.h b/lib/vhost/vhost_internal.h
index 20931b4..0861287 100644
--- a/lib/vhost/vhost_internal.h
+++ b/lib/vhost/vhost_internal.h
@@ -67,6 +67,8 @@
#define SPDK_VHOST_IOVS_MAX 128
+#define MAX_CONNECT_DEVS 64
+
/*
* Rate at which stats are checked for interrupt coalescing.
*/
@@ -134,6 +136,7 @@ struct spdk_vhost_dev_backend {
int (*vhost_remove_controller)(struct spdk_vhost_dev *vdev);
};
+struct spdk_vhost_scsi_dev;
struct spdk_vhost_dev {
struct rte_vhost_memory *mem;
char *name;
@@ -146,6 +149,8 @@ struct spdk_vhost_dev {
enum spdk_vhost_dev_type type;
const struct spdk_vhost_dev_backend *backend;
+ struct spdk_vhost_scsi_dev *svdev;
+ uint8_t status;
uint32_t coalescing_delay_time_base;
@@ -166,6 +171,8 @@ struct spdk_vhost_dev {
};
struct spdk_vhost_dev *spdk_vhost_dev_find(const char *ctrlr_name);
+struct spdk_vhost_dev *spdk_vhost_dev_find_next(struct spdk_vhost_dev *vdev);
+int spdk_vhost_dev_find_all(const char *ctrlr_name, struct spdk_vhost_dev **vdevs);
void spdk_vhost_dev_mem_register(struct spdk_vhost_dev *vdev);
void spdk_vhost_dev_mem_unregister(struct spdk_vhost_dev *vdev);
diff --git a/lib/vhost/vhost_scsi.c b/lib/vhost/vhost_scsi.c
index 488d4d1..b9da29e 100644
--- a/lib/vhost/vhost_scsi.c
+++ b/lib/vhost/vhost_scsi.c
@@ -74,7 +74,9 @@ struct spdk_scsi_dev_vhost_state {
};
struct spdk_vhost_scsi_dev {
- struct spdk_vhost_dev vdev;
+ struct spdk_vhost_dev vdev[MAX_CONNECT_DEVS];
+ int conn_vdev_num;
+ bool scsi_dev_initialized;
struct spdk_scsi_dev *scsi_dev[SPDK_VHOST_SCSI_CTRLR_MAX_DEVS];
struct spdk_scsi_dev_vhost_state scsi_dev_state[SPDK_VHOST_SCSI_CTRLR_MAX_DEVS];
@@ -95,6 +97,7 @@ struct spdk_vhost_scsi_task {
struct spdk_scsi_dev *scsi_dev;
int req_idx;
+ int vdev_idx;
/* If set, the task is currently used for I/O processing. */
bool used;
@@ -115,6 +118,23 @@ const struct spdk_vhost_dev_backend spdk_vhost_scsi_device_backend = {
.vhost_remove_controller = spdk_vhost_scsi_dev_remove,
};
+struct spdk_vhost_dev*
+spdk_vhost_dev_find_next(struct spdk_vhost_dev *vdev)
+{
+ struct spdk_vhost_scsi_dev *svdev = vdev->svdev;
+ int i;
+
+ assert(svdev->conn_vdev_num > 0 && svdev->conn_vdev_num < MAX_CONNECT_DEVS);
+
+ for (i = 0; i < MAX_CONNECT_DEVS; i++) {
+ if (svdev->vdev[i].status == 0) {
+ return &svdev->vdev[i];
+ }
+ }
+
+ return NULL;
+}
+
static void
spdk_vhost_scsi_task_put(struct spdk_vhost_scsi_task *task)
{
@@ -125,9 +145,10 @@ static void
spdk_vhost_scsi_task_free_cb(struct spdk_scsi_task *scsi_task)
{
struct spdk_vhost_scsi_task *task = SPDK_CONTAINEROF(scsi_task, struct spdk_vhost_scsi_task, scsi);
+ int vdev_idx = task->vdev_idx;
- assert(task->svdev->vdev.task_cnt > 0);
- task->svdev->vdev.task_cnt--;
+ assert(task->svdev->vdev[vdev_idx].task_cnt > 0);
+ task->svdev->vdev[vdev_idx].task_cnt--;
task->used = false;
}
@@ -147,16 +168,20 @@ process_removed_devs(struct spdk_vhost_scsi_dev *svdev)
spdk_scsi_dev_destruct(dev);
svdev->scsi_dev[i] = NULL;
if (state->remove_cb) {
- state->remove_cb(&svdev->vdev, state->remove_ctx);
+ /* notify and detach from all impact vdev */
+ int j;
+ for (j = 0; j < MAX_CONNECT_DEVS; j++) {
+ state->remove_cb(&svdev->vdev[j], state->remove_ctx);
+ SPDK_NOTICELOG("%s: hot-detached device 'Dev %u'.\n", svdev->vdev[j].name, i);
+ }
state->remove_cb = NULL;
}
- SPDK_NOTICELOG("%s: hot-detached device 'Dev %u'.\n", svdev->vdev.name, i);
}
}
}
static void
-eventq_enqueue(struct spdk_vhost_scsi_dev *svdev, unsigned scsi_dev_num, uint32_t event,
+eventq_enqueue(struct spdk_vhost_dev* vdev, unsigned scsi_dev_num, uint32_t event,
uint32_t reason)
{
struct spdk_vhost_virtqueue *vq;
@@ -167,25 +192,25 @@ eventq_enqueue(struct spdk_vhost_scsi_dev *svdev, unsigned scsi_dev_num, uint32_
int rc;
assert(scsi_dev_num < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS);
- vq = &svdev->vdev.virtqueue[VIRTIO_SCSI_EVENTQ];
+ vq = &vdev->virtqueue[VIRTIO_SCSI_EVENTQ];
if (spdk_vhost_vq_avail_ring_get(vq, &req, 1) != 1) {
SPDK_ERRLOG("Controller %s: Failed to send virtio event (no avail ring entries?).\n",
- svdev->vdev.name);
+ vdev->name);
return;
}
- rc = spdk_vhost_vq_get_desc(&svdev->vdev, vq, req, &desc, &desc_table, &desc_table_size);
+ rc = spdk_vhost_vq_get_desc(vdev, vq, req, &desc, &desc_table, &desc_table_size);
if (rc != 0 || desc->len < sizeof(*desc_ev)) {
SPDK_ERRLOG("Controller %s: Invalid eventq descriptor at index %"PRIu16".\n",
- svdev->vdev.name, req);
+ vdev->name, req);
goto out;
}
- desc_ev = spdk_vhost_gpa_to_vva(&svdev->vdev, desc->addr);
+ desc_ev = spdk_vhost_gpa_to_vva(vdev, desc->addr);
if (desc_ev == NULL) {
SPDK_ERRLOG("Controller %s: Eventq descriptor at index %"PRIu16" points to unmapped guest memory address %p.\n",
- svdev->vdev.name, req, (void *)(uintptr_t)desc->addr);
+ vdev->name, req, (void *)(uintptr_t)desc->addr);
goto out;
}
@@ -205,13 +230,14 @@ eventq_enqueue(struct spdk_vhost_scsi_dev *svdev, unsigned scsi_dev_num, uint32_
req_size = sizeof(*desc_ev);
out:
- spdk_vhost_vq_used_ring_enqueue(&svdev->vdev, vq, req, req_size);
+ spdk_vhost_vq_used_ring_enqueue(vdev, vq, req, req_size);
}
static void
submit_completion(struct spdk_vhost_scsi_task *task)
{
- spdk_vhost_vq_used_ring_enqueue(&task->svdev->vdev, task->vq, task->req_idx,
+ int vdev_idx = task->vdev_idx;
+ spdk_vhost_vq_used_ring_enqueue(&task->svdev->vdev[vdev_idx], task->vq, task->req_idx,
task->scsi.data_transferred);
SPDK_DEBUGLOG(SPDK_LOG_VHOST_SCSI, "Finished task (%p) req_idx=%d\n", task, task->req_idx);
@@ -262,7 +288,8 @@ mgmt_task_submit(struct spdk_vhost_scsi_task *task, enum spdk_scsi_task_func fun
static void
invalid_request(struct spdk_vhost_scsi_task *task)
{
- spdk_vhost_vq_used_ring_enqueue(&task->svdev->vdev, task->vq, task->req_idx, 0);
+ int vdev_idx = task->vdev_idx;
+ spdk_vhost_vq_used_ring_enqueue(&task->svdev->vdev[vdev_idx], task->vq, task->req_idx, 0);
spdk_vhost_scsi_task_put(task);
SPDK_DEBUGLOG(SPDK_LOG_VHOST_SCSI, "Invalid request (status=%" PRIu8")\n",
@@ -299,7 +326,8 @@ spdk_vhost_scsi_task_init_target(struct spdk_vhost_scsi_task *task, const __u8 *
static void
process_ctrl_request(struct spdk_vhost_scsi_task *task)
{
- struct spdk_vhost_dev *vdev = &task->svdev->vdev;
+ int vdev_idx = task->vdev_idx;
+ struct spdk_vhost_dev *vdev = &task->svdev->vdev[vdev_idx];
struct vring_desc *desc, *desc_table;
struct virtio_scsi_ctrl_tmf_req *ctrl_req;
struct virtio_scsi_ctrl_an_resp *an_resp;
@@ -395,7 +423,8 @@ static int
task_data_setup(struct spdk_vhost_scsi_task *task,
struct virtio_scsi_cmd_req **req)
{
- struct spdk_vhost_dev *vdev = &task->svdev->vdev;
+ int vdev_idx = task->vdev_idx;
+ struct spdk_vhost_dev *vdev = &task->svdev->vdev[vdev_idx];
struct vring_desc *desc, *desc_table;
struct iovec *iovs = task->iovs;
uint16_t iovcnt = 0, iovcnt_max = SPDK_VHOST_IOVS_MAX;
@@ -557,7 +586,7 @@ process_request(struct spdk_vhost_scsi_task *task)
}
static void
-process_controlq(struct spdk_vhost_scsi_dev *svdev, struct spdk_vhost_virtqueue *vq)
+process_controlq(struct spdk_vhost_scsi_dev *svdev, struct spdk_vhost_virtqueue *vq, int idx)
{
struct spdk_vhost_scsi_task *task;
uint16_t reqs[32];
@@ -567,29 +596,30 @@ process_controlq(struct spdk_vhost_scsi_dev *svdev, struct spdk_vhost_virtqueue
for (i = 0; i < reqs_cnt; i++) {
if (spdk_unlikely(reqs[i] >= vq->vring.size)) {
SPDK_ERRLOG("%s: invalid entry in avail ring. Buffer '%"PRIu16"' exceeds virtqueue size (%"PRIu16")\n",
- svdev->vdev.name, reqs[i], vq->vring.size);
- spdk_vhost_vq_used_ring_enqueue(&svdev->vdev, vq, reqs[i], 0);
+ svdev->vdev[idx].name, reqs[i], vq->vring.size);
+ spdk_vhost_vq_used_ring_enqueue(&svdev->vdev[idx], vq, reqs[i], 0);
continue;
}
task = &((struct spdk_vhost_scsi_task *)vq->tasks)[reqs[i]];
if (spdk_unlikely(task->used)) {
SPDK_ERRLOG("%s: invalid entry in avail ring. Buffer '%"PRIu16"' is still in use!\n",
- svdev->vdev.name, reqs[i]);
- spdk_vhost_vq_used_ring_enqueue(&svdev->vdev, vq, reqs[i], 0);
+ svdev->vdev[idx].name, reqs[i]);
+ spdk_vhost_vq_used_ring_enqueue(&svdev->vdev[idx], vq, reqs[i], 0);
continue;
}
- svdev->vdev.task_cnt++;
+ svdev->vdev[idx].task_cnt++;
memset(&task->scsi, 0, sizeof(task->scsi));
task->tmf_resp = NULL;
task->used = true;
+ task->vdev_idx = idx;
process_ctrl_request(task);
}
}
static void
-process_requestq(struct spdk_vhost_scsi_dev *svdev, struct spdk_vhost_virtqueue *vq)
+process_requestq(struct spdk_vhost_scsi_dev *svdev, struct spdk_vhost_virtqueue *vq, int idx)
{
struct spdk_vhost_scsi_task *task;
uint16_t reqs[32];
@@ -605,23 +635,24 @@ process_requestq(struct spdk_vhost_scsi_dev *svdev, struct spdk_vhost_virtqueue
if (spdk_unlikely(reqs[i] >= vq->vring.size)) {
SPDK_ERRLOG("%s: request idx '%"PRIu16"' exceeds virtqueue size (%"PRIu16").\n",
- svdev->vdev.name, reqs[i], vq->vring.size);
- spdk_vhost_vq_used_ring_enqueue(&svdev->vdev, vq, reqs[i], 0);
+ svdev->vdev[idx].name, reqs[i], vq->vring.size);
+ spdk_vhost_vq_used_ring_enqueue(&svdev->vdev[idx], vq, reqs[i], 0);
continue;
}
task = &((struct spdk_vhost_scsi_task *)vq->tasks)[reqs[i]];
if (spdk_unlikely(task->used)) {
SPDK_ERRLOG("%s: request with idx '%"PRIu16"' is already pending.\n",
- svdev->vdev.name, reqs[i]);
- spdk_vhost_vq_used_ring_enqueue(&svdev->vdev, vq, reqs[i], 0);
+ svdev->vdev[idx].name, reqs[i]);
+ spdk_vhost_vq_used_ring_enqueue(&svdev->vdev[idx], vq, reqs[i], 0);
continue;
}
- svdev->vdev.task_cnt++;
+ svdev->vdev[idx].task_cnt++;
memset(&task->scsi, 0, sizeof(task->scsi));
task->resp = NULL;
task->used = true;
+ task->vdev_idx = idx;
result = process_request(task);
if (likely(result == 0)) {
task_submit(task);
@@ -643,12 +674,18 @@ static void
vdev_mgmt_worker(void *arg)
{
struct spdk_vhost_scsi_dev *svdev = arg;
-
- process_removed_devs(svdev);
- spdk_vhost_vq_used_signal(&svdev->vdev, &svdev->vdev.virtqueue[VIRTIO_SCSI_EVENTQ]);
-
- process_controlq(svdev, &svdev->vdev.virtqueue[VIRTIO_SCSI_CONTROLQ]);
- spdk_vhost_vq_used_signal(&svdev->vdev, &svdev->vdev.virtqueue[VIRTIO_SCSI_CONTROLQ]);
+ int i;
+
+ process_removed_devs(svdev);
+ /* notify all existed vdev */
+ for (i = 0; i < MAX_CONNECT_DEVS; i++) {
+ /* only process live vdev */
+ if (svdev->vdev[i].status == 3) {
+ spdk_vhost_vq_used_signal(&svdev->vdev[i], &svdev->vdev[i].virtqueue[VIRTIO_SCSI_EVENTQ]);
+ process_controlq(svdev, &svdev->vdev[i].virtqueue[VIRTIO_SCSI_CONTROLQ], i);
+ spdk_vhost_vq_used_signal(&svdev->vdev[i], &svdev->vdev[i].virtqueue[VIRTIO_SCSI_CONTROLQ]);
+ }
+ }
}
static void
@@ -656,12 +693,17 @@ vdev_worker(void *arg)
{
struct spdk_vhost_scsi_dev *svdev = arg;
uint32_t q_idx;
-
- for (q_idx = VIRTIO_SCSI_REQUESTQ; q_idx < svdev->vdev.num_queues; q_idx++) {
- process_requestq(svdev, &svdev->vdev.virtqueue[q_idx]);
- }
-
- spdk_vhost_dev_used_signal(&svdev->vdev);
+ int i;
+
+ for (i = 0; i < MAX_CONNECT_DEVS; i++) {
+ /* only process live vdev */
+ if (svdev->vdev[i].status == 3) {
+ for (q_idx = VIRTIO_SCSI_REQUESTQ; q_idx < svdev->vdev[i].num_queues; q_idx++) {
+ process_requestq(svdev, &svdev->vdev[i].virtqueue[q_idx], i);
+ }
+ spdk_vhost_dev_used_signal(&svdev->vdev[i]);
+ }
+ }
}
static struct spdk_vhost_scsi_dev *
@@ -677,7 +719,7 @@ to_scsi_dev(struct spdk_vhost_dev *ctrlr)
return NULL;
}
- return (struct spdk_vhost_scsi_dev *)ctrlr;
+ return ctrlr->svdev;
}
int
@@ -685,20 +727,24 @@ spdk_vhost_scsi_dev_construct(const char *name, const char *cpumask)
{
struct spdk_vhost_scsi_dev *svdev = spdk_dma_zmalloc(sizeof(struct spdk_vhost_scsi_dev),
SPDK_CACHE_LINE_SIZE, NULL);
- int rc;
+ int rc, i;
if (svdev == NULL) {
return -ENOMEM;
}
+ assert(svdev->conn_vdev_num == 0);
spdk_vhost_lock();
- rc = spdk_vhost_dev_construct(&svdev->vdev, name, cpumask, SPDK_VHOST_DEV_T_SCSI,
- &spdk_vhost_scsi_device_backend);
-
- if (rc) {
- spdk_dma_free(svdev);
- }
-
+ for (i = 0; i < MAX_CONNECT_DEVS; i++) {
+ rc = spdk_vhost_dev_construct(&svdev->vdev[i], name, cpumask, SPDK_VHOST_DEV_T_SCSI,
+ &spdk_vhost_scsi_device_backend);
+
+ if (rc) {
+ spdk_dma_free(svdev);
+ break;
+ }
+ svdev->vdev[i].svdev = svdev;
+ }
spdk_vhost_unlock();
return rc;
}
@@ -745,31 +791,35 @@ spdk_vhost_scsi_lun_hotremove(const struct spdk_scsi_lun *lun, void *arg)
{
struct spdk_vhost_scsi_dev *svdev = arg;
const struct spdk_scsi_dev *scsi_dev;
- unsigned scsi_dev_num;
+ unsigned scsi_dev_num, i;
assert(lun != NULL);
assert(svdev != NULL);
- if (!spdk_vhost_dev_has_feature(&svdev->vdev, VIRTIO_SCSI_F_HOTPLUG)) {
- SPDK_WARNLOG("%s: hotremove is not enabled for this controller.\n", svdev->vdev.name);
- return;
- }
-
- scsi_dev = spdk_scsi_lun_get_dev(lun);
- for (scsi_dev_num = 0; scsi_dev_num < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; scsi_dev_num++) {
- if (svdev->scsi_dev[scsi_dev_num] == scsi_dev) {
- break;
- }
- }
-
- if (scsi_dev_num == SPDK_VHOST_SCSI_CTRLR_MAX_DEVS) {
- SPDK_ERRLOG("Dev %s is not a part of vhost scsi controller '%s'.\n",
- spdk_scsi_dev_get_name(scsi_dev),
- svdev->vdev.name);
- return;
- }
-
- /* remove entire device */
- spdk_vhost_scsi_dev_remove_tgt(&svdev->vdev, scsi_dev_num, NULL, NULL);
+
+ /* find the scsi_dev to be remove */
+ scsi_dev = spdk_scsi_lun_get_dev(lun);
+ for (scsi_dev_num = 0; scsi_dev_num < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; scsi_dev_num++) {
+ if (svdev->scsi_dev[scsi_dev_num] == scsi_dev) {
+ break;
+ }
+ }
+
+ if (scsi_dev_num == SPDK_VHOST_SCSI_CTRLR_MAX_DEVS) {
+ SPDK_ERRLOG("Dev %s is not a part of vhost scsi controller '%s'.\n",
+ spdk_scsi_dev_get_name(scsi_dev),
+ svdev->vdev[0].name);
+ return;
+ }
+
+ for (i = 0; i < MAX_CONNECT_DEVS; i++) {
+ if (!spdk_vhost_dev_has_feature(&svdev->vdev[i], VIRTIO_SCSI_F_HOTPLUG)) {
+ SPDK_WARNLOG("%s: hotremove is not enabled for this controller.\n", svdev->vdev[i].name);
+ continue;
+ }
+
+ /* remove entire device */
+ spdk_vhost_scsi_dev_remove_tgt(&svdev->vdev[i], scsi_dev_num, NULL, NULL);
+ }
}
int
@@ -786,56 +836,65 @@ spdk_vhost_scsi_dev_add_tgt(struct spdk_vhost_dev *vdev, unsigned scsi_tgt_num,
return -EINVAL;
}
- if (scsi_tgt_num >= SPDK_VHOST_SCSI_CTRLR_MAX_DEVS) {
- SPDK_ERRLOG("Controller %d target number too big (max %d)\n", scsi_tgt_num,
- SPDK_VHOST_SCSI_CTRLR_MAX_DEVS);
- return -EINVAL;
- }
-
- if (lun_name == NULL) {
- SPDK_ERRLOG("No lun name specified \n");
- return -EINVAL;
- } else if (strlen(lun_name) >= SPDK_SCSI_DEV_MAX_NAME) {
- SPDK_ERRLOG("LUN name '%s' too long (max %d).\n", lun_name, SPDK_SCSI_DEV_MAX_NAME - 1);
- return -1;
- }
+ if (!svdev->scsi_dev_initialized) {
+ if (scsi_tgt_num >= SPDK_VHOST_SCSI_CTRLR_MAX_DEVS) {
+ SPDK_ERRLOG("Controller %d target number too big (max %d)\n", scsi_tgt_num,
+ SPDK_VHOST_SCSI_CTRLR_MAX_DEVS);
+ return -EINVAL;
+ }
+
+ if (lun_name == NULL) {
+ SPDK_ERRLOG("No lun name specified \n");
+ return -EINVAL;
+ } else if (strlen(lun_name) >= SPDK_SCSI_DEV_MAX_NAME) {
+ SPDK_ERRLOG("LUN name '%s' too long (max %d).\n", lun_name, SPDK_SCSI_DEV_MAX_NAME - 1);
+ return -1;
+ }
+ }
if (vdev->lcore != -1 && !spdk_vhost_dev_has_feature(vdev, VIRTIO_SCSI_F_HOTPLUG)) {
SPDK_ERRLOG("Controller %s is in use and hotplug is not supported\n", vdev->name);
return -ENOTSUP;
}
- if (svdev->scsi_dev[scsi_tgt_num] != NULL) {
- SPDK_ERRLOG("Controller %s target %u already occupied\n", vdev->name, scsi_tgt_num);
- return -EEXIST;
- }
-
- /*
- * At this stage only one LUN per target
- */
- snprintf(target_name, sizeof(target_name), "Target %u", scsi_tgt_num);
- lun_id_list[0] = 0;
- lun_names_list[0] = (char *)lun_name;
-
- svdev->scsi_dev_state[scsi_tgt_num].removed = false;
- svdev->scsi_dev[scsi_tgt_num] = spdk_scsi_dev_construct(target_name, lun_names_list, lun_id_list,
- 1,
- SPDK_SPC_PROTOCOL_IDENTIFIER_SAS, spdk_vhost_scsi_lun_hotremove, svdev);
-
- if (svdev->scsi_dev[scsi_tgt_num] == NULL) {
- SPDK_ERRLOG("Couldn't create spdk SCSI target '%s' using lun device '%s' in controller: %s\n",
- target_name, lun_name, vdev->name);
- return -EINVAL;
- }
- spdk_scsi_dev_add_port(svdev->scsi_dev[scsi_tgt_num], 0, "vhost");
+ if (!svdev->scsi_dev_initialized) {
+ if (svdev->scsi_dev[scsi_tgt_num] != NULL) {
+ SPDK_ERRLOG("Controller %s target %u already occupied\n", vdev->name, scsi_tgt_num);
+ return -EEXIST;
+ }
+
+ /*
+ * At this stage only one LUN per target
+ */
+ snprintf(target_name, sizeof(target_name), "Target %u", scsi_tgt_num);
+ lun_id_list[0] = 0;
+ lun_names_list[0] = (char *)lun_name;
+
+ svdev->scsi_dev_state[scsi_tgt_num].removed = false;
+ svdev->scsi_dev[scsi_tgt_num] = spdk_scsi_dev_construct(target_name, lun_names_list, lun_id_list,
+ 1,
+ SPDK_SPC_PROTOCOL_IDENTIFIER_SAS, spdk_vhost_scsi_lun_hotremove, svdev);
+
+ if (svdev->scsi_dev[scsi_tgt_num] == NULL) {
+ SPDK_ERRLOG("Couldn't create spdk SCSI target '%s' using lun device '%s' in controller: %s\n",
+ target_name, lun_name, vdev->name);
+ return -EINVAL;
+ }
+ spdk_scsi_dev_add_port(svdev->scsi_dev[scsi_tgt_num], 0, "vhost");
+ }
if (vdev->lcore != -1) {
- spdk_scsi_dev_allocate_io_channels(svdev->scsi_dev[scsi_tgt_num]);
- eventq_enqueue(svdev, scsi_tgt_num, VIRTIO_SCSI_T_TRANSPORT_RESET, VIRTIO_SCSI_EVT_RESET_RESCAN);
+ if (!svdev->scsi_dev_initialized) {
+ spdk_scsi_dev_allocate_io_channels(svdev->scsi_dev[scsi_tgt_num]);
+ }
+ eventq_enqueue(vdev, scsi_tgt_num, VIRTIO_SCSI_T_TRANSPORT_RESET, VIRTIO_SCSI_EVT_RESET_RESCAN);
}
- SPDK_NOTICELOG("Controller %s: defined target '%s' using lun '%s'\n",
- vdev->name, target_name, lun_name);
+ if (!svdev->scsi_dev_initialized) {
+ SPDK_NOTICELOG("Controller %s: defined target '%s' using lun '%s'\n",
+ vdev->name, target_name, lun_name);
+ svdev->scsi_dev_initialized = true;
+ }
return 0;
}
@@ -864,7 +923,7 @@ spdk_vhost_scsi_dev_remove_tgt(struct spdk_vhost_dev *vdev, unsigned scsi_tgt_nu
return -ENODEV;
}
- if (svdev->vdev.lcore == -1) {
+ if (vdev->lcore == -1) {
/* controller is not in use, remove dev and exit */
spdk_scsi_dev_destruct(scsi_dev);
svdev->scsi_dev[scsi_tgt_num] = NULL;
@@ -877,13 +936,13 @@ spdk_vhost_scsi_dev_remove_tgt(struct spdk_vhost_dev *vdev, unsigned scsi_tgt_nu
if (!spdk_vhost_dev_has_feature(vdev, VIRTIO_SCSI_F_HOTPLUG)) {
SPDK_WARNLOG("%s: 'Target %u' is in use and hot-detach is not enabled for this controller.\n",
- svdev->vdev.name, scsi_tgt_num);
+ vdev->name, scsi_tgt_num);
return -ENOTSUP;
}
scsi_dev_state = &svdev->scsi_dev_state[scsi_tgt_num];
if (scsi_dev_state->removed) {
- SPDK_WARNLOG("%s: 'Target %u' has been already marked to hotremove.\n", svdev->vdev.name,
+ SPDK_WARNLOG("%s: 'Target %u' has been already marked to hotremove.\n", vdev->name,
scsi_tgt_num);
return -EBUSY;
}
@@ -891,7 +950,7 @@ spdk_vhost_scsi_dev_remove_tgt(struct spdk_vhost_dev *vdev, unsigned scsi_tgt_nu
scsi_dev_state->remove_cb = cb_fn;
scsi_dev_state->remove_ctx = cb_arg;
scsi_dev_state->removed = true;
- eventq_enqueue(svdev, scsi_tgt_num, VIRTIO_SCSI_T_TRANSPORT_RESET, VIRTIO_SCSI_EVT_RESET_REMOVED);
+ eventq_enqueue(vdev, scsi_tgt_num, VIRTIO_SCSI_T_TRANSPORT_RESET, VIRTIO_SCSI_EVT_RESET_REMOVED);
SPDK_NOTICELOG("%s: queued 'Target %u' for hot-detach.\n", vdev->name, scsi_tgt_num);
return 0;
@@ -901,8 +960,8 @@ int
spdk_vhost_scsi_controller_construct(void)
{
struct spdk_conf_section *sp = spdk_conf_first_section(NULL);
- struct spdk_vhost_dev *vdev;
- int i, dev_num;
+ struct spdk_vhost_dev *vdev[MAX_CONNECT_DEVS];
+ int i, dev_num, vdev_num = 0;
unsigned ctrlr_num = 0;
char *lun_name, *tgt_num_str;
char *cpumask;
@@ -929,8 +988,9 @@ spdk_vhost_scsi_controller_construct(void)
return -1;
}
- vdev = spdk_vhost_dev_find(name);
- assert(vdev);
+ /* multiple vdev supported */
+ vdev_num = spdk_vhost_dev_find_all(name, vdev);
+ assert(vdev_num > 0);
dev = spdk_conf_section_get_nval(sp, "Dev", 0);
tgt = spdk_conf_section_get_nval(sp, "Target", 0);
@@ -969,10 +1029,13 @@ spdk_vhost_scsi_controller_construct(void)
SPDK_ERRLOG("%s: Only one LUN per vhost SCSI device supported\n", name);
return -1;
}
-
- if (spdk_vhost_scsi_dev_add_tgt(vdev, dev_num, lun_name) < 0) {
- return -1;
- }
+
+ int j;
+ for (j = 0; j < vdev_num; j++) {
+ if (spdk_vhost_scsi_dev_add_tgt(vdev[j], dev_num, lun_name) < 0) {
+ return -1;
+ }
+ }
}
sp = spdk_conf_next_section(sp);
@@ -982,13 +1045,13 @@ spdk_vhost_scsi_controller_construct(void)
}
static void
-free_task_pool(struct spdk_vhost_scsi_dev *svdev)
+free_task_pool(struct spdk_vhost_dev *vdev)
{
struct spdk_vhost_virtqueue *vq;
uint16_t i;
- for (i = 0; i < svdev->vdev.num_queues; i++) {
- vq = &svdev->vdev.virtqueue[i];
+ for (i = 0; i < vdev->num_queues; i++) {
+ vq = &vdev->virtqueue[i];
if (vq->tasks == NULL) {
continue;
}
@@ -999,7 +1062,7 @@ free_task_pool(struct spdk_vhost_scsi_dev *svdev)
}
static int
-alloc_task_pool(struct spdk_vhost_scsi_dev *svdev)
+alloc_task_pool(struct spdk_vhost_scsi_dev *svdev, struct spdk_vhost_dev *vdev)
{
struct spdk_vhost_virtqueue *vq;
struct spdk_vhost_scsi_task *task;
@@ -1007,23 +1070,23 @@ alloc_task_pool(struct spdk_vhost_scsi_dev *svdev)
uint16_t i;
uint32_t j;
- for (i = 0; i < svdev->vdev.num_queues; i++) {
- vq = &svdev->vdev.virtqueue[i];
+ for (i = 0; i < vdev->num_queues; i++) {
+ vq = &vdev->virtqueue[i];
task_cnt = vq->vring.size;
if (task_cnt > SPDK_VHOST_MAX_VQ_SIZE) {
/* sanity check */
SPDK_ERRLOG("Controller %s: virtuque %"PRIu16" is too big. (size = %"PRIu32", max = %"PRIu32")\n",
- svdev->vdev.name, i, task_cnt, SPDK_VHOST_MAX_VQ_SIZE);
- free_task_pool(svdev);
+ vdev->name, i, task_cnt, SPDK_VHOST_MAX_VQ_SIZE);
+ free_task_pool(vdev);
return -1;
}
vq->tasks = spdk_dma_zmalloc(sizeof(struct spdk_vhost_scsi_task) * task_cnt,
SPDK_CACHE_LINE_SIZE, NULL);
if (vq->tasks == NULL) {
SPDK_ERRLOG("Controller %s: failed to allocate %"PRIu32" tasks for virtqueue %"PRIu16"\n",
- svdev->vdev.name, task_cnt, i);
- free_task_pool(svdev);
- return -1;
+ vdev->name, task_cnt, i);
+ free_task_pool(vdev);
+ return -1;
}
for (j = 0; j < task_cnt; j++) {
@@ -1055,25 +1118,30 @@ spdk_vhost_scsi_start(struct spdk_vhost_dev *vdev, void *event_ctx)
goto out;
}
- rc = alloc_task_pool(svdev);
+ rc = alloc_task_pool(svdev, vdev);
if (rc != 0) {
SPDK_ERRLOG("%s: failed to alloc task pool.\n", vdev->name);
goto out;
}
- for (i = 0; i < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; i++) {
- if (svdev->scsi_dev[i] == NULL) {
- continue;
- }
- spdk_scsi_dev_allocate_io_channels(svdev->scsi_dev[i]);
- }
SPDK_NOTICELOG("Started poller for vhost controller %s on lcore %d\n", vdev->name, vdev->lcore);
spdk_vhost_dev_mem_register(vdev);
- svdev->requestq_poller = spdk_poller_register(vdev_worker, svdev, 0);
- svdev->mgmt_poller = spdk_poller_register(vdev_mgmt_worker, svdev,
- MGMT_POLL_PERIOD_US);
+ /* register svdev poller if it's not registered */
+ if (svdev->conn_vdev_num == 0) {
+ for (i = 0; i < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; i++) {
+ if (svdev->scsi_dev[i] == NULL) {
+ continue;
+ }
+ spdk_scsi_dev_allocate_io_channels(svdev->scsi_dev[i]);
+ }
+
+ svdev->requestq_poller = spdk_poller_register(vdev_worker, svdev, 0);
+ svdev->mgmt_poller = spdk_poller_register(vdev_mgmt_worker, svdev,
+ MGMT_POLL_PERIOD_US);
+ }
+ svdev->conn_vdev_num++;
out:
spdk_vhost_dev_backend_event_done(event_ctx, rc);
return rc;
@@ -1081,6 +1149,7 @@ out:
struct spdk_vhost_dev_destroy_ctx {
struct spdk_vhost_scsi_dev *svdev;
+ struct spdk_vhost_dev *vdev;
struct spdk_poller *poller;
void *event_ctx;
};
@@ -1090,29 +1159,33 @@ destroy_device_poller_cb(void *arg)
{
struct spdk_vhost_dev_destroy_ctx *ctx = arg;
struct spdk_vhost_scsi_dev *svdev = ctx->svdev;
+ struct spdk_vhost_dev *vdev = ctx->vdev;
uint32_t i;
- if (svdev->vdev.task_cnt > 0) {
+ if (vdev->task_cnt > 0) {
return;
}
-
- for (i = 0; i < svdev->vdev.num_queues; i++) {
- spdk_vhost_vq_used_signal(&svdev->vdev, &svdev->vdev.virtqueue[i]);
- }
-
- for (i = 0; i < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; i++) {
- if (svdev->scsi_dev[i] == NULL) {
- continue;
- }
- spdk_scsi_dev_free_io_channels(svdev->scsi_dev[i]);
+ assert(vdev->status == 3);
+ for (i = 0; i < vdev->num_queues; i++) {
+ spdk_vhost_vq_used_signal(vdev, &vdev->virtqueue[i]);
}
- SPDK_NOTICELOG("Stopping poller for vhost controller %s\n", svdev->vdev.name);
- spdk_vhost_dev_mem_unregister(&svdev->vdev);
-
- free_task_pool(svdev);
-
+ if (svdev->conn_vdev_num == 0) {
+ for (i = 0; i < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; i++) {
+ if (svdev->scsi_dev[i] == NULL) {
+ continue;
+ }
+ spdk_scsi_dev_free_io_channels(svdev->scsi_dev[i]);
+ }
+ }
+
+ SPDK_NOTICELOG("Stopping poller for vhost controller %s\n", vdev->name);
+ spdk_vhost_dev_mem_unregister(vdev);
+
+ free_task_pool(vdev);
+ vdev->status = 0;
+
spdk_poller_unregister(&ctx->poller);
spdk_vhost_dev_backend_event_done(ctx->event_ctx, 0);
spdk_dma_free(ctx);
@@ -1130,6 +1203,10 @@ spdk_vhost_scsi_stop(struct spdk_vhost_dev *vdev, void *event_ctx)
goto err;
}
+ /* update active connected vdev count */
+ svdev->conn_vdev_num--;
+ assert(svdev->conn_vdev_num >= 0);
+
destroy_ctx = spdk_dma_zmalloc(sizeof(*destroy_ctx), SPDK_CACHE_LINE_SIZE, NULL);
if (destroy_ctx == NULL) {
SPDK_ERRLOG("Failed to alloc memory for destroying device.\n");
@@ -1137,10 +1214,18 @@ spdk_vhost_scsi_stop(struct spdk_vhost_dev *vdev, void *event_ctx)
}
destroy_ctx->svdev = svdev;
+ destroy_ctx->vdev = vdev;
destroy_ctx->event_ctx = event_ctx;
- spdk_poller_unregister(&svdev->requestq_poller);
- spdk_poller_unregister(&svdev->mgmt_poller);
+ /* stop svdev poller only when no live vdev exist */
+ if (svdev->conn_vdev_num == 0) {
+ SPDK_NOTICELOG("Stopping svdev, live vdev = %d\n", svdev->conn_vdev_num);
+ spdk_poller_unregister(&svdev->requestq_poller);
+ spdk_poller_unregister(&svdev->mgmt_poller);
+ } else {
+ SPDK_NOTICELOG("Stopping one vdev, live vdev = %d\n", svdev->conn_vdev_num);
+ }
+
destroy_ctx->poller = spdk_poller_register(destroy_device_poller_cb, destroy_ctx,
1000);
--
1.9.1
^ permalink raw reply related [flat|nested] only message in thread
only message in thread, other threads:[~2018-05-11 3:25 UTC | newest]
Thread overview: (only message) (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2018-05-11 3:25 [SPDK] [RFC PATCH 1/4] changed vhost to support multi-process access same target Ming Lin
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox