[PATCH 6/8] virtio-usb: rework endpoint lifecycle to an async split-phase state machine

[Igor Skalkin <igor.skalkin@oss.qualcomm.com>]

The USB gadget API's usb_ep_ops/usb_gadget_ops callbacks are documented
as atomic - they must not sleep. This driver's endpoint and command
handling needs to send a virtio command and wait for the backend's
response, which is exactly what those callbacks cannot do.

We previously worked around this with a small out-of-tree patch to
the gadget core itself: a gadget.nonatomic flag that, when set, swapped
the UDC core's spinlocks for mutexes so a driver could sleep in these
callbacks. That patch was never going to pass upstream review, so it
had to go - which meant this driver needed to actually honor the
atomic contract instead of relying on a modified core.

This commit is that fix: an explicit endpoint state machine
(EP_DISABLED, EP_ENABLING, EP_ENABLED, EP_HALTED) plus per-UDC/
per-request work items and a pending-request queue, so every
usb_ep_ops/usb_gadget_ops callback returns immediately and the actual
virtio round trip happens asynchronously on a workqueue.
ep_enable()/ep_disable()/set_selfpowered()/vbus_draw() all move to
this split-phase pattern; ep_queue()/ep_dequeue() are reworked on top
of the same state machine. gadget.nonatomic itself is removed, since
nothing needs it anymore.

Also fixes a handful of bugs found while doing this: preserve
-ECONNRESET/-ESHUTDOWN completion statuses instead of always
overwriting them, support early-completed requests without a double
giveback(), serialize SETUP handling against in-flight EP0 completions
and still-EP_ENABLING endpoints to avoid overlapping control transfers
exposed by dummy_hcd's tighter timing, and fix EP0's descriptor
initialization and a spinlock-unbalance bug in ep_enable().

Signed-off-by: Igor Skalkin <igor.skalkin@oss.qualcomm.com>
---
 drivers/usb/virtio_usb/device.c |  774 +++++++++++++++++++++++++++++++---------
 drivers/usb/virtio_usb/device.h |   24 +
 2 files changed, 639 insertions(+), 159 deletions(-)

diff --git a/drivers/usb/virtio_usb/device.c b/drivers/usb/virtio_usb/device.c
index 5ae0fc8..9ee1f05 100644
--- a/drivers/usb/virtio_usb/device.c
+++ b/drivers/usb/virtio_usb/device.c
@@ -11,6 +11,7 @@
 #include "device.h"
 
 #define GADGET_NAME "virtio_usb_dc"
+#define VIRTIO_USB_EP0_IDLE_TIMEOUT_MS 5000
 
 /**
  * struct virtio_usb_dc_priv - Device controller data priv
@@ -23,6 +24,8 @@ struct virtio_usb_dc_priv {
 	struct usb_request req;
 	struct virtio_usb_ep *vep;
 	struct virtio_usb_data *vreq;
+	struct work_struct cancel_work;
+	bool completed_early;
 };
 
 /**
@@ -72,6 +75,34 @@ static struct virtio_usb_data *usb_req_to_virtio_data(struct usb_request *req)
 	return priv->vreq;
 }
 
+static bool virtio_usb_dc_req_queue_empty(struct virtio_usb_ep *vep)
+{
+	struct virtio_usb_dc *vudc = vep->vudc;
+	unsigned long flags;
+	bool empty;
+
+	spin_lock_irqsave(&vudc->lock, flags);
+	empty = list_empty(&vep->req_queue);
+	spin_unlock_irqrestore(&vudc->lock, flags);
+
+	return empty;
+}
+
+static int virtio_usb_dc_wait_ep0_idle(struct virtio_usb_dc *vudc)
+{
+	struct virtio_usb_ep *vep = usb_ep_to_virtio_ep(vudc->gadget.ep0);
+	long ret;
+
+	if (virtio_usb_dc_req_queue_empty(vep))
+		return 0;
+
+	ret = wait_event_timeout(
+		vudc->ep0_idle, virtio_usb_dc_req_queue_empty(vep),
+		msecs_to_jiffies(VIRTIO_USB_EP0_IDLE_TIMEOUT_MS));
+
+	return ret ? 0 : -ETIMEDOUT;
+}
+
 /**
  * virtio_usb_dc_complete_req() - Completes a usb request
  * @vreq: virtio usb data message.
@@ -89,6 +120,8 @@ static void virtio_usb_dc_complete_req(struct virtio_usb_data *vreq)
 	struct virtio_usb_data *vreq_iter;
 	unsigned int found = 0;
 	unsigned long flags;
+	bool wake_ep0 = false;
+	bool early;
 
 	spin_lock_irqsave(&vudc->lock, flags);
 
@@ -98,10 +131,15 @@ static void virtio_usb_dc_complete_req(struct virtio_usb_data *vreq)
 			break;
 		}
 	}
+	early = priv->completed_early;
 	if (!found) {
 		spin_unlock_irqrestore(&vudc->lock, flags);
+		/* Arrived after ep_disable() already gave back */
+		if (early)
+			virtio_usb_data_unref(vreq);
 		return;
 	}
+	/* Common case - request still belongs to in-flight queue */
 	if (req->status != -ECONNRESET && req->status != -ESHUTDOWN) {
 		req->status = virtio_error_to_usb(status);
 		if (!req->status)
@@ -109,9 +147,17 @@ static void virtio_usb_dc_complete_req(struct virtio_usb_data *vreq)
 	}
 
 	list_del_init(&vreq->list);
+	wake_ep0 = !vep->ep_id && list_empty(&vep->req_queue);
 	spin_unlock_irqrestore(&vudc->lock, flags);
+	if (wake_ep0)
+		wake_up(&vudc->ep0_idle);
+
+	/* Normal completion: giveback now */
+	if (!early)
+		usb_gadget_giveback_request(&vep->ep, req);
+
+	/* Drop final reference */
 	virtio_usb_data_unref(vreq);
-	usb_gadget_giveback_request(&vep->ep, req);
 }
 
 /**
@@ -128,22 +174,24 @@ void virtio_usb_dc_data_work(struct work_struct *work)
 	struct virtio_usb_queue *dataq =
 		&vusb->vqueues[vusb->dev_vq_base + VIRTIO_USB_VQ_DATA_IDX];
 	struct virtio_usb_data *vreq;
+	unsigned long flags;
 	unsigned int length;
 
-	spin_lock_irq(&dataq->lock);
+	spin_lock_irqsave(&dataq->lock, flags);
 	do {
 		virtqueue_disable_cb(dataq->vqueue);
 		while ((vreq = virtqueue_get_buf(dataq->vqueue, &length))) {
-			spin_unlock_irq(&dataq->lock);
+			spin_unlock_irqrestore(&dataq->lock, flags);
 			virtio_usb_dc_complete_req(vreq);
-			spin_lock_irq(&dataq->lock);
+			spin_lock_irqsave(&dataq->lock, flags);
 		}
 		if (unlikely(virtqueue_is_broken(dataq->vqueue)))
 			break;
 	} while (!virtqueue_enable_cb(dataq->vqueue));
-	spin_unlock_irq(&dataq->lock);
+	spin_unlock_irqrestore(&dataq->lock, flags);
 }
 
+static void virtio_usb_dc_cancel_work(struct work_struct *work);
 /**
  * virtio_usb_dc_data_alloc() - Allocate and initialize a device controller
  * data message.
@@ -174,6 +222,8 @@ static struct virtio_usb_data *virtio_usb_dc_data_alloc(struct usb_ep *ep,
 	priv = virtio_usb_data_priv(vreq);
 	priv->vep = vep;
 	priv->vreq = vreq;
+	priv->completed_early = false;
+	INIT_WORK(&priv->cancel_work, virtio_usb_dc_cancel_work);
 	vreq->msg.queue =
 		&vep->vudc->vusb->vqueues[vep->vudc->vusb->dev_vq_base +
 					  VIRTIO_USB_VQ_DATA_IDX];
@@ -242,6 +292,264 @@ virtio_usb_dc_cmd_alloc(struct virtio_usb_dc *vudc, unsigned int command,
 
 /* Endpoint callbacks */
 
+static int do_real_queue(struct virtio_usb_ep *vep, struct usb_request *req)
+{
+	struct virtio_usb_dc *vudc = vep->vudc;
+	struct virtio_usb_data *vreq = usb_req_to_virtio_data(req);
+	struct virtio_usb_dc_priv *priv = virtio_usb_data_priv(vreq);
+	struct virtio_usb_request *request;
+	struct virtio_usb_response *response;
+	struct scatterlist *out_sgs = NULL, *in_sgs = NULL, *psg_data = NULL;
+	struct scatterlist sg;
+	unsigned long flags;
+	u16 endpoint;
+	u16 transfer_flags = 0;
+	u16 transfer_type = VIRTIO_USB_EP_CONTROL;
+	int rc;
+
+	virtio_usb_data_ref(vreq);
+
+	request = virtio_usb_data_request(vreq);
+	response = virtio_usb_data_response(vreq);
+	response->actual_length = cpu_to_le32(0);
+	response->status = cpu_to_le32(VIRTIO_USB_S_ERR_INTERNAL);
+
+	if (req->short_not_ok)
+		transfer_flags |= VIRTIO_USB_FLAG_SHORT_NOT_OK;
+	else if (req->zero)
+		transfer_flags |= VIRTIO_USB_FLAG_ZERO_PACKET;
+	request->transfer_flags = cpu_to_le16(transfer_flags);
+
+	endpoint = vep->ep_id |
+		   (usb_ep_dir_in(&vep->ep) ? VIRTIO_USB_EP_DIR_IN :
+					      VIRTIO_USB_EP_DIR_OUT);
+	request->endpoint = cpu_to_le16(endpoint);
+	request->port = cpu_to_le16(vudc->port);
+	if (vep->ep_id && vep->ep.desc) {
+		switch (usb_endpoint_type(vep->ep.desc)) {
+		case USB_ENDPOINT_XFER_ISOC:
+			transfer_type = VIRTIO_USB_EP_ISOCHRONOUS;
+			break;
+		case USB_ENDPOINT_XFER_BULK:
+			transfer_type = VIRTIO_USB_EP_BULK;
+			break;
+		case USB_ENDPOINT_XFER_INT:
+			transfer_type = VIRTIO_USB_EP_INTERRUPT;
+			break;
+		default:
+			transfer_type = VIRTIO_USB_EP_CONTROL;
+			break;
+		}
+	}
+	request->transfer_type = cpu_to_le16(transfer_type);
+
+	if (req->length && req->buf) {
+		psg_data = &sg;
+		sg_init_one(psg_data, req->buf, req->length);
+	} else if (req->length && req->num_sgs > 0) {
+		psg_data = req->sg;
+	} else if (req->length && req->sg) {
+		psg_data = &sg;
+		sg_init_one(psg_data, sg_virt(req->sg), req->length);
+	} else {
+		psg_data = NULL;
+	}
+
+	if (usb_ep_dir_in(&vep->ep))
+		out_sgs = psg_data;
+	else
+		in_sgs = psg_data;
+
+	spin_lock_irqsave(&vudc->lock, flags);
+	priv->completed_early = false;
+	req->actual = 0;
+	req->status = -EINPROGRESS;
+	list_add_tail(&vreq->list, &vep->req_queue);
+	spin_unlock_irqrestore(&vudc->lock, flags);
+
+	rc = virtio_usb_data_send(vudc->vusb, vreq, out_sgs, in_sgs);
+	if (rc) {
+		spin_lock_irqsave(&vudc->lock, flags);
+		list_del_init(&vreq->list);
+		spin_unlock_irqrestore(&vudc->lock, flags);
+
+		virtio_usb_data_unref(vreq);
+		return rc;
+	}
+
+	return 0;
+}
+
+static bool virtio_usb_dc_ep_enabling_locked(struct virtio_usb_dc *vudc)
+{
+	u32 i;
+
+	for (i = 1; i < vudc->neps; i++) {
+		if (vudc->veps[i].state == EP_ENABLING)
+			return true;
+	}
+
+	return false;
+}
+
+static void virtio_usb_dc_flush_ep0_pend(struct virtio_usb_dc *vudc)
+{
+	struct virtio_usb_ep *vep = &vudc->veps[0];
+	struct virtio_usb_data *vreq;
+	unsigned long flags;
+
+	spin_lock_irqsave(&vudc->lock, flags);
+	if (virtio_usb_dc_ep_enabling_locked(vudc)) {
+		spin_unlock_irqrestore(&vudc->lock, flags);
+		return;
+	}
+
+	while (!list_empty(&vep->pend_queue)) {
+		struct virtio_usb_dc_priv *priv;
+		int qrc;
+
+		vreq = list_first_entry(&vep->pend_queue,
+					struct virtio_usb_data, list);
+		priv = virtio_usb_data_priv(vreq);
+		list_del_init(&vreq->list);
+		spin_unlock_irqrestore(&vudc->lock, flags);
+
+		qrc = do_real_queue(vep, &priv->req);
+		virtio_usb_data_unref(vreq);
+		if (qrc) {
+			priv->req.status = qrc;
+			usb_gadget_giveback_request(&vep->ep, &priv->req);
+		}
+
+		spin_lock_irqsave(&vudc->lock, flags);
+	}
+	spin_unlock_irqrestore(&vudc->lock, flags);
+}
+
+static bool
+virtio_usb_ep_comp_valid(const struct usb_ss_ep_comp_descriptor *comp)
+{
+	return comp && comp->bLength == USB_DT_SS_EP_COMP_SIZE &&
+	       comp->bDescriptorType == USB_DT_SS_ENDPOINT_COMP;
+}
+
+static void vep_enable_work(struct work_struct *work)
+{
+	struct virtio_usb_ep *vep =
+		container_of(work, struct virtio_usb_ep, enable_work);
+	struct virtio_usb_dc *vudc = vep->vudc;
+	const struct usb_endpoint_descriptor *desc = vep->ep.desc;
+	struct scatterlist sg;
+	struct scatterlist *psg_data = &sg;
+	struct virtio_usb_dev_cmd_hdr *hdr;
+	struct virtio_usb_cmd *cmd;
+	unsigned long flags;
+	u16 endpoint;
+	int rc;
+
+	if (!desc)
+		return;
+
+	endpoint = vep->ep_id |
+		   (usb_ep_dir_in(&vep->ep) ? VIRTIO_USB_EP_DIR_IN :
+					      VIRTIO_USB_EP_DIR_OUT);
+
+	cmd = virtio_usb_dc_cmd_alloc(vudc, VIRTIO_USB_CMD_DEV_EP_ENABLE,
+				      GFP_KERNEL);
+	if (!cmd) {
+		rc = -ENOMEM;
+		goto done;
+	}
+
+	sg_init_one(psg_data, desc, sizeof(*desc));
+	hdr = virtio_usb_cmd_request(cmd);
+	hdr->endpoint = cpu_to_le16(endpoint);
+
+	rc = virtio_usb_cmd_send_sync(vudc->vusb, psg_data, NULL, cmd);
+
+done:
+	spin_lock_irqsave(&vudc->lock, flags);
+	if (!rc) {
+		vep->state = EP_ENABLED;
+		vep->last_err = 0;
+		/* Flush pend_queue */
+		while (!list_empty(&vep->pend_queue)) {
+			struct virtio_usb_data *vreq = list_first_entry(
+				&vep->pend_queue, struct virtio_usb_data, list);
+			struct virtio_usb_dc_priv *priv =
+				virtio_usb_data_priv(vreq);
+			int qrc;
+
+			list_del_init(&vreq->list);
+			spin_unlock_irqrestore(&vudc->lock, flags);
+			qrc = do_real_queue(vep, &priv->req);
+			/*
+			 * do_real_queue() takes its own ref for the
+			 * in-flight/vring lifetime; drop the ref that kept
+			 * vreq alive while it was parked in pend_queue.
+			 */
+			virtio_usb_data_unref(vreq);
+			if (qrc) {
+				/*
+				 * Request was accepted while EP was ENABLING.
+				 * If forwarding now fails, complete it with
+				 * error instead of silently dropping it.
+				 */
+				priv->req.status = qrc;
+				usb_gadget_giveback_request(&vep->ep,
+							    &priv->req);
+			}
+			spin_lock_irqsave(&vudc->lock, flags);
+		}
+	} else {
+		vep->state = EP_HALTED;
+		vep->last_err = rc;
+
+		while (!list_empty(&vep->pend_queue)) {
+			struct virtio_usb_data *vreq = list_first_entry(
+				&vep->pend_queue, struct virtio_usb_data, list);
+			struct virtio_usb_dc_priv *priv =
+				virtio_usb_data_priv(vreq);
+
+			list_del_init(&vreq->list);
+			priv->req.status = rc;
+			spin_unlock_irqrestore(&vudc->lock, flags);
+			virtio_usb_data_unref(vreq);
+			usb_gadget_giveback_request(&vep->ep, &priv->req);
+			spin_lock_irqsave(&vudc->lock, flags);
+		}
+	}
+	spin_unlock_irqrestore(&vudc->lock, flags);
+	virtio_usb_dc_flush_ep0_pend(vudc);
+}
+
+static void vep_disable_work(struct work_struct *work)
+{
+	struct virtio_usb_ep *vep =
+		container_of(work, struct virtio_usb_ep, disable_work);
+	struct virtio_usb_dc *vudc = vep->vudc;
+	struct virtio_usb_cmd *cmd;
+	struct virtio_usb_dev_cmd_hdr *hdr;
+	u16 endpoint;
+
+	if (!READ_ONCE(vudc->driver) || !vudc->registered)
+		return;
+
+	endpoint = vep->ep_id |
+		   (usb_ep_dir_in(&vep->ep) ? VIRTIO_USB_EP_DIR_IN :
+					      VIRTIO_USB_EP_DIR_OUT);
+
+	cmd = virtio_usb_dc_cmd_alloc(vudc, VIRTIO_USB_CMD_DEV_EP_DISABLE,
+				      GFP_KERNEL);
+	if (!cmd)
+		return;
+
+	hdr = virtio_usb_cmd_request(cmd);
+	hdr->endpoint = cpu_to_le16(endpoint);
+
+	(void)virtio_usb_cmd_send_sync(vudc->vusb, NULL, NULL, cmd);
+}
+
 /**
  * virtio_ep_enable() - Enable endpoint
  * This callback is called to configure endpoint and make it usable.
@@ -256,44 +564,30 @@ static int virtio_ep_enable(struct usb_ep *ep,
 {
 	struct virtio_usb_ep *vep = usb_ep_to_virtio_ep(ep);
 	struct virtio_usb_dc *vudc = vep->vudc;
-	struct scatterlist sg;
-	struct scatterlist *psg_data = &sg;
-	struct virtio_usb_dev_cmd_hdr *hdr;
-	struct virtio_usb_cmd *cmd;
 	unsigned long flags;
-	u16 endpoint;
-	int rc;
-
-	if (!vudc->driver)
-		return -ESHUTDOWN;
 
 	spin_lock_irqsave(&vudc->lock, flags);
 
-	if (!ep || !desc || ep->caps.type_control ||
+	if (!vudc->driver || !ep || !desc || ep->caps.type_control ||
 	    desc->bDescriptorType != USB_DT_ENDPOINT) {
 		spin_unlock_irqrestore(&vudc->lock, flags);
 		return -EINVAL;
 	}
 
+	if (vep->state != EP_DISABLED) {
+		spin_unlock_irqrestore(&vudc->lock, flags);
+		return -EBUSY;
+	}
+
 	ep->desc = desc;
 	ep->maxpacket = usb_endpoint_maxp(desc);
-	endpoint = vep->ep_id | (usb_ep_dir_in(ep) ? VIRTIO_USB_EP_DIR_IN :
-						     VIRTIO_USB_EP_DIR_OUT);
-
-	spin_unlock_irqrestore(&vudc->lock, flags);
-
-	cmd = virtio_usb_dc_cmd_alloc(vudc, VIRTIO_USB_CMD_DEV_EP_ENABLE,
-				      GFP_ATOMIC);
-	if (!cmd)
-		return -ENOMEM;
+	vep->state = EP_ENABLING;
 
-	sg_init_one(psg_data, desc, sizeof(*desc));
-	hdr = virtio_usb_cmd_request(cmd);
-	hdr->endpoint = cpu_to_le16(endpoint);
+	queue_work(vudc->ep_cmd_wq, &vep->enable_work);
 
-	rc = virtio_usb_cmd_send_sync(vudc->vusb, psg_data, NULL, cmd);
+	spin_unlock_irqrestore(&vudc->lock, flags);
 
-	return rc;
+	return 0;
 }
 
 /**
@@ -311,42 +605,82 @@ static int virtio_ep_disable(struct usb_ep *ep)
 	struct virtio_usb_data *vreq = NULL, *vreq_tmp;
 	struct virtio_usb_dc *vudc = vep->vudc;
 	struct virtio_usb_dc_priv *priv = NULL;
-	struct virtio_usb_dev_cmd_hdr *hdr;
-	struct virtio_usb_cmd *cmd;
+	LIST_HEAD(giveback_inflight);
+	LIST_HEAD(giveback_pended);
 	unsigned long flags;
-	u16 endpoint;
-	int rc;
 
-	if (!ep || ep->caps.type_control)
+	if (!ep || ep->caps.type_control) // EP0 not called disable
 		return -EINVAL;
 
 	spin_lock_irqsave(&vudc->lock, flags);
 
+	if (vep->state == EP_ENABLING) {
+		/*
+		 * cancel_work_sync() can sleep and must not be called while
+		 * holding vudc->lock: vep_enable_work() only re-acquires
+		 * this same lock after its (possibly blocking) virtio
+		 * command completes, so holding the lock here across
+		 * cancel_work_sync() would deadlock against it.
+		 */
+		spin_unlock_irqrestore(&vudc->lock, flags);
+		cancel_work_sync(&vep->enable_work);
+		spin_lock_irqsave(&vudc->lock, flags);
+	}
+
+	if (vep->state == EP_DISABLED) {
+		spin_unlock_irqrestore(&vudc->lock, flags);
+		return 0;
+	}
+
+	vep->state = EP_DISABLED;
+
+	/*
+	 * All pending (waiting for the ENABLING) - put to local list for
+	 * giveback(-ESHUTDOWN). These requests are not in the virtqueue,
+	 * can be unref now.
+	 */
+	list_for_each_entry_safe(vreq, vreq_tmp, &vep->pend_queue, list) {
+		list_del_init(&vreq->list);
+		list_add_tail(&vreq->list, &giveback_pended);
+	}
+
+	/*
+	 * in-flight requests - make early giveback, remove from req_queue,
+	 * mark completed_early, but do not unref: unref will be in complete
+	 * handler.
+	 */
 	list_for_each_entry_safe(vreq, vreq_tmp, &vep->req_queue, list) {
 		priv = virtio_usb_data_priv(vreq);
-		/**
-		 * When endpoint is disabled, completion handler for all pending
-		 * requests will be called. Make the request status to -ESHUTDOWN
-		 * to prevent requests completes even before the endpoint disable
-		 * is send to the controller.
-		 */
 		priv->req.status = -ESHUTDOWN;
+		priv->completed_early = true;
+		list_del_init(&vreq->list);
+		list_add_tail(&vreq->list, &giveback_inflight);
 	}
-	endpoint = vep->ep_id | (usb_ep_dir_in(ep) ? VIRTIO_USB_EP_DIR_IN :
-						     VIRTIO_USB_EP_DIR_OUT);
+
 	spin_unlock_irqrestore(&vudc->lock, flags);
 
-	cmd = virtio_usb_dc_cmd_alloc(vudc, VIRTIO_USB_CMD_DEV_EP_DISABLE,
-				      GFP_ATOMIC);
-	if (!cmd)
-		return -ENOMEM;
+	/* pending: just giveback and unref */
+	list_for_each_entry_safe(vreq, vreq_tmp, &giveback_pended, list) {
+		struct virtio_usb_dc_priv *p = virtio_usb_data_priv(vreq);
 
-	hdr = virtio_usb_cmd_request(cmd);
-	hdr->endpoint = cpu_to_le16(endpoint);
+		list_del_init(&vreq->list);
+		p->req.status = -ESHUTDOWN;
+		usb_gadget_giveback_request(&vep->ep, &p->req);
+		virtio_usb_data_unref(
+			vreq); /* drop the pend_queue ref taken in virtio_ep_queue() */
+	}
 
-	rc = virtio_usb_cmd_send_sync(vudc->vusb, NULL, NULL, cmd);
+	/* in-flight: giveback now, unref will be in complete_req */
+	list_for_each_entry_safe(vreq, vreq_tmp, &giveback_inflight, list) {
+		struct virtio_usb_dc_priv *p = virtio_usb_data_priv(vreq);
 
-	return rc;
+		list_del_init(&vreq->list);
+		usb_gadget_giveback_request(&vep->ep, &p->req);
+	}
+
+	queue_work(vudc->ep_cmd_wq, &vep->disable_work);
+
+	return 0;
 }
 
 /**
@@ -395,75 +729,52 @@ static void virtio_ep_free_request(struct usb_ep *ep, struct usb_request *req)
 static int virtio_ep_queue(struct usb_ep *ep, struct usb_request *req,
 			   gfp_t mem_flags)
 {
-	struct scatterlist *out_sgs = NULL, *in_sgs = NULL, *psg_data;
 	struct virtio_usb_data *vreq = usb_req_to_virtio_data(req);
 	struct virtio_usb_ep *vep = usb_ep_to_virtio_ep(ep);
 	struct virtio_usb_dc *vudc = vep->vudc;
-	struct virtio_usb_response *response;
-	struct virtio_usb_request *request;
-	u16 transfer_flags = 0;
-	struct scatterlist sg;
 	unsigned long flags;
-	u16 endpoint;
-	int rc;
-
-	virtio_usb_data_ref(vreq);
+	int rc = 0;
 
 	spin_lock_irqsave(&vudc->lock, flags);
-	req->actual = 0;
-	req->status = -EINPROGRESS;
-
-	list_add_tail(&vreq->list, &vep->req_queue);
-	endpoint = vep->ep_id | (usb_ep_dir_in(ep) ? VIRTIO_USB_EP_DIR_IN :
-						     VIRTIO_USB_EP_DIR_OUT);
-	spin_unlock_irqrestore(&vudc->lock, flags);
-
-	request = virtio_usb_data_request(vreq);
-	response = virtio_usb_data_response(vreq);
 
-	response->actual_length = cpu_to_le32(0);
-	response->status = cpu_to_le32(VIRTIO_USB_S_ERR_INTERNAL);
+	switch (vep->state) {
+	case EP_ENABLED:
+		if (!vep->ep_id && virtio_usb_dc_ep_enabling_locked(vudc)) {
+			virtio_usb_data_ref(vreq);
+			req->actual = 0;
+			req->status = -EINPROGRESS;
+			list_add_tail(&vreq->list, &vep->pend_queue);
+			spin_unlock_irqrestore(&vudc->lock, flags);
+			return 0;
+		}
+		spin_unlock_irqrestore(&vudc->lock, flags);
+		rc = do_real_queue(vep, req);
+		/* if rc==0 - unref will be in completion, on failure
+		 * do_real_queue() has already dropped its own ref
+		 */
+		return rc;
 
-	if (req->short_not_ok)
-		transfer_flags |= VIRTIO_USB_FLAG_SHORT_NOT_OK;
-	else if (req->zero)
-		transfer_flags |= VIRTIO_USB_FLAG_ZERO_PACKET;
+	case EP_ENABLING:
+		/* Waiting for ENABLED state */
+		virtio_usb_data_ref(vreq);
+		req->actual = 0;
+		req->status = -EINPROGRESS;
+		list_add_tail(&vreq->list, &vep->pend_queue);
+		spin_unlock_irqrestore(&vudc->lock, flags);
+		// Additional ref waited for the enable_work()
+		return 0;
 
-	request->transfer_flags = cpu_to_le16(transfer_flags);
-	request->endpoint = cpu_to_le16(endpoint);
-	request->port = cpu_to_le16(vudc->port);
+	case EP_HALTED:
+		spin_unlock_irqrestore(&vudc->lock, flags);
+		req->status = vep->last_err ? vep->last_err : -EPIPE;
+		usb_gadget_giveback_request(&vep->ep, req);
+		return req->status;
 
-	if (req->length && req->buf) {
-		psg_data = &sg;
-		sg_init_one(psg_data, req->buf, req->length);
-	} else if (req->length && req->num_sgs > 0) {
-		psg_data = req->sg;
-	} else if (req->sg) {
-		psg_data = &sg;
-		sg_init_one(psg_data, sg_virt(req->sg), req->length);
-	} else {
-		psg_data = NULL;
+	case EP_DISABLED:
+	default:
+		spin_unlock_irqrestore(&vudc->lock, flags);
+		return -ESHUTDOWN;
 	}
-
-	if (usb_ep_dir_in(ep))
-		out_sgs = psg_data;
-	else
-		in_sgs = psg_data;
-
-	rc = virtio_usb_data_send(vudc->vusb, vreq, out_sgs, in_sgs);
-	if (rc)
-		goto on_error_vq;
-
-	return rc;
-
-on_error_vq:
-	spin_lock_irqsave(&vudc->lock, flags);
-	list_del_init(&vreq->list);
-	spin_unlock_irqrestore(&vudc->lock, flags);
-
-	virtio_usb_data_unref(vreq);
-
-	return rc;
 }
 
 /**
@@ -477,20 +788,32 @@ on_error_vq:
 static int virtio_ep_dequeue(struct usb_ep *ep, struct usb_request *req)
 {
 	struct virtio_usb_ep *vep = usb_ep_to_virtio_ep(ep);
-	struct virtio_usb_dev_cmd_cancel *cancel;
 	struct virtio_usb_dc *vudc = vep->vudc;
 	struct virtio_usb_dc_priv *priv;
 	struct virtio_usb_data *vreq;
-	struct virtio_usb_cmd *cmd;
 	unsigned long flags;
 	int rc = -EINVAL;
-	u16 endpoint;
 
 	if (!vudc->driver)
 		return -ESHUTDOWN;
 
 	spin_lock_irqsave(&vudc->lock, flags);
 
+	/* If request is still pending (ENABLING) */
+	list_for_each_entry(vreq, &vep->pend_queue, list) {
+		priv = virtio_usb_data_priv(vreq);
+		if (&priv->req == req) {
+			list_del_init(&vreq->list);
+			req->status = -ECONNRESET;
+			spin_unlock_irqrestore(&vudc->lock, flags);
+
+			virtio_usb_data_unref(vreq);
+			usb_gadget_giveback_request(ep, req);
+			return 0;
+		}
+	}
+
+	/* in-flight - schedule async cancel */
 	list_for_each_entry(vreq, &vep->req_queue, list) {
 		priv = virtio_usb_data_priv(vreq);
 		if (req == &priv->req) {
@@ -502,28 +825,14 @@ static int virtio_ep_dequeue(struct usb_ep *ep, struct usb_request *req)
 			break;
 		}
 	}
-	endpoint = vep->ep_id | (usb_ep_dir_in(ep) ? VIRTIO_USB_EP_DIR_IN :
-						     VIRTIO_USB_EP_DIR_OUT);
 	spin_unlock_irqrestore(&vudc->lock, flags);
 
-	if (rc)
+	if (rc) // request not found in req_queue
 		return rc;
 
-	vreq = priv->vreq;
+	schedule_work(&priv->cancel_work);
 
-	cmd = virtio_usb_dc_cmd_alloc(vudc, VIRTIO_USB_CMD_DEV_CANCEL,
-				      GFP_KERNEL);
-	if (!cmd)
-		return -ENOMEM;
-
-	cancel = virtio_usb_cmd_request(cmd);
-	cancel->hdr.endpoint = cpu_to_le16(endpoint);
-	cancel->hdr.port = cpu_to_le16(vudc->port);
-	cancel->tag = cpu_to_le64((uintptr_t)vreq);
-
-	rc = virtio_usb_cmd_send_sync(vudc->vusb, NULL, NULL, cmd);
-
-	return rc;
+	return 0;
 }
 
 /**
@@ -633,6 +942,109 @@ static const struct usb_ep_ops virtio_usb_ep_ops = {
 /*-------------------------------------------------------------------------*/
 /* UDC callbacks */
 
+static void virtio_usb_dc_cancel_work(struct work_struct *work)
+{
+	struct virtio_usb_dc_priv *priv =
+		container_of(work, struct virtio_usb_dc_priv, cancel_work);
+	struct virtio_usb_ep *vep = priv->vep;
+	struct virtio_usb_dc *vudc = vep->vudc;
+	struct virtio_usb_cmd *cmd;
+	struct virtio_usb_dev_cmd_cancel *cancel;
+	struct virtio_usb_data *vreq = priv->vreq;
+	unsigned long flags;
+	u16 endpoint;
+	struct virtio_usb_data *iter;
+	bool found = false;
+
+	spin_lock_irqsave(&vudc->lock, flags);
+
+	list_for_each_entry(iter, &vep->req_queue, list) {
+		if (iter == vreq) {
+			found = true;
+			break;
+		}
+	}
+	if (!found) {
+		spin_unlock_irqrestore(&vudc->lock, flags);
+		return;
+	}
+	endpoint = vep->ep_id |
+		   (usb_ep_dir_in(&vep->ep) ? VIRTIO_USB_EP_DIR_IN :
+					      VIRTIO_USB_EP_DIR_OUT);
+	spin_unlock_irqrestore(&vudc->lock, flags);
+
+	cmd = virtio_usb_dc_cmd_alloc(vudc, VIRTIO_USB_CMD_DEV_CANCEL,
+				      GFP_KERNEL);
+	if (!cmd)
+		return;
+
+	cancel = virtio_usb_cmd_request(cmd);
+	cancel->hdr.endpoint = cpu_to_le16(endpoint);
+	cancel->hdr.port = cpu_to_le16(vudc->port);
+	cancel->tag = cpu_to_le64((uintptr_t)vreq);
+
+	(void)virtio_usb_cmd_send_sync(vudc->vusb, NULL, NULL, cmd);
+}
+
+static int virtio_usb_dc_send_cmd(struct virtio_usb_dc *vudc,
+				  unsigned int cmd_code, unsigned int val)
+{
+	struct virtio_usb_dev_cmd_set_value *req;
+	struct virtio_usb_cmd *cmd;
+
+	cmd = virtio_usb_dc_cmd_alloc(vudc, cmd_code, GFP_ATOMIC);
+	if (!cmd)
+		return -ENOMEM;
+
+	req = virtio_usb_cmd_request(cmd);
+	req->value = cpu_to_le32(val);
+
+	return virtio_usb_cmd_send_sync(vudc->vusb, NULL, NULL, cmd);
+}
+
+static void virtio_usb_dc_work(struct work_struct *work)
+{
+	struct virtio_usb_dc *vudc =
+		container_of(work, struct virtio_usb_dc, dc_cmd_work);
+	int sp, ma;
+
+	mutex_lock(&vudc->dc_cmd_work_lock);
+
+	sp = atomic_xchg(&vudc->pending_self_pwr, -1);
+	ma = atomic_xchg(&vudc->pending_vbus_ma, -1);
+
+	if (sp >= 0) {
+		bool new_sp = (sp != 0);
+
+		if (new_sp != vudc->cur_self_pwr) {
+			(void)virtio_usb_dc_send_cmd(
+				vudc, VIRTIO_USB_CMD_DEV_SET_SELF_POWERED,
+				new_sp);
+			vudc->cur_self_pwr = new_sp;
+		}
+	}
+
+	if (ma >= 0) {
+		unsigned int new_ma = (unsigned int)ma;
+
+		if (vudc->cur_self_pwr)
+			new_ma = 0;
+
+		if (new_ma != vudc->cur_vbus_ma) {
+			(void)virtio_usb_dc_send_cmd(
+				vudc, VIRTIO_USB_CMD_DEV_VBUS_DRAW, new_ma);
+			vudc->cur_vbus_ma = new_ma;
+		}
+	}
+
+	mutex_unlock(&vudc->dc_cmd_work_lock);
+
+	if (atomic_read(&vudc->pending_self_pwr) >= 0 ||
+	    atomic_read(&vudc->pending_vbus_ma) >= 0) {
+		queue_work(vudc->dc_cmd_wq, &vudc->dc_cmd_work);
+	}
+}
+
 /**
  * virtio_usb_dc_set_selfpowered() - Sets the device selfpowered feature.
  * @gadget: The device being declared as self-powered
@@ -645,18 +1057,11 @@ static int virtio_usb_dc_set_selfpowered(struct usb_gadget *gadget,
 {
 	struct virtio_usb_dc *vudc =
 		container_of(gadget, struct virtio_usb_dc, gadget);
-	unsigned int code = VIRTIO_USB_CMD_DEV_SET_SELF_POWERED;
-	struct virtio_usb_dev_cmd_set_value *req;
-	struct virtio_usb_cmd *cmd;
 
-	cmd = virtio_usb_dc_cmd_alloc(vudc, code, GFP_ATOMIC);
-	if (!cmd)
-		return -ENOMEM;
-
-	req = virtio_usb_cmd_request(cmd);
-	req->value = cpu_to_le32(!!is_selfpowered);
+	atomic_set(&vudc->pending_self_pwr, is_selfpowered ? 1 : 0);
+	queue_work(vudc->dc_cmd_wq, &vudc->dc_cmd_work);
 
-	return virtio_usb_cmd_send_sync(vudc->vusb, NULL, NULL, cmd);
+	return 0;
 }
 
 /**
@@ -774,29 +1179,18 @@ static int virtio_usb_dc_stop(struct usb_gadget *gadget)
 /**
  * virtio_usb_dc_vbus_draw() - Constrain controller's VBUS power usage
  * @gadget: The device whose VBUS usage is being described
- * @mA: How much current to draw, in milliAmperes.
+ * @ma: How much current to draw, in milliAmperes.
  *
  * Return: 0 on success, -errno on failure.
  */
-static int virtio_usb_dc_vbus_draw(struct usb_gadget *gadget, unsigned int mA)
+static int virtio_usb_dc_vbus_draw(struct usb_gadget *gadget, unsigned int ma)
 {
 	struct virtio_usb_dc *vudc =
 		container_of(gadget, struct virtio_usb_dc, gadget);
-	unsigned int code = VIRTIO_USB_CMD_DEV_VBUS_DRAW;
-	struct virtio_usb_dev_cmd_set_value *req;
-	struct virtio_usb_cmd *cmd;
-	int rc;
-
-	cmd = virtio_usb_dc_cmd_alloc(vudc, code, GFP_ATOMIC);
-	if (!cmd)
-		return -ENOMEM;
-
-	req = virtio_usb_cmd_request(cmd);
-	req->value = cpu_to_le32(mA);
-
-	rc = virtio_usb_cmd_send_sync(vudc->vusb, NULL, NULL, cmd);
 
-	return rc;
+	atomic_set(&vudc->pending_vbus_ma, ma);
+	queue_work(vudc->dc_cmd_wq, &vudc->dc_cmd_work);
+	return 0;
 }
 
 /**
@@ -930,6 +1324,11 @@ static int virtio_usb_dc_get_endpoint_info(struct virtio_usb_dc *vudc)
 		vep->ep.name = vep->name;
 		vep->ep.ops = &virtio_usb_ep_ops;
 		INIT_LIST_HEAD(&vep->req_queue);
+		INIT_LIST_HEAD(&vep->pend_queue);
+		vep->state = i ? EP_DISABLED : EP_ENABLED; // EP0 always enabled
+		INIT_WORK(&vep->enable_work, vep_enable_work);
+		INIT_WORK(&vep->disable_work, vep_disable_work);
+		vep->last_err = 0;
 		list_add_tail(&vep->ep.ep_list, &vudc->gadget.ep_list);
 		usb_ep_set_maxpacket_limit(&vep->ep, maxpacket_limit);
 		vep->ep.max_streams = le16_to_cpu(epinfo[i].max_streams);
@@ -1038,7 +1437,6 @@ static void virtio_usb_dc_event_process(struct virtio_usb_event *event)
 		}
 		vudc->gadget.ops = &virtio_gadget_ops;
 		vudc->gadget.max_speed = USB_SPEED_HIGH;
-		vudc->gadget.nonatomic = 1;
 
 		//vudc->gadget.dev.init_name = gadget_name;
 		vudc->gadget.dev.parent = &vudc->pdev->dev;
@@ -1071,6 +1469,14 @@ static void virtio_usb_dc_event_process(struct virtio_usb_event *event)
 
 		vep = usb_ep_to_virtio_ep(vudc->gadget.ep0);
 		setup_evt = virtio_usb_event_buf(event);
+		rc = virtio_usb_dc_wait_ep0_idle(vudc);
+		if (rc) {
+			dev_warn(
+				&vusb->vdev->dev,
+				"SETUP while ep0 request is still pending, timing out\n");
+			virtio_ep_set_halt(vudc->gadget.ep0, 1);
+			break;
+		}
 
 		memcpy(&vep->setup, setup_evt->setup,
 		       sizeof(struct usb_ctrlrequest));
@@ -1102,6 +1508,9 @@ static void virtio_usb_dc_event_process(struct virtio_usb_event *event)
 	case VIRTIO_USB_EVT_DEV_UNBIND: {
 		unsigned int registered;
 
+		pr_info("%s port_id %d VIRTIO_USB_EVT_DEV_UNBIND\n", __func__,
+			port_id);
+
 		spin_lock_irqsave(&vudc->lock, flags);
 		registered = vudc->registered;
 		vudc->registered = 0;
@@ -1113,8 +1522,14 @@ static void virtio_usb_dc_event_process(struct virtio_usb_event *event)
 			vudc->gadget.name = NULL;
 		}
 
-		kfree(vudc->veps);
-		vudc->veps = NULL;
+		if (vudc->veps) {
+			unsigned int i;
+
+			for (i = 0; i < vudc->neps; i++)
+				cancel_work_sync(&vudc->veps[i].enable_work);
+			kfree(vudc->veps);
+			vudc->veps = NULL;
+		}
 
 		if (vudc->pdev) {
 			platform_device_unregister(vudc->pdev);
@@ -1141,7 +1556,6 @@ void virtio_usb_dc_event_work(struct work_struct *work)
 		container_of(work, struct virtio_usb, vq_dev_event_work);
 	struct virtio_usb_queue *evtq =
 		&vusb->vqueues[vusb->dev_vq_base + VIRTIO_USB_VQ_EVENT_IDX];
-
 	virtio_usb_evt_work(evtq, virtio_usb_dc_event_process);
 }
 
@@ -1186,13 +1600,39 @@ int virtio_usb_dc_init(struct virtio_usb *vusb, int port_id)
 	if (!vudc)
 		return -ENOMEM;
 
-	vusb->vports[port_id].vudc = vudc;
 	vudc->vusb = vusb;
 	vudc->port = port_id;
 	for (i = 0; i < VIRTIO_USB_VQ_DEV_MAX; i++)
 		vudc->dcqs[i] = &vusb->vqueues[vusb->dev_vq_base + i];
 
+	vudc->ep_cmd_wq = alloc_ordered_workqueue(
+		"vudc_ep_work_%d", WQ_MEM_RECLAIM | WQ_UNBOUND, port_id);
+	vudc->dc_cmd_wq = alloc_ordered_workqueue(
+		"vudc_dc_work_%d", WQ_MEM_RECLAIM | WQ_UNBOUND, port_id);
+	if (!vudc->ep_cmd_wq || !vudc->dc_cmd_wq) {
+		if (vudc->ep_cmd_wq)
+			destroy_workqueue(vudc->ep_cmd_wq);
+		if (vudc->dc_cmd_wq)
+			destroy_workqueue(vudc->dc_cmd_wq);
+		devm_kfree(&vusb->vdev->dev, vudc);
+		return -ENOMEM;
+	}
+
+	/* Publish only once fully initialized - vports[port_id].vudc must
+	 * never point at a partially-constructed (or already-freed, on the
+	 * failure path above) struct virtio_usb_dc.
+	 */
+	vusb->vports[port_id].vudc = vudc;
+
+	INIT_WORK(&vudc->dc_cmd_work, virtio_usb_dc_work);
+	mutex_init(&vudc->dc_cmd_work_lock);
+	atomic_set(&vudc->pending_self_pwr, -1);
+	atomic_set(&vudc->pending_vbus_ma, -1);
+	vudc->cur_vbus_ma = 0;
+	vudc->cur_self_pwr = false;
+
 	spin_lock_init(&vudc->lock);
+	init_waitqueue_head(&vudc->ep0_idle);
 
 	vudc->registered = 0;
 
@@ -1228,6 +1668,22 @@ int virtio_usb_dc_deinit(struct virtio_usb *vusb)
 		if (registered)
 			usb_del_gadget_udc(&vudc->gadget);
 
+		if (vudc->dc_cmd_wq) {
+			cancel_work_sync(&vudc->dc_cmd_work);
+			destroy_workqueue(vudc->dc_cmd_wq);
+		}
+
+		if (vudc->ep_cmd_wq) {
+			flush_workqueue(vudc->ep_cmd_wq);
+			if (vudc->veps) {
+				unsigned int i;
+
+				for (i = 0; i < vudc->neps; i++)
+					cancel_work_sync(
+						&vudc->veps[i].enable_work);
+			}
+			destroy_workqueue(vudc->ep_cmd_wq);
+		}
 		kfree(vudc->veps);
 		vudc->veps = NULL;
 
diff --git a/drivers/usb/virtio_usb/device.h b/drivers/usb/virtio_usb/device.h
index 9dacc89..2f93f45 100644
--- a/drivers/usb/virtio_usb/device.h
+++ b/drivers/usb/virtio_usb/device.h
@@ -11,6 +11,7 @@
 #include <linux/usb.h>
 #include <linux/list.h>
 #include <linux/platform_device.h>
+#include <linux/wait.h>
 #include <uapi/linux/usb/ch11.h>
 #include <uapi/linux/usb/ch9.h>
 #include <linux/usb/gadget.h>
@@ -37,12 +38,19 @@
 		.dir_out = !!((_dir) & VIRTIO_USB_EP_CAPS_DIR_OUT),            \
 	}
 
+enum vep_state { EP_DISABLED, EP_ENABLING, EP_ENABLED, EP_HALTED };
+
 /**
  * struct virtio_usb_ep - virtio usb device endpoint
  * @ep: usb ep
  * @vusb: VirtIO usb device
  * @setup: setup packet for control endpoint
  * @req_queue: list of usb requests submitted to ep awaiting response
+ * @pend_queue: list of usb requests waiting for submit while EP is ENABLING
+ * @state: endpoint state (necessary to support split-phase async processing)
+ * @enable_work:  workstruct for the async ep_enable()
+ * @disable_work:  workstruct for the async ep_disable()
+ * @last_err: error code if async processeing brings us to HALTED state
  * @ep_id: Id of the endpoint
  * @name: Endpoint name
  */
@@ -51,6 +59,11 @@ struct virtio_usb_ep {
 	struct virtio_usb_dc *vudc;
 	struct usb_ctrlrequest setup;
 	struct list_head req_queue;
+	struct list_head pend_queue;
+	enum vep_state state;
+	struct work_struct enable_work;
+	struct work_struct disable_work;
+	int last_err;
 	u16 ep_id;
 	char name[16];
 };
@@ -66,6 +79,7 @@ struct virtio_usb_ep {
  * @registered: Flag indicating registration status to the UDC core.
  * @pullup:  Software-controlled connect/disconnect status USB host.
  * @lock: Spinlock that protects device state
+ * @ep0_idle: Wait queue for SETUP serialization while ep0 has an in-flight req
  */
 struct virtio_usb_dc {
 	struct platform_device *pdev;
@@ -79,6 +93,16 @@ struct virtio_usb_dc {
 	unsigned registered : 1;
 	unsigned pullup : 1;
 	spinlock_t lock;
+	wait_queue_head_t ep0_idle;
+	/* Workqueue for EP enable/disable/cancel commands */
+	struct workqueue_struct *ep_cmd_wq;
+	struct workqueue_struct *dc_cmd_wq;
+	struct work_struct dc_cmd_work;
+	struct mutex dc_cmd_work_lock;
+	atomic_t pending_self_pwr;
+	atomic_t pending_vbus_ma;
+	unsigned int cur_vbus_ma;
+	bool cur_self_pwr;
 };
 
 extern const struct virtio_usb_vq_desc dev_vqueues[VIRTIO_USB_VQ_DEV_MAX];