[PATCH v3 0/1] evtchn: make EVTCHNOP_reset suitable for kexec

[PATCH v3 0/1] evtchn: make EVTCHNOP_reset suitable for kexec

[Vitaly Kuznetsov]

With the help from David Vrabel (thanks!) I was able to make event channel
rebinding work in both kexec/kdump cases. A guest should do the following
on kexec/kdump:
1) Figure out store/console interdomain mapping
2) Call PHYSDEVOP_unmap_pirq if needed
3) Call EVTCHNOP_reset
4) Rebind store/console channels with EVTCHNOP_bind_interdomain
5) Update HVM_PARAM_CONSOLE_EVTCHN/HVM_PARAM_STORE_EVTCHN

Here is an example of guest code (to be called from kexec/kdump handlers):

void xen_hvm_reset_eventchannels(void)
{
	struct evtchn_status status;
	struct physdev_unmap_pirq unmap_irq;
	struct evtchn_close close;
	struct evtchn_reset reset;
	struct evtchn_bind_interdomain int_bind;
	struct xen_hvm_param xhv;
	uint64_t store_evtchn = 0, console_evtchn = 0;
	domid_t store_domain = 0, console_domain = 0;
	evtchn_port_t store_rem_port = 0, console_rem_port = 0;
	int port, rc = -ENOENT;

	memset(&status, 0, sizeof(status));
	status.dom = DOMID_SELF;
	if (hvm_get_parameter(HVM_PARAM_STORE_EVTCHN, &store_evtchn) == 0) {
		pr_debug("xen: need to rebind store evtchn %d\n",
			 (uint32_t)store_evtchn);
		status.port = store_evtchn;
		rc = HYPERVISOR_event_channel_op(EVTCHNOP_status, &status);
		if (!rc && status.status == EVTCHNSTAT_interdomain) {
			pr_debug("xen: store port: %d, interdomain %d:%d",
				 status.port, status.u.interdomain.dom,
				 status.u.interdomain.port);
			store_domain = status.u.interdomain.dom;
			store_rem_port = status.u.interdomain.port;
		}
	}
	if (hvm_get_parameter(HVM_PARAM_CONSOLE_EVTCHN, &console_evtchn) == 0) {
		pr_debug("xen: need to rebind console evtchn %d\n",
			 (uint32_t)console_evtchn);
		status.port = console_evtchn;
		rc = HYPERVISOR_event_channel_op(EVTCHNOP_status, &status);
		if (!rc && status.status == EVTCHNSTAT_interdomain) {
			pr_debug("xen: console port: %d, interdomain %d:%d",
				 status.port, status.u.interdomain.dom,
				 status.u.interdomain.port);
			console_domain = status.u.interdomain.dom;
			console_rem_port = status.u.interdomain.port;
		}
	}

	for (port = 0; port < xen_evtchn_max_channels(); port++) {
		status.dom = DOMID_SELF;
		status.port = port;
		rc = HYPERVISOR_event_channel_op(EVTCHNOP_status, &status);
		if (rc < 0)
			continue;
		if (status.status != EVTCHNSTAT_closed) {
			close.port = port;
			if (HYPERVISOR_event_channel_op(EVTCHNOP_close,
							&close) != 0)
				pr_warn("xen: failed to close evtchn %d\n",
					port);
		}

		if (status.status == EVTCHNSTAT_pirq) {
			unmap_irq.pirq = status.u.pirq;
			unmap_irq.domid = DOMID_SELF;
			pr_warn("xen: unmapping previously mapped pirq %d\n",
				unmap_irq.pirq);
			if (HYPERVISOR_physdev_op(PHYSDEVOP_unmap_pirq,
						  &unmap_irq) != 0)
				pr_warn("xen: failed to unmap pirq %d\n",
					unmap_irq.pirq);
		}
	}

	reset.dom = DOMID_SELF;
	if (HYPERVISOR_event_channel_op(EVTCHNOP_reset, &reset))
		pr_warn("failed to reset all event channels!\n");

	if (store_rem_port > 0) {
		int_bind.remote_dom = store_domain;
		int_bind.remote_port = store_rem_port;
		pr_debug("rebinding store: %d -> %d:%d\n", int_bind.local_port,
			 int_bind.remote_port, int_bind.remote_dom);
		if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain, &int_bind))
			pr_warn("failed to rebind store!\n");
		if (int_bind.local_port != store_evtchn) {
			xhv.domid = DOMID_SELF;
			xhv.index = HVM_PARAM_STORE_EVTCHN;
			xhv.value = int_bind.local_port;
			if (HYPERVISOR_hvm_op(HVMOP_set_param, &xhv)) {
				pr_warn("Cannot set HVM_PARAM_STORE_EVTCHN!\n");
			}
		}
	}

	if (console_rem_port > 0) {
		int_bind.remote_dom = console_domain;
		int_bind.remote_port = console_rem_port;
		pr_debug("rebinding console: %d -> %d:%d\n", int_bind.local_port,
			 int_bind.remote_port, int_bind.remote_dom);
		if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain, &int_bind))
			pr_warn("failed to rebind console!\n");
		if (int_bind.local_port != console_evtchn) {
			xhv.domid = DOMID_SELF;
			xhv.index = HVM_PARAM_CONSOLE_EVTCHN;
			xhv.value = int_bind.local_port;
			if (HYPERVISOR_hvm_op(HVMOP_set_param, &xhv)) {
				pr_warn("Cannot set HVM_PARAM_CONSOLE_EVTCHN!\n");
			}
		}
	}
}

Vitaly Kuznetsov (1):
  evtchn: make EVTCHNOP_reset suitable for kexec

 xen/common/event_channel.c         | 15 +++++++++++++++
 xen/common/event_fifo.c            |  1 +
 xen/include/public/event_channel.h |  4 ++++
 3 files changed, 20 insertions(+)

-- 
1.9.3

[PATCH v3 1/1] evtchn: make EVTCHNOP_reset suitable for kexec

[Vitaly Kuznetsov]

It would be nice to allow guests to close all event channels in
ABI-agnostic way in case of kexec/kdump. EVTCHNOP_reset looks suitable
for this purpose. However control blocks for vcpus and event array need
cleanup when FIFO ABI is being used.

With this change a guest can simply do EVTCHNOP_reset before kexec in
both 2-level and FIFO cases. It is also important to perform store/console
channel remapping after such call.

The issue can also be solved by introducing a new EVTCHNOP operation but
it seems that EVTCHNOP_reset can be reused.

[The idea was suggested by Ian Campbell, Andrew Cooper, and David Vrabel]

Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com>
---
Changes from v2:
- use d->event_lock when switching ABI [David Vrabel]
- add documentation to xen/include/public/event_channel.h [David Vrabel]
- check for 'dom == DOMID_SELF' instead of 'd == current->domain' to
  allow guests use unaltered behavior [Jan Beulich]

Changes from v1:
- Change EVTCHNOP_reset behavior when it is being called by the guest
  with DOMID_SELF only [Andrew Cooper, Jan Beulich].
- Do not skip interdomain store/console channels, let the guest rebind
  them [David Vrabel].
- Do evtchn_2l_init() after evtchn_fifo_destroy() so the guest can
  perform e.g. EVTCHNOP_bind_interdomain operations after EVTCHNOP_reset.
---
 xen/common/event_channel.c         | 15 +++++++++++++++
 xen/common/event_fifo.c            |  1 +
 xen/include/public/event_channel.h |  4 ++++
 3 files changed, 20 insertions(+)

diff --git a/xen/common/event_channel.c b/xen/common/event_channel.c
index db952af..a7becae 100644
--- a/xen/common/event_channel.c
+++ b/xen/common/event_channel.c
@@ -957,6 +957,21 @@ static long evtchn_reset(evtchn_reset_t *r)
     for ( i = 0; port_is_valid(d, i); i++ )
         (void)__evtchn_close(d, i);
 
+    spin_lock(&d->event_lock);
+
+    if ( (dom == DOMID_SELF) && d->evtchn_fifo )
+    {
+        /*
+         * Guest domain called EVTCHNOP_reset with DOMID_SELF, destroying
+         * FIFO event array and control blocks, resetting evtchn_port_ops to
+         * evtchn_port_ops_2l.
+         */
+        evtchn_fifo_destroy(d);
+        evtchn_2l_init(d);
+    }
+
+    spin_unlock(&d->event_lock);
+
     rc = 0;
 
 out:
diff --git a/xen/common/event_fifo.c b/xen/common/event_fifo.c
index 1fce3f1..51b4ff6 100644
--- a/xen/common/event_fifo.c
+++ b/xen/common/event_fifo.c
@@ -451,6 +451,7 @@ static void cleanup_event_array(struct domain *d)
     for ( i = 0; i < EVTCHN_FIFO_MAX_EVENT_ARRAY_PAGES; i++ )
         unmap_guest_page(d->evtchn_fifo->event_array[i]);
     xfree(d->evtchn_fifo);
+    d->evtchn_fifo = NULL;
 }
 
 static void setup_ports(struct domain *d)
diff --git a/xen/include/public/event_channel.h b/xen/include/public/event_channel.h
index 49ac8cc..05e531d 100644
--- a/xen/include/public/event_channel.h
+++ b/xen/include/public/event_channel.h
@@ -264,6 +264,10 @@ typedef struct evtchn_unmask evtchn_unmask_t;
  * NOTES:
  *  1. <dom> may be specified as DOMID_SELF.
  *  2. Only a sufficiently-privileged domain may specify other than DOMID_SELF.
+ *  3. Destroys all control blocks and event array, resets event channel
+ *     operations to 2-level ABI if called with <dom> == DOMID_SELF and FIFO
+ *     ABI was used. Guests should not bind events during EVTCHNOP_reset call
+ *     as these events are likely to be lost.
  */
 struct evtchn_reset {
     /* IN parameters. */
-- 
1.9.3