* [PATCH V2 01/13] xen: add KERN_DEBUG in printk
2013-02-04 17:23 [PATCH V2] Implement 3-level event channel in Linux Wei Liu
@ 2013-02-04 17:23 ` Wei Liu
2013-02-04 17:23 ` [PATCH V2 02/13] xen: fix output of xen_debug_interrupt Wei Liu
` (11 subsequent siblings)
12 siblings, 0 replies; 15+ messages in thread
From: Wei Liu @ 2013-02-04 17:23 UTC (permalink / raw)
To: xen-devel; +Cc: david.vrabel, Wei Liu, ian.campbell, jbeulich, konrad.wilk
Signed-off-by: Wei Liu <wei.liu2@citrix.com>
---
drivers/xen/events.c | 28 ++++++++++++++--------------
1 file changed, 14 insertions(+), 14 deletions(-)
diff --git a/drivers/xen/events.c b/drivers/xen/events.c
index 7595581..4d9a3a6 100644
--- a/drivers/xen/events.c
+++ b/drivers/xen/events.c
@@ -1176,7 +1176,7 @@ irqreturn_t xen_debug_interrupt(int irq, void *dev_id)
spin_lock_irqsave(&debug_lock, flags);
- printk("\nvcpu %d\n ", cpu);
+ printk(KERN_DEBUG "\nvcpu %d\n ", cpu);
for_each_online_cpu(i) {
int pending;
@@ -1184,51 +1184,51 @@ irqreturn_t xen_debug_interrupt(int irq, void *dev_id)
pending = (get_irq_regs() && i == cpu)
? xen_irqs_disabled(get_irq_regs())
: v->evtchn_upcall_mask;
- printk("%d: masked=%d pending=%d event_sel %0*lx\n ", i,
+ printk(KERN_DEBUG "%d: masked=%d pending=%d event_sel %0*lx\n ", i,
pending, v->evtchn_upcall_pending,
(int)(sizeof(v->evtchn_pending_sel)*2),
v->evtchn_pending_sel);
}
v = per_cpu(xen_vcpu, cpu);
- printk("\npending:\n ");
+ printk(KERN_DEBUG "\npending:\n ");
for (i = ARRAY_SIZE(sh->evtchn_pending)-1; i >= 0; i--)
- printk("%0*lx%s", (int)sizeof(sh->evtchn_pending[0])*2,
+ printk(KERN_DEBUG "%0*lx%s", (int)sizeof(sh->evtchn_pending[0])*2,
sh->evtchn_pending[i],
i % 8 == 0 ? "\n " : " ");
- printk("\nglobal mask:\n ");
+ printk(KERN_DEBUG "\nglobal mask:\n ");
for (i = ARRAY_SIZE(sh->evtchn_mask)-1; i >= 0; i--)
- printk("%0*lx%s",
+ printk(KERN_DEBUG "%0*lx%s",
(int)(sizeof(sh->evtchn_mask[0])*2),
sh->evtchn_mask[i],
i % 8 == 0 ? "\n " : " ");
- printk("\nglobally unmasked:\n ");
+ printk(KERN_DEBUG "\nglobally unmasked:\n ");
for (i = ARRAY_SIZE(sh->evtchn_mask)-1; i >= 0; i--)
- printk("%0*lx%s", (int)(sizeof(sh->evtchn_mask[0])*2),
+ printk(KERN_DEBUG "%0*lx%s", (int)(sizeof(sh->evtchn_mask[0])*2),
sh->evtchn_pending[i] & ~sh->evtchn_mask[i],
i % 8 == 0 ? "\n " : " ");
- printk("\nlocal cpu%d mask:\n ", cpu);
+ printk(KERN_DEBUG "\nlocal cpu%d mask:\n ", cpu);
for (i = (NR_EVENT_CHANNELS/BITS_PER_LONG)-1; i >= 0; i--)
- printk("%0*lx%s", (int)(sizeof(cpu_evtchn[0])*2),
+ printk(KERN_DEBUG "%0*lx%s", (int)(sizeof(cpu_evtchn[0])*2),
cpu_evtchn[i],
i % 8 == 0 ? "\n " : " ");
- printk("\nlocally unmasked:\n ");
+ printk(KERN_DEBUG "\nlocally unmasked:\n ");
for (i = ARRAY_SIZE(sh->evtchn_mask)-1; i >= 0; i--) {
unsigned long pending = sh->evtchn_pending[i]
& ~sh->evtchn_mask[i]
& cpu_evtchn[i];
- printk("%0*lx%s", (int)(sizeof(sh->evtchn_mask[0])*2),
+ printk(KERN_DEBUG "%0*lx%s", (int)(sizeof(sh->evtchn_mask[0])*2),
pending, i % 8 == 0 ? "\n " : " ");
}
- printk("\npending list:\n");
+ printk(KERN_DEBUG "\npending list:\n");
for (i = 0; i < NR_EVENT_CHANNELS; i++) {
if (sync_test_bit(i, sh->evtchn_pending)) {
int word_idx = i / BITS_PER_LONG;
- printk(" %d: event %d -> irq %d%s%s%s\n",
+ printk(KERN_DEBUG " %d: event %d -> irq %d%s%s%s\n",
cpu_from_evtchn(i), i,
evtchn_to_irq[i],
sync_test_bit(word_idx, &v->evtchn_pending_sel)
--
1.7.10.4
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH V2 02/13] xen: fix output of xen_debug_interrupt
2013-02-04 17:23 [PATCH V2] Implement 3-level event channel in Linux Wei Liu
2013-02-04 17:23 ` [PATCH V2 01/13] xen: add KERN_DEBUG in printk Wei Liu
@ 2013-02-04 17:23 ` Wei Liu
2013-02-04 17:23 ` [PATCH V2 03/13] xen: fix error handling path if xen_allocate_irq_dynamic fails Wei Liu
` (10 subsequent siblings)
12 siblings, 0 replies; 15+ messages in thread
From: Wei Liu @ 2013-02-04 17:23 UTC (permalink / raw)
To: xen-devel; +Cc: david.vrabel, Wei Liu, ian.campbell, jbeulich, konrad.wilk
Make the per-cpu selector L1 to be consistent with description in
__xen_evtchn_do_upcall's comment.
Signed-off-by: Wei Liu <wei.liu2@citrix.com>
---
drivers/xen/events.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/drivers/xen/events.c b/drivers/xen/events.c
index 4d9a3a6..d937bf8 100644
--- a/drivers/xen/events.c
+++ b/drivers/xen/events.c
@@ -1232,7 +1232,7 @@ irqreturn_t xen_debug_interrupt(int irq, void *dev_id)
cpu_from_evtchn(i), i,
evtchn_to_irq[i],
sync_test_bit(word_idx, &v->evtchn_pending_sel)
- ? "" : " l2-clear",
+ ? "" : " l1-clear",
!sync_test_bit(i, sh->evtchn_mask)
? "" : " globally-masked",
sync_test_bit(i, cpu_evtchn)
--
1.7.10.4
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH V2 03/13] xen: fix error handling path if xen_allocate_irq_dynamic fails
2013-02-04 17:23 [PATCH V2] Implement 3-level event channel in Linux Wei Liu
2013-02-04 17:23 ` [PATCH V2 01/13] xen: add KERN_DEBUG in printk Wei Liu
2013-02-04 17:23 ` [PATCH V2 02/13] xen: fix output of xen_debug_interrupt Wei Liu
@ 2013-02-04 17:23 ` Wei Liu
2013-02-04 17:23 ` [PATCH V2 04/13] xen: close evtchn port if binding to irq fails Wei Liu
` (9 subsequent siblings)
12 siblings, 0 replies; 15+ messages in thread
From: Wei Liu @ 2013-02-04 17:23 UTC (permalink / raw)
To: xen-devel; +Cc: david.vrabel, Wei Liu, ian.campbell, jbeulich, konrad.wilk
The call to xen_allocate_irq_dynamic() might return -ev error codes, not
lietral -1.
Reviewed-by: David Vrabel <david.vrabel@citrix.com>
Signed-off-by: Wei Liu <wei.liu2@citrix.com>
---
drivers/xen/events.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/drivers/xen/events.c b/drivers/xen/events.c
index d937bf8..96ed76f 100644
--- a/drivers/xen/events.c
+++ b/drivers/xen/events.c
@@ -820,7 +820,7 @@ int bind_evtchn_to_irq(unsigned int evtchn)
if (irq == -1) {
irq = xen_allocate_irq_dynamic();
- if (irq == -1)
+ if (irq < 0)
goto out;
irq_set_chip_and_handler_name(irq, &xen_dynamic_chip,
@@ -923,7 +923,7 @@ int bind_virq_to_irq(unsigned int virq, unsigned int cpu)
if (irq == -1) {
irq = xen_allocate_irq_dynamic();
- if (irq == -1)
+ if (irq < 0)
goto out;
irq_set_chip_and_handler_name(irq, &xen_percpu_chip,
--
1.7.10.4
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH V2 04/13] xen: close evtchn port if binding to irq fails
2013-02-04 17:23 [PATCH V2] Implement 3-level event channel in Linux Wei Liu
` (2 preceding siblings ...)
2013-02-04 17:23 ` [PATCH V2 03/13] xen: fix error handling path if xen_allocate_irq_dynamic fails Wei Liu
@ 2013-02-04 17:23 ` Wei Liu
2013-02-06 8:05 ` Jan Beulich
2013-02-04 17:23 ` [PATCH V2 05/13] xen: sync public headers Wei Liu
` (8 subsequent siblings)
12 siblings, 1 reply; 15+ messages in thread
From: Wei Liu @ 2013-02-04 17:23 UTC (permalink / raw)
To: xen-devel; +Cc: david.vrabel, Wei Liu, ian.campbell, jbeulich, konrad.wilk
Signed-off-by: Wei Liu <wei.liu2@citrix.com>
---
drivers/xen/evtchn.c | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/drivers/xen/evtchn.c b/drivers/xen/evtchn.c
index b1f60a0..ef02dfa 100644
--- a/drivers/xen/evtchn.c
+++ b/drivers/xen/evtchn.c
@@ -269,6 +269,15 @@ static int evtchn_bind_to_user(struct per_user_data *u, int port)
u->name, (void *)(unsigned long)port);
if (rc >= 0)
rc = evtchn_make_refcounted(port);
+ else {
+ /* bind failed, should close the port now */
+ struct evtchn_close close;
+ close.port = port;
+ if (port != 0 && /* port 0 is never used */
+ HYPERVISOR_event_channel_op(EVTCHNOP_close, &close) != 0)
+ BUG();
+ set_port_user(port, NULL);
+ }
return rc;
}
--
1.7.10.4
^ permalink raw reply related [flat|nested] 15+ messages in thread* Re: [PATCH V2 04/13] xen: close evtchn port if binding to irq fails
2013-02-04 17:23 ` [PATCH V2 04/13] xen: close evtchn port if binding to irq fails Wei Liu
@ 2013-02-06 8:05 ` Jan Beulich
0 siblings, 0 replies; 15+ messages in thread
From: Jan Beulich @ 2013-02-06 8:05 UTC (permalink / raw)
To: Wei Liu; +Cc: xen-devel, david.vrabel, konrad.wilk, ian.campbell
>>> On 04.02.13 at 18:23, Wei Liu <wei.liu2@citrix.com> wrote:
> --- a/drivers/xen/evtchn.c
> +++ b/drivers/xen/evtchn.c
> @@ -269,6 +269,15 @@ static int evtchn_bind_to_user(struct per_user_data *u,
> int port)
> u->name, (void *)(unsigned long)port);
> if (rc >= 0)
> rc = evtchn_make_refcounted(port);
> + else {
> + /* bind failed, should close the port now */
> + struct evtchn_close close;
> + close.port = port;
> + if (port != 0 && /* port 0 is never used */
If the port == 0 case really needs handling here (didn't look at
the surrounding code), this ought to be done with a "else if()"
at the top of this change imo.
Jan
> + HYPERVISOR_event_channel_op(EVTCHNOP_close, &close) != 0)
> + BUG();
> + set_port_user(port, NULL);
> + }
>
> return rc;
> }
^ permalink raw reply [flat|nested] 15+ messages in thread
* [PATCH V2 05/13] xen: sync public headers
2013-02-04 17:23 [PATCH V2] Implement 3-level event channel in Linux Wei Liu
` (3 preceding siblings ...)
2013-02-04 17:23 ` [PATCH V2 04/13] xen: close evtchn port if binding to irq fails Wei Liu
@ 2013-02-04 17:23 ` Wei Liu
2013-02-04 17:23 ` [PATCH V2 06/13] xen: introduce test_and_set_mask Wei Liu
` (7 subsequent siblings)
12 siblings, 0 replies; 15+ messages in thread
From: Wei Liu @ 2013-02-04 17:23 UTC (permalink / raw)
To: xen-devel; +Cc: david.vrabel, Wei Liu, ian.campbell, jbeulich, konrad.wilk
Signed-off-by: Wei Liu <wei.liu2@citrix.com>
---
include/xen/interface/event_channel.h | 33 +++++++++++++++++++++++++++++++++
include/xen/interface/xen.h | 9 ++++++++-
2 files changed, 41 insertions(+), 1 deletion(-)
diff --git a/include/xen/interface/event_channel.h b/include/xen/interface/event_channel.h
index f494292..9d8b9e7 100644
--- a/include/xen/interface/event_channel.h
+++ b/include/xen/interface/event_channel.h
@@ -190,6 +190,39 @@ struct evtchn_reset {
};
typedef struct evtchn_reset evtchn_reset_t;
+/*
+ * EVTCHNOP_register_nlevel: Register N-level event channel
+ * NOTES:
+ * 1. Currently only 3-level is supported.
+ * 2. Should fall back to 2-level if this call fails.
+ */
+#define EVTCHNOP_register_nlevel 11
+/* 64 bit guests need 8 pages for evtchn_pending and evtchn_mask for
+ * 256k event channels while 32 bit ones only need 1 page for 32k
+ * event channels. */
+#define EVTCHN_MAX_L3_PAGES 8
+struct evtchn_register_3level {
+ /* IN parameters. */
+ uint32_t nr_pages; /* for evtchn_{pending,mask} */
+ uint32_t nr_vcpus; /* for l2sel_{mfns,mask} */
+ GUEST_HANDLE(ulong) evtchn_pending;
+ GUEST_HANDLE(ulong) evtchn_mask;
+ GUEST_HANDLE(ulong) l2sel_mfns;
+ GUEST_HANDLE(ulong) l2sel_offsets;
+};
+typedef struct evtchn_register_3level evtchn_register_3level_t;
+DEFINE_GUEST_HANDLE(evtchn_register_3level_t);
+
+struct evtchn_register_nlevel {
+ /* IN parameters. */
+ uint32_t level;
+ union {
+ evtchn_register_3level_t l3;
+ } u;
+};
+typedef struct evtchn_register_nlevel evtchn_register_nlevel_t;
+DEFINE_GUEST_HANDLE(evtchn_register_nlevel_t);
+
struct evtchn_op {
uint32_t cmd; /* EVTCHNOP_* */
union {
diff --git a/include/xen/interface/xen.h b/include/xen/interface/xen.h
index a890804..5220e33 100644
--- a/include/xen/interface/xen.h
+++ b/include/xen/interface/xen.h
@@ -283,9 +283,16 @@ DEFINE_GUEST_HANDLE_STRUCT(multicall_entry);
/*
* Event channel endpoints per domain:
+ * 2-level:
* 1024 if a long is 32 bits; 4096 if a long is 64 bits.
+ * 3-level:
+ * 32k if a long is 32 bits; 256k if a long is 64 bits.
*/
-#define NR_EVENT_CHANNELS (sizeof(unsigned long) * sizeof(unsigned long) * 64)
+#define NR_EVENT_CHANNELS_L2 (sizeof(unsigned long) * sizeof(unsigned long) * 64)
+#define NR_EVENT_CHANNELS_L3 (NR_EVENT_CHANNELS_L2 * 64)
+#if !defined(__XEN__) && !defined(__XEN_TOOLS__)
+#define NR_EVENT_CHANNELS NR_EVENT_CHANNELS_L2 /* for compatibility */
+#endif
struct vcpu_time_info {
/*
--
1.7.10.4
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH V2 06/13] xen: introduce test_and_set_mask
2013-02-04 17:23 [PATCH V2] Implement 3-level event channel in Linux Wei Liu
` (4 preceding siblings ...)
2013-02-04 17:23 ` [PATCH V2 05/13] xen: sync public headers Wei Liu
@ 2013-02-04 17:23 ` Wei Liu
2013-02-04 17:23 ` [PATCH V2 07/13] xen: replace raw bit ops with functions Wei Liu
` (6 subsequent siblings)
12 siblings, 0 replies; 15+ messages in thread
From: Wei Liu @ 2013-02-04 17:23 UTC (permalink / raw)
To: xen-devel; +Cc: david.vrabel, Wei Liu, ian.campbell, jbeulich, konrad.wilk
Signed-off-by: Wei Liu <wei.liu2@citrix.com>
---
drivers/xen/events.c | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/drivers/xen/events.c b/drivers/xen/events.c
index 96ed76f..2278851 100644
--- a/drivers/xen/events.c
+++ b/drivers/xen/events.c
@@ -345,6 +345,12 @@ static inline int test_evtchn(int port)
return sync_test_bit(port, &s->evtchn_pending[0]);
}
+static inline int test_and_set_mask(int port)
+{
+ struct shared_info *s = HYPERVISOR_shared_info;
+ return sync_test_and_set_bit(port, &s->evtchn_mask[0]);
+}
+
/**
* notify_remote_via_irq - send event to remote end of event channel via irq
@@ -1464,7 +1470,7 @@ int resend_irq_on_evtchn(unsigned int irq)
if (!VALID_EVTCHN(evtchn))
return 1;
- masked = sync_test_and_set_bit(evtchn, s->evtchn_mask);
+ masked = test_and_set_mask(evtchn);
sync_set_bit(evtchn, s->evtchn_pending);
if (!masked)
unmask_evtchn(evtchn);
@@ -1513,7 +1519,7 @@ static int retrigger_dynirq(struct irq_data *data)
if (VALID_EVTCHN(evtchn)) {
int masked;
- masked = sync_test_and_set_bit(evtchn, sh->evtchn_mask);
+ masked = test_and_set_mask(evtchn);
sync_set_bit(evtchn, sh->evtchn_pending);
if (!masked)
unmask_evtchn(evtchn);
--
1.7.10.4
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH V2 07/13] xen: replace raw bit ops with functions
2013-02-04 17:23 [PATCH V2] Implement 3-level event channel in Linux Wei Liu
` (5 preceding siblings ...)
2013-02-04 17:23 ` [PATCH V2 06/13] xen: introduce test_and_set_mask Wei Liu
@ 2013-02-04 17:23 ` Wei Liu
2013-02-04 17:23 ` [PATCH V2 08/13] xen: generalized event channel operations Wei Liu
` (5 subsequent siblings)
12 siblings, 0 replies; 15+ messages in thread
From: Wei Liu @ 2013-02-04 17:23 UTC (permalink / raw)
To: xen-devel; +Cc: david.vrabel, Wei Liu, ian.campbell, jbeulich, konrad.wilk
Signed-off-by: Wei Liu <wei.liu2@citrix.com>
---
drivers/xen/events.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/drivers/xen/events.c b/drivers/xen/events.c
index 2278851..6019cd9 100644
--- a/drivers/xen/events.c
+++ b/drivers/xen/events.c
@@ -1471,7 +1471,7 @@ int resend_irq_on_evtchn(unsigned int irq)
return 1;
masked = test_and_set_mask(evtchn);
- sync_set_bit(evtchn, s->evtchn_pending);
+ set_evtchn(evtchn);
if (!masked)
unmask_evtchn(evtchn);
@@ -1520,7 +1520,7 @@ static int retrigger_dynirq(struct irq_data *data)
int masked;
masked = test_and_set_mask(evtchn);
- sync_set_bit(evtchn, sh->evtchn_pending);
+ set_evtchn(evtchn);
if (!masked)
unmask_evtchn(evtchn);
ret = 1;
--
1.7.10.4
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH V2 08/13] xen: generalized event channel operations
2013-02-04 17:23 [PATCH V2] Implement 3-level event channel in Linux Wei Liu
` (6 preceding siblings ...)
2013-02-04 17:23 ` [PATCH V2 07/13] xen: replace raw bit ops with functions Wei Liu
@ 2013-02-04 17:23 ` Wei Liu
2013-02-04 17:23 ` [PATCH V2 09/13] xen: dynamically allocate cpu_evtchn_mask Wei Liu
` (4 subsequent siblings)
12 siblings, 0 replies; 15+ messages in thread
From: Wei Liu @ 2013-02-04 17:23 UTC (permalink / raw)
To: xen-devel; +Cc: david.vrabel, Wei Liu, ian.campbell, jbeulich, konrad.wilk
Use global pointers in common operations to allow for better code sharing
between N-level event channel.
Functions which are not suitable for sharing are also taken care of.
Also update drivers/xen/evtchn.c to use exported variable instead of macro.
Signed-off-by: Wei Liu <wei.liu2@citrix.com>
---
drivers/xen/events.c | 180 ++++++++++++++++++++++++++++++++------------------
drivers/xen/evtchn.c | 12 ++--
include/xen/events.h | 2 +
3 files changed, 123 insertions(+), 71 deletions(-)
diff --git a/drivers/xen/events.c b/drivers/xen/events.c
index 6019cd9..355dbda 100644
--- a/drivers/xen/events.c
+++ b/drivers/xen/events.c
@@ -51,6 +51,16 @@
#include <xen/interface/hvm/hvm_op.h>
#include <xen/interface/hvm/params.h>
+/* N-level event channel, starting from 2 */
+unsigned int evtchn_level = 2;
+EXPORT_SYMBOL_GPL(evtchn_level);
+unsigned int nr_event_channels;
+EXPORT_SYMBOL_GPL(nr_event_channels);
+
+/* The following pointers point to pending bitmap and mask bitmap. */
+static unsigned long *evtchn_pending;
+static unsigned long *evtchn_mask;
+
/*
* This lock protects updates to the following mapping and reference-count
* arrays. The lock does not need to be acquired to read the mapping tables.
@@ -113,7 +123,7 @@ static int *evtchn_to_irq;
static unsigned long *pirq_eoi_map;
static bool (*pirq_needs_eoi)(unsigned irq);
-static DEFINE_PER_CPU(unsigned long [NR_EVENT_CHANNELS/BITS_PER_LONG],
+static DEFINE_PER_CPU(unsigned long [NR_EVENT_CHANNELS_L2/BITS_PER_LONG],
cpu_evtchn_mask);
/* Xen will never allocate port zero for any purpose. */
@@ -286,12 +296,11 @@ static bool pirq_needs_eoi_flag(unsigned irq)
}
static inline unsigned long active_evtchns(unsigned int cpu,
- struct shared_info *sh,
unsigned int idx)
{
- return sh->evtchn_pending[idx] &
+ return evtchn_pending[idx] &
per_cpu(cpu_evtchn_mask, cpu)[idx] &
- ~sh->evtchn_mask[idx];
+ ~evtchn_mask[idx];
}
static void bind_evtchn_to_cpu(unsigned int chn, unsigned int cpu)
@@ -329,26 +338,22 @@ static void init_evtchn_cpu_bindings(void)
static inline void clear_evtchn(int port)
{
- struct shared_info *s = HYPERVISOR_shared_info;
- sync_clear_bit(port, &s->evtchn_pending[0]);
+ sync_clear_bit(port, &evtchn_pending[0]);
}
static inline void set_evtchn(int port)
{
- struct shared_info *s = HYPERVISOR_shared_info;
- sync_set_bit(port, &s->evtchn_pending[0]);
+ sync_set_bit(port, &evtchn_pending[0]);
}
static inline int test_evtchn(int port)
{
- struct shared_info *s = HYPERVISOR_shared_info;
- return sync_test_bit(port, &s->evtchn_pending[0]);
+ return sync_test_bit(port, &evtchn_pending[0]);
}
static inline int test_and_set_mask(int port)
{
- struct shared_info *s = HYPERVISOR_shared_info;
- return sync_test_and_set_bit(port, &s->evtchn_mask[0]);
+ return sync_test_and_set_bit(port, &evtchn_mask[0]);
}
@@ -371,13 +376,28 @@ EXPORT_SYMBOL_GPL(notify_remote_via_irq);
static void mask_evtchn(int port)
{
- struct shared_info *s = HYPERVISOR_shared_info;
- sync_set_bit(port, &s->evtchn_mask[0]);
+ sync_set_bit(port, &evtchn_mask[0]);
+}
+
+static inline void __unmask_local_port_l2(int port)
+{
+ struct vcpu_info *vcpu_info = __this_cpu_read(xen_vcpu);
+
+ sync_clear_bit(port, &evtchn_mask[0]);
+
+ /*
+ * The following is basically the equivalent of
+ * 'hw_resend_irq'. Just like a real IO-APIC we 'lose
+ * the interrupt edge' if the channel is masked.
+ */
+ if (sync_test_bit(port, &evtchn_pending[0]) &&
+ !sync_test_and_set_bit(port / BITS_PER_LONG,
+ &vcpu_info->evtchn_pending_sel))
+ vcpu_info->evtchn_upcall_pending = 1;
}
static void unmask_evtchn(int port)
{
- struct shared_info *s = HYPERVISOR_shared_info;
unsigned int cpu = get_cpu();
BUG_ON(!irqs_disabled());
@@ -387,19 +407,13 @@ static void unmask_evtchn(int port)
struct evtchn_unmask unmask = { .port = port };
(void)HYPERVISOR_event_channel_op(EVTCHNOP_unmask, &unmask);
} else {
- struct vcpu_info *vcpu_info = __this_cpu_read(xen_vcpu);
-
- sync_clear_bit(port, &s->evtchn_mask[0]);
-
- /*
- * The following is basically the equivalent of
- * 'hw_resend_irq'. Just like a real IO-APIC we 'lose
- * the interrupt edge' if the channel is masked.
- */
- if (sync_test_bit(port, &s->evtchn_pending[0]) &&
- !sync_test_and_set_bit(port / BITS_PER_LONG,
- &vcpu_info->evtchn_pending_sel))
- vcpu_info->evtchn_upcall_pending = 1;
+ switch (evtchn_level) {
+ case 2:
+ __unmask_local_port_l2(port);
+ break;
+ default:
+ BUG();
+ }
}
put_cpu();
@@ -902,7 +916,7 @@ static int find_virq(unsigned int virq, unsigned int cpu)
int port, rc = -ENOENT;
memset(&status, 0, sizeof(status));
- for (port = 0; port <= NR_EVENT_CHANNELS; port++) {
+ for (port = 0; port <= nr_event_channels; port++) {
status.dom = DOMID_SELF;
status.port = port;
rc = HYPERVISOR_event_channel_op(EVTCHNOP_status, &status);
@@ -1127,7 +1141,7 @@ int evtchn_get(unsigned int evtchn)
struct irq_info *info;
int err = -ENOENT;
- if (evtchn >= NR_EVENT_CHANNELS)
+ if (evtchn >= nr_event_channels)
return -EINVAL;
mutex_lock(&irq_mapping_update_lock);
@@ -1170,15 +1184,16 @@ void xen_send_IPI_one(unsigned int cpu, enum ipi_vector vector)
notify_remote_via_irq(irq);
}
+static irqreturn_t xen_debug_interrupt_l2(int irq, void *dev_id);
+
irqreturn_t xen_debug_interrupt(int irq, void *dev_id)
{
- struct shared_info *sh = HYPERVISOR_shared_info;
- int cpu = smp_processor_id();
- unsigned long *cpu_evtchn = per_cpu(cpu_evtchn_mask, cpu);
- int i;
- unsigned long flags;
+ irqreturn_t rc;
static DEFINE_SPINLOCK(debug_lock);
+ unsigned long flags;
+ int cpu = smp_processor_id();
struct vcpu_info *v;
+ int i;
spin_lock_irqsave(&debug_lock, flags);
@@ -1195,24 +1210,45 @@ irqreturn_t xen_debug_interrupt(int irq, void *dev_id)
(int)(sizeof(v->evtchn_pending_sel)*2),
v->evtchn_pending_sel);
}
+
+ switch (evtchn_level) {
+ case 2:
+ rc = xen_debug_interrupt_l2(irq, dev_id);
+ break;
+ default:
+ BUG();
+ }
+
+ spin_unlock_irqrestore(&debug_lock, flags);
+ return rc;
+}
+
+static irqreturn_t xen_debug_interrupt_l2(int irq, void *dev_id)
+{
+ int cpu = smp_processor_id();
+ unsigned long *cpu_evtchn = per_cpu(cpu_evtchn_mask, cpu);
+ int i;
+ unsigned long nr_elems = NR_EVENT_CHANNELS_L2 / BITS_PER_LONG;
+ struct vcpu_info *v;
+
v = per_cpu(xen_vcpu, cpu);
printk(KERN_DEBUG "\npending:\n ");
- for (i = ARRAY_SIZE(sh->evtchn_pending)-1; i >= 0; i--)
- printk(KERN_DEBUG "%0*lx%s", (int)sizeof(sh->evtchn_pending[0])*2,
- sh->evtchn_pending[i],
+ for (i = nr_elems; i >= 0; i--)
+ printk(KERN_DEBUG "%0*lx%s", (int)sizeof(evtchn_pending[0])*2,
+ evtchn_pending[i],
i % 8 == 0 ? "\n " : " ");
printk(KERN_DEBUG "\nglobal mask:\n ");
- for (i = ARRAY_SIZE(sh->evtchn_mask)-1; i >= 0; i--)
+ for (i = nr_elems-1; i >= 0; i--)
printk(KERN_DEBUG "%0*lx%s",
- (int)(sizeof(sh->evtchn_mask[0])*2),
- sh->evtchn_mask[i],
+ (int)(sizeof(evtchn_mask[0])*2),
+ evtchn_mask[i],
i % 8 == 0 ? "\n " : " ");
printk(KERN_DEBUG "\nglobally unmasked:\n ");
- for (i = ARRAY_SIZE(sh->evtchn_mask)-1; i >= 0; i--)
- printk(KERN_DEBUG "%0*lx%s", (int)(sizeof(sh->evtchn_mask[0])*2),
- sh->evtchn_pending[i] & ~sh->evtchn_mask[i],
+ for (i = nr_elems-1; i >= 0; i--)
+ printk(KERN_DEBUG "%0*lx%s", (int)(sizeof(evtchn_mask[0])*2),
+ evtchn_pending[i] & ~evtchn_mask[i],
i % 8 == 0 ? "\n " : " ");
printk(KERN_DEBUG "\nlocal cpu%d mask:\n ", cpu);
@@ -1222,32 +1258,30 @@ irqreturn_t xen_debug_interrupt(int irq, void *dev_id)
i % 8 == 0 ? "\n " : " ");
printk(KERN_DEBUG "\nlocally unmasked:\n ");
- for (i = ARRAY_SIZE(sh->evtchn_mask)-1; i >= 0; i--) {
- unsigned long pending = sh->evtchn_pending[i]
- & ~sh->evtchn_mask[i]
+ for (i = nr_elems-1; i >= 0; i--) {
+ unsigned long pending = evtchn_pending[i]
+ & ~evtchn_mask[i]
& cpu_evtchn[i];
- printk(KERN_DEBUG "%0*lx%s", (int)(sizeof(sh->evtchn_mask[0])*2),
+ printk(KERN_DEBUG "%0*lx%s", (int)(sizeof(evtchn_mask[0])*2),
pending, i % 8 == 0 ? "\n " : " ");
}
printk(KERN_DEBUG "\npending list:\n");
- for (i = 0; i < NR_EVENT_CHANNELS; i++) {
- if (sync_test_bit(i, sh->evtchn_pending)) {
+ for (i = 0; i < NR_EVENT_CHANNELS_L2; i++) {
+ if (sync_test_bit(i, evtchn_pending)) {
int word_idx = i / BITS_PER_LONG;
printk(KERN_DEBUG " %d: event %d -> irq %d%s%s%s\n",
cpu_from_evtchn(i), i,
evtchn_to_irq[i],
sync_test_bit(word_idx, &v->evtchn_pending_sel)
? "" : " l1-clear",
- !sync_test_bit(i, sh->evtchn_mask)
+ !sync_test_bit(i, evtchn_mask)
? "" : " globally-masked",
sync_test_bit(i, cpu_evtchn)
? "" : " locally-masked");
}
}
- spin_unlock_irqrestore(&debug_lock, flags);
-
return IRQ_HANDLED;
}
@@ -1269,13 +1303,12 @@ static DEFINE_PER_CPU(unsigned int, current_bit_idx);
* a bitset of words which contain pending event bits. The second
* level is a bitset of pending events themselves.
*/
-static void __xen_evtchn_do_upcall(void)
+static void __xen_evtchn_do_upcall_l2(void)
{
int start_word_idx, start_bit_idx;
int word_idx, bit_idx;
int i;
int cpu = get_cpu();
- struct shared_info *s = HYPERVISOR_shared_info;
struct vcpu_info *vcpu_info = __this_cpu_read(xen_vcpu);
unsigned count;
@@ -1314,7 +1347,7 @@ static void __xen_evtchn_do_upcall(void)
}
word_idx = __ffs(words);
- pending_bits = active_evtchns(cpu, s, word_idx);
+ pending_bits = active_evtchns(cpu, word_idx);
bit_idx = 0; /* usually scan entire word from start */
if (word_idx == start_word_idx) {
/* We scan the starting word in two parts */
@@ -1383,7 +1416,13 @@ void xen_evtchn_do_upcall(struct pt_regs *regs)
exit_idle();
irq_enter();
- __xen_evtchn_do_upcall();
+ switch (evtchn_level) {
+ case 2:
+ __xen_evtchn_do_upcall_l2();
+ break;
+ default:
+ BUG();
+ }
irq_exit();
set_irq_regs(old_regs);
@@ -1391,7 +1430,13 @@ void xen_evtchn_do_upcall(struct pt_regs *regs)
void xen_hvm_evtchn_do_upcall(void)
{
- __xen_evtchn_do_upcall();
+ switch (evtchn_level) {
+ case 2:
+ __xen_evtchn_do_upcall_l2();
+ break;
+ default:
+ BUG();
+ }
}
EXPORT_SYMBOL_GPL(xen_hvm_evtchn_do_upcall);
@@ -1465,7 +1510,6 @@ static int set_affinity_irq(struct irq_data *data, const struct cpumask *dest,
int resend_irq_on_evtchn(unsigned int irq)
{
int masked, evtchn = evtchn_from_irq(irq);
- struct shared_info *s = HYPERVISOR_shared_info;
if (!VALID_EVTCHN(evtchn))
return 1;
@@ -1513,7 +1557,6 @@ static void mask_ack_dynirq(struct irq_data *data)
static int retrigger_dynirq(struct irq_data *data)
{
int evtchn = evtchn_from_irq(data->irq);
- struct shared_info *sh = HYPERVISOR_shared_info;
int ret = 0;
if (VALID_EVTCHN(evtchn)) {
@@ -1689,14 +1732,14 @@ void xen_irq_resume(void)
init_evtchn_cpu_bindings();
/* New event-channel space is not 'live' yet. */
- for (evtchn = 0; evtchn < NR_EVENT_CHANNELS; evtchn++)
+ for (evtchn = 0; evtchn < nr_event_channels; evtchn++)
mask_evtchn(evtchn);
/* No IRQ <-> event-channel mappings. */
list_for_each_entry(info, &xen_irq_list_head, list)
info->evtchn = 0; /* zap event-channel binding */
- for (evtchn = 0; evtchn < NR_EVENT_CHANNELS; evtchn++)
+ for (evtchn = 0; evtchn < nr_event_channels; evtchn++)
evtchn_to_irq[evtchn] = -1;
for_each_possible_cpu(cpu) {
@@ -1792,17 +1835,24 @@ void xen_callback_vector(void) {}
void __init xen_init_IRQ(void)
{
int i, rc;
+ struct shared_info *s = HYPERVISOR_shared_info;
+
+ evtchn_pending = s->evtchn_pending;
+ evtchn_mask = s->evtchn_mask;
+
+ evtchn_level = 2;
+ nr_event_channels = NR_EVENT_CHANNELS_L2;
- evtchn_to_irq = kcalloc(NR_EVENT_CHANNELS, sizeof(*evtchn_to_irq),
+ evtchn_to_irq = kcalloc(nr_event_channels, sizeof(*evtchn_to_irq),
GFP_KERNEL);
BUG_ON(!evtchn_to_irq);
- for (i = 0; i < NR_EVENT_CHANNELS; i++)
+ for (i = 0; i < nr_event_channels; i++)
evtchn_to_irq[i] = -1;
init_evtchn_cpu_bindings();
/* No event channels are 'live' right now. */
- for (i = 0; i < NR_EVENT_CHANNELS; i++)
+ for (i = 0; i < nr_event_channels; i++)
mask_evtchn(i);
pirq_needs_eoi = pirq_needs_eoi_flag;
diff --git a/drivers/xen/evtchn.c b/drivers/xen/evtchn.c
index ef02dfa..3eecdd7 100644
--- a/drivers/xen/evtchn.c
+++ b/drivers/xen/evtchn.c
@@ -232,7 +232,7 @@ static ssize_t evtchn_write(struct file *file, const char __user *buf,
for (i = 0; i < (count/sizeof(evtchn_port_t)); i++) {
unsigned port = kbuf[i];
- if (port < NR_EVENT_CHANNELS &&
+ if (port < nr_event_channels &&
get_port_user(port) == u &&
!get_port_enabled(port)) {
set_port_enabled(port, true);
@@ -373,7 +373,7 @@ static long evtchn_ioctl(struct file *file,
break;
rc = -EINVAL;
- if (unbind.port >= NR_EVENT_CHANNELS)
+ if (unbind.port >= nr_event_channels)
break;
spin_lock_irq(&port_user_lock);
@@ -401,7 +401,7 @@ static long evtchn_ioctl(struct file *file,
if (copy_from_user(¬ify, uarg, sizeof(notify)))
break;
- if (notify.port >= NR_EVENT_CHANNELS) {
+ if (notify.port >= nr_event_channels) {
rc = -EINVAL;
} else if (get_port_user(notify.port) != u) {
rc = -ENOTCONN;
@@ -491,7 +491,7 @@ static int evtchn_release(struct inode *inode, struct file *filp)
free_page((unsigned long)u->ring);
- for (i = 0; i < NR_EVENT_CHANNELS; i++) {
+ for (i = 0; i < nr_event_channels; i++) {
if (get_port_user(i) != u)
continue;
@@ -500,7 +500,7 @@ static int evtchn_release(struct inode *inode, struct file *filp)
spin_unlock_irq(&port_user_lock);
- for (i = 0; i < NR_EVENT_CHANNELS; i++) {
+ for (i = 0; i < nr_event_channels; i++) {
if (get_port_user(i) != u)
continue;
@@ -537,7 +537,7 @@ static int __init evtchn_init(void)
if (!xen_domain())
return -ENODEV;
- port_user = kcalloc(NR_EVENT_CHANNELS, sizeof(*port_user), GFP_KERNEL);
+ port_user = kcalloc(nr_event_channels, sizeof(*port_user), GFP_KERNEL);
if (port_user == NULL)
return -ENOMEM;
diff --git a/include/xen/events.h b/include/xen/events.h
index 04399b2..6b117ac 100644
--- a/include/xen/events.h
+++ b/include/xen/events.h
@@ -109,4 +109,6 @@ int xen_irq_from_gsi(unsigned gsi);
/* Determine whether to ignore this IRQ if it is passed to a guest. */
int xen_test_irq_shared(int irq);
+extern unsigned int nr_event_channels;
+
#endif /* _XEN_EVENTS_H */
--
1.7.10.4
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH V2 09/13] xen: dynamically allocate cpu_evtchn_mask
2013-02-04 17:23 [PATCH V2] Implement 3-level event channel in Linux Wei Liu
` (7 preceding siblings ...)
2013-02-04 17:23 ` [PATCH V2 08/13] xen: generalized event channel operations Wei Liu
@ 2013-02-04 17:23 ` Wei Liu
2013-02-04 17:23 ` [PATCH V2 10/13] xen: implement 3-level event channel routines Wei Liu
` (3 subsequent siblings)
12 siblings, 0 replies; 15+ messages in thread
From: Wei Liu @ 2013-02-04 17:23 UTC (permalink / raw)
To: xen-devel; +Cc: david.vrabel, Wei Liu, ian.campbell, jbeulich, konrad.wilk
Signed-off-by: Wei Liu <wei.liu2@citrix.com>
---
drivers/xen/events.c | 18 +++++++++++++++---
1 file changed, 15 insertions(+), 3 deletions(-)
diff --git a/drivers/xen/events.c b/drivers/xen/events.c
index 355dbda..94d6570 100644
--- a/drivers/xen/events.c
+++ b/drivers/xen/events.c
@@ -123,8 +123,7 @@ static int *evtchn_to_irq;
static unsigned long *pirq_eoi_map;
static bool (*pirq_needs_eoi)(unsigned irq);
-static DEFINE_PER_CPU(unsigned long [NR_EVENT_CHANNELS_L2/BITS_PER_LONG],
- cpu_evtchn_mask);
+static DEFINE_PER_CPU(unsigned long *, cpu_evtchn_mask);
/* Xen will never allocate port zero for any purpose. */
#define VALID_EVTCHN(chn) ((chn) != 0)
@@ -333,7 +332,8 @@ static void init_evtchn_cpu_bindings(void)
for_each_possible_cpu(i)
memset(per_cpu(cpu_evtchn_mask, i),
- (i == 0) ? ~0 : 0, sizeof(*per_cpu(cpu_evtchn_mask, i)));
+ (i == 0) ? ~0 : 0,
+ sizeof(unsigned long)*(nr_event_channels/BITS_PER_LONG));
}
static inline void clear_evtchn(int port)
@@ -1835,6 +1835,7 @@ void xen_callback_vector(void) {}
void __init xen_init_IRQ(void)
{
int i, rc;
+ int cpu;
struct shared_info *s = HYPERVISOR_shared_info;
evtchn_pending = s->evtchn_pending;
@@ -1849,6 +1850,17 @@ void __init xen_init_IRQ(void)
for (i = 0; i < nr_event_channels; i++)
evtchn_to_irq[i] = -1;
+ for_each_possible_cpu(cpu) {
+ void *p;
+ unsigned int nr = nr_event_channels / BITS_PER_LONG;
+
+ p = kzalloc_node(sizeof(unsigned long) * nr,
+ GFP_KERNEL,
+ cpu_to_node(cpu));
+ BUG_ON(!p);
+ per_cpu(cpu_evtchn_mask, cpu) = p;
+ }
+
init_evtchn_cpu_bindings();
/* No event channels are 'live' right now. */
--
1.7.10.4
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH V2 10/13] xen: implement 3-level event channel routines
2013-02-04 17:23 [PATCH V2] Implement 3-level event channel in Linux Wei Liu
` (8 preceding siblings ...)
2013-02-04 17:23 ` [PATCH V2 09/13] xen: dynamically allocate cpu_evtchn_mask Wei Liu
@ 2013-02-04 17:23 ` Wei Liu
2013-02-04 17:23 ` [PATCH V2 11/13] xen: introduce xen_event_channel_register_3level Wei Liu
` (2 subsequent siblings)
12 siblings, 0 replies; 15+ messages in thread
From: Wei Liu @ 2013-02-04 17:23 UTC (permalink / raw)
To: xen-devel; +Cc: david.vrabel, Wei Liu, ian.campbell, jbeulich, konrad.wilk
Only do_upcall, debug_interrupt and unmask_evtchn are required.
Signed-off-by: Wei Liu <wei.liu2@citrix.com>
---
drivers/xen/events.c | 291 ++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 291 insertions(+)
diff --git a/drivers/xen/events.c b/drivers/xen/events.c
index 94d6570..f3f0fdc 100644
--- a/drivers/xen/events.c
+++ b/drivers/xen/events.c
@@ -51,6 +51,9 @@
#include <xen/interface/hvm/hvm_op.h>
#include <xen/interface/hvm/params.h>
+/* Helper macro(s) */
+#define LONG_BITORDER (BITS_PER_LONG == 64 ? 6 : 5)
+
/* N-level event channel, starting from 2 */
unsigned int evtchn_level = 2;
EXPORT_SYMBOL_GPL(evtchn_level);
@@ -61,6 +64,9 @@ EXPORT_SYMBOL_GPL(nr_event_channels);
static unsigned long *evtchn_pending;
static unsigned long *evtchn_mask;
+/* 2nd level selector for 3-level event channel */
+static DEFINE_PER_CPU(unsigned long[sizeof(unsigned long) * 8], evtchn_sel_l2);
+
/*
* This lock protects updates to the following mapping and reference-count
* arrays. The lock does not need to be acquired to read the mapping tables.
@@ -396,6 +402,28 @@ static inline void __unmask_local_port_l2(int port)
vcpu_info->evtchn_upcall_pending = 1;
}
+static inline void __unmask_local_port_l3(int port)
+{
+ struct vcpu_info *vcpu_info = __this_cpu_read(xen_vcpu);
+ int cpu = smp_processor_id();
+ unsigned int l1bit = port >> (LONG_BITORDER << 1);
+ unsigned int l2bit = port >> LONG_BITORDER;
+
+ sync_clear_bit(port, &evtchn_mask[0]);
+
+ /*
+ * The following is basically the equivalent of
+ * 'hw_resend_irq'. Just like a real IO-APIC we 'lose
+ * the interrupt edge' if the channel is masked.
+ */
+ if (sync_test_bit(port, &evtchn_pending[0]) &&
+ !sync_test_and_set_bit(l2bit,
+ &per_cpu(evtchn_sel_l2, cpu)[0]) &&
+ !sync_test_and_set_bit(l1bit,
+ &vcpu_info->evtchn_pending_sel))
+ vcpu_info->evtchn_upcall_pending = 1;
+}
+
static void unmask_evtchn(int port)
{
unsigned int cpu = get_cpu();
@@ -411,6 +439,9 @@ static void unmask_evtchn(int port)
case 2:
__unmask_local_port_l2(port);
break;
+ case 3:
+ __unmask_local_port_l3(port);
+ break;
default:
BUG();
}
@@ -1185,6 +1216,7 @@ void xen_send_IPI_one(unsigned int cpu, enum ipi_vector vector)
}
static irqreturn_t xen_debug_interrupt_l2(int irq, void *dev_id);
+static irqreturn_t xen_debug_interrupt_l3(int irq, void *dev_id);
irqreturn_t xen_debug_interrupt(int irq, void *dev_id)
{
@@ -1215,6 +1247,9 @@ irqreturn_t xen_debug_interrupt(int irq, void *dev_id)
case 2:
rc = xen_debug_interrupt_l2(irq, dev_id);
break;
+ case 3:
+ rc = xen_debug_interrupt_l3(irq, dev_id);
+ break;
default:
BUG();
}
@@ -1285,8 +1320,109 @@ static irqreturn_t xen_debug_interrupt_l2(int irq, void *dev_id)
return IRQ_HANDLED;
}
+static irqreturn_t xen_debug_interrupt_l3(int irq, void *dev_id)
+{
+ int cpu = smp_processor_id();
+ unsigned long *cpu_evtchn = per_cpu(cpu_evtchn_mask, cpu);
+ unsigned long nr_elems = NR_EVENT_CHANNELS_L3 / BITS_PER_LONG;
+ int i;
+ struct vcpu_info *v;
+
+ v = per_cpu(xen_vcpu, cpu);
+
+ printk(KERN_DEBUG "\npending (only show words which have bits set to 1):\n ");
+ for (i = nr_elems-1; i >= 0; i--)
+ if (evtchn_pending[i] != 0UL) {
+ printk(KERN_DEBUG " word index %d %0*lx\n",
+ i,
+ (int)sizeof(evtchn_pending[0])*2,
+ evtchn_pending[i]);
+ }
+
+ printk(KERN_DEBUG "\nglobal mask (only show words which have bits set to 0):\n ");
+ for (i = nr_elems-1; i >= 0; i--)
+ if (evtchn_mask[i] != ~0UL) {
+ printk(KERN_DEBUG " word index %d %0*lx\n",
+ i,
+ (int)sizeof(evtchn_mask[0])*2,
+ evtchn_mask[i]);
+ }
+
+ printk(KERN_DEBUG "\nglobally unmasked (only show result words which have bits set to 1):\n ");
+ for (i = nr_elems-1; i >= 0; i--)
+ if ((evtchn_pending[i] & ~evtchn_mask[i]) != 0UL) {
+ printk(KERN_DEBUG " word index %d %0*lx\n",
+ i,
+ (int)(sizeof(evtchn_mask[0])*2),
+ evtchn_pending[i] & ~evtchn_mask[i]);
+ }
+
+ printk(KERN_DEBUG "\nlocal cpu%d mask (only show words which have bits set to 1):\n ", cpu);
+ for (i = (NR_EVENT_CHANNELS_L3/BITS_PER_LONG)-1; i >= 0; i--)
+ if (cpu_evtchn[i] != 0UL) {
+ printk(KERN_DEBUG " word index %d %0*lx\n",
+ i,
+ (int)(sizeof(cpu_evtchn[0])*2),
+ cpu_evtchn[i]);
+ }
+
+ printk(KERN_DEBUG "\nlocally unmasked (only show result words which have bits set to 1):\n ");
+ for (i = nr_elems-1; i >= 0; i--) {
+ unsigned long pending = evtchn_pending[i]
+ & ~evtchn_mask[i]
+ & cpu_evtchn[i];
+ if (pending != 0UL) {
+ printk(KERN_DEBUG " word index %d %0*lx\n",
+ i,
+ (int)(sizeof(evtchn_mask[0])*2),
+ pending);
+ }
+ }
+
+ printk(KERN_DEBUG "\npending list:\n");
+ for (i = 0; i < NR_EVENT_CHANNELS_L3; i++) {
+ if (sync_test_bit(i, evtchn_pending)) {
+ int word_idx = i / (BITS_PER_LONG * BITS_PER_LONG);
+ int word_idx_l2 = i / BITS_PER_LONG;
+ printk(KERN_DEBUG " %d: event %d -> irq %d%s%s%s%s\n",
+ cpu_from_evtchn(i), i,
+ evtchn_to_irq[i],
+ !sync_test_bit(word_idx, &v->evtchn_pending_sel)
+ ? "" : " l1-clear",
+ !sync_test_bit(word_idx_l2, per_cpu(evtchn_sel_l2, cpu))
+ ? "" : " l2-clear",
+ sync_test_bit(i, evtchn_mask)
+ ? "" : " globally-masked",
+ sync_test_bit(i, cpu_evtchn)
+ ? "" : " locally-masked");
+ }
+ }
+
+ return IRQ_HANDLED;
+}
+
+/* The following per-cpu variables are used to save current state of event
+ * processing loop.
+ *
+ * 2-level event channel:
+ * current_word_idx is the bit index in L1 selector indicating the currently
+ * processing word in shared bitmap.
+ * current_bit_idx is the bit index in the currently processing word in shared
+ * bitmap.
+ * N.B. current_word_idx_l2 is not used.
+ *
+ * 3-level event channel:
+ * current_word_idx is the bit index in L1 selector indicating the currently
+ * processing word in L2 selector.
+ * current_word_idx_l2 is the bit index in L2 selector word indicating the
+ * currently processing word in shared bitmap.
+ * current_bit_idx is the bit index in the currently processing word in shared
+ * bitmap.
+ *
+ */
static DEFINE_PER_CPU(unsigned, xed_nesting_count);
static DEFINE_PER_CPU(unsigned int, current_word_idx);
+static DEFINE_PER_CPU(unsigned int, current_word_idx_l2);
static DEFINE_PER_CPU(unsigned int, current_bit_idx);
/*
@@ -1409,6 +1545,155 @@ out:
put_cpu();
}
+/*
+ * In the 3-level event channel implementation, the first level is a
+ * bitset of words which contain pending bits in the second level.
+ * The second level is another bitsets which contain pending bits in
+ * the third level. The third level is a bit set of pending events
+ * themselves.
+ */
+static void __xen_evtchn_do_upcall_l3(void)
+{
+ struct vcpu_info *vcpu_info = __this_cpu_read(xen_vcpu);
+ unsigned count;
+ int start_word_idx_l1, start_word_idx_l2, start_bit_idx;
+ int word_idx_l1, word_idx_l2, bit_idx;
+ int i, j;
+ int cpu = get_cpu();
+
+ do {
+ unsigned long pending_words_l1;
+
+ vcpu_info->evtchn_upcall_pending = 0;
+
+ if (__this_cpu_inc_return(xed_nesting_count) - 1)
+ goto out;
+#ifndef CONFIG_X86
+ /* No need for a barrier -- XCHG is a barrier on x86. */
+ /* Clear master flag /before/ clearing selector flag. */
+ wmb();
+#endif
+ /* here we get l1 pending selector */
+ pending_words_l1 = xchg(&vcpu_info->evtchn_pending_sel, 0);
+
+ start_word_idx_l1 = __this_cpu_read(current_word_idx);
+ start_word_idx_l2 = __this_cpu_read(current_word_idx_l2);
+ start_bit_idx = __this_cpu_read(current_bit_idx);
+
+ word_idx_l1 = start_word_idx_l1;
+
+ /* loop through l1, try to pick up l2 */
+ for (i = 0; pending_words_l1 != 0; i++) {
+ unsigned long words_l1;
+ unsigned long pending_words_l2;
+
+ words_l1 = MASK_LSBS(pending_words_l1, word_idx_l1);
+
+ if (words_l1 == 0) {
+ word_idx_l1 = 0;
+ start_word_idx_l2 = 0;
+ continue;
+ }
+
+ word_idx_l1 = __ffs(words_l1);
+
+ pending_words_l2 =
+ xchg(&per_cpu(evtchn_sel_l2, cpu)[word_idx_l1],
+ 0);
+
+ word_idx_l2 = 0;
+ if (word_idx_l1 == start_word_idx_l1) {
+ if (i == 0)
+ word_idx_l2 = start_word_idx_l2;
+ else
+ word_idx_l2 &= (1UL << start_word_idx_l2) - 1;
+ }
+
+ for (j = 0; pending_words_l2 != 0; j++) {
+ unsigned long pending_bits;
+ unsigned long words_l2;
+ unsigned long idx;
+
+ words_l2 = MASK_LSBS(pending_words_l2,
+ word_idx_l2);
+
+ if (words_l2 == 0) {
+ word_idx_l2 = 0;
+ bit_idx = 0;
+ continue;
+ }
+
+ word_idx_l2 = __ffs(words_l2);
+
+ idx = word_idx_l1*BITS_PER_LONG+word_idx_l2;
+ pending_bits =
+ active_evtchns(cpu, idx);
+
+ bit_idx = 0;
+ if (word_idx_l2 == start_word_idx_l2) {
+ if (j == 0)
+ bit_idx = start_bit_idx;
+ else
+ bit_idx &= (1UL<<start_bit_idx)-1;
+ }
+
+ /* process port */
+ do {
+ unsigned long bits;
+ int port, irq;
+ struct irq_desc *desc;
+
+ bits = MASK_LSBS(pending_bits, bit_idx);
+
+ if (bits == 0)
+ break;
+
+ bit_idx = __ffs(bits);
+
+ port = (word_idx_l1 << (LONG_BITORDER << 1)) +
+ (word_idx_l2 << LONG_BITORDER) +
+ bit_idx;
+
+ irq = evtchn_to_irq[port];
+
+ if (irq != -1) {
+ desc = irq_to_desc(irq);
+ if (desc)
+ generic_handle_irq_desc(irq, desc);
+ }
+
+ bit_idx = (bit_idx + 1) % BITS_PER_LONG;
+
+ __this_cpu_write(current_bit_idx, bit_idx);
+ __this_cpu_write(current_word_idx_l2,
+ bit_idx ? word_idx_l2 :
+ (word_idx_l2+1) % BITS_PER_LONG);
+ __this_cpu_write(current_word_idx_l2,
+ word_idx_l2 ? word_idx_l1 :
+ (word_idx_l1+1) % BITS_PER_LONG);
+ } while (bit_idx != 0);
+
+ if ((word_idx_l2 != start_word_idx_l2) || (j != 0))
+ pending_words_l2 &= ~(1UL << word_idx_l2);
+
+ word_idx_l2 = (word_idx_l2 + 1) % BITS_PER_LONG;
+ }
+
+ if ((word_idx_l1 != start_word_idx_l1) || (i != 0))
+ pending_words_l1 &= ~(1UL << word_idx_l1);
+
+ word_idx_l1 = (word_idx_l1 + 1) % BITS_PER_LONG;
+ }
+
+ BUG_ON(!irqs_disabled());
+ count = __this_cpu_read(xed_nesting_count);
+ __this_cpu_write(xed_nesting_count, 0);
+ } while (count != 1 || vcpu_info->evtchn_upcall_pending);
+
+out:
+ put_cpu();
+}
+
void xen_evtchn_do_upcall(struct pt_regs *regs)
{
struct pt_regs *old_regs = set_irq_regs(regs);
@@ -1420,6 +1705,9 @@ void xen_evtchn_do_upcall(struct pt_regs *regs)
case 2:
__xen_evtchn_do_upcall_l2();
break;
+ case 3:
+ __xen_evtchn_do_upcall_l3();
+ break;
default:
BUG();
}
@@ -1434,6 +1722,9 @@ void xen_hvm_evtchn_do_upcall(void)
case 2:
__xen_evtchn_do_upcall_l2();
break;
+ case 3:
+ __xen_evtchn_do_upcall_l3();
+ break;
default:
BUG();
}
--
1.7.10.4
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH V2 11/13] xen: introduce xen_event_channel_register_3level
2013-02-04 17:23 [PATCH V2] Implement 3-level event channel in Linux Wei Liu
` (9 preceding siblings ...)
2013-02-04 17:23 ` [PATCH V2 10/13] xen: implement 3-level event channel routines Wei Liu
@ 2013-02-04 17:23 ` Wei Liu
2013-02-04 17:23 ` [PATCH V2 12/13] xen: introduce xen_event_channel_register_nlevel Wei Liu
2013-02-04 17:23 ` [PATCH V2 13/13] xen: register 3-level event channel Wei Liu
12 siblings, 0 replies; 15+ messages in thread
From: Wei Liu @ 2013-02-04 17:23 UTC (permalink / raw)
To: xen-devel; +Cc: david.vrabel, Wei Liu, ian.campbell, jbeulich, konrad.wilk
Signed-off-by: Wei Liu <wei.liu2@citrix.com>
---
drivers/xen/events.c | 94 ++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 94 insertions(+)
diff --git a/drivers/xen/events.c b/drivers/xen/events.c
index f3f0fdc..6e0283c 100644
--- a/drivers/xen/events.c
+++ b/drivers/xen/events.c
@@ -53,6 +53,9 @@
/* Helper macro(s) */
#define LONG_BITORDER (BITS_PER_LONG == 64 ? 6 : 5)
+/* event bitmap size: 1 page for 32 bit and 8 pages for 64 bit */
+#define BITMAP_PG_ORDER (BITS_PER_LONG == 64 ? 3 : 1)
+#define BITMAP_NR_PAGES (BITMAP_PG_ORDER == 3 ? 8 : 1)
/* N-level event channel, starting from 2 */
unsigned int evtchn_level = 2;
@@ -2123,6 +2126,97 @@ void xen_callback_vector(void)
void xen_callback_vector(void) {}
#endif
+static int xen_event_channel_register_3level(void)
+{
+ evtchn_register_nlevel_t reg;
+ int i, cpu;
+ unsigned long *_evtchn_pending = NULL;
+ unsigned long *_evtchn_mask = NULL;
+ unsigned long *l2sel_mfns = NULL;
+ unsigned long *l2sel_offsets = NULL;
+ int rc;
+
+ /* If we come from restore path, we don't need to allocate
+ * pages.
+ */
+ if (!evtchn_pending && !evtchn_mask) {
+ evtchn_pending =
+ (unsigned long *)__get_free_pages(GFP_KERNEL,
+ BITMAP_PG_ORDER);
+ evtchn_mask =
+ (unsigned long *)__get_free_pages(GFP_KERNEL,
+ BITMAP_PG_ORDER);
+ if (!evtchn_pending || !evtchn_mask) {
+ free_pages((unsigned long)evtchn_pending, BITMAP_PG_ORDER);
+ free_pages((unsigned long)evtchn_mask, BITMAP_PG_ORDER);
+ evtchn_pending = NULL;
+ evtchn_mask = NULL;
+ rc = -ENOMEM;
+ goto err;
+ }
+ }
+
+ rc = -ENOMEM; /* Common error code for following operations */
+#define __ALLOC_ARRAY(_ptr, _nr) \
+ do { \
+ (_ptr) = kzalloc(sizeof(unsigned long) * (_nr), \
+ GFP_KERNEL); \
+ if (!(_ptr)) \
+ goto out; \
+ } while (0)
+
+ __ALLOC_ARRAY(_evtchn_pending, BITMAP_NR_PAGES);
+ __ALLOC_ARRAY(_evtchn_mask, BITMAP_NR_PAGES);
+ __ALLOC_ARRAY(l2sel_mfns, nr_cpu_ids);
+ __ALLOC_ARRAY(l2sel_offsets, nr_cpu_ids);
+#undef __ALLOC_ARRAY
+
+ memset(®, 0, sizeof(reg));
+
+ for (i = 0; i < BITMAP_NR_PAGES; i++) {
+ unsigned long offset = PAGE_SIZE * i;
+ _evtchn_pending[i] =
+ arbitrary_virt_to_mfn(
+ (void *)((unsigned long)evtchn_pending+offset));
+ _evtchn_mask[i] =
+ arbitrary_virt_to_mfn(
+ (void *)((unsigned long)evtchn_mask+offset));
+ }
+
+ for_each_possible_cpu(cpu) {
+ l2sel_mfns[cpu] =
+ arbitrary_virt_to_mfn(&per_cpu(evtchn_sel_l2, cpu));
+ l2sel_offsets[cpu] =
+ offset_in_page(&per_cpu(evtchn_sel_l2, cpu));
+ }
+
+ reg.u.l3.nr_pages = BITMAP_NR_PAGES;
+ reg.u.l3.evtchn_pending = _evtchn_pending;
+ reg.u.l3.evtchn_mask = _evtchn_mask;
+
+ reg.u.l3.nr_vcpus = nr_cpu_ids;
+ reg.u.l3.l2sel_mfns = l2sel_mfns;
+ reg.u.l3.l2sel_offsets = l2sel_offsets;
+
+ reg.level = 3;
+
+ rc = HYPERVISOR_event_channel_op(EVTCHNOP_register_nlevel, ®);
+ if (rc) {
+ free_pages((unsigned long)evtchn_pending, BITMAP_PG_ORDER);
+ free_pages((unsigned long)evtchn_mask, BITMAP_PG_ORDER);
+ evtchn_pending = NULL;
+ evtchn_mask = NULL;
+ }
+
+out:
+ kfree(_evtchn_pending);
+ kfree(_evtchn_mask);
+ kfree(l2sel_mfns);
+ kfree(l2sel_offsets);
+err:
+ return rc;
+}
+
void __init xen_init_IRQ(void)
{
int i, rc;
--
1.7.10.4
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH V2 12/13] xen: introduce xen_event_channel_register_nlevel
2013-02-04 17:23 [PATCH V2] Implement 3-level event channel in Linux Wei Liu
` (10 preceding siblings ...)
2013-02-04 17:23 ` [PATCH V2 11/13] xen: introduce xen_event_channel_register_3level Wei Liu
@ 2013-02-04 17:23 ` Wei Liu
2013-02-04 17:23 ` [PATCH V2 13/13] xen: register 3-level event channel Wei Liu
12 siblings, 0 replies; 15+ messages in thread
From: Wei Liu @ 2013-02-04 17:23 UTC (permalink / raw)
To: xen-devel; +Cc: david.vrabel, Wei Liu, ian.campbell, jbeulich, konrad.wilk
Signed-off-by: Wei Liu <wei.liu2@citrix.com>
---
drivers/xen/events.c | 55 ++++++++++++++++++++++++++++++++++++++++++++------
include/xen/events.h | 7 +++++++
2 files changed, 56 insertions(+), 6 deletions(-)
diff --git a/drivers/xen/events.c b/drivers/xen/events.c
index 6e0283c..6a0307c 100644
--- a/drivers/xen/events.c
+++ b/drivers/xen/events.c
@@ -2217,17 +2217,60 @@ err:
return rc;
}
+void xen_event_channel_register_nlevel(unsigned int highest_level)
+{
+ int rc;
+
+ switch (highest_level) {
+ case 3:
+ rc = xen_event_channel_register_3level();
+ if (rc == 0) {
+ printk(KERN_INFO "Register 3-level event channel succeed.\n");
+ xen_set_event_channel_nlevel(3);
+ break;
+ } else {
+ printk(KERN_INFO "Regsiter 3-level event channel failed with %d\n",
+ rc);
+ }
+ /* fall through */
+ case 2:
+ printk(KERN_INFO "Using 2-level event channel\n");
+ xen_set_event_channel_nlevel(2);
+ break;
+ default:
+ printk(KERN_INFO "Trying to register unsupported %d-level event channel\n",
+ highest_level);
+ }
+}
+
+void xen_set_event_channel_nlevel(unsigned int level)
+{
+ struct shared_info *s = HYPERVISOR_shared_info;
+ switch (level) {
+ case 2:
+ evtchn_pending = s->evtchn_pending;
+ evtchn_mask = s->evtchn_mask;
+ evtchn_level = 2;
+ nr_event_channels = NR_EVENT_CHANNELS_L2;
+ break;
+ case 3:
+ /* evtchn_pending/mask already set */
+ evtchn_level = 3;
+ nr_event_channels = NR_EVENT_CHANNELS_L3;
+ break;
+ default:
+ printk(KERN_EMERG "Trying to set unsupported %d-level event channel\n",
+ level);
+ BUG();
+ }
+}
+
void __init xen_init_IRQ(void)
{
int i, rc;
int cpu;
- struct shared_info *s = HYPERVISOR_shared_info;
-
- evtchn_pending = s->evtchn_pending;
- evtchn_mask = s->evtchn_mask;
- evtchn_level = 2;
- nr_event_channels = NR_EVENT_CHANNELS_L2;
+ xen_set_event_channel_nlevel(2);
evtchn_to_irq = kcalloc(nr_event_channels, sizeof(*evtchn_to_irq),
GFP_KERNEL);
diff --git a/include/xen/events.h b/include/xen/events.h
index 6b117ac..6b6c4fb 100644
--- a/include/xen/events.h
+++ b/include/xen/events.h
@@ -111,4 +111,11 @@ int xen_test_irq_shared(int irq);
extern unsigned int nr_event_channels;
+/* Register N-level event channel. If register N-level fails, it will try to
+ * register N-1 level, until it reaches the default 2-level event channel. */
+void xen_event_channel_register_nlevel(unsigned int highest_level);
+
+/* Set event channel to N-level if registration succeed */
+void xen_set_event_channel_nlevel(unsigned int level);
+
#endif /* _XEN_EVENTS_H */
--
1.7.10.4
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH V2 13/13] xen: register 3-level event channel
2013-02-04 17:23 [PATCH V2] Implement 3-level event channel in Linux Wei Liu
` (11 preceding siblings ...)
2013-02-04 17:23 ` [PATCH V2 12/13] xen: introduce xen_event_channel_register_nlevel Wei Liu
@ 2013-02-04 17:23 ` Wei Liu
12 siblings, 0 replies; 15+ messages in thread
From: Wei Liu @ 2013-02-04 17:23 UTC (permalink / raw)
To: xen-devel; +Cc: david.vrabel, Wei Liu, ian.campbell, jbeulich, konrad.wilk
3-level event channel is registered in
a) xen_init_IRQ(), when the guest is fresh started;
b) xen_vcpu_restore(), when the guest is restored.
Signed-off-by: Wei Liu <wei.liu2@citrix.com>
---
arch/x86/xen/enlighten.c | 3 +++
drivers/xen/events.c | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index bc893e7..722a994 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -43,6 +43,7 @@
#include <xen/hvm.h>
#include <xen/hvc-console.h>
#include <xen/acpi.h>
+#include <xen/events.h>
#include <asm/paravirt.h>
#include <asm/apic.h>
@@ -195,6 +196,8 @@ void xen_vcpu_restore(void)
HYPERVISOR_vcpu_op(VCPUOP_up, cpu, NULL))
BUG();
}
+
+ xen_event_channel_register_nlevel(3);
}
static void __init xen_banner(void)
diff --git a/drivers/xen/events.c b/drivers/xen/events.c
index 6a0307c..849d049 100644
--- a/drivers/xen/events.c
+++ b/drivers/xen/events.c
@@ -2270,7 +2270,7 @@ void __init xen_init_IRQ(void)
int i, rc;
int cpu;
- xen_set_event_channel_nlevel(2);
+ xen_event_channel_register_nlevel(3);
evtchn_to_irq = kcalloc(nr_event_channels, sizeof(*evtchn_to_irq),
GFP_KERNEL);
--
1.7.10.4
^ permalink raw reply related [flat|nested] 15+ messages in thread