[RFC PATCH V5 07/14] xen: generalized event channel operations

[Wei Liu <wei.liu2@citrix.com>]

Use global pointers in common operations to allow for better code sharing
between 2 and 3 level event channel ABI.

Function pointers are used to deal with functions which are not suitable for
sharing.

Also update drivers/xen/evtchn.c to use exported variable instead of macro.

Signed-off-by: Wei Liu <wei.liu2@citrix.com>
---
 drivers/xen/events.c |  199 +++++++++++++++++++++++++++++++-------------------
 drivers/xen/evtchn.c |   13 ++--
 include/xen/events.h |    3 +
 3 files changed, 135 insertions(+), 80 deletions(-)

diff --git a/drivers/xen/events.c b/drivers/xen/events.c
index 6e226c3..217efb2 100644
--- a/drivers/xen/events.c
+++ b/drivers/xen/events.c
@@ -56,6 +56,27 @@
 #include <xen/interface/sched.h>
 #include <asm/hw_irq.h>
 
+/* extended event channel ABI in use, default is EVTCHN_EXTENDED_NONE */
+uint64_t xen_evtchn_extended = EVTCHN_EXTENDED_NONE;
+EXPORT_SYMBOL_GPL(xen_evtchn_extended);
+/* number of event channels */
+unsigned int xen_nr_event_channels;
+EXPORT_SYMBOL_GPL(xen_nr_event_channels);
+
+struct evtchn_ops {
+	void (*unmask)(int port);
+	irqreturn_t (*debug_interrupt)(int irq, void *dev_id);
+	void (*do_upcall)(void);
+};
+
+static const struct evtchn_ops *eops;
+
+/* The following pointers point to pending bitmap and mask bitmap. */
+static xen_ulong_t *evtchn_pending;
+static xen_ulong_t *evtchn_mask;
+/* The following per-cpu var points to selector(s). */
+static DEFINE_PER_CPU(xen_ulong_t *[1], evtchn_sel);
+
 /*
  * This lock protects updates to the following mapping and reference-count
  * arrays. The lock does not need to be acquired to read the mapping tables.
@@ -135,7 +156,7 @@ static bool (*pirq_needs_eoi)(unsigned irq);
 /* Find the first set bit in a evtchn mask */
 #define EVTCHN_FIRST_BIT(w) find_first_bit(BM(&(w)), BITS_PER_EVTCHN_WORD)
 
-static DEFINE_PER_CPU(xen_ulong_t [NR_EVENT_CHANNELS/BITS_PER_EVTCHN_WORD],
+static DEFINE_PER_CPU(xen_ulong_t [NR_EVENT_CHANNELS_L2/BITS_PER_EVTCHN_WORD],
 		      cpu_evtchn_mask);
 
 /* Xen will never allocate port zero for any purpose. */
@@ -310,12 +331,11 @@ static bool pirq_needs_eoi_flag(unsigned irq)
 }
 
 static inline xen_ulong_t active_evtchns(unsigned int cpu,
-					 struct shared_info *sh,
 					 unsigned int idx)
 {
-	return sh->evtchn_pending[idx] &
+	return evtchn_pending[idx] &
 		per_cpu(cpu_evtchn_mask, cpu)[idx] &
-		~sh->evtchn_mask[idx];
+		~evtchn_mask[idx];
 }
 
 static void bind_evtchn_to_cpu(unsigned int chn, unsigned int cpu)
@@ -353,26 +373,22 @@ static void init_evtchn_cpu_bindings(void)
 
 static inline void clear_evtchn(int port)
 {
-	struct shared_info *s = HYPERVISOR_shared_info;
-	sync_clear_bit(port, BM(&s->evtchn_pending[0]));
+	sync_clear_bit(port, BM(&evtchn_pending[0]));
 }
 
 static inline void set_evtchn(int port)
 {
-	struct shared_info *s = HYPERVISOR_shared_info;
-	sync_set_bit(port, BM(&s->evtchn_pending[0]));
+	sync_set_bit(port, BM(&evtchn_pending[0]));
 }
 
 static inline int test_evtchn(int port)
 {
-	struct shared_info *s = HYPERVISOR_shared_info;
-	return sync_test_bit(port, BM(&s->evtchn_pending[0]));
+	return sync_test_bit(port, BM(&evtchn_pending[0]));
 }
 
 static inline int test_and_set_mask(int port)
 {
-	struct shared_info *s = HYPERVISOR_shared_info;
-	return sync_test_and_set_bit(port, BM(&s->evtchn_mask[0]));
+	return sync_test_and_set_bit(port, BM(&evtchn_mask[0]));
 }
 
 
@@ -395,24 +411,40 @@ EXPORT_SYMBOL_GPL(notify_remote_via_irq);
 
 static void mask_evtchn(int port)
 {
-	struct shared_info *s = HYPERVISOR_shared_info;
-	sync_set_bit(port, BM(&s->evtchn_mask[0]));
+	sync_set_bit(port, BM(&evtchn_mask[0]));
+}
+
+static inline void __unmask_local_port_l2(int port)
+{
+	struct vcpu_info *vcpu_info = __this_cpu_read(xen_vcpu);
+	int cpu = smp_processor_id();
+
+	sync_clear_bit(port, BM(&evtchn_mask[0]));
+
+	/*
+	 * The following is basically the equivalent of
+	 * 'hw_resend_irq'. Just like a real IO-APIC we 'lose
+	 * the interrupt edge' if the channel is masked.
+	 */
+	if (sync_test_bit(port, BM(&evtchn_pending[0])) &&
+	    !sync_test_and_set_bit(port / BITS_PER_EVTCHN_WORD,
+				   BM(per_cpu(evtchn_sel, cpu)[0])))
+		vcpu_info->evtchn_upcall_pending = 1;
 }
 
 static void unmask_evtchn(int port)
 {
-	struct shared_info *s = HYPERVISOR_shared_info;
 	unsigned int cpu = get_cpu();
-	int do_hypercall = 0, evtchn_pending = 0;
+	int do_hypercall = 0, _evtchn_pending = 0;
 
 	BUG_ON(!irqs_disabled());
 
 	if (unlikely((cpu != cpu_from_evtchn(port))))
 		do_hypercall = 1;
 	else
-		evtchn_pending = sync_test_bit(port, BM(&s->evtchn_pending[0]));
+		_evtchn_pending = sync_test_bit(port, BM(&evtchn_pending[0]));
 
-	if (unlikely(evtchn_pending && xen_hvm_domain()))
+	if (unlikely(_evtchn_pending && xen_hvm_domain()))
 		do_hypercall = 1;
 
 	/* Slow path (hypercall) if this is a non-local port or if this is
@@ -421,21 +453,8 @@ static void unmask_evtchn(int port)
 	if (do_hypercall) {
 		struct evtchn_unmask unmask = { .port = port };
 		(void)HYPERVISOR_event_channel_op(EVTCHNOP_unmask, &unmask);
-	} else {
-		struct vcpu_info *vcpu_info = __this_cpu_read(xen_vcpu);
-
-		sync_clear_bit(port, BM(&s->evtchn_mask[0]));
-
-		/*
-		 * The following is basically the equivalent of
-		 * 'hw_resend_irq'. Just like a real IO-APIC we 'lose
-		 * the interrupt edge' if the channel is masked.
-		 */
-		if (evtchn_pending &&
-		    !sync_test_and_set_bit(port / BITS_PER_EVTCHN_WORD,
-					   BM(&vcpu_info->evtchn_pending_sel)))
-			vcpu_info->evtchn_upcall_pending = 1;
-	}
+	} else
+		eops->unmask(port);
 
 	put_cpu();
 }
@@ -938,7 +957,7 @@ static int find_virq(unsigned int virq, unsigned int cpu)
 	int port, rc = -ENOENT;
 
 	memset(&status, 0, sizeof(status));
-	for (port = 0; port <= NR_EVENT_CHANNELS; port++) {
+	for (port = 0; port <= xen_nr_event_channels; port++) {
 		status.dom = DOMID_SELF;
 		status.port = port;
 		rc = HYPERVISOR_event_channel_op(EVTCHNOP_status, &status);
@@ -1163,7 +1182,7 @@ int evtchn_get(unsigned int evtchn)
 	struct irq_info *info;
 	int err = -ENOENT;
 
-	if (evtchn >= NR_EVENT_CHANNELS)
+	if (evtchn >= xen_nr_event_channels)
 		return -EINVAL;
 
 	mutex_lock(&irq_mapping_update_lock);
@@ -1208,13 +1227,12 @@ void xen_send_IPI_one(unsigned int cpu, enum ipi_vector vector)
 
 irqreturn_t xen_debug_interrupt(int irq, void *dev_id)
 {
-	struct shared_info *sh = HYPERVISOR_shared_info;
-	int cpu = smp_processor_id();
-	xen_ulong_t *cpu_evtchn = per_cpu(cpu_evtchn_mask, cpu);
-	int i;
-	unsigned long flags;
+	irqreturn_t rc;
 	static DEFINE_SPINLOCK(debug_lock);
+	unsigned long flags;
+	int cpu = smp_processor_id();
 	struct vcpu_info *v;
+	int i;
 
 	spin_lock_irqsave(&debug_lock, flags);
 
@@ -1228,65 +1246,80 @@ irqreturn_t xen_debug_interrupt(int irq, void *dev_id)
 			: v->evtchn_upcall_mask;
 		printk(KERN_DEBUG "%d: masked=%d pending=%d event_sel %0*"PRI_xen_ulong"\n  ", i,
 		       pending, v->evtchn_upcall_pending,
-		       (int)(sizeof(v->evtchn_pending_sel)*2),
-		       v->evtchn_pending_sel);
+		       (int)(sizeof(*per_cpu(evtchn_sel, cpu)[0])*2),
+		       *per_cpu(evtchn_sel, cpu)[0]);
 	}
+
+	rc = eops->debug_interrupt(irq, dev_id);
+
+	spin_unlock_irqrestore(&debug_lock, flags);
+	return rc;
+}
+
+static irqreturn_t xen_debug_interrupt_l2(int irq, void *dev_id)
+{
+	int cpu = smp_processor_id();
+	xen_ulong_t *cpu_evtchn = per_cpu(cpu_evtchn_mask, cpu);
+	int i;
+	unsigned long nr_elems = NR_EVENT_CHANNELS_L2 / BITS_PER_EVTCHN_WORD;
+	struct vcpu_info *v;
+
 	v = per_cpu(xen_vcpu, cpu);
 
 	printk(KERN_DEBUG "\npending:\n   ");
-	for (i = ARRAY_SIZE(sh->evtchn_pending)-1; i >= 0; i--)
+	for (i = nr_elems; i >= 0; i--)
 		printk(KERN_DEBUG "%0*"PRI_xen_ulong"%s",
-		       (int)sizeof(sh->evtchn_pending[0])*2,
-		       sh->evtchn_pending[i],
+		       (int)sizeof(evtchn_pending[0])*2,
+		       evtchn_pending[i],
 		       i % 8 == 0 ? "\n   " : " ");
 	printk(KERN_DEBUG "\nglobal mask:\n   ");
-	for (i = ARRAY_SIZE(sh->evtchn_mask)-1; i >= 0; i--)
+	for (i = nr_elems; i >= 0; i--)
 		printk(KERN_DEBUG "%0*"PRI_xen_ulong"%s",
-		       (int)(sizeof(sh->evtchn_mask[0])*2),
-		       sh->evtchn_mask[i],
+		       (int)(sizeof(evtchn_mask[0])*2),
+		       evtchn_mask[i],
 		       i % 8 == 0 ? "\n   " : " ");
 
 	printk(KERN_DEBUG "\nglobally unmasked:\n   ");
-	for (i = ARRAY_SIZE(sh->evtchn_mask)-1; i >= 0; i--)
+	for (i = nr_elems; i >= 0; i--)
 		printk("%0*"PRI_xen_ulong"%s",
-		       (int)(sizeof(sh->evtchn_mask[0])*2),
-		       sh->evtchn_pending[i] & ~sh->evtchn_mask[i],
+		       (int)(sizeof(evtchn_mask[0])*2),
+		       evtchn_pending[i] & ~evtchn_mask[i],
 		       i % 8 == 0 ? "\n   " : " ");
 
 	printk("\nlocal cpu%d mask:\n   ", cpu);
-	for (i = (NR_EVENT_CHANNELS/BITS_PER_EVTCHN_WORD)-1; i >= 0; i--)
-		printk(KERN_DEBUG "%0*"PRI_xen_ulong"%s", (int)(sizeof(cpu_evtchn[0])*2),
+	for (i = (NR_EVENT_CHANNELS_L2/BITS_PER_EVTCHN_WORD)-1; i >= 0; i--)
+		printk(KERN_DEBUG "%0*"PRI_xen_ulong"%s",
+		       (int)(sizeof(cpu_evtchn[0])*2),
 		       cpu_evtchn[i],
 		       i % 8 == 0 ? "\n   " : " ");
 
 	printk(KERN_DEBUG "\nlocally unmasked:\n   ");
-	for (i = ARRAY_SIZE(sh->evtchn_mask)-1; i >= 0; i--) {
-		xen_ulong_t pending = sh->evtchn_pending[i]
-			& ~sh->evtchn_mask[i]
+	for (i = nr_elems-1; i >= 0; i--) {
+		xen_ulong_t pending = evtchn_pending[i]
+			& ~evtchn_mask[i]
 			& cpu_evtchn[i];
 		printk(KERN_DEBUG "%0*"PRI_xen_ulong"%s",
-		       (int)(sizeof(sh->evtchn_mask[0])*2),
+		       (int)(sizeof(evtchn_mask[0])*2),
 		       pending, i % 8 == 0 ? "\n   " : " ");
 	}
 
 	printk(KERN_DEBUG "\npending list:\n");
-	for (i = 0; i < NR_EVENT_CHANNELS; i++) {
-		if (sync_test_bit(i, BM(sh->evtchn_pending))) {
+	for (i = 0; i < NR_EVENT_CHANNELS_L2; i++) {
+		if (sync_test_bit(i, BM(evtchn_pending))) {
 			int word_idx = i / BITS_PER_EVTCHN_WORD;
 			printk(KERN_DEBUG "  %d: event %d -> irq %d%s%s%s\n",
 			       cpu_from_evtchn(i), i,
 			       evtchn_to_irq[i],
-			       sync_test_bit(word_idx, BM(&v->evtchn_pending_sel))
+			       sync_test_bit(word_idx,
+				             BM(per_cpu(evtchn_sel, cpu)[0]))
 					     ? "" : " l1-clear",
-			       !sync_test_bit(i, BM(sh->evtchn_mask))
+			       !sync_test_bit(i, BM(evtchn_mask))
 					     ? "" : " globally-masked",
 			       sync_test_bit(i, BM(cpu_evtchn))
 					     ? "" : " locally-masked");
 		}
 	}
 
-	spin_unlock_irqrestore(&debug_lock, flags);
-
 	return IRQ_HANDLED;
 }
 
@@ -1308,13 +1341,12 @@ static DEFINE_PER_CPU(unsigned int, current_bit_idx);
  * a bitset of words which contain pending event bits.  The second
  * level is a bitset of pending events themselves.
  */
-static void __xen_evtchn_do_upcall(void)
+static void __xen_evtchn_do_upcall_l2(void)
 {
 	int start_word_idx, start_bit_idx;
 	int word_idx, bit_idx;
 	int i;
 	int cpu = get_cpu();
-	struct shared_info *s = HYPERVISOR_shared_info;
 	struct vcpu_info *vcpu_info = __this_cpu_read(xen_vcpu);
 	unsigned count;
 
@@ -1331,7 +1363,7 @@ static void __xen_evtchn_do_upcall(void)
 		 * selector flag. xchg_xen_ulong must contain an
 		 * appropriate barrier.
 		 */
-		pending_words = xchg_xen_ulong(&vcpu_info->evtchn_pending_sel, 0);
+		pending_words = xchg_xen_ulong(per_cpu(evtchn_sel, cpu)[0], 0);
 
 		start_word_idx = __this_cpu_read(current_word_idx);
 		start_bit_idx = __this_cpu_read(current_bit_idx);
@@ -1354,7 +1386,7 @@ static void __xen_evtchn_do_upcall(void)
 			}
 			word_idx = EVTCHN_FIRST_BIT(words);
 
-			pending_bits = active_evtchns(cpu, s, word_idx);
+			pending_bits = active_evtchns(cpu, word_idx);
 			bit_idx = 0; /* usually scan entire word from start */
 			if (word_idx == start_word_idx) {
 				/* We scan the starting word in two parts */
@@ -1425,7 +1457,7 @@ void xen_evtchn_do_upcall(struct pt_regs *regs)
 	exit_idle();
 #endif
 
-	__xen_evtchn_do_upcall();
+	eops->do_upcall();
 
 	irq_exit();
 	set_irq_regs(old_regs);
@@ -1433,7 +1465,7 @@ void xen_evtchn_do_upcall(struct pt_regs *regs)
 
 void xen_hvm_evtchn_do_upcall(void)
 {
-	__xen_evtchn_do_upcall();
+	eops->do_upcall();
 }
 EXPORT_SYMBOL_GPL(xen_hvm_evtchn_do_upcall);
 
@@ -1729,14 +1761,14 @@ void xen_irq_resume(void)
 	init_evtchn_cpu_bindings();
 
 	/* New event-channel space is not 'live' yet. */
-	for (evtchn = 0; evtchn < NR_EVENT_CHANNELS; evtchn++)
+	for (evtchn = 0; evtchn < xen_nr_event_channels; evtchn++)
 		mask_evtchn(evtchn);
 
 	/* No IRQ <-> event-channel mappings. */
 	list_for_each_entry(info, &xen_irq_list_head, list)
 		info->evtchn = 0; /* zap event-channel binding */
 
-	for (evtchn = 0; evtchn < NR_EVENT_CHANNELS; evtchn++)
+	for (evtchn = 0; evtchn < xen_nr_event_channels; evtchn++)
 		evtchn_to_irq[evtchn] = -1;
 
 	for_each_possible_cpu(cpu) {
@@ -1829,20 +1861,39 @@ void xen_callback_vector(void)
 void xen_callback_vector(void) {}
 #endif
 
+const struct evtchn_ops evtchn_l2_ops = {
+	.unmask = __unmask_local_port_l2,
+	.debug_interrupt = xen_debug_interrupt_l2,
+	.do_upcall = __xen_evtchn_do_upcall_l2
+};
+
 void __init xen_init_IRQ(void)
 {
 	int i;
+	int cpu;
+	struct shared_info *s = HYPERVISOR_shared_info;
+
+	evtchn_pending = s->evtchn_pending;
+	evtchn_mask = s->evtchn_mask;
+	for_each_possible_cpu(cpu) {
+		struct vcpu_info *vcpu_info = per_cpu(xen_vcpu, cpu);
+		per_cpu(evtchn_sel, cpu)[0] = &vcpu_info->evtchn_pending_sel;
+	}
+
+	xen_evtchn_extended = EVTCHN_EXTENDED_NONE;
+	xen_nr_event_channels = NR_EVENT_CHANNELS_L2;
+	eops = &evtchn_l2_ops;
 
-	evtchn_to_irq = kcalloc(NR_EVENT_CHANNELS, sizeof(*evtchn_to_irq),
+	evtchn_to_irq = kcalloc(xen_nr_event_channels, sizeof(*evtchn_to_irq),
 				    GFP_KERNEL);
 	BUG_ON(!evtchn_to_irq);
-	for (i = 0; i < NR_EVENT_CHANNELS; i++)
+	for (i = 0; i < xen_nr_event_channels; i++)
 		evtchn_to_irq[i] = -1;
 
 	init_evtchn_cpu_bindings();
 
 	/* No event channels are 'live' right now. */
-	for (i = 0; i < NR_EVENT_CHANNELS; i++)
+	for (i = 0; i < xen_nr_event_channels; i++)
 		mask_evtchn(i);
 
 	pirq_needs_eoi = pirq_needs_eoi_flag;
diff --git a/drivers/xen/evtchn.c b/drivers/xen/evtchn.c
index b2db77e..ac7a96e 100644
--- a/drivers/xen/evtchn.c
+++ b/drivers/xen/evtchn.c
@@ -232,7 +232,7 @@ static ssize_t evtchn_write(struct file *file, const char __user *buf,
 	for (i = 0; i < (count/sizeof(evtchn_port_t)); i++) {
 		unsigned port = kbuf[i];
 
-		if (port < NR_EVENT_CHANNELS &&
+		if (port < xen_nr_event_channels &&
 		    get_port_user(port) == u &&
 		    !get_port_enabled(port)) {
 			set_port_enabled(port, true);
@@ -374,7 +374,7 @@ static long evtchn_ioctl(struct file *file,
 			break;
 
 		rc = -EINVAL;
-		if (unbind.port >= NR_EVENT_CHANNELS)
+		if (unbind.port >= xen_nr_event_channels)
 			break;
 
 		spin_lock_irq(&port_user_lock);
@@ -402,7 +402,7 @@ static long evtchn_ioctl(struct file *file,
 		if (copy_from_user(&notify, uarg, sizeof(notify)))
 			break;
 
-		if (notify.port >= NR_EVENT_CHANNELS) {
+		if (notify.port >= xen_nr_event_channels) {
 			rc = -EINVAL;
 		} else if (get_port_user(notify.port) != u) {
 			rc = -ENOTCONN;
@@ -492,7 +492,7 @@ static int evtchn_release(struct inode *inode, struct file *filp)
 
 	free_page((unsigned long)u->ring);
 
-	for (i = 0; i < NR_EVENT_CHANNELS; i++) {
+	for (i = 0; i < xen_nr_event_channels; i++) {
 		if (get_port_user(i) != u)
 			continue;
 
@@ -501,7 +501,7 @@ static int evtchn_release(struct inode *inode, struct file *filp)
 
 	spin_unlock_irq(&port_user_lock);
 
-	for (i = 0; i < NR_EVENT_CHANNELS; i++) {
+	for (i = 0; i < xen_nr_event_channels; i++) {
 		if (get_port_user(i) != u)
 			continue;
 
@@ -538,7 +538,8 @@ static int __init evtchn_init(void)
 	if (!xen_domain())
 		return -ENODEV;
 
-	port_user = kcalloc(NR_EVENT_CHANNELS, sizeof(*port_user), GFP_KERNEL);
+	port_user = kcalloc(xen_nr_event_channels,
+			    sizeof(*port_user), GFP_KERNEL);
 	if (port_user == NULL)
 		return -ENOMEM;
 
diff --git a/include/xen/events.h b/include/xen/events.h
index c6bfe01..24cf421 100644
--- a/include/xen/events.h
+++ b/include/xen/events.h
@@ -111,4 +111,7 @@ int xen_test_irq_shared(int irq);
 
 /* initialize Xen IRQ subsystem */
 void xen_init_IRQ(void);
+extern unsigned int xen_nr_event_channels;
+extern uint64_t xen_evtchn_extended;
+
 #endif	/* _XEN_EVENTS_H */
-- 
1.7.10.4