[PATCH 0/3] input: evdev: Dynamic buffers (rev2)

[PATCH 0/3] input: evdev: Dynamic buffers (rev2)

[Henrik Rydberg]

Ok, second version of the evdev buffer patches.

In the first patch, the locking has been completely reworked. The
buffer locking is now similar to seqlock, except the readers only
block while there is an update affecting the current read. Should be
fairly optimal, but we can always revert to seqlocks in case the
scatter of smp instructions feels inadequate.

The second patch only has trivial changes, and the third patch is
unchanged, but included for completeness.

Cheers,
Henrik

---

Henrik Rydberg (3):
  input: evdev: use multi-reader buffer to save space (rev2)
  input: evdev: convert to dynamic event buffer (rev2)
  input: use driver hint to compute the evdev buffer size

 drivers/input/evdev.c |   90 +++++++++++++++++++++++++++++++++---------------
 include/linux/input.h |    7 ++++
 2 files changed, 69 insertions(+), 28 deletions(-)

[PATCH 1/3] input: evdev: use multi-reader buffer to save space (rev2)

[Henrik Rydberg]

Preparing for larger buffer needs, convert the current per-client
circular buffer to a single buffer with multiple clients. Use a
lock-less mechanism where clients wait during buffer collision only.

Signed-off-by: Henrik Rydberg <rydberg@euromail.se>
---
 drivers/input/evdev.c |   74 +++++++++++++++++++++++++++++-------------------
 1 files changed, 45 insertions(+), 29 deletions(-)

diff --git a/drivers/input/evdev.c b/drivers/input/evdev.c
index 2ee6c7a..9cbed21 100644
--- a/drivers/input/evdev.c
+++ b/drivers/input/evdev.c
@@ -33,13 +33,14 @@ struct evdev {
 	spinlock_t client_lock; /* protects client_list */
 	struct mutex mutex;
 	struct device dev;
+	int head;
+	int next_head;
+	struct input_event buffer[EVDEV_BUFFER_SIZE];
 };
 
 struct evdev_client {
-	struct input_event buffer[EVDEV_BUFFER_SIZE];
 	int head;
 	int tail;
-	spinlock_t buffer_lock; /* protects access to buffer, head and tail */
 	struct fasync_struct *fasync;
 	struct evdev *evdev;
 	struct list_head node;
@@ -48,18 +49,11 @@ struct evdev_client {
 static struct evdev *evdev_table[EVDEV_MINORS];
 static DEFINE_MUTEX(evdev_table_mutex);
 
-static void evdev_pass_event(struct evdev_client *client,
-			     struct input_event *event)
+static inline void evdev_sync_event(struct evdev_client *client,
+				    int head, int type)
 {
-	/*
-	 * Interrupts are disabled, just acquire the lock
-	 */
-	spin_lock(&client->buffer_lock);
-	client->buffer[client->head++] = *event;
-	client->head &= EVDEV_BUFFER_SIZE - 1;
-	spin_unlock(&client->buffer_lock);
-
-	if (event->type == EV_SYN)
+	client->head = head;
+	if (type == EV_SYN)
 		kill_fasync(&client->fasync, SIGIO, POLL_IN);
 }
 
@@ -78,14 +72,22 @@ static void evdev_event(struct input_handle *handle,
 	event.code = code;
 	event.value = value;
 
+	/* lock-less write, interrupts disabled locally */
+	evdev->next_head = (evdev->head + 1) & (EVDEV_BUFFER_SIZE - 1);
+	smp_wmb();
+	evdev->buffer[evdev->head] = event;
+	smp_wmb();
+	evdev->head = evdev->next_head;
+	smp_wmb();
+
 	rcu_read_lock();
 
 	client = rcu_dereference(evdev->grab);
 	if (client)
-		evdev_pass_event(client, &event);
+		evdev_sync_event(client, evdev->head, type);
 	else
 		list_for_each_entry_rcu(client, &evdev->client_list, node)
-			evdev_pass_event(client, &event);
+			evdev_sync_event(client, evdev->head, type);
 
 	rcu_read_unlock();
 
@@ -269,7 +271,6 @@ static int evdev_open(struct inode *inode, struct file *file)
 		goto err_put_evdev;
 	}
 
-	spin_lock_init(&client->buffer_lock);
 	client->evdev = evdev;
 	evdev_attach_client(evdev, client);
 
@@ -324,22 +325,37 @@ static ssize_t evdev_write(struct file *file, const char __user *buffer,
 	return retval;
 }
 
-static int evdev_fetch_next_event(struct evdev_client *client,
+static inline bool write_overlaps_read(int head, int next_head, int tail)
+{
+	if (next_head < head)
+		return tail >= head || tail < next_head;
+	else
+		return tail >= head && tail < next_head;
+}
+
+static int evdev_fetch_next_event(struct evdev *evdev,
+				  struct evdev_client *client,
 				  struct input_event *event)
 {
-	int have_event;
+	int head, next_head;
 
-	spin_lock_irq(&client->buffer_lock);
+	if (client->head == client->tail)
+		return 0;
 
-	have_event = client->head != client->tail;
-	if (have_event) {
-		*event = client->buffer[client->tail++];
-		client->tail &= EVDEV_BUFFER_SIZE - 1;
+ repeat:
+	head = evdev->head;
+	smp_rmb();
+	*event = evdev->buffer[client->tail];
+	smp_rmb();
+	next_head = evdev->next_head;
+	smp_rmb();
+	if (unlikely(write_overlaps_read(head, next_head, client->tail))) {
+		cpu_relax();
+		goto repeat;
 	}
 
-	spin_unlock_irq(&client->buffer_lock);
-
-	return have_event;
+	client->tail = (client->tail + 1) & (EVDEV_BUFFER_SIZE - 1);
+	return 1;
 }
 
 static ssize_t evdev_read(struct file *file, char __user *buffer,
@@ -366,7 +382,7 @@ static ssize_t evdev_read(struct file *file, char __user *buffer,
 		return -ENODEV;
 
 	while (retval + input_event_size() <= count &&
-	       evdev_fetch_next_event(client, &event)) {
+	       evdev_fetch_next_event(evdev, client, &event)) {
 
 		if (input_event_to_user(buffer + retval, &event))
 			return -EFAULT;
-- 
1.6.3.3

[PATCH 2/3] input: evdev: convert to dynamic event buffer (rev2)

[Henrik Rydberg]

Allocate the event buffer dynamically, and prepare to compute the buffer
size in a separate function. This patch defines the size computation to
be identical to the current code, and does not contain any logical changes.

Signed-off-by: Henrik Rydberg <rydberg@euromail.se>
---
 drivers/input/evdev.c |   23 +++++++++++++++++++----
 1 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/drivers/input/evdev.c b/drivers/input/evdev.c
index 9cbed21..327f821 100644
--- a/drivers/input/evdev.c
+++ b/drivers/input/evdev.c
@@ -10,7 +10,7 @@
 
 #define EVDEV_MINOR_BASE	64
 #define EVDEV_MINORS		32
-#define EVDEV_BUFFER_SIZE	64
+#define EVDEV_MIN_BUFFER_SIZE	64
 
 #include <linux/poll.h>
 #include <linux/sched.h>
@@ -35,7 +35,8 @@ struct evdev {
 	struct device dev;
 	int head;
 	int next_head;
-	struct input_event buffer[EVDEV_BUFFER_SIZE];
+	int bufsize;
+	struct input_event *buffer;
 };
 
 struct evdev_client {
@@ -73,7 +74,7 @@ static void evdev_event(struct input_handle *handle,
 	event.value = value;
 
 	/* lock-less write, interrupts disabled locally */
-	evdev->next_head = (evdev->head + 1) & (EVDEV_BUFFER_SIZE - 1);
+	evdev->next_head = (evdev->head + 1) & (evdev->bufsize - 1);
 	smp_wmb();
 	evdev->buffer[evdev->head] = event;
 	smp_wmb();
@@ -125,6 +126,7 @@ static void evdev_free(struct device *dev)
 	struct evdev *evdev = container_of(dev, struct evdev, dev);
 
 	input_put_device(evdev->handle.dev);
+	kfree(evdev->buffer);
 	kfree(evdev);
 }
 
@@ -354,7 +356,7 @@ static int evdev_fetch_next_event(struct evdev *evdev,
 		goto repeat;
 	}
 
-	client->tail = (client->tail + 1) & (EVDEV_BUFFER_SIZE - 1);
+	client->tail = (client->tail + 1) & (evdev->bufsize - 1);
 	return 1;
 }
 
@@ -803,6 +805,11 @@ static void evdev_cleanup(struct evdev *evdev)
 	}
 }
 
+static int evdev_compute_buffer_size(struct input_dev *dev)
+{
+	return EVDEV_MIN_BUFFER_SIZE;
+}
+
 /*
  * Create new evdev device. Note that input core serializes calls
  * to connect and disconnect so we don't need to lock evdev_table here.
@@ -847,6 +854,14 @@ static int evdev_connect(struct input_handler *handler, struct input_dev *dev,
 	evdev->dev.release = evdev_free;
 	device_initialize(&evdev->dev);
 
+	evdev->bufsize = evdev_compute_buffer_size(dev);
+	evdev->buffer = kzalloc(evdev->bufsize * sizeof(struct input_event),
+				GFP_KERNEL);
+	if (!evdev->buffer) {
+		error = -ENOMEM;
+		goto err_free_evdev;
+	}
+
 	error = input_register_handle(&evdev->handle);
 	if (error)
 		goto err_free_evdev;
-- 
1.6.3.3

[PATCH 3/3] input: use driver hint to compute the evdev buffer size

[Henrik Rydberg]

Some devices, in particular MT devices, produce a lot of data.
This leads to a high frequency of lost packets in evdev, which
by default uses a fairly small event buffer. Let the drivers hint
the average number of events per packet for the device by calling
the input_set_events_per_packet(), and use that information when
computing the evdev buffer size.

Signed-off-by: Henrik Rydberg <rydberg@euromail.se>
---
 drivers/input/evdev.c |    5 ++++-
 include/linux/input.h |    7 +++++++
 2 files changed, 11 insertions(+), 1 deletions(-)

diff --git a/drivers/input/evdev.c b/drivers/input/evdev.c
index 327f821..fdddb43 100644
--- a/drivers/input/evdev.c
+++ b/drivers/input/evdev.c
@@ -11,6 +11,7 @@
 #define EVDEV_MINOR_BASE	64
 #define EVDEV_MINORS		32
 #define EVDEV_MIN_BUFFER_SIZE	64
+#define EVDEV_BUF_PACKETS	8
 
 #include <linux/poll.h>
 #include <linux/sched.h>
@@ -807,7 +808,9 @@ static void evdev_cleanup(struct evdev *evdev)
 
 static int evdev_compute_buffer_size(struct input_dev *dev)
 {
-	return EVDEV_MIN_BUFFER_SIZE;
+	int nev = dev->hint_events_per_packet * EVDEV_BUF_PACKETS;
+	nev = max(nev, EVDEV_MIN_BUFFER_SIZE);
+	return roundup_pow_of_two(nev);
 }
 
 /*
diff --git a/include/linux/input.h b/include/linux/input.h
index bd00786..35b015d 100644
--- a/include/linux/input.h
+++ b/include/linux/input.h
@@ -1162,6 +1162,8 @@ struct input_dev {
 	unsigned long ffbit[BITS_TO_LONGS(FF_CNT)];
 	unsigned long swbit[BITS_TO_LONGS(SW_CNT)];
 
+	unsigned int hint_events_per_packet;
+
 	unsigned int keycodemax;
 	unsigned int keycodesize;
 	void *keycode;
@@ -1439,6 +1441,11 @@ static inline void input_mt_slot(struct input_dev *dev, int slot)
 
 void input_set_capability(struct input_dev *dev, unsigned int type, unsigned int code);
 
+static inline void input_set_events_per_packet(struct input_dev *dev, int nev)
+{
+	dev->hint_events_per_packet = nev;
+}
+
 static inline void input_set_abs_params(struct input_dev *dev, int axis, int min, int max, int fuzz, int flat)
 {
 	dev->absmin[axis] = min;
-- 
1.6.3.3