[uml-devel] [PATCH] I/O-Request Token Bucket Limiter

["Christopher S. Aker" <caker@theshore.net>]

[-- Attachment #1: Type: text/plain, Size: 2035 bytes --]

Patch is also available here:
http://www.theshore.net/~caker/patches/

Apply to a tree already patched with 2.4-um or 2.6-um.

-Chris

---token-limiter-v1.README---

This patch implements a simple token bucket filter/limiter on requests
processed through the async io_thread.  It will work with or without COW,
but not on ubd devices in sync mode.

* Two new command line arguments for setting at boot-time:

token_max=60000
    This sets the maximum tokens in the token-bucket I/O request limiter.
    Each io_thread request subtracts one token from the bucket.  The bucket
    is supplied with <token_refill> tokens every second.  When the bucket
    becomes empty, I/O requests are throttled to the <token_refill> rate.

token_refill=2000
    This sets the bucket's refill rate, adding <token_refill> tokens to the
    bucket every second. This becomes the I/O request rate when the bucket
    becomes empty.

* Three new mconsole commands to get status information and change the two 
variables above:

(mconsole) help
    io_status - Return current I/O status and settings
    io_token_max <num> - sets the bucket size
    io_token_refill <num> - number of tokens to add each second

* Example io_status output while:

idle:
io_count=1845202 io_rate=0 io_tokens=50000 token_refill=1024 \
token_max=50000

bursting:
io_count=1861594 io_rate=6806 io_tokens=35754 token_refill=1024 \
token_max=50000

throttled:
io_count=1994811 io_rate=1087 io_tokens=-153 token_refill=1024 \
token_max=50000 
io_count=2052289 io_rate=1057 io_tokens=-286 token_refill=1024 \
token_max=50000
io_count=2067794 io_rate=1015 io_tokens=-132 token_refill=1024 \
token_max=50000

Our sample shows io_tokens being negative, but - while waiting for the bucket 
to be re-supplied, requests will still dribble through no faster than 
token_refill req/second. Or, perhaps this is due to inaccuracies of nanosleep
or a bug in my logic.

* TODO
Stop using an alarm in the io_thread, and instead use gettimeofday to 
calculate the token credits due and io_rate.



[-- Attachment #2: token-limiter-v1.patch --]
[-- Type: application/octet-stream, Size: 7985 bytes --]

diff -aur -X diff-exclude linode23/arch/um/drivers/mconsole_kern.c linode23-limiter/arch/um/drivers/mconsole_kern.c
--- linode23/arch/um/drivers/mconsole_kern.c	2004-02-18 20:04:28.000000000 -0500
+++ linode23-limiter/arch/um/drivers/mconsole_kern.c	2004-03-28 00:04:44.546996347 -0500
@@ -30,6 +30,7 @@
 #include "os.h"
 #include "umid.h"
 #include "irq_kern.h"
+#include "ubd_user.h"
 
 static int do_unlink_socket(struct notifier_block *notifier, 
 			    unsigned long what, void *data)
@@ -219,6 +220,9 @@
     go - continue the UML after a 'stop' \n\
     log <string> - make UML enter <string> into the kernel log\n\
     proc <file> - returns the contents of the UML's /proc/<file>\n\
+    io_status - Return current I/O status and settings\n\
+    io_token_max <num> - sets the bucket size\n\
+    io_token_refill <num> - number of tokens to add each second\n\
 "
 
 void mconsole_help(struct mc_request *req)
@@ -548,6 +552,51 @@
 
 EXPORT_SYMBOL(mconsole_notify_socket);
 
+void mconsole_io_status(struct mc_request *req)
+{
+	char str[256];
+
+	sprintf(str, "io_count=%d io_rate=%d io_tokens=%d token_refill=%d token_max=%d",
+		get_io_count(),
+		get_io_rate(),
+		get_io_tokens(),
+		get_token_refill(),
+		get_token_max()
+	);
+
+	mconsole_reply(req, str, 0, 0);
+}
+
+void mconsole_io_token_refill(struct mc_request *req)
+{
+	char *end;
+	int n;	
+
+	char *ptr = req->request.data;
+	ptr += strlen("io_token_refill ");
+	n = simple_strtoul(ptr, &end, 0);
+
+	if (n>0) 
+		set_token_refill(n);
+
+	mconsole_io_status(req);
+}
+
+void mconsole_io_token_max(struct mc_request *req)
+{
+	char *end;
+	int n;
+
+	char *ptr = req->request.data;
+	ptr += strlen("io_token_max ");
+	n = simple_strtoul(ptr, &end, 0);
+
+	if (n>0) 
+		set_token_max(n);
+
+	mconsole_io_status(req);
+}
+
 /*
  * Overrides for Emacs so that we follow Linus's tabbing style.
  * Emacs will notice this stuff at the end of the file and automatically
diff -aur -X diff-exclude linode23/arch/um/drivers/mconsole_user.c linode23-limiter/arch/um/drivers/mconsole_user.c
--- linode23/arch/um/drivers/mconsole_user.c	2004-02-18 20:04:28.000000000 -0500
+++ linode23-limiter/arch/um/drivers/mconsole_user.c	2004-03-26 20:32:12.129731275 -0500
@@ -30,6 +30,9 @@
 	{ "go", mconsole_go, MCONSOLE_INTR },
 	{ "log", mconsole_log, MCONSOLE_INTR },
 	{ "proc", mconsole_proc, MCONSOLE_PROC },
+	{ "io_status", mconsole_io_status, MCONSOLE_INTR },
+	{ "io_token_refill", mconsole_io_token_refill, MCONSOLE_INTR },
+	{ "io_token_max", mconsole_io_token_max, MCONSOLE_INTR },
 };
 
 /* Initialized in mconsole_init, which is an initcall */
diff -aur -X diff-exclude linode23/arch/um/drivers/ubd_user.c linode23-limiter/arch/um/drivers/ubd_user.c
--- linode23/arch/um/drivers/ubd_user.c	2004-02-18 20:04:28.000000000 -0500
+++ linode23-limiter/arch/um/drivers/ubd_user.c	2004-03-28 01:03:07.265680876 -0500
@@ -26,6 +26,10 @@
 #include <endian.h>
 #include <byteswap.h>
 
+#include <stdlib.h>
+#include <time.h>
+#include "init.h"
+
 static int same_backing_files(char *from_cmdline, char *from_cow, char *cow)
 {
 	struct uml_stat buf1, buf2;
@@ -307,12 +311,23 @@
 /* Only changed by the io thread */
 int io_count = 0;
 
+/* Are set by the kernel and read from the io_thread or vice-versa */
+int token_refill = 2000;// add this many tokens to the bucket every second
+int token_delay = 1000000000 / 2000;
+int token_max = 60000;  // max tokens in bucket
+int io_tokens = 60000;  // token bucket
+int io_count_last = 0;  // snapshot of io_count last time in io_alarm()
+int io_rate = 0;        // calculated io ops per second
+
 int io_thread(void *arg)
 {
 	struct io_thread_req req;
 	int n;
 
 	signal(SIGWINCH, SIG_IGN);
+	signal(SIGALRM, io_alarm);
+	io_alarm(14);
+
 	while(1){
 		n = os_read_file(kernel_fd, &req, sizeof(req));
 		if(n != sizeof(req)){
@@ -327,6 +342,7 @@
 		}
 		io_count++;
 		do_io(&req);
+		io_delay();
 		n = os_write_file(kernel_fd, &req, sizeof(req));
 		if(n != sizeof(req))
 			printk("io_thread - write failed, fd = %d, err = %d\n",
@@ -365,6 +381,103 @@
 	return(err);
 }
 
+void io_alarm(int sig)
+{
+	io_rate = io_count - io_count_last;
+	io_count_last = io_count;
+
+	io_tokens += token_refill;
+	if (io_tokens > token_max){
+		io_tokens = token_max;
+	}
+
+	/* printk("io_alarm - current speed: %d, io_tokens : %d \n", 
+		io_rate, io_tokens); */
+
+	alarm(1);
+}
+
+void io_delay()
+{
+	io_tokens--;
+	if (io_tokens <= 0) {
+		struct timespec ts;
+		ts.tv_sec = 0;
+		ts.tv_nsec = token_delay;
+		nanosleep(&ts, NULL);
+	}
+}
+
+int set_token_refill(int tokens)
+{
+	token_refill = tokens;
+
+	token_delay = 1000;
+	if (token_refill)
+		token_delay = 1000000000 / token_refill;
+
+	return 0;
+}
+
+int get_token_refill()
+{
+	return token_refill;
+}
+
+int get_io_count()
+{
+	return io_count;
+}
+
+int set_token_max(int tokens)
+{
+	token_max = tokens;
+	return 0;
+}
+
+int get_token_max()
+{
+	return token_max;
+}
+
+int get_io_rate()
+{
+	return io_rate;
+}
+
+int get_io_tokens()
+{
+	return io_tokens;
+}
+
+static int cmdline_token_max(char *name, int *add)
+{
+	set_token_max(atoi(name));
+	return(0);
+}
+
+__uml_setup("token_max=", cmdline_token_max,
+"token_max=60000\n" \
+"    This sets the maximum tokens in the token-bucket I/O request limiter.\n"   \
+"    Each io_thread request subtracts one token from the bucket.  The bucket\n" \
+"    is supplied with <token_refill> tokens every second.  When the bucket\n"   \
+"    becomes empty, I/O requests are throttled to the <token_refill> rate.\n\n"
+);
+
+
+static int cmdline_token_refill(char *name, int *add)
+{
+	set_token_refill(atoi(name));
+	return(0);
+}
+
+__uml_setup("token_refill=", cmdline_token_refill,
+"token_refill=2000\n" \
+"    This sets the bucket's refill rate, adding <token_refill> tokens to the\n" \
+"    bucket every second. This becomes the I/O request rate when the bucket\n"  \
+"    becomes empty.\n\n"
+);
+
 /*
  * Overrides for Emacs so that we follow Linus's tabbing style.
  * Emacs will notice this stuff at the end of the file and automatically
diff -aur -X diff-exclude linode23/arch/um/include/mconsole.h linode23-limiter/arch/um/include/mconsole.h
--- linode23/arch/um/include/mconsole.h	2004-02-18 20:04:28.000000000 -0500
+++ linode23-limiter/arch/um/include/mconsole.h	2004-03-26 20:31:32.333430818 -0500
@@ -81,6 +81,9 @@
 extern void mconsole_go(struct mc_request *req);
 extern void mconsole_log(struct mc_request *req);
 extern void mconsole_proc(struct mc_request *req);
+extern void mconsole_io_status(struct mc_request *req);
+extern void mconsole_io_token_refill(struct mc_request *req);
+extern void mconsole_io_token_max(struct mc_request *req);
 
 extern int mconsole_get_request(int fd, struct mc_request *req);
 extern int mconsole_notify(char *sock_name, int type, const void *data, 
diff -aur -X diff-exclude linode23/arch/um/include/ubd_user.h linode23-limiter/arch/um/include/ubd_user.h
--- linode23/arch/um/include/ubd_user.h	2004-02-18 20:04:28.000000000 -0500
+++ linode23-limiter/arch/um/include/ubd_user.h	2004-03-27 22:23:18.413800952 -0500
@@ -42,6 +42,18 @@
 extern int start_io_thread(unsigned long sp, int *fds_out);
 extern void do_io(struct io_thread_req *req);
 
+static int cmdline_token_max(char *name, int *add);
+static int cmdline_token_refill(char *name, int *add);
+extern void io_delay(void);
+extern void io_alarm(int sig);
+extern int set_token_refill(int tokens);
+extern int get_token_refill(void);
+extern int set_token_max(int tokens);
+extern int get_token_max(void);
+extern int get_io_count(void);
+extern int get_io_tokens(void);
+extern int get_io_rate(void);
+
 static inline int ubd_test_bit(__u64 bit, unsigned char *data)
 {
 	__u64 n;