[Cluster-devel] cluster/group/gfs_controld main.c plock.c

[teigland@sourceware.org <teigland@sourceware.org>]

CVSROOT:	/cvs/cluster
Module name:	cluster
Branch: 	RHEL5
Changes by:	teigland at sourceware.org	2006-11-14 20:33:33

Modified files:
	group/gfs_controld: main.c plock.c 

Log message:
	Add plock rate limit option -l <limit>.  Current default is no limit (0).
	If a limit is set, gfs_controld will send no more than <limit> plock
	operations (multicast messages) every second.
	
	Given a limit of 10, one file system where plocks are used, and a program
	that does a tight loop of fcntl lock/unlock operations, the max number of
	loop iterations in 1 second would be 5.  If eight nodes were all doing
	this there would be 80 total network multicasts every second from all
	nodes in the cluster.
	
	We also record the volume of plock messages accepted locally and received
	from the network in the debug log.  A log entry is written for every
	1000 locally accepted plock operations and for every 1000 operations
	received from the network.

Patches:
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/gfs_controld/main.c.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.18&r2=1.18.2.1
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/gfs_controld/plock.c.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.25&r2=1.25.2.1

--- cluster/group/gfs_controld/main.c	2006/10/20 19:32:52	1.18
+++ cluster/group/gfs_controld/main.c	2006/11/14 20:33:32	1.18.2.1
@@ -12,7 +12,7 @@
 
 #include "lock_dlm.h"
 
-#define OPTION_STRING			"DPhVwp"
+#define OPTION_STRING			"DPhVwpl:"
 #define LOCKFILE_NAME			"/var/run/gfs_controld.pid"
 
 struct client {
@@ -32,11 +32,13 @@
 static int groupd_fd;
 static int uevent_fd;
 static int plocks_fd;
+static int plocks_ci;
 
 extern struct list_head mounts;
 extern struct list_head withdrawn_mounts;
 int no_withdraw;
 int no_plock;
+uint32_t plock_rate_limit;
 
 
 int do_write(int fd, void *buf, size_t count)
@@ -158,6 +160,18 @@
 	client[ci].mg = NULL;
 }
 
+static void client_ignore(int ci, int fd)
+{
+	pollfd[ci].fd = -1;
+	pollfd[ci].events = 0;
+}
+
+static void client_back(int ci, int fd)
+{
+	pollfd[ci].fd = fd;
+	pollfd[ci].events = POLLIN;
+}
+
 int client_send(int ci, char *buf, int len)
 {
 	return do_write(client[ci].fd, buf, len);
@@ -401,7 +415,7 @@
 
 int loop(void)
 {
-	int rv, i, f;
+	int rv, i, f, error, poll_timeout = -1, ignore_plocks_fd = 0;
 
 	rv = listen_fd = setup_listen();
 	if (rv < 0)
@@ -431,12 +445,12 @@
 	rv = plocks_fd = setup_plocks();
 	if (rv < 0)
 		goto out;
-	client_add(plocks_fd);
+	plocks_ci = client_add(plocks_fd);
 
 	log_debug("setup done");
 
 	for (;;) {
-		rv = poll(pollfd, client_maxi + 1, -1);
+		rv = poll(pollfd, client_maxi + 1, poll_timeout);
 		if (rv < 0)
 			log_error("poll error %d errno %d", rv, errno);
 
@@ -463,9 +477,15 @@
 					process_cpg();
 				else if (pollfd[i].fd == uevent_fd)
 					process_uevent();
-				else if (pollfd[i].fd == plocks_fd)
-					process_plocks();
-				else
+				else if (pollfd[i].fd == plocks_fd) {
+					error = process_plocks();
+					if (error == -EBUSY) {
+						client_ignore(plocks_ci,
+							      plocks_fd);
+						ignore_plocks_fd = 1;
+						poll_timeout = 100;
+					}
+				} else
 					process_client(i);
 			}
 
@@ -482,6 +502,18 @@
 				}
 				client_dead(i);
 			}
+
+			/* check if our plock rate limit has expired so we
+			   can start taking more local plock requests again */
+
+			if (ignore_plocks_fd) {
+				error = process_plocks();
+				if (error != -EBUSY) {
+					client_back(plocks_ci, plocks_fd);
+					ignore_plocks_fd = 0;
+					poll_timeout = -1;
+				}
+			}
 		}
 	}
 	rv = 0;
@@ -560,6 +592,10 @@
 	printf("Options:\n");
 	printf("\n");
 	printf("  -D	       Enable debugging code and don't fork\n");
+	printf("  -P	       Enable plock debugging\n");
+	printf("  -p	       Disable plocks\n");
+	printf("  -l <limit>   Limit the rate of plock operations\n");
+	printf("  -w	       Disable withdraw\n");
 	printf("  -h	       Print this help, then exit\n");
 	printf("  -V	       Print program version information, then exit\n");
 }
@@ -586,6 +622,10 @@
 			plock_debug_opt = 1;
 			break;
 
+		case 'l':
+			plock_rate_limit = atoi(optarg);
+			break;
+
 		case 'p':
 			no_plock = 1;
 			break;
--- cluster/group/gfs_controld/plock.c	2006/11/03 15:33:46	1.25
+++ cluster/group/gfs_controld/plock.c	2006/11/14 20:33:32	1.25.2.1
@@ -18,6 +18,7 @@
 #include <sys/ioctl.h>
 #include <sys/stat.h>
 #include <sys/utsname.h>
+#include <sys/time.h>
 #include <netinet/in.h>
 #include <arpa/inet.h>
 #include <net/if.h>
@@ -49,6 +50,14 @@
 extern int message_flow_control_on;
 extern int no_plock;
 
+extern uint32_t plock_rate_limit;
+uint32_t plock_read_count;
+uint32_t plock_recv_count;
+uint32_t plock_rate_delays;
+struct timeval plock_read_time;
+struct timeval plock_recv_time;
+struct timeval plock_rate_last;
+
 static SaCkptHandleT ckpt_handle;
 static SaCkptCallbacksT callbacks = { 0, 0 };
 static SaVersionT version = { 'B', 1, 1 };
@@ -276,6 +285,13 @@
 	SaAisErrorT err;
 	int rv;
 
+	plock_read_count = 0;
+	plock_recv_count = 0;
+	plock_rate_delays = 0;
+	gettimeofday(&plock_read_time, NULL);
+	gettimeofday(&plock_recv_time, NULL);
+	gettimeofday(&plock_rate_last, NULL);
+
 	if (no_plock)
 		goto control;
 
@@ -300,6 +316,7 @@
 	struct mountgroup *mg;
 	struct gdlm_plock_info info;
 	struct gdlm_header *hd;
+	struct timeval now;
 	char *buf;
 	int len, rv;
 
@@ -307,6 +324,19 @@
 	if (message_flow_control_on)
 		return 0;
 
+	/* do we want to do something a little more accurate than tv_sec? */
+
+	/* limit plock rate within one second */
+	if (plock_rate_limit && plock_read_count &&
+	    !(plock_read_count % plock_rate_limit)) {
+		gettimeofday(&now, NULL);
+		if (now.tv_sec - plock_rate_last.tv_sec <= 0) {
+			plock_rate_delays++;
+			return -EBUSY;
+		}
+		plock_rate_last = now;
+	}
+
 	memset(&info, 0, sizeof(info));
 
 	rv = read(control_fd, &info, sizeof(info));
@@ -331,6 +361,18 @@
 		  info.nodeid, info.pid, info.owner,
 		  info.wait);
 
+	/* report plock rate and any delays since the last report */
+	plock_read_count++;
+	if (!(plock_read_count % 1000)) {
+		gettimeofday(&now, NULL);
+		log_group(mg, "plock_read_count %u time %us delays %u",
+			  plock_read_count,
+			  (unsigned) (now.tv_sec - plock_read_time.tv_sec),
+			  plock_rate_delays);
+		plock_read_time = now;
+		plock_rate_delays = 0;
+	}
+
 	len = sizeof(struct gdlm_header) + sizeof(struct gdlm_plock_info);
 	buf = malloc(len);
 	if (!buf) {
@@ -878,6 +920,7 @@
 {
 	struct gdlm_plock_info info;
 	struct gdlm_header *hd = (struct gdlm_header *) buf;
+	struct timeval now;
 	int rv = 0;
 
 	memcpy(&info, buf + sizeof(struct gdlm_header), sizeof(info));
@@ -892,6 +935,14 @@
 		  info.nodeid, info.pid, info.owner,
 		  info.wait);
 
+	plock_recv_count++;
+	if (!(plock_recv_count % 1000)) {
+		gettimeofday(&now, NULL);
+		log_group(mg, "plock_recv_count %u time %us", plock_recv_count,
+			  (unsigned) (now.tv_sec - plock_recv_time.tv_sec));
+		plock_recv_time = now;
+	}
+
 	if (info.optype == GDLM_PLOCK_OP_GET && from != our_nodeid)
 		return;