udevd - throttle fork() if we get to many childs in running

udevd - throttle fork() if we get to many childs in running

[Kay Sievers]

[-- Attachment #1: Type: text/plain, Size: 347 bytes --]

We count the number of "R" processes in our session group and stop
forking if we reach a specified limit.

Sleeping events are not counted. Running hotplug.d/ and dev.d/ scripts
are counted, as they belong to our session group.

This should help the "lots of disks" setups where the kernel may spit
out lots of events in parallel.

Thanks,
Kay  


[-- Attachment #2: udevd-throttle-fork-01.patch --]
[-- Type: text/x-patch, Size: 4149 bytes --]

diff -Nru a/udevd.c b/udevd.c
--- a/udevd.c	2004-12-26 03:53:53 +01:00
+++ b/udevd.c	2004-12-26 03:53:53 +01:00
@@ -28,11 +28,13 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <ctype.h>
+#include <dirent.h>
+#include <fcntl.h>
 #include <sys/time.h>
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <sys/un.h>
-#include <fcntl.h>
 #include <sys/sysinfo.h>
 #include <sys/stat.h>
 
@@ -45,6 +47,7 @@
 
 /* global variables*/
 static int udevsendsock;
+static int sid;
 
 static int pipefds[2];
 static long startup_time;
@@ -133,6 +136,8 @@
 		/* child */
 		close(udevsendsock);
 		logging_close();
+
+		setpriority(PRIO_PROCESS, 0, UDEV_PRIORITY);
 		execve(udev_bin, argv, msg->envp);
 		dbg("exec of child failed");
 		_exit(1);
@@ -140,9 +145,6 @@
 	case -1:
 		dbg("fork of child failed");
 		run_queue_delete(msg);
-		/* note: we never managed to run, so we had no impact on 
-		 * running_with_devpath(), so don't bother setting run_exec_q
-		 */
 		break;
 	default:
 		/* get SIGCHLD in main loop */
@@ -151,6 +153,60 @@
 	}
 }
 
+static int running_processes_in_session(void)
+{
+	DIR *dir;
+	struct dirent *dent;
+	int running = 0;
+
+	dir = opendir("/proc");
+	if (!dir)
+		return -1;
+
+	/* read all processes in /proc */
+	for (dent = readdir(dir); dent != NULL; dent = readdir(dir)) {
+		int f;
+		char procdir[64];
+		char line[256];
+		const char *pos;
+		pid_t pid, ppid, pgrp, session;
+		char state;
+
+		if (!isdigit(dent->d_name[0]))
+			continue;
+
+		snprintf(procdir, sizeof(procdir), "/proc/%s/stat", dent->d_name);
+		procdir[sizeof(procdir)-1] = '\0';
+
+		f = open(procdir, O_RDONLY);
+		if (f == -1)
+			continue;
+
+		read(f, line, sizeof(line));
+		close(f);
+
+		/* skip ugly program name */
+		pos = strrchr(line, ')') + 2;
+
+		pid = strtol(line, NULL, 10);
+		if (sscanf(pos, "%c %d %d %d ", &state, &ppid, &pgrp, &session) != 4)
+			continue;
+
+		/* count only processes in our session */
+		if (session != sid)
+			continue;
+
+		/* count only running, no sleeping processes */
+		if (state != 'R')
+			continue;
+
+		running++;
+	}
+	closedir(dir);
+
+	return running;
+}
+
 static int compare_devpath(const char *running, const char *waiting)
 {
 	int i;
@@ -207,13 +263,26 @@
 	struct hotplug_msg *loop_msg;
 	struct hotplug_msg *tmp_msg;
 	struct hotplug_msg *msg;
+	int active;
+	int recheck = THROTTLE_RUNNING_CHILDS_COUNT;
 
 	list_for_each_entry_safe(loop_msg, tmp_msg, &exec_list, list) {
+		/* check running processes in our session and possibly throttle */
+		if (recheck >= THROTTLE_RUNNING_CHILDS_COUNT) {
+			active = running_processes_in_session();
+			if (active > THROTTLE_RUNNING_CHILDS_COUNT) {
+				dbg("skip exec, %d processes already in running state", active);
+				return;
+			}
+			recheck = 0;
+		}
+
 		msg = running_with_devpath(loop_msg);
 		if (!msg) {
 			/* move event to run list */
 			list_move_tail(&loop_msg->list, &running_list);
 			udev_run(loop_msg);
+			recheck++;
 			dbg("moved seq %llu to running list", loop_msg->seqnum);
 		} else {
 			dbg("delay seq %llu (%s), cause seq %llu (%s) is still running",
@@ -490,7 +559,10 @@
 		close(fd);
 
 	/* become session leader */
-	setsid();
+	sid = setsid();
+
+	/*set a reasonable scheduling priority for the daemon */
+	setpriority(PRIO_PROCESS, 0, UDEVD_PRIORITY);
 
 	/* setup signal handler pipe */
 	retval = pipe(pipefds);
diff -Nru a/udevd.h b/udevd.h
--- a/udevd.h	2004-12-26 03:53:53 +01:00
+++ b/udevd.h	2004-12-26 03:53:53 +01:00
@@ -28,9 +28,19 @@
 #define UDEVD_SOCK_PATH			"udevd"
 #define SEND_WAIT_MAX_SECONDS		3
 #define SEND_WAIT_LOOP_PER_SECOND	10
+
+#define UDEVD_PRIORITY			-5
+#define UDEV_PRIORITY			-2
+
+/* duration of initialization phase with shorter timeout */
 #define INIT_TIME_SEC			5
 #define EVENT_INIT_TIMEOUT_SEC		2
+
+/* timeout to wait for missing events */
 #define EVENT_TIMEOUT_SEC		10
+
+/* start to throttle forking if maximum number of running childs in our session is reached */
+#define THROTTLE_RUNNING_CHILDS_COUNT	10
 
 /* environment buffer, should match the kernel's size in lib/kobject_uevent.h */
 #define HOTPLUG_BUFFER_SIZE		1024

Re: udevd - throttle fork() if we get to many childs in running

[Kay Sievers]

On Sun, 2004-12-26 at 04:13 +0100, Kay Sievers wrote:
> We count the number of "R" processes in our session group and stop
> forking if we reach a specified limit.
> 
> Sleeping events are not counted. Running hotplug.d/ and dev.d/ scripts
> are counted, as they belong to our session group.
> 
> This should help the "lots of disks" setups where the kernel may spit
> out lots of events in parallel.

Here is an updated version which avoids scanning /proc if the system
isn't busy:

  If the system reaches a defined limit of processes in running state, udevd
  starts to count its own processes in running state from its session (all
  forked hotplug child processes, subprocesses and callouts) and throttles
  further process forking if the limit is reached.
  
  This should help setups with hundreds of events emitted hotplug events
  in parallel with hundreds of processes in "R" state. which makes the machine
  unresponsible.
  
  I placed a 100% cpu time consuming program in /etc/hotplug.d/ which runs for 5
  seconds. With this patch I can load "scsi_debug add_host\x100" without any major
  problem. Without the patch the box is completly unresponsible for many minutes.

The patch is in my udev tree at:
  bk://vrfy.bkbits.net/udev

diff -Nru a/udevd.c b/udevd.c
--- a/udevd.c	2005-01-16 17:10:37 +01:00
+++ b/udevd.c	2005-01-16 17:10:37 +01:00
@@ -28,11 +28,13 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <ctype.h>
+#include <dirent.h>
+#include <fcntl.h>
 #include <sys/time.h>
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <sys/un.h>
-#include <fcntl.h>
 #include <sys/sysinfo.h>
 #include <sys/stat.h>
 
@@ -45,6 +47,7 @@
 
 /* global variables*/
 static int udevsendsock;
+static pid_t sid;
 
 static int pipefds[2];
 static long startup_time;
@@ -144,6 +147,8 @@
 		/* child */
 		close(udevsendsock);
 		logging_close();
+
+		setpriority(PRIO_PROCESS, 0, UDEV_PRIORITY);
 		execve(udev_bin, argv, msg->envp);
 		dbg("exec of child failed");
 		_exit(1);
@@ -151,9 +156,6 @@
 	case -1:
 		dbg("fork of child failed");
 		run_queue_delete(msg);
-		/* note: we never managed to run, so we had no impact on 
-		 * running_with_devpath(), so don't bother setting run_exec_q
-		 */
 		break;
 	default:
 		/* get SIGCHLD in main loop */
@@ -162,6 +164,100 @@
 	}
 }
 
+static int running_processes(void)
+{
+	int f;
+	static char buf[4096];
+	int len;
+	int running;
+	const char *pos;
+
+	f = open("/proc/stat", O_RDONLY);
+	if (f = -1)
+		return -1;
+
+	len = read(f, buf, sizeof(buf));
+	close(f);
+
+	if (len <= 0)
+		return -1;
+	else
+		buf[len] = '\0';
+
+	pos = strstr(buf, "procs_running ");
+	if (pos = NULL)
+		return -1;
+
+	if (sscanf(pos, "procs_running %u", &running) != 1)
+		return -1;
+
+	return running;
+}
+
+/* return the number of process es in our session, count only until limit */
+static int running_processes_in_session(pid_t session, int limit)
+{
+	DIR *dir;
+	struct dirent *dent;
+	int running = 0;
+
+	dir = opendir("/proc");
+	if (!dir)
+		return -1;
+
+	/* read process info from /proc */
+	for (dent = readdir(dir); dent != NULL; dent = readdir(dir)) {
+		int f;
+		char procdir[64];
+		char line[256];
+		const char *pos;
+		char state;
+		pid_t ppid, pgrp, sess;
+		int len;
+
+		if (!isdigit(dent->d_name[0]))
+			continue;
+
+		snprintf(procdir, sizeof(procdir), "/proc/%s/stat", dent->d_name);
+		procdir[sizeof(procdir)-1] = '\0';
+
+		f = open(procdir, O_RDONLY);
+		if (f = -1)
+			continue;
+
+		len = read(f, line, sizeof(line));
+		close(f);
+
+		if (len <= 0)
+			continue;
+		else
+			line[len] = '\0';
+
+		/* skip ugly program name */
+		pos = strrchr(line, ')') + 2;
+		if (pos = NULL)
+			continue;
+
+		if (sscanf(pos, "%c %d %d %d ", &state, &ppid, &pgrp, &sess) != 4)
+			continue;
+
+		/* count only processes in our session */
+		if (sess != session)
+			continue;
+
+		/* count only running, no sleeping processes */
+		if (state != 'R')
+			continue;
+
+		running++;
+		if (limit > 0 && running >= limit)
+			break;
+	}
+	closedir(dir);
+
+	return running;
+}
+
 static int compare_devpath(const char *running, const char *waiting)
 {
 	int i;
@@ -218,13 +314,30 @@
 	struct hotplug_msg *loop_msg;
 	struct hotplug_msg *tmp_msg;
 	struct hotplug_msg *msg;
+	int running;
+
+	running = running_processes();
+	dbg("%d processes runnning on system", running);
+	if (running < 0)
+		running = THROTTLE_MAX_RUNNING_CHILDS;
 
 	list_for_each_entry_safe(loop_msg, tmp_msg, &exec_list, list) {
+		/* check running processes in our session and possibly throttle */
+		if (running >= THROTTLE_MAX_RUNNING_CHILDS) {
+			running = running_processes_in_session(sid, THROTTLE_MAX_RUNNING_CHILDS+10);
+			dbg("%d processes running in session", running);
+			if (running >= THROTTLE_MAX_RUNNING_CHILDS) {
+				dbg("delay seq %llu, cause too many processes already running", loop_msg->seqnum);
+				return;
+			}
+		}
+
 		msg = running_with_devpath(loop_msg);
 		if (!msg) {
 			/* move event to run list */
 			list_move_tail(&loop_msg->list, &running_list);
 			udev_run(loop_msg);
+			running++;
 			dbg("moved seq %llu to running list", loop_msg->seqnum);
 		} else {
 			dbg("delay seq %llu (%s), cause seq %llu (%s) is still running",
@@ -529,10 +642,17 @@
 		}
 	}
 
+	/* become session leader */
+	sid = setsid();
+	dbg("our session is %d", sid);
+
 	/* make sure we don't lock any path */
 	chdir("/");
 	umask(umask(077) | 022);
 
+	/*set a reasonable scheduling priority for the daemon */
+	setpriority(PRIO_PROCESS, 0, UDEVD_PRIORITY);
+
 	/* Set fds to dev/null */
 	fd = open( "/dev/null", O_RDWR );
 	if (fd >= 0)  {
@@ -544,9 +664,6 @@
 	} else
 		dbg("error opening /dev/null %s", strerror(errno));
 
-	/* become session leader */
-	setsid();
-
 	/* setup signal handler pipe */
 	retval = pipe(pipefds);
 	if (retval < 0) {
@@ -597,14 +714,14 @@
 	else
 		udev_bin = UDEV_BIN;
 
-	/* possible set of expected_seqnum number */
+	/* possible init of expected_seqnum value */
 	udevd_expected_seqnum = getenv("UDEVD_EXPECTED_SEQNUM");
 	if (udevd_expected_seqnum != NULL) {
 		expected_seqnum = strtoull(udevd_expected_seqnum, NULL, 10);
 		dbg("initialize expected_seqnum to %llu", expected_seqnum);
 	}
 
-	/* get current time to provide shorter startup timeout */
+	/* get current time to provide shorter timeout on startup */
 	sysinfo(&info);
 	startup_time = info.uptime;
 
diff -Nru a/udevd.h b/udevd.h
--- a/udevd.h	2005-01-16 17:10:37 +01:00
+++ b/udevd.h	2005-01-16 17:10:37 +01:00
@@ -28,9 +28,19 @@
 #define UDEVD_SOCK_PATH			"udevd"
 #define SEND_WAIT_MAX_SECONDS		3
 #define SEND_WAIT_LOOP_PER_SECOND	10
+
+#define UDEVD_PRIORITY			-4
+#define UDEV_PRIORITY			-2
+
+/* duration of initialization phase with shorter timeout */
 #define INIT_TIME_SEC			5
 #define EVENT_INIT_TIMEOUT_SEC		2
+
+/* timeout to wait for missing events */
 #define EVENT_TIMEOUT_SEC		10
+
+/* start to throttle forking if maximum number of running childs in our session is reached */
+#define THROTTLE_MAX_RUNNING_CHILDS	10
 
 /* environment buffer, should match the kernel's size in lib/kobject_uevent.h */
 #define HOTPLUG_BUFFER_SIZE		1024




-------------------------------------------------------
The SF.Net email is sponsored by: Beat the post-holiday blues
Get a FREE limited edition SourceForge.net t-shirt from ThinkGeek.
It's fun and FREE -- well, almost....http://www.thinkgeek.com/sfshirt
_______________________________________________
Linux-hotplug-devel mailing list  http://linux-hotplug.sourceforge.net
Linux-hotplug-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/linux-hotplug-devel

Re: udevd - throttle fork() if we get to many childs in running state

[Patrick Mansfield]

Hi Kay -

On Sun, Jan 16, 2005 at 05:16:39PM +0100, Kay Sievers wrote:

>   If the system reaches a defined limit of processes in running state, udevd
>   starts to count its own processes in running state from its session (all
>   forked hotplug child processes, subprocesses and callouts) and throttles
>   further process forking if the limit is reached.

Why do you only count running processes? Seems a bit abitrary, since you
don't know when the sleeping processes will become runnable.

Why not increase the total limit, and count all of its processes? I
doubt that would change timings.

-- Patrick Mansfield


-------------------------------------------------------
The SF.Net email is sponsored by: Beat the post-holiday blues
Get a FREE limited edition SourceForge.net t-shirt from ThinkGeek.
It's fun and FREE -- well, almost....http://www.thinkgeek.com/sfshirt
_______________________________________________
Linux-hotplug-devel mailing list  http://linux-hotplug.sourceforge.net
Linux-hotplug-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/linux-hotplug-devel

Re: udevd - throttle fork() if we get to many childs in running

[Kay Sievers]

On Mon, 2005-01-17 at 10:52 -0800, Patrick Mansfield wrote:
> Hi Kay -
> 
> On Sun, Jan 16, 2005 at 05:16:39PM +0100, Kay Sievers wrote:
> 
> >   If the system reaches a defined limit of processes in running state, udevd
> >   starts to count its own processes in running state from its session (all
> >   forked hotplug child processes, subprocesses and callouts) and throttles
> >   further process forking if the limit is reached.
> 
> Why do you only count running processes? Seems a bit abitrary, since you
> don't know when the sleeping processes will become runnable.

Good question! :)

Hmm, yes it's not a hard limit. If we reach the running maximum, we at
least don't fork more childs. My tests with the scsi_debug and a killer-
hotplug-script worked very nice that way.

> Why not increase the total limit, and count all of its processes? I
> doubt that would change timings.

The current scripts use sleep calls and I thought it would be nice to
let other processes run if they do nothing in the meantime.

Some stupid script may completely block the hotplug handling just by
sleeping for a long time with that hard limit.

Thanks,
Kay



-------------------------------------------------------
The SF.Net email is sponsored by: Beat the post-holiday blues
Get a FREE limited edition SourceForge.net t-shirt from ThinkGeek.
It's fun and FREE -- well, almost....http://www.thinkgeek.com/sfshirt
_______________________________________________
Linux-hotplug-devel mailing list  http://linux-hotplug.sourceforge.net
Linux-hotplug-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/linux-hotplug-devel