* [Cluster-devel] cluster/group/gfs_controld lock_dlm.h main.c r ...
@ 2006-07-20 20:19 teigland
0 siblings, 0 replies; 7+ messages in thread
From: teigland @ 2006-07-20 20:19 UTC (permalink / raw)
To: cluster-devel.redhat.com
CVSROOT: /cvs/cluster
Module name: cluster
Changes by: teigland at sourceware.org 2006-07-20 20:19:44
Modified files:
group/gfs_controld: lock_dlm.h main.c recover.c
Log message:
if mount.gfs is unmounting/leaving the group because the kernel mount
failed, then don't wait for the kernel mount to complete before doing
the leave
Patches:
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/gfs_controld/lock_dlm.h.diff?cvsroot=cluster&r1=1.6&r2=1.7
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/gfs_controld/main.c.diff?cvsroot=cluster&r1=1.5&r2=1.6
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/gfs_controld/recover.c.diff?cvsroot=cluster&r1=1.3&r2=1.4
--- cluster/group/gfs_controld/lock_dlm.h 2006/07/19 14:44:40 1.6
+++ cluster/group/gfs_controld/lock_dlm.h 2006/07/20 20:19:44 1.7
@@ -133,6 +133,7 @@
int mount_client_notified;
int mount_client_delay;
int delay_send_journals;
+ int kernel_mount_error;
int got_kernel_mount;
int first_mounter;
int first_mounter_done;
@@ -235,7 +236,7 @@
int do_mount(int ci, char *dir, char *type, char *proto, char *table,
char *options);
-int do_unmount(int ci, char *dir);
+int do_unmount(int ci, char *dir, int mnterr);
int do_remount(int ci, char *dir, char *mode);
int do_withdraw(char *name);
int kernel_recovery_done(char *name);
--- cluster/group/gfs_controld/main.c 2006/07/14 18:56:10 1.5
+++ cluster/group/gfs_controld/main.c 2006/07/20 20:19:44 1.6
@@ -178,7 +178,7 @@
if (!strcmp(cmd, "join"))
rv = do_mount(ci, dir, type, proto, table, extra);
else if (!strcmp(cmd, "leave"))
- rv = do_unmount(ci, dir);
+ rv = do_unmount(ci, dir, atoi(proto));
else if (!strcmp(cmd, "remount"))
rv = do_remount(ci, dir, argv[3]);
else if (!strcmp(cmd, "dump")) {
--- cluster/group/gfs_controld/recover.c 2006/06/15 20:41:46 1.3
+++ cluster/group/gfs_controld/recover.c 2006/07/20 20:19:44 1.4
@@ -1447,7 +1447,7 @@
return 0;
}
-int do_unmount(int ci, char *dir)
+int do_unmount(int ci, char *dir, int mnterr)
{
struct mountgroup *mg;
@@ -1466,11 +1466,17 @@
return -1;
}
+ if (mnterr) {
+ log_group(mg, "do_unmount: kernel mount error %d", mnterr);
+ mg->kernel_mount_error = mnterr;
+ goto out;
+ }
+
if (mg->withdraw) {
log_error("do_unmount: fs on %s is withdrawing", dir);
return -1;
}
-
+
/* Check to see if we're waiting for a kernel recovery_done to do a
start_done(). If so, call the start_done() here because we won't be
getting anything else from gfs-kernel which is now gone. */
@@ -1479,7 +1485,7 @@
log_group(mg, "do_unmount: fill in start_done");
start_done(mg);
}
-
+ out:
group_leave(gh, mg->name);
return 0;
}
@@ -1600,9 +1606,22 @@
if (mg->got_kernel_mount)
break;
- if (mg->mount_client_notified)
- wait_for_kernel_mount(mg);
- else {
+ if (mg->mount_client_notified) {
+
+ /* this kernel_mount_error check isn't perfect, we
+ could still 1) notify mount.gfs, 2) get a stop cb,
+ 3) kernel mount fails, 4) mount.gfs sends a leave
+ with mnterr, 5) we don't recv it and don't set
+ kernel_mount_error because we're stuck in
+ wait_for_kernel_mount() from do_stop */
+
+ if (!mg->kernel_mount_error)
+ wait_for_kernel_mount(mg);
+ else {
+ log_group(mg, "ignore stop, failed mount");
+ break;
+ }
+ } else {
mg->mount_client_delay = 1;
break;
}
^ permalink raw reply [flat|nested] 7+ messages in thread
* [Cluster-devel] cluster/group/gfs_controld lock_dlm.h main.c r ...
@ 2006-12-05 22:19 teigland
0 siblings, 0 replies; 7+ messages in thread
From: teigland @ 2006-12-05 22:19 UTC (permalink / raw)
To: cluster-devel.redhat.com
CVSROOT: /cvs/cluster
Module name: cluster
Changes by: teigland at sourceware.org 2006-12-05 22:19:17
Modified files:
group/gfs_controld: lock_dlm.h main.c recover.c
Log message:
Before doing the mount-group portion of withdraw, fork off a dmsetup to
suspend the fs device. This means gfs doesn't need to call dm_suspend()
in the kernel before calling out to us. The suspend waits for all
outstanding i/o to return on the device which is necessary prior to
telling other nodes to do recovery. (Later we should probably swap
in an error table and resume the device.)
bz 215962
Patches:
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/gfs_controld/lock_dlm.h.diff?cvsroot=cluster&r1=1.23&r2=1.24
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/gfs_controld/main.c.diff?cvsroot=cluster&r1=1.26&r2=1.27
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/gfs_controld/recover.c.diff?cvsroot=cluster&r1=1.23&r2=1.24
--- cluster/group/gfs_controld/lock_dlm.h 2006/11/21 17:28:08 1.23
+++ cluster/group/gfs_controld/lock_dlm.h 2006/12/05 22:19:17 1.24
@@ -30,6 +30,7 @@
#include <sys/un.h>
#include <sys/types.h>
#include <sys/stat.h>
+#include <sys/wait.h>
#include <sys/errno.h>
#include <linux/netlink.h>
@@ -125,6 +126,7 @@
char type[5];
char dir[PATH_MAX+1];
char options[MAX_OPTIONS_LEN+1];
+ char dev[PATH_MAX+1];
int last_stop;
int last_start;
@@ -167,6 +169,8 @@
int readonly;
int rw;
int withdraw;
+ int dmsetup_wait;
+ pid_t dmsetup_pid;
struct list_head saved_messages;
void *start2_fn;
@@ -269,7 +273,7 @@
void exit_cman(void);
int do_mount(int ci, char *dir, char *type, char *proto, char *table,
- char *options, struct mountgroup **mg_ret);
+ char *options, char *dev, struct mountgroup **mg_ret);
int do_unmount(int ci, char *dir, int mnterr);
int do_remount(int ci, char *dir, char *mode);
int do_withdraw(char *name);
@@ -289,5 +293,6 @@
void process_saved_plocks(struct mountgroup *mg);
void purge_plocks(struct mountgroup *mg, int nodeid, int unmount);
int unlink_checkpoint(struct mountgroup *mg);
+void update_dmsetup_wait(void);
#endif
--- cluster/group/gfs_controld/main.c 2006/12/05 16:59:53 1.26
+++ cluster/group/gfs_controld/main.c 2006/12/05 22:19:17 1.27
@@ -41,6 +41,7 @@
int no_withdraw;
int no_plock;
uint32_t plock_rate_limit = DEFAULT_PLOCK_RATE_LIMIT;
+int dmsetup_wait;
int do_read(int fd, void *buf, size_t count)
{
@@ -285,14 +286,16 @@
log_debug("client %d: %s", ci, buf);
- get_args(buf, &argc, argv, ' ', 6);
+ get_args(buf, &argc, argv, ' ', 7);
cmd = argv[0];
rv = 0;
if (!strcmp(cmd, "join")) {
- /* ci, dir, type, proto, table, extra */
+ /* ci, dir (mountpoint), type (gfs/gfs2), proto (lock_dlm),
+ table (fsname:clustername), extra (rw), dev (/dev/sda1) */
+
rv = do_mount(ci, argv[1], argv[2], argv[3], argv[4], argv[5],
- &mg);
+ argv[6], &mg);
fd = client[ci].fd;
fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK);
if (!rv) {
@@ -533,6 +536,17 @@
poll_timeout = -1;
}
}
+
+ if (dmsetup_wait) {
+ update_dmsetup_wait();
+ if (dmsetup_wait) {
+ if (poll_timeout == -1)
+ poll_timeout = 1000;
+ } else {
+ if (poll_timeout == 1000)
+ poll_timeout = -1;
+ }
+ }
}
}
rv = 0;
--- cluster/group/gfs_controld/recover.c 2006/10/23 15:44:33 1.23
+++ cluster/group/gfs_controld/recover.c 2006/12/05 22:19:17 1.24
@@ -19,6 +19,7 @@
extern int our_nodeid;
extern group_handle_t gh;
extern int no_withdraw;
+extern int dmsetup_wait;
struct list_head mounts;
struct list_head withdrawn_mounts;
@@ -1493,15 +1494,15 @@
}
int do_mount(int ci, char *dir, char *type, char *proto, char *table,
- char *options, struct mountgroup **mg_ret)
+ char *options, char *dev, struct mountgroup **mg_ret)
{
struct mountgroup *mg;
char table2[MAXLINE];
char *cluster = NULL, *name = NULL;
int rv;
- log_debug("mount: %s %s %s %s %s",
- dir, type, proto, table, options);
+ log_debug("mount: %s %s %s %s %s %s",
+ dir, type, proto, table, options, dev);
if (strcmp(proto, "lock_dlm")) {
log_error("mount: lockproto %s not supported", proto);
@@ -1554,6 +1555,7 @@
strncpy(mg->type, type, sizeof(mg->type));
strncpy(mg->table, table, sizeof(mg->table));
strncpy(mg->options, options, sizeof(mg->options));
+ strncpy(mg->dev, dev, sizeof(mg->dev));
if (strlen(cluster) != strlen(clustername) ||
strlen(cluster) == 0 || strcmp(cluster, clustername)) {
@@ -2478,6 +2480,41 @@
return 0;
}
+static int run_dmsetup_suspend(struct mountgroup *mg, char *dev)
+{
+ struct sched_param sched_param;
+ char buf[PATH_MAX];
+ pid_t pid;
+ int i, rv;
+
+ memset(buf, 0, sizeof(buf));
+ rv = readlink(dev, buf, PATH_MAX);
+ if (rv < 0)
+ strncpy(buf, dev, sizeof(buf));
+
+ log_group(mg, "run_dmsetup_suspend %s (orig %s)", buf, dev);
+
+ pid = fork();
+ if (pid < 0)
+ return -1;
+
+ if (pid) {
+ mg->dmsetup_wait = 1;
+ mg->dmsetup_pid = pid;
+ return 0;
+ } else {
+ sched_param.sched_priority = 0;
+ sched_setscheduler(0, SCHED_OTHER, &sched_param);
+
+ for (i = 0; i < 50; i++)
+ close(i);
+
+ execlp("dmsetup", "dmsetup", "suspend", buf, NULL);
+ exit(EXIT_FAILURE);
+ }
+ return -1;
+}
+
/* The basic rule of withdraw is that we don't want to tell the kernel to drop
all locks until we know gfs has been stopped/blocked on all nodes. They'll
be stopped for our leave, we just need to know when they've all arrived
@@ -2492,6 +2529,7 @@
{
struct mountgroup *mg;
char *name = strstr(table, ":") + 1;
+ int rv;
if (no_withdraw) {
log_error("withdraw feature not enabled");
@@ -2504,8 +2542,66 @@
return -1;
}
- mg->withdraw = 1;
- send_withdraw(mg);
+ rv = run_dmsetup_suspend(mg, mg->dev);
+ if (rv) {
+ log_error("do_withdraw %s: dmsetup %s error %d", mg->name,
+ mg->dev, rv);
+ return -1;
+ }
+
+ dmsetup_wait = 1;
return 0;
}
+void dmsetup_suspend_done(struct mountgroup *mg, int rv)
+{
+ log_group(mg, "dmsetup_suspend_done result %d", rv);
+ mg->dmsetup_wait = 0;
+ mg->dmsetup_pid = 0;
+
+ if (!rv) {
+ mg->withdraw = 1;
+ send_withdraw(mg);
+ }
+}
+
+void update_dmsetup_wait(void)
+{
+ struct mountgroup *mg;
+ int status;
+ int waiting = 0;
+ pid_t pid;
+
+ list_for_each_entry(mg, &mounts, list) {
+ if (mg->dmsetup_wait) {
+ pid = waitpid(mg->dmsetup_pid, &status, WNOHANG);
+
+ /* process not exited yet */
+ if (!pid) {
+ waiting++;
+ continue;
+ }
+
+ if (pid < 0) {
+ log_error("update_dmsetup_wait %s: waitpid %d "
+ "error %d", mg->name,
+ mg->dmsetup_pid, errno);
+ dmsetup_suspend_done(mg, -2);
+ continue;
+ }
+
+ /* process exited */
+
+ if (!WIFEXITED(status) || WEXITSTATUS(status))
+ dmsetup_suspend_done(mg, -1);
+ else
+ dmsetup_suspend_done(mg, 0);
+ }
+ }
+
+ if (!waiting) {
+ dmsetup_wait = 0;
+ log_debug("dmsetup_wait off");
+ }
+}
+
^ permalink raw reply [flat|nested] 7+ messages in thread
* [Cluster-devel] cluster/group/gfs_controld lock_dlm.h main.c r ...
@ 2006-12-05 22:24 teigland
0 siblings, 0 replies; 7+ messages in thread
From: teigland @ 2006-12-05 22:24 UTC (permalink / raw)
To: cluster-devel.redhat.com
CVSROOT: /cvs/cluster
Module name: cluster
Branch: RHEL5
Changes by: teigland at sourceware.org 2006-12-05 22:24:29
Modified files:
group/gfs_controld: lock_dlm.h main.c recover.c
Log message:
Before doing the mount-group portion of withdraw, fork off a dmsetup to
suspend the fs device. This means gfs doesn't need to call dm_suspend()
in the kernel before calling out to us. The suspend waits for all
outstanding i/o to return on the device which is necessary prior to
telling other nodes to do recovery. (Later we should probably swap
in an error table and resume the device.)
bz 215962
Patches:
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/gfs_controld/lock_dlm.h.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.21.2.2&r2=1.21.2.3
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/gfs_controld/main.c.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.18.2.8&r2=1.18.2.9
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/gfs_controld/recover.c.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.23&r2=1.23.2.1
--- cluster/group/gfs_controld/lock_dlm.h 2006/11/21 17:28:45 1.21.2.2
+++ cluster/group/gfs_controld/lock_dlm.h 2006/12/05 22:24:29 1.21.2.3
@@ -30,6 +30,7 @@
#include <sys/un.h>
#include <sys/types.h>
#include <sys/stat.h>
+#include <sys/wait.h>
#include <sys/errno.h>
#include <linux/netlink.h>
@@ -125,6 +126,7 @@
char type[5];
char dir[PATH_MAX+1];
char options[MAX_OPTIONS_LEN+1];
+ char dev[PATH_MAX+1];
int last_stop;
int last_start;
@@ -167,6 +169,8 @@
int readonly;
int rw;
int withdraw;
+ int dmsetup_wait;
+ pid_t dmsetup_pid;
struct list_head saved_messages;
void *start2_fn;
@@ -269,7 +273,7 @@
void exit_cman(void);
int do_mount(int ci, char *dir, char *type, char *proto, char *table,
- char *options, struct mountgroup **mg_ret);
+ char *options, char *dev, struct mountgroup **mg_ret);
int do_unmount(int ci, char *dir, int mnterr);
int do_remount(int ci, char *dir, char *mode);
int do_withdraw(char *name);
@@ -289,5 +293,6 @@
void process_saved_plocks(struct mountgroup *mg);
void purge_plocks(struct mountgroup *mg, int nodeid, int unmount);
int unlink_checkpoint(struct mountgroup *mg);
+void update_dmsetup_wait(void);
#endif
--- cluster/group/gfs_controld/main.c 2006/12/05 17:26:46 1.18.2.8
+++ cluster/group/gfs_controld/main.c 2006/12/05 22:24:29 1.18.2.9
@@ -41,6 +41,7 @@
int no_withdraw;
int no_plock;
uint32_t plock_rate_limit = DEFAULT_PLOCK_RATE_LIMIT;
+int dmsetup_wait;
int do_read(int fd, void *buf, size_t count)
{
@@ -285,14 +286,16 @@
log_debug("client %d: %s", ci, buf);
- get_args(buf, &argc, argv, ' ', 6);
+ get_args(buf, &argc, argv, ' ', 7);
cmd = argv[0];
rv = 0;
if (!strcmp(cmd, "join")) {
- /* ci, dir, type, proto, table, extra */
+ /* ci, dir (mountpoint), type (gfs/gfs2), proto (lock_dlm),
+ table (fsname:clustername), extra (rw), dev (/dev/sda1) */
+
rv = do_mount(ci, argv[1], argv[2], argv[3], argv[4], argv[5],
- &mg);
+ argv[6], &mg);
fd = client[ci].fd;
fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK);
if (!rv) {
@@ -533,6 +536,17 @@
poll_timeout = -1;
}
}
+
+ if (dmsetup_wait) {
+ update_dmsetup_wait();
+ if (dmsetup_wait) {
+ if (poll_timeout == -1)
+ poll_timeout = 1000;
+ } else {
+ if (poll_timeout == 1000)
+ poll_timeout = -1;
+ }
+ }
}
}
rv = 0;
--- cluster/group/gfs_controld/recover.c 2006/10/23 15:44:33 1.23
+++ cluster/group/gfs_controld/recover.c 2006/12/05 22:24:29 1.23.2.1
@@ -19,6 +19,7 @@
extern int our_nodeid;
extern group_handle_t gh;
extern int no_withdraw;
+extern int dmsetup_wait;
struct list_head mounts;
struct list_head withdrawn_mounts;
@@ -1493,15 +1494,15 @@
}
int do_mount(int ci, char *dir, char *type, char *proto, char *table,
- char *options, struct mountgroup **mg_ret)
+ char *options, char *dev, struct mountgroup **mg_ret)
{
struct mountgroup *mg;
char table2[MAXLINE];
char *cluster = NULL, *name = NULL;
int rv;
- log_debug("mount: %s %s %s %s %s",
- dir, type, proto, table, options);
+ log_debug("mount: %s %s %s %s %s %s",
+ dir, type, proto, table, options, dev);
if (strcmp(proto, "lock_dlm")) {
log_error("mount: lockproto %s not supported", proto);
@@ -1554,6 +1555,7 @@
strncpy(mg->type, type, sizeof(mg->type));
strncpy(mg->table, table, sizeof(mg->table));
strncpy(mg->options, options, sizeof(mg->options));
+ strncpy(mg->dev, dev, sizeof(mg->dev));
if (strlen(cluster) != strlen(clustername) ||
strlen(cluster) == 0 || strcmp(cluster, clustername)) {
@@ -2478,6 +2480,41 @@
return 0;
}
+static int run_dmsetup_suspend(struct mountgroup *mg, char *dev)
+{
+ struct sched_param sched_param;
+ char buf[PATH_MAX];
+ pid_t pid;
+ int i, rv;
+
+ memset(buf, 0, sizeof(buf));
+ rv = readlink(dev, buf, PATH_MAX);
+ if (rv < 0)
+ strncpy(buf, dev, sizeof(buf));
+
+ log_group(mg, "run_dmsetup_suspend %s (orig %s)", buf, dev);
+
+ pid = fork();
+ if (pid < 0)
+ return -1;
+
+ if (pid) {
+ mg->dmsetup_wait = 1;
+ mg->dmsetup_pid = pid;
+ return 0;
+ } else {
+ sched_param.sched_priority = 0;
+ sched_setscheduler(0, SCHED_OTHER, &sched_param);
+
+ for (i = 0; i < 50; i++)
+ close(i);
+
+ execlp("dmsetup", "dmsetup", "suspend", buf, NULL);
+ exit(EXIT_FAILURE);
+ }
+ return -1;
+}
+
/* The basic rule of withdraw is that we don't want to tell the kernel to drop
all locks until we know gfs has been stopped/blocked on all nodes. They'll
be stopped for our leave, we just need to know when they've all arrived
@@ -2492,6 +2529,7 @@
{
struct mountgroup *mg;
char *name = strstr(table, ":") + 1;
+ int rv;
if (no_withdraw) {
log_error("withdraw feature not enabled");
@@ -2504,8 +2542,66 @@
return -1;
}
- mg->withdraw = 1;
- send_withdraw(mg);
+ rv = run_dmsetup_suspend(mg, mg->dev);
+ if (rv) {
+ log_error("do_withdraw %s: dmsetup %s error %d", mg->name,
+ mg->dev, rv);
+ return -1;
+ }
+
+ dmsetup_wait = 1;
return 0;
}
+void dmsetup_suspend_done(struct mountgroup *mg, int rv)
+{
+ log_group(mg, "dmsetup_suspend_done result %d", rv);
+ mg->dmsetup_wait = 0;
+ mg->dmsetup_pid = 0;
+
+ if (!rv) {
+ mg->withdraw = 1;
+ send_withdraw(mg);
+ }
+}
+
+void update_dmsetup_wait(void)
+{
+ struct mountgroup *mg;
+ int status;
+ int waiting = 0;
+ pid_t pid;
+
+ list_for_each_entry(mg, &mounts, list) {
+ if (mg->dmsetup_wait) {
+ pid = waitpid(mg->dmsetup_pid, &status, WNOHANG);
+
+ /* process not exited yet */
+ if (!pid) {
+ waiting++;
+ continue;
+ }
+
+ if (pid < 0) {
+ log_error("update_dmsetup_wait %s: waitpid %d "
+ "error %d", mg->name,
+ mg->dmsetup_pid, errno);
+ dmsetup_suspend_done(mg, -2);
+ continue;
+ }
+
+ /* process exited */
+
+ if (!WIFEXITED(status) || WEXITSTATUS(status))
+ dmsetup_suspend_done(mg, -1);
+ else
+ dmsetup_suspend_done(mg, 0);
+ }
+ }
+
+ if (!waiting) {
+ dmsetup_wait = 0;
+ log_debug("dmsetup_wait off");
+ }
+}
+
^ permalink raw reply [flat|nested] 7+ messages in thread
* [Cluster-devel] cluster/group/gfs_controld lock_dlm.h main.c r ...
@ 2006-12-05 22:24 teigland
0 siblings, 0 replies; 7+ messages in thread
From: teigland @ 2006-12-05 22:24 UTC (permalink / raw)
To: cluster-devel.redhat.com
CVSROOT: /cvs/cluster
Module name: cluster
Branch: RHEL50
Changes by: teigland at sourceware.org 2006-12-05 22:24:37
Modified files:
group/gfs_controld: lock_dlm.h main.c recover.c
Log message:
Before doing the mount-group portion of withdraw, fork off a dmsetup to
suspend the fs device. This means gfs doesn't need to call dm_suspend()
in the kernel before calling out to us. The suspend waits for all
outstanding i/o to return on the device which is necessary prior to
telling other nodes to do recovery. (Later we should probably swap
in an error table and resume the device.)
bz 215962
Patches:
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/gfs_controld/lock_dlm.h.diff?cvsroot=cluster&only_with_tag=RHEL50&r1=1.21.4.2&r2=1.21.4.3
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/gfs_controld/main.c.diff?cvsroot=cluster&only_with_tag=RHEL50&r1=1.18.4.7&r2=1.18.4.8
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/gfs_controld/recover.c.diff?cvsroot=cluster&only_with_tag=RHEL50&r1=1.23&r2=1.23.4.1
--- cluster/group/gfs_controld/lock_dlm.h 2006/11/21 17:28:52 1.21.4.2
+++ cluster/group/gfs_controld/lock_dlm.h 2006/12/05 22:24:37 1.21.4.3
@@ -30,6 +30,7 @@
#include <sys/un.h>
#include <sys/types.h>
#include <sys/stat.h>
+#include <sys/wait.h>
#include <sys/errno.h>
#include <linux/netlink.h>
@@ -125,6 +126,7 @@
char type[5];
char dir[PATH_MAX+1];
char options[MAX_OPTIONS_LEN+1];
+ char dev[PATH_MAX+1];
int last_stop;
int last_start;
@@ -167,6 +169,8 @@
int readonly;
int rw;
int withdraw;
+ int dmsetup_wait;
+ pid_t dmsetup_pid;
struct list_head saved_messages;
void *start2_fn;
@@ -269,7 +273,7 @@
void exit_cman(void);
int do_mount(int ci, char *dir, char *type, char *proto, char *table,
- char *options, struct mountgroup **mg_ret);
+ char *options, char *dev, struct mountgroup **mg_ret);
int do_unmount(int ci, char *dir, int mnterr);
int do_remount(int ci, char *dir, char *mode);
int do_withdraw(char *name);
@@ -289,5 +293,6 @@
void process_saved_plocks(struct mountgroup *mg);
void purge_plocks(struct mountgroup *mg, int nodeid, int unmount);
int unlink_checkpoint(struct mountgroup *mg);
+void update_dmsetup_wait(void);
#endif
--- cluster/group/gfs_controld/main.c 2006/12/05 17:26:52 1.18.4.7
+++ cluster/group/gfs_controld/main.c 2006/12/05 22:24:37 1.18.4.8
@@ -41,6 +41,7 @@
int no_withdraw;
int no_plock;
uint32_t plock_rate_limit = DEFAULT_PLOCK_RATE_LIMIT;
+int dmsetup_wait;
int do_read(int fd, void *buf, size_t count)
{
@@ -285,14 +286,16 @@
log_debug("client %d: %s", ci, buf);
- get_args(buf, &argc, argv, ' ', 6);
+ get_args(buf, &argc, argv, ' ', 7);
cmd = argv[0];
rv = 0;
if (!strcmp(cmd, "join")) {
- /* ci, dir, type, proto, table, extra */
+ /* ci, dir (mountpoint), type (gfs/gfs2), proto (lock_dlm),
+ table (fsname:clustername), extra (rw), dev (/dev/sda1) */
+
rv = do_mount(ci, argv[1], argv[2], argv[3], argv[4], argv[5],
- &mg);
+ argv[6], &mg);
fd = client[ci].fd;
fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK);
if (!rv) {
@@ -533,6 +536,17 @@
poll_timeout = -1;
}
}
+
+ if (dmsetup_wait) {
+ update_dmsetup_wait();
+ if (dmsetup_wait) {
+ if (poll_timeout == -1)
+ poll_timeout = 1000;
+ } else {
+ if (poll_timeout == 1000)
+ poll_timeout = -1;
+ }
+ }
}
}
rv = 0;
--- cluster/group/gfs_controld/recover.c 2006/10/23 15:44:33 1.23
+++ cluster/group/gfs_controld/recover.c 2006/12/05 22:24:37 1.23.4.1
@@ -19,6 +19,7 @@
extern int our_nodeid;
extern group_handle_t gh;
extern int no_withdraw;
+extern int dmsetup_wait;
struct list_head mounts;
struct list_head withdrawn_mounts;
@@ -1493,15 +1494,15 @@
}
int do_mount(int ci, char *dir, char *type, char *proto, char *table,
- char *options, struct mountgroup **mg_ret)
+ char *options, char *dev, struct mountgroup **mg_ret)
{
struct mountgroup *mg;
char table2[MAXLINE];
char *cluster = NULL, *name = NULL;
int rv;
- log_debug("mount: %s %s %s %s %s",
- dir, type, proto, table, options);
+ log_debug("mount: %s %s %s %s %s %s",
+ dir, type, proto, table, options, dev);
if (strcmp(proto, "lock_dlm")) {
log_error("mount: lockproto %s not supported", proto);
@@ -1554,6 +1555,7 @@
strncpy(mg->type, type, sizeof(mg->type));
strncpy(mg->table, table, sizeof(mg->table));
strncpy(mg->options, options, sizeof(mg->options));
+ strncpy(mg->dev, dev, sizeof(mg->dev));
if (strlen(cluster) != strlen(clustername) ||
strlen(cluster) == 0 || strcmp(cluster, clustername)) {
@@ -2478,6 +2480,41 @@
return 0;
}
+static int run_dmsetup_suspend(struct mountgroup *mg, char *dev)
+{
+ struct sched_param sched_param;
+ char buf[PATH_MAX];
+ pid_t pid;
+ int i, rv;
+
+ memset(buf, 0, sizeof(buf));
+ rv = readlink(dev, buf, PATH_MAX);
+ if (rv < 0)
+ strncpy(buf, dev, sizeof(buf));
+
+ log_group(mg, "run_dmsetup_suspend %s (orig %s)", buf, dev);
+
+ pid = fork();
+ if (pid < 0)
+ return -1;
+
+ if (pid) {
+ mg->dmsetup_wait = 1;
+ mg->dmsetup_pid = pid;
+ return 0;
+ } else {
+ sched_param.sched_priority = 0;
+ sched_setscheduler(0, SCHED_OTHER, &sched_param);
+
+ for (i = 0; i < 50; i++)
+ close(i);
+
+ execlp("dmsetup", "dmsetup", "suspend", buf, NULL);
+ exit(EXIT_FAILURE);
+ }
+ return -1;
+}
+
/* The basic rule of withdraw is that we don't want to tell the kernel to drop
all locks until we know gfs has been stopped/blocked on all nodes. They'll
be stopped for our leave, we just need to know when they've all arrived
@@ -2492,6 +2529,7 @@
{
struct mountgroup *mg;
char *name = strstr(table, ":") + 1;
+ int rv;
if (no_withdraw) {
log_error("withdraw feature not enabled");
@@ -2504,8 +2542,66 @@
return -1;
}
- mg->withdraw = 1;
- send_withdraw(mg);
+ rv = run_dmsetup_suspend(mg, mg->dev);
+ if (rv) {
+ log_error("do_withdraw %s: dmsetup %s error %d", mg->name,
+ mg->dev, rv);
+ return -1;
+ }
+
+ dmsetup_wait = 1;
return 0;
}
+void dmsetup_suspend_done(struct mountgroup *mg, int rv)
+{
+ log_group(mg, "dmsetup_suspend_done result %d", rv);
+ mg->dmsetup_wait = 0;
+ mg->dmsetup_pid = 0;
+
+ if (!rv) {
+ mg->withdraw = 1;
+ send_withdraw(mg);
+ }
+}
+
+void update_dmsetup_wait(void)
+{
+ struct mountgroup *mg;
+ int status;
+ int waiting = 0;
+ pid_t pid;
+
+ list_for_each_entry(mg, &mounts, list) {
+ if (mg->dmsetup_wait) {
+ pid = waitpid(mg->dmsetup_pid, &status, WNOHANG);
+
+ /* process not exited yet */
+ if (!pid) {
+ waiting++;
+ continue;
+ }
+
+ if (pid < 0) {
+ log_error("update_dmsetup_wait %s: waitpid %d "
+ "error %d", mg->name,
+ mg->dmsetup_pid, errno);
+ dmsetup_suspend_done(mg, -2);
+ continue;
+ }
+
+ /* process exited */
+
+ if (!WIFEXITED(status) || WEXITSTATUS(status))
+ dmsetup_suspend_done(mg, -1);
+ else
+ dmsetup_suspend_done(mg, 0);
+ }
+ }
+
+ if (!waiting) {
+ dmsetup_wait = 0;
+ log_debug("dmsetup_wait off");
+ }
+}
+
^ permalink raw reply [flat|nested] 7+ messages in thread
* [Cluster-devel] cluster/group/gfs_controld lock_dlm.h main.c r ...
@ 2006-12-20 19:13 teigland
0 siblings, 0 replies; 7+ messages in thread
From: teigland @ 2006-12-20 19:13 UTC (permalink / raw)
To: cluster-devel.redhat.com
CVSROOT: /cvs/cluster
Module name: cluster
Changes by: teigland at sourceware.org 2006-12-20 19:13:13
Modified files:
group/gfs_controld: lock_dlm.h main.c recover.c
Log message:
Support mounting a single fs on multiple mount points.
bz 218560
Patches:
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/gfs_controld/lock_dlm.h.diff?cvsroot=cluster&r1=1.26&r2=1.27
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/gfs_controld/main.c.diff?cvsroot=cluster&r1=1.27&r2=1.28
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/gfs_controld/recover.c.diff?cvsroot=cluster&r1=1.27&r2=1.28
--- cluster/group/gfs_controld/lock_dlm.h 2006/12/19 17:05:59 1.26
+++ cluster/group/gfs_controld/lock_dlm.h 2006/12/20 19:13:13 1.27
@@ -113,6 +113,12 @@
} \
}
+struct mountpoint {
+ struct list_head list;
+ char dir[PATH_MAX+1];
+ int client;
+};
+
struct mountgroup {
struct list_head list;
uint32_t id;
@@ -120,11 +126,11 @@
struct list_head members_gone;
int memb_count;
struct list_head resources; /* for plocks */
+ struct list_head mountpoints;
char name[MAXNAME+1];
char table[MAXNAME+1];
char type[5];
- char dir[PATH_MAX+1];
char options[MAX_OPTIONS_LEN+1];
char dev[PATH_MAX+1];
@@ -280,7 +286,7 @@
int kernel_recovery_done(char *name);
void ping_kernel_mount(char *table);
void save_message(struct mountgroup *mg, char *buf, int len, int from, int type);
-void got_mount_result(struct mountgroup *mg, int result);
+void got_mount_result(struct mountgroup *mg, int result, int ci, int another);
int client_send(int ci, char *buf, int len);
--- cluster/group/gfs_controld/main.c 2006/12/05 22:19:17 1.27
+++ cluster/group/gfs_controld/main.c 2006/12/20 19:13:13 1.28
@@ -21,6 +21,7 @@
int fd;
char type[32];
struct mountgroup *mg;
+ int another_mount;
};
static int client_maxi;
@@ -298,14 +299,15 @@
argv[6], &mg);
fd = client[ci].fd;
fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK);
- if (!rv) {
+ if (!rv || rv == -EALREADY) {
+ client[ci].another_mount = rv;
client[ci].mg = mg;
mg->mount_client_fd = fd;
}
goto reply;
} else if (!strcmp(cmd, "mount_result")) {
- got_mount_result(client[ci].mg, atoi(argv[3]));
-
+ got_mount_result(client[ci].mg, atoi(argv[3]), ci,
+ client[ci].another_mount);
} else if (!strcmp(cmd, "leave")) {
rv = do_unmount(ci, argv[1], atoi(argv[3]));
goto reply;
--- cluster/group/gfs_controld/recover.c 2006/12/19 22:19:01 1.27
+++ cluster/group/gfs_controld/recover.c 2006/12/20 19:13:13 1.28
@@ -1430,23 +1430,36 @@
}
}
-struct mountgroup *create_mg(char *name)
+struct mountgroup *create_mg(char *name, char *dir)
{
struct mountgroup *mg;
+ struct mountpoint *mp;
mg = malloc(sizeof(struct mountgroup));
+ if (!mg)
+ return NULL;
memset(mg, 0, sizeof(struct mountgroup));
INIT_LIST_HEAD(&mg->members);
INIT_LIST_HEAD(&mg->members_gone);
INIT_LIST_HEAD(&mg->resources);
INIT_LIST_HEAD(&mg->saved_messages);
+ INIT_LIST_HEAD(&mg->mountpoints);
mg->init = 1;
mg->master_nodeid = -1;
mg->low_nodeid = -1;
strncpy(mg->name, name, MAXNAME);
+ mp = malloc(sizeof(struct mountpoint));
+ if (!mp) {
+ free(mg);
+ return NULL;
+ }
+ memset(mp, 0, sizeof(struct mountpoint));
+ strncpy(mp->dir, dir, sizeof(mp->dir));
+ list_add(&mp->list, &mg->mountpoints);
+
return mg;
}
@@ -1473,12 +1486,23 @@
return NULL;
}
+struct mountpoint *find_mountpoint(struct mountgroup *mg, char *dir)
+{
+ struct mountpoint *mp;
+
+ list_for_each_entry(mp, &mg->mountpoints, list) {
+ if (!strcmp(mp->dir, dir))
+ return mp;
+ }
+ return NULL;
+}
+
struct mountgroup *find_mg_dir(char *dir)
{
struct mountgroup *mg;
list_for_each_entry(mg, &mounts, list) {
- if (!strcmp(mg->dir, dir))
+ if (find_mountpoint(mg, dir))
return mg;
}
return NULL;
@@ -1501,10 +1525,54 @@
return 0;
}
+int add_another_mountpoint(struct mountgroup *mg, char *dir, char *dev, int ci)
+{
+ struct mountpoint *mp;
+
+ log_group(mg, "add_another_mountpoint dir %s dev %s ci %d",
+ dir, dev, ci);
+
+ /* check if this is the same fs mounted on another dir or a different
+ fs with the same name (which is an error) */
+
+ if (strcmp(mg->dev, dev)) {
+ log_group(mg, "different fs dev %s with same name", mg->dev);
+ return -EINVAL;
+ }
+
+ if (find_mountpoint(mg, dir)) {
+ log_group(mg, "mount point %s already used", dir);
+ return -EBUSY;
+ }
+
+ /* we only really need to check one of these */
+ if (mg->mount_client || mg->mount_client_fd || !mg->kernel_mount_done) {
+ log_group(mg, "other mount in progress client %d fd %d done %d",
+ mg->mount_client, mg->mount_client_fd,
+ mg->kernel_mount_done);
+ return -EBUSY;
+ }
+
+ mp = malloc(sizeof(struct mountpoint));
+ if (!mp)
+ return -ENOMEM;
+
+ memset(mp, 0, sizeof(struct mountpoint));
+ strncpy(mp->dir, dir, sizeof(mp->dir));
+ list_add(&mp->list, &mg->mountpoints);
+ mg->mount_client = ci;
+
+ /* we return this special error to mount.gfs which mount.gfs will
+ recognize as meaning the fs is already mounted, so it shouldn't
+ read any hostdata from us, but just go ahead and mount(2) */
+
+ return -EALREADY;
+}
+
int do_mount(int ci, char *dir, char *type, char *proto, char *table,
char *options, char *dev, struct mountgroup **mg_ret)
{
- struct mountgroup *mg;
+ struct mountgroup *mg = NULL;
char table2[MAXLINE];
char *cluster = NULL, *name = NULL;
int rv;
@@ -1515,7 +1583,7 @@
if (strcmp(proto, "lock_dlm")) {
log_error("mount: lockproto %s not supported", proto);
rv = -EINVAL;
- goto fail;
+ goto out;
}
if (strstr(options, "jid=") ||
@@ -1523,7 +1591,7 @@
strstr(options, "id=")) {
log_error("mount: jid, first and id are reserved options");
rv = -EINVAL;
- goto fail;
+ goto out;
}
/* table is <cluster>:<name> */
@@ -1534,7 +1602,7 @@
name = strstr(table2, ":");
if (!name) {
rv = -EINVAL;
- goto fail;
+ goto out;
}
*name = '\0';
@@ -1543,23 +1611,22 @@
if (strlen(name) > MAXNAME) {
rv = -ENAMETOOLONG;
- goto fail;
+ goto out;
}
mg = find_mg(name);
if (mg) {
- rv = -EEXIST;
- goto fail;
+ rv = add_another_mountpoint(mg, dir, dev, ci);
+ goto out;
}
- mg = create_mg(name);
+ mg = create_mg(name, dir);
if (!mg) {
rv = -ENOMEM;
- goto fail;
+ goto out;
}
mg->mount_client = ci;
- strncpy(mg->dir, dir, sizeof(mg->dir));
strncpy(mg->type, type, sizeof(mg->type));
strncpy(mg->table, table, sizeof(mg->table));
strncpy(mg->options, options, sizeof(mg->options));
@@ -1570,7 +1637,7 @@
rv = -1;
log_error("mount: fs requires cluster=\"%s\" current=\"%s\"",
cluster, clustername);
- goto fail;
+ goto out;
} else
log_group(mg, "cluster name matches: %s", clustername);
@@ -1581,7 +1648,7 @@
if (!we_are_in_fence_domain()) {
rv = -EINVAL;
log_error("mount: not in default fence domain");
- goto fail;
+ goto out;
}
}
@@ -1591,7 +1658,7 @@
if (mg->spectator) {
rv = -EINVAL;
log_error("mount: readonly invalid with spectator");
- goto fail;
+ goto out;
}
mg->readonly = 1;
}
@@ -1599,17 +1666,15 @@
if (strlen(options) > MAX_OPTIONS_LEN-1) {
rv = -EINVAL;
log_error("mount: options too long %d", strlen(options));
- goto fail;
+ goto out;
}
list_add(&mg->list, &mounts);
- *mg_ret = mg;
-
group_join(gh, name);
- return 0;
-
- fail:
- log_error("mount: failed %d", rv);
+ rv = 0;
+ out:
+ *mg_ret = mg;
+ log_group(mg, "do_mount: rv %d", rv);
return rv;
}
@@ -1891,14 +1956,20 @@
int do_unmount(int ci, char *dir, int mnterr)
{
struct mountgroup *mg;
+ struct mountpoint *mp;
list_for_each_entry(mg, &withdrawn_mounts, list) {
- if (!strcmp(mg->dir, dir)) {
- log_group(mg, "unmount withdrawn fs");
+ mp = find_mountpoint(mg, dir);
+ if (!mp)
+ continue;
+ log_group(mg, "unmount %s for withdrawn fs", dir);
+ list_del(&mp->list);
+ free(mp);
+ if (list_empty(&mg->mountpoints)) {
list_del(&mg->list);
free(mg);
- return 0;
}
+ return 0;
}
mg = find_mg_dir(dir);
@@ -1922,10 +1993,27 @@
}
if (mg->withdraw) {
- log_error("do_unmount: fs on %s is withdrawing", dir);
+ log_error("%s do_unmount: fs on %s is withdrawing",
+ mg->name, dir);
+ return -1;
+ }
+
+ if (!mg->kernel_mount_done) {
+ log_error("%s do_unmount: fs on %s is still mounting",
+ mg->name, dir);
return -1;
}
+ mp = find_mountpoint(mg, dir);
+ ASSERT(mp);
+ list_del(&mp->list);
+ free(mp);
+
+ if (!list_empty(&mg->mountpoints)) {
+ log_group(mg, "removed mountpoint %s, more remaining", dir);
+ return 0;
+ }
+
/* Check to see if we're waiting for a kernel recovery_done to do a
start_done(). If so, call the start_done() here because we won't be
getting anything else from gfs-kernel which is now gone. */
@@ -1934,6 +2022,7 @@
log_group(mg, "do_unmount: fill in start_done");
start_done(mg);
}
+
out:
group_leave(gh, mg->name);
return 0;
@@ -1972,8 +2061,6 @@
if (rv < 0)
log_error("notify_mount_client: send failed %d", rv);
- mg->mount_client = 0;
-
if (error) {
log_group(mg, "leaving due to mount error: %s", mg->error_msg);
if (memb->finished)
@@ -2003,14 +2090,42 @@
log_group(mg, "ping_kernel_mount %d", rv);
}
-void got_mount_result(struct mountgroup *mg, int result)
+/* remove the mountpoint that this client added */
+void remove_failed_mountpoint(struct mountgroup *mg, int ci)
+{
+ struct mountpoint *mp;
+ int found = 0;
+
+ list_for_each_entry(mp, &mg->mountpoints, list) {
+ if (mp->client == ci) {
+ list_del(&mp->list);
+ free(mp);
+ found = 1;
+ break;
+ }
+ }
+ ASSERT(found);
+ ASSERT(!list_empty(&mg->mountpoints));
+}
+
+void got_mount_result(struct mountgroup *mg, int result, int ci, int another)
{
struct mg_member *memb;
memb = find_memb_nodeid(mg, our_nodeid);
- log_group(mg, "mount_result: kernel_mount_error %d first_mounter %d "
- "opts %x", result, mg->first_mounter, memb->opts);
+ log_group(mg, "got_mount_result: ci %d result %d another %d "
+ "first_mounter %d opts %x",
+ ci, result, another, mg->first_mounter, memb->opts);
+
+ mg->mount_client = 0;
+ mg->mount_client_fd = 0;
+
+ if (another) {
+ if (result)
+ remove_failed_mountpoint(mg, ci);
+ return;
+ }
mg->kernel_mount_done = 1;
mg->kernel_mount_error = result;
^ permalink raw reply [flat|nested] 7+ messages in thread
* [Cluster-devel] cluster/group/gfs_controld lock_dlm.h main.c r ...
@ 2006-12-20 19:14 teigland
0 siblings, 0 replies; 7+ messages in thread
From: teigland @ 2006-12-20 19:14 UTC (permalink / raw)
To: cluster-devel.redhat.com
CVSROOT: /cvs/cluster
Module name: cluster
Branch: RHEL5
Changes by: teigland at sourceware.org 2006-12-20 19:14:41
Modified files:
group/gfs_controld: lock_dlm.h main.c recover.c
Log message:
Support mounting a single fs on multiple mount points.
bz 218560
Patches:
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/gfs_controld/lock_dlm.h.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.21.2.5&r2=1.21.2.6
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/gfs_controld/main.c.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.18.2.9&r2=1.18.2.10
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/gfs_controld/recover.c.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.23.2.4&r2=1.23.2.5
--- cluster/group/gfs_controld/lock_dlm.h 2006/12/19 17:07:12 1.21.2.5
+++ cluster/group/gfs_controld/lock_dlm.h 2006/12/20 19:14:41 1.21.2.6
@@ -113,6 +113,12 @@
} \
}
+struct mountpoint {
+ struct list_head list;
+ char dir[PATH_MAX+1];
+ int client;
+};
+
struct mountgroup {
struct list_head list;
uint32_t id;
@@ -120,11 +126,11 @@
struct list_head members_gone;
int memb_count;
struct list_head resources; /* for plocks */
+ struct list_head mountpoints;
char name[MAXNAME+1];
char table[MAXNAME+1];
char type[5];
- char dir[PATH_MAX+1];
char options[MAX_OPTIONS_LEN+1];
char dev[PATH_MAX+1];
@@ -280,7 +286,7 @@
int kernel_recovery_done(char *name);
void ping_kernel_mount(char *table);
void save_message(struct mountgroup *mg, char *buf, int len, int from, int type);
-void got_mount_result(struct mountgroup *mg, int result);
+void got_mount_result(struct mountgroup *mg, int result, int ci, int another);
int client_send(int ci, char *buf, int len);
--- cluster/group/gfs_controld/main.c 2006/12/05 22:24:29 1.18.2.9
+++ cluster/group/gfs_controld/main.c 2006/12/20 19:14:41 1.18.2.10
@@ -21,6 +21,7 @@
int fd;
char type[32];
struct mountgroup *mg;
+ int another_mount;
};
static int client_maxi;
@@ -298,14 +299,15 @@
argv[6], &mg);
fd = client[ci].fd;
fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK);
- if (!rv) {
+ if (!rv || rv == -EALREADY) {
+ client[ci].another_mount = rv;
client[ci].mg = mg;
mg->mount_client_fd = fd;
}
goto reply;
} else if (!strcmp(cmd, "mount_result")) {
- got_mount_result(client[ci].mg, atoi(argv[3]));
-
+ got_mount_result(client[ci].mg, atoi(argv[3]), ci,
+ client[ci].another_mount);
} else if (!strcmp(cmd, "leave")) {
rv = do_unmount(ci, argv[1], atoi(argv[3]));
goto reply;
--- cluster/group/gfs_controld/recover.c 2006/12/19 22:19:59 1.23.2.4
+++ cluster/group/gfs_controld/recover.c 2006/12/20 19:14:41 1.23.2.5
@@ -1430,23 +1430,36 @@
}
}
-struct mountgroup *create_mg(char *name)
+struct mountgroup *create_mg(char *name, char *dir)
{
struct mountgroup *mg;
+ struct mountpoint *mp;
mg = malloc(sizeof(struct mountgroup));
+ if (!mg)
+ return NULL;
memset(mg, 0, sizeof(struct mountgroup));
INIT_LIST_HEAD(&mg->members);
INIT_LIST_HEAD(&mg->members_gone);
INIT_LIST_HEAD(&mg->resources);
INIT_LIST_HEAD(&mg->saved_messages);
+ INIT_LIST_HEAD(&mg->mountpoints);
mg->init = 1;
mg->master_nodeid = -1;
mg->low_nodeid = -1;
strncpy(mg->name, name, MAXNAME);
+ mp = malloc(sizeof(struct mountpoint));
+ if (!mp) {
+ free(mg);
+ return NULL;
+ }
+ memset(mp, 0, sizeof(struct mountpoint));
+ strncpy(mp->dir, dir, sizeof(mp->dir));
+ list_add(&mp->list, &mg->mountpoints);
+
return mg;
}
@@ -1473,12 +1486,23 @@
return NULL;
}
+struct mountpoint *find_mountpoint(struct mountgroup *mg, char *dir)
+{
+ struct mountpoint *mp;
+
+ list_for_each_entry(mp, &mg->mountpoints, list) {
+ if (!strcmp(mp->dir, dir))
+ return mp;
+ }
+ return NULL;
+}
+
struct mountgroup *find_mg_dir(char *dir)
{
struct mountgroup *mg;
list_for_each_entry(mg, &mounts, list) {
- if (!strcmp(mg->dir, dir))
+ if (find_mountpoint(mg, dir))
return mg;
}
return NULL;
@@ -1501,10 +1525,54 @@
return 0;
}
+int add_another_mountpoint(struct mountgroup *mg, char *dir, char *dev, int ci)
+{
+ struct mountpoint *mp;
+
+ log_group(mg, "add_another_mountpoint dir %s dev %s ci %d",
+ dir, dev, ci);
+
+ /* check if this is the same fs mounted on another dir or a different
+ fs with the same name (which is an error) */
+
+ if (strcmp(mg->dev, dev)) {
+ log_group(mg, "different fs dev %s with same name", mg->dev);
+ return -EINVAL;
+ }
+
+ if (find_mountpoint(mg, dir)) {
+ log_group(mg, "mount point %s already used", dir);
+ return -EBUSY;
+ }
+
+ /* we only really need to check one of these */
+ if (mg->mount_client || mg->mount_client_fd || !mg->kernel_mount_done) {
+ log_group(mg, "other mount in progress client %d fd %d done %d",
+ mg->mount_client, mg->mount_client_fd,
+ mg->kernel_mount_done);
+ return -EBUSY;
+ }
+
+ mp = malloc(sizeof(struct mountpoint));
+ if (!mp)
+ return -ENOMEM;
+
+ memset(mp, 0, sizeof(struct mountpoint));
+ strncpy(mp->dir, dir, sizeof(mp->dir));
+ list_add(&mp->list, &mg->mountpoints);
+ mg->mount_client = ci;
+
+ /* we return this special error to mount.gfs which mount.gfs will
+ recognize as meaning the fs is already mounted, so it shouldn't
+ read any hostdata from us, but just go ahead and mount(2) */
+
+ return -EALREADY;
+}
+
int do_mount(int ci, char *dir, char *type, char *proto, char *table,
char *options, char *dev, struct mountgroup **mg_ret)
{
- struct mountgroup *mg;
+ struct mountgroup *mg = NULL;
char table2[MAXLINE];
char *cluster = NULL, *name = NULL;
int rv;
@@ -1515,7 +1583,7 @@
if (strcmp(proto, "lock_dlm")) {
log_error("mount: lockproto %s not supported", proto);
rv = -EINVAL;
- goto fail;
+ goto out;
}
if (strstr(options, "jid=") ||
@@ -1523,7 +1591,7 @@
strstr(options, "id=")) {
log_error("mount: jid, first and id are reserved options");
rv = -EINVAL;
- goto fail;
+ goto out;
}
/* table is <cluster>:<name> */
@@ -1534,7 +1602,7 @@
name = strstr(table2, ":");
if (!name) {
rv = -EINVAL;
- goto fail;
+ goto out;
}
*name = '\0';
@@ -1543,23 +1611,22 @@
if (strlen(name) > MAXNAME) {
rv = -ENAMETOOLONG;
- goto fail;
+ goto out;
}
mg = find_mg(name);
if (mg) {
- rv = -EEXIST;
- goto fail;
+ rv = add_another_mountpoint(mg, dir, dev, ci);
+ goto out;
}
- mg = create_mg(name);
+ mg = create_mg(name, dir);
if (!mg) {
rv = -ENOMEM;
- goto fail;
+ goto out;
}
mg->mount_client = ci;
- strncpy(mg->dir, dir, sizeof(mg->dir));
strncpy(mg->type, type, sizeof(mg->type));
strncpy(mg->table, table, sizeof(mg->table));
strncpy(mg->options, options, sizeof(mg->options));
@@ -1570,7 +1637,7 @@
rv = -1;
log_error("mount: fs requires cluster=\"%s\" current=\"%s\"",
cluster, clustername);
- goto fail;
+ goto out;
} else
log_group(mg, "cluster name matches: %s", clustername);
@@ -1581,7 +1648,7 @@
if (!we_are_in_fence_domain()) {
rv = -EINVAL;
log_error("mount: not in default fence domain");
- goto fail;
+ goto out;
}
}
@@ -1591,7 +1658,7 @@
if (mg->spectator) {
rv = -EINVAL;
log_error("mount: readonly invalid with spectator");
- goto fail;
+ goto out;
}
mg->readonly = 1;
}
@@ -1599,17 +1666,15 @@
if (strlen(options) > MAX_OPTIONS_LEN-1) {
rv = -EINVAL;
log_error("mount: options too long %d", strlen(options));
- goto fail;
+ goto out;
}
list_add(&mg->list, &mounts);
- *mg_ret = mg;
-
group_join(gh, name);
- return 0;
-
- fail:
- log_error("mount: failed %d", rv);
+ rv = 0;
+ out:
+ *mg_ret = mg;
+ log_group(mg, "do_mount: rv %d", rv);
return rv;
}
@@ -1891,14 +1956,20 @@
int do_unmount(int ci, char *dir, int mnterr)
{
struct mountgroup *mg;
+ struct mountpoint *mp;
list_for_each_entry(mg, &withdrawn_mounts, list) {
- if (!strcmp(mg->dir, dir)) {
- log_group(mg, "unmount withdrawn fs");
+ mp = find_mountpoint(mg, dir);
+ if (!mp)
+ continue;
+ log_group(mg, "unmount %s for withdrawn fs", dir);
+ list_del(&mp->list);
+ free(mp);
+ if (list_empty(&mg->mountpoints)) {
list_del(&mg->list);
free(mg);
- return 0;
}
+ return 0;
}
mg = find_mg_dir(dir);
@@ -1922,10 +1993,27 @@
}
if (mg->withdraw) {
- log_error("do_unmount: fs on %s is withdrawing", dir);
+ log_error("%s do_unmount: fs on %s is withdrawing",
+ mg->name, dir);
+ return -1;
+ }
+
+ if (!mg->kernel_mount_done) {
+ log_error("%s do_unmount: fs on %s is still mounting",
+ mg->name, dir);
return -1;
}
+ mp = find_mountpoint(mg, dir);
+ ASSERT(mp);
+ list_del(&mp->list);
+ free(mp);
+
+ if (!list_empty(&mg->mountpoints)) {
+ log_group(mg, "removed mountpoint %s, more remaining", dir);
+ return 0;
+ }
+
/* Check to see if we're waiting for a kernel recovery_done to do a
start_done(). If so, call the start_done() here because we won't be
getting anything else from gfs-kernel which is now gone. */
@@ -1934,6 +2022,7 @@
log_group(mg, "do_unmount: fill in start_done");
start_done(mg);
}
+
out:
group_leave(gh, mg->name);
return 0;
@@ -1972,8 +2061,6 @@
if (rv < 0)
log_error("notify_mount_client: send failed %d", rv);
- mg->mount_client = 0;
-
if (error) {
log_group(mg, "leaving due to mount error: %s", mg->error_msg);
if (memb->finished)
@@ -2003,14 +2090,42 @@
log_group(mg, "ping_kernel_mount %d", rv);
}
-void got_mount_result(struct mountgroup *mg, int result)
+/* remove the mountpoint that this client added */
+void remove_failed_mountpoint(struct mountgroup *mg, int ci)
+{
+ struct mountpoint *mp;
+ int found = 0;
+
+ list_for_each_entry(mp, &mg->mountpoints, list) {
+ if (mp->client == ci) {
+ list_del(&mp->list);
+ free(mp);
+ found = 1;
+ break;
+ }
+ }
+ ASSERT(found);
+ ASSERT(!list_empty(&mg->mountpoints));
+}
+
+void got_mount_result(struct mountgroup *mg, int result, int ci, int another)
{
struct mg_member *memb;
memb = find_memb_nodeid(mg, our_nodeid);
- log_group(mg, "mount_result: kernel_mount_error %d first_mounter %d "
- "opts %x", result, mg->first_mounter, memb->opts);
+ log_group(mg, "got_mount_result: ci %d result %d another %d "
+ "first_mounter %d opts %x",
+ ci, result, another, mg->first_mounter, memb->opts);
+
+ mg->mount_client = 0;
+ mg->mount_client_fd = 0;
+
+ if (another) {
+ if (result)
+ remove_failed_mountpoint(mg, ci);
+ return;
+ }
mg->kernel_mount_done = 1;
mg->kernel_mount_error = result;
^ permalink raw reply [flat|nested] 7+ messages in thread
* [Cluster-devel] cluster/group/gfs_controld lock_dlm.h main.c r ...
@ 2006-12-20 19:16 teigland
0 siblings, 0 replies; 7+ messages in thread
From: teigland @ 2006-12-20 19:16 UTC (permalink / raw)
To: cluster-devel.redhat.com
CVSROOT: /cvs/cluster
Module name: cluster
Branch: RHEL50
Changes by: teigland at sourceware.org 2006-12-20 19:16:21
Modified files:
group/gfs_controld: lock_dlm.h main.c recover.c
Log message:
Support mounting a single fs on multiple mount points.
bz 218560
Patches:
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/gfs_controld/lock_dlm.h.diff?cvsroot=cluster&only_with_tag=RHEL50&r1=1.21.4.5&r2=1.21.4.6
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/gfs_controld/main.c.diff?cvsroot=cluster&only_with_tag=RHEL50&r1=1.18.4.8&r2=1.18.4.9
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/gfs_controld/recover.c.diff?cvsroot=cluster&only_with_tag=RHEL50&r1=1.23.4.4&r2=1.23.4.5
--- cluster/group/gfs_controld/lock_dlm.h 2006/12/19 17:07:22 1.21.4.5
+++ cluster/group/gfs_controld/lock_dlm.h 2006/12/20 19:16:21 1.21.4.6
@@ -113,6 +113,12 @@
} \
}
+struct mountpoint {
+ struct list_head list;
+ char dir[PATH_MAX+1];
+ int client;
+};
+
struct mountgroup {
struct list_head list;
uint32_t id;
@@ -120,11 +126,11 @@
struct list_head members_gone;
int memb_count;
struct list_head resources; /* for plocks */
+ struct list_head mountpoints;
char name[MAXNAME+1];
char table[MAXNAME+1];
char type[5];
- char dir[PATH_MAX+1];
char options[MAX_OPTIONS_LEN+1];
char dev[PATH_MAX+1];
@@ -280,7 +286,7 @@
int kernel_recovery_done(char *name);
void ping_kernel_mount(char *table);
void save_message(struct mountgroup *mg, char *buf, int len, int from, int type);
-void got_mount_result(struct mountgroup *mg, int result);
+void got_mount_result(struct mountgroup *mg, int result, int ci, int another);
int client_send(int ci, char *buf, int len);
--- cluster/group/gfs_controld/main.c 2006/12/05 22:24:37 1.18.4.8
+++ cluster/group/gfs_controld/main.c 2006/12/20 19:16:21 1.18.4.9
@@ -21,6 +21,7 @@
int fd;
char type[32];
struct mountgroup *mg;
+ int another_mount;
};
static int client_maxi;
@@ -298,14 +299,15 @@
argv[6], &mg);
fd = client[ci].fd;
fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK);
- if (!rv) {
+ if (!rv || rv == -EALREADY) {
+ client[ci].another_mount = rv;
client[ci].mg = mg;
mg->mount_client_fd = fd;
}
goto reply;
} else if (!strcmp(cmd, "mount_result")) {
- got_mount_result(client[ci].mg, atoi(argv[3]));
-
+ got_mount_result(client[ci].mg, atoi(argv[3]), ci,
+ client[ci].another_mount);
} else if (!strcmp(cmd, "leave")) {
rv = do_unmount(ci, argv[1], atoi(argv[3]));
goto reply;
--- cluster/group/gfs_controld/recover.c 2006/12/19 22:20:08 1.23.4.4
+++ cluster/group/gfs_controld/recover.c 2006/12/20 19:16:21 1.23.4.5
@@ -1430,23 +1430,36 @@
}
}
-struct mountgroup *create_mg(char *name)
+struct mountgroup *create_mg(char *name, char *dir)
{
struct mountgroup *mg;
+ struct mountpoint *mp;
mg = malloc(sizeof(struct mountgroup));
+ if (!mg)
+ return NULL;
memset(mg, 0, sizeof(struct mountgroup));
INIT_LIST_HEAD(&mg->members);
INIT_LIST_HEAD(&mg->members_gone);
INIT_LIST_HEAD(&mg->resources);
INIT_LIST_HEAD(&mg->saved_messages);
+ INIT_LIST_HEAD(&mg->mountpoints);
mg->init = 1;
mg->master_nodeid = -1;
mg->low_nodeid = -1;
strncpy(mg->name, name, MAXNAME);
+ mp = malloc(sizeof(struct mountpoint));
+ if (!mp) {
+ free(mg);
+ return NULL;
+ }
+ memset(mp, 0, sizeof(struct mountpoint));
+ strncpy(mp->dir, dir, sizeof(mp->dir));
+ list_add(&mp->list, &mg->mountpoints);
+
return mg;
}
@@ -1473,12 +1486,23 @@
return NULL;
}
+struct mountpoint *find_mountpoint(struct mountgroup *mg, char *dir)
+{
+ struct mountpoint *mp;
+
+ list_for_each_entry(mp, &mg->mountpoints, list) {
+ if (!strcmp(mp->dir, dir))
+ return mp;
+ }
+ return NULL;
+}
+
struct mountgroup *find_mg_dir(char *dir)
{
struct mountgroup *mg;
list_for_each_entry(mg, &mounts, list) {
- if (!strcmp(mg->dir, dir))
+ if (find_mountpoint(mg, dir))
return mg;
}
return NULL;
@@ -1501,10 +1525,54 @@
return 0;
}
+int add_another_mountpoint(struct mountgroup *mg, char *dir, char *dev, int ci)
+{
+ struct mountpoint *mp;
+
+ log_group(mg, "add_another_mountpoint dir %s dev %s ci %d",
+ dir, dev, ci);
+
+ /* check if this is the same fs mounted on another dir or a different
+ fs with the same name (which is an error) */
+
+ if (strcmp(mg->dev, dev)) {
+ log_group(mg, "different fs dev %s with same name", mg->dev);
+ return -EINVAL;
+ }
+
+ if (find_mountpoint(mg, dir)) {
+ log_group(mg, "mount point %s already used", dir);
+ return -EBUSY;
+ }
+
+ /* we only really need to check one of these */
+ if (mg->mount_client || mg->mount_client_fd || !mg->kernel_mount_done) {
+ log_group(mg, "other mount in progress client %d fd %d done %d",
+ mg->mount_client, mg->mount_client_fd,
+ mg->kernel_mount_done);
+ return -EBUSY;
+ }
+
+ mp = malloc(sizeof(struct mountpoint));
+ if (!mp)
+ return -ENOMEM;
+
+ memset(mp, 0, sizeof(struct mountpoint));
+ strncpy(mp->dir, dir, sizeof(mp->dir));
+ list_add(&mp->list, &mg->mountpoints);
+ mg->mount_client = ci;
+
+ /* we return this special error to mount.gfs which mount.gfs will
+ recognize as meaning the fs is already mounted, so it shouldn't
+ read any hostdata from us, but just go ahead and mount(2) */
+
+ return -EALREADY;
+}
+
int do_mount(int ci, char *dir, char *type, char *proto, char *table,
char *options, char *dev, struct mountgroup **mg_ret)
{
- struct mountgroup *mg;
+ struct mountgroup *mg = NULL;
char table2[MAXLINE];
char *cluster = NULL, *name = NULL;
int rv;
@@ -1515,7 +1583,7 @@
if (strcmp(proto, "lock_dlm")) {
log_error("mount: lockproto %s not supported", proto);
rv = -EINVAL;
- goto fail;
+ goto out;
}
if (strstr(options, "jid=") ||
@@ -1523,7 +1591,7 @@
strstr(options, "id=")) {
log_error("mount: jid, first and id are reserved options");
rv = -EINVAL;
- goto fail;
+ goto out;
}
/* table is <cluster>:<name> */
@@ -1534,7 +1602,7 @@
name = strstr(table2, ":");
if (!name) {
rv = -EINVAL;
- goto fail;
+ goto out;
}
*name = '\0';
@@ -1543,23 +1611,22 @@
if (strlen(name) > MAXNAME) {
rv = -ENAMETOOLONG;
- goto fail;
+ goto out;
}
mg = find_mg(name);
if (mg) {
- rv = -EEXIST;
- goto fail;
+ rv = add_another_mountpoint(mg, dir, dev, ci);
+ goto out;
}
- mg = create_mg(name);
+ mg = create_mg(name, dir);
if (!mg) {
rv = -ENOMEM;
- goto fail;
+ goto out;
}
mg->mount_client = ci;
- strncpy(mg->dir, dir, sizeof(mg->dir));
strncpy(mg->type, type, sizeof(mg->type));
strncpy(mg->table, table, sizeof(mg->table));
strncpy(mg->options, options, sizeof(mg->options));
@@ -1570,7 +1637,7 @@
rv = -1;
log_error("mount: fs requires cluster=\"%s\" current=\"%s\"",
cluster, clustername);
- goto fail;
+ goto out;
} else
log_group(mg, "cluster name matches: %s", clustername);
@@ -1581,7 +1648,7 @@
if (!we_are_in_fence_domain()) {
rv = -EINVAL;
log_error("mount: not in default fence domain");
- goto fail;
+ goto out;
}
}
@@ -1591,7 +1658,7 @@
if (mg->spectator) {
rv = -EINVAL;
log_error("mount: readonly invalid with spectator");
- goto fail;
+ goto out;
}
mg->readonly = 1;
}
@@ -1599,17 +1666,15 @@
if (strlen(options) > MAX_OPTIONS_LEN-1) {
rv = -EINVAL;
log_error("mount: options too long %d", strlen(options));
- goto fail;
+ goto out;
}
list_add(&mg->list, &mounts);
- *mg_ret = mg;
-
group_join(gh, name);
- return 0;
-
- fail:
- log_error("mount: failed %d", rv);
+ rv = 0;
+ out:
+ *mg_ret = mg;
+ log_group(mg, "do_mount: rv %d", rv);
return rv;
}
@@ -1891,14 +1956,20 @@
int do_unmount(int ci, char *dir, int mnterr)
{
struct mountgroup *mg;
+ struct mountpoint *mp;
list_for_each_entry(mg, &withdrawn_mounts, list) {
- if (!strcmp(mg->dir, dir)) {
- log_group(mg, "unmount withdrawn fs");
+ mp = find_mountpoint(mg, dir);
+ if (!mp)
+ continue;
+ log_group(mg, "unmount %s for withdrawn fs", dir);
+ list_del(&mp->list);
+ free(mp);
+ if (list_empty(&mg->mountpoints)) {
list_del(&mg->list);
free(mg);
- return 0;
}
+ return 0;
}
mg = find_mg_dir(dir);
@@ -1922,10 +1993,27 @@
}
if (mg->withdraw) {
- log_error("do_unmount: fs on %s is withdrawing", dir);
+ log_error("%s do_unmount: fs on %s is withdrawing",
+ mg->name, dir);
+ return -1;
+ }
+
+ if (!mg->kernel_mount_done) {
+ log_error("%s do_unmount: fs on %s is still mounting",
+ mg->name, dir);
return -1;
}
+ mp = find_mountpoint(mg, dir);
+ ASSERT(mp);
+ list_del(&mp->list);
+ free(mp);
+
+ if (!list_empty(&mg->mountpoints)) {
+ log_group(mg, "removed mountpoint %s, more remaining", dir);
+ return 0;
+ }
+
/* Check to see if we're waiting for a kernel recovery_done to do a
start_done(). If so, call the start_done() here because we won't be
getting anything else from gfs-kernel which is now gone. */
@@ -1934,6 +2022,7 @@
log_group(mg, "do_unmount: fill in start_done");
start_done(mg);
}
+
out:
group_leave(gh, mg->name);
return 0;
@@ -1972,8 +2061,6 @@
if (rv < 0)
log_error("notify_mount_client: send failed %d", rv);
- mg->mount_client = 0;
-
if (error) {
log_group(mg, "leaving due to mount error: %s", mg->error_msg);
if (memb->finished)
@@ -2003,14 +2090,42 @@
log_group(mg, "ping_kernel_mount %d", rv);
}
-void got_mount_result(struct mountgroup *mg, int result)
+/* remove the mountpoint that this client added */
+void remove_failed_mountpoint(struct mountgroup *mg, int ci)
+{
+ struct mountpoint *mp;
+ int found = 0;
+
+ list_for_each_entry(mp, &mg->mountpoints, list) {
+ if (mp->client == ci) {
+ list_del(&mp->list);
+ free(mp);
+ found = 1;
+ break;
+ }
+ }
+ ASSERT(found);
+ ASSERT(!list_empty(&mg->mountpoints));
+}
+
+void got_mount_result(struct mountgroup *mg, int result, int ci, int another)
{
struct mg_member *memb;
memb = find_memb_nodeid(mg, our_nodeid);
- log_group(mg, "mount_result: kernel_mount_error %d first_mounter %d "
- "opts %x", result, mg->first_mounter, memb->opts);
+ log_group(mg, "got_mount_result: ci %d result %d another %d "
+ "first_mounter %d opts %x",
+ ci, result, another, mg->first_mounter, memb->opts);
+
+ mg->mount_client = 0;
+ mg->mount_client_fd = 0;
+
+ if (another) {
+ if (result)
+ remove_failed_mountpoint(mg, ci);
+ return;
+ }
mg->kernel_mount_done = 1;
mg->kernel_mount_error = result;
^ permalink raw reply [flat|nested] 7+ messages in thread
end of thread, other threads:[~2006-12-20 19:16 UTC | newest]
Thread overview: 7+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2006-12-20 19:14 [Cluster-devel] cluster/group/gfs_controld lock_dlm.h main.c r teigland
-- strict thread matches above, loose matches on Subject: below --
2006-12-20 19:16 teigland
2006-12-20 19:13 teigland
2006-12-05 22:24 teigland
2006-12-05 22:24 teigland
2006-12-05 22:19 teigland
2006-07-20 20:19 teigland
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.