[Cluster-devel] [PATCHv2 dlm-tool 0/4] dlm_controld: support for mark and waitplock_recovery

[Cluster-devel] [PATCHv2 dlm-tool 0/4] dlm_controld: support for mark and waitplock_recovery

[Alexander Aring]

Hi,

this patch series adds support for set the in-kernel socket skb mark
value over dlm_controld. There exists two kinds of socket, one listen
socket and multiple peer sockets. Both can be set via the dlm config
file via "listen_mark" or multiple entries of:

node id=$NODEID mark=$MARK

whereas $NODEID is the corosync assigned nodeid. The given mark number
can be hexadecimal or decimal.

Also it adds support to set the waitplock_recovery per cluster attribute
by setting enable_waitplock_recover over file or argument configuration.

- Alex

changes since v2:
 - remove leftover PRIu32 in nodeid configuration
 - make unsigned int values also work with file configuration, was arg
   only before
 - add support to set the waitplock_recovery switch via dlm_controld
 - remove free function of node_config, may be necessary when implement
   some kind of NOHUP and reparse config file

Alexander Aring (4):
  dlm_controld: add support for unsigned int values
  dlm_controld: set listen skb mark setting
  dlm_controld: add support for per nodeid configuration
  dlm_controld: add support for waitplock_recovery switch

 dlm_controld/Makefile      |  3 +-
 dlm_controld/action.c      | 38 ++++++++++++++++--
 dlm_controld/config.c      | 25 ++++++++++++
 dlm_controld/dlm.conf.5    | 23 +++++++++++
 dlm_controld/dlm_daemon.h  | 12 +++++-
 dlm_controld/main.c        | 17 ++++++++
 dlm_controld/member.c      |  6 ++-
 dlm_controld/node_config.c | 82 ++++++++++++++++++++++++++++++++++++++
 dlm_controld/node_config.h | 31 ++++++++++++++
 9 files changed, 231 insertions(+), 6 deletions(-)
 create mode 100644 dlm_controld/node_config.c
 create mode 100644 dlm_controld/node_config.h

-- 
2.26.2

[Cluster-devel] [PATCHv2 dlm-tool 1/4] dlm_controld: add support for unsigned int values

[Alexander Aring]

This patch adds support for setting a unsigned integer value.
---
 dlm_controld/config.c     | 25 +++++++++++++++++++++++++
 dlm_controld/dlm_daemon.h |  5 +++++
 dlm_controld/main.c       |  3 +++
 3 files changed, 33 insertions(+)

diff --git a/dlm_controld/config.c b/dlm_controld/config.c
index 3ba8a53b..ec269168 100644
--- a/dlm_controld/config.c
+++ b/dlm_controld/config.c
@@ -156,6 +156,19 @@ static void get_val_int(char *line, int *val_out)
 	*val_out = atoi(val);
 }
 
+static void get_val_uint(char *line, unsigned int *val_out)
+{
+	char key[MAX_LINE];
+	char val[MAX_LINE];
+	int rv;
+
+	rv = sscanf(line, "%[^=]=%s", key, val);
+	if (rv != 2)
+		return;
+
+	*val_out = strtoul(val, NULL, 0);
+}
+
 static void get_val_str(char *line, char *val_out)
 {
 	char key[MAX_LINE];
@@ -171,6 +184,7 @@ static void get_val_str(char *line, char *val_out)
 
 void set_opt_file(int update)
 {
+	unsigned int uval = 0;
 	struct dlm_option *o;
 	FILE *file;
 	char line[MAX_LINE];
@@ -238,6 +252,17 @@ void set_opt_file(int update)
 			log_debug("config file %s = %d cli_set %d use %d",
 				  o->name, o->file_int, o->cli_set, o->use_int);
 
+		} else if (o->req_arg == req_arg_uint) {
+			get_val_uint(line, &uval);
+
+			o->file_uint = uval;
+
+			if (!o->cli_set)
+				o->use_uint = o->file_uint;
+
+			log_debug("config file %s = %u cli_set %d use %u",
+				  o->name, o->file_uint, o->cli_set, o->use_uint);
+
 		} else if (o->req_arg == req_arg_bool) {
 			get_val_int(line, &val);
 
diff --git a/dlm_controld/dlm_daemon.h b/dlm_controld/dlm_daemon.h
index 5b9a52da..3dad0bf1 100644
--- a/dlm_controld/dlm_daemon.h
+++ b/dlm_controld/dlm_daemon.h
@@ -86,6 +86,7 @@ enum {
         req_arg_bool = 1,
         req_arg_int = 2,
         req_arg_str = 3,
+        req_arg_uint = 4,
 };
 
 enum {
@@ -125,6 +126,7 @@ struct dlm_option {
 
 	int use_int;
 	char *use_str;
+	unsigned int use_uint;
 
 	int default_int;
 	const char *default_str;
@@ -132,15 +134,18 @@ struct dlm_option {
 	int cli_set;
 	int cli_int;
 	char *cli_str;
+	unsigned int cli_uint;
 
 	int file_set;
 	int file_int;
 	char *file_str;
+	unsigned int file_uint;
 };
 
 EXTERN struct dlm_option dlm_options[dlm_options_max];
 #define opt(x) dlm_options[x].use_int
 #define opts(x) dlm_options[x].use_str
+#define optu(x) dlm_options[x].use_uint
 
 
 /* DLM_LOCKSPACE_LEN: maximum lockspace name length, from linux/dlmconstants.h.
diff --git a/dlm_controld/main.c b/dlm_controld/main.c
index 8be6a4bc..b4f4ffb8 100644
--- a/dlm_controld/main.c
+++ b/dlm_controld/main.c
@@ -1972,6 +1972,9 @@ static void set_opt_cli(int argc, char **argv)
 		} else if (o->req_arg == req_arg_bool) {
 			o->cli_int = atoi(arg_str) ? 1 : 0;
 			o->use_int = o->cli_int;
+		} else if (o->req_arg == req_arg_uint) {
+			o->cli_uint = strtoul(arg_str, NULL, 0);
+			o->use_uint = o->cli_uint;
 		}
 	}
 
-- 
2.26.2

[Cluster-devel] [PATCHv2 dlm-tool 2/4] dlm_controld: set listen skb mark setting

[Alexander Aring]

This patch adds support to set the skb mark value for the in-kernel DLM
listen socket.
---
 dlm_controld/action.c     | 2 ++
 dlm_controld/dlm.conf.5   | 2 ++
 dlm_controld/dlm_daemon.h | 1 +
 dlm_controld/main.c       | 5 +++++
 4 files changed, 10 insertions(+)

diff --git a/dlm_controld/action.c b/dlm_controld/action.c
index ecd0d022..e901d555 100644
--- a/dlm_controld/action.c
+++ b/dlm_controld/action.c
@@ -851,6 +851,8 @@ int setup_configfs_options(void)
 	    dlm_options[timewarn_ind].file_set)
 		set_configfs_cluster("timewarn_cs", NULL, opt(timewarn_ind));
 
+	set_configfs_cluster("mark", NULL, optu(mark_ind));
+
 	proto_name = opts(protocol_ind);
 	proto_num = -1;
 
diff --git a/dlm_controld/dlm.conf.5 b/dlm_controld/dlm.conf.5
index 09492176..771951d4 100644
--- a/dlm_controld/dlm.conf.5
+++ b/dlm_controld/dlm.conf.5
@@ -40,6 +40,8 @@ protocol
 .br
 bind_all
 .br
+mark
+.br
 debug_logfile
 .br
 enable_plock
diff --git a/dlm_controld/dlm_daemon.h b/dlm_controld/dlm_daemon.h
index 3dad0bf1..8816ca75 100644
--- a/dlm_controld/dlm_daemon.h
+++ b/dlm_controld/dlm_daemon.h
@@ -97,6 +97,7 @@ enum {
         protocol_ind,
         debug_logfile_ind,
 	bind_all_ind,
+        mark_ind,
         enable_fscontrol_ind,
         enable_plock_ind,
         plock_debug_ind,
diff --git a/dlm_controld/main.c b/dlm_controld/main.c
index b4f4ffb8..022a6c7c 100644
--- a/dlm_controld/main.c
+++ b/dlm_controld/main.c
@@ -1732,6 +1732,11 @@ static void set_opt_defaults(void)
 			0, NULL,
 			""); /* do not advertise */
 
+	set_opt_default(mark_ind,
+			"mark", '\0', req_arg_uint,
+			0, NULL,
+			"set mark value for the DLM in-kernel listen socket");
+
 	set_opt_default(debug_logfile_ind,
 			"debug_logfile", 'L', no_arg,
 			0, NULL,
-- 
2.26.2

[Cluster-devel] [PATCHv2 dlm-tool 3/4] dlm_controld: add support for per nodeid configuration

[Alexander Aring]

This patch adds support to make a configuration per nodeid and key-value
pairs. As example this patch will introduce the key mark to set via configfs
comms per nodeid the SO_MARK socket option.
---
 dlm_controld/Makefile      |  3 +-
 dlm_controld/action.c      | 31 ++++++++++++--
 dlm_controld/dlm.conf.5    | 19 +++++++++
 dlm_controld/dlm_daemon.h  |  5 ++-
 dlm_controld/main.c        |  4 ++
 dlm_controld/member.c      |  6 ++-
 dlm_controld/node_config.c | 82 ++++++++++++++++++++++++++++++++++++++
 dlm_controld/node_config.h | 31 ++++++++++++++
 8 files changed, 175 insertions(+), 6 deletions(-)
 create mode 100644 dlm_controld/node_config.c
 create mode 100644 dlm_controld/node_config.h

diff --git a/dlm_controld/Makefile b/dlm_controld/Makefile
index 6081cf8b..fbc8926c 100644
--- a/dlm_controld/Makefile
+++ b/dlm_controld/Makefile
@@ -32,7 +32,8 @@ BIN_SOURCE = action.c \
              config.c \
              member.c \
              logging.c \
-             rbtree.c
+             rbtree.c \
+             node_config.c
 LIB_SOURCE = lib.c
 
 CFLAGS += -D_GNU_SOURCE -O2 -ggdb \
diff --git a/dlm_controld/action.c b/dlm_controld/action.c
index e901d555..126e3b62 100644
--- a/dlm_controld/action.c
+++ b/dlm_controld/action.c
@@ -570,15 +570,16 @@ static int add_configfs_base(void)
 	return rv;
 }
 
-int add_configfs_node(int nodeid, char *addr, int addrlen, int local)
+int add_configfs_node(int nodeid, char *addr, int addrlen, int local,
+		      uint32_t mark)
 {
 	char path[PATH_MAX];
 	char padded_addr[sizeof(struct sockaddr_storage)];
 	char buf[32];
 	int rv, fd;
 
-	log_debug("set_configfs_node %d %s local %d",
-		  nodeid, str_ip(addr), local);
+	log_debug("set_configfs_node %d %s local %d mark %" PRIu32,
+		  nodeid, str_ip(addr), local, mark);
 
 	/*
 	 * create comm dir for this node
@@ -639,6 +640,30 @@ int add_configfs_node(int nodeid, char *addr, int addrlen, int local)
 	}
 	close(fd);
 
+	/*
+	 * set skb mark for nodeid
+	 */
+
+	memset(path, 0, PATH_MAX);
+	snprintf(path, PATH_MAX, "%s/%d/mark", COMMS_DIR, nodeid);
+
+	fd = open(path, O_WRONLY);
+	if (fd < 0) {
+		log_error("%s: open failed: %d", path, errno);
+		return -1;
+	}
+
+	memset(buf, 0, sizeof(buf));
+	snprintf(buf, 32, "%" PRIu32, mark);
+
+	rv = do_write(fd, buf, strlen(buf));
+	if (rv < 0) {
+		log_error("%s: write failed: %d, %s", path, errno, buf);
+		close(fd);
+		return -1;
+	}
+	close(fd);
+
 	/*
 	 * set local
 	 */
diff --git a/dlm_controld/dlm.conf.5 b/dlm_controld/dlm.conf.5
index 771951d4..1ce0c644 100644
--- a/dlm_controld/dlm.conf.5
+++ b/dlm_controld/dlm.conf.5
@@ -392,6 +392,25 @@ master    foo node=2 weight=1
 In which case node 1 will master 2/3 of the total resources and node 2
 will master the other 1/3.
 
+.SS Node configuration
+
+Node configurations can be set by the node keyword followed of key-value
+pairs.
+
+.B Keys:
+
+.B mark
+The mark key can be used to set a specific mark value which is then used
+by the in-kernel DLM socket creation. This can be used to match for DLM
+specfic packets for e.g. routing.
+
+Example of setting a per socket value for nodeid 1 and a mark value
+of 42:
+
+node id=1 mark=42
+
+For local nodes this value doesn't have any effect.
+
 .SH SEE ALSO
 .BR dlm_controld (8),
 .BR dlm_tool (8)
diff --git a/dlm_controld/dlm_daemon.h b/dlm_controld/dlm_daemon.h
index 8816ca75..9e7a5fbf 100644
--- a/dlm_controld/dlm_daemon.h
+++ b/dlm_controld/dlm_daemon.h
@@ -40,6 +40,7 @@
 #include <sched.h>
 #include <signal.h>
 #include <dirent.h>
+#include <inttypes.h>
 #include <sys/sysmacros.h>
 
 #include <corosync/cpg.h>
@@ -48,6 +49,7 @@
 #include "libdlmcontrol.h"
 #include "dlm_controld.h"
 #include "fence_config.h"
+#include "node_config.h"
 #include "list.h"
 #include "rbtree.h"
 #include "linux_endian.h"
@@ -364,7 +366,8 @@ int set_sysfs_nodir(char *name, int val);
 int set_configfs_members(struct lockspace *ls, char *name,
 			 int new_count, int *new_members,
 			 int renew_count, int *renew_members);
-int add_configfs_node(int nodeid, char *addr, int addrlen, int local);
+int add_configfs_node(int nodeid, char *addr, int addrlen, int local,
+		      uint32_t mark);
 void del_configfs_node(int nodeid);
 void clear_configfs(void);
 int setup_configfs_options(void);
diff --git a/dlm_controld/main.c b/dlm_controld/main.c
index 022a6c7c..b330f88d 100644
--- a/dlm_controld/main.c
+++ b/dlm_controld/main.c
@@ -2047,6 +2047,10 @@ int main(int argc, char **argv)
 	set_opt_cli(argc, argv);
 	set_opt_file(0);
 
+	rv = node_config_init(CONF_FILE_PATH);
+	if (rv)
+		return 1;
+
 	strcpy(fence_all_device.name, "fence_all");
 	strcpy(fence_all_device.agent, "dlm_stonith");
 	fence_all_device.unfence = 0;
diff --git a/dlm_controld/member.c b/dlm_controld/member.c
index da3a1f5b..1d5bfa3d 100644
--- a/dlm_controld/member.c
+++ b/dlm_controld/member.c
@@ -109,6 +109,7 @@ static void quorum_callback(quorum_handle_t h, uint32_t quorate,
 {
 	corosync_cfg_node_address_t addrs[MAX_NODE_ADDRESSES];
 	corosync_cfg_node_address_t *addrptr = addrs;
+	const struct node_config *nc;
 	cs_error_t err;
 	int i, j, num_addrs;
 	uint64_t now = monotime();
@@ -163,12 +164,15 @@ static void quorum_callback(quorum_handle_t h, uint32_t quorate,
 				continue;
 			}
 
+			nc = node_config_get(quorum_nodes[i]);
+
 			for (j = 0; j < num_addrs; j++) {
 				add_configfs_node(quorum_nodes[i],
 						  addrptr[j].address,
 						  addrptr[j].address_length,
 						  (quorum_nodes[i] ==
-						   our_nodeid));
+						   our_nodeid),
+						  nc->mark);
 			}
 		}
 	}
diff --git a/dlm_controld/node_config.c b/dlm_controld/node_config.c
new file mode 100644
index 00000000..fe794be7
--- /dev/null
+++ b/dlm_controld/node_config.c
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2020 Red Hat, Inc.
+ *
+ * This copyrighted material is made available to anyone wishing to use,
+ * modify, copy, or redistribute it subject to the terms and conditions
+ * of the GNU General Public License v2 or (at your option) any later version.
+ */
+
+#include "dlm_daemon.h"
+
+#define MAX_LINE 4096
+
+static struct node_config nc[MAX_NODES];
+
+static const struct node_config nc_default = {
+	.mark = 0,
+};
+
+int node_config_init(const char *path)
+{
+	char line[MAX_LINE], tmp[MAX_LINE];
+	unsigned long mark;
+	FILE *file;
+	int nodeid;
+	int rv;
+
+	/* if no config file is given we assume default node configuration */
+	file = fopen(path, "r");
+	if (!file) {
+		log_debug("No config file %s, we assume default node configuration: mark %" PRIu32,
+			  path, nc_default.mark);
+		return 0;
+	}
+
+	while (fgets(line, MAX_LINE, file)) {
+		if (line[0] == '#')
+			continue;
+		if (line[0] == '\n')
+			continue;
+
+		if (!strncmp(line, "node", strlen("node"))) {
+			rv = sscanf(line, "node id=%d mark=%s", &nodeid, tmp);
+			if (rv < 2) {
+				log_error("Invalid configuration line: %s", line);
+				rv = -EINVAL;
+				goto out;
+			}
+
+			/* skip invalid nodeid's */
+			if (nodeid <= 0 || nodeid > MAX_NODES - 1)
+				continue;
+
+			mark = strtoul(tmp, NULL, 0);
+			if (mark == ULONG_MAX) {
+				log_error("Failed to pars mark value %s will use %" PRIu32,
+					  tmp, nc_default.mark);
+				mark = nc_default.mark;
+			}
+			nc[nodeid].mark = mark;
+
+			log_debug("parsed node config id=%d mark=%" PRIu32,
+				  nodeid, mark);
+		}
+	}
+
+	fclose(file);
+	return 0;
+
+out:
+	fclose(file);
+	return rv;
+}
+
+const struct node_config *node_config_get(int nodeid)
+{
+	if (nodeid <= 0 || nodeid > MAX_NODES - 1) {
+		log_debug("node config requested for id=%d returning defaults", nodeid);
+		return &nc_default;
+	}
+
+	return &nc[nodeid];
+}
diff --git a/dlm_controld/node_config.h b/dlm_controld/node_config.h
new file mode 100644
index 00000000..f0a4c2a0
--- /dev/null
+++ b/dlm_controld/node_config.h
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2020 Red Hat, Inc.
+ *
+ * This copyrighted material is made available to anyone wishing to use,
+ * modify, copy, or redistribute it subject to the terms and conditions
+ * of the GNU General Public License v2 or (at your option) any later version.
+ */
+
+#ifndef _NODE_CONFIG_H_
+#define _NODE_CONFIG_H_
+
+#include <stdint.h>
+
+struct node_config {
+	uint32_t mark;
+};
+
+/*
+ * Returns -ENOENT if path does not exist or there is no
+ * config for nodeid in the file.
+ *
+ * Returns -EXYZ if there's a problem with the config.
+ *
+ * Returns 0 if a config was found with no problems.
+ */
+
+int node_config_init(const char *path);
+
+const struct node_config *node_config_get(int nodeid);
+
+#endif
-- 
2.26.2

[Cluster-devel] [PATCHv2 dlm-tool 4/4] dlm_controld: add support for waitplock_recovery switch

[Alexander Aring]

This patch adds support to set the cluster attribute waitplock_recovery
via enable_waitplock_recover arg or config file attribute.
---
 dlm_controld/action.c     | 5 +++++
 dlm_controld/dlm.conf.5   | 2 ++
 dlm_controld/dlm_daemon.h | 1 +
 dlm_controld/main.c       | 5 +++++
 4 files changed, 13 insertions(+)

diff --git a/dlm_controld/action.c b/dlm_controld/action.c
index 126e3b62..63040227 100644
--- a/dlm_controld/action.c
+++ b/dlm_controld/action.c
@@ -876,6 +876,11 @@ int setup_configfs_options(void)
 	    dlm_options[timewarn_ind].file_set)
 		set_configfs_cluster("timewarn_cs", NULL, opt(timewarn_ind));
 
+	if (dlm_options[enable_waitplock_recovery_ind].cli_set ||
+	    dlm_options[enable_waitplock_recovery_ind].file_set)
+		set_configfs_cluster("waitplock_recovery", NULL,
+				     opt(enable_waitplock_recovery_ind));
+
 	set_configfs_cluster("mark", NULL, optu(mark_ind));
 
 	proto_name = opts(protocol_ind);
diff --git a/dlm_controld/dlm.conf.5 b/dlm_controld/dlm.conf.5
index 1ce0c644..e92dfc8e 100644
--- a/dlm_controld/dlm.conf.5
+++ b/dlm_controld/dlm.conf.5
@@ -46,6 +46,8 @@ debug_logfile
 .br
 enable_plock
 .br
+enable_waitplock_recovery
+.br
 plock_debug
 .br
 plock_rate_limit
diff --git a/dlm_controld/dlm_daemon.h b/dlm_controld/dlm_daemon.h
index 9e7a5fbf..979aab7a 100644
--- a/dlm_controld/dlm_daemon.h
+++ b/dlm_controld/dlm_daemon.h
@@ -102,6 +102,7 @@ enum {
         mark_ind,
         enable_fscontrol_ind,
         enable_plock_ind,
+        enable_waitplock_recovery_ind,
         plock_debug_ind,
         plock_rate_limit_ind,
         plock_ownership_ind,
diff --git a/dlm_controld/main.c b/dlm_controld/main.c
index b330f88d..3ec318c2 100644
--- a/dlm_controld/main.c
+++ b/dlm_controld/main.c
@@ -1752,6 +1752,11 @@ static void set_opt_defaults(void)
 			1, NULL,
 			"enable/disable posix lock support for cluster fs");
 
+	set_opt_default(enable_waitplock_recovery_ind,
+			"enable_waitplock_recovery", '\0', req_arg_bool,
+			1, NULL,
+			"enable/disable posix lock to wait for dlm recovery after lock acquire");
+
 	set_opt_default(plock_debug_ind,
 			"plock_debug", 'P', no_arg,
 			0, NULL,
-- 
2.26.2

[Cluster-devel] [PATCHv2 dlm-tool 4/4] dlm_controld: add support for waitplock_recovery switch

[Alexander Ahring Oder Aring]

Hi,

On Fri, Jun 26, 2020 at 12:45 PM Alexander Aring <aahringo@redhat.com> wrote:
>
> This patch adds support to set the cluster attribute waitplock_recovery
> via enable_waitplock_recover arg or config file attribute.
...
> diff --git a/dlm_controld/main.c b/dlm_controld/main.c
> index b330f88d..3ec318c2 100644
> --- a/dlm_controld/main.c
> +++ b/dlm_controld/main.c
> @@ -1752,6 +1752,11 @@ static void set_opt_defaults(void)
>                         1, NULL,
>                         "enable/disable posix lock support for cluster fs");
>
> +       set_opt_default(enable_waitplock_recovery_ind,
> +                       "enable_waitplock_recovery", '\0', req_arg_bool,
> +                       1, NULL,

that should be 0 otherwise the userland will activate it by default.
The kernel default value is 0.
I will send a v3, sorry about that.

- Alex